From noreply at buildbot.pypy.org Mon Sep 1 03:38:07 2014 From: noreply at buildbot.pypy.org (rlamy) Date: Mon, 1 Sep 2014 03:38:07 +0200 (CEST) Subject: [pypy-commit] pypy default: rename argument for consistency with RFile Message-ID: <20140901013807.391021C1036@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: Changeset: r73253:71ab012c8eca Date: 2014-09-01 02:37 +0100 http://bitbucket.org/pypy/pypy/changeset/71ab012c8eca/ Log: rename argument for consistency with RFile diff --git a/rpython/rlib/rStringIO.py b/rpython/rlib/rStringIO.py --- a/rpython/rlib/rStringIO.py +++ b/rpython/rlib/rStringIO.py @@ -125,19 +125,19 @@ assert result >= 0 return result - def read(self, n=-1): + def read(self, size=-1): p = self.__pos - if p == 0 and n < 0: + if p == 0 and size < 0: self.__pos = AT_END return self.getvalue() # reading everything - if p == AT_END or n == 0: + if p == AT_END or size == 0: return '' assert p >= 0 self.__copy_into_bigbuffer() mysize = len(self.__bigbuffer) count = mysize - p - if n >= 0: - count = min(n, count) + if size >= 0: + count = min(size, count) if count <= 0: return '' if p == 0 and count == mysize: From noreply at buildbot.pypy.org Mon Sep 1 08:45:10 2014 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 1 Sep 2014 08:45:10 +0200 (CEST) Subject: [pypy-commit] pypy default: Attempt to get the stdout/stderr as well in the written resultlogs. Message-ID: <20140901064510.107701C025B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73254:93a97c795531 Date: 2014-09-01 08:44 +0200 http://bitbucket.org/pypy/pypy/changeset/93a97c795531/ Log: Attempt to get the stdout/stderr as well in the written resultlogs. diff --git a/_pytest/resultlog.py b/_pytest/resultlog.py --- a/_pytest/resultlog.py +++ b/_pytest/resultlog.py @@ -53,16 +53,23 @@ self.config = config self.logfile = logfile # preferably line buffered - def write_log_entry(self, testpath, lettercode, longrepr): + def write_log_entry(self, testpath, lettercode, longrepr, sections=None): py.builtin.print_("%s %s" % (lettercode, testpath), file=self.logfile) for line in longrepr.splitlines(): py.builtin.print_(" %s" % line, file=self.logfile) + if sections is not None: + for title, content in sections: + py.builtin.print_(" ---------- %s ----------" % (title,), + file=self.logfile) + for line in content.splitlines(): + py.builtin.print_(" %s" % line, file=self.logfile) def log_outcome(self, report, lettercode, longrepr): testpath = getattr(report, 'nodeid', None) if testpath is None: testpath = report.fspath - self.write_log_entry(testpath, lettercode, longrepr) + self.write_log_entry(testpath, lettercode, longrepr, + getattr(report, 'sections', None)) def pytest_runtest_logreport(self, report): if report.when != "call" and report.passed: From noreply at buildbot.pypy.org Mon Sep 1 09:00:21 2014 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 1 Sep 2014 09:00:21 +0200 (CEST) Subject: [pypy-commit] pypy default: Reapply fix c6f52c21fe7e Message-ID: <20140901070021.4F68F1C072F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73255:88ad3346a763 Date: 2014-09-01 08:56 +0200 http://bitbucket.org/pypy/pypy/changeset/88ad3346a763/ Log: Reapply fix c6f52c21fe7e diff --git a/py/_path/local.py b/py/_path/local.py --- a/py/_path/local.py +++ b/py/_path/local.py @@ -750,7 +750,8 @@ mkdtemp = classmethod(mkdtemp) def make_numbered_dir(cls, prefix='session-', rootdir=None, keep=3, - lock_timeout = 172800): # two days + lock_timeout = 172800, # two days + min_timeout = 300): # five minutes """ return unique directory with a number greater than the current maximum one. The number is assumed to start directly after prefix. if keep is true directories with a number less than (maxnum-keep) @@ -818,6 +819,20 @@ for path in rootdir.listdir(): num = parse_num(path) if num is not None and num <= (maxnum - keep): + if min_timeout: + # NB: doing this is needed to prevent (or reduce + # a lot the chance of) the following situation: + # 'keep+1' processes call make_numbered_dir() at + # the same time, they create dirs, but then the + # last process notices the first dir doesn't have + # (yet) a .lock in it and kills it. + try: + t1 = path.lstat().mtime + t2 = lockfile.lstat().mtime + if abs(t2-t1) < min_timeout: + continue # skip directories too recent + except py.error.Error: + continue # failure to get a time, better skip lf = path.join('.lock') try: t1 = lf.lstat().mtime From noreply at buildbot.pypy.org Mon Sep 1 09:00:22 2014 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 1 Sep 2014 09:00:22 +0200 (CEST) Subject: [pypy-commit] pypy default: reapply dc81116ff4b7 Message-ID: <20140901070022.8B2171C072F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73256:44c39245f150 Date: 2014-09-01 08:57 +0200 http://bitbucket.org/pypy/pypy/changeset/44c39245f150/ Log: reapply dc81116ff4b7 diff --git a/py/_code/source.py b/py/_code/source.py --- a/py/_code/source.py +++ b/py/_code/source.py @@ -416,6 +416,8 @@ trysource = source[start:end] if trysource.isparseable(): return start, end + if end == start + 100: # XXX otherwise, it takes forever + break # XXX raise SyntaxError("no valid source range around line %d " % (lineno,)) From noreply at buildbot.pypy.org Mon Sep 1 09:06:28 2014 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 1 Sep 2014 09:06:28 +0200 (CEST) Subject: [pypy-commit] pypy default: Add two README files in "py" and "_pytest". PLEASE READ THEM Message-ID: <20140901070628.6127C1C072F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73257:084074ff7fa1 Date: 2014-09-01 09:06 +0200 http://bitbucket.org/pypy/pypy/changeset/084074ff7fa1/ Log: Add two README files in "py" and "_pytest". PLEASE READ THEM when updating these! diff --git a/_pytest/README-BEFORE-UPDATING b/_pytest/README-BEFORE-UPDATING new file mode 100644 --- /dev/null +++ b/_pytest/README-BEFORE-UPDATING @@ -0,0 +1,17 @@ +This is PyPy's code of the pytest lib. We don't expect to upgrade it +very often, but once we do: + + WARNING! + + WE HAVE MADE A FEW TWEAKS HERE! + +Please be sure that you don't just copy the newer version from +upstream without checking the few changes that we did. This +can be done like this: + + cd + hg log . -v | less + +then search for all " _pytest/" in that list to know which are the +relevant checkins. (Look for the checkins that only edit one +or two files in this directory.) diff --git a/py/README-BEFORE-UPDATING b/py/README-BEFORE-UPDATING new file mode 100644 --- /dev/null +++ b/py/README-BEFORE-UPDATING @@ -0,0 +1,17 @@ +This is PyPy's code of the py lib. We don't expect to upgrade it +very often, but once we do: + + WARNING! + + WE HAVE MADE A FEW TWEAKS HERE! + +Please be sure that you don't just copy the newer version from +upstream without checking the few changes that we did. This +can be done like this: + + cd + hg log . -v | less + +then search for all " py/" in that list to know which are the +relevant checkins. (Look for the checkins that only edit one +or two files in this directory.) From noreply at buildbot.pypy.org Mon Sep 1 09:18:22 2014 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 1 Sep 2014 09:18:22 +0200 (CEST) Subject: [pypy-commit] pypy default: Update for some (minimal) constant-folding occurring in the list-to-cdata-struct code Message-ID: <20140901071822.291241C13AE@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73258:3119573b8a41 Date: 2014-09-01 09:17 +0200 http://bitbucket.org/pypy/pypy/changeset/3119573b8a41/ Log: Update for some (minimal) constant-folding occurring in the list-to- cdata-struct code diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py --- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py +++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py @@ -340,30 +340,19 @@ guard_value(p166, ConstPtr(ptr72), descr=...) p167 = call(ConstClass(_ll_0_alloc_with_del___), descr=) guard_no_exception(descr=...) - i168 = call(ConstClass(_ll_1_raw_malloc_varsize__Signed), 6, descr=) - i169 = int_add(i168, i97) - i170 = int_sub(i160, i106) - setfield_gc(p167, i168, descr=) + i112 = int_sub(i160, -32768) setfield_gc(p167, ConstPtr(null), descr=) - setfield_gc(p167, ConstPtr(ptr89), descr=) - i171 = uint_gt(i170, i108) - guard_false(i171, descr=...) - i172 = int_sub(i160, -32768) - i173 = int_and(i172, 65535) - i174 = int_add(i173, -32768) - setarrayitem_raw(i169, 0, i174, descr=) - i175 = int_add(i168, i121) - i176 = int_sub(i160, i130) - i177 = uint_gt(i176, i132) - guard_false(i177, descr=...) - setarrayitem_raw(i175, 0, i174, descr=) - i178 = int_add(i168, i140) - i179 = int_sub(i160, i149) - i180 = uint_gt(i179, i151) - guard_false(i180, descr=...) - setarrayitem_raw(i178, 0, i174, descr=) + setfield_gc(p167, ConstPtr(ptr85), descr=) + i114 = uint_gt(i112, 65535) + guard_false(i114, descr=...) + i115 = int_and(i112, 65535) + i116 = int_add(i115, -32768) --TICK-- - i183 = arraylen_gc(p67, descr=) - i184 = arraylen_gc(p92, descr=) + i119 = call(ConstClass(_ll_1_raw_malloc_varsize__Signed), 6, descr=) + raw_store(i119, 0, i116, descr=) + raw_store(i119, 2, i116, descr=) + raw_store(i119, 4, i116, descr=) + setfield_gc(p167, i119, descr=) + i123 = arraylen_gc(p67, descr=) jump(..., descr=...) """) From noreply at buildbot.pypy.org Mon Sep 1 09:25:32 2014 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 1 Sep 2014 09:25:32 +0200 (CEST) Subject: [pypy-commit] pypy default: fix for e9f0c13de06b: output the jit-backend-count loop numbers Message-ID: <20140901072532.5FC9E1C35CC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73259:625c201e7f9b Date: 2014-09-01 09:25 +0200 http://bitbucket.org/pypy/pypy/changeset/625c201e7f9b/ Log: fix for e9f0c13de06b: output the jit-backend-count loop numbers as unsigned too, to match the newly unsigned Guard0xNNN output. diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py --- a/rpython/jit/backend/llsupport/assembler.py +++ b/rpython/jit/backend/llsupport/assembler.py @@ -294,10 +294,16 @@ struct = self.loop_run_counters[i] if struct.type == 'l': prefix = 'TargetToken(%d)' % struct.number - elif struct.type == 'b': - prefix = 'bridge ' + str(struct.number) else: - prefix = 'entry ' + str(struct.number) + num = struct.number + if num == -1: + num = '-1' + else: + num = str(r_uint(num)) + if struct.type == 'b': + prefix = 'bridge %s' % num + else: + prefix = 'entry %s' % num debug_print(prefix + ':' + str(struct.i)) debug_stop('jit-backend-counts') From noreply at buildbot.pypy.org Mon Sep 1 09:33:11 2014 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 1 Sep 2014 09:33:11 +0200 (CEST) Subject: [pypy-commit] pypy default: Fix test for the 'trace-limit-hack' merge Message-ID: <20140901073311.6ADF01D22E6@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73260:fc98972622d7 Date: 2014-09-01 09:32 +0200 http://bitbucket.org/pypy/pypy/changeset/fc98972622d7/ Log: Fix test for the 'trace-limit-hack' merge diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py --- a/pypy/module/pypyjit/test_pypy_c/test_call.py +++ b/pypy/module/pypyjit/test_pypy_c/test_call.py @@ -17,13 +17,18 @@ # now we can inline it as call assembler i = 0 j = 0 - while i < 20: + while i < 25: i += 1 j += rec(100) # ID: call_rec return j # - log = self.run(fn, [], threshold=18) - loop, = log.loops_by_filename(self.filepath) + # NB. the parameters below are a bit ad-hoc. After 16 iterations, + # the we trace from the "while" and reach a "trace too long". Then + # in the next execution, we trace the "rec" function from start; + # that's "functrace" below. Then after one or two extra iterations + # we try again from "while", and this time we succeed. + log = self.run(fn, [], threshold=20) + functrace, loop = log.loops_by_filename(self.filepath) assert loop.match_by_id('call_rec', """ ... p53 = call_assembler(..., descr=...) From noreply at buildbot.pypy.org Mon Sep 1 09:50:26 2014 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 1 Sep 2014 09:50:26 +0200 (CEST) Subject: [pypy-commit] pypy arm-longlong: uh, there are synonyms in the ARM instruction set? Message-ID: <20140901075026.25CD31C13AE@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: arm-longlong Changeset: r73261:d6eaca4efd43 Date: 2014-09-01 09:49 +0200 http://bitbucket.org/pypy/pypy/changeset/d6eaca4efd43/ Log: uh, there are synonyms in the ARM instruction set? diff --git a/rpython/jit/backend/arm/codebuilder.py b/rpython/jit/backend/arm/codebuilder.py --- a/rpython/jit/backend/arm/codebuilder.py +++ b/rpython/jit/backend/arm/codebuilder.py @@ -340,19 +340,8 @@ MOD = binary_helper_call('int_mod') UDIV = binary_helper_call('uint_div') - def FMDRR(self, dm, rd, rn, c=cond.AL): - self.write32(c << 28 - | 0x0c400b10 - | (dm & 0xF) - | (rd & 0xF) << 12 - | (rn & 0xF) << 16) - - def FMRRD(self, rd, rn, dm, c=cond.AL): - self.write32(c << 28 - | 0x0c500b10 - | (dm & 0xF) - | (rd & 0xF) << 12 - | (rn & 0xF) << 16) + FMDRR = VMOV_cr # uh, there are synonyms? + FMRRD = VMOV_rc def _encode_reg_list(self, instr, regs): for reg in regs: From noreply at buildbot.pypy.org Mon Sep 1 10:08:37 2014 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 1 Sep 2014 10:08:37 +0200 (CEST) Subject: [pypy-commit] pypy default: Move the test to a function and use it from two places. Message-ID: <20140901080837.A39D71D3538@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73262:b97c7d6f7fd8 Date: 2014-09-01 10:07 +0200 http://bitbucket.org/pypy/pypy/changeset/b97c7d6f7fd8/ Log: Move the test to a function and use it from two places. diff --git a/pypy/module/sys/initpath.py b/pypy/module/sys/initpath.py --- a/pypy/module/sys/initpath.py +++ b/pypy/module/sys/initpath.py @@ -18,6 +18,13 @@ _WIN32 = sys.platform == 'win32' +def _exists_and_is_executable(fn): + # os.access checks using the user's real uid and gid. + # Since pypy should not be run setuid/setgid, this + # should be sufficient. + return os.path.isfile(fn) and os.access(fn, os.X_OK) + + def find_executable(executable): """ Return the absolute path of the executable, by looking into PATH and @@ -34,18 +41,14 @@ if path: for dir in path.split(os.pathsep): fn = os.path.join(dir, executable) - if os.path.isfile(fn): - # os.access checks using the user's real uid and gid. - # Since pypy should not be run setuid/setgid, this - # should be sufficient. - if os.access(fn, os.X_OK): - executable = fn - break + if _exists_and_is_executable(fn): + executable = fn + break executable = rpath.rabspath(executable) # 'sys.executable' should not end up being an non-existing file; # just use '' in this case. (CPython issue #7774) - return executable if os.path.isfile(executable) else '' + return executable if _exists_and_is_executable(executable) else '' def _readlink_maybe(filename): From noreply at buildbot.pypy.org Mon Sep 1 11:16:34 2014 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 1 Sep 2014 11:16:34 +0200 (CEST) Subject: [pypy-commit] pypy default: Add a test for unichar comparison. It was not broken but producing Message-ID: <20140901091634.4ABFA1D2B61@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73263:65ac482d28d6 Date: 2014-09-01 11:16 +0200 http://bitbucket.org/pypy/pypy/changeset/65ac482d28d6/ Log: Add a test for unichar comparison. It was not broken but producing very inefficient code. Improve... diff --git a/rpython/rtyper/rstr.py b/rpython/rtyper/rstr.py --- a/rpython/rtyper/rstr.py +++ b/rpython/rtyper/rstr.py @@ -3,6 +3,7 @@ from rpython.rtyper import rint from rpython.rtyper.error import TyperError from rpython.rtyper.lltypesystem.lltype import Signed, Bool, Void, UniChar +from rpython.rtyper.lltypesystem import lltype from rpython.rtyper.rmodel import IteratorRepr, inputconst, Repr from rpython.rtyper.rint import IntegerRepr from rpython.rtyper.rfloat import FloatRepr @@ -384,10 +385,10 @@ unicode_encode_utf_8_impl, 'runicode_encode_utf_8') def rtype_method_upper(self, hop): - raise TypeError("Cannot do toupper on unicode string") + raise TyperError("Cannot do toupper on unicode string") def rtype_method_lower(self, hop): - raise TypeError("Cannot do tolower on unicode string") + raise TyperError("Cannot do tolower on unicode string") @jit.elidable def ll_encode_utf8(self, ll_s): @@ -711,6 +712,11 @@ pairtype(AbstractUniCharRepr, AbstractCharRepr)): def rtype_eq(_, hop): return _rtype_unchr_compare_template(hop, 'eq') def rtype_ne(_, hop): return _rtype_unchr_compare_template(hop, 'ne') + def rtype_lt(_, hop): return _rtype_unchr_compare_template_ord(hop, 'lt') + def rtype_le(_, hop): return _rtype_unchr_compare_template_ord(hop, 'le') + def rtype_gt(_, hop): return _rtype_unchr_compare_template_ord(hop, 'gt') + def rtype_ge(_, hop): return _rtype_unchr_compare_template_ord(hop, 'ge') + #Helper functions for comparisons @@ -719,6 +725,18 @@ vlist = hop.inputargs(unichar_repr, unichar_repr) return hop.genop('unichar_' + func, vlist, resulttype=Bool) +def _rtype_unchr_compare_template_ord(hop, func): + vlist = hop.inputargs(*hop.args_r) + vlist2 = [] + for v in vlist: + if v.concretetype == lltype.Char: + v = hop.genop('cast_char_to_int', [v], resulttype=lltype.Signed) + elif v.concretetype == lltype.UniChar: + v = hop.genop('cast_unichar_to_int', [v], resulttype=lltype.Signed) + else: + assert 0, v.concretetype + vlist2.append(v) + return hop.genop('int_' + func, vlist2, resulttype=Bool) # # _________________________ Conversions _________________________ diff --git a/rpython/rtyper/test/test_runicode.py b/rpython/rtyper/test/test_runicode.py --- a/rpython/rtyper/test/test_runicode.py +++ b/rpython/rtyper/test/test_runicode.py @@ -296,3 +296,13 @@ res = self.interpret(f, [5]) assert res == 0 + + def test_unicode_char_comparison(self): + const = u'abcdef' + def f(n): + return const[n] >= u'c' + + res = self.interpret(f, [1]) + assert res == False + res = self.interpret(f, [2]) + assert res == True From noreply at buildbot.pypy.org Mon Sep 1 11:45:11 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Mon, 1 Sep 2014 11:45:11 +0200 (CEST) Subject: [pypy-commit] stmgc default: introduce stmcb_obj_supports_cards and hopefully make cards finally work as they should Message-ID: <20140901094511.993241C06D8@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1326:83e4c655d31b Date: 2014-09-01 11:46 +0200 http://bitbucket.org/pypy/stmgc/changeset/83e4c655d31b/ Log: introduce stmcb_obj_supports_cards and hopefully make cards finally work as they should diff --git a/c7/demo/demo2.c b/c7/demo/demo2.c --- a/c7/demo/demo2.c +++ b/c7/demo/demo2.c @@ -43,6 +43,10 @@ n = (struct node_s*)obj; visit((object_t **)&n->next); } +long stmcb_obj_supports_cards(struct object_s *obj) +{ + return 0; +} void stmcb_get_card_base_itemsize(struct object_s *obj, uintptr_t offset_itemsize[2]) { diff --git a/c7/demo/demo_largemalloc.c b/c7/demo/demo_largemalloc.c --- a/c7/demo/demo_largemalloc.c +++ b/c7/demo/demo_largemalloc.c @@ -24,7 +24,10 @@ } void stmcb_commit_soon() {} - +long stmcb_obj_supports_cards(struct object_s *obj) +{ + return 0; +} void stmcb_trace_cards(struct object_s *obj, void cb(object_t **), uintptr_t start, uintptr_t stop) { abort(); diff --git a/c7/demo/demo_random.c b/c7/demo/demo_random.c --- a/c7/demo/demo_random.c +++ b/c7/demo/demo_random.c @@ -80,7 +80,10 @@ } void stmcb_commit_soon() {} - +long stmcb_obj_supports_cards(struct object_s *obj) +{ + return 0; +} void stmcb_trace_cards(struct object_s *obj, void cb(object_t **), uintptr_t start, uintptr_t stop) { abort(); diff --git a/c7/demo/demo_random2.c b/c7/demo/demo_random2.c --- a/c7/demo/demo_random2.c +++ b/c7/demo/demo_random2.c @@ -85,6 +85,10 @@ void stmcb_commit_soon() {} +long stmcb_obj_supports_cards(struct object_s *obj) +{ + return 0; +} void stmcb_trace_cards(struct object_s *obj, void cb(object_t **), uintptr_t start, uintptr_t stop) { abort(); diff --git a/c7/demo/demo_simple.c b/c7/demo/demo_simple.c --- a/c7/demo/demo_simple.c +++ b/c7/demo/demo_simple.c @@ -38,7 +38,10 @@ n = (struct node_s*)obj; visit((object_t **)&n->next); } - +long stmcb_obj_supports_cards(struct object_s *obj) +{ + return 0; +} void stmcb_commit_soon() {} void stmcb_trace_cards(struct object_s *obj, void cb(object_t **), diff --git a/c7/demo/test_shadowstack.c b/c7/demo/test_shadowstack.c --- a/c7/demo/test_shadowstack.c +++ b/c7/demo/test_shadowstack.c @@ -18,6 +18,10 @@ void stmcb_trace(struct object_s *obj, void visit(object_t **)) { } +long stmcb_obj_supports_cards(struct object_s *obj) +{ + return 0; +} void stmcb_get_card_base_itemsize(struct object_s *obj, uintptr_t offset_itemsize[2]) { diff --git a/c7/stm/core.c b/c7/stm/core.c --- a/c7/stm/core.c +++ b/c7/stm/core.c @@ -231,8 +231,12 @@ { struct object_s *realobj = (struct object_s *) REAL_ADDRESS(STM_SEGMENT->segment_base, obj); + long supports = stmcb_obj_supports_cards(realobj); + if (!supports) + return 0; + + /* check also if it makes sense: */ size_t size = stmcb_size_rounded_up(realobj); - return (size >= _STM_MIN_CARD_OBJ_SIZE); } @@ -590,13 +594,16 @@ assert(!(obj->stm_flags & GCFLAG_CARDS_SET)); assert(!IS_OVERFLOW_OBJ(STM_PSEGMENT, obj)); + uintptr_t offset_itemsize[2]; struct object_s *realobj = (struct object_s *)REAL_ADDRESS(STM_SEGMENT->segment_base, obj); size_t obj_size = stmcb_size_rounded_up(realobj); assert(obj_size >= 32); + stmcb_get_card_base_itemsize(realobj, offset_itemsize); + size_t real_idx_count = (obj_size - offset_itemsize[0]) / offset_itemsize[1]; uintptr_t first_card_index = get_write_lock_idx((uintptr_t)obj); uintptr_t card_index = 1; - uintptr_t last_card_index = get_index_to_card_index(obj_size - 1); /* max valid index */ + uintptr_t last_card_index = get_index_to_card_index(real_idx_count - 1); /* max valid index */ long i, myself = STM_SEGMENT->segment_num; /* simple heuristic to check if probably the whole object is @@ -617,7 +624,6 @@ /* Combine multiple marked cards and do a memcpy for them. We don't try yet to use page_copy() or otherwise take into account privatization of pages (except _has_private_page_in_range) */ - uintptr_t offset_itemsize[2]; bool all_cards_were_cleared = true; uintptr_t start_card_index = -1; @@ -634,7 +640,6 @@ /* realobj, get_card_index_to_index(card_index)); */ if (all_cards_were_cleared) { all_cards_were_cleared = false; - stmcb_get_card_base_itemsize(realobj, offset_itemsize); } } } diff --git a/c7/stm/nursery.c b/c7/stm/nursery.c --- a/c7/stm/nursery.c +++ b/c7/stm/nursery.c @@ -241,8 +241,12 @@ #undef STM_SEGMENT struct object_s *realobj = (struct object_s *)REAL_ADDRESS(pseg->pub.segment_base, obj); size_t size = stmcb_size_rounded_up(realobj); + OPT_ASSERT(size >= _STM_MIN_CARD_OBJ_SIZE); - OPT_ASSERT(size >= _STM_MIN_CARD_OBJ_SIZE); + uintptr_t offset_itemsize[2]; + stmcb_get_card_base_itemsize(realobj, offset_itemsize); + size = (size - offset_itemsize[0]) / offset_itemsize[1]; + assert(IMPLY(mark_value == CARD_CLEAR, !mark_all)); /* not necessary */ assert(IMPLY(mark_all, mark_value == CARD_MARKED_OLD)); /* set *all* to OLD */ assert(IMPLY(IS_OVERFLOW_OBJ(pseg, realobj), diff --git a/c7/stmgc.h b/c7/stmgc.h --- a/c7/stmgc.h +++ b/c7/stmgc.h @@ -259,6 +259,9 @@ one item */ extern void stmcb_get_card_base_itemsize(struct object_s *, uintptr_t offset_itemsize[2]); +/* returns whether this object supports cards. we will only call + stmcb_get_card_base_itemsize on objs that do so. */ +extern long stmcb_obj_supports_cards(struct object_s *); extern void stmcb_commit_soon(void); @@ -367,7 +370,7 @@ int stm_is_inevitable(void); #else static inline int stm_is_inevitable(void) { - return !rewind_jmp_armed(&STM_SEGMENT->running_thread->rjthread); + return !rewind_jmp_armed(&STM_SEGMENT->running_thread->rjthread); } #endif static inline void stm_become_inevitable(stm_thread_local_t *tl, diff --git a/c7/test/support.py b/c7/test/support.py --- a/c7/test/support.py +++ b/c7/test/support.py @@ -301,7 +301,10 @@ visit(ref); } } - +long stmcb_obj_supports_cards(struct object_s *obj) +{ + return 1; +} void stmcb_trace_cards(struct object_s *obj, void visit(object_t **), uintptr_t start, uintptr_t stop) { From noreply at buildbot.pypy.org Mon Sep 1 12:04:08 2014 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 1 Sep 2014 12:04:08 +0200 (CEST) Subject: [pypy-commit] pypy default: Remove this special case. See comments. Message-ID: <20140901100408.4226C1C35CC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73264:fcbba01341eb Date: 2014-09-01 11:28 +0200 http://bitbucket.org/pypy/pypy/changeset/fcbba01341eb/ Log: Remove this special case. See comments. diff --git a/pypy/module/_pypyjson/interp_encoder.py b/pypy/module/_pypyjson/interp_encoder.py --- a/pypy/module/_pypyjson/interp_encoder.py +++ b/pypy/module/_pypyjson/interp_encoder.py @@ -37,16 +37,14 @@ sb = StringBuilder(len(u)) sb.append_slice(s, 0, first) else: + # We used to check if 'u' contains only safe characters, and return + # 'w_string' directly. But this requires an extra pass over all + # characters, and the expected use case of this function, from + # json.encoder, will anyway re-encode a unicode result back to + # a string (with the ascii encoding). So we may as well directly + # turn it into a string from here, and avoid the extra pass over + # all characters here. u = space.unicode_w(w_string) - for i in range(len(u)): - c = u[i] - if c >= u' ' and c <= u'~' and c != u'"' and c != u'\\': - pass - else: - break - else: - # the input is a unicode with only non-special ascii chars - return w_string sb = StringBuilder(len(u)) first = 0 diff --git a/pypy/module/_pypyjson/test/test__pypyjson.py b/pypy/module/_pypyjson/test/test__pypyjson.py --- a/pypy/module/_pypyjson/test/test__pypyjson.py +++ b/pypy/module/_pypyjson/test/test__pypyjson.py @@ -192,14 +192,14 @@ def test_raw_encode_basestring_ascii(self): import _pypyjson - def check(s, expected_type=str): + def check(s): s = _pypyjson.raw_encode_basestring_ascii(s) - assert type(s) is expected_type + assert type(s) is str return s assert check("") == "" - assert check(u"", expected_type=unicode) == u"" + assert check(u"") == "" assert check("abc ") == "abc " - assert check(u"abc ", expected_type=unicode) == u"abc " + assert check(u"abc ") == "abc " raises(UnicodeDecodeError, check, "\xc0") assert check("\xc2\x84") == "\\u0084" assert check("\xf0\x92\x8d\x85") == "\\ud808\\udf45" From noreply at buildbot.pypy.org Mon Sep 1 12:04:09 2014 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 1 Sep 2014 12:04:09 +0200 (CEST) Subject: [pypy-commit] pypy default: Detect and special-case in the JIT the greater-than kind of comparisons Message-ID: <20140901100409.835871C35CC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73265:8b404466566d Date: 2014-09-01 12:03 +0200 http://bitbucket.org/pypy/pypy/changeset/8b404466566d/ Log: Detect and special-case in the JIT the greater-than kind of comparisons between two single characters (or unichars). diff --git a/rpython/jit/codewriter/effectinfo.py b/rpython/jit/codewriter/effectinfo.py --- a/rpython/jit/codewriter/effectinfo.py +++ b/rpython/jit/codewriter/effectinfo.py @@ -36,6 +36,7 @@ OS_STREQ_NONNULL_CHAR = 29 # s1 == char (assert s1!=NULL) OS_STREQ_CHECKNULL_CHAR = 30 # s1!=NULL and s1==char OS_STREQ_LENGTHOK = 31 # s1 == s2 (assert len(s1)==len(s2)) + OS_STR_CMP = 32 # "stroruni.cmp" # OS_UNI_CONCAT = 42 # OS_UNI_SLICE = 43 # @@ -47,6 +48,7 @@ OS_UNIEQ_NONNULL_CHAR = 49 # (must be the same amount as for OS_UNIEQ_CHECKNULL_CHAR = 50 # STR, in the same order) OS_UNIEQ_LENGTHOK = 51 # + OS_UNI_CMP = 52 _OS_offset_uni = OS_UNI_CONCAT - OS_STR_CONCAT # OS_LIBFFI_CALL = 62 diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py --- a/rpython/jit/codewriter/jtransform.py +++ b/rpython/jit/codewriter/jtransform.py @@ -1767,6 +1767,7 @@ dict = {"stroruni.concat": EffectInfo.OS_STR_CONCAT, "stroruni.slice": EffectInfo.OS_STR_SLICE, "stroruni.equal": EffectInfo.OS_STR_EQUAL, + "stroruni.cmp": EffectInfo.OS_STR_CMP, "stroruni.copy_string_to_raw": EffectInfo.OS_STR_COPY_TO_RAW, } CHR = lltype.Char @@ -1774,6 +1775,7 @@ dict = {"stroruni.concat": EffectInfo.OS_UNI_CONCAT, "stroruni.slice": EffectInfo.OS_UNI_SLICE, "stroruni.equal": EffectInfo.OS_UNI_EQUAL, + "stroruni.cmp": EffectInfo.OS_UNI_CMP, "stroruni.copy_string_to_raw": EffectInfo.OS_UNI_COPY_TO_RAW } CHR = lltype.UniChar diff --git a/rpython/jit/metainterp/optimizeopt/vstring.py b/rpython/jit/metainterp/optimizeopt/vstring.py --- a/rpython/jit/metainterp/optimizeopt/vstring.py +++ b/rpython/jit/metainterp/optimizeopt/vstring.py @@ -733,6 +733,25 @@ return True return False + def opt_call_stroruni_STR_CMP(self, op, mode): + v1 = self.getvalue(op.getarg(1)) + v2 = self.getvalue(op.getarg(2)) + l1box = v1.getstrlen(None, mode, None) + l2box = v2.getstrlen(None, mode, None) + if (l1box is not None and l2box is not None and + isinstance(l1box, ConstInt) and + isinstance(l2box, ConstInt) and + l1box.value == l2box.value == 1): + # comparing two single chars + vchar1 = self.strgetitem(v1, optimizer.CVAL_ZERO, mode) + vchar2 = self.strgetitem(v2, optimizer.CVAL_ZERO, mode) + seo = self.optimizer.send_extra_operation + seo(ResOperation(rop.INT_SUB, [vchar1.force_box(self), + vchar2.force_box(self)], + op.result)) + return True + return False + def opt_call_SHRINK_ARRAY(self, op): v1 = self.getvalue(op.getarg(1)) v2 = self.getvalue(op.getarg(2)) diff --git a/rpython/jit/metainterp/test/test_string.py b/rpython/jit/metainterp/test/test_string.py --- a/rpython/jit/metainterp/test/test_string.py +++ b/rpython/jit/metainterp/test/test_string.py @@ -846,6 +846,27 @@ 'jump': 1, 'guard_true': 2, 'int_ge': 2, 'int_add': 2, 'int_sub': 2 }) + def test_compare_single_char_for_ordering(self): + jitdriver = JitDriver(reds=['result', 'n'], greens=[]) + _str = self._str + constant1 = _str("abcdefghij") + + def cmpstr(x, y): + return x > _str(y) + + def f(n): + cmpstr(_str("abc"), "def") # force x and y to be annot as strings + result = 0 + while n >= 0: + jitdriver.jit_merge_point(n=n, result=result) + c = constant1[n] + result += cmpstr(c, "c") + n -= 1 + return result + + res = self.meta_interp(f, [9]) + assert res == f(9) + self.check_resops(newstr=0, newunicode=0, call=0) class TestLLtype(StringTests, LLJitMixin): diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py --- a/rpython/rtyper/lltypesystem/rstr.py +++ b/rpython/rtyper/lltypesystem/rstr.py @@ -531,6 +531,7 @@ return diff i += 1 return len1 - len2 + ll_strcmp.oopspec = 'stroruni.cmp(s1, s2)' @jit.elidable def ll_streq(s1, s2): From noreply at buildbot.pypy.org Mon Sep 1 12:07:18 2014 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 1 Sep 2014 12:07:18 +0200 (CEST) Subject: [pypy-commit] pypy default: clarify comment Message-ID: <20140901100718.730651C35CC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73266:dffa8ea95d54 Date: 2014-09-01 12:06 +0200 http://bitbucket.org/pypy/pypy/changeset/dffa8ea95d54/ Log: clarify comment diff --git a/pypy/module/_pypyjson/interp_encoder.py b/pypy/module/_pypyjson/interp_encoder.py --- a/pypy/module/_pypyjson/interp_encoder.py +++ b/pypy/module/_pypyjson/interp_encoder.py @@ -41,9 +41,9 @@ # 'w_string' directly. But this requires an extra pass over all # characters, and the expected use case of this function, from # json.encoder, will anyway re-encode a unicode result back to - # a string (with the ascii encoding). So we may as well directly - # turn it into a string from here, and avoid the extra pass over - # all characters here. + # a string (with the ascii encoding). This requires two passes + # over the characters. So we may as well directly turn it into a + # string here --- only one pass. u = space.unicode_w(w_string) sb = StringBuilder(len(u)) first = 0 From noreply at buildbot.pypy.org Mon Sep 1 12:43:06 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Mon, 1 Sep 2014 12:43:06 +0200 (CEST) Subject: [pypy-commit] pypy stmgc-c7: import stmgc Message-ID: <20140901104306.959151C025B@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: stmgc-c7 Changeset: r73267:960172cff05f Date: 2014-09-01 12:43 +0200 http://bitbucket.org/pypy/pypy/changeset/960172cff05f/ Log: import stmgc diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision --- a/rpython/translator/stm/src_stm/revision +++ b/rpython/translator/stm/src_stm/revision @@ -1,1 +1,1 @@ -dbe9b14b252f +83e4c655d31b diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c --- a/rpython/translator/stm/src_stm/stm/core.c +++ b/rpython/translator/stm/src_stm/stm/core.c @@ -232,8 +232,12 @@ { struct object_s *realobj = (struct object_s *) REAL_ADDRESS(STM_SEGMENT->segment_base, obj); + long supports = stmcb_obj_supports_cards(realobj); + if (!supports) + return 0; + + /* check also if it makes sense: */ size_t size = stmcb_size_rounded_up(realobj); - return (size >= _STM_MIN_CARD_OBJ_SIZE); } @@ -591,13 +595,16 @@ assert(!(obj->stm_flags & GCFLAG_CARDS_SET)); assert(!IS_OVERFLOW_OBJ(STM_PSEGMENT, obj)); + uintptr_t offset_itemsize[2]; struct object_s *realobj = (struct object_s *)REAL_ADDRESS(STM_SEGMENT->segment_base, obj); size_t obj_size = stmcb_size_rounded_up(realobj); assert(obj_size >= 32); + stmcb_get_card_base_itemsize(realobj, offset_itemsize); + size_t real_idx_count = (obj_size - offset_itemsize[0]) / offset_itemsize[1]; uintptr_t first_card_index = get_write_lock_idx((uintptr_t)obj); uintptr_t card_index = 1; - uintptr_t last_card_index = get_index_to_card_index(obj_size - 1); /* max valid index */ + uintptr_t last_card_index = get_index_to_card_index(real_idx_count - 1); /* max valid index */ long i, myself = STM_SEGMENT->segment_num; /* simple heuristic to check if probably the whole object is @@ -618,7 +625,6 @@ /* Combine multiple marked cards and do a memcpy for them. We don't try yet to use page_copy() or otherwise take into account privatization of pages (except _has_private_page_in_range) */ - uintptr_t offset_itemsize[2]; bool all_cards_were_cleared = true; uintptr_t start_card_index = -1; @@ -635,7 +641,6 @@ /* realobj, get_card_index_to_index(card_index)); */ if (all_cards_were_cleared) { all_cards_were_cleared = false; - stmcb_get_card_base_itemsize(realobj, offset_itemsize); } } } diff --git a/rpython/translator/stm/src_stm/stm/nursery.c b/rpython/translator/stm/src_stm/stm/nursery.c --- a/rpython/translator/stm/src_stm/stm/nursery.c +++ b/rpython/translator/stm/src_stm/stm/nursery.c @@ -242,8 +242,12 @@ #undef STM_SEGMENT struct object_s *realobj = (struct object_s *)REAL_ADDRESS(pseg->pub.segment_base, obj); size_t size = stmcb_size_rounded_up(realobj); + OPT_ASSERT(size >= _STM_MIN_CARD_OBJ_SIZE); - OPT_ASSERT(size >= _STM_MIN_CARD_OBJ_SIZE); + uintptr_t offset_itemsize[2]; + stmcb_get_card_base_itemsize(realobj, offset_itemsize); + size = (size - offset_itemsize[0]) / offset_itemsize[1]; + assert(IMPLY(mark_value == CARD_CLEAR, !mark_all)); /* not necessary */ assert(IMPLY(mark_all, mark_value == CARD_MARKED_OLD)); /* set *all* to OLD */ assert(IMPLY(IS_OVERFLOW_OBJ(pseg, realobj), diff --git a/rpython/translator/stm/src_stm/stmgc.h b/rpython/translator/stm/src_stm/stmgc.h --- a/rpython/translator/stm/src_stm/stmgc.h +++ b/rpython/translator/stm/src_stm/stmgc.h @@ -260,6 +260,9 @@ one item */ extern void stmcb_get_card_base_itemsize(struct object_s *, uintptr_t offset_itemsize[2]); +/* returns whether this object supports cards. we will only call + stmcb_get_card_base_itemsize on objs that do so. */ +extern long stmcb_obj_supports_cards(struct object_s *); extern void stmcb_commit_soon(void); @@ -368,7 +371,7 @@ int stm_is_inevitable(void); #else static inline int stm_is_inevitable(void) { - return !rewind_jmp_armed(&STM_SEGMENT->running_thread->rjthread); + return !rewind_jmp_armed(&STM_SEGMENT->running_thread->rjthread); } #endif static inline void stm_become_inevitable(stm_thread_local_t *tl, From noreply at buildbot.pypy.org Mon Sep 1 12:43:07 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Mon, 1 Sep 2014 12:43:07 +0200 (CEST) Subject: [pypy-commit] pypy stmgc-c7: add pypy_stmcb_obj_supports_cards Message-ID: <20140901104307.E6CA91C025B@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: stmgc-c7 Changeset: r73268:db94b14e4f34 Date: 2014-09-01 12:43 +0200 http://bitbucket.org/pypy/pypy/changeset/db94b14e4f34/ Log: add pypy_stmcb_obj_supports_cards diff --git a/rpython/memory/gctransform/stmframework.py b/rpython/memory/gctransform/stmframework.py --- a/rpython/memory/gctransform/stmframework.py +++ b/rpython/memory/gctransform/stmframework.py @@ -33,6 +33,14 @@ llannotation.SomePtr(GCClass.VISIT_FPTR)], annmodel.s_None)) # + def pypy_stmcb_obj_supports_cards(obj): + typeid = gc.get_type_id(obj) + return gc.is_varsize(typeid) + pypy_stmcb_obj_supports_cards.c_name = "pypy_stmcb_obj_supports_cards" + self.autoregister_ptrs.append( + getfn(pypy_stmcb_obj_supports_cards, [llannotation.SomeAddress()], + annmodel.SomeInteger())) + # def pypy_stmcb_trace_cards(obj, visit_fn, start, stop): typeid = gc.get_type_id(obj) if not gc.has_gcptr_in_varsize(typeid): diff --git a/rpython/translator/stm/src_stm/stmgcintf.c b/rpython/translator/stm/src_stm/stmgcintf.c --- a/rpython/translator/stm/src_stm/stmgcintf.c +++ b/rpython/translator/stm/src_stm/stmgcintf.c @@ -13,6 +13,7 @@ extern void pypy_stmcb_get_card_base_itemsize(void*, uintptr_t[]); extern void pypy_stmcb_trace(void*, void(*)(void*)); extern void pypy_stmcb_trace_cards(void*, void(*)(void*), uintptr_t, uintptr_t); +extern Signed pypy_stmcb_obj_supports_cards(void*); inline ssize_t stmcb_size_rounded_up(struct object_s *obj) { ssize_t result = pypy_stmcb_size_rounded_up(obj); @@ -30,6 +31,10 @@ pypy_stmcb_trace(obj, (void(*)(void*))visit); } +inline long stmcb_obj_supports_cards(struct object_s *obj) { + return pypy_stmcb_obj_supports_cards(obj); +} + inline void stmcb_trace_cards(struct object_s *obj, void visit(object_t **), uintptr_t start, uintptr_t stop) { pypy_stmcb_trace_cards(obj, (void(*)(void*))visit, start, stop); From noreply at buildbot.pypy.org Mon Sep 1 15:15:57 2014 From: noreply at buildbot.pypy.org (groggi) Date: Mon, 1 Sep 2014 15:15:57 +0200 (CEST) Subject: [pypy-commit] pypy gc-incminimark-pinning: add first simple test for a weakref pointing to a pinned object. Fails right now. Message-ID: <20140901131557.65EA21C025B@cobra.cs.uni-duesseldorf.de> Author: Gregor Wegberg Branch: gc-incminimark-pinning Changeset: r73269:de940c201b84 Date: 2014-09-01 15:14 +0200 http://bitbucket.org/pypy/pypy/changeset/de940c201b84/ Log: add first simple test for a weakref pointing to a pinned object. Fails right now. diff --git a/rpython/memory/test/test_incminimark_gc.py b/rpython/memory/test/test_incminimark_gc.py --- a/rpython/memory/test/test_incminimark_gc.py +++ b/rpython/memory/test/test_incminimark_gc.py @@ -36,3 +36,27 @@ return ref() is b res = self.interpret(f, []) assert res == True + + def test_weakref_to_pinned(self): + import weakref + from rpython.rlib import rgc + class A(object): + pass + def g(): + a = A() + assert rgc.pin(a) + a.x = 100 + wr = weakref.ref(a) + llop.gc__collect(lltype.Void) + assert wr() is not None + assert a.x == 100 + return wr + def f(): + ref = g() + llop.gc__collect(lltype.Void, 1) + b = ref() + assert b is not None + b.x = 101 + return ref() is b + res = self.interpret(f, []) + assert res == True From noreply at buildbot.pypy.org Mon Sep 1 15:20:10 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Mon, 1 Sep 2014 15:20:10 +0200 (CEST) Subject: [pypy-commit] stmgc gc-small-uniform: Somehow merge default Message-ID: <20140901132010.9ED611C328B@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: gc-small-uniform Changeset: r1327:3aa3696e8add Date: 2014-09-01 14:15 +0200 http://bitbucket.org/pypy/stmgc/changeset/3aa3696e8add/ Log: Somehow merge default diff too long, truncating to 2000 out of 5104 lines diff --git a/c7/TODO b/c7/TODO --- a/c7/TODO +++ b/c7/TODO @@ -1,8 +1,6 @@ - use small uniform gcpages -- write barrier for big arrays - - finalizers - the highest_overflow_number can overflow after 2**30 non-collect-time @@ -16,3 +14,16 @@ the unused pages away --- or maybe use consecutive addresses from the lowest ones from segment N, instead of the page corresponding to the page number in segment 0 (possibly a bit messy) + +- possibly messy too, but think about not using N+1 segments but only N + +- use a call/cc-style variant of setjmp/longjmp to avoid inevitable + transactions when we need to return + +- kill "atomic" and use regular lock elision + +- increase the memory limit, currently 2.5GB; this requires, apparently, + more fighting against LLVM bugs + +- avoid __builtin_frame_address(0) in precisely the performance-critical + functions like the interpreter main loop diff --git a/c7/demo/demo2.c b/c7/demo/demo2.c --- a/c7/demo/demo2.c +++ b/c7/demo/demo2.c @@ -43,7 +43,20 @@ n = (struct node_s*)obj; visit((object_t **)&n->next); } - +long stmcb_obj_supports_cards(struct object_s *obj) +{ + return 0; +} +void stmcb_get_card_base_itemsize(struct object_s *obj, + uintptr_t offset_itemsize[2]) +{ + abort(); +} +void stmcb_trace_cards(struct object_s *obj, void visit(object_t **), + uintptr_t start, uintptr_t stop) +{ + abort(); +} void stmcb_commit_soon() {} static void expand_marker(char *base, uintptr_t odd_number, @@ -62,9 +75,8 @@ { nodeptr_t r_n; long prev, sum; - stm_jmpbuf_t here; - STM_START_TRANSACTION(&stm_thread_local, here); + stm_start_transaction(&stm_thread_local); stm_read((objptr_t)global_chained_list); r_n = global_chained_list; @@ -92,11 +104,9 @@ nodeptr_t swap_nodes(nodeptr_t initial) { - stm_jmpbuf_t here; - assert(initial != NULL); - STM_START_TRANSACTION(&stm_thread_local, here); + stm_start_transaction(&stm_thread_local); if (stm_thread_local.longest_marker_state != 0) { fprintf(stderr, "[%p] marker %d for %.6f seconds:\n", @@ -193,7 +203,7 @@ stm_commit_transaction(); - stm_start_inevitable_transaction(&stm_thread_local); + stm_start_transaction(&stm_thread_local); STM_POP_ROOT(stm_thread_local, global_chained_list); /* update value */ assert(global_chained_list->value == -1); STM_PUSH_ROOT(stm_thread_local, global_chained_list); /* remains forever in the shadow stack */ @@ -202,6 +212,11 @@ printf("setup ok\n"); } +void teardown_list(void) +{ + STM_POP_ROOT_RET(stm_thread_local); +} + static sem_t done; @@ -215,7 +230,9 @@ void *demo2(void *arg) { int status; + rewind_jmp_buf rjbuf; stm_register_thread_local(&stm_thread_local); + stm_rewind_jmp_enterframe(&stm_thread_local, &rjbuf); char *org = (char *)stm_thread_local.shadowstack; STM_PUSH_ROOT(stm_thread_local, global_chained_list); /* remains forever in the shadow stack */ @@ -235,6 +252,7 @@ STM_POP_ROOT(stm_thread_local, global_chained_list); OPT_ASSERT(org == (char *)stm_thread_local.shadowstack); + stm_rewind_jmp_leaveframe(&stm_thread_local, &rjbuf); unregister_thread_local(); status = sem_post(&done); assert(status == 0); return NULL; @@ -271,11 +289,13 @@ int main(void) { int status, i; + rewind_jmp_buf rjbuf; status = sem_init(&done, 0, 0); assert(status == 0); stm_setup(); stm_register_thread_local(&stm_thread_local); + stm_rewind_jmp_enterframe(&stm_thread_local, &rjbuf); stmcb_expand_marker = expand_marker; @@ -292,9 +312,11 @@ final_check(); + teardown_list(); + stm_rewind_jmp_leaveframe(&stm_thread_local, &rjbuf); unregister_thread_local(); - stm_teardown(); + //stm_teardown(); return 0; } diff --git a/c7/demo/demo_largemalloc.c b/c7/demo/demo_largemalloc.c --- a/c7/demo/demo_largemalloc.c +++ b/c7/demo/demo_largemalloc.c @@ -24,6 +24,18 @@ } void stmcb_commit_soon() {} +long stmcb_obj_supports_cards(struct object_s *obj) +{ + return 0; +} +void stmcb_trace_cards(struct object_s *obj, void cb(object_t **), + uintptr_t start, uintptr_t stop) { + abort(); +} +void stmcb_get_card_base_itemsize(struct object_s *obj, + uintptr_t offset_itemsize[2]) { + abort(); +} /************************************************************/ @@ -67,7 +79,7 @@ int i; arena_data = malloc(ARENA_SIZE); assert(arena_data != NULL); - _stm_mutex_pages_lock(); + //_stm_mutex_pages_lock(); for (i = 0; i < 25; i++) timing(i); return 0; diff --git a/c7/demo/demo_random.c b/c7/demo/demo_random.c --- a/c7/demo/demo_random.c +++ b/c7/demo/demo_random.c @@ -80,6 +80,18 @@ } void stmcb_commit_soon() {} +long stmcb_obj_supports_cards(struct object_s *obj) +{ + return 0; +} +void stmcb_trace_cards(struct object_s *obj, void cb(object_t **), + uintptr_t start, uintptr_t stop) { + abort(); +} +void stmcb_get_card_base_itemsize(struct object_s *obj, + uintptr_t offset_itemsize[2]) { + abort(); +} int get_rand(int max) { @@ -323,15 +335,15 @@ void *demo_random(void *arg) { int status; + rewind_jmp_buf rjbuf; stm_register_thread_local(&stm_thread_local); + stm_rewind_jmp_enterframe(&stm_thread_local, &rjbuf); setup_thread(); objptr_t p; - stm_jmpbuf_t here; - volatile int call_fork = (arg != NULL); - STM_START_TRANSACTION(&stm_thread_local, here); + stm_start_transaction(&stm_thread_local); assert(td.num_roots >= td.num_roots_at_transaction_start); td.num_roots = td.num_roots_at_transaction_start; p = NULL; @@ -349,11 +361,12 @@ if (p == (objptr_t)-1) { push_roots(); + long call_fork = (arg != NULL && *(long *)arg); if (call_fork == 0) { /* common case */ stm_commit_transaction(); td.num_roots_at_transaction_start = td.num_roots; if (get_rand(100) < 98) { - STM_START_TRANSACTION(&stm_thread_local, here); + stm_start_transaction(&stm_thread_local); } else { stm_start_inevitable_transaction(&stm_thread_local); } @@ -365,7 +378,7 @@ else { /* run a fork() inside the transaction */ printf("========== FORK =========\n"); - call_fork = 0; + *(long*)arg = 0; pid_t child = fork(); printf("=== in process %d thread %lx, fork() returned %d\n", (int)getpid(), (long)pthread_self(), (int)child); @@ -383,8 +396,19 @@ } } } + push_roots(); stm_commit_transaction(); + /* even out the shadow stack before leaveframe: */ + stm_start_inevitable_transaction(&stm_thread_local); + while (td.num_roots > 0) { + td.num_roots--; + objptr_t t; + STM_POP_ROOT(stm_thread_local, t); + } + stm_commit_transaction(); + + stm_rewind_jmp_leaveframe(&stm_thread_local, &rjbuf); stm_unregister_thread_local(&stm_thread_local); status = sem_post(&done); assert(status == 0); @@ -433,6 +457,7 @@ int main(void) { int i, status; + rewind_jmp_buf rjbuf; /* pick a random seed from the time in seconds. A bit pointless for now... because the interleaving of the @@ -446,6 +471,7 @@ stm_setup(); stm_register_thread_local(&stm_thread_local); + stm_rewind_jmp_enterframe(&stm_thread_local, &rjbuf); setup_globals(); @@ -463,7 +489,7 @@ long forkbase = NUMTHREADS * THREAD_STARTS / (FORKS + 1); long _fork = (thread_starts % forkbase) == 0; thread_starts--; - newthread(demo_random, (void *)_fork); + newthread(demo_random, &_fork); } } @@ -483,6 +509,7 @@ printf("Test OK!\n"); + stm_rewind_jmp_leaveframe(&stm_thread_local, &rjbuf); stm_unregister_thread_local(&stm_thread_local); stm_teardown(); diff --git a/c7/demo/demo_random2.c b/c7/demo/demo_random2.c new file mode 100644 --- /dev/null +++ b/c7/demo/demo_random2.c @@ -0,0 +1,540 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "stmgc.h" + +#define NUMTHREADS 3 +#define STEPS_PER_THREAD 500 +#define THREAD_STARTS 1000 // how many restarts of threads +#define PREBUILT_ROOTS 3 +#define FORKS 3 + +#define ACTIVE_ROOTS_SET_SIZE 100 // max num of roots created/alive in one transaction +#define MAX_ROOTS_ON_SS 1000 // max on shadow stack + +// SUPPORT +struct node_s; +typedef TLPREFIX struct node_s node_t; +typedef node_t* nodeptr_t; +typedef object_t* objptr_t; +int num_forked_children = 0; + +struct node_s { + struct object_s hdr; + int sig; + long my_size; + long my_id; + long my_hash; + nodeptr_t next; +}; + +#define SIGNATURE 0x01234567 + + +static sem_t done; +__thread stm_thread_local_t stm_thread_local; +__thread void *thread_may_fork; + +// global and per-thread-data +time_t default_seed; +objptr_t prebuilt_roots[PREBUILT_ROOTS]; + +struct thread_data { + unsigned int thread_seed; + int steps_left; + objptr_t active_roots_set[ACTIVE_ROOTS_SET_SIZE]; + int active_roots_num; + long roots_on_ss; + long roots_on_ss_at_tr_start; +}; +__thread struct thread_data td; + +struct thread_data *_get_td(void) +{ + return &td; /* for gdb */ +} + + +ssize_t stmcb_size_rounded_up(struct object_s *ob) +{ + return ((struct node_s*)ob)->my_size; +} + +void stmcb_trace(struct object_s *obj, void visit(object_t **)) +{ + struct node_s *n; + n = (struct node_s*)obj; + + /* and the same value at the end: */ + /* note, ->next may be the same as last_next */ + nodeptr_t *last_next = (nodeptr_t*)((char*)n + n->my_size - sizeof(void*)); + + assert(n->next == *last_next); + + visit((object_t **)&n->next); + visit((object_t **)last_next); + + assert(n->next == *last_next); +} + +void stmcb_commit_soon() {} + +long stmcb_obj_supports_cards(struct object_s *obj) +{ + return 0; +} +void stmcb_trace_cards(struct object_s *obj, void cb(object_t **), + uintptr_t start, uintptr_t stop) { + abort(); +} +void stmcb_get_card_base_itemsize(struct object_s *obj, + uintptr_t offset_itemsize[2]) { + abort(); +} + +int get_rand(int max) +{ + if (max == 0) + return 0; + return (int)(rand_r(&td.thread_seed) % (unsigned int)max); +} + +objptr_t get_random_root() +{ + /* get some root from shadowstack or active_root_set or prebuilt_roots */ + int num = get_rand(3); + intptr_t ss_size = td.roots_on_ss; + if (num == 0 && ss_size > 0) { + num = get_rand(ss_size); + /* XXX: impl detail: there is already a "-1" on the SS -> +1 */ + objptr_t r = (objptr_t)stm_thread_local.shadowstack_base[num+1].ss; + OPT_ASSERT((((uintptr_t)r) & 3) == 0); + } + + if (num == 1 && td.active_roots_num > 0) { + num = get_rand(td.active_roots_num); + return td.active_roots_set[num]; + } else { + num = get_rand(PREBUILT_ROOTS); + return prebuilt_roots[num]; + } +} + + +long push_roots() +{ + int i; + long to_push = td.active_roots_num; + long not_pushed = 0; + for (i = to_push - 1; i >= 0; i--) { + td.active_roots_num--; + if (td.roots_on_ss < MAX_ROOTS_ON_SS) { + STM_PUSH_ROOT(stm_thread_local, td.active_roots_set[i]); + td.roots_on_ss++; + } else { + not_pushed++; + } + } + return to_push - not_pushed; +} + +void add_root(objptr_t r); +void pop_roots(long to_pop) +{ + int i; + for (i = 0; i < to_pop; i++) { + objptr_t t; + STM_POP_ROOT(stm_thread_local, t); + add_root(t); + td.roots_on_ss--; + } +} + +void del_root(int idx) +{ + int i; + + for (i = idx; i < td.active_roots_num - 1; i++) + td.active_roots_set[i] = td.active_roots_set[i + 1]; + td.active_roots_num--; +} + +void add_root(objptr_t r) +{ + if (r && td.active_roots_num < ACTIVE_ROOTS_SET_SIZE) { + td.active_roots_set[td.active_roots_num++] = r; + } +} + + +void read_barrier(objptr_t p) +{ + if (p != NULL) { + stm_read(p); + } +} + +void write_barrier(objptr_t p) +{ + if (p != NULL) { + stm_write(p); + } +} + +void set_next(objptr_t p, objptr_t v) +{ + if (p != NULL) { + nodeptr_t n = (nodeptr_t)p; + + /* and the same value at the end: */ + nodeptr_t TLPREFIX *last_next = (nodeptr_t TLPREFIX *)((stm_char*)n + n->my_size - sizeof(void*)); + assert(n->next == *last_next); + n->next = (nodeptr_t)v; + *last_next = (nodeptr_t)v; + } +} + +nodeptr_t get_next(objptr_t p) +{ + nodeptr_t n = (nodeptr_t)p; + + /* and the same value at the end: */ + nodeptr_t TLPREFIX *last_next = (nodeptr_t TLPREFIX *)((stm_char*)n + n->my_size - sizeof(void*)); + OPT_ASSERT(n->next == *last_next); + + return n->next; +} + + +objptr_t simple_events(objptr_t p, objptr_t _r) +{ + int k = get_rand(10); + long pushed; + + switch (k) { + case 0: // remove a root + if (td.active_roots_num) { + del_root(get_rand(td.active_roots_num)); + } + break; + case 1: // add 'p' to roots + add_root(p); + break; + case 2: // set 'p' to point to a root + if (_r) + p = _r; + break; + case 3: // allocate fresh 'p' + pushed = push_roots(); + size_t sizes[4] = {sizeof(struct node_s), + sizeof(struct node_s) + (get_rand(100000) & ~15), + sizeof(struct node_s) + 4096, + sizeof(struct node_s) + 4096*70}; + size_t size = sizes[get_rand(4)]; + p = stm_allocate(size); + ((nodeptr_t)p)->sig = SIGNATURE; + ((nodeptr_t)p)->my_size = size; + ((nodeptr_t)p)->my_id = 0; + ((nodeptr_t)p)->my_hash = 0; + pop_roots(pushed); + break; + case 4: // read and validate 'p' + read_barrier(p); + break; + case 5: // only do a stm_write_barrier + write_barrier(p); + break; + case 6: // follow p->next + if (p) { + read_barrier(p); + p = (objptr_t)(get_next(p)); + } + break; + case 7: // set 'p' as *next in one of the roots + write_barrier(_r); + set_next(_r, p); + break; + case 8: // id checking + if (p) { + nodeptr_t n = (nodeptr_t)p; + if (n->my_id == 0) { + write_barrier(p); + n->my_id = stm_id(p); + } + else { + read_barrier(p); + assert(n->my_id == stm_id(p)); + } + } + break; + case 9: + if (p) { + nodeptr_t n = (nodeptr_t)p; + if (n->my_hash == 0) { + write_barrier(p); + n->my_hash = stm_identityhash(p); + } + else { + read_barrier(p); + assert(n->my_hash == stm_identityhash(p)); + } + } + break; + } + return p; +} + +void frame_loop(); +objptr_t do_step(objptr_t p) +{ + objptr_t _r; + int k; + + _r = get_random_root(); + k = get_rand(12); + + if (k < 10) { + p = simple_events(p, _r); + } else if (get_rand(20) == 1) { + long pushed = push_roots(); + stm_commit_transaction(); + td.roots_on_ss_at_tr_start = td.roots_on_ss; + + if (get_rand(100) < 98) { + stm_start_transaction(&stm_thread_local); + } else { + stm_start_inevitable_transaction(&stm_thread_local); + } + td.roots_on_ss = td.roots_on_ss_at_tr_start; + td.active_roots_num = 0; + pop_roots(pushed); + p = NULL; + } else if (get_rand(10) == 1) { + long pushed = push_roots(); + /* leaving our frame */ + frame_loop(); + /* back in our frame */ + pop_roots(pushed); + p = NULL; + } else if (get_rand(20) == 1) { + long pushed = push_roots(); + stm_become_inevitable(&stm_thread_local, "please"); + assert(stm_is_inevitable()); + pop_roots(pushed); + p= NULL; + } else if (get_rand(20) == 1) { + p = (objptr_t)-1; // possibly fork + } else if (get_rand(20) == 1) { + long pushed = push_roots(); + stm_become_globally_unique_transaction(&stm_thread_local, "really"); + fprintf(stderr, "[GUT/%d]", (int)STM_SEGMENT->segment_num); + pop_roots(pushed); + p = NULL; + } + return p; +} + +void frame_loop() +{ + objptr_t p = NULL; + rewind_jmp_buf rjbuf; + + stm_rewind_jmp_enterframe(&stm_thread_local, &rjbuf); + //fprintf(stderr,"%p F: %p\n", STM_SEGMENT->running_thread, __builtin_frame_address(0)); + + long roots_on_ss = td.roots_on_ss; + /* "interpreter main loop": this is one "application-frame" */ + while (td.steps_left-->0 && get_rand(10) != 0) { + if (td.steps_left % 8 == 0) + fprintf(stdout, "#"); + + assert(p == NULL || ((nodeptr_t)p)->sig == SIGNATURE); + + p = do_step(p); + + + if (p == (objptr_t)-1) { + p = NULL; + + long call_fork = (thread_may_fork != NULL && *(long *)thread_may_fork); + if (call_fork) { /* common case */ + long pushed = push_roots(); + /* run a fork() inside the transaction */ + printf("========== FORK =========\n"); + *(long*)thread_may_fork = 0; + pid_t child = fork(); + printf("=== in process %d thread %lx, fork() returned %d\n", + (int)getpid(), (long)pthread_self(), (int)child); + if (child == -1) { + fprintf(stderr, "fork() error: %m\n"); + abort(); + } + if (child != 0) + num_forked_children++; + else + num_forked_children = 0; + + pop_roots(pushed); + } + } + } + OPT_ASSERT(roots_on_ss == td.roots_on_ss); + + stm_rewind_jmp_leaveframe(&stm_thread_local, &rjbuf); +} + + + +void setup_thread() +{ + memset(&td, 0, sizeof(struct thread_data)); + + /* stupid check because gdb shows garbage + in td.roots: */ + int i; + for (i = 0; i < ACTIVE_ROOTS_SET_SIZE; i++) + assert(td.active_roots_set[i] == NULL); + + td.thread_seed = default_seed++; + td.steps_left = STEPS_PER_THREAD; + td.active_roots_num = 0; + td.roots_on_ss = 0; + td.roots_on_ss_at_tr_start = 0; +} + + + +void *demo_random(void *arg) +{ + int status; + rewind_jmp_buf rjbuf; + stm_register_thread_local(&stm_thread_local); + stm_rewind_jmp_enterframe(&stm_thread_local, &rjbuf); + + setup_thread(); + + td.roots_on_ss_at_tr_start = 0; + stm_start_transaction(&stm_thread_local); + td.roots_on_ss = td.roots_on_ss_at_tr_start; + td.active_roots_num = 0; + + thread_may_fork = arg; + while (td.steps_left-->0) { + frame_loop(); + } + + stm_commit_transaction(); + + stm_rewind_jmp_leaveframe(&stm_thread_local, &rjbuf); + stm_unregister_thread_local(&stm_thread_local); + + status = sem_post(&done); assert(status == 0); + return NULL; +} + +void newthread(void*(*func)(void*), void *arg) +{ + pthread_t th; + int status = pthread_create(&th, NULL, func, arg); + if (status != 0) + abort(); + pthread_detach(th); + printf("started new thread\n"); +} + + +void setup_globals() +{ + int i; + + struct node_s prebuilt_template = { + .sig = SIGNATURE, + .my_size = sizeof(struct node_s), + .my_id = 0, + .my_hash = 0, + .next = NULL + }; + + stm_start_inevitable_transaction(&stm_thread_local); + for (i = 0; i < PREBUILT_ROOTS; i++) { + void* new_templ = malloc(sizeof(struct node_s)); + memcpy(new_templ, &prebuilt_template, sizeof(struct node_s)); + prebuilt_roots[i] = stm_setup_prebuilt((objptr_t)(long)new_templ); + + if (i % 2 == 0) { + int hash = i + 5; + stm_set_prebuilt_identityhash(prebuilt_roots[i], + hash); + ((nodeptr_t)prebuilt_roots[i])->my_hash = hash; + } + } + stm_commit_transaction(); +} + +int main(void) +{ + int i, status; + rewind_jmp_buf rjbuf; + + /* pick a random seed from the time in seconds. + A bit pointless for now... because the interleaving of the + threads is really random. */ + default_seed = time(NULL); + printf("running with seed=%lld\n", (long long)default_seed); + + status = sem_init(&done, 0, 0); + assert(status == 0); + + + stm_setup(); + stm_register_thread_local(&stm_thread_local); + stm_rewind_jmp_enterframe(&stm_thread_local, &rjbuf); + + setup_globals(); + + int thread_starts = NUMTHREADS * THREAD_STARTS; + for (i = 0; i < NUMTHREADS; i++) { + newthread(demo_random, NULL); + thread_starts--; + } + + for (i=0; i < NUMTHREADS * THREAD_STARTS; i++) { + status = sem_wait(&done); + assert(status == 0); + printf("thread finished\n"); + if (thread_starts) { + long forkbase = NUMTHREADS * THREAD_STARTS / (FORKS + 1); + long _fork = (thread_starts % forkbase) == 0; + thread_starts--; + newthread(demo_random, &_fork); + } + } + + for (i = 0; i < num_forked_children; i++) { + pid_t child = wait(&status); + if (child == -1) + perror("wait"); + printf("From %d: child %d terminated with exit status %d\n", + (int)getpid(), (int)child, status); + if (WIFEXITED(status) && WEXITSTATUS(status) == 0) + ; + else { + printf("*** error from the child ***\n"); + return 1; + } + } + + printf("Test OK!\n"); + + stm_rewind_jmp_leaveframe(&stm_thread_local, &rjbuf); + stm_unregister_thread_local(&stm_thread_local); + stm_teardown(); + + return 0; +} diff --git a/c7/demo/demo_simple.c b/c7/demo/demo_simple.c --- a/c7/demo/demo_simple.c +++ b/c7/demo/demo_simple.c @@ -10,7 +10,7 @@ # include "stmgc.h" #endif -#define ITERS 1000000 +#define ITERS 100000 #define NTHREADS 2 @@ -38,27 +38,40 @@ n = (struct node_s*)obj; visit((object_t **)&n->next); } - +long stmcb_obj_supports_cards(struct object_s *obj) +{ + return 0; +} void stmcb_commit_soon() {} +void stmcb_trace_cards(struct object_s *obj, void cb(object_t **), + uintptr_t start, uintptr_t stop) { + abort(); +} +void stmcb_get_card_base_itemsize(struct object_s *obj, + uintptr_t offset_itemsize[2]) { + abort(); +} static sem_t done; static __thread int tl_counter = 0; -static int gl_counter = 0; +//static int gl_counter = 0; void *demo2(void *arg) { int status; + rewind_jmp_buf rjbuf; stm_register_thread_local(&stm_thread_local); + stm_rewind_jmp_enterframe(&stm_thread_local, &rjbuf); char *org = (char *)stm_thread_local.shadowstack; tl_counter = 0; object_t *tmp; int i = 0; while (i < ITERS) { - stm_start_inevitable_transaction(&stm_thread_local); + stm_start_transaction(&stm_thread_local); tl_counter++; if (i % 500 < 250) STM_PUSH_ROOT(stm_thread_local, stm_allocate(16));//gl_counter++; @@ -68,8 +81,9 @@ i++; } - assert(org == (char *)stm_thread_local.shadowstack); + OPT_ASSERT(org == (char *)stm_thread_local.shadowstack); + stm_rewind_jmp_leaveframe(&stm_thread_local, &rjbuf); stm_unregister_thread_local(&stm_thread_local); status = sem_post(&done); assert(status == 0); return NULL; diff --git a/c7/demo/test_shadowstack.c b/c7/demo/test_shadowstack.c new file mode 100644 --- /dev/null +++ b/c7/demo/test_shadowstack.c @@ -0,0 +1,74 @@ +#include +#include +#include "stmgc.h" + +stm_thread_local_t stm_thread_local; + +typedef TLPREFIX struct node_s node_t; + +struct node_s { + struct object_s hdr; + long value; +}; + +ssize_t stmcb_size_rounded_up(struct object_s *ob) +{ + return sizeof(struct node_s); +} +void stmcb_trace(struct object_s *obj, void visit(object_t **)) +{ +} +long stmcb_obj_supports_cards(struct object_s *obj) +{ + return 0; +} +void stmcb_get_card_base_itemsize(struct object_s *obj, + uintptr_t offset_itemsize[2]) +{ + abort(); +} +void stmcb_trace_cards(struct object_s *obj, void visit(object_t **), + uintptr_t start, uintptr_t stop) +{ + abort(); +} +void stmcb_commit_soon() {} + + +int main(void) +{ + rewind_jmp_buf rjbuf; + + stm_setup(); + stm_register_thread_local(&stm_thread_local); + stm_rewind_jmp_enterframe(&stm_thread_local, &rjbuf); + + stm_start_transaction(&stm_thread_local); + node_t *node = (node_t *)stm_allocate(sizeof(struct node_s)); + node->value = 129821; + STM_PUSH_ROOT(stm_thread_local, node); + STM_PUSH_ROOT(stm_thread_local, 333); /* odd value */ + stm_commit_transaction(); + + /* now in a new transaction, pop the node off the shadowstack, but + then do a major collection. It should still be found by the + tracing logic. */ + stm_start_transaction(&stm_thread_local); + STM_POP_ROOT_RET(stm_thread_local); + STM_POP_ROOT(stm_thread_local, node); + assert(node->value == 129821); + STM_PUSH_ROOT(stm_thread_local, NULL); + stm_collect(9); + + node_t *node2 = (node_t *)stm_allocate(sizeof(struct node_s)); + assert(node2 != node); + assert(node->value == 129821); + + STM_PUSH_ROOT(stm_thread_local, node2); + stm_collect(0); + STM_POP_ROOT(stm_thread_local, node2); + assert(node2 != node); + assert(node->value == 129821); + + return 0; +} diff --git a/c7/stm/contention.c b/c7/stm/contention.c --- a/c7/stm/contention.c +++ b/c7/stm/contention.c @@ -98,13 +98,14 @@ /************************************************************/ -static void contention_management(uint8_t other_segment_num, +static bool contention_management(uint8_t other_segment_num, enum contention_kind_e kind, object_t *obj) { assert(_has_mutex()); assert(other_segment_num != STM_SEGMENT->segment_num); + bool others_may_have_run = false; if (must_abort()) abort_with_mutex(); @@ -152,6 +153,7 @@ if (contmgr.try_sleep && kind != WRITE_WRITE_CONTENTION && contmgr.other_pseg->safe_point != SP_WAIT_FOR_C_TRANSACTION_DONE) { + others_may_have_run = true; /* Sleep. - Not for write-write contentions, because we're not at a @@ -192,7 +194,7 @@ /* tell the other to commit ASAP, since it causes aborts */ signal_other_to_commit_soon(contmgr.other_pseg); - dprintf(("abort in contention\n")); + dprintf(("abort in contention: kind %d\n", kind)); STM_SEGMENT->nursery_end = abort_category; marker_contention(kind, false, other_segment_num, obj); abort_with_mutex(); @@ -225,6 +227,7 @@ if (must_abort()) abort_with_mutex(); + others_may_have_run = true; dprintf(("contention: wait C_ABORTED...\n")); cond_wait(C_ABORTED); dprintf(("contention: done\n")); @@ -278,6 +281,7 @@ stmcb_commit_soon(); } } + return others_may_have_run; } static void write_write_contention_management(uintptr_t lock_idx, @@ -301,10 +305,10 @@ s_mutex_unlock(); } -static void write_read_contention_management(uint8_t other_segment_num, +static bool write_read_contention_management(uint8_t other_segment_num, object_t *obj) { - contention_management(other_segment_num, WRITE_READ_CONTENTION, obj); + return contention_management(other_segment_num, WRITE_READ_CONTENTION, obj); } static void inevitable_contention_management(uint8_t other_segment_num) diff --git a/c7/stm/contention.h b/c7/stm/contention.h --- a/c7/stm/contention.h +++ b/c7/stm/contention.h @@ -1,7 +1,7 @@ static void write_write_contention_management(uintptr_t lock_idx, object_t *obj); -static void write_read_contention_management(uint8_t other_segment_num, +static bool write_read_contention_management(uint8_t other_segment_num, object_t *obj); static void inevitable_contention_management(uint8_t other_segment_num); diff --git a/c7/stm/core.c b/c7/stm/core.c --- a/c7/stm/core.c +++ b/c7/stm/core.c @@ -40,26 +40,67 @@ #endif } -void _stm_write_slowpath(object_t *obj) +__attribute__((always_inline)) +static void write_slowpath_overflow_obj(object_t *obj, bool mark_card) +{ + /* An overflow object is an object from the same transaction, but + outside the nursery. More precisely, it is no longer young, + i.e. it comes from before the most recent minor collection. + */ + assert(STM_PSEGMENT->objects_pointing_to_nursery != NULL); + + assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); + if (!mark_card) { + /* The basic case, with no card marking. We append the object + into 'objects_pointing_to_nursery', and remove the flag so + that the write_slowpath will not be called again until the + next minor collection. */ + if (obj->stm_flags & GCFLAG_CARDS_SET) { + /* if we clear this flag, we also need to clear the cards */ + _reset_object_cards(get_priv_segment(STM_SEGMENT->segment_num), + obj, CARD_CLEAR, false); + } + obj->stm_flags &= ~(GCFLAG_WRITE_BARRIER | GCFLAG_CARDS_SET); + LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, obj); + } + else { + /* Card marking. Don't remove GCFLAG_WRITE_BARRIER because we + need to come back to _stm_write_slowpath_card() for every + card to mark. Add GCFLAG_CARDS_SET. */ + assert(!(obj->stm_flags & GCFLAG_CARDS_SET)); + obj->stm_flags |= GCFLAG_CARDS_SET; + assert(STM_PSEGMENT->old_objects_with_cards); + LIST_APPEND(STM_PSEGMENT->old_objects_with_cards, obj); + } +} + +__attribute__((always_inline)) +static void write_slowpath_common(object_t *obj, bool mark_card) { assert(_seems_to_be_running_transaction()); assert(!_is_young(obj)); assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); - /* is this an object from the same transaction, outside the nursery? */ - if ((obj->stm_flags & -GCFLAG_OVERFLOW_NUMBER_bit0) == - STM_PSEGMENT->overflow_number) { + uintptr_t base_lock_idx = get_write_lock_idx((uintptr_t)obj); - dprintf_test(("write_slowpath %p -> ovf obj_to_nurs\n", obj)); - obj->stm_flags &= ~GCFLAG_WRITE_BARRIER; - assert(STM_PSEGMENT->objects_pointing_to_nursery != NULL); - LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, obj); + if (IS_OVERFLOW_OBJ(STM_PSEGMENT, obj)) { + assert(write_locks[base_lock_idx] == 0); + write_slowpath_overflow_obj(obj, mark_card); return; } + /* Else, it's an old object and we need to privatise it. + Do a read-barrier now. Note that this must occur before the + safepoints that may be issued in write_write_contention_management(). + */ + stm_read(obj); - /* do a read-barrier now. Note that this must occur before the - safepoints that may be issued in write_write_contention_management(). */ - stm_read(obj); + /* Take the segment's own lock number */ + uint8_t lock_num = STM_PSEGMENT->write_lock_num; + + /* If CARDS_SET, we entered here at least once already, so we + already own the write_lock */ + assert(IMPLY(obj->stm_flags & GCFLAG_CARDS_SET, + write_locks[base_lock_idx] == lock_num)); /* XXX XXX XXX make the logic of write-locking objects optional! */ @@ -68,16 +109,14 @@ 'modified_old_objects' (but, because it had GCFLAG_WRITE_BARRIER, not in 'objects_pointing_to_nursery'). We'll detect this case by finding that we already own the write-lock. */ - uintptr_t lock_idx = (((uintptr_t)obj) >> 4) - WRITELOCK_START; - uint8_t lock_num = STM_PSEGMENT->write_lock_num; - assert(lock_idx < sizeof(write_locks)); + retry: - if (write_locks[lock_idx] == 0) { + if (write_locks[base_lock_idx] == 0) { /* A lock to prevent reading garbage from lookup_other_thread_recorded_marker() */ acquire_marker_lock(STM_SEGMENT->segment_base); - if (UNLIKELY(!__sync_bool_compare_and_swap(&write_locks[lock_idx], + if (UNLIKELY(!__sync_bool_compare_and_swap(&write_locks[base_lock_idx], 0, lock_num))) { release_marker_lock(STM_SEGMENT->segment_base); goto retry; @@ -119,16 +158,15 @@ realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj); obj_size = stmcb_size_rounded_up((struct object_s *)realobj); - /* that's the page *following* the last page with the object */ - end_page = (((uintptr_t)obj) + obj_size + 4095) / 4096UL; + /* get the last page containing data from the object */ + end_page = (((uintptr_t)obj) + obj_size - 1) / 4096UL; - for (i = first_page; i < end_page; i++) { + for (i = first_page; i <= end_page; i++) { page_privatize(i); } } } - else if (write_locks[lock_idx] == lock_num) { - OPT_ASSERT(STM_PSEGMENT->objects_pointing_to_nursery != NULL); + else if (write_locks[base_lock_idx] == lock_num) { #ifdef STM_TESTS bool found = false; LIST_FOREACH_R(STM_PSEGMENT->modified_old_objects, object_t *, @@ -139,17 +177,10 @@ else { /* call the contention manager, and then retry (unless we were aborted). */ - write_write_contention_management(lock_idx, obj); + write_write_contention_management(base_lock_idx, obj); goto retry; } - /* A common case for write_locks[] that was either 0 or lock_num: - we need to add the object to 'objects_pointing_to_nursery' - if there is such a list. */ - if (STM_PSEGMENT->objects_pointing_to_nursery != NULL) { - dprintf_test(("write_slowpath %p -> old obj_to_nurs\n", obj)); - LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, obj); - } /* check that we really have a private page */ assert(is_private_page(STM_SEGMENT->segment_num, @@ -158,16 +189,125 @@ /* check that so far all copies of the object have the flag */ check_flag_write_barrier(obj); - /* remove GCFLAG_WRITE_BARRIER, but only if we succeeded in - getting the write-lock */ assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); - obj->stm_flags &= ~GCFLAG_WRITE_BARRIER; + if (!mark_card) { + /* A common case for write_locks[] that was either 0 or lock_num: + we need to add the object to the appropriate list if there is one. + */ + if (STM_PSEGMENT->objects_pointing_to_nursery != NULL) { + dprintf_test(("write_slowpath %p -> old obj_to_nurs\n", obj)); + LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, obj); + } + + if (obj->stm_flags & GCFLAG_CARDS_SET) { + /* if we clear this flag, we have to tell sync_old_objs that + everything needs to be synced */ + _reset_object_cards(get_priv_segment(STM_SEGMENT->segment_num), + obj, CARD_MARKED_OLD, true); /* mark all */ + } + + /* remove GCFLAG_WRITE_BARRIER if we succeeded in getting the base + write-lock (not for card marking). */ + obj->stm_flags &= ~(GCFLAG_WRITE_BARRIER | GCFLAG_CARDS_SET); + } + else { + /* don't remove WRITE_BARRIER, but add CARDS_SET */ + obj->stm_flags |= GCFLAG_CARDS_SET; + assert(STM_PSEGMENT->old_objects_with_cards); + LIST_APPEND(STM_PSEGMENT->old_objects_with_cards, obj); + } /* for sanity, check again that all other segment copies of this object still have the flag (so privatization worked) */ check_flag_write_barrier(obj); } +void _stm_write_slowpath(object_t *obj) +{ + write_slowpath_common(obj, /*mark_card=*/false); +} + +static bool obj_should_use_cards(object_t *obj) +{ + struct object_s *realobj = (struct object_s *) + REAL_ADDRESS(STM_SEGMENT->segment_base, obj); + long supports = stmcb_obj_supports_cards(realobj); + if (!supports) + return 0; + + /* check also if it makes sense: */ + size_t size = stmcb_size_rounded_up(realobj); + return (size >= _STM_MIN_CARD_OBJ_SIZE); +} + +char _stm_write_slowpath_card_extra(object_t *obj) +{ + /* the PyPy JIT calls this function directly if it finds that an + array doesn't have the GCFLAG_CARDS_SET */ + bool mark_card = obj_should_use_cards(obj); + write_slowpath_common(obj, mark_card); + return mark_card; +} + +long _stm_write_slowpath_card_extra_base(void) +{ + /* for the PyPy JIT: _stm_write_slowpath_card_extra_base[obj >> 4] + is the byte that must be set to CARD_MARKED. The logic below + does the same, but more explicitly. */ + return (((long)write_locks) - WRITELOCK_START + 1) + + 0x4000000000000000L; // <- workaround for a clang bug :-( +} + +void _stm_write_slowpath_card(object_t *obj, uintptr_t index) +{ + /* If CARDS_SET is not set so far, issue a normal write barrier. + If the object is large enough, ask it to set up the object for + card marking instead. + */ + if (!(obj->stm_flags & GCFLAG_CARDS_SET)) { + char mark_card = _stm_write_slowpath_card_extra(obj); + if (!mark_card) + return; + } + + dprintf_test(("write_slowpath_card %p -> index:%lu\n", + obj, index)); + + /* We reach this point if we have to mark the card. + */ + assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); + assert(obj->stm_flags & GCFLAG_CARDS_SET); + assert(!(obj->stm_flags & GCFLAG_SMALL_UNIFORM)); /* not supported/tested */ + +#ifndef NDEBUG + struct object_s *realobj = (struct object_s *) + REAL_ADDRESS(STM_SEGMENT->segment_base, obj); + size_t size = stmcb_size_rounded_up(realobj); + /* we need at least one lock in addition to the STM-reserved object + write-lock */ + assert(size >= 32); + /* the 'index' must be in range(length-of-obj), but we don't have + a direct way to know the length. We know that it is smaller + than the size in bytes. */ + assert(index < size); +#endif + + /* Write into the card's lock. This is used by the next minor + collection to know what parts of the big object may have changed. + We already own the object here or it is an overflow obj. */ + uintptr_t base_lock_idx = get_write_lock_idx((uintptr_t)obj); + uintptr_t card_lock_idx = base_lock_idx + get_index_to_card_index(index); + write_locks[card_lock_idx] = CARD_MARKED; + + /* More debug checks */ + dprintf(("mark %p index %lu, card:%lu with %d\n", + obj, index, get_index_to_card_index(index), CARD_MARKED)); + assert(IMPLY(IS_OVERFLOW_OBJ(STM_PSEGMENT, obj), + write_locks[base_lock_idx] == 0)); + assert(IMPLY(!IS_OVERFLOW_OBJ(STM_PSEGMENT, obj), + write_locks[base_lock_idx] == STM_PSEGMENT->write_lock_num)); +} + static void reset_transaction_read_version(void) { /* force-reset all read markers to 0 */ @@ -188,14 +328,12 @@ STM_SEGMENT->transaction_read_version = 1; } -void _stm_start_transaction(stm_thread_local_t *tl, stm_jmpbuf_t *jmpbuf) +static void _stm_start_transaction(stm_thread_local_t *tl, bool inevitable) { assert(!_stm_in_transaction(tl)); - s_mutex_lock(); - retry: - if (jmpbuf == NULL) { + if (inevitable) { wait_for_end_of_inevitable_transaction(tl); } @@ -209,14 +347,10 @@ STM_PSEGMENT->start_time = tl->_timing_cur_start; STM_PSEGMENT->signalled_to_commit_soon = false; STM_PSEGMENT->safe_point = SP_RUNNING; -#ifndef NDEBUG - STM_PSEGMENT->marker_inev[1] = 99999999999999999L; -#endif - if (jmpbuf == NULL) + STM_PSEGMENT->marker_inev[1] = 0; + if (inevitable) marker_fetch_inev(); - STM_PSEGMENT->transaction_state = (jmpbuf != NULL ? TS_REGULAR - : TS_INEVITABLE); - STM_SEGMENT->jmpbuf_ptr = jmpbuf; + STM_PSEGMENT->transaction_state = (inevitable ? TS_INEVITABLE : TS_REGULAR); #ifndef NDEBUG STM_PSEGMENT->running_pthread = pthread_self(); #endif @@ -245,7 +379,8 @@ assert(list_is_empty(STM_PSEGMENT->young_weakrefs)); assert(tree_is_cleared(STM_PSEGMENT->young_outside_nursery)); assert(tree_is_cleared(STM_PSEGMENT->nursery_objects_shadows)); - assert(tree_is_cleared(STM_PSEGMENT->callbacks_on_abort)); + assert(tree_is_cleared(STM_PSEGMENT->callbacks_on_commit_and_abort[0])); + assert(tree_is_cleared(STM_PSEGMENT->callbacks_on_commit_and_abort[1])); assert(STM_PSEGMENT->objects_pointing_to_nursery == NULL); assert(STM_PSEGMENT->large_overflow_objects == NULL); #ifndef NDEBUG @@ -256,6 +391,24 @@ check_nursery_at_transaction_start(); } +long stm_start_transaction(stm_thread_local_t *tl) +{ + s_mutex_lock(); +#ifdef STM_NO_AUTOMATIC_SETJMP + long repeat_count = 0; /* test/support.py */ +#else + long repeat_count = stm_rewind_jmp_setjmp(tl); +#endif + _stm_start_transaction(tl, false); + return repeat_count; +} + +void stm_start_inevitable_transaction(stm_thread_local_t *tl) +{ + s_mutex_lock(); + _stm_start_transaction(tl, true); +} + /************************************************************/ @@ -286,13 +439,17 @@ ({ if (was_read_remote(remote_base, item, remote_version)) { /* A write-read conflict! */ - write_read_contention_management(i, item); - - /* If we reach this point, we didn't abort, but maybe we - had to wait for the other thread to commit. If we - did, then we have to restart committing from our call - to synchronize_all_threads(). */ - return true; + dprintf(("write-read conflict on %p, our seg: %d, other: %ld\n", + item, STM_SEGMENT->segment_num, i)); + if (write_read_contention_management(i, item)) { + /* If we reach this point, we didn't abort, but we + had to wait for the other thread to commit. If we + did, then we have to restart committing from our call + to synchronize_all_threads(). */ + return true; + } + /* we aborted the other transaction without waiting, so + we can just continue */ } })); } @@ -309,16 +466,16 @@ */ assert(!_is_young(obj)); + char *segment_base = get_segment_base(source_segment_num); uintptr_t start = (uintptr_t)obj; uintptr_t first_page = start / 4096UL; + struct object_s *realobj = (struct object_s *) + REAL_ADDRESS(segment_base, obj); if (is_small_uniform(obj)) { abort();//XXX WRITE THE FAST CASE } else { - char *segment_base = get_segment_base(source_segment_num); - struct object_s *realobj = (struct object_s *) - REAL_ADDRESS(segment_base, obj); ssize_t obj_size = stmcb_size_rounded_up(realobj); assert(obj_size >= 16); uintptr_t end = start + obj_size; @@ -465,6 +622,237 @@ } while (j > 0); } +static void _page_wise_synchronize_object_now_default(object_t *obj) +{ + uintptr_t start = (uintptr_t)obj; + uintptr_t first_page = start / 4096UL; + + char *realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj); + ssize_t obj_size = stmcb_size_rounded_up((struct object_s *)realobj); + assert(obj_size >= 16); + uintptr_t end = start + obj_size; + uintptr_t last_page = (end - 1) / 4096UL; + long i, myself = STM_SEGMENT->segment_num; + + for (; first_page <= last_page; first_page++) { + + uintptr_t copy_size; + if (first_page == last_page) { + /* this is the final fragment */ + copy_size = end - start; + } + else { + /* this is a non-final fragment, going up to the + page's end */ + copy_size = 4096 - (start & 4095); + } + /* double-check that the result fits in one page */ + assert(copy_size > 0); + assert(copy_size + (start & 4095) <= 4096); + + /* First copy the object into the shared page, if needed */ + char *src = REAL_ADDRESS(STM_SEGMENT->segment_base, start); + char *dst = REAL_ADDRESS(stm_object_pages, start); + if (is_private_page(myself, first_page)) { + if (copy_size == 4096) + pagecopy(dst, src); + else + memcpy(dst, src, copy_size); + } + else { + assert(memcmp(dst, src, copy_size) == 0); /* same page */ + } + + for (i = 1; i <= NB_SEGMENTS; i++) { + if (i == myself) + continue; + + /* src = REAL_ADDRESS(stm_object_pages, start); */ + dst = REAL_ADDRESS(get_segment_base(i), start); + if (is_private_page(i, first_page)) { + /* The page is a private page. We need to diffuse this + fragment of object from the shared page to this private + page. */ + if (copy_size == 4096) + pagecopy(dst, src); + else + memcpy(dst, src, copy_size); + } + else { + assert(!memcmp(dst, src, copy_size)); /* same page */ + } + } + + start = (start + 4096) & ~4095; + } +} + +static inline bool _has_private_page_in_range( + long seg_num, uintptr_t start, uintptr_t size) +{ + uintptr_t first_page = start / 4096UL; + uintptr_t last_page = (start + size) / 4096UL; + for (; first_page <= last_page; first_page++) + if (is_private_page(seg_num, first_page)) + return true; + return false; +} + +static void _card_wise_synchronize_object_now_default(object_t *obj) +{ + assert(obj_should_use_cards(obj)); + assert(!(obj->stm_flags & GCFLAG_CARDS_SET)); + assert(!IS_OVERFLOW_OBJ(STM_PSEGMENT, obj)); + + uintptr_t offset_itemsize[2]; + struct object_s *realobj = (struct object_s *)REAL_ADDRESS(STM_SEGMENT->segment_base, obj); + size_t obj_size = stmcb_size_rounded_up(realobj); + assert(obj_size >= 32); + stmcb_get_card_base_itemsize(realobj, offset_itemsize); + size_t real_idx_count = (obj_size - offset_itemsize[0]) / offset_itemsize[1]; + + uintptr_t first_card_index = get_write_lock_idx((uintptr_t)obj); + uintptr_t card_index = 1; + uintptr_t last_card_index = get_index_to_card_index(real_idx_count - 1); /* max valid index */ + long i, myself = STM_SEGMENT->segment_num; + + /* simple heuristic to check if probably the whole object is + marked anyway so we should do page-wise synchronize */ + if (write_locks[first_card_index + 1] == CARD_MARKED_OLD + && write_locks[first_card_index + last_card_index] == CARD_MARKED_OLD + && write_locks[first_card_index + (last_card_index >> 1) + 1] == CARD_MARKED_OLD) { + + dprintf(("card_wise_sync assumes %p,size:%lu is fully marked\n", obj, obj_size)); + _reset_object_cards(get_priv_segment(STM_SEGMENT->segment_num), + obj, CARD_CLEAR, false); + _page_wise_synchronize_object_now(obj); + return; + } + + dprintf(("card_wise_sync syncs %p,size:%lu card-wise\n", obj, obj_size)); + + /* Combine multiple marked cards and do a memcpy for them. We don't + try yet to use page_copy() or otherwise take into account privatization + of pages (except _has_private_page_in_range) */ + bool all_cards_were_cleared = true; + + uintptr_t start_card_index = -1; + while (card_index <= last_card_index) { + uintptr_t card_lock_idx = first_card_index + card_index; + uint8_t card_value = write_locks[card_lock_idx]; + + if (card_value == CARD_MARKED_OLD) { + write_locks[card_lock_idx] = CARD_CLEAR; + + if (start_card_index == -1) { /* first marked card */ + start_card_index = card_index; + /* start = (uintptr_t)obj + stmcb_index_to_byte_offset( */ + /* realobj, get_card_index_to_index(card_index)); */ + if (all_cards_were_cleared) { + all_cards_were_cleared = false; + } + } + } + else { + OPT_ASSERT(card_value == CARD_CLEAR); + } + + if (start_card_index != -1 /* something to copy */ + && (card_value != CARD_MARKED_OLD /* found non-marked card */ + || card_index == last_card_index)) { /* this is the last card */ + /* do the copying: */ + uintptr_t start, copy_size; + uintptr_t next_card_offset; + uintptr_t start_card_offset; + uintptr_t next_card_index = card_index; + + if (card_value == CARD_MARKED_OLD) { + /* card_index is the last card of the object, but we need + to go one further to get the right offset */ + next_card_index++; + } + + start_card_offset = offset_itemsize[0] + + get_card_index_to_index(start_card_index) * offset_itemsize[1]; + + next_card_offset = offset_itemsize[0] + + get_card_index_to_index(next_card_index) * offset_itemsize[1]; + + if (next_card_offset > obj_size) + next_card_offset = obj_size; + + start = (uintptr_t)obj + start_card_offset; + copy_size = next_card_offset - start_card_offset; + OPT_ASSERT(copy_size > 0); + + /* dprintf(("copy %lu bytes\n", copy_size)); */ + + /* since we have marked cards, at least one page here must be private */ + assert(_has_private_page_in_range(myself, start, copy_size)); + + /* copy to shared segment: */ + char *src = REAL_ADDRESS(STM_SEGMENT->segment_base, start); + char *dst = REAL_ADDRESS(stm_object_pages, start); + memcpy(dst, src, copy_size); + + /* copy to other segments */ + for (i = 1; i <= NB_SEGMENTS; i++) { + if (i == myself) + continue; + if (!_has_private_page_in_range(i, start, copy_size)) + continue; + /* src = REAL_ADDRESS(stm_object_pages, start); */ + dst = REAL_ADDRESS(get_segment_base(i), start); + memcpy(dst, src, copy_size); + } + + start_card_index = -1; + } + + card_index++; + } + + if (all_cards_were_cleared) { + /* well, seems like we never called stm_write_card() on it, so actually + we need to fall back to synchronize the whole object */ + _page_wise_synchronize_object_now(obj); + return; + } + +#ifndef NDEBUG + char *src = REAL_ADDRESS(stm_object_pages, (uintptr_t)obj); + char *dst; + for (i = 1; i <= NB_SEGMENTS; i++) { + dst = REAL_ADDRESS(get_segment_base(i), (uintptr_t)obj); + assert(memcmp(dst, src, obj_size) == 0); + } +#endif +} + +static void synchronize_object_now_default(object_t *obj, bool ignore_cards) +{ + /* Copy around the version of 'obj' that lives in our own segment. + It is first copied into the shared pages, and then into other + segments' own private pages. + + Must be called with the privatization lock acquired. + */ + assert(!_is_young(obj)); + assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); + assert(STM_PSEGMENT->privatization_lock == 1); + + if (obj->stm_flags & GCFLAG_SMALL_UNIFORM) { + assert(!(obj->stm_flags & GCFLAG_CARDS_SET)); + abort();//XXX WRITE THE FAST CASE + } else if (ignore_cards || !obj_should_use_cards(obj)) { + _page_wise_synchronize_object_now(obj); + } else { + _card_wise_synchronize_object_now(obj); + } + + _cards_cleared_in_object(get_priv_segment(STM_SEGMENT->segment_num), obj); +} + static void push_overflow_objects_from_privatized_pages(void) { if (STM_PSEGMENT->large_overflow_objects == NULL) @@ -472,7 +860,7 @@ acquire_privatization_lock(); LIST_FOREACH_R(STM_PSEGMENT->large_overflow_objects, object_t *, - synchronize_object_enqueue(item)); + synchronize_object_enqueue(item, true /*ignore_cards*/)); synchronize_objects_flush(); release_privatization_lock(); } @@ -497,7 +885,7 @@ /* copy the object to the shared page, and to the other private pages as needed */ - synchronize_object_enqueue(item); + synchronize_object_enqueue(item, false); /* don't ignore_cards */ })); release_privatization_lock(); @@ -511,8 +899,13 @@ STM_PSEGMENT->safe_point = SP_NO_TRANSACTION; STM_PSEGMENT->transaction_state = TS_NONE; + /* marker_inev is not needed anymore */ + STM_PSEGMENT->marker_inev[1] = 0; + /* reset these lists to NULL for the next transaction */ + _verify_cards_cleared_in_all_lists(get_priv_segment(STM_SEGMENT->segment_num)); LIST_FREE(STM_PSEGMENT->objects_pointing_to_nursery); + list_clear(STM_PSEGMENT->old_objects_with_cards); LIST_FREE(STM_PSEGMENT->large_overflow_objects); timing_end_transaction(attribute_to); @@ -533,6 +926,9 @@ /* the call to minor_collection() above leaves us with STM_TIME_BOOKKEEPING */ + /* synchronize overflow objects living in privatized pages */ + push_overflow_objects_from_privatized_pages(); + s_mutex_lock(); restart: @@ -549,31 +945,29 @@ dprintf(("commit_transaction\n")); assert(STM_SEGMENT->nursery_end == NURSERY_END); - STM_SEGMENT->jmpbuf_ptr = NULL; + stm_rewind_jmp_forget(STM_SEGMENT->running_thread); /* if a major collection is required, do it here */ - if (is_major_collection_requested()) + if (is_major_collection_requested()) { + int oldstate = change_timing_state(STM_TIME_MAJOR_GC); major_collection_now_at_safe_point(); - - /* synchronize overflow objects living in privatized pages */ - push_overflow_objects_from_privatized_pages(); + change_timing_state(oldstate); + } /* synchronize modified old objects to other threads */ push_modified_to_other_segments(); + _verify_cards_cleared_in_all_lists(get_priv_segment(STM_SEGMENT->segment_num)); /* update 'overflow_number' if needed */ if (STM_PSEGMENT->overflow_number_has_been_used) { highest_overflow_number += GCFLAG_OVERFLOW_NUMBER_bit0; - /* Note that the overflow number cannot be entirely 1 bits; - this prevents stm_flags from ever containing the value -1, - which might be confused with GCWORD_MOVED. */ assert(highest_overflow_number != /* XXX else, overflow! */ (uint32_t)-GCFLAG_OVERFLOW_NUMBER_bit0); STM_PSEGMENT->overflow_number = highest_overflow_number; STM_PSEGMENT->overflow_number_has_been_used = false; } - clear_callbacks_on_abort(); + invoke_and_clear_user_callbacks(0); /* for commit */ /* send what is hopefully the correct signals */ if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) { @@ -622,11 +1016,14 @@ ssize_t size = stmcb_size_rounded_up((struct object_s *)src); memcpy(dst, src, size); + if (obj_should_use_cards(item)) + _reset_object_cards(pseg, item, CARD_CLEAR, false); + /* objects in 'modified_old_objects' usually have the WRITE_BARRIER flag, unless they have been modified recently. Ignore the old flag; after copying from the other segment, we should have the flag. */ - assert(item->stm_flags & GCFLAG_WRITE_BARRIER); + assert(((struct object_s *)dst)->stm_flags & GCFLAG_WRITE_BARRIER); /* write all changes to the object before we release the write lock below. This is needed because we need to @@ -650,6 +1047,10 @@ static void abort_data_structures_from_segment_num(int segment_num) { +#pragma push_macro("STM_PSEGMENT") +#pragma push_macro("STM_SEGMENT") +#undef STM_PSEGMENT +#undef STM_SEGMENT /* This function clears the content of the given segment undergoing an abort. It is called from abort_with_mutex(), but also sometimes from other threads that figure out that this segment should abort. @@ -677,25 +1078,74 @@ /* throw away the content of the nursery */ long bytes_in_nursery = throw_away_nursery(pseg); + /* modified_old_objects' cards get cleared in + reset_modified_from_other_segments. Objs in old_objs_with_cards but not + in modified_old_objs are overflow objects and handled here: */ + if (pseg->large_overflow_objects != NULL) { + /* some overflow objects may have cards when aborting, clear them too */ + LIST_FOREACH_R(pseg->large_overflow_objects, object_t * /*item*/, + { + struct object_s *realobj = (struct object_s *) + REAL_ADDRESS(pseg->pub.segment_base, item); + + if (realobj->stm_flags & GCFLAG_CARDS_SET) { + /* CARDS_SET is enough since other HAS_CARDS objs + are already cleared */ + _reset_object_cards(pseg, item, CARD_CLEAR, false); + } + }); + } + /* reset all the modified objects (incl. re-adding GCFLAG_WRITE_BARRIER) */ reset_modified_from_other_segments(segment_num); + _verify_cards_cleared_in_all_lists(pseg); - /* reset the tl->shadowstack and thread_local_obj to their original - value before the transaction start */ + /* reset tl->shadowstack and thread_local_obj to their original + value before the transaction start. Also restore the content + of the shadowstack here. */ stm_thread_local_t *tl = pseg->pub.running_thread; +#ifdef STM_NO_AUTOMATIC_SETJMP + /* In tests, we don't save and restore the shadowstack correctly. + Be sure to not change items below shadowstack_at_start_of_transaction. + There is no such restrictions in non-Python-based tests. */ assert(tl->shadowstack >= pseg->shadowstack_at_start_of_transaction); - pseg->shadowstack_at_abort = tl->shadowstack; tl->shadowstack = pseg->shadowstack_at_start_of_transaction; +#else + /* NB. careful, this function might be called more than once to + abort a given segment. Make sure that + stm_rewind_jmp_restore_shadowstack() is idempotent. */ + /* we need to do this here and not directly in rewind_longjmp() because + that is called when we already released everything (safe point) + and a concurrent major GC could mess things up. */ + if (tl->shadowstack != NULL) + stm_rewind_jmp_restore_shadowstack(tl); + assert(tl->shadowstack == pseg->shadowstack_at_start_of_transaction); +#endif tl->thread_local_obj = pseg->threadlocal_at_start_of_transaction; tl->last_abort__bytes_in_nursery = bytes_in_nursery; /* reset these lists to NULL too on abort */ LIST_FREE(pseg->objects_pointing_to_nursery); + list_clear(pseg->old_objects_with_cards); LIST_FREE(pseg->large_overflow_objects); list_clear(pseg->young_weakrefs); +#pragma pop_macro("STM_SEGMENT") +#pragma pop_macro("STM_PSEGMENT") } -static void abort_with_mutex(void) +#ifdef STM_NO_AUTOMATIC_SETJMP +void _test_run_abort(stm_thread_local_t *tl) __attribute__((noreturn)); +int stm_is_inevitable(void) +{ + switch (STM_PSEGMENT->transaction_state) { + case TS_REGULAR: return 0; + case TS_INEVITABLE: return 1; + default: abort(); + } +} +#endif + +static stm_thread_local_t *abort_with_mutex_no_longjmp(void) { assert(_has_mutex()); dprintf(("~~~ ABORT\n")); @@ -704,15 +1154,14 @@ abort_data_structures_from_segment_num(STM_SEGMENT->segment_num); - stm_jmpbuf_t *jmpbuf_ptr = STM_SEGMENT->jmpbuf_ptr; + stm_thread_local_t *tl = STM_SEGMENT->running_thread; /* clear memory registered on the thread-local */ - stm_thread_local_t *tl = STM_SEGMENT->running_thread; if (tl->mem_clear_on_abort) memset(tl->mem_clear_on_abort, 0, tl->mem_bytes_to_clear_on_abort); /* invoke the callbacks */ - invoke_and_clear_callbacks_on_abort(); + invoke_and_clear_user_callbacks(1); /* for abort */ int attribute_to = STM_TIME_RUN_ABORTED_OTHER; @@ -729,6 +1178,12 @@ /* Broadcast C_ABORTED to wake up contention.c */ cond_broadcast(C_ABORTED); + return tl; +} + +static void abort_with_mutex(void) +{ + stm_thread_local_t *tl = abort_with_mutex_no_longjmp(); s_mutex_unlock(); /* It seems to be a good idea, at least in some examples, to sleep @@ -743,9 +1198,12 @@ */ usleep(1); - assert(jmpbuf_ptr != NULL); - assert(jmpbuf_ptr != (stm_jmpbuf_t *)-1); /* for tests only */ - __builtin_longjmp(*jmpbuf_ptr, 1); +#ifdef STM_NO_AUTOMATIC_SETJMP + _test_run_abort(tl); +#else + s_mutex_lock(); + stm_rewind_jmp_longjmp(tl); +#endif } void _stm_become_inevitable(const char *msg) @@ -759,12 +1217,11 @@ marker_fetch_inev(); wait_for_end_of_inevitable_transaction(NULL); STM_PSEGMENT->transaction_state = TS_INEVITABLE; - STM_SEGMENT->jmpbuf_ptr = NULL; - clear_callbacks_on_abort(); + stm_rewind_jmp_forget(STM_SEGMENT->running_thread); + invoke_and_clear_user_callbacks(0); /* for commit */ } else { assert(STM_PSEGMENT->transaction_state == TS_INEVITABLE); - assert(STM_SEGMENT->jmpbuf_ptr == NULL); } s_mutex_unlock(); diff --git a/c7/stm/core.h b/c7/stm/core.h --- a/c7/stm/core.h +++ b/c7/stm/core.h @@ -14,7 +14,7 @@ #endif -#define NB_PAGES (1500*256) // 1500MB +#define NB_PAGES (2500*256) // 2500MB #define NB_SEGMENTS STM_NB_SEGMENTS #define NB_SEGMENTS_MAX 240 /* don't increase NB_SEGMENTS past this */ #define MAP_PAGES_FLAGS (MAP_SHARED | MAP_ANONYMOUS | MAP_NORESERVE) @@ -35,6 +35,8 @@ #define WRITELOCK_START ((END_NURSERY_PAGE * 4096UL) >> 4) #define WRITELOCK_END READMARKER_END +#define CARD_SIZE _STM_CARD_SIZE + enum /* stm_flags */ { /* This flag is set on non-nursery objects. It forces stm_write() to call _stm_write_slowpath(). @@ -47,7 +49,13 @@ The same flag is abused to mark prebuilt objects whose hash has been taken during translation and is statically recorded just after the object. */ - GCFLAG_HAS_SHADOW = 0x2, + GCFLAG_HAS_SHADOW = 0x02, + + /* Set on objects that are large enough (_STM_MIN_CARD_OBJ_SIZE) + to have multiple cards (at least _STM_MIN_CARD_COUNT), and that + have at least one card marked. This flag implies + GCFLAG_WRITE_BARRIER. */ + GCFLAG_CARDS_SET = _STM_GCFLAG_CARDS_SET, /* All remaining bits of the 32-bit 'stm_flags' field are taken by the "overflow number". This is a number that identifies the @@ -56,7 +64,7 @@ current transaction that have been flushed out of the nursery, which occurs if the same transaction allocates too many objects. */ - GCFLAG_OVERFLOW_NUMBER_bit0 = 0x4 /* must be last */ + GCFLAG_OVERFLOW_NUMBER_bit0 = 0x8 /* must be last */ }; #define SYNC_QUEUE_SIZE 31 @@ -75,9 +83,7 @@ /* List of old objects (older than the current transaction) that the current transaction attempts to modify. This is used to track the STM status: they are old objects that where written to and - that need to be copied to other segments upon commit. Note that - every object takes three list items: the object, and two words for - the location marker. */ + that need to be copied to other segments upon commit. */ struct list_s *modified_old_objects; /* For each entry in 'modified_old_objects', we have two entries @@ -95,6 +101,10 @@ understood as meaning implicitly "this is the same as 'modified_old_objects'". */ struct list_s *objects_pointing_to_nursery; + /* Like objects_pointing_to_nursery it holds the old objects that + we did a stm_write_card() on. Objects can be in both lists. + It is NULL iff objects_pointing_to_nursery is NULL. */ + struct list_s *old_objects_with_cards; /* List of all large, overflowed objects. Only non-NULL after the current transaction spanned a minor collection. */ From noreply at buildbot.pypy.org Mon Sep 1 15:20:11 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Mon, 1 Sep 2014 15:20:11 +0200 (CEST) Subject: [pypy-commit] stmgc gc-small-uniform: more merging Message-ID: <20140901132011.CD6AE1C328B@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: gc-small-uniform Changeset: r1328:ceac696718e6 Date: 2014-09-01 14:51 +0200 http://bitbucket.org/pypy/stmgc/changeset/ceac696718e6/ Log: more merging diff --git a/c7/stm/core.c b/c7/stm/core.c --- a/c7/stm/core.c +++ b/c7/stm/core.c @@ -277,7 +277,7 @@ */ assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); assert(obj->stm_flags & GCFLAG_CARDS_SET); - assert(!(obj->stm_flags & GCFLAG_SMALL_UNIFORM)); /* not supported/tested */ + assert(!is_small_uniform(obj)); /* not supported/tested */ #ifndef NDEBUG struct object_s *realobj = (struct object_s *) @@ -513,6 +513,18 @@ } } +static inline bool _has_private_page_in_range( + long seg_num, uintptr_t start, uintptr_t size) +{ + uintptr_t first_page = start / 4096UL; + uintptr_t last_page = (start + size) / 4096UL; + for (; first_page <= last_page; first_page++) + if (is_private_page(seg_num, first_page)) + return true; + return false; +} + + static inline void _synchronize_fragment(stm_char *frag, ssize_t frag_size) { /* First copy the object into the shared page, if needed */ @@ -533,30 +545,8 @@ ++STM_PSEGMENT->sq_len; } -static void synchronize_object_enqueue(object_t *obj) +static void _page_wise_synchronize_object_now(object_t *obj, ssize_t obj_size) { - /* Copy around the version of 'obj' that lives in our own segment. - It is first copied into the shared pages, and then into other - segments' own private pages. (The second part might be done - later; call synchronize_objects_flush() to flush this queue.) - - Must be called with the privatization lock acquired. - */ - assert(!_is_young(obj)); - assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); - ssize_t obj_size = stmcb_size_rounded_up( - (struct object_s *)REAL_ADDRESS(STM_SEGMENT->segment_base, obj)); - OPT_ASSERT(obj_size >= 16); - assert(STM_PSEGMENT->privatization_lock == 1); - - if (LIKELY(is_small_uniform(obj))) { - _synchronize_fragment((stm_char *)obj, obj_size); - return; - } - - /* else, a more complicated case for large objects, to copy - around data only within the needed pages - */ uintptr_t start = (uintptr_t)obj; uintptr_t end = start + obj_size; @@ -574,140 +564,18 @@ _synchronize_fragment((stm_char *)start, copy_size); start = copy_up_to; - } while (start != end); } -static void synchronize_objects_flush(void) +static void _card_wise_synchronize_object_now(object_t *obj, ssize_t obj_size) { - - /* Do a full memory barrier. We must make sure that other - CPUs see the changes we did to the shared page ("S", in - synchronize_object_enqueue()) before we check the other segments - with is_private_page() (below). Otherwise, we risk the - following: this CPU writes "S" but the writes are not visible yet; - then it checks is_private_page() and gets false, and does nothing - more; just afterwards another CPU sets its own private_page bit - and copies the page; but it risks doing so before seeing the "S" - writes. - */ - long j = STM_PSEGMENT->sq_len; - if (j == 0) - return; - STM_PSEGMENT->sq_len = 0; - - __sync_synchronize(); - - long i, myself = STM_SEGMENT->segment_num; - do { - --j; - stm_char *frag = STM_PSEGMENT->sq_fragments[j]; - uintptr_t page = ((uintptr_t)frag) / 4096UL; - if (!any_other_private_page(myself, page)) - continue; - - ssize_t frag_size = STM_PSEGMENT->sq_fragsizes[j]; - - for (i = 1; i <= NB_SEGMENTS; i++) { - if (i == myself) - continue; - - char *src = REAL_ADDRESS(stm_object_pages, frag); - char *dst = REAL_ADDRESS(get_segment_base(i), frag); - if (is_private_page(i, page)) - memcpy(dst, src, frag_size); - else - EVENTUALLY(memcmp(dst, src, frag_size) == 0); /* same page */ - } - } while (j > 0); -} - -static void _page_wise_synchronize_object_now_default(object_t *obj) -{ - uintptr_t start = (uintptr_t)obj; - uintptr_t first_page = start / 4096UL; - - char *realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj); - ssize_t obj_size = stmcb_size_rounded_up((struct object_s *)realobj); - assert(obj_size >= 16); - uintptr_t end = start + obj_size; - uintptr_t last_page = (end - 1) / 4096UL; - long i, myself = STM_SEGMENT->segment_num; - - for (; first_page <= last_page; first_page++) { - - uintptr_t copy_size; - if (first_page == last_page) { - /* this is the final fragment */ - copy_size = end - start; - } - else { - /* this is a non-final fragment, going up to the - page's end */ - copy_size = 4096 - (start & 4095); - } - /* double-check that the result fits in one page */ - assert(copy_size > 0); - assert(copy_size + (start & 4095) <= 4096); - - /* First copy the object into the shared page, if needed */ - char *src = REAL_ADDRESS(STM_SEGMENT->segment_base, start); - char *dst = REAL_ADDRESS(stm_object_pages, start); - if (is_private_page(myself, first_page)) { - if (copy_size == 4096) - pagecopy(dst, src); - else - memcpy(dst, src, copy_size); - } - else { - assert(memcmp(dst, src, copy_size) == 0); /* same page */ - } - - for (i = 1; i <= NB_SEGMENTS; i++) { - if (i == myself) - continue; - - /* src = REAL_ADDRESS(stm_object_pages, start); */ - dst = REAL_ADDRESS(get_segment_base(i), start); - if (is_private_page(i, first_page)) { - /* The page is a private page. We need to diffuse this - fragment of object from the shared page to this private - page. */ - if (copy_size == 4096) - pagecopy(dst, src); - else - memcpy(dst, src, copy_size); - } - else { - assert(!memcmp(dst, src, copy_size)); /* same page */ - } - } - - start = (start + 4096) & ~4095; - } -} - -static inline bool _has_private_page_in_range( - long seg_num, uintptr_t start, uintptr_t size) -{ - uintptr_t first_page = start / 4096UL; - uintptr_t last_page = (start + size) / 4096UL; - for (; first_page <= last_page; first_page++) - if (is_private_page(seg_num, first_page)) - return true; - return false; -} - -static void _card_wise_synchronize_object_now_default(object_t *obj) -{ + assert(obj_size >= 32); assert(obj_should_use_cards(obj)); assert(!(obj->stm_flags & GCFLAG_CARDS_SET)); assert(!IS_OVERFLOW_OBJ(STM_PSEGMENT, obj)); uintptr_t offset_itemsize[2]; struct object_s *realobj = (struct object_s *)REAL_ADDRESS(STM_SEGMENT->segment_base, obj); - size_t obj_size = stmcb_size_rounded_up(realobj); - assert(obj_size >= 32); stmcb_get_card_base_itemsize(realobj, offset_itemsize); size_t real_idx_count = (obj_size - offset_itemsize[0]) / offset_itemsize[1]; @@ -725,7 +593,7 @@ dprintf(("card_wise_sync assumes %p,size:%lu is fully marked\n", obj, obj_size)); _reset_object_cards(get_priv_segment(STM_SEGMENT->segment_num), obj, CARD_CLEAR, false); - _page_wise_synchronize_object_now(obj); + _page_wise_synchronize_object_now(obj, obj_size); return; } @@ -790,21 +658,8 @@ /* since we have marked cards, at least one page here must be private */ assert(_has_private_page_in_range(myself, start, copy_size)); - /* copy to shared segment: */ - char *src = REAL_ADDRESS(STM_SEGMENT->segment_base, start); - char *dst = REAL_ADDRESS(stm_object_pages, start); - memcpy(dst, src, copy_size); - - /* copy to other segments */ - for (i = 1; i <= NB_SEGMENTS; i++) { - if (i == myself) - continue; - if (!_has_private_page_in_range(i, start, copy_size)) - continue; - /* src = REAL_ADDRESS(stm_object_pages, start); */ - dst = REAL_ADDRESS(get_segment_base(i), start); - memcpy(dst, src, copy_size); - } + /* push to seg0 and enqueue for synchronization */ + _synchronize_fragment((stm_char *)start, copy_size); start_card_index = -1; } @@ -815,7 +670,7 @@ if (all_cards_were_cleared) { /* well, seems like we never called stm_write_card() on it, so actually we need to fall back to synchronize the whole object */ - _page_wise_synchronize_object_now(obj); + _page_wise_synchronize_object_now(obj, obj_size); return; } @@ -829,30 +684,91 @@ #endif } -static void synchronize_object_now_default(object_t *obj, bool ignore_cards) + + + +static void synchronize_object_enqueue(object_t *obj, bool ignore_cards) { /* Copy around the version of 'obj' that lives in our own segment. It is first copied into the shared pages, and then into other - segments' own private pages. + segments' own private pages. (The second part might be done + later; call synchronize_objects_flush() to flush this queue.) Must be called with the privatization lock acquired. */ + char *realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj); + ssize_t obj_size = stmcb_size_rounded_up((struct object_s *)realobj); + assert(obj_size >= 16); assert(!_is_young(obj)); assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); assert(STM_PSEGMENT->privatization_lock == 1); - if (obj->stm_flags & GCFLAG_SMALL_UNIFORM) { + if (LIKELY(is_small_uniform(obj))) { assert(!(obj->stm_flags & GCFLAG_CARDS_SET)); - abort();//XXX WRITE THE FAST CASE + _synchronize_fragment((stm_char *)obj, obj_size); + return; } else if (ignore_cards || !obj_should_use_cards(obj)) { - _page_wise_synchronize_object_now(obj); + /* else, a more complicated case for large objects, to copy + around data only within the needed pages */ + _page_wise_synchronize_object_now(obj, obj_size); } else { - _card_wise_synchronize_object_now(obj); + /* ... or even only cards that need to be updated */ + _card_wise_synchronize_object_now(obj, obj_size); } _cards_cleared_in_object(get_priv_segment(STM_SEGMENT->segment_num), obj); } +static void synchronize_objects_flush(void) +{ + + /* Do a full memory barrier. We must make sure that other + CPUs see the changes we did to the shared page ("S", in + synchronize_object_enqueue()) before we check the other segments + with is_private_page() (below). Otherwise, we risk the + following: this CPU writes "S" but the writes are not visible yet; + then it checks is_private_page() and gets false, and does nothing + more; just afterwards another CPU sets its own private_page bit + and copies the page; but it risks doing so before seeing the "S" + writes. + */ + long j = STM_PSEGMENT->sq_len; + if (j == 0) + return; + STM_PSEGMENT->sq_len = 0; + + __sync_synchronize(); + + long i, myself = STM_SEGMENT->segment_num; + do { + --j; + stm_char *frag = STM_PSEGMENT->sq_fragments[j]; + uintptr_t page = ((uintptr_t)frag) / 4096UL; + if (!any_other_private_page(myself, page)) + continue; + + ssize_t frag_size = STM_PSEGMENT->sq_fragsizes[j]; + char *src = REAL_ADDRESS(stm_object_pages, frag); + + for (i = 1; i <= NB_SEGMENTS; i++) { + if (i == myself) + continue; + + char *dst = REAL_ADDRESS(get_segment_base(i), frag); + if (is_private_page(i, page)) { + if (frag_size == 4096) + pagecopy(dst, src); + else + memcpy(dst, src, frag_size); + } else { + EVENTUALLY(memcmp(dst, src, frag_size) == 0); /* same page */ + } + } + } while (j > 0); +} + + + static void push_overflow_objects_from_privatized_pages(void) { if (STM_PSEGMENT->large_overflow_objects == NULL) diff --git a/c7/stm/core.h b/c7/stm/core.h --- a/c7/stm/core.h +++ b/c7/stm/core.h @@ -304,7 +304,7 @@ } static void copy_object_to_shared(object_t *obj, int source_segment_num); -static void synchronize_object_enqueue(object_t *obj); +static void synchronize_object_enqueue(object_t *obj, bool ignore_cards); static void synchronize_objects_flush(void); static inline void acquire_privatization_lock(void) diff --git a/c7/stm/nursery.c b/c7/stm/nursery.c --- a/c7/stm/nursery.c +++ b/c7/stm/nursery.c @@ -120,7 +120,6 @@ char *realnobj = REAL_ADDRESS(STM_SEGMENT->segment_base, nobj); memcpy(realnobj, realobj, size); - xxx:also for smallobjs? nobj_sync_now = ((uintptr_t)nobj) | FLAG_SYNC_LARGE; /* Done copying the object. */ @@ -377,7 +376,6 @@ { dprintf(("collect_oldrefs_to_nursery\n")); struct list_s *lst = STM_PSEGMENT->objects_pointing_to_nursery; - assert(STM_PSEGMENT->minor_collect_will_commit_now); while (!list_is_empty(lst)) { uintptr_t obj_sync_now = list_pop_item(lst); @@ -550,7 +548,6 @@ else { collect_cardrefs_to_nursery(); num_old = STM_PSEGMENT->modified_old_objects_markers_num_old; - abort(); // handle specially the objects_pointing_to_nursery already there } collect_roots_from_markers(num_old); From noreply at buildbot.pypy.org Mon Sep 1 15:20:12 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Mon, 1 Sep 2014 15:20:12 +0200 (CEST) Subject: [pypy-commit] stmgc gc-small-uniform: merging mostly complete Message-ID: <20140901132012.E19CC1C328B@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: gc-small-uniform Changeset: r1329:fe2595f7a3cc Date: 2014-09-01 15:21 +0200 http://bitbucket.org/pypy/stmgc/changeset/fe2595f7a3cc/ Log: merging mostly complete diff --git a/c7/stm/core.c b/c7/stm/core.c --- a/c7/stm/core.c +++ b/c7/stm/core.c @@ -459,7 +459,6 @@ static void copy_object_to_shared(object_t *obj, int source_segment_num) { - abort(); /* Only used by major GC. XXX There is a lot of code duplication with synchronize_object_now() but I don't completely see how to improve... @@ -471,13 +470,16 @@ uintptr_t first_page = start / 4096UL; struct object_s *realobj = (struct object_s *) REAL_ADDRESS(segment_base, obj); + ssize_t obj_size = stmcb_size_rounded_up(realobj); + assert(obj_size >= 16); + if (is_small_uniform(obj)) { - abort();//XXX WRITE THE FAST CASE + char *src = REAL_ADDRESS(segment_base, start); + char *dst = REAL_ADDRESS(stm_object_pages, start); + memcpy(dst, src, obj_size); } else { - ssize_t obj_size = stmcb_size_rounded_up(realobj); - assert(obj_size >= 16); uintptr_t end = start + obj_size; uintptr_t last_page = (end - 1) / 4096UL; @@ -532,10 +534,14 @@ char *src = REAL_ADDRESS(STM_SEGMENT->segment_base, frag); char *dst = REAL_ADDRESS(stm_object_pages, frag); - if (is_private_page(STM_SEGMENT->segment_num, page)) - memcpy(dst, src, frag_size); - else + if (is_private_page(STM_SEGMENT->segment_num, page)) { + if (frag_size == 4096) + pagecopy(dst, src); + else + memcpy(dst, src, frag_size); + } else { EVENTUALLY(memcmp(dst, src, frag_size) == 0); /* same page */ + } /* Then enqueue this object (or fragemnt of object) */ if (STM_PSEGMENT->sq_len == SYNC_QUEUE_SIZE) @@ -582,7 +588,7 @@ uintptr_t first_card_index = get_write_lock_idx((uintptr_t)obj); uintptr_t card_index = 1; uintptr_t last_card_index = get_index_to_card_index(real_idx_count - 1); /* max valid index */ - long i, myself = STM_SEGMENT->segment_num; + long myself = STM_SEGMENT->segment_num; /* simple heuristic to check if probably the whole object is marked anyway so we should do page-wise synchronize */ @@ -674,14 +680,6 @@ return; } -#ifndef NDEBUG - char *src = REAL_ADDRESS(stm_object_pages, (uintptr_t)obj); - char *dst; - for (i = 1; i <= NB_SEGMENTS; i++) { - dst = REAL_ADDRESS(get_segment_base(i), (uintptr_t)obj); - assert(memcmp(dst, src, obj_size) == 0); - } -#endif } From noreply at buildbot.pypy.org Mon Sep 1 17:06:55 2014 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 1 Sep 2014 17:06:55 +0200 (CEST) Subject: [pypy-commit] pypy arm-longlong: Close branch, ready for merge. Message-ID: <20140901150655.D14951C072F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: arm-longlong Changeset: r73270:e49b397a6028 Date: 2014-09-01 17:02 +0200 http://bitbucket.org/pypy/pypy/changeset/e49b397a6028/ Log: Close branch, ready for merge. Can't test on soft-float ARM for now. We'll see later. From noreply at buildbot.pypy.org Mon Sep 1 17:06:57 2014 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 1 Sep 2014 17:06:57 +0200 (CEST) Subject: [pypy-commit] pypy default: hg merge arm-longlong Message-ID: <20140901150657.5244A1C072F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73271:8ae0ea93f7fe Date: 2014-09-01 17:05 +0200 http://bitbucket.org/pypy/pypy/changeset/8ae0ea93f7fe/ Log: hg merge arm-longlong Reintroduce longlong support in the ARM backend. Tested with the hard-float variant of the calling convention; can't test the soft- float variant for now. I'll test myself at a later date or just handle bug reports --- if any: I think that for soft-float, we simply need to consider longlongs as equivalent to 64-bit floats (as we do now). diff --git a/rpython/jit/backend/arm/callbuilder.py b/rpython/jit/backend/arm/callbuilder.py --- a/rpython/jit/backend/arm/callbuilder.py +++ b/rpython/jit/backend/arm/callbuilder.py @@ -80,15 +80,6 @@ self.mc.gen_load_int(r.ip.value, n) self.mc.SUB_rr(r.sp.value, r.sp.value, r.ip.value) - def _must_remap_fnloc(self): - fnloc = self.fnloc - if fnloc.is_stack(): - return True - if self.is_call_release_gil: - if fnloc is r.r5 or fnloc is r.r6 or fnloc is r.r7: - return True - return False - def call_releasegil_addr_and_move_real_arguments(self, fastgil): assert self.is_call_release_gil assert not self.asm._is_asmgcc() @@ -121,7 +112,7 @@ self.mc.STREX(r.r3.value, r.ip.value, r.r6.value, c=c.EQ) # try to claim the lock self.mc.CMP_ri(r.r3.value, 0, cond=c.EQ) # did this succeed? - self.mc.DMB(c=c.EQ) + self.mc.DMB() # the success of the lock acquisition is defined by # 'EQ is true', or equivalently by 'r3 == 0'. # @@ -268,7 +259,7 @@ # or on the stack, which we can not access later # If this happens to be the case we remap the register to r4 and use r4 # to call the function - if self.fnloc in r.argument_regs or self._must_remap_fnloc(): + if not self.fnloc.is_imm(): non_float_locs.append(self.fnloc) non_float_regs.append(r.r4) self.fnloc = r.r4 @@ -285,29 +276,23 @@ def get_next_vfp(self, tp): assert tp in 'fS' - if self.next_arg_vfp == -1: - return None - if tp == 'S': + if tp == 'f': + # 64bit double + i = max(self.next_arg_vfp, (self.next_arg_svfp + 1) >> 1) + if i >= len(r.vfp_argument_regs): + self.next_arg_svfp = 1000 # stop that sequence too + return None + self.next_arg_vfp = i + 1 + return r.vfp_argument_regs[i] + else: + # 32bit float i = self.next_arg_svfp - next_vfp = (i >> 1) + 1 - if not (i + 1) & 1: # i is even - self.next_arg_vfp = max(self.next_arg_vfp, next_vfp) - self.next_arg_svfp = self.next_arg_vfp << 1 - else: - self.next_arg_svfp += 1 - self.next_arg_vfp = next_vfp - lst = r.svfp_argument_regs - else: # 64bit double - i = self.next_arg_vfp - self.next_arg_vfp += 1 - if self.next_arg_svfp >> 1 == i: - self.next_arg_svfp = self.next_arg_vfp << 1 - lst = r.vfp_argument_regs - try: - return lst[i] - except IndexError: - self.next_arg_vfp = self.next_arg_svfp = -1 - return None + if not (i & 1): # if i is even + i = max(i, self.next_arg_vfp << 1) + if i >= len(r.svfp_argument_regs): + return None + self.next_arg_svfp = i + 1 + return r.svfp_argument_regs[i] def prepare_arguments(self): non_float_locs = [] @@ -316,34 +301,64 @@ float_regs = [] stack_args = [] singlefloats = None + longlong_mask = 0 arglocs = self.arglocs argtypes = self.argtypes - count = 0 # stack alignment counter + r_register_count = 0 on_stack = 0 + for i in range(len(arglocs)): argtype = INT if i < len(argtypes) and argtypes[i] == 'S': argtype = argtypes[i] arg = arglocs[i] + if arg.is_float(): - argtype = FLOAT - reg = self.get_next_vfp(argtype) - if reg: - assert len(float_regs) < len(r.vfp_argument_regs) - float_locs.append(arg) - assert reg not in float_regs - float_regs.append(reg) - else: # float argument that needs to go on the stack - if count % 2 != 0: - stack_args.append(None) - count = 0 - on_stack += 1 - stack_args.append(arg) - on_stack += 2 + if i < len(argtypes) and argtypes[i] == 'L': + # A longlong argument. It uses two regular argument + # positions, but aligned to an even number. This is + # a bit strange, but it is the case even for registers: + # it can be in r0-r1 or in r2-r3 but not in r1-r2. + assert arg.is_float() + if r_register_count == 0: + # will temporarily load the register into d8 + float_locs.append(arg) + float_regs.append(r.d8) + longlong_mask |= 1 + r_register_count = 2 + continue + elif r_register_count <= 2: + # will temporarily load the register into d9 + float_locs.append(arg) + float_regs.append(r.d9) + longlong_mask |= 2 + r_register_count = 4 + continue + elif r_register_count == 3: + r_register_count = 4 + else: + # A 64-bit float argument. Goes into the next free v# + # register, or if none, to the stack aligned to an + # even number of words. + argtype = FLOAT + reg = self.get_next_vfp(argtype) + if reg: + float_locs.append(arg) + assert reg not in float_regs + float_regs.append(reg) + continue + # float or longlong argument that needs to go on the stack + if on_stack & 1: # odd: realign + stack_args.append(None) + on_stack += 1 + stack_args.append(arg) + on_stack += 2 + elif argtype == 'S': - # Singlefloat argument + # Singlefloat (32-bit) argument. Goes into the next free + # v# register, or if none, to the stack in a single word. if singlefloats is None: singlefloats = [] tgt = self.get_next_vfp(argtype) @@ -351,32 +366,36 @@ singlefloats.append((arg, tgt)) else: # Singlefloat argument that needs to go on the stack # treated the same as a regular core register argument - count += 1 + stack_args.append(arg) on_stack += 1 - stack_args.append(arg) else: - if len(non_float_regs) < len(r.argument_regs): - reg = r.argument_regs[len(non_float_regs)] + # Regular one-word argument. Goes into the next register + # free from the list r0, r1, r2, r3, or to the stack. + if r_register_count < len(r.argument_regs): + reg = r.argument_regs[r_register_count] + r_register_count += 1 non_float_locs.append(arg) non_float_regs.append(reg) else: # non-float argument that needs to go on the stack - count += 1 + stack_args.append(arg) on_stack += 1 - stack_args.append(arg) + # align the stack - if count % 2 != 0: + if on_stack & 1: # odd: realign stack_args.append(None) on_stack += 1 self._push_stack_args(stack_args, on_stack*WORD) + # Check that the address of the function we want to call is not # currently stored in one of the registers used to pass the arguments # or on the stack, which we can not access later # If this happens to be the case we remap the register to r4 and use r4 # to call the function - if self.fnloc in non_float_regs or self._must_remap_fnloc(): + if not self.fnloc.is_imm(): non_float_locs.append(self.fnloc) non_float_regs.append(r.r4) self.fnloc = r.r4 + # remap values stored in vfp registers remap_frame_layout(self.asm, float_locs, float_regs, r.vfp_ip) if singlefloats: @@ -392,13 +411,22 @@ src = r.ip if src.is_core_reg(): self.mc.VMOV_cs(dest.value, src.value) + # remap values stored in core registers remap_frame_layout(self.asm, non_float_locs, non_float_regs, r.ip) + if longlong_mask & 1: + self.mc.FMRRD(r.r0.value, r.r1.value, r.d8.value) + if longlong_mask & 2: + self.mc.FMRRD(r.r2.value, r.r3.value, r.d9.value) + def load_result(self): resloc = self.resloc if self.restype == 'S': self.mc.VMOV_sc(resloc.value, r.s0.value) + elif self.restype == 'L': + assert resloc.is_vfp_reg() + self.mc.FMDRR(resloc.value, r.r0.value, r.r1.value) # ensure the result is wellformed and stored in the correct location if resloc is not None and resloc.is_core_reg(): self._ensure_result_bit_extension(resloc, @@ -408,7 +436,10 @@ if self.resloc is None: return [], [] if self.resloc.is_vfp_reg(): - return [], [r.d0] + if self.restype == 'L': # long long + return [r.r0, r.r1], [] + else: + return [], [r.d0] assert self.resloc.is_core_reg() return [r.r0], [] diff --git a/rpython/jit/backend/arm/codebuilder.py b/rpython/jit/backend/arm/codebuilder.py --- a/rpython/jit/backend/arm/codebuilder.py +++ b/rpython/jit/backend/arm/codebuilder.py @@ -332,13 +332,17 @@ | (rd & 0xF) << 12 | (rn & 0xF) << 16) - def DMB(self, c=cond.AL): - self.write32(c << 28 | 0x157ff05f) + def DMB(self): + # note: 'cond' is only permitted on Thumb here + self.write32(0xf57ff05f) DIV = binary_helper_call('int_div') MOD = binary_helper_call('int_mod') UDIV = binary_helper_call('uint_div') + FMDRR = VMOV_cr # uh, there are synonyms? + FMRRD = VMOV_rc + def _encode_reg_list(self, instr, regs): for reg in regs: instr |= 0x1 << reg diff --git a/rpython/jit/backend/arm/runner.py b/rpython/jit/backend/arm/runner.py --- a/rpython/jit/backend/arm/runner.py +++ b/rpython/jit/backend/arm/runner.py @@ -20,7 +20,7 @@ IS_64_BIT = False supports_floats = True - supports_longlong = False # incomplete, notably in callbuilder.py + supports_longlong = True supports_singlefloats = True from rpython.jit.backend.arm.arch import JITFRAME_FIXED_SIZE diff --git a/rpython/jit/backend/arm/test/test_callbuilder.py b/rpython/jit/backend/arm/test/test_callbuilder.py new file mode 100644 --- /dev/null +++ b/rpython/jit/backend/arm/test/test_callbuilder.py @@ -0,0 +1,47 @@ +from rpython.jit.backend.arm.callbuilder import HardFloatCallBuilder +from rpython.jit.backend.arm import registers as r + + + +def test_hf_vfp_registers_all_singlefloat(): + hf = HardFloatCallBuilder.__new__(HardFloatCallBuilder) + got = [hf.get_next_vfp('S') for i in range(18)] + assert got == [r.s0, r.s1, r.s2, r.s3, r.s4, r.s5, r.s6, r.s7, + r.s8, r.s9, r.s10, r.s11, r.s12, r.s13, r.s14, r.s15, + None, None] + +def test_hf_vfp_registers_all_doublefloat(): + hf = HardFloatCallBuilder.__new__(HardFloatCallBuilder) + got = [hf.get_next_vfp('f') for i in range(10)] + assert got == [r.d0, r.d1, r.d2, r.d3, r.d4, r.d5, r.d6, r.d7, + None, None] + +def test_hf_vfp_registers_mixture(): + hf = HardFloatCallBuilder.__new__(HardFloatCallBuilder) + got = [hf.get_next_vfp('S'), hf.get_next_vfp('f'), + hf.get_next_vfp('S'), hf.get_next_vfp('f'), + hf.get_next_vfp('S'), hf.get_next_vfp('f'), + hf.get_next_vfp('S'), hf.get_next_vfp('f'), + hf.get_next_vfp('S'), hf.get_next_vfp('f'), + hf.get_next_vfp('S'), hf.get_next_vfp('f'), + hf.get_next_vfp('S'), hf.get_next_vfp('f')] + assert got == [r.s0, r.d1, + r.s1, r.d2, + r.s6, r.d4, + r.s7, r.d5, + r.s12, r.d7, + r.s13, None, + None, None] + +def test_hf_vfp_registers_mixture_2(): + hf = HardFloatCallBuilder.__new__(HardFloatCallBuilder) + got = [hf.get_next_vfp('f'), hf.get_next_vfp('f'), + hf.get_next_vfp('f'), hf.get_next_vfp('f'), + hf.get_next_vfp('f'), hf.get_next_vfp('f'), + hf.get_next_vfp('f'), hf.get_next_vfp('S'), + hf.get_next_vfp('f'), hf.get_next_vfp('S')] + assert got == [r.d0, r.d1, + r.d2, r.d3, + r.d4, r.d5, + r.d6, r.s14, + None, None] # <- and not r.s15 for the last item diff --git a/rpython/jit/backend/arm/test/test_instr_codebuilder.py b/rpython/jit/backend/arm/test/test_instr_codebuilder.py --- a/rpython/jit/backend/arm/test/test_instr_codebuilder.py +++ b/rpython/jit/backend/arm/test/test_instr_codebuilder.py @@ -199,6 +199,14 @@ self.cb.DMB() self.assert_equal('DMB') + def test_fmdrr(self): + self.cb.FMDRR(r.d11.value, r.r9.value, r.r14.value) + self.assert_equal('FMDRR d11, r9, r14') + + def test_fmrrd(self): + self.cb.FMRRD(r.r9.value, r.r14.value, r.d11.value) + self.assert_equal('FMRRD r9, r14, d11') + def test_size_of_gen_load_int(): for v, n in [(5, 4), (6, 4), (7, 2)]: diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py --- a/rpython/jit/backend/test/runner_test.py +++ b/rpython/jit/backend/test/runner_test.py @@ -2718,12 +2718,11 @@ assert r == result def test_call_release_gil_variable_function_and_arguments(self): - # NOTE NOTE NOTE - # This also works as a test for ctypes and libffi. - # On some platforms, one of these is buggy... + from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.rlib.libffi import types from rpython.rlib.rarithmetic import r_uint, r_longlong, r_ulonglong from rpython.rlib.rarithmetic import r_singlefloat + from rpython.translator.c import primitive cpu = self.cpu rnd = random.Random(525) @@ -2752,25 +2751,76 @@ (types.float, rffi.FLOAT), ] * 4 - for k in range(100): + NB_TESTS = 100 + c_source = [] + all_tests = [] + export_symbols = [] + + def prepare_c_source(): + """Pick a random choice of argument types and length, + and build a C function with these arguments. The C + function will simply copy them all into static global + variables. There are then additional functions to fetch + them, one per argument, with a signature 'void(ARG *)'. + """ POSSIBLE_TYPES = [rnd.choice(ALL_TYPES) for i in range(random.randrange(2, 5))] load_factor = rnd.random() keepalive_factor = rnd.random() # - def pseudo_c_function(*args): - seen.append(list(args)) - # ffitypes = [] ARGTYPES = [] for i in range(rnd.randrange(4, 20)): ffitype, TP = rnd.choice(POSSIBLE_TYPES) ffitypes.append(ffitype) ARGTYPES.append(TP) + fn_name = 'vartest%d' % k + all_tests.append((ARGTYPES, ffitypes, fn_name)) # - FPTR = self.Ptr(self.FuncType(ARGTYPES, lltype.Void)) - func_ptr = llhelper(FPTR, pseudo_c_function) - funcbox = self.get_funcbox(cpu, func_ptr) + fn_args = [] + for i, ARG in enumerate(ARGTYPES): + arg_decl = primitive.cdecl(primitive.PrimitiveType[ARG], + 'x%d' % i) + fn_args.append(arg_decl) + var_name = 'argcopy_%s_x%d' % (fn_name, i) + var_decl = primitive.cdecl(primitive.PrimitiveType[ARG], + var_name) + c_source.append('static %s;' % var_decl) + getter_name = '%s_get%d' % (fn_name, i) + export_symbols.append(getter_name) + c_source.append('void %s(%s) { *p = %s; }' % ( + getter_name, + primitive.cdecl(primitive.PrimitiveType[ARG], '*p'), + var_name)) + export_symbols.append(fn_name) + c_source.append('') + c_source.append('static void real%s(%s)' % ( + fn_name, ', '.join(fn_args))) + c_source.append('{') + for i in range(len(ARGTYPES)): + c_source.append(' argcopy_%s_x%d = x%d;' % (fn_name, i, i)) + c_source.append('}') + c_source.append('void *%s(void)' % fn_name) + c_source.append('{') + c_source.append(' return (void *)&real%s;' % fn_name) + c_source.append('}') + c_source.append('') + + for k in range(NB_TESTS): + prepare_c_source() + + eci = ExternalCompilationInfo( + separate_module_sources=['\n'.join(c_source)], + export_symbols=export_symbols) + + for k in range(NB_TESTS): + ARGTYPES, ffitypes, fn_name = all_tests[k] + func_getter_ptr = rffi.llexternal(fn_name, [], lltype.Signed, + compilation_info=eci, _nowrapper=True) + load_factor = rnd.random() + keepalive_factor = rnd.random() + # + func_raw = func_getter_ptr() calldescr = cpu._calldescr_dynamic_for_tests(ffitypes, types.void) faildescr = BasicFailDescr(1) # @@ -2790,7 +2840,7 @@ print print codes # - argvalues = [funcbox.getint()] + argvalues = [func_raw] for TP in ARGTYPES: r = (rnd.random() - 0.5) * 999999999999.9 r = rffi.cast(TP, r) @@ -2840,16 +2890,26 @@ looptoken = JitCellToken() self.cpu.compile_loop(argboxes, ops, looptoken) # - seen = [] deadframe = self.cpu.execute_token(looptoken, *argvalues_normal) fail = self.cpu.get_latest_descr(deadframe) assert fail.identifier == 0 expected = argvalues[1:] - [got] = seen - different_values = ['%r != %r' % (a, b) - for a, b in zip(got, expected) - if a != b] - assert got == expected, ', '.join(different_values) + got = [] + for i, ARG in enumerate(ARGTYPES): + PARG = rffi.CArrayPtr(ARG) + getter_name = '%s_get%d' % (fn_name, i) + getter_ptr = rffi.llexternal(getter_name, [PARG], lltype.Void, + compilation_info=eci, + _nowrapper=True) + my_arg = lltype.malloc(PARG.TO, 1, zero=True, flavor='raw') + getter_ptr(my_arg) + got.append(my_arg[0]) + lltype.free(my_arg, flavor='raw') + different_values = ['x%d: got %r, expected %r' % (i, a, b) + for i, (a, b) in enumerate(zip(got, expected)) + if a != b] + assert got == expected, '\n'.join( + ['bad args, signature %r' % codes[1:]] + different_values) def test_guard_not_invalidated(self): From noreply at buildbot.pypy.org Mon Sep 1 17:20:00 2014 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 1 Sep 2014 17:20:00 +0200 (CEST) Subject: [pypy-commit] pypy default: add a remark about SoftFloatCallBuilder Message-ID: <20140901152000.E70FA1C072F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73272:f27cde19678a Date: 2014-09-01 17:19 +0200 http://bitbucket.org/pypy/pypy/changeset/f27cde19678a/ Log: add a remark about SoftFloatCallBuilder diff --git a/rpython/jit/backend/arm/callbuilder.py b/rpython/jit/backend/arm/callbuilder.py --- a/rpython/jit/backend/arm/callbuilder.py +++ b/rpython/jit/backend/arm/callbuilder.py @@ -173,6 +173,13 @@ class SoftFloatCallBuilder(ARMCallbuilder): + # XXX Maybe we could kill this class and unify the remaining two + # XXX classes, by carefully checking if all methods here are doing + # XXX the exact same thing as the methods from HardFloatCallBuilder, + # XXX but simply forcing all BoxFloat arguments to be longlongs + # XXX (i.e. ignoring 'f' in favour of 'L'), and the same with + # XXX single-float arguments (ignoring 'S' in favour of 'i'); + # XXX and the same for the return value. def get_result_locs(self): if self.resloc is None: From noreply at buildbot.pypy.org Mon Sep 1 19:24:23 2014 From: noreply at buildbot.pypy.org (rlamy) Date: Mon, 1 Sep 2014 19:24:23 +0200 (CEST) Subject: [pypy-commit] pypy default: Backout 44c3924 : no need to patch dead code Message-ID: <20140901172423.8F53B1D22E6@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: Changeset: r73273:1cea12b7ec07 Date: 2014-09-01 18:20 +0100 http://bitbucket.org/pypy/pypy/changeset/1cea12b7ec07/ Log: Backout 44c3924 : no need to patch dead code diff --git a/py/_code/source.py b/py/_code/source.py --- a/py/_code/source.py +++ b/py/_code/source.py @@ -416,8 +416,6 @@ trysource = source[start:end] if trysource.isparseable(): return start, end - if end == start + 100: # XXX otherwise, it takes forever - break # XXX raise SyntaxError("no valid source range around line %d " % (lineno,)) From noreply at buildbot.pypy.org Mon Sep 1 20:22:15 2014 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 1 Sep 2014 20:22:15 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: a simple hack to avoid writing "uninitialized" to places Message-ID: <20140901182215.BF4B71C025B@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73274:ab09d9e107f6 Date: 2014-09-01 12:21 -0600 http://bitbucket.org/pypy/pypy/changeset/ab09d9e107f6/ Log: a simple hack to avoid writing "uninitialized" to places diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py --- a/rpython/memory/gctransform/framework.py +++ b/rpython/memory/gctransform/framework.py @@ -1264,6 +1264,7 @@ def gen_zero_gc_pointers(TYPE, v, llops, previous_steps=None): + xxxx if previous_steps is None: previous_steps = [] assert (isinstance(TYPE, lltype.Struct) or isinstance(TYPE, lltype.Array)) diff --git a/rpython/rtyper/lltypesystem/llmemory.py b/rpython/rtyper/lltypesystem/llmemory.py --- a/rpython/rtyper/lltypesystem/llmemory.py +++ b/rpython/rtyper/lltypesystem/llmemory.py @@ -1037,7 +1037,8 @@ else: # this is a hack XXX de-hack this llvalue = source._obj.getitem(i, uninitialized_ok=True) - dest._obj.setitem(i, llvalue) + if not isinstance(llvalue, lltype._uninitialized): + dest._obj.setitem(i, llvalue) elif isinstance(T, lltype.Struct): for name in T._names: FIELDTYPE = getattr(T, name) From noreply at buildbot.pypy.org Mon Sep 1 20:53:12 2014 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 1 Sep 2014 20:53:12 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: Refactor some stuff so test_transformed_gc passes (although it's definitely not Message-ID: <20140901185312.D0A381D22E6@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73275:0cfec6969ec9 Date: 2014-09-01 12:52 -0600 http://bitbucket.org/pypy/pypy/changeset/0cfec6969ec9/ Log: Refactor some stuff so test_transformed_gc passes (although it's definitely not exhaustive as shown by missing cases) diff --git a/rpython/memory/gc/test/test_direct.py b/rpython/memory/gc/test/test_direct.py --- a/rpython/memory/gc/test/test_direct.py +++ b/rpython/memory/gc/test/test_direct.py @@ -725,21 +725,22 @@ assert arr_of_ptr_struct[i].next == lltype.nullptr(S) #fail for now - def test_malloc_array_of_ptr_arr(self): + def xxx_test_malloc_array_of_ptr_arr(self): ARR_OF_PTR_ARR = lltype.GcArray(lltype.Ptr(lltype.GcArray(lltype.Ptr(S)))) - arr_of_ptr_arr = lltype.malloc(ARR_OF_PTR_ARR, 10) + arr_of_ptr_arr = self.malloc(ARR_OF_PTR_ARR, 10) self.stackroots.append(arr_of_ptr_arr) for i in range(10): assert arr_of_ptr_arr[i] == lltype.nullptr(lltype.GcArray(lltype.Ptr(S))) for i in range(10): - arr_of_ptr_arr[i] = self.malloc(lltype.GcArray(lltype.Ptr(S)), i) - self.stackroots.append(arr_of_ptr_arr[i]) - debug_print(arr_of_ptr_arr[i]) + self.writearray(arr_of_ptr_arr, i, + self.malloc(lltype.GcArray(lltype.Ptr(S)), i)) + #self.stackroots.append(arr_of_ptr_arr[i]) + #debug_print(arr_of_ptr_arr[i]) for elem in arr_of_ptr_arr[i]: - self.stackroots.append(elem) + #self.stackroots.append(elem) assert elem == lltype.nullptr(S) elem = self.malloc(S) assert elem.prev == lltype.nullptr(S) assert elem.next == lltype.nullptr(S) - \ No newline at end of file + diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py --- a/rpython/memory/gctransform/framework.py +++ b/rpython/memory/gctransform/framework.py @@ -883,7 +883,7 @@ if not self.malloc_zero_filled: v_ob = hop.spaceop.args[0] TYPE = v_ob.concretetype.TO - gen_zero_gc_pointers(TYPE, v_ob, hop.llops) + self.gen_zero_gc_pointers(TYPE, v_ob, hop.llops) def gct_gc_writebarrier_before_copy(self, hop): op = hop.spaceop @@ -1204,6 +1204,74 @@ def pop_roots(self, hop, livevars): raise NotImplementedError + def gen_zero_gc_pointers(self, TYPE, v, llops, previous_steps=None): + if isinstance(TYPE, lltype.Struct): + for name in TYPE._names: + FIELD = getattr(TYPE, name) + if isinstance(FIELD, lltype.Ptr) and FIELD._needsgc(): + c_name = rmodel.inputconst(lltype.Void, name) + c_null = rmodel.inputconst(FIELD, lltype.nullptr(FIELD.TO)) + llops.genop('bare_setfield', [v, c_name, c_null]) + elif (isinstance(FIELD, lltype.Array) and + isinstance(FIELD.OF, lltype.Ptr) and FIELD.OF._needsgc()): + xxx + return + elif isinstance(TYPE, lltype.Array): + ITEM = TYPE.OF + if isinstance(ITEM, lltype.Ptr) and ITEM._needsgc(): + v_size = llops.genop('getarraysize', [v], + resulttype=lltype.Signed) + c_size = rmodel.inputconst(lltype.Signed, llmemory.sizeof(ITEM)) + v_totalsize = llops.genop('int_mul', [v_size, c_size], + resulttype=lltype.Signed) + v_a = llops.genop('cast_ptr_to_adr', [v], + resulttype=llmemory.Address) + c_fixedofs = rmodel.inputconst(lltype.Signed, + llmemory.itemoffsetof(TYPE)) + v_adr = llops.genop('adr_add', [v_a, c_fixedofs], + resulttype=llmemory.Address) + llops.genop('raw_memclear', [v_adr, v_totalsize]) + elif isinstance(TYPE, lltype.Struct): + xxx + return + else: + raise TypeError(TYPE) + xxxx + if previous_steps is None: + previous_steps = [] + assert (isinstance(TYPE, lltype.Struct) or isinstance(TYPE, lltype.Array)) + if isinstance(TYPE, lltype.Struct): + for name in TYPE._names: + c_name = rmodel.inputconst(lltype.Void, name) + FIELD = getattr(TYPE, name) + #handle ptr field in GcStruct + if isinstance(FIELD, lltype.Ptr) and FIELD._needsgc(): + c_null = rmodel.inputconst(FIELD, lltype.nullptr(FIELD.TO)) + if not previous_steps: + llops.genop('bare_setfield', [v, c_name, c_null]) + else: + llops.genop('bare_setinteriorfield', + [v] + previous_steps + [c_name, c_null]) + #handle inside GcStruct field + elif isinstance(FIELD, lltype.Struct): + gen_zero_gc_pointers(FIELD, v, llops, previous_steps + [c_name]) + #handle inside GcArray field + elif isinstance(FIELD, lltype.Array): + gen_zero_gc_pointers(FIELD, v, llops, previous_steps + [c_name]) + if isinstance(TYPE, lltype.Array): + ITEM = TYPE.OF + if previous_steps: + v = llop.genop('getinteriorfield',[v]+previous_steps) + arr_size = llops.genop('getarraysize',[v]) + for i in range(arr_size): + #handle an array of GcPtr + if isinstance(ITEM, lltype.Ptr) and ITEM._needsgc(): + c_null = rmodel.inputconst(ITEM, lltype.nullptr(ITEM.TO)) + llops.genop('bare_setarrayitem',[v, i, c_null]) + if isinstance(ITEM, lltype.Struct) or isinstance(ITEM, lltype.GcArray): + array_item = llops.genop('getarrayitem',[v,i]) + gen_zero_gc_pointers(FIELD, array_item, llops, previous_steps) + class TransformerLayoutBuilder(gctypelayout.TypeLayoutBuilder): @@ -1262,43 +1330,6 @@ [llmemory.Address, llmemory.Address], llmemory.Address) return fptr - -def gen_zero_gc_pointers(TYPE, v, llops, previous_steps=None): - xxxx - if previous_steps is None: - previous_steps = [] - assert (isinstance(TYPE, lltype.Struct) or isinstance(TYPE, lltype.Array)) - if isinstance(TYPE, lltype.Struct): - for name in TYPE._names: - c_name = rmodel.inputconst(lltype.Void, name) - FIELD = getattr(TYPE, name) - #handle ptr field in GcStruct - if isinstance(FIELD, lltype.Ptr) and FIELD._needsgc(): - c_null = rmodel.inputconst(FIELD, lltype.nullptr(FIELD.TO)) - if not previous_steps: - llops.genop('bare_setfield', [v, c_name, c_null]) - else: - llops.genop('bare_setinteriorfield', - [v] + previous_steps + [c_name, c_null]) - #handle inside GcStruct field - elif isinstance(FIELD, lltype.Struct): - gen_zero_gc_pointers(FIELD, v, llops, previous_steps + [c_name]) - #handle inside GcArray field - elif isinstance(FIELD, lltype.Array): - gen_zero_gc_pointers(FIELD, v, llops, previous_steps + [c_name]) - if isinstance(TYPE, lltype.Array): - ITEM = TYPE.OF - if previous_steps: - v = llop.genop('getinteriorfield',[v]+previous_steps) - arr_size = llops.genop('getarraysize',[v]) - for i in range(arr_size): - #handle an array of GcPtr - if isinstance(ITEM, lltype.Ptr) and ITEM._needsgc(): - c_null = rmodel.inputconst(ITEM, lltype.nullptr(ITEM.TO)) - llops.genop('bare_setarrayitem',[v, i, c_null]) - if isinstance(ITEM, lltype.Struct) or isinstance(ITEM, lltype.GcArray): - array_item = llops.genop('getarrayitem',[v,i]) - gen_zero_gc_pointers(FIELD, array_item, llops, previous_steps) # ____________________________________________________________ diff --git a/rpython/memory/test/test_transformed_gc.py b/rpython/memory/test/test_transformed_gc.py --- a/rpython/memory/test/test_transformed_gc.py +++ b/rpython/memory/test/test_transformed_gc.py @@ -1296,8 +1296,7 @@ def test_nursery_hash_base(self): res = self.runner('nursery_hash_base') - assert res >= 195 - assert False + assert res([]) >= 195 class TestIncrementalMiniMarkGC(TestMiniMarkGC): gcname = "incminimark" From noreply at buildbot.pypy.org Mon Sep 1 21:15:27 2014 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 1 Sep 2014 21:15:27 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: merge default Message-ID: <20140901191527.F39511C06D8@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73276:6dd6a6c171b2 Date: 2014-09-01 13:14 -0600 http://bitbucket.org/pypy/pypy/changeset/6dd6a6c171b2/ Log: merge default diff too long, truncating to 2000 out of 63292 lines diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -35,280 +35,290 @@ the beginning of each file) the files in the 'pypy' directory are each copyrighted by one or more of the following people and organizations: - Armin Rigo - Maciej Fijalkowski - Carl Friedrich Bolz - Antonio Cuni - Amaury Forgeot d'Arc - Samuele Pedroni - Alex Gaynor - Michael Hudson - David Schneider - Matti Picus - Brian Kearns - Philip Jenvey - Holger Krekel - Christian Tismer - Hakan Ardo - Benjamin Peterson - Manuel Jacob - Anders Chrigstrom - Eric van Riet Paap - Wim Lavrijsen - Ronan Lamy - Richard Emslie - Alexander Schremmer - Dan Villiom Podlaski Christiansen - Lukas Diekmann - Sven Hager - Anders Lehmann - Aurelien Campeas - Niklaus Haldimann - Camillo Bruni - Laura Creighton - Toon Verwaest - Remi Meier - Leonardo Santagada - Seo Sanghyeon - Romain Guillebert - Justin Peel - Ronny Pfannschmidt - David Edelsohn - Anders Hammarquist - Jakub Gustak - Guido Wesdorp - Lawrence Oluyede - Bartosz Skowron - Daniel Roberts - Niko Matsakis - Adrien Di Mascio - Alexander Hesse - Ludovic Aubry - Jacob Hallen - Jason Creighton - Alex Martelli - Michal Bendowski - Jan de Mooij - stian - Michael Foord - Stephan Diehl - Stefan Schwarzer - Valentino Volonghi - Tomek Meka - Patrick Maupin - Bob Ippolito - Bruno Gola - Jean-Paul Calderone - Timo Paulssen - Squeaky - Alexandre Fayolle - Simon Burton - Marius Gedminas - John Witulski - Konstantin Lopuhin - Greg Price - Dario Bertini - Mark Pearse - Simon Cross - Andreas Stührk - Jean-Philippe St. Pierre - Guido van Rossum - Pavel Vinogradov - Paweł Piotr Przeradowski - Paul deGrandis - Ilya Osadchiy - Tobias Oberstein - Adrian Kuhn - Boris Feigin - Stefano Rivera - tav - Taavi Burns - Georg Brandl - Bert Freudenberg - Stian Andreassen - Laurence Tratt - Wanja Saatkamp - Ivan Sichmann Freitas - Gerald Klix - Mike Blume - Oscar Nierstrasz - Stefan H. Muller - Jeremy Thurgood - Gregor Wegberg - Rami Chowdhury - Tobias Pape - Edd Barrett - David Malcolm - Eugene Oden - Henry Mason - Preston Timmons - Jeff Terrace - David Ripton - Dusty Phillips - Lukas Renggli - Guenter Jantzen - Ned Batchelder - Amit Regmi - Ben Young - Nicolas Chauvat - Andrew Durdin - Andrew Chambers - Michael Schneider - Nicholas Riley - Jason Chu - Igor Trindade Oliveira - Rocco Moretti - Gintautas Miliauskas - Michael Twomey - Lucian Branescu Mihaila - Tim Felgentreff - Tyler Wade - Gabriel Lavoie - Olivier Dormond - Jared Grubb - Karl Bartel - Brian Dorsey - Victor Stinner - Andrews Medina - Stuart Williams - Jasper Schulz - Christian Hudon - Toby Watson - Antoine Pitrou - Aaron Iles - Michael Cheng - Justas Sadzevicius - Mikael Schönenberg - Gasper Zejn - Neil Shepperd - Elmo Mäntynen - Jonathan David Riehl - Stanislaw Halik - Anders Qvist - Chirag Jadwani - Beatrice During - Alex Perry - Vincent Legoll - Alan McIntyre - Alexander Sedov - Corbin Simpson - Christopher Pope - wenzhuman - Christian Tismer - Marc Abramowitz - Dan Stromberg - Stefano Parmesan - Alexis Daboville - Jens-Uwe Mager - Carl Meyer - Karl Ramm - Pieter Zieschang - Gabriel - Lukas Vacek - Andrew Dalke - Sylvain Thenault - Nathan Taylor - Vladimir Kryachko - Jacek Generowicz - Alejandro J. Cura - Jacob Oscarson - Travis Francis Athougies - Ryan Gonzalez - Kristjan Valur Jonsson - Sebastian Pawluś - Neil Blakey-Milner - anatoly techtonik - Lutz Paelike - Lucio Torre - Lars Wassermann - Henrik Vendelbo - Dan Buch - Miguel de Val Borro - Artur Lisiecki - Sergey Kishchenko - Ignas Mikalajunas - Christoph Gerum - Martin Blais - Lene Wagner - Tomo Cocoa - roberto at goyle - Yury V. Zaytsev - Anna Katrina Dominguez - William Leslie - Bobby Impollonia - timo at eistee.fritz.box - Andrew Thompson - Ben Darnell - Roberto De Ioris - Juan Francisco Cantero Hurtado - Godefroid Chappelle - Joshua Gilbert - Dan Colish - Christopher Armstrong - Michael Hudson-Doyle - Anders Sigfridsson - Yasir Suhail - rafalgalczynski at gmail.com - Floris Bruynooghe - Laurens Van Houtven - Akira Li - Gustavo Niemeyer - Stephan Busemann - Rafał Gałczyński - Yusei Tahara - Christian Muirhead - James Lan - shoma hosaka - Daniel Neuh?user - Matthew Miller - Buck Golemon - Konrad Delong - Dinu Gherman - Chris Lambacher - coolbutuseless at gmail.com - Rodrigo Araújo - w31rd0 - Jim Baker - James Robert - Armin Ronacher - Brett Cannon - yrttyr - aliceinwire - OlivierBlanvillain - Zooko Wilcox-O Hearn - Tomer Chachamu - Christopher Groskopf - Asmo Soinio - Stefan Marr - jiaaro - opassembler.py - Antony Lee - Jim Hunziker - Markus Unterwaditzer - Even Wiik Thomassen - jbs - soareschen - Kurt Griffiths - Mike Bayer - Flavio Percoco - Kristoffer Kleine - yasirs - Michael Chermside - Anna Ravencroft - Julien Phalip - Dan Loewenherz + Armin Rigo + Maciej Fijalkowski + Carl Friedrich Bolz + Antonio Cuni + Amaury Forgeot d'Arc + Samuele Pedroni + Alex Gaynor + Michael Hudson + David Schneider + Matti Picus + Brian Kearns + Philip Jenvey + Holger Krekel + Christian Tismer + Hakan Ardo + Benjamin Peterson + Manuel Jacob + Anders Chrigstrom + Eric van Riet Paap + Ronan Lamy + Wim Lavrijsen + Richard Emslie + Alexander Schremmer + Dan Villiom Podlaski Christiansen + Lukas Diekmann + Sven Hager + Anders Lehmann + Aurelien Campeas + Niklaus Haldimann + Remi Meier + Camillo Bruni + Laura Creighton + Toon Verwaest + Leonardo Santagada + Seo Sanghyeon + Romain Guillebert + Justin Peel + Ronny Pfannschmidt + David Edelsohn + Anders Hammarquist + Jakub Gustak + Guido Wesdorp + Lawrence Oluyede + Bartosz Skowron + Gregor Wegberg + Daniel Roberts + Niko Matsakis + Adrien Di Mascio + Alexander Hesse + Ludovic Aubry + Jacob Hallen + Jason Creighton + Alex Martelli + Michal Bendowski + Jan de Mooij + stian + Michael Foord + Stephan Diehl + Tyler Wade + Stefan Schwarzer + Valentino Volonghi + Tomek Meka + Patrick Maupin + Bob Ippolito + Bruno Gola + Jean-Paul Calderone + Timo Paulssen + Squeaky + Alexandre Fayolle + Simon Burton + Marius Gedminas + Martin Matusiak + Konstantin Lopuhin + John Witulski + Wenzhu Man + Greg Price + Dario Bertini + Mark Pearse + Simon Cross + Ivan Sichmann Freitas + Andreas Stührk + Jean-Philippe St. Pierre + Guido van Rossum + Pavel Vinogradov + Stefano Rivera + Paweł Piotr Przeradowski + Paul deGrandis + Ilya Osadchiy + Tobias Oberstein + Adrian Kuhn + Boris Feigin + tav + Taavi Burns + Georg Brandl + Laurence Tratt + Bert Freudenberg + Stian Andreassen + Wanja Saatkamp + Gerald Klix + Mike Blume + Oscar Nierstrasz + Stefan H. Muller + Edd Barrett + Jeremy Thurgood + Rami Chowdhury + Tobias Pape + David Malcolm + Eugene Oden + Henry Mason + Vasily Kuznetsov + Preston Timmons + Jeff Terrace + David Ripton + Dusty Phillips + Lukas Renggli + Guenter Jantzen + Ned Batchelder + Amit Regmi + Ben Young + Nicolas Chauvat + Andrew Durdin + Andrew Chambers + Michael Schneider + Nicholas Riley + Jason Chu + Igor Trindade Oliveira + Tim Felgentreff + Rocco Moretti + Gintautas Miliauskas + Michael Twomey + Lucian Branescu Mihaila + Gabriel Lavoie + Olivier Dormond + Jared Grubb + Karl Bartel + Brian Dorsey + Victor Stinner + Andrews Medina + Stuart Williams + Jasper Schulz + Christian Hudon + Toby Watson + Antoine Pitrou + Aaron Iles + Michael Cheng + Justas Sadzevicius + Gasper Zejn + anatoly techtonik + Neil Shepperd + Mikael Schönenberg + Elmo M?ntynen + Jonathan David Riehl + Stanislaw Halik + Anders Qvist + Corbin Simpson + Chirag Jadwani + Beatrice During + Alex Perry + Vincent Legoll + Alan McIntyre + Alexander Sedov + Christopher Pope + Christian Tismer + Marc Abramowitz + Dan Stromberg + Stefano Parmesan + Alexis Daboville + Jens-Uwe Mager + Carl Meyer + Karl Ramm + Pieter Zieschang + Sebastian Pawluś + Gabriel + Lukas Vacek + Andrew Dalke + Sylvain Thenault + Nathan Taylor + Vladimir Kryachko + Arjun Naik + Attila Gobi + Jacek Generowicz + Alejandro J. Cura + Jacob Oscarson + Travis Francis Athougies + Ryan Gonzalez + Ian Foote + Kristjan Valur Jonsson + Neil Blakey-Milner + Lutz Paelike + Lucio Torre + Lars Wassermann + Valentina Mukhamedzhanova + Henrik Vendelbo + Dan Buch + Miguel de Val Borro + Artur Lisiecki + Sergey Kishchenko + Yichao Yu + Ignas Mikalajunas + Christoph Gerum + Martin Blais + Lene Wagner + Tomo Cocoa + roberto at goyle + Yury V. Zaytsev + Anna Katrina Dominguez + William Leslie + Bobby Impollonia + timo at eistee.fritz.box + Andrew Thompson + Yusei Tahara + Ben Darnell + Roberto De Ioris + Juan Francisco Cantero Hurtado + Godefroid Chappelle + Joshua Gilbert + Dan Colish + Christopher Armstrong + Michael Hudson-Doyle + Anders Sigfridsson + Yasir Suhail + Jason Michalski + rafalgalczynski at gmail.com + Floris Bruynooghe + Laurens Van Houtven + Akira Li + Gustavo Niemeyer + Stephan Busemann + Rafał Gałczyński + Christian Muirhead + James Lan + shoma hosaka + Daniel Neuh?user + Matthew Miller + Buck Golemon + Konrad Delong + Dinu Gherman + Chris Lambacher + coolbutuseless at gmail.com + Rodrigo Araújo + Jim Baker + James Robert + Armin Ronacher + Brett Cannon + yrttyr + aliceinwire + OlivierBlanvillain + Zooko Wilcox-O Hearn + Tomer Chachamu + Christopher Groskopf + Asmo Soinio + Stefan Marr + jiaaro + Mads Kiilerich + opassembler.py + Antony Lee + Jim Hunziker + Markus Unterwaditzer + Even Wiik Thomassen + jbs + soareschen + Kurt Griffiths + Mike Bayer + Matthew Miller + Flavio Percoco + Kristoffer Kleine + yasirs + Michael Chermside + Anna Ravencroft + Dan Crosta + Julien Phalip + Dan Loewenherz - Heinrich-Heine University, Germany - Open End AB (formerly AB Strakt), Sweden - merlinux GmbH, Germany - tismerysoft GmbH, Germany - Logilab Paris, France - DFKI GmbH, Germany - Impara, Germany - Change Maker, Sweden - University of California Berkeley, USA - Google Inc. - King's College London + Heinrich-Heine University, Germany + Open End AB (formerly AB Strakt), Sweden + merlinux GmbH, Germany + tismerysoft GmbH, Germany + Logilab Paris, France + DFKI GmbH, Germany + Impara, Germany + Change Maker, Sweden + University of California Berkeley, USA + Google Inc. + King's College London The PyPy Logo as used by http://speed.pypy.org and others was created by Samuel Reis and is distributed on terms of Creative Commons Share Alike @@ -354,6 +364,6 @@ See the License for the specific language governing permissions and limitations under the License. -Detailled license information is contained in the NOTICE file in the +Detailed license information is contained in the NOTICE file in the directory. diff --git a/_pytest/__init__.py b/_pytest/__init__.py --- a/_pytest/__init__.py +++ b/_pytest/__init__.py @@ -1,2 +1,2 @@ # -__version__ = '2.2.4.dev2' +__version__ = '2.5.2' diff --git a/_pytest/_argcomplete.py b/_pytest/_argcomplete.py new file mode 100644 --- /dev/null +++ b/_pytest/_argcomplete.py @@ -0,0 +1,104 @@ + +"""allow bash-completion for argparse with argcomplete if installed +needs argcomplete>=0.5.6 for python 3.2/3.3 (older versions fail +to find the magic string, so _ARGCOMPLETE env. var is never set, and +this does not need special code. + +argcomplete does not support python 2.5 (although the changes for that +are minor). + +Function try_argcomplete(parser) should be called directly before +the call to ArgumentParser.parse_args(). + +The filescompleter is what you normally would use on the positional +arguments specification, in order to get "dirname/" after "dirn" +instead of the default "dirname ": + + optparser.add_argument(Config._file_or_dir, nargs='*' + ).completer=filescompleter + +Other, application specific, completers should go in the file +doing the add_argument calls as they need to be specified as .completer +attributes as well. (If argcomplete is not installed, the function the +attribute points to will not be used). + +SPEEDUP +======= +The generic argcomplete script for bash-completion +(/etc/bash_completion.d/python-argcomplete.sh ) +uses a python program to determine startup script generated by pip. +You can speed up completion somewhat by changing this script to include + # PYTHON_ARGCOMPLETE_OK +so the the python-argcomplete-check-easy-install-script does not +need to be called to find the entry point of the code and see if that is +marked with PYTHON_ARGCOMPLETE_OK + +INSTALL/DEBUGGING +================= +To include this support in another application that has setup.py generated +scripts: +- add the line: + # PYTHON_ARGCOMPLETE_OK + near the top of the main python entry point +- include in the file calling parse_args(): + from _argcomplete import try_argcomplete, filescompleter + , call try_argcomplete just before parse_args(), and optionally add + filescompleter to the positional arguments' add_argument() +If things do not work right away: +- switch on argcomplete debugging with (also helpful when doing custom + completers): + export _ARC_DEBUG=1 +- run: + python-argcomplete-check-easy-install-script $(which appname) + echo $? + will echo 0 if the magic line has been found, 1 if not +- sometimes it helps to find early on errors using: + _ARGCOMPLETE=1 _ARC_DEBUG=1 appname + which should throw a KeyError: 'COMPLINE' (which is properly set by the + global argcomplete script). +""" + +import sys +import os +from glob import glob + +class FastFilesCompleter: + 'Fast file completer class' + def __init__(self, directories=True): + self.directories = directories + + def __call__(self, prefix, **kwargs): + """only called on non option completions""" + if os.path.sep in prefix[1:]: # + prefix_dir = len(os.path.dirname(prefix) + os.path.sep) + else: + prefix_dir = 0 + completion = [] + globbed = [] + if '*' not in prefix and '?' not in prefix: + if prefix[-1] == os.path.sep: # we are on unix, otherwise no bash + globbed.extend(glob(prefix + '.*')) + prefix += '*' + globbed.extend(glob(prefix)) + for x in sorted(globbed): + if os.path.isdir(x): + x += '/' + # append stripping the prefix (like bash, not like compgen) + completion.append(x[prefix_dir:]) + return completion + +if os.environ.get('_ARGCOMPLETE'): + # argcomplete 0.5.6 is not compatible with python 2.5.6: print/with/format + if sys.version_info[:2] < (2, 6): + sys.exit(1) + try: + import argcomplete.completers + except ImportError: + sys.exit(-1) + filescompleter = FastFilesCompleter() + + def try_argcomplete(parser): + argcomplete.autocomplete(parser) +else: + def try_argcomplete(parser): pass + filescompleter = None diff --git a/_pytest/assertion/__init__.py b/_pytest/assertion/__init__.py --- a/_pytest/assertion/__init__.py +++ b/_pytest/assertion/__init__.py @@ -3,7 +3,6 @@ """ import py import sys -import pytest from _pytest.monkeypatch import monkeypatch from _pytest.assertion import util @@ -19,8 +18,8 @@ to provide assert expression information. """) group.addoption('--no-assert', action="store_true", default=False, dest="noassert", help="DEPRECATED equivalent to --assert=plain") - group.addoption('--nomagic', action="store_true", default=False, - dest="nomagic", help="DEPRECATED equivalent to --assert=plain") + group.addoption('--nomagic', '--no-magic', action="store_true", + default=False, help="DEPRECATED equivalent to --assert=plain") class AssertionState: """State for the assertion plugin.""" @@ -35,22 +34,25 @@ mode = "plain" if mode == "rewrite": try: - import ast + import ast # noqa except ImportError: mode = "reinterp" else: - if sys.platform.startswith('java'): + # Both Jython and CPython 2.6.0 have AST bugs that make the + # assertion rewriting hook malfunction. + if (sys.platform.startswith('java') or + sys.version_info[:3] == (2, 6, 0)): mode = "reinterp" if mode != "plain": _load_modules(mode) m = monkeypatch() config._cleanup.append(m.undo) m.setattr(py.builtin.builtins, 'AssertionError', - reinterpret.AssertionError) + reinterpret.AssertionError) # noqa hook = None if mode == "rewrite": - hook = rewrite.AssertionRewritingHook() - sys.meta_path.append(hook) + hook = rewrite.AssertionRewritingHook() # noqa + sys.meta_path.insert(0, hook) warn_about_missing_assertion(mode) config._assertstate = AssertionState(config, mode) config._assertstate.hook = hook @@ -73,9 +75,16 @@ def callbinrepr(op, left, right): hook_result = item.ihook.pytest_assertrepr_compare( config=item.config, op=op, left=left, right=right) + for new_expl in hook_result: if new_expl: - res = '\n~'.join(new_expl) + # Don't include pageloads of data unless we are very + # verbose (-vv) + if (sum(len(p) for p in new_expl[1:]) > 80*8 + and item.config.option.verbose < 2): + new_expl[1:] = [py.builtin._totext( + 'Detailed information truncated, use "-vv" to show')] + res = py.builtin._totext('\n~').join(new_expl) if item.config.getvalue("assertmode") == "rewrite": # The result will be fed back a python % formatting # operation, which will fail if there are extraneous @@ -95,9 +104,9 @@ def _load_modules(mode): """Lazily import assertion related code.""" global rewrite, reinterpret - from _pytest.assertion import reinterpret + from _pytest.assertion import reinterpret # noqa if mode == "rewrite": - from _pytest.assertion import rewrite + from _pytest.assertion import rewrite # noqa def warn_about_missing_assertion(mode): try: diff --git a/_pytest/assertion/newinterpret.py b/_pytest/assertion/newinterpret.py --- a/_pytest/assertion/newinterpret.py +++ b/_pytest/assertion/newinterpret.py @@ -11,7 +11,7 @@ from _pytest.assertion.reinterpret import BuiltinAssertionError -if sys.platform.startswith("java") and sys.version_info < (2, 5, 2): +if sys.platform.startswith("java"): # See http://bugs.jython.org/issue1497 _exprs = ("BoolOp", "BinOp", "UnaryOp", "Lambda", "IfExp", "Dict", "ListComp", "GeneratorExp", "Yield", "Compare", "Call", diff --git a/_pytest/assertion/oldinterpret.py b/_pytest/assertion/oldinterpret.py --- a/_pytest/assertion/oldinterpret.py +++ b/_pytest/assertion/oldinterpret.py @@ -526,10 +526,13 @@ # example: def f(): return 5 + def g(): return 3 + def h(x): return 'never' + check("f() * g() == 5") check("not f()") check("not (f() and g() or 0)") diff --git a/_pytest/assertion/reinterpret.py b/_pytest/assertion/reinterpret.py --- a/_pytest/assertion/reinterpret.py +++ b/_pytest/assertion/reinterpret.py @@ -1,18 +1,26 @@ import sys import py from _pytest.assertion.util import BuiltinAssertionError +u = py.builtin._totext + class AssertionError(BuiltinAssertionError): def __init__(self, *args): BuiltinAssertionError.__init__(self, *args) if args: + # on Python2.6 we get len(args)==2 for: assert 0, (x,y) + # on Python2.7 and above we always get len(args) == 1 + # with args[0] being the (x,y) tuple. + if len(args) > 1: + toprint = args + else: + toprint = args[0] try: - self.msg = str(args[0]) - except py.builtin._sysex: - raise - except: - self.msg = "<[broken __repr__] %s at %0xd>" %( - args[0].__class__, id(args[0])) + self.msg = u(toprint) + except Exception: + self.msg = u( + "<[broken __repr__] %s at %0xd>" + % (toprint.__class__, id(toprint))) else: f = py.code.Frame(sys._getframe(1)) try: @@ -44,4 +52,3 @@ from _pytest.assertion.newinterpret import interpret as reinterpret else: reinterpret = reinterpret_old - diff --git a/_pytest/assertion/rewrite.py b/_pytest/assertion/rewrite.py --- a/_pytest/assertion/rewrite.py +++ b/_pytest/assertion/rewrite.py @@ -6,6 +6,7 @@ import imp import marshal import os +import re import struct import sys import types @@ -14,13 +15,7 @@ from _pytest.assertion import util -# Windows gives ENOENT in places *nix gives ENOTDIR. -if sys.platform.startswith("win"): - PATH_COMPONENT_NOT_DIR = errno.ENOENT -else: - PATH_COMPONENT_NOT_DIR = errno.ENOTDIR - -# py.test caches rewritten pycs in __pycache__. +# pytest caches rewritten pycs in __pycache__. if hasattr(imp, "get_tag"): PYTEST_TAG = imp.get_tag() + "-PYTEST" else: @@ -34,17 +29,19 @@ PYTEST_TAG = "%s-%s%s-PYTEST" % (impl, ver[0], ver[1]) del ver, impl -PYC_EXT = ".py" + "c" if __debug__ else "o" +PYC_EXT = ".py" + (__debug__ and "c" or "o") PYC_TAIL = "." + PYTEST_TAG + PYC_EXT REWRITE_NEWLINES = sys.version_info[:2] != (2, 7) and sys.version_info < (3, 2) +ASCII_IS_DEFAULT_ENCODING = sys.version_info[0] < 3 class AssertionRewritingHook(object): - """Import hook which rewrites asserts.""" + """PEP302 Import hook which rewrites asserts.""" def __init__(self): self.session = None self.modules = {} + self._register_with_pkg_resources() def set_session(self, session): self.fnpats = session.config.getini("python_files") @@ -59,8 +56,12 @@ names = name.rsplit(".", 1) lastname = names[-1] pth = None - if path is not None and len(path) == 1: - pth = path[0] + if path is not None: + # Starting with Python 3.3, path is a _NamespacePath(), which + # causes problems if not converted to list. + path = list(path) + if len(path) == 1: + pth = path[0] if pth is None: try: fd, fn, desc = imp.find_module(lastname, path) @@ -95,12 +96,13 @@ finally: self.session = sess else: - state.trace("matched test file (was specified on cmdline): %r" % (fn,)) + state.trace("matched test file (was specified on cmdline): %r" % + (fn,)) # The requested module looks like a test file, so rewrite it. This is # the most magical part of the process: load the source, rewrite the # asserts, and load the rewritten source. We also cache the rewritten # module code in a special pyc. We must be aware of the possibility of - # concurrent py.test processes rewriting and loading pycs. To avoid + # concurrent pytest processes rewriting and loading pycs. To avoid # tricky race conditions, we maintain the following invariant: The # cached pyc is always a complete, valid pyc. Operations on it must be # atomic. POSIX's atomic rename comes in handy. @@ -116,19 +118,19 @@ # common case) or it's blocked by a non-dir node. In the # latter case, we'll ignore it in _write_pyc. pass - elif e == PATH_COMPONENT_NOT_DIR: + elif e in [errno.ENOENT, errno.ENOTDIR]: # One of the path components was not a directory, likely # because we're in a zip file. write = False elif e == errno.EACCES: - state.trace("read only directory: %r" % (fn_pypath.dirname,)) + state.trace("read only directory: %r" % fn_pypath.dirname) write = False else: raise cache_name = fn_pypath.basename[:-3] + PYC_TAIL pyc = os.path.join(cache_dir, cache_name) - # Notice that even if we're in a read-only directory, I'm going to check - # for a cached pyc. This may not be optimal... + # Notice that even if we're in a read-only directory, I'm going + # to check for a cached pyc. This may not be optimal... co = _read_pyc(fn_pypath, pyc) if co is None: state.trace("rewriting %r" % (fn,)) @@ -153,27 +155,59 @@ mod.__file__ = co.co_filename # Normally, this attribute is 3.2+. mod.__cached__ = pyc + mod.__loader__ = self py.builtin.exec_(co, mod.__dict__) except: del sys.modules[name] raise return sys.modules[name] -def _write_pyc(co, source_path, pyc): - # Technically, we don't have to have the same pyc format as (C)Python, since - # these "pycs" should never be seen by builtin import. However, there's - # little reason deviate, and I hope sometime to be able to use - # imp.load_compiled to load them. (See the comment in load_module above.) + + + def is_package(self, name): + try: + fd, fn, desc = imp.find_module(name) + except ImportError: + return False + if fd is not None: + fd.close() + tp = desc[2] + return tp == imp.PKG_DIRECTORY + + @classmethod + def _register_with_pkg_resources(cls): + """ + Ensure package resources can be loaded from this loader. May be called + multiple times, as the operation is idempotent. + """ + try: + import pkg_resources + # access an attribute in case a deferred importer is present + pkg_resources.__name__ + except ImportError: + return + + # Since pytest tests are always located in the file system, the + # DefaultProvider is appropriate. + pkg_resources.register_loader_type(cls, pkg_resources.DefaultProvider) + + +def _write_pyc(state, co, source_path, pyc): + # Technically, we don't have to have the same pyc format as + # (C)Python, since these "pycs" should never be seen by builtin + # import. However, there's little reason deviate, and I hope + # sometime to be able to use imp.load_compiled to load them. (See + # the comment in load_module above.) mtime = int(source_path.mtime()) try: fp = open(pyc, "wb") except IOError: err = sys.exc_info()[1].errno - if err == PATH_COMPONENT_NOT_DIR: - # This happens when we get a EEXIST in find_module creating the - # __pycache__ directory and __pycache__ is by some non-dir node. - return False - raise + state.trace("error writing pyc file at %s: errno=%s" %(pyc, err)) + # we ignore any failure to write the cache file + # there are many reasons, permission-denied, __pycache__ being a + # file etc. + return False try: fp.write(imp.get_magic()) fp.write(struct.pack(">", - ast.Add : "+", - ast.Sub : "-", - ast.Mult : "*", - ast.Div : "/", - ast.FloorDiv : "//", - ast.Mod : "%", - ast.Eq : "==", - ast.NotEq : "!=", - ast.Lt : "<", - ast.LtE : "<=", - ast.Gt : ">", - ast.GtE : ">=", - ast.Pow : "**", - ast.Is : "is", - ast.IsNot : "is not", - ast.In : "in", - ast.NotIn : "not in" + ast.BitOr: "|", + ast.BitXor: "^", + ast.BitAnd: "&", + ast.LShift: "<<", + ast.RShift: ">>", + ast.Add: "+", + ast.Sub: "-", + ast.Mult: "*", + ast.Div: "/", + ast.FloorDiv: "//", + ast.Mod: "%%", # escaped for string formatting + ast.Eq: "==", + ast.NotEq: "!=", + ast.Lt: "<", + ast.LtE: "<=", + ast.Gt: ">", + ast.GtE: ">=", + ast.Pow: "**", + ast.Is: "is", + ast.IsNot: "is not", + ast.In: "in", + ast.NotIn: "not in" } @@ -341,7 +408,7 @@ lineno = 0 for item in mod.body: if (expect_docstring and isinstance(item, ast.Expr) and - isinstance(item.value, ast.Str)): + isinstance(item.value, ast.Str)): doc = item.value.s if "PYTEST_DONT_REWRITE" in doc: # The module has disabled assertion rewriting. @@ -462,7 +529,8 @@ body.append(raise_) # Clear temporary variables by setting them to None. if self.variables: - variables = [ast.Name(name, ast.Store()) for name in self.variables] + variables = [ast.Name(name, ast.Store()) + for name in self.variables] clear = ast.Assign(variables, ast.Name("None", ast.Load())) self.statements.append(clear) # Fix line numbers. @@ -471,11 +539,12 @@ return self.statements def visit_Name(self, name): - # Check if the name is local or not. + # Display the repr of the name if it's a local variable or + # _should_repr_global_name() thinks it's acceptable. locs = ast.Call(self.builtin("locals"), [], [], None, None) - globs = ast.Call(self.builtin("globals"), [], [], None, None) - ops = [ast.In(), ast.IsNot()] - test = ast.Compare(ast.Str(name.id), ops, [locs, globs]) + inlocs = ast.Compare(ast.Str(name.id), [ast.In()], [locs]) + dorepr = self.helper("should_repr_global_name", name) + test = ast.BoolOp(ast.Or(), [inlocs, dorepr]) expr = ast.IfExp(test, self.display(name), ast.Str(name.id)) return name, self.explanation_param(expr) @@ -492,7 +561,8 @@ for i, v in enumerate(boolop.values): if i: fail_inner = [] - self.on_failure.append(ast.If(cond, fail_inner, [])) + # cond is set in a prior loop iteration below + self.on_failure.append(ast.If(cond, fail_inner, [])) # noqa self.on_failure = fail_inner self.push_format_context() res, expl = self.visit(v) @@ -548,7 +618,8 @@ new_kwarg, expl = self.visit(call.kwargs) arg_expls.append("**" + expl) expl = "%s(%s)" % (func_expl, ', '.join(arg_expls)) - new_call = ast.Call(new_func, new_args, new_kwargs, new_star, new_kwarg) + new_call = ast.Call(new_func, new_args, new_kwargs, + new_star, new_kwarg) res = self.assign(new_call) res_expl = self.explanation_param(self.display(res)) outer_expl = "%s\n{%s = %s\n}" % (res_expl, res_expl, expl) @@ -584,7 +655,7 @@ res_expr = ast.Compare(left_res, [op], [next_res]) self.statements.append(ast.Assign([store_names[i]], res_expr)) left_res, left_expl = next_res, next_expl - # Use py.code._reprcompare if that's available. + # Use pytest.assertion.util._reprcompare if that's available. expl_call = self.helper("call_reprcompare", ast.Tuple(syms, ast.Load()), ast.Tuple(load_names, ast.Load()), diff --git a/_pytest/assertion/util.py b/_pytest/assertion/util.py --- a/_pytest/assertion/util.py +++ b/_pytest/assertion/util.py @@ -1,8 +1,13 @@ """Utilities for assertion debugging""" import py +try: + from collections import Sequence +except ImportError: + Sequence = list BuiltinAssertionError = py.builtin.builtins.AssertionError +u = py.builtin._totext # The _reprcompare attribute on the util module is used by the new assertion # interpretation code and assertion rewriter to detect this plugin was @@ -10,6 +15,7 @@ # DebugInterpreter. _reprcompare = None + def format_explanation(explanation): """This formats an explanation @@ -20,7 +26,18 @@ for when one explanation needs to span multiple lines, e.g. when displaying diffs. """ - # simplify 'assert False where False = ...' + explanation = _collapse_false(explanation) + lines = _split_explanation(explanation) + result = _format_lines(lines) + return u('\n').join(result) + + +def _collapse_false(explanation): + """Collapse expansions of False + + So this strips out any "assert False\n{where False = ...\n}" + blocks. + """ where = 0 while True: start = where = explanation.find("False\n{False = ", where) @@ -42,28 +59,48 @@ explanation = (explanation[:start] + explanation[start+15:end-1] + explanation[end+1:]) where -= 17 - raw_lines = (explanation or '').split('\n') - # escape newlines not followed by {, } and ~ + return explanation + + +def _split_explanation(explanation): + """Return a list of individual lines in the explanation + + This will return a list of lines split on '\n{', '\n}' and '\n~'. + Any other newlines will be escaped and appear in the line as the + literal '\n' characters. + """ + raw_lines = (explanation or u('')).split('\n') lines = [raw_lines[0]] for l in raw_lines[1:]: if l.startswith('{') or l.startswith('}') or l.startswith('~'): lines.append(l) else: lines[-1] += '\\n' + l + return lines + +def _format_lines(lines): + """Format the individual lines + + This will replace the '{', '}' and '~' characters of our mini + formatting language with the proper 'where ...', 'and ...' and ' + + ...' text, taking care of indentation along the way. + + Return a list of formatted lines. + """ result = lines[:1] stack = [0] stackcnt = [0] for line in lines[1:]: if line.startswith('{'): if stackcnt[-1]: - s = 'and ' + s = u('and ') else: - s = 'where ' + s = u('where ') stack.append(len(result)) stackcnt[-1] += 1 stackcnt.append(0) - result.append(' +' + ' '*(len(stack)-1) + s + line[1:]) + result.append(u(' +') + u(' ')*(len(stack)-1) + s + line[1:]) elif line.startswith('}'): assert line.startswith('}') stack.pop() @@ -71,9 +108,9 @@ result[stack[-1]] += line[1:] else: assert line.startswith('~') - result.append(' '*len(stack) + line[1:]) + result.append(u(' ')*len(stack) + line[1:]) assert len(stack) == 1 - return '\n'.join(result) + return result # Provide basestring in python3 @@ -83,132 +120,163 @@ basestring = str -def assertrepr_compare(op, left, right): - """return specialised explanations for some operators/operands""" - width = 80 - 15 - len(op) - 2 # 15 chars indentation, 1 space around op +def assertrepr_compare(config, op, left, right): + """Return specialised explanations for some operators/operands""" + width = 80 - 15 - len(op) - 2 # 15 chars indentation, 1 space around op left_repr = py.io.saferepr(left, maxsize=int(width/2)) right_repr = py.io.saferepr(right, maxsize=width-len(left_repr)) - summary = '%s %s %s' % (left_repr, op, right_repr) + summary = u('%s %s %s') % (left_repr, op, right_repr) - issequence = lambda x: isinstance(x, (list, tuple)) + issequence = lambda x: (isinstance(x, (list, tuple, Sequence)) + and not isinstance(x, basestring)) istext = lambda x: isinstance(x, basestring) isdict = lambda x: isinstance(x, dict) - isset = lambda x: isinstance(x, set) + isset = lambda x: isinstance(x, (set, frozenset)) + verbose = config.getoption('verbose') explanation = None try: if op == '==': if istext(left) and istext(right): - explanation = _diff_text(left, right) + explanation = _diff_text(left, right, verbose) elif issequence(left) and issequence(right): - explanation = _compare_eq_sequence(left, right) + explanation = _compare_eq_sequence(left, right, verbose) elif isset(left) and isset(right): - explanation = _compare_eq_set(left, right) + explanation = _compare_eq_set(left, right, verbose) elif isdict(left) and isdict(right): - explanation = _diff_text(py.std.pprint.pformat(left), - py.std.pprint.pformat(right)) + explanation = _compare_eq_dict(left, right, verbose) elif op == 'not in': if istext(left) and istext(right): - explanation = _notin_text(left, right) - except py.builtin._sysex: - raise - except: + explanation = _notin_text(left, right, verbose) + except Exception: excinfo = py.code.ExceptionInfo() - explanation = ['(pytest_assertion plugin: representation of ' - 'details failed. Probably an object has a faulty __repr__.)', - str(excinfo) - ] - + explanation = [ + u('(pytest_assertion plugin: representation of details failed. ' + 'Probably an object has a faulty __repr__.)'), + u(excinfo)] if not explanation: return None - # Don't include pageloads of data, should be configurable - if len(''.join(explanation)) > 80*8: - explanation = ['Detailed information too verbose, truncated'] - return [summary] + explanation -def _diff_text(left, right): - """Return the explanation for the diff between text +def _diff_text(left, right, verbose=False): + """Return the explanation for the diff between text or bytes - This will skip leading and trailing characters which are - identical to keep the diff minimal. + Unless --verbose is used this will skip leading and trailing + characters which are identical to keep the diff minimal. + + If the input are bytes they will be safely converted to text. """ explanation = [] - i = 0 # just in case left or right has zero length - for i in range(min(len(left), len(right))): - if left[i] != right[i]: - break - if i > 42: - i -= 10 # Provide some context - explanation = ['Skipping %s identical ' - 'leading characters in diff' % i] - left = left[i:] - right = right[i:] - if len(left) == len(right): - for i in range(len(left)): - if left[-i] != right[-i]: + if isinstance(left, py.builtin.bytes): + left = u(repr(left)[1:-1]).replace(r'\n', '\n') + if isinstance(right, py.builtin.bytes): + right = u(repr(right)[1:-1]).replace(r'\n', '\n') + if not verbose: + i = 0 # just in case left or right has zero length + for i in range(min(len(left), len(right))): + if left[i] != right[i]: break if i > 42: - i -= 10 # Provide some context - explanation += ['Skipping %s identical ' - 'trailing characters in diff' % i] - left = left[:-i] - right = right[:-i] + i -= 10 # Provide some context + explanation = [u('Skipping %s identical leading ' + 'characters in diff, use -v to show') % i] + left = left[i:] + right = right[i:] + if len(left) == len(right): + for i in range(len(left)): + if left[-i] != right[-i]: + break + if i > 42: + i -= 10 # Provide some context + explanation += [u('Skipping %s identical trailing ' + 'characters in diff, use -v to show') % i] + left = left[:-i] + right = right[:-i] explanation += [line.strip('\n') for line in py.std.difflib.ndiff(left.splitlines(), right.splitlines())] return explanation -def _compare_eq_sequence(left, right): +def _compare_eq_sequence(left, right, verbose=False): explanation = [] for i in range(min(len(left), len(right))): if left[i] != right[i]: - explanation += ['At index %s diff: %r != %r' % - (i, left[i], right[i])] + explanation += [u('At index %s diff: %r != %r') + % (i, left[i], right[i])] break if len(left) > len(right): - explanation += ['Left contains more items, ' - 'first extra item: %s' % py.io.saferepr(left[len(right)],)] + explanation += [u('Left contains more items, first extra item: %s') + % py.io.saferepr(left[len(right)],)] elif len(left) < len(right): - explanation += ['Right contains more items, ' - 'first extra item: %s' % py.io.saferepr(right[len(left)],)] - return explanation # + _diff_text(py.std.pprint.pformat(left), - # py.std.pprint.pformat(right)) + explanation += [ + u('Right contains more items, first extra item: %s') % + py.io.saferepr(right[len(left)],)] + return explanation # + _diff_text(py.std.pprint.pformat(left), + # py.std.pprint.pformat(right)) -def _compare_eq_set(left, right): +def _compare_eq_set(left, right, verbose=False): explanation = [] diff_left = left - right diff_right = right - left if diff_left: - explanation.append('Extra items in the left set:') + explanation.append(u('Extra items in the left set:')) for item in diff_left: explanation.append(py.io.saferepr(item)) if diff_right: - explanation.append('Extra items in the right set:') + explanation.append(u('Extra items in the right set:')) for item in diff_right: explanation.append(py.io.saferepr(item)) return explanation -def _notin_text(term, text): +def _compare_eq_dict(left, right, verbose=False): + explanation = [] + common = set(left).intersection(set(right)) + same = dict((k, left[k]) for k in common if left[k] == right[k]) + if same and not verbose: + explanation += [u('Omitting %s identical items, use -v to show') % + len(same)] + elif same: + explanation += [u('Common items:')] + explanation += py.std.pprint.pformat(same).splitlines() + diff = set(k for k in common if left[k] != right[k]) + if diff: + explanation += [u('Differing items:')] + for k in diff: + explanation += [py.io.saferepr({k: left[k]}) + ' != ' + + py.io.saferepr({k: right[k]})] + extra_left = set(left) - set(right) + if extra_left: + explanation.append(u('Left contains more items:')) + explanation.extend(py.std.pprint.pformat( + dict((k, left[k]) for k in extra_left)).splitlines()) + extra_right = set(right) - set(left) + if extra_right: + explanation.append(u('Right contains more items:')) + explanation.extend(py.std.pprint.pformat( + dict((k, right[k]) for k in extra_right)).splitlines()) + return explanation + + +def _notin_text(term, text, verbose=False): index = text.find(term) head = text[:index] tail = text[index+len(term):] correct_text = head + tail - diff = _diff_text(correct_text, text) - newdiff = ['%s is contained here:' % py.io.saferepr(term, maxsize=42)] + diff = _diff_text(correct_text, text, verbose) + newdiff = [u('%s is contained here:') % py.io.saferepr(term, maxsize=42)] for line in diff: - if line.startswith('Skipping'): + if line.startswith(u('Skipping')): continue - if line.startswith('- '): + if line.startswith(u('- ')): continue - if line.startswith('+ '): - newdiff.append(' ' + line[2:]) + if line.startswith(u('+ ')): + newdiff.append(u(' ') + line[2:]) else: newdiff.append(line) return newdiff diff --git a/_pytest/capture.py b/_pytest/capture.py --- a/_pytest/capture.py +++ b/_pytest/capture.py @@ -1,43 +1,114 @@ -""" per-test stdout/stderr capturing mechanisms, ``capsys`` and ``capfd`` function arguments. """ +""" + per-test stdout/stderr capturing mechanisms, + ``capsys`` and ``capfd`` function arguments. +""" +# note: py.io capture was where copied from +# pylib 1.4.20.dev2 (rev 13d9af95547e) +import sys +import os +import tempfile -import pytest, py -import os +import py +import pytest + +try: + from io import StringIO +except ImportError: + from StringIO import StringIO + +try: + from io import BytesIO +except ImportError: + class BytesIO(StringIO): + def write(self, data): + if isinstance(data, unicode): + raise TypeError("not a byte value: %r" % (data,)) + StringIO.write(self, data) + +if sys.version_info < (3, 0): + class TextIO(StringIO): + def write(self, data): + if not isinstance(data, unicode): + enc = getattr(self, '_encoding', 'UTF-8') + data = unicode(data, enc, 'replace') + StringIO.write(self, data) +else: + TextIO = StringIO + + +patchsysdict = {0: 'stdin', 1: 'stdout', 2: 'stderr'} + def pytest_addoption(parser): group = parser.getgroup("general") - group._addoption('--capture', action="store", default=None, - metavar="method", type="choice", choices=['fd', 'sys', 'no'], + group._addoption( + '--capture', action="store", default=None, + metavar="method", choices=['fd', 'sys', 'no'], help="per-test capturing method: one of fd (default)|sys|no.") - group._addoption('-s', action="store_const", const="no", dest="capture", + group._addoption( + '-s', action="store_const", const="no", dest="capture", help="shortcut for --capture=no.") + @pytest.mark.tryfirst -def pytest_cmdline_parse(pluginmanager, args): - # we want to perform capturing already for plugin/conftest loading - if '-s' in args or "--capture=no" in args: - method = "no" - elif hasattr(os, 'dup') and '--capture=sys' not in args: +def pytest_load_initial_conftests(early_config, parser, args, __multicall__): + ns = parser.parse_known_args(args) + method = ns.capture + if not method: method = "fd" - else: + if method == "fd" and not hasattr(os, "dup"): method = "sys" capman = CaptureManager(method) - pluginmanager.register(capman, "capturemanager") + early_config.pluginmanager.register(capman, "capturemanager") + + # make sure that capturemanager is properly reset at final shutdown + def teardown(): + try: + capman.reset_capturings() + except ValueError: + pass + + early_config.pluginmanager.add_shutdown(teardown) + + # make sure logging does not raise exceptions at the end + def silence_logging_at_shutdown(): + if "logging" in sys.modules: + sys.modules["logging"].raiseExceptions = False + early_config.pluginmanager.add_shutdown(silence_logging_at_shutdown) + + # finally trigger conftest loading but while capturing (issue93) + capman.resumecapture() + try: + try: + return __multicall__.execute() + finally: + out, err = capman.suspendcapture() + except: + sys.stdout.write(out) + sys.stderr.write(err) + raise + def addouterr(rep, outerr): for secname, content in zip(["out", "err"], outerr): if content: rep.sections.append(("Captured std%s" % secname, content)) + class NoCapture: def startall(self): pass + def resume(self): pass + def reset(self): pass + def suspend(self): return "", "" + class CaptureManager: def __init__(self, defaultmethod=None): self._method2capture = {} @@ -45,21 +116,23 @@ def _maketempfile(self): f = py.std.tempfile.TemporaryFile() - newf = py.io.dupfile(f, encoding="UTF-8") + newf = dupfile(f, encoding="UTF-8") f.close() return newf def _makestringio(self): - return py.io.TextIO() + return TextIO() def _getcapture(self, method): if method == "fd": - return py.io.StdCaptureFD(now=False, - out=self._maketempfile(), err=self._maketempfile() + return StdCaptureFD( + out=self._maketempfile(), + err=self._maketempfile(), ) elif method == "sys": - return py.io.StdCapture(now=False, - out=self._makestringio(), err=self._makestringio() + return StdCapture( + out=self._makestringio(), + err=self._makestringio(), ) elif method == "no": return NoCapture() @@ -74,23 +147,24 @@ method = config._conftest.rget("option_capture", path=fspath) except KeyError: method = "fd" - if method == "fd" and not hasattr(os, 'dup'): # e.g. jython + if method == "fd" and not hasattr(os, 'dup'): # e.g. jython method = "sys" return method def reset_capturings(self): - for name, cap in self._method2capture.items(): + for cap in self._method2capture.values(): cap.reset() def resumecapture_item(self, item): method = self._getmethod(item.config, item.fspath) if not hasattr(item, 'outerr'): - item.outerr = ('', '') # we accumulate outerr on the item + item.outerr = ('', '') # we accumulate outerr on the item return self.resumecapture(method) def resumecapture(self, method=None): if hasattr(self, '_capturing'): - raise ValueError("cannot resume, already capturing with %r" % + raise ValueError( + "cannot resume, already capturing with %r" % (self._capturing,)) if method is None: method = self._defaultmethod @@ -119,30 +193,29 @@ return "", "" def activate_funcargs(self, pyfuncitem): - if not hasattr(pyfuncitem, 'funcargs'): - return - assert not hasattr(self, '_capturing_funcargs') - self._capturing_funcargs = capturing_funcargs = [] - for name, capfuncarg in pyfuncitem.funcargs.items(): - if name in ('capsys', 'capfd'): - capturing_funcargs.append(capfuncarg) - capfuncarg._start() + funcargs = getattr(pyfuncitem, "funcargs", None) + if funcargs is not None: + for name, capfuncarg in funcargs.items(): + if name in ('capsys', 'capfd'): + assert not hasattr(self, '_capturing_funcarg') + self._capturing_funcarg = capfuncarg + capfuncarg._start() def deactivate_funcargs(self): - capturing_funcargs = getattr(self, '_capturing_funcargs', None) - if capturing_funcargs is not None: - while capturing_funcargs: - capfuncarg = capturing_funcargs.pop() - capfuncarg._finalize() - del self._capturing_funcargs + capturing_funcarg = getattr(self, '_capturing_funcarg', None) + if capturing_funcarg: + outerr = capturing_funcarg._finalize() + del self._capturing_funcarg + return outerr def pytest_make_collect_report(self, __multicall__, collector): method = self._getmethod(collector.config, collector.fspath) try: self.resumecapture(method) except ValueError: - return # recursive collect, XXX refactor capturing - # to allow for more lightweight recursive capturing + # recursive collect, XXX refactor capturing + # to allow for more lightweight recursive capturing + return try: rep = __multicall__.execute() finally: @@ -169,46 +242,371 @@ @pytest.mark.tryfirst def pytest_runtest_makereport(self, __multicall__, item, call): - self.deactivate_funcargs() + funcarg_outerr = self.deactivate_funcargs() rep = __multicall__.execute() outerr = self.suspendcapture(item) - if not rep.passed: - addouterr(rep, outerr) + if funcarg_outerr is not None: + outerr = (outerr[0] + funcarg_outerr[0], + outerr[1] + funcarg_outerr[1]) + addouterr(rep, outerr) if not rep.passed or rep.when == "teardown": outerr = ('', '') item.outerr = outerr return rep +error_capsysfderror = "cannot use capsys and capfd at the same time" + + def pytest_funcarg__capsys(request): """enables capturing of writes to sys.stdout/sys.stderr and makes captured output available via ``capsys.readouterr()`` method calls which return a ``(out, err)`` tuple. """ - return CaptureFuncarg(py.io.StdCapture) + if "capfd" in request._funcargs: + raise request.raiseerror(error_capsysfderror) + return CaptureFixture(StdCapture) + def pytest_funcarg__capfd(request): """enables capturing of writes to file descriptors 1 and 2 and makes captured output available via ``capsys.readouterr()`` method calls which return a ``(out, err)`` tuple. """ + if "capsys" in request._funcargs: + request.raiseerror(error_capsysfderror) if not hasattr(os, 'dup'): - py.test.skip("capfd funcarg needs os.dup") - return CaptureFuncarg(py.io.StdCaptureFD) + pytest.skip("capfd funcarg needs os.dup") + return CaptureFixture(StdCaptureFD) -class CaptureFuncarg: + +class CaptureFixture: def __init__(self, captureclass): - self.capture = captureclass(now=False) + self._capture = captureclass() def _start(self): - self.capture.startall() + self._capture.startall() def _finalize(self): - if hasattr(self, 'capture'): - self.capture.reset() - del self.capture + if hasattr(self, '_capture'): + outerr = self._outerr = self._capture.reset() + del self._capture + return outerr def readouterr(self): - return self.capture.readouterr() + try: + return self._capture.readouterr() + except AttributeError: + return self._outerr def close(self): self._finalize() + + +class FDCapture: + """ Capture IO to/from a given os-level filedescriptor. """ + + def __init__(self, targetfd, tmpfile=None, patchsys=False): + """ save targetfd descriptor, and open a new + temporary file there. If no tmpfile is + specified a tempfile.Tempfile() will be opened + in text mode. + """ + self.targetfd = targetfd + if tmpfile is None and targetfd != 0: + f = tempfile.TemporaryFile('wb+') + tmpfile = dupfile(f, encoding="UTF-8") + f.close() + self.tmpfile = tmpfile + self._savefd = os.dup(self.targetfd) + if patchsys: + self._oldsys = getattr(sys, patchsysdict[targetfd]) + + def start(self): + try: + os.fstat(self._savefd) + except OSError: + raise ValueError( + "saved filedescriptor not valid, " + "did you call start() twice?") + if self.targetfd == 0 and not self.tmpfile: + fd = os.open(os.devnull, os.O_RDONLY) + os.dup2(fd, 0) + os.close(fd) + if hasattr(self, '_oldsys'): + setattr(sys, patchsysdict[self.targetfd], DontReadFromInput()) + else: + os.dup2(self.tmpfile.fileno(), self.targetfd) + if hasattr(self, '_oldsys'): + setattr(sys, patchsysdict[self.targetfd], self.tmpfile) + + def done(self): + """ unpatch and clean up, returns the self.tmpfile (file object) + """ + os.dup2(self._savefd, self.targetfd) + os.close(self._savefd) + if self.targetfd != 0: + self.tmpfile.seek(0) + if hasattr(self, '_oldsys'): + setattr(sys, patchsysdict[self.targetfd], self._oldsys) + return self.tmpfile + + def writeorg(self, data): + """ write a string to the original file descriptor + """ + tempfp = tempfile.TemporaryFile() + try: + os.dup2(self._savefd, tempfp.fileno()) + tempfp.write(data) + finally: + tempfp.close() + + +def dupfile(f, mode=None, buffering=0, raising=False, encoding=None): + """ return a new open file object that's a duplicate of f + + mode is duplicated if not given, 'buffering' controls + buffer size (defaulting to no buffering) and 'raising' + defines whether an exception is raised when an incompatible + file object is passed in (if raising is False, the file + object itself will be returned) + """ + try: + fd = f.fileno() + mode = mode or f.mode + except AttributeError: + if raising: + raise + return f + newfd = os.dup(fd) + if sys.version_info >= (3, 0): + if encoding is not None: + mode = mode.replace("b", "") + buffering = True + return os.fdopen(newfd, mode, buffering, encoding, closefd=True) + else: + f = os.fdopen(newfd, mode, buffering) + if encoding is not None: + return EncodedFile(f, encoding) + return f + + +class EncodedFile(object): + def __init__(self, _stream, encoding): + self._stream = _stream + self.encoding = encoding + + def write(self, obj): + if isinstance(obj, unicode): + obj = obj.encode(self.encoding) + self._stream.write(obj) + + def writelines(self, linelist): + data = ''.join(linelist) + self.write(data) + + def __getattr__(self, name): + return getattr(self._stream, name) + + +class Capture(object): + def reset(self): + """ reset sys.stdout/stderr and return captured output as strings. """ + if hasattr(self, '_reset'): + raise ValueError("was already reset") + self._reset = True + outfile, errfile = self.done(save=False) + out, err = "", "" + if outfile and not outfile.closed: From noreply at buildbot.pypy.org Mon Sep 1 21:16:22 2014 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 1 Sep 2014 21:16:22 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: merge default Message-ID: <20140901191622.AC5E51C06D8@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73277:a9dc0d3a9f93 Date: 2014-09-01 13:15 -0600 http://bitbucket.org/pypy/pypy/changeset/a9dc0d3a9f93/ Log: merge default diff --git a/_pytest/README-BEFORE-UPDATING b/_pytest/README-BEFORE-UPDATING new file mode 100644 --- /dev/null +++ b/_pytest/README-BEFORE-UPDATING @@ -0,0 +1,17 @@ +This is PyPy's code of the pytest lib. We don't expect to upgrade it +very often, but once we do: + + WARNING! + + WE HAVE MADE A FEW TWEAKS HERE! + +Please be sure that you don't just copy the newer version from +upstream without checking the few changes that we did. This +can be done like this: + + cd + hg log . -v | less + +then search for all " _pytest/" in that list to know which are the +relevant checkins. (Look for the checkins that only edit one +or two files in this directory.) diff --git a/_pytest/resultlog.py b/_pytest/resultlog.py --- a/_pytest/resultlog.py +++ b/_pytest/resultlog.py @@ -53,16 +53,23 @@ self.config = config self.logfile = logfile # preferably line buffered - def write_log_entry(self, testpath, lettercode, longrepr): + def write_log_entry(self, testpath, lettercode, longrepr, sections=None): py.builtin.print_("%s %s" % (lettercode, testpath), file=self.logfile) for line in longrepr.splitlines(): py.builtin.print_(" %s" % line, file=self.logfile) + if sections is not None: + for title, content in sections: + py.builtin.print_(" ---------- %s ----------" % (title,), + file=self.logfile) + for line in content.splitlines(): + py.builtin.print_(" %s" % line, file=self.logfile) def log_outcome(self, report, lettercode, longrepr): testpath = getattr(report, 'nodeid', None) if testpath is None: testpath = report.fspath - self.write_log_entry(testpath, lettercode, longrepr) + self.write_log_entry(testpath, lettercode, longrepr, + getattr(report, 'sections', None)) def pytest_runtest_logreport(self, report): if report.when != "call" and report.passed: diff --git a/py/README-BEFORE-UPDATING b/py/README-BEFORE-UPDATING new file mode 100644 --- /dev/null +++ b/py/README-BEFORE-UPDATING @@ -0,0 +1,17 @@ +This is PyPy's code of the py lib. We don't expect to upgrade it +very often, but once we do: + + WARNING! + + WE HAVE MADE A FEW TWEAKS HERE! + +Please be sure that you don't just copy the newer version from +upstream without checking the few changes that we did. This +can be done like this: + + cd + hg log . -v | less + +then search for all " py/" in that list to know which are the +relevant checkins. (Look for the checkins that only edit one +or two files in this directory.) diff --git a/py/_path/local.py b/py/_path/local.py --- a/py/_path/local.py +++ b/py/_path/local.py @@ -750,7 +750,8 @@ mkdtemp = classmethod(mkdtemp) def make_numbered_dir(cls, prefix='session-', rootdir=None, keep=3, - lock_timeout = 172800): # two days + lock_timeout = 172800, # two days + min_timeout = 300): # five minutes """ return unique directory with a number greater than the current maximum one. The number is assumed to start directly after prefix. if keep is true directories with a number less than (maxnum-keep) @@ -818,6 +819,20 @@ for path in rootdir.listdir(): num = parse_num(path) if num is not None and num <= (maxnum - keep): + if min_timeout: + # NB: doing this is needed to prevent (or reduce + # a lot the chance of) the following situation: + # 'keep+1' processes call make_numbered_dir() at + # the same time, they create dirs, but then the + # last process notices the first dir doesn't have + # (yet) a .lock in it and kills it. + try: + t1 = path.lstat().mtime + t2 = lockfile.lstat().mtime + if abs(t2-t1) < min_timeout: + continue # skip directories too recent + except py.error.Error: + continue # failure to get a time, better skip lf = path.join('.lock') try: t1 = lf.lstat().mtime diff --git a/pypy/module/_pypyjson/interp_encoder.py b/pypy/module/_pypyjson/interp_encoder.py --- a/pypy/module/_pypyjson/interp_encoder.py +++ b/pypy/module/_pypyjson/interp_encoder.py @@ -37,16 +37,14 @@ sb = StringBuilder(len(u)) sb.append_slice(s, 0, first) else: + # We used to check if 'u' contains only safe characters, and return + # 'w_string' directly. But this requires an extra pass over all + # characters, and the expected use case of this function, from + # json.encoder, will anyway re-encode a unicode result back to + # a string (with the ascii encoding). This requires two passes + # over the characters. So we may as well directly turn it into a + # string here --- only one pass. u = space.unicode_w(w_string) - for i in range(len(u)): - c = u[i] - if c >= u' ' and c <= u'~' and c != u'"' and c != u'\\': - pass - else: - break - else: - # the input is a unicode with only non-special ascii chars - return w_string sb = StringBuilder(len(u)) first = 0 diff --git a/pypy/module/_pypyjson/test/test__pypyjson.py b/pypy/module/_pypyjson/test/test__pypyjson.py --- a/pypy/module/_pypyjson/test/test__pypyjson.py +++ b/pypy/module/_pypyjson/test/test__pypyjson.py @@ -192,14 +192,14 @@ def test_raw_encode_basestring_ascii(self): import _pypyjson - def check(s, expected_type=str): + def check(s): s = _pypyjson.raw_encode_basestring_ascii(s) - assert type(s) is expected_type + assert type(s) is str return s assert check("") == "" - assert check(u"", expected_type=unicode) == u"" + assert check(u"") == "" assert check("abc ") == "abc " - assert check(u"abc ", expected_type=unicode) == u"abc " + assert check(u"abc ") == "abc " raises(UnicodeDecodeError, check, "\xc0") assert check("\xc2\x84") == "\\u0084" assert check("\xf0\x92\x8d\x85") == "\\ud808\\udf45" diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py --- a/pypy/module/pypyjit/test_pypy_c/test_call.py +++ b/pypy/module/pypyjit/test_pypy_c/test_call.py @@ -17,13 +17,18 @@ # now we can inline it as call assembler i = 0 j = 0 - while i < 20: + while i < 25: i += 1 j += rec(100) # ID: call_rec return j # - log = self.run(fn, [], threshold=18) - loop, = log.loops_by_filename(self.filepath) + # NB. the parameters below are a bit ad-hoc. After 16 iterations, + # the we trace from the "while" and reach a "trace too long". Then + # in the next execution, we trace the "rec" function from start; + # that's "functrace" below. Then after one or two extra iterations + # we try again from "while", and this time we succeed. + log = self.run(fn, [], threshold=20) + functrace, loop = log.loops_by_filename(self.filepath) assert loop.match_by_id('call_rec', """ ... p53 = call_assembler(..., descr=...) diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py --- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py +++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py @@ -340,30 +340,19 @@ guard_value(p166, ConstPtr(ptr72), descr=...) p167 = call(ConstClass(_ll_0_alloc_with_del___), descr=) guard_no_exception(descr=...) - i168 = call(ConstClass(_ll_1_raw_malloc_varsize__Signed), 6, descr=) - i169 = int_add(i168, i97) - i170 = int_sub(i160, i106) - setfield_gc(p167, i168, descr=) + i112 = int_sub(i160, -32768) setfield_gc(p167, ConstPtr(null), descr=) - setfield_gc(p167, ConstPtr(ptr89), descr=) - i171 = uint_gt(i170, i108) - guard_false(i171, descr=...) - i172 = int_sub(i160, -32768) - i173 = int_and(i172, 65535) - i174 = int_add(i173, -32768) - setarrayitem_raw(i169, 0, i174, descr=) - i175 = int_add(i168, i121) - i176 = int_sub(i160, i130) - i177 = uint_gt(i176, i132) - guard_false(i177, descr=...) - setarrayitem_raw(i175, 0, i174, descr=) - i178 = int_add(i168, i140) - i179 = int_sub(i160, i149) - i180 = uint_gt(i179, i151) - guard_false(i180, descr=...) - setarrayitem_raw(i178, 0, i174, descr=) + setfield_gc(p167, ConstPtr(ptr85), descr=) + i114 = uint_gt(i112, 65535) + guard_false(i114, descr=...) + i115 = int_and(i112, 65535) + i116 = int_add(i115, -32768) --TICK-- - i183 = arraylen_gc(p67, descr=) - i184 = arraylen_gc(p92, descr=) + i119 = call(ConstClass(_ll_1_raw_malloc_varsize__Signed), 6, descr=) + raw_store(i119, 0, i116, descr=) + raw_store(i119, 2, i116, descr=) + raw_store(i119, 4, i116, descr=) + setfield_gc(p167, i119, descr=) + i123 = arraylen_gc(p67, descr=) jump(..., descr=...) """) diff --git a/pypy/module/sys/initpath.py b/pypy/module/sys/initpath.py --- a/pypy/module/sys/initpath.py +++ b/pypy/module/sys/initpath.py @@ -18,6 +18,13 @@ _WIN32 = sys.platform == 'win32' +def _exists_and_is_executable(fn): + # os.access checks using the user's real uid and gid. + # Since pypy should not be run setuid/setgid, this + # should be sufficient. + return os.path.isfile(fn) and os.access(fn, os.X_OK) + + def find_executable(executable): """ Return the absolute path of the executable, by looking into PATH and @@ -34,14 +41,14 @@ if path: for dir in path.split(os.pathsep): fn = os.path.join(dir, executable) - if os.path.isfile(fn): + if _exists_and_is_executable(fn): executable = fn break executable = rpath.rabspath(executable) # 'sys.executable' should not end up being an non-existing file; # just use '' in this case. (CPython issue #7774) - return executable if os.path.isfile(executable) else '' + return executable if _exists_and_is_executable(executable) else '' def _readlink_maybe(filename): diff --git a/pypy/module/sys/test/test_initpath.py b/pypy/module/sys/test/test_initpath.py --- a/pypy/module/sys/test/test_initpath.py +++ b/pypy/module/sys/test/test_initpath.py @@ -57,6 +57,7 @@ a.join('pypy').ensure(file=True) b.join('pypy').ensure(file=True) # + monkeypatch.setattr(os, 'access', lambda x, y: True) # if there is already a slash, don't do anything monkeypatch.chdir(tmpdir) assert find_executable('a/pypy') == a.join('pypy') @@ -82,7 +83,11 @@ # if pypy is found but it's not a file, ignore it c.join('pypy').ensure(dir=True) assert find_executable('pypy') == a.join('pypy') + # if pypy is found but it's not executable, ignore it + monkeypatch.setattr(os, 'access', lambda x, y: False) + assert find_executable('pypy') == '' # + monkeypatch.setattr(os, 'access', lambda x, y: True) monkeypatch.setattr(initpath, 'we_are_translated', lambda: True) monkeypatch.setattr(initpath, '_WIN32', True) monkeypatch.setenv('PATH', str(a)) diff --git a/rpython/jit/backend/arm/callbuilder.py b/rpython/jit/backend/arm/callbuilder.py --- a/rpython/jit/backend/arm/callbuilder.py +++ b/rpython/jit/backend/arm/callbuilder.py @@ -80,15 +80,6 @@ self.mc.gen_load_int(r.ip.value, n) self.mc.SUB_rr(r.sp.value, r.sp.value, r.ip.value) - def _must_remap_fnloc(self): - fnloc = self.fnloc - if fnloc.is_stack(): - return True - if self.is_call_release_gil: - if fnloc is r.r5 or fnloc is r.r6 or fnloc is r.r7: - return True - return False - def call_releasegil_addr_and_move_real_arguments(self, fastgil): assert self.is_call_release_gil assert not self.asm._is_asmgcc() @@ -121,7 +112,7 @@ self.mc.STREX(r.r3.value, r.ip.value, r.r6.value, c=c.EQ) # try to claim the lock self.mc.CMP_ri(r.r3.value, 0, cond=c.EQ) # did this succeed? - self.mc.DMB(c=c.EQ) + self.mc.DMB() # the success of the lock acquisition is defined by # 'EQ is true', or equivalently by 'r3 == 0'. # @@ -182,6 +173,13 @@ class SoftFloatCallBuilder(ARMCallbuilder): + # XXX Maybe we could kill this class and unify the remaining two + # XXX classes, by carefully checking if all methods here are doing + # XXX the exact same thing as the methods from HardFloatCallBuilder, + # XXX but simply forcing all BoxFloat arguments to be longlongs + # XXX (i.e. ignoring 'f' in favour of 'L'), and the same with + # XXX single-float arguments (ignoring 'S' in favour of 'i'); + # XXX and the same for the return value. def get_result_locs(self): if self.resloc is None: @@ -268,7 +266,7 @@ # or on the stack, which we can not access later # If this happens to be the case we remap the register to r4 and use r4 # to call the function - if self.fnloc in r.argument_regs or self._must_remap_fnloc(): + if not self.fnloc.is_imm(): non_float_locs.append(self.fnloc) non_float_regs.append(r.r4) self.fnloc = r.r4 @@ -285,29 +283,23 @@ def get_next_vfp(self, tp): assert tp in 'fS' - if self.next_arg_vfp == -1: - return None - if tp == 'S': + if tp == 'f': + # 64bit double + i = max(self.next_arg_vfp, (self.next_arg_svfp + 1) >> 1) + if i >= len(r.vfp_argument_regs): + self.next_arg_svfp = 1000 # stop that sequence too + return None + self.next_arg_vfp = i + 1 + return r.vfp_argument_regs[i] + else: + # 32bit float i = self.next_arg_svfp - next_vfp = (i >> 1) + 1 - if not (i + 1) & 1: # i is even - self.next_arg_vfp = max(self.next_arg_vfp, next_vfp) - self.next_arg_svfp = self.next_arg_vfp << 1 - else: - self.next_arg_svfp += 1 - self.next_arg_vfp = next_vfp - lst = r.svfp_argument_regs - else: # 64bit double - i = self.next_arg_vfp - self.next_arg_vfp += 1 - if self.next_arg_svfp >> 1 == i: - self.next_arg_svfp = self.next_arg_vfp << 1 - lst = r.vfp_argument_regs - try: - return lst[i] - except IndexError: - self.next_arg_vfp = self.next_arg_svfp = -1 - return None + if not (i & 1): # if i is even + i = max(i, self.next_arg_vfp << 1) + if i >= len(r.svfp_argument_regs): + return None + self.next_arg_svfp = i + 1 + return r.svfp_argument_regs[i] def prepare_arguments(self): non_float_locs = [] @@ -316,34 +308,64 @@ float_regs = [] stack_args = [] singlefloats = None + longlong_mask = 0 arglocs = self.arglocs argtypes = self.argtypes - count = 0 # stack alignment counter + r_register_count = 0 on_stack = 0 + for i in range(len(arglocs)): argtype = INT if i < len(argtypes) and argtypes[i] == 'S': argtype = argtypes[i] arg = arglocs[i] + if arg.is_float(): - argtype = FLOAT - reg = self.get_next_vfp(argtype) - if reg: - assert len(float_regs) < len(r.vfp_argument_regs) - float_locs.append(arg) - assert reg not in float_regs - float_regs.append(reg) - else: # float argument that needs to go on the stack - if count % 2 != 0: - stack_args.append(None) - count = 0 - on_stack += 1 - stack_args.append(arg) - on_stack += 2 + if i < len(argtypes) and argtypes[i] == 'L': + # A longlong argument. It uses two regular argument + # positions, but aligned to an even number. This is + # a bit strange, but it is the case even for registers: + # it can be in r0-r1 or in r2-r3 but not in r1-r2. + assert arg.is_float() + if r_register_count == 0: + # will temporarily load the register into d8 + float_locs.append(arg) + float_regs.append(r.d8) + longlong_mask |= 1 + r_register_count = 2 + continue + elif r_register_count <= 2: + # will temporarily load the register into d9 + float_locs.append(arg) + float_regs.append(r.d9) + longlong_mask |= 2 + r_register_count = 4 + continue + elif r_register_count == 3: + r_register_count = 4 + else: + # A 64-bit float argument. Goes into the next free v# + # register, or if none, to the stack aligned to an + # even number of words. + argtype = FLOAT + reg = self.get_next_vfp(argtype) + if reg: + float_locs.append(arg) + assert reg not in float_regs + float_regs.append(reg) + continue + # float or longlong argument that needs to go on the stack + if on_stack & 1: # odd: realign + stack_args.append(None) + on_stack += 1 + stack_args.append(arg) + on_stack += 2 + elif argtype == 'S': - # Singlefloat argument + # Singlefloat (32-bit) argument. Goes into the next free + # v# register, or if none, to the stack in a single word. if singlefloats is None: singlefloats = [] tgt = self.get_next_vfp(argtype) @@ -351,32 +373,36 @@ singlefloats.append((arg, tgt)) else: # Singlefloat argument that needs to go on the stack # treated the same as a regular core register argument - count += 1 + stack_args.append(arg) on_stack += 1 - stack_args.append(arg) else: - if len(non_float_regs) < len(r.argument_regs): - reg = r.argument_regs[len(non_float_regs)] + # Regular one-word argument. Goes into the next register + # free from the list r0, r1, r2, r3, or to the stack. + if r_register_count < len(r.argument_regs): + reg = r.argument_regs[r_register_count] + r_register_count += 1 non_float_locs.append(arg) non_float_regs.append(reg) else: # non-float argument that needs to go on the stack - count += 1 + stack_args.append(arg) on_stack += 1 - stack_args.append(arg) + # align the stack - if count % 2 != 0: + if on_stack & 1: # odd: realign stack_args.append(None) on_stack += 1 self._push_stack_args(stack_args, on_stack*WORD) + # Check that the address of the function we want to call is not # currently stored in one of the registers used to pass the arguments # or on the stack, which we can not access later # If this happens to be the case we remap the register to r4 and use r4 # to call the function - if self.fnloc in non_float_regs or self._must_remap_fnloc(): + if not self.fnloc.is_imm(): non_float_locs.append(self.fnloc) non_float_regs.append(r.r4) self.fnloc = r.r4 + # remap values stored in vfp registers remap_frame_layout(self.asm, float_locs, float_regs, r.vfp_ip) if singlefloats: @@ -392,13 +418,22 @@ src = r.ip if src.is_core_reg(): self.mc.VMOV_cs(dest.value, src.value) + # remap values stored in core registers remap_frame_layout(self.asm, non_float_locs, non_float_regs, r.ip) + if longlong_mask & 1: + self.mc.FMRRD(r.r0.value, r.r1.value, r.d8.value) + if longlong_mask & 2: + self.mc.FMRRD(r.r2.value, r.r3.value, r.d9.value) + def load_result(self): resloc = self.resloc if self.restype == 'S': self.mc.VMOV_sc(resloc.value, r.s0.value) + elif self.restype == 'L': + assert resloc.is_vfp_reg() + self.mc.FMDRR(resloc.value, r.r0.value, r.r1.value) # ensure the result is wellformed and stored in the correct location if resloc is not None and resloc.is_core_reg(): self._ensure_result_bit_extension(resloc, @@ -408,7 +443,10 @@ if self.resloc is None: return [], [] if self.resloc.is_vfp_reg(): - return [], [r.d0] + if self.restype == 'L': # long long + return [r.r0, r.r1], [] + else: + return [], [r.d0] assert self.resloc.is_core_reg() return [r.r0], [] diff --git a/rpython/jit/backend/arm/codebuilder.py b/rpython/jit/backend/arm/codebuilder.py --- a/rpython/jit/backend/arm/codebuilder.py +++ b/rpython/jit/backend/arm/codebuilder.py @@ -332,13 +332,17 @@ | (rd & 0xF) << 12 | (rn & 0xF) << 16) - def DMB(self, c=cond.AL): - self.write32(c << 28 | 0x157ff05f) + def DMB(self): + # note: 'cond' is only permitted on Thumb here + self.write32(0xf57ff05f) DIV = binary_helper_call('int_div') MOD = binary_helper_call('int_mod') UDIV = binary_helper_call('uint_div') + FMDRR = VMOV_cr # uh, there are synonyms? + FMRRD = VMOV_rc + def _encode_reg_list(self, instr, regs): for reg in regs: instr |= 0x1 << reg diff --git a/rpython/jit/backend/arm/runner.py b/rpython/jit/backend/arm/runner.py --- a/rpython/jit/backend/arm/runner.py +++ b/rpython/jit/backend/arm/runner.py @@ -20,7 +20,7 @@ IS_64_BIT = False supports_floats = True - supports_longlong = False # incomplete, notably in callbuilder.py + supports_longlong = True supports_singlefloats = True from rpython.jit.backend.arm.arch import JITFRAME_FIXED_SIZE diff --git a/rpython/jit/backend/arm/test/test_callbuilder.py b/rpython/jit/backend/arm/test/test_callbuilder.py new file mode 100644 --- /dev/null +++ b/rpython/jit/backend/arm/test/test_callbuilder.py @@ -0,0 +1,47 @@ +from rpython.jit.backend.arm.callbuilder import HardFloatCallBuilder +from rpython.jit.backend.arm import registers as r + + + +def test_hf_vfp_registers_all_singlefloat(): + hf = HardFloatCallBuilder.__new__(HardFloatCallBuilder) + got = [hf.get_next_vfp('S') for i in range(18)] + assert got == [r.s0, r.s1, r.s2, r.s3, r.s4, r.s5, r.s6, r.s7, + r.s8, r.s9, r.s10, r.s11, r.s12, r.s13, r.s14, r.s15, + None, None] + +def test_hf_vfp_registers_all_doublefloat(): + hf = HardFloatCallBuilder.__new__(HardFloatCallBuilder) + got = [hf.get_next_vfp('f') for i in range(10)] + assert got == [r.d0, r.d1, r.d2, r.d3, r.d4, r.d5, r.d6, r.d7, + None, None] + +def test_hf_vfp_registers_mixture(): + hf = HardFloatCallBuilder.__new__(HardFloatCallBuilder) + got = [hf.get_next_vfp('S'), hf.get_next_vfp('f'), + hf.get_next_vfp('S'), hf.get_next_vfp('f'), + hf.get_next_vfp('S'), hf.get_next_vfp('f'), + hf.get_next_vfp('S'), hf.get_next_vfp('f'), + hf.get_next_vfp('S'), hf.get_next_vfp('f'), + hf.get_next_vfp('S'), hf.get_next_vfp('f'), + hf.get_next_vfp('S'), hf.get_next_vfp('f')] + assert got == [r.s0, r.d1, + r.s1, r.d2, + r.s6, r.d4, + r.s7, r.d5, + r.s12, r.d7, + r.s13, None, + None, None] + +def test_hf_vfp_registers_mixture_2(): + hf = HardFloatCallBuilder.__new__(HardFloatCallBuilder) + got = [hf.get_next_vfp('f'), hf.get_next_vfp('f'), + hf.get_next_vfp('f'), hf.get_next_vfp('f'), + hf.get_next_vfp('f'), hf.get_next_vfp('f'), + hf.get_next_vfp('f'), hf.get_next_vfp('S'), + hf.get_next_vfp('f'), hf.get_next_vfp('S')] + assert got == [r.d0, r.d1, + r.d2, r.d3, + r.d4, r.d5, + r.d6, r.s14, + None, None] # <- and not r.s15 for the last item diff --git a/rpython/jit/backend/arm/test/test_instr_codebuilder.py b/rpython/jit/backend/arm/test/test_instr_codebuilder.py --- a/rpython/jit/backend/arm/test/test_instr_codebuilder.py +++ b/rpython/jit/backend/arm/test/test_instr_codebuilder.py @@ -199,6 +199,14 @@ self.cb.DMB() self.assert_equal('DMB') + def test_fmdrr(self): + self.cb.FMDRR(r.d11.value, r.r9.value, r.r14.value) + self.assert_equal('FMDRR d11, r9, r14') + + def test_fmrrd(self): + self.cb.FMRRD(r.r9.value, r.r14.value, r.d11.value) + self.assert_equal('FMRRD r9, r14, d11') + def test_size_of_gen_load_int(): for v, n in [(5, 4), (6, 4), (7, 2)]: diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py --- a/rpython/jit/backend/llsupport/assembler.py +++ b/rpython/jit/backend/llsupport/assembler.py @@ -294,10 +294,16 @@ struct = self.loop_run_counters[i] if struct.type == 'l': prefix = 'TargetToken(%d)' % struct.number - elif struct.type == 'b': - prefix = 'bridge ' + str(struct.number) else: - prefix = 'entry ' + str(struct.number) + num = struct.number + if num == -1: + num = '-1' + else: + num = str(r_uint(num)) + if struct.type == 'b': + prefix = 'bridge %s' % num + else: + prefix = 'entry %s' % num debug_print(prefix + ':' + str(struct.i)) debug_stop('jit-backend-counts') diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py --- a/rpython/jit/backend/test/runner_test.py +++ b/rpython/jit/backend/test/runner_test.py @@ -2718,12 +2718,11 @@ assert r == result def test_call_release_gil_variable_function_and_arguments(self): - # NOTE NOTE NOTE - # This also works as a test for ctypes and libffi. - # On some platforms, one of these is buggy... + from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.rlib.libffi import types from rpython.rlib.rarithmetic import r_uint, r_longlong, r_ulonglong from rpython.rlib.rarithmetic import r_singlefloat + from rpython.translator.c import primitive cpu = self.cpu rnd = random.Random(525) @@ -2752,25 +2751,76 @@ (types.float, rffi.FLOAT), ] * 4 - for k in range(100): + NB_TESTS = 100 + c_source = [] + all_tests = [] + export_symbols = [] + + def prepare_c_source(): + """Pick a random choice of argument types and length, + and build a C function with these arguments. The C + function will simply copy them all into static global + variables. There are then additional functions to fetch + them, one per argument, with a signature 'void(ARG *)'. + """ POSSIBLE_TYPES = [rnd.choice(ALL_TYPES) for i in range(random.randrange(2, 5))] load_factor = rnd.random() keepalive_factor = rnd.random() # - def pseudo_c_function(*args): - seen.append(list(args)) - # ffitypes = [] ARGTYPES = [] for i in range(rnd.randrange(4, 20)): ffitype, TP = rnd.choice(POSSIBLE_TYPES) ffitypes.append(ffitype) ARGTYPES.append(TP) + fn_name = 'vartest%d' % k + all_tests.append((ARGTYPES, ffitypes, fn_name)) # - FPTR = self.Ptr(self.FuncType(ARGTYPES, lltype.Void)) - func_ptr = llhelper(FPTR, pseudo_c_function) - funcbox = self.get_funcbox(cpu, func_ptr) + fn_args = [] + for i, ARG in enumerate(ARGTYPES): + arg_decl = primitive.cdecl(primitive.PrimitiveType[ARG], + 'x%d' % i) + fn_args.append(arg_decl) + var_name = 'argcopy_%s_x%d' % (fn_name, i) + var_decl = primitive.cdecl(primitive.PrimitiveType[ARG], + var_name) + c_source.append('static %s;' % var_decl) + getter_name = '%s_get%d' % (fn_name, i) + export_symbols.append(getter_name) + c_source.append('void %s(%s) { *p = %s; }' % ( + getter_name, + primitive.cdecl(primitive.PrimitiveType[ARG], '*p'), + var_name)) + export_symbols.append(fn_name) + c_source.append('') + c_source.append('static void real%s(%s)' % ( + fn_name, ', '.join(fn_args))) + c_source.append('{') + for i in range(len(ARGTYPES)): + c_source.append(' argcopy_%s_x%d = x%d;' % (fn_name, i, i)) + c_source.append('}') + c_source.append('void *%s(void)' % fn_name) + c_source.append('{') + c_source.append(' return (void *)&real%s;' % fn_name) + c_source.append('}') + c_source.append('') + + for k in range(NB_TESTS): + prepare_c_source() + + eci = ExternalCompilationInfo( + separate_module_sources=['\n'.join(c_source)], + export_symbols=export_symbols) + + for k in range(NB_TESTS): + ARGTYPES, ffitypes, fn_name = all_tests[k] + func_getter_ptr = rffi.llexternal(fn_name, [], lltype.Signed, + compilation_info=eci, _nowrapper=True) + load_factor = rnd.random() + keepalive_factor = rnd.random() + # + func_raw = func_getter_ptr() calldescr = cpu._calldescr_dynamic_for_tests(ffitypes, types.void) faildescr = BasicFailDescr(1) # @@ -2790,7 +2840,7 @@ print print codes # - argvalues = [funcbox.getint()] + argvalues = [func_raw] for TP in ARGTYPES: r = (rnd.random() - 0.5) * 999999999999.9 r = rffi.cast(TP, r) @@ -2840,16 +2890,26 @@ looptoken = JitCellToken() self.cpu.compile_loop(argboxes, ops, looptoken) # - seen = [] deadframe = self.cpu.execute_token(looptoken, *argvalues_normal) fail = self.cpu.get_latest_descr(deadframe) assert fail.identifier == 0 expected = argvalues[1:] - [got] = seen - different_values = ['%r != %r' % (a, b) - for a, b in zip(got, expected) - if a != b] - assert got == expected, ', '.join(different_values) + got = [] + for i, ARG in enumerate(ARGTYPES): + PARG = rffi.CArrayPtr(ARG) + getter_name = '%s_get%d' % (fn_name, i) + getter_ptr = rffi.llexternal(getter_name, [PARG], lltype.Void, + compilation_info=eci, + _nowrapper=True) + my_arg = lltype.malloc(PARG.TO, 1, zero=True, flavor='raw') + getter_ptr(my_arg) + got.append(my_arg[0]) + lltype.free(my_arg, flavor='raw') + different_values = ['x%d: got %r, expected %r' % (i, a, b) + for i, (a, b) in enumerate(zip(got, expected)) + if a != b] + assert got == expected, '\n'.join( + ['bad args, signature %r' % codes[1:]] + different_values) def test_guard_not_invalidated(self): diff --git a/rpython/jit/codewriter/effectinfo.py b/rpython/jit/codewriter/effectinfo.py --- a/rpython/jit/codewriter/effectinfo.py +++ b/rpython/jit/codewriter/effectinfo.py @@ -36,6 +36,7 @@ OS_STREQ_NONNULL_CHAR = 29 # s1 == char (assert s1!=NULL) OS_STREQ_CHECKNULL_CHAR = 30 # s1!=NULL and s1==char OS_STREQ_LENGTHOK = 31 # s1 == s2 (assert len(s1)==len(s2)) + OS_STR_CMP = 32 # "stroruni.cmp" # OS_UNI_CONCAT = 42 # OS_UNI_SLICE = 43 # @@ -47,6 +48,7 @@ OS_UNIEQ_NONNULL_CHAR = 49 # (must be the same amount as for OS_UNIEQ_CHECKNULL_CHAR = 50 # STR, in the same order) OS_UNIEQ_LENGTHOK = 51 # + OS_UNI_CMP = 52 _OS_offset_uni = OS_UNI_CONCAT - OS_STR_CONCAT # OS_LIBFFI_CALL = 62 diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py --- a/rpython/jit/codewriter/jtransform.py +++ b/rpython/jit/codewriter/jtransform.py @@ -1767,6 +1767,7 @@ dict = {"stroruni.concat": EffectInfo.OS_STR_CONCAT, "stroruni.slice": EffectInfo.OS_STR_SLICE, "stroruni.equal": EffectInfo.OS_STR_EQUAL, + "stroruni.cmp": EffectInfo.OS_STR_CMP, "stroruni.copy_string_to_raw": EffectInfo.OS_STR_COPY_TO_RAW, } CHR = lltype.Char @@ -1774,6 +1775,7 @@ dict = {"stroruni.concat": EffectInfo.OS_UNI_CONCAT, "stroruni.slice": EffectInfo.OS_UNI_SLICE, "stroruni.equal": EffectInfo.OS_UNI_EQUAL, + "stroruni.cmp": EffectInfo.OS_UNI_CMP, "stroruni.copy_string_to_raw": EffectInfo.OS_UNI_COPY_TO_RAW } CHR = lltype.UniChar diff --git a/rpython/jit/metainterp/optimizeopt/vstring.py b/rpython/jit/metainterp/optimizeopt/vstring.py --- a/rpython/jit/metainterp/optimizeopt/vstring.py +++ b/rpython/jit/metainterp/optimizeopt/vstring.py @@ -733,6 +733,25 @@ return True return False + def opt_call_stroruni_STR_CMP(self, op, mode): + v1 = self.getvalue(op.getarg(1)) + v2 = self.getvalue(op.getarg(2)) + l1box = v1.getstrlen(None, mode, None) + l2box = v2.getstrlen(None, mode, None) + if (l1box is not None and l2box is not None and + isinstance(l1box, ConstInt) and + isinstance(l2box, ConstInt) and + l1box.value == l2box.value == 1): + # comparing two single chars + vchar1 = self.strgetitem(v1, optimizer.CVAL_ZERO, mode) + vchar2 = self.strgetitem(v2, optimizer.CVAL_ZERO, mode) + seo = self.optimizer.send_extra_operation + seo(ResOperation(rop.INT_SUB, [vchar1.force_box(self), + vchar2.force_box(self)], + op.result)) + return True + return False + def opt_call_SHRINK_ARRAY(self, op): v1 = self.getvalue(op.getarg(1)) v2 = self.getvalue(op.getarg(2)) diff --git a/rpython/jit/metainterp/test/test_string.py b/rpython/jit/metainterp/test/test_string.py --- a/rpython/jit/metainterp/test/test_string.py +++ b/rpython/jit/metainterp/test/test_string.py @@ -846,6 +846,27 @@ 'jump': 1, 'guard_true': 2, 'int_ge': 2, 'int_add': 2, 'int_sub': 2 }) + def test_compare_single_char_for_ordering(self): + jitdriver = JitDriver(reds=['result', 'n'], greens=[]) + _str = self._str + constant1 = _str("abcdefghij") + + def cmpstr(x, y): + return x > _str(y) + + def f(n): + cmpstr(_str("abc"), "def") # force x and y to be annot as strings + result = 0 + while n >= 0: + jitdriver.jit_merge_point(n=n, result=result) + c = constant1[n] + result += cmpstr(c, "c") + n -= 1 + return result + + res = self.meta_interp(f, [9]) + assert res == f(9) + self.check_resops(newstr=0, newunicode=0, call=0) class TestLLtype(StringTests, LLJitMixin): diff --git a/rpython/rlib/rStringIO.py b/rpython/rlib/rStringIO.py --- a/rpython/rlib/rStringIO.py +++ b/rpython/rlib/rStringIO.py @@ -125,19 +125,19 @@ assert result >= 0 return result - def read(self, n=-1): + def read(self, size=-1): p = self.__pos - if p == 0 and n < 0: + if p == 0 and size < 0: self.__pos = AT_END return self.getvalue() # reading everything - if p == AT_END or n == 0: + if p == AT_END or size == 0: return '' assert p >= 0 self.__copy_into_bigbuffer() mysize = len(self.__bigbuffer) count = mysize - p - if n >= 0: - count = min(n, count) + if size >= 0: + count = min(size, count) if count <= 0: return '' if p == 0 and count == mysize: diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py --- a/rpython/rtyper/lltypesystem/rstr.py +++ b/rpython/rtyper/lltypesystem/rstr.py @@ -531,6 +531,7 @@ return diff i += 1 return len1 - len2 + ll_strcmp.oopspec = 'stroruni.cmp(s1, s2)' @jit.elidable def ll_streq(s1, s2): diff --git a/rpython/rtyper/rstr.py b/rpython/rtyper/rstr.py --- a/rpython/rtyper/rstr.py +++ b/rpython/rtyper/rstr.py @@ -3,6 +3,7 @@ from rpython.rtyper import rint from rpython.rtyper.error import TyperError from rpython.rtyper.lltypesystem.lltype import Signed, Bool, Void, UniChar +from rpython.rtyper.lltypesystem import lltype from rpython.rtyper.rmodel import IteratorRepr, inputconst, Repr from rpython.rtyper.rint import IntegerRepr from rpython.rtyper.rfloat import FloatRepr @@ -384,10 +385,10 @@ unicode_encode_utf_8_impl, 'runicode_encode_utf_8') def rtype_method_upper(self, hop): - raise TypeError("Cannot do toupper on unicode string") + raise TyperError("Cannot do toupper on unicode string") def rtype_method_lower(self, hop): - raise TypeError("Cannot do tolower on unicode string") + raise TyperError("Cannot do tolower on unicode string") @jit.elidable def ll_encode_utf8(self, ll_s): @@ -711,6 +712,11 @@ pairtype(AbstractUniCharRepr, AbstractCharRepr)): def rtype_eq(_, hop): return _rtype_unchr_compare_template(hop, 'eq') def rtype_ne(_, hop): return _rtype_unchr_compare_template(hop, 'ne') + def rtype_lt(_, hop): return _rtype_unchr_compare_template_ord(hop, 'lt') + def rtype_le(_, hop): return _rtype_unchr_compare_template_ord(hop, 'le') + def rtype_gt(_, hop): return _rtype_unchr_compare_template_ord(hop, 'gt') + def rtype_ge(_, hop): return _rtype_unchr_compare_template_ord(hop, 'ge') + #Helper functions for comparisons @@ -719,6 +725,18 @@ vlist = hop.inputargs(unichar_repr, unichar_repr) return hop.genop('unichar_' + func, vlist, resulttype=Bool) +def _rtype_unchr_compare_template_ord(hop, func): + vlist = hop.inputargs(*hop.args_r) + vlist2 = [] + for v in vlist: + if v.concretetype == lltype.Char: + v = hop.genop('cast_char_to_int', [v], resulttype=lltype.Signed) + elif v.concretetype == lltype.UniChar: + v = hop.genop('cast_unichar_to_int', [v], resulttype=lltype.Signed) + else: + assert 0, v.concretetype + vlist2.append(v) + return hop.genop('int_' + func, vlist2, resulttype=Bool) # # _________________________ Conversions _________________________ diff --git a/rpython/rtyper/test/test_runicode.py b/rpython/rtyper/test/test_runicode.py --- a/rpython/rtyper/test/test_runicode.py +++ b/rpython/rtyper/test/test_runicode.py @@ -296,3 +296,13 @@ res = self.interpret(f, [5]) assert res == 0 + + def test_unicode_char_comparison(self): + const = u'abcdef' + def f(n): + return const[n] >= u'c' + + res = self.interpret(f, [1]) + assert res == False + res = self.interpret(f, [2]) + assert res == True From noreply at buildbot.pypy.org Mon Sep 1 23:04:53 2014 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 1 Sep 2014 23:04:53 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: Handle the uninitialized value better while translating (it can happen) Message-ID: <20140901210453.377071C025B@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73278:5c7aa94e6c89 Date: 2014-09-01 15:04 -0600 http://bitbucket.org/pypy/pypy/changeset/5c7aa94e6c89/ Log: Handle the uninitialized value better while translating (it can happen) diff --git a/rpython/memory/test/test_transformed_gc.py b/rpython/memory/test/test_transformed_gc.py --- a/rpython/memory/test/test_transformed_gc.py +++ b/rpython/memory/test/test_transformed_gc.py @@ -1331,7 +1331,7 @@ run = self.runner('malloc_array_of_gcptr') res = run([]) assert not res - ''' + def define_malloc_struct_of_gcptr(cls): S1 = lltype.GcStruct('S', ('x', lltype.Signed)) S = lltype.GcStruct('S', @@ -1347,25 +1347,7 @@ run = self.runner("malloc_struct_of_gcptr") res = run([]) assert res - ''' - ''' - def define_malloc_struct_of_gcptr(cls): - S = lltype.GcForwardReference() - S.become(lltype.GcStruct('S', - ('x', lltype.Signed), - ('prev', lltype.Ptr(S)), - ('next', lltype.Ptr(S)))) - s0 = lltype.malloc(S,zero = False) - def f(): - return s0.next == lltype.nullptr(S) - return f - def test_malloc_struct_of_gcptr(self): - run = self.runner("malloc_struct_of_gcptr") - pdb.set_trace() - res = run([]) - assert res - ''' # ________________________________________________________________ # tagged pointers diff --git a/rpython/translator/c/database.py b/rpython/translator/c/database.py --- a/rpython/translator/c/database.py +++ b/rpython/translator/c/database.py @@ -286,6 +286,8 @@ for value in newdependencies: #if isinstance(value, _uninitialized): # continue + if isinstance(value, lltype._uninitialized): + continue if isinstance(typeOf(value), ContainerType): node = self.getcontainernode(value) if parent and node._funccodegen_owner is not None: diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py --- a/rpython/translator/c/test/test_newgc.py +++ b/rpython/translator/c/test/test_newgc.py @@ -1381,20 +1381,29 @@ for length in range(3, 76, 5)]) def define_nursery_hash_base(cls): + from rpython.rlib.debug import debug_print + class A: pass def fn(): objects = [] hashes = [] for i in range(200): + debug_print("starting nursery collection", i) rgc.collect(0) # nursery-only collection, if possible + debug_print("finishing nursery collection", i) obj = A() objects.append(obj) hashes.append(compute_identity_hash(obj)) unique = {} + debug_print("objects", len(objects)) for i in range(len(objects)): + debug_print(i) assert compute_identity_hash(objects[i]) == hashes[i] + debug_print("storing in dict") unique[hashes[i]] = None + debug_print("done") + debug_print("finished") return len(unique) return fn From noreply at buildbot.pypy.org Mon Sep 1 23:11:05 2014 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 1 Sep 2014 23:11:05 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: fix some tests Message-ID: <20140901211105.568601C025B@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73279:b3beb2273ee2 Date: 2014-09-01 15:10 -0600 http://bitbucket.org/pypy/pypy/changeset/b3beb2273ee2/ Log: fix some tests diff --git a/rpython/memory/test/test_transformed_gc.py b/rpython/memory/test/test_transformed_gc.py --- a/rpython/memory/test/test_transformed_gc.py +++ b/rpython/memory/test/test_transformed_gc.py @@ -1319,7 +1319,7 @@ S = lltype.GcStruct('S', ('x', lltype.Signed)) A = lltype.GcArray(lltype.Ptr(S)) def f(): - lst = lltype.malloc(A, 5, zero= False) + lst = lltype.malloc(A, 5) return (lst[0] == lltype.nullptr(S) and lst[1] == lltype.nullptr(S) and lst[2] == lltype.nullptr(S) @@ -1330,7 +1330,7 @@ def test_malloc_array_of_gcptr(self): run = self.runner('malloc_array_of_gcptr') res = run([]) - assert not res + assert res def define_malloc_struct_of_gcptr(cls): S1 = lltype.GcStruct('S', ('x', lltype.Signed)) From noreply at buildbot.pypy.org Tue Sep 2 11:43:37 2014 From: noreply at buildbot.pypy.org (groggi) Date: Tue, 2 Sep 2014 11:43:37 +0200 (CEST) Subject: [pypy-commit] pypy gc-incminimark-pinning: don't allow objects with weakrefs to be pinned Message-ID: <20140902094337.1714D1C072F@cobra.cs.uni-duesseldorf.de> Author: Gregor Wegberg Branch: gc-incminimark-pinning Changeset: r73280:109d47eb6e54 Date: 2014-09-02 11:38 +0200 http://bitbucket.org/pypy/pypy/changeset/109d47eb6e54/ Log: don't allow objects with weakrefs to be pinned diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -970,16 +970,21 @@ # makes no sense. If you run into this case, you may forgot # to check can_move(obj). return False - if self.has_gcptr(self.get_type_id(obj)): + if self._is_pinned(obj): + # already pinned, we do not allow to pin it again. + # Reason: It would be possible that the first caller unpins + # while the second caller thinks it's still pinned. + return False + # + obj_type_id = self.get_type_id(obj) + if self.has_gcptr(obj_type_id): # objects containing GC pointers can't be pinned. If we would add # it, we would have to track all pinned objects and trace them # every minor collection to make sure the referenced object are # kept alive. Right now this is not a use case that's needed. return False - if self._is_pinned(obj): - # already pinned, we do not allow to pin it again. - # Reason: It would be possible that the first caller unpins - # while the second caller thinks it's still pinned. + if self.weakpointer_offset(obj_type_id) >= 0: + # for now we don't support pinning objects with weak pointers. return False # self.header(obj).tid |= GCFLAG_PINNED diff --git a/rpython/memory/test/test_incminimark_gc.py b/rpython/memory/test/test_incminimark_gc.py --- a/rpython/memory/test/test_incminimark_gc.py +++ b/rpython/memory/test/test_incminimark_gc.py @@ -1,5 +1,6 @@ from rpython.rtyper.lltypesystem import lltype from rpython.rtyper.lltypesystem.lloperation import llop +from rpython.rlib import rgc from rpython.memory.test import test_minimark_gc @@ -37,6 +38,16 @@ res = self.interpret(f, []) assert res == True + def test_pin_weakref_not_implemented(self): + import weakref + class A: + pass + def f(): + a = A() + ref = weakref.ref(a) + assert not rgc.pin(ref) + self.interpret(f, []) + def test_weakref_to_pinned(self): import weakref from rpython.rlib import rgc From noreply at buildbot.pypy.org Tue Sep 2 15:42:25 2014 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 2 Sep 2014 15:42:25 +0200 (CEST) Subject: [pypy-commit] pypy default: backout 62f4648aed86 Message-ID: <20140902134225.C71271C0EE4@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73281:b7ef31a82744 Date: 2014-09-02 15:41 +0200 http://bitbucket.org/pypy/pypy/changeset/b7ef31a82744/ Log: backout 62f4648aed86 add an explanation and a link to the full commit for why diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py --- a/rpython/rlib/rfile.py +++ b/rpython/rlib/rfile.py @@ -60,8 +60,11 @@ c_setvbuf = llexternal('setvbuf', [FILEP, rffi.CCHARP, rffi.INT, rffi.SIZE_T], rffi.INT) -c_fclose = llexternal('fclose', [FILEP], rffi.INT) -c_pclose = llexternal('pclose', [FILEP], rffi.INT) +# Note: the following two functions are called from __del__ methods, +# so must be 'releasegil=False'. Otherwise, a program using both +# threads and the RFile class cannot translate. See c684bf704d1f +c_fclose = llexternal('fclose', [FILEP], rffi.INT, releasegil=False) +c_pclose = llexternal('pclose', [FILEP], rffi.INT, releasegil=False) c_getc = llexternal('getc', [FILEP], rffi.INT, macro=True) c_fgets = llexternal('fgets', [rffi.CCHARP, rffi.INT, FILEP], rffi.CCHARP) From noreply at buildbot.pypy.org Tue Sep 2 16:05:00 2014 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 2 Sep 2014 16:05:00 +0200 (CEST) Subject: [pypy-commit] pypy default: From RFile.close(), call the C function fclose() or pclose() by Message-ID: <20140902140500.6D9061C0EE4@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73282:fd7c85c9370f Date: 2014-09-02 16:04 +0200 http://bitbucket.org/pypy/pypy/changeset/fd7c85c9370f/ Log: From RFile.close(), call the C function fclose() or pclose() by releasing the GIL. But from RFile.__del__(), we can't and don't release the GIL, so we use a different pair of llexternal functions. diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py --- a/rpython/rlib/rfile.py +++ b/rpython/rlib/rfile.py @@ -60,11 +60,16 @@ c_setvbuf = llexternal('setvbuf', [FILEP, rffi.CCHARP, rffi.INT, rffi.SIZE_T], rffi.INT) +c_fclose = llexternal('fclose', [FILEP], rffi.INT) +c_pclose = llexternal('pclose', [FILEP], rffi.INT) + # Note: the following two functions are called from __del__ methods, # so must be 'releasegil=False'. Otherwise, a program using both # threads and the RFile class cannot translate. See c684bf704d1f -c_fclose = llexternal('fclose', [FILEP], rffi.INT, releasegil=False) -c_pclose = llexternal('pclose', [FILEP], rffi.INT, releasegil=False) +c_fclose_in_del = llexternal('fclose', [FILEP], rffi.INT, releasegil=False) +c_pclose_in_del = llexternal('pclose', [FILEP], rffi.INT, releasegil=False) +_fclose2 = (c_fclose, c_fclose_in_del) +_pclose2 = (c_pclose, c_pclose_in_del) c_getc = llexternal('getc', [FILEP], rffi.INT, macro=True) c_fgets = llexternal('fgets', [rffi.CCHARP, rffi.INT, FILEP], rffi.CCHARP) @@ -183,16 +188,23 @@ lltype.free(ll_type, flavor='raw') finally: lltype.free(ll_command, flavor='raw') - return RFile(ll_file, c_pclose) + return RFile(ll_file, _pclose2) class RFile(object): - def __init__(self, ll_file, do_close=c_fclose): + def __init__(self, ll_file, close2=_fclose2): self._ll_file = ll_file - self._do_close = do_close + self._close2 = close2 def __del__(self): - self.close() + """Closes the described file when the object's last reference + goes away. Unlike an explicit call to close(), this is meant + as a last-resort solution and cannot release the GIL or return + an error code.""" + ll_file = self._ll_file + if ll_file: + do_close = self._close2[1] + do_close(ll_file) # return value ignored def close(self): """Closes the described file. @@ -207,7 +219,8 @@ if ll_file: # double close is allowed self._ll_file = lltype.nullptr(FILEP.TO) - res = self._do_close(ll_file) + do_close = self._close2[0] + res = do_close(ll_file) if res == -1: errno = rposix.get_errno() raise OSError(errno, os.strerror(errno)) diff --git a/rpython/rlib/test/test_rfile.py b/rpython/rlib/test/test_rfile.py --- a/rpython/rlib/test/test_rfile.py +++ b/rpython/rlib/test/test_rfile.py @@ -1,4 +1,4 @@ -import os, sys, py, errno +import os, sys, py, errno, gc from rpython.rtyper.test.tool import BaseRtypingTest from rpython.tool.udir import udir from rpython.rlib import rfile @@ -267,6 +267,18 @@ cls.tmpdir = udir.join('test_rfile_direct') cls.tmpdir.ensure(dir=True) + def test_auto_close(self): + fname = str(self.tmpdir.join('file_auto_close')) + f = rfile.create_file(fname, 'w') + f.write('a') # remains in buffers + assert os.path.getsize(fname) == 0 + del f + for i in range(5): + if os.path.getsize(fname) != 0: + break + gc.collect() + assert os.path.getsize(fname) == 1 + def test_read_a_lot(self): fname = str(self.tmpdir.join('file_read_a_lot')) with open(fname, 'w') as f: From noreply at buildbot.pypy.org Tue Sep 2 17:21:38 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Tue, 2 Sep 2014 17:21:38 +0200 (CEST) Subject: [pypy-commit] stmgc default: yes, bear with us... Message-ID: <20140902152138.533AC1C072F@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1330:0afeb56953d5 Date: 2014-09-02 17:22 +0200 http://bitbucket.org/pypy/stmgc/changeset/0afeb56953d5/ Log: yes, bear with us... diff --git a/c8/doc/high-level-c8.pdf b/c8/doc/high-level-c8.pdf new file mode 100644 index 0000000000000000000000000000000000000000..12cc5dc631834f1dabfe1b533c2dcb01f9a79c74 GIT binary patch [cut] From noreply at buildbot.pypy.org Tue Sep 2 20:33:58 2014 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 2 Sep 2014 20:33:58 +0200 (CEST) Subject: [pypy-commit] pypy default: Fix test, and improve it to check that a non-executable fake exe is not Message-ID: <20140902183358.6C68F1D2577@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73283:912dd9df99a8 Date: 2014-09-02 20:33 +0200 http://bitbucket.org/pypy/pypy/changeset/912dd9df99a8/ Log: Fix test, and improve it to check that a non-executable fake exe is not picked up any more, but an executable one is (after issue #1856). diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py --- a/pypy/interpreter/test/test_app_main.py +++ b/pypy/interpreter/test/test_app_main.py @@ -985,6 +985,11 @@ assert sys.path == old_sys_path + [self.goal_dir] app_main.setup_bootstrap_path(self.fake_exe) + assert sys.executable == '' # not executable! + assert sys.path == old_sys_path + [self.goal_dir] + + os.chmod(self.fake_exe, 0755) + app_main.setup_bootstrap_path(self.fake_exe) assert sys.executable == self.fake_exe assert self.goal_dir not in sys.path From noreply at buildbot.pypy.org Wed Sep 3 10:02:25 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Wed, 3 Sep 2014 10:02:25 +0200 (CEST) Subject: [pypy-commit] stmgc default: update hgignore Message-ID: <20140903080225.629431C072F@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1331:56d393181533 Date: 2014-09-02 17:28 +0200 http://bitbucket.org/pypy/stmgc/changeset/56d393181533/ Log: update hgignore diff --git a/.hgignore b/.hgignore --- a/.hgignore +++ b/.hgignore @@ -7,3 +7,4 @@ *.orig */__pycache__ *.out.* +*/\#*\# From noreply at buildbot.pypy.org Wed Sep 3 10:02:26 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Wed, 3 Sep 2014 10:02:26 +0200 (CEST) Subject: [pypy-commit] stmgc default: WIP Message-ID: <20140903080226.9AFD51C072F@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1332:e250803b835e Date: 2014-09-03 09:54 +0200 http://bitbucket.org/pypy/stmgc/changeset/e250803b835e/ Log: WIP diff too long, truncating to 2000 out of 2893 lines diff --git a/.hgignore b/.hgignore --- a/.hgignore +++ b/.hgignore @@ -8,3 +8,4 @@ */__pycache__ *.out.* */\#*\# +*/.\#* diff --git a/c8/stm/atomic.h b/c8/stm/atomic.h new file mode 100644 --- /dev/null +++ b/c8/stm/atomic.h @@ -0,0 +1,47 @@ +#ifndef _STM_ATOMIC_H +#define _STM_ATOMIC_H + +/* spin_loop() corresponds to the PAUSE instruction on x86. On + other architectures, we generate no instruction (but still need + the compiler barrier); if on another architecture you find the + corresponding instruction, feel free to add it here. +*/ + +/* write_fence() is a function that inserts a "write fence". The + goal is to make sure that past writes are really pushed to memory + before the future writes. We assume that the corresponding "read + fence" effect is done automatically by a corresponding + __sync_bool_compare_and_swap(). + + On x86, this is done automatically by the CPU; we only need a + compiler barrier (asm("memory")). + + On other architectures, we use __sync_synchronize() as a general + fall-back, but we might have more efficient alternative on some other + platforms too. +*/ + + +#if defined(__i386__) || defined(__amd64__) + +# define HAVE_FULL_EXCHANGE_INSN + static inline void spin_loop(void) { asm("pause" : : : "memory"); } + static inline void write_fence(void) { asm("" : : : "memory"); } + +#else + + static inline void spin_loop(void) { asm("" : : : "memory"); } + static inline void write_fence(void) { __sync_synchronize(); } + +#endif + + +#define spinlock_acquire(lock) \ + do { if (LIKELY(__sync_lock_test_and_set(&(lock), 1) == 0)) break; \ + spin_loop(); } while (1) +#define spinlock_release(lock) \ + do { assert((lock) == 1); \ + __sync_lock_release(&(lock)); } while (0) + + +#endif /* _STM_ATOMIC_H */ diff --git a/c8/stm/core.c b/c8/stm/core.c new file mode 100644 --- /dev/null +++ b/c8/stm/core.c @@ -0,0 +1,101 @@ +#ifndef _STM_CORE_H_ +# error "must be compiled via stmgc.c" +#endif + + +void _stm_write_slowpath(object_t *obj) +{ + +} + +static void reset_transaction_read_version(void) +{ + /* force-reset all read markers to 0 */ + + char *readmarkers = REAL_ADDRESS(STM_SEGMENT->segment_base, + FIRST_READMARKER_PAGE * 4096UL); + dprintf(("reset_transaction_read_version: %p %ld\n", readmarkers, + (long)(NB_READMARKER_PAGES * 4096UL))); + if (mmap(readmarkers, NB_READMARKER_PAGES * 4096UL, + PROT_READ | PROT_WRITE, + MAP_FIXED | MAP_PAGES_FLAGS, -1, 0) != readmarkers) { + /* fall-back */ +#if STM_TESTS + stm_fatalerror("reset_transaction_read_version: %m"); +#endif + memset(readmarkers, 0, NB_READMARKER_PAGES * 4096UL); + } + STM_SEGMENT->transaction_read_version = 1; +} + + +static void _stm_start_transaction(stm_thread_local_t *tl, bool inevitable) +{ + assert(!_stm_in_transaction(tl)); + + retry: + + if (!acquire_thread_segment(tl)) + goto retry; + /* GS invalid before this point! */ + +#ifndef NDEBUG + STM_PSEGMENT->running_pthread = pthread_self(); +#endif + + dprintf(("start_transaction\n")); + + s_mutex_unlock(); + + uint8_t old_rv = STM_SEGMENT->transaction_read_version; + STM_SEGMENT->transaction_read_version = old_rv + 1; + if (UNLIKELY(old_rv == 0xff)) { + reset_transaction_read_version(); + } + + assert(list_is_empty(STM_PSEGMENT->modified_old_objects)); + check_nursery_at_transaction_start(); +} + +long stm_start_transaction(stm_thread_local_t *tl) +{ + s_mutex_lock(); +#ifdef STM_NO_AUTOMATIC_SETJMP + long repeat_count = 0; /* test/support.py */ +#else + long repeat_count = stm_rewind_jmp_setjmp(tl); +#endif + _stm_start_transaction(tl, false); + return repeat_count; +} + + +/************************************************************/ + +static void _finish_transaction() +{ + stm_thread_local_t *tl = STM_SEGMENT->running_thread; + release_thread_segment(tl); + /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */ +} + +void stm_commit_transaction(void) +{ + assert(!_has_mutex()); + assert(STM_PSEGMENT->running_pthread == pthread_self()); + + minor_collection(1); + abort(); + + s_mutex_lock(); + + assert(STM_SEGMENT->nursery_end == NURSERY_END); + stm_rewind_jmp_forget(STM_SEGMENT->running_thread); + + + /* done */ + _finish_transaction(); + /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */ + + s_mutex_unlock(); +} diff --git a/c8/stm/core.h b/c8/stm/core.h new file mode 100644 --- /dev/null +++ b/c8/stm/core.h @@ -0,0 +1,97 @@ +#define _STM_CORE_H_ + +#include +#include +#include +#include +#include +#include + +/************************************************************/ + +#ifndef STM_GC_NURSERY +# define STM_GC_NURSERY 4096 // 4MB +#endif + + +#define NB_PAGES (2500*256) // 2500MB +#define NB_SEGMENTS STM_NB_SEGMENTS +#define NB_SEGMENTS_MAX 240 /* don't increase NB_SEGMENTS past this */ +#define MAP_PAGES_FLAGS (MAP_SHARED | MAP_ANONYMOUS | MAP_NORESERVE) +#define NB_NURSERY_PAGES (STM_GC_NURSERY/4) + +#define TOTAL_MEMORY (NB_PAGES * 4096UL * (1 + NB_SEGMENTS)) +#define READMARKER_END ((NB_PAGES * 4096UL) >> 4) +#define FIRST_OBJECT_PAGE ((READMARKER_END + 4095) / 4096UL) +#define FIRST_NURSERY_PAGE FIRST_OBJECT_PAGE +#define END_NURSERY_PAGE (FIRST_NURSERY_PAGE + NB_NURSERY_PAGES) + +#define READMARKER_START ((FIRST_OBJECT_PAGE * 4096UL) >> 4) +#define FIRST_READMARKER_PAGE (READMARKER_START / 4096UL) +#define OLD_RM_START ((END_NURSERY_PAGE * 4096UL) >> 4) +#define FIRST_OLD_RM_PAGE (OLD_RM_START / 4096UL) +#define NB_READMARKER_PAGES (FIRST_OBJECT_PAGE - FIRST_READMARKER_PAGE) + + +#define USE_REMAP_FILE_PAGES 1 + +enum /* stm_flags */ { + GCFLAG_WRITE_BARRIER = _STM_GCFLAG_WRITE_BARRIER, +}; + + +/************************************************************/ + + +#define STM_PSEGMENT ((stm_priv_segment_info_t *)STM_SEGMENT) + +typedef TLPREFIX struct stm_priv_segment_info_s stm_priv_segment_info_t; + +struct stm_priv_segment_info_s { + struct stm_segment_info_s pub; + + struct list_s *modified_old_objects; + + /* For debugging */ +#ifndef NDEBUG + pthread_t running_pthread; +#endif +}; + + +static char *stm_object_pages; +static int stm_object_pages_fd; +static stm_thread_local_t *stm_all_thread_locals = NULL; + + +#define REAL_ADDRESS(segment_base, src) ((segment_base) + (uintptr_t)(src)) + + +static inline char *get_segment_base(long segment_num) { + return stm_object_pages + segment_num * (NB_PAGES * 4096UL); +} + +static inline +struct stm_segment_info_s *get_segment(long segment_num) { + return (struct stm_segment_info_s *)REAL_ADDRESS( + get_segment_base(segment_num), STM_PSEGMENT); +} + +static inline +struct stm_priv_segment_info_s *get_priv_segment(long segment_num) { + return (struct stm_priv_segment_info_s *)REAL_ADDRESS( + get_segment_base(segment_num), STM_PSEGMENT); +} + +static bool _is_tl_registered(stm_thread_local_t *tl); +static bool _seems_to_be_running_transaction(void); + + +static inline void _duck(void) { + /* put a call to _duck() between two instructions that set 0 into + a %gs-prefixed address and that may otherwise be replaced with + llvm.memset --- it fails later because of the prefix... + This is not needed any more after applying the patch + llvmfix/no-memset-creation-with-addrspace.diff. */ + asm("/* workaround for llvm bug */"); +} diff --git a/c8/stm/fprintcolor.c b/c8/stm/fprintcolor.c new file mode 100644 --- /dev/null +++ b/c8/stm/fprintcolor.c @@ -0,0 +1,48 @@ +/* ------------------------------------------------------------ */ +#ifdef STM_DEBUGPRINT +/* ------------------------------------------------------------ */ + + +static int threadcolor_printf(const char *format, ...) +{ + char buffer[2048]; + va_list ap; + int result; + int size = (int)sprintf(buffer, "\033[%dm[%d,%lx] ", dprintfcolor(), + (int)getpid(), (long)pthread_self()); + assert(size >= 0); + + va_start(ap, format); + result = vsnprintf(buffer + size, 2000, format, ap); + assert(result >= 0); + va_end(ap); + + strcpy(buffer + size + result, "\033[0m"); + fputs(buffer, stderr); + + return result; +} + + +/* ------------------------------------------------------------ */ +#endif +/* ------------------------------------------------------------ */ + + +static void stm_fatalerror(const char *format, ...) +{ + va_list ap; + +#ifdef STM_DEBUGPRINT + dprintf(("STM Subsystem: Fatal Error\n")); +#else + fprintf(stderr, "STM Subsystem: Fatal Error\n"); +#endif + + va_start(ap, format); + vfprintf(stderr, format, ap); + fprintf(stderr, "\n"); + va_end(ap); + + abort(); +} diff --git a/c8/stm/fprintcolor.h b/c8/stm/fprintcolor.h new file mode 100644 --- /dev/null +++ b/c8/stm/fprintcolor.h @@ -0,0 +1,41 @@ +/* ------------------------------------------------------------ */ +#ifdef STM_DEBUGPRINT +/* ------------------------------------------------------------ */ + + +#include + + +#define dprintf(args) threadcolor_printf args +static inline int dprintfcolor(void) +{ + return 31 + (STM_SEGMENT->segment_num + 5) % 6; +} + +static int threadcolor_printf(const char *format, ...) + __attribute__((format (printf, 1, 2))); + +#ifdef STM_TESTS +# define dprintf_test(args) dprintf(args) +#else +# define dprintf_test(args) do { } while(0) +#endif + + +/* ------------------------------------------------------------ */ +#else +/* ------------------------------------------------------------ */ + + +#define dprintf(args) do { } while(0) +#define dprintf_test(args) do { } while(0) +#define dprintfcolor() 0 + + +/* ------------------------------------------------------------ */ +#endif +/* ------------------------------------------------------------ */ + + +static void stm_fatalerror(const char *format, ...) + __attribute__((format (printf, 1, 2), noreturn)); diff --git a/c8/stm/list.c b/c8/stm/list.c new file mode 100644 --- /dev/null +++ b/c8/stm/list.c @@ -0,0 +1,180 @@ +#ifndef _STM_CORE_H_ +# error "must be compiled via stmgc.c" +#endif + + +#define LIST_SETSIZE(n) (sizeof(struct list_s) + LIST_ITEMSSIZE(n)) +#define LIST_ITEMSSIZE(n) ((n) * sizeof(uintptr_t)) +#define LIST_OVERCNT(n) (33 + ((((n) / 2) * 3) | 1)) + +static struct list_s *list_create(void) +{ + uintptr_t initial_allocation = 32; + struct list_s *lst = malloc(LIST_SETSIZE(initial_allocation)); + if (lst == NULL) + stm_fatalerror("out of memory in list_create"); /* XXX */ + + lst->count = 0; + lst->last_allocated = initial_allocation - 1; + return lst; +} + +static struct list_s *_list_grow(struct list_s *lst, uintptr_t nalloc) +{ + nalloc = LIST_OVERCNT(nalloc); + lst = realloc(lst, LIST_SETSIZE(nalloc)); + if (lst == NULL) + stm_fatalerror("out of memory in _list_grow"); /* XXX */ + + lst->last_allocated = nalloc - 1; + return lst; +} + + +/************************************************************/ + +static void _tree_clear_node(wlog_node_t *node) +{ + memset(node, 0, sizeof(wlog_node_t)); +} + +static void tree_clear(struct tree_s *tree) +{ + if (tree->raw_current != tree->raw_start) { + _tree_clear_node(&tree->toplevel); + tree->raw_current = tree->raw_start; + } +} + +static struct tree_s *tree_create(void) +{ + return (struct tree_s *)calloc(1, sizeof(struct tree_s)); +} + +static void tree_free(struct tree_s *tree) +{ + free(tree->raw_start); + assert(memset(tree, 0xDD, sizeof(struct tree_s))); + free(tree); +} + +static void _tree_compress(struct tree_s *tree) +{ + wlog_t *item; + struct tree_s tree_copy; + memset(&tree_copy, 0, sizeof(struct tree_s)); + + TREE_LOOP_FORWARD(*tree, item) { + tree_insert(&tree_copy, item->addr, item->val); + + } TREE_LOOP_END; + + free(tree->raw_start); + *tree = tree_copy; +} + +static wlog_t *_tree_find(char *entry, uintptr_t addr) +{ + uintptr_t key = TREE_HASH(addr); + while (((long)entry) & 1) { + /* points to a further level */ + key >>= TREE_BITS; + entry = *(char **)((entry - 1) + (key & TREE_MASK)); + } + return (wlog_t *)entry; /* may be NULL */ +} + +static void _tree_grow(struct tree_s *tree, long extra) +{ + struct tree_s newtree; + wlog_t *item; + long alloc = tree->raw_end - tree->raw_start; + long newalloc = (alloc + extra + (alloc >> 2) + 31) & ~15; + //fprintf(stderr, "growth: %ld\n", newalloc); + char *newitems = malloc(newalloc); + if (newitems == NULL) { + stm_fatalerror("out of memory!"); /* XXX */ + } + newtree.raw_start = newitems; + newtree.raw_current = newitems; + newtree.raw_end = newitems + newalloc; + _tree_clear_node(&newtree.toplevel); + TREE_LOOP_FORWARD(*tree, item) + { + tree_insert(&newtree, item->addr, item->val); + } TREE_LOOP_END; + free(tree->raw_start); + *tree = newtree; +} + +static char *_tree_grab(struct tree_s *tree, long size) +{ + char *result; + result = tree->raw_current; + tree->raw_current += size; + if (tree->raw_current > tree->raw_end) { + _tree_grow(tree, size); + return NULL; + } + return result; +} + +static void tree_insert(struct tree_s *tree, uintptr_t addr, uintptr_t val) +{ + assert(addr != 0); /* the NULL key is reserved */ + retry:; + wlog_t *wlog; + uintptr_t key = TREE_HASH(addr); + int shift = 0; + char *p = (char *)(tree->toplevel.items); + char *entry; + while (1) { + assert(shift < TREE_DEPTH_MAX * TREE_BITS); + p += (key >> shift) & TREE_MASK; + shift += TREE_BITS; + entry = *(char **)p; + if (entry == NULL) + break; + else if (((long)entry) & 1) { + /* points to a further level */ + p = entry - 1; + } + else { + wlog_t *wlog1 = (wlog_t *)entry; + if (wlog1->addr == 0) { + /* reuse the deleted entry and that's it */ + wlog1->addr = addr; + wlog1->val = val; + return; + } + /* the key must not already be present */ + assert(wlog1->addr != addr); + /* collision: there is already a different wlog here */ + wlog_node_t *node = (wlog_node_t *) + _tree_grab(tree, sizeof(wlog_node_t)); + if (node == NULL) goto retry; + _tree_clear_node(node); + uintptr_t key1 = TREE_HASH(wlog1->addr); + char *p1 = (char *)(node->items); + *(wlog_t **)(p1 + ((key1 >> shift) & TREE_MASK)) = wlog1; + *(char **)p = ((char *)node) + 1; + p = p1; + } + } + wlog = (wlog_t *)_tree_grab(tree, sizeof(wlog_t)); + if (wlog == NULL) goto retry; + wlog->addr = addr; + wlog->val = val; + *(char **)p = (char *)wlog; +} + +static bool tree_delete_item(struct tree_s *tree, uintptr_t addr) +{ + wlog_t *entry; + TREE_FIND(*tree, addr, entry, goto missing); + entry->addr = 0; + return true; + + missing: + return false; +} diff --git a/c8/stm/list.h b/c8/stm/list.h new file mode 100644 --- /dev/null +++ b/c8/stm/list.h @@ -0,0 +1,217 @@ +#include +#include + +/************************************************************/ + +struct list_s { + uintptr_t count; + uintptr_t last_allocated; + uintptr_t items[]; +}; + +static struct list_s *list_create(void) __attribute__((unused)); + +static inline void list_free(struct list_s *lst) +{ + free(lst); +} + +#define LIST_CREATE(lst) ((lst) = list_create()) +#define LIST_FREE(lst) (list_free(lst), (lst) = NULL) + + +static struct list_s *_list_grow(struct list_s *, uintptr_t); + +static inline struct list_s *list_append(struct list_s *lst, uintptr_t item) +{ + uintptr_t index = lst->count++; + if (UNLIKELY(index > lst->last_allocated)) + lst = _list_grow(lst, index); + lst->items[index] = item; + return lst; +} + +#define LIST_APPEND(lst, e) ((lst) = list_append((lst), (uintptr_t)(e))) + +static inline struct list_s *list_append2(struct list_s *lst, + uintptr_t item0, uintptr_t item1) +{ + uintptr_t index = lst->count; + lst->count += 2; + if (UNLIKELY(index >= lst->last_allocated)) + lst = _list_grow(lst, index + 1); + lst->items[index + 0] = item0; + lst->items[index + 1] = item1; + return lst; +} + + +static inline void list_clear(struct list_s *lst) +{ + lst->count = 0; +} + +static inline bool list_is_empty(struct list_s *lst) +{ + return (lst->count == 0); +} + +static inline uintptr_t list_count(struct list_s *lst) +{ + return lst->count; +} + +static inline uintptr_t list_pop_item(struct list_s *lst) +{ + assert(lst->count > 0); + return lst->items[--lst->count]; +} + +static inline uintptr_t list_item(struct list_s *lst, uintptr_t index) +{ + return lst->items[index]; +} + +static inline void list_set_item(struct list_s *lst, uintptr_t index, + uintptr_t newitem) +{ + lst->items[index] = newitem; +} + +static inline uintptr_t *list_ptr_to_item(struct list_s *lst, uintptr_t index) +{ + return &lst->items[index]; +} + +#define LIST_FOREACH_R(lst, TYPE, CODE) \ + do { \ + struct list_s *_lst = (lst); \ + uintptr_t _i; \ + for (_i = _lst->count; _i--; ) { \ + TYPE item = (TYPE)_lst->items[_i]; \ + CODE; \ + } \ + } while (0) + +/************************************************************/ + +/* The tree_xx functions are, like the name hints, implemented as a tree, + supporting very high performance in TREE_FIND in the common case where + there are no or few elements in the tree, but scaling correctly + if the number of items becomes large (logarithmically, rather + than almost-constant-time with hash maps, but with low constants). + The value 0 cannot be used as a key. +*/ + +#define TREE_BITS 4 +#define TREE_ARITY (1 << TREE_BITS) + +#define TREE_DEPTH_MAX ((sizeof(void*)*8 + TREE_BITS-1) / TREE_BITS) +/* sizeof(void*)*8 = total number of bits + (x + TREE_BITS-1) / TREE_BITS = divide by TREE_BITS, rounding up +*/ + +#define TREE_HASH(key) ((key) ^ ((key) << 4)) +#define TREE_MASK ((TREE_ARITY - 1) * sizeof(void*)) + +typedef struct { + uintptr_t addr; + uintptr_t val; +} wlog_t; + +typedef struct { + char *items[TREE_ARITY]; +} wlog_node_t; + +struct tree_s { + char *raw_start, *raw_current, *raw_end; + wlog_node_t toplevel; +}; + +static struct tree_s *tree_create(void) __attribute__((unused)); +static void tree_free(struct tree_s *tree) __attribute__((unused)); +static void tree_clear(struct tree_s *tree) __attribute__((unused)); +//static inline void tree_delete_not_used_any_more(struct tree_s *tree)... + +static inline bool tree_is_cleared(struct tree_s *tree) { + return tree->raw_current == tree->raw_start; +} + +#define _TREE_LOOP(tree, item, INITIAL, _PLUS_) \ +{ \ + struct { char **next; char **end; } _stack[TREE_DEPTH_MAX], *_stackp; \ + char **_next, **_end, *_entry; \ + long _deleted_factor = 0; \ + struct tree_s *_tree = &(tree); \ + /* initialization */ \ + _stackp = _stack; /* empty stack */ \ + _next = _tree->toplevel.items + INITIAL; \ + _end = _next _PLUS_ TREE_ARITY; \ + /* loop */ \ + while (1) \ + { \ + if (_next == _end) \ + { \ + if (_stackp == _stack) \ + break; /* done */ \ + /* finished with this level, go to the next one */ \ + _stackp--; \ + _next = _stackp->next; \ + _end = _stackp->end; \ + continue; \ + } \ + _entry = *_next; \ + _next = _next _PLUS_ 1; \ + if (_entry == NULL) /* empty entry */ \ + continue; \ + if (((long)_entry) & 1) \ + { /* points to a further level: enter it */ \ + assert(_stackp - _stack < TREE_DEPTH_MAX); \ + _stackp->next = _next; \ + _stackp->end = _end; \ + _stackp++; \ + _next = ((wlog_node_t *)(_entry - 1))->items + INITIAL; \ + _end = _next _PLUS_ TREE_ARITY; \ + continue; \ + } \ + /* points to a wlog_t item */ \ + if (((wlog_t *)_entry)->addr == 0) { /* deleted entry */ \ + _deleted_factor += 3; \ + continue; \ + } \ + _deleted_factor -= 4; \ + item = (wlog_t *)_entry; + +#define TREE_LOOP_FORWARD(tree, item) \ + _TREE_LOOP(tree, item, 0, +) +#define TREE_LOOP_BACKWARD(tree, item) \ + _TREE_LOOP(tree, item, (TREE_ARITY-1), -) +#define TREE_LOOP_END } } +#define TREE_LOOP_END_AND_COMPRESS \ + } if (_deleted_factor > 9) _tree_compress(_tree); } +#define TREE_LOOP_DELETE(item) { (item)->addr = NULL; _deleted_factor += 6; } + +#define TREE_FIND(tree, addr1, result, goto_not_found) \ +{ \ + uintptr_t _key = TREE_HASH(addr1); \ + char *_p = (char *)((tree).toplevel.items); \ + char *_entry = *(char **)(_p + (_key & TREE_MASK)); \ + if (_entry == NULL) \ + goto_not_found; /* common case, hopefully */ \ + result = _tree_find(_entry, addr1); \ + if (result == NULL || result->addr != (addr1)) \ + goto_not_found; \ +} + +static wlog_t *_tree_find(char *entry, uintptr_t addr); +static void _tree_compress(struct tree_s *tree) __attribute__((unused)); +static void tree_insert(struct tree_s *tree, uintptr_t addr, uintptr_t val); +static bool tree_delete_item(struct tree_s *tree, uintptr_t addr) + __attribute__((unused)); + +static inline bool tree_contains(struct tree_s *tree, uintptr_t addr) +{ + wlog_t *result; + TREE_FIND(*tree, addr, result, return false); + return true; +} diff --git a/c8/stm/misc.c b/c8/stm/misc.c new file mode 100644 --- /dev/null +++ b/c8/stm/misc.c @@ -0,0 +1,61 @@ +#ifndef _STM_CORE_H_ +# error "must be compiled via stmgc.c" +#endif + + +char *_stm_real_address(object_t *o) +{ + if (o == NULL) + return NULL; + + assert(FIRST_OBJECT_PAGE * 4096UL <= (uintptr_t)o + && (uintptr_t)o < NB_PAGES * 4096UL); + return REAL_ADDRESS(STM_SEGMENT->segment_base, o); +} + +char *_stm_get_segment_base(long index) +{ + return get_segment_base(index); +} + +struct stm_priv_segment_info_s *_stm_segment(void) +{ + char *info = REAL_ADDRESS(STM_SEGMENT->segment_base, STM_PSEGMENT); + return (struct stm_priv_segment_info_s *)info; +} + +stm_thread_local_t *_stm_thread(void) +{ + return STM_SEGMENT->running_thread; +} + +bool _stm_was_read(object_t *obj) +{ + uint8_t rm = ((struct stm_read_marker_s *) + (STM_SEGMENT->segment_base + (((uintptr_t)obj) >> 4)))->rm; + assert(rm <= STM_SEGMENT->transaction_read_version); + return rm == STM_SEGMENT->transaction_read_version; +} + +bool _stm_was_written(object_t *obj) +{ + return (obj->stm_flags & _STM_GCFLAG_WRITE_BARRIER) == 0; +} + + +#ifdef STM_TESTS + +long _stm_count_modified_old_objects(void) +{ + if (STM_PSEGMENT->modified_old_objects == NULL) + return -1; + return list_count(STM_PSEGMENT->modified_old_objects); +} + + +object_t *_stm_enum_modified_old_objects(long index) +{ + return (object_t *)list_item( + STM_PSEGMENT->modified_old_objects, index); +} +#endif diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c new file mode 100644 --- /dev/null +++ b/c8/stm/nursery.c @@ -0,0 +1,143 @@ +#ifndef _STM_CORE_H_ +# error "must be compiled via stmgc.c" +#endif + +/************************************************************/ + +#define NURSERY_START (FIRST_NURSERY_PAGE * 4096UL) +#define NURSERY_SIZE (NB_NURSERY_PAGES * 4096UL) +#define NURSERY_END (NURSERY_START + NURSERY_SIZE) + +static uintptr_t _stm_nursery_start; + + +/************************************************************/ + +static void setup_nursery(void) +{ + _stm_nursery_start = NURSERY_START; + + long i; + for (i = 1; i <= NB_SEGMENTS; i++) { + get_segment(i)->nursery_current = (stm_char *)NURSERY_START; + get_segment(i)->nursery_end = NURSERY_END; + } +} + +static inline bool _is_in_nursery(object_t *obj) +{ + assert((uintptr_t)obj >= NURSERY_START); + return (uintptr_t)obj < NURSERY_END; +} + + +/************************************************************/ + +static size_t throw_away_nursery(struct stm_priv_segment_info_s *pseg) +{ +#pragma push_macro("STM_PSEGMENT") +#pragma push_macro("STM_SEGMENT") +#undef STM_PSEGMENT +#undef STM_SEGMENT + dprintf(("throw_away_nursery\n")); + /* reset the nursery by zeroing it */ + size_t nursery_used; + char *realnursery; + + realnursery = REAL_ADDRESS(pseg->pub.segment_base, _stm_nursery_start); + nursery_used = pseg->pub.nursery_current - (stm_char *)_stm_nursery_start; + if (nursery_used > NB_NURSERY_PAGES * 4096) { + /* possible in rare cases when the program artificially advances + its own nursery_current */ + nursery_used = NB_NURSERY_PAGES * 4096; + } + OPT_ASSERT((nursery_used & 7) == 0); + memset(realnursery, 0, nursery_used); + + /* assert that the rest of the nursery still contains only zeroes */ + assert_memset_zero(realnursery + nursery_used, + (NURSERY_END - _stm_nursery_start) - nursery_used); + + pseg->pub.nursery_current = (stm_char *)_stm_nursery_start; + + + return nursery_used; +#pragma pop_macro("STM_SEGMENT") +#pragma pop_macro("STM_PSEGMENT") +} + + +static void _do_minor_collection(bool commit) +{ + dprintf(("minor_collection commit=%d\n", (int)commit)); + + throw_away_nursery(get_priv_segment(STM_SEGMENT->segment_num)); +} + +static void minor_collection(bool commit) +{ + assert(!_has_mutex()); + + _do_minor_collection(commit); +} + + +/************************************************************/ + + +object_t *_stm_allocate_slowpath(ssize_t size_rounded_up) +{ + /* may collect! */ + STM_SEGMENT->nursery_current -= size_rounded_up; /* restore correct val */ + + restart: + + OPT_ASSERT(size_rounded_up >= 16); + OPT_ASSERT((size_rounded_up & 7) == 0); + + stm_char *p = STM_SEGMENT->nursery_current; + stm_char *end = p + size_rounded_up; + if ((uintptr_t)end <= NURSERY_END) { + STM_SEGMENT->nursery_current = end; + return (object_t *)p; + } + + abort();//stm_collect(0); + goto restart; +} + +#ifdef STM_TESTS +void _stm_set_nursery_free_count(uint64_t free_count) +{ + assert(free_count <= NURSERY_SIZE); + assert((free_count & 7) == 0); + _stm_nursery_start = NURSERY_END - free_count; + + long i; + for (i = 1; i <= NB_SEGMENTS; i++) { + if ((uintptr_t)get_segment(i)->nursery_current < _stm_nursery_start) + get_segment(i)->nursery_current = (stm_char *)_stm_nursery_start; + } +} +#endif + +static void assert_memset_zero(void *s, size_t n) +{ +#ifndef NDEBUG + size_t i; +# ifndef STM_TESTS + if (n > 5000) n = 5000; +# endif + n /= 8; + for (i = 0; i < n; i++) + assert(((uint64_t *)s)[i] == 0); +#endif +} + +static void check_nursery_at_transaction_start(void) +{ + assert((uintptr_t)STM_SEGMENT->nursery_current == _stm_nursery_start); + assert_memset_zero(REAL_ADDRESS(STM_SEGMENT->segment_base, + STM_SEGMENT->nursery_current), + NURSERY_END - _stm_nursery_start); +} diff --git a/c8/stm/nursery.h b/c8/stm/nursery.h new file mode 100644 --- /dev/null +++ b/c8/stm/nursery.h @@ -0,0 +1,5 @@ +static void minor_collection(bool commit); +static void check_nursery_at_transaction_start(void); +static size_t throw_away_nursery(struct stm_priv_segment_info_s *pseg); + +static void assert_memset_zero(void *s, size_t n); diff --git a/c8/stm/rewind_setjmp.c b/c8/stm/rewind_setjmp.c new file mode 100644 --- /dev/null +++ b/c8/stm/rewind_setjmp.c @@ -0,0 +1,217 @@ +#include "rewind_setjmp.h" +#include +#include +#include +#include + + +struct _rewind_jmp_moved_s { + struct _rewind_jmp_moved_s *next; + size_t stack_size; + size_t shadowstack_size; +}; +#define RJM_HEADER sizeof(struct _rewind_jmp_moved_s) + +#ifndef RJBUF_CUSTOM_MALLOC +#define rj_malloc malloc +#define rj_free free +#else +void *rj_malloc(size_t); +void rj_free(void *); +#endif + +/* XXX: currently synchronized with global mutex!! */ + +static void copy_stack(rewind_jmp_thread *rjthread, char *base, void *ssbase) +{ + /* Copy away part of the stack and shadowstack. Sets moved_off_base to + the current frame_base. + + The stack is copied between 'base' (lower limit, i.e. newest bytes) + and 'rjthread->head->frame_base' (upper limit, i.e. oldest bytes). + The shadowstack is copied between 'ssbase' (upper limit, newest) + and 'rjthread->head->shadowstack_base' (lower limit, oldest). + */ + struct _rewind_jmp_moved_s *next; + char *stop; + void *ssstop; + size_t stack_size, ssstack_size; + + assert(rjthread->head != NULL); + ssstop = rjthread->head->shadowstack_base; + if (((long)ssstop) & 1) { + /* PyPy's JIT: 'head->frame_base' is missing; use directly 'head', + which should be at the end of the frame (and doesn't need itself + to be copied because it contains immutable data only) */ + ssstop = ((char *)ssstop) - 1; + stop = (char *)rjthread->head; + } + else { + stop = rjthread->head->frame_base; + } + assert(stop >= base); + assert(ssstop <= ssbase); + stack_size = stop - base; + ssstack_size = ssbase - ssstop; + + next = (struct _rewind_jmp_moved_s *) + rj_malloc(RJM_HEADER + stack_size + ssstack_size); + assert(next != NULL); /* XXX out of memory */ + next->next = rjthread->moved_off; + next->stack_size = stack_size; + next->shadowstack_size = ssstack_size; + + memcpy(((char *)next) + RJM_HEADER, base, stack_size); + memcpy(((char *)next) + RJM_HEADER + stack_size, ssstop, + ssstack_size); + + rjthread->moved_off_base = stop; + rjthread->moved_off_ssbase = ssstop; + rjthread->moved_off = next; +} + +__attribute__((noinline)) +long rewind_jmp_setjmp(rewind_jmp_thread *rjthread, void *ss) +{ + /* saves the current stack frame to the list of slices and + calls setjmp(). It returns the number of times a longjmp() + jumped back to this setjmp() */ + if (rjthread->moved_off) { + /* old stack slices are not needed anymore (next longjmp() + will restore only to this setjmp()) */ + _rewind_jmp_free_stack_slices(rjthread); + } + /* all locals of this function that need to be saved and restored + across the setjmp() should be stored inside this structure */ + struct { void *ss1; rewind_jmp_thread *rjthread1; } volatile saved = + { ss, rjthread }; + + int result; + if (__builtin_setjmp(rjthread->jmpbuf) == 0) { + rjthread = saved.rjthread1; + rjthread->initial_head = rjthread->head; + result = 0; + } + else { + rjthread = saved.rjthread1; + rjthread->head = rjthread->initial_head; + result = rjthread->repeat_count + 1; + } + rjthread->repeat_count = result; + + /* snapshot of top frame: needed every time because longjmp() frees + the previous one. Note that this function is called with the + mutex already acquired. Although it's not the job of this file, + we assert it is indeed acquired here. This is needed, otherwise a + concurrent GC may get garbage while saving shadow stack */ +#ifdef _STM_CORE_H_ + assert(_has_mutex()); +#endif + copy_stack(rjthread, (char *)&saved, saved.ss1); + + return result; +} + +__attribute__((noinline, noreturn)) +static void do_longjmp(rewind_jmp_thread *rjthread, char *stack_free) +{ + /* go through list of copied stack-slices and copy them back to the + current stack, expanding it if necessary. The shadowstack should + already be restored at this point (restore_shadowstack()) */ + assert(rjthread->moved_off_base != NULL); + + while (rjthread->moved_off) { + struct _rewind_jmp_moved_s *p = rjthread->moved_off; + char *target = rjthread->moved_off_base; + /* CPU stack grows downwards: */ + target -= p->stack_size; + if (target < stack_free) { + /* need more stack space! */ + do_longjmp(rjthread, alloca(stack_free - target)); + abort(); /* unreachable */ + } + memcpy(target, ((char *)p) + RJM_HEADER, p->stack_size); + + rjthread->moved_off_base = target; + rjthread->moved_off = p->next; + rj_free(p); + } + +#ifdef _STM_CORE_H_ + /* This function must be called with the mutex held. It will + remain held across the longjmp that follows and into the + target rewind_jmp_setjmp() function. */ + assert(_has_mutex()); +#endif + __builtin_longjmp(rjthread->jmpbuf, 1); +} + +__attribute__((noreturn)) +void rewind_jmp_longjmp(rewind_jmp_thread *rjthread) +{ + char _rewind_jmp_marker; + do_longjmp(rjthread, &_rewind_jmp_marker); +} + + +char *rewind_jmp_enum_shadowstack(rewind_jmp_thread *rjthread, + void *callback(void *, const void *, size_t)) +{ + /* enumerate all saved shadow-stack slices */ + struct _rewind_jmp_moved_s *p = rjthread->moved_off; + char *sstarget = rjthread->moved_off_ssbase; + +#ifdef _STM_CORE_H_ + assert(_has_mutex()); +#endif + + while (p) { + if (p->shadowstack_size) { + void *ss_slice = ((char *)p) + RJM_HEADER + p->stack_size; + callback(sstarget, ss_slice, p->shadowstack_size); + + sstarget += p->shadowstack_size; + } + p = p->next; + } + return sstarget; +} + + +char *rewind_jmp_restore_shadowstack(rewind_jmp_thread *rjthread) +{ + return rewind_jmp_enum_shadowstack(rjthread, memcpy); +} + +__attribute__((noinline)) +void _rewind_jmp_copy_stack_slice(rewind_jmp_thread *rjthread) +{ + /* called when leaving a frame. copies the now-current frame + to the list of stack-slices */ +#ifdef _STM_CORE_H_ + /* A transaction should be running now. This means in particular + that it's not possible that a major GC runs concurrently with + this code (and tries to read the shadowstack slice). */ + assert(_seems_to_be_running_transaction()); +#endif + if (rjthread->head == NULL) { + _rewind_jmp_free_stack_slices(rjthread); + return; + } + assert(rjthread->moved_off_base < (char *)rjthread->head); + copy_stack(rjthread, rjthread->moved_off_base, rjthread->moved_off_ssbase); +} + +void _rewind_jmp_free_stack_slices(rewind_jmp_thread *rjthread) +{ + /* frees all saved stack copies */ + struct _rewind_jmp_moved_s *p = rjthread->moved_off; + while (p) { + struct _rewind_jmp_moved_s *pnext = p->next; + rj_free(p); + p = pnext; + } + rjthread->moved_off = NULL; + rjthread->moved_off_base = NULL; + rjthread->moved_off_ssbase = NULL; +} diff --git a/c8/stm/rewind_setjmp.h b/c8/stm/rewind_setjmp.h new file mode 100644 --- /dev/null +++ b/c8/stm/rewind_setjmp.h @@ -0,0 +1,117 @@ +#ifndef _REWIND_SETJMP_H_ +#define _REWIND_SETJMP_H_ + + +#include + +/************************************************************ +There is a singly-linked list of frames in each thread +rjthread->head->prev->prev->prev + +Another singly-linked list is the list of copied stack-slices. +When doing a setjmp(), we copy the top-frame, free all old +stack-slices, and link it to the top-frame->moved_off. +When returning from the top-frame while moved_off still points +to a slice, we also need to copy the top-frame->prev frame/slice +and add it to this list (pointed to by moved_off). +-------------------------------------------------------------- + + : : ^^^^^ + |-------------------| older frames in the stack + | prev=0 | + ,---> | rewind_jmp_buf | + | |-------------------| + | | | + | : : + | : : + | | | + | |-------------------| + `---------prev | + ,----> | rewind_jmp_buf | + | +-------------------| + | | | + | : : + | | | + | |-------------------| + `----------prev | + ,---> | rewind_jmp_buf | <--------------- MOVED_OFF_BASE + | |---------------- +-------------+ + | | | STACK COPY | + | | : : + | : | size | + | | | next | <---- MOVED_OFF + | | +---|------ +-------------+ + | | | | | STACK COPY | + | |-------------------| | : (SEQUEL) : + `---------prev | | : : +HEAD-----> | rewind_jmp_buf | | | | + |-------------------| | | size | + `------> | next=0 | + +-------------+ + + +************************************************************/ + +typedef struct _rewind_jmp_buf { + char *shadowstack_base; + struct _rewind_jmp_buf *prev; + char *frame_base; + /* NB: PyPy's JIT has got details of this structure hard-coded, + as follows: it uses 2 words only (so frame_base is invalid) + and sets the lowest bit of 'shadowstack_base' to tell this */ +} rewind_jmp_buf; + +typedef struct { + rewind_jmp_buf *head; + rewind_jmp_buf *initial_head; + char *moved_off_base; + char *moved_off_ssbase; + struct _rewind_jmp_moved_s *moved_off; + void *jmpbuf[5]; + long repeat_count; +} rewind_jmp_thread; + + +/* remember the current stack and ss_stack positions */ +#define rewind_jmp_enterframe(rjthread, rjbuf, ss) do { \ + rewind_jmp_prepareframe(rjbuf); \ + rewind_jmp_enterprepframe(rjthread, rjbuf, ss); \ +} while (0) +#define rewind_jmp_prepareframe(rjbuf) \ + ((rjbuf)->frame_base = __builtin_frame_address(0)) +#define rewind_jmp_enterprepframe(rjthread, rjbuf, ss) do { \ + assert((((long)(ss)) & 1) == 0); \ + (rjbuf)->shadowstack_base = (char *)(ss); \ + (rjbuf)->prev = (rjthread)->head; \ + (rjthread)->head = (rjbuf); \ +} while (0) + +/* go up one frame. if there was a setjmp call in this frame, + */ +#define rewind_jmp_leaveframe(rjthread, rjbuf, ss) do { \ + assert((rjbuf)->shadowstack_base == (char *)(ss)); \ + (rjthread)->head = (rjbuf)->prev; \ + if ((rjbuf)->frame_base == (rjthread)->moved_off_base) { \ + assert((rjthread)->moved_off_ssbase == (char *)(ss));\ + _rewind_jmp_copy_stack_slice(rjthread); \ + } \ +} while (0) + +long rewind_jmp_setjmp(rewind_jmp_thread *rjthread, void *ss); +void rewind_jmp_longjmp(rewind_jmp_thread *rjthread) __attribute__((noreturn)); +char *rewind_jmp_restore_shadowstack(rewind_jmp_thread *rjthread); +char *rewind_jmp_enum_shadowstack(rewind_jmp_thread *rjthread, + void *callback(void *, const void *, size_t)); + +#define rewind_jmp_forget(rjthread) do { \ + if ((rjthread)->moved_off) _rewind_jmp_free_stack_slices(rjthread); \ + (rjthread)->moved_off_base = 0; \ + (rjthread)->moved_off_ssbase = 0; \ +} while (0) + +void _rewind_jmp_copy_stack_slice(rewind_jmp_thread *); +void _rewind_jmp_free_stack_slices(rewind_jmp_thread *); + +#define rewind_jmp_armed(rjthread) ((rjthread)->moved_off_base != 0) + +#endif diff --git a/c8/stm/setup.c b/c8/stm/setup.c new file mode 100644 --- /dev/null +++ b/c8/stm/setup.c @@ -0,0 +1,207 @@ +#ifndef _STM_CORE_H_ +# error "must be compiled via stmgc.c" +#endif + + +#ifdef USE_REMAP_FILE_PAGES +static char *setup_mmap(char *reason, int *ignored) +{ + char *result = mmap(NULL, TOTAL_MEMORY, + PROT_READ | PROT_WRITE, + MAP_PAGES_FLAGS, -1, 0); + if (result == MAP_FAILED) + stm_fatalerror("%s failed: %m", reason); + + return result; +} +static void close_fd_mmap(int ignored) +{ +} +#else +#include /* For O_* constants */ +static char *setup_mmap(char *reason, int *map_fd) +{ + char name[128]; + + /* Create the big shared memory object, and immediately unlink it. + There is a small window where if this process is killed the + object is left around. It doesn't seem possible to do anything + about it... + */ + int fd = shm_open(name, O_RDWR | O_CREAT | O_EXCL, 0600); + shm_unlink(name); + + if (fd == -1) { + stm_fatalerror("%s failed (stm_open): %m", reason); + } + if (ftruncate(fd, TOTAL_MEMORY) != 0) { + stm_fatalerror("%s failed (ftruncate): %m", reason); + } + char *result = mmap(NULL, TOTAL_MEMORY, + PROT_READ | PROT_WRITE, + MAP_PAGES_FLAGS & ~MAP_ANONYMOUS, fd, 0); + if (result == MAP_FAILED) { + stm_fatalerror("%s failed (mmap): %m", reason); + } + *map_fd = fd; + return result; +} +static void close_fd_mmap(int map_fd) +{ + close(map_fd); +} +#endif + +static void setup_protection_settings(void) +{ + /* The segment 0 is not used to run transactions, but contains the + shared copy of the pages. We mprotect all pages before so that + accesses fail, up to and including the pages corresponding to the + nurseries of the other segments. */ + mprotect(stm_object_pages, END_NURSERY_PAGE * 4096UL, PROT_NONE); + + long i; + for (i = 1; i <= NB_SEGMENTS; i++) { + char *segment_base = get_segment_base(i); + + /* In each segment, the first page is where TLPREFIX'ed + NULL accesses land. We mprotect it so that accesses fail. */ + mprotect(segment_base, 4096, PROT_NONE); + + /* Pages in range(2, FIRST_READMARKER_PAGE) are never used */ + if (FIRST_READMARKER_PAGE > 2) + mprotect(segment_base + 8192, + (FIRST_READMARKER_PAGE - 2) * 4096UL, + PROT_NONE); + } +} + +void stm_setup(void) +{ + /* Check that some values are acceptable */ + assert(NB_SEGMENTS <= NB_SEGMENTS_MAX); + assert(4096 <= ((uintptr_t)STM_SEGMENT)); + assert((uintptr_t)STM_SEGMENT == (uintptr_t)STM_PSEGMENT); + assert(((uintptr_t)STM_PSEGMENT) + sizeof(*STM_PSEGMENT) <= 8192); + assert(2 <= FIRST_READMARKER_PAGE); + assert(FIRST_READMARKER_PAGE * 4096UL <= READMARKER_START); + assert(READMARKER_START < READMARKER_END); + assert(READMARKER_END <= 4096UL * FIRST_OBJECT_PAGE); + assert(FIRST_OBJECT_PAGE < NB_PAGES); + assert((NB_PAGES * 4096UL) >> 8 <= (FIRST_OBJECT_PAGE * 4096UL) >> 4); + assert((END_NURSERY_PAGE * 4096UL) >> 8 <= + (FIRST_READMARKER_PAGE * 4096UL)); + + stm_object_pages = setup_mmap("initial stm_object_pages mmap()", + &stm_object_pages_fd); + setup_protection_settings(); + + long i; + for (i = 1; i <= NB_SEGMENTS; i++) { + char *segment_base = get_segment_base(i); + + /* Fill the TLS page (page 1) with 0xDC, for debugging */ + memset(REAL_ADDRESS(segment_base, 4096), 0xDC, 4096); + /* Make a "hole" at STM_PSEGMENT (which includes STM_SEGMENT) */ + memset(REAL_ADDRESS(segment_base, STM_PSEGMENT), 0, + sizeof(*STM_PSEGMENT)); + + /* Initialize STM_PSEGMENT */ + struct stm_priv_segment_info_s *pr = get_priv_segment(i); + assert(1 <= i && i < 255); /* 255 is WL_VISITED in gcpage.c */ + pr->pub.segment_num = i; + pr->pub.segment_base = segment_base; + pr->modified_old_objects = list_create(); + pr->pub.transaction_read_version = 0xff; + } + + /* The pages are shared lazily, as remap_file_pages() takes a relatively + long time for each page. + + The read markers are initially zero, but we set anyway + transaction_read_version to 0xff in order to force the first + transaction to "clear" the read markers by mapping a different, + private range of addresses. + */ + + setup_sync(); + setup_nursery(); +} + +void stm_teardown(void) +{ + /* This function is called during testing, but normal programs don't + need to call it. */ + assert(!_has_mutex()); + + long i; + for (i = 1; i <= NB_SEGMENTS; i++) { + struct stm_priv_segment_info_s *pr = get_priv_segment(i); + list_free(pr->modified_old_objects); + } + + munmap(stm_object_pages, TOTAL_MEMORY); + stm_object_pages = NULL; + close_fd_mmap(stm_object_pages_fd); + + teardown_sync(); +} + +static pthread_t *_get_cpth(stm_thread_local_t *tl) +{ + assert(sizeof(pthread_t) <= sizeof(tl->creating_pthread)); + return (pthread_t *)(tl->creating_pthread); +} + +void stm_register_thread_local(stm_thread_local_t *tl) +{ + int num; + s_mutex_lock(); + if (stm_all_thread_locals == NULL) { + stm_all_thread_locals = tl->next = tl->prev = tl; + num = 0; + } + else { + tl->next = stm_all_thread_locals; + tl->prev = stm_all_thread_locals->prev; + stm_all_thread_locals->prev->next = tl; + stm_all_thread_locals->prev = tl; + num = tl->prev->associated_segment_num; + } + + /* assign numbers consecutively, but that's for tests; we could also + assign the same number to all of them and they would get their own + numbers automatically. */ + num = (num % NB_SEGMENTS) + 1; + tl->associated_segment_num = num; + *_get_cpth(tl) = pthread_self(); + set_gs_register(get_segment_base(num)); + s_mutex_unlock(); +} + +void stm_unregister_thread_local(stm_thread_local_t *tl) +{ + s_mutex_lock(); + assert(tl->prev != NULL); + assert(tl->next != NULL); + + if (tl == stm_all_thread_locals) { + stm_all_thread_locals = stm_all_thread_locals->next; + if (tl == stm_all_thread_locals) { + stm_all_thread_locals = NULL; + s_mutex_unlock(); + return; + } + } + tl->prev->next = tl->next; + tl->next->prev = tl->prev; + tl->prev = NULL; + tl->next = NULL; + s_mutex_unlock(); +} + +__attribute__((unused)) +static bool _is_tl_registered(stm_thread_local_t *tl) +{ + return tl->next != NULL; +} diff --git a/c8/stm/setup.h b/c8/stm/setup.h new file mode 100644 --- /dev/null +++ b/c8/stm/setup.h @@ -0,0 +1,4 @@ +static char *setup_mmap(char *reason, int *map_fd); +static void close_fd_mmap(int map_fd); +static void setup_protection_settings(void); +static pthread_t *_get_cpth(stm_thread_local_t *); diff --git a/c8/stm/sync.c b/c8/stm/sync.c new file mode 100644 --- /dev/null +++ b/c8/stm/sync.c @@ -0,0 +1,144 @@ +#include +#include +#include + +#ifndef _STM_CORE_H_ +# error "must be compiled via stmgc.c" +#endif + + + +static union { + struct { + pthread_mutex_t global_mutex; + /* some additional pieces of global state follow */ + uint8_t in_use1[NB_SEGMENTS]; /* 1 if running a pthread */ + }; + char reserved[192]; +} sync_ctl __attribute__((aligned(64))); + + +static void setup_sync(void) +{ + if (pthread_mutex_init(&sync_ctl.global_mutex, NULL) != 0) + stm_fatalerror("mutex initialization: %m"); +} + +static void teardown_sync(void) +{ + if (pthread_mutex_destroy(&sync_ctl.global_mutex) != 0) + stm_fatalerror("mutex destroy: %m"); + + memset(&sync_ctl, 0, sizeof(sync_ctl)); +} + +#ifndef NDEBUG +__thread bool _has_mutex_here; +static inline bool _has_mutex(void) +{ + return _has_mutex_here; +} +#endif + +static void set_gs_register(char *value) +{ + if (UNLIKELY(syscall(SYS_arch_prctl, ARCH_SET_GS, (uint64_t)value) != 0)) + stm_fatalerror("syscall(arch_prctl, ARCH_SET_GS): %m"); +} + +static inline void s_mutex_lock(void) +{ + assert(!_has_mutex_here); + if (UNLIKELY(pthread_mutex_lock(&sync_ctl.global_mutex) != 0)) + stm_fatalerror("pthread_mutex_lock: %m"); + assert((_has_mutex_here = true, 1)); +} + +static inline void s_mutex_unlock(void) +{ + assert(_has_mutex_here); + if (UNLIKELY(pthread_mutex_unlock(&sync_ctl.global_mutex) != 0)) + stm_fatalerror("pthread_mutex_unlock: %m"); + assert((_has_mutex_here = false, 1)); +} + +/************************************************************/ + + +static bool acquire_thread_segment(stm_thread_local_t *tl) +{ + /* This function acquires a segment for the currently running thread, + and set up the GS register if it changed. */ + assert(_has_mutex()); + assert(_is_tl_registered(tl)); + + int num = tl->associated_segment_num; + if (sync_ctl.in_use1[num - 1] == 0) { + /* fast-path: we can get the same segment number than the one + we had before. The value stored in GS is still valid. */ +#ifdef STM_TESTS + /* that can be optimized away, except during tests, because + they use only one thread */ + set_gs_register(get_segment_base(num)); +#endif + dprintf(("acquired same segment: %d\n", num)); + goto got_num; + } + /* Look for the next free segment. If there is none, wait for + the condition variable. */ + int retries; + for (retries = 0; retries < NB_SEGMENTS; retries++) { + num = (num % NB_SEGMENTS) + 1; + if (sync_ctl.in_use1[num - 1] == 0) { + /* we're getting 'num', a different number. */ + dprintf(("acquired different segment: %d->%d\n", tl->associated_segment_num, num)); + tl->associated_segment_num = num; + set_gs_register(get_segment_base(num)); + goto got_num; + } + } + /* No segment available. Wait until release_thread_segment() + signals that one segment has been freed. */ + abort(); /* XXX */ + + /* Return false to the caller, which will call us again */ + return false; + + got_num: + sync_ctl.in_use1[num - 1] = 1; + assert(STM_SEGMENT->segment_num == num); + assert(STM_SEGMENT->running_thread == NULL); + STM_SEGMENT->running_thread = tl; + return true; +} + +static void release_thread_segment(stm_thread_local_t *tl) +{ + assert(_has_mutex()); + + assert(STM_SEGMENT->running_thread == tl); + STM_SEGMENT->running_thread = NULL; + + assert(sync_ctl.in_use1[tl->associated_segment_num - 1] == 1); + sync_ctl.in_use1[tl->associated_segment_num - 1] = 0; +} + +__attribute__((unused)) +static bool _seems_to_be_running_transaction(void) +{ + return (STM_SEGMENT->running_thread != NULL); +} + +bool _stm_in_transaction(stm_thread_local_t *tl) +{ + int num = tl->associated_segment_num; + assert(1 <= num && num <= NB_SEGMENTS); + return get_segment(num)->running_thread == tl; +} + +void _stm_test_switch(stm_thread_local_t *tl) +{ + assert(_stm_in_transaction(tl)); + set_gs_register(get_segment_base(tl->associated_segment_num)); + assert(STM_SEGMENT->running_thread == tl); +} diff --git a/c8/stm/sync.h b/c8/stm/sync.h new file mode 100644 --- /dev/null +++ b/c8/stm/sync.h @@ -0,0 +1,16 @@ +static void setup_sync(void); +static void teardown_sync(void); + + +static void s_mutex_lock(void); +static void s_mutex_unlock(void); +#ifndef NDEBUG +static bool _has_mutex(void); +#endif +static void set_gs_register(char *value); + + +/* acquire and release one of the segments for running the given thread + (must have the mutex acquired!) */ +static bool acquire_thread_segment(stm_thread_local_t *tl); +static void release_thread_segment(stm_thread_local_t *tl); diff --git a/c8/stmgc.c b/c8/stmgc.c new file mode 100644 --- /dev/null +++ b/c8/stmgc.c @@ -0,0 +1,19 @@ +#define _GNU_SOURCE 1 +#include "stmgc.h" +#include "stm/atomic.h" +#include "stm/list.h" +#include "stm/core.h" +#include "stm/nursery.h" +#include "stm/sync.h" +#include "stm/setup.h" +#include "stm/fprintcolor.h" +#include "stm/rewind_setjmp.h" + +#include "stm/list.c" +#include "stm/nursery.c" +#include "stm/core.c" +#include "stm/sync.c" +#include "stm/setup.c" +#include "stm/fprintcolor.c" +#include "stm/rewind_setjmp.c" +#include "stm/misc.c" diff --git a/c8/stmgc.h b/c8/stmgc.h new file mode 100644 --- /dev/null +++ b/c8/stmgc.h @@ -0,0 +1,162 @@ +#ifndef _STMGC_H +#define _STMGC_H + + +/* ==================== INTERNAL ==================== */ + +/* See "API" below. */ + + +#include +#include +#include +#include +#include + +#include "stm/rewind_setjmp.h" + +#if LONG_MAX == 2147483647 +# error "Requires a 64-bit environment" +#endif + + +#define TLPREFIX __attribute__((address_space(256))) + +typedef TLPREFIX struct object_s object_t; +typedef TLPREFIX struct stm_segment_info_s stm_segment_info_t; +typedef TLPREFIX struct stm_read_marker_s stm_read_marker_t; +typedef TLPREFIX char stm_char; + +struct stm_read_marker_s { + /* In every segment, every object has a corresponding read marker. + We assume that objects are at least 16 bytes long, and use + their address divided by 16. The read marker is equal to + 'STM_SEGMENT->transaction_read_version' if and only if the + object was read in the current transaction. The nurseries + also have corresponding read markers, but they are never used. */ + uint8_t rm; +}; + +struct stm_segment_info_s { + uint8_t transaction_read_version; + int segment_num; + char *segment_base; + stm_char *nursery_current; + uintptr_t nursery_end; + struct stm_thread_local_s *running_thread; +}; +#define STM_SEGMENT ((stm_segment_info_t *)4352) + + +typedef struct stm_thread_local_s { + /* rewind_setjmp's interface */ + rewind_jmp_thread rjthread; + /* the next fields are handled internally by the library */ + int associated_segment_num; + struct stm_thread_local_s *prev, *next; + void *creating_pthread[2]; +} stm_thread_local_t; + +#define _STM_GCFLAG_WRITE_BARRIER 0x01 + + +void _stm_write_slowpath(object_t *); +object_t *_stm_allocate_slowpath(ssize_t); +#ifdef STM_TESTS +#include +bool _stm_was_read(object_t *obj); +bool _stm_was_written(object_t *obj); + +char *_stm_get_segment_base(long index); +bool _stm_in_transaction(stm_thread_local_t *tl); +void _stm_set_nursery_free_count(uint64_t free_count); +long _stm_count_modified_old_objects(void); +object_t *_stm_enum_modified_old_objects(long index); +#endif + +/* ==================== HELPERS ==================== */ +#ifdef NDEBUG +#define OPT_ASSERT(cond) do { if (!(cond)) __builtin_unreachable(); } while (0) +#else +#define OPT_ASSERT(cond) assert(cond) +#endif +#define LIKELY(x) __builtin_expect(x, 1) +#define UNLIKELY(x) __builtin_expect(x, 0) +#define IMPLY(a, b) (!(a) || (b)) + + +/* ==================== PUBLIC API ==================== */ + +/* Number of segments (i.e. how many transactions can be executed in + parallel, in maximum). If you try to start transactions in more + threads than the number of segments, it will block, waiting for the + next segment to become free. +*/ +#define STM_NB_SEGMENTS 4 + + +struct object_s { + uint32_t stm_flags; /* reserved for the STM library */ +}; + +extern ssize_t stmcb_size_rounded_up(struct object_s *); + +__attribute__((always_inline)) +static inline void stm_read(object_t *obj) +{ + ((stm_read_marker_t *)(((uintptr_t)obj) >> 4))->rm = + STM_SEGMENT->transaction_read_version; +} + +__attribute__((always_inline)) +static inline void stm_write(object_t *obj) +{ + if (UNLIKELY((obj->stm_flags & _STM_GCFLAG_WRITE_BARRIER) != 0)) + _stm_write_slowpath(obj); +} + + +__attribute__((always_inline)) +static inline object_t *stm_allocate(ssize_t size_rounded_up) +{ + OPT_ASSERT(size_rounded_up >= 16); + OPT_ASSERT((size_rounded_up & 7) == 0); + + stm_char *p = STM_SEGMENT->nursery_current; + stm_char *end = p + size_rounded_up; + STM_SEGMENT->nursery_current = end; + if (UNLIKELY((uintptr_t)end > STM_SEGMENT->nursery_end)) + return _stm_allocate_slowpath(size_rounded_up); + + return (object_t *)p; +} + + +void stm_setup(void); +void stm_teardown(void); + + +void stm_register_thread_local(stm_thread_local_t *tl); +void stm_unregister_thread_local(stm_thread_local_t *tl); + +#define stm_rewind_jmp_enterprepframe(tl, rjbuf) \ + rewind_jmp_enterprepframe(&(tl)->rjthread, rjbuf, (tl)->shadowstack) +#define stm_rewind_jmp_enterframe(tl, rjbuf) \ + rewind_jmp_enterframe(&(tl)->rjthread, rjbuf, (tl)->shadowstack) +#define stm_rewind_jmp_leaveframe(tl, rjbuf) \ + rewind_jmp_leaveframe(&(tl)->rjthread, rjbuf, (tl)->shadowstack) +#define stm_rewind_jmp_setjmp(tl) \ + rewind_jmp_setjmp(&(tl)->rjthread, (tl)->shadowstack) +#define stm_rewind_jmp_longjmp(tl) \ + rewind_jmp_longjmp(&(tl)->rjthread) +#define stm_rewind_jmp_forget(tl) \ + rewind_jmp_forget(&(tl)->rjthread) + + +long stm_start_transaction(stm_thread_local_t *tl); +void stm_commit_transaction(void); + + +/* ==================== END ==================== */ + +#endif diff --git a/c8/test/common.py b/c8/test/common.py new file mode 100644 --- /dev/null +++ b/c8/test/common.py @@ -0,0 +1,27 @@ +import os +import sys +assert sys.maxint == 9223372036854775807, "requires a 64-bit environment" + +# ---------- +os.environ['CC'] = 'clang' + +parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +# ---------- + +source_files = [os.path.join(parent_dir, "stmgc.c")] +all_files = [os.path.join(parent_dir, "stmgc.h"), + os.path.join(parent_dir, "stmgc.c")] + [ + os.path.join(parent_dir, 'stm', _n) + for _n in os.listdir(os.path.join(parent_dir, 'stm')) + if (_n.endswith('.h') or _n.endswith('.c')) and not _n.startswith('.')] + +_pycache_ = os.path.join(parent_dir, 'test', '__pycache__') +if os.path.exists(_pycache_): + _fs = [_f for _f in os.listdir(_pycache_) if _f.startswith('_cffi_')] + if _fs: + _fsmtime = min(os.stat(os.path.join(_pycache_, _f)).st_mtime + for _f in _fs) + if any(os.stat(src).st_mtime >= _fsmtime for src in all_files): + import shutil + shutil.rmtree(_pycache_) diff --git a/c8/test/support.py b/c8/test/support.py new file mode 100644 --- /dev/null +++ b/c8/test/support.py @@ -0,0 +1,463 @@ +import os +import cffi, weakref +from common import parent_dir, source_files + +# ---------- + +ffi = cffi.FFI() +ffi.cdef(""" +typedef ... object_t; +#define SIZEOF_MYOBJ ... +#define STM_NB_SEGMENTS ... +#define _STM_GCFLAG_WRITE_BARRIER ... + +typedef struct { +...; +} rewind_jmp_thread; + +typedef struct { + rewind_jmp_thread rjthread; + int associated_segment_num; + struct stm_thread_local_s *prev, *next; + void *creating_pthread[2]; + ...; +} stm_thread_local_t; + +void stm_read(object_t *obj); +/*void stm_write(object_t *obj); use _checked_stm_write() instead */ +object_t *stm_allocate(ssize_t size_rounded_up); + +void stm_setup(void); +void stm_teardown(void); +void stm_register_thread_local(stm_thread_local_t *tl); +void stm_unregister_thread_local(stm_thread_local_t *tl); + +bool _checked_stm_write(object_t *obj); +bool _stm_was_read(object_t *obj); +bool _stm_was_written(object_t *obj); +char *_stm_get_segment_base(long index); +bool _stm_in_transaction(stm_thread_local_t *tl); From noreply at buildbot.pypy.org Wed Sep 3 10:02:27 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Wed, 3 Sep 2014 10:02:27 +0200 (CEST) Subject: [pypy-commit] stmgc default: pass 2 simple tests Message-ID: <20140903080227.B07631C072F@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1333:8ee2f1a4c200 Date: 2014-09-03 10:03 +0200 http://bitbucket.org/pypy/stmgc/changeset/8ee2f1a4c200/ Log: pass 2 simple tests diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c --- a/c8/stm/nursery.c +++ b/c8/stm/nursery.c @@ -30,6 +30,12 @@ return (uintptr_t)obj < NURSERY_END; } +long stm_can_move(object_t *obj) +{ + /* 'long' return value to avoid using 'bool' in the public interface */ + return _is_in_nursery(obj); +} + /************************************************************/ diff --git a/c8/stmgc.h b/c8/stmgc.h --- a/c8/stmgc.h +++ b/c8/stmgc.h @@ -67,6 +67,10 @@ bool _stm_was_read(object_t *obj); bool _stm_was_written(object_t *obj); +long stm_can_move(object_t *obj); +char *_stm_real_address(object_t *o); +void _stm_test_switch(stm_thread_local_t *tl); + char *_stm_get_segment_base(long index); bool _stm_in_transaction(stm_thread_local_t *tl); void _stm_set_nursery_free_count(uint64_t free_count); diff --git a/c8/test/support.py b/c8/test/support.py --- a/c8/test/support.py +++ b/c8/test/support.py @@ -39,6 +39,10 @@ bool _stm_in_transaction(stm_thread_local_t *tl); int _stm_get_flags(object_t *obj); +long stm_can_move(object_t *obj); +char *_stm_real_address(object_t *o); +void _stm_test_switch(stm_thread_local_t *tl); + void clear_jmpbuf(stm_thread_local_t *tl); long stm_start_transaction(stm_thread_local_t *tl); bool _check_commit_transaction(void); @@ -330,15 +334,10 @@ -SHADOWSTACK_LENGTH = 1000 _keepalive = weakref.WeakKeyDictionary() def _allocate_thread_local(): tl = ffi.new("stm_thread_local_t *") - ss = ffi.new("struct stm_shadowentry_s[]", SHADOWSTACK_LENGTH) - _keepalive[tl] = ss - tl.shadowstack = ss - tl.shadowstack_base = ss lib.stm_register_thread_local(tl) return tl @@ -355,17 +354,10 @@ lib.stmcb_expand_marker = ffi.NULL lib.stmcb_debug_print = ffi.NULL tl = self.tls[self.current_thread] - if lib._stm_in_transaction(tl) and lib.stm_is_inevitable(): - self.commit_transaction() # must succeed! + assert not lib._stm_in_transaction(tl) # for n, tl in enumerate(self.tls): - if lib._stm_in_transaction(tl): - if self.current_thread != n: - self.switch(n) - if lib.stm_is_inevitable(): - self.commit_transaction() # must succeed! - else: - self.abort_transaction() + assert not lib._stm_in_transaction(tl) # for tl in self.tls: lib.stm_unregister_thread_local(tl) @@ -405,34 +397,18 @@ def switch(self, thread_num): assert thread_num != self.current_thread - tl = self.tls[self.current_thread] - if lib._stm_in_transaction(tl): - stm_start_safe_point() # self.current_thread = thread_num tl2 = self.tls[thread_num] # if lib._stm_in_transaction(tl2): lib._stm_test_switch(tl2) - stm_stop_safe_point() # can raise Conflict def push_root(self, o): - assert ffi.typeof(o) == ffi.typeof("object_t *") - tl = self.tls[self.current_thread] - curlength = tl.shadowstack - tl.shadowstack_base - assert 0 <= curlength < SHADOWSTACK_LENGTH - tl.shadowstack[0].ss = ffi.cast("object_t *", o) - tl.shadowstack += 1 + assert 0 def pop_root(self): - tl = self.tls[self.current_thread] - curlength = tl.shadowstack - tl.shadowstack_base - assert curlength >= 1 - if curlength == 1: - raise EmptyStack - assert 0 < curlength <= SHADOWSTACK_LENGTH - tl.shadowstack -= 1 - return ffi.cast("object_t *", tl.shadowstack[0].ss) + assert 0 def push_root_no_gc(self): "Pushes an invalid object, to crash in case the GC is called" @@ -451,13 +427,3 @@ def set_thread_local_obj(self, newobj): tl = self.tls[self.current_thread] tl.thread_local_obj = newobj - - def become_inevitable(self): - tl = self.tls[self.current_thread] - if lib._check_become_inevitable(tl): - raise Conflict() - - def become_globally_unique_transaction(self): - tl = self.tls[self.current_thread] - if lib._check_become_globally_unique_transaction(tl): - raise Conflict() From noreply at buildbot.pypy.org Wed Sep 3 10:25:52 2014 From: noreply at buildbot.pypy.org (malthe) Date: Wed, 3 Sep 2014 10:25:52 +0200 (CEST) Subject: [pypy-commit] cffi tmpdir-from-environ: The 'Verifier' class already falls back to '_caller_dir_pycache' Message-ID: <20140903082552.9B0F61C320C@cobra.cs.uni-duesseldorf.de> Author: malthe Branch: tmpdir-from-environ Changeset: r1561:b59844f99880 Date: 2014-09-02 13:59 +0200 http://bitbucket.org/cffi/cffi/changeset/b59844f99880/ Log: The 'Verifier' class already falls back to '_caller_dir_pycache' diff --git a/cffi/api.py b/cffi/api.py --- a/cffi/api.py +++ b/cffi/api.py @@ -334,8 +334,7 @@ (including calling macros). This is unlike 'ffi.dlopen()', which requires binary compatibility in the signatures. """ - from .verifier import Verifier, _caller_dir_pycache - tmpdir = tmpdir or _caller_dir_pycache() + from .verifier import Verifier self.verifier = Verifier(self, source, tmpdir, **kwargs) lib = self.verifier.load_library() self._libraries.append(lib) From noreply at buildbot.pypy.org Wed Sep 3 10:25:53 2014 From: noreply at buildbot.pypy.org (malthe) Date: Wed, 3 Sep 2014 10:25:53 +0200 (CEST) Subject: [pypy-commit] cffi tmpdir-from-environ: Add support for 'CFFI_TMPDIR' environment variable Message-ID: <20140903082553.B25AD1C320C@cobra.cs.uni-duesseldorf.de> Author: malthe Branch: tmpdir-from-environ Changeset: r1562:a9fa3e1d222d Date: 2014-09-02 14:07 +0200 http://bitbucket.org/cffi/cffi/changeset/a9fa3e1d222d/ Log: Add support for 'CFFI_TMPDIR' environment variable If set, the value is used as the 'tmpdir' instead of the caller directory's '__pycache__'. This allows correct run-time operation when the package source is read-only. diff --git a/cffi/verifier.py b/cffi/verifier.py --- a/cffi/verifier.py +++ b/cffi/verifier.py @@ -42,7 +42,7 @@ modulename = '_cffi_%s_%s%s%s' % (tag, self._vengine._class_key, k1, k2) suffix = _get_so_suffixes()[0] - self.tmpdir = tmpdir or _caller_dir_pycache() + self.tmpdir = tmpdir or os.environ.get('CFFI_TMPDIR') or _caller_dir_pycache() self.sourcefilename = os.path.join(self.tmpdir, modulename + '.c') self.modulefilename = os.path.join(self.tmpdir, modulename + suffix) self.ext_package = ext_package diff --git a/doc/source/index.rst b/doc/source/index.rst --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -381,7 +381,10 @@ Alternatively, you can just completely remove the ``__pycache__`` directory. - +An alternative cache directory can be given as the ``tmpdir`` argument +to ``verify()``, via the environment variable ``CFFI_TMPDIR``, or by +calling ``cffi.verifier.set_tmpdir(path)`` prior to calling +``verify``. ======================================================= From noreply at buildbot.pypy.org Wed Sep 3 10:25:54 2014 From: noreply at buildbot.pypy.org (malthe) Date: Wed, 3 Sep 2014 10:25:54 +0200 (CEST) Subject: [pypy-commit] cffi tmpdir-from-environ: Mention 'CFFI_TMPDIR' in changelog Message-ID: <20140903082554.CBA511C320C@cobra.cs.uni-duesseldorf.de> Author: malthe Branch: tmpdir-from-environ Changeset: r1563:77488a5815a5 Date: 2014-09-02 14:59 +0200 http://bitbucket.org/cffi/cffi/changeset/77488a5815a5/ Log: Mention 'CFFI_TMPDIR' in changelog diff --git a/doc/source/index.rst b/doc/source/index.rst --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -640,8 +640,9 @@ .. versionadded:: 0.4 The ``tmpdir`` argument to ``verify()`` controls where the C - files are created and compiled. By default it is - ``directory_containing_the_py_file/__pycache__``, using the + files are created and compiled. Unless the ``CFFI_TMPDIR`` environment + variable is set, the default is + ``directory_containing_the_py_file/__pycache__`` using the directory name of the .py file that contains the actual call to ``ffi.verify()``. (This is a bit of a hack but is generally consistent with the location of the .pyc files for your library. From noreply at buildbot.pypy.org Wed Sep 3 10:25:55 2014 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 3 Sep 2014 10:25:55 +0200 (CEST) Subject: [pypy-commit] cffi default: Merged in malthe/cffi/tmpdir-from-environ (pull request #46) Message-ID: <20140903082555.D6D8E1C320C@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1564:66970d83f3a0 Date: 2014-09-03 10:26 +0200 http://bitbucket.org/cffi/cffi/changeset/66970d83f3a0/ Log: Merged in malthe/cffi/tmpdir-from-environ (pull request #46) Add support for alternative "tmpdir" diff --git a/cffi/api.py b/cffi/api.py --- a/cffi/api.py +++ b/cffi/api.py @@ -334,8 +334,7 @@ (including calling macros). This is unlike 'ffi.dlopen()', which requires binary compatibility in the signatures. """ - from .verifier import Verifier, _caller_dir_pycache - tmpdir = tmpdir or _caller_dir_pycache() + from .verifier import Verifier self.verifier = Verifier(self, source, tmpdir, **kwargs) lib = self.verifier.load_library() self._libraries.append(lib) diff --git a/cffi/verifier.py b/cffi/verifier.py --- a/cffi/verifier.py +++ b/cffi/verifier.py @@ -42,7 +42,7 @@ modulename = '_cffi_%s_%s%s%s' % (tag, self._vengine._class_key, k1, k2) suffix = _get_so_suffixes()[0] - self.tmpdir = tmpdir or _caller_dir_pycache() + self.tmpdir = tmpdir or os.environ.get('CFFI_TMPDIR') or _caller_dir_pycache() self.sourcefilename = os.path.join(self.tmpdir, modulename + '.c') self.modulefilename = os.path.join(self.tmpdir, modulename + suffix) self.ext_package = ext_package diff --git a/doc/source/index.rst b/doc/source/index.rst --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -381,7 +381,10 @@ Alternatively, you can just completely remove the ``__pycache__`` directory. - +An alternative cache directory can be given as the ``tmpdir`` argument +to ``verify()``, via the environment variable ``CFFI_TMPDIR``, or by +calling ``cffi.verifier.set_tmpdir(path)`` prior to calling +``verify``. ======================================================= @@ -637,8 +640,9 @@ .. versionadded:: 0.4 The ``tmpdir`` argument to ``verify()`` controls where the C - files are created and compiled. By default it is - ``directory_containing_the_py_file/__pycache__``, using the + files are created and compiled. Unless the ``CFFI_TMPDIR`` environment + variable is set, the default is + ``directory_containing_the_py_file/__pycache__`` using the directory name of the .py file that contains the actual call to ``ffi.verify()``. (This is a bit of a hack but is generally consistent with the location of the .pyc files for your library. From noreply at buildbot.pypy.org Wed Sep 3 10:55:34 2014 From: noreply at buildbot.pypy.org (groggi) Date: Wed, 3 Sep 2014 10:55:34 +0200 (CEST) Subject: [pypy-commit] pypy gc-incminimark-pinning: update comments in rffi Message-ID: <20140903085534.831A31C320C@cobra.cs.uni-duesseldorf.de> Author: Gregor Wegberg Branch: gc-incminimark-pinning Changeset: r73284:f7d67105c780 Date: 2014-09-03 10:54 +0200 http://bitbucket.org/pypy/pypy/changeset/f7d67105c780/ Log: update comments in rffi diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py --- a/rpython/rtyper/lltypesystem/rffi.py +++ b/rpython/rtyper/lltypesystem/rffi.py @@ -763,14 +763,13 @@ """ Either returns a non-moving copy or performs neccessary pointer arithmetic to return a pointer to the characters of a string if the - string is already nonmovable. Must be followed by a + string is already nonmovable or could be pinned. Must be followed by a free_nonmovingbuffer call. First bool returned indicates if 'data' was pinned. Second bool returned - indicates if we did a raw alloc because pinning didn't work. Bot bools + indicates if we did a raw alloc because pinning failed. Both bools should never be true at the same time. """ - # XXX update doc string lldata = llstrtype(data) count = len(data) @@ -801,7 +800,8 @@ @jit.dont_look_inside def free_nonmovingbuffer(data, buf, is_pinned, is_raw): """ - Either free a non-moving buffer or keep the original storage alive. + Keep 'data' alive and unpin it if it was pinned ('is_pinned' is true). + Otherwise free the non-moving copy ('is_raw' is true). """ if is_pinned: rgc.unpin(data) From noreply at buildbot.pypy.org Wed Sep 3 11:29:03 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Wed, 3 Sep 2014 11:29:03 +0200 (CEST) Subject: [pypy-commit] stmgc default: really pass 2nd test Message-ID: <20140903092903.9F3021C072F@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1334:5eb79da39a17 Date: 2014-09-03 10:16 +0200 http://bitbucket.org/pypy/stmgc/changeset/5eb79da39a17/ Log: really pass 2nd test diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -99,3 +99,57 @@ s_mutex_unlock(); } + + +static void abort_data_structures_from_segment_num(int segment_num) +{ +#pragma push_macro("STM_PSEGMENT") +#pragma push_macro("STM_SEGMENT") +#undef STM_PSEGMENT +#undef STM_SEGMENT + /* struct stm_priv_segment_info_s *pseg = get_priv_segment(segment_num); */ + + /* throw_away_nursery(pseg); */ + + /* reset_modified_from_other_segments(segment_num); */ + +#pragma pop_macro("STM_SEGMENT") +#pragma pop_macro("STM_PSEGMENT") +} + + +static stm_thread_local_t *abort_with_mutex_no_longjmp(void) +{ + assert(_has_mutex()); + dprintf(("~~~ ABORT\n")); + + assert(STM_PSEGMENT->running_pthread == pthread_self()); + + abort_data_structures_from_segment_num(STM_SEGMENT->segment_num); + + stm_thread_local_t *tl = STM_SEGMENT->running_thread; + + _finish_transaction(); + /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */ + + return tl; +} + + +#ifdef STM_NO_AUTOMATIC_SETJMP +void _test_run_abort(stm_thread_local_t *tl) __attribute__((noreturn)); +#endif + +void stm_abort_transaction(void) +{ + s_mutex_lock(); + stm_thread_local_t *tl = abort_with_mutex_no_longjmp(); + s_mutex_unlock(); + +#ifdef STM_NO_AUTOMATIC_SETJMP + _test_run_abort(tl); +#else + s_mutex_lock(); + stm_rewind_jmp_longjmp(tl); +#endif +} diff --git a/c8/stmgc.h b/c8/stmgc.h --- a/c8/stmgc.h +++ b/c8/stmgc.h @@ -159,6 +159,7 @@ long stm_start_transaction(stm_thread_local_t *tl); void stm_commit_transaction(void); +void stm_abort_transaction(void) __attribute__((noreturn)); /* ==================== END ==================== */ diff --git a/c8/test/support.py b/c8/test/support.py --- a/c8/test/support.py +++ b/c8/test/support.py @@ -46,6 +46,7 @@ void clear_jmpbuf(stm_thread_local_t *tl); long stm_start_transaction(stm_thread_local_t *tl); bool _check_commit_transaction(void); +bool _check_abort_transaction(void); void _set_type_id(object_t *obj, uint32_t h); uint32_t _get_type_id(object_t *obj); @@ -113,6 +114,10 @@ CHECKED(stm_commit_transaction()); } +bool _check_abort_transaction(void) { + CHECKED(stm_abort_transaction()); +} + #undef CHECKED @@ -353,11 +358,12 @@ def teardown_method(self, meth): lib.stmcb_expand_marker = ffi.NULL lib.stmcb_debug_print = ffi.NULL - tl = self.tls[self.current_thread] - assert not lib._stm_in_transaction(tl) # for n, tl in enumerate(self.tls): - assert not lib._stm_in_transaction(tl) + if lib._stm_in_transaction(tl): + if self.current_thread != n: + self.switch(n) + self.abort_transaction() # for tl in self.tls: lib.stm_unregister_thread_local(tl) From noreply at buildbot.pypy.org Wed Sep 3 11:29:04 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Wed, 3 Sep 2014 11:29:04 +0200 (CEST) Subject: [pypy-commit] stmgc default: +1 Message-ID: <20140903092904.C25111C072F@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1335:40e30c73642d Date: 2014-09-03 10:19 +0200 http://bitbucket.org/pypy/stmgc/changeset/40e30c73642d/ Log: +1 diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -85,7 +85,6 @@ assert(STM_PSEGMENT->running_pthread == pthread_self()); minor_collection(1); - abort(); s_mutex_lock(); diff --git a/c8/test/support.py b/c8/test/support.py --- a/c8/test/support.py +++ b/c8/test/support.py @@ -53,6 +53,8 @@ void _set_ptr(object_t *obj, int n, object_t *v); object_t * _get_ptr(object_t *obj, int n); +long _stm_count_modified_old_objects(void); +object_t *_stm_enum_modified_old_objects(long index); void *memset(void *s, int c, size_t n); From noreply at buildbot.pypy.org Wed Sep 3 11:29:05 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Wed, 3 Sep 2014 11:29:05 +0200 (CEST) Subject: [pypy-commit] stmgc default: re-introduce objects_pointing_to_nursery Message-ID: <20140903092905.CC2D21C072F@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1336:3efac6fcb465 Date: 2014-09-03 10:35 +0200 http://bitbucket.org/pypy/stmgc/changeset/3efac6fcb465/ Log: re-introduce objects_pointing_to_nursery diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -5,7 +5,16 @@ void _stm_write_slowpath(object_t *obj) { + assert(_seems_to_be_running_transaction()); + assert(!_is_in_nursery(obj)); + assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); + stm_read(obj); + + LIST_APPEND(STM_PSEGMENT->modified_old_objects, obj); + + LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, obj); + obj->stm_flags &= ~GCFLAG_WRITE_BARRIER; } static void reset_transaction_read_version(void) @@ -54,6 +63,7 @@ } assert(list_is_empty(STM_PSEGMENT->modified_old_objects)); + assert(list_is_empty(STM_PSEGMENT->objects_pointing_to_nursery)); check_nursery_at_transaction_start(); } @@ -75,6 +85,9 @@ static void _finish_transaction() { stm_thread_local_t *tl = STM_SEGMENT->running_thread; + + list_clear(STM_PSEGMENT->objects_pointing_to_nursery); + release_thread_segment(tl); /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */ } diff --git a/c8/stm/core.h b/c8/stm/core.h --- a/c8/stm/core.h +++ b/c8/stm/core.h @@ -51,6 +51,7 @@ struct stm_segment_info_s pub; struct list_s *modified_old_objects; + struct list_s *objects_pointing_to_nursery; /* For debugging */ #ifndef NDEBUG diff --git a/c8/stm/misc.c b/c8/stm/misc.c --- a/c8/stm/misc.c +++ b/c8/stm/misc.c @@ -44,7 +44,6 @@ #ifdef STM_TESTS - long _stm_count_modified_old_objects(void) { if (STM_PSEGMENT->modified_old_objects == NULL) @@ -52,10 +51,22 @@ return list_count(STM_PSEGMENT->modified_old_objects); } +long _stm_count_objects_pointing_to_nursery(void) +{ + if (STM_PSEGMENT->objects_pointing_to_nursery == NULL) + return -1; + return list_count(STM_PSEGMENT->objects_pointing_to_nursery); +} object_t *_stm_enum_modified_old_objects(long index) { return (object_t *)list_item( STM_PSEGMENT->modified_old_objects, index); } + +object_t *_stm_enum_objects_pointing_to_nursery(long index) +{ + return (object_t *)list_item( + STM_PSEGMENT->objects_pointing_to_nursery, index); +} #endif diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c --- a/c8/stm/nursery.c +++ b/c8/stm/nursery.c @@ -77,6 +77,8 @@ { dprintf(("minor_collection commit=%d\n", (int)commit)); + assert(list_is_empty(STM_PSEGMENT->objects_pointing_to_nursery)); + throw_away_nursery(get_priv_segment(STM_SEGMENT->segment_num)); } diff --git a/c8/stm/setup.c b/c8/stm/setup.c --- a/c8/stm/setup.c +++ b/c8/stm/setup.c @@ -112,6 +112,7 @@ pr->pub.segment_num = i; pr->pub.segment_base = segment_base; pr->modified_old_objects = list_create(); + pr->objects_pointing_to_nursery = list_create(); pr->pub.transaction_read_version = 0xff; } @@ -137,6 +138,8 @@ long i; for (i = 1; i <= NB_SEGMENTS; i++) { struct stm_priv_segment_info_s *pr = get_priv_segment(i); + assert(list_is_empty(pr->objects_pointing_to_nursery)); + list_free(pr->objects_pointing_to_nursery); list_free(pr->modified_old_objects); } diff --git a/c8/stmgc.h b/c8/stmgc.h --- a/c8/stmgc.h +++ b/c8/stmgc.h @@ -75,7 +75,9 @@ bool _stm_in_transaction(stm_thread_local_t *tl); void _stm_set_nursery_free_count(uint64_t free_count); long _stm_count_modified_old_objects(void); +long _stm_count_objects_pointing_to_nursery(void); object_t *_stm_enum_modified_old_objects(long index); +object_t *_stm_enum_objects_pointing_to_nursery(long index); #endif /* ==================== HELPERS ==================== */ diff --git a/c8/test/support.py b/c8/test/support.py --- a/c8/test/support.py +++ b/c8/test/support.py @@ -54,7 +54,9 @@ object_t * _get_ptr(object_t *obj, int n); long _stm_count_modified_old_objects(void); +long _stm_count_objects_pointing_to_nursery(void); object_t *_stm_enum_modified_old_objects(long index); +object_t *_stm_enum_objects_pointing_to_nursery(long index); void *memset(void *s, int c, size_t n); diff --git a/c8/test/test_basic.py b/c8/test/test_basic.py --- a/c8/test/test_basic.py +++ b/c8/test/test_basic.py @@ -51,7 +51,7 @@ stm_write(lp1) assert stm_was_written(lp1) assert modified_old_objects() == [] # object not old - assert objects_pointing_to_nursery() == None # short transaction + assert objects_pointing_to_nursery() == [] # short transaction self.commit_transaction() def test_allocate_old(self): @@ -214,7 +214,7 @@ self.commit_transaction() - + def test_start_transaction_updates(self): self.start_transaction() lp1 = stm_allocate(16) @@ -233,7 +233,7 @@ self.switch(0) self.start_transaction() assert stm_get_char(lp1) == 'b' - + def test_resolve_no_conflict_empty(self): self.start_transaction() @@ -254,11 +254,11 @@ self.commit_transaction() lp1 = self.pop_root() # 'a' in SHARED_PAGE - + self.start_transaction() - + self.switch(1) - + self.start_transaction() stm_write(lp1) # privatize page p1 = stm_get_real_address(lp1) @@ -307,7 +307,7 @@ self.push_root(lp1) self.commit_transaction() lp1 = self.pop_root() - + self.start_transaction() stm_read(lp1) # @@ -325,7 +325,7 @@ self.push_root(lp1) self.commit_transaction() lp1 = self.pop_root() - + self.start_transaction() # self.switch(1) @@ -344,7 +344,7 @@ self.push_root(lp1) self.commit_transaction() lp1 = self.pop_root() - + self.start_transaction() stm_write(lp1) # acquire lock # From noreply at buildbot.pypy.org Wed Sep 3 11:29:06 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Wed, 3 Sep 2014 11:29:06 +0200 (CEST) Subject: [pypy-commit] stmgc default: remove special segment 0 and introduce simple stm_allocate_old for tests Message-ID: <20140903092906.DF3521C072F@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1337:0d99d2f929cd Date: 2014-09-03 11:30 +0200 http://bitbucket.org/pypy/stmgc/changeset/0d99d2f929cd/ Log: remove special segment 0 and introduce simple stm_allocate_old for tests diff --git a/c8/stm/core.h b/c8/stm/core.h --- a/c8/stm/core.h +++ b/c8/stm/core.h @@ -33,8 +33,6 @@ #define NB_READMARKER_PAGES (FIRST_OBJECT_PAGE - FIRST_READMARKER_PAGE) -#define USE_REMAP_FILE_PAGES 1 - enum /* stm_flags */ { GCFLAG_WRITE_BARRIER = _STM_GCFLAG_WRITE_BARRIER, }; @@ -52,6 +50,7 @@ struct list_s *modified_old_objects; struct list_s *objects_pointing_to_nursery; + uint8_t privatization_lock; /* For debugging */ #ifndef NDEBUG @@ -96,3 +95,17 @@ llvmfix/no-memset-creation-with-addrspace.diff. */ asm("/* workaround for llvm bug */"); } + +static inline void acquire_privatization_lock(void) +{ + uint8_t *lock = (uint8_t *)REAL_ADDRESS(STM_SEGMENT->segment_base, + &STM_PSEGMENT->privatization_lock); + spinlock_acquire(*lock); +} + +static inline void release_privatization_lock(void) +{ + uint8_t *lock = (uint8_t *)REAL_ADDRESS(STM_SEGMENT->segment_base, + &STM_PSEGMENT->privatization_lock); + spinlock_release(*lock); +} diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c new file mode 100644 --- /dev/null +++ b/c8/stm/gcpage.c @@ -0,0 +1,55 @@ +#ifndef _STM_CORE_H_ +# error "must be compiled via stmgc.c" +#endif + + +static void setup_gcpage(void) +{ + uninitialized_page_start = stm_object_pages + END_NURSERY_PAGE * 4096UL; + uninitialized_page_stop = stm_object_pages + NB_PAGES * 4096UL; +} + +static void teardown_gcpage(void) +{ +} + +static void setup_N_pages(char *pages_addr, uint64_t num) +{ + pages_initialize_private((pages_addr - stm_object_pages) / 4096UL, num); +} + + +static char *allocate_outside_nursery_large(uint64_t size) +{ + /* XXX: real allocation */ + char *addr = uninitialized_page_start; + + char *start = uninitialized_page_start; + if (addr + size > start) { /* XXX: always for now */ + uintptr_t npages; + npages = (addr + size - start) / 4096UL + 1; + if (uninitialized_page_stop - start < npages * 4096UL) { + stm_fatalerror("out of memory!"); /* XXX */ + } + setup_N_pages(start, npages); + if (!__sync_bool_compare_and_swap(&uninitialized_page_start, + start, + start + npages * 4096UL)) { + stm_fatalerror("uninitialized_page_start changed?"); + } + } + + return addr; +} + +object_t *_stm_allocate_old(ssize_t size_rounded_up) +{ + /* only for tests xxx but stm_setup_prebuilt() uses this now too */ + char *p = allocate_outside_nursery_large(size_rounded_up); + memset(p, 0, size_rounded_up); + + object_t *o = (object_t *)(p - stm_object_pages); + o->stm_flags = GCFLAG_WRITE_BARRIER; + + return o; +} diff --git a/c8/stm/gcpage.h b/c8/stm/gcpage.h new file mode 100644 --- /dev/null +++ b/c8/stm/gcpage.h @@ -0,0 +1,7 @@ + + +static char *uninitialized_page_start; /* within segment 0 */ +static char *uninitialized_page_stop; + +static void setup_gcpage(void); +static void teardown_gcpage(void); diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c --- a/c8/stm/nursery.c +++ b/c8/stm/nursery.c @@ -18,7 +18,7 @@ _stm_nursery_start = NURSERY_START; long i; - for (i = 1; i <= NB_SEGMENTS; i++) { + for (i = 0; i < NB_SEGMENTS; i++) { get_segment(i)->nursery_current = (stm_char *)NURSERY_START; get_segment(i)->nursery_end = NURSERY_END; } @@ -122,7 +122,7 @@ _stm_nursery_start = NURSERY_END - free_count; long i; - for (i = 1; i <= NB_SEGMENTS; i++) { + for (i = 0; i < NB_SEGMENTS; i++) { if ((uintptr_t)get_segment(i)->nursery_current < _stm_nursery_start) get_segment(i)->nursery_current = (stm_char *)_stm_nursery_start; } diff --git a/c8/stm/pagecopy.c b/c8/stm/pagecopy.c new file mode 100644 --- /dev/null +++ b/c8/stm/pagecopy.c @@ -0,0 +1,68 @@ + +#define PAGECOPY_128(dest, src) \ + asm volatile("movdqa (%0), %%xmm0\n" \ + "movdqa 16(%0), %%xmm1\n" \ + "movdqa 32(%0), %%xmm2\n" \ + "movdqa 48(%0), %%xmm3\n" \ + "movdqa %%xmm0, (%1)\n" \ + "movdqa %%xmm1, 16(%1)\n" \ + "movdqa %%xmm2, 32(%1)\n" \ + "movdqa %%xmm3, 48(%1)\n" \ + "movdqa 64(%0), %%xmm0\n" \ + "movdqa 80(%0), %%xmm1\n" \ + "movdqa 96(%0), %%xmm2\n" \ + "movdqa 112(%0), %%xmm3\n" \ + "movdqa %%xmm0, 64(%1)\n" \ + "movdqa %%xmm1, 80(%1)\n" \ + "movdqa %%xmm2, 96(%1)\n" \ + "movdqa %%xmm3, 112(%1)\n" \ + : \ + : "r"(src), "r"(dest) \ + : "xmm0", "xmm1", "xmm2", "xmm3", "memory") + +static void pagecopy(void *dest, const void *src) +{ + unsigned long i; + for (i = 0; i < 4096 / 128; i++) { + PAGECOPY_128(dest + 128*i, src + 128*i); + } +} + +#if 0 +static void pagecopy_256(void *dest, const void *src) +{ + PAGECOPY_128(dest, src ); + PAGECOPY_128(dest + 128, src + 128); +} +#endif + +#if 0 /* XXX enable if detected on the cpu */ +static void pagecopy_ymm8(void *dest, const void *src) +{ + asm volatile("0:\n" + "vmovdqa (%0), %%ymm0\n" + "vmovdqa 32(%0), %%ymm1\n" + "vmovdqa 64(%0), %%ymm2\n" + "vmovdqa 96(%0), %%ymm3\n" + "vmovdqa 128(%0), %%ymm4\n" + "vmovdqa 160(%0), %%ymm5\n" + "vmovdqa 192(%0), %%ymm6\n" + "vmovdqa 224(%0), %%ymm7\n" + "addq $256, %0\n" + "vmovdqa %%ymm0, (%1)\n" + "vmovdqa %%ymm1, 32(%1)\n" + "vmovdqa %%ymm2, 64(%1)\n" + "vmovdqa %%ymm3, 96(%1)\n" + "vmovdqa %%ymm4, 128(%1)\n" + "vmovdqa %%ymm5, 160(%1)\n" + "vmovdqa %%ymm6, 192(%1)\n" + "vmovdqa %%ymm7, 224(%1)\n" + "addq $256, %1\n" + "cmpq %2, %0\n" + "jne 0b" + : "=r"(src), "=r"(dest) + : "r"((char *)src + 4096), "0"(src), "1"(dest) + : "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm5", "xmm6", "xmm7"); +} +#endif diff --git a/c8/stm/pagecopy.h b/c8/stm/pagecopy.h new file mode 100644 --- /dev/null +++ b/c8/stm/pagecopy.h @@ -0,0 +1,2 @@ + +static void pagecopy(void *dest, const void *src); // 4096 bytes diff --git a/c8/stm/pages.c b/c8/stm/pages.c new file mode 100644 --- /dev/null +++ b/c8/stm/pages.c @@ -0,0 +1,46 @@ +#ifndef _STM_CORE_H_ +# error "must be compiled via stmgc.c" +#endif + + +/************************************************************/ + +static void setup_pages(void) +{ +} + +static void teardown_pages(void) +{ + memset(pages_privatized, 0, sizeof(pages_privatized)); +} + +/************************************************************/ + + +static void pages_initialize_private(uintptr_t pagenum, uintptr_t count) +{ + dprintf(("pages_initialize_private: 0x%ld - 0x%ld\n", pagenum, + pagenum + count)); + assert(pagenum < NB_PAGES); + if (count == 0) + return; + + long i; + for (i = 0; i < NB_SEGMENTS; i++) { + spinlock_acquire(get_priv_segment(i)->privatization_lock); + } + + while (count-->0) { + for (i = 0; i < NB_SEGMENTS; i++) { + uint64_t bitmask = 1UL << i; + volatile struct page_shared_s *ps = (volatile struct page_shared_s *) + &pages_privatized[pagenum + count - PAGE_FLAG_START]; + + ps->by_segment |= bitmask; + } + } + + for (i = NB_SEGMENTS-1; i >= 0; i--) { + spinlock_release(get_priv_segment(i)->privatization_lock); + } +} diff --git a/c8/stm/pages.h b/c8/stm/pages.h new file mode 100644 --- /dev/null +++ b/c8/stm/pages.h @@ -0,0 +1,31 @@ + +#define PAGE_FLAG_START END_NURSERY_PAGE +#define PAGE_FLAG_END NB_PAGES + +#define USE_REMAP_FILE_PAGES + +struct page_shared_s { +#if NB_SEGMENTS <= 8 + uint8_t by_segment; +#elif NB_SEGMENTS <= 16 + uint16_t by_segment; +#elif NB_SEGMENTS <= 32 + uint32_t by_segment; +#elif NB_SEGMENTS <= 64 + uint64_t by_segment; +#else +# error "NB_SEGMENTS > 64 not supported right now" +#endif +}; + +static struct page_shared_s pages_privatized[PAGE_FLAG_END - PAGE_FLAG_START]; + +static void pages_initialize_private(uintptr_t pagenum, uintptr_t count); + + +static inline bool is_private_page(long segnum, uintptr_t pagenum) +{ + assert(pagenum >= PAGE_FLAG_START); + uint64_t bitmask = 1UL << segnum; + return (pages_privatized[pagenum - PAGE_FLAG_START].by_segment & bitmask); +} diff --git a/c8/stm/setup.c b/c8/stm/setup.c --- a/c8/stm/setup.c +++ b/c8/stm/setup.c @@ -54,14 +54,8 @@ static void setup_protection_settings(void) { - /* The segment 0 is not used to run transactions, but contains the - shared copy of the pages. We mprotect all pages before so that - accesses fail, up to and including the pages corresponding to the - nurseries of the other segments. */ - mprotect(stm_object_pages, END_NURSERY_PAGE * 4096UL, PROT_NONE); - long i; - for (i = 1; i <= NB_SEGMENTS; i++) { + for (i = 0; i < NB_SEGMENTS; i++) { char *segment_base = get_segment_base(i); /* In each segment, the first page is where TLPREFIX'ed @@ -97,7 +91,7 @@ setup_protection_settings(); long i; - for (i = 1; i <= NB_SEGMENTS; i++) { + for (i = 0; i < NB_SEGMENTS; i++) { char *segment_base = get_segment_base(i); /* Fill the TLS page (page 1) with 0xDC, for debugging */ @@ -108,7 +102,7 @@ /* Initialize STM_PSEGMENT */ struct stm_priv_segment_info_s *pr = get_priv_segment(i); - assert(1 <= i && i < 255); /* 255 is WL_VISITED in gcpage.c */ + assert(0 <= i && i < 255); /* 255 is WL_VISITED in gcpage.c */ pr->pub.segment_num = i; pr->pub.segment_base = segment_base; pr->modified_old_objects = list_create(); @@ -127,6 +121,8 @@ setup_sync(); setup_nursery(); + setup_gcpage(); + setup_pages(); } void stm_teardown(void) @@ -136,7 +132,7 @@ assert(!_has_mutex()); long i; - for (i = 1; i <= NB_SEGMENTS; i++) { + for (i = 0; i < NB_SEGMENTS; i++) { struct stm_priv_segment_info_s *pr = get_priv_segment(i); assert(list_is_empty(pr->objects_pointing_to_nursery)); list_free(pr->objects_pointing_to_nursery); @@ -148,6 +144,8 @@ close_fd_mmap(stm_object_pages_fd); teardown_sync(); + teardown_gcpage(); + teardown_pages(); } static pthread_t *_get_cpth(stm_thread_local_t *tl) @@ -175,7 +173,7 @@ /* assign numbers consecutively, but that's for tests; we could also assign the same number to all of them and they would get their own numbers automatically. */ - num = (num % NB_SEGMENTS) + 1; + num = (num + 1) % NB_SEGMENTS; tl->associated_segment_num = num; *_get_cpth(tl) = pthread_self(); set_gs_register(get_segment_base(num)); diff --git a/c8/stm/sync.c b/c8/stm/sync.c --- a/c8/stm/sync.c +++ b/c8/stm/sync.c @@ -73,7 +73,7 @@ assert(_is_tl_registered(tl)); int num = tl->associated_segment_num; - if (sync_ctl.in_use1[num - 1] == 0) { + if (sync_ctl.in_use1[num] == 0) { /* fast-path: we can get the same segment number than the one we had before. The value stored in GS is still valid. */ #ifdef STM_TESTS @@ -88,8 +88,8 @@ the condition variable. */ int retries; for (retries = 0; retries < NB_SEGMENTS; retries++) { - num = (num % NB_SEGMENTS) + 1; - if (sync_ctl.in_use1[num - 1] == 0) { + num = num % NB_SEGMENTS; + if (sync_ctl.in_use1[num] == 0) { /* we're getting 'num', a different number. */ dprintf(("acquired different segment: %d->%d\n", tl->associated_segment_num, num)); tl->associated_segment_num = num; @@ -105,7 +105,7 @@ return false; got_num: - sync_ctl.in_use1[num - 1] = 1; + sync_ctl.in_use1[num] = 1; assert(STM_SEGMENT->segment_num == num); assert(STM_SEGMENT->running_thread == NULL); STM_SEGMENT->running_thread = tl; @@ -119,8 +119,8 @@ assert(STM_SEGMENT->running_thread == tl); STM_SEGMENT->running_thread = NULL; - assert(sync_ctl.in_use1[tl->associated_segment_num - 1] == 1); - sync_ctl.in_use1[tl->associated_segment_num - 1] = 0; + assert(sync_ctl.in_use1[tl->associated_segment_num] == 1); + sync_ctl.in_use1[tl->associated_segment_num] = 0; } __attribute__((unused)) @@ -132,7 +132,7 @@ bool _stm_in_transaction(stm_thread_local_t *tl) { int num = tl->associated_segment_num; - assert(1 <= num && num <= NB_SEGMENTS); + assert(0 <= num && num < NB_SEGMENTS); return get_segment(num)->running_thread == tl; } diff --git a/c8/stmgc.c b/c8/stmgc.c --- a/c8/stmgc.c +++ b/c8/stmgc.c @@ -2,7 +2,10 @@ #include "stmgc.h" #include "stm/atomic.h" #include "stm/list.h" +#include "stm/pagecopy.h" #include "stm/core.h" +#include "stm/pages.h" +#include "stm/gcpage.h" #include "stm/nursery.h" #include "stm/sync.h" #include "stm/setup.h" @@ -10,8 +13,11 @@ #include "stm/rewind_setjmp.h" #include "stm/list.c" +#include "stm/pagecopy.c" #include "stm/nursery.c" #include "stm/core.c" +#include "stm/pages.c" +#include "stm/gcpage.c" #include "stm/sync.c" #include "stm/setup.c" #include "stm/fprintcolor.c" diff --git a/c8/stmgc.h b/c8/stmgc.h --- a/c8/stmgc.h +++ b/c8/stmgc.h @@ -62,13 +62,15 @@ void _stm_write_slowpath(object_t *); object_t *_stm_allocate_slowpath(ssize_t); + +object_t *_stm_allocate_old(ssize_t size_rounded_up); +char *_stm_real_address(object_t *o); #ifdef STM_TESTS #include bool _stm_was_read(object_t *obj); bool _stm_was_written(object_t *obj); long stm_can_move(object_t *obj); -char *_stm_real_address(object_t *o); void _stm_test_switch(stm_thread_local_t *tl); char *_stm_get_segment_base(long index); diff --git a/c8/test/support.py b/c8/test/support.py --- a/c8/test/support.py +++ b/c8/test/support.py @@ -39,6 +39,7 @@ bool _stm_in_transaction(stm_thread_local_t *tl); int _stm_get_flags(object_t *obj); +object_t *_stm_allocate_old(ssize_t size_rounded_up); long stm_can_move(object_t *obj); char *_stm_real_address(object_t *o); void _stm_test_switch(stm_thread_local_t *tl); From noreply at buildbot.pypy.org Wed Sep 3 13:16:08 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Wed, 3 Sep 2014 13:16:08 +0200 (CEST) Subject: [pypy-commit] stmgc default: deal with unshared-pages-by-default with an explicit obj-sync after allocate_old Message-ID: <20140903111608.221E71C072F@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1338:410a8bb1bea6 Date: 2014-09-03 13:17 +0200 http://bitbucket.org/pypy/stmgc/changeset/410a8bb1bea6/ Log: deal with unshared-pages-by-default with an explicit obj-sync after allocate_old diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -82,6 +82,66 @@ /************************************************************/ +static void _page_wise_synchronize_object_now(object_t *obj) +{ + uintptr_t start = (uintptr_t)obj; + uintptr_t first_page = start / 4096UL; + + char *realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj); + ssize_t obj_size = stmcb_size_rounded_up((struct object_s *)realobj); + assert(obj_size >= 16); + uintptr_t end = start + obj_size; + uintptr_t last_page = (end - 1) / 4096UL; + long i, myself = STM_SEGMENT->segment_num; + + for (; first_page <= last_page; first_page++) { + + uintptr_t copy_size; + if (first_page == last_page) { + /* this is the final fragment */ + copy_size = end - start; + } + else { + /* this is a non-final fragment, going up to the + page's end */ + copy_size = 4096 - (start & 4095); + } + /* double-check that the result fits in one page */ + assert(copy_size > 0); + assert(copy_size + (start & 4095) <= 4096); + + char *dst, *src; + src = REAL_ADDRESS(STM_SEGMENT->segment_base, start); + for (i = 0; i < NB_SEGMENTS; i++) { + if (i == myself) + continue; + + dst = REAL_ADDRESS(get_segment_base(i), start); + if (is_private_page(i, first_page)) { + /* The page is a private page. We need to diffuse this + fragment of object from the shared page to this private + page. */ + if (copy_size == 4096) + pagecopy(dst, src); + else + memcpy(dst, src, copy_size); + } + else { + assert(!memcmp(dst, src, copy_size)); /* same page */ + } + } + + start = (start + 4096) & ~4095; + } +} + +void _push_obj_to_other_segments(object_t *obj) +{ + acquire_privatization_lock(); + _page_wise_synchronize_object_now(obj); + release_privatization_lock(); +} + static void _finish_transaction() { stm_thread_local_t *tl = STM_SEGMENT->running_thread; diff --git a/c8/stmgc.h b/c8/stmgc.h --- a/c8/stmgc.h +++ b/c8/stmgc.h @@ -72,6 +72,7 @@ long stm_can_move(object_t *obj); void _stm_test_switch(stm_thread_local_t *tl); +void _push_obj_to_other_segments(object_t *obj); char *_stm_get_segment_base(long index); bool _stm_in_transaction(stm_thread_local_t *tl); diff --git a/c8/test/support.py b/c8/test/support.py --- a/c8/test/support.py +++ b/c8/test/support.py @@ -38,6 +38,7 @@ char *_stm_get_segment_base(long index); bool _stm_in_transaction(stm_thread_local_t *tl); int _stm_get_flags(object_t *obj); +void _push_obj_to_other_segments(object_t *obj); object_t *_stm_allocate_old(ssize_t size_rounded_up); long stm_can_move(object_t *obj); @@ -215,12 +216,14 @@ o = lib._stm_allocate_old(size) tid = 42 + size lib._set_type_id(o, tid) + lib._push_obj_to_other_segments(o) return o def stm_allocate_old_refs(n): o = lib._stm_allocate_old(HDR + n * WORD) tid = 421420 + n lib._set_type_id(o, tid) + lib._push_obj_to_other_segments(o) return o def stm_allocate(size): From noreply at buildbot.pypy.org Wed Sep 3 13:33:56 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Wed, 3 Sep 2014 13:33:56 +0200 (CEST) Subject: [pypy-commit] stmgc default: do simple minor collections Message-ID: <20140903113356.12C691C320C@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1339:99ad13731c25 Date: 2014-09-03 13:33 +0200 http://bitbucket.org/pypy/stmgc/changeset/99ad13731c25/ Log: do simple minor collections diff --git a/c8/stm/gcpage.h b/c8/stm/gcpage.h --- a/c8/stm/gcpage.h +++ b/c8/stm/gcpage.h @@ -5,3 +5,4 @@ static void setup_gcpage(void); static void teardown_gcpage(void); +static char *allocate_outside_nursery_large(uint64_t size); diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c --- a/c8/stm/nursery.c +++ b/c8/stm/nursery.c @@ -30,6 +30,11 @@ return (uintptr_t)obj < NURSERY_END; } +static inline bool _is_young(object_t *obj) +{ + return _is_in_nursery(obj); /* XXX: young_outside_nursery */ +} + long stm_can_move(object_t *obj) { /* 'long' return value to avoid using 'bool' in the public interface */ @@ -38,6 +43,83 @@ /************************************************************/ +#define GCWORD_MOVED ((object_t *) -1) + +static void minor_trace_if_young(object_t **pobj) +{ + /* takes a normal pointer to a thread-local pointer to an object */ + object_t *obj = *pobj; + object_t *nobj; + char *realobj; + size_t size; + + if (obj == NULL) + return; + assert((uintptr_t)obj < NB_PAGES * 4096UL); + + if (_is_in_nursery(obj)) { + /* If the object was already seen here, its first word was set + to GCWORD_MOVED. In that case, the forwarding location, i.e. + where the object moved to, is stored in the second word in 'obj'. */ + object_t *TLPREFIX *pforwarded_array = (object_t *TLPREFIX *)obj; + + realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj); + size = stmcb_size_rounded_up((struct object_s *)realobj); + + /* XXX: small objs */ + char *allocated = allocate_outside_nursery_large(size); + nobj = (object_t *)(allocated - stm_object_pages); + + char *realnobj = REAL_ADDRESS(STM_SEGMENT->segment_base, nobj); + memcpy(realnobj, realobj, size); + + pforwarded_array[0] = GCWORD_MOVED; + pforwarded_array[1] = nobj; + *pobj = nobj; + } + else { + /* XXX: young_outside_nursery */ + return; + } + + /* Must trace the object later */ + LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, (uintptr_t)nobj); +} + + +static inline void _collect_now(object_t *obj) +{ + assert(!_is_young(obj)); + + dprintf(("_collect_now: %p\n", obj)); + + assert(!(obj->stm_flags & GCFLAG_WRITE_BARRIER)); + + char *realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj); + stmcb_trace((struct object_s *)realobj, &minor_trace_if_young); + + obj->stm_flags |= GCFLAG_WRITE_BARRIER; +} + + +static void collect_oldrefs_to_nursery(void) +{ + dprintf(("collect_oldrefs_to_nursery\n")); + struct list_s *lst = STM_PSEGMENT->objects_pointing_to_nursery; + + while (!list_is_empty(lst)) { + object_t *obj = (object_t *)list_pop_item(lst);; + + _collect_now(obj); + + /* XXX: only if commit now and only for big objs */ + _push_obj_to_other_segments(obj); + + /* the list could have moved while appending */ + lst = STM_PSEGMENT->objects_pointing_to_nursery; + } +} + static size_t throw_away_nursery(struct stm_priv_segment_info_s *pseg) { @@ -77,6 +159,8 @@ { dprintf(("minor_collection commit=%d\n", (int)commit)); + collect_oldrefs_to_nursery(); + assert(list_is_empty(STM_PSEGMENT->objects_pointing_to_nursery)); throw_away_nursery(get_priv_segment(STM_SEGMENT->segment_num)); diff --git a/c8/stmgc.h b/c8/stmgc.h --- a/c8/stmgc.h +++ b/c8/stmgc.h @@ -109,6 +109,7 @@ }; extern ssize_t stmcb_size_rounded_up(struct object_s *); +void stmcb_trace(struct object_s *obj, void visit(object_t **)); __attribute__((always_inline)) static inline void stm_read(object_t *obj) diff --git a/c8/test/support.py b/c8/test/support.py --- a/c8/test/support.py +++ b/c8/test/support.py @@ -182,6 +182,22 @@ } } + +void stmcb_trace(struct object_s *obj, void visit(object_t **)) +{ + int i; + struct myobj_s *myobj = (struct myobj_s*)obj; + if (myobj->type_id < 421420) { + /* basic case: no references */ + return; + } + for (i=0; i < myobj->type_id - 421420; i++) { + object_t **ref = ((object_t **)(myobj + 1)) + i; + visit(ref); + } +} + + ''', sources=source_files, define_macros=[('STM_TESTS', '1'), ('STM_NO_AUTOMATIC_SETJMP', '1'), diff --git a/c8/test/test_basic.py b/c8/test/test_basic.py --- a/c8/test/test_basic.py +++ b/c8/test/test_basic.py @@ -62,6 +62,7 @@ def test_write_on_old(self): lp1 = stm_allocate_old(16) + self.start_transaction() assert stm_get_char(lp1) == '\0' stm_write(lp1) @@ -80,6 +81,8 @@ def test_read_write_1(self): lp1 = stm_allocate_old(16) stm_get_real_address(lp1)[HDR] = 'a' #setchar + lib._push_obj_to_other_segments(lp1) + # self.start_transaction() self.commit_transaction() # @@ -88,7 +91,7 @@ assert modified_old_objects() == [] stm_write(lp1) assert modified_old_objects() == [lp1] - assert objects_pointing_to_nursery() == None + assert objects_pointing_to_nursery() == [lp1] assert stm_get_char(lp1) == 'a' stm_set_char(lp1, 'b') # From noreply at buildbot.pypy.org Wed Sep 3 14:55:41 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Wed, 3 Sep 2014 14:55:41 +0200 (CEST) Subject: [pypy-commit] stmgc default: start using a global commit log Message-ID: <20140903125541.4FB321D3538@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1340:03fc557e29dd Date: 2014-09-03 14:56 +0200 http://bitbucket.org/pypy/stmgc/changeset/03fc557e29dd/ Log: start using a global commit log diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -3,6 +3,85 @@ #endif +/* ############# commit log ############# */ + +void _dbg_print_commit_log() +{ + struct stm_commit_log_entry_s *cl = &commit_log_root; + + fprintf(stderr, "root (%p, %d)\n", cl->next, cl->segment_num); + while ((cl = cl->next)) { + size_t i = 0; + fprintf(stderr, "elem (%p, %d)\n", cl->next, cl->segment_num); + object_t *obj; + do { + obj = cl->written[i]; + fprintf(stderr, "-> %p\n", obj); + i++; + } while ((obj = cl->written[i])); + } +} + +void stm_validate(void *free_if_abort) +{ + struct stm_commit_log_entry_s *cl = STM_PSEGMENT->last_commit_log_entry; + + /* Don't check 'cl'. This entry is already checked */ + while ((cl = cl->next)) { + size_t i = 0; + OPT_ASSERT(cl->segment_num >= 0 && cl->segment_num < NB_SEGMENTS); + + object_t *obj; + while ((obj = cl->written[i])) { + /* XXX: update our copies */ + + if (_stm_was_read(obj)) { + free(free_if_abort); + stm_abort_transaction(); + } + + i++; + }; + + /* last fully validated entry */ + STM_PSEGMENT->last_commit_log_entry = cl; + } +} + +static struct stm_commit_log_entry_s *_create_commit_log_entry() +{ + struct list_s *lst = STM_PSEGMENT->modified_old_objects; + size_t count = list_count(lst); + size_t byte_len = sizeof(struct stm_commit_log_entry_s) + (count + 1) * sizeof(object_t*); + struct stm_commit_log_entry_s *result = malloc(byte_len); + int i; + + result->next = NULL; + result->segment_num = STM_SEGMENT->segment_num; + for (i = 0; i < count; i++) { + result->written[i] = (object_t*)list_item(lst, i); + } + result->written[count] = NULL; + + return result; +} + +static void _validate_and_add_to_commit_log() +{ + struct stm_commit_log_entry_s *new; + void* *to; + + new = _create_commit_log_entry(); + fprintf(stderr,"%p\n", new); + do { + stm_validate(new); + + to = (void **)&(STM_PSEGMENT->last_commit_log_entry->next); + } while (!__sync_bool_compare_and_swap((void**)to, (void**)NULL, (void**)new)); +} + +/* ############# STM ############# */ + void _stm_write_slowpath(object_t *obj) { assert(_seems_to_be_running_transaction()); @@ -159,6 +238,8 @@ minor_collection(1); + _validate_and_add_to_commit_log(); + s_mutex_lock(); assert(STM_SEGMENT->nursery_end == NURSERY_END); diff --git a/c8/stm/core.h b/c8/stm/core.h --- a/c8/stm/core.h +++ b/c8/stm/core.h @@ -52,12 +52,22 @@ struct list_s *objects_pointing_to_nursery; uint8_t privatization_lock; + struct stm_commit_log_entry_s *last_commit_log_entry; + /* For debugging */ #ifndef NDEBUG pthread_t running_pthread; #endif }; +/* Commit Log things */ +struct stm_commit_log_entry_s { + struct stm_commit_log_entry_s *next; + int segment_num; + object_t *written[]; /* terminated with a NULL ptr */ +}; +static struct stm_commit_log_entry_s commit_log_root = {NULL, -1}; + static char *stm_object_pages; static int stm_object_pages_fd; diff --git a/c8/stm/setup.c b/c8/stm/setup.c --- a/c8/stm/setup.c +++ b/c8/stm/setup.c @@ -107,6 +107,7 @@ pr->pub.segment_base = segment_base; pr->modified_old_objects = list_create(); pr->objects_pointing_to_nursery = list_create(); + pr->last_commit_log_entry = &commit_log_root; pr->pub.transaction_read_version = 0xff; } diff --git a/c8/stmgc.h b/c8/stmgc.h --- a/c8/stmgc.h +++ b/c8/stmgc.h @@ -67,6 +67,7 @@ char *_stm_real_address(object_t *o); #ifdef STM_TESTS #include +void stm_validate(void *free_if_abort); bool _stm_was_read(object_t *obj); bool _stm_was_written(object_t *obj); diff --git a/c8/test/support.py b/c8/test/support.py --- a/c8/test/support.py +++ b/c8/test/support.py @@ -31,6 +31,8 @@ void stm_teardown(void); void stm_register_thread_local(stm_thread_local_t *tl); void stm_unregister_thread_local(stm_thread_local_t *tl); +void stm_validate(void *free_if_abort); +bool _check_stm_validate(); bool _checked_stm_write(object_t *obj); bool _stm_was_read(object_t *obj); @@ -124,6 +126,10 @@ CHECKED(stm_abort_transaction()); } +bool _check_stm_validate(void) { + CHECKED(stm_validate(NULL)); +} + #undef CHECKED @@ -314,6 +320,9 @@ def stm_was_written_card(o): return lib._stm_was_written_card(o) +def stm_validate(): + if lib._check_stm_validate(): + raise Conflict() def stm_start_safe_point(): lib._stm_start_safe_point() @@ -433,6 +442,7 @@ # if lib._stm_in_transaction(tl2): lib._stm_test_switch(tl2) + stm_validate() # can raise def push_root(self, o): assert 0 From noreply at buildbot.pypy.org Wed Sep 3 15:44:40 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Wed, 3 Sep 2014 15:44:40 +0200 (CEST) Subject: [pypy-commit] stmgc default: re-add shadowstack Message-ID: <20140903134440.DA4381D2678@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1341:4a48705ca8ce Date: 2014-09-03 15:45 +0200 http://bitbucket.org/pypy/stmgc/changeset/4a48705ca8ce/ Log: re-add shadowstack diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -130,6 +130,7 @@ #ifndef NDEBUG STM_PSEGMENT->running_pthread = pthread_self(); #endif + STM_PSEGMENT->shadowstack_at_start_of_transaction = tl->shadowstack; dprintf(("start_transaction\n")); @@ -260,11 +261,30 @@ #pragma push_macro("STM_SEGMENT") #undef STM_PSEGMENT #undef STM_SEGMENT - /* struct stm_priv_segment_info_s *pseg = get_priv_segment(segment_num); */ + struct stm_priv_segment_info_s *pseg = get_priv_segment(segment_num); - /* throw_away_nursery(pseg); */ + throw_away_nursery(pseg); - /* reset_modified_from_other_segments(segment_num); */ + /* XXX: reset_modified_from_other_segments(segment_num); */ + + stm_thread_local_t *tl = pseg->pub.running_thread; +#ifdef STM_NO_AUTOMATIC_SETJMP + /* In tests, we don't save and restore the shadowstack correctly. + Be sure to not change items below shadowstack_at_start_of_transaction. + There is no such restrictions in non-Python-based tests. */ + assert(tl->shadowstack >= pseg->shadowstack_at_start_of_transaction); + tl->shadowstack = pseg->shadowstack_at_start_of_transaction; +#else + /* NB. careful, this function might be called more than once to + abort a given segment. Make sure that + stm_rewind_jmp_restore_shadowstack() is idempotent. */ + /* we need to do this here and not directly in rewind_longjmp() because + that is called when we already released everything (safe point) + and a concurrent major GC could mess things up. */ + if (tl->shadowstack != NULL) + stm_rewind_jmp_restore_shadowstack(tl); + assert(tl->shadowstack == pseg->shadowstack_at_start_of_transaction); +#endif #pragma pop_macro("STM_SEGMENT") #pragma pop_macro("STM_PSEGMENT") diff --git a/c8/stm/core.h b/c8/stm/core.h --- a/c8/stm/core.h +++ b/c8/stm/core.h @@ -54,6 +54,8 @@ struct stm_commit_log_entry_s *last_commit_log_entry; + struct stm_shadowentry_s *shadowstack_at_start_of_transaction; + /* For debugging */ #ifndef NDEBUG pthread_t running_pthread; diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c --- a/c8/stm/nursery.c +++ b/c8/stm/nursery.c @@ -87,6 +87,33 @@ } +static void collect_roots_in_nursery(void) +{ + stm_thread_local_t *tl = STM_SEGMENT->running_thread; + struct stm_shadowentry_s *current = tl->shadowstack; + struct stm_shadowentry_s *finalbase = tl->shadowstack_base; + struct stm_shadowentry_s *ssbase; + ssbase = (struct stm_shadowentry_s *)tl->rjthread.moved_off_ssbase; + if (ssbase == NULL) + ssbase = finalbase; + else + assert(finalbase <= ssbase && ssbase <= current); + + while (current > ssbase) { + --current; + uintptr_t x = (uintptr_t)current->ss; + + if ((x & 3) == 0) { + /* the stack entry is a regular pointer (possibly NULL) */ + minor_trace_if_young(¤t->ss); + } + else { + /* it is an odd-valued marker, ignore */ + } + } +} + + static inline void _collect_now(object_t *obj) { assert(!_is_young(obj)); @@ -159,6 +186,8 @@ { dprintf(("minor_collection commit=%d\n", (int)commit)); + collect_roots_in_nursery(); + collect_oldrefs_to_nursery(); assert(list_is_empty(STM_PSEGMENT->objects_pointing_to_nursery)); diff --git a/c8/stm/setup.c b/c8/stm/setup.c --- a/c8/stm/setup.c +++ b/c8/stm/setup.c @@ -149,6 +149,45 @@ teardown_pages(); } +static void _shadowstack_trap_page(char *start, int prot) +{ + size_t bsize = STM_SHADOW_STACK_DEPTH * sizeof(struct stm_shadowentry_s); + char *end = start + bsize + 4095; + end -= (((uintptr_t)end) & 4095); + mprotect(end, 4096, prot); +} + +static void _init_shadow_stack(stm_thread_local_t *tl) +{ + size_t bsize = STM_SHADOW_STACK_DEPTH * sizeof(struct stm_shadowentry_s); + char *start = malloc(bsize + 8192); /* for the trap page, plus rounding */ + if (!start) + stm_fatalerror("can't allocate shadow stack"); + + /* set up a trap page: if the shadowstack overflows, it will + crash in a clean segfault */ + _shadowstack_trap_page(start, PROT_NONE); + + struct stm_shadowentry_s *s = (struct stm_shadowentry_s *)start; + tl->shadowstack = s; + tl->shadowstack_base = s; + STM_PUSH_ROOT(*tl, -1); +} + +static void _done_shadow_stack(stm_thread_local_t *tl) +{ + assert(tl->shadowstack > tl->shadowstack_base); + assert(tl->shadowstack_base->ss == (object_t *)-1); + + char *start = (char *)tl->shadowstack_base; + _shadowstack_trap_page(start, PROT_READ | PROT_WRITE); + + free(tl->shadowstack_base); + tl->shadowstack = NULL; + tl->shadowstack_base = NULL; +} + + static pthread_t *_get_cpth(stm_thread_local_t *tl) { assert(sizeof(pthread_t) <= sizeof(tl->creating_pthread)); @@ -177,6 +216,7 @@ num = (num + 1) % NB_SEGMENTS; tl->associated_segment_num = num; *_get_cpth(tl) = pthread_self(); + _init_shadow_stack(tl); set_gs_register(get_segment_base(num)); s_mutex_unlock(); } @@ -186,7 +226,7 @@ s_mutex_lock(); assert(tl->prev != NULL); assert(tl->next != NULL); - + _done_shadow_stack(tl); if (tl == stm_all_thread_locals) { stm_all_thread_locals = stm_all_thread_locals->next; if (tl == stm_all_thread_locals) { diff --git a/c8/stmgc.h b/c8/stmgc.h --- a/c8/stmgc.h +++ b/c8/stmgc.h @@ -48,9 +48,16 @@ #define STM_SEGMENT ((stm_segment_info_t *)4352) +struct stm_shadowentry_s { + /* Like stm_read_marker_s, this is a struct to enable better + aliasing analysis in the C code. */ + object_t *ss; +}; + typedef struct stm_thread_local_s { /* rewind_setjmp's interface */ rewind_jmp_thread rjthread; + struct stm_shadowentry_s *shadowstack, *shadowstack_base; /* the next fields are handled internally by the library */ int associated_segment_num; struct stm_thread_local_s *prev, *next; @@ -146,6 +153,10 @@ void stm_setup(void); void stm_teardown(void); +#define STM_SHADOW_STACK_DEPTH 163840 +#define STM_PUSH_ROOT(tl, p) ((tl).shadowstack++->ss = (object_t *)(p)) +#define STM_POP_ROOT(tl, p) ((p) = (typeof(p))((--(tl).shadowstack)->ss)) +#define STM_POP_ROOT_RET(tl) ((--(tl).shadowstack)->ss) void stm_register_thread_local(stm_thread_local_t *tl); void stm_unregister_thread_local(stm_thread_local_t *tl); @@ -162,6 +173,13 @@ rewind_jmp_longjmp(&(tl)->rjthread) #define stm_rewind_jmp_forget(tl) \ rewind_jmp_forget(&(tl)->rjthread) +#define stm_rewind_jmp_restore_shadowstack(tl) do { \ + assert(rewind_jmp_armed(&(tl)->rjthread)); \ + (tl)->shadowstack = (struct stm_shadowentry_s *) \ + rewind_jmp_restore_shadowstack(&(tl)->rjthread); \ +} while (0) +#define stm_rewind_jmp_enum_shadowstack(tl, callback) \ + rewind_jmp_enum_shadowstack(&(tl)->rjthread, callback) long stm_start_transaction(stm_thread_local_t *tl); diff --git a/c8/test/support.py b/c8/test/support.py --- a/c8/test/support.py +++ b/c8/test/support.py @@ -15,8 +15,14 @@ ...; } rewind_jmp_thread; +struct stm_shadowentry_s { + object_t *ss; +}; + + typedef struct { rewind_jmp_thread rjthread; + struct stm_shadowentry_s *shadowstack, *shadowstack_base; int associated_segment_num; struct stm_thread_local_s *prev, *next; void *creating_pthread[2]; @@ -372,10 +378,15 @@ +SHADOWSTACK_LENGTH = 1000 _keepalive = weakref.WeakKeyDictionary() def _allocate_thread_local(): tl = ffi.new("stm_thread_local_t *") + ss = ffi.new("struct stm_shadowentry_s[]", SHADOWSTACK_LENGTH) + _keepalive[tl] = ss + tl.shadowstack = ss + tl.shadowstack_base = ss lib.stm_register_thread_local(tl) return tl @@ -445,10 +456,22 @@ stm_validate() # can raise def push_root(self, o): - assert 0 + assert ffi.typeof(o) == ffi.typeof("object_t *") + tl = self.tls[self.current_thread] + curlength = tl.shadowstack - tl.shadowstack_base + assert 0 <= curlength < SHADOWSTACK_LENGTH + tl.shadowstack[0].ss = ffi.cast("object_t *", o) + tl.shadowstack += 1 def pop_root(self): - assert 0 + tl = self.tls[self.current_thread] + curlength = tl.shadowstack - tl.shadowstack_base + assert curlength >= 1 + if curlength == 1: + raise EmptyStack + assert 0 < curlength <= SHADOWSTACK_LENGTH + tl.shadowstack -= 1 + return ffi.cast("object_t *", tl.shadowstack[0].ss) def push_root_no_gc(self): "Pushes an invalid object, to crash in case the GC is called" From noreply at buildbot.pypy.org Wed Sep 3 16:11:13 2014 From: noreply at buildbot.pypy.org (ISF) Date: Wed, 3 Sep 2014 16:11:13 +0200 (CEST) Subject: [pypy-commit] pypy ppc-updated-backend: Don't use autopath, set PYTHONPATH instead Message-ID: <20140903141113.88B101C320C@cobra.cs.uni-duesseldorf.de> Author: Ivan Sichmann Freitas Branch: ppc-updated-backend Changeset: r73285:9ed110ab86ea Date: 2014-08-28 13:44 -0300 http://bitbucket.org/pypy/pypy/changeset/9ed110ab86ea/ Log: Don't use autopath, set PYTHONPATH instead diff --git a/rpython/jit/backend/ppc/tool/viewcode.py b/rpython/jit/backend/ppc/tool/viewcode.py --- a/rpython/jit/backend/ppc/tool/viewcode.py +++ b/rpython/jit/backend/ppc/tool/viewcode.py @@ -8,7 +8,6 @@ ./viewcode.py log # also includes a pygame viewer """ -import autopath import new import operator import py From noreply at buildbot.pypy.org Wed Sep 3 16:11:14 2014 From: noreply at buildbot.pypy.org (ISF) Date: Wed, 3 Sep 2014 16:11:14 +0200 (CEST) Subject: [pypy-commit] pypy ppc-updated-backend: Add support for disassembling big endian ppc dumps Message-ID: <20140903141114.C29B61C320C@cobra.cs.uni-duesseldorf.de> Author: Ivan Sichmann Freitas Branch: ppc-updated-backend Changeset: r73286:1ce94dc202c5 Date: 2014-09-03 14:10 -0300 http://bitbucket.org/pypy/pypy/changeset/1ce94dc202c5/ Log: Add support for disassembling big endian ppc dumps diff --git a/rpython/jit/backend/tool/viewcode.py b/rpython/jit/backend/tool/viewcode.py --- a/rpython/jit/backend/tool/viewcode.py +++ b/rpython/jit/backend/tool/viewcode.py @@ -60,9 +60,22 @@ 'i386': 'i386', 'arm': 'arm', 'arm_32': 'arm', + 'ppc' : 'powerpc:common64', + } + machine_endianness = { + 'x86': 'little', + 'x86-without-sse2': 'little', + 'x86_32': 'little', + 'x86_64': 'little', + 'x86-64': 'little', + 'i386': 'little', + 'arm': 'little', + 'arm_32': 'little', + 'ppc' : 'big', } cmd = find_objdump() objdump = ('%(command)s -b binary -m %(machine)s ' + '--endian=%(endianness)s ' '--disassembler-options=intel-mnemonics ' '--adjust-vma=%(origin)d -D %(file)s') # @@ -74,6 +87,7 @@ 'file': tmpfile, 'origin': originaddr, 'machine': objdump_machine_option[backend_name], + 'endianness': machine_endianness[backend_name], }, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = p.communicate() assert not p.returncode, ('Encountered an error running objdump: %s' % From noreply at buildbot.pypy.org Wed Sep 3 16:44:20 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Wed, 3 Sep 2014 16:44:20 +0200 (CEST) Subject: [pypy-commit] stmgc default: start with some signal handling Message-ID: <20140903144420.0AB841D2678@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1342:d02e51a1b9e1 Date: 2014-09-03 16:45 +0200 http://bitbucket.org/pypy/stmgc/changeset/d02e51a1b9e1/ Log: start with some signal handling diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -2,9 +2,23 @@ # error "must be compiled via stmgc.c" #endif +#include + + +/* ############# signal handler ############# */ + +void _signal_handler(int sig, siginfo_t *siginfo, void *context) +{ + + if (siginfo->si_addr == NULL) { + /* send to GDB */ + kill(getpid(), SIGINT); + } +} /* ############# commit log ############# */ + void _dbg_print_commit_log() { struct stm_commit_log_entry_s *cl = &commit_log_root; @@ -90,6 +104,26 @@ stm_read(obj); + /* make other segments trap if accessing this object */ + uintptr_t first_page = ((uintptr_t)obj) / 4096UL; + char *realobj; + size_t obj_size; + uintptr_t i, end_page; + + realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj); + obj_size = stmcb_size_rounded_up((struct object_s *)realobj); + end_page = (((uintptr_t)obj) + obj_size - 1) / 4096UL; + + for (i = 0; i < NB_SEGMENTS; i++) { + if (i == STM_SEGMENT->segment_num) + continue; + + char *segment_base = get_segment_base(i); + mprotect(segment_base + first_page * 4096, + (end_page - first_page + 1) * 4096, PROT_NONE); + dprintf(("prot %lu, len=%lu in seg %d\n", first_page, (end_page - first_page + 1), i)); + } + LIST_APPEND(STM_PSEGMENT->modified_old_objects, obj); LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, obj); @@ -162,66 +196,6 @@ /************************************************************/ -static void _page_wise_synchronize_object_now(object_t *obj) -{ - uintptr_t start = (uintptr_t)obj; - uintptr_t first_page = start / 4096UL; - - char *realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj); - ssize_t obj_size = stmcb_size_rounded_up((struct object_s *)realobj); - assert(obj_size >= 16); - uintptr_t end = start + obj_size; - uintptr_t last_page = (end - 1) / 4096UL; - long i, myself = STM_SEGMENT->segment_num; - - for (; first_page <= last_page; first_page++) { - - uintptr_t copy_size; - if (first_page == last_page) { - /* this is the final fragment */ - copy_size = end - start; - } - else { - /* this is a non-final fragment, going up to the - page's end */ - copy_size = 4096 - (start & 4095); - } - /* double-check that the result fits in one page */ - assert(copy_size > 0); - assert(copy_size + (start & 4095) <= 4096); - - char *dst, *src; - src = REAL_ADDRESS(STM_SEGMENT->segment_base, start); - for (i = 0; i < NB_SEGMENTS; i++) { - if (i == myself) - continue; - - dst = REAL_ADDRESS(get_segment_base(i), start); - if (is_private_page(i, first_page)) { - /* The page is a private page. We need to diffuse this - fragment of object from the shared page to this private - page. */ - if (copy_size == 4096) - pagecopy(dst, src); - else - memcpy(dst, src, copy_size); - } - else { - assert(!memcmp(dst, src, copy_size)); /* same page */ - } - } - - start = (start + 4096) & ~4095; - } -} - -void _push_obj_to_other_segments(object_t *obj) -{ - acquire_privatization_lock(); - _page_wise_synchronize_object_now(obj); - release_privatization_lock(); -} - static void _finish_transaction() { stm_thread_local_t *tl = STM_SEGMENT->running_thread; diff --git a/c8/stm/core.h b/c8/stm/core.h --- a/c8/stm/core.h +++ b/c8/stm/core.h @@ -20,7 +20,7 @@ #define MAP_PAGES_FLAGS (MAP_SHARED | MAP_ANONYMOUS | MAP_NORESERVE) #define NB_NURSERY_PAGES (STM_GC_NURSERY/4) -#define TOTAL_MEMORY (NB_PAGES * 4096UL * (1 + NB_SEGMENTS)) +#define TOTAL_MEMORY (NB_PAGES * 4096UL * NB_SEGMENTS) #define READMARKER_END ((NB_PAGES * 4096UL) >> 4) #define FIRST_OBJECT_PAGE ((READMARKER_END + 4095) / 4096UL) #define FIRST_NURSERY_PAGE FIRST_OBJECT_PAGE diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c --- a/c8/stm/gcpage.c +++ b/c8/stm/gcpage.c @@ -15,7 +15,7 @@ static void setup_N_pages(char *pages_addr, uint64_t num) { - pages_initialize_private((pages_addr - stm_object_pages) / 4096UL, num); + pages_initialize_shared((pages_addr - stm_object_pages) / 4096UL, num); } diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c --- a/c8/stm/nursery.c +++ b/c8/stm/nursery.c @@ -139,9 +139,6 @@ _collect_now(obj); - /* XXX: only if commit now and only for big objs */ - _push_obj_to_other_segments(obj); - /* the list could have moved while appending */ lst = STM_PSEGMENT->objects_pointing_to_nursery; } diff --git a/c8/stm/pages.c b/c8/stm/pages.c --- a/c8/stm/pages.c +++ b/c8/stm/pages.c @@ -16,31 +16,55 @@ /************************************************************/ +static void d_remap_file_pages(char *addr, size_t size, ssize_t pgoff) +{ + dprintf(("remap_file_pages: 0x%lx bytes: (seg%ld %p) --> (seg%ld %p)\n", + (long)size, + (long)((addr - stm_object_pages) / 4096UL) / NB_PAGES, + (void *)((addr - stm_object_pages) % (4096UL * NB_PAGES)), + (long)pgoff / NB_PAGES, + (void *)((pgoff % NB_PAGES) * 4096UL))); + assert(size % 4096 == 0); + assert(size <= TOTAL_MEMORY); + assert(((uintptr_t)addr) % 4096 == 0); + assert(addr >= stm_object_pages); + assert(addr <= stm_object_pages + TOTAL_MEMORY - size); + assert(pgoff >= 0); + assert(pgoff <= (TOTAL_MEMORY - size) / 4096UL); -static void pages_initialize_private(uintptr_t pagenum, uintptr_t count) + /* assert remappings follow the rule that page N in one segment + can only be remapped to page N in another segment */ + assert(((addr - stm_object_pages) / 4096UL - pgoff) % NB_PAGES == 0); + +#ifdef USE_REMAP_FILE_PAGES + int res = remap_file_pages(addr, size, 0, pgoff, 0); + if (UNLIKELY(res < 0)) + stm_fatalerror("remap_file_pages: %m"); +#else + char *res = mmap(addr, size, + PROT_READ | PROT_WRITE, + (MAP_PAGES_FLAGS & ~MAP_ANONYMOUS) | MAP_FIXED, + stm_object_pages_fd, pgoff * 4096UL); + if (UNLIKELY(res != addr)) + stm_fatalerror("mmap (remapping page): %m"); +#endif +} + + +static void pages_initialize_shared(uintptr_t pagenum, uintptr_t count) { - dprintf(("pages_initialize_private: 0x%ld - 0x%ld\n", pagenum, + /* call remap_file_pages() to make all pages in the range(pagenum, + pagenum+count) refer to the same physical range of pages from + segment 0. */ + dprintf(("pages_initialize_shared: 0x%ld - 0x%ld\n", pagenum, pagenum + count)); assert(pagenum < NB_PAGES); if (count == 0) return; - - long i; - for (i = 0; i < NB_SEGMENTS; i++) { - spinlock_acquire(get_priv_segment(i)->privatization_lock); - } - - while (count-->0) { - for (i = 0; i < NB_SEGMENTS; i++) { - uint64_t bitmask = 1UL << i; - volatile struct page_shared_s *ps = (volatile struct page_shared_s *) - &pages_privatized[pagenum + count - PAGE_FLAG_START]; - - ps->by_segment |= bitmask; - } - } - - for (i = NB_SEGMENTS-1; i >= 0; i--) { - spinlock_release(get_priv_segment(i)->privatization_lock); + uintptr_t i; + for (i = 1; i < NB_SEGMENTS; i++) { + char *segment_base = get_segment_base(i); + d_remap_file_pages(segment_base + pagenum * 4096UL, + count * 4096UL, pagenum); } } diff --git a/c8/stm/pages.h b/c8/stm/pages.h --- a/c8/stm/pages.h +++ b/c8/stm/pages.h @@ -20,8 +20,7 @@ static struct page_shared_s pages_privatized[PAGE_FLAG_END - PAGE_FLAG_START]; -static void pages_initialize_private(uintptr_t pagenum, uintptr_t count); - +static void pages_initialize_shared(uintptr_t pagenum, uintptr_t count); static inline bool is_private_page(long segnum, uintptr_t pagenum) { diff --git a/c8/stm/setup.c b/c8/stm/setup.c --- a/c8/stm/setup.c +++ b/c8/stm/setup.c @@ -2,6 +2,7 @@ # error "must be compiled via stmgc.c" #endif +#include #ifdef USE_REMAP_FILE_PAGES static char *setup_mmap(char *reason, int *ignored) @@ -67,9 +68,32 @@ mprotect(segment_base + 8192, (FIRST_READMARKER_PAGE - 2) * 4096UL, PROT_NONE); + + if (i != 0) { + /* let's give all pages to segment 0 at first and make them + write-inaccessible everywhere else */ + mprotect(segment_base + END_NURSERY_PAGE * 4096, + (NB_PAGES - END_NURSERY_PAGE) * 4096, + PROT_READ); + } } } +static void setup_signal_handler(void) +{ + struct sigaction act; + memset(&act, 0, sizeof(act)); + + act.sa_sigaction = &_signal_handler; + /* The SA_SIGINFO flag tells sigaction() to use the sa_sigaction field, not sa_handler. */ + act.sa_flags = SA_SIGINFO; + + if (sigaction(SIGSEGV, &act, NULL) < 0) { + perror ("sigaction"); + abort(); + } +} + void stm_setup(void) { /* Check that some values are acceptable */ @@ -89,6 +113,7 @@ stm_object_pages = setup_mmap("initial stm_object_pages mmap()", &stm_object_pages_fd); setup_protection_settings(); + setup_signal_handler(); long i; for (i = 0; i < NB_SEGMENTS; i++) { diff --git a/c8/test/support.py b/c8/test/support.py --- a/c8/test/support.py +++ b/c8/test/support.py @@ -46,7 +46,6 @@ char *_stm_get_segment_base(long index); bool _stm_in_transaction(stm_thread_local_t *tl); int _stm_get_flags(object_t *obj); -void _push_obj_to_other_segments(object_t *obj); object_t *_stm_allocate_old(ssize_t size_rounded_up); long stm_can_move(object_t *obj); @@ -244,14 +243,12 @@ o = lib._stm_allocate_old(size) tid = 42 + size lib._set_type_id(o, tid) - lib._push_obj_to_other_segments(o) return o def stm_allocate_old_refs(n): o = lib._stm_allocate_old(HDR + n * WORD) tid = 421420 + n lib._set_type_id(o, tid) - lib._push_obj_to_other_segments(o) return o def stm_allocate(size): From noreply at buildbot.pypy.org Wed Sep 3 16:55:15 2014 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 3 Sep 2014 16:55:15 +0200 (CEST) Subject: [pypy-commit] pypy default: Be safe: it seems we can get unicode strings here, and the print_() function Message-ID: <20140903145515.69F741D2678@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73287:2765b0c6e842 Date: 2014-09-03 16:54 +0200 http://bitbucket.org/pypy/pypy/changeset/2765b0c6e842/ Log: Be safe: it seems we can get unicode strings here, and the print_() function just calls str(x) on them. diff --git a/_pytest/resultlog.py b/_pytest/resultlog.py --- a/_pytest/resultlog.py +++ b/_pytest/resultlog.py @@ -54,15 +54,15 @@ self.logfile = logfile # preferably line buffered def write_log_entry(self, testpath, lettercode, longrepr, sections=None): - py.builtin.print_("%s %s" % (lettercode, testpath), file=self.logfile) + _safeprint("%s %s" % (lettercode, testpath), file=self.logfile) for line in longrepr.splitlines(): - py.builtin.print_(" %s" % line, file=self.logfile) + _safeprint(" %s" % line, file=self.logfile) if sections is not None: for title, content in sections: - py.builtin.print_(" ---------- %s ----------" % (title,), - file=self.logfile) + _safeprint(" ---------- %s ----------" % (title,), + file=self.logfile) for line in content.splitlines(): - py.builtin.print_(" %s" % line, file=self.logfile) + _safeprint(" %s" % line, file=self.logfile) def log_outcome(self, report, lettercode, longrepr): testpath = getattr(report, 'nodeid', None) @@ -105,3 +105,8 @@ if path is None: path = "cwd:%s" % py.path.local() self.write_log_entry(path, '!', str(excrepr)) + +def _safeprint(s, file): + if isinstance(s, unicode): + s = s.encode('utf-8') + py.builtin.print_(s) From noreply at buildbot.pypy.org Wed Sep 3 16:58:05 2014 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 3 Sep 2014 16:58:05 +0200 (CEST) Subject: [pypy-commit] pypy default: oups Message-ID: <20140903145805.D75B41D2678@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73288:fd526e56da5a Date: 2014-09-03 16:57 +0200 http://bitbucket.org/pypy/pypy/changeset/fd526e56da5a/ Log: oups diff --git a/_pytest/resultlog.py b/_pytest/resultlog.py --- a/_pytest/resultlog.py +++ b/_pytest/resultlog.py @@ -109,4 +109,4 @@ def _safeprint(s, file): if isinstance(s, unicode): s = s.encode('utf-8') - py.builtin.print_(s) + py.builtin.print_(s, file=file) From noreply at buildbot.pypy.org Wed Sep 3 16:59:52 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Wed, 3 Sep 2014 16:59:52 +0200 (CEST) Subject: [pypy-commit] stmgc default: add some XXXs Message-ID: <20140903145952.431391D2678@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1343:0bb3f6af0fa4 Date: 2014-09-03 17:01 +0200 http://bitbucket.org/pypy/stmgc/changeset/0bb3f6af0fa4/ Log: add some XXXs diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -9,11 +9,14 @@ void _signal_handler(int sig, siginfo_t *siginfo, void *context) { + /* make PROT_READWRITE again and validate */ - if (siginfo->si_addr == NULL) { + if (siginfo->si_addr == NULL) { /* actual segfault */ /* send to GDB */ kill(getpid(), SIGINT); } + /* didn't know what to do with it: send to GDB */ + kill(getpid(), SIGINT); } /* ############# commit log ############# */ @@ -104,6 +107,11 @@ stm_read(obj); + /* XXX: misses synchronisation with other write_barriers + on same page */ + /* XXX: make backup copy */ + /* XXX: privatize pages if necessary */ + /* make other segments trap if accessing this object */ uintptr_t first_page = ((uintptr_t)obj) / 4096UL; char *realobj; @@ -215,6 +223,8 @@ _validate_and_add_to_commit_log(); + /* XXX:discard backup copies */ + s_mutex_lock(); assert(STM_SEGMENT->nursery_end == NURSERY_END); From noreply at buildbot.pypy.org Wed Sep 3 19:29:14 2014 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 3 Sep 2014 19:29:14 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: kill malloc_nonmovable - it turned out to be a bad idea to start with Message-ID: <20140903172914.7B8931D2934@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73289:b1e335a09328 Date: 2014-09-03 11:28 -0600 http://bitbucket.org/pypy/pypy/changeset/b1e335a09328/ Log: kill malloc_nonmovable - it turned out to be a bad idea to start with diff --git a/pypy/tool/pypyjit_child.py b/pypy/tool/pypyjit_child.py --- a/pypy/tool/pypyjit_child.py +++ b/pypy/tool/pypyjit_child.py @@ -10,10 +10,6 @@ graph = loc['graph'] interp.malloc_check = False - def returns_null(T, *args, **kwds): - return lltype.nullptr(T) - interp.heap.malloc_nonmovable = returns_null # XXX - from rpython.jit.backend.llgraph.runner import LLGraphCPU #LLtypeCPU.supports_floats = False # for now apply_jit(interp, graph, LLGraphCPU) diff --git a/rpython/jit/backend/llsupport/test/zrpy_gc_test.py b/rpython/jit/backend/llsupport/test/zrpy_gc_test.py --- a/rpython/jit/backend/llsupport/test/zrpy_gc_test.py +++ b/rpython/jit/backend/llsupport/test/zrpy_gc_test.py @@ -223,7 +223,7 @@ ## return None, f, None def define_compile_framework_1(cls): - # a moving GC. Supports malloc_varsize_nonmovable. Simple test, works + # a moving GC. Simple test, works # without write_barriers and root stack enumeration. def f(n, x, *args): y = X() diff --git a/rpython/jit/tl/jittest.py b/rpython/jit/tl/jittest.py --- a/rpython/jit/tl/jittest.py +++ b/rpython/jit/tl/jittest.py @@ -6,7 +6,6 @@ import os from rpython import conftest -from rpython.rtyper.lltypesystem import lltype from rpython.rtyper.llinterp import LLInterpreter from rpython.rtyper.annlowlevel import llstr from rpython.jit.metainterp import warmspot @@ -24,10 +23,6 @@ graph = driver.translator._graphof(driver.entry_point) interp = LLInterpreter(driver.translator.rtyper) - def returns_null(T, *args, **kwds): - return lltype.nullptr(T) - interp.heap.malloc_nonmovable = returns_null # XXX - get_policy = driver.extra['jitpolicy'] jitpolicy = get_policy(driver) diff --git a/rpython/memory/gc/base.py b/rpython/memory/gc/base.py --- a/rpython/memory/gc/base.py +++ b/rpython/memory/gc/base.py @@ -47,9 +47,6 @@ def _teardown(self): pass - def can_malloc_nonmovable(self): - return not self.moving_gc - def can_optimize_clean_setarrayitems(self): return True # False in case of card marking @@ -168,9 +165,6 @@ # lots of cast and reverse-cast around... return llmemory.cast_ptr_to_adr(ref) - def malloc_nonmovable(self, typeid, length=0, zero=False): - return self.malloc(typeid, length, zero) - def id(self, ptr): return lltype.cast_ptr_to_int(ptr) diff --git a/rpython/memory/gc/hybrid.py b/rpython/memory/gc/hybrid.py --- a/rpython/memory/gc/hybrid.py +++ b/rpython/memory/gc/hybrid.py @@ -167,7 +167,7 @@ llmemory.GCREF) return self.malloc_varsize_slowpath(typeid, length) - def malloc_varsize_slowpath(self, typeid, length, force_nonmovable=False): + def malloc_varsize_slowpath(self, typeid, length): # For objects that are too large, or when the nursery is exhausted. # In order to keep malloc_varsize_clear() as compact as possible, # we recompute what we need in this slow path instead of passing @@ -185,7 +185,7 @@ nonlarge_max = self.nonlarge_gcptrs_max else: nonlarge_max = self.nonlarge_max - if force_nonmovable or raw_malloc_usage(totalsize) > nonlarge_max: + if raw_malloc_usage(totalsize) > nonlarge_max: result = self.malloc_varsize_marknsweep(totalsize) flags = self.GCFLAGS_FOR_NEW_EXTERNAL_OBJECTS | GCFLAG_UNVISITED else: @@ -197,17 +197,6 @@ malloc_varsize_slowpath._dont_inline_ = True - def malloc_varsize_nonmovable(self, typeid, length): - return self.malloc_varsize_slowpath(typeid, length, True) - - def malloc_nonmovable(self, typeid, length, zero): - # helper for testing, same as GCBase.malloc - if self.is_varsize(typeid): - gcref = self.malloc_varsize_slowpath(typeid, length, True) - else: - raise NotImplementedError("Not supported") - return llmemory.cast_ptr_to_adr(gcref) - def can_move(self, addr): tid = self.header(addr).tid return not (tid & GCFLAG_EXTERNAL) @@ -554,6 +543,3 @@ "gen3: unexpected GCFLAG_UNVISITED") ll_assert((tid & GCFLAG_AGE_MASK) == GCFLAG_AGE_MAX, "gen3: wrong age field") - - def can_malloc_nonmovable(self): - return True diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -427,7 +427,7 @@ # the start of the nursery: we actually allocate a bit more for # the nursery than really needed, to simplify pointer arithmetic # in malloc_fixedsize_clear(). The few extra pages are never used - # anyway so it doesn't even count. + # anyway so it doesn't even counct. nursery = llarena.arena_malloc(self._nursery_memory_size(), 0) if not nursery: raise MemoryError("cannot allocate nursery") @@ -830,9 +830,6 @@ # that one will occur very soon self.nursery_free = self.nursery_top - def can_malloc_nonmovable(self): - return True - def can_optimize_clean_setarrayitems(self): if self.card_page_indices > 0: return False @@ -868,20 +865,6 @@ (obj + offset_to_length).signed[0] = smallerlength return True - - def malloc_fixedsize_nonmovable(self, typeid): - obj = self.external_malloc(typeid, 0) - return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF) - - def malloc_varsize_nonmovable(self, typeid, length): - obj = self.external_malloc(typeid, length) - return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF) - - def malloc_nonmovable(self, typeid, length, zero): - # helper for testing, same as GCBase.malloc - return self.external_malloc(typeid, length or 0) # None -> 0 - - # ---------- # Simple helpers diff --git a/rpython/memory/gc/minimark.py b/rpython/memory/gc/minimark.py --- a/rpython/memory/gc/minimark.py +++ b/rpython/memory/gc/minimark.py @@ -822,9 +822,6 @@ self.nursery_top = self.nursery_real_top self.nursery_free = self.nursery_real_top - def can_malloc_nonmovable(self): - return True - def can_optimize_clean_setarrayitems(self): if self.card_page_indices > 0: return False @@ -861,19 +858,6 @@ return True - def malloc_fixedsize_nonmovable(self, typeid): - obj = self.external_malloc(typeid, 0) - return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF) - - def malloc_varsize_nonmovable(self, typeid, length): - obj = self.external_malloc(typeid, length) - return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF) - - def malloc_nonmovable(self, typeid, length, zero): - # helper for testing, same as GCBase.malloc - return self.external_malloc(typeid, length or 0) # None -> 0 - - # ---------- # Simple helpers diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py --- a/rpython/memory/gctransform/framework.py +++ b/rpython/memory/gctransform/framework.py @@ -388,17 +388,6 @@ annmodel.SomeInteger(nonneg=True)], s_gcref, inline = True) - if getattr(GCClass, 'malloc_varsize_nonmovable', False): - malloc_nonmovable = func_with_new_name( - GCClass.malloc_varsize_nonmovable.im_func, - "malloc_varsize_nonmovable") - self.malloc_varsize_nonmovable_ptr = getfn( - malloc_nonmovable, - [s_gc, s_typeid16, - annmodel.SomeInteger(nonneg=True)], s_gcref) - else: - self.malloc_varsize_nonmovable_ptr = None - if getattr(GCClass, 'raw_malloc_memory_pressure', False): def raw_malloc_memory_pressure_varsize(length, itemsize): totalmem = length * itemsize @@ -665,7 +654,6 @@ has_light_finalizer) if not op.opname.endswith('_varsize') and not flags.get('varsize'): - #TODO:check if it's safe to remove the zero-flag zero = flags.get('zero', False) if (self.malloc_fast_ptr is not None and not c_has_finalizer.value and @@ -684,18 +672,12 @@ info_varsize.ofstolength) c_varitemsize = rmodel.inputconst(lltype.Signed, info_varsize.varitemsize) - if flags.get('nonmovable') and self.malloc_varsize_nonmovable_ptr: - # we don't have tests for such cases, let's fail - # explicitely - malloc_ptr = self.malloc_varsize_nonmovable_ptr - args = [self.c_const_gc, c_type_id, v_length] + if self.malloc_varsize_fast_ptr is not None: + malloc_ptr = self.malloc_varsize_fast_ptr else: - if self.malloc_varsize_fast_ptr is not None: - malloc_ptr = self.malloc_varsize_fast_ptr - else: - malloc_ptr = self.malloc_varsize_ptr - args = [self.c_const_gc, c_type_id, v_length, c_size, - c_varitemsize, c_ofstolength] + malloc_ptr = self.malloc_varsize_ptr + args = [self.c_const_gc, c_type_id, v_length, c_size, + c_varitemsize, c_ofstolength] livevars = self.push_roots(hop) v_result = hop.genop("direct_call", [malloc_ptr] + args, resulttype=llmemory.GCREF) @@ -1076,20 +1058,6 @@ resultvar=hop.spaceop.result) self.pop_roots(hop, livevars) - def gct_malloc_nonmovable_varsize(self, hop): - TYPE = hop.spaceop.result.concretetype - if self.gcdata.gc.can_malloc_nonmovable(): - return self.gct_malloc_varsize(hop, {'nonmovable':True}) - c = rmodel.inputconst(TYPE, lltype.nullptr(TYPE.TO)) - return hop.cast_result(c) - - def gct_malloc_nonmovable(self, hop): - TYPE = hop.spaceop.result.concretetype - if self.gcdata.gc.can_malloc_nonmovable(): - return self.gct_malloc(hop, {'nonmovable':True}) - c = rmodel.inputconst(TYPE, lltype.nullptr(TYPE.TO)) - return hop.cast_result(c) - def _set_into_gc_array_part(self, op): if op.opname == 'setarrayitem': return op.args[1] @@ -1231,7 +1199,7 @@ v_adr = llops.genop('adr_add', [v_a, c_fixedofs], resulttype=llmemory.Address) llops.genop('raw_memclear', [v_adr, v_totalsize]) - elif isinstance(TYPE, lltype.Struct): + elif isinstance(ITEM, lltype.Struct): xxx return else: diff --git a/rpython/memory/gctransform/transform.py b/rpython/memory/gctransform/transform.py --- a/rpython/memory/gctransform/transform.py +++ b/rpython/memory/gctransform/transform.py @@ -510,12 +510,6 @@ assert meth, "%s has no support for malloc_varsize with flavor %r" % (self, flavor) return self.varsize_malloc_helper(hop, flags, meth, []) - def gct_malloc_nonmovable(self, *args, **kwds): - return self.gct_malloc(*args, **kwds) - - def gct_malloc_nonmovable_varsize(self, *args, **kwds): - return self.gct_malloc_varsize(*args, **kwds) - def gct_gc_add_memory_pressure(self, hop): if hasattr(self, 'raw_malloc_memory_pressure_ptr'): op = hop.spaceop diff --git a/rpython/memory/gcwrapper.py b/rpython/memory/gcwrapper.py --- a/rpython/memory/gcwrapper.py +++ b/rpython/memory/gcwrapper.py @@ -57,16 +57,6 @@ return lltype.malloc(TYPE, n, flavor=flavor, zero=zero, track_allocation=track_allocation) - def malloc_nonmovable(self, TYPE, n=None, zero=False): - typeid = self.get_type_id(TYPE) - if not self.gc.can_malloc_nonmovable(): - return lltype.nullptr(TYPE) - addr = self.gc.malloc_nonmovable(typeid, n, zero=zero) - result = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(TYPE)) - if not self.gc.malloc_zero_filled: - gctypelayout.zero_gc_pointers(result) - return result - def add_memory_pressure(self, size): if hasattr(self.gc, 'raw_malloc_memory_pressure'): self.gc.raw_malloc_memory_pressure(size) diff --git a/rpython/memory/test/gc_test_base.py b/rpython/memory/test/gc_test_base.py --- a/rpython/memory/test/gc_test_base.py +++ b/rpython/memory/test/gc_test_base.py @@ -25,7 +25,6 @@ class GCTest(object): GC_PARAMS = {} GC_CAN_MOVE = False - GC_CAN_MALLOC_NONMOVABLE = True GC_CAN_SHRINK_ARRAY = False GC_CAN_SHRINK_BIG_ARRAY = False BUT_HOW_BIG_IS_A_BIG_STRING = 3*WORD @@ -614,34 +613,6 @@ return rgc.can_move(lltype.malloc(TP, 1)) assert self.interpret(func, []) == self.GC_CAN_MOVE - - def test_malloc_nonmovable(self): - TP = lltype.GcArray(lltype.Char) - def func(): - a = rgc.malloc_nonmovable(TP, 3) - if a: - assert not rgc.can_move(a) - return 1 - return 0 - - assert self.interpret(func, []) == int(self.GC_CAN_MALLOC_NONMOVABLE) - - def test_malloc_nonmovable_fixsize(self): - S = lltype.GcStruct('S', ('x', lltype.Float)) - TP = lltype.GcStruct('T', ('s', lltype.Ptr(S))) - def func(): - try: - a = rgc.malloc_nonmovable(TP) - rgc.collect() - if a: - assert not rgc.can_move(a) - return 1 - return 0 - except Exception: - return 2 - - assert self.interpret(func, []) == int(self.GC_CAN_MALLOC_NONMOVABLE) - def test_shrink_array(self): from rpython.rtyper.lltypesystem.rstr import STR diff --git a/rpython/memory/test/test_hybrid_gc.py b/rpython/memory/test/test_hybrid_gc.py --- a/rpython/memory/test/test_hybrid_gc.py +++ b/rpython/memory/test/test_hybrid_gc.py @@ -8,7 +8,6 @@ class TestHybridGC(test_generational_gc.TestGenerationalGC): from rpython.memory.gc.hybrid import HybridGC as GCClass - GC_CAN_MALLOC_NONMOVABLE = True GC_CAN_SHRINK_BIG_ARRAY = False def test_ref_from_rawmalloced_to_regular(self): @@ -71,6 +70,3 @@ return b.num_deleted res = self.interpret(f, [15]) assert res == 16 - - def test_malloc_nonmovable_fixsize(self): - py.test.skip("Not supported") diff --git a/rpython/memory/test/test_hybrid_gc_smallheap.py b/rpython/memory/test/test_hybrid_gc_smallheap.py --- a/rpython/memory/test/test_hybrid_gc_smallheap.py +++ b/rpython/memory/test/test_hybrid_gc_smallheap.py @@ -10,7 +10,6 @@ from rpython.memory.gc.hybrid import HybridGC as GCClass GC_CAN_MOVE = False # with this size of heap, stuff gets allocated # in 3rd gen. - GC_CAN_MALLOC_NONMOVABLE = True GC_PARAMS = {'space_size': 48*WORD, 'min_nursery_size': 12*WORD, 'nursery_size': 12*WORD, @@ -51,6 +50,3 @@ return i res = self.interpret(f, [200]) assert res == 401 - - def test_malloc_nonmovable_fixsize(self): - py.test.skip("Not supported") diff --git a/rpython/memory/test/test_transformed_gc.py b/rpython/memory/test/test_transformed_gc.py --- a/rpython/memory/test/test_transformed_gc.py +++ b/rpython/memory/test/test_transformed_gc.py @@ -44,7 +44,6 @@ class GCTest(object): gcpolicy = None GC_CAN_MOVE = False - GC_CAN_MALLOC_NONMOVABLE = True taggedpointers = False def setup_class(cls): @@ -622,45 +621,6 @@ res = run([]) assert res == self.GC_CAN_MOVE - def define_malloc_nonmovable(cls): - TP = lltype.GcArray(lltype.Char) - def func(): - #try: - a = rgc.malloc_nonmovable(TP, 3, zero=True) - rgc.collect() - if a: - assert not rgc.can_move(a) - return 1 - return 0 - #except Exception, e: - # return 2 - - return func - - def test_malloc_nonmovable(self): - run = self.runner("malloc_nonmovable") - assert int(self.GC_CAN_MALLOC_NONMOVABLE) == run([]) - - def define_malloc_nonmovable_fixsize(cls): - S = lltype.GcStruct('S', ('x', lltype.Float)) - TP = lltype.GcStruct('T', ('s', lltype.Ptr(S))) - def func(): - try: - a = rgc.malloc_nonmovable(TP) - rgc.collect() - if a: - assert not rgc.can_move(a) - return 1 - return 0 - except Exception, e: - return 2 - - return func - - def test_malloc_nonmovable_fixsize(self): - run = self.runner("malloc_nonmovable_fixsize") - assert run([]) == int(self.GC_CAN_MALLOC_NONMOVABLE) - def define_shrink_array(cls): from rpython.rtyper.lltypesystem.rstr import STR @@ -708,7 +668,6 @@ class GenericMovingGCTests(GenericGCTests): GC_CAN_MOVE = True - GC_CAN_MALLOC_NONMOVABLE = False GC_CAN_TEST_ID = False def define_many_ids(cls): class A(object): @@ -1169,7 +1128,6 @@ class TestHybridGC(TestGenerationGC): gcname = "hybrid" - GC_CAN_MALLOC_NONMOVABLE = True class gcpolicy(gc.BasicFrameworkGcPolicy): class transformerclass(shadowstack.ShadowStackFrameworkGCTransformer): @@ -1230,9 +1188,6 @@ run = self.runner("write_barrier_direct") res = run([]) assert res == 42 - - def test_malloc_nonmovable_fixsize(self): - py.test.skip("not supported") class TestMiniMarkGC(TestHybridGC): gcname = "minimark" diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py --- a/rpython/rlib/rgc.py +++ b/rpython/rlib/rgc.py @@ -101,40 +101,6 @@ hop.exception_is_here() return hop.genop('gc_heap_stats', [], resulttype=hop.r_result) -def malloc_nonmovable(TP, n=None, zero=False): - """ Allocate a non-moving buffer or return nullptr. - When running directly, will pretend that gc is always - moving (might be configurable in a future) - """ - return lltype.nullptr(TP) - -class MallocNonMovingEntry(ExtRegistryEntry): - _about_ = malloc_nonmovable - - def compute_result_annotation(self, s_TP, s_n=None, s_zero=None): - # basically return the same as malloc - from rpython.annotator.builtin import BUILTIN_ANALYZERS - return BUILTIN_ANALYZERS[lltype.malloc](s_TP, s_n, s_zero=s_zero) - - def specialize_call(self, hop, i_zero=None): - # XXX assume flavor and zero to be None by now - assert hop.args_s[0].is_constant() - vlist = [hop.inputarg(lltype.Void, arg=0)] - opname = 'malloc_nonmovable' - flags = {'flavor': 'gc'} - if i_zero is not None: - flags['zero'] = hop.args_s[i_zero].const - nb_args = hop.nb_args - 1 - else: - nb_args = hop.nb_args - vlist.append(hop.inputconst(lltype.Void, flags)) - - if nb_args == 2: - vlist.append(hop.inputarg(lltype.Signed, arg=1)) - opname += '_varsize' - - hop.exception_cannot_occur() - return hop.genop(opname, vlist, resulttype = hop.r_result.lowleveltype) def copy_struct_item(source, dest, si, di): TP = lltype.typeOf(source).TO.OF diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py --- a/rpython/rtyper/llinterp.py +++ b/rpython/rtyper/llinterp.py @@ -682,18 +682,6 @@ except MemoryError: self.make_llexception() - def op_malloc_nonmovable(self, TYPE, flags): - flavor = flags['flavor'] - assert flavor == 'gc' - zero = flags.get('zero', False) - return self.heap.malloc_nonmovable(TYPE, zero=zero) - - def op_malloc_nonmovable_varsize(self, TYPE, flags, size): - flavor = flags['flavor'] - assert flavor == 'gc' - zero = flags.get('zero', False) - return self.heap.malloc_nonmovable(TYPE, size, zero=zero) - def op_free(self, obj, flags): assert flags['flavor'] == 'raw' track_allocation = flags.get('track_allocation', True) diff --git a/rpython/rtyper/lltypesystem/llheap.py b/rpython/rtyper/lltypesystem/llheap.py --- a/rpython/rtyper/lltypesystem/llheap.py +++ b/rpython/rtyper/lltypesystem/llheap.py @@ -18,7 +18,6 @@ def weakref_create_getlazy(objgetter): return weakref_create(objgetter()) -malloc_nonmovable = malloc def shrink_array(p, smallersize): return False diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py --- a/rpython/rtyper/lltypesystem/lloperation.py +++ b/rpython/rtyper/lltypesystem/lloperation.py @@ -368,8 +368,6 @@ 'malloc': LLOp(canmallocgc=True), 'malloc_varsize': LLOp(canmallocgc=True), - 'malloc_nonmovable': LLOp(canmallocgc=True), - 'malloc_nonmovable_varsize':LLOp(canmallocgc=True), 'shrink_array': LLOp(canrun=True), 'zero_gc_pointers_inside': LLOp(), 'free': LLOp(), diff --git a/rpython/rtyper/lltypesystem/rclass.py b/rpython/rtyper/lltypesystem/rclass.py --- a/rpython/rtyper/lltypesystem/rclass.py +++ b/rpython/rtyper/lltypesystem/rclass.py @@ -505,13 +505,7 @@ ctype = inputconst(Void, self.object_type) cflags = inputconst(Void, flags) vlist = [ctype, cflags] - cnonmovable = self.classdef.classdesc.read_attribute( - '_alloc_nonmovable_', Constant(False)) - if cnonmovable.value: - opname = 'malloc_nonmovable' - else: - opname = 'malloc' - vptr = llops.genop(opname, vlist, + vptr = llops.genop('malloc', vlist, resulttype = Ptr(self.object_type)) ctypeptr = inputconst(CLASSTYPE, self.rclass.getvtable()) self.setfield(vptr, '__class__', ctypeptr, llops) diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py --- a/rpython/rtyper/lltypesystem/rdict.py +++ b/rpython/rtyper/lltypesystem/rdict.py @@ -2,6 +2,7 @@ from rpython.flowspace.model import Constant from rpython.rtyper.rdict import AbstractDictRepr, AbstractDictIteratorRepr from rpython.rtyper.lltypesystem import lltype +from rpython.rtyper.lltypesystem.lloperation import llop from rpython.rlib import objectmodel, jit from rpython.rlib.debug import ll_assert from rpython.rlib.rarithmetic import r_uint, intmask, LONG_BIT @@ -615,18 +616,22 @@ i = i & mask # keep 'i' as a signed number here, to consistently pass signed # arguments to the small helper methods. + llop.debug_print(lltype.Void, "looking in", i) if not entries.everused(i): if freeslot == -1: freeslot = intmask(i) return r_uint(freeslot) | HIGHEST_BIT elif entries.valid(i): + llop.debug_print(lltype.Void, "found!", hash, entries.hash(i)) checkingkey = entries[i].key if direct_compare and checkingkey == key: return i if d.keyeq is not None and entries.hash(i) == hash: # correct hash, maybe the key is e.g. a different pointer to # an equal object + llop.debug_print(lltype.Void, "hashes compare") found = d.keyeq(checkingkey, key) + llop.debug_print(lltype.Void, "found", found) if d.paranoia: if (entries != d.entries or not entries.valid(i) or entries[i].key != checkingkey): diff --git a/rpython/rtyper/test/test_rclass.py b/rpython/rtyper/test/test_rclass.py --- a/rpython/rtyper/test/test_rclass.py +++ b/rpython/rtyper/test/test_rclass.py @@ -1144,17 +1144,6 @@ assert sorted([u]) == [6] # 32-bit types assert sorted([i, r, d, l]) == [2, 3, 4, 5] # 64-bit types - def test_nonmovable(self): - for (nonmovable, opname) in [(True, 'malloc_nonmovable'), - (False, 'malloc')]: - class A(object): - _alloc_nonmovable_ = nonmovable - def f(): - return A() - t, typer, graph = self.gengraph(f, []) - assert summary(graph) == {opname: 1, - 'cast_pointer': 1, - 'setfield': 1} def test_iter(self): class Iterable(object): diff --git a/rpython/translator/c/test/test_boehm.py b/rpython/translator/c/test/test_boehm.py --- a/rpython/translator/c/test/test_boehm.py +++ b/rpython/translator/c/test/test_boehm.py @@ -336,22 +336,6 @@ c_fn = self.getcompiled(fn, []) assert not c_fn() - def test_malloc_nonmovable(self): - TP = lltype.GcArray(lltype.Char) - def func(): - try: - a = rgc.malloc_nonmovable(TP, 3) - rgc.collect() - if a: - assert not rgc.can_move(a) - return 0 - return 1 - except Exception: - return 2 - - run = self.getcompiled(func) - assert run() == 0 - def test_shrink_array(self): def f(): ptr = lltype.malloc(STR, 3) diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py --- a/rpython/translator/c/test/test_newgc.py +++ b/rpython/translator/c/test/test_newgc.py @@ -22,7 +22,6 @@ removetypeptr = False taggedpointers = False GC_CAN_MOVE = False - GC_CAN_MALLOC_NONMOVABLE = True GC_CAN_SHRINK_ARRAY = False _isolated_func = None @@ -721,25 +720,6 @@ def test_can_move(self): assert self.run('can_move') == self.GC_CAN_MOVE - def define_malloc_nonmovable(cls): - TP = lltype.GcArray(lltype.Char) - def func(): - try: - a = rgc.malloc_nonmovable(TP, 3) - rgc.collect() - if a: - assert not rgc.can_move(a) - return 1 - return 0 - except Exception: - return 2 - - return func - - def test_malloc_nonmovable(self): - res = self.run('malloc_nonmovable') - assert res == self.GC_CAN_MALLOC_NONMOVABLE - def define_resizable_buffer(cls): from rpython.rtyper.lltypesystem.rstr import STR @@ -1219,7 +1199,6 @@ gcpolicy = "semispace" should_be_moving = True GC_CAN_MOVE = True - GC_CAN_MALLOC_NONMOVABLE = False GC_CAN_SHRINK_ARRAY = True # for snippets @@ -1420,7 +1399,6 @@ class TestHybridGC(TestGenerationalGC): gcpolicy = "hybrid" should_be_moving = True - GC_CAN_MALLOC_NONMOVABLE = True def test_gc_set_max_heap_size(self): py.test.skip("not implemented") @@ -1433,7 +1411,6 @@ class TestMiniMarkGC(TestSemiSpaceGC): gcpolicy = "minimark" should_be_moving = True - GC_CAN_MALLOC_NONMOVABLE = True GC_CAN_SHRINK_ARRAY = True def test_gc_heap_stats(self): diff --git a/rpython/translator/exceptiontransform.py b/rpython/translator/exceptiontransform.py --- a/rpython/translator/exceptiontransform.py +++ b/rpython/translator/exceptiontransform.py @@ -251,8 +251,7 @@ len(block.operations) and (block.exits[0].args[0].concretetype is lltype.Void or block.exits[0].args[0] is block.operations[-1].result) and - block.operations[-1].opname not in ('malloc', # special cases - 'malloc_nonmovable')): + block.operations[-1].opname != 'malloc'): # special cases last_operation -= 1 lastblock = block for i in range(last_operation, -1, -1): @@ -423,14 +422,6 @@ flavor = spaceop.args[1].value['flavor'] if flavor == 'gc': insert_zeroing_op = True - elif spaceop.opname == 'malloc_nonmovable': - # xxx we cannot insert zero_gc_pointers_inside after - # malloc_nonmovable, because it can return null. For now - # we simply always force the zero=True flag on - # malloc_nonmovable. - c_flags = spaceop.args[1] - c_flags.value = c_flags.value.copy() - spaceop.args[1].value['zero'] = True # NB. when inserting more special-cases here, keep in mind that # you also need to list the opnames in transform_block() # (see "special cases") From noreply at buildbot.pypy.org Wed Sep 3 21:50:12 2014 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 3 Sep 2014 21:50:12 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: fixes Message-ID: <20140903195012.041D41C320C@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73290:6a7d16b9825a Date: 2014-09-03 13:49 -0600 http://bitbucket.org/pypy/pypy/changeset/6a7d16b9825a/ Log: fixes diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py --- a/rpython/memory/gctransform/framework.py +++ b/rpython/memory/gctransform/framework.py @@ -867,6 +867,12 @@ TYPE = v_ob.concretetype.TO self.gen_zero_gc_pointers(TYPE, v_ob, hop.llops) + def gct_zero_everything_inside(self, hop): + if not self.malloc_zero_filled: + v_ob = hop.spaceop.args[0] + TYPE = v_ob.concretetype.TO + self.gen_zero_gc_pointers(TYPE, v_ob, hop.llops, everything=True) + def gct_gc_writebarrier_before_copy(self, hop): op = hop.spaceop if not hasattr(self, 'wb_before_copy_ptr'): @@ -1172,21 +1178,37 @@ def pop_roots(self, hop, livevars): raise NotImplementedError - def gen_zero_gc_pointers(self, TYPE, v, llops, previous_steps=None): + def gen_zero_gc_pointers(self, TYPE, v, llops, previous_steps=None, + everything=False): if isinstance(TYPE, lltype.Struct): for name in TYPE._names: FIELD = getattr(TYPE, name) - if isinstance(FIELD, lltype.Ptr) and FIELD._needsgc(): + if ((isinstance(FIELD, lltype.Ptr) and FIELD._needsgc()) + or everything): c_name = rmodel.inputconst(lltype.Void, name) - c_null = rmodel.inputconst(FIELD, lltype.nullptr(FIELD.TO)) + c_null = rmodel.inputconst(FIELD, FIELD._defl()) llops.genop('bare_setfield', [v, c_name, c_null]) elif (isinstance(FIELD, lltype.Array) and isinstance(FIELD.OF, lltype.Ptr) and FIELD.OF._needsgc()): xxx + return elif isinstance(TYPE, lltype.Array): ITEM = TYPE.OF - if isinstance(ITEM, lltype.Ptr) and ITEM._needsgc(): + if everything: + needs_clearing = True + elif isinstance(ITEM, lltype.Ptr) and ITEM._needsgc(): + needs_clearing = True + elif isinstance(ITEM, lltype.Struct): + for SUBITEM in ITEM._flds.values(): + if isinstance(SUBITEM, lltype.Ptr) and SUBITEM._needsgc(): + needs_clearing = True + break + else: + needs_clearing = False + else: + needs_clearing = False + if needs_clearing: v_size = llops.genop('getarraysize', [v], resulttype=lltype.Signed) c_size = rmodel.inputconst(lltype.Signed, llmemory.sizeof(ITEM)) @@ -1199,8 +1221,6 @@ v_adr = llops.genop('adr_add', [v_a, c_fixedofs], resulttype=llmemory.Address) llops.genop('raw_memclear', [v_adr, v_totalsize]) - elif isinstance(ITEM, lltype.Struct): - xxx return else: raise TypeError(TYPE) diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py --- a/rpython/translator/c/test/test_newgc.py +++ b/rpython/translator/c/test/test_newgc.py @@ -1194,6 +1194,20 @@ def test_gcflag_extra(self): self.run("gcflag_extra") + def define_check_zero_works(self): + S = lltype.GcStruct("s", ('x', lltype.Signed)) + A = lltype.GcArray(lltype.Signed) + + def fn(): + s = lltype.malloc(S, zero=True) + assert s.x == 0 + a = lltype.malloc(A, 3, zero=True) + assert a[2] == 0 + return 0 + return fn + + def test_check_zero_works(self): + self.run("check_zero_works") class TestSemiSpaceGC(UsingFrameworkTest, snippet.SemiSpaceGCTestDefines): gcpolicy = "semispace" diff --git a/rpython/translator/exceptiontransform.py b/rpython/translator/exceptiontransform.py --- a/rpython/translator/exceptiontransform.py +++ b/rpython/translator/exceptiontransform.py @@ -251,7 +251,7 @@ len(block.operations) and (block.exits[0].args[0].concretetype is lltype.Void or block.exits[0].args[0] is block.operations[-1].result) and - block.operations[-1].opname != 'malloc'): # special cases + block.operations[-1].opname not in ('malloc', 'malloc_varsize')): # special cases last_operation -= 1 lastblock = block for i in range(last_operation, -1, -1): @@ -422,6 +422,7 @@ flavor = spaceop.args[1].value['flavor'] if flavor == 'gc': insert_zeroing_op = True + true_zero = spaceop.args[1].value.get('zero', False) # NB. when inserting more special-cases here, keep in mind that # you also need to list the opnames in transform_block() # (see "special cases") @@ -437,9 +438,12 @@ v_result_after = copyvar(None, v_result) l0.args.append(v_result) normalafterblock.inputargs.append(v_result_after) + if true_zero: + opname = "zero_everything_inside" + else: + opname = "zero_gc_pointers_inside" normalafterblock.operations.insert( - 0, SpaceOperation('zero_gc_pointers_inside', - [v_result_after], + 0, SpaceOperation(opname, [v_result_after], varoftype(lltype.Void))) def setup_excdata(self): From noreply at buildbot.pypy.org Wed Sep 3 22:34:14 2014 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 3 Sep 2014 22:34:14 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: start working on the JIT - first part is that we don't optimize away storage Message-ID: <20140903203414.37D4A1D2678@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73291:0038daf24dd5 Date: 2014-09-03 14:33 -0600 http://bitbucket.org/pypy/pypy/changeset/0038daf24dd5/ Log: start working on the JIT - first part is that we don't optimize away storage of nulls diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py @@ -1056,7 +1056,7 @@ """ self.optimize_loop(ops, expected) - def test_nonvirtual_dont_write_null_fields_on_force(self): + def test_nonvirtual_write_null_fields_on_force(self): ops = """ [i] p1 = new_with_vtable(ConstClass(node_vtable)) @@ -1070,6 +1070,7 @@ expected = """ [i] p1 = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(p1, 0, descr=valuedescr) escape(p1) i2 = getfield_gc(p1, descr=valuedescr) jump(i2) @@ -1176,7 +1177,7 @@ """ self.optimize_loop(ops, expected) - def test_nonvirtual_array_dont_write_null_fields_on_force(self): + def test_nonvirtual_array_write_null_fields_on_force(self): ops = """ [i1] p1 = new_array(5, descr=arraydescr) @@ -1189,6 +1190,7 @@ [i1] p1 = new_array(5, descr=arraydescr) setarrayitem_gc(p1, 0, i1, descr=arraydescr) + setarrayitem_gc(p1, 1, 0, descr=arraydescr) escape(p1) jump(i1) """ diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -1397,7 +1397,7 @@ """ self.optimize_loop(ops, expected) - def test_nonvirtual_dont_write_null_fields_on_force(self): + def test_nonvirtual_write_null_fields_on_force(self): ops = """ [i] p1 = new_with_vtable(ConstClass(node_vtable)) @@ -1411,6 +1411,7 @@ expected = """ [i] p1 = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(p1, 0, descr=valuedescr) escape(p1) i2 = getfield_gc(p1, descr=valuedescr) jump(i2) @@ -1608,7 +1609,7 @@ """ self.optimize_loop(ops, expected) - def test_nonvirtual_array_dont_write_null_fields_on_force(self): + def test_nonvirtual_array_write_null_fields_on_force(self): ops = """ [i1] p1 = new_array(5, descr=arraydescr) @@ -1621,6 +1622,7 @@ [i1] p1 = new_array(5, descr=arraydescr) setarrayitem_gc(p1, 0, i1, descr=arraydescr) + setarrayitem_gc(p1, 1, 0, descr=arraydescr) escape(p1) jump(i1) """ diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py --- a/rpython/jit/metainterp/optimizeopt/virtualize.py +++ b/rpython/jit/metainterp/optimizeopt/virtualize.py @@ -158,8 +158,6 @@ iteritems = list(iteritems) iteritems.sort(key=lambda (x, y): x.sort_key()) for ofs, value in iteritems: - if value.is_null(): - continue subbox = value.force_box(optforce) op = ResOperation(rop.SETFIELD_GC, [box, subbox], None, descr=ofs) @@ -310,8 +308,6 @@ for index in range(len(self._items)): subvalue = self._items[index] if subvalue is not self.constvalue: - if subvalue.is_null(): - continue subbox = subvalue.force_box(optforce) op = ResOperation(rop.SETARRAYITEM_GC, [box, ConstInt(index), subbox], None, From noreply at buildbot.pypy.org Wed Sep 3 22:41:48 2014 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 3 Sep 2014 22:41:48 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: kill debug prints from dict Message-ID: <20140903204148.2331F1C320C@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73292:2efd57cc1412 Date: 2014-09-03 14:41 -0600 http://bitbucket.org/pypy/pypy/changeset/2efd57cc1412/ Log: kill debug prints from dict diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py --- a/rpython/rtyper/lltypesystem/rdict.py +++ b/rpython/rtyper/lltypesystem/rdict.py @@ -616,22 +616,18 @@ i = i & mask # keep 'i' as a signed number here, to consistently pass signed # arguments to the small helper methods. - llop.debug_print(lltype.Void, "looking in", i) if not entries.everused(i): if freeslot == -1: freeslot = intmask(i) return r_uint(freeslot) | HIGHEST_BIT elif entries.valid(i): - llop.debug_print(lltype.Void, "found!", hash, entries.hash(i)) checkingkey = entries[i].key if direct_compare and checkingkey == key: return i if d.keyeq is not None and entries.hash(i) == hash: # correct hash, maybe the key is e.g. a different pointer to # an equal object - llop.debug_print(lltype.Void, "hashes compare") found = d.keyeq(checkingkey, key) - llop.debug_print(lltype.Void, "found", found) if d.paranoia: if (entries != d.entries or not entries.valid(i) or entries[i].key != checkingkey): From noreply at buildbot.pypy.org Thu Sep 4 00:32:07 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 4 Sep 2014 00:32:07 +0200 (CEST) Subject: [pypy-commit] pypy default: Oups, it seems the buildbots use too much memory with this setting? Message-ID: <20140903223207.C799F1D38C0@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73293:41824e6d3136 Date: 2014-09-04 00:31 +0200 http://bitbucket.org/pypy/pypy/changeset/41824e6d3136/ Log: Oups, it seems the buildbots use too much memory with this setting? diff --git a/_pytest/resultlog.py b/_pytest/resultlog.py --- a/_pytest/resultlog.py +++ b/_pytest/resultlog.py @@ -57,7 +57,7 @@ _safeprint("%s %s" % (lettercode, testpath), file=self.logfile) for line in longrepr.splitlines(): _safeprint(" %s" % line, file=self.logfile) - if sections is not None: + if 0 and sections is not None: # XXX XXX USES TOO MUCH MEMORY?? for title, content in sections: _safeprint(" ---------- %s ----------" % (title,), file=self.logfile) From noreply at buildbot.pypy.org Thu Sep 4 02:02:08 2014 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 4 Sep 2014 02:02:08 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: introduce zero_contents and zero_gc_pointers as SpaceOperations in jtransform Message-ID: <20140904000208.64EB81D2798@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73294:80638b7a13c4 Date: 2014-09-03 18:01 -0600 http://bitbucket.org/pypy/pypy/changeset/80638b7a13c4/ Log: introduce zero_contents and zero_gc_pointers as SpaceOperations in jtransform diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py --- a/rpython/jit/codewriter/jtransform.py +++ b/rpython/jit/codewriter/jtransform.py @@ -612,8 +612,14 @@ # XXX only strings or simple arrays for now ARRAY = op.args[0].value arraydescr = self.cpu.arraydescrof(ARRAY) - return SpaceOperation('new_array', [op.args[2], arraydescr], - op.result) + op1 = SpaceOperation('new_array', [op.args[2], arraydescr], + op.result) + if self._has_gcptrs_in(ARRAY): + return [op1, SpaceOperation('zero_gc_pointers', [op.result], + None)] + if op.args[1].value.get('zero', False): + return [op1, SpaceOperation('zero_contents', [op.result], None)] + return op1 def rewrite_op_free(self, op): d = op.args[1].value.copy() @@ -860,7 +866,11 @@ if op.args[1].value['flavor'] == 'raw': return self._rewrite_raw_malloc(op, 'raw_malloc_fixedsize', []) # - assert op.args[1].value == {'flavor': 'gc'} + if op.args[1].value.get('zero', False): + true_zero = True + else: + assert op.args[1].value == {'flavor': 'gc'} + true_zero = False STRUCT = op.args[0].value vtable = heaptracker.get_vtable_for_gcstruct(self.cpu, STRUCT) if vtable: @@ -881,7 +891,24 @@ else: opname = 'new' sizedescr = self.cpu.sizeof(STRUCT) - return SpaceOperation(opname, [sizedescr], op.result) + op1 = SpaceOperation(opname, [sizedescr], op.result) + if true_zero: + return [op1, SpaceOperation('zero_contents', [op.result], None)] + if self._has_gcptrs_in(STRUCT): + return [op1, SpaceOperation('zero_gc_pointers', [op.result], None)] + return op1 + + def _has_gcptrs_in(self, STRUCT): + if isinstance(STRUCT, lltype.Array): + ITEM = STRUCT.OF + if isinstance(ITEM, lltype.Struct): + STRUCT = ITEM + else: + return isinstance(ITEM, lltype.Ptr) and ITEM._needsgc() + for FIELD in STRUCT._flds.values(): + if isinstance(FIELD, lltype.Ptr) and FIELD._needsgc(): + return True + return False def rewrite_op_getinteriorarraysize(self, op): # only supports strings and unicodes diff --git a/rpython/jit/codewriter/test/test_jtransform.py b/rpython/jit/codewriter/test/test_jtransform.py --- a/rpython/jit/codewriter/test/test_jtransform.py +++ b/rpython/jit/codewriter/test/test_jtransform.py @@ -529,6 +529,31 @@ assert op1.opname == 'new' assert op1.args == [('sizedescr', S)] +def test_malloc_new_zero(): + SS = lltype.GcStruct('SS') + S = lltype.GcStruct('S', ('x', lltype.Ptr(SS))) + v = varoftype(lltype.Ptr(S)) + op = SpaceOperation('malloc', [Constant(S, lltype.Void), + Constant({'flavor': 'gc'}, lltype.Void)], v) + op1, op2 = Transformer(FakeCPU()).rewrite_operation(op) + assert op1.opname == 'new' + assert op1.args == [('sizedescr', S)] + assert op2.opname == 'zero_gc_pointers' + assert op2.args == [v] + +def test_malloc_new_zero_2(): + SS = lltype.GcStruct('SS') + S = lltype.GcStruct('S', ('x', lltype.Ptr(SS))) + v = varoftype(lltype.Ptr(S)) + op = SpaceOperation('malloc', [Constant(S, lltype.Void), + Constant({'flavor': 'gc', + 'zero': True}, lltype.Void)], v) + op1, op2 = Transformer(FakeCPU()).rewrite_operation(op) + assert op1.opname == 'new' + assert op1.args == [('sizedescr', S)] + assert op2.opname == 'zero_contents' + assert op2.args == [v] + def test_malloc_new_with_vtable(): vtable = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True) S = lltype.GcStruct('S', ('parent', rclass.OBJECT)) @@ -1026,6 +1051,20 @@ assert op1.args == [v1] assert op1.result == v2 +def test_malloc_varsize_zero(): + c_A = Constant(lltype.GcArray(lltype.Signed), lltype.Void) + c_flags = Constant({"flavor": "gc"}, lltype.Void) + v1 = varoftype(lltype.Signed) + v2 = varoftype(c_A.value) + op = SpaceOperation('malloc_varsize', [c_A, c_flags, v1], v2) + op1 = Transformer(FakeCPU()).rewrite_operation(op) + assert op1.opname == 'new_array' + c_flags = Constant({"flavor": "gc", "zero": True}, lltype.Void) + op = SpaceOperation('malloc_varsize', [c_A, c_flags, v1], v2) + op1, op2 = Transformer(FakeCPU()).rewrite_operation(op) + assert op1.opname == 'new_array' + assert op2.opname == 'zero_contents' + def test_str_concat(): # test that the oopspec is present and correctly transformed PSTR = lltype.Ptr(rstr.STR) From noreply at buildbot.pypy.org Thu Sep 4 10:34:04 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Thu, 4 Sep 2014 10:34:04 +0200 (CEST) Subject: [pypy-commit] stmgc default: add minimal privatization in signal handler Message-ID: <20140904083404.6D8A81C243B@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1344:69b4538466fc Date: 2014-09-04 10:35 +0200 http://bitbucket.org/pypy/stmgc/changeset/69b4538466fc/ Log: add minimal privatization in signal handler diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -9,14 +9,31 @@ void _signal_handler(int sig, siginfo_t *siginfo, void *context) { - /* make PROT_READWRITE again and validate */ - - if (siginfo->si_addr == NULL) { /* actual segfault */ + char *addr = siginfo->si_addr; + dprintf(("si_addr: %p\n", addr)); + if (addr == NULL) { /* actual segfault */ /* send to GDB */ kill(getpid(), SIGINT); } - /* didn't know what to do with it: send to GDB */ - kill(getpid(), SIGINT); + /* make PROT_READWRITE again and validate */ + int segnum = get_segment_of_linear_address(addr); + OPT_ASSERT(segnum == STM_SEGMENT->segment_num); + dprintf(("-> segment: %d\n", segnum)); + char *seg_base = STM_SEGMENT->segment_base; + uintptr_t pagenum = ((char*)addr - seg_base) / 4096UL; + + /* XXX: missing synchronisation: we may change protection, then + another thread changes it back, then we try to privatize that + calls page_copy() and traps */ + mprotect(seg_base + pagenum * 4096UL, 4096, PROT_READ|PROT_WRITE); + page_privatize(pagenum); + + /* XXX: ... what can go wrong when we abort from inside + the signal handler? */ + + /* make sure we are up to date in this (and all other) pages */ + stm_validate(NULL); + return; } /* ############# commit log ############# */ @@ -125,11 +142,11 @@ for (i = 0; i < NB_SEGMENTS; i++) { if (i == STM_SEGMENT->segment_num) continue; - + /* XXX: only do it if not already PROT_NONE */ char *segment_base = get_segment_base(i); mprotect(segment_base + first_page * 4096, (end_page - first_page + 1) * 4096, PROT_NONE); - dprintf(("prot %lu, len=%lu in seg %d\n", first_page, (end_page - first_page + 1), i)); + dprintf(("prot %lu, len=%lu in seg %lu\n", first_page, (end_page - first_page + 1), i)); } LIST_APPEND(STM_PSEGMENT->modified_old_objects, obj); diff --git a/c8/stm/core.h b/c8/stm/core.h --- a/c8/stm/core.h +++ b/c8/stm/core.h @@ -95,6 +95,12 @@ get_segment_base(segment_num), STM_PSEGMENT); } +static inline int get_segment_of_linear_address(char *addr) { + assert(addr > stm_object_pages && addr < stm_object_pages + TOTAL_MEMORY); + return (addr - stm_object_pages) / (NB_PAGES * 4096UL); +} + + static bool _is_tl_registered(stm_thread_local_t *tl); static bool _seems_to_be_running_transaction(void); diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c --- a/c8/stm/gcpage.c +++ b/c8/stm/gcpage.c @@ -51,5 +51,7 @@ object_t *o = (object_t *)(p - stm_object_pages); o->stm_flags = GCFLAG_WRITE_BARRIER; + dprintf(("allocate_old(%lu): %p, seg=%d\n", size_rounded_up, p, + get_segment_of_linear_address(p))); return o; } diff --git a/c8/stm/pages.c b/c8/stm/pages.c --- a/c8/stm/pages.c +++ b/c8/stm/pages.c @@ -68,3 +68,38 @@ count * 4096UL, pagenum); } } + +static void page_privatize(uintptr_t pagenum) +{ + /* check this thread's 'pages_privatized' bit */ + uint64_t bitmask = 1UL << (STM_SEGMENT->segment_num - 1); + volatile struct page_shared_s *ps = (volatile struct page_shared_s *) + &pages_privatized[pagenum - PAGE_FLAG_START]; + if (ps->by_segment & bitmask) { + /* the page is already privatized; nothing to do */ + return; + } + + long i; + for (i = 0; i < NB_SEGMENTS; i++) { + spinlock_acquire(get_priv_segment(i)->privatization_lock); + } + + /* add this thread's 'pages_privatized' bit */ + ps->by_segment |= bitmask; + + /* "unmaps" the page to make the address space location correspond + again to its underlying file offset (XXX later we should again + attempt to group together many calls to d_remap_file_pages() in + succession) */ + uintptr_t pagenum_in_file = NB_PAGES * STM_SEGMENT->segment_num + pagenum; + char *new_page = stm_object_pages + pagenum_in_file * 4096UL; + d_remap_file_pages(new_page, 4096, pagenum_in_file); + + /* copy the content from the shared (segment 0) source */ + pagecopy(new_page, stm_object_pages + pagenum * 4096UL); + + for (i = NB_SEGMENTS-1; i >= 0; i--) { + spinlock_release(get_priv_segment(i)->privatization_lock); + } +} diff --git a/c8/stm/pages.h b/c8/stm/pages.h --- a/c8/stm/pages.h +++ b/c8/stm/pages.h @@ -21,6 +21,8 @@ static struct page_shared_s pages_privatized[PAGE_FLAG_END - PAGE_FLAG_START]; static void pages_initialize_shared(uintptr_t pagenum, uintptr_t count); +static void page_privatize(uintptr_t pagenum); + static inline bool is_private_page(long segnum, uintptr_t pagenum) { diff --git a/c8/stm/setup.c b/c8/stm/setup.c --- a/c8/stm/setup.c +++ b/c8/stm/setup.c @@ -70,11 +70,11 @@ PROT_NONE); if (i != 0) { - /* let's give all pages to segment 0 at first and make them - write-inaccessible everywhere else */ + /* let's give all pages to segment 0 at first, all others + need to trap and look for the backup copy */ mprotect(segment_base + END_NURSERY_PAGE * 4096, (NB_PAGES - END_NURSERY_PAGE) * 4096, - PROT_READ); + PROT_NONE); } } } @@ -232,13 +232,12 @@ tl->prev = stm_all_thread_locals->prev; stm_all_thread_locals->prev->next = tl; stm_all_thread_locals->prev = tl; - num = tl->prev->associated_segment_num; + num = (tl->prev->associated_segment_num + 1) % NB_SEGMENTS; } /* assign numbers consecutively, but that's for tests; we could also assign the same number to all of them and they would get their own numbers automatically. */ - num = (num + 1) % NB_SEGMENTS; tl->associated_segment_num = num; *_get_cpth(tl) = pthread_self(); _init_shadow_stack(tl); diff --git a/c8/stm/sync.c b/c8/stm/sync.c --- a/c8/stm/sync.c +++ b/c8/stm/sync.c @@ -142,3 +142,8 @@ set_gs_register(get_segment_base(tl->associated_segment_num)); assert(STM_SEGMENT->running_thread == tl); } + +void _stm_test_switch_segment(int segnum) +{ + set_gs_register(get_segment_base(segnum)); +} diff --git a/c8/stmgc.h b/c8/stmgc.h --- a/c8/stmgc.h +++ b/c8/stmgc.h @@ -80,6 +80,7 @@ long stm_can_move(object_t *obj); void _stm_test_switch(stm_thread_local_t *tl); +void _stm_test_switch_segment(int segnum); void _push_obj_to_other_segments(object_t *obj); char *_stm_get_segment_base(long index); diff --git a/c8/test/support.py b/c8/test/support.py --- a/c8/test/support.py +++ b/c8/test/support.py @@ -51,6 +51,7 @@ long stm_can_move(object_t *obj); char *_stm_real_address(object_t *o); void _stm_test_switch(stm_thread_local_t *tl); +void _stm_test_switch_segment(int segnum); void clear_jmpbuf(stm_thread_local_t *tl); long stm_start_transaction(stm_thread_local_t *tl); @@ -395,6 +396,9 @@ lib.stm_setup() self.tls = [_allocate_thread_local() for i in range(self.NB_THREADS)] self.current_thread = 0 + # force-switch back to segment 0 so that when we do something + # outside of transactions before the test, it happens in seg0 + self.switch_to_segment(0) def teardown_method(self, meth): lib.stmcb_expand_marker = ffi.NULL @@ -452,6 +456,9 @@ lib._stm_test_switch(tl2) stm_validate() # can raise + def switch_to_segment(self, seg_num): + lib._stm_test_switch_segment(seg_num) + def push_root(self, o): assert ffi.typeof(o) == ffi.typeof("object_t *") tl = self.tls[self.current_thread] diff --git a/c8/test/test_basic.py b/c8/test/test_basic.py --- a/c8/test/test_basic.py +++ b/c8/test/test_basic.py @@ -56,7 +56,7 @@ def test_allocate_old(self): lp1 = stm_allocate_old(16) - self.switch(1) + self.switch(1) # actually has not much of an effect... lp2 = stm_allocate_old(16) assert lp1 != lp2 From noreply at buildbot.pypy.org Thu Sep 4 10:34:21 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 4 Sep 2014 10:34:21 +0200 (CEST) Subject: [pypy-commit] pypy default: Trying to output the stdout/stderr but only in failure cases Message-ID: <20140904083421.784381C243B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73295:59dec5f6241b Date: 2014-09-04 10:33 +0200 http://bitbucket.org/pypy/pypy/changeset/59dec5f6241b/ Log: Trying to output the stdout/stderr but only in failure cases diff --git a/_pytest/resultlog.py b/_pytest/resultlog.py --- a/_pytest/resultlog.py +++ b/_pytest/resultlog.py @@ -57,7 +57,8 @@ _safeprint("%s %s" % (lettercode, testpath), file=self.logfile) for line in longrepr.splitlines(): _safeprint(" %s" % line, file=self.logfile) - if 0 and sections is not None: # XXX XXX USES TOO MUCH MEMORY?? + if sections is not None and ( + lettercode in ('E', 'F')): # to limit the size of logs for title, content in sections: _safeprint(" ---------- %s ----------" % (title,), file=self.logfile) From noreply at buildbot.pypy.org Thu Sep 4 10:38:13 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 4 Sep 2014 10:38:13 +0200 (CEST) Subject: [pypy-commit] pypy default: Translation fix Message-ID: <20140904083813.90AD01C243B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73296:fe132a89e725 Date: 2014-09-04 10:37 +0200 http://bitbucket.org/pypy/pypy/changeset/fe132a89e725/ Log: Translation fix diff --git a/rpython/jit/backend/arm/codebuilder.py b/rpython/jit/backend/arm/codebuilder.py --- a/rpython/jit/backend/arm/codebuilder.py +++ b/rpython/jit/backend/arm/codebuilder.py @@ -333,8 +333,11 @@ | (rn & 0xF) << 16) def DMB(self): - # note: 'cond' is only permitted on Thumb here - self.write32(0xf57ff05f) + # note: 'cond' is only permitted on Thumb here, but don't + # write literally 0xf57ff05f, because it's larger than 31 bits + c = cond.AL + self.write32(c << 28 + | 0x157ff05f) DIV = binary_helper_call('int_div') MOD = binary_helper_call('int_mod') From noreply at buildbot.pypy.org Thu Sep 4 11:13:42 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Thu, 4 Sep 2014 11:13:42 +0200 (CEST) Subject: [pypy-commit] stmgc default: more questions Message-ID: <20140904091342.3A7C71C129D@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1345:72825d611f91 Date: 2014-09-04 10:51 +0200 http://bitbucket.org/pypy/stmgc/changeset/72825d611f91/ Log: more questions diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -15,6 +15,8 @@ /* send to GDB */ kill(getpid(), SIGINT); } + /* XXX: should we save 'errno'? */ + /* make PROT_READWRITE again and validate */ int segnum = get_segment_of_linear_address(addr); OPT_ASSERT(segnum == STM_SEGMENT->segment_num); @@ -25,6 +27,8 @@ /* XXX: missing synchronisation: we may change protection, then another thread changes it back, then we try to privatize that calls page_copy() and traps */ + /* XXX: mprotect is not reentrant and interruptible by signals, + so it needs additional synchronisation.*/ mprotect(seg_base + pagenum * 4096UL, 4096, PROT_READ|PROT_WRITE); page_privatize(pagenum); From noreply at buildbot.pypy.org Thu Sep 4 11:13:43 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Thu, 4 Sep 2014 11:13:43 +0200 (CEST) Subject: [pypy-commit] stmgc default: attempt which ended in more questions Message-ID: <20140904091343.5857D1C129D@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1346:0fa234f3b87a Date: 2014-09-04 11:14 +0200 http://bitbucket.org/pypy/stmgc/changeset/0fa234f3b87a/ Log: attempt which ended in more questions diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -60,6 +60,14 @@ } } +static void _update_obj_from(int from_seg, object_t *obj) +{ + /* check if its pages are private, only then we need + to update them. If they are also still read-protected, + we may trigger the signal handler. This would cause + it to also enter stm_validate()..... */ +} + void stm_validate(void *free_if_abort) { struct stm_commit_log_entry_s *cl = STM_PSEGMENT->last_commit_log_entry; @@ -71,7 +79,12 @@ object_t *obj; while ((obj = cl->written[i])) { - /* XXX: update our copies */ + /* in case this entry's transaction has not yet discarded + the backup copies, wait. */ + while (cl->committing) + spin_loop(); + + _update_obj_from(cl->segment_num, obj); if (_stm_was_read(obj)) { free(free_if_abort); @@ -100,11 +113,12 @@ result->written[i] = (object_t*)list_item(lst, i); } result->written[count] = NULL; + spinlock_acquire(result->committing); /* =true */ return result; } -static void _validate_and_add_to_commit_log() +static struct stm_commit_log_entry_s *_validate_and_add_to_commit_log() { struct stm_commit_log_entry_s *new; void* *to; @@ -116,6 +130,8 @@ to = (void **)&(STM_PSEGMENT->last_commit_log_entry->next); } while (!__sync_bool_compare_and_swap((void**)to, (void**)NULL, (void**)new)); + + return new; } /* ############# STM ############# */ @@ -242,9 +258,9 @@ minor_collection(1); - _validate_and_add_to_commit_log(); - + struct stm_commit_log_entry_s* entry = _validate_and_add_to_commit_log(); /* XXX:discard backup copies */ + spinlock_release(entry->committing); s_mutex_lock(); diff --git a/c8/stm/core.h b/c8/stm/core.h --- a/c8/stm/core.h +++ b/c8/stm/core.h @@ -65,6 +65,7 @@ /* Commit Log things */ struct stm_commit_log_entry_s { struct stm_commit_log_entry_s *next; + bool committing; /* true while not yet removed backup copies */ int segment_num; object_t *written[]; /* terminated with a NULL ptr */ }; From noreply at buildbot.pypy.org Thu Sep 4 11:19:02 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 4 Sep 2014 11:19:02 +0200 (CEST) Subject: [pypy-commit] stmgc default: Simplify the typecasts Message-ID: <20140904091902.912711C024B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1347:b2171f4a0a0b Date: 2014-09-04 11:19 +0200 http://bitbucket.org/pypy/stmgc/changeset/b2171f4a0a0b/ Log: Simplify the typecasts diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -120,16 +120,15 @@ static struct stm_commit_log_entry_s *_validate_and_add_to_commit_log() { - struct stm_commit_log_entry_s *new; - void* *to; + struct stm_commit_log_entry_s *new, **to; new = _create_commit_log_entry(); fprintf(stderr,"%p\n", new); do { stm_validate(new); - to = (void **)&(STM_PSEGMENT->last_commit_log_entry->next); - } while (!__sync_bool_compare_and_swap((void**)to, (void**)NULL, (void**)new)); + to = &(STM_PSEGMENT->last_commit_log_entry->next); + } while (!__sync_bool_compare_and_swap(to, NULL, new)); return new; } From noreply at buildbot.pypy.org Thu Sep 4 11:19:50 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Thu, 4 Sep 2014 11:19:50 +0200 (CEST) Subject: [pypy-commit] stmgc default: let's just do that for now Message-ID: <20140904091950.337DF1C024B@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1348:45b6f201404c Date: 2014-09-04 11:20 +0200 http://bitbucket.org/pypy/stmgc/changeset/45b6f201404c/ Log: let's just do that for now diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -113,6 +113,7 @@ result->written[i] = (object_t*)list_item(lst, i); } result->written[count] = NULL; + result->committing = false; spinlock_acquire(result->committing); /* =true */ return result; From noreply at buildbot.pypy.org Thu Sep 4 11:19:51 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Thu, 4 Sep 2014 11:19:51 +0200 (CEST) Subject: [pypy-commit] stmgc default: Merge Message-ID: <20140904091951.50C771C024B@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1349:272ec2f42b17 Date: 2014-09-04 11:20 +0200 http://bitbucket.org/pypy/stmgc/changeset/272ec2f42b17/ Log: Merge diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -121,16 +121,15 @@ static struct stm_commit_log_entry_s *_validate_and_add_to_commit_log() { - struct stm_commit_log_entry_s *new; - void* *to; + struct stm_commit_log_entry_s *new, **to; new = _create_commit_log_entry(); fprintf(stderr,"%p\n", new); do { stm_validate(new); - to = (void **)&(STM_PSEGMENT->last_commit_log_entry->next); - } while (!__sync_bool_compare_and_swap((void**)to, (void**)NULL, (void**)new)); + to = &(STM_PSEGMENT->last_commit_log_entry->next); + } while (!__sync_bool_compare_and_swap(to, NULL, new)); return new; } From noreply at buildbot.pypy.org Thu Sep 4 13:25:00 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Thu, 4 Sep 2014 13:25:00 +0200 (CEST) Subject: [pypy-commit] stmgc default: make modified_old_objs a tree Message-ID: <20140904112500.296CA1C30CB@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1350:8a58897dff57 Date: 2014-09-04 13:25 +0200 http://bitbucket.org/pypy/stmgc/changeset/8a58897dff57/ Log: make modified_old_objs a tree diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -45,7 +45,8 @@ void _dbg_print_commit_log() { - struct stm_commit_log_entry_s *cl = &commit_log_root; + volatile struct stm_commit_log_entry_s *cl = (volatile struct stm_commit_log_entry_s *) + &commit_log_root; fprintf(stderr, "root (%p, %d)\n", cl->next, cl->segment_num); while ((cl = cl->next)) { @@ -70,7 +71,8 @@ void stm_validate(void *free_if_abort) { - struct stm_commit_log_entry_s *cl = STM_PSEGMENT->last_commit_log_entry; + volatile struct stm_commit_log_entry_s *cl = (volatile struct stm_commit_log_entry_s *) + STM_PSEGMENT->last_commit_log_entry; /* Don't check 'cl'. This entry is already checked */ while ((cl = cl->next)) { @@ -79,10 +81,6 @@ object_t *obj; while ((obj = cl->written[i])) { - /* in case this entry's transaction has not yet discarded - the backup copies, wait. */ - while (cl->committing) - spin_loop(); _update_obj_from(cl->segment_num, obj); @@ -95,33 +93,37 @@ }; /* last fully validated entry */ - STM_PSEGMENT->last_commit_log_entry = cl; + STM_PSEGMENT->last_commit_log_entry = (struct stm_commit_log_entry_s *)cl; } } static struct stm_commit_log_entry_s *_create_commit_log_entry() { - struct list_s *lst = STM_PSEGMENT->modified_old_objects; - size_t count = list_count(lst); + struct tree_s *tree = STM_PSEGMENT->modified_old_objects; + size_t count = tree_count(tree); size_t byte_len = sizeof(struct stm_commit_log_entry_s) + (count + 1) * sizeof(object_t*); struct stm_commit_log_entry_s *result = malloc(byte_len); - int i; result->next = NULL; result->segment_num = STM_SEGMENT->segment_num; - for (i = 0; i < count; i++) { - result->written[i] = (object_t*)list_item(lst, i); - } + + int i = 0; + wlog_t *item; + TREE_LOOP_FORWARD(tree, item); + result->written[i] = (object_t*)item->addr; + i++; + TREE_LOOP_END; + + OPT_ASSERT(count == i); result->written[count] = NULL; - result->committing = false; - spinlock_acquire(result->committing); /* =true */ return result; } static struct stm_commit_log_entry_s *_validate_and_add_to_commit_log() { - struct stm_commit_log_entry_s *new, **to; + struct stm_commit_log_entry_s *new; + volatile struct stm_commit_log_entry_s **to; new = _create_commit_log_entry(); fprintf(stderr,"%p\n", new); @@ -169,7 +171,7 @@ dprintf(("prot %lu, len=%lu in seg %lu\n", first_page, (end_page - first_page + 1), i)); } - LIST_APPEND(STM_PSEGMENT->modified_old_objects, obj); + tree_insert(STM_PSEGMENT->modified_old_objects, (uintptr_t)obj, 0); LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, obj); obj->stm_flags &= ~GCFLAG_WRITE_BARRIER; @@ -221,7 +223,7 @@ reset_transaction_read_version(); } - assert(list_is_empty(STM_PSEGMENT->modified_old_objects)); + assert(tree_count(STM_PSEGMENT->modified_old_objects) == 0); assert(list_is_empty(STM_PSEGMENT->objects_pointing_to_nursery)); check_nursery_at_transaction_start(); } @@ -260,7 +262,6 @@ struct stm_commit_log_entry_s* entry = _validate_and_add_to_commit_log(); /* XXX:discard backup copies */ - spinlock_release(entry->committing); s_mutex_lock(); diff --git a/c8/stm/core.h b/c8/stm/core.h --- a/c8/stm/core.h +++ b/c8/stm/core.h @@ -48,7 +48,7 @@ struct stm_priv_segment_info_s { struct stm_segment_info_s pub; - struct list_s *modified_old_objects; + struct tree_s *modified_old_objects; struct list_s *objects_pointing_to_nursery; uint8_t privatization_lock; @@ -64,8 +64,7 @@ /* Commit Log things */ struct stm_commit_log_entry_s { - struct stm_commit_log_entry_s *next; - bool committing; /* true while not yet removed backup copies */ + volatile struct stm_commit_log_entry_s *next; int segment_num; object_t *written[]; /* terminated with a NULL ptr */ }; diff --git a/c8/stm/list.c b/c8/stm/list.c --- a/c8/stm/list.c +++ b/c8/stm/list.c @@ -43,6 +43,7 @@ if (tree->raw_current != tree->raw_start) { _tree_clear_node(&tree->toplevel); tree->raw_current = tree->raw_start; + tree->count = 0; } } @@ -64,7 +65,7 @@ struct tree_s tree_copy; memset(&tree_copy, 0, sizeof(struct tree_s)); - TREE_LOOP_FORWARD(*tree, item) { + TREE_LOOP_FORWARD(tree, item) { tree_insert(&tree_copy, item->addr, item->val); } TREE_LOOP_END; @@ -99,7 +100,7 @@ newtree.raw_current = newitems; newtree.raw_end = newitems + newalloc; _tree_clear_node(&newtree.toplevel); - TREE_LOOP_FORWARD(*tree, item) + TREE_LOOP_FORWARD(tree, item) { tree_insert(&newtree, item->addr, item->val); } TREE_LOOP_END; @@ -145,6 +146,7 @@ /* reuse the deleted entry and that's it */ wlog1->addr = addr; wlog1->val = val; + tree->count++; return; } /* the key must not already be present */ @@ -166,15 +168,29 @@ wlog->addr = addr; wlog->val = val; *(char **)p = (char *)wlog; + tree->count++; } static bool tree_delete_item(struct tree_s *tree, uintptr_t addr) { wlog_t *entry; - TREE_FIND(*tree, addr, entry, goto missing); + TREE_FIND(tree, addr, entry, goto missing); entry->addr = 0; + tree->count--; return true; missing: return false; } + +static wlog_t *tree_item(struct tree_s *tree, int index) +{ + int i = 0; + wlog_t *item; + TREE_LOOP_FORWARD(tree, item); + if (i == index) + return item; + i++; + TREE_LOOP_END; + return NULL; +} diff --git a/c8/stm/list.h b/c8/stm/list.h --- a/c8/stm/list.h +++ b/c8/stm/list.h @@ -125,6 +125,7 @@ struct tree_s { char *raw_start, *raw_current, *raw_end; + uintptr_t count; wlog_node_t toplevel; }; @@ -137,12 +138,16 @@ return tree->raw_current == tree->raw_start; } +static inline uintptr_t tree_count(struct tree_s *tree) { + return tree->count; +} + #define _TREE_LOOP(tree, item, INITIAL, _PLUS_) \ { \ struct { char **next; char **end; } _stack[TREE_DEPTH_MAX], *_stackp; \ char **_next, **_end, *_entry; \ long _deleted_factor = 0; \ - struct tree_s *_tree = &(tree); \ + struct tree_s *_tree = (tree); \ /* initialization */ \ _stackp = _stack; /* empty stack */ \ _next = _tree->toplevel.items + INITIAL; \ @@ -189,12 +194,12 @@ #define TREE_LOOP_END } } #define TREE_LOOP_END_AND_COMPRESS \ } if (_deleted_factor > 9) _tree_compress(_tree); } -#define TREE_LOOP_DELETE(item) { (item)->addr = NULL; _deleted_factor += 6; } +#define TREE_LOOP_DELETE(tree, item) { (tree)->count--; (item)->addr = NULL; _deleted_factor += 6; } #define TREE_FIND(tree, addr1, result, goto_not_found) \ { \ uintptr_t _key = TREE_HASH(addr1); \ - char *_p = (char *)((tree).toplevel.items); \ + char *_p = (char *)((tree)->toplevel.items); \ char *_entry = *(char **)(_p + (_key & TREE_MASK)); \ if (_entry == NULL) \ goto_not_found; /* common case, hopefully */ \ @@ -208,10 +213,11 @@ static void tree_insert(struct tree_s *tree, uintptr_t addr, uintptr_t val); static bool tree_delete_item(struct tree_s *tree, uintptr_t addr) __attribute__((unused)); +static wlog_t *tree_item(struct tree_s *tree, int index); /* SLOW */ static inline bool tree_contains(struct tree_s *tree, uintptr_t addr) { wlog_t *result; - TREE_FIND(*tree, addr, result, return false); + TREE_FIND(tree, addr, result, return false); return true; } diff --git a/c8/stm/misc.c b/c8/stm/misc.c --- a/c8/stm/misc.c +++ b/c8/stm/misc.c @@ -48,7 +48,7 @@ { if (STM_PSEGMENT->modified_old_objects == NULL) return -1; - return list_count(STM_PSEGMENT->modified_old_objects); + return tree_count(STM_PSEGMENT->modified_old_objects); } long _stm_count_objects_pointing_to_nursery(void) @@ -60,8 +60,8 @@ object_t *_stm_enum_modified_old_objects(long index) { - return (object_t *)list_item( - STM_PSEGMENT->modified_old_objects, index); + wlog_t* entry = tree_item(STM_PSEGMENT->modified_old_objects, index); + return (object_t*)entry->addr; } object_t *_stm_enum_objects_pointing_to_nursery(long index) diff --git a/c8/stm/setup.c b/c8/stm/setup.c --- a/c8/stm/setup.c +++ b/c8/stm/setup.c @@ -130,7 +130,7 @@ assert(0 <= i && i < 255); /* 255 is WL_VISITED in gcpage.c */ pr->pub.segment_num = i; pr->pub.segment_base = segment_base; - pr->modified_old_objects = list_create(); + pr->modified_old_objects = tree_create(); pr->objects_pointing_to_nursery = list_create(); pr->last_commit_log_entry = &commit_log_root; pr->pub.transaction_read_version = 0xff; @@ -162,7 +162,7 @@ struct stm_priv_segment_info_s *pr = get_priv_segment(i); assert(list_is_empty(pr->objects_pointing_to_nursery)); list_free(pr->objects_pointing_to_nursery); - list_free(pr->modified_old_objects); + tree_free(pr->modified_old_objects); } munmap(stm_object_pages, TOTAL_MEMORY); From noreply at buildbot.pypy.org Thu Sep 4 14:26:37 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 4 Sep 2014 14:26:37 +0200 (CEST) Subject: [pypy-commit] pypy default: Fix test Message-ID: <20140904122637.1DA181C024B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73297:b15ac44c3739 Date: 2014-09-04 14:21 +0200 http://bitbucket.org/pypy/pypy/changeset/b15ac44c3739/ Log: Fix test diff --git a/pypy/module/pypyjit/test_pypy_c/test_cprofile.py b/pypy/module/pypyjit/test_pypy_c/test_cprofile.py --- a/pypy/module/pypyjit/test_pypy_c/test_cprofile.py +++ b/pypy/module/pypyjit/test_pypy_c/test_cprofile.py @@ -1,4 +1,4 @@ -import py, sys +import py, sys, re from pypy.module.pypyjit.test_pypy_c.test_00_model import BaseTestPyPyC class TestCProfile(BaseTestPyPyC): @@ -28,8 +28,18 @@ print loop.ops_by_id(method) # on 32-bit, there is f1=call(read_timestamp); ...; # f2=call(read_timestamp); f3=call(llong_sub,f1,f2) - # but all calls can be special-cased by the backend if supported - if sys.maxint != 2147483647: - assert ' call(' not in repr(loop.ops_by_id(method)) + # but all calls can be special-cased by the backend if + # supported. On 64-bit there is only the two calls to + # read_timestamp. + r = re.compile(r" call[(]ConstClass[(](.+?)[)]") + calls = r.findall(repr(loop.ops_by_id(method))) + if sys.maxint == 2147483647: + assert len(calls) == 6 + else: + assert len(calls) == 2 + for x in calls: + assert ('ll_read_timestamp' in x or 'llong_sub' in x + or 'llong_add' in x) + # assert ' call_may_force(' not in repr(loop.ops_by_id(method)) assert ' cond_call(' in repr(loop.ops_by_id(method)) From noreply at buildbot.pypy.org Thu Sep 4 14:26:38 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 4 Sep 2014 14:26:38 +0200 (CEST) Subject: [pypy-commit] pypy default: 32-bit JIT: avoid one obscure "call(llong_sub, x, 0)" Message-ID: <20140904122638.42D6B1C024B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73298:3f18f8fae752 Date: 2014-09-04 14:26 +0200 http://bitbucket.org/pypy/pypy/changeset/3f18f8fae752/ Log: 32-bit JIT: avoid one obscure "call(llong_sub, x, 0)" diff --git a/pypy/module/_lsprof/interp_lsprof.py b/pypy/module/_lsprof/interp_lsprof.py --- a/pypy/module/_lsprof/interp_lsprof.py +++ b/pypy/module/_lsprof/interp_lsprof.py @@ -175,7 +175,11 @@ def _stop(self, profobj, entry): tt = profobj.ll_timer() - self.ll_t0 - it = tt - self.ll_subt + ll_subt = self.ll_subt + if jit.isconstant(ll_subt) and not ll_subt: + it = tt # for 32-bit JIT: avoid the 'call(llong_sub, tt, 0)' + else: + it = tt - ll_subt if self.previous: self.previous.ll_subt += tt entry._stop(tt, it) diff --git a/pypy/module/pypyjit/test_pypy_c/test_cprofile.py b/pypy/module/pypyjit/test_pypy_c/test_cprofile.py --- a/pypy/module/pypyjit/test_pypy_c/test_cprofile.py +++ b/pypy/module/pypyjit/test_pypy_c/test_cprofile.py @@ -34,7 +34,7 @@ r = re.compile(r" call[(]ConstClass[(](.+?)[)]") calls = r.findall(repr(loop.ops_by_id(method))) if sys.maxint == 2147483647: - assert len(calls) == 6 + assert len(calls) == 5 else: assert len(calls) == 2 for x in calls: From noreply at buildbot.pypy.org Thu Sep 4 15:30:13 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 4 Sep 2014 15:30:13 +0200 (CEST) Subject: [pypy-commit] pypy default: The list comprehension optimization was not enabled in RPython loops Message-ID: <20140904133013.05B701C024B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73299:3ffe46ffd7ba Date: 2014-09-04 15:29 +0200 http://bitbucket.org/pypy/pypy/changeset/3ffe46ffd7ba/ Log: The list comprehension optimization was not enabled in RPython loops over strings (only lists and dicts). diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py --- a/rpython/annotator/unaryop.py +++ b/rpython/annotator/unaryop.py @@ -304,10 +304,10 @@ def hint(self, *args_s): hints = args_s[-1].const if 'maxlength' in hints: - # only for iteration over lists or dicts at the moment, + # only for iteration over lists or dicts or strs at the moment, # not over an iterator object (because it has no known length) s_iterable = args_s[0] - if isinstance(s_iterable, (SomeList, SomeDict)): + if isinstance(s_iterable, (SomeList, SomeDict, SomeString)): self = SomeList(self.listdef) # create a fresh copy self.listdef.resize() self.listdef.listitem.hint_maxlength = True diff --git a/rpython/translator/test/test_simplify.py b/rpython/translator/test/test_simplify.py --- a/rpython/translator/test/test_simplify.py +++ b/rpython/translator/test/test_simplify.py @@ -328,6 +328,12 @@ interp = LLInterpreter(t.rtyper) return interp, graph + def no_resize(self, graph): + for block in graph.iterblocks(): + for op in block.operations: + if op.opname == 'direct_call': + assert 'list_resize' not in repr(op.args[0]) + def test_simple(self): def main(n): lst = [x*17 for x in range(n)] @@ -335,6 +341,16 @@ interp, graph = self.specialize(main, [int]) res = interp.eval_graph(graph, [10]) assert res == 5 * 17 + self.no_resize(graph) + + def test_str2list(self): + def main(n): + lst = [c for c in str(n)] + return len(lst) + interp, graph = self.specialize(main, [int]) + res = interp.eval_graph(graph, [1091283]) + assert res == 7 + self.no_resize(graph) def test_simple_non_exact(self): def main(n): @@ -343,6 +359,7 @@ interp, graph = self.specialize(main, [int]) res = interp.eval_graph(graph, [10]) assert res == 5 + self.no_resize(graph) def test_mutated_after_listcomp(self): def main(n): From noreply at buildbot.pypy.org Thu Sep 4 15:34:35 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 4 Sep 2014 15:34:35 +0200 (CEST) Subject: [pypy-commit] pypy default: Iterating over an RPython string can load the length of the string only once Message-ID: <20140904133435.0A3C01C024B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73300:433eafa3297b Date: 2014-09-04 15:34 +0200 http://bitbucket.org/pypy/pypy/changeset/433eafa3297b/ Log: Iterating over an RPython string can load the length of the string only once diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py --- a/rpython/rtyper/lltypesystem/rstr.py +++ b/rpython/rtyper/lltypesystem/rstr.py @@ -1213,6 +1213,7 @@ external_item_repr = char_repr lowleveltype = Ptr(GcStruct('stringiter', ('string', string_repr.lowleveltype), + ('length', Signed), ('index', Signed))) class UnicodeIteratorRepr(BaseStringIteratorRepr): @@ -1220,6 +1221,7 @@ external_item_repr = unichar_repr lowleveltype = Ptr(GcStruct('unicodeiter', ('string', unicode_repr.lowleveltype), + ('length', Signed), ('index', Signed))) def ll_striter(string): @@ -1231,16 +1233,16 @@ raise TypeError("Unknown string type %s" % (typeOf(string),)) iter = malloc(TP) iter.string = string + iter.length = len(string.chars) # load this value only once iter.index = 0 return iter def ll_strnext(iter): - chars = iter.string.chars index = iter.index - if index >= len(chars): + if index >= iter.length: raise StopIteration iter.index = index + 1 - return chars[index] + return iter.string.chars[index] def ll_getnextindex(iter): return iter.index From noreply at buildbot.pypy.org Thu Sep 4 15:59:17 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 4 Sep 2014 15:59:17 +0200 (CEST) Subject: [pypy-commit] pypy.org extradoc: Link to the matching funding blog post Message-ID: <20140904135917.3561D1C024B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r530:17666b561e9b Date: 2014-09-04 15:58 +0200 http://bitbucket.org/pypy/pypy.org/changeset/17666b561e9b/ Log: Link to the matching funding blog post diff --git a/don1.html b/don1.html --- a/don1.html +++ b/don1.html @@ -20,6 +20,7 @@ +
  • September only: double donations!
    • diff --git a/don2.html b/don2.html --- a/don2.html +++ b/don2.html @@ -8,6 +8,7 @@
    • +
  • September only: double donations!
  • diff --git a/don3.html b/don3.html --- a/don3.html +++ b/don3.html @@ -20,6 +20,7 @@
    • +
  • September only: double donations!
    • diff --git a/don4.html b/don4.html --- a/don4.html +++ b/don4.html @@ -22,6 +22,7 @@ +
  • September only: double donations!
    • From noreply at buildbot.pypy.org Thu Sep 4 15:59:18 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 4 Sep 2014 15:59:18 +0200 (CEST) Subject: [pypy-commit] pypy.org extradoc: merge heads Message-ID: <20140904135918.528EA1C024B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r531:434c019b914d Date: 2014-09-04 15:59 +0200 http://bitbucket.org/pypy/pypy.org/changeset/434c019b914d/ Log: merge heads diff --git a/don1.html b/don1.html --- a/don1.html +++ b/don1.html @@ -9,13 +9,13 @@ - $52313 of $105000 (49.8%) + $52380 of $105000 (49.9%)
    diff --git a/don3.html b/don3.html --- a/don3.html +++ b/don3.html @@ -15,7 +15,7 @@ - $48398 of $60000 (80.7%) + $48408 of $60000 (80.7%)
    diff --git a/don4.html b/don4.html --- a/don4.html +++ b/don4.html @@ -17,7 +17,7 @@ 2nd call: - $13914 of $80000 (17.4%) + $13939 of $80000 (17.4%)
    From noreply at buildbot.pypy.org Thu Sep 4 16:04:37 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 4 Sep 2014 16:04:37 +0200 (CEST) Subject: [pypy-commit] pypy default: Backed out changeset 3f18f8fae752 Message-ID: <20140904140437.87AEC1C024B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73301:e12c22423464 Date: 2014-09-04 16:04 +0200 http://bitbucket.org/pypy/pypy/changeset/e12c22423464/ Log: Backed out changeset 3f18f8fae752 Doesn't work. I guess ll_subt is not constant early enough. diff --git a/pypy/module/_lsprof/interp_lsprof.py b/pypy/module/_lsprof/interp_lsprof.py --- a/pypy/module/_lsprof/interp_lsprof.py +++ b/pypy/module/_lsprof/interp_lsprof.py @@ -175,11 +175,7 @@ def _stop(self, profobj, entry): tt = profobj.ll_timer() - self.ll_t0 - ll_subt = self.ll_subt - if jit.isconstant(ll_subt) and not ll_subt: - it = tt # for 32-bit JIT: avoid the 'call(llong_sub, tt, 0)' - else: - it = tt - ll_subt + it = tt - self.ll_subt if self.previous: self.previous.ll_subt += tt entry._stop(tt, it) diff --git a/pypy/module/pypyjit/test_pypy_c/test_cprofile.py b/pypy/module/pypyjit/test_pypy_c/test_cprofile.py --- a/pypy/module/pypyjit/test_pypy_c/test_cprofile.py +++ b/pypy/module/pypyjit/test_pypy_c/test_cprofile.py @@ -34,7 +34,7 @@ r = re.compile(r" call[(]ConstClass[(](.+?)[)]") calls = r.findall(repr(loop.ops_by_id(method))) if sys.maxint == 2147483647: - assert len(calls) == 5 + assert len(calls) == 6 else: assert len(calls) == 2 for x in calls: From noreply at buildbot.pypy.org Thu Sep 4 17:31:34 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Thu, 4 Sep 2014 17:31:34 +0200 (CEST) Subject: [pypy-commit] stmgc default: some refactoring Message-ID: <20140904153134.CE55C1C129D@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1351:419893208d27 Date: 2014-09-04 16:11 +0200 http://bitbucket.org/pypy/stmgc/changeset/419893208d27/ Log: some refactoring diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -29,7 +29,8 @@ calls page_copy() and traps */ /* XXX: mprotect is not reentrant and interruptible by signals, so it needs additional synchronisation.*/ - mprotect(seg_base + pagenum * 4096UL, 4096, PROT_READ|PROT_WRITE); + pages_set_protection(segnum, pagenum, 1, PROT_READ|PROT_WRITE); + page_privatize(pagenum); /* XXX: ... what can go wrong when we abort from inside @@ -74,6 +75,7 @@ volatile struct stm_commit_log_entry_s *cl = (volatile struct stm_commit_log_entry_s *) STM_PSEGMENT->last_commit_log_entry; + bool needs_abort = false; /* Don't check 'cl'. This entry is already checked */ while ((cl = cl->next)) { size_t i = 0; @@ -81,12 +83,10 @@ object_t *obj; while ((obj = cl->written[i])) { - _update_obj_from(cl->segment_num, obj); - if (_stm_was_read(obj)) { - free(free_if_abort); - stm_abort_transaction(); + if (!needs_abort &&_stm_was_read(obj)) { + needs_abort = true; } i++; @@ -95,6 +95,11 @@ /* last fully validated entry */ STM_PSEGMENT->last_commit_log_entry = (struct stm_commit_log_entry_s *)cl; } + + if (needs_abort) { + free(free_if_abort); + stm_abort_transaction(); + } } static struct stm_commit_log_entry_s *_create_commit_log_entry() @@ -156,22 +161,27 @@ char *realobj; size_t obj_size; uintptr_t i, end_page; + int my_segnum = STM_SEGMENT->segment_num; realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj); obj_size = stmcb_size_rounded_up((struct object_s *)realobj); end_page = (((uintptr_t)obj) + obj_size - 1) / 4096UL; for (i = 0; i < NB_SEGMENTS; i++) { - if (i == STM_SEGMENT->segment_num) + if (i == my_segnum) + continue; + if (!is_readable_page(i, first_page)) continue; /* XXX: only do it if not already PROT_NONE */ char *segment_base = get_segment_base(i); - mprotect(segment_base + first_page * 4096, - (end_page - first_page + 1) * 4096, PROT_NONE); + pages_set_protection(i, first_page, end_page - first_page + 1, + PROT_NONE); dprintf(("prot %lu, len=%lu in seg %lu\n", first_page, (end_page - first_page + 1), i)); } + acquire_modified_objs_lock(my_segnum); tree_insert(STM_PSEGMENT->modified_old_objects, (uintptr_t)obj, 0); + release_modified_objs_lock(my_segnum); LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, obj); obj->stm_flags &= ~GCFLAG_WRITE_BARRIER; @@ -261,7 +271,9 @@ minor_collection(1); struct stm_commit_log_entry_s* entry = _validate_and_add_to_commit_log(); + acquire_modified_objs_lock(STM_SEGMENT->segment_num); /* XXX:discard backup copies */ + release_modified_objs_lock(STM_SEGMENT->segment_num); s_mutex_lock(); diff --git a/c8/stm/core.h b/c8/stm/core.h --- a/c8/stm/core.h +++ b/c8/stm/core.h @@ -48,6 +48,7 @@ struct stm_priv_segment_info_s { struct stm_segment_info_s pub; + uint8_t modified_objs_lock; struct tree_s *modified_old_objects; struct list_s *objects_pointing_to_nursery; uint8_t privatization_lock; @@ -127,3 +128,17 @@ &STM_PSEGMENT->privatization_lock); spinlock_release(*lock); } + +static inline void acquire_modified_objs_lock(int segnum) +{ + uint8_t *lock = (uint8_t *)REAL_ADDRESS(get_segment_base(segnum), + &STM_PSEGMENT->modified_objs_lock); + spinlock_acquire(*lock); +} + +static inline void release_modified_objs_lock(int segnum) +{ + uint8_t *lock = (uint8_t *)REAL_ADDRESS(get_segment_base(segnum), + &STM_PSEGMENT->modified_objs_lock); + spinlock_release(*lock); +} diff --git a/c8/stm/pages.c b/c8/stm/pages.c --- a/c8/stm/pages.c +++ b/c8/stm/pages.c @@ -1,7 +1,7 @@ #ifndef _STM_CORE_H_ # error "must be compiled via stmgc.c" #endif - +#include /************************************************************/ @@ -67,12 +67,28 @@ d_remap_file_pages(segment_base + pagenum * 4096UL, count * 4096UL, pagenum); } + + for (i = 0; i < NB_SEGMENTS; i++) { + uint64_t bitmask = 1UL << i; + uintptr_t amount = count; + while (amount-->0) { + volatile struct page_shared_s *ps = (volatile struct page_shared_s *) + &pages_readable[pagenum + amount - PAGE_FLAG_START]; + if (i == 0) { + /* readable */ + ps->by_segment |= bitmask; + } else { + /* not readable (ensured in setup.c) */ + ps->by_segment &= ~bitmask; + } + } + } } static void page_privatize(uintptr_t pagenum) { /* check this thread's 'pages_privatized' bit */ - uint64_t bitmask = 1UL << (STM_SEGMENT->segment_num - 1); + uint64_t bitmask = 1UL << STM_SEGMENT->segment_num; volatile struct page_shared_s *ps = (volatile struct page_shared_s *) &pages_privatized[pagenum - PAGE_FLAG_START]; if (ps->by_segment & bitmask) { @@ -103,3 +119,27 @@ spinlock_release(get_priv_segment(i)->privatization_lock); } } + +static void pages_set_protection(int segnum, uintptr_t pagenum, + uintptr_t count, int prot) +{ + char *addr = get_segment_base(segnum) + pagenum * 4096UL; + mprotect(addr, count * 4096UL, prot); + + long i; + for (i = 0; i < NB_SEGMENTS; i++) { + uint64_t bitmask = 1UL << i; + uintptr_t amount = count; + while (amount-->0) { + volatile struct page_shared_s *ps = (volatile struct page_shared_s *) + &pages_readable[pagenum + amount - PAGE_FLAG_START]; + if (prot == PROT_NONE) { + /* not readable */ + ps->by_segment &= ~bitmask; + } else { + assert(prot == (PROT_READ|PROT_WRITE)); + ps->by_segment |= bitmask; + } + } + } +} diff --git a/c8/stm/pages.h b/c8/stm/pages.h --- a/c8/stm/pages.h +++ b/c8/stm/pages.h @@ -19,10 +19,12 @@ }; static struct page_shared_s pages_privatized[PAGE_FLAG_END - PAGE_FLAG_START]; +static struct page_shared_s pages_readable[PAGE_FLAG_END - PAGE_FLAG_START]; static void pages_initialize_shared(uintptr_t pagenum, uintptr_t count); static void page_privatize(uintptr_t pagenum); - +static void pages_set_protection(int segnum, uintptr_t pagenum, + uintptr_t count, int prot); static inline bool is_private_page(long segnum, uintptr_t pagenum) { @@ -30,3 +32,10 @@ uint64_t bitmask = 1UL << segnum; return (pages_privatized[pagenum - PAGE_FLAG_START].by_segment & bitmask); } + +static inline bool is_readable_page(long segnum, uintptr_t pagenum) +{ + assert(pagenum >= PAGE_FLAG_START); + uint64_t bitmask = 1UL << segnum; + return (pages_readable[pagenum - PAGE_FLAG_START].by_segment & bitmask); +} diff --git a/c8/stm/setup.c b/c8/stm/setup.c --- a/c8/stm/setup.c +++ b/c8/stm/setup.c @@ -75,6 +75,8 @@ mprotect(segment_base + END_NURSERY_PAGE * 4096, (NB_PAGES - END_NURSERY_PAGE) * 4096, PROT_NONE); + /* pages_initialize_shared() makes sure pages_readable + is initialized correctly */ } } } From noreply at buildbot.pypy.org Thu Sep 4 17:31:35 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Thu, 4 Sep 2014 17:31:35 +0200 (CEST) Subject: [pypy-commit] stmgc default: progress Message-ID: <20140904153135.F06D71C129D@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1352:6db6026fb9a6 Date: 2014-09-04 17:32 +0200 http://bitbucket.org/pypy/stmgc/changeset/6db6026fb9a6/ Log: progress diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -7,6 +7,28 @@ /* ############# signal handler ############# */ +static void bring_page_up_to_date(uintptr_t pagenum) +{ + /* pagecopy from somewhere readable, then update + all written objs from that segment */ + long i; + int my_segnum = STM_SEGMENT->segment_num; + + for (i = 0; i < NB_SEGMENTS; i++) { + if (i == my_segnum) + continue; + if (!is_readable_page(i, first_page)) + continue; + + acquire_privatization_lock(i); + + /* copy the content from there to our segment */ + pagecopy(new_page, get_segment_base(from_segnum) + pagenum * 4096UL); + + release_privatization_lock(i); + } +} + void _signal_handler(int sig, siginfo_t *siginfo, void *context) { char *addr = siginfo->si_addr; @@ -31,7 +53,7 @@ so it needs additional synchronisation.*/ pages_set_protection(segnum, pagenum, 1, PROT_READ|PROT_WRITE); - page_privatize(pagenum); + bring_page_up_to_date(pagenum); /* XXX: ... what can go wrong when we abort from inside the signal handler? */ @@ -143,20 +165,11 @@ /* ############# STM ############# */ -void _stm_write_slowpath(object_t *obj) +void _privatize_and_protect_other_segments(object_t *obj) { - assert(_seems_to_be_running_transaction()); - assert(!_is_in_nursery(obj)); - assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); + assert(STM_PSEGMENT->privatization_lock); /* we hold it */ + assert(obj_size < 4096); /* too lazy right now (part of the code is ready) */ - stm_read(obj); - - /* XXX: misses synchronisation with other write_barriers - on same page */ - /* XXX: make backup copy */ - /* XXX: privatize pages if necessary */ - - /* make other segments trap if accessing this object */ uintptr_t first_page = ((uintptr_t)obj) / 4096UL; char *realobj; size_t obj_size; @@ -167,24 +180,85 @@ obj_size = stmcb_size_rounded_up((struct object_s *)realobj); end_page = (((uintptr_t)obj) + obj_size - 1) / 4096UL; + /* privatize pages: */ + /* get the last page containing data from the object */ + end_page = (((uintptr_t)obj) + obj_size - 1) / 4096UL; + for (i = first_page; i <= end_page; i++) { + if (is_private_page(my_segnum, i)) + continue; + page_privatize(i); + bring_page_up_to_date(i); + } + + /* then PROT_NONE everyone else that doesn't have a private + page already */ for (i = 0; i < NB_SEGMENTS; i++) { if (i == my_segnum) continue; - if (!is_readable_page(i, first_page)) + if (!is_readable_page(i, first_page) || is_private_page(i, first_page)) continue; - /* XXX: only do it if not already PROT_NONE */ - char *segment_base = get_segment_base(i); + + acquire_privatization_lock(i); pages_set_protection(i, first_page, end_page - first_page + 1, PROT_NONE); + release_privatization_lock(i); + dprintf(("prot %lu, len=%lu in seg %lu\n", first_page, (end_page - first_page + 1), i)); } +} +void _stm_write_slowpath(object_t *obj) +{ + assert(_seems_to_be_running_transaction()); + assert(!_is_in_nursery(obj)); + assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); + + int my_segnum = STM_SEGMENT->segment_num; + uintptr_t first_page = ((uintptr_t)obj) / 4096UL; + char *realobj; + size_t obj_size; + + realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj); + obj_size = stmcb_size_rounded_up((struct object_s *)realobj); + + /* add to read set: */ + stm_read(obj); + + /* create backup copy: */ + struct object_s *bk_obj = malloc(obj_size); + memcpy(bk_obj, realobj, obj_size); + + assert(obj_size < 4096); /* too lazy right now (part of the code is ready) */ + + retry: + acquire_privatization_lock(my_segnum); + if (!is_readable_page(my_segnum, first_page)) { + release_privatization_lock(my_segnum); + + bring_page_up_to_date(first_page); + + spin_loop(); + goto retry; + } + /* page is not PROT_NONE for us, we can PROT_NONE all others */ + _privatize_and_protect_other_segments(obj); + + /* remove the WRITE_BARRIER flag (could be done later, but I + think we find more bugs this way) */ + obj->stm_flags &= ~GCFLAG_WRITE_BARRIER; + + /* done fiddling with protection and privatization */ + release_privatization_lock(my_segnum); + + /* phew, now add the obj to the write-set and register the + backup copy. */ acquire_modified_objs_lock(my_segnum); - tree_insert(STM_PSEGMENT->modified_old_objects, (uintptr_t)obj, 0); + tree_insert(STM_PSEGMENT->modified_old_objects, + (uintptr_t)obj, (uintptr_t)bk_obj); release_modified_objs_lock(my_segnum); + /* also add it to the GC list for minor collections */ LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, obj); - obj->stm_flags &= ~GCFLAG_WRITE_BARRIER; } static void reset_transaction_read_version(void) diff --git a/c8/stm/core.h b/c8/stm/core.h --- a/c8/stm/core.h +++ b/c8/stm/core.h @@ -115,30 +115,22 @@ asm("/* workaround for llvm bug */"); } -static inline void acquire_privatization_lock(void) +static inline void acquire_privatization_lock(int segnum) { - uint8_t *lock = (uint8_t *)REAL_ADDRESS(STM_SEGMENT->segment_base, - &STM_PSEGMENT->privatization_lock); - spinlock_acquire(*lock); + spinlock_acquire(get_priv_segment(segnum)->privatization_lock); } -static inline void release_privatization_lock(void) +static inline void release_privatization_lock(int segnum) { - uint8_t *lock = (uint8_t *)REAL_ADDRESS(STM_SEGMENT->segment_base, - &STM_PSEGMENT->privatization_lock); - spinlock_release(*lock); + spinlock_release(get_priv_segment(segnum)->privatization_lock); } static inline void acquire_modified_objs_lock(int segnum) { - uint8_t *lock = (uint8_t *)REAL_ADDRESS(get_segment_base(segnum), - &STM_PSEGMENT->modified_objs_lock); - spinlock_acquire(*lock); + spinlock_acquire(get_priv_segment(segnum)->modified_objs_lock); } static inline void release_modified_objs_lock(int segnum) { - uint8_t *lock = (uint8_t *)REAL_ADDRESS(get_segment_base(segnum), - &STM_PSEGMENT->modified_objs_lock); - spinlock_release(*lock); + spinlock_release(get_priv_segment(segnum)->modified_objs_lock); } diff --git a/c8/stm/pages.c b/c8/stm/pages.c --- a/c8/stm/pages.c +++ b/c8/stm/pages.c @@ -74,12 +74,17 @@ while (amount-->0) { volatile struct page_shared_s *ps = (volatile struct page_shared_s *) &pages_readable[pagenum + amount - PAGE_FLAG_START]; + volatile struct page_shared_s *ps2 = (volatile struct page_shared_s *) + &pages_privatized[pagenum + amount - PAGE_FLAG_START]; + if (i == 0) { - /* readable */ + /* readable & private */ ps->by_segment |= bitmask; + ps2->by_segment |= bitmask; } else { - /* not readable (ensured in setup.c) */ + /* not readable (ensured in setup.c), not private */ ps->by_segment &= ~bitmask; + ps2->by_segment &= ~bitmask; } } } @@ -87,6 +92,9 @@ static void page_privatize(uintptr_t pagenum) { + /* hopefully holding the lock */ + assert(STM_PSEGMENT->privatization_lock); + /* check this thread's 'pages_privatized' bit */ uint64_t bitmask = 1UL << STM_SEGMENT->segment_num; volatile struct page_shared_s *ps = (volatile struct page_shared_s *) @@ -96,11 +104,6 @@ return; } - long i; - for (i = 0; i < NB_SEGMENTS; i++) { - spinlock_acquire(get_priv_segment(i)->privatization_lock); - } - /* add this thread's 'pages_privatized' bit */ ps->by_segment |= bitmask; @@ -111,18 +114,14 @@ uintptr_t pagenum_in_file = NB_PAGES * STM_SEGMENT->segment_num + pagenum; char *new_page = stm_object_pages + pagenum_in_file * 4096UL; d_remap_file_pages(new_page, 4096, pagenum_in_file); - - /* copy the content from the shared (segment 0) source */ - pagecopy(new_page, stm_object_pages + pagenum * 4096UL); - - for (i = NB_SEGMENTS-1; i >= 0; i--) { - spinlock_release(get_priv_segment(i)->privatization_lock); - } } static void pages_set_protection(int segnum, uintptr_t pagenum, uintptr_t count, int prot) { + /* we hopefully hold the privatization lock: */ + assert(get_priv_segment(segnum)->privatization_lock); + char *addr = get_segment_base(segnum) + pagenum * 4096UL; mprotect(addr, count * 4096UL, prot); From noreply at buildbot.pypy.org Thu Sep 4 17:36:20 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 4 Sep 2014 17:36:20 +0200 (CEST) Subject: [pypy-commit] pypy default: Rename some variables. Before this, keepalive_until_there() was Message-ID: <20140904153620.1A5DA1C129D@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73302:f917e235b8ac Date: 2014-09-04 17:17 +0200 http://bitbucket.org/pypy/pypy/changeset/f917e235b8ac/ Log: Rename some variables. Before this, keepalive_until_there() was mistakenly called on the address variables, which has no effect! diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py --- a/rpython/rtyper/lltypesystem/rstr.py +++ b/rpython/rtyper/lltypesystem/rstr.py @@ -82,9 +82,9 @@ ll_assert(dststart >= 0, "copystrc: negative dststart") ll_assert(dststart + length <= len(dst.chars), "copystrc: dst ovf") # from here, no GC operations can happen - src = _get_raw_buf(SRC_TP, src, srcstart) - dst = _get_raw_buf(DST_TP, dst, dststart) - llmemory.raw_memcopy(src, dst, llmemory.sizeof(CHAR_TP) * length) + asrc = _get_raw_buf(SRC_TP, src, srcstart) + adst = _get_raw_buf(DST_TP, dst, dststart) + llmemory.raw_memcopy(asrc, adst, llmemory.sizeof(CHAR_TP) * length) # end of "no GC" section keepalive_until_here(src) keepalive_until_here(dst) @@ -102,10 +102,10 @@ # xxx Warning: same note as above apply: don't do this at home assert length >= 0 # from here, no GC operations can happen - src = _get_raw_buf(SRC_TP, src, srcstart) - adr = llmemory.cast_ptr_to_adr(ptrdst) - dstbuf = adr + llmemory.itemoffsetof(typeOf(ptrdst).TO, 0) - llmemory.raw_memcopy(src, dstbuf, llmemory.sizeof(CHAR_TP) * length) + asrc = _get_raw_buf(SRC_TP, src, srcstart) + adst = llmemory.cast_ptr_to_adr(ptrdst) + adst = adst + llmemory.itemoffsetof(typeOf(ptrdst).TO, 0) + llmemory.raw_memcopy(asrc, adst, llmemory.sizeof(CHAR_TP) * length) # end of "no GC" section keepalive_until_here(src) copy_string_to_raw._always_inline_ = True @@ -118,11 +118,11 @@ # xxx Warning: same note as above apply: don't do this at home assert length >= 0 # from here, no GC operations can happen - dst = _get_raw_buf(SRC_TP, dst, dststart) - adr = llmemory.cast_ptr_to_adr(ptrsrc) + adst = _get_raw_buf(SRC_TP, dst, dststart) + asrc = llmemory.cast_ptr_to_adr(ptrsrc) - srcbuf = adr + llmemory.itemoffsetof(typeOf(ptrsrc).TO, 0) - llmemory.raw_memcopy(srcbuf, dst, llmemory.sizeof(CHAR_TP) * length) + asrc = asrc + llmemory.itemoffsetof(typeOf(ptrsrc).TO, 0) + llmemory.raw_memcopy(asrc, adst, llmemory.sizeof(CHAR_TP) * length) # end of "no GC" section keepalive_until_here(dst) copy_raw_to_string._always_inline_ = True From noreply at buildbot.pypy.org Thu Sep 4 17:36:21 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 4 Sep 2014 17:36:21 +0200 (CEST) Subject: [pypy-commit] pypy default: Support list(some_string) and list(some_unicode) in RPython. Message-ID: <20140904153621.47D941C129D@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73303:4b4f0a70d03c Date: 2014-09-04 17:18 +0200 http://bitbucket.org/pypy/pypy/changeset/4b4f0a70d03c/ Log: Support list(some_string) and list(some_unicode) in RPython. diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py --- a/rpython/rtyper/lltypesystem/rstr.py +++ b/rpython/rtyper/lltypesystem/rstr.py @@ -1152,6 +1152,24 @@ return hop.gendirectcall(cls.ll_join_strs, size, vtemp) do_stringformat = classmethod(do_stringformat) + def ll_string2list(RESLIST, src): + length = len(src.chars) + lst = RESLIST.ll_newlist(length) + dst = lst.ll_items() + SRC = typeOf(src).TO # STR or UNICODE + DST = typeOf(dst).TO # GcArray + assert DST.OF is SRC.chars.OF + # from here, no GC operations can happen + asrc = llmemory.cast_ptr_to_adr(src) + ( + llmemory.offsetof(SRC, 'chars') + + llmemory.itemoffsetof(SRC.chars, 0)) + adst = llmemory.cast_ptr_to_adr(dst) + llmemory.itemoffsetof(DST, 0) + llmemory.raw_memcopy(asrc, adst, llmemory.sizeof(DST.OF) * length) + # end of "no GC" section + keepalive_until_here(src) + keepalive_until_here(dst) + return lst + TEMP = GcArray(Ptr(STR)) TEMP_UNICODE = GcArray(Ptr(UNICODE)) diff --git a/rpython/rtyper/rstr.py b/rpython/rtyper/rstr.py --- a/rpython/rtyper/rstr.py +++ b/rpython/rtyper/rstr.py @@ -372,6 +372,17 @@ ll_fn = getattr(r_str.ll, 'll_stringslice_%s' % (kind,)) return hop.gendirectcall(ll_fn, v_str, *vlist) + def rtype_bltn_list(self, hop): + string_repr = hop.args_r[0].repr + if hop.r_result.LIST.ITEM != string_repr.lowleveltype.TO.chars.OF: + raise TyperError("list(str-or-unicode) returns a list of chars; " + "it cannot return a list of %r" % ( + hop.r_result.LIST.ITEM,)) + v_str, = hop.inputargs(string_repr) + cRESLIST = hop.inputconst(Void, hop.r_result.LIST) + hop.exception_is_here() + return hop.gendirectcall(self.ll.ll_string2list, cRESLIST, v_str) + class AbstractUnicodeRepr(AbstractStringRepr): diff --git a/rpython/rtyper/test/test_rlist.py b/rpython/rtyper/test/test_rlist.py --- a/rpython/rtyper/test/test_rlist.py +++ b/rpython/rtyper/test/test_rlist.py @@ -430,6 +430,36 @@ res = self.interpret(dummyfn, ()) assert res == 42 + def test_bltn_list_from_string(self): + def dummyfn(n): + l1 = list(str(n)) + return ord(l1[0]) + res = self.interpret(dummyfn, [71234]) + assert res == ord('7') + + def test_bltn_list_from_unicode(self): + def dummyfn(n): + l1 = list(unicode(str(n))) + return ord(l1[0]) + res = self.interpret(dummyfn, [71234]) + assert res == ord('7') + + def test_bltn_list_from_string_resize(self): + def dummyfn(n): + l1 = list(str(n)) + l1.append('X') + return ord(l1[0]) + res = self.interpret(dummyfn, [71234]) + assert res == ord('7') + + def test_bltn_list_from_unicode_resize(self): + def dummyfn(n): + l1 = list(unicode(str(n))) + l1.append(u'X') + return ord(l1[0]) + res = self.interpret(dummyfn, [71234]) + assert res == ord('7') + def test_is_true(self): def is_true(lst): if lst: From noreply at buildbot.pypy.org Thu Sep 4 17:36:22 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 4 Sep 2014 17:36:22 +0200 (CEST) Subject: [pypy-commit] pypy default: Use the newly RPython list(string) here Message-ID: <20140904153622.714CF1C129D@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73304:1ee3f38d059e Date: 2014-09-04 17:35 +0200 http://bitbucket.org/pypy/pypy/changeset/1ee3f38d059e/ Log: Use the newly RPython list(string) here diff --git a/pypy/module/_io/interp_stringio.py b/pypy/module/_io/interp_stringio.py --- a/pypy/module/_io/interp_stringio.py +++ b/pypy/module/_io/interp_stringio.py @@ -86,7 +86,7 @@ initval = space.unicode_w(w_initval) size = len(initval) self.resize_buffer(size) - self.buf = [c for c in initval] + self.buf = list(initval) pos = space.getindex_w(w_pos, space.w_TypeError) if pos < 0: raise OperationError(space.w_ValueError, diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py --- a/pypy/objspace/std/bytearrayobject.py +++ b/pypy/objspace/std/bytearrayobject.py @@ -534,7 +534,7 @@ if not e.match(space, space.w_TypeError): raise else: - return [c for c in buf.as_str()] + return list(buf.as_str()) # sequence of bytes w_iter = space.iter(w_source) From noreply at buildbot.pypy.org Thu Sep 4 17:42:04 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Thu, 4 Sep 2014 17:42:04 +0200 (CEST) Subject: [pypy-commit] stmgc default: some private, shared, logical page, virtual page, file page, explanation Message-ID: <20140904154204.24F9D1C129D@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1353:7d666a0a78f2 Date: 2014-09-04 17:43 +0200 http://bitbucket.org/pypy/stmgc/changeset/7d666a0a78f2/ Log: some private, shared, logical page, virtual page, file page, explanation diff --git a/c8/stm/pages.h b/c8/stm/pages.h --- a/c8/stm/pages.h +++ b/c8/stm/pages.h @@ -1,3 +1,22 @@ +/* + We have logical pages: one %gs relative pointer can point in some + logical page + We have virtual pages: one virtual address can point in some + virtual page. We have NB_SEGMENTS virtual pages per logical page. + We have file pages: they correspond mostly to physical memory pages + used for mmap/remap_file_pages + + A logical page is SHARED iff all NB_SEGMENTS virtual pages point to + one file page, and thus to the same logical page. + + A logical page becomes PRIVATE if one virtual page still maps to the + original file page, and all others turn read protected. + -> only one can modify it. + + A logical page can also be "PRIVATE IN A SEGMENT", referring to + the virtual page of the segment having its own file page backing. + It also implies the logical page is not read protected. +*/ #define PAGE_FLAG_START END_NURSERY_PAGE #define PAGE_FLAG_END NB_PAGES From noreply at buildbot.pypy.org Thu Sep 4 18:08:40 2014 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 4 Sep 2014 18:08:40 +0200 (CEST) Subject: [pypy-commit] pypy default: rename this file Message-ID: <20140904160840.889901C129D@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r73305:c2604106be80 Date: 2014-09-04 10:07 -0600 http://bitbucket.org/pypy/pypy/changeset/c2604106be80/ Log: rename this file diff --git a/pypy/doc/release-2.4.rst b/pypy/doc/release-2.4.0.rst rename from pypy/doc/release-2.4.rst rename to pypy/doc/release-2.4.0.rst From noreply at buildbot.pypy.org Thu Sep 4 18:26:44 2014 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 4 Sep 2014 18:26:44 +0200 (CEST) Subject: [pypy-commit] pypy default: work on the release announcement, not very happy yet Message-ID: <20140904162644.1C7E11C129D@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r73306:f87daa98dd77 Date: 2014-09-04 10:25 -0600 http://bitbucket.org/pypy/pypy/changeset/f87daa98dd77/ Log: work on the release announcement, not very happy yet diff --git a/pypy/doc/release-2.4.0.rst b/pypy/doc/release-2.4.0.rst --- a/pypy/doc/release-2.4.0.rst +++ b/pypy/doc/release-2.4.0.rst @@ -5,6 +5,8 @@ We're pleased to announce PyPy 2.4, a significant milestone on it's own right and the proud parent of our recent PyPy3 and STM releases. +XXX this sentence is confusing, refactor + This release contains several improvements and bugfixes. You can download the PyPy 2.4 release here: @@ -16,7 +18,9 @@ We've shown quite a bit of progress but we're slowly running out of funds. Please consider donating more, or even better convince your employer to donate, -so we can finish those projects! The three sub-projects are: +so we can finish those projects! We would like to also point out that in +September, `the Python Software Foundation`_ will `match funds`_ for +any donations up to $10k! The three sub-projects are: * `Py3k`_ (supporting Python 3.x): We have released a Python 3.2.5 compatable version we call PyPy3 2.3.1, and are working toward a Python 3.3 compatable version @@ -30,15 +34,19 @@ .. _`STM`: http://pypy.org/tmdonate2.html .. _`NumPy`: http://pypy.org/numpydonate.html .. _`on bitbucket`: https://www.bitbucket.org/pypy/numpy +.. _`the Python Software Foundation`: https://www.python.org/psf/ +.. _`match funds`: http://morepypy.blogspot.com/2014/09/python-software-foundation-matching.html What is PyPy? ============= PyPy is a very compliant Python interpreter, almost a drop-in replacement for -CPython 2.7. It's fast (`pypy 2.3 and cpython 2.7.x`_ performance comparison; +CPython 2.7. It's fast (`pypy 2.4 and cpython 2.7.x`_ performance comparison; note that cpython's speed has not changed since 2.7.2) due to its integrated tracing JIT compiler. +XXX confusing sentence, rewrite + This release supports x86 machines running Linux 32/64, Mac OS X 64, Windows, and OpenBSD, as well as newer ARM hardware (ARMv6 or ARMv7, with VFPv3) running Linux. @@ -47,21 +55,25 @@ bit python is still stalling, we would welcome a volunteer to `handle that`_. -.. _`pypy 2.3 and cpython 2.7.x`: http://speed.pypy.org +.. _`pypy 2.4 and cpython 2.7.x`: http://speed.pypy.org .. _`handle that`: http://doc.pypy.org/en/latest/windows.html#what-is-missing-for-a-full-64-bit-translation Highlights ========== -Benchmarks improved after internal improvements in string and bytearray handling, -and a major rewrite of the GIL handling. Many of these improvements are offshoots -of the STM work. +Benchmarks improved after internal improvements in string and +bytearray handling, and a major rewrite of the GIL handling. This means +that external calls are now a lot faster, especially the CFFI ones. It also +means that a lot of corner cases when handling strings or bytearrays have +better performance. -We merged in Python's 2.7.8 stdlib in a record time of one week, proving the -maturity of our underlying RPython code base and PyPy interpreter. +PyPy now uses Python 2.7.8 standard library. -We welcomed more than 12 new contributors, and conducted two Google Summer of Code -projects XXX details? +We welcomed more than 12 new contributors, and conducted two Google +Summer of Code as well as other student projects not directly related +to Summer of Code. + +XXX mention the work is ongoing and did not make it to 2.4? Issues reported with our previous release were fixed after reports from users on our new issue tracker at https://bitbucket.org/pypy/pypy/issues or on IRC at @@ -74,7 +86,9 @@ handling, which becomes advantageous on large programs at the cost of slightly slower small *benchmark* type programs. -* Boost performance of thread-local variables in both unjitted and jitted code +* Boost performance of thread-local variables in both unjitted and jitted code, + this mostly affects errno handling on linux, which makes external calls + faster. * Move to a mixed polling and mutex GIL model that make mutli-threaded jitted code run *much* faster @@ -85,14 +99,12 @@ * Fix performance regression on ufunc(, ) in numpy -* Classes in the ast module are now distinct from structures used by the compiler, - which simplifies and speeds up translation of our source code to the PyPy binary - interpreter +* Classes in the ast module are now distinct from structures used by + the compiler, which simplifies and speeds up translation of our + source code to the PyPy binary interpreter * Upgrade stdlib from 2.7.5 to 2.7.8 -* - * Many issues were resolved_ since the 2.3.1 release on June 8 .. _`whats-new`: http://doc.pypy.org/en/latest/whatsnew-2.3.1.html From noreply at buildbot.pypy.org Thu Sep 4 22:43:20 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 4 Sep 2014 22:43:20 +0200 (CEST) Subject: [pypy-commit] pypy.org extradoc: update the values Message-ID: <20140904204320.911E81C024B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r532:b47553d7e443 Date: 2014-09-04 22:43 +0200 http://bitbucket.org/pypy/pypy.org/changeset/b47553d7e443/ Log: update the values diff --git a/don3.html b/don3.html --- a/don3.html +++ b/don3.html @@ -15,7 +15,7 @@ - $48408 of $60000 (80.7%) + $48412 of $60000 (80.7%)
    From noreply at buildbot.pypy.org Fri Sep 5 08:13:56 2014 From: noreply at buildbot.pypy.org (mattip) Date: Fri, 5 Sep 2014 08:13:56 +0200 (CEST) Subject: [pypy-commit] pypy release-2.4.x: start release branch Message-ID: <20140905061356.1E7911C0793@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: release-2.4.x Changeset: r73308:854a257b6f28 Date: 2014-09-05 08:30 +0300 http://bitbucket.org/pypy/pypy/changeset/854a257b6f28/ Log: start release branch diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -65,9 +65,9 @@ # built documents. # # The short X.Y version. -version = '2.3' +version = '2.4' # The full version, including alpha/beta/rc tags. -release = '2.3.0' +release = '2.4.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst --- a/pypy/doc/index.rst +++ b/pypy/doc/index.rst @@ -40,7 +40,7 @@ * `FAQ`_: some frequently asked questions. -* `Release 2.3.1`_: the latest official release +* `Release 2.4.0`_: the latest official release * `PyPy Blog`_: news and status info about PyPy @@ -110,7 +110,7 @@ .. _`Getting Started`: getting-started.html .. _`Papers`: extradoc.html .. _`Videos`: video-index.html -.. _`Release 2.3.1`: http://pypy.org/download.html +.. _`Release 2.4.0`: http://pypy.org/download.html .. _`speed.pypy.org`: http://speed.pypy.org .. _`RPython toolchain`: translation.html .. _`potential project ideas`: project-ideas.html diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-2.4.0.rst rename from pypy/doc/whatsnew-head.rst rename to pypy/doc/whatsnew-2.4.0.rst diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h --- a/pypy/module/cpyext/include/patchlevel.h +++ b/pypy/module/cpyext/include/patchlevel.h @@ -29,7 +29,7 @@ #define PY_VERSION "2.7.8" /* PyPy version as a string */ -#define PYPY_VERSION "2.4.0-alpha0" +#define PYPY_VERSION "2.4.0" /* Subversion Revision number of this file (not of the repository). * Empty since Mercurial migration. */ diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -10,7 +10,7 @@ #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py CPYTHON_API_VERSION = 1013 #XXX # sync with include/modsupport.h -PYPY_VERSION = (2, 4, 0, "alpha", 0) #XXX # sync patchlevel.h +PYPY_VERSION = (2, 4, 0, "final", 0) #XXX # sync patchlevel.h if platform.name == 'msvc': COMPILER_INFO = 'MSC v.%d 32 bit' % (platform.version * 10 + 600) From noreply at buildbot.pypy.org Fri Sep 5 08:13:57 2014 From: noreply at buildbot.pypy.org (mattip) Date: Fri, 5 Sep 2014 08:13:57 +0200 (CEST) Subject: [pypy-commit] pypy release-2.4.x: restart whatsnew Message-ID: <20140905061357.716B11C0793@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: release-2.4.x Changeset: r73309:f7a2f261cebb Date: 2014-09-05 08:33 +0300 http://bitbucket.org/pypy/pypy/changeset/f7a2f261cebb/ Log: restart whatsnew diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/whatsnew-head.rst @@ -0,0 +1,8 @@ + +======================= +What's new in PyPy 2.5+ +======================= + +.. this is a revision shortly after release-2.4.x +.. startrev: 204b550079b0 + From noreply at buildbot.pypy.org Fri Sep 5 08:13:58 2014 From: noreply at buildbot.pypy.org (mattip) Date: Fri, 5 Sep 2014 08:13:58 +0200 (CEST) Subject: [pypy-commit] pypy default: restart default as 2.5+ Message-ID: <20140905061358.B12681C0793@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r73310:f751f7e19308 Date: 2014-09-05 09:09 +0300 http://bitbucket.org/pypy/pypy/changeset/f751f7e19308/ Log: restart default as 2.5+ diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -65,9 +65,9 @@ # built documents. # # The short X.Y version. -version = '2.3' +version = '2.4' # The full version, including alpha/beta/rc tags. -release = '2.3.0' +release = '2.4.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst --- a/pypy/doc/index.rst +++ b/pypy/doc/index.rst @@ -40,7 +40,7 @@ * `FAQ`_: some frequently asked questions. -* `Release 2.3.1`_: the latest official release +* `Release 2.4.0`_: the latest official release * `PyPy Blog`_: news and status info about PyPy @@ -110,7 +110,7 @@ .. _`Getting Started`: getting-started.html .. _`Papers`: extradoc.html .. _`Videos`: video-index.html -.. _`Release 2.3.1`: http://pypy.org/download.html +.. _`Release 2.4.0`: http://pypy.org/download.html .. _`speed.pypy.org`: http://speed.pypy.org .. _`RPython toolchain`: translation.html .. _`potential project ideas`: project-ideas.html diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-2.4.0.rst rename from pypy/doc/whatsnew-head.rst rename to pypy/doc/whatsnew-2.4.0.rst diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h --- a/pypy/module/cpyext/include/patchlevel.h +++ b/pypy/module/cpyext/include/patchlevel.h @@ -29,7 +29,7 @@ #define PY_VERSION "2.7.8" /* PyPy version as a string */ -#define PYPY_VERSION "2.4.0-alpha0" +#define PYPY_VERSION "2.5.0-alpha0" /* Subversion Revision number of this file (not of the repository). * Empty since Mercurial migration. */ diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -10,7 +10,7 @@ #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py CPYTHON_API_VERSION = 1013 #XXX # sync with include/modsupport.h -PYPY_VERSION = (2, 4, 0, "alpha", 0) #XXX # sync patchlevel.h +PYPY_VERSION = (2, 5, 0, "alpha", 0) #XXX # sync patchlevel.h if platform.name == 'msvc': COMPILER_INFO = 'MSC v.%d 32 bit' % (platform.version * 10 + 600) From noreply at buildbot.pypy.org Fri Sep 5 08:13:59 2014 From: noreply at buildbot.pypy.org (mattip) Date: Fri, 5 Sep 2014 08:13:59 +0200 (CEST) Subject: [pypy-commit] pypy default: restart whatsnew Message-ID: <20140905061359.E065F1C0793@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r73311:47781ae69774 Date: 2014-09-05 08:33 +0300 http://bitbucket.org/pypy/pypy/changeset/47781ae69774/ Log: restart whatsnew diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/whatsnew-head.rst @@ -0,0 +1,8 @@ + +======================= +What's new in PyPy 2.5+ +======================= + +.. this is a revision shortly after release-2.4.x +.. startrev: 204b550079b0 + From noreply at buildbot.pypy.org Fri Sep 5 08:40:35 2014 From: noreply at buildbot.pypy.org (mattip) Date: Fri, 5 Sep 2014 08:40:35 +0200 (CEST) Subject: [pypy-commit] pypy default: more editing Message-ID: <20140905064035.7EFBE1C1299@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r73312:c0bbec06aee1 Date: 2014-09-05 09:40 +0300 http://bitbucket.org/pypy/pypy/changeset/c0bbec06aee1/ Log: more editing diff --git a/pypy/doc/release-2.4.0.rst b/pypy/doc/release-2.4.0.rst --- a/pypy/doc/release-2.4.0.rst +++ b/pypy/doc/release-2.4.0.rst @@ -2,12 +2,8 @@ PyPy 2.4 - ???????? ================================================= -We're pleased to announce PyPy 2.4, a significant milestone on it's own right -and the proud parent of our recent PyPy3 and STM releases. - -XXX this sentence is confusing, refactor - -This release contains several improvements and bugfixes. +We're pleased to announce PyPy 2.4, which contains significant performance +enhancements and bug fixes. You can download the PyPy 2.4 release here: @@ -28,7 +24,8 @@ * `STM`_ (software transactional memory): We have release a first working version, and continue to try out new promising paths of acheiving a fast multithreaded python -* `NumPy`_ which requires installation of our fork of upstream numpy, available `on bitbucket`_ +* `NumPy`_ which requires installation of our fork of upstream numpy, +available `on bitbucket`_ .. _`Py3k`: http://pypy.org/py3donate.html .. _`STM`: http://pypy.org/tmdonate2.html @@ -41,15 +38,12 @@ ============= PyPy is a very compliant Python interpreter, almost a drop-in replacement for -CPython 2.7. It's fast (`pypy 2.4 and cpython 2.7.x`_ performance comparison; -note that cpython's speed has not changed since 2.7.2) +CPython 2.7. It's fast (`pypy 2.4 and cpython 2.7.x`_ performance comparison) due to its integrated tracing JIT compiler. -XXX confusing sentence, rewrite - -This release supports x86 machines running Linux 32/64, Mac OS X 64, Windows, -and OpenBSD, -as well as newer ARM hardware (ARMv6 or ARMv7, with VFPv3) running Linux. +This release supports **x86** machines on most common operating systems +(Linux 32/64, Mac OS X 64, Windows, and OpenBSD), +as well as newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux. While we support 32 bit python on Windows, work on the native Windows 64 bit python is still stalling, we would welcome a volunteer @@ -70,10 +64,9 @@ PyPy now uses Python 2.7.8 standard library. We welcomed more than 12 new contributors, and conducted two Google -Summer of Code as well as other student projects not directly related -to Summer of Code. +Summer of Code projects, as well as other student projects not +directly related to Summer of Code. -XXX mention the work is ongoing and did not make it to 2.4? Issues reported with our previous release were fixed after reports from users on our new issue tracker at https://bitbucket.org/pypy/pypy/issues or on IRC at @@ -110,6 +103,9 @@ .. _`whats-new`: http://doc.pypy.org/en/latest/whatsnew-2.3.1.html .. _resolved: https://bitbucket.org/pypy/pypy/issues?status=resolved +We have further improvements on the way: rpython file handling and +usable numpy linalg compatabiity should be merged soon. + Please try it out and let us know what you think. We especially welcome success stories, we know you are using PyPy, please tell us about it! From noreply at buildbot.pypy.org Fri Sep 5 10:18:45 2014 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 5 Sep 2014 10:18:45 +0200 (CEST) Subject: [pypy-commit] pypy default: issue #1259 Message-ID: <20140905081845.B93AD1C0793@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73313:1a1f484e66a8 Date: 2014-09-05 10:18 +0200 http://bitbucket.org/pypy/pypy/changeset/1a1f484e66a8/ Log: issue #1259 We don't have many prebuilt pure-Python modules, but for the ones that show up, remove their __file__ rather than translate it statically inside the executable. diff --git a/pypy/interpreter/module.py b/pypy/interpreter/module.py --- a/pypy/interpreter/module.py +++ b/pypy/interpreter/module.py @@ -29,6 +29,17 @@ space.w_None) self.startup_called = False + def _cleanup_(self): + """Called by the annotator on prebuilt Module instances. + We don't have many such modules, but for the ones that + show up, remove their __file__ rather than translate it + statically inside the executable.""" + try: + space = self.space + space.delitem(self.w_dict, space.wrap('__file__')) + except OperationError: + pass + def install(self): """NOT_RPYTHON: installs this module into space.builtin_modules""" w_mod = self.space.wrap(self) diff --git a/pypy/interpreter/test/test_module.py b/pypy/interpreter/test/test_module.py --- a/pypy/interpreter/test/test_module.py +++ b/pypy/interpreter/test/test_module.py @@ -1,4 +1,5 @@ - +import py +from pypy.interpreter.error import OperationError from pypy.interpreter.module import Module class TestModule: @@ -17,6 +18,18 @@ space.raises_w(space.w_AttributeError, space.delattr, w_m, w('x')) + def test___file__(self, space): + w = space.wrap + m = Module(space, space.wrap('m')) + py.test.raises(OperationError, space.getattr, w(m), w('__file__')) + m._cleanup_() + py.test.raises(OperationError, space.getattr, w(m), w('__file__')) + space.setattr(w(m), w('__file__'), w('m.py')) + space.getattr(w(m), w('__file__')) # does not raise + m._cleanup_() + py.test.raises(OperationError, space.getattr, w(m), w('__file__')) + + class AppTest_ModuleObject: def test_attr(self): m = __import__('__builtin__') From noreply at buildbot.pypy.org Fri Sep 5 10:40:52 2014 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 5 Sep 2014 10:40:52 +0200 (CEST) Subject: [pypy-commit] pypy default: increase the resistance of this test (a bit) Message-ID: <20140905084052.43CB51C243B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73314:9de2b25ab1f6 Date: 2014-09-05 10:40 +0200 http://bitbucket.org/pypy/pypy/changeset/9de2b25ab1f6/ Log: increase the resistance of this test (a bit) diff --git a/pypy/module/thread/test/test_thread.py b/pypy/module/thread/test/test_thread.py --- a/pypy/module/thread/test/test_thread.py +++ b/pypy/module/thread/test/test_thread.py @@ -227,7 +227,7 @@ import signal def f(): - for x in range(5): + for x in range(40): if waiting: thread.interrupt_main() return @@ -236,7 +236,7 @@ def busy_wait(): waiting.append(None) - for x in range(10): + for x in range(50): print 'tick...', x # <-force the GIL to be released, as time.sleep(0.1) # time.sleep doesn't do non-translated waiting.pop() @@ -245,6 +245,8 @@ signal.signal(signal.SIGINT, signal.default_int_handler) for i in range(100): + print + print "loop", i waiting = [] thread.start_new_thread(f, ()) raises(KeyboardInterrupt, busy_wait) From noreply at buildbot.pypy.org Fri Sep 5 11:37:55 2014 From: noreply at buildbot.pypy.org (groggi) Date: Fri, 5 Sep 2014 11:37:55 +0200 (CEST) Subject: [pypy-commit] pypy gc-incminimark-pinning: solve "black->white" pointer problem for pinned objects Message-ID: <20140905093755.995CD1C0793@cobra.cs.uni-duesseldorf.de> Author: Gregor Wegberg Branch: gc-incminimark-pinning Changeset: r73315:e9a6ac6462c4 Date: 2014-09-05 11:28 +0200 http://bitbucket.org/pypy/pypy/changeset/e9a6ac6462c4/ Log: solve "black->white" pointer problem for pinned objects diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -1215,6 +1215,12 @@ pass # black -> gray elif self.header(obj).tid & GCFLAG_NO_HEAP_PTRS != 0: pass # black -> white-but-prebuilt-so-dont-care + elif self._is_pinned(obj): + # black -> pinned: the pinned object is a white one as + # every minor collection visits them and takes care of + # visiting pinned objects. + # XXX (groggi) double check with fijal/armin + pass # black -> pinned else: ll_assert(False, "black -> white pointer found") From noreply at buildbot.pypy.org Fri Sep 5 11:37:56 2014 From: noreply at buildbot.pypy.org (groggi) Date: Fri, 5 Sep 2014 11:37:56 +0200 (CEST) Subject: [pypy-commit] pypy gc-incminimark-pinning: add XXX with possible solution for a nursery with too many pinned objects Message-ID: <20140905093756.D608C1C0793@cobra.cs.uni-duesseldorf.de> Author: Gregor Wegberg Branch: gc-incminimark-pinning Changeset: r73316:24a441449f40 Date: 2014-09-05 11:32 +0200 http://bitbucket.org/pypy/pypy/changeset/24a441449f40/ Log: add XXX with possible solution for a nursery with too many pinned objects diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -775,6 +775,11 @@ # enough space to allocate the object pass else: + # XXX (groggi) possible solutions: + # - gc-incminimark-pinning-arealimit branch (pretty simple, downsides in regard what can be pinned) + # - do raw_malloc (probably best idea, but also quit some code changes needed) + # - make in some way sure that after a minor collection + # enough space can be reserved (probably a bad idea in regard to performance) ll_assert(minor_collection_count == 2, "Seeing minor_collection() at least twice. " "Too many pinned objects?") From noreply at buildbot.pypy.org Fri Sep 5 11:37:58 2014 From: noreply at buildbot.pypy.org (groggi) Date: Fri, 5 Sep 2014 11:37:58 +0200 (CEST) Subject: [pypy-commit] pypy gc-incminimark-pinning: add XXX for weakref pointing to pinned object support Message-ID: <20140905093758.1BE9D1C0793@cobra.cs.uni-duesseldorf.de> Author: Gregor Wegberg Branch: gc-incminimark-pinning Changeset: r73317:f047eb414d3a Date: 2014-09-05 11:33 +0200 http://bitbucket.org/pypy/pypy/changeset/f047eb414d3a/ Log: add XXX for weakref pointing to pinned object support diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -2578,6 +2578,9 @@ # ---------- # Weakrefs + # XXX (groggi): weakref pointing to pinned object not supported. + # XXX (groggi): missing asserts/checks for the missing feature. + # The code relies on the fact that no weakref can be an old object # weakly pointing to a young object. Indeed, weakrefs are immutable # so they cannot point to an object that was created after it. From noreply at buildbot.pypy.org Fri Sep 5 11:37:59 2014 From: noreply at buildbot.pypy.org (groggi) Date: Fri, 5 Sep 2014 11:37:59 +0200 (CEST) Subject: [pypy-commit] pypy gc-incminimark-pinning: Merge heads Message-ID: <20140905093759.4D1E51C0793@cobra.cs.uni-duesseldorf.de> Author: Gregor Wegberg Branch: gc-incminimark-pinning Changeset: r73318:19a470acba00 Date: 2014-09-05 11:37 +0200 http://bitbucket.org/pypy/pypy/changeset/19a470acba00/ Log: Merge heads diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py --- a/rpython/rtyper/lltypesystem/rffi.py +++ b/rpython/rtyper/lltypesystem/rffi.py @@ -763,14 +763,13 @@ """ Either returns a non-moving copy or performs neccessary pointer arithmetic to return a pointer to the characters of a string if the - string is already nonmovable. Must be followed by a + string is already nonmovable or could be pinned. Must be followed by a free_nonmovingbuffer call. First bool returned indicates if 'data' was pinned. Second bool returned - indicates if we did a raw alloc because pinning didn't work. Bot bools + indicates if we did a raw alloc because pinning failed. Both bools should never be true at the same time. """ - # XXX update doc string lldata = llstrtype(data) count = len(data) @@ -801,7 +800,8 @@ @jit.dont_look_inside def free_nonmovingbuffer(data, buf, is_pinned, is_raw): """ - Either free a non-moving buffer or keep the original storage alive. + Keep 'data' alive and unpin it if it was pinned ('is_pinned' is true). + Otherwise free the non-moving copy ('is_raw' is true). """ if is_pinned: rgc.unpin(data) From noreply at buildbot.pypy.org Fri Sep 5 11:51:59 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Fri, 5 Sep 2014 11:51:59 +0200 (CEST) Subject: [pypy-commit] stmgc default: progress Message-ID: <20140905095159.51D1F1C129D@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1354:b58cd671cb54 Date: 2014-09-05 10:18 +0200 http://bitbucket.org/pypy/stmgc/changeset/b58cd671cb54/ Log: progress diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -14,19 +14,41 @@ long i; int my_segnum = STM_SEGMENT->segment_num; + assert(!is_readable_log_page_in(my_segnum, pagenum)); + + /* make readable */ + assert(STM_PSEGMENT->privatization_lock); /* we hold it, nobody + will privatize a page, + necessary? */ + pages_set_protection(my_segnum, pagenum, 1, PROT_READ|PROT_WRITE); + + assert(!is_shared_log_page(pagenum)); + for (i = 0; i < NB_SEGMENTS; i++) { if (i == my_segnum) continue; - if (!is_readable_page(i, first_page)) + if (!is_readable_log_page_in(i, pagenum)) continue; acquire_privatization_lock(i); + assert(is_readable_log_page_in(i, pagenum)); /* still... */ /* copy the content from there to our segment */ - pagecopy(new_page, get_segment_base(from_segnum) + pagenum * 4096UL); + pagecopy((char*)(get_virt_page_of(my_segnum, pagenum) * 4096UL), + (char*)(get_virt_page_of(i, pagenum) * 4096UL)); + + /* get valid state from backup copies of written objs in + the range of this page: */ + acquire_modified_objs_lock(i); + + release_modified_objs_lock(i); release_privatization_lock(i); + + return; } + + abort(); /* didn't find a page to copy from?? */ } void _signal_handler(int sig, siginfo_t *siginfo, void *context) @@ -39,27 +61,17 @@ } /* XXX: should we save 'errno'? */ - /* make PROT_READWRITE again and validate */ + int segnum = get_segment_of_linear_address(addr); OPT_ASSERT(segnum == STM_SEGMENT->segment_num); dprintf(("-> segment: %d\n", segnum)); char *seg_base = STM_SEGMENT->segment_base; uintptr_t pagenum = ((char*)addr - seg_base) / 4096UL; - /* XXX: missing synchronisation: we may change protection, then - another thread changes it back, then we try to privatize that - calls page_copy() and traps */ - /* XXX: mprotect is not reentrant and interruptible by signals, - so it needs additional synchronisation.*/ - pages_set_protection(segnum, pagenum, 1, PROT_READ|PROT_WRITE); + acquire_privatization_lock(segnum); + bring_page_up_to_date(pagenum); + release_privatization_lock(segnum); - bring_page_up_to_date(pagenum); - - /* XXX: ... what can go wrong when we abort from inside - the signal handler? */ - - /* make sure we are up to date in this (and all other) pages */ - stm_validate(NULL); return; } @@ -167,43 +179,40 @@ void _privatize_and_protect_other_segments(object_t *obj) { - assert(STM_PSEGMENT->privatization_lock); /* we hold it */ - assert(obj_size < 4096); /* too lazy right now (part of the code is ready) */ +#ifndef NDEBUG + long l; + for (l = 0; l < NB_SEGMENTS; l++) { + assert(get_priv_segment(l)->privatization_lock); + } +#endif uintptr_t first_page = ((uintptr_t)obj) / 4096UL; char *realobj; size_t obj_size; - uintptr_t i, end_page; + uintptr_t i; int my_segnum = STM_SEGMENT->segment_num; realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj); obj_size = stmcb_size_rounded_up((struct object_s *)realobj); - end_page = (((uintptr_t)obj) + obj_size - 1) / 4096UL; + assert(obj_size < 4096); /* XXX */ /* privatize pages: */ - /* get the last page containing data from the object */ - end_page = (((uintptr_t)obj) + obj_size - 1) / 4096UL; - for (i = first_page; i <= end_page; i++) { - if (is_private_page(my_segnum, i)) - continue; - page_privatize(i); - bring_page_up_to_date(i); + assert(is_shared_log_page(first_page)); + /* XXX: change this logic: + right now, privatization means private in seg0 and private + in my_segnum */ + for (i = 0; i < NB_SEGMENTS; i++) { + assert(!is_private_log_page_in(i, first_page)); + + if (i != my_segnum && i != 0) + pages_set_protection(i, first_page, 1, PROT_NONE); } - /* then PROT_NONE everyone else that doesn't have a private - page already */ - for (i = 0; i < NB_SEGMENTS; i++) { - if (i == my_segnum) - continue; - if (!is_readable_page(i, first_page) || is_private_page(i, first_page)) - continue; - - acquire_privatization_lock(i); - pages_set_protection(i, first_page, end_page - first_page + 1, - PROT_NONE); - release_privatization_lock(i); - - dprintf(("prot %lu, len=%lu in seg %lu\n", first_page, (end_page - first_page + 1), i)); + /* remap pages for my_segnum and copy the contents */ + if (i != 0) { + page_privatize(first_page); + pagecopy((char*)(get_virt_page_of(my_segnum, first_page) * 4096UL), + (char*)(get_virt_page_of(0, first_page) * 4096UL)); } } @@ -230,21 +239,28 @@ assert(obj_size < 4096); /* too lazy right now (part of the code is ready) */ - retry: + if (is_shared_log_page(first_page)) { + /* acquire all privatization locks, make private and + read protect others */ + long i; + for (i = 0; i < NB_SEGMENTS; i++) { + acquire_privatization_lock(i); + } + if (is_shared_log_page(first_page)) + _privatize_and_protect_other_segments(obj); + for (i = NB_SEGMENTS-1; i >= 0; i--) { + release_privatization_lock(i); + } + } + /* page not shared anymore. but we still may have + only a read protected page ourselves: */ + acquire_privatization_lock(my_segnum); - if (!is_readable_page(my_segnum, first_page)) { - release_privatization_lock(my_segnum); + if (!is_readable_log_page_in(my_segnum, first_page)) + bring_page_up_to_date(first_page); + /* page is not PROT_NONE for us */ - bring_page_up_to_date(first_page); - - spin_loop(); - goto retry; - } - /* page is not PROT_NONE for us, we can PROT_NONE all others */ - _privatize_and_protect_other_segments(obj); - - /* remove the WRITE_BARRIER flag (could be done later, but I - think we find more bugs this way) */ + /* remove the WRITE_BARRIER flag */ obj->stm_flags &= ~GCFLAG_WRITE_BARRIER; /* done fiddling with protection and privatization */ diff --git a/c8/stm/pages.h b/c8/stm/pages.h --- a/c8/stm/pages.h +++ b/c8/stm/pages.h @@ -45,14 +45,28 @@ static void pages_set_protection(int segnum, uintptr_t pagenum, uintptr_t count, int prot); -static inline bool is_private_page(long segnum, uintptr_t pagenum) + +static inline uintptr_t get_virt_page_of(long segnum, uintptr_t pagenum) +{ + /* logical page -> virtual page */ + return (uintptr_t)get_segment_base(segnum) / 4096UL + pagenum; +} + +static inline bool is_shared_log_page(uintptr_t pagenum) +{ + assert(pagenum >= PAGE_FLAG_START); + return pages_privatized[pagenum - PAGE_FLAG_START].by_segment == 0; +} + + +static inline bool is_private_log_page_in(long segnum, uintptr_t pagenum) { assert(pagenum >= PAGE_FLAG_START); uint64_t bitmask = 1UL << segnum; return (pages_privatized[pagenum - PAGE_FLAG_START].by_segment & bitmask); } -static inline bool is_readable_page(long segnum, uintptr_t pagenum) +static inline bool is_readable_log_page_in(long segnum, uintptr_t pagenum) { assert(pagenum >= PAGE_FLAG_START); uint64_t bitmask = 1UL << segnum; From noreply at buildbot.pypy.org Fri Sep 5 11:52:00 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Fri, 5 Sep 2014 11:52:00 +0200 (CEST) Subject: [pypy-commit] stmgc default: more progress Message-ID: <20140905095200.784F41C129D@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1355:3107d8e61582 Date: 2014-09-05 11:07 +0200 http://bitbucket.org/pypy/stmgc/changeset/3107d8e61582/ Log: more progress diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -15,12 +15,14 @@ int my_segnum = STM_SEGMENT->segment_num; assert(!is_readable_log_page_in(my_segnum, pagenum)); + assert(!is_private_log_page_in(my_segnum, pagenum)); /* make readable */ assert(STM_PSEGMENT->privatization_lock); /* we hold it, nobody will privatize a page, necessary? */ pages_set_protection(my_segnum, pagenum, 1, PROT_READ|PROT_WRITE); + page_privatize(pagenum); assert(!is_shared_log_page(pagenum)); @@ -34,12 +36,30 @@ assert(is_readable_log_page_in(i, pagenum)); /* still... */ /* copy the content from there to our segment */ + dprintf(("pagecopy pagenum:%lu, src: %lu, dst:%lu\n", pagenum, i, my_segnum)); pagecopy((char*)(get_virt_page_of(my_segnum, pagenum) * 4096UL), (char*)(get_virt_page_of(i, pagenum) * 4096UL)); /* get valid state from backup copies of written objs in the range of this page: */ acquire_modified_objs_lock(i); + struct tree_s *tree = get_priv_segment(i)->modified_old_objects; + wlog_t *item; + TREE_LOOP_FORWARD(tree, item); + if (item->addr >= pagenum * 4096UL && item->addr < (pagenum + 1) * 4096UL) { + object_t *obj = (object_t*)item->addr; + struct object_s* bk_obj = (struct object_s *)item->val; + size_t obj_size; + + obj_size = stmcb_size_rounded_up(bk_obj); + assert(obj_size < 4096); /* XXX */ + + assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); /* not written here */ + memcpy(REAL_ADDRESS(STM_SEGMENT->segment_base, obj), + bk_obj, obj_size); + assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); /* still not written */ + } + TREE_LOOP_END; release_modified_objs_lock(i); @@ -206,14 +226,21 @@ if (i != my_segnum && i != 0) pages_set_protection(i, first_page, 1, PROT_NONE); + else /* both, seg0 and my_segnum: */ + pages_set_protection(i, first_page, 1, PROT_READ|PROT_WRITE); } /* remap pages for my_segnum and copy the contents */ - if (i != 0) { + set_page_private_in(0, first_page); + /* seg0 already up-to-date */ + if (my_segnum != 0) { page_privatize(first_page); pagecopy((char*)(get_virt_page_of(my_segnum, first_page) * 4096UL), (char*)(get_virt_page_of(0, first_page) * 4096UL)); } + + assert(is_private_log_page_in(my_segnum, first_page)); + assert(is_readable_log_page_in(my_segnum, first_page)); } void _stm_write_slowpath(object_t *obj) diff --git a/c8/stm/pages.c b/c8/stm/pages.c --- a/c8/stm/pages.c +++ b/c8/stm/pages.c @@ -77,15 +77,8 @@ volatile struct page_shared_s *ps2 = (volatile struct page_shared_s *) &pages_privatized[pagenum + amount - PAGE_FLAG_START]; - if (i == 0) { - /* readable & private */ - ps->by_segment |= bitmask; - ps2->by_segment |= bitmask; - } else { - /* not readable (ensured in setup.c), not private */ - ps->by_segment &= ~bitmask; - ps2->by_segment &= ~bitmask; - } + ps->by_segment |= bitmask; /* readable */ + ps2->by_segment = 0; /* not private */ } } } @@ -104,6 +97,8 @@ return; } + dprintf(("page_privatize(%lu) in seg:%d\n", pagenum, STM_SEGMENT->segment_num)); + /* add this thread's 'pages_privatized' bit */ ps->by_segment |= bitmask; @@ -122,23 +117,23 @@ /* we hopefully hold the privatization lock: */ assert(get_priv_segment(segnum)->privatization_lock); - char *addr = get_segment_base(segnum) + pagenum * 4096UL; + char *addr = get_virt_page_of(segnum, pagenum); mprotect(addr, count * 4096UL, prot); - long i; - for (i = 0; i < NB_SEGMENTS; i++) { - uint64_t bitmask = 1UL << i; - uintptr_t amount = count; - while (amount-->0) { - volatile struct page_shared_s *ps = (volatile struct page_shared_s *) - &pages_readable[pagenum + amount - PAGE_FLAG_START]; - if (prot == PROT_NONE) { - /* not readable */ - ps->by_segment &= ~bitmask; - } else { - assert(prot == (PROT_READ|PROT_WRITE)); - ps->by_segment |= bitmask; - } + dprintf(("pages_set_protection(%d, %lu, %lu, %d)\n", + segnum, pagenum, count, prot)); + + uint64_t bitmask = 1UL << segnum; + uintptr_t amount = count; + while (amount-->0) { + volatile struct page_shared_s *ps = (volatile struct page_shared_s *) + &pages_readable[pagenum + amount - PAGE_FLAG_START]; + if (prot == PROT_NONE) { + /* not readable */ + ps->by_segment &= ~bitmask; + } else { + assert(prot == (PROT_READ|PROT_WRITE)); + ps->by_segment |= bitmask; } } } diff --git a/c8/stm/pages.h b/c8/stm/pages.h --- a/c8/stm/pages.h +++ b/c8/stm/pages.h @@ -58,6 +58,14 @@ return pages_privatized[pagenum - PAGE_FLAG_START].by_segment == 0; } +static inline void set_page_private_in(long segnum, uintptr_t pagenum) +{ + uint64_t bitmask = 1UL << segnum; + volatile struct page_shared_s *ps = (volatile struct page_shared_s *) + &pages_privatized[pagenum - PAGE_FLAG_START]; + assert(!(ps->by_segment & bitmask)); + ps->by_segment |= bitmask; +} static inline bool is_private_log_page_in(long segnum, uintptr_t pagenum) { diff --git a/c8/stm/setup.c b/c8/stm/setup.c --- a/c8/stm/setup.c +++ b/c8/stm/setup.c @@ -68,16 +68,6 @@ mprotect(segment_base + 8192, (FIRST_READMARKER_PAGE - 2) * 4096UL, PROT_NONE); - - if (i != 0) { - /* let's give all pages to segment 0 at first, all others - need to trap and look for the backup copy */ - mprotect(segment_base + END_NURSERY_PAGE * 4096, - (NB_PAGES - END_NURSERY_PAGE) * 4096, - PROT_NONE); - /* pages_initialize_shared() makes sure pages_readable - is initialized correctly */ - } } } From noreply at buildbot.pypy.org Fri Sep 5 11:52:01 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Fri, 5 Sep 2014 11:52:01 +0200 (CEST) Subject: [pypy-commit] stmgc default: pass first isolation tests Message-ID: <20140905095201.807B11C129D@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1356:45cc9257f06e Date: 2014-09-05 11:53 +0200 http://bitbucket.org/pypy/stmgc/changeset/45cc9257f06e/ Log: pass first isolation tests diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -36,7 +36,7 @@ assert(is_readable_log_page_in(i, pagenum)); /* still... */ /* copy the content from there to our segment */ - dprintf(("pagecopy pagenum:%lu, src: %lu, dst:%lu\n", pagenum, i, my_segnum)); + dprintf(("pagecopy pagenum:%lu, src: %lu, dst:%d\n", pagenum, i, my_segnum)); pagecopy((char*)(get_virt_page_of(my_segnum, pagenum) * 4096UL), (char*)(get_virt_page_of(i, pagenum) * 4096UL)); @@ -54,10 +54,9 @@ obj_size = stmcb_size_rounded_up(bk_obj); assert(obj_size < 4096); /* XXX */ - assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); /* not written here */ memcpy(REAL_ADDRESS(STM_SEGMENT->segment_base, obj), bk_obj, obj_size); - assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); /* still not written */ + assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); /* not written */ } TREE_LOOP_END; @@ -75,7 +74,8 @@ { char *addr = siginfo->si_addr; dprintf(("si_addr: %p\n", addr)); - if (addr == NULL) { /* actual segfault */ + if (addr == NULL || addr < stm_object_pages || addr > stm_object_pages+TOTAL_MEMORY) { + /* actual segfault */ /* send to GDB */ kill(getpid(), SIGINT); } @@ -350,7 +350,7 @@ reset_transaction_read_version(); } - assert(tree_count(STM_PSEGMENT->modified_old_objects) == 0); + assert(tree_is_cleared(STM_PSEGMENT->modified_old_objects)); assert(list_is_empty(STM_PSEGMENT->objects_pointing_to_nursery)); check_nursery_at_transaction_start(); } @@ -387,9 +387,21 @@ minor_collection(1); - struct stm_commit_log_entry_s* entry = _validate_and_add_to_commit_log(); + _validate_and_add_to_commit_log(); + acquire_modified_objs_lock(STM_SEGMENT->segment_num); - /* XXX:discard backup copies */ + + struct tree_s *tree = STM_PSEGMENT->modified_old_objects; + wlog_t *item; + TREE_LOOP_FORWARD(tree, item); + object_t *obj = (object_t*)item->addr; + struct object_s* bk_obj = (struct object_s *)item->val; + free(bk_obj); + obj->stm_flags |= GCFLAG_WRITE_BARRIER; + TREE_LOOP_END; + + tree_clear(tree); + release_modified_objs_lock(STM_SEGMENT->segment_num); s_mutex_lock(); diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c --- a/c8/stm/gcpage.c +++ b/c8/stm/gcpage.c @@ -51,7 +51,9 @@ object_t *o = (object_t *)(p - stm_object_pages); o->stm_flags = GCFLAG_WRITE_BARRIER; - dprintf(("allocate_old(%lu): %p, seg=%d\n", size_rounded_up, p, - get_segment_of_linear_address(p))); + dprintf(("allocate_old(%lu): %p, seg=%d, page=%lu\n", + size_rounded_up, p, + get_segment_of_linear_address(p), + (p - STM_SEGMENT->segment_base) / 4096UL)); return o; } diff --git a/c8/stm/list.c b/c8/stm/list.c --- a/c8/stm/list.c +++ b/c8/stm/list.c @@ -45,6 +45,7 @@ tree->raw_current = tree->raw_start; tree->count = 0; } + assert(tree->count == 0); } static struct tree_s *tree_create(void) @@ -99,6 +100,7 @@ newtree.raw_start = newitems; newtree.raw_current = newitems; newtree.raw_end = newitems + newalloc; + newtree.count = 0; _tree_clear_node(&newtree.toplevel); TREE_LOOP_FORWARD(tree, item) { @@ -146,7 +148,7 @@ /* reuse the deleted entry and that's it */ wlog1->addr = addr; wlog1->val = val; - tree->count++; + (tree->count)++; return; } /* the key must not already be present */ @@ -168,7 +170,7 @@ wlog->addr = addr; wlog->val = val; *(char **)p = (char *)wlog; - tree->count++; + (tree->count)++; } static bool tree_delete_item(struct tree_s *tree, uintptr_t addr) diff --git a/c8/stm/list.h b/c8/stm/list.h --- a/c8/stm/list.h +++ b/c8/stm/list.h @@ -124,8 +124,8 @@ } wlog_node_t; struct tree_s { + uintptr_t count; char *raw_start, *raw_current, *raw_end; - uintptr_t count; wlog_node_t toplevel; }; @@ -135,10 +135,12 @@ //static inline void tree_delete_not_used_any_more(struct tree_s *tree)... static inline bool tree_is_cleared(struct tree_s *tree) { + assert((tree->raw_current == tree->raw_start) == (tree->count == 0)); return tree->raw_current == tree->raw_start; } static inline uintptr_t tree_count(struct tree_s *tree) { + assert(tree->count >= 0); return tree->count; } diff --git a/c8/stm/misc.c b/c8/stm/misc.c --- a/c8/stm/misc.c +++ b/c8/stm/misc.c @@ -46,8 +46,8 @@ #ifdef STM_TESTS long _stm_count_modified_old_objects(void) { - if (STM_PSEGMENT->modified_old_objects == NULL) - return -1; + assert(STM_PSEGMENT->modified_old_objects); + assert(tree_count(STM_PSEGMENT->modified_old_objects) < 10000); return tree_count(STM_PSEGMENT->modified_old_objects); } diff --git a/c8/stm/pages.c b/c8/stm/pages.c --- a/c8/stm/pages.c +++ b/c8/stm/pages.c @@ -117,11 +117,16 @@ /* we hopefully hold the privatization lock: */ assert(get_priv_segment(segnum)->privatization_lock); - char *addr = get_virt_page_of(segnum, pagenum); + if ((prot == PROT_NONE && !is_readable_log_page_in(segnum, pagenum)) + || (prot == (PROT_READ|PROT_WRITE) && is_readable_log_page_in(segnum, pagenum))) + return; + + char *addr = (char*)(get_virt_page_of(segnum, pagenum) * 4096UL); mprotect(addr, count * 4096UL, prot); - dprintf(("pages_set_protection(%d, %lu, %lu, %d)\n", - segnum, pagenum, count, prot)); + dprintf(("pages_set_protection(%d, %lu, %lu, %d), virtpage:%lu\n", + segnum, pagenum, count, prot, + get_virt_page_of(segnum, pagenum))); uint64_t bitmask = 1UL << segnum; uintptr_t amount = count; diff --git a/c8/test/test_basic.py b/c8/test/test_basic.py --- a/c8/test/test_basic.py +++ b/c8/test/test_basic.py @@ -81,7 +81,6 @@ def test_read_write_1(self): lp1 = stm_allocate_old(16) stm_get_real_address(lp1)[HDR] = 'a' #setchar - lib._push_obj_to_other_segments(lp1) # self.start_transaction() self.commit_transaction() From noreply at buildbot.pypy.org Fri Sep 5 11:55:41 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Fri, 5 Sep 2014 11:55:41 +0200 (CEST) Subject: [pypy-commit] stmgc default: pass a few more by validating on start_transaction() Message-ID: <20140905095541.BFD5D1C129D@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1357:6dd4a007769a Date: 2014-09-05 11:56 +0200 http://bitbucket.org/pypy/stmgc/changeset/6dd4a007769a/ Log: pass a few more by validating on start_transaction() diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -353,6 +353,7 @@ assert(tree_is_cleared(STM_PSEGMENT->modified_old_objects)); assert(list_is_empty(STM_PSEGMENT->objects_pointing_to_nursery)); check_nursery_at_transaction_start(); + stm_validate(NULL); } long stm_start_transaction(stm_thread_local_t *tl) From noreply at buildbot.pypy.org Fri Sep 5 12:51:49 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Fri, 5 Sep 2014 12:51:49 +0200 (CEST) Subject: [pypy-commit] stmgc default: actually update to current state in stm_validate Message-ID: <20140905105149.564151C129D@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1358:afd9564842de Date: 2014-09-05 12:53 +0200 http://bitbucket.org/pypy/stmgc/changeset/afd9564842de/ Log: actually update to current state in stm_validate diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -118,10 +118,37 @@ static void _update_obj_from(int from_seg, object_t *obj) { - /* check if its pages are private, only then we need - to update them. If they are also still read-protected, - we may trigger the signal handler. This would cause - it to also enter stm_validate()..... */ + char *realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj); + size_t obj_size; + + uintptr_t pagenum = (uintptr_t)obj / 4096UL; + if (!is_private_log_page_in(STM_SEGMENT->segment_num, pagenum)) + return; /* only do in sighandler */ + + /* should be readable & private (XXX: maybe not after major GCs) */ + assert(is_readable_log_page_in(from_seg, pagenum)); + assert(is_private_log_page_in(from_seg, pagenum)); + + acquire_modified_objs_lock(from_seg); + + struct tree_s *tree = get_priv_segment(from_seg)->modified_old_objects; + wlog_t *item; + TREE_FIND(tree, (uintptr_t)obj, item, goto not_found); + + obj_size = stmcb_size_rounded_up((struct object_s*)item->val); + memcpy(realobj, (char*)item->val, obj_size); + assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); + release_modified_objs_lock(from_seg); + return; + + not_found: + /* copy from page */ + obj_size = stmcb_size_rounded_up(REAL_ADDRESS(get_segment_base(from_seg), obj)); + memcpy(realobj, + REAL_ADDRESS(get_segment_base(from_seg), obj), + obj_size); + obj->stm_flags |= GCFLAG_WRITE_BARRIER; /* may already be gone */ + release_modified_objs_lock(from_seg); } void stm_validate(void *free_if_abort) @@ -129,6 +156,7 @@ volatile struct stm_commit_log_entry_s *cl = (volatile struct stm_commit_log_entry_s *) STM_PSEGMENT->last_commit_log_entry; + acquire_privatization_lock(STM_SEGMENT->segment_num); bool needs_abort = false; /* Don't check 'cl'. This entry is already checked */ while ((cl = cl->next)) { @@ -136,10 +164,11 @@ OPT_ASSERT(cl->segment_num >= 0 && cl->segment_num < NB_SEGMENTS); object_t *obj; + while ((obj = cl->written[i])) { _update_obj_from(cl->segment_num, obj); - if (!needs_abort &&_stm_was_read(obj)) { + if (!needs_abort && _stm_was_read(obj)) { needs_abort = true; } @@ -150,6 +179,7 @@ STM_PSEGMENT->last_commit_log_entry = (struct stm_commit_log_entry_s *)cl; } + release_privatization_lock(STM_SEGMENT->segment_num); if (needs_abort) { free(free_if_abort); stm_abort_transaction(); From noreply at buildbot.pypy.org Fri Sep 5 13:01:36 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Fri, 5 Sep 2014 13:01:36 +0200 (CEST) Subject: [pypy-commit] stmgc default: fix Message-ID: <20140905110136.BEFA11C129D@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1359:0308c7a328d1 Date: 2014-09-05 13:02 +0200 http://bitbucket.org/pypy/stmgc/changeset/0308c7a328d1/ Log: fix diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -108,11 +108,10 @@ size_t i = 0; fprintf(stderr, "elem (%p, %d)\n", cl->next, cl->segment_num); object_t *obj; - do { - obj = cl->written[i]; + while ((obj = cl->written[i])) { fprintf(stderr, "-> %p\n", obj); i++; - } while ((obj = cl->written[i])); + }; } } @@ -122,8 +121,11 @@ size_t obj_size; uintptr_t pagenum = (uintptr_t)obj / 4096UL; - if (!is_private_log_page_in(STM_SEGMENT->segment_num, pagenum)) + if (!is_private_log_page_in(STM_SEGMENT->segment_num, pagenum)) { + assert(!is_shared_log_page(pagenum)); + assert(!is_readable_log_page_in(STM_SEGMENT->segment_num, pagenum)); return; /* only do in sighandler */ + } /* should be readable & private (XXX: maybe not after major GCs) */ assert(is_readable_log_page_in(from_seg, pagenum)); diff --git a/c8/stm/setup.c b/c8/stm/setup.c --- a/c8/stm/setup.c +++ b/c8/stm/setup.c @@ -159,6 +159,8 @@ munmap(stm_object_pages, TOTAL_MEMORY); stm_object_pages = NULL; + commit_log_root.next = NULL; /* xxx:free them */ + commit_log_root.segment_num = -1; close_fd_mmap(stm_object_pages_fd); teardown_sync(); From noreply at buildbot.pypy.org Fri Sep 5 15:16:52 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Fri, 5 Sep 2014 15:16:52 +0200 (CEST) Subject: [pypy-commit] stmgc default: fix some tests Message-ID: <20140905131652.487551D3956@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1360:6bb0732c0c48 Date: 2014-09-05 15:18 +0200 http://bitbucket.org/pypy/stmgc/changeset/6bb0732c0c48/ Log: fix some tests diff --git a/c8/test/test_basic.py b/c8/test/test_basic.py --- a/c8/test/test_basic.py +++ b/c8/test/test_basic.py @@ -317,8 +317,9 @@ self.start_transaction() stm_write(lp1) stm_set_char(lp1, 'b') - # transaction from thread 1 is newer than from thread 0 - py.test.raises(Conflict, self.commit_transaction) + self.commit_transaction() + + py.test.raises(Conflict, self.switch, 0) # fails validation def test_resolve_write_read_conflict(self): self.start_transaction() @@ -352,7 +353,11 @@ # self.switch(1) self.start_transaction() - py.test.raises(Conflict, stm_write, lp1) # write-write conflict + stm_write(lp1) + self.switch(0) + self.commit_transaction() + + py.test.raises(Conflict, self.switch, 1) # write-write conflict def test_abort_cleanup(self): self.start_transaction() From noreply at buildbot.pypy.org Fri Sep 5 15:24:47 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Fri, 5 Sep 2014 15:24:47 +0200 (CEST) Subject: [pypy-commit] stmgc default: reset modified objs on abort Message-ID: <20140905132447.EB8161D3957@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1361:758cb4ddefea Date: 2014-09-05 15:26 +0200 http://bitbucket.org/pypy/stmgc/changeset/758cb4ddefea/ Log: reset modified objs on abort diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -145,7 +145,8 @@ not_found: /* copy from page */ - obj_size = stmcb_size_rounded_up(REAL_ADDRESS(get_segment_base(from_seg), obj)); + obj_size = stmcb_size_rounded_up( + (struct object_s*)REAL_ADDRESS(get_segment_base(from_seg), obj)); memcpy(realobj, REAL_ADDRESS(get_segment_base(from_seg), obj), obj_size); @@ -450,6 +451,39 @@ s_mutex_unlock(); } +void reset_modified_from_backup_copies(int segment_num) +{ +#pragma push_macro("STM_PSEGMENT") +#pragma push_macro("STM_SEGMENT") +#undef STM_PSEGMENT +#undef STM_SEGMENT + acquire_modified_objs_lock(segment_num); + + struct stm_priv_segment_info_s *pseg = get_priv_segment(segment_num); + struct tree_s *tree = pseg->modified_old_objects; + wlog_t *item; + TREE_LOOP_FORWARD(tree, item); { + object_t *obj = (object_t*)item->addr; + struct object_s* bk_obj = (struct object_s *)item->val; + size_t obj_size; + + obj_size = stmcb_size_rounded_up(bk_obj); + assert(obj_size < 4096); /* XXX */ + + memcpy(REAL_ADDRESS(pseg->pub.segment_base, obj), + bk_obj, obj_size); + assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); /* not written */ + + free(bk_obj); + } TREE_LOOP_END; + + tree_clear(tree); + + release_modified_objs_lock(segment_num); + +#pragma pop_macro("STM_SEGMENT") +#pragma pop_macro("STM_PSEGMENT") +} static void abort_data_structures_from_segment_num(int segment_num) { @@ -461,7 +495,7 @@ throw_away_nursery(pseg); - /* XXX: reset_modified_from_other_segments(segment_num); */ + reset_modified_from_backup_copies(segment_num); stm_thread_local_t *tl = pseg->pub.running_thread; #ifdef STM_NO_AUTOMATIC_SETJMP From noreply at buildbot.pypy.org Fri Sep 5 15:47:03 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Fri, 5 Sep 2014 15:47:03 +0200 (CEST) Subject: [pypy-commit] stmgc default: some cleanup and comments Message-ID: <20140905134703.30F551D387E@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1362:248d91849433 Date: 2014-09-05 15:48 +0200 http://bitbucket.org/pypy/stmgc/changeset/248d91849433/ Log: some cleanup and comments diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -9,23 +9,29 @@ static void bring_page_up_to_date(uintptr_t pagenum) { - /* pagecopy from somewhere readable, then update + /* Assuming pagenum is not yet private in this segment and + currently read-protected: + + pagecopy from somewhere readable, then update all written objs from that segment */ + + assert(!is_shared_log_page(pagenum)); + + /* XXX: this may not even be necessary: */ + assert(STM_PSEGMENT->privatization_lock); + long i; int my_segnum = STM_SEGMENT->segment_num; assert(!is_readable_log_page_in(my_segnum, pagenum)); assert(!is_private_log_page_in(my_segnum, pagenum)); - /* make readable */ - assert(STM_PSEGMENT->privatization_lock); /* we hold it, nobody - will privatize a page, - necessary? */ + /* privatize and make readable */ pages_set_protection(my_segnum, pagenum, 1, PROT_READ|PROT_WRITE); page_privatize(pagenum); - assert(!is_shared_log_page(pagenum)); - + /* look for a segment where the page is readable (and + therefore private): */ for (i = 0; i < NB_SEGMENTS; i++) { if (i == my_segnum) continue; @@ -33,7 +39,9 @@ continue; acquire_privatization_lock(i); - assert(is_readable_log_page_in(i, pagenum)); /* still... */ + /* nobody should be able to change this because we have + our own privatization lock: */ + assert(is_readable_log_page_in(i, pagenum)); /* copy the content from there to our segment */ dprintf(("pagecopy pagenum:%lu, src: %lu, dst:%d\n", pagenum, i, my_segnum)); @@ -45,23 +53,25 @@ acquire_modified_objs_lock(i); struct tree_s *tree = get_priv_segment(i)->modified_old_objects; wlog_t *item; - TREE_LOOP_FORWARD(tree, item); - if (item->addr >= pagenum * 4096UL && item->addr < (pagenum + 1) * 4096UL) { - object_t *obj = (object_t*)item->addr; - struct object_s* bk_obj = (struct object_s *)item->val; - size_t obj_size; + TREE_LOOP_FORWARD(tree, item); { + if (item->addr >= pagenum * 4096UL && item->addr < (pagenum + 1) * 4096UL) { + /* obj is in range. XXX: no page overlapping objs allowed yet */ - obj_size = stmcb_size_rounded_up(bk_obj); - assert(obj_size < 4096); /* XXX */ + object_t *obj = (object_t*)item->addr; + struct object_s* bk_obj = (struct object_s *)item->val; + size_t obj_size; - memcpy(REAL_ADDRESS(STM_SEGMENT->segment_base, obj), - bk_obj, obj_size); - assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); /* not written */ - } - TREE_LOOP_END; + obj_size = stmcb_size_rounded_up(bk_obj); + assert(obj_size < 4096); /* XXX */ + + memcpy(REAL_ADDRESS(STM_SEGMENT->segment_base, obj), + bk_obj, obj_size); + + assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); /* bk_obj never written */ + } + } TREE_LOOP_END; release_modified_objs_lock(i); - release_privatization_lock(i); return; @@ -76,7 +86,7 @@ dprintf(("si_addr: %p\n", addr)); if (addr == NULL || addr < stm_object_pages || addr > stm_object_pages+TOTAL_MEMORY) { /* actual segfault */ - /* send to GDB */ + /* send to GDB (XXX) */ kill(getpid(), SIGINT); } /* XXX: should we save 'errno'? */ @@ -117,20 +127,26 @@ static void _update_obj_from(int from_seg, object_t *obj) { + /* during validation this looks up the obj in the + from_seg (backup or normal) and copies the version + over the current segment's one */ char *realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj); size_t obj_size; uintptr_t pagenum = (uintptr_t)obj / 4096UL; if (!is_private_log_page_in(STM_SEGMENT->segment_num, pagenum)) { + /* done in signal handler on first access anyway */ assert(!is_shared_log_page(pagenum)); assert(!is_readable_log_page_in(STM_SEGMENT->segment_num, pagenum)); - return; /* only do in sighandler */ + return; } /* should be readable & private (XXX: maybe not after major GCs) */ assert(is_readable_log_page_in(from_seg, pagenum)); assert(is_private_log_page_in(from_seg, pagenum)); + /* look the obj up in the other segment's modified_old_objects to + get its backup copy: */ acquire_modified_objs_lock(from_seg); struct tree_s *tree = get_priv_segment(from_seg)->modified_old_objects; @@ -144,7 +160,7 @@ return; not_found: - /* copy from page */ + /* copy from page directly (obj is unmodified) */ obj_size = stmcb_size_rounded_up( (struct object_s*)REAL_ADDRESS(get_segment_base(from_seg), obj)); memcpy(realobj, @@ -156,6 +172,10 @@ void stm_validate(void *free_if_abort) { + /* go from last known entry in commit log to the + most current one and apply all changes done + by other transactions. Abort if we read one of + the committed objs. */ volatile struct stm_commit_log_entry_s *cl = (volatile struct stm_commit_log_entry_s *) STM_PSEGMENT->last_commit_log_entry; @@ -163,11 +183,10 @@ bool needs_abort = false; /* Don't check 'cl'. This entry is already checked */ while ((cl = cl->next)) { - size_t i = 0; OPT_ASSERT(cl->segment_num >= 0 && cl->segment_num < NB_SEGMENTS); object_t *obj; - + size_t i = 0; while ((obj = cl->written[i])) { _update_obj_from(cl->segment_num, obj); @@ -181,8 +200,8 @@ /* last fully validated entry */ STM_PSEGMENT->last_commit_log_entry = (struct stm_commit_log_entry_s *)cl; } + release_privatization_lock(STM_SEGMENT->segment_num); - release_privatization_lock(STM_SEGMENT->segment_num); if (needs_abort) { free(free_if_abort); stm_abort_transaction(); @@ -191,6 +210,10 @@ static struct stm_commit_log_entry_s *_create_commit_log_entry() { + /* puts all modified_old_objects in a new commit log entry */ + + // we don't need the privatization lock, as we are only + // reading from modified_old_objs and nobody but us can change it struct tree_s *tree = STM_PSEGMENT->modified_old_objects; size_t count = tree_count(tree); size_t byte_len = sizeof(struct stm_commit_log_entry_s) + (count + 1) * sizeof(object_t*); @@ -201,10 +224,10 @@ int i = 0; wlog_t *item; - TREE_LOOP_FORWARD(tree, item); - result->written[i] = (object_t*)item->addr; - i++; - TREE_LOOP_END; + TREE_LOOP_FORWARD(tree, item); { + result->written[i] = (object_t*)item->addr; + i++; + } TREE_LOOP_END; OPT_ASSERT(count == i); result->written[count] = NULL; @@ -212,26 +235,26 @@ return result; } -static struct stm_commit_log_entry_s *_validate_and_add_to_commit_log() +static void _validate_and_add_to_commit_log() { struct stm_commit_log_entry_s *new; volatile struct stm_commit_log_entry_s **to; new = _create_commit_log_entry(); - fprintf(stderr,"%p\n", new); do { stm_validate(new); + /* try attaching to commit log: */ to = &(STM_PSEGMENT->last_commit_log_entry->next); } while (!__sync_bool_compare_and_swap(to, NULL, new)); - - return new; } /* ############# STM ############# */ void _privatize_and_protect_other_segments(object_t *obj) { + /* privatize pages of obj for our segment iff previously + the pages were fully shared. */ #ifndef NDEBUG long l; for (l = 0; l < NB_SEGMENTS; l++) { @@ -249,7 +272,8 @@ obj_size = stmcb_size_rounded_up((struct object_s *)realobj); assert(obj_size < 4096); /* XXX */ - /* privatize pages: */ + /* privatize pages by read-protecting them in other + segments and giving us a private copy: */ assert(is_shared_log_page(first_page)); /* XXX: change this logic: right now, privatization means private in seg0 and private @@ -423,21 +447,23 @@ _validate_and_add_to_commit_log(); + /* clear WRITE_BARRIER flags, free all backup copies, + and clear the tree: */ acquire_modified_objs_lock(STM_SEGMENT->segment_num); struct tree_s *tree = STM_PSEGMENT->modified_old_objects; wlog_t *item; - TREE_LOOP_FORWARD(tree, item); - object_t *obj = (object_t*)item->addr; - struct object_s* bk_obj = (struct object_s *)item->val; - free(bk_obj); - obj->stm_flags |= GCFLAG_WRITE_BARRIER; - TREE_LOOP_END; - + TREE_LOOP_FORWARD(tree, item); { + object_t *obj = (object_t*)item->addr; + struct object_s* bk_obj = (struct object_s *)item->val; + free(bk_obj); + obj->stm_flags |= GCFLAG_WRITE_BARRIER; + } TREE_LOOP_END; tree_clear(tree); release_modified_objs_lock(STM_SEGMENT->segment_num); + s_mutex_lock(); assert(STM_SEGMENT->nursery_end == NURSERY_END); From noreply at buildbot.pypy.org Fri Sep 5 16:11:38 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Fri, 5 Sep 2014 16:11:38 +0200 (CEST) Subject: [pypy-commit] stmgc default: add some very simple commit-log testing Message-ID: <20140905141138.CA1F11C0793@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1363:a5c6dc8a08d5 Date: 2014-09-05 16:12 +0200 http://bitbucket.org/pypy/stmgc/changeset/a5c6dc8a08d5/ Log: add some very simple commit-log testing diff --git a/c8/stm/misc.c b/c8/stm/misc.c --- a/c8/stm/misc.c +++ b/c8/stm/misc.c @@ -69,4 +69,25 @@ return (object_t *)list_item( STM_PSEGMENT->objects_pointing_to_nursery, index); } + +static volatile struct stm_commit_log_entry_s *_last_cl_entry; +static long _last_cl_entry_index; +void _stm_start_enum_last_cl_entry() +{ + _last_cl_entry = &commit_log_root; + volatile struct stm_commit_log_entry_s *cl = (volatile struct stm_commit_log_entry_s *) + &commit_log_root; + + while ((cl = cl->next)) { + _last_cl_entry = cl; + } + _last_cl_entry_index = 0; +} + +object_t *_stm_next_last_cl_entry() +{ + if (_last_cl_entry != &commit_log_root) + return _last_cl_entry->written[_last_cl_entry_index++]; + return NULL; +} #endif diff --git a/c8/stmgc.h b/c8/stmgc.h --- a/c8/stmgc.h +++ b/c8/stmgc.h @@ -90,6 +90,8 @@ long _stm_count_objects_pointing_to_nursery(void); object_t *_stm_enum_modified_old_objects(long index); object_t *_stm_enum_objects_pointing_to_nursery(long index); +object_t *_stm_next_last_cl_entry(); +void _stm_start_enum_last_cl_entry(); #endif /* ==================== HELPERS ==================== */ diff --git a/c8/test/support.py b/c8/test/support.py --- a/c8/test/support.py +++ b/c8/test/support.py @@ -67,6 +67,8 @@ long _stm_count_objects_pointing_to_nursery(void); object_t *_stm_enum_modified_old_objects(long index); object_t *_stm_enum_objects_pointing_to_nursery(long index); +object_t *_stm_next_last_cl_entry(); +void _stm_start_enum_last_cl_entry(); void *memset(void *s, int c, size_t n); @@ -373,6 +375,14 @@ return None return map(lib._stm_enum_old_objects_with_cards, range(count)) +def last_commit_log_entries(): + lib._stm_start_enum_last_cl_entry() + res = [] + obj = lib._stm_next_last_cl_entry() + while obj != ffi.NULL: + res.append(obj) + obj = lib._stm_next_last_cl_entry() + return res diff --git a/c8/test/test_basic.py b/c8/test/test_basic.py --- a/c8/test/test_basic.py +++ b/c8/test/test_basic.py @@ -33,9 +33,12 @@ self.switch(1) self.start_transaction() self.commit_transaction() + assert last_commit_log_entries() == [] + self.switch(0) self.commit_transaction() + assert last_commit_log_entries() == [] def test_simple_read(self): self.start_transaction() @@ -43,6 +46,7 @@ stm_read(lp1) assert stm_was_read(lp1) self.commit_transaction() + assert last_commit_log_entries() == [] def test_simple_write(self): self.start_transaction() @@ -53,6 +57,7 @@ assert modified_old_objects() == [] # object not old assert objects_pointing_to_nursery() == [] # short transaction self.commit_transaction() + assert last_commit_log_entries() == [] def test_allocate_old(self): lp1 = stm_allocate_old(16) @@ -101,6 +106,7 @@ # self.switch(1) self.commit_transaction() + assert last_commit_log_entries() == [lp1] # py.test.raises(Conflict, self.switch, 0) # detects rw conflict @@ -123,6 +129,7 @@ assert (p2 - p1) % 4096 == 0 assert stm_get_char(lp) == 'u' self.commit_transaction() + assert last_commit_log_entries() == [lp] def test_commit_fresh_objects2(self): self.switch(1) From noreply at buildbot.pypy.org Fri Sep 5 16:43:00 2014 From: noreply at buildbot.pypy.org (bivab) Date: Fri, 5 Sep 2014 16:43:00 +0200 (CEST) Subject: [pypy-commit] buildbot default: drop ARMHF/Raring test runs related to geenbox3-node0 builder, which is not availible anymore Message-ID: <20140905144300.0615D1C1299@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: Changeset: r916:942e14a6d1de Date: 2014-09-05 12:51 +0200 http://bitbucket.org/pypy/buildbot/changeset/942e14a6d1de/ Log: drop ARMHF/Raring test runs related to geenbox3-node0 builder, which is not availible anymore diff --git a/bot2/pypybuildbot/arm_master.py b/bot2/pypybuildbot/arm_master.py --- a/bot2/pypybuildbot/arm_master.py +++ b/bot2/pypybuildbot/arm_master.py @@ -122,7 +122,6 @@ BUILDJITLINUXARMHF_RARING = "build-pypy-c-jit-linux-armhf-raring" builderNames = [ - LINUXARMHF, APPLVLLINUXARM, APPLVLLINUXARMHF_v7, APPLVLLINUXARMHF_RASPBIAN, @@ -148,8 +147,6 @@ BUILDLINUXARM, # on hhu-cross-armel, uses 1 core BUILDLINUXARMHF_RASPBIAN, # on hhu-cross-raspbianhf, uses 1 core - LINUXARMHF, # onw tests on greenbox3-node0 - JITBACKENDONLYLINUXARMEL, # on hhu-imx.53 JITBACKENDONLYLINUXARMHF, JITBACKENDONLYLINUXARMHF_v7, # on cubieboard-bob @@ -172,8 +169,7 @@ JITLINUXARMHF_v7, # triggered by BUILDJITLINUXARMHF_RASPBIAN, on cubieboard-bob ]), - Triggerable("JITLINUXARMHF_RARING_scheduler", [ - JITLINUXARMHF_RARING, # triggered by BUILDJITLINUXARMHF_RARING + Triggerable("JITLINUXARMHF_RARING_scheduler", [ # triggered by BUILDJITLINUXARMHF_RARING ]) ] @@ -198,12 +194,6 @@ "locks": [ARMBoardLock.access('counting')], }, ## armv7 - {"name": LINUXARMHF, - "slavenames": ["greenbox3-node0"], - "builddir": LINUXARMHF, - "factory": pypyOwnTestFactoryARM, - "category": 'linux-armhf', - }, {"name": JITBACKENDONLYLINUXARMHF_v7, "slavenames": ['cubieboard-bob'], "builddir": JITBACKENDONLYLINUXARMHF_v7, @@ -257,12 +247,6 @@ 'category': 'linux-armhf', "locks": [ARMBoardLock.access('counting')], }, - {"name": JITLINUXARMHF_RARING, - "slavenames": ["greenbox3-node0"], - 'builddir': JITLINUXARMHF_RARING, - 'factory': pypyARMHF_RARING_JITTranslatedTestFactory, - 'category': 'linux-armhf', - }, # Translation Builders for ARM {"name": BUILDLINUXARM, "slavenames": ['hhu-cross-armel'], From noreply at buildbot.pypy.org Fri Sep 5 17:10:35 2014 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 5 Sep 2014 17:10:35 +0200 (CEST) Subject: [pypy-commit] pypy default: Add a readline() method to speed up reading lines out of a text file on Message-ID: <20140905151035.CC4FD1C0793@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73319:316def1241d1 Date: 2014-09-05 17:09 +0200 http://bitbucket.org/pypy/pypy/changeset/316def1241d1/ Log: Add a readline() method to speed up reading lines out of a text file on Windows diff --git a/rpython/rlib/streamio.py b/rpython/rlib/streamio.py --- a/rpython/rlib/streamio.py +++ b/rpython/rlib/streamio.py @@ -900,6 +900,13 @@ return '\n'.join(result) + def readline(self): + line = self.base.readline() + limit = len(line) - 2 + if limit >= 0 and line[limit] == '\r' and line[limit + 1] == '\n': + line = line[:limit] + '\n' + return line + def tell(self): pos = self.base.tell() return pos - len(self.lfbuffer) From noreply at buildbot.pypy.org Fri Sep 5 20:19:02 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Fri, 5 Sep 2014 20:19:02 +0200 (CEST) Subject: [pypy-commit] pypy default: support IOError in rpython Message-ID: <20140905181902.622C11C0793@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r73320:14a5e9603b7a Date: 2014-09-05 13:35 -0400 http://bitbucket.org/pypy/pypy/changeset/14a5e9603b7a/ Log: support IOError in rpython diff --git a/rpython/annotator/builtin.py b/rpython/annotator/builtin.py --- a/rpython/annotator/builtin.py +++ b/rpython/annotator/builtin.py @@ -255,6 +255,10 @@ BUILTIN_ANALYZERS[original] = value + at analyzer_for(getattr(IOError.__init__, 'im_func', IOError.__init__)) +def IOError_init(s_self, *args): + pass + @analyzer_for(getattr(OSError.__init__, 'im_func', OSError.__init__)) def OSError_init(s_self, *args): pass diff --git a/rpython/annotator/classdef.py b/rpython/annotator/classdef.py --- a/rpython/annotator/classdef.py +++ b/rpython/annotator/classdef.py @@ -438,8 +438,9 @@ # ____________________________________________________________ FORCE_ATTRIBUTES_INTO_CLASSES = { + IOError: {'errno': SomeInteger()}, OSError: {'errno': SomeInteger()}, - } +} try: WindowsError diff --git a/rpython/rtyper/rbuiltin.py b/rpython/rtyper/rbuiltin.py --- a/rpython/rtyper/rbuiltin.py +++ b/rpython/rtyper/rbuiltin.py @@ -264,6 +264,17 @@ def rtype_object__init__(hop): hop.exception_cannot_occur() +def rtype_IOError__init__(hop): + hop.exception_cannot_occur() + if hop.nb_args == 2: + raise TyperError("IOError() should not be called with " + "a single argument") + if hop.nb_args >= 3: + v_self = hop.args_v[0] + r_self = hop.args_r[0] + v_errno = hop.inputarg(lltype.Signed, arg=1) + r_self.setfield(v_self, 'errno', v_errno, hop.llops) + def rtype_OSError__init__(hop): hop.exception_cannot_occur() if hop.nb_args == 2: @@ -333,6 +344,9 @@ original = getattr(__builtin__, name[14:]) BUILTIN_TYPER[original] = value +BUILTIN_TYPER[getattr(IOError.__init__, 'im_func', IOError.__init__)] = ( + rtype_IOError__init__) + BUILTIN_TYPER[getattr(OSError.__init__, 'im_func', OSError.__init__)] = ( rtype_OSError__init__) diff --git a/rpython/rtyper/test/test_exception.py b/rpython/rtyper/test/test_exception.py --- a/rpython/rtyper/test/test_exception.py +++ b/rpython/rtyper/test/test_exception.py @@ -36,14 +36,23 @@ class TestException(BaseRtypingTest): def test_exception_with_arg(self): def g(n): + raise IOError(n, "?") + def h(n): raise OSError(n, "?") def f(n): try: g(n) + except IOError, e: + assert e.errno == 42 + else: + assert False + try: + h(n) except OSError, e: - return e.errno - res = self.interpret(f, [42]) - assert res == 42 + assert e.errno == 42 + else: + assert False + self.interpret(f, [42]) def test_catch_incompatible_class(self): class MyError(Exception): From noreply at buildbot.pypy.org Fri Sep 5 20:19:03 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Fri, 5 Sep 2014 20:19:03 +0200 (CEST) Subject: [pypy-commit] pypy default: these seem obsolete for rpython-level OSError Message-ID: <20140905181903.B87BA1C0793@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r73321:3bf469df25da Date: 2014-09-05 13:46 -0400 http://bitbucket.org/pypy/pypy/changeset/3bf469df25da/ Log: these seem obsolete for rpython-level OSError diff --git a/rpython/rtyper/exceptiondata.py b/rpython/rtyper/exceptiondata.py --- a/rpython/rtyper/exceptiondata.py +++ b/rpython/rtyper/exceptiondata.py @@ -44,13 +44,6 @@ classdef = bk.getuniqueclassdef(cls) rclass.getclassrepr(rtyper, classdef).setup() - def make_raise_OSError(self, rtyper): - # ll_raise_OSError(errno) - def ll_raise_OSError(errno): - raise OSError(errno, None) - helper_fn = rtyper.annotate_helper_fn(ll_raise_OSError, [annmodel.SomeInteger()]) - return helper_fn - def get_standard_ll_exc_instance(self, rtyper, clsdef): from rpython.rtyper.lltypesystem.rclass import getinstancerepr r_inst = getinstancerepr(rtyper, clsdef) @@ -69,7 +62,6 @@ # create helper functionptrs self.fn_exception_match = self.make_exception_matcher(rtyper) self.fn_type_of_exc_inst = self.make_type_of_exc_inst(rtyper) - self.fn_raise_OSError = self.make_raise_OSError(rtyper) def make_exception_matcher(self, rtyper): # ll_exception_matcher(real_exception_vtable, match_exception_vtable) diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py --- a/rpython/rtyper/llinterp.py +++ b/rpython/rtyper/llinterp.py @@ -441,12 +441,8 @@ extraargs = () typer = self.llinterpreter.typer exdata = typer.exceptiondata - if isinstance(exc, OSError): - self.op_direct_call(exdata.fn_raise_OSError, exc.errno) - assert False, "op_direct_call above should have raised" - else: - evalue = exdata.get_standard_ll_exc_instance_by_class(exc.__class__) - etype = self.op_direct_call(exdata.fn_type_of_exc_inst, evalue) + evalue = exdata.get_standard_ll_exc_instance_by_class(exc.__class__) + etype = self.op_direct_call(exdata.fn_type_of_exc_inst, evalue) raise LLException(etype, evalue, *extraargs) def invoke_callable_with_pyexceptions(self, fptr, *args): diff --git a/rpython/translator/c/extfunc.py b/rpython/translator/c/extfunc.py --- a/rpython/translator/c/extfunc.py +++ b/rpython/translator/c/extfunc.py @@ -90,7 +90,6 @@ yield ('RPYTHON_EXCEPTION_MATCH', exceptiondata.fn_exception_match) yield ('RPYTHON_TYPE_OF_EXC_INST', exceptiondata.fn_type_of_exc_inst) - yield ('RPYTHON_RAISE_OSERROR', exceptiondata.fn_raise_OSError) yield ('RPyExceptionOccurred1', exctransformer.rpyexc_occured_ptr.value) yield ('RPyFetchExceptionType', exctransformer.rpyexc_fetch_type_ptr.value) From noreply at buildbot.pypy.org Fri Sep 5 20:45:52 2014 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 5 Sep 2014 20:45:52 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: implement clear array contents Message-ID: <20140905184552.48C201C0793@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73322:381a8a3875fd Date: 2014-09-05 12:45 -0600 http://bitbucket.org/pypy/pypy/changeset/381a8a3875fd/ Log: implement clear array contents diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py --- a/rpython/jit/backend/llgraph/runner.py +++ b/rpython/jit/backend/llgraph/runner.py @@ -630,19 +630,22 @@ def bh_new(self, sizedescr): return lltype.cast_opaque_ptr(llmemory.GCREF, - lltype.malloc(sizedescr.S, zero=True)) + lltype.malloc(sizedescr.S)) def bh_new_with_vtable(self, vtable, descr): - result = lltype.malloc(descr.S, zero=True) + result = lltype.malloc(descr.S) result_as_objptr = lltype.cast_pointer(rclass.OBJECTPTR, result) result_as_objptr.typeptr = support.cast_from_int(rclass.CLASSTYPE, vtable) return lltype.cast_opaque_ptr(llmemory.GCREF, result) def bh_new_array(self, length, arraydescr): - array = lltype.malloc(arraydescr.A, length, zero=True) + array = lltype.malloc(arraydescr.A, length) return lltype.cast_opaque_ptr(llmemory.GCREF, array) + def bh_clear_array_contents(self, array, arraydescr): + pass # I think arrays are zero-initialized here anyhow + def bh_classof(self, struct): struct = lltype.cast_opaque_ptr(rclass.OBJECTPTR, struct) result_adr = llmemory.cast_ptr_to_adr(struct.typeptr) diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py --- a/rpython/jit/backend/llsupport/gc.py +++ b/rpython/jit/backend/llsupport/gc.py @@ -18,10 +18,12 @@ from rpython.jit.backend.llsupport.descr import get_call_descr from rpython.jit.backend.llsupport.rewrite import GcRewriterAssembler from rpython.memory.gctransform import asmgcroot +from rpython.jit.codewriter.effectinfo import EffectInfo # ____________________________________________________________ class GcLLDescription(GcCache): + malloc_zero_filled = True def __init__(self, gcdescr, translator=None, rtyper=None): GcCache.__init__(self, translator is not None, rtyper) @@ -32,6 +34,15 @@ self.fielddescr_vtable = get_field_descr(self, rclass.OBJECT, 'typeptr') self._generated_functions = [] + self.memset_ptr = rffi.llexternal('memset', [lltype.Signed, rffi.INT, + rffi.SIZE_T], lltype.Void, + sandboxsafe=True, + _nowrapper=True) + self.memset_ptr_as_int = heaptracker.adr2int( + llmemory.cast_ptr_to_adr(self.memset_ptr)) + ei = EffectInfo([], [], [], [], [], [], EffectInfo.EF_CANNOT_RAISE) + self.memset_descr = get_call_descr(self, [lltype.Signed, rffi.INT, + rffi.SIZE_T], lltype.Void, ei) def _setup_str(self): self.str_descr = get_array_descr(self, rstr.STR) @@ -377,6 +388,7 @@ from rpython.memory.gcheader import GCHeaderBuilder self.GCClass = self.layoutbuilder.GCClass self.moving_gc = self.GCClass.moving_gc + self.malloc_zero_filled = self.GCClass.malloc_zero_filled self.HDRPTR = lltype.Ptr(self.GCClass.HDR) self.gcheaderbuilder = GCHeaderBuilder(self.HDRPTR.TO) self.max_size_of_young_obj = self.GCClass.JIT_max_size_of_young_obj() diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py --- a/rpython/jit/backend/llsupport/llmodel.py +++ b/rpython/jit/backend/llsupport/llmodel.py @@ -591,6 +591,13 @@ def bh_new(self, sizedescr): return self.gc_ll_descr.gc_malloc(sizedescr) + def bh_clear_array_contents(self, ref, arraydescr): + ofs, size, _ = self.unpack_arraydescr_size(arraydescr) + arraysize = self.bh_arraylen_gc(ref, arraydescr) + totalsize = size * arraysize + adr = rffi.cast(lltype.Signed, ref) + ofs + self.gc_ll_descr.memset_ptr(adr, 1, totalsize) + def bh_new_with_vtable(self, vtable, sizedescr): res = self.gc_ll_descr.gc_malloc(sizedescr) if self.vtable_offset is not None: diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -1,7 +1,7 @@ from rpython.rlib.rarithmetic import ovfcheck from rpython.rtyper.lltypesystem import llmemory from rpython.jit.metainterp import history -from rpython.jit.metainterp.history import ConstInt, BoxPtr, ConstPtr +from rpython.jit.metainterp.history import ConstInt, BoxPtr, ConstPtr, BoxInt from rpython.jit.metainterp.resoperation import ResOperation, rop from rpython.jit.codewriter import heaptracker from rpython.jit.backend.llsupport.symbolic import WORD @@ -59,6 +59,10 @@ if op.is_malloc(): self.handle_malloc_operation(op) continue + elif op.getopnum() == rop.CLEAR_ARRAY_CONTENTS: + if not self.gc_ll_descr.malloc_zero_filled: + self.handle_clear_array_contents(op) + continue elif op.can_malloc(): self.emitting_an_operation_that_can_collect() elif op.getopnum() == rop.LABEL: @@ -157,6 +161,26 @@ else: raise NotImplementedError(op.getopname()) + def handle_clear_array_contents(self, op): + # XXX this maybe should go to optimizer, so we can remove extra ops? + arraydescr = op.getdescr() + ofs, size, _ = self.cpu.unpack_arraydescr_size(arraydescr) + v_arr = op.getarg(0) + v_arr_plus_ofs = BoxInt() + v_arrsize = BoxInt() + v_totalsize = BoxInt() + gcdescr = self.gc_ll_descr + ops = [ + ResOperation(rop.INT_ADD, [v_arr, ConstInt(size)], v_arr_plus_ofs), + ResOperation(rop.ARRAYLEN_GC, [v_arr], v_arrsize, descr=arraydescr), + ResOperation(rop.INT_MUL, [v_arrsize, ConstInt(size)], v_totalsize), + ResOperation(rop.CALL, [ConstInt(gcdescr.memset_ptr_as_int), + v_arr_plus_ofs, + ConstInt(0), v_totalsize], None, + descr=gcdescr.memset_descr), + ] + self.newops.extend(ops) + def gen_malloc_frame(self, frame_info, frame, size_box): descrs = self.gc_ll_descr.getframedescrs(self.cpu) if self.gc_ll_descr.kind == 'boehm': diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py --- a/rpython/jit/backend/test/runner_test.py +++ b/rpython/jit/backend/test/runner_test.py @@ -4439,3 +4439,21 @@ res = self.execute_operation(rop.CAST_FLOAT_TO_SINGLEFLOAT, [boxfloat(12.5)], 'int') assert res.getint() == struct.unpack("I", struct.pack("f", 12.5))[0] + + def test_clear_array_contents(self): + from rpython.jit.backend.llsupport.llmodel import AbstractLLCPU + if not isinstance(self.cpu, AbstractLLCPU): + py.test.skip("pointless test on non-asm") + oldval = self.cpu.gc_ll_descr.malloc_zero_filled + self.cpu.gc_ll_descr.malloc_zero_filled = False + try: + A = lltype.GcArray(lltype.Signed) + a = lltype.malloc(A, 3) + a[1] = 13 + descr = self.cpu.arraydescrof(A) + ref = lltype.cast_opaque_ptr(llmemory.GCREF, a) + self.execute_operation(rop.CLEAR_ARRAY_CONTENTS, + [BoxPtr(ref)], 'void', descr=descr) + assert a[1] == 0 + finally: + self.cpu.gc_ll_descr.malloc_zero_filled = oldval diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -315,7 +315,8 @@ self.assembler.mc.mark_op(op) self.rm.position = i self.xrm.position = i - if op.has_no_side_effect() and op.result not in self.longevity: + if (op.has_no_side_effect() and op.result not in self.longevity + and op.opnum != rop.CLEAR_ARRAY_CONTENTS): i += 1 self.possibly_free_vars_for_op(op) continue diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py --- a/rpython/jit/codewriter/jtransform.py +++ b/rpython/jit/codewriter/jtransform.py @@ -615,12 +615,41 @@ op1 = SpaceOperation('new_array', [op.args[2], arraydescr], op.result) if self._has_gcptrs_in(ARRAY): - return [op1, SpaceOperation('zero_gc_pointers', [op.result], - None)] + return self.zero_contents(op1, op.result, ARRAY, + only_gc_pointers=True) if op.args[1].value.get('zero', False): - return [op1, SpaceOperation('zero_contents', [op.result], None)] + return self.zero_contents(op1, op.result, ARRAY) return op1 + def zero_contents(self, prev_op, v, TYPE, only_gc_pointers=False): + ops = [prev_op] + if isinstance(TYPE, lltype.Struct): + for name, FIELD in TYPE._flds.iteritems(): + if (not only_gc_pointers or + isinstance(FIELD, lltype.Ptr) and FIELD._needsgc()): + c_name = Constant(name, lltype.Void) + c_null = Constant(FIELD._defl(), FIELD) + op = SpaceOperation('setfield', [v, c_name, c_null], + None) + self.extend_with(ops, self.rewrite_op_setfield(op)) + elif isinstance(TYPE, lltype.Array): + arraydescr = self.cpu.arraydescrof(TYPE) + ops.append(SpaceOperation('clear_array_contents', + [v, arraydescr], None)) + else: + raise TypeError("Expected struct or array, got '%r'", (TYPE,)) + if len(ops) == 1: + return ops[0] + return ops + + def extend_with(self, l, ops): + if ops is None: + return + if isinstance(ops, list): + l.extend(ops) + else: + l.append(ops) + def rewrite_op_free(self, op): d = op.args[1].value.copy() assert d['flavor'] == 'raw' @@ -893,9 +922,10 @@ sizedescr = self.cpu.sizeof(STRUCT) op1 = SpaceOperation(opname, [sizedescr], op.result) if true_zero: - return [op1, SpaceOperation('zero_contents', [op.result], None)] + return self.zero_contents(op1, op.result, STRUCT) if self._has_gcptrs_in(STRUCT): - return [op1, SpaceOperation('zero_gc_pointers', [op.result], None)] + return self.zero_contents(op1, op.result, STRUCT, + only_gc_pointers=True) return op1 def _has_gcptrs_in(self, STRUCT): diff --git a/rpython/jit/codewriter/test/test_jtransform.py b/rpython/jit/codewriter/test/test_jtransform.py --- a/rpython/jit/codewriter/test/test_jtransform.py +++ b/rpython/jit/codewriter/test/test_jtransform.py @@ -538,12 +538,11 @@ op1, op2 = Transformer(FakeCPU()).rewrite_operation(op) assert op1.opname == 'new' assert op1.args == [('sizedescr', S)] - assert op2.opname == 'zero_gc_pointers' - assert op2.args == [v] + assert op2.opname == 'setfield_gc_r' + assert op2.args[0] == v def test_malloc_new_zero_2(): - SS = lltype.GcStruct('SS') - S = lltype.GcStruct('S', ('x', lltype.Ptr(SS))) + S = lltype.GcStruct('S', ('x', lltype.Signed)) v = varoftype(lltype.Ptr(S)) op = SpaceOperation('malloc', [Constant(S, lltype.Void), Constant({'flavor': 'gc', @@ -551,8 +550,8 @@ op1, op2 = Transformer(FakeCPU()).rewrite_operation(op) assert op1.opname == 'new' assert op1.args == [('sizedescr', S)] - assert op2.opname == 'zero_contents' - assert op2.args == [v] + assert op2.opname == 'setfield_gc_i' + assert op2.args[0] == v def test_malloc_new_with_vtable(): vtable = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True) @@ -1063,7 +1062,7 @@ op = SpaceOperation('malloc_varsize', [c_A, c_flags, v1], v2) op1, op2 = Transformer(FakeCPU()).rewrite_operation(op) assert op1.opname == 'new_array' - assert op2.opname == 'zero_contents' + assert op2.opname == 'clear_array_contents' def test_str_concat(): # test that the oopspec is present and correctly transformed diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py --- a/rpython/jit/metainterp/blackhole.py +++ b/rpython/jit/metainterp/blackhole.py @@ -1345,6 +1345,10 @@ vtable = heaptracker.descr2vtable(cpu, descr) return cpu.bh_new_with_vtable(vtable, descr) + @arguments("cpu", "r", "d") + def bhimpl_clear_array_contents(cpu, ref, descr): + cpu.bh_clear_array_contents(ref, descr) + @arguments("cpu", "r", returns="i") def bhimpl_guard_class(cpu, struct): return cpu.bh_classof(struct) diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py --- a/rpython/jit/metainterp/executor.py +++ b/rpython/jit/metainterp/executor.py @@ -335,6 +335,7 @@ rop.CALL_MALLOC_NURSERY, rop.CALL_MALLOC_NURSERY_VARSIZE, rop.CALL_MALLOC_NURSERY_VARSIZE_FRAME, + rop.CLEAR_ARRAY_CONTENTS, rop.LABEL, ): # list of opcodes never executed by pyjitpl continue diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -396,6 +396,10 @@ def opimpl_new(self, sizedescr): return self.metainterp.execute_new(sizedescr) + @arguments("box", "descr") + def opimpl_clear_array_contents(self, box, descr): + self.metainterp.execute_and_record(rop.CLEAR_ARRAY_CONTENTS, descr, box) + @arguments("descr") def opimpl_new_with_vtable(self, sizedescr): cpu = self.metainterp.cpu diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py --- a/rpython/jit/metainterp/resoperation.py +++ b/rpython/jit/metainterp/resoperation.py @@ -492,6 +492,9 @@ 'MARK_OPAQUE_PTR/1b', # this one has no *visible* side effect, since the virtualizable # must be forced, however we need to execute it anyway + 'CLEAR_ARRAY_CONTENTS/1d', + # this one does not *really* have a side effect since it's equivalent + # to array just coming zeroed '_NOSIDEEFFECT_LAST', # ----- end of no_side_effect operations ----- 'INCREMENT_DEBUG_COUNTER/1', From noreply at buildbot.pypy.org Fri Sep 5 20:48:34 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Fri, 5 Sep 2014 20:48:34 +0200 (CEST) Subject: [pypy-commit] pypy rpython-enverror: attempt to support EnvironmentError properly in rpython Message-ID: <20140905184834.90E671C0793@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: rpython-enverror Changeset: r73323:9984536b0e51 Date: 2014-09-05 14:47 -0400 http://bitbucket.org/pypy/pypy/changeset/9984536b0e51/ Log: attempt to support EnvironmentError properly in rpython diff --git a/rpython/annotator/builtin.py b/rpython/annotator/builtin.py --- a/rpython/annotator/builtin.py +++ b/rpython/annotator/builtin.py @@ -255,12 +255,8 @@ BUILTIN_ANALYZERS[original] = value - at analyzer_for(getattr(IOError.__init__, 'im_func', IOError.__init__)) -def IOError_init(s_self, *args): - pass - - at analyzer_for(getattr(OSError.__init__, 'im_func', OSError.__init__)) -def OSError_init(s_self, *args): + at analyzer_for(getattr(EnvironmentError.__init__, 'im_func', EnvironmentError.__init__)) +def EnvironmentError_init(s_self, *args): pass try: diff --git a/rpython/annotator/classdef.py b/rpython/annotator/classdef.py --- a/rpython/annotator/classdef.py +++ b/rpython/annotator/classdef.py @@ -438,8 +438,7 @@ # ____________________________________________________________ FORCE_ATTRIBUTES_INTO_CLASSES = { - IOError: {'errno': SomeInteger()}, - OSError: {'errno': SomeInteger()}, + EnvironmentError: {'errno': SomeInteger(), 'strerror': SomeString(can_be_None=True), 'filename': SomeString(can_be_None=True)}, } try: diff --git a/rpython/rtyper/rbuiltin.py b/rpython/rtyper/rbuiltin.py --- a/rpython/rtyper/rbuiltin.py +++ b/rpython/rtyper/rbuiltin.py @@ -264,27 +264,18 @@ def rtype_object__init__(hop): hop.exception_cannot_occur() -def rtype_IOError__init__(hop): +def rtype_EnvironmentError__init__(hop): hop.exception_cannot_occur() - if hop.nb_args == 2: - raise TyperError("IOError() should not be called with " - "a single argument") - if hop.nb_args >= 3: + if hop.nb_args >= 2: v_self = hop.args_v[0] r_self = hop.args_r[0] v_errno = hop.inputarg(lltype.Signed, arg=1) r_self.setfield(v_self, 'errno', v_errno, hop.llops) - -def rtype_OSError__init__(hop): - hop.exception_cannot_occur() - if hop.nb_args == 2: - raise TyperError("OSError() should not be called with " - "a single argument") - if hop.nb_args >= 3: - v_self = hop.args_v[0] - r_self = hop.args_r[0] - v_errno = hop.inputarg(lltype.Signed, arg=1) - r_self.setfield(v_self, 'errno', v_errno, hop.llops) + if hop.nb_args >= 3: + v_strerror = hop.inputarg(hop.args_r[2], arg=2) + else: + v_strerror = None + r_self.setfield(v_self, 'strerror', v_strerror, hop.llops) def rtype_WindowsError__init__(hop): hop.exception_cannot_occur() @@ -344,11 +335,8 @@ original = getattr(__builtin__, name[14:]) BUILTIN_TYPER[original] = value -BUILTIN_TYPER[getattr(IOError.__init__, 'im_func', IOError.__init__)] = ( - rtype_IOError__init__) - -BUILTIN_TYPER[getattr(OSError.__init__, 'im_func', OSError.__init__)] = ( - rtype_OSError__init__) +BUILTIN_TYPER[getattr(EnvironmentError.__init__, 'im_func', EnvironmentError.__init__)] = ( + rtype_EnvironmentError__init__) try: WindowsError diff --git a/rpython/rtyper/test/test_exception.py b/rpython/rtyper/test/test_exception.py --- a/rpython/rtyper/test/test_exception.py +++ b/rpython/rtyper/test/test_exception.py @@ -36,20 +36,34 @@ class TestException(BaseRtypingTest): def test_exception_with_arg(self): def g(n): - raise IOError(n, "?") + raise IOError(n) def h(n): raise OSError(n, "?") + def i(n): + raise EnvironmentError(n, "?", "test") def f(n): try: g(n) except IOError, e: assert e.errno == 42 + assert e.strerror is None + assert e.filename is None else: assert False try: h(n) except OSError, e: assert e.errno == 42 + assert e.strerror == "?" + assert e.filename is None + else: + assert False + try: + i(n) + except EnvironmentError as e: + assert e.errno == 42 + assert e.strerror == "?" + assert e.filename == "test" else: assert False self.interpret(f, [42]) From noreply at buildbot.pypy.org Fri Sep 5 20:59:04 2014 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 5 Sep 2014 20:59:04 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: fix some tests Message-ID: <20140905185904.185531C0793@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73324:0026f4ea693a Date: 2014-09-05 12:58 -0600 http://bitbucket.org/pypy/pypy/changeset/0026f4ea693a/ Log: fix some tests diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py --- a/rpython/jit/backend/llgraph/runner.py +++ b/rpython/jit/backend/llgraph/runner.py @@ -643,8 +643,19 @@ array = lltype.malloc(arraydescr.A, length) return lltype.cast_opaque_ptr(llmemory.GCREF, array) - def bh_clear_array_contents(self, array, arraydescr): - pass # I think arrays are zero-initialized here anyhow + def bh_clear_array_contents(self, a, descr): + a = support.cast_arg(lltype.Ptr(descr.A), a) + ITEM = descr.A.OF + array = a._obj + if isinstance(ITEM, lltype.Struct): + for i in xrange(array.getlength()): + for name, FIELD in ITEM._flds.iteritems(): + null = FIELD._defl() + setattr(array.getitem(i), name, null) + else: + null = ITEM._defl() + for i in xrange(array.getlength()): + array.setitem(i, null) def bh_classof(self, struct): struct = lltype.cast_opaque_ptr(rclass.OBJECTPTR, struct) diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py --- a/rpython/jit/backend/llsupport/llmodel.py +++ b/rpython/jit/backend/llsupport/llmodel.py @@ -596,7 +596,8 @@ arraysize = self.bh_arraylen_gc(ref, arraydescr) totalsize = size * arraysize adr = rffi.cast(lltype.Signed, ref) + ofs - self.gc_ll_descr.memset_ptr(adr, 1, totalsize) + self.gc_ll_descr.memset_ptr(adr, rffi.cast(rffi.INT, 0), + rffi.cast(rffi.SIZE_T, totalsize)) def bh_new_with_vtable(self, vtable, sizedescr): res = self.gc_ll_descr.gc_malloc(sizedescr) From noreply at buildbot.pypy.org Fri Sep 5 21:12:59 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Fri, 5 Sep 2014 21:12:59 +0200 (CEST) Subject: [pypy-commit] pypy rpython-enverror: fix Message-ID: <20140905191259.2B9191C0793@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: rpython-enverror Changeset: r73325:1ff37a253216 Date: 2014-09-05 15:12 -0400 http://bitbucket.org/pypy/pypy/changeset/1ff37a253216/ Log: fix diff --git a/rpython/rtyper/rbuiltin.py b/rpython/rtyper/rbuiltin.py --- a/rpython/rtyper/rbuiltin.py +++ b/rpython/rtyper/rbuiltin.py @@ -3,9 +3,10 @@ from rpython.rlib import rarithmetic, objectmodel from rpython.rtyper import raddress, rptr, extregistry, rrange from rpython.rtyper.error import TyperError -from rpython.rtyper.lltypesystem import lltype, llmemory, rclass +from rpython.rtyper.lltypesystem import lltype, llmemory, rclass, rstr from rpython.rtyper.lltypesystem.rdict import rtype_r_dict from rpython.rtyper.rmodel import Repr +from rpython.rtyper.rstr import AbstractStringRepr from rpython.tool.pairtype import pairtype @@ -272,10 +273,11 @@ v_errno = hop.inputarg(lltype.Signed, arg=1) r_self.setfield(v_self, 'errno', v_errno, hop.llops) if hop.nb_args >= 3: - v_strerror = hop.inputarg(hop.args_r[2], arg=2) - else: - v_strerror = None - r_self.setfield(v_self, 'strerror', v_strerror, hop.llops) + v_strerror = hop.inputarg(rstr.string_repr, arg=2) + r_self.setfield(v_self, 'strerror', v_strerror, hop.llops) + if hop.nb_args >= 4: + v_filename = hop.inputarg(rstr.string_repr, arg=3) + r_self.setfield(v_self, 'filename', v_filename, hop.llops) def rtype_WindowsError__init__(hop): hop.exception_cannot_occur() From noreply at buildbot.pypy.org Fri Sep 5 21:21:36 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Fri, 5 Sep 2014 21:21:36 +0200 (CEST) Subject: [pypy-commit] pypy rpython-enverror: enhance test Message-ID: <20140905192136.DDC2F1C1299@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: rpython-enverror Changeset: r73326:b98721991115 Date: 2014-09-05 15:16 -0400 http://bitbucket.org/pypy/pypy/changeset/b98721991115/ Log: enhance test diff --git a/rpython/rtyper/test/test_exception.py b/rpython/rtyper/test/test_exception.py --- a/rpython/rtyper/test/test_exception.py +++ b/rpython/rtyper/test/test_exception.py @@ -38,9 +38,11 @@ def g(n): raise IOError(n) def h(n): - raise OSError(n, "?") + raise OSError(n, "?", None) def i(n): raise EnvironmentError(n, "?", "test") + def j(n): + raise IOError(0, "test") def f(n): try: g(n) @@ -66,6 +68,12 @@ assert e.filename == "test" else: assert False + try: + j(n) + except (IOError, OSError) as e: + assert e.errno == 0 + assert e.strerror == "test" + assert e.filename is None self.interpret(f, [42]) def test_catch_incompatible_class(self): From noreply at buildbot.pypy.org Fri Sep 5 21:21:38 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Fri, 5 Sep 2014 21:21:38 +0200 (CEST) Subject: [pypy-commit] pypy rpython-enverror: fail cleanly without args Message-ID: <20140905192138.2B3C31C1299@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: rpython-enverror Changeset: r73327:1e5fec237272 Date: 2014-09-05 15:21 -0400 http://bitbucket.org/pypy/pypy/changeset/1e5fec237272/ Log: fail cleanly without args diff --git a/rpython/rtyper/rbuiltin.py b/rpython/rtyper/rbuiltin.py --- a/rpython/rtyper/rbuiltin.py +++ b/rpython/rtyper/rbuiltin.py @@ -267,17 +267,19 @@ def rtype_EnvironmentError__init__(hop): hop.exception_cannot_occur() - if hop.nb_args >= 2: - v_self = hop.args_v[0] - r_self = hop.args_r[0] - v_errno = hop.inputarg(lltype.Signed, arg=1) - r_self.setfield(v_self, 'errno', v_errno, hop.llops) - if hop.nb_args >= 3: - v_strerror = hop.inputarg(rstr.string_repr, arg=2) - r_self.setfield(v_self, 'strerror', v_strerror, hop.llops) - if hop.nb_args >= 4: - v_filename = hop.inputarg(rstr.string_repr, arg=3) - r_self.setfield(v_self, 'filename', v_filename, hop.llops) + if hop.nb_args <= 1: + raise TyperError("EnvironmentError() should be called with " + "at least one argument") + v_self = hop.args_v[0] + r_self = hop.args_r[0] + v_errno = hop.inputarg(lltype.Signed, arg=1) + r_self.setfield(v_self, 'errno', v_errno, hop.llops) + if hop.nb_args >= 3: + v_strerror = hop.inputarg(rstr.string_repr, arg=2) + r_self.setfield(v_self, 'strerror', v_strerror, hop.llops) + if hop.nb_args >= 4: + v_filename = hop.inputarg(rstr.string_repr, arg=3) + r_self.setfield(v_self, 'filename', v_filename, hop.llops) def rtype_WindowsError__init__(hop): hop.exception_cannot_occur() From noreply at buildbot.pypy.org Fri Sep 5 21:24:57 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Fri, 5 Sep 2014 21:24:57 +0200 (CEST) Subject: [pypy-commit] pypy rpython-enverror: close branch before merge Message-ID: <20140905192457.1130A1C1299@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: rpython-enverror Changeset: r73328:d042af34b299 Date: 2014-09-05 15:23 -0400 http://bitbucket.org/pypy/pypy/changeset/d042af34b299/ Log: close branch before merge From noreply at buildbot.pypy.org Fri Sep 5 21:24:58 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Fri, 5 Sep 2014 21:24:58 +0200 (CEST) Subject: [pypy-commit] pypy default: properly support EnvironmentError in rpython Message-ID: <20140905192458.4429A1C1299@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r73329:34ddb6cb97b1 Date: 2014-09-05 15:23 -0400 http://bitbucket.org/pypy/pypy/changeset/34ddb6cb97b1/ Log: properly support EnvironmentError in rpython diff --git a/rpython/annotator/builtin.py b/rpython/annotator/builtin.py --- a/rpython/annotator/builtin.py +++ b/rpython/annotator/builtin.py @@ -255,12 +255,8 @@ BUILTIN_ANALYZERS[original] = value - at analyzer_for(getattr(IOError.__init__, 'im_func', IOError.__init__)) -def IOError_init(s_self, *args): - pass - - at analyzer_for(getattr(OSError.__init__, 'im_func', OSError.__init__)) -def OSError_init(s_self, *args): + at analyzer_for(getattr(EnvironmentError.__init__, 'im_func', EnvironmentError.__init__)) +def EnvironmentError_init(s_self, *args): pass try: diff --git a/rpython/annotator/classdef.py b/rpython/annotator/classdef.py --- a/rpython/annotator/classdef.py +++ b/rpython/annotator/classdef.py @@ -438,8 +438,7 @@ # ____________________________________________________________ FORCE_ATTRIBUTES_INTO_CLASSES = { - IOError: {'errno': SomeInteger()}, - OSError: {'errno': SomeInteger()}, + EnvironmentError: {'errno': SomeInteger(), 'strerror': SomeString(can_be_None=True), 'filename': SomeString(can_be_None=True)}, } try: diff --git a/rpython/rtyper/rbuiltin.py b/rpython/rtyper/rbuiltin.py --- a/rpython/rtyper/rbuiltin.py +++ b/rpython/rtyper/rbuiltin.py @@ -3,9 +3,10 @@ from rpython.rlib import rarithmetic, objectmodel from rpython.rtyper import raddress, rptr, extregistry, rrange from rpython.rtyper.error import TyperError -from rpython.rtyper.lltypesystem import lltype, llmemory, rclass +from rpython.rtyper.lltypesystem import lltype, llmemory, rclass, rstr from rpython.rtyper.lltypesystem.rdict import rtype_r_dict from rpython.rtyper.rmodel import Repr +from rpython.rtyper.rstr import AbstractStringRepr from rpython.tool.pairtype import pairtype @@ -264,27 +265,21 @@ def rtype_object__init__(hop): hop.exception_cannot_occur() -def rtype_IOError__init__(hop): +def rtype_EnvironmentError__init__(hop): hop.exception_cannot_occur() - if hop.nb_args == 2: - raise TyperError("IOError() should not be called with " - "a single argument") + if hop.nb_args <= 1: + raise TyperError("EnvironmentError() should be called with " + "at least one argument") + v_self = hop.args_v[0] + r_self = hop.args_r[0] + v_errno = hop.inputarg(lltype.Signed, arg=1) + r_self.setfield(v_self, 'errno', v_errno, hop.llops) if hop.nb_args >= 3: - v_self = hop.args_v[0] - r_self = hop.args_r[0] - v_errno = hop.inputarg(lltype.Signed, arg=1) - r_self.setfield(v_self, 'errno', v_errno, hop.llops) - -def rtype_OSError__init__(hop): - hop.exception_cannot_occur() - if hop.nb_args == 2: - raise TyperError("OSError() should not be called with " - "a single argument") - if hop.nb_args >= 3: - v_self = hop.args_v[0] - r_self = hop.args_r[0] - v_errno = hop.inputarg(lltype.Signed, arg=1) - r_self.setfield(v_self, 'errno', v_errno, hop.llops) + v_strerror = hop.inputarg(rstr.string_repr, arg=2) + r_self.setfield(v_self, 'strerror', v_strerror, hop.llops) + if hop.nb_args >= 4: + v_filename = hop.inputarg(rstr.string_repr, arg=3) + r_self.setfield(v_self, 'filename', v_filename, hop.llops) def rtype_WindowsError__init__(hop): hop.exception_cannot_occur() @@ -344,11 +339,8 @@ original = getattr(__builtin__, name[14:]) BUILTIN_TYPER[original] = value -BUILTIN_TYPER[getattr(IOError.__init__, 'im_func', IOError.__init__)] = ( - rtype_IOError__init__) - -BUILTIN_TYPER[getattr(OSError.__init__, 'im_func', OSError.__init__)] = ( - rtype_OSError__init__) +BUILTIN_TYPER[getattr(EnvironmentError.__init__, 'im_func', EnvironmentError.__init__)] = ( + rtype_EnvironmentError__init__) try: WindowsError diff --git a/rpython/rtyper/test/test_exception.py b/rpython/rtyper/test/test_exception.py --- a/rpython/rtyper/test/test_exception.py +++ b/rpython/rtyper/test/test_exception.py @@ -36,22 +36,44 @@ class TestException(BaseRtypingTest): def test_exception_with_arg(self): def g(n): - raise IOError(n, "?") + raise IOError(n) def h(n): - raise OSError(n, "?") + raise OSError(n, "?", None) + def i(n): + raise EnvironmentError(n, "?", "test") + def j(n): + raise IOError(0, "test") def f(n): try: g(n) except IOError, e: assert e.errno == 42 + assert e.strerror is None + assert e.filename is None else: assert False try: h(n) except OSError, e: assert e.errno == 42 + assert e.strerror == "?" + assert e.filename is None else: assert False + try: + i(n) + except EnvironmentError as e: + assert e.errno == 42 + assert e.strerror == "?" + assert e.filename == "test" + else: + assert False + try: + j(n) + except (IOError, OSError) as e: + assert e.errno == 0 + assert e.strerror == "test" + assert e.filename is None self.interpret(f, [42]) def test_catch_incompatible_class(self): From noreply at buildbot.pypy.org Fri Sep 5 21:41:27 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Fri, 5 Sep 2014 21:41:27 +0200 (CEST) Subject: [pypy-commit] pypy default: use IOError where appropriate in rfile Message-ID: <20140905194128.011481C0793@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r73330:351bf5327716 Date: 2014-09-05 15:39 -0400 http://bitbucket.org/pypy/pypy/changeset/351bf5327716/ Log: use IOError where appropriate in rfile diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py --- a/rpython/rlib/rfile.py +++ b/rpython/rlib/rfile.py @@ -93,7 +93,7 @@ def _error(ll_file): err = c_ferror(ll_file) c_clearerr(ll_file) - raise OSError(err, os.strerror(err)) + raise IOError(err, os.strerror(err)) def _dircheck(ll_file): @@ -104,7 +104,7 @@ else: if stat.S_ISDIR(st[0]): err = errno.EISDIR - raise OSError(err, os.strerror(err)) + raise IOError(err, os.strerror(err)) def _sanitize_mode(mode): @@ -136,7 +136,7 @@ ll_file = c_fopen(ll_name, ll_mode) if not ll_file: errno = rposix.get_errno() - raise OSError(errno, os.strerror(errno)) + raise IOError(errno, os.strerror(errno)) finally: lltype.free(ll_mode, flavor='raw') finally: @@ -223,7 +223,7 @@ res = do_close(ll_file) if res == -1: errno = rposix.get_errno() - raise OSError(errno, os.strerror(errno)) + raise IOError(errno, os.strerror(errno)) return res def _check_closed(self): @@ -341,7 +341,7 @@ bytes = c_fwrite(ll_value, 1, length, self._ll_file) if bytes != length: errno = rposix.get_errno() - raise OSError(errno, os.strerror(errno)) + raise IOError(errno, os.strerror(errno)) finally: rffi.free_nonmovingbuffer(value, ll_value) @@ -350,7 +350,7 @@ res = c_fflush(self._ll_file) if res != 0: errno = rposix.get_errno() - raise OSError(errno, os.strerror(errno)) + raise IOError(errno, os.strerror(errno)) def truncate(self, arg=-1): self._check_closed() @@ -360,21 +360,21 @@ res = c_ftruncate(self.fileno(), arg) if res == -1: errno = rposix.get_errno() - raise OSError(errno, os.strerror(errno)) + raise IOError(errno, os.strerror(errno)) def seek(self, pos, whence=0): self._check_closed() res = c_fseek(self._ll_file, pos, whence) if res == -1: errno = rposix.get_errno() - raise OSError(errno, os.strerror(errno)) + raise IOError(errno, os.strerror(errno)) def tell(self): self._check_closed() res = intmask(c_ftell(self._ll_file)) if res == -1: errno = rposix.get_errno() - raise OSError(errno, os.strerror(errno)) + raise IOError(errno, os.strerror(errno)) return res def fileno(self): diff --git a/rpython/rlib/test/test_rfile.py b/rpython/rlib/test/test_rfile.py --- a/rpython/rlib/test/test_rfile.py +++ b/rpython/rlib/test/test_rfile.py @@ -25,61 +25,67 @@ f.close() f() + assert open(fname, "r").read() == "dupa" self.interpret(f, []) assert open(fname, "r").read() == "dupa" def test_open_errors(self): - def f(exc): - def g(run): - try: - open('zzz', 'badmode') - except ValueError: - pass + def f(run): + try: + open('zzz', 'badmode') + except ValueError: + pass + else: + assert False + + try: + open('zzz') + except IOError as e: + assert e.errno == errno.ENOENT + else: + assert False + + try: + open('.') + except IOError as e: + if os.name == 'posix': + assert e.errno == errno.EISDIR else: - assert False + assert e.errno == errno.EACCES + else: + assert False - try: - open('zzz') - except exc as e: - assert e.errno == errno.ENOENT - else: - assert False + try: + os.fdopen(42, "badmode") + except ValueError: + pass + else: + assert False - try: - open('.') - except exc as e: - if os.name == 'posix': + try: + fd = os.open('.', os.O_RDONLY, 0777) + except OSError as e: + assert os.name == 'nt' and e.errno == errno.EACCES + else: + assert os.name != 'nt' + if run: + try: + os.fdopen(fd) + except IOError as e: assert e.errno == errno.EISDIR else: - assert e.errno == errno.EACCES - else: - assert False + assert False + os.close(fd) - try: - os.fdopen(42, "badmode") - except ValueError: - pass - else: - assert False + try: + os.fdopen(12345) + except OSError as e: + assert e.errno == errno.EBADF + else: + assert False - try: - fd = os.open('.', os.O_RDONLY, 0777) - except OSError as e: - assert os.name == 'nt' and e.errno == errno.EACCES - else: - assert os.name != 'nt' - if run: - try: - os.fdopen(fd) - except exc as e: - assert e.errno == errno.EISDIR - else: - assert False - os.close(fd) - return g - - f(IOError)(sys.version_info >= (2, 7, 9)) - self.interpret(f(OSError), [True]) + f(sys.version_info >= (2, 7, 9)) + self.interpret(f, [True]) @py.test.mark.skipif("sys.platform == 'win32'") # http://msdn.microsoft.com/en-us/library/86cebhfs.aspx @@ -120,6 +126,12 @@ def f(): f = open(fname, "w") + try: + f.read() + except IOError as e: + pass + else: + assert False f.write("dupa\x00dupb") f.close() for mode in ['r', 'U']: @@ -162,6 +174,7 @@ assert d == "a" assert e == "" + f() self.interpret(f, []) def test_seek(self): @@ -172,6 +185,12 @@ f.write("xxx") f.seek(0) assert f.read() == "xxx" + try: + f.seek(0, 42) + except IOError as e: + assert e.errno == errno.EINVAL + else: + assert False f.close() f() @@ -214,6 +233,8 @@ finally: f.close() + res = f() + assert res > 2 res = self.interpret(f, []) assert res > 2 @@ -228,6 +249,8 @@ finally: f.close() + res = f() + assert res == 3 res = self.interpret(f, []) assert res == 3 @@ -243,6 +266,7 @@ f2.close() f.close() + f() self.interpret(f, []) def test_truncate(self): From noreply at buildbot.pypy.org Fri Sep 5 22:44:05 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Fri, 5 Sep 2014 22:44:05 +0200 (CEST) Subject: [pypy-commit] pypy default: fix raising EnvironmentError without args Message-ID: <20140905204405.C26F91C0793@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r73331:08c4d8862d38 Date: 2014-09-05 16:43 -0400 http://bitbucket.org/pypy/pypy/changeset/08c4d8862d38/ Log: fix raising EnvironmentError without args diff --git a/rpython/rtyper/rbuiltin.py b/rpython/rtyper/rbuiltin.py --- a/rpython/rtyper/rbuiltin.py +++ b/rpython/rtyper/rbuiltin.py @@ -267,12 +267,12 @@ def rtype_EnvironmentError__init__(hop): hop.exception_cannot_occur() - if hop.nb_args <= 1: - raise TyperError("EnvironmentError() should be called with " - "at least one argument") v_self = hop.args_v[0] r_self = hop.args_r[0] - v_errno = hop.inputarg(lltype.Signed, arg=1) + if hop.nb_args >= 2: + v_errno = hop.inputarg(lltype.Signed, arg=1) + else: + v_errno = hop.inputconst(lltype.Signed, 0) r_self.setfield(v_self, 'errno', v_errno, hop.llops) if hop.nb_args >= 3: v_strerror = hop.inputarg(rstr.string_repr, arg=2) diff --git a/rpython/rtyper/test/test_exception.py b/rpython/rtyper/test/test_exception.py --- a/rpython/rtyper/test/test_exception.py +++ b/rpython/rtyper/test/test_exception.py @@ -43,6 +43,8 @@ raise EnvironmentError(n, "?", "test") def j(n): raise IOError(0, "test") + def k(n): + raise OSError def f(n): try: g(n) @@ -74,6 +76,12 @@ assert e.errno == 0 assert e.strerror == "test" assert e.filename is None + try: + k(n) + except EnvironmentError as e: + assert e.errno == 0 + assert e.strerror is None + assert e.filename is None self.interpret(f, [42]) def test_catch_incompatible_class(self): From noreply at buildbot.pypy.org Fri Sep 5 23:10:10 2014 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 5 Sep 2014 23:10:10 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: write a test showing the problem Message-ID: <20140905211010.8CC211C0793@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73332:189f50ca3955 Date: 2014-09-05 15:09 -0600 http://bitbucket.org/pypy/pypy/changeset/189f50ca3955/ Log: write a test showing the problem diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py --- a/rpython/memory/gctransform/framework.py +++ b/rpython/memory/gctransform/framework.py @@ -1180,14 +1180,26 @@ def gen_zero_gc_pointers(self, TYPE, v, llops, previous_steps=None, everything=False): + if previous_steps is None: + previous_steps = [] if isinstance(TYPE, lltype.Struct): for name in TYPE._names: FIELD = getattr(TYPE, name) + c_name = rmodel.inputconst(lltype.Void, name) + if isinstance(FIELD, lltype.Struct): + # parent + self.gen_zero_gc_pointers(FIELD, v, llops, + previous_steps + [c_name], + everything=everything) + continue if ((isinstance(FIELD, lltype.Ptr) and FIELD._needsgc()) or everything): - c_name = rmodel.inputconst(lltype.Void, name) c_null = rmodel.inputconst(FIELD, FIELD._defl()) - llops.genop('bare_setfield', [v, c_name, c_null]) + if previous_steps: + llops.genop('bare_setinteriorfield', + [v] + previous_steps + [c_name, c_null]) + else: + llops.genop('bare_setfield', [v, c_name, c_null]) elif (isinstance(FIELD, lltype.Array) and isinstance(FIELD.OF, lltype.Ptr) and FIELD.OF._needsgc()): xxx diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py --- a/rpython/translator/c/test/test_newgc.py +++ b/rpython/translator/c/test/test_newgc.py @@ -1194,13 +1194,40 @@ def test_gcflag_extra(self): self.run("gcflag_extra") + def define_zeroing_class_works(self): + class A(object): + def __init__(self, x): + self.x = x + + class BABA(A): + def __init__(self, y): + self.y = y + + def fn(x): + if x > 3: + a = BABA(str(x)) + else: + a = A(x) + assert not a.y + return 0 + + return fn + + def test_zeroing_class_works(self): + self.run("zeroing_class_works", 13) + def define_check_zero_works(self): S = lltype.GcStruct("s", ('x', lltype.Signed)) + S2 = lltype.GcStruct("s2", ('parent', + lltype.Struct("s1", ("x", lltype.Signed))), + ('y', lltype.Signed)) A = lltype.GcArray(lltype.Signed) def fn(): s = lltype.malloc(S, zero=True) assert s.x == 0 + s2 = lltype.malloc(S2, zero=True) + assert s2.parent.x == 0 a = lltype.malloc(A, 3, zero=True) assert a[2] == 0 return 0 From noreply at buildbot.pypy.org Fri Sep 5 23:13:07 2014 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 5 Sep 2014 23:13:07 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: the test is bogus, fix seems to be ok Message-ID: <20140905211307.30F121C0793@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73333:ef37f9ae099a Date: 2014-09-05 15:12 -0600 http://bitbucket.org/pypy/pypy/changeset/ef37f9ae099a/ Log: the test is bogus, fix seems to be ok diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py --- a/rpython/translator/c/test/test_newgc.py +++ b/rpython/translator/c/test/test_newgc.py @@ -1194,28 +1194,6 @@ def test_gcflag_extra(self): self.run("gcflag_extra") - def define_zeroing_class_works(self): - class A(object): - def __init__(self, x): - self.x = x - - class BABA(A): - def __init__(self, y): - self.y = y - - def fn(x): - if x > 3: - a = BABA(str(x)) - else: - a = A(x) - assert not a.y - return 0 - - return fn - - def test_zeroing_class_works(self): - self.run("zeroing_class_works", 13) - def define_check_zero_works(self): S = lltype.GcStruct("s", ('x', lltype.Signed)) S2 = lltype.GcStruct("s2", ('parent', From noreply at buildbot.pypy.org Sat Sep 6 10:11:01 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 6 Sep 2014 10:11:01 +0200 (CEST) Subject: [pypy-commit] pypy cpybug-seq-radd-rmul: issue1861 Message-ID: <20140906081101.7664C1C305D@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cpybug-seq-radd-rmul Changeset: r73334:d3f11fcdfbd6 Date: 2014-09-06 10:09 +0200 http://bitbucket.org/pypy/pypy/changeset/d3f11fcdfbd6/ Log: issue1861 From noreply at buildbot.pypy.org Sat Sep 6 10:11:02 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 6 Sep 2014 10:11:02 +0200 (CEST) Subject: [pypy-commit] pypy cpybug-seq-radd-rmul: Trying out a bug compatibility fix... Message-ID: <20140906081102.EAD751C305D@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cpybug-seq-radd-rmul Changeset: r73335:2b4dc5840823 Date: 2014-09-06 10:10 +0200 http://bitbucket.org/pypy/pypy/changeset/2b4dc5840823/ Log: Trying out a bug compatibility fix... diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py --- a/pypy/objspace/descroperation.py +++ b/pypy/objspace/descroperation.py @@ -671,6 +671,7 @@ left, right = specialnames errormsg = "unsupported operand type(s) for %s: '%%N' and '%%N'" % ( symbol.replace('%', '%%'),) + seq_bug_compat = (symbol == '+' or symbol == '*') def binop_impl(space, w_obj1, w_obj2): w_typ1 = space.type(w_obj1) @@ -686,20 +687,16 @@ # __xxx__ and __rxxx__ methods where found by identity. # Note that space.is_w() is potentially not happy if one of them # is None... - if w_left_src is not w_right_src: # XXX - # -- cpython bug compatibility: see objspace/std/test/ - # -- test_unicodeobject.test_str_unicode_concat_overrides. - # -- The following handles "unicode + string subclass" by - # -- pretending that the unicode is a superclass of the - # -- string, thus giving priority to the string subclass' - # -- __radd__() method. The case "string + unicode subclass" - # -- is handled directly by add__String_Unicode(). - if symbol == '+' and space.is_w(w_typ1, space.w_unicode): - w_typ1 = space.w_basestring - # -- end of bug compatibility - if space.is_true(space.issubtype(w_typ2, w_typ1)): - if (w_left_src and w_right_src and - not space.abstract_issubclass_w(w_left_src, w_right_src) and + if w_right_src and (w_left_src is not w_right_src) and w_left_src: + # 'seq_bug_compat' is for cpython bug-to-bug compatibility: + # see objspace/std/test/test_unicodeobject.*concat_overrides + # and objspace/test/test_descrobject.*rmul_overrides. + # For cases like "unicode + string subclass". + if ((seq_bug_compat and w_typ1.flag_sequence_bug_compat + and not w_typ2.flag_sequence_bug_compat) + # the non-bug-compat part is the following check: + or space.is_true(space.issubtype(w_typ2, w_typ1))): + if (not space.abstract_issubclass_w(w_left_src, w_right_src) and not space.abstract_issubclass_w(w_typ1, w_right_src)): w_obj1, w_obj2 = w_obj2, w_obj1 w_left_impl, w_right_impl = w_right_impl, w_left_impl diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py --- a/pypy/objspace/std/bytearrayobject.py +++ b/pypy/objspace/std/bytearrayobject.py @@ -1131,6 +1131,7 @@ reverse = interp2app(W_BytearrayObject.descr_reverse, doc=BytearrayDocstrings.reverse.__doc__), ) +W_BytearrayObject.typedef.flag_sequence_bug_compat = True init_signature = Signature(['source', 'encoding', 'errors'], None, None) init_defaults = [None, None, None] diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py --- a/pypy/objspace/std/bytesobject.py +++ b/pypy/objspace/std/bytesobject.py @@ -951,6 +951,7 @@ _formatter_field_name_split = interp2app(W_BytesObject.descr_formatter_field_name_split), ) +W_BytesObject.typedef.flag_sequence_bug_compat = True def string_escape_encode(s, quote): diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py --- a/pypy/objspace/std/listobject.py +++ b/pypy/objspace/std/listobject.py @@ -1874,3 +1874,4 @@ insert = interp2app(W_ListObject.descr_insert), remove = interp2app(W_ListObject.descr_remove), ) +W_ListObject.typedef.flag_sequence_bug_compat = True diff --git a/pypy/objspace/std/stdtypedef.py b/pypy/objspace/std/stdtypedef.py --- a/pypy/objspace/std/stdtypedef.py +++ b/pypy/objspace/std/stdtypedef.py @@ -93,6 +93,8 @@ overridetypedef=overridetypedef) if typedef is not overridetypedef: w_type.w_doc = space.wrap(typedef.doc) + if hasattr(typedef, 'flag_sequence_bug_compat'): + w_type.flag_sequence_bug_compat = typedef.flag_sequence_bug_compat w_type.lazyloaders = lazyloaders return w_type diff --git a/pypy/objspace/std/tupleobject.py b/pypy/objspace/std/tupleobject.py --- a/pypy/objspace/std/tupleobject.py +++ b/pypy/objspace/std/tupleobject.py @@ -244,6 +244,7 @@ count = interp2app(W_AbstractTupleObject.descr_count), index = interp2app(W_AbstractTupleObject.descr_index) ) +W_AbstractTupleObject.typedef.flag_sequence_bug_compat = True class W_TupleObject(W_AbstractTupleObject): diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py --- a/pypy/objspace/std/typeobject.py +++ b/pypy/objspace/std/typeobject.py @@ -67,6 +67,7 @@ _immutable_fields_ = ["flag_heaptype", "flag_cpytype", "flag_abstract?", + "flag_sequence_bug_compat", 'needsdel', 'weakrefable', 'hasdict', @@ -104,6 +105,7 @@ w_self.flag_heaptype = False w_self.flag_cpytype = False w_self.flag_abstract = False + w_self.flag_sequence_bug_compat = False w_self.instancetypedef = overridetypedef if overridetypedef is not None: diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -1068,6 +1068,7 @@ _formatter_field_name_split = interp2app(W_UnicodeObject.descr_formatter_field_name_split), ) +W_UnicodeObject.typedef.flag_sequence_bug_compat = True def _create_list_from_unicode(value): diff --git a/pypy/objspace/test/test_descroperation.py b/pypy/objspace/test/test_descroperation.py --- a/pypy/objspace/test/test_descroperation.py +++ b/pypy/objspace/test/test_descroperation.py @@ -734,6 +734,30 @@ assert X() == 'hello' + def test_sequence_rmul_overrides(self): + class oops(object): + def __rmul__(self, other): + return 42 + def __index__(self): + return 3 + assert '2' * oops() == 42 + assert [2] * oops() == 42 + assert (2,) * oops() == 42 + assert u'2' * oops() == 42 + assert bytearray('2') * oops() == 42 + assert 1000 * oops() == 42 + assert '2'.__mul__(oops()) == '222' + + def test_sequence_radd_overrides(self): + class A1(list): + pass + class A2(list): + def __radd__(self, other): + return 42 + assert [2] + A1([3]) == [2, 3] + assert type([2] + A1([3])) is list + assert [2] + A2([3]) == 42 + class AppTestWithBuiltinShortcut(AppTest_Descroperation): spaceconfig = {'objspace.std.builtinshortcut': True} From noreply at buildbot.pypy.org Sat Sep 6 10:28:52 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 6 Sep 2014 10:28:52 +0200 (CEST) Subject: [pypy-commit] pypy cpybug-seq-radd-rmul: passing test: the same with an old-style class Message-ID: <20140906082852.9453D1C305D@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cpybug-seq-radd-rmul Changeset: r73336:f7e76c39d341 Date: 2014-09-06 10:28 +0200 http://bitbucket.org/pypy/pypy/changeset/f7e76c39d341/ Log: passing test: the same with an old-style class diff --git a/pypy/objspace/test/test_descroperation.py b/pypy/objspace/test/test_descroperation.py --- a/pypy/objspace/test/test_descroperation.py +++ b/pypy/objspace/test/test_descroperation.py @@ -748,6 +748,20 @@ assert 1000 * oops() == 42 assert '2'.__mul__(oops()) == '222' + def test_sequence_rmul_overrides_oldstyle(self): + class oops: + def __rmul__(self, other): + return 42 + def __index__(self): + return 3 + assert '2' * oops() == 42 + assert [2] * oops() == 42 + assert (2,) * oops() == 42 + assert u'2' * oops() == 42 + assert bytearray('2') * oops() == 42 + assert 1000 * oops() == 42 + assert '2'.__mul__(oops()) == '222' + def test_sequence_radd_overrides(self): class A1(list): pass From noreply at buildbot.pypy.org Sat Sep 6 12:19:57 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 6 Sep 2014 12:19:57 +0200 (CEST) Subject: [pypy-commit] pypy cpybug-seq-radd-rmul: ready to merge Message-ID: <20140906101957.627E41C030E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: cpybug-seq-radd-rmul Changeset: r73337:98e74f0c90f3 Date: 2014-09-06 12:17 +0200 http://bitbucket.org/pypy/pypy/changeset/98e74f0c90f3/ Log: ready to merge From noreply at buildbot.pypy.org Sat Sep 6 12:19:58 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 6 Sep 2014 12:19:58 +0200 (CEST) Subject: [pypy-commit] pypy default: issue1861: trying to remove the "unicode + string_subclass" hack and Message-ID: <20140906101958.92CAE1C030E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73338:6a380bc4ae37 Date: 2014-09-06 12:19 +0200 http://bitbucket.org/pypy/pypy/changeset/6a380bc4ae37/ Log: issue1861: trying to remove the "unicode + string_subclass" hack and replace it with a more general hack, based on detecting that the left-hand-side argument is a "sequence". This is meant to follow CPython, which uses sq_xxx methods which mess up with the implementation of "+" and "*". diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py --- a/pypy/objspace/descroperation.py +++ b/pypy/objspace/descroperation.py @@ -671,6 +671,7 @@ left, right = specialnames errormsg = "unsupported operand type(s) for %s: '%%N' and '%%N'" % ( symbol.replace('%', '%%'),) + seq_bug_compat = (symbol == '+' or symbol == '*') def binop_impl(space, w_obj1, w_obj2): w_typ1 = space.type(w_obj1) @@ -686,20 +687,16 @@ # __xxx__ and __rxxx__ methods where found by identity. # Note that space.is_w() is potentially not happy if one of them # is None... - if w_left_src is not w_right_src: # XXX - # -- cpython bug compatibility: see objspace/std/test/ - # -- test_unicodeobject.test_str_unicode_concat_overrides. - # -- The following handles "unicode + string subclass" by - # -- pretending that the unicode is a superclass of the - # -- string, thus giving priority to the string subclass' - # -- __radd__() method. The case "string + unicode subclass" - # -- is handled directly by add__String_Unicode(). - if symbol == '+' and space.is_w(w_typ1, space.w_unicode): - w_typ1 = space.w_basestring - # -- end of bug compatibility - if space.is_true(space.issubtype(w_typ2, w_typ1)): - if (w_left_src and w_right_src and - not space.abstract_issubclass_w(w_left_src, w_right_src) and + if w_right_src and (w_left_src is not w_right_src) and w_left_src: + # 'seq_bug_compat' is for cpython bug-to-bug compatibility: + # see objspace/std/test/test_unicodeobject.*concat_overrides + # and objspace/test/test_descrobject.*rmul_overrides. + # For cases like "unicode + string subclass". + if ((seq_bug_compat and w_typ1.flag_sequence_bug_compat + and not w_typ2.flag_sequence_bug_compat) + # the non-bug-compat part is the following check: + or space.is_true(space.issubtype(w_typ2, w_typ1))): + if (not space.abstract_issubclass_w(w_left_src, w_right_src) and not space.abstract_issubclass_w(w_typ1, w_right_src)): w_obj1, w_obj2 = w_obj2, w_obj1 w_left_impl, w_right_impl = w_right_impl, w_left_impl diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py --- a/pypy/objspace/std/bytearrayobject.py +++ b/pypy/objspace/std/bytearrayobject.py @@ -1131,6 +1131,7 @@ reverse = interp2app(W_BytearrayObject.descr_reverse, doc=BytearrayDocstrings.reverse.__doc__), ) +W_BytearrayObject.typedef.flag_sequence_bug_compat = True init_signature = Signature(['source', 'encoding', 'errors'], None, None) init_defaults = [None, None, None] diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py --- a/pypy/objspace/std/bytesobject.py +++ b/pypy/objspace/std/bytesobject.py @@ -951,6 +951,7 @@ _formatter_field_name_split = interp2app(W_BytesObject.descr_formatter_field_name_split), ) +W_BytesObject.typedef.flag_sequence_bug_compat = True def string_escape_encode(s, quote): diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py --- a/pypy/objspace/std/listobject.py +++ b/pypy/objspace/std/listobject.py @@ -1874,3 +1874,4 @@ insert = interp2app(W_ListObject.descr_insert), remove = interp2app(W_ListObject.descr_remove), ) +W_ListObject.typedef.flag_sequence_bug_compat = True diff --git a/pypy/objspace/std/stdtypedef.py b/pypy/objspace/std/stdtypedef.py --- a/pypy/objspace/std/stdtypedef.py +++ b/pypy/objspace/std/stdtypedef.py @@ -93,6 +93,8 @@ overridetypedef=overridetypedef) if typedef is not overridetypedef: w_type.w_doc = space.wrap(typedef.doc) + if hasattr(typedef, 'flag_sequence_bug_compat'): + w_type.flag_sequence_bug_compat = typedef.flag_sequence_bug_compat w_type.lazyloaders = lazyloaders return w_type diff --git a/pypy/objspace/std/tupleobject.py b/pypy/objspace/std/tupleobject.py --- a/pypy/objspace/std/tupleobject.py +++ b/pypy/objspace/std/tupleobject.py @@ -244,6 +244,7 @@ count = interp2app(W_AbstractTupleObject.descr_count), index = interp2app(W_AbstractTupleObject.descr_index) ) +W_AbstractTupleObject.typedef.flag_sequence_bug_compat = True class W_TupleObject(W_AbstractTupleObject): diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py --- a/pypy/objspace/std/typeobject.py +++ b/pypy/objspace/std/typeobject.py @@ -67,6 +67,7 @@ _immutable_fields_ = ["flag_heaptype", "flag_cpytype", "flag_abstract?", + "flag_sequence_bug_compat", 'needsdel', 'weakrefable', 'hasdict', @@ -104,6 +105,7 @@ w_self.flag_heaptype = False w_self.flag_cpytype = False w_self.flag_abstract = False + w_self.flag_sequence_bug_compat = False w_self.instancetypedef = overridetypedef if overridetypedef is not None: diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -1068,6 +1068,7 @@ _formatter_field_name_split = interp2app(W_UnicodeObject.descr_formatter_field_name_split), ) +W_UnicodeObject.typedef.flag_sequence_bug_compat = True def _create_list_from_unicode(value): diff --git a/pypy/objspace/test/test_descroperation.py b/pypy/objspace/test/test_descroperation.py --- a/pypy/objspace/test/test_descroperation.py +++ b/pypy/objspace/test/test_descroperation.py @@ -734,6 +734,44 @@ assert X() == 'hello' + def test_sequence_rmul_overrides(self): + class oops(object): + def __rmul__(self, other): + return 42 + def __index__(self): + return 3 + assert '2' * oops() == 42 + assert [2] * oops() == 42 + assert (2,) * oops() == 42 + assert u'2' * oops() == 42 + assert bytearray('2') * oops() == 42 + assert 1000 * oops() == 42 + assert '2'.__mul__(oops()) == '222' + + def test_sequence_rmul_overrides_oldstyle(self): + class oops: + def __rmul__(self, other): + return 42 + def __index__(self): + return 3 + assert '2' * oops() == 42 + assert [2] * oops() == 42 + assert (2,) * oops() == 42 + assert u'2' * oops() == 42 + assert bytearray('2') * oops() == 42 + assert 1000 * oops() == 42 + assert '2'.__mul__(oops()) == '222' + + def test_sequence_radd_overrides(self): + class A1(list): + pass + class A2(list): + def __radd__(self, other): + return 42 + assert [2] + A1([3]) == [2, 3] + assert type([2] + A1([3])) is list + assert [2] + A2([3]) == 42 + class AppTestWithBuiltinShortcut(AppTest_Descroperation): spaceconfig = {'objspace.std.builtinshortcut': True} From noreply at buildbot.pypy.org Sat Sep 6 13:40:36 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 6 Sep 2014 13:40:36 +0200 (CEST) Subject: [pypy-commit] pypy default: issue1860: if we have a DONT_TRACE_HERE function, but it fails to trace Message-ID: <20140906114036.01D341C305D@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73339:ee3ca13580fc Date: 2014-09-06 13:39 +0200 http://bitbucket.org/pypy/pypy/changeset/ee3ca13580fc/ Log: issue1860: if we have a DONT_TRACE_HERE function, but it fails to trace the next time (e.g. too large or vable escape), then the previous logic would try to retrace again and again every single iteration. This creates a huge overhead. Instead, try to trace immediately only the first time, and then fall back to the regular counters. diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py --- a/rpython/jit/metainterp/warmstate.py +++ b/rpython/jit/metainterp/warmstate.py @@ -127,6 +127,7 @@ JC_TRACING = 0x01 JC_DONT_TRACE_HERE = 0x02 JC_TEMPORARY = 0x04 +JC_TRACING_OCCURRED= 0x08 class BaseJitCell(object): """Subclasses of BaseJitCell are used in tandem with the single @@ -160,6 +161,8 @@ JC_TRACING: we are now tracing the loop from this greenkey. We'll likely end up with a wref_procedure_token, soonish. + JC_TRACING_OCCURRED: set if JC_TRACING was set at least once. + JC_TEMPORARY: a "temporary" wref_procedure_token. It's the procedure_token of a dummy loop that simply calls back the interpreter. Used for a CALL_ASSEMBLER where the @@ -206,7 +209,7 @@ # if we have this flag, and we *had* a procedure_token but # we no longer have one, then remove me. this prevents this # JitCell from being immortal. - return self.has_seen_a_procedure_token() + return self.has_seen_a_procedure_token() # i.e. dead weakref return True # Other JitCells can be removed. # ____________________________________________________________ @@ -374,7 +377,7 @@ if cell is None: cell = JitCell(*greenargs) jitcounter.install_new_cell(hash, cell) - cell.flags |= JC_TRACING + cell.flags |= JC_TRACING | JC_TRACING_OCCURRED try: metainterp.compile_and_run_once(jitdriver_sd, *args) finally: @@ -418,9 +421,15 @@ if procedure_token is None: if cell.flags & JC_DONT_TRACE_HERE: if not cell.has_seen_a_procedure_token(): - # we're seeing a fresh JC_DONT_TRACE_HERE with no - # procedure_token. Compile now. - bound_reached(hash, cell, *args) + # A JC_DONT_TRACE_HERE, i.e. a non-inlinable function. + # If we never tried to trace it, try it now immediately. + # Otherwise, count normally. + if cell.flags & JC_TRACING_OCCURRED: + tick = jitcounter.tick(hash, increment_threshold) + else: + tick = True + if tick: + bound_reached(hash, cell, *args) return # it was an aborted compilation, or maybe a weakref that # has been freed From noreply at buildbot.pypy.org Sat Sep 6 18:07:21 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 6 Sep 2014 18:07:21 +0200 (CEST) Subject: [pypy-commit] pypy default: precision Message-ID: <20140906160721.AB8971C129D@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73340:490b99a71a59 Date: 2014-09-06 18:06 +0200 http://bitbucket.org/pypy/pypy/changeset/490b99a71a59/ Log: precision diff --git a/pypy/doc/release-2.4.0.rst b/pypy/doc/release-2.4.0.rst --- a/pypy/doc/release-2.4.0.rst +++ b/pypy/doc/release-2.4.0.rst @@ -86,7 +86,7 @@ * Move to a mixed polling and mutex GIL model that make mutli-threaded jitted code run *much* faster -* Optimize errno handling in linux +* Optimize errno handling in linux (x86 and x86-64 only) * Remove ctypes pythonapi and ctypes.PyDLL, which never worked on PyPy From noreply at buildbot.pypy.org Sat Sep 6 18:24:44 2014 From: noreply at buildbot.pypy.org (techtonik) Date: Sat, 6 Sep 2014 18:24:44 +0200 (CEST) Subject: [pypy-commit] pypy techtonik/release240rst-fix-indentation-and-some-o-1410020236989: release-2.4.0.rst fix indentation and some obvious mistakes Message-ID: <20140906162444.5D7D31C129D@cobra.cs.uni-duesseldorf.de> Author: anatoly techtonik Branch: techtonik/release240rst-fix-indentation-and-some-o-1410020236989 Changeset: r73341:fc2906300cb4 Date: 2014-09-06 16:20 +0000 http://bitbucket.org/pypy/pypy/changeset/fc2906300cb4/ Log: release-2.4.0.rst fix indentation and some obvious mistakes diff --git a/pypy/doc/release-2.4.0.rst b/pypy/doc/release-2.4.0.rst --- a/pypy/doc/release-2.4.0.rst +++ b/pypy/doc/release-2.4.0.rst @@ -11,21 +11,20 @@ We would like to thank our donors for the continued support of the PyPy project, and for those who donate to our three sub-projects. -We've shown quite a bit of progress -but we're slowly running out of funds. +We've shown quite a bit of progress, but we're slowly running out of funds. Please consider donating more, or even better convince your employer to donate, so we can finish those projects! We would like to also point out that in September, `the Python Software Foundation`_ will `match funds`_ for any donations up to $10k! The three sub-projects are: -* `Py3k`_ (supporting Python 3.x): We have released a Python 3.2.5 compatable version - we call PyPy3 2.3.1, and are working toward a Python 3.3 compatable version +* `Py3k`_ (supporting Python 3.x): We have released a Python 3.2.5 compatible version + we call PyPy3 2.3.1, and are working toward a Python 3.3 compatible version -* `STM`_ (software transactional memory): We have release a first working version, and -continue to try out new promising paths of acheiving a fast multithreaded python +* `STM`_ (software transactional memory): We have released a first working version, + and continue to try out new promising paths of achieving a fast multithreaded Python * `NumPy`_ which requires installation of our fork of upstream numpy, -available `on bitbucket`_ + available `on bitbucket`_ .. _`Py3k`: http://pypy.org/py3donate.html .. _`STM`: http://pypy.org/tmdonate2.html @@ -55,11 +54,11 @@ Highlights ========== -Benchmarks improved after internal improvements in string and +Benchmarks improved after internal enhancements in string and bytearray handling, and a major rewrite of the GIL handling. This means that external calls are now a lot faster, especially the CFFI ones. It also -means that a lot of corner cases when handling strings or bytearrays have -better performance. +means better performance in a lot of corner cases with handling strings or +bytearrays. PyPy now uses Python 2.7.8 standard library. @@ -76,8 +75,8 @@ * Reduced internal copying of bytearray operations * Tweak the internal structure of StringBuilder to speed up large string -handling, which becomes advantageous on large programs at the cost of slightly -slower small *benchmark* type programs. + handling, which becomes advantageous on large programs at the cost of slightly + slower small *benchmark* type programs. * Boost performance of thread-local variables in both unjitted and jitted code, this mostly affects errno handling on linux, which makes external calls From noreply at buildbot.pypy.org Sat Sep 6 18:24:45 2014 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Sat, 6 Sep 2014 18:24:45 +0200 (CEST) Subject: [pypy-commit] pypy default: Merged in techtonik/pypy-1/techtonik/release240rst-fix-indentation-and-some-o-1410020236989 (pull request #277) Message-ID: <20140906162445.8F3BA1C129D@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: Changeset: r73342:4701289f5ac2 Date: 2014-09-06 09:24 -0700 http://bitbucket.org/pypy/pypy/changeset/4701289f5ac2/ Log: Merged in techtonik/pypy-1/techtonik/release240rst-fix-indentation- and-some-o-1410020236989 (pull request #277) release-2.4.0.rst fix indentation and some obvious mistakes diff --git a/pypy/doc/release-2.4.0.rst b/pypy/doc/release-2.4.0.rst --- a/pypy/doc/release-2.4.0.rst +++ b/pypy/doc/release-2.4.0.rst @@ -11,21 +11,20 @@ We would like to thank our donors for the continued support of the PyPy project, and for those who donate to our three sub-projects. -We've shown quite a bit of progress -but we're slowly running out of funds. +We've shown quite a bit of progress, but we're slowly running out of funds. Please consider donating more, or even better convince your employer to donate, so we can finish those projects! We would like to also point out that in September, `the Python Software Foundation`_ will `match funds`_ for any donations up to $10k! The three sub-projects are: -* `Py3k`_ (supporting Python 3.x): We have released a Python 3.2.5 compatable version - we call PyPy3 2.3.1, and are working toward a Python 3.3 compatable version +* `Py3k`_ (supporting Python 3.x): We have released a Python 3.2.5 compatible version + we call PyPy3 2.3.1, and are working toward a Python 3.3 compatible version -* `STM`_ (software transactional memory): We have release a first working version, and -continue to try out new promising paths of acheiving a fast multithreaded python +* `STM`_ (software transactional memory): We have released a first working version, + and continue to try out new promising paths of achieving a fast multithreaded Python * `NumPy`_ which requires installation of our fork of upstream numpy, -available `on bitbucket`_ + available `on bitbucket`_ .. _`Py3k`: http://pypy.org/py3donate.html .. _`STM`: http://pypy.org/tmdonate2.html @@ -55,11 +54,11 @@ Highlights ========== -Benchmarks improved after internal improvements in string and +Benchmarks improved after internal enhancements in string and bytearray handling, and a major rewrite of the GIL handling. This means that external calls are now a lot faster, especially the CFFI ones. It also -means that a lot of corner cases when handling strings or bytearrays have -better performance. +means better performance in a lot of corner cases with handling strings or +bytearrays. PyPy now uses Python 2.7.8 standard library. @@ -76,8 +75,8 @@ * Reduced internal copying of bytearray operations * Tweak the internal structure of StringBuilder to speed up large string -handling, which becomes advantageous on large programs at the cost of slightly -slower small *benchmark* type programs. + handling, which becomes advantageous on large programs at the cost of slightly + slower small *benchmark* type programs. * Boost performance of thread-local variables in both unjitted and jitted code, this mostly affects errno handling on linux, which makes external calls From noreply at buildbot.pypy.org Sat Sep 6 21:15:46 2014 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 6 Sep 2014 21:15:46 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: implement the arena debugging and fix for asmgcc Message-ID: <20140906191546.5E5E01C305D@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73343:3fc11182cb52 Date: 2014-09-06 13:15 -0600 http://bitbucket.org/pypy/pypy/changeset/3fc11182cb52/ Log: implement the arena debugging and fix for asmgcc diff --git a/rpython/jit/backend/llsupport/jitframe.py b/rpython/jit/backend/llsupport/jitframe.py --- a/rpython/jit/backend/llsupport/jitframe.py +++ b/rpython/jit/backend/llsupport/jitframe.py @@ -47,6 +47,7 @@ def jitframe_allocate(frame_info): frame = lltype.malloc(JITFRAME, frame_info.jfi_frame_depth) frame.jf_frame_info = frame_info + frame.jf_extra_stack_depth = 0 return frame def jitframe_resolve(frame): diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -6,9 +6,8 @@ '4M'. Small values (like 1 or 1KB) are useful for debugging. - PYPY_GC_NURSERY_CLEANUP The interval at which nursery is cleaned up. Must - be smaller than the nursery size and bigger than the - biggest object we can allotate in the nursery. + PYPY_GC_NURSERY_DEBUG If set to non-zero, will fill nursery with garbage, + to help debugging. PYPY_GC_INCREMENT_STEP The size of memory marked during the marking step. Default is size of nursery * 2. If you mark it too high @@ -358,6 +357,7 @@ if not self.read_from_env: self.allocate_nursery() self.gc_increment_step = self.nursery_size * 4 + self.gc_nursery_debug = False else: # defaultsize = self.nursery_size @@ -414,6 +414,9 @@ else: self.gc_increment_step = newsize * 4 # + nursery_debug = env.read_uint_from_env('PYPY_GC_NURSERY_DEBUG') + if nursery_debug > 0: + self.gc_nursery_debug = True self.minor_collection() # to empty the nursery llarena.arena_free(self.nursery) self.nursery_size = newsize @@ -1390,7 +1393,10 @@ # # All live nursery objects are out, and the rest dies. Fill # the nursery up to the cleanup point with zeros - llarena.arena_reset(self.nursery, self.nursery_size, 0) + if self.gc_nursery_debug: + llarena.arena_reset(self.nursery, self.nursery_size, 3) + else: + llarena.arena_reset(self.nursery, self.nursery_size, 0) self.debug_rotate_nursery() self.nursery_free = self.nursery self.nursery_top = self.nursery + self.nursery_size diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py --- a/rpython/rtyper/llinterp.py +++ b/rpython/rtyper/llinterp.py @@ -953,6 +953,9 @@ checkadr(toaddr) llmemory.raw_memcopy(fromaddr, toaddr, size) + def op_raw_memset(self, addr, byte, size): + raise NotImplementedError + op_raw_memmove = op_raw_memcopy # this is essentially the same here def op_raw_load(self, RESTYPE, addr, offset): diff --git a/rpython/rtyper/lltypesystem/llarena.py b/rpython/rtyper/lltypesystem/llarena.py --- a/rpython/rtyper/lltypesystem/llarena.py +++ b/rpython/rtyper/lltypesystem/llarena.py @@ -1,6 +1,7 @@ import array, weakref from rpython.rtyper.lltypesystem import llmemory from rpython.rlib.rarithmetic import is_valid_int +from rpython.rtyper.lltypesystem.lloperation import llop # An "arena" is a large area of memory which can hold a number of @@ -53,7 +54,7 @@ del self.objectptrs[offset] del self.objectsizes[offset] obj._free() - if zero: + if zero and zero != 3: initialbyte = "0" else: initialbyte = "#" @@ -346,6 +347,7 @@ * 0: don't fill the area with zeroes * 1: clear, optimized for a very large area of memory * 2: clear, optimized for a small or medium area of memory + * 3: fill with garbage """ arena_addr = getfakearenaaddress(arena_addr) arena_addr.arena.reset(zero, arena_addr.offset, size) @@ -518,6 +520,8 @@ if zero: if zero == 1: clear_large_memory_chunk(arena_addr, size) + elif zero == 3: + llop.raw_memset(lltype.Void, arena_addr, ord('#'), size) else: llmemory.raw_memclear(arena_addr, size) llimpl_arena_reset._always_inline_ = True diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py --- a/rpython/rtyper/lltypesystem/lloperation.py +++ b/rpython/rtyper/lltypesystem/lloperation.py @@ -406,6 +406,7 @@ 'raw_malloc_usage': LLOp(sideeffects=False), 'raw_free': LLOp(), 'raw_memclear': LLOp(), + 'raw_memset': LLOp(), 'raw_memcopy': LLOp(), 'raw_memmove': LLOp(), 'raw_load': LLOp(sideeffects=False, canrun=True), diff --git a/rpython/translator/c/src/mem.h b/rpython/translator/c/src/mem.h --- a/rpython/translator/c/src/mem.h +++ b/rpython/translator/c/src/mem.h @@ -18,6 +18,7 @@ #define OP_RAW_FREE(p, r) PyObject_Free(p); COUNT_FREE; #define OP_RAW_MEMCLEAR(p, size, r) memset((void*)p, 0, size) +#define OP_RAW_MEMSET(p, byte, size, r) memset((void*)p, byte, size) #define OP_RAW_MALLOC_USAGE(size, r) r = size From noreply at buildbot.pypy.org Sat Sep 6 22:28:29 2014 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 6 Sep 2014 22:28:29 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: damn Message-ID: <20140906202829.C3E591C305D@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73344:511e2f195b48 Date: 2014-09-06 14:28 -0600 http://bitbucket.org/pypy/pypy/changeset/511e2f195b48/ Log: damn diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -417,6 +417,8 @@ nursery_debug = env.read_uint_from_env('PYPY_GC_NURSERY_DEBUG') if nursery_debug > 0: self.gc_nursery_debug = True + else: + self.gc_nursery_debug = False self.minor_collection() # to empty the nursery llarena.arena_free(self.nursery) self.nursery_size = newsize From noreply at buildbot.pypy.org Sat Sep 6 22:45:06 2014 From: noreply at buildbot.pypy.org (waedt) Date: Sat, 6 Sep 2014 22:45:06 +0200 (CEST) Subject: [pypy-commit] pypy utf8-unicode2: Remove this test again. It passes translated, but times out untranslated Message-ID: <20140906204506.6AEB91C305D@cobra.cs.uni-duesseldorf.de> Author: Tyler Wade Branch: utf8-unicode2 Changeset: r73345:c12f7021bd3a Date: 2014-08-14 02:52 -0500 http://bitbucket.org/pypy/pypy/changeset/c12f7021bd3a/ Log: Remove this test again. It passes translated, but times out untranslated diff --git a/pypy/module/_io/test/test_textio.py b/pypy/module/_io/test/test_textio.py --- a/pypy/module/_io/test/test_textio.py +++ b/pypy/module/_io/test/test_textio.py @@ -281,58 +281,6 @@ t.read() == u'a' - def test_interrupted_write(self): - import _io - import os - import threading - import sys - import signal - import errno - - item = u'xy' - bytes = 'xy' - - signal.signal(signal.SIGALRM, lambda x, y: 1 // 0) - - read_results = [] - def _read(): - s = os.read(r, 1) - read_results.append(s) - t = threading.Thread(target=_read) - t.daemon = True - r, w = os.pipe() - try: - wio = _io.open(w, mode='w', encoding="ascii") - t.start() - signal.alarm(1) - # Fill the pipe enough that the write will be blocking. - # It will be interrupted by the timer armed above. Since the - # other thread has read one byte, the low-level write will - # return with a successful (partial) result rather than an EINTR. - # The buffered IO layer must check for pending signal - # handlers, which in this case will invoke alarm_interrupt(). - - raises(ZeroDivisionError, wio.write, item * (4194305 // len(item) + 1)) - - t.join() - # We got one byte, get another one and check that it isn't a - # repeat of the first one. - read_results.append(os.read(r, 1)) - - assert read_results == [bytes[0:1], bytes[1:2]] - finally: - os.close(w) - os.close(r) - # This is deliberate. If we didn't close the file descriptor - # before closing wio, wio would try to flush its internal - # buffer, and block again. - try: - wio.close() - except IOError as e: - if e.errno != errno.EBADF: - raise - - class AppTestIncrementalNewlineDecoder: def test_newline_decoder(self): import _io From noreply at buildbot.pypy.org Sat Sep 6 22:45:07 2014 From: noreply at buildbot.pypy.org (waedt) Date: Sat, 6 Sep 2014 22:45:07 +0200 (CEST) Subject: [pypy-commit] pypy utf8-unicode2: Utf8Iterator: Don't recalculate byte_pos until its needed Message-ID: <20140906204507.A362A1C305D@cobra.cs.uni-duesseldorf.de> Author: Tyler Wade Branch: utf8-unicode2 Changeset: r73346:c294b2a1b07c Date: 2014-08-16 21:25 -0500 http://bitbucket.org/pypy/pypy/changeset/c294b2a1b07c/ Log: Utf8Iterator: Don't recalculate byte_pos until its needed diff --git a/pypy/interpreter/utf8.py b/pypy/interpreter/utf8.py --- a/pypy/interpreter/utf8.py +++ b/pypy/interpreter/utf8.py @@ -814,18 +814,30 @@ self._str = str self._pos = start + self._calculated_pos = start self._byte_pos = str.index_of_char(start) - - self._calc_current() + self._current = utf8ord_bytes(self._str.bytes, self._byte_pos) def _calc_current(self): if self._pos >= len(self._str) or self._pos < 0: raise IndexError() - else: + + count = self._pos - self._calculated_pos + self._calculated_pos = self._pos + if count > 0: + while count != 0: + self._byte_pos = self._str.next_char(self._byte_pos) + count -= 1 + self._current = utf8ord_bytes(self._str.bytes, self._byte_pos) + + elif count < 0: + while count < 0: + self._byte_pos = self._str.prev_char(self._byte_pos) + count += 1 self._current = utf8ord_bytes(self._str.bytes, self._byte_pos) def current(self): - if self._current == -1: + if self._calculated_pos != self._pos: self._calc_current() return self._current @@ -833,32 +845,12 @@ return self._pos def byte_pos(self): + if self._calculated_pos != self._pos: + self._calc_current() return self._byte_pos def move(self, count): - # TODO: As an optimization, we could delay moving byte_pos until we - # _calc_current - if count > 0: - self._pos += count - - if self._pos < 0: - self._byte_pos = 0 - else: - while count != 0: - self._byte_pos = self._str.next_char(self._byte_pos) - count -= 1 - self._current = -1 - - elif count < 0: - self._pos += count - - if self._pos < 0: - self._byte_pos = 0 - else: - while count < 0: - self._byte_pos = self._str.prev_char(self._byte_pos) - count += 1 - self._current = -1 + self._pos += count def finished(self): return self._pos >= len(self._str) @@ -866,6 +858,7 @@ def copy(self): i = Utf8Iterator(self._str) i._pos = self._pos + i._calculated_pos = self._calculated_pos i._byte_pos = self._byte_pos i._current = self._current return i From noreply at buildbot.pypy.org Sat Sep 6 22:45:08 2014 From: noreply at buildbot.pypy.org (waedt) Date: Sat, 6 Sep 2014 22:45:08 +0200 (CEST) Subject: [pypy-commit] pypy utf8-unicode2: Give app-level unicode objects a specialized iterator Message-ID: <20140906204508.CA0E81C305D@cobra.cs.uni-duesseldorf.de> Author: Tyler Wade Branch: utf8-unicode2 Changeset: r73347:7b8e8bfabdc3 Date: 2014-08-17 01:21 -0500 http://bitbucket.org/pypy/pypy/changeset/7b8e8bfabdc3/ Log: Give app-level unicode objects a specialized iterator diff --git a/pypy/objspace/std/iterobject.py b/pypy/objspace/std/iterobject.py --- a/pypy/objspace/std/iterobject.py +++ b/pypy/objspace/std/iterobject.py @@ -113,6 +113,19 @@ self.index = index + 1 return w_item +class W_FastUnicodeIterObject(W_AbstractSeqIterObject): + def __init__(self, w_uni, index=0): + W_AbstractSeqIterObject.__init__(self, w_uni, index) + self.iter = iter(w_uni._value) + + def descr_next(self, space): + try: + next = self.iter.next() + except StopIteration: + raise OperationError(space.w_StopIteration, space.w_None) + self.index += 1 + return space.wrap(next) + class W_ReverseSeqIterObject(W_Root): def __init__(self, space, w_seq, index=-1): diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py --- a/pypy/objspace/std/objspace.py +++ b/pypy/objspace/std/objspace.py @@ -22,7 +22,7 @@ from pypy.objspace.std.dictmultiobject import W_DictMultiObject from pypy.objspace.std.floatobject import W_FloatObject from pypy.objspace.std.intobject import W_IntObject -from pypy.objspace.std.iterobject import W_AbstractSeqIterObject +from pypy.objspace.std.iterobject import W_AbstractSeqIterObject, W_FastUnicodeIterObject from pypy.objspace.std.listobject import W_ListObject from pypy.objspace.std.longobject import W_LongObject, newlong from pypy.objspace.std.bufferobject import W_Buffer @@ -317,6 +317,8 @@ return W_SliceObject(w_start, w_end, w_step) def newseqiter(self, w_obj): + if isinstance(w_obj, W_UnicodeObject): + return W_FastUnicodeIterObject(w_obj) return W_SeqIterObject(w_obj) def newbuffer(self, w_obj): From noreply at buildbot.pypy.org Sat Sep 6 22:45:09 2014 From: noreply at buildbot.pypy.org (waedt) Date: Sat, 6 Sep 2014 22:45:09 +0200 (CEST) Subject: [pypy-commit] pypy utf8-unicode2: Fix MBCS codecs on Windows Message-ID: <20140906204509.EC8B71C305D@cobra.cs.uni-duesseldorf.de> Author: Tyler Wade Branch: utf8-unicode2 Changeset: r73348:19502a3ecd04 Date: 2014-08-28 23:48 -0500 http://bitbucket.org/pypy/pypy/changeset/19502a3ecd04/ Log: Fix MBCS codecs on Windows diff --git a/pypy/interpreter/test/test_utf8_codecs.py b/pypy/interpreter/test/test_utf8_codecs.py --- a/pypy/interpreter/test/test_utf8_codecs.py +++ b/pypy/interpreter/test/test_utf8_codecs.py @@ -727,7 +727,7 @@ def test_mbcs_encode_force_replace(self): if sys.platform != 'win32': py.test.skip("mbcs encoding is win32-specific") - u = u'@test_2224_tmp-?L??\udc80' + u = Utf8Str.from_unicode(u'@test_2224_tmp-?L??\udc80') encoder = self.getencoder('mbcs') assert encoder(u, len(u), 'strict') == '@test_2224_tmp-?L???' py.test.raises(UnicodeEncodeError, encoder, u, len(u), 'strict', diff --git a/pypy/interpreter/utf8.py b/pypy/interpreter/utf8.py --- a/pypy/interpreter/utf8.py +++ b/pypy/interpreter/utf8.py @@ -9,12 +9,18 @@ wchar_rint = rffi.r_int +wchar_ruint = rffi.r_uint WCHAR_INTP = rffi.INTP +WCHAR_UINTP = rffi.UINTP WCHAR_INT = rffi.INT +WCHAR_UINT = rffi.UINT if rffi.sizeof(rffi.WCHAR_T) == 2: wchar_rint = rffi.r_short + wchar_ruint = rffi.r_ushort WCHAR_INTP = rffi.SHORTP + WCHAR_UINTP = rffi.USHORTP WCHAR_INT = rffi.SHORT + WCHAR_UINT = rffi.USHORT def utf8chr(value): @@ -541,11 +547,11 @@ if c > 0xFFFF: length += 1 - array = lltype.malloc(WCHAR_INTP.TO, length, flavor='raw', + array = lltype.malloc(WCHAR_UINTP.TO, length, flavor='raw', track_allocation=track_allocation) self.copy_to_wcharp(array, 0, length) - array[length - 1] = wchar_rint(0) + array[length - 1] = wchar_ruint(0) array = rffi.cast(rffi.CWCHARP, array) return array @@ -560,12 +566,12 @@ if rffi.sizeof(rffi.WCHAR_T) == 2: c1, c2 = create_surrogate_pair(c) - dst[i + dststart] = wchar_rint(c1) + dst[i + dststart] = wchar_ruint(c1) if c2: i += 1 - dst[i + dststart] = wchar_rint(c2) + dst[i + dststart] = wchar_ruint(c2) else: - dst[i + dststart] = wchar_rint(c) + dst[i + dststart] = wchar_ruint(c) i += 1 @@ -574,7 +580,7 @@ @staticmethod def from_wcharp(wcharp): - array = rffi.cast(WCHAR_INTP, wcharp) + array = rffi.cast(WCHAR_UINTP, wcharp) builder = Utf8Builder() i = 0; while True: @@ -602,7 +608,7 @@ @staticmethod def from_wcharpn(wcharp, size): - array = rffi.cast(WCHAR_INTP, wcharp) + array = rffi.cast(WCHAR_UINTP, wcharp) builder = Utf8Builder() i = 0; while i < size: @@ -630,7 +636,7 @@ @staticmethod def from_wcharpsize(wcharp, size): - array = rffi.cast(WCHAR_INTP, wcharp) + array = rffi.cast(WCHAR_UINTP, wcharp) builder = Utf8Builder() i = 0; while i < size: diff --git a/pypy/interpreter/utf8_codecs.py b/pypy/interpreter/utf8_codecs.py --- a/pypy/interpreter/utf8_codecs.py +++ b/pypy/interpreter/utf8_codecs.py @@ -1483,7 +1483,9 @@ if MultiByteToWideChar(CP_ACP, flags, dataptr, size, buf.raw, usize) == 0: _decode_mbcs_error(s, errorhandler) - return buf.str(usize), size + # TODO Is this cast necessary for rpython static-typing? + #return Utf8Str.from_wcharp(rffi.cast(rffi.CWCHARP, buf.raw)), size + return Utf8Str.from_wcharpsize(buf.raw, size), size def unicode_encode_mbcs(s, size, errors, errorhandler=None, force_replace=True): @@ -1507,7 +1509,7 @@ used_default_p[0] = rffi.cast(rwin32.BOOL, False) try: - with rffi.scoped_nonmoving_unicodebuffer(s) as dataptr: + with s.scoped_wcharp_copy() as dataptr: # first get the size of the result mbcssize = WideCharToMultiByte(CP_ACP, flags, dataptr, size, None, 0, From noreply at buildbot.pypy.org Sat Sep 6 22:45:11 2014 From: noreply at buildbot.pypy.org (waedt) Date: Sat, 6 Sep 2014 22:45:11 +0200 (CEST) Subject: [pypy-commit] pypy utf8-unicode2: Mostly fix _cffi_backend on Windows Message-ID: <20140906204511.2F5021C305D@cobra.cs.uni-duesseldorf.de> Author: Tyler Wade Branch: utf8-unicode2 Changeset: r73349:039ddc451153 Date: 2014-08-29 00:00 -0500 http://bitbucket.org/pypy/pypy/changeset/039ddc451153/ Log: Mostly fix _cffi_backend on Windows diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -10,6 +10,7 @@ from rpython.rlib.rarithmetic import r_uint, SHRT_MIN, SHRT_MAX, \ INT_MIN, INT_MAX, UINT_MAX +from pypy.interpreter import utf8 from pypy.interpreter.utf8 import Utf8Str from pypy.interpreter.executioncontext import (ExecutionContext, ActionFlag, UserDelAction) @@ -1567,7 +1568,7 @@ "Like unicode_w, but rejects strings with NUL bytes." from rpython.rlib import rstring result = w_obj.unicode_w(self) - if Utf8Str('\x00') in result: + if utf8.IN('\00', result): raise OperationError(self.w_TypeError, self.wrap( 'argument must be a unicode string without NUL characters')) return rstring.assert_str0(result) diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py --- a/pypy/module/_cffi_backend/ctypeptr.py +++ b/pypy/module/_cffi_backend/ctypeptr.py @@ -101,10 +101,10 @@ "initializer unicode string is too long for '%s' " "(got %d characters)", self.name, n) - unichardata = rffi.cast(utf8.WCHAR_INTP, cdata) + unichardata = rffi.cast(utf8.WCHAR_UINTP, cdata) s.copy_to_wcharp(unichardata, 0, n) if n != self.length: - unichardata[n] = utf8.wchar_rint(0) + unichardata[n] = utf8.wchar_ruint(0) else: raise self._convert_error("list or tuple", w_ob) From noreply at buildbot.pypy.org Sat Sep 6 22:45:12 2014 From: noreply at buildbot.pypy.org (waedt) Date: Sat, 6 Sep 2014 22:45:12 +0200 (CEST) Subject: [pypy-commit] pypy utf8-unicode2: Fix some tests on Windows Message-ID: <20140906204512.5541E1C305D@cobra.cs.uni-duesseldorf.de> Author: Tyler Wade Branch: utf8-unicode2 Changeset: r73350:249da6fb0a96 Date: 2014-08-29 00:31 -0500 http://bitbucket.org/pypy/pypy/changeset/249da6fb0a96/ Log: Fix some tests on Windows diff --git a/pypy/module/_rawffi/test/test__rawffi.py b/pypy/module/_rawffi/test/test__rawffi.py --- a/pypy/module/_rawffi/test/test__rawffi.py +++ b/pypy/module/_rawffi/test/test__rawffi.py @@ -861,7 +861,7 @@ a[2] = u'z' assert a[0] == u'x' b = _rawffi.Array('c').fromaddress(a.buffer, 38) - if sys.maxunicode > 65535: + if _rawffi.sizeof('u') == 4: # UCS4 build assert b[0] == 'x' assert b[1] == '\x00' diff --git a/pypy/module/array/test/test_array.py b/pypy/module/array/test/test_array.py --- a/pypy/module/array/test/test_array.py +++ b/pypy/module/array/test/test_array.py @@ -1031,9 +1031,13 @@ assert str(buffer(self.array('i'))) == '' def test_unicode_outofrange(self): + a = self.array('u', unicode(r'\x01\u263a\x00\ufeff', 'unicode-escape')) b = self.array('u', unicode(r'\x01\u263a\x00\ufeff', 'unicode-escape')) b.byteswap() - raises(ValueError, "b[0]") + if b.itemsize == 4: + raises(ValueError, "b[0]") + else: + assert a != b class AppTestArrayBuiltinShortcut(AppTestArray): From noreply at buildbot.pypy.org Sat Sep 6 22:45:13 2014 From: noreply at buildbot.pypy.org (waedt) Date: Sat, 6 Sep 2014 22:45:13 +0200 (CEST) Subject: [pypy-commit] pypy utf8-unicode2: Use an iterator here so its O(n) Message-ID: <20140906204513.731EC1C305D@cobra.cs.uni-duesseldorf.de> Author: Tyler Wade Branch: utf8-unicode2 Changeset: r73351:4c43dd3a4c7b Date: 2014-08-29 01:23 -0500 http://bitbucket.org/pypy/pypy/changeset/4c43dd3a4c7b/ Log: Use an iterator here so its O(n) diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py --- a/pypy/module/_codecs/interp_codecs.py +++ b/pypy/module/_codecs/interp_codecs.py @@ -625,9 +625,10 @@ def charmap_build(space, chars): # XXX CPython sometimes uses a three-level trie w_charmap = space.newdict() - for num in range(len(chars)): - elem = chars[num] + num = 0 + for elem in chars: space.setitem(w_charmap, space.newint(utf8ord(elem)), space.newint(num)) + num += 1 return w_charmap # ____________________________________________________________ From noreply at buildbot.pypy.org Sat Sep 6 22:45:14 2014 From: noreply at buildbot.pypy.org (waedt) Date: Sat, 6 Sep 2014 22:45:14 +0200 (CEST) Subject: [pypy-commit] pypy utf8-unicode2: Add a simple index caching scheme Message-ID: <20140906204514.967B91C305D@cobra.cs.uni-duesseldorf.de> Author: Tyler Wade Branch: utf8-unicode2 Changeset: r73352:26fe9ddc3df9 Date: 2014-09-06 15:38 -0500 http://bitbucket.org/pypy/pypy/changeset/26fe9ddc3df9/ Log: Add a simple index caching scheme diff --git a/pypy/interpreter/test/test_utf8.py b/pypy/interpreter/test/test_utf8.py --- a/pypy/interpreter/test/test_utf8.py +++ b/pypy/interpreter/test/test_utf8.py @@ -2,7 +2,8 @@ import py import sys -from pypy.interpreter.utf8 import Utf8Str, Utf8Builder, utf8chr, utf8ord +from pypy.interpreter.utf8 import ( + Utf8Str, Utf8Builder, utf8chr, utf8ord, utf8ord_bytes, LastAccessCache) from rpython.rtyper.lltypesystem import rffi from rpython.rtyper.test.test_llinterp import interpret @@ -221,3 +222,14 @@ rffi.free_wcharp(wcharp) +def test_mru_cache(): + s = Utf8Str.from_unicode(u'abcdefg') + cacher = LastAccessCache(s) + + for i in range(len(s)): + assert i == cacher.byte_index_of_char(i) + assert cacher.byte_index_of_char(1) == 1 + + l = [unichr(utf8ord_bytes(s.bytes, cacher.byte_index_of_char(i))) + for i in range(len(s))] + assert u''.join(l) diff --git a/pypy/interpreter/utf8.py b/pypy/interpreter/utf8.py --- a/pypy/interpreter/utf8.py +++ b/pypy/interpreter/utf8.py @@ -4,6 +4,7 @@ from rpython.rlib.runicode import utf8_code_length from rpython.rlib.unicodedata import unicodedb_5_2_0 as unicodedb from rpython.rlib.rarithmetic import r_uint, intmask, base_int +from rpython.rlib.jit import elidable from rpython.rtyper.lltypesystem import rffi, lltype from rpython.tool.sourcetools import func_with_new_name @@ -132,6 +133,7 @@ # have to iterate the bytes while checking for (& 0b01000000) self.bytes = data + self._cache_scheme = LastAccessCache(self) self._is_ascii = is_ascii if length != -1: @@ -160,22 +162,51 @@ self._len = length def index_of_char(self, char): - if char >= len(self): - return len(self.bytes) - byte = 0 - pos = 0 - while pos < char: - pos += 1 - byte += utf8_code_length[ord(self.bytes[byte])] + return self._cache_scheme.byte_index_of_char(char) - return byte + def index_of_char_from_known(self, char, start_char, start_byte): + if start_char > char: + pos = start_char + byte_pos = start_byte + while pos != char: + byte_pos -= 1 + if utf8_code_length[ord(self.bytes[byte_pos])]: + pos -= 1 - def char_index_of_byte(self, byte_): - byte = 0 - pos = 0 - while byte < byte_: - pos += 1 - byte += utf8_code_length[ord(self.bytes[byte])] + elif start_char < char: + diff = char - start_char + byte_pos = start_byte + while diff: + byte_pos = self.next_char(byte_pos) + diff -= 1 + + else: + return start_byte + + return byte_pos + + + def char_index_of_byte(self, byte_pos): + return self._cache_scheme.char_index_of_byte(byte_pos) + + def char_index_of_byte_from_known(self, byte_pos, start_char, start_byte): + if start_byte > byte_pos: + pos = start_char - 1 + cur_byte_pos = start_byte - 1 + while byte_pos != cur_byte_pos: + if utf8_code_length[ord(self.bytes[cur_byte_pos])]: + pos -= 1 + cur_byte_pos -= 1 + + elif start_byte < byte_pos: + pos = start_char + cur_byte_pos = start_byte + while byte_pos != cur_byte_pos: + cur_byte_pos = self.next_char(cur_byte_pos) + pos += 1 + + else: + return start_char return pos @@ -870,3 +901,60 @@ return i +#__________________ + + +class LastAccessCache(object): + _immutable_fields_ = ['str'] + def __init__(self, str): + self.str = str + self.prev_byte_pos = 0 + self.prev_pos = 0 + + def byte_index_of_char(self, pos): + if pos == 0: + return 0 + + # Calculate the distance from the start, the last known position, and + # the end + # (cost, known char, known byte) + start_dist = (pos, 0, 0) + end_dist = (2 * (len(self.str) - pos), len(self.str), + len(self.str.bytes)) + + if pos <= self.prev_pos: + min = (2 * (self.prev_pos - pos), self.prev_pos, self.prev_byte_pos) + else: + min = (pos - self.prev_pos, self.prev_pos, self.prev_byte_pos) + + if start_dist[0] < min[0]: + min = start_dist + if end_dist[0] < min[0]: + min = end_dist + + b = self.str.index_of_char_from_known(pos, min[1], min[2]) + self.prev_pos = pos + self.prev_byte_pos = b + return b + + def char_index_of_byte(self, byte_pos): + if byte_pos == 0: + return 0 + + # (cost, known char, known byte) + start_dist = (byte_pos, 0, 0) + end_dist = (2 * (len(self.str.bytes) - byte_pos), len(self.str), + len(self.str.bytes)) + + min = (2 * abs(byte_pos - self.prev_byte_pos), self.prev_pos, + self.prev_byte_pos) + + if start_dist[0] < min[0]: + min = start_dist + if end_dist[0] < min[0]: + min = end_dist + + i = self.str.char_index_of_byte_from_known(byte_pos, min[1], min[2]) + self.prev_pos = i + self.prev_byte_pos = byte_pos + return i From noreply at buildbot.pypy.org Sat Sep 6 22:45:15 2014 From: noreply at buildbot.pypy.org (waedt) Date: Sat, 6 Sep 2014 22:45:15 +0200 (CEST) Subject: [pypy-commit] pypy utf8-unicode2: Rename this method Message-ID: <20140906204515.BCEF01C305D@cobra.cs.uni-duesseldorf.de> Author: Tyler Wade Branch: utf8-unicode2 Changeset: r73353:20c49b3d0e22 Date: 2014-09-06 15:42 -0500 http://bitbucket.org/pypy/pypy/changeset/20c49b3d0e22/ Log: Rename this method diff --git a/pypy/interpreter/test/test_utf8.py b/pypy/interpreter/test/test_utf8.py --- a/pypy/interpreter/test/test_utf8.py +++ b/pypy/interpreter/test/test_utf8.py @@ -109,13 +109,13 @@ def test_convert_indices(): s = build_utf8str() - assert s.index_of_char(0) == 0 - assert s.index_of_char(1) == 1 - assert s.index_of_char(2) == 3 - assert s.index_of_char(3) == 6 + assert s.byte_index_of_char(0) == 0 + assert s.byte_index_of_char(1) == 1 + assert s.byte_index_of_char(2) == 3 + assert s.byte_index_of_char(3) == 6 for i in range(len(s)): - assert s.char_index_of_byte(s.index_of_char(i)) == i + assert s.char_index_of_byte(s.byte_index_of_char(i)) == i def test_join(): s = Utf8Str(' ') diff --git a/pypy/interpreter/utf8.py b/pypy/interpreter/utf8.py --- a/pypy/interpreter/utf8.py +++ b/pypy/interpreter/utf8.py @@ -57,7 +57,7 @@ return res def utf8ord(ustr, start=0): - start = ustr.index_of_char(start) + start = ustr.byte_index_of_char(start) return utf8ord_bytes(ustr.bytes, start) @specialize.argtype(0) @@ -161,10 +161,10 @@ self._len = length - def index_of_char(self, char): + def byte_index_of_char(self, char): return self._cache_scheme.byte_index_of_char(char) - def index_of_char_from_known(self, char, start_char, start_byte): + def byte_index_of_char_from_known(self, char, start_char, start_byte): if start_char > char: pos = start_char byte_pos = start_byte @@ -246,7 +246,7 @@ if self._is_ascii: return Utf8Str(self.bytes[start:stop], True) - start_byte = self.index_of_char(start) + start_byte = self.byte_index_of_char(start) stop_byte = start_byte stop_pos = start # TODO: Is detecting ascii-ness here actually useful? If it will @@ -364,11 +364,11 @@ if start < 0: start = 0 else: - start = self.index_of_char(start) + start = self.byte_index_of_char(start) elif start > len(self): start = -1 else: - start = self.index_of_char(start) + start = self.byte_index_of_char(start) if end is None or end >= len(self): end = len(self.bytes) @@ -377,11 +377,11 @@ if end < 0: end = 0 else: - end = self.index_of_char(end) + end = self.byte_index_of_char(end) elif end > len(self): end = len(self.bytes) else: - end = self.index_of_char(end) + end = self.byte_index_of_char(end) return start, end @@ -755,8 +755,8 @@ if isinstance(s, str): self._builder.append_slice(s, start, end) elif isinstance(s, Utf8Str): - self._builder.append_slice(s.bytes, s.index_of_char(start), - s.index_of_char(end)) + self._builder.append_slice(s.bytes, s.byte_index_of_char(start), + s.byte_index_of_char(end)) if not s._is_ascii: self._is_ascii = False else: @@ -852,7 +852,7 @@ self._pos = start self._calculated_pos = start - self._byte_pos = str.index_of_char(start) + self._byte_pos = str.byte_index_of_char(start) self._current = utf8ord_bytes(self._str.bytes, self._byte_pos) def _calc_current(self): @@ -932,7 +932,7 @@ if end_dist[0] < min[0]: min = end_dist - b = self.str.index_of_char_from_known(pos, min[1], min[2]) + b = self.str.byte_index_of_char_from_known(pos, min[1], min[2]) self.prev_pos = pos self.prev_byte_pos = b return b diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py --- a/pypy/module/cpyext/unicodeobject.py +++ b/pypy/module/cpyext/unicodeobject.py @@ -693,8 +693,8 @@ suffix match), 0 otherwise. Return -1 if an error occurred.""" str = space.unicode_w(w_str) substr = space.unicode_w(w_substr) - start = str.index_of_char(start) - end = str.index_of_char(end) + start = str.byte_index_of_char(start) + end = str.byte_index_of_char(end) if rffi.cast(lltype.Signed, direction) <= 0: return rstring.startswith(str.bytes, substr.bytes, start, end) else: diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -334,11 +334,13 @@ def _startswith(self, space, value, w_prefix, start, end): return startswith(value.bytes, self._op_val(space, w_prefix).bytes, - value.index_of_char(start), value.index_of_char(end)) + value.byte_index_of_char(start), + value.byte_index_of_char(end)) def _endswith(self, space, value, w_prefix, start, end): return endswith(value.bytes, self._op_val(space, w_prefix).bytes, - value.index_of_char(start), value.index_of_char(end)) + value.byte_index_of_char(start), + value.byte_index_of_char(end)) @staticmethod def _split(value, sep=None, maxsplit=-1): From noreply at buildbot.pypy.org Sat Sep 6 23:36:31 2014 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 6 Sep 2014 23:36:31 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: Remove clear_array_contents on virtual values. Since they're initialized Message-ID: <20140906213631.406681C320C@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73354:bf18cdaa79cf Date: 2014-09-06 15:36 -0600 http://bitbucket.org/pypy/pypy/changeset/bf18cdaa79cf/ Log: Remove clear_array_contents on virtual values. Since they're initialized to zeros (and we force zeros on force), it should be about it diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py --- a/rpython/jit/codewriter/jtransform.py +++ b/rpython/jit/codewriter/jtransform.py @@ -615,14 +615,13 @@ op1 = SpaceOperation('new_array', [op.args[2], arraydescr], op.result) if self._has_gcptrs_in(ARRAY): - return self.zero_contents(op1, op.result, ARRAY, + return self.zero_contents([op1], op.result, ARRAY, only_gc_pointers=True) if op.args[1].value.get('zero', False): - return self.zero_contents(op1, op.result, ARRAY) + return self.zero_contents([op1], op.result, ARRAY) return op1 - def zero_contents(self, prev_op, v, TYPE, only_gc_pointers=False): - ops = [prev_op] + def zero_contents(self, ops, v, TYPE, only_gc_pointers=False): if isinstance(TYPE, lltype.Struct): for name, FIELD in TYPE._flds.iteritems(): if (not only_gc_pointers or @@ -922,9 +921,9 @@ sizedescr = self.cpu.sizeof(STRUCT) op1 = SpaceOperation(opname, [sizedescr], op.result) if true_zero: - return self.zero_contents(op1, op.result, STRUCT) + return self.zero_contents([op1], op.result, STRUCT) if self._has_gcptrs_in(STRUCT): - return self.zero_contents(op1, op.result, STRUCT, + return self.zero_contents([op1], op.result, STRUCT, only_gc_pointers=True) return op1 @@ -1664,6 +1663,8 @@ v.concretetype = lltype.Signed ops.append(SpaceOperation('int_force_ge_zero', [v_length], v)) ops.append(SpaceOperation('new_array', [v, arraydescr], op.result)) + if self._has_gcptrs_in(op.result.concretetype.TO): + self.zero_contents(ops, op.result, op.result.concretetype.TO) return ops def do_fixed_list_len(self, op, args, arraydescr): diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py @@ -5461,6 +5461,17 @@ """ self.optimize_loop(ops, expected) + def test_virtual_clear_array_contents(self): + ops = """ + [] + p0 = new_array(2, descr=arraydescr) + clear_array_contents(p0, descr=arraydescr) + """ + expected = """ + [] + """ + self.optimize_loop(ops, expected) + class TestLLtype(BaseTestOptimizeBasic, LLtypeMixin): pass diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py --- a/rpython/jit/metainterp/optimizeopt/virtualize.py +++ b/rpython/jit/metainterp/optimizeopt/virtualize.py @@ -284,7 +284,7 @@ def setitem(self, index, itemvalue): assert isinstance(itemvalue, optimizer.OptValue) - self._items[index] = itemvalue + self._items[index] = itemvalue def force_at_end_of_preamble(self, already_forced, optforce): # note that this method is on VArrayValue instead of @@ -660,6 +660,12 @@ else: self.emit_operation(op) + def optimize_CLEAR_ARRAY_CONTENTS(self, op): + v = self.getvalue(op.getarg(0)) + if v.is_virtual(): + return + self.emit_operation(op) + def optimize_CALL(self, op): effectinfo = op.getdescr().get_extra_info() if effectinfo.oopspecindex == EffectInfo.OS_RAW_MALLOC_VARSIZE_CHAR: diff --git a/rpython/jit/metainterp/test/test_list.py b/rpython/jit/metainterp/test/test_list.py --- a/rpython/jit/metainterp/test/test_list.py +++ b/rpython/jit/metainterp/test/test_list.py @@ -8,7 +8,7 @@ def check_all_virtualized(self): self.check_resops(setarrayitem_gc=0, new_array=0, arraylen_gc=0, - getarrayitem_gc=0) + getarrayitem_gc=0) def test_simple_array(self): jitdriver = JitDriver(greens = [], reds = ['n']) From noreply at buildbot.pypy.org Sun Sep 7 00:33:11 2014 From: noreply at buildbot.pypy.org (waedt) Date: Sun, 7 Sep 2014 00:33:11 +0200 (CEST) Subject: [pypy-commit] pypy utf8-unicode2: Fix translation Message-ID: <20140906223311.8B1F41C129D@cobra.cs.uni-duesseldorf.de> Author: Tyler Wade Branch: utf8-unicode2 Changeset: r73355:327917b8c2e5 Date: 2014-09-06 17:32 -0500 http://bitbucket.org/pypy/pypy/changeset/327917b8c2e5/ Log: Fix translation diff --git a/pypy/interpreter/utf8.py b/pypy/interpreter/utf8.py --- a/pypy/interpreter/utf8.py +++ b/pypy/interpreter/utf8.py @@ -124,7 +124,7 @@ return s1 in s2 class Utf8Str(object): - _immutable_fields_ = ['bytes', '_is_ascii', '_len'] + _immutable_fields_ = ['bytes', '_is_ascii', '_len', '_cache_scheme'] def __init__(self, data, is_ascii=False, length=-1): # TODO: Maybe I can determine is_ascii rather than have it passed in? @@ -162,7 +162,12 @@ self._len = length def byte_index_of_char(self, char): - return self._cache_scheme.byte_index_of_char(char) + if self._is_ascii: + return char + + res = self._cache_scheme.byte_index_of_char(char) + assert res >= 0 + return res def byte_index_of_char_from_known(self, char, start_char, start_byte): if start_char > char: @@ -187,6 +192,9 @@ def char_index_of_byte(self, byte_pos): + if self._is_ascii: + return byte_pos + return self._cache_scheme.char_index_of_byte(byte_pos) def char_index_of_byte_from_known(self, byte_pos, start_char, start_byte): @@ -915,24 +923,21 @@ if pos == 0: return 0 - # Calculate the distance from the start, the last known position, and - # the end - # (cost, known char, known byte) - start_dist = (pos, 0, 0) - end_dist = (2 * (len(self.str) - pos), len(self.str), - len(self.str.bytes)) + if pos < 2 * (len(self.str) - pos): + cost = pos + known_char = 0 + known_byte = 0 + else: + cost = 2 * (len(self.str) - pos) + known_char = len(self.str) + known_byte = len(self.str.bytes) - if pos <= self.prev_pos: - min = (2 * (self.prev_pos - pos), self.prev_pos, self.prev_byte_pos) - else: - min = (pos - self.prev_pos, self.prev_pos, self.prev_byte_pos) + if 2 * abs(pos - self.prev_pos) < cost: + known_char = self.prev_pos + known_byte = self.prev_byte_pos - if start_dist[0] < min[0]: - min = start_dist - if end_dist[0] < min[0]: - min = end_dist - - b = self.str.byte_index_of_char_from_known(pos, min[1], min[2]) + b = self.str.byte_index_of_char_from_known(pos, known_char, + known_byte) self.prev_pos = pos self.prev_byte_pos = b return b @@ -942,19 +947,21 @@ return 0 # (cost, known char, known byte) - start_dist = (byte_pos, 0, 0) - end_dist = (2 * (len(self.str.bytes) - byte_pos), len(self.str), - len(self.str.bytes)) + if byte_pos < 2 * (len(self.str.bytes) - byte_pos): + cost = byte_pos + known_char = 0 + known_byte = 0 + else: + cost = 2 * (len(self.str.bytes) - byte_pos) + known_char = len(self.str) + known_byte = len(self.str.bytes) - min = (2 * abs(byte_pos - self.prev_byte_pos), self.prev_pos, - self.prev_byte_pos) + if 2 * abs(byte_pos - self.prev_byte_pos) < cost: + known_char = self.prev_pos + known_byte = self.prev_byte_pos - if start_dist[0] < min[0]: - min = start_dist - if end_dist[0] < min[0]: - min = end_dist - - i = self.str.char_index_of_byte_from_known(byte_pos, min[1], min[2]) + i = self.str.char_index_of_byte_from_known(byte_pos, known_char, + known_byte) self.prev_pos = i self.prev_byte_pos = byte_pos return i From noreply at buildbot.pypy.org Sun Sep 7 03:12:51 2014 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 7 Sep 2014 03:12:51 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: hint for ll_mul Message-ID: <20140907011251.B5CDB1C030E@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73356:6fc92c728d84 Date: 2014-09-06 19:12 -0600 http://bitbucket.org/pypy/pypy/changeset/6fc92c728d84/ Log: hint for ll_mul diff --git a/rpython/rtyper/rlist.py b/rpython/rtyper/rlist.py --- a/rpython/rtyper/rlist.py +++ b/rpython/rtyper/rlist.py @@ -1014,7 +1014,8 @@ return res ll_inplace_mul.oopspec = 'list.inplace_mul(l, factor)' - + at jit.look_inside_iff(lambda _, l, factor: jit.isvirtual(l) and + jit.isconstant(factor) and factor < 10) def ll_mul(RESLIST, l, factor): length = l.ll_length() if factor < 0: From noreply at buildbot.pypy.org Sun Sep 7 09:30:34 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 7 Sep 2014 09:30:34 +0200 (CEST) Subject: [pypy-commit] pypy default: Add a check Message-ID: <20140907073034.D40A11C072F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73357:a54f4f1af1fc Date: 2014-09-07 09:25 +0200 http://bitbucket.org/pypy/pypy/changeset/a54f4f1af1fc/ Log: Add a check diff --git a/rpython/memory/gctransform/asmgcroot.py b/rpython/memory/gctransform/asmgcroot.py --- a/rpython/memory/gctransform/asmgcroot.py +++ b/rpython/memory/gctransform/asmgcroot.py @@ -517,6 +517,8 @@ "anywhere I know, bug in asmgcc") # fish the depth extra_stack_depth = (ebp_in_caller + STACK_DEPTH_OFS).signed[0] + ll_assert((extra_stack_depth & (rffi.sizeof(lltype.Signed) - 1)) + == 0, "asmgcc: misaligned extra_stack_depth") extra_stack_depth //= rffi.sizeof(lltype.Signed) self._shape_decompressor.setjitframe(extra_stack_depth) return From noreply at buildbot.pypy.org Sun Sep 7 09:30:36 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 7 Sep 2014 09:30:36 +0200 (CEST) Subject: [pypy-commit] pypy default: Waiting 0.5 seconds is by far not enough on Windows. Sleep until all Message-ID: <20140907073036.0E0631C072F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73358:d5571baf1ff6 Date: 2014-09-07 09:30 +0200 http://bitbucket.org/pypy/pypy/changeset/d5571baf1ff6/ Log: Waiting 0.5 seconds is by far not enough on Windows. Sleep until all threads are finished. diff --git a/pypy/module/thread/test/test_thread.py b/pypy/module/thread/test/test_thread.py --- a/pypy/module/thread/test/test_thread.py +++ b/pypy/module/thread/test/test_thread.py @@ -13,18 +13,26 @@ def f(): lock.acquire() lock.release() + start = thread._count() try: try: for i in range(1000): thread.start_new_thread(f, ()) finally: lock.release() - # wait a bit to allow most threads to finish now - time.sleep(0.5) except (thread.error, MemoryError): cls.w_can_start_many_threads = space.wrap(False) else: cls.w_can_start_many_threads = space.wrap(True) + # wait a bit to allow all threads to finish now + remaining = thread._count() + retries = 0 + while remaining > start: + retries += 1 + if retries == 200: + raise Exception("the test's threads don't stop!") + time.sleep(0.2) + remaining = thread._count() def test_start_new_thread(self): import thread From noreply at buildbot.pypy.org Sun Sep 7 15:16:37 2014 From: noreply at buildbot.pypy.org (mattip) Date: Sun, 7 Sep 2014 15:16:37 +0200 (CEST) Subject: [pypy-commit] pypy release-2.4.x: merge default into branch Message-ID: <20140907131637.26F321C0A66@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: release-2.4.x Changeset: r73359:9f425c60afdf Date: 2014-09-07 14:38 +0300 http://bitbucket.org/pypy/pypy/changeset/9f425c60afdf/ Log: merge default into branch diff --git a/pypy/doc/release-2.4.0.rst b/pypy/doc/release-2.4.0.rst --- a/pypy/doc/release-2.4.0.rst +++ b/pypy/doc/release-2.4.0.rst @@ -2,12 +2,8 @@ PyPy 2.4 - ???????? ================================================= -We're pleased to announce PyPy 2.4, a significant milestone on it's own right -and the proud parent of our recent PyPy3 and STM releases. - -XXX this sentence is confusing, refactor - -This release contains several improvements and bugfixes. +We're pleased to announce PyPy 2.4, which contains significant performance +enhancements and bug fixes. You can download the PyPy 2.4 release here: @@ -15,20 +11,20 @@ We would like to thank our donors for the continued support of the PyPy project, and for those who donate to our three sub-projects. -We've shown quite a bit of progress -but we're slowly running out of funds. +We've shown quite a bit of progress, but we're slowly running out of funds. Please consider donating more, or even better convince your employer to donate, so we can finish those projects! We would like to also point out that in September, `the Python Software Foundation`_ will `match funds`_ for any donations up to $10k! The three sub-projects are: -* `Py3k`_ (supporting Python 3.x): We have released a Python 3.2.5 compatable version - we call PyPy3 2.3.1, and are working toward a Python 3.3 compatable version +* `Py3k`_ (supporting Python 3.x): We have released a Python 3.2.5 compatible version + we call PyPy3 2.3.1, and are working toward a Python 3.3 compatible version -* `STM`_ (software transactional memory): We have release a first working version, and -continue to try out new promising paths of acheiving a fast multithreaded python +* `STM`_ (software transactional memory): We have released a first working version, + and continue to try out new promising paths of achieving a fast multithreaded Python -* `NumPy`_ which requires installation of our fork of upstream numpy, available `on bitbucket`_ +* `NumPy`_ which requires installation of our fork of upstream numpy, + available `on bitbucket`_ .. _`Py3k`: http://pypy.org/py3donate.html .. _`STM`: http://pypy.org/tmdonate2.html @@ -41,15 +37,12 @@ ============= PyPy is a very compliant Python interpreter, almost a drop-in replacement for -CPython 2.7. It's fast (`pypy 2.4 and cpython 2.7.x`_ performance comparison; -note that cpython's speed has not changed since 2.7.2) +CPython 2.7. It's fast (`pypy 2.4 and cpython 2.7.x`_ performance comparison) due to its integrated tracing JIT compiler. -XXX confusing sentence, rewrite - -This release supports x86 machines running Linux 32/64, Mac OS X 64, Windows, -and OpenBSD, -as well as newer ARM hardware (ARMv6 or ARMv7, with VFPv3) running Linux. +This release supports **x86** machines on most common operating systems +(Linux 32/64, Mac OS X 64, Windows, and OpenBSD), +as well as newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux. While we support 32 bit python on Windows, work on the native Windows 64 bit python is still stalling, we would welcome a volunteer @@ -61,19 +54,18 @@ Highlights ========== -Benchmarks improved after internal improvements in string and +Benchmarks improved after internal enhancements in string and bytearray handling, and a major rewrite of the GIL handling. This means that external calls are now a lot faster, especially the CFFI ones. It also -means that a lot of corner cases when handling strings or bytearrays have -better performance. +means better performance in a lot of corner cases with handling strings or +bytearrays. PyPy now uses Python 2.7.8 standard library. We welcomed more than 12 new contributors, and conducted two Google -Summer of Code as well as other student projects not directly related -to Summer of Code. +Summer of Code projects, as well as other student projects not +directly related to Summer of Code. -XXX mention the work is ongoing and did not make it to 2.4? Issues reported with our previous release were fixed after reports from users on our new issue tracker at https://bitbucket.org/pypy/pypy/issues or on IRC at @@ -83,8 +75,8 @@ * Reduced internal copying of bytearray operations * Tweak the internal structure of StringBuilder to speed up large string -handling, which becomes advantageous on large programs at the cost of slightly -slower small *benchmark* type programs. + handling, which becomes advantageous on large programs at the cost of slightly + slower small *benchmark* type programs. * Boost performance of thread-local variables in both unjitted and jitted code, this mostly affects errno handling on linux, which makes external calls @@ -93,7 +85,7 @@ * Move to a mixed polling and mutex GIL model that make mutli-threaded jitted code run *much* faster -* Optimize errno handling in linux +* Optimize errno handling in linux (x86 and x86-64 only) * Remove ctypes pythonapi and ctypes.PyDLL, which never worked on PyPy @@ -110,6 +102,9 @@ .. _`whats-new`: http://doc.pypy.org/en/latest/whatsnew-2.3.1.html .. _resolved: https://bitbucket.org/pypy/pypy/issues?status=resolved +We have further improvements on the way: rpython file handling and +usable numpy linalg compatabiity should be merged soon. + Please try it out and let us know what you think. We especially welcome success stories, we know you are using PyPy, please tell us about it! diff --git a/pypy/interpreter/module.py b/pypy/interpreter/module.py --- a/pypy/interpreter/module.py +++ b/pypy/interpreter/module.py @@ -29,6 +29,17 @@ space.w_None) self.startup_called = False + def _cleanup_(self): + """Called by the annotator on prebuilt Module instances. + We don't have many such modules, but for the ones that + show up, remove their __file__ rather than translate it + statically inside the executable.""" + try: + space = self.space + space.delitem(self.w_dict, space.wrap('__file__')) + except OperationError: + pass + def install(self): """NOT_RPYTHON: installs this module into space.builtin_modules""" w_mod = self.space.wrap(self) diff --git a/pypy/interpreter/test/test_module.py b/pypy/interpreter/test/test_module.py --- a/pypy/interpreter/test/test_module.py +++ b/pypy/interpreter/test/test_module.py @@ -1,4 +1,5 @@ - +import py +from pypy.interpreter.error import OperationError from pypy.interpreter.module import Module class TestModule: @@ -17,6 +18,18 @@ space.raises_w(space.w_AttributeError, space.delattr, w_m, w('x')) + def test___file__(self, space): + w = space.wrap + m = Module(space, space.wrap('m')) + py.test.raises(OperationError, space.getattr, w(m), w('__file__')) + m._cleanup_() + py.test.raises(OperationError, space.getattr, w(m), w('__file__')) + space.setattr(w(m), w('__file__'), w('m.py')) + space.getattr(w(m), w('__file__')) # does not raise + m._cleanup_() + py.test.raises(OperationError, space.getattr, w(m), w('__file__')) + + class AppTest_ModuleObject: def test_attr(self): m = __import__('__builtin__') diff --git a/pypy/module/thread/test/test_thread.py b/pypy/module/thread/test/test_thread.py --- a/pypy/module/thread/test/test_thread.py +++ b/pypy/module/thread/test/test_thread.py @@ -13,18 +13,26 @@ def f(): lock.acquire() lock.release() + start = thread._count() try: try: for i in range(1000): thread.start_new_thread(f, ()) finally: lock.release() - # wait a bit to allow most threads to finish now - time.sleep(0.5) except (thread.error, MemoryError): cls.w_can_start_many_threads = space.wrap(False) else: cls.w_can_start_many_threads = space.wrap(True) + # wait a bit to allow all threads to finish now + remaining = thread._count() + retries = 0 + while remaining > start: + retries += 1 + if retries == 200: + raise Exception("the test's threads don't stop!") + time.sleep(0.2) + remaining = thread._count() def test_start_new_thread(self): import thread @@ -227,7 +235,7 @@ import signal def f(): - for x in range(5): + for x in range(40): if waiting: thread.interrupt_main() return @@ -236,7 +244,7 @@ def busy_wait(): waiting.append(None) - for x in range(10): + for x in range(50): print 'tick...', x # <-force the GIL to be released, as time.sleep(0.1) # time.sleep doesn't do non-translated waiting.pop() @@ -245,6 +253,8 @@ signal.signal(signal.SIGINT, signal.default_int_handler) for i in range(100): + print + print "loop", i waiting = [] thread.start_new_thread(f, ()) raises(KeyboardInterrupt, busy_wait) diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py --- a/pypy/objspace/descroperation.py +++ b/pypy/objspace/descroperation.py @@ -671,6 +671,7 @@ left, right = specialnames errormsg = "unsupported operand type(s) for %s: '%%N' and '%%N'" % ( symbol.replace('%', '%%'),) + seq_bug_compat = (symbol == '+' or symbol == '*') def binop_impl(space, w_obj1, w_obj2): w_typ1 = space.type(w_obj1) @@ -686,20 +687,16 @@ # __xxx__ and __rxxx__ methods where found by identity. # Note that space.is_w() is potentially not happy if one of them # is None... - if w_left_src is not w_right_src: # XXX - # -- cpython bug compatibility: see objspace/std/test/ - # -- test_unicodeobject.test_str_unicode_concat_overrides. - # -- The following handles "unicode + string subclass" by - # -- pretending that the unicode is a superclass of the - # -- string, thus giving priority to the string subclass' - # -- __radd__() method. The case "string + unicode subclass" - # -- is handled directly by add__String_Unicode(). - if symbol == '+' and space.is_w(w_typ1, space.w_unicode): - w_typ1 = space.w_basestring - # -- end of bug compatibility - if space.is_true(space.issubtype(w_typ2, w_typ1)): - if (w_left_src and w_right_src and - not space.abstract_issubclass_w(w_left_src, w_right_src) and + if w_right_src and (w_left_src is not w_right_src) and w_left_src: + # 'seq_bug_compat' is for cpython bug-to-bug compatibility: + # see objspace/std/test/test_unicodeobject.*concat_overrides + # and objspace/test/test_descrobject.*rmul_overrides. + # For cases like "unicode + string subclass". + if ((seq_bug_compat and w_typ1.flag_sequence_bug_compat + and not w_typ2.flag_sequence_bug_compat) + # the non-bug-compat part is the following check: + or space.is_true(space.issubtype(w_typ2, w_typ1))): + if (not space.abstract_issubclass_w(w_left_src, w_right_src) and not space.abstract_issubclass_w(w_typ1, w_right_src)): w_obj1, w_obj2 = w_obj2, w_obj1 w_left_impl, w_right_impl = w_right_impl, w_left_impl diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py --- a/pypy/objspace/std/bytearrayobject.py +++ b/pypy/objspace/std/bytearrayobject.py @@ -1131,6 +1131,7 @@ reverse = interp2app(W_BytearrayObject.descr_reverse, doc=BytearrayDocstrings.reverse.__doc__), ) +W_BytearrayObject.typedef.flag_sequence_bug_compat = True init_signature = Signature(['source', 'encoding', 'errors'], None, None) init_defaults = [None, None, None] diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py --- a/pypy/objspace/std/bytesobject.py +++ b/pypy/objspace/std/bytesobject.py @@ -951,6 +951,7 @@ _formatter_field_name_split = interp2app(W_BytesObject.descr_formatter_field_name_split), ) +W_BytesObject.typedef.flag_sequence_bug_compat = True def string_escape_encode(s, quote): diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py --- a/pypy/objspace/std/listobject.py +++ b/pypy/objspace/std/listobject.py @@ -1874,3 +1874,4 @@ insert = interp2app(W_ListObject.descr_insert), remove = interp2app(W_ListObject.descr_remove), ) +W_ListObject.typedef.flag_sequence_bug_compat = True diff --git a/pypy/objspace/std/stdtypedef.py b/pypy/objspace/std/stdtypedef.py --- a/pypy/objspace/std/stdtypedef.py +++ b/pypy/objspace/std/stdtypedef.py @@ -93,6 +93,8 @@ overridetypedef=overridetypedef) if typedef is not overridetypedef: w_type.w_doc = space.wrap(typedef.doc) + if hasattr(typedef, 'flag_sequence_bug_compat'): + w_type.flag_sequence_bug_compat = typedef.flag_sequence_bug_compat w_type.lazyloaders = lazyloaders return w_type diff --git a/pypy/objspace/std/tupleobject.py b/pypy/objspace/std/tupleobject.py --- a/pypy/objspace/std/tupleobject.py +++ b/pypy/objspace/std/tupleobject.py @@ -244,6 +244,7 @@ count = interp2app(W_AbstractTupleObject.descr_count), index = interp2app(W_AbstractTupleObject.descr_index) ) +W_AbstractTupleObject.typedef.flag_sequence_bug_compat = True class W_TupleObject(W_AbstractTupleObject): diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py --- a/pypy/objspace/std/typeobject.py +++ b/pypy/objspace/std/typeobject.py @@ -67,6 +67,7 @@ _immutable_fields_ = ["flag_heaptype", "flag_cpytype", "flag_abstract?", + "flag_sequence_bug_compat", 'needsdel', 'weakrefable', 'hasdict', @@ -104,6 +105,7 @@ w_self.flag_heaptype = False w_self.flag_cpytype = False w_self.flag_abstract = False + w_self.flag_sequence_bug_compat = False w_self.instancetypedef = overridetypedef if overridetypedef is not None: diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -1068,6 +1068,7 @@ _formatter_field_name_split = interp2app(W_UnicodeObject.descr_formatter_field_name_split), ) +W_UnicodeObject.typedef.flag_sequence_bug_compat = True def _create_list_from_unicode(value): diff --git a/pypy/objspace/test/test_descroperation.py b/pypy/objspace/test/test_descroperation.py --- a/pypy/objspace/test/test_descroperation.py +++ b/pypy/objspace/test/test_descroperation.py @@ -734,6 +734,44 @@ assert X() == 'hello' + def test_sequence_rmul_overrides(self): + class oops(object): + def __rmul__(self, other): + return 42 + def __index__(self): + return 3 + assert '2' * oops() == 42 + assert [2] * oops() == 42 + assert (2,) * oops() == 42 + assert u'2' * oops() == 42 + assert bytearray('2') * oops() == 42 + assert 1000 * oops() == 42 + assert '2'.__mul__(oops()) == '222' + + def test_sequence_rmul_overrides_oldstyle(self): + class oops: + def __rmul__(self, other): + return 42 + def __index__(self): + return 3 + assert '2' * oops() == 42 + assert [2] * oops() == 42 + assert (2,) * oops() == 42 + assert u'2' * oops() == 42 + assert bytearray('2') * oops() == 42 + assert 1000 * oops() == 42 + assert '2'.__mul__(oops()) == '222' + + def test_sequence_radd_overrides(self): + class A1(list): + pass + class A2(list): + def __radd__(self, other): + return 42 + assert [2] + A1([3]) == [2, 3] + assert type([2] + A1([3])) is list + assert [2] + A2([3]) == 42 + class AppTestWithBuiltinShortcut(AppTest_Descroperation): spaceconfig = {'objspace.std.builtinshortcut': True} diff --git a/rpython/annotator/builtin.py b/rpython/annotator/builtin.py --- a/rpython/annotator/builtin.py +++ b/rpython/annotator/builtin.py @@ -255,8 +255,8 @@ BUILTIN_ANALYZERS[original] = value - at analyzer_for(getattr(OSError.__init__, 'im_func', OSError.__init__)) -def OSError_init(s_self, *args): + at analyzer_for(getattr(EnvironmentError.__init__, 'im_func', EnvironmentError.__init__)) +def EnvironmentError_init(s_self, *args): pass try: diff --git a/rpython/annotator/classdef.py b/rpython/annotator/classdef.py --- a/rpython/annotator/classdef.py +++ b/rpython/annotator/classdef.py @@ -438,8 +438,8 @@ # ____________________________________________________________ FORCE_ATTRIBUTES_INTO_CLASSES = { - OSError: {'errno': SomeInteger()}, - } + EnvironmentError: {'errno': SomeInteger(), 'strerror': SomeString(can_be_None=True), 'filename': SomeString(can_be_None=True)}, +} try: WindowsError diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py --- a/rpython/jit/metainterp/warmstate.py +++ b/rpython/jit/metainterp/warmstate.py @@ -127,6 +127,7 @@ JC_TRACING = 0x01 JC_DONT_TRACE_HERE = 0x02 JC_TEMPORARY = 0x04 +JC_TRACING_OCCURRED= 0x08 class BaseJitCell(object): """Subclasses of BaseJitCell are used in tandem with the single @@ -160,6 +161,8 @@ JC_TRACING: we are now tracing the loop from this greenkey. We'll likely end up with a wref_procedure_token, soonish. + JC_TRACING_OCCURRED: set if JC_TRACING was set at least once. + JC_TEMPORARY: a "temporary" wref_procedure_token. It's the procedure_token of a dummy loop that simply calls back the interpreter. Used for a CALL_ASSEMBLER where the @@ -206,7 +209,7 @@ # if we have this flag, and we *had* a procedure_token but # we no longer have one, then remove me. this prevents this # JitCell from being immortal. - return self.has_seen_a_procedure_token() + return self.has_seen_a_procedure_token() # i.e. dead weakref return True # Other JitCells can be removed. # ____________________________________________________________ @@ -374,7 +377,7 @@ if cell is None: cell = JitCell(*greenargs) jitcounter.install_new_cell(hash, cell) - cell.flags |= JC_TRACING + cell.flags |= JC_TRACING | JC_TRACING_OCCURRED try: metainterp.compile_and_run_once(jitdriver_sd, *args) finally: @@ -418,9 +421,15 @@ if procedure_token is None: if cell.flags & JC_DONT_TRACE_HERE: if not cell.has_seen_a_procedure_token(): - # we're seeing a fresh JC_DONT_TRACE_HERE with no - # procedure_token. Compile now. - bound_reached(hash, cell, *args) + # A JC_DONT_TRACE_HERE, i.e. a non-inlinable function. + # If we never tried to trace it, try it now immediately. + # Otherwise, count normally. + if cell.flags & JC_TRACING_OCCURRED: + tick = jitcounter.tick(hash, increment_threshold) + else: + tick = True + if tick: + bound_reached(hash, cell, *args) return # it was an aborted compilation, or maybe a weakref that # has been freed diff --git a/rpython/memory/gctransform/asmgcroot.py b/rpython/memory/gctransform/asmgcroot.py --- a/rpython/memory/gctransform/asmgcroot.py +++ b/rpython/memory/gctransform/asmgcroot.py @@ -517,6 +517,8 @@ "anywhere I know, bug in asmgcc") # fish the depth extra_stack_depth = (ebp_in_caller + STACK_DEPTH_OFS).signed[0] + ll_assert((extra_stack_depth & (rffi.sizeof(lltype.Signed) - 1)) + == 0, "asmgcc: misaligned extra_stack_depth") extra_stack_depth //= rffi.sizeof(lltype.Signed) self._shape_decompressor.setjitframe(extra_stack_depth) return diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py --- a/rpython/rlib/rfile.py +++ b/rpython/rlib/rfile.py @@ -93,7 +93,7 @@ def _error(ll_file): err = c_ferror(ll_file) c_clearerr(ll_file) - raise OSError(err, os.strerror(err)) + raise IOError(err, os.strerror(err)) def _dircheck(ll_file): @@ -104,7 +104,7 @@ else: if stat.S_ISDIR(st[0]): err = errno.EISDIR - raise OSError(err, os.strerror(err)) + raise IOError(err, os.strerror(err)) def _sanitize_mode(mode): @@ -136,7 +136,7 @@ ll_file = c_fopen(ll_name, ll_mode) if not ll_file: errno = rposix.get_errno() - raise OSError(errno, os.strerror(errno)) + raise IOError(errno, os.strerror(errno)) finally: lltype.free(ll_mode, flavor='raw') finally: @@ -223,7 +223,7 @@ res = do_close(ll_file) if res == -1: errno = rposix.get_errno() - raise OSError(errno, os.strerror(errno)) + raise IOError(errno, os.strerror(errno)) return res def _check_closed(self): @@ -341,7 +341,7 @@ bytes = c_fwrite(ll_value, 1, length, self._ll_file) if bytes != length: errno = rposix.get_errno() - raise OSError(errno, os.strerror(errno)) + raise IOError(errno, os.strerror(errno)) finally: rffi.free_nonmovingbuffer(value, ll_value) @@ -350,7 +350,7 @@ res = c_fflush(self._ll_file) if res != 0: errno = rposix.get_errno() - raise OSError(errno, os.strerror(errno)) + raise IOError(errno, os.strerror(errno)) def truncate(self, arg=-1): self._check_closed() @@ -360,21 +360,21 @@ res = c_ftruncate(self.fileno(), arg) if res == -1: errno = rposix.get_errno() - raise OSError(errno, os.strerror(errno)) + raise IOError(errno, os.strerror(errno)) def seek(self, pos, whence=0): self._check_closed() res = c_fseek(self._ll_file, pos, whence) if res == -1: errno = rposix.get_errno() - raise OSError(errno, os.strerror(errno)) + raise IOError(errno, os.strerror(errno)) def tell(self): self._check_closed() res = intmask(c_ftell(self._ll_file)) if res == -1: errno = rposix.get_errno() - raise OSError(errno, os.strerror(errno)) + raise IOError(errno, os.strerror(errno)) return res def fileno(self): diff --git a/rpython/rlib/streamio.py b/rpython/rlib/streamio.py --- a/rpython/rlib/streamio.py +++ b/rpython/rlib/streamio.py @@ -900,6 +900,13 @@ return '\n'.join(result) + def readline(self): + line = self.base.readline() + limit = len(line) - 2 + if limit >= 0 and line[limit] == '\r' and line[limit + 1] == '\n': + line = line[:limit] + '\n' + return line + def tell(self): pos = self.base.tell() return pos - len(self.lfbuffer) diff --git a/rpython/rlib/test/test_rfile.py b/rpython/rlib/test/test_rfile.py --- a/rpython/rlib/test/test_rfile.py +++ b/rpython/rlib/test/test_rfile.py @@ -25,61 +25,67 @@ f.close() f() + assert open(fname, "r").read() == "dupa" self.interpret(f, []) assert open(fname, "r").read() == "dupa" def test_open_errors(self): - def f(exc): - def g(run): - try: - open('zzz', 'badmode') - except ValueError: - pass + def f(run): + try: + open('zzz', 'badmode') + except ValueError: + pass + else: + assert False + + try: + open('zzz') + except IOError as e: + assert e.errno == errno.ENOENT + else: + assert False + + try: + open('.') + except IOError as e: + if os.name == 'posix': + assert e.errno == errno.EISDIR else: - assert False + assert e.errno == errno.EACCES + else: + assert False - try: - open('zzz') - except exc as e: - assert e.errno == errno.ENOENT - else: - assert False + try: + os.fdopen(42, "badmode") + except ValueError: + pass + else: + assert False - try: - open('.') - except exc as e: - if os.name == 'posix': + try: + fd = os.open('.', os.O_RDONLY, 0777) + except OSError as e: + assert os.name == 'nt' and e.errno == errno.EACCES + else: + assert os.name != 'nt' + if run: + try: + os.fdopen(fd) + except IOError as e: assert e.errno == errno.EISDIR else: - assert e.errno == errno.EACCES - else: - assert False + assert False + os.close(fd) - try: - os.fdopen(42, "badmode") - except ValueError: - pass - else: - assert False + try: + os.fdopen(12345) + except OSError as e: + assert e.errno == errno.EBADF + else: + assert False - try: - fd = os.open('.', os.O_RDONLY, 0777) - except OSError as e: - assert os.name == 'nt' and e.errno == errno.EACCES - else: - assert os.name != 'nt' - if run: - try: - os.fdopen(fd) - except exc as e: - assert e.errno == errno.EISDIR - else: - assert False - os.close(fd) - return g - - f(IOError)(sys.version_info >= (2, 7, 9)) - self.interpret(f(OSError), [True]) + f(sys.version_info >= (2, 7, 9)) + self.interpret(f, [True]) @py.test.mark.skipif("sys.platform == 'win32'") # http://msdn.microsoft.com/en-us/library/86cebhfs.aspx @@ -120,6 +126,12 @@ def f(): f = open(fname, "w") + try: + f.read() + except IOError as e: + pass + else: + assert False f.write("dupa\x00dupb") f.close() for mode in ['r', 'U']: @@ -162,6 +174,7 @@ assert d == "a" assert e == "" + f() self.interpret(f, []) def test_seek(self): @@ -172,6 +185,12 @@ f.write("xxx") f.seek(0) assert f.read() == "xxx" + try: + f.seek(0, 42) + except IOError as e: + assert e.errno == errno.EINVAL + else: + assert False f.close() f() @@ -214,6 +233,8 @@ finally: f.close() + res = f() + assert res > 2 res = self.interpret(f, []) assert res > 2 @@ -228,6 +249,8 @@ finally: f.close() + res = f() + assert res == 3 res = self.interpret(f, []) assert res == 3 @@ -243,6 +266,7 @@ f2.close() f.close() + f() self.interpret(f, []) def test_truncate(self): diff --git a/rpython/rtyper/exceptiondata.py b/rpython/rtyper/exceptiondata.py --- a/rpython/rtyper/exceptiondata.py +++ b/rpython/rtyper/exceptiondata.py @@ -44,13 +44,6 @@ classdef = bk.getuniqueclassdef(cls) rclass.getclassrepr(rtyper, classdef).setup() - def make_raise_OSError(self, rtyper): - # ll_raise_OSError(errno) - def ll_raise_OSError(errno): - raise OSError(errno, None) - helper_fn = rtyper.annotate_helper_fn(ll_raise_OSError, [annmodel.SomeInteger()]) - return helper_fn - def get_standard_ll_exc_instance(self, rtyper, clsdef): from rpython.rtyper.lltypesystem.rclass import getinstancerepr r_inst = getinstancerepr(rtyper, clsdef) @@ -69,7 +62,6 @@ # create helper functionptrs self.fn_exception_match = self.make_exception_matcher(rtyper) self.fn_type_of_exc_inst = self.make_type_of_exc_inst(rtyper) - self.fn_raise_OSError = self.make_raise_OSError(rtyper) def make_exception_matcher(self, rtyper): # ll_exception_matcher(real_exception_vtable, match_exception_vtable) diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py --- a/rpython/rtyper/llinterp.py +++ b/rpython/rtyper/llinterp.py @@ -441,12 +441,8 @@ extraargs = () typer = self.llinterpreter.typer exdata = typer.exceptiondata - if isinstance(exc, OSError): - self.op_direct_call(exdata.fn_raise_OSError, exc.errno) - assert False, "op_direct_call above should have raised" - else: - evalue = exdata.get_standard_ll_exc_instance_by_class(exc.__class__) - etype = self.op_direct_call(exdata.fn_type_of_exc_inst, evalue) + evalue = exdata.get_standard_ll_exc_instance_by_class(exc.__class__) + etype = self.op_direct_call(exdata.fn_type_of_exc_inst, evalue) raise LLException(etype, evalue, *extraargs) def invoke_callable_with_pyexceptions(self, fptr, *args): diff --git a/rpython/rtyper/rbuiltin.py b/rpython/rtyper/rbuiltin.py --- a/rpython/rtyper/rbuiltin.py +++ b/rpython/rtyper/rbuiltin.py @@ -3,9 +3,10 @@ from rpython.rlib import rarithmetic, objectmodel from rpython.rtyper import raddress, rptr, extregistry, rrange from rpython.rtyper.error import TyperError -from rpython.rtyper.lltypesystem import lltype, llmemory, rclass +from rpython.rtyper.lltypesystem import lltype, llmemory, rclass, rstr from rpython.rtyper.lltypesystem.rdict import rtype_r_dict from rpython.rtyper.rmodel import Repr +from rpython.rtyper.rstr import AbstractStringRepr from rpython.tool.pairtype import pairtype @@ -264,16 +265,21 @@ def rtype_object__init__(hop): hop.exception_cannot_occur() -def rtype_OSError__init__(hop): +def rtype_EnvironmentError__init__(hop): hop.exception_cannot_occur() - if hop.nb_args == 2: - raise TyperError("OSError() should not be called with " - "a single argument") + v_self = hop.args_v[0] + r_self = hop.args_r[0] + if hop.nb_args >= 2: + v_errno = hop.inputarg(lltype.Signed, arg=1) + else: + v_errno = hop.inputconst(lltype.Signed, 0) + r_self.setfield(v_self, 'errno', v_errno, hop.llops) if hop.nb_args >= 3: - v_self = hop.args_v[0] - r_self = hop.args_r[0] - v_errno = hop.inputarg(lltype.Signed, arg=1) - r_self.setfield(v_self, 'errno', v_errno, hop.llops) + v_strerror = hop.inputarg(rstr.string_repr, arg=2) + r_self.setfield(v_self, 'strerror', v_strerror, hop.llops) + if hop.nb_args >= 4: + v_filename = hop.inputarg(rstr.string_repr, arg=3) + r_self.setfield(v_self, 'filename', v_filename, hop.llops) def rtype_WindowsError__init__(hop): hop.exception_cannot_occur() @@ -333,8 +339,8 @@ original = getattr(__builtin__, name[14:]) BUILTIN_TYPER[original] = value -BUILTIN_TYPER[getattr(OSError.__init__, 'im_func', OSError.__init__)] = ( - rtype_OSError__init__) +BUILTIN_TYPER[getattr(EnvironmentError.__init__, 'im_func', EnvironmentError.__init__)] = ( + rtype_EnvironmentError__init__) try: WindowsError diff --git a/rpython/rtyper/test/test_exception.py b/rpython/rtyper/test/test_exception.py --- a/rpython/rtyper/test/test_exception.py +++ b/rpython/rtyper/test/test_exception.py @@ -36,14 +36,53 @@ class TestException(BaseRtypingTest): def test_exception_with_arg(self): def g(n): - raise OSError(n, "?") + raise IOError(n) + def h(n): + raise OSError(n, "?", None) + def i(n): + raise EnvironmentError(n, "?", "test") + def j(n): + raise IOError(0, "test") + def k(n): + raise OSError def f(n): try: g(n) + except IOError, e: + assert e.errno == 42 + assert e.strerror is None + assert e.filename is None + else: + assert False + try: + h(n) except OSError, e: - return e.errno - res = self.interpret(f, [42]) - assert res == 42 + assert e.errno == 42 + assert e.strerror == "?" + assert e.filename is None + else: + assert False + try: + i(n) + except EnvironmentError as e: + assert e.errno == 42 + assert e.strerror == "?" + assert e.filename == "test" + else: + assert False + try: + j(n) + except (IOError, OSError) as e: + assert e.errno == 0 + assert e.strerror == "test" + assert e.filename is None + try: + k(n) + except EnvironmentError as e: + assert e.errno == 0 + assert e.strerror is None + assert e.filename is None + self.interpret(f, [42]) def test_catch_incompatible_class(self): class MyError(Exception): diff --git a/rpython/translator/c/extfunc.py b/rpython/translator/c/extfunc.py --- a/rpython/translator/c/extfunc.py +++ b/rpython/translator/c/extfunc.py @@ -90,7 +90,6 @@ yield ('RPYTHON_EXCEPTION_MATCH', exceptiondata.fn_exception_match) yield ('RPYTHON_TYPE_OF_EXC_INST', exceptiondata.fn_type_of_exc_inst) - yield ('RPYTHON_RAISE_OSERROR', exceptiondata.fn_raise_OSError) yield ('RPyExceptionOccurred1', exctransformer.rpyexc_occured_ptr.value) yield ('RPyFetchExceptionType', exctransformer.rpyexc_fetch_type_ptr.value) From noreply at buildbot.pypy.org Sun Sep 7 18:50:16 2014 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 7 Sep 2014 18:50:16 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: fix codewriter for nested structs (a bit hackish) Message-ID: <20140907165016.7EC031C072F@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73360:be3eeb6dcff1 Date: 2014-09-07 10:49 -0600 http://bitbucket.org/pypy/pypy/changeset/be3eeb6dcff1/ Log: fix codewriter for nested structs (a bit hackish) diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py --- a/rpython/jit/codewriter/jtransform.py +++ b/rpython/jit/codewriter/jtransform.py @@ -630,7 +630,12 @@ c_null = Constant(FIELD._defl(), FIELD) op = SpaceOperation('setfield', [v, c_name, c_null], None) - self.extend_with(ops, self.rewrite_op_setfield(op)) + self.extend_with(ops, self.rewrite_op_setfield(op, + override_type=TYPE)) + elif isinstance(FIELD, lltype.Struct): + # substruct + self.zero_contents(ops, v, FIELD, + only_gc_pointers=only_gc_pointers) elif isinstance(TYPE, lltype.Array): arraydescr = self.cpu.arraydescrof(TYPE) ops.append(SpaceOperation('clear_array_contents', @@ -793,13 +798,17 @@ op1] return op1 - def rewrite_op_setfield(self, op): + def rewrite_op_setfield(self, op, override_type=None): if self.is_typeptr_getset(op): # ignore the operation completely -- instead, it's done by 'new' return # turn the flow graph 'setfield' operation into our own version [v_inst, c_fieldname, v_value] = op.args RESULT = v_value.concretetype + if override_type is not None: + TYPE = override_type + else: + TYPE = v_inst.concretetype.TO if RESULT is lltype.Void: return # check for virtualizable @@ -809,10 +818,12 @@ return [SpaceOperation('-live-', [], None), SpaceOperation('setfield_vable_%s' % kind, [v_inst, v_value, descr], None)] - self.check_field_access(v_inst.concretetype.TO) - argname = getattr(v_inst.concretetype.TO, '_gckind', 'gc') - descr = self.cpu.fielddescrof(v_inst.concretetype.TO, - c_fieldname.value) + self.check_field_access(TYPE) + if override_type: + argname = 'gc' + else: + argname = getattr(TYPE, '_gckind', 'gc') + descr = self.cpu.fielddescrof(TYPE, c_fieldname.value) kind = getkind(RESULT)[0] if argname == 'raw' and kind == 'r': raise Exception("setfield_raw_r not supported") diff --git a/rpython/jit/codewriter/test/test_jtransform.py b/rpython/jit/codewriter/test/test_jtransform.py --- a/rpython/jit/codewriter/test/test_jtransform.py +++ b/rpython/jit/codewriter/test/test_jtransform.py @@ -553,6 +553,22 @@ assert op2.opname == 'setfield_gc_i' assert op2.args[0] == v +def test_malloc_new_zero_nested(): + S0 = lltype.GcStruct('S0') + S = lltype.Struct('S', ('x', lltype.Ptr(S0))) + S2 = lltype.GcStruct('S2', ('parent', S), + ('xx', lltype.Ptr(S0))) + v = varoftype(lltype.Ptr(S2)) + op = SpaceOperation('malloc', [Constant(S2, lltype.Void), + Constant({'flavor': 'gc'}, lltype.Void)], v) + op1, op2, op3 = Transformer(FakeCPU()).rewrite_operation(op) + assert op1.opname == 'new' + assert op1.args == [('sizedescr', S2)] + assert op2.opname == 'setfield_gc_r' + assert op2.args[0] == v + assert op3.opname == 'setfield_gc_r' + assert op3.args[0] == v + def test_malloc_new_with_vtable(): vtable = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True) S = lltype.GcStruct('S', ('parent', rclass.OBJECT)) From noreply at buildbot.pypy.org Sun Sep 7 19:39:18 2014 From: noreply at buildbot.pypy.org (waedt) Date: Sun, 7 Sep 2014 19:39:18 +0200 (CEST) Subject: [pypy-commit] pypy utf8-unicode2: Fix translation, but better Message-ID: <20140907173918.6ABD11C0A66@cobra.cs.uni-duesseldorf.de> Author: Tyler Wade Branch: utf8-unicode2 Changeset: r73361:53178aaa779b Date: 2014-09-07 12:38 -0500 http://bitbucket.org/pypy/pypy/changeset/53178aaa779b/ Log: Fix translation, but better diff --git a/pypy/interpreter/utf8.py b/pypy/interpreter/utf8.py --- a/pypy/interpreter/utf8.py +++ b/pypy/interpreter/utf8.py @@ -4,7 +4,6 @@ from rpython.rlib.runicode import utf8_code_length from rpython.rlib.unicodedata import unicodedb_5_2_0 as unicodedb from rpython.rlib.rarithmetic import r_uint, intmask, base_int -from rpython.rlib.jit import elidable from rpython.rtyper.lltypesystem import rffi, lltype from rpython.tool.sourcetools import func_with_new_name @@ -190,7 +189,6 @@ return byte_pos - def char_index_of_byte(self, byte_pos): if self._is_ascii: return byte_pos @@ -268,6 +266,8 @@ is_ascii = False stop_byte += increment + assert start_byte >= 0 + assert stop_byte >= 0 return Utf8Str(self.bytes[start_byte:stop_byte], is_ascii, stop - start) @@ -399,6 +399,7 @@ if start < 0: return -1 + assert end >= 0 if isinstance(other, Utf8Str): pos = self.bytes.find(other.bytes, start, end) elif isinstance(other, str): @@ -419,6 +420,7 @@ if start < 0: return -1 + assert end >= 0 if isinstance(other, Utf8Str): pos = self.bytes.rfind(other.bytes, start, end) elif isinstance(other, unicode): @@ -438,6 +440,7 @@ if start < 0: return 0 + assert end >= 0 if isinstance(other, Utf8Str): count = self.bytes.count(other.bytes, start, end) elif isinstance(other, unicode): From noreply at buildbot.pypy.org Sun Sep 7 21:14:56 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Sun, 7 Sep 2014 21:14:56 +0200 (CEST) Subject: [pypy-commit] pypy default: rfile fdopen needs validate_fd for win32 Message-ID: <20140907191456.9BF671C3382@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r73362:7026746cbb1b Date: 2014-09-07 12:14 -0700 http://bitbucket.org/pypy/pypy/changeset/7026746cbb1b/ Log: rfile fdopen needs validate_fd for win32 diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py --- a/rpython/rlib/rfile.py +++ b/rpython/rlib/rfile.py @@ -155,9 +155,11 @@ def create_fdopen_rfile(fd, mode="r"): mode = _sanitize_mode(mode) + fd = rffi.cast(rffi.INT, fd) + rposix.validate_fd(fd) ll_mode = rffi.str2charp(mode) try: - ll_file = c_fdopen(rffi.cast(rffi.INT, fd), ll_mode) + ll_file = c_fdopen(fd, ll_mode) if not ll_file: errno = rposix.get_errno() raise OSError(errno, os.strerror(errno)) diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -119,7 +119,7 @@ return 1 def validate_fd(fd): - return 1 + pass def closerange(fd_low, fd_high): # this behaves like os.closerange() from Python 2.6. From noreply at buildbot.pypy.org Sun Sep 7 21:39:42 2014 From: noreply at buildbot.pypy.org (mattip) Date: Sun, 7 Sep 2014 21:39:42 +0200 (CEST) Subject: [pypy-commit] pypy release-2.4.x: Added tag release-2.4-beta1 for changeset 9f425c60afdf Message-ID: <20140907193942.8853D1C072F@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: release-2.4.x Changeset: r73363:a631385c7117 Date: 2014-09-07 22:38 +0300 http://bitbucket.org/pypy/pypy/changeset/a631385c7117/ Log: Added tag release-2.4-beta1 for changeset 9f425c60afdf diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -14,3 +14,4 @@ 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.3.1 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1 0000000000000000000000000000000000000000 release-2.2=3.1 +9f425c60afdf2c1d38ee016071cffc55c28048b9 release-2.4-beta1 From noreply at buildbot.pypy.org Sun Sep 7 22:49:02 2014 From: noreply at buildbot.pypy.org (mattip) Date: Sun, 7 Sep 2014 22:49:02 +0200 (CEST) Subject: [pypy-commit] jitviewer default: Added tag pypy-2.4 for changeset 62ad3e746dac Message-ID: <20140907204902.C23511C072F@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r259:0184a76341f4 Date: 2014-09-07 23:49 +0300 http://bitbucket.org/pypy/jitviewer/changeset/0184a76341f4/ Log: Added tag pypy-2.4 for changeset 62ad3e746dac diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -1,3 +1,4 @@ 24adc3403cd8fdcd9e3f76f31a8dc2c145471002 release-0.1 13e1f8c97ca7c47f807ea93f44392c3f48102675 pypy-1.9 15e03325a227c4c7145a56e841b6a8a3c59730ed pypy-2.2 +62ad3e746dacc21c8e5dff2a37738659e1b61b7a pypy-2.4 From noreply at buildbot.pypy.org Sun Sep 7 22:49:03 2014 From: noreply at buildbot.pypy.org (mattip) Date: Sun, 7 Sep 2014 22:49:03 +0200 (CEST) Subject: [pypy-commit] jitviewer default: Added tag pypy-2.3 for changeset 62ad3e746dac Message-ID: <20140907204903.D45311C072F@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r260:0aed93257081 Date: 2014-09-07 23:49 +0300 http://bitbucket.org/pypy/jitviewer/changeset/0aed93257081/ Log: Added tag pypy-2.3 for changeset 62ad3e746dac diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -2,3 +2,4 @@ 13e1f8c97ca7c47f807ea93f44392c3f48102675 pypy-1.9 15e03325a227c4c7145a56e841b6a8a3c59730ed pypy-2.2 62ad3e746dacc21c8e5dff2a37738659e1b61b7a pypy-2.4 +62ad3e746dacc21c8e5dff2a37738659e1b61b7a pypy-2.3 From noreply at buildbot.pypy.org Sun Sep 7 23:31:59 2014 From: noreply at buildbot.pypy.org (mattip) Date: Sun, 7 Sep 2014 23:31:59 +0200 (CEST) Subject: [pypy-commit] pypy default: close and trunctate file on test failure Message-ID: <20140907213159.CAC971C0F86@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r73364:3e93f5de92ea Date: 2014-09-08 00:22 +0300 http://bitbucket.org/pypy/pypy/changeset/3e93f5de92ea/ Log: close and trunctate file on test failure diff --git a/lib-python/2.7/test/test_mmap.py b/lib-python/2.7/test/test_mmap.py --- a/lib-python/2.7/test/test_mmap.py +++ b/lib-python/2.7/test/test_mmap.py @@ -186,6 +186,10 @@ # later tests assume that the length hasn't changed. We need to # repair that. if sys.platform.startswith('win'): + f.close() + f = open(TESTFN, 'r+b') + f.truncate(mapsize) + f.close() self.fail("Opening mmap with size+1 should work on Windows.") else: # we expect a ValueError on Unix, but not on Windows From noreply at buildbot.pypy.org Mon Sep 8 00:04:21 2014 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 8 Sep 2014 00:04:21 +0200 (CEST) Subject: [pypy-commit] pypy default: start 2.4-beta1 release cycle Message-ID: <20140907220421.CA35A1C0F86@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r73365:52e09c6c7a8b Date: 2014-09-08 00:55 +0300 http://bitbucket.org/pypy/pypy/changeset/52e09c6c7a8b/ Log: start 2.4-beta1 release cycle diff --git a/pypy/doc/release-2.4.0.rst b/pypy/doc/release-2.4.0.rst --- a/pypy/doc/release-2.4.0.rst +++ b/pypy/doc/release-2.4.0.rst @@ -1,13 +1,14 @@ ================================================= -PyPy 2.4 - ???????? +PyPy 2.4 - Snow White ================================================= We're pleased to announce PyPy 2.4, which contains significant performance enhancements and bug fixes. -You can download the PyPy 2.4 release here: +You can already download the PyPy 2.4-beta1 pre-release here: - http://pypy.org/download.html + ~~http://pypy.org/download.html~~ + https://bitbucket.org/pypy/pypy/downloads We would like to thank our donors for the continued support of the PyPy project, and for those who donate to our three sub-projects. @@ -97,6 +98,9 @@ * Upgrade stdlib from 2.7.5 to 2.7.8 +* Win32 now links statically to zlib, expat, bzip, and openssl-1.0.1i. + No more missing DLLs + * Many issues were resolved_ since the 2.3.1 release on June 8 .. _`whats-new`: http://doc.pypy.org/en/latest/whatsnew-2.3.1.html diff --git a/pypy/doc/whatsnew-2.4.0.rst b/pypy/doc/whatsnew-2.4.0.rst --- a/pypy/doc/whatsnew-2.4.0.rst +++ b/pypy/doc/whatsnew-2.4.0.rst @@ -60,3 +60,7 @@ .. branch: stdlib-2.7.8 Upgrades from 2.7.6 to 2.7.8 + +.. branch: cpybug-seq-radd-rmul +Fix issue #1861 - cpython compatability madness + diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -4,5 +4,5 @@ ======================= .. this is a revision shortly after release-2.4.x -.. startrev: 204b550079b0 +.. startrev: 7026746cbb1b From noreply at buildbot.pypy.org Mon Sep 8 00:12:04 2014 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 8 Sep 2014 00:12:04 +0200 (CEST) Subject: [pypy-commit] pypy default: reformat Message-ID: <20140907221204.053661C0A66@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r73366:7806811a71d1 Date: 2014-09-08 01:11 +0300 http://bitbucket.org/pypy/pypy/changeset/7806811a71d1/ Log: reformat diff --git a/pypy/doc/release-2.4.0.rst b/pypy/doc/release-2.4.0.rst --- a/pypy/doc/release-2.4.0.rst +++ b/pypy/doc/release-2.4.0.rst @@ -7,8 +7,8 @@ You can already download the PyPy 2.4-beta1 pre-release here: - ~~http://pypy.org/download.html~~ - https://bitbucket.org/pypy/pypy/downloads +http://pypy.org/download.html +~~~~https://bitbucket.org/pypy/pypy/downloads~~~~ We would like to thank our donors for the continued support of the PyPy project, and for those who donate to our three sub-projects. From noreply at buildbot.pypy.org Mon Sep 8 00:22:43 2014 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 8 Sep 2014 00:22:43 +0200 (CEST) Subject: [pypy-commit] pypy default: give up tryng to get strikethrough in markdown Message-ID: <20140907222243.A40641C0A66@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r73367:6ea60888d8f7 Date: 2014-09-08 01:22 +0300 http://bitbucket.org/pypy/pypy/changeset/6ea60888d8f7/ Log: give up tryng to get strikethrough in markdown diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -6,6 +6,7 @@ .. toctree:: + release-2.4.0.rst release-2.3.1.rst release-2.3.0.rst release-2.2.1.rst diff --git a/pypy/doc/release-2.4.0.rst b/pypy/doc/release-2.4.0.rst --- a/pypy/doc/release-2.4.0.rst +++ b/pypy/doc/release-2.4.0.rst @@ -7,8 +7,11 @@ You can already download the PyPy 2.4-beta1 pre-release here: -http://pypy.org/download.html -~~~~https://bitbucket.org/pypy/pypy/downloads~~~~ + https://bitbucket.org/pypy/pypy/downloads + +It should also soon be available on: + + http://pypy.org/download.html We would like to thank our donors for the continued support of the PyPy project, and for those who donate to our three sub-projects. From noreply at buildbot.pypy.org Mon Sep 8 00:24:17 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 8 Sep 2014 00:24:17 +0200 (CEST) Subject: [pypy-commit] pypy default: use a try finally instead Message-ID: <20140907222417.A2A611C0A66@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r73368:596c682e4f53 Date: 2014-09-07 18:22 -0400 http://bitbucket.org/pypy/pypy/changeset/596c682e4f53/ Log: use a try finally instead diff --git a/lib-python/2.7/test/test_mmap.py b/lib-python/2.7/test/test_mmap.py --- a/lib-python/2.7/test/test_mmap.py +++ b/lib-python/2.7/test/test_mmap.py @@ -179,29 +179,27 @@ import sys f = open(TESTFN, "r+b") try: - m = mmap.mmap(f.fileno(), mapsize+1) - except ValueError: - # we do not expect a ValueError on Windows - # CAUTION: This also changes the size of the file on disk, and - # later tests assume that the length hasn't changed. We need to - # repair that. + try: + m = mmap.mmap(f.fileno(), mapsize+1) + except ValueError: + # we do not expect a ValueError on Windows + # CAUTION: This also changes the size of the file on disk, and + # later tests assume that the length hasn't changed. We need to + # repair that. + if sys.platform.startswith('win'): + self.fail("Opening mmap with size+1 should work on Windows.") + else: + # we expect a ValueError on Unix, but not on Windows + if not sys.platform.startswith('win'): + self.fail("Opening mmap with size+1 should raise ValueError.") + m.close() + finally: + f.close() if sys.platform.startswith('win'): - f.close() + # Repair damage from the resizing test. f = open(TESTFN, 'r+b') f.truncate(mapsize) f.close() - self.fail("Opening mmap with size+1 should work on Windows.") - else: - # we expect a ValueError on Unix, but not on Windows - if not sys.platform.startswith('win'): - self.fail("Opening mmap with size+1 should raise ValueError.") - m.close() - f.close() - if sys.platform.startswith('win'): - # Repair damage from the resizing test. - f = open(TESTFN, 'r+b') - f.truncate(mapsize) - f.close() # Opening mmap with access=ACCESS_WRITE f = open(TESTFN, "r+b") From noreply at buildbot.pypy.org Mon Sep 8 00:24:18 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 8 Sep 2014 00:24:18 +0200 (CEST) Subject: [pypy-commit] pypy default: merge heads Message-ID: <20140907222418.DF2CA1C0A66@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r73369:6861cf16a798 Date: 2014-09-07 18:24 -0400 http://bitbucket.org/pypy/pypy/changeset/6861cf16a798/ Log: merge heads diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -6,6 +6,7 @@ .. toctree:: + release-2.4.0.rst release-2.3.1.rst release-2.3.0.rst release-2.2.1.rst diff --git a/pypy/doc/release-2.4.0.rst b/pypy/doc/release-2.4.0.rst --- a/pypy/doc/release-2.4.0.rst +++ b/pypy/doc/release-2.4.0.rst @@ -7,8 +7,11 @@ You can already download the PyPy 2.4-beta1 pre-release here: -http://pypy.org/download.html -~~~~https://bitbucket.org/pypy/pypy/downloads~~~~ + https://bitbucket.org/pypy/pypy/downloads + +It should also soon be available on: + + http://pypy.org/download.html We would like to thank our donors for the continued support of the PyPy project, and for those who donate to our three sub-projects. From noreply at buildbot.pypy.org Mon Sep 8 00:41:52 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 8 Sep 2014 00:41:52 +0200 (CEST) Subject: [pypy-commit] pypy default: test file mode in rfile Message-ID: <20140907224152.A27271C072F@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r73370:44d6afd69be3 Date: 2014-09-07 18:41 -0400 http://bitbucket.org/pypy/pypy/changeset/44d6afd69be3/ Log: test file mode in rfile diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py --- a/rpython/rlib/rfile.py +++ b/rpython/rlib/rfile.py @@ -150,7 +150,7 @@ c_setvbuf(ll_file, buf, _IOLBF, BUFSIZ) else: c_setvbuf(ll_file, buf, _IOFBF, buffering) - return RFile(ll_file) + return RFile(ll_file, mode) def create_fdopen_rfile(fd, mode="r"): @@ -166,7 +166,7 @@ finally: lltype.free(ll_mode, flavor='raw') _dircheck(ll_file) - return RFile(ll_file) + return RFile(ll_file, mode) def create_temp_rfile(): @@ -190,12 +190,21 @@ lltype.free(ll_type, flavor='raw') finally: lltype.free(ll_command, flavor='raw') - return RFile(ll_file, _pclose2) + return RFile(ll_file, close2=_pclose2) class RFile(object): - def __init__(self, ll_file, close2=_fclose2): + _readable = False + _writable = False + + def __init__(self, ll_file, mode='+', close2=_fclose2): self._ll_file = ll_file + if 'r' in mode: + self._readable = True + if 'w' in mode or 'a' in mode: + self._writable = True + if '+' in mode: + self._readable = self._writable = True self._close2 = close2 def __del__(self): @@ -232,9 +241,18 @@ if not self._ll_file: raise ValueError("I/O operation on closed file") + def _check_reading(self): + if not self._readable: + raise IOError(0, "File not open for reading") + + def _check_writing(self): + if not self._writable: + raise IOError(0, "File not open for writing") + def read(self, size=-1): # XXX CPython uses a more delicate logic here self._check_closed() + self._check_reading() ll_file = self._ll_file if size == 0: return "" @@ -300,6 +318,7 @@ def readline(self, size=-1): self._check_closed() + self._check_reading() if size == 0: return "" elif size < 0: @@ -335,6 +354,7 @@ @enforceargs(None, str) def write(self, value): self._check_closed() + self._check_writing() ll_value = rffi.get_nonmovingbuffer(value) try: # note that since we got a nonmoving buffer, it is either raw @@ -356,6 +376,7 @@ def truncate(self, arg=-1): self._check_closed() + self._check_writing() if arg == -1: arg = self.tell() self.flush() diff --git a/rpython/rlib/test/test_rfile.py b/rpython/rlib/test/test_rfile.py --- a/rpython/rlib/test/test_rfile.py +++ b/rpython/rlib/test/test_rfile.py @@ -129,13 +129,25 @@ try: f.read() except IOError as e: - pass + assert not e.errno + else: + assert False + try: + f.readline() + except IOError as e: + assert not e.errno else: assert False f.write("dupa\x00dupb") f.close() for mode in ['r', 'U']: f2 = open(fname, mode) + try: + f2.write('') + except IOError as e: + assert not e.errno + else: + assert False dupa = f2.read(0) assert dupa == "" dupa = f2.read() @@ -215,6 +227,12 @@ new_fno = os.dup(f.fileno()) f2 = os.fdopen(new_fno, "w") f.close() + try: + f2.read() + except IOError as e: + assert not e.errno + else: + assert False f2.write("xxx") f2.close() @@ -281,6 +299,14 @@ data = f.read() assert data == "hello w" f.close() + f = open(fname) + try: + f.truncate() + except IOError as e: + assert not e.errno + else: + assert False + f.close() f() self.interpret(f, []) From noreply at buildbot.pypy.org Mon Sep 8 01:00:51 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 8 Sep 2014 01:00:51 +0200 (CEST) Subject: [pypy-commit] pypy use-file-star-for-file: merge default Message-ID: <20140907230051.3C9761C320C@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: use-file-star-for-file Changeset: r73371:224dfb42e83d Date: 2014-09-07 18:42 -0400 http://bitbucket.org/pypy/pypy/changeset/224dfb42e83d/ Log: merge default diff too long, truncating to 2000 out of 5130 lines diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -35,280 +35,290 @@ the beginning of each file) the files in the 'pypy' directory are each copyrighted by one or more of the following people and organizations: - Armin Rigo - Maciej Fijalkowski - Carl Friedrich Bolz - Antonio Cuni - Amaury Forgeot d'Arc - Samuele Pedroni - Alex Gaynor - Michael Hudson - David Schneider - Matti Picus - Brian Kearns - Philip Jenvey - Holger Krekel - Christian Tismer - Hakan Ardo - Benjamin Peterson - Manuel Jacob - Anders Chrigstrom - Eric van Riet Paap - Wim Lavrijsen - Ronan Lamy - Richard Emslie - Alexander Schremmer - Dan Villiom Podlaski Christiansen - Lukas Diekmann - Sven Hager - Anders Lehmann - Aurelien Campeas - Niklaus Haldimann - Camillo Bruni - Laura Creighton - Toon Verwaest - Remi Meier - Leonardo Santagada - Seo Sanghyeon - Romain Guillebert - Justin Peel - Ronny Pfannschmidt - David Edelsohn - Anders Hammarquist - Jakub Gustak - Guido Wesdorp - Lawrence Oluyede - Bartosz Skowron - Daniel Roberts - Niko Matsakis - Adrien Di Mascio - Alexander Hesse - Ludovic Aubry - Jacob Hallen - Jason Creighton - Alex Martelli - Michal Bendowski - Jan de Mooij - stian - Michael Foord - Stephan Diehl - Stefan Schwarzer - Valentino Volonghi - Tomek Meka - Patrick Maupin - Bob Ippolito - Bruno Gola - Jean-Paul Calderone - Timo Paulssen - Squeaky - Alexandre Fayolle - Simon Burton - Marius Gedminas - John Witulski - Konstantin Lopuhin - Greg Price - Dario Bertini - Mark Pearse - Simon Cross - Andreas Stührk - Jean-Philippe St. Pierre - Guido van Rossum - Pavel Vinogradov - Paweł Piotr Przeradowski - Paul deGrandis - Ilya Osadchiy - Tobias Oberstein - Adrian Kuhn - Boris Feigin - Stefano Rivera - tav - Taavi Burns - Georg Brandl - Bert Freudenberg - Stian Andreassen - Laurence Tratt - Wanja Saatkamp - Ivan Sichmann Freitas - Gerald Klix - Mike Blume - Oscar Nierstrasz - Stefan H. Muller - Jeremy Thurgood - Gregor Wegberg - Rami Chowdhury - Tobias Pape - Edd Barrett - David Malcolm - Eugene Oden - Henry Mason - Preston Timmons - Jeff Terrace - David Ripton - Dusty Phillips - Lukas Renggli - Guenter Jantzen - Ned Batchelder - Amit Regmi - Ben Young - Nicolas Chauvat - Andrew Durdin - Andrew Chambers - Michael Schneider - Nicholas Riley - Jason Chu - Igor Trindade Oliveira - Rocco Moretti - Gintautas Miliauskas - Michael Twomey - Lucian Branescu Mihaila - Tim Felgentreff - Tyler Wade - Gabriel Lavoie - Olivier Dormond - Jared Grubb - Karl Bartel - Brian Dorsey - Victor Stinner - Andrews Medina - Stuart Williams - Jasper Schulz - Christian Hudon - Toby Watson - Antoine Pitrou - Aaron Iles - Michael Cheng - Justas Sadzevicius - Mikael Schönenberg - Gasper Zejn - Neil Shepperd - Elmo Mäntynen - Jonathan David Riehl - Stanislaw Halik - Anders Qvist - Chirag Jadwani - Beatrice During - Alex Perry - Vincent Legoll - Alan McIntyre - Alexander Sedov - Corbin Simpson - Christopher Pope - wenzhuman - Christian Tismer - Marc Abramowitz - Dan Stromberg - Stefano Parmesan - Alexis Daboville - Jens-Uwe Mager - Carl Meyer - Karl Ramm - Pieter Zieschang - Gabriel - Lukas Vacek - Andrew Dalke - Sylvain Thenault - Nathan Taylor - Vladimir Kryachko - Jacek Generowicz - Alejandro J. Cura - Jacob Oscarson - Travis Francis Athougies - Ryan Gonzalez - Kristjan Valur Jonsson - Sebastian Pawluś - Neil Blakey-Milner - anatoly techtonik - Lutz Paelike - Lucio Torre - Lars Wassermann - Henrik Vendelbo - Dan Buch - Miguel de Val Borro - Artur Lisiecki - Sergey Kishchenko - Ignas Mikalajunas - Christoph Gerum - Martin Blais - Lene Wagner - Tomo Cocoa - roberto at goyle - Yury V. Zaytsev - Anna Katrina Dominguez - William Leslie - Bobby Impollonia - timo at eistee.fritz.box - Andrew Thompson - Ben Darnell - Roberto De Ioris - Juan Francisco Cantero Hurtado - Godefroid Chappelle - Joshua Gilbert - Dan Colish - Christopher Armstrong - Michael Hudson-Doyle - Anders Sigfridsson - Yasir Suhail - rafalgalczynski at gmail.com - Floris Bruynooghe - Laurens Van Houtven - Akira Li - Gustavo Niemeyer - Stephan Busemann - Rafał Gałczyński - Yusei Tahara - Christian Muirhead - James Lan - shoma hosaka - Daniel Neuh?user - Matthew Miller - Buck Golemon - Konrad Delong - Dinu Gherman - Chris Lambacher - coolbutuseless at gmail.com - Rodrigo Araújo - w31rd0 - Jim Baker - James Robert - Armin Ronacher - Brett Cannon - yrttyr - aliceinwire - OlivierBlanvillain - Zooko Wilcox-O Hearn - Tomer Chachamu - Christopher Groskopf - Asmo Soinio - Stefan Marr - jiaaro - opassembler.py - Antony Lee - Jim Hunziker - Markus Unterwaditzer - Even Wiik Thomassen - jbs - soareschen - Kurt Griffiths - Mike Bayer - Flavio Percoco - Kristoffer Kleine - yasirs - Michael Chermside - Anna Ravencroft - Julien Phalip - Dan Loewenherz + Armin Rigo + Maciej Fijalkowski + Carl Friedrich Bolz + Antonio Cuni + Amaury Forgeot d'Arc + Samuele Pedroni + Alex Gaynor + Michael Hudson + David Schneider + Matti Picus + Brian Kearns + Philip Jenvey + Holger Krekel + Christian Tismer + Hakan Ardo + Benjamin Peterson + Manuel Jacob + Anders Chrigstrom + Eric van Riet Paap + Ronan Lamy + Wim Lavrijsen + Richard Emslie + Alexander Schremmer + Dan Villiom Podlaski Christiansen + Lukas Diekmann + Sven Hager + Anders Lehmann + Aurelien Campeas + Niklaus Haldimann + Remi Meier + Camillo Bruni + Laura Creighton + Toon Verwaest + Leonardo Santagada + Seo Sanghyeon + Romain Guillebert + Justin Peel + Ronny Pfannschmidt + David Edelsohn + Anders Hammarquist + Jakub Gustak + Guido Wesdorp + Lawrence Oluyede + Bartosz Skowron + Gregor Wegberg + Daniel Roberts + Niko Matsakis + Adrien Di Mascio + Alexander Hesse + Ludovic Aubry + Jacob Hallen + Jason Creighton + Alex Martelli + Michal Bendowski + Jan de Mooij + stian + Michael Foord + Stephan Diehl + Tyler Wade + Stefan Schwarzer + Valentino Volonghi + Tomek Meka + Patrick Maupin + Bob Ippolito + Bruno Gola + Jean-Paul Calderone + Timo Paulssen + Squeaky + Alexandre Fayolle + Simon Burton + Marius Gedminas + Martin Matusiak + Konstantin Lopuhin + John Witulski + Wenzhu Man + Greg Price + Dario Bertini + Mark Pearse + Simon Cross + Ivan Sichmann Freitas + Andreas Stührk + Jean-Philippe St. Pierre + Guido van Rossum + Pavel Vinogradov + Stefano Rivera + Paweł Piotr Przeradowski + Paul deGrandis + Ilya Osadchiy + Tobias Oberstein + Adrian Kuhn + Boris Feigin + tav + Taavi Burns + Georg Brandl + Laurence Tratt + Bert Freudenberg + Stian Andreassen + Wanja Saatkamp + Gerald Klix + Mike Blume + Oscar Nierstrasz + Stefan H. Muller + Edd Barrett + Jeremy Thurgood + Rami Chowdhury + Tobias Pape + David Malcolm + Eugene Oden + Henry Mason + Vasily Kuznetsov + Preston Timmons + Jeff Terrace + David Ripton + Dusty Phillips + Lukas Renggli + Guenter Jantzen + Ned Batchelder + Amit Regmi + Ben Young + Nicolas Chauvat + Andrew Durdin + Andrew Chambers + Michael Schneider + Nicholas Riley + Jason Chu + Igor Trindade Oliveira + Tim Felgentreff + Rocco Moretti + Gintautas Miliauskas + Michael Twomey + Lucian Branescu Mihaila + Gabriel Lavoie + Olivier Dormond + Jared Grubb + Karl Bartel + Brian Dorsey + Victor Stinner + Andrews Medina + Stuart Williams + Jasper Schulz + Christian Hudon + Toby Watson + Antoine Pitrou + Aaron Iles + Michael Cheng + Justas Sadzevicius + Gasper Zejn + anatoly techtonik + Neil Shepperd + Mikael Schönenberg + Elmo M?ntynen + Jonathan David Riehl + Stanislaw Halik + Anders Qvist + Corbin Simpson + Chirag Jadwani + Beatrice During + Alex Perry + Vincent Legoll + Alan McIntyre + Alexander Sedov + Christopher Pope + Christian Tismer + Marc Abramowitz + Dan Stromberg + Stefano Parmesan + Alexis Daboville + Jens-Uwe Mager + Carl Meyer + Karl Ramm + Pieter Zieschang + Sebastian Pawluś + Gabriel + Lukas Vacek + Andrew Dalke + Sylvain Thenault + Nathan Taylor + Vladimir Kryachko + Arjun Naik + Attila Gobi + Jacek Generowicz + Alejandro J. Cura + Jacob Oscarson + Travis Francis Athougies + Ryan Gonzalez + Ian Foote + Kristjan Valur Jonsson + Neil Blakey-Milner + Lutz Paelike + Lucio Torre + Lars Wassermann + Valentina Mukhamedzhanova + Henrik Vendelbo + Dan Buch + Miguel de Val Borro + Artur Lisiecki + Sergey Kishchenko + Yichao Yu + Ignas Mikalajunas + Christoph Gerum + Martin Blais + Lene Wagner + Tomo Cocoa + roberto at goyle + Yury V. Zaytsev + Anna Katrina Dominguez + William Leslie + Bobby Impollonia + timo at eistee.fritz.box + Andrew Thompson + Yusei Tahara + Ben Darnell + Roberto De Ioris + Juan Francisco Cantero Hurtado + Godefroid Chappelle + Joshua Gilbert + Dan Colish + Christopher Armstrong + Michael Hudson-Doyle + Anders Sigfridsson + Yasir Suhail + Jason Michalski + rafalgalczynski at gmail.com + Floris Bruynooghe + Laurens Van Houtven + Akira Li + Gustavo Niemeyer + Stephan Busemann + Rafał Gałczyński + Christian Muirhead + James Lan + shoma hosaka + Daniel Neuh?user + Matthew Miller + Buck Golemon + Konrad Delong + Dinu Gherman + Chris Lambacher + coolbutuseless at gmail.com + Rodrigo Araújo + Jim Baker + James Robert + Armin Ronacher + Brett Cannon + yrttyr + aliceinwire + OlivierBlanvillain + Zooko Wilcox-O Hearn + Tomer Chachamu + Christopher Groskopf + Asmo Soinio + Stefan Marr + jiaaro + Mads Kiilerich + opassembler.py + Antony Lee + Jim Hunziker + Markus Unterwaditzer + Even Wiik Thomassen + jbs + soareschen + Kurt Griffiths + Mike Bayer + Matthew Miller + Flavio Percoco + Kristoffer Kleine + yasirs + Michael Chermside + Anna Ravencroft + Dan Crosta + Julien Phalip + Dan Loewenherz - Heinrich-Heine University, Germany - Open End AB (formerly AB Strakt), Sweden - merlinux GmbH, Germany - tismerysoft GmbH, Germany - Logilab Paris, France - DFKI GmbH, Germany - Impara, Germany - Change Maker, Sweden - University of California Berkeley, USA - Google Inc. - King's College London + Heinrich-Heine University, Germany + Open End AB (formerly AB Strakt), Sweden + merlinux GmbH, Germany + tismerysoft GmbH, Germany + Logilab Paris, France + DFKI GmbH, Germany + Impara, Germany + Change Maker, Sweden + University of California Berkeley, USA + Google Inc. + King's College London The PyPy Logo as used by http://speed.pypy.org and others was created by Samuel Reis and is distributed on terms of Creative Commons Share Alike diff --git a/_pytest/README-BEFORE-UPDATING b/_pytest/README-BEFORE-UPDATING new file mode 100644 --- /dev/null +++ b/_pytest/README-BEFORE-UPDATING @@ -0,0 +1,17 @@ +This is PyPy's code of the pytest lib. We don't expect to upgrade it +very often, but once we do: + + WARNING! + + WE HAVE MADE A FEW TWEAKS HERE! + +Please be sure that you don't just copy the newer version from +upstream without checking the few changes that we did. This +can be done like this: + + cd + hg log . -v | less + +then search for all " _pytest/" in that list to know which are the +relevant checkins. (Look for the checkins that only edit one +or two files in this directory.) diff --git a/_pytest/resultlog.py b/_pytest/resultlog.py --- a/_pytest/resultlog.py +++ b/_pytest/resultlog.py @@ -53,16 +53,24 @@ self.config = config self.logfile = logfile # preferably line buffered - def write_log_entry(self, testpath, lettercode, longrepr): - py.builtin.print_("%s %s" % (lettercode, testpath), file=self.logfile) + def write_log_entry(self, testpath, lettercode, longrepr, sections=None): + _safeprint("%s %s" % (lettercode, testpath), file=self.logfile) for line in longrepr.splitlines(): - py.builtin.print_(" %s" % line, file=self.logfile) + _safeprint(" %s" % line, file=self.logfile) + if sections is not None and ( + lettercode in ('E', 'F')): # to limit the size of logs + for title, content in sections: + _safeprint(" ---------- %s ----------" % (title,), + file=self.logfile) + for line in content.splitlines(): + _safeprint(" %s" % line, file=self.logfile) def log_outcome(self, report, lettercode, longrepr): testpath = getattr(report, 'nodeid', None) if testpath is None: testpath = report.fspath - self.write_log_entry(testpath, lettercode, longrepr) + self.write_log_entry(testpath, lettercode, longrepr, + getattr(report, 'sections', None)) def pytest_runtest_logreport(self, report): if report.when != "call" and report.passed: @@ -98,3 +106,8 @@ if path is None: path = "cwd:%s" % py.path.local() self.write_log_entry(path, '!', str(excrepr)) + +def _safeprint(s, file): + if isinstance(s, unicode): + s = s.encode('utf-8') + py.builtin.print_(s, file=file) diff --git a/lib-python/2.7/sysconfig.py b/lib-python/2.7/sysconfig.py --- a/lib-python/2.7/sysconfig.py +++ b/lib-python/2.7/sysconfig.py @@ -7,30 +7,30 @@ _INSTALL_SCHEMES = { 'posix_prefix': { - 'stdlib': '{base}/lib/python{py_version_short}', - 'platstdlib': '{platbase}/lib/python{py_version_short}', - 'purelib': '{base}/lib/python{py_version_short}/site-packages', - 'platlib': '{platbase}/lib/python{py_version_short}/site-packages', - 'include': '{base}/include/python{py_version_short}', - 'platinclude': '{platbase}/include/python{py_version_short}', + 'stdlib': '{base}/lib/{implementation_lower}{py_version_short}', + 'platstdlib': '{platbase}/lib/{implementation_lower}{py_version_short}', + 'purelib': '{base}/lib/{implementation_lower}{py_version_short}/site-packages', + 'platlib': '{platbase}/lib/{implementation_lower}{py_version_short}/site-packages', + 'include': '{base}/include/{implementation_lower}{py_version_short}', + 'platinclude': '{platbase}/include/{implementation_lower}{py_version_short}', 'scripts': '{base}/bin', 'data': '{base}', }, 'posix_home': { - 'stdlib': '{base}/lib/python', - 'platstdlib': '{base}/lib/python', - 'purelib': '{base}/lib/python', - 'platlib': '{base}/lib/python', - 'include': '{base}/include/python', - 'platinclude': '{base}/include/python', + 'stdlib': '{base}/lib/{implementation_lower}', + 'platstdlib': '{base}/lib/{implementation_lower}', + 'purelib': '{base}/lib/{implementation_lower}', + 'platlib': '{base}/lib/{implementation_lower}', + 'include': '{base}/include/{implementation_lower}', + 'platinclude': '{base}/include/{implementation_lower}', 'scripts': '{base}/bin', 'data' : '{base}', }, 'pypy': { - 'stdlib': '{base}/lib-python/{py_version_short}', - 'platstdlib': '{base}/lib-python/{py_version_short}', - 'purelib': '{base}/lib-python/{py_version_short}', - 'platlib': '{base}/lib-python/{py_version_short}', + 'stdlib': '{base}/lib-{implementation_lower}/{py_version_short}', + 'platstdlib': '{base}/lib-{implementation_lower}/{py_version_short}', + 'purelib': '{base}/lib-{implementation_lower}/{py_version_short}', + 'platlib': '{base}/lib-{implementation_lower}/{py_version_short}', 'include': '{base}/include', 'platinclude': '{base}/include', 'scripts': '{base}/bin', @@ -57,37 +57,37 @@ 'data' : '{base}', }, 'os2_home': { - 'stdlib': '{userbase}/lib/python{py_version_short}', - 'platstdlib': '{userbase}/lib/python{py_version_short}', - 'purelib': '{userbase}/lib/python{py_version_short}/site-packages', - 'platlib': '{userbase}/lib/python{py_version_short}/site-packages', - 'include': '{userbase}/include/python{py_version_short}', + 'stdlib': '{userbase}/lib/{implementation_lower}{py_version_short}', + 'platstdlib': '{userbase}/lib/{implementation_lower}{py_version_short}', + 'purelib': '{userbase}/lib/{implementation_lower}{py_version_short}/site-packages', + 'platlib': '{userbase}/lib/{implementation_lower}{py_version_short}/site-packages', + 'include': '{userbase}/include/{implementation_lower}{py_version_short}', 'scripts': '{userbase}/bin', 'data' : '{userbase}', }, 'nt_user': { - 'stdlib': '{userbase}/Python{py_version_nodot}', - 'platstdlib': '{userbase}/Python{py_version_nodot}', - 'purelib': '{userbase}/Python{py_version_nodot}/site-packages', - 'platlib': '{userbase}/Python{py_version_nodot}/site-packages', - 'include': '{userbase}/Python{py_version_nodot}/Include', + 'stdlib': '{userbase}/{implementation}{py_version_nodot}', + 'platstdlib': '{userbase}/{implementation}{py_version_nodot}', + 'purelib': '{userbase}/{implementation}{py_version_nodot}/site-packages', + 'platlib': '{userbase}/{implementation}{py_version_nodot}/site-packages', + 'include': '{userbase}/{implementation}{py_version_nodot}/Include', 'scripts': '{userbase}/Scripts', 'data' : '{userbase}', }, 'posix_user': { - 'stdlib': '{userbase}/lib/python{py_version_short}', - 'platstdlib': '{userbase}/lib/python{py_version_short}', - 'purelib': '{userbase}/lib/python{py_version_short}/site-packages', - 'platlib': '{userbase}/lib/python{py_version_short}/site-packages', - 'include': '{userbase}/include/python{py_version_short}', + 'stdlib': '{userbase}/lib/{implementation_lower}{py_version_short}', + 'platstdlib': '{userbase}/lib/{implementation_lower}{py_version_short}', + 'purelib': '{userbase}/lib/{implementation_lower}{py_version_short}/site-packages', + 'platlib': '{userbase}/lib/{implementation_lower}{py_version_short}/site-packages', + 'include': '{userbase}/include/{implementation_lower}{py_version_short}', 'scripts': '{userbase}/bin', 'data' : '{userbase}', }, 'osx_framework_user': { - 'stdlib': '{userbase}/lib/python', - 'platstdlib': '{userbase}/lib/python', - 'purelib': '{userbase}/lib/python/site-packages', - 'platlib': '{userbase}/lib/python/site-packages', + 'stdlib': '{userbase}/lib/{implementation_lower}', + 'platstdlib': '{userbase}/lib/{implementation_lower}', + 'purelib': '{userbase}/lib/{implementation_lower}/site-packages', + 'platlib': '{userbase}/lib/{implementation_lower}/site-packages', 'include': '{userbase}/include', 'scripts': '{userbase}/bin', 'data' : '{userbase}', @@ -104,6 +104,11 @@ _CONFIG_VARS = None _USER_BASE = None +def _get_implementation(): + if '__pypy__' in sys.builtin_module_names: + return 'PyPy' + return 'Python' + def _safe_realpath(path): try: return realpath(path) @@ -475,6 +480,8 @@ _CONFIG_VARS['base'] = _PREFIX _CONFIG_VARS['platbase'] = _EXEC_PREFIX _CONFIG_VARS['projectbase'] = _PROJECT_BASE + _CONFIG_VARS['implementation'] = _get_implementation() + _CONFIG_VARS['implementation_lower'] = _get_implementation().lower() if os.name in ('nt', 'os2'): _init_non_posix(_CONFIG_VARS) @@ -644,6 +651,8 @@ _print_dict('Paths', get_paths()) print _print_dict('Variables', get_config_vars()) + print + _print_dict('User', get_paths('%s_user' % os.name)) if __name__ == '__main__': diff --git a/lib-python/2.7/test/pickletester.py b/lib-python/2.7/test/pickletester.py --- a/lib-python/2.7/test/pickletester.py +++ b/lib-python/2.7/test/pickletester.py @@ -6,14 +6,16 @@ import pickletools import copy_reg -from test.test_support import TestFailed, verbose, have_unicode, TESTFN, impl_detail +from test.test_support import TestFailed, verbose, have_unicode, TESTFN try: - from test.test_support import _2G, _1M, precisionbigmemtest + from test.test_support import _2G, _1M, precisionbigmemtest, impl_detail except ImportError: # this import might fail when run on older Python versions by test_xpickle _2G = _1M = 0 def precisionbigmemtest(*args, **kwargs): return lambda self: None + def impl_detail(*args, **kwargs): + return lambda self: None # Tests that try a number of pickle protocols should have a # for proto in protocols: diff --git a/lib-python/2.7/test/test_mmap.py b/lib-python/2.7/test/test_mmap.py --- a/lib-python/2.7/test/test_mmap.py +++ b/lib-python/2.7/test/test_mmap.py @@ -179,25 +179,27 @@ import sys f = open(TESTFN, "r+b") try: - m = mmap.mmap(f.fileno(), mapsize+1) - except ValueError: - # we do not expect a ValueError on Windows - # CAUTION: This also changes the size of the file on disk, and - # later tests assume that the length hasn't changed. We need to - # repair that. + try: + m = mmap.mmap(f.fileno(), mapsize+1) + except ValueError: + # we do not expect a ValueError on Windows + # CAUTION: This also changes the size of the file on disk, and + # later tests assume that the length hasn't changed. We need to + # repair that. + if sys.platform.startswith('win'): + self.fail("Opening mmap with size+1 should work on Windows.") + else: + # we expect a ValueError on Unix, but not on Windows + if not sys.platform.startswith('win'): + self.fail("Opening mmap with size+1 should raise ValueError.") + m.close() + finally: + f.close() if sys.platform.startswith('win'): - self.fail("Opening mmap with size+1 should work on Windows.") - else: - # we expect a ValueError on Unix, but not on Windows - if not sys.platform.startswith('win'): - self.fail("Opening mmap with size+1 should raise ValueError.") - m.close() - f.close() - if sys.platform.startswith('win'): - # Repair damage from the resizing test. - f = open(TESTFN, 'r+b') - f.truncate(mapsize) - f.close() + # Repair damage from the resizing test. + f = open(TESTFN, 'r+b') + f.truncate(mapsize) + f.close() # Opening mmap with access=ACCESS_WRITE f = open(TESTFN, "r+b") diff --git a/py/README-BEFORE-UPDATING b/py/README-BEFORE-UPDATING new file mode 100644 --- /dev/null +++ b/py/README-BEFORE-UPDATING @@ -0,0 +1,17 @@ +This is PyPy's code of the py lib. We don't expect to upgrade it +very often, but once we do: + + WARNING! + + WE HAVE MADE A FEW TWEAKS HERE! + +Please be sure that you don't just copy the newer version from +upstream without checking the few changes that we did. This +can be done like this: + + cd + hg log . -v | less + +then search for all " py/" in that list to know which are the +relevant checkins. (Look for the checkins that only edit one +or two files in this directory.) diff --git a/py/_code/code.py b/py/_code/code.py --- a/py/_code/code.py +++ b/py/_code/code.py @@ -588,7 +588,7 @@ class TerminalRepr: def __str__(self): s = self.__unicode__() - if sys.version_info[0] < 3: + if sys.version_info[0] < 3 and isinstance(s, unicode): s = s.encode('utf-8') return s diff --git a/py/_path/local.py b/py/_path/local.py --- a/py/_path/local.py +++ b/py/_path/local.py @@ -750,7 +750,8 @@ mkdtemp = classmethod(mkdtemp) def make_numbered_dir(cls, prefix='session-', rootdir=None, keep=3, - lock_timeout = 172800): # two days + lock_timeout = 172800, # two days + min_timeout = 300): # five minutes """ return unique directory with a number greater than the current maximum one. The number is assumed to start directly after prefix. if keep is true directories with a number less than (maxnum-keep) @@ -818,6 +819,20 @@ for path in rootdir.listdir(): num = parse_num(path) if num is not None and num <= (maxnum - keep): + if min_timeout: + # NB: doing this is needed to prevent (or reduce + # a lot the chance of) the following situation: + # 'keep+1' processes call make_numbered_dir() at + # the same time, they create dirs, but then the + # last process notices the first dir doesn't have + # (yet) a .lock in it and kills it. + try: + t1 = path.lstat().mtime + t2 = lockfile.lstat().mtime + if abs(t2-t1) < min_timeout: + continue # skip directories too recent + except py.error.Error: + continue # failure to get a time, better skip lf = path.join('.lock') try: t1 = lf.lstat().mtime diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py --- a/pypy/config/pypyoption.py +++ b/pypy/config/pypyoption.py @@ -52,11 +52,10 @@ if sys.platform == "win32": working_modules.add("_winreg") # unix only modules - working_modules.remove("crypt") - working_modules.remove("fcntl") - working_modules.remove("pwd") - working_modules.remove("termios") - working_modules.remove("_minimal_curses") + for name in ["crypt", "fcntl", "pwd", "termios", "_minimal_curses"]: + working_modules.remove(name) + if name in translation_modules: + translation_modules.remove(name) if "cppyy" in working_modules: working_modules.remove("cppyy") # not tested on win32 diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -65,9 +65,9 @@ # built documents. # # The short X.Y version. -version = '2.3' +version = '2.4' # The full version, including alpha/beta/rc tags. -release = '2.3.0' +release = '2.4.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst --- a/pypy/doc/contributor.rst +++ b/pypy/doc/contributor.rst @@ -80,7 +80,7 @@ Martin Matusiak Konstantin Lopuhin John Witulski - wenzhu man + Wenzhu Man Greg Price Dario Bertini Mark Pearse diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -6,6 +6,7 @@ .. toctree:: + release-2.4.0.rst release-2.3.1.rst release-2.3.0.rst release-2.2.1.rst diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst --- a/pypy/doc/index.rst +++ b/pypy/doc/index.rst @@ -40,7 +40,7 @@ * `FAQ`_: some frequently asked questions. -* `Release 2.3.1`_: the latest official release +* `Release 2.4.0`_: the latest official release * `PyPy Blog`_: news and status info about PyPy @@ -110,7 +110,7 @@ .. _`Getting Started`: getting-started.html .. _`Papers`: extradoc.html .. _`Videos`: video-index.html -.. _`Release 2.3.1`: http://pypy.org/download.html +.. _`Release 2.4.0`: http://pypy.org/download.html .. _`speed.pypy.org`: http://speed.pypy.org .. _`RPython toolchain`: translation.html .. _`potential project ideas`: project-ideas.html diff --git a/pypy/doc/release-2.4.rst b/pypy/doc/release-2.4.0.rst rename from pypy/doc/release-2.4.rst rename to pypy/doc/release-2.4.0.rst --- a/pypy/doc/release-2.4.rst +++ b/pypy/doc/release-2.4.0.rst @@ -1,67 +1,75 @@ ================================================= -PyPy 2.4 - ???????? +PyPy 2.4 - Snow White ================================================= -We're pleased to announce PyPy 2.4, a significant milestone on it's own right -and the proud parent of our recent PyPy3 and STM releases. +We're pleased to announce PyPy 2.4, which contains significant performance +enhancements and bug fixes. -This release contains several improvements and bugfixes. +You can already download the PyPy 2.4-beta1 pre-release here: -You can download the PyPy 2.4 release here: + https://bitbucket.org/pypy/pypy/downloads + +It should also soon be available on: http://pypy.org/download.html We would like to thank our donors for the continued support of the PyPy project, and for those who donate to our three sub-projects. -We've shown quite a bit of progress -but we're slowly running out of funds. +We've shown quite a bit of progress, but we're slowly running out of funds. Please consider donating more, or even better convince your employer to donate, -so we can finish those projects! The three sub-projects are: +so we can finish those projects! We would like to also point out that in +September, `the Python Software Foundation`_ will `match funds`_ for +any donations up to $10k! The three sub-projects are: -* `Py3k`_ (supporting Python 3.x): We have released a Python 3.2.5 compatable version - we call PyPy3 2.3.1, and are working toward a Python 3.3 compatable version +* `Py3k`_ (supporting Python 3.x): We have released a Python 3.2.5 compatible version + we call PyPy3 2.3.1, and are working toward a Python 3.3 compatible version -* `STM`_ (software transactional memory): We have release a first working version, and -continue to try out new promising paths of acheiving a fast multithreaded python +* `STM`_ (software transactional memory): We have released a first working version, + and continue to try out new promising paths of achieving a fast multithreaded Python -* `NumPy`_ which requires installation of our fork of upstream numpy, available `on bitbucket`_ +* `NumPy`_ which requires installation of our fork of upstream numpy, + available `on bitbucket`_ .. _`Py3k`: http://pypy.org/py3donate.html .. _`STM`: http://pypy.org/tmdonate2.html .. _`NumPy`: http://pypy.org/numpydonate.html .. _`on bitbucket`: https://www.bitbucket.org/pypy/numpy +.. _`the Python Software Foundation`: https://www.python.org/psf/ +.. _`match funds`: http://morepypy.blogspot.com/2014/09/python-software-foundation-matching.html What is PyPy? ============= PyPy is a very compliant Python interpreter, almost a drop-in replacement for -CPython 2.7. It's fast (`pypy 2.3 and cpython 2.7.x`_ performance comparison; -note that cpython's speed has not changed since 2.7.2) +CPython 2.7. It's fast (`pypy 2.4 and cpython 2.7.x`_ performance comparison) due to its integrated tracing JIT compiler. -This release supports x86 machines running Linux 32/64, Mac OS X 64, Windows, -and OpenBSD, -as well as newer ARM hardware (ARMv6 or ARMv7, with VFPv3) running Linux. +This release supports **x86** machines on most common operating systems +(Linux 32/64, Mac OS X 64, Windows, and OpenBSD), +as well as newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux. While we support 32 bit python on Windows, work on the native Windows 64 bit python is still stalling, we would welcome a volunteer to `handle that`_. -.. _`pypy 2.3 and cpython 2.7.x`: http://speed.pypy.org +.. _`pypy 2.4 and cpython 2.7.x`: http://speed.pypy.org .. _`handle that`: http://doc.pypy.org/en/latest/windows.html#what-is-missing-for-a-full-64-bit-translation Highlights ========== -Benchmarks improved after internal improvements in string and bytearray handling, -and a major rewrite of the GIL handling. Many of these improvements are offshoots -of the STM work. +Benchmarks improved after internal enhancements in string and +bytearray handling, and a major rewrite of the GIL handling. This means +that external calls are now a lot faster, especially the CFFI ones. It also +means better performance in a lot of corner cases with handling strings or +bytearrays. -We merged in Python's 2.7.8 stdlib in a record time of one week, proving the -maturity of our underlying RPython code base and PyPy interpreter. +PyPy now uses Python 2.7.8 standard library. -We welcomed more than 12 new contributors, and conducted two Google Summer of Code -projects XXX details? +We welcomed more than 12 new contributors, and conducted two Google +Summer of Code projects, as well as other student projects not +directly related to Summer of Code. + Issues reported with our previous release were fixed after reports from users on our new issue tracker at https://bitbucket.org/pypy/pypy/issues or on IRC at @@ -71,33 +79,39 @@ * Reduced internal copying of bytearray operations * Tweak the internal structure of StringBuilder to speed up large string -handling, which becomes advantageous on large programs at the cost of slightly -slower small *benchmark* type programs. + handling, which becomes advantageous on large programs at the cost of slightly + slower small *benchmark* type programs. -* Boost performance of thread-local variables in both unjitted and jitted code +* Boost performance of thread-local variables in both unjitted and jitted code, + this mostly affects errno handling on linux, which makes external calls + faster. * Move to a mixed polling and mutex GIL model that make mutli-threaded jitted code run *much* faster -* Optimize errno handling in linux +* Optimize errno handling in linux (x86 and x86-64 only) * Remove ctypes pythonapi and ctypes.PyDLL, which never worked on PyPy * Fix performance regression on ufunc(, ) in numpy -* Classes in the ast module are now distinct from structures used by the compiler, - which simplifies and speeds up translation of our source code to the PyPy binary - interpreter +* Classes in the ast module are now distinct from structures used by + the compiler, which simplifies and speeds up translation of our + source code to the PyPy binary interpreter * Upgrade stdlib from 2.7.5 to 2.7.8 -* - +* Win32 now links statically to zlib, expat, bzip, and openssl-1.0.1i. + No more missing DLLs + * Many issues were resolved_ since the 2.3.1 release on June 8 .. _`whats-new`: http://doc.pypy.org/en/latest/whatsnew-2.3.1.html .. _resolved: https://bitbucket.org/pypy/pypy/issues?status=resolved +We have further improvements on the way: rpython file handling and +usable numpy linalg compatabiity should be merged soon. + Please try it out and let us know what you think. We especially welcome success stories, we know you are using PyPy, please tell us about it! diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py --- a/pypy/doc/tool/makecontributor.py +++ b/pypy/doc/tool/makecontributor.py @@ -68,7 +68,7 @@ 'Manuel Jacob': ['mjacob'], 'Rami Chowdhury': ['necaris'], 'Stanislaw Halik':['w31rd0'], - 'wenzhu man':['wenzhuman'], + 'Wenzhu Man':['wenzhu man', 'wenzhuman'], } alias_map = {} diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-2.4.0.rst copy from pypy/doc/whatsnew-head.rst copy to pypy/doc/whatsnew-2.4.0.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-2.4.0.rst @@ -60,3 +60,7 @@ .. branch: stdlib-2.7.8 Upgrades from 2.7.6 to 2.7.8 + +.. branch: cpybug-seq-radd-rmul +Fix issue #1861 - cpython compatability madness + diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -1,62 +1,8 @@ + ======================= -What's new in PyPy 2.4+ +What's new in PyPy 2.5+ ======================= -.. this is a revision shortly after release-2.3.x -.. startrev: ca9b7cf02cf4 +.. this is a revision shortly after release-2.4.x +.. startrev: 7026746cbb1b -.. branch: fix-bytearray-complexity -Bytearray operations no longer copy the bytearray unnecessarily - -Added support for ``__getitem__``, ``__setitem__``, ``__getslice__``, -``__setslice__``, and ``__len__`` to RPython - -.. branch: stringbuilder2-perf -Give the StringBuilder a more flexible internal structure, with a -chained list of strings instead of just one string. This make it -more efficient when building large strings, e.g. with cStringIO(). - -Also, use systematically jit.conditional_call() instead of regular -branches. This lets the JIT make more linear code, at the cost of -forcing a bit more data (to be passed as arguments to -conditional_calls). I would expect the net result to be a slight -slow-down on some simple benchmarks and a speed-up on bigger -programs. - -.. branch: ec-threadlocal -Change the executioncontext's lookup to be done by reading a thread- -local variable (which is implemented in C using '__thread' if -possible, and pthread_getspecific() otherwise). On Linux x86 and -x86-64, the JIT backend has a special optimization that lets it emit -directly a single MOV from a %gs- or %fs-based address. It seems -actually to give a good boost in performance. - -.. branch: fast-gil -A faster way to handle the GIL, particularly in JIT code. The GIL is -now a composite of two concepts: a global number (it's just set from -1 to 0 and back around CALL_RELEASE_GIL), and a real mutex. If there -are threads waiting to acquire the GIL, one of them is actively -checking the global number every 0.1 ms to 1 ms. Overall, JIT loops -full of external function calls now run a bit faster (if no thread was -started yet), or a *lot* faster (if threads were started already). - -.. branch: jit-get-errno -Optimize the errno handling in the JIT, notably around external -function calls. Linux-only. - -.. branch: disable_pythonapi -Remove non-functioning ctypes.pyhonapi and ctypes.PyDLL, document this -incompatibility with cpython. Recast sys.dllhandle to an int. - -.. branch: scalar-operations -Fix performance regression on ufunc(, ) in numpy. - -.. branch: pytest-25 -Update our copies of py.test and pylib to versions 2.5.2 and 1.4.20, -respectively. - -.. branch: split-ast-classes -Classes in the ast module are now distinct from structures used by the compiler. - -.. branch: stdlib-2.7.8 -Upgrades from 2.7.6 to 2.7.8 diff --git a/pypy/interpreter/mixedmodule.py b/pypy/interpreter/mixedmodule.py --- a/pypy/interpreter/mixedmodule.py +++ b/pypy/interpreter/mixedmodule.py @@ -7,7 +7,6 @@ class MixedModule(Module): applevel_name = None - expose__file__attribute = True # The following attribute is None as long as the module has not been # imported yet, and when it has been, it is mod.__dict__.items() just @@ -144,8 +143,6 @@ for name, spec in cls.appleveldefs.items(): loaders[name] = getappfileloader(pkgroot, appname, spec) assert '__file__' not in loaders - if cls.expose__file__attribute: - loaders['__file__'] = cls.get__file__ if '__doc__' not in loaders: loaders['__doc__'] = cls.get__doc__ @@ -159,28 +156,6 @@ w_obj = loader(space) space.setattr(space.wrap(self), space.wrap(name), w_obj) - def get__file__(cls, space): - """ NOT_RPYTHON. - return the __file__ attribute of a MixedModule - which is the root-directory for the various - applevel and interplevel snippets that make - up the module. - """ - try: - fname = cls._fname - except AttributeError: - pkgroot = cls.__module__ - mod = __import__(pkgroot, None, None, ['__doc__']) - fname = mod.__file__ - assert os.path.basename(fname).startswith('__init__.py') - # make it clear that it's not really the interp-level module - # at this path that we are seeing, but an app-level version of it - fname = os.path.dirname(fname) - cls._fname = fname - return space.wrap(fname) - - get__file__ = classmethod(get__file__) - def get__doc__(cls, space): return space.wrap(cls.__doc__) get__doc__ = classmethod(get__doc__) diff --git a/pypy/interpreter/module.py b/pypy/interpreter/module.py --- a/pypy/interpreter/module.py +++ b/pypy/interpreter/module.py @@ -29,6 +29,17 @@ space.w_None) self.startup_called = False + def _cleanup_(self): + """Called by the annotator on prebuilt Module instances. + We don't have many such modules, but for the ones that + show up, remove their __file__ rather than translate it + statically inside the executable.""" + try: + space = self.space + space.delitem(self.w_dict, space.wrap('__file__')) + except OperationError: + pass + def install(self): """NOT_RPYTHON: installs this module into space.builtin_modules""" w_mod = self.space.wrap(self) diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py --- a/pypy/interpreter/test/test_app_main.py +++ b/pypy/interpreter/test/test_app_main.py @@ -985,6 +985,11 @@ assert sys.path == old_sys_path + [self.goal_dir] app_main.setup_bootstrap_path(self.fake_exe) + assert sys.executable == '' # not executable! + assert sys.path == old_sys_path + [self.goal_dir] + + os.chmod(self.fake_exe, 0755) + app_main.setup_bootstrap_path(self.fake_exe) assert sys.executable == self.fake_exe assert self.goal_dir not in sys.path diff --git a/pypy/interpreter/test/test_extmodules.py b/pypy/interpreter/test/test_extmodules.py --- a/pypy/interpreter/test/test_extmodules.py +++ b/pypy/interpreter/test/test_extmodules.py @@ -64,5 +64,5 @@ @pytest.mark.skipif("config.option.runappdirect") def test_import(self): import extmod - assert extmod.__file__.endswith('extmod') + assert not hasattr(extmod, '__file__') assert type(extmod.time()) is float diff --git a/pypy/interpreter/test/test_module.py b/pypy/interpreter/test/test_module.py --- a/pypy/interpreter/test/test_module.py +++ b/pypy/interpreter/test/test_module.py @@ -1,4 +1,5 @@ - +import py +from pypy.interpreter.error import OperationError from pypy.interpreter.module import Module class TestModule: @@ -17,6 +18,18 @@ space.raises_w(space.w_AttributeError, space.delattr, w_m, w('x')) + def test___file__(self, space): + w = space.wrap + m = Module(space, space.wrap('m')) + py.test.raises(OperationError, space.getattr, w(m), w('__file__')) + m._cleanup_() + py.test.raises(OperationError, space.getattr, w(m), w('__file__')) + space.setattr(w(m), w('__file__'), w('m.py')) + space.getattr(w(m), w('__file__')) # does not raise + m._cleanup_() + py.test.raises(OperationError, space.getattr, w(m), w('__file__')) + + class AppTest_ModuleObject: def test_attr(self): m = __import__('__builtin__') @@ -42,12 +55,9 @@ bar = type(sys)('bar','docstring') assert bar.__doc__ == 'docstring' - def test___file__(self): - import sys, os - if not hasattr(sys, "pypy_objspaceclass"): - skip("need PyPy for sys.__file__ checking") - assert sys.__file__ - assert os.path.basename(sys.__file__) == 'sys' + def test___file__(self): + import sys + assert not hasattr(sys, '__file__') def test_repr(self): import sys diff --git a/pypy/module/__builtin__/__init__.py b/pypy/module/__builtin__/__init__.py --- a/pypy/module/__builtin__/__init__.py +++ b/pypy/module/__builtin__/__init__.py @@ -7,7 +7,6 @@ class Module(MixedModule): """Built-in functions, exceptions, and other objects.""" - expose__file__attribute = False appleveldefs = { 'execfile' : 'app_io.execfile', diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py --- a/pypy/module/_cffi_backend/ctypeprim.py +++ b/pypy/module/_cffi_backend/ctypeprim.py @@ -182,8 +182,12 @@ value = misc.read_raw_long_data(cdata, self.size) return self.space.wrap(value) else: - value = misc.read_raw_signed_data(cdata, self.size) - return self.space.wrap(value) # r_longlong => on 32-bit, 'long' + return self._convert_to_object_longlong(cdata) + + def _convert_to_object_longlong(self, cdata): + # in its own function: LONGLONG may make the whole function jit-opaque + value = misc.read_raw_signed_data(cdata, self.size) + return self.space.wrap(value) # r_longlong => on 32-bit, 'long' def convert_from_object(self, cdata, w_ob): if self.value_fits_long: @@ -193,8 +197,12 @@ self._overflow(w_ob) misc.write_raw_signed_data(cdata, value, self.size) else: - value = misc.as_long_long(self.space, w_ob) - misc.write_raw_signed_data(cdata, value, self.size) + self._convert_from_object_longlong(cdata, w_ob) + + def _convert_from_object_longlong(self, cdata, w_ob): + # in its own function: LONGLONG may make the whole function jit-opaque + value = misc.as_long_long(self.space, w_ob) + misc.write_raw_signed_data(cdata, value, self.size) def get_vararg_type(self): if self.size < rffi.sizeof(rffi.INT): @@ -264,8 +272,12 @@ self._overflow(w_ob) misc.write_raw_unsigned_data(cdata, value, self.size) else: - value = misc.as_unsigned_long_long(self.space, w_ob, strict=True) - misc.write_raw_unsigned_data(cdata, value, self.size) + self._convert_from_object_longlong(cdata, w_ob) + + def _convert_from_object_longlong(self, cdata, w_ob): + # in its own function: LONGLONG may make the whole function jit-opaque + value = misc.as_unsigned_long_long(self.space, w_ob, strict=True) + misc.write_raw_unsigned_data(cdata, value, self.size) def convert_to_object(self, cdata): if self.value_fits_ulong: @@ -275,8 +287,12 @@ else: return self.space.wrap(value) # r_uint => 'long' object else: - value = misc.read_raw_unsigned_data(cdata, self.size) - return self.space.wrap(value) # r_ulonglong => 'long' object + return self._convert_to_object_longlong(cdata) + + def _convert_to_object_longlong(self, cdata): + # in its own function: LONGLONG may make the whole function jit-opaque + value = misc.read_raw_unsigned_data(cdata, self.size) + return self.space.wrap(value) # r_ulonglong => 'long' object def get_vararg_type(self): if self.size < rffi.sizeof(rffi.INT): diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py --- a/pypy/module/_cffi_backend/ctypestruct.py +++ b/pypy/module/_cffi_backend/ctypestruct.py @@ -17,7 +17,7 @@ class W_CTypeStructOrUnion(W_CType): - _immutable_fields_ = ['alignment?', 'fields_list?', 'fields_dict?', + _immutable_fields_ = ['alignment?', 'fields_list?[*]', 'fields_dict?', 'custom_field_pos?', 'with_var_array?'] # fields added by complete_struct_or_union(): alignment = -1 diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py --- a/pypy/module/_cffi_backend/newtype.py +++ b/pypy/module/_cffi_backend/newtype.py @@ -389,7 +389,7 @@ w_ctype.size = totalsize w_ctype.alignment = totalalignment - w_ctype.fields_list = fields_list + w_ctype.fields_list = fields_list[:] w_ctype.fields_dict = fields_dict w_ctype.custom_field_pos = custom_field_pos w_ctype.with_var_array = with_var_array diff --git a/pypy/module/_io/interp_stringio.py b/pypy/module/_io/interp_stringio.py --- a/pypy/module/_io/interp_stringio.py +++ b/pypy/module/_io/interp_stringio.py @@ -86,7 +86,7 @@ initval = space.unicode_w(w_initval) size = len(initval) self.resize_buffer(size) - self.buf = [c for c in initval] + self.buf = list(initval) pos = space.getindex_w(w_pos, space.w_TypeError) if pos < 0: raise OperationError(space.w_ValueError, diff --git a/pypy/module/_pypyjson/interp_encoder.py b/pypy/module/_pypyjson/interp_encoder.py --- a/pypy/module/_pypyjson/interp_encoder.py +++ b/pypy/module/_pypyjson/interp_encoder.py @@ -1,4 +1,6 @@ from rpython.rlib.rstring import StringBuilder +from rpython.rlib.runicode import str_decode_utf_8 +from pypy.interpreter import unicodehelper HEX = '0123456789abcdef' @@ -17,20 +19,37 @@ def raw_encode_basestring_ascii(space, w_string): if space.isinstance_w(w_string, space.w_str): s = space.str_w(w_string) - for c in s: + for i in range(len(s)): + c = s[i] if c >= ' ' and c <= '~' and c != '"' and c != '\\': pass else: + first = i break else: # the input is a string with only non-special ascii chars return w_string - w_string = space.call_method(w_string, 'decode', space.wrap('utf-8')) + eh = unicodehelper.decode_error_handler(space) + u = str_decode_utf_8( + s, len(s), None, final=True, errorhandler=eh, + allow_surrogates=True)[0] + sb = StringBuilder(len(u)) + sb.append_slice(s, 0, first) + else: + # We used to check if 'u' contains only safe characters, and return + # 'w_string' directly. But this requires an extra pass over all + # characters, and the expected use case of this function, from + # json.encoder, will anyway re-encode a unicode result back to + # a string (with the ascii encoding). This requires two passes + # over the characters. So we may as well directly turn it into a + # string here --- only one pass. + u = space.unicode_w(w_string) + sb = StringBuilder(len(u)) + first = 0 - u = space.unicode_w(w_string) - sb = StringBuilder(len(u)) - for c in u: + for i in range(first, len(u)): + c = u[i] if c <= u'~': if c == u'"' or c == u'\\': sb.append('\\') diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h --- a/pypy/module/cpyext/include/patchlevel.h +++ b/pypy/module/cpyext/include/patchlevel.h @@ -29,7 +29,7 @@ #define PY_VERSION "2.7.8" /* PyPy version as a string */ -#define PYPY_VERSION "2.4.0-alpha0" +#define PYPY_VERSION "2.5.0-alpha0" /* Subversion Revision number of this file (not of the repository). * Empty since Mercurial migration. */ diff --git a/pypy/module/errno/test/test_errno.py b/pypy/module/errno/test/test_errno.py --- a/pypy/module/errno/test/test_errno.py +++ b/pypy/module/errno/test/test_errno.py @@ -8,7 +8,7 @@ cls.w_errorcode = cls.space.wrap(errno.errorcode) def test_posix(self): - assert self.errno.__file__ + assert not hasattr(self.errno, '__file__') def test_constants(self): for code, name in self.errorcode.iteritems(): diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -96,7 +96,7 @@ need_sparse_files() def test_posix_is_pypy_s(self): - assert self.posix.__file__ + assert hasattr(self.posix, '_statfields') def test_some_posix_basic_operation(self): path = self.path @@ -282,13 +282,8 @@ f = posix.fdopen(fd, "r") f.close() - # Ensure that fcntl is not faked - try: - import fcntl - except ImportError: - pass - else: - assert fcntl.__file__.endswith('pypy/module/fcntl') + # There used to be code here to ensure that fcntl is not faked + # but we can't do that cleanly any more exc = raises(OSError, posix.fdopen, fd) assert exc.value.errno == errno.EBADF diff --git a/pypy/module/posix/test/test_posix_libfile.py b/pypy/module/posix/test/test_posix_libfile.py --- a/pypy/module/posix/test/test_posix_libfile.py +++ b/pypy/module/posix/test/test_posix_libfile.py @@ -19,7 +19,7 @@ cls.w_path = cls.space.wrap(str(path)) def test_posix_is_pypy_s(self): - assert self.posix.__file__ + assert hasattr(self.posix, '_statfields') def test_fdopen(self): path = self.path diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py --- a/pypy/module/pypyjit/test_pypy_c/test_call.py +++ b/pypy/module/pypyjit/test_pypy_c/test_call.py @@ -17,13 +17,18 @@ # now we can inline it as call assembler i = 0 j = 0 - while i < 20: + while i < 25: i += 1 j += rec(100) # ID: call_rec return j # - log = self.run(fn, [], threshold=18) - loop, = log.loops_by_filename(self.filepath) + # NB. the parameters below are a bit ad-hoc. After 16 iterations, + # the we trace from the "while" and reach a "trace too long". Then + # in the next execution, we trace the "rec" function from start; + # that's "functrace" below. Then after one or two extra iterations + # we try again from "while", and this time we succeed. + log = self.run(fn, [], threshold=20) + functrace, loop = log.loops_by_filename(self.filepath) assert loop.match_by_id('call_rec', """ ... p53 = call_assembler(..., descr=...) diff --git a/pypy/module/pypyjit/test_pypy_c/test_cprofile.py b/pypy/module/pypyjit/test_pypy_c/test_cprofile.py --- a/pypy/module/pypyjit/test_pypy_c/test_cprofile.py +++ b/pypy/module/pypyjit/test_pypy_c/test_cprofile.py @@ -1,4 +1,4 @@ -import py, sys +import py, sys, re from pypy.module.pypyjit.test_pypy_c.test_00_model import BaseTestPyPyC class TestCProfile(BaseTestPyPyC): @@ -26,10 +26,20 @@ for method in ['append', 'pop']: loop, = log.loops_by_id(method) print loop.ops_by_id(method) - # on 32-bit, there is f1=read_timestamp(); ...; - # f2=read_timestamp(); f3=call(llong_sub,f1,f2) - # which should turn into a single PADDQ/PSUBQ - if sys.maxint != 2147483647: - assert ' call(' not in repr(loop.ops_by_id(method)) + # on 32-bit, there is f1=call(read_timestamp); ...; + # f2=call(read_timestamp); f3=call(llong_sub,f1,f2) + # but all calls can be special-cased by the backend if + # supported. On 64-bit there is only the two calls to + # read_timestamp. + r = re.compile(r" call[(]ConstClass[(](.+?)[)]") + calls = r.findall(repr(loop.ops_by_id(method))) + if sys.maxint == 2147483647: + assert len(calls) == 6 + else: + assert len(calls) == 2 + for x in calls: + assert ('ll_read_timestamp' in x or 'llong_sub' in x + or 'llong_add' in x) + # assert ' call_may_force(' not in repr(loop.ops_by_id(method)) assert ' cond_call(' in repr(loop.ops_by_id(method)) diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py --- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py +++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py @@ -340,30 +340,19 @@ guard_value(p166, ConstPtr(ptr72), descr=...) p167 = call(ConstClass(_ll_0_alloc_with_del___), descr=) guard_no_exception(descr=...) - i168 = call(ConstClass(_ll_1_raw_malloc_varsize__Signed), 6, descr=) - i169 = int_add(i168, i97) - i170 = int_sub(i160, i106) - setfield_gc(p167, i168, descr=) + i112 = int_sub(i160, -32768) setfield_gc(p167, ConstPtr(null), descr=) - setfield_gc(p167, ConstPtr(ptr89), descr=) - i171 = uint_gt(i170, i108) - guard_false(i171, descr=...) - i172 = int_sub(i160, -32768) - i173 = int_and(i172, 65535) - i174 = int_add(i173, -32768) - setarrayitem_raw(i169, 0, i174, descr=) - i175 = int_add(i168, i121) - i176 = int_sub(i160, i130) - i177 = uint_gt(i176, i132) - guard_false(i177, descr=...) - setarrayitem_raw(i175, 0, i174, descr=) - i178 = int_add(i168, i140) - i179 = int_sub(i160, i149) - i180 = uint_gt(i179, i151) - guard_false(i180, descr=...) - setarrayitem_raw(i178, 0, i174, descr=) + setfield_gc(p167, ConstPtr(ptr85), descr=) + i114 = uint_gt(i112, 65535) + guard_false(i114, descr=...) + i115 = int_and(i112, 65535) + i116 = int_add(i115, -32768) --TICK-- - i183 = arraylen_gc(p67, descr=) - i184 = arraylen_gc(p92, descr=) + i119 = call(ConstClass(_ll_1_raw_malloc_varsize__Signed), 6, descr=) + raw_store(i119, 0, i116, descr=) + raw_store(i119, 2, i116, descr=) + raw_store(i119, 4, i116, descr=) + setfield_gc(p167, i119, descr=) + i123 = arraylen_gc(p67, descr=) jump(..., descr=...) """) diff --git a/pypy/module/select/interp_select.py b/pypy/module/select/interp_select.py --- a/pypy/module/select/interp_select.py +++ b/pypy/module/select/interp_select.py @@ -98,6 +98,9 @@ for w_f in list_w: fd = space.c_filedescriptor_w(w_f) if fd > nfds: + if _c.MAX_FD_SIZE is not None and fd >= _c.MAX_FD_SIZE: + raise oefmt(space.w_ValueError, + "file descriptor out of range in select()") nfds = fd _c.FD_SET(fd, ll_list) fdlist.append(fd) diff --git a/pypy/module/select/test/test_select.py b/pypy/module/select/test/test_select.py --- a/pypy/module/select/test/test_select.py +++ b/pypy/module/select/test/test_select.py @@ -198,6 +198,16 @@ finally: writeend.close() + def test_select_descr_out_of_bounds(self): + import sys, select + raises(ValueError, select.select, [-1], [], []) + raises(ValueError, select.select, [], [-2], []) + raises(ValueError, select.select, [], [], [-3]) + if sys.platform != 'win32': + raises(ValueError, select.select, [2000000000], [], []) + raises(ValueError, select.select, [], [2000000000], []) + raises(ValueError, select.select, [], [], [2000000000]) + def test_poll(self): import select if not hasattr(select, 'poll'): diff --git a/pypy/module/sys/initpath.py b/pypy/module/sys/initpath.py --- a/pypy/module/sys/initpath.py +++ b/pypy/module/sys/initpath.py @@ -18,6 +18,13 @@ _WIN32 = sys.platform == 'win32' +def _exists_and_is_executable(fn): + # os.access checks using the user's real uid and gid. + # Since pypy should not be run setuid/setgid, this + # should be sufficient. + return os.path.isfile(fn) and os.access(fn, os.X_OK) + + def find_executable(executable): """ Return the absolute path of the executable, by looking into PATH and @@ -34,14 +41,14 @@ if path: for dir in path.split(os.pathsep): fn = os.path.join(dir, executable) - if os.path.isfile(fn): + if _exists_and_is_executable(fn): executable = fn break executable = rpath.rabspath(executable) # 'sys.executable' should not end up being an non-existing file; # just use '' in this case. (CPython issue #7774) - return executable if os.path.isfile(executable) else '' + return executable if _exists_and_is_executable(executable) else '' def _readlink_maybe(filename): diff --git a/pypy/module/sys/test/test_initpath.py b/pypy/module/sys/test/test_initpath.py --- a/pypy/module/sys/test/test_initpath.py +++ b/pypy/module/sys/test/test_initpath.py @@ -57,6 +57,7 @@ a.join('pypy').ensure(file=True) b.join('pypy').ensure(file=True) # + monkeypatch.setattr(os, 'access', lambda x, y: True) # if there is already a slash, don't do anything monkeypatch.chdir(tmpdir) assert find_executable('a/pypy') == a.join('pypy') @@ -82,7 +83,11 @@ # if pypy is found but it's not a file, ignore it c.join('pypy').ensure(dir=True) assert find_executable('pypy') == a.join('pypy') + # if pypy is found but it's not executable, ignore it + monkeypatch.setattr(os, 'access', lambda x, y: False) + assert find_executable('pypy') == '' # + monkeypatch.setattr(os, 'access', lambda x, y: True) monkeypatch.setattr(initpath, 'we_are_translated', lambda: True) monkeypatch.setattr(initpath, '_WIN32', True) monkeypatch.setenv('PATH', str(a)) diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -10,7 +10,7 @@ #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py CPYTHON_API_VERSION = 1013 #XXX # sync with include/modsupport.h -PYPY_VERSION = (2, 4, 0, "alpha", 0) #XXX # sync patchlevel.h +PYPY_VERSION = (2, 5, 0, "alpha", 0) #XXX # sync patchlevel.h if platform.name == 'msvc': COMPILER_INFO = 'MSC v.%d 32 bit' % (platform.version * 10 + 600) diff --git a/pypy/module/thread/test/test_thread.py b/pypy/module/thread/test/test_thread.py --- a/pypy/module/thread/test/test_thread.py +++ b/pypy/module/thread/test/test_thread.py @@ -13,18 +13,26 @@ def f(): lock.acquire() lock.release() + start = thread._count() try: try: for i in range(1000): thread.start_new_thread(f, ()) finally: lock.release() - # wait a bit to allow most threads to finish now - time.sleep(0.5) except (thread.error, MemoryError): cls.w_can_start_many_threads = space.wrap(False) else: cls.w_can_start_many_threads = space.wrap(True) + # wait a bit to allow all threads to finish now + remaining = thread._count() + retries = 0 + while remaining > start: + retries += 1 + if retries == 200: + raise Exception("the test's threads don't stop!") + time.sleep(0.2) + remaining = thread._count() def test_start_new_thread(self): import thread @@ -227,7 +235,7 @@ import signal def f(): - for x in range(5): + for x in range(40): if waiting: thread.interrupt_main() return @@ -236,7 +244,7 @@ def busy_wait(): waiting.append(None) - for x in range(10): + for x in range(50): print 'tick...', x # <-force the GIL to be released, as time.sleep(0.1) # time.sleep doesn't do non-translated waiting.pop() @@ -245,6 +253,8 @@ signal.signal(signal.SIGINT, signal.default_int_handler) for i in range(100): + print + print "loop", i waiting = [] thread.start_new_thread(f, ()) raises(KeyboardInterrupt, busy_wait) diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py --- a/pypy/objspace/descroperation.py +++ b/pypy/objspace/descroperation.py @@ -671,6 +671,7 @@ left, right = specialnames errormsg = "unsupported operand type(s) for %s: '%%N' and '%%N'" % ( symbol.replace('%', '%%'),) + seq_bug_compat = (symbol == '+' or symbol == '*') def binop_impl(space, w_obj1, w_obj2): w_typ1 = space.type(w_obj1) @@ -686,20 +687,16 @@ # __xxx__ and __rxxx__ methods where found by identity. # Note that space.is_w() is potentially not happy if one of them # is None... - if w_left_src is not w_right_src: # XXX - # -- cpython bug compatibility: see objspace/std/test/ - # -- test_unicodeobject.test_str_unicode_concat_overrides. - # -- The following handles "unicode + string subclass" by - # -- pretending that the unicode is a superclass of the - # -- string, thus giving priority to the string subclass' - # -- __radd__() method. The case "string + unicode subclass" - # -- is handled directly by add__String_Unicode(). - if symbol == '+' and space.is_w(w_typ1, space.w_unicode): - w_typ1 = space.w_basestring - # -- end of bug compatibility - if space.is_true(space.issubtype(w_typ2, w_typ1)): - if (w_left_src and w_right_src and - not space.abstract_issubclass_w(w_left_src, w_right_src) and + if w_right_src and (w_left_src is not w_right_src) and w_left_src: + # 'seq_bug_compat' is for cpython bug-to-bug compatibility: + # see objspace/std/test/test_unicodeobject.*concat_overrides + # and objspace/test/test_descrobject.*rmul_overrides. + # For cases like "unicode + string subclass". + if ((seq_bug_compat and w_typ1.flag_sequence_bug_compat + and not w_typ2.flag_sequence_bug_compat) + # the non-bug-compat part is the following check: + or space.is_true(space.issubtype(w_typ2, w_typ1))): + if (not space.abstract_issubclass_w(w_left_src, w_right_src) and not space.abstract_issubclass_w(w_typ1, w_right_src)): w_obj1, w_obj2 = w_obj2, w_obj1 w_left_impl, w_right_impl = w_right_impl, w_left_impl diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py --- a/pypy/objspace/std/bytearrayobject.py +++ b/pypy/objspace/std/bytearrayobject.py @@ -534,7 +534,7 @@ if not e.match(space, space.w_TypeError): raise else: - return [c for c in buf.as_str()] + return list(buf.as_str()) # sequence of bytes w_iter = space.iter(w_source) @@ -1131,6 +1131,7 @@ reverse = interp2app(W_BytearrayObject.descr_reverse, doc=BytearrayDocstrings.reverse.__doc__), ) +W_BytearrayObject.typedef.flag_sequence_bug_compat = True init_signature = Signature(['source', 'encoding', 'errors'], None, None) init_defaults = [None, None, None] diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py --- a/pypy/objspace/std/bytesobject.py +++ b/pypy/objspace/std/bytesobject.py @@ -951,6 +951,7 @@ _formatter_field_name_split = interp2app(W_BytesObject.descr_formatter_field_name_split), ) +W_BytesObject.typedef.flag_sequence_bug_compat = True def string_escape_encode(s, quote): diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py --- a/pypy/objspace/std/listobject.py +++ b/pypy/objspace/std/listobject.py @@ -1874,3 +1874,4 @@ insert = interp2app(W_ListObject.descr_insert), remove = interp2app(W_ListObject.descr_remove), ) +W_ListObject.typedef.flag_sequence_bug_compat = True diff --git a/pypy/objspace/std/stdtypedef.py b/pypy/objspace/std/stdtypedef.py --- a/pypy/objspace/std/stdtypedef.py +++ b/pypy/objspace/std/stdtypedef.py @@ -93,6 +93,8 @@ overridetypedef=overridetypedef) if typedef is not overridetypedef: w_type.w_doc = space.wrap(typedef.doc) + if hasattr(typedef, 'flag_sequence_bug_compat'): + w_type.flag_sequence_bug_compat = typedef.flag_sequence_bug_compat w_type.lazyloaders = lazyloaders return w_type diff --git a/pypy/objspace/std/tupleobject.py b/pypy/objspace/std/tupleobject.py --- a/pypy/objspace/std/tupleobject.py +++ b/pypy/objspace/std/tupleobject.py @@ -244,6 +244,7 @@ count = interp2app(W_AbstractTupleObject.descr_count), index = interp2app(W_AbstractTupleObject.descr_index) ) +W_AbstractTupleObject.typedef.flag_sequence_bug_compat = True class W_TupleObject(W_AbstractTupleObject): diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py --- a/pypy/objspace/std/typeobject.py +++ b/pypy/objspace/std/typeobject.py @@ -67,6 +67,7 @@ _immutable_fields_ = ["flag_heaptype", "flag_cpytype", "flag_abstract?", + "flag_sequence_bug_compat", 'needsdel', 'weakrefable', 'hasdict', @@ -104,6 +105,7 @@ w_self.flag_heaptype = False w_self.flag_cpytype = False w_self.flag_abstract = False + w_self.flag_sequence_bug_compat = False w_self.instancetypedef = overridetypedef if overridetypedef is not None: diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -1068,6 +1068,7 @@ _formatter_field_name_split = interp2app(W_UnicodeObject.descr_formatter_field_name_split), ) +W_UnicodeObject.typedef.flag_sequence_bug_compat = True From noreply at buildbot.pypy.org Mon Sep 8 01:00:52 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 8 Sep 2014 01:00:52 +0200 (CEST) Subject: [pypy-commit] pypy use-file-star-for-file: adjust for rfile raising IOError instead of OSError Message-ID: <20140907230052.73F371C320C@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: use-file-star-for-file Changeset: r73372:63372c8f1af9 Date: 2014-09-07 19:00 -0400 http://bitbucket.org/pypy/pypy/changeset/63372c8f1af9/ Log: adjust for rfile raising IOError instead of OSError diff --git a/pypy/module/_file/interp_file.py b/pypy/module/_file/interp_file.py --- a/pypy/module/_file/interp_file.py +++ b/pypy/module/_file/interp_file.py @@ -10,7 +10,6 @@ from pypy.interpreter.typedef import (TypeDef, GetSetProperty, interp_attrproperty, make_weakref_descr, interp_attrproperty_w) from pypy.interpreter.gateway import interp2app, unwrap_spec -from pypy.interpreter.streamutil import wrap_oserror_as_ioerror class W_File(W_AbstractStream): @@ -52,8 +51,10 @@ assert isinstance(self, W_File) try: self.direct_close() - except OSError as e: - raise wrap_oserror_as_ioerror(self.space, e, self.w_name) + except IOError as e: + space = self.space + w_error = space.call_function(space.w_IOError, space.wrap(e.errno), space.wrap(e.strerror), self.w_name) + raise OperationError(space.w_IOError, w_error) def fdopenstream(self, stream, mode): self.stream = stream @@ -276,8 +277,9 @@ result = self.direct_%(name)s(%(callsig)s) except ValueError as e: raise OperationError(space.w_ValueError, space.wrap(str(e))) - except OSError as e: - raise wrap_oserror_as_ioerror(self.space, e, self.w_name) + except IOError as e: + w_error = space.call_function(space.w_IOError, space.wrap(e.errno), space.wrap(e.strerror), self.w_name) + raise OperationError(space.w_IOError, w_error) finally: self.unlock() return %(wrapresult)s From noreply at buildbot.pypy.org Mon Sep 8 11:20:39 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Mon, 8 Sep 2014 11:20:39 +0200 (CEST) Subject: [pypy-commit] stmgc default: add inevitable transactions Message-ID: <20140908092039.11EA71C320C@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1364:5b5ed3773ccf Date: 2014-09-08 11:22 +0200 http://bitbucket.org/pypy/stmgc/changeset/5b5ed3773ccf/ Log: add inevitable transactions diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -110,13 +110,17 @@ void _dbg_print_commit_log() { - volatile struct stm_commit_log_entry_s *cl = (volatile struct stm_commit_log_entry_s *) - &commit_log_root; + volatile struct stm_commit_log_entry_s *cl; + cl = (volatile struct stm_commit_log_entry_s *)&commit_log_root; fprintf(stderr, "root (%p, %d)\n", cl->next, cl->segment_num); while ((cl = cl->next)) { + if ((uintptr_t)cl == -1) { + fprintf(stderr, "INEVITABLE\n"); + return; + } size_t i = 0; - fprintf(stderr, "elem (%p, %d)\n", cl->next, cl->segment_num); + fprintf(stderr, " elem (%p, %d)\n", cl->next, cl->segment_num); object_t *obj; while ((obj = cl->written[i])) { fprintf(stderr, "-> %p\n", obj); @@ -176,13 +180,21 @@ most current one and apply all changes done by other transactions. Abort if we read one of the committed objs. */ - volatile struct stm_commit_log_entry_s *cl = (volatile struct stm_commit_log_entry_s *) + volatile struct stm_commit_log_entry_s *cl, *prev_cl; + cl = prev_cl = (volatile struct stm_commit_log_entry_s *) STM_PSEGMENT->last_commit_log_entry; - acquire_privatization_lock(STM_SEGMENT->segment_num); bool needs_abort = false; /* Don't check 'cl'. This entry is already checked */ while ((cl = cl->next)) { + if ((uintptr_t)cl == -1) { + /* there is an inevitable transaction running */ + cl = prev_cl; + usleep(1); /* XXX */ + continue; + } + prev_cl = cl; + OPT_ASSERT(cl->segment_num >= 0 && cl->segment_num < NB_SEGMENTS); object_t *obj; @@ -200,7 +212,6 @@ /* last fully validated entry */ STM_PSEGMENT->last_commit_log_entry = (struct stm_commit_log_entry_s *)cl; } - release_privatization_lock(STM_SEGMENT->segment_num); if (needs_abort) { free(free_if_abort); @@ -241,6 +252,16 @@ volatile struct stm_commit_log_entry_s **to; new = _create_commit_log_entry(); + if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) { + OPT_ASSERT((uintptr_t)STM_PSEGMENT->last_commit_log_entry->next == -1); + + to = &(STM_PSEGMENT->last_commit_log_entry->next); + bool yes = __sync_bool_compare_and_swap(to, -1, new); + OPT_ASSERT(yes); + return; + } + + /* regular transaction: */ do { stm_validate(new); @@ -249,6 +270,20 @@ } while (!__sync_bool_compare_and_swap(to, NULL, new)); } +static void _validate_and_turn_inevitable() +{ + struct stm_commit_log_entry_s *new; + volatile struct stm_commit_log_entry_s **to; + + new = (struct stm_commit_log_entry_s*)-1; + do { + stm_validate(NULL); + + /* try attaching to commit log: */ + to = &(STM_PSEGMENT->last_commit_log_entry->next); + } while (!__sync_bool_compare_and_swap(to, NULL, new)); +} + /* ############# STM ############# */ void _privatize_and_protect_other_segments(object_t *obj) @@ -382,7 +417,7 @@ } -static void _stm_start_transaction(stm_thread_local_t *tl, bool inevitable) +static void _stm_start_transaction(stm_thread_local_t *tl) { assert(!_stm_in_transaction(tl)); @@ -392,6 +427,8 @@ goto retry; /* GS invalid before this point! */ + assert(STM_PSEGMENT->transaction_state == TS_NONE); + STM_PSEGMENT->transaction_state = TS_REGULAR; #ifndef NDEBUG STM_PSEGMENT->running_pthread = pthread_self(); #endif @@ -421,10 +458,28 @@ #else long repeat_count = stm_rewind_jmp_setjmp(tl); #endif - _stm_start_transaction(tl, false); + _stm_start_transaction(tl); return repeat_count; } +void stm_start_inevitable_transaction(stm_thread_local_t *tl) +{ + s_mutex_lock(); + _stm_start_transaction(tl); + _stm_become_inevitable("stm_start_inevitable_transaction"); +} + +#ifdef STM_NO_AUTOMATIC_SETJMP +void _test_run_abort(stm_thread_local_t *tl) __attribute__((noreturn)); +int stm_is_inevitable(void) +{ + switch (STM_PSEGMENT->transaction_state) { + case TS_REGULAR: return 0; + case TS_INEVITABLE: return 1; + default: abort(); + } +} +#endif /************************************************************/ @@ -432,6 +487,7 @@ { stm_thread_local_t *tl = STM_SEGMENT->running_thread; + STM_PSEGMENT->transaction_state = TS_NONE; list_clear(STM_PSEGMENT->objects_pointing_to_nursery); release_thread_segment(tl); @@ -443,6 +499,7 @@ assert(!_has_mutex()); assert(STM_PSEGMENT->running_pthread == pthread_self()); + dprintf(("stm_commit_transaction()\n")); minor_collection(1); _validate_and_add_to_commit_log(); @@ -469,7 +526,6 @@ assert(STM_SEGMENT->nursery_end == NURSERY_END); stm_rewind_jmp_forget(STM_SEGMENT->running_thread); - /* done */ _finish_transaction(); /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */ @@ -519,6 +575,16 @@ #undef STM_SEGMENT struct stm_priv_segment_info_s *pseg = get_priv_segment(segment_num); + switch (pseg->transaction_state) { + case TS_REGULAR: + break; + case TS_INEVITABLE: + stm_fatalerror("abort: transaction_state == TS_INEVITABLE"); + default: + stm_fatalerror("abort: bad transaction_state == %d", + (int)pseg->transaction_state); + } + throw_away_nursery(pseg); reset_modified_from_backup_copies(segment_num); @@ -582,3 +648,22 @@ stm_rewind_jmp_longjmp(tl); #endif } + + +void _stm_become_inevitable(const char *msg) +{ + s_mutex_lock(); + + if (STM_PSEGMENT->transaction_state == TS_REGULAR) { + dprintf(("become_inevitable: %s\n", msg)); + + _validate_and_turn_inevitable(); + STM_PSEGMENT->transaction_state = TS_INEVITABLE; + stm_rewind_jmp_forget(STM_SEGMENT->running_thread); + } + else { + assert(STM_PSEGMENT->transaction_state == TS_INEVITABLE); + } + + s_mutex_unlock(); +} diff --git a/c8/stm/core.h b/c8/stm/core.h --- a/c8/stm/core.h +++ b/c8/stm/core.h @@ -53,6 +53,8 @@ struct list_s *objects_pointing_to_nursery; uint8_t privatization_lock; + uint8_t transaction_state; + struct stm_commit_log_entry_s *last_commit_log_entry; struct stm_shadowentry_s *shadowstack_at_start_of_transaction; @@ -63,6 +65,12 @@ #endif }; +enum /* transaction_state */ { + TS_NONE=0, + TS_REGULAR, + TS_INEVITABLE, +}; + /* Commit Log things */ struct stm_commit_log_entry_s { volatile struct stm_commit_log_entry_s *next; diff --git a/c8/stmgc.h b/c8/stmgc.h --- a/c8/stmgc.h +++ b/c8/stmgc.h @@ -69,6 +69,7 @@ void _stm_write_slowpath(object_t *); object_t *_stm_allocate_slowpath(ssize_t); +void _stm_become_inevitable(const char*); object_t *_stm_allocate_old(ssize_t size_rounded_up); char *_stm_real_address(object_t *o); @@ -186,10 +187,27 @@ long stm_start_transaction(stm_thread_local_t *tl); +void stm_start_inevitable_transaction(stm_thread_local_t *tl); + void stm_commit_transaction(void); void stm_abort_transaction(void) __attribute__((noreturn)); +#ifdef STM_NO_AUTOMATIC_SETJMP +int stm_is_inevitable(void); +#else +static inline int stm_is_inevitable(void) { + return !rewind_jmp_armed(&STM_SEGMENT->running_thread->rjthread); +} +#endif +static inline void stm_become_inevitable(stm_thread_local_t *tl, + const char* msg) { + assert(STM_SEGMENT->running_thread == tl); + if (!stm_is_inevitable()) + _stm_become_inevitable(msg); +} + + /* ==================== END ==================== */ #endif diff --git a/c8/test/support.py b/c8/test/support.py --- a/c8/test/support.py +++ b/c8/test/support.py @@ -57,6 +57,8 @@ long stm_start_transaction(stm_thread_local_t *tl); bool _check_commit_transaction(void); bool _check_abort_transaction(void); +bool _check_become_inevitable(stm_thread_local_t *tl); +int stm_is_inevitable(void); void _set_type_id(object_t *obj, uint32_t h); uint32_t _get_type_id(object_t *obj); @@ -134,6 +136,10 @@ CHECKED(stm_abort_transaction()); } +bool _check_become_inevitable(stm_thread_local_t *tl) { + CHECKED(stm_become_inevitable(tl, "TEST")); +} + bool _check_stm_validate(void) { CHECKED(stm_validate(NULL)); } @@ -413,12 +419,18 @@ def teardown_method(self, meth): lib.stmcb_expand_marker = ffi.NULL lib.stmcb_debug_print = ffi.NULL + tl = self.tls[self.current_thread] + if lib._stm_in_transaction(tl) and lib.stm_is_inevitable(): + self.commit_transaction() # must succeed! # for n, tl in enumerate(self.tls): if lib._stm_in_transaction(tl): if self.current_thread != n: self.switch(n) - self.abort_transaction() + if lib.stm_is_inevitable(): + self.commit_transaction() # must succeed! + else: + self.abort_transaction() # for tl in self.tls: lib.stm_unregister_thread_local(tl) @@ -504,3 +516,8 @@ def set_thread_local_obj(self, newobj): tl = self.tls[self.current_thread] tl.thread_local_obj = newobj + + def become_inevitable(self): + tl = self.tls[self.current_thread] + if lib._check_become_inevitable(tl): + raise Conflict() From noreply at buildbot.pypy.org Mon Sep 8 12:28:16 2014 From: noreply at buildbot.pypy.org (groggi) Date: Mon, 8 Sep 2014 12:28:16 +0200 (CEST) Subject: [pypy-commit] pypy gc-incminimark-pinning: add comment (XXX) for possible improvement of rzlib by using pinning Message-ID: <20140908102816.1F16A1C3455@cobra.cs.uni-duesseldorf.de> Author: Gregor Wegberg Branch: gc-incminimark-pinning Changeset: r73373:0d68a93239a5 Date: 2014-09-08 12:27 +0200 http://bitbucket.org/pypy/pypy/changeset/0d68a93239a5/ Log: add comment (XXX) for possible improvement of rzlib by using pinning diff --git a/rpython/rlib/rzlib.py b/rpython/rlib/rzlib.py --- a/rpython/rlib/rzlib.py +++ b/rpython/rlib/rzlib.py @@ -346,6 +346,8 @@ """ # Prepare the input buffer for the stream with lltype.scoped_alloc(rffi.CCHARP.TO, len(data)) as inbuf: + # XXX (groggi) should be possible to improve this with pinning by + # not performing the 'copy_string_to_raw' if non-movable/pinned copy_string_to_raw(llstr(data), inbuf, 0, len(data)) stream.c_next_in = rffi.cast(Bytefp, inbuf) rffi.setintfield(stream, 'c_avail_in', len(data)) From noreply at buildbot.pypy.org Mon Sep 8 12:39:20 2014 From: noreply at buildbot.pypy.org (groggi) Date: Mon, 8 Sep 2014 12:39:20 +0200 (CEST) Subject: [pypy-commit] pypy gc-incminimark-pinning: ups, reapply change to rfile from d6c6003d1ae93df5b54e50229989ced778502694 Message-ID: <20140908103920.811041C009A@cobra.cs.uni-duesseldorf.de> Author: Gregor Wegberg Branch: gc-incminimark-pinning Changeset: r73374:ba0833e510ae Date: 2014-09-08 12:38 +0200 http://bitbucket.org/pypy/pypy/changeset/ba0833e510ae/ Log: ups, reapply change to rfile from d6c6003d1ae93df5b54e50229989ced778502694 diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py --- a/rpython/rlib/rfile.py +++ b/rpython/rlib/rfile.py @@ -317,7 +317,7 @@ @enforceargs(None, str) def write(self, value): self._check_closed() - ll_value = rffi.get_nonmovingbuffer(value) + ll_value, is_pinned, is_raw = rffi.get_nonmovingbuffer(value) try: # note that since we got a nonmoving buffer, it is either raw # or already cannot move, so the arithmetics below are fine @@ -327,7 +327,7 @@ errno = rposix.get_errno() raise OSError(errno, os.strerror(errno)) finally: - rffi.free_nonmovingbuffer(value, ll_value) + rffi.free_nonmovingbuffer(value, ll_value, is_pinned, is_raw) def flush(self): self._check_closed() From noreply at buildbot.pypy.org Mon Sep 8 12:56:23 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Mon, 8 Sep 2014 12:56:23 +0200 (CEST) Subject: [pypy-commit] stmgc default: add two failing tests Message-ID: <20140908105623.7B5261C31B7@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1365:7f4ea36607f9 Date: 2014-09-08 12:57 +0200 http://bitbucket.org/pypy/stmgc/changeset/7f4ea36607f9/ Log: add two failing tests diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -7,6 +7,32 @@ /* ############# signal handler ############# */ +static void copy_bk_objs_from(int from_segnum, uintptr_t pagenum) +{ + acquire_modified_objs_lock(from_segnum); + struct tree_s *tree = get_priv_segment(from_segnum)->modified_old_objects; + wlog_t *item; + TREE_LOOP_FORWARD(tree, item); { + if (item->addr >= pagenum * 4096UL && item->addr < (pagenum + 1) * 4096UL) { + /* obj is in range. XXX: no page overlapping objs allowed yet */ + + object_t *obj = (object_t*)item->addr; + struct object_s* bk_obj = (struct object_s *)item->val; + size_t obj_size; + + obj_size = stmcb_size_rounded_up(bk_obj); + assert(obj_size < 4096); /* XXX */ + + memcpy(REAL_ADDRESS(STM_SEGMENT->segment_base, obj), + bk_obj, obj_size); + + assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); /* bk_obj never written */ + } + } TREE_LOOP_END; + + release_modified_objs_lock(from_segnum); +} + static void bring_page_up_to_date(uintptr_t pagenum) { /* Assuming pagenum is not yet private in this segment and @@ -50,28 +76,8 @@ /* get valid state from backup copies of written objs in the range of this page: */ - acquire_modified_objs_lock(i); - struct tree_s *tree = get_priv_segment(i)->modified_old_objects; - wlog_t *item; - TREE_LOOP_FORWARD(tree, item); { - if (item->addr >= pagenum * 4096UL && item->addr < (pagenum + 1) * 4096UL) { - /* obj is in range. XXX: no page overlapping objs allowed yet */ + copy_bk_objs_from(i, pagenum); - object_t *obj = (object_t*)item->addr; - struct object_s* bk_obj = (struct object_s *)item->val; - size_t obj_size; - - obj_size = stmcb_size_rounded_up(bk_obj); - assert(obj_size < 4096); /* XXX */ - - memcpy(REAL_ADDRESS(STM_SEGMENT->segment_base, obj), - bk_obj, obj_size); - - assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); /* bk_obj never written */ - } - } TREE_LOOP_END; - - release_modified_objs_lock(i); release_privatization_lock(i); return; diff --git a/c8/test/support.py b/c8/test/support.py --- a/c8/test/support.py +++ b/c8/test/support.py @@ -65,6 +65,9 @@ void _set_ptr(object_t *obj, int n, object_t *v); object_t * _get_ptr(object_t *obj, int n); + +void _stm_set_nursery_free_count(uint64_t free_count); + long _stm_count_modified_old_objects(void); long _stm_count_objects_pointing_to_nursery(void); object_t *_stm_enum_modified_old_objects(long index); @@ -406,7 +409,7 @@ class BaseTest(object): - NB_THREADS = 2 + NB_THREADS = 4 def setup_method(self, meth): lib.stm_setup() diff --git a/c8/test/test_basic.py b/c8/test/test_basic.py --- a/c8/test/test_basic.py +++ b/c8/test/test_basic.py @@ -110,6 +110,73 @@ # py.test.raises(Conflict, self.switch, 0) # detects rw conflict + def test_read_write_11(self): + # test that stm_validate() and the SEGV-handler + # always ensure up-to-date views of pages: + lp1 = stm_allocate_old(16) + stm_get_real_address(lp1)[HDR] = 'a' #setchar + # + self.start_transaction() + stm_set_char(lp1, '0') # shared->private + # prot_none in seg: 1,2,3 + # + self.switch(1) + self.start_transaction() + stm_set_char(lp1, '1') + # prot_none in seg: 2,3 + # + self.switch(0) + self.commit_transaction() + assert last_commit_log_entries() == [lp1] # commit '0' + # + py.test.raises(Conflict, self.switch, 1) + self.start_transaction() # updates to '0' + stm_set_char(lp1, 'x') + self.commit_transaction() + assert last_commit_log_entries() == [lp1] # commit 'x' + # + # + self.switch(2) + self.start_transaction() # stm_validate() + res = stm_get_char(lp1) # should be 'x' + self.commit_transaction() + assert res == 'x' + # if fails, segfault-handler copied from seg0 which + # is out-of-date because seg1 committed 'x' + # (seg1 hasn't done stm_validate() since) + + + def test_read_write_12(self): + # test that stm_validate() and the SEGV-handler + # always ensure up-to-date views of pages: + lp1 = stm_allocate_old(16) + stm_get_real_address(lp1)[HDR] = 'a' #setchar + # + self.switch(1) + self.start_transaction() + stm_set_char(lp1, '1') # shared->private + # prot_none in seg: 0,2,3 + # + self.switch(0) + self.start_transaction() + stm_set_char(lp1, '0') + # prot_none in seg: 2,3 + # + self.switch(1) + self.commit_transaction() + assert last_commit_log_entries() == [lp1] + # '1' is committed + # + self.switch(2) + self.start_transaction() # stm_validate() + res = stm_get_char(lp1) # should be '1' + self.commit_transaction() + assert res == '1' + # if res=='a', then we got the outdated page-view + # of segment 0 that didn't do stm_validate() and + # therefore is still outdated. + py.test.raises(Conflict, self.switch, 0) + def test_commit_fresh_objects(self): self.start_transaction() lp = stm_allocate(16) From noreply at buildbot.pypy.org Mon Sep 8 14:16:58 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Mon, 8 Sep 2014 14:16:58 +0200 (CEST) Subject: [pypy-commit] stmgc default: ensure that the write of an stm_read() happens before the actual reading by making the C compiler believe that the write may alias the read (prevents some optimizations) Message-ID: <20140908121658.A0D301C0328@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1366:e0c1255cc369 Date: 2014-09-08 14:18 +0200 http://bitbucket.org/pypy/stmgc/changeset/e0c1255cc369/ Log: ensure that the write of an stm_read() happens before the actual reading by making the C compiler believe that the write may alias the read (prevents some optimizations) diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -262,7 +262,7 @@ OPT_ASSERT((uintptr_t)STM_PSEGMENT->last_commit_log_entry->next == -1); to = &(STM_PSEGMENT->last_commit_log_entry->next); - bool yes = __sync_bool_compare_and_swap(to, -1, new); + bool yes = __sync_bool_compare_and_swap(to, (void*)-1, new); OPT_ASSERT(yes); return; } diff --git a/c8/stm/misc.c b/c8/stm/misc.c --- a/c8/stm/misc.c +++ b/c8/stm/misc.c @@ -31,8 +31,7 @@ bool _stm_was_read(object_t *obj) { - uint8_t rm = ((struct stm_read_marker_s *) - (STM_SEGMENT->segment_base + (((uintptr_t)obj) >> 4)))->rm; + uint8_t rm = *((char *)(STM_SEGMENT->segment_base + (((uintptr_t)obj) >> 4))); assert(rm <= STM_SEGMENT->transaction_read_version); return rm == STM_SEGMENT->transaction_read_version; } diff --git a/c8/stmgc.h b/c8/stmgc.h --- a/c8/stmgc.h +++ b/c8/stmgc.h @@ -24,19 +24,8 @@ typedef TLPREFIX struct object_s object_t; typedef TLPREFIX struct stm_segment_info_s stm_segment_info_t; -typedef TLPREFIX struct stm_read_marker_s stm_read_marker_t; typedef TLPREFIX char stm_char; -struct stm_read_marker_s { - /* In every segment, every object has a corresponding read marker. - We assume that objects are at least 16 bytes long, and use - their address divided by 16. The read marker is equal to - 'STM_SEGMENT->transaction_read_version' if and only if the - object was read in the current transaction. The nurseries - also have corresponding read markers, but they are never used. */ - uint8_t rm; -}; - struct stm_segment_info_s { uint8_t transaction_read_version; int segment_num; @@ -126,8 +115,7 @@ __attribute__((always_inline)) static inline void stm_read(object_t *obj) { - ((stm_read_marker_t *)(((uintptr_t)obj) >> 4))->rm = - STM_SEGMENT->transaction_read_version; + *((stm_char *)(((uintptr_t)obj) >> 4)) = STM_SEGMENT->transaction_read_version; } __attribute__((always_inline)) From noreply at buildbot.pypy.org Mon Sep 8 14:29:52 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Mon, 8 Sep 2014 14:29:52 +0200 (CEST) Subject: [pypy-commit] stmgc default: we may need SA_NODEFER if we want to use longjmp() in the signal handler (in the future) Message-ID: <20140908122952.ECBF21C0328@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1367:4b84c28290c5 Date: 2014-09-08 14:30 +0200 http://bitbucket.org/pypy/stmgc/changeset/4b84c28290c5/ Log: we may need SA_NODEFER if we want to use longjmp() in the signal handler (in the future) diff --git a/c8/stm/setup.c b/c8/stm/setup.c --- a/c8/stm/setup.c +++ b/c8/stm/setup.c @@ -78,7 +78,7 @@ act.sa_sigaction = &_signal_handler; /* The SA_SIGINFO flag tells sigaction() to use the sa_sigaction field, not sa_handler. */ - act.sa_flags = SA_SIGINFO; + act.sa_flags = SA_SIGINFO | SA_NODEFER; if (sigaction(SIGSEGV, &act, NULL) < 0) { perror ("sigaction"); From noreply at buildbot.pypy.org Mon Sep 8 15:02:02 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Mon, 8 Sep 2014 15:02:02 +0200 (CEST) Subject: [pypy-commit] stmgc default: remove signal handler again (for now) Message-ID: <20140908130202.AF42C1C009A@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1368:e314b91fcfec Date: 2014-09-08 15:03 +0200 http://bitbucket.org/pypy/stmgc/changeset/e314b91fcfec/ Log: remove signal handler again (for now) diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -5,7 +5,6 @@ #include -/* ############# signal handler ############# */ static void copy_bk_objs_from(int from_segnum, uintptr_t pagenum) { @@ -33,83 +32,7 @@ release_modified_objs_lock(from_segnum); } -static void bring_page_up_to_date(uintptr_t pagenum) -{ - /* Assuming pagenum is not yet private in this segment and - currently read-protected: - pagecopy from somewhere readable, then update - all written objs from that segment */ - - assert(!is_shared_log_page(pagenum)); - - /* XXX: this may not even be necessary: */ - assert(STM_PSEGMENT->privatization_lock); - - long i; - int my_segnum = STM_SEGMENT->segment_num; - - assert(!is_readable_log_page_in(my_segnum, pagenum)); - assert(!is_private_log_page_in(my_segnum, pagenum)); - - /* privatize and make readable */ - pages_set_protection(my_segnum, pagenum, 1, PROT_READ|PROT_WRITE); - page_privatize(pagenum); - - /* look for a segment where the page is readable (and - therefore private): */ - for (i = 0; i < NB_SEGMENTS; i++) { - if (i == my_segnum) - continue; - if (!is_readable_log_page_in(i, pagenum)) - continue; - - acquire_privatization_lock(i); - /* nobody should be able to change this because we have - our own privatization lock: */ - assert(is_readable_log_page_in(i, pagenum)); - - /* copy the content from there to our segment */ - dprintf(("pagecopy pagenum:%lu, src: %lu, dst:%d\n", pagenum, i, my_segnum)); - pagecopy((char*)(get_virt_page_of(my_segnum, pagenum) * 4096UL), - (char*)(get_virt_page_of(i, pagenum) * 4096UL)); - - /* get valid state from backup copies of written objs in - the range of this page: */ - copy_bk_objs_from(i, pagenum); - - release_privatization_lock(i); - - return; - } - - abort(); /* didn't find a page to copy from?? */ -} - -void _signal_handler(int sig, siginfo_t *siginfo, void *context) -{ - char *addr = siginfo->si_addr; - dprintf(("si_addr: %p\n", addr)); - if (addr == NULL || addr < stm_object_pages || addr > stm_object_pages+TOTAL_MEMORY) { - /* actual segfault */ - /* send to GDB (XXX) */ - kill(getpid(), SIGINT); - } - /* XXX: should we save 'errno'? */ - - - int segnum = get_segment_of_linear_address(addr); - OPT_ASSERT(segnum == STM_SEGMENT->segment_num); - dprintf(("-> segment: %d\n", segnum)); - char *seg_base = STM_SEGMENT->segment_base; - uintptr_t pagenum = ((char*)addr - seg_base) / 4096UL; - - acquire_privatization_lock(segnum); - bring_page_up_to_date(pagenum); - release_privatization_lock(segnum); - - return; -} /* ############# commit log ############# */ @@ -144,15 +67,9 @@ size_t obj_size; uintptr_t pagenum = (uintptr_t)obj / 4096UL; - if (!is_private_log_page_in(STM_SEGMENT->segment_num, pagenum)) { - /* done in signal handler on first access anyway */ - assert(!is_shared_log_page(pagenum)); - assert(!is_readable_log_page_in(STM_SEGMENT->segment_num, pagenum)); - return; - } + assert(is_private_log_page_in(STM_SEGMENT->segment_num, pagenum)); + assert(!is_shared_log_page(pagenum)); - /* should be readable & private (XXX: maybe not after major GCs) */ - assert(is_readable_log_page_in(from_seg, pagenum)); assert(is_private_log_page_in(from_seg, pagenum)); /* look the obj up in the other segment's modified_old_objects to @@ -292,7 +209,7 @@ /* ############# STM ############# */ -void _privatize_and_protect_other_segments(object_t *obj) +void _privatize_shared_page(uintptr_t pagenum) { /* privatize pages of obj for our segment iff previously the pages were fully shared. */ @@ -303,42 +220,22 @@ } #endif - uintptr_t first_page = ((uintptr_t)obj) / 4096UL; - char *realobj; - size_t obj_size; uintptr_t i; int my_segnum = STM_SEGMENT->segment_num; - realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj); - obj_size = stmcb_size_rounded_up((struct object_s *)realobj); - assert(obj_size < 4096); /* XXX */ + assert(is_shared_log_page(pagenum)); + char *src = (char*)(get_virt_page_of(0, pagenum) * 4096UL); - /* privatize pages by read-protecting them in other - segments and giving us a private copy: */ - assert(is_shared_log_page(first_page)); - /* XXX: change this logic: - right now, privatization means private in seg0 and private - in my_segnum */ - for (i = 0; i < NB_SEGMENTS; i++) { - assert(!is_private_log_page_in(i, first_page)); + for (i = 1; i < NB_SEGMENTS; i++) { + assert(!is_private_log_page_in(i, pagenum)); - if (i != my_segnum && i != 0) - pages_set_protection(i, first_page, 1, PROT_NONE); - else /* both, seg0 and my_segnum: */ - pages_set_protection(i, first_page, 1, PROT_READ|PROT_WRITE); + page_privatize_in(i, pagenum); + pagecopy((char*)(get_virt_page_of(i, pagenum) * 4096UL), src); } + set_page_private_in(0, pagenum); - /* remap pages for my_segnum and copy the contents */ - set_page_private_in(0, first_page); - /* seg0 already up-to-date */ - if (my_segnum != 0) { - page_privatize(first_page); - pagecopy((char*)(get_virt_page_of(my_segnum, first_page) * 4096UL), - (char*)(get_virt_page_of(0, first_page) * 4096UL)); - } - - assert(is_private_log_page_in(my_segnum, first_page)); - assert(is_readable_log_page_in(my_segnum, first_page)); + assert(is_private_log_page_in(my_segnum, pagenum)); + assert(!is_shared_log_page(pagenum)); } void _stm_write_slowpath(object_t *obj) @@ -372,7 +269,7 @@ acquire_privatization_lock(i); } if (is_shared_log_page(first_page)) - _privatize_and_protect_other_segments(obj); + _privatize_shared_page(first_page); for (i = NB_SEGMENTS-1; i >= 0; i--) { release_privatization_lock(i); } @@ -381,9 +278,7 @@ only a read protected page ourselves: */ acquire_privatization_lock(my_segnum); - if (!is_readable_log_page_in(my_segnum, first_page)) - bring_page_up_to_date(first_page); - /* page is not PROT_NONE for us */ + OPT_ASSERT(is_private_log_page_in(my_segnum, first_page)); /* remove the WRITE_BARRIER flag */ obj->stm_flags &= ~GCFLAG_WRITE_BARRIER; diff --git a/c8/stm/pages.c b/c8/stm/pages.c --- a/c8/stm/pages.c +++ b/c8/stm/pages.c @@ -72,24 +72,21 @@ uint64_t bitmask = 1UL << i; uintptr_t amount = count; while (amount-->0) { - volatile struct page_shared_s *ps = (volatile struct page_shared_s *) - &pages_readable[pagenum + amount - PAGE_FLAG_START]; volatile struct page_shared_s *ps2 = (volatile struct page_shared_s *) &pages_privatized[pagenum + amount - PAGE_FLAG_START]; - ps->by_segment |= bitmask; /* readable */ ps2->by_segment = 0; /* not private */ } } } -static void page_privatize(uintptr_t pagenum) +static void page_privatize_in(int segnum, uintptr_t pagenum) { /* hopefully holding the lock */ - assert(STM_PSEGMENT->privatization_lock); + assert(get_priv_segment(segnum)->privatization_lock); /* check this thread's 'pages_privatized' bit */ - uint64_t bitmask = 1UL << STM_SEGMENT->segment_num; + uint64_t bitmask = 1UL << segnum; volatile struct page_shared_s *ps = (volatile struct page_shared_s *) &pages_privatized[pagenum - PAGE_FLAG_START]; if (ps->by_segment & bitmask) { @@ -97,7 +94,7 @@ return; } - dprintf(("page_privatize(%lu) in seg:%d\n", pagenum, STM_SEGMENT->segment_num)); + dprintf(("page_privatize(%lu) in seg:%d\n", pagenum, segnum)); /* add this thread's 'pages_privatized' bit */ ps->by_segment |= bitmask; @@ -106,39 +103,7 @@ again to its underlying file offset (XXX later we should again attempt to group together many calls to d_remap_file_pages() in succession) */ - uintptr_t pagenum_in_file = NB_PAGES * STM_SEGMENT->segment_num + pagenum; + uintptr_t pagenum_in_file = NB_PAGES * segnum + pagenum; char *new_page = stm_object_pages + pagenum_in_file * 4096UL; d_remap_file_pages(new_page, 4096, pagenum_in_file); } - -static void pages_set_protection(int segnum, uintptr_t pagenum, - uintptr_t count, int prot) -{ - /* we hopefully hold the privatization lock: */ - assert(get_priv_segment(segnum)->privatization_lock); - - if ((prot == PROT_NONE && !is_readable_log_page_in(segnum, pagenum)) - || (prot == (PROT_READ|PROT_WRITE) && is_readable_log_page_in(segnum, pagenum))) - return; - - char *addr = (char*)(get_virt_page_of(segnum, pagenum) * 4096UL); - mprotect(addr, count * 4096UL, prot); - - dprintf(("pages_set_protection(%d, %lu, %lu, %d), virtpage:%lu\n", - segnum, pagenum, count, prot, - get_virt_page_of(segnum, pagenum))); - - uint64_t bitmask = 1UL << segnum; - uintptr_t amount = count; - while (amount-->0) { - volatile struct page_shared_s *ps = (volatile struct page_shared_s *) - &pages_readable[pagenum + amount - PAGE_FLAG_START]; - if (prot == PROT_NONE) { - /* not readable */ - ps->by_segment &= ~bitmask; - } else { - assert(prot == (PROT_READ|PROT_WRITE)); - ps->by_segment |= bitmask; - } - } -} diff --git a/c8/stm/pages.h b/c8/stm/pages.h --- a/c8/stm/pages.h +++ b/c8/stm/pages.h @@ -38,13 +38,9 @@ }; static struct page_shared_s pages_privatized[PAGE_FLAG_END - PAGE_FLAG_START]; -static struct page_shared_s pages_readable[PAGE_FLAG_END - PAGE_FLAG_START]; static void pages_initialize_shared(uintptr_t pagenum, uintptr_t count); -static void page_privatize(uintptr_t pagenum); -static void pages_set_protection(int segnum, uintptr_t pagenum, - uintptr_t count, int prot); - +static void page_privatize_in(int segnum, uintptr_t pagenum); static inline uintptr_t get_virt_page_of(long segnum, uintptr_t pagenum) { @@ -73,10 +69,3 @@ uint64_t bitmask = 1UL << segnum; return (pages_privatized[pagenum - PAGE_FLAG_START].by_segment & bitmask); } - -static inline bool is_readable_log_page_in(long segnum, uintptr_t pagenum) -{ - assert(pagenum >= PAGE_FLAG_START); - uint64_t bitmask = 1UL << segnum; - return (pages_readable[pagenum - PAGE_FLAG_START].by_segment & bitmask); -} diff --git a/c8/stm/setup.c b/c8/stm/setup.c --- a/c8/stm/setup.c +++ b/c8/stm/setup.c @@ -71,20 +71,6 @@ } } -static void setup_signal_handler(void) -{ - struct sigaction act; - memset(&act, 0, sizeof(act)); - - act.sa_sigaction = &_signal_handler; - /* The SA_SIGINFO flag tells sigaction() to use the sa_sigaction field, not sa_handler. */ - act.sa_flags = SA_SIGINFO | SA_NODEFER; - - if (sigaction(SIGSEGV, &act, NULL) < 0) { - perror ("sigaction"); - abort(); - } -} void stm_setup(void) { @@ -105,7 +91,6 @@ stm_object_pages = setup_mmap("initial stm_object_pages mmap()", &stm_object_pages_fd); setup_protection_settings(); - setup_signal_handler(); long i; for (i = 0; i < NB_SEGMENTS; i++) { From noreply at buildbot.pypy.org Mon Sep 8 15:22:46 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Mon, 8 Sep 2014 15:22:46 +0200 (CEST) Subject: [pypy-commit] stmgc default: remove another leftover Message-ID: <20140908132246.BA47E1C009A@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1369:b9cd2622e3d1 Date: 2014-09-08 15:12 +0200 http://bitbucket.org/pypy/stmgc/changeset/b9cd2622e3d1/ Log: remove another leftover diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -6,34 +6,6 @@ -static void copy_bk_objs_from(int from_segnum, uintptr_t pagenum) -{ - acquire_modified_objs_lock(from_segnum); - struct tree_s *tree = get_priv_segment(from_segnum)->modified_old_objects; - wlog_t *item; - TREE_LOOP_FORWARD(tree, item); { - if (item->addr >= pagenum * 4096UL && item->addr < (pagenum + 1) * 4096UL) { - /* obj is in range. XXX: no page overlapping objs allowed yet */ - - object_t *obj = (object_t*)item->addr; - struct object_s* bk_obj = (struct object_s *)item->val; - size_t obj_size; - - obj_size = stmcb_size_rounded_up(bk_obj); - assert(obj_size < 4096); /* XXX */ - - memcpy(REAL_ADDRESS(STM_SEGMENT->segment_base, obj), - bk_obj, obj_size); - - assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); /* bk_obj never written */ - } - } TREE_LOOP_END; - - release_modified_objs_lock(from_segnum); -} - - - /* ############# commit log ############# */ From noreply at buildbot.pypy.org Mon Sep 8 15:22:47 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Mon, 8 Sep 2014 15:22:47 +0200 (CEST) Subject: [pypy-commit] stmgc default: support multi-page objects Message-ID: <20140908132247.E4AC81C009A@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1370:b8275f2293c3 Date: 2014-09-08 15:24 +0200 http://bitbucket.org/pypy/stmgc/changeset/b8275f2293c3/ Log: support multi-page objects diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -35,21 +35,20 @@ /* during validation this looks up the obj in the from_seg (backup or normal) and copies the version over the current segment's one */ + size_t obj_size; char *realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj); - size_t obj_size; + uintptr_t pagenum = (uintptr_t)obj / 4096UL; - uintptr_t pagenum = (uintptr_t)obj / 4096UL; + assert(!is_shared_log_page(pagenum)); assert(is_private_log_page_in(STM_SEGMENT->segment_num, pagenum)); - assert(!is_shared_log_page(pagenum)); - assert(is_private_log_page_in(from_seg, pagenum)); /* look the obj up in the other segment's modified_old_objects to get its backup copy: */ acquire_modified_objs_lock(from_seg); + wlog_t *item; struct tree_s *tree = get_priv_segment(from_seg)->modified_old_objects; - wlog_t *item; TREE_FIND(tree, (uintptr_t)obj, item, goto not_found); obj_size = stmcb_size_rounded_up((struct object_s*)item->val); @@ -217,12 +216,14 @@ assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); int my_segnum = STM_SEGMENT->segment_num; - uintptr_t first_page = ((uintptr_t)obj) / 4096UL; + uintptr_t end_page, first_page = ((uintptr_t)obj) / 4096UL; char *realobj; size_t obj_size; realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj); obj_size = stmcb_size_rounded_up((struct object_s *)realobj); + /* get the last page containing data from the object */ + end_page = (((uintptr_t)obj) + obj_size - 1) / 4096UL; /* add to read set: */ stm_read(obj); @@ -231,17 +232,18 @@ struct object_s *bk_obj = malloc(obj_size); memcpy(bk_obj, realobj, obj_size); - assert(obj_size < 4096); /* too lazy right now (part of the code is ready) */ - if (is_shared_log_page(first_page)) { /* acquire all privatization locks, make private and read protect others */ long i; + uintptr_t page; for (i = 0; i < NB_SEGMENTS; i++) { acquire_privatization_lock(i); } - if (is_shared_log_page(first_page)) - _privatize_shared_page(first_page); + for (page = first_page; page <= end_page; page++) { + if (is_shared_log_page(page)) + _privatize_shared_page(page); + } for (i = NB_SEGMENTS-1; i >= 0; i--) { release_privatization_lock(i); } From noreply at buildbot.pypy.org Mon Sep 8 18:15:05 2014 From: noreply at buildbot.pypy.org (rguillebert) Date: Mon, 8 Sep 2014 18:15:05 +0200 (CEST) Subject: [pypy-commit] extradoc extradoc: Draft of the status of PyPy proposal for PyCon 2015 Message-ID: <20140908161505.AB17B1C0328@cobra.cs.uni-duesseldorf.de> Author: Romain Guillebert Branch: extradoc Changeset: r5391:e97b47cafad7 Date: 2014-09-08 18:15 +0200 http://bitbucket.org/pypy/extradoc/changeset/e97b47cafad7/ Log: Draft of the status of PyPy proposal for PyCon 2015 diff --git a/talk/pycon2015/status/abstract.rst b/talk/pycon2015/status/abstract.rst new file mode 100644 --- /dev/null +++ b/talk/pycon2015/status/abstract.rst @@ -0,0 +1,14 @@ +Status of PyPy and its ecosystem +================================ + +Description +----------- + +PyPy is a fast alternative implementation of Python. This talk will describe +what happened in the PyPy ecosystem in the last 2 years and what the future +holds. Topics such as JIT improvements, garbage collection, CFFI, Numpy, STM +and Python 3 will be covered. + +Abstract +-------- + From noreply at buildbot.pypy.org Mon Sep 8 18:19:18 2014 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 8 Sep 2014 18:19:18 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: add a field name to the repr of FieldDescr Message-ID: <20140908161918.76BEF1C0328@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73375:4af5a49f9130 Date: 2014-09-08 10:18 -0600 http://bitbucket.org/pypy/pypy/changeset/4af5a49f9130/ Log: add a field name to the repr of FieldDescr diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py --- a/rpython/jit/backend/llsupport/descr.py +++ b/rpython/jit/backend/llsupport/descr.py @@ -95,6 +95,9 @@ self.field_size = field_size self.flag = flag + def __repr__(self): + return 'FieldDescr<%s:%d>' % (self.name, self.offset) + def is_pointer_field(self): return self.flag == FLAG_POINTER From noreply at buildbot.pypy.org Mon Sep 8 18:41:48 2014 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 8 Sep 2014 18:41:48 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: oops Message-ID: <20140908164148.6DDF51C0612@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73376:11af6c309524 Date: 2014-09-08 10:41 -0600 http://bitbucket.org/pypy/pypy/changeset/11af6c309524/ Log: oops diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py --- a/rpython/jit/backend/llsupport/descr.py +++ b/rpython/jit/backend/llsupport/descr.py @@ -96,7 +96,7 @@ self.flag = flag def __repr__(self): - return 'FieldDescr<%s:%d>' % (self.name, self.offset) + return 'FieldDescr<%s>' % (self.name,) def is_pointer_field(self): return self.flag == FLAG_POINTER From noreply at buildbot.pypy.org Mon Sep 8 19:10:18 2014 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 8 Sep 2014 19:10:18 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: I knew I forgot abvout something Message-ID: <20140908171018.86C491C0328@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73377:807c0457f3f6 Date: 2014-09-08 11:09 -0600 http://bitbucket.org/pypy/pypy/changeset/807c0457f3f6/ Log: I knew I forgot abvout something diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py --- a/rpython/jit/codewriter/jtransform.py +++ b/rpython/jit/codewriter/jtransform.py @@ -948,6 +948,9 @@ for FIELD in STRUCT._flds.values(): if isinstance(FIELD, lltype.Ptr) and FIELD._needsgc(): return True + elif isinstance(FIELD, lltype.Struct): + if self._has_gcptrs_in(FIELD): + return True return False def rewrite_op_getinteriorarraysize(self, op): From noreply at buildbot.pypy.org Mon Sep 8 19:45:19 2014 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 8 Sep 2014 19:45:19 +0200 (CEST) Subject: [pypy-commit] pypy.org extradoc: update for 2.4-beta Message-ID: <20140908174519.91E3D1C0350@cobra.cs.uni-duesseldorf.de> Author: Matti Picus Branch: extradoc Changeset: r533:ba0ccd283f5e Date: 2014-09-08 20:42 +0300 http://bitbucket.org/pypy/pypy.org/changeset/ba0ccd283f5e/ Log: update for 2.4-beta diff --git a/compat.html b/compat.html --- a/compat.html +++ b/compat.html @@ -45,12 +45,12 @@

    Python compatibility

    -

    PyPy implements the Python language version 2.7.6. It supports all of the core +

    PyPy implements the Python language version 2.7.8. It supports all of the core language, passing Python test suite (with minor modifications that were already accepted in the main python in newer versions). It supports most of the commonly used Python standard library modules; details below.

    -

    PyPy has alpha/beta-level support for the CPython C API, however, as of -2.3.1 release this feature is not yet complete. We strongly advise use of CFFI +

    PyPy has alpha/beta-level support for the CPython C API, however, +this feature is not yet complete. We strongly advise use of CFFI instead. CFFI come builtin with PyPy. Many libraries will require a bit of effort to work, but there are known success stories. Check out PyPy blog for updates, as well as the Compatibility Wiki.

    diff --git a/download.html b/download.html --- a/download.html +++ b/download.html @@ -50,10 +50,12 @@ performance improvements. Note that the OS X nightly builds (but not the release) are slightly miscompiled due to buildslave being old. Contributions are welcomed.

    -

    Here are the binaries for the current release — PyPy 2.3.1 — -(what's new in PyPy 2.3.1? ) for x86 and ARM Linux, Mac OS/X, Windows -and — PyPy3 2.3.1 — -(what's new in PyPy3 2.3.1?).

    +

    We provide binaries for x86 and ARM Linux, Mac OS/X and Windows for:

    +
    • Download
      • Default (with a JIT Compiler)
        • @@ -72,13 +74,11 @@

        These binaries include a Just-in-Time compiler. They only work on x86 CPUs that have the SSE2 instruction set (most of them do, nowadays), or on x86-64 CPUs. They also contain stackless -extensions, like greenlets. -(This is the official release 2.3.1; -for the most up-to-date version see below.)

        +extensions, like greenlets.

    -
    -

    2.3.1

    -

    Note that Linux binaries are dynamically linked, as is usual, and thus might +

    +

    Linux binaries and common distributions

    +

    Linux binaries are dynamically linked, as is usual, and thus might not be usable due to the sad story of linux binary compatibility. This means that Linux binaries are only usable on the distributions written next to them unless you're ready to hack your system by adding symlinks to the @@ -88,6 +88,9 @@ Fedora, Gentoo and Arch are known to package PyPy, with various degrees of being up-to-date. You may have more luck trying out Squeaky's portable Linux binaries.

    +
    +
    +

    Python2.7 compatible PyPy 2.3.1

    -
    -

    PyPy3 2.3.1

    -

    Note that Linux binaries are dynamically linked, as is usual, and thus might -not be usable due to the sad story of linux binary compatibility. This means -that Linux binaries are only usable on the distributions written next to -them unless you're ready to hack your system by adding symlinks to the -libraries it tries to open.

    + +
    +

    Python 3.3 compatible PyPy3 2.3.1

    @@ -158,7 +171,7 @@ uncompressed, they run in-place. For now you can uncompress them either somewhere in your home directory or, say, in /opt, and if you want, put a symlink from somewhere like -/usr/local/bin/pypy to /path/to/pypy-2.3.1/bin/pypy. Do +/usr/local/bin/pypy to /path/to/pypy-2.4-beta1/bin/pypy. Do not move or copy the executable pypy outside the tree – put a symlink to it, otherwise it will not find its libraries.

    @@ -259,7 +272,8 @@

    Once PyPy is translated from source the binary package similar to those provided in the section Default (with a JIT Compiler) above could be easily created with package.py script as following:

     cd ./pypy/pypy/tool/release/
-python package.py ../../.. pypy-my-own-package-name
+python package.py --help #for information
+python package.py --archive-name pypy-my-own-package-name
    @@ -275,6 +289,13 @@ 5dab108869a039b754da5c07046fb17c pypy-2.3.1-linux-armel.tar.bz2 2b9aeccef1587a42fb5a4cc304b5d881 pypy-2.3.1-src.tar.bz2 15c068c357d60719086b23e0bf9d0a5b pypy-2.3.1-src.zip +cc19c6807d6fa3e0d6e0e8c9f5331aa5 pypy-2.4-beta1-linux64.tar.bz2 +03597850c1179a55870da11bc1f89d3b pypy-2.4-beta1-linux-armel.tar.bz2 +3fe7f30be24796812538beec1df67154 pypy-2.4-beta1-linux-armhf-raring.tar.bz2 +f68438ea507d8236ec2c330078c4c8fe pypy-2.4-beta1-linux-armhf-raspbian.tar.bz2 +f59f63ea54a7f53bf21e58d49ca1be75 pypy-2.4-beta1-linux.tar.bz2 +d4d9c315e38fd52564d57add80804707 pypy-2.4-beta1-osx64.tar.bz2 +1191e71a86beef48861a61327c4ff4fc pypy-2.4-beta1-win32.zip a86da5688dfd84e0485239f8991af308 pypy3-2.3.1-linux64.tar.bz2 b0d6a0491e9c9be39d3c314c0823a039 pypy3-2.3.1-linux-armel.tar.bz2 f79f7b5588d2b5a68d2781908bc8f9af pypy3-2.3.1-linux-armhf-raring.tar.bz2 @@ -295,6 +316,13 @@ ad8ebf67c5ccf354513a9cdb0586080b5964a5bd pypy-2.3.1-linux-armel.tar.bz2 833b33042456fe381cae4481b2eb536c5787d6c7 pypy-2.3.1-src.tar.bz2 0d3f750fc28713eca77a91388c5a63843406d631 pypy-2.3.1-src.zip +7bb3e8fa4b459f15046c2604161c0191032a9e9b pypy-2.4-beta1-linux64.tar.bz2 +99bcafcf0c30adbc9a228ebefc6499a645dda27e pypy-2.4-beta1-linux-armel.tar.bz2 +6f9828cfbe58029e2c82fb60631e9abe931fd5f3 pypy-2.4-beta1-linux-armhf-raring.tar.bz2 +d8cf8d20abb8209c580a930a1fcc76aa2c0a8084 pypy-2.4-beta1-linux-armhf-raspbian.tar.bz2 +c405dc962357d5a7f62b4c693980ad533bca93d5 pypy-2.4-beta1-linux.tar.bz2 +3606211ba23f3b1ffafbd707097c57d83f959d4e pypy-2.4-beta1-osx64.tar.bz2 +4d6f30261a4cdde55372f1d1f7e3764c9724d1a9 pypy-2.4-beta1-win32.zip 7276a9e97229e754f66863a0793c59066120ec51 pypy3-2.3.1-linux64.tar.bz2 fb52a30be0fd4c7d8686c98e03595a8b48b11e82 pypy3-2.3.1-linux-armel.tar.bz2 0239677fe28a4c22a70853242368456b98ac665a pypy3-2.3.1-linux-armhf-raring.tar.bz2 diff --git a/features.html b/features.html --- a/features.html +++ b/features.html @@ -45,7 +45,7 @@

    Features

    -

    PyPy 2.3.1 implements Python 2.7.6 and runs on Intel +

    PyPy 2.4 implements Python 2.7.8 and runs on Intel x86 (IA-32) , x86_64 and ARM platforms, with PPC being stalled. It supports all of the core language, passing the Python test suite (with minor modifications that were already accepted in the main python diff --git a/index.html b/index.html --- a/index.html +++ b/index.html @@ -46,7 +46,7 @@

    Welcome to PyPy

    PyPy is a fast, compliant alternative implementation of the Python -language (2.7.6 and 3.2.5). It has several advantages and distinct features:

    +language (2.7.8 and 3.2.5). It has several advantages and distinct features:

    • Speed: thanks to its Just-in-Time compiler, Python programs @@ -63,7 +63,7 @@
    • As well as other features.
    -

    Download and try out the PyPy or PyPy3 release 2.3.1!

    +

    Download and try out the PyPy or PyPy3 release 2.3.1, or the 2.4-beta!

    Want to know more? A good place to start is our detailed speed and compatibility reports!

    diff --git a/source/compat.txt b/source/compat.txt --- a/source/compat.txt +++ b/source/compat.txt @@ -3,13 +3,13 @@ title: Python compatibility --- -PyPy implements the Python language version 2.7.6. It supports all of the core +PyPy implements the Python language version 2.7.8. It supports all of the core language, passing Python test suite (with minor modifications that were already accepted in the main python in newer versions). It supports most of the commonly used Python `standard library modules`_; details below. -PyPy has **alpha/beta-level** support for the `CPython C API`_, however, as of -2.3.1 release this feature is not yet complete. We strongly advise use of `CFFI`_ +PyPy has **alpha/beta-level** support for the `CPython C API`_, however, +this feature is not yet complete. We strongly advise use of `CFFI`_ instead. CFFI come builtin with PyPy. Many libraries will require a bit of effort to work, but there are known success stories. Check out PyPy blog for updates, as well as the `Compatibility Wiki`__. diff --git a/source/download.txt b/source/download.txt --- a/source/download.txt +++ b/source/download.txt @@ -14,13 +14,15 @@ (but not the release) are slightly miscompiled due to buildslave being old. Contributions are welcomed**. -Here are the binaries for the current release — **PyPy 2.3.1** — -(`what's new in PyPy 2.3.1?`_ ) for x86 and ARM Linux, Mac OS/X, Windows -and — **PyPy3 2.3.1** — -(`what's new in PyPy3 2.3.1?`_). +We provide binaries for x86 and ARM Linux, Mac OS/X and Windows for: + +* the Python2.7 compatible release — **PyPy 2.3.1** — (`what's new in PyPy 2.3.1?`_ ) +* the Python2.7 compatible beta — **PyPy 2.4-beta1** — (`what's new in PyPy 2.4?`_ ) +* the Python3.3 compatible release — **PyPy3 2.3.1** — (`what's new in PyPy3 2.3.1?`_). .. _what's new in PyPy 2.3.1?: http://doc.pypy.org/en/latest/release-2.3.1.html .. _what's new in PyPy3 2.3.1?: http://doc.pypy.org/en/latest/release-pypy3-2.3.1.html +.. _what's new in PyPy 2.4?: http://doc.pypy.org/en/latest/release-2.4.0.html .. class:: download_menu @@ -46,13 +48,11 @@ x86 CPUs that have the SSE2_ instruction set (most of them do, nowadays), or on x86-64 CPUs. They also contain `stackless`_ extensions, like `greenlets`_. -(This is the official release 2.3.1; -for the most up-to-date version see below.) -2.3.1 ------ +Linux binaries and common distributions +--------------------------------------- -Note that Linux binaries are dynamically linked, as is usual, and thus might +Linux binaries are dynamically linked, as is usual, and thus might not be usable due to the sad story of linux binary compatibility. This means that **Linux binaries are only usable on the distributions written next to them** unless you're ready to hack your system by adding symlinks to the @@ -72,6 +72,10 @@ .. _`Arch`: https://wiki.archlinux.org/index.php/PyPy .. _`portable Linux binaries`: https://github.com/squeaky-pl/portable-pypy + +Python2.7 compatible PyPy 2.3.1 +------------------------------- + * `Linux x86 binary (32bit, tar.bz2 built on Ubuntu 10.04.4 LTS)`__ (see ``[1]`` below) * `Linux x86-64 binary (64bit, tar.bz2 built on Ubuntu 12.04 - 14.04)`__ (see ``[1]`` below) * `ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Raspbian)`__ (see ``[1]`` below) @@ -98,14 +102,33 @@ .. __: https://bitbucket.org/pypy/pypy/downloads .. _mirror: http://cobra.cs.uni-duesseldorf.de/~buildmaster/mirror/ -PyPy3 2.3.1 ------------ +Python2.7 compatible PyPy 2.4-beta1 +----------------------------------- -Note that Linux binaries are dynamically linked, as is usual, and thus might -not be usable due to the sad story of linux binary compatibility. This means -that **Linux binaries are only usable on the distributions written next to -them** unless you're ready to hack your system by adding symlinks to the -libraries it tries to open. +* `Linux x86 binary (32bit, tar.bz2 built on Ubuntu 12.04 - 14.04)`__ (see ``[1]`` below) +* `Linux x86-64 binary (64bit, tar.bz2 built on Ubuntu 12.04 - 14.04)`__ (see ``[1]`` below) +* `ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Raspbian)`__ (see ``[1]`` below) +* `ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Ubuntu Raring)`__ (see ``[1]`` below) +* `ARM Softfloat Linux binary (ARMEL/gnueabi, tar.bz2, Ubuntu Precise)`__ (see ``[1]`` below) +* `Mac OS/X binary (64bit)`__ +* `Windows binary (32bit)`__ (you might need the `VS 2008 runtime library + installer vcredist_x86.exe`_.) +* `All our downloads,`__ including previous versions. We also have a + mirror_, but please use only if you have troubles accessing the links above + +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.4-beta1-linux.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.4-beta1-linux64.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.4-beta1-linux-armhf-raspbian.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.4-beta1-linux-armhf-raring.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.4-beta1-linux-armel.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.4-beta1-osx64.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.4-beta1-win32.zip +.. _`VS 2008 runtime library installer vcredist_x86.exe`: http://www.microsoft.com/en-us/download/details.aspx?id=5582 +.. __: https://bitbucket.org/pypy/pypy/downloads +.. _mirror: http://cobra.cs.uni-duesseldorf.de/~buildmaster/mirror/ + +Python 3.3 compatible PyPy3 2.3.1 +--------------------------------- * `Linux x86 binary (32bit, tar.bz2 built on Ubuntu 10.04.4 LTS)`__ (see ``[1]`` below) * `Linux x86-64 binary (64bit, tar.bz2 built on Ubuntu 12.04 - 14.04)`__ (see ``[1]`` below) @@ -137,7 +160,7 @@ ``[1]:`` stating it again: the Linux binaries are provided for the distributions listed here. **If your distribution is not exactly this -one, it won't work,** likely: ``pypy: error while loading shared +one, it won't work,** you will probably see: ``pypy: error while loading shared libraries: ...``. Unless you want to hack a lot, try out the `portable Linux binaries`_. @@ -178,7 +201,7 @@ uncompressed, they run in-place. For now you can uncompress them either somewhere in your home directory or, say, in ``/opt``, and if you want, put a symlink from somewhere like -``/usr/local/bin/pypy`` to ``/path/to/pypy-2.3.1/bin/pypy``. Do +``/usr/local/bin/pypy`` to ``/path/to/pypy-2.4-beta1/bin/pypy``. Do not move or copy the executable ``pypy`` outside the tree --- put a symlink to it, otherwise it will not find its libraries. @@ -289,7 +312,7 @@ .. _`contact us`: contact.html .. _`sandboxing`: features.html#sandboxing .. _`stackless`: http://www.stackless.com/ -.. _`greenlets`: http://codespeak.net/svn/greenlet/trunk/doc/greenlet.txt +.. _`greenlets`: http://pypy.readthedocs.org/en/latest/stackless.html#greenlets .. _Mercurial: http://mercurial.selenic.com/ .. _`nightly binary builds`: http://buildbot.pypy.org/nightly/trunk/ .. _`shadow stack`: http://pypy.readthedocs.org/en/latest/config/translation.gcrootfinder.html @@ -301,7 +324,8 @@ Once PyPy is translated from source the binary package similar to those provided in the section `Default (with a JIT Compiler)`_ above could be easily created with ``package.py`` script as following:: cd ./pypy/pypy/tool/release/ - python package.py ../../.. pypy-my-own-package-name + python package.py --help #for information + python package.py --archive-name pypy-my-own-package-name Checksums --------- @@ -318,6 +342,15 @@ 5dab108869a039b754da5c07046fb17c pypy-2.3.1-linux-armel.tar.bz2 2b9aeccef1587a42fb5a4cc304b5d881 pypy-2.3.1-src.tar.bz2 15c068c357d60719086b23e0bf9d0a5b pypy-2.3.1-src.zip + + cc19c6807d6fa3e0d6e0e8c9f5331aa5 pypy-2.4-beta1-linux64.tar.bz2 + 03597850c1179a55870da11bc1f89d3b pypy-2.4-beta1-linux-armel.tar.bz2 + 3fe7f30be24796812538beec1df67154 pypy-2.4-beta1-linux-armhf-raring.tar.bz2 + f68438ea507d8236ec2c330078c4c8fe pypy-2.4-beta1-linux-armhf-raspbian.tar.bz2 + f59f63ea54a7f53bf21e58d49ca1be75 pypy-2.4-beta1-linux.tar.bz2 + d4d9c315e38fd52564d57add80804707 pypy-2.4-beta1-osx64.tar.bz2 + 1191e71a86beef48861a61327c4ff4fc pypy-2.4-beta1-win32.zip + a86da5688dfd84e0485239f8991af308 pypy3-2.3.1-linux64.tar.bz2 b0d6a0491e9c9be39d3c314c0823a039 pypy3-2.3.1-linux-armel.tar.bz2 f79f7b5588d2b5a68d2781908bc8f9af pypy3-2.3.1-linux-armhf-raring.tar.bz2 @@ -339,6 +372,15 @@ ad8ebf67c5ccf354513a9cdb0586080b5964a5bd pypy-2.3.1-linux-armel.tar.bz2 833b33042456fe381cae4481b2eb536c5787d6c7 pypy-2.3.1-src.tar.bz2 0d3f750fc28713eca77a91388c5a63843406d631 pypy-2.3.1-src.zip + + 7bb3e8fa4b459f15046c2604161c0191032a9e9b pypy-2.4-beta1-linux64.tar.bz2 + 99bcafcf0c30adbc9a228ebefc6499a645dda27e pypy-2.4-beta1-linux-armel.tar.bz2 + 6f9828cfbe58029e2c82fb60631e9abe931fd5f3 pypy-2.4-beta1-linux-armhf-raring.tar.bz2 + d8cf8d20abb8209c580a930a1fcc76aa2c0a8084 pypy-2.4-beta1-linux-armhf-raspbian.tar.bz2 + c405dc962357d5a7f62b4c693980ad533bca93d5 pypy-2.4-beta1-linux.tar.bz2 + 3606211ba23f3b1ffafbd707097c57d83f959d4e pypy-2.4-beta1-osx64.tar.bz2 + 4d6f30261a4cdde55372f1d1f7e3764c9724d1a9 pypy-2.4-beta1-win32.zip + 7276a9e97229e754f66863a0793c59066120ec51 pypy3-2.3.1-linux64.tar.bz2 fb52a30be0fd4c7d8686c98e03595a8b48b11e82 pypy3-2.3.1-linux-armel.tar.bz2 0239677fe28a4c22a70853242368456b98ac665a pypy3-2.3.1-linux-armhf-raring.tar.bz2 diff --git a/source/features.txt b/source/features.txt --- a/source/features.txt +++ b/source/features.txt @@ -6,7 +6,7 @@ PyPy features =========================================================== -**PyPy 2.3.1** implements **Python 2.7.6** and runs on Intel +**PyPy 2.4** implements **Python 2.7.8** and runs on Intel `x86 (IA-32)`_ , `x86_64`_ and `ARM`_ platforms, with PPC being stalled. It supports all of the core language, passing the Python test suite (with minor modifications that were already accepted in the main python diff --git a/source/index.txt b/source/index.txt --- a/source/index.txt +++ b/source/index.txt @@ -4,7 +4,7 @@ --- PyPy is a `fast`_, `compliant`_ alternative implementation of the `Python`_ -language (2.7.6 and 3.2.5). It has several advantages and distinct features: +language (2.7.8 and 3.2.5). It has several advantages and distinct features: * **Speed:** thanks to its Just-in-Time compiler, Python programs often run `faster`_ on PyPy. `(What is a JIT compiler?)`_ @@ -26,7 +26,7 @@ .. class:: download -`Download and try out the PyPy or PyPy3 release 2.3.1!`__ +`Download and try out the PyPy or PyPy3 release 2.3.1, or the 2.4-beta!`__ .. __: download.html @@ -40,7 +40,7 @@ .. _`(What is a JIT compiler?)`: http://en.wikipedia.org/wiki/Just-in-time_compilation .. _`run untrusted code`: features.html#sandboxing .. _`compliant`: compat.html -.. _`Python docs`: http://docs.python.org/release/2.7.3/ +.. _`Python docs`: http://docs.python.org/2.7 .. _`twisted`: https://twistedmatrix.com/ .. _`django`: https://www.djangoproject.com/ .. _`cffi`: https://cffi.readthedocs.org From noreply at buildbot.pypy.org Mon Sep 8 20:01:45 2014 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 8 Sep 2014 20:01:45 +0200 (CEST) Subject: [pypy-commit] pypy default: some fixes Message-ID: <20140908180145.99BD11C0350@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r73378:403da2e83293 Date: 2014-09-08 12:01 -0600 http://bitbucket.org/pypy/pypy/changeset/403da2e83293/ Log: some fixes diff --git a/pypy/doc/release-2.4.0.rst b/pypy/doc/release-2.4.0.rst --- a/pypy/doc/release-2.4.0.rst +++ b/pypy/doc/release-2.4.0.rst @@ -62,7 +62,8 @@ bytearray handling, and a major rewrite of the GIL handling. This means that external calls are now a lot faster, especially the CFFI ones. It also means better performance in a lot of corner cases with handling strings or -bytearrays. +bytearrays. The main bugfix is handling of many socket objects in your +program which in the long run used to "leak" memory. PyPy now uses Python 2.7.8 standard library. @@ -110,7 +111,8 @@ .. _resolved: https://bitbucket.org/pypy/pypy/issues?status=resolved We have further improvements on the way: rpython file handling and -usable numpy linalg compatabiity should be merged soon. +usable numpy linalg compatabiity should be merged soon as well +as improved GC and a bunch of smaller improvements. Please try it out and let us know what you think. We especially welcome success stories, we know you are using PyPy, please tell us about it! From noreply at buildbot.pypy.org Mon Sep 8 20:13:25 2014 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 8 Sep 2014 20:13:25 +0200 (CEST) Subject: [pypy-commit] pypy default: typos and wording Message-ID: <20140908181325.1B4661C0328@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r73379:508330e3b0e2 Date: 2014-09-08 21:13 +0300 http://bitbucket.org/pypy/pypy/changeset/508330e3b0e2/ Log: typos and wording diff --git a/pypy/doc/release-2.4.0.rst b/pypy/doc/release-2.4.0.rst --- a/pypy/doc/release-2.4.0.rst +++ b/pypy/doc/release-2.4.0.rst @@ -110,9 +110,9 @@ .. _`whats-new`: http://doc.pypy.org/en/latest/whatsnew-2.3.1.html .. _resolved: https://bitbucket.org/pypy/pypy/issues?status=resolved -We have further improvements on the way: rpython file handling and -usable numpy linalg compatabiity should be merged soon as well -as improved GC and a bunch of smaller improvements. +We have further improvements on the way: rpython file handling, +numpy linalg compatability, as well +as improved GC and many smaller improvements. Please try it out and let us know what you think. We especially welcome success stories, we know you are using PyPy, please tell us about it! From noreply at buildbot.pypy.org Mon Sep 8 20:23:52 2014 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 8 Sep 2014 20:23:52 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: one forgotten spot about zeroing Message-ID: <20140908182352.C95C81C0350@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73380:6eebdba3529c Date: 2014-09-08 12:23 -0600 http://bitbucket.org/pypy/pypy/changeset/6eebdba3529c/ Log: one forgotten spot about zeroing diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py --- a/rpython/jit/metainterp/optimizeopt/virtualize.py +++ b/rpython/jit/metainterp/optimizeopt/virtualize.py @@ -2,7 +2,7 @@ from rpython.jit.metainterp.executor import execute from rpython.jit.codewriter.heaptracker import vtable2descr from rpython.jit.metainterp.history import Const, ConstInt, BoxInt -from rpython.jit.metainterp.history import CONST_NULL, BoxPtr +from rpython.jit.metainterp.history import CONST_NULL, BoxPtr, CONST_FALSE from rpython.jit.metainterp.optimizeopt import optimizer from rpython.jit.metainterp.optimizeopt.optimizer import OptValue, REMOVED from rpython.jit.metainterp.optimizeopt.util import (make_dispatcher_method, @@ -556,6 +556,7 @@ vrefinfo = self.optimizer.metainterp_sd.virtualref_info c_cls = vrefinfo.jit_virtual_ref_const_class descr_virtual_token = vrefinfo.descr_virtual_token + descr_forced = vrefinfo.descr_forced # # Replace the VIRTUAL_REF operation with a virtual structure of type # 'jit_virtual_ref'. The jit_virtual_ref structure may be forced soon, @@ -565,6 +566,7 @@ tokenbox = BoxPtr() self.emit_operation(ResOperation(rop.FORCE_TOKEN, [], tokenbox)) vrefvalue.setfield(descr_virtual_token, self.getvalue(tokenbox)) + vrefvalue.setfield(descr_forced, CONST_FALSE) def optimize_VIRTUAL_REF_FINISH(self, op): # This operation is used in two cases. In normal cases, it From noreply at buildbot.pypy.org Mon Sep 8 20:41:19 2014 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 8 Sep 2014 20:41:19 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: fix Message-ID: <20140908184119.6D6221C0350@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73381:aff39db87e89 Date: 2014-09-08 12:40 -0600 http://bitbucket.org/pypy/pypy/changeset/aff39db87e89/ Log: fix diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py --- a/rpython/jit/metainterp/optimizeopt/virtualize.py +++ b/rpython/jit/metainterp/optimizeopt/virtualize.py @@ -2,7 +2,7 @@ from rpython.jit.metainterp.executor import execute from rpython.jit.codewriter.heaptracker import vtable2descr from rpython.jit.metainterp.history import Const, ConstInt, BoxInt -from rpython.jit.metainterp.history import CONST_NULL, BoxPtr, CONST_FALSE +from rpython.jit.metainterp.history import CONST_NULL, BoxPtr from rpython.jit.metainterp.optimizeopt import optimizer from rpython.jit.metainterp.optimizeopt.optimizer import OptValue, REMOVED from rpython.jit.metainterp.optimizeopt.util import (make_dispatcher_method, @@ -566,7 +566,7 @@ tokenbox = BoxPtr() self.emit_operation(ResOperation(rop.FORCE_TOKEN, [], tokenbox)) vrefvalue.setfield(descr_virtual_token, self.getvalue(tokenbox)) - vrefvalue.setfield(descr_forced, CONST_FALSE) + vrefvalue.setfield(descr_forced, self.optimizer.cpu.ts.CVAL_NULLREF) def optimize_VIRTUAL_REF_FINISH(self, op): # This operation is used in two cases. In normal cases, it From noreply at buildbot.pypy.org Mon Sep 8 20:54:14 2014 From: noreply at buildbot.pypy.org (rguillebert) Date: Mon, 8 Sep 2014 20:54:14 +0200 (CEST) Subject: [pypy-commit] extradoc extradoc: Add the beginning of an abstract Message-ID: <20140908185414.090821C0350@cobra.cs.uni-duesseldorf.de> Author: Romain Guillebert Branch: extradoc Changeset: r5392:b4c2529c96c0 Date: 2014-09-08 20:54 +0200 http://bitbucket.org/pypy/extradoc/changeset/b4c2529c96c0/ Log: Add the beginning of an abstract diff --git a/talk/pycon2015/status/abstract.rst b/talk/pycon2015/status/abstract.rst --- a/talk/pycon2015/status/abstract.rst +++ b/talk/pycon2015/status/abstract.rst @@ -12,3 +12,17 @@ Abstract -------- +This talk will cover what has happened in the PyPy world during the last 2 +years : + +- Progress in Python 3 support and Numpy support + +- Performance improvements + +- An attempt to remove the GIL using Software Transactional Memory + +- CFFI : The best way to interface with C code on PyPy + +- Status of the crowdfunding + +- ARM From noreply at buildbot.pypy.org Mon Sep 8 21:03:31 2014 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 8 Sep 2014 21:03:31 +0200 (CEST) Subject: [pypy-commit] pypy default: The instructions "vadd.i64" etc. are only ARMv7. Message-ID: <20140908190331.6C1A31C0350@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73382:b39b2e55cd23 Date: 2014-09-08 21:02 +0200 http://bitbucket.org/pypy/pypy/changeset/b39b2e55cd23/ Log: The instructions "vadd.i64" etc. are only ARMv7. diff --git a/rpython/jit/backend/arm/detect.py b/rpython/jit/backend/arm/detect.py --- a/rpython/jit/backend/arm/detect.py +++ b/rpython/jit/backend/arm/detect.py @@ -38,9 +38,9 @@ try: buf = os.read(fd, 2048) if not buf: + n = 6 # we assume ARMv6 as base case debug_print("Could not detect ARM architecture " "version, assuming", "ARMv%d" % n) - n = 6 # we asume ARMv6 as base case finally: os.close(fd) # "Processor : ARMv%d-compatible processor rev 7 (v6l)" diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py --- a/rpython/jit/backend/arm/opassembler.py +++ b/rpython/jit/backend/arm/opassembler.py @@ -1128,6 +1128,8 @@ self.mc.VCVT_int_to_float(res.value, r.svfp_ip.value) return fcond + # the following five instructions are only ARMv7; + # regalloc.py won't call them at all on ARMv6 emit_op_llong_add = gen_emit_float_op('llong_add', 'VADD_i64') emit_op_llong_sub = gen_emit_float_op('llong_sub', 'VSUB_i64') emit_op_llong_and = gen_emit_float_op('llong_and', 'VAND_i64') diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py --- a/rpython/jit/backend/arm/regalloc.py +++ b/rpython/jit/backend/arm/regalloc.py @@ -184,7 +184,7 @@ class Regalloc(BaseRegalloc): - def __init__(self, assembler=None): + def __init__(self, assembler): self.cpu = assembler.cpu self.assembler = assembler self.frame_manager = None @@ -290,7 +290,7 @@ return self.vfprm.convert_to_imm(value) def _prepare(self, inputargs, operations, allgcrefs): - cpu = self.assembler.cpu + cpu = self.cpu self.fm = ARMFrameManager(cpu.get_baseofs_of_frame_field()) self.frame_manager = self.fm operations = cpu.gc_ll_descr.rewrite_assembler(cpu, operations, @@ -550,18 +550,19 @@ EffectInfo.OS_LLONG_AND, EffectInfo.OS_LLONG_OR, EffectInfo.OS_LLONG_XOR): - args = self._prepare_llong_binop_xx(op, fcond) - self.perform_llong(op, args, fcond) - return - if oopspecindex == EffectInfo.OS_LLONG_TO_INT: + if self.cpu.cpuinfo.arch_version >= 7: + args = self._prepare_llong_binop_xx(op, fcond) + self.perform_llong(op, args, fcond) + return + elif oopspecindex == EffectInfo.OS_LLONG_TO_INT: args = self._prepare_llong_to_int(op, fcond) self.perform_llong(op, args, fcond) return - if oopspecindex == EffectInfo.OS_MATH_SQRT: + elif oopspecindex == EffectInfo.OS_MATH_SQRT: args = self.prepare_op_math_sqrt(op, fcond) self.perform_math(op, args, fcond) return - #if oopspecindex == EffectInfo.OS_MATH_READ_TIMESTAMP: + #elif oopspecindex == EffectInfo.OS_MATH_READ_TIMESTAMP: # ... return self._prepare_call(op) @@ -590,7 +591,7 @@ # spill variables that need to be saved around calls self.vfprm.before_call(save_all_regs=save_all_regs) if not save_all_regs: - gcrootmap = self.assembler.cpu.gc_ll_descr.gcrootmap + gcrootmap = self.cpu.gc_ll_descr.gcrootmap if gcrootmap and gcrootmap.is_shadow_stack: save_all_regs = 2 self.rm.before_call(save_all_regs=save_all_regs) @@ -1082,7 +1083,7 @@ gcmap = self.get_gcmap([r.r0, r.r1]) self.possibly_free_var(t) # - gc_ll_descr = self.assembler.cpu.gc_ll_descr + gc_ll_descr = self.cpu.gc_ll_descr self.assembler.malloc_cond_varsize_frame( gc_ll_descr.get_nursery_free_addr(), gc_ll_descr.get_nursery_top_addr(), @@ -1092,7 +1093,7 @@ self.assembler._alignment_check() def prepare_op_call_malloc_nursery_varsize(self, op, fcond): - gc_ll_descr = self.assembler.cpu.gc_ll_descr + gc_ll_descr = self.cpu.gc_ll_descr if not hasattr(gc_ll_descr, 'max_size_of_young_obj'): raise Exception("unreachable code") # for boehm, this function should never be called From noreply at buildbot.pypy.org Tue Sep 9 01:37:21 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 9 Sep 2014 01:37:21 +0200 (CEST) Subject: [pypy-commit] pypy default: undo test file mode in rfile Message-ID: <20140908233721.90C161C3455@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r73383:5f89db247c03 Date: 2014-09-08 13:01 -0400 http://bitbucket.org/pypy/pypy/changeset/5f89db247c03/ Log: undo test file mode in rfile diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py --- a/rpython/rlib/rfile.py +++ b/rpython/rlib/rfile.py @@ -150,7 +150,7 @@ c_setvbuf(ll_file, buf, _IOLBF, BUFSIZ) else: c_setvbuf(ll_file, buf, _IOFBF, buffering) - return RFile(ll_file, mode) + return RFile(ll_file) def create_fdopen_rfile(fd, mode="r"): @@ -166,7 +166,7 @@ finally: lltype.free(ll_mode, flavor='raw') _dircheck(ll_file) - return RFile(ll_file, mode) + return RFile(ll_file) def create_temp_rfile(): @@ -194,17 +194,8 @@ class RFile(object): - _readable = False - _writable = False - - def __init__(self, ll_file, mode='+', close2=_fclose2): + def __init__(self, ll_file, close2=_fclose2): self._ll_file = ll_file - if 'r' in mode: - self._readable = True - if 'w' in mode or 'a' in mode: - self._writable = True - if '+' in mode: - self._readable = self._writable = True self._close2 = close2 def __del__(self): @@ -241,18 +232,9 @@ if not self._ll_file: raise ValueError("I/O operation on closed file") - def _check_reading(self): - if not self._readable: - raise IOError(0, "File not open for reading") - - def _check_writing(self): - if not self._writable: - raise IOError(0, "File not open for writing") - def read(self, size=-1): # XXX CPython uses a more delicate logic here self._check_closed() - self._check_reading() ll_file = self._ll_file if size == 0: return "" @@ -318,7 +300,6 @@ def readline(self, size=-1): self._check_closed() - self._check_reading() if size == 0: return "" elif size < 0: @@ -354,7 +335,6 @@ @enforceargs(None, str) def write(self, value): self._check_closed() - self._check_writing() ll_value = rffi.get_nonmovingbuffer(value) try: # note that since we got a nonmoving buffer, it is either raw @@ -376,7 +356,6 @@ def truncate(self, arg=-1): self._check_closed() - self._check_writing() if arg == -1: arg = self.tell() self.flush() diff --git a/rpython/rlib/test/test_rfile.py b/rpython/rlib/test/test_rfile.py --- a/rpython/rlib/test/test_rfile.py +++ b/rpython/rlib/test/test_rfile.py @@ -129,13 +129,13 @@ try: f.read() except IOError as e: - assert not e.errno + pass else: assert False try: f.readline() except IOError as e: - assert not e.errno + pass else: assert False f.write("dupa\x00dupb") @@ -145,7 +145,7 @@ try: f2.write('') except IOError as e: - assert not e.errno + pass else: assert False dupa = f2.read(0) @@ -230,7 +230,7 @@ try: f2.read() except IOError as e: - assert not e.errno + pass else: assert False f2.write("xxx") @@ -303,7 +303,7 @@ try: f.truncate() except IOError as e: - assert not e.errno + pass else: assert False f.close() From noreply at buildbot.pypy.org Tue Sep 9 01:37:22 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 9 Sep 2014 01:37:22 +0200 (CEST) Subject: [pypy-commit] pypy default: need to clear file error here Message-ID: <20140908233722.C97AF1C3455@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r73384:369f2b3d76f7 Date: 2014-09-08 13:58 -0400 http://bitbucket.org/pypy/pypy/changeset/369f2b3d76f7/ Log: need to clear file error here diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py --- a/rpython/rlib/rfile.py +++ b/rpython/rlib/rfile.py @@ -343,6 +343,7 @@ bytes = c_fwrite(ll_value, 1, length, self._ll_file) if bytes != length: errno = rposix.get_errno() + c_clearerr(self._ll_file) raise IOError(errno, os.strerror(errno)) finally: rffi.free_nonmovingbuffer(value, ll_value) diff --git a/rpython/rlib/test/test_rfile.py b/rpython/rlib/test/test_rfile.py --- a/rpython/rlib/test/test_rfile.py +++ b/rpython/rlib/test/test_rfile.py @@ -143,7 +143,7 @@ for mode in ['r', 'U']: f2 = open(fname, mode) try: - f2.write('') + f2.write('z') except IOError as e: pass else: From noreply at buildbot.pypy.org Tue Sep 9 01:37:24 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 9 Sep 2014 01:37:24 +0200 (CEST) Subject: [pypy-commit] pypy default: univ newline support for rfile read Message-ID: <20140908233724.040BE1C3455@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r73385:4d7e9cbb05f7 Date: 2014-09-08 18:12 -0400 http://bitbucket.org/pypy/pypy/changeset/4d7e9cbb05f7/ Log: univ newline support for rfile read diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py --- a/rpython/rlib/rfile.py +++ b/rpython/rlib/rfile.py @@ -47,6 +47,11 @@ BASE_BUF_SIZE = 4096 BASE_LINE_SIZE = 100 +NEWLINE_UNKNOWN = 0 +NEWLINE_CR = 1 +NEWLINE_LF = 2 +NEWLINE_CRLF = 4 + def llexternal(*args, **kwargs): return rffi.llexternal(*args, compilation_info=eci, **kwargs) @@ -128,10 +133,10 @@ def create_file(filename, mode="r", buffering=-1): - mode = _sanitize_mode(mode) + newmode = _sanitize_mode(mode) ll_name = rffi.str2charp(filename) try: - ll_mode = rffi.str2charp(mode) + ll_mode = rffi.str2charp(newmode) try: ll_file = c_fopen(ll_name, ll_mode) if not ll_file: @@ -150,14 +155,14 @@ c_setvbuf(ll_file, buf, _IOLBF, BUFSIZ) else: c_setvbuf(ll_file, buf, _IOFBF, buffering) - return RFile(ll_file) + return RFile(ll_file, mode) def create_fdopen_rfile(fd, mode="r"): - mode = _sanitize_mode(mode) + newmode = _sanitize_mode(mode) fd = rffi.cast(rffi.INT, fd) rposix.validate_fd(fd) - ll_mode = rffi.str2charp(mode) + ll_mode = rffi.str2charp(newmode) try: ll_file = c_fdopen(fd, ll_mode) if not ll_file: @@ -166,7 +171,7 @@ finally: lltype.free(ll_mode, flavor='raw') _dircheck(ll_file) - return RFile(ll_file) + return RFile(ll_file, mode) def create_temp_rfile(): @@ -194,8 +199,14 @@ class RFile(object): - def __init__(self, ll_file, close2=_fclose2): + _univ_newline = False + _newlinetypes = NEWLINE_UNKNOWN + _skipnextlf = False + + def __init__(self, ll_file, mode=None, close2=_fclose2): self._ll_file = ll_file + if mode is not None: + self._univ_newline = 'U' in mode self._close2 = close2 def __del__(self): @@ -232,6 +243,52 @@ if not self._ll_file: raise ValueError("I/O operation on closed file") + def _fread(self, buf, n, stream): + if not self._univ_newline: + return c_fread(buf, 1, n, stream) + + i = 0 + dst = buf + newlinetypes = self._newlinetypes + skipnextlf = self._skipnextlf + while n: + nread = c_fread(dst, 1, n, stream) + if nread == 0: + break + + src = dst + n -= nread + shortread = n != 0 + while nread: + nread -= 1 + c = src[0] + src = rffi.ptradd(src, 1) + if c == '\r': + dst[0] = '\n' + dst = rffi.ptradd(dst, 1) + i += 1 + skipnextlf = True + elif skipnextlf and c == '\n': + skipnextlf = False + newlinetypes |= NEWLINE_CRLF + n += 1 + else: + if c == '\n': + newlinetypes |= NEWLINE_LF + elif skipnextlf: + newlinetypes |= NEWLINE_CR + dst[0] = c + dst = rffi.ptradd(dst, 1) + i += 1 + skipnextlf = False + if shortread: + if skipnextlf and c_feof(stream): + newlinetypes |= NEWLINE_CR + break + self._newlinetypes = newlinetypes + self._skipnextlf = skipnextlf + return i + def read(self, size=-1): # XXX CPython uses a more delicate logic here self._check_closed() @@ -244,7 +301,7 @@ try: s = StringBuilder() while True: - returned_size = c_fread(buf, 1, BASE_BUF_SIZE, ll_file) + returned_size = self._fread(buf, BASE_BUF_SIZE, ll_file) returned_size = intmask(returned_size) # is between 0 and BASE_BUF_SIZE if returned_size == 0: if c_feof(ll_file): @@ -256,7 +313,7 @@ lltype.free(buf, flavor='raw') else: # size > 0 with rffi.scoped_alloc_buffer(size) as buf: - returned_size = c_fread(buf.raw, 1, size, ll_file) + returned_size = self._fread(buf.raw, size, ll_file) returned_size = intmask(returned_size) # is between 0 and size if returned_size == 0: if not c_feof(ll_file): diff --git a/rpython/rlib/test/test_rfile.py b/rpython/rlib/test/test_rfile.py --- a/rpython/rlib/test/test_rfile.py +++ b/rpython/rlib/test/test_rfile.py @@ -189,6 +189,23 @@ f() self.interpret(f, []) + def test_read_universal(self): + fname = self.tmpdir.join('read_univ') + fname.write("dupa\ndupb\r\ndupc") + fname = str(fname) + + def f(): + f = open(fname, 'U') + assert f.read() == "dupa\ndupb\ndupc" + f.close() + f = open(fname, 'U') + assert f.read(9) == "dupa\ndupb" + assert f.read(42) == "\ndupc" + f.close() + + f() + self.interpret(f, []) + def test_seek(self): fname = str(self.tmpdir.join('file_4')) From noreply at buildbot.pypy.org Tue Sep 9 01:37:25 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 9 Sep 2014 01:37:25 +0200 (CEST) Subject: [pypy-commit] pypy default: univ newline support for rfile readline Message-ID: <20140908233725.2F2441C3455@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r73386:d2dc19e8a7b4 Date: 2014-09-08 19:01 -0400 http://bitbucket.org/pypy/pypy/changeset/d2dc19e8a7b4/ Log: univ newline support for rfile readline diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py --- a/rpython/rlib/rfile.py +++ b/rpython/rlib/rfile.py @@ -359,7 +359,7 @@ self._check_closed() if size == 0: return "" - elif size < 0: + elif size < 0 and not self._univ_newline: with rffi.scoped_alloc_buffer(BASE_LINE_SIZE) as buf: c = self._readline1(buf.raw) if c >= 0: @@ -374,19 +374,50 @@ break s.append_charpsize(buf.raw, c) return s.build() - else: # size > 0 + else: # size > 0 or self._univ_newline ll_file = self._ll_file + c = 0 s = StringBuilder() - while s.getlength() < size: - c = c_getc(ll_file) + if self._univ_newline: + newlinetypes = self._newlinetypes + skipnextlf = self._skipnextlf + while size < 0 or s.getlength() < size: + c = c_getc(ll_file) + if c == EOF: + break + if skipnextlf: + skipnextlf = False + if c == ord('\n'): + newlinetypes |= NEWLINE_CRLF + c = c_getc(ll_file) + if c == EOF: + break + else: + newlinetypes |= NEWLINE_CR + if c == ord('\r'): + skipnextlf = True + c = ord('\n') + elif c == ord('\n'): + newlinetypes |= NEWLINE_LF + s.append(chr(c)) + if c == ord('\n'): + break if c == EOF: - if c_ferror(ll_file): - raise _error(ll_file) - break - c = chr(c) - s.append(c) - if c == '\n': - break + if skipnextlf: + newlinetypes |= NEWLINE_CR + self._newlinetypes = newlinetypes + self._skipnextlf = skipnextlf + else: + while s.getlength() < size: + c = c_getc(ll_file) + if c == EOF: + break + s.append(chr(c)) + if c == ord('\n'): + break + if c == EOF: + if c_ferror(ll_file): + raise _error(ll_file) return s.build() @enforceargs(None, str) diff --git a/rpython/rlib/test/test_rfile.py b/rpython/rlib/test/test_rfile.py --- a/rpython/rlib/test/test_rfile.py +++ b/rpython/rlib/test/test_rfile.py @@ -197,10 +197,16 @@ def f(): f = open(fname, 'U') assert f.read() == "dupa\ndupb\ndupc" - f.close() - f = open(fname, 'U') + assert f.read() == "" + f.seek(0) assert f.read(9) == "dupa\ndupb" assert f.read(42) == "\ndupc" + assert f.read(1) == "" + f.seek(0) + assert f.readline() == "dupa\n" + assert f.readline() == "dupb\n" + assert f.readline() == "dupc" + assert f.readline() == "" f.close() f() From noreply at buildbot.pypy.org Tue Sep 9 01:37:26 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 9 Sep 2014 01:37:26 +0200 (CEST) Subject: [pypy-commit] pypy default: test/fix rfile tell with universal newlines Message-ID: <20140908233726.6B3D31C3455@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r73387:dc540ed089f2 Date: 2014-09-08 19:35 -0400 http://bitbucket.org/pypy/pypy/changeset/dc540ed089f2/ Log: test/fix rfile tell with universal newlines diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py --- a/rpython/rlib/rfile.py +++ b/rpython/rlib/rfile.py @@ -77,6 +77,7 @@ _pclose2 = (c_pclose, c_pclose_in_del) c_getc = llexternal('getc', [FILEP], rffi.INT, macro=True) +c_ungetc = llexternal('ungetc', [rffi.INT, FILEP], rffi.INT) c_fgets = llexternal('fgets', [rffi.CCHARP, rffi.INT, FILEP], rffi.CCHARP) c_fread = llexternal('fread', [rffi.CCHARP, rffi.SIZE_T, rffi.SIZE_T, FILEP], rffi.SIZE_T) @@ -466,6 +467,14 @@ if res == -1: errno = rposix.get_errno() raise IOError(errno, os.strerror(errno)) + if self._skipnextlf: + c = c_getc(self._ll_file) + if c == ord('\n'): + self._newlinetypes |= NEWLINE_CRLF + res += 1 + self._skipnextlf = False + elif c != EOF: + c_ungetc(c, self._ll_file) return res def fileno(self): diff --git a/rpython/rlib/test/test_rfile.py b/rpython/rlib/test/test_rfile.py --- a/rpython/rlib/test/test_rfile.py +++ b/rpython/rlib/test/test_rfile.py @@ -191,21 +191,26 @@ def test_read_universal(self): fname = self.tmpdir.join('read_univ') - fname.write("dupa\ndupb\r\ndupc") + fname.write("dupa\ndupb\r\ndupc\rdupd") fname = str(fname) def f(): f = open(fname, 'U') - assert f.read() == "dupa\ndupb\ndupc" + assert f.read() == "dupa\ndupb\ndupc\ndupd" assert f.read() == "" f.seek(0) - assert f.read(9) == "dupa\ndupb" - assert f.read(42) == "\ndupc" + assert f.read(10) == "dupa\ndupb\n" + assert f.read(42) == "dupc\ndupd" assert f.read(1) == "" f.seek(0) assert f.readline() == "dupa\n" + assert f.tell() == 5 assert f.readline() == "dupb\n" - assert f.readline() == "dupc" + assert f.tell() == 11 + assert f.readline() == "dupc\n" + assert f.tell() == 16 + assert f.readline() == "dupd" + assert f.tell() == 20 assert f.readline() == "" f.close() From noreply at buildbot.pypy.org Tue Sep 9 01:37:27 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 9 Sep 2014 01:37:27 +0200 (CEST) Subject: [pypy-commit] pypy default: merge heads Message-ID: <20140908233727.C685D1C3455@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r73388:76b2985de3ea Date: 2014-09-08 19:36 -0400 http://bitbucket.org/pypy/pypy/changeset/76b2985de3ea/ Log: merge heads diff --git a/pypy/doc/release-2.4.0.rst b/pypy/doc/release-2.4.0.rst --- a/pypy/doc/release-2.4.0.rst +++ b/pypy/doc/release-2.4.0.rst @@ -62,7 +62,8 @@ bytearray handling, and a major rewrite of the GIL handling. This means that external calls are now a lot faster, especially the CFFI ones. It also means better performance in a lot of corner cases with handling strings or -bytearrays. +bytearrays. The main bugfix is handling of many socket objects in your +program which in the long run used to "leak" memory. PyPy now uses Python 2.7.8 standard library. @@ -109,8 +110,9 @@ .. _`whats-new`: http://doc.pypy.org/en/latest/whatsnew-2.3.1.html .. _resolved: https://bitbucket.org/pypy/pypy/issues?status=resolved -We have further improvements on the way: rpython file handling and -usable numpy linalg compatabiity should be merged soon. +We have further improvements on the way: rpython file handling, +numpy linalg compatability, as well +as improved GC and many smaller improvements. Please try it out and let us know what you think. We especially welcome success stories, we know you are using PyPy, please tell us about it! diff --git a/rpython/jit/backend/arm/detect.py b/rpython/jit/backend/arm/detect.py --- a/rpython/jit/backend/arm/detect.py +++ b/rpython/jit/backend/arm/detect.py @@ -38,9 +38,9 @@ try: buf = os.read(fd, 2048) if not buf: + n = 6 # we assume ARMv6 as base case debug_print("Could not detect ARM architecture " "version, assuming", "ARMv%d" % n) - n = 6 # we asume ARMv6 as base case finally: os.close(fd) # "Processor : ARMv%d-compatible processor rev 7 (v6l)" diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py --- a/rpython/jit/backend/arm/opassembler.py +++ b/rpython/jit/backend/arm/opassembler.py @@ -1128,6 +1128,8 @@ self.mc.VCVT_int_to_float(res.value, r.svfp_ip.value) return fcond + # the following five instructions are only ARMv7; + # regalloc.py won't call them at all on ARMv6 emit_op_llong_add = gen_emit_float_op('llong_add', 'VADD_i64') emit_op_llong_sub = gen_emit_float_op('llong_sub', 'VSUB_i64') emit_op_llong_and = gen_emit_float_op('llong_and', 'VAND_i64') diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py --- a/rpython/jit/backend/arm/regalloc.py +++ b/rpython/jit/backend/arm/regalloc.py @@ -184,7 +184,7 @@ class Regalloc(BaseRegalloc): - def __init__(self, assembler=None): + def __init__(self, assembler): self.cpu = assembler.cpu self.assembler = assembler self.frame_manager = None @@ -290,7 +290,7 @@ return self.vfprm.convert_to_imm(value) def _prepare(self, inputargs, operations, allgcrefs): - cpu = self.assembler.cpu + cpu = self.cpu self.fm = ARMFrameManager(cpu.get_baseofs_of_frame_field()) self.frame_manager = self.fm operations = cpu.gc_ll_descr.rewrite_assembler(cpu, operations, @@ -550,18 +550,19 @@ EffectInfo.OS_LLONG_AND, EffectInfo.OS_LLONG_OR, EffectInfo.OS_LLONG_XOR): - args = self._prepare_llong_binop_xx(op, fcond) - self.perform_llong(op, args, fcond) - return - if oopspecindex == EffectInfo.OS_LLONG_TO_INT: + if self.cpu.cpuinfo.arch_version >= 7: + args = self._prepare_llong_binop_xx(op, fcond) + self.perform_llong(op, args, fcond) + return + elif oopspecindex == EffectInfo.OS_LLONG_TO_INT: args = self._prepare_llong_to_int(op, fcond) self.perform_llong(op, args, fcond) return - if oopspecindex == EffectInfo.OS_MATH_SQRT: + elif oopspecindex == EffectInfo.OS_MATH_SQRT: args = self.prepare_op_math_sqrt(op, fcond) self.perform_math(op, args, fcond) return - #if oopspecindex == EffectInfo.OS_MATH_READ_TIMESTAMP: + #elif oopspecindex == EffectInfo.OS_MATH_READ_TIMESTAMP: # ... return self._prepare_call(op) @@ -590,7 +591,7 @@ # spill variables that need to be saved around calls self.vfprm.before_call(save_all_regs=save_all_regs) if not save_all_regs: - gcrootmap = self.assembler.cpu.gc_ll_descr.gcrootmap + gcrootmap = self.cpu.gc_ll_descr.gcrootmap if gcrootmap and gcrootmap.is_shadow_stack: save_all_regs = 2 self.rm.before_call(save_all_regs=save_all_regs) @@ -1082,7 +1083,7 @@ gcmap = self.get_gcmap([r.r0, r.r1]) self.possibly_free_var(t) # - gc_ll_descr = self.assembler.cpu.gc_ll_descr + gc_ll_descr = self.cpu.gc_ll_descr self.assembler.malloc_cond_varsize_frame( gc_ll_descr.get_nursery_free_addr(), gc_ll_descr.get_nursery_top_addr(), @@ -1092,7 +1093,7 @@ self.assembler._alignment_check() def prepare_op_call_malloc_nursery_varsize(self, op, fcond): - gc_ll_descr = self.assembler.cpu.gc_ll_descr + gc_ll_descr = self.cpu.gc_ll_descr if not hasattr(gc_ll_descr, 'max_size_of_young_obj'): raise Exception("unreachable code") # for boehm, this function should never be called From noreply at buildbot.pypy.org Tue Sep 9 01:37:55 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 9 Sep 2014 01:37:55 +0200 (CEST) Subject: [pypy-commit] pypy use-file-star-for-file: merge default Message-ID: <20140908233755.D47C41C3455@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: use-file-star-for-file Changeset: r73389:2c58cdc115fd Date: 2014-09-08 19:12 -0400 http://bitbucket.org/pypy/pypy/changeset/2c58cdc115fd/ Log: merge default diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py --- a/rpython/rlib/rfile.py +++ b/rpython/rlib/rfile.py @@ -47,6 +47,11 @@ BASE_BUF_SIZE = 4096 BASE_LINE_SIZE = 100 +NEWLINE_UNKNOWN = 0 +NEWLINE_CR = 1 +NEWLINE_LF = 2 +NEWLINE_CRLF = 4 + def llexternal(*args, **kwargs): return rffi.llexternal(*args, compilation_info=eci, **kwargs) @@ -128,10 +133,10 @@ def create_file(filename, mode="r", buffering=-1): - mode = _sanitize_mode(mode) + newmode = _sanitize_mode(mode) ll_name = rffi.str2charp(filename) try: - ll_mode = rffi.str2charp(mode) + ll_mode = rffi.str2charp(newmode) try: ll_file = c_fopen(ll_name, ll_mode) if not ll_file: @@ -154,10 +159,10 @@ def create_fdopen_rfile(fd, mode="r"): - mode = _sanitize_mode(mode) + newmode = _sanitize_mode(mode) fd = rffi.cast(rffi.INT, fd) rposix.validate_fd(fd) - ll_mode = rffi.str2charp(mode) + ll_mode = rffi.str2charp(newmode) try: ll_file = c_fdopen(fd, ll_mode) if not ll_file: @@ -194,17 +199,14 @@ class RFile(object): - _readable = False - _writable = False + _univ_newline = False + _newlinetypes = NEWLINE_UNKNOWN + _skipnextlf = False - def __init__(self, ll_file, mode='+', close2=_fclose2): + def __init__(self, ll_file, mode=None, close2=_fclose2): self._ll_file = ll_file - if 'r' in mode: - self._readable = True - if 'w' in mode or 'a' in mode: - self._writable = True - if '+' in mode: - self._readable = self._writable = True + if mode is not None: + self._univ_newline = 'U' in mode self._close2 = close2 def __del__(self): @@ -241,18 +243,55 @@ if not self._ll_file: raise ValueError("I/O operation on closed file") - def _check_reading(self): - if not self._readable: - raise IOError(0, "File not open for reading") + def _fread(self, buf, n, stream): + if not self._univ_newline: + return c_fread(buf, 1, n, stream) - def _check_writing(self): - if not self._writable: - raise IOError(0, "File not open for writing") + i = 0 + dst = buf + newlinetypes = self._newlinetypes + skipnextlf = self._skipnextlf + while n: + nread = c_fread(dst, 1, n, stream) + if nread == 0: + break + + src = dst + n -= nread + shortread = n != 0 + while nread: + nread -= 1 + c = src[0] + src = rffi.ptradd(src, 1) + if c == '\r': + dst[0] = '\n' + dst = rffi.ptradd(dst, 1) + i += 1 + skipnextlf = True + elif skipnextlf and c == '\n': + skipnextlf = False + newlinetypes |= NEWLINE_CRLF + n += 1 + else: + if c == '\n': + newlinetypes |= NEWLINE_LF + elif skipnextlf: + newlinetypes |= NEWLINE_CR + dst[0] = c + dst = rffi.ptradd(dst, 1) + i += 1 + skipnextlf = False + if shortread: + if skipnextlf and c_feof(stream): + newlinetypes |= NEWLINE_CR + break + self._newlinetypes = newlinetypes + self._skipnextlf = skipnextlf + return i def read(self, size=-1): # XXX CPython uses a more delicate logic here self._check_closed() - self._check_reading() ll_file = self._ll_file if size == 0: return "" @@ -262,7 +301,7 @@ try: s = StringBuilder() while True: - returned_size = c_fread(buf, 1, BASE_BUF_SIZE, ll_file) + returned_size = self._fread(buf, BASE_BUF_SIZE, ll_file) returned_size = intmask(returned_size) # is between 0 and BASE_BUF_SIZE if returned_size == 0: if c_feof(ll_file): @@ -274,7 +313,7 @@ lltype.free(buf, flavor='raw') else: # size > 0 with rffi.scoped_alloc_buffer(size) as buf: - returned_size = c_fread(buf.raw, 1, size, ll_file) + returned_size = self._fread(buf.raw, size, ll_file) returned_size = intmask(returned_size) # is between 0 and size if returned_size == 0: if not c_feof(ll_file): @@ -318,10 +357,9 @@ def readline(self, size=-1): self._check_closed() - self._check_reading() if size == 0: return "" - elif size < 0: + elif size < 0 and not self._univ_newline: with rffi.scoped_alloc_buffer(BASE_LINE_SIZE) as buf: c = self._readline1(buf.raw) if c >= 0: @@ -336,25 +374,55 @@ break s.append_charpsize(buf.raw, c) return s.build() - else: # size > 0 + else: # size > 0 or self._univ_newline ll_file = self._ll_file + c = 0 s = StringBuilder() - while s.getlength() < size: - c = c_getc(ll_file) + if self._univ_newline: + newlinetypes = self._newlinetypes + skipnextlf = self._skipnextlf + while size < 0 or s.getlength() < size: + c = c_getc(ll_file) + if c == EOF: + break + if skipnextlf: + skipnextlf = False + if c == ord('\n'): + newlinetypes |= NEWLINE_CRLF + c = c_getc(ll_file) + if c == EOF: + break + else: + newlinetypes |= NEWLINE_CR + if c == ord('\r'): + skipnextlf = True + c = ord('\n') + elif c == ord('\n'): + newlinetypes |= NEWLINE_LF + s.append(chr(c)) + if c == ord('\n'): + break if c == EOF: - if c_ferror(ll_file): - raise _error(ll_file) - break - c = chr(c) - s.append(c) - if c == '\n': - break + if skipnextlf: + newlinetypes |= NEWLINE_CR + self._newlinetypes = newlinetypes + self._skipnextlf = skipnextlf + else: + while s.getlength() < size: + c = c_getc(ll_file) + if c == EOF: + break + s.append(chr(c)) + if c == ord('\n'): + break + if c == EOF: + if c_ferror(ll_file): + raise _error(ll_file) return s.build() @enforceargs(None, str) def write(self, value): self._check_closed() - self._check_writing() ll_value = rffi.get_nonmovingbuffer(value) try: # note that since we got a nonmoving buffer, it is either raw @@ -363,6 +431,7 @@ bytes = c_fwrite(ll_value, 1, length, self._ll_file) if bytes != length: errno = rposix.get_errno() + c_clearerr(self._ll_file) raise IOError(errno, os.strerror(errno)) finally: rffi.free_nonmovingbuffer(value, ll_value) @@ -376,7 +445,6 @@ def truncate(self, arg=-1): self._check_closed() - self._check_writing() if arg == -1: arg = self.tell() self.flush() diff --git a/rpython/rlib/test/test_rfile.py b/rpython/rlib/test/test_rfile.py --- a/rpython/rlib/test/test_rfile.py +++ b/rpython/rlib/test/test_rfile.py @@ -129,13 +129,13 @@ try: f.read() except IOError as e: - assert not e.errno + pass else: assert False try: f.readline() except IOError as e: - assert not e.errno + pass else: assert False f.write("dupa\x00dupb") @@ -143,9 +143,9 @@ for mode in ['r', 'U']: f2 = open(fname, mode) try: - f2.write('') + f2.write('z') except IOError as e: - assert not e.errno + pass else: assert False dupa = f2.read(0) @@ -189,6 +189,29 @@ f() self.interpret(f, []) + def test_read_universal(self): + fname = self.tmpdir.join('read_univ') + fname.write("dupa\ndupb\r\ndupc") + fname = str(fname) + + def f(): + f = open(fname, 'U') + assert f.read() == "dupa\ndupb\ndupc" + assert f.read() == "" + f.seek(0) + assert f.read(9) == "dupa\ndupb" + assert f.read(42) == "\ndupc" + assert f.read(1) == "" + f.seek(0) + assert f.readline() == "dupa\n" + assert f.readline() == "dupb\n" + assert f.readline() == "dupc" + assert f.readline() == "" + f.close() + + f() + self.interpret(f, []) + def test_seek(self): fname = str(self.tmpdir.join('file_4')) @@ -230,7 +253,7 @@ try: f2.read() except IOError as e: - assert not e.errno + pass else: assert False f2.write("xxx") @@ -303,7 +326,7 @@ try: f.truncate() except IOError as e: - assert not e.errno + pass else: assert False f.close() From noreply at buildbot.pypy.org Tue Sep 9 01:37:57 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 9 Sep 2014 01:37:57 +0200 (CEST) Subject: [pypy-commit] pypy use-file-star-for-file: use newlinetypes attribute from rfile Message-ID: <20140908233757.45F271C3455@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: use-file-star-for-file Changeset: r73390:b7e915440ad5 Date: 2014-09-08 19:12 -0400 http://bitbucket.org/pypy/pypy/changeset/b7e915440ad5/ Log: use newlinetypes attribute from rfile diff --git a/pypy/module/_file/interp_file.py b/pypy/module/_file/interp_file.py --- a/pypy/module/_file/interp_file.py +++ b/pypy/module/_file/interp_file.py @@ -121,7 +121,7 @@ def direct_close(self): stream = self.stream if stream is not None: - #self.newlines = self.stream.getnewlines() + self.newlines = self.stream._newlinetypes self.stream = None openstreams = getopenstreams(self.space) try: @@ -452,8 +452,7 @@ def descr_file_newlines(space, file): if file.stream: - #newlines = file.stream.getnewlines() - newlines = file.newlines + newlines = file.stream._newlinetypes else: newlines = file.newlines if newlines == 0: From noreply at buildbot.pypy.org Tue Sep 9 01:37:58 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 9 Sep 2014 01:37:58 +0200 (CEST) Subject: [pypy-commit] pypy use-file-star-for-file: merge default Message-ID: <20140908233758.7DBDA1C3455@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: use-file-star-for-file Changeset: r73391:21e58ff89bd9 Date: 2014-09-08 19:35 -0400 http://bitbucket.org/pypy/pypy/changeset/21e58ff89bd9/ Log: merge default diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py --- a/rpython/rlib/rfile.py +++ b/rpython/rlib/rfile.py @@ -77,6 +77,7 @@ _pclose2 = (c_pclose, c_pclose_in_del) c_getc = llexternal('getc', [FILEP], rffi.INT, macro=True) +c_ungetc = llexternal('ungetc', [rffi.INT, FILEP], rffi.INT) c_fgets = llexternal('fgets', [rffi.CCHARP, rffi.INT, FILEP], rffi.CCHARP) c_fread = llexternal('fread', [rffi.CCHARP, rffi.SIZE_T, rffi.SIZE_T, FILEP], rffi.SIZE_T) @@ -466,6 +467,14 @@ if res == -1: errno = rposix.get_errno() raise IOError(errno, os.strerror(errno)) + if self._skipnextlf: + c = c_getc(self._ll_file) + if c == ord('\n'): + self._newlinetypes |= NEWLINE_CRLF + res += 1 + self._skipnextlf = False + elif c != EOF: + c_ungetc(c, self._ll_file) return res def fileno(self): diff --git a/rpython/rlib/test/test_rfile.py b/rpython/rlib/test/test_rfile.py --- a/rpython/rlib/test/test_rfile.py +++ b/rpython/rlib/test/test_rfile.py @@ -191,21 +191,26 @@ def test_read_universal(self): fname = self.tmpdir.join('read_univ') - fname.write("dupa\ndupb\r\ndupc") + fname.write("dupa\ndupb\r\ndupc\rdupd") fname = str(fname) def f(): f = open(fname, 'U') - assert f.read() == "dupa\ndupb\ndupc" + assert f.read() == "dupa\ndupb\ndupc\ndupd" assert f.read() == "" f.seek(0) - assert f.read(9) == "dupa\ndupb" - assert f.read(42) == "\ndupc" + assert f.read(10) == "dupa\ndupb\n" + assert f.read(42) == "dupc\ndupd" assert f.read(1) == "" f.seek(0) assert f.readline() == "dupa\n" + assert f.tell() == 5 assert f.readline() == "dupb\n" - assert f.readline() == "dupc" + assert f.tell() == 11 + assert f.readline() == "dupc\n" + assert f.tell() == 16 + assert f.readline() == "dupd" + assert f.tell() == 20 assert f.readline() == "" f.close() From noreply at buildbot.pypy.org Tue Sep 9 09:56:49 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Tue, 9 Sep 2014 09:56:49 +0200 (CEST) Subject: [pypy-commit] stmgc default: pass all of test_basic Message-ID: <20140909075649.984ED1C02BF@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1371:87edf3694c7e Date: 2014-09-09 09:58 +0200 http://bitbucket.org/pypy/stmgc/changeset/87edf3694c7e/ Log: pass all of test_basic diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -460,7 +460,7 @@ (int)pseg->transaction_state); } - throw_away_nursery(pseg); + long bytes_in_nursery = throw_away_nursery(pseg); reset_modified_from_backup_copies(segment_num); @@ -482,6 +482,7 @@ stm_rewind_jmp_restore_shadowstack(tl); assert(tl->shadowstack == pseg->shadowstack_at_start_of_transaction); #endif +tl->last_abort__bytes_in_nursery = bytes_in_nursery; #pragma pop_macro("STM_SEGMENT") #pragma pop_macro("STM_PSEGMENT") diff --git a/c8/stmgc.h b/c8/stmgc.h --- a/c8/stmgc.h +++ b/c8/stmgc.h @@ -47,6 +47,7 @@ /* rewind_setjmp's interface */ rewind_jmp_thread rjthread; struct stm_shadowentry_s *shadowstack, *shadowstack_base; + long last_abort__bytes_in_nursery; /* the next fields are handled internally by the library */ int associated_segment_num; struct stm_thread_local_s *prev, *next; diff --git a/c8/test/support.py b/c8/test/support.py --- a/c8/test/support.py +++ b/c8/test/support.py @@ -23,6 +23,7 @@ typedef struct { rewind_jmp_thread rjthread; struct stm_shadowentry_s *shadowstack, *shadowstack_base; + long last_abort__bytes_in_nursery; int associated_segment_num; struct stm_thread_local_s *prev, *next; void *creating_pthread[2]; From noreply at buildbot.pypy.org Tue Sep 9 10:45:26 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Tue, 9 Sep 2014 10:45:26 +0200 (CEST) Subject: [pypy-commit] stmgc default: start with test_nursery (fix missing GCWORD_MOVED test) Message-ID: <20140909084526.9BC921C06A7@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1372:f7891821148c Date: 2014-09-09 10:15 +0200 http://bitbucket.org/pypy/stmgc/changeset/f7891821148c/ Log: start with test_nursery (fix missing GCWORD_MOVED test) diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c --- a/c8/stm/gcpage.c +++ b/c8/stm/gcpage.c @@ -38,6 +38,10 @@ stm_fatalerror("uninitialized_page_start changed?"); } } + dprintf(("allocate_outside_nursery_large(%lu): %p, seg=%d, page=%lu\n", + size, addr, get_segment_of_linear_address(addr), + (addr - STM_SEGMENT->segment_base) / 4096UL)); + return addr; } diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c --- a/c8/stm/nursery.c +++ b/c8/stm/nursery.c @@ -63,6 +63,11 @@ where the object moved to, is stored in the second word in 'obj'. */ object_t *TLPREFIX *pforwarded_array = (object_t *TLPREFIX *)obj; + if (LIKELY(pforwarded_array[0] == GCWORD_MOVED)) { + *pobj = pforwarded_array[1]; /* already moved */ + return; + } + realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj); size = stmcb_size_rounded_up((struct object_s *)realobj); @@ -199,6 +204,16 @@ _do_minor_collection(commit); } +void stm_collect(long level) +{ + if (level > 0) + abort(); + + minor_collection(/*commit=*/ false); + /* XXX: major_collection_if_requested(); */ +} + + /************************************************************/ @@ -220,7 +235,7 @@ return (object_t *)p; } - abort();//stm_collect(0); + stm_collect(0); goto restart; } diff --git a/c8/stmgc.h b/c8/stmgc.h --- a/c8/stmgc.h +++ b/c8/stmgc.h @@ -181,6 +181,8 @@ void stm_commit_transaction(void); void stm_abort_transaction(void) __attribute__((noreturn)); +void stm_collect(long level); + #ifdef STM_NO_AUTOMATIC_SETJMP int stm_is_inevitable(void); diff --git a/c8/test/test_nursery.py b/c8/test/test_nursery.py new file mode 100644 --- /dev/null +++ b/c8/test/test_nursery.py @@ -0,0 +1,250 @@ +from support import * +import py + +class TestNursery(BaseTest): + + def test_nursery_full(self): + lib._stm_set_nursery_free_count(2048) + self.start_transaction() + self.push_root_no_gc() + lp1 = stm_allocate(2048) # no collection here + self.pop_root() + # + self.push_root(lp1) + lp2 = stm_allocate(2048) + lp1b = self.pop_root() + assert lp1b != lp1 # collection occurred + + def test_several_minor_collections(self): + # make a long, ever-growing linked list of objects, in one transaction + lib._stm_set_nursery_free_count(2048) + self.start_transaction() + lp1 = stm_allocate_refs(1) + self.push_root(lp1) + prev = lp1 + prevprev = None + FIT = 2048 / 16 - 1 # without 'lp1' above + N = 4096 / 16 + 41 + for i in range(N): + if prevprev: + assert stm_get_ref(prevprev, 0) == prev + self.push_root(prevprev) + self.push_root(prev) + lp3 = stm_allocate_refs(1) + prev = self.pop_root() + if prevprev: + prevprev = self.pop_root() + assert prevprev != prev + stm_set_ref(prev, 0, lp3) + + #assert modified_old_objects() == [] # only 1 transaction + opn = objects_pointing_to_nursery() + if i < FIT: + assert opn == [] # no minor collection so far + else: + assert len(opn) == 1 + + prevprev = prev + prev = lp3 + + lp1 = self.pop_root() + #assert modified_old_objects() == [] + + lp2 = lp1 + for i in range(N): + assert lp2 + prev = lp2 + lp2 = stm_get_ref(lp2, 0) + assert lp2 == lp3 + + def test_many_allocs(self): + lib._stm_set_nursery_free_count(32768) + obj_size = 512 + num = 65536 / obj_size + 41 + + self.start_transaction() + for i in range(num): + new = stm_allocate(obj_size) + stm_set_char(new, chr(i % 255)) + self.push_root(new) + + old = [] + young = [] + for i in reversed(range(num)): + r = self.pop_root() + assert stm_get_char(r) == chr(i % 255) + if is_in_nursery(r): + young.append(r) + else: + old.append(r) + + assert old + assert young + + def test_larger_than_limit_for_nursery_die(self): + obj_size = lib._STM_FAST_ALLOC + 16 + + self.start_transaction() + assert lib._stm_total_allocated() == 0 + seen = set() + for i in range(10): + stm_minor_collect() + new = stm_allocate(obj_size) + assert not is_in_nursery(new) + assert lib._stm_total_allocated() == obj_size + 16 + seen.add(new) + assert len(seen) < 5 # addresses are reused + stm_minor_collect() + assert lib._stm_total_allocated() == 0 + + def test_larger_than_limit_for_nursery_dont_die(self): + obj_nrefs = (lib._STM_FAST_ALLOC + 16) // 8 + + self.start_transaction() + lp1 = ffi.cast("object_t *", 0) + seen = set() + for i in range(100): + self.push_root(lp1) + stm_minor_collect() + lp1 = self.pop_root() + new = stm_allocate_refs(obj_nrefs) + assert not is_in_nursery(new) + seen.add(new) + stm_set_ref(new, i, lp1) + lp1 = new + assert len(seen) == 100 # addresses are not reused + + for i in reversed(range(100)): + assert lp1 + lp1 = stm_get_ref(lp1, i) + assert not lp1 + + def test_account_for_privatized_page(self): + self.start_transaction() + obj = stm_allocate(16) + self.push_root(obj) + self.commit_transaction() + obj = self.pop_root() + base = lib._stm_total_allocated() + assert base <= 4096 + + self.start_transaction() + stm_write(obj) + assert lib._stm_total_allocated() == base + 4096 + + def test_reset_partial_alloc_pages(self): + py.test.skip("a would-be-nice feature, but not actually needed: " + "the next major GC will take care of it") + self.start_transaction() + new = stm_allocate(16) + stm_set_char(new, 'a') + self.push_root(new) + stm_minor_collect() + new = self.pop_root() + self.abort_transaction() + + self.start_transaction() + newer = stm_allocate(16) + self.push_root(newer) + stm_minor_collect() + newer = self.pop_root() + assert stm_get_real_address(new) == stm_get_real_address(newer) + assert stm_get_char(newer) == '\0' + + def test_reuse_page(self): + py.test.skip("a would-be-nice feature, but not actually needed: " + "the next major GC will take care of it") + self.start_transaction() + new = stm_allocate(16) + self.push_root(new) + stm_minor_collect() + new = self.pop_root() + # assert stm_get_page_flag(stm_get_obj_pages(new)[0]) == lib.UNCOMMITTED_SHARED_PAGE + self.abort_transaction() + + self.start_transaction() + newer = stm_allocate(16) + self.push_root(newer) + stm_minor_collect() + newer = self.pop_root() + assert new == newer + + def test_write_to_old_after_minor(self): + self.start_transaction() + new = stm_allocate(16) + self.push_root(new) + stm_minor_collect() + old = self.pop_root() + self.commit_transaction() + + self.start_transaction() + stm_write(old) # old objs to trace + stm_set_char(old, 'x') + stm_minor_collect() + stm_write(old) # old objs to trace + stm_set_char(old, 'y') + self.commit_transaction() + + def test_can_move(self): + self.start_transaction() + new = stm_allocate(16) + assert lib.stm_can_move(new) == 1 + self.push_root(new) + stm_minor_collect() + old = self.pop_root() + assert lib.stm_can_move(old) == 0 + self.commit_transaction() + + self.start_transaction() + assert lib.stm_can_move(old) == 0 + + def test_marker_1(self): + self.start_transaction() + p1 = stm_allocate(600) + stm_set_char(p1, 'o') + self.push_root(p1) + self.push_root(ffi.cast("object_t *", 123)) + p2 = stm_allocate(600) + stm_set_char(p2, 't') + self.push_root(p2) + stm_minor_collect() + assert lib._stm_total_allocated() == 2 * 616 + # + p2 = self.pop_root() + m = self.pop_root() + assert m == ffi.cast("object_t *", 123) + p1 = self.pop_root() + assert stm_get_char(p1) == 'o' + assert stm_get_char(p2) == 't' + + def test_marker_2(self): + py.test.skip("testing this requires working shadowstack saving logic") + self.start_transaction() + p1 = stm_allocate(600) + stm_set_char(p1, 'o') + self.push_root(p1) + self.push_root(ffi.cast("object_t *", lib.STM_STACK_MARKER_OLD)) + p2 = stm_allocate(600) + stm_set_char(p2, 't') + self.push_root(p2) + stm_minor_collect() + assert lib._stm_total_allocated() == 1 * 616 + # + p2 = self.pop_root() + m = self.pop_root() + assert m == ffi.cast("object_t *", lib.STM_STACK_MARKER_OLD) + assert stm_get_char(p2) == 't' + # the 'p1' reference is invalid now, don't try to read it. + # we check that it's invalid because _stm_total_allocated() + # only records one of the two objects. + + def test_clear_read_marker_for_external_young(self): + self.start_transaction() + big = stm_allocate(FAST_ALLOC + 1000) # young outside nursery + stm_read(big) + assert stm_was_read(big) + stm_minor_collect() # free young outside + assert not stm_was_read(big) + # if the read marker is not cleared, we get false conflicts + # with later transactions using the same large-malloced slot + # as our outside-nursery-obj From noreply at buildbot.pypy.org Tue Sep 9 10:45:27 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Tue, 9 Sep 2014 10:45:27 +0200 (CEST) Subject: [pypy-commit] stmgc default: re-add young_outside_nursery and pass all expected tests in test_nursery.py Message-ID: <20140909084527.DE2E31C06A7@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1373:6d1bc582253e Date: 2014-09-09 10:46 +0200 http://bitbucket.org/pypy/stmgc/changeset/6d1bc582253e/ Log: re-add young_outside_nursery and pass all expected tests in test_nursery.py diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -257,18 +257,22 @@ /* remove the WRITE_BARRIER flag */ obj->stm_flags &= ~GCFLAG_WRITE_BARRIER; + /* also add it to the GC list for minor collections */ + LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, obj); + /* done fiddling with protection and privatization */ release_privatization_lock(my_segnum); /* phew, now add the obj to the write-set and register the backup copy. */ - acquire_modified_objs_lock(my_segnum); - tree_insert(STM_PSEGMENT->modified_old_objects, - (uintptr_t)obj, (uintptr_t)bk_obj); - release_modified_objs_lock(my_segnum); + /* XXX: possibly slow check; try overflow objs again? */ + if (!tree_contains(STM_PSEGMENT->modified_old_objects, (uintptr_t)obj)) { + acquire_modified_objs_lock(my_segnum); + tree_insert(STM_PSEGMENT->modified_old_objects, + (uintptr_t)obj, (uintptr_t)bk_obj); + release_modified_objs_lock(my_segnum); + } - /* also add it to the GC list for minor collections */ - LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, obj); } static void reset_transaction_read_version(void) @@ -321,7 +325,10 @@ assert(tree_is_cleared(STM_PSEGMENT->modified_old_objects)); assert(list_is_empty(STM_PSEGMENT->objects_pointing_to_nursery)); + assert(tree_is_cleared(STM_PSEGMENT->young_outside_nursery)); + check_nursery_at_transaction_start(); + stm_validate(NULL); } diff --git a/c8/stm/core.h b/c8/stm/core.h --- a/c8/stm/core.h +++ b/c8/stm/core.h @@ -51,6 +51,8 @@ uint8_t modified_objs_lock; struct tree_s *modified_old_objects; struct list_s *objects_pointing_to_nursery; + struct tree_s *young_outside_nursery; + uint8_t privatization_lock; uint8_t transaction_state; diff --git a/c8/stm/list.h b/c8/stm/list.h --- a/c8/stm/list.h +++ b/c8/stm/list.h @@ -135,10 +135,13 @@ //static inline void tree_delete_not_used_any_more(struct tree_s *tree)... static inline bool tree_is_cleared(struct tree_s *tree) { - assert((tree->raw_current == tree->raw_start) == (tree->count == 0)); return tree->raw_current == tree->raw_start; } +static inline bool tree_is_empty(struct tree_s *tree) { + return tree->count == 0; +} + static inline uintptr_t tree_count(struct tree_s *tree) { assert(tree->count >= 0); return tree->count; diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c --- a/c8/stm/nursery.c +++ b/c8/stm/nursery.c @@ -15,6 +15,7 @@ static void setup_nursery(void) { + assert(_STM_FAST_ALLOC <= NURSERY_SIZE); _stm_nursery_start = NURSERY_START; long i; @@ -32,7 +33,8 @@ static inline bool _is_young(object_t *obj) { - return _is_in_nursery(obj); /* XXX: young_outside_nursery */ + return (_is_in_nursery(obj) || + tree_contains(STM_PSEGMENT->young_outside_nursery, (uintptr_t)obj)); } long stm_can_move(object_t *obj) @@ -83,8 +85,14 @@ *pobj = nobj; } else { - /* XXX: young_outside_nursery */ - return; + /* The object was not in the nursery at all */ + if (LIKELY(!tree_contains(STM_PSEGMENT->young_outside_nursery, + (uintptr_t)obj))) + return; /* common case: it was an old object, nothing to do */ + + /* a young object outside the nursery */ + nobj = obj; + tree_delete_item(STM_PSEGMENT->young_outside_nursery, (uintptr_t)nobj); } /* Must trace the object later */ @@ -177,12 +185,37 @@ pseg->pub.nursery_current = (stm_char *)_stm_nursery_start; + /* free any object left from 'young_outside_nursery' */ + if (!tree_is_cleared(pseg->young_outside_nursery)) { + wlog_t *item; + + if (!tree_is_empty(pseg->young_outside_nursery)) { + /* tree may still be empty even if not cleared */ + TREE_LOOP_FORWARD(pseg->young_outside_nursery, item) { + object_t *obj = (object_t*)item->addr; + assert(!_is_in_nursery(obj)); + + /* mark slot as unread (it can only have the read marker + in this segment) */ + *((char *)(pseg->pub.segment_base + (((uintptr_t)obj) >> 4))) = 0; + + /* XXX: _stm_large_free(stm_object_pages + item->addr); */ + } TREE_LOOP_END; + } + + tree_clear(pseg->young_outside_nursery); + } + return nursery_used; #pragma pop_macro("STM_SEGMENT") #pragma pop_macro("STM_PSEGMENT") } +#define MINOR_NOTHING_TO_DO(pseg) \ + ((pseg)->pub.nursery_current == (stm_char *)_stm_nursery_start && \ + tree_is_cleared((pseg)->young_outside_nursery)) + static void _do_minor_collection(bool commit) { @@ -195,6 +228,8 @@ assert(list_is_empty(STM_PSEGMENT->objects_pointing_to_nursery)); throw_away_nursery(get_priv_segment(STM_SEGMENT->segment_num)); + + assert(MINOR_NOTHING_TO_DO(STM_PSEGMENT)); } static void minor_collection(bool commit) @@ -227,6 +262,7 @@ OPT_ASSERT(size_rounded_up >= 16); OPT_ASSERT((size_rounded_up & 7) == 0); + OPT_ASSERT(size_rounded_up < _STM_FAST_ALLOC); stm_char *p = STM_SEGMENT->nursery_current; stm_char *end = p + size_rounded_up; @@ -239,6 +275,26 @@ goto restart; } +object_t *_stm_allocate_external(ssize_t size_rounded_up) +{ + /* /\* first, force a collection if needed *\/ */ + /* if (is_major_collection_requested()) { */ + /* /\* use stm_collect() with level 0: if another thread does a major GC */ + /* in-between, is_major_collection_requested() will become false */ + /* again, and we'll avoid doing yet another one afterwards. *\/ */ + /* stm_collect(0); */ + /* } */ + + char *result = allocate_outside_nursery_large(size_rounded_up); + object_t *o = (object_t *)(result - stm_object_pages); + + tree_insert(STM_PSEGMENT->young_outside_nursery, (uintptr_t)o, 0); + + memset(REAL_ADDRESS(STM_SEGMENT->segment_base, o), 0, size_rounded_up); + return o; +} + + #ifdef STM_TESTS void _stm_set_nursery_free_count(uint64_t free_count) { diff --git a/c8/stm/setup.c b/c8/stm/setup.c --- a/c8/stm/setup.c +++ b/c8/stm/setup.c @@ -87,6 +87,7 @@ assert((NB_PAGES * 4096UL) >> 8 <= (FIRST_OBJECT_PAGE * 4096UL) >> 4); assert((END_NURSERY_PAGE * 4096UL) >> 8 <= (FIRST_READMARKER_PAGE * 4096UL)); + assert(_STM_FAST_ALLOC <= NB_NURSERY_PAGES * 4096); stm_object_pages = setup_mmap("initial stm_object_pages mmap()", &stm_object_pages_fd); @@ -109,6 +110,7 @@ pr->pub.segment_base = segment_base; pr->modified_old_objects = tree_create(); pr->objects_pointing_to_nursery = list_create(); + pr->young_outside_nursery = tree_create(); pr->last_commit_log_entry = &commit_log_root; pr->pub.transaction_read_version = 0xff; } @@ -140,6 +142,7 @@ assert(list_is_empty(pr->objects_pointing_to_nursery)); list_free(pr->objects_pointing_to_nursery); tree_free(pr->modified_old_objects); + tree_free(pr->young_outside_nursery); } munmap(stm_object_pages, TOTAL_MEMORY); diff --git a/c8/stmgc.h b/c8/stmgc.h --- a/c8/stmgc.h +++ b/c8/stmgc.h @@ -55,10 +55,11 @@ } stm_thread_local_t; #define _STM_GCFLAG_WRITE_BARRIER 0x01 - +#define _STM_FAST_ALLOC (66*1024) void _stm_write_slowpath(object_t *); object_t *_stm_allocate_slowpath(ssize_t); +object_t *_stm_allocate_external(ssize_t); void _stm_become_inevitable(const char*); object_t *_stm_allocate_old(ssize_t size_rounded_up); @@ -133,6 +134,9 @@ OPT_ASSERT(size_rounded_up >= 16); OPT_ASSERT((size_rounded_up & 7) == 0); + if (UNLIKELY(size_rounded_up >= _STM_FAST_ALLOC)) + return _stm_allocate_external(size_rounded_up); + stm_char *p = STM_SEGMENT->nursery_current; stm_char *end = p + size_rounded_up; STM_SEGMENT->nursery_current = end; diff --git a/c8/test/support.py b/c8/test/support.py --- a/c8/test/support.py +++ b/c8/test/support.py @@ -10,6 +10,7 @@ #define SIZEOF_MYOBJ ... #define STM_NB_SEGMENTS ... #define _STM_GCFLAG_WRITE_BARRIER ... +#define _STM_FAST_ALLOC ... typedef struct { ...; @@ -66,6 +67,7 @@ void _set_ptr(object_t *obj, int n, object_t *v); object_t * _get_ptr(object_t *obj, int n); +void stm_collect(long level); void _stm_set_nursery_free_count(uint64_t free_count); @@ -242,6 +244,7 @@ assert HDR == 8 GCFLAG_WRITE_BARRIER = lib._STM_GCFLAG_WRITE_BARRIER NB_SEGMENTS = lib.STM_NB_SEGMENTS +FAST_ALLOC = lib._STM_FAST_ALLOC class Conflict(Exception): pass diff --git a/c8/test/test_nursery.py b/c8/test/test_nursery.py --- a/c8/test/test_nursery.py +++ b/c8/test/test_nursery.py @@ -82,6 +82,7 @@ assert young def test_larger_than_limit_for_nursery_die(self): + py.test.xfail() obj_size = lib._STM_FAST_ALLOC + 16 self.start_transaction() @@ -120,6 +121,7 @@ assert not lp1 def test_account_for_privatized_page(self): + py.test.xfail() self.start_transaction() obj = stm_allocate(16) self.push_root(obj) @@ -180,6 +182,7 @@ self.start_transaction() stm_write(old) # old objs to trace stm_set_char(old, 'x') + assert objects_pointing_to_nursery() == [old] stm_minor_collect() stm_write(old) # old objs to trace stm_set_char(old, 'y') @@ -199,6 +202,7 @@ assert lib.stm_can_move(old) == 0 def test_marker_1(self): + py.test.xfail() self.start_transaction() p1 = stm_allocate(600) stm_set_char(p1, 'o') From noreply at buildbot.pypy.org Tue Sep 9 11:33:13 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Tue, 9 Sep 2014 11:33:13 +0200 (CEST) Subject: [pypy-commit] stmgc default: add hash/id/shadow Message-ID: <20140909093313.B86DE1C02BF@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1374:30234e0f06ae Date: 2014-09-09 11:07 +0200 http://bitbucket.org/pypy/stmgc/changeset/30234e0f06ae/ Log: add hash/id/shadow diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -326,6 +326,7 @@ assert(tree_is_cleared(STM_PSEGMENT->modified_old_objects)); assert(list_is_empty(STM_PSEGMENT->objects_pointing_to_nursery)); assert(tree_is_cleared(STM_PSEGMENT->young_outside_nursery)); + assert(tree_is_cleared(STM_PSEGMENT->nursery_objects_shadows)); check_nursery_at_transaction_start(); diff --git a/c8/stm/core.h b/c8/stm/core.h --- a/c8/stm/core.h +++ b/c8/stm/core.h @@ -35,6 +35,7 @@ enum /* stm_flags */ { GCFLAG_WRITE_BARRIER = _STM_GCFLAG_WRITE_BARRIER, + GCFLAG_HAS_SHADOW = 0x02, }; @@ -52,6 +53,7 @@ struct tree_s *modified_old_objects; struct list_s *objects_pointing_to_nursery; struct tree_s *young_outside_nursery; + struct tree_s *nursery_objects_shadows; uint8_t privatization_lock; diff --git a/c8/stm/hash_id.c b/c8/stm/hash_id.c new file mode 100644 --- /dev/null +++ b/c8/stm/hash_id.c @@ -0,0 +1,69 @@ +#ifndef _STM_CORE_H_ +# error "must be compiled via stmgc.c" +#endif + + +static long mangle_hash(long i) +{ + /* To hash pointers in dictionaries. Assumes that i shows some + alignment (to 8, 16, maybe 32 bytes), so we use the following + formula to avoid the trailing bits being always 0. */ + return i ^ (i >> 5); +} + +static long id_or_identityhash(object_t *obj, bool is_hash) +{ + long result; + + if (obj != NULL) { + if (_is_in_nursery(obj)) { + obj = find_shadow(obj); + } + else if (is_hash) { + if (obj->stm_flags & GCFLAG_HAS_SHADOW) { + + /* For identityhash(), we need a special case for some + prebuilt objects: their hash must be the same before + and after translation. It is stored as an extra word + after the object. But we cannot use it for id() + because the stored value might clash with a real one. + */ + struct object_s *realobj = (struct object_s *) + REAL_ADDRESS(STM_SEGMENT->segment_base, obj); + size_t size = stmcb_size_rounded_up(realobj); + result = *(long *)(((char *)realobj) + size); + /* Important: the returned value is not mangle_hash()ed! */ + return result; + } + } + } + + result = (long)(uintptr_t)obj; + if (is_hash) { + result = mangle_hash(result); + } + return result; +} + +long stm_id(object_t *obj) +{ + return id_or_identityhash(obj, false); +} + +long stm_identityhash(object_t *obj) +{ + return id_or_identityhash(obj, true); +} + +void stm_set_prebuilt_identityhash(object_t *obj, long hash) +{ + struct object_s *realobj = (struct object_s *) + REAL_ADDRESS(stm_object_pages, obj); + + assert(realobj->stm_flags == GCFLAG_WRITE_BARRIER); + realobj->stm_flags |= GCFLAG_HAS_SHADOW; + + size_t size = stmcb_size_rounded_up(realobj); + assert(*(long *)(((char *)realobj) + size) == 0); + *(long *)(((char *)realobj) + size) = hash; +} diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c --- a/c8/stm/nursery.c +++ b/c8/stm/nursery.c @@ -45,6 +45,7 @@ /************************************************************/ +static object_t *find_existing_shadow(object_t *obj); #define GCWORD_MOVED ((object_t *) -1) static void minor_trace_if_young(object_t **pobj) @@ -65,9 +66,23 @@ where the object moved to, is stored in the second word in 'obj'. */ object_t *TLPREFIX *pforwarded_array = (object_t *TLPREFIX *)obj; - if (LIKELY(pforwarded_array[0] == GCWORD_MOVED)) { - *pobj = pforwarded_array[1]; /* already moved */ - return; + if (obj->stm_flags & GCFLAG_HAS_SHADOW) { + /* ^^ the single check above detects both already-moved objects + and objects with HAS_SHADOW. This is because GCWORD_MOVED + overrides completely the stm_flags field with 1's bits. */ + + if (LIKELY(pforwarded_array[0] == GCWORD_MOVED)) { + *pobj = pforwarded_array[1]; /* already moved */ + return; + } + else { + /* really has a shadow */ + nobj = find_existing_shadow(obj); + obj->stm_flags &= ~GCFLAG_HAS_SHADOW; + realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj); + size = stmcb_size_rounded_up((struct object_s *)realobj); + goto copy_large_object; + } } realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj); @@ -77,6 +92,7 @@ char *allocated = allocate_outside_nursery_large(size); nobj = (object_t *)(allocated - stm_object_pages); + copy_large_object:; char *realnobj = REAL_ADDRESS(STM_SEGMENT->segment_base, nobj); memcpy(realnobj, realobj, size); @@ -206,6 +222,7 @@ tree_clear(pseg->young_outside_nursery); } + tree_clear(pseg->nursery_objects_shadows); return nursery_used; #pragma pop_macro("STM_SEGMENT") @@ -330,3 +347,62 @@ STM_SEGMENT->nursery_current), NURSERY_END - _stm_nursery_start); } + + +static object_t *allocate_shadow(object_t *obj) +{ + char *realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj); + size_t size = stmcb_size_rounded_up((struct object_s *)realobj); + + /* always gets outside as a large object for now */ + char *allocated = allocate_outside_nursery_large(size); + object_t *nobj = (object_t *)(allocated - stm_object_pages); + + /* Initialize the shadow enough to be considered a valid gc object. + If the original object stays alive at the next minor collection, + it will anyway be copied over the shadow and overwrite the + following fields. But if the object dies, then the shadow will + stay around and only be freed at the next major collection, at + which point we want it to look valid (but ready to be freed). + + Here, in the general case, it requires copying the whole object. + It could be more optimized in special cases like in PyPy, by + copying only the typeid and (for var-sized objects) the length + field. It's probably overkill to add a special stmcb_xxx + interface just for that. + */ + char *realnobj = REAL_ADDRESS(STM_SEGMENT->segment_base, nobj); + memcpy(realnobj, realobj, size); + + obj->stm_flags |= GCFLAG_HAS_SHADOW; + + tree_insert(STM_PSEGMENT->nursery_objects_shadows, + (uintptr_t)obj, (uintptr_t)nobj); + return nobj; +} + +static object_t *find_existing_shadow(object_t *obj) +{ + wlog_t *item; + + TREE_FIND(STM_PSEGMENT->nursery_objects_shadows, + (uintptr_t)obj, item, goto not_found); + + /* The answer is the address of the shadow. */ + return (object_t *)item->val; + + not_found: + stm_fatalerror("GCFLAG_HAS_SHADOW but no shadow found"); +} + +static object_t *find_shadow(object_t *obj) +{ + /* The object 'obj' is still in the nursery. Find or allocate a + "shadow" object, which is where the object will be moved by the + next minor collection + */ + if (obj->stm_flags & GCFLAG_HAS_SHADOW) + return find_existing_shadow(obj); + else + return allocate_shadow(obj); +} diff --git a/c8/stm/pages.c b/c8/stm/pages.c --- a/c8/stm/pages.c +++ b/c8/stm/pages.c @@ -69,7 +69,6 @@ } for (i = 0; i < NB_SEGMENTS; i++) { - uint64_t bitmask = 1UL << i; uintptr_t amount = count; while (amount-->0) { volatile struct page_shared_s *ps2 = (volatile struct page_shared_s *) diff --git a/c8/stm/setup.c b/c8/stm/setup.c --- a/c8/stm/setup.c +++ b/c8/stm/setup.c @@ -111,6 +111,7 @@ pr->modified_old_objects = tree_create(); pr->objects_pointing_to_nursery = list_create(); pr->young_outside_nursery = tree_create(); + pr->nursery_objects_shadows = tree_create(); pr->last_commit_log_entry = &commit_log_root; pr->pub.transaction_read_version = 0xff; } @@ -143,6 +144,7 @@ list_free(pr->objects_pointing_to_nursery); tree_free(pr->modified_old_objects); tree_free(pr->young_outside_nursery); + tree_free(pr->nursery_objects_shadows); } munmap(stm_object_pages, TOTAL_MEMORY); diff --git a/c8/stmgc.c b/c8/stmgc.c --- a/c8/stmgc.c +++ b/c8/stmgc.c @@ -22,4 +22,5 @@ #include "stm/setup.c" #include "stm/fprintcolor.c" #include "stm/rewind_setjmp.c" +#include "stm/hash_id.c" #include "stm/misc.c" diff --git a/c8/test/support.py b/c8/test/support.py --- a/c8/test/support.py +++ b/c8/test/support.py @@ -71,6 +71,11 @@ void _stm_set_nursery_free_count(uint64_t free_count); +long stm_identityhash(object_t *obj); +long stm_id(object_t *obj); +void stm_set_prebuilt_identityhash(object_t *obj, uint64_t hash); + + long _stm_count_modified_old_objects(void); long _stm_count_objects_pointing_to_nursery(void); object_t *_stm_enum_modified_old_objects(long index); diff --git a/c8/test/test_hash_id.py b/c8/test/test_hash_id.py new file mode 100644 --- /dev/null +++ b/c8/test/test_hash_id.py @@ -0,0 +1,74 @@ +from support import * +from test_prebuilt import prebuilt +import py + +class TestHashId(BaseTest): + + def test_hash_old_object(self): + lp1 = stm_allocate_old(16) + lp2 = stm_allocate_old(16) + lp3 = stm_allocate_old(16) + lp4 = stm_allocate_old(16) + self.start_transaction() + h1 = lib.stm_identityhash(lp1) + h2 = lib.stm_identityhash(lp2) + h3 = lib.stm_identityhash(lp3) + h4 = lib.stm_identityhash(lp4) + assert len(set([h1, h2, h3, h4])) == 4 # guaranteed by the algo + + def test_id_old_object(self): + lp1 = stm_allocate_old(16) + self.start_transaction() + h1 = lib.stm_id(lp1) + assert h1 == int(ffi.cast("long", lp1)) + + def test_set_prebuilt_identityhash(self): + static1 = prebuilt(16) + static2 = prebuilt(16) + lp1 = lib.stm_setup_prebuilt(static1) + lp2 = lib.stm_setup_prebuilt(static2) + lib.stm_set_prebuilt_identityhash(lp1, 42) + self.start_transaction() + h1 = lib.stm_identityhash(lp1) + h2 = lib.stm_identityhash(lp2) + assert h1 == 42 + assert h2 != 0 + h1 = lib.stm_id(lp1) + h2 = lib.stm_id(lp2) + assert h1 == int(ffi.cast("long", lp1)) + assert h2 == int(ffi.cast("long", lp2)) + + def test_hash_nursery(self): + self.start_transaction() + lp1 = stm_allocate(16) + lp2 = stm_allocate(16) + lp3 = stm_allocate(16) + lp4 = stm_allocate(16) + h1 = lib.stm_identityhash(lp1) + h2 = lib.stm_identityhash(lp2) + h3 = lib.stm_identityhash(lp3) + h4 = lib.stm_identityhash(lp4) + assert len(set([h1, h2, h3, h4])) == 4 # guaranteed by the algo + + def test_hash_lower_bits(self): + self.start_transaction() + lp1 = stm_allocate(32) + lp2 = stm_allocate(32) + lp3 = stm_allocate(32) + lp4 = stm_allocate(32) + h1 = lib.stm_identityhash(lp1) + h2 = lib.stm_identityhash(lp2) + h3 = lib.stm_identityhash(lp3) + h4 = lib.stm_identityhash(lp4) + assert len(set([h1 & 15, h2 & 15, h3 & 15, h4 & 15])) == 4 + + def test_hash_around_minor_collect(self): + self.start_transaction() + lp = stm_allocate(16) + h1 = lib.stm_identityhash(lp) + self.push_root(lp) + stm_minor_collect() + lp = self.pop_root() + h2 = lib.stm_identityhash(lp) + assert h2 == h1 + assert h2 != lib.stm_id(lp) From noreply at buildbot.pypy.org Tue Sep 9 11:33:14 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Tue, 9 Sep 2014 11:33:14 +0200 (CEST) Subject: [pypy-commit] stmgc default: add prebuilt-support Message-ID: <20140909093314.DA6441C02BF@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1375:9817d13b7e38 Date: 2014-09-09 11:16 +0200 http://bitbucket.org/pypy/stmgc/changeset/9817d13b7e38/ Log: add prebuilt-support diff --git a/c8/stm/prebuilt.c b/c8/stm/prebuilt.c new file mode 100644 --- /dev/null +++ b/c8/stm/prebuilt.c @@ -0,0 +1,73 @@ +#ifndef _STM_CORE_H_ +# error "must be compiled via stmgc.c" +#endif + + +static struct list_s *prebuilt_objects_to_trace; +static struct tree_s *tree_prebuilt_objs; /* XXX from gcpage.c */ + + +static void prebuilt_trace(object_t **pstaticobj_invalid) +{ + uintptr_t objaddr = (uintptr_t)*pstaticobj_invalid; + struct object_s *obj = (struct object_s *)objaddr; + + if (obj == NULL) + return; + + /* If the object was already moved, it is stored in 'tree_prebuilt_objs'. + For now we use this dictionary, with keys being equal to the numeric + address of the prebuilt object. + */ + wlog_t *item; + TREE_FIND(tree_prebuilt_objs, (uintptr_t)obj, item, goto not_found); + + *pstaticobj_invalid = (object_t *)item->val; /* already moved */ + return; + + not_found:; + /* We need to make a copy of this object. The extra "long" is for + the prebuilt hash. */ + size_t size = stmcb_size_rounded_up(obj); + object_t *nobj = _stm_allocate_old(size + sizeof(long)); + + /* Copy the object */ + char *realnobj = REAL_ADDRESS(stm_object_pages, nobj); + memcpy(realnobj, (char *)objaddr, size); + + /* Fix the flags in the copied object, asserting that it was zero so far */ + assert(nobj->stm_flags == 0); + nobj->stm_flags = GCFLAG_WRITE_BARRIER; + + /* Add the object to the tree */ + tree_insert(tree_prebuilt_objs, (uintptr_t)obj, (uintptr_t)nobj); + + /* Done */ + *pstaticobj_invalid = nobj; + LIST_APPEND(prebuilt_objects_to_trace, realnobj); +} + +object_t *stm_setup_prebuilt(object_t *staticobj_invalid) +{ + /* All variable names in "_invalid" here mean that although the + type is really "object_t *", it should not actually be accessed + via %gs. + */ + if (tree_prebuilt_objs == NULL) + tree_prebuilt_objs = tree_create(); + + LIST_CREATE(prebuilt_objects_to_trace); + + object_t *obj = staticobj_invalid; + prebuilt_trace(&obj); + + while (!list_is_empty(prebuilt_objects_to_trace)) { + struct object_s *realobj1 = + (struct object_s *)list_pop_item(prebuilt_objects_to_trace); + stmcb_trace(realobj1, &prebuilt_trace); + } + + LIST_FREE(prebuilt_objects_to_trace); + + return obj; +} diff --git a/c8/stmgc.c b/c8/stmgc.c --- a/c8/stmgc.c +++ b/c8/stmgc.c @@ -23,4 +23,5 @@ #include "stm/fprintcolor.c" #include "stm/rewind_setjmp.c" #include "stm/hash_id.c" +#include "stm/prebuilt.c" #include "stm/misc.c" diff --git a/c8/stmgc.h b/c8/stmgc.h --- a/c8/stmgc.h +++ b/c8/stmgc.h @@ -187,6 +187,12 @@ void stm_collect(long level); +long stm_identityhash(object_t *obj); +long stm_id(object_t *obj); +void stm_set_prebuilt_identityhash(object_t *obj, long hash); + +object_t *stm_setup_prebuilt(object_t *); + #ifdef STM_NO_AUTOMATIC_SETJMP int stm_is_inevitable(void); diff --git a/c8/test/support.py b/c8/test/support.py --- a/c8/test/support.py +++ b/c8/test/support.py @@ -42,6 +42,9 @@ void stm_validate(void *free_if_abort); bool _check_stm_validate(); +object_t *stm_setup_prebuilt(object_t *); + + bool _checked_stm_write(object_t *obj); bool _stm_was_read(object_t *obj); bool _stm_was_written(object_t *obj); diff --git a/c8/test/test_prebuilt.py b/c8/test/test_prebuilt.py new file mode 100644 --- /dev/null +++ b/c8/test/test_prebuilt.py @@ -0,0 +1,99 @@ +from support import * +import py +import weakref + + +prebuilt_dict = weakref.WeakKeyDictionary() + +def _prebuilt(size, tid): + assert size >= 16 + assert (size & 7) == 0 + myobj1 = ffi.new("char[]", size) + myobj = ffi.cast("object_t *", myobj1) + prebuilt_dict[myobj] = myobj1 + ffi.cast("uint32_t *", myobj)[1] = tid + return myobj + +def prebuilt(size): + return _prebuilt(size, 42 + size) + +def prebuilt_refs(n): + return _prebuilt(HDR + n * WORD, 421420 + n) + + +class TestPrebuilt(BaseTest): + + def test_simple_prebuilt(self): + static1 = prebuilt(16) + ffi.cast("char *", static1)[8:11] = 'ABC' + print static1 + lp = lib.stm_setup_prebuilt(static1) + # + self.start_transaction() + assert stm_get_char(lp, 8) == 'A' + assert stm_get_char(lp, 9) == 'B' + assert stm_get_char(lp, 10) == 'C' + + def test_prebuilt_rec(self): + static1 = prebuilt_refs(2) + static2 = prebuilt(16) + ffi.cast("char *", static2)[8:11] = 'ABC' + ffi.cast("object_t **", static1)[1] = static2 + lp1 = lib.stm_setup_prebuilt(static1) + # + self.start_transaction() + assert not stm_get_ref(lp1, 1) + lp2 = stm_get_ref(lp1, 0) + print lp2 + assert stm_get_char(lp2, 8) == 'A' + assert stm_get_char(lp2, 9) == 'B' + assert stm_get_char(lp2, 10) == 'C' + + def test_prebuilt_rec_cycle(self): + static1 = prebuilt_refs(1) + static2 = prebuilt_refs(1) + ffi.cast("object_t **", static1)[1] = static2 + ffi.cast("object_t **", static2)[1] = static1 + lp1 = lib.stm_setup_prebuilt(static1) + # + self.start_transaction() + lp2 = stm_get_ref(lp1, 0) + print lp2 + assert lp2 != lp1 + assert stm_get_ref(lp2, 0) == lp1 + assert lib._stm_get_flags(lp1) == lib._STM_GCFLAG_WRITE_BARRIER + assert lib._stm_get_flags(lp2) == lib._STM_GCFLAG_WRITE_BARRIER + + def test_multiple_calls_to_stm_setup_prebuilt_1(self, reverse=False): + static1 = prebuilt_refs(1) + static2 = prebuilt_refs(1) + ffi.cast("object_t **", static1)[1] = static2 + if not reverse: + lp1 = lib.stm_setup_prebuilt(static1) + lp2 = lib.stm_setup_prebuilt(static2) + else: + lp2 = lib.stm_setup_prebuilt(static2) + lp1 = lib.stm_setup_prebuilt(static1) + # + self.start_transaction() + assert stm_get_ref(lp1, 0) == lp2 + assert stm_get_ref(lp2, 0) == ffi.NULL + assert lib._stm_get_flags(lp1) == lib._STM_GCFLAG_WRITE_BARRIER + assert lib._stm_get_flags(lp2) == lib._STM_GCFLAG_WRITE_BARRIER + + def test_multiple_calls_to_stm_setup_prebuilt_2(self): + self.test_multiple_calls_to_stm_setup_prebuilt_1(reverse=True) + + def test_prebuilt_align_4_byte(self): + static0 = prebuilt(16) + p0 = ffi.cast("char *", static0) + for i in reversed(range(12)): + p0[i + 4] = p0[i] + static1 = ffi.cast("object_t *", p0 + 4) + ffi.cast("char *", static1)[8:11] = 'ABC' + lp = lib.stm_setup_prebuilt(static1) + # + self.start_transaction() + assert stm_get_char(lp, 8) == 'A' + assert stm_get_char(lp, 9) == 'B' + assert stm_get_char(lp, 10) == 'C' From noreply at buildbot.pypy.org Tue Sep 9 11:33:15 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Tue, 9 Sep 2014 11:33:15 +0200 (CEST) Subject: [pypy-commit] stmgc default: add test_rewind Message-ID: <20140909093315.E36771C02BF@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1376:ac63f9383783 Date: 2014-09-09 11:19 +0200 http://bitbucket.org/pypy/stmgc/changeset/ac63f9383783/ Log: add test_rewind diff --git a/c8/test/test_rewind.c b/c8/test/test_rewind.c new file mode 100644 --- /dev/null +++ b/c8/test/test_rewind.c @@ -0,0 +1,365 @@ +#include +#include +#include +#include +#include "rewind_setjmp.h" + + +rewind_jmp_thread gthread; +int gevents[1000]; +int num_gevents = 0; + +void gevent(int num) +{ + assert(num_gevents <= sizeof(gevents) / sizeof(int)); + gevents[num_gevents++] = num; +} + +void check_gevents(int expected[], int expected_size) +{ + int i; + int expected_count = expected_size / sizeof(int); + for (i = 0; i < expected_count && i < num_gevents; i++) { + assert(gevents[i] == expected[i]); + } + assert(num_gevents == expected_count); +} + +#define CHECK(expected) check_gevents(expected, sizeof(expected)) + +/************************************************************/ + +__attribute__((noinline)) +void f1(int x) +{ + gevent(1); + if (x < 10) { + rewind_jmp_longjmp(>hread); + } +} + +static int test1_x; + +void test1(void) +{ + rewind_jmp_buf buf; + rewind_jmp_enterframe(>hread, &buf, NULL); + + test1_x = 0; + rewind_jmp_setjmp(>hread, NULL); + + test1_x++; + f1(test1_x); + + assert(test1_x == 10); + int expected[] = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; + CHECK(expected); + + assert(rewind_jmp_armed(>hread)); + rewind_jmp_forget(>hread); + assert(!rewind_jmp_armed(>hread)); + + rewind_jmp_leaveframe(>hread, &buf, NULL); +} + +/************************************************************/ + +static int test2_x; + +__attribute__((noinline)) +int f2(void) +{ + rewind_jmp_buf buf; + rewind_jmp_enterframe(>hread, &buf, NULL); + test2_x = 0; + rewind_jmp_setjmp(>hread, NULL); + rewind_jmp_leaveframe(>hread, &buf, NULL); + return ++test2_x; +} + +void test2(void) +{ + rewind_jmp_buf buf; + rewind_jmp_enterframe(>hread, &buf, NULL); + int x = f2(); + gevent(x); + if (x < 10) + rewind_jmp_longjmp(>hread); + rewind_jmp_leaveframe(>hread, &buf, NULL); + int expected[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + CHECK(expected); +} + +/************************************************************/ + +__attribute__((noinline)) +int f3(int rec) +{ + if (rec > 0) + return f3(rec - 1); + else + return f2(); +} + +void test3(void) +{ + rewind_jmp_buf buf; + rewind_jmp_enterframe(>hread, &buf, NULL); + int x = f3(50); + gevent(x); + if (x < 10) + rewind_jmp_longjmp(>hread); + rewind_jmp_leaveframe(>hread, &buf, NULL); + int expected[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + CHECK(expected); +} + +/************************************************************/ + +__attribute__((noinline)) +int f4(int rec) +{ + rewind_jmp_buf buf; + rewind_jmp_enterframe(>hread, &buf, NULL); + int res; + if (rec > 0) + res = f4(rec - 1); + else + res = f2(); + rewind_jmp_leaveframe(>hread, &buf, NULL); + return res; +} + +void test4(void) +{ + rewind_jmp_buf buf; + rewind_jmp_enterframe(>hread, &buf, NULL); + int x = f4(5); + gevent(x); + if (x < 10) + rewind_jmp_longjmp(>hread); + rewind_jmp_leaveframe(>hread, &buf, NULL); + int expected[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + CHECK(expected); +} + +/************************************************************/ + +void test5(void) +{ + struct { int a; rewind_jmp_buf buf; int b; } sbuf; + rewind_jmp_enterframe(>hread, &sbuf.buf, NULL); + sbuf.a = 42; + sbuf.b = -42; + test2_x = 0; + rewind_jmp_setjmp(>hread, NULL); + sbuf.a++; + sbuf.b--; + gevent(sbuf.a); + gevent(sbuf.b); + if (test2_x == 0) { + test2_x++; + rewind_jmp_longjmp(>hread); + } + int expected[] = {43, -43, 43, -43}; + CHECK(expected); + rewind_jmp_leaveframe(>hread, &sbuf.buf, NULL); +} + +/************************************************************/ + +static int test6_x; + +__attribute__((noinline)) +void foo(int *x) { ++*x; } + +__attribute__((noinline)) +void f6(int a1, int a2, int a3, int a4, int a5, int a6, int a7, + int a8, int a9, int a10, int a11, int a12, int a13) +{ + rewind_jmp_buf buf; + rewind_jmp_enterframe(>hread, &buf, NULL); + + rewind_jmp_setjmp(>hread, NULL); + gevent(a1); gevent(a2); gevent(a3); gevent(a4); + gevent(a5); gevent(a6); gevent(a7); gevent(a8); + gevent(a9); gevent(a10); gevent(a11); gevent(a12); + gevent(a13); + if (++test6_x < 4) { + foo(&a1); + foo(&a2); + foo(&a3); + foo(&a4); + foo(&a5); + foo(&a6); + foo(&a7); + foo(&a8); + foo(&a9); + foo(&a10); + foo(&a11); + foo(&a12); + foo(&a13); + rewind_jmp_longjmp(>hread); + } + rewind_jmp_leaveframe(>hread, &buf, NULL); +} + +void test6(void) +{ + rewind_jmp_buf buf; + rewind_jmp_enterframe(>hread, &buf, NULL); + test6_x = 0; + f6(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13); + rewind_jmp_leaveframe(>hread, &buf, NULL); + int expected[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13}; + CHECK(expected); +} + +/************************************************************/ + +static void *ssarray[99]; + +void testTL1(void) +{ + void *a4, *a5; + rewind_jmp_buf buf; + rewind_jmp_enterframe(>hread, &buf, ssarray+5); + + a4 = (void *)444444; + a5 = (void *)555555; + ssarray[4] = a4; + ssarray[5] = a5; + + if (rewind_jmp_setjmp(>hread, ssarray+6) == 0) { + /* first path */ + assert(ssarray[4] == a4); + assert(ssarray[5] == a5); + ssarray[4] = NULL; + ssarray[5] = NULL; + rewind_jmp_restore_shadowstack(>hread); + rewind_jmp_longjmp(>hread); + } + /* second path */ + assert(ssarray[4] == NULL); /* was not saved */ + assert(ssarray[5] == a5); /* saved and restored */ + + rewind_jmp_leaveframe(>hread, &buf, ssarray+5); +} + +__attribute__((noinline)) +int gtl2(void) +{ + rewind_jmp_buf buf; + rewind_jmp_enterframe(>hread, &buf, ssarray+5); + ssarray[5] = (void *)555555; + + int result = rewind_jmp_setjmp(>hread, ssarray+6); + + assert(ssarray[4] == (void *)444444); + assert(ssarray[5] == (void *)555555); + ssarray[5] = NULL; + + rewind_jmp_leaveframe(>hread, &buf, ssarray+5); + return result; +} + +void testTL2(void) +{ + rewind_jmp_buf buf; + rewind_jmp_enterframe(>hread, &buf, ssarray+4); + + ssarray[4] = (void *)444444; + int result = gtl2(); + ssarray[4] = NULL; + + if (result == 0) { + rewind_jmp_restore_shadowstack(>hread); + rewind_jmp_longjmp(>hread); + } + + rewind_jmp_leaveframe(>hread, &buf, ssarray+4); +} + +/************************************************************/ + +__attribute__((noinline)) +int _7start_transaction() +{ + int result = rewind_jmp_setjmp(>hread, NULL); + return result; +} + +__attribute__((noinline)) +int _7enter_callback(rewind_jmp_buf *buf) +{ + rewind_jmp_enterprepframe(>hread, buf, NULL); + return _7start_transaction(); +} + +__attribute__((noinline)) +int _7bootstrap() +{ + rewind_jmp_longjmp(>hread); + return 0; +} + +__attribute__((noinline)) +int _7leave_callback(rewind_jmp_buf *buf) +{ + rewind_jmp_leaveframe(>hread, buf, NULL); + return 0; +} + +void test7(void) +{ + rewind_jmp_buf buf; + register long bla = 3; + rewind_jmp_prepareframe(&buf); + if (_7enter_callback(&buf) == 0) { + _7bootstrap(); + } + _7leave_callback(&buf); + assert(bla == 3); +} + +/************************************************************/ + +int rj_malloc_count = 0; + +void *rj_malloc(size_t size) +{ + rj_malloc_count++; + void *ptr = malloc(size); + fprintf(stderr, "malloc(%ld) -> %p\n", (long)size, ptr); + return ptr; +} + +void rj_free(void *ptr) +{ + if (ptr) + rj_malloc_count--; + fprintf(stderr, "free(%p)\n", ptr); + free(ptr); +} + + +int main(int argc, char *argv[]) +{ + assert(argc > 1); + if (!strcmp(argv[1], "1")) test1(); + else if (!strcmp(argv[1], "2")) test2(); + else if (!strcmp(argv[1], "3")) test3(); + else if (!strcmp(argv[1], "4")) test4(); + else if (!strcmp(argv[1], "5")) test5(); + else if (!strcmp(argv[1], "6")) test6(); + else if (!strcmp(argv[1], "7")) test7(); + else if (!strcmp(argv[1], "TL1")) testTL1(); + else if (!strcmp(argv[1], "TL2")) testTL2(); + else + assert(!"bad argv[1]"); + assert(rj_malloc_count == 0); + return 0; +} diff --git a/c8/test/test_rewind.py b/c8/test/test_rewind.py new file mode 100644 --- /dev/null +++ b/c8/test/test_rewind.py @@ -0,0 +1,19 @@ +import os + +def run_test(opt): + err = os.system("clang -g -O%s -Werror -DRJBUF_CUSTOM_MALLOC -I../stm" + " -o test_rewind_O%s test_rewind.c ../stm/rewind_setjmp.c" + % (opt, opt)) + if err != 0: + raise OSError("clang failed on test_rewind.c") + for testnum in [1, 2, 3, 4, 5, 6, 7, "TL1", "TL2"]: + print '=== O%s: RUNNING TEST %s ===' % (opt, testnum) + err = os.system("./test_rewind_O%s %s" % (opt, testnum)) + if err != 0: + raise OSError("'test_rewind_O%s %s' failed" % (opt, testnum)) + os.unlink("./test_rewind_O%s" % (opt,)) + +def test_O0(): run_test(0) +def test_O1(): run_test(1) +def test_O2(): run_test(2) +def test_O3(): run_test(3) From noreply at buildbot.pypy.org Tue Sep 9 11:33:17 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Tue, 9 Sep 2014 11:33:17 +0200 (CEST) Subject: [pypy-commit] stmgc default: add test_list.py Message-ID: <20140909093317.0A56A1C02BF@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1377:c823a30acbc0 Date: 2014-09-09 11:21 +0200 http://bitbucket.org/pypy/stmgc/changeset/c823a30acbc0/ Log: add test_list.py diff --git a/c8/stm/list.h b/c8/stm/list.h --- a/c8/stm/list.h +++ b/c8/stm/list.h @@ -218,7 +218,7 @@ static void tree_insert(struct tree_s *tree, uintptr_t addr, uintptr_t val); static bool tree_delete_item(struct tree_s *tree, uintptr_t addr) __attribute__((unused)); -static wlog_t *tree_item(struct tree_s *tree, int index); /* SLOW */ +static wlog_t *tree_item(struct tree_s *tree, int index) __attribute__((unused)); /* SLOW */ static inline bool tree_contains(struct tree_s *tree, uintptr_t addr) { diff --git a/c8/test/test_list.py b/c8/test/test_list.py new file mode 100644 --- /dev/null +++ b/c8/test/test_list.py @@ -0,0 +1,129 @@ +import random +import cffi +from common import parent_dir + + +ffi = cffi.FFI() +ffi.cdef(""" +struct list_s *list_create(void); + +struct tree_s *tree_create(void); +void tree_free(struct tree_s *tree); +void tree_clear(struct tree_s *tree); +bool tree_is_cleared(struct tree_s *tree); +bool tree_contains(struct tree_s *tree, uintptr_t addr); +void tree_insert(struct tree_s *tree, uintptr_t addr, uintptr_t val); +bool tree_delete_item(struct tree_s *tree, uintptr_t addr); +int test_tree_walk(struct tree_s *tree, uintptr_t addrs[]); +""") + +lib = ffi.verify(''' +#include +#include +#include + +#define LIKELY(x) (x) +#define UNLIKELY(x) (x) +#define stm_fatalerror(x) abort() + +#include "stm/list.h" + +#define _STM_CORE_H_ +#include "stm/list.c" + +int test_tree_walk(struct tree_s *tree, uintptr_t addrs[]) +{ + int result = 0; + wlog_t *item; + TREE_LOOP_FORWARD(tree, item) { + addrs[result++] = item->addr; + } TREE_LOOP_END; + int i = result; + TREE_LOOP_BACKWARD(tree, item) { + assert(i > 0); + i--; + assert(addrs[i] == item->addr); + } TREE_LOOP_END; + assert(i == 0); + return result; +} +''', define_macros=[('STM_TESTS', '1')], + undef_macros=['NDEBUG'], + include_dirs=[parent_dir], + extra_compile_args=['-g', '-O0', '-Werror', '-ferror-limit=1'], + force_generic_engine=True) + +# ____________________________________________________________ + +# XXX need tests for list_xxx too + +def test_tree_empty(): + t = lib.tree_create() + for i in range(100): + assert lib.tree_contains(t, i) == False + lib.tree_free(t) + +def test_tree_add(): + t = lib.tree_create() + lib.tree_insert(t, 23, 456) + for i in range(0, 100): + assert lib.tree_contains(t, i) == (i == 23) + lib.tree_free(t) + +def test_tree_is_cleared(): + t = lib.tree_create() + assert lib.tree_is_cleared(t) + lib.tree_insert(t, 23, 456) + assert not lib.tree_is_cleared(t) + lib.tree_free(t) + +def test_tree_delete_item(): + t = lib.tree_create() + lib.tree_insert(t, 23, 456) + lib.tree_insert(t, 42, 34289) + assert not lib.tree_is_cleared(t) + assert lib.tree_contains(t, 23) + res = lib.tree_delete_item(t, 23) + assert res + assert not lib.tree_contains(t, 23) + res = lib.tree_delete_item(t, 23) + assert not res + res = lib.tree_delete_item(t, 21) + assert not res + assert not lib.tree_is_cleared(t) + assert lib.tree_contains(t, 42) + res = lib.tree_delete_item(t, 42) + assert res + assert not lib.tree_is_cleared(t) # not cleared, but still empty + for i in range(100): + assert not lib.tree_contains(t, i) + lib.tree_free(t) + +def test_tree_walk(): + t = lib.tree_create() + lib.tree_insert(t, 23, 456) + lib.tree_insert(t, 42, 34289) + a = ffi.new("uintptr_t[10]") + res = lib.test_tree_walk(t, a) + assert res == 2 + assert ((a[0] == 23 and a[1] == 42) or + (a[0] == 42 and a[1] == 23)) + lib.tree_free(t) + +def test_tree_walk_big(): + t = lib.tree_create() + values = random.sample(xrange(1, 100000), 300) + for x in values: + lib.tree_insert(t, x, x) + a = ffi.new("uintptr_t[1000]") + res = lib.test_tree_walk(t, a) + assert res == 300 + found = set() + for i in range(res): + found.add(a[i]) + assert found == set(values) + lib.tree_free(t) + +def test_hash_permutation(): + hashes = [((n ^ (n << 4)) & 0xFF0) for n in range(256)] + assert set(hashes) == set(range(0, 4096, 16)) From noreply at buildbot.pypy.org Tue Sep 9 11:33:18 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Tue, 9 Sep 2014 11:33:18 +0200 (CEST) Subject: [pypy-commit] stmgc default: add abort/commit callbacks Message-ID: <20140909093318.2D7FF1C02BF@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1378:0e5c6ce3c7bd Date: 2014-09-09 11:31 +0200 http://bitbucket.org/pypy/stmgc/changeset/0e5c6ce3c7bd/ Log: add abort/commit callbacks diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -327,6 +327,8 @@ assert(list_is_empty(STM_PSEGMENT->objects_pointing_to_nursery)); assert(tree_is_cleared(STM_PSEGMENT->young_outside_nursery)); assert(tree_is_cleared(STM_PSEGMENT->nursery_objects_shadows)); + assert(tree_is_cleared(STM_PSEGMENT->callbacks_on_commit_and_abort[0])); + assert(tree_is_cleared(STM_PSEGMENT->callbacks_on_commit_and_abort[1])); check_nursery_at_transaction_start(); @@ -403,6 +405,7 @@ release_modified_objs_lock(STM_SEGMENT->segment_num); + invoke_and_clear_user_callbacks(0); /* for commit */ s_mutex_lock(); @@ -508,6 +511,11 @@ stm_thread_local_t *tl = STM_SEGMENT->running_thread; + if (tl->mem_clear_on_abort) + memset(tl->mem_clear_on_abort, 0, tl->mem_bytes_to_clear_on_abort); + + invoke_and_clear_user_callbacks(1); /* for abort */ + _finish_transaction(); /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */ @@ -544,6 +552,7 @@ _validate_and_turn_inevitable(); STM_PSEGMENT->transaction_state = TS_INEVITABLE; stm_rewind_jmp_forget(STM_SEGMENT->running_thread); + invoke_and_clear_user_callbacks(0); /* for commit */ } else { assert(STM_PSEGMENT->transaction_state == TS_INEVITABLE); diff --git a/c8/stm/core.h b/c8/stm/core.h --- a/c8/stm/core.h +++ b/c8/stm/core.h @@ -59,6 +59,8 @@ uint8_t transaction_state; + struct tree_s *callbacks_on_commit_and_abort[2]; + struct stm_commit_log_entry_s *last_commit_log_entry; struct stm_shadowentry_s *shadowstack_at_start_of_transaction; diff --git a/c8/stm/extra.c b/c8/stm/extra.c new file mode 100644 --- /dev/null +++ b/c8/stm/extra.c @@ -0,0 +1,82 @@ +#ifndef _STM_CORE_H_ +# error "must be compiled via stmgc.c" +#endif + + +static long register_callbacks(stm_thread_local_t *tl, + void *key, void callback(void *), long index) +{ + if (!_stm_in_transaction(tl)) { + /* check that the current thread-local is really running a + transaction, and do nothing otherwise. */ + return -1; + } + + if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) { + /* ignore callbacks if we're in an inevitable transaction + (which cannot abort) */ + return -1; + } + + struct tree_s *callbacks; + callbacks = STM_PSEGMENT->callbacks_on_commit_and_abort[index]; + + if (callback == NULL) { + /* double-unregistering works, but return 0 */ + return tree_delete_item(callbacks, (uintptr_t)key); + } + else { + /* double-registering the same key will crash */ + tree_insert(callbacks, (uintptr_t)key, (uintptr_t)callback); + return 1; + } +} + +long stm_call_on_commit(stm_thread_local_t *tl, + void *key, void callback(void *)) +{ + long result = register_callbacks(tl, key, callback, 0); + if (result < 0 && callback != NULL) { + /* no regular transaction running, invoke the callback + immediately */ + callback(key); + } + return result; +} + +long stm_call_on_abort(stm_thread_local_t *tl, + void *key, void callback(void *)) +{ + return register_callbacks(tl, key, callback, 1); +} + +static void invoke_and_clear_user_callbacks(long index) +{ + struct tree_s *callbacks; + + /* clear the callbacks that we don't want to invoke at all */ + callbacks = STM_PSEGMENT->callbacks_on_commit_and_abort[1 - index]; + if (!tree_is_cleared(callbacks)) + tree_clear(callbacks); + + /* invoke the callbacks from the other group */ + callbacks = STM_PSEGMENT->callbacks_on_commit_and_abort[index]; + if (tree_is_cleared(callbacks)) + return; + STM_PSEGMENT->callbacks_on_commit_and_abort[index] = tree_create(); + + wlog_t *item; + TREE_LOOP_FORWARD(callbacks, item) { + void *key = (void *)item->addr; + void (*callback)(void *) = (void(*)(void *))item->val; + assert(key != NULL); + assert(callback != NULL); + + /* The callback may call stm_call_on_abort(key, NULL). It is ignored, + because 'callbacks_on_commit_and_abort' was cleared already. */ + callback(key); + + } TREE_LOOP_END; + + tree_free(callbacks); +} diff --git a/c8/stm/extra.h b/c8/stm/extra.h new file mode 100644 --- /dev/null +++ b/c8/stm/extra.h @@ -0,0 +1,3 @@ + +static void invoke_and_clear_user_callbacks(long index); +/* 0 = for commit, 1 = for abort */ diff --git a/c8/stm/setup.c b/c8/stm/setup.c --- a/c8/stm/setup.c +++ b/c8/stm/setup.c @@ -112,6 +112,9 @@ pr->objects_pointing_to_nursery = list_create(); pr->young_outside_nursery = tree_create(); pr->nursery_objects_shadows = tree_create(); + pr->callbacks_on_commit_and_abort[0] = tree_create(); + pr->callbacks_on_commit_and_abort[1] = tree_create(); + pr->last_commit_log_entry = &commit_log_root; pr->pub.transaction_read_version = 0xff; } @@ -145,6 +148,8 @@ tree_free(pr->modified_old_objects); tree_free(pr->young_outside_nursery); tree_free(pr->nursery_objects_shadows); + tree_free(pr->callbacks_on_commit_and_abort[0]); + tree_free(pr->callbacks_on_commit_and_abort[1]); } munmap(stm_object_pages, TOTAL_MEMORY); diff --git a/c8/stmgc.c b/c8/stmgc.c --- a/c8/stmgc.c +++ b/c8/stmgc.c @@ -11,6 +11,7 @@ #include "stm/setup.h" #include "stm/fprintcolor.h" #include "stm/rewind_setjmp.h" +#include "stm/extra.h" #include "stm/list.c" #include "stm/pagecopy.c" @@ -25,3 +26,4 @@ #include "stm/hash_id.c" #include "stm/prebuilt.c" #include "stm/misc.c" +#include "stm/extra.c" diff --git a/c8/stmgc.h b/c8/stmgc.h --- a/c8/stmgc.h +++ b/c8/stmgc.h @@ -47,6 +47,9 @@ /* rewind_setjmp's interface */ rewind_jmp_thread rjthread; struct stm_shadowentry_s *shadowstack, *shadowstack_base; + + char *mem_clear_on_abort; + size_t mem_bytes_to_clear_on_abort; long last_abort__bytes_in_nursery; /* the next fields are handled internally by the library */ int associated_segment_num; @@ -194,6 +197,11 @@ object_t *stm_setup_prebuilt(object_t *); +long stm_call_on_abort(stm_thread_local_t *, void *key, void callback(void *)); +long stm_call_on_commit(stm_thread_local_t *, void *key, void callback(void *)); + + + #ifdef STM_NO_AUTOMATIC_SETJMP int stm_is_inevitable(void); #else diff --git a/c8/test/support.py b/c8/test/support.py --- a/c8/test/support.py +++ b/c8/test/support.py @@ -24,6 +24,8 @@ typedef struct { rewind_jmp_thread rjthread; struct stm_shadowentry_s *shadowstack, *shadowstack_base; + char *mem_clear_on_abort; + size_t mem_bytes_to_clear_on_abort; long last_abort__bytes_in_nursery; int associated_segment_num; struct stm_thread_local_s *prev, *next; @@ -79,6 +81,10 @@ void stm_set_prebuilt_identityhash(object_t *obj, uint64_t hash); +long stm_call_on_abort(stm_thread_local_t *, void *key, void callback(void *)); +long stm_call_on_commit(stm_thread_local_t *, void *key, void callback(void *)); + + long _stm_count_modified_old_objects(void); long _stm_count_objects_pointing_to_nursery(void); object_t *_stm_enum_modified_old_objects(long index); diff --git a/c8/test/test_extra.py b/c8/test/test_extra.py new file mode 100644 --- /dev/null +++ b/c8/test/test_extra.py @@ -0,0 +1,194 @@ +from support import * +import py + +def ffi_new_aligned(string): + ALIGN = ffi.sizeof("void *") + p1 = ffi.new("void *[]", (len(string) + ALIGN) // ALIGN) + p2 = ffi.gc(ffi.cast("char *", p1), lambda p2: p1) + p2[0:len(string)+1] = string + '\x00' + assert ffi.string(p2) == string + return p2 + + +class TestExtra(BaseTest): + + def test_clear_on_abort(self): + p = ffi.new("char[]", "hello") + tl = self.get_stm_thread_local() + tl.mem_clear_on_abort = p + tl.mem_bytes_to_clear_on_abort = 2 + # + self.start_transaction() + assert ffi.string(p) == "hello" + self.abort_transaction() + assert p[0] == '\0' + assert p[1] == '\0' + assert p[2] == 'l' + assert p[3] == 'l' + assert p[4] == 'o' + + def test_call_on_abort(self): + p0 = ffi_new_aligned("aaa") + p1 = ffi_new_aligned("hello") + p2 = ffi_new_aligned("removed") + p3 = ffi_new_aligned("world") + p4 = ffi_new_aligned("00") + # + @ffi.callback("void(void *)") + def clear_me(p): + p = ffi.cast("char *", p) + p[0] = chr(ord(p[0]) + 1) + # + self.start_transaction() + x = lib.stm_call_on_abort(self.get_stm_thread_local(), p0, clear_me) + assert x != 0 + # the registered callbacks are removed on + # successful commit + self.commit_transaction() + assert ffi.string(p0) == "aaa" + # + self.start_transaction() + x = lib.stm_call_on_abort(self.get_stm_thread_local(), p1, clear_me) + assert x != 0 + x = lib.stm_call_on_abort(self.get_stm_thread_local(), p2, clear_me) + assert x != 0 + x = lib.stm_call_on_abort(self.get_stm_thread_local(), p3, clear_me) + assert x != 0 + x = lib.stm_call_on_abort(self.get_stm_thread_local(), p2, ffi.NULL) + assert x != 0 + x = lib.stm_call_on_abort(self.get_stm_thread_local(), p2, ffi.NULL) + assert x == 0 + x = lib.stm_call_on_abort(self.get_stm_thread_local(), p4, ffi.NULL) + assert x == 0 + assert ffi.string(p0) == "aaa" + assert ffi.string(p1) == "hello" + assert ffi.string(p2) == "removed" + assert ffi.string(p3) == "world" + self.abort_transaction() + # + assert ffi.string(p0) == "aaa" + assert ffi.string(p1) == "iello" + assert ffi.string(p2) == "removed" + assert ffi.string(p3) == "xorld" + # + # the registered callbacks are removed on abort + self.start_transaction() + self.abort_transaction() + assert ffi.string(p0) == "aaa" + assert ffi.string(p1) == "iello" + assert ffi.string(p2) == "removed" + assert ffi.string(p3) == "xorld" + assert ffi.string(p4) == "00" + + def test_ignores_if_outside_transaction(self): + @ffi.callback("void(void *)") + def dont_see_me(p): + seen.append(p) + # + seen = [] + p0 = ffi_new_aligned("aaa") + x = lib.stm_call_on_abort(self.get_stm_thread_local(), p0, dont_see_me) + assert x != 0 + self.start_transaction() + self.abort_transaction() + assert seen == [] + + def test_call_on_commit(self): + p0 = ffi_new_aligned("aaa") + p1 = ffi_new_aligned("hello") + p2 = ffi_new_aligned("removed") + p3 = ffi_new_aligned("world") + p4 = ffi_new_aligned("00") + # + @ffi.callback("void(void *)") + def clear_me(p): + p = ffi.cast("char *", p) + p[0] = chr(ord(p[0]) + 1) + # + self.start_transaction() + x = lib.stm_call_on_commit(self.get_stm_thread_local(), p0, clear_me) + assert x != 0 + # the registered callbacks are not called on abort + self.abort_transaction() + assert ffi.string(p0) == "aaa" + # + self.start_transaction() + x = lib.stm_call_on_commit(self.get_stm_thread_local(), p1, clear_me) + assert x != 0 + x = lib.stm_call_on_commit(self.get_stm_thread_local(), p2, clear_me) + assert x != 0 + x = lib.stm_call_on_commit(self.get_stm_thread_local(), p3, clear_me) + assert x != 0 + x = lib.stm_call_on_commit(self.get_stm_thread_local(), p2, ffi.NULL) + assert x != 0 + x = lib.stm_call_on_commit(self.get_stm_thread_local(), p2, ffi.NULL) + assert x == 0 + x = lib.stm_call_on_commit(self.get_stm_thread_local(), p4, ffi.NULL) + assert x == 0 + assert ffi.string(p0) == "aaa" + assert ffi.string(p1) == "hello" + assert ffi.string(p2) == "removed" + assert ffi.string(p3) == "world" + self.commit_transaction() + # + assert ffi.string(p0) == "aaa" + assert ffi.string(p1) == "iello" + assert ffi.string(p2) == "removed" + assert ffi.string(p3) == "xorld" + assert ffi.string(p4) == "00" + + def test_call_on_commit_immediately_if_inevitable(self): + p0 = ffi_new_aligned("aaa") + self.start_transaction() + self.become_inevitable() + # + @ffi.callback("void(void *)") + def clear_me(p): + p = ffi.cast("char *", p) + p[0] = chr(ord(p[0]) + 1) + # + lib.stm_call_on_commit(self.get_stm_thread_local(), p0, clear_me) + assert ffi.string(p0) == "baa" + self.commit_transaction() + assert ffi.string(p0) == "baa" + + def test_call_on_commit_as_soon_as_inevitable(self): + p0 = ffi_new_aligned("aaa") + self.start_transaction() + # + @ffi.callback("void(void *)") + def clear_me(p): + p = ffi.cast("char *", p) + p[0] = chr(ord(p[0]) + 1) + # + lib.stm_call_on_commit(self.get_stm_thread_local(), p0, clear_me) + assert ffi.string(p0) == "aaa" + self.become_inevitable() + assert ffi.string(p0) == "baa" + self.commit_transaction() + assert ffi.string(p0) == "baa" + + def test_call_on_commit_immediately_if_outside_transaction(self): + p0 = ffi_new_aligned("aaa") + # + @ffi.callback("void(void *)") + def clear_me(p): + p = ffi.cast("char *", p) + p[0] = chr(ord(p[0]) + 1) + # + lib.stm_call_on_commit(self.get_stm_thread_local(), p0, clear_me) + assert ffi.string(p0) == "baa" + self.start_transaction() + assert ffi.string(p0) == "baa" + self.commit_transaction() + assert ffi.string(p0) == "baa" + + def test_stm_become_globally_unique_transaction(self): + self.start_transaction() + # + self.switch(1) + self.start_transaction() + self.become_globally_unique_transaction() + assert lib.stm_is_inevitable() + # + py.test.raises(Conflict, self.switch, 0) From noreply at buildbot.pypy.org Tue Sep 9 14:06:00 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Tue, 9 Sep 2014 14:06:00 +0200 (CEST) Subject: [pypy-commit] stmgc default: add safe-point mechanism again & globally_unique_transaction Message-ID: <20140909120600.CB61E1C06A7@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1379:06a773048b9a Date: 2014-09-09 14:07 +0200 http://bitbucket.org/pypy/stmgc/changeset/06a773048b9a/ Log: add safe-point mechanism again & globally_unique_transaction diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -74,6 +74,11 @@ most current one and apply all changes done by other transactions. Abort if we read one of the committed objs. */ + if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) { + assert((uintptr_t)STM_PSEGMENT->last_commit_log_entry->next == -1); + return; + } + volatile struct stm_commit_log_entry_s *cl, *prev_cl; cl = prev_cl = (volatile struct stm_commit_log_entry_s *) STM_PSEGMENT->last_commit_log_entry; @@ -83,6 +88,9 @@ while ((cl = cl->next)) { if ((uintptr_t)cl == -1) { /* there is an inevitable transaction running */ +#if STM_TESTS + stm_abort_transaction(); +#endif cl = prev_cl; usleep(1); /* XXX */ continue; @@ -306,13 +314,16 @@ goto retry; /* GS invalid before this point! */ + assert(STM_PSEGMENT->safe_point == SP_NO_TRANSACTION); assert(STM_PSEGMENT->transaction_state == TS_NONE); STM_PSEGMENT->transaction_state = TS_REGULAR; + STM_PSEGMENT->safe_point = SP_RUNNING; #ifndef NDEBUG STM_PSEGMENT->running_pthread = pthread_self(); #endif STM_PSEGMENT->shadowstack_at_start_of_transaction = tl->shadowstack; + enter_safe_point_if_requested(); dprintf(("start_transaction\n")); s_mutex_unlock(); @@ -372,6 +383,7 @@ { stm_thread_local_t *tl = STM_SEGMENT->running_thread; + STM_PSEGMENT->safe_point = SP_NO_TRANSACTION; STM_PSEGMENT->transaction_state = TS_NONE; list_clear(STM_PSEGMENT->objects_pointing_to_nursery); @@ -382,6 +394,7 @@ void stm_commit_transaction(void) { assert(!_has_mutex()); + assert(STM_PSEGMENT->safe_point == SP_RUNNING); assert(STM_PSEGMENT->running_pthread == pthread_self()); dprintf(("stm_commit_transaction()\n")); @@ -412,6 +425,10 @@ assert(STM_SEGMENT->nursery_end == NURSERY_END); stm_rewind_jmp_forget(STM_SEGMENT->running_thread); + if (globally_unique_transaction && STM_PSEGMENT->transaction_state == TS_INEVITABLE) { + committed_globally_unique_transaction(); + } + /* done */ _finish_transaction(); /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */ @@ -516,23 +533,25 @@ invoke_and_clear_user_callbacks(1); /* for abort */ + if (is_abort(STM_SEGMENT->nursery_end)) { + /* done aborting */ + STM_SEGMENT->nursery_end = pause_signalled ? NSE_SIGPAUSE + : NURSERY_END; + } + _finish_transaction(); /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */ return tl; } - -#ifdef STM_NO_AUTOMATIC_SETJMP -void _test_run_abort(stm_thread_local_t *tl) __attribute__((noreturn)); -#endif - -void stm_abort_transaction(void) +static void abort_with_mutex(void) { - s_mutex_lock(); stm_thread_local_t *tl = abort_with_mutex_no_longjmp(); s_mutex_unlock(); + usleep(1); + #ifdef STM_NO_AUTOMATIC_SETJMP _test_run_abort(tl); #else @@ -542,9 +561,22 @@ } + +#ifdef STM_NO_AUTOMATIC_SETJMP +void _test_run_abort(stm_thread_local_t *tl) __attribute__((noreturn)); +#endif + +void stm_abort_transaction(void) +{ + s_mutex_lock(); + abort_with_mutex(); +} + + void _stm_become_inevitable(const char *msg) { s_mutex_lock(); + enter_safe_point_if_requested(); if (STM_PSEGMENT->transaction_state == TS_REGULAR) { dprintf(("become_inevitable: %s\n", msg)); @@ -560,3 +592,13 @@ s_mutex_unlock(); } + +void stm_become_globally_unique_transaction(stm_thread_local_t *tl, + const char *msg) +{ + stm_become_inevitable(tl, msg); /* may still abort */ + + s_mutex_lock(); + synchronize_all_threads(STOP_OTHERS_AND_BECOME_GLOBALLY_UNIQUE); + s_mutex_unlock(); +} diff --git a/c8/stm/core.h b/c8/stm/core.h --- a/c8/stm/core.h +++ b/c8/stm/core.h @@ -57,6 +57,7 @@ uint8_t privatization_lock; + uint8_t safe_point; uint8_t transaction_state; struct tree_s *callbacks_on_commit_and_abort[2]; @@ -71,6 +72,16 @@ #endif }; +enum /* safe_point */ { + SP_NO_TRANSACTION=0, + SP_RUNNING, + SP_WAIT_FOR_C_REQUEST_REMOVED, + SP_WAIT_FOR_C_AT_SAFE_POINT, +#ifdef STM_TESTS + SP_WAIT_FOR_OTHER_THREAD, +#endif +}; + enum /* transaction_state */ { TS_NONE=0, TS_REGULAR, @@ -119,6 +130,12 @@ static bool _is_tl_registered(stm_thread_local_t *tl); static bool _seems_to_be_running_transaction(void); +static void teardown_core(void); +static void abort_with_mutex(void) __attribute__((noreturn)); +static stm_thread_local_t *abort_with_mutex_no_longjmp(void); +static void abort_data_structures_from_segment_num(int segment_num); + + static inline void _duck(void) { /* put a call to _duck() between two instructions that set 0 into diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c --- a/c8/stm/nursery.c +++ b/c8/stm/nursery.c @@ -253,6 +253,8 @@ { assert(!_has_mutex()); + stm_safe_point(); + _do_minor_collection(commit); } @@ -276,6 +278,7 @@ STM_SEGMENT->nursery_current -= size_rounded_up; /* restore correct val */ restart: + stm_safe_point(); OPT_ASSERT(size_rounded_up >= 16); OPT_ASSERT((size_rounded_up & 7) == 0); diff --git a/c8/stm/nursery.h b/c8/stm/nursery.h --- a/c8/stm/nursery.h +++ b/c8/stm/nursery.h @@ -1,5 +1,22 @@ + +#define NSE_SIGPAUSE _STM_NSE_SIGNAL_MAX + static void minor_collection(bool commit); static void check_nursery_at_transaction_start(void); static size_t throw_away_nursery(struct stm_priv_segment_info_s *pseg); static void assert_memset_zero(void *s, size_t n); + + +static inline bool is_abort(uintptr_t nursery_end) { + return (nursery_end <= _STM_NSE_SIGNAL_MAX && nursery_end != NSE_SIGPAUSE); +} + +static inline bool is_aborting_now(uint8_t other_segment_num) { + return (is_abort(get_segment(other_segment_num)->nursery_end) && + get_priv_segment(other_segment_num)->safe_point != SP_RUNNING); +} + + +#define must_abort() is_abort(STM_SEGMENT->nursery_end) +static object_t *find_shadow(object_t *obj); diff --git a/c8/stm/sync.c b/c8/stm/sync.c --- a/c8/stm/sync.c +++ b/c8/stm/sync.c @@ -11,6 +11,7 @@ static union { struct { pthread_mutex_t global_mutex; + pthread_cond_t cond[_C_TOTAL]; /* some additional pieces of global state follow */ uint8_t in_use1[NB_SEGMENTS]; /* 1 if running a pthread */ }; @@ -22,6 +23,12 @@ { if (pthread_mutex_init(&sync_ctl.global_mutex, NULL) != 0) stm_fatalerror("mutex initialization: %m"); + + long i; + for (i = 0; i < _C_TOTAL; i++) { + if (pthread_cond_init(&sync_ctl.cond[i], NULL) != 0) + stm_fatalerror("cond initialization: %m"); + } } static void teardown_sync(void) @@ -29,6 +36,12 @@ if (pthread_mutex_destroy(&sync_ctl.global_mutex) != 0) stm_fatalerror("mutex destroy: %m"); + long i; + for (i = 0; i < _C_TOTAL; i++) { + if (pthread_cond_destroy(&sync_ctl.cond[i]) != 0) + stm_fatalerror("cond destroy: %m"); + } + memset(&sync_ctl, 0, sizeof(sync_ctl)); } @@ -62,6 +75,31 @@ assert((_has_mutex_here = false, 1)); } + +static inline void cond_wait(enum cond_type_e ctype) +{ +#ifdef STM_NO_COND_WAIT + stm_fatalerror("*** cond_wait/%d called!", (int)ctype); +#endif + + assert(_has_mutex_here); + if (UNLIKELY(pthread_cond_wait(&sync_ctl.cond[ctype], + &sync_ctl.global_mutex) != 0)) + stm_fatalerror("pthread_cond_wait/%d: %m", (int)ctype); +} + +static inline void cond_signal(enum cond_type_e ctype) +{ + if (UNLIKELY(pthread_cond_signal(&sync_ctl.cond[ctype]) != 0)) + stm_fatalerror("pthread_cond_signal/%d: %m", (int)ctype); +} + +static inline void cond_broadcast(enum cond_type_e ctype) +{ + if (UNLIKELY(pthread_cond_broadcast(&sync_ctl.cond[ctype]) != 0)) + stm_fatalerror("pthread_cond_broadcast/%d: %m", (int)ctype); +} + /************************************************************/ @@ -147,3 +185,169 @@ { set_gs_register(get_segment_base(segnum)); } + +#if STM_TESTS +void _stm_start_safe_point(void) +{ + assert(STM_PSEGMENT->safe_point == SP_RUNNING); + STM_PSEGMENT->safe_point = SP_WAIT_FOR_OTHER_THREAD; +} + +void _stm_stop_safe_point(void) +{ + assert(STM_PSEGMENT->safe_point == SP_WAIT_FOR_OTHER_THREAD); + STM_PSEGMENT->safe_point = SP_RUNNING; + + stm_safe_point(); +} +#endif + + + +/************************************************************/ + + +#ifndef NDEBUG +static bool _safe_points_requested = false; +#endif + +static void signal_everybody_to_pause_running(void) +{ + assert(_safe_points_requested == false); + assert((_safe_points_requested = true, 1)); + assert(_has_mutex()); + + long i; + for (i = 1; i <= NB_SEGMENTS; i++) { + if (get_segment(i)->nursery_end == NURSERY_END) + get_segment(i)->nursery_end = NSE_SIGPAUSE; + } + assert(!pause_signalled); + pause_signalled = true; +} + +static inline long count_other_threads_sp_running(void) +{ + /* Return the number of other threads in SP_RUNNING. + Asserts that SP_RUNNING threads still have the NSE_SIGxxx. */ + long i; + long result = 0; + int my_num = STM_SEGMENT->segment_num; + + for (i = 1; i <= NB_SEGMENTS; i++) { + if (i != my_num && get_priv_segment(i)->safe_point == SP_RUNNING) { + assert(get_segment(i)->nursery_end <= _STM_NSE_SIGNAL_MAX); + result++; + } + } + return result; +} + +static void remove_requests_for_safe_point(void) +{ + assert(pause_signalled); + pause_signalled = false; + assert(_safe_points_requested == true); + assert((_safe_points_requested = false, 1)); + + long i; + for (i = 1; i <= NB_SEGMENTS; i++) { + assert(get_segment(i)->nursery_end != NURSERY_END); + if (get_segment(i)->nursery_end == NSE_SIGPAUSE) + get_segment(i)->nursery_end = NURSERY_END; + } + cond_broadcast(C_REQUEST_REMOVED); +} + +static void enter_safe_point_if_requested(void) +{ + if (STM_SEGMENT->nursery_end == NURSERY_END) + return; /* fast path: no safe point requested */ + + assert(_seems_to_be_running_transaction()); + assert(_has_mutex()); + while (1) { + if (must_abort()) + abort_with_mutex(); + + if (STM_SEGMENT->nursery_end == NURSERY_END) + break; /* no safe point requested */ + + assert(STM_SEGMENT->nursery_end == NSE_SIGPAUSE); + assert(pause_signalled); + + /* If we are requested to enter a safe-point, we cannot proceed now. + Wait until the safe-point request is removed for us. */ +#ifdef STM_TESTS + abort_with_mutex(); +#endif + cond_signal(C_AT_SAFE_POINT); + STM_PSEGMENT->safe_point = SP_WAIT_FOR_C_REQUEST_REMOVED; + cond_wait(C_REQUEST_REMOVED); + STM_PSEGMENT->safe_point = SP_RUNNING; + } +} + +static void synchronize_all_threads(enum sync_type_e sync_type) +{ + enter_safe_point_if_requested(); + + /* Only one thread should reach this point concurrently. This is + why: if several threads call this function, the first one that + goes past this point will set the "request safe point" on all + other threads; then none of the other threads will go past the + enter_safe_point_if_requested() above. + */ + if (UNLIKELY(globally_unique_transaction)) { + assert(count_other_threads_sp_running() == 0); + return; + } + + signal_everybody_to_pause_running(); + + /* If some other threads are SP_RUNNING, we cannot proceed now. + Wait until all other threads are suspended. */ + while (count_other_threads_sp_running() > 0) { + STM_PSEGMENT->safe_point = SP_WAIT_FOR_C_AT_SAFE_POINT; + cond_wait(C_AT_SAFE_POINT); + STM_PSEGMENT->safe_point = SP_RUNNING; + + if (must_abort()) { + remove_requests_for_safe_point(); /* => C_REQUEST_REMOVED */ + abort_with_mutex(); + } + } + + if (UNLIKELY(sync_type == STOP_OTHERS_AND_BECOME_GLOBALLY_UNIQUE)) { + globally_unique_transaction = true; + assert(STM_SEGMENT->nursery_end == NSE_SIGPAUSE); + STM_SEGMENT->nursery_end = NURSERY_END; + return; /* don't remove the requests for safe-points in this case */ + } + + /* Remove the requests for safe-points now. In principle we should + remove it later, when the caller is done, but this is equivalent + as long as we hold the mutex. + */ + remove_requests_for_safe_point(); /* => C_REQUEST_REMOVED */ +} + +static void committed_globally_unique_transaction(void) +{ + assert(globally_unique_transaction); + assert(STM_SEGMENT->nursery_end == NURSERY_END); + STM_SEGMENT->nursery_end = NSE_SIGPAUSE; + globally_unique_transaction = false; + remove_requests_for_safe_point(); +} + +void _stm_collectable_safe_point(void) +{ + /* If 'nursery_end' was set to NSE_SIGxxx by another thread, + we end up here as soon as we try to call stm_allocate() or do + a call to stm_safe_point(). + */ + s_mutex_lock(); + enter_safe_point_if_requested(); + s_mutex_unlock(); +} diff --git a/c8/stm/sync.h b/c8/stm/sync.h --- a/c8/stm/sync.h +++ b/c8/stm/sync.h @@ -1,9 +1,17 @@ static void setup_sync(void); static void teardown_sync(void); +enum cond_type_e { + C_AT_SAFE_POINT, + C_REQUEST_REMOVED, + _C_TOTAL +}; static void s_mutex_lock(void); static void s_mutex_unlock(void); +static void cond_wait(enum cond_type_e); +static void cond_signal(enum cond_type_e); +static void cond_broadcast(enum cond_type_e); #ifndef NDEBUG static bool _has_mutex(void); #endif @@ -14,3 +22,13 @@ (must have the mutex acquired!) */ static bool acquire_thread_segment(stm_thread_local_t *tl); static void release_thread_segment(stm_thread_local_t *tl); + +enum sync_type_e { + STOP_OTHERS_UNTIL_MUTEX_UNLOCK, + STOP_OTHERS_AND_BECOME_GLOBALLY_UNIQUE, +}; +static void synchronize_all_threads(enum sync_type_e sync_type); +static void committed_globally_unique_transaction(void); + +static bool pause_signalled, globally_unique_transaction; +static void enter_safe_point_if_requested(void); diff --git a/c8/stmgc.h b/c8/stmgc.h --- a/c8/stmgc.h +++ b/c8/stmgc.h @@ -59,11 +59,13 @@ #define _STM_GCFLAG_WRITE_BARRIER 0x01 #define _STM_FAST_ALLOC (66*1024) +#define _STM_NSE_SIGNAL_MAX 1 void _stm_write_slowpath(object_t *); object_t *_stm_allocate_slowpath(ssize_t); object_t *_stm_allocate_external(ssize_t); void _stm_become_inevitable(const char*); +void _stm_collectable_safe_point(); object_t *_stm_allocate_old(ssize_t size_rounded_up); char *_stm_real_address(object_t *o); @@ -78,6 +80,9 @@ void _stm_test_switch_segment(int segnum); void _push_obj_to_other_segments(object_t *obj); +void _stm_start_safe_point(void); +void _stm_stop_safe_point(void); + char *_stm_get_segment_base(long index); bool _stm_in_transaction(stm_thread_local_t *tl); void _stm_set_nursery_free_count(uint64_t free_count); @@ -200,6 +205,10 @@ long stm_call_on_abort(stm_thread_local_t *, void *key, void callback(void *)); long stm_call_on_commit(stm_thread_local_t *, void *key, void callback(void *)); +static inline void stm_safe_point(void) { + if (STM_SEGMENT->nursery_end <= _STM_NSE_SIGNAL_MAX) + _stm_collectable_safe_point(); +} #ifdef STM_NO_AUTOMATIC_SETJMP @@ -216,6 +225,7 @@ _stm_become_inevitable(msg); } +void stm_become_globally_unique_transaction(stm_thread_local_t *tl, const char *msg); /* ==================== END ==================== */ diff --git a/c8/test/support.py b/c8/test/support.py --- a/c8/test/support.py +++ b/c8/test/support.py @@ -45,6 +45,8 @@ bool _check_stm_validate(); object_t *stm_setup_prebuilt(object_t *); +void _stm_start_safe_point(void); +bool _check_stop_safe_point(void); bool _checked_stm_write(object_t *obj); @@ -65,6 +67,7 @@ bool _check_commit_transaction(void); bool _check_abort_transaction(void); bool _check_become_inevitable(stm_thread_local_t *tl); +bool _check_become_globally_unique_transaction(stm_thread_local_t *tl); int stm_is_inevitable(void); void _set_type_id(object_t *obj, uint32_t h); @@ -152,6 +155,10 @@ CHECKED(stm_commit_transaction()); } +bool _check_stop_safe_point(void) { + CHECKED(_stm_stop_safe_point()); +} + bool _check_abort_transaction(void) { CHECKED(stm_abort_transaction()); } @@ -160,6 +167,10 @@ CHECKED(stm_become_inevitable(tl, "TEST")); } +bool _check_become_globally_unique_transaction(stm_thread_local_t *tl) { + CHECKED(stm_become_globally_unique_transaction(tl, "TESTGUT")); +} + bool _check_stm_validate(void) { CHECKED(stm_validate(NULL)); } @@ -491,13 +502,17 @@ def switch(self, thread_num): assert thread_num != self.current_thread + tl = self.tls[self.current_thread] + if lib._stm_in_transaction(tl): + stm_start_safe_point() # self.current_thread = thread_num tl2 = self.tls[thread_num] # if lib._stm_in_transaction(tl2): lib._stm_test_switch(tl2) - stm_validate() # can raise + stm_stop_safe_point() # can raise Conflict + stm_validate() # can raise Conflict def switch_to_segment(self, seg_num): lib._stm_test_switch_segment(seg_num) @@ -542,3 +557,8 @@ tl = self.tls[self.current_thread] if lib._check_become_inevitable(tl): raise Conflict() + + def become_globally_unique_transaction(self): + tl = self.tls[self.current_thread] + if lib._check_become_globally_unique_transaction(tl): + raise Conflict() From noreply at buildbot.pypy.org Tue Sep 9 15:14:03 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 9 Sep 2014 15:14:03 +0200 (CEST) Subject: [pypy-commit] pypy default: fix test_read_universal on win32 Message-ID: <20140909131403.C3B361C0350@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r73392:6cf4196d8d3a Date: 2014-09-09 06:13 -0700 http://bitbucket.org/pypy/pypy/changeset/6cf4196d8d3a/ Log: fix test_read_universal on win32 diff --git a/rpython/rlib/test/test_rfile.py b/rpython/rlib/test/test_rfile.py --- a/rpython/rlib/test/test_rfile.py +++ b/rpython/rlib/test/test_rfile.py @@ -190,9 +190,9 @@ self.interpret(f, []) def test_read_universal(self): - fname = self.tmpdir.join('read_univ') - fname.write("dupa\ndupb\r\ndupc\rdupd") - fname = str(fname) + fname = str(self.tmpdir.join('read_univ')) + with open(fname, 'wb') as f: + f.write("dupa\ndupb\r\ndupc\rdupd") def f(): f = open(fname, 'U') From noreply at buildbot.pypy.org Tue Sep 9 17:38:53 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 9 Sep 2014 17:38:53 +0200 (CEST) Subject: [pypy-commit] pypy default: test/fix rfile seek case with univ newline Message-ID: <20140909153853.844501C06A7@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r73393:c4e64d36778a Date: 2014-09-09 11:34 -0400 http://bitbucket.org/pypy/pypy/changeset/c4e64d36778a/ Log: test/fix rfile seek case with univ newline diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py --- a/rpython/rlib/rfile.py +++ b/rpython/rlib/rfile.py @@ -460,6 +460,7 @@ if res == -1: errno = rposix.get_errno() raise IOError(errno, os.strerror(errno)) + self._skipnextlf = False def tell(self): self._check_closed() diff --git a/rpython/rlib/test/test_rfile.py b/rpython/rlib/test/test_rfile.py --- a/rpython/rlib/test/test_rfile.py +++ b/rpython/rlib/test/test_rfile.py @@ -212,6 +212,11 @@ assert f.readline() == "dupd" assert f.tell() == 20 assert f.readline() == "" + f.seek(0) + assert f.readline() == "dupa\n" + assert f.readline() == "dupb\n" + f.seek(4) + assert f.read(1) == "\n" f.close() f() From noreply at buildbot.pypy.org Tue Sep 9 17:38:54 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 9 Sep 2014 17:38:54 +0200 (CEST) Subject: [pypy-commit] pypy use-file-star-for-file: merge default Message-ID: <20140909153854.D88B21C06A7@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: use-file-star-for-file Changeset: r73394:ffc6b3c013ca Date: 2014-09-09 11:35 -0400 http://bitbucket.org/pypy/pypy/changeset/ffc6b3c013ca/ Log: merge default diff --git a/pypy/doc/release-2.4.0.rst b/pypy/doc/release-2.4.0.rst --- a/pypy/doc/release-2.4.0.rst +++ b/pypy/doc/release-2.4.0.rst @@ -62,7 +62,8 @@ bytearray handling, and a major rewrite of the GIL handling. This means that external calls are now a lot faster, especially the CFFI ones. It also means better performance in a lot of corner cases with handling strings or -bytearrays. +bytearrays. The main bugfix is handling of many socket objects in your +program which in the long run used to "leak" memory. PyPy now uses Python 2.7.8 standard library. @@ -109,8 +110,9 @@ .. _`whats-new`: http://doc.pypy.org/en/latest/whatsnew-2.3.1.html .. _resolved: https://bitbucket.org/pypy/pypy/issues?status=resolved -We have further improvements on the way: rpython file handling and -usable numpy linalg compatabiity should be merged soon. +We have further improvements on the way: rpython file handling, +numpy linalg compatability, as well +as improved GC and many smaller improvements. Please try it out and let us know what you think. We especially welcome success stories, we know you are using PyPy, please tell us about it! diff --git a/rpython/jit/backend/arm/detect.py b/rpython/jit/backend/arm/detect.py --- a/rpython/jit/backend/arm/detect.py +++ b/rpython/jit/backend/arm/detect.py @@ -38,9 +38,9 @@ try: buf = os.read(fd, 2048) if not buf: + n = 6 # we assume ARMv6 as base case debug_print("Could not detect ARM architecture " "version, assuming", "ARMv%d" % n) - n = 6 # we asume ARMv6 as base case finally: os.close(fd) # "Processor : ARMv%d-compatible processor rev 7 (v6l)" diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py --- a/rpython/jit/backend/arm/opassembler.py +++ b/rpython/jit/backend/arm/opassembler.py @@ -1128,6 +1128,8 @@ self.mc.VCVT_int_to_float(res.value, r.svfp_ip.value) return fcond + # the following five instructions are only ARMv7; + # regalloc.py won't call them at all on ARMv6 emit_op_llong_add = gen_emit_float_op('llong_add', 'VADD_i64') emit_op_llong_sub = gen_emit_float_op('llong_sub', 'VSUB_i64') emit_op_llong_and = gen_emit_float_op('llong_and', 'VAND_i64') diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py --- a/rpython/jit/backend/arm/regalloc.py +++ b/rpython/jit/backend/arm/regalloc.py @@ -184,7 +184,7 @@ class Regalloc(BaseRegalloc): - def __init__(self, assembler=None): + def __init__(self, assembler): self.cpu = assembler.cpu self.assembler = assembler self.frame_manager = None @@ -290,7 +290,7 @@ return self.vfprm.convert_to_imm(value) def _prepare(self, inputargs, operations, allgcrefs): - cpu = self.assembler.cpu + cpu = self.cpu self.fm = ARMFrameManager(cpu.get_baseofs_of_frame_field()) self.frame_manager = self.fm operations = cpu.gc_ll_descr.rewrite_assembler(cpu, operations, @@ -550,18 +550,19 @@ EffectInfo.OS_LLONG_AND, EffectInfo.OS_LLONG_OR, EffectInfo.OS_LLONG_XOR): - args = self._prepare_llong_binop_xx(op, fcond) - self.perform_llong(op, args, fcond) - return - if oopspecindex == EffectInfo.OS_LLONG_TO_INT: + if self.cpu.cpuinfo.arch_version >= 7: + args = self._prepare_llong_binop_xx(op, fcond) + self.perform_llong(op, args, fcond) + return + elif oopspecindex == EffectInfo.OS_LLONG_TO_INT: args = self._prepare_llong_to_int(op, fcond) self.perform_llong(op, args, fcond) return - if oopspecindex == EffectInfo.OS_MATH_SQRT: + elif oopspecindex == EffectInfo.OS_MATH_SQRT: args = self.prepare_op_math_sqrt(op, fcond) self.perform_math(op, args, fcond) return - #if oopspecindex == EffectInfo.OS_MATH_READ_TIMESTAMP: + #elif oopspecindex == EffectInfo.OS_MATH_READ_TIMESTAMP: # ... return self._prepare_call(op) @@ -590,7 +591,7 @@ # spill variables that need to be saved around calls self.vfprm.before_call(save_all_regs=save_all_regs) if not save_all_regs: - gcrootmap = self.assembler.cpu.gc_ll_descr.gcrootmap + gcrootmap = self.cpu.gc_ll_descr.gcrootmap if gcrootmap and gcrootmap.is_shadow_stack: save_all_regs = 2 self.rm.before_call(save_all_regs=save_all_regs) @@ -1082,7 +1083,7 @@ gcmap = self.get_gcmap([r.r0, r.r1]) self.possibly_free_var(t) # - gc_ll_descr = self.assembler.cpu.gc_ll_descr + gc_ll_descr = self.cpu.gc_ll_descr self.assembler.malloc_cond_varsize_frame( gc_ll_descr.get_nursery_free_addr(), gc_ll_descr.get_nursery_top_addr(), @@ -1092,7 +1093,7 @@ self.assembler._alignment_check() def prepare_op_call_malloc_nursery_varsize(self, op, fcond): - gc_ll_descr = self.assembler.cpu.gc_ll_descr + gc_ll_descr = self.cpu.gc_ll_descr if not hasattr(gc_ll_descr, 'max_size_of_young_obj'): raise Exception("unreachable code") # for boehm, this function should never be called diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py --- a/rpython/rlib/rfile.py +++ b/rpython/rlib/rfile.py @@ -460,6 +460,7 @@ if res == -1: errno = rposix.get_errno() raise IOError(errno, os.strerror(errno)) + self._skipnextlf = False def tell(self): self._check_closed() diff --git a/rpython/rlib/test/test_rfile.py b/rpython/rlib/test/test_rfile.py --- a/rpython/rlib/test/test_rfile.py +++ b/rpython/rlib/test/test_rfile.py @@ -190,9 +190,9 @@ self.interpret(f, []) def test_read_universal(self): - fname = self.tmpdir.join('read_univ') - fname.write("dupa\ndupb\r\ndupc\rdupd") - fname = str(fname) + fname = str(self.tmpdir.join('read_univ')) + with open(fname, 'wb') as f: + f.write("dupa\ndupb\r\ndupc\rdupd") def f(): f = open(fname, 'U') @@ -212,6 +212,11 @@ assert f.readline() == "dupd" assert f.tell() == 20 assert f.readline() == "" + f.seek(0) + assert f.readline() == "dupa\n" + assert f.readline() == "dupb\n" + f.seek(4) + assert f.read(1) == "\n" f.close() f() From noreply at buildbot.pypy.org Tue Sep 9 19:29:18 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 9 Sep 2014 19:29:18 +0200 (CEST) Subject: [pypy-commit] pypy use-file-star-for-file: remove obsolete test Message-ID: <20140909172918.789611D2DDE@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: use-file-star-for-file Changeset: r73395:830102f2bf6d Date: 2014-09-09 11:48 -0400 http://bitbucket.org/pypy/pypy/changeset/830102f2bf6d/ Log: remove obsolete test diff --git a/pypy/module/_file/test/test_file.py b/pypy/module/_file/test/test_file.py --- a/pypy/module/_file/test/test_file.py +++ b/pypy/module/_file/test/test_file.py @@ -282,38 +282,6 @@ f.close() -class AppTestNonblocking(object): - def setup_class(cls): - from pypy.module._file.interp_file import W_File - - cls.old_read = os.read - - if cls.runappdirect: - py.test.skip("works with internals of _file impl on py.py") - state = [0] - def read(fd, n=None): - if fd != 42: - return cls.old_read(fd, n) - if state[0] == 0: - state[0] += 1 - return "xyz" - if state[0] < 3: - state[0] += 1 - raise OSError(errno.EAGAIN, "xyz") - return '' - os.read = read - stdin = W_File(cls.space) - stdin.file_fdopen(42, 'rb', 1) - stdin.name = '' - cls.w_stream = stdin - - def teardown_class(cls): - os.read = cls.old_read - - def test_nonblocking_file(self): - res = self.stream.read() - assert res == 'xyz' - class AppTestConcurrency(object): # these tests only really make sense on top of a translated pypy-c, # because on top of py.py the inner calls to os.write() don't From noreply at buildbot.pypy.org Tue Sep 9 19:29:19 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 9 Sep 2014 19:29:19 +0200 (CEST) Subject: [pypy-commit] pypy use-file-star-for-file: replace usage of streamio in imp Message-ID: <20140909172919.ABDD01D2DDE@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: use-file-star-for-file Changeset: r73396:b1d15f933761 Date: 2014-09-09 12:54 -0400 http://bitbucket.org/pypy/pypy/changeset/b1d15f933761/ Log: replace usage of streamio in imp diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py --- a/pypy/module/imp/importing.py +++ b/pypy/module/imp/importing.py @@ -11,8 +11,7 @@ from pypy.interpreter.baseobjspace import W_Root, CannotHaveLock from pypy.interpreter.eval import Code from pypy.interpreter.pycode import PyCode -from rpython.rlib import streamio, jit -from rpython.rlib.streamio import StreamErrors +from rpython.rlib import rfile, jit from rpython.rlib.objectmodel import we_are_translated, specialize from pypy.module.sys.version import PYPY_VERSION @@ -535,14 +534,15 @@ try: if modtype in (PY_SOURCE, PY_COMPILED, C_EXTENSION): assert suffix is not None + assert filemode is not None filename = filepart + suffix - stream = streamio.open_file_as_stream(filename, filemode) + stream = rfile.create_file(filename, filemode) try: return FindInfo(modtype, filename, stream, suffix, filemode) except: stream.close() raise - except StreamErrors: + except EnvironmentError: pass # XXX! must not eat all exceptions, e.g. # Out of file descriptors. @@ -602,14 +602,14 @@ if find_info.modtype == PY_SOURCE: load_source_module( space, w_modulename, w_mod, - find_info.filename, find_info.stream.readall(), - find_info.stream.try_to_find_file_descriptor()) + find_info.filename, find_info.stream.read(), + find_info.stream.fileno()) return w_mod elif find_info.modtype == PY_COMPILED: magic = _r_long(find_info.stream) timestamp = _r_long(find_info.stream) load_compiled_module(space, w_modulename, w_mod, find_info.filename, - magic, timestamp, find_info.stream.readall()) + magic, timestamp, find_info.stream.read()) return w_mod elif find_info.modtype == PKG_DIRECTORY: w_path = space.newlist([space.wrap(find_info.filename)]) @@ -879,7 +879,7 @@ if stream: # existing and up-to-date .pyc file try: - code_w = read_compiled_module(space, cpathname, stream.readall()) + code_w = read_compiled_module(space, cpathname, stream.read()) finally: stream.close() space.setattr(w_mod, w('__file__'), w(cpathname)) @@ -925,17 +925,10 @@ d -= 0x100 return a | (b<<8) | (c<<16) | (d<<24) -def _read_n(stream, n): - buf = '' - while len(buf) < n: - data = stream.read(n - len(buf)) - if not data: - raise streamio.StreamError("end of file") - buf += data - return buf - def _r_long(stream): - s = _read_n(stream, 4) + s = stream.read(4) + if len(s) != 4: + raise IOError("end of file") return _get_long(s) def _w_long(stream, x): @@ -954,7 +947,7 @@ """ stream = None try: - stream = streamio.open_file_as_stream(pycfilename, "rb") + stream = rfile.create_file(pycfilename, "rb") magic = _r_long(stream) if magic != get_pyc_magic(space): stream.close() @@ -964,7 +957,7 @@ stream.close() return None return stream - except StreamErrors: + except EnvironmentError: if stream: stream.close() return None # XXX! must not eat all exceptions, e.g. @@ -1002,16 +995,17 @@ return w_mod +O_BINARY = getattr(os, 'O_BINARY', 0) + def open_exclusive(space, cpathname, mode): try: os.unlink(cpathname) except OSError: pass - flags = (os.O_EXCL|os.O_CREAT|os.O_WRONLY|os.O_TRUNC| - streamio.O_BINARY) + flags = (os.O_EXCL|os.O_CREAT|os.O_WRONLY|os.O_TRUNC|O_BINARY) fd = os.open(cpathname, flags, mode) - return streamio.fdopen_as_stream(fd, "wb") + return rfile.create_fdopen_rfile(fd, "wb") def write_compiled_module(space, co, cpathname, src_mode, src_mtime): """ @@ -1037,7 +1031,7 @@ mode = src_mode & ~0111 try: stream = open_exclusive(space, cpathname, mode) - except (OSError, StreamErrors): + except EnvironmentError: try: os.unlink(cpathname) except OSError: @@ -1058,7 +1052,7 @@ _w_long(stream, src_mtime) finally: stream.close() - except StreamErrors: + except EnvironmentError: try: os.unlink(cpathname) except OSError: diff --git a/pypy/module/imp/interp_imp.py b/pypy/module/imp/interp_imp.py --- a/pypy/module/imp/interp_imp.py +++ b/pypy/module/imp/interp_imp.py @@ -1,7 +1,6 @@ +from rpython.rlib import rfile from pypy.module.imp import importing from pypy.module._file.interp_file import W_File -from rpython.rlib import streamio -from rpython.rlib.streamio import StreamErrors from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter.module import Module from pypy.interpreter.gateway import unwrap_spec @@ -32,15 +31,9 @@ d = x & 0xff return space.wrap(chr(a) + chr(b) + chr(c) + chr(d)) -def get_file(space, w_file, filename, filemode): +def get_file(space, w_file, filename, mode): if space.is_none(w_file): - try: - return streamio.open_file_as_stream(filename, filemode) - except StreamErrors, e: - # XXX this is not quite the correct place, but it will do for now. - # XXX see the issue which I'm sure exists already but whose number - # XXX I cannot find any more... - raise wrap_streamerror(space, e) + return rfile.create_file(filename, mode) else: return space.interp_w(W_File, w_file).stream @@ -60,8 +53,7 @@ if stream is not None: fileobj = W_File(space) fileobj.fdopenstream( - stream, stream.try_to_find_file_descriptor(), - find_info.filemode, w_filename) + stream, find_info.filemode) w_fileobj = space.wrap(fileobj) else: w_fileobj = space.w_None @@ -100,7 +92,7 @@ importing.load_source_module( space, w_modulename, w_mod, - filename, stream.readall(), stream.try_to_find_file_descriptor()) + filename, stream.read(), stream.fileno()) if space.is_none(w_file): stream.close() return w_mod @@ -115,7 +107,7 @@ importing.load_compiled_module( space, w_modulename, w_module, filename, magic, timestamp, - stream.readall()) + stream.read()) if space.is_none(w_file): stream.close() From noreply at buildbot.pypy.org Tue Sep 9 19:29:20 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 9 Sep 2014 19:29:20 +0200 (CEST) Subject: [pypy-commit] pypy use-file-star-for-file: remove usage of streamio in imp tests Message-ID: <20140909172920.DCFF61D2DDE@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: use-file-star-for-file Changeset: r73397:db0f83d4b6c3 Date: 2014-09-09 12:48 -0400 http://bitbucket.org/pypy/pypy/changeset/db0f83d4b6c3/ Log: remove usage of streamio in imp tests diff --git a/pypy/module/imp/interp_imp.py b/pypy/module/imp/interp_imp.py --- a/pypy/module/imp/interp_imp.py +++ b/pypy/module/imp/interp_imp.py @@ -4,7 +4,6 @@ from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter.module import Module from pypy.interpreter.gateway import unwrap_spec -from pypy.interpreter.streamutil import wrap_streamerror def get_suffixes(space): diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py --- a/pypy/module/imp/test/test_import.py +++ b/pypy/module/imp/test/test_import.py @@ -4,7 +4,6 @@ from pypy.interpreter.error import OperationError import pypy.interpreter.pycode from rpython.tool.udir import udir -from rpython.rlib import streamio from pypy.tool.option import make_config from pypy.tool.pytest.objspace import maketestobjspace import pytest @@ -103,12 +102,11 @@ w = space.wrap w_modname = w("compiled.x") filename = str(p.join("x.py")) - stream = streamio.open_file_as_stream(filename, "r") + stream = open(filename, "r") try: importing.load_source_module( space, w_modname, w(importing.Module(space, w_modname)), - filename, stream.readall(), - stream.try_to_find_file_descriptor()) + filename, stream.read(), stream.fileno()) finally: stream.close() if space.config.objspace.usepycfiles: @@ -757,11 +755,11 @@ mtime = 12345 co = compile('x = 42', '?', 'exec') cpathname = _testfile(importing.get_pyc_magic(space), mtime, co) - stream = streamio.open_file_as_stream(cpathname, "rb") + stream = open(cpathname, "rb") try: stream.seek(8, 0) w_code = importing.read_compiled_module( - space, cpathname, stream.readall()) + space, cpathname, stream.read()) pycode = w_code finally: stream.close() @@ -778,7 +776,7 @@ co = compile('x = 42', '?', 'exec') cpathname = _testfile(importing.get_pyc_magic(space), mtime, co) w_modulename = space.wrap('somemodule') - stream = streamio.open_file_as_stream(cpathname, "rb") + stream = open(cpathname, "rb") try: w_mod = space.wrap(Module(space, w_modulename)) magic = importing._r_long(stream) @@ -789,7 +787,7 @@ cpathname, magic, timestamp, - stream.readall()) + stream.read()) finally: stream.close() assert w_mod is w_ret @@ -800,11 +798,11 @@ def test_parse_source_module(self): space = self.space pathname = _testfilesource() - stream = streamio.open_file_as_stream(pathname, "r") + stream = open(pathname, "r") try: w_ret = importing.parse_source_module(space, pathname, - stream.readall()) + stream.read()) finally: stream.close() pycode = w_ret @@ -817,14 +815,14 @@ def test_long_writes(self): pathname = str(udir.join('test.dat')) - stream = streamio.open_file_as_stream(pathname, "wb") + stream = open(pathname, "wb") try: importing._w_long(stream, 42) importing._w_long(stream, 12312) importing._w_long(stream, 128397198) finally: stream.close() - stream = streamio.open_file_as_stream(pathname, "rb") + stream = open(pathname, "rb") try: res = importing._r_long(stream) assert res == 42 @@ -840,12 +838,11 @@ w_modulename = space.wrap('somemodule') w_mod = space.wrap(Module(space, w_modulename)) pathname = _testfilesource() - stream = streamio.open_file_as_stream(pathname, "r") + stream = open(pathname, "r") try: w_ret = importing.load_source_module( space, w_modulename, w_mod, - pathname, stream.readall(), - stream.try_to_find_file_descriptor()) + pathname, stream.read(), stream.fileno()) finally: stream.close() assert w_mod is w_ret @@ -862,12 +859,11 @@ w_modulename = space.wrap('somemodule') w_mod = space.wrap(Module(space, w_modulename)) pathname = _testfilesource() - stream = streamio.open_file_as_stream(pathname, "r") + stream = open(pathname, "r") try: w_ret = importing.load_source_module( space, w_modulename, w_mod, - pathname, stream.readall(), - stream.try_to_find_file_descriptor(), + pathname, stream.read(), stream.fileno(), write_pyc=False) finally: stream.close() @@ -879,14 +875,13 @@ w_modulename = space.wrap('somemodule') w_mod = space.wrap(Module(space, w_modulename)) pathname = _testfilesource() - stream = streamio.open_file_as_stream(pathname, "r") + stream = open(pathname, "r") try: space.setattr(space.sys, space.wrap('dont_write_bytecode'), space.w_True) w_ret = importing.load_source_module( space, w_modulename, w_mod, - pathname, stream.readall(), - stream.try_to_find_file_descriptor()) + pathname, stream.read(), stream.fileno()) finally: space.setattr(space.sys, space.wrap('dont_write_bytecode'), space.w_False) @@ -900,12 +895,11 @@ w_modulename = space.wrap('somemodule') w_mod = space.wrap(Module(space, w_modulename)) pathname = _testfilesource(source="") - stream = streamio.open_file_as_stream(pathname, "r") + stream = open(pathname, "r") try: w_ret = importing.load_source_module( space, w_modulename, w_mod, - pathname, stream.readall(), - stream.try_to_find_file_descriptor()) + pathname, stream.read(), stream.fileno()) except OperationError: # OperationError("Syntax Error") pass @@ -913,19 +907,18 @@ cpathname = udir.join('test.pyc') assert not cpathname.check() - + def test_load_source_module_importerror(self): # the .pyc file is created before executing the module space = self.space w_modulename = space.wrap('somemodule') w_mod = space.wrap(Module(space, w_modulename)) pathname = _testfilesource(source="a = unknown_name") - stream = streamio.open_file_as_stream(pathname, "r") + stream = open(pathname, "r") try: w_ret = importing.load_source_module( space, w_modulename, w_mod, - pathname, stream.readall(), - stream.try_to_find_file_descriptor()) + pathname, stream.read(), stream.fileno()) except OperationError: # OperationError("NameError", "global name 'unknown_name' is not defined") pass @@ -939,11 +932,11 @@ space = self.space pathname = _testfilesource() os.chmod(pathname, 0777) - stream = streamio.open_file_as_stream(pathname, "r") + stream = open(pathname, "r") try: w_ret = importing.parse_source_module(space, pathname, - stream.readall()) + stream.read()) finally: stream.close() pycode = w_ret @@ -969,11 +962,11 @@ assert os.stat(cpathname).st_mode & 0111 == 0 # read compiled module - stream = streamio.open_file_as_stream(cpathname, "rb") + stream = open(cpathname, "rb") try: stream.seek(8, 0) w_code = importing.read_compiled_module(space, cpathname, - stream.readall()) + stream.read()) pycode = w_code finally: stream.close() @@ -1004,7 +997,7 @@ cpathname = _testfile(importing.get_pyc_magic(space1), mtime, co) w_modulename = space2.wrap('somemodule') - stream = streamio.open_file_as_stream(cpathname, "rb") + stream = open(cpathname, "rb") try: w_mod = space2.wrap(Module(space2, w_modulename)) magic = importing._r_long(stream) @@ -1017,7 +1010,7 @@ cpathname, magic, timestamp, - stream.readall()) + stream.read()) finally: stream.close() From noreply at buildbot.pypy.org Tue Sep 9 19:29:22 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 9 Sep 2014 19:29:22 +0200 (CEST) Subject: [pypy-commit] pypy use-file-star-for-file: handle exception here Message-ID: <20140909172922.088911D2DDE@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: use-file-star-for-file Changeset: r73398:d1feca94030c Date: 2014-09-09 13:02 -0400 http://bitbucket.org/pypy/pypy/changeset/d1feca94030c/ Log: handle exception here diff --git a/pypy/module/imp/interp_imp.py b/pypy/module/imp/interp_imp.py --- a/pypy/module/imp/interp_imp.py +++ b/pypy/module/imp/interp_imp.py @@ -1,7 +1,7 @@ from rpython.rlib import rfile from pypy.module.imp import importing from pypy.module._file.interp_file import W_File -from pypy.interpreter.error import OperationError, oefmt +from pypy.interpreter.error import OperationError, oefmt, exception_from_errno from pypy.interpreter.module import Module from pypy.interpreter.gateway import unwrap_spec @@ -32,7 +32,10 @@ def get_file(space, w_file, filename, mode): if space.is_none(w_file): - return rfile.create_file(filename, mode) + try: + return rfile.create_file(filename, mode) + except: + raise exception_from_errno(space, space.w_IOError) else: return space.interp_w(W_File, w_file).stream From noreply at buildbot.pypy.org Tue Sep 9 19:29:23 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 9 Sep 2014 19:29:23 +0200 (CEST) Subject: [pypy-commit] pypy use-file-star-for-file: fix translation Message-ID: <20140909172923.2E51D1D2DDE@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: use-file-star-for-file Changeset: r73399:6b14933abfd0 Date: 2014-09-09 13:27 -0400 http://bitbucket.org/pypy/pypy/changeset/6b14933abfd0/ Log: fix translation diff --git a/pypy/module/_file/interp_file.py b/pypy/module/_file/interp_file.py --- a/pypy/module/_file/interp_file.py +++ b/pypy/module/_file/interp_file.py @@ -217,6 +217,7 @@ self.do_direct_write(data) def do_direct_write(self, data): + assert data is not None self.softspace = 0 self.getstream().write(data) From noreply at buildbot.pypy.org Tue Sep 9 23:26:18 2014 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 9 Sep 2014 23:26:18 +0200 (CEST) Subject: [pypy-commit] buildbot default: remove aurora, add allegro_win32 for windows builds Message-ID: <20140909212618.48D571C06A7@cobra.cs.uni-duesseldorf.de> Author: Matti Picus Branch: Changeset: r917:3b13288e2ae9 Date: 2014-09-10 00:26 +0300 http://bitbucket.org/pypy/buildbot/changeset/3b13288e2ae9/ Log: remove aurora, add allegro_win32 for windows builds diff --git a/bot2/pypybuildbot/master.py b/bot2/pypybuildbot/master.py --- a/bot2/pypybuildbot/master.py +++ b/bot2/pypybuildbot/master.py @@ -385,7 +385,7 @@ 'category' : 'mac64', }, {"name": WIN32, - "slavenames": ["aurora", "SalsaSalsa"], + "slavenames": ["SalsaSalsa", "allegro_win32"], "builddir": WIN32, "factory": pypyOwnTestFactoryWin, "locks": [WinSlaveLock.access('counting')], @@ -398,14 +398,14 @@ "category": 'win32' }, {"name": APPLVLWIN32, - "slavenames": ["aurora", "SalsaSalsa"], + "slavenames": ["SalsaSalsa", "allegro_win32"], "builddir": APPLVLWIN32, "factory": pypyTranslatedAppLevelTestFactoryWin, "locks": [WinSlaveLock.access('counting')], "category": "win32", }, {"name" : JITWIN32, - "slavenames": ["aurora", "SalsaSalsa"], + "slavenames": ["SalsaSalsa", "allegro_win32"], 'builddir' : JITWIN32, 'factory' : pypyJITTranslatedTestFactoryWin, "locks": [WinSlaveLock.access('counting')], From noreply at buildbot.pypy.org Wed Sep 10 01:03:58 2014 From: noreply at buildbot.pypy.org (mattip) Date: Wed, 10 Sep 2014 01:03:58 +0200 (CEST) Subject: [pypy-commit] pypy default: typos Message-ID: <20140909230358.0B25B1D22E6@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r73400:9c30b3d71ad1 Date: 2014-09-08 23:14 +0300 http://bitbucket.org/pypy/pypy/changeset/9c30b3d71ad1/ Log: typos diff --git a/pypy/doc/release-2.4.0.rst b/pypy/doc/release-2.4.0.rst --- a/pypy/doc/release-2.4.0.rst +++ b/pypy/doc/release-2.4.0.rst @@ -7,10 +7,6 @@ You can already download the PyPy 2.4-beta1 pre-release here: - https://bitbucket.org/pypy/pypy/downloads - -It should also soon be available on: - http://pypy.org/download.html We would like to thank our donors for the continued support of the PyPy @@ -87,7 +83,7 @@ this mostly affects errno handling on linux, which makes external calls faster. -* Move to a mixed polling and mutex GIL model that make mutli-threaded jitted +* Move to a mixed polling and mutex GIL model that make mutlithreaded jitted code run *much* faster * Optimize errno handling in linux (x86 and x86-64 only) @@ -111,7 +107,7 @@ .. _resolved: https://bitbucket.org/pypy/pypy/issues?status=resolved We have further improvements on the way: rpython file handling, -numpy linalg compatability, as well +numpy linalg compatibility, as well as improved GC and many smaller improvements. Please try it out and let us know what you think. We especially welcome From noreply at buildbot.pypy.org Wed Sep 10 01:03:59 2014 From: noreply at buildbot.pypy.org (mattip) Date: Wed, 10 Sep 2014 01:03:59 +0200 (CEST) Subject: [pypy-commit] pypy default: always create pdb debug info in windows Message-ID: <20140909230359.5931D1D22E6@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r73401:81319f2bd5bf Date: 2014-09-10 02:02 +0300 http://bitbucket.org/pypy/pypy/changeset/81319f2bd5bf/ Log: always create pdb debug info in windows diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -100,8 +100,8 @@ cc = 'cl.exe' link = 'link.exe' - cflags = ('/MD', '/O2') - link_flags = () + cflags = ('/MD', '/O2', '/Zi') + link_flags = ('/debug',) standalone_only = () shared_only = () environ = None @@ -143,7 +143,6 @@ # Install debug options only when interpreter is in debug mode if sys.executable.lower().endswith('_d.exe'): self.cflags = ['/MDd', '/Z7', '/Od'] - self.link_flags = ['/debug'] # Increase stack size, for the linker and the stack check code. stack_size = 8 << 20 # 8 Mb From noreply at buildbot.pypy.org Wed Sep 10 01:37:23 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Wed, 10 Sep 2014 01:37:23 +0200 (CEST) Subject: [pypy-commit] pypy default: IOError with single arg takes string Message-ID: <20140909233723.88B151D2317@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r73402:67a100b33b4a Date: 2014-09-09 16:27 -0400 http://bitbucket.org/pypy/pypy/changeset/67a100b33b4a/ Log: IOError with single arg takes string diff --git a/rpython/rtyper/rbuiltin.py b/rpython/rtyper/rbuiltin.py --- a/rpython/rtyper/rbuiltin.py +++ b/rpython/rtyper/rbuiltin.py @@ -269,17 +269,19 @@ hop.exception_cannot_occur() v_self = hop.args_v[0] r_self = hop.args_r[0] - if hop.nb_args >= 2: + if hop.nb_args <= 2: + v_errno = hop.inputconst(lltype.Signed, 0) + if hop.nb_args == 2: + v_strerror = hop.inputarg(rstr.string_repr, arg=1) + r_self.setfield(v_self, 'strerror', v_strerror, hop.llops) + else: v_errno = hop.inputarg(lltype.Signed, arg=1) - else: - v_errno = hop.inputconst(lltype.Signed, 0) - r_self.setfield(v_self, 'errno', v_errno, hop.llops) - if hop.nb_args >= 3: v_strerror = hop.inputarg(rstr.string_repr, arg=2) r_self.setfield(v_self, 'strerror', v_strerror, hop.llops) if hop.nb_args >= 4: v_filename = hop.inputarg(rstr.string_repr, arg=3) r_self.setfield(v_self, 'filename', v_filename, hop.llops) + r_self.setfield(v_self, 'errno', v_errno, hop.llops) def rtype_WindowsError__init__(hop): hop.exception_cannot_occur() diff --git a/rpython/rtyper/test/test_exception.py b/rpython/rtyper/test/test_exception.py --- a/rpython/rtyper/test/test_exception.py +++ b/rpython/rtyper/test/test_exception.py @@ -36,7 +36,7 @@ class TestException(BaseRtypingTest): def test_exception_with_arg(self): def g(n): - raise IOError(n) + raise IOError("test") def h(n): raise OSError(n, "?", None) def i(n): @@ -49,8 +49,8 @@ try: g(n) except IOError, e: - assert e.errno == 42 - assert e.strerror is None + assert e.errno == 0 + assert e.strerror == "test" assert e.filename is None else: assert False From noreply at buildbot.pypy.org Wed Sep 10 01:37:24 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Wed, 10 Sep 2014 01:37:24 +0200 (CEST) Subject: [pypy-commit] pypy default: merge heads Message-ID: <20140909233724.C708D1D2317@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r73403:b18809f448df Date: 2014-09-09 19:37 -0400 http://bitbucket.org/pypy/pypy/changeset/b18809f448df/ Log: merge heads diff --git a/pypy/doc/release-2.4.0.rst b/pypy/doc/release-2.4.0.rst --- a/pypy/doc/release-2.4.0.rst +++ b/pypy/doc/release-2.4.0.rst @@ -7,10 +7,6 @@ You can already download the PyPy 2.4-beta1 pre-release here: - https://bitbucket.org/pypy/pypy/downloads - -It should also soon be available on: - http://pypy.org/download.html We would like to thank our donors for the continued support of the PyPy @@ -87,7 +83,7 @@ this mostly affects errno handling on linux, which makes external calls faster. -* Move to a mixed polling and mutex GIL model that make mutli-threaded jitted +* Move to a mixed polling and mutex GIL model that make mutlithreaded jitted code run *much* faster * Optimize errno handling in linux (x86 and x86-64 only) @@ -111,7 +107,7 @@ .. _resolved: https://bitbucket.org/pypy/pypy/issues?status=resolved We have further improvements on the way: rpython file handling, -numpy linalg compatability, as well +numpy linalg compatibility, as well as improved GC and many smaller improvements. Please try it out and let us know what you think. We especially welcome diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -100,8 +100,8 @@ cc = 'cl.exe' link = 'link.exe' - cflags = ('/MD', '/O2') - link_flags = () + cflags = ('/MD', '/O2', '/Zi') + link_flags = ('/debug',) standalone_only = () shared_only = () environ = None @@ -143,7 +143,6 @@ # Install debug options only when interpreter is in debug mode if sys.executable.lower().endswith('_d.exe'): self.cflags = ['/MDd', '/Z7', '/Od'] - self.link_flags = ['/debug'] # Increase stack size, for the linker and the stack check code. stack_size = 8 << 20 # 8 Mb From noreply at buildbot.pypy.org Wed Sep 10 01:37:57 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Wed, 10 Sep 2014 01:37:57 +0200 (CEST) Subject: [pypy-commit] pypy use-file-star-for-file: ensure imp.find_module fobj has a name Message-ID: <20140909233757.BCFE91D2317@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: use-file-star-for-file Changeset: r73404:6d1e13b03f9a Date: 2014-09-09 13:56 -0400 http://bitbucket.org/pypy/pypy/changeset/6d1e13b03f9a/ Log: ensure imp.find_module fobj has a name diff --git a/pypy/module/_file/interp_file.py b/pypy/module/_file/interp_file.py --- a/pypy/module/_file/interp_file.py +++ b/pypy/module/_file/interp_file.py @@ -56,7 +56,7 @@ w_error = space.call_function(space.w_IOError, space.wrap(e.errno), space.wrap(e.strerror), self.w_name) raise OperationError(space.w_IOError, w_error) - def fdopenstream(self, stream, mode): + def fdopenstream(self, stream, mode, w_name=None): self.stream = stream self.mode = mode self.binary = "b" in mode @@ -66,6 +66,8 @@ self.writable = True if '+' in mode: self.readable = self.writable = True + if w_name is not None: + self.w_name = w_name getopenstreams(self.space)[stream] = None def check_closed(self): diff --git a/pypy/module/imp/interp_imp.py b/pypy/module/imp/interp_imp.py --- a/pypy/module/imp/interp_imp.py +++ b/pypy/module/imp/interp_imp.py @@ -55,7 +55,7 @@ if stream is not None: fileobj = W_File(space) fileobj.fdopenstream( - stream, find_info.filemode) + stream, find_info.filemode, w_filename) w_fileobj = space.wrap(fileobj) else: w_fileobj = space.w_None diff --git a/pypy/module/imp/test/test_app.py b/pypy/module/imp/test/test_app.py --- a/pypy/module/imp/test/test_app.py +++ b/pypy/module/imp/test/test_app.py @@ -32,6 +32,7 @@ import os file, pathname, description = self.imp.find_module('StringIO') assert file is not None + assert file.name.endswith("StringIO.py") file.close() assert os.path.exists(pathname) pathname = pathname.lower() From noreply at buildbot.pypy.org Wed Sep 10 01:37:58 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Wed, 10 Sep 2014 01:37:58 +0200 (CEST) Subject: [pypy-commit] pypy use-file-star-for-file: merge default Message-ID: <20140909233758.F21D41D2317@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: use-file-star-for-file Changeset: r73405:74ed616baf65 Date: 2014-09-09 16:28 -0400 http://bitbucket.org/pypy/pypy/changeset/74ed616baf65/ Log: merge default diff --git a/rpython/rtyper/rbuiltin.py b/rpython/rtyper/rbuiltin.py --- a/rpython/rtyper/rbuiltin.py +++ b/rpython/rtyper/rbuiltin.py @@ -269,17 +269,19 @@ hop.exception_cannot_occur() v_self = hop.args_v[0] r_self = hop.args_r[0] - if hop.nb_args >= 2: + if hop.nb_args <= 2: + v_errno = hop.inputconst(lltype.Signed, 0) + if hop.nb_args == 2: + v_strerror = hop.inputarg(rstr.string_repr, arg=1) + r_self.setfield(v_self, 'strerror', v_strerror, hop.llops) + else: v_errno = hop.inputarg(lltype.Signed, arg=1) - else: - v_errno = hop.inputconst(lltype.Signed, 0) - r_self.setfield(v_self, 'errno', v_errno, hop.llops) - if hop.nb_args >= 3: v_strerror = hop.inputarg(rstr.string_repr, arg=2) r_self.setfield(v_self, 'strerror', v_strerror, hop.llops) if hop.nb_args >= 4: v_filename = hop.inputarg(rstr.string_repr, arg=3) r_self.setfield(v_self, 'filename', v_filename, hop.llops) + r_self.setfield(v_self, 'errno', v_errno, hop.llops) def rtype_WindowsError__init__(hop): hop.exception_cannot_occur() diff --git a/rpython/rtyper/test/test_exception.py b/rpython/rtyper/test/test_exception.py --- a/rpython/rtyper/test/test_exception.py +++ b/rpython/rtyper/test/test_exception.py @@ -36,7 +36,7 @@ class TestException(BaseRtypingTest): def test_exception_with_arg(self): def g(n): - raise IOError(n) + raise IOError("test") def h(n): raise OSError(n, "?", None) def i(n): @@ -49,8 +49,8 @@ try: g(n) except IOError, e: - assert e.errno == 42 - assert e.strerror is None + assert e.errno == 0 + assert e.strerror == "test" assert e.filename is None else: assert False From noreply at buildbot.pypy.org Wed Sep 10 01:38:00 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Wed, 10 Sep 2014 01:38:00 +0200 (CEST) Subject: [pypy-commit] pypy use-file-star-for-file: include a copy of old W_File to make bz2 happy Message-ID: <20140909233800.3E1291D2317@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: use-file-star-for-file Changeset: r73406:5a43c448d34d Date: 2014-09-09 17:02 -0400 http://bitbucket.org/pypy/pypy/changeset/5a43c448d34d/ Log: include a copy of old W_File to make bz2 happy diff --git a/pypy/module/bz2/interp_bz2.py b/pypy/module/bz2/interp_bz2.py --- a/pypy/module/bz2/interp_bz2.py +++ b/pypy/module/bz2/interp_bz2.py @@ -237,7 +237,7 @@ # Make the BZ2File type by internally inheriting from W_File. # XXX this depends on internal details of W_File to work properly. -from pypy.module._file.interp_file import W_File +from pypy.module.bz2.interp_file import W_File class W_BZ2File(W_File): diff --git a/pypy/module/bz2/interp_file.py b/pypy/module/bz2/interp_file.py new file mode 100644 --- /dev/null +++ b/pypy/module/bz2/interp_file.py @@ -0,0 +1,640 @@ +import py +import os +import stat +import errno +from rpython.rlib import streamio +from rpython.rlib.objectmodel import specialize +from rpython.rlib.rarithmetic import r_longlong +from rpython.rlib.rstring import StringBuilder +from pypy.module._file.interp_stream import W_AbstractStream, StreamErrors +from pypy.module.posix.interp_posix import dispatch_filename +from pypy.interpreter.error import OperationError, oefmt, wrap_oserror +from pypy.interpreter.typedef import (TypeDef, GetSetProperty, + interp_attrproperty, make_weakref_descr, interp_attrproperty_w) +from pypy.interpreter.gateway import interp2app, unwrap_spec +from pypy.interpreter.streamutil import wrap_streamerror, wrap_oserror_as_ioerror + + +class W_File(W_AbstractStream): + """An interp-level file object. This implements the same interface than + the app-level files, with the following differences: + + * method names are prefixed with 'file_' + * the 'normal' app-level constructor is implemented by file___init__(). + * the methods with the 'direct_' prefix should be used if the caller + locks and unlocks the file itself, and takes care of StreamErrors. + """ + + # Default values until the file is successfully opened + stream = None + w_name = None + mode = "" + binary = False + readable = False + writable = False + softspace= 0 # Required according to file object docs + encoding = None + errors = None + fd = -1 + cffi_fileobj = None # pypy/module/_cffi_backend + + newlines = 0 # Updated when the stream is closed + + def __init__(self, space): + self.space = space + + def __del__(self): + # assume that the file and stream objects are only visible in the + # thread that runs __del__, so no race condition should be possible + self.clear_all_weakrefs() + if self.stream is not None: + self.enqueue_for_destruction(self.space, W_File.destructor, + 'close() method of ') + + def destructor(self): + assert isinstance(self, W_File) + try: + self.direct_close() + except StreamErrors, e: + operr = wrap_streamerror(self.space, e, self.w_name) + raise operr + + def fdopenstream(self, stream, fd, mode, w_name=None): + self.fd = fd + self.mode = mode + self.binary = "b" in mode + if 'r' in mode or 'U' in mode: + self.readable = True + if 'w' in mode or 'a' in mode: + self.writable = True + if '+' in mode: + self.readable = self.writable = True + if w_name is not None: + self.w_name = w_name + self.stream = stream + if stream.flushable(): + getopenstreams(self.space)[stream] = None + + def check_not_dir(self, fd): + try: + st = os.fstat(fd) + except OSError: + pass + else: + if (stat.S_ISDIR(st[0])): + ose = OSError(errno.EISDIR, '') + raise wrap_oserror_as_ioerror(self.space, ose, self.w_name) + + def check_mode_ok(self, mode): + if (not mode or mode[0] not in ['r', 'w', 'a', 'U'] or + ('U' in mode and ('w' in mode or 'a' in mode))): + space = self.space + raise oefmt(space.w_ValueError, "invalid mode: '%s'", mode) + + def check_closed(self): + if self.stream is None: + raise OperationError(self.space.w_ValueError, + self.space.wrap("I/O operation on closed file") + ) + + def check_readable(self): + if not self.readable: + raise OperationError(self.space.w_IOError, self.space.wrap( + "File not open for reading")) + + def check_writable(self): + if not self.writable: + raise OperationError(self.space.w_IOError, self.space.wrap( + "File not open for writing")) + + def getstream(self): + """Return self.stream or raise an app-level ValueError if missing + (i.e. if the file is closed).""" + self.check_closed() + return self.stream + + def _when_reading_first_flush(self, otherfile): + """Flush otherfile before reading from self.""" + self.stream = streamio.CallbackReadFilter(self.stream, + otherfile._try_to_flush) + + def _try_to_flush(self): + stream = self.stream + if stream is not None: + stream.flush() + + # ____________________________________________________________ + # + # The 'direct_' methods assume that the caller already acquired the + # file lock. They don't convert StreamErrors to OperationErrors, too. + + @unwrap_spec(mode=str, buffering=int) + def direct___init__(self, w_name, mode='r', buffering=-1): + self.direct_close() + self.w_name = w_name + self.check_mode_ok(mode) + stream = dispatch_filename(streamio.open_file_as_stream)( + self.space, w_name, mode, buffering, signal_checker(self.space)) + fd = stream.try_to_find_file_descriptor() + self.check_not_dir(fd) + self.fdopenstream(stream, fd, mode) + + def direct___enter__(self): + self.check_closed() + return self + + def file__exit__(self, __args__): + """__exit__(*excinfo) -> None. Closes the file.""" + self.space.call_method(self, "close") + # can't return close() value + return None + + def direct_fdopen(self, fd, mode='r', buffering=-1): + self.direct_close() + self.w_name = self.space.wrap('') + self.check_mode_ok(mode) + stream = streamio.fdopen_as_stream(fd, mode, buffering, + signal_checker(self.space)) + self.check_not_dir(fd) + self.fdopenstream(stream, fd, mode) + + def direct_close(self): + stream = self.stream + if stream is not None: + self.newlines = self.stream.getnewlines() + self.stream = None + self.fd = -1 + openstreams = getopenstreams(self.space) + try: + del openstreams[stream] + except KeyError: + pass + # close the stream. If cffi_fileobj is None, we close the + # underlying fileno too. Otherwise, we leave that to + # cffi_fileobj.close(). + cffifo = self.cffi_fileobj + self.cffi_fileobj = None + stream.close1(cffifo is None) + if cffifo is not None: + cffifo.close() + + def direct_fileno(self): + self.getstream() # check if the file is still open + return self.fd + + def direct_flush(self): + self.getstream().flush() + + def direct_next(self): + line = self.getstream().readline() + if line == '': + raise OperationError(self.space.w_StopIteration, self.space.w_None) + return line + + @unwrap_spec(n=int) + def direct_read(self, n=-1): + stream = self.getstream() + self.check_readable() + if n < 0: + return stream.readall() + else: + result = StringBuilder(n) + while n > 0: + try: + data = stream.read(n) + except OSError, e: + # a special-case only for read() (similar to CPython, which + # also loses partial data with other methods): if we get + # EAGAIN after already some data was received, return it. + if is_wouldblock_error(e): + got = result.build() + if len(got) > 0: + return got + raise + if not data: + break + n -= len(data) + result.append(data) + return result.build() + + @unwrap_spec(size=int) + def direct_readline(self, size=-1): + stream = self.getstream() + self.check_readable() + if size < 0: + return stream.readline() + else: + # very inefficient unless there is a peek() + result = StringBuilder() + while size > 0: + # "peeks" on the underlying stream to see how many chars + # we can safely read without reading past an end-of-line + startindex, peeked = stream.peek() + assert 0 <= startindex <= len(peeked) + endindex = startindex + size + pn = peeked.find("\n", startindex, endindex) + if pn < 0: + pn = min(endindex - 1, len(peeked)) + c = stream.read(pn - startindex + 1) + if not c: + break + result.append(c) + if c.endswith('\n'): + break + size -= len(c) + return result.build() + + @unwrap_spec(size=int) + def direct_readlines(self, size=0): + stream = self.getstream() + self.check_readable() + # this is implemented as: .read().split('\n') + # except that it keeps the \n in the resulting strings + if size <= 0: + data = stream.readall() + else: + data = stream.read(size) + result = [] + splitfrom = 0 + for i in range(len(data)): + if data[i] == '\n': + result.append(data[splitfrom : i + 1]) + splitfrom = i + 1 + # + if splitfrom < len(data): + # there is a partial line at the end. If size > 0, it is likely + # to be because the 'read(size)' returned data up to the middle + # of a line. In that case, use 'readline()' to read until the + # end of the current line. + data = data[splitfrom:] + if size > 0: + data += stream.readline() + result.append(data) + return result + + @unwrap_spec(offset=r_longlong, whence=int) + def direct_seek(self, offset, whence=0): + self.getstream().seek(offset, whence) + + def direct_tell(self): + return self.getstream().tell() + + def direct_truncate(self, w_size=None): # note: a wrapped size! + stream = self.getstream() + self.check_writable() + space = self.space + if space.is_none(w_size): + size = stream.tell() + else: + size = space.r_longlong_w(w_size) + stream.truncate(size) + + def direct_write(self, w_data): + space = self.space + self.check_writable() + if self.binary: + data = space.getarg_w('s*', w_data).as_str() + else: + if space.isinstance_w(w_data, space.w_unicode): + w_data = space.call_method(w_data, "encode", + space.wrap(self.encoding), + space.wrap(self.errors)) + data = space.charbuf_w(w_data) + self.do_direct_write(data) + + def do_direct_write(self, data): + self.softspace = 0 + self.getstream().write(data) + + def direct___iter__(self): + self.getstream() + return self + direct_xreadlines = direct___iter__ + + def direct_isatty(self): + self.getstream() # check if the file is still open + return os.isatty(self.fd) + + # ____________________________________________________________ + # + # The 'file_' methods are the one exposed to app-level. + + def file_fdopen(self, fd, mode="r", buffering=-1): + try: + self.direct_fdopen(fd, mode, buffering) + except OSError as e: + raise wrap_oserror(self.space, e) + + _exposed_method_names = [] + + def _decl(class_scope, name, docstring, + wrapresult="space.wrap(result)"): + # hack hack to build a wrapper around the direct_xxx methods. + # The wrapper adds lock/unlock calls and a space.wrap() on + # the result, conversion of stream errors to OperationErrors, + # and has the specified docstring and unwrap_spec. + direct_fn = class_scope['direct_' + name] + co = direct_fn.func_code + argnames = co.co_varnames[:co.co_argcount] + defaults = direct_fn.func_defaults or () + unwrap_spec = getattr(direct_fn, 'unwrap_spec', None) + + args = [] + for i, argname in enumerate(argnames): + try: + default = defaults[-len(argnames) + i] + except IndexError: + args.append(argname) + else: + args.append('%s=%r' % (argname, default)) + sig = ', '.join(args) + assert argnames[0] == 'self' + callsig = ', '.join(argnames[1:]) + + src = py.code.Source(""" + def file_%(name)s(%(sig)s): + %(docstring)r + space = self.space + self.lock() + try: + try: + result = self.direct_%(name)s(%(callsig)s) + except StreamErrors, e: + raise wrap_streamerror(space, e, self.w_name) + finally: + self.unlock() + return %(wrapresult)s + """ % locals()) + exec str(src) in globals(), class_scope + if unwrap_spec is not None: + class_scope['file_' + name].unwrap_spec = unwrap_spec + class_scope['_exposed_method_names'].append(name) + + + _decl(locals(), "__init__", """Opens a file.""") + + _decl(locals(), "__enter__", """__enter__() -> self.""") + + _decl(locals(), "close", + """close() -> None or (perhaps) an integer. Close the file. + +Sets data attribute .closed to True. A closed file cannot be used for +further I/O operations. close() may be called more than once without +error. Some kinds of file objects (for example, opened by popen()) +may return an exit status upon closing.""") + # NB. close() needs to use the stream lock to avoid double-closes or + # close-while-another-thread-uses-it. + + + _decl(locals(), "fileno", + '''fileno() -> integer "file descriptor". + +This is needed for lower-level file interfaces, such os.read().''') + + _decl(locals(), "flush", + """flush() -> None. Flush the internal I/O buffer.""") + + _decl(locals(), "isatty", + """isatty() -> true or false. True if the file is connected to a tty device.""") + + _decl(locals(), "next", + """next() -> the next line in the file, or raise StopIteration""") + + _decl(locals(), "read", + """read([size]) -> read at most size bytes, returned as a string. + +If the size argument is negative or omitted, read until EOF is reached. +Notice that when in non-blocking mode, less data than what was requested +may be returned, even if no size parameter was given.""") + + _decl(locals(), "readline", + """readline([size]) -> next line from the file, as a string. + +Retain newline. A non-negative size argument limits the maximum +number of bytes to return (an incomplete line may be returned then). +Return an empty string at EOF.""") + + _decl(locals(), "readlines", + """readlines([size]) -> list of strings, each a line from the file. + +Call readline() repeatedly and return a list of the lines so read. +The optional size argument, if given, is an approximate bound on the +total number of bytes in the lines returned.""", + wrapresult = "wrap_list_of_str(space, result)") + + _decl(locals(), "seek", + """seek(offset[, whence]) -> None. Move to new file position. + +Argument offset is a byte count. Optional argument whence defaults to +0 (offset from start of file, offset should be >= 0); other values are 1 +(move relative to current position, positive or negative), and 2 (move +relative to end of file, usually negative, although many platforms allow +seeking beyond the end of a file). If the file is opened in text mode, +only offsets returned by tell() are legal. Use of other offsets causes +undefined behavior. +Note that not all file objects are seekable.""") + + _decl(locals(), "tell", + "tell() -> current file position, an integer (may be a long integer).") + + _decl(locals(), "truncate", + """truncate([size]) -> None. Truncate the file to at most size bytes. + +Size defaults to the current file position, as returned by tell().""") + + _decl(locals(), "write", + """write(str) -> None. Write string str to file. + +Note that due to buffering, flush() or close() may be needed before +the file on disk reflects the data written.""") + + _decl(locals(), "__iter__", + """Iterating over files, as in 'for line in f:', returns each line of +the file one by one.""") + + _decl(locals(), "xreadlines", + """xreadlines() -> returns self. + +For backward compatibility. File objects now include the performance +optimizations previously implemented in the xreadlines module.""") + + def file__repr__(self): + if self.stream is None: + head = "closed" + else: + head = "open" + info = "%s file %s, mode '%s'" % ( + head, + self.getdisplayname(), + self.mode) + return self.getrepr(self.space, info) + + def getdisplayname(self): + space = self.space + w_name = self.w_name + if w_name is None: + return '?' + else: + return space.str_w(space.repr(w_name)) + + def file_writelines(self, w_lines): + """writelines(sequence_of_strings) -> None. Write the strings to the file. + +Note that newlines are not added. The sequence can be any iterable object +producing strings. This is equivalent to calling write() for each string.""" + + space = self.space + self.check_closed() + self.check_writable() + lines = space.fixedview(w_lines) + for i, w_line in enumerate(lines): + if not space.isinstance_w(w_line, space.w_str): + try: + if self.binary: + line = w_line.readbuf_w(space).as_str() + else: + line = w_line.charbuf_w(space) + except TypeError: + raise OperationError(space.w_TypeError, space.wrap( + "writelines() argument must be a sequence of strings")) + else: + lines[i] = space.wrap(line) + for w_line in lines: + self.file_write(w_line) + + def file_readinto(self, w_rwbuffer): + """readinto() -> Undocumented. Don't use this; it may go away.""" + # XXX not the most efficient solution as it doesn't avoid the copying + space = self.space + rwbuffer = space.writebuf_w(w_rwbuffer) + w_data = self.file_read(rwbuffer.getlength()) + data = space.str_w(w_data) + rwbuffer.setslice(0, data) + return space.wrap(len(data)) + + +# ____________________________________________________________ + + +def descr_file__new__(space, w_subtype, __args__): + file = space.allocate_instance(W_File, w_subtype) + W_File.__init__(file, space) + return space.wrap(file) + + at unwrap_spec(fd=int, mode=str, buffering=int) +def descr_file_fdopen(space, w_subtype, fd, mode='r', buffering=-1): + file = space.allocate_instance(W_File, w_subtype) + W_File.__init__(file, space) + file.file_fdopen(fd, mode, buffering) + return space.wrap(file) + +def descr_file_closed(space, file): + return space.wrap(file.stream is None) + +def descr_file_newlines(space, file): + if file.stream: + newlines = file.stream.getnewlines() + else: + newlines = file.newlines + if newlines == 0: + return space.w_None + elif newlines == 1: + return space.wrap("\r") + elif newlines == 2: + return space.wrap("\n") + elif newlines == 4: + return space.wrap("\r\n") + result = [] + if newlines & 1: + result.append(space.wrap('\r')) + if newlines & 2: + result.append(space.wrap('\n')) + if newlines & 4: + result.append(space.wrap('\r\n')) + return space.newtuple(result[:]) + +def descr_file_softspace(space, file): + return space.wrap(file.softspace) + +def descr_file_setsoftspace(space, file, w_newvalue): + file.softspace = space.int_w(w_newvalue) + +# ____________________________________________________________ + +W_File.typedef = TypeDef( + "file", + __doc__ = """file(name[, mode[, buffering]]) -> file object + +Open a file. The mode can be 'r', 'w' or 'a' for reading (default), +writing or appending. The file will be created if it doesn't exist +when opened for writing or appending; it will be truncated when +opened for writing. Add a 'b' to the mode for binary files. +Add a '+' to the mode to allow simultaneous reading and writing. +If the buffering argument is given, 0 means unbuffered, 1 means line +buffered, and larger numbers specify the buffer size. +Add a 'U' to mode to open the file for input with universal newline +support. Any line ending in the input file will be seen as a '\n' +in Python. Also, a file so opened gains the attribute 'newlines'; +the value for this attribute is one of None (no newline read yet), +'\r', '\n', '\r\n' or a tuple containing all the newline types seen. + +Note: open() is an alias for file(). +""", + __new__ = interp2app(descr_file__new__), + fdopen = interp2app(descr_file_fdopen, as_classmethod=True), + name = interp_attrproperty_w('w_name', cls=W_File, doc="file name"), + mode = interp_attrproperty('mode', cls=W_File, + doc = "file mode ('r', 'U', 'w', 'a', " + "possibly with 'b' or '+' added)"), + encoding = interp_attrproperty('encoding', cls=W_File), + errors = interp_attrproperty('errors', cls=W_File), + closed = GetSetProperty(descr_file_closed, cls=W_File, + doc="True if the file is closed"), + newlines = GetSetProperty(descr_file_newlines, cls=W_File, + doc="end-of-line convention used in this file"), + softspace= GetSetProperty(descr_file_softspace, + descr_file_setsoftspace, + cls=W_File, + doc="Support for 'print'."), + __repr__ = interp2app(W_File.file__repr__), + readinto = interp2app(W_File.file_readinto), + writelines = interp2app(W_File.file_writelines), + __exit__ = interp2app(W_File.file__exit__), + __weakref__ = make_weakref_descr(W_File), + **dict([(name, interp2app(getattr(W_File, 'file_' + name))) + for name in W_File._exposed_method_names]) + ) + +# ____________________________________________________________ + +def wrap_list_of_str(space, lst): + return space.newlist([space.wrap(s) for s in lst]) + +class FileState: + def __init__(self, space): + self.openstreams = {} + +def getopenstreams(space): + return space.fromcache(FileState).openstreams + + at specialize.memo() +def signal_checker(space): + def checksignals(): + space.getexecutioncontext().checksignals() + return checksignals + +MAYBE_EAGAIN = getattr(errno, 'EAGAIN', None) +MAYBE_EWOULDBLOCK = getattr(errno, 'EWOULDBLOCK', None) + +def is_wouldblock_error(e): + if MAYBE_EAGAIN is not None and e.errno == MAYBE_EAGAIN: + return True + if MAYBE_EWOULDBLOCK is not None and e.errno == MAYBE_EWOULDBLOCK: + return True + return False + + + at unwrap_spec(w_file=W_File, encoding="str_or_None", errors="str_or_None") +def set_file_encoding(space, w_file, encoding=None, errors=None): + w_file.encoding = encoding + w_file.errors = errors From noreply at buildbot.pypy.org Wed Sep 10 03:08:46 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Wed, 10 Sep 2014 03:08:46 +0200 (CEST) Subject: [pypy-commit] pypy default: fix translation Message-ID: <20140910010846.D199E1D386F@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r73407:615fa520e47e Date: 2014-09-09 20:56 -0400 http://bitbucket.org/pypy/pypy/changeset/615fa520e47e/ Log: fix translation diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py --- a/rpython/rlib/rfile.py +++ b/rpython/rlib/rfile.py @@ -220,6 +220,9 @@ do_close = self._close2[1] do_close(ll_file) # return value ignored + def _cleanup_(self): + self._ll_file = lltype.nullptr(FILEP.TO) + def close(self): """Closes the described file. From noreply at buildbot.pypy.org Wed Sep 10 03:08:48 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Wed, 10 Sep 2014 03:08:48 +0200 (CEST) Subject: [pypy-commit] pypy default: support rfile.isatty() Message-ID: <20140910010848.0FCE01D386F@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r73408:6647fa7b2e68 Date: 2014-09-09 21:00 -0400 http://bitbucket.org/pypy/pypy/changeset/6647fa7b2e68/ Log: support rfile.isatty() diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py --- a/rpython/rlib/rfile.py +++ b/rpython/rlib/rfile.py @@ -484,3 +484,7 @@ def fileno(self): self._check_closed() return intmask(c_fileno(self._ll_file)) + + def isatty(self): + self._check_closed() + return os.isatty(c_fileno(self._ll_file)) diff --git a/rpython/rlib/test/test_rfile.py b/rpython/rlib/test/test_rfile.py --- a/rpython/rlib/test/test_rfile.py +++ b/rpython/rlib/test/test_rfile.py @@ -279,6 +279,7 @@ def f(): f = open(fname, "w") + assert not f.isatty() try: return f.fileno() finally: From noreply at buildbot.pypy.org Wed Sep 10 03:08:49 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Wed, 10 Sep 2014 03:08:49 +0200 (CEST) Subject: [pypy-commit] pypy use-file-star-for-file: merge default Message-ID: <20140910010849.632511D386F@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: use-file-star-for-file Changeset: r73409:fc2acceb8c60 Date: 2014-09-09 21:00 -0400 http://bitbucket.org/pypy/pypy/changeset/fc2acceb8c60/ Log: merge default diff --git a/pypy/doc/release-2.4.0.rst b/pypy/doc/release-2.4.0.rst --- a/pypy/doc/release-2.4.0.rst +++ b/pypy/doc/release-2.4.0.rst @@ -7,10 +7,6 @@ You can already download the PyPy 2.4-beta1 pre-release here: - https://bitbucket.org/pypy/pypy/downloads - -It should also soon be available on: - http://pypy.org/download.html We would like to thank our donors for the continued support of the PyPy @@ -87,7 +83,7 @@ this mostly affects errno handling on linux, which makes external calls faster. -* Move to a mixed polling and mutex GIL model that make mutli-threaded jitted +* Move to a mixed polling and mutex GIL model that make mutlithreaded jitted code run *much* faster * Optimize errno handling in linux (x86 and x86-64 only) @@ -111,7 +107,7 @@ .. _resolved: https://bitbucket.org/pypy/pypy/issues?status=resolved We have further improvements on the way: rpython file handling, -numpy linalg compatability, as well +numpy linalg compatibility, as well as improved GC and many smaller improvements. Please try it out and let us know what you think. We especially welcome diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py --- a/rpython/rlib/rfile.py +++ b/rpython/rlib/rfile.py @@ -220,6 +220,9 @@ do_close = self._close2[1] do_close(ll_file) # return value ignored + def _cleanup_(self): + self._ll_file = lltype.nullptr(FILEP.TO) + def close(self): """Closes the described file. @@ -481,3 +484,7 @@ def fileno(self): self._check_closed() return intmask(c_fileno(self._ll_file)) + + def isatty(self): + self._check_closed() + return os.isatty(c_fileno(self._ll_file)) diff --git a/rpython/rlib/test/test_rfile.py b/rpython/rlib/test/test_rfile.py --- a/rpython/rlib/test/test_rfile.py +++ b/rpython/rlib/test/test_rfile.py @@ -279,6 +279,7 @@ def f(): f = open(fname, "w") + assert not f.isatty() try: return f.fileno() finally: diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -100,8 +100,8 @@ cc = 'cl.exe' link = 'link.exe' - cflags = ('/MD', '/O2') - link_flags = () + cflags = ('/MD', '/O2', '/Zi') + link_flags = ('/debug',) standalone_only = () shared_only = () environ = None @@ -143,7 +143,6 @@ # Install debug options only when interpreter is in debug mode if sys.executable.lower().endswith('_d.exe'): self.cflags = ['/MDd', '/Z7', '/Od'] - self.link_flags = ['/debug'] # Increase stack size, for the linker and the stack check code. stack_size = 8 << 20 # 8 Mb From noreply at buildbot.pypy.org Wed Sep 10 03:08:50 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Wed, 10 Sep 2014 03:08:50 +0200 (CEST) Subject: [pypy-commit] pypy use-file-star-for-file: use rfile.isatty() Message-ID: <20140910010850.A27421D386F@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: use-file-star-for-file Changeset: r73410:4f4af93bcb92 Date: 2014-09-09 21:02 -0400 http://bitbucket.org/pypy/pypy/changeset/4f4af93bcb92/ Log: use rfile.isatty() diff --git a/pypy/module/_file/interp_file.py b/pypy/module/_file/interp_file.py --- a/pypy/module/_file/interp_file.py +++ b/pypy/module/_file/interp_file.py @@ -229,8 +229,7 @@ direct_xreadlines = direct___iter__ def direct_isatty(self): - stream = self.getstream() # check if the file is still open - return os.isatty(stream.fileno()) + return self.getstream().isatty() # ____________________________________________________________ # From noreply at buildbot.pypy.org Wed Sep 10 03:12:01 2014 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 10 Sep 2014 03:12:01 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: Fix the *_clear variants by calling raw_memclear and use it from the JIT Message-ID: <20140910011201.E82291D386F@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73411:5e81f151b049 Date: 2014-09-09 19:11 -0600 http://bitbucket.org/pypy/pypy/changeset/5e81f151b049/ Log: Fix the *_clear variants by calling raw_memclear and use it from the JIT (it's important for blackhole and tracing) diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py --- a/rpython/jit/backend/llsupport/gc.py +++ b/rpython/jit/backend/llsupport/gc.py @@ -423,8 +423,8 @@ self._random_usage_of_xmm_registers() type_id = rffi.cast(llgroup.HALFWORD, 0) # missing here return llop1.do_malloc_fixedsize(llmemory.GCREF, - type_id, size, - False, False, False) + type_id, size, + False, False, False) self.generate_function('malloc_nursery', malloc_nursery_slowpath, [lltype.Signed]) @@ -435,7 +435,7 @@ assert num_elem >= 0, 'num_elem should be >= 0' type_id = llop.extract_ushort(llgroup.HALFWORD, tid) check_typeid(type_id) - return llop1.do_malloc_varsize( + return llop1.do_malloc_varsize_clear( llmemory.GCREF, type_id, num_elem, self.standard_array_basesize, itemsize, self.standard_array_length_ofs) @@ -449,7 +449,7 @@ occur e.g. with arrays of floats on Win32.""" type_id = llop.extract_ushort(llgroup.HALFWORD, tid) check_typeid(type_id) - return llop1.do_malloc_varsize( + return llop1.do_malloc_varsize_clear( llmemory.GCREF, type_id, num_elem, basesize, itemsize, lengthofs) self.generate_function('malloc_array_nonstandard', @@ -491,7 +491,7 @@ self._random_usage_of_xmm_registers() type_id = llop.extract_ushort(llgroup.HALFWORD, tid) check_typeid(type_id) - return llop1.do_malloc_fixedsize(llmemory.GCREF, + return llop1.do_malloc_fixedsize_clear(llmemory.GCREF, type_id, size, False, False, False) self.generate_function('malloc_big_fixedsize', malloc_big_fixedsize, @@ -502,7 +502,7 @@ llop1 = self.llop1 type_id = llop.extract_ushort(llgroup.HALFWORD, sizedescr.tid) check_typeid(type_id) - return llop1.do_malloc_fixedsize(llmemory.GCREF, + return llop1.do_malloc_fixedsize_clear(llmemory.GCREF, type_id, sizedescr.size, False, False, False) @@ -511,7 +511,7 @@ llop1 = self.llop1 type_id = llop.extract_ushort(llgroup.HALFWORD, arraydescr.tid) check_typeid(type_id) - return llop1.do_malloc_varsize(llmemory.GCREF, + return llop1.do_malloc_varsize_clear(llmemory.GCREF, type_id, num_elem, arraydescr.basesize, arraydescr.itemsize, diff --git a/rpython/jit/codewriter/codewriter.py b/rpython/jit/codewriter/codewriter.py --- a/rpython/jit/codewriter/codewriter.py +++ b/rpython/jit/codewriter/codewriter.py @@ -13,7 +13,7 @@ class CodeWriter(object): callcontrol = None # for tests - debug = False + debug = True def __init__(self, cpu=None, jitdrivers_sd=[]): self.cpu = cpu diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py --- a/rpython/memory/gctransform/framework.py +++ b/rpython/memory/gctransform/framework.py @@ -835,6 +835,21 @@ resultvar=op.result) self.pop_roots(hop, livevars) + def gct_do_malloc_fixedsize_clear(self, hop): + # used by the JIT (see rpython.jit.backend.llsupport.gc) + self.gct_do_malloc_fixedsize(hop) + if not self.malloc_zero_filled: + op = hop.spaceop + v_size = op.args[1] + c_fixedofs = rmodel.inputconst(lltype.Signed, + rffi.sizeof(lltype.Signed)) + c_size = rmodel.inputconst(lltype.Signed, 1) + v_a = op.result + v_real_size = hop.genop('int_sub', [v_size, c_fixedofs], + resulttype=lltype.Signed) + self.emit_raw_memclear(hop.llops, v_real_size, c_size, + c_fixedofs, v_a) + def gct_do_malloc_varsize(self, hop): # used by the JIT (see rpython.jit.backend.llsupport.gc) op = hop.spaceop @@ -848,6 +863,17 @@ resultvar=op.result) self.pop_roots(hop, livevars) + def gct_do_malloc_varsize_clear(self, hop): + # used by the JIT (see rpython.jit.backend.llsupport.gc) + self.gct_do_malloc_fixedsize(hop) + if not self.malloc_zero_filled: + op = hop.spaceop + v_size = op.args[1] + c_fixedofs = op.args[2] + c_size = op.args[3] + v_a = op.result + self.emit_raw_memclear(hop.llops, v_size, c_size, c_fixedofs, v_a) + def gct_get_write_barrier_failing_case(self, hop): op = hop.spaceop hop.genop("same_as", @@ -1224,53 +1250,21 @@ v_size = llops.genop('getarraysize', [v], resulttype=lltype.Signed) c_size = rmodel.inputconst(lltype.Signed, llmemory.sizeof(ITEM)) - v_totalsize = llops.genop('int_mul', [v_size, c_size], - resulttype=lltype.Signed) v_a = llops.genop('cast_ptr_to_adr', [v], resulttype=llmemory.Address) c_fixedofs = rmodel.inputconst(lltype.Signed, llmemory.itemoffsetof(TYPE)) - v_adr = llops.genop('adr_add', [v_a, c_fixedofs], - resulttype=llmemory.Address) - llops.genop('raw_memclear', [v_adr, v_totalsize]) + self.emit_raw_memclear(llops, v_size, c_size, c_fixedofs, v_a) return else: - raise TypeError(TYPE) - xxxx - if previous_steps is None: - previous_steps = [] - assert (isinstance(TYPE, lltype.Struct) or isinstance(TYPE, lltype.Array)) - if isinstance(TYPE, lltype.Struct): - for name in TYPE._names: - c_name = rmodel.inputconst(lltype.Void, name) - FIELD = getattr(TYPE, name) - #handle ptr field in GcStruct - if isinstance(FIELD, lltype.Ptr) and FIELD._needsgc(): - c_null = rmodel.inputconst(FIELD, lltype.nullptr(FIELD.TO)) - if not previous_steps: - llops.genop('bare_setfield', [v, c_name, c_null]) - else: - llops.genop('bare_setinteriorfield', - [v] + previous_steps + [c_name, c_null]) - #handle inside GcStruct field - elif isinstance(FIELD, lltype.Struct): - gen_zero_gc_pointers(FIELD, v, llops, previous_steps + [c_name]) - #handle inside GcArray field - elif isinstance(FIELD, lltype.Array): - gen_zero_gc_pointers(FIELD, v, llops, previous_steps + [c_name]) - if isinstance(TYPE, lltype.Array): - ITEM = TYPE.OF - if previous_steps: - v = llop.genop('getinteriorfield',[v]+previous_steps) - arr_size = llops.genop('getarraysize',[v]) - for i in range(arr_size): - #handle an array of GcPtr - if isinstance(ITEM, lltype.Ptr) and ITEM._needsgc(): - c_null = rmodel.inputconst(ITEM, lltype.nullptr(ITEM.TO)) - llops.genop('bare_setarrayitem',[v, i, c_null]) - if isinstance(ITEM, lltype.Struct) or isinstance(ITEM, lltype.GcArray): - array_item = llops.genop('getarrayitem',[v,i]) - gen_zero_gc_pointers(FIELD, array_item, llops, previous_steps) + raise TypeError(TYPE) + + def emit_raw_memclear(self, llops, v_size, c_size, c_fixedofs, v_a): + v_totalsize = llops.genop('int_mul', [v_size, c_size], + resulttype=lltype.Signed) + v_adr = llops.genop('adr_add', [v_a, c_fixedofs], + resulttype=llmemory.Address) + llops.genop('raw_memclear', [v_adr, v_totalsize]) class TransformerLayoutBuilder(gctypelayout.TypeLayoutBuilder): diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py --- a/rpython/rtyper/lltypesystem/lloperation.py +++ b/rpython/rtyper/lltypesystem/lloperation.py @@ -450,7 +450,9 @@ 'get_exception_addr': LLOp(), 'get_exc_value_addr': LLOp(), 'do_malloc_fixedsize':LLOp(canmallocgc=True), + 'do_malloc_fixedsize_clear': LLOp(canmallocgc=True), 'do_malloc_varsize': LLOp(canmallocgc=True), + 'do_malloc_varsize_clear': LLOp(canmallocgc=True), 'get_write_barrier_failing_case': LLOp(sideeffects=False), 'get_write_barrier_from_array_failing_case': LLOp(sideeffects=False), 'gc_get_type_info_group': LLOp(sideeffects=False), From noreply at buildbot.pypy.org Wed Sep 10 04:06:02 2014 From: noreply at buildbot.pypy.org (rlamy) Date: Wed, 10 Sep 2014 04:06:02 +0200 (CEST) Subject: [pypy-commit] pypy default: Tell the annotator that RFile.read cannot return None Message-ID: <20140910020602.97F251C02BF@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: Changeset: r73412:2268afdc52e2 Date: 2014-09-10 03:04 +0100 http://bitbucket.org/pypy/pypy/changeset/2268afdc52e2/ Log: Tell the annotator that RFile.read cannot return None Since the return value of fread is checked, buf cannot be NULL and s is therefore really a string. diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py --- a/rpython/rlib/rfile.py +++ b/rpython/rlib/rfile.py @@ -323,6 +323,7 @@ if not c_feof(ll_file): raise _error(ll_file) s = buf.str(returned_size) + assert s is not None return s def _readline1(self, raw_buf): From noreply at buildbot.pypy.org Wed Sep 10 04:08:14 2014 From: noreply at buildbot.pypy.org (rlamy) Date: Wed, 10 Sep 2014 04:08:14 +0200 (CEST) Subject: [pypy-commit] pypy default: Add enforceargs to rpython.rlib.rpath functions Message-ID: <20140910020814.702911C02BF@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: Changeset: r73413:5b5351812a83 Date: 2014-09-10 03:07 +0100 http://bitbucket.org/pypy/pypy/changeset/5b5351812a83/ Log: Add enforceargs to rpython.rlib.rpath functions diff --git a/rpython/rlib/rpath.py b/rpython/rlib/rpath.py --- a/rpython/rlib/rpath.py +++ b/rpython/rlib/rpath.py @@ -4,6 +4,7 @@ import os, stat from rpython.rlib import rposix +from rpython.rlib.objectmodel import enforceargs # ____________________________________________________________ @@ -11,6 +12,7 @@ # Generic implementations in RPython for both POSIX and NT # + at enforceargs(str) def risdir(s): """Return true if the pathname refers to an existing directory.""" try: @@ -25,10 +27,12 @@ # POSIX-only implementations # + at enforceargs(str) def _posix_risabs(s): """Test whether a path is absolute""" return s.startswith('/') + at enforceargs(str) def _posix_rnormpath(path): """Normalize path, eliminating double slashes, etc.""" slash, dot = '/', '.' @@ -56,6 +60,7 @@ path = slash*initial_slashes + path return path or dot + at enforceargs(str) def _posix_rabspath(path): """Return an absolute, **non-normalized** path. **This version does not let exceptions propagate.**""" @@ -67,6 +72,7 @@ except OSError: return path + at enforceargs(str, str) def _posix_rjoin(a, b): """Join two pathname components, inserting '/' as needed. If the second component is an absolute path, the first one @@ -87,11 +93,13 @@ # NT-only implementations # + at enforceargs(str) def _nt_risabs(s): """Test whether a path is absolute""" s = _nt_rsplitdrive(s)[1] return s.startswith('/') or s.startswith('\\') + at enforceargs(str) def _nt_rnormpath(path): """Normalize path, eliminating double slashes, etc.""" backslash, dot = '\\', '.' @@ -142,6 +150,7 @@ comps.append(dot) return prefix + backslash.join(comps) + at enforceargs(str) def _nt_rabspath(path): try: if path == '': @@ -150,6 +159,7 @@ except OSError: return path + at enforceargs(str) def _nt_rsplitdrive(p): """Split a pathname into drive/UNC sharepoint and relative path specifiers. @@ -177,6 +187,7 @@ return p[:2], p[2:] return '', p + at enforceargs(str, str) def _nt_rjoin(path, p): """Join two or more pathname components, inserting "\\" as needed.""" result_drive, result_path = _nt_rsplitdrive(path) From noreply at buildbot.pypy.org Wed Sep 10 05:28:03 2014 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 10 Sep 2014 05:28:03 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: kill the current approach Message-ID: <20140910032803.091AC1D22E6@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73414:fdb0c7732a79 Date: 2014-09-09 19:44 -0600 http://bitbucket.org/pypy/pypy/changeset/fdb0c7732a79/ Log: kill the current approach diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py --- a/rpython/jit/backend/llgraph/runner.py +++ b/rpython/jit/backend/llgraph/runner.py @@ -643,20 +643,6 @@ array = lltype.malloc(arraydescr.A, length) return lltype.cast_opaque_ptr(llmemory.GCREF, array) - def bh_clear_array_contents(self, a, descr): - a = support.cast_arg(lltype.Ptr(descr.A), a) - ITEM = descr.A.OF - array = a._obj - if isinstance(ITEM, lltype.Struct): - for i in xrange(array.getlength()): - for name, FIELD in ITEM._flds.iteritems(): - null = FIELD._defl() - setattr(array.getitem(i), name, null) - else: - null = ITEM._defl() - for i in xrange(array.getlength()): - array.setitem(i, null) - def bh_classof(self, struct): struct = lltype.cast_opaque_ptr(rclass.OBJECTPTR, struct) result_adr = llmemory.cast_ptr_to_adr(struct.typeptr) diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py --- a/rpython/jit/backend/llsupport/llmodel.py +++ b/rpython/jit/backend/llsupport/llmodel.py @@ -591,14 +591,6 @@ def bh_new(self, sizedescr): return self.gc_ll_descr.gc_malloc(sizedescr) - def bh_clear_array_contents(self, ref, arraydescr): - ofs, size, _ = self.unpack_arraydescr_size(arraydescr) - arraysize = self.bh_arraylen_gc(ref, arraydescr) - totalsize = size * arraysize - adr = rffi.cast(lltype.Signed, ref) + ofs - self.gc_ll_descr.memset_ptr(adr, rffi.cast(rffi.INT, 0), - rffi.cast(rffi.SIZE_T, totalsize)) - def bh_new_with_vtable(self, vtable, sizedescr): res = self.gc_ll_descr.gc_malloc(sizedescr) if self.vtable_offset is not None: diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -59,10 +59,6 @@ if op.is_malloc(): self.handle_malloc_operation(op) continue - elif op.getopnum() == rop.CLEAR_ARRAY_CONTENTS: - if not self.gc_ll_descr.malloc_zero_filled: - self.handle_clear_array_contents(op) - continue elif op.can_malloc(): self.emitting_an_operation_that_can_collect() elif op.getopnum() == rop.LABEL: @@ -161,26 +157,6 @@ else: raise NotImplementedError(op.getopname()) - def handle_clear_array_contents(self, op): - # XXX this maybe should go to optimizer, so we can remove extra ops? - arraydescr = op.getdescr() - ofs, size, _ = self.cpu.unpack_arraydescr_size(arraydescr) - v_arr = op.getarg(0) - v_arr_plus_ofs = BoxInt() - v_arrsize = BoxInt() - v_totalsize = BoxInt() - gcdescr = self.gc_ll_descr - ops = [ - ResOperation(rop.INT_ADD, [v_arr, ConstInt(size)], v_arr_plus_ofs), - ResOperation(rop.ARRAYLEN_GC, [v_arr], v_arrsize, descr=arraydescr), - ResOperation(rop.INT_MUL, [v_arrsize, ConstInt(size)], v_totalsize), - ResOperation(rop.CALL, [ConstInt(gcdescr.memset_ptr_as_int), - v_arr_plus_ofs, - ConstInt(0), v_totalsize], None, - descr=gcdescr.memset_descr), - ] - self.newops.extend(ops) - def gen_malloc_frame(self, frame_info, frame, size_box): descrs = self.gc_ll_descr.getframedescrs(self.cpu) if self.gc_ll_descr.kind == 'boehm': diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py --- a/rpython/jit/backend/test/runner_test.py +++ b/rpython/jit/backend/test/runner_test.py @@ -4439,21 +4439,3 @@ res = self.execute_operation(rop.CAST_FLOAT_TO_SINGLEFLOAT, [boxfloat(12.5)], 'int') assert res.getint() == struct.unpack("I", struct.pack("f", 12.5))[0] - - def test_clear_array_contents(self): - from rpython.jit.backend.llsupport.llmodel import AbstractLLCPU - if not isinstance(self.cpu, AbstractLLCPU): - py.test.skip("pointless test on non-asm") - oldval = self.cpu.gc_ll_descr.malloc_zero_filled - self.cpu.gc_ll_descr.malloc_zero_filled = False - try: - A = lltype.GcArray(lltype.Signed) - a = lltype.malloc(A, 3) - a[1] = 13 - descr = self.cpu.arraydescrof(A) - ref = lltype.cast_opaque_ptr(llmemory.GCREF, a) - self.execute_operation(rop.CLEAR_ARRAY_CONTENTS, - [BoxPtr(ref)], 'void', descr=descr) - assert a[1] == 0 - finally: - self.cpu.gc_ll_descr.malloc_zero_filled = oldval diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -315,8 +315,7 @@ self.assembler.mc.mark_op(op) self.rm.position = i self.xrm.position = i - if (op.has_no_side_effect() and op.result not in self.longevity - and op.opnum != rop.CLEAR_ARRAY_CONTENTS): + if op.has_no_side_effect() and op.result not in self.longevity: i += 1 self.possibly_free_vars_for_op(op) continue diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py --- a/rpython/jit/codewriter/jtransform.py +++ b/rpython/jit/codewriter/jtransform.py @@ -612,39 +612,8 @@ # XXX only strings or simple arrays for now ARRAY = op.args[0].value arraydescr = self.cpu.arraydescrof(ARRAY) - op1 = SpaceOperation('new_array', [op.args[2], arraydescr], - op.result) - if self._has_gcptrs_in(ARRAY): - return self.zero_contents([op1], op.result, ARRAY, - only_gc_pointers=True) - if op.args[1].value.get('zero', False): - return self.zero_contents([op1], op.result, ARRAY) - return op1 - - def zero_contents(self, ops, v, TYPE, only_gc_pointers=False): - if isinstance(TYPE, lltype.Struct): - for name, FIELD in TYPE._flds.iteritems(): - if (not only_gc_pointers or - isinstance(FIELD, lltype.Ptr) and FIELD._needsgc()): - c_name = Constant(name, lltype.Void) - c_null = Constant(FIELD._defl(), FIELD) - op = SpaceOperation('setfield', [v, c_name, c_null], - None) - self.extend_with(ops, self.rewrite_op_setfield(op, - override_type=TYPE)) - elif isinstance(FIELD, lltype.Struct): - # substruct - self.zero_contents(ops, v, FIELD, - only_gc_pointers=only_gc_pointers) - elif isinstance(TYPE, lltype.Array): - arraydescr = self.cpu.arraydescrof(TYPE) - ops.append(SpaceOperation('clear_array_contents', - [v, arraydescr], None)) - else: - raise TypeError("Expected struct or array, got '%r'", (TYPE,)) - if len(ops) == 1: - return ops[0] - return ops + return SpaceOperation('new_array', [op.args[2], arraydescr], + op.result) def extend_with(self, l, ops): if ops is None: @@ -930,28 +899,7 @@ else: opname = 'new' sizedescr = self.cpu.sizeof(STRUCT) - op1 = SpaceOperation(opname, [sizedescr], op.result) - if true_zero: - return self.zero_contents([op1], op.result, STRUCT) - if self._has_gcptrs_in(STRUCT): - return self.zero_contents([op1], op.result, STRUCT, - only_gc_pointers=True) - return op1 - - def _has_gcptrs_in(self, STRUCT): - if isinstance(STRUCT, lltype.Array): - ITEM = STRUCT.OF - if isinstance(ITEM, lltype.Struct): - STRUCT = ITEM - else: - return isinstance(ITEM, lltype.Ptr) and ITEM._needsgc() - for FIELD in STRUCT._flds.values(): - if isinstance(FIELD, lltype.Ptr) and FIELD._needsgc(): - return True - elif isinstance(FIELD, lltype.Struct): - if self._has_gcptrs_in(FIELD): - return True - return False + return SpaceOperation(opname, [sizedescr], op.result) def rewrite_op_getinteriorarraysize(self, op): # only supports strings and unicodes @@ -1677,8 +1625,6 @@ v.concretetype = lltype.Signed ops.append(SpaceOperation('int_force_ge_zero', [v_length], v)) ops.append(SpaceOperation('new_array', [v, arraydescr], op.result)) - if self._has_gcptrs_in(op.result.concretetype.TO): - self.zero_contents(ops, op.result, op.result.concretetype.TO) return ops def do_fixed_list_len(self, op, args, arraydescr): diff --git a/rpython/jit/codewriter/test/test_jtransform.py b/rpython/jit/codewriter/test/test_jtransform.py --- a/rpython/jit/codewriter/test/test_jtransform.py +++ b/rpython/jit/codewriter/test/test_jtransform.py @@ -529,46 +529,6 @@ assert op1.opname == 'new' assert op1.args == [('sizedescr', S)] -def test_malloc_new_zero(): - SS = lltype.GcStruct('SS') - S = lltype.GcStruct('S', ('x', lltype.Ptr(SS))) - v = varoftype(lltype.Ptr(S)) - op = SpaceOperation('malloc', [Constant(S, lltype.Void), - Constant({'flavor': 'gc'}, lltype.Void)], v) - op1, op2 = Transformer(FakeCPU()).rewrite_operation(op) - assert op1.opname == 'new' - assert op1.args == [('sizedescr', S)] - assert op2.opname == 'setfield_gc_r' - assert op2.args[0] == v - -def test_malloc_new_zero_2(): - S = lltype.GcStruct('S', ('x', lltype.Signed)) - v = varoftype(lltype.Ptr(S)) - op = SpaceOperation('malloc', [Constant(S, lltype.Void), - Constant({'flavor': 'gc', - 'zero': True}, lltype.Void)], v) - op1, op2 = Transformer(FakeCPU()).rewrite_operation(op) - assert op1.opname == 'new' - assert op1.args == [('sizedescr', S)] - assert op2.opname == 'setfield_gc_i' - assert op2.args[0] == v - -def test_malloc_new_zero_nested(): - S0 = lltype.GcStruct('S0') - S = lltype.Struct('S', ('x', lltype.Ptr(S0))) - S2 = lltype.GcStruct('S2', ('parent', S), - ('xx', lltype.Ptr(S0))) - v = varoftype(lltype.Ptr(S2)) - op = SpaceOperation('malloc', [Constant(S2, lltype.Void), - Constant({'flavor': 'gc'}, lltype.Void)], v) - op1, op2, op3 = Transformer(FakeCPU()).rewrite_operation(op) - assert op1.opname == 'new' - assert op1.args == [('sizedescr', S2)] - assert op2.opname == 'setfield_gc_r' - assert op2.args[0] == v - assert op3.opname == 'setfield_gc_r' - assert op3.args[0] == v - def test_malloc_new_with_vtable(): vtable = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True) S = lltype.GcStruct('S', ('parent', rclass.OBJECT)) @@ -1066,20 +1026,6 @@ assert op1.args == [v1] assert op1.result == v2 -def test_malloc_varsize_zero(): - c_A = Constant(lltype.GcArray(lltype.Signed), lltype.Void) - c_flags = Constant({"flavor": "gc"}, lltype.Void) - v1 = varoftype(lltype.Signed) - v2 = varoftype(c_A.value) - op = SpaceOperation('malloc_varsize', [c_A, c_flags, v1], v2) - op1 = Transformer(FakeCPU()).rewrite_operation(op) - assert op1.opname == 'new_array' - c_flags = Constant({"flavor": "gc", "zero": True}, lltype.Void) - op = SpaceOperation('malloc_varsize', [c_A, c_flags, v1], v2) - op1, op2 = Transformer(FakeCPU()).rewrite_operation(op) - assert op1.opname == 'new_array' - assert op2.opname == 'clear_array_contents' - def test_str_concat(): # test that the oopspec is present and correctly transformed PSTR = lltype.Ptr(rstr.STR) diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py --- a/rpython/jit/metainterp/blackhole.py +++ b/rpython/jit/metainterp/blackhole.py @@ -1345,10 +1345,6 @@ vtable = heaptracker.descr2vtable(cpu, descr) return cpu.bh_new_with_vtable(vtable, descr) - @arguments("cpu", "r", "d") - def bhimpl_clear_array_contents(cpu, ref, descr): - cpu.bh_clear_array_contents(ref, descr) - @arguments("cpu", "r", returns="i") def bhimpl_guard_class(cpu, struct): return cpu.bh_classof(struct) diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py --- a/rpython/jit/metainterp/executor.py +++ b/rpython/jit/metainterp/executor.py @@ -335,7 +335,6 @@ rop.CALL_MALLOC_NURSERY, rop.CALL_MALLOC_NURSERY_VARSIZE, rop.CALL_MALLOC_NURSERY_VARSIZE_FRAME, - rop.CLEAR_ARRAY_CONTENTS, rop.LABEL, ): # list of opcodes never executed by pyjitpl continue diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py @@ -5461,17 +5461,6 @@ """ self.optimize_loop(ops, expected) - def test_virtual_clear_array_contents(self): - ops = """ - [] - p0 = new_array(2, descr=arraydescr) - clear_array_contents(p0, descr=arraydescr) - """ - expected = """ - [] - """ - self.optimize_loop(ops, expected) - class TestLLtype(BaseTestOptimizeBasic, LLtypeMixin): pass diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py --- a/rpython/jit/metainterp/optimizeopt/virtualize.py +++ b/rpython/jit/metainterp/optimizeopt/virtualize.py @@ -662,12 +662,6 @@ else: self.emit_operation(op) - def optimize_CLEAR_ARRAY_CONTENTS(self, op): - v = self.getvalue(op.getarg(0)) - if v.is_virtual(): - return - self.emit_operation(op) - def optimize_CALL(self, op): effectinfo = op.getdescr().get_extra_info() if effectinfo.oopspecindex == EffectInfo.OS_RAW_MALLOC_VARSIZE_CHAR: diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -396,10 +396,6 @@ def opimpl_new(self, sizedescr): return self.metainterp.execute_new(sizedescr) - @arguments("box", "descr") - def opimpl_clear_array_contents(self, box, descr): - self.metainterp.execute_and_record(rop.CLEAR_ARRAY_CONTENTS, descr, box) - @arguments("descr") def opimpl_new_with_vtable(self, sizedescr): cpu = self.metainterp.cpu diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py --- a/rpython/jit/metainterp/resoperation.py +++ b/rpython/jit/metainterp/resoperation.py @@ -492,9 +492,6 @@ 'MARK_OPAQUE_PTR/1b', # this one has no *visible* side effect, since the virtualizable # must be forced, however we need to execute it anyway - 'CLEAR_ARRAY_CONTENTS/1d', - # this one does not *really* have a side effect since it's equivalent - # to array just coming zeroed '_NOSIDEEFFECT_LAST', # ----- end of no_side_effect operations ----- 'INCREMENT_DEBUG_COUNTER/1', From noreply at buildbot.pypy.org Wed Sep 10 05:28:04 2014 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 10 Sep 2014 05:28:04 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: implement size descr storing offsets of gc pointers Message-ID: <20140910032804.3EC6D1D22E6@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73415:6dfd216b0e8d Date: 2014-09-09 21:27 -0600 http://bitbucket.org/pypy/pypy/changeset/6dfd216b0e8d/ Log: implement size descr storing offsets of gc pointers diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py --- a/rpython/jit/backend/llgraph/runner.py +++ b/rpython/jit/backend/llgraph/runner.py @@ -630,17 +630,17 @@ def bh_new(self, sizedescr): return lltype.cast_opaque_ptr(llmemory.GCREF, - lltype.malloc(sizedescr.S)) + lltype.malloc(sizedescr.S, zero=True)) def bh_new_with_vtable(self, vtable, descr): - result = lltype.malloc(descr.S) + result = lltype.malloc(descr.S, zero=True) result_as_objptr = lltype.cast_pointer(rclass.OBJECTPTR, result) result_as_objptr.typeptr = support.cast_from_int(rclass.CLASSTYPE, vtable) return lltype.cast_opaque_ptr(llmemory.GCREF, result) def bh_new_array(self, length, arraydescr): - array = lltype.malloc(arraydescr.A, length) + array = lltype.malloc(arraydescr.A, length, zero=True) return lltype.cast_opaque_ptr(llmemory.GCREF, array) def bh_classof(self, struct): diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py --- a/rpython/jit/backend/llsupport/descr.py +++ b/rpython/jit/backend/llsupport/descr.py @@ -35,9 +35,11 @@ size = 0 # help translation tid = llop.combine_ushort(lltype.Signed, 0, 0) - def __init__(self, size, count_fields_if_immut=-1): + def __init__(self, size, count_fields_if_immut=-1, + offsets_of_gcptrs=None): self.size = size self.count_fields_if_immut = count_fields_if_immut + self.offsets_of_gcptrs = offsets_of_gcptrs def count_fields_if_immutable(self): return self.count_fields_if_immut @@ -58,10 +60,13 @@ except KeyError: size = symbolic.get_size(STRUCT, gccache.translate_support_code) count_fields_if_immut = heaptracker.count_fields_if_immutable(STRUCT) + offsets_of_gcptrs = heaptracker.offsets_of_gcptrs(gccache, STRUCT) if heaptracker.has_gcstruct_a_vtable(STRUCT): - sizedescr = SizeDescrWithVTable(size, count_fields_if_immut) + sizedescr = SizeDescrWithVTable(size, count_fields_if_immut, + offsets_of_gcptrs) else: - sizedescr = SizeDescr(size, count_fields_if_immut) + sizedescr = SizeDescr(size, count_fields_if_immut, + offsets_of_gcptrs) gccache.init_size_descr(STRUCT, sizedescr) cache[STRUCT] = sizedescr return sizedescr diff --git a/rpython/jit/backend/llsupport/test/test_descr.py b/rpython/jit/backend/llsupport/test/test_descr.py --- a/rpython/jit/backend/llsupport/test/test_descr.py +++ b/rpython/jit/backend/llsupport/test/test_descr.py @@ -15,6 +15,9 @@ ('y', lltype.Ptr(T))) descr_s = get_size_descr(c0, S) descr_t = get_size_descr(c0, T) + assert descr_t.offsets_of_gcptrs == [] + assert descr_s.offsets_of_gcptrs == [ + symbolic.get_field_token(S, "y", False)[0]] assert descr_s.size == symbolic.get_size(S, False) assert descr_t.size == symbolic.get_size(T, False) assert descr_s.count_fields_if_immutable() == -1 diff --git a/rpython/jit/codewriter/heaptracker.py b/rpython/jit/codewriter/heaptracker.py --- a/rpython/jit/codewriter/heaptracker.py +++ b/rpython/jit/codewriter/heaptracker.py @@ -1,5 +1,6 @@ from rpython.rtyper.lltypesystem import lltype, llmemory, rclass from rpython.rlib.objectmodel import we_are_translated +from rpython.jit.backend.llsupport import symbolic def adr2int(addr): @@ -125,3 +126,16 @@ vtable = descr.as_vtable_size_descr()._corresponding_vtable vtable = llmemory.cast_ptr_to_adr(vtable) return adr2int(vtable) + +def offsets_of_gcptrs(gccache, STRUCT, res=None): + if res is None: + res = [] + for name, FIELD in STRUCT._flds.iteritems(): + if isinstance(FIELD, lltype.Ptr) and FIELD._needsgc(): + + offset, _ = symbolic.get_field_token(STRUCT, name, + gccache.translate_support_code) + res.append(offset) + elif isinstance(FIELD, lltype.Struct): + offsets_of_gcptrs(gccache, FIELD, res) + return res From noreply at buildbot.pypy.org Wed Sep 10 11:35:53 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Wed, 10 Sep 2014 11:35:53 +0200 (CEST) Subject: [pypy-commit] stmgc default: adapt test_random.py and fix overwriting up-to-date objs during abort Message-ID: <20140910093553.05C841D38C8@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1380:a1c63ae8d467 Date: 2014-09-10 11:37 +0200 http://bitbucket.org/pypy/stmgc/changeset/a1c63ae8d467/ Log: adapt test_random.py and fix overwriting up-to-date objs during abort diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -89,10 +89,11 @@ if ((uintptr_t)cl == -1) { /* there is an inevitable transaction running */ #if STM_TESTS + free(free_if_abort); stm_abort_transaction(); #endif cl = prev_cl; - usleep(1); /* XXX */ + _stm_collectable_safe_point(); continue; } prev_cl = cl; @@ -104,8 +105,23 @@ while ((obj = cl->written[i])) { _update_obj_from(cl->segment_num, obj); - if (!needs_abort && _stm_was_read(obj)) { + if (_stm_was_read(obj)) { needs_abort = true; + + /* if we wrote this obj, we need to free its backup and + remove it from modified_old_objects because + we would otherwise overwrite the updated obj on abort */ + acquire_modified_objs_lock(STM_SEGMENT->segment_num); + wlog_t *item; + struct tree_s *tree = STM_PSEGMENT->modified_old_objects; + TREE_FIND(tree, (uintptr_t)obj, item, goto not_found); + + free((void*)item->val); + TREE_FIND_DELETE(tree, item); + + not_found: + /* nothing todo */ + release_modified_objs_lock(STM_SEGMENT->segment_num); } i++; @@ -453,7 +469,6 @@ size_t obj_size; obj_size = stmcb_size_rounded_up(bk_obj); - assert(obj_size < 4096); /* XXX */ memcpy(REAL_ADDRESS(pseg->pub.segment_base, obj), bk_obj, obj_size); diff --git a/c8/stm/list.h b/c8/stm/list.h --- a/c8/stm/list.h +++ b/c8/stm/list.h @@ -200,6 +200,8 @@ #define TREE_LOOP_END_AND_COMPRESS \ } if (_deleted_factor > 9) _tree_compress(_tree); } #define TREE_LOOP_DELETE(tree, item) { (tree)->count--; (item)->addr = NULL; _deleted_factor += 6; } +#define TREE_FIND_DELETE(tree, item) { (tree)->count--; (item)->addr = NULL; } + #define TREE_FIND(tree, addr1, result, goto_not_found) \ { \ diff --git a/c8/test/support.py b/c8/test/support.py --- a/c8/test/support.py +++ b/c8/test/support.py @@ -63,7 +63,7 @@ void _stm_test_switch_segment(int segnum); void clear_jmpbuf(stm_thread_local_t *tl); -long stm_start_transaction(stm_thread_local_t *tl); +long _check_start_transaction(stm_thread_local_t *tl); bool _check_commit_transaction(void); bool _check_abort_transaction(void); bool _check_become_inevitable(stm_thread_local_t *tl); @@ -155,6 +155,17 @@ CHECKED(stm_commit_transaction()); } +long _check_start_transaction(stm_thread_local_t *tl) { + void **jmpbuf = tl->rjthread.jmpbuf; \ + if (__builtin_setjmp(jmpbuf) == 0) { /* returned directly */\ + stm_start_transaction(tl); \ + clear_jmpbuf(tl); \ + return 0; \ + } \ + clear_jmpbuf(tl); \ + return 1; +} + bool _check_stop_safe_point(void) { CHECKED(_stm_stop_safe_point()); } @@ -474,8 +485,8 @@ def start_transaction(self): tl = self.tls[self.current_thread] assert not lib._stm_in_transaction(tl) - res = lib.stm_start_transaction(tl) - assert res == 0 + if lib._check_start_transaction(tl): + raise Conflict() lib.clear_jmpbuf(tl) assert lib._stm_in_transaction(tl) # diff --git a/c8/test/test_random.py b/c8/test/test_random.py new file mode 100644 --- /dev/null +++ b/c8/test/test_random.py @@ -0,0 +1,617 @@ +from support import * +import sys, random +import py + + + +class Exec(object): + def __init__(self, test): + self.content = {'self': test} + self.thread_num = 0 + + def do(self, cmd): + color = "\033[%dm" % (31 + self.thread_num % 6) + print >> sys.stderr, color + cmd + "\033[0m" + exec cmd in globals(), self.content + + +def raising_call(conflict, func, *args): + arguments = ", ".join(map(str, args)) + if conflict: + return "py.test.raises(Conflict, %s, %s)" % (func, arguments) + return "%s(%s)" % (func, arguments) + + +class WriteWriteConflictNotTestable(Exception): + # How can I test a write-write conflict between + # an inevitable and a normal transaction? The + # inevitable transaction would have to wait, + # but now for tests we simply abort. Of course + # aborting the inevitable transaction is not possible.. + pass + +def contention_management(our_trs, other_trs, wait=False, objs_in_conflict=None): + """exact copy of logic in contention.c""" + if our_trs.inevitable and wait: + # we win but cannot wait in tests... + raise WriteWriteConflictNotTestable + + + if our_trs.start_time >= other_trs.start_time: + abort_other = False + else: + abort_other = True + + if other_trs.check_must_abort(): + abort_other = True + elif our_trs.inevitable: + abort_other = True + elif other_trs.inevitable: + abort_other = False + + if not abort_other: + our_trs.set_must_abort(objs_in_conflict) + else: + other_trs.set_must_abort(objs_in_conflict) + + +class TransactionState(object): + """State of a transaction running in a thread, + e.g. maintains read/write sets. The state will be + discarded on abort or pushed to other threads""" + + def __init__(self, start_time, thread_num=None): + self.read_set = set() + self.write_set = set() + self.values = {} + self._must_abort = False + self.start_time = start_time + self.objs_in_conflict = set() + self.inevitable = False + self.created_in_this_transaction = set() + self.thread_num = thread_num + + def get_old_modified(self): + # returns only the ones that are modified and not from + # this transaction + return self.write_set.difference(self.created_in_this_transaction) + + def set_must_abort(self, objs_in_conflict=None): + assert not self.inevitable + if objs_in_conflict is not None: + self.objs_in_conflict |= objs_in_conflict + self._must_abort = True + color = "\033[%dm" % (31 + self.thread_num % 6) + print >> sys.stderr, color + "# must abort: %r\033[0m" % (objs_in_conflict,) + + def check_must_abort(self): + return self._must_abort + + def has_conflict_with(self, committed): + return bool(self.read_set & committed.write_set) + + def update_from_committed(self, committed, only_new=False): + """returns True if conflict""" + if only_new: + for w in committed.write_set: + self.values[w] = committed.values[w] + for w in committed.created_in_this_transaction: + self.values[w] = committed.values[w] + else: + self.values.update(committed.values) + + if self.has_conflict_with(committed): + # we are too late + self.set_must_abort(objs_in_conflict=self.read_set & committed.write_set) + return self.check_must_abort() + + def read_root(self, r): + self.read_set.add(r) + return self.values[r] + + def add_root(self, r, v, created_in_this_transaction): + assert self.values.get(r, None) is None + self.values[r] = v + if created_in_this_transaction: + self.created_in_this_transaction.add(r) + + def write_root(self, r, v): + if r not in self.created_in_this_transaction: + self.read_set.add(r) + self.write_set.add(r) + old = self.values.get(r, None) + self.values[r] = v + return old + + +class ThreadState(object): + """Maintains state for one thread. Mostly manages things + to be kept between transactions (e.g. saved roots) and + handles discarding/reseting states on transaction abort""" + + def __init__(self, num, global_state): + self.num = num + self.saved_roots = [] + self.roots_on_stack = 0 + self.roots_on_transaction_start = 0 + self.transaction_state = None + self.global_state = global_state + + def register_root(self, r): + self.saved_roots.append(r) + assert len(self.saved_roots) < SHADOWSTACK_LENGTH + + def forget_random_root(self): + if self.transaction_state.inevitable: + # forget *all* roots + self.roots_on_stack = 0 + self.roots_on_transaction_start = 0 + res = str(self.saved_roots) + del self.saved_roots[:] + else: + # forget all non-pushed roots for now + assert self.roots_on_stack == self.roots_on_transaction_start + res = str(self.saved_roots[self.roots_on_stack:]) + del self.saved_roots[self.roots_on_stack:] + return res + + def get_random_root(self): + rnd = self.global_state.rnd + if self.saved_roots: + return rnd.choice([rnd.choice(self.global_state.prebuilt_roots), + rnd.choice(self.saved_roots)]) + return rnd.choice(self.global_state.prebuilt_roots) + + def push_roots(self, ex): + assert self.roots_on_stack == self.roots_on_transaction_start + for r in self.saved_roots[self.roots_on_transaction_start:]: + ex.do('self.push_root(%s)' % r) + self.roots_on_stack += 1 + + def pop_roots(self, ex): + for r in reversed(self.saved_roots[self.roots_on_transaction_start:]): + ex.do('%s = self.pop_root()' % r) + ex.do('# 0x%x, size %d' % ( + int(ffi.cast("uintptr_t", ex.content[r])), + stm_get_obj_size(ex.content[r]))) + self.roots_on_stack -= 1 + assert self.roots_on_stack == self.roots_on_transaction_start + + def reload_roots(self, ex): + assert self.roots_on_stack == self.roots_on_transaction_start + to_reload = self.saved_roots[:self.roots_on_stack] + if to_reload: + ex.do("# reload roots on stack:") + for r in reversed(to_reload): + ex.do('%s = self.pop_root()' % r) + for r in to_reload: + ex.do('self.push_root(%s) # 0x%x, size %d' % ( + r, int(ffi.cast("uintptr_t", ex.content[r])), + stm_get_obj_size(ex.content[r]))) + + def start_transaction(self, thread_num): + assert self.transaction_state is None + if self.global_state.is_inevitable_transaction_running(): + return False + start_time = self.global_state.inc_and_get_global_time() + trs = TransactionState(start_time, thread_num) + trs.update_from_committed( + self.global_state.committed_transaction_state) + self.transaction_state = trs + self.roots_on_transaction_start = self.roots_on_stack + return True + + def commit_transaction(self): + trs = self.transaction_state + gtrs = self.global_state.committed_transaction_state + # + # check for inevitable transaction, must_abort otherwise + self.global_state.check_if_can_become_inevitable(trs) + if not trs.check_must_abort(): + # abort all others in conflict + self.global_state.check_for_write_read_conflicts(trs) + conflicts = trs.check_must_abort() + if not conflicts: + # update global committed state w/o conflict + assert not gtrs.update_from_committed(trs) + self.global_state.push_state_to_other_threads(trs) + self.transaction_state = None + return conflicts + + def abort_transaction(self): + assert self.transaction_state.check_must_abort() + assert not self.transaction_state.inevitable + self.roots_on_stack = self.roots_on_transaction_start + del self.saved_roots[self.roots_on_stack:] + self.transaction_state = None + + +class GlobalState(object): + """Maintains the global view (in a TransactionState) on + objects and threads. It also handles checking for conflicts + between threads and pushing state to other threads""" + + def __init__(self, ex, rnd): + self.ex = ex + self.rnd = rnd + self.thread_states = [] + self.prebuilt_roots = [] + self.committed_transaction_state = TransactionState(0) + self.global_time = 0 + self.root_numbering = 0 + self.ref_type_map = {} + self.root_sizes = {} + + def get_new_root_name(self, is_ref_type, size): + self.root_numbering += 1 + r = "lp_%s_%d" % ("ref" if is_ref_type else "char", self.root_numbering) + self.ref_type_map[r] = is_ref_type + self.root_sizes[r] = size + return r + + def has_ref_type(self, r): + return self.ref_type_map[r] + + def get_root_size(self, r): + return self.root_sizes[r] + + def inc_and_get_global_time(self): + self.global_time += 1 + return self.global_time + + def push_state_to_other_threads(self, trs): + assert not trs.check_must_abort() + for ts in self.thread_states: + other_trs = ts.transaction_state + if other_trs is None or other_trs is trs: + continue + other_trs.update_from_committed(trs, only_new=True) + + if trs.check_must_abort(): + self.ex.do('# conflict while pushing to other threads: %s' % + trs.objs_in_conflict) + + def is_inevitable_transaction_running(self): + for ts in self.thread_states: + other_trs = ts.transaction_state + if (other_trs and other_trs.inevitable): + self.ex.do("# there is another inevitable transaction:") + return True + return False + + def check_if_can_become_inevitable(self, trs): + assert not trs.check_must_abort() + for ts in self.thread_states: + other_trs = ts.transaction_state + if (other_trs and trs is not other_trs + and other_trs.inevitable): + self.ex.do("# there is another inevitable transaction:") + trs.set_must_abort() + break + + def check_for_write_read_conflicts(self, trs): + assert not trs.check_must_abort() + for ts in self.thread_states: + other_trs = ts.transaction_state + if other_trs is None or other_trs is trs: + continue + + confl_set = other_trs.read_set & trs.write_set + if confl_set: + # trs wins! + other_trs.set_must_abort(objs_in_conflict=confl_set) + assert not trs.check_must_abort() + + assert not trs.check_must_abort() + + +################################################################### +################################################################### +######################## STM OPERATIONS ########################### +################################################################### +################################################################### + + +def op_start_transaction(ex, global_state, thread_state): + if thread_state.start_transaction(ex.thread_num): + ex.do('self.start_transaction()') + thread_state.reload_roots(ex) + # + # assert that everything known is old: + old_objs = thread_state.saved_roots + for o in old_objs: + ex.do("assert not is_in_nursery(%s)" % o) + else: + ex.do('py.test.raises(Conflict, self.start_transaction)') + + +def op_commit_transaction(ex, global_state, thread_state): + # + # push all new roots + ex.do("# push new objs before commit:") + thread_state.push_roots(ex) + aborts = thread_state.commit_transaction() + # + if aborts: + thread_state.abort_transaction() + ex.do(raising_call(aborts, "self.commit_transaction")) + +def op_abort_transaction(ex, global_state, thread_state): + trs = thread_state.transaction_state + if trs.inevitable: + return + trs.set_must_abort() + thread_state.abort_transaction() + ex.do('self.abort_transaction()') + +def op_become_inevitable(ex, global_state, thread_state): + trs = thread_state.transaction_state + global_state.check_if_can_become_inevitable(trs) + + thread_state.push_roots(ex) + ex.do(raising_call(trs.check_must_abort(), + "self.become_inevitable")) + if trs.check_must_abort(): + thread_state.abort_transaction() + else: + trs.inevitable = True + thread_state.pop_roots(ex) + thread_state.reload_roots(ex) + + +def op_allocate(ex, global_state, thread_state): + size = global_state.rnd.choice([ + "16", + str(4096+16), + str(80*1024+16), + #"SOME_MEDIUM_SIZE+16", + #"SOME_LARGE_SIZE+16", + ]) + r = global_state.get_new_root_name(False, size) + thread_state.push_roots(ex) + + ex.do('%s = stm_allocate(%s)' % (r, size)) + ex.do('# 0x%x' % (int(ffi.cast("uintptr_t", ex.content[r])))) + thread_state.transaction_state.add_root(r, 0, True) + + thread_state.pop_roots(ex) + thread_state.reload_roots(ex) + thread_state.register_root(r) + +def op_allocate_ref(ex, global_state, thread_state): + num = str(global_state.rnd.randrange(1, 1000)) + r = global_state.get_new_root_name(True, num) + thread_state.push_roots(ex) + ex.do('%s = stm_allocate_refs(%s)' % (r, num)) + ex.do('# 0x%x' % (int(ffi.cast("uintptr_t", ex.content[r])))) + thread_state.transaction_state.add_root(r, "ffi.NULL", True) + + thread_state.pop_roots(ex) + thread_state.reload_roots(ex) + thread_state.register_root(r) + +def op_minor_collect(ex, global_state, thread_state): + thread_state.push_roots(ex) + ex.do('stm_minor_collect()') + thread_state.pop_roots(ex) + thread_state.reload_roots(ex) + +def op_major_collect(ex, global_state, thread_state): + thread_state.push_roots(ex) + ex.do('stm_major_collect()') + thread_state.pop_roots(ex) + thread_state.reload_roots(ex) + + +def op_forget_root(ex, global_state, thread_state): + r = thread_state.forget_random_root() + if thread_state.transaction_state.inevitable: + ex.do('# inevitable forget %s' % r) + else: + ex.do('# forget %s' % r) + +def op_write(ex, global_state, thread_state): + r = thread_state.get_random_root() + trs = thread_state.transaction_state + is_ref = global_state.has_ref_type(r) + try_cards = global_state.rnd.randrange(1, 100) > 5 and False + # + # decide on a value to write + if is_ref: + v = thread_state.get_random_root() + else: + v = ord(global_state.rnd.choice("abcdefghijklmnop")) + assert trs.write_root(r, v) is not None + # + aborts = trs.check_must_abort() + if aborts: + thread_state.abort_transaction() + offset = global_state.get_root_size(r) + " - 1" + if is_ref: + ex.do(raising_call(aborts, "stm_set_ref", r, offset, v, try_cards)) + if not aborts: + ex.do(raising_call(False, "stm_set_ref", r, "0", v, try_cards)) + else: + ex.do(raising_call(aborts, "stm_set_char", r, repr(chr(v)), offset, try_cards)) + if not aborts: + ex.do(raising_call(False, "stm_set_char", r, repr(chr(v)), "HDR", try_cards)) + +def op_read(ex, global_state, thread_state): + r = thread_state.get_random_root() + trs = thread_state.transaction_state + v = trs.read_root(r) + # + offset = global_state.get_root_size(r) + " - 1" + if global_state.has_ref_type(r): + if v in thread_state.saved_roots or v in global_state.prebuilt_roots: + # v = root known to this transaction; or prebuilt + ex.do("assert stm_get_ref(%s, %s) == %s" % (r, offset, v)) + ex.do("assert stm_get_ref(%s, 0) == %s" % (r, v)) + elif v != "ffi.NULL": + global_trs = global_state.committed_transaction_state + if v not in trs.values: + # not from this transaction AND not known at the start of this + # transaction AND not pushed to us by a commit + assert False + elif v not in global_trs.values: + # created and forgotten earlier in this transaction, we still + # know its latest value (v in trs.values) + ex.do("# revive %r in this transaction" % v) + else: + # created in an earlier transaction, now also known here. We + # know its value (v in trs.values) + ex.do("# register %r in this thread" % v) + # + ex.do("%s = stm_get_ref(%s, %s)" % (v, r, offset)) + ex.do("%s = stm_get_ref(%s, 0)" % (v, r)) + thread_state.register_root(v) + else: + # v is NULL; we still need to read it (as it should be in the read-set): + ex.do("assert stm_get_ref(%s, %s) == %s" % (r,offset,v)) + ex.do("assert stm_get_ref(%s, 0) == %s" % (r,v)) + else: + ex.do("assert stm_get_char(%s, %s) == %s" % (r, offset, repr(chr(v)))) + ex.do("assert stm_get_char(%s, HDR) == %s" % (r, repr(chr(v)))) + +def op_assert_size(ex, global_state, thread_state): + r = thread_state.get_random_root() + size = global_state.get_root_size(r) + if global_state.has_ref_type(r): + ex.do("assert stm_get_obj_size(%s) == %s" % (r, size + " * WORD + HDR")) + else: + ex.do("assert stm_get_obj_size(%s) == %s" % (r, size)) + +def op_assert_modified(ex, global_state, thread_state): + trs = thread_state.transaction_state + modified = trs.get_old_modified() + ex.do("# modified = %s" % modified) + ex.do("modified = modified_old_objects()") + # check that all definitely old objs we wrote to and that were saved + # are in modified_old_objs() + saved = [m for m in modified + if m in thread_state.saved_roots or m in global_state.prebuilt_roots] + ex.do("assert set([%s]).issubset(set(modified))" % ( + ", ".join(saved) + )) + + +def op_switch_thread(ex, global_state, thread_state, new_thread_state=None): + if new_thread_state is None: + new_thread_state = global_state.rnd.choice(global_state.thread_states) + + if new_thread_state != thread_state: + if thread_state.transaction_state: + thread_state.push_roots(ex) + ex.do('#') + # + trs = new_thread_state.transaction_state + if trs is not None and not trs.inevitable: + if global_state.is_inevitable_transaction_running(): + trs.set_must_abort() + conflicts = trs is not None and trs.check_must_abort() + ex.thread_num = new_thread_state.num + # + ex.do(raising_call(conflicts, + "self.switch", new_thread_state.num)) + if conflicts: + new_thread_state.abort_transaction() + elif trs: + new_thread_state.pop_roots(ex) + new_thread_state.reload_roots(ex) + + return new_thread_state + + +################################################################### +################################################################### +####################### TEST GENERATION ########################### +################################################################### +################################################################### + + +class TestRandom(BaseTest): + NB_THREADS = NB_SEGMENTS + + def test_fixed_16_bytes_objects(self, seed=1010): + rnd = random.Random(seed) + + N_OBJECTS = 3 + N_THREADS = self.NB_THREADS + ex = Exec(self) + ex.do("################################################################\n"*10) + ex.do('# initialization') + + global_state = GlobalState(ex, rnd) + for i in range(N_THREADS): + global_state.thread_states.append( + ThreadState(i, global_state)) + curr_thread = global_state.thread_states[0] + + for i in range(N_OBJECTS): + r = global_state.get_new_root_name(False, "384") + ex.do('%s = stm_allocate_old(384)' % r) + global_state.committed_transaction_state.add_root(r, 0, False) + global_state.prebuilt_roots.append(r) + + r = global_state.get_new_root_name(True, "50") + ex.do('%s = stm_allocate_old_refs(50)' % r) + global_state.committed_transaction_state.add_root(r, "ffi.NULL", False) + global_state.prebuilt_roots.append(r) + global_state.committed_transaction_state.write_set = set() + global_state.committed_transaction_state.read_set = set() + + # random steps: + possible_actions = [ + op_allocate, + op_allocate_ref, op_allocate_ref, + op_write, op_write, op_write, + op_read, op_read, op_read, op_read, op_read, op_read, op_read, op_read, + op_commit_transaction, + op_abort_transaction, + op_forget_root, + op_become_inevitable, + op_assert_size, + op_assert_modified, + op_minor_collect, + #op_major_collect, + ] + for _ in range(200): + # make sure we are in a transaction: + curr_thread = op_switch_thread(ex, global_state, curr_thread) + + if (global_state.is_inevitable_transaction_running() + and curr_thread.transaction_state is None): + continue # don't bother trying to start a transaction + + if curr_thread.transaction_state is None: + op_start_transaction(ex, global_state, curr_thread) + assert curr_thread.transaction_state is not None + + # do something random + action = rnd.choice(possible_actions) + action(ex, global_state, curr_thread) + + # to make sure we don't have aborts in the test's teardown method, + # we will simply stop all running transactions + for ts in global_state.thread_states: + if ts.transaction_state is not None: + if curr_thread != ts: + ex.do('#') + curr_thread = op_switch_thread(ex, global_state, curr_thread, + new_thread_state=ts) + + # could have aborted in the switch() above: + if curr_thread.transaction_state: + op_commit_transaction(ex, global_state, curr_thread) + + + + def _make_fun(seed): + def test_fun(self): + self.test_fixed_16_bytes_objects(seed) + test_fun.__name__ = 'test_random_%d' % seed + return test_fun + + for _seed in range(5000, 5200): + _fn = _make_fun(_seed) + locals()[_fn.__name__] = _fn From noreply at buildbot.pypy.org Wed Sep 10 13:22:56 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Wed, 10 Sep 2014 13:22:56 +0200 (CEST) Subject: [pypy-commit] stmgc default: fix some demos and some bugs Message-ID: <20140910112256.B33331C0350@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1381:9adaf26befd4 Date: 2014-09-10 13:24 +0200 http://bitbucket.org/pypy/stmgc/changeset/9adaf26befd4/ Log: fix some demos and some bugs diff --git a/c8/demo/Makefile b/c8/demo/Makefile new file mode 100644 --- /dev/null +++ b/c8/demo/Makefile @@ -0,0 +1,36 @@ +# +# Makefile for the demos. +# + +DEBUG_EXE = debug-demo2 +BUILD_EXE = build-demo2 +RELEASE_EXE = release-demo2 + +debug: $(DEBUG_EXE) # with prints and asserts +build: $(BUILD_EXE) # without prints, but with asserts +release: $(RELEASE_EXE) # without prints nor asserts + +clean: + rm -f $(BUILD_EXE) $(DEBUG_EXE) $(RELEASE_EXE) + + +H_FILES = ../stmgc.h ../stm/*.h +C_FILES = ../stmgc.c ../stm/*.c + +COMMON = -I.. -pthread -lrt -g -Wall -Werror -DSTM_LARGEMALLOC_TEST + + +# note that 'build' is partially optimized but still contains all asserts +debug-%: %.c ${H_FILES} ${C_FILES} + clang $(COMMON) -DSTM_DEBUGPRINT -DSTM_GC_NURSERY=128 -O0 \ + $< -o debug-$* ../stmgc.c + +build-%: %.c ${H_FILES} ${C_FILES} + clang $(COMMON) -DSTM_GC_NURSERY=128 -O1 $< -o build-$* ../stmgc.c + +release-%: %.c ${H_FILES} ${C_FILES} + clang $(COMMON) -DNDEBUG -O2 $< -o release-$* ../stmgc.c + + +release-htm-%: %.c ../../htm-c7/stmgc.? ../../htm-c7/htm.h + clang $(COMMON) -O2 $< -o release-htm-$* ../../htm-c7/stmgc.c -DUSE_HTM diff --git a/c8/demo/demo2.c b/c8/demo/demo2.c new file mode 100644 --- /dev/null +++ b/c8/demo/demo2.c @@ -0,0 +1,322 @@ +#include +#include +#include +#include +#include + +#ifdef USE_HTM +# include "../../htm-c7/stmgc.h" +#else +# include "stmgc.h" +#endif + +#define LIST_LENGTH 4000 +#define NTHREADS 2 + +#ifdef USE_HTM +# define BUNCH 200 +#else +# define BUNCH 200 +#endif + +typedef TLPREFIX struct node_s node_t; +typedef node_t* nodeptr_t; +typedef object_t* objptr_t; + +struct node_s { + struct object_s hdr; + long value; + nodeptr_t next; +}; + +__thread stm_thread_local_t stm_thread_local; + + +ssize_t stmcb_size_rounded_up(struct object_s *ob) +{ + return sizeof(struct node_s); +} + +void stmcb_trace(struct object_s *obj, void visit(object_t **)) +{ + struct node_s *n; + n = (struct node_s*)obj; + visit((object_t **)&n->next); +} +long stmcb_obj_supports_cards(struct object_s *obj) +{ + return 0; +} +void stmcb_get_card_base_itemsize(struct object_s *obj, + uintptr_t offset_itemsize[2]) +{ + abort(); +} +void stmcb_trace_cards(struct object_s *obj, void visit(object_t **), + uintptr_t start, uintptr_t stop) +{ + abort(); +} +void stmcb_commit_soon() {} + +static void expand_marker(char *base, uintptr_t odd_number, + object_t *following_object, + char *outputbuf, size_t outputbufsize) +{ + assert(following_object == NULL); + snprintf(outputbuf, outputbufsize, "<%p %lu>", base, odd_number); +} + + +nodeptr_t global_chained_list; + + +long check_sorted(void) +{ + nodeptr_t r_n; + long prev, sum; + + stm_start_transaction(&stm_thread_local); + + stm_read((objptr_t)global_chained_list); + r_n = global_chained_list; + assert(r_n->value == -1); + + prev = -1; + sum = 0; + while (r_n->next) { + r_n = r_n->next; + stm_read((objptr_t)r_n); + sum += r_n->value; + + stm_safe_point(); + if (prev >= r_n->value) { + stm_commit_transaction(); + return -1; + } + + prev = r_n->value; + } + + stm_commit_transaction(); + return sum; +} + +nodeptr_t swap_nodes(nodeptr_t initial) +{ + assert(initial != NULL); + + stm_start_transaction(&stm_thread_local); + + if (stm_thread_local.longest_marker_state != 0) { + fprintf(stderr, "[%p] marker %d for %.6f seconds:\n", + &stm_thread_local, + stm_thread_local.longest_marker_state, + stm_thread_local.longest_marker_time); + fprintf(stderr, "\tself:\t\"%s\"\n\tother:\t\"%s\"\n", + stm_thread_local.longest_marker_self, + stm_thread_local.longest_marker_other); + stm_thread_local.longest_marker_state = 0; + stm_thread_local.longest_marker_time = 0.0; + } + + nodeptr_t prev = initial; + stm_read((objptr_t)prev); + + int i; + for (i=0; inext; + if (current == NULL) { + stm_commit_transaction(); + return NULL; + } + stm_read((objptr_t)current); + nodeptr_t next = current->next; + if (next == NULL) { + stm_commit_transaction(); + return NULL; + } + stm_read((objptr_t)next); + + if (next->value < current->value) { + stm_write((objptr_t)prev); + stm_write((objptr_t)current); + stm_write((objptr_t)next); + + prev->next = next; + current->next = next->next; + next->next = current; + + stm_safe_point(); + } + prev = current; + } + + stm_commit_transaction(); + return prev; +} + + + +void bubble_run(void) +{ + nodeptr_t r_current; + + r_current = global_chained_list; + while (r_current) { + r_current = swap_nodes(r_current); + } +} + + +/* initialize list with values in decreasing order */ +void setup_list(void) +{ + int i; + nodeptr_t w_newnode, w_prev; + + stm_start_inevitable_transaction(&stm_thread_local); + + global_chained_list = (nodeptr_t)stm_allocate(sizeof(struct node_s)); + global_chained_list->value = -1; + global_chained_list->next = NULL; + + STM_PUSH_ROOT(stm_thread_local, global_chained_list); + + w_prev = global_chained_list; + for (i = 0; i < LIST_LENGTH; i++) { + STM_PUSH_ROOT(stm_thread_local, w_prev); + w_newnode = (nodeptr_t)stm_allocate(sizeof(struct node_s)); + + STM_POP_ROOT(stm_thread_local, w_prev); + w_newnode->value = LIST_LENGTH - i; + w_newnode->next = NULL; + + stm_write((objptr_t)w_prev); + w_prev->next = w_newnode; + w_prev = w_newnode; + } + + STM_POP_ROOT(stm_thread_local, global_chained_list); /* update value */ + assert(global_chained_list->value == -1); + STM_PUSH_ROOT(stm_thread_local, global_chained_list); + + stm_commit_transaction(); + + stm_start_transaction(&stm_thread_local); + STM_POP_ROOT(stm_thread_local, global_chained_list); /* update value */ + assert(global_chained_list->value == -1); + STM_PUSH_ROOT(stm_thread_local, global_chained_list); /* remains forever in the shadow stack */ + stm_commit_transaction(); + + printf("setup ok\n"); +} + +void teardown_list(void) +{ + STM_POP_ROOT_RET(stm_thread_local); +} + + +static sem_t done; + + +void unregister_thread_local(void) +{ + stm_flush_timing(&stm_thread_local, 1); + stm_unregister_thread_local(&stm_thread_local); +} + +void *demo2(void *arg) +{ + int status; + rewind_jmp_buf rjbuf; + stm_register_thread_local(&stm_thread_local); + stm_rewind_jmp_enterframe(&stm_thread_local, &rjbuf); + char *org = (char *)stm_thread_local.shadowstack; + + STM_PUSH_ROOT(stm_thread_local, global_chained_list); /* remains forever in the shadow stack */ + + int loops = 0; + + while (check_sorted() == -1) { + + STM_PUSH_MARKER(stm_thread_local, 2 * loops + 1, NULL); + + bubble_run(); + + STM_POP_MARKER(stm_thread_local); + loops++; + } + + STM_POP_ROOT(stm_thread_local, global_chained_list); + OPT_ASSERT(org == (char *)stm_thread_local.shadowstack); + + stm_rewind_jmp_leaveframe(&stm_thread_local, &rjbuf); + unregister_thread_local(); + status = sem_post(&done); assert(status == 0); + return NULL; +} + +void final_check(void) +{ + long sum; + + printf("final check\n"); + + sum = check_sorted(); + + // little Gauss: + if (sum == (1 + LIST_LENGTH) * (LIST_LENGTH / 2)) + printf("check ok\n"); + else + printf("check ERROR\n"); +} + + +void newthread(void*(*func)(void*), void *arg) +{ + pthread_t th; + int status = pthread_create(&th, NULL, func, arg); + if (status != 0) + abort(); + pthread_detach(th); + printf("started new thread\n"); +} + + + +int main(void) +{ + int status, i; + rewind_jmp_buf rjbuf; + + status = sem_init(&done, 0, 0); assert(status == 0); + + stm_setup(); + stm_register_thread_local(&stm_thread_local); + stm_rewind_jmp_enterframe(&stm_thread_local, &rjbuf); + stmcb_expand_marker = expand_marker; + + + setup_list(); + + + for (i = 1; i <= NTHREADS; i++) { + newthread(demo2, (void*)(uintptr_t)i); + } + + for (i = 1; i <= NTHREADS; i++) { + status = sem_wait(&done); assert(status == 0); + } + + final_check(); + + teardown_list(); + + stm_rewind_jmp_leaveframe(&stm_thread_local, &rjbuf); + unregister_thread_local(); + //stm_teardown(); + + return 0; +} diff --git a/c8/demo/demo_largemalloc.c b/c8/demo/demo_largemalloc.c new file mode 100644 --- /dev/null +++ b/c8/demo/demo_largemalloc.c @@ -0,0 +1,86 @@ +#include +#include +#include +#include + +#include "stmgc.h" +#include "../stm/largemalloc.h" + +static inline double get_stm_time(void) +{ + struct timespec tp; + clock_gettime(CLOCK_MONOTONIC, &tp); + return tp.tv_sec + tp.tv_nsec * 0.000000001; +} + +ssize_t stmcb_size_rounded_up(struct object_s *ob) +{ + abort(); +} + +void stmcb_trace(struct object_s *obj, void visit(object_t **)) +{ + abort(); +} + +void stmcb_commit_soon() {} +long stmcb_obj_supports_cards(struct object_s *obj) +{ + return 0; +} +void stmcb_trace_cards(struct object_s *obj, void cb(object_t **), + uintptr_t start, uintptr_t stop) { + abort(); +} +void stmcb_get_card_base_itemsize(struct object_s *obj, + uintptr_t offset_itemsize[2]) { + abort(); +} + +/************************************************************/ + +#define ARENA_SIZE (1024*1024*1024) + +static char *arena_data; +extern bool (*_stm_largemalloc_keep)(char *data); /* a hook for tests */ +void _stm_mutex_pages_lock(void); + + +static bool keep_me(char *data) { + static bool last_answer = false; + last_answer = !last_answer; + return last_answer; +} + +void timing(int scale) +{ + long limit = 1L << scale; + _stm_largemalloc_init_arena(arena_data, ARENA_SIZE); + double start = get_stm_time(); + + long i; + for (i = 0; i < limit; i++) { + _stm_large_malloc(16 + 8 * (i % 4)); /* may return NULL */ + } + _stm_largemalloc_keep = keep_me; + _stm_largemalloc_sweep(); + for (i = 0; i < limit; i++) { + _stm_large_malloc(16 + 8 * (i % 4)); /* may return NULL */ + } + + double stop = get_stm_time(); + printf("scale %2d: %.9f\n", scale, stop - start); +} + + + +int main(void) +{ + int i; + arena_data = malloc(ARENA_SIZE); + assert(arena_data != NULL); + //_stm_mutex_pages_lock(); + for (i = 0; i < 25; i++) + timing(i); + return 0; +} diff --git a/c8/demo/demo_random.c b/c8/demo/demo_random.c new file mode 100644 --- /dev/null +++ b/c8/demo/demo_random.c @@ -0,0 +1,517 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "stmgc.h" + +#define NUMTHREADS 3 +#define STEPS_PER_THREAD 500 +#define THREAD_STARTS 1000 // how many restarts of threads +#define PREBUILT_ROOTS 3 +#define MAXROOTS 1000 +#define FORKS 3 + +// SUPPORT +struct node_s; +typedef TLPREFIX struct node_s node_t; +typedef node_t* nodeptr_t; +typedef object_t* objptr_t; +int num_forked_children = 0; + +struct node_s { + struct object_s hdr; + int sig; + long my_size; + long my_id; + long my_hash; + nodeptr_t next; +}; + +#define SIGNATURE 0x01234567 + + +static sem_t done; +__thread stm_thread_local_t stm_thread_local; + +// global and per-thread-data +time_t default_seed; +objptr_t prebuilt_roots[PREBUILT_ROOTS]; + +struct thread_data { + unsigned int thread_seed; + objptr_t roots[MAXROOTS]; + int num_roots; + int num_roots_at_transaction_start; + int steps_left; +}; +__thread struct thread_data td; + +struct thread_data *_get_td(void) +{ + return &td; /* for gdb */ +} + + +ssize_t stmcb_size_rounded_up(struct object_s *ob) +{ + return ((struct node_s*)ob)->my_size; +} + +void stmcb_trace(struct object_s *obj, void visit(object_t **)) +{ + struct node_s *n; + n = (struct node_s*)obj; + + /* and the same value at the end: */ + /* note, ->next may be the same as last_next */ + nodeptr_t *last_next = (nodeptr_t*)((char*)n + n->my_size - sizeof(void*)); + + assert(n->next == *last_next); + + visit((object_t **)&n->next); + visit((object_t **)last_next); + + assert(n->next == *last_next); +} + +void stmcb_commit_soon() {} +long stmcb_obj_supports_cards(struct object_s *obj) +{ + return 0; +} +void stmcb_trace_cards(struct object_s *obj, void cb(object_t **), + uintptr_t start, uintptr_t stop) { + abort(); +} +void stmcb_get_card_base_itemsize(struct object_s *obj, + uintptr_t offset_itemsize[2]) { + abort(); +} + +int get_rand(int max) +{ + if (max == 0) + return 0; + return (int)(rand_r(&td.thread_seed) % (unsigned int)max); +} + +objptr_t get_random_root() +{ + int num = get_rand(2); + if (num == 0 && td.num_roots > 0) { + num = get_rand(td.num_roots); + return td.roots[num]; + } + else { + num = get_rand(PREBUILT_ROOTS); + return prebuilt_roots[num]; + } +} + +void reload_roots() +{ + int i; + assert(td.num_roots == td.num_roots_at_transaction_start); + for (i = td.num_roots_at_transaction_start - 1; i >= 0; i--) { + if (td.roots[i]) + STM_POP_ROOT(stm_thread_local, td.roots[i]); + } + + for (i = 0; i < td.num_roots_at_transaction_start; i++) { + if (td.roots[i]) + STM_PUSH_ROOT(stm_thread_local, td.roots[i]); + } +} + +void push_roots() +{ + int i; + for (i = td.num_roots_at_transaction_start; i < td.num_roots; i++) { + if (td.roots[i]) + STM_PUSH_ROOT(stm_thread_local, td.roots[i]); + } +} + +void pop_roots() +{ + int i; + for (i = td.num_roots - 1; i >= td.num_roots_at_transaction_start; i--) { + if (td.roots[i]) + STM_POP_ROOT(stm_thread_local, td.roots[i]); + } +} + +void del_root(int idx) +{ + int i; + assert(idx >= td.num_roots_at_transaction_start); + + for (i = idx; i < td.num_roots - 1; i++) + td.roots[i] = td.roots[i + 1]; + td.num_roots--; +} + +void add_root(objptr_t r) +{ + if (r && td.num_roots < MAXROOTS) { + td.roots[td.num_roots++] = r; + } +} + + +void read_barrier(objptr_t p) +{ + if (p != NULL) { + stm_read(p); + } +} + +void write_barrier(objptr_t p) +{ + if (p != NULL) { + stm_write(p); + } +} + +void set_next(objptr_t p, objptr_t v) +{ + if (p != NULL) { + nodeptr_t n = (nodeptr_t)p; + + /* and the same value at the end: */ + nodeptr_t TLPREFIX *last_next = (nodeptr_t TLPREFIX *)((stm_char*)n + n->my_size - sizeof(void*)); + assert(n->next == *last_next); + n->next = (nodeptr_t)v; + *last_next = (nodeptr_t)v; + } +} + +nodeptr_t get_next(objptr_t p) +{ + nodeptr_t n = (nodeptr_t)p; + + /* and the same value at the end: */ + nodeptr_t TLPREFIX *last_next = (nodeptr_t TLPREFIX *)((stm_char*)n + n->my_size - sizeof(void*)); + OPT_ASSERT(n->next == *last_next); + + return n->next; +} + + +objptr_t simple_events(objptr_t p, objptr_t _r) +{ + int k = get_rand(10); + int num; + + switch (k) { + case 0: // remove a root + if (td.num_roots > td.num_roots_at_transaction_start) { + num = td.num_roots_at_transaction_start + + get_rand(td.num_roots - td.num_roots_at_transaction_start); + del_root(num); + } + break; + case 1: // add 'p' to roots + add_root(p); + break; + case 2: // set 'p' to point to a root + if (_r) + p = _r; + break; + case 3: // allocate fresh 'p' + push_roots(); + size_t sizes[4] = {sizeof(struct node_s), + sizeof(struct node_s) + (get_rand(100000) & ~15), + sizeof(struct node_s) + 4096, + sizeof(struct node_s) + 4096*70}; + size_t size = sizes[get_rand(4)]; + p = stm_allocate(size); + ((nodeptr_t)p)->sig = SIGNATURE; + ((nodeptr_t)p)->my_size = size; + ((nodeptr_t)p)->my_id = 0; + ((nodeptr_t)p)->my_hash = 0; + pop_roots(); + /* reload_roots not necessary, all are old after start_transaction */ + break; + case 4: // read and validate 'p' + read_barrier(p); + break; + case 5: // only do a stm_write_barrier + write_barrier(p); + break; + case 6: // follow p->next + if (p) { + read_barrier(p); + p = (objptr_t)(get_next(p)); + } + break; + case 7: // set 'p' as *next in one of the roots + write_barrier(_r); + set_next(_r, p); + break; + case 8: // id checking + if (p) { + nodeptr_t n = (nodeptr_t)p; + if (n->my_id == 0) { + write_barrier(p); + n->my_id = stm_id(p); + } + else { + read_barrier(p); + assert(n->my_id == stm_id(p)); + } + } + break; + case 9: + if (p) { + nodeptr_t n = (nodeptr_t)p; + if (n->my_hash == 0) { + write_barrier(p); + n->my_hash = stm_identityhash(p); + } + else { + read_barrier(p); + assert(n->my_hash == stm_identityhash(p)); + } + } + break; + } + return p; +} + + +objptr_t do_step(objptr_t p) +{ + objptr_t _r; + int k; + + _r = get_random_root(); + k = get_rand(12); + + if (k < 10) { + p = simple_events(p, _r); + } else if (get_rand(20) == 1) { + return (objptr_t)-1; // break current + } else if (get_rand(20) == 1) { + push_roots(); + stm_become_inevitable(&stm_thread_local, "please"); + pop_roots(); + return NULL; + } else if (get_rand(240) == 1) { + push_roots(); + stm_become_globally_unique_transaction(&stm_thread_local, "really"); + fprintf(stderr, "[GUT/%d]", (int)STM_SEGMENT->segment_num); + pop_roots(); + return NULL; + } + return p; +} + + + +void setup_thread() +{ + memset(&td, 0, sizeof(struct thread_data)); + + /* stupid check because gdb shows garbage + in td.roots: */ + int i; + for (i = 0; i < MAXROOTS; i++) + assert(td.roots[i] == NULL); + + td.thread_seed = default_seed++; + td.steps_left = STEPS_PER_THREAD; + td.num_roots = 0; + td.num_roots_at_transaction_start = 0; +} + + + +void *demo_random(void *arg) +{ + int status; + rewind_jmp_buf rjbuf; + stm_register_thread_local(&stm_thread_local); + stm_rewind_jmp_enterframe(&stm_thread_local, &rjbuf); + + setup_thread(); + + objptr_t p; + + stm_start_transaction(&stm_thread_local); + assert(td.num_roots >= td.num_roots_at_transaction_start); + td.num_roots = td.num_roots_at_transaction_start; + p = NULL; + pop_roots(); /* does nothing.. */ + reload_roots(); + + while (td.steps_left-->0) { + if (td.steps_left % 8 == 0) + fprintf(stdout, "#"); + + assert(p == NULL || ((nodeptr_t)p)->sig == SIGNATURE); + + p = do_step(p); + + if (p == (objptr_t)-1) { + push_roots(); + + long call_fork = (arg != NULL && *(long *)arg); + if (call_fork == 0) { /* common case */ + stm_commit_transaction(); + td.num_roots_at_transaction_start = td.num_roots; + if (get_rand(100) < 98) { + stm_start_transaction(&stm_thread_local); + } else { + stm_start_inevitable_transaction(&stm_thread_local); + } + td.num_roots = td.num_roots_at_transaction_start; + p = NULL; + pop_roots(); + reload_roots(); + } + else { + /* run a fork() inside the transaction */ + printf("========== FORK =========\n"); + *(long*)arg = 0; + pid_t child = fork(); + printf("=== in process %d thread %lx, fork() returned %d\n", + (int)getpid(), (long)pthread_self(), (int)child); + if (child == -1) { + fprintf(stderr, "fork() error: %m\n"); + abort(); + } + if (child != 0) + num_forked_children++; + else + num_forked_children = 0; + + pop_roots(); + p = NULL; + } + } + } + push_roots(); + stm_commit_transaction(); + + /* even out the shadow stack before leaveframe: */ + stm_start_inevitable_transaction(&stm_thread_local); + while (td.num_roots > 0) { + td.num_roots--; + objptr_t t; + STM_POP_ROOT(stm_thread_local, t); + } + stm_commit_transaction(); + + stm_rewind_jmp_leaveframe(&stm_thread_local, &rjbuf); + stm_unregister_thread_local(&stm_thread_local); + + status = sem_post(&done); assert(status == 0); + return NULL; +} + +void newthread(void*(*func)(void*), void *arg) +{ + pthread_t th; + int status = pthread_create(&th, NULL, func, arg); + if (status != 0) + abort(); + pthread_detach(th); + printf("started new thread\n"); +} + + +void setup_globals() +{ + int i; + + struct node_s prebuilt_template = { + .sig = SIGNATURE, + .my_size = sizeof(struct node_s), + .my_id = 0, + .my_hash = 0, + .next = NULL + }; + + stm_start_inevitable_transaction(&stm_thread_local); + for (i = 0; i < PREBUILT_ROOTS; i++) { + void* new_templ = malloc(sizeof(struct node_s)); + memcpy(new_templ, &prebuilt_template, sizeof(struct node_s)); + prebuilt_roots[i] = stm_setup_prebuilt((objptr_t)(long)new_templ); + + if (i % 2 == 0) { + int hash = i + 5; + stm_set_prebuilt_identityhash(prebuilt_roots[i], + hash); + ((nodeptr_t)prebuilt_roots[i])->my_hash = hash; + } + } + stm_commit_transaction(); +} + +int main(void) +{ + int i, status; + rewind_jmp_buf rjbuf; + + /* pick a random seed from the time in seconds. + A bit pointless for now... because the interleaving of the + threads is really random. */ + default_seed = time(NULL); + printf("running with seed=%lld\n", (long long)default_seed); + + status = sem_init(&done, 0, 0); + assert(status == 0); + + + stm_setup(); + stm_register_thread_local(&stm_thread_local); + stm_rewind_jmp_enterframe(&stm_thread_local, &rjbuf); + + setup_globals(); + + int thread_starts = NUMTHREADS * THREAD_STARTS; + for (i = 0; i < NUMTHREADS; i++) { + newthread(demo_random, NULL); + thread_starts--; + } + + for (i=0; i < NUMTHREADS * THREAD_STARTS; i++) { + status = sem_wait(&done); + assert(status == 0); + printf("thread finished\n"); + if (thread_starts) { + long forkbase = NUMTHREADS * THREAD_STARTS / (FORKS + 1); + long _fork = (thread_starts % forkbase) == 0; + thread_starts--; + newthread(demo_random, &_fork); + } + } + + for (i = 0; i < num_forked_children; i++) { + pid_t child = wait(&status); + if (child == -1) + perror("wait"); + printf("From %d: child %d terminated with exit status %d\n", + (int)getpid(), (int)child, status); + if (WIFEXITED(status) && WEXITSTATUS(status) == 0) + ; + else { + printf("*** error from the child ***\n"); + return 1; + } + } + + printf("Test OK!\n"); + + stm_rewind_jmp_leaveframe(&stm_thread_local, &rjbuf); + stm_unregister_thread_local(&stm_thread_local); + stm_teardown(); + + return 0; +} diff --git a/c8/demo/demo_random2.c b/c8/demo/demo_random2.c new file mode 100644 --- /dev/null +++ b/c8/demo/demo_random2.c @@ -0,0 +1,540 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "stmgc.h" + +#define NUMTHREADS 3 +#define STEPS_PER_THREAD 500 +#define THREAD_STARTS 1000 // how many restarts of threads +#define PREBUILT_ROOTS 3 +#define FORKS 3 + +#define ACTIVE_ROOTS_SET_SIZE 100 // max num of roots created/alive in one transaction +#define MAX_ROOTS_ON_SS 1000 // max on shadow stack + +// SUPPORT +struct node_s; +typedef TLPREFIX struct node_s node_t; +typedef node_t* nodeptr_t; +typedef object_t* objptr_t; +int num_forked_children = 0; + +struct node_s { + struct object_s hdr; + int sig; + long my_size; + long my_id; + long my_hash; + nodeptr_t next; +}; + +#define SIGNATURE 0x01234567 + + +static sem_t done; +__thread stm_thread_local_t stm_thread_local; +__thread void *thread_may_fork; + +// global and per-thread-data +time_t default_seed; +objptr_t prebuilt_roots[PREBUILT_ROOTS]; + +struct thread_data { + unsigned int thread_seed; + int steps_left; + objptr_t active_roots_set[ACTIVE_ROOTS_SET_SIZE]; + int active_roots_num; + long roots_on_ss; + long roots_on_ss_at_tr_start; +}; +__thread struct thread_data td; + +struct thread_data *_get_td(void) +{ + return &td; /* for gdb */ +} + + +ssize_t stmcb_size_rounded_up(struct object_s *ob) +{ + return ((struct node_s*)ob)->my_size; +} + +void stmcb_trace(struct object_s *obj, void visit(object_t **)) +{ + struct node_s *n; + n = (struct node_s*)obj; + + /* and the same value at the end: */ + /* note, ->next may be the same as last_next */ + nodeptr_t *last_next = (nodeptr_t*)((char*)n + n->my_size - sizeof(void*)); + + assert(n->next == *last_next); + + visit((object_t **)&n->next); + visit((object_t **)last_next); + + assert(n->next == *last_next); +} + +void stmcb_commit_soon() {} + +long stmcb_obj_supports_cards(struct object_s *obj) +{ + return 0; +} +void stmcb_trace_cards(struct object_s *obj, void cb(object_t **), + uintptr_t start, uintptr_t stop) { + abort(); +} +void stmcb_get_card_base_itemsize(struct object_s *obj, + uintptr_t offset_itemsize[2]) { + abort(); +} + +int get_rand(int max) +{ + if (max == 0) + return 0; + return (int)(rand_r(&td.thread_seed) % (unsigned int)max); +} + +objptr_t get_random_root() +{ + /* get some root from shadowstack or active_root_set or prebuilt_roots */ + int num = get_rand(3); + intptr_t ss_size = td.roots_on_ss; + if (num == 0 && ss_size > 0) { + num = get_rand(ss_size); + /* XXX: impl detail: there is already a "-1" on the SS -> +1 */ + objptr_t r = (objptr_t)stm_thread_local.shadowstack_base[num+1].ss; + OPT_ASSERT((((uintptr_t)r) & 3) == 0); + } + + if (num == 1 && td.active_roots_num > 0) { + num = get_rand(td.active_roots_num); + return td.active_roots_set[num]; + } else { + num = get_rand(PREBUILT_ROOTS); + return prebuilt_roots[num]; + } +} + + +long push_roots() +{ + int i; + long to_push = td.active_roots_num; + long not_pushed = 0; + for (i = to_push - 1; i >= 0; i--) { + td.active_roots_num--; + if (td.roots_on_ss < MAX_ROOTS_ON_SS) { + STM_PUSH_ROOT(stm_thread_local, td.active_roots_set[i]); + td.roots_on_ss++; + } else { + not_pushed++; + } + } + return to_push - not_pushed; +} + +void add_root(objptr_t r); +void pop_roots(long to_pop) +{ + int i; + for (i = 0; i < to_pop; i++) { + objptr_t t; + STM_POP_ROOT(stm_thread_local, t); + add_root(t); + td.roots_on_ss--; + } +} + +void del_root(int idx) +{ + int i; + + for (i = idx; i < td.active_roots_num - 1; i++) + td.active_roots_set[i] = td.active_roots_set[i + 1]; + td.active_roots_num--; +} + +void add_root(objptr_t r) +{ + if (r && td.active_roots_num < ACTIVE_ROOTS_SET_SIZE) { + td.active_roots_set[td.active_roots_num++] = r; + } +} + + +void read_barrier(objptr_t p) +{ + if (p != NULL) { + stm_read(p); + } +} + +void write_barrier(objptr_t p) +{ + if (p != NULL) { + stm_write(p); + } +} + +void set_next(objptr_t p, objptr_t v) +{ + if (p != NULL) { + nodeptr_t n = (nodeptr_t)p; + + /* and the same value at the end: */ + nodeptr_t TLPREFIX *last_next = (nodeptr_t TLPREFIX *)((stm_char*)n + n->my_size - sizeof(void*)); + assert(n->next == *last_next); + n->next = (nodeptr_t)v; + *last_next = (nodeptr_t)v; + } +} + +nodeptr_t get_next(objptr_t p) +{ + nodeptr_t n = (nodeptr_t)p; + + /* and the same value at the end: */ + nodeptr_t TLPREFIX *last_next = (nodeptr_t TLPREFIX *)((stm_char*)n + n->my_size - sizeof(void*)); + OPT_ASSERT(n->next == *last_next); + + return n->next; +} + + +objptr_t simple_events(objptr_t p, objptr_t _r) +{ + int k = get_rand(10); + long pushed; + + switch (k) { + case 0: // remove a root + if (td.active_roots_num) { + del_root(get_rand(td.active_roots_num)); + } + break; + case 1: // add 'p' to roots + add_root(p); + break; + case 2: // set 'p' to point to a root + if (_r) + p = _r; + break; + case 3: // allocate fresh 'p' + pushed = push_roots(); + size_t sizes[4] = {sizeof(struct node_s), + sizeof(struct node_s) + (get_rand(100000) & ~15), + sizeof(struct node_s) + 4096, + sizeof(struct node_s) + 4096*70}; + size_t size = sizes[get_rand(4)]; + p = stm_allocate(size); + ((nodeptr_t)p)->sig = SIGNATURE; + ((nodeptr_t)p)->my_size = size; + ((nodeptr_t)p)->my_id = 0; + ((nodeptr_t)p)->my_hash = 0; + pop_roots(pushed); + break; + case 4: // read and validate 'p' + read_barrier(p); + break; + case 5: // only do a stm_write_barrier + write_barrier(p); + break; + case 6: // follow p->next + if (p) { + read_barrier(p); + p = (objptr_t)(get_next(p)); + } + break; + case 7: // set 'p' as *next in one of the roots + write_barrier(_r); + set_next(_r, p); + break; + case 8: // id checking + if (p) { + nodeptr_t n = (nodeptr_t)p; + if (n->my_id == 0) { + write_barrier(p); + n->my_id = stm_id(p); + } + else { + read_barrier(p); + assert(n->my_id == stm_id(p)); + } + } + break; + case 9: + if (p) { + nodeptr_t n = (nodeptr_t)p; + if (n->my_hash == 0) { + write_barrier(p); + n->my_hash = stm_identityhash(p); + } + else { + read_barrier(p); + assert(n->my_hash == stm_identityhash(p)); + } + } + break; + } + return p; +} + +void frame_loop(); +objptr_t do_step(objptr_t p) +{ + objptr_t _r; + int k; + + _r = get_random_root(); + k = get_rand(12); + + if (k < 10) { + p = simple_events(p, _r); + } else if (get_rand(20) == 1) { + long pushed = push_roots(); + stm_commit_transaction(); + td.roots_on_ss_at_tr_start = td.roots_on_ss; + + if (get_rand(100) < 98) { + stm_start_transaction(&stm_thread_local); + } else { + stm_start_inevitable_transaction(&stm_thread_local); + } + td.roots_on_ss = td.roots_on_ss_at_tr_start; + td.active_roots_num = 0; + pop_roots(pushed); + p = NULL; + } else if (get_rand(10) == 1) { + long pushed = push_roots(); + /* leaving our frame */ + frame_loop(); + /* back in our frame */ + pop_roots(pushed); + p = NULL; + } else if (get_rand(20) == 1) { + long pushed = push_roots(); + stm_become_inevitable(&stm_thread_local, "please"); + assert(stm_is_inevitable()); + pop_roots(pushed); + p= NULL; + } else if (get_rand(20) == 1) { + p = (objptr_t)-1; // possibly fork + } else if (get_rand(20) == 1) { + long pushed = push_roots(); + stm_become_globally_unique_transaction(&stm_thread_local, "really"); + fprintf(stderr, "[GUT/%d]", (int)STM_SEGMENT->segment_num); + pop_roots(pushed); + p = NULL; + } + return p; +} + +void frame_loop() +{ + objptr_t p = NULL; + rewind_jmp_buf rjbuf; + + stm_rewind_jmp_enterframe(&stm_thread_local, &rjbuf); + //fprintf(stderr,"%p F: %p\n", STM_SEGMENT->running_thread, __builtin_frame_address(0)); + + long roots_on_ss = td.roots_on_ss; + /* "interpreter main loop": this is one "application-frame" */ + while (td.steps_left-->0 && get_rand(10) != 0) { + if (td.steps_left % 8 == 0) + fprintf(stdout, "#"); + + assert(p == NULL || ((nodeptr_t)p)->sig == SIGNATURE); + + p = do_step(p); + + + if (p == (objptr_t)-1) { + p = NULL; + + long call_fork = (thread_may_fork != NULL && *(long *)thread_may_fork); + if (call_fork) { /* common case */ + long pushed = push_roots(); + /* run a fork() inside the transaction */ + printf("========== FORK =========\n"); + *(long*)thread_may_fork = 0; + pid_t child = fork(); + printf("=== in process %d thread %lx, fork() returned %d\n", + (int)getpid(), (long)pthread_self(), (int)child); + if (child == -1) { + fprintf(stderr, "fork() error: %m\n"); + abort(); + } + if (child != 0) + num_forked_children++; + else + num_forked_children = 0; + + pop_roots(pushed); + } + } + } + OPT_ASSERT(roots_on_ss == td.roots_on_ss); + + stm_rewind_jmp_leaveframe(&stm_thread_local, &rjbuf); +} + + + +void setup_thread() +{ + memset(&td, 0, sizeof(struct thread_data)); + + /* stupid check because gdb shows garbage + in td.roots: */ + int i; + for (i = 0; i < ACTIVE_ROOTS_SET_SIZE; i++) + assert(td.active_roots_set[i] == NULL); + + td.thread_seed = default_seed++; + td.steps_left = STEPS_PER_THREAD; + td.active_roots_num = 0; + td.roots_on_ss = 0; + td.roots_on_ss_at_tr_start = 0; +} + + + +void *demo_random(void *arg) +{ + int status; + rewind_jmp_buf rjbuf; + stm_register_thread_local(&stm_thread_local); + stm_rewind_jmp_enterframe(&stm_thread_local, &rjbuf); + + setup_thread(); + + td.roots_on_ss_at_tr_start = 0; + stm_start_transaction(&stm_thread_local); + td.roots_on_ss = td.roots_on_ss_at_tr_start; + td.active_roots_num = 0; + + thread_may_fork = arg; + while (td.steps_left-->0) { + frame_loop(); + } + + stm_commit_transaction(); + + stm_rewind_jmp_leaveframe(&stm_thread_local, &rjbuf); + stm_unregister_thread_local(&stm_thread_local); + + status = sem_post(&done); assert(status == 0); + return NULL; +} + +void newthread(void*(*func)(void*), void *arg) +{ + pthread_t th; + int status = pthread_create(&th, NULL, func, arg); + if (status != 0) + abort(); + pthread_detach(th); + printf("started new thread\n"); +} + + +void setup_globals() +{ + int i; + + struct node_s prebuilt_template = { + .sig = SIGNATURE, + .my_size = sizeof(struct node_s), + .my_id = 0, + .my_hash = 0, + .next = NULL + }; + + stm_start_inevitable_transaction(&stm_thread_local); + for (i = 0; i < PREBUILT_ROOTS; i++) { + void* new_templ = malloc(sizeof(struct node_s)); + memcpy(new_templ, &prebuilt_template, sizeof(struct node_s)); + prebuilt_roots[i] = stm_setup_prebuilt((objptr_t)(long)new_templ); + + if (i % 2 == 0) { + int hash = i + 5; + stm_set_prebuilt_identityhash(prebuilt_roots[i], + hash); + ((nodeptr_t)prebuilt_roots[i])->my_hash = hash; + } + } + stm_commit_transaction(); +} + +int main(void) +{ + int i, status; + rewind_jmp_buf rjbuf; + + /* pick a random seed from the time in seconds. + A bit pointless for now... because the interleaving of the + threads is really random. */ + default_seed = time(NULL); + printf("running with seed=%lld\n", (long long)default_seed); + + status = sem_init(&done, 0, 0); + assert(status == 0); + + + stm_setup(); + stm_register_thread_local(&stm_thread_local); + stm_rewind_jmp_enterframe(&stm_thread_local, &rjbuf); + + setup_globals(); + + int thread_starts = NUMTHREADS * THREAD_STARTS; + for (i = 0; i < NUMTHREADS; i++) { + newthread(demo_random, NULL); + thread_starts--; + } + + for (i=0; i < NUMTHREADS * THREAD_STARTS; i++) { + status = sem_wait(&done); + assert(status == 0); + printf("thread finished\n"); + if (thread_starts) { + long forkbase = NUMTHREADS * THREAD_STARTS / (FORKS + 1); + long _fork = (thread_starts % forkbase) == 0; + thread_starts--; + newthread(demo_random, &_fork); + } + } + + for (i = 0; i < num_forked_children; i++) { + pid_t child = wait(&status); + if (child == -1) + perror("wait"); + printf("From %d: child %d terminated with exit status %d\n", + (int)getpid(), (int)child, status); + if (WIFEXITED(status) && WEXITSTATUS(status) == 0) + ; + else { + printf("*** error from the child ***\n"); + return 1; + } + } + + printf("Test OK!\n"); + + stm_rewind_jmp_leaveframe(&stm_thread_local, &rjbuf); + stm_unregister_thread_local(&stm_thread_local); + stm_teardown(); + + return 0; +} diff --git a/c8/demo/demo_simple.c b/c8/demo/demo_simple.c new file mode 100644 --- /dev/null +++ b/c8/demo/demo_simple.c @@ -0,0 +1,128 @@ +#include +#include +#include +#include +#include + +#ifdef USE_HTM +# include "../../htm-c7/stmgc.h" +#else +# include "stmgc.h" +#endif + +#define ITERS 100000 +#define NTHREADS 2 + + +typedef TLPREFIX struct node_s node_t; +typedef node_t* nodeptr_t; +typedef object_t* objptr_t; + +struct node_s { + struct object_s hdr; + long value; + nodeptr_t next; +}; + +__thread stm_thread_local_t stm_thread_local; + + +ssize_t stmcb_size_rounded_up(struct object_s *ob) +{ + return sizeof(struct node_s); +} + +void stmcb_trace(struct object_s *obj, void visit(object_t **)) +{ + struct node_s *n; + n = (struct node_s*)obj; + visit((object_t **)&n->next); +} +long stmcb_obj_supports_cards(struct object_s *obj) +{ + return 0; +} +void stmcb_commit_soon() {} + +void stmcb_trace_cards(struct object_s *obj, void cb(object_t **), + uintptr_t start, uintptr_t stop) { + abort(); +} +void stmcb_get_card_base_itemsize(struct object_s *obj, + uintptr_t offset_itemsize[2]) { + abort(); +} + + +static sem_t done; + +static __thread int tl_counter = 0; +//static int gl_counter = 0; + +void *demo2(void *arg) +{ + int status; + rewind_jmp_buf rjbuf; + stm_register_thread_local(&stm_thread_local); + stm_rewind_jmp_enterframe(&stm_thread_local, &rjbuf); + char *org = (char *)stm_thread_local.shadowstack; + tl_counter = 0; + + object_t *tmp; + int i = 0; + while (i < ITERS) { + stm_start_transaction(&stm_thread_local); + tl_counter++; + if (i % 500 < 250) + STM_PUSH_ROOT(stm_thread_local, stm_allocate(16));//gl_counter++; + else + STM_POP_ROOT(stm_thread_local, tmp); + stm_commit_transaction(); + i++; + } + + OPT_ASSERT(org == (char *)stm_thread_local.shadowstack); + + stm_rewind_jmp_leaveframe(&stm_thread_local, &rjbuf); + stm_unregister_thread_local(&stm_thread_local); + status = sem_post(&done); assert(status == 0); + return NULL; +} + + +void newthread(void*(*func)(void*), void *arg) +{ + pthread_t th; + int status = pthread_create(&th, NULL, func, arg); + if (status != 0) + abort(); + pthread_detach(th); + printf("started new thread\n"); +} + + + +int main(void) +{ + int status, i; + + status = sem_init(&done, 0, 0); assert(status == 0); + + stm_setup(); + stm_register_thread_local(&stm_thread_local); + + + for (i = 1; i <= NTHREADS; i++) { + newthread(demo2, (void*)(uintptr_t)i); + } + + for (i = 1; i <= NTHREADS; i++) { + status = sem_wait(&done); assert(status == 0); + } + + + stm_unregister_thread_local(&stm_thread_local); + stm_teardown(); + + return 0; +} diff --git a/c8/demo/test_shadowstack.c b/c8/demo/test_shadowstack.c new file mode 100644 --- /dev/null +++ b/c8/demo/test_shadowstack.c @@ -0,0 +1,74 @@ +#include +#include +#include "stmgc.h" + +stm_thread_local_t stm_thread_local; + +typedef TLPREFIX struct node_s node_t; + +struct node_s { + struct object_s hdr; + long value; +}; + +ssize_t stmcb_size_rounded_up(struct object_s *ob) +{ + return sizeof(struct node_s); +} +void stmcb_trace(struct object_s *obj, void visit(object_t **)) +{ +} +long stmcb_obj_supports_cards(struct object_s *obj) +{ + return 0; +} +void stmcb_get_card_base_itemsize(struct object_s *obj, + uintptr_t offset_itemsize[2]) +{ + abort(); +} +void stmcb_trace_cards(struct object_s *obj, void visit(object_t **), + uintptr_t start, uintptr_t stop) +{ + abort(); +} +void stmcb_commit_soon() {} + + +int main(void) +{ + rewind_jmp_buf rjbuf; + + stm_setup(); + stm_register_thread_local(&stm_thread_local); + stm_rewind_jmp_enterframe(&stm_thread_local, &rjbuf); + + stm_start_transaction(&stm_thread_local); + node_t *node = (node_t *)stm_allocate(sizeof(struct node_s)); + node->value = 129821; + STM_PUSH_ROOT(stm_thread_local, node); + STM_PUSH_ROOT(stm_thread_local, 333); /* odd value */ + stm_commit_transaction(); + + /* now in a new transaction, pop the node off the shadowstack, but + then do a major collection. It should still be found by the + tracing logic. */ + stm_start_transaction(&stm_thread_local); + STM_POP_ROOT_RET(stm_thread_local); + STM_POP_ROOT(stm_thread_local, node); + assert(node->value == 129821); + STM_PUSH_ROOT(stm_thread_local, NULL); + stm_collect(9); + + node_t *node2 = (node_t *)stm_allocate(sizeof(struct node_s)); + assert(node2 != node); + assert(node->value == 129821); + + STM_PUSH_ROOT(stm_thread_local, node2); + stm_collect(0); + STM_POP_ROOT(stm_thread_local, node2); + assert(node2 != node); + assert(node->value == 129821); + + return 0; +} diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -39,7 +39,7 @@ char *realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj); uintptr_t pagenum = (uintptr_t)obj / 4096UL; - assert(!is_shared_log_page(pagenum)); + OPT_ASSERT(!is_shared_log_page(pagenum)); assert(is_private_log_page_in(STM_SEGMENT->segment_num, pagenum)); assert(is_private_log_page_in(from_seg, pagenum)); @@ -229,7 +229,7 @@ } set_page_private_in(0, pagenum); - assert(is_private_log_page_in(my_segnum, pagenum)); + OPT_ASSERT(is_private_log_page_in(my_segnum, pagenum)); assert(!is_shared_log_page(pagenum)); } @@ -437,8 +437,9 @@ invoke_and_clear_user_callbacks(0); /* for commit */ s_mutex_lock(); + enter_safe_point_if_requested(); + assert(STM_SEGMENT->nursery_end == NURSERY_END); - assert(STM_SEGMENT->nursery_end == NURSERY_END); stm_rewind_jmp_forget(STM_SEGMENT->running_thread); if (globally_unique_transaction && STM_PSEGMENT->transaction_state == TS_INEVITABLE) { @@ -590,11 +591,9 @@ void _stm_become_inevitable(const char *msg) { - s_mutex_lock(); - enter_safe_point_if_requested(); - if (STM_PSEGMENT->transaction_state == TS_REGULAR) { dprintf(("become_inevitable: %s\n", msg)); + _stm_collectable_safe_point(); _validate_and_turn_inevitable(); STM_PSEGMENT->transaction_state = TS_INEVITABLE; @@ -604,8 +603,6 @@ else { assert(STM_PSEGMENT->transaction_state == TS_INEVITABLE); } - - s_mutex_unlock(); } void stm_become_globally_unique_transaction(stm_thread_local_t *tl, diff --git a/c8/stm/core.h b/c8/stm/core.h --- a/c8/stm/core.h +++ b/c8/stm/core.h @@ -130,7 +130,6 @@ static bool _is_tl_registered(stm_thread_local_t *tl); static bool _seems_to_be_running_transaction(void); -static void teardown_core(void); static void abort_with_mutex(void) __attribute__((noreturn)); static stm_thread_local_t *abort_with_mutex_no_longjmp(void); static void abort_data_structures_from_segment_num(int segment_num); diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c --- a/c8/stm/gcpage.c +++ b/c8/stm/gcpage.c @@ -19,9 +19,12 @@ } +static int lock_growth_large = 0; + static char *allocate_outside_nursery_large(uint64_t size) { /* XXX: real allocation */ + spinlock_acquire(lock_growth_large); char *addr = uninitialized_page_start; char *start = uninitialized_page_start; @@ -42,7 +45,7 @@ size, addr, get_segment_of_linear_address(addr), (addr - STM_SEGMENT->segment_base) / 4096UL)); - + spinlock_release(lock_growth_large); return addr; } diff --git a/c8/stm/list.h b/c8/stm/list.h --- a/c8/stm/list.h +++ b/c8/stm/list.h @@ -199,8 +199,8 @@ #define TREE_LOOP_END } } #define TREE_LOOP_END_AND_COMPRESS \ } if (_deleted_factor > 9) _tree_compress(_tree); } -#define TREE_LOOP_DELETE(tree, item) { (tree)->count--; (item)->addr = NULL; _deleted_factor += 6; } -#define TREE_FIND_DELETE(tree, item) { (tree)->count--; (item)->addr = NULL; } +#define TREE_LOOP_DELETE(tree, item) { (tree)->count--; (item)->addr = 0; _deleted_factor += 6; } +#define TREE_FIND_DELETE(tree, item) { (tree)->count--; (item)->addr = 0; } #define TREE_FIND(tree, addr1, result, goto_not_found) \ diff --git a/c8/stm/sync.c b/c8/stm/sync.c --- a/c8/stm/sync.c +++ b/c8/stm/sync.c @@ -137,11 +137,10 @@ } /* No segment available. Wait until release_thread_segment() signals that one segment has been freed. */ - abort(); /* XXX */ + cond_wait(C_SEGMENT_FREE); /* Return false to the caller, which will call us again */ return false; - got_num: sync_ctl.in_use1[num] = 1; assert(STM_SEGMENT->segment_num == num); @@ -154,6 +153,8 @@ { assert(_has_mutex()); + cond_signal(C_SEGMENT_FREE); + assert(STM_SEGMENT->running_thread == tl); STM_SEGMENT->running_thread = NULL; @@ -218,7 +219,7 @@ assert(_has_mutex()); long i; - for (i = 1; i <= NB_SEGMENTS; i++) { + for (i = 0; i < NB_SEGMENTS; i++) { if (get_segment(i)->nursery_end == NURSERY_END) get_segment(i)->nursery_end = NSE_SIGPAUSE; } @@ -234,7 +235,7 @@ long result = 0; int my_num = STM_SEGMENT->segment_num; - for (i = 1; i <= NB_SEGMENTS; i++) { + for (i = 0; i < NB_SEGMENTS; i++) { if (i != my_num && get_priv_segment(i)->safe_point == SP_RUNNING) { assert(get_segment(i)->nursery_end <= _STM_NSE_SIGNAL_MAX); result++; @@ -251,7 +252,7 @@ assert((_safe_points_requested = false, 1)); long i; - for (i = 1; i <= NB_SEGMENTS; i++) { + for (i = 0; i < NB_SEGMENTS; i++) { assert(get_segment(i)->nursery_end != NURSERY_END); if (get_segment(i)->nursery_end == NSE_SIGPAUSE) get_segment(i)->nursery_end = NURSERY_END; diff --git a/c8/stm/sync.h b/c8/stm/sync.h --- a/c8/stm/sync.h +++ b/c8/stm/sync.h @@ -4,6 +4,7 @@ enum cond_type_e { C_AT_SAFE_POINT, C_REQUEST_REMOVED, + C_SEGMENT_FREE, _C_TOTAL }; diff --git a/c8/stmgc.c b/c8/stmgc.c --- a/c8/stmgc.c +++ b/c8/stmgc.c @@ -2,28 +2,28 @@ #include "stmgc.h" #include "stm/atomic.h" #include "stm/list.h" +#include "stm/core.h" #include "stm/pagecopy.h" -#include "stm/core.h" #include "stm/pages.h" #include "stm/gcpage.h" -#include "stm/nursery.h" #include "stm/sync.h" #include "stm/setup.h" +#include "stm/nursery.h" +#include "stm/extra.h" #include "stm/fprintcolor.h" #include "stm/rewind_setjmp.h" -#include "stm/extra.h" +#include "stm/misc.c" #include "stm/list.c" #include "stm/pagecopy.c" +#include "stm/pages.c" +#include "stm/prebuilt.c" +#include "stm/gcpage.c" #include "stm/nursery.c" -#include "stm/core.c" -#include "stm/pages.c" -#include "stm/gcpage.c" #include "stm/sync.c" #include "stm/setup.c" +#include "stm/hash_id.c" +#include "stm/core.c" +#include "stm/extra.c" #include "stm/fprintcolor.c" #include "stm/rewind_setjmp.c" -#include "stm/hash_id.c" -#include "stm/prebuilt.c" -#include "stm/misc.c" -#include "stm/extra.c" diff --git a/c8/test/test_demo.py b/c8/test/test_demo.py new file mode 100644 --- /dev/null +++ b/c8/test/test_demo.py @@ -0,0 +1,40 @@ +import py +import os + + +class TestDemo: + + def _do(self, cmd): + print cmd + err = os.system(cmd) + if err: py.test.fail("'%s' failed (result %r)" % (cmd, err)) + + def make_and_run(self, target): + self._do("make -C ../demo %s" % target) + self._do("../demo/%s > /dev/null" % target) + + def test_shadowstack(self): + py.test.xfail("no major gc yet") + self.make_and_run("debug-test_shadowstack") + + def test_demo_simple_build(self): self.make_and_run("build-demo_simple") + + def test_demo_largemalloc_build(self): + py.test.xfail("no largemalloc") + self.make_and_run("build-demo_largemalloc") + + + + # def test_demo2_debug(self): self.make_and_run("debug-demo2") + def test_demo2_build(self): + py.test.xfail("no markers yet") + self.make_and_run("build-demo2") + # def test_demo2_release(self): self.make_and_run("release-demo2") + + # def test_demo_random_debug(self): self.make_and_run("debug-demo_random") + def test_demo_random_build(self): self.make_and_run("build-demo_random") + def test_demo_random_release(self): self.make_and_run("release-demo_random") + + # def test_demo_random2_debug(self): self.make_and_run("debug-demo_random2") + def test_demo_random2_build(self): self.make_and_run("build-demo_random2") + def test_demo_random2_release(self): self.make_and_run("release-demo_random2") From noreply at buildbot.pypy.org Wed Sep 10 15:25:35 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Wed, 10 Sep 2014 15:25:35 +0200 (CEST) Subject: [pypy-commit] stmgc default: uhm, try to do atomic privatization of pages Message-ID: <20140910132535.236811C073F@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1382:777200bdb987 Date: 2014-09-10 15:27 +0200 http://bitbucket.org/pypy/stmgc/changeset/777200bdb987/ Log: uhm, try to do atomic privatization of pages diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -224,8 +224,7 @@ for (i = 1; i < NB_SEGMENTS; i++) { assert(!is_private_log_page_in(i, pagenum)); - page_privatize_in(i, pagenum); - pagecopy((char*)(get_virt_page_of(i, pagenum) * 4096UL), src); + page_privatize_in(i, pagenum, src); } set_page_private_in(0, pagenum); @@ -256,23 +255,23 @@ struct object_s *bk_obj = malloc(obj_size); memcpy(bk_obj, realobj, obj_size); - if (is_shared_log_page(first_page)) { - /* acquire all privatization locks, make private and - read protect others */ - long i; - uintptr_t page; - for (i = 0; i < NB_SEGMENTS; i++) { - acquire_privatization_lock(i); - } - for (page = first_page; page <= end_page; page++) { + /* if there are shared pages, privatize them */ + + uintptr_t page; + for (page = first_page; page <= end_page; page++) { + if (is_shared_log_page(page)) { + long i; + for (i = 0; i < NB_SEGMENTS; i++) { + acquire_privatization_lock(i); + } if (is_shared_log_page(page)) _privatize_shared_page(page); - } - for (i = NB_SEGMENTS-1; i >= 0; i--) { - release_privatization_lock(i); + for (i = NB_SEGMENTS-1; i >= 0; i--) { + release_privatization_lock(i); + } } } - /* page not shared anymore. but we still may have + /* pages not shared anymore. but we still may have only a read protected page ourselves: */ acquire_privatization_lock(my_segnum); diff --git a/c8/stm/core.h b/c8/stm/core.h --- a/c8/stm/core.h +++ b/c8/stm/core.h @@ -32,6 +32,7 @@ #define FIRST_OLD_RM_PAGE (OLD_RM_START / 4096UL) #define NB_READMARKER_PAGES (FIRST_OBJECT_PAGE - FIRST_READMARKER_PAGE) +#define TMP_COPY_PAGE 1 /* HACK */ enum /* stm_flags */ { GCFLAG_WRITE_BARRIER = _STM_GCFLAG_WRITE_BARRIER, diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c --- a/c8/stm/gcpage.c +++ b/c8/stm/gcpage.c @@ -15,7 +15,14 @@ static void setup_N_pages(char *pages_addr, uint64_t num) { + long i; + for (i = 0; i < NB_SEGMENTS; i++) { + acquire_privatization_lock(i); + } pages_initialize_shared((pages_addr - stm_object_pages) / 4096UL, num); + for (i = NB_SEGMENTS-1; i >= 0; i--) { + release_privatization_lock(i); + } } @@ -43,7 +50,7 @@ } dprintf(("allocate_outside_nursery_large(%lu): %p, seg=%d, page=%lu\n", size, addr, get_segment_of_linear_address(addr), - (addr - STM_SEGMENT->segment_base) / 4096UL)); + (addr - get_segment_base(get_segment_of_linear_address(addr))) / 4096UL)); spinlock_release(lock_growth_large); return addr; diff --git a/c8/stm/pages.c b/c8/stm/pages.c --- a/c8/stm/pages.c +++ b/c8/stm/pages.c @@ -34,7 +34,8 @@ /* assert remappings follow the rule that page N in one segment can only be remapped to page N in another segment */ - assert(((addr - stm_object_pages) / 4096UL - pgoff) % NB_PAGES == 0); + assert(IMPLY(((addr - stm_object_pages) / 4096UL) != TMP_COPY_PAGE, + ((addr - stm_object_pages) / 4096UL - pgoff) % NB_PAGES == 0)); #ifdef USE_REMAP_FILE_PAGES int res = remap_file_pages(addr, size, 0, pgoff, 0); @@ -58,6 +59,12 @@ segment 0. */ dprintf(("pages_initialize_shared: 0x%ld - 0x%ld\n", pagenum, pagenum + count)); +#ifndef NDEBUG + long l; + for (l = 0; l < NB_SEGMENTS; l++) { + assert(get_priv_segment(l)->privatization_lock); + } +#endif assert(pagenum < NB_PAGES); if (count == 0) return; @@ -79,10 +86,15 @@ } } -static void page_privatize_in(int segnum, uintptr_t pagenum) + +static void page_privatize_in(int segnum, uintptr_t pagenum, char *initialize_from) { - /* hopefully holding the lock */ - assert(get_priv_segment(segnum)->privatization_lock); +#ifndef NDEBUG + long l; + for (l = 0; l < NB_SEGMENTS; l++) { + assert(get_priv_segment(l)->privatization_lock); + } +#endif /* check this thread's 'pages_privatized' bit */ uint64_t bitmask = 1UL << segnum; @@ -103,6 +115,14 @@ attempt to group together many calls to d_remap_file_pages() in succession) */ uintptr_t pagenum_in_file = NB_PAGES * segnum + pagenum; + char *tmp_page = stm_object_pages + TMP_COPY_PAGE * 4096UL; + /* first remap to TMP_PAGE, then copy stuff there (to the underlying + file page), then remap this file-page hopefully atomically to the + segnum's virtual page */ + d_remap_file_pages(tmp_page, 4096, pagenum_in_file); + pagecopy(tmp_page, initialize_from); + write_fence(); + char *new_page = stm_object_pages + pagenum_in_file * 4096UL; d_remap_file_pages(new_page, 4096, pagenum_in_file); } diff --git a/c8/stm/pages.h b/c8/stm/pages.h --- a/c8/stm/pages.h +++ b/c8/stm/pages.h @@ -40,7 +40,7 @@ static struct page_shared_s pages_privatized[PAGE_FLAG_END - PAGE_FLAG_START]; static void pages_initialize_shared(uintptr_t pagenum, uintptr_t count); -static void page_privatize_in(int segnum, uintptr_t pagenum); +static void page_privatize_in(int segnum, uintptr_t pagenum, char *initialize_from); static inline uintptr_t get_virt_page_of(long segnum, uintptr_t pagenum) { diff --git a/c8/stm/setup.c b/c8/stm/setup.c --- a/c8/stm/setup.c +++ b/c8/stm/setup.c @@ -63,11 +63,19 @@ NULL accesses land. We mprotect it so that accesses fail. */ mprotect(segment_base, 4096, PROT_NONE); + /* TMP_COPY_PAGE is used for atomic privatization */ + mprotect(segment_base + TMP_COPY_PAGE * 4096UL, + 4096UL, PROT_READ|PROT_WRITE); + /* Pages in range(2, FIRST_READMARKER_PAGE) are never used */ - if (FIRST_READMARKER_PAGE > 2) - mprotect(segment_base + 8192, - (FIRST_READMARKER_PAGE - 2) * 4096UL, + if (FIRST_READMARKER_PAGE > TMP_COPY_PAGE + 1) + mprotect(segment_base + (TMP_COPY_PAGE + 1) * 4096, + (FIRST_READMARKER_PAGE - TMP_COPY_PAGE - 1) * 4096UL, PROT_NONE); + + /* STM_SEGMENT */ + mprotect(segment_base + ((uintptr_t)STM_SEGMENT / 4096UL) * 4096UL, + 4096UL, PROT_READ|PROT_WRITE); } } @@ -75,11 +83,13 @@ void stm_setup(void) { /* Check that some values are acceptable */ + assert(TMP_COPY_PAGE > 0 && TMP_COPY_PAGE <= 1); + assert(TMP_COPY_PAGE * 4096 + 4096 <= ((uintptr_t)STM_SEGMENT)); + assert((uintptr_t)STM_SEGMENT == (uintptr_t)STM_PSEGMENT); + assert(((uintptr_t)STM_PSEGMENT) + sizeof(*STM_PSEGMENT) <= FIRST_READMARKER_PAGE*4096); + assert(NB_SEGMENTS <= NB_SEGMENTS_MAX); - assert(4096 <= ((uintptr_t)STM_SEGMENT)); - assert((uintptr_t)STM_SEGMENT == (uintptr_t)STM_PSEGMENT); - assert(((uintptr_t)STM_PSEGMENT) + sizeof(*STM_PSEGMENT) <= 8192); - assert(2 <= FIRST_READMARKER_PAGE); + assert(TMP_COPY_PAGE < FIRST_READMARKER_PAGE); assert(FIRST_READMARKER_PAGE * 4096UL <= READMARKER_START); assert(READMARKER_START < READMARKER_END); assert(READMARKER_END <= 4096UL * FIRST_OBJECT_PAGE); @@ -98,7 +108,7 @@ char *segment_base = get_segment_base(i); /* Fill the TLS page (page 1) with 0xDC, for debugging */ - memset(REAL_ADDRESS(segment_base, 4096), 0xDC, 4096); + memset(REAL_ADDRESS(segment_base, ((uintptr_t)STM_PSEGMENT/4096) * 4096), 0xDC, 4096); /* Make a "hole" at STM_PSEGMENT (which includes STM_SEGMENT) */ memset(REAL_ADDRESS(segment_base, STM_PSEGMENT), 0, sizeof(*STM_PSEGMENT)); diff --git a/c8/stmgc.h b/c8/stmgc.h --- a/c8/stmgc.h +++ b/c8/stmgc.h @@ -34,7 +34,7 @@ uintptr_t nursery_end; struct stm_thread_local_s *running_thread; }; -#define STM_SEGMENT ((stm_segment_info_t *)4352) +#define STM_SEGMENT ((stm_segment_info_t *)8192) struct stm_shadowentry_s { From noreply at buildbot.pypy.org Wed Sep 10 16:00:49 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Wed, 10 Sep 2014 16:00:49 +0200 (CEST) Subject: [pypy-commit] stmgc default: use pwrite & mmap instead of extra remap_file_pages call (not sure if actually better) Message-ID: <20140910140049.1B0761D235C@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1383:a20e5e7e942c Date: 2014-09-10 16:02 +0200 http://bitbucket.org/pypy/stmgc/changeset/a20e5e7e942c/ Log: use pwrite & mmap instead of extra remap_file_pages call (not sure if actually better) diff --git a/c8/stm/core.h b/c8/stm/core.h --- a/c8/stm/core.h +++ b/c8/stm/core.h @@ -32,8 +32,6 @@ #define FIRST_OLD_RM_PAGE (OLD_RM_START / 4096UL) #define NB_READMARKER_PAGES (FIRST_OBJECT_PAGE - FIRST_READMARKER_PAGE) -#define TMP_COPY_PAGE 1 /* HACK */ - enum /* stm_flags */ { GCFLAG_WRITE_BARRIER = _STM_GCFLAG_WRITE_BARRIER, GCFLAG_HAS_SHADOW = 0x02, diff --git a/c8/stm/pagecopy.h b/c8/stm/pagecopy.h --- a/c8/stm/pagecopy.h +++ b/c8/stm/pagecopy.h @@ -1,2 +1,2 @@ -static void pagecopy(void *dest, const void *src); // 4096 bytes +static void pagecopy(void *dest, const void *src) __attribute__((unused)); // 4096 bytes diff --git a/c8/stm/pages.c b/c8/stm/pages.c --- a/c8/stm/pages.c +++ b/c8/stm/pages.c @@ -1,8 +1,8 @@ #ifndef _STM_CORE_H_ # error "must be compiled via stmgc.c" #endif -#include +#include /************************************************************/ static void setup_pages(void) @@ -34,21 +34,14 @@ /* assert remappings follow the rule that page N in one segment can only be remapped to page N in another segment */ - assert(IMPLY(((addr - stm_object_pages) / 4096UL) != TMP_COPY_PAGE, - ((addr - stm_object_pages) / 4096UL - pgoff) % NB_PAGES == 0)); + assert(((addr - stm_object_pages) / 4096UL - pgoff) % NB_PAGES == 0); -#ifdef USE_REMAP_FILE_PAGES - int res = remap_file_pages(addr, size, 0, pgoff, 0); - if (UNLIKELY(res < 0)) - stm_fatalerror("remap_file_pages: %m"); -#else char *res = mmap(addr, size, PROT_READ | PROT_WRITE, (MAP_PAGES_FLAGS & ~MAP_ANONYMOUS) | MAP_FIXED, stm_object_pages_fd, pgoff * 4096UL); if (UNLIKELY(res != addr)) stm_fatalerror("mmap (remapping page): %m"); -#endif } @@ -115,14 +108,15 @@ attempt to group together many calls to d_remap_file_pages() in succession) */ uintptr_t pagenum_in_file = NB_PAGES * segnum + pagenum; - char *tmp_page = stm_object_pages + TMP_COPY_PAGE * 4096UL; - /* first remap to TMP_PAGE, then copy stuff there (to the underlying - file page), then remap this file-page hopefully atomically to the - segnum's virtual page */ - d_remap_file_pages(tmp_page, 4096, pagenum_in_file); - pagecopy(tmp_page, initialize_from); + char *new_page = stm_object_pages + pagenum_in_file * 4096UL; + + /* first write to the file page directly: */ + ssize_t written = pwrite(stm_object_pages_fd, initialize_from, 4096UL, + pagenum_in_file * 4096UL); + if (written != 4096) + stm_fatalerror("pwrite didn't write the whole page: %zd", written); + + /* now remap virtual page in segment to the new file page */ write_fence(); - - char *new_page = stm_object_pages + pagenum_in_file * 4096UL; d_remap_file_pages(new_page, 4096, pagenum_in_file); } diff --git a/c8/stm/pages.h b/c8/stm/pages.h --- a/c8/stm/pages.h +++ b/c8/stm/pages.h @@ -21,7 +21,6 @@ #define PAGE_FLAG_START END_NURSERY_PAGE #define PAGE_FLAG_END NB_PAGES -#define USE_REMAP_FILE_PAGES struct page_shared_s { #if NB_SEGMENTS <= 8 diff --git a/c8/stm/setup.c b/c8/stm/setup.c --- a/c8/stm/setup.c +++ b/c8/stm/setup.c @@ -2,27 +2,11 @@ # error "must be compiled via stmgc.c" #endif -#include -#ifdef USE_REMAP_FILE_PAGES -static char *setup_mmap(char *reason, int *ignored) -{ - char *result = mmap(NULL, TOTAL_MEMORY, - PROT_READ | PROT_WRITE, - MAP_PAGES_FLAGS, -1, 0); - if (result == MAP_FAILED) - stm_fatalerror("%s failed: %m", reason); - - return result; -} -static void close_fd_mmap(int ignored) -{ -} -#else #include /* For O_* constants */ static char *setup_mmap(char *reason, int *map_fd) { - char name[128]; + char name[128] = "/__stmgc_c8__"; /* Create the big shared memory object, and immediately unlink it. There is a small window where if this process is killed the @@ -51,7 +35,6 @@ { close(map_fd); } -#endif static void setup_protection_settings(void) { @@ -63,19 +46,13 @@ NULL accesses land. We mprotect it so that accesses fail. */ mprotect(segment_base, 4096, PROT_NONE); - /* TMP_COPY_PAGE is used for atomic privatization */ - mprotect(segment_base + TMP_COPY_PAGE * 4096UL, - 4096UL, PROT_READ|PROT_WRITE); - /* Pages in range(2, FIRST_READMARKER_PAGE) are never used */ - if (FIRST_READMARKER_PAGE > TMP_COPY_PAGE + 1) - mprotect(segment_base + (TMP_COPY_PAGE + 1) * 4096, - (FIRST_READMARKER_PAGE - TMP_COPY_PAGE - 1) * 4096UL, + if (FIRST_READMARKER_PAGE > 2) + mprotect(segment_base + 2 * 4096, + (FIRST_READMARKER_PAGE - 2) * 4096UL, PROT_NONE); - /* STM_SEGMENT */ - mprotect(segment_base + ((uintptr_t)STM_SEGMENT / 4096UL) * 4096UL, - 4096UL, PROT_READ|PROT_WRITE); + /* STM_SEGMENT is in page 1 */ } } @@ -83,13 +60,11 @@ void stm_setup(void) { /* Check that some values are acceptable */ - assert(TMP_COPY_PAGE > 0 && TMP_COPY_PAGE <= 1); - assert(TMP_COPY_PAGE * 4096 + 4096 <= ((uintptr_t)STM_SEGMENT)); + assert(4096 <= ((uintptr_t)STM_SEGMENT)); assert((uintptr_t)STM_SEGMENT == (uintptr_t)STM_PSEGMENT); assert(((uintptr_t)STM_PSEGMENT) + sizeof(*STM_PSEGMENT) <= FIRST_READMARKER_PAGE*4096); assert(NB_SEGMENTS <= NB_SEGMENTS_MAX); - assert(TMP_COPY_PAGE < FIRST_READMARKER_PAGE); assert(FIRST_READMARKER_PAGE * 4096UL <= READMARKER_START); assert(READMARKER_START < READMARKER_END); assert(READMARKER_END <= 4096UL * FIRST_OBJECT_PAGE); diff --git a/c8/stmgc.h b/c8/stmgc.h --- a/c8/stmgc.h +++ b/c8/stmgc.h @@ -34,7 +34,7 @@ uintptr_t nursery_end; struct stm_thread_local_s *running_thread; }; -#define STM_SEGMENT ((stm_segment_info_t *)8192) +#define STM_SEGMENT ((stm_segment_info_t *)4352) struct stm_shadowentry_s { From noreply at buildbot.pypy.org Wed Sep 10 16:47:38 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Wed, 10 Sep 2014 16:47:38 +0200 (CEST) Subject: [pypy-commit] pypy default: backout 5b5351812a83, breaks translation Message-ID: <20140910144738.217481D38F6@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r73416:2e37426702e6 Date: 2014-09-10 10:32 -0400 http://bitbucket.org/pypy/pypy/changeset/2e37426702e6/ Log: backout 5b5351812a83, breaks translation diff --git a/rpython/rlib/rpath.py b/rpython/rlib/rpath.py --- a/rpython/rlib/rpath.py +++ b/rpython/rlib/rpath.py @@ -4,7 +4,6 @@ import os, stat from rpython.rlib import rposix -from rpython.rlib.objectmodel import enforceargs # ____________________________________________________________ @@ -12,7 +11,6 @@ # Generic implementations in RPython for both POSIX and NT # - at enforceargs(str) def risdir(s): """Return true if the pathname refers to an existing directory.""" try: @@ -27,12 +25,10 @@ # POSIX-only implementations # - at enforceargs(str) def _posix_risabs(s): """Test whether a path is absolute""" return s.startswith('/') - at enforceargs(str) def _posix_rnormpath(path): """Normalize path, eliminating double slashes, etc.""" slash, dot = '/', '.' @@ -60,7 +56,6 @@ path = slash*initial_slashes + path return path or dot - at enforceargs(str) def _posix_rabspath(path): """Return an absolute, **non-normalized** path. **This version does not let exceptions propagate.**""" @@ -72,7 +67,6 @@ except OSError: return path - at enforceargs(str, str) def _posix_rjoin(a, b): """Join two pathname components, inserting '/' as needed. If the second component is an absolute path, the first one @@ -93,13 +87,11 @@ # NT-only implementations # - at enforceargs(str) def _nt_risabs(s): """Test whether a path is absolute""" s = _nt_rsplitdrive(s)[1] return s.startswith('/') or s.startswith('\\') - at enforceargs(str) def _nt_rnormpath(path): """Normalize path, eliminating double slashes, etc.""" backslash, dot = '\\', '.' @@ -150,7 +142,6 @@ comps.append(dot) return prefix + backslash.join(comps) - at enforceargs(str) def _nt_rabspath(path): try: if path == '': @@ -159,7 +150,6 @@ except OSError: return path - at enforceargs(str) def _nt_rsplitdrive(p): """Split a pathname into drive/UNC sharepoint and relative path specifiers. @@ -187,7 +177,6 @@ return p[:2], p[2:] return '', p - at enforceargs(str, str) def _nt_rjoin(path, p): """Join two or more pathname components, inserting "\\" as needed.""" result_drive, result_path = _nt_rsplitdrive(path) From noreply at buildbot.pypy.org Wed Sep 10 18:27:49 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Wed, 10 Sep 2014 18:27:49 +0200 (CEST) Subject: [pypy-commit] pypy use-file-star-for-file: merge default Message-ID: <20140910162749.1BEA01D236C@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: use-file-star-for-file Changeset: r73417:29584b2dd0e9 Date: 2014-09-10 11:05 -0400 http://bitbucket.org/pypy/pypy/changeset/29584b2dd0e9/ Log: merge default diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py --- a/rpython/rlib/rfile.py +++ b/rpython/rlib/rfile.py @@ -323,6 +323,7 @@ if not c_feof(ll_file): raise _error(ll_file) s = buf.str(returned_size) + assert s is not None return s def _readline1(self, raw_buf): From noreply at buildbot.pypy.org Wed Sep 10 18:27:50 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Wed, 10 Sep 2014 18:27:50 +0200 (CEST) Subject: [pypy-commit] pypy use-file-star-for-file: try to init sys streams at module startup Message-ID: <20140910162750.43F7C1D236C@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: use-file-star-for-file Changeset: r73418:6b12e0690a8b Date: 2014-09-10 12:27 -0400 http://bitbucket.org/pypy/pypy/changeset/6b12e0690a8b/ Log: try to init sys streams at module startup diff --git a/pypy/module/_file/interp_file.py b/pypy/module/_file/interp_file.py --- a/pypy/module/_file/interp_file.py +++ b/pypy/module/_file/interp_file.py @@ -533,6 +533,9 @@ class FileState: def __init__(self, space): + self._cleanup_() + + def _cleanup_(self): self.openstreams = {} def getopenstreams(space): diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py --- a/pypy/module/sys/__init__.py +++ b/pypy/module/sys/__init__.py @@ -104,6 +104,9 @@ } def startup(self, space): + from pypy.module.sys.state import getio + getio(space).startup(space) + if space.config.translating and not we_are_translated(): # don't get the filesystemencoding at translation time assert self.filesystemencoding is None diff --git a/pypy/module/sys/state.py b/pypy/module/sys/state.py --- a/pypy/module/sys/state.py +++ b/pypy/module/sys/state.py @@ -31,8 +31,10 @@ class IOState: def __init__(self, space): + pass + + def startup(self, space): from pypy.module._file.interp_file import W_File - self.space = space stdin = W_File(space) stdin.file_fdopen(0, "r", 1) From noreply at buildbot.pypy.org Wed Sep 10 23:25:30 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Wed, 10 Sep 2014 23:25:30 +0200 (CEST) Subject: [pypy-commit] pypy use-file-star-for-file: expect IOError here Message-ID: <20140910212530.241021C0350@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: use-file-star-for-file Changeset: r73419:9ac52f581888 Date: 2014-09-10 15:31 -0400 http://bitbucket.org/pypy/pypy/changeset/9ac52f581888/ Log: expect IOError here diff --git a/pypy/module/_file/__init__.py b/pypy/module/_file/__init__.py --- a/pypy/module/_file/__init__.py +++ b/pypy/module/_file/__init__.py @@ -24,7 +24,7 @@ else: try: stream.flush() - except OSError: + except IOError: pass def setup_after_space_initialization(self): From noreply at buildbot.pypy.org Wed Sep 10 23:25:31 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Wed, 10 Sep 2014 23:25:31 +0200 (CEST) Subject: [pypy-commit] pypy default: cleanup Message-ID: <20140910212531.560821C0350@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r73420:29beaf084002 Date: 2014-09-10 15:31 -0400 http://bitbucket.org/pypy/pypy/changeset/29beaf084002/ Log: cleanup diff --git a/rpython/annotator/builtin.py b/rpython/annotator/builtin.py --- a/rpython/annotator/builtin.py +++ b/rpython/annotator/builtin.py @@ -255,6 +255,11 @@ BUILTIN_ANALYZERS[original] = value + at analyzer_for(getattr(object.__init__, 'im_func', object.__init__)) +def object_init(s_self, *args): + # ignore - mostly used for abstract classes initialization + pass + @analyzer_for(getattr(EnvironmentError.__init__, 'im_func', EnvironmentError.__init__)) def EnvironmentError_init(s_self, *args): pass @@ -268,11 +273,6 @@ def WindowsError_init(s_self, *args): pass - at analyzer_for(getattr(object.__init__, 'im_func', object.__init__)) -def object_init(s_self, *args): - # ignore - mostly used for abstract classes initialization - pass - @analyzer_for(sys.getdefaultencoding) def conf(): diff --git a/rpython/annotator/classdef.py b/rpython/annotator/classdef.py --- a/rpython/annotator/classdef.py +++ b/rpython/annotator/classdef.py @@ -438,7 +438,9 @@ # ____________________________________________________________ FORCE_ATTRIBUTES_INTO_CLASSES = { - EnvironmentError: {'errno': SomeInteger(), 'strerror': SomeString(can_be_None=True), 'filename': SomeString(can_be_None=True)}, + EnvironmentError: {'errno': SomeInteger(), + 'strerror': SomeString(can_be_None=True), + 'filename': SomeString(can_be_None=True)}, } try: @@ -455,4 +457,3 @@ else: FORCE_ATTRIBUTES_INTO_CLASSES[termios.error] = \ {'args': SomeTuple([SomeInteger(), SomeString()])} - diff --git a/rpython/rtyper/rbuiltin.py b/rpython/rtyper/rbuiltin.py --- a/rpython/rtyper/rbuiltin.py +++ b/rpython/rtyper/rbuiltin.py @@ -259,9 +259,6 @@ return i1 return i2 -def rtype_Exception__init__(hop): - hop.exception_cannot_occur() - def rtype_object__init__(hop): hop.exception_cannot_occur() @@ -341,6 +338,9 @@ original = getattr(__builtin__, name[14:]) BUILTIN_TYPER[original] = value +BUILTIN_TYPER[getattr(object.__init__, 'im_func', object.__init__)] = ( + rtype_object__init__) + BUILTIN_TYPER[getattr(EnvironmentError.__init__, 'im_func', EnvironmentError.__init__)] = ( rtype_EnvironmentError__init__) @@ -353,8 +353,6 @@ getattr(WindowsError.__init__, 'im_func', WindowsError.__init__)] = ( rtype_WindowsError__init__) -BUILTIN_TYPER[getattr(object.__init__, 'im_func', object.__init__)] = ( - rtype_object__init__) # annotation of low-level types def rtype_malloc(hop, i_flavor=None, i_zero=None, i_track_allocation=None, From noreply at buildbot.pypy.org Wed Sep 10 23:25:32 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Wed, 10 Sep 2014 23:25:32 +0200 (CEST) Subject: [pypy-commit] pypy use-file-star-for-file: merge default Message-ID: <20140910212532.74C5B1C0350@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: use-file-star-for-file Changeset: r73421:c5f07d6d5b66 Date: 2014-09-10 17:24 -0400 http://bitbucket.org/pypy/pypy/changeset/c5f07d6d5b66/ Log: merge default diff --git a/rpython/annotator/builtin.py b/rpython/annotator/builtin.py --- a/rpython/annotator/builtin.py +++ b/rpython/annotator/builtin.py @@ -255,6 +255,11 @@ BUILTIN_ANALYZERS[original] = value + at analyzer_for(getattr(object.__init__, 'im_func', object.__init__)) +def object_init(s_self, *args): + # ignore - mostly used for abstract classes initialization + pass + @analyzer_for(getattr(EnvironmentError.__init__, 'im_func', EnvironmentError.__init__)) def EnvironmentError_init(s_self, *args): pass @@ -268,11 +273,6 @@ def WindowsError_init(s_self, *args): pass - at analyzer_for(getattr(object.__init__, 'im_func', object.__init__)) -def object_init(s_self, *args): - # ignore - mostly used for abstract classes initialization - pass - @analyzer_for(sys.getdefaultencoding) def conf(): diff --git a/rpython/annotator/classdef.py b/rpython/annotator/classdef.py --- a/rpython/annotator/classdef.py +++ b/rpython/annotator/classdef.py @@ -438,7 +438,9 @@ # ____________________________________________________________ FORCE_ATTRIBUTES_INTO_CLASSES = { - EnvironmentError: {'errno': SomeInteger(), 'strerror': SomeString(can_be_None=True), 'filename': SomeString(can_be_None=True)}, + EnvironmentError: {'errno': SomeInteger(), + 'strerror': SomeString(can_be_None=True), + 'filename': SomeString(can_be_None=True)}, } try: @@ -455,4 +457,3 @@ else: FORCE_ATTRIBUTES_INTO_CLASSES[termios.error] = \ {'args': SomeTuple([SomeInteger(), SomeString()])} - diff --git a/rpython/rtyper/rbuiltin.py b/rpython/rtyper/rbuiltin.py --- a/rpython/rtyper/rbuiltin.py +++ b/rpython/rtyper/rbuiltin.py @@ -259,9 +259,6 @@ return i1 return i2 -def rtype_Exception__init__(hop): - hop.exception_cannot_occur() - def rtype_object__init__(hop): hop.exception_cannot_occur() @@ -341,6 +338,9 @@ original = getattr(__builtin__, name[14:]) BUILTIN_TYPER[original] = value +BUILTIN_TYPER[getattr(object.__init__, 'im_func', object.__init__)] = ( + rtype_object__init__) + BUILTIN_TYPER[getattr(EnvironmentError.__init__, 'im_func', EnvironmentError.__init__)] = ( rtype_EnvironmentError__init__) @@ -353,8 +353,6 @@ getattr(WindowsError.__init__, 'im_func', WindowsError.__init__)] = ( rtype_WindowsError__init__) -BUILTIN_TYPER[getattr(object.__init__, 'im_func', object.__init__)] = ( - rtype_object__init__) # annotation of low-level types def rtype_malloc(hop, i_flavor=None, i_zero=None, i_track_allocation=None, From noreply at buildbot.pypy.org Thu Sep 11 00:39:12 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 11 Sep 2014 00:39:12 +0200 (CEST) Subject: [pypy-commit] pypy use-file-star-for-file: another approach for sys io init Message-ID: <20140910223912.8E1001D235C@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: use-file-star-for-file Changeset: r73422:1899f1699c85 Date: 2014-09-10 18:24 -0400 http://bitbucket.org/pypy/pypy/changeset/1899f1699c85/ Log: another approach for sys io init diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py --- a/pypy/module/sys/__init__.py +++ b/pypy/module/sys/__init__.py @@ -104,9 +104,6 @@ } def startup(self, space): - from pypy.module.sys.state import getio - getio(space).startup(space) - if space.config.translating and not we_are_translated(): # don't get the filesystemencoding at translation time assert self.filesystemencoding is None diff --git a/pypy/module/sys/state.py b/pypy/module/sys/state.py --- a/pypy/module/sys/state.py +++ b/pypy/module/sys/state.py @@ -31,9 +31,15 @@ class IOState: def __init__(self, space): - pass + self._cleanup_() + + def _cleanup_(self): + self.w_stdin = self.w_stdout = self.w_stderr = None def startup(self, space): + if self.w_stdout is not None: + return + from pypy.module._file.interp_file import W_File stdin = W_File(space) @@ -52,7 +58,9 @@ self.w_stderr = space.wrap(stderr) def getio(space): - return space.fromcache(IOState) + io = space.fromcache(IOState) + io.startup(space) + return io def pypy_getudir(space): """NOT_RPYTHON From noreply at buildbot.pypy.org Thu Sep 11 00:39:13 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 11 Sep 2014 00:39:13 +0200 (CEST) Subject: [pypy-commit] pypy default: cleanup Message-ID: <20140910223913.BFD611D235C@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r73423:b5d8f4d2a81e Date: 2014-09-10 18:25 -0400 http://bitbucket.org/pypy/pypy/changeset/b5d8f4d2a81e/ Log: cleanup diff --git a/pypy/module/sys/state.py b/pypy/module/sys/state.py --- a/pypy/module/sys/state.py +++ b/pypy/module/sys/state.py @@ -7,15 +7,15 @@ # ____________________________________________________________ # -class State: - def __init__(self, space): - self.space = space +class State: + def __init__(self, space): + self.space = space self.w_modules = space.newdict(module=True) - self.w_warnoptions = space.newlist([]) self.w_argv = space.newlist([]) - self.setinitialpath(space) + + self.setinitialpath(space) def setinitialpath(self, space): from pypy.module.sys.initpath import compute_stdlib_path @@ -25,10 +25,10 @@ path = compute_stdlib_path(self, srcdir) self.w_path = space.newlist([space.wrap(p) for p in path]) - def get(space): return space.fromcache(State) + class IOState: def __init__(self, space): from pypy.module._file.interp_file import W_File @@ -54,9 +54,9 @@ def getio(space): return space.fromcache(IOState) + def pypy_getudir(space): """NOT_RPYTHON (should be removed from interpleveldefs before translation)""" from rpython.tool.udir import udir return space.wrap(str(udir)) - From noreply at buildbot.pypy.org Thu Sep 11 00:39:14 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 11 Sep 2014 00:39:14 +0200 (CEST) Subject: [pypy-commit] pypy use-file-star-for-file: merge default Message-ID: <20140910223914.EE62F1D235C@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: use-file-star-for-file Changeset: r73424:adf0c81a2d15 Date: 2014-09-10 18:25 -0400 http://bitbucket.org/pypy/pypy/changeset/adf0c81a2d15/ Log: merge default diff --git a/pypy/module/sys/state.py b/pypy/module/sys/state.py --- a/pypy/module/sys/state.py +++ b/pypy/module/sys/state.py @@ -7,15 +7,15 @@ # ____________________________________________________________ # -class State: - def __init__(self, space): - self.space = space +class State: + def __init__(self, space): + self.space = space self.w_modules = space.newdict(module=True) - self.w_warnoptions = space.newlist([]) self.w_argv = space.newlist([]) - self.setinitialpath(space) + + self.setinitialpath(space) def setinitialpath(self, space): from pypy.module.sys.initpath import compute_stdlib_path @@ -25,10 +25,10 @@ path = compute_stdlib_path(self, srcdir) self.w_path = space.newlist([space.wrap(p) for p in path]) - def get(space): return space.fromcache(State) + class IOState: def __init__(self, space): self._cleanup_() @@ -62,9 +62,9 @@ io.startup(space) return io + def pypy_getudir(space): """NOT_RPYTHON (should be removed from interpleveldefs before translation)""" from rpython.tool.udir import udir return space.wrap(str(udir)) - From noreply at buildbot.pypy.org Thu Sep 11 01:26:24 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 11 Sep 2014 01:26:24 +0200 (CEST) Subject: [pypy-commit] pypy default: support stdin/stdout/stderr in rfile Message-ID: <20140910232624.50B121C0350@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r73425:500069cd3e5f Date: 2014-09-10 18:57 -0400 http://bitbucket.org/pypy/pypy/changeset/500069cd3e5f/ Log: support stdin/stdout/stderr in rfile diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py --- a/rpython/rlib/rfile.py +++ b/rpython/rlib/rfile.py @@ -38,6 +38,7 @@ FILEP = rffi.COpaquePtr("FILE") OFF_T = config['off_t'] + _IONBF = config['_IONBF'] _IOLBF = config['_IOLBF'] _IOFBF = config['_IOFBF'] @@ -95,6 +96,10 @@ c_ferror = llexternal('ferror', [FILEP], rffi.INT) c_clearerr = llexternal('clearerr', [FILEP], lltype.Void) +c_stdin = rffi.CExternVariable(FILEP, 'stdin', eci, c_type='FILE*')[0] +c_stdout = rffi.CExternVariable(FILEP, 'stdout', eci, c_type='FILE*')[0] +c_stderr = rffi.CExternVariable(FILEP, 'stderr', eci, c_type='FILE*')[0] + def _error(ll_file): err = c_ferror(ll_file) @@ -199,6 +204,14 @@ return RFile(ll_file, close2=_pclose2) +def create_stdio(): + close2 = [None, None] + stdin = RFile(c_stdin(), close2=close2) + stdout = RFile(c_stdout(), close2=close2) + stderr = RFile(c_stderr(), close2=close2) + return stdin, stdout, stderr + + class RFile(object): _univ_newline = False _newlinetypes = NEWLINE_UNKNOWN @@ -218,7 +231,8 @@ ll_file = self._ll_file if ll_file: do_close = self._close2[1] - do_close(ll_file) # return value ignored + if do_close: + do_close(ll_file) # return value ignored def _cleanup_(self): self._ll_file = lltype.nullptr(FILEP.TO) @@ -237,10 +251,11 @@ # double close is allowed self._ll_file = lltype.nullptr(FILEP.TO) do_close = self._close2[0] - res = do_close(ll_file) - if res == -1: - errno = rposix.get_errno() - raise IOError(errno, os.strerror(errno)) + if do_close: + res = do_close(ll_file) + if res == -1: + errno = rposix.get_errno() + raise IOError(errno, os.strerror(errno)) return res def _check_closed(self): diff --git a/rpython/rlib/test/test_rfile.py b/rpython/rlib/test/test_rfile.py --- a/rpython/rlib/test/test_rfile.py +++ b/rpython/rlib/test/test_rfile.py @@ -351,6 +351,13 @@ cls.tmpdir = udir.join('test_rfile_direct') cls.tmpdir.ensure(dir=True) + def test_stdio(self): + i, o, e = rfile.create_stdio() + o.write("test\n") + i.close() + o.close() + e.close() + def test_auto_close(self): fname = str(self.tmpdir.join('file_auto_close')) f = rfile.create_file(fname, 'w') From noreply at buildbot.pypy.org Thu Sep 11 01:26:25 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 11 Sep 2014 01:26:25 +0200 (CEST) Subject: [pypy-commit] pypy use-file-star-for-file: merge default Message-ID: <20140910232625.842B11C0350@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: use-file-star-for-file Changeset: r73426:ee8225bda67c Date: 2014-09-10 18:59 -0400 http://bitbucket.org/pypy/pypy/changeset/ee8225bda67c/ Log: merge default diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py --- a/rpython/rlib/rfile.py +++ b/rpython/rlib/rfile.py @@ -38,6 +38,7 @@ FILEP = rffi.COpaquePtr("FILE") OFF_T = config['off_t'] + _IONBF = config['_IONBF'] _IOLBF = config['_IOLBF'] _IOFBF = config['_IOFBF'] @@ -95,6 +96,10 @@ c_ferror = llexternal('ferror', [FILEP], rffi.INT) c_clearerr = llexternal('clearerr', [FILEP], lltype.Void) +c_stdin = rffi.CExternVariable(FILEP, 'stdin', eci, c_type='FILE*')[0] +c_stdout = rffi.CExternVariable(FILEP, 'stdout', eci, c_type='FILE*')[0] +c_stderr = rffi.CExternVariable(FILEP, 'stderr', eci, c_type='FILE*')[0] + def _error(ll_file): err = c_ferror(ll_file) @@ -199,6 +204,14 @@ return RFile(ll_file, close2=_pclose2) +def create_stdio(): + close2 = [None, None] + stdin = RFile(c_stdin(), close2=close2) + stdout = RFile(c_stdout(), close2=close2) + stderr = RFile(c_stderr(), close2=close2) + return stdin, stdout, stderr + + class RFile(object): _univ_newline = False _newlinetypes = NEWLINE_UNKNOWN @@ -218,7 +231,8 @@ ll_file = self._ll_file if ll_file: do_close = self._close2[1] - do_close(ll_file) # return value ignored + if do_close: + do_close(ll_file) # return value ignored def _cleanup_(self): self._ll_file = lltype.nullptr(FILEP.TO) @@ -237,10 +251,11 @@ # double close is allowed self._ll_file = lltype.nullptr(FILEP.TO) do_close = self._close2[0] - res = do_close(ll_file) - if res == -1: - errno = rposix.get_errno() - raise IOError(errno, os.strerror(errno)) + if do_close: + res = do_close(ll_file) + if res == -1: + errno = rposix.get_errno() + raise IOError(errno, os.strerror(errno)) return res def _check_closed(self): diff --git a/rpython/rlib/test/test_rfile.py b/rpython/rlib/test/test_rfile.py --- a/rpython/rlib/test/test_rfile.py +++ b/rpython/rlib/test/test_rfile.py @@ -351,6 +351,13 @@ cls.tmpdir = udir.join('test_rfile_direct') cls.tmpdir.ensure(dir=True) + def test_stdio(self): + i, o, e = rfile.create_stdio() + o.write("test\n") + i.close() + o.close() + e.close() + def test_auto_close(self): fname = str(self.tmpdir.join('file_auto_close')) f = rfile.create_file(fname, 'w') From noreply at buildbot.pypy.org Thu Sep 11 01:26:26 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 11 Sep 2014 01:26:26 +0200 (CEST) Subject: [pypy-commit] pypy default: test/fix stdin/stdout/stderr file name Message-ID: <20140910232626.B40D81C0350@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r73427:f9915af38176 Date: 2014-09-10 19:08 -0400 http://bitbucket.org/pypy/pypy/changeset/f9915af38176/ Log: test/fix stdin/stdout/stderr file name diff --git a/pypy/module/sys/state.py b/pypy/module/sys/state.py --- a/pypy/module/sys/state.py +++ b/pypy/module/sys/state.py @@ -36,17 +36,17 @@ stdin = W_File(space) stdin.file_fdopen(0, "r", 1) - stdin.name = '' + stdin.w_name = space.wrap('') self.w_stdin = space.wrap(stdin) stdout = W_File(space) stdout.file_fdopen(1, "w", 1) - stdout.name = '' + stdout.w_name = space.wrap('') self.w_stdout = space.wrap(stdout) stderr = W_File(space) stderr.file_fdopen(2, "w", 0) - stderr.name = '' + stderr.w_name = space.wrap('') self.w_stderr = space.wrap(stderr) stdin._when_reading_first_flush(stdout) diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py --- a/pypy/module/sys/test/test_sysmodule.py +++ b/pypy/module/sys/test/test_sysmodule.py @@ -91,6 +91,10 @@ assert isinstance(sys.__stderr__, file) assert isinstance(sys.__stdin__, file) + assert sys.__stdin__.name == "" + assert sys.__stdout__.name == "" + assert sys.__stderr__.name == "" + if self.appdirect and not isinstance(sys.stdin, file): return From noreply at buildbot.pypy.org Thu Sep 11 01:26:27 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 11 Sep 2014 01:26:27 +0200 (CEST) Subject: [pypy-commit] pypy use-file-star-for-file: use rfile.create_stdio() in sys Message-ID: <20140910232627.E71CF1C0350@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: use-file-star-for-file Changeset: r73428:036547102c48 Date: 2014-09-10 19:09 -0400 http://bitbucket.org/pypy/pypy/changeset/036547102c48/ Log: use rfile.create_stdio() in sys diff --git a/pypy/module/sys/state.py b/pypy/module/sys/state.py --- a/pypy/module/sys/state.py +++ b/pypy/module/sys/state.py @@ -4,6 +4,9 @@ import os import pypy +from rpython.rlib import rfile +from pypy.module._file.interp_file import W_File + # ____________________________________________________________ # @@ -40,21 +43,18 @@ if self.w_stdout is not None: return - from pypy.module._file.interp_file import W_File + i, o, e = rfile.create_stdio() stdin = W_File(space) - stdin.file_fdopen(0, "r", 1) - stdin.name = '' + stdin.fdopenstream(i, "r", space.wrap("")) self.w_stdin = space.wrap(stdin) stdout = W_File(space) - stdout.file_fdopen(1, "w", 1) - stdout.name = '' + stdout.fdopenstream(o, "w", space.wrap("")) self.w_stdout = space.wrap(stdout) stderr = W_File(space) - stderr.file_fdopen(2, "w", 0) - stderr.name = '' + stderr.fdopenstream(e, "w", space.wrap("")) self.w_stderr = space.wrap(stderr) def getio(space): From noreply at buildbot.pypy.org Thu Sep 11 01:26:29 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 11 Sep 2014 01:26:29 +0200 (CEST) Subject: [pypy-commit] pypy use-file-star-for-file: merge default Message-ID: <20140910232629.2649D1C0350@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: use-file-star-for-file Changeset: r73429:4ff11b806c16 Date: 2014-09-10 19:13 -0400 http://bitbucket.org/pypy/pypy/changeset/4ff11b806c16/ Log: merge default diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py --- a/pypy/module/sys/test/test_sysmodule.py +++ b/pypy/module/sys/test/test_sysmodule.py @@ -91,6 +91,10 @@ assert isinstance(sys.__stderr__, file) assert isinstance(sys.__stdin__, file) + assert sys.__stdin__.name == "" + assert sys.__stdout__.name == "" + assert sys.__stderr__.name == "" + if self.appdirect and not isinstance(sys.stdin, file): return From noreply at buildbot.pypy.org Thu Sep 11 01:33:13 2014 From: noreply at buildbot.pypy.org (dansanders) Date: Thu, 11 Sep 2014 01:33:13 +0200 (CEST) Subject: [pypy-commit] pypy default: Fix memory leak in sandbox I/O. Message-ID: <20140910233313.A40AF1C0350@cobra.cs.uni-duesseldorf.de> Author: Dan Sanders Branch: Changeset: r73430:7f1e404f5238 Date: 2014-09-10 19:04 +0000 http://bitbucket.org/pypy/pypy/changeset/7f1e404f5238/ Log: Fix memory leak in sandbox I/O. diff --git a/rpython/translator/sandbox/rsandbox.py b/rpython/translator/sandbox/rsandbox.py --- a/rpython/translator/sandbox/rsandbox.py +++ b/rpython/translator/sandbox/rsandbox.py @@ -61,13 +61,16 @@ def need_more_data(self): buflen = self.buflen buf = lltype.malloc(rffi.CCHARP.TO, buflen, flavor='raw') - buflen = rffi.cast(rffi.SIZE_T, buflen) - count = ll_read_not_sandboxed(self.fd, buf, buflen) - count = rffi.cast(lltype.Signed, count) - if count <= 0: - raise IOError - self.buf += ''.join([buf[i] for i in range(count)]) - self.buflen *= 2 + try: + buflen = rffi.cast(rffi.SIZE_T, buflen) + count = ll_read_not_sandboxed(self.fd, buf, buflen) + count = rffi.cast(lltype.Signed, count) + if count <= 0: + raise IOError + self.buf += ''.join([buf[i] for i in range(count)]) + self.buflen *= 2 + finally: + lltype.free(buf, flavor='raw') def sandboxed_io(buf): STDIN = 0 From noreply at buildbot.pypy.org Thu Sep 11 01:33:14 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 11 Sep 2014 01:33:14 +0200 (CEST) Subject: [pypy-commit] pypy default: Merged in dansanders/pypy (pull request #278) Message-ID: <20140910233314.EBD761C0350@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r73431:6cfe51b6af4c Date: 2014-09-10 16:32 -0700 http://bitbucket.org/pypy/pypy/changeset/6cfe51b6af4c/ Log: Merged in dansanders/pypy (pull request #278) Fix memory leak in sandbox I/O. diff --git a/rpython/translator/sandbox/rsandbox.py b/rpython/translator/sandbox/rsandbox.py --- a/rpython/translator/sandbox/rsandbox.py +++ b/rpython/translator/sandbox/rsandbox.py @@ -61,13 +61,16 @@ def need_more_data(self): buflen = self.buflen buf = lltype.malloc(rffi.CCHARP.TO, buflen, flavor='raw') - buflen = rffi.cast(rffi.SIZE_T, buflen) - count = ll_read_not_sandboxed(self.fd, buf, buflen) - count = rffi.cast(lltype.Signed, count) - if count <= 0: - raise IOError - self.buf += ''.join([buf[i] for i in range(count)]) - self.buflen *= 2 + try: + buflen = rffi.cast(rffi.SIZE_T, buflen) + count = ll_read_not_sandboxed(self.fd, buf, buflen) + count = rffi.cast(lltype.Signed, count) + if count <= 0: + raise IOError + self.buf += ''.join([buf[i] for i in range(count)]) + self.buflen *= 2 + finally: + lltype.free(buf, flavor='raw') def sandboxed_io(buf): STDIN = 0 From noreply at buildbot.pypy.org Thu Sep 11 02:23:00 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 11 Sep 2014 02:23:00 +0200 (CEST) Subject: [pypy-commit] pypy use-file-star-for-file: backout 1899f1699c85 Message-ID: <20140911002301.0AF9C1D3984@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: use-file-star-for-file Changeset: r73432:0bcc5f7964fd Date: 2014-09-10 19:56 -0400 http://bitbucket.org/pypy/pypy/changeset/0bcc5f7964fd/ Log: backout 1899f1699c85 diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py --- a/pypy/module/sys/__init__.py +++ b/pypy/module/sys/__init__.py @@ -104,6 +104,9 @@ } def startup(self, space): + from pypy.module.sys.state import getio + getio(space).startup(space) + if space.config.translating and not we_are_translated(): # don't get the filesystemencoding at translation time assert self.filesystemencoding is None diff --git a/pypy/module/sys/state.py b/pypy/module/sys/state.py --- a/pypy/module/sys/state.py +++ b/pypy/module/sys/state.py @@ -34,15 +34,9 @@ class IOState: def __init__(self, space): - self._cleanup_() - - def _cleanup_(self): - self.w_stdin = self.w_stdout = self.w_stderr = None + pass def startup(self, space): - if self.w_stdout is not None: - return - i, o, e = rfile.create_stdio() stdin = W_File(space) @@ -58,9 +52,7 @@ self.w_stderr = space.wrap(stderr) def getio(space): - io = space.fromcache(IOState) - io.startup(space) - return io + return space.fromcache(IOState) def pypy_getudir(space): From noreply at buildbot.pypy.org Thu Sep 11 02:23:02 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 11 Sep 2014 02:23:02 +0200 (CEST) Subject: [pypy-commit] pypy use-file-star-for-file: try another sys stdio init approach Message-ID: <20140911002302.435E01D3984@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: use-file-star-for-file Changeset: r73433:9ed68e57bad0 Date: 2014-09-10 20:06 -0400 http://bitbucket.org/pypy/pypy/changeset/9ed68e57bad0/ Log: try another sys stdio init approach diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py --- a/pypy/module/sys/__init__.py +++ b/pypy/module/sys/__init__.py @@ -29,12 +29,6 @@ 'maxsize' : 'space.wrap(sys.maxint)', 'byteorder' : 'space.wrap(sys.byteorder)', 'maxunicode' : 'space.wrap(vm.MAXUNICODE)', - 'stdin' : 'state.getio(space).w_stdin', - '__stdin__' : 'state.getio(space).w_stdin', - 'stdout' : 'state.getio(space).w_stdout', - '__stdout__' : 'state.getio(space).w_stdout', - 'stderr' : 'state.getio(space).w_stderr', - '__stderr__' : 'state.getio(space).w_stderr', 'pypy_objspaceclass' : 'space.wrap(repr(space))', #'prefix' : # added by pypy_initial_path() when it #'exec_prefix' : # succeeds, pointing to trunk or /usr @@ -105,7 +99,15 @@ def startup(self, space): from pypy.module.sys.state import getio - getio(space).startup(space) + io = getio(space) + io.startup(space) + + for k in ["stdin", "__stdin__"]: + space.setitem(self.w_dict, space.wrap(k), io.stdin) + for k in ["stdout", "__stdout__"]: + space.setitem(self.w_dict, space.wrap(k), io.stdout) + for k in ["stderr", "__stderr__"]: + space.setitem(self.w_dict, space.wrap(k), io.stderr) if space.config.translating and not we_are_translated(): # don't get the filesystemencoding at translation time diff --git a/pypy/module/sys/state.py b/pypy/module/sys/state.py --- a/pypy/module/sys/state.py +++ b/pypy/module/sys/state.py @@ -41,15 +41,15 @@ stdin = W_File(space) stdin.fdopenstream(i, "r", space.wrap("")) - self.w_stdin = space.wrap(stdin) + self.stdin = stdin stdout = W_File(space) stdout.fdopenstream(o, "w", space.wrap("")) - self.w_stdout = space.wrap(stdout) + self.stdout = stdout stderr = W_File(space) stderr.fdopenstream(e, "w", space.wrap("")) - self.w_stderr = space.wrap(stderr) + self.stderr = stderr def getio(space): return space.fromcache(IOState) From noreply at buildbot.pypy.org Thu Sep 11 02:23:04 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 11 Sep 2014 02:23:04 +0200 (CEST) Subject: [pypy-commit] pypy default: fix translation Message-ID: <20140911002304.207481D3984@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r73434:ba969ea17c8e Date: 2014-09-10 20:22 -0400 http://bitbucket.org/pypy/pypy/changeset/ba969ea17c8e/ Log: fix translation diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py --- a/rpython/rlib/rfile.py +++ b/rpython/rlib/rfile.py @@ -205,7 +205,7 @@ def create_stdio(): - close2 = [None, None] + close2 = (None, None) stdin = RFile(c_stdin(), close2=close2) stdout = RFile(c_stdout(), close2=close2) stderr = RFile(c_stderr(), close2=close2) From noreply at buildbot.pypy.org Thu Sep 11 02:23:05 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 11 Sep 2014 02:23:05 +0200 (CEST) Subject: [pypy-commit] pypy use-file-star-for-file: merge default Message-ID: <20140911002305.B084F1D3984@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: use-file-star-for-file Changeset: r73435:99c40ef45b66 Date: 2014-09-10 20:22 -0400 http://bitbucket.org/pypy/pypy/changeset/99c40ef45b66/ Log: merge default diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py --- a/rpython/rlib/rfile.py +++ b/rpython/rlib/rfile.py @@ -205,7 +205,7 @@ def create_stdio(): - close2 = [None, None] + close2 = (None, None) stdin = RFile(c_stdin(), close2=close2) stdout = RFile(c_stdout(), close2=close2) stderr = RFile(c_stderr(), close2=close2) diff --git a/rpython/translator/sandbox/rsandbox.py b/rpython/translator/sandbox/rsandbox.py --- a/rpython/translator/sandbox/rsandbox.py +++ b/rpython/translator/sandbox/rsandbox.py @@ -61,13 +61,16 @@ def need_more_data(self): buflen = self.buflen buf = lltype.malloc(rffi.CCHARP.TO, buflen, flavor='raw') - buflen = rffi.cast(rffi.SIZE_T, buflen) - count = ll_read_not_sandboxed(self.fd, buf, buflen) - count = rffi.cast(lltype.Signed, count) - if count <= 0: - raise IOError - self.buf += ''.join([buf[i] for i in range(count)]) - self.buflen *= 2 + try: + buflen = rffi.cast(rffi.SIZE_T, buflen) + count = ll_read_not_sandboxed(self.fd, buf, buflen) + count = rffi.cast(lltype.Signed, count) + if count <= 0: + raise IOError + self.buf += ''.join([buf[i] for i in range(count)]) + self.buflen *= 2 + finally: + lltype.free(buf, flavor='raw') def sandboxed_io(buf): STDIN = 0 From noreply at buildbot.pypy.org Thu Sep 11 03:13:26 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 11 Sep 2014 03:13:26 +0200 (CEST) Subject: [pypy-commit] pypy default: test/fix fdopen buffering Message-ID: <20140911011326.E9CA61C073F@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r73436:d00e4c12507a Date: 2014-09-10 21:11 -0400 http://bitbucket.org/pypy/pypy/changeset/d00e4c12507a/ Log: test/fix fdopen buffering diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py --- a/rpython/rlib/rfile.py +++ b/rpython/rlib/rfile.py @@ -153,18 +153,12 @@ finally: lltype.free(ll_name, flavor='raw') _dircheck(ll_file) - if buffering >= 0: - buf = lltype.nullptr(rffi.CCHARP.TO) - if buffering == 0: - c_setvbuf(ll_file, buf, _IONBF, 0) - elif buffering == 1: - c_setvbuf(ll_file, buf, _IOLBF, BUFSIZ) - else: - c_setvbuf(ll_file, buf, _IOFBF, buffering) - return RFile(ll_file, mode) + f = RFile(ll_file, mode) + f._setbufsize(buffering) + return f -def create_fdopen_rfile(fd, mode="r"): +def create_fdopen_rfile(fd, mode="r", buffering=-1): newmode = _sanitize_mode(mode) fd = rffi.cast(rffi.INT, fd) rposix.validate_fd(fd) @@ -177,7 +171,9 @@ finally: lltype.free(ll_mode, flavor='raw') _dircheck(ll_file) - return RFile(ll_file, mode) + f = RFile(ll_file, mode) + f._setbufsize(buffering) + return f def create_temp_rfile(): @@ -213,6 +209,7 @@ class RFile(object): + _setbuf = lltype.nullptr(rffi.CCHARP.TO) _univ_newline = False _newlinetypes = NEWLINE_UNKNOWN _skipnextlf = False @@ -223,6 +220,23 @@ self._univ_newline = 'U' in mode self._close2 = close2 + def _setbufsize(self, bufsize): + if bufsize >= 0: + if bufsize == 0: + mode = _IONBF + elif bufsize == 1: + mode = _IOLBF + bufsize = BUFSIZ + else: + mode = _IOFBF + if self._setbuf: + lltype.free(self._setbuf, flavor='raw') + if mode == _IONBF: + self._setbuf = lltype.nullptr(rffi.CCHARP.TO) + else: + self._setbuf = lltype.malloc(rffi.CCHARP.TO, bufsize, flavor='raw') + c_setvbuf(self._ll_file, self._setbuf, mode, bufsize) + def __del__(self): """Closes the described file when the object's last reference goes away. Unlike an explicit call to close(), this is meant @@ -233,6 +247,8 @@ do_close = self._close2[1] if do_close: do_close(ll_file) # return value ignored + if self._setbuf: + lltype.free(self._setbuf, flavor='raw') def _cleanup_(self): self._ll_file = lltype.nullptr(FILEP.TO) @@ -251,11 +267,16 @@ # double close is allowed self._ll_file = lltype.nullptr(FILEP.TO) do_close = self._close2[0] - if do_close: - res = do_close(ll_file) - if res == -1: - errno = rposix.get_errno() - raise IOError(errno, os.strerror(errno)) + try: + if do_close: + res = do_close(ll_file) + if res == -1: + errno = rposix.get_errno() + raise IOError(errno, os.strerror(errno)) + finally: + if self._setbuf: + lltype.free(self._setbuf, flavor='raw') + self._setbuf = lltype.nullptr(rffi.CCHARP.TO) return res def _check_closed(self): diff --git a/rpython/rlib/test/test_rfile.py b/rpython/rlib/test/test_rfile.py --- a/rpython/rlib/test/test_rfile.py +++ b/rpython/rlib/test/test_rfile.py @@ -104,15 +104,53 @@ f() self.interpret(f, []) + @py.test.mark.skipif("sys.platform == 'win32'") + # http://msdn.microsoft.com/en-us/library/86cebhfs.aspx + def test_fdopen_buffering_line(self): + fname = str(self.tmpdir.join('file_1a')) + + def f(): + g = open(fname, 'w') + f = os.fdopen(os.dup(g.fileno()), 'w', 1) + g.close() + f.write('dupa\ndupb') + f2 = open(fname, 'r') + assert f2.read() == 'dupa\n' + f.close() + assert f2.read() == 'dupb' + f2.close() + + f() + self.interpret(f, []) + def test_open_buffering_full(self): fname = str(self.tmpdir.join('file_1b')) def f(): f = open(fname, 'w', 128) - f.write('dupa') + f.write('dupa\ndupb') f2 = open(fname, 'r') assert f2.read() == '' - f.write('z' * 5000) + f.write('z' * 120) + assert f2.read() != '' + f.close() + assert f2.read() != '' + f2.close() + + f() + self.interpret(f, []) + + def test_fdopen_buffering_full(self): + fname = str(self.tmpdir.join('file_1b')) + + def f(): + g = open(fname, 'w') + f = os.fdopen(os.dup(g.fileno()), 'w', 128) + g.close() + f.write('dupa\ndupb') + f2 = open(fname, 'r') + assert f2.read() == '' + f.write('z' * 120) assert f2.read() != '' f.close() assert f2.read() != '' From noreply at buildbot.pypy.org Thu Sep 11 03:13:28 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 11 Sep 2014 03:13:28 +0200 (CEST) Subject: [pypy-commit] pypy use-file-star-for-file: merge default Message-ID: <20140911011328.3B6F51C073F@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: use-file-star-for-file Changeset: r73437:c9689dd4d663 Date: 2014-09-10 21:11 -0400 http://bitbucket.org/pypy/pypy/changeset/c9689dd4d663/ Log: merge default diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py --- a/rpython/rlib/rfile.py +++ b/rpython/rlib/rfile.py @@ -153,18 +153,12 @@ finally: lltype.free(ll_name, flavor='raw') _dircheck(ll_file) - if buffering >= 0: - buf = lltype.nullptr(rffi.CCHARP.TO) - if buffering == 0: - c_setvbuf(ll_file, buf, _IONBF, 0) - elif buffering == 1: - c_setvbuf(ll_file, buf, _IOLBF, BUFSIZ) - else: - c_setvbuf(ll_file, buf, _IOFBF, buffering) - return RFile(ll_file, mode) + f = RFile(ll_file, mode) + f._setbufsize(buffering) + return f -def create_fdopen_rfile(fd, mode="r"): +def create_fdopen_rfile(fd, mode="r", buffering=-1): newmode = _sanitize_mode(mode) fd = rffi.cast(rffi.INT, fd) rposix.validate_fd(fd) @@ -177,7 +171,9 @@ finally: lltype.free(ll_mode, flavor='raw') _dircheck(ll_file) - return RFile(ll_file, mode) + f = RFile(ll_file, mode) + f._setbufsize(buffering) + return f def create_temp_rfile(): @@ -213,6 +209,7 @@ class RFile(object): + _setbuf = lltype.nullptr(rffi.CCHARP.TO) _univ_newline = False _newlinetypes = NEWLINE_UNKNOWN _skipnextlf = False @@ -223,6 +220,23 @@ self._univ_newline = 'U' in mode self._close2 = close2 + def _setbufsize(self, bufsize): + if bufsize >= 0: + if bufsize == 0: + mode = _IONBF + elif bufsize == 1: + mode = _IOLBF + bufsize = BUFSIZ + else: + mode = _IOFBF + if self._setbuf: + lltype.free(self._setbuf, flavor='raw') + if mode == _IONBF: + self._setbuf = lltype.nullptr(rffi.CCHARP.TO) + else: + self._setbuf = lltype.malloc(rffi.CCHARP.TO, bufsize, flavor='raw') + c_setvbuf(self._ll_file, self._setbuf, mode, bufsize) + def __del__(self): """Closes the described file when the object's last reference goes away. Unlike an explicit call to close(), this is meant @@ -233,6 +247,8 @@ do_close = self._close2[1] if do_close: do_close(ll_file) # return value ignored + if self._setbuf: + lltype.free(self._setbuf, flavor='raw') def _cleanup_(self): self._ll_file = lltype.nullptr(FILEP.TO) @@ -251,11 +267,16 @@ # double close is allowed self._ll_file = lltype.nullptr(FILEP.TO) do_close = self._close2[0] - if do_close: - res = do_close(ll_file) - if res == -1: - errno = rposix.get_errno() - raise IOError(errno, os.strerror(errno)) + try: + if do_close: + res = do_close(ll_file) + if res == -1: + errno = rposix.get_errno() + raise IOError(errno, os.strerror(errno)) + finally: + if self._setbuf: + lltype.free(self._setbuf, flavor='raw') + self._setbuf = lltype.nullptr(rffi.CCHARP.TO) return res def _check_closed(self): diff --git a/rpython/rlib/test/test_rfile.py b/rpython/rlib/test/test_rfile.py --- a/rpython/rlib/test/test_rfile.py +++ b/rpython/rlib/test/test_rfile.py @@ -104,15 +104,53 @@ f() self.interpret(f, []) + @py.test.mark.skipif("sys.platform == 'win32'") + # http://msdn.microsoft.com/en-us/library/86cebhfs.aspx + def test_fdopen_buffering_line(self): + fname = str(self.tmpdir.join('file_1a')) + + def f(): + g = open(fname, 'w') + f = os.fdopen(os.dup(g.fileno()), 'w', 1) + g.close() + f.write('dupa\ndupb') + f2 = open(fname, 'r') + assert f2.read() == 'dupa\n' + f.close() + assert f2.read() == 'dupb' + f2.close() + + f() + self.interpret(f, []) + def test_open_buffering_full(self): fname = str(self.tmpdir.join('file_1b')) def f(): f = open(fname, 'w', 128) - f.write('dupa') + f.write('dupa\ndupb') f2 = open(fname, 'r') assert f2.read() == '' - f.write('z' * 5000) + f.write('z' * 120) + assert f2.read() != '' + f.close() + assert f2.read() != '' + f2.close() + + f() + self.interpret(f, []) + + def test_fdopen_buffering_full(self): + fname = str(self.tmpdir.join('file_1b')) + + def f(): + g = open(fname, 'w') + f = os.fdopen(os.dup(g.fileno()), 'w', 128) + g.close() + f.write('dupa\ndupb') + f2 = open(fname, 'r') + assert f2.read() == '' + f.write('z' * 120) assert f2.read() != '' f.close() assert f2.read() != '' From noreply at buildbot.pypy.org Thu Sep 11 03:33:18 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 11 Sep 2014 03:33:18 +0200 (CEST) Subject: [pypy-commit] pypy use-file-star-for-file: allow leakfinder.start_tracking_allocations to be called in test setup Message-ID: <20140911013318.970311C073F@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: use-file-star-for-file Changeset: r73438:8f2bf775fa49 Date: 2014-09-10 21:31 -0400 http://bitbucket.org/pypy/pypy/changeset/8f2bf775fa49/ Log: allow leakfinder.start_tracking_allocations to be called in test setup diff --git a/pypy/module/_file/test/test_file_extra.py b/pypy/module/_file/test/test_file_extra.py --- a/pypy/module/_file/test/test_file_extra.py +++ b/pypy/module/_file/test/test_file_extra.py @@ -1,5 +1,6 @@ import os, random, sys import rpython.tool.udir +from rpython.tool import leakfinder import py udir = rpython.tool.udir.udir.ensure('test_file_extra', dir=1) @@ -230,15 +231,16 @@ w_filetype = space.gettypeobject(W_File.typedef) else: w_filetype = file # TinyObjSpace, for "py.test -A" + self.w_sample = space.wrap(self.sample) + self.w_expected_filename = space.wrap(self.expected_filename) + self.w_expected_mode = space.wrap(self.expected_mode) + self.w_expected_lines = space.wrap(self.get_expected_lines()) + leakfinder.start_tracking_allocations() self.w_file = space.call_function( w_filetype, space.wrap(self.expected_filename), space.wrap(self.expected_mode), *[space.wrap(a) for a in self.extra_args]) - self.w_sample = space.wrap(self.sample) - self.w_expected_filename = space.wrap(self.expected_filename) - self.w_expected_mode = space.wrap(self.expected_mode) - self.w_expected_lines = space.wrap(self.get_expected_lines()) def teardown_method(self, method): self.space.call_method(self.w_file, 'close') diff --git a/rpython/conftest.py b/rpython/conftest.py --- a/rpython/conftest.py +++ b/rpython/conftest.py @@ -64,8 +64,9 @@ __multicall__.execute() if not isinstance(item, py.test.collect.Function): return - if not getattr(item.obj, 'dont_track_allocations', False): - leakfinder.start_tracking_allocations() + if (not getattr(item.obj, 'dont_track_allocations', False) + and not leakfinder.TRACK_ALLOCATIONS): + leakfinder.start_tracking_allocations() def pytest_runtest_call(self, __multicall__, item): __multicall__.execute() From noreply at buildbot.pypy.org Thu Sep 11 04:53:07 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 11 Sep 2014 04:53:07 +0200 (CEST) Subject: [pypy-commit] pypy use-file-star-for-file: expose file._setbufsize Message-ID: <20140911025307.032CC1C0A66@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: use-file-star-for-file Changeset: r73439:cb5bd9b58319 Date: 2014-09-10 22:46 -0400 http://bitbucket.org/pypy/pypy/changeset/cb5bd9b58319/ Log: expose file._setbufsize diff --git a/pypy/module/_file/interp_file.py b/pypy/module/_file/interp_file.py --- a/pypy/module/_file/interp_file.py +++ b/pypy/module/_file/interp_file.py @@ -231,6 +231,10 @@ def direct_isatty(self): return self.getstream().isatty() + @unwrap_spec(bufsize=int) + def descr_setbufsize(self, bufsize): + self.getstream()._setbufsize(bufsize) + # ____________________________________________________________ # # The 'file_' methods are the one exposed to app-level. @@ -520,6 +524,7 @@ __repr__ = interp2app(W_File.file__repr__), readinto = interp2app(W_File.file_readinto), writelines = interp2app(W_File.file_writelines), + _setbufsize = interp2app(W_File.descr_setbufsize), __exit__ = interp2app(W_File.file__exit__), __weakref__ = make_weakref_descr(W_File), **dict([(name, interp2app(getattr(W_File, 'file_' + name))) diff --git a/pypy/module/_file/test/test_file.py b/pypy/module/_file/test/test_file.py --- a/pypy/module/_file/test/test_file.py +++ b/pypy/module/_file/test/test_file.py @@ -453,6 +453,16 @@ s = os.read(f.fileno(), 10) assert s == 'bar\n' + import sys + if '__pypy__' in sys.builtin_module_names: + with self.file(self.temppath, 'rb') as f: + f._setbufsize(0) + s = f.readline() + assert s == 'foo\n' + s = os.read(f.fileno(), 10) + assert s == 'bar\n' + + def test_flush_at_exit(): from pypy import conftest from pypy.tool.option import make_config, make_objspace From noreply at buildbot.pypy.org Thu Sep 11 04:53:08 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 11 Sep 2014 04:53:08 +0200 (CEST) Subject: [pypy-commit] pypy use-file-star-for-file: use f._setbufsize in interpreter init Message-ID: <20140911025308.3A9C51C0A66@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: use-file-star-for-file Changeset: r73440:6a7cc1c84664 Date: 2014-09-10 22:50 -0400 http://bitbucket.org/pypy/pypy/changeset/6a7cc1c84664/ Log: use f._setbufsize in interpreter init diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py --- a/pypy/interpreter/app_main.py +++ b/pypy/interpreter/app_main.py @@ -222,20 +222,24 @@ print >> sys.stderr, 'usage: %s [options]' % (sys.executable,) print >> sys.stderr, 'Try `%s -h` for more information.' % (sys.executable,) -def fdopen(fd, mode, bufsize=-1): +def set_unbuffered_io(): try: - fdopen = file.fdopen - except AttributeError: # only on top of CPython, running tests + for f in [sys.__stdin__, sys.__stdout__, sys.__stderr__]: + f._setbufsize(0) + except AttributeError: from os import fdopen - return fdopen(fd, mode, bufsize) + sys.stdin = sys.__stdin__ = fdopen(0, 'rb', 0) + sys.stdout = sys.__stdout__ = fdopen(1, 'wb', 0) + sys.stderr = sys.__stderr__ = fdopen(2, 'wb', 0) -def set_unbuffered_io(): - sys.stdin = sys.__stdin__ = fdopen(0, 'rb', 0) - sys.stdout = sys.__stdout__ = fdopen(1, 'wb', 0) - sys.stderr = sys.__stderr__ = fdopen(2, 'wb', 0) - -def set_fully_buffered_io(): - sys.stdout = sys.__stdout__ = fdopen(1, 'w') +def set_line_buffered_io(): + try: + for f in [sys.__stdin__, sys.__stdout__]: + f._setbufsize(1) + except AttributeError: + from os import fdopen + sys.stdin = sys.__stdin__ = fdopen(0, 'rb', 1) + sys.stdout = sys.__stdout__ = fdopen(1, 'wb', 1) # ____________________________________________________________ # Main entry point @@ -511,8 +515,8 @@ if unbuffered: set_unbuffered_io() - elif not sys.stdout.isatty(): - set_fully_buffered_io() + elif interactive: + set_line_buffered_io() mainmodule = type(sys)('__main__') sys.modules['__main__'] = mainmodule diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py --- a/pypy/interpreter/test/test_app_main.py +++ b/pypy/interpreter/test/test_app_main.py @@ -755,7 +755,7 @@ time.sleep(1) # stdout flushed automatically here """) - cmdline = '%s -u "%s" %s' % (sys.executable, app_main, path) + cmdline = '%s "%s" %s' % (sys.executable, app_main, path) print 'POPEN:', cmdline child_in, child_out_err = os.popen4(cmdline) data = child_out_err.read(11) @@ -765,6 +765,32 @@ assert data == '\x00(STDOUT)\n\x00' # from stdout child_out_err.close() + def test_non_interactive_stdout_line_buffered(self): + if os.name == 'nt': + try: + import __pypy__ + except: + py.test.skip('app_main cannot run on non-pypy for windows') + path = getscript(r""" + import sys, time + sys.stdout.write('\x00(STDOUT)\n\x00') + time.sleep(1) + sys.stderr.write('\x00[STDERR]\n\x00') + time.sleep(1) + # stdout flushed automatically here + """) + cmdline = '%s "%s" -i %s' % (sys.executable, app_main, path) + print 'POPEN:', cmdline + child_in, child_out_err = os.popen4(cmdline) + data = child_out_err.read(10) + assert data == '\x00(STDOUT)\n' # from stdout + data = child_out_err.read(11) + assert data == '\x00[STDERR]\n\x00' # from stderr + child_in.close() + data = child_out_err.read(2) + assert data == '\n\x00' + child_out_err.close() + def test_non_interactive_stdout_unbuffered(self, monkeypatch): monkeypatch.setenv('PYTHONUNBUFFERED', '1') if os.name == 'nt': @@ -787,8 +813,8 @@ assert data == '\x00(STDOUT)\n\x00' # from stderr data = child_out_err.read(11) assert data == '\x00[STDERR]\n\x00' # from stdout + child_in.close() child_out_err.close() - child_in.close() def test_proper_sys_path(self, tmpdir): data = self.run('-c "import _ctypes"', python_flags='-S') From noreply at buildbot.pypy.org Thu Sep 11 04:56:23 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 11 Sep 2014 04:56:23 +0200 (CEST) Subject: [pypy-commit] pypy use-file-star-for-file: fix fdopen directory Message-ID: <20140911025623.CC0351C0A66@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: use-file-star-for-file Changeset: r73441:5cd637dc010a Date: 2014-09-10 22:56 -0400 http://bitbucket.org/pypy/pypy/changeset/5cd637dc010a/ Log: fix fdopen directory diff --git a/pypy/module/_file/interp_file.py b/pypy/module/_file/interp_file.py --- a/pypy/module/_file/interp_file.py +++ b/pypy/module/_file/interp_file.py @@ -244,6 +244,10 @@ self.direct_fdopen(fd, mode, buffering) except ValueError as e: raise OperationError(self.space.w_ValueError, self.space.wrap(str(e))) + except IOError as e: + space = self.space + w_error = space.call_function(space.w_IOError, space.wrap(e.errno), space.wrap(e.strerror), self.w_name) + raise OperationError(space.w_IOError, w_error) except OSError as e: raise wrap_oserror(self.space, e) From noreply at buildbot.pypy.org Thu Sep 11 05:07:36 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 11 Sep 2014 05:07:36 +0200 (CEST) Subject: [pypy-commit] pypy use-file-star-for-file: fix test_pytrace Message-ID: <20140911030736.AF2EE1D235C@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: use-file-star-for-file Changeset: r73442:05cdfb9505fa Date: 2014-09-10 23:07 -0400 http://bitbucket.org/pypy/pypy/changeset/05cdfb9505fa/ Log: fix test_pytrace diff --git a/pypy/interpreter/test/test_zpy.py b/pypy/interpreter/test/test_zpy.py --- a/pypy/interpreter/test/test_zpy.py +++ b/pypy/interpreter/test/test_zpy.py @@ -114,7 +114,7 @@ '>>>> ') in output assert ('\t: LOAD_NAME 0 (x)\n' '\t: PRINT_EXPR 0 \n' - # '5\n' --- this line sent to stderr + '5\n' '\t: LOAD_CONST 0 (None)\n' '\t: RETURN_VALUE 0 \n' '>>>> ') in output From noreply at buildbot.pypy.org Thu Sep 11 07:15:17 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 11 Sep 2014 07:15:17 +0200 (CEST) Subject: [pypy-commit] pypy use-file-star-for-file: fix test_bz2_file Message-ID: <20140911051517.9DCD61D2970@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: use-file-star-for-file Changeset: r73443:6c6872eb8092 Date: 2014-09-11 00:51 -0400 http://bitbucket.org/pypy/pypy/changeset/6c6872eb8092/ Log: fix test_bz2_file diff --git a/pypy/module/bz2/test/test_bz2_file.py b/pypy/module/bz2/test/test_bz2_file.py --- a/pypy/module/bz2/test/test_bz2_file.py +++ b/pypy/module/bz2/test/test_bz2_file.py @@ -28,6 +28,11 @@ data = DATA[:100] f.write(data, 'wb') + def create_short_temp_file(): + f = py.test.ensuretemp("bz2").join("foo") + DATA = 'BZh91AY&SY\xd9b\x89]\x00\x00\x00\x03\x80\x04\x00\x02\x00\x0c\x00 \x00!\x9ah3M\x13<]\xc9\x14\xe1BCe\x8a%t' + f.write(DATA, 'wb') + @unwrap_spec(data=str) def decompress(space, data): import popen2 @@ -43,10 +48,11 @@ mod.TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' mod.DATA = DATA - mod.DATA_CRLF = DATA_CRLF + mod.DATA_CRLF = DATA_CRLF mod.create_temp_file = create_temp_file + mod.create_broken_temp_file = create_broken_temp_file + mod.create_short_temp_file = create_short_temp_file mod.decompress = decompress - mod.create_broken_temp_file = create_broken_temp_file s = 'abcdefghijklmnop' mod.RANDOM_DATA = ''.join([s[int(random.random() * len(s))] for i in range(30000)]) @@ -65,10 +71,12 @@ if cls.runappdirect: cls.w_create_temp_file = create_temp_file cls.w_create_broken_temp_file = lambda self: create_broken_temp_file() + cls.w_create_short_temp_file = lambda self: create_short_temp_file() cls.w_decompress = lambda self, *args: decompress(cls.space, *args) else: cls.w_create_temp_file = cls.space.wrap(interp2app(create_temp_file)) cls.w_create_broken_temp_file = cls.space.wrap(interp2app(create_broken_temp_file)) + cls.w_create_short_temp_file = cls.space.wrap(interp2app(create_short_temp_file)) cls.w_decompress = cls.space.wrap(interp2app(decompress)) cls.w_random_data = cls.space.wrap(RANDOM_DATA) @@ -362,11 +370,7 @@ def test_readlines_bug_1191043(self): # readlines()/xreadlines() for files containing no newline from bz2 import BZ2File - - DATA = 'BZh91AY&SY\xd9b\x89]\x00\x00\x00\x03\x80\x04\x00\x02\x00\x0c\x00 \x00!\x9ah3M\x13<]\xc9\x14\xe1BCe\x8a%t' - f = open(self.temppath, "wb") - f.write(DATA) - f.close() + self.create_short_temp_file() bz2f = BZ2File(self.temppath) lines = bz2f.readlines() From noreply at buildbot.pypy.org Thu Sep 11 07:15:18 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 11 Sep 2014 07:15:18 +0200 (CEST) Subject: [pypy-commit] pypy use-file-star-for-file: cleanup bz2 tests Message-ID: <20140911051518.D9C511D2970@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: use-file-star-for-file Changeset: r73444:e1b2e70e183e Date: 2014-09-11 01:09 -0400 http://bitbucket.org/pypy/pypy/changeset/e1b2e70e183e/ Log: cleanup bz2 tests diff --git a/pypy/module/bz2/test/test_bz2_file.py b/pypy/module/bz2/test/test_bz2_file.py --- a/pypy/module/bz2/test/test_bz2_file.py +++ b/pypy/module/bz2/test/test_bz2_file.py @@ -14,25 +14,22 @@ skip("bz2 module is not available on Windows") def setup_module(mod): - DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' - DATA_CRLF = 'BZh91AY&SY\xaez\xbbN\x00\x01H\xdf\x80\x00\x12@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe0@\x01\xbc\xc6`\x86*\x8d=M\xa9\x9a\x86\xd0L@\x0fI\xa6!\xa1\x13\xc8\x88jdi\x8d@\x03@\x1a\x1a\x0c\x0c\x83 \x00\xc4h2\x19\x01\x82D\x84e\t\xe8\x99\x89\x19\x1ah\x00\r\x1a\x11\xaf\x9b\x0fG\xf5(\x1b\x1f?\t\x12\xcf\xb5\xfc\x95E\x00ps\x89\x12^\xa4\xdd\xa2&\x05(\x87\x04\x98\x89u\xe40%\xb6\x19\'\x8c\xc4\x89\xca\x07\x0e\x1b!\x91UIFU%C\x994!DI\xd2\xfa\xf0\xf1N8W\xde\x13A\xf5\x9cr%?\x9f3;I45A\xd1\x8bT\xb1\xa4\xc7\x8d\x1a\\"\xad\xa1\xabyBg\x15\xb9l\x88\x88\x91k"\x94\xa4\xd4\x89\xae*\xa6\x0b\x10\x0c\xd6\xd4m\xe86\xec\xb5j\x8a\x86j\';\xca.\x01I\xf2\xaaJ\xe8\x88\x8cU+t3\xfb\x0c\n\xa33\x13r2\r\x16\xe0\xb3(\xbf\x1d\x83r\xe7M\xf0D\x1365\xd8\x88\xd3\xa4\x92\xcb2\x06\x04\\\xc1\xb0\xea//\xbek&\xd8\xe6+t\xe5\xa1\x13\xada\x16\xder5"w]\xa2i\xb7[\x97R \xe2IT\xcd;Z\x04dk4\xad\x8a\t\xd3\x81z\x10\xf1:^`\xab\x1f\xc5\xdc\x91N\x14$+\x9e\xae\xd3\x80' + DATA = { + 'normal': 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`', + 'crlf': 'BZh91AY&SY\xaez\xbbN\x00\x01H\xdf\x80\x00\x12@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe0@\x01\xbc\xc6`\x86*\x8d=M\xa9\x9a\x86\xd0L@\x0fI\xa6!\xa1\x13\xc8\x88jdi\x8d@\x03@\x1a\x1a\x0c\x0c\x83 \x00\xc4h2\x19\x01\x82D\x84e\t\xe8\x99\x89\x19\x1ah\x00\r\x1a\x11\xaf\x9b\x0fG\xf5(\x1b\x1f?\t\x12\xcf\xb5\xfc\x95E\x00ps\x89\x12^\xa4\xdd\xa2&\x05(\x87\x04\x98\x89u\xe40%\xb6\x19\'\x8c\xc4\x89\xca\x07\x0e\x1b!\x91UIFU%C\x994!DI\xd2\xfa\xf0\xf1N8W\xde\x13A\xf5\x9cr%?\x9f3;I45A\xd1\x8bT\xb1\xa4\xc7\x8d\x1a\\"\xad\xa1\xabyBg\x15\xb9l\x88\x88\x91k"\x94\xa4\xd4\x89\xae*\xa6\x0b\x10\x0c\xd6\xd4m\xe86\xec\xb5j\x8a\x86j\';\xca.\x01I\xf2\xaaJ\xe8\x88\x8cU+t3\xfb\x0c\n\xa33\x13r2\r\x16\xe0\xb3(\xbf\x1d\x83r\xe7M\xf0D\x1365\xd8\x88\xd3\xa4\x92\xcb2\x06\x04\\\xc1\xb0\xea//\xbek&\xd8\xe6+t\xe5\xa1\x13\xada\x16\xder5"w]\xa2i\xb7[\x97R \xe2IT\xcd;Z\x04dk4\xad\x8a\t\xd3\x81z\x10\xf1:^`\xab\x1f\xc5\xdc\x91N\x14$+\x9e\xae\xd3\x80', + 'short': 'BZh91AY&SY\xd9b\x89]\x00\x00\x00\x03\x80\x04\x00\x02\x00\x0c\x00 \x00!\x9ah3M\x13<]\xc9\x14\xe1BCe\x8a%t', + } + DATA['broken'] = DATA['normal'][:100] - @unwrap_spec(crlf=bool) - def create_temp_file(space, crlf=False): + s = 'abcdefghijklmnop' + DATA['random'] = ''.join([s[int(random.random() * len(s))] for i in range(30000)]) + + @unwrap_spec(key=str) + def create_temp_file(space, key='normal'): f = py.test.ensuretemp("bz2").join("foo") - data = (DATA, DATA_CRLF)[crlf] + data = DATA[key] f.write(data, 'wb') - def create_broken_temp_file(): - f = py.test.ensuretemp("bz2").join("foo") - data = DATA[:100] - f.write(data, 'wb') - - def create_short_temp_file(): - f = py.test.ensuretemp("bz2").join("foo") - DATA = 'BZh91AY&SY\xd9b\x89]\x00\x00\x00\x03\x80\x04\x00\x02\x00\x0c\x00 \x00!\x9ah3M\x13<]\xc9\x14\xe1BCe\x8a%t' - f.write(DATA, 'wb') - @unwrap_spec(data=str) def decompress(space, data): import popen2 @@ -48,13 +45,8 @@ mod.TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' mod.DATA = DATA - mod.DATA_CRLF = DATA_CRLF mod.create_temp_file = create_temp_file - mod.create_broken_temp_file = create_broken_temp_file - mod.create_short_temp_file = create_short_temp_file mod.decompress = decompress - s = 'abcdefghijklmnop' - mod.RANDOM_DATA = ''.join([s[int(random.random() * len(s))] for i in range(30000)]) class AppTestBZ2File(CheckAllocation): @@ -65,20 +57,14 @@ def setup_class(cls): cls.w_TEXT = cls.space.wrap(TEXT) cls.w_DATA = cls.space.wrap(DATA) - cls.w_DATA_CRLF = cls.space.wrap(DATA_CRLF) cls.w_temppath = cls.space.wrap( str(py.test.ensuretemp("bz2").join("foo"))) if cls.runappdirect: cls.w_create_temp_file = create_temp_file - cls.w_create_broken_temp_file = lambda self: create_broken_temp_file() - cls.w_create_short_temp_file = lambda self: create_short_temp_file() cls.w_decompress = lambda self, *args: decompress(cls.space, *args) else: cls.w_create_temp_file = cls.space.wrap(interp2app(create_temp_file)) - cls.w_create_broken_temp_file = cls.space.wrap(interp2app(create_broken_temp_file)) - cls.w_create_short_temp_file = cls.space.wrap(interp2app(create_short_temp_file)) cls.w_decompress = cls.space.wrap(interp2app(decompress)) - cls.w_random_data = cls.space.wrap(RANDOM_DATA) def test_attributes(self): from bz2 import BZ2File @@ -181,7 +167,7 @@ f = open(self.temppath) f.seek(0, 2) f.read() - assert f.tell() == len(self.DATA) + assert f.tell() == len(self.DATA['normal']) f.close() def test_seek_forward(self): @@ -270,13 +256,13 @@ def test_silently_closes(self): from bz2 import BZ2File - self.create_broken_temp_file() + self.create_temp_file('broken') BZ2File(self.temppath) # check that no C-level malloc is left behind def test_read_broken_file(self): from bz2 import BZ2File - self.create_broken_temp_file() + self.create_temp_file('broken') bz2f = BZ2File(self.temppath) raises(EOFError, bz2f.read) del bz2f # delete from this frame, which is captured in the traceback @@ -284,7 +270,7 @@ def test_subsequent_read_broken_file(self): from bz2 import BZ2File counter = 0 - self.create_broken_temp_file() + self.create_temp_file('broken') bz2f = BZ2File(self.temppath) try: bz2f.read(10) @@ -328,7 +314,7 @@ def test_universal_newlines_crlf(self): from bz2 import BZ2File - self.create_temp_file(crlf=True) + self.create_temp_file('crlf') bz2f = BZ2File(self.temppath, "rU") data = bz2f.read() @@ -370,7 +356,7 @@ def test_readlines_bug_1191043(self): # readlines()/xreadlines() for files containing no newline from bz2 import BZ2File - self.create_short_temp_file() + self.create_temp_file('short') bz2f = BZ2File(self.temppath) lines = bz2f.readlines() @@ -437,10 +423,10 @@ from bz2 import BZ2File import random bz2f = BZ2File(self.temppath, 'w') - bz2f.write(self.random_data) + bz2f.write(self.DATA['random']) bz2f.close() bz2f = BZ2File(self.temppath, 'r') - assert bz2f.read() == self.random_data + assert bz2f.read() == self.DATA['random'] del bz2f # delete from this frame, which is captured in the traceback def test_context_manager(self): From noreply at buildbot.pypy.org Thu Sep 11 07:15:19 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 11 Sep 2014 07:15:19 +0200 (CEST) Subject: [pypy-commit] pypy use-file-star-for-file: try to fix test_write_full Message-ID: <20140911051519.F37461D2970@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: use-file-star-for-file Changeset: r73445:a337ec6aba73 Date: 2014-09-11 01:14 -0400 http://bitbucket.org/pypy/pypy/changeset/a337ec6aba73/ Log: try to fix test_write_full diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py --- a/rpython/rlib/rfile.py +++ b/rpython/rlib/rfile.py @@ -468,9 +468,9 @@ try: # note that since we got a nonmoving buffer, it is either raw # or already cannot move, so the arithmetics below are fine - length = len(value) - bytes = c_fwrite(ll_value, 1, length, self._ll_file) - if bytes != length: + n = len(value) + n2 = c_fwrite(ll_value, 1, n, self._ll_file) + if n2 != n or c_ferror(self._ll_file): errno = rposix.get_errno() c_clearerr(self._ll_file) raise IOError(errno, os.strerror(errno)) From noreply at buildbot.pypy.org Thu Sep 11 08:36:28 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 11 Sep 2014 08:36:28 +0200 (CEST) Subject: [pypy-commit] pypy use-file-star-for-file: add a comment Message-ID: <20140911063628.9929A1C1292@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: use-file-star-for-file Changeset: r73446:a03b54d0c322 Date: 2014-09-11 01:17 -0400 http://bitbucket.org/pypy/pypy/changeset/a03b54d0c322/ Log: add a comment diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py --- a/rpython/rlib/rfile.py +++ b/rpython/rlib/rfile.py @@ -287,7 +287,7 @@ if not self._univ_newline: return c_fread(buf, 1, n, stream) - i = 0 + i = 0 # XXX how to do ptrdiff (dst - buf) instead? dst = buf newlinetypes = self._newlinetypes skipnextlf = self._skipnextlf From noreply at buildbot.pypy.org Thu Sep 11 08:36:29 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 11 Sep 2014 08:36:29 +0200 (CEST) Subject: [pypy-commit] pypy use-file-star-for-file: implement rfile read logic to match cpython Message-ID: <20140911063629.CFF2D1C1292@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: use-file-star-for-file Changeset: r73447:0c381d8d106c Date: 2014-09-11 02:18 -0400 http://bitbucket.org/pypy/pypy/changeset/0c381d8d106c/ Log: implement rfile read logic to match cpython diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py --- a/rpython/rlib/rfile.py +++ b/rpython/rlib/rfile.py @@ -330,37 +330,49 @@ return i def read(self, size=-1): - # XXX CPython uses a more delicate logic here self._check_closed() ll_file = self._ll_file - if size == 0: - return "" - elif size < 0: - # read the entire contents - buf = lltype.malloc(rffi.CCHARP.TO, BASE_BUF_SIZE, flavor='raw') - try: - s = StringBuilder() - while True: - returned_size = self._fread(buf, BASE_BUF_SIZE, ll_file) - returned_size = intmask(returned_size) # is between 0 and BASE_BUF_SIZE - if returned_size == 0: - if c_feof(ll_file): - # ok, finished - return s.build() - raise _error(ll_file) - s.append_charpsize(buf, returned_size) - finally: - lltype.free(buf, flavor='raw') - else: # size > 0 - with rffi.scoped_alloc_buffer(size) as buf: - returned_size = self._fread(buf.raw, size, ll_file) - returned_size = intmask(returned_size) # is between 0 and size - if returned_size == 0: - if not c_feof(ll_file): - raise _error(ll_file) - s = buf.str(returned_size) - assert s is not None - return s + + bytesrequested = size + if bytesrequested < 0: + buffersize = BASE_BUF_SIZE + else: + buffersize = bytesrequested + bytesread = 0 + + s = StringBuilder() + buf = lltype.malloc(rffi.CCHARP.TO, buffersize, flavor='raw') + try: + while True: + if bytesrequested >= 0: + buffersize = bytesrequested - bytesread + assert buffersize >= 0 + chunksize = intmask(self._fread(buf, buffersize, ll_file)) + interrupted = (c_ferror(ll_file) and + rposix.get_errno() == errno.EINTR) + if interrupted: + c_clearerr(ll_file) + # XXX check signals + if chunksize == 0: + if interrupted: + continue + if not c_ferror(ll_file): + break + c_clearerr(ll_file) + if bytesread > 0 and rposix.get_errno() == errno.EAGAIN: + break + err = rposix.get_errno() + raise IOError(err, os.strerror(err)) + s.append_charpsize(buf, chunksize) + bytesread += chunksize + if chunksize < buffersize and not interrupted: + c_clearerr(ll_file) + break + if bytesrequested >= 0: + break + finally: + lltype.free(buf, flavor='raw') + return s.build() def _readline1(self, raw_buf): ll_file = self._ll_file From noreply at buildbot.pypy.org Thu Sep 11 09:50:18 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 11 Sep 2014 09:50:18 +0200 (CEST) Subject: [pypy-commit] pypy use-file-star-for-file: cleanup rfile exceptions Message-ID: <20140911075018.41E7E1D27E9@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: use-file-star-for-file Changeset: r73448:443bb45634a8 Date: 2014-09-11 03:18 -0400 http://bitbucket.org/pypy/pypy/changeset/443bb45634a8/ Log: cleanup rfile exceptions diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py --- a/rpython/rlib/rfile.py +++ b/rpython/rlib/rfile.py @@ -101,10 +101,9 @@ c_stderr = rffi.CExternVariable(FILEP, 'stderr', eci, c_type='FILE*')[0] -def _error(ll_file): - err = c_ferror(ll_file) - c_clearerr(ll_file) - raise IOError(err, os.strerror(err)) +def _from_errno(exc): + err = rposix.get_errno() + return exc(err, os.strerror(err)) def _dircheck(ll_file): @@ -146,8 +145,7 @@ try: ll_file = c_fopen(ll_name, ll_mode) if not ll_file: - errno = rposix.get_errno() - raise IOError(errno, os.strerror(errno)) + raise _from_errno(IOError) finally: lltype.free(ll_mode, flavor='raw') finally: @@ -166,8 +164,7 @@ try: ll_file = c_fdopen(fd, ll_mode) if not ll_file: - errno = rposix.get_errno() - raise OSError(errno, os.strerror(errno)) + raise _from_errno(OSError) finally: lltype.free(ll_mode, flavor='raw') _dircheck(ll_file) @@ -179,8 +176,7 @@ def create_temp_rfile(): res = c_tmpfile() if not res: - errno = rposix.get_errno() - raise OSError(errno, os.strerror(errno)) + raise _from_errno(OSError) return RFile(res) @@ -191,8 +187,7 @@ try: ll_file = c_popen(ll_command, ll_type) if not ll_file: - errno = rposix.get_errno() - raise OSError(errno, os.strerror(errno)) + raise _from_errno(OSError) finally: lltype.free(ll_type, flavor='raw') finally: @@ -271,8 +266,7 @@ if do_close: res = do_close(ll_file) if res == -1: - errno = rposix.get_errno() - raise IOError(errno, os.strerror(errno)) + raise _from_errno(IOError) finally: if self._setbuf: lltype.free(self._setbuf, flavor='raw') @@ -361,8 +355,7 @@ c_clearerr(ll_file) if bytesread > 0 and rposix.get_errno() == errno.EAGAIN: break - err = rposix.get_errno() - raise IOError(err, os.strerror(err)) + raise _from_errno(IOError) s.append_charpsize(buf, chunksize) bytesread += chunksize if chunksize < buffersize and not interrupted: @@ -381,9 +374,9 @@ result = c_fgets(raw_buf, BASE_LINE_SIZE, ll_file) if not result: - if c_feof(ll_file): # ok - return 0 - raise _error(ll_file) + c_clearerr(ll_file) + # XXX check signals + return 0 # Assume that fgets() works as documented, and additionally # never writes beyond the final \0, which the CPython @@ -470,7 +463,9 @@ break if c == EOF: if c_ferror(ll_file): - raise _error(ll_file) + c_clearerr(ll_file) + raise _from_errno(IOError) + c_clearerr(ll_file) return s.build() @enforceargs(None, str) @@ -483,9 +478,8 @@ n = len(value) n2 = c_fwrite(ll_value, 1, n, self._ll_file) if n2 != n or c_ferror(self._ll_file): - errno = rposix.get_errno() c_clearerr(self._ll_file) - raise IOError(errno, os.strerror(errno)) + raise _from_errno(IOError) finally: rffi.free_nonmovingbuffer(value, ll_value) @@ -493,8 +487,8 @@ self._check_closed() res = c_fflush(self._ll_file) if res != 0: - errno = rposix.get_errno() - raise IOError(errno, os.strerror(errno)) + c_clearerr(self._ll_file) + raise _from_errno(IOError) def truncate(self, arg=-1): self._check_closed() @@ -502,24 +496,24 @@ arg = self.tell() self.flush() res = c_ftruncate(self.fileno(), arg) - if res == -1: - errno = rposix.get_errno() - raise IOError(errno, os.strerror(errno)) + if res != 0: + c_clearerr(self._ll_file) + raise _from_errno(IOError) def seek(self, pos, whence=0): self._check_closed() res = c_fseek(self._ll_file, pos, whence) - if res == -1: - errno = rposix.get_errno() - raise IOError(errno, os.strerror(errno)) + if res != 0: + c_clearerr(self._ll_file) + raise _from_errno(IOError) self._skipnextlf = False def tell(self): self._check_closed() res = intmask(c_ftell(self._ll_file)) if res == -1: - errno = rposix.get_errno() - raise IOError(errno, os.strerror(errno)) + c_clearerr(self._ll_file) + raise _from_errno(IOError) if self._skipnextlf: c = c_getc(self._ll_file) if c == ord('\n'): From noreply at buildbot.pypy.org Thu Sep 11 09:50:19 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 11 Sep 2014 09:50:19 +0200 (CEST) Subject: [pypy-commit] pypy use-file-star-for-file: check rfile mode readable/writable Message-ID: <20140911075019.703211C1292@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: use-file-star-for-file Changeset: r73449:2cdd638ae53b Date: 2014-09-11 03:31 -0400 http://bitbucket.org/pypy/pypy/changeset/2cdd638ae53b/ Log: check rfile mode readable/writable diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py --- a/rpython/rlib/rfile.py +++ b/rpython/rlib/rfile.py @@ -204,6 +204,8 @@ class RFile(object): + _readable = True + _writable = True _setbuf = lltype.nullptr(rffi.CCHARP.TO) _univ_newline = False _newlinetypes = NEWLINE_UNKNOWN @@ -213,6 +215,13 @@ self._ll_file = ll_file if mode is not None: self._univ_newline = 'U' in mode + self._readable = self._writable = False + if 'r' in mode or self._univ_newline: + self._readable = True + if 'w' in mode or 'a' in mode: + self._writable = True + if '+' in mode: + self._readable = self._writable = True self._close2 = close2 def _setbufsize(self, bufsize): @@ -277,6 +286,14 @@ if not self._ll_file: raise ValueError("I/O operation on closed file") + def _check_readable(self): + if not self._readable: + raise IOError("File not open for reading") + + def _check_writable(self): + if not self._writable: + raise IOError("File not open for writing") + def _fread(self, buf, n, stream): if not self._univ_newline: return c_fread(buf, 1, n, stream) @@ -325,6 +342,7 @@ def read(self, size=-1): self._check_closed() + self._check_readable() ll_file = self._ll_file bytesrequested = size @@ -403,6 +421,7 @@ def readline(self, size=-1): self._check_closed() + self._check_readable() if size == 0: return "" elif size < 0 and not self._univ_newline: @@ -471,6 +490,7 @@ @enforceargs(None, str) def write(self, value): self._check_closed() + self._check_writable() ll_value = rffi.get_nonmovingbuffer(value) try: # note that since we got a nonmoving buffer, it is either raw @@ -492,6 +512,7 @@ def truncate(self, arg=-1): self._check_closed() + self._check_writable() if arg == -1: arg = self.tell() self.flush() From noreply at buildbot.pypy.org Thu Sep 11 10:29:36 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 11 Sep 2014 10:29:36 +0200 (CEST) Subject: [pypy-commit] pypy.org extradoc: Update Message-ID: <20140911082936.691E81D29F9@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r534:77b88426748a Date: 2014-09-11 10:29 +0200 http://bitbucket.org/pypy/pypy.org/changeset/77b88426748a/ Log: Update diff --git a/download.html b/download.html --- a/download.html +++ b/download.html @@ -226,7 +226,8 @@

  • Run the rpython script. Here are the common combinations -of options (works also with python instead of pypy):

    +of options (works also with python instead of pypy; +requires Python 2.x or PyPy 2):

     pypy ../../rpython/bin/rpython -Ojit targetpypystandalone           # get the JIT version
 pypy ../../rpython/bin/rpython -O2 targetpypystandalone             # get the no-jit version
diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -270,7 +270,8 @@
      cd pypy/pypy/goal
 
 4. Run the ``rpython`` script.  Here are the common combinations
-   of options (works also with ``python`` instead of ``pypy``)::
+   of options (works also with ``python`` instead of ``pypy``;
+   requires Python 2.x or PyPy 2)::
 
      pypy ../../rpython/bin/rpython -Ojit targetpypystandalone           # get the JIT version
      pypy ../../rpython/bin/rpython -O2 targetpypystandalone             # get the no-jit version

From noreply at buildbot.pypy.org  Thu Sep 11 10:29:53 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 11 Sep 2014 10:29:53 +0200 (CEST)
Subject: [pypy-commit] pypy default: Update
Message-ID: <20140911082953.DDB931D29F9@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r73450:ab723fa4e9f7
Date: 2014-09-11 10:29 +0200
http://bitbucket.org/pypy/pypy/changeset/ab723fa4e9f7/

Log:	Update

diff --git a/pypy/doc/getting-started-python.rst b/pypy/doc/getting-started-python.rst
--- a/pypy/doc/getting-started-python.rst
+++ b/pypy/doc/getting-started-python.rst
@@ -111,6 +111,10 @@
    of your choice.  Typical example: ``--opt=2`` gives a good (but of
    course slower) Python interpreter without the JIT.
 
+   Consider using PyPy instead of CPython in the above command line,
+   as it is much faster.  (Note that ``rpython`` is a Python 2 program,
+   not Python 3; you need to run either PyPy 2 or CPython 2.)
+
 .. _`optimization level`: config/opt.html
 
 If everything works correctly this will create an executable

From noreply at buildbot.pypy.org  Thu Sep 11 10:37:57 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu, 11 Sep 2014 10:37:57 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: test/fix stdio flush on
	close
Message-ID: <20140911083757.4E6801D29E7@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73451:dc9cc4507de8
Date: 2014-09-11 04:14 -0400
http://bitbucket.org/pypy/pypy/changeset/dc9cc4507de8/

Log:	test/fix stdio flush on close

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -86,6 +86,7 @@
 c_fwrite = llexternal('fwrite', [rffi.CCHARP, rffi.SIZE_T, rffi.SIZE_T, FILEP],
                       rffi.SIZE_T)
 c_fflush = llexternal('fflush', [FILEP], rffi.INT)
+c_fflush_nogil = llexternal('fflush', [FILEP], rffi.INT, releasegil=False)
 c_ftruncate = llexternal(ftruncate, [rffi.INT, OFF_T], rffi.INT, macro=True)
 
 c_fseek = llexternal('fseek', [FILEP, rffi.LONG, rffi.INT], rffi.INT)
@@ -195,9 +196,14 @@
     return RFile(ll_file, close2=_pclose2)
 
 
+def _check_and_flush(ll_file):
+    prev_fail = c_ferror(ll_file)
+    return c_fflush(ll_file) or (EOF if prev_fail else 0)
+
+
 def create_stdio():
-    close2 = (None, None)
-    stdin = RFile(c_stdin(), close2=close2)
+    close2 = (_check_and_flush, c_fflush_nogil)
+    stdin = RFile(c_stdin(), close2=(None, None))
     stdout = RFile(c_stdout(), close2=close2)
     stderr = RFile(c_stderr(), close2=close2)
     return stdin, stdout, stderr
diff --git a/rpython/rlib/test/test_rfile.py b/rpython/rlib/test/test_rfile.py
--- a/rpython/rlib/test/test_rfile.py
+++ b/rpython/rlib/test/test_rfile.py
@@ -389,12 +389,23 @@
         cls.tmpdir = udir.join('test_rfile_direct')
         cls.tmpdir.ensure(dir=True)
 
-    def test_stdio(self):
+    def test_stdio_close(self, capfd):
         i, o, e = rfile.create_stdio()
         o.write("test\n")
         i.close()
         o.close()
         e.close()
+        out, err = capfd.readouterr()
+        assert out == "test\n"
+
+    def test_stdio_del(self, capfd):
+        i, o, e = rfile.create_stdio()
+        o.write("test\n")
+        del i
+        del o
+        del e
+        out, err = capfd.readouterr()
+        assert out == "test\n"
 
     def test_auto_close(self):
         fname = str(self.tmpdir.join('file_auto_close'))

From noreply at buildbot.pypy.org  Thu Sep 11 10:37:58 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu, 11 Sep 2014 10:37:58 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: fix cpyext
 test_file_writestring when running with fully buffered stdout
Message-ID: <20140911083758.A0F0B1D29E7@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73452:3a09b44f41ab
Date: 2014-09-11 04:21 -0400
http://bitbucket.org/pypy/pypy/changeset/3a09b44f41ab/

Log:	fix cpyext test_file_writestring when running with fully buffered
	stdout

diff --git a/pypy/module/cpyext/test/test_pyfile.py b/pypy/module/cpyext/test/test_pyfile.py
--- a/pypy/module/cpyext/test/test_pyfile.py
+++ b/pypy/module/cpyext/test/test_pyfile.py
@@ -74,6 +74,7 @@
             api.PyFile_WriteString(s, space.sys.get("stdout"))
         finally:
             rffi.free_charp(s)
+        space.call_method(space.sys.get("stdout"), "flush")
         out, err = capfd.readouterr()
         out = out.replace('\r\n', '\n')
         assert out == "test\n"

From noreply at buildbot.pypy.org  Thu Sep 11 10:37:59 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu, 11 Sep 2014 10:37:59 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: use a growing buffer for
	rfile read
Message-ID: <20140911083759.C119F1D29E7@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73453:81a458fbb16b
Date: 2014-09-11 04:37 -0400
http://bitbucket.org/pypy/pypy/changeset/81a458fbb16b/

Log:	use a growing buffer for rfile read

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -45,7 +45,6 @@
 BUFSIZ = config['BUFSIZ']
 EOF = config['EOF']
 
-BASE_BUF_SIZE = 4096
 BASE_LINE_SIZE = 100
 
 NEWLINE_UNKNOWN = 0
@@ -346,6 +345,10 @@
         self._skipnextlf = skipnextlf
         return i
 
+    def _new_buffersize(self, currentsize):
+        # XXX use fstat if possible
+        return currentsize + (currentsize >> 3) + 6
+
     def read(self, size=-1):
         self._check_closed()
         self._check_readable()
@@ -353,7 +356,7 @@
 
         bytesrequested = size
         if bytesrequested < 0:
-            buffersize = BASE_BUF_SIZE
+            buffersize = self._new_buffersize(0)
         else:
             buffersize = bytesrequested
         bytesread = 0
@@ -387,6 +390,10 @@
                     break
                 if bytesrequested >= 0:
                     break
+                else:
+                    buffersize = self._new_buffersize(buffersize)
+                    lltype.free(buf, flavor='raw')
+                    buf = lltype.malloc(rffi.CCHARP.TO, buffersize, flavor='raw')
         finally:
             lltype.free(buf, flavor='raw')
         return s.build()

From noreply at buildbot.pypy.org  Thu Sep 11 11:15:13 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu, 11 Sep 2014 11:15:13 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: use fstat for sizing
 rfile read buffer when possible
Message-ID: <20140911091513.C4C1E1C1292@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73454:02830ee6a82d
Date: 2014-09-11 04:54 -0400
http://bitbucket.org/pypy/pypy/changeset/02830ee6a82d/

Log:	use fstat for sizing rfile read buffer when possible

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -346,7 +346,22 @@
         return i
 
     def _new_buffersize(self, currentsize):
-        # XXX use fstat if possible
+        ll_file = self._ll_file
+        try:
+            st = os.fstat(c_fileno(ll_file))
+        except OSError:
+            pass
+        else:
+            end = st.st_size
+            try:
+                pos = os.lseek(c_fileno(ll_file), 0, os.SEEK_CUR)
+            except OSError:
+                c_clearerr(ll_file)
+            else:
+                pos = c_ftell(ll_file)
+                if end > pos and pos >= 0:
+                    return currentsize + end - pos + 1
+        # fstat didn't work
         return currentsize + (currentsize >> 3) + 6
 
     def read(self, size=-1):
@@ -367,7 +382,7 @@
             while True:
                 if bytesrequested >= 0:
                     buffersize = bytesrequested - bytesread
-                    assert buffersize >= 0
+                assert buffersize >= 0
                 chunksize = intmask(self._fread(buf, buffersize, ll_file))
                 interrupted = (c_ferror(ll_file) and
                                rposix.get_errno() == errno.EINTR)

From noreply at buildbot.pypy.org  Thu Sep 11 11:15:14 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu, 11 Sep 2014 11:15:14 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: simplify
Message-ID: <20140911091514.ECAF41C1292@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73455:216182f96c93
Date: 2014-09-11 05:14 -0400
http://bitbucket.org/pypy/pypy/changeset/216182f96c93/

Log:	simplify

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -369,19 +369,11 @@
         self._check_readable()
         ll_file = self._ll_file
 
-        bytesrequested = size
-        if bytesrequested < 0:
-            buffersize = self._new_buffersize(0)
-        else:
-            buffersize = bytesrequested
-        bytesread = 0
-
         s = StringBuilder()
+        buffersize = size if size >= 0 else self._new_buffersize(0)
         buf = lltype.malloc(rffi.CCHARP.TO, buffersize, flavor='raw')
         try:
             while True:
-                if bytesrequested >= 0:
-                    buffersize = bytesrequested - bytesread
                 assert buffersize >= 0
                 chunksize = intmask(self._fread(buf, buffersize, ll_file))
                 interrupted = (c_ferror(ll_file) and
@@ -395,20 +387,18 @@
                     if not c_ferror(ll_file):
                         break
                     c_clearerr(ll_file)
-                    if bytesread > 0 and rposix.get_errno() == errno.EAGAIN:
+                    if s.getlength() > 0 and rposix.get_errno() == errno.EAGAIN:
                         break
                     raise _from_errno(IOError)
                 s.append_charpsize(buf, chunksize)
-                bytesread += chunksize
                 if chunksize < buffersize and not interrupted:
                     c_clearerr(ll_file)
                     break
-                if bytesrequested >= 0:
+                if size >= 0:
                     break
-                else:
-                    buffersize = self._new_buffersize(buffersize)
-                    lltype.free(buf, flavor='raw')
-                    buf = lltype.malloc(rffi.CCHARP.TO, buffersize, flavor='raw')
+                buffersize = self._new_buffersize(buffersize)
+                lltype.free(buf, flavor='raw')
+                buf = lltype.malloc(rffi.CCHARP.TO, buffersize, flavor='raw')
         finally:
             lltype.free(buf, flavor='raw')
         return s.build()

From noreply at buildbot.pypy.org  Thu Sep 11 12:04:38 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 11 Sep 2014 12:04:38 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Added empty file with info from
	the web site
Message-ID: <20140911100438.9C76C1D2CA9@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: extradoc
Changeset: r5393:74f9b3662e1c
Date: 2014-09-11 12:04 +0200
http://bitbucket.org/pypy/extradoc/changeset/74f9b3662e1c/

Log:	Added empty file with info from the web site

diff --git a/talk/pycon2015/stm/submitted.txt b/talk/pycon2015/stm/submitted.txt
new file mode 100644
--- /dev/null
+++ b/talk/pycon2015/stm/submitted.txt
@@ -0,0 +1,43 @@
+PyCon 2015 proposal for STM
+===========================
+
+Title
+
+Category
+
+Duration
+
+Description
+...If your talk is accepted this will be made public and printed in the program. Should be one paragraph, maximum 400 characters.
+
+Audience
+...Who is the intended audience for your talk? (Be specific; "Python programmers" is not a good answer to this question.)
+
+Python level
+...Level of audience expertise assumed in Python.
+
+Objectives
+...What will attendees get out of your talk? When they leave the room, what will they know that they didn't know before?
+... Who is the intended audience for your talk? (Be specific; "Python programmers" is not a good answer to this question.)
+... What will attendees get out of your talk? When they leave the room, what will they know that they didn't know before?
+
+Detailed Abstract
+...Detailed description. Will be made public if your talk is accepted.
+... Include links to source code, articles, blog posts, or other writing that adds context to the presentation.
+
+Outline
+... Your outline should be an enumeration of what you intend to say, along with time estimates.
+... It is not necessary to have completely written or planned your talk already, but you should have a basic idea of what the over-arching points you intend to make are, and roughly how long you will spend on each one.
+
+Additional notes
+...Anything else you'd like the program committee to know when making their selection: your past speaking experience, open source community experience, etc.
+... If you've given a talk, tutorial, or other presentation before, especially at an earlier PyCon or another conference, include that information as well as a link to slides or a video if they're available.
+
+Additional requirements
+...Please let us know if you have any specific needs (A/V requirements, multiple microphones, a table, etc). Note for example that 'audio out' is not provided for your computer unless you tell us in advance.
+
+Recording release
+...By submitting your talk proposal, you agree to give permission to the Python Software Foundation to record, edit, and release audio and/or video of your presentation. If you do not agree to this, please uncheck this box. See PyCon 2015 Recording Release for details.
+
+-----------------
+You will be able to edit your proposal after it has been submitted. The program committee may ask questions, provide feedback, and even suggest changes to your proposal as part of the review processes.

From noreply at buildbot.pypy.org  Thu Sep 11 12:13:29 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu, 11 Sep 2014 12:13:29 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: fix buffer size to match
	cpython
Message-ID: <20140911101329.A9B9B1C1036@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73456:1f46f61322f3
Date: 2014-09-11 06:06 -0400
http://bitbucket.org/pypy/pypy/changeset/1f46f61322f3/

Log:	fix buffer size to match cpython

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -307,8 +307,10 @@
         dst = buf
         newlinetypes = self._newlinetypes
         skipnextlf = self._skipnextlf
+        assert n >= 0
         while n:
             nread = c_fread(dst, 1, n, stream)
+            assert nread <= n
             if nread == 0:
                 break
 
@@ -371,11 +373,11 @@
 
         s = StringBuilder()
         buffersize = size if size >= 0 else self._new_buffersize(0)
-        buf = lltype.malloc(rffi.CCHARP.TO, buffersize, flavor='raw')
+        remainsize = buffersize
+        buf = lltype.malloc(rffi.CCHARP.TO, remainsize, flavor='raw')
         try:
             while True:
-                assert buffersize >= 0
-                chunksize = intmask(self._fread(buf, buffersize, ll_file))
+                chunksize = intmask(self._fread(buf, remainsize, ll_file))
                 interrupted = (c_ferror(ll_file) and
                                rposix.get_errno() == errno.EINTR)
                 if interrupted:
@@ -391,14 +393,15 @@
                         break
                     raise _from_errno(IOError)
                 s.append_charpsize(buf, chunksize)
-                if chunksize < buffersize and not interrupted:
+                if chunksize < remainsize and not interrupted:
                     c_clearerr(ll_file)
                     break
                 if size >= 0:
                     break
                 buffersize = self._new_buffersize(buffersize)
+                remainsize = buffersize - s.getlength()
                 lltype.free(buf, flavor='raw')
-                buf = lltype.malloc(rffi.CCHARP.TO, buffersize, flavor='raw')
+                buf = lltype.malloc(rffi.CCHARP.TO, remainsize, flavor='raw')
         finally:
             lltype.free(buf, flavor='raw')
         return s.build()

From noreply at buildbot.pypy.org  Thu Sep 11 12:40:40 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu, 11 Sep 2014 12:40:40 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: group writing ops
	together
Message-ID: <20140911104040.DE0321C1036@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73457:67e45cc32904
Date: 2014-09-11 06:25 -0400
http://bitbucket.org/pypy/pypy/changeset/67e45cc32904/

Log:	group writing ops together

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -524,13 +524,6 @@
         finally:
             rffi.free_nonmovingbuffer(value, ll_value)
 
-    def flush(self):
-        self._check_closed()
-        res = c_fflush(self._ll_file)
-        if res != 0:
-            c_clearerr(self._ll_file)
-            raise _from_errno(IOError)
-
     def truncate(self, arg=-1):
         self._check_closed()
         self._check_writable()
@@ -542,6 +535,13 @@
             c_clearerr(self._ll_file)
             raise _from_errno(IOError)
 
+    def flush(self):
+        self._check_closed()
+        res = c_fflush(self._ll_file)
+        if res != 0:
+            c_clearerr(self._ll_file)
+            raise _from_errno(IOError)
+
     def seek(self, pos, whence=0):
         self._check_closed()
         res = c_fseek(self._ll_file, pos, whence)

From noreply at buildbot.pypy.org  Thu Sep 11 12:40:42 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu, 11 Sep 2014 12:40:42 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: check signals in rfile
Message-ID: <20140911104042.2562F1C1036@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73458:c95e690fc570
Date: 2014-09-11 06:36 -0400
http://bitbucket.org/pypy/pypy/changeset/c95e690fc570/

Log:	check signals in rfile

diff --git a/pypy/module/_file/interp_file.py b/pypy/module/_file/interp_file.py
--- a/pypy/module/_file/interp_file.py
+++ b/pypy/module/_file/interp_file.py
@@ -57,6 +57,7 @@
             raise OperationError(space.w_IOError, w_error)
 
     def fdopenstream(self, stream, mode, w_name=None):
+        stream._signal_checker = signal_checker(self.space)
         self.stream = stream
         self.mode = mode
         self.binary = "b" in mode
diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -209,6 +209,7 @@
 
 
 class RFile(object):
+    _signal_checker = None
     _readable = True
     _writable = True
     _setbuf = lltype.nullptr(rffi.CCHARP.TO)
@@ -382,7 +383,8 @@
                                rposix.get_errno() == errno.EINTR)
                 if interrupted:
                     c_clearerr(ll_file)
-                    # XXX check signals
+                    if self._signal_checker is not None:
+                        self._signal_checker()
                 if chunksize == 0:
                     if interrupted:
                         continue
@@ -414,7 +416,8 @@
         result = c_fgets(raw_buf, BASE_LINE_SIZE, ll_file)
         if not result:
             c_clearerr(ll_file)
-            # XXX check signals
+            if self._signal_checker is not None:
+                self._signal_checker()
             return 0
 
         # Assume that fgets() works as documented, and additionally

From noreply at buildbot.pypy.org  Thu Sep 11 13:13:00 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu, 11 Sep 2014 13:13:00 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: use alloc_buffer to
 avoid extra copy when possible in read
Message-ID: <20140911111300.79E9A1D3899@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73459:fd6110e214ca
Date: 2014-09-11 07:11 -0400
http://bitbucket.org/pypy/pypy/changeset/fd6110e214ca/

Log:	use alloc_buffer to avoid extra copy when possible in read

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -375,10 +375,10 @@
         s = StringBuilder()
         buffersize = size if size >= 0 else self._new_buffersize(0)
         remainsize = buffersize
-        buf = lltype.malloc(rffi.CCHARP.TO, remainsize, flavor='raw')
+        raw_buf, gc_buf = rffi.alloc_buffer(remainsize)
         try:
             while True:
-                chunksize = intmask(self._fread(buf, remainsize, ll_file))
+                chunksize = intmask(self._fread(raw_buf, remainsize, ll_file))
                 interrupted = (c_ferror(ll_file) and
                                rposix.get_errno() == errno.EINTR)
                 if interrupted:
@@ -394,7 +394,11 @@
                     if s.getlength() > 0 and rposix.get_errno() == errno.EAGAIN:
                         break
                     raise _from_errno(IOError)
-                s.append_charpsize(buf, chunksize)
+                elif chunksize == size:
+                    # we read everything in one call, try to avoid copy
+                    # (remainsize == size if chunksize == size)
+                    return rffi.str_from_buffer(raw_buf, gc_buf, remainsize, size)
+                s.append_charpsize(raw_buf, chunksize)
                 if chunksize < remainsize and not interrupted:
                     c_clearerr(ll_file)
                     break
@@ -402,10 +406,10 @@
                     break
                 buffersize = self._new_buffersize(buffersize)
                 remainsize = buffersize - s.getlength()
-                lltype.free(buf, flavor='raw')
-                buf = lltype.malloc(rffi.CCHARP.TO, remainsize, flavor='raw')
+                rffi.keep_buffer_alive_until_here(raw_buf, gc_buf)
+                raw_buf, gc_buf = rffi.alloc_buffer(remainsize)
         finally:
-            lltype.free(buf, flavor='raw')
+            rffi.keep_buffer_alive_until_here(raw_buf, gc_buf)
         return s.build()
 
     def _readline1(self, raw_buf):

From noreply at buildbot.pypy.org  Thu Sep 11 18:57:01 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 11 Sep 2014 18:57:01 +0200 (CEST)
Subject: [pypy-commit] pypy default: poll can fail. Raise SSLError in that
	case
Message-ID: <20140911165701.124851D2CA9@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r73460:8a07f6d2a9a0
Date: 2014-09-11 10:56 -0600
http://bitbucket.org/pypy/pypy/changeset/8a07f6d2a9a0/

Log:	poll can fail. Raise SSLError in that case

diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -759,7 +759,11 @@
 
         # socket's timeout is in seconds, poll's timeout in ms
         timeout = int(sock_timeout * 1000 + 0.5)
-        ready = rpoll.poll(fddict, timeout)
+        try:
+            ready = rpoll.poll(fddict, timeout)
+        except rpoll.PollError, e:
+            message = e.get_msg()
+            raise ssl_error(space, message, e.errno)
     else:
         if MAX_FD_SIZE is not None and sock_fd >= MAX_FD_SIZE:
             return SOCKET_TOO_LARGE_FOR_SELECT

From noreply at buildbot.pypy.org  Thu Sep 11 18:58:20 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 11 Sep 2014 18:58:20 +0200 (CEST)
Subject: [pypy-commit] pypy default: reraise the applevel in the select case
	too; 
Message-ID: <20140911165820.341571D2CA9@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r73461:22e1280d5d03
Date: 2014-09-11 10:57 -0600
http://bitbucket.org/pypy/pypy/changeset/22e1280d5d03/

Log:	reraise the applevel in the select case too;

diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -768,12 +768,16 @@
         if MAX_FD_SIZE is not None and sock_fd >= MAX_FD_SIZE:
             return SOCKET_TOO_LARGE_FOR_SELECT
 
-        if writing:
-            r, w, e = rpoll.select([], [sock_fd], [], sock_timeout)
-            ready = w
-        else:
-            r, w, e = rpoll.select([sock_fd], [], [], sock_timeout)
-            ready = r
+        try:
+            if writing:
+                r, w, e = rpoll.select([], [sock_fd], [], sock_timeout)
+                ready = w
+            else:
+                r, w, e = rpoll.select([sock_fd], [], [], sock_timeout)
+                ready = r
+        except SelectError, e:
+            message = e.get_msg()
+            raise ssl_error(space, message, e.errno)
     if ready:
         return SOCKET_OPERATION_OK
     else:

From noreply at buildbot.pypy.org  Thu Sep 11 19:13:32 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu, 11 Sep 2014 19:13:32 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: rfile fixes for win32
Message-ID: <20140911171332.6D1B41C1292@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73462:b17c5146eb5d
Date: 2014-09-11 10:10 -0700
http://bitbucket.org/pypy/pypy/changeset/b17c5146eb5d/

Log:	rfile fixes for win32

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -12,6 +12,7 @@
 from rpython.rtyper.tool import rffi_platform as platform
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 
+
 includes = ['stdio.h', 'sys/types.h']
 if os.name == "posix":
     includes += ['unistd.h']
@@ -20,7 +21,15 @@
 else:
     ftruncate = '_chsize'
     fileno = '_fileno'
-eci = ExternalCompilationInfo(includes=includes)
+
+stdio_streams = ['stdin', 'stdout', 'stderr']
+separate_module_sources = ['\n'.join('FILE* get_%s() { return %s; }' % (s, s)
+                                     for s in stdio_streams)]
+export_symbols = ['get_%s' % s for s in stdio_streams]
+
+eci = ExternalCompilationInfo(includes=includes,
+                              separate_module_sources=separate_module_sources,
+                              export_symbols=export_symbols)
 
 
 class CConfig(object):
@@ -96,9 +105,9 @@
 c_ferror = llexternal('ferror', [FILEP], rffi.INT)
 c_clearerr = llexternal('clearerr', [FILEP], lltype.Void)
 
-c_stdin = rffi.CExternVariable(FILEP, 'stdin', eci, c_type='FILE*')[0]
-c_stdout = rffi.CExternVariable(FILEP, 'stdout', eci, c_type='FILE*')[0]
-c_stderr = rffi.CExternVariable(FILEP, 'stderr', eci, c_type='FILE*')[0]
+c_stdin = llexternal('get_stdin', [], FILEP, _nowrapper=True)
+c_stdout = llexternal('get_stdout', [], FILEP, _nowrapper=True)
+c_stderr = llexternal('get_stderr', [], FILEP, _nowrapper=True)
 
 
 def _from_errno(exc):
@@ -363,7 +372,7 @@
             else:
                 pos = c_ftell(ll_file)
                 if end > pos and pos >= 0:
-                    return currentsize + end - pos + 1
+                    return intmask(currentsize + end - pos + 1)
         # fstat didn't work
         return currentsize + (currentsize >> 3) + 6
 

From noreply at buildbot.pypy.org  Thu Sep 11 19:29:21 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 11 Sep 2014 19:29:21 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: work on the proposal
Message-ID: <20140911172921.11F3F1C3637@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: extradoc
Changeset: r5394:4b47262f49ca
Date: 2014-09-11 11:29 -0600
http://bitbucket.org/pypy/extradoc/changeset/4b47262f49ca/

Log:	work on the proposal

diff --git a/talk/pycon2015/status/abstract.rst b/talk/pycon2015/status/abstract.rst
--- a/talk/pycon2015/status/abstract.rst
+++ b/talk/pycon2015/status/abstract.rst
@@ -1,13 +1,24 @@
-Status of PyPy and its ecosystem
-================================
+PyPy - the last 2 years of progress
+===================================
 
-Description
------------
+PyPy has been in the works for more than ten years and has reached relative
+maturity with more and more libraries working under PyPy and more deployments
+happening. Right now it entertains between 0.5-1.0% of PyPI package downloads
+(with CPython taking virtually all of the rest), used mostly for
+high-performance web servers.
 
-PyPy is a fast alternative implementation of Python. This talk will describe
-what happened in the PyPy ecosystem in the last 2 years and what the future
-holds. Topics such as JIT improvements, garbage collection, CFFI, Numpy, STM
-and Python 3 will be covered.
+Since no PyPy talk happened at PyCon 2014, we would like to present what
+we have achieved during the two years between talks. We would like to cover
+advancements in the PyPy performance landscape, but more importantly how
+we're addresssing the community needs and building the ecosystem. These days
+a lot of libraries that used to bind to C using CPython C API are either
+using cffi or have alternatives using cffi.
+
+We would also like to walk through a few success stories that we have
+experienced. Unforunately the biggest chunk of PyPy clients are very
+secretive (e.g. trading companies), but we can still present a few case studies.
+
+XXXX
 
 Abstract
 --------
diff --git a/talk/uct2014/example/demo1.py b/talk/uct2014/example/demo1.py
--- a/talk/uct2014/example/demo1.py
+++ b/talk/uct2014/example/demo1.py
@@ -12,7 +12,7 @@
     except ValueError:
         print "Requires ints"
         return 1
-    print "Added", arg0 + arg1
+    print "Got", adder(arg0, arg1)
     return 0
 
 def target(*args):
diff --git a/talk/uct2014/example/interp.py b/talk/uct2014/example/interp.py
--- a/talk/uct2014/example/interp.py
+++ b/talk/uct2014/example/interp.py
@@ -46,3 +46,7 @@
 
 def target(*args):
     return entry_point
+
+if __name__ == '__main__':
+    import sys
+    entry_point(sys.argv)

From noreply at buildbot.pypy.org  Thu Sep 11 19:49:02 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu, 11 Sep 2014 19:49:02 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: cleanup
Message-ID: <20140911174902.161A71C1036@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73463:ba680c1a8f5a
Date: 2014-09-11 11:57 -0400
http://bitbucket.org/pypy/pypy/changeset/ba680c1a8f5a/

Log:	cleanup

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -269,9 +269,6 @@
             if self._setbuf:
                 lltype.free(self._setbuf, flavor='raw')
 
-    def _cleanup_(self):
-        self._ll_file = lltype.nullptr(FILEP.TO)
-
     def close(self):
         """Closes the described file.
 
@@ -297,6 +294,9 @@
                     self._setbuf = lltype.nullptr(rffi.CCHARP.TO)
         return res
 
+    def _cleanup_(self):
+        self._ll_file = lltype.nullptr(FILEP.TO)
+
     def _check_closed(self):
         if not self._ll_file:
             raise ValueError("I/O operation on closed file")

From noreply at buildbot.pypy.org  Thu Sep 11 19:49:03 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu, 11 Sep 2014 19:49:03 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: handle eintr in rfile
	readline
Message-ID: <20140911174903.5525B1C1036@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73464:12e45d6f7dbf
Date: 2014-09-11 13:48 -0400
http://bitbucket.org/pypy/pypy/changeset/12e45d6f7dbf/

Log:	handle eintr in rfile readline

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -478,50 +478,71 @@
             return s.build()
         else:  # size > 0 or self._univ_newline
             ll_file = self._ll_file
+            newlinetypes = self._newlinetypes
+            skipnextlf = self._skipnextlf
             c = 0
             s = StringBuilder()
-            if self._univ_newline:
-                newlinetypes = self._newlinetypes
-                skipnextlf = self._skipnextlf
-                while size < 0 or s.getlength() < size:
-                    c = c_getc(ll_file)
+            while True:
+                if self._univ_newline:
+                    while size < 0 or s.getlength() < size:
+                        c = c_getc(ll_file)
+                        if c == EOF:
+                            break
+                        if skipnextlf:
+                            skipnextlf = False
+                            if c == ord('\n'):
+                                newlinetypes |= NEWLINE_CRLF
+                                c = c_getc(ll_file)
+                                if c == EOF:
+                                    break
+                            else:
+                                newlinetypes |= NEWLINE_CR
+                        if c == ord('\r'):
+                            skipnextlf = True
+                            c = ord('\n')
+                        elif c == ord('\n'):
+                            newlinetypes |= NEWLINE_LF
+                        s.append(chr(c))
+                        if c == ord('\n'):
+                            break
                     if c == EOF:
-                        break
-                    if skipnextlf:
-                        skipnextlf = False
+                        if c_ferror(ll_file) and rposix.get_errno() == errno.EINTR:
+                            self._newlinetypes = newlinetypes
+                            self._skipnextlf = skipnextlf
+                            if self._signal_checker is not None:
+                                self._signal_checker()
+                            c_clearerr(ll_file)
+                            continue
+                        if skipnextlf:
+                            newlinetypes |= NEWLINE_CR
+                else:
+                    while s.getlength() < size:
+                        c = c_getc(ll_file)
+                        if c == EOF:
+                            break
+                        s.append(chr(c))
                         if c == ord('\n'):
-                            newlinetypes |= NEWLINE_CRLF
-                            c = c_getc(ll_file)
-                            if c == EOF:
-                                break
-                        else:
-                            newlinetypes |= NEWLINE_CR
-                    if c == ord('\r'):
-                        skipnextlf = True
-                        c = ord('\n')
-                    elif c == ord('\n'):
-                        newlinetypes |= NEWLINE_LF
-                    s.append(chr(c))
-                    if c == ord('\n'):
-                        break
-                if c == EOF:
-                    if skipnextlf:
-                        newlinetypes |= NEWLINE_CR
+                            break
                 self._newlinetypes = newlinetypes
                 self._skipnextlf = skipnextlf
-            else:
-                while s.getlength() < size:
-                    c = c_getc(ll_file)
-                    if c == EOF:
-                        break
-                    s.append(chr(c))
-                    if c == ord('\n'):
-                        break
-            if c == EOF:
-                if c_ferror(ll_file):
+                if c == ord('\n'):
+                    break
+                elif c == EOF:
+                    if c_ferror(ll_file):
+                        if rposix.get_errno() == errno.EINTR:
+                            if self._signal_checker is not None:
+                                self._signal_checker()
+                            c_clearerr(ll_file)
+                            continue
+                        c_clearerr(ll_file)
+                        raise _from_errno(IOError)
                     c_clearerr(ll_file)
-                    raise _from_errno(IOError)
-                c_clearerr(ll_file)
+                    if self._signal_checker is not None:
+                        self._signal_checker()
+                    break
+                else:
+                    assert s.getlength() == size
+                    break
             return s.build()
 
     @enforceargs(None, str)

From noreply at buildbot.pypy.org  Thu Sep 11 20:01:33 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 11 Sep 2014 20:01:33 +0200 (CEST)
Subject: [pypy-commit] pypy gc_no_cleanup_nursery: Backed out changeset
	6dfd216b0e8d
Message-ID: <20140911180133.0FD8C1D2AD4@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: gc_no_cleanup_nursery
Changeset: r73465:3bb2c92cc2d3
Date: 2014-09-11 11:36 -0600
http://bitbucket.org/pypy/pypy/changeset/3bb2c92cc2d3/

Log:	Backed out changeset 6dfd216b0e8d

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -630,17 +630,17 @@
 
     def bh_new(self, sizedescr):
         return lltype.cast_opaque_ptr(llmemory.GCREF,
-                                      lltype.malloc(sizedescr.S, zero=True))
+                                      lltype.malloc(sizedescr.S))
 
     def bh_new_with_vtable(self, vtable, descr):
-        result = lltype.malloc(descr.S, zero=True)
+        result = lltype.malloc(descr.S)
         result_as_objptr = lltype.cast_pointer(rclass.OBJECTPTR, result)
         result_as_objptr.typeptr = support.cast_from_int(rclass.CLASSTYPE,
                                                          vtable)
         return lltype.cast_opaque_ptr(llmemory.GCREF, result)
 
     def bh_new_array(self, length, arraydescr):
-        array = lltype.malloc(arraydescr.A, length, zero=True)
+        array = lltype.malloc(arraydescr.A, length)
         return lltype.cast_opaque_ptr(llmemory.GCREF, array)
 
     def bh_classof(self, struct):
diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py
--- a/rpython/jit/backend/llsupport/descr.py
+++ b/rpython/jit/backend/llsupport/descr.py
@@ -35,11 +35,9 @@
     size = 0      # help translation
     tid = llop.combine_ushort(lltype.Signed, 0, 0)
 
-    def __init__(self, size, count_fields_if_immut=-1,
-                 offsets_of_gcptrs=None):
+    def __init__(self, size, count_fields_if_immut=-1):
         self.size = size
         self.count_fields_if_immut = count_fields_if_immut
-        self.offsets_of_gcptrs = offsets_of_gcptrs
 
     def count_fields_if_immutable(self):
         return self.count_fields_if_immut
@@ -60,13 +58,10 @@
     except KeyError:
         size = symbolic.get_size(STRUCT, gccache.translate_support_code)
         count_fields_if_immut = heaptracker.count_fields_if_immutable(STRUCT)
-        offsets_of_gcptrs = heaptracker.offsets_of_gcptrs(gccache, STRUCT)
         if heaptracker.has_gcstruct_a_vtable(STRUCT):
-            sizedescr = SizeDescrWithVTable(size, count_fields_if_immut,
-                                            offsets_of_gcptrs)
+            sizedescr = SizeDescrWithVTable(size, count_fields_if_immut)
         else:
-            sizedescr = SizeDescr(size, count_fields_if_immut,
-                                  offsets_of_gcptrs)
+            sizedescr = SizeDescr(size, count_fields_if_immut)
         gccache.init_size_descr(STRUCT, sizedescr)
         cache[STRUCT] = sizedescr
         return sizedescr
diff --git a/rpython/jit/backend/llsupport/test/test_descr.py b/rpython/jit/backend/llsupport/test/test_descr.py
--- a/rpython/jit/backend/llsupport/test/test_descr.py
+++ b/rpython/jit/backend/llsupport/test/test_descr.py
@@ -15,9 +15,6 @@
                              ('y', lltype.Ptr(T)))
     descr_s = get_size_descr(c0, S)
     descr_t = get_size_descr(c0, T)
-    assert descr_t.offsets_of_gcptrs == []
-    assert descr_s.offsets_of_gcptrs == [
-        symbolic.get_field_token(S, "y", False)[0]]
     assert descr_s.size == symbolic.get_size(S, False)
     assert descr_t.size == symbolic.get_size(T, False)
     assert descr_s.count_fields_if_immutable() == -1
diff --git a/rpython/jit/codewriter/heaptracker.py b/rpython/jit/codewriter/heaptracker.py
--- a/rpython/jit/codewriter/heaptracker.py
+++ b/rpython/jit/codewriter/heaptracker.py
@@ -1,6 +1,5 @@
 from rpython.rtyper.lltypesystem import lltype, llmemory, rclass
 from rpython.rlib.objectmodel import we_are_translated
-from rpython.jit.backend.llsupport import symbolic
 
 
 def adr2int(addr):
@@ -126,16 +125,3 @@
     vtable = descr.as_vtable_size_descr()._corresponding_vtable
     vtable = llmemory.cast_ptr_to_adr(vtable)
     return adr2int(vtable)
-
-def offsets_of_gcptrs(gccache, STRUCT, res=None):
-    if res is None:
-        res = []
-    for name, FIELD in STRUCT._flds.iteritems():
-        if isinstance(FIELD, lltype.Ptr) and FIELD._needsgc():
-            
-            offset, _ = symbolic.get_field_token(STRUCT, name,
-                                        gccache.translate_support_code)
-            res.append(offset)
-        elif isinstance(FIELD, lltype.Struct):
-            offsets_of_gcptrs(gccache, FIELD, res)
-    return res

From noreply at buildbot.pypy.org  Thu Sep 11 20:01:34 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 11 Sep 2014 20:01:34 +0200 (CEST)
Subject: [pypy-commit] pypy gc_no_cleanup_nursery: Backout the changes,
 come back to the previous approach
Message-ID: <20140911180134.981201D2AD4@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: gc_no_cleanup_nursery
Changeset: r73466:1066bc5225af
Date: 2014-09-11 11:36 -0600
http://bitbucket.org/pypy/pypy/changeset/1066bc5225af/

Log:	Backout the changes, come back to the previous approach

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -643,6 +643,20 @@
         array = lltype.malloc(arraydescr.A, length)
         return lltype.cast_opaque_ptr(llmemory.GCREF, array)
 
+    def bh_clear_array_contents(self, a, descr):
+        a = support.cast_arg(lltype.Ptr(descr.A), a)
+        ITEM = descr.A.OF
+        array = a._obj
+        if isinstance(ITEM, lltype.Struct):
+            for i in xrange(array.getlength()):
+                for name, FIELD in ITEM._flds.iteritems():
+                    null = FIELD._defl()
+                    setattr(array.getitem(i), name, null)
+        else:
+            null = ITEM._defl()
+            for i in xrange(array.getlength()):
+                array.setitem(i, null)
+
     def bh_classof(self, struct):
         struct = lltype.cast_opaque_ptr(rclass.OBJECTPTR, struct)
         result_adr = llmemory.cast_ptr_to_adr(struct.typeptr)
diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -591,6 +591,14 @@
     def bh_new(self, sizedescr):
         return self.gc_ll_descr.gc_malloc(sizedescr)
 
+    def bh_clear_array_contents(self, ref, arraydescr):
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        arraysize = self.bh_arraylen_gc(ref, arraydescr)
+        totalsize = size * arraysize
+        adr = rffi.cast(lltype.Signed, ref) + ofs
+        self.gc_ll_descr.memset_ptr(adr, rffi.cast(rffi.INT, 0),
+                                    rffi.cast(rffi.SIZE_T, totalsize))
+
     def bh_new_with_vtable(self, vtable, sizedescr):
         res = self.gc_ll_descr.gc_malloc(sizedescr)
         if self.vtable_offset is not None:
diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -59,6 +59,10 @@
             if op.is_malloc():
                 self.handle_malloc_operation(op)
                 continue
+            elif op.getopnum() == rop.CLEAR_ARRAY_CONTENTS:
+                if not self.gc_ll_descr.malloc_zero_filled:
+                    self.handle_clear_array_contents(op)
+                continue
             elif op.can_malloc():
                 self.emitting_an_operation_that_can_collect()
             elif op.getopnum() == rop.LABEL:
@@ -157,6 +161,26 @@
             else:
                 raise NotImplementedError(op.getopname())
 
+    def handle_clear_array_contents(self, op):
+        # XXX this maybe should go to optimizer, so we can remove extra ops?
+        arraydescr = op.getdescr()
+        ofs, size, _ = self.cpu.unpack_arraydescr_size(arraydescr)
+        v_arr = op.getarg(0)
+        v_arr_plus_ofs = BoxInt()
+        v_arrsize = BoxInt()
+        v_totalsize = BoxInt()
+        gcdescr = self.gc_ll_descr
+        ops = [
+            ResOperation(rop.INT_ADD, [v_arr, ConstInt(size)], v_arr_plus_ofs),
+            ResOperation(rop.ARRAYLEN_GC, [v_arr], v_arrsize, descr=arraydescr),
+            ResOperation(rop.INT_MUL, [v_arrsize, ConstInt(size)], v_totalsize),
+            ResOperation(rop.CALL, [ConstInt(gcdescr.memset_ptr_as_int),
+                                    v_arr_plus_ofs,
+                                    ConstInt(0), v_totalsize], None,
+                                    descr=gcdescr.memset_descr),
+        ]
+        self.newops.extend(ops)
+
     def gen_malloc_frame(self, frame_info, frame, size_box):
         descrs = self.gc_ll_descr.getframedescrs(self.cpu)
         if self.gc_ll_descr.kind == 'boehm':
diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -4439,3 +4439,21 @@
         res = self.execute_operation(rop.CAST_FLOAT_TO_SINGLEFLOAT,
                                    [boxfloat(12.5)], 'int')
         assert res.getint() == struct.unpack("I", struct.pack("f", 12.5))[0]
+
+    def test_clear_array_contents(self):
+        from rpython.jit.backend.llsupport.llmodel import AbstractLLCPU
+        if not isinstance(self.cpu, AbstractLLCPU):
+            py.test.skip("pointless test on non-asm")
+        oldval = self.cpu.gc_ll_descr.malloc_zero_filled
+        self.cpu.gc_ll_descr.malloc_zero_filled = False
+        try:
+            A = lltype.GcArray(lltype.Signed)
+            a = lltype.malloc(A, 3)
+            a[1] = 13
+            descr = self.cpu.arraydescrof(A)
+            ref = lltype.cast_opaque_ptr(llmemory.GCREF, a)
+            self.execute_operation(rop.CLEAR_ARRAY_CONTENTS,
+                                   [BoxPtr(ref)], 'void', descr=descr)
+            assert a[1] == 0
+        finally:
+            self.cpu.gc_ll_descr.malloc_zero_filled = oldval
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -315,7 +315,8 @@
             self.assembler.mc.mark_op(op)
             self.rm.position = i
             self.xrm.position = i
-            if op.has_no_side_effect() and op.result not in self.longevity:
+            if (op.has_no_side_effect() and op.result not in self.longevity
+                and op.opnum != rop.CLEAR_ARRAY_CONTENTS):
                 i += 1
                 self.possibly_free_vars_for_op(op)
                 continue
diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -612,8 +612,39 @@
             # XXX only strings or simple arrays for now
             ARRAY = op.args[0].value
             arraydescr = self.cpu.arraydescrof(ARRAY)
-            return SpaceOperation('new_array', [op.args[2], arraydescr],
-                                  op.result)
+            op1 = SpaceOperation('new_array', [op.args[2], arraydescr],
+                                 op.result)
+            if self._has_gcptrs_in(ARRAY):
+                return self.zero_contents([op1], op.result, ARRAY,
+                                          only_gc_pointers=True)
+            if op.args[1].value.get('zero', False):
+                return self.zero_contents([op1], op.result, ARRAY)
+            return op1
+
+    def zero_contents(self, ops, v, TYPE, only_gc_pointers=False):
+        if isinstance(TYPE, lltype.Struct):
+            for name, FIELD in TYPE._flds.iteritems():
+                if (not only_gc_pointers or
+                    isinstance(FIELD, lltype.Ptr) and FIELD._needsgc()):
+                    c_name = Constant(name, lltype.Void)
+                    c_null = Constant(FIELD._defl(), FIELD)
+                    op = SpaceOperation('setfield', [v, c_name, c_null],
+                                        None)
+                    self.extend_with(ops, self.rewrite_op_setfield(op,
+                                          override_type=TYPE))
+                elif isinstance(FIELD, lltype.Struct):
+                    # substruct
+                    self.zero_contents(ops, v, FIELD,
+                                       only_gc_pointers=only_gc_pointers)
+        elif isinstance(TYPE, lltype.Array):
+            arraydescr = self.cpu.arraydescrof(TYPE)
+            ops.append(SpaceOperation('clear_array_contents',
+                                      [v, arraydescr], None))
+        else:
+            raise TypeError("Expected struct or array, got '%r'", (TYPE,))
+        if len(ops) == 1:
+            return ops[0]
+        return ops
 
     def extend_with(self, l, ops):
         if ops is None:
@@ -899,7 +930,28 @@
         else:
             opname = 'new'
         sizedescr = self.cpu.sizeof(STRUCT)
-        return SpaceOperation(opname, [sizedescr], op.result)
+        op1 = SpaceOperation(opname, [sizedescr], op.result)
+        if true_zero:
+            return self.zero_contents([op1], op.result, STRUCT)
+        if self._has_gcptrs_in(STRUCT):
+            return self.zero_contents([op1], op.result, STRUCT,
+                                      only_gc_pointers=True)
+        return op1
+
+    def _has_gcptrs_in(self, STRUCT):
+        if isinstance(STRUCT, lltype.Array):
+            ITEM = STRUCT.OF
+            if isinstance(ITEM, lltype.Struct):
+                STRUCT = ITEM
+            else:
+                return isinstance(ITEM, lltype.Ptr) and ITEM._needsgc()
+        for FIELD in STRUCT._flds.values():
+            if isinstance(FIELD, lltype.Ptr) and FIELD._needsgc():
+                return True
+            elif isinstance(FIELD, lltype.Struct):
+                if self._has_gcptrs_in(FIELD):
+                    return True
+        return False
 
     def rewrite_op_getinteriorarraysize(self, op):
         # only supports strings and unicodes
@@ -1625,6 +1677,8 @@
             v.concretetype = lltype.Signed
             ops.append(SpaceOperation('int_force_ge_zero', [v_length], v))
         ops.append(SpaceOperation('new_array', [v, arraydescr], op.result))
+        if self._has_gcptrs_in(op.result.concretetype.TO):
+            self.zero_contents(ops, op.result, op.result.concretetype.TO)
         return ops
 
     def do_fixed_list_len(self, op, args, arraydescr):
diff --git a/rpython/jit/codewriter/test/test_jtransform.py b/rpython/jit/codewriter/test/test_jtransform.py
--- a/rpython/jit/codewriter/test/test_jtransform.py
+++ b/rpython/jit/codewriter/test/test_jtransform.py
@@ -529,6 +529,46 @@
     assert op1.opname == 'new'
     assert op1.args == [('sizedescr', S)]
 
+def test_malloc_new_zero():
+    SS = lltype.GcStruct('SS')
+    S = lltype.GcStruct('S', ('x', lltype.Ptr(SS)))
+    v = varoftype(lltype.Ptr(S))
+    op = SpaceOperation('malloc', [Constant(S, lltype.Void),
+                                   Constant({'flavor': 'gc'}, lltype.Void)], v)
+    op1, op2 = Transformer(FakeCPU()).rewrite_operation(op)
+    assert op1.opname == 'new'
+    assert op1.args == [('sizedescr', S)]
+    assert op2.opname == 'setfield_gc_r'
+    assert op2.args[0] == v
+
+def test_malloc_new_zero_2():
+    S = lltype.GcStruct('S', ('x', lltype.Signed))
+    v = varoftype(lltype.Ptr(S))
+    op = SpaceOperation('malloc', [Constant(S, lltype.Void),
+                                   Constant({'flavor': 'gc',
+                                             'zero': True}, lltype.Void)], v)
+    op1, op2 = Transformer(FakeCPU()).rewrite_operation(op)
+    assert op1.opname == 'new'
+    assert op1.args == [('sizedescr', S)]
+    assert op2.opname == 'setfield_gc_i'
+    assert op2.args[0] == v
+
+def test_malloc_new_zero_nested():
+    S0 = lltype.GcStruct('S0')
+    S = lltype.Struct('S', ('x', lltype.Ptr(S0)))
+    S2 = lltype.GcStruct('S2', ('parent', S),
+                         ('xx', lltype.Ptr(S0)))
+    v = varoftype(lltype.Ptr(S2))
+    op = SpaceOperation('malloc', [Constant(S2, lltype.Void),
+                                   Constant({'flavor': 'gc'}, lltype.Void)], v)
+    op1, op2, op3 = Transformer(FakeCPU()).rewrite_operation(op)
+    assert op1.opname == 'new'
+    assert op1.args == [('sizedescr', S2)]
+    assert op2.opname == 'setfield_gc_r'
+    assert op2.args[0] == v
+    assert op3.opname == 'setfield_gc_r'
+    assert op3.args[0] == v
+
 def test_malloc_new_with_vtable():
     vtable = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True)
     S = lltype.GcStruct('S', ('parent', rclass.OBJECT))
@@ -1026,6 +1066,20 @@
     assert op1.args == [v1]
     assert op1.result == v2
 
+def test_malloc_varsize_zero():
+    c_A = Constant(lltype.GcArray(lltype.Signed), lltype.Void)
+    c_flags = Constant({"flavor": "gc"}, lltype.Void)
+    v1 = varoftype(lltype.Signed)
+    v2 = varoftype(c_A.value)
+    op = SpaceOperation('malloc_varsize', [c_A, c_flags, v1], v2)
+    op1 = Transformer(FakeCPU()).rewrite_operation(op)
+    assert op1.opname == 'new_array'
+    c_flags = Constant({"flavor": "gc", "zero": True}, lltype.Void)
+    op = SpaceOperation('malloc_varsize', [c_A, c_flags, v1], v2)
+    op1, op2 = Transformer(FakeCPU()).rewrite_operation(op)
+    assert op1.opname == 'new_array'
+    assert op2.opname == 'clear_array_contents'
+
 def test_str_concat():
     # test that the oopspec is present and correctly transformed
     PSTR = lltype.Ptr(rstr.STR)
diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py
--- a/rpython/jit/metainterp/blackhole.py
+++ b/rpython/jit/metainterp/blackhole.py
@@ -1345,6 +1345,10 @@
         vtable = heaptracker.descr2vtable(cpu, descr)
         return cpu.bh_new_with_vtable(vtable, descr)
 
+    @arguments("cpu", "r", "d")
+    def bhimpl_clear_array_contents(cpu, ref, descr):
+        cpu.bh_clear_array_contents(ref, descr)    
+
     @arguments("cpu", "r", returns="i")
     def bhimpl_guard_class(cpu, struct):
         return cpu.bh_classof(struct)
diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py
--- a/rpython/jit/metainterp/executor.py
+++ b/rpython/jit/metainterp/executor.py
@@ -335,6 +335,7 @@
                          rop.CALL_MALLOC_NURSERY,
                          rop.CALL_MALLOC_NURSERY_VARSIZE,
                          rop.CALL_MALLOC_NURSERY_VARSIZE_FRAME,
+                         rop.CLEAR_ARRAY_CONTENTS,
                          rop.LABEL,
                          ):      # list of opcodes never executed by pyjitpl
                 continue
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -5461,6 +5461,17 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_virtual_clear_array_contents(self):
+        ops = """
+        []
+        p0 = new_array(2, descr=arraydescr)
+        clear_array_contents(p0, descr=arraydescr)
+        """
+        expected = """
+        []
+        """
+        self.optimize_loop(ops, expected)
+
 
 class TestLLtype(BaseTestOptimizeBasic, LLtypeMixin):
     pass
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -662,6 +662,12 @@
         else:
             self.emit_operation(op)
 
+    def optimize_CLEAR_ARRAY_CONTENTS(self, op):
+        v = self.getvalue(op.getarg(0))
+        if v.is_virtual():
+            return
+        self.emit_operation(op)
+
     def optimize_CALL(self, op):
         effectinfo = op.getdescr().get_extra_info()
         if effectinfo.oopspecindex == EffectInfo.OS_RAW_MALLOC_VARSIZE_CHAR:
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -396,6 +396,10 @@
     def opimpl_new(self, sizedescr):
         return self.metainterp.execute_new(sizedescr)
 
+    @arguments("box", "descr")
+    def opimpl_clear_array_contents(self, box, descr):
+        self.metainterp.execute_and_record(rop.CLEAR_ARRAY_CONTENTS, descr, box)
+
     @arguments("descr")
     def opimpl_new_with_vtable(self, sizedescr):
         cpu = self.metainterp.cpu
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -492,6 +492,9 @@
     'MARK_OPAQUE_PTR/1b',
     # this one has no *visible* side effect, since the virtualizable
     # must be forced, however we need to execute it anyway
+    'CLEAR_ARRAY_CONTENTS/1d',
+    # this one does not *really* have a side effect since it's equivalent
+    # to array just coming zeroed
     '_NOSIDEEFFECT_LAST', # ----- end of no_side_effect operations -----
 
     'INCREMENT_DEBUG_COUNTER/1',

From noreply at buildbot.pypy.org  Thu Sep 11 20:01:35 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 11 Sep 2014 20:01:35 +0200 (CEST)
Subject: [pypy-commit] pypy gc_no_cleanup_nursery: oops
Message-ID: <20140911180135.CB4D81D2AD4@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: gc_no_cleanup_nursery
Changeset: r73467:91d9dc7a4a1c
Date: 2014-09-11 12:00 -0600
http://bitbucket.org/pypy/pypy/changeset/91d9dc7a4a1c/

Log:	oops

diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -865,7 +865,7 @@
 
     def gct_do_malloc_varsize_clear(self, hop):
         # used by the JIT (see rpython.jit.backend.llsupport.gc)
-        self.gct_do_malloc_fixedsize(hop)
+        self.gct_do_malloc_varsize(hop)
         if not self.malloc_zero_filled:
             op = hop.spaceop
             v_size = op.args[1]

From noreply at buildbot.pypy.org  Thu Sep 11 21:09:11 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 11 Sep 2014 21:09:11 +0200 (CEST)
Subject: [pypy-commit] pypy no-write-barrier-in-const-ptrs: We don't need
 write barrier for ConstPtrs (since they're always old anyhow)
Message-ID: <20140911190911.5A3351C1292@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: no-write-barrier-in-const-ptrs
Changeset: r73468:e7e97150a2cd
Date: 2014-09-11 13:08 -0600
http://bitbucket.org/pypy/pypy/changeset/e7e97150a2cd/

Log:	We don't need write barrier for ConstPtrs (since they're always old
	anyhow)

diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -390,8 +390,7 @@
         val = op.getarg(0)
         if val not in self.write_barrier_applied:
             v = op.getarg(1)
-            if isinstance(v, BoxPtr) or (isinstance(v, ConstPtr) and
-                                         bool(v.value)): # store a non-NULL
+            if isinstance(v, BoxPtr) or not isinstance(v, ConstPtr):
                 self.gen_write_barrier(val)
                 #op = op.copy_and_change(rop.SETFIELD_RAW)
         self.newops.append(op)
@@ -400,8 +399,7 @@
         val = op.getarg(0)
         if val not in self.write_barrier_applied:
             v = op.getarg(2)
-            if isinstance(v, BoxPtr) or (isinstance(v, ConstPtr) and
-                                         bool(v.value)): # store a non-NULL
+            if isinstance(v, BoxPtr) or not isinstance(v, ConstPtr):
                 self.gen_write_barrier_array(val, op.getarg(1))
                 #op = op.copy_and_change(rop.SET{ARRAYITEM,INTERIORFIELD}_RAW)
         self.newops.append(op)

From noreply at buildbot.pypy.org  Thu Sep 11 21:35:38 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu, 11 Sep 2014 21:35:38 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: fix get_stdio wrappers
Message-ID: <20140911193538.99E601C3637@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73469:fdf663e56443
Date: 2014-09-11 15:34 -0400
http://bitbucket.org/pypy/pypy/changeset/fdf663e56443/

Log:	fix get_stdio wrappers

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -25,10 +25,12 @@
 stdio_streams = ['stdin', 'stdout', 'stderr']
 separate_module_sources = ['\n'.join('FILE* get_%s() { return %s; }' % (s, s)
                                      for s in stdio_streams)]
+post_include_bits = ['FILE* get_%s();' % s for s in stdio_streams]
 export_symbols = ['get_%s' % s for s in stdio_streams]
 
 eci = ExternalCompilationInfo(includes=includes,
                               separate_module_sources=separate_module_sources,
+                              post_include_bits=post_include_bits,
                               export_symbols=export_symbols)
 
 

From noreply at buildbot.pypy.org  Thu Sep 11 21:40:59 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu, 11 Sep 2014 21:40:59 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: add simple target to
	test rfile
Message-ID: <20140911194059.B21441C3637@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73470:e42707eeae0d
Date: 2014-09-11 15:40 -0400
http://bitbucket.org/pypy/pypy/changeset/e42707eeae0d/

Log:	add simple target to test rfile

diff --git a/rpython/translator/goal/targetrfile.py b/rpython/translator/goal/targetrfile.py
new file mode 100644
--- /dev/null
+++ b/rpython/translator/goal/targetrfile.py
@@ -0,0 +1,15 @@
+from rpython.rlib import rfile
+
+def entry_point(argv):
+    i, o, e = rfile.create_stdio()
+    o.write('test\n')
+    return 0
+
+# _____ Define and setup target ___
+
+def target(*args):
+    return entry_point, None
+
+if __name__ == '__main__':
+    import sys
+    sys.exit(entry_point(sys.argv))

From noreply at buildbot.pypy.org  Thu Sep 11 22:44:37 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu, 11 Sep 2014 22:44:37 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: simplify rfile fread
Message-ID: <20140911204437.4829D1C31C8@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73471:51dc6795d2f6
Date: 2014-09-11 16:03 -0400
http://bitbucket.org/pypy/pypy/changeset/51dc6795d2f6/

Log:	simplify rfile fread

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -315,27 +315,26 @@
         if not self._univ_newline:
             return c_fread(buf, 1, n, stream)
 
-        i = 0  # XXX how to do ptrdiff (dst - buf) instead?
-        dst = buf
+        i = 0
         newlinetypes = self._newlinetypes
         skipnextlf = self._skipnextlf
         assert n >= 0
         while n:
+            dst = rffi.ptradd(buf, i)
             nread = c_fread(dst, 1, n, stream)
             assert nread <= n
             if nread == 0:
                 break
 
-            src = dst
+            j = 0
             n -= nread
             shortread = n != 0
             while nread:
                 nread -= 1
-                c = src[0]
-                src = rffi.ptradd(src, 1)
+                c = dst[j]
+                j += 1
                 if c == '\r':
-                    dst[0] = '\n'
-                    dst = rffi.ptradd(dst, 1)
+                    buf[i] = '\n'
                     i += 1
                     skipnextlf = True
                 elif skipnextlf and c == '\n':
@@ -347,8 +346,7 @@
                         newlinetypes |= NEWLINE_LF
                     elif skipnextlf:
                         newlinetypes |= NEWLINE_CR
-                    dst[0] = c
-                    dst = rffi.ptradd(dst, 1)
+                    buf[i] = c
                     i += 1
                     skipnextlf = False
             if shortread:

From noreply at buildbot.pypy.org  Thu Sep 11 22:44:38 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu, 11 Sep 2014 22:44:38 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: avoid releasing gil for
 some rfile functions
Message-ID: <20140911204438.771901C31C8@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73472:851a85c58d90
Date: 2014-09-11 16:27 -0400
http://bitbucket.org/pypy/pypy/changeset/851a85c58d90/

Log:	avoid releasing gil for some rfile functions

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -87,25 +87,26 @@
 _fclose2 = (c_fclose, c_fclose_in_del)
 _pclose2 = (c_pclose, c_pclose_in_del)
 
-c_getc = llexternal('getc', [FILEP], rffi.INT, macro=True)
-c_ungetc = llexternal('ungetc', [rffi.INT, FILEP], rffi.INT)
+c_getc = llexternal('getc', [FILEP], rffi.INT, macro=True, releasegil=False)
+c_ungetc = llexternal('ungetc', [rffi.INT, FILEP], rffi.INT, releasegil=False)
+
 c_fgets = llexternal('fgets', [rffi.CCHARP, rffi.INT, FILEP], rffi.CCHARP)
 c_fread = llexternal('fread', [rffi.CCHARP, rffi.SIZE_T, rffi.SIZE_T, FILEP],
                      rffi.SIZE_T)
-
 c_fwrite = llexternal('fwrite', [rffi.CCHARP, rffi.SIZE_T, rffi.SIZE_T, FILEP],
                       rffi.SIZE_T)
+
 c_fflush = llexternal('fflush', [FILEP], rffi.INT)
 c_fflush_nogil = llexternal('fflush', [FILEP], rffi.INT, releasegil=False)
 c_ftruncate = llexternal(ftruncate, [rffi.INT, OFF_T], rffi.INT, macro=True)
 
 c_fseek = llexternal('fseek', [FILEP, rffi.LONG, rffi.INT], rffi.INT)
 c_ftell = llexternal('ftell', [FILEP], rffi.LONG)
-c_fileno = llexternal(fileno, [FILEP], rffi.INT)
 
-c_feof = llexternal('feof', [FILEP], rffi.INT)
-c_ferror = llexternal('ferror', [FILEP], rffi.INT)
-c_clearerr = llexternal('clearerr', [FILEP], lltype.Void)
+c_fileno = llexternal(fileno, [FILEP], rffi.INT, releasegil=False)
+c_feof = llexternal('feof', [FILEP], rffi.INT, releasegil=False)
+c_ferror = llexternal('ferror', [FILEP], rffi.INT, releasegil=False)
+c_clearerr = llexternal('clearerr', [FILEP], lltype.Void, releasegil=False)
 
 c_stdin = llexternal('get_stdin', [], FILEP, _nowrapper=True)
 c_stdout = llexternal('get_stdout', [], FILEP, _nowrapper=True)

From noreply at buildbot.pypy.org  Thu Sep 11 22:44:39 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu, 11 Sep 2014 22:44:39 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: cleanup
Message-ID: <20140911204439.A559C1C31C8@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73473:6641180dd7a4
Date: 2014-09-11 16:36 -0400
http://bitbucket.org/pypy/pypy/changeset/6641180dd7a4/

Log:	cleanup

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -76,16 +76,21 @@
 c_setvbuf = llexternal('setvbuf', [FILEP, rffi.CCHARP, rffi.INT, rffi.SIZE_T],
                        rffi.INT)
 
-c_fclose = llexternal('fclose', [FILEP], rffi.INT)
-c_pclose = llexternal('pclose', [FILEP], rffi.INT)
-
-# Note: the following two functions are called from __del__ methods,
+# Note: the following functions are called from __del__ methods,
 # so must be 'releasegil=False'.  Otherwise, a program using both
 # threads and the RFile class cannot translate.  See c684bf704d1f
-c_fclose_in_del = llexternal('fclose', [FILEP], rffi.INT, releasegil=False)
-c_pclose_in_del = llexternal('pclose', [FILEP], rffi.INT, releasegil=False)
-_fclose2 = (c_fclose, c_fclose_in_del)
-_pclose2 = (c_pclose, c_pclose_in_del)
+_fflush = ('fflush', [FILEP], rffi.INT)
+c_fflush, c_fflush_nogil = (llexternal(*_fflush),
+                            llexternal(*_fflush, releasegil=False))
+_fclose = ('fclose', [FILEP], rffi.INT)
+c_fclose, c_fclose_nogil = (llexternal(*_fclose),
+                            llexternal(*_fclose, releasegil=False))
+_pclose = ('pclose', [FILEP], rffi.INT)
+c_pclose, c_pclose_nogil = (llexternal(*_pclose),
+                            llexternal(*_pclose, releasegil=False))
+
+_fclose2 = (c_fclose, c_fclose_nogil)
+_pclose2 = (c_pclose, c_pclose_nogil)
 
 c_getc = llexternal('getc', [FILEP], rffi.INT, macro=True, releasegil=False)
 c_ungetc = llexternal('ungetc', [rffi.INT, FILEP], rffi.INT, releasegil=False)
@@ -96,8 +101,6 @@
 c_fwrite = llexternal('fwrite', [rffi.CCHARP, rffi.SIZE_T, rffi.SIZE_T, FILEP],
                       rffi.SIZE_T)
 
-c_fflush = llexternal('fflush', [FILEP], rffi.INT)
-c_fflush_nogil = llexternal('fflush', [FILEP], rffi.INT, releasegil=False)
 c_ftruncate = llexternal(ftruncate, [rffi.INT, OFF_T], rffi.INT, macro=True)
 
 c_fseek = llexternal('fseek', [FILEP, rffi.LONG, rffi.INT], rffi.INT)

From noreply at buildbot.pypy.org  Thu Sep 11 22:48:31 2014
From: noreply at buildbot.pypy.org (agobi)
Date: Thu, 11 Sep 2014 22:48:31 +0200 (CEST)
Subject: [pypy-commit] pypy default: Introducing the missing method
 _compare_digest in the module operator.
Message-ID: <20140911204831.27D5F1C31C8@cobra.cs.uni-duesseldorf.de>

Author: Attila Gobi 
Branch: 
Changeset: r73474:ac26684c69de
Date: 2014-07-27 13:23 +0200
http://bitbucket.org/pypy/pypy/changeset/ac26684c69de/

Log:	Introducing the missing method _compare_digest in the module
	operator. The method is used in the hmac module. (grafted from
	b453ad72cd9a80f665d4dd4a0565f1321c9bc3be)

diff --git a/pypy/module/operator/__init__.py b/pypy/module/operator/__init__.py
--- a/pypy/module/operator/__init__.py
+++ b/pypy/module/operator/__init__.py
@@ -39,7 +39,7 @@
                     'irshift', 'isub', 'itruediv', 'ixor', '_length_hint']
 
     interpleveldefs = {
-        '_compare_digest': 'interp_operator.compare_digest',
+        '_compare_digest': 'tscmp.compare_digest',
     }
 
     for name in interp_names:
diff --git a/pypy/module/operator/test/test_operator.py b/pypy/module/operator/test/test_operator.py
--- a/pypy/module/operator/test/test_operator.py
+++ b/pypy/module/operator/test/test_operator.py
@@ -334,3 +334,15 @@
         assert operator._compare_digest(a, b)
         a, b = mybytes(b"foobar"), mybytes(b"foobaz")
         assert not operator._compare_digest(a, b)
+
+    def test_compare_digest_buffer(self):
+        import operator
+        assert operator._compare_digest(b'asd', b'asd')
+        assert not operator._compare_digest(b'asd', b'qwe')
+        assert not operator._compare_digest(b'asd', b'asdq')
+
+    def test_compare_digest_ascii(self):
+        import operator
+        assert operator._compare_digest('asd', 'asd')
+        assert not operator._compare_digest('asd', 'qwe')
+        assert not operator._compare_digest('asd', 'asdq')
diff --git a/pypy/module/operator/test/test_tscmp.py b/pypy/module/operator/test/test_tscmp.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/operator/test/test_tscmp.py
@@ -0,0 +1,15 @@
+from pypy.module.operator.tscmp import pypy_tscmp
+from rpython.rtyper.lltypesystem.rffi import scoped_nonmovingbuffer
+
+class TestTimingSafeCompare:
+    def test_tscmp_neq(self):
+        assert not pypy_tscmp('asd', 'qwe', 3, 3)
+
+    def test_tscmp_eq(self):
+        assert pypy_tscmp('asd', 'asd', 3, 3)
+
+    def test_tscmp_len(self):
+        assert pypy_tscmp('asdp', 'asdq', 3, 3)
+
+    def test_tscmp_nlen(self):
+        assert not pypy_tscmp('asd', 'asd', 2, 3)
diff --git a/pypy/module/operator/tscmp.c b/pypy/module/operator/tscmp.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/operator/tscmp.c
@@ -0,0 +1,42 @@
+/* From CPython 3.3.5's operator.c
+ */
+
+#include 
+#include "tscmp.h"
+
+int
+pypy_tscmp(const unsigned char *a, const unsigned char *b, long len_a, long len_b)
+{
+    /* The volatile type declarations make sure that the compiler has no
+     * chance to optimize and fold the code in any way that may change
+     * the timing.
+     */
+    volatile long length;
+    volatile const unsigned char *left;
+    volatile const unsigned char *right;
+    long i;
+    unsigned char result;
+
+    /* loop count depends on length of b */
+    length = len_b;
+    left = NULL;
+    right = b;
+
+    /* don't use else here to keep the amount of CPU instructions constant,
+     * volatile forces re-evaluation
+     *  */
+    if (len_a == length) {
+        left = *((volatile const unsigned char**)&a);
+        result = 0;
+    }
+    if (len_a != length) {
+        left = b;
+        result = 1;
+    }
+
+    for (i=0; i < length; i++) {
+        result |= *left++ ^ *right++;
+    }
+
+    return (result == 0);
+}
diff --git a/pypy/module/operator/tscmp.h b/pypy/module/operator/tscmp.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/operator/tscmp.h
@@ -0,0 +1,1 @@
+int pypy_tscmp(const unsigned char *a, const unsigned char *b, long len_a, long len_b);
diff --git a/pypy/module/operator/tscmp.py b/pypy/module/operator/tscmp.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/operator/tscmp.py
@@ -0,0 +1,42 @@
+"""
+Provides _compare_digest method, which is a safe comparing to prevent timing
+attacks for the hmac module.
+"""
+import py
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
+from pypy.interpreter.error import OperationError
+
+cwd = py.path.local(__file__).dirpath()
+eci = ExternalCompilationInfo(
+    includes=[cwd.join('tscmp.h')],
+    separate_module_files=[cwd.join('tscmp.c')],
+    export_symbols=['pypy_tscmp'])
+
+def llexternal(*args, **kwargs):
+    kwargs.setdefault('compilation_info', eci)
+    kwargs.setdefault('sandboxsafe', True)
+    return rffi.llexternal(*args, **kwargs)
+
+pypy_tscmp = llexternal('pypy_tscmp', [rffi.CCHARP, rffi.CCHARP, rffi.LONG, rffi.LONG], rffi.INT)
+
+def compare_digest(space, w_a, w_b):
+    if space.isinstance_w(w_a, space.w_unicode) and space.isinstance_w(w_b, space.w_unicode):
+        try:
+            a_value = space.call_method(w_a, "encode", space.wrap("ascii"))
+            b_value = space.call_method(w_b, "encode", space.wrap("ascii"))
+            return compare_digest_buffer(space, a_value, b_value)
+        except OperationError as e:
+            if not e.match(space, space.w_UnicodeEncodeError):
+                raise
+            raise OperationError(space.w_TypeError,
+                    space.wrap("comparing strings with non-ASCII characters is not supported"))
+    else:
+        return compare_digest_buffer(space, w_a, w_b)
+
+def compare_digest_buffer(space, w_a, w_b):
+    a = space.bufferstr_w(w_a)
+    b = space.bufferstr_w(w_b)
+    with rffi.scoped_nonmovingbuffer(a) as a_buffer:
+        with rffi.scoped_nonmovingbuffer(b) as b_buffer:
+            return space.wrap(pypy_tscmp(a_buffer, b_buffer, len(a), len(b)))

From noreply at buildbot.pypy.org  Thu Sep 11 22:48:32 2014
From: noreply at buildbot.pypy.org (agobi)
Date: Thu, 11 Sep 2014 22:48:32 +0200 (CEST)
Subject: [pypy-commit] pypy default: fixing return type of _compare_digest
Message-ID: <20140911204832.ABAD21C31C8@cobra.cs.uni-duesseldorf.de>

Author: Attila Gobi 
Branch: 
Changeset: r73475:718ffad76ab8
Date: 2014-07-27 13:36 +0200
http://bitbucket.org/pypy/pypy/changeset/718ffad76ab8/

Log:	fixing return type of _compare_digest (grafted from
	69a3e8128a9c8255ccfc4df889dc07a806167b20)

diff --git a/pypy/module/operator/tscmp.py b/pypy/module/operator/tscmp.py
--- a/pypy/module/operator/tscmp.py
+++ b/pypy/module/operator/tscmp.py
@@ -39,4 +39,4 @@
     b = space.bufferstr_w(w_b)
     with rffi.scoped_nonmovingbuffer(a) as a_buffer:
         with rffi.scoped_nonmovingbuffer(b) as b_buffer:
-            return space.wrap(pypy_tscmp(a_buffer, b_buffer, len(a), len(b)))
+            return space.wrap(rffi.cast(lltype.Bool, pypy_tscmp(a_buffer, b_buffer, len(a), len(b))))

From noreply at buildbot.pypy.org  Thu Sep 11 22:48:33 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Thu, 11 Sep 2014 22:48:33 +0200 (CEST)
Subject: [pypy-commit] pypy default: pep8/use oefmt
Message-ID: <20140911204833.CD6351C31C8@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey 
Branch: 
Changeset: r73476:d5039133da03
Date: 2014-07-27 11:35 -0700
http://bitbucket.org/pypy/pypy/changeset/d5039133da03/

Log:	pep8/use oefmt (grafted from
	6704fc912ee12bb7be511d77fd192537ca3d75e3)

diff --git a/pypy/module/operator/test/test_tscmp.py b/pypy/module/operator/test/test_tscmp.py
--- a/pypy/module/operator/test/test_tscmp.py
+++ b/pypy/module/operator/test/test_tscmp.py
@@ -1,5 +1,4 @@
 from pypy.module.operator.tscmp import pypy_tscmp
-from rpython.rtyper.lltypesystem.rffi import scoped_nonmovingbuffer
 
 class TestTimingSafeCompare:
     def test_tscmp_neq(self):
diff --git a/pypy/module/operator/tscmp.py b/pypy/module/operator/tscmp.py
--- a/pypy/module/operator/tscmp.py
+++ b/pypy/module/operator/tscmp.py
@@ -3,9 +3,11 @@
 attacks for the hmac module.
 """
 import py
+
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from pypy.interpreter.error import OperationError
+
+from pypy.interpreter.error import OperationError, oefmt
 
 cwd = py.path.local(__file__).dirpath()
 eci = ExternalCompilationInfo(
@@ -13,15 +15,21 @@
     separate_module_files=[cwd.join('tscmp.c')],
     export_symbols=['pypy_tscmp'])
 
+
 def llexternal(*args, **kwargs):
     kwargs.setdefault('compilation_info', eci)
     kwargs.setdefault('sandboxsafe', True)
     return rffi.llexternal(*args, **kwargs)
 
-pypy_tscmp = llexternal('pypy_tscmp', [rffi.CCHARP, rffi.CCHARP, rffi.LONG, rffi.LONG], rffi.INT)
+
+pypy_tscmp = llexternal('pypy_tscmp',
+                        [rffi.CCHARP, rffi.CCHARP, rffi.LONG, rffi.LONG],
+                        rffi.INT)
+
 
 def compare_digest(space, w_a, w_b):
-    if space.isinstance_w(w_a, space.w_unicode) and space.isinstance_w(w_b, space.w_unicode):
+    if (space.isinstance_w(w_a, space.w_unicode) and
+        space.isinstance_w(w_b, space.w_unicode)):
         try:
             a_value = space.call_method(w_a, "encode", space.wrap("ascii"))
             b_value = space.call_method(w_b, "encode", space.wrap("ascii"))
@@ -29,14 +37,17 @@
         except OperationError as e:
             if not e.match(space, space.w_UnicodeEncodeError):
                 raise
-            raise OperationError(space.w_TypeError,
-                    space.wrap("comparing strings with non-ASCII characters is not supported"))
+            raise oefmt(space.w_TypeError,
+                        "comparing strings with non-ASCII characters is not "
+                        "supported")
     else:
         return compare_digest_buffer(space, w_a, w_b)
 
+
 def compare_digest_buffer(space, w_a, w_b):
     a = space.bufferstr_w(w_a)
     b = space.bufferstr_w(w_b)
     with rffi.scoped_nonmovingbuffer(a) as a_buffer:
         with rffi.scoped_nonmovingbuffer(b) as b_buffer:
-            return space.wrap(rffi.cast(lltype.Bool, pypy_tscmp(a_buffer, b_buffer, len(a), len(b))))
+            result = pypy_tscmp(a_buffer, b_buffer, len(a), len(b))
+            return space.wrap(rffi.cast(lltype.Bool, result))

From noreply at buildbot.pypy.org  Thu Sep 11 22:48:35 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Thu, 11 Sep 2014 22:48:35 +0200 (CEST)
Subject: [pypy-commit] pypy default: simplify
Message-ID: <20140911204835.1322C1C31C8@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey 
Branch: 
Changeset: r73477:9580ada38c4c
Date: 2014-07-27 11:36 -0700
http://bitbucket.org/pypy/pypy/changeset/9580ada38c4c/

Log:	simplify (grafted from 50402333035102e8dcce27b9d01a3f14a26f8c73)

diff --git a/pypy/module/operator/tscmp.py b/pypy/module/operator/tscmp.py
--- a/pypy/module/operator/tscmp.py
+++ b/pypy/module/operator/tscmp.py
@@ -31,17 +31,15 @@
     if (space.isinstance_w(w_a, space.w_unicode) and
         space.isinstance_w(w_b, space.w_unicode)):
         try:
-            a_value = space.call_method(w_a, "encode", space.wrap("ascii"))
-            b_value = space.call_method(w_b, "encode", space.wrap("ascii"))
-            return compare_digest_buffer(space, a_value, b_value)
+            w_a = space.call_method(w_a, 'encode', space.wrap('ascii'))
+            w_b = space.call_method(w_b, 'encode', space.wrap('ascii'))
         except OperationError as e:
             if not e.match(space, space.w_UnicodeEncodeError):
                 raise
             raise oefmt(space.w_TypeError,
                         "comparing strings with non-ASCII characters is not "
                         "supported")
-    else:
-        return compare_digest_buffer(space, w_a, w_b)
+    return compare_digest_buffer(space, w_a, w_b)
 
 
 def compare_digest_buffer(space, w_a, w_b):

From noreply at buildbot.pypy.org  Thu Sep 11 22:48:36 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Thu, 11 Sep 2014 22:48:36 +0200 (CEST)
Subject: [pypy-commit] pypy default: hopefully fix translation
 (tscmp.c:5:10: fatal error: 'tscmp.h' file not found)
Message-ID: <20140911204836.2F68D1C31C8@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey 
Branch: 
Changeset: r73478:37dd94f30100
Date: 2014-07-27 14:06 -0700
http://bitbucket.org/pypy/pypy/changeset/37dd94f30100/

Log:	hopefully fix translation (tscmp.c:5:10: fatal error: 'tscmp.h' file
	not found) (grafted from 871e353f33fc88e4047051410899940eccac5dd2)

diff --git a/pypy/module/operator/tscmp.py b/pypy/module/operator/tscmp.py
--- a/pypy/module/operator/tscmp.py
+++ b/pypy/module/operator/tscmp.py
@@ -12,6 +12,7 @@
 cwd = py.path.local(__file__).dirpath()
 eci = ExternalCompilationInfo(
     includes=[cwd.join('tscmp.h')],
+    include_dirs=[str(cwd)],
     separate_module_files=[cwd.join('tscmp.c')],
     export_symbols=['pypy_tscmp'])
 

From noreply at buildbot.pypy.org  Thu Sep 11 22:48:37 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Thu, 11 Sep 2014 22:48:37 +0200 (CEST)
Subject: [pypy-commit] pypy default: adapt the compare_digest impl from 3.3
	to 2.7
Message-ID: <20140911204837.617D51C31C8@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey 
Branch: 
Changeset: r73479:9831468e1882
Date: 2014-09-11 12:20 -0700
http://bitbucket.org/pypy/pypy/changeset/9831468e1882/

Log:	adapt the compare_digest impl from 3.3 to 2.7

diff --git a/pypy/module/operator/interp_operator.py b/pypy/module/operator/interp_operator.py
--- a/pypy/module/operator/interp_operator.py
+++ b/pypy/module/operator/interp_operator.py
@@ -1,6 +1,4 @@
-from rpython.rlib.objectmodel import specialize
-
-from pypy.interpreter.error import OperationError, oefmt
+from pypy.interpreter.error import OperationError
 from pypy.interpreter.gateway import unwrap_spec
 
 
@@ -249,33 +247,3 @@
 @unwrap_spec(default=int)
 def _length_hint(space, w_iterable, default):
     return space.wrap(space.length_hint(w_iterable, default))
-
-def compare_digest(space, w_a, w_b):
-    if (
-        space.isinstance_w(w_a, space.w_unicode) and
-        space.isinstance_w(w_b, space.w_unicode)
-    ):
-        return space.wrap(tscmp(space.unicode_w(w_a), space.unicode_w(w_b)))
-    if (
-        space.isinstance_w(w_a, space.w_unicode) or
-        space.isinstance_w(w_b, space.w_unicode)
-    ):
-        raise oefmt(
-            space.w_TypeError,
-            "unsupported operand types(s) or combination of types: '%N' and '%N'",
-            w_a,
-            w_b,
-        )
-    else:
-        return space.wrap(tscmp(space.bufferstr_w(w_a), space.bufferstr_w(w_b)))
-
-
- at specialize.argtype(0, 1)
-def tscmp(a, b):
-    len_a = len(a)
-    len_b = len(b)
-    length = min(len(a), len(b))
-    res = len_a ^ len_b
-    for i in xrange(length):
-        res |= ord(a[i]) ^ ord(b[i])
-    return res == 0
diff --git a/pypy/module/operator/test/test_operator.py b/pypy/module/operator/test/test_operator.py
--- a/pypy/module/operator/test/test_operator.py
+++ b/pypy/module/operator/test/test_operator.py
@@ -335,14 +335,8 @@
         a, b = mybytes(b"foobar"), mybytes(b"foobaz")
         assert not operator._compare_digest(a, b)
 
-    def test_compare_digest_buffer(self):
+    def test_compare_digest_unicode(self):
         import operator
-        assert operator._compare_digest(b'asd', b'asd')
-        assert not operator._compare_digest(b'asd', b'qwe')
-        assert not operator._compare_digest(b'asd', b'asdq')
-
-    def test_compare_digest_ascii(self):
-        import operator
-        assert operator._compare_digest('asd', 'asd')
-        assert not operator._compare_digest('asd', 'qwe')
-        assert not operator._compare_digest('asd', 'asdq')
+        assert operator._compare_digest(u'asd', u'asd')
+        assert not operator._compare_digest(u'asd', u'qwe')
+        raises(TypeError, operator._compare_digest, u'asd', b'qwe')
diff --git a/pypy/module/operator/test/test_tscmp.py b/pypy/module/operator/test/test_tscmp.py
--- a/pypy/module/operator/test/test_tscmp.py
+++ b/pypy/module/operator/test/test_tscmp.py
@@ -1,14 +1,28 @@
-from pypy.module.operator.tscmp import pypy_tscmp
+from pypy.module.operator.tscmp import pypy_tscmp, pypy_tscmp_wide
 
 class TestTimingSafeCompare:
+    tostr = str
+    tscmp = staticmethod(pypy_tscmp)
+
     def test_tscmp_neq(self):
-        assert not pypy_tscmp('asd', 'qwe', 3, 3)
+        assert not self.tscmp(self.tostr('asd'), self.tostr('qwe'), 3, 3)
 
     def test_tscmp_eq(self):
-        assert pypy_tscmp('asd', 'asd', 3, 3)
+        assert self.tscmp(self.tostr('asd'), self.tostr('asd'), 3, 3)
 
     def test_tscmp_len(self):
-        assert pypy_tscmp('asdp', 'asdq', 3, 3)
+        assert self.tscmp(self.tostr('asdp'), self.tostr('asdq'), 3, 3)
 
     def test_tscmp_nlen(self):
-        assert not pypy_tscmp('asd', 'asd', 2, 3)
+        assert not self.tscmp(self.tostr('asd'), self.tostr('asd'), 2, 3)
+
+
+class TestTimingSafeCompareWide(TestTimingSafeCompare):
+    tostr = unicode
+    tscmp = staticmethod(pypy_tscmp_wide)
+
+    def test_tscmp_wide_nonascii(self):
+        a, b = u"\ud808\udf45", u"\ud808\udf45"
+        assert self.tscmp(a, b, len(a), len(b))
+        a, b = u"\ud808\udf45", u"\ud808\udf45 "
+        assert not self.tscmp(a, b, len(a), len(b))
diff --git a/pypy/module/operator/tscmp.c b/pypy/module/operator/tscmp.c
--- a/pypy/module/operator/tscmp.c
+++ b/pypy/module/operator/tscmp.c
@@ -1,21 +1,22 @@
-/* From CPython 3.3.5's operator.c
+/* Derived from CPython 3.3.5's operator.c::_tscmp
  */
 
 #include 
+#include 
 #include "tscmp.h"
 
 int
-pypy_tscmp(const unsigned char *a, const unsigned char *b, long len_a, long len_b)
+pypy_tscmp(const char *a, const char *b, long len_a, long len_b)
 {
     /* The volatile type declarations make sure that the compiler has no
      * chance to optimize and fold the code in any way that may change
      * the timing.
      */
     volatile long length;
-    volatile const unsigned char *left;
-    volatile const unsigned char *right;
+    volatile const char *left;
+    volatile const char *right;
     long i;
-    unsigned char result;
+    char result;
 
     /* loop count depends on length of b */
     length = len_b;
@@ -26,7 +27,7 @@
      * volatile forces re-evaluation
      *  */
     if (len_a == length) {
-        left = *((volatile const unsigned char**)&a);
+        left = *((volatile const char**)&a);
         result = 0;
     }
     if (len_a != length) {
@@ -40,3 +41,40 @@
 
     return (result == 0);
 }
+
+int
+pypy_tscmp_wide(const wchar_t *a, const wchar_t *b, long len_a, long len_b)
+{
+    /* The volatile type declarations make sure that the compiler has no
+     * chance to optimize and fold the code in any way that may change
+     * the timing.
+     */
+    volatile long length;
+    volatile const wchar_t *left;
+    volatile const wchar_t *right;
+    long i;
+    wchar_t result;
+
+    /* loop count depends on length of b */
+    length = len_b;
+    left = NULL;
+    right = b;
+
+    /* don't use else here to keep the amount of CPU instructions constant,
+     * volatile forces re-evaluation
+     *  */
+    if (len_a == length) {
+        left = *((volatile const wchar_t**)&a);
+        result = 0;
+    }
+    if (len_a != length) {
+        left = b;
+        result = 1;
+    }
+
+    for (i=0; i < length; i++) {
+        result |= *left++ ^ *right++;
+    }
+
+    return (result == 0);
+}
diff --git a/pypy/module/operator/tscmp.h b/pypy/module/operator/tscmp.h
--- a/pypy/module/operator/tscmp.h
+++ b/pypy/module/operator/tscmp.h
@@ -1,1 +1,2 @@
-int pypy_tscmp(const unsigned char *a, const unsigned char *b, long len_a, long len_b);
+int pypy_tscmp(const char *, const char *, long, long);
+int pypy_tscmp_wide(const wchar_t *, const wchar_t *, long, long);
diff --git a/pypy/module/operator/tscmp.py b/pypy/module/operator/tscmp.py
--- a/pypy/module/operator/tscmp.py
+++ b/pypy/module/operator/tscmp.py
@@ -7,14 +7,14 @@
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 
-from pypy.interpreter.error import OperationError, oefmt
+from pypy.interpreter.error import oefmt
 
 cwd = py.path.local(__file__).dirpath()
 eci = ExternalCompilationInfo(
     includes=[cwd.join('tscmp.h')],
     include_dirs=[str(cwd)],
     separate_module_files=[cwd.join('tscmp.c')],
-    export_symbols=['pypy_tscmp'])
+    export_symbols=['pypy_tscmp', 'pypy_tscmp_wide'])
 
 
 def llexternal(*args, **kwargs):
@@ -23,30 +23,51 @@
     return rffi.llexternal(*args, **kwargs)
 
 
-pypy_tscmp = llexternal('pypy_tscmp',
-                        [rffi.CCHARP, rffi.CCHARP, rffi.LONG, rffi.LONG],
-                        rffi.INT)
+pypy_tscmp = llexternal(
+    'pypy_tscmp',
+    [rffi.CCHARP, rffi.CCHARP, rffi.LONG, rffi.LONG],
+    rffi.INT)
+pypy_tscmp_wide = llexternal(
+    'pypy_tscmp_wide',
+    [rffi.CWCHARP, rffi.CWCHARP, rffi.LONG, rffi.LONG],
+    rffi.INT)
 
 
 def compare_digest(space, w_a, w_b):
+    """compare_digest(a, b) -> bool
+
+    Return 'a == b'.  This function uses an approach designed to prevent
+    timing analysis, making it appropriate for cryptography.  a and b
+    must both be of the same type: either str (ASCII only), or any type
+    that supports the buffer protocol (e.g. bytes).
+
+    Note: If a and b are of different lengths, or if an error occurs, a
+    timing attack could theoretically reveal information about the types
+    and lengths of a and b--but not their values.
+    """
     if (space.isinstance_w(w_a, space.w_unicode) and
         space.isinstance_w(w_b, space.w_unicode)):
-        try:
-            w_a = space.call_method(w_a, 'encode', space.wrap('ascii'))
-            w_b = space.call_method(w_b, 'encode', space.wrap('ascii'))
-        except OperationError as e:
-            if not e.match(space, space.w_UnicodeEncodeError):
-                raise
-            raise oefmt(space.w_TypeError,
-                        "comparing strings with non-ASCII characters is not "
-                        "supported")
+        a = space.unicode_w(w_a)
+        b = space.unicode_w(w_b)
+        with rffi.scoped_nonmoving_unicodebuffer(a) as a_buf:
+            with rffi.scoped_nonmoving_unicodebuffer(b) as b_buf:
+                result = pypy_tscmp_wide(a_buf, b_buf, len(a), len(b))
+        return space.wrap(rffi.cast(lltype.Bool, result))
     return compare_digest_buffer(space, w_a, w_b)
 
 
 def compare_digest_buffer(space, w_a, w_b):
-    a = space.bufferstr_w(w_a)
-    b = space.bufferstr_w(w_b)
-    with rffi.scoped_nonmovingbuffer(a) as a_buffer:
-        with rffi.scoped_nonmovingbuffer(b) as b_buffer:
-            result = pypy_tscmp(a_buffer, b_buffer, len(a), len(b))
-            return space.wrap(rffi.cast(lltype.Bool, result))
+    try:
+        a_buf = w_a.buffer_w(space, space.BUF_SIMPLE)
+        b_buf = w_b.buffer_w(space, space.BUF_SIMPLE)
+    except TypeError:
+        raise oefmt(space.w_TypeError,
+                    "unsupported operand types(s) or combination of types: "
+                    "'%T' and '%T'", w_a, w_b)
+
+    a = a_buf.as_str()
+    b = b_buf.as_str()
+    with rffi.scoped_nonmovingbuffer(a) as a_buf:
+        with rffi.scoped_nonmovingbuffer(b) as b_buf:
+            result = pypy_tscmp(a_buf, b_buf, len(a), len(b))
+    return space.wrap(rffi.cast(lltype.Bool, result))

From noreply at buildbot.pypy.org  Thu Sep 11 22:48:39 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Thu, 11 Sep 2014 22:48:39 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge upstream
Message-ID: <20140911204839.F0F641C31C8@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey 
Branch: 
Changeset: r73480:a552e9ed86d6
Date: 2014-09-11 12:43 -0700
http://bitbucket.org/pypy/pypy/changeset/a552e9ed86d6/

Log:	merge upstream

diff too long, truncating to 2000 out of 2511 lines

diff --git a/_pytest/resultlog.py b/_pytest/resultlog.py
--- a/_pytest/resultlog.py
+++ b/_pytest/resultlog.py
@@ -54,15 +54,16 @@
         self.logfile = logfile # preferably line buffered
 
     def write_log_entry(self, testpath, lettercode, longrepr, sections=None):
-        py.builtin.print_("%s %s" % (lettercode, testpath), file=self.logfile)
+        _safeprint("%s %s" % (lettercode, testpath), file=self.logfile)
         for line in longrepr.splitlines():
-            py.builtin.print_(" %s" % line, file=self.logfile)
-        if sections is not None:
+            _safeprint(" %s" % line, file=self.logfile)
+        if sections is not None and (
+                lettercode in ('E', 'F')):    # to limit the size of logs
             for title, content in sections:
-                py.builtin.print_(" ---------- %s ----------" % (title,),
-                                  file=self.logfile)
+                _safeprint(" ---------- %s ----------" % (title,),
+                           file=self.logfile)
                 for line in content.splitlines():
-                    py.builtin.print_(" %s" % line, file=self.logfile)
+                    _safeprint(" %s" % line, file=self.logfile)
 
     def log_outcome(self, report, lettercode, longrepr):
         testpath = getattr(report, 'nodeid', None)
@@ -105,3 +106,8 @@
         if path is None:
             path = "cwd:%s" % py.path.local()
         self.write_log_entry(path, '!', str(excrepr))
+
+def _safeprint(s, file):
+    if isinstance(s, unicode):
+        s = s.encode('utf-8')
+    py.builtin.print_(s, file=file)
diff --git a/lib-python/2.7/test/test_mmap.py b/lib-python/2.7/test/test_mmap.py
--- a/lib-python/2.7/test/test_mmap.py
+++ b/lib-python/2.7/test/test_mmap.py
@@ -179,25 +179,27 @@
         import sys
         f = open(TESTFN, "r+b")
         try:
-            m = mmap.mmap(f.fileno(), mapsize+1)
-        except ValueError:
-            # we do not expect a ValueError on Windows
-            # CAUTION:  This also changes the size of the file on disk, and
-            # later tests assume that the length hasn't changed.  We need to
-            # repair that.
+            try:
+                m = mmap.mmap(f.fileno(), mapsize+1)
+            except ValueError:
+                # we do not expect a ValueError on Windows
+                # CAUTION:  This also changes the size of the file on disk, and
+                # later tests assume that the length hasn't changed.  We need to
+                # repair that.
+                if sys.platform.startswith('win'):
+                    self.fail("Opening mmap with size+1 should work on Windows.")
+            else:
+                # we expect a ValueError on Unix, but not on Windows
+                if not sys.platform.startswith('win'):
+                    self.fail("Opening mmap with size+1 should raise ValueError.")
+                m.close()
+        finally:
+            f.close()
             if sys.platform.startswith('win'):
-                self.fail("Opening mmap with size+1 should work on Windows.")
-        else:
-            # we expect a ValueError on Unix, but not on Windows
-            if not sys.platform.startswith('win'):
-                self.fail("Opening mmap with size+1 should raise ValueError.")
-            m.close()
-        f.close()
-        if sys.platform.startswith('win'):
-            # Repair damage from the resizing test.
-            f = open(TESTFN, 'r+b')
-            f.truncate(mapsize)
-            f.close()
+                # Repair damage from the resizing test.
+                f = open(TESTFN, 'r+b')
+                f.truncate(mapsize)
+                f.close()
 
         # Opening mmap with access=ACCESS_WRITE
         f = open(TESTFN, "r+b")
diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -65,9 +65,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '2.3'
+version = '2.4'
 # The full version, including alpha/beta/rc tags.
-release = '2.3.0'
+release = '2.4.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/pypy/doc/getting-started-python.rst b/pypy/doc/getting-started-python.rst
--- a/pypy/doc/getting-started-python.rst
+++ b/pypy/doc/getting-started-python.rst
@@ -111,6 +111,10 @@
    of your choice.  Typical example: ``--opt=2`` gives a good (but of
    course slower) Python interpreter without the JIT.
 
+   Consider using PyPy instead of CPython in the above command line,
+   as it is much faster.  (Note that ``rpython`` is a Python 2 program,
+   not Python 3; you need to run either PyPy 2 or CPython 2.)
+
 .. _`optimization level`: config/opt.html
 
 If everything works correctly this will create an executable
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-2.4.0.rst
    release-2.3.1.rst
    release-2.3.0.rst
    release-2.2.1.rst
diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst
--- a/pypy/doc/index.rst
+++ b/pypy/doc/index.rst
@@ -40,7 +40,7 @@
 
 * `FAQ`_: some frequently asked questions.
 
-* `Release 2.3.1`_: the latest official release
+* `Release 2.4.0`_: the latest official release
 
 * `PyPy Blog`_: news and status info about PyPy 
 
@@ -110,7 +110,7 @@
 .. _`Getting Started`: getting-started.html
 .. _`Papers`: extradoc.html
 .. _`Videos`: video-index.html
-.. _`Release 2.3.1`: http://pypy.org/download.html
+.. _`Release 2.4.0`: http://pypy.org/download.html
 .. _`speed.pypy.org`: http://speed.pypy.org
 .. _`RPython toolchain`: translation.html
 .. _`potential project ideas`: project-ideas.html
diff --git a/pypy/doc/release-2.4.0.rst b/pypy/doc/release-2.4.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-2.4.0.rst
@@ -0,0 +1,119 @@
+=================================================
+PyPy 2.4 - Snow White
+=================================================
+
+We're pleased to announce PyPy 2.4, which contains significant performance
+enhancements and bug fixes. 
+
+You can already download the PyPy 2.4-beta1 pre-release here:
+
+    http://pypy.org/download.html
+
+We would like to thank our donors for the continued support of the PyPy
+project, and for those who donate to our three sub-projects.
+We've shown quite a bit of progress, but we're slowly running out of funds.
+Please consider donating more, or even better convince your employer to donate,
+so we can finish those projects! We would like to also point out that in
+September, `the Python Software Foundation`_ will `match funds`_ for
+any donations up to $10k!  The three sub-projects are:
+
+* `Py3k`_ (supporting Python 3.x): We have released a Python 3.2.5 compatible version
+   we call PyPy3 2.3.1, and are working toward a Python 3.3 compatible version
+
+* `STM`_ (software transactional memory): We have released a first working version,
+  and continue to try out new promising paths of achieving a fast multithreaded Python
+
+* `NumPy`_ which requires installation of our fork of upstream numpy, 
+  available `on bitbucket`_
+
+.. _`Py3k`: http://pypy.org/py3donate.html
+.. _`STM`: http://pypy.org/tmdonate2.html
+.. _`NumPy`: http://pypy.org/numpydonate.html
+.. _`on bitbucket`: https://www.bitbucket.org/pypy/numpy   
+.. _`the Python Software Foundation`: https://www.python.org/psf/
+.. _`match funds`: http://morepypy.blogspot.com/2014/09/python-software-foundation-matching.html
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`pypy 2.4 and cpython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+This release supports **x86** machines on most common operating systems 
+(Linux 32/64, Mac OS X 64, Windows, and OpenBSD),
+as well as newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux. 
+
+While we support 32 bit python on Windows, work on the native Windows 64
+bit python is still stalling, we would welcome a volunteer
+to `handle that`_.
+
+.. _`pypy 2.4 and cpython 2.7.x`: http://speed.pypy.org
+.. _`handle that`: http://doc.pypy.org/en/latest/windows.html#what-is-missing-for-a-full-64-bit-translation
+
+Highlights
+==========
+
+Benchmarks improved after internal enhancements in string and
+bytearray handling, and a major rewrite of the GIL handling. This means
+that external calls are now a lot faster, especially the CFFI ones. It also
+means better performance in a lot of corner cases with handling strings or
+bytearrays. The main bugfix is handling of many socket objects in your
+program which in the long run used to "leak" memory.
+
+PyPy now uses Python 2.7.8 standard library.
+
+We welcomed more than 12 new contributors, and conducted two Google
+Summer of Code projects, as well as other student projects not
+directly related to Summer of Code.
+
+
+Issues reported with our previous release were fixed after reports from users on
+our new issue tracker at https://bitbucket.org/pypy/pypy/issues or on IRC at
+#pypy. Here is a summary of the user-facing changes;
+for more information see `whats-new`_:
+
+* Reduced internal copying of bytearray operations
+
+* Tweak the internal structure of StringBuilder to speed up large string
+  handling, which becomes advantageous on large programs at the cost of slightly
+  slower small *benchmark* type programs.
+
+* Boost performance of thread-local variables in both unjitted and jitted code,
+  this mostly affects errno handling on linux, which makes external calls
+  faster.
+
+* Move to a mixed polling and mutex GIL model that make mutlithreaded jitted
+  code run *much* faster
+
+* Optimize errno handling in linux (x86 and x86-64 only)
+
+* Remove ctypes pythonapi and ctypes.PyDLL, which never worked on PyPy
+
+* Fix performance regression on ufunc(, ) in numpy
+
+* Classes in the ast module are now distinct from structures used by
+  the compiler, which simplifies and speeds up translation of our
+  source code to the PyPy binary interpreter
+
+* Upgrade stdlib from 2.7.5 to 2.7.8
+
+* Win32 now links statically to zlib, expat, bzip, and openssl-1.0.1i.
+  No more missing DLLs
+  
+* Many issues were resolved_ since the 2.3.1 release on June 8
+
+.. _`whats-new`: http://doc.pypy.org/en/latest/whatsnew-2.3.1.html
+.. _resolved: https://bitbucket.org/pypy/pypy/issues?status=resolved
+
+We have further improvements on the way: rpython file handling,
+numpy linalg compatibility, as well
+as improved GC and many smaller improvements.
+
+Please try it out and let us know what you think. We especially welcome
+success stories, we know you are using PyPy, please tell us about it!
+
+Cheers
+
+The PyPy Team
+
diff --git a/pypy/doc/release-2.4.rst b/pypy/doc/release-2.4.rst
deleted file mode 100644
--- a/pypy/doc/release-2.4.rst
+++ /dev/null
@@ -1,107 +0,0 @@
-=================================================
-PyPy 2.4 - ????????
-=================================================
-
-We're pleased to announce PyPy 2.4, a significant milestone on it's own right
-and the proud parent of our recent PyPy3 and STM releases.
-
-This release contains several improvements and bugfixes.
-
-You can download the PyPy 2.4 release here:
-
-    http://pypy.org/download.html
-
-We would like to thank our donors for the continued support of the PyPy
-project, and for those who donate to our three sub-projects.
-We've shown quite a bit of progress 
-but we're slowly running out of funds.
-Please consider donating more, or even better convince your employer to donate,
-so we can finish those projects!  The three sub-projects are:
-
-* `Py3k`_ (supporting Python 3.x): We have released a Python 3.2.5 compatable version
-   we call PyPy3 2.3.1, and are working toward a Python 3.3 compatable version
-
-* `STM`_ (software transactional memory): We have release a first working version, and
-continue to try out new promising paths of acheiving a fast multithreaded python
-
-* `NumPy`_ which requires installation of our fork of upstream numpy, available `on bitbucket`_
-
-.. _`Py3k`: http://pypy.org/py3donate.html
-.. _`STM`: http://pypy.org/tmdonate2.html
-.. _`NumPy`: http://pypy.org/numpydonate.html
-.. _`on bitbucket`: https://www.bitbucket.org/pypy/numpy   
-
-What is PyPy?
-=============
-
-PyPy is a very compliant Python interpreter, almost a drop-in replacement for
-CPython 2.7. It's fast (`pypy 2.3 and cpython 2.7.x`_ performance comparison;
-note that cpython's speed has not changed since 2.7.2)
-due to its integrated tracing JIT compiler.
-
-This release supports x86 machines running Linux 32/64, Mac OS X 64, Windows,
-and OpenBSD,
-as well as newer ARM hardware (ARMv6 or ARMv7, with VFPv3) running Linux. 
-
-While we support 32 bit python on Windows, work on the native Windows 64
-bit python is still stalling, we would welcome a volunteer
-to `handle that`_.
-
-.. _`pypy 2.3 and cpython 2.7.x`: http://speed.pypy.org
-.. _`handle that`: http://doc.pypy.org/en/latest/windows.html#what-is-missing-for-a-full-64-bit-translation
-
-Highlights
-==========
-
-Benchmarks improved after internal improvements in string and bytearray handling,
-and a major rewrite of the GIL handling. Many of these improvements are offshoots
-of the STM work.
-
-We merged in Python's 2.7.8 stdlib in a record time of one week, proving the
-maturity of our underlying RPython code base and PyPy interpreter.
-
-We welcomed more than 12 new contributors, and conducted two Google Summer of Code
-projects XXX details?
-
-Issues reported with our previous release were fixed after reports from users on
-our new issue tracker at https://bitbucket.org/pypy/pypy/issues or on IRC at
-#pypy. Here is a summary of the user-facing changes;
-for more information see `whats-new`_:
-
-* Reduced internal copying of bytearray operations
-
-* Tweak the internal structure of StringBuilder to speed up large string
-handling, which becomes advantageous on large programs at the cost of slightly
-slower small *benchmark* type programs.
-
-* Boost performance of thread-local variables in both unjitted and jitted code
-
-* Move to a mixed polling and mutex GIL model that make mutli-threaded jitted
-  code run *much* faster
-
-* Optimize errno handling in linux
-
-* Remove ctypes pythonapi and ctypes.PyDLL, which never worked on PyPy
-
-* Fix performance regression on ufunc(, ) in numpy
-
-* Classes in the ast module are now distinct from structures used by the compiler,
-  which simplifies and speeds up translation of our source code to the PyPy binary
-  interpreter
-
-* Upgrade stdlib from 2.7.5 to 2.7.8
-
-* 
-
-* Many issues were resolved_ since the 2.3.1 release on June 8
-
-.. _`whats-new`: http://doc.pypy.org/en/latest/whatsnew-2.3.1.html
-.. _resolved: https://bitbucket.org/pypy/pypy/issues?status=resolved
-
-Please try it out and let us know what you think. We especially welcome
-success stories, we know you are using PyPy, please tell us about it!
-
-Cheers
-
-The PyPy Team
-
diff --git a/pypy/doc/whatsnew-2.4.0.rst b/pypy/doc/whatsnew-2.4.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-2.4.0.rst
@@ -0,0 +1,66 @@
+=======================
+What's new in PyPy 2.4+
+=======================
+
+.. this is a revision shortly after release-2.3.x
+.. startrev: ca9b7cf02cf4
+
+.. branch: fix-bytearray-complexity
+Bytearray operations no longer copy the bytearray unnecessarily
+
+Added support for ``__getitem__``, ``__setitem__``, ``__getslice__``,
+``__setslice__``,  and ``__len__`` to RPython
+
+.. branch: stringbuilder2-perf
+Give the StringBuilder a more flexible internal structure, with a
+chained list of strings instead of just one string. This make it
+more efficient when building large strings, e.g. with cStringIO().
+
+Also, use systematically jit.conditional_call() instead of regular
+branches. This lets the JIT make more linear code, at the cost of
+forcing a bit more data (to be passed as arguments to
+conditional_calls). I would expect the net result to be a slight
+slow-down on some simple benchmarks and a speed-up on bigger
+programs.
+
+.. branch: ec-threadlocal
+Change the executioncontext's lookup to be done by reading a thread-
+local variable (which is implemented in C using '__thread' if
+possible, and pthread_getspecific() otherwise). On Linux x86 and
+x86-64, the JIT backend has a special optimization that lets it emit
+directly a single MOV from a %gs- or %fs-based address. It seems
+actually to give a good boost in performance.
+
+.. branch: fast-gil
+A faster way to handle the GIL, particularly in JIT code. The GIL is
+now a composite of two concepts: a global number (it's just set from
+1 to 0 and back around CALL_RELEASE_GIL), and a real mutex. If there
+are threads waiting to acquire the GIL, one of them is actively
+checking the global number every 0.1 ms to 1 ms.  Overall, JIT loops
+full of external function calls now run a bit faster (if no thread was
+started yet), or a *lot* faster (if threads were started already).
+
+.. branch: jit-get-errno
+Optimize the errno handling in the JIT, notably around external
+function calls. Linux-only.
+
+.. branch: disable_pythonapi
+Remove non-functioning ctypes.pyhonapi and ctypes.PyDLL, document this
+incompatibility with cpython. Recast sys.dllhandle to an int.
+
+.. branch: scalar-operations
+Fix performance regression on ufunc(, ) in numpy.
+
+.. branch: pytest-25
+Update our copies of py.test and pylib to versions 2.5.2 and 1.4.20, 
+respectively.
+
+.. branch: split-ast-classes
+Classes in the ast module are now distinct from structures used by the compiler.
+
+.. branch: stdlib-2.7.8
+Upgrades from 2.7.6 to 2.7.8
+
+.. branch: cpybug-seq-radd-rmul
+Fix issue #1861 - cpython compatability madness
+
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -1,62 +1,8 @@
+
 =======================
-What's new in PyPy 2.4+
+What's new in PyPy 2.5+
 =======================
 
-.. this is a revision shortly after release-2.3.x
-.. startrev: ca9b7cf02cf4
+.. this is a revision shortly after release-2.4.x
+.. startrev: 7026746cbb1b
 
-.. branch: fix-bytearray-complexity
-Bytearray operations no longer copy the bytearray unnecessarily
-
-Added support for ``__getitem__``, ``__setitem__``, ``__getslice__``,
-``__setslice__``,  and ``__len__`` to RPython
-
-.. branch: stringbuilder2-perf
-Give the StringBuilder a more flexible internal structure, with a
-chained list of strings instead of just one string. This make it
-more efficient when building large strings, e.g. with cStringIO().
-
-Also, use systematically jit.conditional_call() instead of regular
-branches. This lets the JIT make more linear code, at the cost of
-forcing a bit more data (to be passed as arguments to
-conditional_calls). I would expect the net result to be a slight
-slow-down on some simple benchmarks and a speed-up on bigger
-programs.
-
-.. branch: ec-threadlocal
-Change the executioncontext's lookup to be done by reading a thread-
-local variable (which is implemented in C using '__thread' if
-possible, and pthread_getspecific() otherwise). On Linux x86 and
-x86-64, the JIT backend has a special optimization that lets it emit
-directly a single MOV from a %gs- or %fs-based address. It seems
-actually to give a good boost in performance.
-
-.. branch: fast-gil
-A faster way to handle the GIL, particularly in JIT code. The GIL is
-now a composite of two concepts: a global number (it's just set from
-1 to 0 and back around CALL_RELEASE_GIL), and a real mutex. If there
-are threads waiting to acquire the GIL, one of them is actively
-checking the global number every 0.1 ms to 1 ms.  Overall, JIT loops
-full of external function calls now run a bit faster (if no thread was
-started yet), or a *lot* faster (if threads were started already).
-
-.. branch: jit-get-errno
-Optimize the errno handling in the JIT, notably around external
-function calls. Linux-only.
-
-.. branch: disable_pythonapi
-Remove non-functioning ctypes.pyhonapi and ctypes.PyDLL, document this
-incompatibility with cpython. Recast sys.dllhandle to an int.
-
-.. branch: scalar-operations
-Fix performance regression on ufunc(, ) in numpy.
-
-.. branch: pytest-25
-Update our copies of py.test and pylib to versions 2.5.2 and 1.4.20, 
-respectively.
-
-.. branch: split-ast-classes
-Classes in the ast module are now distinct from structures used by the compiler.
-
-.. branch: stdlib-2.7.8
-Upgrades from 2.7.6 to 2.7.8
diff --git a/pypy/interpreter/module.py b/pypy/interpreter/module.py
--- a/pypy/interpreter/module.py
+++ b/pypy/interpreter/module.py
@@ -29,6 +29,17 @@
                           space.w_None)
         self.startup_called = False
 
+    def _cleanup_(self):
+        """Called by the annotator on prebuilt Module instances.
+        We don't have many such modules, but for the ones that
+        show up, remove their __file__ rather than translate it
+        statically inside the executable."""
+        try:
+            space = self.space
+            space.delitem(self.w_dict, space.wrap('__file__'))
+        except OperationError:
+            pass
+
     def install(self):
         """NOT_RPYTHON: installs this module into space.builtin_modules"""
         w_mod = self.space.wrap(self)
diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -985,6 +985,11 @@
             assert sys.path == old_sys_path + [self.goal_dir]
 
             app_main.setup_bootstrap_path(self.fake_exe)
+            assert sys.executable == ''      # not executable!
+            assert sys.path == old_sys_path + [self.goal_dir]
+
+            os.chmod(self.fake_exe, 0755)
+            app_main.setup_bootstrap_path(self.fake_exe)
             assert sys.executable == self.fake_exe
             assert self.goal_dir not in sys.path
 
diff --git a/pypy/interpreter/test/test_module.py b/pypy/interpreter/test/test_module.py
--- a/pypy/interpreter/test/test_module.py
+++ b/pypy/interpreter/test/test_module.py
@@ -1,4 +1,5 @@
-
+import py
+from pypy.interpreter.error import OperationError
 from pypy.interpreter.module import Module
 
 class TestModule: 
@@ -17,6 +18,18 @@
         space.raises_w(space.w_AttributeError,
                        space.delattr, w_m, w('x'))
 
+    def test___file__(self, space):
+        w = space.wrap
+        m = Module(space, space.wrap('m'))
+        py.test.raises(OperationError, space.getattr, w(m), w('__file__'))
+        m._cleanup_()
+        py.test.raises(OperationError, space.getattr, w(m), w('__file__'))
+        space.setattr(w(m), w('__file__'), w('m.py'))
+        space.getattr(w(m), w('__file__'))   # does not raise
+        m._cleanup_()
+        py.test.raises(OperationError, space.getattr, w(m), w('__file__'))
+
+
 class AppTest_ModuleObject: 
     def test_attr(self):
         m = __import__('__builtin__')
diff --git a/pypy/module/_io/interp_stringio.py b/pypy/module/_io/interp_stringio.py
--- a/pypy/module/_io/interp_stringio.py
+++ b/pypy/module/_io/interp_stringio.py
@@ -86,7 +86,7 @@
         initval = space.unicode_w(w_initval)
         size = len(initval)
         self.resize_buffer(size)
-        self.buf = [c for c in initval]
+        self.buf = list(initval)
         pos = space.getindex_w(w_pos, space.w_TypeError)
         if pos < 0:
             raise OperationError(space.w_ValueError,
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -759,17 +759,25 @@
 
         # socket's timeout is in seconds, poll's timeout in ms
         timeout = int(sock_timeout * 1000 + 0.5)
-        ready = rpoll.poll(fddict, timeout)
+        try:
+            ready = rpoll.poll(fddict, timeout)
+        except rpoll.PollError, e:
+            message = e.get_msg()
+            raise ssl_error(space, message, e.errno)
     else:
         if MAX_FD_SIZE is not None and sock_fd >= MAX_FD_SIZE:
             return SOCKET_TOO_LARGE_FOR_SELECT
 
-        if writing:
-            r, w, e = rpoll.select([], [sock_fd], [], sock_timeout)
-            ready = w
-        else:
-            r, w, e = rpoll.select([sock_fd], [], [], sock_timeout)
-            ready = r
+        try:
+            if writing:
+                r, w, e = rpoll.select([], [sock_fd], [], sock_timeout)
+                ready = w
+            else:
+                r, w, e = rpoll.select([sock_fd], [], [], sock_timeout)
+                ready = r
+        except SelectError, e:
+            message = e.get_msg()
+            raise ssl_error(space, message, e.errno)
     if ready:
         return SOCKET_OPERATION_OK
     else:
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,7 +29,7 @@
 #define PY_VERSION		"2.7.8"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "2.4.0-alpha0"
+#define PYPY_VERSION "2.5.0-alpha0"
 
 /* Subversion Revision number of this file (not of the repository).
  * Empty since Mercurial migration. */
diff --git a/pypy/module/pypyjit/test_pypy_c/test_cprofile.py b/pypy/module/pypyjit/test_pypy_c/test_cprofile.py
--- a/pypy/module/pypyjit/test_pypy_c/test_cprofile.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_cprofile.py
@@ -1,4 +1,4 @@
-import py, sys
+import py, sys, re
 from pypy.module.pypyjit.test_pypy_c.test_00_model import BaseTestPyPyC
 
 class TestCProfile(BaseTestPyPyC):
@@ -28,8 +28,18 @@
             print loop.ops_by_id(method)
             # on 32-bit, there is f1=call(read_timestamp); ...;
             # f2=call(read_timestamp); f3=call(llong_sub,f1,f2)
-            # but all calls can be special-cased by the backend if supported
-            if sys.maxint != 2147483647:
-                assert ' call(' not in repr(loop.ops_by_id(method))
+            # but all calls can be special-cased by the backend if
+            # supported.  On 64-bit there is only the two calls to
+            # read_timestamp.
+            r = re.compile(r" call[(]ConstClass[(](.+?)[)]")
+            calls = r.findall(repr(loop.ops_by_id(method)))
+            if sys.maxint == 2147483647:
+                assert len(calls) == 6
+            else:
+                assert len(calls) == 2
+            for x in calls:
+                assert ('ll_read_timestamp' in x or 'llong_sub' in x
+                        or 'llong_add' in x)
+            #
             assert ' call_may_force(' not in repr(loop.ops_by_id(method))
             assert ' cond_call(' in repr(loop.ops_by_id(method))
diff --git a/pypy/module/sys/state.py b/pypy/module/sys/state.py
--- a/pypy/module/sys/state.py
+++ b/pypy/module/sys/state.py
@@ -7,15 +7,15 @@
 # ____________________________________________________________
 #
 
-class State: 
-    def __init__(self, space): 
-        self.space = space 
+class State:
+    def __init__(self, space):
+        self.space = space
 
         self.w_modules = space.newdict(module=True)
-
         self.w_warnoptions = space.newlist([])
         self.w_argv = space.newlist([])
-        self.setinitialpath(space) 
+
+        self.setinitialpath(space)
 
     def setinitialpath(self, space):
         from pypy.module.sys.initpath import compute_stdlib_path
@@ -25,10 +25,10 @@
         path = compute_stdlib_path(self, srcdir)
         self.w_path = space.newlist([space.wrap(p) for p in path])
 
-
 def get(space):
     return space.fromcache(State)
 
+
 class IOState:
     def __init__(self, space):
         from pypy.module._file.interp_file import W_File
@@ -36,17 +36,17 @@
 
         stdin = W_File(space)
         stdin.file_fdopen(0, "r", 1)
-        stdin.name = ''
+        stdin.w_name = space.wrap('')
         self.w_stdin = space.wrap(stdin)
 
         stdout = W_File(space)
         stdout.file_fdopen(1, "w", 1)
-        stdout.name = ''
+        stdout.w_name = space.wrap('')
         self.w_stdout = space.wrap(stdout)
 
         stderr = W_File(space)
         stderr.file_fdopen(2, "w", 0)
-        stderr.name = ''
+        stderr.w_name = space.wrap('')
         self.w_stderr = space.wrap(stderr)
 
         stdin._when_reading_first_flush(stdout)
@@ -54,9 +54,9 @@
 def getio(space):
     return space.fromcache(IOState)
 
+
 def pypy_getudir(space):
     """NOT_RPYTHON
     (should be removed from interpleveldefs before translation)"""
     from rpython.tool.udir import udir
     return space.wrap(str(udir))
-
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -91,6 +91,10 @@
         assert isinstance(sys.__stderr__, file)
         assert isinstance(sys.__stdin__, file)
 
+        assert sys.__stdin__.name == ""
+        assert sys.__stdout__.name == ""
+        assert sys.__stderr__.name == ""
+
         if self.appdirect and not isinstance(sys.stdin, file):
             return
 
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (2, 4, 0, "alpha", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (2, 5, 0, "alpha", 0)    #XXX # sync patchlevel.h
 
 if platform.name == 'msvc':
     COMPILER_INFO = 'MSC v.%d 32 bit' % (platform.version * 10 + 600)
diff --git a/pypy/module/thread/test/test_thread.py b/pypy/module/thread/test/test_thread.py
--- a/pypy/module/thread/test/test_thread.py
+++ b/pypy/module/thread/test/test_thread.py
@@ -13,18 +13,26 @@
         def f():
             lock.acquire()
             lock.release()
+        start = thread._count()
         try:
             try:
                 for i in range(1000):
                     thread.start_new_thread(f, ())
             finally:
                 lock.release()
-                # wait a bit to allow most threads to finish now
-                time.sleep(0.5)
         except (thread.error, MemoryError):
             cls.w_can_start_many_threads = space.wrap(False)
         else:
             cls.w_can_start_many_threads = space.wrap(True)
+        # wait a bit to allow all threads to finish now
+        remaining = thread._count()
+        retries = 0
+        while remaining > start:
+            retries += 1
+            if retries == 200:
+                raise Exception("the test's threads don't stop!")
+            time.sleep(0.2)
+            remaining = thread._count()
 
     def test_start_new_thread(self):
         import thread
@@ -227,7 +235,7 @@
         import signal
 
         def f():
-            for x in range(5):
+            for x in range(40):
                 if waiting:
                     thread.interrupt_main()
                     return
@@ -236,7 +244,7 @@
 
         def busy_wait():
             waiting.append(None)
-            for x in range(10):
+            for x in range(50):
                 print 'tick...', x  # <-force the GIL to be released, as
                 time.sleep(0.1)    #   time.sleep doesn't do non-translated
             waiting.pop()
@@ -245,6 +253,8 @@
         signal.signal(signal.SIGINT, signal.default_int_handler)
 
         for i in range(100):
+            print
+            print "loop", i
             waiting = []
             thread.start_new_thread(f, ())
             raises(KeyboardInterrupt, busy_wait)
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -671,6 +671,7 @@
     left, right = specialnames
     errormsg = "unsupported operand type(s) for %s: '%%N' and '%%N'" % (
         symbol.replace('%', '%%'),)
+    seq_bug_compat = (symbol == '+' or symbol == '*')
 
     def binop_impl(space, w_obj1, w_obj2):
         w_typ1 = space.type(w_obj1)
@@ -686,20 +687,16 @@
             # __xxx__ and __rxxx__ methods where found by identity.
             # Note that space.is_w() is potentially not happy if one of them
             # is None...
-            if w_left_src is not w_right_src:    # XXX
-                # -- cpython bug compatibility: see objspace/std/test/
-                # -- test_unicodeobject.test_str_unicode_concat_overrides.
-                # -- The following handles "unicode + string subclass" by
-                # -- pretending that the unicode is a superclass of the
-                # -- string, thus giving priority to the string subclass'
-                # -- __radd__() method.  The case "string + unicode subclass"
-                # -- is handled directly by add__String_Unicode().
-                if symbol == '+' and space.is_w(w_typ1, space.w_unicode):
-                    w_typ1 = space.w_basestring
-                # -- end of bug compatibility
-                if space.is_true(space.issubtype(w_typ2, w_typ1)):
-                    if (w_left_src and w_right_src and
-                        not space.abstract_issubclass_w(w_left_src, w_right_src) and
+            if w_right_src and (w_left_src is not w_right_src) and w_left_src:
+                # 'seq_bug_compat' is for cpython bug-to-bug compatibility:
+                # see objspace/std/test/test_unicodeobject.*concat_overrides
+                # and objspace/test/test_descrobject.*rmul_overrides.
+                # For cases like "unicode + string subclass".
+                if ((seq_bug_compat and w_typ1.flag_sequence_bug_compat
+                                    and not w_typ2.flag_sequence_bug_compat)
+                        # the non-bug-compat part is the following check:
+                        or space.is_true(space.issubtype(w_typ2, w_typ1))):
+                    if (not space.abstract_issubclass_w(w_left_src, w_right_src) and
                         not space.abstract_issubclass_w(w_typ1, w_right_src)):
                         w_obj1, w_obj2 = w_obj2, w_obj1
                         w_left_impl, w_right_impl = w_right_impl, w_left_impl
diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -534,7 +534,7 @@
         if not e.match(space, space.w_TypeError):
             raise
     else:
-        return [c for c in buf.as_str()]
+        return list(buf.as_str())
 
     # sequence of bytes
     w_iter = space.iter(w_source)
@@ -1131,6 +1131,7 @@
     reverse = interp2app(W_BytearrayObject.descr_reverse,
                          doc=BytearrayDocstrings.reverse.__doc__),
 )
+W_BytearrayObject.typedef.flag_sequence_bug_compat = True
 
 init_signature = Signature(['source', 'encoding', 'errors'], None, None)
 init_defaults = [None, None, None]
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -951,6 +951,7 @@
     _formatter_field_name_split =
         interp2app(W_BytesObject.descr_formatter_field_name_split),
 )
+W_BytesObject.typedef.flag_sequence_bug_compat = True
 
 
 def string_escape_encode(s, quote):
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -1874,3 +1874,4 @@
     insert = interp2app(W_ListObject.descr_insert),
     remove = interp2app(W_ListObject.descr_remove),
 )
+W_ListObject.typedef.flag_sequence_bug_compat = True
diff --git a/pypy/objspace/std/stdtypedef.py b/pypy/objspace/std/stdtypedef.py
--- a/pypy/objspace/std/stdtypedef.py
+++ b/pypy/objspace/std/stdtypedef.py
@@ -93,6 +93,8 @@
                               overridetypedef=overridetypedef)
         if typedef is not overridetypedef:
             w_type.w_doc = space.wrap(typedef.doc)
+        if hasattr(typedef, 'flag_sequence_bug_compat'):
+            w_type.flag_sequence_bug_compat = typedef.flag_sequence_bug_compat
         w_type.lazyloaders = lazyloaders
         return w_type
 
diff --git a/pypy/objspace/std/tupleobject.py b/pypy/objspace/std/tupleobject.py
--- a/pypy/objspace/std/tupleobject.py
+++ b/pypy/objspace/std/tupleobject.py
@@ -244,6 +244,7 @@
     count = interp2app(W_AbstractTupleObject.descr_count),
     index = interp2app(W_AbstractTupleObject.descr_index)
 )
+W_AbstractTupleObject.typedef.flag_sequence_bug_compat = True
 
 
 class W_TupleObject(W_AbstractTupleObject):
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -67,6 +67,7 @@
     _immutable_fields_ = ["flag_heaptype",
                           "flag_cpytype",
                           "flag_abstract?",
+                          "flag_sequence_bug_compat",
                           'needsdel',
                           'weakrefable',
                           'hasdict',
@@ -104,6 +105,7 @@
         w_self.flag_heaptype = False
         w_self.flag_cpytype = False
         w_self.flag_abstract = False
+        w_self.flag_sequence_bug_compat = False
         w_self.instancetypedef = overridetypedef
 
         if overridetypedef is not None:
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -1068,6 +1068,7 @@
     _formatter_field_name_split =
         interp2app(W_UnicodeObject.descr_formatter_field_name_split),
 )
+W_UnicodeObject.typedef.flag_sequence_bug_compat = True
 
 
 def _create_list_from_unicode(value):
diff --git a/pypy/objspace/test/test_descroperation.py b/pypy/objspace/test/test_descroperation.py
--- a/pypy/objspace/test/test_descroperation.py
+++ b/pypy/objspace/test/test_descroperation.py
@@ -734,6 +734,44 @@
 
         assert X() == 'hello'
 
+    def test_sequence_rmul_overrides(self):
+        class oops(object):
+            def __rmul__(self, other):
+                return 42
+            def __index__(self):
+                return 3
+        assert '2' * oops() == 42
+        assert [2] * oops() == 42
+        assert (2,) * oops() == 42
+        assert u'2' * oops() == 42
+        assert bytearray('2') * oops() == 42
+        assert 1000 * oops() == 42
+        assert '2'.__mul__(oops()) == '222'
+
+    def test_sequence_rmul_overrides_oldstyle(self):
+        class oops:
+            def __rmul__(self, other):
+                return 42
+            def __index__(self):
+                return 3
+        assert '2' * oops() == 42
+        assert [2] * oops() == 42
+        assert (2,) * oops() == 42
+        assert u'2' * oops() == 42
+        assert bytearray('2') * oops() == 42
+        assert 1000 * oops() == 42
+        assert '2'.__mul__(oops()) == '222'
+
+    def test_sequence_radd_overrides(self):
+        class A1(list):
+            pass
+        class A2(list):
+            def __radd__(self, other):
+                return 42
+        assert [2] + A1([3]) == [2, 3]
+        assert type([2] + A1([3])) is list
+        assert [2] + A2([3]) == 42
+
 
 class AppTestWithBuiltinShortcut(AppTest_Descroperation):
     spaceconfig = {'objspace.std.builtinshortcut': True}
diff --git a/rpython/annotator/builtin.py b/rpython/annotator/builtin.py
--- a/rpython/annotator/builtin.py
+++ b/rpython/annotator/builtin.py
@@ -255,8 +255,13 @@
         BUILTIN_ANALYZERS[original] = value
 
 
- at analyzer_for(getattr(OSError.__init__, 'im_func', OSError.__init__))
-def OSError_init(s_self, *args):
+ at analyzer_for(getattr(object.__init__, 'im_func', object.__init__))
+def object_init(s_self, *args):
+    # ignore - mostly used for abstract classes initialization
+    pass
+
+ at analyzer_for(getattr(EnvironmentError.__init__, 'im_func', EnvironmentError.__init__))
+def EnvironmentError_init(s_self, *args):
     pass
 
 try:
@@ -268,11 +273,6 @@
     def WindowsError_init(s_self, *args):
         pass
 
- at analyzer_for(getattr(object.__init__, 'im_func', object.__init__))
-def object_init(s_self, *args):
-    # ignore - mostly used for abstract classes initialization
-    pass
-
 
 @analyzer_for(sys.getdefaultencoding)
 def conf():
diff --git a/rpython/annotator/classdef.py b/rpython/annotator/classdef.py
--- a/rpython/annotator/classdef.py
+++ b/rpython/annotator/classdef.py
@@ -438,8 +438,10 @@
 # ____________________________________________________________
 
 FORCE_ATTRIBUTES_INTO_CLASSES = {
-    OSError: {'errno': SomeInteger()},
-    }
+    EnvironmentError: {'errno': SomeInteger(),
+                       'strerror': SomeString(can_be_None=True),
+                       'filename': SomeString(can_be_None=True)},
+}
 
 try:
     WindowsError
@@ -455,4 +457,3 @@
 else:
     FORCE_ATTRIBUTES_INTO_CLASSES[termios.error] = \
         {'args': SomeTuple([SomeInteger(), SomeString()])}
-
diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py
--- a/rpython/annotator/unaryop.py
+++ b/rpython/annotator/unaryop.py
@@ -304,10 +304,10 @@
     def hint(self, *args_s):
         hints = args_s[-1].const
         if 'maxlength' in hints:
-            # only for iteration over lists or dicts at the moment,
+            # only for iteration over lists or dicts or strs at the moment,
             # not over an iterator object (because it has no known length)
             s_iterable = args_s[0]
-            if isinstance(s_iterable, (SomeList, SomeDict)):
+            if isinstance(s_iterable, (SomeList, SomeDict, SomeString)):
                 self = SomeList(self.listdef) # create a fresh copy
                 self.listdef.resize()
                 self.listdef.listitem.hint_maxlength = True
diff --git a/rpython/jit/backend/arm/codebuilder.py b/rpython/jit/backend/arm/codebuilder.py
--- a/rpython/jit/backend/arm/codebuilder.py
+++ b/rpython/jit/backend/arm/codebuilder.py
@@ -333,8 +333,11 @@
                     | (rn & 0xF) << 16)
 
     def DMB(self):
-        # note: 'cond' is only permitted on Thumb here
-        self.write32(0xf57ff05f)
+        # note: 'cond' is only permitted on Thumb here, but don't
+        # write literally 0xf57ff05f, because it's larger than 31 bits
+        c = cond.AL
+        self.write32(c << 28
+                    | 0x157ff05f)
 
     DIV = binary_helper_call('int_div')
     MOD = binary_helper_call('int_mod')
diff --git a/rpython/jit/backend/arm/detect.py b/rpython/jit/backend/arm/detect.py
--- a/rpython/jit/backend/arm/detect.py
+++ b/rpython/jit/backend/arm/detect.py
@@ -38,9 +38,9 @@
     try:
         buf = os.read(fd, 2048)
         if not buf:
+            n = 6  # we assume ARMv6 as base case
             debug_print("Could not detect ARM architecture "
                         "version, assuming", "ARMv%d" % n)
-            n = 6  # we asume ARMv6 as base case
     finally:
         os.close(fd)
     # "Processor       : ARMv%d-compatible processor rev 7 (v6l)"
diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py
--- a/rpython/jit/backend/arm/opassembler.py
+++ b/rpython/jit/backend/arm/opassembler.py
@@ -1128,6 +1128,8 @@
         self.mc.VCVT_int_to_float(res.value, r.svfp_ip.value)
         return fcond
 
+    # the following five instructions are only ARMv7;
+    # regalloc.py won't call them at all on ARMv6
     emit_op_llong_add = gen_emit_float_op('llong_add', 'VADD_i64')
     emit_op_llong_sub = gen_emit_float_op('llong_sub', 'VSUB_i64')
     emit_op_llong_and = gen_emit_float_op('llong_and', 'VAND_i64')
diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py
--- a/rpython/jit/backend/arm/regalloc.py
+++ b/rpython/jit/backend/arm/regalloc.py
@@ -184,7 +184,7 @@
 
 class Regalloc(BaseRegalloc):
 
-    def __init__(self, assembler=None):
+    def __init__(self, assembler):
         self.cpu = assembler.cpu
         self.assembler = assembler
         self.frame_manager = None
@@ -290,7 +290,7 @@
             return self.vfprm.convert_to_imm(value)
 
     def _prepare(self, inputargs, operations, allgcrefs):
-        cpu = self.assembler.cpu
+        cpu = self.cpu
         self.fm = ARMFrameManager(cpu.get_baseofs_of_frame_field())
         self.frame_manager = self.fm
         operations = cpu.gc_ll_descr.rewrite_assembler(cpu, operations,
@@ -550,18 +550,19 @@
                             EffectInfo.OS_LLONG_AND,
                             EffectInfo.OS_LLONG_OR,
                             EffectInfo.OS_LLONG_XOR):
-                args = self._prepare_llong_binop_xx(op, fcond)
-                self.perform_llong(op, args, fcond)
-                return
-            if oopspecindex == EffectInfo.OS_LLONG_TO_INT:
+                if self.cpu.cpuinfo.arch_version >= 7:
+                    args = self._prepare_llong_binop_xx(op, fcond)
+                    self.perform_llong(op, args, fcond)
+                    return
+            elif oopspecindex == EffectInfo.OS_LLONG_TO_INT:
                 args = self._prepare_llong_to_int(op, fcond)
                 self.perform_llong(op, args, fcond)
                 return
-            if oopspecindex == EffectInfo.OS_MATH_SQRT:
+            elif oopspecindex == EffectInfo.OS_MATH_SQRT:
                 args = self.prepare_op_math_sqrt(op, fcond)
                 self.perform_math(op, args, fcond)
                 return
-            #if oopspecindex == EffectInfo.OS_MATH_READ_TIMESTAMP:
+            #elif oopspecindex == EffectInfo.OS_MATH_READ_TIMESTAMP:
             #    ...
         return self._prepare_call(op)
 
@@ -590,7 +591,7 @@
         # spill variables that need to be saved around calls
         self.vfprm.before_call(save_all_regs=save_all_regs)
         if not save_all_regs:
-            gcrootmap = self.assembler.cpu.gc_ll_descr.gcrootmap
+            gcrootmap = self.cpu.gc_ll_descr.gcrootmap
             if gcrootmap and gcrootmap.is_shadow_stack:
                 save_all_regs = 2
         self.rm.before_call(save_all_regs=save_all_regs)
@@ -1082,7 +1083,7 @@
         gcmap = self.get_gcmap([r.r0, r.r1])
         self.possibly_free_var(t)
         #
-        gc_ll_descr = self.assembler.cpu.gc_ll_descr
+        gc_ll_descr = self.cpu.gc_ll_descr
         self.assembler.malloc_cond_varsize_frame(
             gc_ll_descr.get_nursery_free_addr(),
             gc_ll_descr.get_nursery_top_addr(),
@@ -1092,7 +1093,7 @@
         self.assembler._alignment_check()
 
     def prepare_op_call_malloc_nursery_varsize(self, op, fcond):
-        gc_ll_descr = self.assembler.cpu.gc_ll_descr
+        gc_ll_descr = self.cpu.gc_ll_descr
         if not hasattr(gc_ll_descr, 'max_size_of_young_obj'):
             raise Exception("unreachable code")
             # for boehm, this function should never be called
diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py
--- a/rpython/jit/metainterp/warmstate.py
+++ b/rpython/jit/metainterp/warmstate.py
@@ -127,6 +127,7 @@
 JC_TRACING         = 0x01
 JC_DONT_TRACE_HERE = 0x02
 JC_TEMPORARY       = 0x04
+JC_TRACING_OCCURRED= 0x08
 
 class BaseJitCell(object):
     """Subclasses of BaseJitCell are used in tandem with the single
@@ -160,6 +161,8 @@
         JC_TRACING: we are now tracing the loop from this greenkey.
         We'll likely end up with a wref_procedure_token, soonish.
 
+        JC_TRACING_OCCURRED: set if JC_TRACING was set at least once.
+
         JC_TEMPORARY: a "temporary" wref_procedure_token.
         It's the procedure_token of a dummy loop that simply calls
         back the interpreter.  Used for a CALL_ASSEMBLER where the
@@ -206,7 +209,7 @@
             # if we have this flag, and we *had* a procedure_token but
             # we no longer have one, then remove me.  this prevents this
             # JitCell from being immortal.
-            return self.has_seen_a_procedure_token()
+            return self.has_seen_a_procedure_token()     # i.e. dead weakref
         return True   # Other JitCells can be removed.
 
 # ____________________________________________________________
@@ -374,7 +377,7 @@
             if cell is None:
                 cell = JitCell(*greenargs)
                 jitcounter.install_new_cell(hash, cell)
-            cell.flags |= JC_TRACING
+            cell.flags |= JC_TRACING | JC_TRACING_OCCURRED
             try:
                 metainterp.compile_and_run_once(jitdriver_sd, *args)
             finally:
@@ -418,9 +421,15 @@
             if procedure_token is None:
                 if cell.flags & JC_DONT_TRACE_HERE:
                     if not cell.has_seen_a_procedure_token():
-                        # we're seeing a fresh JC_DONT_TRACE_HERE with no
-                        # procedure_token.  Compile now.
-                        bound_reached(hash, cell, *args)
+                        # A JC_DONT_TRACE_HERE, i.e. a non-inlinable function.
+                        # If we never tried to trace it, try it now immediately.
+                        # Otherwise, count normally.
+                        if cell.flags & JC_TRACING_OCCURRED:
+                            tick = jitcounter.tick(hash, increment_threshold)
+                        else:
+                            tick = True
+                        if tick:
+                            bound_reached(hash, cell, *args)
                         return
                 # it was an aborted compilation, or maybe a weakref that
                 # has been freed
diff --git a/rpython/memory/gctransform/asmgcroot.py b/rpython/memory/gctransform/asmgcroot.py
--- a/rpython/memory/gctransform/asmgcroot.py
+++ b/rpython/memory/gctransform/asmgcroot.py
@@ -517,6 +517,8 @@
                       "anywhere I know, bug in asmgcc")
             # fish the depth
             extra_stack_depth = (ebp_in_caller + STACK_DEPTH_OFS).signed[0]
+            ll_assert((extra_stack_depth & (rffi.sizeof(lltype.Signed) - 1))
+                       == 0, "asmgcc: misaligned extra_stack_depth")
             extra_stack_depth //= rffi.sizeof(lltype.Signed)
             self._shape_decompressor.setjitframe(extra_stack_depth)
             return
diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -38,6 +38,7 @@
 
 FILEP = rffi.COpaquePtr("FILE")
 OFF_T = config['off_t']
+
 _IONBF = config['_IONBF']
 _IOLBF = config['_IOLBF']
 _IOFBF = config['_IOFBF']
@@ -47,6 +48,11 @@
 BASE_BUF_SIZE = 4096
 BASE_LINE_SIZE = 100
 
+NEWLINE_UNKNOWN = 0
+NEWLINE_CR = 1
+NEWLINE_LF = 2
+NEWLINE_CRLF = 4
+
 
 def llexternal(*args, **kwargs):
     return rffi.llexternal(*args, compilation_info=eci, **kwargs)
@@ -63,7 +69,16 @@
 c_fclose = llexternal('fclose', [FILEP], rffi.INT)
 c_pclose = llexternal('pclose', [FILEP], rffi.INT)
 
+# Note: the following two functions are called from __del__ methods,
+# so must be 'releasegil=False'.  Otherwise, a program using both
+# threads and the RFile class cannot translate.  See c684bf704d1f
+c_fclose_in_del = llexternal('fclose', [FILEP], rffi.INT, releasegil=False)
+c_pclose_in_del = llexternal('pclose', [FILEP], rffi.INT, releasegil=False)
+_fclose2 = (c_fclose, c_fclose_in_del)
+_pclose2 = (c_pclose, c_pclose_in_del)
+
 c_getc = llexternal('getc', [FILEP], rffi.INT, macro=True)
+c_ungetc = llexternal('ungetc', [rffi.INT, FILEP], rffi.INT)
 c_fgets = llexternal('fgets', [rffi.CCHARP, rffi.INT, FILEP], rffi.CCHARP)
 c_fread = llexternal('fread', [rffi.CCHARP, rffi.SIZE_T, rffi.SIZE_T, FILEP],
                      rffi.SIZE_T)
@@ -81,11 +96,15 @@
 c_ferror = llexternal('ferror', [FILEP], rffi.INT)
 c_clearerr = llexternal('clearerr', [FILEP], lltype.Void)
 
+c_stdin = rffi.CExternVariable(FILEP, 'stdin', eci, c_type='FILE*')[0]
+c_stdout = rffi.CExternVariable(FILEP, 'stdout', eci, c_type='FILE*')[0]
+c_stderr = rffi.CExternVariable(FILEP, 'stderr', eci, c_type='FILE*')[0]
+
 
 def _error(ll_file):
     err = c_ferror(ll_file)
     c_clearerr(ll_file)
-    raise OSError(err, os.strerror(err))
+    raise IOError(err, os.strerror(err))
 
 
 def _dircheck(ll_file):
@@ -96,7 +115,7 @@
     else:
         if stat.S_ISDIR(st[0]):
             err = errno.EISDIR
-            raise OSError(err, os.strerror(err))
+            raise IOError(err, os.strerror(err))
 
 
 def _sanitize_mode(mode):
@@ -120,43 +139,41 @@
 
 
 def create_file(filename, mode="r", buffering=-1):
-    mode = _sanitize_mode(mode)
+    newmode = _sanitize_mode(mode)
     ll_name = rffi.str2charp(filename)
     try:
-        ll_mode = rffi.str2charp(mode)
+        ll_mode = rffi.str2charp(newmode)
         try:
             ll_file = c_fopen(ll_name, ll_mode)
             if not ll_file:
                 errno = rposix.get_errno()
-                raise OSError(errno, os.strerror(errno))
+                raise IOError(errno, os.strerror(errno))
         finally:
             lltype.free(ll_mode, flavor='raw')
     finally:
         lltype.free(ll_name, flavor='raw')
     _dircheck(ll_file)
-    if buffering >= 0:
-        buf = lltype.nullptr(rffi.CCHARP.TO)
-        if buffering == 0:
-            c_setvbuf(ll_file, buf, _IONBF, 0)
-        elif buffering == 1:
-            c_setvbuf(ll_file, buf, _IOLBF, BUFSIZ)
-        else:
-            c_setvbuf(ll_file, buf, _IOFBF, buffering)
-    return RFile(ll_file)
+    f = RFile(ll_file, mode)
+    f._setbufsize(buffering)
+    return f
 
 
-def create_fdopen_rfile(fd, mode="r"):
-    mode = _sanitize_mode(mode)
-    ll_mode = rffi.str2charp(mode)
+def create_fdopen_rfile(fd, mode="r", buffering=-1):
+    newmode = _sanitize_mode(mode)
+    fd = rffi.cast(rffi.INT, fd)
+    rposix.validate_fd(fd)
+    ll_mode = rffi.str2charp(newmode)
     try:
-        ll_file = c_fdopen(rffi.cast(rffi.INT, fd), ll_mode)
+        ll_file = c_fdopen(fd, ll_mode)
         if not ll_file:
             errno = rposix.get_errno()
             raise OSError(errno, os.strerror(errno))
     finally:
         lltype.free(ll_mode, flavor='raw')
     _dircheck(ll_file)
-    return RFile(ll_file)
+    f = RFile(ll_file, mode)
+    f._setbufsize(buffering)
+    return f
 
 
 def create_temp_rfile():
@@ -180,16 +197,61 @@
             lltype.free(ll_type, flavor='raw')
     finally:
         lltype.free(ll_command, flavor='raw')
-    return RFile(ll_file, c_pclose)
+    return RFile(ll_file, close2=_pclose2)
+
+
+def create_stdio():
+    close2 = (None, None)
+    stdin = RFile(c_stdin(), close2=close2)
+    stdout = RFile(c_stdout(), close2=close2)
+    stderr = RFile(c_stderr(), close2=close2)
+    return stdin, stdout, stderr
 
 
 class RFile(object):
-    def __init__(self, ll_file, do_close=c_fclose):
+    _setbuf = lltype.nullptr(rffi.CCHARP.TO)
+    _univ_newline = False
+    _newlinetypes = NEWLINE_UNKNOWN
+    _skipnextlf = False
+
+    def __init__(self, ll_file, mode=None, close2=_fclose2):
         self._ll_file = ll_file
-        self._do_close = do_close
+        if mode is not None:
+            self._univ_newline = 'U' in mode
+        self._close2 = close2
+
+    def _setbufsize(self, bufsize):
+        if bufsize >= 0:
+            if bufsize == 0:
+                mode = _IONBF
+            elif bufsize == 1:
+                mode = _IOLBF
+                bufsize = BUFSIZ
+            else:
+                mode = _IOFBF
+            if self._setbuf:
+                lltype.free(self._setbuf, flavor='raw')
+            if mode == _IONBF:
+                self._setbuf = lltype.nullptr(rffi.CCHARP.TO)
+            else:
+                self._setbuf = lltype.malloc(rffi.CCHARP.TO, bufsize, flavor='raw')
+            c_setvbuf(self._ll_file, self._setbuf, mode, bufsize)
 
     def __del__(self):
-        self.close()
+        """Closes the described file when the object's last reference
+        goes away.  Unlike an explicit call to close(), this is meant
+        as a last-resort solution and cannot release the GIL or return
+        an error code."""
+        ll_file = self._ll_file
+        if ll_file:
+            do_close = self._close2[1]
+            if do_close:
+                do_close(ll_file)       # return value ignored
+            if self._setbuf:
+                lltype.free(self._setbuf, flavor='raw')
+
+    def _cleanup_(self):
+        self._ll_file = lltype.nullptr(FILEP.TO)
 
     def close(self):
         """Closes the described file.
@@ -204,16 +266,69 @@
         if ll_file:
             # double close is allowed
             self._ll_file = lltype.nullptr(FILEP.TO)
-            res = self._do_close(ll_file)
-            if res == -1:
-                errno = rposix.get_errno()
-                raise OSError(errno, os.strerror(errno))
+            do_close = self._close2[0]
+            try:
+                if do_close:
+                    res = do_close(ll_file)
+                    if res == -1:
+                        errno = rposix.get_errno()
+                        raise IOError(errno, os.strerror(errno))
+            finally:
+                if self._setbuf:
+                    lltype.free(self._setbuf, flavor='raw')
+                    self._setbuf = lltype.nullptr(rffi.CCHARP.TO)
         return res
 
     def _check_closed(self):
         if not self._ll_file:
             raise ValueError("I/O operation on closed file")
 
+    def _fread(self, buf, n, stream):
+        if not self._univ_newline:
+            return c_fread(buf, 1, n, stream)
+
+        i = 0
+        dst = buf
+        newlinetypes = self._newlinetypes
+        skipnextlf = self._skipnextlf
+        while n:
+            nread = c_fread(dst, 1, n, stream)
+            if nread == 0:
+                break
+
+            src = dst
+            n -= nread
+            shortread = n != 0
+            while nread:
+                nread -= 1
+                c = src[0]
+                src = rffi.ptradd(src, 1)
+                if c == '\r':
+                    dst[0] = '\n'
+                    dst = rffi.ptradd(dst, 1)
+                    i += 1
+                    skipnextlf = True
+                elif skipnextlf and c == '\n':
+                    skipnextlf = False
+                    newlinetypes |= NEWLINE_CRLF
+                    n += 1
+                else:
+                    if c == '\n':
+                        newlinetypes |= NEWLINE_LF
+                    elif skipnextlf:
+                        newlinetypes |= NEWLINE_CR
+                    dst[0] = c
+                    dst = rffi.ptradd(dst, 1)
+                    i += 1
+                    skipnextlf = False
+            if shortread:
+                if skipnextlf and c_feof(stream):
+                    newlinetypes |= NEWLINE_CR
+                break
+        self._newlinetypes = newlinetypes
+        self._skipnextlf = skipnextlf
+        return i
+
     def read(self, size=-1):
         # XXX CPython uses a more delicate logic here
         self._check_closed()
@@ -226,7 +341,7 @@
             try:
                 s = StringBuilder()
                 while True:
-                    returned_size = c_fread(buf, 1, BASE_BUF_SIZE, ll_file)
+                    returned_size = self._fread(buf, BASE_BUF_SIZE, ll_file)
                     returned_size = intmask(returned_size)  # is between 0 and BASE_BUF_SIZE
                     if returned_size == 0:
                         if c_feof(ll_file):
@@ -238,12 +353,13 @@
                 lltype.free(buf, flavor='raw')
         else:  # size > 0
             with rffi.scoped_alloc_buffer(size) as buf:
-                returned_size = c_fread(buf.raw, 1, size, ll_file)
+                returned_size = self._fread(buf.raw, size, ll_file)
                 returned_size = intmask(returned_size)  # is between 0 and size
                 if returned_size == 0:
                     if not c_feof(ll_file):
                         raise _error(ll_file)
                 s = buf.str(returned_size)
+                assert s is not None
             return s
 
     def _readline1(self, raw_buf):
@@ -284,7 +400,7 @@
         self._check_closed()
         if size == 0:
             return ""
-        elif size < 0:
+        elif size < 0 and not self._univ_newline:
             with rffi.scoped_alloc_buffer(BASE_LINE_SIZE) as buf:
                 c = self._readline1(buf.raw)
                 if c >= 0:
@@ -299,19 +415,50 @@
                         break
                 s.append_charpsize(buf.raw, c)
             return s.build()
-        else:  # size > 0
+        else:  # size > 0 or self._univ_newline
             ll_file = self._ll_file
+            c = 0
             s = StringBuilder()
-            while s.getlength() < size:
-                c = c_getc(ll_file)
+            if self._univ_newline:
+                newlinetypes = self._newlinetypes
+                skipnextlf = self._skipnextlf
+                while size < 0 or s.getlength() < size:
+                    c = c_getc(ll_file)
+                    if c == EOF:
+                        break
+                    if skipnextlf:
+                        skipnextlf = False
+                        if c == ord('\n'):
+                            newlinetypes |= NEWLINE_CRLF
+                            c = c_getc(ll_file)
+                            if c == EOF:
+                                break
+                        else:
+                            newlinetypes |= NEWLINE_CR
+                    if c == ord('\r'):
+                        skipnextlf = True
+                        c = ord('\n')
+                    elif c == ord('\n'):
+                        newlinetypes |= NEWLINE_LF
+                    s.append(chr(c))
+                    if c == ord('\n'):
+                        break
                 if c == EOF:
-                    if c_ferror(ll_file):
-                        raise _error(ll_file)
-                    break
-                c = chr(c)
-                s.append(c)
-                if c == '\n':
-                    break
+                    if skipnextlf:
+                        newlinetypes |= NEWLINE_CR
+                self._newlinetypes = newlinetypes
+                self._skipnextlf = skipnextlf
+            else:
+                while s.getlength() < size:
+                    c = c_getc(ll_file)
+                    if c == EOF:
+                        break
+                    s.append(chr(c))
+                    if c == ord('\n'):
+                        break
+            if c == EOF:
+                if c_ferror(ll_file):
+                    raise _error(ll_file)
             return s.build()
 
     @enforceargs(None, str)
@@ -325,7 +472,8 @@
             bytes = c_fwrite(ll_value, 1, length, self._ll_file)
             if bytes != length:
                 errno = rposix.get_errno()
-                raise OSError(errno, os.strerror(errno))
+                c_clearerr(self._ll_file)
+                raise IOError(errno, os.strerror(errno))
         finally:
             rffi.free_nonmovingbuffer(value, ll_value)
 
@@ -334,7 +482,7 @@
         res = c_fflush(self._ll_file)
         if res != 0:
             errno = rposix.get_errno()
-            raise OSError(errno, os.strerror(errno))
+            raise IOError(errno, os.strerror(errno))
 
     def truncate(self, arg=-1):
         self._check_closed()
@@ -344,23 +492,36 @@
         res = c_ftruncate(self.fileno(), arg)
         if res == -1:
             errno = rposix.get_errno()
-            raise OSError(errno, os.strerror(errno))
+            raise IOError(errno, os.strerror(errno))
 
     def seek(self, pos, whence=0):
         self._check_closed()
         res = c_fseek(self._ll_file, pos, whence)
         if res == -1:
             errno = rposix.get_errno()
-            raise OSError(errno, os.strerror(errno))
+            raise IOError(errno, os.strerror(errno))
+        self._skipnextlf = False
 
     def tell(self):
         self._check_closed()
         res = intmask(c_ftell(self._ll_file))
         if res == -1:
             errno = rposix.get_errno()
-            raise OSError(errno, os.strerror(errno))
+            raise IOError(errno, os.strerror(errno))
+        if self._skipnextlf:
+            c = c_getc(self._ll_file)
+            if c == ord('\n'):
+                self._newlinetypes |= NEWLINE_CRLF
+                res += 1
+                self._skipnextlf = False
+            elif c != EOF:
+                c_ungetc(c, self._ll_file)
         return res
 
     def fileno(self):
         self._check_closed()
         return intmask(c_fileno(self._ll_file))
+
+    def isatty(self):
+        self._check_closed()
+        return os.isatty(c_fileno(self._ll_file))
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -119,7 +119,7 @@
         return 1
 
     def validate_fd(fd):
-        return 1
+        pass
 
 def closerange(fd_low, fd_high):
     # this behaves like os.closerange() from Python 2.6.
diff --git a/rpython/rlib/streamio.py b/rpython/rlib/streamio.py
--- a/rpython/rlib/streamio.py
+++ b/rpython/rlib/streamio.py
@@ -900,6 +900,13 @@
 
         return '\n'.join(result)
 
+    def readline(self):
+        line = self.base.readline()
+        limit = len(line) - 2
+        if limit >= 0 and line[limit] == '\r' and line[limit + 1] == '\n':
+            line = line[:limit] + '\n'
+        return line
+
     def tell(self):
         pos = self.base.tell()
         return pos - len(self.lfbuffer)
diff --git a/rpython/rlib/test/test_rfile.py b/rpython/rlib/test/test_rfile.py
--- a/rpython/rlib/test/test_rfile.py
+++ b/rpython/rlib/test/test_rfile.py
@@ -1,4 +1,4 @@
-import os, sys, py, errno
+import os, sys, py, errno, gc
 from rpython.rtyper.test.tool import BaseRtypingTest
 from rpython.tool.udir import udir
 from rpython.rlib import rfile
@@ -25,61 +25,67 @@
             f.close()
 
         f()
+        assert open(fname, "r").read() == "dupa"
         self.interpret(f, [])
         assert open(fname, "r").read() == "dupa"
 
     def test_open_errors(self):
-        def f(exc):
-            def g(run):
-                try:
-                    open('zzz', 'badmode')
-                except ValueError:
-                    pass
+        def f(run):
+            try:
+                open('zzz', 'badmode')
+            except ValueError:
+                pass
+            else:
+                assert False
+
+            try:
+                open('zzz')
+            except IOError as e:
+                assert e.errno == errno.ENOENT
+            else:
+                assert False
+
+            try:
+                open('.')
+            except IOError as e:
+                if os.name == 'posix':
+                    assert e.errno == errno.EISDIR
                 else:
-                    assert False
+                    assert e.errno == errno.EACCES
+            else:
+                assert False
 
-                try:
-                    open('zzz')
-                except exc as e:
-                    assert e.errno == errno.ENOENT
-                else:
-                    assert False
+            try:
+                os.fdopen(42, "badmode")
+            except ValueError:
+                pass
+            else:
+                assert False
 
-                try:
-                    open('.')
-                except exc as e:
-                    if os.name == 'posix':
+            try:
+                fd = os.open('.', os.O_RDONLY, 0777)
+            except OSError as e:
+                assert os.name == 'nt' and e.errno == errno.EACCES
+            else:
+                assert os.name != 'nt'
+                if run:
+                    try:
+                        os.fdopen(fd)
+                    except IOError as e:
                         assert e.errno == errno.EISDIR
                     else:
-                        assert e.errno == errno.EACCES
-                else:
-                    assert False
+                        assert False
+                os.close(fd)
 
-                try:
-                    os.fdopen(42, "badmode")
-                except ValueError:
-                    pass
-                else:
-                    assert False
+            try:
+                os.fdopen(12345)
+            except OSError as e:
+                assert e.errno == errno.EBADF
+            else:
+                assert False
 
-                try:
-                    fd = os.open('.', os.O_RDONLY, 0777)
-                except OSError as e:
-                    assert os.name == 'nt' and e.errno == errno.EACCES
-                else:
-                    assert os.name != 'nt'
-                    if run:
-                        try:
-                            os.fdopen(fd)
-                        except exc as e:
-                            assert e.errno == errno.EISDIR
-                        else:
-                            assert False
-                    os.close(fd)
-            return g
-
-        f(IOError)(sys.version_info >= (2, 7, 9))
-        self.interpret(f(OSError), [True])
+        f(sys.version_info >= (2, 7, 9))
+        self.interpret(f, [True])
 
     @py.test.mark.skipif("sys.platform == 'win32'")
     # http://msdn.microsoft.com/en-us/library/86cebhfs.aspx
@@ -98,15 +104,53 @@
         f()
         self.interpret(f, [])
 
+    @py.test.mark.skipif("sys.platform == 'win32'")
+    # http://msdn.microsoft.com/en-us/library/86cebhfs.aspx
+    def test_fdopen_buffering_line(self):
+        fname = str(self.tmpdir.join('file_1a'))
+
+        def f():
+            g = open(fname, 'w')
+            f = os.fdopen(os.dup(g.fileno()), 'w', 1)
+            g.close()
+            f.write('dupa\ndupb')
+            f2 = open(fname, 'r')
+            assert f2.read() == 'dupa\n'
+            f.close()
+            assert f2.read() == 'dupb'
+            f2.close()
+
+        f()
+        self.interpret(f, [])
+
     def test_open_buffering_full(self):
         fname = str(self.tmpdir.join('file_1b'))
 
         def f():
             f = open(fname, 'w', 128)
-            f.write('dupa')
+            f.write('dupa\ndupb')
             f2 = open(fname, 'r')
             assert f2.read() == ''
-            f.write('z' * 5000)
+            f.write('z' * 120)
+            assert f2.read() != ''
+            f.close()
+            assert f2.read() != ''
+            f2.close()
+
+        f()
+        self.interpret(f, [])
+
+    def test_fdopen_buffering_full(self):
+        fname = str(self.tmpdir.join('file_1b'))
+
+        def f():
+            g = open(fname, 'w')
+            f = os.fdopen(os.dup(g.fileno()), 'w', 128)
+            g.close()
+            f.write('dupa\ndupb')
+            f2 = open(fname, 'r')
+            assert f2.read() == ''
+            f.write('z' * 120)
             assert f2.read() != ''
             f.close()
             assert f2.read() != ''
@@ -120,10 +164,28 @@
 
         def f():
             f = open(fname, "w")
+            try:
+                f.read()
+            except IOError as e:
+                pass
+            else:
+                assert False
+            try:
+                f.readline()
+            except IOError as e:
+                pass
+            else:
+                assert False
             f.write("dupa\x00dupb")
             f.close()
             for mode in ['r', 'U']:
                 f2 = open(fname, mode)
+                try:
+                    f2.write('z')
+                except IOError as e:
+                    pass
+                else:
+                    assert False
                 dupa = f2.read(0)
                 assert dupa == ""
                 dupa = f2.read()
@@ -162,6 +224,40 @@
             assert d == "a"
             assert e == ""
 
+        f()
+        self.interpret(f, [])
+
+    def test_read_universal(self):
+        fname = str(self.tmpdir.join('read_univ'))
+        with open(fname, 'wb') as f:
+            f.write("dupa\ndupb\r\ndupc\rdupd")
+
+        def f():
+            f = open(fname, 'U')
+            assert f.read() == "dupa\ndupb\ndupc\ndupd"
+            assert f.read() == ""
+            f.seek(0)
+            assert f.read(10) == "dupa\ndupb\n"
+            assert f.read(42) == "dupc\ndupd"
+            assert f.read(1) == ""
+            f.seek(0)
+            assert f.readline() == "dupa\n"
+            assert f.tell() == 5
+            assert f.readline() == "dupb\n"
+            assert f.tell() == 11
+            assert f.readline() == "dupc\n"
+            assert f.tell() == 16
+            assert f.readline() == "dupd"
+            assert f.tell() == 20
+            assert f.readline() == ""
+            f.seek(0)
+            assert f.readline() == "dupa\n"
+            assert f.readline() == "dupb\n"
+            f.seek(4)
+            assert f.read(1) == "\n"
+            f.close()
+
+        f()
         self.interpret(f, [])
 
     def test_seek(self):
@@ -172,6 +268,12 @@
             f.write("xxx")
             f.seek(0)
             assert f.read() == "xxx"
+            try:
+                f.seek(0, 42)
+            except IOError as e:
+                assert e.errno == errno.EINVAL
+            else:
+                assert False
             f.close()
 
         f()
@@ -196,6 +298,12 @@
             new_fno = os.dup(f.fileno())
             f2 = os.fdopen(new_fno, "w")
             f.close()
+            try:
+                f2.read()
+            except IOError as e:
+                pass
+            else:
+                assert False
             f2.write("xxx")
             f2.close()
 
@@ -209,11 +317,14 @@
 
         def f():
             f = open(fname, "w")
+            assert not f.isatty()
             try:
                 return f.fileno()
             finally:
                 f.close()
 
+        res = f()
+        assert res > 2
         res = self.interpret(f, [])

From noreply at buildbot.pypy.org  Thu Sep 11 22:48:43 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Thu, 11 Sep 2014 22:48:43 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: merge default
Message-ID: <20140911204843.4962A1C31C8@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey 
Branch: py3k
Changeset: r73481:1568c757e7cb
Date: 2014-09-11 13:47 -0700
http://bitbucket.org/pypy/pypy/changeset/1568c757e7cb/

Log:	merge default

diff too long, truncating to 2000 out of 6687 lines

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -35,280 +35,290 @@
 the beginning of each file) the files in the 'pypy' directory are each
 copyrighted by one or more of the following people and organizations:    
 
-    Armin Rigo
-    Maciej Fijalkowski
-    Carl Friedrich Bolz
-    Antonio Cuni
-    Amaury Forgeot d'Arc
-    Samuele Pedroni
-    Alex Gaynor
-    Michael Hudson
-    David Schneider
-    Matti Picus
-    Brian Kearns
-    Philip Jenvey
-    Holger Krekel
-    Christian Tismer
-    Hakan Ardo
-    Benjamin Peterson
-    Manuel Jacob
-    Anders Chrigstrom
-    Eric van Riet Paap
-    Wim Lavrijsen
-    Ronan Lamy
-    Richard Emslie
-    Alexander Schremmer
-    Dan Villiom Podlaski Christiansen
-    Lukas Diekmann
-    Sven Hager
-    Anders Lehmann
-    Aurelien Campeas
-    Niklaus Haldimann
-    Camillo Bruni
-    Laura Creighton
-    Toon Verwaest
-    Remi Meier
-    Leonardo Santagada
-    Seo Sanghyeon
-    Romain Guillebert
-    Justin Peel
-    Ronny Pfannschmidt
-    David Edelsohn
-    Anders Hammarquist
-    Jakub Gustak
-    Guido Wesdorp
-    Lawrence Oluyede
-    Bartosz Skowron
-    Daniel Roberts
-    Niko Matsakis
-    Adrien Di Mascio
-    Alexander Hesse
-    Ludovic Aubry
-    Jacob Hallen
-    Jason Creighton
-    Alex Martelli
-    Michal Bendowski
-    Jan de Mooij
-    stian
-    Michael Foord
-    Stephan Diehl
-    Stefan Schwarzer
-    Valentino Volonghi
-    Tomek Meka
-    Patrick Maupin
-    Bob Ippolito
-    Bruno Gola
-    Jean-Paul Calderone
-    Timo Paulssen
-    Squeaky
-    Alexandre Fayolle
-    Simon Burton
-    Marius Gedminas
-    John Witulski
-    Konstantin Lopuhin
-    Greg Price
-    Dario Bertini
-    Mark Pearse
-    Simon Cross
-    Andreas Stührk
-    Jean-Philippe St. Pierre
-    Guido van Rossum
-    Pavel Vinogradov
-    Paweł Piotr Przeradowski
-    Paul deGrandis
-    Ilya Osadchiy
-    Tobias Oberstein
-    Adrian Kuhn
-    Boris Feigin
-    Stefano Rivera
-    tav
-    Taavi Burns
-    Georg Brandl
-    Bert Freudenberg
-    Stian Andreassen
-    Laurence Tratt
-    Wanja Saatkamp
-    Ivan Sichmann Freitas
-    Gerald Klix
-    Mike Blume
-    Oscar Nierstrasz
-    Stefan H. Muller
-    Jeremy Thurgood
-    Gregor Wegberg
-    Rami Chowdhury
-    Tobias Pape
-    Edd Barrett
-    David Malcolm
-    Eugene Oden
-    Henry Mason
-    Preston Timmons
-    Jeff Terrace
-    David Ripton
-    Dusty Phillips
-    Lukas Renggli
-    Guenter Jantzen
-    Ned Batchelder
-    Amit Regmi
-    Ben Young
-    Nicolas Chauvat
-    Andrew Durdin
-    Andrew Chambers
-    Michael Schneider
-    Nicholas Riley
-    Jason Chu
-    Igor Trindade Oliveira
-    Rocco Moretti
-    Gintautas Miliauskas
-    Michael Twomey
-    Lucian Branescu Mihaila
-    Tim Felgentreff
-    Tyler Wade
-    Gabriel Lavoie
-    Olivier Dormond
-    Jared Grubb
-    Karl Bartel
-    Brian Dorsey
-    Victor Stinner
-    Andrews Medina
-    Stuart Williams
-    Jasper Schulz
-    Christian Hudon
-    Toby Watson
-    Antoine Pitrou
-    Aaron Iles
-    Michael Cheng
-    Justas Sadzevicius
-    Mikael Schönenberg
-    Gasper Zejn
-    Neil Shepperd
-    Elmo Mäntynen
-    Jonathan David Riehl
-    Stanislaw Halik
-    Anders Qvist
-    Chirag Jadwani
-    Beatrice During
-    Alex Perry
-    Vincent Legoll
-    Alan McIntyre
-    Alexander Sedov
-    Corbin Simpson
-    Christopher Pope
-    wenzhuman
-    Christian Tismer 
-    Marc Abramowitz
-    Dan Stromberg
-    Stefano Parmesan
-    Alexis Daboville
-    Jens-Uwe Mager
-    Carl Meyer
-    Karl Ramm
-    Pieter Zieschang
-    Gabriel
-    Lukas Vacek
-    Andrew Dalke
-    Sylvain Thenault
-    Nathan Taylor
-    Vladimir Kryachko
-    Jacek Generowicz
-    Alejandro J. Cura
-    Jacob Oscarson
-    Travis Francis Athougies
-    Ryan Gonzalez
-    Kristjan Valur Jonsson
-    Sebastian Pawluś
-    Neil Blakey-Milner
-    anatoly techtonik
-    Lutz Paelike
-    Lucio Torre
-    Lars Wassermann
-    Henrik Vendelbo
-    Dan Buch
-    Miguel de Val Borro
-    Artur Lisiecki
-    Sergey Kishchenko
-    Ignas Mikalajunas
-    Christoph Gerum
-    Martin Blais
-    Lene Wagner
-    Tomo Cocoa
-    roberto at goyle
-    Yury V. Zaytsev
-    Anna Katrina Dominguez
-    William Leslie
-    Bobby Impollonia
-    timo at eistee.fritz.box
-    Andrew Thompson
-    Ben Darnell
-    Roberto De Ioris
-    Juan Francisco Cantero Hurtado
-    Godefroid Chappelle
-    Joshua Gilbert
-    Dan Colish
-    Christopher Armstrong
-    Michael Hudson-Doyle
-    Anders Sigfridsson
-    Yasir Suhail
-    rafalgalczynski at gmail.com
-    Floris Bruynooghe
-    Laurens Van Houtven
-    Akira Li
-    Gustavo Niemeyer
-    Stephan Busemann
-    Rafał Gałczyński
-    Yusei Tahara
-    Christian Muirhead
-    James Lan
-    shoma hosaka
-    Daniel Neuh?user
-    Matthew Miller
-    Buck Golemon
-    Konrad Delong
-    Dinu Gherman
-    Chris Lambacher
-    coolbutuseless at gmail.com
-    Rodrigo Araújo
-    w31rd0
-    Jim Baker
-    James Robert
-    Armin Ronacher
-    Brett Cannon
-    yrttyr
-    aliceinwire
-    OlivierBlanvillain
-    Zooko Wilcox-O Hearn
-    Tomer Chachamu
-    Christopher Groskopf
-    Asmo Soinio
-    Stefan Marr
-    jiaaro
-    opassembler.py
-    Antony Lee
-    Jim Hunziker
-    Markus Unterwaditzer
-    Even Wiik Thomassen
-    jbs
-    soareschen
-    Kurt Griffiths
-    Mike Bayer
-    Flavio Percoco
-    Kristoffer Kleine
-    yasirs
-    Michael Chermside
-    Anna Ravencroft
-    Julien Phalip
-    Dan Loewenherz
+  Armin Rigo
+  Maciej Fijalkowski
+  Carl Friedrich Bolz
+  Antonio Cuni
+  Amaury Forgeot d'Arc
+  Samuele Pedroni
+  Alex Gaynor
+  Michael Hudson
+  David Schneider
+  Matti Picus
+  Brian Kearns
+  Philip Jenvey
+  Holger Krekel
+  Christian Tismer
+  Hakan Ardo
+  Benjamin Peterson
+  Manuel Jacob
+  Anders Chrigstrom
+  Eric van Riet Paap
+  Ronan Lamy
+  Wim Lavrijsen
+  Richard Emslie
+  Alexander Schremmer
+  Dan Villiom Podlaski Christiansen
+  Lukas Diekmann
+  Sven Hager
+  Anders Lehmann
+  Aurelien Campeas
+  Niklaus Haldimann
+  Remi Meier
+  Camillo Bruni
+  Laura Creighton
+  Toon Verwaest
+  Leonardo Santagada
+  Seo Sanghyeon
+  Romain Guillebert
+  Justin Peel
+  Ronny Pfannschmidt
+  David Edelsohn
+  Anders Hammarquist
+  Jakub Gustak
+  Guido Wesdorp
+  Lawrence Oluyede
+  Bartosz Skowron
+  Gregor Wegberg
+  Daniel Roberts
+  Niko Matsakis
+  Adrien Di Mascio
+  Alexander Hesse
+  Ludovic Aubry
+  Jacob Hallen
+  Jason Creighton
+  Alex Martelli
+  Michal Bendowski
+  Jan de Mooij
+  stian
+  Michael Foord
+  Stephan Diehl
+  Tyler Wade
+  Stefan Schwarzer
+  Valentino Volonghi
+  Tomek Meka
+  Patrick Maupin
+  Bob Ippolito
+  Bruno Gola
+  Jean-Paul Calderone
+  Timo Paulssen
+  Squeaky
+  Alexandre Fayolle
+  Simon Burton
+  Marius Gedminas
+  Martin Matusiak
+  Konstantin Lopuhin
+  John Witulski
+  Wenzhu Man
+  Greg Price
+  Dario Bertini
+  Mark Pearse
+  Simon Cross
+  Ivan Sichmann Freitas
+  Andreas Stührk
+  Jean-Philippe St. Pierre
+  Guido van Rossum
+  Pavel Vinogradov
+  Stefano Rivera
+  Paweł Piotr Przeradowski
+  Paul deGrandis
+  Ilya Osadchiy
+  Tobias Oberstein
+  Adrian Kuhn
+  Boris Feigin
+  tav
+  Taavi Burns
+  Georg Brandl
+  Laurence Tratt
+  Bert Freudenberg
+  Stian Andreassen
+  Wanja Saatkamp
+  Gerald Klix
+  Mike Blume
+  Oscar Nierstrasz
+  Stefan H. Muller
+  Edd Barrett
+  Jeremy Thurgood
+  Rami Chowdhury
+  Tobias Pape
+  David Malcolm
+  Eugene Oden
+  Henry Mason
+  Vasily Kuznetsov
+  Preston Timmons
+  Jeff Terrace
+  David Ripton
+  Dusty Phillips
+  Lukas Renggli
+  Guenter Jantzen
+  Ned Batchelder
+  Amit Regmi
+  Ben Young
+  Nicolas Chauvat
+  Andrew Durdin
+  Andrew Chambers
+  Michael Schneider
+  Nicholas Riley
+  Jason Chu
+  Igor Trindade Oliveira
+  Tim Felgentreff
+  Rocco Moretti
+  Gintautas Miliauskas
+  Michael Twomey
+  Lucian Branescu Mihaila
+  Gabriel Lavoie
+  Olivier Dormond
+  Jared Grubb
+  Karl Bartel
+  Brian Dorsey
+  Victor Stinner
+  Andrews Medina
+  Stuart Williams
+  Jasper Schulz
+  Christian Hudon
+  Toby Watson
+  Antoine Pitrou
+  Aaron Iles
+  Michael Cheng
+  Justas Sadzevicius
+  Gasper Zejn
+  anatoly techtonik
+  Neil Shepperd
+  Mikael Schönenberg
+  Elmo M?ntynen
+  Jonathan David Riehl
+  Stanislaw Halik
+  Anders Qvist
+  Corbin Simpson
+  Chirag Jadwani
+  Beatrice During
+  Alex Perry
+  Vincent Legoll
+  Alan McIntyre
+  Alexander Sedov
+  Christopher Pope
+  Christian Tismer 
+  Marc Abramowitz
+  Dan Stromberg
+  Stefano Parmesan
+  Alexis Daboville
+  Jens-Uwe Mager
+  Carl Meyer
+  Karl Ramm
+  Pieter Zieschang
+  Sebastian Pawluś
+  Gabriel
+  Lukas Vacek
+  Andrew Dalke
+  Sylvain Thenault
+  Nathan Taylor
+  Vladimir Kryachko
+  Arjun Naik
+  Attila Gobi
+  Jacek Generowicz
+  Alejandro J. Cura
+  Jacob Oscarson
+  Travis Francis Athougies
+  Ryan Gonzalez
+  Ian Foote
+  Kristjan Valur Jonsson
+  Neil Blakey-Milner
+  Lutz Paelike
+  Lucio Torre
+  Lars Wassermann
+  Valentina Mukhamedzhanova
+  Henrik Vendelbo
+  Dan Buch
+  Miguel de Val Borro
+  Artur Lisiecki
+  Sergey Kishchenko
+  Yichao Yu
+  Ignas Mikalajunas
+  Christoph Gerum
+  Martin Blais
+  Lene Wagner
+  Tomo Cocoa
+  roberto at goyle
+  Yury V. Zaytsev
+  Anna Katrina Dominguez
+  William Leslie
+  Bobby Impollonia
+  timo at eistee.fritz.box
+  Andrew Thompson
+  Yusei Tahara
+  Ben Darnell
+  Roberto De Ioris
+  Juan Francisco Cantero Hurtado
+  Godefroid Chappelle
+  Joshua Gilbert
+  Dan Colish
+  Christopher Armstrong
+  Michael Hudson-Doyle
+  Anders Sigfridsson
+  Yasir Suhail
+  Jason Michalski
+  rafalgalczynski at gmail.com
+  Floris Bruynooghe
+  Laurens Van Houtven
+  Akira Li
+  Gustavo Niemeyer
+  Stephan Busemann
+  Rafał Gałczyński
+  Christian Muirhead
+  James Lan
+  shoma hosaka
+  Daniel Neuh?user
+  Matthew Miller
+  Buck Golemon
+  Konrad Delong
+  Dinu Gherman
+  Chris Lambacher
+  coolbutuseless at gmail.com
+  Rodrigo Araújo
+  Jim Baker
+  James Robert
+  Armin Ronacher
+  Brett Cannon
+  yrttyr
+  aliceinwire
+  OlivierBlanvillain
+  Zooko Wilcox-O Hearn
+  Tomer Chachamu
+  Christopher Groskopf
+  Asmo Soinio
+  Stefan Marr
+  jiaaro
+  Mads Kiilerich
+  opassembler.py
+  Antony Lee
+  Jim Hunziker
+  Markus Unterwaditzer
+  Even Wiik Thomassen
+  jbs
+  soareschen
+  Kurt Griffiths
+  Mike Bayer
+  Matthew Miller
+  Flavio Percoco
+  Kristoffer Kleine
+  yasirs
+  Michael Chermside
+  Anna Ravencroft
+  Dan Crosta
+  Julien Phalip
+  Dan Loewenherz
 
-    Heinrich-Heine University, Germany 
-    Open End AB (formerly AB Strakt), Sweden
-    merlinux GmbH, Germany 
-    tismerysoft GmbH, Germany 
-    Logilab Paris, France 
-    DFKI GmbH, Germany 
-    Impara, Germany
-    Change Maker, Sweden 
-    University of California Berkeley, USA
-    Google Inc.
-    King's College London
+  Heinrich-Heine University, Germany 
+  Open End AB (formerly AB Strakt), Sweden
+  merlinux GmbH, Germany 
+  tismerysoft GmbH, Germany 
+  Logilab Paris, France 
+  DFKI GmbH, Germany 
+  Impara, Germany
+  Change Maker, Sweden 
+  University of California Berkeley, USA
+  Google Inc.
+  King's College London
 
 The PyPy Logo as used by http://speed.pypy.org and others was created
 by Samuel Reis and is distributed on terms of Creative Commons Share Alike
diff --git a/_pytest/README-BEFORE-UPDATING b/_pytest/README-BEFORE-UPDATING
new file mode 100644
--- /dev/null
+++ b/_pytest/README-BEFORE-UPDATING
@@ -0,0 +1,17 @@
+This is PyPy's code of the pytest lib.  We don't expect to upgrade it
+very often, but once we do:
+
+    WARNING!
+
+    WE HAVE MADE A FEW TWEAKS HERE!
+
+Please be sure that you don't just copy the newer version from
+upstream without checking the few changes that we did.  This
+can be done like this:
+
+    cd 
+    hg log . -v | less
+
+then search for all " _pytest/" in that list to know which are the
+relevant checkins.  (Look for the checkins that only edit one
+or two files in this directory.)
diff --git a/_pytest/resultlog.py b/_pytest/resultlog.py
--- a/_pytest/resultlog.py
+++ b/_pytest/resultlog.py
@@ -53,16 +53,24 @@
         self.config = config
         self.logfile = logfile # preferably line buffered
 
-    def write_log_entry(self, testpath, lettercode, longrepr):
-        py.builtin.print_("%s %s" % (lettercode, testpath), file=self.logfile)
+    def write_log_entry(self, testpath, lettercode, longrepr, sections=None):
+        _safeprint("%s %s" % (lettercode, testpath), file=self.logfile)
         for line in longrepr.splitlines():
-            py.builtin.print_(" %s" % line, file=self.logfile)
+            _safeprint(" %s" % line, file=self.logfile)
+        if sections is not None and (
+                lettercode in ('E', 'F')):    # to limit the size of logs
+            for title, content in sections:
+                _safeprint(" ---------- %s ----------" % (title,),
+                           file=self.logfile)
+                for line in content.splitlines():
+                    _safeprint(" %s" % line, file=self.logfile)
 
     def log_outcome(self, report, lettercode, longrepr):
         testpath = getattr(report, 'nodeid', None)
         if testpath is None:
             testpath = report.fspath
-        self.write_log_entry(testpath, lettercode, longrepr)
+        self.write_log_entry(testpath, lettercode, longrepr,
+                             getattr(report, 'sections', None))
 
     def pytest_runtest_logreport(self, report):
         if report.when != "call" and report.passed:
@@ -98,3 +106,8 @@
         if path is None:
             path = "cwd:%s" % py.path.local()
         self.write_log_entry(path, '!', str(excrepr))
+
+def _safeprint(s, file):
+    if isinstance(s, unicode):
+        s = s.encode('utf-8')
+    py.builtin.print_(s, file=file)
diff --git a/lib-python/2.7/json/encoder.py b/lib-python/2.7/json/encoder.py
--- a/lib-python/2.7/json/encoder.py
+++ b/lib-python/2.7/json/encoder.py
@@ -529,3 +529,10 @@
                                           _current_indent_level):
                 yield chunk
             self.__remove_markers(markers, o)
+
+
+# overwrite some helpers here with more efficient versions
+try:
+    from _pypyjson import raw_encode_basestring_ascii
+except ImportError:
+    pass
diff --git a/lib-python/2.7/sysconfig.py b/lib-python/2.7/sysconfig.py
--- a/lib-python/2.7/sysconfig.py
+++ b/lib-python/2.7/sysconfig.py
@@ -7,30 +7,30 @@
 
 _INSTALL_SCHEMES = {
     'posix_prefix': {
-        'stdlib': '{base}/lib/python{py_version_short}',
-        'platstdlib': '{platbase}/lib/python{py_version_short}',
-        'purelib': '{base}/lib/python{py_version_short}/site-packages',
-        'platlib': '{platbase}/lib/python{py_version_short}/site-packages',
-        'include': '{base}/include/python{py_version_short}',
-        'platinclude': '{platbase}/include/python{py_version_short}',
+        'stdlib': '{base}/lib/{implementation_lower}{py_version_short}',
+        'platstdlib': '{platbase}/lib/{implementation_lower}{py_version_short}',
+        'purelib': '{base}/lib/{implementation_lower}{py_version_short}/site-packages',
+        'platlib': '{platbase}/lib/{implementation_lower}{py_version_short}/site-packages',
+        'include': '{base}/include/{implementation_lower}{py_version_short}',
+        'platinclude': '{platbase}/include/{implementation_lower}{py_version_short}',
         'scripts': '{base}/bin',
         'data': '{base}',
         },
     'posix_home': {
-        'stdlib': '{base}/lib/python',
-        'platstdlib': '{base}/lib/python',
-        'purelib': '{base}/lib/python',
-        'platlib': '{base}/lib/python',
-        'include': '{base}/include/python',
-        'platinclude': '{base}/include/python',
+        'stdlib': '{base}/lib/{implementation_lower}',
+        'platstdlib': '{base}/lib/{implementation_lower}',
+        'purelib': '{base}/lib/{implementation_lower}',
+        'platlib': '{base}/lib/{implementation_lower}',
+        'include': '{base}/include/{implementation_lower}',
+        'platinclude': '{base}/include/{implementation_lower}',
         'scripts': '{base}/bin',
         'data'   : '{base}',
         },
     'pypy': {
-        'stdlib': '{base}/lib-python/{py_version_short}',
-        'platstdlib': '{base}/lib-python/{py_version_short}',
-        'purelib': '{base}/lib-python/{py_version_short}',
-        'platlib': '{base}/lib-python/{py_version_short}',
+        'stdlib': '{base}/lib-{implementation_lower}/{py_version_short}',
+        'platstdlib': '{base}/lib-{implementation_lower}/{py_version_short}',
+        'purelib': '{base}/lib-{implementation_lower}/{py_version_short}',
+        'platlib': '{base}/lib-{implementation_lower}/{py_version_short}',
         'include': '{base}/include',
         'platinclude': '{base}/include',
         'scripts': '{base}/bin',
@@ -57,37 +57,37 @@
         'data'   : '{base}',
         },
     'os2_home': {
-        'stdlib': '{userbase}/lib/python{py_version_short}',
-        'platstdlib': '{userbase}/lib/python{py_version_short}',
-        'purelib': '{userbase}/lib/python{py_version_short}/site-packages',
-        'platlib': '{userbase}/lib/python{py_version_short}/site-packages',
-        'include': '{userbase}/include/python{py_version_short}',
+        'stdlib': '{userbase}/lib/{implementation_lower}{py_version_short}',
+        'platstdlib': '{userbase}/lib/{implementation_lower}{py_version_short}',
+        'purelib': '{userbase}/lib/{implementation_lower}{py_version_short}/site-packages',
+        'platlib': '{userbase}/lib/{implementation_lower}{py_version_short}/site-packages',
+        'include': '{userbase}/include/{implementation_lower}{py_version_short}',
         'scripts': '{userbase}/bin',
         'data'   : '{userbase}',
         },
     'nt_user': {
-        'stdlib': '{userbase}/Python{py_version_nodot}',
-        'platstdlib': '{userbase}/Python{py_version_nodot}',
-        'purelib': '{userbase}/Python{py_version_nodot}/site-packages',
-        'platlib': '{userbase}/Python{py_version_nodot}/site-packages',
-        'include': '{userbase}/Python{py_version_nodot}/Include',
+        'stdlib': '{userbase}/{implementation}{py_version_nodot}',
+        'platstdlib': '{userbase}/{implementation}{py_version_nodot}',
+        'purelib': '{userbase}/{implementation}{py_version_nodot}/site-packages',
+        'platlib': '{userbase}/{implementation}{py_version_nodot}/site-packages',
+        'include': '{userbase}/{implementation}{py_version_nodot}/Include',
         'scripts': '{userbase}/Scripts',
         'data'   : '{userbase}',
         },
     'posix_user': {
-        'stdlib': '{userbase}/lib/python{py_version_short}',
-        'platstdlib': '{userbase}/lib/python{py_version_short}',
-        'purelib': '{userbase}/lib/python{py_version_short}/site-packages',
-        'platlib': '{userbase}/lib/python{py_version_short}/site-packages',
-        'include': '{userbase}/include/python{py_version_short}',
+        'stdlib': '{userbase}/lib/{implementation_lower}{py_version_short}',
+        'platstdlib': '{userbase}/lib/{implementation_lower}{py_version_short}',
+        'purelib': '{userbase}/lib/{implementation_lower}{py_version_short}/site-packages',
+        'platlib': '{userbase}/lib/{implementation_lower}{py_version_short}/site-packages',
+        'include': '{userbase}/include/{implementation_lower}{py_version_short}',
         'scripts': '{userbase}/bin',
         'data'   : '{userbase}',
         },
     'osx_framework_user': {
-        'stdlib': '{userbase}/lib/python',
-        'platstdlib': '{userbase}/lib/python',
-        'purelib': '{userbase}/lib/python/site-packages',
-        'platlib': '{userbase}/lib/python/site-packages',
+        'stdlib': '{userbase}/lib/{implementation_lower}',
+        'platstdlib': '{userbase}/lib/{implementation_lower}',
+        'purelib': '{userbase}/lib/{implementation_lower}/site-packages',
+        'platlib': '{userbase}/lib/{implementation_lower}/site-packages',
         'include': '{userbase}/include',
         'scripts': '{userbase}/bin',
         'data'   : '{userbase}',
@@ -104,6 +104,11 @@
 _CONFIG_VARS = None
 _USER_BASE = None
 
+def _get_implementation():
+    if '__pypy__' in sys.builtin_module_names:
+        return 'PyPy'
+    return 'Python'
+
 def _safe_realpath(path):
     try:
         return realpath(path)
@@ -475,6 +480,8 @@
         _CONFIG_VARS['base'] = _PREFIX
         _CONFIG_VARS['platbase'] = _EXEC_PREFIX
         _CONFIG_VARS['projectbase'] = _PROJECT_BASE
+        _CONFIG_VARS['implementation'] = _get_implementation()
+        _CONFIG_VARS['implementation_lower'] = _get_implementation().lower()
 
         if os.name in ('nt', 'os2'):
             _init_non_posix(_CONFIG_VARS)
@@ -644,6 +651,8 @@
     _print_dict('Paths', get_paths())
     print
     _print_dict('Variables', get_config_vars())
+    print
+    _print_dict('User', get_paths('%s_user' % os.name))
 
 
 if __name__ == '__main__':
diff --git a/lib-python/2.7/test/pickletester.py b/lib-python/2.7/test/pickletester.py
--- a/lib-python/2.7/test/pickletester.py
+++ b/lib-python/2.7/test/pickletester.py
@@ -6,14 +6,16 @@
 import pickletools
 import copy_reg
 
-from test.test_support import TestFailed, verbose, have_unicode, TESTFN, impl_detail
+from test.test_support import TestFailed, verbose, have_unicode, TESTFN
 try:
-    from test.test_support import _2G, _1M, precisionbigmemtest
+    from test.test_support import _2G, _1M, precisionbigmemtest, impl_detail
 except ImportError:
     # this import might fail when run on older Python versions by test_xpickle
     _2G = _1M = 0
     def precisionbigmemtest(*args, **kwargs):
         return lambda self: None
+    def impl_detail(*args, **kwargs):
+        return lambda self: None
 
 # Tests that try a number of pickle protocols should have a
 #     for proto in protocols:
diff --git a/lib-python/2.7/test/test_mmap.py b/lib-python/2.7/test/test_mmap.py
--- a/lib-python/2.7/test/test_mmap.py
+++ b/lib-python/2.7/test/test_mmap.py
@@ -179,25 +179,27 @@
         import sys
         f = open(TESTFN, "r+b")
         try:
-            m = mmap.mmap(f.fileno(), mapsize+1)
-        except ValueError:
-            # we do not expect a ValueError on Windows
-            # CAUTION:  This also changes the size of the file on disk, and
-            # later tests assume that the length hasn't changed.  We need to
-            # repair that.
+            try:
+                m = mmap.mmap(f.fileno(), mapsize+1)
+            except ValueError:
+                # we do not expect a ValueError on Windows
+                # CAUTION:  This also changes the size of the file on disk, and
+                # later tests assume that the length hasn't changed.  We need to
+                # repair that.
+                if sys.platform.startswith('win'):
+                    self.fail("Opening mmap with size+1 should work on Windows.")
+            else:
+                # we expect a ValueError on Unix, but not on Windows
+                if not sys.platform.startswith('win'):
+                    self.fail("Opening mmap with size+1 should raise ValueError.")
+                m.close()
+        finally:
+            f.close()
             if sys.platform.startswith('win'):
-                self.fail("Opening mmap with size+1 should work on Windows.")
-        else:
-            # we expect a ValueError on Unix, but not on Windows
-            if not sys.platform.startswith('win'):
-                self.fail("Opening mmap with size+1 should raise ValueError.")
-            m.close()
-        f.close()
-        if sys.platform.startswith('win'):
-            # Repair damage from the resizing test.
-            f = open(TESTFN, 'r+b')
-            f.truncate(mapsize)
-            f.close()
+                # Repair damage from the resizing test.
+                f = open(TESTFN, 'r+b')
+                f.truncate(mapsize)
+                f.close()
 
         # Opening mmap with access=ACCESS_WRITE
         f = open(TESTFN, "r+b")
diff --git a/py/README-BEFORE-UPDATING b/py/README-BEFORE-UPDATING
new file mode 100644
--- /dev/null
+++ b/py/README-BEFORE-UPDATING
@@ -0,0 +1,17 @@
+This is PyPy's code of the py lib.  We don't expect to upgrade it
+very often, but once we do:
+
+    WARNING!
+
+    WE HAVE MADE A FEW TWEAKS HERE!
+
+Please be sure that you don't just copy the newer version from
+upstream without checking the few changes that we did.  This
+can be done like this:
+
+    cd 
+    hg log . -v | less
+
+then search for all " py/" in that list to know which are the
+relevant checkins.  (Look for the checkins that only edit one
+or two files in this directory.)
diff --git a/py/_code/code.py b/py/_code/code.py
--- a/py/_code/code.py
+++ b/py/_code/code.py
@@ -588,7 +588,7 @@
 class TerminalRepr:
     def __str__(self):
         s = self.__unicode__()
-        if sys.version_info[0] < 3:
+        if sys.version_info[0] < 3 and isinstance(s, unicode):
             s = s.encode('utf-8')
         return s
 
diff --git a/py/_path/local.py b/py/_path/local.py
--- a/py/_path/local.py
+++ b/py/_path/local.py
@@ -750,7 +750,8 @@
     mkdtemp = classmethod(mkdtemp)
 
     def make_numbered_dir(cls, prefix='session-', rootdir=None, keep=3,
-                          lock_timeout = 172800):   # two days
+                          lock_timeout = 172800,   # two days
+                          min_timeout = 300):      # five minutes
         """ return unique directory with a number greater than the current
             maximum one.  The number is assumed to start directly after prefix.
             if keep is true directories with a number less than (maxnum-keep)
@@ -818,6 +819,20 @@
             for path in rootdir.listdir():
                 num = parse_num(path)
                 if num is not None and num <= (maxnum - keep):
+                    if min_timeout:
+                        # NB: doing this is needed to prevent (or reduce
+                        # a lot the chance of) the following situation:
+                        # 'keep+1' processes call make_numbered_dir() at
+                        # the same time, they create dirs, but then the
+                        # last process notices the first dir doesn't have
+                        # (yet) a .lock in it and kills it.
+                        try:
+                            t1 = path.lstat().mtime
+                            t2 = lockfile.lstat().mtime
+                            if abs(t2-t1) < min_timeout:
+                                continue   # skip directories too recent
+                        except py.error.Error:
+                            continue   # failure to get a time, better skip
                     lf = path.join('.lock')
                     try:
                         t1 = lf.lstat().mtime
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -53,12 +53,11 @@
 if sys.platform == "win32":
     working_modules.add("_winreg")
     # unix only modules
-    working_modules.remove("crypt")
-    working_modules.remove("fcntl")
-    working_modules.remove("pwd")
-    working_modules.remove("termios")
-    working_modules.remove("_minimal_curses")
-    working_modules.remove("_posixsubprocess")
+    for name in ["crypt", "fcntl", "pwd", "termios", "_minimal_curses",
+                 "_posixsubprocess"]:
+        working_modules.remove(name)
+        if name in translation_modules:
+            translation_modules.remove(name)
 
     if "cppyy" in working_modules:
         working_modules.remove("cppyy")  # not tested on win32
diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -65,9 +65,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '2.3'
+version = '2.4'
 # The full version, including alpha/beta/rc tags.
-release = '2.3.0'
+release = '2.4.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -25,8 +25,8 @@
   Manuel Jacob
   Anders Chrigstrom
   Eric van Riet Paap
+  Ronan Lamy
   Wim Lavrijsen
-  Ronan Lamy
   Richard Emslie
   Alexander Schremmer
   Dan Villiom Podlaski Christiansen
@@ -35,10 +35,10 @@
   Anders Lehmann
   Aurelien Campeas
   Niklaus Haldimann
+  Remi Meier
   Camillo Bruni
   Laura Creighton
   Toon Verwaest
-  Remi Meier
   Leonardo Santagada
   Seo Sanghyeon
   Romain Guillebert
@@ -50,6 +50,7 @@
   Guido Wesdorp
   Lawrence Oluyede
   Bartosz Skowron
+  Gregor Wegberg
   Daniel Roberts
   Niko Matsakis
   Adrien Di Mascio
@@ -63,6 +64,7 @@
   stian
   Michael Foord
   Stephan Diehl
+  Tyler Wade
   Stefan Schwarzer
   Valentino Volonghi
   Tomek Meka
@@ -75,43 +77,45 @@
   Alexandre Fayolle
   Simon Burton
   Marius Gedminas
+  Martin Matusiak
+  Konstantin Lopuhin
   John Witulski
-  Konstantin Lopuhin
+  Wenzhu Man
   Greg Price
   Dario Bertini
   Mark Pearse
   Simon Cross
+  Ivan Sichmann Freitas
   Andreas Stührk
   Jean-Philippe St. Pierre
   Guido van Rossum
   Pavel Vinogradov
+  Stefano Rivera
   Paweł Piotr Przeradowski
   Paul deGrandis
   Ilya Osadchiy
   Tobias Oberstein
   Adrian Kuhn
   Boris Feigin
-  Stefano Rivera
   tav
   Taavi Burns
   Georg Brandl
+  Laurence Tratt
   Bert Freudenberg
   Stian Andreassen
-  Laurence Tratt
   Wanja Saatkamp
-  Ivan Sichmann Freitas
   Gerald Klix
   Mike Blume
   Oscar Nierstrasz
   Stefan H. Muller
+  Edd Barrett
   Jeremy Thurgood
-  Gregor Wegberg
   Rami Chowdhury
   Tobias Pape
-  Edd Barrett
   David Malcolm
   Eugene Oden
   Henry Mason
+  Vasily Kuznetsov
   Preston Timmons
   Jeff Terrace
   David Ripton
@@ -128,12 +132,11 @@
   Nicholas Riley
   Jason Chu
   Igor Trindade Oliveira
+  Tim Felgentreff
   Rocco Moretti
   Gintautas Miliauskas
   Michael Twomey
   Lucian Branescu Mihaila
-  Tim Felgentreff
-  Tyler Wade
   Gabriel Lavoie
   Olivier Dormond
   Jared Grubb
@@ -149,22 +152,22 @@
   Aaron Iles
   Michael Cheng
   Justas Sadzevicius
+  Gasper Zejn
+  anatoly techtonik
+  Neil Shepperd
   Mikael Schönenberg
-  Gasper Zejn
-  Neil Shepperd
-  Elmo Mäntynen
+  Elmo M?ntynen
   Jonathan David Riehl
   Stanislaw Halik
   Anders Qvist
+  Corbin Simpson
   Chirag Jadwani
   Beatrice During
   Alex Perry
   Vincent Legoll
   Alan McIntyre
   Alexander Sedov
-  Corbin Simpson
   Christopher Pope
-  wenzhuman
   Christian Tismer 
   Marc Abramowitz
   Dan Stromberg
@@ -174,29 +177,33 @@
   Carl Meyer
   Karl Ramm
   Pieter Zieschang
+  Sebastian Pawluś
   Gabriel
   Lukas Vacek
   Andrew Dalke
   Sylvain Thenault
   Nathan Taylor
   Vladimir Kryachko
+  Arjun Naik
+  Attila Gobi
   Jacek Generowicz
   Alejandro J. Cura
   Jacob Oscarson
   Travis Francis Athougies
   Ryan Gonzalez
+  Ian Foote
   Kristjan Valur Jonsson
-  Sebastian Pawluś
   Neil Blakey-Milner
-  anatoly techtonik
   Lutz Paelike
   Lucio Torre
   Lars Wassermann
+  Valentina Mukhamedzhanova
   Henrik Vendelbo
   Dan Buch
   Miguel de Val Borro
   Artur Lisiecki
   Sergey Kishchenko
+  Yichao Yu
   Ignas Mikalajunas
   Christoph Gerum
   Martin Blais
@@ -209,6 +216,7 @@
   Bobby Impollonia
   timo at eistee.fritz.box
   Andrew Thompson
+  Yusei Tahara
   Ben Darnell
   Roberto De Ioris
   Juan Francisco Cantero Hurtado
@@ -219,6 +227,7 @@
   Michael Hudson-Doyle
   Anders Sigfridsson
   Yasir Suhail
+  Jason Michalski
   rafalgalczynski at gmail.com
   Floris Bruynooghe
   Laurens Van Houtven
@@ -226,7 +235,6 @@
   Gustavo Niemeyer
   Stephan Busemann
   Rafał Gałczyński
-  Yusei Tahara
   Christian Muirhead
   James Lan
   shoma hosaka
@@ -238,7 +246,6 @@
   Chris Lambacher
   coolbutuseless at gmail.com
   Rodrigo Araújo
-  w31rd0
   Jim Baker
   James Robert
   Armin Ronacher
@@ -252,6 +259,7 @@
   Asmo Soinio
   Stefan Marr
   jiaaro
+  Mads Kiilerich
   opassembler.py
   Antony Lee
   Jim Hunziker
@@ -261,11 +269,13 @@
   soareschen
   Kurt Griffiths
   Mike Bayer
+  Matthew Miller
   Flavio Percoco
   Kristoffer Kleine
   yasirs
   Michael Chermside
   Anna Ravencroft
+  Dan Crosta
   Julien Phalip
   Dan Loewenherz
 
diff --git a/pypy/doc/getting-started-python.rst b/pypy/doc/getting-started-python.rst
--- a/pypy/doc/getting-started-python.rst
+++ b/pypy/doc/getting-started-python.rst
@@ -111,6 +111,10 @@
    of your choice.  Typical example: ``--opt=2`` gives a good (but of
    course slower) Python interpreter without the JIT.
 
+   Consider using PyPy instead of CPython in the above command line,
+   as it is much faster.  (Note that ``rpython`` is a Python 2 program,
+   not Python 3; you need to run either PyPy 2 or CPython 2.)
+
 .. _`optimization level`: config/opt.html
 
 If everything works correctly this will create an executable
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-2.4.0.rst
    release-2.3.1.rst
    release-2.3.0.rst
    release-2.2.1.rst
diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst
--- a/pypy/doc/index.rst
+++ b/pypy/doc/index.rst
@@ -40,7 +40,7 @@
 
 * `FAQ`_: some frequently asked questions.
 
-* `Release 2.3.1`_: the latest official release
+* `Release 2.4.0`_: the latest official release
 
 * `PyPy Blog`_: news and status info about PyPy 
 
@@ -110,7 +110,7 @@
 .. _`Getting Started`: getting-started.html
 .. _`Papers`: extradoc.html
 .. _`Videos`: video-index.html
-.. _`Release 2.3.1`: http://pypy.org/download.html
+.. _`Release 2.4.0`: http://pypy.org/download.html
 .. _`speed.pypy.org`: http://speed.pypy.org
 .. _`RPython toolchain`: translation.html
 .. _`potential project ideas`: project-ideas.html
diff --git a/pypy/doc/release-2.4.0.rst b/pypy/doc/release-2.4.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-2.4.0.rst
@@ -0,0 +1,119 @@
+=================================================
+PyPy 2.4 - Snow White
+=================================================
+
+We're pleased to announce PyPy 2.4, which contains significant performance
+enhancements and bug fixes. 
+
+You can already download the PyPy 2.4-beta1 pre-release here:
+
+    http://pypy.org/download.html
+
+We would like to thank our donors for the continued support of the PyPy
+project, and for those who donate to our three sub-projects.
+We've shown quite a bit of progress, but we're slowly running out of funds.
+Please consider donating more, or even better convince your employer to donate,
+so we can finish those projects! We would like to also point out that in
+September, `the Python Software Foundation`_ will `match funds`_ for
+any donations up to $10k!  The three sub-projects are:
+
+* `Py3k`_ (supporting Python 3.x): We have released a Python 3.2.5 compatible version
+   we call PyPy3 2.3.1, and are working toward a Python 3.3 compatible version
+
+* `STM`_ (software transactional memory): We have released a first working version,
+  and continue to try out new promising paths of achieving a fast multithreaded Python
+
+* `NumPy`_ which requires installation of our fork of upstream numpy, 
+  available `on bitbucket`_
+
+.. _`Py3k`: http://pypy.org/py3donate.html
+.. _`STM`: http://pypy.org/tmdonate2.html
+.. _`NumPy`: http://pypy.org/numpydonate.html
+.. _`on bitbucket`: https://www.bitbucket.org/pypy/numpy   
+.. _`the Python Software Foundation`: https://www.python.org/psf/
+.. _`match funds`: http://morepypy.blogspot.com/2014/09/python-software-foundation-matching.html
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`pypy 2.4 and cpython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+This release supports **x86** machines on most common operating systems 
+(Linux 32/64, Mac OS X 64, Windows, and OpenBSD),
+as well as newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux. 
+
+While we support 32 bit python on Windows, work on the native Windows 64
+bit python is still stalling, we would welcome a volunteer
+to `handle that`_.
+
+.. _`pypy 2.4 and cpython 2.7.x`: http://speed.pypy.org
+.. _`handle that`: http://doc.pypy.org/en/latest/windows.html#what-is-missing-for-a-full-64-bit-translation
+
+Highlights
+==========
+
+Benchmarks improved after internal enhancements in string and
+bytearray handling, and a major rewrite of the GIL handling. This means
+that external calls are now a lot faster, especially the CFFI ones. It also
+means better performance in a lot of corner cases with handling strings or
+bytearrays. The main bugfix is handling of many socket objects in your
+program which in the long run used to "leak" memory.
+
+PyPy now uses Python 2.7.8 standard library.
+
+We welcomed more than 12 new contributors, and conducted two Google
+Summer of Code projects, as well as other student projects not
+directly related to Summer of Code.
+
+
+Issues reported with our previous release were fixed after reports from users on
+our new issue tracker at https://bitbucket.org/pypy/pypy/issues or on IRC at
+#pypy. Here is a summary of the user-facing changes;
+for more information see `whats-new`_:
+
+* Reduced internal copying of bytearray operations
+
+* Tweak the internal structure of StringBuilder to speed up large string
+  handling, which becomes advantageous on large programs at the cost of slightly
+  slower small *benchmark* type programs.
+
+* Boost performance of thread-local variables in both unjitted and jitted code,
+  this mostly affects errno handling on linux, which makes external calls
+  faster.
+
+* Move to a mixed polling and mutex GIL model that make mutlithreaded jitted
+  code run *much* faster
+
+* Optimize errno handling in linux (x86 and x86-64 only)
+
+* Remove ctypes pythonapi and ctypes.PyDLL, which never worked on PyPy
+
+* Fix performance regression on ufunc(, ) in numpy
+
+* Classes in the ast module are now distinct from structures used by
+  the compiler, which simplifies and speeds up translation of our
+  source code to the PyPy binary interpreter
+
+* Upgrade stdlib from 2.7.5 to 2.7.8
+
+* Win32 now links statically to zlib, expat, bzip, and openssl-1.0.1i.
+  No more missing DLLs
+  
+* Many issues were resolved_ since the 2.3.1 release on June 8
+
+.. _`whats-new`: http://doc.pypy.org/en/latest/whatsnew-2.3.1.html
+.. _resolved: https://bitbucket.org/pypy/pypy/issues?status=resolved
+
+We have further improvements on the way: rpython file handling,
+numpy linalg compatibility, as well
+as improved GC and many smaller improvements.
+
+Please try it out and let us know what you think. We especially welcome
+success stories, we know you are using PyPy, please tell us about it!
+
+Cheers
+
+The PyPy Team
+
diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py
--- a/pypy/doc/tool/makecontributor.py
+++ b/pypy/doc/tool/makecontributor.py
@@ -67,6 +67,8 @@
     'Edd Barrett': ['edd'],
     'Manuel Jacob': ['mjacob'],
     'Rami Chowdhury': ['necaris'],
+    'Stanislaw Halik':['w31rd0'],
+    'Wenzhu Man':['wenzhu man', 'wenzhuman'],
     }
 
 alias_map = {}
diff --git a/pypy/doc/whatsnew-2.4.0.rst b/pypy/doc/whatsnew-2.4.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-2.4.0.rst
@@ -0,0 +1,66 @@
+=======================
+What's new in PyPy 2.4+
+=======================
+
+.. this is a revision shortly after release-2.3.x
+.. startrev: ca9b7cf02cf4
+
+.. branch: fix-bytearray-complexity
+Bytearray operations no longer copy the bytearray unnecessarily
+
+Added support for ``__getitem__``, ``__setitem__``, ``__getslice__``,
+``__setslice__``,  and ``__len__`` to RPython
+
+.. branch: stringbuilder2-perf
+Give the StringBuilder a more flexible internal structure, with a
+chained list of strings instead of just one string. This make it
+more efficient when building large strings, e.g. with cStringIO().
+
+Also, use systematically jit.conditional_call() instead of regular
+branches. This lets the JIT make more linear code, at the cost of
+forcing a bit more data (to be passed as arguments to
+conditional_calls). I would expect the net result to be a slight
+slow-down on some simple benchmarks and a speed-up on bigger
+programs.
+
+.. branch: ec-threadlocal
+Change the executioncontext's lookup to be done by reading a thread-
+local variable (which is implemented in C using '__thread' if
+possible, and pthread_getspecific() otherwise). On Linux x86 and
+x86-64, the JIT backend has a special optimization that lets it emit
+directly a single MOV from a %gs- or %fs-based address. It seems
+actually to give a good boost in performance.
+
+.. branch: fast-gil
+A faster way to handle the GIL, particularly in JIT code. The GIL is
+now a composite of two concepts: a global number (it's just set from
+1 to 0 and back around CALL_RELEASE_GIL), and a real mutex. If there
+are threads waiting to acquire the GIL, one of them is actively
+checking the global number every 0.1 ms to 1 ms.  Overall, JIT loops
+full of external function calls now run a bit faster (if no thread was
+started yet), or a *lot* faster (if threads were started already).
+
+.. branch: jit-get-errno
+Optimize the errno handling in the JIT, notably around external
+function calls. Linux-only.
+
+.. branch: disable_pythonapi
+Remove non-functioning ctypes.pyhonapi and ctypes.PyDLL, document this
+incompatibility with cpython. Recast sys.dllhandle to an int.
+
+.. branch: scalar-operations
+Fix performance regression on ufunc(, ) in numpy.
+
+.. branch: pytest-25
+Update our copies of py.test and pylib to versions 2.5.2 and 1.4.20, 
+respectively.
+
+.. branch: split-ast-classes
+Classes in the ast module are now distinct from structures used by the compiler.
+
+.. branch: stdlib-2.7.8
+Upgrades from 2.7.6 to 2.7.8
+
+.. branch: cpybug-seq-radd-rmul
+Fix issue #1861 - cpython compatability madness
+
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -1,59 +1,8 @@
+
 =======================
-What's new in PyPy 2.4+
+What's new in PyPy 2.5+
 =======================
 
-.. this is a revision shortly after release-2.3.x
-.. startrev: ca9b7cf02cf4
+.. this is a revision shortly after release-2.4.x
+.. startrev: 7026746cbb1b
 
-.. branch: fix-bytearray-complexity
-Bytearray operations no longer copy the bytearray unnecessarily
-
-Added support for ``__getitem__``, ``__setitem__``, ``__getslice__``,
-``__setslice__``,  and ``__len__`` to RPython
-
-.. branch: stringbuilder2-perf
-Give the StringBuilder a more flexible internal structure, with a
-chained list of strings instead of just one string. This make it
-more efficient when building large strings, e.g. with cStringIO().
-
-Also, use systematically jit.conditional_call() instead of regular
-branches. This lets the JIT make more linear code, at the cost of
-forcing a bit more data (to be passed as arguments to
-conditional_calls). I would expect the net result to be a slight
-slow-down on some simple benchmarks and a speed-up on bigger
-programs.
-
-.. branch: ec-threadlocal
-Change the executioncontext's lookup to be done by reading a thread-
-local variable (which is implemented in C using '__thread' if
-possible, and pthread_getspecific() otherwise). On Linux x86 and
-x86-64, the JIT backend has a special optimization that lets it emit
-directly a single MOV from a %gs- or %fs-based address. It seems
-actually to give a good boost in performance.
-
-.. branch: fast-gil
-A faster way to handle the GIL, particularly in JIT code. The GIL is
-now a composite of two concepts: a global number (it's just set from
-1 to 0 and back around CALL_RELEASE_GIL), and a real mutex. If there
-are threads waiting to acquire the GIL, one of them is actively
-checking the global number every 0.1 ms to 1 ms.  Overall, JIT loops
-full of external function calls now run a bit faster (if no thread was
-started yet), or a *lot* faster (if threads were started already).
-
-.. branch: jit-get-errno
-Optimize the errno handling in the JIT, notably around external
-function calls. Linux-only.
-
-.. branch: disable_pythonapi
-Remove non-functioning ctypes.pyhonapi and ctypes.PyDLL, document this
-incompatibility with cpython. Recast sys.dllhandle to an int.
-
-.. branch: scalar-operations
-Fix performance regression on ufunc(, ) in numpy.
-
-.. branch: pytest-25
-Update our copies of py.test and pylib to versions 2.5.2 and 1.4.20, 
-respectively.
-
-.. branch: split-ast-classes
-Classes in the ast module are now distinct from structures used by the compiler.
diff --git a/pypy/interpreter/mixedmodule.py b/pypy/interpreter/mixedmodule.py
--- a/pypy/interpreter/mixedmodule.py
+++ b/pypy/interpreter/mixedmodule.py
@@ -7,7 +7,6 @@
 
 class MixedModule(Module):
     applevel_name = None
-    expose__file__attribute = True
 
     # The following attribute is None as long as the module has not been
     # imported yet, and when it has been, it is mod.__dict__.items() just
@@ -144,8 +143,6 @@
             for name, spec in cls.appleveldefs.items():
                 loaders[name] = getappfileloader(pkgroot, appname, spec)
             assert '__file__' not in loaders
-            if cls.expose__file__attribute:
-                loaders['__file__'] = cls.get__file__
             if '__doc__' not in loaders:
                 loaders['__doc__'] = cls.get__doc__
 
@@ -159,28 +156,6 @@
         w_obj = loader(space)
         space.setattr(space.wrap(self), space.wrap(name), w_obj)
 
-    def get__file__(cls, space):
-        """ NOT_RPYTHON.
-        return the __file__ attribute of a MixedModule
-        which is the root-directory for the various
-        applevel and interplevel snippets that make
-        up the module.
-        """
-        try:
-            fname = cls._fname
-        except AttributeError:
-            pkgroot = cls.__module__
-            mod = __import__(pkgroot, None, None, ['__doc__'])
-            fname = mod.__file__
-            assert os.path.basename(fname).startswith('__init__.py')
-            # make it clear that it's not really the interp-level module
-            # at this path that we are seeing, but an app-level version of it
-            fname = os.path.dirname(fname)
-            cls._fname = fname
-        return space.wrap(fname)
-
-    get__file__ = classmethod(get__file__)
-
     def get__doc__(cls, space):
         return space.wrap(cls.__doc__)
     get__doc__ = classmethod(get__doc__)
diff --git a/pypy/interpreter/module.py b/pypy/interpreter/module.py
--- a/pypy/interpreter/module.py
+++ b/pypy/interpreter/module.py
@@ -29,6 +29,17 @@
                           space.w_None)
         self.startup_called = False
 
+    def _cleanup_(self):
+        """Called by the annotator on prebuilt Module instances.
+        We don't have many such modules, but for the ones that
+        show up, remove their __file__ rather than translate it
+        statically inside the executable."""
+        try:
+            space = self.space
+            space.delitem(self.w_dict, space.wrap('__file__'))
+        except OperationError:
+            pass
+
     def install(self):
         """NOT_RPYTHON: installs this module into space.builtin_modules"""
         w_mod = self.space.wrap(self)
diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -1104,6 +1104,11 @@
             assert sys.path == old_sys_path + [self.goal_dir]
 
             app_main.setup_bootstrap_path(self.fake_exe)
+            assert sys.executable == ''      # not executable!
+            assert sys.path == old_sys_path + [self.goal_dir]
+
+            os.chmod(self.fake_exe, 0755)
+            app_main.setup_bootstrap_path(self.fake_exe)
             assert sys.executable == self.fake_exe
             assert self.goal_dir not in sys.path
 
diff --git a/pypy/interpreter/test/test_extmodules.py b/pypy/interpreter/test/test_extmodules.py
--- a/pypy/interpreter/test/test_extmodules.py
+++ b/pypy/interpreter/test/test_extmodules.py
@@ -64,5 +64,5 @@
     @pytest.mark.skipif("config.option.runappdirect")
     def test_import(self):
         import extmod
-        assert extmod.__file__.endswith('extmod')
+        assert not hasattr(extmod, '__file__')
         assert type(extmod.time()) is float
diff --git a/pypy/interpreter/test/test_module.py b/pypy/interpreter/test/test_module.py
--- a/pypy/interpreter/test/test_module.py
+++ b/pypy/interpreter/test/test_module.py
@@ -1,4 +1,6 @@
 # encoding: utf-8
+import py
+from pypy.interpreter.error import OperationError
 from pypy.interpreter.module import Module
 
 class TestModule: 
@@ -17,6 +19,18 @@
         space.raises_w(space.w_AttributeError,
                        space.delattr, w_m, w('x'))
 
+    def test___file__(self, space):
+        w = space.wrap
+        m = Module(space, space.wrap('m'))
+        py.test.raises(OperationError, space.getattr, w(m), w('__file__'))
+        m._cleanup_()
+        py.test.raises(OperationError, space.getattr, w(m), w('__file__'))
+        space.setattr(w(m), w('__file__'), w('m.py'))
+        space.getattr(w(m), w('__file__'))   # does not raise
+        m._cleanup_()
+        py.test.raises(OperationError, space.getattr, w(m), w('__file__'))
+
+
 class AppTest_ModuleObject: 
     def test_attr(self):
         m = __import__('builtins')
@@ -42,12 +56,9 @@
         bar = type(sys)('bar','docstring')
         assert bar.__doc__ == 'docstring'
 
-    def test___file__(self): 
-        import sys, os
-        if not hasattr(sys, "pypy_objspaceclass"):
-            skip("need PyPy for sys.__file__ checking")
-        assert sys.__file__ 
-        assert os.path.basename(sys.__file__) == 'sys'
+    def test___file__(self):
+        import sys
+        assert not hasattr(sys, '__file__')
 
     def test_repr(self):
         import sys
diff --git a/pypy/interpreter/test/test_typedef.py b/pypy/interpreter/test/test_typedef.py
--- a/pypy/interpreter/test/test_typedef.py
+++ b/pypy/interpreter/test/test_typedef.py
@@ -385,6 +385,7 @@
         assert bm.__self__ is obj
         assert bm.__doc__ == "aaa"
         assert bm.x == 3
+        assert type(bm).__doc__ == "instancemethod(function, instance, class)\n\nCreate an instance method object."
         raises(AttributeError, setattr, bm, 'x', 15)
         l = []
         assert l.append.__self__ is l
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -837,6 +837,9 @@
 
 Method.typedef = TypeDef(
     "method",
+    __doc__ = """instancemethod(function, instance, class)
+
+Create an instance method object.""",
     __new__ = interp2app(Method.descr_method__new__.im_func),
     __call__ = interp2app(Method.descr_method_call),
     __get__ = interp2app(Method.descr_method_get),
diff --git a/pypy/module/__builtin__/__init__.py b/pypy/module/__builtin__/__init__.py
--- a/pypy/module/__builtin__/__init__.py
+++ b/pypy/module/__builtin__/__init__.py
@@ -9,8 +9,6 @@
     """Built-in functions, exceptions, and other objects."""
     applevel_name = 'builtins'
 
-    expose__file__attribute = False
-
     appleveldefs = {
         'input'         : 'app_io.input',
         'print'         : 'app_io.print_',
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -182,8 +182,12 @@
             value = misc.read_raw_long_data(cdata, self.size)
             return self.space.wrap(value)
         else:
-            value = misc.read_raw_signed_data(cdata, self.size)
-            return self.space.wrap(value)    # r_longlong => on 32-bit, 'long'
+            return self._convert_to_object_longlong(cdata)
+
+    def _convert_to_object_longlong(self, cdata):
+        # in its own function: LONGLONG may make the whole function jit-opaque
+        value = misc.read_raw_signed_data(cdata, self.size)
+        return self.space.wrap(value)    # r_longlong => on 32-bit, 'long'
 
     def convert_from_object(self, cdata, w_ob):
         if self.value_fits_long:
@@ -193,8 +197,12 @@
                     self._overflow(w_ob)
             misc.write_raw_signed_data(cdata, value, self.size)
         else:
-            value = misc.as_long_long(self.space, w_ob)
-            misc.write_raw_signed_data(cdata, value, self.size)
+            self._convert_from_object_longlong(cdata, w_ob)
+
+    def _convert_from_object_longlong(self, cdata, w_ob):
+        # in its own function: LONGLONG may make the whole function jit-opaque
+        value = misc.as_long_long(self.space, w_ob)
+        misc.write_raw_signed_data(cdata, value, self.size)
 
     def get_vararg_type(self):
         if self.size < rffi.sizeof(rffi.INT):
@@ -264,8 +272,12 @@
                     self._overflow(w_ob)
             misc.write_raw_unsigned_data(cdata, value, self.size)
         else:
-            value = misc.as_unsigned_long_long(self.space, w_ob, strict=True)
-            misc.write_raw_unsigned_data(cdata, value, self.size)
+            self._convert_from_object_longlong(cdata, w_ob)
+
+    def _convert_from_object_longlong(self, cdata, w_ob):
+        # in its own function: LONGLONG may make the whole function jit-opaque
+        value = misc.as_unsigned_long_long(self.space, w_ob, strict=True)
+        misc.write_raw_unsigned_data(cdata, value, self.size)
 
     def convert_to_object(self, cdata):
         if self.value_fits_ulong:
@@ -275,8 +287,12 @@
             else:
                 return self.space.wrap(value)    # r_uint => 'long' object
         else:
-            value = misc.read_raw_unsigned_data(cdata, self.size)
-            return self.space.wrap(value)    # r_ulonglong => 'long' object
+            return self._convert_to_object_longlong(cdata)
+
+    def _convert_to_object_longlong(self, cdata):
+        # in its own function: LONGLONG may make the whole function jit-opaque
+        value = misc.read_raw_unsigned_data(cdata, self.size)
+        return self.space.wrap(value)    # r_ulonglong => 'long' object
 
     def get_vararg_type(self):
         if self.size < rffi.sizeof(rffi.INT):
diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py
--- a/pypy/module/_cffi_backend/ctypestruct.py
+++ b/pypy/module/_cffi_backend/ctypestruct.py
@@ -17,7 +17,7 @@
 
 
 class W_CTypeStructOrUnion(W_CType):
-    _immutable_fields_ = ['alignment?', 'fields_list?', 'fields_dict?',
+    _immutable_fields_ = ['alignment?', 'fields_list?[*]', 'fields_dict?',
                           'custom_field_pos?', 'with_var_array?']
     # fields added by complete_struct_or_union():
     alignment = -1
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -389,7 +389,7 @@
 
     w_ctype.size = totalsize
     w_ctype.alignment = totalalignment
-    w_ctype.fields_list = fields_list
+    w_ctype.fields_list = fields_list[:]
     w_ctype.fields_dict = fields_dict
     w_ctype.custom_field_pos = custom_field_pos
     w_ctype.with_var_array = with_var_array
diff --git a/pypy/module/_io/interp_stringio.py b/pypy/module/_io/interp_stringio.py
--- a/pypy/module/_io/interp_stringio.py
+++ b/pypy/module/_io/interp_stringio.py
@@ -86,7 +86,7 @@
         initval = space.unicode_w(w_initval)
         size = len(initval)
         self.resize_buffer(size)
-        self.buf = [c for c in initval]
+        self.buf = list(initval)
         pos = space.getindex_w(w_pos, space.w_TypeError)
         if pos < 0:
             raise OperationError(space.w_ValueError,
diff --git a/pypy/module/_pypyjson/__init__.py b/pypy/module/_pypyjson/__init__.py
--- a/pypy/module/_pypyjson/__init__.py
+++ b/pypy/module/_pypyjson/__init__.py
@@ -7,4 +7,6 @@
 
     interpleveldefs = {
         'loads' : 'interp_decoder.loads',
+        'raw_encode_basestring_ascii':
+            'interp_encoder.raw_encode_basestring_ascii',
         }
diff --git a/pypy/module/_pypyjson/interp_encoder.py b/pypy/module/_pypyjson/interp_encoder.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_pypyjson/interp_encoder.py
@@ -0,0 +1,82 @@
+from rpython.rlib.rstring import StringBuilder
+from rpython.rlib.runicode import str_decode_utf_8
+from pypy.interpreter import unicodehelper
+
+
+HEX = '0123456789abcdef'
+
+ESCAPE_DICT = {
+    '\b': '\\b',
+    '\f': '\\f',
+    '\n': '\\n',
+    '\r': '\\r',
+    '\t': '\\t',
+}
+ESCAPE_BEFORE_SPACE = [ESCAPE_DICT.get(chr(_i), '\\u%04x' % _i)
+                       for _i in range(32)]
+
+
+def raw_encode_basestring_ascii(space, w_string):
+    if space.isinstance_w(w_string, space.w_str):
+        s = space.str_w(w_string)
+        for i in range(len(s)):
+            c = s[i]
+            if c >= ' ' and c <= '~' and c != '"' and c != '\\':
+                pass
+            else:
+                first = i
+                break
+        else:
+            # the input is a string with only non-special ascii chars
+            return w_string
+
+        eh = unicodehelper.decode_error_handler(space)
+        u = str_decode_utf_8(
+                s, len(s), None, final=True, errorhandler=eh,
+                allow_surrogates=True)[0]
+        sb = StringBuilder(len(u))
+        sb.append_slice(s, 0, first)
+    else:
+        # We used to check if 'u' contains only safe characters, and return
+        # 'w_string' directly.  But this requires an extra pass over all
+        # characters, and the expected use case of this function, from
+        # json.encoder, will anyway re-encode a unicode result back to
+        # a string (with the ascii encoding).  This requires two passes
+        # over the characters.  So we may as well directly turn it into a
+        # string here --- only one pass.
+        u = space.unicode_w(w_string)
+        sb = StringBuilder(len(u))
+        first = 0
+
+    for i in range(first, len(u)):
+        c = u[i]
+        if c <= u'~':
+            if c == u'"' or c == u'\\':
+                sb.append('\\')
+            elif c < u' ':
+                sb.append(ESCAPE_BEFORE_SPACE[ord(c)])
+                continue
+            sb.append(chr(ord(c)))
+        else:
+            if c <= u'\uffff':
+                sb.append('\\u')
+                sb.append(HEX[ord(c) >> 12])
+                sb.append(HEX[(ord(c) >> 8) & 0x0f])
+                sb.append(HEX[(ord(c) >> 4) & 0x0f])
+                sb.append(HEX[ord(c) & 0x0f])
+            else:
+                # surrogate pair
+                n = ord(c) - 0x10000
+                s1 = 0xd800 | ((n >> 10) & 0x3ff)
+                sb.append('\\ud')
+                sb.append(HEX[(s1 >> 8) & 0x0f])
+                sb.append(HEX[(s1 >> 4) & 0x0f])
+                sb.append(HEX[s1 & 0x0f])
+                s2 = 0xdc00 | (n & 0x3ff)
+                sb.append('\\ud')
+                sb.append(HEX[(s2 >> 8) & 0x0f])
+                sb.append(HEX[(s2 >> 4) & 0x0f])
+                sb.append(HEX[s2 & 0x0f])
+
+    res = sb.build()
+    return space.wrap(res)
diff --git a/pypy/module/_pypyjson/test/test__pypyjson.py b/pypy/module/_pypyjson/test/test__pypyjson.py
--- a/pypy/module/_pypyjson/test/test__pypyjson.py
+++ b/pypy/module/_pypyjson/test/test__pypyjson.py
@@ -189,6 +189,25 @@
         s = '["\ttab\tcharacter\tin\tstring\t"]'
         raises(ValueError, "_pypyjson.loads(s)")
 
+    def test_raw_encode_basestring_ascii(self):
+        import _pypyjson
+        def check(s):
+            s = _pypyjson.raw_encode_basestring_ascii(s)
+            assert type(s) is str
+            return s
+        assert check("") == ""
+        assert check(u"") == ""
+        assert check("abc ") == "abc "
+        assert check(u"abc ") == "abc "
+        raises(UnicodeDecodeError, check, "\xc0")
+        assert check("\xc2\x84") == "\\u0084"
+        assert check("\xf0\x92\x8d\x85") == "\\ud808\\udf45"
+        assert check(u"\ud808\udf45") == "\\ud808\\udf45"
+        assert check(u"\U00012345") == "\\ud808\\udf45"
+        assert check("a\"c") == "a\\\"c"
+        assert check("\\\"\b\f\n\r\t") == '\\\\\\"\\b\\f\\n\\r\\t'
+        assert check("\x07") == "\\u0007"
+
     def test_keys_reuse(self):
         import _pypyjson
         s = '[{"a_key": 1, "b_\xe9": 2}, {"a_key": 3, "b_\xe9": 4}]'
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -907,17 +907,25 @@
 
         # socket's timeout is in seconds, poll's timeout in ms
         timeout = int(sock_timeout * 1000 + 0.5)
-        ready = rpoll.poll(fddict, timeout)
+        try:
+            ready = rpoll.poll(fddict, timeout)
+        except rpoll.PollError, e:
+            message = e.get_msg()
+            raise ssl_error(space, message, e.errno)
     else:
         if MAX_FD_SIZE is not None and sock_fd >= MAX_FD_SIZE:
             return SOCKET_TOO_LARGE_FOR_SELECT
 
-        if writing:
-            r, w, e = rpoll.select([], [sock_fd], [], sock_timeout)
-            ready = w
-        else:
-            r, w, e = rpoll.select([sock_fd], [], [], sock_timeout)
-            ready = r
+        try:
+            if writing:
+                r, w, e = rpoll.select([], [sock_fd], [], sock_timeout)
+                ready = w
+            else:
+                r, w, e = rpoll.select([sock_fd], [], [], sock_timeout)
+                ready = r
+        except SelectError, e:
+            message = e.get_msg()
+            raise ssl_error(space, message, e.errno)
     if ready:
         return SOCKET_OPERATION_OK
     else:
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,7 +29,7 @@
 #define PY_VERSION		"3.2.5"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "2.4.0-alpha0"
+#define PYPY_VERSION "2.5.0-alpha0"
 
 /* Subversion Revision number of this file (not of the repository).
  * Empty since Mercurial migration. */
diff --git a/pypy/module/errno/test/test_errno.py b/pypy/module/errno/test/test_errno.py
--- a/pypy/module/errno/test/test_errno.py
+++ b/pypy/module/errno/test/test_errno.py
@@ -8,7 +8,7 @@
         cls.w_errorcode = cls.space.wrap(errno.errorcode)
 
     def test_posix(self):
-        assert self.errno.__file__
+        assert not hasattr(self.errno, '__file__')
 
     def test_constants(self):
         for code, name in self.errorcode.items():
diff --git a/pypy/module/operator/__init__.py b/pypy/module/operator/__init__.py
--- a/pypy/module/operator/__init__.py
+++ b/pypy/module/operator/__init__.py
@@ -35,7 +35,7 @@
                     'indexOf']
 
     interpleveldefs = {
-        '_compare_digest': 'interp_operator.compare_digest',
+        '_compare_digest': 'tscmp.compare_digest',
     }
 
     for name in interp_names:
diff --git a/pypy/module/operator/interp_operator.py b/pypy/module/operator/interp_operator.py
--- a/pypy/module/operator/interp_operator.py
+++ b/pypy/module/operator/interp_operator.py
@@ -1,6 +1,4 @@
-from rpython.rlib.objectmodel import specialize
-
-from pypy.interpreter.error import OperationError, oefmt
+from pypy.interpreter.error import OperationError
 from pypy.interpreter.gateway import unwrap_spec
 
 
@@ -208,33 +206,3 @@
 @unwrap_spec(default=int)
 def _length_hint(space, w_iterable, default):
     return space.wrap(space.length_hint(w_iterable, default))
-
-def compare_digest(space, w_a, w_b):
-    if (
-        space.isinstance_w(w_a, space.w_unicode) and
-        space.isinstance_w(w_b, space.w_unicode)
-    ):
-        return space.wrap(tscmp(space.unicode_w(w_a), space.unicode_w(w_b)))
-    if (
-        space.isinstance_w(w_a, space.w_unicode) or
-        space.isinstance_w(w_b, space.w_unicode)
-    ):

From noreply at buildbot.pypy.org  Thu Sep 11 22:50:38 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu, 11 Sep 2014 22:50:38 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: fixes for rfile
	setbufsize
Message-ID: <20140911205038.30B2C1C31C8@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73482:aa030b8c29a6
Date: 2014-09-11 16:49 -0400
http://bitbucket.org/pypy/pypy/changeset/aa030b8c29a6/

Log:	fixes for rfile setbufsize

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -73,9 +73,6 @@
                       [rffi.INT, rffi.CCHARP], FILEP)
 c_tmpfile = llexternal('tmpfile', [], FILEP)
 
-c_setvbuf = llexternal('setvbuf', [FILEP, rffi.CCHARP, rffi.INT, rffi.SIZE_T],
-                       rffi.INT)
-
 # Note: the following functions are called from __del__ methods,
 # so must be 'releasegil=False'.  Otherwise, a program using both
 # threads and the RFile class cannot translate.  See c684bf704d1f
@@ -110,6 +107,8 @@
 c_feof = llexternal('feof', [FILEP], rffi.INT, releasegil=False)
 c_ferror = llexternal('ferror', [FILEP], rffi.INT, releasegil=False)
 c_clearerr = llexternal('clearerr', [FILEP], lltype.Void, releasegil=False)
+c_setvbuf = llexternal('setvbuf', [FILEP, rffi.CCHARP, rffi.INT, rffi.SIZE_T],
+                       rffi.INT, releasegil=False)
 
 c_stdin = llexternal('get_stdin', [], FILEP, _nowrapper=True)
 c_stdout = llexternal('get_stdout', [], FILEP, _nowrapper=True)
@@ -254,6 +253,7 @@
                 bufsize = BUFSIZ
             else:
                 mode = _IOFBF
+            c_fflush_nogil(self._ll_file)
             if self._setbuf:
                 lltype.free(self._setbuf, flavor='raw')
             if mode == _IONBF:

From noreply at buildbot.pypy.org  Thu Sep 11 23:08:10 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Thu, 11 Sep 2014 23:08:10 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix untested commit to default
Message-ID: <20140911210810.4E6DB1C31C8@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: 
Changeset: r73483:e4b10708718c
Date: 2014-09-12 00:07 +0300
http://bitbucket.org/pypy/pypy/changeset/e4b10708718c/

Log:	fix untested commit to default

diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -775,7 +775,7 @@
             else:
                 r, w, e = rpoll.select([sock_fd], [], [], sock_timeout)
                 ready = r
-        except SelectError, e:
+        except rpoll.SelectError as e:
             message = e.get_msg()
             raise ssl_error(space, message, e.errno)
     if ready:

From noreply at buildbot.pypy.org  Thu Sep 11 23:14:07 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu, 11 Sep 2014 23:14:07 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: test/fix truncate on
	win32
Message-ID: <20140911211407.094511C31C8@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73484:e8ce802885c3
Date: 2014-09-11 14:12 -0700
http://bitbucket.org/pypy/pypy/changeset/e8ce802885c3/

Log:	test/fix truncate on win32

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -107,6 +107,7 @@
 c_feof = llexternal('feof', [FILEP], rffi.INT, releasegil=False)
 c_ferror = llexternal('ferror', [FILEP], rffi.INT, releasegil=False)
 c_clearerr = llexternal('clearerr', [FILEP], lltype.Void, releasegil=False)
+
 c_setvbuf = llexternal('setvbuf', [FILEP, rffi.CCHARP, rffi.INT, rffi.SIZE_T],
                        rffi.INT, releasegil=False)
 
@@ -568,13 +569,24 @@
     def truncate(self, arg=-1):
         self._check_closed()
         self._check_writable()
+        pos = intmask(c_ftell(self._ll_file))
+        if pos == -1:
+            c_clearerr(self._ll_file)
+            raise _from_errno(IOError)
         if arg == -1:
-            arg = self.tell()
-        self.flush()
+            arg = pos
+        res = c_fflush(self._ll_file)
+        if res != 0:
+            c_clearerr(self._ll_file)
+            raise _from_errno(IOError)
         res = c_ftruncate(self.fileno(), arg)
         if res != 0:
             c_clearerr(self._ll_file)
             raise _from_errno(IOError)
+        res = c_fseek(self._ll_file, pos, os.SEEK_SET)
+        if res != 0:
+            c_clearerr(self._ll_file)
+            raise _from_errno(IOError)
 
     def flush(self):
         self._check_closed()
diff --git a/rpython/rlib/test/test_rfile.py b/rpython/rlib/test/test_rfile.py
--- a/rpython/rlib/test/test_rfile.py
+++ b/rpython/rlib/test/test_rfile.py
@@ -365,11 +365,14 @@
         def f():
             f = open(fname, "w+b")
             f.write("hello world")
-            f.seek(7)
+            f.seek(6)
+            assert f.read(1) == 'w'
             f.truncate()
+            assert f.read(1) == ''
+            f.write('w')
             f.seek(0)
             data = f.read()
-            assert data == "hello w"
+            assert data == "hello ww"
             f.close()
             f = open(fname)
             try:

From noreply at buildbot.pypy.org  Thu Sep 11 23:20:42 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu, 11 Sep 2014 23:20:42 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: cleanup
Message-ID: <20140911212042.50D4A1C31C8@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73485:bc60013638fd
Date: 2014-09-11 17:18 -0400
http://bitbucket.org/pypy/pypy/changeset/bc60013638fd/

Log:	cleanup

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -554,14 +554,15 @@
     def write(self, value):
         self._check_closed()
         self._check_writable()
+        ll_file = self._ll_file
         ll_value = rffi.get_nonmovingbuffer(value)
         try:
             # note that since we got a nonmoving buffer, it is either raw
             # or already cannot move, so the arithmetics below are fine
             n = len(value)
-            n2 = c_fwrite(ll_value, 1, n, self._ll_file)
-            if n2 != n or c_ferror(self._ll_file):
-                c_clearerr(self._ll_file)
+            n2 = c_fwrite(ll_value, 1, n, ll_file)
+            if n2 != n or c_ferror(ll_file):
+                c_clearerr(ll_file)
                 raise _from_errno(IOError)
         finally:
             rffi.free_nonmovingbuffer(value, ll_value)
@@ -569,23 +570,24 @@
     def truncate(self, arg=-1):
         self._check_closed()
         self._check_writable()
-        pos = intmask(c_ftell(self._ll_file))
+        ll_file = self._ll_file
+        pos = intmask(c_ftell(ll_file))
         if pos == -1:
-            c_clearerr(self._ll_file)
+            c_clearerr(ll_file)
             raise _from_errno(IOError)
         if arg == -1:
             arg = pos
-        res = c_fflush(self._ll_file)
+        res = c_fflush(ll_file)
         if res != 0:
-            c_clearerr(self._ll_file)
+            c_clearerr(ll_file)
             raise _from_errno(IOError)
-        res = c_ftruncate(self.fileno(), arg)
+        res = c_ftruncate(c_fileno(ll_file), arg)
         if res != 0:
-            c_clearerr(self._ll_file)
+            c_clearerr(ll_file)
             raise _from_errno(IOError)
-        res = c_fseek(self._ll_file, pos, os.SEEK_SET)
+        res = c_fseek(ll_file, pos, os.SEEK_SET)
         if res != 0:
-            c_clearerr(self._ll_file)
+            c_clearerr(ll_file)
             raise _from_errno(IOError)
 
     def flush(self):

From noreply at buildbot.pypy.org  Fri Sep 12 00:15:19 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Fri, 12 Sep 2014 00:15:19 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: merge py3k
Message-ID: <20140911221519.CB6EE1D2D6C@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey 
Branch: py3.3
Changeset: r73486:d25d0c52e0f8
Date: 2014-09-11 14:57 -0700
http://bitbucket.org/pypy/pypy/changeset/d25d0c52e0f8/

Log:	merge py3k

diff too long, truncating to 2000 out of 41356 lines

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -35,280 +35,290 @@
 the beginning of each file) the files in the 'pypy' directory are each
 copyrighted by one or more of the following people and organizations:    
 
-    Armin Rigo
-    Maciej Fijalkowski
-    Carl Friedrich Bolz
-    Antonio Cuni
-    Amaury Forgeot d'Arc
-    Samuele Pedroni
-    Alex Gaynor
-    Michael Hudson
-    David Schneider
-    Matti Picus
-    Brian Kearns
-    Philip Jenvey
-    Holger Krekel
-    Christian Tismer
-    Hakan Ardo
-    Benjamin Peterson
-    Manuel Jacob
-    Anders Chrigstrom
-    Eric van Riet Paap
-    Wim Lavrijsen
-    Ronan Lamy
-    Richard Emslie
-    Alexander Schremmer
-    Dan Villiom Podlaski Christiansen
-    Lukas Diekmann
-    Sven Hager
-    Anders Lehmann
-    Aurelien Campeas
-    Niklaus Haldimann
-    Camillo Bruni
-    Laura Creighton
-    Toon Verwaest
-    Remi Meier
-    Leonardo Santagada
-    Seo Sanghyeon
-    Romain Guillebert
-    Justin Peel
-    Ronny Pfannschmidt
-    David Edelsohn
-    Anders Hammarquist
-    Jakub Gustak
-    Guido Wesdorp
-    Lawrence Oluyede
-    Bartosz Skowron
-    Daniel Roberts
-    Niko Matsakis
-    Adrien Di Mascio
-    Alexander Hesse
-    Ludovic Aubry
-    Jacob Hallen
-    Jason Creighton
-    Alex Martelli
-    Michal Bendowski
-    Jan de Mooij
-    stian
-    Michael Foord
-    Stephan Diehl
-    Stefan Schwarzer
-    Valentino Volonghi
-    Tomek Meka
-    Patrick Maupin
-    Bob Ippolito
-    Bruno Gola
-    Jean-Paul Calderone
-    Timo Paulssen
-    Squeaky
-    Alexandre Fayolle
-    Simon Burton
-    Marius Gedminas
-    John Witulski
-    Konstantin Lopuhin
-    Greg Price
-    Dario Bertini
-    Mark Pearse
-    Simon Cross
-    Andreas Stührk
-    Jean-Philippe St. Pierre
-    Guido van Rossum
-    Pavel Vinogradov
-    Paweł Piotr Przeradowski
-    Paul deGrandis
-    Ilya Osadchiy
-    Tobias Oberstein
-    Adrian Kuhn
-    Boris Feigin
-    Stefano Rivera
-    tav
-    Taavi Burns
-    Georg Brandl
-    Bert Freudenberg
-    Stian Andreassen
-    Laurence Tratt
-    Wanja Saatkamp
-    Ivan Sichmann Freitas
-    Gerald Klix
-    Mike Blume
-    Oscar Nierstrasz
-    Stefan H. Muller
-    Jeremy Thurgood
-    Gregor Wegberg
-    Rami Chowdhury
-    Tobias Pape
-    Edd Barrett
-    David Malcolm
-    Eugene Oden
-    Henry Mason
-    Preston Timmons
-    Jeff Terrace
-    David Ripton
-    Dusty Phillips
-    Lukas Renggli
-    Guenter Jantzen
-    Ned Batchelder
-    Amit Regmi
-    Ben Young
-    Nicolas Chauvat
-    Andrew Durdin
-    Andrew Chambers
-    Michael Schneider
-    Nicholas Riley
-    Jason Chu
-    Igor Trindade Oliveira
-    Rocco Moretti
-    Gintautas Miliauskas
-    Michael Twomey
-    Lucian Branescu Mihaila
-    Tim Felgentreff
-    Tyler Wade
-    Gabriel Lavoie
-    Olivier Dormond
-    Jared Grubb
-    Karl Bartel
-    Brian Dorsey
-    Victor Stinner
-    Andrews Medina
-    Stuart Williams
-    Jasper Schulz
-    Christian Hudon
-    Toby Watson
-    Antoine Pitrou
-    Aaron Iles
-    Michael Cheng
-    Justas Sadzevicius
-    Mikael Schönenberg
-    Gasper Zejn
-    Neil Shepperd
-    Elmo Mäntynen
-    Jonathan David Riehl
-    Stanislaw Halik
-    Anders Qvist
-    Chirag Jadwani
-    Beatrice During
-    Alex Perry
-    Vincent Legoll
-    Alan McIntyre
-    Alexander Sedov
-    Corbin Simpson
-    Christopher Pope
-    wenzhuman
-    Christian Tismer 
-    Marc Abramowitz
-    Dan Stromberg
-    Stefano Parmesan
-    Alexis Daboville
-    Jens-Uwe Mager
-    Carl Meyer
-    Karl Ramm
-    Pieter Zieschang
-    Gabriel
-    Lukas Vacek
-    Andrew Dalke
-    Sylvain Thenault
-    Nathan Taylor
-    Vladimir Kryachko
-    Jacek Generowicz
-    Alejandro J. Cura
-    Jacob Oscarson
-    Travis Francis Athougies
-    Ryan Gonzalez
-    Kristjan Valur Jonsson
-    Sebastian Pawluś
-    Neil Blakey-Milner
-    anatoly techtonik
-    Lutz Paelike
-    Lucio Torre
-    Lars Wassermann
-    Henrik Vendelbo
-    Dan Buch
-    Miguel de Val Borro
-    Artur Lisiecki
-    Sergey Kishchenko
-    Ignas Mikalajunas
-    Christoph Gerum
-    Martin Blais
-    Lene Wagner
-    Tomo Cocoa
-    roberto at goyle
-    Yury V. Zaytsev
-    Anna Katrina Dominguez
-    William Leslie
-    Bobby Impollonia
-    timo at eistee.fritz.box
-    Andrew Thompson
-    Ben Darnell
-    Roberto De Ioris
-    Juan Francisco Cantero Hurtado
-    Godefroid Chappelle
-    Joshua Gilbert
-    Dan Colish
-    Christopher Armstrong
-    Michael Hudson-Doyle
-    Anders Sigfridsson
-    Yasir Suhail
-    rafalgalczynski at gmail.com
-    Floris Bruynooghe
-    Laurens Van Houtven
-    Akira Li
-    Gustavo Niemeyer
-    Stephan Busemann
-    Rafał Gałczyński
-    Yusei Tahara
-    Christian Muirhead
-    James Lan
-    shoma hosaka
-    Daniel Neuh?user
-    Matthew Miller
-    Buck Golemon
-    Konrad Delong
-    Dinu Gherman
-    Chris Lambacher
-    coolbutuseless at gmail.com
-    Rodrigo Araújo
-    w31rd0
-    Jim Baker
-    James Robert
-    Armin Ronacher
-    Brett Cannon
-    yrttyr
-    aliceinwire
-    OlivierBlanvillain
-    Zooko Wilcox-O Hearn
-    Tomer Chachamu
-    Christopher Groskopf
-    Asmo Soinio
-    Stefan Marr
-    jiaaro
-    opassembler.py
-    Antony Lee
-    Jim Hunziker
-    Markus Unterwaditzer
-    Even Wiik Thomassen
-    jbs
-    soareschen
-    Kurt Griffiths
-    Mike Bayer
-    Flavio Percoco
-    Kristoffer Kleine
-    yasirs
-    Michael Chermside
-    Anna Ravencroft
-    Julien Phalip
-    Dan Loewenherz
+  Armin Rigo
+  Maciej Fijalkowski
+  Carl Friedrich Bolz
+  Antonio Cuni
+  Amaury Forgeot d'Arc
+  Samuele Pedroni
+  Alex Gaynor
+  Michael Hudson
+  David Schneider
+  Matti Picus
+  Brian Kearns
+  Philip Jenvey
+  Holger Krekel
+  Christian Tismer
+  Hakan Ardo
+  Benjamin Peterson
+  Manuel Jacob
+  Anders Chrigstrom
+  Eric van Riet Paap
+  Ronan Lamy
+  Wim Lavrijsen
+  Richard Emslie
+  Alexander Schremmer
+  Dan Villiom Podlaski Christiansen
+  Lukas Diekmann
+  Sven Hager
+  Anders Lehmann
+  Aurelien Campeas
+  Niklaus Haldimann
+  Remi Meier
+  Camillo Bruni
+  Laura Creighton
+  Toon Verwaest
+  Leonardo Santagada
+  Seo Sanghyeon
+  Romain Guillebert
+  Justin Peel
+  Ronny Pfannschmidt
+  David Edelsohn
+  Anders Hammarquist
+  Jakub Gustak
+  Guido Wesdorp
+  Lawrence Oluyede
+  Bartosz Skowron
+  Gregor Wegberg
+  Daniel Roberts
+  Niko Matsakis
+  Adrien Di Mascio
+  Alexander Hesse
+  Ludovic Aubry
+  Jacob Hallen
+  Jason Creighton
+  Alex Martelli
+  Michal Bendowski
+  Jan de Mooij
+  stian
+  Michael Foord
+  Stephan Diehl
+  Tyler Wade
+  Stefan Schwarzer
+  Valentino Volonghi
+  Tomek Meka
+  Patrick Maupin
+  Bob Ippolito
+  Bruno Gola
+  Jean-Paul Calderone
+  Timo Paulssen
+  Squeaky
+  Alexandre Fayolle
+  Simon Burton
+  Marius Gedminas
+  Martin Matusiak
+  Konstantin Lopuhin
+  John Witulski
+  Wenzhu Man
+  Greg Price
+  Dario Bertini
+  Mark Pearse
+  Simon Cross
+  Ivan Sichmann Freitas
+  Andreas Stührk
+  Jean-Philippe St. Pierre
+  Guido van Rossum
+  Pavel Vinogradov
+  Stefano Rivera
+  Paweł Piotr Przeradowski
+  Paul deGrandis
+  Ilya Osadchiy
+  Tobias Oberstein
+  Adrian Kuhn
+  Boris Feigin
+  tav
+  Taavi Burns
+  Georg Brandl
+  Laurence Tratt
+  Bert Freudenberg
+  Stian Andreassen
+  Wanja Saatkamp
+  Gerald Klix
+  Mike Blume
+  Oscar Nierstrasz
+  Stefan H. Muller
+  Edd Barrett
+  Jeremy Thurgood
+  Rami Chowdhury
+  Tobias Pape
+  David Malcolm
+  Eugene Oden
+  Henry Mason
+  Vasily Kuznetsov
+  Preston Timmons
+  Jeff Terrace
+  David Ripton
+  Dusty Phillips
+  Lukas Renggli
+  Guenter Jantzen
+  Ned Batchelder
+  Amit Regmi
+  Ben Young
+  Nicolas Chauvat
+  Andrew Durdin
+  Andrew Chambers
+  Michael Schneider
+  Nicholas Riley
+  Jason Chu
+  Igor Trindade Oliveira
+  Tim Felgentreff
+  Rocco Moretti
+  Gintautas Miliauskas
+  Michael Twomey
+  Lucian Branescu Mihaila
+  Gabriel Lavoie
+  Olivier Dormond
+  Jared Grubb
+  Karl Bartel
+  Brian Dorsey
+  Victor Stinner
+  Andrews Medina
+  Stuart Williams
+  Jasper Schulz
+  Christian Hudon
+  Toby Watson
+  Antoine Pitrou
+  Aaron Iles
+  Michael Cheng
+  Justas Sadzevicius
+  Gasper Zejn
+  anatoly techtonik
+  Neil Shepperd
+  Mikael Schönenberg
+  Elmo M?ntynen
+  Jonathan David Riehl
+  Stanislaw Halik
+  Anders Qvist
+  Corbin Simpson
+  Chirag Jadwani
+  Beatrice During
+  Alex Perry
+  Vincent Legoll
+  Alan McIntyre
+  Alexander Sedov
+  Christopher Pope
+  Christian Tismer 
+  Marc Abramowitz
+  Dan Stromberg
+  Stefano Parmesan
+  Alexis Daboville
+  Jens-Uwe Mager
+  Carl Meyer
+  Karl Ramm
+  Pieter Zieschang
+  Sebastian Pawluś
+  Gabriel
+  Lukas Vacek
+  Andrew Dalke
+  Sylvain Thenault
+  Nathan Taylor
+  Vladimir Kryachko
+  Arjun Naik
+  Attila Gobi
+  Jacek Generowicz
+  Alejandro J. Cura
+  Jacob Oscarson
+  Travis Francis Athougies
+  Ryan Gonzalez
+  Ian Foote
+  Kristjan Valur Jonsson
+  Neil Blakey-Milner
+  Lutz Paelike
+  Lucio Torre
+  Lars Wassermann
+  Valentina Mukhamedzhanova
+  Henrik Vendelbo
+  Dan Buch
+  Miguel de Val Borro
+  Artur Lisiecki
+  Sergey Kishchenko
+  Yichao Yu
+  Ignas Mikalajunas
+  Christoph Gerum
+  Martin Blais
+  Lene Wagner
+  Tomo Cocoa
+  roberto at goyle
+  Yury V. Zaytsev
+  Anna Katrina Dominguez
+  William Leslie
+  Bobby Impollonia
+  timo at eistee.fritz.box
+  Andrew Thompson
+  Yusei Tahara
+  Ben Darnell
+  Roberto De Ioris
+  Juan Francisco Cantero Hurtado
+  Godefroid Chappelle
+  Joshua Gilbert
+  Dan Colish
+  Christopher Armstrong
+  Michael Hudson-Doyle
+  Anders Sigfridsson
+  Yasir Suhail
+  Jason Michalski
+  rafalgalczynski at gmail.com
+  Floris Bruynooghe
+  Laurens Van Houtven
+  Akira Li
+  Gustavo Niemeyer
+  Stephan Busemann
+  Rafał Gałczyński
+  Christian Muirhead
+  James Lan
+  shoma hosaka
+  Daniel Neuh?user
+  Matthew Miller
+  Buck Golemon
+  Konrad Delong
+  Dinu Gherman
+  Chris Lambacher
+  coolbutuseless at gmail.com
+  Rodrigo Araújo
+  Jim Baker
+  James Robert
+  Armin Ronacher
+  Brett Cannon
+  yrttyr
+  aliceinwire
+  OlivierBlanvillain
+  Zooko Wilcox-O Hearn
+  Tomer Chachamu
+  Christopher Groskopf
+  Asmo Soinio
+  Stefan Marr
+  jiaaro
+  Mads Kiilerich
+  opassembler.py
+  Antony Lee
+  Jim Hunziker
+  Markus Unterwaditzer
+  Even Wiik Thomassen
+  jbs
+  soareschen
+  Kurt Griffiths
+  Mike Bayer
+  Matthew Miller
+  Flavio Percoco
+  Kristoffer Kleine
+  yasirs
+  Michael Chermside
+  Anna Ravencroft
+  Dan Crosta
+  Julien Phalip
+  Dan Loewenherz
 
-    Heinrich-Heine University, Germany 
-    Open End AB (formerly AB Strakt), Sweden
-    merlinux GmbH, Germany 
-    tismerysoft GmbH, Germany 
-    Logilab Paris, France 
-    DFKI GmbH, Germany 
-    Impara, Germany
-    Change Maker, Sweden 
-    University of California Berkeley, USA
-    Google Inc.
-    King's College London
+  Heinrich-Heine University, Germany 
+  Open End AB (formerly AB Strakt), Sweden
+  merlinux GmbH, Germany 
+  tismerysoft GmbH, Germany 
+  Logilab Paris, France 
+  DFKI GmbH, Germany 
+  Impara, Germany
+  Change Maker, Sweden 
+  University of California Berkeley, USA
+  Google Inc.
+  King's College London
 
 The PyPy Logo as used by http://speed.pypy.org and others was created
 by Samuel Reis and is distributed on terms of Creative Commons Share Alike
diff --git a/_pytest/README-BEFORE-UPDATING b/_pytest/README-BEFORE-UPDATING
new file mode 100644
--- /dev/null
+++ b/_pytest/README-BEFORE-UPDATING
@@ -0,0 +1,17 @@
+This is PyPy's code of the pytest lib.  We don't expect to upgrade it
+very often, but once we do:
+
+    WARNING!
+
+    WE HAVE MADE A FEW TWEAKS HERE!
+
+Please be sure that you don't just copy the newer version from
+upstream without checking the few changes that we did.  This
+can be done like this:
+
+    cd 
+    hg log . -v | less
+
+then search for all " _pytest/" in that list to know which are the
+relevant checkins.  (Look for the checkins that only edit one
+or two files in this directory.)
diff --git a/_pytest/resultlog.py b/_pytest/resultlog.py
--- a/_pytest/resultlog.py
+++ b/_pytest/resultlog.py
@@ -53,16 +53,24 @@
         self.config = config
         self.logfile = logfile # preferably line buffered
 
-    def write_log_entry(self, testpath, lettercode, longrepr):
-        py.builtin.print_("%s %s" % (lettercode, testpath), file=self.logfile)
+    def write_log_entry(self, testpath, lettercode, longrepr, sections=None):
+        _safeprint("%s %s" % (lettercode, testpath), file=self.logfile)
         for line in longrepr.splitlines():
-            py.builtin.print_(" %s" % line, file=self.logfile)
+            _safeprint(" %s" % line, file=self.logfile)
+        if sections is not None and (
+                lettercode in ('E', 'F')):    # to limit the size of logs
+            for title, content in sections:
+                _safeprint(" ---------- %s ----------" % (title,),
+                           file=self.logfile)
+                for line in content.splitlines():
+                    _safeprint(" %s" % line, file=self.logfile)
 
     def log_outcome(self, report, lettercode, longrepr):
         testpath = getattr(report, 'nodeid', None)
         if testpath is None:
             testpath = report.fspath
-        self.write_log_entry(testpath, lettercode, longrepr)
+        self.write_log_entry(testpath, lettercode, longrepr,
+                             getattr(report, 'sections', None))
 
     def pytest_runtest_logreport(self, report):
         if report.when != "call" and report.passed:
@@ -98,3 +106,8 @@
         if path is None:
             path = "cwd:%s" % py.path.local()
         self.write_log_entry(path, '!', str(excrepr))
+
+def _safeprint(s, file):
+    if isinstance(s, unicode):
+        s = s.encode('utf-8')
+    py.builtin.print_(s, file=file)
diff --git a/lib-python/2.7/CGIHTTPServer.py b/lib-python/2.7/CGIHTTPServer.py
--- a/lib-python/2.7/CGIHTTPServer.py
+++ b/lib-python/2.7/CGIHTTPServer.py
@@ -84,7 +84,7 @@
         path begins with one of the strings in self.cgi_directories
         (and the next character is a '/' or the end of the string).
         """
-        collapsed_path = _url_collapse_path(self.path)
+        collapsed_path = _url_collapse_path(urllib.unquote(self.path))
         dir_sep = collapsed_path.find('/', 1)
         head, tail = collapsed_path[:dir_sep], collapsed_path[dir_sep+1:]
         if head in self.cgi_directories:
diff --git a/lib-python/2.7/Cookie.py b/lib-python/2.7/Cookie.py
--- a/lib-python/2.7/Cookie.py
+++ b/lib-python/2.7/Cookie.py
@@ -1,6 +1,3 @@
-#!/usr/bin/env python
-#
-
 ####
 # Copyright 2000 by Timothy O'Malley 
 #
diff --git a/lib-python/2.7/HTMLParser.py b/lib-python/2.7/HTMLParser.py
--- a/lib-python/2.7/HTMLParser.py
+++ b/lib-python/2.7/HTMLParser.py
@@ -22,9 +22,12 @@
 starttagopen = re.compile('<[a-zA-Z]')
 piclose = re.compile('>')
 commentclose = re.compile(r'--\s*>')
-tagfind = re.compile('([a-zA-Z][-.a-zA-Z0-9:_]*)(?:\s|/(?!>))*')
+
 # see http://www.w3.org/TR/html5/tokenization.html#tag-open-state
 # and http://www.w3.org/TR/html5/tokenization.html#tag-name-state
+# note: if you change tagfind/attrfind remember to update locatestarttagend too
+tagfind = re.compile('([a-zA-Z][^\t\n\r\f />\x00]*)(?:\s|/(?!>))*')
+# this regex is currently unused, but left for backward compatibility
 tagfind_tolerant = re.compile('[a-zA-Z][^\t\n\r\f />\x00]*')
 
 attrfind = re.compile(
@@ -32,7 +35,7 @@
     r'(\'[^\']*\'|"[^"]*"|(?![\'"])[^>\s]*))?(?:\s|/(?!>))*')
 
 locatestarttagend = re.compile(r"""
-  <[a-zA-Z][-.a-zA-Z0-9:_]*          # tag name
+  <[a-zA-Z][^\t\n\r\f />\x00]*       # tag name
   (?:[\s/]*                          # optional whitespace before attribute name
     (?:(?<=['"\s/])[^\s/>][^\s/=>]*  # attribute name
       (?:\s*=+\s*                    # value indicator
@@ -192,9 +195,9 @@
                     i = self.updatepos(i, k)
                     continue
                 else:
-                    if ";" in rawdata[i:]: #bail by consuming &#
-                        self.handle_data(rawdata[0:2])
-                        i = self.updatepos(i, 2)
+                    if ";" in rawdata[i:]:  # bail by consuming '&#'
+                        self.handle_data(rawdata[i:i+2])
+                        i = self.updatepos(i, i+2)
                     break
             elif startswith('&', i):
                 match = entityref.match(rawdata, i)
@@ -373,14 +376,14 @@
                 self.handle_data(rawdata[i:gtpos])
                 return gtpos
             # find the name: w3.org/TR/html5/tokenization.html#tag-name-state
-            namematch = tagfind_tolerant.match(rawdata, i+2)
+            namematch = tagfind.match(rawdata, i+2)
             if not namematch:
                 # w3.org/TR/html5/tokenization.html#end-tag-open-state
                 if rawdata[i:i+3] == '':
                     return i+3
                 else:
                     return self.parse_bogus_comment(i)
-            tagname = namematch.group().lower()
+            tagname = namematch.group(1).lower()
             # consume and ignore other stuff between the name and the >
             # Note: this is not 100% correct, since we might have things like
             # , but looking for > after tha name should cover
diff --git a/lib-python/2.7/SimpleHTTPServer.py b/lib-python/2.7/SimpleHTTPServer.py
--- a/lib-python/2.7/SimpleHTTPServer.py
+++ b/lib-python/2.7/SimpleHTTPServer.py
@@ -43,8 +43,10 @@
         """Serve a GET request."""
         f = self.send_head()
         if f:
-            self.copyfile(f, self.wfile)
-            f.close()
+            try:
+                self.copyfile(f, self.wfile)
+            finally:
+                f.close()
 
     def do_HEAD(self):
         """Serve a HEAD request."""
@@ -88,13 +90,17 @@
         except IOError:
             self.send_error(404, "File not found")
             return None
-        self.send_response(200)
-        self.send_header("Content-type", ctype)
-        fs = os.fstat(f.fileno())
-        self.send_header("Content-Length", str(fs[6]))
-        self.send_header("Last-Modified", self.date_time_string(fs.st_mtime))
-        self.end_headers()
-        return f
+        try:
+            self.send_response(200)
+            self.send_header("Content-type", ctype)
+            fs = os.fstat(f.fileno())
+            self.send_header("Content-Length", str(fs[6]))
+            self.send_header("Last-Modified", self.date_time_string(fs.st_mtime))
+            self.end_headers()
+            return f
+        except:
+            f.close()
+            raise
 
     def list_directory(self, path):
         """Helper to produce a directory listing (absent index.html).
diff --git a/lib-python/2.7/SimpleXMLRPCServer.py b/lib-python/2.7/SimpleXMLRPCServer.py
--- a/lib-python/2.7/SimpleXMLRPCServer.py
+++ b/lib-python/2.7/SimpleXMLRPCServer.py
@@ -704,4 +704,5 @@
     server = SimpleXMLRPCServer(("localhost", 8000))
     server.register_function(pow)
     server.register_function(lambda x,y: x+y, 'add')
+    server.register_multicall_functions()
     server.serve_forever()
diff --git a/lib-python/2.7/SocketServer.py b/lib-python/2.7/SocketServer.py
--- a/lib-python/2.7/SocketServer.py
+++ b/lib-python/2.7/SocketServer.py
@@ -513,35 +513,37 @@
 
     def collect_children(self):
         """Internal routine to wait for children that have exited."""
-        if self.active_children is None: return
+        if self.active_children is None:
+            return
+
+        # If we're above the max number of children, wait and reap them until
+        # we go back below threshold. Note that we use waitpid(-1) below to be
+        # able to collect children in size() syscalls instead
+        # of size(): the downside is that this might reap children
+        # which we didn't spawn, which is why we only resort to this when we're
+        # above max_children.
         while len(self.active_children) >= self.max_children:
-            # XXX: This will wait for any child process, not just ones
-            # spawned by this library. This could confuse other
-            # libraries that expect to be able to wait for their own
-            # children.
             try:
-                pid, status = os.waitpid(0, 0)
-            except os.error:
-                pid = None
-            if pid not in self.active_children: continue
-            self.active_children.remove(pid)
+                pid, _ = os.waitpid(-1, 0)
+                self.active_children.discard(pid)
+            except OSError as e:
+                if e.errno == errno.ECHILD:
+                    # we don't have any children, we're done
+                    self.active_children.clear()
+                elif e.errno != errno.EINTR:
+                    break
 
-        # XXX: This loop runs more system calls than it ought
-        # to. There should be a way to put the active_children into a
-        # process group and then use os.waitpid(-pgid) to wait for any
-        # of that set, but I couldn't find a way to allocate pgids
-        # that couldn't collide.
-        for child in self.active_children:
+        # Now reap all defunct children.
+        for pid in self.active_children.copy():
             try:
-                pid, status = os.waitpid(child, os.WNOHANG)
-            except os.error:
-                pid = None
-            if not pid: continue
-            try:
-                self.active_children.remove(pid)
-            except ValueError, e:
-                raise ValueError('%s. x=%d and list=%r' % (e.message, pid,
-                                                           self.active_children))
+                pid, _ = os.waitpid(pid, os.WNOHANG)
+                # if the child hasn't exited yet, pid will be 0 and ignored by
+                # discard() below
+                self.active_children.discard(pid)
+            except OSError as e:
+                if e.errno == errno.ECHILD:
+                    # someone else reaped it
+                    self.active_children.discard(pid)
 
     def handle_timeout(self):
         """Wait for zombies after self.timeout seconds of inactivity.
@@ -557,8 +559,8 @@
         if pid:
             # Parent process
             if self.active_children is None:
-                self.active_children = []
-            self.active_children.append(pid)
+                self.active_children = set()
+            self.active_children.add(pid)
             self.close_request(request) #close handle in parent process
             return
         else:
diff --git a/lib-python/2.7/_MozillaCookieJar.py b/lib-python/2.7/_MozillaCookieJar.py
--- a/lib-python/2.7/_MozillaCookieJar.py
+++ b/lib-python/2.7/_MozillaCookieJar.py
@@ -39,7 +39,7 @@
     magic_re = "#( Netscape)? HTTP Cookie File"
     header = """\
 # Netscape HTTP Cookie File
-# http://www.netscape.com/newsref/std/cookie_spec.html
+# http://curl.haxx.se/rfc/cookie_spec.html
 # This is a generated file!  Do not edit.
 
 """
diff --git a/lib-python/2.7/_abcoll.py b/lib-python/2.7/_abcoll.py
--- a/lib-python/2.7/_abcoll.py
+++ b/lib-python/2.7/_abcoll.py
@@ -165,12 +165,17 @@
     def __gt__(self, other):
         if not isinstance(other, Set):
             return NotImplemented
-        return other < self
+        return len(self) > len(other) and self.__ge__(other)
 
     def __ge__(self, other):
         if not isinstance(other, Set):
             return NotImplemented
-        return other <= self
+        if len(self) < len(other):
+            return False
+        for elem in other:
+            if elem not in self:
+                return False
+        return True
 
     def __eq__(self, other):
         if not isinstance(other, Set):
@@ -194,6 +199,8 @@
             return NotImplemented
         return self._from_iterable(value for value in other if value in self)
 
+    __rand__ = __and__
+
     def isdisjoint(self, other):
         'Return True if two sets have a null intersection.'
         for value in other:
@@ -207,6 +214,8 @@
         chain = (e for s in (self, other) for e in s)
         return self._from_iterable(chain)
 
+    __ror__ = __or__
+
     def __sub__(self, other):
         if not isinstance(other, Set):
             if not isinstance(other, Iterable):
@@ -215,6 +224,14 @@
         return self._from_iterable(value for value in self
                                    if value not in other)
 
+    def __rsub__(self, other):
+        if not isinstance(other, Set):
+            if not isinstance(other, Iterable):
+                return NotImplemented
+            other = self._from_iterable(other)
+        return self._from_iterable(value for value in other
+                                   if value not in self)
+
     def __xor__(self, other):
         if not isinstance(other, Set):
             if not isinstance(other, Iterable):
@@ -222,6 +239,8 @@
             other = self._from_iterable(other)
         return (self - other) | (other - self)
 
+    __rxor__ = __xor__
+
     # Sets are not hashable by default, but subclasses can change this
     __hash__ = None
 
diff --git a/lib-python/2.7/_osx_support.py b/lib-python/2.7/_osx_support.py
--- a/lib-python/2.7/_osx_support.py
+++ b/lib-python/2.7/_osx_support.py
@@ -182,7 +182,7 @@
         # Compiler is GCC, check if it is LLVM-GCC
         data = _read_output("'%s' --version"
                              % (cc.replace("'", "'\"'\"'"),))
-        if 'llvm-gcc' in data:
+        if data and 'llvm-gcc' in data:
             # Found LLVM-GCC, fall back to clang
             cc = _find_build_tool('clang')
 
@@ -450,8 +450,16 @@
         # case and disallow installs.
         cflags = _config_vars.get(_INITPRE+'CFLAGS',
                                     _config_vars.get('CFLAGS', ''))
-        if ((macrelease + '.') >= '10.4.' and
-            '-arch' in cflags.strip()):
+        if macrelease:
+            try:
+                macrelease = tuple(int(i) for i in macrelease.split('.')[0:2])
+            except ValueError:
+                macrelease = (10, 0)
+        else:
+            # assume no universal support
+            macrelease = (10, 0)
+
+        if (macrelease >= (10, 4)) and '-arch' in cflags.strip():
             # The universal build will build fat binaries, but not on
             # systems before 10.4
 
diff --git a/lib-python/2.7/_pyio.py b/lib-python/2.7/_pyio.py
--- a/lib-python/2.7/_pyio.py
+++ b/lib-python/2.7/_pyio.py
@@ -192,38 +192,45 @@
                  (appending and "a" or "") +
                  (updating and "+" or ""),
                  closefd)
-    line_buffering = False
-    if buffering == 1 or buffering < 0 and raw.isatty():
-        buffering = -1
-        line_buffering = True
-    if buffering < 0:
-        buffering = DEFAULT_BUFFER_SIZE
-        try:
-            bs = os.fstat(raw.fileno()).st_blksize
-        except (os.error, AttributeError):
-            pass
+    result = raw
+    try:
+        line_buffering = False
+        if buffering == 1 or buffering < 0 and raw.isatty():
+            buffering = -1
+            line_buffering = True
+        if buffering < 0:
+            buffering = DEFAULT_BUFFER_SIZE
+            try:
+                bs = os.fstat(raw.fileno()).st_blksize
+            except (os.error, AttributeError):
+                pass
+            else:
+                if bs > 1:
+                    buffering = bs
+        if buffering < 0:
+            raise ValueError("invalid buffering size")
+        if buffering == 0:
+            if binary:
+                return result
+            raise ValueError("can't have unbuffered text I/O")
+        if updating:
+            buffer = BufferedRandom(raw, buffering)
+        elif writing or appending:
+            buffer = BufferedWriter(raw, buffering)
+        elif reading:
+            buffer = BufferedReader(raw, buffering)
         else:
-            if bs > 1:
-                buffering = bs
-    if buffering < 0:
-        raise ValueError("invalid buffering size")
-    if buffering == 0:
+            raise ValueError("unknown mode: %r" % mode)
+        result = buffer
         if binary:
-            return raw
-        raise ValueError("can't have unbuffered text I/O")
-    if updating:
-        buffer = BufferedRandom(raw, buffering)
-    elif writing or appending:
-        buffer = BufferedWriter(raw, buffering)
-    elif reading:
-        buffer = BufferedReader(raw, buffering)
-    else:
-        raise ValueError("unknown mode: %r" % mode)
-    if binary:
-        return buffer
-    text = TextIOWrapper(buffer, encoding, errors, newline, line_buffering)
-    text.mode = mode
-    return text
+            return result
+        text = TextIOWrapper(buffer, encoding, errors, newline, line_buffering)
+        result = text
+        text.mode = mode
+        return result
+    except:
+        result.close()
+        raise
 
 
 class DocDescriptor:
@@ -1997,7 +2004,13 @@
 
     def getvalue(self):
         self.flush()
-        return self.buffer.getvalue().decode(self._encoding, self._errors)
+        decoder = self._decoder or self._get_decoder()
+        old_state = decoder.getstate()
+        decoder.reset()
+        try:
+            return decoder.decode(self.buffer.getvalue(), final=True)
+        finally:
+            decoder.setstate(old_state)
 
     def __repr__(self):
         # TextIOWrapper tells the encoding in its repr. In StringIO,
diff --git a/lib-python/2.7/_weakrefset.py b/lib-python/2.7/_weakrefset.py
--- a/lib-python/2.7/_weakrefset.py
+++ b/lib-python/2.7/_weakrefset.py
@@ -60,6 +60,8 @@
             for itemref in self.data:
                 item = itemref()
                 if item is not None:
+                    # Caveat: the iterator will keep a strong reference to
+                    # `item` until it is resumed or closed.
                     yield item
 
     def __len__(self):
diff --git a/lib-python/2.7/aifc.py b/lib-python/2.7/aifc.py
--- a/lib-python/2.7/aifc.py
+++ b/lib-python/2.7/aifc.py
@@ -778,7 +778,7 @@
 
     def _ensure_header_written(self, datasize):
         if not self._nframeswritten:
-            if self._comptype in ('ULAW', 'ALAW'):
+            if self._comptype in ('ULAW', 'ulaw', 'ALAW', 'alaw'):
                 if not self._sampwidth:
                     self._sampwidth = 2
                 if self._sampwidth != 2:
@@ -844,7 +844,7 @@
         if self._datalength & 1:
             self._datalength = self._datalength + 1
         if self._aifc:
-            if self._comptype in ('ULAW', 'ALAW'):
+            if self._comptype in ('ULAW', 'ulaw', 'ALAW', 'alaw'):
                 self._datalength = self._datalength // 2
                 if self._datalength & 1:
                     self._datalength = self._datalength + 1
@@ -852,7 +852,10 @@
                 self._datalength = (self._datalength + 3) // 4
                 if self._datalength & 1:
                     self._datalength = self._datalength + 1
-        self._form_length_pos = self._file.tell()
+        try:
+            self._form_length_pos = self._file.tell()
+        except (AttributeError, IOError):
+            self._form_length_pos = None
         commlength = self._write_form_length(self._datalength)
         if self._aifc:
             self._file.write('AIFC')
@@ -864,7 +867,8 @@
         self._file.write('COMM')
         _write_ulong(self._file, commlength)
         _write_short(self._file, self._nchannels)
-        self._nframes_pos = self._file.tell()
+        if self._form_length_pos is not None:
+            self._nframes_pos = self._file.tell()
         _write_ulong(self._file, self._nframes)
         if self._comptype in ('ULAW', 'ulaw', 'ALAW', 'alaw', 'G722'):
             _write_short(self._file, 8)
@@ -875,7 +879,8 @@
             self._file.write(self._comptype)
             _write_string(self._file, self._compname)
         self._file.write('SSND')
-        self._ssnd_length_pos = self._file.tell()
+        if self._form_length_pos is not None:
+            self._ssnd_length_pos = self._file.tell()
         _write_ulong(self._file, self._datalength + 8)
         _write_ulong(self._file, 0)
         _write_ulong(self._file, 0)
diff --git a/lib-python/2.7/argparse.py b/lib-python/2.7/argparse.py
--- a/lib-python/2.7/argparse.py
+++ b/lib-python/2.7/argparse.py
@@ -168,6 +168,8 @@
         self._prog = prog
         self._indent_increment = indent_increment
         self._max_help_position = max_help_position
+        self._max_help_position = min(max_help_position,
+                                      max(width - 20, indent_increment * 2))
         self._width = width
 
         self._current_indent = 0
@@ -339,7 +341,7 @@
                     else:
                         line_len = len(indent) - 1
                     for part in parts:
-                        if line_len + 1 + len(part) > text_width:
+                        if line_len + 1 + len(part) > text_width and line:
                             lines.append(indent + ' '.join(line))
                             line = []
                             line_len = len(indent) - 1
@@ -478,7 +480,7 @@
     def _format_text(self, text):
         if '%(prog)' in text:
             text = text % dict(prog=self._prog)
-        text_width = self._width - self._current_indent
+        text_width = max(self._width - self._current_indent, 11)
         indent = ' ' * self._current_indent
         return self._fill_text(text, text_width, indent) + '\n\n'
 
@@ -486,7 +488,7 @@
         # determine the required width and the entry label
         help_position = min(self._action_max_length + 2,
                             self._max_help_position)
-        help_width = self._width - help_position
+        help_width = max(self._width - help_position, 11)
         action_width = help_position - self._current_indent - 2
         action_header = self._format_action_invocation(action)
 
@@ -1155,9 +1157,13 @@
     __hash__ = None
 
     def __eq__(self, other):
+        if not isinstance(other, Namespace):
+            return NotImplemented
         return vars(self) == vars(other)
 
     def __ne__(self, other):
+        if not isinstance(other, Namespace):
+            return NotImplemented
         return not (self == other)
 
     def __contains__(self, key):
diff --git a/lib-python/2.7/bsddb/dbshelve.py b/lib-python/2.7/bsddb/dbshelve.py
--- a/lib-python/2.7/bsddb/dbshelve.py
+++ b/lib-python/2.7/bsddb/dbshelve.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 #------------------------------------------------------------------------
 #           Copyright (c) 1997-2001 by Total Control Software
 #                         All Rights Reserved
diff --git a/lib-python/2.7/bsddb/test/test_dbtables.py b/lib-python/2.7/bsddb/test/test_dbtables.py
--- a/lib-python/2.7/bsddb/test/test_dbtables.py
+++ b/lib-python/2.7/bsddb/test/test_dbtables.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-#
 #-----------------------------------------------------------------------
 # A test suite for the table interface built on bsddb.db
 #-----------------------------------------------------------------------
diff --git a/lib-python/2.7/codecs.py b/lib-python/2.7/codecs.py
--- a/lib-python/2.7/codecs.py
+++ b/lib-python/2.7/codecs.py
@@ -456,15 +456,12 @@
 
         # read until we get the required number of characters (if available)
         while True:
-            # can the request can be satisfied from the character buffer?
-            if chars < 0:
-                if size < 0:
-                    if self.charbuffer:
-                        break
-                elif len(self.charbuffer) >= size:
+            # can the request be satisfied from the character buffer?
+            if chars >= 0:
+                if len(self.charbuffer) >= chars:
                     break
-            else:
-                if len(self.charbuffer) >= chars:
+            elif size >= 0:
+                if len(self.charbuffer) >= size:
                     break
             # we need more data
             if size < 0:
diff --git a/lib-python/2.7/collections.py b/lib-python/2.7/collections.py
--- a/lib-python/2.7/collections.py
+++ b/lib-python/2.7/collections.py
@@ -319,6 +319,7 @@
     if isinstance(field_names, basestring):
         field_names = field_names.replace(',', ' ').split()
     field_names = map(str, field_names)
+    typename = str(typename)
     if rename:
         seen = set()
         for index, name in enumerate(field_names):
@@ -331,6 +332,8 @@
                 field_names[index] = '_%d' % index
             seen.add(name)
     for name in [typename] + field_names:
+        if type(name) != str:
+            raise TypeError('Type names and field names must be strings')
         if not all(c.isalnum() or c=='_' for c in name):
             raise ValueError('Type names and field names can only contain '
                              'alphanumeric characters and underscores: %r' % name)
diff --git a/lib-python/2.7/csv.py b/lib-python/2.7/csv.py
--- a/lib-python/2.7/csv.py
+++ b/lib-python/2.7/csv.py
@@ -93,6 +93,10 @@
         self.line_num = self.reader.line_num
         return self._fieldnames
 
+    # Issue 20004: Because DictReader is a classic class, this setter is
+    # ignored.  At this point in 2.7's lifecycle, it is too late to change the
+    # base class for fear of breaking working code.  If you want to change
+    # fieldnames without overwriting the getter, set _fieldnames directly.
     @fieldnames.setter
     def fieldnames(self, value):
         self._fieldnames = value
@@ -140,8 +144,8 @@
         if self.extrasaction == "raise":
             wrong_fields = [k for k in rowdict if k not in self.fieldnames]
             if wrong_fields:
-                raise ValueError("dict contains fields not in fieldnames: " +
-                                 ", ".join(wrong_fields))
+                raise ValueError("dict contains fields not in fieldnames: "
+                                 + ", ".join([repr(x) for x in wrong_fields]))
         return [rowdict.get(key, self.restval) for key in self.fieldnames]
 
     def writerow(self, rowdict):
diff --git a/lib-python/2.7/ctypes/test/__init__.py b/lib-python/2.7/ctypes/test/__init__.py
--- a/lib-python/2.7/ctypes/test/__init__.py
+++ b/lib-python/2.7/ctypes/test/__init__.py
@@ -2,7 +2,15 @@
 
 use_resources = []
 
-class ResourceDenied(Exception):
+import ctypes
+ctypes_symbols = dir(ctypes)
+
+def need_symbol(name):
+    return unittest.skipUnless(name in ctypes_symbols,
+                               '{!r} is required'.format(name))
+
+
+class ResourceDenied(unittest.SkipTest):
     """Test skipped because it requested a disallowed resource.
 
     This is raised when a test calls requires() for a resource that
diff --git a/lib-python/2.7/ctypes/test/test_arrays.py b/lib-python/2.7/ctypes/test/test_arrays.py
--- a/lib-python/2.7/ctypes/test/test_arrays.py
+++ b/lib-python/2.7/ctypes/test/test_arrays.py
@@ -2,6 +2,8 @@
 from ctypes import *
 from test.test_support import impl_detail
 
+from ctypes.test import need_symbol
+
 formats = "bBhHiIlLqQfd"
 
 # c_longdouble commented out for PyPy, look at the commend in test_longdouble
@@ -98,8 +100,8 @@
         self.assertEqual(values, [1, 2, 3, 4, 5])
 
     def test_classcache(self):
-        self.assertTrue(not ARRAY(c_int, 3) is ARRAY(c_int, 4))
-        self.assertTrue(ARRAY(c_int, 3) is ARRAY(c_int, 3))
+        self.assertIsNot(ARRAY(c_int, 3), ARRAY(c_int, 4))
+        self.assertIs(ARRAY(c_int, 3), ARRAY(c_int, 3))
 
     def test_from_address(self):
         # Failed with 0.9.8, reported by JUrner
@@ -112,20 +114,16 @@
         self.assertEqual(sz[1:4:2], "o")
         self.assertEqual(sz.value, "foo")
 
-    try:
-        create_unicode_buffer
-    except NameError:
-        pass
-    else:
-        def test_from_addressW(self):
-            p = create_unicode_buffer("foo")
-            sz = (c_wchar * 3).from_address(addressof(p))
-            self.assertEqual(sz[:], "foo")
-            self.assertEqual(sz[::], "foo")
-            self.assertEqual(sz[::-1], "oof")
-            self.assertEqual(sz[::3], "f")
-            self.assertEqual(sz[1:4:2], "o")
-            self.assertEqual(sz.value, "foo")
+    @need_symbol('create_unicode_buffer')
+    def test_from_addressW(self):
+        p = create_unicode_buffer("foo")
+        sz = (c_wchar * 3).from_address(addressof(p))
+        self.assertEqual(sz[:], "foo")
+        self.assertEqual(sz[::], "foo")
+        self.assertEqual(sz[::-1], "oof")
+        self.assertEqual(sz[::3], "f")
+        self.assertEqual(sz[1:4:2], "o")
+        self.assertEqual(sz.value, "foo")
 
     def test_cache(self):
         # Array types are cached internally in the _ctypes extension,
@@ -139,7 +137,7 @@
         # Create a new array type based on it:
         t1 = my_int * 1
         t2 = my_int * 1
-        self.assertTrue(t1 is t2)
+        self.assertIs(t1, t2)
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_as_parameter.py b/lib-python/2.7/ctypes/test/test_as_parameter.py
--- a/lib-python/2.7/ctypes/test/test_as_parameter.py
+++ b/lib-python/2.7/ctypes/test/test_as_parameter.py
@@ -1,5 +1,6 @@
 import unittest
 from ctypes import *
+from ctypes.test import need_symbol
 import _ctypes_test
 
 dll = CDLL(_ctypes_test.__file__)
@@ -17,11 +18,8 @@
     def wrap(self, param):
         return param
 
+    @need_symbol('c_wchar')
     def test_wchar_parm(self):
-        try:
-            c_wchar
-        except NameError:
-            return
         f = dll._testfunc_i_bhilfd
         f.argtypes = [c_byte, c_wchar, c_int, c_long, c_float, c_double]
         result = f(self.wrap(1), self.wrap(u"x"), self.wrap(3), self.wrap(4), self.wrap(5.0), self.wrap(6.0))
@@ -134,7 +132,7 @@
         f.argtypes = [c_longlong, MyCallback]
 
         def callback(value):
-            self.assertTrue(isinstance(value, (int, long)))
+            self.assertIsInstance(value, (int, long))
             return value & 0x7FFFFFFF
 
         cb = MyCallback(callback)
diff --git a/lib-python/2.7/ctypes/test/test_bitfields.py b/lib-python/2.7/ctypes/test/test_bitfields.py
--- a/lib-python/2.7/ctypes/test/test_bitfields.py
+++ b/lib-python/2.7/ctypes/test/test_bitfields.py
@@ -1,4 +1,5 @@
 from ctypes import *
+from ctypes.test import need_symbol
 import unittest
 import os
 
@@ -131,15 +132,6 @@
         self.assertEqual(result[0], TypeError)
         self.assertIn('bit fields not allowed for type', result[1])
 
-        try:
-            c_wchar
-        except NameError:
-            pass
-        else:
-            result = self.fail_fields(("a", c_wchar, 1))
-            self.assertEqual(result[0], TypeError)
-            self.assertIn('bit fields not allowed for type', result[1])
-
         class Dummy(Structure):
             _fields_ = []
 
@@ -147,6 +139,12 @@
         self.assertEqual(result[0], TypeError)
         self.assertIn('bit fields not allowed for type', result[1])
 
+    @need_symbol('c_wchar')
+    def test_c_wchar(self):
+        result = self.fail_fields(("a", c_wchar, 1))
+        self.assertEqual(result,
+                (TypeError, 'bit fields not allowed for type c_wchar'))
+
     def test_single_bitfield_size(self):
         for c_typ in int_types:
             result = self.fail_fields(("a", c_typ, -1))
@@ -213,7 +211,7 @@
         class X(Structure):
             _fields_ = [("a", c_byte, 4),
                         ("b", c_int, 32)]
-        self.assertEqual(sizeof(X), sizeof(c_int)*2)
+        self.assertEqual(sizeof(X), alignment(c_int)+sizeof(c_int))
 
     def test_mixed_3(self):
         class X(Structure):
@@ -246,7 +244,7 @@
             _anonymous_ = ["_"]
             _fields_ = [("_", X)]
 
-    @unittest.skipUnless(hasattr(ctypes, "c_uint32"), "c_int32 is required")
+    @need_symbol('c_uint32')
     def test_uint32(self):
         class X(Structure):
             _fields_ = [("a", c_uint32, 32)]
@@ -256,7 +254,7 @@
         x.a = 0xFDCBA987
         self.assertEqual(x.a, 0xFDCBA987)
 
-    @unittest.skipUnless(hasattr(ctypes, "c_uint64"), "c_int64 is required")
+    @need_symbol('c_uint64')
     def test_uint64(self):
         class X(Structure):
             _fields_ = [("a", c_uint64, 64)]
diff --git a/lib-python/2.7/ctypes/test/test_buffers.py b/lib-python/2.7/ctypes/test/test_buffers.py
--- a/lib-python/2.7/ctypes/test/test_buffers.py
+++ b/lib-python/2.7/ctypes/test/test_buffers.py
@@ -1,4 +1,5 @@
 from ctypes import *
+from ctypes.test import need_symbol
 import unittest
 
 class StringBufferTestCase(unittest.TestCase):
@@ -7,12 +8,12 @@
         b = create_string_buffer(32)
         self.assertEqual(len(b), 32)
         self.assertEqual(sizeof(b), 32 * sizeof(c_char))
-        self.assertTrue(type(b[0]) is str)
+        self.assertIs(type(b[0]), str)
 
         b = create_string_buffer("abc")
         self.assertEqual(len(b), 4) # trailing nul char
         self.assertEqual(sizeof(b), 4 * sizeof(c_char))
-        self.assertTrue(type(b[0]) is str)
+        self.assertIs(type(b[0]), str)
         self.assertEqual(b[0], "a")
         self.assertEqual(b[:], "abc\0")
         self.assertEqual(b[::], "abc\0")
@@ -36,39 +37,36 @@
         self.assertEqual(b[::2], "ac")
         self.assertEqual(b[::5], "a")
 
-    try:
-        c_wchar
-    except NameError:
-        pass
-    else:
-        def test_unicode_buffer(self):
-            b = create_unicode_buffer(32)
-            self.assertEqual(len(b), 32)
-            self.assertEqual(sizeof(b), 32 * sizeof(c_wchar))
-            self.assertTrue(type(b[0]) is unicode)
+    @need_symbol('c_wchar')
+    def test_unicode_buffer(self):
+        b = create_unicode_buffer(32)
+        self.assertEqual(len(b), 32)
+        self.assertEqual(sizeof(b), 32 * sizeof(c_wchar))
+        self.assertIs(type(b[0]), unicode)
 
-            b = create_unicode_buffer(u"abc")
-            self.assertEqual(len(b), 4) # trailing nul char
-            self.assertEqual(sizeof(b), 4 * sizeof(c_wchar))
-            self.assertTrue(type(b[0]) is unicode)
-            self.assertEqual(b[0], u"a")
-            self.assertEqual(b[:], "abc\0")
-            self.assertEqual(b[::], "abc\0")
-            self.assertEqual(b[::-1], "\0cba")
-            self.assertEqual(b[::2], "ac")
-            self.assertEqual(b[::5], "a")
+        b = create_unicode_buffer(u"abc")
+        self.assertEqual(len(b), 4) # trailing nul char
+        self.assertEqual(sizeof(b), 4 * sizeof(c_wchar))
+        self.assertIs(type(b[0]), unicode)
+        self.assertEqual(b[0], u"a")
+        self.assertEqual(b[:], "abc\0")
+        self.assertEqual(b[::], "abc\0")
+        self.assertEqual(b[::-1], "\0cba")
+        self.assertEqual(b[::2], "ac")
+        self.assertEqual(b[::5], "a")
 
-        def test_unicode_conversion(self):
-            b = create_unicode_buffer("abc")
-            self.assertEqual(len(b), 4) # trailing nul char
-            self.assertEqual(sizeof(b), 4 * sizeof(c_wchar))
-            self.assertTrue(type(b[0]) is unicode)
-            self.assertEqual(b[0], u"a")
-            self.assertEqual(b[:], "abc\0")
-            self.assertEqual(b[::], "abc\0")
-            self.assertEqual(b[::-1], "\0cba")
-            self.assertEqual(b[::2], "ac")
-            self.assertEqual(b[::5], "a")
+    @need_symbol('c_wchar')
+    def test_unicode_conversion(self):
+        b = create_unicode_buffer("abc")
+        self.assertEqual(len(b), 4) # trailing nul char
+        self.assertEqual(sizeof(b), 4 * sizeof(c_wchar))
+        self.assertIs(type(b[0]), unicode)
+        self.assertEqual(b[0], u"a")
+        self.assertEqual(b[:], "abc\0")
+        self.assertEqual(b[::], "abc\0")
+        self.assertEqual(b[::-1], "\0cba")
+        self.assertEqual(b[::2], "ac")
+        self.assertEqual(b[::5], "a")
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_byteswap.py b/lib-python/2.7/ctypes/test/test_byteswap.py
--- a/lib-python/2.7/ctypes/test/test_byteswap.py
+++ b/lib-python/2.7/ctypes/test/test_byteswap.py
@@ -15,7 +15,8 @@
 # For Structures and Unions, these types are created on demand.
 
 class Test(unittest.TestCase):
-    def X_test(self):
+    @unittest.skip('test disabled')
+    def test_X(self):
         print >> sys.stderr,  sys.byteorder
         for i in range(32):
             bits = BITS()
@@ -25,11 +26,11 @@
     @xfail
     def test_endian_short(self):
         if sys.byteorder == "little":
-            self.assertTrue(c_short.__ctype_le__ is c_short)
-            self.assertTrue(c_short.__ctype_be__.__ctype_le__ is c_short)
+            self.assertIs(c_short.__ctype_le__, c_short)
+            self.assertIs(c_short.__ctype_be__.__ctype_le__, c_short)
         else:
-            self.assertTrue(c_short.__ctype_be__ is c_short)
-            self.assertTrue(c_short.__ctype_le__.__ctype_be__ is c_short)
+            self.assertIs(c_short.__ctype_be__, c_short)
+            self.assertIs(c_short.__ctype_le__.__ctype_be__, c_short)
         s = c_short.__ctype_be__(0x1234)
         self.assertEqual(bin(struct.pack(">h", 0x1234)), "1234")
         self.assertEqual(bin(s), "1234")
@@ -53,11 +54,11 @@
     @xfail
     def test_endian_int(self):
         if sys.byteorder == "little":
-            self.assertTrue(c_int.__ctype_le__ is c_int)
-            self.assertTrue(c_int.__ctype_be__.__ctype_le__ is c_int)
+            self.assertIs(c_int.__ctype_le__, c_int)
+            self.assertIs(c_int.__ctype_be__.__ctype_le__, c_int)
         else:
-            self.assertTrue(c_int.__ctype_be__ is c_int)
-            self.assertTrue(c_int.__ctype_le__.__ctype_be__ is c_int)
+            self.assertIs(c_int.__ctype_be__, c_int)
+            self.assertIs(c_int.__ctype_le__.__ctype_be__, c_int)
 
         s = c_int.__ctype_be__(0x12345678)
         self.assertEqual(bin(struct.pack(">i", 0x12345678)), "12345678")
@@ -82,11 +83,11 @@
     @xfail
     def test_endian_longlong(self):
         if sys.byteorder == "little":
-            self.assertTrue(c_longlong.__ctype_le__ is c_longlong)
-            self.assertTrue(c_longlong.__ctype_be__.__ctype_le__ is c_longlong)
+            self.assertIs(c_longlong.__ctype_le__, c_longlong)
+            self.assertIs(c_longlong.__ctype_be__.__ctype_le__, c_longlong)
         else:
-            self.assertTrue(c_longlong.__ctype_be__ is c_longlong)
-            self.assertTrue(c_longlong.__ctype_le__.__ctype_be__ is c_longlong)
+            self.assertIs(c_longlong.__ctype_be__, c_longlong)
+            self.assertIs(c_longlong.__ctype_le__.__ctype_be__, c_longlong)
 
         s = c_longlong.__ctype_be__(0x1234567890ABCDEF)
         self.assertEqual(bin(struct.pack(">q", 0x1234567890ABCDEF)), "1234567890ABCDEF")
@@ -111,11 +112,11 @@
     @xfail
     def test_endian_float(self):
         if sys.byteorder == "little":
-            self.assertTrue(c_float.__ctype_le__ is c_float)
-            self.assertTrue(c_float.__ctype_be__.__ctype_le__ is c_float)
+            self.assertIs(c_float.__ctype_le__, c_float)
+            self.assertIs(c_float.__ctype_be__.__ctype_le__, c_float)
         else:
-            self.assertTrue(c_float.__ctype_be__ is c_float)
-            self.assertTrue(c_float.__ctype_le__.__ctype_be__ is c_float)
+            self.assertIs(c_float.__ctype_be__, c_float)
+            self.assertIs(c_float.__ctype_le__.__ctype_be__, c_float)
         s = c_float(math.pi)
         self.assertEqual(bin(struct.pack("f", math.pi)), bin(s))
         # Hm, what's the precision of a float compared to a double?
@@ -130,11 +131,11 @@
     @xfail
     def test_endian_double(self):
         if sys.byteorder == "little":
-            self.assertTrue(c_double.__ctype_le__ is c_double)
-            self.assertTrue(c_double.__ctype_be__.__ctype_le__ is c_double)
+            self.assertIs(c_double.__ctype_le__, c_double)
+            self.assertIs(c_double.__ctype_be__.__ctype_le__, c_double)
         else:
-            self.assertTrue(c_double.__ctype_be__ is c_double)
-            self.assertTrue(c_double.__ctype_le__.__ctype_be__ is c_double)
+            self.assertIs(c_double.__ctype_be__, c_double)
+            self.assertIs(c_double.__ctype_le__.__ctype_be__, c_double)
         s = c_double(math.pi)
         self.assertEqual(s.value, math.pi)
         self.assertEqual(bin(struct.pack("d", math.pi)), bin(s))
@@ -146,14 +147,14 @@
         self.assertEqual(bin(struct.pack(">d", math.pi)), bin(s))
 
     def test_endian_other(self):
-        self.assertTrue(c_byte.__ctype_le__ is c_byte)
-        self.assertTrue(c_byte.__ctype_be__ is c_byte)
+        self.assertIs(c_byte.__ctype_le__, c_byte)
+        self.assertIs(c_byte.__ctype_be__, c_byte)
 
-        self.assertTrue(c_ubyte.__ctype_le__ is c_ubyte)
-        self.assertTrue(c_ubyte.__ctype_be__ is c_ubyte)
+        self.assertIs(c_ubyte.__ctype_le__, c_ubyte)
+        self.assertIs(c_ubyte.__ctype_be__, c_ubyte)
 
-        self.assertTrue(c_char.__ctype_le__ is c_char)
-        self.assertTrue(c_char.__ctype_be__ is c_char)
+        self.assertIs(c_char.__ctype_le__, c_char)
+        self.assertIs(c_char.__ctype_be__, c_char)
 
     @xfail
     def test_struct_fields_1(self):
diff --git a/lib-python/2.7/ctypes/test/test_callbacks.py b/lib-python/2.7/ctypes/test/test_callbacks.py
--- a/lib-python/2.7/ctypes/test/test_callbacks.py
+++ b/lib-python/2.7/ctypes/test/test_callbacks.py
@@ -1,5 +1,6 @@
 import unittest
 from ctypes import *
+from ctypes.test import need_symbol
 from ctypes.test import xfail
 import _ctypes_test
 
@@ -95,9 +96,10 @@
     # disabled: would now (correctly) raise a RuntimeWarning about
     # a memory leak.  A callback function cannot return a non-integral
     # C type without causing a memory leak.
-##    def test_char_p(self):
-##        self.check_type(c_char_p, "abc")
-##        self.check_type(c_char_p, "def")
+    @unittest.skip('test disabled')
+    def test_char_p(self):
+        self.check_type(c_char_p, "abc")
+        self.check_type(c_char_p, "def")
 
     @xfail
     def test_pyobject(self):
@@ -150,13 +152,12 @@
         CFUNCTYPE(None)(lambda x=Nasty(): None)
 
 
-try:
-    WINFUNCTYPE
-except NameError:
-    pass
-else:
-    class StdcallCallbacks(Callbacks):
+ at need_symbol('WINFUNCTYPE')
+class StdcallCallbacks(Callbacks):
+    try:
         functype = WINFUNCTYPE
+    except NameError:
+        pass
 
 ################################################################
 
@@ -186,7 +187,7 @@
         from ctypes.util import find_library
         libc_path = find_library("c")
         if not libc_path:
-            return # cannot test
+            self.skipTest('could not find libc')
         libc = CDLL(libc_path)
 
         @CFUNCTYPE(c_int, POINTER(c_int), POINTER(c_int))
@@ -198,23 +199,19 @@
         libc.qsort(array, len(array), sizeof(c_int), cmp_func)
         self.assertEqual(array[:], [1, 5, 7, 33, 99])
 
-    try:
-        WINFUNCTYPE
-    except NameError:
-        pass
-    else:
-        def test_issue_8959_b(self):
-            from ctypes.wintypes import BOOL, HWND, LPARAM
+    @need_symbol('WINFUNCTYPE')
+    def test_issue_8959_b(self):
+        from ctypes.wintypes import BOOL, HWND, LPARAM
+        global windowCount
+        windowCount = 0
+
+        @WINFUNCTYPE(BOOL, HWND, LPARAM)
+        def EnumWindowsCallbackFunc(hwnd, lParam):
             global windowCount
-            windowCount = 0
+            windowCount += 1
+            return True #Allow windows to keep enumerating
 
-            @WINFUNCTYPE(BOOL, HWND, LPARAM)
-            def EnumWindowsCallbackFunc(hwnd, lParam):
-                global windowCount
-                windowCount += 1
-                return True #Allow windows to keep enumerating
-
-            windll.user32.EnumWindows(EnumWindowsCallbackFunc, 0)
+        windll.user32.EnumWindows(EnumWindowsCallbackFunc, 0)
 
     def test_callback_register_int(self):
         # Issue #8275: buggy handling of callback args under Win64
diff --git a/lib-python/2.7/ctypes/test/test_cast.py b/lib-python/2.7/ctypes/test/test_cast.py
--- a/lib-python/2.7/ctypes/test/test_cast.py
+++ b/lib-python/2.7/ctypes/test/test_cast.py
@@ -1,4 +1,5 @@
 from ctypes import *
+from ctypes.test import need_symbol
 import unittest
 import sys
 
@@ -38,14 +39,14 @@
 
         p = cast(array, POINTER(c_char_p))
         # array and p share a common _objects attribute
-        self.assertTrue(p._objects is array._objects)
+        self.assertIs(p._objects, array._objects)
         self.assertEqual(array._objects, {'0': "foo bar", id(array): array})
         p[0] = "spam spam"
         self.assertEqual(p._objects, {'0': "spam spam", id(array): array})
-        self.assertTrue(array._objects is p._objects)
+        self.assertIs(array._objects, p._objects)
         p[1] = "foo bar"
         self.assertEqual(p._objects, {'1': 'foo bar', '0': "spam spam", id(array): array})
-        self.assertTrue(array._objects is p._objects)
+        self.assertIs(array._objects, p._objects)
 
     def test_other(self):
         p = cast((c_int * 4)(1, 2, 3, 4), POINTER(c_int))
@@ -75,15 +76,11 @@
         self.assertEqual(cast(cast(s, c_void_p), c_char_p).value,
                              "hiho")
 
-    try:
-        c_wchar_p
-    except NameError:
-        pass
-    else:
-        def test_wchar_p(self):
-            s = c_wchar_p("hiho")
-            self.assertEqual(cast(cast(s, c_void_p), c_wchar_p).value,
-                                 "hiho")
+    @need_symbol('c_wchar_p')
+    def test_wchar_p(self):
+        s = c_wchar_p("hiho")
+        self.assertEqual(cast(cast(s, c_void_p), c_wchar_p).value,
+                             "hiho")
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_cfuncs.py b/lib-python/2.7/ctypes/test/test_cfuncs.py
--- a/lib-python/2.7/ctypes/test/test_cfuncs.py
+++ b/lib-python/2.7/ctypes/test/test_cfuncs.py
@@ -3,6 +3,7 @@
 
 import unittest
 from ctypes import *
+from ctypes.test import need_symbol
 
 import _ctypes_test
 from test.test_support import impl_detail
@@ -196,7 +197,7 @@
 try:
     WinDLL
 except NameError:
-    pass
+    def stdcall_dll(*_): pass
 else:
     class stdcall_dll(WinDLL):
         def __getattr__(self, name):
@@ -206,9 +207,9 @@
             setattr(self, name, func)
             return func
 
-    class stdcallCFunctions(CFunctions):
-        _dll = stdcall_dll(_ctypes_test.__file__)
-        pass
+ at need_symbol('WinDLL')
+class stdcallCFunctions(CFunctions):
+    _dll = stdcall_dll(_ctypes_test.__file__)
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_checkretval.py b/lib-python/2.7/ctypes/test/test_checkretval.py
--- a/lib-python/2.7/ctypes/test/test_checkretval.py
+++ b/lib-python/2.7/ctypes/test/test_checkretval.py
@@ -1,6 +1,7 @@
 import unittest
 
 from ctypes import *
+from ctypes.test import need_symbol
 
 class CHECKED(c_int):
     def _check_retval_(value):
@@ -25,15 +26,11 @@
         del dll._testfunc_p_p.restype
         self.assertEqual(42, dll._testfunc_p_p(42))
 
-    try:
-        oledll
-    except NameError:
-        pass
-    else:
-        def test_oledll(self):
-            self.assertRaises(WindowsError,
-                                  oledll.oleaut32.CreateTypeLib2,
-                                  0, None, None)
+    @need_symbol('oledll')
+    def test_oledll(self):
+        self.assertRaises(WindowsError,
+                              oledll.oleaut32.CreateTypeLib2,
+                              0, None, None)
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_errcheck.py b/lib-python/2.7/ctypes/test/test_errcheck.py
deleted file mode 100644
--- a/lib-python/2.7/ctypes/test/test_errcheck.py
+++ /dev/null
@@ -1,19 +0,0 @@
-import sys
-from ctypes import *
-
-##class HMODULE(Structure):
-##    _fields_ = [("value", c_void_p)]
-
-##    def __repr__(self):
-##        return "" % self.value
-
-##windll.kernel32.GetModuleHandleA.restype = HMODULE
-
-##print windll.kernel32.GetModuleHandleA("python23.dll")
-##print hex(sys.dllhandle)
-
-##def nonzero(handle):
-##    return (GetLastError(), handle)
-
-##windll.kernel32.GetModuleHandleA.errcheck = nonzero
-##print windll.kernel32.GetModuleHandleA("spam")
diff --git a/lib-python/2.7/ctypes/test/test_find.py b/lib-python/2.7/ctypes/test/test_find.py
--- a/lib-python/2.7/ctypes/test/test_find.py
+++ b/lib-python/2.7/ctypes/test/test_find.py
@@ -1,4 +1,5 @@
 import unittest
+import os
 import sys
 from ctypes import *
 from ctypes.util import find_library
@@ -40,43 +41,43 @@
             except OSError:
                 pass
 
-    if lib_gl:
-        def test_gl(self):
-            if self.gl:
-                self.gl.glClearIndex
+    @unittest.skipUnless(lib_gl, 'lib_gl not available')
+    def test_gl(self):
+        if self.gl:
+            self.gl.glClearIndex
 
-    if lib_glu:
-        def test_glu(self):
-            if self.glu:
-                self.glu.gluBeginCurve
+    @unittest.skipUnless(lib_glu, 'lib_glu not available')
+    def test_glu(self):
+        if self.glu:
+            self.glu.gluBeginCurve
 
-    if lib_gle:
-        def test_gle(self):
-            if self.gle:
-                self.gle.gleGetJoinStyle
+    @unittest.skipUnless(lib_gle, 'lib_gle not available')
+    def test_gle(self):
+        if self.gle:
+            self.gle.gleGetJoinStyle
 
-##if os.name == "posix" and sys.platform != "darwin":
-
-##    # On platforms where the default shared library suffix is '.so',
-##    # at least some libraries can be loaded as attributes of the cdll
-##    # object, since ctypes now tries loading the lib again
-##    # with '.so' appended of the first try fails.
-##    #
-##    # Won't work for libc, unfortunately.  OTOH, it isn't
-##    # needed for libc since this is already mapped into the current
-##    # process (?)
-##    #
-##    # On MAC OSX, it won't work either, because dlopen() needs a full path,
-##    # and the default suffix is either none or '.dylib'.
-
-##    class LoadLibs(unittest.TestCase):
-##        def test_libm(self):
-##            import math
-##            libm = cdll.libm
-##            sqrt = libm.sqrt
-##            sqrt.argtypes = (c_double,)
-##            sqrt.restype = c_double
-##            self.assertEqual(sqrt(2), math.sqrt(2))
+# On platforms where the default shared library suffix is '.so',
+# at least some libraries can be loaded as attributes of the cdll
+# object, since ctypes now tries loading the lib again
+# with '.so' appended of the first try fails.
+#
+# Won't work for libc, unfortunately.  OTOH, it isn't
+# needed for libc since this is already mapped into the current
+# process (?)
+#
+# On MAC OSX, it won't work either, because dlopen() needs a full path,
+# and the default suffix is either none or '.dylib'.
+ at unittest.skip('test disabled')
+ at unittest.skipUnless(os.name=="posix" and sys.platform != "darwin",
+                     'test not suitable for this platform')
+class LoadLibs(unittest.TestCase):
+    def test_libm(self):
+        import math
+        libm = cdll.libm
+        sqrt = libm.sqrt
+        sqrt.argtypes = (c_double,)
+        sqrt.restype = c_double
+        self.assertEqual(sqrt(2), math.sqrt(2))
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_frombuffer.py b/lib-python/2.7/ctypes/test/test_frombuffer.py
--- a/lib-python/2.7/ctypes/test/test_frombuffer.py
+++ b/lib-python/2.7/ctypes/test/test_frombuffer.py
@@ -25,7 +25,7 @@
         a[0], a[-1] = 200, -200
         self.assertEqual(x[:], a.tolist())
 
-        self.assertTrue(a in x._objects.values())
+        self.assertIn(a, x._objects.values())
 
         self.assertRaises(ValueError,
                           c_int.from_buffer, a, -1)
diff --git a/lib-python/2.7/ctypes/test/test_funcptr.py b/lib-python/2.7/ctypes/test/test_funcptr.py
--- a/lib-python/2.7/ctypes/test/test_funcptr.py
+++ b/lib-python/2.7/ctypes/test/test_funcptr.py
@@ -75,7 +75,7 @@
         ##                  "lpfnWndProc", WNDPROC_2(wndproc))
         # instead:
 
-        self.assertTrue(WNDPROC is WNDPROC_2)
+        self.assertIs(WNDPROC, WNDPROC_2)
         # 'wndclass.lpfnWndProc' leaks 94 references.  Why?
         self.assertEqual(wndclass.lpfnWndProc(1, 2, 3, 4), 10)
 
diff --git a/lib-python/2.7/ctypes/test/test_functions.py b/lib-python/2.7/ctypes/test/test_functions.py
--- a/lib-python/2.7/ctypes/test/test_functions.py
+++ b/lib-python/2.7/ctypes/test/test_functions.py
@@ -6,6 +6,7 @@
 """
 
 from ctypes import *
+from ctypes.test import need_symbol
 import sys, unittest
 from ctypes.test import xfail
 from test.test_support import impl_detail
@@ -65,22 +66,16 @@
             pass
 
 
+    @need_symbol('c_wchar')
     def test_wchar_parm(self):
-        try:
-            c_wchar
-        except NameError:
-            return
         f = dll._testfunc_i_bhilfd
         f.argtypes = [c_byte, c_wchar, c_int, c_long, c_float, c_double]
         result = f(1, u"x", 3, 4, 5.0, 6.0)
         self.assertEqual(result, 139)
         self.assertEqual(type(result), int)
 
+    @need_symbol('c_wchar')
     def test_wchar_result(self):
-        try:
-            c_wchar
-        except NameError:
-            return
         f = dll._testfunc_i_bhilfd
         f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double]
         f.restype = c_wchar
@@ -158,11 +153,8 @@
         self.assertEqual(result, -21)

From noreply at buildbot.pypy.org  Fri Sep 12 00:15:21 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Fri, 12 Sep 2014 00:15:21 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: partially backout 9831468e1882: readapt to
	py3
Message-ID: <20140911221521.3B5F81D2D6C@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey 
Branch: py3k
Changeset: r73487:e577a2f0be5d
Date: 2014-09-11 15:07 -0700
http://bitbucket.org/pypy/pypy/changeset/e577a2f0be5d/

Log:	partially backout 9831468e1882: readapt to py3

diff --git a/pypy/module/operator/test/test_tscmp.py b/pypy/module/operator/test/test_tscmp.py
--- a/pypy/module/operator/test/test_tscmp.py
+++ b/pypy/module/operator/test/test_tscmp.py
@@ -1,28 +1,14 @@
-from pypy.module.operator.tscmp import pypy_tscmp, pypy_tscmp_wide
+from pypy.module.operator.tscmp import pypy_tscmp
 
 class TestTimingSafeCompare:
-    tostr = str
-    tscmp = staticmethod(pypy_tscmp)
-
     def test_tscmp_neq(self):
-        assert not self.tscmp(self.tostr('asd'), self.tostr('qwe'), 3, 3)
+        assert not pypy_tscmp('asd', 'qwe', 3, 3)
 
     def test_tscmp_eq(self):
-        assert self.tscmp(self.tostr('asd'), self.tostr('asd'), 3, 3)
+        assert pypy_tscmp('asd', 'asd', 3, 3)
 
     def test_tscmp_len(self):
-        assert self.tscmp(self.tostr('asdp'), self.tostr('asdq'), 3, 3)
+        assert pypy_tscmp('asdp', 'asdq', 3, 3)
 
     def test_tscmp_nlen(self):
-        assert not self.tscmp(self.tostr('asd'), self.tostr('asd'), 2, 3)
-
-
-class TestTimingSafeCompareWide(TestTimingSafeCompare):
-    tostr = unicode
-    tscmp = staticmethod(pypy_tscmp_wide)
-
-    def test_tscmp_wide_nonascii(self):
-        a, b = u"\ud808\udf45", u"\ud808\udf45"
-        assert self.tscmp(a, b, len(a), len(b))
-        a, b = u"\ud808\udf45", u"\ud808\udf45 "
-        assert not self.tscmp(a, b, len(a), len(b))
+        assert not pypy_tscmp('asd', 'asd', 2, 3)
diff --git a/pypy/module/operator/tscmp.c b/pypy/module/operator/tscmp.c
--- a/pypy/module/operator/tscmp.c
+++ b/pypy/module/operator/tscmp.c
@@ -2,7 +2,6 @@
  */
 
 #include 
-#include 
 #include "tscmp.h"
 
 int
@@ -41,40 +40,3 @@
 
     return (result == 0);
 }
-
-int
-pypy_tscmp_wide(const wchar_t *a, const wchar_t *b, long len_a, long len_b)
-{
-    /* The volatile type declarations make sure that the compiler has no
-     * chance to optimize and fold the code in any way that may change
-     * the timing.
-     */
-    volatile long length;
-    volatile const wchar_t *left;
-    volatile const wchar_t *right;
-    long i;
-    wchar_t result;
-
-    /* loop count depends on length of b */
-    length = len_b;
-    left = NULL;
-    right = b;
-
-    /* don't use else here to keep the amount of CPU instructions constant,
-     * volatile forces re-evaluation
-     *  */
-    if (len_a == length) {
-        left = *((volatile const wchar_t**)&a);
-        result = 0;
-    }
-    if (len_a != length) {
-        left = b;
-        result = 1;
-    }
-
-    for (i=0; i < length; i++) {
-        result |= *left++ ^ *right++;
-    }
-
-    return (result == 0);
-}
diff --git a/pypy/module/operator/tscmp.h b/pypy/module/operator/tscmp.h
--- a/pypy/module/operator/tscmp.h
+++ b/pypy/module/operator/tscmp.h
@@ -1,2 +1,1 @@
 int pypy_tscmp(const char *, const char *, long, long);
-int pypy_tscmp_wide(const wchar_t *, const wchar_t *, long, long);
diff --git a/pypy/module/operator/tscmp.py b/pypy/module/operator/tscmp.py
--- a/pypy/module/operator/tscmp.py
+++ b/pypy/module/operator/tscmp.py
@@ -7,14 +7,14 @@
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 
-from pypy.interpreter.error import oefmt
+from pypy.interpreter.error import OperationError, oefmt
 
 cwd = py.path.local(__file__).dirpath()
 eci = ExternalCompilationInfo(
     includes=[cwd.join('tscmp.h')],
     include_dirs=[str(cwd)],
     separate_module_files=[cwd.join('tscmp.c')],
-    export_symbols=['pypy_tscmp', 'pypy_tscmp_wide'])
+    export_symbols=['pypy_tscmp'])
 
 
 def llexternal(*args, **kwargs):
@@ -27,10 +27,6 @@
     'pypy_tscmp',
     [rffi.CCHARP, rffi.CCHARP, rffi.LONG, rffi.LONG],
     rffi.INT)
-pypy_tscmp_wide = llexternal(
-    'pypy_tscmp_wide',
-    [rffi.CWCHARP, rffi.CWCHARP, rffi.LONG, rffi.LONG],
-    rffi.INT)
 
 
 def compare_digest(space, w_a, w_b):
@@ -47,26 +43,21 @@
     """
     if (space.isinstance_w(w_a, space.w_unicode) and
         space.isinstance_w(w_b, space.w_unicode)):
-        a = space.unicode_w(w_a)
-        b = space.unicode_w(w_b)
-        with rffi.scoped_nonmoving_unicodebuffer(a) as a_buf:
-            with rffi.scoped_nonmoving_unicodebuffer(b) as b_buf:
-                result = pypy_tscmp_wide(a_buf, b_buf, len(a), len(b))
-        return space.wrap(rffi.cast(lltype.Bool, result))
+        try:
+            w_a = space.call_method(w_a, 'encode', space.wrap('ascii'))
+            w_b = space.call_method(w_b, 'encode', space.wrap('ascii'))
+        except OperationError as e:
+            if not e.match(space, space.w_UnicodeEncodeError):
+                raise
+            raise oefmt(space.w_TypeError,
+                        "comparing strings with non-ASCII characters is not "
+                        "supported")
     return compare_digest_buffer(space, w_a, w_b)
 
 
 def compare_digest_buffer(space, w_a, w_b):
-    try:
-        a_buf = w_a.buffer_w(space, space.BUF_SIMPLE)
-        b_buf = w_b.buffer_w(space, space.BUF_SIMPLE)
-    except TypeError:
-        raise oefmt(space.w_TypeError,
-                    "unsupported operand types(s) or combination of types: "
-                    "'%T' and '%T'", w_a, w_b)
-
-    a = a_buf.as_str()
-    b = b_buf.as_str()
+    a = space.bufferstr_w(w_a)
+    b = space.bufferstr_w(w_b)
     with rffi.scoped_nonmovingbuffer(a) as a_buf:
         with rffi.scoped_nonmovingbuffer(b) as b_buf:
             result = pypy_tscmp(a_buf, b_buf, len(a), len(b))

From noreply at buildbot.pypy.org  Fri Sep 12 00:15:22 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Fri, 12 Sep 2014 00:15:22 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: adapt to py3
Message-ID: <20140911221522.7BE641D2D6C@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey 
Branch: py3k
Changeset: r73488:ebbb9950dfa9
Date: 2014-09-11 15:07 -0700
http://bitbucket.org/pypy/pypy/changeset/ebbb9950dfa9/

Log:	adapt to py3

diff --git a/pypy/module/operator/test/test_operator.py b/pypy/module/operator/test/test_operator.py
--- a/pypy/module/operator/test/test_operator.py
+++ b/pypy/module/operator/test/test_operator.py
@@ -272,7 +272,7 @@
         a, b = 100, 200
         raises(TypeError, operator._compare_digest, a, b)
         a, b = "fooä", "fooä"
-        assert operator._compare_digest(a, b)
+        raises(TypeError, operator._compare_digest, a, b)
 
         # subclasses are supported by ignore __eq__
         class mystr(str):

From noreply at buildbot.pypy.org  Fri Sep 12 00:15:23 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Fri, 12 Sep 2014 00:15:23 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: avoid the overridable via subclass encode
	method
Message-ID: <20140911221523.C4F651D2D6C@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey 
Branch: py3k
Changeset: r73489:a9fef4fe325f
Date: 2014-09-11 15:12 -0700
http://bitbucket.org/pypy/pypy/changeset/a9fef4fe325f/

Log:	avoid the overridable via subclass encode method

diff --git a/pypy/module/operator/tscmp.py b/pypy/module/operator/tscmp.py
--- a/pypy/module/operator/tscmp.py
+++ b/pypy/module/operator/tscmp.py
@@ -8,6 +8,7 @@
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 
 from pypy.interpreter.error import OperationError, oefmt
+from pypy.interpreter.unicodehelper import encode
 
 cwd = py.path.local(__file__).dirpath()
 eci = ExternalCompilationInfo(
@@ -44,8 +45,8 @@
     if (space.isinstance_w(w_a, space.w_unicode) and
         space.isinstance_w(w_b, space.w_unicode)):
         try:
-            w_a = space.call_method(w_a, 'encode', space.wrap('ascii'))
-            w_b = space.call_method(w_b, 'encode', space.wrap('ascii'))
+            w_a = encode(space, w_a, 'ascii')
+            w_b = encode(space, w_b, 'ascii')
         except OperationError as e:
             if not e.match(space, space.w_UnicodeEncodeError):
                 raise

From noreply at buildbot.pypy.org  Fri Sep 12 00:15:25 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Fri, 12 Sep 2014 00:15:25 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: 2to3
Message-ID: <20140911221525.0DD011D2D6C@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey 
Branch: py3k
Changeset: r73490:71b29c6796ae
Date: 2014-09-11 15:14 -0700
http://bitbucket.org/pypy/pypy/changeset/71b29c6796ae/

Log:	2to3

diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -1107,7 +1107,7 @@
             assert sys.executable == ''      # not executable!
             assert sys.path == old_sys_path + [self.goal_dir]
 
-            os.chmod(self.fake_exe, 0755)
+            os.chmod(self.fake_exe, 0o755)
             app_main.setup_bootstrap_path(self.fake_exe)
             assert sys.executable == self.fake_exe
             assert self.goal_dir not in sys.path

From noreply at buildbot.pypy.org  Fri Sep 12 00:15:26 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Fri, 12 Sep 2014 00:15:26 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: merge py3k
Message-ID: <20140911221526.4C8941D2D6C@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey 
Branch: py3.3
Changeset: r73491:a4b792a114c2
Date: 2014-09-11 15:14 -0700
http://bitbucket.org/pypy/pypy/changeset/a4b792a114c2/

Log:	merge py3k

diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -1107,7 +1107,7 @@
             assert sys.executable == ''      # not executable!
             assert sys.path == old_sys_path + [self.goal_dir]
 
-            os.chmod(self.fake_exe, 0755)
+            os.chmod(self.fake_exe, 0o755)
             app_main.setup_bootstrap_path(self.fake_exe)
             assert sys.executable == self.fake_exe
             assert self.goal_dir not in sys.path
diff --git a/pypy/module/operator/test/test_tscmp.py b/pypy/module/operator/test/test_tscmp.py
--- a/pypy/module/operator/test/test_tscmp.py
+++ b/pypy/module/operator/test/test_tscmp.py
@@ -1,28 +1,14 @@
-from pypy.module.operator.tscmp import pypy_tscmp, pypy_tscmp_wide
+from pypy.module.operator.tscmp import pypy_tscmp
 
 class TestTimingSafeCompare:
-    tostr = str
-    tscmp = staticmethod(pypy_tscmp)
-
     def test_tscmp_neq(self):
-        assert not self.tscmp(self.tostr('asd'), self.tostr('qwe'), 3, 3)
+        assert not pypy_tscmp('asd', 'qwe', 3, 3)
 
     def test_tscmp_eq(self):
-        assert self.tscmp(self.tostr('asd'), self.tostr('asd'), 3, 3)
+        assert pypy_tscmp('asd', 'asd', 3, 3)
 
     def test_tscmp_len(self):
-        assert self.tscmp(self.tostr('asdp'), self.tostr('asdq'), 3, 3)
+        assert pypy_tscmp('asdp', 'asdq', 3, 3)
 
     def test_tscmp_nlen(self):
-        assert not self.tscmp(self.tostr('asd'), self.tostr('asd'), 2, 3)
-
-
-class TestTimingSafeCompareWide(TestTimingSafeCompare):
-    tostr = unicode
-    tscmp = staticmethod(pypy_tscmp_wide)
-
-    def test_tscmp_wide_nonascii(self):
-        a, b = u"\ud808\udf45", u"\ud808\udf45"
-        assert self.tscmp(a, b, len(a), len(b))
-        a, b = u"\ud808\udf45", u"\ud808\udf45 "
-        assert not self.tscmp(a, b, len(a), len(b))
+        assert not pypy_tscmp('asd', 'asd', 2, 3)
diff --git a/pypy/module/operator/tscmp.c b/pypy/module/operator/tscmp.c
--- a/pypy/module/operator/tscmp.c
+++ b/pypy/module/operator/tscmp.c
@@ -2,7 +2,6 @@
  */
 
 #include 
-#include 
 #include "tscmp.h"
 
 int
@@ -41,40 +40,3 @@
 
     return (result == 0);
 }
-
-int
-pypy_tscmp_wide(const wchar_t *a, const wchar_t *b, long len_a, long len_b)
-{
-    /* The volatile type declarations make sure that the compiler has no
-     * chance to optimize and fold the code in any way that may change
-     * the timing.
-     */
-    volatile long length;
-    volatile const wchar_t *left;
-    volatile const wchar_t *right;
-    long i;
-    wchar_t result;
-
-    /* loop count depends on length of b */
-    length = len_b;
-    left = NULL;
-    right = b;
-
-    /* don't use else here to keep the amount of CPU instructions constant,
-     * volatile forces re-evaluation
-     *  */
-    if (len_a == length) {
-        left = *((volatile const wchar_t**)&a);
-        result = 0;
-    }
-    if (len_a != length) {
-        left = b;
-        result = 1;
-    }
-
-    for (i=0; i < length; i++) {
-        result |= *left++ ^ *right++;
-    }
-
-    return (result == 0);
-}
diff --git a/pypy/module/operator/tscmp.h b/pypy/module/operator/tscmp.h
--- a/pypy/module/operator/tscmp.h
+++ b/pypy/module/operator/tscmp.h
@@ -1,2 +1,1 @@
 int pypy_tscmp(const char *, const char *, long, long);
-int pypy_tscmp_wide(const wchar_t *, const wchar_t *, long, long);
diff --git a/pypy/module/operator/tscmp.py b/pypy/module/operator/tscmp.py
--- a/pypy/module/operator/tscmp.py
+++ b/pypy/module/operator/tscmp.py
@@ -7,14 +7,15 @@
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 
-from pypy.interpreter.error import oefmt
+from pypy.interpreter.error import OperationError, oefmt
+from pypy.interpreter.unicodehelper import encode
 
 cwd = py.path.local(__file__).dirpath()
 eci = ExternalCompilationInfo(
     includes=[cwd.join('tscmp.h')],
     include_dirs=[str(cwd)],
     separate_module_files=[cwd.join('tscmp.c')],
-    export_symbols=['pypy_tscmp', 'pypy_tscmp_wide'])
+    export_symbols=['pypy_tscmp'])
 
 
 def llexternal(*args, **kwargs):
@@ -27,10 +28,6 @@
     'pypy_tscmp',
     [rffi.CCHARP, rffi.CCHARP, rffi.LONG, rffi.LONG],
     rffi.INT)
-pypy_tscmp_wide = llexternal(
-    'pypy_tscmp_wide',
-    [rffi.CWCHARP, rffi.CWCHARP, rffi.LONG, rffi.LONG],
-    rffi.INT)
 
 
 def compare_digest(space, w_a, w_b):
@@ -47,26 +44,21 @@
     """
     if (space.isinstance_w(w_a, space.w_unicode) and
         space.isinstance_w(w_b, space.w_unicode)):
-        a = space.unicode_w(w_a)
-        b = space.unicode_w(w_b)
-        with rffi.scoped_nonmoving_unicodebuffer(a) as a_buf:
-            with rffi.scoped_nonmoving_unicodebuffer(b) as b_buf:
-                result = pypy_tscmp_wide(a_buf, b_buf, len(a), len(b))
-        return space.wrap(rffi.cast(lltype.Bool, result))
+        try:
+            w_a = encode(space, w_a, 'ascii')
+            w_b = encode(space, w_b, 'ascii')
+        except OperationError as e:
+            if not e.match(space, space.w_UnicodeEncodeError):
+                raise
+            raise oefmt(space.w_TypeError,
+                        "comparing strings with non-ASCII characters is not "
+                        "supported")
     return compare_digest_buffer(space, w_a, w_b)
 
 
 def compare_digest_buffer(space, w_a, w_b):
-    try:
-        a_buf = w_a.buffer_w(space, space.BUF_SIMPLE)
-        b_buf = w_b.buffer_w(space, space.BUF_SIMPLE)
-    except TypeError:
-        raise oefmt(space.w_TypeError,
-                    "unsupported operand types(s) or combination of types: "
-                    "'%T' and '%T'", w_a, w_b)
-
-    a = a_buf.as_str()
-    b = b_buf.as_str()
+    a = space.bufferstr_w(w_a)
+    b = space.bufferstr_w(w_b)
     with rffi.scoped_nonmovingbuffer(a) as a_buf:
         with rffi.scoped_nonmovingbuffer(b) as b_buf:
             result = pypy_tscmp(a_buf, b_buf, len(a), len(b))

From noreply at buildbot.pypy.org  Fri Sep 12 00:58:18 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Fri, 12 Sep 2014 00:58:18 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: merge default
Message-ID: <20140911225818.B589A1C3637@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey 
Branch: py3k
Changeset: r73492:8fbe4a728f3e
Date: 2014-09-11 15:57 -0700
http://bitbucket.org/pypy/pypy/changeset/8fbe4a728f3e/

Log:	merge default

diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -923,7 +923,7 @@
             else:
                 r, w, e = rpoll.select([sock_fd], [], [], sock_timeout)
                 ready = r
-        except SelectError, e:
+        except rpoll.SelectError as e:
             message = e.get_msg()
             raise ssl_error(space, message, e.errno)
     if ready:

From noreply at buildbot.pypy.org  Fri Sep 12 00:58:20 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Fri, 12 Sep 2014 00:58:20 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: merge py3k
Message-ID: <20140911225820.23B571C3637@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey 
Branch: py3.3
Changeset: r73493:6a2b74eae1e7
Date: 2014-09-11 15:57 -0700
http://bitbucket.org/pypy/pypy/changeset/6a2b74eae1e7/

Log:	merge py3k

diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -930,7 +930,7 @@
             else:
                 r, w, e = rpoll.select([sock_fd], [], [], sock_timeout)
                 ready = r
-        except SelectError, e:
+        except rpoll.SelectError as e:
             message = e.get_msg()
             raise ssl_error(space, message, e.errno)
     if ready:

From noreply at buildbot.pypy.org  Fri Sep 12 01:11:19 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri, 12 Sep 2014 01:11:19 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: use getc_unlocked when
	possible
Message-ID: <20140911231119.F25641C1036@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73494:06cf208f624d
Date: 2014-09-11 16:10 -0700
http://bitbucket.org/pypy/pypy/changeset/06cf208f624d/

Log:	use getc_unlocked when possible

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -89,9 +89,18 @@
 _fclose2 = (c_fclose, c_fclose_nogil)
 _pclose2 = (c_pclose, c_pclose_nogil)
 
-c_getc = llexternal('getc', [FILEP], rffi.INT, macro=True, releasegil=False)
+c_flockfile = llexternal('flockfile', [FILEP], lltype.Void, releasegil=False)
+c_funlockfile = llexternal('funlockfile', [FILEP], lltype.Void, releasegil=False)
+
+c_getc = llexternal('getc', [FILEP], rffi.INT, releasegil=False)
+c_getc_unlocked = llexternal('getc_unlocked', [FILEP], rffi.INT, releasegil=False)
 c_ungetc = llexternal('ungetc', [rffi.INT, FILEP], rffi.INT, releasegil=False)
 
+if os.name == 'nt':
+    c_flockfile = lambda ll_file: None
+    c_funlockfile = lambda ll_file: None
+    c_getc_unlocked = c_getc
+
 c_fgets = llexternal('fgets', [rffi.CCHARP, rffi.INT, FILEP], rffi.CCHARP)
 c_fread = llexternal('fread', [rffi.CCHARP, rffi.SIZE_T, rffi.SIZE_T, FILEP],
                      rffi.SIZE_T)
@@ -488,16 +497,17 @@
             c = 0
             s = StringBuilder()
             while True:
+                c_flockfile(ll_file)
                 if self._univ_newline:
                     while size < 0 or s.getlength() < size:
-                        c = c_getc(ll_file)
+                        c = c_getc_unlocked(ll_file)
                         if c == EOF:
                             break
                         if skipnextlf:
                             skipnextlf = False
                             if c == ord('\n'):
                                 newlinetypes |= NEWLINE_CRLF
-                                c = c_getc(ll_file)
+                                c = c_getc_unlocked(ll_file)
                                 if c == EOF:
                                     break
                             else:
@@ -512,6 +522,7 @@
                             break
                     if c == EOF:
                         if c_ferror(ll_file) and rposix.get_errno() == errno.EINTR:
+                            c_funlockfile(ll_file)
                             self._newlinetypes = newlinetypes
                             self._skipnextlf = skipnextlf
                             if self._signal_checker is not None:
@@ -522,12 +533,13 @@
                             newlinetypes |= NEWLINE_CR
                 else:
                     while s.getlength() < size:
-                        c = c_getc(ll_file)
+                        c = c_getc_unlocked(ll_file)
                         if c == EOF:
                             break
                         s.append(chr(c))
                         if c == ord('\n'):
                             break
+                c_funlockfile(ll_file)
                 self._newlinetypes = newlinetypes
                 self._skipnextlf = skipnextlf
                 if c == ord('\n'):

From noreply at buildbot.pypy.org  Fri Sep 12 01:28:47 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri, 12 Sep 2014 01:28:47 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: prefer getc_unlocked
 over fgets in readline
Message-ID: <20140911232847.0C08F1C1036@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73495:a92bdb7b7707
Date: 2014-09-11 19:26 -0400
http://bitbucket.org/pypy/pypy/changeset/a92bdb7b7707/

Log:	prefer getc_unlocked over fgets in readline

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -100,6 +100,9 @@
     c_flockfile = lambda ll_file: None
     c_funlockfile = lambda ll_file: None
     c_getc_unlocked = c_getc
+    USE_FGETS_IN_GETLINE = True
+else:
+    USE_FGETS_IN_GETLINE = False
 
 c_fgets = llexternal('fgets', [rffi.CCHARP, rffi.INT, FILEP], rffi.CCHARP)
 c_fread = llexternal('fread', [rffi.CCHARP, rffi.SIZE_T, rffi.SIZE_T, FILEP],
@@ -435,7 +438,7 @@
             rffi.keep_buffer_alive_until_here(raw_buf, gc_buf)
         return s.build()
 
-    def _readline1(self, raw_buf):
+    def _get_line_fgets_single(self, raw_buf):
         ll_file = self._ll_file
         for i in range(BASE_LINE_SIZE):
             raw_buf[i] = '\n'
@@ -470,97 +473,103 @@
             assert p > 0 and raw_buf[p - 1] == '\0'
             return p - 1
 
+    def _get_line_fgets(self):
+        with rffi.scoped_alloc_buffer(BASE_LINE_SIZE) as buf:
+            c = self._get_line_fgets_single(buf.raw)
+            if c >= 0:
+                return buf.str(c)
+
+            # this is the rare case: the line is longer than BASE_LINE_SIZE
+            s = StringBuilder()
+            while True:
+                s.append_charpsize(buf.raw, BASE_LINE_SIZE - 1)
+                c = self._get_line_fgets_single(buf.raw)
+                if c >= 0:
+                    break
+            s.append_charpsize(buf.raw, c)
+        return s.build()
+
+    def _get_line(self, size):
+        if USE_FGETS_IN_GETLINE and size < 0 and not self._univ_newline:
+            return self._get_line_fgets()
+
+        ll_file = self._ll_file
+        newlinetypes = self._newlinetypes
+        skipnextlf = self._skipnextlf
+        c = 0
+        s = StringBuilder()
+        while True:
+            c_flockfile(ll_file)
+            if self._univ_newline:
+                while size < 0 or s.getlength() < size:
+                    c = c_getc_unlocked(ll_file)
+                    if c == EOF:
+                        break
+                    if skipnextlf:
+                        skipnextlf = False
+                        if c == ord('\n'):
+                            newlinetypes |= NEWLINE_CRLF
+                            c = c_getc_unlocked(ll_file)
+                            if c == EOF:
+                                break
+                        else:
+                            newlinetypes |= NEWLINE_CR
+                    if c == ord('\r'):
+                        skipnextlf = True
+                        c = ord('\n')
+                    elif c == ord('\n'):
+                        newlinetypes |= NEWLINE_LF
+                    s.append(chr(c))
+                    if c == ord('\n'):
+                        break
+                if c == EOF:
+                    if c_ferror(ll_file) and rposix.get_errno() == errno.EINTR:
+                        c_funlockfile(ll_file)
+                        self._newlinetypes = newlinetypes
+                        self._skipnextlf = skipnextlf
+                        if self._signal_checker is not None:
+                            self._signal_checker()
+                        c_clearerr(ll_file)
+                        continue
+                    if skipnextlf:
+                        newlinetypes |= NEWLINE_CR
+            else:
+                while size < 0 or s.getlength() < size:
+                    c = c_getc_unlocked(ll_file)
+                    if c == EOF:
+                        break
+                    s.append(chr(c))
+                    if c == ord('\n'):
+                        break
+            c_funlockfile(ll_file)
+            self._newlinetypes = newlinetypes
+            self._skipnextlf = skipnextlf
+            if c == ord('\n'):
+                break
+            elif c == EOF:
+                if c_ferror(ll_file):
+                    if rposix.get_errno() == errno.EINTR:
+                        if self._signal_checker is not None:
+                            self._signal_checker()
+                        c_clearerr(ll_file)
+                        continue
+                    c_clearerr(ll_file)
+                    raise _from_errno(IOError)
+                c_clearerr(ll_file)
+                if self._signal_checker is not None:
+                    self._signal_checker()
+                break
+            else:
+                assert s.getlength() == size
+                break
+        return s.build()
+
     def readline(self, size=-1):
         self._check_closed()
         self._check_readable()
         if size == 0:
             return ""
-        elif size < 0 and not self._univ_newline:
-            with rffi.scoped_alloc_buffer(BASE_LINE_SIZE) as buf:
-                c = self._readline1(buf.raw)
-                if c >= 0:
-                    return buf.str(c)
-
-                # this is the rare case: the line is longer than BASE_LINE_SIZE
-                s = StringBuilder()
-                while True:
-                    s.append_charpsize(buf.raw, BASE_LINE_SIZE - 1)
-                    c = self._readline1(buf.raw)
-                    if c >= 0:
-                        break
-                s.append_charpsize(buf.raw, c)
-            return s.build()
-        else:  # size > 0 or self._univ_newline
-            ll_file = self._ll_file
-            newlinetypes = self._newlinetypes
-            skipnextlf = self._skipnextlf
-            c = 0
-            s = StringBuilder()
-            while True:
-                c_flockfile(ll_file)
-                if self._univ_newline:
-                    while size < 0 or s.getlength() < size:
-                        c = c_getc_unlocked(ll_file)
-                        if c == EOF:
-                            break
-                        if skipnextlf:
-                            skipnextlf = False
-                            if c == ord('\n'):
-                                newlinetypes |= NEWLINE_CRLF
-                                c = c_getc_unlocked(ll_file)
-                                if c == EOF:
-                                    break
-                            else:
-                                newlinetypes |= NEWLINE_CR
-                        if c == ord('\r'):
-                            skipnextlf = True
-                            c = ord('\n')
-                        elif c == ord('\n'):
-                            newlinetypes |= NEWLINE_LF
-                        s.append(chr(c))
-                        if c == ord('\n'):
-                            break
-                    if c == EOF:
-                        if c_ferror(ll_file) and rposix.get_errno() == errno.EINTR:
-                            c_funlockfile(ll_file)
-                            self._newlinetypes = newlinetypes
-                            self._skipnextlf = skipnextlf
-                            if self._signal_checker is not None:
-                                self._signal_checker()
-                            c_clearerr(ll_file)
-                            continue
-                        if skipnextlf:
-                            newlinetypes |= NEWLINE_CR
-                else:
-                    while s.getlength() < size:
-                        c = c_getc_unlocked(ll_file)
-                        if c == EOF:
-                            break
-                        s.append(chr(c))
-                        if c == ord('\n'):
-                            break
-                c_funlockfile(ll_file)
-                self._newlinetypes = newlinetypes
-                self._skipnextlf = skipnextlf
-                if c == ord('\n'):
-                    break
-                elif c == EOF:
-                    if c_ferror(ll_file):
-                        if rposix.get_errno() == errno.EINTR:
-                            if self._signal_checker is not None:
-                                self._signal_checker()
-                            c_clearerr(ll_file)
-                            continue
-                        c_clearerr(ll_file)
-                        raise _from_errno(IOError)
-                    c_clearerr(ll_file)
-                    if self._signal_checker is not None:
-                        self._signal_checker()
-                    break
-                else:
-                    assert s.getlength() == size
-                    break
-            return s.build()
+        return self._get_line(size)
 
     @enforceargs(None, str)
     def write(self, value):

From noreply at buildbot.pypy.org  Fri Sep 12 02:44:34 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri, 12 Sep 2014 02:44:34 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: this test fails on
 cpython on mac/win, adjust
Message-ID: <20140912004434.9DC9C1C31C8@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73496:120342e28cf1
Date: 2014-09-11 17:43 -0700
http://bitbucket.org/pypy/pypy/changeset/120342e28cf1/

Log:	this test fails on cpython on mac/win, adjust

diff --git a/pypy/module/_file/test/test_file_extra.py b/pypy/module/_file/test/test_file_extra.py
--- a/pypy/module/_file/test/test_file_extra.py
+++ b/pypy/module/_file/test/test_file_extra.py
@@ -431,6 +431,7 @@
         f.close()
 
     def test_rw_bin(self):
+        import sys
         import random
         flags = 'w+b'
         checkflags = 'rb'
@@ -440,44 +441,49 @@
         f = file(fn, flags)
         expected = ''
         pos = 0
-        for i in range(5000):
-            x = random.random()
-            if x < 0.4:
-                l = int(x*100)
-                buf = f.read(l)
-                assert buf == expected[pos:pos+l]
-                pos += len(buf)
-            elif x < 0.75:
-                writeeol, expecteol = random.choice(eolstyles)
-                x = str(x)
-                buf1 = x+writeeol
-                buf2 = x+expecteol
-                f.write(buf1)
-                expected = expected[:pos] + buf2 + expected[pos+len(buf2):]
-                pos += len(buf2)
-            elif x < 0.80:
-                pos = random.randint(0, len(expected))
-                f.seek(pos)
-            elif x < 0.85:
-                pos = random.randint(0, len(expected))
-                f.seek(pos - len(expected), 2)
-            elif x < 0.90:
-                currentpos = pos
-                pos = random.randint(0, len(expected))
-                f.seek(pos - currentpos, 1)
-            elif x < 0.95:
-                assert f.tell() == pos
-            else:
-                f.flush()
-                g = open(fn, checkflags)
-                buf = g.read()
-                g.close()
-                assert buf == expected
-        f.close()
-        g = open(fn, checkflags)
-        buf = g.read()
-        g.close()
-        assert buf == expected
+        try:
+            for i in range(5000):
+                x = random.random()
+                if x < 0.4:
+                    l = int(x*100)
+                    buf = f.read(l)
+                    assert buf == expected[pos:pos+l]
+                    pos += len(buf)
+                elif x < 0.75:
+                    writeeol, expecteol = random.choice(eolstyles)
+                    x = str(x)
+                    buf1 = x+writeeol
+                    buf2 = x+expecteol
+                    f.write(buf1)
+                    expected = expected[:pos] + buf2 + expected[pos+len(buf2):]
+                    pos += len(buf2)
+                elif x < 0.80:
+                    pos = random.randint(0, len(expected))
+                    f.seek(pos)
+                elif x < 0.85:
+                    pos = random.randint(0, len(expected))
+                    f.seek(pos - len(expected), 2)
+                elif x < 0.90:
+                    currentpos = pos
+                    pos = random.randint(0, len(expected))
+                    f.seek(pos - currentpos, 1)
+                elif x < 0.95:
+                    assert f.tell() == pos
+                else:
+                    f.flush()
+                    g = open(fn, checkflags)
+                    buf = g.read()
+                    g.close()
+                    assert buf == expected
+            f.close()
+            g = open(fn, checkflags)
+            buf = g.read()
+            g.close()
+            assert buf == expected
+        except AssertionError:
+            assert sys.platform in {'darwin', 'win32'}
+        else:
+            assert sys.platform.startswith('linux')
 
     def test_rw(self):
         fn = self.temptestfile

From noreply at buildbot.pypy.org  Fri Sep 12 04:26:39 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri, 12 Sep 2014 04:26:39 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: fix this test on win32
Message-ID: <20140912022639.41F0C1C1036@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73497:369b70435fcb
Date: 2014-09-11 19:25 -0700
http://bitbucket.org/pypy/pypy/changeset/369b70435fcb/

Log:	fix this test on win32

diff --git a/pypy/module/_file/test/test_file_extra.py b/pypy/module/_file/test/test_file_extra.py
--- a/pypy/module/_file/test/test_file_extra.py
+++ b/pypy/module/_file/test/test_file_extra.py
@@ -503,24 +503,26 @@
         f.close()
 
     def test_flush(self):
-        import os
+        import os, sys
         fn = self.temptestfile
         f = file(fn, 'w', 0)
         f.write('x')
         assert os.stat(fn).st_size == 1
         f.close()
 
-        f = file(fn, 'wb', 1)
-        f.write('x')
-        assert os.stat(fn).st_size == 0
-        f.write('\n')
-        assert os.stat(fn).st_size == 2
-        f.write('x')
-        assert os.stat(fn).st_size == 2
-        f.flush()
-        assert os.stat(fn).st_size == 3
-        f.close()
-        assert os.stat(fn).st_size == 3
+        # http://msdn.microsoft.com/en-us/library/86cebhfs.aspx
+        if sys.platform != 'win32':
+            f = file(fn, 'wb', 1)
+            f.write('x')
+            assert os.stat(fn).st_size == 0
+            f.write('\n')
+            assert os.stat(fn).st_size == 2
+            f.write('x')
+            assert os.stat(fn).st_size == 2
+            f.flush()
+            assert os.stat(fn).st_size == 3
+            f.close()
+            assert os.stat(fn).st_size == 3
 
         f = file(fn, 'wb', 1000)
         f.write('x')

From noreply at buildbot.pypy.org  Fri Sep 12 04:42:54 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri, 12 Sep 2014 04:42:54 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: release gil around read
	ops
Message-ID: <20140912024254.F2A541D29E7@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73498:506ac75b968f
Date: 2014-09-11 22:34 -0400
http://bitbucket.org/pypy/pypy/changeset/506ac75b968f/

Log:	release gil around read ops

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -89,12 +89,12 @@
 _fclose2 = (c_fclose, c_fclose_nogil)
 _pclose2 = (c_pclose, c_pclose_nogil)
 
-c_flockfile = llexternal('flockfile', [FILEP], lltype.Void, releasegil=False)
-c_funlockfile = llexternal('funlockfile', [FILEP], lltype.Void, releasegil=False)
+c_flockfile = llexternal('flockfile', [FILEP], lltype.Void)
+c_funlockfile = llexternal('funlockfile', [FILEP], lltype.Void)
 
-c_getc = llexternal('getc', [FILEP], rffi.INT, releasegil=False)
-c_getc_unlocked = llexternal('getc_unlocked', [FILEP], rffi.INT, releasegil=False)
-c_ungetc = llexternal('ungetc', [rffi.INT, FILEP], rffi.INT, releasegil=False)
+c_getc = llexternal('getc', [FILEP], rffi.INT)
+c_getc_unlocked = llexternal('getc_unlocked', [FILEP], rffi.INT)
+c_ungetc = llexternal('ungetc', [rffi.INT, FILEP], rffi.INT)
 
 if os.name == 'nt':
     c_flockfile = lambda ll_file: None

From noreply at buildbot.pypy.org  Fri Sep 12 11:13:46 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 12 Sep 2014 11:13:46 +0200 (CEST)
Subject: [pypy-commit] pypy default: update for latest external dependencies
Message-ID: <20140912091346.7066F1C0547@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: 
Changeset: r73499:608405b649e1
Date: 2014-09-12 12:13 +0300
http://bitbucket.org/pypy/pypy/changeset/608405b649e1/

Log:	update for latest external dependencies

diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -85,10 +85,13 @@
 
 Abridged method (for -Ojit builds using Visual Studio 2008)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Download the versions of all the external packages
-from 
+Download the versions of all the external packages from 
+https://bitbucket.org/pypy/pypy/downloads/local_2.4.zip
+(for 2.4 release and later) or
 https://bitbucket.org/pypy/pypy/downloads/local.zip
-Then expand it into the base directory (base_dir) and modify your environment to reflect this::
+(for pre-2.4 versions)
+Then expand it into the base directory (base_dir) and modify your environment
+to reflect this::
 
     set PATH=\bin;\tcltk\bin;%PATH%
     set INCLUDE=\include;\tcltk\include;%INCLUDE%

From noreply at buildbot.pypy.org  Fri Sep 12 14:07:35 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 12 Sep 2014 14:07:35 +0200 (CEST)
Subject: [pypy-commit] pypy win32-fixes5: make msvc happier for 0-length
 unions (test_raw_array_field_prebuilt)
Message-ID: <20140912120735.2FF681C3641@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: win32-fixes5
Changeset: r73500:d69db21b9810
Date: 2014-08-28 23:32 +0300
http://bitbucket.org/pypy/pypy/changeset/d69db21b9810/

Log:	make msvc happier for 0-length unions
	(test_raw_array_field_prebuilt)

diff --git a/rpython/translator/c/node.py b/rpython/translator/c/node.py
--- a/rpython/translator/c/node.py
+++ b/rpython/translator/c/node.py
@@ -521,12 +521,16 @@
             return []
         lines = list(self.initializationexpr())
         type, name = self.get_declaration()
-        if name != self.name:
-            lines[0] = '{ ' + lines[0]    # extra braces around the 'a' part
-            lines[-1] += ' }'             # of the union
-        lines[0] = '%s = %s' % (
-            cdecl(type, name, self.is_thread_local()),
-            lines[0])
+        if name != self.name and len(lines) < 2:
+            # a union with length 0
+            lines[0] = cdecl(type, name, self.is_thread_local())
+        else:
+            if name != self.name:    
+                lines[0] = '{ ' + lines[0]    # extra braces around the 'a' part
+                lines[-1] += ' }'             # of the union
+            lines[0] = '%s = %s' % (
+                cdecl(type, name, self.is_thread_local()),
+                lines[0])
         lines[-1] += ';'
         return lines
 
@@ -563,6 +567,11 @@
     def initializationexpr(self, decoration=''):
         T = self.getTYPE()
         is_empty = True
+        type, name = self.get_declaration()
+        if name != self.name and self.getvarlength() < 1:
+            # an empty union
+            yield ''
+            return
         yield '{'
         defnode = self.db.gettypedefnode(T)
 

From noreply at buildbot.pypy.org  Fri Sep 12 14:07:36 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 12 Sep 2014 14:07:36 +0200 (CEST)
Subject: [pypy-commit] pypy win32-fixes5: skip tests if no objdump available
Message-ID: <20140912120736.814381C3641@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: win32-fixes5
Changeset: r73501:6e4b2dfe8ed7
Date: 2014-08-29 00:24 +0300
http://bitbucket.org/pypy/pypy/changeset/6e4b2dfe8ed7/

Log:	skip tests if no objdump available

diff --git a/rpython/tool/jitlogparser/test/test_parser.py b/rpython/tool/jitlogparser/test/test_parser.py
--- a/rpython/tool/jitlogparser/test/test_parser.py
+++ b/rpython/tool/jitlogparser/test/test_parser.py
@@ -5,6 +5,7 @@
 from rpython.tool.jitlogparser.storage import LoopStorage
 import py, sys
 from rpython.jit.backend.detect_cpu import autodetect
+from rpython.jit.backend.tool.viewcode import ObjdumpNotFound
 
 def parse(input, **kwds):
     return SimpleParser.parse_from_input(input, **kwds)
@@ -193,7 +194,8 @@
         py.test.skip('x86 only test')
     backend_dump = "554889E5534154415541564157488DA500000000488B042590C5540148C7042590C554010000000048898570FFFFFF488B042598C5540148C7042598C554010000000048898568FFFFFF488B0425A0C5540148C70425A0C554010000000048898560FFFFFF488B0425A8C5540148C70425A8C554010000000048898558FFFFFF4C8B3C2550525B0149BB30E06C96FC7F00004D8B334983C60149BB30E06C96FC7F00004D89334981FF102700000F8D000000004983C7014C8B342580F76A024983EE014C89342580F76A024983FE000F8C00000000E9AEFFFFFF488B042588F76A024829E0483B042580EC3C01760D49BB05F30894FC7F000041FFD3554889E5534154415541564157488DA550FFFFFF4889BD70FFFFFF4889B568FFFFFF48899560FFFFFF48898D58FFFFFF4D89C7E954FFFFFF49BB00F00894FC7F000041FFD34440484C3D030300000049BB00F00894FC7F000041FFD34440484C3D070304000000"
     dump_start = 0x7f3b0b2e63d5
-    loop = parse("""
+    try:
+        loop = parse("""
     # Loop 0 : loop with 19 ops
     [p0, p1, p2, p3, i4]
     debug_merge_point(0, 0, ' #15 COMPARE_OP')
@@ -212,6 +214,8 @@
     +218: --end of the loop--""", backend_dump=backend_dump,
                  dump_start=dump_start,
                  backend_tp='x86_64')
+    except ObjdumpNotFound:
+        py.test.skip('no objdump found on path')
     cmp = loop.operations[1]
     assert 'jge' in cmp.asm
     assert '0x2710' in cmp.asm
@@ -276,8 +280,11 @@
         py.test.skip('x86 only test')
     _, loops = import_log(str(py.path.local(__file__).join('..',
                                                            'logtest.log')))
-    for loop in loops:
-        loop.force_asm()
+    try:
+        for loop in loops:
+            loop.force_asm()
+    except ObjdumpNotFound:
+        py.test.skip('no objdump found on path')
     assert 'jge' in loops[0].operations[3].asm
 
 def test_import_log_2():
@@ -285,8 +292,11 @@
         py.test.skip('x86 only test')
     _, loops = import_log(str(py.path.local(__file__).join('..',
                                                            'logtest2.log')))
-    for loop in loops:
-        loop.force_asm()
+    try:
+        for loop in loops:
+            loop.force_asm()
+    except ObjdumpNotFound:
+        py.test.skip('no objdump found on path')
     assert 'cmp' in loops[1].operations[2].asm
 
 def test_Op_repr_is_pure():

From noreply at buildbot.pypy.org  Fri Sep 12 14:07:37 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 12 Sep 2014 14:07:37 +0200 (CEST)
Subject: [pypy-commit] pypy default: any file on windows is "executable",
 skip test added by pull request #276 for issue #1856
Message-ID: <20140912120737.9D7D91C3641@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: 
Changeset: r73502:25c65744b4b0
Date: 2014-09-12 14:09 +0300
http://bitbucket.org/pypy/pypy/changeset/25c65744b4b0/

Log:	any file on windows is "executable", skip test added by pull request
	#276 for issue #1856

diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -945,7 +945,7 @@
         prefix = udir.join('pathtest').ensure(dir=1)
         fake_exe = 'bin/pypy-c'
         if sys.platform == 'win32':
-            fake_exe += '.exe'
+            fake_exe = 'pypy-c.exe'
         fake_exe = prefix.join(fake_exe).ensure(file=1)
         expected_path = [str(prefix.join(subdir).ensure(dir=1))
                          for subdir in ('lib_pypy',
@@ -985,8 +985,10 @@
             assert sys.path == old_sys_path + [self.goal_dir]
 
             app_main.setup_bootstrap_path(self.fake_exe)
-            assert sys.executable == ''      # not executable!
-            assert sys.path == old_sys_path + [self.goal_dir]
+            if not sys.platform == 'win32':
+                # an existing file is always 'executable' on windows
+                assert sys.executable == ''      # not executable!
+                assert sys.path == old_sys_path + [self.goal_dir]
 
             os.chmod(self.fake_exe, 0755)
             app_main.setup_bootstrap_path(self.fake_exe)

From noreply at buildbot.pypy.org  Fri Sep 12 14:07:38 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 12 Sep 2014 14:07:38 +0200 (CEST)
Subject: [pypy-commit] pypy default: emit pdb file in same directory as obj
 for parallel test runs
Message-ID: <20140912120738.C30CC1C3641@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: 
Changeset: r73503:b18c07d4bffd
Date: 2014-09-12 15:04 +0300
http://bitbucket.org/pypy/pypy/changeset/b18c07d4bffd/

Log:	emit pdb file in same directory as obj for parallel test runs

diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py
--- a/rpython/translator/platform/windows.py
+++ b/rpython/translator/platform/windows.py
@@ -203,7 +203,9 @@
         # the assembler still has the old behavior that all options
         # must come first, and after the file name all options are ignored.
         # So please be careful with the order of parameters! ;-)
-        args = ['/nologo', '/c'] + compile_args + ['/Fo%s' % (oname,), str(cfile)]
+        pdb_dir = oname.dirname
+        args = ['/nologo', '/c'] + compile_args + ['/Fd%s\\' % (pdb_dir,),
+                        '/Fo%s' % (oname,), str(cfile)]
         self._execute_c_compiler(cc, args, oname)
         return oname
 
@@ -345,7 +347,7 @@
                '$(CREATE_PCH) $(INCLUDEDIRS)'))
             rules.append(('.c.obj', '',
                     '$(CC) /nologo $(CFLAGS) $(CFLAGSEXTRA) $(USE_PCH) '
-                    '/Fo$@ /c $< $(INCLUDEDIRS)'))
+                    '/Fd$(@D)\\ /Fo$@ /c $< $(INCLUDEDIRS)'))
             #Do not use precompiled headers for some files
             #rules.append((r'{..\module_cache}.c{..\module_cache}.obj', '',
             #        '$(CC) /nologo $(CFLAGS) $(CFLAGSEXTRA) /Fo$@ /c $< $(INCLUDEDIRS)'))
@@ -360,12 +362,13 @@
                     target = f[:-1] + 'obj'
                     rules.append((target, f,
                         '$(CC) /nologo $(CFLAGS) $(CFLAGSEXTRA) '
-                        '/Fo%s /c %s $(INCLUDEDIRS)' %(target, f)))
+                        '/Fd%s\\ /Fo%s /c %s $(INCLUDEDIRS)' %(
+                                os.path.dirname(target), target, f)))
 
         else:
             rules.append(('.c.obj', '',
                           '$(CC) /nologo $(CFLAGS) $(CFLAGSEXTRA) '
-                          '/Fo$@ /c $< $(INCLUDEDIRS)'))
+                          '/Fd$(@D)\\ /Fo$@ /c $< $(INCLUDEDIRS)'))
 
 
         for args in definitions:

From noreply at buildbot.pypy.org  Fri Sep 12 14:07:47 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 12 Sep 2014 14:07:47 +0200 (CEST)
Subject: [pypy-commit] pypy win32-fixes5: merge default into branch
Message-ID: <20140912120747.B3AC71C3641@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: win32-fixes5
Changeset: r73504:68a792269472
Date: 2014-09-12 15:07 +0300
http://bitbucket.org/pypy/pypy/changeset/68a792269472/

Log:	merge default into branch

diff too long, truncating to 2000 out of 43849 lines

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -35,280 +35,290 @@
 the beginning of each file) the files in the 'pypy' directory are each
 copyrighted by one or more of the following people and organizations:    
 
-    Armin Rigo
-    Maciej Fijalkowski
-    Carl Friedrich Bolz
-    Antonio Cuni
-    Amaury Forgeot d'Arc
-    Samuele Pedroni
-    Alex Gaynor
-    Michael Hudson
-    David Schneider
-    Matti Picus
-    Brian Kearns
-    Philip Jenvey
-    Holger Krekel
-    Christian Tismer
-    Hakan Ardo
-    Benjamin Peterson
-    Manuel Jacob
-    Anders Chrigstrom
-    Eric van Riet Paap
-    Wim Lavrijsen
-    Ronan Lamy
-    Richard Emslie
-    Alexander Schremmer
-    Dan Villiom Podlaski Christiansen
-    Lukas Diekmann
-    Sven Hager
-    Anders Lehmann
-    Aurelien Campeas
-    Niklaus Haldimann
-    Camillo Bruni
-    Laura Creighton
-    Toon Verwaest
-    Remi Meier
-    Leonardo Santagada
-    Seo Sanghyeon
-    Romain Guillebert
-    Justin Peel
-    Ronny Pfannschmidt
-    David Edelsohn
-    Anders Hammarquist
-    Jakub Gustak
-    Guido Wesdorp
-    Lawrence Oluyede
-    Bartosz Skowron
-    Daniel Roberts
-    Niko Matsakis
-    Adrien Di Mascio
-    Alexander Hesse
-    Ludovic Aubry
-    Jacob Hallen
-    Jason Creighton
-    Alex Martelli
-    Michal Bendowski
-    Jan de Mooij
-    stian
-    Michael Foord
-    Stephan Diehl
-    Stefan Schwarzer
-    Valentino Volonghi
-    Tomek Meka
-    Patrick Maupin
-    Bob Ippolito
-    Bruno Gola
-    Jean-Paul Calderone
-    Timo Paulssen
-    Squeaky
-    Alexandre Fayolle
-    Simon Burton
-    Marius Gedminas
-    John Witulski
-    Konstantin Lopuhin
-    Greg Price
-    Dario Bertini
-    Mark Pearse
-    Simon Cross
-    Andreas Stührk
-    Jean-Philippe St. Pierre
-    Guido van Rossum
-    Pavel Vinogradov
-    Paweł Piotr Przeradowski
-    Paul deGrandis
-    Ilya Osadchiy
-    Tobias Oberstein
-    Adrian Kuhn
-    Boris Feigin
-    Stefano Rivera
-    tav
-    Taavi Burns
-    Georg Brandl
-    Bert Freudenberg
-    Stian Andreassen
-    Laurence Tratt
-    Wanja Saatkamp
-    Ivan Sichmann Freitas
-    Gerald Klix
-    Mike Blume
-    Oscar Nierstrasz
-    Stefan H. Muller
-    Jeremy Thurgood
-    Gregor Wegberg
-    Rami Chowdhury
-    Tobias Pape
-    Edd Barrett
-    David Malcolm
-    Eugene Oden
-    Henry Mason
-    Preston Timmons
-    Jeff Terrace
-    David Ripton
-    Dusty Phillips
-    Lukas Renggli
-    Guenter Jantzen
-    Ned Batchelder
-    Amit Regmi
-    Ben Young
-    Nicolas Chauvat
-    Andrew Durdin
-    Andrew Chambers
-    Michael Schneider
-    Nicholas Riley
-    Jason Chu
-    Igor Trindade Oliveira
-    Rocco Moretti
-    Gintautas Miliauskas
-    Michael Twomey
-    Lucian Branescu Mihaila
-    Tim Felgentreff
-    Tyler Wade
-    Gabriel Lavoie
-    Olivier Dormond
-    Jared Grubb
-    Karl Bartel
-    Brian Dorsey
-    Victor Stinner
-    Andrews Medina
-    Stuart Williams
-    Jasper Schulz
-    Christian Hudon
-    Toby Watson
-    Antoine Pitrou
-    Aaron Iles
-    Michael Cheng
-    Justas Sadzevicius
-    Mikael Schönenberg
-    Gasper Zejn
-    Neil Shepperd
-    Elmo Mäntynen
-    Jonathan David Riehl
-    Stanislaw Halik
-    Anders Qvist
-    Chirag Jadwani
-    Beatrice During
-    Alex Perry
-    Vincent Legoll
-    Alan McIntyre
-    Alexander Sedov
-    Corbin Simpson
-    Christopher Pope
-    wenzhuman
-    Christian Tismer 
-    Marc Abramowitz
-    Dan Stromberg
-    Stefano Parmesan
-    Alexis Daboville
-    Jens-Uwe Mager
-    Carl Meyer
-    Karl Ramm
-    Pieter Zieschang
-    Gabriel
-    Lukas Vacek
-    Andrew Dalke
-    Sylvain Thenault
-    Nathan Taylor
-    Vladimir Kryachko
-    Jacek Generowicz
-    Alejandro J. Cura
-    Jacob Oscarson
-    Travis Francis Athougies
-    Ryan Gonzalez
-    Kristjan Valur Jonsson
-    Sebastian Pawluś
-    Neil Blakey-Milner
-    anatoly techtonik
-    Lutz Paelike
-    Lucio Torre
-    Lars Wassermann
-    Henrik Vendelbo
-    Dan Buch
-    Miguel de Val Borro
-    Artur Lisiecki
-    Sergey Kishchenko
-    Ignas Mikalajunas
-    Christoph Gerum
-    Martin Blais
-    Lene Wagner
-    Tomo Cocoa
-    roberto at goyle
-    Yury V. Zaytsev
-    Anna Katrina Dominguez
-    William Leslie
-    Bobby Impollonia
-    timo at eistee.fritz.box
-    Andrew Thompson
-    Ben Darnell
-    Roberto De Ioris
-    Juan Francisco Cantero Hurtado
-    Godefroid Chappelle
-    Joshua Gilbert
-    Dan Colish
-    Christopher Armstrong
-    Michael Hudson-Doyle
-    Anders Sigfridsson
-    Yasir Suhail
-    rafalgalczynski at gmail.com
-    Floris Bruynooghe
-    Laurens Van Houtven
-    Akira Li
-    Gustavo Niemeyer
-    Stephan Busemann
-    Rafał Gałczyński
-    Yusei Tahara
-    Christian Muirhead
-    James Lan
-    shoma hosaka
-    Daniel Neuh?user
-    Matthew Miller
-    Buck Golemon
-    Konrad Delong
-    Dinu Gherman
-    Chris Lambacher
-    coolbutuseless at gmail.com
-    Rodrigo Araújo
-    w31rd0
-    Jim Baker
-    James Robert
-    Armin Ronacher
-    Brett Cannon
-    yrttyr
-    aliceinwire
-    OlivierBlanvillain
-    Zooko Wilcox-O Hearn
-    Tomer Chachamu
-    Christopher Groskopf
-    Asmo Soinio
-    Stefan Marr
-    jiaaro
-    opassembler.py
-    Antony Lee
-    Jim Hunziker
-    Markus Unterwaditzer
-    Even Wiik Thomassen
-    jbs
-    soareschen
-    Kurt Griffiths
-    Mike Bayer
-    Flavio Percoco
-    Kristoffer Kleine
-    yasirs
-    Michael Chermside
-    Anna Ravencroft
-    Julien Phalip
-    Dan Loewenherz
+  Armin Rigo
+  Maciej Fijalkowski
+  Carl Friedrich Bolz
+  Antonio Cuni
+  Amaury Forgeot d'Arc
+  Samuele Pedroni
+  Alex Gaynor
+  Michael Hudson
+  David Schneider
+  Matti Picus
+  Brian Kearns
+  Philip Jenvey
+  Holger Krekel
+  Christian Tismer
+  Hakan Ardo
+  Benjamin Peterson
+  Manuel Jacob
+  Anders Chrigstrom
+  Eric van Riet Paap
+  Ronan Lamy
+  Wim Lavrijsen
+  Richard Emslie
+  Alexander Schremmer
+  Dan Villiom Podlaski Christiansen
+  Lukas Diekmann
+  Sven Hager
+  Anders Lehmann
+  Aurelien Campeas
+  Niklaus Haldimann
+  Remi Meier
+  Camillo Bruni
+  Laura Creighton
+  Toon Verwaest
+  Leonardo Santagada
+  Seo Sanghyeon
+  Romain Guillebert
+  Justin Peel
+  Ronny Pfannschmidt
+  David Edelsohn
+  Anders Hammarquist
+  Jakub Gustak
+  Guido Wesdorp
+  Lawrence Oluyede
+  Bartosz Skowron
+  Gregor Wegberg
+  Daniel Roberts
+  Niko Matsakis
+  Adrien Di Mascio
+  Alexander Hesse
+  Ludovic Aubry
+  Jacob Hallen
+  Jason Creighton
+  Alex Martelli
+  Michal Bendowski
+  Jan de Mooij
+  stian
+  Michael Foord
+  Stephan Diehl
+  Tyler Wade
+  Stefan Schwarzer
+  Valentino Volonghi
+  Tomek Meka
+  Patrick Maupin
+  Bob Ippolito
+  Bruno Gola
+  Jean-Paul Calderone
+  Timo Paulssen
+  Squeaky
+  Alexandre Fayolle
+  Simon Burton
+  Marius Gedminas
+  Martin Matusiak
+  Konstantin Lopuhin
+  John Witulski
+  Wenzhu Man
+  Greg Price
+  Dario Bertini
+  Mark Pearse
+  Simon Cross
+  Ivan Sichmann Freitas
+  Andreas Stührk
+  Jean-Philippe St. Pierre
+  Guido van Rossum
+  Pavel Vinogradov
+  Stefano Rivera
+  Paweł Piotr Przeradowski
+  Paul deGrandis
+  Ilya Osadchiy
+  Tobias Oberstein
+  Adrian Kuhn
+  Boris Feigin
+  tav
+  Taavi Burns
+  Georg Brandl
+  Laurence Tratt
+  Bert Freudenberg
+  Stian Andreassen
+  Wanja Saatkamp
+  Gerald Klix
+  Mike Blume
+  Oscar Nierstrasz
+  Stefan H. Muller
+  Edd Barrett
+  Jeremy Thurgood
+  Rami Chowdhury
+  Tobias Pape
+  David Malcolm
+  Eugene Oden
+  Henry Mason
+  Vasily Kuznetsov
+  Preston Timmons
+  Jeff Terrace
+  David Ripton
+  Dusty Phillips
+  Lukas Renggli
+  Guenter Jantzen
+  Ned Batchelder
+  Amit Regmi
+  Ben Young
+  Nicolas Chauvat
+  Andrew Durdin
+  Andrew Chambers
+  Michael Schneider
+  Nicholas Riley
+  Jason Chu
+  Igor Trindade Oliveira
+  Tim Felgentreff
+  Rocco Moretti
+  Gintautas Miliauskas
+  Michael Twomey
+  Lucian Branescu Mihaila
+  Gabriel Lavoie
+  Olivier Dormond
+  Jared Grubb
+  Karl Bartel
+  Brian Dorsey
+  Victor Stinner
+  Andrews Medina
+  Stuart Williams
+  Jasper Schulz
+  Christian Hudon
+  Toby Watson
+  Antoine Pitrou
+  Aaron Iles
+  Michael Cheng
+  Justas Sadzevicius
+  Gasper Zejn
+  anatoly techtonik
+  Neil Shepperd
+  Mikael Schönenberg
+  Elmo M?ntynen
+  Jonathan David Riehl
+  Stanislaw Halik
+  Anders Qvist
+  Corbin Simpson
+  Chirag Jadwani
+  Beatrice During
+  Alex Perry
+  Vincent Legoll
+  Alan McIntyre
+  Alexander Sedov
+  Christopher Pope
+  Christian Tismer 
+  Marc Abramowitz
+  Dan Stromberg
+  Stefano Parmesan
+  Alexis Daboville
+  Jens-Uwe Mager
+  Carl Meyer
+  Karl Ramm
+  Pieter Zieschang
+  Sebastian Pawluś
+  Gabriel
+  Lukas Vacek
+  Andrew Dalke
+  Sylvain Thenault
+  Nathan Taylor
+  Vladimir Kryachko
+  Arjun Naik
+  Attila Gobi
+  Jacek Generowicz
+  Alejandro J. Cura
+  Jacob Oscarson
+  Travis Francis Athougies
+  Ryan Gonzalez
+  Ian Foote
+  Kristjan Valur Jonsson
+  Neil Blakey-Milner
+  Lutz Paelike
+  Lucio Torre
+  Lars Wassermann
+  Valentina Mukhamedzhanova
+  Henrik Vendelbo
+  Dan Buch
+  Miguel de Val Borro
+  Artur Lisiecki
+  Sergey Kishchenko
+  Yichao Yu
+  Ignas Mikalajunas
+  Christoph Gerum
+  Martin Blais
+  Lene Wagner
+  Tomo Cocoa
+  roberto at goyle
+  Yury V. Zaytsev
+  Anna Katrina Dominguez
+  William Leslie
+  Bobby Impollonia
+  timo at eistee.fritz.box
+  Andrew Thompson
+  Yusei Tahara
+  Ben Darnell
+  Roberto De Ioris
+  Juan Francisco Cantero Hurtado
+  Godefroid Chappelle
+  Joshua Gilbert
+  Dan Colish
+  Christopher Armstrong
+  Michael Hudson-Doyle
+  Anders Sigfridsson
+  Yasir Suhail
+  Jason Michalski
+  rafalgalczynski at gmail.com
+  Floris Bruynooghe
+  Laurens Van Houtven
+  Akira Li
+  Gustavo Niemeyer
+  Stephan Busemann
+  Rafał Gałczyński
+  Christian Muirhead
+  James Lan
+  shoma hosaka
+  Daniel Neuh?user
+  Matthew Miller
+  Buck Golemon
+  Konrad Delong
+  Dinu Gherman
+  Chris Lambacher
+  coolbutuseless at gmail.com
+  Rodrigo Araújo
+  Jim Baker
+  James Robert
+  Armin Ronacher
+  Brett Cannon
+  yrttyr
+  aliceinwire
+  OlivierBlanvillain
+  Zooko Wilcox-O Hearn
+  Tomer Chachamu
+  Christopher Groskopf
+  Asmo Soinio
+  Stefan Marr
+  jiaaro
+  Mads Kiilerich
+  opassembler.py
+  Antony Lee
+  Jim Hunziker
+  Markus Unterwaditzer
+  Even Wiik Thomassen
+  jbs
+  soareschen
+  Kurt Griffiths
+  Mike Bayer
+  Matthew Miller
+  Flavio Percoco
+  Kristoffer Kleine
+  yasirs
+  Michael Chermside
+  Anna Ravencroft
+  Dan Crosta
+  Julien Phalip
+  Dan Loewenherz
 
-    Heinrich-Heine University, Germany 
-    Open End AB (formerly AB Strakt), Sweden
-    merlinux GmbH, Germany 
-    tismerysoft GmbH, Germany 
-    Logilab Paris, France 
-    DFKI GmbH, Germany 
-    Impara, Germany
-    Change Maker, Sweden 
-    University of California Berkeley, USA
-    Google Inc.
-    King's College London
+  Heinrich-Heine University, Germany 
+  Open End AB (formerly AB Strakt), Sweden
+  merlinux GmbH, Germany 
+  tismerysoft GmbH, Germany 
+  Logilab Paris, France 
+  DFKI GmbH, Germany 
+  Impara, Germany
+  Change Maker, Sweden 
+  University of California Berkeley, USA
+  Google Inc.
+  King's College London
 
 The PyPy Logo as used by http://speed.pypy.org and others was created
 by Samuel Reis and is distributed on terms of Creative Commons Share Alike
diff --git a/_pytest/README-BEFORE-UPDATING b/_pytest/README-BEFORE-UPDATING
new file mode 100644
--- /dev/null
+++ b/_pytest/README-BEFORE-UPDATING
@@ -0,0 +1,17 @@
+This is PyPy's code of the pytest lib.  We don't expect to upgrade it
+very often, but once we do:
+
+    WARNING!
+
+    WE HAVE MADE A FEW TWEAKS HERE!
+
+Please be sure that you don't just copy the newer version from
+upstream without checking the few changes that we did.  This
+can be done like this:
+
+    cd 
+    hg log . -v | less
+
+then search for all " _pytest/" in that list to know which are the
+relevant checkins.  (Look for the checkins that only edit one
+or two files in this directory.)
diff --git a/_pytest/resultlog.py b/_pytest/resultlog.py
--- a/_pytest/resultlog.py
+++ b/_pytest/resultlog.py
@@ -53,16 +53,24 @@
         self.config = config
         self.logfile = logfile # preferably line buffered
 
-    def write_log_entry(self, testpath, lettercode, longrepr):
-        py.builtin.print_("%s %s" % (lettercode, testpath), file=self.logfile)
+    def write_log_entry(self, testpath, lettercode, longrepr, sections=None):
+        _safeprint("%s %s" % (lettercode, testpath), file=self.logfile)
         for line in longrepr.splitlines():
-            py.builtin.print_(" %s" % line, file=self.logfile)
+            _safeprint(" %s" % line, file=self.logfile)
+        if sections is not None and (
+                lettercode in ('E', 'F')):    # to limit the size of logs
+            for title, content in sections:
+                _safeprint(" ---------- %s ----------" % (title,),
+                           file=self.logfile)
+                for line in content.splitlines():
+                    _safeprint(" %s" % line, file=self.logfile)
 
     def log_outcome(self, report, lettercode, longrepr):
         testpath = getattr(report, 'nodeid', None)
         if testpath is None:
             testpath = report.fspath
-        self.write_log_entry(testpath, lettercode, longrepr)
+        self.write_log_entry(testpath, lettercode, longrepr,
+                             getattr(report, 'sections', None))
 
     def pytest_runtest_logreport(self, report):
         if report.when != "call" and report.passed:
@@ -98,3 +106,8 @@
         if path is None:
             path = "cwd:%s" % py.path.local()
         self.write_log_entry(path, '!', str(excrepr))
+
+def _safeprint(s, file):
+    if isinstance(s, unicode):
+        s = s.encode('utf-8')
+    py.builtin.print_(s, file=file)
diff --git a/lib-python/2.7/CGIHTTPServer.py b/lib-python/2.7/CGIHTTPServer.py
--- a/lib-python/2.7/CGIHTTPServer.py
+++ b/lib-python/2.7/CGIHTTPServer.py
@@ -84,7 +84,7 @@
         path begins with one of the strings in self.cgi_directories
         (and the next character is a '/' or the end of the string).
         """
-        collapsed_path = _url_collapse_path(self.path)
+        collapsed_path = _url_collapse_path(urllib.unquote(self.path))
         dir_sep = collapsed_path.find('/', 1)
         head, tail = collapsed_path[:dir_sep], collapsed_path[dir_sep+1:]
         if head in self.cgi_directories:
diff --git a/lib-python/2.7/Cookie.py b/lib-python/2.7/Cookie.py
--- a/lib-python/2.7/Cookie.py
+++ b/lib-python/2.7/Cookie.py
@@ -1,6 +1,3 @@
-#!/usr/bin/env python
-#
-
 ####
 # Copyright 2000 by Timothy O'Malley 
 #
diff --git a/lib-python/2.7/HTMLParser.py b/lib-python/2.7/HTMLParser.py
--- a/lib-python/2.7/HTMLParser.py
+++ b/lib-python/2.7/HTMLParser.py
@@ -22,9 +22,12 @@
 starttagopen = re.compile('<[a-zA-Z]')
 piclose = re.compile('>')
 commentclose = re.compile(r'--\s*>')
-tagfind = re.compile('([a-zA-Z][-.a-zA-Z0-9:_]*)(?:\s|/(?!>))*')
+
 # see http://www.w3.org/TR/html5/tokenization.html#tag-open-state
 # and http://www.w3.org/TR/html5/tokenization.html#tag-name-state
+# note: if you change tagfind/attrfind remember to update locatestarttagend too
+tagfind = re.compile('([a-zA-Z][^\t\n\r\f />\x00]*)(?:\s|/(?!>))*')
+# this regex is currently unused, but left for backward compatibility
 tagfind_tolerant = re.compile('[a-zA-Z][^\t\n\r\f />\x00]*')
 
 attrfind = re.compile(
@@ -32,7 +35,7 @@
     r'(\'[^\']*\'|"[^"]*"|(?![\'"])[^>\s]*))?(?:\s|/(?!>))*')
 
 locatestarttagend = re.compile(r"""
-  <[a-zA-Z][-.a-zA-Z0-9:_]*          # tag name
+  <[a-zA-Z][^\t\n\r\f />\x00]*       # tag name
   (?:[\s/]*                          # optional whitespace before attribute name
     (?:(?<=['"\s/])[^\s/>][^\s/=>]*  # attribute name
       (?:\s*=+\s*                    # value indicator
@@ -192,9 +195,9 @@
                     i = self.updatepos(i, k)
                     continue
                 else:
-                    if ";" in rawdata[i:]: #bail by consuming &#
-                        self.handle_data(rawdata[0:2])
-                        i = self.updatepos(i, 2)
+                    if ";" in rawdata[i:]:  # bail by consuming '&#'
+                        self.handle_data(rawdata[i:i+2])
+                        i = self.updatepos(i, i+2)
                     break
             elif startswith('&', i):
                 match = entityref.match(rawdata, i)
@@ -373,14 +376,14 @@
                 self.handle_data(rawdata[i:gtpos])
                 return gtpos
             # find the name: w3.org/TR/html5/tokenization.html#tag-name-state
-            namematch = tagfind_tolerant.match(rawdata, i+2)
+            namematch = tagfind.match(rawdata, i+2)
             if not namematch:
                 # w3.org/TR/html5/tokenization.html#end-tag-open-state
                 if rawdata[i:i+3] == '':
                     return i+3
                 else:
                     return self.parse_bogus_comment(i)
-            tagname = namematch.group().lower()
+            tagname = namematch.group(1).lower()
             # consume and ignore other stuff between the name and the >
             # Note: this is not 100% correct, since we might have things like
             # , but looking for > after tha name should cover
diff --git a/lib-python/2.7/SimpleHTTPServer.py b/lib-python/2.7/SimpleHTTPServer.py
--- a/lib-python/2.7/SimpleHTTPServer.py
+++ b/lib-python/2.7/SimpleHTTPServer.py
@@ -43,8 +43,10 @@
         """Serve a GET request."""
         f = self.send_head()
         if f:
-            self.copyfile(f, self.wfile)
-            f.close()
+            try:
+                self.copyfile(f, self.wfile)
+            finally:
+                f.close()
 
     def do_HEAD(self):
         """Serve a HEAD request."""
@@ -88,13 +90,17 @@
         except IOError:
             self.send_error(404, "File not found")
             return None
-        self.send_response(200)
-        self.send_header("Content-type", ctype)
-        fs = os.fstat(f.fileno())
-        self.send_header("Content-Length", str(fs[6]))
-        self.send_header("Last-Modified", self.date_time_string(fs.st_mtime))
-        self.end_headers()
-        return f
+        try:
+            self.send_response(200)
+            self.send_header("Content-type", ctype)
+            fs = os.fstat(f.fileno())
+            self.send_header("Content-Length", str(fs[6]))
+            self.send_header("Last-Modified", self.date_time_string(fs.st_mtime))
+            self.end_headers()
+            return f
+        except:
+            f.close()
+            raise
 
     def list_directory(self, path):
         """Helper to produce a directory listing (absent index.html).
diff --git a/lib-python/2.7/SimpleXMLRPCServer.py b/lib-python/2.7/SimpleXMLRPCServer.py
--- a/lib-python/2.7/SimpleXMLRPCServer.py
+++ b/lib-python/2.7/SimpleXMLRPCServer.py
@@ -704,4 +704,5 @@
     server = SimpleXMLRPCServer(("localhost", 8000))
     server.register_function(pow)
     server.register_function(lambda x,y: x+y, 'add')
+    server.register_multicall_functions()
     server.serve_forever()
diff --git a/lib-python/2.7/SocketServer.py b/lib-python/2.7/SocketServer.py
--- a/lib-python/2.7/SocketServer.py
+++ b/lib-python/2.7/SocketServer.py
@@ -513,35 +513,37 @@
 
     def collect_children(self):
         """Internal routine to wait for children that have exited."""
-        if self.active_children is None: return
+        if self.active_children is None:
+            return
+
+        # If we're above the max number of children, wait and reap them until
+        # we go back below threshold. Note that we use waitpid(-1) below to be
+        # able to collect children in size() syscalls instead
+        # of size(): the downside is that this might reap children
+        # which we didn't spawn, which is why we only resort to this when we're
+        # above max_children.
         while len(self.active_children) >= self.max_children:
-            # XXX: This will wait for any child process, not just ones
-            # spawned by this library. This could confuse other
-            # libraries that expect to be able to wait for their own
-            # children.
             try:
-                pid, status = os.waitpid(0, 0)
-            except os.error:
-                pid = None
-            if pid not in self.active_children: continue
-            self.active_children.remove(pid)
+                pid, _ = os.waitpid(-1, 0)
+                self.active_children.discard(pid)
+            except OSError as e:
+                if e.errno == errno.ECHILD:
+                    # we don't have any children, we're done
+                    self.active_children.clear()
+                elif e.errno != errno.EINTR:
+                    break
 
-        # XXX: This loop runs more system calls than it ought
-        # to. There should be a way to put the active_children into a
-        # process group and then use os.waitpid(-pgid) to wait for any
-        # of that set, but I couldn't find a way to allocate pgids
-        # that couldn't collide.
-        for child in self.active_children:
+        # Now reap all defunct children.
+        for pid in self.active_children.copy():
             try:
-                pid, status = os.waitpid(child, os.WNOHANG)
-            except os.error:
-                pid = None
-            if not pid: continue
-            try:
-                self.active_children.remove(pid)
-            except ValueError, e:
-                raise ValueError('%s. x=%d and list=%r' % (e.message, pid,
-                                                           self.active_children))
+                pid, _ = os.waitpid(pid, os.WNOHANG)
+                # if the child hasn't exited yet, pid will be 0 and ignored by
+                # discard() below
+                self.active_children.discard(pid)
+            except OSError as e:
+                if e.errno == errno.ECHILD:
+                    # someone else reaped it
+                    self.active_children.discard(pid)
 
     def handle_timeout(self):
         """Wait for zombies after self.timeout seconds of inactivity.
@@ -557,8 +559,8 @@
         if pid:
             # Parent process
             if self.active_children is None:
-                self.active_children = []
-            self.active_children.append(pid)
+                self.active_children = set()
+            self.active_children.add(pid)
             self.close_request(request) #close handle in parent process
             return
         else:
diff --git a/lib-python/2.7/_MozillaCookieJar.py b/lib-python/2.7/_MozillaCookieJar.py
--- a/lib-python/2.7/_MozillaCookieJar.py
+++ b/lib-python/2.7/_MozillaCookieJar.py
@@ -39,7 +39,7 @@
     magic_re = "#( Netscape)? HTTP Cookie File"
     header = """\
 # Netscape HTTP Cookie File
-# http://www.netscape.com/newsref/std/cookie_spec.html
+# http://curl.haxx.se/rfc/cookie_spec.html
 # This is a generated file!  Do not edit.
 
 """
diff --git a/lib-python/2.7/_abcoll.py b/lib-python/2.7/_abcoll.py
--- a/lib-python/2.7/_abcoll.py
+++ b/lib-python/2.7/_abcoll.py
@@ -165,12 +165,17 @@
     def __gt__(self, other):
         if not isinstance(other, Set):
             return NotImplemented
-        return other < self
+        return len(self) > len(other) and self.__ge__(other)
 
     def __ge__(self, other):
         if not isinstance(other, Set):
             return NotImplemented
-        return other <= self
+        if len(self) < len(other):
+            return False
+        for elem in other:
+            if elem not in self:
+                return False
+        return True
 
     def __eq__(self, other):
         if not isinstance(other, Set):
@@ -194,6 +199,8 @@
             return NotImplemented
         return self._from_iterable(value for value in other if value in self)
 
+    __rand__ = __and__
+
     def isdisjoint(self, other):
         'Return True if two sets have a null intersection.'
         for value in other:
@@ -207,6 +214,8 @@
         chain = (e for s in (self, other) for e in s)
         return self._from_iterable(chain)
 
+    __ror__ = __or__
+
     def __sub__(self, other):
         if not isinstance(other, Set):
             if not isinstance(other, Iterable):
@@ -215,6 +224,14 @@
         return self._from_iterable(value for value in self
                                    if value not in other)
 
+    def __rsub__(self, other):
+        if not isinstance(other, Set):
+            if not isinstance(other, Iterable):
+                return NotImplemented
+            other = self._from_iterable(other)
+        return self._from_iterable(value for value in other
+                                   if value not in self)
+
     def __xor__(self, other):
         if not isinstance(other, Set):
             if not isinstance(other, Iterable):
@@ -222,6 +239,8 @@
             other = self._from_iterable(other)
         return (self - other) | (other - self)
 
+    __rxor__ = __xor__
+
     # Sets are not hashable by default, but subclasses can change this
     __hash__ = None
 
diff --git a/lib-python/2.7/_osx_support.py b/lib-python/2.7/_osx_support.py
--- a/lib-python/2.7/_osx_support.py
+++ b/lib-python/2.7/_osx_support.py
@@ -182,7 +182,7 @@
         # Compiler is GCC, check if it is LLVM-GCC
         data = _read_output("'%s' --version"
                              % (cc.replace("'", "'\"'\"'"),))
-        if 'llvm-gcc' in data:
+        if data and 'llvm-gcc' in data:
             # Found LLVM-GCC, fall back to clang
             cc = _find_build_tool('clang')
 
@@ -450,8 +450,16 @@
         # case and disallow installs.
         cflags = _config_vars.get(_INITPRE+'CFLAGS',
                                     _config_vars.get('CFLAGS', ''))
-        if ((macrelease + '.') >= '10.4.' and
-            '-arch' in cflags.strip()):
+        if macrelease:
+            try:
+                macrelease = tuple(int(i) for i in macrelease.split('.')[0:2])
+            except ValueError:
+                macrelease = (10, 0)
+        else:
+            # assume no universal support
+            macrelease = (10, 0)
+
+        if (macrelease >= (10, 4)) and '-arch' in cflags.strip():
             # The universal build will build fat binaries, but not on
             # systems before 10.4
 
diff --git a/lib-python/2.7/_pyio.py b/lib-python/2.7/_pyio.py
--- a/lib-python/2.7/_pyio.py
+++ b/lib-python/2.7/_pyio.py
@@ -192,38 +192,45 @@
                  (appending and "a" or "") +
                  (updating and "+" or ""),
                  closefd)
-    line_buffering = False
-    if buffering == 1 or buffering < 0 and raw.isatty():
-        buffering = -1
-        line_buffering = True
-    if buffering < 0:
-        buffering = DEFAULT_BUFFER_SIZE
-        try:
-            bs = os.fstat(raw.fileno()).st_blksize
-        except (os.error, AttributeError):
-            pass
+    result = raw
+    try:
+        line_buffering = False
+        if buffering == 1 or buffering < 0 and raw.isatty():
+            buffering = -1
+            line_buffering = True
+        if buffering < 0:
+            buffering = DEFAULT_BUFFER_SIZE
+            try:
+                bs = os.fstat(raw.fileno()).st_blksize
+            except (os.error, AttributeError):
+                pass
+            else:
+                if bs > 1:
+                    buffering = bs
+        if buffering < 0:
+            raise ValueError("invalid buffering size")
+        if buffering == 0:
+            if binary:
+                return result
+            raise ValueError("can't have unbuffered text I/O")
+        if updating:
+            buffer = BufferedRandom(raw, buffering)
+        elif writing or appending:
+            buffer = BufferedWriter(raw, buffering)
+        elif reading:
+            buffer = BufferedReader(raw, buffering)
         else:
-            if bs > 1:
-                buffering = bs
-    if buffering < 0:
-        raise ValueError("invalid buffering size")
-    if buffering == 0:
+            raise ValueError("unknown mode: %r" % mode)
+        result = buffer
         if binary:
-            return raw
-        raise ValueError("can't have unbuffered text I/O")
-    if updating:
-        buffer = BufferedRandom(raw, buffering)
-    elif writing or appending:
-        buffer = BufferedWriter(raw, buffering)
-    elif reading:
-        buffer = BufferedReader(raw, buffering)
-    else:
-        raise ValueError("unknown mode: %r" % mode)
-    if binary:
-        return buffer
-    text = TextIOWrapper(buffer, encoding, errors, newline, line_buffering)
-    text.mode = mode
-    return text
+            return result
+        text = TextIOWrapper(buffer, encoding, errors, newline, line_buffering)
+        result = text
+        text.mode = mode
+        return result
+    except:
+        result.close()
+        raise
 
 
 class DocDescriptor:
@@ -1997,7 +2004,13 @@
 
     def getvalue(self):
         self.flush()
-        return self.buffer.getvalue().decode(self._encoding, self._errors)
+        decoder = self._decoder or self._get_decoder()
+        old_state = decoder.getstate()
+        decoder.reset()
+        try:
+            return decoder.decode(self.buffer.getvalue(), final=True)
+        finally:
+            decoder.setstate(old_state)
 
     def __repr__(self):
         # TextIOWrapper tells the encoding in its repr. In StringIO,
diff --git a/lib-python/2.7/_weakrefset.py b/lib-python/2.7/_weakrefset.py
--- a/lib-python/2.7/_weakrefset.py
+++ b/lib-python/2.7/_weakrefset.py
@@ -60,6 +60,8 @@
             for itemref in self.data:
                 item = itemref()
                 if item is not None:
+                    # Caveat: the iterator will keep a strong reference to
+                    # `item` until it is resumed or closed.
                     yield item
 
     def __len__(self):
diff --git a/lib-python/2.7/aifc.py b/lib-python/2.7/aifc.py
--- a/lib-python/2.7/aifc.py
+++ b/lib-python/2.7/aifc.py
@@ -778,7 +778,7 @@
 
     def _ensure_header_written(self, datasize):
         if not self._nframeswritten:
-            if self._comptype in ('ULAW', 'ALAW'):
+            if self._comptype in ('ULAW', 'ulaw', 'ALAW', 'alaw'):
                 if not self._sampwidth:
                     self._sampwidth = 2
                 if self._sampwidth != 2:
@@ -844,7 +844,7 @@
         if self._datalength & 1:
             self._datalength = self._datalength + 1
         if self._aifc:
-            if self._comptype in ('ULAW', 'ALAW'):
+            if self._comptype in ('ULAW', 'ulaw', 'ALAW', 'alaw'):
                 self._datalength = self._datalength // 2
                 if self._datalength & 1:
                     self._datalength = self._datalength + 1
@@ -852,7 +852,10 @@
                 self._datalength = (self._datalength + 3) // 4
                 if self._datalength & 1:
                     self._datalength = self._datalength + 1
-        self._form_length_pos = self._file.tell()
+        try:
+            self._form_length_pos = self._file.tell()
+        except (AttributeError, IOError):
+            self._form_length_pos = None
         commlength = self._write_form_length(self._datalength)
         if self._aifc:
             self._file.write('AIFC')
@@ -864,7 +867,8 @@
         self._file.write('COMM')
         _write_ulong(self._file, commlength)
         _write_short(self._file, self._nchannels)
-        self._nframes_pos = self._file.tell()
+        if self._form_length_pos is not None:
+            self._nframes_pos = self._file.tell()
         _write_ulong(self._file, self._nframes)
         if self._comptype in ('ULAW', 'ulaw', 'ALAW', 'alaw', 'G722'):
             _write_short(self._file, 8)
@@ -875,7 +879,8 @@
             self._file.write(self._comptype)
             _write_string(self._file, self._compname)
         self._file.write('SSND')
-        self._ssnd_length_pos = self._file.tell()
+        if self._form_length_pos is not None:
+            self._ssnd_length_pos = self._file.tell()
         _write_ulong(self._file, self._datalength + 8)
         _write_ulong(self._file, 0)
         _write_ulong(self._file, 0)
diff --git a/lib-python/2.7/argparse.py b/lib-python/2.7/argparse.py
--- a/lib-python/2.7/argparse.py
+++ b/lib-python/2.7/argparse.py
@@ -168,6 +168,8 @@
         self._prog = prog
         self._indent_increment = indent_increment
         self._max_help_position = max_help_position
+        self._max_help_position = min(max_help_position,
+                                      max(width - 20, indent_increment * 2))
         self._width = width
 
         self._current_indent = 0
@@ -339,7 +341,7 @@
                     else:
                         line_len = len(indent) - 1
                     for part in parts:
-                        if line_len + 1 + len(part) > text_width:
+                        if line_len + 1 + len(part) > text_width and line:
                             lines.append(indent + ' '.join(line))
                             line = []
                             line_len = len(indent) - 1
@@ -478,7 +480,7 @@
     def _format_text(self, text):
         if '%(prog)' in text:
             text = text % dict(prog=self._prog)
-        text_width = self._width - self._current_indent
+        text_width = max(self._width - self._current_indent, 11)
         indent = ' ' * self._current_indent
         return self._fill_text(text, text_width, indent) + '\n\n'
 
@@ -486,7 +488,7 @@
         # determine the required width and the entry label
         help_position = min(self._action_max_length + 2,
                             self._max_help_position)
-        help_width = self._width - help_position
+        help_width = max(self._width - help_position, 11)
         action_width = help_position - self._current_indent - 2
         action_header = self._format_action_invocation(action)
 
@@ -1155,9 +1157,13 @@
     __hash__ = None
 
     def __eq__(self, other):
+        if not isinstance(other, Namespace):
+            return NotImplemented
         return vars(self) == vars(other)
 
     def __ne__(self, other):
+        if not isinstance(other, Namespace):
+            return NotImplemented
         return not (self == other)
 
     def __contains__(self, key):
diff --git a/lib-python/2.7/bsddb/dbshelve.py b/lib-python/2.7/bsddb/dbshelve.py
--- a/lib-python/2.7/bsddb/dbshelve.py
+++ b/lib-python/2.7/bsddb/dbshelve.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 #------------------------------------------------------------------------
 #           Copyright (c) 1997-2001 by Total Control Software
 #                         All Rights Reserved
diff --git a/lib-python/2.7/bsddb/test/test_dbtables.py b/lib-python/2.7/bsddb/test/test_dbtables.py
--- a/lib-python/2.7/bsddb/test/test_dbtables.py
+++ b/lib-python/2.7/bsddb/test/test_dbtables.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-#
 #-----------------------------------------------------------------------
 # A test suite for the table interface built on bsddb.db
 #-----------------------------------------------------------------------
diff --git a/lib-python/2.7/codecs.py b/lib-python/2.7/codecs.py
--- a/lib-python/2.7/codecs.py
+++ b/lib-python/2.7/codecs.py
@@ -456,15 +456,12 @@
 
         # read until we get the required number of characters (if available)
         while True:
-            # can the request can be satisfied from the character buffer?
-            if chars < 0:
-                if size < 0:
-                    if self.charbuffer:
-                        break
-                elif len(self.charbuffer) >= size:
+            # can the request be satisfied from the character buffer?
+            if chars >= 0:
+                if len(self.charbuffer) >= chars:
                     break
-            else:
-                if len(self.charbuffer) >= chars:
+            elif size >= 0:
+                if len(self.charbuffer) >= size:
                     break
             # we need more data
             if size < 0:
diff --git a/lib-python/2.7/collections.py b/lib-python/2.7/collections.py
--- a/lib-python/2.7/collections.py
+++ b/lib-python/2.7/collections.py
@@ -319,6 +319,7 @@
     if isinstance(field_names, basestring):
         field_names = field_names.replace(',', ' ').split()
     field_names = map(str, field_names)
+    typename = str(typename)
     if rename:
         seen = set()
         for index, name in enumerate(field_names):
@@ -331,6 +332,8 @@
                 field_names[index] = '_%d' % index
             seen.add(name)
     for name in [typename] + field_names:
+        if type(name) != str:
+            raise TypeError('Type names and field names must be strings')
         if not all(c.isalnum() or c=='_' for c in name):
             raise ValueError('Type names and field names can only contain '
                              'alphanumeric characters and underscores: %r' % name)
diff --git a/lib-python/2.7/csv.py b/lib-python/2.7/csv.py
--- a/lib-python/2.7/csv.py
+++ b/lib-python/2.7/csv.py
@@ -93,6 +93,10 @@
         self.line_num = self.reader.line_num
         return self._fieldnames
 
+    # Issue 20004: Because DictReader is a classic class, this setter is
+    # ignored.  At this point in 2.7's lifecycle, it is too late to change the
+    # base class for fear of breaking working code.  If you want to change
+    # fieldnames without overwriting the getter, set _fieldnames directly.
     @fieldnames.setter
     def fieldnames(self, value):
         self._fieldnames = value
@@ -140,8 +144,8 @@
         if self.extrasaction == "raise":
             wrong_fields = [k for k in rowdict if k not in self.fieldnames]
             if wrong_fields:
-                raise ValueError("dict contains fields not in fieldnames: " +
-                                 ", ".join(wrong_fields))
+                raise ValueError("dict contains fields not in fieldnames: "
+                                 + ", ".join([repr(x) for x in wrong_fields]))
         return [rowdict.get(key, self.restval) for key in self.fieldnames]
 
     def writerow(self, rowdict):
diff --git a/lib-python/2.7/ctypes/test/__init__.py b/lib-python/2.7/ctypes/test/__init__.py
--- a/lib-python/2.7/ctypes/test/__init__.py
+++ b/lib-python/2.7/ctypes/test/__init__.py
@@ -2,7 +2,15 @@
 
 use_resources = []
 
-class ResourceDenied(Exception):
+import ctypes
+ctypes_symbols = dir(ctypes)
+
+def need_symbol(name):
+    return unittest.skipUnless(name in ctypes_symbols,
+                               '{!r} is required'.format(name))
+
+
+class ResourceDenied(unittest.SkipTest):
     """Test skipped because it requested a disallowed resource.
 
     This is raised when a test calls requires() for a resource that
diff --git a/lib-python/2.7/ctypes/test/test_arrays.py b/lib-python/2.7/ctypes/test/test_arrays.py
--- a/lib-python/2.7/ctypes/test/test_arrays.py
+++ b/lib-python/2.7/ctypes/test/test_arrays.py
@@ -2,6 +2,8 @@
 from ctypes import *
 from test.test_support import impl_detail
 
+from ctypes.test import need_symbol
+
 formats = "bBhHiIlLqQfd"
 
 # c_longdouble commented out for PyPy, look at the commend in test_longdouble
@@ -98,8 +100,8 @@
         self.assertEqual(values, [1, 2, 3, 4, 5])
 
     def test_classcache(self):
-        self.assertTrue(not ARRAY(c_int, 3) is ARRAY(c_int, 4))
-        self.assertTrue(ARRAY(c_int, 3) is ARRAY(c_int, 3))
+        self.assertIsNot(ARRAY(c_int, 3), ARRAY(c_int, 4))
+        self.assertIs(ARRAY(c_int, 3), ARRAY(c_int, 3))
 
     def test_from_address(self):
         # Failed with 0.9.8, reported by JUrner
@@ -112,20 +114,16 @@
         self.assertEqual(sz[1:4:2], "o")
         self.assertEqual(sz.value, "foo")
 
-    try:
-        create_unicode_buffer
-    except NameError:
-        pass
-    else:
-        def test_from_addressW(self):
-            p = create_unicode_buffer("foo")
-            sz = (c_wchar * 3).from_address(addressof(p))
-            self.assertEqual(sz[:], "foo")
-            self.assertEqual(sz[::], "foo")
-            self.assertEqual(sz[::-1], "oof")
-            self.assertEqual(sz[::3], "f")
-            self.assertEqual(sz[1:4:2], "o")
-            self.assertEqual(sz.value, "foo")
+    @need_symbol('create_unicode_buffer')
+    def test_from_addressW(self):
+        p = create_unicode_buffer("foo")
+        sz = (c_wchar * 3).from_address(addressof(p))
+        self.assertEqual(sz[:], "foo")
+        self.assertEqual(sz[::], "foo")
+        self.assertEqual(sz[::-1], "oof")
+        self.assertEqual(sz[::3], "f")
+        self.assertEqual(sz[1:4:2], "o")
+        self.assertEqual(sz.value, "foo")
 
     def test_cache(self):
         # Array types are cached internally in the _ctypes extension,
@@ -139,7 +137,7 @@
         # Create a new array type based on it:
         t1 = my_int * 1
         t2 = my_int * 1
-        self.assertTrue(t1 is t2)
+        self.assertIs(t1, t2)
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_as_parameter.py b/lib-python/2.7/ctypes/test/test_as_parameter.py
--- a/lib-python/2.7/ctypes/test/test_as_parameter.py
+++ b/lib-python/2.7/ctypes/test/test_as_parameter.py
@@ -1,5 +1,6 @@
 import unittest
 from ctypes import *
+from ctypes.test import need_symbol
 import _ctypes_test
 
 dll = CDLL(_ctypes_test.__file__)
@@ -17,11 +18,8 @@
     def wrap(self, param):
         return param
 
+    @need_symbol('c_wchar')
     def test_wchar_parm(self):
-        try:
-            c_wchar
-        except NameError:
-            return
         f = dll._testfunc_i_bhilfd
         f.argtypes = [c_byte, c_wchar, c_int, c_long, c_float, c_double]
         result = f(self.wrap(1), self.wrap(u"x"), self.wrap(3), self.wrap(4), self.wrap(5.0), self.wrap(6.0))
@@ -134,7 +132,7 @@
         f.argtypes = [c_longlong, MyCallback]
 
         def callback(value):
-            self.assertTrue(isinstance(value, (int, long)))
+            self.assertIsInstance(value, (int, long))
             return value & 0x7FFFFFFF
 
         cb = MyCallback(callback)
diff --git a/lib-python/2.7/ctypes/test/test_bitfields.py b/lib-python/2.7/ctypes/test/test_bitfields.py
--- a/lib-python/2.7/ctypes/test/test_bitfields.py
+++ b/lib-python/2.7/ctypes/test/test_bitfields.py
@@ -1,4 +1,5 @@
 from ctypes import *
+from ctypes.test import need_symbol
 import unittest
 import os
 
@@ -131,15 +132,6 @@
         self.assertEqual(result[0], TypeError)
         self.assertIn('bit fields not allowed for type', result[1])
 
-        try:
-            c_wchar
-        except NameError:
-            pass
-        else:
-            result = self.fail_fields(("a", c_wchar, 1))
-            self.assertEqual(result[0], TypeError)
-            self.assertIn('bit fields not allowed for type', result[1])
-
         class Dummy(Structure):
             _fields_ = []
 
@@ -147,6 +139,12 @@
         self.assertEqual(result[0], TypeError)
         self.assertIn('bit fields not allowed for type', result[1])
 
+    @need_symbol('c_wchar')
+    def test_c_wchar(self):
+        result = self.fail_fields(("a", c_wchar, 1))
+        self.assertEqual(result,
+                (TypeError, 'bit fields not allowed for type c_wchar'))
+
     def test_single_bitfield_size(self):
         for c_typ in int_types:
             result = self.fail_fields(("a", c_typ, -1))
@@ -213,7 +211,7 @@
         class X(Structure):
             _fields_ = [("a", c_byte, 4),
                         ("b", c_int, 32)]
-        self.assertEqual(sizeof(X), sizeof(c_int)*2)
+        self.assertEqual(sizeof(X), alignment(c_int)+sizeof(c_int))
 
     def test_mixed_3(self):
         class X(Structure):
@@ -246,7 +244,7 @@
             _anonymous_ = ["_"]
             _fields_ = [("_", X)]
 
-    @unittest.skipUnless(hasattr(ctypes, "c_uint32"), "c_int32 is required")
+    @need_symbol('c_uint32')
     def test_uint32(self):
         class X(Structure):
             _fields_ = [("a", c_uint32, 32)]
@@ -256,7 +254,7 @@
         x.a = 0xFDCBA987
         self.assertEqual(x.a, 0xFDCBA987)
 
-    @unittest.skipUnless(hasattr(ctypes, "c_uint64"), "c_int64 is required")
+    @need_symbol('c_uint64')
     def test_uint64(self):
         class X(Structure):
             _fields_ = [("a", c_uint64, 64)]
diff --git a/lib-python/2.7/ctypes/test/test_buffers.py b/lib-python/2.7/ctypes/test/test_buffers.py
--- a/lib-python/2.7/ctypes/test/test_buffers.py
+++ b/lib-python/2.7/ctypes/test/test_buffers.py
@@ -1,4 +1,5 @@
 from ctypes import *
+from ctypes.test import need_symbol
 import unittest
 
 class StringBufferTestCase(unittest.TestCase):
@@ -7,12 +8,12 @@
         b = create_string_buffer(32)
         self.assertEqual(len(b), 32)
         self.assertEqual(sizeof(b), 32 * sizeof(c_char))
-        self.assertTrue(type(b[0]) is str)
+        self.assertIs(type(b[0]), str)
 
         b = create_string_buffer("abc")
         self.assertEqual(len(b), 4) # trailing nul char
         self.assertEqual(sizeof(b), 4 * sizeof(c_char))
-        self.assertTrue(type(b[0]) is str)
+        self.assertIs(type(b[0]), str)
         self.assertEqual(b[0], "a")
         self.assertEqual(b[:], "abc\0")
         self.assertEqual(b[::], "abc\0")
@@ -36,39 +37,36 @@
         self.assertEqual(b[::2], "ac")
         self.assertEqual(b[::5], "a")
 
-    try:
-        c_wchar
-    except NameError:
-        pass
-    else:
-        def test_unicode_buffer(self):
-            b = create_unicode_buffer(32)
-            self.assertEqual(len(b), 32)
-            self.assertEqual(sizeof(b), 32 * sizeof(c_wchar))
-            self.assertTrue(type(b[0]) is unicode)
+    @need_symbol('c_wchar')
+    def test_unicode_buffer(self):
+        b = create_unicode_buffer(32)
+        self.assertEqual(len(b), 32)
+        self.assertEqual(sizeof(b), 32 * sizeof(c_wchar))
+        self.assertIs(type(b[0]), unicode)
 
-            b = create_unicode_buffer(u"abc")
-            self.assertEqual(len(b), 4) # trailing nul char
-            self.assertEqual(sizeof(b), 4 * sizeof(c_wchar))
-            self.assertTrue(type(b[0]) is unicode)
-            self.assertEqual(b[0], u"a")
-            self.assertEqual(b[:], "abc\0")
-            self.assertEqual(b[::], "abc\0")
-            self.assertEqual(b[::-1], "\0cba")
-            self.assertEqual(b[::2], "ac")
-            self.assertEqual(b[::5], "a")
+        b = create_unicode_buffer(u"abc")
+        self.assertEqual(len(b), 4) # trailing nul char
+        self.assertEqual(sizeof(b), 4 * sizeof(c_wchar))
+        self.assertIs(type(b[0]), unicode)
+        self.assertEqual(b[0], u"a")
+        self.assertEqual(b[:], "abc\0")
+        self.assertEqual(b[::], "abc\0")
+        self.assertEqual(b[::-1], "\0cba")
+        self.assertEqual(b[::2], "ac")
+        self.assertEqual(b[::5], "a")
 
-        def test_unicode_conversion(self):
-            b = create_unicode_buffer("abc")
-            self.assertEqual(len(b), 4) # trailing nul char
-            self.assertEqual(sizeof(b), 4 * sizeof(c_wchar))
-            self.assertTrue(type(b[0]) is unicode)
-            self.assertEqual(b[0], u"a")
-            self.assertEqual(b[:], "abc\0")
-            self.assertEqual(b[::], "abc\0")
-            self.assertEqual(b[::-1], "\0cba")
-            self.assertEqual(b[::2], "ac")
-            self.assertEqual(b[::5], "a")
+    @need_symbol('c_wchar')
+    def test_unicode_conversion(self):
+        b = create_unicode_buffer("abc")
+        self.assertEqual(len(b), 4) # trailing nul char
+        self.assertEqual(sizeof(b), 4 * sizeof(c_wchar))
+        self.assertIs(type(b[0]), unicode)
+        self.assertEqual(b[0], u"a")
+        self.assertEqual(b[:], "abc\0")
+        self.assertEqual(b[::], "abc\0")
+        self.assertEqual(b[::-1], "\0cba")
+        self.assertEqual(b[::2], "ac")
+        self.assertEqual(b[::5], "a")
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_byteswap.py b/lib-python/2.7/ctypes/test/test_byteswap.py
--- a/lib-python/2.7/ctypes/test/test_byteswap.py
+++ b/lib-python/2.7/ctypes/test/test_byteswap.py
@@ -15,7 +15,8 @@
 # For Structures and Unions, these types are created on demand.
 
 class Test(unittest.TestCase):
-    def X_test(self):
+    @unittest.skip('test disabled')
+    def test_X(self):
         print >> sys.stderr,  sys.byteorder
         for i in range(32):
             bits = BITS()
@@ -25,11 +26,11 @@
     @xfail
     def test_endian_short(self):
         if sys.byteorder == "little":
-            self.assertTrue(c_short.__ctype_le__ is c_short)
-            self.assertTrue(c_short.__ctype_be__.__ctype_le__ is c_short)
+            self.assertIs(c_short.__ctype_le__, c_short)
+            self.assertIs(c_short.__ctype_be__.__ctype_le__, c_short)
         else:
-            self.assertTrue(c_short.__ctype_be__ is c_short)
-            self.assertTrue(c_short.__ctype_le__.__ctype_be__ is c_short)
+            self.assertIs(c_short.__ctype_be__, c_short)
+            self.assertIs(c_short.__ctype_le__.__ctype_be__, c_short)
         s = c_short.__ctype_be__(0x1234)
         self.assertEqual(bin(struct.pack(">h", 0x1234)), "1234")
         self.assertEqual(bin(s), "1234")
@@ -53,11 +54,11 @@
     @xfail
     def test_endian_int(self):
         if sys.byteorder == "little":
-            self.assertTrue(c_int.__ctype_le__ is c_int)
-            self.assertTrue(c_int.__ctype_be__.__ctype_le__ is c_int)
+            self.assertIs(c_int.__ctype_le__, c_int)
+            self.assertIs(c_int.__ctype_be__.__ctype_le__, c_int)
         else:
-            self.assertTrue(c_int.__ctype_be__ is c_int)
-            self.assertTrue(c_int.__ctype_le__.__ctype_be__ is c_int)
+            self.assertIs(c_int.__ctype_be__, c_int)
+            self.assertIs(c_int.__ctype_le__.__ctype_be__, c_int)
 
         s = c_int.__ctype_be__(0x12345678)
         self.assertEqual(bin(struct.pack(">i", 0x12345678)), "12345678")
@@ -82,11 +83,11 @@
     @xfail
     def test_endian_longlong(self):
         if sys.byteorder == "little":
-            self.assertTrue(c_longlong.__ctype_le__ is c_longlong)
-            self.assertTrue(c_longlong.__ctype_be__.__ctype_le__ is c_longlong)
+            self.assertIs(c_longlong.__ctype_le__, c_longlong)
+            self.assertIs(c_longlong.__ctype_be__.__ctype_le__, c_longlong)
         else:
-            self.assertTrue(c_longlong.__ctype_be__ is c_longlong)
-            self.assertTrue(c_longlong.__ctype_le__.__ctype_be__ is c_longlong)
+            self.assertIs(c_longlong.__ctype_be__, c_longlong)
+            self.assertIs(c_longlong.__ctype_le__.__ctype_be__, c_longlong)
 
         s = c_longlong.__ctype_be__(0x1234567890ABCDEF)
         self.assertEqual(bin(struct.pack(">q", 0x1234567890ABCDEF)), "1234567890ABCDEF")
@@ -111,11 +112,11 @@
     @xfail
     def test_endian_float(self):
         if sys.byteorder == "little":
-            self.assertTrue(c_float.__ctype_le__ is c_float)
-            self.assertTrue(c_float.__ctype_be__.__ctype_le__ is c_float)
+            self.assertIs(c_float.__ctype_le__, c_float)
+            self.assertIs(c_float.__ctype_be__.__ctype_le__, c_float)
         else:
-            self.assertTrue(c_float.__ctype_be__ is c_float)
-            self.assertTrue(c_float.__ctype_le__.__ctype_be__ is c_float)
+            self.assertIs(c_float.__ctype_be__, c_float)
+            self.assertIs(c_float.__ctype_le__.__ctype_be__, c_float)
         s = c_float(math.pi)
         self.assertEqual(bin(struct.pack("f", math.pi)), bin(s))
         # Hm, what's the precision of a float compared to a double?
@@ -130,11 +131,11 @@
     @xfail
     def test_endian_double(self):
         if sys.byteorder == "little":
-            self.assertTrue(c_double.__ctype_le__ is c_double)
-            self.assertTrue(c_double.__ctype_be__.__ctype_le__ is c_double)
+            self.assertIs(c_double.__ctype_le__, c_double)
+            self.assertIs(c_double.__ctype_be__.__ctype_le__, c_double)
         else:
-            self.assertTrue(c_double.__ctype_be__ is c_double)
-            self.assertTrue(c_double.__ctype_le__.__ctype_be__ is c_double)
+            self.assertIs(c_double.__ctype_be__, c_double)
+            self.assertIs(c_double.__ctype_le__.__ctype_be__, c_double)
         s = c_double(math.pi)
         self.assertEqual(s.value, math.pi)
         self.assertEqual(bin(struct.pack("d", math.pi)), bin(s))
@@ -146,14 +147,14 @@
         self.assertEqual(bin(struct.pack(">d", math.pi)), bin(s))
 
     def test_endian_other(self):
-        self.assertTrue(c_byte.__ctype_le__ is c_byte)
-        self.assertTrue(c_byte.__ctype_be__ is c_byte)
+        self.assertIs(c_byte.__ctype_le__, c_byte)
+        self.assertIs(c_byte.__ctype_be__, c_byte)
 
-        self.assertTrue(c_ubyte.__ctype_le__ is c_ubyte)
-        self.assertTrue(c_ubyte.__ctype_be__ is c_ubyte)
+        self.assertIs(c_ubyte.__ctype_le__, c_ubyte)
+        self.assertIs(c_ubyte.__ctype_be__, c_ubyte)
 
-        self.assertTrue(c_char.__ctype_le__ is c_char)
-        self.assertTrue(c_char.__ctype_be__ is c_char)
+        self.assertIs(c_char.__ctype_le__, c_char)
+        self.assertIs(c_char.__ctype_be__, c_char)
 
     @xfail
     def test_struct_fields_1(self):
diff --git a/lib-python/2.7/ctypes/test/test_callbacks.py b/lib-python/2.7/ctypes/test/test_callbacks.py
--- a/lib-python/2.7/ctypes/test/test_callbacks.py
+++ b/lib-python/2.7/ctypes/test/test_callbacks.py
@@ -1,5 +1,6 @@
 import unittest
 from ctypes import *
+from ctypes.test import need_symbol
 from ctypes.test import xfail
 import _ctypes_test
 
@@ -95,9 +96,10 @@
     # disabled: would now (correctly) raise a RuntimeWarning about
     # a memory leak.  A callback function cannot return a non-integral
     # C type without causing a memory leak.
-##    def test_char_p(self):
-##        self.check_type(c_char_p, "abc")
-##        self.check_type(c_char_p, "def")
+    @unittest.skip('test disabled')
+    def test_char_p(self):
+        self.check_type(c_char_p, "abc")
+        self.check_type(c_char_p, "def")
 
     @xfail
     def test_pyobject(self):
@@ -150,13 +152,12 @@
         CFUNCTYPE(None)(lambda x=Nasty(): None)
 
 
-try:
-    WINFUNCTYPE
-except NameError:
-    pass
-else:
-    class StdcallCallbacks(Callbacks):
+ at need_symbol('WINFUNCTYPE')
+class StdcallCallbacks(Callbacks):
+    try:
         functype = WINFUNCTYPE
+    except NameError:
+        pass
 
 ################################################################
 
@@ -186,7 +187,7 @@
         from ctypes.util import find_library
         libc_path = find_library("c")
         if not libc_path:
-            return # cannot test
+            self.skipTest('could not find libc')
         libc = CDLL(libc_path)
 
         @CFUNCTYPE(c_int, POINTER(c_int), POINTER(c_int))
@@ -198,23 +199,19 @@
         libc.qsort(array, len(array), sizeof(c_int), cmp_func)
         self.assertEqual(array[:], [1, 5, 7, 33, 99])
 
-    try:
-        WINFUNCTYPE
-    except NameError:
-        pass
-    else:
-        def test_issue_8959_b(self):
-            from ctypes.wintypes import BOOL, HWND, LPARAM
+    @need_symbol('WINFUNCTYPE')
+    def test_issue_8959_b(self):
+        from ctypes.wintypes import BOOL, HWND, LPARAM
+        global windowCount
+        windowCount = 0
+
+        @WINFUNCTYPE(BOOL, HWND, LPARAM)
+        def EnumWindowsCallbackFunc(hwnd, lParam):
             global windowCount
-            windowCount = 0
+            windowCount += 1
+            return True #Allow windows to keep enumerating
 
-            @WINFUNCTYPE(BOOL, HWND, LPARAM)
-            def EnumWindowsCallbackFunc(hwnd, lParam):
-                global windowCount
-                windowCount += 1
-                return True #Allow windows to keep enumerating
-
-            windll.user32.EnumWindows(EnumWindowsCallbackFunc, 0)
+        windll.user32.EnumWindows(EnumWindowsCallbackFunc, 0)
 
     def test_callback_register_int(self):
         # Issue #8275: buggy handling of callback args under Win64
diff --git a/lib-python/2.7/ctypes/test/test_cast.py b/lib-python/2.7/ctypes/test/test_cast.py
--- a/lib-python/2.7/ctypes/test/test_cast.py
+++ b/lib-python/2.7/ctypes/test/test_cast.py
@@ -1,4 +1,5 @@
 from ctypes import *
+from ctypes.test import need_symbol
 import unittest
 import sys
 
@@ -38,14 +39,14 @@
 
         p = cast(array, POINTER(c_char_p))
         # array and p share a common _objects attribute
-        self.assertTrue(p._objects is array._objects)
+        self.assertIs(p._objects, array._objects)
         self.assertEqual(array._objects, {'0': "foo bar", id(array): array})
         p[0] = "spam spam"
         self.assertEqual(p._objects, {'0': "spam spam", id(array): array})
-        self.assertTrue(array._objects is p._objects)
+        self.assertIs(array._objects, p._objects)
         p[1] = "foo bar"
         self.assertEqual(p._objects, {'1': 'foo bar', '0': "spam spam", id(array): array})
-        self.assertTrue(array._objects is p._objects)
+        self.assertIs(array._objects, p._objects)
 
     def test_other(self):
         p = cast((c_int * 4)(1, 2, 3, 4), POINTER(c_int))
@@ -75,15 +76,11 @@
         self.assertEqual(cast(cast(s, c_void_p), c_char_p).value,
                              "hiho")
 
-    try:
-        c_wchar_p
-    except NameError:
-        pass
-    else:
-        def test_wchar_p(self):
-            s = c_wchar_p("hiho")
-            self.assertEqual(cast(cast(s, c_void_p), c_wchar_p).value,
-                                 "hiho")
+    @need_symbol('c_wchar_p')
+    def test_wchar_p(self):
+        s = c_wchar_p("hiho")
+        self.assertEqual(cast(cast(s, c_void_p), c_wchar_p).value,
+                             "hiho")
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_cfuncs.py b/lib-python/2.7/ctypes/test/test_cfuncs.py
--- a/lib-python/2.7/ctypes/test/test_cfuncs.py
+++ b/lib-python/2.7/ctypes/test/test_cfuncs.py
@@ -3,6 +3,7 @@
 
 import unittest
 from ctypes import *
+from ctypes.test import need_symbol
 
 import _ctypes_test
 from test.test_support import impl_detail
@@ -196,7 +197,7 @@
 try:
     WinDLL
 except NameError:
-    pass
+    def stdcall_dll(*_): pass
 else:
     class stdcall_dll(WinDLL):
         def __getattr__(self, name):
@@ -206,9 +207,9 @@
             setattr(self, name, func)
             return func
 
-    class stdcallCFunctions(CFunctions):
-        _dll = stdcall_dll(_ctypes_test.__file__)
-        pass
+ at need_symbol('WinDLL')
+class stdcallCFunctions(CFunctions):
+    _dll = stdcall_dll(_ctypes_test.__file__)
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_checkretval.py b/lib-python/2.7/ctypes/test/test_checkretval.py
--- a/lib-python/2.7/ctypes/test/test_checkretval.py
+++ b/lib-python/2.7/ctypes/test/test_checkretval.py
@@ -1,6 +1,7 @@
 import unittest
 
 from ctypes import *
+from ctypes.test import need_symbol
 
 class CHECKED(c_int):
     def _check_retval_(value):
@@ -25,15 +26,11 @@
         del dll._testfunc_p_p.restype
         self.assertEqual(42, dll._testfunc_p_p(42))
 
-    try:
-        oledll
-    except NameError:
-        pass
-    else:
-        def test_oledll(self):
-            self.assertRaises(WindowsError,
-                                  oledll.oleaut32.CreateTypeLib2,
-                                  0, None, None)
+    @need_symbol('oledll')
+    def test_oledll(self):
+        self.assertRaises(WindowsError,
+                              oledll.oleaut32.CreateTypeLib2,
+                              0, None, None)
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_errcheck.py b/lib-python/2.7/ctypes/test/test_errcheck.py
deleted file mode 100644
--- a/lib-python/2.7/ctypes/test/test_errcheck.py
+++ /dev/null
@@ -1,19 +0,0 @@
-import sys
-from ctypes import *
-
-##class HMODULE(Structure):
-##    _fields_ = [("value", c_void_p)]
-
-##    def __repr__(self):
-##        return "" % self.value
-
-##windll.kernel32.GetModuleHandleA.restype = HMODULE
-
-##print windll.kernel32.GetModuleHandleA("python23.dll")
-##print hex(sys.dllhandle)
-
-##def nonzero(handle):
-##    return (GetLastError(), handle)
-
-##windll.kernel32.GetModuleHandleA.errcheck = nonzero
-##print windll.kernel32.GetModuleHandleA("spam")
diff --git a/lib-python/2.7/ctypes/test/test_find.py b/lib-python/2.7/ctypes/test/test_find.py
--- a/lib-python/2.7/ctypes/test/test_find.py
+++ b/lib-python/2.7/ctypes/test/test_find.py
@@ -1,4 +1,5 @@
 import unittest
+import os
 import sys
 from ctypes import *
 from ctypes.util import find_library
@@ -40,43 +41,43 @@
             except OSError:
                 pass
 
-    if lib_gl:
-        def test_gl(self):
-            if self.gl:
-                self.gl.glClearIndex
+    @unittest.skipUnless(lib_gl, 'lib_gl not available')
+    def test_gl(self):
+        if self.gl:
+            self.gl.glClearIndex
 
-    if lib_glu:
-        def test_glu(self):
-            if self.glu:
-                self.glu.gluBeginCurve
+    @unittest.skipUnless(lib_glu, 'lib_glu not available')
+    def test_glu(self):
+        if self.glu:
+            self.glu.gluBeginCurve
 
-    if lib_gle:
-        def test_gle(self):
-            if self.gle:
-                self.gle.gleGetJoinStyle
+    @unittest.skipUnless(lib_gle, 'lib_gle not available')
+    def test_gle(self):
+        if self.gle:
+            self.gle.gleGetJoinStyle
 
-##if os.name == "posix" and sys.platform != "darwin":
-
-##    # On platforms where the default shared library suffix is '.so',
-##    # at least some libraries can be loaded as attributes of the cdll
-##    # object, since ctypes now tries loading the lib again
-##    # with '.so' appended of the first try fails.
-##    #
-##    # Won't work for libc, unfortunately.  OTOH, it isn't
-##    # needed for libc since this is already mapped into the current
-##    # process (?)
-##    #
-##    # On MAC OSX, it won't work either, because dlopen() needs a full path,
-##    # and the default suffix is either none or '.dylib'.
-
-##    class LoadLibs(unittest.TestCase):
-##        def test_libm(self):
-##            import math
-##            libm = cdll.libm
-##            sqrt = libm.sqrt
-##            sqrt.argtypes = (c_double,)
-##            sqrt.restype = c_double
-##            self.assertEqual(sqrt(2), math.sqrt(2))
+# On platforms where the default shared library suffix is '.so',
+# at least some libraries can be loaded as attributes of the cdll
+# object, since ctypes now tries loading the lib again
+# with '.so' appended of the first try fails.
+#
+# Won't work for libc, unfortunately.  OTOH, it isn't
+# needed for libc since this is already mapped into the current
+# process (?)
+#
+# On MAC OSX, it won't work either, because dlopen() needs a full path,
+# and the default suffix is either none or '.dylib'.
+ at unittest.skip('test disabled')
+ at unittest.skipUnless(os.name=="posix" and sys.platform != "darwin",
+                     'test not suitable for this platform')
+class LoadLibs(unittest.TestCase):
+    def test_libm(self):
+        import math
+        libm = cdll.libm
+        sqrt = libm.sqrt
+        sqrt.argtypes = (c_double,)
+        sqrt.restype = c_double
+        self.assertEqual(sqrt(2), math.sqrt(2))
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_frombuffer.py b/lib-python/2.7/ctypes/test/test_frombuffer.py
--- a/lib-python/2.7/ctypes/test/test_frombuffer.py
+++ b/lib-python/2.7/ctypes/test/test_frombuffer.py
@@ -25,7 +25,7 @@
         a[0], a[-1] = 200, -200
         self.assertEqual(x[:], a.tolist())
 
-        self.assertTrue(a in x._objects.values())
+        self.assertIn(a, x._objects.values())
 
         self.assertRaises(ValueError,
                           c_int.from_buffer, a, -1)
diff --git a/lib-python/2.7/ctypes/test/test_funcptr.py b/lib-python/2.7/ctypes/test/test_funcptr.py
--- a/lib-python/2.7/ctypes/test/test_funcptr.py
+++ b/lib-python/2.7/ctypes/test/test_funcptr.py
@@ -75,7 +75,7 @@
         ##                  "lpfnWndProc", WNDPROC_2(wndproc))
         # instead:
 
-        self.assertTrue(WNDPROC is WNDPROC_2)
+        self.assertIs(WNDPROC, WNDPROC_2)
         # 'wndclass.lpfnWndProc' leaks 94 references.  Why?
         self.assertEqual(wndclass.lpfnWndProc(1, 2, 3, 4), 10)
 
diff --git a/lib-python/2.7/ctypes/test/test_functions.py b/lib-python/2.7/ctypes/test/test_functions.py
--- a/lib-python/2.7/ctypes/test/test_functions.py
+++ b/lib-python/2.7/ctypes/test/test_functions.py
@@ -6,6 +6,7 @@
 """
 
 from ctypes import *
+from ctypes.test import need_symbol
 import sys, unittest
 from ctypes.test import xfail
 from test.test_support import impl_detail
@@ -65,22 +66,16 @@
             pass
 
 
+    @need_symbol('c_wchar')
     def test_wchar_parm(self):
-        try:
-            c_wchar
-        except NameError:
-            return
         f = dll._testfunc_i_bhilfd
         f.argtypes = [c_byte, c_wchar, c_int, c_long, c_float, c_double]
         result = f(1, u"x", 3, 4, 5.0, 6.0)
         self.assertEqual(result, 139)
         self.assertEqual(type(result), int)
 
+    @need_symbol('c_wchar')
     def test_wchar_result(self):
-        try:
-            c_wchar
-        except NameError:
-            return
         f = dll._testfunc_i_bhilfd
         f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double]
         f.restype = c_wchar
@@ -158,11 +153,8 @@
         self.assertEqual(result, -21)

From noreply at buildbot.pypy.org  Fri Sep 12 16:45:36 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 12 Sep 2014 16:45:36 +0200 (CEST)
Subject: [pypy-commit] pypy win32-fixes5: check more carefully for empty
	unions
Message-ID: <20140912144536.E91031D2970@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: win32-fixes5
Changeset: r73505:f2ddd548054d
Date: 2014-09-12 17:45 +0300
http://bitbucket.org/pypy/pypy/changeset/f2ddd548054d/

Log:	check more carefully for empty unions

diff --git a/rpython/translator/c/node.py b/rpython/translator/c/node.py
--- a/rpython/translator/c/node.py
+++ b/rpython/translator/c/node.py
@@ -525,7 +525,7 @@
             # a union with length 0
             lines[0] = cdecl(type, name, self.is_thread_local())
         else:
-            if name != self.name:    
+            if name != self.name:
                 lines[0] = '{ ' + lines[0]    # extra braces around the 'a' part
                 lines[-1] += ' }'             # of the union
             lines[0] = '%s = %s' % (
@@ -567,12 +567,6 @@
     def initializationexpr(self, decoration=''):
         T = self.getTYPE()
         is_empty = True
-        type, name = self.get_declaration()
-        if name != self.name and self.getvarlength() < 1:
-            # an empty union
-            yield ''
-            return
-        yield '{'
         defnode = self.db.gettypedefnode(T)
 
         data = []
@@ -601,7 +595,13 @@
             padding_drop = T._hints['get_padding_drop'](d)
         else:
             padding_drop = []
+        type, name = self.get_declaration()
+        if name != self.name and self.getvarlength() < 1 and len(data) < 1:
+            # an empty union
+            yield ''
+            return
 
+        yield '{'
         for name, value in data:
             if name in padding_drop:
                 continue

From noreply at buildbot.pypy.org  Fri Sep 12 17:15:45 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri, 12 Sep 2014 17:15:45 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: fix buffering arg to
	fdopen
Message-ID: <20140912151545.D2D351C073F@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73506:645e06e76b50
Date: 2014-09-12 11:13 -0400
http://bitbucket.org/pypy/pypy/changeset/645e06e76b50/

Log:	fix buffering arg to fdopen

diff --git a/pypy/module/_file/interp_file.py b/pypy/module/_file/interp_file.py
--- a/pypy/module/_file/interp_file.py
+++ b/pypy/module/_file/interp_file.py
@@ -105,6 +105,12 @@
         stream = rfile.create_file(fsencode_w(self.space, w_name), mode, buffering)
         self.fdopenstream(stream, mode)
 
+    def direct_fdopen(self, fd, mode='r', buffering=-1):
+        self.direct_close()
+        self.w_name = self.space.wrap('')
+        stream = rfile.create_fdopen_rfile(fd, mode, buffering)
+        self.fdopenstream(stream, mode)
+
     def direct___enter__(self):
         self.check_closed()
         return self
@@ -115,12 +121,6 @@
         # can't return close() value
         return None
 
-    def direct_fdopen(self, fd, mode='r', buffering=-1):
-        self.direct_close()
-        self.w_name = self.space.wrap('')
-        stream = rfile.create_fdopen_rfile(fd, mode)
-        self.fdopenstream(stream, mode)
-
     def direct_close(self):
         stream = self.stream
         if stream is not None:
diff --git a/pypy/module/_file/test/test_file.py b/pypy/module/_file/test/test_file.py
--- a/pypy/module/_file/test/test_file.py
+++ b/pypy/module/_file/test/test_file.py
@@ -267,7 +267,7 @@
         with self.file(self.temppath, 'r') as f:
             raises(IOError, f.truncate, 100)
 
-    def test_write_full(self):
+    def test_write_full_regular(self):
         try:
             f = self.file('/dev/full', 'w', 1)
         except IOError:
@@ -281,6 +281,22 @@
         finally:
             f.close()
 
+    def test_write_full_fdopen(self):
+        import os
+        fd = os.open('/dev/full', os.O_WRONLY)
+        try:
+            f = os.fdopen(fd, 'w', 1)
+        except IOError:
+            skip("requires '/dev/full'")
+        try:
+            f.write('hello')
+            raises(IOError, f.write, '\n')
+            f.write('zzz')
+            raises(IOError, f.flush)
+            f.flush()
+        finally:
+            f.close()
+
 
 class AppTestConcurrency(object):
     # these tests only really make sense on top of a translated pypy-c,

From noreply at buildbot.pypy.org  Fri Sep 12 17:29:08 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri, 12 Sep 2014 17:29:08 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: fix buffered fdopen
 tests in text_file_extra
Message-ID: <20140912152908.62D461C073F@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73507:4cb30452dec3
Date: 2014-09-12 11:28 -0400
http://bitbucket.org/pypy/pypy/changeset/4cb30452dec3/

Log:	fix buffered fdopen tests in text_file_extra

diff --git a/pypy/module/_file/test/test_file_extra.py b/pypy/module/_file/test/test_file_extra.py
--- a/pypy/module/_file/test/test_file_extra.py
+++ b/pypy/module/_file/test/test_file_extra.py
@@ -295,17 +295,18 @@
             w_filetype = os    # TinyObjSpace, for "py.test -A"
                                # (CPython has no file.fdopen, only os.fdopen)
         fd = os.open(AppTestFile.expected_filename, os.O_RDONLY | O_BINARY)
+        self.w_fd = space.wrap(fd)
+        self.w_sample = space.wrap(self.sample)
+        self.w_expected_filename = space.wrap(self.expected_filename)
+        self.w_expected_mode = space.wrap(self.expected_mode)
+        self.w_expected_lines = space.wrap(self.get_expected_lines())
+        leakfinder.start_tracking_allocations()
         self.w_file = space.call_method(
             w_filetype,
             'fdopen',
             space.wrap(fd),
             space.wrap(self.expected_mode),
             *[space.wrap(a) for a in self.extra_args])
-        self.w_fd = space.wrap(fd)
-        self.w_sample = space.wrap(self.sample)
-        self.w_expected_filename = space.wrap(self.expected_filename)
-        self.w_expected_mode = space.wrap(self.expected_mode)
-        self.w_expected_lines = space.wrap(self.get_expected_lines())
 
     def teardown_method(self, method):
         self.space.call_method(self.w_file, 'close')

From noreply at buildbot.pypy.org  Fri Sep 12 17:45:10 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri, 12 Sep 2014 17:45:10 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: do proper error handling
	in getline_fgets
Message-ID: <20140912154510.C55381C3641@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73508:9cc506ff40ab
Date: 2014-09-12 11:42 -0400
http://bitbucket.org/pypy/pypy/changeset/9cc506ff40ab/

Log:	do proper error handling in getline_fgets

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -445,6 +445,14 @@
 
         result = c_fgets(raw_buf, BASE_LINE_SIZE, ll_file)
         if not result:
+            if c_ferror(ll_file):
+                if rposix.get_errno() == errno.EINTR:
+                    if self._signal_checker is not None:
+                        self._signal_checker()
+                    c_clearerr(ll_file)
+                    return -1
+                c_clearerr(ll_file)
+                raise _from_errno(IOError)
             c_clearerr(ll_file)
             if self._signal_checker is not None:
                 self._signal_checker()
diff --git a/rpython/rlib/test/test_rfile.py b/rpython/rlib/test/test_rfile.py
--- a/rpython/rlib/test/test_rfile.py
+++ b/rpython/rlib/test/test_rfile.py
@@ -307,6 +307,28 @@
             f2.write("xxx")
             f2.close()
 
+            fd = os.open(fname, os.O_RDONLY, 0777)
+            f = os.fdopen(fd, 'r')
+            os.close(fd)
+            try:
+                f.read()
+            except IOError as e:
+                pass
+            else:
+                assert False
+            try:
+                f.readline()
+            except IOError as e:
+                pass
+            else:
+                assert False
+            try:
+                f.close()
+            except IOError as e:
+                pass
+            else:
+                assert False
+
         f()
         assert open(fname).read() == "xxx"
         self.interpret(f, [])

From noreply at buildbot.pypy.org  Fri Sep 12 18:01:57 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri, 12 Sep 2014 18:01:57 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: backout proper error
 handling here, as cpython doesn't do it
Message-ID: <20140912160157.EDE461D2970@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73509:4fb50815c912
Date: 2014-09-12 12:01 -0400
http://bitbucket.org/pypy/pypy/changeset/4fb50815c912/

Log:	backout proper error handling here, as cpython doesn't do it

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -445,14 +445,6 @@
 
         result = c_fgets(raw_buf, BASE_LINE_SIZE, ll_file)
         if not result:
-            if c_ferror(ll_file):
-                if rposix.get_errno() == errno.EINTR:
-                    if self._signal_checker is not None:
-                        self._signal_checker()
-                    c_clearerr(ll_file)
-                    return -1
-                c_clearerr(ll_file)
-                raise _from_errno(IOError)
             c_clearerr(ll_file)
             if self._signal_checker is not None:
                 self._signal_checker()
diff --git a/rpython/rlib/test/test_rfile.py b/rpython/rlib/test/test_rfile.py
--- a/rpython/rlib/test/test_rfile.py
+++ b/rpython/rlib/test/test_rfile.py
@@ -316,12 +316,15 @@
                 pass
             else:
                 assert False
-            try:
-                f.readline()
-            except IOError as e:
-                pass
+            if os.name != 'nt':
+                try:
+                    f.readline()
+                except IOError as e:
+                    pass
+                else:
+                    assert False
             else:
-                assert False
+                assert f.readline() == ''
             try:
                 f.close()
             except IOError as e:

From noreply at buildbot.pypy.org  Fri Sep 12 20:07:36 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri, 12 Sep 2014 20:07:36 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: retry when necessary in
	file.readinto
Message-ID: <20140912180736.C73BA1D23D3@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73510:a2cb5424ebf5
Date: 2014-09-12 14:06 -0400
http://bitbucket.org/pypy/pypy/changeset/a2cb5424ebf5/

Log:	retry when necessary in file.readinto

diff --git a/pypy/module/_file/interp_file.py b/pypy/module/_file/interp_file.py
--- a/pypy/module/_file/interp_file.py
+++ b/pypy/module/_file/interp_file.py
@@ -437,10 +437,18 @@
         # XXX not the most efficient solution as it doesn't avoid the copying
         space = self.space
         rwbuffer = space.writebuf_w(w_rwbuffer)
-        w_data = self.file_read(rwbuffer.getlength())
-        data = space.str_w(w_data)
-        rwbuffer.setslice(0, data)
-        return space.wrap(len(data))
+        ntodo = rwbuffer.getlength()
+        ndone = 0
+        while ntodo:
+            w_data = self.file_read(ntodo)
+            data = space.str_w(w_data)
+            nnow = len(data)
+            if nnow == 0:
+                break
+            rwbuffer.setslice(ndone, data)
+            ndone += nnow
+            ntodo -= nnow
+        return space.wrap(ndone)
 
 
 # ____________________________________________________________

From noreply at buildbot.pypy.org  Fri Sep 12 20:36:07 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri, 12 Sep 2014 20:36:07 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: skip /dev/full tests
	better
Message-ID: <20140912183607.DD8B91D2577@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73511:9227e33346d4
Date: 2014-09-12 11:35 -0700
http://bitbucket.org/pypy/pypy/changeset/9227e33346d4/

Log:	skip /dev/full tests better

diff --git a/pypy/module/_file/test/test_file.py b/pypy/module/_file/test/test_file.py
--- a/pypy/module/_file/test/test_file.py
+++ b/pypy/module/_file/test/test_file.py
@@ -267,11 +267,9 @@
         with self.file(self.temppath, 'r') as f:
             raises(IOError, f.truncate, 100)
 
+    @py.test.mark.skipif("not os.path.exists('/dev/full')")
     def test_write_full_regular(self):
-        try:
-            f = self.file('/dev/full', 'w', 1)
-        except IOError:
-            skip("requires '/dev/full'")
+        f = self.file('/dev/full', 'w', 1)
         try:
             f.write('hello')
             raises(IOError, f.write, '\n')
@@ -281,13 +279,11 @@
         finally:
             f.close()
 
+    @py.test.mark.skipif("not os.path.exists('/dev/full')")
     def test_write_full_fdopen(self):
         import os
         fd = os.open('/dev/full', os.O_WRONLY)
-        try:
-            f = os.fdopen(fd, 'w', 1)
-        except IOError:
-            skip("requires '/dev/full'")
+        f = os.fdopen(fd, 'w', 1)
         try:
             f.write('hello')
             raises(IOError, f.write, '\n')

From noreply at buildbot.pypy.org  Fri Sep 12 20:48:59 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri, 12 Sep 2014 20:48:59 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: these tests fail on mac
 even against cpython
Message-ID: <20140912184859.8CCBC1C073F@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73512:802c516c3dc6
Date: 2014-09-12 11:41 -0700
http://bitbucket.org/pypy/pypy/changeset/802c516c3dc6/

Log:	these tests fail on mac even against cpython

diff --git a/rpython/rlib/test/test_rfile.py b/rpython/rlib/test/test_rfile.py
--- a/rpython/rlib/test/test_rfile.py
+++ b/rpython/rlib/test/test_rfile.py
@@ -123,6 +123,7 @@
         f()
         self.interpret(f, [])
 
+    @py.test.mark.xfail("sys.platform == 'darwin'")
     def test_open_buffering_full(self):
         fname = str(self.tmpdir.join('file_1b'))
 
@@ -140,6 +141,7 @@
         f()
         self.interpret(f, [])
 
+    @py.test.mark.xfail("sys.platform == 'darwin'")
     def test_fdopen_buffering_full(self):
         fname = str(self.tmpdir.join('file_1b'))
 

From noreply at buildbot.pypy.org  Fri Sep 12 21:14:59 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Fri, 12 Sep 2014 21:14:59 +0200 (CEST)
Subject: [pypy-commit] pypy default: utilize scoped allocation
Message-ID: <20140912191459.1C44E1C1292@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey 
Branch: 
Changeset: r73513:e6aa1e2c6877
Date: 2014-09-12 12:13 -0700
http://bitbucket.org/pypy/pypy/changeset/e6aa1e2c6877/

Log:	utilize scoped allocation

diff --git a/rpython/translator/sandbox/rsandbox.py b/rpython/translator/sandbox/rsandbox.py
--- a/rpython/translator/sandbox/rsandbox.py
+++ b/rpython/translator/sandbox/rsandbox.py
@@ -60,8 +60,7 @@
 
     def need_more_data(self):
         buflen = self.buflen
-        buf = lltype.malloc(rffi.CCHARP.TO, buflen, flavor='raw')
-        try:
+        with lltype.scoped_alloc(rffi.CCHARP.TO, buflen) as buf:
             buflen = rffi.cast(rffi.SIZE_T, buflen)
             count = ll_read_not_sandboxed(self.fd, buf, buflen)
             count = rffi.cast(lltype.Signed, count)
@@ -69,20 +68,15 @@
                 raise IOError
             self.buf += ''.join([buf[i] for i in range(count)])
             self.buflen *= 2
-        finally:
-            lltype.free(buf, flavor='raw')
 
 def sandboxed_io(buf):
     STDIN = 0
     STDOUT = 1
     # send the buffer with the marshalled fnname and input arguments to STDOUT
-    p = lltype.malloc(rffi.CCHARP.TO, len(buf), flavor='raw')
-    try:
+    with lltype.scoped_alloc(rffi.CCHARP.TO, len(buf)) as p:
         for i in range(len(buf)):
             p[i] = buf[i]
         writeall_not_sandboxed(STDOUT, p, len(buf))
-    finally:
-        lltype.free(p, flavor='raw')
     # build a Loader that will get the answer from STDIN
     loader = FdLoader(STDIN)
     # check for errors
@@ -108,9 +102,8 @@
 @signature(types.str(), returns=types.impossible())
 def not_implemented_stub(msg):
     STDERR = 2
-    buf = rffi.str2charp(msg + '\n')
-    writeall_not_sandboxed(STDERR, buf, len(msg) + 1)
-    rffi.free_charp(buf)
+    with rffi.scoped_str2charp(msg + '\n') as buf:
+        writeall_not_sandboxed(STDERR, buf, len(msg) + 1)
     raise RuntimeError(msg)  # XXX in RPython, the msg is ignored at the moment
 
 dump_string = rmarshal.get_marshaller(str)

From noreply at buildbot.pypy.org  Fri Sep 12 21:17:45 2014
From: noreply at buildbot.pypy.org (rguillebert)
Date: Fri, 12 Sep 2014 21:17:45 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: typos
Message-ID: <20140912191745.AD2B01C1292@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert 
Branch: extradoc
Changeset: r5395:0a13f83cb05d
Date: 2014-09-12 21:17 +0200
http://bitbucket.org/pypy/extradoc/changeset/0a13f83cb05d/

Log:	typos

diff --git a/talk/pycon2015/status/abstract.rst b/talk/pycon2015/status/abstract.rst
--- a/talk/pycon2015/status/abstract.rst
+++ b/talk/pycon2015/status/abstract.rst
@@ -11,11 +11,11 @@
 we have achieved during the two years between talks. We would like to cover
 advancements in the PyPy performance landscape, but more importantly how
 we're addresssing the community needs and building the ecosystem. These days
-a lot of libraries that used to bind to C using CPython C API are either
+a lot of libraries that used to bind to C using the CPython C API are either
 using cffi or have alternatives using cffi.
 
 We would also like to walk through a few success stories that we have
-experienced. Unforunately the biggest chunk of PyPy clients are very
+experienced. Unfortunately the biggest chunk of PyPy clients are very
 secretive (e.g. trading companies), but we can still present a few case studies.
 
 XXXX

From noreply at buildbot.pypy.org  Fri Sep 12 22:20:16 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri, 12 Sep 2014 22:20:16 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: fix large seek/tell
	offsets
Message-ID: <20140912202016.C7D711C1292@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73514:9190ab687bde
Date: 2014-09-12 13:19 -0700
http://bitbucket.org/pypy/pypy/changeset/9190ab687bde/

Log:	fix large seek/tell offsets

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -112,8 +112,11 @@
 
 c_ftruncate = llexternal(ftruncate, [rffi.INT, OFF_T], rffi.INT, macro=True)
 
-c_fseek = llexternal('fseek', [FILEP, rffi.LONG, rffi.INT], rffi.INT)
-c_ftell = llexternal('ftell', [FILEP], rffi.LONG)
+c_fseek = llexternal('fseeko', [FILEP, OFF_T, rffi.INT], rffi.INT)
+c_ftell = llexternal('ftello', [FILEP], OFF_T)
+if os.name == 'nt':
+    c_fseek = llexternal('_fseeki64', [FILEP, rffi.LONGLONG, rffi.INT], rffi.INT)
+    c_ftell = llexternal('_ftelli64', [FILEP], rffi.LONGLONG)
 
 c_fileno = llexternal(fileno, [FILEP], rffi.INT, releasegil=False)
 c_feof = llexternal('feof', [FILEP], rffi.INT, releasegil=False)
@@ -592,7 +595,7 @@
         self._check_closed()
         self._check_writable()
         ll_file = self._ll_file
-        pos = intmask(c_ftell(ll_file))
+        pos = c_ftell(ll_file)
         if pos == -1:
             c_clearerr(ll_file)
             raise _from_errno(IOError)
@@ -628,7 +631,7 @@
 
     def tell(self):
         self._check_closed()
-        res = intmask(c_ftell(self._ll_file))
+        res = c_ftell(self._ll_file)
         if res == -1:
             c_clearerr(self._ll_file)
             raise _from_errno(IOError)

From noreply at buildbot.pypy.org  Fri Sep 12 23:24:49 2014
From: noreply at buildbot.pypy.org (rguillebert)
Date: Fri, 12 Sep 2014 23:24:49 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Add the talk description (WIP)
Message-ID: <20140912212449.8050B1D3495@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert 
Branch: extradoc
Changeset: r5396:f584b11b5444
Date: 2014-09-12 23:24 +0200
http://bitbucket.org/pypy/extradoc/changeset/f584b11b5444/

Log:	Add the talk description (WIP)

diff --git a/talk/pycon2015/status/abstract.rst b/talk/pycon2015/status/abstract.rst
--- a/talk/pycon2015/status/abstract.rst
+++ b/talk/pycon2015/status/abstract.rst
@@ -1,6 +1,17 @@
 PyPy - the last 2 years of progress
 ===================================
 
+Description (one paragraph, 400 chars max)
+------------------------------------------
+
+This talk describes what happened in the PyPy ecosystem in the last 2 years, a
+timeframe in which PyPy has been successfully deployed multiple times while
+yielding interesting performance improvements and a timeframe in which more
+libraries started being compatible with PyPy through the use of cffi.
+
+Detailed Abstract
+-----------------
+
 PyPy has been in the works for more than ten years and has reached relative
 maturity with more and more libraries working under PyPy and more deployments
 happening. Right now it entertains between 0.5-1.0% of PyPI package downloads
@@ -19,21 +30,3 @@
 secretive (e.g. trading companies), but we can still present a few case studies.
 
 XXXX
-
-Abstract
---------
-
-This talk will cover what has happened in the PyPy world during the last 2
-years :
-
-- Progress in Python 3 support and Numpy support
-
-- Performance improvements
-
-- An attempt to remove the GIL using Software Transactional Memory
-
-- CFFI : The best way to interface with C code on PyPy
-
-- Status of the crowdfunding
-
-- ARM

From noreply at buildbot.pypy.org  Sat Sep 13 00:19:11 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Sat, 13 Sep 2014 00:19:11 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: workaround lack of __future__.__file__,
 as __future__'s pre-imported during
Message-ID: <20140912221911.BD9731C1292@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey 
Branch: py3k
Changeset: r73516:73c7d67b9c5f
Date: 2014-09-12 15:18 -0700
http://bitbucket.org/pypy/pypy/changeset/73c7d67b9c5f/

Log:	workaround lack of __future__.__file__, as __future__'s pre-imported
	during translation

diff --git a/lib-python/3/test/test_modulefinder.py b/lib-python/3/test/test_modulefinder.py
--- a/lib-python/3/test/test_modulefinder.py
+++ b/lib-python/3/test/test_modulefinder.py
@@ -9,7 +9,7 @@
 import modulefinder
 
 TEST_DIR = tempfile.mkdtemp()
-TEST_PATH = [TEST_DIR, os.path.dirname(__future__.__file__)]
+TEST_PATH = [TEST_DIR, os.path.dirname(tempfile.__file__)]
 
 # Each test description is a list of 5 items:
 #

From noreply at buildbot.pypy.org  Sat Sep 13 03:14:53 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Sat, 13 Sep 2014 03:14:53 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: verify exception raised
	here
Message-ID: <20140913011453.BBA761D386F@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73517:56bb9af6b44b
Date: 2014-09-12 16:45 -0700
http://bitbucket.org/pypy/pypy/changeset/56bb9af6b44b/

Log:	verify exception raised here

diff --git a/rpython/rtyper/test/test_exception.py b/rpython/rtyper/test/test_exception.py
--- a/rpython/rtyper/test/test_exception.py
+++ b/rpython/rtyper/test/test_exception.py
@@ -76,12 +76,16 @@
                 assert e.errno == 0
                 assert e.strerror == "test"
                 assert e.filename is None
+            else:
+                assert False
             try:
                 k(n)
             except EnvironmentError as e:
                 assert e.errno == 0
                 assert e.strerror is None
                 assert e.filename is None
+            else:
+                assert False
         self.interpret(f, [42])
 
     def test_catch_incompatible_class(self):

From noreply at buildbot.pypy.org  Sat Sep 13 03:14:55 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Sat, 13 Sep 2014 03:14:55 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: fobj.seek is emulated on
 win32, doesn't catch invalid mode
Message-ID: <20140913011455.0045C1D386F@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73518:e0367fae2428
Date: 2014-09-12 16:49 -0700
http://bitbucket.org/pypy/pypy/changeset/e0367fae2428/

Log:	fobj.seek is emulated on win32, doesn't catch invalid mode

diff --git a/rpython/rlib/test/test_rfile.py b/rpython/rlib/test/test_rfile.py
--- a/rpython/rlib/test/test_rfile.py
+++ b/rpython/rlib/test/test_rfile.py
@@ -262,6 +262,7 @@
         f()
         self.interpret(f, [])
 
+    @py.test.mark.xfail("os.name == 'nt'")
     def test_seek(self):
         fname = str(self.tmpdir.join('file_4'))
 

From noreply at buildbot.pypy.org  Sat Sep 13 03:14:56 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Sat, 13 Sep 2014 03:14:56 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: these tests crash
	interpreter in win32 vm
Message-ID: <20140913011456.4166C1D386F@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73519:28cc8dda39ba
Date: 2014-09-12 16:57 -0700
http://bitbucket.org/pypy/pypy/changeset/28cc8dda39ba/

Log:	these tests crash interpreter in win32 vm

diff --git a/pypy/module/_file/test/test_file.py b/pypy/module/_file/test/test_file.py
--- a/pypy/module/_file/test/test_file.py
+++ b/pypy/module/_file/test/test_file.py
@@ -236,12 +236,14 @@
             data = f.read()
             assert data == "15"
 
+    @py.test.mark.skipif("sys.platform == 'win32'")
     def test_exception_from_close(self):
         import os
         f = self.file(self.temppath, 'w')
         os.close(f.fileno())
         raises(IOError, f.close)    # bad file descriptor
 
+    @py.test.mark.skipif("sys.platform == 'win32'")
     def test_exception_from_del(self):
         import os, gc, sys, cStringIO
         f = self.file(self.temppath, 'w')
diff --git a/rpython/rlib/test/test_rfile.py b/rpython/rlib/test/test_rfile.py
--- a/rpython/rlib/test/test_rfile.py
+++ b/rpython/rlib/test/test_rfile.py
@@ -293,7 +293,7 @@
         f()
         self.interpret(f, [])
 
-    def test_fdopen(self):
+    def test_fdopen_normal(self):
         fname = str(self.tmpdir.join('file_4a'))
 
         def f():
@@ -310,6 +310,17 @@
             f2.write("xxx")
             f2.close()
 
+        f()
+        assert open(fname).read() == "xxx"
+        self.interpret(f, [])
+        assert open(fname).read() == "xxx"
+
+    @py.test.mark.skipif("sys.platform == 'win32'")
+    def test_fdopen_errors(self):
+        fname = str(self.tmpdir.join('file'))
+
+        def f():
+            open(fname, 'w').close()
             fd = os.open(fname, os.O_RDONLY, 0777)
             f = os.fdopen(fd, 'r')
             os.close(fd)
@@ -336,9 +347,7 @@
                 assert False
 
         f()
-        assert open(fname).read() == "xxx"
         self.interpret(f, [])
-        assert open(fname).read() == "xxx"
 
     def test_fileno(self):
         fname = str(self.tmpdir.join('file_5'))

From noreply at buildbot.pypy.org  Sat Sep 13 03:23:14 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Sat, 13 Sep 2014 03:23:14 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: verify size of OFF_T
Message-ID: <20140913012314.686A71D38B8@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73520:fc130e159c9d
Date: 2014-09-12 21:23 -0400
http://bitbucket.org/pypy/pypy/changeset/fc130e159c9d/

Log:	verify size of OFF_T

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -117,6 +117,8 @@
 if os.name == 'nt':
     c_fseek = llexternal('_fseeki64', [FILEP, rffi.LONGLONG, rffi.INT], rffi.INT)
     c_ftell = llexternal('_ftelli64', [FILEP], rffi.LONGLONG)
+else:
+    assert rffi.sizeof(OFF_T) == 8
 
 c_fileno = llexternal(fileno, [FILEP], rffi.INT, releasegil=False)
 c_feof = llexternal('feof', [FILEP], rffi.INT, releasegil=False)
@@ -605,6 +607,7 @@
         if res != 0:
             c_clearerr(ll_file)
             raise _from_errno(IOError)
+        # XXX use fseek/SetEndOfFile on windows
         res = c_ftruncate(c_fileno(ll_file), arg)
         if res != 0:
             c_clearerr(ll_file)

From noreply at buildbot.pypy.org  Sat Sep 13 04:02:28 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Sat, 13 Sep 2014 04:02:28 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: set macro=True for
	fseeko/ftello
Message-ID: <20140913020228.5193E1D23D3@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73521:0fa6a641a60a
Date: 2014-09-13 02:02 +0000
http://bitbucket.org/pypy/pypy/changeset/0fa6a641a60a/

Log:	set macro=True for fseeko/ftello

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -112,8 +112,8 @@
 
 c_ftruncate = llexternal(ftruncate, [rffi.INT, OFF_T], rffi.INT, macro=True)
 
-c_fseek = llexternal('fseeko', [FILEP, OFF_T, rffi.INT], rffi.INT)
-c_ftell = llexternal('ftello', [FILEP], OFF_T)
+c_fseek = llexternal('fseeko', [FILEP, OFF_T, rffi.INT], rffi.INT, macro=True)
+c_ftell = llexternal('ftello', [FILEP], OFF_T, macro=True)
 if os.name == 'nt':
     c_fseek = llexternal('_fseeki64', [FILEP, rffi.LONGLONG, rffi.INT], rffi.INT)
     c_ftell = llexternal('_ftelli64', [FILEP], rffi.LONGLONG)

From noreply at buildbot.pypy.org  Sat Sep 13 17:18:20 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Sat, 13 Sep 2014 17:18:20 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: avoid macro=True
Message-ID: <20140913151820.A5ABF1C3233@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73522:5853c33ae109
Date: 2014-09-13 15:17 +0000
http://bitbucket.org/pypy/pypy/changeset/5853c33ae109/

Log:	avoid macro=True

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -3,7 +3,7 @@
 python builtin open()
 """
 
-import os, stat, errno
+import os, sys, stat, errno
 from rpython.rlib import rposix
 from rpython.rlib.objectmodel import enforceargs
 from rpython.rlib.rarithmetic import intmask
@@ -112,13 +112,17 @@
 
 c_ftruncate = llexternal(ftruncate, [rffi.INT, OFF_T], rffi.INT, macro=True)
 
-c_fseek = llexternal('fseeko', [FILEP, OFF_T, rffi.INT], rffi.INT, macro=True)
-c_ftell = llexternal('ftello', [FILEP], OFF_T, macro=True)
-if os.name == 'nt':
+if os.name != 'nt':
+    assert rffi.sizeof(OFF_T) == 8
+    if sys.platform.startswith('linux'):
+        c_fseek = llexternal('fseeko64', [FILEP, OFF_T, rffi.INT], rffi.INT)
+        c_ftell = llexternal('ftello64', [FILEP], OFF_T)
+    else:
+        c_fseek = llexternal('fseeko', [FILEP, OFF_T, rffi.INT], rffi.INT)
+        c_ftell = llexternal('ftello', [FILEP], OFF_T)
+else:
     c_fseek = llexternal('_fseeki64', [FILEP, rffi.LONGLONG, rffi.INT], rffi.INT)
     c_ftell = llexternal('_ftelli64', [FILEP], rffi.LONGLONG)
-else:
-    assert rffi.sizeof(OFF_T) == 8
 
 c_fileno = llexternal(fileno, [FILEP], rffi.INT, releasegil=False)
 c_feof = llexternal('feof', [FILEP], rffi.INT, releasegil=False)

From noreply at buildbot.pypy.org  Sat Sep 13 17:19:24 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Sat, 13 Sep 2014 17:19:24 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: add comment
Message-ID: <20140913151924.8FFC01C3233@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73523:05306b1932a1
Date: 2014-09-13 11:08 -0400
http://bitbucket.org/pypy/pypy/changeset/05306b1932a1/

Log:	add comment

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -508,6 +508,7 @@
         c = 0
         s = StringBuilder()
         while True:
+            # XXX release gil for all of this rather than each getc
             c_flockfile(ll_file)
             if self._univ_newline:
                 while size < 0 or s.getlength() < size:

From noreply at buildbot.pypy.org  Sat Sep 13 17:28:37 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Sat, 13 Sep 2014 17:28:37 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: same for ftruncate
Message-ID: <20140913152837.CD4CC1C3334@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73524:4a1f0a83d60b
Date: 2014-09-13 11:28 -0400
http://bitbucket.org/pypy/pypy/changeset/4a1f0a83d60b/

Log:	same for ftruncate

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -16,10 +16,8 @@
 includes = ['stdio.h', 'sys/types.h']
 if os.name == "posix":
     includes += ['unistd.h']
-    ftruncate = 'ftruncate'
     fileno = 'fileno'
 else:
-    ftruncate = '_chsize'
     fileno = '_fileno'
 
 stdio_streams = ['stdin', 'stdout', 'stderr']
@@ -110,17 +108,18 @@
 c_fwrite = llexternal('fwrite', [rffi.CCHARP, rffi.SIZE_T, rffi.SIZE_T, FILEP],
                       rffi.SIZE_T)
 
-c_ftruncate = llexternal(ftruncate, [rffi.INT, OFF_T], rffi.INT, macro=True)
-
 if os.name != 'nt':
     assert rffi.sizeof(OFF_T) == 8
     if sys.platform.startswith('linux'):
+        c_ftruncate = llexternal('ftruncate64', [rffi.INT, OFF_T], rffi.INT)
         c_fseek = llexternal('fseeko64', [FILEP, OFF_T, rffi.INT], rffi.INT)
         c_ftell = llexternal('ftello64', [FILEP], OFF_T)
     else:
+        c_ftruncate = llexternal('ftruncate', [rffi.INT, OFF_T], rffi.INT)
         c_fseek = llexternal('fseeko', [FILEP, OFF_T, rffi.INT], rffi.INT)
         c_ftell = llexternal('ftello', [FILEP], OFF_T)
 else:
+    c_ftruncate = llexternal('_chsize', [rffi.INT, OFF_T], rffi.INT)
     c_fseek = llexternal('_fseeki64', [FILEP, rffi.LONGLONG, rffi.INT], rffi.INT)
     c_ftell = llexternal('_ftelli64', [FILEP], rffi.LONGLONG)
 

From noreply at buildbot.pypy.org  Sat Sep 13 18:05:41 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 13 Sep 2014 18:05:41 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Fill in the proposal
Message-ID: <20140913160541.89D761C35E3@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: extradoc
Changeset: r5397:b3742ec418b5
Date: 2014-09-13 17:58 +0200
http://bitbucket.org/pypy/extradoc/changeset/b3742ec418b5/

Log:	Fill in the proposal

diff --git a/talk/pycon2015/stm/submitted.txt b/talk/pycon2015/stm/submitted.txt
--- a/talk/pycon2015/stm/submitted.txt
+++ b/talk/pycon2015/stm/submitted.txt
@@ -2,42 +2,124 @@
 ===========================
 
 Title
+-----
+
+PyPy-STM: Using multiple cores in your existing Python programs
+
 
 Category
+--------
+
+???
+
 
 Duration
+--------
+
+30 minutes
+
 
 Description
-...If your talk is accepted this will be made public and printed in the program. Should be one paragraph, maximum 400 characters.
+-----------
+
+PyPy is a fast alternative Python implementation.  Software
+Transactional Memory is a current academic research topic.  Put the two
+together --brew for a few years-- and we get a version of PyPy
+that runs on multiple cores, without the infamous Global Interpreter
+Lock (GIL).  It has been released in 2013 in beta, including
+integration with the Just-in-Time compiler.
+
 
 Audience
-...Who is the intended audience for your talk? (Be specific; "Python programmers" is not a good answer to this question.)
+--------
+
+Regular Python programmers looking for simpler ways to use multiple
+cores; or people looking for the theory behind PyPy-STM.
+
 
 Python level
-...Level of audience expertise assumed in Python.
+------------
+
+Medium
+
 
 Objectives
-...What will attendees get out of your talk? When they leave the room, what will they know that they didn't know before?
-... Who is the intended audience for your talk? (Be specific; "Python programmers" is not a good answer to this question.)
-... What will attendees get out of your talk? When they leave the room, what will they know that they didn't know before?
+----------
+
+Attendees will get a quick overview about how PyPy-STM manages to
+remove the GIL.  They will also learn about how this approach actually
+enables a new way to use multiple cores in their own applications, and
+how it differs from other solutions (which are generally about running
+independent processes).
+
 
 Detailed Abstract
-...Detailed description. Will be made public if your talk is accepted.
-... Include links to source code, articles, blog posts, or other writing that adds context to the presentation.
+-----------------
+
+PyPy-STM is, in one sentence, a special version of PyPy that runs on
+multiple cores without the infamous Global Interpreter Lock (GIL).
+Unlike CPython or the regular PyPy, your multithreaded CPU-intensive
+programs will run using multiple cores.  Assuming that you have any
+such program.
+
+Look closer, though, and see that PyPy-STM is not actually *removing*
+the GIL.  It is instead using STM --Software Transactional Memory-- to
+try to run several threads in parallel even though they all acquire
+the same lock.  This lock can be the GIL, or it can be a custom lock.
+So PyPy-STM gives naturally a new way to use multiple cores, where
+race conditions and other pitfalls of multithreaded programming are
+almost absent: run multiple threads, but run them all using
+coarse-grained locking, with only one lock.  PyPy-STM actually
+executes such threads in parallel, as long as they modify independent
+data.  In case of an actual "conflict" between two threads, one of
+them is allowed to proceed; the other "aborts" and restart its work
+from a bit earlier.  (This is conceptually similar to transactions in
+databases, but more transparent.)
+
+The typical application that can benefit is doing some computation on
+all objects in some list, but where many of the computations turn out
+to be independent.  A lot of applications follow this pattern: for
+example, Twisted-based web servers, handling concurrent connections,
+can often answer different requests in an independent way.  This can
+be written by extending the core of Twisted to add a pool of threads.
+The actual Twisted programs remain unchanged: they need not be aware
+of the multiple threads, because they are all protected by the same
+lock.  Logically, they are serialized; in practice, with PyPy-STM,
+they often run in parallel.
+
+How does PyPy-STM actually work?  The 10000-feet view is to give to
+each thread its own copy of the memory, where it sees and updates its
+own version, or "view", of the objects.  Changes that are "committed"
+by a transaction are eventually propagated into the other threads'
+views.  The main trick on top of that is the OS's support for viewing
+the same physical page of memory at multiple address.  This allows the
+views to share most of their pages with each other.
+
+As of July 2013, the performance is that PyPy-STM is around 40% slower
+than a regular PyPy on one thread, but scales nicely.  Running an
+application on two threads already is more than enough to regain that
+loss.  So far, our target is around 4 threads; right now, performance
+degrades if you try to run more threads, but we are working on
+improving this as well.
+
+For more details and the current status, follow STM on the PyPy blog:
+http://morepypy.blogspot.com/search/label/stm
+
 
 Outline
-... Your outline should be an enumeration of what you intend to say, along with time estimates.
-... It is not necessary to have completely written or planned your talk already, but you should have a basic idea of what the over-arching points you intend to make are, and roughly how long you will spend on each one.
+-------
+
+1. Intro (5 min): PyPy, STM
+
+2. User Programming model (15 min): using multithreads with coarse locking;
+   examples of applications; how to structure them best for PyPy-STM
+   
+3. How things work under the cover (5 min): overview.
+
+4. Questions (5 min).
+
 
 Additional notes
-...Anything else you'd like the program committee to know when making their selection: your past speaking experience, open source community experience, etc.
-... If you've given a talk, tutorial, or other presentation before, especially at an earlier PyCon or another conference, include that information as well as a link to slides or a video if they're available.
+----------------
 
-Additional requirements
-...Please let us know if you have any specific needs (A/V requirements, multiple microphones, a table, etc). Note for example that 'audio out' is not provided for your computer unless you tell us in advance.
-
-Recording release
-...By submitting your talk proposal, you agree to give permission to the Python Software Foundation to record, edit, and release audio and/or video of your presentation. If you do not agree to this, please uncheck this box. See PyCon 2015 Recording Release for details.
-
------------------
-You will be able to edit your proposal after it has been submitted. The program committee may ask questions, provide feedback, and even suggest changes to your proposal as part of the review processes.
+Long-time PyPy speaker at numerous conferences (PyCon, EuroPython, etc.)

From noreply at buildbot.pypy.org  Sat Sep 13 18:05:42 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 13 Sep 2014 18:05:42 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: merge heads
Message-ID: <20140913160542.BFB511C35E3@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: extradoc
Changeset: r5398:347ebc4365d2
Date: 2014-09-13 18:05 +0200
http://bitbucket.org/pypy/extradoc/changeset/347ebc4365d2/

Log:	merge heads

diff --git a/talk/pycon2015/status/abstract.rst b/talk/pycon2015/status/abstract.rst
--- a/talk/pycon2015/status/abstract.rst
+++ b/talk/pycon2015/status/abstract.rst
@@ -1,28 +1,32 @@
-Status of PyPy and its ecosystem
-================================
+PyPy - the last 2 years of progress
+===================================
 
-Description
------------
+Description (one paragraph, 400 chars max)
+------------------------------------------
 
-PyPy is a fast alternative implementation of Python. This talk will describe
-what happened in the PyPy ecosystem in the last 2 years and what the future
-holds. Topics such as JIT improvements, garbage collection, CFFI, Numpy, STM
-and Python 3 will be covered.
+This talk describes what happened in the PyPy ecosystem in the last 2 years, a
+timeframe in which PyPy has been successfully deployed multiple times while
+yielding interesting performance improvements and a timeframe in which more
+libraries started being compatible with PyPy through the use of cffi.
 
-Abstract
---------
+Detailed Abstract
+-----------------
 
-This talk will cover what has happened in the PyPy world during the last 2
-years :
+PyPy has been in the works for more than ten years and has reached relative
+maturity with more and more libraries working under PyPy and more deployments
+happening. Right now it entertains between 0.5-1.0% of PyPI package downloads
+(with CPython taking virtually all of the rest), used mostly for
+high-performance web servers.
 
-- Progress in Python 3 support and Numpy support
+Since no PyPy talk happened at PyCon 2014, we would like to present what
+we have achieved during the two years between talks. We would like to cover
+advancements in the PyPy performance landscape, but more importantly how
+we're addresssing the community needs and building the ecosystem. These days
+a lot of libraries that used to bind to C using the CPython C API are either
+using cffi or have alternatives using cffi.
 
-- Performance improvements
+We would also like to walk through a few success stories that we have
+experienced. Unfortunately the biggest chunk of PyPy clients are very
+secretive (e.g. trading companies), but we can still present a few case studies.
 
-- An attempt to remove the GIL using Software Transactional Memory
-
-- CFFI : The best way to interface with C code on PyPy
-
-- Status of the crowdfunding
-
-- ARM
+XXXX
diff --git a/talk/uct2014/example/demo1.py b/talk/uct2014/example/demo1.py
--- a/talk/uct2014/example/demo1.py
+++ b/talk/uct2014/example/demo1.py
@@ -12,7 +12,7 @@
     except ValueError:
         print "Requires ints"
         return 1
-    print "Added", arg0 + arg1
+    print "Got", adder(arg0, arg1)
     return 0
 
 def target(*args):
diff --git a/talk/uct2014/example/interp.py b/talk/uct2014/example/interp.py
--- a/talk/uct2014/example/interp.py
+++ b/talk/uct2014/example/interp.py
@@ -46,3 +46,7 @@
 
 def target(*args):
     return entry_point
+
+if __name__ == '__main__':
+    import sys
+    entry_point(sys.argv)

From noreply at buildbot.pypy.org  Sat Sep 13 18:15:14 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 13 Sep 2014 18:15:14 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: tweaks
Message-ID: <20140913161514.7A9E41C3233@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: extradoc
Changeset: r5399:a008dc94c13f
Date: 2014-09-13 18:15 +0200
http://bitbucket.org/pypy/extradoc/changeset/a008dc94c13f/

Log:	tweaks

diff --git a/talk/pycon2015/stm/submitted.txt b/talk/pycon2015/stm/submitted.txt
--- a/talk/pycon2015/stm/submitted.txt
+++ b/talk/pycon2015/stm/submitted.txt
@@ -47,7 +47,7 @@
 ----------
 
 Attendees will get a quick overview about how PyPy-STM manages to
-remove the GIL.  They will also learn about how this approach actually
+remove the GIL.  They will also learn about how this approach
 enables a new way to use multiple cores in their own applications, and
 how it differs from other solutions (which are generally about running
 independent processes).
@@ -76,8 +76,8 @@
 from a bit earlier.  (This is conceptually similar to transactions in
 databases, but more transparent.)
 
-The typical application that can benefit is doing some computation on
-all objects in some list, but where many of the computations turn out
+The typical application that can benefit is doing some computations
+based on a number of inputs, where many of the computations turn out
 to be independent.  A lot of applications follow this pattern: for
 example, Twisted-based web servers, handling concurrent connections,
 can often answer different requests in an independent way.  This can

From noreply at buildbot.pypy.org  Sat Sep 13 23:45:25 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Sat, 13 Sep 2014 23:45:25 +0200 (CEST)
Subject: [pypy-commit] pypy default: only specify Fd for non-makefile build
	(windows)
Message-ID: <20140913214525.C33BF1C3233@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: 
Changeset: r73525:1adc566ec0bf
Date: 2014-09-13 23:44 +0200
http://bitbucket.org/pypy/pypy/changeset/1adc566ec0bf/

Log:	only specify Fd for non-makefile build (windows)

diff --git a/rpython/translator/platform/test/test_makefile.py b/rpython/translator/platform/test/test_makefile.py
--- a/rpython/translator/platform/test/test_makefile.py
+++ b/rpython/translator/platform/test/test_makefile.py
@@ -44,6 +44,7 @@
         assert res.returncode == 0        
     
     def test_900_files(self):
+        tmpdir = udir.join('test_900_files').ensure(dir=1)
         txt = '#include \n'
         for i in range(900):
             txt += 'int func%03d();\n' % i
@@ -52,11 +53,11 @@
             txt += '    j += func%03d();\n' % i
         txt += '    printf("%d\\n", j);\n'
         txt += '    return 0;};\n'
-        cfile = udir.join('test_900_files.c')
+        cfile = tmpdir.join('test_900_files.c')
         cfile.write(txt)
         cfiles = [cfile]
         for i in range(900):
-            cfile2 = udir.join('implement%03d.c' %i)
+            cfile2 = tmpdir.join('implement%03d.c' %i)
             cfile2.write('''
                 int func%03d()
             {
@@ -64,10 +65,10 @@
             }
             ''' % (i, i))
             cfiles.append(cfile2)
-        mk = self.platform.gen_makefile(cfiles, ExternalCompilationInfo(), path=udir)
+        mk = self.platform.gen_makefile(cfiles, ExternalCompilationInfo(), path=tmpdir)
         mk.write()
         self.platform.execute_makefile(mk)
-        res = self.platform.execute(udir.join('test_900_files'))
+        res = self.platform.execute(tmpdir.join('test_900_files'))
         self.check_res(res, '%d\n' %sum(range(900)))
 
     def test_precompiled_headers(self):
diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py
--- a/rpython/translator/platform/windows.py
+++ b/rpython/translator/platform/windows.py
@@ -204,8 +204,9 @@
         # must come first, and after the file name all options are ignored.
         # So please be careful with the order of parameters! ;-)
         pdb_dir = oname.dirname
-        args = ['/nologo', '/c'] + compile_args + ['/Fd%s\\' % (pdb_dir,),
-                        '/Fo%s' % (oname,), str(cfile)]
+	if pdb_dir:
+	        compile_args += ['/Fd%s\\' % (pdb_dir,)]
+	args = ['/nologo', '/c'] + compile_args + ['/Fo%s' % (oname,), str(cfile)]
         self._execute_c_compiler(cc, args, oname)
         return oname
 
@@ -347,7 +348,7 @@
                '$(CREATE_PCH) $(INCLUDEDIRS)'))
             rules.append(('.c.obj', '',
                     '$(CC) /nologo $(CFLAGS) $(CFLAGSEXTRA) $(USE_PCH) '
-                    '/Fd$(@D)\\ /Fo$@ /c $< $(INCLUDEDIRS)'))
+                    '/Fo$@ /c $< $(INCLUDEDIRS)'))
             #Do not use precompiled headers for some files
             #rules.append((r'{..\module_cache}.c{..\module_cache}.obj', '',
             #        '$(CC) /nologo $(CFLAGS) $(CFLAGSEXTRA) /Fo$@ /c $< $(INCLUDEDIRS)'))
@@ -362,13 +363,12 @@
                     target = f[:-1] + 'obj'
                     rules.append((target, f,
                         '$(CC) /nologo $(CFLAGS) $(CFLAGSEXTRA) '
-                        '/Fd%s\\ /Fo%s /c %s $(INCLUDEDIRS)' %(
-                                os.path.dirname(target), target, f)))
+                        '/Fo%s /c %s $(INCLUDEDIRS)' %(target, f)))
 
         else:
             rules.append(('.c.obj', '',
                           '$(CC) /nologo $(CFLAGS) $(CFLAGSEXTRA) '
-                          '/Fd$(@D)\\ /Fo$@ /c $< $(INCLUDEDIRS)'))
+                          '/Fo$@ /c $< $(INCLUDEDIRS)'))
 
 
         for args in definitions:

From noreply at buildbot.pypy.org  Sun Sep 14 00:54:39 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Sun, 14 Sep 2014 00:54:39 +0200 (CEST)
Subject: [pypy-commit] buildbot default: don't explicitly specify the binary
 name of 'pypy' so that py3k branches can
Message-ID: <20140913225440.115151C3CC1@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey 
Branch: 
Changeset: r918:345d88ba452d
Date: 2014-09-13 15:53 -0700
http://bitbucket.org/pypy/buildbot/changeset/345d88ba452d/

Log:	don't explicitly specify the binary name of 'pypy' so that py3k
	branches can default to a value of 'pypy3'

diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py
--- a/bot2/pypybuildbot/builds.py
+++ b/bot2/pypybuildbot/builds.py
@@ -491,8 +491,7 @@
             description="compress pypy-c",
             haltOnFailure=False,
             command=prefix + ["python", "pypy/tool/release/package.py",
-                     ".", WithProperties(name), 'pypy',
-                     '.'],
+                              "--targetdir=.", ".", WithProperties(name)],
             workdir='build'))
         nightly = '~/nightly/'
         extension = get_extension(platform)
@@ -600,8 +599,7 @@
         self.addStep(ShellCmd(
             description="compress pypy-c",
             command=prefix + ["python", "pypy/tool/release/package.py",
-                     ".", WithProperties(name), 'pypy',
-                     '.'],
+                              "--targetdir=.", ".", WithProperties(name)],
             haltOnFailure=True,
             workdir='build'))
         nightly = '~/nightly/'

From noreply at buildbot.pypy.org  Sun Sep 14 00:59:01 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Sun, 14 Sep 2014 00:59:01 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: issue1796: default the packaged pypy
 binary/shared lib name to 'pypy3'
Message-ID: <20140913225902.046281C3CC1@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey 
Branch: py3k
Changeset: r73526:86ac335daec2
Date: 2014-09-13 15:57 -0700
http://bitbucket.org/pypy/pypy/changeset/86ac335daec2/

Log:	issue1796: default the packaged pypy binary/shared lib name to
	'pypy3'

diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -306,10 +306,10 @@
         import imp
         argparse = imp.load_source('argparse', 'lib-python/2.7/argparse.py')
     if sys.platform == 'win32':
-        pypy_exe = 'pypy.exe'
+        pypy_exe = 'pypy3.exe'
         license_base = os.path.join(basedir, r'..\..\..\local') # as on buildbot YMMV
     else:
-        pypy_exe = 'pypy'
+        pypy_exe = 'pypy3'
         license_base = '/usr/share/doc'
     parser = argparse.ArgumentParser()
     args = list(args)

From noreply at buildbot.pypy.org  Sun Sep 14 01:31:13 2014
From: noreply at buildbot.pypy.org (mjacob)
Date: Sun, 14 Sep 2014 01:31:13 +0200 (CEST)
Subject: [pypy-commit] pypy improve-docs: Revert content and location of
 release notes to how they are in default.
Message-ID: <20140913233113.EF8C81D27FC@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: improve-docs
Changeset: r73527:8abae94daa2f
Date: 2014-09-13 19:31 +0200
http://bitbucket.org/pypy/pypy/changeset/8abae94daa2f/

Log:	Revert content and location of release notes to how they are in
	default.

diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/index-of-release-notes.rst
@@ -0,0 +1,44 @@
+Historical release notes
+========================
+
+Cpython 2.7 compatible versions
+===============================
+
+.. toctree::
+
+   release-2.4.0.rst
+   release-2.3.1.rst
+   release-2.3.0.rst
+   release-2.2.1.rst
+   release-2.2.0.rst
+   release-2.1.0.rst
+   release-2.1.0-beta2.rst
+   release-2.1.0-beta1.rst
+   release-2.1.0.rst
+   release-2.0.2.rst
+   release-2.0.1.rst
+   release-2.0.0.rst
+   release-2.0.0-beta2.rst
+   release-2.0.0-beta1.rst
+   release-1.9.0.rst
+   release-1.8.0.rst
+   release-1.7.0.rst
+   release-1.6.0.rst
+   release-1.5.0.rst
+   release-1.4.1.rst
+   release-1.4.0beta.rst
+   release-1.4.0.rst
+   release-1.3.0.rst
+   release-1.2.0.rst
+   release-1.1.0.rst
+   release-1.0.0.rst
+   release-0.99.0.rst
+   release-0.9.0.rst
+   release-0.8.0.rst
+   release-0.7.0.rst
+   release-0.6
+
+Cpython 3.2 compatible versions
+===============================
+.. toctree::
+   release-pypy3-2.1.0-beta1.rst
diff --git a/pypy/doc/releases/0.6.rst b/pypy/doc/release-0.6.rst
rename from pypy/doc/releases/0.6.rst
rename to pypy/doc/release-0.6.rst
--- a/pypy/doc/releases/0.6.rst
+++ b/pypy/doc/release-0.6.rst
@@ -1,12 +1,12 @@
 The PyPy 0.6 release
-====================
+-------------------- 
 
-*The PyPy Development Team is happy to announce the first
+*The PyPy Development Team is happy to announce the first 
 public release of PyPy after two years of spare-time and
-half a year of EU funded development.  The 0.6 release
-is eminently a preview release.*
+half a year of EU funded development.  The 0.6 release 
+is eminently a preview release.*  
 
-What it is and where to start
+What it is and where to start 
 -----------------------------
 
 Getting started:    getting-started.html
@@ -29,7 +29,7 @@
 that functionality is still made available by PyPy piggy-backing on
 the host CPython interpreter.  Double interpretation and abstractions
 in the code-base make it so that PyPy running on CPython is quite slow
-(around 2000x slower than CPython), this is expected.
+(around 2000x slower than CPython ), this is expected.  
 
 This release is intended for people that want to look and get a feel
 into what we are doing, playing with interpreter and perusing the
@@ -38,8 +38,8 @@
 Interesting bits and highlights
 ---------------------------------
 
-The release is also a snap-shot of our ongoing efforts towards
-low-level translation and experimenting with unique features.
+The release is also a snap-shot of our ongoing efforts towards 
+low-level translation and experimenting with unique features. 
 
 * By default, PyPy is a Python version that works completely with
   new-style-classes semantics.  However, support for old-style classes
@@ -47,9 +47,9 @@
   their metaclass and instance object are included and can be re-made
   the default with the ``--oldstyle`` option.
 
-* In PyPy, bytecode interpretation and object manipulations
-  are well separated between a bytecode interpreter and an
-  *object space* which implements operations on objects.
+* In PyPy, bytecode interpretation and object manipulations 
+  are well separated between a bytecode interpreter and an 
+  *object space* which implements operations on objects. 
   PyPy comes with experimental object spaces augmenting the
   standard one through delegation:
 
@@ -58,17 +58,17 @@
 
   * the 'thunk' object space that implements lazy values and a 'become'
     operation that can exchange object identities.
-
+  
   These spaces already give a glimpse in the flexibility potential of
   PyPy.  See demo/fibonacci.py and demo/sharedref.py for examples
   about the 'thunk' object space.
 
-* The 0.6 release also contains a snapshot of our translation-efforts
+* The 0.6 release also contains a snapshot of our translation-efforts 
   to lower level languages.  For that we have developed an
   annotator which is capable of inferring type information
   across our code base.  The annotator right now is already
   capable of successfully type annotating basically *all* of
-  PyPy code-base, and is included with 0.6.
+  PyPy code-base, and is included with 0.6.  
 
 * From type annotated code, low-level code needs to be generated.
   Backends for various targets (C, LLVM,...) are included; they are
@@ -80,35 +80,35 @@
 ---------------------------------
 
 Generating low-level code is the main area we are hammering on in the
-next months; our plan is to produce a PyPy version in August/September
-that does not need to be interpreted by CPython anymore and will
-thus run considerably faster than the 0.6 preview release.
+next months; our plan is to produce a PyPy version in August/September 
+that does not need to be interpreted by CPython anymore and will 
+thus run considerably faster than the 0.6 preview release. 
 
 PyPy has been a community effort from the start and it would
 not have got that far without the coding and feedback support
-from numerous people.   Please feel free to give feedback and
-raise questions.
+from numerous people.   Please feel free to give feedback and 
+raise questions. 
 
     contact points: http://pypy.org/contact.html
 
     contributor list: contributor.html
 
-have fun,
+have fun, 
 
-    Armin Rigo, Samuele Pedroni,
+    Armin Rigo, Samuele Pedroni, 
 
-    Holger Krekel, Christian Tismer,
+    Holger Krekel, Christian Tismer, 
 
-    Carl Friedrich Bolz
+    Carl Friedrich Bolz 
 
 
-    PyPy development and activities happen as an open source project
-    and with the support of a consortium funded by a two year EU IST
-    research grant. Here is a list of partners of the EU project:
-
+    PyPy development and activities happen as an open source project  
+    and with the support of a consortium funded by a two year EU IST 
+    research grant. Here is a list of partners of the EU project: 
+        
         Heinrich-Heine University (Germany), AB Strakt (Sweden)
 
-        merlinux GmbH (Germany), tismerysoft GmbH(Germany)
+        merlinux GmbH (Germany), tismerysoft GmbH(Germany) 
 
         Logilab Paris (France), DFKI GmbH (Germany)
 
diff --git a/pypy/doc/releases/0.7.0.rst b/pypy/doc/release-0.7.0.rst
rename from pypy/doc/releases/0.7.0.rst
rename to pypy/doc/release-0.7.0.rst
--- a/pypy/doc/releases/0.7.0.rst
+++ b/pypy/doc/release-0.7.0.rst
@@ -1,28 +1,28 @@
 pypy-0.7.0: first PyPy-generated Python Implementations
-=======================================================
+==============================================================
 
-What was once just an idea between a few people discussing
-on some nested mailing list thread and in a pub became reality ...
+What was once just an idea between a few people discussing 
+on some nested mailing list thread and in a pub became reality ... 
 the PyPy development team is happy to announce its first
 public release of a fully translatable self contained Python
 implementation.  The 0.7 release showcases the results of our
 efforts in the last few months since the 0.6 preview release
 which have been partially funded by the European Union:
 
-- whole program type inference on our Python Interpreter
-  implementation with full translation to two different
-  machine-level targets: C and LLVM
+- whole program type inference on our Python Interpreter 
+  implementation with full translation to two different 
+  machine-level targets: C and LLVM 
 
-- a translation choice of using a refcounting or Boehm
+- a translation choice of using a refcounting or Boehm 
   garbage collectors
 
-- the ability to translate with or without thread support
+- the ability to translate with or without thread support 
 
-- very complete language-level compliance with CPython 2.4.1
+- very complete language-level compliance with CPython 2.4.1 
 
 
-What is PyPy (about)?
----------------------
+What is PyPy (about)? 
+------------------------------------------------
 
 PyPy is a MIT-licensed research-oriented reimplementation of
 Python written in Python itself, flexible and easy to
@@ -39,23 +39,23 @@
 Note that PyPy is mainly a research and development project
 and does not by itself focus on getting a production-ready
 Python implementation although we do hope and expect it to
-become a viable contender in that area sometime next year.
+become a viable contender in that area sometime next year. 
 
 
-Where to start?
----------------
+Where to start? 
+-----------------------------
 
 Getting started:    http://codespeak.net/pypy/dist/pypy/doc/getting-started.html
 
-PyPy Documentation: http://codespeak.net/pypy/dist/pypy/doc/
+PyPy Documentation: http://codespeak.net/pypy/dist/pypy/doc/ 
 
 PyPy Homepage:      http://codespeak.net/pypy/
 
 The interpreter and object model implementations shipped with
 the 0.7 version can run on their own and implement the core
 language features of Python as of CPython 2.4.  However, we still
-do not recommend using PyPy for anything else than for education,
-playing or research purposes.
+do not recommend using PyPy for anything else than for education, 
+playing or research purposes.  
 
 Ongoing work and near term goals
 ---------------------------------
@@ -63,42 +63,42 @@
 PyPy has been developed during approximately 15 coding sprints
 across Europe and the US.  It continues to be a very
 dynamically and incrementally evolving project with many
-one-week meetings to follow.  You are invited to consider coming to
-the next such meeting in Paris mid October 2005 where we intend to
+one-week meetings to follow.  You are invited to consider coming to 
+the next such meeting in Paris mid October 2005 where we intend to 
 plan and head for an even more intense phase of the project
 involving building a JIT-Compiler and enabling unique
 features not found in other Python language implementations.
 
 PyPy has been a community effort from the start and it would
 not have got that far without the coding and feedback support
-from numerous people.   Please feel free to give feedback and
-raise questions.
+from numerous people.   Please feel free to give feedback and 
+raise questions. 
 
     contact points: http://codespeak.net/pypy/dist/pypy/doc/contact.html
 
     contributor list: http://codespeak.net/pypy/dist/pypy/doc/contributor.html
 
-have fun,
+have fun, 
+    
+    the pypy team, of which here is a partial snapshot
+    of mainly involved persons: 
 
-    the pypy team, of which here is a partial snapshot
-    of mainly involved persons:
+    Armin Rigo, Samuele Pedroni, 
+    Holger Krekel, Christian Tismer, 
+    Carl Friedrich Bolz, Michael Hudson, 
+    Eric van Riet Paap, Richard Emslie, 
+    Anders Chrigstroem, Anders Lehmann, 
+    Ludovic Aubry, Adrien Di Mascio, 
+    Niklaus Haldimann, Jacob Hallen, 
+    Bea During, Laura Creighton, 
+    and many contributors ... 
 
-    Armin Rigo, Samuele Pedroni,
-    Holger Krekel, Christian Tismer,
-    Carl Friedrich Bolz, Michael Hudson,
-    Eric van Riet Paap, Richard Emslie,
-    Anders Chrigstroem, Anders Lehmann,
-    Ludovic Aubry, Adrien Di Mascio,
-    Niklaus Haldimann, Jacob Hallen,
-    Bea During, Laura Creighton,
-    and many contributors ...
-
-PyPy development and activities happen as an open source project
-and with the support of a consortium partially funded by a two
-year European Union IST research grant. Here is a list of
-the full partners of that consortium:
-
+PyPy development and activities happen as an open source project  
+and with the support of a consortium partially funded by a two 
+year European Union IST research grant. Here is a list of 
+the full partners of that consortium: 
+        
     Heinrich-Heine University (Germany), AB Strakt (Sweden)
-    merlinux GmbH (Germany), tismerysoft GmbH(Germany)
+    merlinux GmbH (Germany), tismerysoft GmbH(Germany) 
     Logilab Paris (France), DFKI GmbH (Germany)
     ChangeMaker (Sweden), Impara (Germany)
diff --git a/pypy/doc/releases/0.8.0.rst b/pypy/doc/release-0.8.0.rst
rename from pypy/doc/releases/0.8.0.rst
rename to pypy/doc/release-0.8.0.rst
--- a/pypy/doc/releases/0.8.0.rst
+++ b/pypy/doc/release-0.8.0.rst
@@ -1,24 +1,24 @@
-pypy-0.8.0: Translatable compiler/parser and some more speed
+pypy-0.8.0: Translatable compiler/parser and some more speed 
 ==============================================================
 
-The PyPy development team has been busy working and we've now packaged
-our latest improvements, completed work and new experiments as
+The PyPy development team has been busy working and we've now packaged 
+our latest improvements, completed work and new experiments as 
 version 0.8.0, our third public release.
 
 The highlights of this third release of PyPy are:
 
 - Translatable parser and AST compiler. PyPy now integrates its own
   compiler based on Python own 'compiler' package but with a number
-  of fixes and code simplifications in order to get it translated
-  with the rest of PyPy.  This makes using the translated pypy
-  interactively much more pleasant, as compilation is considerably
+  of fixes and code simplifications in order to get it translated 
+  with the rest of PyPy.  This makes using the translated pypy 
+  interactively much more pleasant, as compilation is considerably 
   faster than in 0.7.0.
 
 - Some Speed enhancements. Translated PyPy is now about 10 times
   faster than 0.7 but still 10-20 times slower than
-  CPython on pystones and other benchmarks.  At the same time,
+  CPython on pystones and other benchmarks.  At the same time, 
   language compliance has been slightly increased compared to 0.7
-  which had already reached major CPython compliance goals.
+  which had already reached major CPython compliance goals. 
 
 - Some experimental features are now translatable.  Since 0.6.0, PyPy
   shipped with an experimental Object Space (the part of PyPy
@@ -27,7 +27,7 @@
   object space can also be translated preserving its feature
   additions.
 
-What is PyPy (about)?
+What is PyPy (about)? 
 ------------------------------------------------
 
 PyPy is a MIT-licensed research-oriented reimplementation of
@@ -45,25 +45,25 @@
 Note that PyPy is mainly a research and development project
 and does not by itself focus on getting a production-ready
 Python implementation although we do hope and expect it to
-become a viable contender in that area sometime next year.
+become a viable contender in that area sometime next year. 
 
-PyPy is partially funded as a research project under the
-European Union's IST programme.
+PyPy is partially funded as a research project under the 
+European Union's IST programme. 
 
-Where to start?
+Where to start? 
 -----------------------------
 
 Getting started:    http://codespeak.net/pypy/dist/pypy/doc/getting-started.html
 
-PyPy Documentation: http://codespeak.net/pypy/dist/pypy/doc/
+PyPy Documentation: http://codespeak.net/pypy/dist/pypy/doc/ 
 
 PyPy Homepage:      http://codespeak.net/pypy/
 
 The interpreter and object model implementations shipped with
 the 0.8 version can run on their own and implement the core
 language features of Python as of CPython 2.4.  However, we still
-do not recommend using PyPy for anything else than for education,
-playing or research purposes.
+do not recommend using PyPy for anything else than for education, 
+playing or research purposes.  
 
 Ongoing work and near term goals
 ---------------------------------
@@ -74,11 +74,11 @@
 which we still expect will be the major source of our future speed
 improvements and some successful amount of work has been done on the
 support needed for stackless-like features.
-
+  
 This release also includes the snapshots in preliminary or embryonic
 form of the following interesting but yet not completed sub projects:
 
-- The OOtyper, a RTyper variation for higher-level backends
+- The OOtyper, a RTyper variation for higher-level backends 
   (Squeak, ...)
 - A JavaScript backend
 - A limited (PPC) assembler backend (this related to the JIT)
@@ -91,25 +91,25 @@
 
 PyPy has been a community effort from the start and it would
 not have got that far without the coding and feedback support
-from numerous people.   Please feel free to give feedback and
-raise questions.
+from numerous people.   Please feel free to give feedback and 
+raise questions. 
 
     contact points: http://codespeak.net/pypy/dist/pypy/doc/contact.html
 
 
-have fun,
-
-    the pypy team, (Armin Rigo, Samuele Pedroni,
-    Holger Krekel, Christian Tismer,
-    Carl Friedrich Bolz, Michael Hudson,
+have fun, 
+    
+    the pypy team, (Armin Rigo, Samuele Pedroni, 
+    Holger Krekel, Christian Tismer, 
+    Carl Friedrich Bolz, Michael Hudson, 
     and many others: http://codespeak.net/pypy/dist/pypy/doc/contributor.html)
 
-PyPy development and activities happen as an open source project
-and with the support of a consortium partially funded by a two
-year European Union IST research grant. The full partners of that
-consortium are:
-
+PyPy development and activities happen as an open source project  
+and with the support of a consortium partially funded by a two 
+year European Union IST research grant. The full partners of that 
+consortium are: 
+        
     Heinrich-Heine University (Germany), AB Strakt (Sweden)
-    merlinux GmbH (Germany), tismerysoft GmbH (Germany)
+    merlinux GmbH (Germany), tismerysoft GmbH (Germany) 
     Logilab Paris (France), DFKI GmbH (Germany)
     ChangeMaker (Sweden), Impara (Germany)
diff --git a/pypy/doc/releases/0.9.0.rst b/pypy/doc/release-0.9.0.rst
rename from pypy/doc/releases/0.9.0.rst
rename to pypy/doc/release-0.9.0.rst
--- a/pypy/doc/releases/0.9.0.rst
+++ b/pypy/doc/release-0.9.0.rst
@@ -1,8 +1,8 @@
 pypy-0.9.0: stackless, new extension compiler
 ==============================================================
 
-The PyPy development team has been busy working and we've now packaged
-our latest improvements, completed work and new experiments as
+The PyPy development team has been busy working and we've now packaged 
+our latest improvements, completed work and new experiments as 
 version 0.9.0, our fourth public release.
 
 The highlights of this fourth release of PyPy are:
@@ -28,7 +28,7 @@
     standard-in-Python-2.5 ctypes module.  See its documentation for more:
     http://codespeak.net/pypy/dist/pypy/doc/rctypes.html
 
-**framework GCs**
+**framework GCs** 
     PyPy's interpreter can now be compiled to use a garbage collector
     written in RPython.  This added control over PyPy's execution makes the
     implementation of new and interesting features possible, apart from
@@ -61,7 +61,7 @@
     We now run all our tests every night, and you can see the summary at:
     http://buildbot.pypy.org/summary
 
-What is PyPy (about)?
+What is PyPy (about)? 
 ------------------------------------------------
 
 PyPy is a MIT-licensed research-oriented reimplementation of Python
@@ -83,12 +83,12 @@
 PyPy is partially funded as a research project under the European
 Union's IST programme.
 
-Where to start?
+Where to start? 
 -----------------------------
 
 Getting started:    http://codespeak.net/pypy/dist/pypy/doc/getting-started.html
 
-PyPy Documentation: http://codespeak.net/pypy/dist/pypy/doc/
+PyPy Documentation: http://codespeak.net/pypy/dist/pypy/doc/ 
 
 PyPy Homepage:      http://codespeak.net/pypy/
 
@@ -115,24 +115,24 @@
 
 PyPy has been a community effort from the start and it would
 not have got that far without the coding and feedback support
-from numerous people.   Please feel free to give feedback and
-raise questions.
+from numerous people.   Please feel free to give feedback and 
+raise questions. 
 
     contact points: http://codespeak.net/pypy/dist/pypy/doc/contact.html
 
-have fun,
-
-    the pypy team, (Armin Rigo, Samuele Pedroni,
-    Holger Krekel, Christian Tismer,
-    Carl Friedrich Bolz, Michael Hudson,
+have fun, 
+    
+    the pypy team, (Armin Rigo, Samuele Pedroni, 
+    Holger Krekel, Christian Tismer, 
+    Carl Friedrich Bolz, Michael Hudson, 
     and many others: http://codespeak.net/pypy/dist/pypy/doc/contributor.html)
 
-PyPy development and activities happen as an open source project
-and with the support of a consortium partially funded by a two
-year European Union IST research grant. The full partners of that
-consortium are:
-
+PyPy development and activities happen as an open source project  
+and with the support of a consortium partially funded by a two 
+year European Union IST research grant. The full partners of that 
+consortium are: 
+        
     Heinrich-Heine University (Germany), AB Strakt (Sweden)
-    merlinux GmbH (Germany), tismerysoft GmbH (Germany)
+    merlinux GmbH (Germany), tismerysoft GmbH (Germany) 
     Logilab Paris (France), DFKI GmbH (Germany)
     ChangeMaker (Sweden), Impara (Germany)
diff --git a/pypy/doc/releases/0.99.0.rst b/pypy/doc/release-0.99.0.rst
rename from pypy/doc/releases/0.99.0.rst
rename to pypy/doc/release-0.99.0.rst
--- a/pypy/doc/releases/0.99.0.rst
+++ b/pypy/doc/release-0.99.0.rst
@@ -1,37 +1,37 @@
 ======================================================================
-pypy-0.99.0: new object spaces, optimizations, configuration ...
+pypy-0.99.0: new object spaces, optimizations, configuration ... 
 ======================================================================
 
 Welcome to the PyPy 0.99.0 release - a major snapshot
-and milestone of the last 8 months of work and contributions
+and milestone of the last 8 months of work and contributions 
 since PyPy-0.9.0 came out in June 2006!
 
-Main entry point for getting-started/download and documentation:
+Main entry point for getting-started/download and documentation: 
 
     http://codespeak.net/pypy/dist/pypy/doc/index.html
 
 Further below you'll find some notes about PyPy,
-the 0.99.0 highlights and our aims for PyPy 1.0.
+the 0.99.0 highlights and our aims for PyPy 1.0. 
 
-have fun,
+have fun, 
 
-    the PyPy team,
+    the PyPy team, 
     Samuele Pedroni, Carl Friedrich Bolz, Armin Rigo, Michael Hudson,
     Maciej Fijalkowski, Anders Chrigstroem, Holger Krekel,
     Guido Wesdorp
 
-    and many others:
+    and many others: 
     http://codespeak.net/pypy/dist/pypy/doc/contributor.html
 
 
-What is PyPy?
+What is PyPy? 
 ================================
 
-Technically, PyPy is both a Python Interpreter implementation
-and an advanced Compiler, actually a framework for implementing
+Technically, PyPy is both a Python Interpreter implementation 
+and an advanced Compiler, actually a framework for implementing 
 dynamic languages and generating virtual machines for them.
 The Framework allows for alternative frontends and
-for alternative backends, currently C, LLVM and .NET.
+for alternative backends, currently C, LLVM and .NET.  
 For our main target "C", we can can "mix in" different Garbage
 Collectors and threading models, including micro-threads aka
 "Stackless".  The inherent complexity that arises from this
@@ -41,36 +41,36 @@
 Socially, PyPy is a collaborative effort of many individuals
 working together in a distributed and sprint-driven way since
 2003.  PyPy would not have gotten as far without the coding,
-feedback and general support from numerous people.
+feedback and general support from numerous people. 
 
 Formally, many of the current developers are involved in
 executing an EU contract with the goal of exploring and
 researching new approaches to Language/Compiler development and
 software engineering.  This contract's duration is about to
 end March 2007 and we are working and preparing the according
-final review which is scheduled for May 2007.
+final review which is scheduled for May 2007.  
 
 
-Key 0.99.0 Features
+Key 0.99.0 Features 
 =====================
 
 * new object spaces:
 
-  - Tainting: a 270-line proxy object space tracking
-    and boxing sensitive information within an application.
-    A tainted object is completely barred from crossing
+  - Tainting: a 270-line proxy object space tracking 
+    and boxing sensitive information within an application. 
+    A tainted object is completely barred from crossing 
     an I/O barrier, such as writing to files, databases
-    or sockets.  This allows to significantly reduce the
-    effort of e.g. security reviews to the few places where
-    objects are "declassified" in order to send information
-    across I/O barriers.
+    or sockets.  This allows to significantly reduce the 
+    effort of e.g. security reviews to the few places where 
+    objects are "declassified" in order to send information 
+    across I/O barriers. 
 
   - Transparent proxies: allow to customize both application and
     builtin objects from application level code.  Works as an addition
     to the Standard Object Space (and is translatable). For details see
     http://codespeak.net/pypy/dist/pypy/doc/proxy.html
-
-* optimizations:
+ 
+* optimizations: 
 
   - Experimental new optimized implementations for various built in Python
     types (strings, dicts, lists)
@@ -81,7 +81,7 @@
   - Improved inlining (now also working for higher level
     backends) and malloc removal.
 
-  - twice the speed of the 0.9 release, overall 2-3 slower than CPython
+  - twice the speed of the 0.9 release, overall 2-3 slower than CPython 
 
 * High level backends:
 
@@ -92,15 +92,15 @@
   - the JavaScript backend has evolved to a point where it can be used to write
     AJAX web applications with it. This is still an experimental technique,
     though. For demo applications see:
-    http://play1.codespeak.net/
+    http://play1.codespeak.net/ 
 
-* new configuration system:
-  There is a new comprehensive configuration system that allows
+* new configuration system: 
+  There is a new comprehensive configuration system that allows 
   fine-grained configuration of the PyPy standard interpreter and the
-  translation process.
+  translation process. 
 
 * new and improved modules: Since the last release, the signal, mmap, bz2
-  and fcntl standard library modules have been implemented for PyPy. The socket,
+  and fcntl standard library modules have been implemented for PyPy. The socket, 
   _sre and os modules have been greatly improved. In addition we added a the
   pypymagic module that contains PyPy-specific functionality.
 
@@ -116,60 +116,60 @@
   http://codespeak.net/pypy/dist/pypy/doc/rlib.html
 
 * improved documentation:
-
+  
   - extended documentation about stackless features:
     http://codespeak.net/pypy/dist/pypy/doc/stackless.html
-
+  
   - PyPy video documentation: eight hours of talks, interviews and features:
     http://codespeak.net/pypy/dist/pypy/doc/video-index.html
 
   - technical reports about various aspects of PyPy:
     http://codespeak.net/pypy/dist/pypy/doc/index-report.html
-
+    
   The entry point to all our documentation is:
   http://codespeak.net/pypy/dist/pypy/doc/index.html
 
 
 
-What about 1.0?
+What about 1.0? 
 ======================
 
 In the last week leading up to the release, we decided
 to go for tagging the release as 0.99.0, mainly because
-we have some efforts pending to integrate and complete
-research and coding work:
+we have some efforts pending to integrate and complete 
+research and coding work: 
 
 * the JIT Compiler Generator is ready, but not fully integrated
   with the PyPy interpreter.  As a result, the JIT does not give
   actual speed improvements yet, so we chose to leave it out of the
   0.99 release: the result doesn't meet yet the speed expectations
-  that we set for ourselves - and which some blogs and people
+  that we set for ourselves - and which some blogs and people 
   have chosen as the main criterium for looking at PyPy.
 
 * the extension enabling runtime changes of the Python grammar is not
   yet integrated. This will be used to provide Aspect-Oriented
-  Programming extensions and Design by Contract facilities in PyPy.
+  Programming extensions and Design by Contract facilities in PyPy. 
 
 * the Logic object space, which provides Logic Variables in PyPy,
   needs to undergo a bit more testing. A constraint problem solver
-  extension module is ready, and needs to be integrated with the codebase.
+  extension module is ready, and needs to be integrated with the codebase. 
 
 PyPy 0.99 is the start for getting to 1.0 end of March 2007,
-which we intend to become a base for a longer (and more relaxed :)
-time to come.
+which we intend to become a base for a longer (and more relaxed :) 
+time to come. 
 
 
 
 Funding partners and organizations
 =====================================================
-
-PyPy development and activities happen as an open source project
+    
+PyPy development and activities happen as an open source project  
 and with the support of a consortium partially funded by a 28 months
-European Union IST research grant. The full partners of that
-consortium are:
-
+European Union IST research grant. The full partners of that 
+consortium are: 
+        
     Heinrich-Heine University (Germany), Open End (Sweden)
-    merlinux GmbH (Germany), tismerysoft GmbH (Germany)
+    merlinux GmbH (Germany), tismerysoft GmbH (Germany) 
     Logilab Paris (France), DFKI GmbH (Germany)
     ChangeMaker (Sweden), Impara (Germany)
 
diff --git a/pypy/doc/releases/1.0.0.rst b/pypy/doc/release-1.0.0.rst
rename from pypy/doc/releases/1.0.0.rst
rename to pypy/doc/release-1.0.0.rst
--- a/pypy/doc/releases/1.0.0.rst
+++ b/pypy/doc/release-1.0.0.rst
@@ -2,8 +2,8 @@
 PyPy 1.0: JIT compilers for free and more
 ==========================================
 
-Welcome to the PyPy 1.0 release - a milestone integrating the results
-of four years of research, engineering, management and sprinting
+Welcome to the PyPy 1.0 release - a milestone integrating the results 
+of four years of research, engineering, management and sprinting 
 efforts, concluding the 28 months phase of EU co-funding!
 
 Although still not mature enough for general use, PyPy 1.0 materializes
diff --git a/pypy/doc/releases/1.1.0.rst b/pypy/doc/release-1.1.0.rst
rename from pypy/doc/releases/1.1.0.rst
rename to pypy/doc/release-1.1.0.rst
--- a/pypy/doc/releases/1.1.0.rst
+++ b/pypy/doc/release-1.1.0.rst
@@ -8,7 +8,7 @@
 interpreter more stable and bug-free.
 
 Download page:
-
+    
    http://codespeak.net/pypy/dist/pypy/doc/download.html
 
 PyPy's Getting Started lives at:
@@ -19,7 +19,7 @@
 ==========================
 
   - More of CPython's standard library extension modules are supported,
-    among them ctypes, sqlite3, csv, and many more. Most of these extension
+    among them ctypes, sqlite3, csv, and many more. Most of these extension 
     modules are fully supported under Windows as well.
 
     http://codespeak.net/pypy/dist/pypy/doc/cpython_differences.html
@@ -51,7 +51,7 @@
 
   - Sandboxing support: It is possible to translate the Python
     interpreter in a special way so that the result is fully sandboxed.
-
+    
     http://codespeak.net/pypy/dist/pypy/doc/sandbox.html
     http://blog.sandbox.lt/en/WSGI%20and%20PyPy%20sandbox
 
@@ -137,7 +137,7 @@
 Have fun,
 
     the PyPy release team, [in alphabetical order]
-
+    
     Amaury Forgeot d'Arc, Anders Hammerquist, Antonio Cuni, Armin Rigo,
     Carl Friedrich Bolz, Christian Tismer, Holger Krekel,
     Maciek Fijalkowski, Samuele Pedroni
diff --git a/pypy/doc/releases/1.2.0.rst b/pypy/doc/release-1.2.0.rst
rename from pypy/doc/releases/1.2.0.rst
rename to pypy/doc/release-1.2.0.rst
diff --git a/pypy/doc/releases/1.3.0.rst b/pypy/doc/release-1.3.0.rst
rename from pypy/doc/releases/1.3.0.rst
rename to pypy/doc/release-1.3.0.rst
diff --git a/pypy/doc/releases/1.4.0.rst b/pypy/doc/release-1.4.0.rst
rename from pypy/doc/releases/1.4.0.rst
rename to pypy/doc/release-1.4.0.rst
--- a/pypy/doc/releases/1.4.0.rst
+++ b/pypy/doc/release-1.4.0.rst
@@ -47,11 +47,11 @@
 * Other speed improvements, like JITted calls to functions like map().
 
 .. _virtualenv: http://pypi.python.org/pypi/virtualenv
-.. _Virtualenv support: http://morepypy.blogspot.com/2010/08/using-virtualenv-with-pypy.html
-.. _in production: http://morepypy.blogspot.com/2010/11/running-large-radio-telescope-software.html
-.. _our blog: http://morepypy.blogspot.com
-.. _pypy 1.4 and pypy 1.3: http://speed.pypy.org/comparison/?exe=1%2B41,1%2B172&ben=1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20&env=1&hor=false&bas=1%2B41&chart=normal+bars
-.. _pypy 1.4 and cpython 2.6: http://speed.pypy.org/comparison/?exe=2%2B35,1%2B172&ben=1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20&env=1&hor=false&bas=2%2B35&chart=normal+bars
+.. _`Virtualenv support`: http://morepypy.blogspot.com/2010/08/using-virtualenv-with-pypy.html
+.. _`in production`: http://morepypy.blogspot.com/2010/11/running-large-radio-telescope-software.html
+.. _`our blog`: http://morepypy.blogspot.com
+.. _`pypy 1.4 and pypy 1.3`: http://speed.pypy.org/comparison/?exe=1%2B41,1%2B172&ben=1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20&env=1&hor=false&bas=1%2B41&chart=normal+bars
+.. _`pypy 1.4 and cpython 2.6`: http://speed.pypy.org/comparison/?exe=2%2B35,1%2B172&ben=1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20&env=1&hor=false&bas=2%2B35&chart=normal+bars
 
 Cheers,
 
diff --git a/pypy/doc/releases/1.4.0beta.rst b/pypy/doc/release-1.4.0beta.rst
rename from pypy/doc/releases/1.4.0beta.rst
rename to pypy/doc/release-1.4.0beta.rst
--- a/pypy/doc/releases/1.4.0beta.rst
+++ b/pypy/doc/release-1.4.0beta.rst
@@ -33,4 +33,4 @@
 Cheers,
 The PyPy team
 
-.. _list of patches: https://bitbucket.org/pypy/pypy/src/tip/pypy/module/cpyext/patches/
+.. _`list of patches`: https://bitbucket.org/pypy/pypy/src/tip/pypy/module/cpyext/patches/
diff --git a/pypy/doc/releases/1.4.1.rst b/pypy/doc/release-1.4.1.rst
rename from pypy/doc/releases/1.4.1.rst
rename to pypy/doc/release-1.4.1.rst
diff --git a/pypy/doc/releases/1.5.0.rst b/pypy/doc/release-1.5.0.rst
rename from pypy/doc/releases/1.5.0.rst
rename to pypy/doc/release-1.5.0.rst
--- a/pypy/doc/releases/1.5.0.rst
+++ b/pypy/doc/release-1.5.0.rst
@@ -59,15 +59,15 @@
 Amaury Forgeot d'Arc, Alex Gaynor, Armin Rigo and the PyPy team
 
 
-.. _CPython 2.6: http://docs.python.org/dev/whatsnew/2.6.html
-.. _CPython 2.7: http://docs.python.org/dev/whatsnew/2.7.html
+.. _`CPython 2.6`: http://docs.python.org/dev/whatsnew/2.6.html
+.. _`CPython 2.7`: http://docs.python.org/dev/whatsnew/2.7.html
 
-.. _our blog: http://morepypy.blogspot.com
-.. _pypy 1.5 and pypy 1.4: http://bit.ly/joPhHo
-.. _pypy 1.5 and cpython 2.6.2: http://bit.ly/mbVWwJ
+.. _`our blog`: http://morepypy.blogspot.com
+.. _`pypy 1.5 and pypy 1.4`: http://bit.ly/joPhHo
+.. _`pypy 1.5 and cpython 2.6.2`: http://bit.ly/mbVWwJ
 
-.. _loop invariant code motion: http://morepypy.blogspot.com/2011/01/loop-invariant-code-motion.html
-.. _compatibility wiki: https://bitbucket.org/pypy/compatibility/wiki/Home
-.. _Tkinter and IDLE: http://morepypy.blogspot.com/2011/04/using-tkinter-and-idle-with-pypy.html
-.. _cProfile: http://docs.python.org/library/profile.html
-.. _external fork: https://bitbucket.org/alex_gaynor/pypy-postgresql
+.. _`loop invariant code motion`: http://morepypy.blogspot.com/2011/01/loop-invariant-code-motion.html
+.. _`compatibility wiki`: https://bitbucket.org/pypy/compatibility/wiki/Home
+.. _`Tkinter and IDLE`: http://morepypy.blogspot.com/2011/04/using-tkinter-and-idle-with-pypy.html
+.. _`cProfile`: http://docs.python.org/library/profile.html
+.. _`external fork`: https://bitbucket.org/alex_gaynor/pypy-postgresql
diff --git a/pypy/doc/releases/1.6.0.rst b/pypy/doc/release-1.6.0.rst
rename from pypy/doc/releases/1.6.0.rst
rename to pypy/doc/release-1.6.0.rst
--- a/pypy/doc/releases/1.6.0.rst
+++ b/pypy/doc/release-1.6.0.rst
@@ -30,7 +30,7 @@
 the JIT warmup time, the optimizations performed by the JIT, the quality of
 the generated machine code and the implementation of our Python interpreter.
 
-.. _pypy 1.5 and cpython 2.6.2: http://speed.pypy.org
+.. _`pypy 1.5 and cpython 2.6.2`: http://speed.pypy.org
 
 
 Highlights
@@ -89,7 +89,7 @@
 Maciej Fijalkowski, Amaury Forgeot d'Arc, Alex Gaynor,
 Armin Rigo and the PyPy team
 
-.. _jitviewer: http://morepypy.blogspot.com/2011/08/visualization-of-jitted-code.html
-.. _bug tracker: https://bugs.pypy.org
-.. _compatibility wiki: https://bitbucket.org/pypy/compatibility/wiki/Home
+.. _`jitviewer`: http://morepypy.blogspot.com/2011/08/visualization-of-jitted-code.html
+.. _`bug tracker`: https://bugs.pypy.org
+.. _`compatibility wiki`: https://bitbucket.org/pypy/compatibility/wiki/Home
 
diff --git a/pypy/doc/releases/1.7.0.rst b/pypy/doc/release-1.7.0.rst
rename from pypy/doc/releases/1.7.0.rst
rename to pypy/doc/release-1.7.0.rst
--- a/pypy/doc/releases/1.7.0.rst
+++ b/pypy/doc/release-1.7.0.rst
@@ -26,7 +26,7 @@
 our benchmark suite, PyPy 1.7 is around **30%** faster than PyPy 1.6 and up
 to **20 times** faster on some benchmarks.
 
-.. _pypy 1.7 and cpython 2.7.1: http://speed.pypy.org
+.. _`pypy 1.7 and cpython 2.7.1`: http://speed.pypy.org
 
 
 Highlights
@@ -89,6 +89,6 @@
 `py3k proposal`_. In case you want PyPy to progress, but you trust us with
 the general direction, you can always donate to the `general pot`_.
 
-.. _numpy proposal: http://pypy.org/numpydonate.html
-.. _py3k proposal: http://pypy.org/py3donate.html
-.. _general pot: http://pypy.org
+.. _`numpy proposal`: http://pypy.org/numpydonate.html
+.. _`py3k proposal`: http://pypy.org/py3donate.html
+.. _`general pot`: http://pypy.org
diff --git a/pypy/doc/releases/1.8.0.rst b/pypy/doc/release-1.8.0.rst
rename from pypy/doc/releases/1.8.0.rst
rename to pypy/doc/release-1.8.0.rst
--- a/pypy/doc/releases/1.8.0.rst
+++ b/pypy/doc/release-1.8.0.rst
@@ -15,7 +15,7 @@
 
     http://pypy.org/download.html
 
-.. _list strategies: http://morepypy.blogspot.com/2011/10/more-compact-lists-with-list-strategies.html
+.. _`list strategies`: http://morepypy.blogspot.com/2011/10/more-compact-lists-with-list-strategies.html
 
 What is PyPy?
 =============
@@ -28,7 +28,7 @@
 Windows 32. Windows 64 work has been stalled, we would welcome a volunteer
 to handle that.
 
-.. _pypy 1.8 and cpython 2.7.1: http://speed.pypy.org
+.. _`pypy 1.8 and cpython 2.7.1`: http://speed.pypy.org
 
 
 Highlights
@@ -90,9 +90,9 @@
 Cheers,
 The PyPy Team
 
-.. _brief overview: http://doc.pypy.org/en/latest/jit-hooks.html
-.. _numpy status page: http://buildbot.pypy.org/numpy-status/latest.html
-.. _numpy status update blog report: http://morepypy.blogspot.com/2012/01/numpypy-status-update.html
-.. _numpypy: http://pypy.org/numpydonate.html
-.. _py3k: http://pypy.org/py3donate.html
-.. _our plans: http://morepypy.blogspot.com/2012/01/transactional-memory-ii.html
+.. _`brief overview`: http://doc.pypy.org/en/latest/jit-hooks.html
+.. _`numpy status page`: http://buildbot.pypy.org/numpy-status/latest.html
+.. _`numpy status update blog report`: http://morepypy.blogspot.com/2012/01/numpypy-status-update.html
+.. _`numpypy`: http://pypy.org/numpydonate.html
+.. _`py3k`: http://pypy.org/py3donate.html
+.. _`our plans`: http://morepypy.blogspot.com/2012/01/transactional-memory-ii.html
diff --git a/pypy/doc/releases/1.9.0.rst b/pypy/doc/release-1.9.0.rst
rename from pypy/doc/releases/1.9.0.rst
rename to pypy/doc/release-1.9.0.rst
--- a/pypy/doc/releases/1.9.0.rst
+++ b/pypy/doc/release-1.9.0.rst
@@ -9,9 +9,9 @@
 
 You can download the PyPy 1.9 release here:
 
-    http://pypy.org/download.html
+    http://pypy.org/download.html 
 
-.. _numpypy: http://pypy.org/numpydonate.html
+.. _`numpypy`: http://pypy.org/numpydonate.html
 
 
 What is PyPy?
@@ -25,7 +25,7 @@
 Windows 32.  Windows 64 work is still stalling, we would welcome a volunteer
 to handle that.
 
-.. _pypy 1.9 and cpython 2.7.2: http://speed.pypy.org
+.. _`pypy 1.9 and cpython 2.7.2`: http://speed.pypy.org
 
 
 Thanks to our donors
@@ -51,9 +51,9 @@
 PyPy from the corresponding branches (respectively ``py3k`` and
 ``stm-thread``).
 
-.. _NumPy in PyPy: http://pypy.org/numpydonate.html
-.. _Py3k (Python 3): http://pypy.org/py3donate.html
-.. _Software Transactional Memory: http://pypy.org/tmdonate.html
+.. _`NumPy in PyPy`: http://pypy.org/numpydonate.html
+.. _`Py3k (Python 3)`: http://pypy.org/py3donate.html
+.. _`Software Transactional Memory`: http://pypy.org/tmdonate.html
 
 Highlights
 ==========
@@ -96,8 +96,8 @@
 
 * List comprehension has been improved.
 
-.. _numpy-status: http://buildbot.pypy.org/numpy-status/latest.html
-.. _JIT hooks documentation: http://doc.pypy.org/en/latest/jit-hooks.html
+.. _`numpy-status`: http://buildbot.pypy.org/numpy-status/latest.html
+.. _`JIT hooks documentation`: http://doc.pypy.org/en/latest/jit-hooks.html
 
 JitViewer
 =========
@@ -105,7 +105,7 @@
 There will be a corresponding 1.9 release of JitViewer which is guaranteed
 to work with PyPy 1.9. See the `JitViewer docs`_ for details.
 
-.. _JitViewer docs: http://bitbucket.org/pypy/jitviewer
+.. _`JitViewer docs`: http://bitbucket.org/pypy/jitviewer
 
 Cheers,
 The PyPy Team
diff --git a/pypy/doc/releases/2.0.0-beta1.rst b/pypy/doc/release-2.0.0-beta1.rst
rename from pypy/doc/releases/2.0.0-beta1.rst
rename to pypy/doc/release-2.0.0-beta1.rst
--- a/pypy/doc/releases/2.0.0-beta1.rst
+++ b/pypy/doc/release-2.0.0-beta1.rst
@@ -14,7 +14,7 @@
 
 You can download the PyPy 2.0 beta 1 release here:
 
-    http://pypy.org/download.html
+    http://pypy.org/download.html 
 
 What is PyPy?
 =============
@@ -28,7 +28,7 @@
 Windows 64 work is still stalling, we would welcome a volunteer
 to handle that.
 
-.. _pypy 2.0 beta 1 and cpython 2.7.3: http://bit.ly/USXqpP
+.. _`pypy 2.0 beta 1 and cpython 2.7.3`: http://bit.ly/USXqpP
 
 How to use PyPy?
 ================
@@ -37,10 +37,10 @@
 installed, you can follow instructions from `pypy documentation`_ on how
 to proceed. This document also covers other `installation schemes`_.
 
-.. _pypy documentation: http://doc.pypy.org/en/latest/getting-started.html#installing-using-virtualenv
-.. _virtualenv: http://www.virtualenv.org/en/latest/
-.. _installation schemes: http://doc.pypy.org/en/latest/getting-started.html#installing-pypy
-.. _PyPy and pip: http://doc.pypy.org/en/latest/getting-started.html#installing-pypy
+.. _`pypy documentation`: http://doc.pypy.org/en/latest/getting-started.html#installing-using-virtualenv
+.. _`virtualenv`: http://www.virtualenv.org/en/latest/
+.. _`installation schemes`: http://doc.pypy.org/en/latest/getting-started.html#installing-pypy
+.. _`PyPy and pip`: http://doc.pypy.org/en/latest/getting-started.html#installing-pypy
 
 Regressions
 ===========
@@ -101,8 +101,8 @@
   unicode strings, which means that now such collections will be both faster
   and more compact.
 
-.. _cpython issue tracker: http://bugs.python.org/issue14621
-.. _jit hooks: http://doc.pypy.org/en/latest/jit-hooks.html
+.. _`cpython issue tracker`: http://bugs.python.org/issue14621
+.. _`jit hooks`: http://doc.pypy.org/en/latest/jit-hooks.html
 
 Things we're working on
 =======================
diff --git a/pypy/doc/releases/2.0.0-beta2.rst b/pypy/doc/release-2.0.0-beta2.rst
rename from pypy/doc/releases/2.0.0-beta2.rst
rename to pypy/doc/release-2.0.0-beta2.rst
--- a/pypy/doc/releases/2.0.0-beta2.rst
+++ b/pypy/doc/release-2.0.0-beta2.rst
@@ -23,7 +23,7 @@
 
 You can download the PyPy 2.0 beta 2 release here:
 
-    http://pypy.org/download.html
+    http://pypy.org/download.html 
 
 What is PyPy?
 =============
@@ -38,7 +38,7 @@
 Windows 64 work is still stalling, we would welcome a volunteer
 to handle that.
 
-.. _pypy 2.0 beta 2 and cpython 2.7.3: http://bit.ly/USXqpP
+.. _`pypy 2.0 beta 2 and cpython 2.7.3`: http://bit.ly/USXqpP
 
 How to use PyPy?
 ================
@@ -47,9 +47,9 @@
 installed, you can follow instructions from `pypy documentation`_ on how
 to proceed. This document also covers other `installation schemes`_.
 
-.. _pypy documentation: http://doc.pypy.org/en/latest/getting-started.html#installing-using-virtualenv
-.. _virtualenv: http://www.virtualenv.org/en/latest/
-.. _installation schemes: http://doc.pypy.org/en/latest/getting-started.html#installing-pypy
+.. _`pypy documentation`: http://doc.pypy.org/en/latest/getting-started.html#installing-using-virtualenv
+.. _`virtualenv`: http://www.virtualenv.org/en/latest/
+.. _`installation schemes`: http://doc.pypy.org/en/latest/getting-started.html#installing-pypy
 
 Highlights
 ==========
@@ -83,12 +83,12 @@
   unicode strings, which means that now such collections will be both faster
   and more compact.
 
-.. _eventlet: http://eventlet.net/
-.. _gevent: http://www.gevent.org/
-.. _cffi: http://cffi.readthedocs.org/en/release-0.6/
-.. _JIT hooks: http://doc.pypy.org/en/latest/jit-hooks.html
-.. _pypycore: https://github.com/gevent-on-pypy/pypycore
-.. _pypy-hacks: https://github.com/schmir/gevent/tree/pypy-hacks
-.. _\_curses.py: https://bitbucket.org/pypy/pypy/src/aefddd47f224e3c12e2ea74f5c796d76f4355bdb/lib_pypy/_curses.py?at=default
-.. _\_sqlite3.py: https://bitbucket.org/pypy/pypy/src/aefddd47f224e3c12e2ea74f5c796d76f4355bdb/lib_pypy/_sqlite3.py?at=default
+.. _`eventlet`: http://eventlet.net/
+.. _`gevent`: http://www.gevent.org/
+.. _`cffi`: http://cffi.readthedocs.org/en/release-0.6/
+.. _`JIT hooks`: http://doc.pypy.org/en/latest/jit-hooks.html
+.. _`pypycore`: https://github.com/gevent-on-pypy/pypycore
+.. _`pypy-hacks`: https://github.com/schmir/gevent/tree/pypy-hacks
+.. _`_curses.py`: https://bitbucket.org/pypy/pypy/src/aefddd47f224e3c12e2ea74f5c796d76f4355bdb/lib_pypy/_curses.py?at=default
+.. _`_sqlite3.py`: https://bitbucket.org/pypy/pypy/src/aefddd47f224e3c12e2ea74f5c796d76f4355bdb/lib_pypy/_sqlite3.py?at=default
 
diff --git a/pypy/doc/releases/2.0.0.rst b/pypy/doc/release-2.0.0.rst
rename from pypy/doc/releases/2.0.0.rst
rename to pypy/doc/release-2.0.0.rst
--- a/pypy/doc/releases/2.0.0.rst
+++ b/pypy/doc/release-2.0.0.rst
@@ -19,7 +19,7 @@
 * PyPy now contains release 0.6 of `cffi`_ as a builtin module, which
   is preferred way of calling C from Python that works well on PyPy
 
-.. _cffi: http://cffi.readthedocs.org
+.. _`cffi`: http://cffi.readthedocs.org
 
 If you're using PyPy for anything, it would help us immensely if you fill out
 the following survey: http://bit.ly/pypysurvey This is for the developers
@@ -37,7 +37,7 @@
 to handle that. ARM support is on the way, as you can see from the recently
 released alpha for ARM.
 
-.. _pypy 2.0 and cpython 2.7.3: http://speed.pypy.org
+.. _`pypy 2.0 and cpython 2.7.3`: http://speed.pypy.org
 
 Highlights
 ==========
@@ -64,8 +64,8 @@
   lazy expression evaluation. On the other hand, we now have more complete
   dtype support and support more array attributes.  
 
-.. _pypycore: https://github.com/gevent-on-pypy/pypycore/
-.. _pypy-hacks: https://github.com/schmir/gevent/tree/pypy-hacks
+.. _`pypycore`: https://github.com/gevent-on-pypy/pypycore/
+.. _`pypy-hacks`: https://github.com/schmir/gevent/tree/pypy-hacks
 
 Cheers,
 fijal, arigo and the PyPy team
diff --git a/pypy/doc/release-2.4.0.rst b/pypy/doc/release-2.4.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-2.4.0.rst
@@ -0,0 +1,119 @@
+=================================================
+PyPy 2.4 - Snow White
+=================================================
+
+We're pleased to announce PyPy 2.4, which contains significant performance
+enhancements and bug fixes. 
+
+You can already download the PyPy 2.4-beta1 pre-release here:
+
+    http://pypy.org/download.html
+
+We would like to thank our donors for the continued support of the PyPy
+project, and for those who donate to our three sub-projects.
+We've shown quite a bit of progress, but we're slowly running out of funds.
+Please consider donating more, or even better convince your employer to donate,
+so we can finish those projects! We would like to also point out that in
+September, `the Python Software Foundation`_ will `match funds`_ for
+any donations up to $10k!  The three sub-projects are:
+
+* `Py3k`_ (supporting Python 3.x): We have released a Python 3.2.5 compatible version
+   we call PyPy3 2.3.1, and are working toward a Python 3.3 compatible version
+
+* `STM`_ (software transactional memory): We have released a first working version,
+  and continue to try out new promising paths of achieving a fast multithreaded Python
+
+* `NumPy`_ which requires installation of our fork of upstream numpy, 
+  available `on bitbucket`_
+
+.. _`Py3k`: http://pypy.org/py3donate.html
+.. _`STM`: http://pypy.org/tmdonate2.html
+.. _`NumPy`: http://pypy.org/numpydonate.html
+.. _`on bitbucket`: https://www.bitbucket.org/pypy/numpy   
+.. _`the Python Software Foundation`: https://www.python.org/psf/
+.. _`match funds`: http://morepypy.blogspot.com/2014/09/python-software-foundation-matching.html
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`pypy 2.4 and cpython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+This release supports **x86** machines on most common operating systems 
+(Linux 32/64, Mac OS X 64, Windows, and OpenBSD),
+as well as newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux. 
+
+While we support 32 bit python on Windows, work on the native Windows 64
+bit python is still stalling, we would welcome a volunteer
+to `handle that`_.
+
+.. _`pypy 2.4 and cpython 2.7.x`: http://speed.pypy.org
+.. _`handle that`: http://doc.pypy.org/en/latest/windows.html#what-is-missing-for-a-full-64-bit-translation
+
+Highlights
+==========
+
+Benchmarks improved after internal enhancements in string and
+bytearray handling, and a major rewrite of the GIL handling. This means
+that external calls are now a lot faster, especially the CFFI ones. It also
+means better performance in a lot of corner cases with handling strings or
+bytearrays. The main bugfix is handling of many socket objects in your
+program which in the long run used to "leak" memory.
+
+PyPy now uses Python 2.7.8 standard library.
+
+We welcomed more than 12 new contributors, and conducted two Google
+Summer of Code projects, as well as other student projects not
+directly related to Summer of Code.
+
+
+Issues reported with our previous release were fixed after reports from users on
+our new issue tracker at https://bitbucket.org/pypy/pypy/issues or on IRC at
+#pypy. Here is a summary of the user-facing changes;
+for more information see `whats-new`_:
+
+* Reduced internal copying of bytearray operations
+
+* Tweak the internal structure of StringBuilder to speed up large string
+  handling, which becomes advantageous on large programs at the cost of slightly
+  slower small *benchmark* type programs.
+
+* Boost performance of thread-local variables in both unjitted and jitted code,
+  this mostly affects errno handling on linux, which makes external calls
+  faster.
+
+* Move to a mixed polling and mutex GIL model that make mutlithreaded jitted
+  code run *much* faster
+
+* Optimize errno handling in linux (x86 and x86-64 only)
+
+* Remove ctypes pythonapi and ctypes.PyDLL, which never worked on PyPy
+
+* Fix performance regression on ufunc(, ) in numpy
+
+* Classes in the ast module are now distinct from structures used by
+  the compiler, which simplifies and speeds up translation of our
+  source code to the PyPy binary interpreter
+
+* Upgrade stdlib from 2.7.5 to 2.7.8
+
+* Win32 now links statically to zlib, expat, bzip, and openssl-1.0.1i.
+  No more missing DLLs
+  
+* Many issues were resolved_ since the 2.3.1 release on June 8
+
+.. _`whats-new`: http://doc.pypy.org/en/latest/whatsnew-2.3.1.html
+.. _resolved: https://bitbucket.org/pypy/pypy/issues?status=resolved
+
+We have further improvements on the way: rpython file handling,
+numpy linalg compatibility, as well
+as improved GC and many smaller improvements.
+
+Please try it out and let us know what you think. We especially welcome
+success stories, we know you are using PyPy, please tell us about it!
+
+Cheers
+
+The PyPy Team
+
diff --git a/pypy/doc/releases/index.rst b/pypy/doc/releases/index.rst
deleted file mode 100644
--- a/pypy/doc/releases/index.rst
+++ /dev/null
@@ -1,25 +0,0 @@
-Release Announcements
-=====================
-
-.. toctree::
-
-   2.0.0
-   2.0.0-beta2
-   2.0.0-beta1
-   1.9.0
-   1.8.0
-   1.7.0
-   1.6.0
-   1.5.0
-   1.4.1
-   1.4.0beta
-   1.4.0
-   1.3.0
-   1.2.0
-   1.1.0
-   1.0.0
-   0.99.0
-   0.9.0
-   0.8.0
-   0.7.0
-   0.6

From noreply at buildbot.pypy.org  Sun Sep 14 04:23:35 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Sun, 14 Sep 2014 04:23:35 +0200 (CEST)
Subject: [pypy-commit] pypy default: cleanup
Message-ID: <20140914022335.614AF1C3233@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey 
Branch: 
Changeset: r73528:5e1d74cb3416
Date: 2014-09-13 18:10 -0700
http://bitbucket.org/pypy/pypy/changeset/5e1d74cb3416/

Log:	cleanup

diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -38,18 +38,15 @@
 def cpython_code_signature(code):
     "([list-of-arg-names], vararg-name-or-None, kwarg-name-or-None)."
     argcount = code.co_argcount
+    varnames = code.co_varnames
     assert argcount >= 0     # annotator hint
-    argnames = list(code.co_varnames[:argcount])
+    argnames = list(varnames[:argcount])
     if code.co_flags & CO_VARARGS:
-        varargname = code.co_varnames[argcount]
+        varargname = varnames[argcount]
         argcount += 1
     else:
         varargname = None
-    if code.co_flags & CO_VARKEYWORDS:
-        kwargname = code.co_varnames[argcount]
-        argcount += 1
-    else:
-        kwargname = None
+    kwargname = varnames[argcount] if code.co_flags & CO_VARKEYWORDS else None
     return Signature(argnames, varargname, kwargname)
 
 

From noreply at buildbot.pypy.org  Sun Sep 14 04:23:36 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Sun, 14 Sep 2014 04:23:36 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: merge default
Message-ID: <20140914022336.CDC571C3233@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey 
Branch: py3k
Changeset: r73529:201aee996695
Date: 2014-09-13 18:26 -0700
http://bitbucket.org/pypy/pypy/changeset/201aee996695/

Log:	merge default

diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -85,10 +85,13 @@
 
 Abridged method (for -Ojit builds using Visual Studio 2008)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Download the versions of all the external packages
-from 
+Download the versions of all the external packages from 
+https://bitbucket.org/pypy/pypy/downloads/local_2.4.zip
+(for 2.4 release and later) or
 https://bitbucket.org/pypy/pypy/downloads/local.zip
-Then expand it into the base directory (base_dir) and modify your environment to reflect this::
+(for pre-2.4 versions)
+Then expand it into the base directory (base_dir) and modify your environment
+to reflect this::
 
     set PATH=\bin;\tcltk\bin;%PATH%
     set INCLUDE=\include;\tcltk\include;%INCLUDE%
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -45,6 +45,7 @@
 def cpython_code_signature(code):
     "([list-of-arg-names], vararg-name-or-None, kwarg-name-or-None)."
     argcount = code.co_argcount
+    varnames = code.co_varnames
     if we_are_translated():
         kwonlyargcount = code.co_kwonlyargcount
     else:
@@ -52,18 +53,14 @@
         kwonlyargcount = getattr(code, 'co_kwonlyargcount', 0)
     assert argcount >= 0     # annotator hint
     assert kwonlyargcount >= 0
-    argnames = list(code.co_varnames[:argcount])
-    kwonlyargs = list(code.co_varnames[argcount:argcount + kwonlyargcount])
+    argnames = list(varnames[:argcount])
+    kwonlyargs = list(varnames[argcount:argcount + kwonlyargcount])
     if code.co_flags & CO_VARARGS:
-        varargname = code.co_varnames[argcount]
+        varargname = varnames[argcount]
         argcount += 1
     else:
         varargname = None
-    if code.co_flags & CO_VARKEYWORDS:
-        kwargname = code.co_varnames[argcount]
-        argcount += 1
-    else:
-        kwargname = None
+    kwargname = varnames[argcount] if code.co_flags & CO_VARKEYWORDS else None
     return Signature(argnames, varargname, kwargname, kwonlyargs)
 
 
diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -1064,7 +1064,7 @@
         prefix = udir.join('pathtest').ensure(dir=1)
         fake_exe = 'bin/pypy-c'
         if sys.platform == 'win32':
-            fake_exe += '.exe'
+            fake_exe = 'pypy-c.exe'
         fake_exe = prefix.join(fake_exe).ensure(file=1)
         expected_path = [str(prefix.join(subdir).ensure(dir=1))
                          for subdir in ('lib_pypy',
@@ -1104,8 +1104,10 @@
             assert sys.path == old_sys_path + [self.goal_dir]
 
             app_main.setup_bootstrap_path(self.fake_exe)
-            assert sys.executable == ''      # not executable!
-            assert sys.path == old_sys_path + [self.goal_dir]
+            if not sys.platform == 'win32':
+                # an existing file is always 'executable' on windows
+                assert sys.executable == ''      # not executable!
+                assert sys.path == old_sys_path + [self.goal_dir]
 
             os.chmod(self.fake_exe, 0o755)
             app_main.setup_bootstrap_path(self.fake_exe)
diff --git a/rpython/translator/platform/test/test_makefile.py b/rpython/translator/platform/test/test_makefile.py
--- a/rpython/translator/platform/test/test_makefile.py
+++ b/rpython/translator/platform/test/test_makefile.py
@@ -44,6 +44,7 @@
         assert res.returncode == 0        
     
     def test_900_files(self):
+        tmpdir = udir.join('test_900_files').ensure(dir=1)
         txt = '#include \n'
         for i in range(900):
             txt += 'int func%03d();\n' % i
@@ -52,11 +53,11 @@
             txt += '    j += func%03d();\n' % i
         txt += '    printf("%d\\n", j);\n'
         txt += '    return 0;};\n'
-        cfile = udir.join('test_900_files.c')
+        cfile = tmpdir.join('test_900_files.c')
         cfile.write(txt)
         cfiles = [cfile]
         for i in range(900):
-            cfile2 = udir.join('implement%03d.c' %i)
+            cfile2 = tmpdir.join('implement%03d.c' %i)
             cfile2.write('''
                 int func%03d()
             {
@@ -64,10 +65,10 @@
             }
             ''' % (i, i))
             cfiles.append(cfile2)
-        mk = self.platform.gen_makefile(cfiles, ExternalCompilationInfo(), path=udir)
+        mk = self.platform.gen_makefile(cfiles, ExternalCompilationInfo(), path=tmpdir)
         mk.write()
         self.platform.execute_makefile(mk)
-        res = self.platform.execute(udir.join('test_900_files'))
+        res = self.platform.execute(tmpdir.join('test_900_files'))
         self.check_res(res, '%d\n' %sum(range(900)))
 
     def test_precompiled_headers(self):
diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py
--- a/rpython/translator/platform/windows.py
+++ b/rpython/translator/platform/windows.py
@@ -203,7 +203,10 @@
         # the assembler still has the old behavior that all options
         # must come first, and after the file name all options are ignored.
         # So please be careful with the order of parameters! ;-)
-        args = ['/nologo', '/c'] + compile_args + ['/Fo%s' % (oname,), str(cfile)]
+        pdb_dir = oname.dirname
+	if pdb_dir:
+	        compile_args += ['/Fd%s\\' % (pdb_dir,)]
+	args = ['/nologo', '/c'] + compile_args + ['/Fo%s' % (oname,), str(cfile)]
         self._execute_c_compiler(cc, args, oname)
         return oname
 
diff --git a/rpython/translator/sandbox/rsandbox.py b/rpython/translator/sandbox/rsandbox.py
--- a/rpython/translator/sandbox/rsandbox.py
+++ b/rpython/translator/sandbox/rsandbox.py
@@ -60,8 +60,7 @@
 
     def need_more_data(self):
         buflen = self.buflen
-        buf = lltype.malloc(rffi.CCHARP.TO, buflen, flavor='raw')
-        try:
+        with lltype.scoped_alloc(rffi.CCHARP.TO, buflen) as buf:
             buflen = rffi.cast(rffi.SIZE_T, buflen)
             count = ll_read_not_sandboxed(self.fd, buf, buflen)
             count = rffi.cast(lltype.Signed, count)
@@ -69,20 +68,15 @@
                 raise IOError
             self.buf += ''.join([buf[i] for i in range(count)])
             self.buflen *= 2
-        finally:
-            lltype.free(buf, flavor='raw')
 
 def sandboxed_io(buf):
     STDIN = 0
     STDOUT = 1
     # send the buffer with the marshalled fnname and input arguments to STDOUT
-    p = lltype.malloc(rffi.CCHARP.TO, len(buf), flavor='raw')
-    try:
+    with lltype.scoped_alloc(rffi.CCHARP.TO, len(buf)) as p:
         for i in range(len(buf)):
             p[i] = buf[i]
         writeall_not_sandboxed(STDOUT, p, len(buf))
-    finally:
-        lltype.free(p, flavor='raw')
     # build a Loader that will get the answer from STDIN
     loader = FdLoader(STDIN)
     # check for errors
@@ -108,9 +102,8 @@
 @signature(types.str(), returns=types.impossible())
 def not_implemented_stub(msg):
     STDERR = 2
-    buf = rffi.str2charp(msg + '\n')
-    writeall_not_sandboxed(STDERR, buf, len(msg) + 1)
-    rffi.free_charp(buf)
+    with rffi.scoped_str2charp(msg + '\n') as buf:
+        writeall_not_sandboxed(STDERR, buf, len(msg) + 1)
     raise RuntimeError(msg)  # XXX in RPython, the msg is ignored at the moment
 
 dump_string = rmarshal.get_marshaller(str)

From noreply at buildbot.pypy.org  Sun Sep 14 04:23:38 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Sun, 14 Sep 2014 04:23:38 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: issue1844: guard against invalid code
	object argcounts
Message-ID: <20140914022338.2A5D51C3233@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey 
Branch: py3k
Changeset: r73530:013ccdd884c9
Date: 2014-09-13 19:22 -0700
http://bitbucket.org/pypy/pypy/changeset/013ccdd884c9/

Log:	issue1844: guard against invalid code object argcounts

diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -54,7 +54,10 @@
     assert argcount >= 0     # annotator hint
     assert kwonlyargcount >= 0
     argnames = list(varnames[:argcount])
-    kwonlyargs = list(varnames[argcount:argcount + kwonlyargcount])
+    if argcount < len(varnames):
+        kwonlyargs = list(varnames[argcount:argcount + kwonlyargcount])
+    else:
+        kwonlyargs = None
     if code.co_flags & CO_VARARGS:
         varargname = varnames[argcount]
         argcount += 1
diff --git a/pypy/interpreter/test/test_code.py b/pypy/interpreter/test/test_code.py
--- a/pypy/interpreter/test/test_code.py
+++ b/pypy/interpreter/test/test_code.py
@@ -194,3 +194,9 @@
         # CO_NESTED
         assert d['f'](4).__code__.co_flags & 0x10
         assert d['f'].__code__.co_flags & 0x10 == 0
+
+    def test_issue1844(self):
+        import types
+        args = (1, 0, 1, 0, 0, b'', (), (), (), '', 'operator', 0, b'')
+        # previously raised a MemoryError when translated
+        types.CodeType(*args)

From noreply at buildbot.pypy.org  Sun Sep 14 05:31:17 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 14 Sep 2014 05:31:17 +0200 (CEST)
Subject: [pypy-commit] pypy default: untabify
Message-ID: <20140914033117.2D8551C35E3@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: 
Changeset: r73531:3fb1090bc6e1
Date: 2014-09-14 06:30 +0300
http://bitbucket.org/pypy/pypy/changeset/3fb1090bc6e1/

Log:	untabify

diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py
--- a/rpython/translator/platform/windows.py
+++ b/rpython/translator/platform/windows.py
@@ -204,9 +204,9 @@
         # must come first, and after the file name all options are ignored.
         # So please be careful with the order of parameters! ;-)
         pdb_dir = oname.dirname
-	if pdb_dir:
-	        compile_args += ['/Fd%s\\' % (pdb_dir,)]
-	args = ['/nologo', '/c'] + compile_args + ['/Fo%s' % (oname,), str(cfile)]
+        if pdb_dir:
+                compile_args += ['/Fd%s\\' % (pdb_dir,)]
+        args = ['/nologo', '/c'] + compile_args + ['/Fo%s' % (oname,), str(cfile)]
         self._execute_c_compiler(cc, args, oname)
         return oname
 

From noreply at buildbot.pypy.org  Sun Sep 14 20:22:14 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 14 Sep 2014 20:22:14 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: work a bit on the proposal
Message-ID: <20140914182214.B65841D2970@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: extradoc
Changeset: r5400:3f6667e580dc
Date: 2014-09-14 12:21 -0600
http://bitbucket.org/pypy/extradoc/changeset/3f6667e580dc/

Log:	work a bit on the proposal

diff --git a/talk/pycon2015/status/abstract.rst b/talk/pycon2015/status/abstract.rst
--- a/talk/pycon2015/status/abstract.rst
+++ b/talk/pycon2015/status/abstract.rst
@@ -1,5 +1,19 @@
+Pycon 2015 proposal for PyPy status
+
+Title
+-----
+
 PyPy - the last 2 years of progress
-===================================
+
+Category
+--------
+
+Implementations, Virtual Machines
+
+Duration
+--------
+
+30min
 
 Description (one paragraph, 400 chars max)
 ------------------------------------------
@@ -9,6 +23,24 @@
 yielding interesting performance improvements and a timeframe in which more
 libraries started being compatible with PyPy through the use of cffi.
 
+Audience
+--------
+
+Regular Python programmers interested in performance,
+looking to learn more about PyPy. No virtual machine knowledge necessary
+
+Python level
+------------
+
+Medium
+
+Objectives
+----------
+
+We want to explain what we have done during the two years between our talks
+at Pycon and how we expanded PyPy community and the ability to use PyPy
+in various contexts.
+
 Detailed Abstract
 -----------------
 
@@ -29,4 +61,14 @@
 experienced. Unfortunately the biggest chunk of PyPy clients are very
 secretive (e.g. trading companies), but we can still present a few case studies.
 
-XXXX
+Outline
+-------
+
+It's really hard to come up with the outline given the time to Pycon. We'll
+definitely talk about cffi and the community, but the exact extend
+will be decided later.
+
+Additional notes
+----------------
+
+Long-time PyPy speaker at numerous conferences (PyCon, EuroPython, etc.)

From noreply at buildbot.pypy.org  Sun Sep 14 20:41:52 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 14 Sep 2014 20:41:52 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: expand
Message-ID: <20140914184152.0ED8C1C04FA@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: extradoc
Changeset: r5401:359443316deb
Date: 2014-09-14 12:41 -0600
http://bitbucket.org/pypy/extradoc/changeset/359443316deb/

Log:	expand

diff --git a/talk/pycon2015/status/abstract.rst b/talk/pycon2015/status/abstract.rst
--- a/talk/pycon2015/status/abstract.rst
+++ b/talk/pycon2015/status/abstract.rst
@@ -72,3 +72,10 @@
 ----------------
 
 Long-time PyPy speaker at numerous conferences (PyCon, EuroPython, etc.)
+
+While this is "state of the project" talk, which is explicitely mentioned as
+"don't" in the talk guildelines, the PyPy talk at pycon has been
+always very well attended. We also believe that enough has happened in the past
+two years to move PyPy from a mostly interesting project to something
+that people are actually using in production in various interesting
+deployments.

From noreply at buildbot.pypy.org  Sun Sep 14 23:40:36 2014
From: noreply at buildbot.pypy.org (mjacob)
Date: Sun, 14 Sep 2014 23:40:36 +0200 (CEST)
Subject: [pypy-commit] pypy improve-docs: hg merge default
Message-ID: <20140914214036.7ABEB1C04FA@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: improve-docs
Changeset: r73532:2d9e0893a2ec
Date: 2014-09-14 23:36 +0200
http://bitbucket.org/pypy/pypy/changeset/2d9e0893a2ec/

Log:	hg merge default

diff too long, truncating to 2000 out of 65600 lines

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -35,280 +35,290 @@
 the beginning of each file) the files in the 'pypy' directory are each
 copyrighted by one or more of the following people and organizations:    
 
-    Armin Rigo
-    Maciej Fijalkowski
-    Carl Friedrich Bolz
-    Antonio Cuni
-    Amaury Forgeot d'Arc
-    Samuele Pedroni
-    Alex Gaynor
-    Michael Hudson
-    David Schneider
-    Matti Picus
-    Brian Kearns
-    Philip Jenvey
-    Holger Krekel
-    Christian Tismer
-    Hakan Ardo
-    Benjamin Peterson
-    Manuel Jacob
-    Anders Chrigstrom
-    Eric van Riet Paap
-    Wim Lavrijsen
-    Ronan Lamy
-    Richard Emslie
-    Alexander Schremmer
-    Dan Villiom Podlaski Christiansen
-    Lukas Diekmann
-    Sven Hager
-    Anders Lehmann
-    Aurelien Campeas
-    Niklaus Haldimann
-    Camillo Bruni
-    Laura Creighton
-    Toon Verwaest
-    Remi Meier
-    Leonardo Santagada
-    Seo Sanghyeon
-    Romain Guillebert
-    Justin Peel
-    Ronny Pfannschmidt
-    David Edelsohn
-    Anders Hammarquist
-    Jakub Gustak
-    Guido Wesdorp
-    Lawrence Oluyede
-    Bartosz Skowron
-    Daniel Roberts
-    Niko Matsakis
-    Adrien Di Mascio
-    Alexander Hesse
-    Ludovic Aubry
-    Jacob Hallen
-    Jason Creighton
-    Alex Martelli
-    Michal Bendowski
-    Jan de Mooij
-    stian
-    Michael Foord
-    Stephan Diehl
-    Stefan Schwarzer
-    Valentino Volonghi
-    Tomek Meka
-    Patrick Maupin
-    Bob Ippolito
-    Bruno Gola
-    Jean-Paul Calderone
-    Timo Paulssen
-    Squeaky
-    Alexandre Fayolle
-    Simon Burton
-    Marius Gedminas
-    John Witulski
-    Konstantin Lopuhin
-    Greg Price
-    Dario Bertini
-    Mark Pearse
-    Simon Cross
-    Andreas Stührk
-    Jean-Philippe St. Pierre
-    Guido van Rossum
-    Pavel Vinogradov
-    Paweł Piotr Przeradowski
-    Paul deGrandis
-    Ilya Osadchiy
-    Tobias Oberstein
-    Adrian Kuhn
-    Boris Feigin
-    Stefano Rivera
-    tav
-    Taavi Burns
-    Georg Brandl
-    Bert Freudenberg
-    Stian Andreassen
-    Laurence Tratt
-    Wanja Saatkamp
-    Ivan Sichmann Freitas
-    Gerald Klix
-    Mike Blume
-    Oscar Nierstrasz
-    Stefan H. Muller
-    Jeremy Thurgood
-    Gregor Wegberg
-    Rami Chowdhury
-    Tobias Pape
-    Edd Barrett
-    David Malcolm
-    Eugene Oden
-    Henry Mason
-    Preston Timmons
-    Jeff Terrace
-    David Ripton
-    Dusty Phillips
-    Lukas Renggli
-    Guenter Jantzen
-    Ned Batchelder
-    Amit Regmi
-    Ben Young
-    Nicolas Chauvat
-    Andrew Durdin
-    Andrew Chambers
-    Michael Schneider
-    Nicholas Riley
-    Jason Chu
-    Igor Trindade Oliveira
-    Rocco Moretti
-    Gintautas Miliauskas
-    Michael Twomey
-    Lucian Branescu Mihaila
-    Tim Felgentreff
-    Tyler Wade
-    Gabriel Lavoie
-    Olivier Dormond
-    Jared Grubb
-    Karl Bartel
-    Brian Dorsey
-    Victor Stinner
-    Andrews Medina
-    Stuart Williams
-    Jasper Schulz
-    Christian Hudon
-    Toby Watson
-    Antoine Pitrou
-    Aaron Iles
-    Michael Cheng
-    Justas Sadzevicius
-    Mikael Schönenberg
-    Gasper Zejn
-    Neil Shepperd
-    Elmo Mäntynen
-    Jonathan David Riehl
-    Stanislaw Halik
-    Anders Qvist
-    Chirag Jadwani
-    Beatrice During
-    Alex Perry
-    Vincent Legoll
-    Alan McIntyre
-    Alexander Sedov
-    Corbin Simpson
-    Christopher Pope
-    wenzhuman
-    Christian Tismer 
-    Marc Abramowitz
-    Dan Stromberg
-    Stefano Parmesan
-    Alexis Daboville
-    Jens-Uwe Mager
-    Carl Meyer
-    Karl Ramm
-    Pieter Zieschang
-    Gabriel
-    Lukas Vacek
-    Andrew Dalke
-    Sylvain Thenault
-    Nathan Taylor
-    Vladimir Kryachko
-    Jacek Generowicz
-    Alejandro J. Cura
-    Jacob Oscarson
-    Travis Francis Athougies
-    Ryan Gonzalez
-    Kristjan Valur Jonsson
-    Sebastian Pawluś
-    Neil Blakey-Milner
-    anatoly techtonik
-    Lutz Paelike
-    Lucio Torre
-    Lars Wassermann
-    Henrik Vendelbo
-    Dan Buch
-    Miguel de Val Borro
-    Artur Lisiecki
-    Sergey Kishchenko
-    Ignas Mikalajunas
-    Christoph Gerum
-    Martin Blais
-    Lene Wagner
-    Tomo Cocoa
-    roberto at goyle
-    Yury V. Zaytsev
-    Anna Katrina Dominguez
-    William Leslie
-    Bobby Impollonia
-    timo at eistee.fritz.box
-    Andrew Thompson
-    Ben Darnell
-    Roberto De Ioris
-    Juan Francisco Cantero Hurtado
-    Godefroid Chappelle
-    Joshua Gilbert
-    Dan Colish
-    Christopher Armstrong
-    Michael Hudson-Doyle
-    Anders Sigfridsson
-    Yasir Suhail
-    rafalgalczynski at gmail.com
-    Floris Bruynooghe
-    Laurens Van Houtven
-    Akira Li
-    Gustavo Niemeyer
-    Stephan Busemann
-    Rafał Gałczyński
-    Yusei Tahara
-    Christian Muirhead
-    James Lan
-    shoma hosaka
-    Daniel Neuh?user
-    Matthew Miller
-    Buck Golemon
-    Konrad Delong
-    Dinu Gherman
-    Chris Lambacher
-    coolbutuseless at gmail.com
-    Rodrigo Araújo
-    w31rd0
-    Jim Baker
-    James Robert
-    Armin Ronacher
-    Brett Cannon
-    yrttyr
-    aliceinwire
-    OlivierBlanvillain
-    Zooko Wilcox-O Hearn
-    Tomer Chachamu
-    Christopher Groskopf
-    Asmo Soinio
-    Stefan Marr
-    jiaaro
-    opassembler.py
-    Antony Lee
-    Jim Hunziker
-    Markus Unterwaditzer
-    Even Wiik Thomassen
-    jbs
-    soareschen
-    Kurt Griffiths
-    Mike Bayer
-    Flavio Percoco
-    Kristoffer Kleine
-    yasirs
-    Michael Chermside
-    Anna Ravencroft
-    Julien Phalip
-    Dan Loewenherz
+  Armin Rigo
+  Maciej Fijalkowski
+  Carl Friedrich Bolz
+  Antonio Cuni
+  Amaury Forgeot d'Arc
+  Samuele Pedroni
+  Alex Gaynor
+  Michael Hudson
+  David Schneider
+  Matti Picus
+  Brian Kearns
+  Philip Jenvey
+  Holger Krekel
+  Christian Tismer
+  Hakan Ardo
+  Benjamin Peterson
+  Manuel Jacob
+  Anders Chrigstrom
+  Eric van Riet Paap
+  Ronan Lamy
+  Wim Lavrijsen
+  Richard Emslie
+  Alexander Schremmer
+  Dan Villiom Podlaski Christiansen
+  Lukas Diekmann
+  Sven Hager
+  Anders Lehmann
+  Aurelien Campeas
+  Niklaus Haldimann
+  Remi Meier
+  Camillo Bruni
+  Laura Creighton
+  Toon Verwaest
+  Leonardo Santagada
+  Seo Sanghyeon
+  Romain Guillebert
+  Justin Peel
+  Ronny Pfannschmidt
+  David Edelsohn
+  Anders Hammarquist
+  Jakub Gustak
+  Guido Wesdorp
+  Lawrence Oluyede
+  Bartosz Skowron
+  Gregor Wegberg
+  Daniel Roberts
+  Niko Matsakis
+  Adrien Di Mascio
+  Alexander Hesse
+  Ludovic Aubry
+  Jacob Hallen
+  Jason Creighton
+  Alex Martelli
+  Michal Bendowski
+  Jan de Mooij
+  stian
+  Michael Foord
+  Stephan Diehl
+  Tyler Wade
+  Stefan Schwarzer
+  Valentino Volonghi
+  Tomek Meka
+  Patrick Maupin
+  Bob Ippolito
+  Bruno Gola
+  Jean-Paul Calderone
+  Timo Paulssen
+  Squeaky
+  Alexandre Fayolle
+  Simon Burton
+  Marius Gedminas
+  Martin Matusiak
+  Konstantin Lopuhin
+  John Witulski
+  Wenzhu Man
+  Greg Price
+  Dario Bertini
+  Mark Pearse
+  Simon Cross
+  Ivan Sichmann Freitas
+  Andreas Stührk
+  Jean-Philippe St. Pierre
+  Guido van Rossum
+  Pavel Vinogradov
+  Stefano Rivera
+  Paweł Piotr Przeradowski
+  Paul deGrandis
+  Ilya Osadchiy
+  Tobias Oberstein
+  Adrian Kuhn
+  Boris Feigin
+  tav
+  Taavi Burns
+  Georg Brandl
+  Laurence Tratt
+  Bert Freudenberg
+  Stian Andreassen
+  Wanja Saatkamp
+  Gerald Klix
+  Mike Blume
+  Oscar Nierstrasz
+  Stefan H. Muller
+  Edd Barrett
+  Jeremy Thurgood
+  Rami Chowdhury
+  Tobias Pape
+  David Malcolm
+  Eugene Oden
+  Henry Mason
+  Vasily Kuznetsov
+  Preston Timmons
+  Jeff Terrace
+  David Ripton
+  Dusty Phillips
+  Lukas Renggli
+  Guenter Jantzen
+  Ned Batchelder
+  Amit Regmi
+  Ben Young
+  Nicolas Chauvat
+  Andrew Durdin
+  Andrew Chambers
+  Michael Schneider
+  Nicholas Riley
+  Jason Chu
+  Igor Trindade Oliveira
+  Tim Felgentreff
+  Rocco Moretti
+  Gintautas Miliauskas
+  Michael Twomey
+  Lucian Branescu Mihaila
+  Gabriel Lavoie
+  Olivier Dormond
+  Jared Grubb
+  Karl Bartel
+  Brian Dorsey
+  Victor Stinner
+  Andrews Medina
+  Stuart Williams
+  Jasper Schulz
+  Christian Hudon
+  Toby Watson
+  Antoine Pitrou
+  Aaron Iles
+  Michael Cheng
+  Justas Sadzevicius
+  Gasper Zejn
+  anatoly techtonik
+  Neil Shepperd
+  Mikael Schönenberg
+  Elmo M?ntynen
+  Jonathan David Riehl
+  Stanislaw Halik
+  Anders Qvist
+  Corbin Simpson
+  Chirag Jadwani
+  Beatrice During
+  Alex Perry
+  Vincent Legoll
+  Alan McIntyre
+  Alexander Sedov
+  Christopher Pope
+  Christian Tismer 
+  Marc Abramowitz
+  Dan Stromberg
+  Stefano Parmesan
+  Alexis Daboville
+  Jens-Uwe Mager
+  Carl Meyer
+  Karl Ramm
+  Pieter Zieschang
+  Sebastian Pawluś
+  Gabriel
+  Lukas Vacek
+  Andrew Dalke
+  Sylvain Thenault
+  Nathan Taylor
+  Vladimir Kryachko
+  Arjun Naik
+  Attila Gobi
+  Jacek Generowicz
+  Alejandro J. Cura
+  Jacob Oscarson
+  Travis Francis Athougies
+  Ryan Gonzalez
+  Ian Foote
+  Kristjan Valur Jonsson
+  Neil Blakey-Milner
+  Lutz Paelike
+  Lucio Torre
+  Lars Wassermann
+  Valentina Mukhamedzhanova
+  Henrik Vendelbo
+  Dan Buch
+  Miguel de Val Borro
+  Artur Lisiecki
+  Sergey Kishchenko
+  Yichao Yu
+  Ignas Mikalajunas
+  Christoph Gerum
+  Martin Blais
+  Lene Wagner
+  Tomo Cocoa
+  roberto at goyle
+  Yury V. Zaytsev
+  Anna Katrina Dominguez
+  William Leslie
+  Bobby Impollonia
+  timo at eistee.fritz.box
+  Andrew Thompson
+  Yusei Tahara
+  Ben Darnell
+  Roberto De Ioris
+  Juan Francisco Cantero Hurtado
+  Godefroid Chappelle
+  Joshua Gilbert
+  Dan Colish
+  Christopher Armstrong
+  Michael Hudson-Doyle
+  Anders Sigfridsson
+  Yasir Suhail
+  Jason Michalski
+  rafalgalczynski at gmail.com
+  Floris Bruynooghe
+  Laurens Van Houtven
+  Akira Li
+  Gustavo Niemeyer
+  Stephan Busemann
+  Rafał Gałczyński
+  Christian Muirhead
+  James Lan
+  shoma hosaka
+  Daniel Neuh?user
+  Matthew Miller
+  Buck Golemon
+  Konrad Delong
+  Dinu Gherman
+  Chris Lambacher
+  coolbutuseless at gmail.com
+  Rodrigo Araújo
+  Jim Baker
+  James Robert
+  Armin Ronacher
+  Brett Cannon
+  yrttyr
+  aliceinwire
+  OlivierBlanvillain
+  Zooko Wilcox-O Hearn
+  Tomer Chachamu
+  Christopher Groskopf
+  Asmo Soinio
+  Stefan Marr
+  jiaaro
+  Mads Kiilerich
+  opassembler.py
+  Antony Lee
+  Jim Hunziker
+  Markus Unterwaditzer
+  Even Wiik Thomassen
+  jbs
+  soareschen
+  Kurt Griffiths
+  Mike Bayer
+  Matthew Miller
+  Flavio Percoco
+  Kristoffer Kleine
+  yasirs
+  Michael Chermside
+  Anna Ravencroft
+  Dan Crosta
+  Julien Phalip
+  Dan Loewenherz
 
-    Heinrich-Heine University, Germany 
-    Open End AB (formerly AB Strakt), Sweden
-    merlinux GmbH, Germany 
-    tismerysoft GmbH, Germany 
-    Logilab Paris, France 
-    DFKI GmbH, Germany 
-    Impara, Germany
-    Change Maker, Sweden 
-    University of California Berkeley, USA
-    Google Inc.
-    King's College London
+  Heinrich-Heine University, Germany 
+  Open End AB (formerly AB Strakt), Sweden
+  merlinux GmbH, Germany 
+  tismerysoft GmbH, Germany 
+  Logilab Paris, France 
+  DFKI GmbH, Germany 
+  Impara, Germany
+  Change Maker, Sweden 
+  University of California Berkeley, USA
+  Google Inc.
+  King's College London
 
 The PyPy Logo as used by http://speed.pypy.org and others was created
 by Samuel Reis and is distributed on terms of Creative Commons Share Alike
@@ -354,6 +364,6 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 
-Detailled license information is contained in the NOTICE file in the
+Detailed license information is contained in the NOTICE file in the
 directory.
 
diff --git a/_pytest/README-BEFORE-UPDATING b/_pytest/README-BEFORE-UPDATING
new file mode 100644
--- /dev/null
+++ b/_pytest/README-BEFORE-UPDATING
@@ -0,0 +1,17 @@
+This is PyPy's code of the pytest lib.  We don't expect to upgrade it
+very often, but once we do:
+
+    WARNING!
+
+    WE HAVE MADE A FEW TWEAKS HERE!
+
+Please be sure that you don't just copy the newer version from
+upstream without checking the few changes that we did.  This
+can be done like this:
+
+    cd 
+    hg log . -v | less
+
+then search for all " _pytest/" in that list to know which are the
+relevant checkins.  (Look for the checkins that only edit one
+or two files in this directory.)
diff --git a/_pytest/__init__.py b/_pytest/__init__.py
--- a/_pytest/__init__.py
+++ b/_pytest/__init__.py
@@ -1,2 +1,2 @@
 #
-__version__ = '2.2.4.dev2'
+__version__ = '2.5.2'
diff --git a/_pytest/_argcomplete.py b/_pytest/_argcomplete.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_argcomplete.py
@@ -0,0 +1,104 @@
+
+"""allow bash-completion for argparse with argcomplete if installed
+needs argcomplete>=0.5.6 for python 3.2/3.3 (older versions fail
+to find the magic string, so _ARGCOMPLETE env. var is never set, and
+this does not need special code.
+
+argcomplete does not support python 2.5 (although the changes for that
+are minor).
+
+Function try_argcomplete(parser) should be called directly before
+the call to ArgumentParser.parse_args().
+
+The filescompleter is what you normally would use on the positional
+arguments specification, in order to get "dirname/" after "dirn"
+instead of the default "dirname ":
+
+   optparser.add_argument(Config._file_or_dir, nargs='*'
+                               ).completer=filescompleter
+
+Other, application specific, completers should go in the file
+doing the add_argument calls as they need to be specified as .completer
+attributes as well. (If argcomplete is not installed, the function the
+attribute points to will not be used).
+
+SPEEDUP
+=======
+The generic argcomplete script for bash-completion
+(/etc/bash_completion.d/python-argcomplete.sh )
+uses a python program to determine startup script generated by pip.
+You can speed up completion somewhat by changing this script to include
+  # PYTHON_ARGCOMPLETE_OK
+so the the python-argcomplete-check-easy-install-script does not
+need to be called to find the entry point of the code and see if that is
+marked  with PYTHON_ARGCOMPLETE_OK
+
+INSTALL/DEBUGGING
+=================
+To include this support in another application that has setup.py generated
+scripts:
+- add the line:
+    # PYTHON_ARGCOMPLETE_OK
+  near the top of the main python entry point
+- include in the file calling parse_args():
+    from _argcomplete import try_argcomplete, filescompleter
+   , call try_argcomplete just before parse_args(), and optionally add
+   filescompleter to the positional arguments' add_argument()
+If things do not work right away:
+- switch on argcomplete debugging with (also helpful when doing custom
+  completers):
+    export _ARC_DEBUG=1
+- run:
+    python-argcomplete-check-easy-install-script $(which appname)
+    echo $?
+  will echo 0 if the magic line has been found, 1 if not
+- sometimes it helps to find early on errors using:
+    _ARGCOMPLETE=1 _ARC_DEBUG=1 appname
+  which should throw a KeyError: 'COMPLINE' (which is properly set by the
+  global argcomplete script).
+"""
+
+import sys
+import os
+from glob import glob
+
+class FastFilesCompleter:
+    'Fast file completer class'
+    def __init__(self, directories=True):
+        self.directories = directories
+
+    def __call__(self, prefix, **kwargs):
+        """only called on non option completions"""
+        if os.path.sep in prefix[1:]: #
+            prefix_dir = len(os.path.dirname(prefix) + os.path.sep)
+        else:
+            prefix_dir = 0
+        completion = []
+        globbed = []
+        if '*' not in prefix and '?' not in prefix:
+            if prefix[-1] == os.path.sep:  # we are on unix, otherwise no bash
+                globbed.extend(glob(prefix + '.*'))
+            prefix += '*'
+        globbed.extend(glob(prefix))
+        for x in sorted(globbed):
+            if os.path.isdir(x):
+                x += '/'
+            # append stripping the prefix (like bash, not like compgen)
+            completion.append(x[prefix_dir:])
+        return completion
+
+if os.environ.get('_ARGCOMPLETE'):
+    # argcomplete 0.5.6 is not compatible with python 2.5.6: print/with/format
+    if sys.version_info[:2] < (2, 6):
+        sys.exit(1)
+    try:
+        import argcomplete.completers
+    except ImportError:
+        sys.exit(-1)
+    filescompleter = FastFilesCompleter()
+
+    def try_argcomplete(parser):
+        argcomplete.autocomplete(parser)
+else:
+    def try_argcomplete(parser): pass
+    filescompleter = None
diff --git a/_pytest/assertion/__init__.py b/_pytest/assertion/__init__.py
--- a/_pytest/assertion/__init__.py
+++ b/_pytest/assertion/__init__.py
@@ -3,7 +3,6 @@
 """
 import py
 import sys
-import pytest
 from _pytest.monkeypatch import monkeypatch
 from _pytest.assertion import util
 
@@ -19,8 +18,8 @@
 to provide assert expression information. """)
     group.addoption('--no-assert', action="store_true", default=False,
         dest="noassert", help="DEPRECATED equivalent to --assert=plain")
-    group.addoption('--nomagic', action="store_true", default=False,
-        dest="nomagic", help="DEPRECATED equivalent to --assert=plain")
+    group.addoption('--nomagic', '--no-magic', action="store_true",
+        default=False, help="DEPRECATED equivalent to --assert=plain")
 
 class AssertionState:
     """State for the assertion plugin."""
@@ -35,22 +34,25 @@
         mode = "plain"
     if mode == "rewrite":
         try:
-            import ast
+            import ast  # noqa
         except ImportError:
             mode = "reinterp"
         else:
-            if sys.platform.startswith('java'):
+            # Both Jython and CPython 2.6.0 have AST bugs that make the
+            # assertion rewriting hook malfunction.
+            if (sys.platform.startswith('java') or
+                sys.version_info[:3] == (2, 6, 0)):
                 mode = "reinterp"
     if mode != "plain":
         _load_modules(mode)
         m = monkeypatch()
         config._cleanup.append(m.undo)
         m.setattr(py.builtin.builtins, 'AssertionError',
-                  reinterpret.AssertionError)
+                  reinterpret.AssertionError)  # noqa
     hook = None
     if mode == "rewrite":
-        hook = rewrite.AssertionRewritingHook()
-        sys.meta_path.append(hook)
+        hook = rewrite.AssertionRewritingHook()  # noqa
+        sys.meta_path.insert(0, hook)
     warn_about_missing_assertion(mode)
     config._assertstate = AssertionState(config, mode)
     config._assertstate.hook = hook
@@ -73,9 +75,16 @@
     def callbinrepr(op, left, right):
         hook_result = item.ihook.pytest_assertrepr_compare(
             config=item.config, op=op, left=left, right=right)
+
         for new_expl in hook_result:
             if new_expl:
-                res = '\n~'.join(new_expl)
+                # Don't include pageloads of data unless we are very
+                # verbose (-vv)
+                if (sum(len(p) for p in new_expl[1:]) > 80*8
+                        and item.config.option.verbose < 2):
+                    new_expl[1:] = [py.builtin._totext(
+                        'Detailed information truncated, use "-vv" to show')]
+                res = py.builtin._totext('\n~').join(new_expl)
                 if item.config.getvalue("assertmode") == "rewrite":
                     # The result will be fed back a python % formatting
                     # operation, which will fail if there are extraneous
@@ -95,9 +104,9 @@
 def _load_modules(mode):
     """Lazily import assertion related code."""
     global rewrite, reinterpret
-    from _pytest.assertion import reinterpret
+    from _pytest.assertion import reinterpret  # noqa
     if mode == "rewrite":
-        from _pytest.assertion import rewrite
+        from _pytest.assertion import rewrite  # noqa
 
 def warn_about_missing_assertion(mode):
     try:
diff --git a/_pytest/assertion/newinterpret.py b/_pytest/assertion/newinterpret.py
--- a/_pytest/assertion/newinterpret.py
+++ b/_pytest/assertion/newinterpret.py
@@ -11,7 +11,7 @@
 from _pytest.assertion.reinterpret import BuiltinAssertionError
 
 
-if sys.platform.startswith("java") and sys.version_info < (2, 5, 2):
+if sys.platform.startswith("java"):
     # See http://bugs.jython.org/issue1497
     _exprs = ("BoolOp", "BinOp", "UnaryOp", "Lambda", "IfExp", "Dict",
               "ListComp", "GeneratorExp", "Yield", "Compare", "Call",
diff --git a/_pytest/assertion/oldinterpret.py b/_pytest/assertion/oldinterpret.py
--- a/_pytest/assertion/oldinterpret.py
+++ b/_pytest/assertion/oldinterpret.py
@@ -526,10 +526,13 @@
     # example:
     def f():
         return 5
+
     def g():
         return 3
+
     def h(x):
         return 'never'
+
     check("f() * g() == 5")
     check("not f()")
     check("not (f() and g() or 0)")
diff --git a/_pytest/assertion/reinterpret.py b/_pytest/assertion/reinterpret.py
--- a/_pytest/assertion/reinterpret.py
+++ b/_pytest/assertion/reinterpret.py
@@ -1,18 +1,26 @@
 import sys
 import py
 from _pytest.assertion.util import BuiltinAssertionError
+u = py.builtin._totext
+
 
 class AssertionError(BuiltinAssertionError):
     def __init__(self, *args):
         BuiltinAssertionError.__init__(self, *args)
         if args:
+            # on Python2.6 we get len(args)==2 for: assert 0, (x,y)
+            # on Python2.7 and above we always get len(args) == 1
+            # with args[0] being the (x,y) tuple.
+            if len(args) > 1:
+                toprint = args
+            else:
+                toprint = args[0]
             try:
-                self.msg = str(args[0])
-            except py.builtin._sysex:
-                raise
-            except:
-                self.msg = "<[broken __repr__] %s at %0xd>" %(
-                    args[0].__class__, id(args[0]))
+                self.msg = u(toprint)
+            except Exception:
+                self.msg = u(
+                    "<[broken __repr__] %s at %0xd>"
+                    % (toprint.__class__, id(toprint)))
         else:
             f = py.code.Frame(sys._getframe(1))
             try:
@@ -44,4 +52,3 @@
     from _pytest.assertion.newinterpret import interpret as reinterpret
 else:
     reinterpret = reinterpret_old
-
diff --git a/_pytest/assertion/rewrite.py b/_pytest/assertion/rewrite.py
--- a/_pytest/assertion/rewrite.py
+++ b/_pytest/assertion/rewrite.py
@@ -6,6 +6,7 @@
 import imp
 import marshal
 import os
+import re
 import struct
 import sys
 import types
@@ -14,13 +15,7 @@
 from _pytest.assertion import util
 
 
-# Windows gives ENOENT in places *nix gives ENOTDIR.
-if sys.platform.startswith("win"):
-    PATH_COMPONENT_NOT_DIR = errno.ENOENT
-else:
-    PATH_COMPONENT_NOT_DIR = errno.ENOTDIR
-
-# py.test caches rewritten pycs in __pycache__.
+# pytest caches rewritten pycs in __pycache__.
 if hasattr(imp, "get_tag"):
     PYTEST_TAG = imp.get_tag() + "-PYTEST"
 else:
@@ -34,17 +29,19 @@
     PYTEST_TAG = "%s-%s%s-PYTEST" % (impl, ver[0], ver[1])
     del ver, impl
 
-PYC_EXT = ".py" + "c" if __debug__ else "o"
+PYC_EXT = ".py" + (__debug__ and "c" or "o")
 PYC_TAIL = "." + PYTEST_TAG + PYC_EXT
 
 REWRITE_NEWLINES = sys.version_info[:2] != (2, 7) and sys.version_info < (3, 2)
+ASCII_IS_DEFAULT_ENCODING = sys.version_info[0] < 3
 
 class AssertionRewritingHook(object):
-    """Import hook which rewrites asserts."""
+    """PEP302 Import hook which rewrites asserts."""
 
     def __init__(self):
         self.session = None
         self.modules = {}
+        self._register_with_pkg_resources()
 
     def set_session(self, session):
         self.fnpats = session.config.getini("python_files")
@@ -59,8 +56,12 @@
         names = name.rsplit(".", 1)
         lastname = names[-1]
         pth = None
-        if path is not None and len(path) == 1:
-            pth = path[0]
+        if path is not None:
+            # Starting with Python 3.3, path is a _NamespacePath(), which
+            # causes problems if not converted to list.
+            path = list(path)
+            if len(path) == 1:
+                pth = path[0]
         if pth is None:
             try:
                 fd, fn, desc = imp.find_module(lastname, path)
@@ -95,12 +96,13 @@
             finally:
                 self.session = sess
         else:
-            state.trace("matched test file (was specified on cmdline): %r" % (fn,))
+            state.trace("matched test file (was specified on cmdline): %r" %
+                        (fn,))
         # The requested module looks like a test file, so rewrite it. This is
         # the most magical part of the process: load the source, rewrite the
         # asserts, and load the rewritten source. We also cache the rewritten
         # module code in a special pyc. We must be aware of the possibility of
-        # concurrent py.test processes rewriting and loading pycs. To avoid
+        # concurrent pytest processes rewriting and loading pycs. To avoid
         # tricky race conditions, we maintain the following invariant: The
         # cached pyc is always a complete, valid pyc. Operations on it must be
         # atomic. POSIX's atomic rename comes in handy.
@@ -116,19 +118,19 @@
                     # common case) or it's blocked by a non-dir node. In the
                     # latter case, we'll ignore it in _write_pyc.
                     pass
-                elif e == PATH_COMPONENT_NOT_DIR:
+                elif e in [errno.ENOENT, errno.ENOTDIR]:
                     # One of the path components was not a directory, likely
                     # because we're in a zip file.
                     write = False
                 elif e == errno.EACCES:
-                    state.trace("read only directory: %r" % (fn_pypath.dirname,))
+                    state.trace("read only directory: %r" % fn_pypath.dirname)
                     write = False
                 else:
                     raise
         cache_name = fn_pypath.basename[:-3] + PYC_TAIL
         pyc = os.path.join(cache_dir, cache_name)
-        # Notice that even if we're in a read-only directory, I'm going to check
-        # for a cached pyc. This may not be optimal...
+        # Notice that even if we're in a read-only directory, I'm going
+        # to check for a cached pyc. This may not be optimal...
         co = _read_pyc(fn_pypath, pyc)
         if co is None:
             state.trace("rewriting %r" % (fn,))
@@ -153,27 +155,59 @@
             mod.__file__ = co.co_filename
             # Normally, this attribute is 3.2+.
             mod.__cached__ = pyc
+            mod.__loader__ = self
             py.builtin.exec_(co, mod.__dict__)
         except:
             del sys.modules[name]
             raise
         return sys.modules[name]
 
-def _write_pyc(co, source_path, pyc):
-    # Technically, we don't have to have the same pyc format as (C)Python, since
-    # these "pycs" should never be seen by builtin import. However, there's
-    # little reason deviate, and I hope sometime to be able to use
-    # imp.load_compiled to load them. (See the comment in load_module above.)
+
+
+    def is_package(self, name):
+        try:
+            fd, fn, desc = imp.find_module(name)
+        except ImportError:
+            return False
+        if fd is not None:
+            fd.close()
+        tp = desc[2]
+        return tp == imp.PKG_DIRECTORY
+
+    @classmethod
+    def _register_with_pkg_resources(cls):
+        """
+        Ensure package resources can be loaded from this loader. May be called
+        multiple times, as the operation is idempotent.
+        """
+        try:
+            import pkg_resources
+            # access an attribute in case a deferred importer is present
+            pkg_resources.__name__
+        except ImportError:
+            return
+
+        # Since pytest tests are always located in the file system, the
+        #  DefaultProvider is appropriate.
+        pkg_resources.register_loader_type(cls, pkg_resources.DefaultProvider)
+
+
+def _write_pyc(state, co, source_path, pyc):
+    # Technically, we don't have to have the same pyc format as
+    # (C)Python, since these "pycs" should never be seen by builtin
+    # import. However, there's little reason deviate, and I hope
+    # sometime to be able to use imp.load_compiled to load them. (See
+    # the comment in load_module above.)
     mtime = int(source_path.mtime())
     try:
         fp = open(pyc, "wb")
     except IOError:
         err = sys.exc_info()[1].errno
-        if err == PATH_COMPONENT_NOT_DIR:
-            # This happens when we get a EEXIST in find_module creating the
-            # __pycache__ directory and __pycache__ is by some non-dir node.
-            return False
-        raise
+        state.trace("error writing pyc file at %s: errno=%s" %(pyc, err))
+        # we ignore any failure to write the cache file
+        # there are many reasons, permission-denied, __pycache__ being a
+        # file etc.
+        return False
     try:
         fp.write(imp.get_magic())
         fp.write(struct.pack(">",
-    ast.Add : "+",
-    ast.Sub : "-",
-    ast.Mult : "*",
-    ast.Div : "/",
-    ast.FloorDiv : "//",
-    ast.Mod : "%",
-    ast.Eq : "==",
-    ast.NotEq : "!=",
-    ast.Lt : "<",
-    ast.LtE : "<=",
-    ast.Gt : ">",
-    ast.GtE : ">=",
-    ast.Pow : "**",
-    ast.Is : "is",
-    ast.IsNot : "is not",
-    ast.In : "in",
-    ast.NotIn : "not in"
+    ast.BitOr: "|",
+    ast.BitXor: "^",
+    ast.BitAnd: "&",
+    ast.LShift: "<<",
+    ast.RShift: ">>",
+    ast.Add: "+",
+    ast.Sub: "-",
+    ast.Mult: "*",
+    ast.Div: "/",
+    ast.FloorDiv: "//",
+    ast.Mod: "%%", # escaped for string formatting
+    ast.Eq: "==",
+    ast.NotEq: "!=",
+    ast.Lt: "<",
+    ast.LtE: "<=",
+    ast.Gt: ">",
+    ast.GtE: ">=",
+    ast.Pow: "**",
+    ast.Is: "is",
+    ast.IsNot: "is not",
+    ast.In: "in",
+    ast.NotIn: "not in"
 }
 
 
@@ -341,7 +408,7 @@
         lineno = 0
         for item in mod.body:
             if (expect_docstring and isinstance(item, ast.Expr) and
-                isinstance(item.value, ast.Str)):
+                    isinstance(item.value, ast.Str)):
                 doc = item.value.s
                 if "PYTEST_DONT_REWRITE" in doc:
                     # The module has disabled assertion rewriting.
@@ -462,7 +529,8 @@
         body.append(raise_)
         # Clear temporary variables by setting them to None.
         if self.variables:
-            variables = [ast.Name(name, ast.Store()) for name in self.variables]
+            variables = [ast.Name(name, ast.Store())
+                         for name in self.variables]
             clear = ast.Assign(variables, ast.Name("None", ast.Load()))
             self.statements.append(clear)
         # Fix line numbers.
@@ -471,11 +539,12 @@
         return self.statements
 
     def visit_Name(self, name):
-        # Check if the name is local or not.
+        # Display the repr of the name if it's a local variable or
+        # _should_repr_global_name() thinks it's acceptable.
         locs = ast.Call(self.builtin("locals"), [], [], None, None)
-        globs = ast.Call(self.builtin("globals"), [], [], None, None)
-        ops = [ast.In(), ast.IsNot()]
-        test = ast.Compare(ast.Str(name.id), ops, [locs, globs])
+        inlocs = ast.Compare(ast.Str(name.id), [ast.In()], [locs])
+        dorepr = self.helper("should_repr_global_name", name)
+        test = ast.BoolOp(ast.Or(), [inlocs, dorepr])
         expr = ast.IfExp(test, self.display(name), ast.Str(name.id))
         return name, self.explanation_param(expr)
 
@@ -492,7 +561,8 @@
         for i, v in enumerate(boolop.values):
             if i:
                 fail_inner = []
-                self.on_failure.append(ast.If(cond, fail_inner, []))
+                # cond is set in a prior loop iteration below
+                self.on_failure.append(ast.If(cond, fail_inner, [])) # noqa
                 self.on_failure = fail_inner
             self.push_format_context()
             res, expl = self.visit(v)
@@ -548,7 +618,8 @@
             new_kwarg, expl = self.visit(call.kwargs)
             arg_expls.append("**" + expl)
         expl = "%s(%s)" % (func_expl, ', '.join(arg_expls))
-        new_call = ast.Call(new_func, new_args, new_kwargs, new_star, new_kwarg)
+        new_call = ast.Call(new_func, new_args, new_kwargs,
+                            new_star, new_kwarg)
         res = self.assign(new_call)
         res_expl = self.explanation_param(self.display(res))
         outer_expl = "%s\n{%s = %s\n}" % (res_expl, res_expl, expl)
@@ -584,7 +655,7 @@
             res_expr = ast.Compare(left_res, [op], [next_res])
             self.statements.append(ast.Assign([store_names[i]], res_expr))
             left_res, left_expl = next_res, next_expl
-        # Use py.code._reprcompare if that's available.
+        # Use pytest.assertion.util._reprcompare if that's available.
         expl_call = self.helper("call_reprcompare",
                                 ast.Tuple(syms, ast.Load()),
                                 ast.Tuple(load_names, ast.Load()),
diff --git a/_pytest/assertion/util.py b/_pytest/assertion/util.py
--- a/_pytest/assertion/util.py
+++ b/_pytest/assertion/util.py
@@ -1,8 +1,13 @@
 """Utilities for assertion debugging"""
 
 import py
+try:
+    from collections import Sequence
+except ImportError:
+    Sequence = list
 
 BuiltinAssertionError = py.builtin.builtins.AssertionError
+u = py.builtin._totext
 
 # The _reprcompare attribute on the util module is used by the new assertion
 # interpretation code and assertion rewriter to detect this plugin was
@@ -10,6 +15,7 @@
 # DebugInterpreter.
 _reprcompare = None
 
+
 def format_explanation(explanation):
     """This formats an explanation
 
@@ -20,7 +26,18 @@
     for when one explanation needs to span multiple lines, e.g. when
     displaying diffs.
     """
-    # simplify 'assert False where False = ...'
+    explanation = _collapse_false(explanation)
+    lines = _split_explanation(explanation)
+    result = _format_lines(lines)
+    return u('\n').join(result)
+
+
+def _collapse_false(explanation):
+    """Collapse expansions of False
+
+    So this strips out any "assert False\n{where False = ...\n}"
+    blocks.
+    """
     where = 0
     while True:
         start = where = explanation.find("False\n{False = ", where)
@@ -42,28 +59,48 @@
             explanation = (explanation[:start] + explanation[start+15:end-1] +
                            explanation[end+1:])
             where -= 17
-    raw_lines = (explanation or '').split('\n')
-    # escape newlines not followed by {, } and ~
+    return explanation
+
+
+def _split_explanation(explanation):
+    """Return a list of individual lines in the explanation
+
+    This will return a list of lines split on '\n{', '\n}' and '\n~'.
+    Any other newlines will be escaped and appear in the line as the
+    literal '\n' characters.
+    """
+    raw_lines = (explanation or u('')).split('\n')
     lines = [raw_lines[0]]
     for l in raw_lines[1:]:
         if l.startswith('{') or l.startswith('}') or l.startswith('~'):
             lines.append(l)
         else:
             lines[-1] += '\\n' + l
+    return lines
 
+
+def _format_lines(lines):
+    """Format the individual lines
+
+    This will replace the '{', '}' and '~' characters of our mini
+    formatting language with the proper 'where ...', 'and ...' and ' +
+    ...' text, taking care of indentation along the way.
+
+    Return a list of formatted lines.
+    """
     result = lines[:1]
     stack = [0]
     stackcnt = [0]
     for line in lines[1:]:
         if line.startswith('{'):
             if stackcnt[-1]:
-                s = 'and   '
+                s = u('and   ')
             else:
-                s = 'where '
+                s = u('where ')
             stack.append(len(result))
             stackcnt[-1] += 1
             stackcnt.append(0)
-            result.append(' +' + '  '*(len(stack)-1) + s + line[1:])
+            result.append(u(' +') + u('  ')*(len(stack)-1) + s + line[1:])
         elif line.startswith('}'):
             assert line.startswith('}')
             stack.pop()
@@ -71,9 +108,9 @@
             result[stack[-1]] += line[1:]
         else:
             assert line.startswith('~')
-            result.append('  '*len(stack) + line[1:])
+            result.append(u('  ')*len(stack) + line[1:])
     assert len(stack) == 1
-    return '\n'.join(result)
+    return result
 
 
 # Provide basestring in python3
@@ -83,132 +120,163 @@
     basestring = str
 
 
-def assertrepr_compare(op, left, right):
-    """return specialised explanations for some operators/operands"""
-    width = 80 - 15 - len(op) - 2 # 15 chars indentation, 1 space around op
+def assertrepr_compare(config, op, left, right):
+    """Return specialised explanations for some operators/operands"""
+    width = 80 - 15 - len(op) - 2  # 15 chars indentation, 1 space around op
     left_repr = py.io.saferepr(left, maxsize=int(width/2))
     right_repr = py.io.saferepr(right, maxsize=width-len(left_repr))
-    summary = '%s %s %s' % (left_repr, op, right_repr)
+    summary = u('%s %s %s') % (left_repr, op, right_repr)
 
-    issequence = lambda x: isinstance(x, (list, tuple))
+    issequence = lambda x: (isinstance(x, (list, tuple, Sequence))
+                            and not isinstance(x, basestring))
     istext = lambda x: isinstance(x, basestring)
     isdict = lambda x: isinstance(x, dict)
-    isset = lambda x: isinstance(x, set)
+    isset = lambda x: isinstance(x, (set, frozenset))
 
+    verbose = config.getoption('verbose')
     explanation = None
     try:
         if op == '==':
             if istext(left) and istext(right):
-                explanation = _diff_text(left, right)
+                explanation = _diff_text(left, right, verbose)
             elif issequence(left) and issequence(right):
-                explanation = _compare_eq_sequence(left, right)
+                explanation = _compare_eq_sequence(left, right, verbose)
             elif isset(left) and isset(right):
-                explanation = _compare_eq_set(left, right)
+                explanation = _compare_eq_set(left, right, verbose)
             elif isdict(left) and isdict(right):
-                explanation = _diff_text(py.std.pprint.pformat(left),
-                                         py.std.pprint.pformat(right))
+                explanation = _compare_eq_dict(left, right, verbose)
         elif op == 'not in':
             if istext(left) and istext(right):
-                explanation = _notin_text(left, right)
-    except py.builtin._sysex:
-        raise
-    except:
+                explanation = _notin_text(left, right, verbose)
+    except Exception:
         excinfo = py.code.ExceptionInfo()
-        explanation = ['(pytest_assertion plugin: representation of '
-            'details failed. Probably an object has a faulty __repr__.)',
-            str(excinfo)
-            ]
-
+        explanation = [
+            u('(pytest_assertion plugin: representation of details failed.  '
+              'Probably an object has a faulty __repr__.)'),
+            u(excinfo)]
 
     if not explanation:
         return None
 
-    # Don't include pageloads of data, should be configurable
-    if len(''.join(explanation)) > 80*8:
-        explanation = ['Detailed information too verbose, truncated']
-
     return [summary] + explanation
 
 
-def _diff_text(left, right):
-    """Return the explanation for the diff between text
+def _diff_text(left, right, verbose=False):
+    """Return the explanation for the diff between text or bytes
 
-    This will skip leading and trailing characters which are
-    identical to keep the diff minimal.
+    Unless --verbose is used this will skip leading and trailing
+    characters which are identical to keep the diff minimal.
+
+    If the input are bytes they will be safely converted to text.
     """
     explanation = []
-    i = 0 # just in case left or right has zero length
-    for i in range(min(len(left), len(right))):
-        if left[i] != right[i]:
-            break
-    if i > 42:
-        i -= 10                 # Provide some context
-        explanation = ['Skipping %s identical '
-                       'leading characters in diff' % i]
-        left = left[i:]
-        right = right[i:]
-    if len(left) == len(right):
-        for i in range(len(left)):
-            if left[-i] != right[-i]:
+    if isinstance(left, py.builtin.bytes):
+        left = u(repr(left)[1:-1]).replace(r'\n', '\n')
+    if isinstance(right, py.builtin.bytes):
+        right = u(repr(right)[1:-1]).replace(r'\n', '\n')
+    if not verbose:
+        i = 0  # just in case left or right has zero length
+        for i in range(min(len(left), len(right))):
+            if left[i] != right[i]:
                 break
         if i > 42:
-            i -= 10     # Provide some context
-            explanation += ['Skipping %s identical '
-                            'trailing characters in diff' % i]
-            left = left[:-i]
-            right = right[:-i]
+            i -= 10                 # Provide some context
+            explanation = [u('Skipping %s identical leading '
+                             'characters in diff, use -v to show') % i]
+            left = left[i:]
+            right = right[i:]
+        if len(left) == len(right):
+            for i in range(len(left)):
+                if left[-i] != right[-i]:
+                    break
+            if i > 42:
+                i -= 10     # Provide some context
+                explanation += [u('Skipping %s identical trailing '
+                                  'characters in diff, use -v to show') % i]
+                left = left[:-i]
+                right = right[:-i]
     explanation += [line.strip('\n')
                     for line in py.std.difflib.ndiff(left.splitlines(),
                                                      right.splitlines())]
     return explanation
 
 
-def _compare_eq_sequence(left, right):
+def _compare_eq_sequence(left, right, verbose=False):
     explanation = []
     for i in range(min(len(left), len(right))):
         if left[i] != right[i]:
-            explanation += ['At index %s diff: %r != %r' %
-                            (i, left[i], right[i])]
+            explanation += [u('At index %s diff: %r != %r')
+                            % (i, left[i], right[i])]
             break
     if len(left) > len(right):
-        explanation += ['Left contains more items, '
-            'first extra item: %s' % py.io.saferepr(left[len(right)],)]
+        explanation += [u('Left contains more items, first extra item: %s')
+                        % py.io.saferepr(left[len(right)],)]
     elif len(left) < len(right):
-        explanation += ['Right contains more items, '
-            'first extra item: %s' % py.io.saferepr(right[len(left)],)]
-    return explanation # + _diff_text(py.std.pprint.pformat(left),
-                       #             py.std.pprint.pformat(right))
+        explanation += [
+            u('Right contains more items, first extra item: %s') %
+            py.io.saferepr(right[len(left)],)]
+    return explanation  # + _diff_text(py.std.pprint.pformat(left),
+                        #             py.std.pprint.pformat(right))
 
 
-def _compare_eq_set(left, right):
+def _compare_eq_set(left, right, verbose=False):
     explanation = []
     diff_left = left - right
     diff_right = right - left
     if diff_left:
-        explanation.append('Extra items in the left set:')
+        explanation.append(u('Extra items in the left set:'))
         for item in diff_left:
             explanation.append(py.io.saferepr(item))
     if diff_right:
-        explanation.append('Extra items in the right set:')
+        explanation.append(u('Extra items in the right set:'))
         for item in diff_right:
             explanation.append(py.io.saferepr(item))
     return explanation
 
 
-def _notin_text(term, text):
+def _compare_eq_dict(left, right, verbose=False):
+    explanation = []
+    common = set(left).intersection(set(right))
+    same = dict((k, left[k]) for k in common if left[k] == right[k])
+    if same and not verbose:
+        explanation += [u('Omitting %s identical items, use -v to show') %
+                        len(same)]
+    elif same:
+        explanation += [u('Common items:')]
+        explanation += py.std.pprint.pformat(same).splitlines()
+    diff = set(k for k in common if left[k] != right[k])
+    if diff:
+        explanation += [u('Differing items:')]
+        for k in diff:
+            explanation += [py.io.saferepr({k: left[k]}) + ' != ' +
+                            py.io.saferepr({k: right[k]})]
+    extra_left = set(left) - set(right)
+    if extra_left:
+        explanation.append(u('Left contains more items:'))
+        explanation.extend(py.std.pprint.pformat(
+            dict((k, left[k]) for k in extra_left)).splitlines())
+    extra_right = set(right) - set(left)
+    if extra_right:
+        explanation.append(u('Right contains more items:'))
+        explanation.extend(py.std.pprint.pformat(
+            dict((k, right[k]) for k in extra_right)).splitlines())
+    return explanation
+
+
+def _notin_text(term, text, verbose=False):
     index = text.find(term)
     head = text[:index]
     tail = text[index+len(term):]
     correct_text = head + tail
-    diff = _diff_text(correct_text, text)
-    newdiff = ['%s is contained here:' % py.io.saferepr(term, maxsize=42)]
+    diff = _diff_text(correct_text, text, verbose)
+    newdiff = [u('%s is contained here:') % py.io.saferepr(term, maxsize=42)]
     for line in diff:
-        if line.startswith('Skipping'):
+        if line.startswith(u('Skipping')):
             continue
-        if line.startswith('- '):
+        if line.startswith(u('- ')):
             continue
-        if line.startswith('+ '):
-            newdiff.append('  ' + line[2:])
+        if line.startswith(u('+ ')):
+            newdiff.append(u('  ') + line[2:])
         else:
             newdiff.append(line)
     return newdiff
diff --git a/_pytest/capture.py b/_pytest/capture.py
--- a/_pytest/capture.py
+++ b/_pytest/capture.py
@@ -1,43 +1,114 @@
-""" per-test stdout/stderr capturing mechanisms, ``capsys`` and ``capfd`` function arguments.  """
+"""
+    per-test stdout/stderr capturing mechanisms,
+    ``capsys`` and ``capfd`` function arguments.
+"""
+# note: py.io capture was where copied from
+# pylib 1.4.20.dev2 (rev 13d9af95547e)
+import sys
+import os
+import tempfile
 
-import pytest, py
-import os
+import py
+import pytest
+
+try:
+    from io import StringIO
+except ImportError:
+    from StringIO import StringIO
+
+try:
+    from io import BytesIO
+except ImportError:
+    class BytesIO(StringIO):
+        def write(self, data):
+            if isinstance(data, unicode):
+                raise TypeError("not a byte value: %r" % (data,))
+            StringIO.write(self, data)
+
+if sys.version_info < (3, 0):
+    class TextIO(StringIO):
+        def write(self, data):
+            if not isinstance(data, unicode):
+                enc = getattr(self, '_encoding', 'UTF-8')
+                data = unicode(data, enc, 'replace')
+            StringIO.write(self, data)
+else:
+    TextIO = StringIO
+
+
+patchsysdict = {0: 'stdin', 1: 'stdout', 2: 'stderr'}
+
 
 def pytest_addoption(parser):
     group = parser.getgroup("general")
-    group._addoption('--capture', action="store", default=None,
-        metavar="method", type="choice", choices=['fd', 'sys', 'no'],
+    group._addoption(
+        '--capture', action="store", default=None,
+        metavar="method", choices=['fd', 'sys', 'no'],
         help="per-test capturing method: one of fd (default)|sys|no.")
-    group._addoption('-s', action="store_const", const="no", dest="capture",
+    group._addoption(
+        '-s', action="store_const", const="no", dest="capture",
         help="shortcut for --capture=no.")
 
+
 @pytest.mark.tryfirst
-def pytest_cmdline_parse(pluginmanager, args):
-    # we want to perform capturing already for plugin/conftest loading
-    if '-s' in args or "--capture=no" in args:
-        method = "no"
-    elif hasattr(os, 'dup') and '--capture=sys' not in args:
+def pytest_load_initial_conftests(early_config, parser, args, __multicall__):
+    ns = parser.parse_known_args(args)
+    method = ns.capture
+    if not method:
         method = "fd"
-    else:
+    if method == "fd" and not hasattr(os, "dup"):
         method = "sys"
     capman = CaptureManager(method)
-    pluginmanager.register(capman, "capturemanager")
+    early_config.pluginmanager.register(capman, "capturemanager")
+
+    # make sure that capturemanager is properly reset at final shutdown
+    def teardown():
+        try:
+            capman.reset_capturings()
+        except ValueError:
+            pass
+
+    early_config.pluginmanager.add_shutdown(teardown)
+
+    # make sure logging does not raise exceptions at the end
+    def silence_logging_at_shutdown():
+        if "logging" in sys.modules:
+            sys.modules["logging"].raiseExceptions = False
+    early_config.pluginmanager.add_shutdown(silence_logging_at_shutdown)
+
+    # finally trigger conftest loading but while capturing (issue93)
+    capman.resumecapture()
+    try:
+        try:
+            return __multicall__.execute()
+        finally:
+            out, err = capman.suspendcapture()
+    except:
+        sys.stdout.write(out)
+        sys.stderr.write(err)
+        raise
+
 
 def addouterr(rep, outerr):
     for secname, content in zip(["out", "err"], outerr):
         if content:
             rep.sections.append(("Captured std%s" % secname, content))
 
+
 class NoCapture:
     def startall(self):
         pass
+
     def resume(self):
         pass
+
     def reset(self):
         pass
+
     def suspend(self):
         return "", ""
 
+
 class CaptureManager:
     def __init__(self, defaultmethod=None):
         self._method2capture = {}
@@ -45,21 +116,23 @@
 
     def _maketempfile(self):
         f = py.std.tempfile.TemporaryFile()
-        newf = py.io.dupfile(f, encoding="UTF-8")
+        newf = dupfile(f, encoding="UTF-8")
         f.close()
         return newf
 
     def _makestringio(self):
-        return py.io.TextIO()
+        return TextIO()
 
     def _getcapture(self, method):
         if method == "fd":
-            return py.io.StdCaptureFD(now=False,
-                out=self._maketempfile(), err=self._maketempfile()
+            return StdCaptureFD(
+                out=self._maketempfile(),
+                err=self._maketempfile(),
             )
         elif method == "sys":
-            return py.io.StdCapture(now=False,
-                out=self._makestringio(), err=self._makestringio()
+            return StdCapture(
+                out=self._makestringio(),
+                err=self._makestringio(),
             )
         elif method == "no":
             return NoCapture()
@@ -74,23 +147,24 @@
                 method = config._conftest.rget("option_capture", path=fspath)
             except KeyError:
                 method = "fd"
-        if method == "fd" and not hasattr(os, 'dup'): # e.g. jython
+        if method == "fd" and not hasattr(os, 'dup'):  # e.g. jython
             method = "sys"
         return method
 
     def reset_capturings(self):
-        for name, cap in self._method2capture.items():
+        for cap in self._method2capture.values():
             cap.reset()
 
     def resumecapture_item(self, item):
         method = self._getmethod(item.config, item.fspath)
         if not hasattr(item, 'outerr'):
-            item.outerr = ('', '') # we accumulate outerr on the item
+            item.outerr = ('', '')  # we accumulate outerr on the item
         return self.resumecapture(method)
 
     def resumecapture(self, method=None):
         if hasattr(self, '_capturing'):
-            raise ValueError("cannot resume, already capturing with %r" %
+            raise ValueError(
+                "cannot resume, already capturing with %r" %
                 (self._capturing,))
         if method is None:
             method = self._defaultmethod
@@ -119,30 +193,29 @@
         return "", ""
 
     def activate_funcargs(self, pyfuncitem):
-        if not hasattr(pyfuncitem, 'funcargs'):
-            return
-        assert not hasattr(self, '_capturing_funcargs')
-        self._capturing_funcargs = capturing_funcargs = []
-        for name, capfuncarg in pyfuncitem.funcargs.items():
-            if name in ('capsys', 'capfd'):
-                capturing_funcargs.append(capfuncarg)
-                capfuncarg._start()
+        funcargs = getattr(pyfuncitem, "funcargs", None)
+        if funcargs is not None:
+            for name, capfuncarg in funcargs.items():
+                if name in ('capsys', 'capfd'):
+                    assert not hasattr(self, '_capturing_funcarg')
+                    self._capturing_funcarg = capfuncarg
+                    capfuncarg._start()
 
     def deactivate_funcargs(self):
-        capturing_funcargs = getattr(self, '_capturing_funcargs', None)
-        if capturing_funcargs is not None:
-            while capturing_funcargs:
-                capfuncarg = capturing_funcargs.pop()
-                capfuncarg._finalize()
-            del self._capturing_funcargs
+        capturing_funcarg = getattr(self, '_capturing_funcarg', None)
+        if capturing_funcarg:
+            outerr = capturing_funcarg._finalize()
+            del self._capturing_funcarg
+            return outerr
 
     def pytest_make_collect_report(self, __multicall__, collector):
         method = self._getmethod(collector.config, collector.fspath)
         try:
             self.resumecapture(method)
         except ValueError:
-            return # recursive collect, XXX refactor capturing
-                   # to allow for more lightweight recursive capturing
+            # recursive collect, XXX refactor capturing
+            # to allow for more lightweight recursive capturing
+            return
         try:
             rep = __multicall__.execute()
         finally:
@@ -169,46 +242,371 @@
 
     @pytest.mark.tryfirst
     def pytest_runtest_makereport(self, __multicall__, item, call):
-        self.deactivate_funcargs()
+        funcarg_outerr = self.deactivate_funcargs()
         rep = __multicall__.execute()
         outerr = self.suspendcapture(item)
-        if not rep.passed:
-            addouterr(rep, outerr)
+        if funcarg_outerr is not None:
+            outerr = (outerr[0] + funcarg_outerr[0],
+                      outerr[1] + funcarg_outerr[1])
+        addouterr(rep, outerr)
         if not rep.passed or rep.when == "teardown":
             outerr = ('', '')
         item.outerr = outerr
         return rep
 
+error_capsysfderror = "cannot use capsys and capfd at the same time"
+
+
 def pytest_funcarg__capsys(request):
     """enables capturing of writes to sys.stdout/sys.stderr and makes
     captured output available via ``capsys.readouterr()`` method calls
     which return a ``(out, err)`` tuple.
     """
-    return CaptureFuncarg(py.io.StdCapture)
+    if "capfd" in request._funcargs:
+        raise request.raiseerror(error_capsysfderror)
+    return CaptureFixture(StdCapture)
+
 
 def pytest_funcarg__capfd(request):
     """enables capturing of writes to file descriptors 1 and 2 and makes
     captured output available via ``capsys.readouterr()`` method calls
     which return a ``(out, err)`` tuple.
     """
+    if "capsys" in request._funcargs:
+        request.raiseerror(error_capsysfderror)
     if not hasattr(os, 'dup'):
-        py.test.skip("capfd funcarg needs os.dup")
-    return CaptureFuncarg(py.io.StdCaptureFD)
+        pytest.skip("capfd funcarg needs os.dup")
+    return CaptureFixture(StdCaptureFD)
 
-class CaptureFuncarg:
+
+class CaptureFixture:
     def __init__(self, captureclass):
-        self.capture = captureclass(now=False)
+        self._capture = captureclass()
 
     def _start(self):
-        self.capture.startall()
+        self._capture.startall()
 
     def _finalize(self):
-        if hasattr(self, 'capture'):
-            self.capture.reset()
-            del self.capture
+        if hasattr(self, '_capture'):
+            outerr = self._outerr = self._capture.reset()
+            del self._capture
+            return outerr
 
     def readouterr(self):
-        return self.capture.readouterr()
+        try:
+            return self._capture.readouterr()
+        except AttributeError:
+            return self._outerr
 
     def close(self):
         self._finalize()
+
+
+class FDCapture:
+    """ Capture IO to/from a given os-level filedescriptor. """
+
+    def __init__(self, targetfd, tmpfile=None, patchsys=False):
+        """ save targetfd descriptor, and open a new
+            temporary file there.  If no tmpfile is
+            specified a tempfile.Tempfile() will be opened
+            in text mode.
+        """
+        self.targetfd = targetfd
+        if tmpfile is None and targetfd != 0:
+            f = tempfile.TemporaryFile('wb+')
+            tmpfile = dupfile(f, encoding="UTF-8")
+            f.close()
+        self.tmpfile = tmpfile
+        self._savefd = os.dup(self.targetfd)
+        if patchsys:
+            self._oldsys = getattr(sys, patchsysdict[targetfd])
+
+    def start(self):
+        try:
+            os.fstat(self._savefd)
+        except OSError:
+            raise ValueError(
+                "saved filedescriptor not valid, "
+                "did you call start() twice?")
+        if self.targetfd == 0 and not self.tmpfile:
+            fd = os.open(os.devnull, os.O_RDONLY)
+            os.dup2(fd, 0)
+            os.close(fd)
+            if hasattr(self, '_oldsys'):
+                setattr(sys, patchsysdict[self.targetfd], DontReadFromInput())
+        else:
+            os.dup2(self.tmpfile.fileno(), self.targetfd)
+            if hasattr(self, '_oldsys'):
+                setattr(sys, patchsysdict[self.targetfd], self.tmpfile)
+
+    def done(self):
+        """ unpatch and clean up, returns the self.tmpfile (file object)
+        """
+        os.dup2(self._savefd, self.targetfd)
+        os.close(self._savefd)
+        if self.targetfd != 0:
+            self.tmpfile.seek(0)
+        if hasattr(self, '_oldsys'):
+            setattr(sys, patchsysdict[self.targetfd], self._oldsys)
+        return self.tmpfile
+
+    def writeorg(self, data):
+        """ write a string to the original file descriptor
+        """
+        tempfp = tempfile.TemporaryFile()
+        try:
+            os.dup2(self._savefd, tempfp.fileno())
+            tempfp.write(data)
+        finally:
+            tempfp.close()
+
+
+def dupfile(f, mode=None, buffering=0, raising=False, encoding=None):
+    """ return a new open file object that's a duplicate of f
+
+        mode is duplicated if not given, 'buffering' controls
+        buffer size (defaulting to no buffering) and 'raising'
+        defines whether an exception is raised when an incompatible
+        file object is passed in (if raising is False, the file
+        object itself will be returned)
+    """
+    try:
+        fd = f.fileno()
+        mode = mode or f.mode
+    except AttributeError:
+        if raising:
+            raise
+        return f
+    newfd = os.dup(fd)
+    if sys.version_info >= (3, 0):
+        if encoding is not None:
+            mode = mode.replace("b", "")
+            buffering = True
+        return os.fdopen(newfd, mode, buffering, encoding, closefd=True)
+    else:
+        f = os.fdopen(newfd, mode, buffering)
+        if encoding is not None:
+            return EncodedFile(f, encoding)
+        return f
+
+
+class EncodedFile(object):
+    def __init__(self, _stream, encoding):
+        self._stream = _stream
+        self.encoding = encoding
+

From noreply at buildbot.pypy.org  Sun Sep 14 23:46:22 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 14 Sep 2014 23:46:22 +0200 (CEST)
Subject: [pypy-commit] pypy no-write-barrier-in-const-ptrs: will merge it
	differently
Message-ID: <20140914214622.460BD1C04FA@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: no-write-barrier-in-const-ptrs
Changeset: r73533:3d26064d218e
Date: 2014-09-14 15:45 -0600
http://bitbucket.org/pypy/pypy/changeset/3d26064d218e/

Log:	will merge it differently


From noreply at buildbot.pypy.org  Sun Sep 14 23:56:12 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 14 Sep 2014 23:56:12 +0200 (CEST)
Subject: [pypy-commit] pypy default: Don't emit write barriers for constants
 in the JIT - they all can't move. Be a bit future-proof by calling can_move,
 which is going to always be true, until we merge pinning
Message-ID: <20140914215612.935CC1C371B@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r73534:8707febbf25b
Date: 2014-09-14 15:54 -0600
http://bitbucket.org/pypy/pypy/changeset/8707febbf25b/

Log:	Don't emit write barriers for constants in the JIT - they all can't
	move. Be a bit future-proof by calling can_move, which is going to
	always be true, until we merge pinning

diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -1,3 +1,4 @@
+from rpython.rlib import rgc
 from rpython.rlib.rarithmetic import ovfcheck
 from rpython.rtyper.lltypesystem import llmemory
 from rpython.jit.metainterp import history
@@ -390,8 +391,8 @@
         val = op.getarg(0)
         if val not in self.write_barrier_applied:
             v = op.getarg(1)
-            if isinstance(v, BoxPtr) or (isinstance(v, ConstPtr) and
-                                         bool(v.value)): # store a non-NULL
+            if (isinstance(v, BoxPtr) or (isinstance(v, ConstPtr) and
+                               rgc.needs_write_barrier(v.value))):
                 self.gen_write_barrier(val)
                 #op = op.copy_and_change(rop.SETFIELD_RAW)
         self.newops.append(op)
@@ -400,8 +401,8 @@
         val = op.getarg(0)
         if val not in self.write_barrier_applied:
             v = op.getarg(2)
-            if isinstance(v, BoxPtr) or (isinstance(v, ConstPtr) and
-                                         bool(v.value)): # store a non-NULL
+            if (isinstance(v, BoxPtr) or (isinstance(v, ConstPtr) and
+                                         rgc.needs_write_barrier(v.value))):
                 self.gen_write_barrier_array(val, op.getarg(1))
                 #op = op.copy_and_change(rop.SET{ARRAYITEM,INTERIORFIELD}_RAW)
         self.newops.append(op)
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -86,6 +86,14 @@
         collect(i)
         i += 1
 
+def needs_write_barrier(obj):
+    """ We need to emit write barrier if the right hand of assignment
+    is in nursery, used by the JIT for handling set*_gc(Const)
+    """
+    if not obj:
+        return False
+    return can_move(obj)
+
 def _heap_stats():
     raise NotImplementedError # can't be run directly
 

From noreply at buildbot.pypy.org  Sun Sep 14 23:56:13 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 14 Sep 2014 23:56:13 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20140914215613.DD51B1C371B@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r73535:06bb85dd0910
Date: 2014-09-14 15:54 -0600
http://bitbucket.org/pypy/pypy/changeset/06bb85dd0910/

Log:	merge

diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -85,10 +85,13 @@
 
 Abridged method (for -Ojit builds using Visual Studio 2008)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Download the versions of all the external packages
-from 
+Download the versions of all the external packages from 
+https://bitbucket.org/pypy/pypy/downloads/local_2.4.zip
+(for 2.4 release and later) or
 https://bitbucket.org/pypy/pypy/downloads/local.zip
-Then expand it into the base directory (base_dir) and modify your environment to reflect this::
+(for pre-2.4 versions)
+Then expand it into the base directory (base_dir) and modify your environment
+to reflect this::
 
     set PATH=\bin;\tcltk\bin;%PATH%
     set INCLUDE=\include;\tcltk\include;%INCLUDE%
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -38,18 +38,15 @@
 def cpython_code_signature(code):
     "([list-of-arg-names], vararg-name-or-None, kwarg-name-or-None)."
     argcount = code.co_argcount
+    varnames = code.co_varnames
     assert argcount >= 0     # annotator hint
-    argnames = list(code.co_varnames[:argcount])
+    argnames = list(varnames[:argcount])
     if code.co_flags & CO_VARARGS:
-        varargname = code.co_varnames[argcount]
+        varargname = varnames[argcount]
         argcount += 1
     else:
         varargname = None
-    if code.co_flags & CO_VARKEYWORDS:
-        kwargname = code.co_varnames[argcount]
-        argcount += 1
-    else:
-        kwargname = None
+    kwargname = varnames[argcount] if code.co_flags & CO_VARKEYWORDS else None
     return Signature(argnames, varargname, kwargname)
 
 
diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -945,7 +945,7 @@
         prefix = udir.join('pathtest').ensure(dir=1)
         fake_exe = 'bin/pypy-c'
         if sys.platform == 'win32':
-            fake_exe += '.exe'
+            fake_exe = 'pypy-c.exe'
         fake_exe = prefix.join(fake_exe).ensure(file=1)
         expected_path = [str(prefix.join(subdir).ensure(dir=1))
                          for subdir in ('lib_pypy',
@@ -985,8 +985,10 @@
             assert sys.path == old_sys_path + [self.goal_dir]
 
             app_main.setup_bootstrap_path(self.fake_exe)
-            assert sys.executable == ''      # not executable!
-            assert sys.path == old_sys_path + [self.goal_dir]
+            if not sys.platform == 'win32':
+                # an existing file is always 'executable' on windows
+                assert sys.executable == ''      # not executable!
+                assert sys.path == old_sys_path + [self.goal_dir]
 
             os.chmod(self.fake_exe, 0755)
             app_main.setup_bootstrap_path(self.fake_exe)
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -775,7 +775,7 @@
             else:
                 r, w, e = rpoll.select([sock_fd], [], [], sock_timeout)
                 ready = r
-        except SelectError, e:
+        except rpoll.SelectError as e:
             message = e.get_msg()
             raise ssl_error(space, message, e.errno)
     if ready:
diff --git a/pypy/module/operator/__init__.py b/pypy/module/operator/__init__.py
--- a/pypy/module/operator/__init__.py
+++ b/pypy/module/operator/__init__.py
@@ -39,7 +39,7 @@
                     'irshift', 'isub', 'itruediv', 'ixor', '_length_hint']
 
     interpleveldefs = {
-        '_compare_digest': 'interp_operator.compare_digest',
+        '_compare_digest': 'tscmp.compare_digest',
     }
 
     for name in interp_names:
diff --git a/pypy/module/operator/interp_operator.py b/pypy/module/operator/interp_operator.py
--- a/pypy/module/operator/interp_operator.py
+++ b/pypy/module/operator/interp_operator.py
@@ -1,6 +1,4 @@
-from rpython.rlib.objectmodel import specialize
-
-from pypy.interpreter.error import OperationError, oefmt
+from pypy.interpreter.error import OperationError
 from pypy.interpreter.gateway import unwrap_spec
 
 
@@ -249,33 +247,3 @@
 @unwrap_spec(default=int)
 def _length_hint(space, w_iterable, default):
     return space.wrap(space.length_hint(w_iterable, default))
-
-def compare_digest(space, w_a, w_b):
-    if (
-        space.isinstance_w(w_a, space.w_unicode) and
-        space.isinstance_w(w_b, space.w_unicode)
-    ):
-        return space.wrap(tscmp(space.unicode_w(w_a), space.unicode_w(w_b)))
-    if (
-        space.isinstance_w(w_a, space.w_unicode) or
-        space.isinstance_w(w_b, space.w_unicode)
-    ):
-        raise oefmt(
-            space.w_TypeError,
-            "unsupported operand types(s) or combination of types: '%N' and '%N'",
-            w_a,
-            w_b,
-        )
-    else:
-        return space.wrap(tscmp(space.bufferstr_w(w_a), space.bufferstr_w(w_b)))
-
-
- at specialize.argtype(0, 1)
-def tscmp(a, b):
-    len_a = len(a)
-    len_b = len(b)
-    length = min(len(a), len(b))
-    res = len_a ^ len_b
-    for i in xrange(length):
-        res |= ord(a[i]) ^ ord(b[i])
-    return res == 0
diff --git a/pypy/module/operator/test/test_operator.py b/pypy/module/operator/test/test_operator.py
--- a/pypy/module/operator/test/test_operator.py
+++ b/pypy/module/operator/test/test_operator.py
@@ -334,3 +334,9 @@
         assert operator._compare_digest(a, b)
         a, b = mybytes(b"foobar"), mybytes(b"foobaz")
         assert not operator._compare_digest(a, b)
+
+    def test_compare_digest_unicode(self):
+        import operator
+        assert operator._compare_digest(u'asd', u'asd')
+        assert not operator._compare_digest(u'asd', u'qwe')
+        raises(TypeError, operator._compare_digest, u'asd', b'qwe')
diff --git a/pypy/module/operator/test/test_tscmp.py b/pypy/module/operator/test/test_tscmp.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/operator/test/test_tscmp.py
@@ -0,0 +1,28 @@
+from pypy.module.operator.tscmp import pypy_tscmp, pypy_tscmp_wide
+
+class TestTimingSafeCompare:
+    tostr = str
+    tscmp = staticmethod(pypy_tscmp)
+
+    def test_tscmp_neq(self):
+        assert not self.tscmp(self.tostr('asd'), self.tostr('qwe'), 3, 3)
+
+    def test_tscmp_eq(self):
+        assert self.tscmp(self.tostr('asd'), self.tostr('asd'), 3, 3)
+
+    def test_tscmp_len(self):
+        assert self.tscmp(self.tostr('asdp'), self.tostr('asdq'), 3, 3)
+
+    def test_tscmp_nlen(self):
+        assert not self.tscmp(self.tostr('asd'), self.tostr('asd'), 2, 3)
+
+
+class TestTimingSafeCompareWide(TestTimingSafeCompare):
+    tostr = unicode
+    tscmp = staticmethod(pypy_tscmp_wide)
+
+    def test_tscmp_wide_nonascii(self):
+        a, b = u"\ud808\udf45", u"\ud808\udf45"
+        assert self.tscmp(a, b, len(a), len(b))
+        a, b = u"\ud808\udf45", u"\ud808\udf45 "
+        assert not self.tscmp(a, b, len(a), len(b))
diff --git a/pypy/module/operator/tscmp.c b/pypy/module/operator/tscmp.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/operator/tscmp.c
@@ -0,0 +1,80 @@
+/* Derived from CPython 3.3.5's operator.c::_tscmp
+ */
+
+#include 
+#include 
+#include "tscmp.h"
+
+int
+pypy_tscmp(const char *a, const char *b, long len_a, long len_b)
+{
+    /* The volatile type declarations make sure that the compiler has no
+     * chance to optimize and fold the code in any way that may change
+     * the timing.
+     */
+    volatile long length;
+    volatile const char *left;
+    volatile const char *right;
+    long i;
+    char result;
+
+    /* loop count depends on length of b */
+    length = len_b;
+    left = NULL;
+    right = b;
+
+    /* don't use else here to keep the amount of CPU instructions constant,
+     * volatile forces re-evaluation
+     *  */
+    if (len_a == length) {
+        left = *((volatile const char**)&a);
+        result = 0;
+    }
+    if (len_a != length) {
+        left = b;
+        result = 1;
+    }
+
+    for (i=0; i < length; i++) {
+        result |= *left++ ^ *right++;
+    }
+
+    return (result == 0);
+}
+
+int
+pypy_tscmp_wide(const wchar_t *a, const wchar_t *b, long len_a, long len_b)
+{
+    /* The volatile type declarations make sure that the compiler has no
+     * chance to optimize and fold the code in any way that may change
+     * the timing.
+     */
+    volatile long length;
+    volatile const wchar_t *left;
+    volatile const wchar_t *right;
+    long i;
+    wchar_t result;
+
+    /* loop count depends on length of b */
+    length = len_b;
+    left = NULL;
+    right = b;
+
+    /* don't use else here to keep the amount of CPU instructions constant,
+     * volatile forces re-evaluation
+     *  */
+    if (len_a == length) {
+        left = *((volatile const wchar_t**)&a);
+        result = 0;
+    }
+    if (len_a != length) {
+        left = b;
+        result = 1;
+    }
+
+    for (i=0; i < length; i++) {
+        result |= *left++ ^ *right++;
+    }
+
+    return (result == 0);
+}
diff --git a/pypy/module/operator/tscmp.h b/pypy/module/operator/tscmp.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/operator/tscmp.h
@@ -0,0 +1,2 @@
+int pypy_tscmp(const char *, const char *, long, long);
+int pypy_tscmp_wide(const wchar_t *, const wchar_t *, long, long);
diff --git a/pypy/module/operator/tscmp.py b/pypy/module/operator/tscmp.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/operator/tscmp.py
@@ -0,0 +1,73 @@
+"""
+Provides _compare_digest method, which is a safe comparing to prevent timing
+attacks for the hmac module.
+"""
+import py
+
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
+
+from pypy.interpreter.error import oefmt
+
+cwd = py.path.local(__file__).dirpath()
+eci = ExternalCompilationInfo(
+    includes=[cwd.join('tscmp.h')],
+    include_dirs=[str(cwd)],
+    separate_module_files=[cwd.join('tscmp.c')],
+    export_symbols=['pypy_tscmp', 'pypy_tscmp_wide'])
+
+
+def llexternal(*args, **kwargs):
+    kwargs.setdefault('compilation_info', eci)
+    kwargs.setdefault('sandboxsafe', True)
+    return rffi.llexternal(*args, **kwargs)
+
+
+pypy_tscmp = llexternal(
+    'pypy_tscmp',
+    [rffi.CCHARP, rffi.CCHARP, rffi.LONG, rffi.LONG],
+    rffi.INT)
+pypy_tscmp_wide = llexternal(
+    'pypy_tscmp_wide',
+    [rffi.CWCHARP, rffi.CWCHARP, rffi.LONG, rffi.LONG],
+    rffi.INT)
+
+
+def compare_digest(space, w_a, w_b):
+    """compare_digest(a, b) -> bool
+
+    Return 'a == b'.  This function uses an approach designed to prevent
+    timing analysis, making it appropriate for cryptography.  a and b
+    must both be of the same type: either str (ASCII only), or any type
+    that supports the buffer protocol (e.g. bytes).
+
+    Note: If a and b are of different lengths, or if an error occurs, a
+    timing attack could theoretically reveal information about the types
+    and lengths of a and b--but not their values.
+    """
+    if (space.isinstance_w(w_a, space.w_unicode) and
+        space.isinstance_w(w_b, space.w_unicode)):
+        a = space.unicode_w(w_a)
+        b = space.unicode_w(w_b)
+        with rffi.scoped_nonmoving_unicodebuffer(a) as a_buf:
+            with rffi.scoped_nonmoving_unicodebuffer(b) as b_buf:
+                result = pypy_tscmp_wide(a_buf, b_buf, len(a), len(b))
+        return space.wrap(rffi.cast(lltype.Bool, result))
+    return compare_digest_buffer(space, w_a, w_b)
+
+
+def compare_digest_buffer(space, w_a, w_b):
+    try:
+        a_buf = w_a.buffer_w(space, space.BUF_SIMPLE)
+        b_buf = w_b.buffer_w(space, space.BUF_SIMPLE)
+    except TypeError:
+        raise oefmt(space.w_TypeError,
+                    "unsupported operand types(s) or combination of types: "
+                    "'%T' and '%T'", w_a, w_b)
+
+    a = a_buf.as_str()
+    b = b_buf.as_str()
+    with rffi.scoped_nonmovingbuffer(a) as a_buf:
+        with rffi.scoped_nonmovingbuffer(b) as b_buf:
+            result = pypy_tscmp(a_buf, b_buf, len(a), len(b))
+    return space.wrap(rffi.cast(lltype.Bool, result))
diff --git a/rpython/translator/platform/test/test_makefile.py b/rpython/translator/platform/test/test_makefile.py
--- a/rpython/translator/platform/test/test_makefile.py
+++ b/rpython/translator/platform/test/test_makefile.py
@@ -44,6 +44,7 @@
         assert res.returncode == 0        
     
     def test_900_files(self):
+        tmpdir = udir.join('test_900_files').ensure(dir=1)
         txt = '#include \n'
         for i in range(900):
             txt += 'int func%03d();\n' % i
@@ -52,11 +53,11 @@
             txt += '    j += func%03d();\n' % i
         txt += '    printf("%d\\n", j);\n'
         txt += '    return 0;};\n'
-        cfile = udir.join('test_900_files.c')
+        cfile = tmpdir.join('test_900_files.c')
         cfile.write(txt)
         cfiles = [cfile]
         for i in range(900):
-            cfile2 = udir.join('implement%03d.c' %i)
+            cfile2 = tmpdir.join('implement%03d.c' %i)
             cfile2.write('''
                 int func%03d()
             {
@@ -64,10 +65,10 @@
             }
             ''' % (i, i))
             cfiles.append(cfile2)
-        mk = self.platform.gen_makefile(cfiles, ExternalCompilationInfo(), path=udir)
+        mk = self.platform.gen_makefile(cfiles, ExternalCompilationInfo(), path=tmpdir)
         mk.write()
         self.platform.execute_makefile(mk)
-        res = self.platform.execute(udir.join('test_900_files'))
+        res = self.platform.execute(tmpdir.join('test_900_files'))
         self.check_res(res, '%d\n' %sum(range(900)))
 
     def test_precompiled_headers(self):
diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py
--- a/rpython/translator/platform/windows.py
+++ b/rpython/translator/platform/windows.py
@@ -203,6 +203,9 @@
         # the assembler still has the old behavior that all options
         # must come first, and after the file name all options are ignored.
         # So please be careful with the order of parameters! ;-)
+        pdb_dir = oname.dirname
+        if pdb_dir:
+                compile_args += ['/Fd%s\\' % (pdb_dir,)]
         args = ['/nologo', '/c'] + compile_args + ['/Fo%s' % (oname,), str(cfile)]
         self._execute_c_compiler(cc, args, oname)
         return oname
diff --git a/rpython/translator/sandbox/rsandbox.py b/rpython/translator/sandbox/rsandbox.py
--- a/rpython/translator/sandbox/rsandbox.py
+++ b/rpython/translator/sandbox/rsandbox.py
@@ -60,8 +60,7 @@
 
     def need_more_data(self):
         buflen = self.buflen
-        buf = lltype.malloc(rffi.CCHARP.TO, buflen, flavor='raw')
-        try:
+        with lltype.scoped_alloc(rffi.CCHARP.TO, buflen) as buf:
             buflen = rffi.cast(rffi.SIZE_T, buflen)
             count = ll_read_not_sandboxed(self.fd, buf, buflen)
             count = rffi.cast(lltype.Signed, count)
@@ -69,20 +68,15 @@
                 raise IOError
             self.buf += ''.join([buf[i] for i in range(count)])
             self.buflen *= 2
-        finally:
-            lltype.free(buf, flavor='raw')
 
 def sandboxed_io(buf):
     STDIN = 0
     STDOUT = 1
     # send the buffer with the marshalled fnname and input arguments to STDOUT
-    p = lltype.malloc(rffi.CCHARP.TO, len(buf), flavor='raw')
-    try:
+    with lltype.scoped_alloc(rffi.CCHARP.TO, len(buf)) as p:
         for i in range(len(buf)):
             p[i] = buf[i]
         writeall_not_sandboxed(STDOUT, p, len(buf))
-    finally:
-        lltype.free(p, flavor='raw')
     # build a Loader that will get the answer from STDIN
     loader = FdLoader(STDIN)
     # check for errors
@@ -108,9 +102,8 @@
 @signature(types.str(), returns=types.impossible())
 def not_implemented_stub(msg):
     STDERR = 2
-    buf = rffi.str2charp(msg + '\n')
-    writeall_not_sandboxed(STDERR, buf, len(msg) + 1)
-    rffi.free_charp(buf)
+    with rffi.scoped_str2charp(msg + '\n') as buf:
+        writeall_not_sandboxed(STDERR, buf, len(msg) + 1)
     raise RuntimeError(msg)  # XXX in RPython, the msg is ignored at the moment
 
 dump_string = rmarshal.get_marshaller(str)

From noreply at buildbot.pypy.org  Mon Sep 15 00:04:33 2014
From: noreply at buildbot.pypy.org (mjacob)
Date: Mon, 15 Sep 2014 00:04:33 +0200 (CEST)
Subject: [pypy-commit] pypy improve-docs: Move TODO-docs to pypy/doc.
Message-ID: <20140914220433.CCC081C371B@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: improve-docs
Changeset: r73536:270d713b5e12
Date: 2014-09-14 23:45 +0200
http://bitbucket.org/pypy/pypy/changeset/270d713b5e12/

Log:	Move TODO-docs to pypy/doc.

diff --git a/TODO-docs b/pypy/doc/TODO
rename from TODO-docs
rename to pypy/doc/TODO

From noreply at buildbot.pypy.org  Mon Sep 15 00:14:17 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 15 Sep 2014 00:14:17 +0200 (CEST)
Subject: [pypy-commit] pypy gc_no_cleanup_nursery: Remove the zeroing of gc
 pointers here, however, still remember we call
Message-ID: <20140914221417.DF71C1C304E@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: gc_no_cleanup_nursery
Changeset: r73537:6f0916ba2510
Date: 2014-09-14 16:13 -0600
http://bitbucket.org/pypy/pypy/changeset/6f0916ba2510/

Log:	Remove the zeroing of gc pointers here, however, still remember we
	call malloc(zero=True) and reflect it in jtransform

diff --git a/rpython/jit/codewriter/heaptracker.py b/rpython/jit/codewriter/heaptracker.py
--- a/rpython/jit/codewriter/heaptracker.py
+++ b/rpython/jit/codewriter/heaptracker.py
@@ -125,3 +125,4 @@
     vtable = descr.as_vtable_size_descr()._corresponding_vtable
     vtable = llmemory.cast_ptr_to_adr(vtable)
     return adr2int(vtable)
+    
diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -614,28 +614,23 @@
             arraydescr = self.cpu.arraydescrof(ARRAY)
             op1 = SpaceOperation('new_array', [op.args[2], arraydescr],
                                  op.result)
-            if self._has_gcptrs_in(ARRAY):
-                return self.zero_contents([op1], op.result, ARRAY,
-                                          only_gc_pointers=True)
             if op.args[1].value.get('zero', False):
                 return self.zero_contents([op1], op.result, ARRAY)
             return op1
 
-    def zero_contents(self, ops, v, TYPE, only_gc_pointers=False):
+    def zero_contents(self, ops, v, TYPE):
         if isinstance(TYPE, lltype.Struct):
             for name, FIELD in TYPE._flds.iteritems():
-                if (not only_gc_pointers or
-                    isinstance(FIELD, lltype.Ptr) and FIELD._needsgc()):
+                if isinstance(FIELD, lltype.Struct):
+                    # substruct
+                    self.zero_contents(ops, v, FIELD)
+                else:
                     c_name = Constant(name, lltype.Void)
                     c_null = Constant(FIELD._defl(), FIELD)
                     op = SpaceOperation('setfield', [v, c_name, c_null],
                                         None)
                     self.extend_with(ops, self.rewrite_op_setfield(op,
                                           override_type=TYPE))
-                elif isinstance(FIELD, lltype.Struct):
-                    # substruct
-                    self.zero_contents(ops, v, FIELD,
-                                       only_gc_pointers=only_gc_pointers)
         elif isinstance(TYPE, lltype.Array):
             arraydescr = self.cpu.arraydescrof(TYPE)
             ops.append(SpaceOperation('clear_array_contents',
@@ -906,10 +901,9 @@
             return self._rewrite_raw_malloc(op, 'raw_malloc_fixedsize', [])
         #
         if op.args[1].value.get('zero', False):
-            true_zero = True
+            zero = True
         else:
-            assert op.args[1].value == {'flavor': 'gc'}
-            true_zero = False
+            zero = False
         STRUCT = op.args[0].value
         vtable = heaptracker.get_vtable_for_gcstruct(self.cpu, STRUCT)
         if vtable:
@@ -931,11 +925,8 @@
             opname = 'new'
         sizedescr = self.cpu.sizeof(STRUCT)
         op1 = SpaceOperation(opname, [sizedescr], op.result)
-        if true_zero:
+        if zero:
             return self.zero_contents([op1], op.result, STRUCT)
-        if self._has_gcptrs_in(STRUCT):
-            return self.zero_contents([op1], op.result, STRUCT,
-                                      only_gc_pointers=True)
         return op1
 
     def _has_gcptrs_in(self, STRUCT):
@@ -1677,8 +1668,6 @@
             v.concretetype = lltype.Signed
             ops.append(SpaceOperation('int_force_ge_zero', [v_length], v))
         ops.append(SpaceOperation('new_array', [v, arraydescr], op.result))
-        if self._has_gcptrs_in(op.result.concretetype.TO):
-            self.zero_contents(ops, op.result, op.result.concretetype.TO)
         return ops
 
     def do_fixed_list_len(self, op, args, arraydescr):
diff --git a/rpython/jit/codewriter/test/test_jtransform.py b/rpython/jit/codewriter/test/test_jtransform.py
--- a/rpython/jit/codewriter/test/test_jtransform.py
+++ b/rpython/jit/codewriter/test/test_jtransform.py
@@ -529,18 +529,6 @@
     assert op1.opname == 'new'
     assert op1.args == [('sizedescr', S)]
 
-def test_malloc_new_zero():
-    SS = lltype.GcStruct('SS')
-    S = lltype.GcStruct('S', ('x', lltype.Ptr(SS)))
-    v = varoftype(lltype.Ptr(S))
-    op = SpaceOperation('malloc', [Constant(S, lltype.Void),
-                                   Constant({'flavor': 'gc'}, lltype.Void)], v)
-    op1, op2 = Transformer(FakeCPU()).rewrite_operation(op)
-    assert op1.opname == 'new'
-    assert op1.args == [('sizedescr', S)]
-    assert op2.opname == 'setfield_gc_r'
-    assert op2.args[0] == v
-
 def test_malloc_new_zero_2():
     S = lltype.GcStruct('S', ('x', lltype.Signed))
     v = varoftype(lltype.Ptr(S))
@@ -560,7 +548,8 @@
                          ('xx', lltype.Ptr(S0)))
     v = varoftype(lltype.Ptr(S2))
     op = SpaceOperation('malloc', [Constant(S2, lltype.Void),
-                                   Constant({'flavor': 'gc'}, lltype.Void)], v)
+                                   Constant({'flavor': 'gc',
+                                             'zero': True}, lltype.Void)], v)
     op1, op2, op3 = Transformer(FakeCPU()).rewrite_operation(op)
     assert op1.opname == 'new'
     assert op1.args == [('sizedescr', S2)]
@@ -1068,12 +1057,8 @@
 
 def test_malloc_varsize_zero():
     c_A = Constant(lltype.GcArray(lltype.Signed), lltype.Void)
-    c_flags = Constant({"flavor": "gc"}, lltype.Void)
     v1 = varoftype(lltype.Signed)
     v2 = varoftype(c_A.value)
-    op = SpaceOperation('malloc_varsize', [c_A, c_flags, v1], v2)
-    op1 = Transformer(FakeCPU()).rewrite_operation(op)
-    assert op1.opname == 'new_array'
     c_flags = Constant({"flavor": "gc", "zero": True}, lltype.Void)
     op = SpaceOperation('malloc_varsize', [c_A, c_flags, v1], v2)
     op1, op2 = Transformer(FakeCPU()).rewrite_operation(op)

From noreply at buildbot.pypy.org  Mon Sep 15 00:37:27 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 15 Sep 2014 00:37:27 +0200 (CEST)
Subject: [pypy-commit] pypy gc_no_cleanup_nursery: implement
	offsets_of_gcfields in sizedescr
Message-ID: <20140914223727.D0C2A1C319E@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: gc_no_cleanup_nursery
Changeset: r73538:5ba9f2d2ffbe
Date: 2014-09-14 16:36 -0600
http://bitbucket.org/pypy/pypy/changeset/5ba9f2d2ffbe/

Log:	implement offsets_of_gcfields in sizedescr

diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py
--- a/rpython/jit/backend/llsupport/descr.py
+++ b/rpython/jit/backend/llsupport/descr.py
@@ -35,9 +35,11 @@
     size = 0      # help translation
     tid = llop.combine_ushort(lltype.Signed, 0, 0)
 
-    def __init__(self, size, count_fields_if_immut=-1):
+    def __init__(self, size, count_fields_if_immut=-1,
+                 offsets_of_gcfields=None):
         self.size = size
         self.count_fields_if_immut = count_fields_if_immut
+        self.offsets_of_gcfields = offsets_of_gcfields
 
     def count_fields_if_immutable(self):
         return self.count_fields_if_immut
@@ -58,10 +60,13 @@
     except KeyError:
         size = symbolic.get_size(STRUCT, gccache.translate_support_code)
         count_fields_if_immut = heaptracker.count_fields_if_immutable(STRUCT)
+        offsets_of_gcfields = heaptracker.offsets_of_gcfields(gccache, STRUCT)
         if heaptracker.has_gcstruct_a_vtable(STRUCT):
-            sizedescr = SizeDescrWithVTable(size, count_fields_if_immut)
+            sizedescr = SizeDescrWithVTable(size, count_fields_if_immut,
+                                            offsets_of_gcfields)
         else:
-            sizedescr = SizeDescr(size, count_fields_if_immut)
+            sizedescr = SizeDescr(size, count_fields_if_immut,
+                                  offsets_of_gcfields)
         gccache.init_size_descr(STRUCT, sizedescr)
         cache[STRUCT] = sizedescr
         return sizedescr
diff --git a/rpython/jit/backend/llsupport/test/test_descr.py b/rpython/jit/backend/llsupport/test/test_descr.py
--- a/rpython/jit/backend/llsupport/test/test_descr.py
+++ b/rpython/jit/backend/llsupport/test/test_descr.py
@@ -19,6 +19,9 @@
     assert descr_t.size == symbolic.get_size(T, False)
     assert descr_s.count_fields_if_immutable() == -1
     assert descr_t.count_fields_if_immutable() == -1
+    assert descr_t.offsets_of_gcfields == []
+    assert descr_s.offsets_of_gcfields == [
+        symbolic.get_field_token(S, 'y', False)[0]]
     assert descr_s == get_size_descr(c0, S)
     assert descr_s != get_size_descr(c1, S)
     #
@@ -26,6 +29,14 @@
     assert isinstance(descr_s.size, Symbolic)
     assert descr_s.count_fields_if_immutable() == -1
 
+    PARENT = lltype.Struct('P', ('x', lltype.Ptr(T)))
+    STRUCT = lltype.GcStruct('S', ('parent', PARENT), ('y', lltype.Ptr(T)))
+    descr_struct = get_size_descr(c0, STRUCT)
+    assert descr_struct.offsets_of_gcfields == [
+        symbolic.get_field_token(PARENT, 'x', False)[0],
+        symbolic.get_field_token(STRUCT, 'y', False)[0],
+    ]
+
 def test_get_size_descr_immut():
     S = lltype.GcStruct('S', hints={'immutable': True})
     T = lltype.GcStruct('T', ('parent', S),
diff --git a/rpython/jit/backend/llsupport/test/test_gc.py b/rpython/jit/backend/llsupport/test/test_gc.py
--- a/rpython/jit/backend/llsupport/test/test_gc.py
+++ b/rpython/jit/backend/llsupport/test/test_gc.py
@@ -70,6 +70,8 @@
         self.record.append(("fixedsize", repr(size), tid, p))
         return p
 
+    do_malloc_fixedsize_clear = do_malloc_fixedsize
+
     def do_malloc_varsize(self, RESTYPE, type_id, length, size,
                                 itemsize, offset_to_length):
         p, tid = self._malloc(type_id, size + itemsize * length)
@@ -80,6 +82,8 @@
                             repr(offset_to_length), p))
         return p
 
+    do_malloc_varsize_clear = do_malloc_varsize
+
     def _write_barrier_failing_case(self, adr_struct):
         self.record.append(('barrier', adr_struct))
 
diff --git a/rpython/jit/codewriter/heaptracker.py b/rpython/jit/codewriter/heaptracker.py
--- a/rpython/jit/codewriter/heaptracker.py
+++ b/rpython/jit/codewriter/heaptracker.py
@@ -126,3 +126,19 @@
     vtable = llmemory.cast_ptr_to_adr(vtable)
     return adr2int(vtable)
     
+def offsets_of_gcfields(gccache, STRUCT, res=None):
+    from rpython.jit.backend.llsupport import symbolic
+
+    if res is None:
+        res = []
+    # order is not relevant, except for tests
+    for name, FIELD in STRUCT._flds.iteritems():
+        if isinstance(FIELD, lltype.Ptr) and FIELD._needsgc():
+            offset, _ = symbolic.get_field_token(STRUCT, name,
+                                                gccache.translate_support_code)
+            res.append(offset)
+        elif isinstance(FIELD, lltype.Struct):
+            offsets_of_gcfields(gccache, FIELD, res)
+    if not we_are_translated():
+        res.sort()
+    return res
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -161,7 +161,6 @@
                 subbox = value.force_box(optforce)
                 op = ResOperation(rop.SETFIELD_GC, [box, subbox], None,
                                   descr=ofs)
-
                 optforce.emit_operation(op)
 
     def _get_field_descr_list(self):

From noreply at buildbot.pypy.org  Mon Sep 15 02:05:18 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 15 Sep 2014 02:05:18 +0200 (CEST)
Subject: [pypy-commit] pypy gc_no_cleanup_nursery: implement zero_ptr_field
 and eager clearing of fields
Message-ID: <20140915000518.E5F411C371B@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: gc_no_cleanup_nursery
Changeset: r73539:b1948ec73a81
Date: 2014-09-14 18:03 -0600
http://bitbucket.org/pypy/pypy/changeset/b1948ec73a81/

Log:	implement zero_ptr_field and eager clearing of fields

diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -61,7 +61,8 @@
                 continue
             elif op.getopnum() == rop.CLEAR_ARRAY_CONTENTS:
                 if not self.gc_ll_descr.malloc_zero_filled:
-                    self.handle_clear_array_contents(op)
+                    self.handle_clear_array_contents(op.getdescr(),
+                                                     op.getarg(0))
                 continue
             elif op.can_malloc():
                 self.emitting_an_operation_that_can_collect()
@@ -115,6 +116,18 @@
         else:
             raise NotImplementedError(op.getopname())
 
+    def clear_gc_fields(self, descr, result):
+        if self.gc_ll_descr.malloc_zero_filled:
+            return
+        for ofs in descr.offsets_of_gcfields:
+            o = ResOperation(rop.ZERO_PTR_FIELD, [result, ConstInt(ofs)], None)
+            self.newops.append(o)
+
+    def clear_varsize_gc_fields(self, descr, result, v_length=None):
+        if self.gc_ll_descr.malloc_zero_filled:
+            return
+        self.handle_clear_array_contents(descr, result, v_length)
+
     def handle_new_fixedsize(self, descr, op):
         assert isinstance(descr, SizeDescr)
         size = descr.size
@@ -122,6 +135,7 @@
             self.gen_initialize_tid(op.result, descr.tid)
         else:
             self.gen_malloc_fixedsize(size, descr.tid, op.result)
+        self.clear_gc_fields(descr, op.result)
 
     def handle_new_array(self, arraydescr, op, kind=FLAG_ARRAY):
         v_length = op.getarg(0)
@@ -143,6 +157,7 @@
             # might end up being allocated by malloc_external or some
             # stuff that initializes GC header fields differently
             self.gen_initialize_len(op.result, v_length, arraydescr.lendescr)
+            self.clear_varsize_gc_fields(op.getdescr(), op.result, v_length)
             return
         if (total_size >= 0 and
                 self.gen_malloc_nursery(total_size, op.result)):
@@ -160,19 +175,24 @@
                 self.gen_malloc_unicode(v_length, op.result)
             else:
                 raise NotImplementedError(op.getopname())
+        self.clear_varsize_gc_fields(op.getdescr(), op.result, v_length)
 
-    def handle_clear_array_contents(self, op):
+    def handle_clear_array_contents(self, arraydescr, v_arr, v_arrsize=None):
         # XXX this maybe should go to optimizer, so we can remove extra ops?
-        arraydescr = op.getdescr()
         ofs, size, _ = self.cpu.unpack_arraydescr_size(arraydescr)
-        v_arr = op.getarg(0)
         v_arr_plus_ofs = BoxInt()
         v_arrsize = BoxInt()
         v_totalsize = BoxInt()
         gcdescr = self.gc_ll_descr
         ops = [
             ResOperation(rop.INT_ADD, [v_arr, ConstInt(size)], v_arr_plus_ofs),
-            ResOperation(rop.ARRAYLEN_GC, [v_arr], v_arrsize, descr=arraydescr),
+        ]
+
+        if v_arrsize is None:
+            o = ResOperation(rop.ARRAYLEN_GC, [v_arr], v_arrsize,
+                             descr=arraydescr)
+            ops.append(o)
+        ops += [
             ResOperation(rop.INT_MUL, [v_arrsize, ConstInt(size)], v_totalsize),
             ResOperation(rop.CALL, [ConstInt(gcdescr.memset_ptr_as_int),
                                     v_arr_plus_ofs,
diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py
--- a/rpython/jit/backend/llsupport/test/test_rewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_rewrite.py
@@ -144,7 +144,7 @@
     def setup_method(self, meth):
         class FakeCPU(BaseFakeCPU):
             def sizeof(self, STRUCT):
-                return SizeDescrWithVTable(102)
+                return SizeDescrWithVTable(102, offsets_of_gcfields=[])
         self.cpu = FakeCPU()
         self.gc_ll_descr = GcLLDescr_boehm(None, None, None)
 
@@ -277,10 +277,11 @@
                                                really_not_translated=True)
         self.gc_ll_descr.write_barrier_descr.has_write_barrier_from_array = (
             lambda cpu: True)
+        self.gc_ll_descr.malloc_zero_filled = False
         #
         class FakeCPU(BaseFakeCPU):
             def sizeof(self, STRUCT):
-                descr = SizeDescrWithVTable(104)
+                descr = SizeDescrWithVTable(104, offsets_of_gcfields=[])
                 descr.tid = 9315
                 return descr
         self.cpu = FakeCPU()
@@ -311,6 +312,7 @@
             setfield_gc(p0, 1234, descr=tiddescr)
             p1 = int_add(p0, %(sdescr.size)d)
             setfield_gc(p1, 5678, descr=tiddescr)
+            zero_ptr_field(p1, %(tdescr.offsets_of_gcfields[0])s)
             p2 = int_add(p1, %(tdescr.size)d)
             setfield_gc(p2, 1234, descr=tiddescr)
             jump()
diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -4457,3 +4457,24 @@
             assert a[1] == 0
         finally:
             self.cpu.gc_ll_descr.malloc_zero_filled = oldval
+
+    def test_zero_ptr_field(self):
+        T = lltype.GcStruct('T')
+        S = lltype.GcStruct('S', ('x', lltype.Ptr(T)))
+        tdescr = self.cpu.sizeof(T)
+        sdescr = self.cpu.sizeof(S)
+        fielddescr = self.cpu.fielddescrof(S, 'x')
+        loop = parse("""
+        []
+        p0 = new(descr=tdescr)
+        p1 = new(descr=sdescr)
+        setfield_gc(p1, p0, descr=fielddescr)
+        zero_ptr_field(p1, %d)
+        finish(p1)
+        """ % fielddescr.offset, namespace=locals())
+        looptoken = JitCellToken()
+        self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken)
+        deadframe = self.cpu.execute_token(looptoken)
+        ref = self.cpu.get_ref_value(deadframe, 0)
+        s = lltype.cast_opaque_ptr(lltype.Ptr(S), ref)
+        assert not s.x
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -1486,6 +1486,8 @@
         dest_addr = AddressLoc(base_loc, ofs_loc)
         self.save_into_mem(dest_addr, value_loc, size_loc)
 
+    genop_discard_zero_ptr_field = genop_discard_setfield_gc
+
     def genop_discard_setinteriorfield_gc(self, op, arglocs):
         (base_loc, ofs_loc, itemsize_loc, fieldsize_loc,
             index_loc, temp_loc, value_loc) = arglocs
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -959,6 +959,13 @@
                                               need_lower_byte=need_lower_byte)
         self.perform_discard(op, [base_loc, ofs_loc, size_loc, value_loc])
 
+    def consider_zero_ptr_field(self, op):
+        ofs_loc = imm(op.getarg(1).getint())
+        size_loc = imm(WORD)
+        base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), [])
+        value_loc = imm(0)
+        self.perform_discard(op, [base_loc, ofs_loc, size_loc, value_loc])
+
     consider_setfield_raw = consider_setfield_gc
 
     def consider_setinteriorfield_gc(self, op):
diff --git a/rpython/jit/codewriter/heaptracker.py b/rpython/jit/codewriter/heaptracker.py
--- a/rpython/jit/codewriter/heaptracker.py
+++ b/rpython/jit/codewriter/heaptracker.py
@@ -139,6 +139,6 @@
             res.append(offset)
         elif isinstance(FIELD, lltype.Struct):
             offsets_of_gcfields(gccache, FIELD, res)
-    if not we_are_translated():
+    if not gccache.translate_support_code:
         res.sort()
     return res
diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py
--- a/rpython/jit/metainterp/executor.py
+++ b/rpython/jit/metainterp/executor.py
@@ -325,6 +325,7 @@
                          rop.INCREMENT_DEBUG_COUNTER,
                          rop.COND_CALL_GC_WB,
                          rop.COND_CALL_GC_WB_ARRAY,
+                         rop.ZERO_PTR_FIELD,
                          rop.DEBUG_MERGE_POINT,
                          rop.JIT_DEBUG,
                          rop.SETARRAYITEM_RAW,
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -504,6 +504,8 @@
     'SETINTERIORFIELD_RAW/3d',    # right now, only used by tests
     'RAW_STORE/3d',
     'SETFIELD_GC/2d',
+    'ZERO_PTR_FIELD/2', # only emitted by the rewrite, sets a pointer field
+                        # at a given offset, no descr
     'SETFIELD_RAW/2d',
     'STRSETITEM/3',
     'UNICODESETITEM/3',

From noreply at buildbot.pypy.org  Mon Sep 15 02:18:54 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Mon, 15 Sep 2014 02:18:54 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: fix test_mmap on linux32
Message-ID: <20140915001854.3F41D1C101E@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73540:7f465c1fe072
Date: 2014-09-15 00:17 +0000
http://bitbucket.org/pypy/pypy/changeset/7f465c1fe072/

Log:	fix test_mmap on linux32

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -65,7 +65,7 @@
 def llexternal(*args, **kwargs):
     return rffi.llexternal(*args, compilation_info=eci, **kwargs)
 
-c_fopen = llexternal('fopen', [rffi.CCHARP, rffi.CCHARP], FILEP)
+c_fopen = llexternal('fopen', [rffi.CCHARP, rffi.CCHARP], FILEP, macro=True)
 c_popen = llexternal('popen', [rffi.CCHARP, rffi.CCHARP], FILEP)
 c_fdopen = llexternal(('_' if os.name == 'nt' else '') + 'fdopen',
                       [rffi.INT, rffi.CCHARP], FILEP)

From noreply at buildbot.pypy.org  Mon Sep 15 03:08:45 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 15 Sep 2014 03:08:45 +0200 (CEST)
Subject: [pypy-commit] pypy gc_no_cleanup_nursery: ZERO_PTR_FIELD in
	test_ll_random
Message-ID: <20140915010845.6D57E1D29E6@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: gc_no_cleanup_nursery
Changeset: r73541:9708b696cace
Date: 2014-09-14 19:07 -0600
http://bitbucket.org/pypy/pypy/changeset/9708b696cace/

Log:	ZERO_PTR_FIELD in test_ll_random

diff --git a/rpython/jit/backend/test/test_ll_random.py b/rpython/jit/backend/test/test_ll_random.py
--- a/rpython/jit/backend/test/test_ll_random.py
+++ b/rpython/jit/backend/test/test_ll_random.py
@@ -95,7 +95,10 @@
             fields.append(('parent', rclass.OBJECT))
             kwds['hints'] = {'vtable': with_vtable._obj}
         for i in range(r.randrange(1, 5)):
-            TYPE = self.get_random_primitive_type(r)
+            if r.random() < 0.1:
+                TYPE = llmemory.GCREF
+            else:
+                TYPE = self.get_random_primitive_type(r)
             fields.append(('f%d' % i, TYPE))
         S = type('S%d' % self.counter, *fields, **kwds)
         self.counter += 1
@@ -246,13 +249,41 @@
             op = ResOperation(self.opnum, [v, c_vtable2], None)
             return op, False
 
+class ZeroPtrFieldOperation(test_random.AbstractOperation):
+    def field_descr(self, builder, r):
+        v, S = builder.get_structptr_var(r, )
+        names = S._names
+        if names[0] == 'parent':
+            names = names[1:]
+        choice = []
+        for name in names:
+            FIELD = getattr(S, name)
+            if isinstance(FIELD, lltype.Ptr) and FIELD._needsgc():
+                choice.append(name)
+        if not choice:
+            raise test_random.CannotProduceOperation
+        name = r.choice(choice)
+        descr = builder.cpu.fielddescrof(S, name)
+        return v, descr.offset
+        
+    def produce_into(self, builder, r):
+        v, offset = self.field_descr(builder, r)
+        builder.do(self.opnum, [v, ConstInt(offset)], None)
+
 class GetFieldOperation(test_random.AbstractOperation):
     def field_descr(self, builder, r):
         v, S = builder.get_structptr_var(r, )
         names = S._names
         if names[0] == 'parent':
             names = names[1:]
-        name = r.choice(names)
+        choice = []
+        for name in names:
+            FIELD = getattr(S, name)
+            if not isinstance(FIELD, lltype.Ptr):
+                choice.append(name)
+        if not choice:
+            raise test_random.CannotProduceOperation
+        name = r.choice(choice)
         descr = builder.cpu.fielddescrof(S, name)
         descr._random_info = 'cpu.fielddescrof(..., %r)' % (name,)
         descr._random_type = S
@@ -274,7 +305,14 @@
                                          array_of_structs=True)
         array = v.getref(lltype.Ptr(A))
         v_index = builder.get_index(len(array), r)
-        name = r.choice(A.OF._names)
+        choice = []
+        for name in A.OF._names:
+            FIELD = getattr(A.OF, name)
+            if not isinstance(FIELD, lltype.Ptr):
+                choice.append(name)
+        if not choice:
+            raise test_random.CannotProduceOperation
+        name = r.choice(choice)
         descr = builder.cpu.interiorfielddescrof(A, name)
         descr._random_info = 'cpu.interiorfielddescrof(..., %r)' % (name,)
         descr._random_type = A
@@ -682,6 +720,7 @@
     OPERATIONS.append(GetFieldOperation(rop.GETFIELD_GC))
     OPERATIONS.append(GetInteriorFieldOperation(rop.GETINTERIORFIELD_GC))
     OPERATIONS.append(SetFieldOperation(rop.SETFIELD_GC))
+    OPERATIONS.append(ZeroPtrFieldOperation(rop.ZERO_PTR_FIELD))
     OPERATIONS.append(SetInteriorFieldOperation(rop.SETINTERIORFIELD_GC))
     OPERATIONS.append(NewOperation(rop.NEW))
     OPERATIONS.append(NewOperation(rop.NEW_WITH_VTABLE))
diff --git a/rpython/jit/backend/test/test_random.py b/rpython/jit/backend/test/test_random.py
--- a/rpython/jit/backend/test/test_random.py
+++ b/rpython/jit/backend/test/test_random.py
@@ -52,10 +52,13 @@
 
     def do(self, opnum, argboxes, descr=None):
         self.fakemetainterp._got_exc = None
-        v_result = execute_nonspec(self.cpu, self.fakemetainterp,
-                                   opnum, argboxes, descr)
-        if isinstance(v_result, Const):
-            v_result = v_result.clonebox()
+        if opnum == rop.ZERO_PTR_FIELD:
+            v_result = None
+        else:
+            v_result = execute_nonspec(self.cpu, self.fakemetainterp,
+                                       opnum, argboxes, descr)
+            if isinstance(v_result, Const):
+                v_result = v_result.clonebox()
         self.loop.operations.append(ResOperation(opnum, argboxes, v_result,
                                                  descr))
         return v_result

From noreply at buildbot.pypy.org  Mon Sep 15 09:05:19 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 15 Sep 2014 09:05:19 +0200 (CEST)
Subject: [pypy-commit] pypy default: Issue #1865: missing keepalive for the
	list of arguments
Message-ID: <20140915070519.8360C1C371B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r73542:d0dd7df65ca3
Date: 2014-09-15 09:04 +0200
http://bitbucket.org/pypy/pypy/changeset/d0dd7df65ca3/

Log:	Issue #1865: missing keepalive for the list of arguments

diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py
--- a/pypy/module/_cffi_backend/ctypefunc.py
+++ b/pypy/module/_cffi_backend/ctypefunc.py
@@ -8,6 +8,7 @@
 from rpython.rlib.jit_libffi import (CIF_DESCRIPTION, CIF_DESCRIPTION_P,
     FFI_TYPE, FFI_TYPE_P, FFI_TYPE_PP, SIZE_OF_FFI_ARG)
 from rpython.rlib.objectmodel import we_are_translated, instantiate
+from rpython.rlib.objectmodel import keepalive_until_here
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
 
 from pypy.interpreter.error import OperationError, oefmt
@@ -160,6 +161,7 @@
                         raw_cdata = rffi.cast(rffi.CCHARPP, data)[0]
                         lltype.free(raw_cdata, flavor='raw')
             lltype.free(buffer, flavor='raw')
+            keepalive_until_here(args_w)
         return w_res
 
 def get_mustfree_flag(data):

From noreply at buildbot.pypy.org  Mon Sep 15 18:31:40 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 15 Sep 2014 18:31:40 +0200 (CEST)
Subject: [pypy-commit] creflect default: Pass the 004 tests
Message-ID: <20140915163140.B12141C3366@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r35:3dfc3004ad46
Date: 2014-09-15 18:31 +0200
http://bitbucket.org/cffi/creflect/changeset/3dfc3004ad46/

Log:	Pass the 004 tests

diff --git a/test/codegen/004.c b/test/codegen/004.c
--- a/test/codegen/004.c
+++ b/test/codegen/004.c
@@ -5,13 +5,13 @@
 int test004(char *r)
 {
     if (!r)
-        return 8 + 18 + 8 + 1;
+        return 8 + 18 + 10;
     {
         num_t *p1;
         char *p2;
         char b[sizeof(**p1)];  /* check that '*num_t' is a valid type */
         p1 = (void *)&p2;
-        *p1 = (void *)b;    /* check that 'num_t' is a pointer type, not array */
+        *p1 = (void *)b;    /* check that 'num_t' is a pointer type */
         (void)(**p1 << 1);  /* check that '*num_t' is an integer type */
         r += sprintf(r, "typedef ");
         **p1 = -1;
@@ -29,7 +29,7 @@
             else if (sizeof(**p1) == sizeof(unsigned long long))
                 r += sprintf(r, "unsigned long long");
             else
-                r += sprintf(r, "uint%lu_t", (long)sizeof(**p1) * 8);
+                r += sprintf(r, "uint%u_t", (int)sizeof(**p1) * 8);
         }
         else {
             if (sizeof(**p1) == sizeof(int))
@@ -43,9 +43,9 @@
             else if (sizeof(**p1) == sizeof(long long))
                 r += sprintf(r, "long long");
             else
-                r += sprintf(r, "int%lu_t", (long)sizeof(**p1) * 8);
+                r += sprintf(r, "int%u_t", (int)sizeof(**p1) * 8);
         }
+        r += sprintf(r, " *num_t;\n");
     }
-    r += sprintf(r, " *num_t;\n");
     return 0;
 }
diff --git a/test/codegen/004b.c b/test/codegen/004b.c
--- a/test/codegen/004b.c
+++ b/test/codegen/004b.c
@@ -5,17 +5,17 @@
 int test004b(char *r)
 {
     if (!r)
-        return 8 + 18 + 8 + 1;
-    r += sprintf(r, "typedef ");
+        return 8 + 18 + 11;
     {
         num_t *p1;
         char *p2;
         char *p3;
-        char b[sizeof(**p1)];  /* check that '**num_t' is a valid type */
+        char b[sizeof(***p1)];  /* check that '**num_t' is a valid type */
         p1 = (void *)&p2;
         *p1 = (void *)&p3;  /* check that 'num_t' is a pointer type */
         **p1 = (void *)b;    /* check that '*num_t' is a pointer type */
         (void)(***p1 << 1);  /* check that '**num_t' is an integer type */
+        r += sprintf(r, "typedef ");
         ***p1 = -1;
         if (***p1 > 0) {
             if (sizeof(***p1) == 1 && ***p1 == 1)
@@ -31,7 +31,7 @@
             else if (sizeof(***p1) == sizeof(unsigned long long))
                 r += sprintf(r, "unsigned long long");
             else
-                r += sprintf(r, "uint%lu_t", (long)sizeof(***p1) * 8);
+                r += sprintf(r, "uint%u_t", (int)sizeof(***p1) * 8);
         }
         else {
             if (sizeof(***p1) == sizeof(int))
@@ -45,9 +45,9 @@
             else if (sizeof(***p1) == sizeof(long long))
                 r += sprintf(r, "long long");
             else
-                r += sprintf(r, "int%lu_t", (long)sizeof(***p1) * 8);
+                r += sprintf(r, "int%u_t", (int)sizeof(***p1) * 8);
         }
+        r += sprintf(r, " **num_t;\n");
     }
-    r += sprintf(r, " **num_t;\n");
     return 0;
 }

From noreply at buildbot.pypy.org  Mon Sep 15 19:19:46 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 15 Sep 2014 19:19:46 +0200 (CEST)
Subject: [pypy-commit] creflect default: The test 005 passes
Message-ID: <20140915171946.27EA81C101E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r36:8a3909328cd4
Date: 2014-09-15 19:19 +0200
http://bitbucket.org/cffi/creflect/changeset/8a3909328cd4/

Log:	The test 005 passes

diff --git a/creflect/codegen.py b/creflect/codegen.py
--- a/creflect/codegen.py
+++ b/creflect/codegen.py
@@ -10,6 +10,7 @@
         self.codelines = []
         self.sprintfleft = ""
         self.sprintfright = ""
+        self.sprintfright_extra = ("", 0)
         self.sprintfmax = []
 
     def add_include(self, includename):
@@ -26,24 +27,29 @@
         assert '&' not in msg
         self.sprintfleft = self.sprintfleft + msg
 
-    def sprintf_add_right(self, msg):
+    def sprintf_add_right(self, msg, extra="", extralength=None):
         assert '&' not in msg
         self.sprintfright = msg + self.sprintfright
+        if extra:
+            assert extralength is not None, "specify 'extralength' explicitly"
+            extra = self.sprintfright_extra[0] + extra
+            extralength = self.sprintfright_extra[1] + extralength
+            self.sprintfright_extra = (extra, extralength)
 
     def sprintf_add_both_sides(self, marker):
         i = marker.index('&')
         self.sprintf_add_left(marker[:i])
         self.sprintf_add_right(marker[i+1:])
 
-    def sprintf(self, buf, extra="", maxlength=None, indent=0):
-        if maxlength is None:
-            assert not extra, "specify 'maxlength' explicitly"
-            maxlength = len(buf)
+    def sprintf(self, buf, extra="", extralength=None, indent=0):
+        if extralength is None:
+            assert not extra, "specify 'extralength' explicitly"
+            extralength = 0
         if extra:
             extra = ', ' + extra
         self.writeline('%sr += sprintf(r, "%s"%s);' %
                        (' ' * indent, buf, extra))
-        self.sprintfmax.append(maxlength)
+        self.sprintfmax.append(len(buf) + extralength)
 
     def flushleft(self):
         buf = self.sprintfleft
@@ -60,12 +66,18 @@
         self.sprintfmax[startindex:] = [m]
 
     def flush(self):
-        self.sprintfleft += self.sprintfright
-        self.sprintfright = ""
-        self.flushleft()
+        buf = self.sprintfleft + self.sprintfright
+        if buf:
+            self.sprintfleft = ""
+            self.sprintfright = ""
+            extra, extralength = self.sprintfright_extra
+            self.sprintfright_extra = ("", 0)
+            self.sprintf(buf, extra=extra, extralength=extralength)
 
     def get_skip_label(self):
-        self.skip_label_active = True
+        if not self.skip_label_active:
+            self.skip_label_active = True
+            self.writedecl('char *r0 = r;')
         return 'f%d' % (self.blocknum,)
 
     def close(self):
@@ -92,17 +104,24 @@
         self.skip_label_counter = 0
         self.skip_label_active = False
         self.block_counter = 0
+        self.return_value = False
 
     def next_block_num(self):
         r = self.block_counter
         self.block_counter = r + 1
         return r
 
+    def need_return_value(self):
+        if not self.return_value:
+            self.return_value = True
+            self.funcblock.writedecl("int r1 = 0;")
+
     def generate(self, funcname):
         outerblock = CodeBlock(self)
         outerblock.writeline('int %s(char *r)' % (funcname,))
 
         funcblock = CodeBlock(self)
+        self.funcblock = funcblock
 
         for decl in self.csource.get_declarations():
             decl.write_declaration(funcblock)
@@ -114,7 +133,10 @@
         else:
             funcblock.writedecl('    return 1;')
             funcblock.writedecl('*r = 0;')
-        funcblock.writeline('return 0;')
+        if self.return_value:
+            funcblock.writeline('return r1;')
+        else:
+            funcblock.writeline('return 0;')
 
         outerblock.write_subblock(funcblock)
         outerblock.writeline('')
diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -106,7 +106,7 @@
             block.sprintf(ut, indent=8)
         block.writeline("    else")
         block.sprintf("uint%u_t", "(int)sizeof(%s) * 8" % (star_p1,),
-                      maxlength=18, indent=8)
+                      extralength=10, indent=8)
         block.writeline("}")
         block.writeline("else {")
         elsecase = ""
@@ -119,7 +119,7 @@
             elsecase = "else "
         block.writeline("    else")
         block.sprintf("int%u_t", "(int)sizeof(%s) * 8" % (star_p1,),
-                      maxlength=18, indent=8)
+                      extralength=10, indent=8)
         block.writeline("}")
         block.end_disjoint_cases(disjunction)
         block.sprintf_add_left(" ")
@@ -176,29 +176,32 @@
         self.c_name_with_marker = (
             self.item.c_name_with_marker.replace('&', brackets))
 
-    def outer_decl(self, outer):
-        star_p1 = outer.fetch_star_p1()
+    def inspect_type(self, block, inspect):
+        star_p1 = inspect.fetch_star_p1()
         errmsg = "type '%s' is not an array, but a pointer type" % (
-            outer.get_comment_type(0, False),)
+            inspect.get_comment_type(0, False),)
         def after():
             ampersand_star_p1 = '&' + star_p1
             if ampersand_star_p1.startswith('&*'):
                 ampersand_star_p1 = ampersand_star_p1[2:]
-            outer.codelines.append("if ((void *)%s != (void *)%s) {" % (
+            block.writeline("if ((void *)%s != (void *)%s) {" % (
                 ampersand_star_p1, star_p1))
-            outer.codelines.append("    r(\"E:%s\", 0, 0);" % (errmsg,))
-            outer.codelines.append("    goto %s;" % (
-                outer.tr.get_skip_label(),))
-            outer.codelines.append("}")
+            block.writeline("    r = r0;")
+            block.sprintf(r"#error %s\n" % errmsg, indent=4)
+            block.tr.need_return_value()
+            block.writeline("    r1 = -1;")
+            block.writeline("    goto %s;" % block.get_skip_label())
+            block.writeline("}")
         def at_end():
             outer.codelines.append("a[%d] = (void *)(sizeof(%s) / sizeof(*%s));"
                                    % (outer.tr.fetch_next_a_index(),
                                       star_p1, star_p1))
-        outer.levels.append('[]')
-        outer.after_star_p1_assignment.append(after)
-        outer.at_end.append(at_end)
-        typeexpr = self.item.outer_decl(outer)
-        return typeexpr.replace('&', '&[?]')
+        inspect.levels.append('[]')
+        inspect.after_star_p1_assignment.append(after)
+        block.sprintf_add_right('[%lld]',
+            extra='(long long)(sizeof(*p1) / sizeof(**p1))',
+            extralength=20)
+        self.item.inspect_type(block, inspect)
 
 
 class StructOrUnionOrEnum(BaseTypeByIdentity):
diff --git a/test/codegen/005.c b/test/codegen/005.c
--- a/test/codegen/005.c
+++ b/test/codegen/005.c
@@ -4,22 +4,55 @@
 
 int test005(char *r)
 {
+    int r1 = 0;
     if (!r)
-        return 8 + 18 + 8 + 1;
+        return 57 + 8 + 18 + 35;
     {
         foo_t *p1;
         char b[sizeof(**p1)];  /* check that 'foo_t[]' is a valid type */
+        char *r0 = r;
         p1 = (void *)b;
         if ((void *)p1 != (void *)*p1) {
-            r("E:type 'foo_t' is not an array, but a pointer type", 0, 0);
-            goto f1;
+            r = r0;
+            r += sprintf(r, "#error type 'foo_t' is not an array, but a pointer type\n");
+            r1 = -1;
+            goto f2;
         }
         (void)(**p1 << 1);  /* check that 'foo_t[]' is an integer type */
         r += sprintf(r, "typedef ");
         **p1 = -1;
-        a[0] = (void *)(**p1 > 0 ? sizeof(**p1) : -sizeof(**p1));
-        a[1] = (void *)(sizeof(*p1) / sizeof(**p1));
+        if (**p1 > 0) {
+            if (sizeof(**p1) == 1 && **p1 == 1)
+                r += sprintf(r, "_Bool");
+            else if (sizeof(**p1) == sizeof(unsigned int))
+                r += sprintf(r, "unsigned int");
+            else if (sizeof(**p1) == sizeof(unsigned short))
+                r += sprintf(r, "unsigned short");
+            else if (sizeof(**p1) == sizeof(unsigned char))
+                r += sprintf(r, "unsigned char");
+            else if (sizeof(**p1) == sizeof(unsigned long))
+                r += sprintf(r, "unsigned long");
+            else if (sizeof(**p1) == sizeof(unsigned long long))
+                r += sprintf(r, "unsigned long long");
+            else
+                r += sprintf(r, "uint%u_t", (int)sizeof(**p1) * 8);
+        }
+        else {
+            if (sizeof(**p1) == sizeof(int))
+                r += sprintf(r, "int");
+            else if (sizeof(**p1) == sizeof(short))
+                r += sprintf(r, "short");
+            else if (sizeof(**p1) == sizeof(signed char))
+                r += sprintf(r, "signed char");
+            else if (sizeof(**p1) == sizeof(long))
+                r += sprintf(r, "long");
+            else if (sizeof(**p1) == sizeof(long long))
+                r += sprintf(r, "long long");
+            else
+                r += sprintf(r, "int%u_t", (int)sizeof(**p1) * 8);
+        }
+        r += sprintf(r, " foo_t[%lld];\n", (long long)(sizeof(*p1) / sizeof(**p1)));
+        f2:;
     }
-    r("=foo_t:int?[?]", a, 2);
-  f1:;
+    return r1;
 }

From noreply at buildbot.pypy.org  Mon Sep 15 19:43:25 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 15 Sep 2014 19:43:25 +0200 (CEST)
Subject: [pypy-commit] creflect default: 005d fails
Message-ID: <20140915174325.AAE221D26D3@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r37:c6573ab87077
Date: 2014-09-15 19:43 +0200
http://bitbucket.org/cffi/creflect/changeset/c6573ab87077/

Log:	005d fails

diff --git a/creflect/codegen.py b/creflect/codegen.py
--- a/creflect/codegen.py
+++ b/creflect/codegen.py
@@ -32,6 +32,8 @@
         self.sprintfright = msg + self.sprintfright
         if extra:
             assert extralength is not None, "specify 'extralength' explicitly"
+            if self.sprintfright_extra[0]:
+                extra = ', ' + extra
             extra = self.sprintfright_extra[0] + extra
             extralength = self.sprintfright_extra[1] + extralength
             self.sprintfright_extra = (extra, extralength)
diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -192,15 +192,12 @@
             block.writeline("    r1 = -1;")
             block.writeline("    goto %s;" % block.get_skip_label())
             block.writeline("}")
-        def at_end():
-            outer.codelines.append("a[%d] = (void *)(sizeof(%s) / sizeof(*%s));"
-                                   % (outer.tr.fetch_next_a_index(),
-                                      star_p1, star_p1))
+            block.sprintf_add_right('[%lld]',
+                extra='(long long)(sizeof(%s) / sizeof(*%s))' % (
+                    star_p1, star_p1),
+                extralength=20)
         inspect.levels.append('[]')
         inspect.after_star_p1_assignment.append(after)
-        block.sprintf_add_right('[%lld]',
-            extra='(long long)(sizeof(*p1) / sizeof(**p1))',
-            extralength=20)
         self.item.inspect_type(block, inspect)
 
 
diff --git a/test/codegen/005b.c b/test/codegen/005b.c
--- a/test/codegen/005b.c
+++ b/test/codegen/005b.c
@@ -2,27 +2,59 @@
 
 # ____________________________________________________________
 
-static void __creflect1(void r(char *, void**, int))
+int test005b(char *r)
 {
-    void *a[2];
-    __CREFLECT_PREV(r);
+    int r1 = 0;
+    if (!r)
+        return 57 + 8 + 18 + 36;
     {
         foo_t *p1;
         char *p3;
+        char *r0 = r;
         char b[sizeof(***p1)];  /* check that '*foo_t[]' is a valid type */
         p1 = (void *)&p3;
         if ((void *)p1 != (void *)*p1) {
-            r("E:type 'foo_t' is not an array, but a pointer type", 0, 0);
-            goto f1;
+            r = r0;
+            r += sprintf(r, "#error type 'foo_t' is not an array, but a pointer type\n");
+            r1 = -1;
+            goto f2;
         }
         **p1 = (void *)b;    /* check that 'foo_t[]' is a pointer type */
         (void)(***p1 << 1);  /* check that '*foo_t[]' is an integer type */
+        r += sprintf(r, "typedef ");
         ***p1 = -1;
-        a[0] = (void *)(***p1 > 0 ? sizeof(***p1) : -sizeof(***p1));
-        a[1] = (void *)(sizeof(*p1) / sizeof(**p1));
+        if (***p1 > 0) {
+            if (sizeof(***p1) == 1 && ***p1 == 1)
+                r += sprintf(r, "_Bool");
+            else if (sizeof(***p1) == sizeof(unsigned int))
+                r += sprintf(r, "unsigned int");
+            else if (sizeof(***p1) == sizeof(unsigned short))
+                r += sprintf(r, "unsigned short");
+            else if (sizeof(***p1) == sizeof(unsigned char))
+                r += sprintf(r, "unsigned char");
+            else if (sizeof(***p1) == sizeof(unsigned long))
+                r += sprintf(r, "unsigned long");
+            else if (sizeof(***p1) == sizeof(unsigned long long))
+                r += sprintf(r, "unsigned long long");
+            else
+                r += sprintf(r, "uint%u_t", (int)sizeof(***p1) * 8);
+        }
+        else {
+            if (sizeof(***p1) == sizeof(int))
+                r += sprintf(r, "int");
+            else if (sizeof(***p1) == sizeof(short))
+                r += sprintf(r, "short");
+            else if (sizeof(***p1) == sizeof(signed char))
+                r += sprintf(r, "signed char");
+            else if (sizeof(***p1) == sizeof(long))
+                r += sprintf(r, "long");
+            else if (sizeof(***p1) == sizeof(long long))
+                r += sprintf(r, "long long");
+            else
+                r += sprintf(r, "int%u_t", (int)sizeof(***p1) * 8);
+        }
+        r += sprintf(r, " *foo_t[%lld];\n", (long long)(sizeof(*p1) / sizeof(**p1)));
+        f2:;
     }
-    r("=foo_t:int?*[?]", a, 2);
-  f1:;
+    return r1;
 }
-
-#expect  =foo_t:int?*[?]  -4  27
diff --git a/test/codegen/005c.c b/test/codegen/005c.c
--- a/test/codegen/005c.c
+++ b/test/codegen/005c.c
@@ -2,27 +2,59 @@
 
 # ____________________________________________________________
 
-static void __creflect1(void r(char *, void**, int))
+int test005c(char *r)
 {
-    void *a[2];
-    __CREFLECT_PREV(r);
+    int r1 = 0;
+    if (!r)
+        return 58 + 8 + 18 + 38;
     {
         foo_t *p1;
         char *p2;
         char b[sizeof(***p1)];  /* check that '(*foo_t)[]' is a valid type */
+        char *r0 = r;
         p1 = (void *)&p2;
         *p1 = (void *)b;    /* check that 'foo_t' is a pointer type */
         if ((void *)*p1 != (void *)**p1) {
-            r("E:type '*foo_t' is not an array, but a pointer type", 0, 0);
-            goto f1;
+            r = r0;
+            r += sprintf(r, "#error type '*foo_t' is not an array, but a pointer type\n");
+            r1 = -1;
+            goto f2;
         }
         (void)(***p1 << 1);  /* check that '(*foo_t)[]' is an integer type */
+        r += sprintf(r, "typedef ");
         ***p1 = -1;
-        a[0] = (void *)(***p1 > 0 ? sizeof(***p1) : -sizeof(***p1));
-        a[1] = (void *)(sizeof(**p1) / sizeof(***p1));
+        if (***p1 > 0) {
+            if (sizeof(***p1) == 1 && ***p1 == 1)
+                r += sprintf(r, "_Bool");
+            else if (sizeof(***p1) == sizeof(unsigned int))
+                r += sprintf(r, "unsigned int");
+            else if (sizeof(***p1) == sizeof(unsigned short))
+                r += sprintf(r, "unsigned short");
+            else if (sizeof(***p1) == sizeof(unsigned char))
+                r += sprintf(r, "unsigned char");
+            else if (sizeof(***p1) == sizeof(unsigned long))
+                r += sprintf(r, "unsigned long");
+            else if (sizeof(***p1) == sizeof(unsigned long long))
+                r += sprintf(r, "unsigned long long");
+            else
+                r += sprintf(r, "uint%u_t", (int)sizeof(***p1) * 8);
+        }
+        else {
+            if (sizeof(***p1) == sizeof(int))
+                r += sprintf(r, "int");
+            else if (sizeof(***p1) == sizeof(short))
+                r += sprintf(r, "short");
+            else if (sizeof(***p1) == sizeof(signed char))
+                r += sprintf(r, "signed char");
+            else if (sizeof(***p1) == sizeof(long))
+                r += sprintf(r, "long");
+            else if (sizeof(***p1) == sizeof(long long))
+                r += sprintf(r, "long long");
+            else
+                r += sprintf(r, "int%u_t", (int)sizeof(***p1) * 8);
+        }
+        r += sprintf(r, " (*foo_t)[%lld];\n", (long long)(sizeof(**p1) / sizeof(***p1)));
+        f2:;
     }
-    r("=foo_t:int?(*)[?]", a, 2);
-  f1:;
+    return r1;
 }
-
-#expect  =foo_t:int?(*)[?]  -4  27
diff --git a/test/codegen/005d.c b/test/codegen/005d.c
--- a/test/codegen/005d.c
+++ b/test/codegen/005d.c
@@ -2,13 +2,15 @@
 
 # ____________________________________________________________
 
-static void __creflect1(void r(char *, void**, int))
+int test005d(char *r)
 {
-    void *a[5];
-    __CREFLECT_PREV(r);
+    int r1 = 0;
+    if (!r)
+        return 57 + 59 + 64 + 68 + 8 + 18 + 120;
     {
         foo_t *p1;
         char *p4;
+        char *r0 = r;
         char *p5;
         char *p6;
         char *p9;
@@ -16,36 +18,69 @@
         char b[sizeof(**********p1)];  /* check that '**(***foo_t[][])[][]' is a valid type */
         p1 = (void *)&p4;
         if ((void *)p1 != (void *)*p1) {
-            r("E:type 'foo_t' is not an array, but a pointer type", 0, 0);
-            goto f1;
+            r = r0;
+            r += sprintf(r, "#error type 'foo_t' is not an array, but a pointer type\n");
+            r1 = -1;
+            goto f2;
         }
         if ((void *)*p1 != (void *)**p1) {
-            r("E:type 'foo_t[]' is not an array, but a pointer type", 0, 0);
-            goto f1;
+            r = r0;
+            r += sprintf(r, "#error type 'foo_t[]' is not an array, but a pointer type\n");
+            r1 = -1;
+            goto f2;
         }
         ***p1 = (void *)&p5;  /* check that 'foo_t[][]' is a pointer type */
         ****p1 = (void *)&p6;  /* check that '*foo_t[][]' is a pointer type */
         *****p1 = (void *)&p9;  /* check that '**foo_t[][]' is a pointer type */
         if ((void *)*****p1 != (void *)******p1) {
-            r("E:type '***foo_t[][]' is not an array, but a pointer type", 0, 0);
-            goto f1;
+            r = r0;
+            r += sprintf(r, "#error type '***foo_t[][]' is not an array, but a pointer type\n");
+            r1 = -1;
+            goto f2;
         }
         if ((void *)******p1 != (void *)*******p1) {
-            r("E:type '(***foo_t[][])[]' is not an array, but a pointer type", 0, 0);
-            goto f1;
+            r = r0;
+            r += sprintf(r, "#error type '(***foo_t[][])[]' is not an array, but a pointer type\n");
+            r1 = -1;
+            goto f2;
         }
         ********p1 = (void *)&p10;  /* check that '(***foo_t[][])[][]' is a pointer type */
         *********p1 = (void *)b;    /* check that '*(***foo_t[][])[][]' is a pointer type */
         (void)(**********p1 << 1);  /* check that '**(***foo_t[][])[][]' is an integer type */
+        r += sprintf(r, "typedef ");
         **********p1 = -1;
-        a[0] = (void *)(**********p1 > 0 ? sizeof(**********p1) : -sizeof(**********p1));
-        a[1] = (void *)(sizeof(*p1) / sizeof(**p1));
-        a[2] = (void *)(sizeof(**p1) / sizeof(***p1));
-        a[3] = (void *)(sizeof(******p1) / sizeof(*******p1));
-        a[4] = (void *)(sizeof(*******p1) / sizeof(********p1));
+        if (**********p1 > 0) {
+            if (sizeof(**********p1) == 1 && **********p1 == 1)
+                r += sprintf(r, "_Bool");
+            else if (sizeof(**********p1) == sizeof(unsigned int))
+                r += sprintf(r, "unsigned int");
+            else if (sizeof(**********p1) == sizeof(unsigned short))
+                r += sprintf(r, "unsigned short");
+            else if (sizeof(**********p1) == sizeof(unsigned char))
+                r += sprintf(r, "unsigned char");
+            else if (sizeof(**********p1) == sizeof(unsigned long))
+                r += sprintf(r, "unsigned long");
+            else if (sizeof(**********p1) == sizeof(unsigned long long))
+                r += sprintf(r, "unsigned long long");
+            else
+                r += sprintf(r, "uint%u_t", (int)sizeof(**********p1) * 8);
+        }
+        else {
+            if (sizeof(**********p1) == sizeof(int))
+                r += sprintf(r, "int");
+            else if (sizeof(**********p1) == sizeof(short))
+                r += sprintf(r, "short");
+            else if (sizeof(**********p1) == sizeof(signed char))
+                r += sprintf(r, "signed char");
+            else if (sizeof(**********p1) == sizeof(long))
+                r += sprintf(r, "long");
+            else if (sizeof(**********p1) == sizeof(long long))
+                r += sprintf(r, "long long");
+            else
+                r += sprintf(r, "int%u_t", (int)sizeof(**********p1) * 8);
+        }
+        r += sprintf(r, " **(***foo_t[%lld][%lld])[%lld][%lld];\n", (long long)(sizeof(*p1) / sizeof(**p1)), (long long)(sizeof(**p1) / sizeof(***p1)), (long long)(sizeof(******p1) / sizeof(*******p1)), (long long)(sizeof(*******p1) / sizeof(********p1)));
+        f2:;
     }
-    r("=foo_t:int?**(***[?][?])[?][?]", a, 5);
-  f1:;
+    return r1;
 }
-
-#expect  =foo_t:int?**(***[?][?])[?][?]  -4  4  3  27  91
diff --git a/test/test_cgcompile.py b/test/test_cgcompile.py
--- a/test/test_cgcompile.py
+++ b/test/test_cgcompile.py
@@ -43,7 +43,6 @@
     for (i = 0; i <= r+3; i++)
         buffer[r] = 0xfd;
     r2 = TESTFN(buffer);
-    assert(r2 == 0);
     for (i = 0; i < r; i++) {
         assert(buffer[i] != (char)0xfd);
         if (buffer[i] == 0)
@@ -52,7 +51,7 @@
     assert(i < r);
     assert(i == 0 || buffer[i-1] == '\n');
     printf("%s", buffer);
-    return 0;
+    return (r2 != 0);
 }
 ''')
     f.close()

From noreply at buildbot.pypy.org  Mon Sep 15 19:50:34 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 15 Sep 2014 19:50:34 +0200 (CEST)
Subject: [pypy-commit] creflect default: Fix 005d
Message-ID: <20140915175034.CDFF01D2750@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r38:bd7d04b35494
Date: 2014-09-15 19:50 +0200
http://bitbucket.org/cffi/creflect/changeset/bd7d04b35494/

Log:	Fix 005d

diff --git a/creflect/codegen.py b/creflect/codegen.py
--- a/creflect/codegen.py
+++ b/creflect/codegen.py
@@ -33,10 +33,10 @@
         if extra:
             assert extralength is not None, "specify 'extralength' explicitly"
             if self.sprintfright_extra[0]:
-                extra = ', ' + extra
-            extra = self.sprintfright_extra[0] + extra
-            extralength = self.sprintfright_extra[1] + extralength
-            self.sprintfright_extra = (extra, extralength)
+                extra += ', '
+            self.sprintfright_extra = (
+                extra + self.sprintfright_extra[0],
+                extralength + self.sprintfright_extra[1])
 
     def sprintf_add_both_sides(self, marker):
         i = marker.index('&')
diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -192,13 +192,13 @@
             block.writeline("    r1 = -1;")
             block.writeline("    goto %s;" % block.get_skip_label())
             block.writeline("}")
-            block.sprintf_add_right('[%lld]',
-                extra='(long long)(sizeof(%s) / sizeof(*%s))' % (
-                    star_p1, star_p1),
-                extralength=20)
         inspect.levels.append('[]')
         inspect.after_star_p1_assignment.append(after)
         self.item.inspect_type(block, inspect)
+        block.sprintf_add_right('[%lld]',
+            extra='(long long)(sizeof(%s) / sizeof(*%s))' % (
+                star_p1, star_p1),
+            extralength=20)
 
 
 class StructOrUnionOrEnum(BaseTypeByIdentity):

From noreply at buildbot.pypy.org  Mon Sep 15 19:54:06 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 15 Sep 2014 19:54:06 +0200 (CEST)
Subject: [pypy-commit] creflect default: 005e passes
Message-ID: <20140915175406.81E961D2750@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r39:fb367039b305
Date: 2014-09-15 19:54 +0200
http://bitbucket.org/cffi/creflect/changeset/fb367039b305/

Log:	005e passes

diff --git a/test/codegen/005e.c b/test/codegen/005e.c
--- a/test/codegen/005e.c
+++ b/test/codegen/005e.c
@@ -1,42 +1,106 @@
-typedef int foo_t[55], bar_t[66];
+typedef int foo_t[55];
+typedef int bar_t[66];
 
 # ____________________________________________________________
 
-static void __creflect1(void r(char *, void**, int))
+int test005e(char *r)
 {
-    void *a[2];
-    __CREFLECT_PREV(r);
+    int r1 = 0;
+    if (!r)
+        return 57 + 8 + 18 + 35 + 57 + 8 + 18 + 35;
     {
         foo_t *p1;
         char b[sizeof(**p1)];  /* check that 'foo_t[]' is a valid type */
+        char *r0 = r;
         p1 = (void *)b;
         if ((void *)p1 != (void *)*p1) {
-            r("E:type 'foo_t' is not an array, but a pointer type", 0, 0);
-            goto f1;
+            r = r0;
+            r += sprintf(r, "#error type 'foo_t' is not an array, but a pointer type\n");
+            r1 = -1;
+            goto f2;
         }
         (void)(**p1 << 1);  /* check that 'foo_t[]' is an integer type */
+        r += sprintf(r, "typedef ");
         **p1 = -1;
-        a[0] = (void *)(**p1 > 0 ? sizeof(**p1) : -sizeof(**p1));
-        a[1] = (void *)(sizeof(*p1) / sizeof(**p1));
+        if (**p1 > 0) {
+            if (sizeof(**p1) == 1 && **p1 == 1)
+                r += sprintf(r, "_Bool");
+            else if (sizeof(**p1) == sizeof(unsigned int))
+                r += sprintf(r, "unsigned int");
+            else if (sizeof(**p1) == sizeof(unsigned short))
+                r += sprintf(r, "unsigned short");
+            else if (sizeof(**p1) == sizeof(unsigned char))
+                r += sprintf(r, "unsigned char");
+            else if (sizeof(**p1) == sizeof(unsigned long))
+                r += sprintf(r, "unsigned long");
+            else if (sizeof(**p1) == sizeof(unsigned long long))
+                r += sprintf(r, "unsigned long long");
+            else
+                r += sprintf(r, "uint%u_t", (int)sizeof(**p1) * 8);
+        }
+        else {
+            if (sizeof(**p1) == sizeof(int))
+                r += sprintf(r, "int");
+            else if (sizeof(**p1) == sizeof(short))
+                r += sprintf(r, "short");
+            else if (sizeof(**p1) == sizeof(signed char))
+                r += sprintf(r, "signed char");
+            else if (sizeof(**p1) == sizeof(long))
+                r += sprintf(r, "long");
+            else if (sizeof(**p1) == sizeof(long long))
+                r += sprintf(r, "long long");
+            else
+                r += sprintf(r, "int%u_t", (int)sizeof(**p1) * 8);
+        }
+        r += sprintf(r, " foo_t[%lld];\n", (long long)(sizeof(*p1) / sizeof(**p1)));
+        f2:;
     }
-    r("=foo_t:int?[?]", a, 2);
-  f1:;
     {
         bar_t *p1;
         char b[sizeof(**p1)];  /* check that 'bar_t[]' is a valid type */
+        char *r0 = r;
         p1 = (void *)b;
         if ((void *)p1 != (void *)*p1) {
-            r("E:type 'bar_t' is not an array, but a pointer type", 0, 0);
-            goto f2;
+            r = r0;
+            r += sprintf(r, "#error type 'bar_t' is not an array, but a pointer type\n");
+            r1 = -1;
+            goto f3;
         }
         (void)(**p1 << 1);  /* check that 'bar_t[]' is an integer type */
+        r += sprintf(r, "typedef ");
         **p1 = -1;
-        a[0] = (void *)(**p1 > 0 ? sizeof(**p1) : -sizeof(**p1));
-        a[1] = (void *)(sizeof(*p1) / sizeof(**p1));
+        if (**p1 > 0) {
+            if (sizeof(**p1) == 1 && **p1 == 1)
+                r += sprintf(r, "_Bool");
+            else if (sizeof(**p1) == sizeof(unsigned int))
+                r += sprintf(r, "unsigned int");
+            else if (sizeof(**p1) == sizeof(unsigned short))
+                r += sprintf(r, "unsigned short");
+            else if (sizeof(**p1) == sizeof(unsigned char))
+                r += sprintf(r, "unsigned char");
+            else if (sizeof(**p1) == sizeof(unsigned long))
+                r += sprintf(r, "unsigned long");
+            else if (sizeof(**p1) == sizeof(unsigned long long))
+                r += sprintf(r, "unsigned long long");
+            else
+                r += sprintf(r, "uint%u_t", (int)sizeof(**p1) * 8);
+        }
+        else {
+            if (sizeof(**p1) == sizeof(int))
+                r += sprintf(r, "int");
+            else if (sizeof(**p1) == sizeof(short))
+                r += sprintf(r, "short");
+            else if (sizeof(**p1) == sizeof(signed char))
+                r += sprintf(r, "signed char");
+            else if (sizeof(**p1) == sizeof(long))
+                r += sprintf(r, "long");
+            else if (sizeof(**p1) == sizeof(long long))
+                r += sprintf(r, "long long");
+            else
+                r += sprintf(r, "int%u_t", (int)sizeof(**p1) * 8);
+        }
+        r += sprintf(r, " bar_t[%lld];\n", (long long)(sizeof(*p1) / sizeof(**p1)));
+        f3:;
     }
-    r("=bar_t:int?[?]", a, 2);
-  f2:;
+    return r1;
 }
-
-#expect  =foo_t:int?[?]  -4  55
-#expect  =bar_t:int?[?]  -4  66

From noreply at buildbot.pypy.org  Mon Sep 15 20:28:13 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 15 Sep 2014 20:28:13 +0200 (CEST)
Subject: [pypy-commit] creflect default: Start support for 'const'
Message-ID: <20140915182813.F389C1C0492@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r40:604ad41bc138
Date: 2014-09-15 20:27 +0200
http://bitbucket.org/cffi/creflect/changeset/604ad41bc138/

Log:	Start support for 'const'

diff --git a/creflect/codegen.py b/creflect/codegen.py
--- a/creflect/codegen.py
+++ b/creflect/codegen.py
@@ -145,7 +145,7 @@
         #
         if outerblock.includes:
             extra = ['#include <%s>' % includename
-                     for includename in self.includes]
+                     for includename in outerblock.includes]
             extra.append('')
         else:
             extra = []
diff --git a/creflect/commontypes.py b/creflect/commontypes.py
--- a/creflect/commontypes.py
+++ b/creflect/commontypes.py
@@ -19,7 +19,7 @@
     '_Bool':              'i',
     }
 
-def resolve_common_type(names):
+def resolve_common_type(names, const):
     # reduce synonyms to a single chosen combination
     names = list(names)
     if names != ['signed', 'char']:    # keep this unmodified
@@ -44,8 +44,11 @@
         names = newnames + names
     ident = ' '.join(names)
     if ident == 'void':
-        return model.void_type
+        if const:
+            return model.const_void_type
+        else:
+            return model.void_type
     if ident == '__dotdotdot__':   # XXX
         raise api.FFIError(':%d: bad usage of "..."' %
                 typenode.coord.line)
-    return model.PrimitiveType(ident)
+    return model.PrimitiveType(ident, const)
diff --git a/creflect/cparser.py b/creflect/cparser.py
--- a/creflect/cparser.py
+++ b/creflect/cparser.py
@@ -136,11 +136,9 @@
             realtype = self._get_type(decl.type, name=decl.name)
         self.declarations.append(model.TypeDef(decl.name, realtype))
 
-    def _get_type_pointer(self, type, const=False):
+    def _get_type_pointer(self, type):
         if isinstance(type, model.RawFunctionType):
             return type.as_function_pointer()
-        if const:
-            return model.ConstPointerType(type)
         return model.PointerType(type)
 
     def _get_type(self, typenode, name=None, partial_length_ok=False):
@@ -164,15 +162,14 @@
         #
         if isinstance(typenode, pycparser.c_ast.PtrDecl):
             # pointer type
-            const = (isinstance(typenode.type, pycparser.c_ast.TypeDecl)
-                     and 'const' in typenode.type.quals)
-            return self._get_type_pointer(self._get_type(typenode.type), const)
+            return self._get_type_pointer(self._get_type(typenode.type))
         #
         if isinstance(typenode, pycparser.c_ast.TypeDecl):
+            const = 'const' in typenode.quals
             type = typenode.type
             if isinstance(type, pycparser.c_ast.IdentifierType):
                 # assume a primitive type.
-                return resolve_common_type(type.names)
+                return resolve_common_type(type.names, const)
             #
             if isinstance(type, pycparser.c_ast.Struct):
                 # 'struct foobar'
diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -11,7 +11,8 @@
         return '<%s>' % (self._get_c_name(),)
 
     def _get_items(self):
-        return [(name, getattr(self, name)) for name in self._attrs_]
+        return [(name, getattr(self, name))
+                for name in (self._attrs_ + 'const')]
 
     def inspect_type(self, block, inspect):
         block.sprintf_add_both_sides(self.c_name_with_marker)
@@ -32,14 +33,20 @@
 
 class VoidType(BaseType):
     _attrs_ = ()
-    c_name_with_marker = 'void &'
+
+    def __init__(self, const):
+        self.const = const
+        self.c_name_with_marker = 'void &'
+        if const:
+            self.c_name_with_marker = 'const ' + self.c_name_with_marker
 
     def inspect_type(self, block, inspect):
         if inspect.started:
             inspect.assign_to_p1('0')
         BaseType.inspect_type(self, block, inspect)
 
-void_type = VoidType()
+void_type = VoidType(const=False)
+const_void_type = VoidType(const=True)
 
 
 class PrimitiveType(BaseType):
@@ -63,10 +70,13 @@
         '_Bool':              'i',
         }
 
-    def __init__(self, name):
+    def __init__(self, name, const):
         assert name in self.ALL_PRIMITIVE_TYPES
         self.name = name
+        self.const = const
         self.c_name_with_marker = name + ' &'
+        if const:
+            self.c_name_with_marker = 'const ' + self.c_name_with_marker
 
     def is_char_type(self):
         return self.ALL_PRIMITIVE_TYPES[self.name] == 'c'
@@ -79,11 +89,18 @@
         star_p1 = inspect.fetch_star_p1()
         comment1 = inspect.get_comment(0, False, "a valid type", minlevel=1)
         comment2 = inspect.get_comment(0, False, "an integer type")
+        comment3 = inspect.get_comment(0, False, "not declared 'const'")
         block.writedecl("char b[sizeof(%s)];%s" % (star_p1, comment1))
         inspect.assign_to_p1("b")
         block.writeline("(void)(%s << 1);%s" % (star_p1, comment2))
+        if self.const:
+            block.sprintf_add_left("const ")
         disjunction = block.flushleft()
-        block.writeline("%s = -1;" % (star_p1,))
+        if self.const:
+            block.add_include("string.h")
+            block.writeline("memset(b, -1, sizeof(b));")
+        else:
+            block.writeline("%s = -1;%s" % (star_p1, comment3))
         block.writeline("if (%s > 0) {" % (star_p1,))
         block.writeline("    if (sizeof(%s) == 1 && %s == 1)" %
                         (star_p1, star_p1))
diff --git a/test/codegen/002c.c b/test/codegen/002c.c
new file mode 100644
--- /dev/null
+++ b/test/codegen/002c.c
@@ -0,0 +1,17 @@
+typedef const void *num_t;
+
+# ____________________________________________________________
+
+int test002c(char *r)
+{
+    if (!r)
+        return 28;
+    {
+        num_t *p1;
+        char *p2;
+        p1 = (void *)&p2;
+        *p1 = (void *)0;    /* check that 'num_t' is a pointer type */
+        r += sprintf(r, "typedef const void *num_t;\n");
+    }
+    return 0;
+}
diff --git a/test/codegen/003.c b/test/codegen/003.c
--- a/test/codegen/003.c
+++ b/test/codegen/003.c
@@ -12,7 +12,7 @@
         p1 = (void *)b;
         (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
         r += sprintf(r, "typedef ");
-        *p1 = -1;
+        *p1 = -1;  /* check that 'num_t' is not declared 'const' */
         if (*p1 > 0) {
             if (sizeof(*p1) == 1 && *p1 == 1)
                 r += sprintf(r, "_Bool");
diff --git a/test/codegen/003b.c b/test/codegen/003b.c
--- a/test/codegen/003b.c
+++ b/test/codegen/003b.c
@@ -12,7 +12,7 @@
         p1 = (void *)b;
         (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
         r += sprintf(r, "typedef ");
-        *p1 = -1;
+        *p1 = -1;  /* check that 'num_t' is not declared 'const' */
         if (*p1 > 0) {
             if (sizeof(*p1) == 1 && *p1 == 1)
                 r += sprintf(r, "_Bool");
diff --git a/test/codegen/003c.c b/test/codegen/003c.c
--- a/test/codegen/003c.c
+++ b/test/codegen/003c.c
@@ -12,7 +12,7 @@
         p1 = (void *)b;
         (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
         r += sprintf(r, "typedef ");
-        *p1 = -1;
+        *p1 = -1;  /* check that 'num_t' is not declared 'const' */
         if (*p1 > 0) {
             if (sizeof(*p1) == 1 && *p1 == 1)
                 r += sprintf(r, "_Bool");
diff --git a/test/codegen/003d.c b/test/codegen/003d.c
--- a/test/codegen/003d.c
+++ b/test/codegen/003d.c
@@ -12,7 +12,7 @@
         p1 = (void *)b;
         (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
         r += sprintf(r, "typedef ");
-        *p1 = -1;
+        *p1 = -1;  /* check that 'num_t' is not declared 'const' */
         if (*p1 > 0) {
             if (sizeof(*p1) == 1 && *p1 == 1)
                 r += sprintf(r, "_Bool");
diff --git a/test/codegen/003e.c b/test/codegen/003e.c
--- a/test/codegen/003e.c
+++ b/test/codegen/003e.c
@@ -12,7 +12,7 @@
         p1 = (void *)b;
         (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
         r += sprintf(r, "typedef ");
-        *p1 = -1;
+        *p1 = -1;  /* check that 'num_t' is not declared 'const' */
         if (*p1 > 0) {
             if (sizeof(*p1) == 1 && *p1 == 1)
                 r += sprintf(r, "_Bool");
diff --git a/test/codegen/003f.c b/test/codegen/003f.c
--- a/test/codegen/003f.c
+++ b/test/codegen/003f.c
@@ -12,7 +12,7 @@
         p1 = (void *)b;
         (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
         r += sprintf(r, "typedef ");
-        *p1 = -1;
+        *p1 = -1;  /* check that 'num_t' is not declared 'const' */
         if (*p1 > 0) {
             if (sizeof(*p1) == 1 && *p1 == 1)
                 r += sprintf(r, "_Bool");
diff --git a/test/codegen/004.c b/test/codegen/004.c
--- a/test/codegen/004.c
+++ b/test/codegen/004.c
@@ -14,7 +14,7 @@
         *p1 = (void *)b;    /* check that 'num_t' is a pointer type */
         (void)(**p1 << 1);  /* check that '*num_t' is an integer type */
         r += sprintf(r, "typedef ");
-        **p1 = -1;
+        **p1 = -1;  /* check that '*num_t' is not declared 'const' */
         if (**p1 > 0) {
             if (sizeof(**p1) == 1 && **p1 == 1)
                 r += sprintf(r, "_Bool");
diff --git a/test/codegen/004b.c b/test/codegen/004b.c
--- a/test/codegen/004b.c
+++ b/test/codegen/004b.c
@@ -16,7 +16,7 @@
         **p1 = (void *)b;    /* check that '*num_t' is a pointer type */
         (void)(***p1 << 1);  /* check that '**num_t' is an integer type */
         r += sprintf(r, "typedef ");
-        ***p1 = -1;
+        ***p1 = -1;  /* check that '**num_t' is not declared 'const' */
         if (***p1 > 0) {
             if (sizeof(***p1) == 1 && ***p1 == 1)
                 r += sprintf(r, "_Bool");
diff --git a/test/codegen/005.c b/test/codegen/005.c
--- a/test/codegen/005.c
+++ b/test/codegen/005.c
@@ -20,7 +20,7 @@
         }
         (void)(**p1 << 1);  /* check that 'foo_t[]' is an integer type */
         r += sprintf(r, "typedef ");
-        **p1 = -1;
+        **p1 = -1;  /* check that 'foo_t[]' is not declared 'const' */
         if (**p1 > 0) {
             if (sizeof(**p1) == 1 && **p1 == 1)
                 r += sprintf(r, "_Bool");
diff --git a/test/codegen/005b.c b/test/codegen/005b.c
--- a/test/codegen/005b.c
+++ b/test/codegen/005b.c
@@ -22,7 +22,7 @@
         **p1 = (void *)b;    /* check that 'foo_t[]' is a pointer type */
         (void)(***p1 << 1);  /* check that '*foo_t[]' is an integer type */
         r += sprintf(r, "typedef ");
-        ***p1 = -1;
+        ***p1 = -1;  /* check that '*foo_t[]' is not declared 'const' */
         if (***p1 > 0) {
             if (sizeof(***p1) == 1 && ***p1 == 1)
                 r += sprintf(r, "_Bool");
diff --git a/test/codegen/005c.c b/test/codegen/005c.c
--- a/test/codegen/005c.c
+++ b/test/codegen/005c.c
@@ -22,7 +22,7 @@
         }
         (void)(***p1 << 1);  /* check that '(*foo_t)[]' is an integer type */
         r += sprintf(r, "typedef ");
-        ***p1 = -1;
+        ***p1 = -1;  /* check that '(*foo_t)[]' is not declared 'const' */
         if (***p1 > 0) {
             if (sizeof(***p1) == 1 && ***p1 == 1)
                 r += sprintf(r, "_Bool");
diff --git a/test/codegen/005d.c b/test/codegen/005d.c
--- a/test/codegen/005d.c
+++ b/test/codegen/005d.c
@@ -48,7 +48,7 @@
         *********p1 = (void *)b;    /* check that '*(***foo_t[][])[][]' is a pointer type */
         (void)(**********p1 << 1);  /* check that '**(***foo_t[][])[][]' is an integer type */
         r += sprintf(r, "typedef ");
-        **********p1 = -1;
+        **********p1 = -1;  /* check that '**(***foo_t[][])[][]' is not declared 'const' */
         if (**********p1 > 0) {
             if (sizeof(**********p1) == 1 && **********p1 == 1)
                 r += sprintf(r, "_Bool");
diff --git a/test/codegen/005e.c b/test/codegen/005e.c
--- a/test/codegen/005e.c
+++ b/test/codegen/005e.c
@@ -21,7 +21,7 @@
         }
         (void)(**p1 << 1);  /* check that 'foo_t[]' is an integer type */
         r += sprintf(r, "typedef ");
-        **p1 = -1;
+        **p1 = -1;  /* check that 'foo_t[]' is not declared 'const' */
         if (**p1 > 0) {
             if (sizeof(**p1) == 1 && **p1 == 1)
                 r += sprintf(r, "_Bool");
@@ -68,7 +68,7 @@
         }
         (void)(**p1 << 1);  /* check that 'bar_t[]' is an integer type */
         r += sprintf(r, "typedef ");
-        **p1 = -1;
+        **p1 = -1;  /* check that 'bar_t[]' is not declared 'const' */
         if (**p1 > 0) {
             if (sizeof(**p1) == 1 && **p1 == 1)
                 r += sprintf(r, "_Bool");
diff --git a/test/codegen/006.c b/test/codegen/006.c
new file mode 100644
--- /dev/null
+++ b/test/codegen/006.c
@@ -0,0 +1,50 @@
+typedef const unsigned int num_t;
+
+# ____________________________________________________________
+#include 
+
+int test006(char *r)
+{
+    if (!r)
+        return 14 + 18 + 9;
+    {
+        num_t *p1;
+        char b[sizeof(*p1)];
+        p1 = (void *)b;
+        (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
+        r += sprintf(r, "typedef const ");
+        memset(b, -1, sizeof(b));
+        if (*p1 > 0) {
+            if (sizeof(*p1) == 1 && *p1 == 1)
+                r += sprintf(r, "_Bool");
+            else if (sizeof(*p1) == sizeof(unsigned int))
+                r += sprintf(r, "unsigned int");
+            else if (sizeof(*p1) == sizeof(unsigned short))
+                r += sprintf(r, "unsigned short");
+            else if (sizeof(*p1) == sizeof(unsigned char))
+                r += sprintf(r, "unsigned char");
+            else if (sizeof(*p1) == sizeof(unsigned long))
+                r += sprintf(r, "unsigned long");
+            else if (sizeof(*p1) == sizeof(unsigned long long))
+                r += sprintf(r, "unsigned long long");
+            else
+                r += sprintf(r, "uint%u_t", (int)sizeof(*p1) * 8);
+        }
+        else {
+            if (sizeof(*p1) == sizeof(int))
+                r += sprintf(r, "int");
+            else if (sizeof(*p1) == sizeof(short))
+                r += sprintf(r, "short");
+            else if (sizeof(*p1) == sizeof(signed char))
+                r += sprintf(r, "signed char");
+            else if (sizeof(*p1) == sizeof(long))
+                r += sprintf(r, "long");
+            else if (sizeof(*p1) == sizeof(long long))
+                r += sprintf(r, "long long");
+            else
+                r += sprintf(r, "int%u_t", (int)sizeof(*p1) * 8);
+        }
+        r += sprintf(r, " num_t;\n");
+    }
+    return 0;
+}

From noreply at buildbot.pypy.org  Mon Sep 15 20:35:53 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 15 Sep 2014 20:35:53 +0200 (CEST)
Subject: [pypy-commit] pypy gc_no_cleanup_nursery: small fixes
Message-ID: <20140915183553.0C3D21C0492@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: gc_no_cleanup_nursery
Changeset: r73543:45d9a5a7c5f5
Date: 2014-09-15 12:34 -0600
http://bitbucket.org/pypy/pypy/changeset/45d9a5a7c5f5/

Log:	small fixes

diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -5,7 +5,8 @@
 from rpython.jit.metainterp.resoperation import ResOperation, rop
 from rpython.jit.codewriter import heaptracker
 from rpython.jit.backend.llsupport.symbolic import WORD
-from rpython.jit.backend.llsupport.descr import SizeDescr, ArrayDescr
+from rpython.jit.backend.llsupport.descr import SizeDescr, ArrayDescr,\
+     FLAG_POINTER
 from rpython.jit.metainterp.history import JitCellToken
 
 FLAG_ARRAY = 0
@@ -60,9 +61,7 @@
                 self.handle_malloc_operation(op)
                 continue
             elif op.getopnum() == rop.CLEAR_ARRAY_CONTENTS:
-                if not self.gc_ll_descr.malloc_zero_filled:
-                    self.handle_clear_array_contents(op.getdescr(),
-                                                     op.getarg(0))
+                self.handle_clear_array_contents(op.getdescr(), op.getarg(0))
                 continue
             elif op.can_malloc():
                 self.emitting_an_operation_that_can_collect()
@@ -126,6 +125,8 @@
     def clear_varsize_gc_fields(self, descr, result, v_length=None):
         if self.gc_ll_descr.malloc_zero_filled:
             return
+        if not descr.is_array_of_pointers():
+            return
         self.handle_clear_array_contents(descr, result, v_length)
 
     def handle_new_fixedsize(self, descr, op):
@@ -157,7 +158,8 @@
             # might end up being allocated by malloc_external or some
             # stuff that initializes GC header fields differently
             self.gen_initialize_len(op.result, v_length, arraydescr.lendescr)
-            self.clear_varsize_gc_fields(op.getdescr(), op.result, v_length)
+            if kind == FLAG_ARRAY:
+                self.clear_varsize_gc_fields(op.getdescr(), op.result, v_length)
             return
         if (total_size >= 0 and
                 self.gen_malloc_nursery(total_size, op.result)):
@@ -175,7 +177,8 @@
                 self.gen_malloc_unicode(v_length, op.result)
             else:
                 raise NotImplementedError(op.getopname())
-        self.clear_varsize_gc_fields(op.getdescr(), op.result, v_length)
+        if kind == FLAG_ARRAY:
+            self.clear_varsize_gc_fields(op.getdescr(), op.result, v_length)
 
     def handle_clear_array_contents(self, arraydescr, v_arr, v_arrsize=None):
         # XXX this maybe should go to optimizer, so we can remove extra ops?

From noreply at buildbot.pypy.org  Mon Sep 15 20:53:45 2014
From: noreply at buildbot.pypy.org (mjacob)
Date: Mon, 15 Sep 2014 20:53:45 +0200 (CEST)
Subject: [pypy-commit] pypy improve-docs: Fix reference.
Message-ID: <20140915185345.70FDF1D24F6@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: improve-docs
Changeset: r73544:7c9843cd4958
Date: 2014-09-15 20:52 +0200
http://bitbucket.org/pypy/pypy/changeset/7c9843cd4958/

Log:	Fix reference.

diff --git a/pypy/doc/install.rst b/pypy/doc/install.rst
--- a/pypy/doc/install.rst
+++ b/pypy/doc/install.rst
@@ -75,4 +75,4 @@
 ~~~~~~~~~~~~~~~~~~~~~~
 
 If you're interested in getting more involved, or doing something different with
-PyPy, consult `the build instructions `.
+PyPy, consult :doc:`the build instructions `.

From noreply at buildbot.pypy.org  Mon Sep 15 20:56:30 2014
From: noreply at buildbot.pypy.org (mjacob)
Date: Mon, 15 Sep 2014 20:56:30 +0200 (CEST)
Subject: [pypy-commit] pypy improve-docs: Change wording a bit for style and
	consistency.
Message-ID: <20140915185630.87CA21D24F6@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: improve-docs
Changeset: r73545:7c0472302251
Date: 2014-09-15 20:55 +0200
http://bitbucket.org/pypy/pypy/changeset/7c0472302251/

Log:	Change wording a bit for style and consistency.

diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -7,7 +7,7 @@
 If you prefer to compile your own PyPy, or if you want to modify it, you
 will need to obtain a copy of the sources.  This can be done either by
 `downloading them from the download page`_ or by checking them out from the
-repository using mercurial.  We suggest using mercurial if one wants to access
+repository using mercurial.  We suggest using mercurial if you want to access
 the current development.
 
 .. _downloading them from the download page: http://pypy.org/download.html

From noreply at buildbot.pypy.org  Mon Sep 15 21:00:13 2014
From: noreply at buildbot.pypy.org (mjacob)
Date: Mon, 15 Sep 2014 21:00:13 +0200 (CEST)
Subject: [pypy-commit] pypy improve-docs: Move these paragraphs up.
Message-ID: <20140915190013.B4DD61D24F6@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: improve-docs
Changeset: r73546:460039f9b847
Date: 2014-09-15 20:58 +0200
http://bitbucket.org/pypy/pypy/changeset/460039f9b847/

Log:	Move these paragraphs up.

diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -1,6 +1,16 @@
 Building PyPy from Source
 =========================
 
+For building PyPy, we recommend installing a pre-built PyPy first (see
+:doc:`install`). It is possible to build PyPy with CPython, but it will take a
+lot longer to run -- depending on your architecture, between two and three
+times as long.
+
+Even when using PyPy to build PyPy, translation is time-consuming -- 30
+minutes on a fast machine -- and RAM-hungry.  You will need **at least** 2 GB
+of memory on a 32-bit machine and 4GB on a 64-bit machine.
+
+
 Clone the repository
 --------------------
 
@@ -35,16 +45,6 @@
 .. _our nightly tests: http://buildbot.pypy.org/summary?branch=
 
 
-For building PyPy, we recommend installing a pre-built PyPy first (see
-:doc:`install`). It is possible to build PyPy with CPython, but it will take a
-lot longer to run -- depending on your architecture, between two and three
-times as long.
-
-Even when using PyPy to build PyPy, translation is time-consuming -- 30
-minutes on a fast machine -- and RAM-hungry.  You will need **at least** 2 GB
-of memory on a 32-bit machine and 4GB on a 64-bit machine.
-
-
 Install build-time dependencies
 -------------------------------
 

From noreply at buildbot.pypy.org  Mon Sep 15 21:14:13 2014
From: noreply at buildbot.pypy.org (mjacob)
Date: Mon, 15 Sep 2014 21:14:13 +0200 (CEST)
Subject: [pypy-commit] pypy improve-docs: Fix heading.
Message-ID: <20140915191413.683141C023E@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: improve-docs
Changeset: r73547:5e89c19edc37
Date: 2014-09-15 21:04 +0200
http://bitbucket.org/pypy/pypy/changeset/5e89c19edc37/

Log:	Fix heading.

diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -133,7 +133,7 @@
 
 .. _`Software Transactional Memory`: stm.html
 
---------------------------------------------------
+
 Is PyPy more clever than CPython about Tail Calls?
 --------------------------------------------------
 

From noreply at buildbot.pypy.org  Mon Sep 15 21:14:14 2014
From: noreply at buildbot.pypy.org (mjacob)
Date: Mon, 15 Sep 2014 21:14:14 +0200 (CEST)
Subject: [pypy-commit] pypy improve-docs: Use :doc: reST role.
Message-ID: <20140915191414.9C0851C023E@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: improve-docs
Changeset: r73548:653b6044022b
Date: 2014-09-15 21:12 +0200
http://bitbucket.org/pypy/pypy/changeset/653b6044022b/

Log:	Use :doc: reST role.

diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -126,13 +126,11 @@
 programmer).
 
 Instead, since 2012, there is work going on on a still very experimental
-`Software Transactional Memory`_ (STM) version of PyPy.  This should give
+:doc:`Software Transactional Memory ` (STM) version of PyPy.  This should give
 an alternative PyPy which works without a GIL, while at the same time
 continuing to give the Python programmer the complete illusion of having
 one.
 
-.. _`Software Transactional Memory`: stm.html
-
 
 Is PyPy more clever than CPython about Tail Calls?
 --------------------------------------------------

From noreply at buildbot.pypy.org  Mon Sep 15 22:13:02 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 15 Sep 2014 22:13:02 +0200 (CEST)
Subject: [pypy-commit] pypy default: ARMv6 fix
Message-ID: <20140915201302.6D2881C023E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r73549:a9ed7770eda6
Date: 2014-09-15 22:12 +0200
http://bitbucket.org/pypy/pypy/changeset/a9ed7770eda6/

Log:	ARMv6 fix

diff --git a/rpython/jit/backend/arm/callbuilder.py b/rpython/jit/backend/arm/callbuilder.py
--- a/rpython/jit/backend/arm/callbuilder.py
+++ b/rpython/jit/backend/arm/callbuilder.py
@@ -92,7 +92,8 @@
             self.mc.LDR_ri(r.r7.value, r.r5.value)
 
         # change 'rpy_fastgil' to 0 (it should be non-zero right now)
-        self.mc.DMB()
+        if self.asm.cpu.cpuinfo.arch_version >= 7:
+            self.mc.DMB()
         self.mc.gen_load_int(r.r6.value, fastgil)
         self.mc.MOV_ri(r.ip.value, 0)
         self.mc.STR_ri(r.ip.value, r.r6.value)
@@ -112,7 +113,8 @@
         self.mc.STREX(r.r3.value, r.ip.value, r.r6.value, c=c.EQ)
                                                  # try to claim the lock
         self.mc.CMP_ri(r.r3.value, 0, cond=c.EQ) # did this succeed?
-        self.mc.DMB()
+        if self.asm.cpu.cpuinfo.arch_version >= 7:
+            self.mc.DMB()
         # the success of the lock acquisition is defined by
         # 'EQ is true', or equivalently by 'r3 == 0'.
         #
diff --git a/rpython/jit/backend/arm/codebuilder.py b/rpython/jit/backend/arm/codebuilder.py
--- a/rpython/jit/backend/arm/codebuilder.py
+++ b/rpython/jit/backend/arm/codebuilder.py
@@ -333,6 +333,8 @@
                     | (rn & 0xF) << 16)
 
     def DMB(self):
+        # ARMv7 only.  I guess ARMv6 CPUs cannot be used in symmetric
+        # multi-processing at all? That would make this instruction unneeded.
         # note: 'cond' is only permitted on Thumb here, but don't
         # write literally 0xf57ff05f, because it's larger than 31 bits
         c = cond.AL
diff --git a/rpython/jit/backend/arm/instructions.py b/rpython/jit/backend/arm/instructions.py
--- a/rpython/jit/backend/arm/instructions.py
+++ b/rpython/jit/backend/arm/instructions.py
@@ -142,6 +142,7 @@
     #'VCVT' : {'opc1':0xB, 'opc2':0xE, 'opc3':0x1, 'base': False},
 }
 
+# ARMv7 only
 simd_instructions_3regs = {
     'VADD_i64': {'A': 0x8, 'B': 0, 'U': 0},
     'VSUB_i64': {'A': 0x8, 'B': 0, 'U': 1},

From noreply at buildbot.pypy.org  Tue Sep 16 09:13:38 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 16 Sep 2014 09:13:38 +0200 (CEST)
Subject: [pypy-commit] creflect default: Const pointers
Message-ID: <20140916071338.266F41C32CA@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r41:a913f8ec3a30
Date: 2014-09-16 09:13 +0200
http://bitbucket.org/cffi/creflect/changeset/a913f8ec3a30/

Log:	Const pointers

diff --git a/creflect/cparser.py b/creflect/cparser.py
--- a/creflect/cparser.py
+++ b/creflect/cparser.py
@@ -136,10 +136,10 @@
             realtype = self._get_type(decl.type, name=decl.name)
         self.declarations.append(model.TypeDef(decl.name, realtype))
 
-    def _get_type_pointer(self, type):
+    def _get_type_pointer(self, type, const):
         if isinstance(type, model.RawFunctionType):
             return type.as_function_pointer()
-        return model.PointerType(type)
+        return model.PointerType(type, const)
 
     def _get_type(self, typenode, name=None, partial_length_ok=False):
         # first, dereference typedefs, if we have it already parsed, we're good
@@ -162,7 +162,8 @@
         #
         if isinstance(typenode, pycparser.c_ast.PtrDecl):
             # pointer type
-            return self._get_type_pointer(self._get_type(typenode.type))
+            const = 'const' in typenode.quals
+            return self._get_type_pointer(self._get_type(typenode.type), const)
         #
         if isinstance(typenode, pycparser.c_ast.TypeDecl):
             const = 'const' in typenode.quals
diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -150,16 +150,18 @@
 
 class PointerType(BaseType):
     _attrs_ = ('totype',)
-    _base_pattern       = "*&"
-    _base_pattern_array = "(*&)"
+    _base_pattern       = {
+        (False, False): "*&",
+        (True, False): "(*&)",
+        (False, True): "*const &",
+        (True, True): "(*const &)",
+        }
 
-    def __init__(self, totype):
+    def __init__(self, totype, const):
         self.totype = totype
+        self.const = const
         base = self.totype.c_name_with_marker
-        if self.totype.is_array_type:
-            extra = self._base_pattern_array
-        else:
-            extra = self._base_pattern
+        extra = self._base_pattern[self.totype.is_array_type, self.const]
         self.c_name_with_marker = base.replace('&', extra)
 
     def inspect_type(self, block, inspect):
@@ -167,12 +169,28 @@
         new_var = 'p%d' % (len(inspect.levels) + 2,)
         block.writedecl("char *%s;" % (new_var,))
         inspect.assign_to_p1("&%s" % (new_var,))
+        #
+        if self.const:
+            errmsg = "type '%s' is not a pointer, but an array type" % (
+                inspect.get_comment_type(0, False),)
+            inspect.assign_target = new_var
+            def after():
+                ampersand_star_p1 = '&' + star_p1
+                if ampersand_star_p1.startswith('&*'):
+                    ampersand_star_p1 = ampersand_star_p1[2:]
+                block.writeline("if ((void *)%s == (void *)%s) {" % (
+                    ampersand_star_p1, star_p1))
+                block.writeline("    r = r0;")
+                block.sprintf(r"#error %s\n" % errmsg, indent=4)
+                block.tr.need_return_value()
+                block.writeline("    r1 = -1;")
+                block.writeline("    goto %s;" % block.get_skip_label())
+                block.writeline("}")
+            inspect.after_star_p1_assignment.append(after)
         inspect.levels.append('*')
         self.totype.inspect_type(block, inspect)
-        if self.totype.is_array_type:
-            extra = self._base_pattern_array
-        else:
-            extra = self._base_pattern
+        #
+        extra = self._base_pattern[self.totype.is_array_type, self.const]
         block.sprintf_add_both_sides(extra)
 
 
@@ -292,6 +310,7 @@
             self.started = True
             self.after_star_p1_assignment = []
             self.at_end = []
+            self.assign_target = None
 
     def get_comment_type(self, levels_delta, ignore_array, minlevel=0):
         end = len(self.levels)
@@ -333,20 +352,25 @@
         return star_p1
 
     def assign_to_p1(self, expr):
-        length = len(self.levels)
-        while length > 0 and self.levels[length - 1] == '[]':
-            length -= 1
-        comment = self.get_comment(-1, True, "a pointer type")
-        spaces = comment and (' ' * (3 - len(expr)))
+        if self.assign_target is None:
+            length = len(self.levels)
+            while length > 0 and self.levels[length - 1] == '[]':
+                length -= 1
+            comment = self.get_comment(-1, True, "a pointer type")
+            spaces = comment and (' ' * (3 - len(expr)))
+            self.assign_target = '%sp1' % ('*' * length,)
+        else:
+            spaces = comment = ''
         if self.structfield is None:
-            line = '%sp1 = (void *)%s;%s%s' % ('*' * length, expr, spaces,
-                                               comment)
+            line = '%s = (void *)%s;%s%s' % (self.assign_target, expr, spaces,
+                                             comment)
         else:
             if length > 0:
                 line = '%sp1->%s = (void *)%s;%s%s' % (
                     '*' * (length - 1), self.structfield, expr, spaces, comment)
             else:
                 line = 'p1 = (void *)(((char *)%s) - (long)o);' % (expr,)
+        self.assign_target = None
         self.block.writeline(line.rstrip())
         for fn in self.after_star_p1_assignment:
             fn()
diff --git a/test/codegen/006b.c b/test/codegen/006b.c
new file mode 100644
--- /dev/null
+++ b/test/codegen/006b.c
@@ -0,0 +1,60 @@
+typedef int *const num_t;
+
+# ____________________________________________________________
+
+int test006b(char *r)
+{
+    int r1 = 0;
+    if (!r)
+        return 57 + 8 + 18 + 16;
+    {
+        num_t *p1;
+        char *p2;
+        char b[sizeof(**p1)];  /* check that '*num_t' is a valid type */
+        char *r0 = r;
+        p1 = (void *)&p2;
+        p2 = (void *)b;
+        if ((void *)p1 == (void *)*p1) {
+            r = r0;
+            r += sprintf(r, "#error type 'num_t' is not a pointer, but an array type\n");
+            r1 = -1;
+            goto f2;
+        }
+        (void)(**p1 << 1);  /* check that '*num_t' is an integer type */
+        r += sprintf(r, "typedef ");
+        **p1 = -1;  /* check that '*num_t' is not declared 'const' */
+        if (**p1 > 0) {
+            if (sizeof(**p1) == 1 && **p1 == 1)
+                r += sprintf(r, "_Bool");
+            else if (sizeof(**p1) == sizeof(unsigned int))
+                r += sprintf(r, "unsigned int");
+            else if (sizeof(**p1) == sizeof(unsigned short))
+                r += sprintf(r, "unsigned short");
+            else if (sizeof(**p1) == sizeof(unsigned char))
+                r += sprintf(r, "unsigned char");
+            else if (sizeof(**p1) == sizeof(unsigned long))
+                r += sprintf(r, "unsigned long");
+            else if (sizeof(**p1) == sizeof(unsigned long long))
+                r += sprintf(r, "unsigned long long");
+            else
+                r += sprintf(r, "uint%u_t", (int)sizeof(**p1) * 8);
+        }
+        else {
+            if (sizeof(**p1) == sizeof(int))
+                r += sprintf(r, "int");
+            else if (sizeof(**p1) == sizeof(short))
+                r += sprintf(r, "short");
+            else if (sizeof(**p1) == sizeof(signed char))
+                r += sprintf(r, "signed char");
+            else if (sizeof(**p1) == sizeof(long))
+                r += sprintf(r, "long");
+            else if (sizeof(**p1) == sizeof(long long))
+                r += sprintf(r, "long long");
+            else
+                r += sprintf(r, "int%u_t", (int)sizeof(**p1) * 8);
+        }
+        r += sprintf(r, " *const num_t;\n");
+        f2:;
+    }
+    return r1;
+}
diff --git a/test/codegen/006c.c b/test/codegen/006c.c
new file mode 100644
--- /dev/null
+++ b/test/codegen/006c.c
@@ -0,0 +1,66 @@
+typedef int (*const foo_t)[27];
+
+# ____________________________________________________________
+
+int test006c(char *r)
+{
+    int r1 = 0;
+    if (!r)
+        return 57 + 58 + 8 + 18 + 44;
+    {
+        foo_t *p1;
+        char *p2;
+        char b[sizeof(***p1)];  /* check that '(*foo_t)[]' is a valid type */
+        char *r0 = r;
+        p1 = (void *)&p2;
+        p2 = (void *)b;
+        if ((void *)p1 == (void *)*p1) {
+            r = r0;
+            r += sprintf(r, "#error type 'foo_t' is not a pointer, but an array type\n");
+            r1 = -1;
+            goto f2;
+        }
+        if ((void *)*p1 != (void *)**p1) {
+            r = r0;
+            r += sprintf(r, "#error type '*foo_t' is not an array, but a pointer type\n");
+            r1 = -1;
+            goto f2;
+        }
+        (void)(***p1 << 1);  /* check that '(*foo_t)[]' is an integer type */
+        r += sprintf(r, "typedef ");
+        ***p1 = -1;  /* check that '(*foo_t)[]' is not declared 'const' */
+        if (***p1 > 0) {
+            if (sizeof(***p1) == 1 && ***p1 == 1)
+                r += sprintf(r, "_Bool");
+            else if (sizeof(***p1) == sizeof(unsigned int))
+                r += sprintf(r, "unsigned int");
+            else if (sizeof(***p1) == sizeof(unsigned short))
+                r += sprintf(r, "unsigned short");
+            else if (sizeof(***p1) == sizeof(unsigned char))
+                r += sprintf(r, "unsigned char");
+            else if (sizeof(***p1) == sizeof(unsigned long))
+                r += sprintf(r, "unsigned long");
+            else if (sizeof(***p1) == sizeof(unsigned long long))
+                r += sprintf(r, "unsigned long long");
+            else
+                r += sprintf(r, "uint%u_t", (int)sizeof(***p1) * 8);
+        }
+        else {
+            if (sizeof(***p1) == sizeof(int))
+                r += sprintf(r, "int");
+            else if (sizeof(***p1) == sizeof(short))
+                r += sprintf(r, "short");
+            else if (sizeof(***p1) == sizeof(signed char))
+                r += sprintf(r, "signed char");
+            else if (sizeof(***p1) == sizeof(long))
+                r += sprintf(r, "long");
+            else if (sizeof(***p1) == sizeof(long long))
+                r += sprintf(r, "long long");
+            else
+                r += sprintf(r, "int%u_t", (int)sizeof(***p1) * 8);
+        }
+        r += sprintf(r, " (*const foo_t)[%lld];\n", (long long)(sizeof(**p1) / sizeof(***p1)));
+        f2:;
+    }
+    return r1;
+}

From noreply at buildbot.pypy.org  Tue Sep 16 10:58:38 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 16 Sep 2014 10:58:38 +0200 (CEST)
Subject: [pypy-commit] creflect default: Starting to fix the struct tests
Message-ID: <20140916085838.B7A561C32CA@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r42:52a14bdda091
Date: 2014-09-16 10:58 +0200
http://bitbucket.org/cffi/creflect/changeset/52a14bdda091/

Log:	Starting to fix the struct tests

diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -1,7 +1,7 @@
 from .codegen import CodeBlock
 
 
-class BaseTypeByIdentity(object):
+class BaseType(object):
     is_array_type = False
 
     def _get_c_name(self):
@@ -17,9 +17,6 @@
     def inspect_type(self, block, inspect):
         block.sprintf_add_both_sides(self.c_name_with_marker)
 
-
-class BaseType(BaseTypeByIdentity):
-
     def __eq__(self, other):
         return (self.__class__ == other.__class__ and
                 self._get_items() == other._get_items())
@@ -197,6 +194,7 @@
 class ArrayType(BaseType):
     _attrs_ = ('item', 'length')
     is_array_type = True
+    const = True     # not really applicable, but must have one
 
     def __init__(self, item, length):
         self.item = item
@@ -236,57 +234,16 @@
             extralength=20)
 
 
-class StructOrUnionOrEnum(BaseTypeByIdentity):
-    _attrs_ = ('name',)
+class StructOrUnionOrEnum(BaseType):
+    _attrs_ = ('kind', 'name')
 
-
-class StructOrUnion(StructOrUnionOrEnum):
-
-    def __init__(self, name, fldnames, fldtypes, fldbitsize):
+    def __init__(self, kind, name, const):
+        self.kind = kind
         self.name = name
-        self.fldnames = fldnames
-        self.fldtypes = fldtypes
-        self.fldbitsize = fldbitsize
+        self.const = const
         self.c_name_with_marker = '%s %s&' % (self.kind, self.name)
-
-    def write_declaration(self, tr):
-        tr.reset_r_output()
-        typename = "%s %s" % (self.kind, self.name)
-        tr.write_msg(typename)
-        tr.write_section("a[%d] = (void *)sizeof(%s);  /* size */"
-                          % (tr.fetch_next_a_index(), typename), indent=4)
-        tr.write_section("a[%d] = &((struct{char a; %s b;}*)0)->b;  /* align */"
-                          % (tr.fetch_next_a_index(), typename), indent=4)
-        tr.flush()
-        for fldname, fldtype in zip(self.fldnames, self.fldtypes):
-            tr.reset_r_output()
-            tr.write_msg("{%s:" % (fldname,))
-            tr.fetch_next_a_index()   # ignored
-            outer = OuterDecl(tr, typename, fldname)
-            outer.start()
-            #
-            xtra = ''
-            checktype = fldtype
-            while isinstance(checktype, ArrayType):
-                xtra += '[0]'
-                checktype = checktype.item
-            if xtra:
-                tr.add_include('stddef.h')
-                outer.decllines.append("void *o = (void *)offsetof(%s, %s%s);"
-                                       "  /* offset */" %
-                                       (typename, fldname, xtra))
-            else:
-                outer.decllines.append("void *o = &((%s *)0)->%s;  /* offset */"
-                                       % (typename, fldname))
-            outer.codelines.append("a[%d] = o;" % (tr.fetch_next_a_index(),))
-            typeexpr = fldtype.outer_decl(outer)
-            outer.stop()
-            tr.write_msg(typeexpr.replace('&', ''))
-            tr.flush()
-
-
-class StructType(StructOrUnion):
-    kind = 'struct'
+        if const:
+            self.c_name_with_marker = 'const ' + self.c_name_with_marker
 
 
 # ____________________________________________________________
@@ -384,6 +341,50 @@
         del self.started
 
 
+class StructOrUnionDecl(object):
+
+    def __init__(self, type, fldnames, fldtypes, fldbitsize):
+        self.type = type
+        self.fldnames = fldnames
+        self.fldtypes = fldtypes
+        self.fldbitsize = fldbitsize
+
+    def write_declaration(self, tr):
+        tr.reset_r_output()
+        typename = "%s %s" % (self.kind, self.name)
+        tr.write_msg(typename)
+        tr.write_section("a[%d] = (void *)sizeof(%s);  /* size */"
+                          % (tr.fetch_next_a_index(), typename), indent=4)
+        tr.write_section("a[%d] = &((struct{char a; %s b;}*)0)->b;  /* align */"
+                          % (tr.fetch_next_a_index(), typename), indent=4)
+        tr.flush()
+        for fldname, fldtype in zip(self.fldnames, self.fldtypes):
+            tr.reset_r_output()
+            tr.write_msg("{%s:" % (fldname,))
+            tr.fetch_next_a_index()   # ignored
+            outer = OuterDecl(tr, typename, fldname)
+            outer.start()
+            #
+            xtra = ''
+            checktype = fldtype
+            while isinstance(checktype, ArrayType):
+                xtra += '[0]'
+                checktype = checktype.item
+            if xtra:
+                tr.add_include('stddef.h')
+                outer.decllines.append("void *o = (void *)offsetof(%s, %s%s);"
+                                       "  /* offset */" %
+                                       (typename, fldname, xtra))
+            else:
+                outer.decllines.append("void *o = &((%s *)0)->%s;  /* offset */"
+                                       % (typename, fldname))
+            outer.codelines.append("a[%d] = o;" % (tr.fetch_next_a_index(),))
+            typeexpr = fldtype.outer_decl(outer)
+            outer.stop()
+            tr.write_msg(typeexpr.replace('&', ''))
+            tr.flush()
+
+
 class TypeDef(object):
     def __init__(self, name, type):
         self.name = name
diff --git a/test/codegen/struct-001.c b/test/codegen/struct-001.c
--- a/test/codegen/struct-001.c
+++ b/test/codegen/struct-001.c
@@ -1,40 +1,95 @@
 struct foo_s {
-    int aa, bb;
+  int aa;
+  int bb;
 };
 
 # ____________________________________________________________
 
-static void __creflect1(void r(char *, void**, int))
+int teststruct_001(char *r)
 {
-    void *a[3];
-    __CREFLECT_PREV(r);
-    a[0] = (void *)sizeof(struct foo_s);  /* size */
-    a[1] = &((struct{char a; struct foo_s b;}*)0)->b;  /* align */
-    r("struct foo_s", a, 2);
+    if (!r)
+        return 57 + 8 + 18 + 16;
+    r += sprintf(r, "struct foo_s {/*%lld,%lld*/\n", (long long)sizeof(struct foo_s), (long long)(((char *)&((struct{char a; struct foo_s b;} *)0)->b) - (char *)0));
     {
+        long long o = ((char *)&((struct foo_s *)0)->aa) - (char *)0;  /* check that 'struct foo_s::aa' is not an array */
         struct foo_s *p1;
-        void *o = &((struct foo_s *)0)->aa;  /* offset */
         char b[sizeof(p1->aa)];
-        a[1] = o;
-        p1 = (void *)(((char *)b) - (long)o);
+        p1 = (void *)(((char *)b) - o);
         (void)(p1->aa << 1);  /* check that 'struct foo_s::aa' is an integer type */
-        p1->aa = -1;
-        a[2] = (void *)(p1->aa > 0 ? sizeof(p1->aa) : -sizeof(p1->aa));
+        r += sprintf(r, "  /*%lld*/", o);
+        p1->aa = -1;  /* check that 'struct foo_s::aa' is not declared 'const' */
+        if (p1->aa > 0) {
+            if (sizeof(p1->aa) == 1 && p1->aa == 1)
+                r += sprintf(r, "_Bool");
+            else if (sizeof(p1->aa) == sizeof(unsigned int))
+                r += sprintf(r, "unsigned int");
+            else if (sizeof(p1->aa) == sizeof(unsigned short))
+                r += sprintf(r, "unsigned short");
+            else if (sizeof(p1->aa) == sizeof(unsigned char))
+                r += sprintf(r, "unsigned char");
+            else if (sizeof(p1->aa) == sizeof(unsigned long))
+                r += sprintf(r, "unsigned long");
+            else if (sizeof(p1->aa) == sizeof(unsigned long long))
+                r += sprintf(r, "unsigned long long");
+            else
+                r += sprintf(r, "uint%u_t", (int)sizeof(p1->aa) * 8);
+        }
+        else {
+            if (sizeof(p1->aa) == sizeof(int))
+                r += sprintf(r, "int");
+            else if (sizeof(p1->aa) == sizeof(short))
+                r += sprintf(r, "short");
+            else if (sizeof(p1->aa) == sizeof(signed char))
+                r += sprintf(r, "signed char");
+            else if (sizeof(p1->aa) == sizeof(long))
+                r += sprintf(r, "long");
+            else if (sizeof(p1->aa) == sizeof(long long))
+                r += sprintf(r, "long long");
+            else
+                r += sprintf(r, "int%u_t", (int)sizeof(p1->aa) * 8);
+        }
+        r += sprintf(r, " aa;\n");
     }
-    r("{aa:int?", a, 3);
     {
+        long long o = ((char *)&((struct foo_s *)0)->bb) - (char *)0;  /* check that 'struct foo_s::bb' is not an array */
         struct foo_s *p1;
-        void *o = &((struct foo_s *)0)->bb;  /* offset */
         char b[sizeof(p1->bb)];
-        a[1] = o;
-        p1 = (void *)(((char *)b) - (long)o);
+        p1 = (void *)(((char *)b) - o);
         (void)(p1->bb << 1);  /* check that 'struct foo_s::bb' is an integer type */
-        p1->bb = -1;
-        a[2] = (void *)(p1->bb > 0 ? sizeof(p1->bb) : -sizeof(p1->bb));
+        r += sprintf(r, "  /*%lld*/", o);
+        p1->bb = -1;  /* check that 'struct foo_s::bb' is not declared 'const' */
+        if (p1->bb > 0) {
+            if (sizeof(p1->bb) == 1 && p1->bb == 1)
+                r += sprintf(r, "_Bool");
+            else if (sizeof(p1->bb) == sizeof(unsigned int))
+                r += sprintf(r, "unsigned int");
+            else if (sizeof(p1->bb) == sizeof(unsigned short))
+                r += sprintf(r, "unsigned short");
+            else if (sizeof(p1->bb) == sizeof(unsigned char))
+                r += sprintf(r, "unsigned char");
+            else if (sizeof(p1->bb) == sizeof(unsigned long))
+                r += sprintf(r, "unsigned long");
+            else if (sizeof(p1->bb) == sizeof(unsigned long long))
+                r += sprintf(r, "unsigned long long");
+            else
+                r += sprintf(r, "uint%u_t", (int)sizeof(p1->bb) * 8);
+        }
+        else {
+            if (sizeof(p1->bb) == sizeof(int))
+                r += sprintf(r, "int");
+            else if (sizeof(p1->bb) == sizeof(short))
+                r += sprintf(r, "short");
+            else if (sizeof(p1->bb) == sizeof(signed char))
+                r += sprintf(r, "signed char");
+            else if (sizeof(p1->bb) == sizeof(long))
+                r += sprintf(r, "long");
+            else if (sizeof(p1->bb) == sizeof(long long))
+                r += sprintf(r, "long long");
+            else
+                r += sprintf(r, "int%u_t", (int)sizeof(p1->bb) * 8);
+        }
+        r += sprintf(r, " bb;\n");
     }
-    r("{bb:int?", a, 3);
+    r += sprintf(r, "};\n");
+    return 0;
 }
-
-#expect  struct foo_s  8  4
-#expect  {aa:int?  99  0  -4
-#expect  {bb:int?  99  4  -4
diff --git a/test/test_cgcompile.py b/test/test_cgcompile.py
--- a/test/test_cgcompile.py
+++ b/test/test_cgcompile.py
@@ -63,7 +63,7 @@
     lines = g.readlines()
     err = g.close()
     assert not err
-    r_remove_addr = re.compile(r"/[*]-?0x[0-9a-f]+[*]/")
+    r_remove_addr = re.compile(r"/[*][-0-9a-fx,]+[*]/")
     got = [r_remove_addr.sub('', line.strip()) for line in lines]
     compare_lists(got, expected)
 

From noreply at buildbot.pypy.org  Tue Sep 16 11:32:55 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 16 Sep 2014 11:32:55 +0200 (CEST)
Subject: [pypy-commit] creflect default: Pass struct-001
Message-ID: <20140916093255.CE9A51D260E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r43:183e7881542b
Date: 2014-09-16 11:33 +0200
http://bitbucket.org/cffi/creflect/changeset/183e7881542b/

Log:	Pass struct-001

diff --git a/creflect/cparser.py b/creflect/cparser.py
--- a/creflect/cparser.py
+++ b/creflect/cparser.py
@@ -54,8 +54,7 @@
     def __init__(self, cdefblock, first_lineno=1):
         self.cdefblock = cdefblock
         self.first_lineno = first_lineno
-        self._structnode2type = {}
-        self._structunionenum = {}
+        self._struct_union_enum_decl = {}
 
     def to_ast(self):
         cparser = pycparser.CParser()
@@ -99,16 +98,16 @@
             tp = self._get_type_pointer(tp)
             self._declare('function ' + decl.name, tp)
         else:
+            const = 'const' in decl.quals
             if isinstance(node, pycparser.c_ast.Struct):
-                # XXX do we need self._declare in any of those?
                 if node.decls is not None:
-                    self._get_struct_union_enum_type('struct', node)
+                    self._get_struct_union_enum_type('struct', node, const)
             elif isinstance(node, pycparser.c_ast.Union):
                 if node.decls is not None:
-                    self._get_struct_union_enum_type('union', node)
+                    self._get_struct_union_enum_type('union', node, const)
             elif isinstance(node, pycparser.c_ast.Enum):
                 if node.values is not None:
-                    self._get_struct_union_enum_type('enum', node)
+                    self._get_struct_union_enum_type('enum', node, const)
             elif not decl.name:
                 raise api.CDefError("construct does not declare any variable",
                                     decl)
@@ -199,87 +198,38 @@
         raise api.FFIError(":%d: bad or unsupported type declaration" %
                 typenode.coord.line)
 
-    def _get_struct_union_enum_type(self, kind, type, name=None, nested=False):
-        # First, a level of caching on the exact 'type' node of the AST.
-        # This is obscure, but needed because pycparser "unrolls" declarations
-        # such as "typedef struct { } foo_t, *foo_p" and we end up with
-        # an AST that is not a tree, but a DAG, with the "type" node of the
-        # two branches foo_t and foo_p of the trees being the same node.
-        # It's a bit silly but detecting "DAG-ness" in the AST tree seems
-        # to be the only way to distinguish this case from two independent
-        # structs.  See test_struct_with_two_usages.
-        try:
-            return self._structnode2type[type]
-        except KeyError:
-            pass
+    def _get_struct_union_enum_type(self, kind, type, const):
+        name = type.name
+        assert name   # XXX
+        result = model.StructOrUnionOrEnum(kind, name, const)
         #
-        # Note that this must handle parsing "struct foo" any number of
-        # times and always return the same StructType object.  Additionally,
-        # one of these times (not necessarily the first), the fields of
-        # the struct can be specified with "struct foo { ...fields... }".
-        # If no name is given, then we have to create a new anonymous struct
-        # with no caching; in this case, the fields are either specified
-        # right now or never.
+        # get the type declaration or create it if needed
+        key = '%s %s' % (kind, name)
+        typedecl = self._struct_union_enum_decl.get(key, None)
         #
-        force_name = name
-        name = type.name
-        #
-        # get the type or create it if needed
-        if name is None:
-            # 'force_name' is used to guess a more readable name for
-            # anonymous structs, for the common case "typedef struct { } foo".
-            if force_name is not None:
-                explicit_name = '$%s' % force_name
-            else:
-                self._anonymous_counter += 1
-                explicit_name = '$%d' % self._anonymous_counter
-            tp = None
-        else:
-            explicit_name = name
-            key = '%s %s' % (kind, name)
-            tp = self._structunionenum.get(key, None)
-        #
-        if tp is None:
-            if kind == 'struct':
-                tp = model.StructType(explicit_name, None, None, None)
-            elif kind == 'union':
-                tp = model.UnionType(explicit_name, None, None, None)
+        if typedecl is None:
+            if kind == 'struct' or kind == 'union':
+                typedecl = model.StructOrUnionDecl(result)
             elif kind == 'enum':
-                tp = self._build_enum_type(explicit_name, type.values)
+                typedecl = XXX
             else:
                 raise AssertionError("kind = %r" % (kind,))
-            if name is not None:
-                self._structunionenum[key] = tp
-                self.declarations.append(tp)
-        else:
-            if kind == 'enum' and type.values is not None:
-                raise NotImplementedError(
-                    "enum %s: the '{}' declaration should appear on the first "
-                    "time the enum is mentioned, not later" % explicit_name)
-        ## XXX
-        ## if not tp.forcename:
-        ##     tp.force_the_name(force_name)
-        ## if tp.forcename and '$' in tp.name:
-        ##     self._declare('anonymous %s' % tp.forcename, tp)
-        #
-        self._structnode2type[type] = tp
-        #
-        # enums: done here
-        if kind == 'enum':
-            return tp
+            self._struct_union_enum_decl[key] = typedecl
+            self.declarations.append(typedecl)
         #
         # is there a 'type.decls'?  If yes, then this is the place in the
-        # C sources that declare the fields.  If no, then just return the
-        # existing type, possibly still incomplete.
-        if type.decls is None:
-            return tp
-        #
-        if tp.fldnames is not None:
+        # C sources that declare the fields.
+        if type.decls is not None:
+            self._add_fields_declaration(typedecl, type.decls)
+        return result
+
+    def _add_fields_declaration(self, typedecl, fields):
+        if typedecl.fldnames is not None:
             raise api.CDefError("duplicate declaration of struct %s" % name)
         fldnames = []
         fldtypes = []
         fldbitsize = []
-        for decl in type.decls:
+        for decl in fields:
             if (isinstance(decl.type, pycparser.c_ast.IdentifierType) and
                     ''.join(decl.type.names) == '__dotdotdot__'):
                 # XXX pycparser is inconsistent: 'names' should be a list
@@ -295,19 +245,19 @@
             type = self._get_type(decl.type, partial_length_ok=True)
             if self._partial_length:
                 self._make_partial(tp, nested)
-            if isinstance(type, model.StructType) and type.partial:
-                self._make_partial(tp, nested)
+            #if isinstance(type, model.StructType) and type.partial:
+            #    self._make_partial(tp, nested)
             fldnames.append(decl.name or '')
             fldtypes.append(type)
             fldbitsize.append(bitsize)
-        tp.fldnames = tuple(fldnames)
-        tp.fldtypes = tuple(fldtypes)
-        tp.fldbitsize = tuple(fldbitsize)
+        typedecl.fldnames = tuple(fldnames)
+        typedecl.fldtypes = tuple(fldtypes)
+        typedecl.fldbitsize = tuple(fldbitsize)
         if fldbitsize != [-1] * len(fldbitsize):
+            xxxx
             if isinstance(tp, model.StructType) and tp.partial:
                 raise NotImplementedError("%s: using both bitfields and '...;'"
                                           % (tp,))
-        return tp
 
     def _parse_constant(self, *args, **kwds):
         return None     # XXX
diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -326,7 +326,7 @@
                 line = '%sp1->%s = (void *)%s;%s%s' % (
                     '*' * (length - 1), self.structfield, expr, spaces, comment)
             else:
-                line = 'p1 = (void *)(((char *)%s) - (long)o);' % (expr,)
+                line = 'p1 = (void *)(((char *)%s) - o);' % (expr,)
         self.assign_target = None
         self.block.writeline(line.rstrip())
         for fn in self.after_star_p1_assignment:
@@ -343,46 +343,52 @@
 
 class StructOrUnionDecl(object):
 
-    def __init__(self, type, fldnames, fldtypes, fldbitsize):
+    def __init__(self, type):
         self.type = type
-        self.fldnames = fldnames
-        self.fldtypes = fldtypes
-        self.fldbitsize = fldbitsize
+        self.fldnames = None
+        self.fldtypes = None
+        self.fldbitsize = None
 
-    def write_declaration(self, tr):
-        tr.reset_r_output()
-        typename = "%s %s" % (self.kind, self.name)
-        tr.write_msg(typename)
-        tr.write_section("a[%d] = (void *)sizeof(%s);  /* size */"
-                          % (tr.fetch_next_a_index(), typename), indent=4)
-        tr.write_section("a[%d] = &((struct{char a; %s b;}*)0)->b;  /* align */"
-                          % (tr.fetch_next_a_index(), typename), indent=4)
-        tr.flush()
+    def write_declaration(self, funcblock):
+        tp = "%s %s" % (self.type.kind, self.type.name)
+        extra1 = "(long long)sizeof(%s)" % (tp,)    # total size
+        extra2 = ("(long long)(((char *)&((struct{char a; %s b;} *)0)->b)"
+                  " - (char *)0)" % (tp,))          # alignment
+        funcblock.sprintf(tp + r" {/*%lld,%lld*/\n",
+                          extra="%s, %s" % (extra1, extra2),
+                          extralength=40)
+        #
         for fldname, fldtype in zip(self.fldnames, self.fldtypes):
-            tr.reset_r_output()
-            tr.write_msg("{%s:" % (fldname,))
-            tr.fetch_next_a_index()   # ignored
-            outer = OuterDecl(tr, typename, fldname)
-            outer.start()
-            #
-            xtra = ''
+            block = CodeBlock(funcblock.tr)
+            inspect = TypeInspector(block, tp, fldname)
+            inspect.start()
+            # get the offset of the field
+            arraylevels = 0
             checktype = fldtype
             while isinstance(checktype, ArrayType):
-                xtra += '[0]'
+                arraylevels += 1
                 checktype = checktype.item
-            if xtra:
-                tr.add_include('stddef.h')
-                outer.decllines.append("void *o = (void *)offsetof(%s, %s%s);"
-                                       "  /* offset */" %
-                                       (typename, fldname, xtra))
+            if arraylevels:
+                block.add_include('stddef.h')
+                comment = ""
+                if arraylevels >= 1:
+                    comment = " (%dx)" % arraylevels
+                comment = inspect.get_comment(0, False, "an array%s" % comment)
+                block.writedecl("long long o = offsetof(%s, %s%s);%s"
+                                % (tp, fldname, "[0]" * arraylevels, comment))
             else:
-                outer.decllines.append("void *o = &((%s *)0)->%s;  /* offset */"
-                                       % (typename, fldname))
-            outer.codelines.append("a[%d] = o;" % (tr.fetch_next_a_index(),))
-            typeexpr = fldtype.outer_decl(outer)
-            outer.stop()
-            tr.write_msg(typeexpr.replace('&', ''))
-            tr.flush()
+                comment = inspect.get_comment(0, False, "not an array")
+                block.writedecl("long long o = ((char *)&((%s *)0)->%s)"
+                                " - (char *)0;%s" % (tp, fldname, comment))
+            #
+            block.sprintf("  /*%lld*/", extra="o", extralength=20)
+            block.sprintf_add_right(r';\n')
+            fldtype.inspect_type(block, inspect)
+            inspect.stop()
+            block.sprintf_add_left(fldname)
+            funcblock.write_subblock(block)
+        #
+        funcblock.sprintf(r"};\n")
 
 
 class TypeDef(object):
diff --git a/test/codegen/struct-001.c b/test/codegen/struct-001.c
--- a/test/codegen/struct-001.c
+++ b/test/codegen/struct-001.c
@@ -8,15 +8,15 @@
 int teststruct_001(char *r)
 {
     if (!r)
-        return 57 + 8 + 18 + 16;
+        return 69 + 30 + 18 + 6 + 30 + 18 + 6 + 4;
     r += sprintf(r, "struct foo_s {/*%lld,%lld*/\n", (long long)sizeof(struct foo_s), (long long)(((char *)&((struct{char a; struct foo_s b;} *)0)->b) - (char *)0));
     {
+        struct foo_s *p1;
         long long o = ((char *)&((struct foo_s *)0)->aa) - (char *)0;  /* check that 'struct foo_s::aa' is not an array */
-        struct foo_s *p1;
         char b[sizeof(p1->aa)];
+        r += sprintf(r, "  /*%lld*/", o);
         p1 = (void *)(((char *)b) - o);
         (void)(p1->aa << 1);  /* check that 'struct foo_s::aa' is an integer type */
-        r += sprintf(r, "  /*%lld*/", o);
         p1->aa = -1;  /* check that 'struct foo_s::aa' is not declared 'const' */
         if (p1->aa > 0) {
             if (sizeof(p1->aa) == 1 && p1->aa == 1)
@@ -51,12 +51,12 @@
         r += sprintf(r, " aa;\n");
     }
     {
+        struct foo_s *p1;
         long long o = ((char *)&((struct foo_s *)0)->bb) - (char *)0;  /* check that 'struct foo_s::bb' is not an array */
-        struct foo_s *p1;
         char b[sizeof(p1->bb)];
+        r += sprintf(r, "  /*%lld*/", o);
         p1 = (void *)(((char *)b) - o);
         (void)(p1->bb << 1);  /* check that 'struct foo_s::bb' is an integer type */
-        r += sprintf(r, "  /*%lld*/", o);
         p1->bb = -1;  /* check that 'struct foo_s::bb' is not declared 'const' */
         if (p1->bb > 0) {
             if (sizeof(p1->bb) == 1 && p1->bb == 1)

From noreply at buildbot.pypy.org  Tue Sep 16 14:38:32 2014
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 16 Sep 2014 14:38:32 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-small-uniform: add files from c7 branch
Message-ID: <20140916123832.44CC51D260E@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: c8-small-uniform
Changeset: r1384:c8192510bcee
Date: 2014-09-16 12:42 +0200
http://bitbucket.org/pypy/stmgc/changeset/c8192510bcee/

Log:	add files from c7 branch

diff --git a/c8/stm/smallmalloc.c b/c8/stm/smallmalloc.c
new file mode 100644
--- /dev/null
+++ b/c8/stm/smallmalloc.c
@@ -0,0 +1,310 @@
+#ifndef _STM_CORE_H_
+# error "must be compiled via stmgc.c"
+#endif
+
+
+#define PAGE_SMSIZE_START   END_NURSERY_PAGE
+#define PAGE_SMSIZE_END     NB_PAGES
+
+typedef struct {
+    uint8_t sz;
+} fpsz_t;
+
+static fpsz_t full_pages_object_size[PAGE_SMSIZE_END - PAGE_SMSIZE_START];
+/* ^^^ This array contains the size (in number of words) of the objects
+   in the given page, provided it's a "full page of small objects".  It
+   is 0 if it's not such a page, if it's fully free, or if it's in
+   small_page_lists.  It is not 0 as soon as the page enters the
+   segment's 'small_malloc_data.loc_free' (even if the page is not
+   technically full yet, it will be very soon in this case).
+*/
+
+static fpsz_t *get_fpsz(char *smallpage)
+{
+    uintptr_t pagenum = (((char *)smallpage) - stm_object_pages) / 4096;
+    assert(PAGE_SMSIZE_START <= pagenum && pagenum < PAGE_SMSIZE_END);
+    return &full_pages_object_size[pagenum - PAGE_SMSIZE_START];
+}
+
+
+#ifdef STM_TESTS
+bool (*_stm_smallmalloc_keep)(char *data);   /* a hook for tests */
+#endif
+
+static void teardown_smallmalloc(void)
+{
+    memset(small_page_lists, 0, sizeof(small_page_lists));
+    assert(free_uniform_pages == NULL);   /* done by the previous line */
+    first_small_uniform_loc = (uintptr_t) -1;
+#ifdef STM_TESTS
+    _stm_smallmalloc_keep = NULL;
+#endif
+    memset(full_pages_object_size, 0, sizeof(full_pages_object_size));
+}
+
+static int gmfp_lock = 0;
+
+static void grab_more_free_pages_for_small_allocations(void)
+{
+    /* Grab GCPAGE_NUM_PAGES pages out of the top addresses.  Use the
+       lock of pages.c to prevent any remapping from occurring under our
+       feet.
+    */
+    spinlock_acquire(gmfp_lock);
+
+    if (free_uniform_pages == NULL) {
+
+        uintptr_t decrease_by = GCPAGE_NUM_PAGES * 4096;
+        if (uninitialized_page_stop - uninitialized_page_start < decrease_by)
+            goto out_of_memory;
+
+        uninitialized_page_stop -= decrease_by;
+        first_small_uniform_loc = uninitialized_page_stop - stm_object_pages;
+
+        char *base = stm_object_pages + END_NURSERY_PAGE * 4096UL;
+        if (!_stm_largemalloc_resize_arena(uninitialized_page_stop - base))
+            goto out_of_memory;
+
+        setup_N_pages(uninitialized_page_stop, GCPAGE_NUM_PAGES);
+
+        char *p = uninitialized_page_stop;
+        long i;
+        for (i = 0; i < GCPAGE_NUM_PAGES; i++) {
+            ((struct small_free_loc_s *)p)->nextpage = free_uniform_pages;
+            free_uniform_pages = (struct small_free_loc_s *)p;
+            p += 4096;
+        }
+    }
+
+    spinlock_release(gmfp_lock);
+    return;
+
+ out_of_memory:
+    stm_fatalerror("out of memory!\n");   /* XXX */
+}
+
+static char *_allocate_small_slowpath(uint64_t size)
+{
+    long n = size / 8;
+    struct small_free_loc_s *smallpage;
+    struct small_free_loc_s *TLPREFIX *fl =
+        &STM_PSEGMENT->small_malloc_data.loc_free[n];
+    assert(*fl == NULL);
+
+ retry:
+    /* First try to grab the next page from the global 'small_page_list'
+     */
+    smallpage = small_page_lists[n];
+    if (smallpage != NULL) {
+        if (UNLIKELY(!__sync_bool_compare_and_swap(&small_page_lists[n],
+                                                   smallpage,
+                                                   smallpage->nextpage)))
+            goto retry;
+
+        /* Succeeded: we have a page in 'smallpage' */
+        *fl = smallpage->next;
+        get_fpsz((char *)smallpage)->sz = n;
+        return (char *)smallpage;
+    }
+
+    /* There is no more page waiting for the correct size of objects.
+       Maybe we can pick one from free_uniform_pages.
+     */
+    smallpage = free_uniform_pages;
+    if (smallpage != NULL) {
+        if (UNLIKELY(!__sync_bool_compare_and_swap(&free_uniform_pages,
+                                                   smallpage,
+                                                   smallpage->nextpage)))
+            goto retry;
+
+        /* Succeeded: we have a page in 'smallpage', which is not
+           initialized so far, apart from the 'nextpage' field read
+           above.  Initialize it.
+        */
+        struct small_free_loc_s *p, **previous;
+        assert(!(((uintptr_t)smallpage) & 4095));
+        previous = (struct small_free_loc_s **)
+            REAL_ADDRESS(STM_SEGMENT->segment_base, fl);
+
+        /* Initialize all slots from the second one to the last one to
+           contain a chained list */
+        uintptr_t i = size;
+        while (i <= 4096 - size) {
+            p = (struct small_free_loc_s *)(((char *)smallpage) + i);
+            *previous = p;
+            previous = &p->next;
+            i += size;
+        }
+        *previous = NULL;
+
+        /* The first slot is immediately returned */
+        get_fpsz((char *)smallpage)->sz = n;
+        return (char *)smallpage;
+    }
+
+    /* Not a single free page left.  Grab some more free pages and retry.
+     */
+    grab_more_free_pages_for_small_allocations();
+    goto retry;
+}
+
+__attribute__((always_inline))
+static inline char *allocate_outside_nursery_small(uint64_t size)
+{
+    OPT_ASSERT((size & 7) == 0);
+    OPT_ASSERT(16 <= size && size <= GC_LAST_SMALL_SIZE);
+
+    struct small_free_loc_s *TLPREFIX *fl =
+        &STM_PSEGMENT->small_malloc_data.loc_free[size / 8];
+
+    struct small_free_loc_s *result = *fl;
+
+    if (UNLIKELY(result == NULL))
+        return _allocate_small_slowpath(size);
+
+    *fl = result->next;
+    return (char *)result;
+}
+
+object_t *_stm_allocate_old_small(ssize_t size_rounded_up)
+{
+    char *p = allocate_outside_nursery_small(size_rounded_up);
+    return (object_t *)(p - stm_object_pages);
+}
+
+/************************************************************/
+
+static inline bool _smallmalloc_sweep_keep(char *p)
+{
+#ifdef STM_TESTS
+    if (_stm_smallmalloc_keep != NULL)
+        return _stm_smallmalloc_keep(p);
+#endif
+    abort();
+    //return smallmalloc_keep_object_at(p);
+}
+
+void check_order_inside_small_page(struct small_free_loc_s *page)
+{
+#ifndef NDEBUG
+    /* the free locations are supposed to be in increasing order */
+    while (page->next != NULL) {
+        assert(page->next > page);
+        page = page->next;
+    }
+#endif
+}
+
+static char *getbaseptr(struct small_free_loc_s *fl)
+{
+    return (char *)(((uintptr_t)fl) & ~4095);
+}
+
+void sweep_small_page(char *baseptr, struct small_free_loc_s *page_free,
+                      long szword)
+{
+    if (page_free != NULL)
+        check_order_inside_small_page(page_free);
+
+    /* for every non-free location, ask if we must free it */
+    uintptr_t i, size = szword * 8;
+    bool any_object_remaining = false, any_object_dying = false;
+    struct small_free_loc_s *fl = page_free;
+    struct small_free_loc_s *flprev = NULL;
+
+    /* XXX could optimize for the case where all objects die: we don't
+       need to painfully rebuild the free list in the whole page, just
+       to have it ignored in the end because we put the page into
+       'free_uniform_pages' */
+
+    for (i = 0; i <= 4096 - size; i += size) {
+        char *p = baseptr + i;
+        if (p == (char *)fl) {
+            /* location is already free */
+            flprev = fl;
+            fl = fl->next;
+            any_object_dying = true;
+        }
+        else if (!_smallmalloc_sweep_keep(p)) {
+            /* the location should be freed now */
+            if (flprev == NULL) {
+                flprev = (struct small_free_loc_s *)p;
+                flprev->next = fl;
+                page_free = flprev;
+            }
+            else {
+                assert(flprev->next == fl);
+                flprev->next = (struct small_free_loc_s *)p;
+                flprev = (struct small_free_loc_s *)p;
+                flprev->next = fl;
+            }
+            any_object_dying = true;
+        }
+        else {
+            any_object_remaining = true;
+        }
+    }
+    if (!any_object_remaining) {
+        ((struct small_free_loc_s *)baseptr)->nextpage = free_uniform_pages;
+        free_uniform_pages = (struct small_free_loc_s *)baseptr;
+    }
+    else if (!any_object_dying) {
+        get_fpsz(baseptr)->sz = szword;
+    }
+    else {
+        check_order_inside_small_page(page_free);
+        page_free->nextpage = small_page_lists[szword];
+        small_page_lists[szword] = page_free;
+    }
+}
+
+void _stm_smallmalloc_sweep(void)
+{
+    long i, szword;
+    for (szword = 2; szword < GC_N_SMALL_REQUESTS; szword++) {
+        struct small_free_loc_s *page = small_page_lists[szword];
+        struct small_free_loc_s *nextpage;
+        small_page_lists[szword] = NULL;
+
+        /* process the pages that the various segments are busy filling */
+        for (i = 1; i <= NB_SEGMENTS; i++) {
+            struct stm_priv_segment_info_s *pseg = get_priv_segment(i);
+            struct small_free_loc_s **fl =
+                    &pseg->small_malloc_data.loc_free[szword];
+            if (*fl != NULL) {
+                /* the entry in full_pages_object_size[] should already be
+                   szword.  We reset it to 0. */
+                fpsz_t *fpsz = get_fpsz((char *)*fl);
+                assert(fpsz->sz == szword);
+                fpsz->sz = 0;
+                sweep_small_page(getbaseptr(*fl), *fl, szword);
+                *fl = NULL;
+            }
+        }
+
+        /* process all the other partially-filled pages */
+        while (page != NULL) {
+            /* for every page in small_page_lists: assert that the
+               corresponding full_pages_object_size[] entry is 0 */
+            assert(get_fpsz((char *)page)->sz == 0);
+            nextpage = page->nextpage;
+            sweep_small_page(getbaseptr(page), page, szword);
+            page = nextpage;
+        }
+    }
+
+    /* process the really full pages, which are the ones which still
+       have a non-zero full_pages_object_size[] entry */
+    char *pageptr = uninitialized_page_stop;
+    fpsz_t *fpsz_start = get_fpsz(pageptr);
+    fpsz_t *fpsz_end = &full_pages_object_size[PAGE_SMSIZE_END -
+                                               PAGE_SMSIZE_START];
+    fpsz_t *fpsz;
+    for (fpsz = fpsz_start; fpsz < fpsz_end; fpsz++, pageptr += 4096) {
+        uint8_t sz = fpsz->sz;
+        if (sz != 0) {
+            fpsz->sz = 0;
+            sweep_small_page(pageptr, NULL, sz);
+        }
+    }
+}
diff --git a/c8/stm/smallmalloc.h b/c8/stm/smallmalloc.h
new file mode 100644
--- /dev/null
+++ b/c8/stm/smallmalloc.h
@@ -0,0 +1,66 @@
+
+/* Outside the nursery, we are taking from the highest addresses
+   complete pages, one at a time, which uniformly contain objects of
+   size "8 * N" for some N in range(2, GC_N_SMALL_REQUESTS).  We are
+   taking from the lowest addresses "large" objects, which are at least
+   288 bytes long, allocated by largemalloc.c.  The limit is the same
+   as used in PyPy's default GC.
+*/
+
+#define GC_N_SMALL_REQUESTS    36
+#define GC_LAST_SMALL_SIZE     (8 * (GC_N_SMALL_REQUESTS - 1))
+
+
+struct small_free_loc_s {
+    /* A chained list of locations within the same page which are
+       free. */
+    struct small_free_loc_s *next;
+
+    /* A chained list of all small pages containing objects of a given
+       small size, and that have at least one free object.  It points
+       *inside* the next page, to another struct small_free_loc_s.  This
+       field is only meaningful on the first small_free_loc_s of a given
+       page! */
+    struct small_free_loc_s *nextpage;
+
+    /* This structure is only two words, so it always fits inside one
+       free slot inside the page. */
+};
+
+
+/* For every size from 16 bytes to 8*(GC_N_SMALL_REQUESTS-1), this is
+   a list of pages that contain objects of that size and have at least
+   one free location.  Additionally, the item 0 in the following list
+   is a chained list of fully-free pages (which can be reused for a
+   different size than the one they originally contained).
+*/
+static struct small_free_loc_s *small_page_lists[GC_N_SMALL_REQUESTS];
+
+#define free_uniform_pages   (small_page_lists[0])
+
+
+/* For is_small_uniform(). */
+static uintptr_t first_small_uniform_loc = (uintptr_t) -1;
+
+
+/* This is a definition for 'STM_PSEGMENT->small_malloc_data'.  Each
+   segment grabs one page at a time from the global list, and then
+   requests for data are answered locally.
+*/
+struct small_malloc_data_s {
+    struct small_free_loc_s *loc_free[GC_N_SMALL_REQUESTS];
+};
+
+
+/* Functions
+ */
+static inline char *allocate_outside_nursery_small(uint64_t size)
+     __attribute__((always_inline));
+
+void _stm_smallmalloc_sweep(void);
+
+static void teardown_smallmalloc(void);
+
+static inline bool is_small_uniform(object_t *obj) {
+    return ((uintptr_t)obj) >= first_small_uniform_loc;
+}
diff --git a/c8/test/test_smallmalloc.py b/c8/test/test_smallmalloc.py
new file mode 100644
--- /dev/null
+++ b/c8/test/test_smallmalloc.py
@@ -0,0 +1,76 @@
+from support import *
+import random
+
+
+def pageof(p):
+    return int(ffi.cast("uintptr_t", p)) >> 12
+
+
+class TestSmallMalloc(BaseTest):
+
+    def setup_method(self, method):
+        BaseTest.setup_method(self, method)
+        @ffi.callback("bool(char *)")
+        def keep(data):
+            p = ffi.cast("object_t *", data - lib.stm_object_pages)
+            self.has_been_asked_for.append(p)
+            return p in self.keep_me
+        lib._stm_smallmalloc_keep = keep
+        self._keepalive_keep_function = keep
+        self.keep_me = set()
+        self.has_been_asked_for = []
+
+    def test_simple_uniform(self):
+        page0 = [stm_allocate_old_small(16) for i in range(0, 4096, 16)]
+        assert len(set(map(pageof, page0))) == 1
+        #
+        page1 = [stm_allocate_old_small(16) for i in range(0, 4096, 16)]
+        assert len(set(map(pageof, page1))) == 1
+        #
+        assert len(set(map(pageof, page0 + page1))) == 2
+
+    def test_different_sizes_different_pages(self):
+        seen = []
+        for i in range(2, GC_N_SMALL_REQUESTS):
+            p = pageof(stm_allocate_old_small(8 * i))
+            assert p not in seen
+            seen.append(p)
+        for i in range(2, GC_N_SMALL_REQUESTS):
+            p = pageof(stm_allocate_old_small(8 * i))
+            assert p == seen[0]
+            seen.pop(0)
+
+    def test_sweep_freeing_simple(self):
+        p1 = stm_allocate_old_small(16)
+        lib._stm_smallmalloc_sweep()
+
+    def test_sweep_freeing_random_subset(self):
+        for i in range(50):
+            page0 = [stm_allocate_old_small(16) for i in range(0, 4096, 16)]
+            assert len(set(map(pageof, page0))) == 1
+            tid = lib._get_type_id(page0[0])
+            while len(page0) > 0:
+                self.keep_me = set(random.sample(page0, len(page0) // 2))
+                self.has_been_asked_for = []
+                lib._stm_smallmalloc_sweep()
+                assert sorted(page0) == self.has_been_asked_for
+                page0r = []
+                for p in page0:
+                    if p in self.keep_me:
+                        assert lib._get_type_id(p) == tid
+                        page0r.append(p)
+                    else:
+                        assert lib._get_type_id(p) != tid
+                page0 = page0r
+                if len(page0) > 10:
+                    p = stm_allocate_old_small(16)
+                    assert pageof(p) == pageof(page0[0])
+                    page0.append(p)
+
+    def test_sweep_full_page_remains_full(self):
+        page0 = [stm_allocate_old_small(16) for i in range(0, 4096, 16)]
+        tid = lib._get_type_id(page0[0])
+        self.keep_me = set(page0)
+        lib._stm_smallmalloc_sweep()
+        for p in page0:
+            assert lib._get_type_id(p) == tid

From noreply at buildbot.pypy.org  Tue Sep 16 14:38:33 2014
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 16 Sep 2014 14:38:33 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-small-uniform: pass test_smallmalloc.py again
Message-ID: <20140916123833.79D9B1D260E@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: c8-small-uniform
Changeset: r1385:5a540496172b
Date: 2014-09-16 13:10 +0200
http://bitbucket.org/pypy/stmgc/changeset/5a540496172b/

Log:	pass test_smallmalloc.py again

diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -69,6 +69,9 @@
 #ifndef NDEBUG
     pthread_t running_pthread;
 #endif
+
+    /* This is for smallmalloc.c */
+    struct small_malloc_data_s small_malloc_data;
 };
 
 enum /* safe_point */ {
@@ -95,8 +98,10 @@
 };
 static struct stm_commit_log_entry_s commit_log_root = {NULL, -1};
 
-
-static char *stm_object_pages;
+#ifndef STM_TESTS
+static
+#endif
+       char *stm_object_pages;
 static int stm_object_pages_fd;
 static stm_thread_local_t *stm_all_thread_locals = NULL;
 
diff --git a/c8/stm/gcpage.h b/c8/stm/gcpage.h
--- a/c8/stm/gcpage.h
+++ b/c8/stm/gcpage.h
@@ -1,8 +1,11 @@
 
+/* Granularity when grabbing more unused pages: take 50 at a time */
+#define GCPAGE_NUM_PAGES   50
 
 static char *uninitialized_page_start;   /* within segment 0 */
 static char *uninitialized_page_stop;
 
 static void setup_gcpage(void);
 static void teardown_gcpage(void);
+static void setup_N_pages(char *pages_addr, uint64_t num);
 static char *allocate_outside_nursery_large(uint64_t size);
diff --git a/c8/stm/setup.c b/c8/stm/setup.c
--- a/c8/stm/setup.c
+++ b/c8/stm/setup.c
@@ -145,6 +145,7 @@
 
     teardown_sync();
     teardown_gcpage();
+    teardown_smallmalloc();
     teardown_pages();
 }
 
diff --git a/c8/stm/smallmalloc.c b/c8/stm/smallmalloc.c
--- a/c8/stm/smallmalloc.c
+++ b/c8/stm/smallmalloc.c
@@ -61,9 +61,10 @@
         uninitialized_page_stop -= decrease_by;
         first_small_uniform_loc = uninitialized_page_stop - stm_object_pages;
 
-        char *base = stm_object_pages + END_NURSERY_PAGE * 4096UL;
-        if (!_stm_largemalloc_resize_arena(uninitialized_page_stop - base))
-            goto out_of_memory;
+        /* XXX: */
+        /* char *base = stm_object_pages + END_NURSERY_PAGE * 4096UL; */
+        /* if (!_stm_largemalloc_resize_arena(uninitialized_page_stop - base)) */
+        /*     goto out_of_memory; */
 
         setup_N_pages(uninitialized_page_stop, GCPAGE_NUM_PAGES);
 
@@ -267,7 +268,7 @@
         small_page_lists[szword] = NULL;
 
         /* process the pages that the various segments are busy filling */
-        for (i = 1; i <= NB_SEGMENTS; i++) {
+        for (i = 0; i < NB_SEGMENTS; i++) {
             struct stm_priv_segment_info_s *pseg = get_priv_segment(i);
             struct small_free_loc_s **fl =
                     &pseg->small_malloc_data.loc_free[szword];
diff --git a/c8/stmgc.c b/c8/stmgc.c
--- a/c8/stmgc.c
+++ b/c8/stmgc.c
@@ -2,6 +2,7 @@
 #include "stmgc.h"
 #include "stm/atomic.h"
 #include "stm/list.h"
+#include "stm/smallmalloc.h"
 #include "stm/core.h"
 #include "stm/pagecopy.h"
 #include "stm/pages.h"
@@ -15,6 +16,7 @@
 
 #include "stm/misc.c"
 #include "stm/list.c"
+#include "stm/smallmalloc.c"
 #include "stm/pagecopy.c"
 #include "stm/pages.c"
 #include "stm/prebuilt.c"
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -80,6 +80,11 @@
 void _stm_test_switch_segment(int segnum);
 void _push_obj_to_other_segments(object_t *obj);
 
+char *stm_object_pages;
+object_t *_stm_allocate_old_small(ssize_t size_rounded_up);
+bool (*_stm_smallmalloc_keep)(char *data);
+void _stm_smallmalloc_sweep(void);
+
 void _stm_start_safe_point(void);
 void _stm_stop_safe_point(void);
 
diff --git a/c8/test/support.py b/c8/test/support.py
--- a/c8/test/support.py
+++ b/c8/test/support.py
@@ -33,6 +33,8 @@
     ...;
 } stm_thread_local_t;
 
+char *stm_object_pages;
+
 void stm_read(object_t *obj);
 /*void stm_write(object_t *obj); use _checked_stm_write() instead */
 object_t *stm_allocate(ssize_t size_rounded_up);
@@ -100,6 +102,10 @@
 ssize_t stmcb_size_rounded_up(struct object_s *obj);
 
 
+object_t *_stm_allocate_old_small(ssize_t size_rounded_up);
+bool (*_stm_smallmalloc_keep)(char *data);
+void _stm_smallmalloc_sweep(void);
+
 """)
 
 
@@ -303,6 +309,12 @@
     lib._set_type_id(o, tid)
     return o
 
+def stm_allocate_old_small(size):
+    o = lib._stm_allocate_old_small(size)
+    tid = 42 + size
+    lib._set_type_id(o, tid)
+    return o
+
 def stm_allocate(size):
     o = lib.stm_allocate(size)
     tid = 42 + size

From noreply at buildbot.pypy.org  Tue Sep 16 14:38:34 2014
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 16 Sep 2014 14:38:34 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-small-uniform: use small obj allocation (not
 yet synchronizing to other segments on commit)
Message-ID: <20140916123834.91FE91D260E@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: c8-small-uniform
Changeset: r1386:a1300b7dd558
Date: 2014-09-16 13:40 +0200
http://bitbucket.org/pypy/stmgc/changeset/a1300b7dd558/

Log:	use small obj allocation (not yet synchronizing to other segments on
	commit)

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -246,7 +246,11 @@
     realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj);
     obj_size = stmcb_size_rounded_up((struct object_s *)realobj);
     /* get the last page containing data from the object */
-    end_page = (((uintptr_t)obj) + obj_size - 1) / 4096UL;
+    if (LIKELY(is_small_uniform(obj))) {
+        end_page = first_page;
+    } else {
+        end_page = (((uintptr_t)obj) + obj_size - 1) / 4096UL;
+    }
 
     /* add to read set: */
     stm_read(obj);
@@ -256,10 +260,9 @@
     memcpy(bk_obj, realobj, obj_size);
 
     /* if there are shared pages, privatize them */
-
-    uintptr_t page;
+    uintptr_t page = first_page;
     for (page = first_page; page <= end_page; page++) {
-        if (is_shared_log_page(page)) {
+        if (UNLIKELY(is_shared_log_page(page))) {
             long i;
             for (i = 0; i < NB_SEGMENTS; i++) {
                 acquire_privatization_lock(i);
diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -88,10 +88,20 @@
         realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj);
         size = stmcb_size_rounded_up((struct object_s *)realobj);
 
-        /* XXX: small objs */
-        char *allocated = allocate_outside_nursery_large(size);
-        nobj = (object_t *)(allocated - stm_object_pages);
+        if (size > GC_LAST_SMALL_SIZE) {
+            /* case 1: object is not small enough.
+               Ask gcpage.c for an allocation via largemalloc. */
+            char *allocated = allocate_outside_nursery_large(size);
+            nobj = (object_t *)(allocated - stm_object_pages);
+        }
+        else {
+            /* case "small enough" */
+            char *allocated = allocate_outside_nursery_small(size);
+            dprintf(("outside small %p or %p, sz=%lu\n", allocated, allocated - stm_object_pages, size));
+            nobj = (object_t *)(allocated - stm_object_pages);
+        }
 
+        /* copy the object */
     copy_large_object:;
         char *realnobj = REAL_ADDRESS(STM_SEGMENT->segment_base, nobj);
         memcpy(realnobj, realobj, size);
diff --git a/c8/test/test_nursery.py b/c8/test/test_nursery.py
--- a/c8/test/test_nursery.py
+++ b/c8/test/test_nursery.py
@@ -252,3 +252,31 @@
         # if the read marker is not cleared, we get false conflicts
         # with later transactions using the same large-malloced slot
         # as our outside-nursery-obj
+
+    def test_synchronize_small_obj(self):
+        # make a shared page, and privatize it
+        self.start_transaction()
+        new = stm_allocate(16)
+        self.push_root(new)
+        self.commit_transaction()
+        new = self.pop_root()
+        self.push_root(new)
+
+        self.start_transaction()
+        stm_set_char(new, 'A')
+        self.commit_transaction()
+
+        # make a new object of the same size, which should end in the
+        # same page
+        self.start_transaction()
+        new2 = stm_allocate(16)
+        stm_set_char(new2, 'a')
+        self.push_root(new2)
+        self.commit_transaction()
+        new2 = self.pop_root()
+        print "new2", new2
+
+        # check that this new object was correctly sychronized
+        self.switch(1)
+        self.start_transaction()
+        assert stm_get_char(new2) == 'a'

From noreply at buildbot.pypy.org  Tue Sep 16 14:38:35 2014
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 16 Sep 2014 14:38:35 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-small-uniform: add synchronization to other
 segments on allocation of new (old) objects
Message-ID: <20140916123835.942BF1D260E@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: c8-small-uniform
Changeset: r1387:22150f9efdc0
Date: 2014-09-16 14:27 +0200
http://bitbucket.org/pypy/stmgc/changeset/22150f9efdc0/

Log:	add synchronization to other segments on allocation of new (old)
	objects

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -5,6 +5,19 @@
 #include 
 
 
+#ifdef NDEBUG
+#define EVENTUALLY(condition)    {/* nothing */}
+#else
+#define EVENTUALLY(condition)                                   \
+    {                                                           \
+        if (!(condition)) {                                     \
+            acquire_privatization_lock(STM_SEGMENT->segment_num);\
+            if (!(condition))                                   \
+                stm_fatalerror("fails: " #condition);           \
+            release_privatization_lock(STM_SEGMENT->segment_num);\
+        }                                                       \
+    }
+#endif
 
 /* ############# commit log ############# */
 
@@ -616,3 +629,102 @@
     synchronize_all_threads(STOP_OTHERS_AND_BECOME_GLOBALLY_UNIQUE);
     s_mutex_unlock();
 }
+
+
+
+static inline void _synchronize_fragment(stm_char *frag, ssize_t frag_size)
+{
+    /* double-check that the result fits in one page */
+    assert(frag_size > 0);
+    assert(frag_size + ((uintptr_t)frag & 4095) <= 4096);
+
+    /* if the page of the fragment is fully shared, nothing to do */
+    assert(STM_PSEGMENT->privatization_lock);
+    if (is_shared_log_page((uintptr_t)frag / 4096))
+        return;                 /* nothing to do */
+
+    /* Enqueue this object (or fragemnt of object) */
+    if (STM_PSEGMENT->sq_len == SYNC_QUEUE_SIZE)
+        synchronize_objects_flush();
+    STM_PSEGMENT->sq_fragments[STM_PSEGMENT->sq_len] = frag;
+    STM_PSEGMENT->sq_fragsizes[STM_PSEGMENT->sq_len] = frag_size;
+    ++STM_PSEGMENT->sq_len;
+}
+
+static void synchronize_object_enqueue(object_t *obj)
+{
+    assert(!_is_young(obj));
+    assert(STM_PSEGMENT->privatization_lock);
+    assert(obj->stm_flags & GCFLAG_WRITE_BARRIER);
+    ssize_t obj_size = stmcb_size_rounded_up(
+        (struct object_s *)REAL_ADDRESS(STM_SEGMENT->segment_base, obj));
+    OPT_ASSERT(obj_size >= 16);
+
+    if (LIKELY(is_small_uniform(obj))) {
+        _synchronize_fragment((stm_char *)obj, obj_size);
+        return;
+    }
+
+    /* else, a more complicated case for large objects, to copy
+       around data only within the needed pages
+    */
+    uintptr_t start = (uintptr_t)obj;
+    uintptr_t end = start + obj_size;
+
+    do {
+        uintptr_t copy_up_to = (start + 4096) & ~4095;   /* end of page */
+        if (copy_up_to >= end) {
+            copy_up_to = end;        /* this is the last fragment */
+        }
+        uintptr_t copy_size = copy_up_to - start;
+
+        _synchronize_fragment((stm_char *)start, copy_size);
+
+        start = copy_up_to;
+    } while (start != end);
+}
+
+static void synchronize_objects_flush(void)
+{
+
+    /* Do a full memory barrier.  We must make sure that other
+       CPUs see the changes we did to the shared page ("S", in
+       synchronize_object_enqueue()) before we check the other segments
+       with is_private_page() (below).  Otherwise, we risk the
+       following: this CPU writes "S" but the writes are not visible yet;
+       then it checks is_private_page() and gets false, and does nothing
+       more; just afterwards another CPU sets its own private_page bit
+       and copies the page; but it risks doing so before seeing the "S"
+       writes.
+    */
+    /* XXX: not sure this applies anymore.  */
+    long j = STM_PSEGMENT->sq_len;
+    if (j == 0)
+        return;
+    STM_PSEGMENT->sq_len = 0;
+
+    __sync_synchronize();
+
+    long i, myself = STM_SEGMENT->segment_num;
+    do {
+        --j;
+        stm_char *frag = STM_PSEGMENT->sq_fragments[j];
+        uintptr_t page = ((uintptr_t)frag) / 4096UL;
+        if (is_shared_log_page(page))
+            continue;
+
+        ssize_t frag_size = STM_PSEGMENT->sq_fragsizes[j];
+
+        char *src = REAL_ADDRESS(STM_SEGMENT->segment_base, frag);
+        for (i = 0; i < NB_SEGMENTS; i++) {
+            if (i == myself)
+                continue;
+
+            char *dst = REAL_ADDRESS(get_segment_base(i), frag);
+            if (is_private_log_page_in(i, page))
+                memcpy(dst, src, frag_size);
+            else
+                EVENTUALLY(memcmp(dst, src, frag_size) == 0);  /* same page */
+        }
+    } while (j > 0);
+}
diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -38,6 +38,10 @@
 };
 
 
+
+#define SYNC_QUEUE_SIZE    31
+
+
 /************************************************************/
 
 
@@ -72,8 +76,15 @@
 
     /* This is for smallmalloc.c */
     struct small_malloc_data_s small_malloc_data;
+
+    /* The sync queue used to synchronize newly allocated objs to
+       other segments */
+    stm_char *sq_fragments[SYNC_QUEUE_SIZE];
+    int sq_fragsizes[SYNC_QUEUE_SIZE];
+    int sq_len;
 };
 
+
 enum /* safe_point */ {
     SP_NO_TRANSACTION=0,
     SP_RUNNING,
@@ -138,6 +149,8 @@
 static stm_thread_local_t *abort_with_mutex_no_longjmp(void);
 static void abort_data_structures_from_segment_num(int segment_num);
 
+static void synchronize_object_enqueue(object_t *obj);
+static void synchronize_objects_flush(void);
 
 
 static inline void _duck(void) {
diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -47,11 +47,14 @@
 /************************************************************/
 static object_t *find_existing_shadow(object_t *obj);
 #define GCWORD_MOVED  ((object_t *) -1)
+#define FLAG_SYNC_LARGE       0x01
+
 
 static void minor_trace_if_young(object_t **pobj)
 {
     /* takes a normal pointer to a thread-local pointer to an object */
     object_t *obj = *pobj;
+    uintptr_t nobj_sync_now;
     object_t *nobj;
     char *realobj;
     size_t size;
@@ -97,7 +100,6 @@
         else {
             /* case "small enough" */
             char *allocated = allocate_outside_nursery_small(size);
-            dprintf(("outside small %p or %p, sz=%lu\n", allocated, allocated - stm_object_pages, size));
             nobj = (object_t *)(allocated - stm_object_pages);
         }
 
@@ -106,6 +108,8 @@
         char *realnobj = REAL_ADDRESS(STM_SEGMENT->segment_base, nobj);
         memcpy(realnobj, realobj, size);
 
+        nobj_sync_now = ((uintptr_t)nobj) | FLAG_SYNC_LARGE;
+
         pforwarded_array[0] = GCWORD_MOVED;
         pforwarded_array[1] = nobj;
         *pobj = nobj;
@@ -119,10 +123,12 @@
         /* a young object outside the nursery */
         nobj = obj;
         tree_delete_item(STM_PSEGMENT->young_outside_nursery, (uintptr_t)nobj);
+
+        nobj_sync_now = ((uintptr_t)nobj) | FLAG_SYNC_LARGE;
     }
 
     /* Must trace the object later */
-    LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, (uintptr_t)nobj);
+    LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, nobj_sync_now);
 }
 
 
@@ -174,13 +180,24 @@
     struct list_s *lst = STM_PSEGMENT->objects_pointing_to_nursery;
 
     while (!list_is_empty(lst)) {
-        object_t *obj = (object_t *)list_pop_item(lst);;
+        uintptr_t obj_sync_now = list_pop_item(lst);
+        object_t *obj = (object_t *)(obj_sync_now & ~FLAG_SYNC_LARGE);
 
         _collect_now(obj);
 
+        if (obj_sync_now & FLAG_SYNC_LARGE) {
+            /* this is a newly allocated object. We must synchronize it
+               to other segments (after we added WRITE_BARRIER). */
+            acquire_privatization_lock(STM_SEGMENT->segment_num);
+            synchronize_object_enqueue(obj);
+            release_privatization_lock(STM_SEGMENT->segment_num);
+        }
+
         /* the list could have moved while appending */
         lst = STM_PSEGMENT->objects_pointing_to_nursery;
     }
+
+    synchronize_objects_flush();
 }
 
 
diff --git a/c8/test/test_nursery.py b/c8/test/test_nursery.py
--- a/c8/test/test_nursery.py
+++ b/c8/test/test_nursery.py
@@ -274,7 +274,6 @@
         self.push_root(new2)
         self.commit_transaction()
         new2 = self.pop_root()
-        print "new2", new2
 
         # check that this new object was correctly sychronized
         self.switch(1)
diff --git a/c8/test/test_random.py b/c8/test/test_random.py
--- a/c8/test/test_random.py
+++ b/c8/test/test_random.py
@@ -361,7 +361,7 @@
 
 def op_allocate(ex, global_state, thread_state):
     size = global_state.rnd.choice([
-        "16",
+        "16", "48", "288",
         str(4096+16),
         str(80*1024+16),
         #"SOME_MEDIUM_SIZE+16",

From noreply at buildbot.pypy.org  Tue Sep 16 14:38:36 2014
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 16 Sep 2014 14:38:36 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-small-uniform: Backed out changeset:
 a20e5e7e942c. I receive SIGBUS before running out of memory using the
 pwrite()-atomic-privatization. Investigate at some point.
Message-ID: <20140916123836.978C01D260E@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: c8-small-uniform
Changeset: r1388:7d8934d22965
Date: 2014-09-16 14:40 +0200
http://bitbucket.org/pypy/stmgc/changeset/7d8934d22965/

Log:	Backed out changeset: a20e5e7e942c. I receive SIGBUS before running
	out of memory using the pwrite()-atomic-privatization. Investigate
	at some point.

diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -32,6 +32,8 @@
 #define FIRST_OLD_RM_PAGE     (OLD_RM_START / 4096UL)
 #define NB_READMARKER_PAGES   (FIRST_OBJECT_PAGE - FIRST_READMARKER_PAGE)
 
+#define TMP_COPY_PAGE         1 /* HACK */
+
 enum /* stm_flags */ {
     GCFLAG_WRITE_BARRIER = _STM_GCFLAG_WRITE_BARRIER,
     GCFLAG_HAS_SHADOW = 0x02,
diff --git a/c8/stm/pagecopy.h b/c8/stm/pagecopy.h
--- a/c8/stm/pagecopy.h
+++ b/c8/stm/pagecopy.h
@@ -1,2 +1,2 @@
 
-static void pagecopy(void *dest, const void *src) __attribute__((unused));      // 4096 bytes
+static void pagecopy(void *dest, const void *src);      // 4096 bytes
diff --git a/c8/stm/pages.c b/c8/stm/pages.c
--- a/c8/stm/pages.c
+++ b/c8/stm/pages.c
@@ -1,8 +1,8 @@
 #ifndef _STM_CORE_H_
 # error "must be compiled via stmgc.c"
 #endif
+#include 
 
-#include 
 /************************************************************/
 
 static void setup_pages(void)
@@ -34,14 +34,21 @@
 
     /* assert remappings follow the rule that page N in one segment
        can only be remapped to page N in another segment */
-    assert(((addr - stm_object_pages) / 4096UL - pgoff) % NB_PAGES == 0);
+    assert(IMPLY(((addr - stm_object_pages) / 4096UL) != TMP_COPY_PAGE,
+                 ((addr - stm_object_pages) / 4096UL - pgoff) % NB_PAGES == 0));
 
+#ifdef USE_REMAP_FILE_PAGES
+    int res = remap_file_pages(addr, size, 0, pgoff, 0);
+    if (UNLIKELY(res < 0))
+        stm_fatalerror("remap_file_pages: %m");
+#else
     char *res = mmap(addr, size,
                      PROT_READ | PROT_WRITE,
                      (MAP_PAGES_FLAGS & ~MAP_ANONYMOUS) | MAP_FIXED,
                      stm_object_pages_fd, pgoff * 4096UL);
     if (UNLIKELY(res != addr))
         stm_fatalerror("mmap (remapping page): %m");
+#endif
 }
 
 
@@ -108,15 +115,14 @@
        attempt to group together many calls to d_remap_file_pages() in
        succession) */
     uintptr_t pagenum_in_file = NB_PAGES * segnum + pagenum;
+    char *tmp_page = stm_object_pages + TMP_COPY_PAGE * 4096UL;
+    /* first remap to TMP_PAGE, then copy stuff there (to the underlying
+       file page), then remap this file-page hopefully atomically to the
+       segnum's virtual page */
+    d_remap_file_pages(tmp_page, 4096, pagenum_in_file);
+    pagecopy(tmp_page, initialize_from);
+    write_fence();
+
     char *new_page = stm_object_pages + pagenum_in_file * 4096UL;
-
-    /* first write to the file page directly: */
-    ssize_t written = pwrite(stm_object_pages_fd, initialize_from, 4096UL,
-                             pagenum_in_file * 4096UL);
-    if (written != 4096)
-        stm_fatalerror("pwrite didn't write the whole page: %zd", written);
-
-    /* now remap virtual page in segment to the new file page */
-    write_fence();
     d_remap_file_pages(new_page, 4096, pagenum_in_file);
 }
diff --git a/c8/stm/pages.h b/c8/stm/pages.h
--- a/c8/stm/pages.h
+++ b/c8/stm/pages.h
@@ -21,6 +21,7 @@
 #define PAGE_FLAG_START   END_NURSERY_PAGE
 #define PAGE_FLAG_END     NB_PAGES
 
+#define USE_REMAP_FILE_PAGES
 
 struct page_shared_s {
 #if NB_SEGMENTS <= 8
diff --git a/c8/stm/setup.c b/c8/stm/setup.c
--- a/c8/stm/setup.c
+++ b/c8/stm/setup.c
@@ -2,11 +2,27 @@
 # error "must be compiled via stmgc.c"
 #endif
 
+#include 
 
+#ifdef USE_REMAP_FILE_PAGES
+static char *setup_mmap(char *reason, int *ignored)
+{
+    char *result = mmap(NULL, TOTAL_MEMORY,
+                        PROT_READ | PROT_WRITE,
+                        MAP_PAGES_FLAGS, -1, 0);
+    if (result == MAP_FAILED)
+        stm_fatalerror("%s failed: %m", reason);
+
+    return result;
+}
+static void close_fd_mmap(int ignored)
+{
+}
+#else
 #include            /* For O_* constants */
 static char *setup_mmap(char *reason, int *map_fd)
 {
-    char name[128] = "/__stmgc_c8__";
+    char name[128];
 
     /* Create the big shared memory object, and immediately unlink it.
        There is a small window where if this process is killed the
@@ -35,6 +51,7 @@
 {
     close(map_fd);
 }
+#endif
 
 static void setup_protection_settings(void)
 {
@@ -46,13 +63,19 @@
            NULL accesses land.  We mprotect it so that accesses fail. */
         mprotect(segment_base, 4096, PROT_NONE);
 
+        /* TMP_COPY_PAGE is used for atomic privatization */
+        mprotect(segment_base + TMP_COPY_PAGE * 4096UL,
+                 4096UL, PROT_READ|PROT_WRITE);
+
         /* Pages in range(2, FIRST_READMARKER_PAGE) are never used */
-        if (FIRST_READMARKER_PAGE > 2)
-            mprotect(segment_base + 2 * 4096,
-                     (FIRST_READMARKER_PAGE - 2) * 4096UL,
+        if (FIRST_READMARKER_PAGE > TMP_COPY_PAGE + 1)
+            mprotect(segment_base + (TMP_COPY_PAGE + 1) * 4096,
+                     (FIRST_READMARKER_PAGE - TMP_COPY_PAGE - 1) * 4096UL,
                      PROT_NONE);
 
-        /* STM_SEGMENT is in page 1 */
+        /* STM_SEGMENT */
+        mprotect(segment_base + ((uintptr_t)STM_SEGMENT / 4096UL) * 4096UL,
+                 4096UL, PROT_READ|PROT_WRITE);
     }
 }
 
@@ -60,11 +83,13 @@
 void stm_setup(void)
 {
     /* Check that some values are acceptable */
-    assert(4096 <= ((uintptr_t)STM_SEGMENT));
+    assert(TMP_COPY_PAGE > 0 && TMP_COPY_PAGE <= 1);
+    assert(TMP_COPY_PAGE * 4096 + 4096 <= ((uintptr_t)STM_SEGMENT));
     assert((uintptr_t)STM_SEGMENT == (uintptr_t)STM_PSEGMENT);
     assert(((uintptr_t)STM_PSEGMENT) + sizeof(*STM_PSEGMENT) <= FIRST_READMARKER_PAGE*4096);
 
     assert(NB_SEGMENTS <= NB_SEGMENTS_MAX);
+    assert(TMP_COPY_PAGE < FIRST_READMARKER_PAGE);
     assert(FIRST_READMARKER_PAGE * 4096UL <= READMARKER_START);
     assert(READMARKER_START < READMARKER_END);
     assert(READMARKER_END <= 4096UL * FIRST_OBJECT_PAGE);
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -34,7 +34,7 @@
     uintptr_t nursery_end;
     struct stm_thread_local_s *running_thread;
 };
-#define STM_SEGMENT           ((stm_segment_info_t *)4352)
+#define STM_SEGMENT           ((stm_segment_info_t *)8192)
 
 
 struct stm_shadowentry_s {

From noreply at buildbot.pypy.org  Tue Sep 16 15:30:10 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 16 Sep 2014 15:30:10 +0200 (CEST)
Subject: [pypy-commit] creflect default: Anonymous structure,
	but directly from a 'typedef'
Message-ID: <20140916133010.681811D2304@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r44:c1f276dfda7b
Date: 2014-09-16 11:54 +0200
http://bitbucket.org/cffi/creflect/changeset/c1f276dfda7b/

Log:	Anonymous structure, but directly from a 'typedef'

diff --git a/creflect/cparser.py b/creflect/cparser.py
--- a/creflect/cparser.py
+++ b/creflect/cparser.py
@@ -55,6 +55,7 @@
         self.cdefblock = cdefblock
         self.first_lineno = first_lineno
         self._struct_union_enum_decl = {}
+        self._count_anonymous_struct = 0
 
     def to_ast(self):
         cparser = pycparser.CParser()
@@ -173,15 +174,18 @@
             #
             if isinstance(type, pycparser.c_ast.Struct):
                 # 'struct foobar'
-                return self._get_struct_union_enum_type('struct', type, name)
+                return self._get_struct_union_enum_type('struct', type, const,
+                                                        name)
             #
             if isinstance(type, pycparser.c_ast.Union):
                 # 'union foobar'
-                return self._get_struct_union_enum_type('union', type, name)
+                return self._get_struct_union_enum_type('union', type, const,
+                                                        name)
             #
             if isinstance(type, pycparser.c_ast.Enum):
                 # 'enum foobar'
-                return self._get_struct_union_enum_type('enum', type, name)
+                return self._get_struct_union_enum_type('enum', type, const,
+                                                        name)
         #
         if isinstance(typenode, pycparser.c_ast.FuncDecl):
             # a function type
@@ -189,18 +193,22 @@
         #
         # nested anonymous structs or unions end up here
         if isinstance(typenode, pycparser.c_ast.Struct):
-            return self._get_struct_union_enum_type('struct', typenode, name,
-                                                    nested=True)
+            return self._get_struct_union_enum_type('struct', typenode, const,
+                                                    name, nested=True)
         if isinstance(typenode, pycparser.c_ast.Union):
-            return self._get_struct_union_enum_type('union', typenode, name,
-                                                    nested=True)
+            return self._get_struct_union_enum_type('union', typenode, const,
+                                                    name, nested=True)
         #
         raise api.FFIError(":%d: bad or unsupported type declaration" %
                 typenode.coord.line)
 
-    def _get_struct_union_enum_type(self, kind, type, const):
+    def _get_struct_union_enum_type(self, kind, type, const, approx_name=None):
         name = type.name
-        assert name   # XXX
+        if not name:
+            if not approx_name:
+                self._count_anonymous_struct += 1
+                approx_name = '%d' % (self._count_anonymous_struct,)
+            name = '$%s' % approx_name
         result = model.StructOrUnionOrEnum(kind, name, const)
         #
         # get the type declaration or create it if needed
diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -241,7 +241,7 @@
         self.kind = kind
         self.name = name
         self.const = const
-        self.c_name_with_marker = '%s %s&' % (self.kind, self.name)
+        self.c_name_with_marker = '%s %s &' % (self.kind, self.name)
         if const:
             self.c_name_with_marker = 'const ' + self.c_name_with_marker
 
@@ -351,16 +351,20 @@
 
     def write_declaration(self, funcblock):
         tp = "%s %s" % (self.type.kind, self.type.name)
-        extra1 = "(long long)sizeof(%s)" % (tp,)    # total size
+        realtp = tp
+        if self.type.name.startswith('$'):
+            realtp = self.type.name[1:]
+            assert not realtp.isdigit()    # XXX
+        extra1 = "(long long)sizeof(%s)" % (realtp,)    # total size
         extra2 = ("(long long)(((char *)&((struct{char a; %s b;} *)0)->b)"
-                  " - (char *)0)" % (tp,))          # alignment
+                  " - (char *)0)" % (realtp,))          # alignment
         funcblock.sprintf(tp + r" {/*%lld,%lld*/\n",
                           extra="%s, %s" % (extra1, extra2),
                           extralength=40)
         #
         for fldname, fldtype in zip(self.fldnames, self.fldtypes):
             block = CodeBlock(funcblock.tr)
-            inspect = TypeInspector(block, tp, fldname)
+            inspect = TypeInspector(block, realtp, fldname)
             inspect.start()
             # get the offset of the field
             arraylevels = 0
@@ -375,11 +379,12 @@
                     comment = " (%dx)" % arraylevels
                 comment = inspect.get_comment(0, False, "an array%s" % comment)
                 block.writedecl("long long o = offsetof(%s, %s%s);%s"
-                                % (tp, fldname, "[0]" * arraylevels, comment))
+                                % (realtp, fldname, "[0]" * arraylevels,
+                                   comment))
             else:
                 comment = inspect.get_comment(0, False, "not an array")
                 block.writedecl("long long o = ((char *)&((%s *)0)->%s)"
-                                " - (char *)0;%s" % (tp, fldname, comment))
+                                " - (char *)0;%s" % (realtp, fldname, comment))
             #
             block.sprintf("  /*%lld*/", extra="o", extralength=20)
             block.sprintf_add_right(r';\n')
diff --git a/test/codegen/struct-001.c b/test/codegen/struct-001.c
--- a/test/codegen/struct-001.c
+++ b/test/codegen/struct-001.c
@@ -1,6 +1,6 @@
 struct foo_s {
   int aa;
-  int bb;
+  unsigned int bb;
 };
 
 # ____________________________________________________________
diff --git a/test/codegen/struct-005.c b/test/codegen/struct-005.c
new file mode 100644
--- /dev/null
+++ b/test/codegen/struct-005.c
@@ -0,0 +1,60 @@
+typedef struct { int aa; } foo_t;
+
+# ____________________________________________________________
+
+int teststruct_005(char *r)
+{
+    if (!r)
+        return 70 + 30 + 18 + 6 + 4 + 30;
+    r += sprintf(r, "struct $foo_t {/*%lld,%lld*/\n", (long long)sizeof(foo_t), (long long)(((char *)&((struct{char a; foo_t b;} *)0)->b) - (char *)0));
+    {
+        foo_t *p1;
+        long long o = ((char *)&((foo_t *)0)->aa) - (char *)0;  /* check that 'foo_t::aa' is not an array */
+        char b[sizeof(p1->aa)];
+        r += sprintf(r, "  /*%lld*/", o);
+        p1 = (void *)(((char *)b) - o);
+        (void)(p1->aa << 1);  /* check that 'foo_t::aa' is an integer type */
+        p1->aa = -1;  /* check that 'foo_t::aa' is not declared 'const' */
+        if (p1->aa > 0) {
+            if (sizeof(p1->aa) == 1 && p1->aa == 1)
+                r += sprintf(r, "_Bool");
+            else if (sizeof(p1->aa) == sizeof(unsigned int))
+                r += sprintf(r, "unsigned int");
+            else if (sizeof(p1->aa) == sizeof(unsigned short))
+                r += sprintf(r, "unsigned short");
+            else if (sizeof(p1->aa) == sizeof(unsigned char))
+                r += sprintf(r, "unsigned char");
+            else if (sizeof(p1->aa) == sizeof(unsigned long))
+                r += sprintf(r, "unsigned long");
+            else if (sizeof(p1->aa) == sizeof(unsigned long long))
+                r += sprintf(r, "unsigned long long");
+            else
+                r += sprintf(r, "uint%u_t", (int)sizeof(p1->aa) * 8);
+        }
+        else {
+            if (sizeof(p1->aa) == sizeof(int))
+                r += sprintf(r, "int");
+            else if (sizeof(p1->aa) == sizeof(short))
+                r += sprintf(r, "short");
+            else if (sizeof(p1->aa) == sizeof(signed char))
+                r += sprintf(r, "signed char");
+            else if (sizeof(p1->aa) == sizeof(long))
+                r += sprintf(r, "long");
+            else if (sizeof(p1->aa) == sizeof(long long))
+                r += sprintf(r, "long long");
+            else
+                r += sprintf(r, "int%u_t", (int)sizeof(p1->aa) * 8);
+        }
+        r += sprintf(r, " aa;\n");
+    }
+    r += sprintf(r, "};\n");
+    {
+        r += sprintf(r, "typedef struct $foo_t foo_t;\n");
+    }
+    return 0;
+}
+
+#expect struct $foo_t {
+#expect   int aa;
+#expect };
+#expect typedef struct $foo_t foo_t;
diff --git a/test/test_cgcompile.py b/test/test_cgcompile.py
--- a/test/test_cgcompile.py
+++ b/test/test_cgcompile.py
@@ -12,13 +12,17 @@
     #
     expected = []
     for line in inputlines:
-        if line.startswith('# _______'):
-            break
-        line = line.split('//')[0].strip()
-        if line:
-            expected.append(line)
-    else:
-        raise ValueError("no '# _______' found in %r" % (filename,))
+        if line.startswith('#expect'):
+            expected.append(line[8:].rstrip())
+    if not expected:
+        for line in inputlines:
+            if line.startswith('# _______'):
+                break
+            line = line.split('//')[0].rstrip()
+            if line:
+                expected.append(line)
+        else:
+            raise ValueError("no '# _______' found in %r" % (filename,))
     #
     basename = os.path.splitext(filename)[0]
     infile = str(udir.join('cg-' + filename))
@@ -64,7 +68,7 @@
     err = g.close()
     assert not err
     r_remove_addr = re.compile(r"/[*][-0-9a-fx,]+[*]/")
-    got = [r_remove_addr.sub('', line.strip()) for line in lines]
+    got = [r_remove_addr.sub('', line.rstrip()) for line in lines]
     compare_lists(got, expected)
 
 

From noreply at buildbot.pypy.org  Tue Sep 16 15:30:11 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 16 Sep 2014 15:30:11 +0200 (CEST)
Subject: [pypy-commit] creflect default: Next test
Message-ID: <20140916133011.75F701C3CF4@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r45:02683c15be69
Date: 2014-09-16 15:30 +0200
http://bitbucket.org/cffi/creflect/changeset/02683c15be69/

Log:	Next test

diff --git a/creflect/cparser.py b/creflect/cparser.py
--- a/creflect/cparser.py
+++ b/creflect/cparser.py
@@ -55,7 +55,6 @@
         self.cdefblock = cdefblock
         self.first_lineno = first_lineno
         self._struct_union_enum_decl = {}
-        self._count_anonymous_struct = 0
 
     def to_ast(self):
         cparser = pycparser.CParser()
@@ -94,7 +93,7 @@
     def _parse_decl(self, decl):
         node = decl.type
         if isinstance(node, pycparser.c_ast.FuncDecl):
-            tp = self._get_type(node, name=decl.name)
+            tp = self._get_type(node)
             assert isinstance(tp, model.RawFunctionType)
             tp = self._get_type_pointer(tp)
             self._declare('function ' + decl.name, tp)
@@ -133,7 +132,7 @@
               decl.type.type.type.names == ['__dotdotdot__']):
             realtype = model.unknown_ptr_type(decl.name)
         else:
-            realtype = self._get_type(decl.type, name=decl.name)
+            realtype = self._get_type(decl.type, approx_name = '$' + decl.name)
         self.declarations.append(model.TypeDef(decl.name, realtype))
 
     def _get_type_pointer(self, type, const):
@@ -141,7 +140,7 @@
             return type.as_function_pointer()
         return model.PointerType(type, const)
 
-    def _get_type(self, typenode, name=None, partial_length_ok=False):
+    def _get_type(self, typenode, approx_name=None, partial_length_ok=False):
         # first, dereference typedefs, if we have it already parsed, we're good
         if (isinstance(typenode, pycparser.c_ast.TypeDecl) and
             isinstance(typenode.type, pycparser.c_ast.IdentifierType) and
@@ -163,7 +162,10 @@
         if isinstance(typenode, pycparser.c_ast.PtrDecl):
             # pointer type
             const = 'const' in typenode.quals
-            return self._get_type_pointer(self._get_type(typenode.type), const)
+            if approx_name:
+                approx_name = '$' + approx_name
+            realtype = self._get_type(typenode.type, approx_name)
+            return self._get_type_pointer(realtype, const)
         #
         if isinstance(typenode, pycparser.c_ast.TypeDecl):
             const = 'const' in typenode.quals
@@ -175,17 +177,17 @@
             if isinstance(type, pycparser.c_ast.Struct):
                 # 'struct foobar'
                 return self._get_struct_union_enum_type('struct', type, const,
-                                                        name)
+                                                        approx_name)
             #
             if isinstance(type, pycparser.c_ast.Union):
                 # 'union foobar'
                 return self._get_struct_union_enum_type('union', type, const,
-                                                        name)
+                                                        approx_name)
             #
             if isinstance(type, pycparser.c_ast.Enum):
                 # 'enum foobar'
                 return self._get_struct_union_enum_type('enum', type, const,
-                                                        name)
+                                                        approx_name)
         #
         if isinstance(typenode, pycparser.c_ast.FuncDecl):
             # a function type
@@ -203,12 +205,11 @@
                 typenode.coord.line)
 
     def _get_struct_union_enum_type(self, kind, type, const, approx_name=None):
-        name = type.name
-        if not name:
-            if not approx_name:
-                self._count_anonymous_struct += 1
-                approx_name = '%d' % (self._count_anonymous_struct,)
-            name = '$%s' % approx_name
+        name = type.name or approx_name
+        if not name or name.startswith('$$$'):
+            self._parse_error("not implemented: anonymous 'struct' elsewhere "
+                              "than in 'typedef struct { ... } typename;' or "
+                              "'typedef struct { ... } *typename;'", type)
         result = model.StructOrUnionOrEnum(kind, name, const)
         #
         # get the type declaration or create it if needed
@@ -223,12 +224,16 @@
             else:
                 raise AssertionError("kind = %r" % (kind,))
             self._struct_union_enum_decl[key] = typedecl
-            self.declarations.append(typedecl)
+            must_add = True
+        else:
+            must_add = False
         #
         # is there a 'type.decls'?  If yes, then this is the place in the
         # C sources that declare the fields.
         if type.decls is not None:
             self._add_fields_declaration(typedecl, type.decls)
+        if must_add:
+            self.declarations.append(typedecl)
         return result
 
     def _add_fields_declaration(self, typedecl, fields):
diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -15,6 +15,8 @@
                 for name in (self._attrs_ + 'const')]
 
     def inspect_type(self, block, inspect):
+        if inspect.started:
+            inspect.assign_to_p1('0')
         block.sprintf_add_both_sides(self.c_name_with_marker)
 
     def __eq__(self, other):
@@ -37,11 +39,6 @@
         if const:
             self.c_name_with_marker = 'const ' + self.c_name_with_marker
 
-    def inspect_type(self, block, inspect):
-        if inspect.started:
-            inspect.assign_to_p1('0')
-        BaseType.inspect_type(self, block, inspect)
-
 void_type = VoidType(const=False)
 const_void_type = VoidType(const=True)
 
@@ -262,7 +259,10 @@
 
     def start(self):
         if not self.started:
-            self.block.writedecl('%s *p1;' % (self.typename,))
+            if not self.typename.startswith('*'):
+                self.block.writedecl('%s *p1;' % (self.typename,))
+            else:
+                self.block.writedecl('%s p1;' % (self.typename[1:],))
             self.levels = []
             self.started = True
             self.after_star_p1_assignment = []
@@ -351,20 +351,37 @@
 
     def write_declaration(self, funcblock):
         tp = "%s %s" % (self.type.kind, self.type.name)
-        realtp = tp
-        if self.type.name.startswith('$'):
+        if not self.type.name.startswith('$'):
+            realtp = tp
+        else:
             realtp = self.type.name[1:]
-            assert not realtp.isdigit()    # XXX
-        extra1 = "(long long)sizeof(%s)" % (realtp,)    # total size
-        extra2 = ("(long long)(((char *)&((struct{char a; %s b;} *)0)->b)"
-                  " - (char *)0)" % (realtp,))          # alignment
-        funcblock.sprintf(tp + r" {/*%lld,%lld*/\n",
-                          extra="%s, %s" % (extra1, extra2),
-                          extralength=40)
+        if not realtp.startswith('$'):
+            sizetp = realtp
+            insptp = realtp
+            ptrtp = '%s *' % (realtp,)
+            include_alignment = True
+        else:
+            ptrtp = realtp[1:]
+            assert not ptrtp.startswith('$')
+            sizetp = '*(%s)0' % ptrtp
+            insptp = '*%s' % ptrtp
+            include_alignment = False
+        #
+        extra1 = "(long long)sizeof(%s)" % (sizetp,)    # total size
+        if include_alignment:
+            extra2 = ("(long long)(((char *)&((struct{char a; %s b;} *)0)->b)"
+                    " - (char *)0)" % (realtp,))          # alignment
+            funcblock.sprintf(tp + r" {/*%lld,%lld*/\n",
+                              extra="%s, %s" % (extra1, extra2),
+                              extralength=40)
+        else:
+            funcblock.sprintf(tp + r" {/*%lld*/\n",
+                              extra=extra1,
+                              extralength=20)
         #
         for fldname, fldtype in zip(self.fldnames, self.fldtypes):
             block = CodeBlock(funcblock.tr)
-            inspect = TypeInspector(block, realtp, fldname)
+            inspect = TypeInspector(block, insptp, fldname)
             inspect.start()
             # get the offset of the field
             arraylevels = 0
@@ -383,8 +400,8 @@
                                    comment))
             else:
                 comment = inspect.get_comment(0, False, "not an array")
-                block.writedecl("long long o = ((char *)&((%s *)0)->%s)"
-                                " - (char *)0;%s" % (realtp, fldname, comment))
+                block.writedecl("long long o = ((char *)&((%s)0)->%s)"
+                                " - (char *)0;%s" % (ptrtp, fldname, comment))
             #
             block.sprintf("  /*%lld*/", extra="o", extralength=20)
             block.sprintf_add_right(r';\n')
diff --git a/test/codegen/struct-005b.c b/test/codegen/struct-005b.c
new file mode 100644
--- /dev/null
+++ b/test/codegen/struct-005b.c
@@ -0,0 +1,64 @@
+typedef struct { int aa; } *foo_p;
+
+# ____________________________________________________________
+
+int teststruct_005b(char *r)
+{
+    if (!r)
+        return 46 + 30 + 18 + 6 + 4 + 32;
+    r += sprintf(r, "struct $$foo_p {/*%lld*/\n", (long long)sizeof(*(foo_p)0));
+    {
+        foo_p p1;
+        long long o = ((char *)&((foo_p)0)->aa) - (char *)0;  /* check that '*foo_p::aa' is not an array */
+        char b[sizeof(p1->aa)];
+        r += sprintf(r, "  /*%lld*/", o);
+        p1 = (void *)(((char *)b) - o);
+        (void)(p1->aa << 1);  /* check that '*foo_p::aa' is an integer type */
+        p1->aa = -1;  /* check that '*foo_p::aa' is not declared 'const' */
+        if (p1->aa > 0) {
+            if (sizeof(p1->aa) == 1 && p1->aa == 1)
+                r += sprintf(r, "_Bool");
+            else if (sizeof(p1->aa) == sizeof(unsigned int))
+                r += sprintf(r, "unsigned int");
+            else if (sizeof(p1->aa) == sizeof(unsigned short))
+                r += sprintf(r, "unsigned short");
+            else if (sizeof(p1->aa) == sizeof(unsigned char))
+                r += sprintf(r, "unsigned char");
+            else if (sizeof(p1->aa) == sizeof(unsigned long))
+                r += sprintf(r, "unsigned long");
+            else if (sizeof(p1->aa) == sizeof(unsigned long long))
+                r += sprintf(r, "unsigned long long");
+            else
+                r += sprintf(r, "uint%u_t", (int)sizeof(p1->aa) * 8);
+        }
+        else {
+            if (sizeof(p1->aa) == sizeof(int))
+                r += sprintf(r, "int");
+            else if (sizeof(p1->aa) == sizeof(short))
+                r += sprintf(r, "short");
+            else if (sizeof(p1->aa) == sizeof(signed char))
+                r += sprintf(r, "signed char");
+            else if (sizeof(p1->aa) == sizeof(long))
+                r += sprintf(r, "long");
+            else if (sizeof(p1->aa) == sizeof(long long))
+                r += sprintf(r, "long long");
+            else
+                r += sprintf(r, "int%u_t", (int)sizeof(p1->aa) * 8);
+        }
+        r += sprintf(r, " aa;\n");
+    }
+    r += sprintf(r, "};\n");
+    {
+        foo_p *p1;
+        char *p2;
+        p1 = (void *)&p2;
+        *p1 = (void *)0;    /* check that 'foo_p' is a pointer type */
+        r += sprintf(r, "typedef struct $$foo_p *foo_p;\n");
+    }
+    return 0;
+}
+
+#expect struct $$foo_p {
+#expect   int aa;
+#expect };
+#expect typedef struct $$foo_p *foo_p;

From noreply at buildbot.pypy.org  Tue Sep 16 15:51:50 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 16 Sep 2014 15:51:50 +0200 (CEST)
Subject: [pypy-commit] creflect default: Move the size,
	align comment before the '{'
Message-ID: <20140916135150.32AA21D23C3@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r46:d6f83b05a000
Date: 2014-09-16 15:37 +0200
http://bitbucket.org/cffi/creflect/changeset/d6f83b05a000/

Log:	Move the size,align comment before the '{'

diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -371,11 +371,11 @@
         if include_alignment:
             extra2 = ("(long long)(((char *)&((struct{char a; %s b;} *)0)->b)"
                     " - (char *)0)" % (realtp,))          # alignment
-            funcblock.sprintf(tp + r" {/*%lld,%lld*/\n",
+            funcblock.sprintf(tp + r" /*%lld,%lld*/{\n",
                               extra="%s, %s" % (extra1, extra2),
                               extralength=40)
         else:
-            funcblock.sprintf(tp + r" {/*%lld*/\n",
+            funcblock.sprintf(tp + r" /*%lld*/{\n",
                               extra=extra1,
                               extralength=20)
         #
diff --git a/test/codegen/struct-001.c b/test/codegen/struct-001.c
--- a/test/codegen/struct-001.c
+++ b/test/codegen/struct-001.c
@@ -9,7 +9,7 @@
 {
     if (!r)
         return 69 + 30 + 18 + 6 + 30 + 18 + 6 + 4;
-    r += sprintf(r, "struct foo_s {/*%lld,%lld*/\n", (long long)sizeof(struct foo_s), (long long)(((char *)&((struct{char a; struct foo_s b;} *)0)->b) - (char *)0));
+    r += sprintf(r, "struct foo_s /*%lld,%lld*/{\n", (long long)sizeof(struct foo_s), (long long)(((char *)&((struct{char a; struct foo_s b;} *)0)->b) - (char *)0));
     {
         struct foo_s *p1;
         long long o = ((char *)&((struct foo_s *)0)->aa) - (char *)0;  /* check that 'struct foo_s::aa' is not an array */
diff --git a/test/codegen/struct-005.c b/test/codegen/struct-005.c
--- a/test/codegen/struct-005.c
+++ b/test/codegen/struct-005.c
@@ -6,7 +6,7 @@
 {
     if (!r)
         return 70 + 30 + 18 + 6 + 4 + 30;
-    r += sprintf(r, "struct $foo_t {/*%lld,%lld*/\n", (long long)sizeof(foo_t), (long long)(((char *)&((struct{char a; foo_t b;} *)0)->b) - (char *)0));
+    r += sprintf(r, "struct $foo_t /*%lld,%lld*/{\n", (long long)sizeof(foo_t), (long long)(((char *)&((struct{char a; foo_t b;} *)0)->b) - (char *)0));
     {
         foo_t *p1;
         long long o = ((char *)&((foo_t *)0)->aa) - (char *)0;  /* check that 'foo_t::aa' is not an array */
diff --git a/test/codegen/struct-005b.c b/test/codegen/struct-005b.c
--- a/test/codegen/struct-005b.c
+++ b/test/codegen/struct-005b.c
@@ -6,7 +6,7 @@
 {
     if (!r)
         return 46 + 30 + 18 + 6 + 4 + 32;
-    r += sprintf(r, "struct $$foo_p {/*%lld*/\n", (long long)sizeof(*(foo_p)0));
+    r += sprintf(r, "struct $$foo_p /*%lld*/{\n", (long long)sizeof(*(foo_p)0));
     {
         foo_p p1;
         long long o = ((char *)&((foo_p)0)->aa) - (char *)0;  /* check that '*foo_p::aa' is not an array */

From noreply at buildbot.pypy.org  Tue Sep 16 15:51:51 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 16 Sep 2014 15:51:51 +0200 (CEST)
Subject: [pypy-commit] creflect default: Uniformize the class names
Message-ID: <20140916135151.35BB01D23C3@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r47:08e8a5eea3b4
Date: 2014-09-16 15:39 +0200
http://bitbucket.org/cffi/creflect/changeset/08e8a5eea3b4/

Log:	Uniformize the class names

diff --git a/creflect/cparser.py b/creflect/cparser.py
--- a/creflect/cparser.py
+++ b/creflect/cparser.py
@@ -133,7 +133,7 @@
             realtype = model.unknown_ptr_type(decl.name)
         else:
             realtype = self._get_type(decl.type, approx_name = '$' + decl.name)
-        self.declarations.append(model.TypeDef(decl.name, realtype))
+        self.declarations.append(model.TypeDefDecl(decl.name, realtype))
 
     def _get_type_pointer(self, type, const):
         if isinstance(type, model.RawFunctionType):
diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -413,7 +413,7 @@
         funcblock.sprintf(r"};\n")
 
 
-class TypeDef(object):
+class TypeDefDecl(object):
     def __init__(self, name, type):
         self.name = name
         self.type = type

From noreply at buildbot.pypy.org  Tue Sep 16 18:23:28 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 16 Sep 2014 18:23:28 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: check file mode in pypy
 until we can use rfile's check
Message-ID: <20140916162328.EDD0B1C03C7@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73550:2db3a031ff88
Date: 2014-09-16 12:21 -0400
http://bitbucket.org/pypy/pypy/changeset/2db3a031ff88/

Log:	check file mode in pypy until we can use rfile's check

diff --git a/pypy/module/_file/interp_file.py b/pypy/module/_file/interp_file.py
--- a/pypy/module/_file/interp_file.py
+++ b/pypy/module/_file/interp_file.py
@@ -71,6 +71,24 @@
             self.w_name = w_name
         getopenstreams(self.space)[stream] = None
 
+    def check_mode_ok(self, mode):
+        # XXX get ValueError message from rpython working so we can use rfile's
+        space = self.space
+        if len(mode) == 0:
+            raise oefmt(space.w_ValueError, "empty mode string")
+        upos = mode.find('U')
+        if upos >= 0:
+            mode = mode[:upos] + mode[upos+1:]
+            first = mode[0:1]
+            if first == 'w' or first == 'a':
+                raise oefmt(space.w_ValueError,
+                            "universal newline mode can only be used with "
+                            "modes starting with 'r'")
+        elif mode[0] != 'r' and mode[0] != 'w' and mode[0] != 'a':
+            raise oefmt(space.w_ValueError,
+                        "mode string must begin with one of 'r', 'w', 'a' "
+                        "or 'U', not '%s'", mode)
+
     def check_closed(self):
         if self.stream is None:
             raise OperationError(self.space.w_ValueError,
@@ -101,12 +119,14 @@
     @unwrap_spec(mode=str, buffering=int)
     def direct___init__(self, w_name, mode='r', buffering=-1):
         self.direct_close()
+        self.check_mode_ok(mode)
         self.w_name = w_name
         stream = rfile.create_file(fsencode_w(self.space, w_name), mode, buffering)
         self.fdopenstream(stream, mode)
 
     def direct_fdopen(self, fd, mode='r', buffering=-1):
         self.direct_close()
+        self.check_mode_ok(mode)
         self.w_name = self.space.wrap('')
         stream = rfile.create_fdopen_rfile(fd, mode, buffering)
         self.fdopenstream(stream, mode)
@@ -243,8 +263,6 @@
     def file_fdopen(self, fd, mode="r", buffering=-1):
         try:
             self.direct_fdopen(fd, mode, buffering)
-        except ValueError as e:
-            raise OperationError(self.space.w_ValueError, self.space.wrap(str(e)))
         except IOError as e:
             space = self.space
             w_error = space.call_function(space.w_IOError, space.wrap(e.errno), space.wrap(e.strerror), self.w_name)
@@ -286,8 +304,6 @@
                 try:
                     try:
                         result = self.direct_%(name)s(%(callsig)s)
-                    except ValueError as e:
-                        raise OperationError(space.w_ValueError, space.wrap(str(e)))
                     except IOError as e:
                         w_error = space.call_function(space.w_IOError, space.wrap(e.errno), space.wrap(e.strerror), self.w_name)
                         raise OperationError(space.w_IOError, w_error)

From noreply at buildbot.pypy.org  Tue Sep 16 18:43:37 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Tue, 16 Sep 2014 18:43:37 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: bootstrap Pycon UK talk by copying
	pycon-italy-2014
Message-ID: <20140916164337.5D2C51C0F86@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: extradoc
Changeset: r5402:6d62743996ef
Date: 2014-09-16 05:13 +0100
http://bitbucket.org/pypy/extradoc/changeset/6d62743996ef/

Log:	bootstrap Pycon UK talk by copying pycon-italy-2014

diff --git a/talk/ep2014/status/Makefile b/talk/pycon-uk-2014/Makefile
copy from talk/ep2014/status/Makefile
copy to talk/pycon-uk-2014/Makefile
--- a/talk/ep2014/status/Makefile
+++ b/talk/pycon-uk-2014/Makefile
@@ -1,11 +1,5 @@
-# you can find rst2beamer.py here:
-# https://bitbucket.org/antocuni/env/raw/default/bin/rst2beamer.py
-
-# WARNING: to work, it needs this patch for docutils
-# https://sourceforge.net/tracker/?func=detail&atid=422032&aid=1459707&group_id=38414
-
 talk.pdf: talk.rst author.latex stylesheet.latex
-	python `which rst2beamer.py` --stylesheet=stylesheet.latex --documentoptions=14pt talk.rst talk.latex || exit
+	rst2beamer --stylesheet=stylesheet.latex --documentoptions=14pt talk.rst talk.latex || exit
 	#/home/antocuni/.virtualenvs/rst2beamer/bin/python `which rst2beamer.py` --stylesheet=stylesheet.latex --documentoptions=14pt talk.rst talk.latex || exit
 	sed 's/\\date{}/\\input{author.latex}/' -i talk.latex || exit
 	#sed 's/\\maketitle/\\input{title.latex}/' -i talk.latex || exit
diff --git a/talk/pycon-italy-2014/author.latex b/talk/pycon-uk-2014/author.latex
copy from talk/pycon-italy-2014/author.latex
copy to talk/pycon-uk-2014/author.latex
diff --git a/talk/pycon-italy-2014/beamerdefs.txt b/talk/pycon-uk-2014/beamerdefs.txt
copy from talk/pycon-italy-2014/beamerdefs.txt
copy to talk/pycon-uk-2014/beamerdefs.txt
diff --git a/talk/pycon-italy-2014/speed.png b/talk/pycon-uk-2014/speed.png
copy from talk/pycon-italy-2014/speed.png
copy to talk/pycon-uk-2014/speed.png
diff --git a/talk/pycon-italy-2014/stylesheet.latex b/talk/pycon-uk-2014/stylesheet.latex
copy from talk/pycon-italy-2014/stylesheet.latex
copy to talk/pycon-uk-2014/stylesheet.latex
diff --git a/talk/pycon-italy-2014/talk.pdf.info b/talk/pycon-uk-2014/talk.pdf.info
copy from talk/pycon-italy-2014/talk.pdf.info
copy to talk/pycon-uk-2014/talk.pdf.info
diff --git a/talk/pycon-italy-2014/talk.rst b/talk/pycon-uk-2014/talk.rst
copy from talk/pycon-italy-2014/talk.rst
copy to talk/pycon-uk-2014/talk.rst
diff --git a/talk/pycon-italy-2014/title.latex b/talk/pycon-uk-2014/title.latex
copy from talk/pycon-italy-2014/title.latex
copy to talk/pycon-uk-2014/title.latex

From noreply at buildbot.pypy.org  Tue Sep 16 18:47:27 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Tue, 16 Sep 2014 18:47:27 +0200 (CEST)
Subject: [pypy-commit] buildbot default: typo, escape arguments to schroot
Message-ID: <20140916164727.B27EB1C335E@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus 
Branch: 
Changeset: r919:2b6cb72277b1
Date: 2014-09-16 19:48 +0300
http://bitbucket.org/pypy/buildbot/changeset/2b6cb72277b1/

Log:	typo, escape arguments to schroot

diff --git a/bbhook/run.py b/bbhook/run.py
--- a/bbhook/run.py
+++ b/bbhook/run.py
@@ -17,7 +17,7 @@
     HOST_NAME = ''
     PORT_NUMBER = 9237
     # WARNING:
-    # deply is meant as a argument for running public facing,
+    # deploy is meant as an argument for running public facing,
     # its not supposed to be used when running a proxy setup
     main.app.run(
         host = HOST_NAME if 'deploy' in sys.argv else '127.0.0.1',
diff --git a/bot2/pypybuildbot/arm_master.py b/bot2/pypybuildbot/arm_master.py
--- a/bot2/pypybuildbot/arm_master.py
+++ b/bot2/pypybuildbot/arm_master.py
@@ -23,7 +23,7 @@
     translationArgs=crosstranslationargs + ['-O2'],
     platform='linux-armel',
     interpreter='pypy',
-    prefix=['schroot', '-c', 'armel'])
+    prefix=['schroot', '-c', 'armel', '--'])
 
 pypyJITCrossTranslationFactoryARM = pypybuilds.NightlyBuild(
     translationArgs=(crosstranslationargs
@@ -31,14 +31,14 @@
                         + crosstranslationjitargs),
     platform='linux-armel',
     interpreter='pypy',
-    prefix=['schroot', '-c', 'armel'],
+    prefix=['schroot', '-c', 'armel', '--'],
     trigger='JITLINUXARM_scheduler')
 
 pypyCrossTranslationFactoryRaspbianHF = pypybuilds.NightlyBuild(
     translationArgs=crosstranslationargs + ['-O2'],
     platform='linux-armhf-raspbian',
     interpreter='pypy',
-    prefix=['schroot', '-c', 'raspbian'],
+    prefix=['schroot', '-c', 'raspbian', '--'],
     trigger='APPLVLLINUXARMHF_RASPBIAN_scheduler')
 
 pypyJITCrossTranslationFactoryRaspbianHF = pypybuilds.NightlyBuild(
@@ -47,7 +47,7 @@
                         + crosstranslationjitargs),
     platform='linux-armhf-raspbian',
     interpreter='pypy',
-    prefix=['schroot', '-c', 'raspbian'],
+    prefix=['schroot', '-c', 'raspbian', '--'],
     trigger='JITLINUXARMHF_RASPBIAN_scheduler')
 
 pypyJITCrossTranslationFactoryRaringHF = pypybuilds.NightlyBuild(
@@ -56,7 +56,7 @@
                         + crosstranslationjitargs),
     platform='linux-armhf-raring',
     interpreter='pypy',
-    prefix=['schroot', '-c', 'raring'],
+    prefix=['schroot', '-c', 'raring', '--'],
     trigger='JITLINUXARMHF_RARING_scheduler')
 
 pypyARMJITTranslatedTestFactory = pypybuilds.TranslatedTests(

From noreply at buildbot.pypy.org  Tue Sep 16 18:49:13 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 16 Sep 2014 18:49:13 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: give this test more time
	to pass
Message-ID: <20140916164913.552F01C335E@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73551:7fa3a1ce2684
Date: 2014-09-16 12:48 -0400
http://bitbucket.org/pypy/pypy/changeset/7fa3a1ce2684/

Log:	give this test more time to pass

diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -1325,7 +1325,7 @@
             for i in range(5):
                 thread.start_new_thread(check, ())
 
-            for n in range(100):
+            for n in range(200):
                 for i in range(105): time.sleep(0.001)
                 if len(got) == 5:
                     break

From noreply at buildbot.pypy.org  Tue Sep 16 18:58:18 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 16 Sep 2014 18:58:18 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: add a comment
Message-ID: <20140916165818.4F1861C3CF4@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73552:99cbbfbb8ded
Date: 2014-09-16 12:57 -0400
http://bitbucket.org/pypy/pypy/changeset/99cbbfbb8ded/

Log:	add a comment

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -482,6 +482,7 @@
             return p - 1
 
     def _get_line_fgets(self):
+        # XXX use buffer size logic from cpython
         with rffi.scoped_alloc_buffer(BASE_LINE_SIZE) as buf:
             c = self._get_line_fgets_single(buf.raw)
             if c >= 0:

From noreply at buildbot.pypy.org  Tue Sep 16 19:10:19 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 16 Sep 2014 19:10:19 +0200 (CEST)
Subject: [pypy-commit] pypy gc_no_cleanup_nursery: easy fix, thanks armin
Message-ID: <20140916171019.2A4BC1C023E@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: gc_no_cleanup_nursery
Changeset: r73553:ba4d93c75370
Date: 2014-09-16 11:08 -0600
http://bitbucket.org/pypy/pypy/changeset/ba4d93c75370/

Log:	easy fix, thanks armin

diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -184,7 +184,6 @@
         # XXX this maybe should go to optimizer, so we can remove extra ops?
         ofs, size, _ = self.cpu.unpack_arraydescr_size(arraydescr)
         v_arr_plus_ofs = BoxInt()
-        v_arrsize = BoxInt()
         v_totalsize = BoxInt()
         gcdescr = self.gc_ll_descr
         ops = [
@@ -192,6 +191,7 @@
         ]
 
         if v_arrsize is None:
+            v_arrsize = BoxInt()
             o = ResOperation(rop.ARRAYLEN_GC, [v_arr], v_arrsize,
                              descr=arraydescr)
             ops.append(o)

From noreply at buildbot.pypy.org  Tue Sep 16 19:29:07 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 16 Sep 2014 19:29:07 +0200 (CEST)
Subject: [pypy-commit] pypy gc_no_cleanup_nursery: anothe rfix
Message-ID: <20140916172907.5353D1D23C3@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: gc_no_cleanup_nursery
Changeset: r73554:de5bd5fa89c6
Date: 2014-09-16 11:27 -0600
http://bitbucket.org/pypy/pypy/changeset/de5bd5fa89c6/

Log:	anothe rfix

diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -187,7 +187,7 @@
         v_totalsize = BoxInt()
         gcdescr = self.gc_ll_descr
         ops = [
-            ResOperation(rop.INT_ADD, [v_arr, ConstInt(size)], v_arr_plus_ofs),
+            ResOperation(rop.INT_ADD, [v_arr, ConstInt(ofs)], v_arr_plus_ofs),
         ]
 
         if v_arrsize is None:

From noreply at buildbot.pypy.org  Tue Sep 16 19:31:14 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 16 Sep 2014 19:31:14 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: cleanup
Message-ID: <20140916173114.AE3221D23C3@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73555:951e6f0afdcb
Date: 2014-09-16 13:15 -0400
http://bitbucket.org/pypy/pypy/changeset/951e6f0afdcb/

Log:	cleanup

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -12,13 +12,11 @@
 from rpython.rtyper.tool import rffi_platform as platform
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 
+MS_WINDOWS = os.name == 'nt'
 
 includes = ['stdio.h', 'sys/types.h']
 if os.name == "posix":
     includes += ['unistd.h']
-    fileno = 'fileno'
-else:
-    fileno = '_fileno'
 
 stdio_streams = ['stdin', 'stdout', 'stderr']
 separate_module_sources = ['\n'.join('FILE* get_%s() { return %s; }' % (s, s)
@@ -65,10 +63,12 @@
 def llexternal(*args, **kwargs):
     return rffi.llexternal(*args, compilation_info=eci, **kwargs)
 
+_fdopen = ('_' if MS_WINDOWS else '') + 'fdopen'
+_fileno = ('_' if MS_WINDOWS else '') + 'fileno'
+
 c_fopen = llexternal('fopen', [rffi.CCHARP, rffi.CCHARP], FILEP, macro=True)
 c_popen = llexternal('popen', [rffi.CCHARP, rffi.CCHARP], FILEP)
-c_fdopen = llexternal(('_' if os.name == 'nt' else '') + 'fdopen',
-                      [rffi.INT, rffi.CCHARP], FILEP)
+c_fdopen = llexternal(_fdopen, [rffi.INT, rffi.CCHARP], FILEP)
 c_tmpfile = llexternal('tmpfile', [], FILEP)
 
 # Note: the following functions are called from __del__ methods,
@@ -94,7 +94,7 @@
 c_getc_unlocked = llexternal('getc_unlocked', [FILEP], rffi.INT)
 c_ungetc = llexternal('ungetc', [rffi.INT, FILEP], rffi.INT)
 
-if os.name == 'nt':
+if MS_WINDOWS:
     c_flockfile = lambda ll_file: None
     c_funlockfile = lambda ll_file: None
     c_getc_unlocked = c_getc
@@ -108,7 +108,7 @@
 c_fwrite = llexternal('fwrite', [rffi.CCHARP, rffi.SIZE_T, rffi.SIZE_T, FILEP],
                       rffi.SIZE_T)
 
-if os.name != 'nt':
+if not MS_WINDOWS:
     assert rffi.sizeof(OFF_T) == 8
     if sys.platform.startswith('linux'):
         c_ftruncate = llexternal('ftruncate64', [rffi.INT, OFF_T], rffi.INT)
@@ -123,7 +123,7 @@
     c_fseek = llexternal('_fseeki64', [FILEP, rffi.LONGLONG, rffi.INT], rffi.INT)
     c_ftell = llexternal('_ftelli64', [FILEP], rffi.LONGLONG)
 
-c_fileno = llexternal(fileno, [FILEP], rffi.INT, releasegil=False)
+c_fileno = llexternal(_fileno, [FILEP], rffi.INT, releasegil=False)
 c_feof = llexternal('feof', [FILEP], rffi.INT, releasegil=False)
 c_ferror = llexternal('ferror', [FILEP], rffi.INT, releasegil=False)
 c_clearerr = llexternal('clearerr', [FILEP], lltype.Void, releasegil=False)

From noreply at buildbot.pypy.org  Tue Sep 16 19:35:14 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 16 Sep 2014 19:35:14 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: fix ftruncate on win32
Message-ID: <20140916173514.C27621D23C3@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73556:43946fb28835
Date: 2014-09-16 10:34 -0700
http://bitbucket.org/pypy/pypy/changeset/43946fb28835/

Log:	fix ftruncate on win32

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -13,6 +13,8 @@
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 
 MS_WINDOWS = os.name == 'nt'
+if MS_WINDOWS:
+    from rpython.rlib.rwin32 import _get_osfhandle, SetEndOfFile, INVALID_HANDLE_VALUE
 
 includes = ['stdio.h', 'sys/types.h']
 if os.name == "posix":
@@ -612,11 +614,25 @@
         if res != 0:
             c_clearerr(ll_file)
             raise _from_errno(IOError)
-        # XXX use fseek/SetEndOfFile on windows
-        res = c_ftruncate(c_fileno(ll_file), arg)
-        if res != 0:
-            c_clearerr(ll_file)
-            raise _from_errno(IOError)
+        if MS_WINDOWS:
+            ret = c_fseek(ll_file, arg, os.SEEK_SET)
+            if ret:
+                c_clearerr(ll_file)
+                raise _from_errno(IOError)
+            handle = _get_osfhandle(c_fileno(ll_file))
+            ret = handle == INVALID_HANDLE_VALUE
+            if ret == 0:
+                ret = SetEndOfFile(handle) == 0
+                if ret:
+                    rposix.set_errno(errno.EACCES)
+            if ret:
+                c_clearerr(ll_file)
+                raise _from_errno(IOError)
+        else:
+            ret = c_ftruncate(c_fileno(ll_file), arg)
+            if ret != 0:
+                c_clearerr(ll_file)
+                raise _from_errno(IOError)
         res = c_fseek(ll_file, pos, os.SEEK_SET)
         if res != 0:
             c_clearerr(ll_file)
diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py
--- a/rpython/rlib/rwin32.py
+++ b/rpython/rlib/rwin32.py
@@ -154,6 +154,8 @@
             raise WindowsError(ERROR_INVALID_HANDLE, "Invalid file handle")
         return handle
 
+    SetEndOfFile = rffi.llexternal('SetEndOfFile', [HANDLE], BOOL)
+
     def build_winerror_to_errno():
         """Build a dictionary mapping windows error numbers to POSIX errno.
         The function returns the dict, and the default value for codes not
diff --git a/rpython/rlib/streamio.py b/rpython/rlib/streamio.py
--- a/rpython/rlib/streamio.py
+++ b/rpython/rlib/streamio.py
@@ -188,15 +188,13 @@
 
 
 if sys.platform == "win32":
-    from rpython.rlib.rwin32 import BOOL, HANDLE, get_osfhandle, GetLastError
+    from rpython.rlib.rwin32 import get_osfhandle, GetLastError, SetEndOfFile
     from rpython.translator.tool.cbuild import ExternalCompilationInfo
     from rpython.rtyper.lltypesystem import rffi
 
     _eci = ExternalCompilationInfo()
     _setmode = rffi.llexternal('_setmode', [rffi.INT, rffi.INT], rffi.INT,
                                compilation_info=_eci)
-    SetEndOfFile = rffi.llexternal('SetEndOfFile', [HANDLE], BOOL,
-                                   compilation_info=_eci)
 
     def _setfd_binary(fd):
         # Allow this to succeed on invalid fd's

From noreply at buildbot.pypy.org  Tue Sep 16 20:12:58 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 16 Sep 2014 20:12:58 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: fix readinto sanity
	checks
Message-ID: <20140916181258.DDDF01C023E@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73557:a35017e41180
Date: 2014-09-16 14:12 -0400
http://bitbucket.org/pypy/pypy/changeset/a35017e41180/

Log:	fix readinto sanity checks

diff --git a/pypy/module/_file/interp_file.py b/pypy/module/_file/interp_file.py
--- a/pypy/module/_file/interp_file.py
+++ b/pypy/module/_file/interp_file.py
@@ -452,6 +452,8 @@
         """readinto() -> Undocumented.  Don't use this; it may go away."""
         # XXX not the most efficient solution as it doesn't avoid the copying
         space = self.space
+        self.check_closed()
+        self.check_readable()
         rwbuffer = space.writebuf_w(w_rwbuffer)
         ntodo = rwbuffer.getlength()
         ndone = 0
diff --git a/pypy/module/_file/test/test_file_extra.py b/pypy/module/_file/test/test_file_extra.py
--- a/pypy/module/_file/test/test_file_extra.py
+++ b/pypy/module/_file/test/test_file_extra.py
@@ -582,6 +582,12 @@
         assert n == 6
         assert len(a) == 10
         assert a.tostring() == 'foobar6789'
+        exc = raises(ValueError, f.readinto, bytearray(''))
+        assert str(exc.value) == "I/O operation on closed file"
+        f = open(fn, 'wb')
+        exc = raises(IOError, f.readinto, bytearray(''))
+        assert str(exc.value) == "File not open for reading"
+        f.close()
 
     def test_weakref(self):
         """Files are weakrefable."""

From noreply at buildbot.pypy.org  Tue Sep 16 20:26:25 2014
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 16 Sep 2014 20:26:25 +0200 (CEST)
Subject: [pypy-commit] jitviewer default: Don't try so many tricks to find
 rpython, just give a good error message
Message-ID: <20140916182625.908401D2304@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor 
Branch: 
Changeset: r261:d416bac6c441
Date: 2014-09-16 11:25 -0700
http://bitbucket.org/pypy/jitviewer/changeset/d416bac6c441/

Log:	Don't try so many tricks to find rpython, just give a good error
	message

diff --git a/_jitviewer/app.py b/_jitviewer/app.py
--- a/_jitviewer/app.py
+++ b/_jitviewer/app.py
@@ -31,13 +31,8 @@
 try:
     import rpython
 except ImportError:
-    import __pypy__
-    sys.path.append(os.path.join(__pypy__.__file__, '..', '..', '..'))
-    try:
-        import rpython
-    except ImportError:
-        failout('Could not import pypy module, make sure to '
-            'add the pypy module to PYTHONPATH')
+    failout('Could not import the rpython module, make sure to add the '
+            'rpython module to PYTHONPATH')
 
 import jinja2
 if jinja2.__version__ < '2.6':

From noreply at buildbot.pypy.org  Tue Sep 16 20:33:28 2014
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 16 Sep 2014 20:33:28 +0200 (CEST)
Subject: [pypy-commit] pypy default: Issue #1867 -- fixed performance and
 memory issues with operator.methodcaller
Message-ID: <20140916183328.523EC1D2304@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor 
Branch: 
Changeset: r73558:c7fcbd1bcedd
Date: 2014-09-16 11:32 -0700
http://bitbucket.org/pypy/pypy/changeset/c7fcbd1bcedd/

Log:	Issue #1867 -- fixed performance and memory issues with
	operator.methodcaller

diff --git a/pypy/module/operator/app_operator.py b/pypy/module/operator/app_operator.py
--- a/pypy/module/operator/app_operator.py
+++ b/pypy/module/operator/app_operator.py
@@ -120,7 +120,11 @@
     return builtinify(getter)
 
 
-def methodcaller(method_name, *args, **kwargs):
-    def call(obj):
-        return getattr(obj, method_name)(*args, **kwargs)
-    return builtinify(call)
+class methodcaller(object):
+    def __init__(self, method_name, *args, **kwargs):
+        self._method_name = method_name
+        self._args = args
+        self._kwargs = kwargs
+
+    def __call__(self, obj):
+        return getattr(obj, self._method_name)(*self._args, **self._kwargs)

From noreply at buildbot.pypy.org  Tue Sep 16 20:33:29 2014
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 16 Sep 2014 20:33:29 +0200 (CEST)
Subject: [pypy-commit] pypy default: merged upstream
Message-ID: <20140916183329.BA0001D2304@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor 
Branch: 
Changeset: r73559:b126182576d1
Date: 2014-09-16 11:32 -0700
http://bitbucket.org/pypy/pypy/changeset/b126182576d1/

Log:	merged upstream

diff --git a/rpython/jit/backend/arm/callbuilder.py b/rpython/jit/backend/arm/callbuilder.py
--- a/rpython/jit/backend/arm/callbuilder.py
+++ b/rpython/jit/backend/arm/callbuilder.py
@@ -92,7 +92,8 @@
             self.mc.LDR_ri(r.r7.value, r.r5.value)
 
         # change 'rpy_fastgil' to 0 (it should be non-zero right now)
-        self.mc.DMB()
+        if self.asm.cpu.cpuinfo.arch_version >= 7:
+            self.mc.DMB()
         self.mc.gen_load_int(r.r6.value, fastgil)
         self.mc.MOV_ri(r.ip.value, 0)
         self.mc.STR_ri(r.ip.value, r.r6.value)
@@ -112,7 +113,8 @@
         self.mc.STREX(r.r3.value, r.ip.value, r.r6.value, c=c.EQ)
                                                  # try to claim the lock
         self.mc.CMP_ri(r.r3.value, 0, cond=c.EQ) # did this succeed?
-        self.mc.DMB()
+        if self.asm.cpu.cpuinfo.arch_version >= 7:
+            self.mc.DMB()
         # the success of the lock acquisition is defined by
         # 'EQ is true', or equivalently by 'r3 == 0'.
         #
diff --git a/rpython/jit/backend/arm/codebuilder.py b/rpython/jit/backend/arm/codebuilder.py
--- a/rpython/jit/backend/arm/codebuilder.py
+++ b/rpython/jit/backend/arm/codebuilder.py
@@ -333,6 +333,8 @@
                     | (rn & 0xF) << 16)
 
     def DMB(self):
+        # ARMv7 only.  I guess ARMv6 CPUs cannot be used in symmetric
+        # multi-processing at all? That would make this instruction unneeded.
         # note: 'cond' is only permitted on Thumb here, but don't
         # write literally 0xf57ff05f, because it's larger than 31 bits
         c = cond.AL
diff --git a/rpython/jit/backend/arm/instructions.py b/rpython/jit/backend/arm/instructions.py
--- a/rpython/jit/backend/arm/instructions.py
+++ b/rpython/jit/backend/arm/instructions.py
@@ -142,6 +142,7 @@
     #'VCVT' : {'opc1':0xB, 'opc2':0xE, 'opc3':0x1, 'base': False},
 }
 
+# ARMv7 only
 simd_instructions_3regs = {
     'VADD_i64': {'A': 0x8, 'B': 0, 'U': 0},
     'VSUB_i64': {'A': 0x8, 'B': 0, 'U': 1},

From noreply at buildbot.pypy.org  Tue Sep 16 22:09:30 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue, 16 Sep 2014 22:09:30 +0200 (CEST)
Subject: [pypy-commit] pypy default: silence some compiler warnings
Message-ID: <20140916200930.04D171C023E@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: 
Changeset: r73560:1100b9ee9458
Date: 2014-09-16 16:08 -0400
http://bitbucket.org/pypy/pypy/changeset/1100b9ee9458/

Log:	silence some compiler warnings

diff --git a/rpython/rtyper/module/ll_os_stat.py b/rpython/rtyper/module/ll_os_stat.py
--- a/rpython/rtyper/module/ll_os_stat.py
+++ b/rpython/rtyper/module/ll_os_stat.py
@@ -186,7 +186,7 @@
     _name_struct_stat = '_stati64'
     INCLUDES = ['sys/types.h', 'sys/stat.h', 'sys/statvfs.h']
 else:
-    _name_struct_stat = 'stat'
+    _name_struct_stat = 'stat64'
     INCLUDES = ['sys/types.h', 'sys/stat.h', 'sys/statvfs.h', 'unistd.h']
 
 compilation_info = ExternalCompilationInfo(

From noreply at buildbot.pypy.org  Tue Sep 16 22:45:34 2014
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 16 Sep 2014 22:45:34 +0200 (CEST)
Subject: [pypy-commit] pypy default: speed up operator.itemgetter
Message-ID: <20140916204534.595631C03C7@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor 
Branch: 
Changeset: r73561:eeb1dde6e052
Date: 2014-09-16 13:44 -0700
http://bitbucket.org/pypy/pypy/changeset/eeb1dde6e052/

Log:	speed up operator.itemgetter

diff --git a/pypy/module/operator/app_operator.py b/pypy/module/operator/app_operator.py
--- a/pypy/module/operator/app_operator.py
+++ b/pypy/module/operator/app_operator.py
@@ -5,6 +5,7 @@
 equivalent to x+y.
 '''
 from __pypy__ import builtinify
+
 import types
 
 
@@ -27,7 +28,7 @@
     'getslice(a, b, c) -- Same as a[b:c].'
     if not isinstance(start, int) or not isinstance(end, int):
         raise TypeError("an integer is expected")
-    return a[start:end] 
+    return a[start:end]
 __getslice__ = getslice
 
 def indexOf(a, b):
@@ -37,7 +38,7 @@
         if x == b:
             return index
         index += 1
-    raise ValueError, 'sequence.index(x): x not in sequence'
+    raise ValueError('sequence.index(x): x not in sequence')
 
 def isMappingType(obj,):
     'isMappingType(a) -- Return True if a has a mapping type, False otherwise.'
@@ -58,9 +59,9 @@
 def repeat(obj, num):
     'repeat(a, b) -- Return a * b, where a is a sequence, and b is an integer.'
     if not isinstance(num, (int, long)):
-        raise TypeError, 'an integer is required'
+        raise TypeError('an integer is required')
     if not isSequenceType(obj):
-        raise TypeError, "non-sequence object can't be repeated"
+        raise TypeError("non-sequence object can't be repeated")
 
     return obj * num
 
@@ -68,7 +69,7 @@
 
 def setslice(a, b, c, d):
     'setslice(a, b, c, d) -- Same as a[b:c] = d.'
-    a[b:c] = d 
+    a[b:c] = d
 __setslice__ = setslice
 
 
@@ -109,15 +110,19 @@
     return make_getter(attr[last:], getter)
 
 
-def itemgetter(item, *items):
-    if items:
-        list_of_indices = [item] + list(items)
-        def getter(obj):
-            return tuple([obj[i] for i in list_of_indices])
-    else:
-        def getter(obj):
-            return obj[item]
-    return builtinify(getter)
+class itemgetter(object):
+    def __init__(self, item, *items):
+        self._single = not bool(items)
+        if self._single:
+            self._idx = item
+        else:
+            self._idx = [item] + list(items)
+
+    def __call__(self, obj):
+        if self._single:
+            return obj[self._idx]
+        else:
+            return tuple([obj[i] for i in self._idx])
 
 
 class methodcaller(object):

From noreply at buildbot.pypy.org  Tue Sep 16 22:45:35 2014
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 16 Sep 2014 22:45:35 +0200 (CEST)
Subject: [pypy-commit] pypy default: merged upstream
Message-ID: <20140916204535.770581C03C7@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor 
Branch: 
Changeset: r73562:d1d26f2b783b
Date: 2014-09-16 13:44 -0700
http://bitbucket.org/pypy/pypy/changeset/d1d26f2b783b/

Log:	merged upstream

diff --git a/rpython/rtyper/module/ll_os_stat.py b/rpython/rtyper/module/ll_os_stat.py
--- a/rpython/rtyper/module/ll_os_stat.py
+++ b/rpython/rtyper/module/ll_os_stat.py
@@ -186,7 +186,7 @@
     _name_struct_stat = '_stati64'
     INCLUDES = ['sys/types.h', 'sys/stat.h', 'sys/statvfs.h']
 else:
-    _name_struct_stat = 'stat'
+    _name_struct_stat = 'stat64'
     INCLUDES = ['sys/types.h', 'sys/stat.h', 'sys/statvfs.h', 'unistd.h']
 
 compilation_info = ExternalCompilationInfo(

From noreply at buildbot.pypy.org  Wed Sep 17 00:20:48 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 17 Sep 2014 00:20:48 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix translation on freebsd
Message-ID: <20140916222048.CA5CD1D2304@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: 
Changeset: r73563:a3bb59563fe7
Date: 2014-09-16 18:19 -0400
http://bitbucket.org/pypy/pypy/changeset/a3bb59563fe7/

Log:	fix translation on freebsd

diff --git a/rpython/rtyper/module/ll_os_stat.py b/rpython/rtyper/module/ll_os_stat.py
--- a/rpython/rtyper/module/ll_os_stat.py
+++ b/rpython/rtyper/module/ll_os_stat.py
@@ -186,7 +186,10 @@
     _name_struct_stat = '_stati64'
     INCLUDES = ['sys/types.h', 'sys/stat.h', 'sys/statvfs.h']
 else:
-    _name_struct_stat = 'stat64'
+    if 'bsd' in sys.platform:
+        _name_struct_stat = 'stat'
+    else:
+        _name_struct_stat = 'stat64'
     INCLUDES = ['sys/types.h', 'sys/stat.h', 'sys/statvfs.h', 'unistd.h']
 
 compilation_info = ExternalCompilationInfo(

From noreply at buildbot.pypy.org  Wed Sep 17 00:20:50 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 17 Sep 2014 00:20:50 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20140916222050.1D5101D2304@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: 
Changeset: r73564:0a17e2784faa
Date: 2014-09-16 18:20 -0400
http://bitbucket.org/pypy/pypy/changeset/0a17e2784faa/

Log:	merge heads

diff --git a/pypy/module/operator/app_operator.py b/pypy/module/operator/app_operator.py
--- a/pypy/module/operator/app_operator.py
+++ b/pypy/module/operator/app_operator.py
@@ -5,6 +5,7 @@
 equivalent to x+y.
 '''
 from __pypy__ import builtinify
+
 import types
 
 
@@ -27,7 +28,7 @@
     'getslice(a, b, c) -- Same as a[b:c].'
     if not isinstance(start, int) or not isinstance(end, int):
         raise TypeError("an integer is expected")
-    return a[start:end] 
+    return a[start:end]
 __getslice__ = getslice
 
 def indexOf(a, b):
@@ -37,7 +38,7 @@
         if x == b:
             return index
         index += 1
-    raise ValueError, 'sequence.index(x): x not in sequence'
+    raise ValueError('sequence.index(x): x not in sequence')
 
 def isMappingType(obj,):
     'isMappingType(a) -- Return True if a has a mapping type, False otherwise.'
@@ -58,9 +59,9 @@
 def repeat(obj, num):
     'repeat(a, b) -- Return a * b, where a is a sequence, and b is an integer.'
     if not isinstance(num, (int, long)):
-        raise TypeError, 'an integer is required'
+        raise TypeError('an integer is required')
     if not isSequenceType(obj):
-        raise TypeError, "non-sequence object can't be repeated"
+        raise TypeError("non-sequence object can't be repeated")
 
     return obj * num
 
@@ -68,7 +69,7 @@
 
 def setslice(a, b, c, d):
     'setslice(a, b, c, d) -- Same as a[b:c] = d.'
-    a[b:c] = d 
+    a[b:c] = d
 __setslice__ = setslice
 
 
@@ -109,15 +110,19 @@
     return make_getter(attr[last:], getter)
 
 
-def itemgetter(item, *items):
-    if items:
-        list_of_indices = [item] + list(items)
-        def getter(obj):
-            return tuple([obj[i] for i in list_of_indices])
-    else:
-        def getter(obj):
-            return obj[item]
-    return builtinify(getter)
+class itemgetter(object):
+    def __init__(self, item, *items):
+        self._single = not bool(items)
+        if self._single:
+            self._idx = item
+        else:
+            self._idx = [item] + list(items)
+
+    def __call__(self, obj):
+        if self._single:
+            return obj[self._idx]
+        else:
+            return tuple([obj[i] for i in self._idx])
 
 
 class methodcaller(object):

From noreply at buildbot.pypy.org  Wed Sep 17 01:30:41 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 17 Sep 2014 01:30:41 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: cleanup
Message-ID: <20140916233041.8AF311D23C3@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73565:8cb71f9cf52d
Date: 2014-09-16 15:55 -0400
http://bitbucket.org/pypy/pypy/changeset/8cb71f9cf52d/

Log:	cleanup

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -13,8 +13,6 @@
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 
 MS_WINDOWS = os.name == 'nt'
-if MS_WINDOWS:
-    from rpython.rlib.rwin32 import _get_osfhandle, SetEndOfFile, INVALID_HANDLE_VALUE
 
 includes = ['stdio.h', 'sys/types.h']
 if os.name == "posix":
@@ -121,7 +119,7 @@
         c_fseek = llexternal('fseeko', [FILEP, OFF_T, rffi.INT], rffi.INT)
         c_ftell = llexternal('ftello', [FILEP], OFF_T)
 else:
-    c_ftruncate = llexternal('_chsize', [rffi.INT, OFF_T], rffi.INT)
+    from rpython.rlib.rwin32 import _get_osfhandle, SetEndOfFile, INVALID_HANDLE_VALUE
     c_fseek = llexternal('_fseeki64', [FILEP, rffi.LONGLONG, rffi.INT], rffi.INT)
     c_ftell = llexternal('_ftelli64', [FILEP], rffi.LONGLONG)
 

From noreply at buildbot.pypy.org  Wed Sep 17 01:30:42 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 17 Sep 2014 01:30:42 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: update rfile test goal
Message-ID: <20140916233042.D1C271D23C3@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73566:9256384c7f1c
Date: 2014-09-16 16:01 -0400
http://bitbucket.org/pypy/pypy/changeset/9256384c7f1c/

Log:	update rfile test goal

diff --git a/rpython/translator/goal/targetrfile.py b/rpython/translator/goal/targetrfile.py
--- a/rpython/translator/goal/targetrfile.py
+++ b/rpython/translator/goal/targetrfile.py
@@ -1,8 +1,11 @@
 from rpython.rlib import rfile
 
 def entry_point(argv):
-    i, o, e = rfile.create_stdio()
-    o.write('test\n')
+    f = rfile.create_file(argv[1], argv[2])
+    while True:
+        line = f.readline()
+        if line == '':
+            break
     return 0
 
 # _____ Define and setup target ___

From noreply at buildbot.pypy.org  Wed Sep 17 01:30:44 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 17 Sep 2014 01:30:44 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: don't release gil for
	getc/ungetc
Message-ID: <20140916233044.0A3ED1D23C3@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73567:9e9dbf9b29af
Date: 2014-09-16 16:17 -0400
http://bitbucket.org/pypy/pypy/changeset/9e9dbf9b29af/

Log:	don't release gil for getc/ungetc

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -87,20 +87,19 @@
 _fclose2 = (c_fclose, c_fclose_nogil)
 _pclose2 = (c_pclose, c_pclose_nogil)
 
-c_flockfile = llexternal('flockfile', [FILEP], lltype.Void)
-c_funlockfile = llexternal('funlockfile', [FILEP], lltype.Void)
-
-c_getc = llexternal('getc', [FILEP], rffi.INT)
-c_getc_unlocked = llexternal('getc_unlocked', [FILEP], rffi.INT)
-c_ungetc = llexternal('ungetc', [rffi.INT, FILEP], rffi.INT)
+c_getc = llexternal('getc', [FILEP], rffi.INT, releasegil=False)
+c_ungetc = llexternal('ungetc', [rffi.INT, FILEP], rffi.INT, releasegil=False)
 
 if MS_WINDOWS:
+    USE_FGETS_IN_GETLINE = True
     c_flockfile = lambda ll_file: None
     c_funlockfile = lambda ll_file: None
-    c_getc_unlocked = c_getc
-    USE_FGETS_IN_GETLINE = True
+    c_getc_unlocked = llexternal('getc', [FILEP], rffi.INT)
 else:
     USE_FGETS_IN_GETLINE = False
+    c_flockfile = llexternal('flockfile', [FILEP], lltype.Void)
+    c_funlockfile = llexternal('funlockfile', [FILEP], lltype.Void)
+    c_getc_unlocked = llexternal('getc_unlocked', [FILEP], rffi.INT)
 
 c_fgets = llexternal('fgets', [rffi.CCHARP, rffi.INT, FILEP], rffi.CCHARP)
 c_fread = llexternal('fread', [rffi.CCHARP, rffi.SIZE_T, rffi.SIZE_T, FILEP],

From noreply at buildbot.pypy.org  Wed Sep 17 01:30:45 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 17 Sep 2014 01:30:45 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: use rawbuf for readline
	inner loop
Message-ID: <20140916233045.3F65F1D23C3@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73568:f96acbdfe57a
Date: 2014-09-16 18:09 -0400
http://bitbucket.org/pypy/pypy/changeset/f96acbdfe57a/

Log:	use rawbuf for readline inner loop

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -504,73 +504,93 @@
         ll_file = self._ll_file
         newlinetypes = self._newlinetypes
         skipnextlf = self._skipnextlf
-        c = 0
+
         s = StringBuilder()
-        while True:
-            # XXX release gil for all of this rather than each getc
-            c_flockfile(ll_file)
-            if self._univ_newline:
-                while size < 0 or s.getlength() < size:
-                    c = c_getc_unlocked(ll_file)
+        buffersize = size if size > 0 else 100
+        remainsize = buffersize
+        raw_buf, gc_buf = rffi.alloc_buffer(remainsize)
+        try:
+            i = 0
+            c = ord('x')
+            while True:
+                # XXX release gil for all of this rather than each getc
+                c_flockfile(ll_file)
+                if self._univ_newline:
+                    while i < remainsize:
+                        c = c_getc_unlocked(ll_file)
+                        if c == EOF:
+                            break
+                        if skipnextlf:
+                            skipnextlf = False
+                            if c == ord('\n'):
+                                newlinetypes |= NEWLINE_CRLF
+                                c = c_getc_unlocked(ll_file)
+                                if c == EOF:
+                                    break
+                            else:
+                                newlinetypes |= NEWLINE_CR
+                        if c == ord('\r'):
+                            skipnextlf = True
+                            c = ord('\n')
+                        elif c == ord('\n'):
+                            newlinetypes |= NEWLINE_LF
+                        raw_buf[i] = chr(c)
+                        i += 1
+                        if c == ord('\n'):
+                            break
                     if c == EOF:
-                        break
-                    if skipnextlf:
-                        skipnextlf = False
+                        if c_ferror(ll_file) and rposix.get_errno() == errno.EINTR:
+                            c_funlockfile(ll_file)
+                            self._newlinetypes = newlinetypes
+                            self._skipnextlf = skipnextlf
+                            if self._signal_checker is not None:
+                                self._signal_checker()
+                            c_clearerr(ll_file)
+                            continue
+                        if skipnextlf:
+                            newlinetypes |= NEWLINE_CR
+                else:
+                    while i < remainsize:
+                        c = c_getc_unlocked(ll_file)
+                        if c == EOF:
+                            break
+                        raw_buf[i] = chr(c)
+                        i += 1
                         if c == ord('\n'):
-                            newlinetypes |= NEWLINE_CRLF
-                            c = c_getc_unlocked(ll_file)
-                            if c == EOF:
-                                break
-                        else:
-                            newlinetypes |= NEWLINE_CR
-                    if c == ord('\r'):
-                        skipnextlf = True
-                        c = ord('\n')
-                    elif c == ord('\n'):
-                        newlinetypes |= NEWLINE_LF
-                    s.append(chr(c))
-                    if c == ord('\n'):
-                        break
-                if c == EOF:
-                    if c_ferror(ll_file) and rposix.get_errno() == errno.EINTR:
-                        c_funlockfile(ll_file)
-                        self._newlinetypes = newlinetypes
-                        self._skipnextlf = skipnextlf
-                        if self._signal_checker is not None:
-                            self._signal_checker()
+                            break
+                c_funlockfile(ll_file)
+                self._newlinetypes = newlinetypes
+                self._skipnextlf = skipnextlf
+                if i == size:
+                    # we read everything in one call, try to avoid copy
+                    # (remainsize == size if i == size)
+                    return rffi.str_from_buffer(raw_buf, gc_buf, remainsize, size)
+                s.append_charpsize(raw_buf, i)
+                if c == ord('\n'):
+                    break
+                elif c == EOF:
+                    if c_ferror(ll_file):
+                        if rposix.get_errno() == errno.EINTR:
+                            if self._signal_checker is not None:
+                                self._signal_checker()
+                            c_clearerr(ll_file)
+                            continue
                         c_clearerr(ll_file)
-                        continue
-                    if skipnextlf:
-                        newlinetypes |= NEWLINE_CR
-            else:
-                while size < 0 or s.getlength() < size:
-                    c = c_getc_unlocked(ll_file)
-                    if c == EOF:
-                        break
-                    s.append(chr(c))
-                    if c == ord('\n'):
-                        break
-            c_funlockfile(ll_file)
-            self._newlinetypes = newlinetypes
-            self._skipnextlf = skipnextlf
-            if c == ord('\n'):
-                break
-            elif c == EOF:
-                if c_ferror(ll_file):
-                    if rposix.get_errno() == errno.EINTR:
-                        if self._signal_checker is not None:
-                            self._signal_checker()
-                        c_clearerr(ll_file)
-                        continue
+                        raise _from_errno(IOError)
                     c_clearerr(ll_file)
-                    raise _from_errno(IOError)
-                c_clearerr(ll_file)
-                if self._signal_checker is not None:
-                    self._signal_checker()
-                break
-            else:
-                assert s.getlength() == size
-                break
+                    if self._signal_checker is not None:
+                        self._signal_checker()
+                    break
+                if size > 0:
+                    assert s.getlength() == size
+                    break
+                buffersize += buffersize >> 2
+                remainsize = buffersize - s.getlength()
+                rffi.keep_buffer_alive_until_here(raw_buf, gc_buf)
+                raw_buf, gc_buf = rffi.alloc_buffer(remainsize)
+                i = 0
+        finally:
+            rffi.keep_buffer_alive_until_here(raw_buf, gc_buf)
         return s.build()
 
     def readline(self, size=-1):

From noreply at buildbot.pypy.org  Wed Sep 17 01:30:46 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 17 Sep 2014 01:30:46 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: release gil around
	entire readline loop
Message-ID: <20140916233046.7CB591D23C3@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73569:b06d8ebb3fa4
Date: 2014-09-16 17:42 -0400
http://bitbucket.org/pypy/pypy/changeset/b06d8ebb3fa4/

Log:	release gil around entire readline loop

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -94,12 +94,12 @@
     USE_FGETS_IN_GETLINE = True
     c_flockfile = lambda ll_file: None
     c_funlockfile = lambda ll_file: None
-    c_getc_unlocked = llexternal('getc', [FILEP], rffi.INT)
+    c_getc_unlocked = c_getc
 else:
     USE_FGETS_IN_GETLINE = False
-    c_flockfile = llexternal('flockfile', [FILEP], lltype.Void)
-    c_funlockfile = llexternal('funlockfile', [FILEP], lltype.Void)
-    c_getc_unlocked = llexternal('getc_unlocked', [FILEP], rffi.INT)
+    c_flockfile = llexternal('flockfile', [FILEP], lltype.Void, releasegil=False)
+    c_funlockfile = llexternal('funlockfile', [FILEP], lltype.Void, releasegil=False)
+    c_getc_unlocked = llexternal('getc_unlocked', [FILEP], rffi.INT, releasegil=False)
 
 c_fgets = llexternal('fgets', [rffi.CCHARP, rffi.INT, FILEP], rffi.CCHARP)
 c_fread = llexternal('fread', [rffi.CCHARP, rffi.SIZE_T, rffi.SIZE_T, FILEP],
@@ -513,7 +513,8 @@
             i = 0
             c = ord('x')
             while True:
-                # XXX release gil for all of this rather than each getc
+                before = rffi.aroundstate.before
+                if before: before()
                 c_flockfile(ll_file)
                 if self._univ_newline:
                     while i < remainsize:
@@ -541,6 +542,8 @@
                     if c == EOF:
                         if c_ferror(ll_file) and rposix.get_errno() == errno.EINTR:
                             c_funlockfile(ll_file)
+                            after = rffi.aroundstate.after
+                            if after: after()
                             self._newlinetypes = newlinetypes
                             self._skipnextlf = skipnextlf
                             if self._signal_checker is not None:
@@ -559,6 +562,8 @@
                         if c == ord('\n'):
                             break
                 c_funlockfile(ll_file)
+                after = rffi.aroundstate.after
+                if after: after()
                 self._newlinetypes = newlinetypes
                 self._skipnextlf = skipnextlf
                 if i == size:

From noreply at buildbot.pypy.org  Wed Sep 17 01:30:47 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 17 Sep 2014 01:30:47 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: use explicit asserts here
Message-ID: <20140916233047.A5BA81D23C3@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73570:27bcb64887c4
Date: 2014-09-16 17:49 -0400
http://bitbucket.org/pypy/pypy/changeset/27bcb64887c4/

Log:	use explicit asserts here

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -429,7 +429,7 @@
                     raise _from_errno(IOError)
                 elif chunksize == size:
                     # we read everything in one call, try to avoid copy
-                    # (remainsize == size if chunksize == size)
+                    assert remainsize == size
                     return rffi.str_from_buffer(raw_buf, gc_buf, remainsize, size)
                 s.append_charpsize(raw_buf, chunksize)
                 if chunksize < remainsize and not interrupted:
@@ -568,7 +568,7 @@
                 self._skipnextlf = skipnextlf
                 if i == size:
                     # we read everything in one call, try to avoid copy
-                    # (remainsize == size if i == size)
+                    assert remainsize == size
                     return rffi.str_from_buffer(raw_buf, gc_buf, remainsize, size)
                 s.append_charpsize(raw_buf, i)
                 if c == ord('\n'):

From noreply at buildbot.pypy.org  Wed Sep 17 01:30:48 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 17 Sep 2014 01:30:48 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: avoid copies in more
	cases
Message-ID: <20140916233048.C25B51D23C3@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73571:16f3d4e99a89
Date: 2014-09-16 18:58 -0400
http://bitbucket.org/pypy/pypy/changeset/16f3d4e99a89/

Log:	avoid copies in more cases

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -427,20 +427,19 @@
                     if s.getlength() > 0 and rposix.get_errno() == errno.EAGAIN:
                         break
                     raise _from_errno(IOError)
-                elif chunksize == size:
-                    # we read everything in one call, try to avoid copy
-                    assert remainsize == size
-                    return rffi.str_from_buffer(raw_buf, gc_buf, remainsize, size)
-                s.append_charpsize(raw_buf, chunksize)
                 if chunksize < remainsize and not interrupted:
                     c_clearerr(ll_file)
                     break
                 if size >= 0:
                     break
+                s.append_charpsize(raw_buf, chunksize)
                 buffersize = self._new_buffersize(buffersize)
                 remainsize = buffersize - s.getlength()
                 rffi.keep_buffer_alive_until_here(raw_buf, gc_buf)
                 raw_buf, gc_buf = rffi.alloc_buffer(remainsize)
+            if s.getlength() == 0:
+                return rffi.str_from_buffer(raw_buf, gc_buf, remainsize, chunksize)
+            s.append_charpsize(raw_buf, chunksize)
         finally:
             rffi.keep_buffer_alive_until_here(raw_buf, gc_buf)
         return s.build()
@@ -566,11 +565,6 @@
                 if after: after()
                 self._newlinetypes = newlinetypes
                 self._skipnextlf = skipnextlf
-                if i == size:
-                    # we read everything in one call, try to avoid copy
-                    assert remainsize == size
-                    return rffi.str_from_buffer(raw_buf, gc_buf, remainsize, size)
-                s.append_charpsize(raw_buf, i)
                 if c == ord('\n'):
                     break
                 elif c == EOF:
@@ -587,13 +581,16 @@
                         self._signal_checker()
                     break
                 if size > 0:
-                    assert s.getlength() == size
                     break
+                s.append_charpsize(raw_buf, i)
                 buffersize += buffersize >> 2
                 remainsize = buffersize - s.getlength()
                 rffi.keep_buffer_alive_until_here(raw_buf, gc_buf)
                 raw_buf, gc_buf = rffi.alloc_buffer(remainsize)
                 i = 0
+            if s.getlength() == 0:
+                return rffi.str_from_buffer(raw_buf, gc_buf, remainsize, i)
+            s.append_charpsize(raw_buf, i)
         finally:
             rffi.keep_buffer_alive_until_here(raw_buf, gc_buf)
         return s.build()

From noreply at buildbot.pypy.org  Wed Sep 17 01:33:27 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 17 Sep 2014 01:33:27 +0200 (CEST)
Subject: [pypy-commit] pypy default: change this for linux only
Message-ID: <20140916233327.37A2B1D23C3@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: 
Changeset: r73572:2cf9f31fe14c
Date: 2014-09-16 16:33 -0700
http://bitbucket.org/pypy/pypy/changeset/2cf9f31fe14c/

Log:	change this for linux only

diff --git a/rpython/rtyper/module/ll_os_stat.py b/rpython/rtyper/module/ll_os_stat.py
--- a/rpython/rtyper/module/ll_os_stat.py
+++ b/rpython/rtyper/module/ll_os_stat.py
@@ -186,10 +186,10 @@
     _name_struct_stat = '_stati64'
     INCLUDES = ['sys/types.h', 'sys/stat.h', 'sys/statvfs.h']
 else:
-    if 'bsd' in sys.platform:
+    if sys.platform.startswith('linux'):
+        _name_struct_stat = 'stat64'
+    else:
         _name_struct_stat = 'stat'
-    else:
-        _name_struct_stat = 'stat64'
     INCLUDES = ['sys/types.h', 'sys/stat.h', 'sys/statvfs.h', 'unistd.h']
 
 compilation_info = ExternalCompilationInfo(

From noreply at buildbot.pypy.org  Wed Sep 17 06:58:17 2014
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Wed, 17 Sep 2014 06:58:17 +0200 (CEST)
Subject: [pypy-commit] pypy default: optimize operator.attrgetter
Message-ID: <20140917045817.EB3F01C337D@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor 
Branch: 
Changeset: r73573:5a423327c96a
Date: 2014-09-16 21:57 -0700
http://bitbucket.org/pypy/pypy/changeset/5a423327c96a/

Log:	optimize operator.attrgetter

diff --git a/pypy/module/operator/app_operator.py b/pypy/module/operator/app_operator.py
--- a/pypy/module/operator/app_operator.py
+++ b/pypy/module/operator/app_operator.py
@@ -73,41 +73,59 @@
 __setslice__ = setslice
 
 
+def _resolve_attr_chain(chain, obj, idx=0):
+    obj = getattr(obj, chain[idx])
+    if idx + 1 == len(chain):
+        return obj
+    else:
+        return _resolve_attr_chain(chain, obj, idx + 1)
+
+
+class _simple_attrgetter(object):
+    def __init__(self, attr):
+        self._attr = attr
+
+    def __call__(self, obj):
+        return getattr(obj, self._attr)
+
+
+class _single_attrgetter(object):
+    def __init__(self, attrs):
+        self._attrs = attrs
+
+    def __call__(self, obj):
+        return _resolve_attr_chain(self._attrs, obj)
+
+
+class _multi_attrgetter(object):
+    def __init__(self, attrs):
+        self._attrs = attrs
+
+    def __call__(self, obj):
+        return tuple([
+            _resolve_attr_chain(attrs, obj)
+            for attrs in self._attrs
+        ])
+
+
 def attrgetter(attr, *attrs):
+    if (
+        not isinstance(attr, basestring) or
+        not all(isinstance(a, basestring) for a in attrs)
+    ):
+        def _raise_typeerror(obj):
+            raise TypeError(
+                "argument must be a string, not %r" % type(attr).__name__
+            )
+        return _raise_typeerror
     if attrs:
-        getters = [single_attr_getter(a) for a in (attr,) + attrs]
-        def getter(obj):
-            return tuple([getter(obj) for getter in getters])
+        return _multi_attrgetter([
+            a.split(".") for a in [attr] + list(attrs)
+        ])
+    elif "." not in attr:
+        return _simple_attrgetter(attr)
     else:
-        getter = single_attr_getter(attr)
-    return builtinify(getter)
-
-def single_attr_getter(attr):
-    if not isinstance(attr, str):
-        if not isinstance(attr, unicode):
-            def _raise_typeerror(obj):
-                raise TypeError("argument must be a string, not %r" %
-                                (type(attr).__name__,))
-            return _raise_typeerror
-        attr = attr.encode('ascii')
-    #
-    def make_getter(name, prevfn=None):
-        if prevfn is None:
-            def getter(obj):
-                return getattr(obj, name)
-        else:
-            def getter(obj):
-                return getattr(prevfn(obj), name)
-        return getter
-    #
-    last = 0
-    getter = None
-    while True:
-        dot = attr.find(".", last)
-        if dot < 0: break
-        getter = make_getter(attr[last:dot], getter)
-        last = dot + 1
-    return make_getter(attr[last:], getter)
+        return _single_attrgetter(attr.split("."))
 
 
 class itemgetter(object):

From noreply at buildbot.pypy.org  Wed Sep 17 16:19:07 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 17 Sep 2014 16:19:07 +0200 (CEST)
Subject: [pypy-commit] stmgc default: Add a draft for how a simple page
	handling could work.
Message-ID: <20140917141907.E03291C0350@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r1389:4e6fcbe06dfd
Date: 2014-09-17 16:19 +0200
http://bitbucket.org/pypy/stmgc/changeset/4e6fcbe06dfd/

Log:	Add a draft for how a simple page handling could work.

diff --git a/c8/PAGES b/c8/PAGES
new file mode 100644
--- /dev/null
+++ b/c8/PAGES
@@ -0,0 +1,52 @@
+Handling of pages in stmgc
+--------------------------
+
+(Proposal)
+
+Each segment corresponds to one range of virtual addresses, of NB_PAGES.
+
+There is additionally one file descriptor corresponding to a
+memory-mapped file of length "15/16 * NB_PAGES".  In each segment, the
+final 15/16th of it is normally remapped with MAP_SHARED to correspond
+to this file.
+
+For each page of the file and each segment, we store two status bits
+that give one of three possible states: the NO_ACCESS state means that
+the corresponding page was mprotected to PROT_NONE; the SHARED state
+means the page is mapped to the file; the PRIVATE state means the page
+was remapped to an anonymous MAP_PRIVATE page.
+
+When a segment allocates new objects (out of its nursery), they go into
+pages that are initially set to SHARED in this segment, and NO_ACCESS in
+all other segments.
+
+When we try to read the same object from another segment, we get a
+segfault.  In the segment that receives the segfault, the page is
+NO_ACCESS.  At that point, two cases: either we can use the same data,
+or we can't.  We can if the data in the shared page is the unmodified
+data from the current revision (or from a later revision, in which case
+we just update to this later revision).  We can't if the shared data
+comes from a past revision or if it contains currently-modified data.
+
+If we can: we mprotect our segment's page back to SHARED.
+
+If we can't: we remap the page to PRIVATE.
+
+Finally, here is the write barrier logic.  When we're about to write to
+a page in our segment: first, we make sure it's not a NO_ACCESS page (by
+forcing a segfault to occur, I suppose).  Then, if it's a PRIVATE page,
+nothing to do; but if it's a SHARED page, we first check the other
+segments.  If none of them has also the page in the SHARED status (all
+have NO_ACCESS or PRIVATE), then we don't do anything either.  Only if
+there is a different segment that also has the page SHARED do we need
+more care.  There are two solutions in this case:
+
+1. We can change our page to PRIVATE.
+
+2. We can change the other segment's pages to NO_ACCESS.
+
+Which solution to choose in which case is unclear so far.
+
+The end result is that we end with as few PRIVATE pages as reasonably
+possible: it occurs only if one segment has currently changes in the
+page *and* a different segment is currently viewing the same data.

From noreply at buildbot.pypy.org  Wed Sep 17 17:56:12 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 17 Sep 2014 17:56:12 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: prevent close during
 concurrent readline operation
Message-ID: <20140917155612.059121C03C7@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73574:846b090781b8
Date: 2014-09-16 20:58 -0400
http://bitbucket.org/pypy/pypy/changeset/846b090781b8/

Log:	prevent close during concurrent readline operation

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -243,6 +243,7 @@
 
 
 class RFile(object):
+    _unlocked_count = 0
     _signal_checker = None
     _readable = True
     _writable = True
@@ -307,8 +308,11 @@
         ll_file = self._ll_file
         if ll_file:
             # double close is allowed
+            do_close = self._close2[0]
+            if do_close and self._unlocked_count > 0:
+                raise IOError("close() called during concurrent "
+                              "operation on the same file object.")
             self._ll_file = lltype.nullptr(FILEP.TO)
-            do_close = self._close2[0]
             try:
                 if do_close:
                     res = do_close(ll_file)
@@ -513,6 +517,7 @@
             c = ord('x')
             while True:
                 before = rffi.aroundstate.before
+                self._unlocked_count += 1
                 if before: before()
                 c_flockfile(ll_file)
                 if self._univ_newline:
@@ -543,6 +548,7 @@
                             c_funlockfile(ll_file)
                             after = rffi.aroundstate.after
                             if after: after()
+                            self._unlocked_count -= 1
                             self._newlinetypes = newlinetypes
                             self._skipnextlf = skipnextlf
                             if self._signal_checker is not None:
@@ -563,6 +569,7 @@
                 c_funlockfile(ll_file)
                 after = rffi.aroundstate.after
                 if after: after()
+                self._unlocked_count -= 1
                 self._newlinetypes = newlinetypes
                 self._skipnextlf = skipnextlf
                 if c == ord('\n'):

From noreply at buildbot.pypy.org  Wed Sep 17 17:56:13 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 17 Sep 2014 17:56:13 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: only construct
	stringbuilder if necessary
Message-ID: <20140917155613.299341C03C7@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73575:855e42d77388
Date: 2014-09-16 21:27 -0400
http://bitbucket.org/pypy/pypy/changeset/855e42d77388/

Log:	only construct stringbuilder if necessary

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -409,7 +409,7 @@
         self._check_readable()
         ll_file = self._ll_file
 
-        s = StringBuilder()
+        s = None
         buffersize = size if size >= 0 else self._new_buffersize(0)
         remainsize = buffersize
         raw_buf, gc_buf = rffi.alloc_buffer(remainsize)
@@ -428,7 +428,7 @@
                     if not c_ferror(ll_file):
                         break
                     c_clearerr(ll_file)
-                    if s.getlength() > 0 and rposix.get_errno() == errno.EAGAIN:
+                    if s is not None and rposix.get_errno() == errno.EAGAIN:
                         break
                     raise _from_errno(IOError)
                 if chunksize < remainsize and not interrupted:
@@ -436,12 +436,14 @@
                     break
                 if size >= 0:
                     break
+                if s is None:
+                    s = StringBuilder()
                 s.append_charpsize(raw_buf, chunksize)
                 buffersize = self._new_buffersize(buffersize)
                 remainsize = buffersize - s.getlength()
                 rffi.keep_buffer_alive_until_here(raw_buf, gc_buf)
                 raw_buf, gc_buf = rffi.alloc_buffer(remainsize)
-            if s.getlength() == 0:
+            if s is None:
                 return rffi.str_from_buffer(raw_buf, gc_buf, remainsize, chunksize)
             s.append_charpsize(raw_buf, chunksize)
         finally:
@@ -508,7 +510,7 @@
         newlinetypes = self._newlinetypes
         skipnextlf = self._skipnextlf
 
-        s = StringBuilder()
+        s = None
         buffersize = size if size > 0 else 100
         remainsize = buffersize
         raw_buf, gc_buf = rffi.alloc_buffer(remainsize)
@@ -589,13 +591,15 @@
                     break
                 if size > 0:
                     break
+                if s is None:
+                    s = StringBuilder()
                 s.append_charpsize(raw_buf, i)
                 buffersize += buffersize >> 2
                 remainsize = buffersize - s.getlength()
                 rffi.keep_buffer_alive_until_here(raw_buf, gc_buf)
                 raw_buf, gc_buf = rffi.alloc_buffer(remainsize)
                 i = 0
-            if s.getlength() == 0:
+            if s is None:
                 return rffi.str_from_buffer(raw_buf, gc_buf, remainsize, i)
             s.append_charpsize(raw_buf, i)
         finally:

From noreply at buildbot.pypy.org  Wed Sep 17 17:56:14 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 17 Sep 2014 17:56:14 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: move inner getc loop to
	its own function
Message-ID: <20140917155614.493651C03C7@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73576:c5298ca1bb92
Date: 2014-09-16 22:50 -0400
http://bitbucket.org/pypy/pypy/changeset/c5298ca1bb92/

Log:	move inner getc loop to its own function

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -502,13 +502,68 @@
             s.append_charpsize(buf.raw, c)
         return s.build()
 
+    def _get_line_getc(self, raw_buf, remainsize, i):
+        ll_file = self._ll_file
+        c = ord('x')
+
+        self._unlocked_count += 1
+        before = rffi.aroundstate.before
+        if before: before()
+        c_flockfile(ll_file)
+
+        if self._univ_newline:
+            newlinetypes = self._newlinetypes
+            skipnextlf = self._skipnextlf
+            while i < remainsize:
+                c = c_getc_unlocked(ll_file)
+                if c == EOF:
+                    break
+                if skipnextlf:
+                    skipnextlf = False
+                    if c == ord('\n'):
+                        newlinetypes |= NEWLINE_CRLF
+                        c = c_getc_unlocked(ll_file)
+                        if c == EOF:
+                            break
+                    else:
+                        newlinetypes |= NEWLINE_CR
+                if c == ord('\r'):
+                    skipnextlf = True
+                    c = ord('\n')
+                elif c == ord('\n'):
+                    newlinetypes |= NEWLINE_LF
+                raw_buf[i] = chr(c)
+                i += 1
+                if c == ord('\n'):
+                    break
+            if c == EOF and skipnextlf:
+                newlinetypes |= NEWLINE_CR
+            self._newlinetypes = newlinetypes
+            self._skipnextlf = skipnextlf
+        else:
+            while i < remainsize:
+                c = c_getc_unlocked(ll_file)
+                if c == EOF:
+                    break
+                raw_buf[i] = chr(c)
+                i += 1
+                if c == ord('\n'):
+                    break
+
+        c_funlockfile(ll_file)
+        after = rffi.aroundstate.after
+        if after: after()
+        self._unlocked_count -= 1
+
+        return i, c
+    _get_line_getc._gctransformer_hint_close_stack_ = True
+    _get_line_getc._dont_inline_ = True
+
     def _get_line(self, size):
         if USE_FGETS_IN_GETLINE and size < 0 and not self._univ_newline:
             return self._get_line_fgets()
 
         ll_file = self._ll_file
-        newlinetypes = self._newlinetypes
-        skipnextlf = self._skipnextlf
 
         s = None
         buffersize = size if size > 0 else 100
@@ -516,64 +571,8 @@
         raw_buf, gc_buf = rffi.alloc_buffer(remainsize)
         try:
             i = 0
-            c = ord('x')
             while True:
-                before = rffi.aroundstate.before
-                self._unlocked_count += 1
-                if before: before()
-                c_flockfile(ll_file)
-                if self._univ_newline:
-                    while i < remainsize:
-                        c = c_getc_unlocked(ll_file)
-                        if c == EOF:
-                            break
-                        if skipnextlf:
-                            skipnextlf = False
-                            if c == ord('\n'):
-                                newlinetypes |= NEWLINE_CRLF
-                                c = c_getc_unlocked(ll_file)
-                                if c == EOF:
-                                    break
-                            else:
-                                newlinetypes |= NEWLINE_CR
-                        if c == ord('\r'):
-                            skipnextlf = True
-                            c = ord('\n')
-                        elif c == ord('\n'):
-                            newlinetypes |= NEWLINE_LF
-                        raw_buf[i] = chr(c)
-                        i += 1
-                        if c == ord('\n'):
-                            break
-                    if c == EOF:
-                        if c_ferror(ll_file) and rposix.get_errno() == errno.EINTR:
-                            c_funlockfile(ll_file)
-                            after = rffi.aroundstate.after
-                            if after: after()
-                            self._unlocked_count -= 1
-                            self._newlinetypes = newlinetypes
-                            self._skipnextlf = skipnextlf
-                            if self._signal_checker is not None:
-                                self._signal_checker()
-                            c_clearerr(ll_file)
-                            continue
-                        if skipnextlf:
-                            newlinetypes |= NEWLINE_CR
-                else:
-                    while i < remainsize:
-                        c = c_getc_unlocked(ll_file)
-                        if c == EOF:
-                            break
-                        raw_buf[i] = chr(c)
-                        i += 1
-                        if c == ord('\n'):
-                            break
-                c_funlockfile(ll_file)
-                after = rffi.aroundstate.after
-                if after: after()
-                self._unlocked_count -= 1
-                self._newlinetypes = newlinetypes
-                self._skipnextlf = skipnextlf
+                i, c = self._get_line_getc(raw_buf, remainsize, i)
                 if c == ord('\n'):
                     break
                 elif c == EOF:

From noreply at buildbot.pypy.org  Wed Sep 17 17:56:15 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 17 Sep 2014 17:56:15 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: fix get_line_getc with
	jit
Message-ID: <20140917155615.749D01C03C7@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73577:82e7aa02b4a7
Date: 2014-09-17 00:25 -0400
http://bitbucket.org/pypy/pypy/changeset/82e7aa02b4a7/

Log:	fix get_line_getc with jit

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -502,9 +502,10 @@
             s.append_charpsize(buf.raw, c)
         return s.build()
 
-    def _get_line_getc(self, raw_buf, remainsize, i):
+    def _get_line_getc(self, raw_buf, remainsize, state):
         ll_file = self._ll_file
-        c = ord('x')
+        i = state[0]
+        c = state[1]
 
         self._unlocked_count += 1
         before = rffi.aroundstate.before
@@ -555,7 +556,8 @@
         if after: after()
         self._unlocked_count -= 1
 
-        return i, c
+        state[0] = i
+        state[1] = c
     _get_line_getc._gctransformer_hint_close_stack_ = True
     _get_line_getc._dont_inline_ = True
 
@@ -572,7 +574,9 @@
         try:
             i = 0
             while True:
-                i, c = self._get_line_getc(raw_buf, remainsize, i)
+                state = [i, ord('x')]
+                self._get_line_getc(raw_buf, remainsize, state)
+                i, c = state
                 if c == ord('\n'):
                     break
                 elif c == EOF:
diff --git a/rpython/translator/goal/targetrfile.py b/rpython/translator/goal/targetrfile.py
--- a/rpython/translator/goal/targetrfile.py
+++ b/rpython/translator/goal/targetrfile.py
@@ -1,8 +1,11 @@
-from rpython.rlib import rfile
+from rpython.rlib import rfile, jit
+
+driver = jit.JitDriver(greens=[], reds='auto')
 
 def entry_point(argv):
     f = rfile.create_file(argv[1], argv[2])
     while True:
+        driver.jit_merge_point()
         line = f.readline()
         if line == '':
             break

From noreply at buildbot.pypy.org  Wed Sep 17 17:56:16 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 17 Sep 2014 17:56:16 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: rfile unlocked_count not
	needed
Message-ID: <20140917155616.B06A21C03C7@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73578:1acef6fb809c
Date: 2014-09-17 01:54 -0400
http://bitbucket.org/pypy/pypy/changeset/1acef6fb809c/

Log:	rfile unlocked_count not needed

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -243,7 +243,6 @@
 
 
 class RFile(object):
-    _unlocked_count = 0
     _signal_checker = None
     _readable = True
     _writable = True
@@ -309,9 +308,6 @@
         if ll_file:
             # double close is allowed
             do_close = self._close2[0]
-            if do_close and self._unlocked_count > 0:
-                raise IOError("close() called during concurrent "
-                              "operation on the same file object.")
             self._ll_file = lltype.nullptr(FILEP.TO)
             try:
                 if do_close:
@@ -507,7 +503,6 @@
         i = state[0]
         c = state[1]
 
-        self._unlocked_count += 1
         before = rffi.aroundstate.before
         if before: before()
         c_flockfile(ll_file)
@@ -554,7 +549,6 @@
         c_funlockfile(ll_file)
         after = rffi.aroundstate.after
         if after: after()
-        self._unlocked_count -= 1
 
         state[0] = i
         state[1] = c

From noreply at buildbot.pypy.org  Wed Sep 17 17:56:17 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 17 Sep 2014 17:56:17 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: rewrite fgets loop more
	like cpython
Message-ID: <20140917155617.DDF341C03C7@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73579:01b77e9f28ef
Date: 2014-09-17 11:55 -0400
http://bitbucket.org/pypy/pypy/changeset/01b77e9f28ef/

Log:	rewrite fgets loop more like cpython

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -52,8 +52,6 @@
 BUFSIZ = config['BUFSIZ']
 EOF = config['EOF']
 
-BASE_LINE_SIZE = 100
-
 NEWLINE_UNKNOWN = 0
 NEWLINE_CR = 1
 NEWLINE_LF = 2
@@ -446,56 +444,66 @@
             rffi.keep_buffer_alive_until_here(raw_buf, gc_buf)
         return s.build()
 
-    def _get_line_fgets_single(self, raw_buf):
+    def _get_line_fgets(self):
         ll_file = self._ll_file
-        for i in range(BASE_LINE_SIZE):
-            raw_buf[i] = '\n'
 
-        result = c_fgets(raw_buf, BASE_LINE_SIZE, ll_file)
-        if not result:
-            c_clearerr(ll_file)
-            if self._signal_checker is not None:
-                self._signal_checker()
-            return 0
+        s = None
+        buffersize = 100
+        remainsize = buffersize
+        raw_buf, gc_buf = rffi.alloc_buffer(remainsize)
+        try:
+            while True:
+                for i in range(remainsize):
+                    raw_buf[i] = '\n'
 
-        # Assume that fgets() works as documented, and additionally
-        # never writes beyond the final \0, which the CPython
-        # fileobject.c says appears to be the case everywhere.
-        # The only case where the buffer was not big enough is the
-        # case where the buffer is full, ends with \0, and doesn't
-        # end with \n\0.
+                p = c_fgets(raw_buf, remainsize, ll_file)
+                if not p:
+                    c_clearerr(ll_file)
+                    if self._signal_checker is not None:
+                        self._signal_checker()
+                    p = 0
+                    break
 
-        p = 0
-        while raw_buf[p] != '\n':
-            p += 1
-            if p == BASE_LINE_SIZE:
-                # fgets read whole buffer without finding newline
-                return -1
-        # p points to first \n
+                # Assume that fgets() works as documented, and additionally
+                # never writes beyond the final \0, which the CPython
+                # fileobject.c says appears to be the case everywhere.
+                # The only case where the buffer was not big enough is the
+                # case where the buffer is full, ends with \0, and doesn't
+                # end with \n\0.
 
-        if p + 1 < BASE_LINE_SIZE and raw_buf[p + 1] == '\0':
-            # \n followed by \0, fgets read and found newline
-            return p + 1
-        else:
-            # \n not followed by \0, fgets read but didnt find newline
-            assert p > 0 and raw_buf[p - 1] == '\0'
-            return p - 1
+                p = 0
+                while raw_buf[p] != '\n':
+                    p += 1
+                    if p == remainsize:
+                        # fgets read whole buffer without finding newline
+                        p = -1
+                        break
 
-    def _get_line_fgets(self):
-        # XXX use buffer size logic from cpython
-        with rffi.scoped_alloc_buffer(BASE_LINE_SIZE) as buf:
-            c = self._get_line_fgets_single(buf.raw)
-            if c >= 0:
-                return buf.str(c)
+                if p != -1:
+                    # p points to first \n
+                    if p + 1 < remainsize and raw_buf[p + 1] == '\0':
+                        # \n followed by \0, fgets read and found newline
+                        p += 1
+                    else:
+                        # \n not followed by \0, fgets read but didnt find newline
+                        assert p > 0 and raw_buf[p - 1] == '\0'
+                        p -= 1
+                    break
 
-            # this is the rare case: the line is longer than BASE_LINE_SIZE
-            s = StringBuilder()
-            while True:
-                s.append_charpsize(buf.raw, BASE_LINE_SIZE - 1)
-                c = self._get_line_fgets_single(buf.raw)
-                if c >= 0:
-                    break
-            s.append_charpsize(buf.raw, c)
+                # fgets overwrote all the newlines (entire buffer)
+                assert raw_buf[remainsize - 1] == '\0'
+                if s is None:
+                    s = StringBuilder()
+                s.append_charpsize(raw_buf, remainsize - 1)
+                buffersize += buffersize >> 2
+                remainsize = buffersize - s.getlength()
+                rffi.keep_buffer_alive_until_here(raw_buf, gc_buf)
+                raw_buf, gc_buf = rffi.alloc_buffer(remainsize)
+            if s is None:
+                return rffi.str_from_buffer(raw_buf, gc_buf, remainsize, p)
+            s.append_charpsize(raw_buf, p)
+        finally:
+            rffi.keep_buffer_alive_until_here(raw_buf, gc_buf)
         return s.build()
 
     def _get_line_getc(self, raw_buf, remainsize, state):

From noreply at buildbot.pypy.org  Wed Sep 17 17:56:19 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 17 Sep 2014 17:56:19 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: merge default
Message-ID: <20140917155619.A0DB41C03C7@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73580:2ea2f1aeea40
Date: 2014-09-17 11:55 -0400
http://bitbucket.org/pypy/pypy/changeset/2ea2f1aeea40/

Log:	merge default

diff --git a/pypy/doc/getting-started-python.rst b/pypy/doc/getting-started-python.rst
--- a/pypy/doc/getting-started-python.rst
+++ b/pypy/doc/getting-started-python.rst
@@ -111,6 +111,10 @@
    of your choice.  Typical example: ``--opt=2`` gives a good (but of
    course slower) Python interpreter without the JIT.
 
+   Consider using PyPy instead of CPython in the above command line,
+   as it is much faster.  (Note that ``rpython`` is a Python 2 program,
+   not Python 3; you need to run either PyPy 2 or CPython 2.)
+
 .. _`optimization level`: config/opt.html
 
 If everything works correctly this will create an executable
diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -85,10 +85,13 @@
 
 Abridged method (for -Ojit builds using Visual Studio 2008)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Download the versions of all the external packages
-from 
+Download the versions of all the external packages from 
+https://bitbucket.org/pypy/pypy/downloads/local_2.4.zip
+(for 2.4 release and later) or
 https://bitbucket.org/pypy/pypy/downloads/local.zip
-Then expand it into the base directory (base_dir) and modify your environment to reflect this::
+(for pre-2.4 versions)
+Then expand it into the base directory (base_dir) and modify your environment
+to reflect this::
 
     set PATH=\bin;\tcltk\bin;%PATH%
     set INCLUDE=\include;\tcltk\include;%INCLUDE%
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -38,18 +38,15 @@
 def cpython_code_signature(code):
     "([list-of-arg-names], vararg-name-or-None, kwarg-name-or-None)."
     argcount = code.co_argcount
+    varnames = code.co_varnames
     assert argcount >= 0     # annotator hint
-    argnames = list(code.co_varnames[:argcount])
+    argnames = list(varnames[:argcount])
     if code.co_flags & CO_VARARGS:
-        varargname = code.co_varnames[argcount]
+        varargname = varnames[argcount]
         argcount += 1
     else:
         varargname = None
-    if code.co_flags & CO_VARKEYWORDS:
-        kwargname = code.co_varnames[argcount]
-        argcount += 1
-    else:
-        kwargname = None
+    kwargname = varnames[argcount] if code.co_flags & CO_VARKEYWORDS else None
     return Signature(argnames, varargname, kwargname)
 
 
diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -971,7 +971,7 @@
         prefix = udir.join('pathtest').ensure(dir=1)
         fake_exe = 'bin/pypy-c'
         if sys.platform == 'win32':
-            fake_exe += '.exe'
+            fake_exe = 'pypy-c.exe'
         fake_exe = prefix.join(fake_exe).ensure(file=1)
         expected_path = [str(prefix.join(subdir).ensure(dir=1))
                          for subdir in ('lib_pypy',
@@ -1011,8 +1011,10 @@
             assert sys.path == old_sys_path + [self.goal_dir]
 
             app_main.setup_bootstrap_path(self.fake_exe)
-            assert sys.executable == ''      # not executable!
-            assert sys.path == old_sys_path + [self.goal_dir]
+            if not sys.platform == 'win32':
+                # an existing file is always 'executable' on windows
+                assert sys.executable == ''      # not executable!
+                assert sys.path == old_sys_path + [self.goal_dir]
 
             os.chmod(self.fake_exe, 0755)
             app_main.setup_bootstrap_path(self.fake_exe)
diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py
--- a/pypy/module/_cffi_backend/ctypefunc.py
+++ b/pypy/module/_cffi_backend/ctypefunc.py
@@ -8,6 +8,7 @@
 from rpython.rlib.jit_libffi import (CIF_DESCRIPTION, CIF_DESCRIPTION_P,
     FFI_TYPE, FFI_TYPE_P, FFI_TYPE_PP, SIZE_OF_FFI_ARG)
 from rpython.rlib.objectmodel import we_are_translated, instantiate
+from rpython.rlib.objectmodel import keepalive_until_here
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
 
 from pypy.interpreter.error import OperationError, oefmt
@@ -160,6 +161,7 @@
                         raw_cdata = rffi.cast(rffi.CCHARPP, data)[0]
                         lltype.free(raw_cdata, flavor='raw')
             lltype.free(buffer, flavor='raw')
+            keepalive_until_here(args_w)
         return w_res
 
 def get_mustfree_flag(data):
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -759,17 +759,25 @@
 
         # socket's timeout is in seconds, poll's timeout in ms
         timeout = int(sock_timeout * 1000 + 0.5)
-        ready = rpoll.poll(fddict, timeout)
+        try:
+            ready = rpoll.poll(fddict, timeout)
+        except rpoll.PollError, e:
+            message = e.get_msg()
+            raise ssl_error(space, message, e.errno)
     else:
         if MAX_FD_SIZE is not None and sock_fd >= MAX_FD_SIZE:
             return SOCKET_TOO_LARGE_FOR_SELECT
 
-        if writing:
-            r, w, e = rpoll.select([], [sock_fd], [], sock_timeout)
-            ready = w
-        else:
-            r, w, e = rpoll.select([sock_fd], [], [], sock_timeout)
-            ready = r
+        try:
+            if writing:
+                r, w, e = rpoll.select([], [sock_fd], [], sock_timeout)
+                ready = w
+            else:
+                r, w, e = rpoll.select([sock_fd], [], [], sock_timeout)
+                ready = r
+        except rpoll.SelectError as e:
+            message = e.get_msg()
+            raise ssl_error(space, message, e.errno)
     if ready:
         return SOCKET_OPERATION_OK
     else:
diff --git a/pypy/module/operator/__init__.py b/pypy/module/operator/__init__.py
--- a/pypy/module/operator/__init__.py
+++ b/pypy/module/operator/__init__.py
@@ -39,7 +39,7 @@
                     'irshift', 'isub', 'itruediv', 'ixor', '_length_hint']
 
     interpleveldefs = {
-        '_compare_digest': 'interp_operator.compare_digest',
+        '_compare_digest': 'tscmp.compare_digest',
     }
 
     for name in interp_names:
diff --git a/pypy/module/operator/app_operator.py b/pypy/module/operator/app_operator.py
--- a/pypy/module/operator/app_operator.py
+++ b/pypy/module/operator/app_operator.py
@@ -5,6 +5,7 @@
 equivalent to x+y.
 '''
 from __pypy__ import builtinify
+
 import types
 
 
@@ -27,7 +28,7 @@
     'getslice(a, b, c) -- Same as a[b:c].'
     if not isinstance(start, int) or not isinstance(end, int):
         raise TypeError("an integer is expected")
-    return a[start:end] 
+    return a[start:end]
 __getslice__ = getslice
 
 def indexOf(a, b):
@@ -37,7 +38,7 @@
         if x == b:
             return index
         index += 1
-    raise ValueError, 'sequence.index(x): x not in sequence'
+    raise ValueError('sequence.index(x): x not in sequence')
 
 def isMappingType(obj,):
     'isMappingType(a) -- Return True if a has a mapping type, False otherwise.'
@@ -58,9 +59,9 @@
 def repeat(obj, num):
     'repeat(a, b) -- Return a * b, where a is a sequence, and b is an integer.'
     if not isinstance(num, (int, long)):
-        raise TypeError, 'an integer is required'
+        raise TypeError('an integer is required')
     if not isSequenceType(obj):
-        raise TypeError, "non-sequence object can't be repeated"
+        raise TypeError("non-sequence object can't be repeated")
 
     return obj * num
 
@@ -68,59 +69,85 @@
 
 def setslice(a, b, c, d):
     'setslice(a, b, c, d) -- Same as a[b:c] = d.'
-    a[b:c] = d 
+    a[b:c] = d
 __setslice__ = setslice
 
 
+def _resolve_attr_chain(chain, obj, idx=0):
+    obj = getattr(obj, chain[idx])
+    if idx + 1 == len(chain):
+        return obj
+    else:
+        return _resolve_attr_chain(chain, obj, idx + 1)
+
+
+class _simple_attrgetter(object):
+    def __init__(self, attr):
+        self._attr = attr
+
+    def __call__(self, obj):
+        return getattr(obj, self._attr)
+
+
+class _single_attrgetter(object):
+    def __init__(self, attrs):
+        self._attrs = attrs
+
+    def __call__(self, obj):
+        return _resolve_attr_chain(self._attrs, obj)
+
+
+class _multi_attrgetter(object):
+    def __init__(self, attrs):
+        self._attrs = attrs
+
+    def __call__(self, obj):
+        return tuple([
+            _resolve_attr_chain(attrs, obj)
+            for attrs in self._attrs
+        ])
+
+
 def attrgetter(attr, *attrs):
+    if (
+        not isinstance(attr, basestring) or
+        not all(isinstance(a, basestring) for a in attrs)
+    ):
+        def _raise_typeerror(obj):
+            raise TypeError(
+                "argument must be a string, not %r" % type(attr).__name__
+            )
+        return _raise_typeerror
     if attrs:
-        getters = [single_attr_getter(a) for a in (attr,) + attrs]
-        def getter(obj):
-            return tuple([getter(obj) for getter in getters])
+        return _multi_attrgetter([
+            a.split(".") for a in [attr] + list(attrs)
+        ])
+    elif "." not in attr:
+        return _simple_attrgetter(attr)
     else:
-        getter = single_attr_getter(attr)
-    return builtinify(getter)
+        return _single_attrgetter(attr.split("."))
 
-def single_attr_getter(attr):
-    if not isinstance(attr, str):
-        if not isinstance(attr, unicode):
-            def _raise_typeerror(obj):
-                raise TypeError("argument must be a string, not %r" %
-                                (type(attr).__name__,))
-            return _raise_typeerror
-        attr = attr.encode('ascii')
-    #
-    def make_getter(name, prevfn=None):
-        if prevfn is None:
-            def getter(obj):
-                return getattr(obj, name)
+
+class itemgetter(object):
+    def __init__(self, item, *items):
+        self._single = not bool(items)
+        if self._single:
+            self._idx = item
         else:
-            def getter(obj):
-                return getattr(prevfn(obj), name)
-        return getter
-    #
-    last = 0
-    getter = None
-    while True:
-        dot = attr.find(".", last)
-        if dot < 0: break
-        getter = make_getter(attr[last:dot], getter)
-        last = dot + 1
-    return make_getter(attr[last:], getter)
+            self._idx = [item] + list(items)
 
+    def __call__(self, obj):
+        if self._single:
+            return obj[self._idx]
+        else:
+            return tuple([obj[i] for i in self._idx])
 
-def itemgetter(item, *items):
-    if items:
-        list_of_indices = [item] + list(items)
-        def getter(obj):
-            return tuple([obj[i] for i in list_of_indices])
-    else:
-        def getter(obj):
-            return obj[item]
-    return builtinify(getter)
 
+class methodcaller(object):
+    def __init__(self, method_name, *args, **kwargs):
+        self._method_name = method_name
+        self._args = args
+        self._kwargs = kwargs
 
-def methodcaller(method_name, *args, **kwargs):
-    def call(obj):
-        return getattr(obj, method_name)(*args, **kwargs)
-    return builtinify(call)
+    def __call__(self, obj):
+        return getattr(obj, self._method_name)(*self._args, **self._kwargs)
diff --git a/pypy/module/operator/interp_operator.py b/pypy/module/operator/interp_operator.py
--- a/pypy/module/operator/interp_operator.py
+++ b/pypy/module/operator/interp_operator.py
@@ -1,6 +1,4 @@
-from rpython.rlib.objectmodel import specialize
-
-from pypy.interpreter.error import OperationError, oefmt
+from pypy.interpreter.error import OperationError
 from pypy.interpreter.gateway import unwrap_spec
 
 
@@ -249,33 +247,3 @@
 @unwrap_spec(default=int)
 def _length_hint(space, w_iterable, default):
     return space.wrap(space.length_hint(w_iterable, default))
-
-def compare_digest(space, w_a, w_b):
-    if (
-        space.isinstance_w(w_a, space.w_unicode) and
-        space.isinstance_w(w_b, space.w_unicode)
-    ):
-        return space.wrap(tscmp(space.unicode_w(w_a), space.unicode_w(w_b)))
-    if (
-        space.isinstance_w(w_a, space.w_unicode) or
-        space.isinstance_w(w_b, space.w_unicode)
-    ):
-        raise oefmt(
-            space.w_TypeError,
-            "unsupported operand types(s) or combination of types: '%N' and '%N'",
-            w_a,
-            w_b,
-        )
-    else:
-        return space.wrap(tscmp(space.bufferstr_w(w_a), space.bufferstr_w(w_b)))
-
-
- at specialize.argtype(0, 1)
-def tscmp(a, b):
-    len_a = len(a)
-    len_b = len(b)
-    length = min(len(a), len(b))
-    res = len_a ^ len_b
-    for i in xrange(length):
-        res |= ord(a[i]) ^ ord(b[i])
-    return res == 0
diff --git a/pypy/module/operator/test/test_operator.py b/pypy/module/operator/test/test_operator.py
--- a/pypy/module/operator/test/test_operator.py
+++ b/pypy/module/operator/test/test_operator.py
@@ -334,3 +334,9 @@
         assert operator._compare_digest(a, b)
         a, b = mybytes(b"foobar"), mybytes(b"foobaz")
         assert not operator._compare_digest(a, b)
+
+    def test_compare_digest_unicode(self):
+        import operator
+        assert operator._compare_digest(u'asd', u'asd')
+        assert not operator._compare_digest(u'asd', u'qwe')
+        raises(TypeError, operator._compare_digest, u'asd', b'qwe')
diff --git a/pypy/module/operator/test/test_tscmp.py b/pypy/module/operator/test/test_tscmp.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/operator/test/test_tscmp.py
@@ -0,0 +1,28 @@
+from pypy.module.operator.tscmp import pypy_tscmp, pypy_tscmp_wide
+
+class TestTimingSafeCompare:
+    tostr = str
+    tscmp = staticmethod(pypy_tscmp)
+
+    def test_tscmp_neq(self):
+        assert not self.tscmp(self.tostr('asd'), self.tostr('qwe'), 3, 3)
+
+    def test_tscmp_eq(self):
+        assert self.tscmp(self.tostr('asd'), self.tostr('asd'), 3, 3)
+
+    def test_tscmp_len(self):
+        assert self.tscmp(self.tostr('asdp'), self.tostr('asdq'), 3, 3)
+
+    def test_tscmp_nlen(self):
+        assert not self.tscmp(self.tostr('asd'), self.tostr('asd'), 2, 3)
+
+
+class TestTimingSafeCompareWide(TestTimingSafeCompare):
+    tostr = unicode
+    tscmp = staticmethod(pypy_tscmp_wide)
+
+    def test_tscmp_wide_nonascii(self):
+        a, b = u"\ud808\udf45", u"\ud808\udf45"
+        assert self.tscmp(a, b, len(a), len(b))
+        a, b = u"\ud808\udf45", u"\ud808\udf45 "
+        assert not self.tscmp(a, b, len(a), len(b))
diff --git a/pypy/module/operator/tscmp.c b/pypy/module/operator/tscmp.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/operator/tscmp.c
@@ -0,0 +1,80 @@
+/* Derived from CPython 3.3.5's operator.c::_tscmp
+ */
+
+#include 
+#include 
+#include "tscmp.h"
+
+int
+pypy_tscmp(const char *a, const char *b, long len_a, long len_b)
+{
+    /* The volatile type declarations make sure that the compiler has no
+     * chance to optimize and fold the code in any way that may change
+     * the timing.
+     */
+    volatile long length;
+    volatile const char *left;
+    volatile const char *right;
+    long i;
+    char result;
+
+    /* loop count depends on length of b */
+    length = len_b;
+    left = NULL;
+    right = b;
+
+    /* don't use else here to keep the amount of CPU instructions constant,
+     * volatile forces re-evaluation
+     *  */
+    if (len_a == length) {
+        left = *((volatile const char**)&a);
+        result = 0;
+    }
+    if (len_a != length) {
+        left = b;
+        result = 1;
+    }
+
+    for (i=0; i < length; i++) {
+        result |= *left++ ^ *right++;
+    }
+
+    return (result == 0);
+}
+
+int
+pypy_tscmp_wide(const wchar_t *a, const wchar_t *b, long len_a, long len_b)
+{
+    /* The volatile type declarations make sure that the compiler has no
+     * chance to optimize and fold the code in any way that may change
+     * the timing.
+     */
+    volatile long length;
+    volatile const wchar_t *left;
+    volatile const wchar_t *right;
+    long i;
+    wchar_t result;
+
+    /* loop count depends on length of b */
+    length = len_b;
+    left = NULL;
+    right = b;
+
+    /* don't use else here to keep the amount of CPU instructions constant,
+     * volatile forces re-evaluation
+     *  */
+    if (len_a == length) {
+        left = *((volatile const wchar_t**)&a);
+        result = 0;
+    }
+    if (len_a != length) {
+        left = b;
+        result = 1;
+    }
+
+    for (i=0; i < length; i++) {
+        result |= *left++ ^ *right++;
+    }
+
+    return (result == 0);
+}
diff --git a/pypy/module/operator/tscmp.h b/pypy/module/operator/tscmp.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/operator/tscmp.h
@@ -0,0 +1,2 @@
+int pypy_tscmp(const char *, const char *, long, long);
+int pypy_tscmp_wide(const wchar_t *, const wchar_t *, long, long);
diff --git a/pypy/module/operator/tscmp.py b/pypy/module/operator/tscmp.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/operator/tscmp.py
@@ -0,0 +1,73 @@
+"""
+Provides _compare_digest method, which is a safe comparing to prevent timing
+attacks for the hmac module.
+"""
+import py
+
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
+
+from pypy.interpreter.error import oefmt
+
+cwd = py.path.local(__file__).dirpath()
+eci = ExternalCompilationInfo(
+    includes=[cwd.join('tscmp.h')],
+    include_dirs=[str(cwd)],
+    separate_module_files=[cwd.join('tscmp.c')],
+    export_symbols=['pypy_tscmp', 'pypy_tscmp_wide'])
+
+
+def llexternal(*args, **kwargs):
+    kwargs.setdefault('compilation_info', eci)
+    kwargs.setdefault('sandboxsafe', True)
+    return rffi.llexternal(*args, **kwargs)
+
+
+pypy_tscmp = llexternal(
+    'pypy_tscmp',
+    [rffi.CCHARP, rffi.CCHARP, rffi.LONG, rffi.LONG],
+    rffi.INT)
+pypy_tscmp_wide = llexternal(
+    'pypy_tscmp_wide',
+    [rffi.CWCHARP, rffi.CWCHARP, rffi.LONG, rffi.LONG],
+    rffi.INT)
+
+
+def compare_digest(space, w_a, w_b):
+    """compare_digest(a, b) -> bool
+
+    Return 'a == b'.  This function uses an approach designed to prevent
+    timing analysis, making it appropriate for cryptography.  a and b
+    must both be of the same type: either str (ASCII only), or any type
+    that supports the buffer protocol (e.g. bytes).
+
+    Note: If a and b are of different lengths, or if an error occurs, a
+    timing attack could theoretically reveal information about the types
+    and lengths of a and b--but not their values.
+    """
+    if (space.isinstance_w(w_a, space.w_unicode) and
+        space.isinstance_w(w_b, space.w_unicode)):
+        a = space.unicode_w(w_a)
+        b = space.unicode_w(w_b)
+        with rffi.scoped_nonmoving_unicodebuffer(a) as a_buf:
+            with rffi.scoped_nonmoving_unicodebuffer(b) as b_buf:
+                result = pypy_tscmp_wide(a_buf, b_buf, len(a), len(b))
+        return space.wrap(rffi.cast(lltype.Bool, result))
+    return compare_digest_buffer(space, w_a, w_b)
+
+
+def compare_digest_buffer(space, w_a, w_b):
+    try:
+        a_buf = w_a.buffer_w(space, space.BUF_SIMPLE)
+        b_buf = w_b.buffer_w(space, space.BUF_SIMPLE)
+    except TypeError:
+        raise oefmt(space.w_TypeError,
+                    "unsupported operand types(s) or combination of types: "
+                    "'%T' and '%T'", w_a, w_b)
+
+    a = a_buf.as_str()
+    b = b_buf.as_str()
+    with rffi.scoped_nonmovingbuffer(a) as a_buf:
+        with rffi.scoped_nonmovingbuffer(b) as b_buf:
+            result = pypy_tscmp(a_buf, b_buf, len(a), len(b))
+    return space.wrap(rffi.cast(lltype.Bool, result))
diff --git a/rpython/jit/backend/arm/callbuilder.py b/rpython/jit/backend/arm/callbuilder.py
--- a/rpython/jit/backend/arm/callbuilder.py
+++ b/rpython/jit/backend/arm/callbuilder.py
@@ -92,7 +92,8 @@
             self.mc.LDR_ri(r.r7.value, r.r5.value)
 
         # change 'rpy_fastgil' to 0 (it should be non-zero right now)
-        self.mc.DMB()
+        if self.asm.cpu.cpuinfo.arch_version >= 7:
+            self.mc.DMB()
         self.mc.gen_load_int(r.r6.value, fastgil)
         self.mc.MOV_ri(r.ip.value, 0)
         self.mc.STR_ri(r.ip.value, r.r6.value)
@@ -112,7 +113,8 @@
         self.mc.STREX(r.r3.value, r.ip.value, r.r6.value, c=c.EQ)
                                                  # try to claim the lock
         self.mc.CMP_ri(r.r3.value, 0, cond=c.EQ) # did this succeed?
-        self.mc.DMB()
+        if self.asm.cpu.cpuinfo.arch_version >= 7:
+            self.mc.DMB()
         # the success of the lock acquisition is defined by
         # 'EQ is true', or equivalently by 'r3 == 0'.
         #
diff --git a/rpython/jit/backend/arm/codebuilder.py b/rpython/jit/backend/arm/codebuilder.py
--- a/rpython/jit/backend/arm/codebuilder.py
+++ b/rpython/jit/backend/arm/codebuilder.py
@@ -333,6 +333,8 @@
                     | (rn & 0xF) << 16)
 
     def DMB(self):
+        # ARMv7 only.  I guess ARMv6 CPUs cannot be used in symmetric
+        # multi-processing at all? That would make this instruction unneeded.
         # note: 'cond' is only permitted on Thumb here, but don't
         # write literally 0xf57ff05f, because it's larger than 31 bits
         c = cond.AL
diff --git a/rpython/jit/backend/arm/instructions.py b/rpython/jit/backend/arm/instructions.py
--- a/rpython/jit/backend/arm/instructions.py
+++ b/rpython/jit/backend/arm/instructions.py
@@ -142,6 +142,7 @@
     #'VCVT' : {'opc1':0xB, 'opc2':0xE, 'opc3':0x1, 'base': False},
 }
 
+# ARMv7 only
 simd_instructions_3regs = {
     'VADD_i64': {'A': 0x8, 'B': 0, 'U': 0},
     'VSUB_i64': {'A': 0x8, 'B': 0, 'U': 1},
diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -1,3 +1,4 @@
+from rpython.rlib import rgc
 from rpython.rlib.rarithmetic import ovfcheck
 from rpython.rtyper.lltypesystem import llmemory
 from rpython.jit.metainterp import history
@@ -390,8 +391,8 @@
         val = op.getarg(0)
         if val not in self.write_barrier_applied:
             v = op.getarg(1)
-            if isinstance(v, BoxPtr) or (isinstance(v, ConstPtr) and
-                                         bool(v.value)): # store a non-NULL
+            if (isinstance(v, BoxPtr) or (isinstance(v, ConstPtr) and
+                               rgc.needs_write_barrier(v.value))):
                 self.gen_write_barrier(val)
                 #op = op.copy_and_change(rop.SETFIELD_RAW)
         self.newops.append(op)
@@ -400,8 +401,8 @@
         val = op.getarg(0)
         if val not in self.write_barrier_applied:
             v = op.getarg(2)
-            if isinstance(v, BoxPtr) or (isinstance(v, ConstPtr) and
-                                         bool(v.value)): # store a non-NULL
+            if (isinstance(v, BoxPtr) or (isinstance(v, ConstPtr) and
+                                         rgc.needs_write_barrier(v.value))):
                 self.gen_write_barrier_array(val, op.getarg(1))
                 #op = op.copy_and_change(rop.SET{ARRAYITEM,INTERIORFIELD}_RAW)
         self.newops.append(op)
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -86,6 +86,14 @@
         collect(i)
         i += 1
 
+def needs_write_barrier(obj):
+    """ We need to emit write barrier if the right hand of assignment
+    is in nursery, used by the JIT for handling set*_gc(Const)
+    """
+    if not obj:
+        return False
+    return can_move(obj)
+
 def _heap_stats():
     raise NotImplementedError # can't be run directly
 
diff --git a/rpython/rtyper/module/ll_os_stat.py b/rpython/rtyper/module/ll_os_stat.py
--- a/rpython/rtyper/module/ll_os_stat.py
+++ b/rpython/rtyper/module/ll_os_stat.py
@@ -186,7 +186,10 @@
     _name_struct_stat = '_stati64'
     INCLUDES = ['sys/types.h', 'sys/stat.h', 'sys/statvfs.h']
 else:
-    _name_struct_stat = 'stat'
+    if sys.platform.startswith('linux'):
+        _name_struct_stat = 'stat64'
+    else:
+        _name_struct_stat = 'stat'
     INCLUDES = ['sys/types.h', 'sys/stat.h', 'sys/statvfs.h', 'unistd.h']
 
 compilation_info = ExternalCompilationInfo(
diff --git a/rpython/translator/platform/test/test_makefile.py b/rpython/translator/platform/test/test_makefile.py
--- a/rpython/translator/platform/test/test_makefile.py
+++ b/rpython/translator/platform/test/test_makefile.py
@@ -44,6 +44,7 @@
         assert res.returncode == 0        
     
     def test_900_files(self):
+        tmpdir = udir.join('test_900_files').ensure(dir=1)
         txt = '#include \n'
         for i in range(900):
             txt += 'int func%03d();\n' % i
@@ -52,11 +53,11 @@
             txt += '    j += func%03d();\n' % i
         txt += '    printf("%d\\n", j);\n'
         txt += '    return 0;};\n'
-        cfile = udir.join('test_900_files.c')
+        cfile = tmpdir.join('test_900_files.c')
         cfile.write(txt)
         cfiles = [cfile]
         for i in range(900):
-            cfile2 = udir.join('implement%03d.c' %i)
+            cfile2 = tmpdir.join('implement%03d.c' %i)
             cfile2.write('''
                 int func%03d()
             {
@@ -64,10 +65,10 @@
             }
             ''' % (i, i))
             cfiles.append(cfile2)
-        mk = self.platform.gen_makefile(cfiles, ExternalCompilationInfo(), path=udir)
+        mk = self.platform.gen_makefile(cfiles, ExternalCompilationInfo(), path=tmpdir)
         mk.write()
         self.platform.execute_makefile(mk)
-        res = self.platform.execute(udir.join('test_900_files'))
+        res = self.platform.execute(tmpdir.join('test_900_files'))
         self.check_res(res, '%d\n' %sum(range(900)))
 
     def test_precompiled_headers(self):
diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py
--- a/rpython/translator/platform/windows.py
+++ b/rpython/translator/platform/windows.py
@@ -203,6 +203,9 @@
         # the assembler still has the old behavior that all options
         # must come first, and after the file name all options are ignored.
         # So please be careful with the order of parameters! ;-)
+        pdb_dir = oname.dirname
+        if pdb_dir:
+                compile_args += ['/Fd%s\\' % (pdb_dir,)]
         args = ['/nologo', '/c'] + compile_args + ['/Fo%s' % (oname,), str(cfile)]
         self._execute_c_compiler(cc, args, oname)
         return oname
diff --git a/rpython/translator/sandbox/rsandbox.py b/rpython/translator/sandbox/rsandbox.py
--- a/rpython/translator/sandbox/rsandbox.py
+++ b/rpython/translator/sandbox/rsandbox.py
@@ -60,8 +60,7 @@
 
     def need_more_data(self):
         buflen = self.buflen
-        buf = lltype.malloc(rffi.CCHARP.TO, buflen, flavor='raw')
-        try:
+        with lltype.scoped_alloc(rffi.CCHARP.TO, buflen) as buf:
             buflen = rffi.cast(rffi.SIZE_T, buflen)
             count = ll_read_not_sandboxed(self.fd, buf, buflen)
             count = rffi.cast(lltype.Signed, count)
@@ -69,20 +68,15 @@
                 raise IOError
             self.buf += ''.join([buf[i] for i in range(count)])
             self.buflen *= 2
-        finally:
-            lltype.free(buf, flavor='raw')
 
 def sandboxed_io(buf):
     STDIN = 0
     STDOUT = 1
     # send the buffer with the marshalled fnname and input arguments to STDOUT
-    p = lltype.malloc(rffi.CCHARP.TO, len(buf), flavor='raw')
-    try:
+    with lltype.scoped_alloc(rffi.CCHARP.TO, len(buf)) as p:
         for i in range(len(buf)):
             p[i] = buf[i]
         writeall_not_sandboxed(STDOUT, p, len(buf))
-    finally:
-        lltype.free(p, flavor='raw')
     # build a Loader that will get the answer from STDIN
     loader = FdLoader(STDIN)
     # check for errors
@@ -108,9 +102,8 @@
 @signature(types.str(), returns=types.impossible())
 def not_implemented_stub(msg):
     STDERR = 2
-    buf = rffi.str2charp(msg + '\n')
-    writeall_not_sandboxed(STDERR, buf, len(msg) + 1)
-    rffi.free_charp(buf)
+    with rffi.scoped_str2charp(msg + '\n') as buf:
+        writeall_not_sandboxed(STDERR, buf, len(msg) + 1)
     raise RuntimeError(msg)  # XXX in RPython, the msg is ignored at the moment
 
 dump_string = rmarshal.get_marshaller(str)

From noreply at buildbot.pypy.org  Wed Sep 17 19:05:15 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 17 Sep 2014 19:05:15 +0200 (CEST)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <20140917170515.925F81C03C7@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: extradoc
Changeset: r535:0d8a1ce2e400
Date: 2014-09-17 19:05 +0200
http://bitbucket.org/pypy/pypy.org/changeset/0d8a1ce2e400/

Log:	update the values

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -9,13 +9,13 @@
 
 
    
-   $52380 of $105000 (49.9%)
+   $53651 of $105000 (51.1%)
    
    
    
    
 
diff --git a/don3.html b/don3.html
--- a/don3.html
+++ b/don3.html
@@ -9,13 +9,13 @@
 
 
    
-   $48412 of $60000 (80.7%)
+   $49869 of $60000 (83.1%)
    
    
    
    
 
diff --git a/don4.html b/don4.html
--- a/don4.html
+++ b/don4.html
@@ -9,7 +9,7 @@
 
@@ -17,7 +17,7 @@
    2nd call:
    
-   $13939 of $80000 (17.4%)
+   $16253 of $80000 (20.3%)
    
    
    
    
 

From noreply at buildbot.pypy.org  Wed Sep 17 19:20:37 2014
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Wed, 17 Sep 2014 19:20:37 +0200 (CEST)
Subject: [pypy-commit] pypy default: import is now unused
Message-ID: <20140917172038.036E21C023E@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor 
Branch: 
Changeset: r73581:8fac16ad89ab
Date: 2014-09-17 10:20 -0700
http://bitbucket.org/pypy/pypy/changeset/8fac16ad89ab/

Log:	import is now unused

diff --git a/pypy/module/operator/app_operator.py b/pypy/module/operator/app_operator.py
--- a/pypy/module/operator/app_operator.py
+++ b/pypy/module/operator/app_operator.py
@@ -4,7 +4,6 @@
 This module exports a set of operators as functions. E.g. operator.add(x,y) is
 equivalent to x+y.
 '''
-from __pypy__ import builtinify
 
 import types
 

From noreply at buildbot.pypy.org  Wed Sep 17 19:25:02 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 17 Sep 2014 19:25:02 +0200 (CEST)
Subject: [pypy-commit] creflect default: In-progress: change the outer layers
Message-ID: <20140917172502.BAA721C023E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r48:f76ddeee4479
Date: 2014-09-17 19:25 +0200
http://bitbucket.org/cffi/creflect/changeset/f76ddeee4479/

Log:	In-progress: change the outer layers

diff --git a/bin/creflect b/bin/creflect
--- a/bin/creflect
+++ b/bin/creflect
@@ -1,5 +1,5 @@
 #! /usr/bin/env python
 
 import sys
-import creflect
-sys.exit(creflect.main(sys.argv[1:]))
+from creflect.cmdline import main
+sys.exit(main(sys.argv[1:]))
diff --git a/creflect/__init__.py b/creflect/__init__.py
--- a/creflect/__init__.py
+++ b/creflect/__init__.py
@@ -1,3 +1,4 @@
-from cmdline import main
+from .driver import expand_cdef, CDefError
 
-__version__ = (1, 0)
+__version__ = "1.0.0"
+__version_info__ = (1, 0, 0)
diff --git a/creflect/codegen.py b/creflect/codegen.py
--- a/creflect/codegen.py
+++ b/creflect/codegen.py
@@ -120,6 +120,7 @@
 
     def generate(self, funcname):
         outerblock = CodeBlock(self)
+        outerblock.add_include('stdio.h')
         outerblock.writeline('int %s(char *r)' % (funcname,))
 
         funcblock = CodeBlock(self)
diff --git a/creflect/driver.py b/creflect/driver.py
--- a/creflect/driver.py
+++ b/creflect/driver.py
@@ -2,6 +2,11 @@
 from cparser import CSource, CDefError
 
 
+def expand_cdef(input, reflection_func_name, first_lineno=1):
+    csource = CSource(input, first_lineno)
+    return transform_cdef(csource, reflection_func_name)
+
+
 def get_preprocessor_directive(line):
     if line.startswith('#'):
         line = line[1:].split()
@@ -51,24 +56,19 @@
 ''' % (reflection_func_name,))
 
 
-def output_main(outputf):
-    outputf.write(r'''
+DEBUG_TEMPLATE = r'''
 /* debugging only */
 #include 
 #include 
-static void __creflect_report(char *cmd, void **args, int nargs) {
-    int i;
-    printf("%s", cmd);
-    for (i = 0; i < nargs; i++) {
-         printf("  %ld", (long)args[i]);
-         args[i] = (void *)0;
-    }
-    printf("\n");
-    if (cmd[0] == 'E')
-        exit(1);
+#define _CREFLECT_MAIN  %s
+int main(void) {
+    int size = _CREFLECT_MAIN((char *)0);
+    char *result = malloc(size);
+    int err = _CREFLECT_MAIN(result);
+    printf("%%s", result);
+    return (err != 0);
 }
-int main(void) {   /* for debugging only: compile and run as an executable */
-    __CREFLECT_PREV(__creflect_report);
-    return 0;
-}
-''')
+'''
+
+def get_debug_code(reflection_func_name):
+    return DEBUG_TEMPLATE % (reflection_func_name,)
diff --git a/test/test_driver.py b/test/test_driver.py
--- a/test/test_driver.py
+++ b/test/test_driver.py
@@ -1,9 +1,37 @@
 import os
 from StringIO import StringIO
-from creflect import driver
+from creflect.driver import expand_cdef, get_debug_code
 from .udir import udir
 
-def test_driver():
+def compile_and_run(real_code, generated_code, funcname):
+    f = open(str(udir.join(funcname + '.c')), 'w')
+    print >> f, '/* real code */'
+    print >> f, real_code
+    print >> f, '/* generated code */'
+    print >> f, generated_code
+    print >> f, get_debug_code(funcname)
+    f.close()
+    #
+    err = os.system(
+        "cd '%s' && gcc -fPIC -Werror -Wall %s.c -o %s"
+        % (str(udir), funcname, funcname))
+    assert err == 0
+    #
+    g = os.popen("cd '%s' && %s" % (str(udir), funcname), 'r')
+    data = g.read()
+    err = g.close()
+    assert err is None
+    return data
+
+
+def test_expand_cdef():
+    real_code = "typedef short a_t[42];\n"
+    gen = expand_cdef("typedef long a_t[20];", "test_expand_cdef")
+    data = compile_and_run(real_code, gen, "test_expand_cdef")
+    assert data == real_code
+
+
+def XXXtest_driver():
     f = StringIO("""
 typedef unsigned int a_t[5];
 

From noreply at buildbot.pypy.org  Wed Sep 17 20:49:07 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 17 Sep 2014 20:49:07 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: release gil around fread
Message-ID: <20140917184907.265061C03C7@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73582:77e9e65012f3
Date: 2014-09-17 13:08 -0400
http://bitbucket.org/pypy/pypy/changeset/77e9e65012f3/

Log:	release gil around fread

diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -101,7 +101,7 @@
 
 c_fgets = llexternal('fgets', [rffi.CCHARP, rffi.INT, FILEP], rffi.CCHARP)
 c_fread = llexternal('fread', [rffi.CCHARP, rffi.SIZE_T, rffi.SIZE_T, FILEP],
-                     rffi.SIZE_T)
+                     rffi.SIZE_T, releasegil=False)
 c_fwrite = llexternal('fwrite', [rffi.CCHARP, rffi.SIZE_T, rffi.SIZE_T, FILEP],
                       rffi.SIZE_T)
 
@@ -334,50 +334,58 @@
             raise IOError("File not open for writing")
 
     def _fread(self, buf, n, stream):
+        before = rffi.aroundstate.before
+        if before: before()
+
         if not self._univ_newline:
-            return c_fread(buf, 1, n, stream)
+            i = c_fread(buf, 1, n, stream)
+        else:
+            i = 0
+            newlinetypes = self._newlinetypes
+            skipnextlf = self._skipnextlf
+            assert n >= 0
+            while n:
+                dst = rffi.ptradd(buf, i)
+                nread = c_fread(dst, 1, n, stream)
+                assert nread <= n
+                if nread == 0:
+                    break
 
-        i = 0
-        newlinetypes = self._newlinetypes
-        skipnextlf = self._skipnextlf
-        assert n >= 0
-        while n:
-            dst = rffi.ptradd(buf, i)
-            nread = c_fread(dst, 1, n, stream)
-            assert nread <= n
-            if nread == 0:
-                break
+                j = 0
+                n -= nread
+                shortread = n != 0
+                while nread:
+                    nread -= 1
+                    c = dst[j]
+                    j += 1
+                    if c == '\r':
+                        buf[i] = '\n'
+                        i += 1
+                        skipnextlf = True
+                    elif skipnextlf and c == '\n':
+                        skipnextlf = False
+                        newlinetypes |= NEWLINE_CRLF
+                        n += 1
+                    else:
+                        if c == '\n':
+                            newlinetypes |= NEWLINE_LF
+                        elif skipnextlf:
+                            newlinetypes |= NEWLINE_CR
+                        buf[i] = c
+                        i += 1
+                        skipnextlf = False
+                if shortread:
+                    if skipnextlf and c_feof(stream):
+                        newlinetypes |= NEWLINE_CR
+                    break
+            self._newlinetypes = newlinetypes
+            self._skipnextlf = skipnextlf
 
-            j = 0
-            n -= nread
-            shortread = n != 0
-            while nread:
-                nread -= 1
-                c = dst[j]
-                j += 1
-                if c == '\r':
-                    buf[i] = '\n'
-                    i += 1
-                    skipnextlf = True
-                elif skipnextlf and c == '\n':
-                    skipnextlf = False
-                    newlinetypes |= NEWLINE_CRLF
-                    n += 1
-                else:
-                    if c == '\n':
-                        newlinetypes |= NEWLINE_LF
-                    elif skipnextlf:
-                        newlinetypes |= NEWLINE_CR
-                    buf[i] = c
-                    i += 1
-                    skipnextlf = False
-            if shortread:
-                if skipnextlf and c_feof(stream):
-                    newlinetypes |= NEWLINE_CR
-                break
-        self._newlinetypes = newlinetypes
-        self._skipnextlf = skipnextlf
+        after = rffi.aroundstate.after
+        if after: after()
         return i
+    _fread._gctransformer_hint_close_stack_ = True
+    _fread._dont_inline_ = True
 
     def _new_buffersize(self, currentsize):
         ll_file = self._ll_file

From noreply at buildbot.pypy.org  Wed Sep 17 21:51:45 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 17 Sep 2014 21:51:45 +0200 (CEST)
Subject: [pypy-commit] pypy default: this test only passes on use-file-star
Message-ID: <20140917195145.7EA611C023E@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: 
Changeset: r73583:d1947525102a
Date: 2014-09-17 15:50 -0400
http://bitbucket.org/pypy/pypy/changeset/d1947525102a/

Log:	this test only passes on use-file-star

diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -91,9 +91,9 @@
         assert isinstance(sys.__stderr__, file)
         assert isinstance(sys.__stdin__, file)
 
-        assert sys.__stdin__.name == ""
-        assert sys.__stdout__.name == ""
-        assert sys.__stderr__.name == ""
+        #assert sys.__stdin__.name == ""
+        #assert sys.__stdout__.name == ""
+        #assert sys.__stderr__.name == ""
 
         if self.appdirect and not isinstance(sys.stdin, file):
             return

From noreply at buildbot.pypy.org  Wed Sep 17 21:51:46 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 17 Sep 2014 21:51:46 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20140917195146.B27361C023E@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: 
Changeset: r73584:201000b16c85
Date: 2014-09-17 15:51 -0400
http://bitbucket.org/pypy/pypy/changeset/201000b16c85/

Log:	merge heads

diff --git a/pypy/module/operator/app_operator.py b/pypy/module/operator/app_operator.py
--- a/pypy/module/operator/app_operator.py
+++ b/pypy/module/operator/app_operator.py
@@ -4,7 +4,6 @@
 This module exports a set of operators as functions. E.g. operator.add(x,y) is
 equivalent to x+y.
 '''
-from __pypy__ import builtinify
 
 import types
 

From noreply at buildbot.pypy.org  Wed Sep 17 21:52:15 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 17 Sep 2014 21:52:15 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: merge default
Message-ID: <20140917195215.4C96C1C023E@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73585:e5f4f9875bdf
Date: 2014-09-17 15:50 -0400
http://bitbucket.org/pypy/pypy/changeset/e5f4f9875bdf/

Log:	merge default

diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -91,9 +91,9 @@
         assert isinstance(sys.__stderr__, file)
         assert isinstance(sys.__stdin__, file)
 
-        assert sys.__stdin__.name == ""
-        assert sys.__stdout__.name == ""
-        assert sys.__stderr__.name == ""
+        #assert sys.__stdin__.name == ""
+        #assert sys.__stdout__.name == ""
+        #assert sys.__stderr__.name == ""
 
         if self.appdirect and not isinstance(sys.stdin, file):
             return

From noreply at buildbot.pypy.org  Wed Sep 17 21:52:16 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 17 Sep 2014 21:52:16 +0200 (CEST)
Subject: [pypy-commit] pypy use-file-star-for-file: reenable this test
Message-ID: <20140917195216.795121C023E@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns 
Branch: use-file-star-for-file
Changeset: r73586:ef3178a40f55
Date: 2014-09-17 15:50 -0400
http://bitbucket.org/pypy/pypy/changeset/ef3178a40f55/

Log:	reenable this test

diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -91,9 +91,9 @@
         assert isinstance(sys.__stderr__, file)
         assert isinstance(sys.__stdin__, file)
 
-        #assert sys.__stdin__.name == ""
-        #assert sys.__stdout__.name == ""
-        #assert sys.__stderr__.name == ""
+        assert sys.__stdin__.name == ""
+        assert sys.__stdout__.name == ""
+        assert sys.__stderr__.name == ""
 
         if self.appdirect and not isinstance(sys.stdin, file):
             return

From noreply at buildbot.pypy.org  Wed Sep 17 21:56:06 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Wed, 17 Sep 2014 21:56:06 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: issue1798: add nt._getfinalpathname. add
 _getfileinformation as well
Message-ID: <20140917195606.DC93F1C023E@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey 
Branch: py3k
Changeset: r73587:1900e5ea6ec4
Date: 2014-09-17 12:53 -0700
http://bitbucket.org/pypy/pypy/changeset/1900e5ea6ec4/

Log:	issue1798: add nt._getfinalpathname. add _getfileinformation as well

diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py
--- a/pypy/module/posix/__init__.py
+++ b/pypy/module/posix/__init__.py
@@ -164,6 +164,11 @@
     # not visible via os, inconsistency in nt:
     if hasattr(posix, '_getfullpathname'):
         interpleveldefs['_getfullpathname'] = 'interp_posix._getfullpathname'
+    if os.name == 'nt':
+        interpleveldefs.update({
+                '_getfileinformation': 'interp_posix._getfileinformation',
+                '_getfinalpathname': 'interp_posix._getfinalpathname',
+        })
     if hasattr(os, 'chroot'):
         interpleveldefs['chroot'] = 'interp_posix.chroot'
 
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -5,8 +5,7 @@
 from rpython.rlib.objectmodel import specialize
 from rpython.rlib.rarithmetic import r_longlong
 from rpython.rlib.unroll import unrolling_iterable
-from rpython.rtyper.module import ll_os_stat
-from rpython.rtyper.module.ll_os import RegisterOs
+from rpython.rtyper.module import ll_os, ll_os_stat
 
 from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
 from pypy.interpreter.error import OperationError, wrap_oserror, wrap_oserror2
@@ -1205,7 +1204,7 @@
         raise wrap_oserror(space, e)
 
 def declare_new_w_star(name):
-    if name in RegisterOs.w_star_returning_int:
+    if name in ll_os.RegisterOs.w_star_returning_int:
         @unwrap_spec(status=c_int)
         def WSTAR(space, status):
             return space.wrap(getattr(os, name)(status))
@@ -1217,7 +1216,7 @@
     WSTAR.func_name = name
     return WSTAR
 
-for name in RegisterOs.w_star:
+for name in ll_os.RegisterOs.w_star:
     if hasattr(os, name):
         func = declare_new_w_star(name)
         globals()[name] = func
@@ -1383,3 +1382,25 @@
         if codeset:
             return space.wrap(codeset)
     return space.w_None
+
+if _WIN32:
+    @unwrap_spec(fd=c_int)
+    def _getfileinformation(space, fd):
+        try:
+            info = ll_os._getfileinformation(fd)
+        except OSError as e:
+            raise wrap_oserror(space, e)
+        return space.newtuple([space.wrap(info[0]),
+                               space.wrap(info[1]),
+                               space.wrap(info[2])])
+
+    def _getfinalpathname(space, w_path):
+        path = space.unicode_w(w_path)
+        try:
+            result = ll_os._getfinalpathname(path)
+        except ll_os.LLNotImplemented as e:
+            raise OperationError(space.w_NotImplementedError,
+                                 space.wrap(e.msg))
+        except OSError as e:
+            raise wrap_oserror2(space, e, w_path)
+        return space.wrap(result)
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -1040,6 +1040,21 @@
         # just ensure it returns something reasonable
         assert encoding is None or type(encoding) is str
 
+    if os.name == 'nt':
+        def test__getfileinformation(self):
+            import os
+            path = os.path.join(self.pdir, 'file1')
+            with open(path) as fp:
+                info = self.posix._getfileinformation(fp.fileno())
+            assert len(info) == 3
+            assert all(isinstance(obj, int) for obj in info)
+
+        def test__getfinalpathname(self):
+            import os
+            path = os.path.join(self.pdir, 'file1')
+            result = self.posix._getfinalpathname(path)
+            assert os.path.exists(result)
+
 
 class AppTestEnvironment(object):
     def setup_class(cls):
diff --git a/rpython/rlib/rdynload.py b/rpython/rlib/rdynload.py
--- a/rpython/rlib/rdynload.py
+++ b/rpython/rlib/rdynload.py
@@ -158,3 +158,4 @@
         return res
 
     LoadLibrary = rwin32.LoadLibrary
+    GetModuleHandle = rwin32.GetModuleHandle
diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py
--- a/rpython/rlib/rwin32.py
+++ b/rpython/rlib/rwin32.py
@@ -130,6 +130,7 @@
     # is hidden by operations in ll2ctypes.  Call it now.
     GetLastError()
 
+    GetModuleHandle = winexternal('GetModuleHandleA', [rffi.CCHARP], HMODULE)
     LoadLibrary = winexternal('LoadLibraryA', [rffi.CCHARP], HMODULE)
     GetProcAddress = winexternal('GetProcAddress',
                                  [HMODULE, rffi.CCHARP],
diff --git a/rpython/rtyper/lltypesystem/ll2ctypes.py b/rpython/rtyper/lltypesystem/ll2ctypes.py
--- a/rpython/rtyper/lltypesystem/ll2ctypes.py
+++ b/rpython/rtyper/lltypesystem/ll2ctypes.py
@@ -361,7 +361,9 @@
             functype = ctypes.CFUNCTYPE
             if sys.platform == 'win32':
                 from rpython.rlib.clibffi import FFI_STDCALL, FFI_DEFAULT_ABI
-                if getattr(T.TO, 'ABI', FFI_DEFAULT_ABI) == FFI_STDCALL:
+                # XXX:
+                #if getattr(T.TO, 'ABI', FFI_DEFAULT_ABI) == FFI_STDCALL:
+                if getattr(T.TO, 'ABI', FFI_DEFAULT_ABI) == 'FFI_STDCALL':
                     # for win32 system call
                     functype = ctypes.WINFUNCTYPE
             argtypes = [get_ctypes_type(ARG) for ARG in T.TO.ARGS
diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py
--- a/rpython/rtyper/module/ll_os.py
+++ b/rpython/rtyper/module/ll_os.py
@@ -105,6 +105,12 @@
 
 _CYGWIN = sys.platform == 'cygwin'
 
+# plain NotImplementedError is invalid RPython
+class LLNotImplemented(NotImplementedError):
+
+    def __init__(self, msg):
+        self.msg = msg
+
 class CConfig:
     """
     Definitions for platform integration.
@@ -1179,7 +1185,7 @@
                              condition=sys.platform=='win32')
     def register_posix__getfullpathname(self, traits):
         # this nt function is not exposed via os, but needed
-        # to get a correct implementation of os.abspath
+        # to get a correct implementation of os.path.abspath
         from rpython.rtyper.module.ll_win32file import make_getfullpathname_impl
         getfullpathname_llimpl = make_getfullpathname_impl(traits)
 
@@ -1963,10 +1969,12 @@
         return OsEnvironController()
 
 # ____________________________________________________________
-# Support for the WindowsError exception
+# Support for the WindowsError exception and misc functions
 
 if sys.platform == 'win32':
     from rpython.rlib import rwin32
+    from rpython.rtyper.module.ll_win32file import (
+        make__getfileinformation_impl, make__getfinalpathname_impl)
 
     class RegisterFormatError(BaseLazyRegistering):
         def __init__(self):
@@ -1977,3 +1985,6 @@
             return extdef([lltype.Signed], str,
                           "rwin32_FormatError",
                           llimpl=rwin32.llimpl_FormatError)
+
+    _getfileinformation = make__getfileinformation_impl(UnicodeTraits())
+    _getfinalpathname = make__getfinalpathname_impl(UnicodeTraits())
diff --git a/rpython/rtyper/module/ll_win32file.py b/rpython/rtyper/module/ll_win32file.py
--- a/rpython/rtyper/module/ll_win32file.py
+++ b/rpython/rtyper/module/ll_win32file.py
@@ -55,6 +55,15 @@
         FILE_TYPE_CHAR = platform.ConstantInteger('FILE_TYPE_CHAR')
         FILE_TYPE_PIPE = platform.ConstantInteger('FILE_TYPE_PIPE')
 
+        FILE_WRITE_ATTRIBUTES = platform.ConstantInteger(
+            'FILE_WRITE_ATTRIBUTES')
+        OPEN_EXISTING = platform.ConstantInteger(
+            'OPEN_EXISTING')
+        FILE_FLAG_BACKUP_SEMANTICS = platform.ConstantInteger(
+            'FILE_FLAG_BACKUP_SEMANTICS')
+        VOLUME_NAME_DOS = platform.ConstantInteger('VOLUME_NAME_DOS')
+        VOLUME_NAME_NT = platform.ConstantInteger('VOLUME_NAME_NT')
+
         WIN32_FILE_ATTRIBUTE_DATA = platform.Struct(
             'WIN32_FILE_ATTRIBUTE_DATA',
             [('dwFileAttributes', rwin32.DWORD),
@@ -67,14 +76,15 @@
         BY_HANDLE_FILE_INFORMATION = platform.Struct(
             'BY_HANDLE_FILE_INFORMATION',
             [('dwFileAttributes', rwin32.DWORD),
+             ('ftCreationTime', rwin32.FILETIME),
+             ('ftLastAccessTime', rwin32.FILETIME),
+             ('ftLastWriteTime', rwin32.FILETIME),
+             ('dwVolumeSerialNumber', rwin32.DWORD),
              ('nFileSizeHigh', rwin32.DWORD),
              ('nFileSizeLow', rwin32.DWORD),
              ('nNumberOfLinks', rwin32.DWORD),
              ('nFileIndexHigh', rwin32.DWORD),
-             ('nFileIndexLow', rwin32.DWORD),
-             ('ftCreationTime', rwin32.FILETIME),
-             ('ftLastAccessTime', rwin32.FILETIME),
-             ('ftLastWriteTime', rwin32.FILETIME)])
+             ('nFileIndexLow', rwin32.DWORD)])
 
     config = platform.configure(CConfig)
 
@@ -92,6 +102,8 @@
                        INVALID_FILE_ATTRIBUTES
                        _S_IFDIR _S_IFREG _S_IFCHR _S_IFIFO
                        FILE_TYPE_UNKNOWN FILE_TYPE_CHAR FILE_TYPE_PIPE
+                       FILE_WRITE_ATTRIBUTES OPEN_EXISTING FILE_FLAG_BACKUP_SEMANTICS
+                       VOLUME_NAME_DOS VOLUME_NAME_NT
                        ERROR_FILE_NOT_FOUND ERROR_NO_MORE_FILES
                        ERROR_SHARING_VIOLATION
                     '''.split():
@@ -163,6 +175,13 @@
             [traits.CCHARP, traits.CCHARP],
             rwin32.BOOL)
 
+        CreateFile = external(
+            'CreateFile' + apisuffix,
+            [traits.CCHARP, rwin32.DWORD, rwin32.DWORD,
+             rwin32.LPSECURITY_ATTRIBUTES, rwin32.DWORD, rwin32.DWORD,
+             rwin32.HANDLE],
+            rwin32.HANDLE)
+
         DeleteFile = external(
             'DeleteFile' + suffix,
             [traits.CCHARP],
@@ -173,6 +192,27 @@
             [traits.CCHARP, traits.CCHARP],
             rwin32.BOOL)
 
+        # dynamically loaded
+        GetFinalPathNameByHandle = None
+
+        @staticmethod
+        def check_GetFinalPathNameByHandle():
+            if Win32Traits.GetFinalPathNameByHandle:
+                return True
+
+            from rpython.rlib.rdynload import GetModuleHandle, dlsym
+            hKernel32 = GetModuleHandle("KERNEL32")
+            try:
+                func = dlsym(hKernel32, 'GetFinalPathNameByHandle' + suffix)
+            except KeyError:
+                return False
+
+            TYPE = lltype.Ptr(lltype.FuncType(
+                    [rwin32.HANDLE, traits.CCHARP, rwin32.DWORD, rwin32.DWORD],
+                    rwin32.DWORD, abi='FFI_STDCALL'))
+            Win32Traits.GetFinalPathNameByHandle = rffi.cast(TYPE, func)
+            return True
+
     return Win32Traits
 
 #_______________________________________________________________
@@ -336,27 +376,6 @@
     win32traits = make_win32_traits(traits)
     from rpython.rtyper.module.ll_os_stat import time_t_to_FILE_TIME
 
-    class CConfig:
-        _compilation_info_ = ExternalCompilationInfo(
-            includes = ['windows.h'],
-            )
-
-        FILE_WRITE_ATTRIBUTES = platform.ConstantInteger(
-            'FILE_WRITE_ATTRIBUTES')
-        OPEN_EXISTING = platform.ConstantInteger(
-            'OPEN_EXISTING')
-        FILE_FLAG_BACKUP_SEMANTICS = platform.ConstantInteger(
-            'FILE_FLAG_BACKUP_SEMANTICS')
-    globals().update(platform.configure(CConfig))
-
-    CreateFile = rffi.llexternal(
-        'CreateFile' + win32traits.apisuffix,
-        [traits.CCHARP, rwin32.DWORD, rwin32.DWORD,
-         rwin32.LPSECURITY_ATTRIBUTES, rwin32.DWORD, rwin32.DWORD,
-         rwin32.HANDLE],
-        rwin32.HANDLE,
-        calling_conv='win')
-
     GetSystemTime = rffi.llexternal(
         'GetSystemTime',
         [lltype.Ptr(rwin32.SYSTEMTIME)],
@@ -381,10 +400,10 @@
 
     @specialize.argtype(1)
     def os_utime_llimpl(path, tp):
-        hFile = CreateFile(path,
-                           FILE_WRITE_ATTRIBUTES, 0,
-                           None, OPEN_EXISTING,
-                           FILE_FLAG_BACKUP_SEMANTICS,
+        hFile = win32traits.CreateFile(path,
+                           win32traits.FILE_WRITE_ATTRIBUTES, 0,
+                           None, win32traits.OPEN_EXISTING,
+                           win32traits.FILE_FLAG_BACKUP_SEMANTICS,
                            rwin32.NULL_HANDLE)
         if hFile == rwin32.INVALID_HANDLE_VALUE:
             raise rwin32.lastWindowsError()
@@ -413,3 +432,68 @@
             lltype.free(mtime, flavor='raw')
 
     return os_utime_llimpl
+
+#_______________________________________________________________
+# _getfileinformation (py3)
+
+def make__getfileinformation_impl(traits):
+    from rpython.rlib import rwin32
+    win32traits = make_win32_traits(traits)
+
+    def _getfileinformation_llimpl(fd):
+        hFile = rwin32.get_osfhandle(fd)
+        with lltype.scoped_alloc(
+            win32traits.BY_HANDLE_FILE_INFORMATION) as info:
+            if win32traits.GetFileInformationByHandle(hFile, info) == 0:
+                raise lastWindowsError("_getfileinformation")
+            return (rffi.cast(lltype.Signed, info.c_dwVolumeSerialNumber),
+                    rffi.cast(lltype.Signed, info.c_nFileIndexHigh),
+                    rffi.cast(lltype.Signed, info.c_nFileIndexLow))
+
+    return _getfileinformation_llimpl
+
+#_______________________________________________________________
+# _getfinalpathname (py3)
+
+def make__getfinalpathname_impl(traits):
+    from rpython.rlib import rwin32
+    from rpython.rtyper.module.ll_os import LLNotImplemented
+    assert traits.str is unicode, 'Currently only handles unicode paths'
+    win32traits = make_win32_traits(traits)
+
+    def _getfinalpathname_llimpl(path):
+        if not win32traits.check_GetFinalPathNameByHandle():
+            raise LLNotImplemented("GetFinalPathNameByHandle not available on "
+                                   "this platform")
+
+        hFile = win32traits.CreateFile(path, 0, 0, None,
+                                       win32traits.OPEN_EXISTING,
+                                       win32traits.FILE_FLAG_BACKUP_SEMANTICS,
+                                       rwin32.NULL_HANDLE)
+        if hFile == rwin32.INVALID_HANDLE_VALUE:
+            raise rwin32.lastWindowsError("CreateFile")
+
+        VOLUME_NAME_DOS = rffi.cast(rwin32.DWORD, win32traits.VOLUME_NAME_DOS)
+        try:
+            size = win32traits.GetFinalPathNameByHandle(
+                hFile,
+                lltype.nullptr(traits.CCHARP.TO),
+                rffi.cast(rwin32.DWORD, 0),
+                VOLUME_NAME_DOS)
+            if size == 0:
+                raise rwin32.lastWindowsError("GetFinalPathNameByHandle")
+
+            with lltype.scoped_alloc(traits.CCHARP.TO, size + 1) as target_path:
+                result = win32traits.GetFinalPathNameByHandle(
+                    hFile,
+                    target_path,
+                    size,
+                    VOLUME_NAME_DOS)
+                if result == 0:
+                    raise rwin32.lastWindowsError("GetFinalPathNameByHandle")
+                return traits.charpsize2str(target_path,
+                                            rffi.cast(lltype.Signed, result))
+        finally:
+            rwin32.CloseHandle(hFile)
+
+    return _getfinalpathname_llimpl
diff --git a/rpython/rtyper/module/support.py b/rpython/rtyper/module/support.py
--- a/rpython/rtyper/module/support.py
+++ b/rpython/rtyper/module/support.py
@@ -49,6 +49,7 @@
     CHAR = rffi.CHAR
     CCHARP = rffi.CCHARP
     charp2str = staticmethod(rffi.charp2str)
+    charpsize2str = staticmethod(rffi.charpsize2str)
     scoped_str2charp = staticmethod(rffi.scoped_str2charp)
     str2charp = staticmethod(rffi.str2charp)
     free_charp = staticmethod(rffi.free_charp)
@@ -68,6 +69,7 @@
     CHAR = rffi.WCHAR_T
     CCHARP = rffi.CWCHARP
     charp2str = staticmethod(rffi.wcharp2unicode)
+    charpsize2str = staticmethod(rffi.wcharpsize2unicode)
     str2charp = staticmethod(rffi.unicode2wcharp)
     scoped_str2charp = staticmethod(rffi.scoped_unicode2wcharp)
     free_charp = staticmethod(rffi.free_wcharp)
diff --git a/rpython/rtyper/module/test/test_ll_win32file.py b/rpython/rtyper/module/test/test_ll_win32file.py
new file mode 100644
--- /dev/null
+++ b/rpython/rtyper/module/test/test_ll_win32file.py
@@ -0,0 +1,26 @@
+import os
+
+import py
+
+from rpython.rtyper.module import ll_os
+
+if not ll_os._WIN32:
+    py.test.skip("requires Windows")
+
+
+def test__getfinalpathname():
+    path = __file__.decode('mbcs')
+    try:
+        result = ll_os._getfinalpathname(path)
+    except ll_os.LLNotImplemented:
+        py.test.skip("_getfinalpathname not supported on this platform")
+    assert os.path.exists(result)
+
+
+def test__getfileinformation():
+    with open(__file__) as fp:
+        stat = os.fstat(fp.fileno())
+        info = ll_os._getfileinformation(fp.fileno())
+    serial, high, low = info
+    assert type(serial) in (int, long)
+    assert (high << 32) + low == stat.st_ino

From noreply at buildbot.pypy.org  Wed Sep 17 22:01:14 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Wed, 17 Sep 2014 22:01:14 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: rearrange
Message-ID: <20140917200114.39B091C023E@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey 
Branch: py3k
Changeset: r73588:d0c26086a7f6
Date: 2014-09-17 12:57 -0700
http://bitbucket.org/pypy/pypy/changeset/d0c26086a7f6/

Log:	rearrange

diff --git a/rpython/rtyper/module/test/test_ll_win32file.py b/rpython/rtyper/module/test/test_ll_win32file.py
--- a/rpython/rtyper/module/test/test_ll_win32file.py
+++ b/rpython/rtyper/module/test/test_ll_win32file.py
@@ -8,6 +8,15 @@
     py.test.skip("requires Windows")
 
 
+def test__getfileinformation():
+    with open(__file__) as fp:
+        stat = os.fstat(fp.fileno())
+        info = ll_os._getfileinformation(fp.fileno())
+    serial, high, low = info
+    assert type(serial) in (int, long)
+    assert (high << 32) + low == stat.st_ino
+
+
 def test__getfinalpathname():
     path = __file__.decode('mbcs')
     try:
@@ -15,12 +24,3 @@
     except ll_os.LLNotImplemented:
         py.test.skip("_getfinalpathname not supported on this platform")
     assert os.path.exists(result)
-
-
-def test__getfileinformation():
-    with open(__file__) as fp:
-        stat = os.fstat(fp.fileno())
-        info = ll_os._getfileinformation(fp.fileno())
-    serial, high, low = info
-    assert type(serial) in (int, long)
-    assert (high << 32) + low == stat.st_ino

From noreply at buildbot.pypy.org  Wed Sep 17 22:01:15 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Wed, 17 Sep 2014 22:01:15 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: handle older windows versions
Message-ID: <20140917200115.8CD5C1C023E@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey 
Branch: py3k
Changeset: r73589:80ee4756ddc3
Date: 2014-09-17 13:00 -0700
http://bitbucket.org/pypy/pypy/changeset/80ee4756ddc3/

Log:	handle older windows versions

diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -1052,7 +1052,10 @@
         def test__getfinalpathname(self):
             import os
             path = os.path.join(self.pdir, 'file1')
-            result = self.posix._getfinalpathname(path)
+            try:
+                result = self.posix._getfinalpathname(path)
+            except NotImplementedError:
+                skip("_getfinalpathname not supported on this platform")
             assert os.path.exists(result)
 
 

From noreply at buildbot.pypy.org  Wed Sep 17 22:11:06 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 17 Sep 2014 22:11:06 +0200 (CEST)
Subject: [pypy-commit] pypy default: Be less inefficient when we decode
	string literals:
Message-ID: <20140917201106.8A4691C33F0@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: 
Changeset: r73590:10fd806838e2
Date: 2014-09-17 22:01 +0200
http://bitbucket.org/pypy/pypy/changeset/10fd806838e2/

Log:	Be less inefficient when we decode string literals: use RPython
	unicode instead of the full codec machinery, just to write a \U
	escape sequence!

diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py
--- a/pypy/interpreter/pyparser/parsestring.py
+++ b/pypy/interpreter/pyparser/parsestring.py
@@ -83,12 +83,6 @@
     v = PyString_DecodeEscape(space, substr, 'strict', enc)
     return space.wrap(v)
 
-def hexbyte(val):
-    result = "%x" % val
-    if len(result) == 1:
-        result = "0" + result
-    return result
-
 def decode_unicode_utf8(space, s, ps, q):
     # ****The Python 2.7 version, producing UTF-32 escapes****
     # String is utf8-encoded, but 'unicode_escape' expects
@@ -108,15 +102,14 @@
                 # instead.
                 lis.append("u005c")
         if ord(s[ps]) & 0x80: # XXX inefficient
-            w, ps = decode_utf8(space, s, ps, end, "utf-32-be")
-            rn = len(w)
-            assert rn % 4 == 0
-            for i in range(0, rn, 4):
-                lis.append('\\U')
-                lis.append(hexbyte(ord(w[i])))
-                lis.append(hexbyte(ord(w[i+1])))
-                lis.append(hexbyte(ord(w[i+2])))
-                lis.append(hexbyte(ord(w[i+3])))
+            w, ps = decode_utf8(space, s, ps, end)
+            for c in w:
+                # The equivalent of %08x, which is not supported by RPython.
+                # 7 zeroes are enough for the unicode range, and the
+                # result still fits in 32-bit.
+                hexa = hex(ord(c) + 0x10000000)
+                lis.append('\\U0')
+                lis.append(hexa[3:])  # Skip 0x and the leading 1
         else:
             lis.append(s[ps])
             ps += 1
@@ -136,7 +129,7 @@
             # note that the C code has a label here.
             # the logic is the same.
             if recode_encoding and ord(s[ps]) & 0x80:
-                w, ps = decode_utf8(space, s, ps, end, recode_encoding)
+                w, ps = decode_utf8_recode(space, s, ps, end, recode_encoding)
                 # Append bytes to output buffer.
                 builder.append(w)
             else:
@@ -222,15 +215,19 @@
             ch >= 'A' and ch <= 'F')
 
 
-def decode_utf8(space, s, ps, end, encoding):
+def decode_utf8(space, s, ps, end):
     assert ps >= 0
     pt = ps
     # while (s < end && *s != '\\') s++; */ /* inefficient for u".."
     while ps < end and ord(s[ps]) & 0x80:
         ps += 1
-    w_u = space.wrap(unicodehelper.decode_utf8(space, s[pt:ps]))
-    w_v = unicodehelper.encode(space, w_u, encoding)
-    v = space.str_w(w_v)
+    u = unicodehelper.decode_utf8(space, s[pt:ps])
+    return u, ps
+
+def decode_utf8_recode(space, s, ps, end, recode_encoding):
+    u, ps = decode_utf8(space, s, ps, end)
+    w_v = unicodehelper.encode(space, space.wrap(u), recode_encoding)
+    v = space.bytes_w(w_v)
     return v, ps
 
 def raise_app_valueerror(space, msg):
diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -5,6 +5,7 @@
 
 @specialize.memo()
 def decode_error_handler(space):
+    # Fast version of the "strict" errors handler.
     def raise_unicode_exception_decode(errors, encoding, msg, s,
                                        startingpos, endingpos):
         raise OperationError(space.w_UnicodeDecodeError,
@@ -17,6 +18,7 @@
 
 @specialize.memo()
 def encode_error_handler(space):
+    # Fast version of the "strict" errors handler.
     def raise_unicode_exception_encode(errors, encoding, msg, u,
                                        startingpos, endingpos):
         raise OperationError(space.w_UnicodeEncodeError,

From noreply at buildbot.pypy.org  Wed Sep 17 22:19:42 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 17 Sep 2014 22:19:42 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix copy/paste mistake.
Message-ID: <20140917201942.14B391D22E6@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: 
Changeset: r73591:568f6f6ef786
Date: 2014-09-17 22:19 +0200
http://bitbucket.org/pypy/pypy/changeset/568f6f6ef786/

Log:	Fix copy/paste mistake. Unit test will follow.

diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py
--- a/pypy/interpreter/pyparser/parsestring.py
+++ b/pypy/interpreter/pyparser/parsestring.py
@@ -227,7 +227,7 @@
 def decode_utf8_recode(space, s, ps, end, recode_encoding):
     u, ps = decode_utf8(space, s, ps, end)
     w_v = unicodehelper.encode(space, space.wrap(u), recode_encoding)
-    v = space.bytes_w(w_v)
+    v = space.str_w(w_v)
     return v, ps
 
 def raise_app_valueerror(space, msg):

From noreply at buildbot.pypy.org  Wed Sep 17 22:31:44 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 17 Sep 2014 22:31:44 +0200 (CEST)
Subject: [pypy-commit] pypy default: Increase code coverage,
 would have caught the previous mistake.
Message-ID: <20140917203144.8F70C1D22E6@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: 
Changeset: r73592:31228b608991
Date: 2014-09-17 22:31 +0200
http://bitbucket.org/pypy/pypy/changeset/31228b608991/

Log:	Increase code coverage, would have caught the previous mistake.

diff --git a/pypy/interpreter/pyparser/test/test_parsestring.py b/pypy/interpreter/pyparser/test/test_parsestring.py
--- a/pypy/interpreter/pyparser/test/test_parsestring.py
+++ b/pypy/interpreter/pyparser/test/test_parsestring.py
@@ -73,11 +73,11 @@
 
     def test_simple_enc_roundtrip(self):
         space = self.space
-        s = "'\x81'"
+        s = "'\x81\\t'"
         s = s.decode("koi8-u").encode("utf8")
         w_ret = parsestring.parsestr(self.space, 'koi8-u', s)
         ret = space.unwrap(w_ret)
-        assert ret == eval("# -*- coding: koi8-u -*-\n'\x81'") 
+        assert ret == eval("# -*- coding: koi8-u -*-\n'\x81\\t'") 
 
     def test_multiline_unicode_strings_with_backslash(self):
         space = self.space

From noreply at buildbot.pypy.org  Thu Sep 18 00:05:06 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 18 Sep 2014 00:05:06 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: Python 3.3 acknowledged that
 sys.defaultencoding is utf-8 and will never change.
Message-ID: <20140917220506.33D3C1C023E@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: py3.3
Changeset: r73593:be8eea97342c
Date: 2014-09-17 20:24 +0200
http://bitbucket.org/pypy/pypy/changeset/be8eea97342c/

Log:	Python 3.3 acknowledged that sys.defaultencoding is utf-8 and will
	never change.

	Simplify encode_object(), this also removes an assertion which
	prevented module initialization from calling compile() (used by
	namedtuple for example)

diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -7,7 +7,7 @@
 
 class Module(MixedModule):
     """Sys Builtin Module. """
-    _immutable_fields_ = ["defaultencoding?", "debug?"]
+    _immutable_fields_ = ["defaultencoding", "debug?"]
 
     def __init__(self, space, w_name):
         """NOT_RPYTHON""" # because parent __init__ isn't
@@ -15,7 +15,6 @@
             del self.__class__.interpleveldefs['pypy_getudir']
         super(Module, self).__init__(space, w_name)
         self.recursionlimit = 100
-        self.w_default_encoder = None
         self.defaultencoding = "utf-8"
         self.filesystemencoding = None
         self.debug = True
@@ -199,16 +198,6 @@
                 return space.wrap(operror.get_traceback())
         return None
 
-    def get_w_default_encoder(self):
-        if self.w_default_encoder is not None:
-            # XXX is this level of caching ok?  CPython has some shortcuts
-            # for common encodings, but as far as I can see it has no general
-            # cache.
-            return self.w_default_encoder
-        else:
-            from pypy.module.sys.interp_encoding import get_w_default_encoder
-            return get_w_default_encoder(self.space)
-
     def get_flag(self, name):
         space = self.space
         return space.int_w(space.getattr(self.get('flags'), space.wrap(name)))
diff --git a/pypy/module/sys/interp_encoding.py b/pypy/module/sys/interp_encoding.py
--- a/pypy/module/sys/interp_encoding.py
+++ b/pypy/module/sys/interp_encoding.py
@@ -7,17 +7,6 @@
 implementation."""
     return space.wrap(space.sys.defaultencoding)
 
-def get_w_default_encoder(space):
-    assert not (space.config.translating and not we_are_translated()), \
-        "get_w_default_encoder() should not be called during translation"
-    w_encoding = space.wrap(space.sys.defaultencoding)
-    mod = space.getbuiltinmodule("_codecs")
-    w_lookup = space.getattr(mod, space.wrap("lookup"))
-    w_functuple = space.call_function(w_lookup, w_encoding)
-    w_encoder = space.getitem(w_functuple, space.wrap(0))
-    space.sys.w_default_encoder = w_encoder    # cache it
-    return w_encoder
-
 if sys.platform == "win32":
     base_encoding = "mbcs"
 elif sys.platform == "darwin":
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -481,24 +481,20 @@
 
 
 def encode_object(space, w_object, encoding, errors):
-    if encoding is None:
-        # Get the encoder functions as a wrapped object.
-        # This lookup is cached.
-        w_encoder = space.sys.get_w_default_encoder()
-    else:
-        if errors is None or errors == 'strict':
-            if encoding == 'ascii':
-                u = space.unicode_w(w_object)
-                eh = unicodehelper.encode_error_handler(space)
-                return space.wrapbytes(unicode_encode_ascii(
-                        u, len(u), None, errorhandler=eh))
-            if encoding == 'utf-8':
-                u = space.unicode_w(w_object)
-                eh = unicodehelper.encode_error_handler(space)
-                return space.wrapbytes(unicode_encode_utf_8(
-                        u, len(u), None, errorhandler=eh))
-        from pypy.module._codecs.interp_codecs import lookup_codec
-        w_encoder = space.getitem(lookup_codec(space, encoding), space.wrap(0))
+    if errors is None or errors == 'strict':
+        if encoding is None or encoding == 'utf-8':
+            u = space.unicode_w(w_object)
+            eh = unicodehelper.encode_error_handler(space)
+            return space.wrapbytes(unicode_encode_utf_8(
+                    u, len(u), errors, errorhandler=eh))
+        elif encoding == 'ascii':
+            u = space.unicode_w(w_object)
+            eh = unicodehelper.encode_error_handler(space)
+            return space.wrapbytes(unicode_encode_ascii(
+                    u, len(u), errors, errorhandler=eh))
+
+    from pypy.module._codecs.interp_codecs import lookup_codec
+    w_encoder = space.getitem(lookup_codec(space, encoding), space.wrap(0))
     if errors is None:
         w_errors = space.wrap('strict')
     else:

From noreply at buildbot.pypy.org  Thu Sep 18 00:05:07 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 18 Sep 2014 00:05:07 +0200 (CEST)
Subject: [pypy-commit] pypy decimal-libmpdec: Translation fixes.
Message-ID: <20140917220507.A81A71C023E@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: decimal-libmpdec
Changeset: r73594:237e55507f16
Date: 2014-09-16 11:15 +0200
http://bitbucket.org/pypy/pypy/changeset/237e55507f16/

Log:	Translation fixes.

diff --git a/pypy/module/_decimal/interp_context.py b/pypy/module/_decimal/interp_context.py
--- a/pypy/module/_decimal/interp_context.py
+++ b/pypy/module/_decimal/interp_context.py
@@ -1,6 +1,7 @@
 from rpython.rlib import rmpdec
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.tool.sourcetools import func_renamer
 from pypy.interpreter.error import oefmt, OperationError
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.gateway import interp2app, unwrap_spec
@@ -288,9 +289,10 @@
     W_Context.__init__(w_result, space)
     return w_result
 
-def make_unary_method(mpd_func_name):
+def make_unary_method(mpd_func_name, tag=''):
     mpd_func = getattr(rmpdec, mpd_func_name)
     @unwrap_spec(w_context=W_Context)
+    @func_renamer('descr_%s%s' % (mpd_func_name, tag))
     def func_w(space, w_context, w_x):
         from pypy.module._decimal import interp_decimal
         w_a = interp_decimal.convert_op_raise(space, w_context, w_x)
@@ -303,6 +305,7 @@
 def make_binary_method(mpd_func_name):
     mpd_func = getattr(rmpdec, mpd_func_name)
     @unwrap_spec(w_context=W_Context)
+    @func_renamer('descr_%s' % mpd_func_name)
     def func_w(space, w_context, w_x, w_y):
         from pypy.module._decimal import interp_decimal
         w_a, w_b = interp_decimal.convert_binop_raise(
@@ -316,6 +319,7 @@
 def make_bool_method(mpd_func_name):
     mpd_func = getattr(rmpdec, mpd_func_name)
     @unwrap_spec(w_context=W_Context)
+    @func_renamer('descr_%s' % mpd_func_name)
     def func_w(space, w_context, w_x):
         from pypy.module._decimal import interp_decimal
         w_x = interp_decimal.convert_op_raise(space, w_context, w_x)
@@ -326,6 +330,7 @@
 def make_bool_method_noctx(mpd_func_name):
     mpd_func = getattr(rmpdec, mpd_func_name)
     @unwrap_spec(w_context=W_Context)
+    @func_renamer('descr_%s' % mpd_func_name)
     def func_w(space, w_context, w_x):
         from pypy.module._decimal import interp_decimal
         w_x = interp_decimal.convert_op_raise(space, w_context, w_x)
@@ -366,7 +371,7 @@
     plus=make_unary_method('mpd_qplus'),
     to_integral=make_unary_method('mpd_qround_to_int'),
     to_integral_exact=make_unary_method('mpd_qround_to_intx'),
-    to_integral_value=make_unary_method('mpd_qround_to_int'),
+    to_integral_value=make_unary_method('mpd_qround_to_int', tag='value'),
     sqrt=make_unary_method('mpd_qsqrt'),
     logical_invert=make_unary_method('mpd_qinvert'),
     # Binary Operations
diff --git a/pypy/module/_decimal/interp_decimal.py b/pypy/module/_decimal/interp_decimal.py
--- a/pypy/module/_decimal/interp_decimal.py
+++ b/pypy/module/_decimal/interp_decimal.py
@@ -2,6 +2,7 @@
 from rpython.rlib.objectmodel import specialize
 from rpython.rlib.rstring import StringBuilder
 from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.tool.sourcetools import func_renamer
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import oefmt, OperationError
 from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
@@ -519,6 +520,7 @@
 
 def make_unary_number_method(mpd_func_name):
     mpd_func = getattr(rmpdec, mpd_func_name)
+    @func_renamer('descr_%s' % mpd_func_name)
     def descr_method(space, w_self):
         self = space.interp_w(W_Decimal, w_self)
         context = interp_context.getcontext(space)
@@ -541,12 +543,14 @@
 
 def make_binary_number_method(mpd_func_name):
     mpd_func = getattr(rmpdec, mpd_func_name)
+    @func_renamer('descr_%s' % mpd_func_name)
     def descr_method(space, w_self, w_other):
         return binary_number_method(space, mpd_func, w_self, w_other)
     return interp2app(descr_method)
 
 def make_binary_number_method_right(mpd_func_name):
     mpd_func = getattr(rmpdec, mpd_func_name)
+    @func_renamer('descr_r_%s' % mpd_func_name)
     def descr_method(space, w_self, w_other):
         return binary_number_method(space, mpd_func, w_other, w_self)
     return interp2app(descr_method)

From noreply at buildbot.pypy.org  Thu Sep 18 00:05:08 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 18 Sep 2014 00:05:08 +0200 (CEST)
Subject: [pypy-commit] pypy decimal-libmpdec: Implement comparisons with
 Rational and Fractions.
Message-ID: <20140917220508.E9C151C023E@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: decimal-libmpdec
Changeset: r73595:b252f18bbe7e
Date: 2014-09-16 16:01 +0200
http://bitbucket.org/pypy/pypy/changeset/b252f18bbe7e/

Log:	Implement comparisons with Rational and Fractions.

diff --git a/pypy/module/_decimal/__init__.py b/pypy/module/_decimal/__init__.py
--- a/pypy/module/_decimal/__init__.py
+++ b/pypy/module/_decimal/__init__.py
@@ -30,7 +30,5 @@
     for name, flag in interp_signals.SIGNAL_MAP:
         interpleveldefs[name] = 'interp_signals.get(space).w_%s' % name
     for name, flag in interp_signals.COND_MAP:
-        if name == 'InvalidOperation':
-            pass
         interpleveldefs[name] = 'interp_signals.get(space).w_%s' % name
         
diff --git a/pypy/module/_decimal/interp_decimal.py b/pypy/module/_decimal/interp_decimal.py
--- a/pypy/module/_decimal/interp_decimal.py
+++ b/pypy/module/_decimal/interp_decimal.py
@@ -30,6 +30,15 @@
 # DEC_MINALLOC >= MPD_MINALLOC
 DEC_MINALLOC = 4
 
+class State:
+    def __init__(self, space):
+        w_import = space.builtin.get('__import__')
+        w_numbers = space.call_function(w_import,
+                                        space.wrap('numbers'))
+        self.w_Rational = space.getattr(w_numbers,
+                                        space.wrap('Rational'))
+
+
 class W_Decimal(W_Root):
     hash = -1
 
@@ -483,6 +492,46 @@
                     space.type(w_y))
     return w_a, w_b
 
+# Convert rationals for comparison
+def _multiply_by_denominator(space, w_v, w_r, context):
+    # w_v is not special, w_r is a rational.
+    w_denom = space.getattr(w_r, space.wrap("denominator"))
+    denom = decimal_from_bigint(space, None, space.bigint_w(w_denom),
+                                context, exact=True)
+    vv = rmpdec.mpd_qncopy(w_v.mpd)
+    if not vv:
+        raise OperationError(space.w_MemoryError, space.w_None)
+    try:
+        result = W_Decimal(space)
+        with lltype.scoped_alloc(rmpdec.MPD_CONTEXT_PTR.TO) as maxctx:
+            rmpdec.mpd_maxcontext(maxctx)
+            # Prevent Overflow in the following multiplication. The
+            # result of the multiplication is only used in mpd_qcmp,
+            # which can handle values that are technically out of
+            # bounds, like (for 32-bit)
+            # 99999999999999999999...99999999e+425000000.
+            exp = vv.c_exp
+            vv.c_exp = 0
+            with lltype.scoped_alloc(rffi.CArrayPtr(rffi.UINT).TO, 1,
+                                     zero=True) as status_ptr:
+                rmpdec.mpd_qmul(result.mpd, vv, denom.mpd, maxctx, status_ptr)
+                # If any status has been accumulated during the multiplication,
+                # the result is invalid. This is very unlikely, since even the
+                # 32-bit version supports 425000000 digits.
+                if status_ptr[0]:
+                    raise oefmt(space.w_ValueError,
+                                "exact conversion for comparison failed")
+            result.mpd.c_exp = exp
+    finally:
+        rmpdec.mpd_del(vv)
+
+    return space.wrap(result)
+
+def _numerator_as_decimal(space, w_r, context):
+    w_numerator = space.getattr(w_r, space.wrap("numerator"))
+    return decimal_from_bigint(space, None, space.bigint_w(w_numerator),
+                               context, exact=True)
+
 def convert_binop_cmp(space, context, op, w_v, w_w):
     if isinstance(w_w, W_Decimal):
         return None, w_v, w_w
@@ -513,6 +562,11 @@
             w_w = decimal_from_float(space, None, w_w, context, exact=True)
         else:
             return space.w_NotImplemented, None, None
+    elif space.isinstance_w(w_w, space.fromcache(State).w_Rational):
+        w_numerator = _numerator_as_decimal(space, w_w, context)
+        if not rmpdec.mpd_isspecial(w_v.mpd):
+            w_v = _multiply_by_denominator(space, w_v, w_w, context)
+        w_w = w_numerator
     else:
         return space.w_NotImplemented, None, None
     return None, w_v, w_w
diff --git a/pypy/module/_decimal/test/test_decimal.py b/pypy/module/_decimal/test/test_decimal.py
--- a/pypy/module/_decimal/test/test_decimal.py
+++ b/pypy/module/_decimal/test/test_decimal.py
@@ -855,6 +855,49 @@
             doit(c, signal=FloatOperation)
             test_containers(c, signal=FloatOperation)
 
+    def test_decimal_fraction_comparison(self):
+        C = self.decimal
+        D = self.decimal.Decimal
+        from fractions import Fraction as F
+        Context = self.decimal.Context
+        localcontext = self.decimal.localcontext
+        InvalidOperation = self.decimal.InvalidOperation
+
+
+        emax = C.MAX_EMAX
+        emin = C.MIN_EMIN
+        etiny = C.MIN_ETINY
+        c = Context(Emax=emax, Emin=emin)
+
+        with localcontext(c):
+            c.prec = emax
+            assert D(0) < F(1,9999999999999999999999999999999999999)
+            assert F(-1,9999999999999999999999999999999999999) < D(0)
+            assert F(0,1) < D("1e" + str(etiny))
+            assert D("-1e" + str(etiny)) < F(0,1)
+            assert F(0,9999999999999999999999999) < D("1e" + str(etiny))
+            assert D("-1e" + str(etiny)) < F(0,9999999999999999999999999)
+
+            assert D("0.1") == F(1,10)
+            assert F(1,10) == D("0.1")
+
+            c.prec = 300
+            assert D(1)/3 != F(1,3)
+            assert F(1,3) != D(1)/3
+
+            assert F(120984237, 9999999999) <= D("9e" + str(emax))
+            assert D("9e" + str(emax)) >= F(120984237, 9999999999)
+
+            assert D('inf') > F(99999999999,123)
+            assert D('inf') > F(-99999999999,123)
+            assert D('-inf') < F(99999999999,123)
+            assert D('-inf') < F(-99999999999,123)
+
+            raises(InvalidOperation, D('nan').__gt__, F(-9,123))
+            assert F(-9,123).__lt__(D('nan')) is NotImplemented
+            assert D('nan') != F(-9,123)
+            assert F(-9,123) != D('nan')
+
     def test_nan_comparisons(self):
         import operator
         # comparisons involving signaling nans signal InvalidOperation

From noreply at buildbot.pypy.org  Thu Sep 18 00:05:10 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 18 Sep 2014 00:05:10 +0200 (CEST)
Subject: [pypy-commit] pypy decimal-libmpdec: One more test,
	many methods to add.
Message-ID: <20140917220510.44F111C023E@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: decimal-libmpdec
Changeset: r73596:c48bbeeb5798
Date: 2014-09-17 22:06 +0200
http://bitbucket.org/pypy/pypy/changeset/c48bbeeb5798/

Log:	One more test, many methods to add.

diff --git a/pypy/module/_decimal/interp_decimal.py b/pypy/module/_decimal/interp_decimal.py
--- a/pypy/module/_decimal/interp_decimal.py
+++ b/pypy/module/_decimal/interp_decimal.py
@@ -360,6 +360,26 @@
     def descr_rpow(self, space, w_other):
         return W_Decimal.pow_impl(space, w_other, self, None)
 
+    def copy_abs_w(self, space):
+        w_result = W_Decimal.allocate(space)
+        with lltype.scoped_alloc(rffi.CArrayPtr(rffi.UINT).TO, 1,
+                                 zero=True) as status_ptr:
+            rmpdec.mpd_qcopy_abs(w_result.mpd, self.mpd, status_ptr)
+            status = rffi.cast(lltype.Signed, status_ptr[0])
+        if status & rmpdec.MPD_Malloc_error:
+            raise OperationError(space.w_MemoryError, space.w_None)
+        return w_result
+
+    def copy_negate_w(self, space):
+        w_result = W_Decimal.allocate(space)
+        with lltype.scoped_alloc(rffi.CArrayPtr(rffi.UINT).TO, 1,
+                                 zero=True) as status_ptr:
+            rmpdec.mpd_qcopy_negate(w_result.mpd, self.mpd, status_ptr)
+            status = rffi.cast(lltype.Signed, status_ptr[0])
+        if status & rmpdec.MPD_Malloc_error:
+            raise OperationError(space.w_MemoryError, space.w_None)
+        return w_result
+
     def copy_sign_w(self, space, w_other, w_context=None):
         context = convert_context(space, w_context)
         w_other = convert_op_raise(space, context, w_other)
@@ -371,6 +391,11 @@
 
     # Unary arithmetic functions, optional context arg
 
+    def number_class_w(self, space, w_context=None):
+        context = interp_context.ensure_context(space, w_context)
+        cp = rmpdec.mpd_class(self.mpd, context.ctx)
+        return space.wrap(rffi.charp2str(cp))
+
     def to_integral_w(self, space, w_rounding=None, w_context=None):
         context = interp_context.ensure_context(space, w_context)
         w_workctx = context.copy_w(space)
@@ -396,13 +421,42 @@
             rmpdec.mpd_qround_to_intx(w_result.mpd, self.mpd,
                                       w_workctx.ctx, status_ptr)
         return w_result
+
+    # Ternary arithmetic functions, optional context arg
+    def fma_w(self, space, w_other, w_third, w_context=None):
+        context = interp_context.ensure_context(space, w_context)
+        w_a, w_b, w_c = convert_ternop_raise(space, context,
+                                             self, w_other, w_third)
+        w_result = W_Decimal.allocate(space)
+        with context.catch_status(space) as (ctx, status_ptr):
+            rmpdec.mpd_qfma(w_result.mpd, w_a.mpd, w_b.mpd, w_c.mpd,
+                            ctx, status_ptr)
+        return w_result
         
-
     # Boolean functions
+    def is_canonical_w(self, space):
+        return space.wrap(bool(rmpdec.mpd_iscanonical(self.mpd)))
+    def is_nan_w(self, space):
+        return space.wrap(bool(rmpdec.mpd_isnan(self.mpd)))
     def is_qnan_w(self, space):
         return space.wrap(bool(rmpdec.mpd_isqnan(self.mpd)))
+    def is_snan_w(self, space):
+        return space.wrap(bool(rmpdec.mpd_issnan(self.mpd)))
     def is_infinite_w(self, space):
         return space.wrap(bool(rmpdec.mpd_isinfinite(self.mpd)))
+    def is_finite_w(self, space):
+        return space.wrap(bool(rmpdec.mpd_isfinite(self.mpd)))
+    def is_signed_w(self, space):
+        return space.wrap(bool(rmpdec.mpd_issigned(self.mpd)))
+    def is_zero_w(self, space):
+        return space.wrap(bool(rmpdec.mpd_iszero(self.mpd)))
+    # Boolean functions, optional context arg
+    def is_normal_w(self, space, w_context=None):
+        context = interp_context.ensure_context(space, w_context)
+        return space.wrap(bool(rmpdec.mpd_isnormal(self.mpd, context.ctx)))
+    def is_subnormal_w(self, space, w_context=None):
+        context = interp_context.ensure_context(space, w_context)
+        return space.wrap(bool(rmpdec.mpd_issubnormal(self.mpd, context.ctx)))
 
     def as_tuple_w(self, space):
         "Return the DecimalTuple representation of a Decimal"
@@ -492,6 +546,24 @@
                     space.type(w_y))
     return w_a, w_b
 
+def convert_ternop_raise(space, context, w_x, w_y, w_z):
+    w_err, w_a = convert_op(space, context, w_x)
+    if w_err:
+        raise oefmt(space.w_TypeError,
+                    "conversion from %N to Decimal is not supported",
+                    space.type(w_x))
+    w_err, w_b = convert_op(space, context, w_y)
+    if w_err:
+        raise oefmt(space.w_TypeError,
+                    "conversion from %N to Decimal is not supported",
+                    space.type(w_y))
+    w_err, w_c = convert_op(space, context, w_z)
+    if w_err:
+        raise oefmt(space.w_TypeError,
+                    "conversion from %N to Decimal is not supported",
+                    space.type(w_z))
+    return w_a, w_b, w_c
+
 # Convert rationals for comparison
 def _multiply_by_denominator(space, w_v, w_r, context):
     # w_v is not special, w_r is a rational.
@@ -502,7 +574,7 @@
     if not vv:
         raise OperationError(space.w_MemoryError, space.w_None)
     try:
-        result = W_Decimal(space)
+        w_result = W_Decimal.allocate(space)
         with lltype.scoped_alloc(rmpdec.MPD_CONTEXT_PTR.TO) as maxctx:
             rmpdec.mpd_maxcontext(maxctx)
             # Prevent Overflow in the following multiplication. The
@@ -514,18 +586,19 @@
             vv.c_exp = 0
             with lltype.scoped_alloc(rffi.CArrayPtr(rffi.UINT).TO, 1,
                                      zero=True) as status_ptr:
-                rmpdec.mpd_qmul(result.mpd, vv, denom.mpd, maxctx, status_ptr)
+                rmpdec.mpd_qmul(w_result.mpd, vv, denom.mpd, maxctx, status_ptr)
                 # If any status has been accumulated during the multiplication,
                 # the result is invalid. This is very unlikely, since even the
                 # 32-bit version supports 425000000 digits.
-                if status_ptr[0]:
-                    raise oefmt(space.w_ValueError,
-                                "exact conversion for comparison failed")
-            result.mpd.c_exp = exp
+                status = rffi.cast(lltype.Signed, status_ptr[0])
+            if status:
+                raise oefmt(space.w_ValueError,
+                            "exact conversion for comparison failed")
+            w_result.mpd.c_exp = exp
     finally:
         rmpdec.mpd_del(vv)
 
-    return space.wrap(result)
+    return w_result
 
 def _numerator_as_decimal(space, w_r, context):
     w_numerator = space.getattr(w_r, space.wrap("numerator"))
@@ -609,6 +682,22 @@
         return binary_number_method(space, mpd_func, w_other, w_self)
     return interp2app(descr_method)
 
+# Unary function with an optional context arg.
+def unary_method(space, mpd_func, w_self, w_context):
+    self = space.interp_w(W_Decimal, w_self)
+    context = convert_context(space, w_context)
+    w_result = W_Decimal.allocate(space)
+    with context.catch_status(space) as (ctx, status_ptr):
+        mpd_func(w_result.mpd, self.mpd, ctx, status_ptr)
+    return w_result
+    
+def make_unary_method(mpd_func_name):
+    mpd_func = getattr(rmpdec, mpd_func_name)
+    @func_renamer('descr_%s' % mpd_func_name)
+    def descr_method(space, w_self, w_context=None):
+        return unary_method(space, mpd_func, w_self, w_context)
+    return interp2app(descr_method)
+
 def convert_context(space, w_context):
     if w_context is None:
         return interp_context.getcontext(space)
@@ -891,14 +980,39 @@
     __rmod__ = make_binary_number_method_right('mpd_qrem'),
     __rdivmod__ = interp2app(W_Decimal.descr_rdivmod),
     __rpow__ = interp2app(W_Decimal.descr_rpow),
+    # Unary functions, optional context arg for conversion errors
+    copy_abs = interp2app(W_Decimal.copy_abs_w),
+    copy_negate = interp2app(W_Decimal.copy_negate_w),
     # Unary arithmetic functions, optional context arg
+    exp = make_unary_method('mpd_qexp'),
+    ln = make_unary_method('mpd_qln'),
+    log10 = make_unary_method('mpd_qlog10'),
+    logb = make_unary_method('mpd_qlogb'),
+    logical_invert = make_unary_method('mpd_qinvert'),
+    next_minus = make_unary_method('mpd_qnext_minus'),
+    next_plus = make_unary_method('mpd_qnext_plus'),
+    normalize = make_unary_method('mpd_qreduce'),
+    number_class = interp2app(W_Decimal.number_class_w),
     to_integral = interp2app(W_Decimal.to_integral_w),
+    to_integral_exact = interp2app(W_Decimal.to_integral_exact_w),
     to_integral_value = interp2app(W_Decimal.to_integral_w),
-    to_integral_exact = interp2app(W_Decimal.to_integral_exact_w),
-    #
+    sqrt = make_unary_method('mpd_qsqrt'),
+    # Ternary arithmetic functions, optional context arg
+    fma = interp2app(W_Decimal.fma_w),
+    # Boolean functions, no context arg
+    is_canonical = interp2app(W_Decimal.is_canonical_w),
+    is_nan = interp2app(W_Decimal.is_nan_w),
+    is_qnan = interp2app(W_Decimal.is_qnan_w),
+    is_snan = interp2app(W_Decimal.is_snan_w),
+    is_infinite = interp2app(W_Decimal.is_infinite_w),
+    is_finite = interp2app(W_Decimal.is_finite_w),
+    is_signed = interp2app(W_Decimal.is_signed_w),
+    is_zero = interp2app(W_Decimal.is_zero_w),
+    # Boolean functions, optional context arg
+    is_normal = interp2app(W_Decimal.is_normal_w),
+    is_subnormal = interp2app(W_Decimal.is_subnormal_w),
+    # Binary functions, optional context arg for conversion errors
     copy_sign = interp2app(W_Decimal.copy_sign_w),
-    is_qnan = interp2app(W_Decimal.is_qnan_w),
-    is_infinite = interp2app(W_Decimal.is_infinite_w),
     #
     as_tuple = interp2app(W_Decimal.as_tuple_w),
     from_float = interp2app(decimal_from_float_w, as_classmethod=True),
diff --git a/pypy/module/_decimal/test/test_context.py b/pypy/module/_decimal/test/test_context.py
--- a/pypy/module/_decimal/test/test_context.py
+++ b/pypy/module/_decimal/test/test_context.py
@@ -21,6 +21,9 @@
         def assertEqual(space, w_x, w_y):
             assert space.eq_w(w_x, w_y)
         cls.w_assertEqual = space.wrap(gateway.interp2app(assertEqual))
+        def assertIs(space, w_x, w_y):
+            assert space.is_w(w_x, w_y)
+        cls.w_assertIs = space.wrap(gateway.interp2app(assertIs))
 
         cls.w_assertRaises = space.appexec([], """(): return raises""")
 
@@ -274,3 +277,84 @@
         self.assertEqual(c.is_zero(10), d)
         self.assertRaises(TypeError, c.is_zero, '10')
 
+    def test_implicit_context(self):
+        Decimal = self.decimal.Decimal
+        localcontext = self.decimal.localcontext
+
+        with localcontext() as c:
+            c.prec = 1
+            c.Emax = 1
+            c.Emin = -1
+
+            # abs
+            self.assertEqual(abs(Decimal("-10")), 10)
+            # add
+            self.assertEqual(Decimal("7") + 1, 8)
+            # divide
+            self.assertEqual(Decimal("10") / 5, 2)
+            # divide_int
+            self.assertEqual(Decimal("10") // 7, 1)
+            # fma
+            self.assertEqual(Decimal("1.2").fma(Decimal("0.01"), 1), 1)
+            self.assertIs(Decimal("NaN").fma(7, 1).is_nan(), True)
+            # three arg power
+            self.assertEqual(pow(Decimal(10), 2, 7), 2)
+            # exp
+            self.assertEqual(Decimal("1.01").exp(), 3)
+            # is_normal
+            self.assertIs(Decimal("0.01").is_normal(), False)
+            # is_subnormal
+            self.assertIs(Decimal("0.01").is_subnormal(), True)
+            # ln
+            self.assertEqual(Decimal("20").ln(), 3)
+            # log10
+            self.assertEqual(Decimal("20").log10(), 1)
+            # logb
+            self.assertEqual(Decimal("580").logb(), 2)
+            # logical_invert
+            self.assertEqual(Decimal("10").logical_invert(), 1)
+            # minus
+            self.assertEqual(-Decimal("-10"), 10)
+            # multiply
+            self.assertEqual(Decimal("2") * 4, 8)
+            # next_minus
+            self.assertEqual(Decimal("10").next_minus(), 9)
+            # next_plus
+            self.assertEqual(Decimal("10").next_plus(), Decimal('2E+1'))
+            # normalize
+            self.assertEqual(Decimal("-10").normalize(), Decimal('-1E+1'))
+            # number_class
+            self.assertEqual(Decimal("10").number_class(), '+Normal')
+            # plus
+            self.assertEqual(+Decimal("-1"), -1)
+            # remainder
+            self.assertEqual(Decimal("10") % 7, 3)
+            # subtract
+            self.assertEqual(Decimal("10") - 7, 3)
+            # to_integral_exact
+            self.assertEqual(Decimal("1.12345").to_integral_exact(), 1)
+
+            # Boolean functions
+            self.assertTrue(Decimal("1").is_canonical())
+            self.assertTrue(Decimal("1").is_finite())
+            self.assertTrue(Decimal("1").is_finite())
+            self.assertTrue(Decimal("snan").is_snan())
+            self.assertTrue(Decimal("-1").is_signed())
+            self.assertTrue(Decimal("0").is_zero())
+            self.assertTrue(Decimal("0").is_zero())
+
+        # Copy
+        with localcontext() as c:
+            c.prec = 10000
+            x = 1228 ** 1523
+            y = -Decimal(x)
+
+            z = y.copy_abs()
+            self.assertEqual(z, x)
+
+            z = y.copy_negate()
+            self.assertEqual(z, x)
+
+            z = y.copy_sign(Decimal(1))
+            self.assertEqual(z, x)
+
diff --git a/rpython/rlib/rmpdec.py b/rpython/rlib/rmpdec.py
--- a/rpython/rlib/rmpdec.py
+++ b/rpython/rlib/rmpdec.py
@@ -41,6 +41,7 @@
         "mpd_qcopy", "mpd_qncopy", "mpd_setspecial", "mpd_clear_flags",
         "mpd_qimport_u32", "mpd_qexport_u32", "mpd_qexport_u16",
         "mpd_set_sign", "mpd_set_positive", "mpd_sign", "mpd_qfinalize",
+        "mpd_class",
         "mpd_getprec", "mpd_getemin",  "mpd_getemax", "mpd_getround", "mpd_getclamp",
         "mpd_qsetprec", "mpd_qsetemin",  "mpd_qsetemax", "mpd_qsetround", "mpd_qsetclamp",
         "mpd_maxcontext",
@@ -48,7 +49,7 @@
         "mpd_to_sci", "mpd_to_sci_size",
         "mpd_iszero", "mpd_isnegative", "mpd_issigned",
         "mpd_isfinite", "mpd_isinfinite",
-        "mpd_isnormal", "mpd_issubnormal", "mpd_isspecial",
+        "mpd_isnormal", "mpd_issubnormal", "mpd_isspecial", "mpd_iscanonical",
         "mpd_isnan", "mpd_issnan", "mpd_isqnan",
         "mpd_qcmp", "mpd_qcompare", "mpd_qcompare_signal",
         "mpd_qmin", "mpd_qmax", "mpd_qmin_mag", "mpd_qmax_mag",
@@ -58,9 +59,10 @@
         "mpd_qadd", "mpd_qsub", "mpd_qmul", "mpd_qdiv", "mpd_qdivint",
         "mpd_qrem", "mpd_qrem_near", "mpd_qdivmod", "mpd_qpow", "mpd_qpowmod", 
         "mpd_qfma",
-        "mpd_qexp", "mpd_qln", "mpd_qlog10", "mpd_qsqrt", "mpd_qinvert",
+        "mpd_qexp", "mpd_qln", "mpd_qlog10", "mpd_qlogb",
+        "mpd_qsqrt", "mpd_qinvert",
         "mpd_qand", "mpd_qor", "mpd_qxor",
-        "mpd_qcopy_sign",
+        "mpd_qcopy_sign", "mpd_qcopy_abs", "mpd_qcopy_negate",
         "mpd_qround_to_int", "mpd_qround_to_intx",
         ],
     compile_extra=compile_extra,
@@ -187,6 +189,8 @@
     'mpd_sign', [MPD_PTR], rffi.UCHAR)
 mpd_qfinalize = external(
     'mpd_qfinalize', [MPD_PTR, MPD_CONTEXT_PTR, rffi.UINTP], lltype.Void)
+mpd_class = external(
+    'mpd_class', [MPD_PTR, MPD_CONTEXT_PTR], rffi.CCHARP)
 
 # Context operations
 mpd_getprec = external(
@@ -247,6 +251,8 @@
     'mpd_issubnormal', [MPD_PTR, MPD_CONTEXT_PTR], rffi.INT)
 mpd_isspecial = external(
     'mpd_isspecial', [MPD_PTR], rffi.INT)
+mpd_iscanonical = external(
+    'mpd_iscanonical', [MPD_PTR], rffi.INT)
 mpd_isnan = external(
     'mpd_isnan', [MPD_PTR], rffi.INT)
 mpd_issnan = external(
@@ -358,6 +364,9 @@
 mpd_qlog10 = external(
     'mpd_qlog10',
     [MPD_PTR, MPD_PTR, MPD_CONTEXT_PTR, rffi.UINTP], lltype.Void)
+mpd_qlogb = external(
+    'mpd_qlogb',
+    [MPD_PTR, MPD_PTR, MPD_CONTEXT_PTR, rffi.UINTP], lltype.Void)
 mpd_qsqrt = external(
     'mpd_qsqrt',
     [MPD_PTR, MPD_PTR, MPD_CONTEXT_PTR, rffi.UINTP], lltype.Void)
@@ -369,6 +378,14 @@
     'mpd_qcopy_sign',
     [MPD_PTR, MPD_PTR, MPD_PTR, rffi.UINTP],
     lltype.Void)
+mpd_qcopy_abs = external(
+    'mpd_qcopy_abs',
+    [MPD_PTR, MPD_PTR, rffi.UINTP],
+    lltype.Void)
+mpd_qcopy_negate = external(
+    'mpd_qcopy_negate',
+    [MPD_PTR, MPD_PTR, rffi.UINTP],
+    lltype.Void)
 
 mpd_qround_to_int = external(
     'mpd_qround_to_int', [MPD_PTR, MPD_PTR, MPD_CONTEXT_PTR, rffi.UINTP],

From noreply at buildbot.pypy.org  Thu Sep 18 00:05:11 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 18 Sep 2014 00:05:11 +0200 (CEST)
Subject: [pypy-commit] pypy decimal-libmpdec: hg graft 10fd806838e2
Message-ID: <20140917220511.74AD91C023E@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: decimal-libmpdec
Changeset: r73597:819424d457d9
Date: 2014-09-17 22:33 +0200
http://bitbucket.org/pypy/pypy/changeset/819424d457d9/

Log:	hg graft 10fd806838e2

diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py
--- a/pypy/interpreter/pyparser/parsestring.py
+++ b/pypy/interpreter/pyparser/parsestring.py
@@ -82,12 +82,6 @@
     v = PyString_DecodeEscape(space, substr, 'strict', encoding)
     return space.wrapbytes(v)
 
-def hexbyte(val):
-    result = "%x" % val
-    if len(result) == 1:
-        result = "0" + result
-    return result
-
 def decode_unicode_utf8(space, s, ps, q):
     # ****The Python 2.7 version, producing UTF-32 escapes****
     # String is utf8-encoded, but 'unicode_escape' expects
@@ -107,15 +101,14 @@
                 # instead.
                 lis.append("u005c")
         if ord(s[ps]) & 0x80: # XXX inefficient
-            w, ps = decode_utf8(space, s, ps, end, "utf-32-be")
-            rn = len(w)
-            assert rn % 4 == 0
-            for i in range(0, rn, 4):
-                lis.append('\\U')
-                lis.append(hexbyte(ord(w[i])))
-                lis.append(hexbyte(ord(w[i+1])))
-                lis.append(hexbyte(ord(w[i+2])))
-                lis.append(hexbyte(ord(w[i+3])))
+            w, ps = decode_utf8(space, s, ps, end)
+            for c in w:
+                # The equivalent of %08x, which is not supported by RPython.
+                # 7 zeroes are enough for the unicode range, and the
+                # result still fits in 32-bit.
+                hexa = hex(ord(c) + 0x10000000)
+                lis.append('\\U0')
+                lis.append(hexa[3:])  # Skip 0x and the leading 1
         else:
             lis.append(s[ps])
             ps += 1
@@ -135,7 +128,7 @@
             # note that the C code has a label here.
             # the logic is the same.
             if recode_encoding and ord(s[ps]) & 0x80:
-                w, ps = decode_utf8(space, s, ps, end, recode_encoding)
+                w, ps = decode_utf8_recode(space, s, ps, end, recode_encoding)
                 # Append bytes to output buffer.
                 builder.append(w)
             else:
@@ -222,14 +215,18 @@
             ch >= 'A' and ch <= 'F')
 
 
-def decode_utf8(space, s, ps, end, encoding):
+def decode_utf8(space, s, ps, end):
     assert ps >= 0
     pt = ps
     # while (s < end && *s != '\\') s++; */ /* inefficient for u".."
     while ps < end and ord(s[ps]) & 0x80:
         ps += 1
-    w_u = space.wrap(unicodehelper.decode_utf8(space, s[pt:ps]))
-    w_v = unicodehelper.encode(space, w_u, encoding)
+    u = unicodehelper.decode_utf8(space, s[pt:ps])
+    return u, ps
+
+def decode_utf8_recode(space, s, ps, end, recode_encoding):
+    u, ps = decode_utf8(space, s, ps, end)
+    w_v = unicodehelper.encode(space, space.wrap(u), recode_encoding)
     v = space.bytes_w(w_v)
     return v, ps
 
diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -13,6 +13,7 @@
 
 @specialize.memo()
 def decode_error_handler(space):
+    # Fast version of the "strict" errors handler.
     def raise_unicode_exception_decode(errors, encoding, msg, s,
                                        startingpos, endingpos):
         raise OperationError(space.w_UnicodeDecodeError,
@@ -25,6 +26,7 @@
 
 @specialize.memo()
 def encode_error_handler(space):
+    # Fast version of the "strict" errors handler.
     def raise_unicode_exception_encode(errors, encoding, msg, u,
                                        startingpos, endingpos):
         raise OperationError(space.w_UnicodeEncodeError,

From noreply at buildbot.pypy.org  Thu Sep 18 00:05:25 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 18 Sep 2014 00:05:25 +0200 (CEST)
Subject: [pypy-commit] pypy decimal-libmpdec: hg merge py3.3
Message-ID: <20140917220525.B76C21C023E@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: decimal-libmpdec
Changeset: r73598:cda472e2b220
Date: 2014-09-17 22:47 +0200
http://bitbucket.org/pypy/pypy/changeset/cda472e2b220/

Log:	hg merge py3.3

diff too long, truncating to 2000 out of 52994 lines

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -35,280 +35,290 @@
 the beginning of each file) the files in the 'pypy' directory are each
 copyrighted by one or more of the following people and organizations:    
 
-    Armin Rigo
-    Maciej Fijalkowski
-    Carl Friedrich Bolz
-    Antonio Cuni
-    Amaury Forgeot d'Arc
-    Samuele Pedroni
-    Alex Gaynor
-    Michael Hudson
-    David Schneider
-    Matti Picus
-    Brian Kearns
-    Philip Jenvey
-    Holger Krekel
-    Christian Tismer
-    Hakan Ardo
-    Benjamin Peterson
-    Manuel Jacob
-    Anders Chrigstrom
-    Eric van Riet Paap
-    Wim Lavrijsen
-    Ronan Lamy
-    Richard Emslie
-    Alexander Schremmer
-    Dan Villiom Podlaski Christiansen
-    Lukas Diekmann
-    Sven Hager
-    Anders Lehmann
-    Aurelien Campeas
-    Niklaus Haldimann
-    Camillo Bruni
-    Laura Creighton
-    Toon Verwaest
-    Remi Meier
-    Leonardo Santagada
-    Seo Sanghyeon
-    Romain Guillebert
-    Justin Peel
-    Ronny Pfannschmidt
-    David Edelsohn
-    Anders Hammarquist
-    Jakub Gustak
-    Guido Wesdorp
-    Lawrence Oluyede
-    Bartosz Skowron
-    Daniel Roberts
-    Niko Matsakis
-    Adrien Di Mascio
-    Alexander Hesse
-    Ludovic Aubry
-    Jacob Hallen
-    Jason Creighton
-    Alex Martelli
-    Michal Bendowski
-    Jan de Mooij
-    stian
-    Michael Foord
-    Stephan Diehl
-    Stefan Schwarzer
-    Valentino Volonghi
-    Tomek Meka
-    Patrick Maupin
-    Bob Ippolito
-    Bruno Gola
-    Jean-Paul Calderone
-    Timo Paulssen
-    Squeaky
-    Alexandre Fayolle
-    Simon Burton
-    Marius Gedminas
-    John Witulski
-    Konstantin Lopuhin
-    Greg Price
-    Dario Bertini
-    Mark Pearse
-    Simon Cross
-    Andreas Stührk
-    Jean-Philippe St. Pierre
-    Guido van Rossum
-    Pavel Vinogradov
-    Paweł Piotr Przeradowski
-    Paul deGrandis
-    Ilya Osadchiy
-    Tobias Oberstein
-    Adrian Kuhn
-    Boris Feigin
-    Stefano Rivera
-    tav
-    Taavi Burns
-    Georg Brandl
-    Bert Freudenberg
-    Stian Andreassen
-    Laurence Tratt
-    Wanja Saatkamp
-    Ivan Sichmann Freitas
-    Gerald Klix
-    Mike Blume
-    Oscar Nierstrasz
-    Stefan H. Muller
-    Jeremy Thurgood
-    Gregor Wegberg
-    Rami Chowdhury
-    Tobias Pape
-    Edd Barrett
-    David Malcolm
-    Eugene Oden
-    Henry Mason
-    Preston Timmons
-    Jeff Terrace
-    David Ripton
-    Dusty Phillips
-    Lukas Renggli
-    Guenter Jantzen
-    Ned Batchelder
-    Amit Regmi
-    Ben Young
-    Nicolas Chauvat
-    Andrew Durdin
-    Andrew Chambers
-    Michael Schneider
-    Nicholas Riley
-    Jason Chu
-    Igor Trindade Oliveira
-    Rocco Moretti
-    Gintautas Miliauskas
-    Michael Twomey
-    Lucian Branescu Mihaila
-    Tim Felgentreff
-    Tyler Wade
-    Gabriel Lavoie
-    Olivier Dormond
-    Jared Grubb
-    Karl Bartel
-    Brian Dorsey
-    Victor Stinner
-    Andrews Medina
-    Stuart Williams
-    Jasper Schulz
-    Christian Hudon
-    Toby Watson
-    Antoine Pitrou
-    Aaron Iles
-    Michael Cheng
-    Justas Sadzevicius
-    Mikael Schönenberg
-    Gasper Zejn
-    Neil Shepperd
-    Elmo Mäntynen
-    Jonathan David Riehl
-    Stanislaw Halik
-    Anders Qvist
-    Chirag Jadwani
-    Beatrice During
-    Alex Perry
-    Vincent Legoll
-    Alan McIntyre
-    Alexander Sedov
-    Corbin Simpson
-    Christopher Pope
-    wenzhuman
-    Christian Tismer 
-    Marc Abramowitz
-    Dan Stromberg
-    Stefano Parmesan
-    Alexis Daboville
-    Jens-Uwe Mager
-    Carl Meyer
-    Karl Ramm
-    Pieter Zieschang
-    Gabriel
-    Lukas Vacek
-    Andrew Dalke
-    Sylvain Thenault
-    Nathan Taylor
-    Vladimir Kryachko
-    Jacek Generowicz
-    Alejandro J. Cura
-    Jacob Oscarson
-    Travis Francis Athougies
-    Ryan Gonzalez
-    Kristjan Valur Jonsson
-    Sebastian Pawluś
-    Neil Blakey-Milner
-    anatoly techtonik
-    Lutz Paelike
-    Lucio Torre
-    Lars Wassermann
-    Henrik Vendelbo
-    Dan Buch
-    Miguel de Val Borro
-    Artur Lisiecki
-    Sergey Kishchenko
-    Ignas Mikalajunas
-    Christoph Gerum
-    Martin Blais
-    Lene Wagner
-    Tomo Cocoa
-    roberto at goyle
-    Yury V. Zaytsev
-    Anna Katrina Dominguez
-    William Leslie
-    Bobby Impollonia
-    timo at eistee.fritz.box
-    Andrew Thompson
-    Ben Darnell
-    Roberto De Ioris
-    Juan Francisco Cantero Hurtado
-    Godefroid Chappelle
-    Joshua Gilbert
-    Dan Colish
-    Christopher Armstrong
-    Michael Hudson-Doyle
-    Anders Sigfridsson
-    Yasir Suhail
-    rafalgalczynski at gmail.com
-    Floris Bruynooghe
-    Laurens Van Houtven
-    Akira Li
-    Gustavo Niemeyer
-    Stephan Busemann
-    Rafał Gałczyński
-    Yusei Tahara
-    Christian Muirhead
-    James Lan
-    shoma hosaka
-    Daniel Neuh?user
-    Matthew Miller
-    Buck Golemon
-    Konrad Delong
-    Dinu Gherman
-    Chris Lambacher
-    coolbutuseless at gmail.com
-    Rodrigo Araújo
-    w31rd0
-    Jim Baker
-    James Robert
-    Armin Ronacher
-    Brett Cannon
-    yrttyr
-    aliceinwire
-    OlivierBlanvillain
-    Zooko Wilcox-O Hearn
-    Tomer Chachamu
-    Christopher Groskopf
-    Asmo Soinio
-    Stefan Marr
-    jiaaro
-    opassembler.py
-    Antony Lee
-    Jim Hunziker
-    Markus Unterwaditzer
-    Even Wiik Thomassen
-    jbs
-    soareschen
-    Kurt Griffiths
-    Mike Bayer
-    Flavio Percoco
-    Kristoffer Kleine
-    yasirs
-    Michael Chermside
-    Anna Ravencroft
-    Julien Phalip
-    Dan Loewenherz
+  Armin Rigo
+  Maciej Fijalkowski
+  Carl Friedrich Bolz
+  Antonio Cuni
+  Amaury Forgeot d'Arc
+  Samuele Pedroni
+  Alex Gaynor
+  Michael Hudson
+  David Schneider
+  Matti Picus
+  Brian Kearns
+  Philip Jenvey
+  Holger Krekel
+  Christian Tismer
+  Hakan Ardo
+  Benjamin Peterson
+  Manuel Jacob
+  Anders Chrigstrom
+  Eric van Riet Paap
+  Ronan Lamy
+  Wim Lavrijsen
+  Richard Emslie
+  Alexander Schremmer
+  Dan Villiom Podlaski Christiansen
+  Lukas Diekmann
+  Sven Hager
+  Anders Lehmann
+  Aurelien Campeas
+  Niklaus Haldimann
+  Remi Meier
+  Camillo Bruni
+  Laura Creighton
+  Toon Verwaest
+  Leonardo Santagada
+  Seo Sanghyeon
+  Romain Guillebert
+  Justin Peel
+  Ronny Pfannschmidt
+  David Edelsohn
+  Anders Hammarquist
+  Jakub Gustak
+  Guido Wesdorp
+  Lawrence Oluyede
+  Bartosz Skowron
+  Gregor Wegberg
+  Daniel Roberts
+  Niko Matsakis
+  Adrien Di Mascio
+  Alexander Hesse
+  Ludovic Aubry
+  Jacob Hallen
+  Jason Creighton
+  Alex Martelli
+  Michal Bendowski
+  Jan de Mooij
+  stian
+  Michael Foord
+  Stephan Diehl
+  Tyler Wade
+  Stefan Schwarzer
+  Valentino Volonghi
+  Tomek Meka
+  Patrick Maupin
+  Bob Ippolito
+  Bruno Gola
+  Jean-Paul Calderone
+  Timo Paulssen
+  Squeaky
+  Alexandre Fayolle
+  Simon Burton
+  Marius Gedminas
+  Martin Matusiak
+  Konstantin Lopuhin
+  John Witulski
+  Wenzhu Man
+  Greg Price
+  Dario Bertini
+  Mark Pearse
+  Simon Cross
+  Ivan Sichmann Freitas
+  Andreas Stührk
+  Jean-Philippe St. Pierre
+  Guido van Rossum
+  Pavel Vinogradov
+  Stefano Rivera
+  Paweł Piotr Przeradowski
+  Paul deGrandis
+  Ilya Osadchiy
+  Tobias Oberstein
+  Adrian Kuhn
+  Boris Feigin
+  tav
+  Taavi Burns
+  Georg Brandl
+  Laurence Tratt
+  Bert Freudenberg
+  Stian Andreassen
+  Wanja Saatkamp
+  Gerald Klix
+  Mike Blume
+  Oscar Nierstrasz
+  Stefan H. Muller
+  Edd Barrett
+  Jeremy Thurgood
+  Rami Chowdhury
+  Tobias Pape
+  David Malcolm
+  Eugene Oden
+  Henry Mason
+  Vasily Kuznetsov
+  Preston Timmons
+  Jeff Terrace
+  David Ripton
+  Dusty Phillips
+  Lukas Renggli
+  Guenter Jantzen
+  Ned Batchelder
+  Amit Regmi
+  Ben Young
+  Nicolas Chauvat
+  Andrew Durdin
+  Andrew Chambers
+  Michael Schneider
+  Nicholas Riley
+  Jason Chu
+  Igor Trindade Oliveira
+  Tim Felgentreff
+  Rocco Moretti
+  Gintautas Miliauskas
+  Michael Twomey
+  Lucian Branescu Mihaila
+  Gabriel Lavoie
+  Olivier Dormond
+  Jared Grubb
+  Karl Bartel
+  Brian Dorsey
+  Victor Stinner
+  Andrews Medina
+  Stuart Williams
+  Jasper Schulz
+  Christian Hudon
+  Toby Watson
+  Antoine Pitrou
+  Aaron Iles
+  Michael Cheng
+  Justas Sadzevicius
+  Gasper Zejn
+  anatoly techtonik
+  Neil Shepperd
+  Mikael Schönenberg
+  Elmo M?ntynen
+  Jonathan David Riehl
+  Stanislaw Halik
+  Anders Qvist
+  Corbin Simpson
+  Chirag Jadwani
+  Beatrice During
+  Alex Perry
+  Vincent Legoll
+  Alan McIntyre
+  Alexander Sedov
+  Christopher Pope
+  Christian Tismer 
+  Marc Abramowitz
+  Dan Stromberg
+  Stefano Parmesan
+  Alexis Daboville
+  Jens-Uwe Mager
+  Carl Meyer
+  Karl Ramm
+  Pieter Zieschang
+  Sebastian Pawluś
+  Gabriel
+  Lukas Vacek
+  Andrew Dalke
+  Sylvain Thenault
+  Nathan Taylor
+  Vladimir Kryachko
+  Arjun Naik
+  Attila Gobi
+  Jacek Generowicz
+  Alejandro J. Cura
+  Jacob Oscarson
+  Travis Francis Athougies
+  Ryan Gonzalez
+  Ian Foote
+  Kristjan Valur Jonsson
+  Neil Blakey-Milner
+  Lutz Paelike
+  Lucio Torre
+  Lars Wassermann
+  Valentina Mukhamedzhanova
+  Henrik Vendelbo
+  Dan Buch
+  Miguel de Val Borro
+  Artur Lisiecki
+  Sergey Kishchenko
+  Yichao Yu
+  Ignas Mikalajunas
+  Christoph Gerum
+  Martin Blais
+  Lene Wagner
+  Tomo Cocoa
+  roberto at goyle
+  Yury V. Zaytsev
+  Anna Katrina Dominguez
+  William Leslie
+  Bobby Impollonia
+  timo at eistee.fritz.box
+  Andrew Thompson
+  Yusei Tahara
+  Ben Darnell
+  Roberto De Ioris
+  Juan Francisco Cantero Hurtado
+  Godefroid Chappelle
+  Joshua Gilbert
+  Dan Colish
+  Christopher Armstrong
+  Michael Hudson-Doyle
+  Anders Sigfridsson
+  Yasir Suhail
+  Jason Michalski
+  rafalgalczynski at gmail.com
+  Floris Bruynooghe
+  Laurens Van Houtven
+  Akira Li
+  Gustavo Niemeyer
+  Stephan Busemann
+  Rafał Gałczyński
+  Christian Muirhead
+  James Lan
+  shoma hosaka
+  Daniel Neuh?user
+  Matthew Miller
+  Buck Golemon
+  Konrad Delong
+  Dinu Gherman
+  Chris Lambacher
+  coolbutuseless at gmail.com
+  Rodrigo Araújo
+  Jim Baker
+  James Robert
+  Armin Ronacher
+  Brett Cannon
+  yrttyr
+  aliceinwire
+  OlivierBlanvillain
+  Zooko Wilcox-O Hearn
+  Tomer Chachamu
+  Christopher Groskopf
+  Asmo Soinio
+  Stefan Marr
+  jiaaro
+  Mads Kiilerich
+  opassembler.py
+  Antony Lee
+  Jim Hunziker
+  Markus Unterwaditzer
+  Even Wiik Thomassen
+  jbs
+  soareschen
+  Kurt Griffiths
+  Mike Bayer
+  Matthew Miller
+  Flavio Percoco
+  Kristoffer Kleine
+  yasirs
+  Michael Chermside
+  Anna Ravencroft
+  Dan Crosta
+  Julien Phalip
+  Dan Loewenherz
 
-    Heinrich-Heine University, Germany 
-    Open End AB (formerly AB Strakt), Sweden
-    merlinux GmbH, Germany 
-    tismerysoft GmbH, Germany 
-    Logilab Paris, France 
-    DFKI GmbH, Germany 
-    Impara, Germany
-    Change Maker, Sweden 
-    University of California Berkeley, USA
-    Google Inc.
-    King's College London
+  Heinrich-Heine University, Germany 
+  Open End AB (formerly AB Strakt), Sweden
+  merlinux GmbH, Germany 
+  tismerysoft GmbH, Germany 
+  Logilab Paris, France 
+  DFKI GmbH, Germany 
+  Impara, Germany
+  Change Maker, Sweden 
+  University of California Berkeley, USA
+  Google Inc.
+  King's College London
 
 The PyPy Logo as used by http://speed.pypy.org and others was created
 by Samuel Reis and is distributed on terms of Creative Commons Share Alike
@@ -354,6 +364,6 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 
-Detailled license information is contained in the NOTICE file in the
+Detailed license information is contained in the NOTICE file in the
 directory.
 
diff --git a/_pytest/README-BEFORE-UPDATING b/_pytest/README-BEFORE-UPDATING
new file mode 100644
--- /dev/null
+++ b/_pytest/README-BEFORE-UPDATING
@@ -0,0 +1,17 @@
+This is PyPy's code of the pytest lib.  We don't expect to upgrade it
+very often, but once we do:
+
+    WARNING!
+
+    WE HAVE MADE A FEW TWEAKS HERE!
+
+Please be sure that you don't just copy the newer version from
+upstream without checking the few changes that we did.  This
+can be done like this:
+
+    cd 
+    hg log . -v | less
+
+then search for all " _pytest/" in that list to know which are the
+relevant checkins.  (Look for the checkins that only edit one
+or two files in this directory.)
diff --git a/_pytest/resultlog.py b/_pytest/resultlog.py
--- a/_pytest/resultlog.py
+++ b/_pytest/resultlog.py
@@ -53,16 +53,24 @@
         self.config = config
         self.logfile = logfile # preferably line buffered
 
-    def write_log_entry(self, testpath, lettercode, longrepr):
-        py.builtin.print_("%s %s" % (lettercode, testpath), file=self.logfile)
+    def write_log_entry(self, testpath, lettercode, longrepr, sections=None):
+        _safeprint("%s %s" % (lettercode, testpath), file=self.logfile)
         for line in longrepr.splitlines():
-            py.builtin.print_(" %s" % line, file=self.logfile)
+            _safeprint(" %s" % line, file=self.logfile)
+        if sections is not None and (
+                lettercode in ('E', 'F')):    # to limit the size of logs
+            for title, content in sections:
+                _safeprint(" ---------- %s ----------" % (title,),
+                           file=self.logfile)
+                for line in content.splitlines():
+                    _safeprint(" %s" % line, file=self.logfile)
 
     def log_outcome(self, report, lettercode, longrepr):
         testpath = getattr(report, 'nodeid', None)
         if testpath is None:
             testpath = report.fspath
-        self.write_log_entry(testpath, lettercode, longrepr)
+        self.write_log_entry(testpath, lettercode, longrepr,
+                             getattr(report, 'sections', None))
 
     def pytest_runtest_logreport(self, report):
         if report.when != "call" and report.passed:
@@ -98,3 +106,8 @@
         if path is None:
             path = "cwd:%s" % py.path.local()
         self.write_log_entry(path, '!', str(excrepr))
+
+def _safeprint(s, file):
+    if isinstance(s, unicode):
+        s = s.encode('utf-8')
+    py.builtin.print_(s, file=file)
diff --git a/lib-python/2.7/CGIHTTPServer.py b/lib-python/2.7/CGIHTTPServer.py
--- a/lib-python/2.7/CGIHTTPServer.py
+++ b/lib-python/2.7/CGIHTTPServer.py
@@ -84,7 +84,7 @@
         path begins with one of the strings in self.cgi_directories
         (and the next character is a '/' or the end of the string).
         """
-        collapsed_path = _url_collapse_path(self.path)
+        collapsed_path = _url_collapse_path(urllib.unquote(self.path))
         dir_sep = collapsed_path.find('/', 1)
         head, tail = collapsed_path[:dir_sep], collapsed_path[dir_sep+1:]
         if head in self.cgi_directories:
diff --git a/lib-python/2.7/Cookie.py b/lib-python/2.7/Cookie.py
--- a/lib-python/2.7/Cookie.py
+++ b/lib-python/2.7/Cookie.py
@@ -1,6 +1,3 @@
-#!/usr/bin/env python
-#
-
 ####
 # Copyright 2000 by Timothy O'Malley 
 #
diff --git a/lib-python/2.7/HTMLParser.py b/lib-python/2.7/HTMLParser.py
--- a/lib-python/2.7/HTMLParser.py
+++ b/lib-python/2.7/HTMLParser.py
@@ -22,9 +22,12 @@
 starttagopen = re.compile('<[a-zA-Z]')
 piclose = re.compile('>')
 commentclose = re.compile(r'--\s*>')
-tagfind = re.compile('([a-zA-Z][-.a-zA-Z0-9:_]*)(?:\s|/(?!>))*')
+
 # see http://www.w3.org/TR/html5/tokenization.html#tag-open-state
 # and http://www.w3.org/TR/html5/tokenization.html#tag-name-state
+# note: if you change tagfind/attrfind remember to update locatestarttagend too
+tagfind = re.compile('([a-zA-Z][^\t\n\r\f />\x00]*)(?:\s|/(?!>))*')
+# this regex is currently unused, but left for backward compatibility
 tagfind_tolerant = re.compile('[a-zA-Z][^\t\n\r\f />\x00]*')
 
 attrfind = re.compile(
@@ -32,7 +35,7 @@
     r'(\'[^\']*\'|"[^"]*"|(?![\'"])[^>\s]*))?(?:\s|/(?!>))*')
 
 locatestarttagend = re.compile(r"""
-  <[a-zA-Z][-.a-zA-Z0-9:_]*          # tag name
+  <[a-zA-Z][^\t\n\r\f />\x00]*       # tag name
   (?:[\s/]*                          # optional whitespace before attribute name
     (?:(?<=['"\s/])[^\s/>][^\s/=>]*  # attribute name
       (?:\s*=+\s*                    # value indicator
@@ -192,9 +195,9 @@
                     i = self.updatepos(i, k)
                     continue
                 else:
-                    if ";" in rawdata[i:]: #bail by consuming &#
-                        self.handle_data(rawdata[0:2])
-                        i = self.updatepos(i, 2)
+                    if ";" in rawdata[i:]:  # bail by consuming '&#'
+                        self.handle_data(rawdata[i:i+2])
+                        i = self.updatepos(i, i+2)
                     break
             elif startswith('&', i):
                 match = entityref.match(rawdata, i)
@@ -373,14 +376,14 @@
                 self.handle_data(rawdata[i:gtpos])
                 return gtpos
             # find the name: w3.org/TR/html5/tokenization.html#tag-name-state
-            namematch = tagfind_tolerant.match(rawdata, i+2)
+            namematch = tagfind.match(rawdata, i+2)
             if not namematch:
                 # w3.org/TR/html5/tokenization.html#end-tag-open-state
                 if rawdata[i:i+3] == '':
                     return i+3
                 else:
                     return self.parse_bogus_comment(i)
-            tagname = namematch.group().lower()
+            tagname = namematch.group(1).lower()
             # consume and ignore other stuff between the name and the >
             # Note: this is not 100% correct, since we might have things like
             # , but looking for > after tha name should cover
diff --git a/lib-python/2.7/SimpleHTTPServer.py b/lib-python/2.7/SimpleHTTPServer.py
--- a/lib-python/2.7/SimpleHTTPServer.py
+++ b/lib-python/2.7/SimpleHTTPServer.py
@@ -43,8 +43,10 @@
         """Serve a GET request."""
         f = self.send_head()
         if f:
-            self.copyfile(f, self.wfile)
-            f.close()
+            try:
+                self.copyfile(f, self.wfile)
+            finally:
+                f.close()
 
     def do_HEAD(self):
         """Serve a HEAD request."""
@@ -88,13 +90,17 @@
         except IOError:
             self.send_error(404, "File not found")
             return None
-        self.send_response(200)
-        self.send_header("Content-type", ctype)
-        fs = os.fstat(f.fileno())
-        self.send_header("Content-Length", str(fs[6]))
-        self.send_header("Last-Modified", self.date_time_string(fs.st_mtime))
-        self.end_headers()
-        return f
+        try:
+            self.send_response(200)
+            self.send_header("Content-type", ctype)
+            fs = os.fstat(f.fileno())
+            self.send_header("Content-Length", str(fs[6]))
+            self.send_header("Last-Modified", self.date_time_string(fs.st_mtime))
+            self.end_headers()
+            return f
+        except:
+            f.close()
+            raise
 
     def list_directory(self, path):
         """Helper to produce a directory listing (absent index.html).
diff --git a/lib-python/2.7/SimpleXMLRPCServer.py b/lib-python/2.7/SimpleXMLRPCServer.py
--- a/lib-python/2.7/SimpleXMLRPCServer.py
+++ b/lib-python/2.7/SimpleXMLRPCServer.py
@@ -704,4 +704,5 @@
     server = SimpleXMLRPCServer(("localhost", 8000))
     server.register_function(pow)
     server.register_function(lambda x,y: x+y, 'add')
+    server.register_multicall_functions()
     server.serve_forever()
diff --git a/lib-python/2.7/SocketServer.py b/lib-python/2.7/SocketServer.py
--- a/lib-python/2.7/SocketServer.py
+++ b/lib-python/2.7/SocketServer.py
@@ -513,35 +513,37 @@
 
     def collect_children(self):
         """Internal routine to wait for children that have exited."""
-        if self.active_children is None: return
+        if self.active_children is None:
+            return
+
+        # If we're above the max number of children, wait and reap them until
+        # we go back below threshold. Note that we use waitpid(-1) below to be
+        # able to collect children in size() syscalls instead
+        # of size(): the downside is that this might reap children
+        # which we didn't spawn, which is why we only resort to this when we're
+        # above max_children.
         while len(self.active_children) >= self.max_children:
-            # XXX: This will wait for any child process, not just ones
-            # spawned by this library. This could confuse other
-            # libraries that expect to be able to wait for their own
-            # children.
             try:
-                pid, status = os.waitpid(0, 0)
-            except os.error:
-                pid = None
-            if pid not in self.active_children: continue
-            self.active_children.remove(pid)
+                pid, _ = os.waitpid(-1, 0)
+                self.active_children.discard(pid)
+            except OSError as e:
+                if e.errno == errno.ECHILD:
+                    # we don't have any children, we're done
+                    self.active_children.clear()
+                elif e.errno != errno.EINTR:
+                    break
 
-        # XXX: This loop runs more system calls than it ought
-        # to. There should be a way to put the active_children into a
-        # process group and then use os.waitpid(-pgid) to wait for any
-        # of that set, but I couldn't find a way to allocate pgids
-        # that couldn't collide.
-        for child in self.active_children:
+        # Now reap all defunct children.
+        for pid in self.active_children.copy():
             try:
-                pid, status = os.waitpid(child, os.WNOHANG)
-            except os.error:
-                pid = None
-            if not pid: continue
-            try:
-                self.active_children.remove(pid)
-            except ValueError, e:
-                raise ValueError('%s. x=%d and list=%r' % (e.message, pid,
-                                                           self.active_children))
+                pid, _ = os.waitpid(pid, os.WNOHANG)
+                # if the child hasn't exited yet, pid will be 0 and ignored by
+                # discard() below
+                self.active_children.discard(pid)
+            except OSError as e:
+                if e.errno == errno.ECHILD:
+                    # someone else reaped it
+                    self.active_children.discard(pid)
 
     def handle_timeout(self):
         """Wait for zombies after self.timeout seconds of inactivity.
@@ -557,8 +559,8 @@
         if pid:
             # Parent process
             if self.active_children is None:
-                self.active_children = []
-            self.active_children.append(pid)
+                self.active_children = set()
+            self.active_children.add(pid)
             self.close_request(request) #close handle in parent process
             return
         else:
diff --git a/lib-python/2.7/_MozillaCookieJar.py b/lib-python/2.7/_MozillaCookieJar.py
--- a/lib-python/2.7/_MozillaCookieJar.py
+++ b/lib-python/2.7/_MozillaCookieJar.py
@@ -39,7 +39,7 @@
     magic_re = "#( Netscape)? HTTP Cookie File"
     header = """\
 # Netscape HTTP Cookie File
-# http://www.netscape.com/newsref/std/cookie_spec.html
+# http://curl.haxx.se/rfc/cookie_spec.html
 # This is a generated file!  Do not edit.
 
 """
diff --git a/lib-python/2.7/_abcoll.py b/lib-python/2.7/_abcoll.py
--- a/lib-python/2.7/_abcoll.py
+++ b/lib-python/2.7/_abcoll.py
@@ -165,12 +165,17 @@
     def __gt__(self, other):
         if not isinstance(other, Set):
             return NotImplemented
-        return other < self
+        return len(self) > len(other) and self.__ge__(other)
 
     def __ge__(self, other):
         if not isinstance(other, Set):
             return NotImplemented
-        return other <= self
+        if len(self) < len(other):
+            return False
+        for elem in other:
+            if elem not in self:
+                return False
+        return True
 
     def __eq__(self, other):
         if not isinstance(other, Set):
@@ -194,6 +199,8 @@
             return NotImplemented
         return self._from_iterable(value for value in other if value in self)
 
+    __rand__ = __and__
+
     def isdisjoint(self, other):
         'Return True if two sets have a null intersection.'
         for value in other:
@@ -207,6 +214,8 @@
         chain = (e for s in (self, other) for e in s)
         return self._from_iterable(chain)
 
+    __ror__ = __or__
+
     def __sub__(self, other):
         if not isinstance(other, Set):
             if not isinstance(other, Iterable):
@@ -215,6 +224,14 @@
         return self._from_iterable(value for value in self
                                    if value not in other)
 
+    def __rsub__(self, other):
+        if not isinstance(other, Set):
+            if not isinstance(other, Iterable):
+                return NotImplemented
+            other = self._from_iterable(other)
+        return self._from_iterable(value for value in other
+                                   if value not in self)
+
     def __xor__(self, other):
         if not isinstance(other, Set):
             if not isinstance(other, Iterable):
@@ -222,6 +239,8 @@
             other = self._from_iterable(other)
         return (self - other) | (other - self)
 
+    __rxor__ = __xor__
+
     # Sets are not hashable by default, but subclasses can change this
     __hash__ = None
 
diff --git a/lib-python/2.7/_osx_support.py b/lib-python/2.7/_osx_support.py
--- a/lib-python/2.7/_osx_support.py
+++ b/lib-python/2.7/_osx_support.py
@@ -182,7 +182,7 @@
         # Compiler is GCC, check if it is LLVM-GCC
         data = _read_output("'%s' --version"
                              % (cc.replace("'", "'\"'\"'"),))
-        if 'llvm-gcc' in data:
+        if data and 'llvm-gcc' in data:
             # Found LLVM-GCC, fall back to clang
             cc = _find_build_tool('clang')
 
@@ -450,8 +450,16 @@
         # case and disallow installs.
         cflags = _config_vars.get(_INITPRE+'CFLAGS',
                                     _config_vars.get('CFLAGS', ''))
-        if ((macrelease + '.') >= '10.4.' and
-            '-arch' in cflags.strip()):
+        if macrelease:
+            try:
+                macrelease = tuple(int(i) for i in macrelease.split('.')[0:2])
+            except ValueError:
+                macrelease = (10, 0)
+        else:
+            # assume no universal support
+            macrelease = (10, 0)
+
+        if (macrelease >= (10, 4)) and '-arch' in cflags.strip():
             # The universal build will build fat binaries, but not on
             # systems before 10.4
 
diff --git a/lib-python/2.7/_pyio.py b/lib-python/2.7/_pyio.py
--- a/lib-python/2.7/_pyio.py
+++ b/lib-python/2.7/_pyio.py
@@ -192,38 +192,45 @@
                  (appending and "a" or "") +
                  (updating and "+" or ""),
                  closefd)
-    line_buffering = False
-    if buffering == 1 or buffering < 0 and raw.isatty():
-        buffering = -1
-        line_buffering = True
-    if buffering < 0:
-        buffering = DEFAULT_BUFFER_SIZE
-        try:
-            bs = os.fstat(raw.fileno()).st_blksize
-        except (os.error, AttributeError):
-            pass
+    result = raw
+    try:
+        line_buffering = False
+        if buffering == 1 or buffering < 0 and raw.isatty():
+            buffering = -1
+            line_buffering = True
+        if buffering < 0:
+            buffering = DEFAULT_BUFFER_SIZE
+            try:
+                bs = os.fstat(raw.fileno()).st_blksize
+            except (os.error, AttributeError):
+                pass
+            else:
+                if bs > 1:
+                    buffering = bs
+        if buffering < 0:
+            raise ValueError("invalid buffering size")
+        if buffering == 0:
+            if binary:
+                return result
+            raise ValueError("can't have unbuffered text I/O")
+        if updating:
+            buffer = BufferedRandom(raw, buffering)
+        elif writing or appending:
+            buffer = BufferedWriter(raw, buffering)
+        elif reading:
+            buffer = BufferedReader(raw, buffering)
         else:
-            if bs > 1:
-                buffering = bs
-    if buffering < 0:
-        raise ValueError("invalid buffering size")
-    if buffering == 0:
+            raise ValueError("unknown mode: %r" % mode)
+        result = buffer
         if binary:
-            return raw
-        raise ValueError("can't have unbuffered text I/O")
-    if updating:
-        buffer = BufferedRandom(raw, buffering)
-    elif writing or appending:
-        buffer = BufferedWriter(raw, buffering)
-    elif reading:
-        buffer = BufferedReader(raw, buffering)
-    else:
-        raise ValueError("unknown mode: %r" % mode)
-    if binary:
-        return buffer
-    text = TextIOWrapper(buffer, encoding, errors, newline, line_buffering)
-    text.mode = mode
-    return text
+            return result
+        text = TextIOWrapper(buffer, encoding, errors, newline, line_buffering)
+        result = text
+        text.mode = mode
+        return result
+    except:
+        result.close()
+        raise
 
 
 class DocDescriptor:
@@ -1997,7 +2004,13 @@
 
     def getvalue(self):
         self.flush()
-        return self.buffer.getvalue().decode(self._encoding, self._errors)
+        decoder = self._decoder or self._get_decoder()
+        old_state = decoder.getstate()
+        decoder.reset()
+        try:
+            return decoder.decode(self.buffer.getvalue(), final=True)
+        finally:
+            decoder.setstate(old_state)
 
     def __repr__(self):
         # TextIOWrapper tells the encoding in its repr. In StringIO,
diff --git a/lib-python/2.7/_weakrefset.py b/lib-python/2.7/_weakrefset.py
--- a/lib-python/2.7/_weakrefset.py
+++ b/lib-python/2.7/_weakrefset.py
@@ -60,6 +60,8 @@
             for itemref in self.data:
                 item = itemref()
                 if item is not None:
+                    # Caveat: the iterator will keep a strong reference to
+                    # `item` until it is resumed or closed.
                     yield item
 
     def __len__(self):
diff --git a/lib-python/2.7/aifc.py b/lib-python/2.7/aifc.py
--- a/lib-python/2.7/aifc.py
+++ b/lib-python/2.7/aifc.py
@@ -778,7 +778,7 @@
 
     def _ensure_header_written(self, datasize):
         if not self._nframeswritten:
-            if self._comptype in ('ULAW', 'ALAW'):
+            if self._comptype in ('ULAW', 'ulaw', 'ALAW', 'alaw'):
                 if not self._sampwidth:
                     self._sampwidth = 2
                 if self._sampwidth != 2:
@@ -844,7 +844,7 @@
         if self._datalength & 1:
             self._datalength = self._datalength + 1
         if self._aifc:
-            if self._comptype in ('ULAW', 'ALAW'):
+            if self._comptype in ('ULAW', 'ulaw', 'ALAW', 'alaw'):
                 self._datalength = self._datalength // 2
                 if self._datalength & 1:
                     self._datalength = self._datalength + 1
@@ -852,7 +852,10 @@
                 self._datalength = (self._datalength + 3) // 4
                 if self._datalength & 1:
                     self._datalength = self._datalength + 1
-        self._form_length_pos = self._file.tell()
+        try:
+            self._form_length_pos = self._file.tell()
+        except (AttributeError, IOError):
+            self._form_length_pos = None
         commlength = self._write_form_length(self._datalength)
         if self._aifc:
             self._file.write('AIFC')
@@ -864,7 +867,8 @@
         self._file.write('COMM')
         _write_ulong(self._file, commlength)
         _write_short(self._file, self._nchannels)
-        self._nframes_pos = self._file.tell()
+        if self._form_length_pos is not None:
+            self._nframes_pos = self._file.tell()
         _write_ulong(self._file, self._nframes)
         if self._comptype in ('ULAW', 'ulaw', 'ALAW', 'alaw', 'G722'):
             _write_short(self._file, 8)
@@ -875,7 +879,8 @@
             self._file.write(self._comptype)
             _write_string(self._file, self._compname)
         self._file.write('SSND')
-        self._ssnd_length_pos = self._file.tell()
+        if self._form_length_pos is not None:
+            self._ssnd_length_pos = self._file.tell()
         _write_ulong(self._file, self._datalength + 8)
         _write_ulong(self._file, 0)
         _write_ulong(self._file, 0)
diff --git a/lib-python/2.7/argparse.py b/lib-python/2.7/argparse.py
--- a/lib-python/2.7/argparse.py
+++ b/lib-python/2.7/argparse.py
@@ -168,6 +168,8 @@
         self._prog = prog
         self._indent_increment = indent_increment
         self._max_help_position = max_help_position
+        self._max_help_position = min(max_help_position,
+                                      max(width - 20, indent_increment * 2))
         self._width = width
 
         self._current_indent = 0
@@ -339,7 +341,7 @@
                     else:
                         line_len = len(indent) - 1
                     for part in parts:
-                        if line_len + 1 + len(part) > text_width:
+                        if line_len + 1 + len(part) > text_width and line:
                             lines.append(indent + ' '.join(line))
                             line = []
                             line_len = len(indent) - 1
@@ -478,7 +480,7 @@
     def _format_text(self, text):
         if '%(prog)' in text:
             text = text % dict(prog=self._prog)
-        text_width = self._width - self._current_indent
+        text_width = max(self._width - self._current_indent, 11)
         indent = ' ' * self._current_indent
         return self._fill_text(text, text_width, indent) + '\n\n'
 
@@ -486,7 +488,7 @@
         # determine the required width and the entry label
         help_position = min(self._action_max_length + 2,
                             self._max_help_position)
-        help_width = self._width - help_position
+        help_width = max(self._width - help_position, 11)
         action_width = help_position - self._current_indent - 2
         action_header = self._format_action_invocation(action)
 
@@ -1155,9 +1157,13 @@
     __hash__ = None
 
     def __eq__(self, other):
+        if not isinstance(other, Namespace):
+            return NotImplemented
         return vars(self) == vars(other)
 
     def __ne__(self, other):
+        if not isinstance(other, Namespace):
+            return NotImplemented
         return not (self == other)
 
     def __contains__(self, key):
diff --git a/lib-python/2.7/bsddb/dbshelve.py b/lib-python/2.7/bsddb/dbshelve.py
--- a/lib-python/2.7/bsddb/dbshelve.py
+++ b/lib-python/2.7/bsddb/dbshelve.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 #------------------------------------------------------------------------
 #           Copyright (c) 1997-2001 by Total Control Software
 #                         All Rights Reserved
diff --git a/lib-python/2.7/bsddb/test/test_dbtables.py b/lib-python/2.7/bsddb/test/test_dbtables.py
--- a/lib-python/2.7/bsddb/test/test_dbtables.py
+++ b/lib-python/2.7/bsddb/test/test_dbtables.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-#
 #-----------------------------------------------------------------------
 # A test suite for the table interface built on bsddb.db
 #-----------------------------------------------------------------------
diff --git a/lib-python/2.7/codecs.py b/lib-python/2.7/codecs.py
--- a/lib-python/2.7/codecs.py
+++ b/lib-python/2.7/codecs.py
@@ -456,15 +456,12 @@
 
         # read until we get the required number of characters (if available)
         while True:
-            # can the request can be satisfied from the character buffer?
-            if chars < 0:
-                if size < 0:
-                    if self.charbuffer:
-                        break
-                elif len(self.charbuffer) >= size:
+            # can the request be satisfied from the character buffer?
+            if chars >= 0:
+                if len(self.charbuffer) >= chars:
                     break
-            else:
-                if len(self.charbuffer) >= chars:
+            elif size >= 0:
+                if len(self.charbuffer) >= size:
                     break
             # we need more data
             if size < 0:
diff --git a/lib-python/2.7/collections.py b/lib-python/2.7/collections.py
--- a/lib-python/2.7/collections.py
+++ b/lib-python/2.7/collections.py
@@ -319,6 +319,7 @@
     if isinstance(field_names, basestring):
         field_names = field_names.replace(',', ' ').split()
     field_names = map(str, field_names)
+    typename = str(typename)
     if rename:
         seen = set()
         for index, name in enumerate(field_names):
@@ -331,6 +332,8 @@
                 field_names[index] = '_%d' % index
             seen.add(name)
     for name in [typename] + field_names:
+        if type(name) != str:
+            raise TypeError('Type names and field names must be strings')
         if not all(c.isalnum() or c=='_' for c in name):
             raise ValueError('Type names and field names can only contain '
                              'alphanumeric characters and underscores: %r' % name)
diff --git a/lib-python/2.7/csv.py b/lib-python/2.7/csv.py
--- a/lib-python/2.7/csv.py
+++ b/lib-python/2.7/csv.py
@@ -93,6 +93,10 @@
         self.line_num = self.reader.line_num
         return self._fieldnames
 
+    # Issue 20004: Because DictReader is a classic class, this setter is
+    # ignored.  At this point in 2.7's lifecycle, it is too late to change the
+    # base class for fear of breaking working code.  If you want to change
+    # fieldnames without overwriting the getter, set _fieldnames directly.
     @fieldnames.setter
     def fieldnames(self, value):
         self._fieldnames = value
@@ -140,8 +144,8 @@
         if self.extrasaction == "raise":
             wrong_fields = [k for k in rowdict if k not in self.fieldnames]
             if wrong_fields:
-                raise ValueError("dict contains fields not in fieldnames: " +
-                                 ", ".join(wrong_fields))
+                raise ValueError("dict contains fields not in fieldnames: "
+                                 + ", ".join([repr(x) for x in wrong_fields]))
         return [rowdict.get(key, self.restval) for key in self.fieldnames]
 
     def writerow(self, rowdict):
diff --git a/lib-python/2.7/ctypes/test/__init__.py b/lib-python/2.7/ctypes/test/__init__.py
--- a/lib-python/2.7/ctypes/test/__init__.py
+++ b/lib-python/2.7/ctypes/test/__init__.py
@@ -2,7 +2,15 @@
 
 use_resources = []
 
-class ResourceDenied(Exception):
+import ctypes
+ctypes_symbols = dir(ctypes)
+
+def need_symbol(name):
+    return unittest.skipUnless(name in ctypes_symbols,
+                               '{!r} is required'.format(name))
+
+
+class ResourceDenied(unittest.SkipTest):
     """Test skipped because it requested a disallowed resource.
 
     This is raised when a test calls requires() for a resource that
diff --git a/lib-python/2.7/ctypes/test/test_arrays.py b/lib-python/2.7/ctypes/test/test_arrays.py
--- a/lib-python/2.7/ctypes/test/test_arrays.py
+++ b/lib-python/2.7/ctypes/test/test_arrays.py
@@ -2,6 +2,8 @@
 from ctypes import *
 from test.test_support import impl_detail
 
+from ctypes.test import need_symbol
+
 formats = "bBhHiIlLqQfd"
 
 # c_longdouble commented out for PyPy, look at the commend in test_longdouble
@@ -98,8 +100,8 @@
         self.assertEqual(values, [1, 2, 3, 4, 5])
 
     def test_classcache(self):
-        self.assertTrue(not ARRAY(c_int, 3) is ARRAY(c_int, 4))
-        self.assertTrue(ARRAY(c_int, 3) is ARRAY(c_int, 3))
+        self.assertIsNot(ARRAY(c_int, 3), ARRAY(c_int, 4))
+        self.assertIs(ARRAY(c_int, 3), ARRAY(c_int, 3))
 
     def test_from_address(self):
         # Failed with 0.9.8, reported by JUrner
@@ -112,20 +114,16 @@
         self.assertEqual(sz[1:4:2], "o")
         self.assertEqual(sz.value, "foo")
 
-    try:
-        create_unicode_buffer
-    except NameError:
-        pass
-    else:
-        def test_from_addressW(self):
-            p = create_unicode_buffer("foo")
-            sz = (c_wchar * 3).from_address(addressof(p))
-            self.assertEqual(sz[:], "foo")
-            self.assertEqual(sz[::], "foo")
-            self.assertEqual(sz[::-1], "oof")
-            self.assertEqual(sz[::3], "f")
-            self.assertEqual(sz[1:4:2], "o")
-            self.assertEqual(sz.value, "foo")
+    @need_symbol('create_unicode_buffer')
+    def test_from_addressW(self):
+        p = create_unicode_buffer("foo")
+        sz = (c_wchar * 3).from_address(addressof(p))
+        self.assertEqual(sz[:], "foo")
+        self.assertEqual(sz[::], "foo")
+        self.assertEqual(sz[::-1], "oof")
+        self.assertEqual(sz[::3], "f")
+        self.assertEqual(sz[1:4:2], "o")
+        self.assertEqual(sz.value, "foo")
 
     def test_cache(self):
         # Array types are cached internally in the _ctypes extension,
@@ -139,7 +137,7 @@
         # Create a new array type based on it:
         t1 = my_int * 1
         t2 = my_int * 1
-        self.assertTrue(t1 is t2)
+        self.assertIs(t1, t2)
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_as_parameter.py b/lib-python/2.7/ctypes/test/test_as_parameter.py
--- a/lib-python/2.7/ctypes/test/test_as_parameter.py
+++ b/lib-python/2.7/ctypes/test/test_as_parameter.py
@@ -1,5 +1,6 @@
 import unittest
 from ctypes import *
+from ctypes.test import need_symbol
 import _ctypes_test
 
 dll = CDLL(_ctypes_test.__file__)
@@ -17,11 +18,8 @@
     def wrap(self, param):
         return param
 
+    @need_symbol('c_wchar')
     def test_wchar_parm(self):
-        try:
-            c_wchar
-        except NameError:
-            return
         f = dll._testfunc_i_bhilfd
         f.argtypes = [c_byte, c_wchar, c_int, c_long, c_float, c_double]
         result = f(self.wrap(1), self.wrap(u"x"), self.wrap(3), self.wrap(4), self.wrap(5.0), self.wrap(6.0))
@@ -134,7 +132,7 @@
         f.argtypes = [c_longlong, MyCallback]
 
         def callback(value):
-            self.assertTrue(isinstance(value, (int, long)))
+            self.assertIsInstance(value, (int, long))
             return value & 0x7FFFFFFF
 
         cb = MyCallback(callback)
diff --git a/lib-python/2.7/ctypes/test/test_bitfields.py b/lib-python/2.7/ctypes/test/test_bitfields.py
--- a/lib-python/2.7/ctypes/test/test_bitfields.py
+++ b/lib-python/2.7/ctypes/test/test_bitfields.py
@@ -1,4 +1,5 @@
 from ctypes import *
+from ctypes.test import need_symbol
 import unittest
 import os
 
@@ -131,15 +132,6 @@
         self.assertEqual(result[0], TypeError)
         self.assertIn('bit fields not allowed for type', result[1])
 
-        try:
-            c_wchar
-        except NameError:
-            pass
-        else:
-            result = self.fail_fields(("a", c_wchar, 1))
-            self.assertEqual(result[0], TypeError)
-            self.assertIn('bit fields not allowed for type', result[1])
-
         class Dummy(Structure):
             _fields_ = []
 
@@ -147,6 +139,12 @@
         self.assertEqual(result[0], TypeError)
         self.assertIn('bit fields not allowed for type', result[1])
 
+    @need_symbol('c_wchar')
+    def test_c_wchar(self):
+        result = self.fail_fields(("a", c_wchar, 1))
+        self.assertEqual(result,
+                (TypeError, 'bit fields not allowed for type c_wchar'))
+
     def test_single_bitfield_size(self):
         for c_typ in int_types:
             result = self.fail_fields(("a", c_typ, -1))
@@ -213,7 +211,7 @@
         class X(Structure):
             _fields_ = [("a", c_byte, 4),
                         ("b", c_int, 32)]
-        self.assertEqual(sizeof(X), sizeof(c_int)*2)
+        self.assertEqual(sizeof(X), alignment(c_int)+sizeof(c_int))
 
     def test_mixed_3(self):
         class X(Structure):
@@ -246,7 +244,7 @@
             _anonymous_ = ["_"]
             _fields_ = [("_", X)]
 
-    @unittest.skipUnless(hasattr(ctypes, "c_uint32"), "c_int32 is required")
+    @need_symbol('c_uint32')
     def test_uint32(self):
         class X(Structure):
             _fields_ = [("a", c_uint32, 32)]
@@ -256,7 +254,7 @@
         x.a = 0xFDCBA987
         self.assertEqual(x.a, 0xFDCBA987)
 
-    @unittest.skipUnless(hasattr(ctypes, "c_uint64"), "c_int64 is required")
+    @need_symbol('c_uint64')
     def test_uint64(self):
         class X(Structure):
             _fields_ = [("a", c_uint64, 64)]
diff --git a/lib-python/2.7/ctypes/test/test_buffers.py b/lib-python/2.7/ctypes/test/test_buffers.py
--- a/lib-python/2.7/ctypes/test/test_buffers.py
+++ b/lib-python/2.7/ctypes/test/test_buffers.py
@@ -1,4 +1,5 @@
 from ctypes import *
+from ctypes.test import need_symbol
 import unittest
 
 class StringBufferTestCase(unittest.TestCase):
@@ -7,12 +8,12 @@
         b = create_string_buffer(32)
         self.assertEqual(len(b), 32)
         self.assertEqual(sizeof(b), 32 * sizeof(c_char))
-        self.assertTrue(type(b[0]) is str)
+        self.assertIs(type(b[0]), str)
 
         b = create_string_buffer("abc")
         self.assertEqual(len(b), 4) # trailing nul char
         self.assertEqual(sizeof(b), 4 * sizeof(c_char))
-        self.assertTrue(type(b[0]) is str)
+        self.assertIs(type(b[0]), str)
         self.assertEqual(b[0], "a")
         self.assertEqual(b[:], "abc\0")
         self.assertEqual(b[::], "abc\0")
@@ -36,39 +37,36 @@
         self.assertEqual(b[::2], "ac")
         self.assertEqual(b[::5], "a")
 
-    try:
-        c_wchar
-    except NameError:
-        pass
-    else:
-        def test_unicode_buffer(self):
-            b = create_unicode_buffer(32)
-            self.assertEqual(len(b), 32)
-            self.assertEqual(sizeof(b), 32 * sizeof(c_wchar))
-            self.assertTrue(type(b[0]) is unicode)
+    @need_symbol('c_wchar')
+    def test_unicode_buffer(self):
+        b = create_unicode_buffer(32)
+        self.assertEqual(len(b), 32)
+        self.assertEqual(sizeof(b), 32 * sizeof(c_wchar))
+        self.assertIs(type(b[0]), unicode)
 
-            b = create_unicode_buffer(u"abc")
-            self.assertEqual(len(b), 4) # trailing nul char
-            self.assertEqual(sizeof(b), 4 * sizeof(c_wchar))
-            self.assertTrue(type(b[0]) is unicode)
-            self.assertEqual(b[0], u"a")
-            self.assertEqual(b[:], "abc\0")
-            self.assertEqual(b[::], "abc\0")
-            self.assertEqual(b[::-1], "\0cba")
-            self.assertEqual(b[::2], "ac")
-            self.assertEqual(b[::5], "a")
+        b = create_unicode_buffer(u"abc")
+        self.assertEqual(len(b), 4) # trailing nul char
+        self.assertEqual(sizeof(b), 4 * sizeof(c_wchar))
+        self.assertIs(type(b[0]), unicode)
+        self.assertEqual(b[0], u"a")
+        self.assertEqual(b[:], "abc\0")
+        self.assertEqual(b[::], "abc\0")
+        self.assertEqual(b[::-1], "\0cba")
+        self.assertEqual(b[::2], "ac")
+        self.assertEqual(b[::5], "a")
 
-        def test_unicode_conversion(self):
-            b = create_unicode_buffer("abc")
-            self.assertEqual(len(b), 4) # trailing nul char
-            self.assertEqual(sizeof(b), 4 * sizeof(c_wchar))
-            self.assertTrue(type(b[0]) is unicode)
-            self.assertEqual(b[0], u"a")
-            self.assertEqual(b[:], "abc\0")
-            self.assertEqual(b[::], "abc\0")
-            self.assertEqual(b[::-1], "\0cba")
-            self.assertEqual(b[::2], "ac")
-            self.assertEqual(b[::5], "a")
+    @need_symbol('c_wchar')
+    def test_unicode_conversion(self):
+        b = create_unicode_buffer("abc")
+        self.assertEqual(len(b), 4) # trailing nul char
+        self.assertEqual(sizeof(b), 4 * sizeof(c_wchar))
+        self.assertIs(type(b[0]), unicode)
+        self.assertEqual(b[0], u"a")
+        self.assertEqual(b[:], "abc\0")
+        self.assertEqual(b[::], "abc\0")
+        self.assertEqual(b[::-1], "\0cba")
+        self.assertEqual(b[::2], "ac")
+        self.assertEqual(b[::5], "a")
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_byteswap.py b/lib-python/2.7/ctypes/test/test_byteswap.py
--- a/lib-python/2.7/ctypes/test/test_byteswap.py
+++ b/lib-python/2.7/ctypes/test/test_byteswap.py
@@ -15,7 +15,8 @@
 # For Structures and Unions, these types are created on demand.
 
 class Test(unittest.TestCase):
-    def X_test(self):
+    @unittest.skip('test disabled')
+    def test_X(self):
         print >> sys.stderr,  sys.byteorder
         for i in range(32):
             bits = BITS()
@@ -25,11 +26,11 @@
     @xfail
     def test_endian_short(self):
         if sys.byteorder == "little":
-            self.assertTrue(c_short.__ctype_le__ is c_short)
-            self.assertTrue(c_short.__ctype_be__.__ctype_le__ is c_short)
+            self.assertIs(c_short.__ctype_le__, c_short)
+            self.assertIs(c_short.__ctype_be__.__ctype_le__, c_short)
         else:
-            self.assertTrue(c_short.__ctype_be__ is c_short)
-            self.assertTrue(c_short.__ctype_le__.__ctype_be__ is c_short)
+            self.assertIs(c_short.__ctype_be__, c_short)
+            self.assertIs(c_short.__ctype_le__.__ctype_be__, c_short)
         s = c_short.__ctype_be__(0x1234)
         self.assertEqual(bin(struct.pack(">h", 0x1234)), "1234")
         self.assertEqual(bin(s), "1234")
@@ -53,11 +54,11 @@
     @xfail
     def test_endian_int(self):
         if sys.byteorder == "little":
-            self.assertTrue(c_int.__ctype_le__ is c_int)
-            self.assertTrue(c_int.__ctype_be__.__ctype_le__ is c_int)
+            self.assertIs(c_int.__ctype_le__, c_int)
+            self.assertIs(c_int.__ctype_be__.__ctype_le__, c_int)
         else:
-            self.assertTrue(c_int.__ctype_be__ is c_int)
-            self.assertTrue(c_int.__ctype_le__.__ctype_be__ is c_int)
+            self.assertIs(c_int.__ctype_be__, c_int)
+            self.assertIs(c_int.__ctype_le__.__ctype_be__, c_int)
 
         s = c_int.__ctype_be__(0x12345678)
         self.assertEqual(bin(struct.pack(">i", 0x12345678)), "12345678")
@@ -82,11 +83,11 @@
     @xfail
     def test_endian_longlong(self):
         if sys.byteorder == "little":
-            self.assertTrue(c_longlong.__ctype_le__ is c_longlong)
-            self.assertTrue(c_longlong.__ctype_be__.__ctype_le__ is c_longlong)
+            self.assertIs(c_longlong.__ctype_le__, c_longlong)
+            self.assertIs(c_longlong.__ctype_be__.__ctype_le__, c_longlong)
         else:
-            self.assertTrue(c_longlong.__ctype_be__ is c_longlong)
-            self.assertTrue(c_longlong.__ctype_le__.__ctype_be__ is c_longlong)
+            self.assertIs(c_longlong.__ctype_be__, c_longlong)
+            self.assertIs(c_longlong.__ctype_le__.__ctype_be__, c_longlong)
 
         s = c_longlong.__ctype_be__(0x1234567890ABCDEF)
         self.assertEqual(bin(struct.pack(">q", 0x1234567890ABCDEF)), "1234567890ABCDEF")
@@ -111,11 +112,11 @@
     @xfail
     def test_endian_float(self):
         if sys.byteorder == "little":
-            self.assertTrue(c_float.__ctype_le__ is c_float)
-            self.assertTrue(c_float.__ctype_be__.__ctype_le__ is c_float)
+            self.assertIs(c_float.__ctype_le__, c_float)
+            self.assertIs(c_float.__ctype_be__.__ctype_le__, c_float)
         else:
-            self.assertTrue(c_float.__ctype_be__ is c_float)
-            self.assertTrue(c_float.__ctype_le__.__ctype_be__ is c_float)
+            self.assertIs(c_float.__ctype_be__, c_float)
+            self.assertIs(c_float.__ctype_le__.__ctype_be__, c_float)
         s = c_float(math.pi)
         self.assertEqual(bin(struct.pack("f", math.pi)), bin(s))
         # Hm, what's the precision of a float compared to a double?
@@ -130,11 +131,11 @@
     @xfail
     def test_endian_double(self):
         if sys.byteorder == "little":
-            self.assertTrue(c_double.__ctype_le__ is c_double)
-            self.assertTrue(c_double.__ctype_be__.__ctype_le__ is c_double)
+            self.assertIs(c_double.__ctype_le__, c_double)
+            self.assertIs(c_double.__ctype_be__.__ctype_le__, c_double)
         else:
-            self.assertTrue(c_double.__ctype_be__ is c_double)
-            self.assertTrue(c_double.__ctype_le__.__ctype_be__ is c_double)
+            self.assertIs(c_double.__ctype_be__, c_double)
+            self.assertIs(c_double.__ctype_le__.__ctype_be__, c_double)
         s = c_double(math.pi)
         self.assertEqual(s.value, math.pi)
         self.assertEqual(bin(struct.pack("d", math.pi)), bin(s))
@@ -146,14 +147,14 @@
         self.assertEqual(bin(struct.pack(">d", math.pi)), bin(s))
 
     def test_endian_other(self):
-        self.assertTrue(c_byte.__ctype_le__ is c_byte)
-        self.assertTrue(c_byte.__ctype_be__ is c_byte)
+        self.assertIs(c_byte.__ctype_le__, c_byte)
+        self.assertIs(c_byte.__ctype_be__, c_byte)
 
-        self.assertTrue(c_ubyte.__ctype_le__ is c_ubyte)
-        self.assertTrue(c_ubyte.__ctype_be__ is c_ubyte)
+        self.assertIs(c_ubyte.__ctype_le__, c_ubyte)
+        self.assertIs(c_ubyte.__ctype_be__, c_ubyte)
 
-        self.assertTrue(c_char.__ctype_le__ is c_char)
-        self.assertTrue(c_char.__ctype_be__ is c_char)
+        self.assertIs(c_char.__ctype_le__, c_char)
+        self.assertIs(c_char.__ctype_be__, c_char)
 
     @xfail
     def test_struct_fields_1(self):
diff --git a/lib-python/2.7/ctypes/test/test_callbacks.py b/lib-python/2.7/ctypes/test/test_callbacks.py
--- a/lib-python/2.7/ctypes/test/test_callbacks.py
+++ b/lib-python/2.7/ctypes/test/test_callbacks.py
@@ -1,5 +1,6 @@
 import unittest
 from ctypes import *
+from ctypes.test import need_symbol
 from ctypes.test import xfail
 import _ctypes_test
 
@@ -95,9 +96,10 @@
     # disabled: would now (correctly) raise a RuntimeWarning about
     # a memory leak.  A callback function cannot return a non-integral
     # C type without causing a memory leak.
-##    def test_char_p(self):
-##        self.check_type(c_char_p, "abc")
-##        self.check_type(c_char_p, "def")
+    @unittest.skip('test disabled')
+    def test_char_p(self):
+        self.check_type(c_char_p, "abc")
+        self.check_type(c_char_p, "def")
 
     @xfail
     def test_pyobject(self):
@@ -150,13 +152,12 @@
         CFUNCTYPE(None)(lambda x=Nasty(): None)
 
 
-try:
-    WINFUNCTYPE
-except NameError:
-    pass
-else:
-    class StdcallCallbacks(Callbacks):
+ at need_symbol('WINFUNCTYPE')
+class StdcallCallbacks(Callbacks):
+    try:
         functype = WINFUNCTYPE
+    except NameError:
+        pass
 
 ################################################################
 
@@ -186,7 +187,7 @@
         from ctypes.util import find_library
         libc_path = find_library("c")
         if not libc_path:
-            return # cannot test
+            self.skipTest('could not find libc')
         libc = CDLL(libc_path)
 
         @CFUNCTYPE(c_int, POINTER(c_int), POINTER(c_int))
@@ -198,23 +199,19 @@
         libc.qsort(array, len(array), sizeof(c_int), cmp_func)
         self.assertEqual(array[:], [1, 5, 7, 33, 99])
 
-    try:
-        WINFUNCTYPE
-    except NameError:
-        pass
-    else:
-        def test_issue_8959_b(self):
-            from ctypes.wintypes import BOOL, HWND, LPARAM
+    @need_symbol('WINFUNCTYPE')
+    def test_issue_8959_b(self):
+        from ctypes.wintypes import BOOL, HWND, LPARAM
+        global windowCount
+        windowCount = 0
+
+        @WINFUNCTYPE(BOOL, HWND, LPARAM)
+        def EnumWindowsCallbackFunc(hwnd, lParam):
             global windowCount
-            windowCount = 0
+            windowCount += 1
+            return True #Allow windows to keep enumerating
 
-            @WINFUNCTYPE(BOOL, HWND, LPARAM)
-            def EnumWindowsCallbackFunc(hwnd, lParam):
-                global windowCount
-                windowCount += 1
-                return True #Allow windows to keep enumerating
-
-            windll.user32.EnumWindows(EnumWindowsCallbackFunc, 0)
+        windll.user32.EnumWindows(EnumWindowsCallbackFunc, 0)
 
     def test_callback_register_int(self):
         # Issue #8275: buggy handling of callback args under Win64
diff --git a/lib-python/2.7/ctypes/test/test_cast.py b/lib-python/2.7/ctypes/test/test_cast.py
--- a/lib-python/2.7/ctypes/test/test_cast.py
+++ b/lib-python/2.7/ctypes/test/test_cast.py
@@ -1,4 +1,5 @@
 from ctypes import *
+from ctypes.test import need_symbol
 import unittest
 import sys
 
@@ -38,14 +39,14 @@
 
         p = cast(array, POINTER(c_char_p))
         # array and p share a common _objects attribute
-        self.assertTrue(p._objects is array._objects)
+        self.assertIs(p._objects, array._objects)
         self.assertEqual(array._objects, {'0': "foo bar", id(array): array})
         p[0] = "spam spam"
         self.assertEqual(p._objects, {'0': "spam spam", id(array): array})
-        self.assertTrue(array._objects is p._objects)
+        self.assertIs(array._objects, p._objects)
         p[1] = "foo bar"
         self.assertEqual(p._objects, {'1': 'foo bar', '0': "spam spam", id(array): array})
-        self.assertTrue(array._objects is p._objects)
+        self.assertIs(array._objects, p._objects)
 
     def test_other(self):
         p = cast((c_int * 4)(1, 2, 3, 4), POINTER(c_int))
@@ -75,15 +76,11 @@
         self.assertEqual(cast(cast(s, c_void_p), c_char_p).value,
                              "hiho")
 
-    try:
-        c_wchar_p
-    except NameError:
-        pass
-    else:
-        def test_wchar_p(self):
-            s = c_wchar_p("hiho")
-            self.assertEqual(cast(cast(s, c_void_p), c_wchar_p).value,
-                                 "hiho")
+    @need_symbol('c_wchar_p')
+    def test_wchar_p(self):
+        s = c_wchar_p("hiho")
+        self.assertEqual(cast(cast(s, c_void_p), c_wchar_p).value,
+                             "hiho")
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_cfuncs.py b/lib-python/2.7/ctypes/test/test_cfuncs.py
--- a/lib-python/2.7/ctypes/test/test_cfuncs.py
+++ b/lib-python/2.7/ctypes/test/test_cfuncs.py
@@ -3,6 +3,7 @@
 
 import unittest
 from ctypes import *
+from ctypes.test import need_symbol
 
 import _ctypes_test
 from test.test_support import impl_detail
@@ -196,7 +197,7 @@
 try:
     WinDLL
 except NameError:
-    pass
+    def stdcall_dll(*_): pass
 else:
     class stdcall_dll(WinDLL):
         def __getattr__(self, name):
@@ -206,9 +207,9 @@
             setattr(self, name, func)
             return func
 
-    class stdcallCFunctions(CFunctions):
-        _dll = stdcall_dll(_ctypes_test.__file__)
-        pass
+ at need_symbol('WinDLL')
+class stdcallCFunctions(CFunctions):
+    _dll = stdcall_dll(_ctypes_test.__file__)
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_checkretval.py b/lib-python/2.7/ctypes/test/test_checkretval.py
--- a/lib-python/2.7/ctypes/test/test_checkretval.py
+++ b/lib-python/2.7/ctypes/test/test_checkretval.py
@@ -1,6 +1,7 @@
 import unittest
 
 from ctypes import *
+from ctypes.test import need_symbol
 
 class CHECKED(c_int):
     def _check_retval_(value):
@@ -25,15 +26,11 @@
         del dll._testfunc_p_p.restype
         self.assertEqual(42, dll._testfunc_p_p(42))
 
-    try:
-        oledll
-    except NameError:
-        pass
-    else:
-        def test_oledll(self):
-            self.assertRaises(WindowsError,
-                                  oledll.oleaut32.CreateTypeLib2,
-                                  0, None, None)
+    @need_symbol('oledll')
+    def test_oledll(self):
+        self.assertRaises(WindowsError,
+                              oledll.oleaut32.CreateTypeLib2,
+                              0, None, None)
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_errcheck.py b/lib-python/2.7/ctypes/test/test_errcheck.py
deleted file mode 100644
--- a/lib-python/2.7/ctypes/test/test_errcheck.py
+++ /dev/null
@@ -1,19 +0,0 @@
-import sys
-from ctypes import *
-
-##class HMODULE(Structure):
-##    _fields_ = [("value", c_void_p)]
-
-##    def __repr__(self):
-##        return "" % self.value
-
-##windll.kernel32.GetModuleHandleA.restype = HMODULE
-
-##print windll.kernel32.GetModuleHandleA("python23.dll")
-##print hex(sys.dllhandle)
-
-##def nonzero(handle):
-##    return (GetLastError(), handle)
-
-##windll.kernel32.GetModuleHandleA.errcheck = nonzero
-##print windll.kernel32.GetModuleHandleA("spam")
diff --git a/lib-python/2.7/ctypes/test/test_find.py b/lib-python/2.7/ctypes/test/test_find.py
--- a/lib-python/2.7/ctypes/test/test_find.py
+++ b/lib-python/2.7/ctypes/test/test_find.py
@@ -1,4 +1,5 @@
 import unittest
+import os
 import sys
 from ctypes import *
 from ctypes.util import find_library
@@ -40,43 +41,43 @@
             except OSError:
                 pass
 
-    if lib_gl:
-        def test_gl(self):
-            if self.gl:
-                self.gl.glClearIndex
+    @unittest.skipUnless(lib_gl, 'lib_gl not available')
+    def test_gl(self):
+        if self.gl:
+            self.gl.glClearIndex
 
-    if lib_glu:
-        def test_glu(self):
-            if self.glu:
-                self.glu.gluBeginCurve
+    @unittest.skipUnless(lib_glu, 'lib_glu not available')
+    def test_glu(self):
+        if self.glu:
+            self.glu.gluBeginCurve
 
-    if lib_gle:
-        def test_gle(self):
-            if self.gle:
-                self.gle.gleGetJoinStyle
+    @unittest.skipUnless(lib_gle, 'lib_gle not available')
+    def test_gle(self):
+        if self.gle:
+            self.gle.gleGetJoinStyle
 
-##if os.name == "posix" and sys.platform != "darwin":
-
-##    # On platforms where the default shared library suffix is '.so',
-##    # at least some libraries can be loaded as attributes of the cdll
-##    # object, since ctypes now tries loading the lib again
-##    # with '.so' appended of the first try fails.
-##    #
-##    # Won't work for libc, unfortunately.  OTOH, it isn't
-##    # needed for libc since this is already mapped into the current
-##    # process (?)
-##    #
-##    # On MAC OSX, it won't work either, because dlopen() needs a full path,
-##    # and the default suffix is either none or '.dylib'.
-
-##    class LoadLibs(unittest.TestCase):
-##        def test_libm(self):
-##            import math
-##            libm = cdll.libm
-##            sqrt = libm.sqrt
-##            sqrt.argtypes = (c_double,)
-##            sqrt.restype = c_double
-##            self.assertEqual(sqrt(2), math.sqrt(2))
+# On platforms where the default shared library suffix is '.so',
+# at least some libraries can be loaded as attributes of the cdll
+# object, since ctypes now tries loading the lib again
+# with '.so' appended of the first try fails.
+#
+# Won't work for libc, unfortunately.  OTOH, it isn't
+# needed for libc since this is already mapped into the current
+# process (?)
+#
+# On MAC OSX, it won't work either, because dlopen() needs a full path,
+# and the default suffix is either none or '.dylib'.
+ at unittest.skip('test disabled')
+ at unittest.skipUnless(os.name=="posix" and sys.platform != "darwin",
+                     'test not suitable for this platform')
+class LoadLibs(unittest.TestCase):
+    def test_libm(self):
+        import math
+        libm = cdll.libm
+        sqrt = libm.sqrt
+        sqrt.argtypes = (c_double,)
+        sqrt.restype = c_double
+        self.assertEqual(sqrt(2), math.sqrt(2))
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_frombuffer.py b/lib-python/2.7/ctypes/test/test_frombuffer.py
--- a/lib-python/2.7/ctypes/test/test_frombuffer.py
+++ b/lib-python/2.7/ctypes/test/test_frombuffer.py
@@ -25,7 +25,7 @@
         a[0], a[-1] = 200, -200
         self.assertEqual(x[:], a.tolist())
 
-        self.assertTrue(a in x._objects.values())
+        self.assertIn(a, x._objects.values())
 
         self.assertRaises(ValueError,
                           c_int.from_buffer, a, -1)
diff --git a/lib-python/2.7/ctypes/test/test_funcptr.py b/lib-python/2.7/ctypes/test/test_funcptr.py
--- a/lib-python/2.7/ctypes/test/test_funcptr.py
+++ b/lib-python/2.7/ctypes/test/test_funcptr.py
@@ -75,7 +75,7 @@
         ##                  "lpfnWndProc", WNDPROC_2(wndproc))
         # instead:
 
-        self.assertTrue(WNDPROC is WNDPROC_2)
+        self.assertIs(WNDPROC, WNDPROC_2)
         # 'wndclass.lpfnWndProc' leaks 94 references.  Why?
         self.assertEqual(wndclass.lpfnWndProc(1, 2, 3, 4), 10)
 
diff --git a/lib-python/2.7/ctypes/test/test_functions.py b/lib-python/2.7/ctypes/test/test_functions.py
--- a/lib-python/2.7/ctypes/test/test_functions.py
+++ b/lib-python/2.7/ctypes/test/test_functions.py
@@ -6,6 +6,7 @@
 """
 
 from ctypes import *
+from ctypes.test import need_symbol
 import sys, unittest
 from ctypes.test import xfail
 from test.test_support import impl_detail
@@ -65,22 +66,16 @@
             pass
 
 
+    @need_symbol('c_wchar')
     def test_wchar_parm(self):
-        try:
-            c_wchar
-        except NameError:
-            return
         f = dll._testfunc_i_bhilfd
         f.argtypes = [c_byte, c_wchar, c_int, c_long, c_float, c_double]
         result = f(1, u"x", 3, 4, 5.0, 6.0)
         self.assertEqual(result, 139)
         self.assertEqual(type(result), int)
 
+    @need_symbol('c_wchar')
     def test_wchar_result(self):
-        try:

From noreply at buildbot.pypy.org  Thu Sep 18 03:16:12 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 18 Sep 2014 03:16:12 +0200 (CEST)
Subject: [pypy-commit] pypy gc_no_cleanup_nursery: implement setarrayitem
 unrolling for constant length just-allocated arrays
Message-ID: <20140918011612.3925E1C023E@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: gc_no_cleanup_nursery
Changeset: r73599:c12587039b65
Date: 2014-09-17 19:13 -0600
http://bitbucket.org/pypy/pypy/changeset/c12587039b65/

Log:	implement setarrayitem unrolling for constant length just-allocated
	arrays

diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -186,6 +186,15 @@
         v_arr_plus_ofs = BoxInt()
         v_totalsize = BoxInt()
         gcdescr = self.gc_ll_descr
+        if isinstance(v_arrsize, ConstInt) and v_arrsize.getint() < 10:
+            # clear it item by item
+            ops = []
+            for i in range(v_arrsize.getint()):
+                ops.append(ResOperation(rop.SETARRAYITEM_GC,
+                                        [v_arr, ConstInt(i)], None,
+                                        descr=arraydescr))
+            self.newops += ops
+            return
         ops = [
             ResOperation(rop.INT_ADD, [v_arr, ConstInt(ofs)], v_arr_plus_ofs),
         ]

From noreply at buildbot.pypy.org  Thu Sep 18 03:24:46 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 18 Sep 2014 03:24:46 +0200 (CEST)
Subject: [pypy-commit] pypy gc_no_cleanup_nursery: oops
Message-ID: <20140918012446.E659A1C03C7@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: gc_no_cleanup_nursery
Changeset: r73600:e21c64624711
Date: 2014-09-17 19:17 -0600
http://bitbucket.org/pypy/pypy/changeset/e21c64624711/

Log:	oops

diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -191,7 +191,7 @@
             ops = []
             for i in range(v_arrsize.getint()):
                 ops.append(ResOperation(rop.SETARRAYITEM_GC,
-                                        [v_arr, ConstInt(i)], None,
+                                        [v_arr, ConstInt(i), self.c_zero], None,
                                         descr=arraydescr))
             self.newops += ops
             return

From noreply at buildbot.pypy.org  Thu Sep 18 06:04:10 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Thu, 18 Sep 2014 06:04:10 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: progress
Message-ID: <20140918040410.83F1F1D24CD@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: extradoc
Changeset: r5403:ef0fa11803c3
Date: 2014-09-17 21:09 +0100
http://bitbucket.org/pypy/extradoc/changeset/ef0fa11803c3/

Log:	progress

diff --git a/talk/pycon-uk-2014/Makefile b/talk/pycon-uk-2014/Makefile
--- a/talk/pycon-uk-2014/Makefile
+++ b/talk/pycon-uk-2014/Makefile
@@ -1,6 +1,5 @@
 talk.pdf: talk.rst author.latex stylesheet.latex
-	rst2beamer --stylesheet=stylesheet.latex --documentoptions=14pt talk.rst talk.latex || exit
-	#/home/antocuni/.virtualenvs/rst2beamer/bin/python `which rst2beamer.py` --stylesheet=stylesheet.latex --documentoptions=14pt talk.rst talk.latex || exit
+	rst2beamer --stylesheet=stylesheet.latex --documentoptions=14pt --overlaybullets=False talk.rst talk.latex || exit
 	sed 's/\\date{}/\\input{author.latex}/' -i talk.latex || exit
 	#sed 's/\\maketitle/\\input{title.latex}/' -i talk.latex || exit
 	pdflatex talk.latex  || exit
diff --git a/talk/pycon-uk-2014/author.latex b/talk/pycon-uk-2014/author.latex
--- a/talk/pycon-uk-2014/author.latex
+++ b/talk/pycon-uk-2014/author.latex
@@ -1,8 +1,8 @@
 \definecolor{rrblitbackground}{rgb}{0.0, 0.0, 0.0}
 
-\title[PyPy Status]{PyPy Status\\\small{(no, PyPy is not dead)}}
-\author[antocuni]
-{Antonio Cuni}
+\title{PyPy and its ecosystem}
+\author[Ronan Lamy]{Ronan Lamy\\
+\includegraphics[width=80px]{../img/py-web-new.png}}
 
-\institute{PyCon Cinque}
-\date{May 24, 2014}
+\institute{Pycon UK}
+\date{19 September 2014}
diff --git a/talk/pycon-uk-2014/talk.rst b/talk/pycon-uk-2014/talk.rst
--- a/talk/pycon-uk-2014/talk.rst
+++ b/talk/pycon-uk-2014/talk.rst
@@ -1,106 +1,53 @@
 .. include:: beamerdefs.txt
 
-================================
-PyPy Status
-================================
+===========================
+PyPy and its ecosystem
+===========================
 
 About me
 ---------
 
 - PyPy core dev
 
-- ``pdb++``, ``fancycompleter``, ...
+- Working on HippyVM
 
-- Consultant, trainer
-
-- http://antocuni.eu
-
-
-PyPy is not dead
-----------------
-
-- No PyPy status talk at EuroPython 2013
-
-  * for the first time since 2004!
-
-  * for no good reason :)
-
-- PyPy is healthy and alive
-
-|pause|
-
-- WARNING: This talk is boring
-
-  * "it just works"
 
 What is PyPy?
 --------------
 
-* RPython toolchain
+- "PyPy is a fast, compliant alternative implementation of the Python language (2.7.8 and 3.2.5)."
 
-  - subset of Python
+- Python interpreter
 
-  - ideal for writing VMs
+  * written in RPython
 
-  - JIT & GC for free
+  * **FAST**
 
-* Python interpreter
 
-  - written in RPython
+What is RPython?
+----------------
 
-  - **FAST**
+- Subset of Python
 
-* Whatever (dynamic) language you want
+  * easy to read
 
-  - smalltalk, prolog, PHP, javascript, ...
+  * easy to test
 
+- JIT & GC for free
 
-PyPy: past two years (1)
------------------------------
+- General framework for dynamic languages
 
-- PyPy 2.0 (May 2013)
 
-  * beta ARM, CFFI, unicode performance
-
-  * stackless + JIT (eventlet, gevent, ...)
-
-|pause|
-
-- PyPy 2.1 (July 2013)
-
-  * stable ARM
-
-  * py3k (3.2.3), numpy, general improvements, bugfixes
-
-|pause|
-
-- PyPy 2.2 (November 2013)
-
-  * incremental GC, faster JSON
-
-  * more JIT, more py3k
-
-  * more numpy, numpy C API
-
-
-PyPy: past two years (2)
+RPython-powered languages
 -------------------------
 
-- PyPy 2.3 (May 2014)
+- **PyPy**
 
-- Lot of internal refactoring
+- HippyVM: implementing PHP
 
-- C API for embedding
+  * ~7x faster than standard PHP
 
-  * pypy + uWSGI (thanks to Roberto De Ioris)
-
-- the usual, boring, general improvements
-
-
-More PyPy-powered languages
-----------------------------
-
-- RPython: general framework for dynamic languages
+  * http://hippyvm.com/
 
 - Topaz: implementing Ruby
 
@@ -110,28 +57,152 @@
 
   * https://github.com/topazproject/topaz
 
-- HippyVM: implementing PHP
+- Pyrolog (Prolog)
 
-  * ~7x faster than standard PHP
+- RTruffleSOM (Smalltalk)
 
-  * http://hippyvm.com/
+- RSqueakVM (Smalltalk)
 
+- lang-js (JavaScript)
 
 
-Fundraising campaign
----------------------
+RPython translation stages
+--------------------------
 
-- py3k: 50'852 $ of 105'000 $ (48.4%)
+- (R)Python code
 
-- numpy: 48'121 $ of 60'000 $ (80.2%)
+|pause|
 
-- STM, 1st call: 25'000 $
+- ``import``
 
-- STM, 2nd call: 2'097 $ of 80'000 $ (2.6%)
+  * Python objects (functions, classes, ...)
 
-  * more on STM later
+|pause|
 
-- thank to all donors!
+- Bytecode analysis, type inference
+
+  * Typed control flow graph
+
+|pause|
+
+- Translator transformations
+
+  * Add GC & JIT
+
+|pause|
+
+- Code generation
+
+  * C code
+
+|pause|
+
+- ``gcc``
+
+  * Compiled executable
+
+
+How does the JIT work?
+----------------------
+
+|pause|
+
+- "Jitcode": very low-level byte code
+
+  * Translates to machine code
+
+- Translation time
+
+  * Add jitcode representation to RPython functions
+
+- Run-time:
+
+  * Detect **hot** loop
+
+  * Trace one code path through the loop
+
+  * Compile (magic!)
+
+  * Profit!
+
+
+RPython example (HippyVM)
+-------------------------
+
+|scriptsize|
+
+.. code:: python
+
+    @wrap(['space', str, W_Root, Optional(int)])
+    def strpos(space, haystack, w_needle, offset=0):
+        """Find the position of the first occurrence of a substring in a string."""
+        if offset < 0 or offset > len(haystack):
+            space.ec.warn("strpos(): Offset not contained in string")
+            return space.w_False
+        try:
+            needle = unwrap_needle(space, w_needle)
+        except ValidationError as exc:
+            space.ec.warn("strpos(): " + exc.msg)
+            return space.w_False
+        if len(needle) == 0:
+            space.ec.warn("strpos(): Empty needle")
+            return space.w_False
+
+        result = haystack.find(needle, offset)
+
+        if result == -1:
+            return space.w_False
+        return space.newint(result)
+
+|end_scriptsize|
+
+PyPy: past two years (1)
+-----------------------------
+
+- PyPy 2.0 (May 2013)
+
+  * Beta ARM, CFFI, unicode performance
+
+  * stackless + JIT (eventlet, gevent, ...)
+
+|pause|
+
+- PyPy 2.1 (July 2013)
+
+  * Stable ARM
+
+  * py3k (3.2.3), numpy, general improvements, bugfixes
+
+|pause|
+
+- PyPy 2.2 (November 2013)
+
+  * Incremental GC, faster JSON
+
+  * More JIT, more py3k
+
+  * More numpy, numpy C API
+
+
+PyPy: past two years (2)
+-------------------------
+
+- PyPy 2.3 (May 2014)
+
+  * Lot of internal refactoring
+
+  * C API for embedding
+
+  * General improvements
+
+|pause|
+
+- PyPy 2.4 (coming soon!)
+
+  * Python 2.7.8 stdlib
+
+  * General fixes and improvements 
+
 
 Current status
 ---------------
@@ -171,25 +242,9 @@
 
 - ~7.5x faster than CPython on ARM
 
-- thanks to Raspberry-Pi foundation
+- Thanks to Raspberry-Pi foundation
 
-- distributed as part of Raspbian OS
-
-
-numpy
------
-
-- as usual, in-progress
-
-- ~80% of numpy implemented
-
-  * 2336 passing tests out of 3265
-
-  * http://buildbot.pypy.org/numpy-status/latest.html
-
-- just try it
-
-- no scipy :-/
+- Distributed as part of Raspbian OS
 
 
 py3k
@@ -199,7 +254,7 @@
 
 - 3.3: branch started, in-progress
 
-- some missing optimizations
+- Some missing optimizations
 
   * getting better
 
@@ -220,6 +275,22 @@
 - Fast on CPython, super-fast on PyPy
 
 
+numpy
+-----
+
+- As usual, in-progress
+
+- ~80% of numpy implemented
+
+  * 2336 passing tests out of 3265
+
+  * http://buildbot.pypy.org/numpy-status/latest.html
+
+- Just try it
+
+- No scipy :-/
+
+
 cppyy
 ------
 
@@ -303,7 +374,7 @@
 
 - With atomic blocks
 
-  * ==> Rollaback
+  * ==> Rollback
 
   * Performance penalty
 
@@ -315,7 +386,7 @@
 Implementation
 ---------------
 
-- Conflicts detection, commit and rollaback is costly
+- Conflicts detection, commit and rollback is costly
 
 - Original goal (2011): 2x-5x slower than PyPy without STM
 
@@ -339,6 +410,20 @@
 - Lots of polishing needed
 
 
+Fundraising campaign
+---------------------
+
+- py3k: 52,380 $ of 105,000 $ (49.9%)
+
+- numpy: 48,412 $ of 60,000 $ (80.7%)
+
+- STM, 1st call: 25,000 $
+
+- STM, 2nd call: 13,939 $ of 80,000 $ (17.4%)
+
+- Thanks to all donors!
+
+
 Contacts, Q&A
 --------------
 
@@ -346,13 +431,6 @@
 
 - http://morepypy.blogspot.com/
 
-- twitter: @antocuni
-
-- Available for consultancy & training:
-
-  * http://antocuni.eu
-
-  * info at antocuni.eu
+- IRC: #pypy at freenode.net
 
 - Any question?
-

From noreply at buildbot.pypy.org  Thu Sep 18 06:04:11 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Thu, 18 Sep 2014 06:04:11 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: progress
Message-ID: <20140918040411.AB95F1D24CD@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: extradoc
Changeset: r5404:d32cb87fb33e
Date: 2014-09-18 05:04 +0100
http://bitbucket.org/pypy/extradoc/changeset/d32cb87fb33e/

Log:	progress

diff --git a/talk/pycon-uk-2014/Makefile b/talk/pycon-uk-2014/Makefile
--- a/talk/pycon-uk-2014/Makefile
+++ b/talk/pycon-uk-2014/Makefile
@@ -1,6 +1,7 @@
 talk.pdf: talk.rst author.latex stylesheet.latex
-	rst2beamer --stylesheet=stylesheet.latex --documentoptions=14pt --overlaybullets=False talk.rst talk.latex || exit
+	rst2beamer --stylesheet=stylesheet.latex --overlaybullets=False --codeblocks-use-pygments --documentoptions=14pt talk.rst talk.latex || exit
 	sed 's/\\date{}/\\input{author.latex}/' -i talk.latex || exit
+	sed 's/XXXFAKETITLE//' -i talk.latex || exit
 	#sed 's/\\maketitle/\\input{title.latex}/' -i talk.latex || exit
 	pdflatex talk.latex  || exit
 
diff --git a/talk/pycon-uk-2014/talk.rst b/talk/pycon-uk-2014/talk.rst
--- a/talk/pycon-uk-2014/talk.rst
+++ b/talk/pycon-uk-2014/talk.rst
@@ -1,7 +1,7 @@
 .. include:: beamerdefs.txt
 
 ===========================
-PyPy and its ecosystem
+XXXFAKETITLE
 ===========================
 
 About me
@@ -43,19 +43,21 @@
 
 - **PyPy**
 
+|pause|
+
 - HippyVM: implementing PHP
 
   * ~7x faster than standard PHP
 
   * http://hippyvm.com/
 
+|pause|
+
 - Topaz: implementing Ruby
 
-  * most of the language implemented
+  * https://github.com/topazproject/topaz
 
-  * "definitely faster than MRI"
-
-  * https://github.com/topazproject/topaz
+|pause|
 
 - Pyrolog (Prolog)
 
@@ -122,6 +124,8 @@
   * Trace one code path through the loop
 
   * Compile (magic!)
+    
+    - with guards to ensure correctness
 
   * Profit!
 
@@ -131,7 +135,7 @@
 
 |scriptsize|
 
-.. code:: python
+.. sourcecode:: python
 
     @wrap(['space', str, W_Root, Optional(int)])
     def strpos(space, haystack, w_needle, offset=0):
@@ -156,6 +160,62 @@
 
 |end_scriptsize|
 
+
+JITting strpos()
+----------------
+
+|example<| strpos_jit.php |>|
+
+|scriptsize|
+
+.. sourcecode:: php
+
+  
+
+|end_scriptsize|
+
+|end_example|
+
+JITting strpos()
+----------------
+
+|scriptsize|
+
+.. parsed-literal::
+
+  label(p0, p1, p3, p7, i48, i52, p33, p26, descr=TargetToken(140099127632144))
+  i57 = int_eq(i52, 100000)
+  guard_false(i57, descr=) [p1, p0, i52, p3, p7, i48]
+  i58 = int_lt(i52, 100000)
+  guard_true(i58, descr=) [p1, p0, p3, p7, i52, i48]
+  guard_not_invalidated(descr=) [p1, p0, p3, p7, p33, i52, i48]
+  **p59 = call(ConstClass(ll_str__IntegerR_SignedConst_Signed), i52, descr=)**
+  guard_no_exception(descr=) [p1, p0, p59, p3, p7, p33, i52, i48]
+  i60 = strlen(p59)
+  **i61 = call(ConstClass(ll_find_char__rpy_stringPtr_Char_Signed_Signed), p59, 49, 0, i60, descr=)**
+  i62 = int_eq(i61, -1)
+  guard_false(i62, descr=) [p1, p0, i61, p3, p7, p33, i52, i48]
+  i63 = int_add_ovf(i48, i61)
+  guard_no_overflow(descr=) [p1, p0, i63, i61, i48, p3, p7, p33, i52, None]
+  i64 = int_add(i52, 1)
+  i65 = getfield_raw(24210432, descr=)
+  i66 = int_lt(i65, 0)
+  guard_false(i66, descr=) [p1, p0, p3, p7, i63, i64, None, None, None]
+  i67 = arraylen_gc(p26, descr=)
+  jump(p0, p1, p3, p7, i63, i64, p33, p26, descr=TargetToken(140099127632144))
+
+|end_scriptsize|
+
 PyPy: past two years (1)
 -----------------------------
 
@@ -209,21 +269,13 @@
 
 - Python code: "it just works"
 
-- C code: better than ever!
+- 2.7: very stable
 
-  * cpyext: more complete, but still slow
+- 3.2: stable
 
-  * CFFI: the future
+  * Some missing optimizations
 
-  * Native PyPy C API for embedding
-
-  * cppyy for C++
-
-- Lots of CFFI modules around:
-
-  * pygame_cffi, psycopg2_cffi, lxml
-
-- numpy: in-progress (more later)
+- 3.3: in-progress
 
 
 Speed: 6.3x faster than CPython
@@ -247,16 +299,22 @@
 - Distributed as part of Raspbian OS
 
 
-py3k
-----
+Compiled extensions
+-------------------
 
-- 3.2: stable
+- cpyext: incomplete, slow
 
-- 3.3: branch started, in-progress
+- ctypes: inconvenient, slow
 
-- Some missing optimizations
+- "Don't write your own C, PyPy is fast enough"
 
-  * getting better
+|pause|
+
+- Native PyPy C API for embedding
+
+- CFFI: the future
+
+- numpy: in-progress (more later)
 
 
 CFFI
@@ -264,148 +322,109 @@
 
 - Python <-> C interfacing done right
 
+  * Inspired by LuaJIT's FFI
+
   * existing shared libraries
 
   * custom C code
 
-- Inspired by LuaJIT's FFI
-
 - Alternative to C-API, ctypes, Cython, etc.
 
 - Fast on CPython, super-fast on PyPy
 
+- Lots of CFFI modules around:
+
+  * pygame_cffi, psycopg2_cffi, lxml, cryptography, pyzmq, ...
+
+
+CFFI
+--------------------
+
+|example<| Simple example |>|
+|scriptsize|
+
+.. code:: python
+
+    >>> from cffi import FFI
+    >>> ffi = FFI()
+    >>> ffi.cdef("""
+    ...     int printf(const char *format, ...);   // from man page
+    ... """)
+    >>> C = ffi.dlopen(None)              # loads whole C namespace
+    >>> arg = ffi.new("char[]", "world")  # char arg[] = "world";
+    >>> C.printf("hi there, %s!\n", arg)  # call printf()
+    hi there, world!
+
+|end_scriptsize|
+|end_example|
+
+- Docs: http://cffi.readthedocs.org
+
 
 numpy
 -----
 
-- As usual, in-progress
+- In-progress
 
 - ~80% of numpy implemented
 
-  * 2336 passing tests out of 3265
-
-  * http://buildbot.pypy.org/numpy-status/latest.html
+  * 2415 passing tests out of 3265
 
 - Just try it
 
 - No scipy :-/
 
 
-cppyy
-------
-
-- Interface to C++
-
-- Based on reflection, no need to write wrappers
-
-- PyPy-only, similar to PyCintex for CPython
-
-- Main use case: ROOT
-
-  * http://root.cern.ch
-
-  * "a set of OO frameworks with all the functionality needed to handle and
-    analyze large amounts of data in a very efficient way"
-
-- 3x faster than CPython
-
-
 The future: STM
 ----------------
 
 - Software Transactional Memory
 
-- Strategy to solve race conditions
-
-- "Finger crossed", rollback in case of conflicts
+- "Easier parallelism for everybody"
 
 - On-going research project
 
   * by Armin Rigo and Remi Meier
 
-STM semantics
--------------
+Transactional Memory
+--------------------
 
-- N threads
+* like GIL, but instead of blocking, each thread runs optimistically
 
-- Each thread split into atomic blocks
+* "easy" to implement:
 
-- Sequential execution in some arbitrary order
+  - GIL acquire -> transaction start
 
-- In practice:
+  - GIL release -> transaction commit
 
-- Parallel execution, conflicts solved by STM
+* overhead: cross-checking conflicting memory reads and writes,
+  and if necessary, cancel and restart transactions
 
 
-Unit of execution (1)
----------------------
-
-- Atomic blocks == 1 Python bytecode
-
-- Threads are executed in arbitrary order, but bytecodes are atomic
-
-- ==> Same semantics as GIL
-
-- "and this will solve the GIL problem" (A. Rigo, EuroPython 2011 lighting talk)
-
-Unit of execution (2)
-----------------------
+Longer transactions
+-------------------
 
 - Larger atomic blocks
 
-- ``with atomic:``
+- threads and application-level locks still needed...
 
-- Much easier to use than explicit locks
+- but *can be very coarse:*
+
+  - two transactions can optimistically run in parallel
+
+  - even if they both *acquire and release the same lock*
 
 - Can be hidden by libraries to provide even higher level paradigms
 
   * e.g.: Twisted apps made parallel out of the box
 
-Race conditions
----------------
 
-- They don't magically disappear
-
-- With explicit locks
-
-  * ==> BOOM
-
-  * you fix bugs by preventing race conditions
-
-- With atomic blocks
-
-  * ==> Rollback
-
-  * Performance penalty
-
-  * You optimize by preventing race conditions
-
-- Fast&broken vs. Slower&correct
-
-
-Implementation
----------------
-
-- Conflicts detection, commit and rollback is costly
-
-- Original goal (2011): 2x-5x slower than PyPy without STM
-
-  * But parallelizable!
-
-|pause|
-
-- Current goal (2014): 25% slower than PyPy without STM
-
-- Yes, that's 10x less overhead than original goal
-
-- mmap black magic
-
-Current status
----------------
+STM status
+----------
 
 - Preliminary versions of pypy-jit-stm available
 
-- The JIT overhead is still a bit too high
+- Best case 25-40% overhead (much better than originally planned)
 
 - Lots of polishing needed
 
@@ -417,9 +436,11 @@
 
 - numpy: 48,412 $ of 60,000 $ (80.7%)
 
-- STM, 1st call: 25,000 $
+- STM (2nd call): 13,939 $ of 80,000 $ (17.4%)
 
-- STM, 2nd call: 13,939 $ of 80,000 $ (17.4%)
+- General PyPy progress
+
+- **September only**: PSF matches donations
 
 - Thanks to all donors!
 
@@ -429,8 +450,8 @@
 
 - http://pypy.org
 
-- http://morepypy.blogspot.com/
+- Commercial support: http://baroquesoftware.com
 
-- IRC: #pypy at freenode.net
+- IRC: ``#pypy`` on freenode.net
 
 - Any question?

From noreply at buildbot.pypy.org  Thu Sep 18 07:20:04 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Thu, 18 Sep 2014 07:20:04 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Pictures!
Message-ID: <20140918052004.567C81C0492@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: extradoc
Changeset: r5405:886d8e2461bd
Date: 2014-09-18 06:20 +0100
http://bitbucket.org/pypy/extradoc/changeset/886d8e2461bd/

Log:	Pictures!

diff --git a/talk/pycon-uk-2014/Makefile b/talk/pycon-uk-2014/Makefile
--- a/talk/pycon-uk-2014/Makefile
+++ b/talk/pycon-uk-2014/Makefile
@@ -1,5 +1,5 @@
 talk.pdf: talk.rst author.latex stylesheet.latex
-	rst2beamer --stylesheet=stylesheet.latex --overlaybullets=False --codeblocks-use-pygments --documentoptions=14pt talk.rst talk.latex || exit
+	rst2beamer --stylesheet=stylesheet.latex --overlaybullets=False --documentoptions=14pt talk.rst talk.latex || exit
 	sed 's/\\date{}/\\input{author.latex}/' -i talk.latex || exit
 	sed 's/XXXFAKETITLE//' -i talk.latex || exit
 	#sed 's/\\maketitle/\\input{title.latex}/' -i talk.latex || exit
diff --git a/talk/pycon-uk-2014/how-jit.png b/talk/pycon-uk-2014/how-jit.png
new file mode 100644
index 0000000000000000000000000000000000000000..ad9ce720343f9b461e202faacd566abc3a331b61
GIT binary patch

[cut]

diff --git a/talk/pycon-uk-2014/how-pypy.png b/talk/pycon-uk-2014/how-pypy.png
new file mode 100644
index 0000000000000000000000000000000000000000..203ad9bc6073ee2fda4d9e624f6bf6bd1941b4cb
GIT binary patch

[cut]

diff --git a/talk/pycon-uk-2014/how-rpython.png b/talk/pycon-uk-2014/how-rpython.png
new file mode 100644
index 0000000000000000000000000000000000000000..a98cd6335f01d9bcfc2638d72a446a3f3af0cba8
GIT binary patch

[cut]

diff --git a/talk/pycon-uk-2014/stylesheet.latex b/talk/pycon-uk-2014/stylesheet.latex
--- a/talk/pycon-uk-2014/stylesheet.latex
+++ b/talk/pycon-uk-2014/stylesheet.latex
@@ -1,5 +1,4 @@
 \usetheme{Boadilla}
-\setbeamercovered{transparent}
 \setbeamertemplate{navigation symbols}{}
 
 \definecolor{darkgreen}{rgb}{0, 0.5, 0.0}
diff --git a/talk/pycon-uk-2014/talk.rst b/talk/pycon-uk-2014/talk.rst
--- a/talk/pycon-uk-2014/talk.rst
+++ b/talk/pycon-uk-2014/talk.rst
@@ -23,6 +23,18 @@
 
   * **FAST**
 
+How does RPython work?
+----------------------
+
+.. image:: how-rpython.png
+
+.. raw:: latex
+
+    {\tiny
+
+Altered from http://abstrusegoose.com/secretarchives/under-the-hood - CC BY-NC 3.0 US 
+
+|>|
 
 What is RPython?
 ----------------
@@ -103,12 +115,25 @@
 
   * Compiled executable
 
-
 How does the JIT work?
 ----------------------
 
 |pause|
 
+.. image:: how-jit.png
+
+.. raw:: latex
+
+    {\tiny
+
+Altered from http://abstrusegoose.com/secretarchives/under-the-hood - CC BY-NC 3.0 US 
+
+|>|
+
+
+How does the JIT work?
+----------------------
+
 - "Jitcode": very low-level byte code
 
   * Translates to machine code
@@ -216,6 +241,20 @@
 
 |end_scriptsize|
 
+How does PyPy work?
+----------------------
+
+.. image:: how-pypy.png
+
+.. raw:: latex
+
+    {\tiny
+
+Altered from http://abstrusegoose.com/secretarchives/under-the-hood - CC BY-NC 3.0 US 
+
+|>|
+
+
 PyPy: past two years (1)
 -----------------------------
 
@@ -298,6 +337,17 @@
 
 - Distributed as part of Raspbian OS
 
+jitviewer
+---------
+
+- Python code: "it just works"
+
+  * but is it fast?
+
+- jitviewer: find out what the JIT does
+
+  * http://bitbucket.org/pypy/jitviewer
+
 
 Compiled extensions
 -------------------
@@ -343,7 +393,7 @@
 |example<| Simple example |>|
 |scriptsize|
 
-.. code:: python
+.. sourcecode:: python
 
     >>> from cffi import FFI
     >>> ffi = FFI()
@@ -364,13 +414,17 @@
 numpy
 -----
 
-- In-progress
+- Core inside PyPy
+
+- The rest at http://bitbucket.org/pypy/numpy
 
 - ~80% of numpy implemented
 
   * 2415 passing tests out of 3265
 
-- Just try it
+- In progress, just try it
+
+- No Python 3 support
 
 - No scipy :-/
 

From noreply at buildbot.pypy.org  Thu Sep 18 13:31:41 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Thu, 18 Sep 2014 13:31:41 +0200 (CEST)
Subject: [pypy-commit] pypy release-2.4.x: merge default into release
Message-ID: <20140918113141.E1ED81D3878@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: release-2.4.x
Changeset: r73601:f5dcc2477b97
Date: 2014-09-18 14:28 +0300
http://bitbucket.org/pypy/pypy/changeset/f5dcc2477b97/

Log:	merge default into release

diff --git a/lib-python/2.7/test/test_mmap.py b/lib-python/2.7/test/test_mmap.py
--- a/lib-python/2.7/test/test_mmap.py
+++ b/lib-python/2.7/test/test_mmap.py
@@ -179,25 +179,27 @@
         import sys
         f = open(TESTFN, "r+b")
         try:
-            m = mmap.mmap(f.fileno(), mapsize+1)
-        except ValueError:
-            # we do not expect a ValueError on Windows
-            # CAUTION:  This also changes the size of the file on disk, and
-            # later tests assume that the length hasn't changed.  We need to
-            # repair that.
+            try:
+                m = mmap.mmap(f.fileno(), mapsize+1)
+            except ValueError:
+                # we do not expect a ValueError on Windows
+                # CAUTION:  This also changes the size of the file on disk, and
+                # later tests assume that the length hasn't changed.  We need to
+                # repair that.
+                if sys.platform.startswith('win'):
+                    self.fail("Opening mmap with size+1 should work on Windows.")
+            else:
+                # we expect a ValueError on Unix, but not on Windows
+                if not sys.platform.startswith('win'):
+                    self.fail("Opening mmap with size+1 should raise ValueError.")
+                m.close()
+        finally:
+            f.close()
             if sys.platform.startswith('win'):
-                self.fail("Opening mmap with size+1 should work on Windows.")
-        else:
-            # we expect a ValueError on Unix, but not on Windows
-            if not sys.platform.startswith('win'):
-                self.fail("Opening mmap with size+1 should raise ValueError.")
-            m.close()
-        f.close()
-        if sys.platform.startswith('win'):
-            # Repair damage from the resizing test.
-            f = open(TESTFN, 'r+b')
-            f.truncate(mapsize)
-            f.close()
+                # Repair damage from the resizing test.
+                f = open(TESTFN, 'r+b')
+                f.truncate(mapsize)
+                f.close()
 
         # Opening mmap with access=ACCESS_WRITE
         f = open(TESTFN, "r+b")
diff --git a/pypy/doc/getting-started-python.rst b/pypy/doc/getting-started-python.rst
--- a/pypy/doc/getting-started-python.rst
+++ b/pypy/doc/getting-started-python.rst
@@ -111,6 +111,10 @@
    of your choice.  Typical example: ``--opt=2`` gives a good (but of
    course slower) Python interpreter without the JIT.
 
+   Consider using PyPy instead of CPython in the above command line,
+   as it is much faster.  (Note that ``rpython`` is a Python 2 program,
+   not Python 3; you need to run either PyPy 2 or CPython 2.)
+
 .. _`optimization level`: config/opt.html
 
 If everything works correctly this will create an executable
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-2.4.0.rst
    release-2.3.1.rst
    release-2.3.0.rst
    release-2.2.1.rst
diff --git a/pypy/doc/release-2.4.0.rst b/pypy/doc/release-2.4.0.rst
--- a/pypy/doc/release-2.4.0.rst
+++ b/pypy/doc/release-2.4.0.rst
@@ -1,11 +1,11 @@
 =================================================
-PyPy 2.4 - ????????
+PyPy 2.4 - Snow White
 =================================================
 
 We're pleased to announce PyPy 2.4, which contains significant performance
 enhancements and bug fixes. 
 
-You can download the PyPy 2.4 release here:
+You can already download the PyPy 2.4-beta1 pre-release here:
 
     http://pypy.org/download.html
 
@@ -58,7 +58,8 @@
 bytearray handling, and a major rewrite of the GIL handling. This means
 that external calls are now a lot faster, especially the CFFI ones. It also
 means better performance in a lot of corner cases with handling strings or
-bytearrays.
+bytearrays. The main bugfix is handling of many socket objects in your
+program which in the long run used to "leak" memory.
 
 PyPy now uses Python 2.7.8 standard library.
 
@@ -82,7 +83,7 @@
   this mostly affects errno handling on linux, which makes external calls
   faster.
 
-* Move to a mixed polling and mutex GIL model that make mutli-threaded jitted
+* Move to a mixed polling and mutex GIL model that make mutlithreaded jitted
   code run *much* faster
 
 * Optimize errno handling in linux (x86 and x86-64 only)
@@ -97,13 +98,17 @@
 
 * Upgrade stdlib from 2.7.5 to 2.7.8
 
+* Win32 now links statically to zlib, expat, bzip, and openssl-1.0.1i.
+  No more missing DLLs
+  
 * Many issues were resolved_ since the 2.3.1 release on June 8
 
 .. _`whats-new`: http://doc.pypy.org/en/latest/whatsnew-2.3.1.html
 .. _resolved: https://bitbucket.org/pypy/pypy/issues?status=resolved
 
-We have further improvements on the way: rpython file handling and
-usable numpy linalg compatabiity should be merged soon.
+We have further improvements on the way: rpython file handling,
+numpy linalg compatibility, as well
+as improved GC and many smaller improvements.
 
 Please try it out and let us know what you think. We especially welcome
 success stories, we know you are using PyPy, please tell us about it!
diff --git a/pypy/doc/whatsnew-2.4.0.rst b/pypy/doc/whatsnew-2.4.0.rst
--- a/pypy/doc/whatsnew-2.4.0.rst
+++ b/pypy/doc/whatsnew-2.4.0.rst
@@ -60,3 +60,7 @@
 
 .. branch: stdlib-2.7.8
 Upgrades from 2.7.6 to 2.7.8
+
+.. branch: cpybug-seq-radd-rmul
+Fix issue #1861 - cpython compatability madness
+
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -4,5 +4,5 @@
 =======================
 
 .. this is a revision shortly after release-2.4.x
-.. startrev: 204b550079b0
+.. startrev: 7026746cbb1b
 
diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -85,10 +85,13 @@
 
 Abridged method (for -Ojit builds using Visual Studio 2008)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Download the versions of all the external packages
-from 
+Download the versions of all the external packages from 
+https://bitbucket.org/pypy/pypy/downloads/local_2.4.zip
+(for 2.4 release and later) or
 https://bitbucket.org/pypy/pypy/downloads/local.zip
-Then expand it into the base directory (base_dir) and modify your environment to reflect this::
+(for pre-2.4 versions)
+Then expand it into the base directory (base_dir) and modify your environment
+to reflect this::
 
     set PATH=\bin;\tcltk\bin;%PATH%
     set INCLUDE=\include;\tcltk\include;%INCLUDE%
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -38,18 +38,15 @@
 def cpython_code_signature(code):
     "([list-of-arg-names], vararg-name-or-None, kwarg-name-or-None)."
     argcount = code.co_argcount
+    varnames = code.co_varnames
     assert argcount >= 0     # annotator hint
-    argnames = list(code.co_varnames[:argcount])
+    argnames = list(varnames[:argcount])
     if code.co_flags & CO_VARARGS:
-        varargname = code.co_varnames[argcount]
+        varargname = varnames[argcount]
         argcount += 1
     else:
         varargname = None
-    if code.co_flags & CO_VARKEYWORDS:
-        kwargname = code.co_varnames[argcount]
-        argcount += 1
-    else:
-        kwargname = None
+    kwargname = varnames[argcount] if code.co_flags & CO_VARKEYWORDS else None
     return Signature(argnames, varargname, kwargname)
 
 
diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py
--- a/pypy/interpreter/pyparser/parsestring.py
+++ b/pypy/interpreter/pyparser/parsestring.py
@@ -83,12 +83,6 @@
     v = PyString_DecodeEscape(space, substr, 'strict', enc)
     return space.wrap(v)
 
-def hexbyte(val):
-    result = "%x" % val
-    if len(result) == 1:
-        result = "0" + result
-    return result
-
 def decode_unicode_utf8(space, s, ps, q):
     # ****The Python 2.7 version, producing UTF-32 escapes****
     # String is utf8-encoded, but 'unicode_escape' expects
@@ -108,15 +102,14 @@
                 # instead.
                 lis.append("u005c")
         if ord(s[ps]) & 0x80: # XXX inefficient
-            w, ps = decode_utf8(space, s, ps, end, "utf-32-be")
-            rn = len(w)
-            assert rn % 4 == 0
-            for i in range(0, rn, 4):
-                lis.append('\\U')
-                lis.append(hexbyte(ord(w[i])))
-                lis.append(hexbyte(ord(w[i+1])))
-                lis.append(hexbyte(ord(w[i+2])))
-                lis.append(hexbyte(ord(w[i+3])))
+            w, ps = decode_utf8(space, s, ps, end)
+            for c in w:
+                # The equivalent of %08x, which is not supported by RPython.
+                # 7 zeroes are enough for the unicode range, and the
+                # result still fits in 32-bit.
+                hexa = hex(ord(c) + 0x10000000)
+                lis.append('\\U0')
+                lis.append(hexa[3:])  # Skip 0x and the leading 1
         else:
             lis.append(s[ps])
             ps += 1
@@ -136,7 +129,7 @@
             # note that the C code has a label here.
             # the logic is the same.
             if recode_encoding and ord(s[ps]) & 0x80:
-                w, ps = decode_utf8(space, s, ps, end, recode_encoding)
+                w, ps = decode_utf8_recode(space, s, ps, end, recode_encoding)
                 # Append bytes to output buffer.
                 builder.append(w)
             else:
@@ -222,14 +215,18 @@
             ch >= 'A' and ch <= 'F')
 
 
-def decode_utf8(space, s, ps, end, encoding):
+def decode_utf8(space, s, ps, end):
     assert ps >= 0
     pt = ps
     # while (s < end && *s != '\\') s++; */ /* inefficient for u".."
     while ps < end and ord(s[ps]) & 0x80:
         ps += 1
-    w_u = space.wrap(unicodehelper.decode_utf8(space, s[pt:ps]))
-    w_v = unicodehelper.encode(space, w_u, encoding)
+    u = unicodehelper.decode_utf8(space, s[pt:ps])
+    return u, ps
+
+def decode_utf8_recode(space, s, ps, end, recode_encoding):
+    u, ps = decode_utf8(space, s, ps, end)
+    w_v = unicodehelper.encode(space, space.wrap(u), recode_encoding)
     v = space.str_w(w_v)
     return v, ps
 
diff --git a/pypy/interpreter/pyparser/test/test_parsestring.py b/pypy/interpreter/pyparser/test/test_parsestring.py
--- a/pypy/interpreter/pyparser/test/test_parsestring.py
+++ b/pypy/interpreter/pyparser/test/test_parsestring.py
@@ -73,11 +73,11 @@
 
     def test_simple_enc_roundtrip(self):
         space = self.space
-        s = "'\x81'"
+        s = "'\x81\\t'"
         s = s.decode("koi8-u").encode("utf8")
         w_ret = parsestring.parsestr(self.space, 'koi8-u', s)
         ret = space.unwrap(w_ret)
-        assert ret == eval("# -*- coding: koi8-u -*-\n'\x81'") 
+        assert ret == eval("# -*- coding: koi8-u -*-\n'\x81\\t'") 
 
     def test_multiline_unicode_strings_with_backslash(self):
         space = self.space
diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -945,7 +945,7 @@
         prefix = udir.join('pathtest').ensure(dir=1)
         fake_exe = 'bin/pypy-c'
         if sys.platform == 'win32':
-            fake_exe += '.exe'
+            fake_exe = 'pypy-c.exe'
         fake_exe = prefix.join(fake_exe).ensure(file=1)
         expected_path = [str(prefix.join(subdir).ensure(dir=1))
                          for subdir in ('lib_pypy',
@@ -985,8 +985,10 @@
             assert sys.path == old_sys_path + [self.goal_dir]
 
             app_main.setup_bootstrap_path(self.fake_exe)
-            assert sys.executable == ''      # not executable!
-            assert sys.path == old_sys_path + [self.goal_dir]
+            if not sys.platform == 'win32':
+                # an existing file is always 'executable' on windows
+                assert sys.executable == ''      # not executable!
+                assert sys.path == old_sys_path + [self.goal_dir]
 
             os.chmod(self.fake_exe, 0755)
             app_main.setup_bootstrap_path(self.fake_exe)
diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -5,6 +5,7 @@
 
 @specialize.memo()
 def decode_error_handler(space):
+    # Fast version of the "strict" errors handler.
     def raise_unicode_exception_decode(errors, encoding, msg, s,
                                        startingpos, endingpos):
         raise OperationError(space.w_UnicodeDecodeError,
@@ -17,6 +18,7 @@
 
 @specialize.memo()
 def encode_error_handler(space):
+    # Fast version of the "strict" errors handler.
     def raise_unicode_exception_encode(errors, encoding, msg, u,
                                        startingpos, endingpos):
         raise OperationError(space.w_UnicodeEncodeError,
diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py
--- a/pypy/module/_cffi_backend/ctypefunc.py
+++ b/pypy/module/_cffi_backend/ctypefunc.py
@@ -8,6 +8,7 @@
 from rpython.rlib.jit_libffi import (CIF_DESCRIPTION, CIF_DESCRIPTION_P,
     FFI_TYPE, FFI_TYPE_P, FFI_TYPE_PP, SIZE_OF_FFI_ARG)
 from rpython.rlib.objectmodel import we_are_translated, instantiate
+from rpython.rlib.objectmodel import keepalive_until_here
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
 
 from pypy.interpreter.error import OperationError, oefmt
@@ -160,6 +161,7 @@
                         raw_cdata = rffi.cast(rffi.CCHARPP, data)[0]
                         lltype.free(raw_cdata, flavor='raw')
             lltype.free(buffer, flavor='raw')
+            keepalive_until_here(args_w)
         return w_res
 
 def get_mustfree_flag(data):
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -759,17 +759,25 @@
 
         # socket's timeout is in seconds, poll's timeout in ms
         timeout = int(sock_timeout * 1000 + 0.5)
-        ready = rpoll.poll(fddict, timeout)
+        try:
+            ready = rpoll.poll(fddict, timeout)
+        except rpoll.PollError, e:
+            message = e.get_msg()
+            raise ssl_error(space, message, e.errno)
     else:
         if MAX_FD_SIZE is not None and sock_fd >= MAX_FD_SIZE:
             return SOCKET_TOO_LARGE_FOR_SELECT
 
-        if writing:
-            r, w, e = rpoll.select([], [sock_fd], [], sock_timeout)
-            ready = w
-        else:
-            r, w, e = rpoll.select([sock_fd], [], [], sock_timeout)
-            ready = r
+        try:
+            if writing:
+                r, w, e = rpoll.select([], [sock_fd], [], sock_timeout)
+                ready = w
+            else:
+                r, w, e = rpoll.select([sock_fd], [], [], sock_timeout)
+                ready = r
+        except rpoll.SelectError as e:
+            message = e.get_msg()
+            raise ssl_error(space, message, e.errno)
     if ready:
         return SOCKET_OPERATION_OK
     else:
diff --git a/pypy/module/operator/__init__.py b/pypy/module/operator/__init__.py
--- a/pypy/module/operator/__init__.py
+++ b/pypy/module/operator/__init__.py
@@ -39,7 +39,7 @@
                     'irshift', 'isub', 'itruediv', 'ixor', '_length_hint']
 
     interpleveldefs = {
-        '_compare_digest': 'interp_operator.compare_digest',
+        '_compare_digest': 'tscmp.compare_digest',
     }
 
     for name in interp_names:
diff --git a/pypy/module/operator/app_operator.py b/pypy/module/operator/app_operator.py
--- a/pypy/module/operator/app_operator.py
+++ b/pypy/module/operator/app_operator.py
@@ -4,7 +4,7 @@
 This module exports a set of operators as functions. E.g. operator.add(x,y) is
 equivalent to x+y.
 '''
-from __pypy__ import builtinify
+
 import types
 
 
@@ -27,7 +27,7 @@
     'getslice(a, b, c) -- Same as a[b:c].'
     if not isinstance(start, int) or not isinstance(end, int):
         raise TypeError("an integer is expected")
-    return a[start:end] 
+    return a[start:end]
 __getslice__ = getslice
 
 def indexOf(a, b):
@@ -37,7 +37,7 @@
         if x == b:
             return index
         index += 1
-    raise ValueError, 'sequence.index(x): x not in sequence'
+    raise ValueError('sequence.index(x): x not in sequence')
 
 def isMappingType(obj,):
     'isMappingType(a) -- Return True if a has a mapping type, False otherwise.'
@@ -58,9 +58,9 @@
 def repeat(obj, num):
     'repeat(a, b) -- Return a * b, where a is a sequence, and b is an integer.'
     if not isinstance(num, (int, long)):
-        raise TypeError, 'an integer is required'
+        raise TypeError('an integer is required')
     if not isSequenceType(obj):
-        raise TypeError, "non-sequence object can't be repeated"
+        raise TypeError("non-sequence object can't be repeated")
 
     return obj * num
 
@@ -68,59 +68,85 @@
 
 def setslice(a, b, c, d):
     'setslice(a, b, c, d) -- Same as a[b:c] = d.'
-    a[b:c] = d 
+    a[b:c] = d
 __setslice__ = setslice
 
 
+def _resolve_attr_chain(chain, obj, idx=0):
+    obj = getattr(obj, chain[idx])
+    if idx + 1 == len(chain):
+        return obj
+    else:
+        return _resolve_attr_chain(chain, obj, idx + 1)
+
+
+class _simple_attrgetter(object):
+    def __init__(self, attr):
+        self._attr = attr
+
+    def __call__(self, obj):
+        return getattr(obj, self._attr)
+
+
+class _single_attrgetter(object):
+    def __init__(self, attrs):
+        self._attrs = attrs
+
+    def __call__(self, obj):
+        return _resolve_attr_chain(self._attrs, obj)
+
+
+class _multi_attrgetter(object):
+    def __init__(self, attrs):
+        self._attrs = attrs
+
+    def __call__(self, obj):
+        return tuple([
+            _resolve_attr_chain(attrs, obj)
+            for attrs in self._attrs
+        ])
+
+
 def attrgetter(attr, *attrs):
+    if (
+        not isinstance(attr, basestring) or
+        not all(isinstance(a, basestring) for a in attrs)
+    ):
+        def _raise_typeerror(obj):
+            raise TypeError(
+                "argument must be a string, not %r" % type(attr).__name__
+            )
+        return _raise_typeerror
     if attrs:
-        getters = [single_attr_getter(a) for a in (attr,) + attrs]
-        def getter(obj):
-            return tuple([getter(obj) for getter in getters])
+        return _multi_attrgetter([
+            a.split(".") for a in [attr] + list(attrs)
+        ])
+    elif "." not in attr:
+        return _simple_attrgetter(attr)
     else:
-        getter = single_attr_getter(attr)
-    return builtinify(getter)
+        return _single_attrgetter(attr.split("."))
 
-def single_attr_getter(attr):
-    if not isinstance(attr, str):
-        if not isinstance(attr, unicode):
-            def _raise_typeerror(obj):
-                raise TypeError("argument must be a string, not %r" %
-                                (type(attr).__name__,))
-            return _raise_typeerror
-        attr = attr.encode('ascii')
-    #
-    def make_getter(name, prevfn=None):
-        if prevfn is None:
-            def getter(obj):
-                return getattr(obj, name)
+
+class itemgetter(object):
+    def __init__(self, item, *items):
+        self._single = not bool(items)
+        if self._single:
+            self._idx = item
         else:
-            def getter(obj):
-                return getattr(prevfn(obj), name)
-        return getter
-    #
-    last = 0
-    getter = None
-    while True:
-        dot = attr.find(".", last)
-        if dot < 0: break
-        getter = make_getter(attr[last:dot], getter)
-        last = dot + 1
-    return make_getter(attr[last:], getter)
+            self._idx = [item] + list(items)
 
+    def __call__(self, obj):
+        if self._single:
+            return obj[self._idx]
+        else:
+            return tuple([obj[i] for i in self._idx])
 
-def itemgetter(item, *items):
-    if items:
-        list_of_indices = [item] + list(items)
-        def getter(obj):
-            return tuple([obj[i] for i in list_of_indices])
-    else:
-        def getter(obj):
-            return obj[item]
-    return builtinify(getter)
 
+class methodcaller(object):
+    def __init__(self, method_name, *args, **kwargs):
+        self._method_name = method_name
+        self._args = args
+        self._kwargs = kwargs
 
-def methodcaller(method_name, *args, **kwargs):
-    def call(obj):
-        return getattr(obj, method_name)(*args, **kwargs)
-    return builtinify(call)
+    def __call__(self, obj):
+        return getattr(obj, self._method_name)(*self._args, **self._kwargs)
diff --git a/pypy/module/operator/interp_operator.py b/pypy/module/operator/interp_operator.py
--- a/pypy/module/operator/interp_operator.py
+++ b/pypy/module/operator/interp_operator.py
@@ -1,6 +1,4 @@
-from rpython.rlib.objectmodel import specialize
-
-from pypy.interpreter.error import OperationError, oefmt
+from pypy.interpreter.error import OperationError
 from pypy.interpreter.gateway import unwrap_spec
 
 
@@ -249,33 +247,3 @@
 @unwrap_spec(default=int)
 def _length_hint(space, w_iterable, default):
     return space.wrap(space.length_hint(w_iterable, default))
-
-def compare_digest(space, w_a, w_b):
-    if (
-        space.isinstance_w(w_a, space.w_unicode) and
-        space.isinstance_w(w_b, space.w_unicode)
-    ):
-        return space.wrap(tscmp(space.unicode_w(w_a), space.unicode_w(w_b)))
-    if (
-        space.isinstance_w(w_a, space.w_unicode) or
-        space.isinstance_w(w_b, space.w_unicode)
-    ):
-        raise oefmt(
-            space.w_TypeError,
-            "unsupported operand types(s) or combination of types: '%N' and '%N'",
-            w_a,
-            w_b,
-        )
-    else:
-        return space.wrap(tscmp(space.bufferstr_w(w_a), space.bufferstr_w(w_b)))
-
-
- at specialize.argtype(0, 1)
-def tscmp(a, b):
-    len_a = len(a)
-    len_b = len(b)
-    length = min(len(a), len(b))
-    res = len_a ^ len_b
-    for i in xrange(length):
-        res |= ord(a[i]) ^ ord(b[i])
-    return res == 0
diff --git a/pypy/module/operator/test/test_operator.py b/pypy/module/operator/test/test_operator.py
--- a/pypy/module/operator/test/test_operator.py
+++ b/pypy/module/operator/test/test_operator.py
@@ -334,3 +334,9 @@
         assert operator._compare_digest(a, b)
         a, b = mybytes(b"foobar"), mybytes(b"foobaz")
         assert not operator._compare_digest(a, b)
+
+    def test_compare_digest_unicode(self):
+        import operator
+        assert operator._compare_digest(u'asd', u'asd')
+        assert not operator._compare_digest(u'asd', u'qwe')
+        raises(TypeError, operator._compare_digest, u'asd', b'qwe')
diff --git a/pypy/module/operator/test/test_tscmp.py b/pypy/module/operator/test/test_tscmp.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/operator/test/test_tscmp.py
@@ -0,0 +1,28 @@
+from pypy.module.operator.tscmp import pypy_tscmp, pypy_tscmp_wide
+
+class TestTimingSafeCompare:
+    tostr = str
+    tscmp = staticmethod(pypy_tscmp)
+
+    def test_tscmp_neq(self):
+        assert not self.tscmp(self.tostr('asd'), self.tostr('qwe'), 3, 3)
+
+    def test_tscmp_eq(self):
+        assert self.tscmp(self.tostr('asd'), self.tostr('asd'), 3, 3)
+
+    def test_tscmp_len(self):
+        assert self.tscmp(self.tostr('asdp'), self.tostr('asdq'), 3, 3)
+
+    def test_tscmp_nlen(self):
+        assert not self.tscmp(self.tostr('asd'), self.tostr('asd'), 2, 3)
+
+
+class TestTimingSafeCompareWide(TestTimingSafeCompare):
+    tostr = unicode
+    tscmp = staticmethod(pypy_tscmp_wide)
+
+    def test_tscmp_wide_nonascii(self):
+        a, b = u"\ud808\udf45", u"\ud808\udf45"
+        assert self.tscmp(a, b, len(a), len(b))
+        a, b = u"\ud808\udf45", u"\ud808\udf45 "
+        assert not self.tscmp(a, b, len(a), len(b))
diff --git a/pypy/module/operator/tscmp.c b/pypy/module/operator/tscmp.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/operator/tscmp.c
@@ -0,0 +1,80 @@
+/* Derived from CPython 3.3.5's operator.c::_tscmp
+ */
+
+#include 
+#include 
+#include "tscmp.h"
+
+int
+pypy_tscmp(const char *a, const char *b, long len_a, long len_b)
+{
+    /* The volatile type declarations make sure that the compiler has no
+     * chance to optimize and fold the code in any way that may change
+     * the timing.
+     */
+    volatile long length;
+    volatile const char *left;
+    volatile const char *right;
+    long i;
+    char result;
+
+    /* loop count depends on length of b */
+    length = len_b;
+    left = NULL;
+    right = b;
+
+    /* don't use else here to keep the amount of CPU instructions constant,
+     * volatile forces re-evaluation
+     *  */
+    if (len_a == length) {
+        left = *((volatile const char**)&a);
+        result = 0;
+    }
+    if (len_a != length) {
+        left = b;
+        result = 1;
+    }
+
+    for (i=0; i < length; i++) {
+        result |= *left++ ^ *right++;
+    }
+
+    return (result == 0);
+}
+
+int
+pypy_tscmp_wide(const wchar_t *a, const wchar_t *b, long len_a, long len_b)
+{
+    /* The volatile type declarations make sure that the compiler has no
+     * chance to optimize and fold the code in any way that may change
+     * the timing.
+     */
+    volatile long length;
+    volatile const wchar_t *left;
+    volatile const wchar_t *right;
+    long i;
+    wchar_t result;
+
+    /* loop count depends on length of b */
+    length = len_b;
+    left = NULL;
+    right = b;
+
+    /* don't use else here to keep the amount of CPU instructions constant,
+     * volatile forces re-evaluation
+     *  */
+    if (len_a == length) {
+        left = *((volatile const wchar_t**)&a);
+        result = 0;
+    }
+    if (len_a != length) {
+        left = b;
+        result = 1;
+    }
+
+    for (i=0; i < length; i++) {
+        result |= *left++ ^ *right++;
+    }
+
+    return (result == 0);
+}
diff --git a/pypy/module/operator/tscmp.h b/pypy/module/operator/tscmp.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/operator/tscmp.h
@@ -0,0 +1,2 @@
+int pypy_tscmp(const char *, const char *, long, long);
+int pypy_tscmp_wide(const wchar_t *, const wchar_t *, long, long);
diff --git a/pypy/module/operator/tscmp.py b/pypy/module/operator/tscmp.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/operator/tscmp.py
@@ -0,0 +1,73 @@
+"""
+Provides _compare_digest method, which is a safe comparing to prevent timing
+attacks for the hmac module.
+"""
+import py
+
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
+
+from pypy.interpreter.error import oefmt
+
+cwd = py.path.local(__file__).dirpath()
+eci = ExternalCompilationInfo(
+    includes=[cwd.join('tscmp.h')],
+    include_dirs=[str(cwd)],
+    separate_module_files=[cwd.join('tscmp.c')],
+    export_symbols=['pypy_tscmp', 'pypy_tscmp_wide'])
+
+
+def llexternal(*args, **kwargs):
+    kwargs.setdefault('compilation_info', eci)
+    kwargs.setdefault('sandboxsafe', True)
+    return rffi.llexternal(*args, **kwargs)
+
+
+pypy_tscmp = llexternal(
+    'pypy_tscmp',
+    [rffi.CCHARP, rffi.CCHARP, rffi.LONG, rffi.LONG],
+    rffi.INT)
+pypy_tscmp_wide = llexternal(
+    'pypy_tscmp_wide',
+    [rffi.CWCHARP, rffi.CWCHARP, rffi.LONG, rffi.LONG],
+    rffi.INT)
+
+
+def compare_digest(space, w_a, w_b):
+    """compare_digest(a, b) -> bool
+
+    Return 'a == b'.  This function uses an approach designed to prevent
+    timing analysis, making it appropriate for cryptography.  a and b
+    must both be of the same type: either str (ASCII only), or any type
+    that supports the buffer protocol (e.g. bytes).
+
+    Note: If a and b are of different lengths, or if an error occurs, a
+    timing attack could theoretically reveal information about the types
+    and lengths of a and b--but not their values.
+    """
+    if (space.isinstance_w(w_a, space.w_unicode) and
+        space.isinstance_w(w_b, space.w_unicode)):
+        a = space.unicode_w(w_a)
+        b = space.unicode_w(w_b)
+        with rffi.scoped_nonmoving_unicodebuffer(a) as a_buf:
+            with rffi.scoped_nonmoving_unicodebuffer(b) as b_buf:
+                result = pypy_tscmp_wide(a_buf, b_buf, len(a), len(b))
+        return space.wrap(rffi.cast(lltype.Bool, result))
+    return compare_digest_buffer(space, w_a, w_b)
+
+
+def compare_digest_buffer(space, w_a, w_b):
+    try:
+        a_buf = w_a.buffer_w(space, space.BUF_SIMPLE)
+        b_buf = w_b.buffer_w(space, space.BUF_SIMPLE)
+    except TypeError:
+        raise oefmt(space.w_TypeError,
+                    "unsupported operand types(s) or combination of types: "
+                    "'%T' and '%T'", w_a, w_b)
+
+    a = a_buf.as_str()
+    b = b_buf.as_str()
+    with rffi.scoped_nonmovingbuffer(a) as a_buf:
+        with rffi.scoped_nonmovingbuffer(b) as b_buf:
+            result = pypy_tscmp(a_buf, b_buf, len(a), len(b))
+    return space.wrap(rffi.cast(lltype.Bool, result))
diff --git a/pypy/module/sys/state.py b/pypy/module/sys/state.py
--- a/pypy/module/sys/state.py
+++ b/pypy/module/sys/state.py
@@ -7,15 +7,15 @@
 # ____________________________________________________________
 #
 
-class State: 
-    def __init__(self, space): 
-        self.space = space 
+class State:
+    def __init__(self, space):
+        self.space = space
 
         self.w_modules = space.newdict(module=True)
-
         self.w_warnoptions = space.newlist([])
         self.w_argv = space.newlist([])
-        self.setinitialpath(space) 
+
+        self.setinitialpath(space)
 
     def setinitialpath(self, space):
         from pypy.module.sys.initpath import compute_stdlib_path
@@ -25,10 +25,10 @@
         path = compute_stdlib_path(self, srcdir)
         self.w_path = space.newlist([space.wrap(p) for p in path])
 
-
 def get(space):
     return space.fromcache(State)
 
+
 class IOState:
     def __init__(self, space):
         from pypy.module._file.interp_file import W_File
@@ -36,17 +36,17 @@
 
         stdin = W_File(space)
         stdin.file_fdopen(0, "r", 1)
-        stdin.name = ''
+        stdin.w_name = space.wrap('')
         self.w_stdin = space.wrap(stdin)
 
         stdout = W_File(space)
         stdout.file_fdopen(1, "w", 1)
-        stdout.name = ''
+        stdout.w_name = space.wrap('')
         self.w_stdout = space.wrap(stdout)
 
         stderr = W_File(space)
         stderr.file_fdopen(2, "w", 0)
-        stderr.name = ''
+        stderr.w_name = space.wrap('')
         self.w_stderr = space.wrap(stderr)
 
         stdin._when_reading_first_flush(stdout)
@@ -54,9 +54,9 @@
 def getio(space):
     return space.fromcache(IOState)
 
+
 def pypy_getudir(space):
     """NOT_RPYTHON
     (should be removed from interpleveldefs before translation)"""
     from rpython.tool.udir import udir
     return space.wrap(str(udir))
-
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -91,6 +91,10 @@
         assert isinstance(sys.__stderr__, file)
         assert isinstance(sys.__stdin__, file)
 
+        #assert sys.__stdin__.name == ""
+        #assert sys.__stdout__.name == ""
+        #assert sys.__stderr__.name == ""
+
         if self.appdirect and not isinstance(sys.stdin, file):
             return
 
diff --git a/rpython/annotator/builtin.py b/rpython/annotator/builtin.py
--- a/rpython/annotator/builtin.py
+++ b/rpython/annotator/builtin.py
@@ -255,6 +255,11 @@
         BUILTIN_ANALYZERS[original] = value
 
 
+ at analyzer_for(getattr(object.__init__, 'im_func', object.__init__))
+def object_init(s_self, *args):
+    # ignore - mostly used for abstract classes initialization
+    pass
+
 @analyzer_for(getattr(EnvironmentError.__init__, 'im_func', EnvironmentError.__init__))
 def EnvironmentError_init(s_self, *args):
     pass
@@ -268,11 +273,6 @@
     def WindowsError_init(s_self, *args):
         pass
 
- at analyzer_for(getattr(object.__init__, 'im_func', object.__init__))
-def object_init(s_self, *args):
-    # ignore - mostly used for abstract classes initialization
-    pass
-
 
 @analyzer_for(sys.getdefaultencoding)
 def conf():
diff --git a/rpython/annotator/classdef.py b/rpython/annotator/classdef.py
--- a/rpython/annotator/classdef.py
+++ b/rpython/annotator/classdef.py
@@ -438,7 +438,9 @@
 # ____________________________________________________________
 
 FORCE_ATTRIBUTES_INTO_CLASSES = {
-    EnvironmentError: {'errno': SomeInteger(), 'strerror': SomeString(can_be_None=True), 'filename': SomeString(can_be_None=True)},
+    EnvironmentError: {'errno': SomeInteger(),
+                       'strerror': SomeString(can_be_None=True),
+                       'filename': SomeString(can_be_None=True)},
 }
 
 try:
@@ -455,4 +457,3 @@
 else:
     FORCE_ATTRIBUTES_INTO_CLASSES[termios.error] = \
         {'args': SomeTuple([SomeInteger(), SomeString()])}
-
diff --git a/rpython/jit/backend/arm/callbuilder.py b/rpython/jit/backend/arm/callbuilder.py
--- a/rpython/jit/backend/arm/callbuilder.py
+++ b/rpython/jit/backend/arm/callbuilder.py
@@ -92,7 +92,8 @@
             self.mc.LDR_ri(r.r7.value, r.r5.value)
 
         # change 'rpy_fastgil' to 0 (it should be non-zero right now)
-        self.mc.DMB()
+        if self.asm.cpu.cpuinfo.arch_version >= 7:
+            self.mc.DMB()
         self.mc.gen_load_int(r.r6.value, fastgil)
         self.mc.MOV_ri(r.ip.value, 0)
         self.mc.STR_ri(r.ip.value, r.r6.value)
@@ -112,7 +113,8 @@
         self.mc.STREX(r.r3.value, r.ip.value, r.r6.value, c=c.EQ)
                                                  # try to claim the lock
         self.mc.CMP_ri(r.r3.value, 0, cond=c.EQ) # did this succeed?
-        self.mc.DMB()
+        if self.asm.cpu.cpuinfo.arch_version >= 7:
+            self.mc.DMB()
         # the success of the lock acquisition is defined by
         # 'EQ is true', or equivalently by 'r3 == 0'.
         #
diff --git a/rpython/jit/backend/arm/codebuilder.py b/rpython/jit/backend/arm/codebuilder.py
--- a/rpython/jit/backend/arm/codebuilder.py
+++ b/rpython/jit/backend/arm/codebuilder.py
@@ -333,6 +333,8 @@
                     | (rn & 0xF) << 16)
 
     def DMB(self):
+        # ARMv7 only.  I guess ARMv6 CPUs cannot be used in symmetric
+        # multi-processing at all? That would make this instruction unneeded.
         # note: 'cond' is only permitted on Thumb here, but don't
         # write literally 0xf57ff05f, because it's larger than 31 bits
         c = cond.AL
diff --git a/rpython/jit/backend/arm/detect.py b/rpython/jit/backend/arm/detect.py
--- a/rpython/jit/backend/arm/detect.py
+++ b/rpython/jit/backend/arm/detect.py
@@ -38,9 +38,9 @@
     try:
         buf = os.read(fd, 2048)
         if not buf:
+            n = 6  # we assume ARMv6 as base case
             debug_print("Could not detect ARM architecture "
                         "version, assuming", "ARMv%d" % n)
-            n = 6  # we asume ARMv6 as base case
     finally:
         os.close(fd)
     # "Processor       : ARMv%d-compatible processor rev 7 (v6l)"
diff --git a/rpython/jit/backend/arm/instructions.py b/rpython/jit/backend/arm/instructions.py
--- a/rpython/jit/backend/arm/instructions.py
+++ b/rpython/jit/backend/arm/instructions.py
@@ -142,6 +142,7 @@
     #'VCVT' : {'opc1':0xB, 'opc2':0xE, 'opc3':0x1, 'base': False},
 }
 
+# ARMv7 only
 simd_instructions_3regs = {
     'VADD_i64': {'A': 0x8, 'B': 0, 'U': 0},
     'VSUB_i64': {'A': 0x8, 'B': 0, 'U': 1},
diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py
--- a/rpython/jit/backend/arm/opassembler.py
+++ b/rpython/jit/backend/arm/opassembler.py
@@ -1128,6 +1128,8 @@
         self.mc.VCVT_int_to_float(res.value, r.svfp_ip.value)
         return fcond
 
+    # the following five instructions are only ARMv7;
+    # regalloc.py won't call them at all on ARMv6
     emit_op_llong_add = gen_emit_float_op('llong_add', 'VADD_i64')
     emit_op_llong_sub = gen_emit_float_op('llong_sub', 'VSUB_i64')
     emit_op_llong_and = gen_emit_float_op('llong_and', 'VAND_i64')
diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py
--- a/rpython/jit/backend/arm/regalloc.py
+++ b/rpython/jit/backend/arm/regalloc.py
@@ -184,7 +184,7 @@
 
 class Regalloc(BaseRegalloc):
 
-    def __init__(self, assembler=None):
+    def __init__(self, assembler):
         self.cpu = assembler.cpu
         self.assembler = assembler
         self.frame_manager = None
@@ -290,7 +290,7 @@
             return self.vfprm.convert_to_imm(value)
 
     def _prepare(self, inputargs, operations, allgcrefs):
-        cpu = self.assembler.cpu
+        cpu = self.cpu
         self.fm = ARMFrameManager(cpu.get_baseofs_of_frame_field())
         self.frame_manager = self.fm
         operations = cpu.gc_ll_descr.rewrite_assembler(cpu, operations,
@@ -550,18 +550,19 @@
                             EffectInfo.OS_LLONG_AND,
                             EffectInfo.OS_LLONG_OR,
                             EffectInfo.OS_LLONG_XOR):
-                args = self._prepare_llong_binop_xx(op, fcond)
-                self.perform_llong(op, args, fcond)
-                return
-            if oopspecindex == EffectInfo.OS_LLONG_TO_INT:
+                if self.cpu.cpuinfo.arch_version >= 7:
+                    args = self._prepare_llong_binop_xx(op, fcond)
+                    self.perform_llong(op, args, fcond)
+                    return
+            elif oopspecindex == EffectInfo.OS_LLONG_TO_INT:
                 args = self._prepare_llong_to_int(op, fcond)
                 self.perform_llong(op, args, fcond)
                 return
-            if oopspecindex == EffectInfo.OS_MATH_SQRT:
+            elif oopspecindex == EffectInfo.OS_MATH_SQRT:
                 args = self.prepare_op_math_sqrt(op, fcond)
                 self.perform_math(op, args, fcond)
                 return
-            #if oopspecindex == EffectInfo.OS_MATH_READ_TIMESTAMP:
+            #elif oopspecindex == EffectInfo.OS_MATH_READ_TIMESTAMP:
             #    ...
         return self._prepare_call(op)
 
@@ -590,7 +591,7 @@
         # spill variables that need to be saved around calls
         self.vfprm.before_call(save_all_regs=save_all_regs)
         if not save_all_regs:
-            gcrootmap = self.assembler.cpu.gc_ll_descr.gcrootmap
+            gcrootmap = self.cpu.gc_ll_descr.gcrootmap
             if gcrootmap and gcrootmap.is_shadow_stack:
                 save_all_regs = 2
         self.rm.before_call(save_all_regs=save_all_regs)
@@ -1082,7 +1083,7 @@
         gcmap = self.get_gcmap([r.r0, r.r1])
         self.possibly_free_var(t)
         #
-        gc_ll_descr = self.assembler.cpu.gc_ll_descr
+        gc_ll_descr = self.cpu.gc_ll_descr
         self.assembler.malloc_cond_varsize_frame(
             gc_ll_descr.get_nursery_free_addr(),
             gc_ll_descr.get_nursery_top_addr(),
@@ -1092,7 +1093,7 @@
         self.assembler._alignment_check()
 
     def prepare_op_call_malloc_nursery_varsize(self, op, fcond):
-        gc_ll_descr = self.assembler.cpu.gc_ll_descr
+        gc_ll_descr = self.cpu.gc_ll_descr
         if not hasattr(gc_ll_descr, 'max_size_of_young_obj'):
             raise Exception("unreachable code")
             # for boehm, this function should never be called
diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -1,3 +1,4 @@
+from rpython.rlib import rgc
 from rpython.rlib.rarithmetic import ovfcheck
 from rpython.rtyper.lltypesystem import llmemory
 from rpython.jit.metainterp import history
@@ -390,8 +391,8 @@
         val = op.getarg(0)
         if val not in self.write_barrier_applied:
             v = op.getarg(1)
-            if isinstance(v, BoxPtr) or (isinstance(v, ConstPtr) and
-                                         bool(v.value)): # store a non-NULL
+            if (isinstance(v, BoxPtr) or (isinstance(v, ConstPtr) and
+                               rgc.needs_write_barrier(v.value))):
                 self.gen_write_barrier(val)
                 #op = op.copy_and_change(rop.SETFIELD_RAW)
         self.newops.append(op)
@@ -400,8 +401,8 @@
         val = op.getarg(0)
         if val not in self.write_barrier_applied:
             v = op.getarg(2)
-            if isinstance(v, BoxPtr) or (isinstance(v, ConstPtr) and
-                                         bool(v.value)): # store a non-NULL
+            if (isinstance(v, BoxPtr) or (isinstance(v, ConstPtr) and
+                                         rgc.needs_write_barrier(v.value))):
                 self.gen_write_barrier_array(val, op.getarg(1))
                 #op = op.copy_and_change(rop.SET{ARRAYITEM,INTERIORFIELD}_RAW)
         self.newops.append(op)
diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -38,6 +38,7 @@
 
 FILEP = rffi.COpaquePtr("FILE")
 OFF_T = config['off_t']
+
 _IONBF = config['_IONBF']
 _IOLBF = config['_IOLBF']
 _IOFBF = config['_IOFBF']
@@ -47,6 +48,11 @@
 BASE_BUF_SIZE = 4096
 BASE_LINE_SIZE = 100
 
+NEWLINE_UNKNOWN = 0
+NEWLINE_CR = 1
+NEWLINE_LF = 2
+NEWLINE_CRLF = 4
+
 
 def llexternal(*args, **kwargs):
     return rffi.llexternal(*args, compilation_info=eci, **kwargs)
@@ -72,6 +78,7 @@
 _pclose2 = (c_pclose, c_pclose_in_del)
 
 c_getc = llexternal('getc', [FILEP], rffi.INT, macro=True)
+c_ungetc = llexternal('ungetc', [rffi.INT, FILEP], rffi.INT)
 c_fgets = llexternal('fgets', [rffi.CCHARP, rffi.INT, FILEP], rffi.CCHARP)
 c_fread = llexternal('fread', [rffi.CCHARP, rffi.SIZE_T, rffi.SIZE_T, FILEP],
                      rffi.SIZE_T)
@@ -89,6 +96,10 @@
 c_ferror = llexternal('ferror', [FILEP], rffi.INT)
 c_clearerr = llexternal('clearerr', [FILEP], lltype.Void)
 
+c_stdin = rffi.CExternVariable(FILEP, 'stdin', eci, c_type='FILE*')[0]
+c_stdout = rffi.CExternVariable(FILEP, 'stdout', eci, c_type='FILE*')[0]
+c_stderr = rffi.CExternVariable(FILEP, 'stderr', eci, c_type='FILE*')[0]
+
 
 def _error(ll_file):
     err = c_ferror(ll_file)
@@ -128,10 +139,10 @@
 
 
 def create_file(filename, mode="r", buffering=-1):
-    mode = _sanitize_mode(mode)
+    newmode = _sanitize_mode(mode)
     ll_name = rffi.str2charp(filename)
     try:
-        ll_mode = rffi.str2charp(mode)
+        ll_mode = rffi.str2charp(newmode)
         try:
             ll_file = c_fopen(ll_name, ll_mode)
             if not ll_file:
@@ -142,29 +153,27 @@
     finally:
         lltype.free(ll_name, flavor='raw')
     _dircheck(ll_file)
-    if buffering >= 0:
-        buf = lltype.nullptr(rffi.CCHARP.TO)
-        if buffering == 0:
-            c_setvbuf(ll_file, buf, _IONBF, 0)
-        elif buffering == 1:
-            c_setvbuf(ll_file, buf, _IOLBF, BUFSIZ)
-        else:
-            c_setvbuf(ll_file, buf, _IOFBF, buffering)
-    return RFile(ll_file)
+    f = RFile(ll_file, mode)
+    f._setbufsize(buffering)
+    return f
 
 
-def create_fdopen_rfile(fd, mode="r"):
-    mode = _sanitize_mode(mode)
-    ll_mode = rffi.str2charp(mode)
+def create_fdopen_rfile(fd, mode="r", buffering=-1):
+    newmode = _sanitize_mode(mode)
+    fd = rffi.cast(rffi.INT, fd)
+    rposix.validate_fd(fd)
+    ll_mode = rffi.str2charp(newmode)
     try:
-        ll_file = c_fdopen(rffi.cast(rffi.INT, fd), ll_mode)
+        ll_file = c_fdopen(fd, ll_mode)
         if not ll_file:
             errno = rposix.get_errno()
             raise OSError(errno, os.strerror(errno))
     finally:
         lltype.free(ll_mode, flavor='raw')
     _dircheck(ll_file)
-    return RFile(ll_file)
+    f = RFile(ll_file, mode)
+    f._setbufsize(buffering)
+    return f
 
 
 def create_temp_rfile():
@@ -188,14 +197,46 @@
             lltype.free(ll_type, flavor='raw')
     finally:
         lltype.free(ll_command, flavor='raw')
-    return RFile(ll_file, _pclose2)
+    return RFile(ll_file, close2=_pclose2)
+
+
+def create_stdio():
+    close2 = (None, None)
+    stdin = RFile(c_stdin(), close2=close2)
+    stdout = RFile(c_stdout(), close2=close2)
+    stderr = RFile(c_stderr(), close2=close2)
+    return stdin, stdout, stderr
 
 
 class RFile(object):
-    def __init__(self, ll_file, close2=_fclose2):
+    _setbuf = lltype.nullptr(rffi.CCHARP.TO)
+    _univ_newline = False
+    _newlinetypes = NEWLINE_UNKNOWN
+    _skipnextlf = False
+
+    def __init__(self, ll_file, mode=None, close2=_fclose2):
         self._ll_file = ll_file
+        if mode is not None:
+            self._univ_newline = 'U' in mode
         self._close2 = close2
 
+    def _setbufsize(self, bufsize):
+        if bufsize >= 0:
+            if bufsize == 0:
+                mode = _IONBF
+            elif bufsize == 1:
+                mode = _IOLBF
+                bufsize = BUFSIZ
+            else:
+                mode = _IOFBF
+            if self._setbuf:
+                lltype.free(self._setbuf, flavor='raw')
+            if mode == _IONBF:
+                self._setbuf = lltype.nullptr(rffi.CCHARP.TO)
+            else:
+                self._setbuf = lltype.malloc(rffi.CCHARP.TO, bufsize, flavor='raw')
+            c_setvbuf(self._ll_file, self._setbuf, mode, bufsize)
+
     def __del__(self):
         """Closes the described file when the object's last reference
         goes away.  Unlike an explicit call to close(), this is meant
@@ -204,7 +245,13 @@
         ll_file = self._ll_file
         if ll_file:
             do_close = self._close2[1]
-            do_close(ll_file)       # return value ignored
+            if do_close:
+                do_close(ll_file)       # return value ignored
+            if self._setbuf:
+                lltype.free(self._setbuf, flavor='raw')
+
+    def _cleanup_(self):
+        self._ll_file = lltype.nullptr(FILEP.TO)
 
     def close(self):
         """Closes the described file.
@@ -220,16 +267,68 @@
             # double close is allowed
             self._ll_file = lltype.nullptr(FILEP.TO)
             do_close = self._close2[0]
-            res = do_close(ll_file)
-            if res == -1:
-                errno = rposix.get_errno()
-                raise IOError(errno, os.strerror(errno))
+            try:
+                if do_close:
+                    res = do_close(ll_file)
+                    if res == -1:
+                        errno = rposix.get_errno()
+                        raise IOError(errno, os.strerror(errno))
+            finally:
+                if self._setbuf:
+                    lltype.free(self._setbuf, flavor='raw')
+                    self._setbuf = lltype.nullptr(rffi.CCHARP.TO)
         return res
 
     def _check_closed(self):
         if not self._ll_file:
             raise ValueError("I/O operation on closed file")
 
+    def _fread(self, buf, n, stream):
+        if not self._univ_newline:
+            return c_fread(buf, 1, n, stream)
+
+        i = 0
+        dst = buf
+        newlinetypes = self._newlinetypes
+        skipnextlf = self._skipnextlf
+        while n:
+            nread = c_fread(dst, 1, n, stream)
+            if nread == 0:
+                break
+
+            src = dst
+            n -= nread
+            shortread = n != 0
+            while nread:
+                nread -= 1
+                c = src[0]
+                src = rffi.ptradd(src, 1)
+                if c == '\r':
+                    dst[0] = '\n'
+                    dst = rffi.ptradd(dst, 1)
+                    i += 1
+                    skipnextlf = True
+                elif skipnextlf and c == '\n':
+                    skipnextlf = False
+                    newlinetypes |= NEWLINE_CRLF
+                    n += 1
+                else:
+                    if c == '\n':
+                        newlinetypes |= NEWLINE_LF
+                    elif skipnextlf:
+                        newlinetypes |= NEWLINE_CR
+                    dst[0] = c
+                    dst = rffi.ptradd(dst, 1)
+                    i += 1
+                    skipnextlf = False
+            if shortread:
+                if skipnextlf and c_feof(stream):
+                    newlinetypes |= NEWLINE_CR
+                break
+        self._newlinetypes = newlinetypes
+        self._skipnextlf = skipnextlf
+        return i
+
     def read(self, size=-1):
         # XXX CPython uses a more delicate logic here
         self._check_closed()
@@ -242,7 +341,7 @@
             try:
                 s = StringBuilder()
                 while True:
-                    returned_size = c_fread(buf, 1, BASE_BUF_SIZE, ll_file)
+                    returned_size = self._fread(buf, BASE_BUF_SIZE, ll_file)
                     returned_size = intmask(returned_size)  # is between 0 and BASE_BUF_SIZE
                     if returned_size == 0:
                         if c_feof(ll_file):
@@ -254,12 +353,13 @@
                 lltype.free(buf, flavor='raw')
         else:  # size > 0
             with rffi.scoped_alloc_buffer(size) as buf:
-                returned_size = c_fread(buf.raw, 1, size, ll_file)
+                returned_size = self._fread(buf.raw, size, ll_file)
                 returned_size = intmask(returned_size)  # is between 0 and size
                 if returned_size == 0:
                     if not c_feof(ll_file):
                         raise _error(ll_file)
                 s = buf.str(returned_size)
+                assert s is not None
             return s
 
     def _readline1(self, raw_buf):
@@ -300,7 +400,7 @@
         self._check_closed()
         if size == 0:
             return ""
-        elif size < 0:
+        elif size < 0 and not self._univ_newline:
             with rffi.scoped_alloc_buffer(BASE_LINE_SIZE) as buf:
                 c = self._readline1(buf.raw)
                 if c >= 0:
@@ -315,19 +415,50 @@
                         break
                 s.append_charpsize(buf.raw, c)
             return s.build()
-        else:  # size > 0
+        else:  # size > 0 or self._univ_newline
             ll_file = self._ll_file
+            c = 0
             s = StringBuilder()
-            while s.getlength() < size:
-                c = c_getc(ll_file)
+            if self._univ_newline:
+                newlinetypes = self._newlinetypes
+                skipnextlf = self._skipnextlf
+                while size < 0 or s.getlength() < size:
+                    c = c_getc(ll_file)
+                    if c == EOF:
+                        break
+                    if skipnextlf:
+                        skipnextlf = False
+                        if c == ord('\n'):
+                            newlinetypes |= NEWLINE_CRLF
+                            c = c_getc(ll_file)
+                            if c == EOF:
+                                break
+                        else:
+                            newlinetypes |= NEWLINE_CR
+                    if c == ord('\r'):
+                        skipnextlf = True
+                        c = ord('\n')
+                    elif c == ord('\n'):
+                        newlinetypes |= NEWLINE_LF
+                    s.append(chr(c))
+                    if c == ord('\n'):
+                        break
                 if c == EOF:
-                    if c_ferror(ll_file):
-                        raise _error(ll_file)
-                    break
-                c = chr(c)
-                s.append(c)
-                if c == '\n':
-                    break
+                    if skipnextlf:
+                        newlinetypes |= NEWLINE_CR
+                self._newlinetypes = newlinetypes
+                self._skipnextlf = skipnextlf
+            else:
+                while s.getlength() < size:
+                    c = c_getc(ll_file)
+                    if c == EOF:
+                        break
+                    s.append(chr(c))
+                    if c == ord('\n'):
+                        break
+            if c == EOF:
+                if c_ferror(ll_file):
+                    raise _error(ll_file)
             return s.build()
 
     @enforceargs(None, str)
@@ -341,6 +472,7 @@
             bytes = c_fwrite(ll_value, 1, length, self._ll_file)
             if bytes != length:
                 errno = rposix.get_errno()
+                c_clearerr(self._ll_file)
                 raise IOError(errno, os.strerror(errno))
         finally:
             rffi.free_nonmovingbuffer(value, ll_value)
@@ -368,6 +500,7 @@
         if res == -1:
             errno = rposix.get_errno()
             raise IOError(errno, os.strerror(errno))
+        self._skipnextlf = False
 
     def tell(self):
         self._check_closed()
@@ -375,8 +508,20 @@
         if res == -1:
             errno = rposix.get_errno()
             raise IOError(errno, os.strerror(errno))
+        if self._skipnextlf:
+            c = c_getc(self._ll_file)
+            if c == ord('\n'):
+                self._newlinetypes |= NEWLINE_CRLF
+                res += 1
+                self._skipnextlf = False
+            elif c != EOF:
+                c_ungetc(c, self._ll_file)
         return res
 
     def fileno(self):
         self._check_closed()
         return intmask(c_fileno(self._ll_file))
+
+    def isatty(self):
+        self._check_closed()
+        return os.isatty(c_fileno(self._ll_file))
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -86,6 +86,14 @@
         collect(i)
         i += 1
 
+def needs_write_barrier(obj):
+    """ We need to emit write barrier if the right hand of assignment
+    is in nursery, used by the JIT for handling set*_gc(Const)
+    """
+    if not obj:
+        return False
+    return can_move(obj)
+
 def _heap_stats():
     raise NotImplementedError # can't be run directly
 
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -119,7 +119,7 @@
         return 1
 
     def validate_fd(fd):
-        return 1
+        pass
 
 def closerange(fd_low, fd_high):
     # this behaves like os.closerange() from Python 2.6.
diff --git a/rpython/rlib/test/test_rfile.py b/rpython/rlib/test/test_rfile.py
--- a/rpython/rlib/test/test_rfile.py
+++ b/rpython/rlib/test/test_rfile.py
@@ -104,15 +104,53 @@
         f()
         self.interpret(f, [])
 
+    @py.test.mark.skipif("sys.platform == 'win32'")
+    # http://msdn.microsoft.com/en-us/library/86cebhfs.aspx
+    def test_fdopen_buffering_line(self):
+        fname = str(self.tmpdir.join('file_1a'))
+
+        def f():
+            g = open(fname, 'w')
+            f = os.fdopen(os.dup(g.fileno()), 'w', 1)
+            g.close()
+            f.write('dupa\ndupb')
+            f2 = open(fname, 'r')
+            assert f2.read() == 'dupa\n'
+            f.close()
+            assert f2.read() == 'dupb'
+            f2.close()
+
+        f()
+        self.interpret(f, [])
+
     def test_open_buffering_full(self):
         fname = str(self.tmpdir.join('file_1b'))
 
         def f():
             f = open(fname, 'w', 128)
-            f.write('dupa')
+            f.write('dupa\ndupb')
             f2 = open(fname, 'r')
             assert f2.read() == ''
-            f.write('z' * 5000)
+            f.write('z' * 120)
+            assert f2.read() != ''
+            f.close()
+            assert f2.read() != ''
+            f2.close()
+
+        f()
+        self.interpret(f, [])
+
+    def test_fdopen_buffering_full(self):
+        fname = str(self.tmpdir.join('file_1b'))
+
+        def f():
+            g = open(fname, 'w')
+            f = os.fdopen(os.dup(g.fileno()), 'w', 128)
+            g.close()
+            f.write('dupa\ndupb')
+            f2 = open(fname, 'r')
+            assert f2.read() == ''
+            f.write('z' * 120)
             assert f2.read() != ''
             f.close()
             assert f2.read() != ''
@@ -132,10 +170,22 @@
                 pass
             else:
                 assert False
+            try:
+                f.readline()
+            except IOError as e:
+                pass
+            else:
+                assert False
             f.write("dupa\x00dupb")
             f.close()
             for mode in ['r', 'U']:
                 f2 = open(fname, mode)
+                try:
+                    f2.write('z')
+                except IOError as e:
+                    pass
+                else:
+                    assert False
                 dupa = f2.read(0)
                 assert dupa == ""
                 dupa = f2.read()
@@ -177,6 +227,39 @@
         f()
         self.interpret(f, [])
 
+    def test_read_universal(self):
+        fname = str(self.tmpdir.join('read_univ'))
+        with open(fname, 'wb') as f:
+            f.write("dupa\ndupb\r\ndupc\rdupd")
+
+        def f():
+            f = open(fname, 'U')
+            assert f.read() == "dupa\ndupb\ndupc\ndupd"
+            assert f.read() == ""
+            f.seek(0)
+            assert f.read(10) == "dupa\ndupb\n"
+            assert f.read(42) == "dupc\ndupd"
+            assert f.read(1) == ""
+            f.seek(0)
+            assert f.readline() == "dupa\n"
+            assert f.tell() == 5
+            assert f.readline() == "dupb\n"
+            assert f.tell() == 11
+            assert f.readline() == "dupc\n"
+            assert f.tell() == 16
+            assert f.readline() == "dupd"
+            assert f.tell() == 20
+            assert f.readline() == ""
+            f.seek(0)
+            assert f.readline() == "dupa\n"
+            assert f.readline() == "dupb\n"
+            f.seek(4)
+            assert f.read(1) == "\n"
+            f.close()
+
+        f()
+        self.interpret(f, [])
+
     def test_seek(self):
         fname = str(self.tmpdir.join('file_4'))
 
@@ -215,6 +298,12 @@
             new_fno = os.dup(f.fileno())
             f2 = os.fdopen(new_fno, "w")
             f.close()
+            try:
+                f2.read()
+            except IOError as e:
+                pass
+            else:
+                assert False
             f2.write("xxx")
             f2.close()
 
@@ -228,6 +317,7 @@
 
         def f():
             f = open(fname, "w")
+            assert not f.isatty()
             try:
                 return f.fileno()
             finally:
@@ -281,6 +371,14 @@
             data = f.read()
             assert data == "hello w"
             f.close()
+            f = open(fname)
+            try:
+                f.truncate()
+            except IOError as e:
+                pass
+            else:
+                assert False
+            f.close()
 
         f()
         self.interpret(f, [])
@@ -291,6 +389,13 @@
         cls.tmpdir = udir.join('test_rfile_direct')
         cls.tmpdir.ensure(dir=True)
 
+    def test_stdio(self):
+        i, o, e = rfile.create_stdio()
+        o.write("test\n")
+        i.close()
+        o.close()
+        e.close()
+
     def test_auto_close(self):
         fname = str(self.tmpdir.join('file_auto_close'))
         f = rfile.create_file(fname, 'w')
diff --git a/rpython/rtyper/module/ll_os_stat.py b/rpython/rtyper/module/ll_os_stat.py
--- a/rpython/rtyper/module/ll_os_stat.py
+++ b/rpython/rtyper/module/ll_os_stat.py
@@ -186,7 +186,10 @@
     _name_struct_stat = '_stati64'
     INCLUDES = ['sys/types.h', 'sys/stat.h', 'sys/statvfs.h']
 else:
-    _name_struct_stat = 'stat'
+    if sys.platform.startswith('linux'):
+        _name_struct_stat = 'stat64'
+    else:
+        _name_struct_stat = 'stat'
     INCLUDES = ['sys/types.h', 'sys/stat.h', 'sys/statvfs.h', 'unistd.h']
 
 compilation_info = ExternalCompilationInfo(
diff --git a/rpython/rtyper/rbuiltin.py b/rpython/rtyper/rbuiltin.py
--- a/rpython/rtyper/rbuiltin.py
+++ b/rpython/rtyper/rbuiltin.py
@@ -259,9 +259,6 @@
         return i1
     return i2
 
-def rtype_Exception__init__(hop):
-    hop.exception_cannot_occur()
-
 def rtype_object__init__(hop):
     hop.exception_cannot_occur()
 
@@ -269,17 +266,19 @@
     hop.exception_cannot_occur()
     v_self = hop.args_v[0]
     r_self = hop.args_r[0]
-    if hop.nb_args >= 2:
+    if hop.nb_args <= 2:
+        v_errno = hop.inputconst(lltype.Signed, 0)
+        if hop.nb_args == 2:
+            v_strerror = hop.inputarg(rstr.string_repr, arg=1)
+            r_self.setfield(v_self, 'strerror', v_strerror, hop.llops)
+    else:
         v_errno = hop.inputarg(lltype.Signed, arg=1)
-    else:
-        v_errno = hop.inputconst(lltype.Signed, 0)
-    r_self.setfield(v_self, 'errno', v_errno, hop.llops)
-    if hop.nb_args >= 3:
         v_strerror = hop.inputarg(rstr.string_repr, arg=2)
         r_self.setfield(v_self, 'strerror', v_strerror, hop.llops)
         if hop.nb_args >= 4:
             v_filename = hop.inputarg(rstr.string_repr, arg=3)
             r_self.setfield(v_self, 'filename', v_filename, hop.llops)
+    r_self.setfield(v_self, 'errno', v_errno, hop.llops)
 
 def rtype_WindowsError__init__(hop):
     hop.exception_cannot_occur()
@@ -339,6 +338,9 @@
         original = getattr(__builtin__, name[14:])
         BUILTIN_TYPER[original] = value
 
+BUILTIN_TYPER[getattr(object.__init__, 'im_func', object.__init__)] = (
+    rtype_object__init__)
+
 BUILTIN_TYPER[getattr(EnvironmentError.__init__, 'im_func', EnvironmentError.__init__)] = (
     rtype_EnvironmentError__init__)
 
@@ -351,8 +353,6 @@
         getattr(WindowsError.__init__, 'im_func', WindowsError.__init__)] = (
         rtype_WindowsError__init__)
 
-BUILTIN_TYPER[getattr(object.__init__, 'im_func', object.__init__)] = (
-    rtype_object__init__)
 # annotation of low-level types
 
 def rtype_malloc(hop, i_flavor=None, i_zero=None, i_track_allocation=None,
diff --git a/rpython/rtyper/test/test_exception.py b/rpython/rtyper/test/test_exception.py
--- a/rpython/rtyper/test/test_exception.py
+++ b/rpython/rtyper/test/test_exception.py
@@ -36,7 +36,7 @@
 class TestException(BaseRtypingTest):
     def test_exception_with_arg(self):
         def g(n):
-            raise IOError(n)
+            raise IOError("test")
         def h(n):
             raise OSError(n, "?", None)
         def i(n):
@@ -49,8 +49,8 @@
             try:
                 g(n)
             except IOError, e:
-                assert e.errno == 42
-                assert e.strerror is None
+                assert e.errno == 0
+                assert e.strerror == "test"
                 assert e.filename is None
             else:
                 assert False
diff --git a/rpython/translator/platform/test/test_makefile.py b/rpython/translator/platform/test/test_makefile.py
--- a/rpython/translator/platform/test/test_makefile.py
+++ b/rpython/translator/platform/test/test_makefile.py
@@ -44,6 +44,7 @@
         assert res.returncode == 0        
     
     def test_900_files(self):
+        tmpdir = udir.join('test_900_files').ensure(dir=1)
         txt = '#include \n'
         for i in range(900):
             txt += 'int func%03d();\n' % i
@@ -52,11 +53,11 @@
             txt += '    j += func%03d();\n' % i
         txt += '    printf("%d\\n", j);\n'
         txt += '    return 0;};\n'
-        cfile = udir.join('test_900_files.c')
+        cfile = tmpdir.join('test_900_files.c')
         cfile.write(txt)
         cfiles = [cfile]
         for i in range(900):
-            cfile2 = udir.join('implement%03d.c' %i)
+            cfile2 = tmpdir.join('implement%03d.c' %i)
             cfile2.write('''
                 int func%03d()
             {
@@ -64,10 +65,10 @@
             }
             ''' % (i, i))
             cfiles.append(cfile2)
-        mk = self.platform.gen_makefile(cfiles, ExternalCompilationInfo(), path=udir)
+        mk = self.platform.gen_makefile(cfiles, ExternalCompilationInfo(), path=tmpdir)
         mk.write()
         self.platform.execute_makefile(mk)
-        res = self.platform.execute(udir.join('test_900_files'))
+        res = self.platform.execute(tmpdir.join('test_900_files'))
         self.check_res(res, '%d\n' %sum(range(900)))
 
     def test_precompiled_headers(self):
diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py
--- a/rpython/translator/platform/windows.py
+++ b/rpython/translator/platform/windows.py
@@ -100,8 +100,8 @@
     cc = 'cl.exe'
     link = 'link.exe'
 
-    cflags = ('/MD', '/O2')
-    link_flags = ()
+    cflags = ('/MD', '/O2', '/Zi')
+    link_flags = ('/debug',)
     standalone_only = ()
     shared_only = ()
     environ = None
@@ -143,7 +143,6 @@
         # Install debug options only when interpreter is in debug mode
         if sys.executable.lower().endswith('_d.exe'):
             self.cflags = ['/MDd', '/Z7', '/Od']
-            self.link_flags = ['/debug']
 
             # Increase stack size, for the linker and the stack check code.
             stack_size = 8 << 20  # 8 Mb
@@ -204,6 +203,9 @@
         # the assembler still has the old behavior that all options
         # must come first, and after the file name all options are ignored.
         # So please be careful with the order of parameters! ;-)
+        pdb_dir = oname.dirname
+        if pdb_dir:
+                compile_args += ['/Fd%s\\' % (pdb_dir,)]
         args = ['/nologo', '/c'] + compile_args + ['/Fo%s' % (oname,), str(cfile)]
         self._execute_c_compiler(cc, args, oname)
         return oname
diff --git a/rpython/translator/sandbox/rsandbox.py b/rpython/translator/sandbox/rsandbox.py
--- a/rpython/translator/sandbox/rsandbox.py
+++ b/rpython/translator/sandbox/rsandbox.py
@@ -60,26 +60,23 @@
 
     def need_more_data(self):
         buflen = self.buflen
-        buf = lltype.malloc(rffi.CCHARP.TO, buflen, flavor='raw')
-        buflen = rffi.cast(rffi.SIZE_T, buflen)
-        count = ll_read_not_sandboxed(self.fd, buf, buflen)
-        count = rffi.cast(lltype.Signed, count)
-        if count <= 0:
-            raise IOError
-        self.buf += ''.join([buf[i] for i in range(count)])
-        self.buflen *= 2
+        with lltype.scoped_alloc(rffi.CCHARP.TO, buflen) as buf:
+            buflen = rffi.cast(rffi.SIZE_T, buflen)
+            count = ll_read_not_sandboxed(self.fd, buf, buflen)
+            count = rffi.cast(lltype.Signed, count)
+            if count <= 0:
+                raise IOError
+            self.buf += ''.join([buf[i] for i in range(count)])
+            self.buflen *= 2
 
 def sandboxed_io(buf):
     STDIN = 0
     STDOUT = 1
     # send the buffer with the marshalled fnname and input arguments to STDOUT
-    p = lltype.malloc(rffi.CCHARP.TO, len(buf), flavor='raw')
-    try:
+    with lltype.scoped_alloc(rffi.CCHARP.TO, len(buf)) as p:
         for i in range(len(buf)):
             p[i] = buf[i]
         writeall_not_sandboxed(STDOUT, p, len(buf))
-    finally:
-        lltype.free(p, flavor='raw')
     # build a Loader that will get the answer from STDIN
     loader = FdLoader(STDIN)
     # check for errors
@@ -105,9 +102,8 @@
 @signature(types.str(), returns=types.impossible())
 def not_implemented_stub(msg):
     STDERR = 2
-    buf = rffi.str2charp(msg + '\n')
-    writeall_not_sandboxed(STDERR, buf, len(msg) + 1)
-    rffi.free_charp(buf)
+    with rffi.scoped_str2charp(msg + '\n') as buf:
+        writeall_not_sandboxed(STDERR, buf, len(msg) + 1)
     raise RuntimeError(msg)  # XXX in RPython, the msg is ignored at the moment
 
 dump_string = rmarshal.get_marshaller(str)

From noreply at buildbot.pypy.org  Thu Sep 18 13:55:25 2014
From: noreply at buildbot.pypy.org (Raemi)
Date: Thu, 18 Sep 2014 13:55:25 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-small-uniform: add some dummies to make
 duhton-rpython translate
Message-ID: <20140918115525.9AE2B1D3869@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: c8-small-uniform
Changeset: r1390:28b87c884221
Date: 2014-09-18 13:55 +0200
http://bitbucket.org/pypy/stmgc/changeset/28b87c884221/

Log:	add some dummies to make duhton-rpython translate

diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -232,6 +232,27 @@
 
 void stm_become_globally_unique_transaction(stm_thread_local_t *tl, const char *msg);
 
+
+/* dummies for now: */
+__attribute__((always_inline))
+static inline void stm_write_card(object_t *obj, uintptr_t index)
+{
+    stm_write(obj);
+}
+
+static inline object_t *stm_setup_prebuilt_weakref(object_t *pp)
+{
+    return stm_setup_prebuilt(pp);
+}
+
+static inline void stm_flush_timing(stm_thread_local_t *tl, int verbose) {}
 /* ==================== END ==================== */
 
+static void (*stmcb_expand_marker)(char *segment_base, uintptr_t odd_number,
+                            object_t *following_object,
+                            char *outputbuf, size_t outputbufsize);
+
+static void (*stmcb_debug_print)(const char *cause, double time,
+                          const char *marker);
+
 #endif

From noreply at buildbot.pypy.org  Thu Sep 18 16:54:18 2014
From: noreply at buildbot.pypy.org (Raemi)
Date: Thu, 18 Sep 2014 16:54:18 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-new-page-handling: setting up the initial
	configuration
Message-ID: <20140918145418.A6BB41D3869@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: c8-new-page-handling
Changeset: r1391:085233d40f72
Date: 2014-09-18 15:16 +0200
http://bitbucket.org/pypy/stmgc/changeset/085233d40f72/

Log:	setting up the initial configuration

diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -25,6 +25,7 @@
 #define FIRST_OBJECT_PAGE     ((READMARKER_END + 4095) / 4096UL)
 #define FIRST_NURSERY_PAGE    FIRST_OBJECT_PAGE
 #define END_NURSERY_PAGE      (FIRST_NURSERY_PAGE + NB_NURSERY_PAGES)
+#define NB_SHARED_PAGES       (NB_PAGES - END_NURSERY_PAGE)
 
 #define READMARKER_START      ((FIRST_OBJECT_PAGE * 4096UL) >> 4)
 #define FIRST_READMARKER_PAGE (READMARKER_START / 4096UL)
@@ -97,6 +98,7 @@
 
 
 static char *stm_object_pages;
+static char *stm_file_pages;
 static int stm_object_pages_fd;
 static stm_thread_local_t *stm_all_thread_locals = NULL;
 
diff --git a/c8/stm/setup.c b/c8/stm/setup.c
--- a/c8/stm/setup.c
+++ b/c8/stm/setup.c
@@ -4,32 +4,54 @@
 
 
 #include            /* For O_* constants */
-static char *setup_mmap(char *reason, int *map_fd)
+static void setup_mmap(char *reason)
 {
-    char name[128] = "/__stmgc_c8__";
+    char name[] = "/__stmgc_c8__";
 
     /* Create the big shared memory object, and immediately unlink it.
        There is a small window where if this process is killed the
        object is left around.  It doesn't seem possible to do anything
        about it...
     */
-    int fd = shm_open(name, O_RDWR | O_CREAT | O_EXCL, 0600);
+    stm_object_pages_fd = shm_open(name, O_RDWR | O_CREAT | O_EXCL, 0600);
     shm_unlink(name);
 
-    if (fd == -1) {
+    if (stm_object_pages_fd == -1)
         stm_fatalerror("%s failed (stm_open): %m", reason);
+
+    if (ftruncate(stm_object_pages_fd, NB_SHARED_PAGES) != 0)
+        stm_fatalerror("%s failed (ftruncate): %m", reason);
+
+    stm_file_pages = mmap(NULL, NB_SHARED_PAGES * 4096UL,
+                          PROT_READ | PROT_WRITE,
+                          MAP_SHARED | MAP_NORESERVE,
+                          stm_object_pages_fd, 0);
+
+    if (stm_file_pages == MAP_FAILED)
+        stm_fatalerror("%s failed (mmap): %m", reason);
+
+
+    /* reserve the whole virtual memory space of the program for
+       all segments: */
+    stm_object_pages = mmap(NULL, TOTAL_MEMORY,
+                            PROT_READ | PROT_WRITE,
+                            MAP_PRIVATE | MAP_NORESERVE | MAP_ANONYMOUS,
+                            -1, 0);
+    if (stm_object_pages == MAP_FAILED)
+        stm_fatalerror("%s failed (mmap): %m", reason);
+
+    /* remap the shared part of the segments to the file pages */
+    long l;
+    for (l = 0; l < NB_SEGMENTS; l++) {
+        char *result = mmap(
+            stm_object_pages + (l * NB_PAGES + END_NURSERY_PAGE) * 4096UL, /* addr */
+            NB_SHARED_PAGES * 4096UL, /* len */
+            PROT_READ | PROT_WRITE,
+            MAP_FIXED | MAP_SHARED | MAP_NORESERVE,
+            stm_object_pages_fd, 0); /* file & offset */
+        if (result == MAP_FAILED)
+            stm_fatalerror("%s failed (mmap): %m", reason);
     }
-    if (ftruncate(fd, TOTAL_MEMORY) != 0) {
-        stm_fatalerror("%s failed (ftruncate): %m", reason);
-    }
-    char *result = mmap(NULL, TOTAL_MEMORY,
-                        PROT_READ | PROT_WRITE,
-                        MAP_PAGES_FLAGS & ~MAP_ANONYMOUS, fd, 0);
-    if (result == MAP_FAILED) {
-        stm_fatalerror("%s failed (mmap): %m", reason);
-    }
-    *map_fd = fd;
-    return result;
 }
 static void close_fd_mmap(int map_fd)
 {
@@ -52,7 +74,7 @@
                      (FIRST_READMARKER_PAGE - 2) * 4096UL,
                      PROT_NONE);
 
-        /* STM_SEGMENT is in page 1 */
+        /* STM_SEGMENT-TL is in page 1 */
     }
 }
 
@@ -74,8 +96,12 @@
            (FIRST_READMARKER_PAGE * 4096UL));
     assert(_STM_FAST_ALLOC <= NB_NURSERY_PAGES * 4096);
 
-    stm_object_pages = setup_mmap("initial stm_object_pages mmap()",
-                                  &stm_object_pages_fd);
+    setup_mmap("initial stm_object_pages mmap()");
+
+    assert(stm_object_pages_fd);
+    assert(stm_object_pages);
+    assert(stm_file_pages);
+
     setup_protection_settings();
 
     long i;
diff --git a/c8/stm/setup.h b/c8/stm/setup.h
--- a/c8/stm/setup.h
+++ b/c8/stm/setup.h
@@ -1,4 +1,4 @@
-static char *setup_mmap(char *reason, int *map_fd);
+static void setup_mmap(char *reason);
 static void close_fd_mmap(int map_fd);
 static void setup_protection_settings(void);
 static pthread_t *_get_cpth(stm_thread_local_t *);

From noreply at buildbot.pypy.org  Thu Sep 18 16:54:19 2014
From: noreply at buildbot.pypy.org (Raemi)
Date: Thu, 18 Sep 2014 16:54:19 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-new-page-handling: some progress
Message-ID: <20140918145419.CEF7A1D3869@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: c8-new-page-handling
Changeset: r1392:5e2059e06b18
Date: 2014-09-18 16:54 +0200
http://bitbucket.org/pypy/stmgc/changeset/5e2059e06b18/

Log:	some progress

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -39,9 +39,8 @@
     char *realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj);
     uintptr_t pagenum = (uintptr_t)obj / 4096UL;
 
-    OPT_ASSERT(!is_shared_log_page(pagenum));
-    assert(is_private_log_page_in(STM_SEGMENT->segment_num, pagenum));
-    assert(is_private_log_page_in(from_seg, pagenum));
+    assert(get_page_status_in(from_seg, pagenum) != PAGE_NO_ACCESS);
+    assert(get_page_status_in(STM_SEGMENT->segment_num, pagenum) != PAGE_NO_ACCESS);
 
     /* look the obj up in the other segment's modified_old_objects to
        get its backup copy: */
@@ -218,18 +217,17 @@
     uintptr_t i;
     int my_segnum = STM_SEGMENT->segment_num;
 
-    assert(is_shared_log_page(pagenum));
-    char *src = (char*)(get_virt_page_of(0, pagenum) * 4096UL);
+    XXX();
+    //assert(is_shared_log_page(pagenum));
+    /* char *src = (char*)(get_virt_page_of(0, pagenum) * 4096UL); */
 
-    for (i = 1; i < NB_SEGMENTS; i++) {
-        assert(!is_private_log_page_in(i, pagenum));
+    /* for (i = 1; i < NB_SEGMENTS; i++) { */
+    /*     page_privatize_in(i, pagenum, src); */
+    /* } */
+    /* set_page_private_in(0, pagenum); */
 
-        page_privatize_in(i, pagenum, src);
-    }
-    set_page_private_in(0, pagenum);
-
-    OPT_ASSERT(is_private_log_page_in(my_segnum, pagenum));
-    assert(!is_shared_log_page(pagenum));
+    /* OPT_ASSERT(is_private_log_page_in(my_segnum, pagenum)); */
+    /* assert(!is_shared_log_page(pagenum)); */
 }
 
 void _stm_write_slowpath(object_t *obj)
@@ -259,23 +257,23 @@
 
     uintptr_t page;
     for (page = first_page; page <= end_page; page++) {
-        if (is_shared_log_page(page)) {
-            long i;
-            for (i = 0; i < NB_SEGMENTS; i++) {
-                acquire_privatization_lock(i);
-            }
-            if (is_shared_log_page(page))
-                _privatize_shared_page(page);
-            for (i = NB_SEGMENTS-1; i >= 0; i--) {
-                release_privatization_lock(i);
-            }
-        }
+        XXX();
+        /* if (is_shared_log_page(page)) { */
+        /*     long i; */
+        /*     for (i = 0; i < NB_SEGMENTS; i++) { */
+        /*         acquire_privatization_lock(i); */
+        /*     } */
+        /*     if (is_shared_log_page(page)) */
+        /*         _privatize_shared_page(page); */
+        /*     for (i = NB_SEGMENTS-1; i >= 0; i--) { */
+        /*         release_privatization_lock(i); */
+        /*     } */
+        /* } */
     }
     /* pages not shared anymore. but we still may have
        only a read protected page ourselves: */
 
     acquire_privatization_lock(my_segnum);
-    OPT_ASSERT(is_private_log_page_in(my_segnum, first_page));
 
     /* remove the WRITE_BARRIER flag */
     obj->stm_flags &= ~GCFLAG_WRITE_BARRIER;
diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c
--- a/c8/stm/gcpage.c
+++ b/c8/stm/gcpage.c
@@ -19,7 +19,8 @@
     for (i = 0; i < NB_SEGMENTS; i++) {
         acquire_privatization_lock(i);
     }
-    pages_initialize_shared((pages_addr - stm_object_pages) / 4096UL, num);
+    pages_initialize_shared_for(STM_SEGMENT->segment_num,
+                                (pages_addr - stm_object_pages) / 4096UL, num);
     for (i = NB_SEGMENTS-1; i >= 0; i--) {
         release_privatization_lock(i);
     }
diff --git a/c8/stm/pages.c b/c8/stm/pages.c
--- a/c8/stm/pages.c
+++ b/c8/stm/pages.c
@@ -11,12 +11,11 @@
 
 static void teardown_pages(void)
 {
-    memset(pages_privatized, 0, sizeof(pages_privatized));
+    memset(pages_status, 0, sizeof(pages_status));
 }
 
 /************************************************************/
-
-static void d_remap_file_pages(char *addr, size_t size, ssize_t pgoff)
+static void check_remap_makes_sense(char *addr, size_t size, ssize_t pgoff)
 {
     dprintf(("remap_file_pages: 0x%lx bytes: (seg%ld %p) --> (seg%ld %p)\n",
              (long)size,
@@ -35,21 +34,26 @@
     /* assert remappings follow the rule that page N in one segment
        can only be remapped to page N in another segment */
     assert(((addr - stm_object_pages) / 4096UL - pgoff) % NB_PAGES == 0);
+}
+
+static void d_remap_file_pages(char *addr, size_t size, ssize_t pgoff)
+{
+    check_remap_makes_sense(addr, size, pgoff);
 
     char *res = mmap(addr, size,
                      PROT_READ | PROT_WRITE,
-                     (MAP_PAGES_FLAGS & ~MAP_ANONYMOUS) | MAP_FIXED,
+                     MAP_PRIVATE | MAP_NORESERVE | MAP_FIXED,
                      stm_object_pages_fd, pgoff * 4096UL);
     if (UNLIKELY(res != addr))
         stm_fatalerror("mmap (remapping page): %m");
 }
 
 
-static void pages_initialize_shared(uintptr_t pagenum, uintptr_t count)
+static void pages_initialize_shared_for(long segnum, uintptr_t pagenum, uintptr_t count)
 {
     /* call remap_file_pages() to make all pages in the range(pagenum,
-       pagenum+count) refer to the same physical range of pages from
-       segment 0. */
+       pagenum+count) PAGE_SHARED in segnum, and PAGE_NO_ACCESS in other segments */
+
     dprintf(("pages_initialize_shared: 0x%ld - 0x%ld\n", pagenum,
              pagenum + count));
 #ifndef NDEBUG
@@ -61,62 +65,63 @@
     assert(pagenum < NB_PAGES);
     if (count == 0)
         return;
+
+    /* already shared after setup.c (also for the other count-1 pages) */
+    assert(get_page_status_in(segnum, pagenum) == PAGE_SHARED);
+
+    /* make other segments NO_ACCESS: */
     uintptr_t i;
-    for (i = 1; i < NB_SEGMENTS; i++) {
-        char *segment_base = get_segment_base(i);
-        d_remap_file_pages(segment_base + pagenum * 4096UL,
-                           count * 4096UL, pagenum);
-    }
+    for (i = 0; i < NB_SEGMENTS; i++) {
+        if (i != segnum) {
+            char *segment_base = get_segment_base(i);
+            mprotect(segment_base + pagenum * 4096UL,
+                     count * 4096UL, PROT_NONE);
+        }
 
-    for (i = 0; i < NB_SEGMENTS; i++) {
-        uintptr_t amount = count;
-        while (amount-->0) {
-            volatile struct page_shared_s *ps2 = (volatile struct page_shared_s *)
-                &pages_privatized[pagenum + amount - PAGE_FLAG_START];
-
-            ps2->by_segment = 0; /* not private */
-        }
+        long amount = count;
+        while (amount-->0)
+            set_page_status_in(i, pagenum + amount, PAGE_NO_ACCESS);
     }
 }
 
 
-static void page_privatize_in(int segnum, uintptr_t pagenum, char *initialize_from)
-{
-#ifndef NDEBUG
-    long l;
-    for (l = 0; l < NB_SEGMENTS; l++) {
-        assert(get_priv_segment(l)->privatization_lock);
-    }
-#endif
+/* static void page_privatize_in(int segnum, uintptr_t pagenum, char *initialize_from) */
+/* { */
+/* #ifndef NDEBUG */
+/*     long l; */
+/*     for (l = 0; l < NB_SEGMENTS; l++) { */
+/*         assert(get_priv_segment(l)->privatization_lock); */
+/*     } */
+/* #endif */
 
-    /* check this thread's 'pages_privatized' bit */
-    uint64_t bitmask = 1UL << segnum;
-    volatile struct page_shared_s *ps = (volatile struct page_shared_s *)
-        &pages_privatized[pagenum - PAGE_FLAG_START];
-    if (ps->by_segment & bitmask) {
-        /* the page is already privatized; nothing to do */
-        return;
-    }
+/*     /\* check this thread's 'pages_privatized' bit *\/ */
+/*     uint64_t bitmask = 1UL << segnum; */
+/*     volatile struct page_shared_s *ps = (volatile struct page_shared_s *) */
+/*         &pages_privatized[pagenum - PAGE_FLAG_START]; */
+/*     if (ps->by_segment & bitmask) { */
+/*         /\* the page is already privatized; nothing to do *\/ */
+/*         return; */
+/*     } */
 
-    dprintf(("page_privatize(%lu) in seg:%d\n", pagenum, segnum));
+/*     dprintf(("page_privatize(%lu) in seg:%d\n", pagenum, segnum)); */
 
-    /* add this thread's 'pages_privatized' bit */
-    ps->by_segment |= bitmask;
+/*     /\* add this thread's 'pages_privatized' bit *\/ */
+/*     ps->by_segment |= bitmask; */
 
-    /* "unmaps" the page to make the address space location correspond
-       again to its underlying file offset (XXX later we should again
-       attempt to group together many calls to d_remap_file_pages() in
-       succession) */
-    uintptr_t pagenum_in_file = NB_PAGES * segnum + pagenum;
-    char *new_page = stm_object_pages + pagenum_in_file * 4096UL;
+/*     /\* "unmaps" the page to make the address space location correspond */
+/*        again to its underlying file offset (XXX later we should again */
+/*        attempt to group together many calls to d_remap_file_pages() in */
+/*        succession) *\/ */
+/*     uintptr_t pagenum_in_file = NB_PAGES * segnum + pagenum; */
+/*     char *new_page = stm_object_pages + pagenum_in_file * 4096UL; */
 
-    /* first write to the file page directly: */
-    ssize_t written = pwrite(stm_object_pages_fd, initialize_from, 4096UL,
-                             pagenum_in_file * 4096UL);
-    if (written != 4096)
-        stm_fatalerror("pwrite didn't write the whole page: %zd", written);
+/*     /\* first write to the file page directly: *\/ */
+/*     ssize_t written = pwrite(stm_object_pages_fd, initialize_from, 4096UL, */
+/*                              pagenum_in_file * 4096UL); */
+/*     if (written != 4096) */
+/*         stm_fatalerror("pwrite didn't write the whole page: %zd", written); */
 
-    /* now remap virtual page in segment to the new file page */
-    write_fence();
-    d_remap_file_pages(new_page, 4096, pagenum_in_file);
-}
+/*     /\* now remap virtual page in segment to the new file page *\/ */
+/*     write_fence(); */
+/*     d_remap_file_pages(new_page, 4096, pagenum_in_file); */
+/* } */
diff --git a/c8/stm/pages.h b/c8/stm/pages.h
--- a/c8/stm/pages.h
+++ b/c8/stm/pages.h
@@ -20,51 +20,61 @@
 
 #define PAGE_FLAG_START   END_NURSERY_PAGE
 #define PAGE_FLAG_END     NB_PAGES
+/* == NB_SHARED_PAGES */
 
 
 struct page_shared_s {
-#if NB_SEGMENTS <= 8
+#if NB_SEGMENTS <= 4
     uint8_t by_segment;
+#elif NB_SEGMENTS <= 8
+    uint16_t by_segment;
 #elif NB_SEGMENTS <= 16
-    uint16_t by_segment;
+    uint32_t by_segment;
 #elif NB_SEGMENTS <= 32
-    uint32_t by_segment;
-#elif NB_SEGMENTS <= 64
     uint64_t by_segment;
 #else
-#   error "NB_SEGMENTS > 64 not supported right now"
+#   error "NB_SEGMENTS > 32 not supported right now"
 #endif
 };
 
-static struct page_shared_s pages_privatized[PAGE_FLAG_END - PAGE_FLAG_START];
+enum {
+    PAGE_SHARED = 0,
+    PAGE_PRIVATE,
+    PAGE_NO_ACCESS,
+};
 
-static void pages_initialize_shared(uintptr_t pagenum, uintptr_t count);
-static void page_privatize_in(int segnum, uintptr_t pagenum, char *initialize_from);
+static struct page_shared_s pages_status[NB_SHARED_PAGES];
+
+static void pages_initialize_shared_for(long segnum, uintptr_t pagenum, uintptr_t count);
+/* static void page_privatize_in(int segnum, uintptr_t pagenum, char *initialize_from); */
+
+
+static inline uint8_t get_page_status_in(long segnum, uintptr_t pagenum)
+{
+    int seg_shift = segnum * 2;
+    uint64_t bitmask = 3UL << seg_shift;
+    volatile struct page_shared_s *ps = (volatile struct page_shared_s *)
+        &pages_status[pagenum - PAGE_FLAG_START];
+
+    return ((ps->by_segment & bitmask) >> seg_shift) & 3;
+}
+
+static inline void set_page_status_in(long segnum, uintptr_t pagenum, uint8_t status)
+{
+    OPT_ASSERT((status & 3) == status);
+
+    int seg_shift = segnum * 2;
+    volatile struct page_shared_s *ps = (volatile struct page_shared_s *)
+        &pages_status[pagenum - PAGE_FLAG_START];
+
+    assert(status != get_page_status_in(segnum, pagenum));
+    ps->by_segment |= status << seg_shift;
+}
+
+
 
 static inline uintptr_t get_virt_page_of(long segnum, uintptr_t pagenum)
 {
     /* logical page -> virtual page */
     return (uintptr_t)get_segment_base(segnum) / 4096UL + pagenum;
 }
-
-static inline bool is_shared_log_page(uintptr_t pagenum)
-{
-    assert(pagenum >= PAGE_FLAG_START);
-    return pages_privatized[pagenum - PAGE_FLAG_START].by_segment == 0;
-}
-
-static inline void set_page_private_in(long segnum, uintptr_t pagenum)
-{
-    uint64_t bitmask = 1UL << segnum;
-    volatile struct page_shared_s *ps = (volatile struct page_shared_s *)
-        &pages_privatized[pagenum - PAGE_FLAG_START];
-    assert(!(ps->by_segment & bitmask));
-    ps->by_segment |= bitmask;
-}
-
-static inline bool is_private_log_page_in(long segnum, uintptr_t pagenum)
-{
-    assert(pagenum >= PAGE_FLAG_START);
-    uint64_t bitmask = 1UL << segnum;
-    return (pages_privatized[pagenum - PAGE_FLAG_START].by_segment & bitmask);
-}

From noreply at buildbot.pypy.org  Thu Sep 18 17:47:00 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Thu, 18 Sep 2014 17:47:00 +0200 (CEST)
Subject: [pypy-commit] pypy.org extradoc: misleading typo
Message-ID: <20140918154700.190091C337D@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: extradoc
Changeset: r536:61102321c9cb
Date: 2014-09-18 18:46 +0300
http://bitbucket.org/pypy/pypy.org/changeset/61102321c9cb/

Log:	misleading typo

diff --git a/download.html b/download.html
--- a/download.html
+++ b/download.html
@@ -54,7 +54,7 @@
 
 
      
 
    • Download
        
@@ -121,8 +121,8 @@
 mirror, but please use only if you have troubles accessing the links above
    • -
    -

    Python 3.3 compatible PyPy3 2.3.1

    +
    +

    Python 3.2.5 compatible PyPy3 2.3.1

    • Linux x86 binary (32bit, tar.bz2 built on Ubuntu 10.04.4 LTS) (see [1] below)
    • Linux x86-64 binary (64bit, tar.bz2 built on Ubuntu 12.04 - 14.04) (see [1] below)
      • diff --git a/source/download.txt b/source/download.txt --- a/source/download.txt +++ b/source/download.txt @@ -18,7 +18,7 @@ * the Python2.7 compatible release — **PyPy 2.3.1** — (`what's new in PyPy 2.3.1?`_ ) * the Python2.7 compatible beta — **PyPy 2.4-beta1** — (`what's new in PyPy 2.4?`_ ) -* the Python3.3 compatible release — **PyPy3 2.3.1** — (`what's new in PyPy3 2.3.1?`_). +* the Python3.2.5 compatible release — **PyPy3 2.3.1** — (`what's new in PyPy3 2.3.1?`_). .. _what's new in PyPy 2.3.1?: http://doc.pypy.org/en/latest/release-2.3.1.html .. _what's new in PyPy3 2.3.1?: http://doc.pypy.org/en/latest/release-pypy3-2.3.1.html @@ -127,8 +127,8 @@ .. __: https://bitbucket.org/pypy/pypy/downloads .. _mirror: http://cobra.cs.uni-duesseldorf.de/~buildmaster/mirror/ -Python 3.3 compatible PyPy3 2.3.1 ---------------------------------- +Python 3.2.5 compatible PyPy3 2.3.1 +----------------------------------- * `Linux x86 binary (32bit, tar.bz2 built on Ubuntu 10.04.4 LTS)`__ (see ``[1]`` below) * `Linux x86-64 binary (64bit, tar.bz2 built on Ubuntu 12.04 - 14.04)`__ (see ``[1]`` below) From noreply at buildbot.pypy.org Thu Sep 18 19:01:45 2014 From: noreply at buildbot.pypy.org (pjenvey) Date: Thu, 18 Sep 2014 19:01:45 +0200 (CEST) Subject: [pypy-commit] pypy default: pep8/cleanup Message-ID: <20140918170145.E986C1D3869@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: Changeset: r73602:832991d1e798 Date: 2014-09-17 16:54 -0700 http://bitbucket.org/pypy/pypy/changeset/832991d1e798/ Log: pep8/cleanup diff --git a/rpython/rtyper/lltypesystem/lltype.py b/rpython/rtyper/lltypesystem/lltype.py --- a/rpython/rtyper/lltypesystem/lltype.py +++ b/rpython/rtyper/lltypesystem/lltype.py @@ -1,16 +1,17 @@ -from types import NoneType, MethodType import weakref +from types import MethodType, NoneType + +from rpython.annotator.bookkeeper import analyzer_for, immutablevalue from rpython.annotator.model import ( - SomeInteger, SomeBool, SomeObject, AnnotatorError) + AnnotatorError, SomeBool, SomeInteger, SomeObject) +from rpython.rlib.objectmodel import Symbolic from rpython.rlib.rarithmetic import ( - r_int, r_uint, intmask, r_singlefloat, r_ulonglong, r_longlong, - r_longfloat, r_longlonglong, base_int, normalizedinttype, longlongmask, - longlonglongmask, maxint, is_valid_int, is_emulated_long) -from rpython.rlib.objectmodel import Symbolic + base_int, intmask, is_emulated_long, is_valid_int, longlonglongmask, + longlongmask, maxint, normalizedinttype, r_int, r_longfloat, r_longlong, + r_longlonglong, r_singlefloat, r_uint, r_ulonglong) +from rpython.rtyper.extregistry import ExtRegistryEntry +from rpython.tool import leakfinder from rpython.tool.identity_dict import identity_dict -from rpython.tool import leakfinder -from rpython.annotator.bookkeeper import analyzer_for, immutablevalue -from rpython.rtyper.extregistry import ExtRegistryEntry class State(object): pass @@ -313,14 +314,12 @@ except KeyError: return ContainerType.__getattr__(self, name) - def _nofield(self, name): raise AttributeError('struct %s has no field %r' % (self._name, name)) def _names_without_voids(self): - names_without_voids = [name for name in self._names if self._flds[name] is not Void] - return names_without_voids + return [name for name in self._names if self._flds[name] is not Void] def _str_fields_without_voids(self): return ', '.join(['%s: %s' % (name, self._flds[name]) @@ -576,8 +575,10 @@ _gckind = 'raw' def __init__(self, tag, hints={}): - """ if hints['render_structure'] is set, the type is internal and not considered - to come from somewhere else (it should be rendered as a structure) """ + """If hints['render_structure'] is set, the type is internal and + not considered to come from somewhere else (it should be + rendered as a structure) + """ self.tag = tag self.__name__ = tag self.hints = frozendict(hints) @@ -675,7 +676,8 @@ _numbertypes = {int: Number("Signed", int, intmask)} _numbertypes[r_int] = _numbertypes[int] -_numbertypes[r_longlonglong] = Number("SignedLongLongLong", r_longlonglong, longlonglongmask) +_numbertypes[r_longlonglong] = Number("SignedLongLongLong", r_longlonglong, + longlonglongmask) if r_longlong is not r_int: _numbertypes[r_longlong] = Number("SignedLongLong", r_longlong, longlongmask) @@ -702,8 +704,8 @@ UnsignedLongLong = build_number("UnsignedLongLong", r_ulonglong) Float = Primitive("Float", 0.0) # C type 'double' -SingleFloat = Primitive("SingleFloat", r_singlefloat(0.0)) # C type 'float' -LongFloat = Primitive("LongFloat", r_longfloat(0.0)) # C type 'long double' +SingleFloat = Primitive("SingleFloat", r_singlefloat(0.0)) # 'float' +LongFloat = Primitive("LongFloat", r_longfloat(0.0)) # 'long double' r_singlefloat._TYPE = SingleFloat Char = Primitive("Char", '\x00') @@ -876,9 +878,11 @@ @analyzer_for(cast_primitive) def ann_cast_primitive(T, s_v): - from rpython.rtyper.llannotation import annotation_to_lltype, ll_to_annotation + from rpython.rtyper.llannotation import ( + annotation_to_lltype, ll_to_annotation) assert T.is_constant() - return ll_to_annotation(cast_primitive(T.const, annotation_to_lltype(s_v)._defl())) + return ll_to_annotation(cast_primitive(T.const, + annotation_to_lltype(s_v)._defl())) def _cast_whatever(TGT, value): @@ -905,7 +909,8 @@ elif TGT == llmemory.Address and isinstance(ORIG, Ptr): return llmemory.cast_ptr_to_adr(value) elif TGT == Signed and isinstance(ORIG, Ptr) and ORIG.TO._gckind == 'raw': - return llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(value), 'symbolic') + return llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(value), + 'symbolic') raise TypeError("don't know how to cast from %r to %r" % (ORIG, TGT)) @@ -1176,8 +1181,8 @@ except DelayedPointer: return True # assume it's not a delayed null - # _setobj, _getobj and _obj0 are really _internal_ implementations details of _ptr, - # use _obj if necessary instead ! + # _setobj, _getobj and _obj0 are really _internal_ implementations + # details of _ptr, use _obj if necessary instead ! def _setobj(self, pointing_to, solid=False): if pointing_to is None: obj0 = None @@ -1244,12 +1249,12 @@ if T1 == T2: setattr(self._obj, field_name, val) else: - raise TypeError("%r instance field %r:\n" - "expects %r\n" - " got %r" % (self._T, field_name, T1, T2)) + raise TypeError( + "%r instance field %r:\nexpects %r\n got %r" % + (self._T, field_name, T1, T2)) return - raise AttributeError("%r instance has no field %r" % (self._T, - field_name)) + raise AttributeError("%r instance has no field %r" % + (self._T, field_name)) def __getitem__(self, i): # ! can only return basic or ptr ! if isinstance(self._T, (Array, FixedSizeArray)): @@ -1266,7 +1271,8 @@ if isinstance(self._T, (Array, FixedSizeArray)): T1 = self._T.OF if isinstance(T1, ContainerType): - raise TypeError("cannot directly assign to container array items") + raise TypeError("cannot directly assign to container array " + "items") T2 = typeOf(val) if T2 != T1: from rpython.rtyper.lltypesystem import rffi @@ -1316,7 +1322,8 @@ from rpython.rtyper.lltypesystem import rffi if isinstance(self._T, FuncType): if len(args) != len(self._T.ARGS): - raise TypeError("calling %r with wrong argument number: %r" % (self._T, args)) + raise TypeError("calling %r with wrong argument number: %r" % + (self._T, args)) for i, a, ARG in zip(range(len(self._T.ARGS)), args, self._T.ARGS): if typeOf(a) != ARG: # ARG could be Void @@ -1415,11 +1422,13 @@ raise RuntimeError("widening to trash: %r" % self) PARENTTYPE = struc._parent_type if getattr(parent, PARENTTYPE._names[0]) != struc: - raise InvalidCast(CURTYPE, PTRTYPE) # xxx different exception perhaps? + # xxx different exception perhaps? + raise InvalidCast(CURTYPE, PTRTYPE) struc = parent u -= 1 if PARENTTYPE != PTRTYPE.TO: - raise RuntimeError("widening %r inside %r instead of %r" % (CURTYPE, PARENTTYPE, PTRTYPE.TO)) + raise RuntimeError("widening %r inside %r instead of %r" % + (CURTYPE, PARENTTYPE, PTRTYPE.TO)) return _ptr(PTRTYPE, struc, solid=self._solid) def _cast_to_int(self, check=True): @@ -1430,7 +1439,9 @@ return obj # special case for cast_int_to_ptr() results obj = normalizeptr(self, check)._getobj(check) if isinstance(obj, int): - return obj # special case for cast_int_to_ptr() results put into opaques + # special case for cast_int_to_ptr() results put into + # opaques + return obj if getattr(obj, '_read_directly_intval', False): return obj.intval # special case for _llgcopaque result = intmask(obj._getid()) @@ -1468,7 +1479,8 @@ """XXX A nice docstring here""" T = typeOf(val) if isinstance(T, ContainerType): - if self._T._gckind == 'gc' and T._gckind == 'raw' and not isinstance(T, OpaqueType): + if (self._T._gckind == 'gc' and T._gckind == 'raw' and + not isinstance(T, OpaqueType)): val = _interior_ptr(T, self._obj, [offset]) else: val = _ptr(Ptr(T), val, solid=self._solid) @@ -1531,12 +1543,14 @@ setattr(example, s_attr.const, v_lltype._defl()) def call(self, args): - from rpython.rtyper.llannotation import annotation_to_lltype, ll_to_annotation + from rpython.rtyper.llannotation import ( + annotation_to_lltype, ll_to_annotation) args_s, kwds_s = args.unpack() if kwds_s: raise Exception("keyword arguments to call to a low-level fn ptr") info = 'argument to ll function pointer call' - llargs = [annotation_to_lltype(s_arg, info)._defl() for s_arg in args_s] + llargs = [annotation_to_lltype(s_arg, info)._defl() + for s_arg in args_s] v = self.ll_ptrtype._example()(*llargs) return ll_to_annotation(v) @@ -1593,7 +1607,6 @@ return val - assert not '__dict__' in dir(_interior_ptr) class _container(object): @@ -1721,11 +1734,13 @@ __slots__ = ('_hash_cache_', '_compilation_info') - def __new__(self, TYPE, n=None, initialization=None, parent=None, parentindex=None): + def __new__(self, TYPE, n=None, initialization=None, parent=None, + parentindex=None): my_variety = _struct_variety(TYPE._names) return object.__new__(my_variety) - def __init__(self, TYPE, n=None, initialization=None, parent=None, parentindex=None): + def __init__(self, TYPE, n=None, initialization=None, parent=None, + parentindex=None): _parentable.__init__(self, TYPE) if n is not None and TYPE._arrayfld is None: raise TypeError("%r is not variable-sized" % (TYPE,)) @@ -1734,9 +1749,11 @@ first, FIRSTTYPE = TYPE._first_struct() for fld, typ in TYPE._flds.items(): if fld == TYPE._arrayfld: - value = _array(typ, n, initialization=initialization, parent=self, parentindex=fld) + value = _array(typ, n, initialization=initialization, + parent=self, parentindex=fld) else: - value = typ._allocate(initialization=initialization, parent=self, parentindex=fld) + value = typ._allocate(initialization=initialization, + parent=self, parentindex=fld) setattr(self, fld, value) if parent is not None: self._setparentstructure(parent, parentindex) @@ -1795,7 +1812,8 @@ __slots__ = ('items',) - def __init__(self, TYPE, n, initialization=None, parent=None, parentindex=None): + def __init__(self, TYPE, n, initialization=None, parent=None, + parentindex=None): if not is_valid_int(n): raise TypeError("array length must be an int") if n < 0: @@ -1964,7 +1982,8 @@ if not key._was_freed(): newcache[key] = value except RuntimeError: - pass # ignore "accessing subxxx, but already gc-ed parent" + # ignore "accessing subxxx, but already gc-ed parent" + pass if newcache: _subarray._cache[T] = newcache else: @@ -2020,8 +2039,10 @@ attrs.setdefault('_name', '?') attrs.setdefault('_callable', None) self.__dict__.update(attrs) - if '_callable' in attrs and hasattr(attrs['_callable'], '_compilation_info'): - self.__dict__['compilation_info'] = attrs['_callable']._compilation_info + if '_callable' in attrs and hasattr(attrs['_callable'], + '_compilation_info'): + self.__dict__['compilation_info'] = \ + attrs['_callable']._compilation_info def __repr__(self): return '<%s>' % (self,) @@ -2126,8 +2147,8 @@ return _ptr(Ptr(T), o, solid) @analyzer_for(malloc) -def ann_malloc(s_T, s_n=None, s_flavor=None, s_zero=None, s_track_allocation=None, - s_add_memory_pressure=None): +def ann_malloc(s_T, s_n=None, s_flavor=None, s_zero=None, + s_track_allocation=None, s_add_memory_pressure=None): assert (s_n is None or s_n.knowntype == int or issubclass(s_n.knowntype, base_int)) assert s_T.is_constant() @@ -2303,7 +2324,8 @@ @analyzer_for(runtime_type_info) def ann_runtime_type_info(s_p): - assert isinstance(s_p, SomePtr), "runtime_type_info of non-pointer: %r" % s_p + assert isinstance(s_p, SomePtr), \ + "runtime_type_info of non-pointer: %r" % s_p return SomePtr(typeOf(runtime_type_info(s_p.ll_ptrtype._example()))) From noreply at buildbot.pypy.org Thu Sep 18 19:01:47 2014 From: noreply at buildbot.pypy.org (pjenvey) Date: Thu, 18 Sep 2014 19:01:47 +0200 (CEST) Subject: [pypy-commit] pypy py3k: merge default Message-ID: <20140918170147.76D1D1D3869@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r73603:45600f667251 Date: 2014-09-17 17:30 -0700 http://bitbucket.org/pypy/pypy/changeset/45600f667251/ Log: merge default diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py --- a/pypy/interpreter/pyparser/parsestring.py +++ b/pypy/interpreter/pyparser/parsestring.py @@ -82,12 +82,6 @@ v = PyString_DecodeEscape(space, substr, 'strict', encoding) return space.wrapbytes(v) -def hexbyte(val): - result = "%x" % val - if len(result) == 1: - result = "0" + result - return result - def decode_unicode_utf8(space, s, ps, q): # ****The Python 2.7 version, producing UTF-32 escapes**** # String is utf8-encoded, but 'unicode_escape' expects @@ -107,15 +101,14 @@ # instead. lis.append("u005c") if ord(s[ps]) & 0x80: # XXX inefficient - w, ps = decode_utf8(space, s, ps, end, "utf-32-be") - rn = len(w) - assert rn % 4 == 0 - for i in range(0, rn, 4): - lis.append('\\U') - lis.append(hexbyte(ord(w[i]))) - lis.append(hexbyte(ord(w[i+1]))) - lis.append(hexbyte(ord(w[i+2]))) - lis.append(hexbyte(ord(w[i+3]))) + w, ps = decode_utf8(space, s, ps, end) + for c in w: + # The equivalent of %08x, which is not supported by RPython. + # 7 zeroes are enough for the unicode range, and the + # result still fits in 32-bit. + hexa = hex(ord(c) + 0x10000000) + lis.append('\\U0') + lis.append(hexa[3:]) # Skip 0x and the leading 1 else: lis.append(s[ps]) ps += 1 @@ -135,7 +128,7 @@ # note that the C code has a label here. # the logic is the same. if recode_encoding and ord(s[ps]) & 0x80: - w, ps = decode_utf8(space, s, ps, end, recode_encoding) + w, ps = decode_utf8_recode(space, s, ps, end, recode_encoding) # Append bytes to output buffer. builder.append(w) else: @@ -222,14 +215,18 @@ ch >= 'A' and ch <= 'F') -def decode_utf8(space, s, ps, end, encoding): +def decode_utf8(space, s, ps, end): assert ps >= 0 pt = ps # while (s < end && *s != '\\') s++; */ /* inefficient for u".." while ps < end and ord(s[ps]) & 0x80: ps += 1 - w_u = space.wrap(unicodehelper.decode_utf8(space, s[pt:ps])) - w_v = unicodehelper.encode(space, w_u, encoding) + u = unicodehelper.decode_utf8(space, s[pt:ps]) + return u, ps + +def decode_utf8_recode(space, s, ps, end, recode_encoding): + u, ps = decode_utf8(space, s, ps, end) + w_v = unicodehelper.encode(space, space.wrap(u), recode_encoding) v = space.bytes_w(w_v) return v, ps diff --git a/pypy/interpreter/pyparser/test/test_parsestring.py b/pypy/interpreter/pyparser/test/test_parsestring.py --- a/pypy/interpreter/pyparser/test/test_parsestring.py +++ b/pypy/interpreter/pyparser/test/test_parsestring.py @@ -100,11 +100,11 @@ def test_simple_enc_roundtrip(self): space = self.space - s = "'\x81'" + s = "'\x81\\t'" s = s.decode("koi8-u").encode("utf8") w_ret = parsestring.parsestr(self.space, 'koi8-u', s) ret = space.unwrap(w_ret) - assert ret == eval("# -*- coding: koi8-u -*-\nu'\x81'") + assert ret == eval("# -*- coding: koi8-u -*-\nu'\x81\\t'") def test_multiline_unicode_strings_with_backslash(self): space = self.space diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py --- a/pypy/interpreter/unicodehelper.py +++ b/pypy/interpreter/unicodehelper.py @@ -13,6 +13,7 @@ @specialize.memo() def decode_error_handler(space): + # Fast version of the "strict" errors handler. def raise_unicode_exception_decode(errors, encoding, msg, s, startingpos, endingpos): raise OperationError(space.w_UnicodeDecodeError, @@ -25,6 +26,7 @@ @specialize.memo() def encode_error_handler(space): + # Fast version of the "strict" errors handler. def raise_unicode_exception_encode(errors, encoding, msg, u, startingpos, endingpos): raise OperationError(space.w_UnicodeEncodeError, diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py --- a/pypy/module/_cffi_backend/ctypefunc.py +++ b/pypy/module/_cffi_backend/ctypefunc.py @@ -8,6 +8,7 @@ from rpython.rlib.jit_libffi import (CIF_DESCRIPTION, CIF_DESCRIPTION_P, FFI_TYPE, FFI_TYPE_P, FFI_TYPE_PP, SIZE_OF_FFI_ARG) from rpython.rlib.objectmodel import we_are_translated, instantiate +from rpython.rlib.objectmodel import keepalive_until_here from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from pypy.interpreter.error import OperationError, oefmt @@ -160,6 +161,7 @@ raw_cdata = rffi.cast(rffi.CCHARPP, data)[0] lltype.free(raw_cdata, flavor='raw') lltype.free(buffer, flavor='raw') + keepalive_until_here(args_w) return w_res def get_mustfree_flag(data): diff --git a/pypy/module/operator/app_operator.py b/pypy/module/operator/app_operator.py --- a/pypy/module/operator/app_operator.py +++ b/pypy/module/operator/app_operator.py @@ -4,7 +4,8 @@ This module exports a set of operators as functions. E.g. operator.add(x,y) is equivalent to x+y. ''' -from __pypy__ import builtinify + +import types def countOf(a,b): @@ -15,51 +16,78 @@ count += 1 return count +def _resolve_attr_chain(chain, obj, idx=0): + obj = getattr(obj, chain[idx]) + if idx + 1 == len(chain): + return obj + else: + return _resolve_attr_chain(chain, obj, idx + 1) + + +class _simple_attrgetter(object): + def __init__(self, attr): + self._attr = attr + + def __call__(self, obj): + return getattr(obj, self._attr) + + +class _single_attrgetter(object): + def __init__(self, attrs): + self._attrs = attrs + + def __call__(self, obj): + return _resolve_attr_chain(self._attrs, obj) + + +class _multi_attrgetter(object): + def __init__(self, attrs): + self._attrs = attrs + + def __call__(self, obj): + return tuple([ + _resolve_attr_chain(attrs, obj) + for attrs in self._attrs + ]) + + def attrgetter(attr, *attrs): + if ( + not isinstance(attr, str) or + not all(isinstance(a, str) for a in attrs) + ): + raise TypeError("attribute name must be a string, not %r" % + type(attr).__name__) if attrs: - getters = [single_attr_getter(a) for a in (attr,) + attrs] - def getter(obj): - return tuple([getter(obj) for getter in getters]) + return _multi_attrgetter([ + a.split(".") for a in [attr] + list(attrs) + ]) + elif "." not in attr: + return _simple_attrgetter(attr) else: - getter = single_attr_getter(attr) - return builtinify(getter) + return _single_attrgetter(attr.split(".")) -def single_attr_getter(attr): - if not isinstance(attr, str): - raise TypeError("attribute name must be a string, not {!r}".format( - type(attr).__name__)) - # - def make_getter(name, prevfn=None): - if prevfn is None: - def getter(obj): - return getattr(obj, name) + +class itemgetter(object): + def __init__(self, item, *items): + self._single = not bool(items) + if self._single: + self._idx = item else: - def getter(obj): - return getattr(prevfn(obj), name) - return getter - # - last = 0 - getter = None - while True: - dot = attr.find(".", last) - if dot < 0: break - getter = make_getter(attr[last:dot], getter) - last = dot + 1 - return make_getter(attr[last:], getter) + self._idx = [item] + list(items) + def __call__(self, obj): + if self._single: + return obj[self._idx] + else: + return tuple([obj[i] for i in self._idx]) -def itemgetter(item, *items): - if items: - list_of_indices = [item] + list(items) - def getter(obj): - return tuple([obj[i] for i in list_of_indices]) - else: - def getter(obj): - return obj[item] - return builtinify(getter) +class methodcaller(object): + def __init__(self, method_name, *args, **kwargs): + self._method_name = method_name + self._args = args + self._kwargs = kwargs -def methodcaller(method_name, *args, **kwargs): - def call(obj): - return getattr(obj, method_name)(*args, **kwargs) - return builtinify(call) + def __call__(self, obj): + return getattr(obj, self._method_name)(*self._args, **self._kwargs) diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py --- a/pypy/module/sys/test/test_sysmodule.py +++ b/pypy/module/sys/test/test_sysmodule.py @@ -127,9 +127,9 @@ assert isinstance(sys.__stdin__, io.IOBase) assert sys.__stderr__.errors == 'backslashreplace' - assert sys.__stdin__.name == "" - assert sys.__stdout__.name == "" - assert sys.__stderr__.name == "" + #assert sys.__stdin__.name == "" + #assert sys.__stdout__.name == "" + #assert sys.__stderr__.name == "" if self.appdirect and not isinstance(sys.stdin, io.IOBase): return diff --git a/rpython/jit/backend/arm/callbuilder.py b/rpython/jit/backend/arm/callbuilder.py --- a/rpython/jit/backend/arm/callbuilder.py +++ b/rpython/jit/backend/arm/callbuilder.py @@ -92,7 +92,8 @@ self.mc.LDR_ri(r.r7.value, r.r5.value) # change 'rpy_fastgil' to 0 (it should be non-zero right now) - self.mc.DMB() + if self.asm.cpu.cpuinfo.arch_version >= 7: + self.mc.DMB() self.mc.gen_load_int(r.r6.value, fastgil) self.mc.MOV_ri(r.ip.value, 0) self.mc.STR_ri(r.ip.value, r.r6.value) @@ -112,7 +113,8 @@ self.mc.STREX(r.r3.value, r.ip.value, r.r6.value, c=c.EQ) # try to claim the lock self.mc.CMP_ri(r.r3.value, 0, cond=c.EQ) # did this succeed? - self.mc.DMB() + if self.asm.cpu.cpuinfo.arch_version >= 7: + self.mc.DMB() # the success of the lock acquisition is defined by # 'EQ is true', or equivalently by 'r3 == 0'. # diff --git a/rpython/jit/backend/arm/codebuilder.py b/rpython/jit/backend/arm/codebuilder.py --- a/rpython/jit/backend/arm/codebuilder.py +++ b/rpython/jit/backend/arm/codebuilder.py @@ -333,6 +333,8 @@ | (rn & 0xF) << 16) def DMB(self): + # ARMv7 only. I guess ARMv6 CPUs cannot be used in symmetric + # multi-processing at all? That would make this instruction unneeded. # note: 'cond' is only permitted on Thumb here, but don't # write literally 0xf57ff05f, because it's larger than 31 bits c = cond.AL diff --git a/rpython/jit/backend/arm/instructions.py b/rpython/jit/backend/arm/instructions.py --- a/rpython/jit/backend/arm/instructions.py +++ b/rpython/jit/backend/arm/instructions.py @@ -142,6 +142,7 @@ #'VCVT' : {'opc1':0xB, 'opc2':0xE, 'opc3':0x1, 'base': False}, } +# ARMv7 only simd_instructions_3regs = { 'VADD_i64': {'A': 0x8, 'B': 0, 'U': 0}, 'VSUB_i64': {'A': 0x8, 'B': 0, 'U': 1}, diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -1,3 +1,4 @@ +from rpython.rlib import rgc from rpython.rlib.rarithmetic import ovfcheck from rpython.rtyper.lltypesystem import llmemory from rpython.jit.metainterp import history @@ -390,8 +391,8 @@ val = op.getarg(0) if val not in self.write_barrier_applied: v = op.getarg(1) - if isinstance(v, BoxPtr) or (isinstance(v, ConstPtr) and - bool(v.value)): # store a non-NULL + if (isinstance(v, BoxPtr) or (isinstance(v, ConstPtr) and + rgc.needs_write_barrier(v.value))): self.gen_write_barrier(val) #op = op.copy_and_change(rop.SETFIELD_RAW) self.newops.append(op) @@ -400,8 +401,8 @@ val = op.getarg(0) if val not in self.write_barrier_applied: v = op.getarg(2) - if isinstance(v, BoxPtr) or (isinstance(v, ConstPtr) and - bool(v.value)): # store a non-NULL + if (isinstance(v, BoxPtr) or (isinstance(v, ConstPtr) and + rgc.needs_write_barrier(v.value))): self.gen_write_barrier_array(val, op.getarg(1)) #op = op.copy_and_change(rop.SET{ARRAYITEM,INTERIORFIELD}_RAW) self.newops.append(op) diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py --- a/rpython/rlib/rgc.py +++ b/rpython/rlib/rgc.py @@ -86,6 +86,14 @@ collect(i) i += 1 +def needs_write_barrier(obj): + """ We need to emit write barrier if the right hand of assignment + is in nursery, used by the JIT for handling set*_gc(Const) + """ + if not obj: + return False + return can_move(obj) + def _heap_stats(): raise NotImplementedError # can't be run directly diff --git a/rpython/rtyper/lltypesystem/lltype.py b/rpython/rtyper/lltypesystem/lltype.py --- a/rpython/rtyper/lltypesystem/lltype.py +++ b/rpython/rtyper/lltypesystem/lltype.py @@ -1,16 +1,17 @@ -from types import NoneType, MethodType import weakref +from types import MethodType, NoneType + +from rpython.annotator.bookkeeper import analyzer_for, immutablevalue from rpython.annotator.model import ( - SomeInteger, SomeBool, SomeObject, AnnotatorError) + AnnotatorError, SomeBool, SomeInteger, SomeObject) +from rpython.rlib.objectmodel import Symbolic from rpython.rlib.rarithmetic import ( - r_int, r_uint, intmask, r_singlefloat, r_ulonglong, r_longlong, - r_longfloat, r_longlonglong, base_int, normalizedinttype, longlongmask, - longlonglongmask, maxint, is_valid_int, is_emulated_long) -from rpython.rlib.objectmodel import Symbolic + base_int, intmask, is_emulated_long, is_valid_int, longlonglongmask, + longlongmask, maxint, normalizedinttype, r_int, r_longfloat, r_longlong, + r_longlonglong, r_singlefloat, r_uint, r_ulonglong) +from rpython.rtyper.extregistry import ExtRegistryEntry +from rpython.tool import leakfinder from rpython.tool.identity_dict import identity_dict -from rpython.tool import leakfinder -from rpython.annotator.bookkeeper import analyzer_for, immutablevalue -from rpython.rtyper.extregistry import ExtRegistryEntry class State(object): pass @@ -313,14 +314,12 @@ except KeyError: return ContainerType.__getattr__(self, name) - def _nofield(self, name): raise AttributeError('struct %s has no field %r' % (self._name, name)) def _names_without_voids(self): - names_without_voids = [name for name in self._names if self._flds[name] is not Void] - return names_without_voids + return [name for name in self._names if self._flds[name] is not Void] def _str_fields_without_voids(self): return ', '.join(['%s: %s' % (name, self._flds[name]) @@ -576,8 +575,10 @@ _gckind = 'raw' def __init__(self, tag, hints={}): - """ if hints['render_structure'] is set, the type is internal and not considered - to come from somewhere else (it should be rendered as a structure) """ + """If hints['render_structure'] is set, the type is internal and + not considered to come from somewhere else (it should be + rendered as a structure) + """ self.tag = tag self.__name__ = tag self.hints = frozendict(hints) @@ -675,7 +676,8 @@ _numbertypes = {int: Number("Signed", int, intmask)} _numbertypes[r_int] = _numbertypes[int] -_numbertypes[r_longlonglong] = Number("SignedLongLongLong", r_longlonglong, longlonglongmask) +_numbertypes[r_longlonglong] = Number("SignedLongLongLong", r_longlonglong, + longlonglongmask) if r_longlong is not r_int: _numbertypes[r_longlong] = Number("SignedLongLong", r_longlong, longlongmask) @@ -702,8 +704,8 @@ UnsignedLongLong = build_number("UnsignedLongLong", r_ulonglong) Float = Primitive("Float", 0.0) # C type 'double' -SingleFloat = Primitive("SingleFloat", r_singlefloat(0.0)) # C type 'float' -LongFloat = Primitive("LongFloat", r_longfloat(0.0)) # C type 'long double' +SingleFloat = Primitive("SingleFloat", r_singlefloat(0.0)) # 'float' +LongFloat = Primitive("LongFloat", r_longfloat(0.0)) # 'long double' r_singlefloat._TYPE = SingleFloat Char = Primitive("Char", '\x00') @@ -876,9 +878,11 @@ @analyzer_for(cast_primitive) def ann_cast_primitive(T, s_v): - from rpython.rtyper.llannotation import annotation_to_lltype, ll_to_annotation + from rpython.rtyper.llannotation import ( + annotation_to_lltype, ll_to_annotation) assert T.is_constant() - return ll_to_annotation(cast_primitive(T.const, annotation_to_lltype(s_v)._defl())) + return ll_to_annotation(cast_primitive(T.const, + annotation_to_lltype(s_v)._defl())) def _cast_whatever(TGT, value): @@ -905,7 +909,8 @@ elif TGT == llmemory.Address and isinstance(ORIG, Ptr): return llmemory.cast_ptr_to_adr(value) elif TGT == Signed and isinstance(ORIG, Ptr) and ORIG.TO._gckind == 'raw': - return llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(value), 'symbolic') + return llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(value), + 'symbolic') raise TypeError("don't know how to cast from %r to %r" % (ORIG, TGT)) @@ -1176,8 +1181,8 @@ except DelayedPointer: return True # assume it's not a delayed null - # _setobj, _getobj and _obj0 are really _internal_ implementations details of _ptr, - # use _obj if necessary instead ! + # _setobj, _getobj and _obj0 are really _internal_ implementations + # details of _ptr, use _obj if necessary instead ! def _setobj(self, pointing_to, solid=False): if pointing_to is None: obj0 = None @@ -1244,12 +1249,12 @@ if T1 == T2: setattr(self._obj, field_name, val) else: - raise TypeError("%r instance field %r:\n" - "expects %r\n" - " got %r" % (self._T, field_name, T1, T2)) + raise TypeError( + "%r instance field %r:\nexpects %r\n got %r" % + (self._T, field_name, T1, T2)) return - raise AttributeError("%r instance has no field %r" % (self._T, - field_name)) + raise AttributeError("%r instance has no field %r" % + (self._T, field_name)) def __getitem__(self, i): # ! can only return basic or ptr ! if isinstance(self._T, (Array, FixedSizeArray)): @@ -1266,7 +1271,8 @@ if isinstance(self._T, (Array, FixedSizeArray)): T1 = self._T.OF if isinstance(T1, ContainerType): - raise TypeError("cannot directly assign to container array items") + raise TypeError("cannot directly assign to container array " + "items") T2 = typeOf(val) if T2 != T1: from rpython.rtyper.lltypesystem import rffi @@ -1316,7 +1322,8 @@ from rpython.rtyper.lltypesystem import rffi if isinstance(self._T, FuncType): if len(args) != len(self._T.ARGS): - raise TypeError("calling %r with wrong argument number: %r" % (self._T, args)) + raise TypeError("calling %r with wrong argument number: %r" % + (self._T, args)) for i, a, ARG in zip(range(len(self._T.ARGS)), args, self._T.ARGS): if typeOf(a) != ARG: # ARG could be Void @@ -1415,11 +1422,13 @@ raise RuntimeError("widening to trash: %r" % self) PARENTTYPE = struc._parent_type if getattr(parent, PARENTTYPE._names[0]) != struc: - raise InvalidCast(CURTYPE, PTRTYPE) # xxx different exception perhaps? + # xxx different exception perhaps? + raise InvalidCast(CURTYPE, PTRTYPE) struc = parent u -= 1 if PARENTTYPE != PTRTYPE.TO: - raise RuntimeError("widening %r inside %r instead of %r" % (CURTYPE, PARENTTYPE, PTRTYPE.TO)) + raise RuntimeError("widening %r inside %r instead of %r" % + (CURTYPE, PARENTTYPE, PTRTYPE.TO)) return _ptr(PTRTYPE, struc, solid=self._solid) def _cast_to_int(self, check=True): @@ -1430,7 +1439,9 @@ return obj # special case for cast_int_to_ptr() results obj = normalizeptr(self, check)._getobj(check) if isinstance(obj, int): - return obj # special case for cast_int_to_ptr() results put into opaques + # special case for cast_int_to_ptr() results put into + # opaques + return obj if getattr(obj, '_read_directly_intval', False): return obj.intval # special case for _llgcopaque result = intmask(obj._getid()) @@ -1468,7 +1479,8 @@ """XXX A nice docstring here""" T = typeOf(val) if isinstance(T, ContainerType): - if self._T._gckind == 'gc' and T._gckind == 'raw' and not isinstance(T, OpaqueType): + if (self._T._gckind == 'gc' and T._gckind == 'raw' and + not isinstance(T, OpaqueType)): val = _interior_ptr(T, self._obj, [offset]) else: val = _ptr(Ptr(T), val, solid=self._solid) @@ -1531,12 +1543,14 @@ setattr(example, s_attr.const, v_lltype._defl()) def call(self, args): - from rpython.rtyper.llannotation import annotation_to_lltype, ll_to_annotation + from rpython.rtyper.llannotation import ( + annotation_to_lltype, ll_to_annotation) args_s, kwds_s = args.unpack() if kwds_s: raise Exception("keyword arguments to call to a low-level fn ptr") info = 'argument to ll function pointer call' - llargs = [annotation_to_lltype(s_arg, info)._defl() for s_arg in args_s] + llargs = [annotation_to_lltype(s_arg, info)._defl() + for s_arg in args_s] v = self.ll_ptrtype._example()(*llargs) return ll_to_annotation(v) @@ -1593,7 +1607,6 @@ return val - assert not '__dict__' in dir(_interior_ptr) class _container(object): @@ -1721,11 +1734,13 @@ __slots__ = ('_hash_cache_', '_compilation_info') - def __new__(self, TYPE, n=None, initialization=None, parent=None, parentindex=None): + def __new__(self, TYPE, n=None, initialization=None, parent=None, + parentindex=None): my_variety = _struct_variety(TYPE._names) return object.__new__(my_variety) - def __init__(self, TYPE, n=None, initialization=None, parent=None, parentindex=None): + def __init__(self, TYPE, n=None, initialization=None, parent=None, + parentindex=None): _parentable.__init__(self, TYPE) if n is not None and TYPE._arrayfld is None: raise TypeError("%r is not variable-sized" % (TYPE,)) @@ -1734,9 +1749,11 @@ first, FIRSTTYPE = TYPE._first_struct() for fld, typ in TYPE._flds.items(): if fld == TYPE._arrayfld: - value = _array(typ, n, initialization=initialization, parent=self, parentindex=fld) + value = _array(typ, n, initialization=initialization, + parent=self, parentindex=fld) else: - value = typ._allocate(initialization=initialization, parent=self, parentindex=fld) + value = typ._allocate(initialization=initialization, + parent=self, parentindex=fld) setattr(self, fld, value) if parent is not None: self._setparentstructure(parent, parentindex) @@ -1795,7 +1812,8 @@ __slots__ = ('items',) - def __init__(self, TYPE, n, initialization=None, parent=None, parentindex=None): + def __init__(self, TYPE, n, initialization=None, parent=None, + parentindex=None): if not is_valid_int(n): raise TypeError("array length must be an int") if n < 0: @@ -1964,7 +1982,8 @@ if not key._was_freed(): newcache[key] = value except RuntimeError: - pass # ignore "accessing subxxx, but already gc-ed parent" + # ignore "accessing subxxx, but already gc-ed parent" + pass if newcache: _subarray._cache[T] = newcache else: @@ -2020,8 +2039,10 @@ attrs.setdefault('_name', '?') attrs.setdefault('_callable', None) self.__dict__.update(attrs) - if '_callable' in attrs and hasattr(attrs['_callable'], '_compilation_info'): - self.__dict__['compilation_info'] = attrs['_callable']._compilation_info + if '_callable' in attrs and hasattr(attrs['_callable'], + '_compilation_info'): + self.__dict__['compilation_info'] = \ + attrs['_callable']._compilation_info def __repr__(self): return '<%s>' % (self,) @@ -2126,8 +2147,8 @@ return _ptr(Ptr(T), o, solid) @analyzer_for(malloc) -def ann_malloc(s_T, s_n=None, s_flavor=None, s_zero=None, s_track_allocation=None, - s_add_memory_pressure=None): +def ann_malloc(s_T, s_n=None, s_flavor=None, s_zero=None, + s_track_allocation=None, s_add_memory_pressure=None): assert (s_n is None or s_n.knowntype == int or issubclass(s_n.knowntype, base_int)) assert s_T.is_constant() @@ -2303,7 +2324,8 @@ @analyzer_for(runtime_type_info) def ann_runtime_type_info(s_p): - assert isinstance(s_p, SomePtr), "runtime_type_info of non-pointer: %r" % s_p + assert isinstance(s_p, SomePtr), \ + "runtime_type_info of non-pointer: %r" % s_p return SomePtr(typeOf(runtime_type_info(s_p.ll_ptrtype._example()))) diff --git a/rpython/rtyper/module/ll_os_stat.py b/rpython/rtyper/module/ll_os_stat.py --- a/rpython/rtyper/module/ll_os_stat.py +++ b/rpython/rtyper/module/ll_os_stat.py @@ -186,7 +186,10 @@ _name_struct_stat = '_stati64' INCLUDES = ['sys/types.h', 'sys/stat.h', 'sys/statvfs.h'] else: - _name_struct_stat = 'stat' + if sys.platform.startswith('linux'): + _name_struct_stat = 'stat64' + else: + _name_struct_stat = 'stat' INCLUDES = ['sys/types.h', 'sys/stat.h', 'sys/statvfs.h', 'unistd.h'] compilation_info = ExternalCompilationInfo( diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -204,9 +204,9 @@ # must come first, and after the file name all options are ignored. # So please be careful with the order of parameters! ;-) pdb_dir = oname.dirname - if pdb_dir: - compile_args += ['/Fd%s\\' % (pdb_dir,)] - args = ['/nologo', '/c'] + compile_args + ['/Fo%s' % (oname,), str(cfile)] + if pdb_dir: + compile_args += ['/Fd%s\\' % (pdb_dir,)] + args = ['/nologo', '/c'] + compile_args + ['/Fo%s' % (oname,), str(cfile)] self._execute_c_compiler(cc, args, oname) return oname From noreply at buildbot.pypy.org Thu Sep 18 19:01:48 2014 From: noreply at buildbot.pypy.org (pjenvey) Date: Thu, 18 Sep 2014 19:01:48 +0200 (CEST) Subject: [pypy-commit] pypy py3k: fix Message-ID: <20140918170148.9FDBD1D3869@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r73604:e27f287735af Date: 2014-09-18 10:00 -0700 http://bitbucket.org/pypy/pypy/changeset/e27f287735af/ Log: fix diff --git a/rpython/rtyper/module/ll_win32file.py b/rpython/rtyper/module/ll_win32file.py --- a/rpython/rtyper/module/ll_win32file.py +++ b/rpython/rtyper/module/ll_win32file.py @@ -445,7 +445,7 @@ with lltype.scoped_alloc( win32traits.BY_HANDLE_FILE_INFORMATION) as info: if win32traits.GetFileInformationByHandle(hFile, info) == 0: - raise lastWindowsError("_getfileinformation") + raise rwin32.lastWindowsError("_getfileinformation") return (rffi.cast(lltype.Signed, info.c_dwVolumeSerialNumber), rffi.cast(lltype.Signed, info.c_nFileIndexHigh), rffi.cast(lltype.Signed, info.c_nFileIndexLow)) From noreply at buildbot.pypy.org Thu Sep 18 20:12:09 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 18 Sep 2014 20:12:09 +0200 (CEST) Subject: [pypy-commit] pypy gc_zero_array: Write the ZERO_ARRAY operation, with specific support in the x86 backend. Message-ID: <20140918181209.2CEFB1D3869@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: gc_zero_array Changeset: r73605:3ba8153e7746 Date: 2014-09-18 20:11 +0200 http://bitbucket.org/pypy/pypy/changeset/3ba8153e7746/ Log: Write the ZERO_ARRAY operation, with specific support in the x86 backend. Missing tests for ZERO_PTR_FIELD in runner_test, and support for ARM. diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py --- a/rpython/jit/backend/llsupport/assembler.py +++ b/rpython/jit/backend/llsupport/assembler.py @@ -1,5 +1,5 @@ from rpython.jit.backend.llsupport import jitframe -from rpython.jit.backend.llsupport.memcpy import memcpy_fn +from rpython.jit.backend.llsupport.memcpy import memcpy_fn, memset_fn from rpython.jit.backend.llsupport.symbolic import WORD from rpython.jit.metainterp.history import (INT, REF, FLOAT, JitCellToken, ConstInt, BoxInt, AbstractFailDescr) @@ -63,6 +63,7 @@ def __init__(self, cpu, translate_support_code=False): self.cpu = cpu self.memcpy_addr = 0 + self.memset_addr = 0 self.rtyper = cpu.rtyper self._debug = False @@ -79,6 +80,7 @@ else: self.gc_size_of_header = WORD # for tests self.memcpy_addr = self.cpu.cast_ptr_to_int(memcpy_fn) + self.memset_addr = self.cpu.cast_ptr_to_int(memset_fn) self._build_failure_recovery(False, withfloats=False) self._build_failure_recovery(True, withfloats=False) self._build_wb_slowpath(False) diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py --- a/rpython/jit/backend/llsupport/gc.py +++ b/rpython/jit/backend/llsupport/gc.py @@ -34,15 +34,6 @@ self.fielddescr_vtable = get_field_descr(self, rclass.OBJECT, 'typeptr') self._generated_functions = [] - self.memset_ptr = rffi.llexternal('memset', [lltype.Signed, rffi.INT, - rffi.SIZE_T], lltype.Void, - sandboxsafe=True, - _nowrapper=True) - self.memset_ptr_as_int = heaptracker.adr2int( - llmemory.cast_ptr_to_adr(self.memset_ptr)) - ei = EffectInfo([], [], [], [], [], [], EffectInfo.EF_CANNOT_RAISE) - self.memset_descr = get_call_descr(self, [lltype.Signed, rffi.INT, - rffi.SIZE_T], lltype.Void, ei) def _setup_str(self): self.str_descr = get_array_descr(self, rstr.STR) diff --git a/rpython/jit/backend/llsupport/memcpy.py b/rpython/jit/backend/llsupport/memcpy.py --- a/rpython/jit/backend/llsupport/memcpy.py +++ b/rpython/jit/backend/llsupport/memcpy.py @@ -3,3 +3,6 @@ memcpy_fn = rffi.llexternal('memcpy', [llmemory.Address, llmemory.Address, rffi.SIZE_T], lltype.Void, sandboxsafe=True, _nowrapper=True) +memset_fn = rffi.llexternal('memset', [llmemory.Address, rffi.INT, + rffi.SIZE_T], lltype.Void, + sandboxsafe=True, _nowrapper=True) diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -180,29 +180,16 @@ if kind == FLAG_ARRAY: self.clear_varsize_gc_fields(op.getdescr(), op.result, v_length) - def handle_clear_array_contents(self, arraydescr, v_arr, v_arrsize=None): - # XXX this maybe should go to optimizer, so we can remove extra ops? - ofs, size, _ = self.cpu.unpack_arraydescr_size(arraydescr) - v_arr_plus_ofs = BoxInt() - v_totalsize = BoxInt() - gcdescr = self.gc_ll_descr - ops = [ - ResOperation(rop.INT_ADD, [v_arr, ConstInt(ofs)], v_arr_plus_ofs), - ] - - if v_arrsize is None: - v_arrsize = BoxInt() - o = ResOperation(rop.ARRAYLEN_GC, [v_arr], v_arrsize, + def handle_clear_array_contents(self, arraydescr, v_arr, v_length=None): + # XXX more work here to reduce or remove the ZERO_ARRAY in some cases + if v_length is None: + v_length = BoxInt() + o = ResOperation(rop.ARRAYLEN_GC, [v_arr], v_length, descr=arraydescr) - ops.append(o) - ops += [ - ResOperation(rop.INT_MUL, [v_arrsize, ConstInt(size)], v_totalsize), - ResOperation(rop.CALL, [ConstInt(gcdescr.memset_ptr_as_int), - v_arr_plus_ofs, - ConstInt(0), v_totalsize], None, - descr=gcdescr.memset_descr), - ] - self.newops.extend(ops) + self.newops.append(o) + o = ResOperation(rop.ZERO_ARRAY, [v_arr, ConstInt(0), v_length], None, + descr=arraydescr) + self.newops.append(o) def gen_malloc_frame(self, frame_info, frame, size_box): descrs = self.gc_ll_descr.getframedescrs(self.cpu) diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py --- a/rpython/jit/backend/test/runner_test.py +++ b/rpython/jit/backend/test/runner_test.py @@ -4478,3 +4478,49 @@ ref = self.cpu.get_ref_value(deadframe, 0) s = lltype.cast_opaque_ptr(lltype.Ptr(S), ref) assert not s.x + + def test_zero_ptr_field(self): + XXX # write me! + + def test_zero_array(self): + PAIR = lltype.Struct('PAIR', ('a', lltype.Signed), ('b', lltype.Signed)) + for OF in [lltype.Signed, rffi.INT, rffi.SHORT, rffi.UCHAR, PAIR]: + A = lltype.GcArray(OF) + arraydescr = self.cpu.arraydescrof(A) + a = lltype.malloc(A, 100) + addr = llmemory.cast_ptr_to_adr(a) + a_int = heaptracker.adr2int(addr) + a_ref = lltype.cast_opaque_ptr(llmemory.GCREF, a) + for (start, length) in [(0, 100), (49, 49), (1, 98), + (15, 9), (10, 10), (47, 0), + (0, 4)]: + for cls1 in [ConstInt, BoxInt]: + for cls2 in [ConstInt, BoxInt]: + print 'a_int:', a_int + print 'of:', OF + print 'start:', start + print 'length:', length + print 'cls1:', cls1.__name__ + print 'cls2:', cls2.__name__ + for i in range(100): + if OF == PAIR: + a[i].a = a[i].b = -123456789 + else: + a[i] = rffi.cast(OF, -123456789) + startbox = cls1(start) + lengthbox = cls2(length) + if cls1 == cls2 and start == length: + lengthbox = startbox # same box! + self.execute_operation(rop.ZERO_ARRAY, + [BoxPtr(a_ref), + startbox, + lengthbox], + 'void', descr=arraydescr) + assert len(a) == 100 + for i in range(100): + val = (0 if start <= i < start + length + else -123456789) + if OF == PAIR: + assert a[i].a == a[i].b == val + else: + assert a[i] == rffi.cast(OF, val) diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -2363,6 +2363,42 @@ elif IS_X86_64: mc.MOVSX32_rj(loc.value, addr) # memory read, sign-extend + def genop_discard_zero_array(self, op, arglocs): + (base_loc, startindex_loc, bytes_loc, + itemsize_loc, baseofs_loc, null_loc) = arglocs + assert isinstance(bytes_loc, ImmedLoc) + assert isinstance(itemsize_loc, ImmedLoc) + assert isinstance(baseofs_loc, ImmedLoc) + assert isinstance(null_loc, RegLoc) and null_loc.is_xmm + baseofs = baseofs_loc.value + nbytes = bytes_loc.value + if valid_addressing_size(itemsize_loc.value): + scale = get_scale(itemsize_loc.value) + else: + assert isinstance(startindex_loc, ImmedLoc) + assert startindex_loc.value == 0 + scale = 0 + null_reg_cleared = False + i = 0 + while i < nbytes: + addr = addr_add(base_loc, startindex_loc, baseofs + i, scale) + current = nbytes - i + if current >= 16: + current = 16 + if not null_reg_cleared: + self.mc.XORPS_xx(null_loc.value, null_loc.value) + null_reg_cleared = True + self.mc.MOVUPS(addr, null_loc) + else: + if current >= WORD: + current = WORD + elif current >= 4: + current = 4 + elif current >= 2: + current = 2 + self.save_into_mem(addr, imm0, imm(current)) + i += current + genop_discard_list = [Assembler386.not_implemented_op_discard] * rop._LAST genop_list = [Assembler386.not_implemented_op] * rop._LAST diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -8,7 +8,8 @@ unpack_arraydescr, unpack_fielddescr, unpack_interiorfielddescr) from rpython.jit.backend.llsupport.gcmap import allocate_gcmap from rpython.jit.backend.llsupport.regalloc import (FrameManager, BaseRegalloc, - RegisterManager, TempBox, compute_vars_longevity, is_comparison_or_ovf_op) + RegisterManager, TempBox, compute_vars_longevity, is_comparison_or_ovf_op, + valid_addressing_size) from rpython.jit.backend.x86 import rx86 from rpython.jit.backend.x86.arch import (WORD, JITFRAME_FIXED_SIZE, IS_X86_32, IS_X86_64) @@ -1384,6 +1385,71 @@ def consider_keepalive(self, op): pass + def consider_zero_array(self, op): + itemsize, baseofs, _ = unpack_arraydescr(op.getdescr()) + args = op.getarglist() + base_loc = self.rm.make_sure_var_in_reg(args[0], args) + startindex_loc = self.rm.make_sure_var_in_reg(args[1], args) + length_box = op.getarg(2) + if isinstance(length_box, ConstInt): + constbytes = length_box.getint() * itemsize + else: + constbytes = -1 + if 0 <= constbytes <= 16 * 8 and ( + valid_addressing_size(itemsize) or + (isinstance(startindex_loc, ImmedLoc) and + startindex_loc.value == 0)): + if IS_X86_64: + null_loc = X86_64_XMM_SCRATCH_REG + else: + null_box = TempBox() + null_loc = self.xrm.force_allocate_reg(null_box) + self.xrm.possibly_free_var(null_box) + self.perform_discard(op, [base_loc, startindex_loc, + imm(constbytes), imm(itemsize), + imm(baseofs), null_loc]) + else: + # base_loc and startindex_loc are in two regs here (or they are + # immediates). Compute the dstaddr_loc, which is the raw + # address that we will pass as first argument to memset(). + # It can be in the same register as either one, but not in + # args[2], because we're still needing the latter. + dstaddr_box = TempBox() + dstaddr_loc = self.rm.force_allocate_reg(dstaddr_box, [args[2]]) + itemsize_loc = imm(itemsize) + dst_addr = self.assembler._get_interiorfield_addr( + dstaddr_loc, startindex_loc, itemsize_loc, + base_loc, imm(baseofs)) + self.assembler.mc.LEA(dstaddr_loc, dst_addr) + # + if constbytes >= 0: + length_loc = imm(constbytes) + else: + # load length_loc in a register different than dstaddr_loc + length_loc = self.rm.make_sure_var_in_reg(length_box, + [dstaddr_box]) + if itemsize > 1: + # we need a register that is different from dstaddr_loc, + # but which can be identical to length_loc (as usual, + # only if the length_box is not used by future operations) + bytes_box = TempBox() + bytes_loc = self.rm.force_allocate_reg(bytes_box, + [dstaddr_box]) + b_adr = self.assembler._get_interiorfield_addr( + bytes_loc, length_loc, itemsize_loc, imm0, imm0) + self.assembler.mc.LEA(bytes_loc, b_adr) + length_box = bytes_box + length_loc = bytes_loc + # + # call memset() + self.rm.before_call() + self.xrm.before_call() + self.assembler.simple_call_no_collect( + imm(self.assembler.memset_addr), + [dstaddr_loc, imm0, length_loc]) + self.rm.possibly_free_var(length_box) + self.rm.possibly_free_var(dstaddr_box) + def not_implemented_op(self, op): not_implemented("not implemented operation: %s" % op.getopname()) diff --git a/rpython/jit/backend/x86/regloc.py b/rpython/jit/backend/x86/regloc.py --- a/rpython/jit/backend/x86/regloc.py +++ b/rpython/jit/backend/x86/regloc.py @@ -664,6 +664,7 @@ MOVDQ = _binaryop('MOVDQ') MOVD32 = _binaryop('MOVD32') + MOVUPS = _binaryop('MOVUPS') CALL = _relative_unaryop('CALL') JMP = _relative_unaryop('JMP') diff --git a/rpython/jit/backend/x86/rx86.py b/rpython/jit/backend/x86/rx86.py --- a/rpython/jit/backend/x86/rx86.py +++ b/rpython/jit/backend/x86/rx86.py @@ -634,6 +634,9 @@ MOVD32_xs = xmminsn('\x66', rex_nw, '\x0F\x6E', register(1, 8), stack_sp(2)) PSRAD_xi = xmminsn('\x66', rex_nw, '\x0F\x72', register(1), '\xE0', immediate(2, 'b')) + MOVUPS_mx = xmminsn(rex_nw, '\x0F\x11', register(2, 8), mem_reg_plus_const(1)) + MOVUPS_jx = xmminsn(rex_nw, '\x0F\x11', register(2, 8), abs_(1)) + MOVUPS_ax = xmminsn(rex_nw, '\x0F\x11', register(2, 8), mem_reg_plus_scaled_reg_plus_const(1)) # ------------------------------------------------------------ @@ -764,6 +767,7 @@ define_modrm_modes('DIVSD_x*', ['\xF2', rex_nw, '\x0F\x5E', register(1, 8)], regtype='XMM') define_modrm_modes('UCOMISD_x*', ['\x66', rex_nw, '\x0F\x2E', register(1, 8)], regtype='XMM') define_modrm_modes('XORPD_x*', ['\x66', rex_nw, '\x0F\x57', register(1, 8)], regtype='XMM') +define_modrm_modes('XORPS_x*', [rex_nw, '\x0F\x57', register(1, 8)], regtype='XMM') define_modrm_modes('ANDPD_x*', ['\x66', rex_nw, '\x0F\x54', register(1, 8)], regtype='XMM') def define_pxmm_insn(insnname_template, insn_char): diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py --- a/rpython/jit/metainterp/executor.py +++ b/rpython/jit/metainterp/executor.py @@ -326,6 +326,7 @@ rop.COND_CALL_GC_WB, rop.COND_CALL_GC_WB_ARRAY, rop.ZERO_PTR_FIELD, + rop.ZERO_ARRAY, rop.DEBUG_MERGE_POINT, rop.JIT_DEBUG, rop.SETARRAYITEM_RAW, diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py --- a/rpython/jit/metainterp/resoperation.py +++ b/rpython/jit/metainterp/resoperation.py @@ -504,8 +504,10 @@ 'SETINTERIORFIELD_RAW/3d', # right now, only used by tests 'RAW_STORE/3d', 'SETFIELD_GC/2d', - 'ZERO_PTR_FIELD/2', # only emitted by the rewrite, sets a pointer field + 'ZERO_PTR_FIELD/2', # only emitted by the rewrite, clears a pointer field # at a given offset, no descr + 'ZERO_ARRAY/3d', # only emitted by the rewrite, clears (part of) an array + # [arraygcptr, firstindex, length], descr=ArrayDescr 'SETFIELD_RAW/2d', 'STRSETITEM/3', 'UNICODESETITEM/3', From noreply at buildbot.pypy.org Thu Sep 18 22:04:53 2014 From: noreply at buildbot.pypy.org (pjenvey) Date: Thu, 18 Sep 2014 22:04:53 +0200 (CEST) Subject: [pypy-commit] pypy py3k: fix translation Message-ID: <20140918200453.3F79C1D387A@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r73606:95c3cb767044 Date: 2014-09-18 13:04 -0700 http://bitbucket.org/pypy/pypy/changeset/95c3cb767044/ Log: fix translation diff --git a/rpython/rtyper/module/ll_win32file.py b/rpython/rtyper/module/ll_win32file.py --- a/rpython/rtyper/module/ll_win32file.py +++ b/rpython/rtyper/module/ll_win32file.py @@ -194,10 +194,12 @@ # dynamically loaded GetFinalPathNameByHandle = None + GETFINALPATHNAMEBYHANDLE_TP = lltype.Ptr(lltype.FuncType( + [rwin32.HANDLE, traits.CCHARP, rwin32.DWORD, rwin32.DWORD], + rwin32.DWORD, abi='FFI_STDCALL')) - @staticmethod - def check_GetFinalPathNameByHandle(): - if Win32Traits.GetFinalPathNameByHandle: + def check_GetFinalPathNameByHandle(self): + if self.GetFinalPathNameByHandle: return True from rpython.rlib.rdynload import GetModuleHandle, dlsym @@ -207,13 +209,11 @@ except KeyError: return False - TYPE = lltype.Ptr(lltype.FuncType( - [rwin32.HANDLE, traits.CCHARP, rwin32.DWORD, rwin32.DWORD], - rwin32.DWORD, abi='FFI_STDCALL')) - Win32Traits.GetFinalPathNameByHandle = rffi.cast(TYPE, func) + self.GetFinalPathNameByHandle = rffi.cast( + Win32Traits.GETFINALPATHNAMEBYHANDLE_TP, func) return True - return Win32Traits + return Win32Traits() #_______________________________________________________________ # listdir From noreply at buildbot.pypy.org Thu Sep 18 22:27:11 2014 From: noreply at buildbot.pypy.org (mattip) Date: Thu, 18 Sep 2014 22:27:11 +0200 (CEST) Subject: [pypy-commit] pypy release-2.4.x: Added tag release-2.4.0 for changeset f5dcc2477b97 Message-ID: <20140918202711.D13521D3869@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: release-2.4.x Changeset: r73607:6e3fbf0ef860 Date: 2014-09-18 23:24 +0300 http://bitbucket.org/pypy/pypy/changeset/6e3fbf0ef860/ Log: Added tag release-2.4.0 for changeset f5dcc2477b97 diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -15,3 +15,4 @@ 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1 0000000000000000000000000000000000000000 release-2.2=3.1 9f425c60afdf2c1d38ee016071cffc55c28048b9 release-2.4-beta1 +f5dcc2477b97386c11e4b67f08a2d00fbd2fce5d release-2.4.0 From noreply at buildbot.pypy.org Thu Sep 18 22:30:12 2014 From: noreply at buildbot.pypy.org (pjenvey) Date: Thu, 18 Sep 2014 22:30:12 +0200 (CEST) Subject: [pypy-commit] pypy py3k: another translation fix attempt Message-ID: <20140918203012.D324B1D3869@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r73608:c8117677797e Date: 2014-09-18 13:29 -0700 http://bitbucket.org/pypy/pypy/changeset/c8117677797e/ Log: another translation fix attempt diff --git a/rpython/rtyper/module/ll_win32file.py b/rpython/rtyper/module/ll_win32file.py --- a/rpython/rtyper/module/ll_win32file.py +++ b/rpython/rtyper/module/ll_win32file.py @@ -192,11 +192,12 @@ [traits.CCHARP, traits.CCHARP], rwin32.BOOL) - # dynamically loaded - GetFinalPathNameByHandle = None GETFINALPATHNAMEBYHANDLE_TP = lltype.Ptr(lltype.FuncType( [rwin32.HANDLE, traits.CCHARP, rwin32.DWORD, rwin32.DWORD], rwin32.DWORD, abi='FFI_STDCALL')) + # dynamically loaded + GetFinalPathNameByHandle = lltype.nullptr( + GETFINALPATHNAMEBYHANDLE_TP.TO) def check_GetFinalPathNameByHandle(self): if self.GetFinalPathNameByHandle: From noreply at buildbot.pypy.org Fri Sep 19 07:55:34 2014 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 19 Sep 2014 07:55:34 +0200 (CEST) Subject: [pypy-commit] pypy gc_zero_array: close Message-ID: <20140919055535.029751D29E9@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: gc_zero_array Changeset: r73609:94a6aadbcab7 Date: 2014-09-19 07:54 +0200 http://bitbucket.org/pypy/pypy/changeset/94a6aadbcab7/ Log: close From noreply at buildbot.pypy.org Fri Sep 19 07:55:36 2014 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 19 Sep 2014 07:55:36 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: hg merge gc_zero_array Message-ID: <20140919055536.C5AB21D29E9@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: gc_no_cleanup_nursery Changeset: r73610:526a7075bba7 Date: 2014-09-19 07:55 +0200 http://bitbucket.org/pypy/pypy/changeset/526a7075bba7/ Log: hg merge gc_zero_array diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py --- a/rpython/jit/backend/llsupport/assembler.py +++ b/rpython/jit/backend/llsupport/assembler.py @@ -1,5 +1,5 @@ from rpython.jit.backend.llsupport import jitframe -from rpython.jit.backend.llsupport.memcpy import memcpy_fn +from rpython.jit.backend.llsupport.memcpy import memcpy_fn, memset_fn from rpython.jit.backend.llsupport.symbolic import WORD from rpython.jit.metainterp.history import (INT, REF, FLOAT, JitCellToken, ConstInt, BoxInt, AbstractFailDescr) @@ -63,6 +63,7 @@ def __init__(self, cpu, translate_support_code=False): self.cpu = cpu self.memcpy_addr = 0 + self.memset_addr = 0 self.rtyper = cpu.rtyper self._debug = False @@ -79,6 +80,7 @@ else: self.gc_size_of_header = WORD # for tests self.memcpy_addr = self.cpu.cast_ptr_to_int(memcpy_fn) + self.memset_addr = self.cpu.cast_ptr_to_int(memset_fn) self._build_failure_recovery(False, withfloats=False) self._build_failure_recovery(True, withfloats=False) self._build_wb_slowpath(False) diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py --- a/rpython/jit/backend/llsupport/gc.py +++ b/rpython/jit/backend/llsupport/gc.py @@ -34,15 +34,6 @@ self.fielddescr_vtable = get_field_descr(self, rclass.OBJECT, 'typeptr') self._generated_functions = [] - self.memset_ptr = rffi.llexternal('memset', [lltype.Signed, rffi.INT, - rffi.SIZE_T], lltype.Void, - sandboxsafe=True, - _nowrapper=True) - self.memset_ptr_as_int = heaptracker.adr2int( - llmemory.cast_ptr_to_adr(self.memset_ptr)) - ei = EffectInfo([], [], [], [], [], [], EffectInfo.EF_CANNOT_RAISE) - self.memset_descr = get_call_descr(self, [lltype.Signed, rffi.INT, - rffi.SIZE_T], lltype.Void, ei) def _setup_str(self): self.str_descr = get_array_descr(self, rstr.STR) diff --git a/rpython/jit/backend/llsupport/memcpy.py b/rpython/jit/backend/llsupport/memcpy.py --- a/rpython/jit/backend/llsupport/memcpy.py +++ b/rpython/jit/backend/llsupport/memcpy.py @@ -3,3 +3,6 @@ memcpy_fn = rffi.llexternal('memcpy', [llmemory.Address, llmemory.Address, rffi.SIZE_T], lltype.Void, sandboxsafe=True, _nowrapper=True) +memset_fn = rffi.llexternal('memset', [llmemory.Address, rffi.INT, + rffi.SIZE_T], lltype.Void, + sandboxsafe=True, _nowrapper=True) diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -180,38 +180,16 @@ if kind == FLAG_ARRAY: self.clear_varsize_gc_fields(op.getdescr(), op.result, v_length) - def handle_clear_array_contents(self, arraydescr, v_arr, v_arrsize=None): - # XXX this maybe should go to optimizer, so we can remove extra ops? - ofs, size, _ = self.cpu.unpack_arraydescr_size(arraydescr) - v_arr_plus_ofs = BoxInt() - v_totalsize = BoxInt() - gcdescr = self.gc_ll_descr - if isinstance(v_arrsize, ConstInt) and v_arrsize.getint() < 10: - # clear it item by item - ops = [] - for i in range(v_arrsize.getint()): - ops.append(ResOperation(rop.SETARRAYITEM_GC, - [v_arr, ConstInt(i), self.c_zero], None, - descr=arraydescr)) - self.newops += ops - return - ops = [ - ResOperation(rop.INT_ADD, [v_arr, ConstInt(ofs)], v_arr_plus_ofs), - ] - - if v_arrsize is None: - v_arrsize = BoxInt() - o = ResOperation(rop.ARRAYLEN_GC, [v_arr], v_arrsize, + def handle_clear_array_contents(self, arraydescr, v_arr, v_length=None): + # XXX more work here to reduce or remove the ZERO_ARRAY in some cases + if v_length is None: + v_length = BoxInt() + o = ResOperation(rop.ARRAYLEN_GC, [v_arr], v_length, descr=arraydescr) - ops.append(o) - ops += [ - ResOperation(rop.INT_MUL, [v_arrsize, ConstInt(size)], v_totalsize), - ResOperation(rop.CALL, [ConstInt(gcdescr.memset_ptr_as_int), - v_arr_plus_ofs, - ConstInt(0), v_totalsize], None, - descr=gcdescr.memset_descr), - ] - self.newops.extend(ops) + self.newops.append(o) + o = ResOperation(rop.ZERO_ARRAY, [v_arr, ConstInt(0), v_length], None, + descr=arraydescr) + self.newops.append(o) def gen_malloc_frame(self, frame_info, frame, size_box): descrs = self.gc_ll_descr.getframedescrs(self.cpu) diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py --- a/rpython/jit/backend/test/runner_test.py +++ b/rpython/jit/backend/test/runner_test.py @@ -4478,3 +4478,49 @@ ref = self.cpu.get_ref_value(deadframe, 0) s = lltype.cast_opaque_ptr(lltype.Ptr(S), ref) assert not s.x + + def test_zero_ptr_field(self): + XXX # write me! + + def test_zero_array(self): + PAIR = lltype.Struct('PAIR', ('a', lltype.Signed), ('b', lltype.Signed)) + for OF in [lltype.Signed, rffi.INT, rffi.SHORT, rffi.UCHAR, PAIR]: + A = lltype.GcArray(OF) + arraydescr = self.cpu.arraydescrof(A) + a = lltype.malloc(A, 100) + addr = llmemory.cast_ptr_to_adr(a) + a_int = heaptracker.adr2int(addr) + a_ref = lltype.cast_opaque_ptr(llmemory.GCREF, a) + for (start, length) in [(0, 100), (49, 49), (1, 98), + (15, 9), (10, 10), (47, 0), + (0, 4)]: + for cls1 in [ConstInt, BoxInt]: + for cls2 in [ConstInt, BoxInt]: + print 'a_int:', a_int + print 'of:', OF + print 'start:', start + print 'length:', length + print 'cls1:', cls1.__name__ + print 'cls2:', cls2.__name__ + for i in range(100): + if OF == PAIR: + a[i].a = a[i].b = -123456789 + else: + a[i] = rffi.cast(OF, -123456789) + startbox = cls1(start) + lengthbox = cls2(length) + if cls1 == cls2 and start == length: + lengthbox = startbox # same box! + self.execute_operation(rop.ZERO_ARRAY, + [BoxPtr(a_ref), + startbox, + lengthbox], + 'void', descr=arraydescr) + assert len(a) == 100 + for i in range(100): + val = (0 if start <= i < start + length + else -123456789) + if OF == PAIR: + assert a[i].a == a[i].b == val + else: + assert a[i] == rffi.cast(OF, val) diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -2363,6 +2363,42 @@ elif IS_X86_64: mc.MOVSX32_rj(loc.value, addr) # memory read, sign-extend + def genop_discard_zero_array(self, op, arglocs): + (base_loc, startindex_loc, bytes_loc, + itemsize_loc, baseofs_loc, null_loc) = arglocs + assert isinstance(bytes_loc, ImmedLoc) + assert isinstance(itemsize_loc, ImmedLoc) + assert isinstance(baseofs_loc, ImmedLoc) + assert isinstance(null_loc, RegLoc) and null_loc.is_xmm + baseofs = baseofs_loc.value + nbytes = bytes_loc.value + if valid_addressing_size(itemsize_loc.value): + scale = get_scale(itemsize_loc.value) + else: + assert isinstance(startindex_loc, ImmedLoc) + assert startindex_loc.value == 0 + scale = 0 + null_reg_cleared = False + i = 0 + while i < nbytes: + addr = addr_add(base_loc, startindex_loc, baseofs + i, scale) + current = nbytes - i + if current >= 16: + current = 16 + if not null_reg_cleared: + self.mc.XORPS_xx(null_loc.value, null_loc.value) + null_reg_cleared = True + self.mc.MOVUPS(addr, null_loc) + else: + if current >= WORD: + current = WORD + elif current >= 4: + current = 4 + elif current >= 2: + current = 2 + self.save_into_mem(addr, imm0, imm(current)) + i += current + genop_discard_list = [Assembler386.not_implemented_op_discard] * rop._LAST genop_list = [Assembler386.not_implemented_op] * rop._LAST diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -8,7 +8,8 @@ unpack_arraydescr, unpack_fielddescr, unpack_interiorfielddescr) from rpython.jit.backend.llsupport.gcmap import allocate_gcmap from rpython.jit.backend.llsupport.regalloc import (FrameManager, BaseRegalloc, - RegisterManager, TempBox, compute_vars_longevity, is_comparison_or_ovf_op) + RegisterManager, TempBox, compute_vars_longevity, is_comparison_or_ovf_op, + valid_addressing_size) from rpython.jit.backend.x86 import rx86 from rpython.jit.backend.x86.arch import (WORD, JITFRAME_FIXED_SIZE, IS_X86_32, IS_X86_64) @@ -1384,6 +1385,71 @@ def consider_keepalive(self, op): pass + def consider_zero_array(self, op): + itemsize, baseofs, _ = unpack_arraydescr(op.getdescr()) + args = op.getarglist() + base_loc = self.rm.make_sure_var_in_reg(args[0], args) + startindex_loc = self.rm.make_sure_var_in_reg(args[1], args) + length_box = op.getarg(2) + if isinstance(length_box, ConstInt): + constbytes = length_box.getint() * itemsize + else: + constbytes = -1 + if 0 <= constbytes <= 16 * 8 and ( + valid_addressing_size(itemsize) or + (isinstance(startindex_loc, ImmedLoc) and + startindex_loc.value == 0)): + if IS_X86_64: + null_loc = X86_64_XMM_SCRATCH_REG + else: + null_box = TempBox() + null_loc = self.xrm.force_allocate_reg(null_box) + self.xrm.possibly_free_var(null_box) + self.perform_discard(op, [base_loc, startindex_loc, + imm(constbytes), imm(itemsize), + imm(baseofs), null_loc]) + else: + # base_loc and startindex_loc are in two regs here (or they are + # immediates). Compute the dstaddr_loc, which is the raw + # address that we will pass as first argument to memset(). + # It can be in the same register as either one, but not in + # args[2], because we're still needing the latter. + dstaddr_box = TempBox() + dstaddr_loc = self.rm.force_allocate_reg(dstaddr_box, [args[2]]) + itemsize_loc = imm(itemsize) + dst_addr = self.assembler._get_interiorfield_addr( + dstaddr_loc, startindex_loc, itemsize_loc, + base_loc, imm(baseofs)) + self.assembler.mc.LEA(dstaddr_loc, dst_addr) + # + if constbytes >= 0: + length_loc = imm(constbytes) + else: + # load length_loc in a register different than dstaddr_loc + length_loc = self.rm.make_sure_var_in_reg(length_box, + [dstaddr_box]) + if itemsize > 1: + # we need a register that is different from dstaddr_loc, + # but which can be identical to length_loc (as usual, + # only if the length_box is not used by future operations) + bytes_box = TempBox() + bytes_loc = self.rm.force_allocate_reg(bytes_box, + [dstaddr_box]) + b_adr = self.assembler._get_interiorfield_addr( + bytes_loc, length_loc, itemsize_loc, imm0, imm0) + self.assembler.mc.LEA(bytes_loc, b_adr) + length_box = bytes_box + length_loc = bytes_loc + # + # call memset() + self.rm.before_call() + self.xrm.before_call() + self.assembler.simple_call_no_collect( + imm(self.assembler.memset_addr), + [dstaddr_loc, imm0, length_loc]) + self.rm.possibly_free_var(length_box) + self.rm.possibly_free_var(dstaddr_box) + def not_implemented_op(self, op): not_implemented("not implemented operation: %s" % op.getopname()) diff --git a/rpython/jit/backend/x86/regloc.py b/rpython/jit/backend/x86/regloc.py --- a/rpython/jit/backend/x86/regloc.py +++ b/rpython/jit/backend/x86/regloc.py @@ -664,6 +664,7 @@ MOVDQ = _binaryop('MOVDQ') MOVD32 = _binaryop('MOVD32') + MOVUPS = _binaryop('MOVUPS') CALL = _relative_unaryop('CALL') JMP = _relative_unaryop('JMP') diff --git a/rpython/jit/backend/x86/rx86.py b/rpython/jit/backend/x86/rx86.py --- a/rpython/jit/backend/x86/rx86.py +++ b/rpython/jit/backend/x86/rx86.py @@ -634,6 +634,9 @@ MOVD32_xs = xmminsn('\x66', rex_nw, '\x0F\x6E', register(1, 8), stack_sp(2)) PSRAD_xi = xmminsn('\x66', rex_nw, '\x0F\x72', register(1), '\xE0', immediate(2, 'b')) + MOVUPS_mx = xmminsn(rex_nw, '\x0F\x11', register(2, 8), mem_reg_plus_const(1)) + MOVUPS_jx = xmminsn(rex_nw, '\x0F\x11', register(2, 8), abs_(1)) + MOVUPS_ax = xmminsn(rex_nw, '\x0F\x11', register(2, 8), mem_reg_plus_scaled_reg_plus_const(1)) # ------------------------------------------------------------ @@ -764,6 +767,7 @@ define_modrm_modes('DIVSD_x*', ['\xF2', rex_nw, '\x0F\x5E', register(1, 8)], regtype='XMM') define_modrm_modes('UCOMISD_x*', ['\x66', rex_nw, '\x0F\x2E', register(1, 8)], regtype='XMM') define_modrm_modes('XORPD_x*', ['\x66', rex_nw, '\x0F\x57', register(1, 8)], regtype='XMM') +define_modrm_modes('XORPS_x*', [rex_nw, '\x0F\x57', register(1, 8)], regtype='XMM') define_modrm_modes('ANDPD_x*', ['\x66', rex_nw, '\x0F\x54', register(1, 8)], regtype='XMM') def define_pxmm_insn(insnname_template, insn_char): diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py --- a/rpython/jit/metainterp/executor.py +++ b/rpython/jit/metainterp/executor.py @@ -326,6 +326,7 @@ rop.COND_CALL_GC_WB, rop.COND_CALL_GC_WB_ARRAY, rop.ZERO_PTR_FIELD, + rop.ZERO_ARRAY, rop.DEBUG_MERGE_POINT, rop.JIT_DEBUG, rop.SETARRAYITEM_RAW, diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py --- a/rpython/jit/metainterp/resoperation.py +++ b/rpython/jit/metainterp/resoperation.py @@ -504,8 +504,10 @@ 'SETINTERIORFIELD_RAW/3d', # right now, only used by tests 'RAW_STORE/3d', 'SETFIELD_GC/2d', - 'ZERO_PTR_FIELD/2', # only emitted by the rewrite, sets a pointer field + 'ZERO_PTR_FIELD/2', # only emitted by the rewrite, clears a pointer field # at a given offset, no descr + 'ZERO_ARRAY/3d', # only emitted by the rewrite, clears (part of) an array + # [arraygcptr, firstindex, length], descr=ArrayDescr 'SETFIELD_RAW/2d', 'STRSETITEM/3', 'UNICODESETITEM/3', From noreply at buildbot.pypy.org Fri Sep 19 08:33:06 2014 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 19 Sep 2014 08:33:06 +0200 (CEST) Subject: [pypy-commit] extradoc extradoc: slides for brno php, draft Message-ID: <20140919063306.E19821D387A@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: extradoc Changeset: r5406:4ce93a3cd8a0 Date: 2014-09-19 08:32 +0200 http://bitbucket.org/pypy/extradoc/changeset/4ce93a3cd8a0/ Log: slides for brno php, draft diff --git a/talk/brno-php-2014/talk.rst b/talk/brno-php-2014/talk.rst new file mode 100644 --- /dev/null +++ b/talk/brno-php-2014/talk.rst @@ -0,0 +1,160 @@ +HippyVM - yet another attempt at PHP performance +------------------------------------------------ + +Who am I? +--------- + +* Maciej Fijalkowski + +* PyPy developer for about 8 years + +* main author of hippyvm + +* founder of baroquesoftware.com + +HippyVM +------- + +* a PHP interpreter (essentially), runs php code + +|pause| + +* has a just in time compiler, which makes it fast + +|pause| + +* dual licensing, commercial and open source + +HippyVM status +-------------- + +* runs a lot of PHP test suite + +* misses a lot of builtin functions + +* can kind of "run" wordpress, mediawiki, getting there + +* cgi, fastcgi (not open source) + +* quick + +HippyVM status - short +---------------------- + +* fast, compliant + +* not quite ready + +Let's talk about performance +---------------------------- + +* "I want my website to run faster" + +* "I'm going to compare various languages, implementations" + +|pause| + +* "I'll use a recursive fibonacci function" + +Performance +----------- + +* a very complex picture + +* bottlenecks depend on the use case + +* libraries, programmers, styles matter + +Performance - let's try science +------------------------------- + +* get a set of programs (small, medium, large) + +* compare them + +* don't use a single number + +PHP performance +--------------- + +* number of requests per second + +* loading the code + +* accessing the DB + +* gluing it all together + +PHP performance landscape +------------------------- + +* Zend PHP + +* HHVM + +* php-ng + +* HippyVM + +* other projects + +Current HippyVM performance +--------------------------- + +* we can't really compare large things (mediawiki ~on par) + +* small and medium between 2x faster - 2x slower than HHVM + +|pause| + +* very hand-wavy, but you really need to do it yourself + +|pause| + +* consider bottlenecks differ depending on implementation + +Performance - personal opinions +------------------------------- + +* the language should be easy **for a programmer** + +* the language implementation can be complex + +* libraries, patterns and the ecosystem matter for anything non-trivial + +Performance - questions +----------------------- + +* ? + +HippyVM history +--------------- + +* started as a facebook research project + +* got some funding to pursue as a commercial project + +* an offspin of PyPy technology + +PyPy +---- + +* a fast, compliant Python interpreter + +* 0.5%-1% of Python market share + +* a framework for building efficient interpreters + +* 10 years of research + +* fully Open Source, EU funded project + +PyPy history +------------ + +xxxx + +HippyVM history +--------------- + +xxxx From noreply at buildbot.pypy.org Fri Sep 19 08:33:08 2014 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 19 Sep 2014 08:33:08 +0200 (CEST) Subject: [pypy-commit] extradoc extradoc: merge Message-ID: <20140919063308.6E3F01D387A@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: extradoc Changeset: r5407:7adbccfa295d Date: 2014-09-19 08:33 +0200 http://bitbucket.org/pypy/extradoc/changeset/7adbccfa295d/ Log: merge diff --git a/talk/pycon-uk-2014/Makefile b/talk/pycon-uk-2014/Makefile new file mode 100644 --- /dev/null +++ b/talk/pycon-uk-2014/Makefile @@ -0,0 +1,12 @@ +talk.pdf: talk.rst author.latex stylesheet.latex + rst2beamer --stylesheet=stylesheet.latex --overlaybullets=False --documentoptions=14pt talk.rst talk.latex || exit + sed 's/\\date{}/\\input{author.latex}/' -i talk.latex || exit + sed 's/XXXFAKETITLE//' -i talk.latex || exit + #sed 's/\\maketitle/\\input{title.latex}/' -i talk.latex || exit + pdflatex talk.latex || exit + +view: talk.pdf + evince talk.pdf & + +xpdf: talk.pdf + xpdf talk.pdf & diff --git a/talk/pycon-uk-2014/author.latex b/talk/pycon-uk-2014/author.latex new file mode 100644 --- /dev/null +++ b/talk/pycon-uk-2014/author.latex @@ -0,0 +1,8 @@ +\definecolor{rrblitbackground}{rgb}{0.0, 0.0, 0.0} + +\title{PyPy and its ecosystem} +\author[Ronan Lamy]{Ronan Lamy\\ +\includegraphics[width=80px]{../img/py-web-new.png}} + +\institute{Pycon UK} +\date{19 September 2014} diff --git a/talk/pycon-italy-2014/beamerdefs.txt b/talk/pycon-uk-2014/beamerdefs.txt copy from talk/pycon-italy-2014/beamerdefs.txt copy to talk/pycon-uk-2014/beamerdefs.txt diff --git a/talk/pycon-uk-2014/how-jit.png b/talk/pycon-uk-2014/how-jit.png new file mode 100644 index 0000000000000000000000000000000000000000..ad9ce720343f9b461e202faacd566abc3a331b61 GIT binary patch [cut] diff --git a/talk/pycon-uk-2014/how-pypy.png b/talk/pycon-uk-2014/how-pypy.png new file mode 100644 index 0000000000000000000000000000000000000000..203ad9bc6073ee2fda4d9e624f6bf6bd1941b4cb GIT binary patch [cut] diff --git a/talk/pycon-uk-2014/how-rpython.png b/talk/pycon-uk-2014/how-rpython.png new file mode 100644 index 0000000000000000000000000000000000000000..a98cd6335f01d9bcfc2638d72a446a3f3af0cba8 GIT binary patch [cut] diff --git a/talk/pycon-italy-2014/speed.png b/talk/pycon-uk-2014/speed.png copy from talk/pycon-italy-2014/speed.png copy to talk/pycon-uk-2014/speed.png diff --git a/talk/pycon-uk-2014/stylesheet.latex b/talk/pycon-uk-2014/stylesheet.latex new file mode 100644 --- /dev/null +++ b/talk/pycon-uk-2014/stylesheet.latex @@ -0,0 +1,9 @@ +\usetheme{Boadilla} +\setbeamertemplate{navigation symbols}{} + +\definecolor{darkgreen}{rgb}{0, 0.5, 0.0} +\newcommand{\docutilsrolegreen}[1]{\color{darkgreen}#1\normalcolor} +\newcommand{\docutilsrolered}[1]{\color{red}#1\normalcolor} + +\newcommand{\green}[1]{\color{darkgreen}#1\normalcolor} +\newcommand{\red}[1]{\color{red}#1\normalcolor} diff --git a/talk/pycon-italy-2014/talk.pdf.info b/talk/pycon-uk-2014/talk.pdf.info copy from talk/pycon-italy-2014/talk.pdf.info copy to talk/pycon-uk-2014/talk.pdf.info diff --git a/talk/pycon-uk-2014/talk.rst b/talk/pycon-uk-2014/talk.rst new file mode 100644 --- /dev/null +++ b/talk/pycon-uk-2014/talk.rst @@ -0,0 +1,511 @@ +.. include:: beamerdefs.txt + +=========================== +XXXFAKETITLE +=========================== + +About me +--------- + +- PyPy core dev + +- Working on HippyVM + + +What is PyPy? +-------------- + +- "PyPy is a fast, compliant alternative implementation of the Python language (2.7.8 and 3.2.5)." + +- Python interpreter + + * written in RPython + + * **FAST** + +How does RPython work? +---------------------- + +.. image:: how-rpython.png + +.. raw:: latex + + {\tiny + +Altered from http://abstrusegoose.com/secretarchives/under-the-hood - CC BY-NC 3.0 US + +|>| + +What is RPython? +---------------- + +- Subset of Python + + * easy to read + + * easy to test + +- JIT & GC for free + +- General framework for dynamic languages + + +RPython-powered languages +------------------------- + +- **PyPy** + +|pause| + +- HippyVM: implementing PHP + + * ~7x faster than standard PHP + + * http://hippyvm.com/ + +|pause| + +- Topaz: implementing Ruby + + * https://github.com/topazproject/topaz + +|pause| + +- Pyrolog (Prolog) + +- RTruffleSOM (Smalltalk) + +- RSqueakVM (Smalltalk) + +- lang-js (JavaScript) + + +RPython translation stages +-------------------------- + +- (R)Python code + +|pause| + +- ``import`` + + * Python objects (functions, classes, ...) + +|pause| + +- Bytecode analysis, type inference + + * Typed control flow graph + +|pause| + +- Translator transformations + + * Add GC & JIT + +|pause| + +- Code generation + + * C code + +|pause| + +- ``gcc`` + + * Compiled executable + +How does the JIT work? +---------------------- + +|pause| + +.. image:: how-jit.png + +.. raw:: latex + + {\tiny + +Altered from http://abstrusegoose.com/secretarchives/under-the-hood - CC BY-NC 3.0 US + +|>| + + +How does the JIT work? +---------------------- + +- "Jitcode": very low-level byte code + + * Translates to machine code + +- Translation time + + * Add jitcode representation to RPython functions + +- Run-time: + + * Detect **hot** loop + + * Trace one code path through the loop + + * Compile (magic!) + + - with guards to ensure correctness + + * Profit! + + +RPython example (HippyVM) +------------------------- + +|scriptsize| + +.. sourcecode:: python + + @wrap(['space', str, W_Root, Optional(int)]) + def strpos(space, haystack, w_needle, offset=0): + """Find the position of the first occurrence of a substring in a string.""" + if offset < 0 or offset > len(haystack): + space.ec.warn("strpos(): Offset not contained in string") + return space.w_False + try: + needle = unwrap_needle(space, w_needle) + except ValidationError as exc: + space.ec.warn("strpos(): " + exc.msg) + return space.w_False + if len(needle) == 0: + space.ec.warn("strpos(): Empty needle") + return space.w_False + + result = haystack.find(needle, offset) + + if result == -1: + return space.w_False + return space.newint(result) + +|end_scriptsize| + + +JITting strpos() +---------------- + +|example<| strpos_jit.php |>| + +|scriptsize| + +.. sourcecode:: php + + + +|end_scriptsize| + +|end_example| + +JITting strpos() +---------------- + +|scriptsize| + +.. parsed-literal:: + + label(p0, p1, p3, p7, i48, i52, p33, p26, descr=TargetToken(140099127632144)) + i57 = int_eq(i52, 100000) + guard_false(i57, descr=) [p1, p0, i52, p3, p7, i48] + i58 = int_lt(i52, 100000) + guard_true(i58, descr=) [p1, p0, p3, p7, i52, i48] + guard_not_invalidated(descr=) [p1, p0, p3, p7, p33, i52, i48] + **p59 = call(ConstClass(ll_str__IntegerR_SignedConst_Signed), i52, descr=)** + guard_no_exception(descr=) [p1, p0, p59, p3, p7, p33, i52, i48] + i60 = strlen(p59) + **i61 = call(ConstClass(ll_find_char__rpy_stringPtr_Char_Signed_Signed), p59, 49, 0, i60, descr=)** + i62 = int_eq(i61, -1) + guard_false(i62, descr=) [p1, p0, i61, p3, p7, p33, i52, i48] + i63 = int_add_ovf(i48, i61) + guard_no_overflow(descr=) [p1, p0, i63, i61, i48, p3, p7, p33, i52, None] + i64 = int_add(i52, 1) + i65 = getfield_raw(24210432, descr=) + i66 = int_lt(i65, 0) + guard_false(i66, descr=) [p1, p0, p3, p7, i63, i64, None, None, None] + i67 = arraylen_gc(p26, descr=) + jump(p0, p1, p3, p7, i63, i64, p33, p26, descr=TargetToken(140099127632144)) + +|end_scriptsize| + +How does PyPy work? +---------------------- + +.. image:: how-pypy.png + +.. raw:: latex + + {\tiny + +Altered from http://abstrusegoose.com/secretarchives/under-the-hood - CC BY-NC 3.0 US + +|>| + + +PyPy: past two years (1) +----------------------------- + +- PyPy 2.0 (May 2013) + + * Beta ARM, CFFI, unicode performance + + * stackless + JIT (eventlet, gevent, ...) + +|pause| + +- PyPy 2.1 (July 2013) + + * Stable ARM + + * py3k (3.2.3), numpy, general improvements, bugfixes + +|pause| + +- PyPy 2.2 (November 2013) + + * Incremental GC, faster JSON + + * More JIT, more py3k + + * More numpy, numpy C API + + +PyPy: past two years (2) +------------------------- + +- PyPy 2.3 (May 2014) + + * Lot of internal refactoring + + * C API for embedding + + * General improvements + +|pause| + +- PyPy 2.4 (coming soon!) + + * Python 2.7.8 stdlib + + * General fixes and improvements + + +Current status +--------------- + +- Python code: "it just works" + +- 2.7: very stable + +- 3.2: stable + + * Some missing optimizations + +- 3.3: in-progress + + +Speed: 6.3x faster than CPython +-------------------------------- + +.. image:: speed.png + :scale: 47% + + +ARM +---- + +- Official support since PyPy 2.1 + +- "it just works" + +- ~7.5x faster than CPython on ARM + +- Thanks to Raspberry-Pi foundation + +- Distributed as part of Raspbian OS + +jitviewer +--------- + +- Python code: "it just works" + + * but is it fast? + +- jitviewer: find out what the JIT does + + * http://bitbucket.org/pypy/jitviewer + + +Compiled extensions +------------------- + +- cpyext: incomplete, slow + +- ctypes: inconvenient, slow + +- "Don't write your own C, PyPy is fast enough" + +|pause| + +- Native PyPy C API for embedding + +- CFFI: the future + +- numpy: in-progress (more later) + + +CFFI +----- + +- Python <-> C interfacing done right + + * Inspired by LuaJIT's FFI + + * existing shared libraries + + * custom C code + +- Alternative to C-API, ctypes, Cython, etc. + +- Fast on CPython, super-fast on PyPy + +- Lots of CFFI modules around: + + * pygame_cffi, psycopg2_cffi, lxml, cryptography, pyzmq, ... + + +CFFI +-------------------- + +|example<| Simple example |>| +|scriptsize| + +.. sourcecode:: python + + >>> from cffi import FFI + >>> ffi = FFI() + >>> ffi.cdef(""" + ... int printf(const char *format, ...); // from man page + ... """) + >>> C = ffi.dlopen(None) # loads whole C namespace + >>> arg = ffi.new("char[]", "world") # char arg[] = "world"; + >>> C.printf("hi there, %s!\n", arg) # call printf() + hi there, world! + +|end_scriptsize| +|end_example| + +- Docs: http://cffi.readthedocs.org + + +numpy +----- + +- Core inside PyPy + +- The rest at http://bitbucket.org/pypy/numpy + +- ~80% of numpy implemented + + * 2415 passing tests out of 3265 + +- In progress, just try it + +- No Python 3 support + +- No scipy :-/ + + +The future: STM +---------------- + +- Software Transactional Memory + +- "Easier parallelism for everybody" + +- On-going research project + + * by Armin Rigo and Remi Meier + +Transactional Memory +-------------------- + +* like GIL, but instead of blocking, each thread runs optimistically + +* "easy" to implement: + + - GIL acquire -> transaction start + + - GIL release -> transaction commit + +* overhead: cross-checking conflicting memory reads and writes, + and if necessary, cancel and restart transactions + + +Longer transactions +------------------- + +- Larger atomic blocks + +- threads and application-level locks still needed... + +- but *can be very coarse:* + + - two transactions can optimistically run in parallel + + - even if they both *acquire and release the same lock* + +- Can be hidden by libraries to provide even higher level paradigms + + * e.g.: Twisted apps made parallel out of the box + + +STM status +---------- + +- Preliminary versions of pypy-jit-stm available + +- Best case 25-40% overhead (much better than originally planned) + +- Lots of polishing needed + + +Fundraising campaign +--------------------- + +- py3k: 52,380 $ of 105,000 $ (49.9%) + +- numpy: 48,412 $ of 60,000 $ (80.7%) + +- STM (2nd call): 13,939 $ of 80,000 $ (17.4%) + +- General PyPy progress + +- **September only**: PSF matches donations + +- Thanks to all donors! + + +Contacts, Q&A +-------------- + +- http://pypy.org + +- Commercial support: http://baroquesoftware.com + +- IRC: ``#pypy`` on freenode.net + +- Any question? diff --git a/talk/pycon-italy-2014/title.latex b/talk/pycon-uk-2014/title.latex copy from talk/pycon-italy-2014/title.latex copy to talk/pycon-uk-2014/title.latex From noreply at buildbot.pypy.org Fri Sep 19 10:33:59 2014 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 19 Sep 2014 10:33:59 +0200 (CEST) Subject: [pypy-commit] extradoc extradoc: review Message-ID: <20140919083359.7ED491D29E9@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r5408:4d14086777fb Date: 2014-09-19 10:34 +0200 http://bitbucket.org/pypy/extradoc/changeset/4d14086777fb/ Log: review diff --git a/talk/brno-php-2014/talk.rst b/talk/brno-php-2014/talk.rst --- a/talk/brno-php-2014/talk.rst +++ b/talk/brno-php-2014/talk.rst @@ -36,7 +36,7 @@ * cgi, fastcgi (not open source) -* quick +* quick XXX do you mean 'fast'? but next slide already says 'fast' HippyVM status - short ---------------------- From noreply at buildbot.pypy.org Fri Sep 19 10:45:58 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Fri, 19 Sep 2014 10:45:58 +0200 (CEST) Subject: [pypy-commit] stmgc c8-new-page-handling: huge mess in progress Message-ID: <20140919084558.B90FE1C0432@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: c8-new-page-handling Changeset: r1393:b2fd01cfefe6 Date: 2014-09-19 10:46 +0200 http://bitbucket.org/pypy/stmgc/changeset/b2fd01cfefe6/ Log: huge mess in progress diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -2,9 +2,135 @@ # error "must be compiled via stmgc.c" #endif -#include +/* ############# signal handler ############# */ +static void _update_obj_from(int from_seg, object_t *obj); +static void copy_bk_objs_from(int from_segnum, uintptr_t pagenum) +{ + acquire_modified_objs_lock(from_segnum); + struct tree_s *tree = get_priv_segment(from_segnum)->modified_old_objects; + wlog_t *item; + TREE_LOOP_FORWARD(tree, item); { + if (item->addr >= pagenum * 4096UL && item->addr < (pagenum + 1) * 4096UL) { + object_t *obj = (object_t*)item->addr; + struct object_s* bk_obj = (struct object_s *)item->val; + size_t obj_size; + + obj_size = stmcb_size_rounded_up(bk_obj); + + memcpy_to_accessible_pages(STM_SEGMENT->segment_num, + obj, (char*)bk_obj, obj_size); + + assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); /* bk_obj never written */ + } + } TREE_LOOP_END; + + release_modified_objs_lock(from_segnum); +} + +static void update_page_from_to( + uintptr_t pagenum, struct stm_commit_log_entry_s *from, + struct stm_commit_log_entry_s *to) +{ + assert(all_privatization_locks_acquired()); + + volatile struct stm_commit_log_entry_s *cl; + cl = (volatile struct stm_commit_log_entry_s *)from; + + if (from == to) + return; + + while ((cl = cl->next)) { + if ((uintptr_t)cl == -1) + return; + + OPT_ASSERT(cl->segment_num >= 0 && cl->segment_num < NB_SEGMENTS); + + object_t *obj; + size_t i = 0; + while ((obj = cl->written[i])) { + _update_obj_from(cl->segment_num, obj); + + i++; + }; + + /* last fully validated entry */ + if (cl == to) + return; + } +} + +static void bring_page_up_to_date(uintptr_t pagenum) +{ + /* XXX: bad, but no deadlocks: */ + acquire_all_privatization_locks(); + + long i; + int my_segnum = STM_SEGMENT->segment_num; + + assert(get_page_status_in(my_segnum, pagenum) == PAGE_NO_ACCESS); + + /* find who has the PAGE_SHARED */ + int shared_page_holder = -1; + int shared_ref_count = 0; + for (i = 0; i < NB_SEGMENTS; i++) { + if (i == my_segnum) + continue; + if (get_page_status_in(i, pagenum) == PAGE_SHARED) { + shared_page_holder = i; + shared_ref_count++; + } + } + assert(shared_page_holder != -1); + + /* XXX: for now, we don't try to get the single shared page. We simply + regard it as private for its holder. */ + /* this assert should be true for now... */ + assert(shared_ref_count == 1); + + /* make our page private */ + page_privatize_in(STM_SEGMENT->segment_num, pagenum); + volatile char *dummy = REAL_ADDRESS(STM_SEGMENT->segment_base, pagenum*4096UL); + *dummy = *dummy; /* force copy-on-write from shared page */ + + /* if there were modifications in the page, revert them: */ + copy_bk_objs_from(shared_page_holder, pagenum); + + /* if not already newer, update page to our revision */ + update_page_from_to( + pagenum, get_priv_segment(shared_page_holder)->last_commit_log_entry, + STM_PSEGMENT->last_commit_log_entry); + + /* in case page is already newer, validate everything now to have a common + revision for all pages */ + stm_validate(NULL); + + release_all_privatization_locks(); +} + +static void _signal_handler(int sig, siginfo_t *siginfo, void *context) +{ + char *addr = siginfo->si_addr; + dprintf(("si_addr: %p\n", addr)); + if (addr == NULL || addr < stm_object_pages || addr > stm_object_pages+TOTAL_MEMORY) { + /* actual segfault */ + /* send to GDB (XXX) */ + kill(getpid(), SIGINT); + } + /* XXX: should we save 'errno'? */ + + + int segnum = get_segment_of_linear_address(addr); + OPT_ASSERT(segnum == STM_SEGMENT->segment_num); + dprintf(("-> segment: %d\n", segnum)); + char *seg_base = STM_SEGMENT->segment_base; + uintptr_t pagenum = ((char*)addr - seg_base) / 4096UL; + + bring_page_up_to_date(pagenum); + + return; +} /* ############# commit log ############# */ @@ -30,17 +156,10 @@ } } + static void _update_obj_from(int from_seg, object_t *obj) { - /* during validation this looks up the obj in the - from_seg (backup or normal) and copies the version - over the current segment's one */ size_t obj_size; - char *realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj); - uintptr_t pagenum = (uintptr_t)obj / 4096UL; - - assert(get_page_status_in(from_seg, pagenum) != PAGE_NO_ACCESS); - assert(get_page_status_in(STM_SEGMENT->segment_num, pagenum) != PAGE_NO_ACCESS); /* look the obj up in the other segment's modified_old_objects to get its backup copy: */ @@ -51,7 +170,10 @@ TREE_FIND(tree, (uintptr_t)obj, item, goto not_found); obj_size = stmcb_size_rounded_up((struct object_s*)item->val); - memcpy(realobj, (char*)item->val, obj_size); + + memcpy_to_accessible_pages(STM_SEGMENT->segment_num, obj, + (char*)item->val, obj_size); + assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); release_modified_objs_lock(from_seg); return; @@ -60,9 +182,11 @@ /* copy from page directly (obj is unmodified) */ obj_size = stmcb_size_rounded_up( (struct object_s*)REAL_ADDRESS(get_segment_base(from_seg), obj)); - memcpy(realobj, - REAL_ADDRESS(get_segment_base(from_seg), obj), - obj_size); + + memcpy_to_accessible_pages(STM_SEGMENT->segment_num, obj, + REAL_ADDRESS(get_segment_base(from_seg), obj), + obj_size); + obj->stm_flags |= GCFLAG_WRITE_BARRIER; /* may already be gone */ release_modified_objs_lock(from_seg); } @@ -77,6 +201,7 @@ assert((uintptr_t)STM_PSEGMENT->last_commit_log_entry->next == -1); return; } + assert(all_privatization_locks_acquired()); volatile struct stm_commit_log_entry_s *cl, *prev_cl; cl = prev_cl = (volatile struct stm_commit_log_entry_s *) diff --git a/c8/stm/core.h b/c8/stm/core.h --- a/c8/stm/core.h +++ b/c8/stm/core.h @@ -6,6 +6,8 @@ #include #include #include +#include + /************************************************************/ @@ -135,7 +137,7 @@ static stm_thread_local_t *abort_with_mutex_no_longjmp(void); static void abort_data_structures_from_segment_num(int segment_num); - +static void _signal_handler(int sig, siginfo_t *siginfo, void *context); static inline void _duck(void) { /* put a call to _duck() between two instructions that set 0 into @@ -165,3 +167,34 @@ { spinlock_release(get_priv_segment(segnum)->modified_objs_lock); } + + +static inline bool all_privatization_locks_acquired() +{ +#ifndef NDEBUG + long l; + for (l = 0; l < NB_SEGMENTS; l++) { + if (!get_priv_segment(l)->privatization_lock) + return false; + } + return true; +#else + abort(); +#endif +} + +static inline void acquire_all_privatization_locks() +{ + long l; + for (l = 0; l < NB_SEGMENTS; l++) { + acquire_privatization_lock(l); + } +} + +static inline void release_all_privatization_locks() +{ + long l; + for (l = NB_SEGMENTS-1; l >= 0; l--) { + release_privatization_lock(l); + } +} diff --git a/c8/stm/pages.c b/c8/stm/pages.c --- a/c8/stm/pages.c +++ b/c8/stm/pages.c @@ -54,14 +54,10 @@ /* call remap_file_pages() to make all pages in the range(pagenum, pagenum+count) PAGE_SHARED in segnum, and PAGE_NO_ACCESS in other segments */ - dprintf(("pages_initialize_shared: 0x%ld - 0x%ld\n", pagenum, - pagenum + count)); -#ifndef NDEBUG - long l; - for (l = 0; l < NB_SEGMENTS; l++) { - assert(get_priv_segment(l)->privatization_lock); - } -#endif + dprintf(("pages_initialize_shared: 0x%ld - 0x%ld\n", pagenum, pagenum + count)); + + assert(all_privatization_locks_acquired()); + assert(pagenum < NB_PAGES); if (count == 0) return; @@ -85,43 +81,43 @@ } -/* static void page_privatize_in(int segnum, uintptr_t pagenum, char *initialize_from) */ -/* { */ -/* #ifndef NDEBUG */ -/* long l; */ -/* for (l = 0; l < NB_SEGMENTS; l++) { */ -/* assert(get_priv_segment(l)->privatization_lock); */ -/* } */ -/* #endif */ +static void page_privatize_in(int segnum, uintptr_t pagenum) +{ +#ifndef NDEBUG + long l; + for (l = 0; l < NB_SEGMENTS; l++) { + assert(get_priv_segment(l)->privatization_lock); + } +#endif + assert(get_page_status_in(segnum, pagenum) == PAGE_NO_ACCESS); + dprintf(("page_privatize(%lu) in seg:%d\n", pagenum, segnum)); -/* /\* check this thread's 'pages_privatized' bit *\/ */ -/* uint64_t bitmask = 1UL << segnum; */ -/* volatile struct page_shared_s *ps = (volatile struct page_shared_s *) */ -/* &pages_privatized[pagenum - PAGE_FLAG_START]; */ -/* if (ps->by_segment & bitmask) { */ -/* /\* the page is already privatized; nothing to do *\/ */ -/* return; */ -/* } */ + char *addr = (char*)(get_virt_page_of(segnum, pagenum) * 4096UL); + char *result = mmap( + addr, 4096UL, PROT_READ | PROT_WRITE, + MAP_FIXED | MAP_PRIVATE | MAP_NORESERVE, + stm_object_pages_fd, get_file_page_of(pagenum)); + if (result == MAP_FAILED) + stm_fatalerror("page_privatize_in failed (mmap): %m"); -/* dprintf(("page_privatize(%lu) in seg:%d\n", pagenum, segnum)); */ + set_page_status_in(segnum, pagenum, PAGE_PRIVATE); +} -/* /\* add this thread's 'pages_privatized' bit *\/ */ -/* ps->by_segment |= bitmask; */ -/* /\* "unmaps" the page to make the address space location correspond */ -/* again to its underlying file offset (XXX later we should again */ -/* attempt to group together many calls to d_remap_file_pages() in */ -/* succession) *\/ */ -/* uintptr_t pagenum_in_file = NB_PAGES * segnum + pagenum; */ -/* char *new_page = stm_object_pages + pagenum_in_file * 4096UL; */ +static void memcpy_to_accessible_pages( + int dst_segnum, object_t *dst_obj, char *src, size_t len) +{ + /* XXX: optimize */ -/* /\* first write to the file page directly: *\/ */ -/* ssize_t written = pwrite(stm_object_pages_fd, initialize_from, 4096UL, */ -/* pagenum_in_file * 4096UL); */ -/* if (written != 4096) */ -/* stm_fatalerror("pwrite didn't write the whole page: %zd", written); */ + char *realobj = REAL_ADDRESS(get_segment_base(dst_segnum), dst_obj); + char *dst_end = realobj + len; + uintptr_t loc_addr = (uintptr_t)dst_obj; -/* /\* now remap virtual page in segment to the new file page *\/ */ -/* write_fence(); */ -/* d_remap_file_pages(new_page, 4096, pagenum_in_file); */ -/* } */ + while (realobj != dst_end) { + if (get_page_status_in(dst_segnum, loc_addr / 4096UL) != PAGE_NO_ACCESS) + *realobj = *src; + realobj++; + loc_addr++; + src++; + } +} diff --git a/c8/stm/pages.h b/c8/stm/pages.h --- a/c8/stm/pages.h +++ b/c8/stm/pages.h @@ -46,7 +46,23 @@ static struct page_shared_s pages_status[NB_SHARED_PAGES]; static void pages_initialize_shared_for(long segnum, uintptr_t pagenum, uintptr_t count); -/* static void page_privatize_in(int segnum, uintptr_t pagenum, char *initialize_from); */ +static void page_privatize_in(int segnum, uintptr_t pagenum); +static void memcpy_to_accessible_pages(int dst_segnum, object_t *dst_obj, char *src, size_t len); + + + + +static inline uintptr_t get_virt_page_of(long segnum, uintptr_t pagenum) +{ + /* logical page -> virtual page */ + return (uintptr_t)get_segment_base(segnum) / 4096UL + pagenum; +} + +static inline uintptr_t get_file_page_of(uintptr_t pagenum) +{ + /* logical page -> file page */ + return pagenum - PAGE_FLAG_START; +} static inline uint8_t get_page_status_in(long segnum, uintptr_t pagenum) @@ -54,7 +70,7 @@ int seg_shift = segnum * 2; uint64_t bitmask = 3UL << seg_shift; volatile struct page_shared_s *ps = (volatile struct page_shared_s *) - &pages_status[pagenum - PAGE_FLAG_START]; + &pages_status[get_file_page_of(pagenum)]; return ((ps->by_segment & bitmask) >> seg_shift) & 3; } @@ -65,16 +81,8 @@ int seg_shift = segnum * 2; volatile struct page_shared_s *ps = (volatile struct page_shared_s *) - &pages_status[pagenum - PAGE_FLAG_START]; + &pages_status[get_file_page_of(pagenum)]; assert(status != get_page_status_in(segnum, pagenum)); ps->by_segment |= status << seg_shift; } - - - -static inline uintptr_t get_virt_page_of(long segnum, uintptr_t pagenum) -{ - /* logical page -> virtual page */ - return (uintptr_t)get_segment_base(segnum) / 4096UL + pagenum; -} diff --git a/c8/stm/setup.c b/c8/stm/setup.c --- a/c8/stm/setup.c +++ b/c8/stm/setup.c @@ -2,8 +2,9 @@ # error "must be compiled via stmgc.c" #endif +#include +#include /* For O_* constants */ -#include /* For O_* constants */ static void setup_mmap(char *reason) { char name[] = "/__stmgc_c8__"; @@ -79,6 +80,21 @@ } +static void setup_signal_handler(void) +{ + struct sigaction act; + memset(&act, 0, sizeof(act)); + + act.sa_sigaction = &_signal_handler; + /* The SA_SIGINFO flag tells sigaction() to use the sa_sigaction field, not sa_handler. */ + act.sa_flags = SA_SIGINFO | SA_NODEFER; + + if (sigaction(SIGSEGV, &act, NULL) < 0) { + perror ("sigaction"); + abort(); + } +} + void stm_setup(void) { /* Check that some values are acceptable */ @@ -103,6 +119,7 @@ assert(stm_file_pages); setup_protection_settings(); + setup_signal_handler(); long i; for (i = 0; i < NB_SEGMENTS; i++) { From noreply at buildbot.pypy.org Fri Sep 19 12:35:56 2014 From: noreply at buildbot.pypy.org (rlamy) Date: Fri, 19 Sep 2014 12:35:56 +0200 (CEST) Subject: [pypy-commit] extradoc extradoc: tweak Message-ID: <20140919103556.5875C1D29E9@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: extradoc Changeset: r5409:5145ec98b828 Date: 2014-09-19 09:23 +0100 http://bitbucket.org/pypy/extradoc/changeset/5145ec98b828/ Log: tweak diff --git a/talk/pycon-uk-2014/talk.rst b/talk/pycon-uk-2014/talk.rst --- a/talk/pycon-uk-2014/talk.rst +++ b/talk/pycon-uk-2014/talk.rst @@ -4,25 +4,11 @@ XXXFAKETITLE =========================== -About me ---------- - -- PyPy core dev - -- Working on HippyVM - - What is PyPy? -------------- - "PyPy is a fast, compliant alternative implementation of the Python language (2.7.8 and 3.2.5)." -- Python interpreter - - * written in RPython - - * **FAST** - How does RPython work? ---------------------- @@ -39,15 +25,84 @@ What is RPython? ---------------- -- Subset of Python +- General framework for dynamic languages + +- RPython: a subset of Python * easy to read * easy to test -- JIT & GC for free +- RPython toolchain -- General framework for dynamic languages + * Translates RPython code to C + + * Just-in-time compiler (JIT) & garbage collector (GC) for free + + +RPython translation stages +-------------------------- + +- (R)Python code + +- ``import`` + + * Python objects (functions, classes, ...) + +- Bytecode analysis, type inference + + * Typed control flow graph + +- Translator transformations + + * Add GC & **JIT** + +- Code generation + + * C code + +- ``gcc`` + + * Compiled executable + +How does the JIT work? +---------------------- + +|pause| + +.. image:: how-jit.png + +.. raw:: latex + + {\tiny + +Altered from http://abstrusegoose.com/secretarchives/under-the-hood - CC BY-NC 3.0 US + +|>| + + +How does the JIT work? +---------------------- + +- "Jitcode": very low-level byte code + + * Translates to machine code + +- Translation time + + * Add jitcode representation to RPython functions + +- Run-time: + + * Detect **hot** loop + + * Trace one code path through the loop + + * Compile (magic!) + + - with guards to ensure correctness + + * Profit! RPython-powered languages @@ -80,81 +135,6 @@ - lang-js (JavaScript) -RPython translation stages --------------------------- - -- (R)Python code - -|pause| - -- ``import`` - - * Python objects (functions, classes, ...) - -|pause| - -- Bytecode analysis, type inference - - * Typed control flow graph - -|pause| - -- Translator transformations - - * Add GC & JIT - -|pause| - -- Code generation - - * C code - -|pause| - -- ``gcc`` - - * Compiled executable - -How does the JIT work? ----------------------- - -|pause| - -.. image:: how-jit.png - -.. raw:: latex - - {\tiny - -Altered from http://abstrusegoose.com/secretarchives/under-the-hood - CC BY-NC 3.0 US - -|>| - - -How does the JIT work? ----------------------- - -- "Jitcode": very low-level byte code - - * Translates to machine code - -- Translation time - - * Add jitcode representation to RPython functions - -- Run-time: - - * Detect **hot** loop - - * Trace one code path through the loop - - * Compile (magic!) - - - with guards to ensure correctness - - * Profit! - - RPython example (HippyVM) ------------------------- @@ -360,10 +340,12 @@ |pause| +- CFFI: the future + +|pause| + - Native PyPy C API for embedding -- CFFI: the future - - numpy: in-progress (more later) From noreply at buildbot.pypy.org Fri Sep 19 12:35:57 2014 From: noreply at buildbot.pypy.org (rlamy) Date: Fri, 19 Sep 2014 12:35:57 +0200 (CEST) Subject: [pypy-commit] extradoc extradoc: merge heads Message-ID: <20140919103557.7296F1D29E9@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: extradoc Changeset: r5410:ef7b6443a343 Date: 2014-09-19 11:35 +0100 http://bitbucket.org/pypy/extradoc/changeset/ef7b6443a343/ Log: merge heads diff --git a/talk/brno-php-2014/talk.rst b/talk/brno-php-2014/talk.rst new file mode 100644 --- /dev/null +++ b/talk/brno-php-2014/talk.rst @@ -0,0 +1,160 @@ +HippyVM - yet another attempt at PHP performance +------------------------------------------------ + +Who am I? +--------- + +* Maciej Fijalkowski + +* PyPy developer for about 8 years + +* main author of hippyvm + +* founder of baroquesoftware.com + +HippyVM +------- + +* a PHP interpreter (essentially), runs php code + +|pause| + +* has a just in time compiler, which makes it fast + +|pause| + +* dual licensing, commercial and open source + +HippyVM status +-------------- + +* runs a lot of PHP test suite + +* misses a lot of builtin functions + +* can kind of "run" wordpress, mediawiki, getting there + +* cgi, fastcgi (not open source) + +* quick XXX do you mean 'fast'? but next slide already says 'fast' + +HippyVM status - short +---------------------- + +* fast, compliant + +* not quite ready + +Let's talk about performance +---------------------------- + +* "I want my website to run faster" + +* "I'm going to compare various languages, implementations" + +|pause| + +* "I'll use a recursive fibonacci function" + +Performance +----------- + +* a very complex picture + +* bottlenecks depend on the use case + +* libraries, programmers, styles matter + +Performance - let's try science +------------------------------- + +* get a set of programs (small, medium, large) + +* compare them + +* don't use a single number + +PHP performance +--------------- + +* number of requests per second + +* loading the code + +* accessing the DB + +* gluing it all together + +PHP performance landscape +------------------------- + +* Zend PHP + +* HHVM + +* php-ng + +* HippyVM + +* other projects + +Current HippyVM performance +--------------------------- + +* we can't really compare large things (mediawiki ~on par) + +* small and medium between 2x faster - 2x slower than HHVM + +|pause| + +* very hand-wavy, but you really need to do it yourself + +|pause| + +* consider bottlenecks differ depending on implementation + +Performance - personal opinions +------------------------------- + +* the language should be easy **for a programmer** + +* the language implementation can be complex + +* libraries, patterns and the ecosystem matter for anything non-trivial + +Performance - questions +----------------------- + +* ? + +HippyVM history +--------------- + +* started as a facebook research project + +* got some funding to pursue as a commercial project + +* an offspin of PyPy technology + +PyPy +---- + +* a fast, compliant Python interpreter + +* 0.5%-1% of Python market share + +* a framework for building efficient interpreters + +* 10 years of research + +* fully Open Source, EU funded project + +PyPy history +------------ + +xxxx + +HippyVM history +--------------- + +xxxx From noreply at buildbot.pypy.org Fri Sep 19 13:09:29 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Fri, 19 Sep 2014 13:09:29 +0200 (CEST) Subject: [pypy-commit] stmgc c8-new-page-handling: progress Message-ID: <20140919110929.8E4891C34C2@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: c8-new-page-handling Changeset: r1394:5647127245ea Date: 2014-09-19 12:54 +0200 http://bitbucket.org/pypy/stmgc/changeset/5647127245ea/ Log: progress diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -91,8 +91,6 @@ /* make our page private */ page_privatize_in(STM_SEGMENT->segment_num, pagenum); - volatile char *dummy = REAL_ADDRESS(STM_SEGMENT->segment_base, pagenum*4096UL); - *dummy = *dummy; /* force copy-on-write from shared page */ /* if there were modifications in the page, revert them: */ copy_bk_objs_from(shared_page_holder, pagenum); @@ -328,33 +326,6 @@ /* ############# STM ############# */ -void _privatize_shared_page(uintptr_t pagenum) -{ - /* privatize pages of obj for our segment iff previously - the pages were fully shared. */ -#ifndef NDEBUG - long l; - for (l = 0; l < NB_SEGMENTS; l++) { - assert(get_priv_segment(l)->privatization_lock); - } -#endif - - uintptr_t i; - int my_segnum = STM_SEGMENT->segment_num; - - XXX(); - //assert(is_shared_log_page(pagenum)); - /* char *src = (char*)(get_virt_page_of(0, pagenum) * 4096UL); */ - - /* for (i = 1; i < NB_SEGMENTS; i++) { */ - /* page_privatize_in(i, pagenum, src); */ - /* } */ - /* set_page_private_in(0, pagenum); */ - - /* OPT_ASSERT(is_private_log_page_in(my_segnum, pagenum)); */ - /* assert(!is_shared_log_page(pagenum)); */ -} - void _stm_write_slowpath(object_t *obj) { assert(_seems_to_be_running_transaction()); @@ -374,31 +345,39 @@ /* add to read set: */ stm_read(obj); - /* create backup copy: */ + /* create backup copy (this may cause several page faults XXX): */ struct object_s *bk_obj = malloc(obj_size); memcpy(bk_obj, realobj, obj_size); - /* if there are shared pages, privatize them */ + /* privatize pages: */ + acquire_all_privatization_locks(); uintptr_t page; for (page = first_page; page <= end_page; page++) { - XXX(); - /* if (is_shared_log_page(page)) { */ - /* long i; */ - /* for (i = 0; i < NB_SEGMENTS; i++) { */ - /* acquire_privatization_lock(i); */ - /* } */ - /* if (is_shared_log_page(page)) */ - /* _privatize_shared_page(page); */ - /* for (i = NB_SEGMENTS-1; i >= 0; i--) { */ - /* release_privatization_lock(i); */ - /* } */ - /* } */ + /* check if our page is private or we are the only shared-page holder */ + assert(get_page_status_in(my_segnum, page) != PAGE_NO_ACCESS); + + if (get_page_status_in(my_segnum, page) == PAGE_PRIVATE) + continue; + + /* make sure all the others are NO_ACCESS + choosing to make us PRIVATE is harder because then nobody must ever + update the shared page in stm_validate() except if it is the sole + reader of it. But then we don't actually know which revision the page is at. */ + long i; + for (i = 0; i < NB_SEGMENTS; i++) { + if (i == my_segnum) + continue; + + if (get_page_status_in(i, page) == PAGE_SHARED) { + /* xxx: unmap? */ + mprotect((char*)(get_virt_page_of(i, page) * 4096UL), 4096UL, PROT_NONE); + set_page_status_in(i, page, PAGE_NO_ACCESS); + } + } } - /* pages not shared anymore. but we still may have - only a read protected page ourselves: */ - - acquire_privatization_lock(my_segnum); + /* all pages are either private or we were the first to write to a shared + page and therefore got it as our private one */ /* remove the WRITE_BARRIER flag */ obj->stm_flags &= ~GCFLAG_WRITE_BARRIER; @@ -407,7 +386,7 @@ LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, obj); /* done fiddling with protection and privatization */ - release_privatization_lock(my_segnum); + release_all_privatization_locks(); /* phew, now add the obj to the write-set and register the backup copy. */ diff --git a/c8/stm/pages.c b/c8/stm/pages.c --- a/c8/stm/pages.c +++ b/c8/stm/pages.c @@ -15,39 +15,6 @@ } /************************************************************/ -static void check_remap_makes_sense(char *addr, size_t size, ssize_t pgoff) -{ - dprintf(("remap_file_pages: 0x%lx bytes: (seg%ld %p) --> (seg%ld %p)\n", - (long)size, - (long)((addr - stm_object_pages) / 4096UL) / NB_PAGES, - (void *)((addr - stm_object_pages) % (4096UL * NB_PAGES)), - (long)pgoff / NB_PAGES, - (void *)((pgoff % NB_PAGES) * 4096UL))); - assert(size % 4096 == 0); - assert(size <= TOTAL_MEMORY); - assert(((uintptr_t)addr) % 4096 == 0); - assert(addr >= stm_object_pages); - assert(addr <= stm_object_pages + TOTAL_MEMORY - size); - assert(pgoff >= 0); - assert(pgoff <= (TOTAL_MEMORY - size) / 4096UL); - - /* assert remappings follow the rule that page N in one segment - can only be remapped to page N in another segment */ - assert(((addr - stm_object_pages) / 4096UL - pgoff) % NB_PAGES == 0); -} - -static void d_remap_file_pages(char *addr, size_t size, ssize_t pgoff) -{ - check_remap_makes_sense(addr, size, pgoff); - - char *res = mmap(addr, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_NORESERVE | MAP_FIXED, - stm_object_pages_fd, pgoff * 4096UL); - if (UNLIKELY(res != addr)) - stm_fatalerror("mmap (remapping page): %m"); -} - static void pages_initialize_shared_for(long segnum, uintptr_t pagenum, uintptr_t count) { @@ -101,6 +68,9 @@ stm_fatalerror("page_privatize_in failed (mmap): %m"); set_page_status_in(segnum, pagenum, PAGE_PRIVATE); + + volatile char *dummy = REAL_ADDRESS(get_segment_base(segnum), pagenum*4096UL); + *dummy = *dummy; /* force copy-on-write from shared page */ } From noreply at buildbot.pypy.org Fri Sep 19 13:09:30 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Fri, 19 Sep 2014 13:09:30 +0200 (CEST) Subject: [pypy-commit] stmgc c8-new-page-handling: change stm_validate() signature Message-ID: <20140919110930.925BE1C34C2@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: c8-new-page-handling Changeset: r1395:18875506f028 Date: 2014-09-19 13:01 +0200 http://bitbucket.org/pypy/stmgc/changeset/18875506f028/ Log: change stm_validate() signature diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -102,9 +102,7 @@ /* in case page is already newer, validate everything now to have a common revision for all pages */ - stm_validate(NULL); - - release_all_privatization_locks(); + _stm_validate(NULL, true); } static void _signal_handler(int sig, siginfo_t *siginfo, void *context) @@ -189,7 +187,8 @@ release_modified_objs_lock(from_seg); } -void stm_validate(void *free_if_abort) + +static void _stm_validate(void *free_if_abort, bool locks_acquired) { /* go from last known entry in commit log to the most current one and apply all changes done @@ -199,7 +198,12 @@ assert((uintptr_t)STM_PSEGMENT->last_commit_log_entry->next == -1); return; } - assert(all_privatization_locks_acquired()); + + if (locks_acquired) { + assert(all_privatization_locks_acquired()); + } else { + acquire_all_privatization_locks(); + } volatile struct stm_commit_log_entry_s *cl, *prev_cl; cl = prev_cl = (volatile struct stm_commit_log_entry_s *) @@ -253,6 +257,7 @@ STM_PSEGMENT->last_commit_log_entry = (struct stm_commit_log_entry_s *)cl; } + release_all_privatization_locks(); if (needs_abort) { free(free_if_abort); stm_abort_transaction(); @@ -303,7 +308,7 @@ /* regular transaction: */ do { - stm_validate(new); + _stm_validate(new, false); /* try attaching to commit log: */ to = &(STM_PSEGMENT->last_commit_log_entry->next); @@ -317,7 +322,7 @@ new = (struct stm_commit_log_entry_s*)-1; do { - stm_validate(NULL); + stm_validate(); /* try attaching to commit log: */ to = &(STM_PSEGMENT->last_commit_log_entry->next); @@ -325,6 +330,11 @@ } /* ############# STM ############# */ +void stm_validate() +{ + _stm_validate(NULL, false); +} + void _stm_write_slowpath(object_t *obj) { @@ -460,7 +470,7 @@ check_nursery_at_transaction_start(); - stm_validate(NULL); + stm_validate(); } long stm_start_transaction(stm_thread_local_t *tl) diff --git a/c8/stm/core.h b/c8/stm/core.h --- a/c8/stm/core.h +++ b/c8/stm/core.h @@ -138,6 +138,7 @@ static void abort_data_structures_from_segment_num(int segment_num); static void _signal_handler(int sig, siginfo_t *siginfo, void *context); +static void _stm_validate(void *free_if_abort, bool locks_acquired); static inline void _duck(void) { /* put a call to _duck() between two instructions that set 0 into diff --git a/c8/stmgc.h b/c8/stmgc.h --- a/c8/stmgc.h +++ b/c8/stmgc.h @@ -71,7 +71,7 @@ char *_stm_real_address(object_t *o); #ifdef STM_TESTS #include -void stm_validate(void *free_if_abort); +void stm_validate(void); bool _stm_was_read(object_t *obj); bool _stm_was_written(object_t *obj); diff --git a/c8/test/support.py b/c8/test/support.py --- a/c8/test/support.py +++ b/c8/test/support.py @@ -41,7 +41,7 @@ void stm_teardown(void); void stm_register_thread_local(stm_thread_local_t *tl); void stm_unregister_thread_local(stm_thread_local_t *tl); -void stm_validate(void *free_if_abort); +void stm_validate(); bool _check_stm_validate(); object_t *stm_setup_prebuilt(object_t *); @@ -183,7 +183,7 @@ } bool _check_stm_validate(void) { - CHECKED(stm_validate(NULL)); + CHECKED(stm_validate()); } #undef CHECKED From noreply at buildbot.pypy.org Fri Sep 19 13:09:31 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Fri, 19 Sep 2014 13:09:31 +0200 (CEST) Subject: [pypy-commit] stmgc c8-new-page-handling: fix wrong nesting Message-ID: <20140919110931.AC3491C34C2@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: c8-new-page-handling Changeset: r1396:6960f1340af8 Date: 2014-09-19 13:06 +0200 http://bitbucket.org/pypy/stmgc/changeset/6960f1340af8/ Log: fix wrong nesting diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -366,6 +366,8 @@ for (page = first_page; page <= end_page; page++) { /* check if our page is private or we are the only shared-page holder */ assert(get_page_status_in(my_segnum, page) != PAGE_NO_ACCESS); + /* XXX: actually, it can be NO_ACCESS if somebody changed that after we + copied from it */ if (get_page_status_in(my_segnum, page) == PAGE_PRIVATE) continue; diff --git a/c8/stm/pages.c b/c8/stm/pages.c --- a/c8/stm/pages.c +++ b/c8/stm/pages.c @@ -39,11 +39,12 @@ char *segment_base = get_segment_base(i); mprotect(segment_base + pagenum * 4096UL, count * 4096UL, PROT_NONE); + + long amount = count; + while (amount-->0) { + set_page_status_in(i, pagenum + amount, PAGE_NO_ACCESS); + } } - - long amount = count; - while (amount-->0) - set_page_status_in(i, pagenum + amount, PAGE_NO_ACCESS); } } diff --git a/c8/stm/pages.h b/c8/stm/pages.h --- a/c8/stm/pages.h +++ b/c8/stm/pages.h @@ -39,8 +39,8 @@ enum { PAGE_SHARED = 0, - PAGE_PRIVATE, - PAGE_NO_ACCESS, + PAGE_PRIVATE = 1, + PAGE_NO_ACCESS = 2, }; static struct page_shared_s pages_status[NB_SHARED_PAGES]; From noreply at buildbot.pypy.org Fri Sep 19 13:09:33 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Fri, 19 Sep 2014 13:09:33 +0200 (CEST) Subject: [pypy-commit] stmgc c8-new-page-handling: fix future bug Message-ID: <20140919110933.0E84B1C34C2@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: c8-new-page-handling Changeset: r1397:6f4f5fff1904 Date: 2014-09-19 13:09 +0200 http://bitbucket.org/pypy/stmgc/changeset/6f4f5fff1904/ Log: fix future bug diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -365,9 +365,13 @@ uintptr_t page; for (page = first_page; page <= end_page; page++) { /* check if our page is private or we are the only shared-page holder */ + if (get_page_status_in(my_segnum, page) == PAGE_NO_ACCESS) { + /* happens if there is a concurrent WB between us making the backup + and acquiring the locks */ + volatile char *dummy = REAL_ADDRESS(STM_SEGMENT->segment_base, page * 4096UL); + *dummy = *dummy; /* force segfault */ + } assert(get_page_status_in(my_segnum, page) != PAGE_NO_ACCESS); - /* XXX: actually, it can be NO_ACCESS if somebody changed that after we - copied from it */ if (get_page_status_in(my_segnum, page) == PAGE_PRIVATE) continue; From noreply at buildbot.pypy.org Fri Sep 19 13:42:33 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Fri, 19 Sep 2014 13:42:33 +0200 (CEST) Subject: [pypy-commit] stmgc c8-new-page-handling: bugs everywhere... Message-ID: <20140919114233.F13A81C34C2@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: c8-new-page-handling Changeset: r1398:22825b514b8b Date: 2014-09-19 13:41 +0200 http://bitbucket.org/pypy/stmgc/changeset/22825b514b8b/ Log: bugs everywhere... diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -359,6 +359,7 @@ struct object_s *bk_obj = malloc(obj_size); memcpy(bk_obj, realobj, obj_size); + retry: /* privatize pages: */ acquire_all_privatization_locks(); @@ -368,14 +369,19 @@ if (get_page_status_in(my_segnum, page) == PAGE_NO_ACCESS) { /* happens if there is a concurrent WB between us making the backup and acquiring the locks */ + release_all_privatization_locks(); + volatile char *dummy = REAL_ADDRESS(STM_SEGMENT->segment_base, page * 4096UL); *dummy = *dummy; /* force segfault */ + + goto retry; } assert(get_page_status_in(my_segnum, page) != PAGE_NO_ACCESS); if (get_page_status_in(my_segnum, page) == PAGE_PRIVATE) continue; + assert(get_page_status_in(my_segnum, page) == PAGE_SHARED); /* make sure all the others are NO_ACCESS choosing to make us PRIVATE is harder because then nobody must ever update the shared page in stm_validate() except if it is the sole diff --git a/c8/stm/pages.h b/c8/stm/pages.h --- a/c8/stm/pages.h +++ b/c8/stm/pages.h @@ -22,7 +22,6 @@ #define PAGE_FLAG_END NB_PAGES /* == NB_SHARED_PAGES */ - struct page_shared_s { #if NB_SEGMENTS <= 4 uint8_t by_segment; @@ -68,21 +67,23 @@ static inline uint8_t get_page_status_in(long segnum, uintptr_t pagenum) { int seg_shift = segnum * 2; - uint64_t bitmask = 3UL << seg_shift; + OPT_ASSERT(seg_shift < 8 * sizeof(struct page_shared_s)); volatile struct page_shared_s *ps = (volatile struct page_shared_s *) &pages_status[get_file_page_of(pagenum)]; - return ((ps->by_segment & bitmask) >> seg_shift) & 3; + return (ps->by_segment >> seg_shift) & 3; } static inline void set_page_status_in(long segnum, uintptr_t pagenum, uint8_t status) { - OPT_ASSERT((status & 3) == status); + OPT_ASSERT(status < 3); int seg_shift = segnum * 2; + OPT_ASSERT(seg_shift < 8 * sizeof(struct page_shared_s)); volatile struct page_shared_s *ps = (volatile struct page_shared_s *) &pages_status[get_file_page_of(pagenum)]; assert(status != get_page_status_in(segnum, pagenum)); - ps->by_segment |= status << seg_shift; + ps->by_segment &= ~(3UL << seg_shift); /* clear */ + ps->by_segment |= status << seg_shift; /* set */ } diff --git a/c8/stm/setup.c b/c8/stm/setup.c --- a/c8/stm/setup.c +++ b/c8/stm/setup.c @@ -260,6 +260,15 @@ _init_shadow_stack(tl); set_gs_register(get_segment_base(num)); s_mutex_unlock(); + + if (num == 0) { + dprintf(("STM_GC_NURSERY: %d\n", STM_GC_NURSERY)); + dprintf(("NB_PAGES: %d\n", NB_PAGES)); + dprintf(("NB_SEGMENTS: %d\n", NB_SEGMENTS)); + dprintf(("FIRST_OBJECT_PAGE=FIRST_NURSERY_PAGE: %lu\n", FIRST_OBJECT_PAGE)); + dprintf(("END_NURSERY_PAGE: %lu\n", END_NURSERY_PAGE)); + dprintf(("NB_SHARED_PAGES: %lu\n", NB_SHARED_PAGES)); + } } void stm_unregister_thread_local(stm_thread_local_t *tl) From noreply at buildbot.pypy.org Fri Sep 19 13:43:23 2014 From: noreply at buildbot.pypy.org (mattip) Date: Fri, 19 Sep 2014 13:43:23 +0200 (CEST) Subject: [pypy-commit] pypy default: avoid segfault dialog box in packaging, copy pdb after translating (windows) Message-ID: <20140919114323.0AF501C34C2@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r73611:8e1fb8ca5b05 Date: 2014-09-19 14:02 +0300 http://bitbucket.org/pypy/pypy/changeset/8e1fb8ca5b05/ Log: avoid segfault dialog box in packaging, copy pdb after translating (windows) diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -370,5 +370,21 @@ if __name__ == '__main__': import sys + if sys.platform == 'win32': + # Try to avoid opeing a dialog box if one of the + # subprocesses causes a system error + import ctypes + winapi = ctypes.windll.kernel32 + SetErrorMode = winapi.SetErrorMode + SetErrorMode.argtypes=[ctypes.c_int] + + SEM_FAILCRITICALERRORS = 1 + SEM_NOGPFAULTERRORBOX = 2 + SEM_NOOPENFILEERRORBOX = 0x8000 + flags = SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX | SEM_NOOPENFILEERRORBOX + #Since there is no GetErrorMode, do a double Set + old_mode = SetErrorMode(flags) + SetErrorMode(old_mode | flags) + retval, _ = package(*sys.argv[1:]) sys.exit(retval) diff --git a/rpython/translator/driver.py b/rpython/translator/driver.py --- a/rpython/translator/driver.py +++ b/rpython/translator/driver.py @@ -481,6 +481,14 @@ libname = str(newsoname.dirpath().join('python27.lib')) shutil.copyfile(str(soname.new(ext='lib')), libname) self.log.info("copied: %s" % (libname,)) + # XXX TODO : replace the nonsense above with + # ext_to_copy = ['lib', 'pdb'] + ext_to_copy = ['pdb',] + for ext in ext_to_copy: + name = soname.new(ext=ext) + newname = newexename.new(basename=soname.basename) + shutil.copyfile(str(name), str(newname.new(ext=ext))) + self.log.info("copied: %s" % (newname,)) self.c_entryp = newexename self.log.info('usession directory: %s' % (udir,)) self.log.info("created: %s" % (self.c_entryp,)) diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -410,7 +410,7 @@ 'int main(int argc, char* argv[]) ' '{ return $(PYPY_MAIN_FUNCTION)(argc, argv); } > $@') m.rule('$(DEFAULT_TARGET)', ['$(TARGET)', 'main.obj'], - ['$(CC_LINK) /nologo main.obj $(SHARED_IMPORT_LIB) /out:$@ /MANIFEST /MANIFESTFILE:$*.manifest', + ['$(CC_LINK) /nologo /debug main.obj $(SHARED_IMPORT_LIB) /out:$@ /MANIFEST /MANIFESTFILE:$*.manifest', 'mt.exe -nologo -manifest $*.manifest -outputresource:$@;1', ]) m.rule('debugmode_$(DEFAULT_TARGET)', ['debugmode_$(TARGET)', 'main.obj'], diff --git a/rpython/translator/test/test_driver.py b/rpython/translator/test/test_driver.py --- a/rpython/translator/test/test_driver.py +++ b/rpython/translator/test/test_driver.py @@ -55,12 +55,15 @@ src_name = udir.join('src/dydy2.exe') dll_name = udir.join('src/pypy.dll') lib_name = udir.join('src/pypy.lib') + pdb_name = udir.join('src/pypy.pdb') src_name.ensure() src_name.write('exe') dll_name.ensure() dll_name.write('dll') lib_name.ensure() lib_name.write('lib') + pdb_name.ensure() + pdb_name.write('pdb') dst_name.ensure() class CBuilder(object): @@ -75,6 +78,8 @@ assert dst_name.new(purebasename='python27',ext='lib').read() == 'lib' def test_shutil_copy(): + if os.name == 'nt': + py.test.skip('Windows cannot copy or rename to an in-use file') a = udir.join('file_a') b = udir.join('file_a') a.write('hello') From noreply at buildbot.pypy.org Fri Sep 19 13:43:24 2014 From: noreply at buildbot.pypy.org (mattip) Date: Fri, 19 Sep 2014 13:43:24 +0200 (CEST) Subject: [pypy-commit] pypy default: try harder to find license info on windows Message-ID: <20140919114324.337B31C34C2@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r73612:0cfa9e069a4c Date: 2014-09-19 14:29 +0300 http://bitbucket.org/pypy/pypy/changeset/0cfa9e069a4c/ Log: try harder to find license info on windows diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -297,7 +297,13 @@ argparse = imp.load_source('argparse', 'lib-python/2.7/argparse.py') if sys.platform == 'win32': pypy_exe = 'pypy.exe' - license_base = os.path.join(basedir, r'..\..\..\local') # as on buildbot YMMV + for p in [os.path.join(basedir, r'..\..\..\local'), #buildbot + os.path.join(basedir, r'..\local')]: # pypy/doc/windows.rst + if os.path.exists(p): + license_base = p + break + else: + license_base = 'unkown' else: pypy_exe = 'pypy' license_base = '/usr/share/doc' diff --git a/pypy/tool/release/test/test_package.py b/pypy/tool/release/test/test_package.py --- a/pypy/tool/release/test/test_package.py +++ b/pypy/tool/release/test/test_package.py @@ -115,15 +115,21 @@ check(pypy, 0755) def test_generate_license(): - from os.path import dirname, abspath, join + from os.path import dirname, abspath, join, exists class Options(object): pass options = Options() basedir = dirname(dirname(dirname(dirname(dirname(abspath(__file__)))))) options.no_tk = False if sys.platform == 'win32': - # as on buildbot YMMV - options.license_base = join(basedir, r'..\..\..\local') + for p in [join(basedir, r'..\..\..\local'), #buildbot + join(basedir, r'..\local')]: # pypy/doc/windows.rst + if exists(p): + license_base = p + break + else: + license_base = 'unkown' + options.license_base = license_base else: options.license_base = '/usr/share/doc' license = package.generate_license(py.path.local(basedir), options) From noreply at buildbot.pypy.org Fri Sep 19 13:43:26 2014 From: noreply at buildbot.pypy.org (mattip) Date: Fri, 19 Sep 2014 13:43:26 +0200 (CEST) Subject: [pypy-commit] pypy py3k: avoid segfault dialog box in packaging, copy pdb after translating (windows) Message-ID: <20140919114326.C366D1C34C2@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: py3k Changeset: r73613:047dc52c284f Date: 2014-09-19 14:02 +0300 http://bitbucket.org/pypy/pypy/changeset/047dc52c284f/ Log: avoid segfault dialog box in packaging, copy pdb after translating (windows) diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -380,5 +380,21 @@ if __name__ == '__main__': import sys + if sys.platform == 'win32': + # Try to avoid opeing a dialog box if one of the + # subprocesses causes a system error + import ctypes + winapi = ctypes.windll.kernel32 + SetErrorMode = winapi.SetErrorMode + SetErrorMode.argtypes=[ctypes.c_int] + + SEM_FAILCRITICALERRORS = 1 + SEM_NOGPFAULTERRORBOX = 2 + SEM_NOOPENFILEERRORBOX = 0x8000 + flags = SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX | SEM_NOOPENFILEERRORBOX + #Since there is no GetErrorMode, do a double Set + old_mode = SetErrorMode(flags) + SetErrorMode(old_mode | flags) + retval, _ = package(*sys.argv[1:]) sys.exit(retval) diff --git a/rpython/translator/driver.py b/rpython/translator/driver.py --- a/rpython/translator/driver.py +++ b/rpython/translator/driver.py @@ -481,6 +481,14 @@ newlibname = newexename.new(basename=soname.basename) shutil.copyfile(str(libname), str(newlibname.new(ext='lib'))) self.log.info("copied: %s" % (newlibname,)) + # XXX TODO : replace the nonsense above with + # ext_to_copy = ['lib', 'pdb'] + ext_to_copy = ['pdb',] + for ext in ext_to_copy: + name = soname.new(ext=ext) + newname = newexename.new(basename=soname.basename) + shutil.copyfile(str(name), str(newname.new(ext=ext))) + self.log.info("copied: %s" % (newname,)) self.c_entryp = newexename self.log.info('usession directory: %s' % (udir,)) self.log.info("created: %s" % (self.c_entryp,)) diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -410,7 +410,7 @@ 'int main(int argc, char* argv[]) ' '{ return $(PYPY_MAIN_FUNCTION)(argc, argv); } > $@') m.rule('$(DEFAULT_TARGET)', ['$(TARGET)', 'main.obj'], - ['$(CC_LINK) /nologo main.obj $(SHARED_IMPORT_LIB) /out:$@ /MANIFEST /MANIFESTFILE:$*.manifest', + ['$(CC_LINK) /nologo /debug main.obj $(SHARED_IMPORT_LIB) /out:$@ /MANIFEST /MANIFESTFILE:$*.manifest', 'mt.exe -nologo -manifest $*.manifest -outputresource:$@;1', ]) m.rule('debugmode_$(DEFAULT_TARGET)', ['debugmode_$(TARGET)', 'main.obj'], diff --git a/rpython/translator/test/test_driver.py b/rpython/translator/test/test_driver.py --- a/rpython/translator/test/test_driver.py +++ b/rpython/translator/test/test_driver.py @@ -55,12 +55,15 @@ src_name = udir.join('src/dydy2.exe') dll_name = udir.join('src/pypy.dll') lib_name = udir.join('src/pypy.lib') + pdb_name = udir.join('src/pypy.pdb') src_name.ensure() src_name.write('exe') dll_name.ensure() dll_name.write('dll') lib_name.ensure() lib_name.write('lib') + pdb_name.ensure() + pdb_name.write('pdb') dst_name.ensure() class CBuilder(object): @@ -75,6 +78,8 @@ assert dst_name.new(ext='lib').read() == 'lib' def test_shutil_copy(): + if os.name == 'nt': + py.test.skip('Windows cannot copy or rename to an in-use file') a = udir.join('file_a') b = udir.join('file_a') a.write('hello') From noreply at buildbot.pypy.org Fri Sep 19 13:43:28 2014 From: noreply at buildbot.pypy.org (mattip) Date: Fri, 19 Sep 2014 13:43:28 +0200 (CEST) Subject: [pypy-commit] pypy py3k: TODO is done on this branch Message-ID: <20140919114328.065951C34C2@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: py3k Changeset: r73614:84934536fb78 Date: 2014-09-19 14:43 +0300 http://bitbucket.org/pypy/pypy/changeset/84934536fb78/ Log: TODO is done on this branch diff --git a/rpython/translator/driver.py b/rpython/translator/driver.py --- a/rpython/translator/driver.py +++ b/rpython/translator/driver.py @@ -476,14 +476,7 @@ shutil_copy(str(soname), str(newsoname)) self.log.info("copied: %s" % (newsoname,)) if sys.platform == 'win32': - # copy the import library as well - libname = soname.new(ext='lib') - newlibname = newexename.new(basename=soname.basename) - shutil.copyfile(str(libname), str(newlibname.new(ext='lib'))) - self.log.info("copied: %s" % (newlibname,)) - # XXX TODO : replace the nonsense above with - # ext_to_copy = ['lib', 'pdb'] - ext_to_copy = ['pdb',] + ext_to_copy = ['lib', 'pdb'] for ext in ext_to_copy: name = soname.new(ext=ext) newname = newexename.new(basename=soname.basename) From noreply at buildbot.pypy.org Fri Sep 19 13:45:41 2014 From: noreply at buildbot.pypy.org (mattip) Date: Fri, 19 Sep 2014 13:45:41 +0200 (CEST) Subject: [pypy-commit] pypy py3k: try harder to find license info on windows Message-ID: <20140919114541.98D4C1D2CE9@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: py3k Changeset: r73615:6252ee8e03f4 Date: 2014-09-19 14:29 +0300 http://bitbucket.org/pypy/pypy/changeset/6252ee8e03f4/ Log: try harder to find license info on windows diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -307,7 +307,13 @@ argparse = imp.load_source('argparse', 'lib-python/2.7/argparse.py') if sys.platform == 'win32': pypy_exe = 'pypy3.exe' - license_base = os.path.join(basedir, r'..\..\..\local') # as on buildbot YMMV + for p in [os.path.join(basedir, r'..\..\..\local'), #buildbot + os.path.join(basedir, r'..\local')]: # pypy/doc/windows.rst + if os.path.exists(p): + license_base = p + break + else: + license_base = 'unkown' else: pypy_exe = 'pypy3' license_base = '/usr/share/doc' diff --git a/pypy/tool/release/test/test_package.py b/pypy/tool/release/test/test_package.py --- a/pypy/tool/release/test/test_package.py +++ b/pypy/tool/release/test/test_package.py @@ -115,15 +115,21 @@ check(pypy, 0755) def test_generate_license(): - from os.path import dirname, abspath, join + from os.path import dirname, abspath, join, exists class Options(object): pass options = Options() basedir = dirname(dirname(dirname(dirname(dirname(abspath(__file__)))))) options.no_tk = False if sys.platform == 'win32': - # as on buildbot YMMV - options.license_base = join(basedir, r'..\..\..\local') + for p in [join(basedir, r'..\..\..\local'), #buildbot + join(basedir, r'..\local')]: # pypy/doc/windows.rst + if exists(p): + license_base = p + break + else: + license_base = 'unkown' + options.license_base = license_base else: options.license_base = '/usr/share/doc' license = package.generate_license(py.path.local(basedir), options) From noreply at buildbot.pypy.org Fri Sep 19 13:48:36 2014 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 19 Sep 2014 13:48:36 +0200 (CEST) Subject: [pypy-commit] extradoc extradoc: work on slides a little more Message-ID: <20140919114836.34E1D1D2CE9@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: extradoc Changeset: r5411:91173ca8d8c0 Date: 2014-09-19 13:48 +0200 http://bitbucket.org/pypy/extradoc/changeset/91173ca8d8c0/ Log: work on slides a little more diff --git a/talk/brno-php-2014/talk.rst b/talk/brno-php-2014/talk.rst --- a/talk/brno-php-2014/talk.rst +++ b/talk/brno-php-2014/talk.rst @@ -12,6 +12,17 @@ * founder of baroquesoftware.com +This talk +--------- + +* hippyvm project + +* performance and measurments + +* history + +* questions after each part + HippyVM ------- @@ -32,11 +43,11 @@ * misses a lot of builtin functions -* can kind of "run" wordpress, mediawiki, getting there +* can kind of "run" wordpress, mediawiki, squirrelmail, getting there * cgi, fastcgi (not open source) -* quick XXX do you mean 'fast'? but next slide already says 'fast' +* fast HippyVM status - short ---------------------- @@ -45,6 +56,16 @@ * not quite ready +HippyVM - Python bridge +----------------------- + +* demo + +HippyVM - questions +------------------- + +* ? + Let's talk about performance ---------------------------- @@ -56,6 +77,17 @@ * "I'll use a recursive fibonacci function" +Performance - breakdown +----------------------- + +* user code + +* runtime + +* IO, DB, ... - libraries and ecosystem + +* non-trivial interactions between the pieces + Performance ----------- @@ -113,6 +145,13 @@ * consider bottlenecks differ depending on implementation +Good benchmarking - example +--------------------------- + +* speed.pypy.org + +* I strongly encourage people to come up with the same for PHP + Performance - personal opinions ------------------------------- @@ -149,12 +188,58 @@ * fully Open Source, EU funded project -PyPy history ------------- +Let's go back 10 years +---------------------- -xxxx +* Python is (like PHP) a very complex language -HippyVM history ---------------- +* writing an interpreter is hard -xxxx +* writing a just-in-time compiler is even harder + +* we decided to write a framework instead + +A typical example +----------------- + +* interpreter, written in C++ + +* just in time compiler that repeats the semantics, written in C++, + emits assembler + +* another layer, e.g. a method JIT + +|pause| + +* becomes harder and harder to keep up with semantics + +PyPy approach +------------- + +* write an interpreter in machine-readable language + +* just in time compiler gets generated from the description + +* a lot of work once, but prevents a lot of problems + +How well it works? +------------------ + +* HippyVM - 4 people, 1.5 years + +* HHVM - 5 years, team of up to 20 + +|pause| + +* a lot of effort is reusable + +* Truffle is another example of a similar approach + +Questions? +---------- + +* hippyvm.com + +* baroquesoftware.com + +* talk to me! From noreply at buildbot.pypy.org Fri Sep 19 14:30:38 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Fri, 19 Sep 2014 14:30:38 +0200 (CEST) Subject: [pypy-commit] stmgc c8-new-page-handling: don't touch possibly PROT_NONE memory in segfault handler Message-ID: <20140919123038.3CDEB1D2D5A@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: c8-new-page-handling Changeset: r1399:0985c04574ca Date: 2014-09-19 14:13 +0200 http://bitbucket.org/pypy/stmgc/changeset/0985c04574ca/ Log: don't touch possibly PROT_NONE memory in segfault handler diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -157,6 +157,8 @@ { size_t obj_size; + assert(get_priv_segment(from_seg)->privatization_lock); + /* look the obj up in the other segment's modified_old_objects to get its backup copy: */ acquire_modified_objs_lock(from_seg); @@ -170,7 +172,6 @@ memcpy_to_accessible_pages(STM_SEGMENT->segment_num, obj, (char*)item->val, obj_size); - assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); release_modified_objs_lock(from_seg); return; @@ -183,7 +184,6 @@ REAL_ADDRESS(get_segment_base(from_seg), obj), obj_size); - obj->stm_flags |= GCFLAG_WRITE_BARRIER; /* may already be gone */ release_modified_objs_lock(from_seg); } @@ -196,14 +196,15 @@ the committed objs. */ if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) { assert((uintptr_t)STM_PSEGMENT->last_commit_log_entry->next == -1); + if (locks_acquired) + release_all_privatization_locks(); return; } - if (locks_acquired) { + if (locks_acquired) assert(all_privatization_locks_acquired()); - } else { + else acquire_all_privatization_locks(); - } volatile struct stm_commit_log_entry_s *cl, *prev_cl; cl = prev_cl = (volatile struct stm_commit_log_entry_s *) @@ -214,12 +215,14 @@ while ((cl = cl->next)) { if ((uintptr_t)cl == -1) { /* there is an inevitable transaction running */ + release_all_privatization_locks(); #if STM_TESTS free(free_if_abort); stm_abort_transaction(); #endif cl = prev_cl; _stm_collectable_safe_point(); + acquire_all_privatization_locks(); continue; } prev_cl = cl; @@ -407,9 +410,6 @@ /* also add it to the GC list for minor collections */ LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, obj); - /* done fiddling with protection and privatization */ - release_all_privatization_locks(); - /* phew, now add the obj to the write-set and register the backup copy. */ /* XXX: possibly slow check; try overflow objs again? */ @@ -420,6 +420,8 @@ release_modified_objs_lock(my_segnum); } + /* done fiddling with protection and privatization */ + release_all_privatization_locks(); } static void reset_transaction_read_version(void) From noreply at buildbot.pypy.org Fri Sep 19 14:30:39 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Fri, 19 Sep 2014 14:30:39 +0200 (CEST) Subject: [pypy-commit] stmgc c8-new-page-handling: fix invalid argument to mmap Message-ID: <20140919123039.4DF921D2D5A@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: c8-new-page-handling Changeset: r1400:d41bdd1f1bf4 Date: 2014-09-19 14:17 +0200 http://bitbucket.org/pypy/stmgc/changeset/d41bdd1f1bf4/ Log: fix invalid argument to mmap diff --git a/c8/stm/pages.c b/c8/stm/pages.c --- a/c8/stm/pages.c +++ b/c8/stm/pages.c @@ -64,7 +64,7 @@ char *result = mmap( addr, 4096UL, PROT_READ | PROT_WRITE, MAP_FIXED | MAP_PRIVATE | MAP_NORESERVE, - stm_object_pages_fd, get_file_page_of(pagenum)); + stm_object_pages_fd, get_file_page_of(pagenum) * 4096UL); if (result == MAP_FAILED) stm_fatalerror("page_privatize_in failed (mmap): %m"); From noreply at buildbot.pypy.org Fri Sep 19 14:30:40 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Fri, 19 Sep 2014 14:30:40 +0200 (CEST) Subject: [pypy-commit] stmgc c8-new-page-handling: fix overlap condition when trying to find objs that are partially in a page Message-ID: <20140919123040.42BBD1D2D5A@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: c8-new-page-handling Changeset: r1401:740a6aeff2eb Date: 2014-09-19 14:30 +0200 http://bitbucket.org/pypy/stmgc/changeset/740a6aeff2eb/ Log: fix overlap condition when trying to find objs that are partially in a page diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -12,13 +12,11 @@ struct tree_s *tree = get_priv_segment(from_segnum)->modified_old_objects; wlog_t *item; TREE_LOOP_FORWARD(tree, item); { - if (item->addr >= pagenum * 4096UL && item->addr < (pagenum + 1) * 4096UL) { - object_t *obj = (object_t*)item->addr; - struct object_s* bk_obj = (struct object_s *)item->val; - size_t obj_size; + object_t *obj = (object_t*)item->addr; + struct object_s* bk_obj = (struct object_s *)item->val; + size_t obj_size = stmcb_size_rounded_up(bk_obj); - obj_size = stmcb_size_rounded_up(bk_obj); - + if (item->addr < (pagenum + 1) * 4096UL && item->addr + obj_size > pagenum * 4096UL) { memcpy_to_accessible_pages(STM_SEGMENT->segment_num, obj, (char*)bk_obj, obj_size); @@ -91,6 +89,7 @@ /* make our page private */ page_privatize_in(STM_SEGMENT->segment_num, pagenum); + assert(get_page_status_in(my_segnum, pagenum) == PAGE_PRIVATE); /* if there were modifications in the page, revert them: */ copy_bk_objs_from(shared_page_holder, pagenum); @@ -362,6 +361,7 @@ struct object_s *bk_obj = malloc(obj_size); memcpy(bk_obj, realobj, obj_size); + dprintf(("write_slowpath(%p): sz=%lu, bk=%p\n", obj, obj_size, bk_obj)); retry: /* privatize pages: */ acquire_all_privatization_locks(); @@ -398,6 +398,7 @@ /* xxx: unmap? */ mprotect((char*)(get_virt_page_of(i, page) * 4096UL), 4096UL, PROT_NONE); set_page_status_in(i, page, PAGE_NO_ACCESS); + dprintf(("NO_ACCESS in seg %d page %lu\n", i, page)); } } } @@ -465,7 +466,7 @@ STM_PSEGMENT->shadowstack_at_start_of_transaction = tl->shadowstack; enter_safe_point_if_requested(); - dprintf(("start_transaction\n")); + dprintf(("> start_transaction\n")); s_mutex_unlock(); @@ -538,7 +539,7 @@ assert(STM_PSEGMENT->safe_point == SP_RUNNING); assert(STM_PSEGMENT->running_pthread == pthread_self()); - dprintf(("stm_commit_transaction()\n")); + dprintf(("> stm_commit_transaction()\n")); minor_collection(1); _validate_and_add_to_commit_log(); diff --git a/c8/stm/pages.c b/c8/stm/pages.c --- a/c8/stm/pages.c +++ b/c8/stm/pages.c @@ -84,6 +84,8 @@ char *dst_end = realobj + len; uintptr_t loc_addr = (uintptr_t)dst_obj; + dprintf(("memcpy_to_accessible_pages(%d, %p, %p, %lu)\n", dst_segnum, dst_obj, src, len)); + while (realobj != dst_end) { if (get_page_status_in(dst_segnum, loc_addr / 4096UL) != PAGE_NO_ACCESS) *realobj = *src; From noreply at buildbot.pypy.org Fri Sep 19 14:34:54 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Fri, 19 Sep 2014 14:34:54 +0200 (CEST) Subject: [pypy-commit] stmgc c8-new-page-handling: pass test_basic again, yay Message-ID: <20140919123454.59B721D2D5A@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: c8-new-page-handling Changeset: r1402:172d904d4d30 Date: 2014-09-19 14:35 +0200 http://bitbucket.org/pypy/stmgc/changeset/172d904d4d30/ Log: pass test_basic again, yay diff --git a/c8/test/support.py b/c8/test/support.py --- a/c8/test/support.py +++ b/c8/test/support.py @@ -551,11 +551,20 @@ self.push_root(ffi.cast("object_t *", 8)) def check_char_everywhere(self, obj, expected_content, offset=HDR): - for i in range(len(self.tls)): + for i in range(self.NB_THREADS): + if self.current_thread != i: + self.switch(i) + tl = self.tls[i] + if not lib._stm_in_transaction(tl): + self.start_transaction() + + # check: addr = lib._stm_get_segment_base(i) content = addr[int(ffi.cast("uintptr_t", obj)) + offset] assert content == expected_content + self.abort_transaction() + def get_thread_local_obj(self): tl = self.tls[self.current_thread] return tl.thread_local_obj From noreply at buildbot.pypy.org Fri Sep 19 14:54:28 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Fri, 19 Sep 2014 14:54:28 +0200 (CEST) Subject: [pypy-commit] stmgc c8-new-page-handling: add some comments Message-ID: <20140919125428.D2E841C0320@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: c8-new-page-handling Changeset: r1403:07565910b902 Date: 2014-09-19 14:54 +0200 http://bitbucket.org/pypy/stmgc/changeset/07565910b902/ Log: add some comments diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -4,10 +4,48 @@ /* ############# signal handler ############# */ -static void _update_obj_from(int from_seg, object_t *obj); +static void _update_obj_from(int from_seg, object_t *obj) +{ + /* updates 'obj' in our accessible pages from another segment's + page or bk copy. (never touch PROT_NONE memory) */ + /* XXXXXXX: are we sure everything is readable in from_seg??? */ + size_t obj_size; + + assert(get_priv_segment(from_seg)->privatization_lock); + + /* look the obj up in the other segment's modified_old_objects to + get its backup copy: */ + acquire_modified_objs_lock(from_seg); + + wlog_t *item; + struct tree_s *tree = get_priv_segment(from_seg)->modified_old_objects; + TREE_FIND(tree, (uintptr_t)obj, item, goto not_found); + + obj_size = stmcb_size_rounded_up((struct object_s*)item->val); + + memcpy_to_accessible_pages(STM_SEGMENT->segment_num, obj, + (char*)item->val, obj_size); + + release_modified_objs_lock(from_seg); + return; + + not_found: + /* copy from page directly (obj is unmodified) */ + obj_size = stmcb_size_rounded_up( + (struct object_s*)REAL_ADDRESS(get_segment_base(from_seg), obj)); + + memcpy_to_accessible_pages(STM_SEGMENT->segment_num, obj, + REAL_ADDRESS(get_segment_base(from_seg), obj), + obj_size); + + release_modified_objs_lock(from_seg); +} + static void copy_bk_objs_from(int from_segnum, uintptr_t pagenum) { + /* looks at all bk copies of objects overlapping page 'pagenum' and + copies to current segment (never touch PROT_NONE memory) */ acquire_modified_objs_lock(from_segnum); struct tree_s *tree = get_priv_segment(from_segnum)->modified_old_objects; wlog_t *item; @@ -17,6 +55,8 @@ size_t obj_size = stmcb_size_rounded_up(bk_obj); if (item->addr < (pagenum + 1) * 4096UL && item->addr + obj_size > pagenum * 4096UL) { + /* XXX: should actually only write to pagenum, but we validate + afterwards anyway and abort in case we had modifications there */ memcpy_to_accessible_pages(STM_SEGMENT->segment_num, obj, (char*)bk_obj, obj_size); @@ -31,6 +71,8 @@ uintptr_t pagenum, struct stm_commit_log_entry_s *from, struct stm_commit_log_entry_s *to) { + /* walk the commit log and update the page 'pagenum' until we reach + the same revision as our segment, or we reach the HEAD. */ assert(all_privatization_locks_acquired()); volatile struct stm_commit_log_entry_s *cl; @@ -61,6 +103,9 @@ static void bring_page_up_to_date(uintptr_t pagenum) { + /* assumes page 'pagenum' is ACCESS_NONE, privatizes it, + and validates to newest revision */ + /* XXX: bad, but no deadlocks: */ acquire_all_privatization_locks(); @@ -152,41 +197,6 @@ } -static void _update_obj_from(int from_seg, object_t *obj) -{ - size_t obj_size; - - assert(get_priv_segment(from_seg)->privatization_lock); - - /* look the obj up in the other segment's modified_old_objects to - get its backup copy: */ - acquire_modified_objs_lock(from_seg); - - wlog_t *item; - struct tree_s *tree = get_priv_segment(from_seg)->modified_old_objects; - TREE_FIND(tree, (uintptr_t)obj, item, goto not_found); - - obj_size = stmcb_size_rounded_up((struct object_s*)item->val); - - memcpy_to_accessible_pages(STM_SEGMENT->segment_num, obj, - (char*)item->val, obj_size); - - release_modified_objs_lock(from_seg); - return; - - not_found: - /* copy from page directly (obj is unmodified) */ - obj_size = stmcb_size_rounded_up( - (struct object_s*)REAL_ADDRESS(get_segment_base(from_seg), obj)); - - memcpy_to_accessible_pages(STM_SEGMENT->segment_num, obj, - REAL_ADDRESS(get_segment_base(from_seg), obj), - obj_size); - - release_modified_objs_lock(from_seg); -} - - static void _stm_validate(void *free_if_abort, bool locks_acquired) { /* go from last known entry in commit log to the From noreply at buildbot.pypy.org Fri Sep 19 15:15:59 2014 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 19 Sep 2014 15:15:59 +0200 (CEST) Subject: [pypy-commit] creflect default: Adapt the driver's code Message-ID: <20140919131559.59F101C34C2@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r49:70c5769aca75 Date: 2014-09-19 15:16 +0200 http://bitbucket.org/cffi/creflect/changeset/70c5769aca75/ Log: Adapt the driver's code diff --git a/creflect/cmdline.py b/creflect/cmdline.py --- a/creflect/cmdline.py +++ b/creflect/cmdline.py @@ -1,21 +1,21 @@ """Usage: - creflect [options] input.rfl [output.c] + creflect [options] input.rfl.c [output.c] -Read the 'input.rfl' file and expand the #creflect...#end sections in it +Read the 'input.rfl.c' file and expand the CREFLECT sections in it into regular C code. Write the result to 'output.c'. The default output file name is built from the input file name, by -replacing the extension with '.c'. The input and/or output file names +removing the '.rfl' part. The input and/or output file names can be specified as '-' to read from stdin and/or write to stdout. Options: - -n, --name=NAME name of the reflection function to create - (default: "_creflect") - -m, --main for debugging, include a main() function that + -m, --main for debugging, include a main() function that prints the recorded reflection information - -h, --help display this help and exit - --version output version information and exit + -n, --no-copy output only the translated creflect sections, + without copying the text around them + -h, --help display this help and exit + --version output version information and exit """ import sys, os, getopt @@ -28,24 +28,24 @@ def main(argv): try: - options, args = getopt.gnu_getopt(argv, "nmh", - ["name", "main", "help", "version"]) + options, args = getopt.gnu_getopt(argv, "mnh", + ["main", "no-copy", "help", "version"]) except getopt.GetoptError, e: return error(e) # from . import __version__ - reflection_func_name = "_creflect" include_main = False + include_text_outside_creflect = True for option, value in options: - if option == '-n' or option == '--name': - reflection_func_name = value + if option == '-n' or option == '--no-copy': + include_text_outside_creflect = False elif option == '-m' or option == '--main': include_main = True elif option == '-h' or option == '--help': print __doc__ return 0 elif option == '--version': - print 'CReflect %d.%d' % __version__ + print 'CReflect %s' % __version__ return 0 # if len(args) == 0: @@ -58,14 +58,17 @@ else: if inputfile == '-': return error("destination file must be specified if input is '-'") - outputfile, _ = os.path.splitext(inputfile) - outputfile += '.c' + i = inputfile.lower().rfind('.rfl') + if i < 0: + return error("cannot find '.rfl' in the input file name: specify " + "the output file name explicitly") + outputfile = inputfile[:i] + inputfile[i+4:] if inputfile != '-' and inputfile == outputfile: return error("not overwriting the file %r" % (inputfile,)) # from . import driver - print >> sys.stderr, 'CReflect %d.%d converting %r to %r...' % ( - __version__[0], __version__[1], inputfile, outputfile) + print >> sys.stderr, 'CReflect %s converting %r to %r...' % ( + __version__, inputfile, outputfile) if inputfile == '-': inputf = sys.stdin else: @@ -76,9 +79,10 @@ outputf = open(outputfile, 'w') # try: - driver.driver(inputf, outputf, reflection_func_name) + blocks = driver.copy_file_and_expand(inputf, outputf, + include_text_outside_creflect = include_text_outside_creflect) if include_main: - driver.output_main(outputf) + outputf.write(driver.get_debug_code(blocks)) except driver.CDefError, e: print >> sys.stderr, 'FAILED:', e return 1 diff --git a/creflect/driver.py b/creflect/driver.py --- a/creflect/driver.py +++ b/creflect/driver.py @@ -1,3 +1,4 @@ +import re from codegen import transform_cdef from cparser import CSource, CDefError @@ -7,68 +8,91 @@ return transform_cdef(csource, reflection_func_name) -def get_preprocessor_directive(line): - if line.startswith('#'): - line = line[1:].split() - if line: - return line[0] +r_comment1 = re.compile(r'\s*/[*/]\s*CREFLECT\s*:\s*([A-Za-z_][A-Za-z0-9_]*)' + r'\s*[(]\s*[)]') +r_comment2 = re.compile(r'\s*/[*/]\s*CREFLECT\s*:\s*[Ee][Nn][Dd]\b') +END = object() + +def get_creflect_comment(line): + match = r_comment1.match(line) + if match: + return match.group(1) + if r_comment2.match(line): + return END return None -def driver(inputf, outputf, reflection_func_name): - func_num = 0 +def copy_file_and_expand(inputf, outputf, include_text_outside_creflect=True): lineno = 0 + blocks = [] while True: - # outside '#creflect' sections + # outside 'CREFLECT:' sections for line in inputf: lineno += 1 - if get_preprocessor_directive(line) == 'creflect': + funcname = get_creflect_comment(line) + if funcname is not None: break - outputf.write(line) + if include_text_outside_creflect: + outputf.write(line) else: break # end of file + if funcname is END: + raise CDefError("line %d: 'CREFLECT: end' without " + "'CREFLECT: funcname()' before" % lineno) # - # inside a '#creflect' section + # inside a 'CREFLECT:' section csource = [] first_lineno = lineno + 1 for line in inputf: lineno += 1 - if get_preprocessor_directive(line) == 'end': + end = get_creflect_comment(line) + if end is not None: break csource.append(line) else: - raise CDefError("missing '#end' directive after '#creflect'") - csource = CSource(''.join(csource), first_lineno) - func_num += 1 - outputf.write("/***** '#creflect' block start, " - "source line %d *****/\n\n" % (first_lineno - 1,)) - output = transform_cdef(csource, func_num) + raise CDefError("missing 'CREFLECT: end' in the input file") + if end is not END: + raise CDefError("line %d: 'CREFLECT: funcname()' without " + "'CREFLECT: end' before" % lineno) + # + if include_text_outside_creflect: + outputf.write("/***** CREFLECT block start, " + "source line %d *****/\n\n" % (first_lineno - 1,)) + output = expand_cdef(''.join(csource), funcname, first_lineno) outputf.write(output) - outputf.write("\n/***** '#creflect' block end, " - "source line %d *****/\n" % (lineno,)) - if func_num == 0: - raise CDefError("no '#creflect' found") - outputf.write(''' -/* main entry point for getting the reflection function */ -void %s(void r(char *, void**, int)) -{ - __CREFLECT_PREV(r); -} -''' % (reflection_func_name,)) + if include_text_outside_creflect: + outputf.write("\n/***** CREFLECT block end, " + "source line %d *****/\n" % (lineno,)) + blocks.append(funcname) + # + if len(blocks) == 0: + raise CDefError("no 'CREFLECT' block found in the input file") + return blocks -DEBUG_TEMPLATE = r''' -/* debugging only */ +_DEBUG_TEMPLATE1 = ''' +/***** CREFLECT debug code *****/ #include #include -#define _CREFLECT_MAIN %s int main(void) { - int size = _CREFLECT_MAIN((char *)0); - char *result = malloc(size); - int err = _CREFLECT_MAIN(result); - printf("%%s", result); - return (err != 0); +%s return 0; } ''' +_DEBUG_TEMPLATE2 = '''\ + { + int size = %(funcname)s((char *)0); + char *result = malloc(size); + int err = %(funcname)s(result); + printf("%%s", result); + free(result); + if (err != 0) + return 1; + } +''' def get_debug_code(reflection_func_name): - return DEBUG_TEMPLATE % (reflection_func_name,) + if not isinstance(reflection_func_name, list): + reflection_func_name = [reflection_func_name] + parts = [] + for funcname in reflection_func_name: + parts.append(_DEBUG_TEMPLATE2 % {'funcname': funcname}) + return _DEBUG_TEMPLATE1 % (''.join(parts),) diff --git a/test/test_driver.py b/test/test_driver.py --- a/test/test_driver.py +++ b/test/test_driver.py @@ -1,14 +1,11 @@ import os from StringIO import StringIO -from creflect.driver import expand_cdef, get_debug_code +from creflect.driver import expand_cdef, get_debug_code, copy_file_and_expand from .udir import udir -def compile_and_run(real_code, generated_code, funcname): +def compile_and_run(code, funcname): f = open(str(udir.join(funcname + '.c')), 'w') - print >> f, '/* real code */' - print >> f, real_code - print >> f, '/* generated code */' - print >> f, generated_code + print >> f, code print >> f, get_debug_code(funcname) f.close() # @@ -27,47 +24,21 @@ def test_expand_cdef(): real_code = "typedef short a_t[42];\n" gen = expand_cdef("typedef long a_t[20];", "test_expand_cdef") - data = compile_and_run(real_code, gen, "test_expand_cdef") + code = '/* real code */\n%s\n/* generated code */\n%s' % (real_code, gen) + data = compile_and_run(code, "test_expand_cdef") assert data == real_code -def XXXtest_driver(): +def test_copy_file_and_expand(): f = StringIO(""" -typedef unsigned int a_t[5]; +typedef unsigned short a_t[5]; -#creflect -typedef long a_t[20]; -#end - -typedef unsigned short b_t; -# creflect -typedef int b_t; -# end +/* CREFLECT: inspect1() */ +typedef int a_t[20]; +/* CREFLECT: end */ """) g = StringIO() - driver.driver(f, g, "foobar_func") + copy_file_and_expand(f, g) # - f = open(str(udir.join('foobar.c')), 'w') - f.write(g.getvalue()) - f.close() - # - err = os.system( - "cd '%s' && gcc -fPIC -Werror -Wall -shared foobar.c -o foobar.so" - % (str(udir),)) - assert err == 0 - # - import ctypes - cdll = ctypes.CDLL(str(udir.join('foobar.so'))) - RFUNC = ctypes.CFUNCTYPE(None, - ctypes.c_char_p, - ctypes.POINTER(ctypes.c_void_p), - ctypes.c_int) - seen = [] - def rfunc(cmd, args, nargs): - print cmd - seen.append((cmd, [args[i] for i in range(nargs)])) - callback = RFUNC(rfunc) - cdll.foobar_func(callback) - assert seen == [ - ('=a_t:int?[?]', [4, 5]), - ('=b_t:int?', [2])] + data = compile_and_run(g.getvalue(), "inspect1") + assert data == "typedef unsigned short a_t[5];\n" From noreply at buildbot.pypy.org Sat Sep 20 12:40:42 2014 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 20 Sep 2014 12:40:42 +0200 (CEST) Subject: [pypy-commit] extradoc extradoc: talk as it went Message-ID: <20140920104042.A52D61C02D6@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: extradoc Changeset: r5412:9257532b4a94 Date: 2014-09-20 12:40 +0200 http://bitbucket.org/pypy/extradoc/changeset/9257532b4a94/ Log: talk as it went diff --git a/talk/brno-php-2014/Makefile b/talk/brno-php-2014/Makefile new file mode 100644 --- /dev/null +++ b/talk/brno-php-2014/Makefile @@ -0,0 +1,6 @@ +talk.pdf: talk.rst stylesheet.latex author.latex + rst2beamer --stylesheet=stylesheet.latex --documentoptions=14pt talk.rst talk.latex --overlaybullets=false || exit + pdflatex talk.latex + +clean: + rm talk.pdf talk.latex \ No newline at end of file diff --git a/talk/brno-php-2014/stylesheet.latex b/talk/brno-php-2014/stylesheet.latex new file mode 100644 --- /dev/null +++ b/talk/brno-php-2014/stylesheet.latex @@ -0,0 +1,10 @@ +\usetheme{Warsaw} +\setbeamercovered{transparent} +\setbeamertemplate{navigation symbols}{} + +\definecolor{darkgreen}{rgb}{0, 0.5, 0.0} +\newcommand{\docutilsrolegreen}[1]{\color{darkgreen}#1\normalcolor} +\newcommand{\docutilsrolered}[1]{\color{red}#1\normalcolor} + +\newcommand{\green}[1]{\color{darkgreen}#1\normalcolor} +\newcommand{\red}[1]{\color{red}#1\normalcolor} diff --git a/talk/brno-php-2014/talk.pdf b/talk/brno-php-2014/talk.pdf new file mode 100644 index 0000000000000000000000000000000000000000..a2346852cf47c78cad5350a6639917bec93bb0d3 GIT binary patch [cut] diff --git a/talk/brno-php-2014/talk.rst b/talk/brno-php-2014/talk.rst --- a/talk/brno-php-2014/talk.rst +++ b/talk/brno-php-2014/talk.rst @@ -1,27 +1,27 @@ +.. include:: ../default/beamerdefs.txt + HippyVM - yet another attempt at PHP performance ------------------------------------------------- +================================================ Who am I? --------- * Maciej Fijalkowski -* PyPy developer for about 8 years +* PyPy core developer for about 8 years -* main author of hippyvm +* main author of HippyVM * founder of baroquesoftware.com This talk --------- -* hippyvm project +* HippyVM project * performance and measurments -* history - -* questions after each part +* a bit about PyPy, HippyVM and performance oriented virtual machines HippyVM ------- @@ -30,7 +30,7 @@ |pause| -* has a just in time compiler, which makes it fast +* has a **just in time compiler**, which makes it fast |pause| @@ -39,7 +39,7 @@ HippyVM status -------------- -* runs a lot of PHP test suite +* runs a lot of PHP test suite (over half) * misses a lot of builtin functions @@ -47,7 +47,18 @@ * cgi, fastcgi (not open source) -* fast +* performance oriented + +HippyVM demo +------------ + +* famous fibonacci + +* fannkuch + +* richards + +* bigger stuff\* HippyVM status - short ---------------------- @@ -61,11 +72,6 @@ * demo -HippyVM - questions -------------------- - -* ? - Let's talk about performance ---------------------------- @@ -88,6 +94,10 @@ * non-trivial interactions between the pieces +|pause| + +* I can't guess + Performance ----------- @@ -106,6 +116,50 @@ * don't use a single number +Bad benchmarking - example 1 +---------------------------- + +* benchmarks game + +* limited set of benchmarks + +* limited set of bottlenecks + +* relentlessly optimized + +Bad benchmarking - example 2 +---------------------------- + +* pystone + +* the idea - measure time consumed by each operation + +* should give you the idea how the program will operate + +|pause| + +* it's wrong + +Why it's wrong? +--------------- + +* the interaction is non-trivial + +* e.g. garbage collection + +* Python 2.6 vs 2.7, "minor" improvement in the GC + +* PyPy translation toolchain takes 1h instead of 3h + +Good benchmarking - example +--------------------------- + +* speed.pypy.org + +* a decent set of small medium and large benchmarks + +* I strongly encourage people to come up with the same for PHP + PHP performance --------------- @@ -145,27 +199,34 @@ * consider bottlenecks differ depending on implementation -Good benchmarking - example ---------------------------- +PHP performance problems +------------------------ -* speed.pypy.org +* low performance of the reference implementation -* I strongly encourage people to come up with the same for PHP +* relative immaturity of more advanced implementations + +* reload all the code creates problems + +|pause| + +* the language quirks can be worked around + +|pause| + +* in my opinion, dynamism, etc. does not matter Performance - personal opinions ------------------------------- * the language should be easy **for a programmer** +* we, as a PyPy team, never discuss language design + * the language implementation can be complex * libraries, patterns and the ecosystem matter for anything non-trivial -Performance - questions ------------------------ - -* ? - HippyVM history --------------- @@ -199,12 +260,53 @@ * we decided to write a framework instead -A typical example ------------------ +Just in time compiler? +---------------------- -* interpreter, written in C++ +* "lower level" languages, like C, have compilers that turn C into assembler -* just in time compiler that repeats the semantics, written in C++, +* "higher level" have interpreters that run a virtual machine + +* just in time is a hybrid - run in an interpreter, + compile to assembler on the fly + +Why not a compiler? +------------------- + +* "Just compile X to Lisp/C/JavaScript, it'll be fast!" + +|pause| + +* mismatch in semantics + +* hard-to-prove edge cases + +* ends up not being that fast + +Just in time - benefits +----------------------- + +* you can choose what to compile + +* you can ignore the edge cases (but have a way to deoptimize) + +* guesses are a lot easier + +* a lot of things fall in naturally, e.g. late imports + +A simple example +---------------- + +* an interpreter written in C/C++ + +* e.g. CPython, Zend PHP, MRI Ruby, ... + +A typical more advanced example +------------------------------- + +* interpreter, written in C/C++ + +* just in time compiler that repeats the semantics, written in C/C++, emits assembler * another layer, e.g. a method JIT @@ -213,10 +315,12 @@ * becomes harder and harder to keep up with semantics +* examples: V8, HHVM, \*monkey + PyPy approach ------------- -* write an interpreter in machine-readable language +* write an interpreter in a machine-readable language * just in time compiler gets generated from the description @@ -235,6 +339,15 @@ * Truffle is another example of a similar approach +HippyVM - future +---------------- + +* find more funding + +* run open source projects efficiently + +* develop a benchmark suite + Questions? ---------- diff --git a/talk/default/beamerdefs.txt b/talk/default/beamerdefs.txt new file mode 100644 --- /dev/null +++ b/talk/default/beamerdefs.txt @@ -0,0 +1,108 @@ +.. colors +.. =========================== + +.. role:: green +.. role:: red + + +.. general useful commands +.. =========================== + +.. |pause| raw:: latex + + \pause + +.. |small| raw:: latex + + {\small + +.. |end_small| raw:: latex + + } + +.. |scriptsize| raw:: latex + + {\scriptsize + +.. |end_scriptsize| raw:: latex + + } + +.. |strike<| raw:: latex + + \sout{ + +.. closed bracket +.. =========================== + +.. |>| raw:: latex + + } + + +.. example block +.. =========================== + +.. |example<| raw:: latex + + \begin{exampleblock}{ + + +.. |end_example| raw:: latex + + \end{exampleblock} + + + +.. alert block +.. =========================== + +.. |alert<| raw:: latex + + \begin{alertblock}{ + + +.. |end_alert| raw:: latex + + \end{alertblock} + + + +.. columns +.. =========================== + +.. |column1| raw:: latex + + \begin{columns} + \begin{column}{0.45\textwidth} + +.. |column2| raw:: latex + + \end{column} + \begin{column}{0.45\textwidth} + + +.. |end_columns| raw:: latex + + \end{column} + \end{columns} + + + +.. |snake| image:: ../../img/py-web-new.png + :scale: 15% + + + +.. nested blocks +.. =========================== + +.. |nested| raw:: latex + + \begin{columns} + \begin{column}{0.85\textwidth} + +.. |end_nested| raw:: latex + + \end{column} + \end{columns} diff --git a/talk/ep2014/status/stylesheet.latex b/talk/ep2014/status/stylesheet.latex --- a/talk/ep2014/status/stylesheet.latex +++ b/talk/ep2014/status/stylesheet.latex @@ -1,4 +1,3 @@ -\usetheme{Boadilla} \setbeamercovered{transparent} \setbeamertemplate{navigation symbols}{} From noreply at buildbot.pypy.org Sat Sep 20 18:20:10 2014 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 20 Sep 2014 18:20:10 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: clear array of structs too; Message-ID: <20140920162010.996561C02D6@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73616:7b6328a2da99 Date: 2014-09-20 18:16 +0200 http://bitbucket.org/pypy/pypy/changeset/7b6328a2da99/ Log: clear array of structs too; diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -125,9 +125,8 @@ def clear_varsize_gc_fields(self, descr, result, v_length=None): if self.gc_ll_descr.malloc_zero_filled: return - if not descr.is_array_of_pointers(): - return - self.handle_clear_array_contents(descr, result, v_length) + if descr.is_array_of_structs() or descr.is_array_of_pointers(): + self.handle_clear_array_contents(descr, result, v_length) def handle_new_fixedsize(self, descr, op): assert isinstance(descr, SizeDescr) From noreply at buildbot.pypy.org Sat Sep 20 23:16:24 2014 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Sat, 20 Sep 2014 23:16:24 +0200 (CEST) Subject: [pypy-commit] pypy default: Fix curses on OS X -- copy this logic from cpython Message-ID: <20140920211624.7E34B1C142A@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: Changeset: r73617:6948fbcc0042 Date: 2014-09-20 14:16 -0700 http://bitbucket.org/pypy/pypy/changeset/6948fbcc0042/ Log: Fix curses on OS X -- copy this logic from cpython diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py --- a/lib_pypy/_curses.py +++ b/lib_pypy/_curses.py @@ -286,6 +286,13 @@ lib = ffi.verify(""" +#ifdef __APPLE__ +/* the following define is necessary for OS X 10.6+; without it, the + Apple-supplied ncurses.h sets NCURSES_OPAQUE to 1, and then Python + can't get at the WINDOW flags field. */ +#define NCURSES_OPAQUE 0 +#endif + #include #include #include From noreply at buildbot.pypy.org Sat Sep 20 23:19:51 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Sat, 20 Sep 2014 23:19:51 +0200 (CEST) Subject: [pypy-commit] pypy use-file-star-for-file: add comment Message-ID: <20140920211951.59F661C142A@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: use-file-star-for-file Changeset: r73618:509fd1d325f2 Date: 2014-09-20 17:19 -0400 http://bitbucket.org/pypy/pypy/changeset/509fd1d325f2/ Log: add comment diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py --- a/rpython/rlib/rfile.py +++ b/rpython/rlib/rfile.py @@ -443,6 +443,7 @@ s.append_charpsize(raw_buf, chunksize) buffersize = self._new_buffersize(buffersize) remainsize = buffersize - s.getlength() + # XXX add rpython realloc so we can avoid the copy to stringbuilder? rffi.keep_buffer_alive_until_here(raw_buf, gc_buf) raw_buf, gc_buf = rffi.alloc_buffer(remainsize) if s is None: @@ -505,6 +506,7 @@ s.append_charpsize(raw_buf, remainsize - 1) buffersize += buffersize >> 2 remainsize = buffersize - s.getlength() + # XXX add rpython realloc so we can avoid the copy to stringbuilder? rffi.keep_buffer_alive_until_here(raw_buf, gc_buf) raw_buf, gc_buf = rffi.alloc_buffer(remainsize) if s is None: @@ -609,6 +611,7 @@ s.append_charpsize(raw_buf, i) buffersize += buffersize >> 2 remainsize = buffersize - s.getlength() + # XXX add rpython realloc so we can avoid the copy to stringbuilder? rffi.keep_buffer_alive_until_here(raw_buf, gc_buf) raw_buf, gc_buf = rffi.alloc_buffer(remainsize) i = 0 From noreply at buildbot.pypy.org Sun Sep 21 00:00:39 2014 From: noreply at buildbot.pypy.org (mattip) Date: Sun, 21 Sep 2014 00:00:39 +0200 (CEST) Subject: [pypy-commit] pypy default: update for 2.4.0 final Message-ID: <20140920220039.DD9171C3488@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r73619:e0c96ae36f5b Date: 2014-09-21 00:19 +0300 http://bitbucket.org/pypy/pypy/changeset/e0c96ae36f5b/ Log: update for 2.4.0 final diff --git a/pypy/doc/release-2.4.0.rst b/pypy/doc/release-2.4.0.rst --- a/pypy/doc/release-2.4.0.rst +++ b/pypy/doc/release-2.4.0.rst @@ -5,7 +5,7 @@ We're pleased to announce PyPy 2.4, which contains significant performance enhancements and bug fixes. -You can already download the PyPy 2.4-beta1 pre-release here: +You can download the PyPy 2.4.0 release here: http://pypy.org/download.html @@ -63,6 +63,8 @@ PyPy now uses Python 2.7.8 standard library. +We fixed a memory leak in IO in the sandbox_ code + We welcomed more than 12 new contributors, and conducted two Google Summer of Code projects, as well as other student projects not directly related to Summer of Code. @@ -105,6 +107,7 @@ .. _`whats-new`: http://doc.pypy.org/en/latest/whatsnew-2.3.1.html .. _resolved: https://bitbucket.org/pypy/pypy/issues?status=resolved +.. _sandbox: http://doc.pypy.org/en/latest/sandbox.html We have further improvements on the way: rpython file handling, numpy linalg compatibility, as well From noreply at buildbot.pypy.org Sun Sep 21 00:00:41 2014 From: noreply at buildbot.pypy.org (mattip) Date: Sun, 21 Sep 2014 00:00:41 +0200 (CEST) Subject: [pypy-commit] pypy default: fix cffi to remove tmp files, should be ported to upstream Message-ID: <20140920220041.2B9281C3488@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r73620:cdf3042470fb Date: 2014-09-21 01:00 +0300 http://bitbucket.org/pypy/pypy/changeset/cdf3042470fb/ Log: fix cffi to remove tmp files, should be ported to upstream diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_zdistutils.py b/pypy/module/test_lib_pypy/cffi_tests/test_zdistutils.py --- a/pypy/module/test_lib_pypy/cffi_tests/test_zdistutils.py +++ b/pypy/module/test_lib_pypy/cffi_tests/test_zdistutils.py @@ -16,6 +16,10 @@ if distutils.ccompiler.get_default_compiler() == 'msvc': self.lib_m = 'msvcrt' + def teardown_class(self): + if udir.isdir(): + udir.remove() + def test_locate_engine_class(self): cls = _locate_engine_class(FFI(), self.generic) if self.generic: From noreply at buildbot.pypy.org Sun Sep 21 00:29:51 2014 From: noreply at buildbot.pypy.org (mattip) Date: Sun, 21 Sep 2014 00:29:51 +0200 (CEST) Subject: [pypy-commit] cffi default: remove tmp files after test Message-ID: <20140920222951.2E6D91C142A@cobra.cs.uni-duesseldorf.de> Author: Matti Picus Branch: Changeset: r1567:4f869a54f47e Date: 2014-09-21 01:06 +0300 http://bitbucket.org/cffi/cffi/changeset/4f869a54f47e/ Log: remove tmp files after test diff --git a/testing/test_zdistutils.py b/testing/test_zdistutils.py --- a/testing/test_zdistutils.py +++ b/testing/test_zdistutils.py @@ -15,6 +15,10 @@ if distutils.ccompiler.get_default_compiler() == 'msvc': self.lib_m = 'msvcrt' + def teardown_class(self): + if udir.isdir(): + udir.remove() + def test_locate_engine_class(self): cls = _locate_engine_class(FFI(), self.generic) if self.generic: From noreply at buildbot.pypy.org Sun Sep 21 00:29:52 2014 From: noreply at buildbot.pypy.org (mattip) Date: Sun, 21 Sep 2014 00:29:52 +0200 (CEST) Subject: [pypy-commit] cffi default: remove tmp files after test Message-ID: <20140920222952.7283C1C142A@cobra.cs.uni-duesseldorf.de> Author: Matti Picus Branch: Changeset: r1568:bad1c0344487 Date: 2014-09-21 01:29 +0300 http://bitbucket.org/cffi/cffi/changeset/bad1c0344487/ Log: remove tmp files after test diff --git a/testing/test_zintegration.py b/testing/test_zintegration.py --- a/testing/test_zintegration.py +++ b/testing/test_zintegration.py @@ -72,50 +72,55 @@ assert not os.path.exists(str(SNIPPET_DIR.join(dirname, 'lextab.py'))) assert not os.path.exists(str(SNIPPET_DIR.join(dirname, 'yacctab.py'))) -def test_infrastructure(): - run_setup_and_program('infrastructure', ''' - import snip_infrastructure - assert snip_infrastructure.func() == 42 - ''') +class TestZIntegration(object): + def teardown_class(self): + if udir.isdir(): + udir.remove() -def test_distutils_module(): - run_setup_and_program("distutils_module", ''' - import snip_basic_verify - p = snip_basic_verify.C.getpwuid(0) - assert snip_basic_verify.ffi.string(p.pw_name) == b"root" - ''') + def test_infrastructure(self): + run_setup_and_program('infrastructure', ''' + import snip_infrastructure + assert snip_infrastructure.func() == 42 + ''') -def test_distutils_package_1(): - run_setup_and_program("distutils_package_1", ''' - import snip_basic_verify1 - p = snip_basic_verify1.C.getpwuid(0) - assert snip_basic_verify1.ffi.string(p.pw_name) == b"root" - ''') + def test_distutils_module(self): + run_setup_and_program("distutils_module", ''' + import snip_basic_verify + p = snip_basic_verify.C.getpwuid(0) + assert snip_basic_verify.ffi.string(p.pw_name) == b"root" + ''') -def test_distutils_package_2(): - run_setup_and_program("distutils_package_2", ''' - import snip_basic_verify2 - p = snip_basic_verify2.C.getpwuid(0) - assert snip_basic_verify2.ffi.string(p.pw_name) == b"root" - ''') + def test_distutils_package_1(self): + run_setup_and_program("distutils_package_1", ''' + import snip_basic_verify1 + p = snip_basic_verify1.C.getpwuid(0) + assert snip_basic_verify1.ffi.string(p.pw_name) == b"root" + ''') -def test_setuptools_module(): - run_setup_and_program("setuptools_module", ''' - import snip_setuptools_verify - p = snip_setuptools_verify.C.getpwuid(0) - assert snip_setuptools_verify.ffi.string(p.pw_name) == b"root" - ''') + def test_distutils_package_2(self): + run_setup_and_program("distutils_package_2", ''' + import snip_basic_verify2 + p = snip_basic_verify2.C.getpwuid(0) + assert snip_basic_verify2.ffi.string(p.pw_name) == b"root" + ''') -def test_setuptools_package_1(): - run_setup_and_program("setuptools_package_1", ''' - import snip_setuptools_verify1 - p = snip_setuptools_verify1.C.getpwuid(0) - assert snip_setuptools_verify1.ffi.string(p.pw_name) == b"root" - ''') + def test_setuptools_module(self): + run_setup_and_program("setuptools_module", ''' + import snip_setuptools_verify + p = snip_setuptools_verify.C.getpwuid(0) + assert snip_setuptools_verify.ffi.string(p.pw_name) == b"root" + ''') -def test_setuptools_package_2(): - run_setup_and_program("setuptools_package_2", ''' - import snip_setuptools_verify2 - p = snip_setuptools_verify2.C.getpwuid(0) - assert snip_setuptools_verify2.ffi.string(p.pw_name) == b"root" - ''') + def test_setuptools_package_1(self): + run_setup_and_program("setuptools_package_1", ''' + import snip_setuptools_verify1 + p = snip_setuptools_verify1.C.getpwuid(0) + assert snip_setuptools_verify1.ffi.string(p.pw_name) == b"root" + ''') + + def test_setuptools_package_2(self): + run_setup_and_program("setuptools_package_2", ''' + import snip_setuptools_verify2 + p = snip_setuptools_verify2.C.getpwuid(0) + assert snip_setuptools_verify2.ffi.string(p.pw_name) == b"root" + ''') From noreply at buildbot.pypy.org Sun Sep 21 00:46:41 2014 From: noreply at buildbot.pypy.org (mattip) Date: Sun, 21 Sep 2014 00:46:41 +0200 (CEST) Subject: [pypy-commit] pypy default: remove tmp files after test (ported from upstream) Message-ID: <20140920224641.781741C142A@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r73621:0f69f42b7721 Date: 2014-09-21 01:46 +0300 http://bitbucket.org/pypy/pypy/changeset/0f69f42b7721/ Log: remove tmp files after test (ported from upstream) diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_zintegration.py b/pypy/module/test_lib_pypy/cffi_tests/test_zintegration.py --- a/pypy/module/test_lib_pypy/cffi_tests/test_zintegration.py +++ b/pypy/module/test_lib_pypy/cffi_tests/test_zintegration.py @@ -73,50 +73,55 @@ assert not os.path.exists(str(SNIPPET_DIR.join(dirname, 'lextab.py'))) assert not os.path.exists(str(SNIPPET_DIR.join(dirname, 'yacctab.py'))) -def test_infrastructure(): - run_setup_and_program('infrastructure', ''' - import snip_infrastructure - assert snip_infrastructure.func() == 42 - ''') +class TestZIntegration(object): + def teardown_class(self): + if udir.isdir(): + udir.remove() -def test_distutils_module(): - run_setup_and_program("distutils_module", ''' - import snip_basic_verify - p = snip_basic_verify.C.getpwuid(0) - assert snip_basic_verify.ffi.string(p.pw_name) == b"root" - ''') + def test_infrastructure(self): + run_setup_and_program('infrastructure', ''' + import snip_infrastructure + assert snip_infrastructure.func() == 42 + ''') -def test_distutils_package_1(): - run_setup_and_program("distutils_package_1", ''' - import snip_basic_verify1 - p = snip_basic_verify1.C.getpwuid(0) - assert snip_basic_verify1.ffi.string(p.pw_name) == b"root" - ''') + def test_distutils_module(self): + run_setup_and_program("distutils_module", ''' + import snip_basic_verify + p = snip_basic_verify.C.getpwuid(0) + assert snip_basic_verify.ffi.string(p.pw_name) == b"root" + ''') -def test_distutils_package_2(): - run_setup_and_program("distutils_package_2", ''' - import snip_basic_verify2 - p = snip_basic_verify2.C.getpwuid(0) - assert snip_basic_verify2.ffi.string(p.pw_name) == b"root" - ''') + def test_distutils_package_1(self): + run_setup_and_program("distutils_package_1", ''' + import snip_basic_verify1 + p = snip_basic_verify1.C.getpwuid(0) + assert snip_basic_verify1.ffi.string(p.pw_name) == b"root" + ''') -def test_setuptools_module(): - run_setup_and_program("setuptools_module", ''' - import snip_setuptools_verify - p = snip_setuptools_verify.C.getpwuid(0) - assert snip_setuptools_verify.ffi.string(p.pw_name) == b"root" - ''') + def test_distutils_package_2(self): + run_setup_and_program("distutils_package_2", ''' + import snip_basic_verify2 + p = snip_basic_verify2.C.getpwuid(0) + assert snip_basic_verify2.ffi.string(p.pw_name) == b"root" + ''') -def test_setuptools_package_1(): - run_setup_and_program("setuptools_package_1", ''' - import snip_setuptools_verify1 - p = snip_setuptools_verify1.C.getpwuid(0) - assert snip_setuptools_verify1.ffi.string(p.pw_name) == b"root" - ''') + def test_setuptools_module(self): + run_setup_and_program("setuptools_module", ''' + import snip_setuptools_verify + p = snip_setuptools_verify.C.getpwuid(0) + assert snip_setuptools_verify.ffi.string(p.pw_name) == b"root" + ''') -def test_setuptools_package_2(): - run_setup_and_program("setuptools_package_2", ''' - import snip_setuptools_verify2 - p = snip_setuptools_verify2.C.getpwuid(0) - assert snip_setuptools_verify2.ffi.string(p.pw_name) == b"root" - ''') + def test_setuptools_package_1(self): + run_setup_and_program("setuptools_package_1", ''' + import snip_setuptools_verify1 + p = snip_setuptools_verify1.C.getpwuid(0) + assert snip_setuptools_verify1.ffi.string(p.pw_name) == b"root" + ''') + + def test_setuptools_package_2(self): + run_setup_and_program("setuptools_package_2", ''' + import snip_setuptools_verify2 + p = snip_setuptools_verify2.C.getpwuid(0) + assert snip_setuptools_verify2.ffi.string(p.pw_name) == b"root" + ''') From noreply at buildbot.pypy.org Sun Sep 21 00:59:02 2014 From: noreply at buildbot.pypy.org (mattip) Date: Sun, 21 Sep 2014 00:59:02 +0200 (CEST) Subject: [pypy-commit] pypy.org extradoc: update for 2.4.0 Message-ID: <20140920225902.9A49D1C3488@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: extradoc Changeset: r537:171c02d0ea6f Date: 2014-09-21 01:54 +0300 http://bitbucket.org/pypy/pypy.org/changeset/171c02d0ea6f/ Log: update for 2.4.0 diff --git a/download.html b/download.html --- a/download.html +++ b/download.html @@ -52,8 +52,7 @@ miscompiled due to buildslave being old. Contributions are welcomed.

      We provide binaries for x86 and ARM Linux, Mac OS/X and Windows for:

        @@ -89,33 +88,16 @@ degrees of being up-to-date. You may have more luck trying out Squeaky's portable Linux binaries.

    -
    -

    Python2.7 compatible PyPy 2.3.1

    + -
    -

    Python2.7 compatible PyPy 2.4-beta1

    -
    @@ -209,8 +191,8 @@

  • Get the source code. The following packages contain the source at the same revision as the above binaries:

    Or you can checkout the current trunk using Mercurial (the trunk usually works and is of course more up-to-date):

    @@ -281,22 +263,17 @@

    Checksums

    Here are the checksums for each of the downloads (md5 and sha1):

    -3237dcf84b03ddaa0d2c7453531d04f0  pypy-2.3.1-linux64.tar.bz2
-ac6845dba055e0d962ae744e0ef8cee1  pypy-2.3.1-linux-armhf-raring.tar.bz2
-0a96d00edf99d42ac21d46301cffcadb  pypy-2.3.1-linux-armhf-raspbian.tar.bz2
-ef9a6e49124c879aa5495c5a2b90ec3c  pypy-2.3.1-linux.tar.bz2
-39b8c4c7ae8ff9f7dd8b854834b530ef  pypy-2.3.1-osx64.tar.bz2
-aa71e065f89849a9f641cc3ca8d80ebc  pypy-2.3.1-win32.zip
-5dab108869a039b754da5c07046fb17c  pypy-2.3.1-linux-armel.tar.bz2
-2b9aeccef1587a42fb5a4cc304b5d881  pypy-2.3.1-src.tar.bz2
-15c068c357d60719086b23e0bf9d0a5b  pypy-2.3.1-src.zip
-cc19c6807d6fa3e0d6e0e8c9f5331aa5  pypy-2.4-beta1-linux64.tar.bz2
-03597850c1179a55870da11bc1f89d3b  pypy-2.4-beta1-linux-armel.tar.bz2
-3fe7f30be24796812538beec1df67154  pypy-2.4-beta1-linux-armhf-raring.tar.bz2
-f68438ea507d8236ec2c330078c4c8fe  pypy-2.4-beta1-linux-armhf-raspbian.tar.bz2
-f59f63ea54a7f53bf21e58d49ca1be75  pypy-2.4-beta1-linux.tar.bz2
-d4d9c315e38fd52564d57add80804707  pypy-2.4-beta1-osx64.tar.bz2
-1191e71a86beef48861a61327c4ff4fc  pypy-2.4-beta1-win32.zip
+63bd68546f60cf5921ba7654f3fe47aa  pypy-2.4.0-linux64.tar.bz2
+6c9b444a1cd090ab7b43083a24e07734  pypy-2.4.0-linux-armel.tar.bz2
+5ff951da5989a00e01611678c311f8af  pypy-2.4.0-linux-armhf-raring.tar.bz2
+d7540883a52f91433da62b0cdfaaa30f  pypy-2.4.0-linux-armhf-raspbian.tar.bz2
+77a971f5198685ff60528de5def407dd  pypy-2.4.0-linux.tar.bz2
+59ecfb3084e725fb5fd5ccedf8e0686a  pypy-2.4.0-src.zip
+907d6fbabc5bcd5bafdcf02a76a8ca33  pypy-2.4.0-win32.zip
+5487df96f47d93fc1c45265c26f17945  pypy-2.4.0-src.tar.bz2
+62d2894064234c8d010d773534d0c69a  pypy-2.4.0-osx64.tar.bz2
+eaba228fa7ba3f7022804900f9c9f885  pypy-2.4.0-sandbox-linux64.tar.bz2
+9b2aaf173bf424ca12097a13fc58eb0d  pypy-2.4.0-sandbox-linux.tar.bz2
 a86da5688dfd84e0485239f8991af308  pypy3-2.3.1-linux64.tar.bz2
 b0d6a0491e9c9be39d3c314c0823a039  pypy3-2.3.1-linux-armel.tar.bz2
 f79f7b5588d2b5a68d2781908bc8f9af  pypy3-2.3.1-linux-armhf-raring.tar.bz2
@@ -308,22 +285,17 @@
 88f33c7a25996fe4abf342765570a8fe  pypy3-2.3.1-src.zip
 2c9f0054f3b93a6473f10be35277825a  pypy-1.8-sandbox-linux64.tar.bz2
 009c970b5fa75754ae4c32a5d108a8d4  pypy-1.8-sandbox-linux.tar.bz2
-8ef9b71a5fd66bdb598f178602ea195f7c1b0f66  pypy-2.3.1-linux64.tar.bz2
-e76fb3b2e11cc6ffe6004377292621b6a38ddcde  pypy-2.3.1-linux-armhf-raring.tar.bz2
-31b144d77fd8203d8950b25c8ec8e2a28aed450d  pypy-2.3.1-linux-armhf-raspbian.tar.bz2
-18e5bcf259501c99200798f252e5902f29ca2941  pypy-2.3.1-linux.tar.bz2
-4d9cdf801e4c8fb432b17be0edf76eb3d9360f40  pypy-2.3.1-osx64.tar.bz2
-08639771f26188739a82408454188582c6e25ce9  pypy-2.3.1-win32.zip
-ad8ebf67c5ccf354513a9cdb0586080b5964a5bd  pypy-2.3.1-linux-armel.tar.bz2
-833b33042456fe381cae4481b2eb536c5787d6c7  pypy-2.3.1-src.tar.bz2
-0d3f750fc28713eca77a91388c5a63843406d631  pypy-2.3.1-src.zip
-7bb3e8fa4b459f15046c2604161c0191032a9e9b  pypy-2.4-beta1-linux64.tar.bz2
-99bcafcf0c30adbc9a228ebefc6499a645dda27e  pypy-2.4-beta1-linux-armel.tar.bz2
-6f9828cfbe58029e2c82fb60631e9abe931fd5f3  pypy-2.4-beta1-linux-armhf-raring.tar.bz2
-d8cf8d20abb8209c580a930a1fcc76aa2c0a8084  pypy-2.4-beta1-linux-armhf-raspbian.tar.bz2
-c405dc962357d5a7f62b4c693980ad533bca93d5  pypy-2.4-beta1-linux.tar.bz2
-3606211ba23f3b1ffafbd707097c57d83f959d4e  pypy-2.4-beta1-osx64.tar.bz2
-4d6f30261a4cdde55372f1d1f7e3764c9724d1a9  pypy-2.4-beta1-win32.zip
+c362247226f1cde2b957ab5e885f41475381553b  pypy-2.4.0-linux64.tar.bz2
+d542ee549ded9face573ac9fb49a23f5a5b4be60  pypy-2.4.0-linux-armel.tar.bz2
+b8e02dc381e5040e2bf50541e82f0148f9a46a48  pypy-2.4.0-linux-armhf-raring.tar.bz2
+ad65e7ddb1582b465a37090dc4a13bc37a8cd95d  pypy-2.4.0-linux-armhf-raspbian.tar.bz2
+fd52b42069287ca11e816c8e18fc95f53542c73d  pypy-2.4.0-linux.tar.bz2
+fdc25708de28f7e46caa4435507c44f534f6c817  pypy-2.4.0-src.tar.bz2
+16e9f954e2829fc3961c922d016259f44ac55a0e  pypy-2.4.0-src.zip
+b72c3365c23c34ffd35a781fb72d8722e0b7517e  pypy-2.4.0-win32.zip
+b9f87df364617c1a8c43303a467d698e4a3e1935  pypy-2.4.0-osx64.tar.bz2
+17d6cb1818d3d5c4d0a099e1af518a437609f0c1  pypy-2.4.0-sandbox-linux64.tar.bz2
+6a590a4f935b7e5fc95b6a3e783819b5ad2a85ef  pypy-2.4.0-sandbox-linux.tar.bz2
 7276a9e97229e754f66863a0793c59066120ec51  pypy3-2.3.1-linux64.tar.bz2
 fb52a30be0fd4c7d8686c98e03595a8b48b11e82  pypy3-2.3.1-linux-armel.tar.bz2
 0239677fe28a4c22a70853242368456b98ac665a  pypy3-2.3.1-linux-armhf-raring.tar.bz2
diff --git a/index.html b/index.html
--- a/index.html
+++ b/index.html
@@ -63,7 +63,7 @@
 
  • As well as other features.
    • 
 
 
-
    Download and try out the PyPy or PyPy3 release 2.3.1, or the 2.4-beta!
    
+
    Download and try out the PyPy or PyPy3 release 2.4.0
    
 
    Want to know more? A good place to start is our detailed speed and
 compatibility reports!
    • diff --git a/source/download.txt b/source/download.txt --- a/source/download.txt +++ b/source/download.txt @@ -16,11 +16,10 @@ We provide binaries for x86 and ARM Linux, Mac OS/X and Windows for: -* the Python2.7 compatible release — **PyPy 2.3.1** — (`what's new in PyPy 2.3.1?`_ ) -* the Python2.7 compatible beta — **PyPy 2.4-beta1** — (`what's new in PyPy 2.4?`_ ) +* the Python2.7 compatible release — **PyPy 2.4.0** — (`what's new in PyPy 2.4.0?`_ ) * the Python3.2.5 compatible release — **PyPy3 2.3.1** — (`what's new in PyPy3 2.3.1?`_). -.. _what's new in PyPy 2.3.1?: http://doc.pypy.org/en/latest/release-2.3.1.html +.. _what's new in PyPy 2.4.0?: http://doc.pypy.org/en/latest/release-2.4.0.html .. _what's new in PyPy3 2.3.1?: http://doc.pypy.org/en/latest/release-pypy3-2.3.1.html .. _what's new in PyPy 2.4?: http://doc.pypy.org/en/latest/release-2.4.0.html @@ -73,36 +72,7 @@ .. _`portable Linux binaries`: https://github.com/squeaky-pl/portable-pypy -Python2.7 compatible PyPy 2.3.1 -------------------------------- - -* `Linux x86 binary (32bit, tar.bz2 built on Ubuntu 10.04.4 LTS)`__ (see ``[1]`` below) -* `Linux x86-64 binary (64bit, tar.bz2 built on Ubuntu 12.04 - 14.04)`__ (see ``[1]`` below) -* `ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Raspbian)`__ (see ``[1]`` below) -* `ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Ubuntu Raring)`__ (see ``[1]`` below) -* `ARM Softfloat Linux binary (ARMEL/gnueabi, tar.bz2, Ubuntu Precise)`__ (see ``[1]`` below) -* `Mac OS/X binary (64bit)`__ -* `Windows binary (32bit)`__ (you might need the `VS 2008 runtime library - installer vcredist_x86.exe`_.) -* `Source (tar.bz2)`__ -* `Source (zip)`__ -* `All our downloads,`__ including previous versions. We also have a - mirror_, but please use only if you have troubles accessing the links above - -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.3.1-linux.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.3.1-linux64.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.3.1-linux-armhf-raspbian.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.3.1-linux-armhf-raring.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.3.1-linux-armel.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.3.1-osx64.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.3.1-win32.zip -.. _`VS 2008 runtime library installer vcredist_x86.exe`: http://www.microsoft.com/en-us/download/details.aspx?id=5582 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.3.1-src.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.3.1-src.zip -.. __: https://bitbucket.org/pypy/pypy/downloads -.. _mirror: http://cobra.cs.uni-duesseldorf.de/~buildmaster/mirror/ - -Python2.7 compatible PyPy 2.4-beta1 +Python2.7 compatible PyPy 2.4.0 ----------------------------------- * `Linux x86 binary (32bit, tar.bz2 built on Ubuntu 12.04 - 14.04)`__ (see ``[1]`` below) @@ -116,13 +86,13 @@ * `All our downloads,`__ including previous versions. We also have a mirror_, but please use only if you have troubles accessing the links above -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.4-beta1-linux.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.4-beta1-linux64.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.4-beta1-linux-armhf-raspbian.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.4-beta1-linux-armhf-raring.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.4-beta1-linux-armel.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.4-beta1-osx64.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.4-beta1-win32.zip +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.4.0-linux.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.4.0-linux64.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.4.0-linux-armhf-raspbian.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.4.0-linux-armhf-raring.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.4.0-linux-armel.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.4.0-osx64.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.4.0-win32.zip .. _`VS 2008 runtime library installer vcredist_x86.exe`: http://www.microsoft.com/en-us/download/details.aspx?id=5582 .. __: https://bitbucket.org/pypy/pypy/downloads .. _mirror: http://cobra.cs.uni-duesseldorf.de/~buildmaster/mirror/ @@ -201,7 +171,7 @@ uncompressed, they run in-place. For now you can uncompress them either somewhere in your home directory or, say, in ``/opt``, and if you want, put a symlink from somewhere like -``/usr/local/bin/pypy`` to ``/path/to/pypy-2.4-beta1/bin/pypy``. Do +``/usr/local/bin/pypy`` to ``/path/to/pypy-2.4.0/bin/pypy``. Do not move or copy the executable ``pypy`` outside the tree --- put a symlink to it, otherwise it will not find its libraries. @@ -250,11 +220,11 @@ 1. Get the source code. The following packages contain the source at the same revision as the above binaries: - * `pypy-2.3.1-src.tar.bz2`__ (sources, Unix line endings) - * `pypy-2.3.1-src.zip`__ (sources, Unix line endings too) + * `pypy-2.4.0-src.tar.bz2`__ (sources, Unix line endings) + * `pypy-2.4.0-src.zip`__ (sources, Unix line endings too) - .. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.3.1-src.tar.bz2 - .. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.3.1-src.zip + .. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.4.0-src.tar.bz2 + .. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.4.0-src.zip Or you can checkout the current trunk using Mercurial_ (the trunk usually works and is of course more up-to-date):: @@ -334,23 +304,17 @@ Here are the checksums for each of the downloads (md5 and sha1):: - 3237dcf84b03ddaa0d2c7453531d04f0 pypy-2.3.1-linux64.tar.bz2 - ac6845dba055e0d962ae744e0ef8cee1 pypy-2.3.1-linux-armhf-raring.tar.bz2 - 0a96d00edf99d42ac21d46301cffcadb pypy-2.3.1-linux-armhf-raspbian.tar.bz2 - ef9a6e49124c879aa5495c5a2b90ec3c pypy-2.3.1-linux.tar.bz2 - 39b8c4c7ae8ff9f7dd8b854834b530ef pypy-2.3.1-osx64.tar.bz2 - aa71e065f89849a9f641cc3ca8d80ebc pypy-2.3.1-win32.zip - 5dab108869a039b754da5c07046fb17c pypy-2.3.1-linux-armel.tar.bz2 - 2b9aeccef1587a42fb5a4cc304b5d881 pypy-2.3.1-src.tar.bz2 - 15c068c357d60719086b23e0bf9d0a5b pypy-2.3.1-src.zip - - cc19c6807d6fa3e0d6e0e8c9f5331aa5 pypy-2.4-beta1-linux64.tar.bz2 - 03597850c1179a55870da11bc1f89d3b pypy-2.4-beta1-linux-armel.tar.bz2 - 3fe7f30be24796812538beec1df67154 pypy-2.4-beta1-linux-armhf-raring.tar.bz2 - f68438ea507d8236ec2c330078c4c8fe pypy-2.4-beta1-linux-armhf-raspbian.tar.bz2 - f59f63ea54a7f53bf21e58d49ca1be75 pypy-2.4-beta1-linux.tar.bz2 - d4d9c315e38fd52564d57add80804707 pypy-2.4-beta1-osx64.tar.bz2 - 1191e71a86beef48861a61327c4ff4fc pypy-2.4-beta1-win32.zip + 63bd68546f60cf5921ba7654f3fe47aa pypy-2.4.0-linux64.tar.bz2 + 6c9b444a1cd090ab7b43083a24e07734 pypy-2.4.0-linux-armel.tar.bz2 + 5ff951da5989a00e01611678c311f8af pypy-2.4.0-linux-armhf-raring.tar.bz2 + d7540883a52f91433da62b0cdfaaa30f pypy-2.4.0-linux-armhf-raspbian.tar.bz2 + 77a971f5198685ff60528de5def407dd pypy-2.4.0-linux.tar.bz2 + 59ecfb3084e725fb5fd5ccedf8e0686a pypy-2.4.0-src.zip + 907d6fbabc5bcd5bafdcf02a76a8ca33 pypy-2.4.0-win32.zip + 5487df96f47d93fc1c45265c26f17945 pypy-2.4.0-src.tar.bz2 + 62d2894064234c8d010d773534d0c69a pypy-2.4.0-osx64.tar.bz2 + eaba228fa7ba3f7022804900f9c9f885 pypy-2.4.0-sandbox-linux64.tar.bz2 + 9b2aaf173bf424ca12097a13fc58eb0d pypy-2.4.0-sandbox-linux.tar.bz2 a86da5688dfd84e0485239f8991af308 pypy3-2.3.1-linux64.tar.bz2 b0d6a0491e9c9be39d3c314c0823a039 pypy3-2.3.1-linux-armel.tar.bz2 @@ -364,23 +328,18 @@ 2c9f0054f3b93a6473f10be35277825a pypy-1.8-sandbox-linux64.tar.bz2 009c970b5fa75754ae4c32a5d108a8d4 pypy-1.8-sandbox-linux.tar.bz2 - 8ef9b71a5fd66bdb598f178602ea195f7c1b0f66 pypy-2.3.1-linux64.tar.bz2 - e76fb3b2e11cc6ffe6004377292621b6a38ddcde pypy-2.3.1-linux-armhf-raring.tar.bz2 - 31b144d77fd8203d8950b25c8ec8e2a28aed450d pypy-2.3.1-linux-armhf-raspbian.tar.bz2 - 18e5bcf259501c99200798f252e5902f29ca2941 pypy-2.3.1-linux.tar.bz2 - 4d9cdf801e4c8fb432b17be0edf76eb3d9360f40 pypy-2.3.1-osx64.tar.bz2 - 08639771f26188739a82408454188582c6e25ce9 pypy-2.3.1-win32.zip - ad8ebf67c5ccf354513a9cdb0586080b5964a5bd pypy-2.3.1-linux-armel.tar.bz2 - 833b33042456fe381cae4481b2eb536c5787d6c7 pypy-2.3.1-src.tar.bz2 - 0d3f750fc28713eca77a91388c5a63843406d631 pypy-2.3.1-src.zip + c362247226f1cde2b957ab5e885f41475381553b pypy-2.4.0-linux64.tar.bz2 + d542ee549ded9face573ac9fb49a23f5a5b4be60 pypy-2.4.0-linux-armel.tar.bz2 + b8e02dc381e5040e2bf50541e82f0148f9a46a48 pypy-2.4.0-linux-armhf-raring.tar.bz2 + ad65e7ddb1582b465a37090dc4a13bc37a8cd95d pypy-2.4.0-linux-armhf-raspbian.tar.bz2 + fd52b42069287ca11e816c8e18fc95f53542c73d pypy-2.4.0-linux.tar.bz2 + fdc25708de28f7e46caa4435507c44f534f6c817 pypy-2.4.0-src.tar.bz2 + 16e9f954e2829fc3961c922d016259f44ac55a0e pypy-2.4.0-src.zip + b72c3365c23c34ffd35a781fb72d8722e0b7517e pypy-2.4.0-win32.zip + b9f87df364617c1a8c43303a467d698e4a3e1935 pypy-2.4.0-osx64.tar.bz2 + 17d6cb1818d3d5c4d0a099e1af518a437609f0c1 pypy-2.4.0-sandbox-linux64.tar.bz2 + 6a590a4f935b7e5fc95b6a3e783819b5ad2a85ef pypy-2.4.0-sandbox-linux.tar.bz2 - 7bb3e8fa4b459f15046c2604161c0191032a9e9b pypy-2.4-beta1-linux64.tar.bz2 - 99bcafcf0c30adbc9a228ebefc6499a645dda27e pypy-2.4-beta1-linux-armel.tar.bz2 - 6f9828cfbe58029e2c82fb60631e9abe931fd5f3 pypy-2.4-beta1-linux-armhf-raring.tar.bz2 - d8cf8d20abb8209c580a930a1fcc76aa2c0a8084 pypy-2.4-beta1-linux-armhf-raspbian.tar.bz2 - c405dc962357d5a7f62b4c693980ad533bca93d5 pypy-2.4-beta1-linux.tar.bz2 - 3606211ba23f3b1ffafbd707097c57d83f959d4e pypy-2.4-beta1-osx64.tar.bz2 - 4d6f30261a4cdde55372f1d1f7e3764c9724d1a9 pypy-2.4-beta1-win32.zip 7276a9e97229e754f66863a0793c59066120ec51 pypy3-2.3.1-linux64.tar.bz2 fb52a30be0fd4c7d8686c98e03595a8b48b11e82 pypy3-2.3.1-linux-armel.tar.bz2 diff --git a/source/index.txt b/source/index.txt --- a/source/index.txt +++ b/source/index.txt @@ -26,7 +26,7 @@ .. class:: download -`Download and try out the PyPy or PyPy3 release 2.3.1, or the 2.4-beta!`__ +`Download and try out the PyPy or PyPy3 release 2.4.0`__ .. __: download.html From noreply at buildbot.pypy.org Sun Sep 21 07:27:19 2014 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 21 Sep 2014 07:27:19 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: remove spurious zero_ptr_fields + fix tests Message-ID: <20140921052719.2997A1C35EE@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73622:fe96b6f2e560 Date: 2014-09-21 07:26 +0200 http://bitbucket.org/pypy/pypy/changeset/fe96b6f2e560/ Log: remove spurious zero_ptr_fields + fix tests diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -45,6 +45,7 @@ self.newops = [] self.known_lengths = {} self.write_barrier_applied = {} + self.delayed_zero_setfields = {} def rewrite(self, operations): # we can only remember one malloc since the next malloc can possibly @@ -71,6 +72,7 @@ # ---------- write barriers ---------- if self.gc_ll_descr.write_barrier_descr is not None: if op.getopnum() == rop.SETFIELD_GC: + self.consider_setfield_gc(op) self.handle_write_barrier_setfield(op) continue if op.getopnum() == rop.SETINTERIORFIELD_GC: @@ -79,10 +81,18 @@ if op.getopnum() == rop.SETARRAYITEM_GC: self.handle_write_barrier_setarrayitem(op) continue + else: + # this is dead code, but in case we have a gc that does + # not have a write barrier and does not zero memory, we would + # need to clal it + if op.getopnum() == rop.SETFIELD_GC: + self.consider_setfield_gc(op) # ---------- call assembler ----------- if op.getopnum() == rop.CALL_ASSEMBLER: self.handle_call_assembler(op) continue + if op.getopnum() == rop.JUMP or op.getopnum() == rop.FINISH: + self.emit_pending_zeros() # self.newops.append(op) return self.newops @@ -118,14 +128,28 @@ def clear_gc_fields(self, descr, result): if self.gc_ll_descr.malloc_zero_filled: return + try: + d = self.delayed_zero_setfields[result] + except KeyError: + d = {} + self.delayed_zero_setfields[result] = d for ofs in descr.offsets_of_gcfields: - o = ResOperation(rop.ZERO_PTR_FIELD, [result, ConstInt(ofs)], None) - self.newops.append(o) + d[ofs] = None + + def consider_setfield_gc(self, op): + offset = op.getdescr().offset + try: + del self.delayed_zero_setfields[op.getarg(0)][offset] + except KeyError: + pass def clear_varsize_gc_fields(self, descr, result, v_length=None): if self.gc_ll_descr.malloc_zero_filled: return if descr.is_array_of_structs() or descr.is_array_of_pointers(): + # for the case of array of structs, this is for correctness only, + # since in practice all GC arrays of structs are allocated + # with malloc(zero=True) self.handle_clear_array_contents(descr, result, v_length) def handle_new_fixedsize(self, descr, op): @@ -257,8 +281,16 @@ # forgets the previous MALLOC_NURSERY, if any; and empty the # set 'write_barrier_applied', so that future SETFIELDs will generate # a write barrier as usual. + # it also writes down all the pending zero ptr fields self._op_malloc_nursery = None self.write_barrier_applied.clear() + self.emit_pending_zeros() + + def emit_pending_zeros(self): + for v, d in self.delayed_zero_setfields.iteritems(): + for ofs in d.iterkeys(): + op = ResOperation(rop.ZERO_PTR_FIELD, [v, ConstInt(ofs)], None) + self.newops.append(op) def _gen_call_malloc_gc(self, args, v_result, descr): """Generate a CALL_MALLOC_GC with the given args.""" diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py --- a/rpython/jit/backend/llsupport/test/test_rewrite.py +++ b/rpython/jit/backend/llsupport/test/test_rewrite.py @@ -85,7 +85,6 @@ signedframedescr = self.cpu.signedframedescr floatframedescr = self.cpu.floatframedescr casmdescr.compiled_loop_token = clt - tzdescr = None # noone cares # namespace.update(locals()) # @@ -312,9 +311,9 @@ setfield_gc(p0, 1234, descr=tiddescr) p1 = int_add(p0, %(sdescr.size)d) setfield_gc(p1, 5678, descr=tiddescr) - zero_ptr_field(p1, %(tdescr.offsets_of_gcfields[0])s) p2 = int_add(p1, %(tdescr.size)d) setfield_gc(p2, 1234, descr=tiddescr) + zero_ptr_field(p1, %(tdescr.offsets_of_gcfields[0])s) jump() """) @@ -599,6 +598,7 @@ %(cdescr.basesize + 129 * cdescr.itemsize)d) setfield_gc(p1, 8111, descr=tiddescr) setfield_gc(p1, 129, descr=clendescr) + zero_array(p1, 0, 129, descr=cdescr) call(123456) cond_call_gc_wb(p1, descr=wbdescr) setarrayitem_gc(p1, i2, p3, descr=cdescr) @@ -620,6 +620,7 @@ %(cdescr.basesize + 130 * cdescr.itemsize)d) setfield_gc(p1, 8111, descr=tiddescr) setfield_gc(p1, 130, descr=clendescr) + zero_array(p1, 0, 130, descr=cdescr) call(123456) cond_call_gc_wb_array(p1, i2, descr=wbdescr) setarrayitem_gc(p1, i2, p3, descr=cdescr) @@ -651,6 +652,7 @@ %(cdescr.basesize + 5 * cdescr.itemsize)d) setfield_gc(p1, 8111, descr=tiddescr) setfield_gc(p1, 5, descr=clendescr) + zero_array(p1, 0, 5, descr=cdescr) label(p1, i2, p3) cond_call_gc_wb_array(p1, i2, descr=wbdescr) setarrayitem_gc(p1, i2, p3, descr=cdescr) @@ -720,6 +722,7 @@ %(cdescr.basesize + 5 * cdescr.itemsize)d) setfield_gc(p0, 8111, descr=tiddescr) setfield_gc(p0, 5, descr=clendescr) + zero_array(p0, 0, 5, descr=cdescr) setarrayitem_gc(p0, i2, p1, descr=cdescr) jump() """) @@ -753,6 +756,7 @@ [i0] p0 = call_malloc_nursery(%(tdescr.size)d) setfield_gc(p0, 5678, descr=tiddescr) + zero_ptr_field(p0, %(tdescr.offsets_of_gcfields[0])s) p1 = call_malloc_nursery_varsize(1, 1, i0, \ descr=strdescr) setfield_gc(p1, i0, descr=strlendescr) @@ -772,6 +776,7 @@ [p1] p0 = call_malloc_nursery(%(tdescr.size)d) setfield_gc(p0, 5678, descr=tiddescr) + zero_ptr_field(p0, %(tdescr.offsets_of_gcfields[0])s) label(p0, p1) cond_call_gc_wb(p0, descr=wbdescr) setfield_gc(p0, p1, descr=tzdescr) diff --git a/rpython/jit/codewriter/heaptracker.py b/rpython/jit/codewriter/heaptracker.py --- a/rpython/jit/codewriter/heaptracker.py +++ b/rpython/jit/codewriter/heaptracker.py @@ -132,7 +132,8 @@ if res is None: res = [] # order is not relevant, except for tests - for name, FIELD in STRUCT._flds.iteritems(): + for name in STRUCT._names: + FIELD = getattr(STRUCT, name) if isinstance(FIELD, lltype.Ptr) and FIELD._needsgc(): offset, _ = symbolic.get_field_token(STRUCT, name, gccache.translate_support_code) From noreply at buildbot.pypy.org Sun Sep 21 09:07:37 2014 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 21 Sep 2014 09:07:37 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: fix rpython Message-ID: <20140921070737.487511C0FF8@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73623:95eb1ca371e7 Date: 2014-09-21 08:53 +0200 http://bitbucket.org/pypy/pypy/changeset/95eb1ca371e7/ Log: fix rpython diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -137,7 +137,7 @@ d[ofs] = None def consider_setfield_gc(self, op): - offset = op.getdescr().offset + offset = self.cpu.unpack_fielddescr(op.getdescr()) try: del self.delayed_zero_setfields[op.getarg(0)][offset] except KeyError: From noreply at buildbot.pypy.org Sun Sep 21 09:07:38 2014 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 21 Sep 2014 09:07:38 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: more fixes Message-ID: <20140921070738.8701F1C0FF8@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73624:ef9f7e3dc9fa Date: 2014-09-21 09:07 +0200 http://bitbucket.org/pypy/pypy/changeset/ef9f7e3dc9fa/ Log: more fixes diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py --- a/rpython/jit/backend/llsupport/llmodel.py +++ b/rpython/jit/backend/llsupport/llmodel.py @@ -14,6 +14,7 @@ get_call_descr, get_interiorfield_descr, FieldDescr, ArrayDescr, CallDescr, InteriorFieldDescr, FLAG_POINTER, FLAG_FLOAT) +from rpython.jit.backend.llsupport.memcpy import memset_fn from rpython.jit.backend.llsupport.asmmemmgr import AsmMemoryManager from rpython.rlib.unroll import unrolling_iterable @@ -596,8 +597,8 @@ arraysize = self.bh_arraylen_gc(ref, arraydescr) totalsize = size * arraysize adr = rffi.cast(lltype.Signed, ref) + ofs - self.gc_ll_descr.memset_ptr(adr, rffi.cast(rffi.INT, 0), - rffi.cast(rffi.SIZE_T, totalsize)) + memset_fn(rffi.cast(llmemory.Address, adr), rffi.cast(rffi.INT, 0), + rffi.cast(rffi.SIZE_T, totalsize)) def bh_new_with_vtable(self, vtable, sizedescr): res = self.gc_ll_descr.gc_malloc(sizedescr) From noreply at buildbot.pypy.org Sun Sep 21 10:09:40 2014 From: noreply at buildbot.pypy.org (groggi) Date: Sun, 21 Sep 2014 10:09:40 +0200 (CEST) Subject: [pypy-commit] pypy gc-incminimark-pinning: Merge default into gc-incminimark-pinning Message-ID: <20140921080940.21BE51C0130@cobra.cs.uni-duesseldorf.de> Author: Gregor Wegberg Branch: gc-incminimark-pinning Changeset: r73625:55c211d937b2 Date: 2014-09-21 10:08 +0200 http://bitbucket.org/pypy/pypy/changeset/55c211d937b2/ Log: Merge default into gc-incminimark-pinning diff too long, truncating to 2000 out of 5185 lines diff --git a/_pytest/README-BEFORE-UPDATING b/_pytest/README-BEFORE-UPDATING new file mode 100644 --- /dev/null +++ b/_pytest/README-BEFORE-UPDATING @@ -0,0 +1,17 @@ +This is PyPy's code of the pytest lib. We don't expect to upgrade it +very often, but once we do: + + WARNING! + + WE HAVE MADE A FEW TWEAKS HERE! + +Please be sure that you don't just copy the newer version from +upstream without checking the few changes that we did. This +can be done like this: + + cd + hg log . -v | less + +then search for all " _pytest/" in that list to know which are the +relevant checkins. (Look for the checkins that only edit one +or two files in this directory.) diff --git a/_pytest/resultlog.py b/_pytest/resultlog.py --- a/_pytest/resultlog.py +++ b/_pytest/resultlog.py @@ -53,16 +53,24 @@ self.config = config self.logfile = logfile # preferably line buffered - def write_log_entry(self, testpath, lettercode, longrepr): - py.builtin.print_("%s %s" % (lettercode, testpath), file=self.logfile) + def write_log_entry(self, testpath, lettercode, longrepr, sections=None): + _safeprint("%s %s" % (lettercode, testpath), file=self.logfile) for line in longrepr.splitlines(): - py.builtin.print_(" %s" % line, file=self.logfile) + _safeprint(" %s" % line, file=self.logfile) + if sections is not None and ( + lettercode in ('E', 'F')): # to limit the size of logs + for title, content in sections: + _safeprint(" ---------- %s ----------" % (title,), + file=self.logfile) + for line in content.splitlines(): + _safeprint(" %s" % line, file=self.logfile) def log_outcome(self, report, lettercode, longrepr): testpath = getattr(report, 'nodeid', None) if testpath is None: testpath = report.fspath - self.write_log_entry(testpath, lettercode, longrepr) + self.write_log_entry(testpath, lettercode, longrepr, + getattr(report, 'sections', None)) def pytest_runtest_logreport(self, report): if report.when != "call" and report.passed: @@ -98,3 +106,8 @@ if path is None: path = "cwd:%s" % py.path.local() self.write_log_entry(path, '!', str(excrepr)) + +def _safeprint(s, file): + if isinstance(s, unicode): + s = s.encode('utf-8') + py.builtin.print_(s, file=file) diff --git a/lib-python/2.7/test/test_mmap.py b/lib-python/2.7/test/test_mmap.py --- a/lib-python/2.7/test/test_mmap.py +++ b/lib-python/2.7/test/test_mmap.py @@ -179,25 +179,27 @@ import sys f = open(TESTFN, "r+b") try: - m = mmap.mmap(f.fileno(), mapsize+1) - except ValueError: - # we do not expect a ValueError on Windows - # CAUTION: This also changes the size of the file on disk, and - # later tests assume that the length hasn't changed. We need to - # repair that. + try: + m = mmap.mmap(f.fileno(), mapsize+1) + except ValueError: + # we do not expect a ValueError on Windows + # CAUTION: This also changes the size of the file on disk, and + # later tests assume that the length hasn't changed. We need to + # repair that. + if sys.platform.startswith('win'): + self.fail("Opening mmap with size+1 should work on Windows.") + else: + # we expect a ValueError on Unix, but not on Windows + if not sys.platform.startswith('win'): + self.fail("Opening mmap with size+1 should raise ValueError.") + m.close() + finally: + f.close() if sys.platform.startswith('win'): - self.fail("Opening mmap with size+1 should work on Windows.") - else: - # we expect a ValueError on Unix, but not on Windows - if not sys.platform.startswith('win'): - self.fail("Opening mmap with size+1 should raise ValueError.") - m.close() - f.close() - if sys.platform.startswith('win'): - # Repair damage from the resizing test. - f = open(TESTFN, 'r+b') - f.truncate(mapsize) - f.close() + # Repair damage from the resizing test. + f = open(TESTFN, 'r+b') + f.truncate(mapsize) + f.close() # Opening mmap with access=ACCESS_WRITE f = open(TESTFN, "r+b") diff --git a/py/README-BEFORE-UPDATING b/py/README-BEFORE-UPDATING new file mode 100644 --- /dev/null +++ b/py/README-BEFORE-UPDATING @@ -0,0 +1,17 @@ +This is PyPy's code of the py lib. We don't expect to upgrade it +very often, but once we do: + + WARNING! + + WE HAVE MADE A FEW TWEAKS HERE! + +Please be sure that you don't just copy the newer version from +upstream without checking the few changes that we did. This +can be done like this: + + cd + hg log . -v | less + +then search for all " py/" in that list to know which are the +relevant checkins. (Look for the checkins that only edit one +or two files in this directory.) diff --git a/py/_path/local.py b/py/_path/local.py --- a/py/_path/local.py +++ b/py/_path/local.py @@ -750,7 +750,8 @@ mkdtemp = classmethod(mkdtemp) def make_numbered_dir(cls, prefix='session-', rootdir=None, keep=3, - lock_timeout = 172800): # two days + lock_timeout = 172800, # two days + min_timeout = 300): # five minutes """ return unique directory with a number greater than the current maximum one. The number is assumed to start directly after prefix. if keep is true directories with a number less than (maxnum-keep) @@ -818,6 +819,20 @@ for path in rootdir.listdir(): num = parse_num(path) if num is not None and num <= (maxnum - keep): + if min_timeout: + # NB: doing this is needed to prevent (or reduce + # a lot the chance of) the following situation: + # 'keep+1' processes call make_numbered_dir() at + # the same time, they create dirs, but then the + # last process notices the first dir doesn't have + # (yet) a .lock in it and kills it. + try: + t1 = path.lstat().mtime + t2 = lockfile.lstat().mtime + if abs(t2-t1) < min_timeout: + continue # skip directories too recent + except py.error.Error: + continue # failure to get a time, better skip lf = path.join('.lock') try: t1 = lf.lstat().mtime diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -65,9 +65,9 @@ # built documents. # # The short X.Y version. -version = '2.3' +version = '2.4' # The full version, including alpha/beta/rc tags. -release = '2.3.1' +release = '2.4.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pypy/doc/getting-started-python.rst b/pypy/doc/getting-started-python.rst --- a/pypy/doc/getting-started-python.rst +++ b/pypy/doc/getting-started-python.rst @@ -111,6 +111,10 @@ of your choice. Typical example: ``--opt=2`` gives a good (but of course slower) Python interpreter without the JIT. + Consider using PyPy instead of CPython in the above command line, + as it is much faster. (Note that ``rpython`` is a Python 2 program, + not Python 3; you need to run either PyPy 2 or CPython 2.) + .. _`optimization level`: config/opt.html If everything works correctly this will create an executable diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -6,6 +6,7 @@ .. toctree:: + release-2.4.0.rst release-2.3.1.rst release-2.3.0.rst release-2.2.1.rst diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst --- a/pypy/doc/index.rst +++ b/pypy/doc/index.rst @@ -40,7 +40,7 @@ * `FAQ`_: some frequently asked questions. -* `Release 2.3.1`_: the latest official release +* `Release 2.4.0`_: the latest official release * `PyPy Blog`_: news and status info about PyPy @@ -110,7 +110,7 @@ .. _`Getting Started`: getting-started.html .. _`Papers`: extradoc.html .. _`Videos`: video-index.html -.. _`Release 2.3.1`: http://pypy.org/download.html +.. _`Release 2.4.0`: http://pypy.org/download.html .. _`speed.pypy.org`: http://speed.pypy.org .. _`RPython toolchain`: translation.html .. _`potential project ideas`: project-ideas.html diff --git a/pypy/doc/release-2.4.0.rst b/pypy/doc/release-2.4.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-2.4.0.rst @@ -0,0 +1,119 @@ +================================================= +PyPy 2.4 - Snow White +================================================= + +We're pleased to announce PyPy 2.4, which contains significant performance +enhancements and bug fixes. + +You can already download the PyPy 2.4-beta1 pre-release here: + + http://pypy.org/download.html + +We would like to thank our donors for the continued support of the PyPy +project, and for those who donate to our three sub-projects. +We've shown quite a bit of progress, but we're slowly running out of funds. +Please consider donating more, or even better convince your employer to donate, +so we can finish those projects! We would like to also point out that in +September, `the Python Software Foundation`_ will `match funds`_ for +any donations up to $10k! The three sub-projects are: + +* `Py3k`_ (supporting Python 3.x): We have released a Python 3.2.5 compatible version + we call PyPy3 2.3.1, and are working toward a Python 3.3 compatible version + +* `STM`_ (software transactional memory): We have released a first working version, + and continue to try out new promising paths of achieving a fast multithreaded Python + +* `NumPy`_ which requires installation of our fork of upstream numpy, + available `on bitbucket`_ + +.. _`Py3k`: http://pypy.org/py3donate.html +.. _`STM`: http://pypy.org/tmdonate2.html +.. _`NumPy`: http://pypy.org/numpydonate.html +.. _`on bitbucket`: https://www.bitbucket.org/pypy/numpy +.. _`the Python Software Foundation`: https://www.python.org/psf/ +.. _`match funds`: http://morepypy.blogspot.com/2014/09/python-software-foundation-matching.html + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7. It's fast (`pypy 2.4 and cpython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +This release supports **x86** machines on most common operating systems +(Linux 32/64, Mac OS X 64, Windows, and OpenBSD), +as well as newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux. + +While we support 32 bit python on Windows, work on the native Windows 64 +bit python is still stalling, we would welcome a volunteer +to `handle that`_. + +.. _`pypy 2.4 and cpython 2.7.x`: http://speed.pypy.org +.. _`handle that`: http://doc.pypy.org/en/latest/windows.html#what-is-missing-for-a-full-64-bit-translation + +Highlights +========== + +Benchmarks improved after internal enhancements in string and +bytearray handling, and a major rewrite of the GIL handling. This means +that external calls are now a lot faster, especially the CFFI ones. It also +means better performance in a lot of corner cases with handling strings or +bytearrays. The main bugfix is handling of many socket objects in your +program which in the long run used to "leak" memory. + +PyPy now uses Python 2.7.8 standard library. + +We welcomed more than 12 new contributors, and conducted two Google +Summer of Code projects, as well as other student projects not +directly related to Summer of Code. + + +Issues reported with our previous release were fixed after reports from users on +our new issue tracker at https://bitbucket.org/pypy/pypy/issues or on IRC at +#pypy. Here is a summary of the user-facing changes; +for more information see `whats-new`_: + +* Reduced internal copying of bytearray operations + +* Tweak the internal structure of StringBuilder to speed up large string + handling, which becomes advantageous on large programs at the cost of slightly + slower small *benchmark* type programs. + +* Boost performance of thread-local variables in both unjitted and jitted code, + this mostly affects errno handling on linux, which makes external calls + faster. + +* Move to a mixed polling and mutex GIL model that make mutlithreaded jitted + code run *much* faster + +* Optimize errno handling in linux (x86 and x86-64 only) + +* Remove ctypes pythonapi and ctypes.PyDLL, which never worked on PyPy + +* Fix performance regression on ufunc(, ) in numpy + +* Classes in the ast module are now distinct from structures used by + the compiler, which simplifies and speeds up translation of our + source code to the PyPy binary interpreter + +* Upgrade stdlib from 2.7.5 to 2.7.8 + +* Win32 now links statically to zlib, expat, bzip, and openssl-1.0.1i. + No more missing DLLs + +* Many issues were resolved_ since the 2.3.1 release on June 8 + +.. _`whats-new`: http://doc.pypy.org/en/latest/whatsnew-2.3.1.html +.. _resolved: https://bitbucket.org/pypy/pypy/issues?status=resolved + +We have further improvements on the way: rpython file handling, +numpy linalg compatibility, as well +as improved GC and many smaller improvements. + +Please try it out and let us know what you think. We especially welcome +success stories, we know you are using PyPy, please tell us about it! + +Cheers + +The PyPy Team + diff --git a/pypy/doc/release-2.4.rst b/pypy/doc/release-2.4.rst deleted file mode 100644 --- a/pypy/doc/release-2.4.rst +++ /dev/null @@ -1,107 +0,0 @@ -================================================= -PyPy 2.4 - ???????? -================================================= - -We're pleased to announce PyPy 2.4, a significant milestone on it's own right -and the proud parent of our recent PyPy3 and STM releases. - -This release contains several improvements and bugfixes. - -You can download the PyPy 2.4 release here: - - http://pypy.org/download.html - -We would like to thank our donors for the continued support of the PyPy -project, and for those who donate to our three sub-projects. -We've shown quite a bit of progress -but we're slowly running out of funds. -Please consider donating more, or even better convince your employer to donate, -so we can finish those projects! The three sub-projects are: - -* `Py3k`_ (supporting Python 3.x): We have released a Python 3.2.5 compatable version - we call PyPy3 2.3.1, and are working toward a Python 3.3 compatable version - -* `STM`_ (software transactional memory): We have release a first working version, and -continue to try out new promising paths of acheiving a fast multithreaded python - -* `NumPy`_ which requires installation of our fork of upstream numpy, available `on bitbucket`_ - -.. _`Py3k`: http://pypy.org/py3donate.html -.. _`STM`: http://pypy.org/tmdonate2.html -.. _`NumPy`: http://pypy.org/numpydonate.html -.. _`on bitbucket`: https://www.bitbucket.org/pypy/numpy - -What is PyPy? -============= - -PyPy is a very compliant Python interpreter, almost a drop-in replacement for -CPython 2.7. It's fast (`pypy 2.3 and cpython 2.7.x`_ performance comparison; -note that cpython's speed has not changed since 2.7.2) -due to its integrated tracing JIT compiler. - -This release supports x86 machines running Linux 32/64, Mac OS X 64, Windows, -and OpenBSD, -as well as newer ARM hardware (ARMv6 or ARMv7, with VFPv3) running Linux. - -While we support 32 bit python on Windows, work on the native Windows 64 -bit python is still stalling, we would welcome a volunteer -to `handle that`_. - -.. _`pypy 2.3 and cpython 2.7.x`: http://speed.pypy.org -.. _`handle that`: http://doc.pypy.org/en/latest/windows.html#what-is-missing-for-a-full-64-bit-translation - -Highlights -========== - -Benchmarks improved after internal improvements in string and bytearray handling, -and a major rewrite of the GIL handling. Many of these improvements are offshoots -of the STM work. - -We merged in Python's 2.7.8 stdlib in a record time of one week, proving the -maturity of our underlying RPython code base and PyPy interpreter. - -We welcomed more than 12 new contributors, and conducted two Google Summer of Code -projects XXX details? - -Issues reported with our previous release were fixed after reports from users on -our new issue tracker at https://bitbucket.org/pypy/pypy/issues or on IRC at -#pypy. Here is a summary of the user-facing changes; -for more information see `whats-new`_: - -* Reduced internal copying of bytearray operations - -* Tweak the internal structure of StringBuilder to speed up large string -handling, which becomes advantageous on large programs at the cost of slightly -slower small *benchmark* type programs. - -* Boost performance of thread-local variables in both unjitted and jitted code - -* Move to a mixed polling and mutex GIL model that make mutli-threaded jitted - code run *much* faster - -* Optimize errno handling in linux - -* Remove ctypes pythonapi and ctypes.PyDLL, which never worked on PyPy - -* Fix performance regression on ufunc(, ) in numpy - -* Classes in the ast module are now distinct from structures used by the compiler, - which simplifies and speeds up translation of our source code to the PyPy binary - interpreter - -* Upgrade stdlib from 2.7.5 to 2.7.8 - -* - -* Many issues were resolved_ since the 2.3.1 release on June 8 - -.. _`whats-new`: http://doc.pypy.org/en/latest/whatsnew-2.3.1.html -.. _resolved: https://bitbucket.org/pypy/pypy/issues?status=resolved - -Please try it out and let us know what you think. We especially welcome -success stories, we know you are using PyPy, please tell us about it! - -Cheers - -The PyPy Team - diff --git a/pypy/doc/whatsnew-2.4.0.rst b/pypy/doc/whatsnew-2.4.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/whatsnew-2.4.0.rst @@ -0,0 +1,66 @@ +======================= +What's new in PyPy 2.4+ +======================= + +.. this is a revision shortly after release-2.3.x +.. startrev: ca9b7cf02cf4 + +.. branch: fix-bytearray-complexity +Bytearray operations no longer copy the bytearray unnecessarily + +Added support for ``__getitem__``, ``__setitem__``, ``__getslice__``, +``__setslice__``, and ``__len__`` to RPython + +.. branch: stringbuilder2-perf +Give the StringBuilder a more flexible internal structure, with a +chained list of strings instead of just one string. This make it +more efficient when building large strings, e.g. with cStringIO(). + +Also, use systematically jit.conditional_call() instead of regular +branches. This lets the JIT make more linear code, at the cost of +forcing a bit more data (to be passed as arguments to +conditional_calls). I would expect the net result to be a slight +slow-down on some simple benchmarks and a speed-up on bigger +programs. + +.. branch: ec-threadlocal +Change the executioncontext's lookup to be done by reading a thread- +local variable (which is implemented in C using '__thread' if +possible, and pthread_getspecific() otherwise). On Linux x86 and +x86-64, the JIT backend has a special optimization that lets it emit +directly a single MOV from a %gs- or %fs-based address. It seems +actually to give a good boost in performance. + +.. branch: fast-gil +A faster way to handle the GIL, particularly in JIT code. The GIL is +now a composite of two concepts: a global number (it's just set from +1 to 0 and back around CALL_RELEASE_GIL), and a real mutex. If there +are threads waiting to acquire the GIL, one of them is actively +checking the global number every 0.1 ms to 1 ms. Overall, JIT loops +full of external function calls now run a bit faster (if no thread was +started yet), or a *lot* faster (if threads were started already). + +.. branch: jit-get-errno +Optimize the errno handling in the JIT, notably around external +function calls. Linux-only. + +.. branch: disable_pythonapi +Remove non-functioning ctypes.pyhonapi and ctypes.PyDLL, document this +incompatibility with cpython. Recast sys.dllhandle to an int. + +.. branch: scalar-operations +Fix performance regression on ufunc(, ) in numpy. + +.. branch: pytest-25 +Update our copies of py.test and pylib to versions 2.5.2 and 1.4.20, +respectively. + +.. branch: split-ast-classes +Classes in the ast module are now distinct from structures used by the compiler. + +.. branch: stdlib-2.7.8 +Upgrades from 2.7.6 to 2.7.8 + +.. branch: cpybug-seq-radd-rmul +Fix issue #1861 - cpython compatability madness + diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -1,62 +1,8 @@ + ======================= -What's new in PyPy 2.4+ +What's new in PyPy 2.5+ ======================= -.. this is a revision shortly after release-2.3.x -.. startrev: ca9b7cf02cf4 +.. this is a revision shortly after release-2.4.x +.. startrev: 7026746cbb1b -.. branch: fix-bytearray-complexity -Bytearray operations no longer copy the bytearray unnecessarily - -Added support for ``__getitem__``, ``__setitem__``, ``__getslice__``, -``__setslice__``, and ``__len__`` to RPython - -.. branch: stringbuilder2-perf -Give the StringBuilder a more flexible internal structure, with a -chained list of strings instead of just one string. This make it -more efficient when building large strings, e.g. with cStringIO(). - -Also, use systematically jit.conditional_call() instead of regular -branches. This lets the JIT make more linear code, at the cost of -forcing a bit more data (to be passed as arguments to -conditional_calls). I would expect the net result to be a slight -slow-down on some simple benchmarks and a speed-up on bigger -programs. - -.. branch: ec-threadlocal -Change the executioncontext's lookup to be done by reading a thread- -local variable (which is implemented in C using '__thread' if -possible, and pthread_getspecific() otherwise). On Linux x86 and -x86-64, the JIT backend has a special optimization that lets it emit -directly a single MOV from a %gs- or %fs-based address. It seems -actually to give a good boost in performance. - -.. branch: fast-gil -A faster way to handle the GIL, particularly in JIT code. The GIL is -now a composite of two concepts: a global number (it's just set from -1 to 0 and back around CALL_RELEASE_GIL), and a real mutex. If there -are threads waiting to acquire the GIL, one of them is actively -checking the global number every 0.1 ms to 1 ms. Overall, JIT loops -full of external function calls now run a bit faster (if no thread was -started yet), or a *lot* faster (if threads were started already). - -.. branch: jit-get-errno -Optimize the errno handling in the JIT, notably around external -function calls. Linux-only. - -.. branch: disable_pythonapi -Remove non-functioning ctypes.pyhonapi and ctypes.PyDLL, document this -incompatibility with cpython. Recast sys.dllhandle to an int. - -.. branch: scalar-operations -Fix performance regression on ufunc(, ) in numpy. - -.. branch: pytest-25 -Update our copies of py.test and pylib to versions 2.5.2 and 1.4.20, -respectively. - -.. branch: split-ast-classes -Classes in the ast module are now distinct from structures used by the compiler. - -.. branch: stdlib-2.7.8 -Upgrades from 2.7.6 to 2.7.8 diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst --- a/pypy/doc/windows.rst +++ b/pypy/doc/windows.rst @@ -85,10 +85,13 @@ Abridged method (for -Ojit builds using Visual Studio 2008) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Download the versions of all the external packages -from +Download the versions of all the external packages from +https://bitbucket.org/pypy/pypy/downloads/local_2.4.zip +(for 2.4 release and later) or https://bitbucket.org/pypy/pypy/downloads/local.zip -Then expand it into the base directory (base_dir) and modify your environment to reflect this:: +(for pre-2.4 versions) +Then expand it into the base directory (base_dir) and modify your environment +to reflect this:: set PATH=\bin;\tcltk\bin;%PATH% set INCLUDE=\include;\tcltk\include;%INCLUDE% diff --git a/pypy/interpreter/module.py b/pypy/interpreter/module.py --- a/pypy/interpreter/module.py +++ b/pypy/interpreter/module.py @@ -29,6 +29,17 @@ space.w_None) self.startup_called = False + def _cleanup_(self): + """Called by the annotator on prebuilt Module instances. + We don't have many such modules, but for the ones that + show up, remove their __file__ rather than translate it + statically inside the executable.""" + try: + space = self.space + space.delitem(self.w_dict, space.wrap('__file__')) + except OperationError: + pass + def install(self): """NOT_RPYTHON: installs this module into space.builtin_modules""" w_mod = self.space.wrap(self) diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py --- a/pypy/interpreter/pycode.py +++ b/pypy/interpreter/pycode.py @@ -38,18 +38,15 @@ def cpython_code_signature(code): "([list-of-arg-names], vararg-name-or-None, kwarg-name-or-None)." argcount = code.co_argcount + varnames = code.co_varnames assert argcount >= 0 # annotator hint - argnames = list(code.co_varnames[:argcount]) + argnames = list(varnames[:argcount]) if code.co_flags & CO_VARARGS: - varargname = code.co_varnames[argcount] + varargname = varnames[argcount] argcount += 1 else: varargname = None - if code.co_flags & CO_VARKEYWORDS: - kwargname = code.co_varnames[argcount] - argcount += 1 - else: - kwargname = None + kwargname = varnames[argcount] if code.co_flags & CO_VARKEYWORDS else None return Signature(argnames, varargname, kwargname) diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py --- a/pypy/interpreter/pyparser/parsestring.py +++ b/pypy/interpreter/pyparser/parsestring.py @@ -83,12 +83,6 @@ v = PyString_DecodeEscape(space, substr, 'strict', enc) return space.wrap(v) -def hexbyte(val): - result = "%x" % val - if len(result) == 1: - result = "0" + result - return result - def decode_unicode_utf8(space, s, ps, q): # ****The Python 2.7 version, producing UTF-32 escapes**** # String is utf8-encoded, but 'unicode_escape' expects @@ -108,15 +102,14 @@ # instead. lis.append("u005c") if ord(s[ps]) & 0x80: # XXX inefficient - w, ps = decode_utf8(space, s, ps, end, "utf-32-be") - rn = len(w) - assert rn % 4 == 0 - for i in range(0, rn, 4): - lis.append('\\U') - lis.append(hexbyte(ord(w[i]))) - lis.append(hexbyte(ord(w[i+1]))) - lis.append(hexbyte(ord(w[i+2]))) - lis.append(hexbyte(ord(w[i+3]))) + w, ps = decode_utf8(space, s, ps, end) + for c in w: + # The equivalent of %08x, which is not supported by RPython. + # 7 zeroes are enough for the unicode range, and the + # result still fits in 32-bit. + hexa = hex(ord(c) + 0x10000000) + lis.append('\\U0') + lis.append(hexa[3:]) # Skip 0x and the leading 1 else: lis.append(s[ps]) ps += 1 @@ -136,7 +129,7 @@ # note that the C code has a label here. # the logic is the same. if recode_encoding and ord(s[ps]) & 0x80: - w, ps = decode_utf8(space, s, ps, end, recode_encoding) + w, ps = decode_utf8_recode(space, s, ps, end, recode_encoding) # Append bytes to output buffer. builder.append(w) else: @@ -222,14 +215,18 @@ ch >= 'A' and ch <= 'F') -def decode_utf8(space, s, ps, end, encoding): +def decode_utf8(space, s, ps, end): assert ps >= 0 pt = ps # while (s < end && *s != '\\') s++; */ /* inefficient for u".." while ps < end and ord(s[ps]) & 0x80: ps += 1 - w_u = space.wrap(unicodehelper.decode_utf8(space, s[pt:ps])) - w_v = unicodehelper.encode(space, w_u, encoding) + u = unicodehelper.decode_utf8(space, s[pt:ps]) + return u, ps + +def decode_utf8_recode(space, s, ps, end, recode_encoding): + u, ps = decode_utf8(space, s, ps, end) + w_v = unicodehelper.encode(space, space.wrap(u), recode_encoding) v = space.str_w(w_v) return v, ps diff --git a/pypy/interpreter/pyparser/test/test_parsestring.py b/pypy/interpreter/pyparser/test/test_parsestring.py --- a/pypy/interpreter/pyparser/test/test_parsestring.py +++ b/pypy/interpreter/pyparser/test/test_parsestring.py @@ -73,11 +73,11 @@ def test_simple_enc_roundtrip(self): space = self.space - s = "'\x81'" + s = "'\x81\\t'" s = s.decode("koi8-u").encode("utf8") w_ret = parsestring.parsestr(self.space, 'koi8-u', s) ret = space.unwrap(w_ret) - assert ret == eval("# -*- coding: koi8-u -*-\n'\x81'") + assert ret == eval("# -*- coding: koi8-u -*-\n'\x81\\t'") def test_multiline_unicode_strings_with_backslash(self): space = self.space diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py --- a/pypy/interpreter/test/test_app_main.py +++ b/pypy/interpreter/test/test_app_main.py @@ -945,7 +945,7 @@ prefix = udir.join('pathtest').ensure(dir=1) fake_exe = 'bin/pypy-c' if sys.platform == 'win32': - fake_exe += '.exe' + fake_exe = 'pypy-c.exe' fake_exe = prefix.join(fake_exe).ensure(file=1) expected_path = [str(prefix.join(subdir).ensure(dir=1)) for subdir in ('lib_pypy', @@ -985,6 +985,13 @@ assert sys.path == old_sys_path + [self.goal_dir] app_main.setup_bootstrap_path(self.fake_exe) + if not sys.platform == 'win32': + # an existing file is always 'executable' on windows + assert sys.executable == '' # not executable! + assert sys.path == old_sys_path + [self.goal_dir] + + os.chmod(self.fake_exe, 0755) + app_main.setup_bootstrap_path(self.fake_exe) assert sys.executable == self.fake_exe assert self.goal_dir not in sys.path diff --git a/pypy/interpreter/test/test_module.py b/pypy/interpreter/test/test_module.py --- a/pypy/interpreter/test/test_module.py +++ b/pypy/interpreter/test/test_module.py @@ -1,4 +1,5 @@ - +import py +from pypy.interpreter.error import OperationError from pypy.interpreter.module import Module class TestModule: @@ -17,6 +18,18 @@ space.raises_w(space.w_AttributeError, space.delattr, w_m, w('x')) + def test___file__(self, space): + w = space.wrap + m = Module(space, space.wrap('m')) + py.test.raises(OperationError, space.getattr, w(m), w('__file__')) + m._cleanup_() + py.test.raises(OperationError, space.getattr, w(m), w('__file__')) + space.setattr(w(m), w('__file__'), w('m.py')) + space.getattr(w(m), w('__file__')) # does not raise + m._cleanup_() + py.test.raises(OperationError, space.getattr, w(m), w('__file__')) + + class AppTest_ModuleObject: def test_attr(self): m = __import__('__builtin__') diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py --- a/pypy/interpreter/unicodehelper.py +++ b/pypy/interpreter/unicodehelper.py @@ -5,6 +5,7 @@ @specialize.memo() def decode_error_handler(space): + # Fast version of the "strict" errors handler. def raise_unicode_exception_decode(errors, encoding, msg, s, startingpos, endingpos): raise OperationError(space.w_UnicodeDecodeError, @@ -17,6 +18,7 @@ @specialize.memo() def encode_error_handler(space): + # Fast version of the "strict" errors handler. def raise_unicode_exception_encode(errors, encoding, msg, u, startingpos, endingpos): raise OperationError(space.w_UnicodeEncodeError, diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py --- a/pypy/module/_cffi_backend/ctypefunc.py +++ b/pypy/module/_cffi_backend/ctypefunc.py @@ -8,6 +8,7 @@ from rpython.rlib.jit_libffi import (CIF_DESCRIPTION, CIF_DESCRIPTION_P, FFI_TYPE, FFI_TYPE_P, FFI_TYPE_PP, SIZE_OF_FFI_ARG) from rpython.rlib.objectmodel import we_are_translated, instantiate +from rpython.rlib.objectmodel import keepalive_until_here from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from pypy.interpreter.error import OperationError, oefmt @@ -160,6 +161,7 @@ raw_cdata = rffi.cast(rffi.CCHARPP, data)[0] lltype.free(raw_cdata, flavor='raw') lltype.free(buffer, flavor='raw') + keepalive_until_here(args_w) return w_res def get_mustfree_flag(data): diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py --- a/pypy/module/_cffi_backend/ctypeprim.py +++ b/pypy/module/_cffi_backend/ctypeprim.py @@ -182,8 +182,12 @@ value = misc.read_raw_long_data(cdata, self.size) return self.space.wrap(value) else: - value = misc.read_raw_signed_data(cdata, self.size) - return self.space.wrap(value) # r_longlong => on 32-bit, 'long' + return self._convert_to_object_longlong(cdata) + + def _convert_to_object_longlong(self, cdata): + # in its own function: LONGLONG may make the whole function jit-opaque + value = misc.read_raw_signed_data(cdata, self.size) + return self.space.wrap(value) # r_longlong => on 32-bit, 'long' def convert_from_object(self, cdata, w_ob): if self.value_fits_long: @@ -193,8 +197,12 @@ self._overflow(w_ob) misc.write_raw_signed_data(cdata, value, self.size) else: - value = misc.as_long_long(self.space, w_ob) - misc.write_raw_signed_data(cdata, value, self.size) + self._convert_from_object_longlong(cdata, w_ob) + + def _convert_from_object_longlong(self, cdata, w_ob): + # in its own function: LONGLONG may make the whole function jit-opaque + value = misc.as_long_long(self.space, w_ob) + misc.write_raw_signed_data(cdata, value, self.size) def get_vararg_type(self): if self.size < rffi.sizeof(rffi.INT): @@ -264,8 +272,12 @@ self._overflow(w_ob) misc.write_raw_unsigned_data(cdata, value, self.size) else: - value = misc.as_unsigned_long_long(self.space, w_ob, strict=True) - misc.write_raw_unsigned_data(cdata, value, self.size) + self._convert_from_object_longlong(cdata, w_ob) + + def _convert_from_object_longlong(self, cdata, w_ob): + # in its own function: LONGLONG may make the whole function jit-opaque + value = misc.as_unsigned_long_long(self.space, w_ob, strict=True) + misc.write_raw_unsigned_data(cdata, value, self.size) def convert_to_object(self, cdata): if self.value_fits_ulong: @@ -275,8 +287,12 @@ else: return self.space.wrap(value) # r_uint => 'long' object else: - value = misc.read_raw_unsigned_data(cdata, self.size) - return self.space.wrap(value) # r_ulonglong => 'long' object + return self._convert_to_object_longlong(cdata) + + def _convert_to_object_longlong(self, cdata): + # in its own function: LONGLONG may make the whole function jit-opaque + value = misc.read_raw_unsigned_data(cdata, self.size) + return self.space.wrap(value) # r_ulonglong => 'long' object def get_vararg_type(self): if self.size < rffi.sizeof(rffi.INT): diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py --- a/pypy/module/_cffi_backend/ctypestruct.py +++ b/pypy/module/_cffi_backend/ctypestruct.py @@ -17,7 +17,7 @@ class W_CTypeStructOrUnion(W_CType): - _immutable_fields_ = ['alignment?', 'fields_list?', 'fields_dict?', + _immutable_fields_ = ['alignment?', 'fields_list?[*]', 'fields_dict?', 'custom_field_pos?', 'with_var_array?'] # fields added by complete_struct_or_union(): alignment = -1 diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py --- a/pypy/module/_cffi_backend/newtype.py +++ b/pypy/module/_cffi_backend/newtype.py @@ -389,7 +389,7 @@ w_ctype.size = totalsize w_ctype.alignment = totalalignment - w_ctype.fields_list = fields_list + w_ctype.fields_list = fields_list[:] w_ctype.fields_dict = fields_dict w_ctype.custom_field_pos = custom_field_pos w_ctype.with_var_array = with_var_array diff --git a/pypy/module/_io/interp_stringio.py b/pypy/module/_io/interp_stringio.py --- a/pypy/module/_io/interp_stringio.py +++ b/pypy/module/_io/interp_stringio.py @@ -86,7 +86,7 @@ initval = space.unicode_w(w_initval) size = len(initval) self.resize_buffer(size) - self.buf = [c for c in initval] + self.buf = list(initval) pos = space.getindex_w(w_pos, space.w_TypeError) if pos < 0: raise OperationError(space.w_ValueError, diff --git a/pypy/module/_pypyjson/interp_encoder.py b/pypy/module/_pypyjson/interp_encoder.py --- a/pypy/module/_pypyjson/interp_encoder.py +++ b/pypy/module/_pypyjson/interp_encoder.py @@ -37,16 +37,14 @@ sb = StringBuilder(len(u)) sb.append_slice(s, 0, first) else: + # We used to check if 'u' contains only safe characters, and return + # 'w_string' directly. But this requires an extra pass over all + # characters, and the expected use case of this function, from + # json.encoder, will anyway re-encode a unicode result back to + # a string (with the ascii encoding). This requires two passes + # over the characters. So we may as well directly turn it into a + # string here --- only one pass. u = space.unicode_w(w_string) - for i in range(len(u)): - c = u[i] - if c >= u' ' and c <= u'~' and c != u'"' and c != u'\\': - pass - else: - break - else: - # the input is a unicode with only non-special ascii chars - return w_string sb = StringBuilder(len(u)) first = 0 diff --git a/pypy/module/_pypyjson/test/test__pypyjson.py b/pypy/module/_pypyjson/test/test__pypyjson.py --- a/pypy/module/_pypyjson/test/test__pypyjson.py +++ b/pypy/module/_pypyjson/test/test__pypyjson.py @@ -192,14 +192,14 @@ def test_raw_encode_basestring_ascii(self): import _pypyjson - def check(s, expected_type=str): + def check(s): s = _pypyjson.raw_encode_basestring_ascii(s) - assert type(s) is expected_type + assert type(s) is str return s assert check("") == "" - assert check(u"", expected_type=unicode) == u"" + assert check(u"") == "" assert check("abc ") == "abc " - assert check(u"abc ", expected_type=unicode) == u"abc " + assert check(u"abc ") == "abc " raises(UnicodeDecodeError, check, "\xc0") assert check("\xc2\x84") == "\\u0084" assert check("\xf0\x92\x8d\x85") == "\\ud808\\udf45" diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py --- a/pypy/module/_ssl/interp_ssl.py +++ b/pypy/module/_ssl/interp_ssl.py @@ -759,17 +759,25 @@ # socket's timeout is in seconds, poll's timeout in ms timeout = int(sock_timeout * 1000 + 0.5) - ready = rpoll.poll(fddict, timeout) + try: + ready = rpoll.poll(fddict, timeout) + except rpoll.PollError, e: + message = e.get_msg() + raise ssl_error(space, message, e.errno) else: if MAX_FD_SIZE is not None and sock_fd >= MAX_FD_SIZE: return SOCKET_TOO_LARGE_FOR_SELECT - if writing: - r, w, e = rpoll.select([], [sock_fd], [], sock_timeout) - ready = w - else: - r, w, e = rpoll.select([sock_fd], [], [], sock_timeout) - ready = r + try: + if writing: + r, w, e = rpoll.select([], [sock_fd], [], sock_timeout) + ready = w + else: + r, w, e = rpoll.select([sock_fd], [], [], sock_timeout) + ready = r + except rpoll.SelectError as e: + message = e.get_msg() + raise ssl_error(space, message, e.errno) if ready: return SOCKET_OPERATION_OK else: diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h --- a/pypy/module/cpyext/include/patchlevel.h +++ b/pypy/module/cpyext/include/patchlevel.h @@ -29,7 +29,7 @@ #define PY_VERSION "2.7.8" /* PyPy version as a string */ -#define PYPY_VERSION "2.3.1" +#define PYPY_VERSION "2.5.0-alpha0" /* Subversion Revision number of this file (not of the repository). * Empty since Mercurial migration. */ diff --git a/pypy/module/operator/__init__.py b/pypy/module/operator/__init__.py --- a/pypy/module/operator/__init__.py +++ b/pypy/module/operator/__init__.py @@ -39,7 +39,7 @@ 'irshift', 'isub', 'itruediv', 'ixor', '_length_hint'] interpleveldefs = { - '_compare_digest': 'interp_operator.compare_digest', + '_compare_digest': 'tscmp.compare_digest', } for name in interp_names: diff --git a/pypy/module/operator/app_operator.py b/pypy/module/operator/app_operator.py --- a/pypy/module/operator/app_operator.py +++ b/pypy/module/operator/app_operator.py @@ -4,7 +4,7 @@ This module exports a set of operators as functions. E.g. operator.add(x,y) is equivalent to x+y. ''' -from __pypy__ import builtinify + import types @@ -27,7 +27,7 @@ 'getslice(a, b, c) -- Same as a[b:c].' if not isinstance(start, int) or not isinstance(end, int): raise TypeError("an integer is expected") - return a[start:end] + return a[start:end] __getslice__ = getslice def indexOf(a, b): @@ -37,7 +37,7 @@ if x == b: return index index += 1 - raise ValueError, 'sequence.index(x): x not in sequence' + raise ValueError('sequence.index(x): x not in sequence') def isMappingType(obj,): 'isMappingType(a) -- Return True if a has a mapping type, False otherwise.' @@ -58,9 +58,9 @@ def repeat(obj, num): 'repeat(a, b) -- Return a * b, where a is a sequence, and b is an integer.' if not isinstance(num, (int, long)): - raise TypeError, 'an integer is required' + raise TypeError('an integer is required') if not isSequenceType(obj): - raise TypeError, "non-sequence object can't be repeated" + raise TypeError("non-sequence object can't be repeated") return obj * num @@ -68,59 +68,85 @@ def setslice(a, b, c, d): 'setslice(a, b, c, d) -- Same as a[b:c] = d.' - a[b:c] = d + a[b:c] = d __setslice__ = setslice +def _resolve_attr_chain(chain, obj, idx=0): + obj = getattr(obj, chain[idx]) + if idx + 1 == len(chain): + return obj + else: + return _resolve_attr_chain(chain, obj, idx + 1) + + +class _simple_attrgetter(object): + def __init__(self, attr): + self._attr = attr + + def __call__(self, obj): + return getattr(obj, self._attr) + + +class _single_attrgetter(object): + def __init__(self, attrs): + self._attrs = attrs + + def __call__(self, obj): + return _resolve_attr_chain(self._attrs, obj) + + +class _multi_attrgetter(object): + def __init__(self, attrs): + self._attrs = attrs + + def __call__(self, obj): + return tuple([ + _resolve_attr_chain(attrs, obj) + for attrs in self._attrs + ]) + + def attrgetter(attr, *attrs): + if ( + not isinstance(attr, basestring) or + not all(isinstance(a, basestring) for a in attrs) + ): + def _raise_typeerror(obj): + raise TypeError( + "argument must be a string, not %r" % type(attr).__name__ + ) + return _raise_typeerror if attrs: - getters = [single_attr_getter(a) for a in (attr,) + attrs] - def getter(obj): - return tuple([getter(obj) for getter in getters]) + return _multi_attrgetter([ + a.split(".") for a in [attr] + list(attrs) + ]) + elif "." not in attr: + return _simple_attrgetter(attr) else: - getter = single_attr_getter(attr) - return builtinify(getter) + return _single_attrgetter(attr.split(".")) -def single_attr_getter(attr): - if not isinstance(attr, str): - if not isinstance(attr, unicode): - def _raise_typeerror(obj): - raise TypeError("argument must be a string, not %r" % - (type(attr).__name__,)) - return _raise_typeerror - attr = attr.encode('ascii') - # - def make_getter(name, prevfn=None): - if prevfn is None: - def getter(obj): - return getattr(obj, name) + +class itemgetter(object): + def __init__(self, item, *items): + self._single = not bool(items) + if self._single: + self._idx = item else: - def getter(obj): - return getattr(prevfn(obj), name) - return getter - # - last = 0 - getter = None - while True: - dot = attr.find(".", last) - if dot < 0: break - getter = make_getter(attr[last:dot], getter) - last = dot + 1 - return make_getter(attr[last:], getter) + self._idx = [item] + list(items) + def __call__(self, obj): + if self._single: + return obj[self._idx] + else: + return tuple([obj[i] for i in self._idx]) -def itemgetter(item, *items): - if items: - list_of_indices = [item] + list(items) - def getter(obj): - return tuple([obj[i] for i in list_of_indices]) - else: - def getter(obj): - return obj[item] - return builtinify(getter) +class methodcaller(object): + def __init__(self, method_name, *args, **kwargs): + self._method_name = method_name + self._args = args + self._kwargs = kwargs -def methodcaller(method_name, *args, **kwargs): - def call(obj): - return getattr(obj, method_name)(*args, **kwargs) - return builtinify(call) + def __call__(self, obj): + return getattr(obj, self._method_name)(*self._args, **self._kwargs) diff --git a/pypy/module/operator/interp_operator.py b/pypy/module/operator/interp_operator.py --- a/pypy/module/operator/interp_operator.py +++ b/pypy/module/operator/interp_operator.py @@ -1,6 +1,4 @@ -from rpython.rlib.objectmodel import specialize - -from pypy.interpreter.error import OperationError, oefmt +from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import unwrap_spec @@ -249,33 +247,3 @@ @unwrap_spec(default=int) def _length_hint(space, w_iterable, default): return space.wrap(space.length_hint(w_iterable, default)) - -def compare_digest(space, w_a, w_b): - if ( - space.isinstance_w(w_a, space.w_unicode) and - space.isinstance_w(w_b, space.w_unicode) - ): - return space.wrap(tscmp(space.unicode_w(w_a), space.unicode_w(w_b))) - if ( - space.isinstance_w(w_a, space.w_unicode) or - space.isinstance_w(w_b, space.w_unicode) - ): - raise oefmt( - space.w_TypeError, - "unsupported operand types(s) or combination of types: '%N' and '%N'", - w_a, - w_b, - ) - else: - return space.wrap(tscmp(space.bufferstr_w(w_a), space.bufferstr_w(w_b))) - - - at specialize.argtype(0, 1) -def tscmp(a, b): - len_a = len(a) - len_b = len(b) - length = min(len(a), len(b)) - res = len_a ^ len_b - for i in xrange(length): - res |= ord(a[i]) ^ ord(b[i]) - return res == 0 diff --git a/pypy/module/operator/test/test_operator.py b/pypy/module/operator/test/test_operator.py --- a/pypy/module/operator/test/test_operator.py +++ b/pypy/module/operator/test/test_operator.py @@ -334,3 +334,9 @@ assert operator._compare_digest(a, b) a, b = mybytes(b"foobar"), mybytes(b"foobaz") assert not operator._compare_digest(a, b) + + def test_compare_digest_unicode(self): + import operator + assert operator._compare_digest(u'asd', u'asd') + assert not operator._compare_digest(u'asd', u'qwe') + raises(TypeError, operator._compare_digest, u'asd', b'qwe') diff --git a/pypy/module/operator/test/test_tscmp.py b/pypy/module/operator/test/test_tscmp.py new file mode 100644 --- /dev/null +++ b/pypy/module/operator/test/test_tscmp.py @@ -0,0 +1,28 @@ +from pypy.module.operator.tscmp import pypy_tscmp, pypy_tscmp_wide + +class TestTimingSafeCompare: + tostr = str + tscmp = staticmethod(pypy_tscmp) + + def test_tscmp_neq(self): + assert not self.tscmp(self.tostr('asd'), self.tostr('qwe'), 3, 3) + + def test_tscmp_eq(self): + assert self.tscmp(self.tostr('asd'), self.tostr('asd'), 3, 3) + + def test_tscmp_len(self): + assert self.tscmp(self.tostr('asdp'), self.tostr('asdq'), 3, 3) + + def test_tscmp_nlen(self): + assert not self.tscmp(self.tostr('asd'), self.tostr('asd'), 2, 3) + + +class TestTimingSafeCompareWide(TestTimingSafeCompare): + tostr = unicode + tscmp = staticmethod(pypy_tscmp_wide) + + def test_tscmp_wide_nonascii(self): + a, b = u"\ud808\udf45", u"\ud808\udf45" + assert self.tscmp(a, b, len(a), len(b)) + a, b = u"\ud808\udf45", u"\ud808\udf45 " + assert not self.tscmp(a, b, len(a), len(b)) diff --git a/pypy/module/operator/tscmp.c b/pypy/module/operator/tscmp.c new file mode 100644 --- /dev/null +++ b/pypy/module/operator/tscmp.c @@ -0,0 +1,80 @@ +/* Derived from CPython 3.3.5's operator.c::_tscmp + */ + +#include +#include +#include "tscmp.h" + +int +pypy_tscmp(const char *a, const char *b, long len_a, long len_b) +{ + /* The volatile type declarations make sure that the compiler has no + * chance to optimize and fold the code in any way that may change + * the timing. + */ + volatile long length; + volatile const char *left; + volatile const char *right; + long i; + char result; + + /* loop count depends on length of b */ + length = len_b; + left = NULL; + right = b; + + /* don't use else here to keep the amount of CPU instructions constant, + * volatile forces re-evaluation + * */ + if (len_a == length) { + left = *((volatile const char**)&a); + result = 0; + } + if (len_a != length) { + left = b; + result = 1; + } + + for (i=0; i < length; i++) { + result |= *left++ ^ *right++; + } + + return (result == 0); +} + +int +pypy_tscmp_wide(const wchar_t *a, const wchar_t *b, long len_a, long len_b) +{ + /* The volatile type declarations make sure that the compiler has no + * chance to optimize and fold the code in any way that may change + * the timing. + */ + volatile long length; + volatile const wchar_t *left; + volatile const wchar_t *right; + long i; + wchar_t result; + + /* loop count depends on length of b */ + length = len_b; + left = NULL; + right = b; + + /* don't use else here to keep the amount of CPU instructions constant, + * volatile forces re-evaluation + * */ + if (len_a == length) { + left = *((volatile const wchar_t**)&a); + result = 0; + } + if (len_a != length) { + left = b; + result = 1; + } + + for (i=0; i < length; i++) { + result |= *left++ ^ *right++; + } + + return (result == 0); +} diff --git a/pypy/module/operator/tscmp.h b/pypy/module/operator/tscmp.h new file mode 100644 --- /dev/null +++ b/pypy/module/operator/tscmp.h @@ -0,0 +1,2 @@ +int pypy_tscmp(const char *, const char *, long, long); +int pypy_tscmp_wide(const wchar_t *, const wchar_t *, long, long); diff --git a/pypy/module/operator/tscmp.py b/pypy/module/operator/tscmp.py new file mode 100644 --- /dev/null +++ b/pypy/module/operator/tscmp.py @@ -0,0 +1,73 @@ +""" +Provides _compare_digest method, which is a safe comparing to prevent timing +attacks for the hmac module. +""" +import py + +from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.translator.tool.cbuild import ExternalCompilationInfo + +from pypy.interpreter.error import oefmt + +cwd = py.path.local(__file__).dirpath() +eci = ExternalCompilationInfo( + includes=[cwd.join('tscmp.h')], + include_dirs=[str(cwd)], + separate_module_files=[cwd.join('tscmp.c')], + export_symbols=['pypy_tscmp', 'pypy_tscmp_wide']) + + +def llexternal(*args, **kwargs): + kwargs.setdefault('compilation_info', eci) + kwargs.setdefault('sandboxsafe', True) + return rffi.llexternal(*args, **kwargs) + + +pypy_tscmp = llexternal( + 'pypy_tscmp', + [rffi.CCHARP, rffi.CCHARP, rffi.LONG, rffi.LONG], + rffi.INT) +pypy_tscmp_wide = llexternal( + 'pypy_tscmp_wide', + [rffi.CWCHARP, rffi.CWCHARP, rffi.LONG, rffi.LONG], + rffi.INT) + + +def compare_digest(space, w_a, w_b): + """compare_digest(a, b) -> bool + + Return 'a == b'. This function uses an approach designed to prevent + timing analysis, making it appropriate for cryptography. a and b + must both be of the same type: either str (ASCII only), or any type + that supports the buffer protocol (e.g. bytes). + + Note: If a and b are of different lengths, or if an error occurs, a + timing attack could theoretically reveal information about the types + and lengths of a and b--but not their values. + """ + if (space.isinstance_w(w_a, space.w_unicode) and + space.isinstance_w(w_b, space.w_unicode)): + a = space.unicode_w(w_a) + b = space.unicode_w(w_b) + with rffi.scoped_nonmoving_unicodebuffer(a) as a_buf: + with rffi.scoped_nonmoving_unicodebuffer(b) as b_buf: + result = pypy_tscmp_wide(a_buf, b_buf, len(a), len(b)) + return space.wrap(rffi.cast(lltype.Bool, result)) + return compare_digest_buffer(space, w_a, w_b) + + +def compare_digest_buffer(space, w_a, w_b): + try: + a_buf = w_a.buffer_w(space, space.BUF_SIMPLE) + b_buf = w_b.buffer_w(space, space.BUF_SIMPLE) + except TypeError: + raise oefmt(space.w_TypeError, + "unsupported operand types(s) or combination of types: " + "'%T' and '%T'", w_a, w_b) + + a = a_buf.as_str() + b = b_buf.as_str() + with rffi.scoped_nonmovingbuffer(a) as a_buf: + with rffi.scoped_nonmovingbuffer(b) as b_buf: + result = pypy_tscmp(a_buf, b_buf, len(a), len(b)) + return space.wrap(rffi.cast(lltype.Bool, result)) diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py --- a/pypy/module/pypyjit/test_pypy_c/test_call.py +++ b/pypy/module/pypyjit/test_pypy_c/test_call.py @@ -17,13 +17,18 @@ # now we can inline it as call assembler i = 0 j = 0 - while i < 20: + while i < 25: i += 1 j += rec(100) # ID: call_rec return j # - log = self.run(fn, [], threshold=18) - loop, = log.loops_by_filename(self.filepath) + # NB. the parameters below are a bit ad-hoc. After 16 iterations, + # the we trace from the "while" and reach a "trace too long". Then + # in the next execution, we trace the "rec" function from start; + # that's "functrace" below. Then after one or two extra iterations + # we try again from "while", and this time we succeed. + log = self.run(fn, [], threshold=20) + functrace, loop = log.loops_by_filename(self.filepath) assert loop.match_by_id('call_rec', """ ... p53 = call_assembler(..., descr=...) diff --git a/pypy/module/pypyjit/test_pypy_c/test_cprofile.py b/pypy/module/pypyjit/test_pypy_c/test_cprofile.py --- a/pypy/module/pypyjit/test_pypy_c/test_cprofile.py +++ b/pypy/module/pypyjit/test_pypy_c/test_cprofile.py @@ -1,4 +1,4 @@ -import py, sys +import py, sys, re from pypy.module.pypyjit.test_pypy_c.test_00_model import BaseTestPyPyC class TestCProfile(BaseTestPyPyC): @@ -26,10 +26,20 @@ for method in ['append', 'pop']: loop, = log.loops_by_id(method) print loop.ops_by_id(method) - # on 32-bit, there is f1=read_timestamp(); ...; - # f2=read_timestamp(); f3=call(llong_sub,f1,f2) - # which should turn into a single PADDQ/PSUBQ - if sys.maxint != 2147483647: - assert ' call(' not in repr(loop.ops_by_id(method)) + # on 32-bit, there is f1=call(read_timestamp); ...; + # f2=call(read_timestamp); f3=call(llong_sub,f1,f2) + # but all calls can be special-cased by the backend if + # supported. On 64-bit there is only the two calls to + # read_timestamp. + r = re.compile(r" call[(]ConstClass[(](.+?)[)]") + calls = r.findall(repr(loop.ops_by_id(method))) + if sys.maxint == 2147483647: + assert len(calls) == 6 + else: + assert len(calls) == 2 + for x in calls: + assert ('ll_read_timestamp' in x or 'llong_sub' in x + or 'llong_add' in x) + # assert ' call_may_force(' not in repr(loop.ops_by_id(method)) assert ' cond_call(' in repr(loop.ops_by_id(method)) diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py --- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py +++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py @@ -340,30 +340,19 @@ guard_value(p166, ConstPtr(ptr72), descr=...) p167 = call(ConstClass(_ll_0_alloc_with_del___), descr=) guard_no_exception(descr=...) - i168 = call(ConstClass(_ll_1_raw_malloc_varsize__Signed), 6, descr=) - i169 = int_add(i168, i97) - i170 = int_sub(i160, i106) - setfield_gc(p167, i168, descr=) + i112 = int_sub(i160, -32768) setfield_gc(p167, ConstPtr(null), descr=) - setfield_gc(p167, ConstPtr(ptr89), descr=) - i171 = uint_gt(i170, i108) - guard_false(i171, descr=...) - i172 = int_sub(i160, -32768) - i173 = int_and(i172, 65535) - i174 = int_add(i173, -32768) - setarrayitem_raw(i169, 0, i174, descr=) - i175 = int_add(i168, i121) - i176 = int_sub(i160, i130) - i177 = uint_gt(i176, i132) - guard_false(i177, descr=...) - setarrayitem_raw(i175, 0, i174, descr=) - i178 = int_add(i168, i140) - i179 = int_sub(i160, i149) - i180 = uint_gt(i179, i151) - guard_false(i180, descr=...) - setarrayitem_raw(i178, 0, i174, descr=) + setfield_gc(p167, ConstPtr(ptr85), descr=) + i114 = uint_gt(i112, 65535) + guard_false(i114, descr=...) + i115 = int_and(i112, 65535) + i116 = int_add(i115, -32768) --TICK-- - i183 = arraylen_gc(p67, descr=) - i184 = arraylen_gc(p92, descr=) + i119 = call(ConstClass(_ll_1_raw_malloc_varsize__Signed), 6, descr=) + raw_store(i119, 0, i116, descr=) + raw_store(i119, 2, i116, descr=) + raw_store(i119, 4, i116, descr=) + setfield_gc(p167, i119, descr=) + i123 = arraylen_gc(p67, descr=) jump(..., descr=...) """) diff --git a/pypy/module/sys/initpath.py b/pypy/module/sys/initpath.py --- a/pypy/module/sys/initpath.py +++ b/pypy/module/sys/initpath.py @@ -18,6 +18,13 @@ _WIN32 = sys.platform == 'win32' +def _exists_and_is_executable(fn): + # os.access checks using the user's real uid and gid. + # Since pypy should not be run setuid/setgid, this + # should be sufficient. + return os.path.isfile(fn) and os.access(fn, os.X_OK) + + def find_executable(executable): """ Return the absolute path of the executable, by looking into PATH and @@ -34,14 +41,14 @@ if path: for dir in path.split(os.pathsep): fn = os.path.join(dir, executable) - if os.path.isfile(fn): + if _exists_and_is_executable(fn): executable = fn break executable = rpath.rabspath(executable) # 'sys.executable' should not end up being an non-existing file; # just use '' in this case. (CPython issue #7774) - return executable if os.path.isfile(executable) else '' + return executable if _exists_and_is_executable(executable) else '' def _readlink_maybe(filename): diff --git a/pypy/module/sys/state.py b/pypy/module/sys/state.py --- a/pypy/module/sys/state.py +++ b/pypy/module/sys/state.py @@ -7,15 +7,15 @@ # ____________________________________________________________ # -class State: - def __init__(self, space): - self.space = space +class State: + def __init__(self, space): + self.space = space self.w_modules = space.newdict(module=True) - self.w_warnoptions = space.newlist([]) self.w_argv = space.newlist([]) - self.setinitialpath(space) + + self.setinitialpath(space) def setinitialpath(self, space): from pypy.module.sys.initpath import compute_stdlib_path @@ -25,10 +25,10 @@ path = compute_stdlib_path(self, srcdir) self.w_path = space.newlist([space.wrap(p) for p in path]) - def get(space): return space.fromcache(State) + class IOState: def __init__(self, space): from pypy.module._file.interp_file import W_File @@ -36,17 +36,17 @@ stdin = W_File(space) stdin.file_fdopen(0, "r", 1) - stdin.name = '' + stdin.w_name = space.wrap('') self.w_stdin = space.wrap(stdin) stdout = W_File(space) stdout.file_fdopen(1, "w", 1) - stdout.name = '' + stdout.w_name = space.wrap('') self.w_stdout = space.wrap(stdout) stderr = W_File(space) stderr.file_fdopen(2, "w", 0) - stderr.name = '' + stderr.w_name = space.wrap('') self.w_stderr = space.wrap(stderr) stdin._when_reading_first_flush(stdout) @@ -54,9 +54,9 @@ def getio(space): return space.fromcache(IOState) + def pypy_getudir(space): """NOT_RPYTHON (should be removed from interpleveldefs before translation)""" from rpython.tool.udir import udir return space.wrap(str(udir)) - diff --git a/pypy/module/sys/test/test_initpath.py b/pypy/module/sys/test/test_initpath.py --- a/pypy/module/sys/test/test_initpath.py +++ b/pypy/module/sys/test/test_initpath.py @@ -57,6 +57,7 @@ a.join('pypy').ensure(file=True) b.join('pypy').ensure(file=True) # + monkeypatch.setattr(os, 'access', lambda x, y: True) # if there is already a slash, don't do anything monkeypatch.chdir(tmpdir) assert find_executable('a/pypy') == a.join('pypy') @@ -82,7 +83,11 @@ # if pypy is found but it's not a file, ignore it c.join('pypy').ensure(dir=True) assert find_executable('pypy') == a.join('pypy') + # if pypy is found but it's not executable, ignore it + monkeypatch.setattr(os, 'access', lambda x, y: False) + assert find_executable('pypy') == '' # + monkeypatch.setattr(os, 'access', lambda x, y: True) monkeypatch.setattr(initpath, 'we_are_translated', lambda: True) monkeypatch.setattr(initpath, '_WIN32', True) monkeypatch.setenv('PATH', str(a)) diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py --- a/pypy/module/sys/test/test_sysmodule.py +++ b/pypy/module/sys/test/test_sysmodule.py @@ -91,6 +91,10 @@ assert isinstance(sys.__stderr__, file) assert isinstance(sys.__stdin__, file) + #assert sys.__stdin__.name == "" + #assert sys.__stdout__.name == "" + #assert sys.__stderr__.name == "" + if self.appdirect and not isinstance(sys.stdin, file): return diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -10,7 +10,7 @@ #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py CPYTHON_API_VERSION = 1013 #XXX # sync with include/modsupport.h -PYPY_VERSION = (2, 3, 1, "final", 0) #XXX # sync patchlevel.h +PYPY_VERSION = (2, 5, 0, "alpha", 0) #XXX # sync patchlevel.h if platform.name == 'msvc': COMPILER_INFO = 'MSC v.%d 32 bit' % (platform.version * 10 + 600) diff --git a/pypy/module/thread/test/test_thread.py b/pypy/module/thread/test/test_thread.py --- a/pypy/module/thread/test/test_thread.py +++ b/pypy/module/thread/test/test_thread.py @@ -13,18 +13,26 @@ def f(): lock.acquire() lock.release() + start = thread._count() try: try: for i in range(1000): thread.start_new_thread(f, ()) finally: lock.release() - # wait a bit to allow most threads to finish now - time.sleep(0.5) except (thread.error, MemoryError): cls.w_can_start_many_threads = space.wrap(False) else: cls.w_can_start_many_threads = space.wrap(True) + # wait a bit to allow all threads to finish now + remaining = thread._count() + retries = 0 + while remaining > start: + retries += 1 + if retries == 200: + raise Exception("the test's threads don't stop!") + time.sleep(0.2) + remaining = thread._count() def test_start_new_thread(self): import thread @@ -227,7 +235,7 @@ import signal def f(): - for x in range(5): + for x in range(40): if waiting: thread.interrupt_main() return @@ -236,7 +244,7 @@ def busy_wait(): waiting.append(None) - for x in range(10): + for x in range(50): print 'tick...', x # <-force the GIL to be released, as time.sleep(0.1) # time.sleep doesn't do non-translated waiting.pop() @@ -245,6 +253,8 @@ signal.signal(signal.SIGINT, signal.default_int_handler) for i in range(100): + print + print "loop", i waiting = [] thread.start_new_thread(f, ()) raises(KeyboardInterrupt, busy_wait) diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py --- a/pypy/objspace/descroperation.py +++ b/pypy/objspace/descroperation.py @@ -671,6 +671,7 @@ left, right = specialnames errormsg = "unsupported operand type(s) for %s: '%%N' and '%%N'" % ( symbol.replace('%', '%%'),) + seq_bug_compat = (symbol == '+' or symbol == '*') def binop_impl(space, w_obj1, w_obj2): w_typ1 = space.type(w_obj1) @@ -686,20 +687,16 @@ # __xxx__ and __rxxx__ methods where found by identity. # Note that space.is_w() is potentially not happy if one of them # is None... - if w_left_src is not w_right_src: # XXX - # -- cpython bug compatibility: see objspace/std/test/ - # -- test_unicodeobject.test_str_unicode_concat_overrides. - # -- The following handles "unicode + string subclass" by - # -- pretending that the unicode is a superclass of the - # -- string, thus giving priority to the string subclass' - # -- __radd__() method. The case "string + unicode subclass" - # -- is handled directly by add__String_Unicode(). - if symbol == '+' and space.is_w(w_typ1, space.w_unicode): - w_typ1 = space.w_basestring - # -- end of bug compatibility - if space.is_true(space.issubtype(w_typ2, w_typ1)): - if (w_left_src and w_right_src and - not space.abstract_issubclass_w(w_left_src, w_right_src) and + if w_right_src and (w_left_src is not w_right_src) and w_left_src: + # 'seq_bug_compat' is for cpython bug-to-bug compatibility: + # see objspace/std/test/test_unicodeobject.*concat_overrides + # and objspace/test/test_descrobject.*rmul_overrides. + # For cases like "unicode + string subclass". + if ((seq_bug_compat and w_typ1.flag_sequence_bug_compat + and not w_typ2.flag_sequence_bug_compat) + # the non-bug-compat part is the following check: + or space.is_true(space.issubtype(w_typ2, w_typ1))): + if (not space.abstract_issubclass_w(w_left_src, w_right_src) and not space.abstract_issubclass_w(w_typ1, w_right_src)): w_obj1, w_obj2 = w_obj2, w_obj1 w_left_impl, w_right_impl = w_right_impl, w_left_impl diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py --- a/pypy/objspace/std/bytearrayobject.py +++ b/pypy/objspace/std/bytearrayobject.py @@ -534,7 +534,7 @@ if not e.match(space, space.w_TypeError): raise else: - return [c for c in buf.as_str()] + return list(buf.as_str()) # sequence of bytes w_iter = space.iter(w_source) @@ -1131,6 +1131,7 @@ reverse = interp2app(W_BytearrayObject.descr_reverse, doc=BytearrayDocstrings.reverse.__doc__), ) +W_BytearrayObject.typedef.flag_sequence_bug_compat = True init_signature = Signature(['source', 'encoding', 'errors'], None, None) init_defaults = [None, None, None] diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py --- a/pypy/objspace/std/bytesobject.py +++ b/pypy/objspace/std/bytesobject.py @@ -951,6 +951,7 @@ _formatter_field_name_split = interp2app(W_BytesObject.descr_formatter_field_name_split), ) +W_BytesObject.typedef.flag_sequence_bug_compat = True def string_escape_encode(s, quote): diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py --- a/pypy/objspace/std/listobject.py +++ b/pypy/objspace/std/listobject.py @@ -1874,3 +1874,4 @@ insert = interp2app(W_ListObject.descr_insert), remove = interp2app(W_ListObject.descr_remove), ) +W_ListObject.typedef.flag_sequence_bug_compat = True diff --git a/pypy/objspace/std/stdtypedef.py b/pypy/objspace/std/stdtypedef.py --- a/pypy/objspace/std/stdtypedef.py +++ b/pypy/objspace/std/stdtypedef.py @@ -93,6 +93,8 @@ overridetypedef=overridetypedef) if typedef is not overridetypedef: w_type.w_doc = space.wrap(typedef.doc) + if hasattr(typedef, 'flag_sequence_bug_compat'): + w_type.flag_sequence_bug_compat = typedef.flag_sequence_bug_compat w_type.lazyloaders = lazyloaders return w_type diff --git a/pypy/objspace/std/tupleobject.py b/pypy/objspace/std/tupleobject.py --- a/pypy/objspace/std/tupleobject.py +++ b/pypy/objspace/std/tupleobject.py @@ -244,6 +244,7 @@ count = interp2app(W_AbstractTupleObject.descr_count), index = interp2app(W_AbstractTupleObject.descr_index) ) +W_AbstractTupleObject.typedef.flag_sequence_bug_compat = True class W_TupleObject(W_AbstractTupleObject): diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py --- a/pypy/objspace/std/typeobject.py +++ b/pypy/objspace/std/typeobject.py @@ -67,6 +67,7 @@ _immutable_fields_ = ["flag_heaptype", "flag_cpytype", "flag_abstract?", + "flag_sequence_bug_compat", 'needsdel', 'weakrefable', 'hasdict', @@ -104,6 +105,7 @@ w_self.flag_heaptype = False w_self.flag_cpytype = False w_self.flag_abstract = False + w_self.flag_sequence_bug_compat = False w_self.instancetypedef = overridetypedef if overridetypedef is not None: diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -1068,6 +1068,7 @@ _formatter_field_name_split = interp2app(W_UnicodeObject.descr_formatter_field_name_split), ) +W_UnicodeObject.typedef.flag_sequence_bug_compat = True def _create_list_from_unicode(value): diff --git a/pypy/objspace/test/test_descroperation.py b/pypy/objspace/test/test_descroperation.py --- a/pypy/objspace/test/test_descroperation.py +++ b/pypy/objspace/test/test_descroperation.py @@ -734,6 +734,44 @@ assert X() == 'hello' + def test_sequence_rmul_overrides(self): + class oops(object): + def __rmul__(self, other): + return 42 + def __index__(self): + return 3 + assert '2' * oops() == 42 + assert [2] * oops() == 42 + assert (2,) * oops() == 42 + assert u'2' * oops() == 42 + assert bytearray('2') * oops() == 42 + assert 1000 * oops() == 42 + assert '2'.__mul__(oops()) == '222' + + def test_sequence_rmul_overrides_oldstyle(self): + class oops: + def __rmul__(self, other): + return 42 + def __index__(self): + return 3 + assert '2' * oops() == 42 + assert [2] * oops() == 42 + assert (2,) * oops() == 42 + assert u'2' * oops() == 42 + assert bytearray('2') * oops() == 42 + assert 1000 * oops() == 42 + assert '2'.__mul__(oops()) == '222' + + def test_sequence_radd_overrides(self): + class A1(list): + pass + class A2(list): + def __radd__(self, other): + return 42 + assert [2] + A1([3]) == [2, 3] + assert type([2] + A1([3])) is list + assert [2] + A2([3]) == 42 + class AppTestWithBuiltinShortcut(AppTest_Descroperation): spaceconfig = {'objspace.std.builtinshortcut': True} diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -297,7 +297,13 @@ argparse = imp.load_source('argparse', 'lib-python/2.7/argparse.py') if sys.platform == 'win32': pypy_exe = 'pypy.exe' From noreply at buildbot.pypy.org Sun Sep 21 10:09:41 2014 From: noreply at buildbot.pypy.org (groggi) Date: Sun, 21 Sep 2014 10:09:41 +0200 (CEST) Subject: [pypy-commit] pypy gc-incminimark-pinning: Merge default into gc-incminimark-pinning Message-ID: <20140921080941.675041C0130@cobra.cs.uni-duesseldorf.de> Author: Gregor Wegberg Branch: gc-incminimark-pinning Changeset: r73626:1bc63cf2b16c Date: 2014-09-21 10:08 +0200 http://bitbucket.org/pypy/pypy/changeset/1bc63cf2b16c/ Log: Merge default into gc-incminimark-pinning diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py --- a/lib_pypy/_curses.py +++ b/lib_pypy/_curses.py @@ -286,6 +286,13 @@ lib = ffi.verify(""" +#ifdef __APPLE__ +/* the following define is necessary for OS X 10.6+; without it, the + Apple-supplied ncurses.h sets NCURSES_OPAQUE to 1, and then Python + can't get at the WINDOW flags field. */ +#define NCURSES_OPAQUE 0 +#endif + #include #include #include diff --git a/pypy/doc/release-2.4.0.rst b/pypy/doc/release-2.4.0.rst --- a/pypy/doc/release-2.4.0.rst +++ b/pypy/doc/release-2.4.0.rst @@ -5,7 +5,7 @@ We're pleased to announce PyPy 2.4, which contains significant performance enhancements and bug fixes. -You can already download the PyPy 2.4-beta1 pre-release here: +You can download the PyPy 2.4.0 release here: http://pypy.org/download.html @@ -63,6 +63,8 @@ PyPy now uses Python 2.7.8 standard library. +We fixed a memory leak in IO in the sandbox_ code + We welcomed more than 12 new contributors, and conducted two Google Summer of Code projects, as well as other student projects not directly related to Summer of Code. @@ -105,6 +107,7 @@ .. _`whats-new`: http://doc.pypy.org/en/latest/whatsnew-2.3.1.html .. _resolved: https://bitbucket.org/pypy/pypy/issues?status=resolved +.. _sandbox: http://doc.pypy.org/en/latest/sandbox.html We have further improvements on the way: rpython file handling, numpy linalg compatibility, as well diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_zdistutils.py b/pypy/module/test_lib_pypy/cffi_tests/test_zdistutils.py --- a/pypy/module/test_lib_pypy/cffi_tests/test_zdistutils.py +++ b/pypy/module/test_lib_pypy/cffi_tests/test_zdistutils.py @@ -16,6 +16,10 @@ if distutils.ccompiler.get_default_compiler() == 'msvc': self.lib_m = 'msvcrt' + def teardown_class(self): + if udir.isdir(): + udir.remove() + def test_locate_engine_class(self): cls = _locate_engine_class(FFI(), self.generic) if self.generic: diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_zintegration.py b/pypy/module/test_lib_pypy/cffi_tests/test_zintegration.py --- a/pypy/module/test_lib_pypy/cffi_tests/test_zintegration.py +++ b/pypy/module/test_lib_pypy/cffi_tests/test_zintegration.py @@ -73,50 +73,55 @@ assert not os.path.exists(str(SNIPPET_DIR.join(dirname, 'lextab.py'))) assert not os.path.exists(str(SNIPPET_DIR.join(dirname, 'yacctab.py'))) -def test_infrastructure(): - run_setup_and_program('infrastructure', ''' - import snip_infrastructure - assert snip_infrastructure.func() == 42 - ''') +class TestZIntegration(object): + def teardown_class(self): + if udir.isdir(): + udir.remove() -def test_distutils_module(): - run_setup_and_program("distutils_module", ''' - import snip_basic_verify - p = snip_basic_verify.C.getpwuid(0) - assert snip_basic_verify.ffi.string(p.pw_name) == b"root" - ''') + def test_infrastructure(self): + run_setup_and_program('infrastructure', ''' + import snip_infrastructure + assert snip_infrastructure.func() == 42 + ''') -def test_distutils_package_1(): - run_setup_and_program("distutils_package_1", ''' - import snip_basic_verify1 - p = snip_basic_verify1.C.getpwuid(0) - assert snip_basic_verify1.ffi.string(p.pw_name) == b"root" - ''') + def test_distutils_module(self): + run_setup_and_program("distutils_module", ''' + import snip_basic_verify + p = snip_basic_verify.C.getpwuid(0) + assert snip_basic_verify.ffi.string(p.pw_name) == b"root" + ''') -def test_distutils_package_2(): - run_setup_and_program("distutils_package_2", ''' - import snip_basic_verify2 - p = snip_basic_verify2.C.getpwuid(0) - assert snip_basic_verify2.ffi.string(p.pw_name) == b"root" - ''') + def test_distutils_package_1(self): + run_setup_and_program("distutils_package_1", ''' + import snip_basic_verify1 + p = snip_basic_verify1.C.getpwuid(0) + assert snip_basic_verify1.ffi.string(p.pw_name) == b"root" + ''') -def test_setuptools_module(): - run_setup_and_program("setuptools_module", ''' - import snip_setuptools_verify - p = snip_setuptools_verify.C.getpwuid(0) - assert snip_setuptools_verify.ffi.string(p.pw_name) == b"root" - ''') + def test_distutils_package_2(self): + run_setup_and_program("distutils_package_2", ''' + import snip_basic_verify2 + p = snip_basic_verify2.C.getpwuid(0) + assert snip_basic_verify2.ffi.string(p.pw_name) == b"root" + ''') -def test_setuptools_package_1(): - run_setup_and_program("setuptools_package_1", ''' - import snip_setuptools_verify1 - p = snip_setuptools_verify1.C.getpwuid(0) - assert snip_setuptools_verify1.ffi.string(p.pw_name) == b"root" - ''') + def test_setuptools_module(self): + run_setup_and_program("setuptools_module", ''' + import snip_setuptools_verify + p = snip_setuptools_verify.C.getpwuid(0) + assert snip_setuptools_verify.ffi.string(p.pw_name) == b"root" + ''') -def test_setuptools_package_2(): - run_setup_and_program("setuptools_package_2", ''' - import snip_setuptools_verify2 - p = snip_setuptools_verify2.C.getpwuid(0) - assert snip_setuptools_verify2.ffi.string(p.pw_name) == b"root" - ''') + def test_setuptools_package_1(self): + run_setup_and_program("setuptools_package_1", ''' + import snip_setuptools_verify1 + p = snip_setuptools_verify1.C.getpwuid(0) + assert snip_setuptools_verify1.ffi.string(p.pw_name) == b"root" + ''') + + def test_setuptools_package_2(self): + run_setup_and_program("setuptools_package_2", ''' + import snip_setuptools_verify2 + p = snip_setuptools_verify2.C.getpwuid(0) + assert snip_setuptools_verify2.ffi.string(p.pw_name) == b"root" + ''') From noreply at buildbot.pypy.org Sun Sep 21 10:21:55 2014 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 21 Sep 2014 10:21:55 +0200 (CEST) Subject: [pypy-commit] pypy default: a big oops Message-ID: <20140921082155.090231C044F@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r73627:7255c146c5f2 Date: 2014-09-21 10:09 +0200 http://bitbucket.org/pypy/pypy/changeset/7255c146c5f2/ Log: a big oops diff --git a/rpython/tool/jitlogparser/parser.py b/rpython/tool/jitlogparser/parser.py --- a/rpython/tool/jitlogparser/parser.py +++ b/rpython/tool/jitlogparser/parser.py @@ -406,9 +406,9 @@ loops = [] cat = extract_category(log, 'jit-log-opt') if not cat: - extract_category(log, 'jit-log-rewritten') + cat = extract_category(log, 'jit-log-rewritten') if not cat: - extract_category(log, 'jit-log-noopt') + cat = extract_category(log, 'jit-log-noopt') for entry in cat: parser = ParserCls(entry, None, {}, 'lltype', None, nonstrict=True) From noreply at buildbot.pypy.org Sun Sep 21 10:21:56 2014 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 21 Sep 2014 10:21:56 +0200 (CEST) Subject: [pypy-commit] pypy default: merge Message-ID: <20140921082156.2B2A81C044F@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r73628:b6c3792d707a Date: 2014-09-21 10:09 +0200 http://bitbucket.org/pypy/pypy/changeset/b6c3792d707a/ Log: merge diff --git a/rpython/rtyper/lltypesystem/lltype.py b/rpython/rtyper/lltypesystem/lltype.py --- a/rpython/rtyper/lltypesystem/lltype.py +++ b/rpython/rtyper/lltypesystem/lltype.py @@ -1,16 +1,17 @@ -from types import NoneType, MethodType import weakref +from types import MethodType, NoneType + +from rpython.annotator.bookkeeper import analyzer_for, immutablevalue from rpython.annotator.model import ( - SomeInteger, SomeBool, SomeObject, AnnotatorError) + AnnotatorError, SomeBool, SomeInteger, SomeObject) +from rpython.rlib.objectmodel import Symbolic from rpython.rlib.rarithmetic import ( - r_int, r_uint, intmask, r_singlefloat, r_ulonglong, r_longlong, - r_longfloat, r_longlonglong, base_int, normalizedinttype, longlongmask, - longlonglongmask, maxint, is_valid_int, is_emulated_long) -from rpython.rlib.objectmodel import Symbolic + base_int, intmask, is_emulated_long, is_valid_int, longlonglongmask, + longlongmask, maxint, normalizedinttype, r_int, r_longfloat, r_longlong, + r_longlonglong, r_singlefloat, r_uint, r_ulonglong) +from rpython.rtyper.extregistry import ExtRegistryEntry +from rpython.tool import leakfinder from rpython.tool.identity_dict import identity_dict -from rpython.tool import leakfinder -from rpython.annotator.bookkeeper import analyzer_for, immutablevalue -from rpython.rtyper.extregistry import ExtRegistryEntry class State(object): pass @@ -313,14 +314,12 @@ except KeyError: return ContainerType.__getattr__(self, name) - def _nofield(self, name): raise AttributeError('struct %s has no field %r' % (self._name, name)) def _names_without_voids(self): - names_without_voids = [name for name in self._names if self._flds[name] is not Void] - return names_without_voids + return [name for name in self._names if self._flds[name] is not Void] def _str_fields_without_voids(self): return ', '.join(['%s: %s' % (name, self._flds[name]) @@ -576,8 +575,10 @@ _gckind = 'raw' def __init__(self, tag, hints={}): - """ if hints['render_structure'] is set, the type is internal and not considered - to come from somewhere else (it should be rendered as a structure) """ + """If hints['render_structure'] is set, the type is internal and + not considered to come from somewhere else (it should be + rendered as a structure) + """ self.tag = tag self.__name__ = tag self.hints = frozendict(hints) @@ -675,7 +676,8 @@ _numbertypes = {int: Number("Signed", int, intmask)} _numbertypes[r_int] = _numbertypes[int] -_numbertypes[r_longlonglong] = Number("SignedLongLongLong", r_longlonglong, longlonglongmask) +_numbertypes[r_longlonglong] = Number("SignedLongLongLong", r_longlonglong, + longlonglongmask) if r_longlong is not r_int: _numbertypes[r_longlong] = Number("SignedLongLong", r_longlong, longlongmask) @@ -702,8 +704,8 @@ UnsignedLongLong = build_number("UnsignedLongLong", r_ulonglong) Float = Primitive("Float", 0.0) # C type 'double' -SingleFloat = Primitive("SingleFloat", r_singlefloat(0.0)) # C type 'float' -LongFloat = Primitive("LongFloat", r_longfloat(0.0)) # C type 'long double' +SingleFloat = Primitive("SingleFloat", r_singlefloat(0.0)) # 'float' +LongFloat = Primitive("LongFloat", r_longfloat(0.0)) # 'long double' r_singlefloat._TYPE = SingleFloat Char = Primitive("Char", '\x00') @@ -876,9 +878,11 @@ @analyzer_for(cast_primitive) def ann_cast_primitive(T, s_v): - from rpython.rtyper.llannotation import annotation_to_lltype, ll_to_annotation + from rpython.rtyper.llannotation import ( + annotation_to_lltype, ll_to_annotation) assert T.is_constant() - return ll_to_annotation(cast_primitive(T.const, annotation_to_lltype(s_v)._defl())) + return ll_to_annotation(cast_primitive(T.const, + annotation_to_lltype(s_v)._defl())) def _cast_whatever(TGT, value): @@ -905,7 +909,8 @@ elif TGT == llmemory.Address and isinstance(ORIG, Ptr): return llmemory.cast_ptr_to_adr(value) elif TGT == Signed and isinstance(ORIG, Ptr) and ORIG.TO._gckind == 'raw': - return llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(value), 'symbolic') + return llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(value), + 'symbolic') raise TypeError("don't know how to cast from %r to %r" % (ORIG, TGT)) @@ -1176,8 +1181,8 @@ except DelayedPointer: return True # assume it's not a delayed null - # _setobj, _getobj and _obj0 are really _internal_ implementations details of _ptr, - # use _obj if necessary instead ! + # _setobj, _getobj and _obj0 are really _internal_ implementations + # details of _ptr, use _obj if necessary instead ! def _setobj(self, pointing_to, solid=False): if pointing_to is None: obj0 = None @@ -1244,12 +1249,12 @@ if T1 == T2: setattr(self._obj, field_name, val) else: - raise TypeError("%r instance field %r:\n" - "expects %r\n" - " got %r" % (self._T, field_name, T1, T2)) + raise TypeError( + "%r instance field %r:\nexpects %r\n got %r" % + (self._T, field_name, T1, T2)) return - raise AttributeError("%r instance has no field %r" % (self._T, - field_name)) + raise AttributeError("%r instance has no field %r" % + (self._T, field_name)) def __getitem__(self, i): # ! can only return basic or ptr ! if isinstance(self._T, (Array, FixedSizeArray)): @@ -1266,7 +1271,8 @@ if isinstance(self._T, (Array, FixedSizeArray)): T1 = self._T.OF if isinstance(T1, ContainerType): - raise TypeError("cannot directly assign to container array items") + raise TypeError("cannot directly assign to container array " + "items") T2 = typeOf(val) if T2 != T1: from rpython.rtyper.lltypesystem import rffi @@ -1316,7 +1322,8 @@ from rpython.rtyper.lltypesystem import rffi if isinstance(self._T, FuncType): if len(args) != len(self._T.ARGS): - raise TypeError("calling %r with wrong argument number: %r" % (self._T, args)) + raise TypeError("calling %r with wrong argument number: %r" % + (self._T, args)) for i, a, ARG in zip(range(len(self._T.ARGS)), args, self._T.ARGS): if typeOf(a) != ARG: # ARG could be Void @@ -1415,11 +1422,13 @@ raise RuntimeError("widening to trash: %r" % self) PARENTTYPE = struc._parent_type if getattr(parent, PARENTTYPE._names[0]) != struc: - raise InvalidCast(CURTYPE, PTRTYPE) # xxx different exception perhaps? + # xxx different exception perhaps? + raise InvalidCast(CURTYPE, PTRTYPE) struc = parent u -= 1 if PARENTTYPE != PTRTYPE.TO: - raise RuntimeError("widening %r inside %r instead of %r" % (CURTYPE, PARENTTYPE, PTRTYPE.TO)) + raise RuntimeError("widening %r inside %r instead of %r" % + (CURTYPE, PARENTTYPE, PTRTYPE.TO)) return _ptr(PTRTYPE, struc, solid=self._solid) def _cast_to_int(self, check=True): @@ -1430,7 +1439,9 @@ return obj # special case for cast_int_to_ptr() results obj = normalizeptr(self, check)._getobj(check) if isinstance(obj, int): - return obj # special case for cast_int_to_ptr() results put into opaques + # special case for cast_int_to_ptr() results put into + # opaques + return obj if getattr(obj, '_read_directly_intval', False): return obj.intval # special case for _llgcopaque result = intmask(obj._getid()) @@ -1468,7 +1479,8 @@ """XXX A nice docstring here""" T = typeOf(val) if isinstance(T, ContainerType): - if self._T._gckind == 'gc' and T._gckind == 'raw' and not isinstance(T, OpaqueType): + if (self._T._gckind == 'gc' and T._gckind == 'raw' and + not isinstance(T, OpaqueType)): val = _interior_ptr(T, self._obj, [offset]) else: val = _ptr(Ptr(T), val, solid=self._solid) @@ -1531,12 +1543,14 @@ setattr(example, s_attr.const, v_lltype._defl()) def call(self, args): - from rpython.rtyper.llannotation import annotation_to_lltype, ll_to_annotation + from rpython.rtyper.llannotation import ( + annotation_to_lltype, ll_to_annotation) args_s, kwds_s = args.unpack() if kwds_s: raise Exception("keyword arguments to call to a low-level fn ptr") info = 'argument to ll function pointer call' - llargs = [annotation_to_lltype(s_arg, info)._defl() for s_arg in args_s] + llargs = [annotation_to_lltype(s_arg, info)._defl() + for s_arg in args_s] v = self.ll_ptrtype._example()(*llargs) return ll_to_annotation(v) @@ -1593,7 +1607,6 @@ return val - assert not '__dict__' in dir(_interior_ptr) class _container(object): @@ -1721,11 +1734,13 @@ __slots__ = ('_hash_cache_', '_compilation_info') - def __new__(self, TYPE, n=None, initialization=None, parent=None, parentindex=None): + def __new__(self, TYPE, n=None, initialization=None, parent=None, + parentindex=None): my_variety = _struct_variety(TYPE._names) return object.__new__(my_variety) - def __init__(self, TYPE, n=None, initialization=None, parent=None, parentindex=None): + def __init__(self, TYPE, n=None, initialization=None, parent=None, + parentindex=None): _parentable.__init__(self, TYPE) if n is not None and TYPE._arrayfld is None: raise TypeError("%r is not variable-sized" % (TYPE,)) @@ -1734,9 +1749,11 @@ first, FIRSTTYPE = TYPE._first_struct() for fld, typ in TYPE._flds.items(): if fld == TYPE._arrayfld: - value = _array(typ, n, initialization=initialization, parent=self, parentindex=fld) + value = _array(typ, n, initialization=initialization, + parent=self, parentindex=fld) else: - value = typ._allocate(initialization=initialization, parent=self, parentindex=fld) + value = typ._allocate(initialization=initialization, + parent=self, parentindex=fld) setattr(self, fld, value) if parent is not None: self._setparentstructure(parent, parentindex) @@ -1795,7 +1812,8 @@ __slots__ = ('items',) - def __init__(self, TYPE, n, initialization=None, parent=None, parentindex=None): + def __init__(self, TYPE, n, initialization=None, parent=None, + parentindex=None): if not is_valid_int(n): raise TypeError("array length must be an int") if n < 0: @@ -1964,7 +1982,8 @@ if not key._was_freed(): newcache[key] = value except RuntimeError: - pass # ignore "accessing subxxx, but already gc-ed parent" + # ignore "accessing subxxx, but already gc-ed parent" + pass if newcache: _subarray._cache[T] = newcache else: @@ -2020,8 +2039,10 @@ attrs.setdefault('_name', '?') attrs.setdefault('_callable', None) self.__dict__.update(attrs) - if '_callable' in attrs and hasattr(attrs['_callable'], '_compilation_info'): - self.__dict__['compilation_info'] = attrs['_callable']._compilation_info + if '_callable' in attrs and hasattr(attrs['_callable'], + '_compilation_info'): + self.__dict__['compilation_info'] = \ + attrs['_callable']._compilation_info def __repr__(self): return '<%s>' % (self,) @@ -2126,8 +2147,8 @@ return _ptr(Ptr(T), o, solid) @analyzer_for(malloc) -def ann_malloc(s_T, s_n=None, s_flavor=None, s_zero=None, s_track_allocation=None, - s_add_memory_pressure=None): +def ann_malloc(s_T, s_n=None, s_flavor=None, s_zero=None, + s_track_allocation=None, s_add_memory_pressure=None): assert (s_n is None or s_n.knowntype == int or issubclass(s_n.knowntype, base_int)) assert s_T.is_constant() @@ -2303,7 +2324,8 @@ @analyzer_for(runtime_type_info) def ann_runtime_type_info(s_p): - assert isinstance(s_p, SomePtr), "runtime_type_info of non-pointer: %r" % s_p + assert isinstance(s_p, SomePtr), \ + "runtime_type_info of non-pointer: %r" % s_p return SomePtr(typeOf(runtime_type_info(s_p.ll_ptrtype._example()))) From noreply at buildbot.pypy.org Sun Sep 21 10:21:57 2014 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 21 Sep 2014 10:21:57 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: small improvements Message-ID: <20140921082157.465D21C044F@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73629:4739b87c13c1 Date: 2014-09-21 10:17 +0200 http://bitbucket.org/pypy/pypy/changeset/4739b87c13c1/ Log: small improvements diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -46,6 +46,7 @@ self.known_lengths = {} self.write_barrier_applied = {} self.delayed_zero_setfields = {} + self.delayed_zero_setarrayitems = {} def rewrite(self, operations): # we can only remember one malloc since the next malloc can possibly @@ -135,6 +136,7 @@ self.delayed_zero_setfields[result] = d for ofs in descr.offsets_of_gcfields: d[ofs] = None + self.delayed_zero_setfields.clear() def consider_setfield_gc(self, op): offset = self.cpu.unpack_fielddescr(op.getdescr()) diff --git a/rpython/tool/jitlogparser/parser.py b/rpython/tool/jitlogparser/parser.py --- a/rpython/tool/jitlogparser/parser.py +++ b/rpython/tool/jitlogparser/parser.py @@ -406,9 +406,9 @@ loops = [] cat = extract_category(log, 'jit-log-opt') if not cat: - extract_category(log, 'jit-log-rewritten') + cat = extract_category(log, 'jit-log-rewritten') if not cat: - extract_category(log, 'jit-log-noopt') + cat = extract_category(log, 'jit-log-noopt') for entry in cat: parser = ParserCls(entry, None, {}, 'lltype', None, nonstrict=True) From noreply at buildbot.pypy.org Sun Sep 21 10:21:58 2014 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 21 Sep 2014 10:21:58 +0200 (CEST) Subject: [pypy-commit] pypy default: merge Message-ID: <20140921082158.72B9A1C044F@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r73630:d020fd66e8e2 Date: 2014-09-21 10:18 +0200 http://bitbucket.org/pypy/pypy/changeset/d020fd66e8e2/ Log: merge diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py --- a/lib_pypy/_curses.py +++ b/lib_pypy/_curses.py @@ -286,6 +286,13 @@ lib = ffi.verify(""" +#ifdef __APPLE__ +/* the following define is necessary for OS X 10.6+; without it, the + Apple-supplied ncurses.h sets NCURSES_OPAQUE to 1, and then Python + can't get at the WINDOW flags field. */ +#define NCURSES_OPAQUE 0 +#endif + #include #include #include diff --git a/pypy/doc/release-2.4.0.rst b/pypy/doc/release-2.4.0.rst --- a/pypy/doc/release-2.4.0.rst +++ b/pypy/doc/release-2.4.0.rst @@ -5,7 +5,7 @@ We're pleased to announce PyPy 2.4, which contains significant performance enhancements and bug fixes. -You can already download the PyPy 2.4-beta1 pre-release here: +You can download the PyPy 2.4.0 release here: http://pypy.org/download.html @@ -63,6 +63,8 @@ PyPy now uses Python 2.7.8 standard library. +We fixed a memory leak in IO in the sandbox_ code + We welcomed more than 12 new contributors, and conducted two Google Summer of Code projects, as well as other student projects not directly related to Summer of Code. @@ -105,6 +107,7 @@ .. _`whats-new`: http://doc.pypy.org/en/latest/whatsnew-2.3.1.html .. _resolved: https://bitbucket.org/pypy/pypy/issues?status=resolved +.. _sandbox: http://doc.pypy.org/en/latest/sandbox.html We have further improvements on the way: rpython file handling, numpy linalg compatibility, as well diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_zdistutils.py b/pypy/module/test_lib_pypy/cffi_tests/test_zdistutils.py --- a/pypy/module/test_lib_pypy/cffi_tests/test_zdistutils.py +++ b/pypy/module/test_lib_pypy/cffi_tests/test_zdistutils.py @@ -16,6 +16,10 @@ if distutils.ccompiler.get_default_compiler() == 'msvc': self.lib_m = 'msvcrt' + def teardown_class(self): + if udir.isdir(): + udir.remove() + def test_locate_engine_class(self): cls = _locate_engine_class(FFI(), self.generic) if self.generic: diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_zintegration.py b/pypy/module/test_lib_pypy/cffi_tests/test_zintegration.py --- a/pypy/module/test_lib_pypy/cffi_tests/test_zintegration.py +++ b/pypy/module/test_lib_pypy/cffi_tests/test_zintegration.py @@ -73,50 +73,55 @@ assert not os.path.exists(str(SNIPPET_DIR.join(dirname, 'lextab.py'))) assert not os.path.exists(str(SNIPPET_DIR.join(dirname, 'yacctab.py'))) -def test_infrastructure(): - run_setup_and_program('infrastructure', ''' - import snip_infrastructure - assert snip_infrastructure.func() == 42 - ''') +class TestZIntegration(object): + def teardown_class(self): + if udir.isdir(): + udir.remove() -def test_distutils_module(): - run_setup_and_program("distutils_module", ''' - import snip_basic_verify - p = snip_basic_verify.C.getpwuid(0) - assert snip_basic_verify.ffi.string(p.pw_name) == b"root" - ''') + def test_infrastructure(self): + run_setup_and_program('infrastructure', ''' + import snip_infrastructure + assert snip_infrastructure.func() == 42 + ''') -def test_distutils_package_1(): - run_setup_and_program("distutils_package_1", ''' - import snip_basic_verify1 - p = snip_basic_verify1.C.getpwuid(0) - assert snip_basic_verify1.ffi.string(p.pw_name) == b"root" - ''') + def test_distutils_module(self): + run_setup_and_program("distutils_module", ''' + import snip_basic_verify + p = snip_basic_verify.C.getpwuid(0) + assert snip_basic_verify.ffi.string(p.pw_name) == b"root" + ''') -def test_distutils_package_2(): - run_setup_and_program("distutils_package_2", ''' - import snip_basic_verify2 - p = snip_basic_verify2.C.getpwuid(0) - assert snip_basic_verify2.ffi.string(p.pw_name) == b"root" - ''') + def test_distutils_package_1(self): + run_setup_and_program("distutils_package_1", ''' + import snip_basic_verify1 + p = snip_basic_verify1.C.getpwuid(0) + assert snip_basic_verify1.ffi.string(p.pw_name) == b"root" + ''') -def test_setuptools_module(): - run_setup_and_program("setuptools_module", ''' - import snip_setuptools_verify - p = snip_setuptools_verify.C.getpwuid(0) - assert snip_setuptools_verify.ffi.string(p.pw_name) == b"root" - ''') + def test_distutils_package_2(self): + run_setup_and_program("distutils_package_2", ''' + import snip_basic_verify2 + p = snip_basic_verify2.C.getpwuid(0) + assert snip_basic_verify2.ffi.string(p.pw_name) == b"root" + ''') -def test_setuptools_package_1(): - run_setup_and_program("setuptools_package_1", ''' - import snip_setuptools_verify1 - p = snip_setuptools_verify1.C.getpwuid(0) - assert snip_setuptools_verify1.ffi.string(p.pw_name) == b"root" - ''') + def test_setuptools_module(self): + run_setup_and_program("setuptools_module", ''' + import snip_setuptools_verify + p = snip_setuptools_verify.C.getpwuid(0) + assert snip_setuptools_verify.ffi.string(p.pw_name) == b"root" + ''') -def test_setuptools_package_2(): - run_setup_and_program("setuptools_package_2", ''' - import snip_setuptools_verify2 - p = snip_setuptools_verify2.C.getpwuid(0) - assert snip_setuptools_verify2.ffi.string(p.pw_name) == b"root" - ''') + def test_setuptools_package_1(self): + run_setup_and_program("setuptools_package_1", ''' + import snip_setuptools_verify1 + p = snip_setuptools_verify1.C.getpwuid(0) + assert snip_setuptools_verify1.ffi.string(p.pw_name) == b"root" + ''') + + def test_setuptools_package_2(self): + run_setup_and_program("setuptools_package_2", ''' + import snip_setuptools_verify2 + p = snip_setuptools_verify2.C.getpwuid(0) + assert snip_setuptools_verify2.ffi.string(p.pw_name) == b"root" + ''') diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -297,7 +297,13 @@ argparse = imp.load_source('argparse', 'lib-python/2.7/argparse.py') if sys.platform == 'win32': pypy_exe = 'pypy.exe' - license_base = os.path.join(basedir, r'..\..\..\local') # as on buildbot YMMV + for p in [os.path.join(basedir, r'..\..\..\local'), #buildbot + os.path.join(basedir, r'..\local')]: # pypy/doc/windows.rst + if os.path.exists(p): + license_base = p + break + else: + license_base = 'unkown' else: pypy_exe = 'pypy' license_base = '/usr/share/doc' @@ -370,5 +376,21 @@ if __name__ == '__main__': import sys + if sys.platform == 'win32': + # Try to avoid opeing a dialog box if one of the + # subprocesses causes a system error + import ctypes + winapi = ctypes.windll.kernel32 + SetErrorMode = winapi.SetErrorMode + SetErrorMode.argtypes=[ctypes.c_int] + + SEM_FAILCRITICALERRORS = 1 + SEM_NOGPFAULTERRORBOX = 2 + SEM_NOOPENFILEERRORBOX = 0x8000 + flags = SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX | SEM_NOOPENFILEERRORBOX + #Since there is no GetErrorMode, do a double Set + old_mode = SetErrorMode(flags) + SetErrorMode(old_mode | flags) + retval, _ = package(*sys.argv[1:]) sys.exit(retval) diff --git a/pypy/tool/release/test/test_package.py b/pypy/tool/release/test/test_package.py --- a/pypy/tool/release/test/test_package.py +++ b/pypy/tool/release/test/test_package.py @@ -115,15 +115,21 @@ check(pypy, 0755) def test_generate_license(): - from os.path import dirname, abspath, join + from os.path import dirname, abspath, join, exists class Options(object): pass options = Options() basedir = dirname(dirname(dirname(dirname(dirname(abspath(__file__)))))) options.no_tk = False if sys.platform == 'win32': - # as on buildbot YMMV - options.license_base = join(basedir, r'..\..\..\local') + for p in [join(basedir, r'..\..\..\local'), #buildbot + join(basedir, r'..\local')]: # pypy/doc/windows.rst + if exists(p): + license_base = p + break + else: + license_base = 'unkown' + options.license_base = license_base else: options.license_base = '/usr/share/doc' license = package.generate_license(py.path.local(basedir), options) diff --git a/rpython/translator/driver.py b/rpython/translator/driver.py --- a/rpython/translator/driver.py +++ b/rpython/translator/driver.py @@ -481,6 +481,14 @@ libname = str(newsoname.dirpath().join('python27.lib')) shutil.copyfile(str(soname.new(ext='lib')), libname) self.log.info("copied: %s" % (libname,)) + # XXX TODO : replace the nonsense above with + # ext_to_copy = ['lib', 'pdb'] + ext_to_copy = ['pdb',] + for ext in ext_to_copy: + name = soname.new(ext=ext) + newname = newexename.new(basename=soname.basename) + shutil.copyfile(str(name), str(newname.new(ext=ext))) + self.log.info("copied: %s" % (newname,)) self.c_entryp = newexename self.log.info('usession directory: %s' % (udir,)) self.log.info("created: %s" % (self.c_entryp,)) diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -410,7 +410,7 @@ 'int main(int argc, char* argv[]) ' '{ return $(PYPY_MAIN_FUNCTION)(argc, argv); } > $@') m.rule('$(DEFAULT_TARGET)', ['$(TARGET)', 'main.obj'], - ['$(CC_LINK) /nologo main.obj $(SHARED_IMPORT_LIB) /out:$@ /MANIFEST /MANIFESTFILE:$*.manifest', + ['$(CC_LINK) /nologo /debug main.obj $(SHARED_IMPORT_LIB) /out:$@ /MANIFEST /MANIFESTFILE:$*.manifest', 'mt.exe -nologo -manifest $*.manifest -outputresource:$@;1', ]) m.rule('debugmode_$(DEFAULT_TARGET)', ['debugmode_$(TARGET)', 'main.obj'], diff --git a/rpython/translator/test/test_driver.py b/rpython/translator/test/test_driver.py --- a/rpython/translator/test/test_driver.py +++ b/rpython/translator/test/test_driver.py @@ -55,12 +55,15 @@ src_name = udir.join('src/dydy2.exe') dll_name = udir.join('src/pypy.dll') lib_name = udir.join('src/pypy.lib') + pdb_name = udir.join('src/pypy.pdb') src_name.ensure() src_name.write('exe') dll_name.ensure() dll_name.write('dll') lib_name.ensure() lib_name.write('lib') + pdb_name.ensure() + pdb_name.write('pdb') dst_name.ensure() class CBuilder(object): @@ -75,6 +78,8 @@ assert dst_name.new(purebasename='python27',ext='lib').read() == 'lib' def test_shutil_copy(): + if os.name == 'nt': + py.test.skip('Windows cannot copy or rename to an in-use file') a = udir.join('file_a') b = udir.join('file_a') a.write('hello') From noreply at buildbot.pypy.org Sun Sep 21 10:42:06 2014 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 21 Sep 2014 10:42:06 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: optimize away the trivial case Message-ID: <20140921084206.DC29F1C0130@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73631:539491bcd60f Date: 2014-09-21 10:31 +0200 http://bitbucket.org/pypy/pypy/changeset/539491bcd60f/ Log: optimize away the trivial case diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -212,6 +212,8 @@ o = ResOperation(rop.ARRAYLEN_GC, [v_arr], v_length, descr=arraydescr) self.newops.append(o) + elif isinstance(v_length, ConstInt) and v_length.getint() == 0: + return o = ResOperation(rop.ZERO_ARRAY, [v_arr, ConstInt(0), v_length], None, descr=arraydescr) self.newops.append(o) From noreply at buildbot.pypy.org Sun Sep 21 11:17:43 2014 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 21 Sep 2014 11:17:43 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: one more missing setfield for the asmgcc case Message-ID: <20140921091743.739021C31C8@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73632:9189598fd962 Date: 2014-09-21 11:16 +0200 http://bitbucket.org/pypy/pypy/changeset/9189598fd962/ Log: one more missing setfield for the asmgcc case diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -241,6 +241,9 @@ length_box, descr=descrs.jfi_frame_depth) self.newops.append(op1) + op2 = ResOperation(rop.SETFIELD_GC, [frame, ConstInt(0)], + None, descr=descrs.jf_extra_stack_depth) + self.newops.append(op2) self.gen_initialize_len(frame, length_box, descrs.arraydescr.lendescr) From noreply at buildbot.pypy.org Sun Sep 21 13:06:13 2014 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 21 Sep 2014 13:06:13 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: fixes Message-ID: <20140921110613.A5AF91C0FF8@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73633:86b8d1f0b17d Date: 2014-09-21 13:05 +0200 http://bitbucket.org/pypy/pypy/changeset/86b8d1f0b17d/ Log: fixes diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -62,6 +62,8 @@ if op.is_malloc(): self.handle_malloc_operation(op) continue + if op.is_guard(): + self.emit_pending_zeros() elif op.getopnum() == rop.CLEAR_ARRAY_CONTENTS: self.handle_clear_array_contents(op.getdescr(), op.getarg(0)) continue @@ -136,7 +138,6 @@ self.delayed_zero_setfields[result] = d for ofs in descr.offsets_of_gcfields: d[ofs] = None - self.delayed_zero_setfields.clear() def consider_setfield_gc(self, op): offset = self.cpu.unpack_fielddescr(op.getdescr()) @@ -298,6 +299,7 @@ for ofs in d.iterkeys(): op = ResOperation(rop.ZERO_PTR_FIELD, [v, ConstInt(ofs)], None) self.newops.append(op) + self.delayed_zero_setfields.clear() def _gen_call_malloc_gc(self, args, v_result, descr): """Generate a CALL_MALLOC_GC with the given args.""" diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py --- a/rpython/jit/backend/llsupport/test/test_rewrite.py +++ b/rpython/jit/backend/llsupport/test/test_rewrite.py @@ -122,6 +122,9 @@ def unpack_arraydescr_size(self, d): return 0, d.itemsize, 0 + def unpack_fielddescr(self, d): + return d.offset + def arraydescrof(self, ARRAY): try: return self._cache[ARRAY] diff --git a/rpython/jit/codewriter/heaptracker.py b/rpython/jit/codewriter/heaptracker.py --- a/rpython/jit/codewriter/heaptracker.py +++ b/rpython/jit/codewriter/heaptracker.py @@ -140,6 +140,4 @@ res.append(offset) elif isinstance(FIELD, lltype.Struct): offsets_of_gcfields(gccache, FIELD, res) - if not gccache.translate_support_code: - res.sort() return res From noreply at buildbot.pypy.org Sun Sep 21 13:07:38 2014 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 21 Sep 2014 13:07:38 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: fix the test Message-ID: <20140921110738.6F4561C0FF8@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73634:3b1a1ae57720 Date: 2014-09-21 13:07 +0200 http://bitbucket.org/pypy/pypy/changeset/3b1a1ae57720/ Log: fix the test diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py --- a/rpython/jit/backend/llsupport/test/test_rewrite.py +++ b/rpython/jit/backend/llsupport/test/test_rewrite.py @@ -82,6 +82,7 @@ jfi_frame_depth = framedescrs.jfi_frame_depth jfi_frame_size = framedescrs.jfi_frame_size jf_frame_info = framedescrs.jf_frame_info + jf_extra_stack_depth = framedescrs.jf_extra_stack_depth signedframedescr = self.cpu.signedframedescr floatframedescr = self.cpu.floatframedescr casmdescr.compiled_loop_token = clt @@ -810,6 +811,7 @@ p1 = call_malloc_nursery_varsize_frame(i1) setfield_gc(p1, 0, descr=tiddescr) i2 = getfield_gc(ConstClass(frame_info), descr=jfi_frame_depth) + setfield_gc(p1, 0, descr=jf_extra_stack_depth) setfield_gc(p1, i2, descr=framelendescr) setfield_gc(p1, ConstClass(frame_info), descr=jf_frame_info) setarrayitem_gc(p1, 0, i0, descr=signedframedescr) From noreply at buildbot.pypy.org Sun Sep 21 13:49:44 2014 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 21 Sep 2014 13:49:44 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: extra fields that we zero by hand Message-ID: <20140921114944.1115C1C0130@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73635:255d60019595 Date: 2014-09-21 13:49 +0200 http://bitbucket.org/pypy/pypy/changeset/255d60019595/ Log: extra fields that we zero by hand diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py --- a/rpython/jit/backend/llsupport/gc.py +++ b/rpython/jit/backend/llsupport/gc.py @@ -120,7 +120,8 @@ descrs = JitFrameDescrs() descrs.arraydescr = cpu.arraydescrof(jitframe.JITFRAME) for name in ['jf_descr', 'jf_guard_exc', 'jf_force_descr', - 'jf_frame_info', 'jf_gcmap', 'jf_extra_stack_depth']: + 'jf_frame_info', 'jf_gcmap', 'jf_extra_stack_depth', + 'jf_savedata', 'jf_forward']: setattr(descrs, name, cpu.fielddescrof(jitframe.JITFRAME, name)) descrs.jfi_frame_size = cpu.fielddescrof(jitframe.JITFRAMEINFO, 'jfi_frame_size') diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -215,7 +215,7 @@ self.newops.append(o) elif isinstance(v_length, ConstInt) and v_length.getint() == 0: return - o = ResOperation(rop.ZERO_ARRAY, [v_arr, ConstInt(0), v_length], None, + o = ResOperation(rop.ZERO_ARRAY, [v_arr, self.c_zero, v_length], None, descr=arraydescr) self.newops.append(o) @@ -238,13 +238,21 @@ self.gen_malloc_nursery_varsize_frame(size_box, frame) self.gen_initialize_tid(frame, descrs.arraydescr.tid) length_box = history.BoxInt() - op1 = ResOperation(rop.GETFIELD_GC, [history.ConstInt(frame_info)], - length_box, - descr=descrs.jfi_frame_depth) - self.newops.append(op1) - op2 = ResOperation(rop.SETFIELD_GC, [frame, ConstInt(0)], - None, descr=descrs.jf_extra_stack_depth) - self.newops.append(op2) + # we need to explicitely zero all the gc fields, because + # of the unusal malloc pattern + extra_ops = [ + ResOperation(rop.GETFIELD_GC, [history.ConstInt(frame_info)], + length_box, descr=descrs.jfi_frame_depth), + ResOperation(rop.SETFIELD_GC, [frame, self.c_zero], + None, descr=descrs.jf_extra_stack_depth), + ResOperation(rop.SETFIELD_GC, [frame, self.c_zero], + None, descr=descrs.jf_savedata), + ResOperation(rop.SETFIELD_GC, [frame, self.c_zero], + None, descr=descrs.jf_guard_exc), + ResOperation(rop.SETFIELD_GC, [frame, self.c_zero], + None, descr=descrs.jf_forward), + ] + self.newops += extra_ops self.gen_initialize_len(frame, length_box, descrs.arraydescr.lendescr) From noreply at buildbot.pypy.org Sun Sep 21 16:11:37 2014 From: noreply at buildbot.pypy.org (mattip) Date: Sun, 21 Sep 2014 16:11:37 +0200 (CEST) Subject: [pypy-commit] pypy.org extradoc: reword download link text Message-ID: <20140921141137.D119C1C03FC@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: extradoc Changeset: r538:d4a4df1e1753 Date: 2014-09-21 17:11 +0300 http://bitbucket.org/pypy/pypy.org/changeset/d4a4df1e1753/ Log: reword download link text diff --git a/index.html b/index.html --- a/index.html +++ b/index.html @@ -63,7 +63,7 @@
  • As well as other features.
    • -

    Download and try out the PyPy or PyPy3 release 2.4.0

    +

    Download and try out PyPy

    Want to know more? A good place to start is our detailed speed and compatibility reports!

    diff --git a/source/index.txt b/source/index.txt --- a/source/index.txt +++ b/source/index.txt @@ -26,7 +26,7 @@ .. class:: download -`Download and try out the PyPy or PyPy3 release 2.4.0`__ +`Download and try out PyPy`__ .. __: download.html From noreply at buildbot.pypy.org Mon Sep 22 01:21:17 2014 From: noreply at buildbot.pypy.org (pjenvey) Date: Mon, 22 Sep 2014 01:21:17 +0200 (CEST) Subject: [pypy-commit] pypy py3.3: merge py3k Message-ID: <20140921232117.4AE4E1C0130@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3.3 Changeset: r73636:ac2759ae7eb6 Date: 2014-09-18 21:14 -0700 http://bitbucket.org/pypy/pypy/changeset/ac2759ae7eb6/ Log: merge py3k diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst --- a/pypy/doc/windows.rst +++ b/pypy/doc/windows.rst @@ -85,10 +85,13 @@ Abridged method (for -Ojit builds using Visual Studio 2008) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Download the versions of all the external packages -from +Download the versions of all the external packages from +https://bitbucket.org/pypy/pypy/downloads/local_2.4.zip +(for 2.4 release and later) or https://bitbucket.org/pypy/pypy/downloads/local.zip -Then expand it into the base directory (base_dir) and modify your environment to reflect this:: +(for pre-2.4 versions) +Then expand it into the base directory (base_dir) and modify your environment +to reflect this:: set PATH=\bin;\tcltk\bin;%PATH% set INCLUDE=\include;\tcltk\include;%INCLUDE% diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py --- a/pypy/interpreter/pycode.py +++ b/pypy/interpreter/pycode.py @@ -46,6 +46,7 @@ def cpython_code_signature(code): "([list-of-arg-names], vararg-name-or-None, kwarg-name-or-None)." argcount = code.co_argcount + varnames = code.co_varnames if we_are_translated(): kwonlyargcount = code.co_kwonlyargcount else: @@ -53,16 +54,18 @@ kwonlyargcount = getattr(code, 'co_kwonlyargcount', 0) assert argcount >= 0 # annotator hint assert kwonlyargcount >= 0 - argnames = list(code.co_varnames[:argcount]) - kwonlyargs = list(code.co_varnames[argcount:argcount + kwonlyargcount]) + argnames = list(varnames[:argcount]) + if argcount < len(varnames): + kwonlyargs = list(varnames[argcount:argcount + kwonlyargcount]) + else: + kwonlyargs = None if code.co_flags & CO_VARARGS: - varargname = code.co_varnames[argcount] + varargname = varnames[argcount] argcount += 1 else: varargname = None if code.co_flags & CO_VARKEYWORDS: kwargname = code.co_varnames[argcount + kwonlyargcount] - argcount += 1 else: kwargname = None return Signature(argnames, varargname, kwargname, kwonlyargs) diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py --- a/pypy/interpreter/pyparser/parsestring.py +++ b/pypy/interpreter/pyparser/parsestring.py @@ -82,12 +82,6 @@ v = PyString_DecodeEscape(space, substr, 'strict', encoding) return space.wrapbytes(v) -def hexbyte(val): - result = "%x" % val - if len(result) == 1: - result = "0" + result - return result - def decode_unicode_utf8(space, s, ps, q): # ****The Python 2.7 version, producing UTF-32 escapes**** # String is utf8-encoded, but 'unicode_escape' expects @@ -107,15 +101,14 @@ # instead. lis.append("u005c") if ord(s[ps]) & 0x80: # XXX inefficient - w, ps = decode_utf8(space, s, ps, end, "utf-32-be") - rn = len(w) - assert rn % 4 == 0 - for i in range(0, rn, 4): - lis.append('\\U') - lis.append(hexbyte(ord(w[i]))) - lis.append(hexbyte(ord(w[i+1]))) - lis.append(hexbyte(ord(w[i+2]))) - lis.append(hexbyte(ord(w[i+3]))) + w, ps = decode_utf8(space, s, ps, end) + for c in w: + # The equivalent of %08x, which is not supported by RPython. + # 7 zeroes are enough for the unicode range, and the + # result still fits in 32-bit. + hexa = hex(ord(c) + 0x10000000) + lis.append('\\U0') + lis.append(hexa[3:]) # Skip 0x and the leading 1 else: lis.append(s[ps]) ps += 1 @@ -135,7 +128,7 @@ # note that the C code has a label here. # the logic is the same. if recode_encoding and ord(s[ps]) & 0x80: - w, ps = decode_utf8(space, s, ps, end, recode_encoding) + w, ps = decode_utf8_recode(space, s, ps, end, recode_encoding) # Append bytes to output buffer. builder.append(w) else: @@ -222,14 +215,18 @@ ch >= 'A' and ch <= 'F') -def decode_utf8(space, s, ps, end, encoding): +def decode_utf8(space, s, ps, end): assert ps >= 0 pt = ps # while (s < end && *s != '\\') s++; */ /* inefficient for u".." while ps < end and ord(s[ps]) & 0x80: ps += 1 - w_u = space.wrap(unicodehelper.decode_utf8(space, s[pt:ps])) - w_v = unicodehelper.encode(space, w_u, encoding) + u = unicodehelper.decode_utf8(space, s[pt:ps]) + return u, ps + +def decode_utf8_recode(space, s, ps, end, recode_encoding): + u, ps = decode_utf8(space, s, ps, end) + w_v = unicodehelper.encode(space, space.wrap(u), recode_encoding) v = space.bytes_w(w_v) return v, ps diff --git a/pypy/interpreter/pyparser/test/test_parsestring.py b/pypy/interpreter/pyparser/test/test_parsestring.py --- a/pypy/interpreter/pyparser/test/test_parsestring.py +++ b/pypy/interpreter/pyparser/test/test_parsestring.py @@ -100,11 +100,11 @@ def test_simple_enc_roundtrip(self): space = self.space - s = "'\x81'" + s = "'\x81\\t'" s = s.decode("koi8-u").encode("utf8") w_ret = parsestring.parsestr(self.space, 'koi8-u', s) ret = space.unwrap(w_ret) - assert ret == eval("# -*- coding: koi8-u -*-\nu'\x81'") + assert ret == eval("# -*- coding: koi8-u -*-\nu'\x81\\t'") def test_multiline_unicode_strings_with_backslash(self): space = self.space diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py --- a/pypy/interpreter/test/test_app_main.py +++ b/pypy/interpreter/test/test_app_main.py @@ -1064,7 +1064,7 @@ prefix = udir.join('pathtest').ensure(dir=1) fake_exe = 'bin/pypy-c' if sys.platform == 'win32': - fake_exe += '.exe' + fake_exe = 'pypy-c.exe' fake_exe = prefix.join(fake_exe).ensure(file=1) expected_path = [str(prefix.join(subdir).ensure(dir=1)) for subdir in ('lib_pypy', @@ -1104,8 +1104,10 @@ assert sys.path == old_sys_path + [self.goal_dir] app_main.setup_bootstrap_path(self.fake_exe) - assert sys.executable == '' # not executable! - assert sys.path == old_sys_path + [self.goal_dir] + if not sys.platform == 'win32': + # an existing file is always 'executable' on windows + assert sys.executable == '' # not executable! + assert sys.path == old_sys_path + [self.goal_dir] os.chmod(self.fake_exe, 0o755) app_main.setup_bootstrap_path(self.fake_exe) diff --git a/pypy/interpreter/test/test_code.py b/pypy/interpreter/test/test_code.py --- a/pypy/interpreter/test/test_code.py +++ b/pypy/interpreter/test/test_code.py @@ -194,3 +194,9 @@ # CO_NESTED assert d['f'](4).__code__.co_flags & 0x10 assert d['f'].__code__.co_flags & 0x10 == 0 + + def test_issue1844(self): + import types + args = (1, 0, 1, 0, 0, b'', (), (), (), '', 'operator', 0, b'') + # previously raised a MemoryError when translated + types.CodeType(*args) diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py --- a/pypy/interpreter/unicodehelper.py +++ b/pypy/interpreter/unicodehelper.py @@ -13,6 +13,7 @@ @specialize.memo() def decode_error_handler(space): + # Fast version of the "strict" errors handler. def raise_unicode_exception_decode(errors, encoding, msg, s, startingpos, endingpos): raise OperationError(space.w_UnicodeDecodeError, @@ -25,6 +26,7 @@ @specialize.memo() def encode_error_handler(space): + # Fast version of the "strict" errors handler. def raise_unicode_exception_encode(errors, encoding, msg, u, startingpos, endingpos): raise OperationError(space.w_UnicodeEncodeError, diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py --- a/pypy/module/_cffi_backend/ctypefunc.py +++ b/pypy/module/_cffi_backend/ctypefunc.py @@ -8,6 +8,7 @@ from rpython.rlib.jit_libffi import (CIF_DESCRIPTION, CIF_DESCRIPTION_P, FFI_TYPE, FFI_TYPE_P, FFI_TYPE_PP, SIZE_OF_FFI_ARG) from rpython.rlib.objectmodel import we_are_translated, instantiate +from rpython.rlib.objectmodel import keepalive_until_here from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from pypy.interpreter.error import OperationError, oefmt @@ -160,6 +161,7 @@ raw_cdata = rffi.cast(rffi.CCHARPP, data)[0] lltype.free(raw_cdata, flavor='raw') lltype.free(buffer, flavor='raw') + keepalive_until_here(args_w) return w_res def get_mustfree_flag(data): diff --git a/pypy/module/operator/app_operator.py b/pypy/module/operator/app_operator.py --- a/pypy/module/operator/app_operator.py +++ b/pypy/module/operator/app_operator.py @@ -4,7 +4,8 @@ This module exports a set of operators as functions. E.g. operator.add(x,y) is equivalent to x+y. ''' -from __pypy__ import builtinify + +import types def countOf(a,b): @@ -15,51 +16,78 @@ count += 1 return count +def _resolve_attr_chain(chain, obj, idx=0): + obj = getattr(obj, chain[idx]) + if idx + 1 == len(chain): + return obj + else: + return _resolve_attr_chain(chain, obj, idx + 1) + + +class _simple_attrgetter(object): + def __init__(self, attr): + self._attr = attr + + def __call__(self, obj): + return getattr(obj, self._attr) + + +class _single_attrgetter(object): + def __init__(self, attrs): + self._attrs = attrs + + def __call__(self, obj): + return _resolve_attr_chain(self._attrs, obj) + + +class _multi_attrgetter(object): + def __init__(self, attrs): + self._attrs = attrs + + def __call__(self, obj): + return tuple([ + _resolve_attr_chain(attrs, obj) + for attrs in self._attrs + ]) + + def attrgetter(attr, *attrs): + if ( + not isinstance(attr, str) or + not all(isinstance(a, str) for a in attrs) + ): + raise TypeError("attribute name must be a string, not %r" % + type(attr).__name__) if attrs: - getters = [single_attr_getter(a) for a in (attr,) + attrs] - def getter(obj): - return tuple([getter(obj) for getter in getters]) + return _multi_attrgetter([ + a.split(".") for a in [attr] + list(attrs) + ]) + elif "." not in attr: + return _simple_attrgetter(attr) else: - getter = single_attr_getter(attr) - return builtinify(getter) + return _single_attrgetter(attr.split(".")) -def single_attr_getter(attr): - if not isinstance(attr, str): - raise TypeError("attribute name must be a string, not {!r}".format( - type(attr).__name__)) - # - def make_getter(name, prevfn=None): - if prevfn is None: - def getter(obj): - return getattr(obj, name) + +class itemgetter(object): + def __init__(self, item, *items): + self._single = not bool(items) + if self._single: + self._idx = item else: - def getter(obj): - return getattr(prevfn(obj), name) - return getter - # - last = 0 - getter = None - while True: - dot = attr.find(".", last) - if dot < 0: break - getter = make_getter(attr[last:dot], getter) - last = dot + 1 - return make_getter(attr[last:], getter) + self._idx = [item] + list(items) + def __call__(self, obj): + if self._single: + return obj[self._idx] + else: + return tuple([obj[i] for i in self._idx]) -def itemgetter(item, *items): - if items: - list_of_indices = [item] + list(items) - def getter(obj): - return tuple([obj[i] for i in list_of_indices]) - else: - def getter(obj): - return obj[item] - return builtinify(getter) +class methodcaller(object): + def __init__(self, method_name, *args, **kwargs): + self._method_name = method_name + self._args = args + self._kwargs = kwargs -def methodcaller(method_name, *args, **kwargs): - def call(obj): - return getattr(obj, method_name)(*args, **kwargs) - return builtinify(call) + def __call__(self, obj): + return getattr(obj, self._method_name)(*self._args, **self._kwargs) diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py --- a/pypy/module/posix/__init__.py +++ b/pypy/module/posix/__init__.py @@ -168,6 +168,11 @@ # not visible via os, inconsistency in nt: if hasattr(posix, '_getfullpathname'): interpleveldefs['_getfullpathname'] = 'interp_posix._getfullpathname' + if os.name == 'nt': + interpleveldefs.update({ + '_getfileinformation': 'interp_posix._getfileinformation', + '_getfinalpathname': 'interp_posix._getfinalpathname', + }) if hasattr(os, 'chroot'): interpleveldefs['chroot'] = 'interp_posix.chroot' diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py --- a/pypy/module/posix/interp_posix.py +++ b/pypy/module/posix/interp_posix.py @@ -5,8 +5,7 @@ from rpython.rlib.objectmodel import specialize from rpython.rlib.rarithmetic import r_longlong from rpython.rlib.unroll import unrolling_iterable -from rpython.rtyper.module import ll_os_stat -from rpython.rtyper.module.ll_os import RegisterOs +from rpython.rtyper.module import ll_os, ll_os_stat from pypy.interpreter.gateway import unwrap_spec, WrappedDefault from pypy.interpreter.error import OperationError, wrap_oserror, wrap_oserror2 @@ -1234,7 +1233,7 @@ raise wrap_oserror(space, e) def declare_new_w_star(name): - if name in RegisterOs.w_star_returning_int: + if name in ll_os.RegisterOs.w_star_returning_int: @unwrap_spec(status=c_int) def WSTAR(space, status): return space.wrap(getattr(os, name)(status)) @@ -1246,7 +1245,7 @@ WSTAR.func_name = name return WSTAR -for name in RegisterOs.w_star: +for name in ll_os.RegisterOs.w_star: if hasattr(os, name): func = declare_new_w_star(name) globals()[name] = func @@ -1412,3 +1411,25 @@ if codeset: return space.wrap(codeset) return space.w_None + +if _WIN32: + @unwrap_spec(fd=c_int) + def _getfileinformation(space, fd): + try: + info = ll_os._getfileinformation(fd) + except OSError as e: + raise wrap_oserror(space, e) + return space.newtuple([space.wrap(info[0]), + space.wrap(info[1]), + space.wrap(info[2])]) + + def _getfinalpathname(space, w_path): + path = space.unicode_w(w_path) + try: + result = ll_os._getfinalpathname(path) + except ll_os.LLNotImplemented as e: + raise OperationError(space.w_NotImplementedError, + space.wrap(e.msg)) + except OSError as e: + raise wrap_oserror2(space, e, w_path) + return space.wrap(result) diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -1069,6 +1069,24 @@ # just ensure it returns something reasonable assert encoding is None or type(encoding) is str + if os.name == 'nt': + def test__getfileinformation(self): + import os + path = os.path.join(self.pdir, 'file1') + with open(path) as fp: + info = self.posix._getfileinformation(fp.fileno()) + assert len(info) == 3 + assert all(isinstance(obj, int) for obj in info) + + def test__getfinalpathname(self): + import os + path = os.path.join(self.pdir, 'file1') + try: + result = self.posix._getfinalpathname(path) + except NotImplementedError: + skip("_getfinalpathname not supported on this platform") + assert os.path.exists(result) + def test_rtld_constants(self): # check presence of major RTLD_* constants self.posix.RTLD_LAZY @@ -1076,6 +1094,7 @@ self.posix.RTLD_GLOBAL self.posix.RTLD_LOCAL + class AppTestEnvironment(object): def setup_class(cls): cls.space = space diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py --- a/pypy/module/sys/test/test_sysmodule.py +++ b/pypy/module/sys/test/test_sysmodule.py @@ -127,9 +127,9 @@ assert isinstance(sys.__stdin__, io.IOBase) assert sys.__stderr__.errors == 'backslashreplace' - assert sys.__stdin__.name == "" - assert sys.__stdout__.name == "" - assert sys.__stderr__.name == "" + #assert sys.__stdin__.name == "" + #assert sys.__stdout__.name == "" + #assert sys.__stderr__.name == "" if self.appdirect and not isinstance(sys.stdin, io.IOBase): return diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -306,10 +306,10 @@ import imp argparse = imp.load_source('argparse', 'lib-python/2.7/argparse.py') if sys.platform == 'win32': - pypy_exe = 'pypy.exe' + pypy_exe = 'pypy3.exe' license_base = os.path.join(basedir, r'..\..\..\local') # as on buildbot YMMV else: - pypy_exe = 'pypy' + pypy_exe = 'pypy3' license_base = '/usr/share/doc' parser = argparse.ArgumentParser() args = list(args) diff --git a/rpython/jit/backend/arm/callbuilder.py b/rpython/jit/backend/arm/callbuilder.py --- a/rpython/jit/backend/arm/callbuilder.py +++ b/rpython/jit/backend/arm/callbuilder.py @@ -92,7 +92,8 @@ self.mc.LDR_ri(r.r7.value, r.r5.value) # change 'rpy_fastgil' to 0 (it should be non-zero right now) - self.mc.DMB() + if self.asm.cpu.cpuinfo.arch_version >= 7: + self.mc.DMB() self.mc.gen_load_int(r.r6.value, fastgil) self.mc.MOV_ri(r.ip.value, 0) self.mc.STR_ri(r.ip.value, r.r6.value) @@ -112,7 +113,8 @@ self.mc.STREX(r.r3.value, r.ip.value, r.r6.value, c=c.EQ) # try to claim the lock self.mc.CMP_ri(r.r3.value, 0, cond=c.EQ) # did this succeed? - self.mc.DMB() + if self.asm.cpu.cpuinfo.arch_version >= 7: + self.mc.DMB() # the success of the lock acquisition is defined by # 'EQ is true', or equivalently by 'r3 == 0'. # diff --git a/rpython/jit/backend/arm/codebuilder.py b/rpython/jit/backend/arm/codebuilder.py --- a/rpython/jit/backend/arm/codebuilder.py +++ b/rpython/jit/backend/arm/codebuilder.py @@ -333,6 +333,8 @@ | (rn & 0xF) << 16) def DMB(self): + # ARMv7 only. I guess ARMv6 CPUs cannot be used in symmetric + # multi-processing at all? That would make this instruction unneeded. # note: 'cond' is only permitted on Thumb here, but don't # write literally 0xf57ff05f, because it's larger than 31 bits c = cond.AL diff --git a/rpython/jit/backend/arm/instructions.py b/rpython/jit/backend/arm/instructions.py --- a/rpython/jit/backend/arm/instructions.py +++ b/rpython/jit/backend/arm/instructions.py @@ -142,6 +142,7 @@ #'VCVT' : {'opc1':0xB, 'opc2':0xE, 'opc3':0x1, 'base': False}, } +# ARMv7 only simd_instructions_3regs = { 'VADD_i64': {'A': 0x8, 'B': 0, 'U': 0}, 'VSUB_i64': {'A': 0x8, 'B': 0, 'U': 1}, diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -1,3 +1,4 @@ +from rpython.rlib import rgc from rpython.rlib.rarithmetic import ovfcheck from rpython.rtyper.lltypesystem import llmemory from rpython.jit.metainterp import history @@ -390,8 +391,8 @@ val = op.getarg(0) if val not in self.write_barrier_applied: v = op.getarg(1) - if isinstance(v, BoxPtr) or (isinstance(v, ConstPtr) and - bool(v.value)): # store a non-NULL + if (isinstance(v, BoxPtr) or (isinstance(v, ConstPtr) and + rgc.needs_write_barrier(v.value))): self.gen_write_barrier(val) #op = op.copy_and_change(rop.SETFIELD_RAW) self.newops.append(op) @@ -400,8 +401,8 @@ val = op.getarg(0) if val not in self.write_barrier_applied: v = op.getarg(2) - if isinstance(v, BoxPtr) or (isinstance(v, ConstPtr) and - bool(v.value)): # store a non-NULL + if (isinstance(v, BoxPtr) or (isinstance(v, ConstPtr) and + rgc.needs_write_barrier(v.value))): self.gen_write_barrier_array(val, op.getarg(1)) #op = op.copy_and_change(rop.SET{ARRAYITEM,INTERIORFIELD}_RAW) self.newops.append(op) diff --git a/rpython/rlib/rdynload.py b/rpython/rlib/rdynload.py --- a/rpython/rlib/rdynload.py +++ b/rpython/rlib/rdynload.py @@ -158,3 +158,4 @@ return res LoadLibrary = rwin32.LoadLibrary + GetModuleHandle = rwin32.GetModuleHandle diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py --- a/rpython/rlib/rgc.py +++ b/rpython/rlib/rgc.py @@ -86,6 +86,14 @@ collect(i) i += 1 +def needs_write_barrier(obj): + """ We need to emit write barrier if the right hand of assignment + is in nursery, used by the JIT for handling set*_gc(Const) + """ + if not obj: + return False + return can_move(obj) + def _heap_stats(): raise NotImplementedError # can't be run directly diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py --- a/rpython/rlib/rwin32.py +++ b/rpython/rlib/rwin32.py @@ -130,6 +130,7 @@ # is hidden by operations in ll2ctypes. Call it now. GetLastError() + GetModuleHandle = winexternal('GetModuleHandleA', [rffi.CCHARP], HMODULE) LoadLibrary = winexternal('LoadLibraryA', [rffi.CCHARP], HMODULE) GetProcAddress = winexternal('GetProcAddress', [HMODULE, rffi.CCHARP], diff --git a/rpython/rtyper/lltypesystem/ll2ctypes.py b/rpython/rtyper/lltypesystem/ll2ctypes.py --- a/rpython/rtyper/lltypesystem/ll2ctypes.py +++ b/rpython/rtyper/lltypesystem/ll2ctypes.py @@ -361,7 +361,9 @@ functype = ctypes.CFUNCTYPE if sys.platform == 'win32': from rpython.rlib.clibffi import FFI_STDCALL, FFI_DEFAULT_ABI - if getattr(T.TO, 'ABI', FFI_DEFAULT_ABI) == FFI_STDCALL: + # XXX: + #if getattr(T.TO, 'ABI', FFI_DEFAULT_ABI) == FFI_STDCALL: + if getattr(T.TO, 'ABI', FFI_DEFAULT_ABI) == 'FFI_STDCALL': # for win32 system call functype = ctypes.WINFUNCTYPE argtypes = [get_ctypes_type(ARG) for ARG in T.TO.ARGS diff --git a/rpython/rtyper/lltypesystem/lltype.py b/rpython/rtyper/lltypesystem/lltype.py --- a/rpython/rtyper/lltypesystem/lltype.py +++ b/rpython/rtyper/lltypesystem/lltype.py @@ -1,16 +1,17 @@ -from types import NoneType, MethodType import weakref +from types import MethodType, NoneType + +from rpython.annotator.bookkeeper import analyzer_for, immutablevalue from rpython.annotator.model import ( - SomeInteger, SomeBool, SomeObject, AnnotatorError) + AnnotatorError, SomeBool, SomeInteger, SomeObject) +from rpython.rlib.objectmodel import Symbolic from rpython.rlib.rarithmetic import ( - r_int, r_uint, intmask, r_singlefloat, r_ulonglong, r_longlong, - r_longfloat, r_longlonglong, base_int, normalizedinttype, longlongmask, - longlonglongmask, maxint, is_valid_int, is_emulated_long) -from rpython.rlib.objectmodel import Symbolic + base_int, intmask, is_emulated_long, is_valid_int, longlonglongmask, + longlongmask, maxint, normalizedinttype, r_int, r_longfloat, r_longlong, + r_longlonglong, r_singlefloat, r_uint, r_ulonglong) +from rpython.rtyper.extregistry import ExtRegistryEntry +from rpython.tool import leakfinder from rpython.tool.identity_dict import identity_dict -from rpython.tool import leakfinder -from rpython.annotator.bookkeeper import analyzer_for, immutablevalue -from rpython.rtyper.extregistry import ExtRegistryEntry class State(object): pass @@ -313,14 +314,12 @@ except KeyError: return ContainerType.__getattr__(self, name) - def _nofield(self, name): raise AttributeError('struct %s has no field %r' % (self._name, name)) def _names_without_voids(self): - names_without_voids = [name for name in self._names if self._flds[name] is not Void] - return names_without_voids + return [name for name in self._names if self._flds[name] is not Void] def _str_fields_without_voids(self): return ', '.join(['%s: %s' % (name, self._flds[name]) @@ -576,8 +575,10 @@ _gckind = 'raw' def __init__(self, tag, hints={}): - """ if hints['render_structure'] is set, the type is internal and not considered - to come from somewhere else (it should be rendered as a structure) """ + """If hints['render_structure'] is set, the type is internal and + not considered to come from somewhere else (it should be + rendered as a structure) + """ self.tag = tag self.__name__ = tag self.hints = frozendict(hints) @@ -675,7 +676,8 @@ _numbertypes = {int: Number("Signed", int, intmask)} _numbertypes[r_int] = _numbertypes[int] -_numbertypes[r_longlonglong] = Number("SignedLongLongLong", r_longlonglong, longlonglongmask) +_numbertypes[r_longlonglong] = Number("SignedLongLongLong", r_longlonglong, + longlonglongmask) if r_longlong is not r_int: _numbertypes[r_longlong] = Number("SignedLongLong", r_longlong, longlongmask) @@ -702,8 +704,8 @@ UnsignedLongLong = build_number("UnsignedLongLong", r_ulonglong) Float = Primitive("Float", 0.0) # C type 'double' -SingleFloat = Primitive("SingleFloat", r_singlefloat(0.0)) # C type 'float' -LongFloat = Primitive("LongFloat", r_longfloat(0.0)) # C type 'long double' +SingleFloat = Primitive("SingleFloat", r_singlefloat(0.0)) # 'float' +LongFloat = Primitive("LongFloat", r_longfloat(0.0)) # 'long double' r_singlefloat._TYPE = SingleFloat Char = Primitive("Char", '\x00') @@ -876,9 +878,11 @@ @analyzer_for(cast_primitive) def ann_cast_primitive(T, s_v): - from rpython.rtyper.llannotation import annotation_to_lltype, ll_to_annotation + from rpython.rtyper.llannotation import ( + annotation_to_lltype, ll_to_annotation) assert T.is_constant() - return ll_to_annotation(cast_primitive(T.const, annotation_to_lltype(s_v)._defl())) + return ll_to_annotation(cast_primitive(T.const, + annotation_to_lltype(s_v)._defl())) def _cast_whatever(TGT, value): @@ -905,7 +909,8 @@ elif TGT == llmemory.Address and isinstance(ORIG, Ptr): return llmemory.cast_ptr_to_adr(value) elif TGT == Signed and isinstance(ORIG, Ptr) and ORIG.TO._gckind == 'raw': - return llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(value), 'symbolic') + return llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(value), + 'symbolic') raise TypeError("don't know how to cast from %r to %r" % (ORIG, TGT)) @@ -1176,8 +1181,8 @@ except DelayedPointer: return True # assume it's not a delayed null - # _setobj, _getobj and _obj0 are really _internal_ implementations details of _ptr, - # use _obj if necessary instead ! + # _setobj, _getobj and _obj0 are really _internal_ implementations + # details of _ptr, use _obj if necessary instead ! def _setobj(self, pointing_to, solid=False): if pointing_to is None: obj0 = None @@ -1244,12 +1249,12 @@ if T1 == T2: setattr(self._obj, field_name, val) else: - raise TypeError("%r instance field %r:\n" - "expects %r\n" - " got %r" % (self._T, field_name, T1, T2)) + raise TypeError( + "%r instance field %r:\nexpects %r\n got %r" % + (self._T, field_name, T1, T2)) return - raise AttributeError("%r instance has no field %r" % (self._T, - field_name)) + raise AttributeError("%r instance has no field %r" % + (self._T, field_name)) def __getitem__(self, i): # ! can only return basic or ptr ! if isinstance(self._T, (Array, FixedSizeArray)): @@ -1266,7 +1271,8 @@ if isinstance(self._T, (Array, FixedSizeArray)): T1 = self._T.OF if isinstance(T1, ContainerType): - raise TypeError("cannot directly assign to container array items") + raise TypeError("cannot directly assign to container array " + "items") T2 = typeOf(val) if T2 != T1: from rpython.rtyper.lltypesystem import rffi @@ -1316,7 +1322,8 @@ from rpython.rtyper.lltypesystem import rffi if isinstance(self._T, FuncType): if len(args) != len(self._T.ARGS): - raise TypeError("calling %r with wrong argument number: %r" % (self._T, args)) + raise TypeError("calling %r with wrong argument number: %r" % + (self._T, args)) for i, a, ARG in zip(range(len(self._T.ARGS)), args, self._T.ARGS): if typeOf(a) != ARG: # ARG could be Void @@ -1415,11 +1422,13 @@ raise RuntimeError("widening to trash: %r" % self) PARENTTYPE = struc._parent_type if getattr(parent, PARENTTYPE._names[0]) != struc: - raise InvalidCast(CURTYPE, PTRTYPE) # xxx different exception perhaps? + # xxx different exception perhaps? + raise InvalidCast(CURTYPE, PTRTYPE) struc = parent u -= 1 if PARENTTYPE != PTRTYPE.TO: - raise RuntimeError("widening %r inside %r instead of %r" % (CURTYPE, PARENTTYPE, PTRTYPE.TO)) + raise RuntimeError("widening %r inside %r instead of %r" % + (CURTYPE, PARENTTYPE, PTRTYPE.TO)) return _ptr(PTRTYPE, struc, solid=self._solid) def _cast_to_int(self, check=True): @@ -1430,7 +1439,9 @@ return obj # special case for cast_int_to_ptr() results obj = normalizeptr(self, check)._getobj(check) if isinstance(obj, int): - return obj # special case for cast_int_to_ptr() results put into opaques + # special case for cast_int_to_ptr() results put into + # opaques + return obj if getattr(obj, '_read_directly_intval', False): return obj.intval # special case for _llgcopaque result = intmask(obj._getid()) @@ -1468,7 +1479,8 @@ """XXX A nice docstring here""" T = typeOf(val) if isinstance(T, ContainerType): - if self._T._gckind == 'gc' and T._gckind == 'raw' and not isinstance(T, OpaqueType): + if (self._T._gckind == 'gc' and T._gckind == 'raw' and + not isinstance(T, OpaqueType)): val = _interior_ptr(T, self._obj, [offset]) else: val = _ptr(Ptr(T), val, solid=self._solid) @@ -1531,12 +1543,14 @@ setattr(example, s_attr.const, v_lltype._defl()) def call(self, args): - from rpython.rtyper.llannotation import annotation_to_lltype, ll_to_annotation + from rpython.rtyper.llannotation import ( + annotation_to_lltype, ll_to_annotation) args_s, kwds_s = args.unpack() if kwds_s: raise Exception("keyword arguments to call to a low-level fn ptr") info = 'argument to ll function pointer call' - llargs = [annotation_to_lltype(s_arg, info)._defl() for s_arg in args_s] + llargs = [annotation_to_lltype(s_arg, info)._defl() + for s_arg in args_s] v = self.ll_ptrtype._example()(*llargs) return ll_to_annotation(v) @@ -1593,7 +1607,6 @@ return val - assert not '__dict__' in dir(_interior_ptr) class _container(object): @@ -1721,11 +1734,13 @@ __slots__ = ('_hash_cache_', '_compilation_info') - def __new__(self, TYPE, n=None, initialization=None, parent=None, parentindex=None): + def __new__(self, TYPE, n=None, initialization=None, parent=None, + parentindex=None): my_variety = _struct_variety(TYPE._names) return object.__new__(my_variety) - def __init__(self, TYPE, n=None, initialization=None, parent=None, parentindex=None): + def __init__(self, TYPE, n=None, initialization=None, parent=None, + parentindex=None): _parentable.__init__(self, TYPE) if n is not None and TYPE._arrayfld is None: raise TypeError("%r is not variable-sized" % (TYPE,)) @@ -1734,9 +1749,11 @@ first, FIRSTTYPE = TYPE._first_struct() for fld, typ in TYPE._flds.items(): if fld == TYPE._arrayfld: - value = _array(typ, n, initialization=initialization, parent=self, parentindex=fld) + value = _array(typ, n, initialization=initialization, + parent=self, parentindex=fld) else: - value = typ._allocate(initialization=initialization, parent=self, parentindex=fld) + value = typ._allocate(initialization=initialization, + parent=self, parentindex=fld) setattr(self, fld, value) if parent is not None: self._setparentstructure(parent, parentindex) @@ -1795,7 +1812,8 @@ __slots__ = ('items',) - def __init__(self, TYPE, n, initialization=None, parent=None, parentindex=None): + def __init__(self, TYPE, n, initialization=None, parent=None, + parentindex=None): if not is_valid_int(n): raise TypeError("array length must be an int") if n < 0: @@ -1964,7 +1982,8 @@ if not key._was_freed(): newcache[key] = value except RuntimeError: - pass # ignore "accessing subxxx, but already gc-ed parent" + # ignore "accessing subxxx, but already gc-ed parent" + pass if newcache: _subarray._cache[T] = newcache else: @@ -2020,8 +2039,10 @@ attrs.setdefault('_name', '?') attrs.setdefault('_callable', None) self.__dict__.update(attrs) - if '_callable' in attrs and hasattr(attrs['_callable'], '_compilation_info'): - self.__dict__['compilation_info'] = attrs['_callable']._compilation_info + if '_callable' in attrs and hasattr(attrs['_callable'], + '_compilation_info'): + self.__dict__['compilation_info'] = \ + attrs['_callable']._compilation_info def __repr__(self): return '<%s>' % (self,) @@ -2126,8 +2147,8 @@ return _ptr(Ptr(T), o, solid) @analyzer_for(malloc) -def ann_malloc(s_T, s_n=None, s_flavor=None, s_zero=None, s_track_allocation=None, - s_add_memory_pressure=None): +def ann_malloc(s_T, s_n=None, s_flavor=None, s_zero=None, + s_track_allocation=None, s_add_memory_pressure=None): assert (s_n is None or s_n.knowntype == int or issubclass(s_n.knowntype, base_int)) assert s_T.is_constant() @@ -2303,7 +2324,8 @@ @analyzer_for(runtime_type_info) def ann_runtime_type_info(s_p): - assert isinstance(s_p, SomePtr), "runtime_type_info of non-pointer: %r" % s_p + assert isinstance(s_p, SomePtr), \ + "runtime_type_info of non-pointer: %r" % s_p return SomePtr(typeOf(runtime_type_info(s_p.ll_ptrtype._example()))) diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py --- a/rpython/rtyper/module/ll_os.py +++ b/rpython/rtyper/module/ll_os.py @@ -105,6 +105,12 @@ _CYGWIN = sys.platform == 'cygwin' +# plain NotImplementedError is invalid RPython +class LLNotImplemented(NotImplementedError): + + def __init__(self, msg): + self.msg = msg + class CConfig: """ Definitions for platform integration. @@ -1179,7 +1185,7 @@ condition=sys.platform=='win32') def register_posix__getfullpathname(self, traits): # this nt function is not exposed via os, but needed - # to get a correct implementation of os.abspath + # to get a correct implementation of os.path.abspath from rpython.rtyper.module.ll_win32file import make_getfullpathname_impl getfullpathname_llimpl = make_getfullpathname_impl(traits) @@ -1963,10 +1969,12 @@ return OsEnvironController() # ____________________________________________________________ -# Support for the WindowsError exception +# Support for the WindowsError exception and misc functions if sys.platform == 'win32': from rpython.rlib import rwin32 + from rpython.rtyper.module.ll_win32file import ( + make__getfileinformation_impl, make__getfinalpathname_impl) class RegisterFormatError(BaseLazyRegistering): def __init__(self): @@ -1977,3 +1985,6 @@ return extdef([lltype.Signed], str, "rwin32_FormatError", llimpl=rwin32.llimpl_FormatError) + + _getfileinformation = make__getfileinformation_impl(UnicodeTraits()) + _getfinalpathname = make__getfinalpathname_impl(UnicodeTraits()) diff --git a/rpython/rtyper/module/ll_os_stat.py b/rpython/rtyper/module/ll_os_stat.py --- a/rpython/rtyper/module/ll_os_stat.py +++ b/rpython/rtyper/module/ll_os_stat.py @@ -186,7 +186,10 @@ _name_struct_stat = '_stati64' INCLUDES = ['sys/types.h', 'sys/stat.h', 'sys/statvfs.h'] else: - _name_struct_stat = 'stat' + if sys.platform.startswith('linux'): + _name_struct_stat = 'stat64' + else: + _name_struct_stat = 'stat' INCLUDES = ['sys/types.h', 'sys/stat.h', 'sys/statvfs.h', 'unistd.h'] compilation_info = ExternalCompilationInfo( diff --git a/rpython/rtyper/module/ll_win32file.py b/rpython/rtyper/module/ll_win32file.py --- a/rpython/rtyper/module/ll_win32file.py +++ b/rpython/rtyper/module/ll_win32file.py @@ -55,6 +55,15 @@ FILE_TYPE_CHAR = platform.ConstantInteger('FILE_TYPE_CHAR') FILE_TYPE_PIPE = platform.ConstantInteger('FILE_TYPE_PIPE') + FILE_WRITE_ATTRIBUTES = platform.ConstantInteger( + 'FILE_WRITE_ATTRIBUTES') + OPEN_EXISTING = platform.ConstantInteger( + 'OPEN_EXISTING') + FILE_FLAG_BACKUP_SEMANTICS = platform.ConstantInteger( + 'FILE_FLAG_BACKUP_SEMANTICS') + VOLUME_NAME_DOS = platform.ConstantInteger('VOLUME_NAME_DOS') + VOLUME_NAME_NT = platform.ConstantInteger('VOLUME_NAME_NT') + WIN32_FILE_ATTRIBUTE_DATA = platform.Struct( 'WIN32_FILE_ATTRIBUTE_DATA', [('dwFileAttributes', rwin32.DWORD), @@ -67,14 +76,15 @@ BY_HANDLE_FILE_INFORMATION = platform.Struct( 'BY_HANDLE_FILE_INFORMATION', [('dwFileAttributes', rwin32.DWORD), + ('ftCreationTime', rwin32.FILETIME), + ('ftLastAccessTime', rwin32.FILETIME), + ('ftLastWriteTime', rwin32.FILETIME), + ('dwVolumeSerialNumber', rwin32.DWORD), ('nFileSizeHigh', rwin32.DWORD), ('nFileSizeLow', rwin32.DWORD), ('nNumberOfLinks', rwin32.DWORD), ('nFileIndexHigh', rwin32.DWORD), - ('nFileIndexLow', rwin32.DWORD), - ('ftCreationTime', rwin32.FILETIME), - ('ftLastAccessTime', rwin32.FILETIME), - ('ftLastWriteTime', rwin32.FILETIME)]) + ('nFileIndexLow', rwin32.DWORD)]) config = platform.configure(CConfig) @@ -92,6 +102,8 @@ INVALID_FILE_ATTRIBUTES _S_IFDIR _S_IFREG _S_IFCHR _S_IFIFO FILE_TYPE_UNKNOWN FILE_TYPE_CHAR FILE_TYPE_PIPE + FILE_WRITE_ATTRIBUTES OPEN_EXISTING FILE_FLAG_BACKUP_SEMANTICS + VOLUME_NAME_DOS VOLUME_NAME_NT ERROR_FILE_NOT_FOUND ERROR_NO_MORE_FILES ERROR_SHARING_VIOLATION '''.split(): @@ -163,6 +175,13 @@ [traits.CCHARP, traits.CCHARP], rwin32.BOOL) + CreateFile = external( + 'CreateFile' + apisuffix, + [traits.CCHARP, rwin32.DWORD, rwin32.DWORD, + rwin32.LPSECURITY_ATTRIBUTES, rwin32.DWORD, rwin32.DWORD, + rwin32.HANDLE], + rwin32.HANDLE) + DeleteFile = external( 'DeleteFile' + suffix, [traits.CCHARP], @@ -173,7 +192,29 @@ [traits.CCHARP, traits.CCHARP], rwin32.BOOL) - return Win32Traits + GETFINALPATHNAMEBYHANDLE_TP = lltype.Ptr(lltype.FuncType( + [rwin32.HANDLE, traits.CCHARP, rwin32.DWORD, rwin32.DWORD], + rwin32.DWORD, abi='FFI_STDCALL')) + # dynamically loaded + GetFinalPathNameByHandle = lltype.nullptr( + GETFINALPATHNAMEBYHANDLE_TP.TO) + + def check_GetFinalPathNameByHandle(self): + if self.GetFinalPathNameByHandle: + return True + + from rpython.rlib.rdynload import GetModuleHandle, dlsym + hKernel32 = GetModuleHandle("KERNEL32") + try: + func = dlsym(hKernel32, 'GetFinalPathNameByHandle' + suffix) + except KeyError: + return False + + self.GetFinalPathNameByHandle = rffi.cast( + Win32Traits.GETFINALPATHNAMEBYHANDLE_TP, func) + return True + + return Win32Traits() #_______________________________________________________________ # listdir @@ -336,27 +377,6 @@ win32traits = make_win32_traits(traits) from rpython.rtyper.module.ll_os_stat import time_t_to_FILE_TIME - class CConfig: - _compilation_info_ = ExternalCompilationInfo( - includes = ['windows.h'], - ) - - FILE_WRITE_ATTRIBUTES = platform.ConstantInteger( - 'FILE_WRITE_ATTRIBUTES') - OPEN_EXISTING = platform.ConstantInteger( - 'OPEN_EXISTING') - FILE_FLAG_BACKUP_SEMANTICS = platform.ConstantInteger( - 'FILE_FLAG_BACKUP_SEMANTICS') - globals().update(platform.configure(CConfig)) - - CreateFile = rffi.llexternal( - 'CreateFile' + win32traits.apisuffix, - [traits.CCHARP, rwin32.DWORD, rwin32.DWORD, - rwin32.LPSECURITY_ATTRIBUTES, rwin32.DWORD, rwin32.DWORD, - rwin32.HANDLE], - rwin32.HANDLE, - calling_conv='win') - GetSystemTime = rffi.llexternal( 'GetSystemTime', [lltype.Ptr(rwin32.SYSTEMTIME)], @@ -381,10 +401,10 @@ @specialize.argtype(1) def os_utime_llimpl(path, tp): - hFile = CreateFile(path, - FILE_WRITE_ATTRIBUTES, 0, - None, OPEN_EXISTING, - FILE_FLAG_BACKUP_SEMANTICS, + hFile = win32traits.CreateFile(path, + win32traits.FILE_WRITE_ATTRIBUTES, 0, + None, win32traits.OPEN_EXISTING, + win32traits.FILE_FLAG_BACKUP_SEMANTICS, rwin32.NULL_HANDLE) if hFile == rwin32.INVALID_HANDLE_VALUE: raise rwin32.lastWindowsError() @@ -413,3 +433,68 @@ lltype.free(mtime, flavor='raw') return os_utime_llimpl + +#_______________________________________________________________ +# _getfileinformation (py3) + +def make__getfileinformation_impl(traits): + from rpython.rlib import rwin32 + win32traits = make_win32_traits(traits) + + def _getfileinformation_llimpl(fd): + hFile = rwin32.get_osfhandle(fd) + with lltype.scoped_alloc( + win32traits.BY_HANDLE_FILE_INFORMATION) as info: + if win32traits.GetFileInformationByHandle(hFile, info) == 0: + raise rwin32.lastWindowsError("_getfileinformation") + return (rffi.cast(lltype.Signed, info.c_dwVolumeSerialNumber), + rffi.cast(lltype.Signed, info.c_nFileIndexHigh), + rffi.cast(lltype.Signed, info.c_nFileIndexLow)) + + return _getfileinformation_llimpl + +#_______________________________________________________________ +# _getfinalpathname (py3) + +def make__getfinalpathname_impl(traits): + from rpython.rlib import rwin32 + from rpython.rtyper.module.ll_os import LLNotImplemented + assert traits.str is unicode, 'Currently only handles unicode paths' + win32traits = make_win32_traits(traits) + + def _getfinalpathname_llimpl(path): + if not win32traits.check_GetFinalPathNameByHandle(): + raise LLNotImplemented("GetFinalPathNameByHandle not available on " + "this platform") + + hFile = win32traits.CreateFile(path, 0, 0, None, + win32traits.OPEN_EXISTING, + win32traits.FILE_FLAG_BACKUP_SEMANTICS, + rwin32.NULL_HANDLE) + if hFile == rwin32.INVALID_HANDLE_VALUE: + raise rwin32.lastWindowsError("CreateFile") + + VOLUME_NAME_DOS = rffi.cast(rwin32.DWORD, win32traits.VOLUME_NAME_DOS) + try: + size = win32traits.GetFinalPathNameByHandle( + hFile, + lltype.nullptr(traits.CCHARP.TO), + rffi.cast(rwin32.DWORD, 0), + VOLUME_NAME_DOS) + if size == 0: + raise rwin32.lastWindowsError("GetFinalPathNameByHandle") + + with lltype.scoped_alloc(traits.CCHARP.TO, size + 1) as target_path: + result = win32traits.GetFinalPathNameByHandle( + hFile, + target_path, + size, + VOLUME_NAME_DOS) + if result == 0: + raise rwin32.lastWindowsError("GetFinalPathNameByHandle") + return traits.charpsize2str(target_path, + rffi.cast(lltype.Signed, result)) + finally: + rwin32.CloseHandle(hFile) + + return _getfinalpathname_llimpl diff --git a/rpython/rtyper/module/support.py b/rpython/rtyper/module/support.py --- a/rpython/rtyper/module/support.py +++ b/rpython/rtyper/module/support.py @@ -49,6 +49,7 @@ CHAR = rffi.CHAR CCHARP = rffi.CCHARP charp2str = staticmethod(rffi.charp2str) + charpsize2str = staticmethod(rffi.charpsize2str) scoped_str2charp = staticmethod(rffi.scoped_str2charp) str2charp = staticmethod(rffi.str2charp) free_charp = staticmethod(rffi.free_charp) @@ -68,6 +69,7 @@ CHAR = rffi.WCHAR_T CCHARP = rffi.CWCHARP charp2str = staticmethod(rffi.wcharp2unicode) + charpsize2str = staticmethod(rffi.wcharpsize2unicode) str2charp = staticmethod(rffi.unicode2wcharp) scoped_str2charp = staticmethod(rffi.scoped_unicode2wcharp) free_charp = staticmethod(rffi.free_wcharp) diff --git a/rpython/rtyper/module/test/test_ll_win32file.py b/rpython/rtyper/module/test/test_ll_win32file.py new file mode 100644 --- /dev/null +++ b/rpython/rtyper/module/test/test_ll_win32file.py @@ -0,0 +1,26 @@ +import os + +import py + +from rpython.rtyper.module import ll_os + +if not ll_os._WIN32: + py.test.skip("requires Windows") + + +def test__getfileinformation(): + with open(__file__) as fp: + stat = os.fstat(fp.fileno()) + info = ll_os._getfileinformation(fp.fileno()) + serial, high, low = info + assert type(serial) in (int, long) + assert (high << 32) + low == stat.st_ino + + +def test__getfinalpathname(): + path = __file__.decode('mbcs') + try: + result = ll_os._getfinalpathname(path) + except ll_os.LLNotImplemented: + py.test.skip("_getfinalpathname not supported on this platform") + assert os.path.exists(result) diff --git a/rpython/translator/platform/test/test_makefile.py b/rpython/translator/platform/test/test_makefile.py --- a/rpython/translator/platform/test/test_makefile.py +++ b/rpython/translator/platform/test/test_makefile.py @@ -44,6 +44,7 @@ assert res.returncode == 0 def test_900_files(self): + tmpdir = udir.join('test_900_files').ensure(dir=1) txt = '#include \n' for i in range(900): txt += 'int func%03d();\n' % i @@ -52,11 +53,11 @@ txt += ' j += func%03d();\n' % i txt += ' printf("%d\\n", j);\n' txt += ' return 0;};\n' - cfile = udir.join('test_900_files.c') + cfile = tmpdir.join('test_900_files.c') cfile.write(txt) cfiles = [cfile] for i in range(900): - cfile2 = udir.join('implement%03d.c' %i) + cfile2 = tmpdir.join('implement%03d.c' %i) cfile2.write(''' int func%03d() { @@ -64,10 +65,10 @@ } ''' % (i, i)) cfiles.append(cfile2) - mk = self.platform.gen_makefile(cfiles, ExternalCompilationInfo(), path=udir) + mk = self.platform.gen_makefile(cfiles, ExternalCompilationInfo(), path=tmpdir) mk.write() self.platform.execute_makefile(mk) - res = self.platform.execute(udir.join('test_900_files')) + res = self.platform.execute(tmpdir.join('test_900_files')) self.check_res(res, '%d\n' %sum(range(900))) def test_precompiled_headers(self): diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -203,6 +203,9 @@ # the assembler still has the old behavior that all options # must come first, and after the file name all options are ignored. # So please be careful with the order of parameters! ;-) + pdb_dir = oname.dirname + if pdb_dir: + compile_args += ['/Fd%s\\' % (pdb_dir,)] args = ['/nologo', '/c'] + compile_args + ['/Fo%s' % (oname,), str(cfile)] self._execute_c_compiler(cc, args, oname) return oname diff --git a/rpython/translator/sandbox/rsandbox.py b/rpython/translator/sandbox/rsandbox.py --- a/rpython/translator/sandbox/rsandbox.py +++ b/rpython/translator/sandbox/rsandbox.py @@ -60,8 +60,7 @@ def need_more_data(self): buflen = self.buflen - buf = lltype.malloc(rffi.CCHARP.TO, buflen, flavor='raw') - try: + with lltype.scoped_alloc(rffi.CCHARP.TO, buflen) as buf: buflen = rffi.cast(rffi.SIZE_T, buflen) count = ll_read_not_sandboxed(self.fd, buf, buflen) count = rffi.cast(lltype.Signed, count) @@ -69,20 +68,15 @@ raise IOError self.buf += ''.join([buf[i] for i in range(count)]) self.buflen *= 2 - finally: - lltype.free(buf, flavor='raw') def sandboxed_io(buf): STDIN = 0 STDOUT = 1 # send the buffer with the marshalled fnname and input arguments to STDOUT - p = lltype.malloc(rffi.CCHARP.TO, len(buf), flavor='raw') - try: + with lltype.scoped_alloc(rffi.CCHARP.TO, len(buf)) as p: for i in range(len(buf)): p[i] = buf[i] writeall_not_sandboxed(STDOUT, p, len(buf)) - finally: - lltype.free(p, flavor='raw') # build a Loader that will get the answer from STDIN loader = FdLoader(STDIN) # check for errors @@ -108,9 +102,8 @@ @signature(types.str(), returns=types.impossible()) def not_implemented_stub(msg): STDERR = 2 - buf = rffi.str2charp(msg + '\n') - writeall_not_sandboxed(STDERR, buf, len(msg) + 1) - rffi.free_charp(buf) + with rffi.scoped_str2charp(msg + '\n') as buf: + writeall_not_sandboxed(STDERR, buf, len(msg) + 1) raise RuntimeError(msg) # XXX in RPython, the msg is ignored at the moment dump_string = rmarshal.get_marshaller(str) From noreply at buildbot.pypy.org Mon Sep 22 01:21:18 2014 From: noreply at buildbot.pypy.org (pjenvey) Date: Mon, 22 Sep 2014 01:21:18 +0200 (CEST) Subject: [pypy-commit] pypy py3k: _getfinalpathname is broken, disable for now Message-ID: <20140921232118.917901C0130@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r73637:7011366e6fbf Date: 2014-09-21 16:20 -0700 http://bitbucket.org/pypy/pypy/changeset/7011366e6fbf/ Log: _getfinalpathname is broken, disable for now diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py --- a/pypy/module/posix/__init__.py +++ b/pypy/module/posix/__init__.py @@ -167,7 +167,8 @@ if os.name == 'nt': interpleveldefs.update({ '_getfileinformation': 'interp_posix._getfileinformation', - '_getfinalpathname': 'interp_posix._getfinalpathname', + # XXX: currently broken + #'_getfinalpathname': 'interp_posix._getfinalpathname', }) if hasattr(os, 'chroot'): interpleveldefs['chroot'] = 'interp_posix.chroot' From noreply at buildbot.pypy.org Mon Sep 22 01:21:20 2014 From: noreply at buildbot.pypy.org (pjenvey) Date: Mon, 22 Sep 2014 01:21:20 +0200 (CEST) Subject: [pypy-commit] pypy py3.3: merge py3k Message-ID: <20140921232120.1C3741C0130@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3.3 Changeset: r73638:de00ce0a7596 Date: 2014-09-21 16:20 -0700 http://bitbucket.org/pypy/pypy/changeset/de00ce0a7596/ Log: merge py3k diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py --- a/pypy/module/posix/__init__.py +++ b/pypy/module/posix/__init__.py @@ -171,7 +171,8 @@ if os.name == 'nt': interpleveldefs.update({ '_getfileinformation': 'interp_posix._getfileinformation', - '_getfinalpathname': 'interp_posix._getfinalpathname', + # XXX: currently broken + #'_getfinalpathname': 'interp_posix._getfinalpathname', }) if hasattr(os, 'chroot'): interpleveldefs['chroot'] = 'interp_posix.chroot' diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -307,7 +307,13 @@ argparse = imp.load_source('argparse', 'lib-python/2.7/argparse.py') if sys.platform == 'win32': pypy_exe = 'pypy3.exe' - license_base = os.path.join(basedir, r'..\..\..\local') # as on buildbot YMMV + for p in [os.path.join(basedir, r'..\..\..\local'), #buildbot + os.path.join(basedir, r'..\local')]: # pypy/doc/windows.rst + if os.path.exists(p): + license_base = p + break + else: + license_base = 'unkown' else: pypy_exe = 'pypy3' license_base = '/usr/share/doc' @@ -380,5 +386,21 @@ if __name__ == '__main__': import sys + if sys.platform == 'win32': + # Try to avoid opeing a dialog box if one of the + # subprocesses causes a system error + import ctypes + winapi = ctypes.windll.kernel32 + SetErrorMode = winapi.SetErrorMode + SetErrorMode.argtypes=[ctypes.c_int] + + SEM_FAILCRITICALERRORS = 1 + SEM_NOGPFAULTERRORBOX = 2 + SEM_NOOPENFILEERRORBOX = 0x8000 + flags = SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX | SEM_NOOPENFILEERRORBOX + #Since there is no GetErrorMode, do a double Set + old_mode = SetErrorMode(flags) + SetErrorMode(old_mode | flags) + retval, _ = package(*sys.argv[1:]) sys.exit(retval) diff --git a/pypy/tool/release/test/test_package.py b/pypy/tool/release/test/test_package.py --- a/pypy/tool/release/test/test_package.py +++ b/pypy/tool/release/test/test_package.py @@ -115,15 +115,21 @@ check(pypy, 0755) def test_generate_license(): - from os.path import dirname, abspath, join + from os.path import dirname, abspath, join, exists class Options(object): pass options = Options() basedir = dirname(dirname(dirname(dirname(dirname(abspath(__file__)))))) options.no_tk = False if sys.platform == 'win32': - # as on buildbot YMMV - options.license_base = join(basedir, r'..\..\..\local') + for p in [join(basedir, r'..\..\..\local'), #buildbot + join(basedir, r'..\local')]: # pypy/doc/windows.rst + if exists(p): + license_base = p + break + else: + license_base = 'unkown' + options.license_base = license_base else: options.license_base = '/usr/share/doc' license = package.generate_license(py.path.local(basedir), options) diff --git a/rpython/translator/driver.py b/rpython/translator/driver.py --- a/rpython/translator/driver.py +++ b/rpython/translator/driver.py @@ -476,11 +476,12 @@ shutil_copy(str(soname), str(newsoname)) self.log.info("copied: %s" % (newsoname,)) if sys.platform == 'win32': - # copy the import library as well - libname = soname.new(ext='lib') - newlibname = newexename.new(basename=soname.basename) - shutil.copyfile(str(libname), str(newlibname.new(ext='lib'))) - self.log.info("copied: %s" % (newlibname,)) + ext_to_copy = ['lib', 'pdb'] + for ext in ext_to_copy: + name = soname.new(ext=ext) + newname = newexename.new(basename=soname.basename) + shutil.copyfile(str(name), str(newname.new(ext=ext))) + self.log.info("copied: %s" % (newname,)) self.c_entryp = newexename self.log.info('usession directory: %s' % (udir,)) self.log.info("created: %s" % (self.c_entryp,)) diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -410,7 +410,7 @@ 'int main(int argc, char* argv[]) ' '{ return $(PYPY_MAIN_FUNCTION)(argc, argv); } > $@') m.rule('$(DEFAULT_TARGET)', ['$(TARGET)', 'main.obj'], - ['$(CC_LINK) /nologo main.obj $(SHARED_IMPORT_LIB) /out:$@ /MANIFEST /MANIFESTFILE:$*.manifest', + ['$(CC_LINK) /nologo /debug main.obj $(SHARED_IMPORT_LIB) /out:$@ /MANIFEST /MANIFESTFILE:$*.manifest', 'mt.exe -nologo -manifest $*.manifest -outputresource:$@;1', ]) m.rule('debugmode_$(DEFAULT_TARGET)', ['debugmode_$(TARGET)', 'main.obj'], diff --git a/rpython/translator/test/test_driver.py b/rpython/translator/test/test_driver.py --- a/rpython/translator/test/test_driver.py +++ b/rpython/translator/test/test_driver.py @@ -55,12 +55,15 @@ src_name = udir.join('src/dydy2.exe') dll_name = udir.join('src/pypy.dll') lib_name = udir.join('src/pypy.lib') + pdb_name = udir.join('src/pypy.pdb') src_name.ensure() src_name.write('exe') dll_name.ensure() dll_name.write('dll') lib_name.ensure() lib_name.write('lib') + pdb_name.ensure() + pdb_name.write('pdb') dst_name.ensure() class CBuilder(object): @@ -75,6 +78,8 @@ assert dst_name.new(ext='lib').read() == 'lib' def test_shutil_copy(): + if os.name == 'nt': + py.test.skip('Windows cannot copy or rename to an in-use file') a = udir.join('file_a') b = udir.join('file_a') a.write('hello') From noreply at buildbot.pypy.org Mon Sep 22 01:23:13 2014 From: noreply at buildbot.pypy.org (pjenvey) Date: Mon, 22 Sep 2014 01:23:13 +0200 (CEST) Subject: [pypy-commit] pypy py3.3: issue832: linux sys.platform now always "linux" Message-ID: <20140921232313.EFCE11C0130@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3.3 Changeset: r73639:31223ffc5ac1 Date: 2014-09-21 16:22 -0700 http://bitbucket.org/pypy/pypy/changeset/31223ffc5ac1/ Log: issue832: linux sys.platform now always "linux" diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py --- a/pypy/module/sys/__init__.py +++ b/pypy/module/sys/__init__.py @@ -23,7 +23,7 @@ '__name__' : '(space.wrap("sys"))', '__doc__' : '(space.wrap("PyPy sys module"))', - 'platform' : 'space.wrap(sys.platform)', + 'platform' : 'space.wrap(system.PLATFORM)', 'maxsize' : 'space.wrap(sys.maxint)', 'byteorder' : 'space.wrap(sys.byteorder)', 'maxunicode' : 'space.wrap(vm.MAXUNICODE)', diff --git a/pypy/module/sys/system.py b/pypy/module/sys/system.py --- a/pypy/module/sys/system.py +++ b/pypy/module/sys/system.py @@ -1,4 +1,6 @@ """Information about the current system.""" +import sys + from pypy.objspace.std.complexobject import HASH_IMAG from pypy.objspace.std.floatobject import HASH_INF, HASH_NAN from pypy.objspace.std.intobject import HASH_MODULUS @@ -6,6 +8,7 @@ from rpython.rlib import rbigint, rfloat from rpython.rtyper.lltypesystem import lltype, rffi +PLATFORM = 'linux' if sys.platform.startswith('linux') else sys.platform app = gateway.applevel(""" "NOT_RPYTHON" From noreply at buildbot.pypy.org Mon Sep 22 01:32:05 2014 From: noreply at buildbot.pypy.org (pjenvey) Date: Mon, 22 Sep 2014 01:32:05 +0200 (CEST) Subject: [pypy-commit] pypy py3.3: sys.platform linux2 -> linux Message-ID: <20140921233205.C698B1C0130@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3.3 Changeset: r73640:57188ff5ab4e Date: 2014-09-21 16:31 -0700 http://bitbucket.org/pypy/pypy/changeset/57188ff5ab4e/ Log: sys.platform linux2 -> linux diff --git a/lib_pypy/ctypes_support.py b/lib_pypy/ctypes_support.py --- a/lib_pypy/ctypes_support.py +++ b/lib_pypy/ctypes_support.py @@ -22,7 +22,7 @@ def _where_is_errno(): return standard_c_lib._errno() -elif sys.platform in ('linux2', 'freebsd6'): +elif sys.platform in ('linux', 'freebsd6'): standard_c_lib.__errno_location.restype = ctypes.POINTER(ctypes.c_int) standard_c_lib.__errno_location.argtypes = None def _where_is_errno(): From noreply at buildbot.pypy.org Mon Sep 22 06:51:54 2014 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Mon, 22 Sep 2014 06:51:54 +0200 (CEST) Subject: [pypy-commit] pypy release-2.4.x: Fix curses on OS X -- copy this logic from cpython Message-ID: <20140922045154.3D7E01C31C8@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: release-2.4.x Changeset: r73641:c6ad44ecf5d8 Date: 2014-09-20 14:16 -0700 http://bitbucket.org/pypy/pypy/changeset/c6ad44ecf5d8/ Log: Fix curses on OS X -- copy this logic from cpython diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py --- a/lib_pypy/_curses.py +++ b/lib_pypy/_curses.py @@ -286,6 +286,13 @@ lib = ffi.verify(""" +#ifdef __APPLE__ +/* the following define is necessary for OS X 10.6+; without it, the + Apple-supplied ncurses.h sets NCURSES_OPAQUE to 1, and then Python + can't get at the WINDOW flags field. */ +#define NCURSES_OPAQUE 0 +#endif + #include #include #include From noreply at buildbot.pypy.org Mon Sep 22 08:55:11 2014 From: noreply at buildbot.pypy.org (stefanor) Date: Mon, 22 Sep 2014 08:55:11 +0200 (CEST) Subject: [pypy-commit] pypy default: First step to fixing the sanbox: implement os.access Message-ID: <20140922065511.B81A81C072F@cobra.cs.uni-duesseldorf.de> Author: Stefano Rivera Branch: Changeset: r73642:1599b45fa4e1 Date: 2014-09-21 23:52 -0700 http://bitbucket.org/pypy/pypy/changeset/1599b45fa4e1/ Log: First step to fixing the sanbox: implement os.access pypy now (since b97c7d6f7fd8) checks for executability, when searching for its own executable on startup. This requires os.access. diff --git a/rpython/translator/sandbox/sandlib.py b/rpython/translator/sandbox/sandlib.py --- a/rpython/translator/sandbox/sandlib.py +++ b/rpython/translator/sandbox/sandlib.py @@ -459,6 +459,15 @@ do_ll_os__ll_os_lstat = do_ll_os__ll_os_stat + def do_ll_os__ll_os_access(self, vpathname, mode): + try: + node = self.get_node(vpathname) + except OSError, e: + if e.errno == errno.ENOENT: + return False + raise + return node.access(mode) + def do_ll_os__ll_os_isatty(self, fd): return self.virtual_console_isatty and fd in (0, 1, 2) diff --git a/rpython/translator/sandbox/test/test_vfs.py b/rpython/translator/sandbox/test/test_vfs.py --- a/rpython/translator/sandbox/test/test_vfs.py +++ b/rpython/translator/sandbox/test/test_vfs.py @@ -33,6 +33,8 @@ py.test.raises(OSError, d.join, 'bar') st = d.stat() assert stat.S_ISDIR(st.st_mode) + assert d.access(os.R_OK | os.X_OK) + assert not d.access(os.W_OK) def test_file(): f = File('hello world') @@ -46,6 +48,8 @@ st = f.stat() assert stat.S_ISREG(st.st_mode) assert st.st_size == 11 + assert f.access(os.R_OK) + assert not f.access(os.W_OK) def test_realdir_realfile(): for show_dotfiles in [False, True]: @@ -78,6 +82,7 @@ f = v_test_vfs.join('symlink2') assert stat.S_ISREG(f.stat().st_mode) + assert f.access(os.R_OK) assert f.open().read() == 'secret' else: py.test.raises(OSError, v_test_vfs.join, 'symlink1') diff --git a/rpython/translator/sandbox/vfs.py b/rpython/translator/sandbox/vfs.py --- a/rpython/translator/sandbox/vfs.py +++ b/rpython/translator/sandbox/vfs.py @@ -37,6 +37,15 @@ (st_mode, st_ino, st_dev, st_nlink, st_uid, st_gid, st_size, st_atime, st_mtime, st_ctime)) + def access(self, mode): + s = self.stat() + e_mode = s.st_mode & stat.S_IRWXO + if UID == s.st_uid: + e_mode |= (s.st_mode & stat.S_IRWXU) >> 6 + if GID == s.st_gid: + e_mode |= (s.st_mode & stat.S_IRWXG) >> 3 + return (e_mode & mode) == mode + def keys(self): raise OSError(errno.ENOTDIR, self) From noreply at buildbot.pypy.org Mon Sep 22 08:55:12 2014 From: noreply at buildbot.pypy.org (stefanor) Date: Mon, 22 Sep 2014 08:55:12 +0200 (CEST) Subject: [pypy-commit] pypy default: Fix the sandbox Message-ID: <20140922065512.E57C21C072F@cobra.cs.uni-duesseldorf.de> Author: Stefano Rivera Branch: Changeset: r73643:17fcf0cc640d Date: 2014-09-21 23:54 -0700 http://bitbucket.org/pypy/pypy/changeset/17fcf0cc640d/ Log: Fix the sandbox pypy now (since b97c7d6f7fd8) checks for executability, when searching for its own executable on startup. Abuse "kind" to maintain extra mode bits, so we can store the executable state in our vfs. diff --git a/pypy/sandbox/pypy_interact.py b/pypy/sandbox/pypy_interact.py --- a/pypy/sandbox/pypy_interact.py +++ b/pypy/sandbox/pypy_interact.py @@ -55,7 +55,7 @@ return Dir({ 'bin': Dir({ - 'pypy-c': RealFile(self.executable), + 'pypy-c': RealFile(self.executable, mode=0111), 'lib-python': RealDir(os.path.join(libroot, 'lib-python'), exclude=exclude), 'lib_pypy': RealDir(os.path.join(libroot, 'lib_pypy'), diff --git a/rpython/translator/sandbox/vfs.py b/rpython/translator/sandbox/vfs.py --- a/rpython/translator/sandbox/vfs.py +++ b/rpython/translator/sandbox/vfs.py @@ -22,7 +22,7 @@ st_size = self.getsize() st_mode = self.kind st_mode |= stat.S_IWUSR | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH - if self.kind == stat.S_IFDIR: + if stat.S_ISDIR(self.kind): st_mode |= stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH if self.read_only: st_uid = 0 # read-only files are virtually owned by root @@ -123,8 +123,9 @@ return cStringIO.StringIO(self.data) class RealFile(File): - def __init__(self, path): + def __init__(self, path, mode=0): self.path = path + self.kind |= mode def __repr__(self): return '' % (self.path,) def getsize(self): From noreply at buildbot.pypy.org Mon Sep 22 09:32:39 2014 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 22 Sep 2014 09:32:39 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: clear more fields Message-ID: <20140922073239.124D11C072F@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73644:238dd369440d Date: 2014-09-22 09:32 +0200 http://bitbucket.org/pypy/pypy/changeset/238dd369440d/ Log: clear more fields diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -248,6 +248,10 @@ ResOperation(rop.SETFIELD_GC, [frame, self.c_zero], None, descr=descrs.jf_savedata), ResOperation(rop.SETFIELD_GC, [frame, self.c_zero], + None, descr=descrs.jf_force_descr), + ResOperation(rop.SETFIELD_GC, [frame, self.c_zero], + None, descr=descrs.jf_descr), + ResOperation(rop.SETFIELD_GC, [frame, self.c_zero], None, descr=descrs.jf_guard_exc), ResOperation(rop.SETFIELD_GC, [frame, self.c_zero], None, descr=descrs.jf_forward), From noreply at buildbot.pypy.org Mon Sep 22 11:17:25 2014 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 22 Sep 2014 11:17:25 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: Update comment Message-ID: <20140922091725.86E6A1C035D@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: gc_no_cleanup_nursery Changeset: r73645:39024806c7ac Date: 2014-09-22 11:17 +0200 http://bitbucket.org/pypy/pypy/changeset/39024806c7ac/ Log: Update comment diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -423,7 +423,8 @@ def gen_malloc_nursery(self, size, v_result): """Try to generate or update a CALL_MALLOC_NURSERY. - If that fails, generate a plain CALL_MALLOC_GC instead. + If that succeeds, return True; you still need to write the tid. + If that fails, return False. """ size = self.round_up_for_allocation(size) if not self.gc_ll_descr.can_use_nursery_malloc(size): From noreply at buildbot.pypy.org Mon Sep 22 12:25:53 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Mon, 22 Sep 2014 12:25:53 +0200 (CEST) Subject: [pypy-commit] stmgc c8-new-page-handling: fix overwriting changes where we already have some modifications Message-ID: <20140922102553.8B0731C0FF8@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: c8-new-page-handling Changeset: r1404:ef65c8fd67f8 Date: 2014-09-22 12:26 +0200 http://bitbucket.org/pypy/stmgc/changeset/ef65c8fd67f8/ Log: fix overwriting changes where we already have some modifications diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -4,13 +4,42 @@ /* ############# signal handler ############# */ -static void _update_obj_from(int from_seg, object_t *obj) +static void memcpy_to_accessible_pages( + int dst_segnum, object_t *dst_obj, + char *src, size_t len, uintptr_t only_page) +{ + /* XXX: optimize */ + + char *realobj = REAL_ADDRESS(get_segment_base(dst_segnum), dst_obj); + char *dst_end = realobj + len; + uintptr_t loc_addr = (uintptr_t)dst_obj; + + dprintf(("memcpy_to_accessible_pages(%d, %p, %p, %lu, %lu)\n", + dst_segnum, dst_obj, src, len, only_page)); + + while (realobj != dst_end) { + if (get_page_status_in(dst_segnum, loc_addr / 4096UL) != PAGE_NO_ACCESS + && (only_page == -1 || only_page == loc_addr / 4096UL)) { + *realobj = *src; + } + realobj++; + loc_addr++; + src++; + } +} + + +static void _update_obj_from(int from_seg, object_t *obj, uintptr_t only_page) { /* updates 'obj' in our accessible pages from another segment's - page or bk copy. (never touch PROT_NONE memory) */ + page or bk copy. (never touch PROT_NONE memory) + only_page = -1 means update whole obj, only_page=pagenum means only + update memory in page 'pagenum' + */ /* XXXXXXX: are we sure everything is readable in from_seg??? */ size_t obj_size; + dprintf(("_update_obj_from(%d, %p)\n", from_seg, obj)); assert(get_priv_segment(from_seg)->privatization_lock); /* look the obj up in the other segment's modified_old_objects to @@ -24,7 +53,7 @@ obj_size = stmcb_size_rounded_up((struct object_s*)item->val); memcpy_to_accessible_pages(STM_SEGMENT->segment_num, obj, - (char*)item->val, obj_size); + (char*)item->val, obj_size, only_page); release_modified_objs_lock(from_seg); return; @@ -36,7 +65,7 @@ memcpy_to_accessible_pages(STM_SEGMENT->segment_num, obj, REAL_ADDRESS(get_segment_base(from_seg), obj), - obj_size); + obj_size, only_page); release_modified_objs_lock(from_seg); } @@ -45,7 +74,9 @@ static void copy_bk_objs_from(int from_segnum, uintptr_t pagenum) { /* looks at all bk copies of objects overlapping page 'pagenum' and - copies to current segment (never touch PROT_NONE memory) */ + copies to current segment (never touch PROT_NONE memory). */ + dprintf(("copy_bk_objs_from(%d, %lu)\n", from_segnum, pagenum)); + acquire_modified_objs_lock(from_segnum); struct tree_s *tree = get_priv_segment(from_segnum)->modified_old_objects; wlog_t *item; @@ -58,18 +89,16 @@ /* XXX: should actually only write to pagenum, but we validate afterwards anyway and abort in case we had modifications there */ memcpy_to_accessible_pages(STM_SEGMENT->segment_num, - obj, (char*)bk_obj, obj_size); - - assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); /* bk_obj never written */ + obj, (char*)bk_obj, obj_size, pagenum); } } TREE_LOOP_END; release_modified_objs_lock(from_segnum); } -static void update_page_from_to( - uintptr_t pagenum, struct stm_commit_log_entry_s *from, - struct stm_commit_log_entry_s *to) +static void update_page_from_to(uintptr_t pagenum, + struct stm_commit_log_entry_s *from, + struct stm_commit_log_entry_s *to) { /* walk the commit log and update the page 'pagenum' until we reach the same revision as our segment, or we reach the HEAD. */ @@ -90,7 +119,7 @@ object_t *obj; size_t i = 0; while ((obj = cl->written[i])) { - _update_obj_from(cl->segment_num, obj); + _update_obj_from(cl->segment_num, obj, pagenum); i++; }; @@ -106,6 +135,8 @@ /* assumes page 'pagenum' is ACCESS_NONE, privatizes it, and validates to newest revision */ + dprintf(("bring_page_up_to_date(%lu), seg %d\n", pagenum, STM_SEGMENT->segment_num)); + /* XXX: bad, but no deadlocks: */ acquire_all_privatization_locks(); @@ -241,7 +272,7 @@ object_t *obj; size_t i = 0; while ((obj = cl->written[i])) { - _update_obj_from(cl->segment_num, obj); + _update_obj_from(cl->segment_num, obj, -1); if (_stm_was_read(obj)) { needs_abort = true; diff --git a/c8/stm/pages.c b/c8/stm/pages.c --- a/c8/stm/pages.c +++ b/c8/stm/pages.c @@ -73,24 +73,3 @@ volatile char *dummy = REAL_ADDRESS(get_segment_base(segnum), pagenum*4096UL); *dummy = *dummy; /* force copy-on-write from shared page */ } - - -static void memcpy_to_accessible_pages( - int dst_segnum, object_t *dst_obj, char *src, size_t len) -{ - /* XXX: optimize */ - - char *realobj = REAL_ADDRESS(get_segment_base(dst_segnum), dst_obj); - char *dst_end = realobj + len; - uintptr_t loc_addr = (uintptr_t)dst_obj; - - dprintf(("memcpy_to_accessible_pages(%d, %p, %p, %lu)\n", dst_segnum, dst_obj, src, len)); - - while (realobj != dst_end) { - if (get_page_status_in(dst_segnum, loc_addr / 4096UL) != PAGE_NO_ACCESS) - *realobj = *src; - realobj++; - loc_addr++; - src++; - } -} diff --git a/c8/stm/pages.h b/c8/stm/pages.h --- a/c8/stm/pages.h +++ b/c8/stm/pages.h @@ -46,7 +46,6 @@ static void pages_initialize_shared_for(long segnum, uintptr_t pagenum, uintptr_t count); static void page_privatize_in(int segnum, uintptr_t pagenum); -static void memcpy_to_accessible_pages(int dst_segnum, object_t *dst_obj, char *src, size_t len); From noreply at buildbot.pypy.org Mon Sep 22 12:54:14 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Mon, 22 Sep 2014 12:54:14 +0200 (CEST) Subject: [pypy-commit] stmgc c8-new-page-handling: actually make the file big enough Message-ID: <20140922105414.109931C035D@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: c8-new-page-handling Changeset: r1405:592f45721ab1 Date: 2014-09-22 12:54 +0200 http://bitbucket.org/pypy/stmgc/changeset/592f45721ab1/ Log: actually make the file big enough diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -476,7 +476,7 @@ (long)(NB_READMARKER_PAGES * 4096UL))); if (mmap(readmarkers, NB_READMARKER_PAGES * 4096UL, PROT_READ | PROT_WRITE, - MAP_FIXED | MAP_PAGES_FLAGS, -1, 0) != readmarkers) { + MAP_FIXED | MAP_SHARED | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0) != readmarkers) { /* fall-back */ #if STM_TESTS stm_fatalerror("reset_transaction_read_version: %m"); diff --git a/c8/stm/core.h b/c8/stm/core.h --- a/c8/stm/core.h +++ b/c8/stm/core.h @@ -19,7 +19,6 @@ #define NB_PAGES (2500*256) // 2500MB #define NB_SEGMENTS STM_NB_SEGMENTS #define NB_SEGMENTS_MAX 240 /* don't increase NB_SEGMENTS past this */ -#define MAP_PAGES_FLAGS (MAP_SHARED | MAP_ANONYMOUS | MAP_NORESERVE) #define NB_NURSERY_PAGES (STM_GC_NURSERY/4) #define TOTAL_MEMORY (NB_PAGES * 4096UL * NB_SEGMENTS) diff --git a/c8/stm/pages.c b/c8/stm/pages.c --- a/c8/stm/pages.c +++ b/c8/stm/pages.c @@ -40,6 +40,16 @@ mprotect(segment_base + pagenum * 4096UL, count * 4096UL, PROT_NONE); + /* char *result = mmap( */ + /* segment_base + pagenum * 4096UL, */ + /* count * 4096UL, */ + /* PROT_NONE, */ + /* MAP_FIXED|MAP_NORESERVE|MAP_PRIVATE|MAP_ANONYMOUS, */ + /* -1, 0); */ + /* if (result == MAP_FAILED) */ + /* stm_fatalerror("pages_initialize_shared failed (mmap): %m"); */ + + long amount = count; while (amount-->0) { set_page_status_in(i, pagenum + amount, PAGE_NO_ACCESS); diff --git a/c8/stm/setup.c b/c8/stm/setup.c --- a/c8/stm/setup.c +++ b/c8/stm/setup.c @@ -20,7 +20,7 @@ if (stm_object_pages_fd == -1) stm_fatalerror("%s failed (stm_open): %m", reason); - if (ftruncate(stm_object_pages_fd, NB_SHARED_PAGES) != 0) + if (ftruncate(stm_object_pages_fd, NB_SHARED_PAGES * 4096UL) != 0) stm_fatalerror("%s failed (ftruncate): %m", reason); stm_file_pages = mmap(NULL, NB_SHARED_PAGES * 4096UL, From noreply at buildbot.pypy.org Mon Sep 22 13:12:51 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Mon, 22 Sep 2014 13:12:51 +0200 (CEST) Subject: [pypy-commit] stmgc c8-new-page-handling: minor renaming and a fix for demo_random Message-ID: <20140922111251.BA54C1C0FF8@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: c8-new-page-handling Changeset: r1406:7d021fd1f7c1 Date: 2014-09-22 13:13 +0200 http://bitbucket.org/pypy/stmgc/changeset/7d021fd1f7c1/ Log: minor renaming and a fix for demo_random diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -71,11 +71,11 @@ } -static void copy_bk_objs_from(int from_segnum, uintptr_t pagenum) +static void copy_bk_objs_in_page_from(int from_segnum, uintptr_t pagenum) { /* looks at all bk copies of objects overlapping page 'pagenum' and copies to current segment (never touch PROT_NONE memory). */ - dprintf(("copy_bk_objs_from(%d, %lu)\n", from_segnum, pagenum)); + dprintf(("copy_bk_objs_in_page_from(%d, %lu)\n", from_segnum, pagenum)); acquire_modified_objs_lock(from_segnum); struct tree_s *tree = get_priv_segment(from_segnum)->modified_old_objects; @@ -96,9 +96,9 @@ release_modified_objs_lock(from_segnum); } -static void update_page_from_to(uintptr_t pagenum, - struct stm_commit_log_entry_s *from, - struct stm_commit_log_entry_s *to) +static void update_page_revision_from_to(uintptr_t pagenum, + struct stm_commit_log_entry_s *from, + struct stm_commit_log_entry_s *to) { /* walk the commit log and update the page 'pagenum' until we reach the same revision as our segment, or we reach the HEAD. */ @@ -130,12 +130,12 @@ } } -static void bring_page_up_to_date(uintptr_t pagenum) +static void handle_segfault_in_page(uintptr_t pagenum) { /* assumes page 'pagenum' is ACCESS_NONE, privatizes it, and validates to newest revision */ - dprintf(("bring_page_up_to_date(%lu), seg %d\n", pagenum, STM_SEGMENT->segment_num)); + dprintf(("handle_segfault_in_page(%lu), seg %d\n", pagenum, STM_SEGMENT->segment_num)); /* XXX: bad, but no deadlocks: */ acquire_all_privatization_locks(); @@ -168,10 +168,10 @@ assert(get_page_status_in(my_segnum, pagenum) == PAGE_PRIVATE); /* if there were modifications in the page, revert them: */ - copy_bk_objs_from(shared_page_holder, pagenum); + copy_bk_objs_in_page_from(shared_page_holder, pagenum); /* if not already newer, update page to our revision */ - update_page_from_to( + update_page_revision_from_to( pagenum, get_priv_segment(shared_page_holder)->last_commit_log_entry, STM_PSEGMENT->last_commit_log_entry); @@ -198,7 +198,7 @@ char *seg_base = STM_SEGMENT->segment_base; uintptr_t pagenum = ((char*)addr - seg_base) / 4096UL; - bring_page_up_to_date(pagenum); + handle_segfault_in_page(pagenum); return; } diff --git a/c8/stmgc.h b/c8/stmgc.h --- a/c8/stmgc.h +++ b/c8/stmgc.h @@ -71,7 +71,6 @@ char *_stm_real_address(object_t *o); #ifdef STM_TESTS #include -void stm_validate(void); bool _stm_was_read(object_t *obj); bool _stm_was_written(object_t *obj); @@ -226,6 +225,7 @@ } void stm_become_globally_unique_transaction(stm_thread_local_t *tl, const char *msg); +void stm_validate(void); /* ==================== END ==================== */ From noreply at buildbot.pypy.org Mon Sep 22 13:56:37 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Mon, 22 Sep 2014 13:56:37 +0200 (CEST) Subject: [pypy-commit] stmgc c8-new-page-handling: check abort in signal handler Message-ID: <20140922115637.66DED1C35EE@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: c8-new-page-handling Changeset: r1407:3779031defc6 Date: 2014-09-22 13:57 +0200 http://bitbucket.org/pypy/stmgc/changeset/3779031defc6/ Log: check abort in signal handler diff --git a/c8/test/support.py b/c8/test/support.py --- a/c8/test/support.py +++ b/c8/test/support.py @@ -48,6 +48,7 @@ void _stm_start_safe_point(void); bool _check_stop_safe_point(void); +ssize_t _checked_stmcb_size_rounded_up(struct object_s *obj); bool _checked_stm_write(object_t *obj); bool _stm_was_read(object_t *obj); @@ -166,6 +167,20 @@ return 1; } +ssize_t _checked_stmcb_size_rounded_up(struct object_s *obj) +{ + stm_thread_local_t *_tl = STM_SEGMENT->running_thread; \ + void **jmpbuf = _tl->rjthread.jmpbuf; \ + if (__builtin_setjmp(jmpbuf) == 0) { /* returned directly */\ + ssize_t res = stmcb_size_rounded_up(obj); + clear_jmpbuf(_tl); + return res; + } + clear_jmpbuf(_tl); + return 1; +} + + bool _check_stop_safe_point(void) { CHECKED(_stm_stop_safe_point()); } @@ -396,7 +411,10 @@ return lib._stm_get_private_page(pagenum) def stm_get_obj_size(o): - return lib.stmcb_size_rounded_up(stm_get_real_address(o)) + res = lib._checked_stmcb_size_rounded_up(stm_get_real_address(o)) + if res == 1: + raise Conflict() + return res def stm_get_obj_pages(o): start = int(ffi.cast('uintptr_t', o)) @@ -511,7 +529,7 @@ assert res # abort_transaction() didn't abort! assert not lib._stm_in_transaction(tl) - def switch(self, thread_num): + def switch(self, thread_num, validate=True): assert thread_num != self.current_thread tl = self.tls[self.current_thread] if lib._stm_in_transaction(tl): @@ -523,7 +541,8 @@ if lib._stm_in_transaction(tl2): lib._stm_test_switch(tl2) stm_stop_safe_point() # can raise Conflict - stm_validate() # can raise Conflict + if validate: + stm_validate() # can raise Conflict def switch_to_segment(self, seg_num): lib._stm_test_switch_segment(seg_num) diff --git a/c8/test/test_basic.py b/c8/test/test_basic.py --- a/c8/test/test_basic.py +++ b/c8/test/test_basic.py @@ -543,3 +543,34 @@ assert self.get_stm_thread_local().last_abort__bytes_in_nursery == 56 self.abort_transaction() assert self.get_stm_thread_local().last_abort__bytes_in_nursery == 0 + + def test_abort_in_segfault_handler(self): + lp1 = stm_allocate_old(16) + lp2 = stm_allocate_old(16) + + self.start_transaction() + stm_set_char(lp1, 'A') + stm_set_char(lp2, 'B') + self.commit_transaction() + # lp1 = S|N|N|N + # lp2 = S|N|N|N + + self.switch(1) + + self.start_transaction() + assert stm_get_char(lp1) == 'A' + # lp1 = S|P|N|N + # lp2 = S|N|N|N + + self.switch(0) + + self.start_transaction() + stm_set_char(lp1, 'C') + self.commit_transaction() + # lp1 = S|P|N|N + # lp2 = S|N|N|N + + self.switch(1, validate=False) + + # seg1 segfaults, validates, and aborts: + py.test.raises(Conflict, stm_get_char, lp2) diff --git a/c8/test/test_random.py b/c8/test/test_random.py --- a/c8/test/test_random.py +++ b/c8/test/test_random.py @@ -535,7 +535,7 @@ def test_fixed_16_bytes_objects(self, seed=1010): rnd = random.Random(seed) - N_OBJECTS = 3 + N_OBJECTS = 4 N_THREADS = self.NB_THREADS ex = Exec(self) ex.do("################################################################\n"*10) @@ -575,7 +575,7 @@ op_minor_collect, #op_major_collect, ] - for _ in range(200): + for _ in range(500): # make sure we are in a transaction: curr_thread = op_switch_thread(ex, global_state, curr_thread) From noreply at buildbot.pypy.org Mon Sep 22 15:03:24 2014 From: noreply at buildbot.pypy.org (Ben Mather) Date: Mon, 22 Sep 2014 15:03:24 +0200 (CEST) Subject: [pypy-commit] pypy py3.3: add faulthandler to list of required modules for all lib-python 3 tests Message-ID: <20140922130324.8B80E1C0FF8@cobra.cs.uni-duesseldorf.de> Author: Ben Mather Branch: py3.3 Changeset: r73646:789fe7d6428a Date: 2014-09-22 13:51 +0100 http://bitbucket.org/pypy/pypy/changeset/789fe7d6428a/ Log: add faulthandler to list of required modules for all lib-python 3 tests faulthandler is imported by lib-python/3/tests/regrtest.py which is imported before running any tests. faulthandler is new to 3.3. fixes all tests crashing with import error when running with interpreted interpreter (compiled with compiled interpreter) diff --git a/lib-python/conftest.py b/lib-python/conftest.py --- a/lib-python/conftest.py +++ b/lib-python/conftest.py @@ -60,7 +60,8 @@ skip=None): self.basename = basename self._usemodules = usemodules.split() + [ - '_socket', 'binascii', 'rctime', 'select', 'signal'] + '_socket', 'binascii', 'rctime', + 'select', 'signal', 'faulthandler'] if not sys.platform == 'win32': self._usemodules.extend(['_posixsubprocess', 'fcntl']) self._compiler = compiler From noreply at buildbot.pypy.org Mon Sep 22 18:17:01 2014 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 22 Sep 2014 18:17:01 +0200 (CEST) Subject: [pypy-commit] pypy default: Issue #1869: fix error messages to match CPython's datetime.c Message-ID: <20140922161701.902DD1C035D@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73647:e69035610b6b Date: 2014-09-22 18:16 +0200 http://bitbucket.org/pypy/pypy/changeset/e69035610b6b/ Log: Issue #1869: fix error messages to match CPython's datetime.c diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py --- a/lib_pypy/datetime.py +++ b/lib_pypy/datetime.py @@ -1242,7 +1242,7 @@ (other._hour, other._minute, other._second, other._microsecond)) if myoff is None or otoff is None: - raise TypeError("cannot compare naive and aware times") + raise TypeError("can't compare offset-naive and offset-aware datetimes") myhhmm = self._hour * 60 + self._minute - myoff othhmm = other._hour * 60 + other._minute - otoff return _cmp((myhhmm, self._second, self._microsecond), @@ -1838,7 +1838,7 @@ other._hour, other._minute, other._second, other._microsecond)) if myoff is None or otoff is None: - raise TypeError("cannot compare naive and aware datetimes") + raise TypeError("can't compare offset-naive and offset-aware datetimes") # XXX What follows could be done more efficiently... diff = self - other # this will take offsets into account if diff.days < 0: @@ -1885,7 +1885,7 @@ if myoff == otoff: return base if myoff is None or otoff is None: - raise TypeError("cannot mix naive and timezone-aware time") + raise TypeError("can't compare offset-naive and offset-aware datetimes") return base + timedelta(minutes = otoff-myoff) def __hash__(self): diff --git a/pypy/module/test_lib_pypy/test_datetime.py b/pypy/module/test_lib_pypy/test_datetime.py --- a/pypy/module/test_lib_pypy/test_datetime.py +++ b/pypy/module/test_lib_pypy/test_datetime.py @@ -191,3 +191,21 @@ def __radd__(self, other): return "radd" assert datetime.date(10, 10, 10) + X() == "radd" + +def test_raises_if_passed_naive_datetime_and_start_or_end_time_defined(): + class Foo(datetime.tzinfo): + def utcoffset(self, dt): + return datetime.timedelta(0.1) + naive = datetime.datetime(2014, 9, 22) + aware = datetime.datetime(2014, 9, 22, tzinfo=Foo()) + with py.test.raises(TypeError) as e: + naive == aware + assert str(e.value) == "can't compare offset-naive and offset-aware datetimes" + with py.test.raises(TypeError) as e: + naive - aware + assert str(e.value) == "can't compare offset-naive and offset-aware datetimes" + naive = datetime.time(7, 32, 12) + aware = datetime.time(7, 32, 12, tzinfo=Foo()) + with py.test.raises(TypeError) as e: + naive == aware + assert str(e.value) == "can't compare offset-naive and offset-aware datetimes" From noreply at buildbot.pypy.org Mon Sep 22 18:23:57 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 22 Sep 2014 18:23:57 +0200 (CEST) Subject: [pypy-commit] pypy default: fix datetime error messages properly Message-ID: <20140922162357.713191C03FC@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r73648:302f635e170f Date: 2014-09-22 12:23 -0400 http://bitbucket.org/pypy/pypy/changeset/302f635e170f/ Log: fix datetime error messages properly diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py --- a/lib_pypy/datetime.py +++ b/lib_pypy/datetime.py @@ -1242,7 +1242,7 @@ (other._hour, other._minute, other._second, other._microsecond)) if myoff is None or otoff is None: - raise TypeError("can't compare offset-naive and offset-aware datetimes") + raise TypeError("can't compare offset-naive and offset-aware times") myhhmm = self._hour * 60 + self._minute - myoff othhmm = other._hour * 60 + other._minute - otoff return _cmp((myhhmm, self._second, self._microsecond), @@ -1885,7 +1885,7 @@ if myoff == otoff: return base if myoff is None or otoff is None: - raise TypeError("can't compare offset-naive and offset-aware datetimes") + raise TypeError("can't subtract offset-naive and offset-aware datetimes") return base + timedelta(minutes = otoff-myoff) def __hash__(self): diff --git a/pypy/module/test_lib_pypy/test_datetime.py b/pypy/module/test_lib_pypy/test_datetime.py --- a/pypy/module/test_lib_pypy/test_datetime.py +++ b/pypy/module/test_lib_pypy/test_datetime.py @@ -203,9 +203,9 @@ assert str(e.value) == "can't compare offset-naive and offset-aware datetimes" with py.test.raises(TypeError) as e: naive - aware - assert str(e.value) == "can't compare offset-naive and offset-aware datetimes" + assert str(e.value) == "can't subtract offset-naive and offset-aware datetimes" naive = datetime.time(7, 32, 12) aware = datetime.time(7, 32, 12, tzinfo=Foo()) with py.test.raises(TypeError) as e: naive == aware - assert str(e.value) == "can't compare offset-naive and offset-aware datetimes" + assert str(e.value) == "can't compare offset-naive and offset-aware times" From noreply at buildbot.pypy.org Mon Sep 22 18:59:49 2014 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 22 Sep 2014 18:59:49 +0200 (CEST) Subject: [pypy-commit] pypy.org extradoc: update hashes for fix osx package Message-ID: <20140922165949.4FF0F1C03FC@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: extradoc Changeset: r539:b5ad8cbd82e5 Date: 2014-09-22 19:59 +0300 http://bitbucket.org/pypy/pypy.org/changeset/b5ad8cbd82e5/ Log: update hashes for fix osx package diff --git a/download.html b/download.html --- a/download.html +++ b/download.html @@ -271,7 +271,7 @@ 59ecfb3084e725fb5fd5ccedf8e0686a pypy-2.4.0-src.zip 907d6fbabc5bcd5bafdcf02a76a8ca33 pypy-2.4.0-win32.zip 5487df96f47d93fc1c45265c26f17945 pypy-2.4.0-src.tar.bz2 -62d2894064234c8d010d773534d0c69a pypy-2.4.0-osx64.tar.bz2 +07896c0ac37f82884e021c9a4514f479 pypy-2.4.0-osx64.tar.bz2 eaba228fa7ba3f7022804900f9c9f885 pypy-2.4.0-sandbox-linux64.tar.bz2 9b2aaf173bf424ca12097a13fc58eb0d pypy-2.4.0-sandbox-linux.tar.bz2 a86da5688dfd84e0485239f8991af308 pypy3-2.3.1-linux64.tar.bz2 @@ -293,7 +293,7 @@ fdc25708de28f7e46caa4435507c44f534f6c817 pypy-2.4.0-src.tar.bz2 16e9f954e2829fc3961c922d016259f44ac55a0e pypy-2.4.0-src.zip b72c3365c23c34ffd35a781fb72d8722e0b7517e pypy-2.4.0-win32.zip -b9f87df364617c1a8c43303a467d698e4a3e1935 pypy-2.4.0-osx64.tar.bz2 +aa7f9b41d8bfda16239b629cd1b8dc884c2ad808 pypy-2.4.0-osx64.tar.bz2 17d6cb1818d3d5c4d0a099e1af518a437609f0c1 pypy-2.4.0-sandbox-linux64.tar.bz2 6a590a4f935b7e5fc95b6a3e783819b5ad2a85ef pypy-2.4.0-sandbox-linux.tar.bz2 7276a9e97229e754f66863a0793c59066120ec51 pypy3-2.3.1-linux64.tar.bz2 diff --git a/source/download.txt b/source/download.txt --- a/source/download.txt +++ b/source/download.txt @@ -312,7 +312,7 @@ 59ecfb3084e725fb5fd5ccedf8e0686a pypy-2.4.0-src.zip 907d6fbabc5bcd5bafdcf02a76a8ca33 pypy-2.4.0-win32.zip 5487df96f47d93fc1c45265c26f17945 pypy-2.4.0-src.tar.bz2 - 62d2894064234c8d010d773534d0c69a pypy-2.4.0-osx64.tar.bz2 + 07896c0ac37f82884e021c9a4514f479 pypy-2.4.0-osx64.tar.bz2 eaba228fa7ba3f7022804900f9c9f885 pypy-2.4.0-sandbox-linux64.tar.bz2 9b2aaf173bf424ca12097a13fc58eb0d pypy-2.4.0-sandbox-linux.tar.bz2 @@ -336,7 +336,7 @@ fdc25708de28f7e46caa4435507c44f534f6c817 pypy-2.4.0-src.tar.bz2 16e9f954e2829fc3961c922d016259f44ac55a0e pypy-2.4.0-src.zip b72c3365c23c34ffd35a781fb72d8722e0b7517e pypy-2.4.0-win32.zip - b9f87df364617c1a8c43303a467d698e4a3e1935 pypy-2.4.0-osx64.tar.bz2 + aa7f9b41d8bfda16239b629cd1b8dc884c2ad808 pypy-2.4.0-osx64.tar.bz2 17d6cb1818d3d5c4d0a099e1af518a437609f0c1 pypy-2.4.0-sandbox-linux64.tar.bz2 6a590a4f935b7e5fc95b6a3e783819b5ad2a85ef pypy-2.4.0-sandbox-linux.tar.bz2 From noreply at buildbot.pypy.org Mon Sep 22 19:02:58 2014 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 22 Sep 2014 19:02:58 +0200 (CEST) Subject: [pypy-commit] extradoc extradoc: Start the Warsaw sprint announcement Message-ID: <20140922170258.ED4A01C072F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r5413:80e359c7ee8f Date: 2014-09-22 19:03 +0200 http://bitbucket.org/pypy/extradoc/changeset/80e359c7ee8f/ Log: Start the Warsaw sprint announcement diff --git a/sprintinfo/warsaw-2014/announcement.txt b/sprintinfo/warsaw-2014/announcement.txt new file mode 100644 --- /dev/null +++ b/sprintinfo/warsaw-2014/announcement.txt @@ -0,0 +1,60 @@ +===================================================================== + PyPy Warsaw Sprint (October 21-25th, 2014) +===================================================================== + +The next PyPy sprint will be in Warsaw, Poland for the first +time. This is a fully public sprint. PyPy sprints are a very good way +to get into PyPy development and no prior PyPy knowledge is necessary. + + +------------------------------ +Goals and topics of the sprint +------------------------------ + +For newcomers: + +* Bring your application or library and we'll help you port it to PyPy + (if needed), benchmark and profile. + +* The easiest way to start hacking on PyPy is to write support for + some missing Python 3.3 functionality, or to work on numpy. + +We'll also work on more specific topics, depending on who is here +and what their interest is, like some missing GC/JIT optimizations, +software transactional memory, etc. + + +----------- +Exact times +----------- + +The work days should be October 21st - 25th, 2014. There might be +a day or an afternoon of break in the middle. We'll typically start +at 10:00 in the morning. + + +------------ +Location +------------ + +The sprint will happen within a room of Warsaw University. The +address is Pasteura 5 (which is a form of "Pasteur street"), room 550. +The person of contact is Maciej Fijalkowski. + + +-------------- +Registration +-------------- + +If you want to attend, please register by adding yourself to the +"people.txt" file in Mercurial:: + + https://bitbucket.org/pypy/extradoc/ + https://bitbucket.org/pypy/extradoc/raw/extradoc/sprintinfo/warsaw-2014 + +or on the pypy-dev mailing list if you do not yet have check-in rights:: + + http://mail.python.org/mailman/listinfo/pypy-dev + +Remember that Poland is a regular Schengen zone EU country, with +main-EU-zone power adapters. diff --git a/sprintinfo/warsaw-2014/people.txt b/sprintinfo/warsaw-2014/people.txt new file mode 100644 --- /dev/null +++ b/sprintinfo/warsaw-2014/people.txt @@ -0,0 +1,13 @@ +People coming to the Warsaw sprint 2014 +================================================== + +People who have a ``?`` in their arrive/depart or accomodation +column are known to be coming but there are no details +available from them. + + +==================== ============== ======================= + Name Arrive/Depart Accomodation +==================== ============== ======================= +Armin Rigo 20/10-2X/10 with fijal? +==================== ============== ======================= From noreply at buildbot.pypy.org Mon Sep 22 19:12:19 2014 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 22 Sep 2014 19:12:19 +0200 (CEST) Subject: [pypy-commit] pypy.org extradoc: update the values Message-ID: <20140922171219.528DB1C035D@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r540:9c846d4897bf Date: 2014-09-22 19:12 +0200 http://bitbucket.org/pypy/pypy.org/changeset/9c846d4897bf/ Log: update the values diff --git a/don1.html b/don1.html --- a/don1.html +++ b/don1.html @@ -9,13 +9,13 @@ - $53651 of $105000 (51.1%) + $57384 of $105000 (54.7%)
    From noreply at buildbot.pypy.org Mon Sep 22 19:38:43 2014 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 22 Sep 2014 19:38:43 +0200 (CEST) Subject: [pypy-commit] pypy release-2.4.x: Moved tag release-2.4.0 to changeset c6ad44ecf5d8 (from changeset f5dcc2477b97) Message-ID: <20140922173843.AFDEC1C072F@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: release-2.4.x Changeset: r73649:a980ebb26592 Date: 2014-09-22 20:34 +0300 http://bitbucket.org/pypy/pypy/changeset/a980ebb26592/ Log: Moved tag release-2.4.0 to changeset c6ad44ecf5d8 (from changeset f5dcc2477b97) diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -16,3 +16,5 @@ 0000000000000000000000000000000000000000 release-2.2=3.1 9f425c60afdf2c1d38ee016071cffc55c28048b9 release-2.4-beta1 f5dcc2477b97386c11e4b67f08a2d00fbd2fce5d release-2.4.0 +f5dcc2477b97386c11e4b67f08a2d00fbd2fce5d release-2.4.0 +c6ad44ecf5d869284b656518632a1b0cb9db0c27 release-2.4.0 From noreply at buildbot.pypy.org Mon Sep 22 20:01:24 2014 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 22 Sep 2014 20:01:24 +0200 (CEST) Subject: [pypy-commit] pypy.org extradoc: Update the src.tar.bz2. Kill the src.zip. Message-ID: <20140922180124.5BF1C1C072F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r541:575a55f60b74 Date: 2014-09-22 20:01 +0200 http://bitbucket.org/pypy/pypy.org/changeset/575a55f60b74/ Log: Update the src.tar.bz2. Kill the src.zip. diff --git a/download.html b/download.html --- a/download.html +++ b/download.html @@ -191,8 +191,7 @@

  • Get the source code. The following packages contain the source at the same revision as the above binaries:

    Or you can checkout the current trunk using Mercurial (the trunk usually works and is of course more up-to-date):

    @@ -268,9 +267,8 @@ 5ff951da5989a00e01611678c311f8af pypy-2.4.0-linux-armhf-raring.tar.bz2 d7540883a52f91433da62b0cdfaaa30f pypy-2.4.0-linux-armhf-raspbian.tar.bz2 77a971f5198685ff60528de5def407dd pypy-2.4.0-linux.tar.bz2 -59ecfb3084e725fb5fd5ccedf8e0686a pypy-2.4.0-src.zip 907d6fbabc5bcd5bafdcf02a76a8ca33 pypy-2.4.0-win32.zip -5487df96f47d93fc1c45265c26f17945 pypy-2.4.0-src.tar.bz2 +6a25a212e7c5121f1f3988c118d05695 pypy-2.4.0-src.tar.bz2 07896c0ac37f82884e021c9a4514f479 pypy-2.4.0-osx64.tar.bz2 eaba228fa7ba3f7022804900f9c9f885 pypy-2.4.0-sandbox-linux64.tar.bz2 9b2aaf173bf424ca12097a13fc58eb0d pypy-2.4.0-sandbox-linux.tar.bz2 @@ -290,8 +288,7 @@ b8e02dc381e5040e2bf50541e82f0148f9a46a48 pypy-2.4.0-linux-armhf-raring.tar.bz2 ad65e7ddb1582b465a37090dc4a13bc37a8cd95d pypy-2.4.0-linux-armhf-raspbian.tar.bz2 fd52b42069287ca11e816c8e18fc95f53542c73d pypy-2.4.0-linux.tar.bz2 -fdc25708de28f7e46caa4435507c44f534f6c817 pypy-2.4.0-src.tar.bz2 -16e9f954e2829fc3961c922d016259f44ac55a0e pypy-2.4.0-src.zip +e2e0bcf8457c0ae5a24f126a60aa921dabfe60fb pypy-2.4.0-src.tar.bz2 b72c3365c23c34ffd35a781fb72d8722e0b7517e pypy-2.4.0-win32.zip aa7f9b41d8bfda16239b629cd1b8dc884c2ad808 pypy-2.4.0-osx64.tar.bz2 17d6cb1818d3d5c4d0a099e1af518a437609f0c1 pypy-2.4.0-sandbox-linux64.tar.bz2 diff --git a/source/download.txt b/source/download.txt --- a/source/download.txt +++ b/source/download.txt @@ -220,11 +220,9 @@ 1. Get the source code. The following packages contain the source at the same revision as the above binaries: - * `pypy-2.4.0-src.tar.bz2`__ (sources, Unix line endings) - * `pypy-2.4.0-src.zip`__ (sources, Unix line endings too) + * `pypy-2.4.0-src.tar.bz2`__ (sources) .. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.4.0-src.tar.bz2 - .. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.4.0-src.zip Or you can checkout the current trunk using Mercurial_ (the trunk usually works and is of course more up-to-date):: @@ -309,9 +307,8 @@ 5ff951da5989a00e01611678c311f8af pypy-2.4.0-linux-armhf-raring.tar.bz2 d7540883a52f91433da62b0cdfaaa30f pypy-2.4.0-linux-armhf-raspbian.tar.bz2 77a971f5198685ff60528de5def407dd pypy-2.4.0-linux.tar.bz2 - 59ecfb3084e725fb5fd5ccedf8e0686a pypy-2.4.0-src.zip 907d6fbabc5bcd5bafdcf02a76a8ca33 pypy-2.4.0-win32.zip - 5487df96f47d93fc1c45265c26f17945 pypy-2.4.0-src.tar.bz2 + 6a25a212e7c5121f1f3988c118d05695 pypy-2.4.0-src.tar.bz2 07896c0ac37f82884e021c9a4514f479 pypy-2.4.0-osx64.tar.bz2 eaba228fa7ba3f7022804900f9c9f885 pypy-2.4.0-sandbox-linux64.tar.bz2 9b2aaf173bf424ca12097a13fc58eb0d pypy-2.4.0-sandbox-linux.tar.bz2 @@ -333,8 +330,7 @@ b8e02dc381e5040e2bf50541e82f0148f9a46a48 pypy-2.4.0-linux-armhf-raring.tar.bz2 ad65e7ddb1582b465a37090dc4a13bc37a8cd95d pypy-2.4.0-linux-armhf-raspbian.tar.bz2 fd52b42069287ca11e816c8e18fc95f53542c73d pypy-2.4.0-linux.tar.bz2 - fdc25708de28f7e46caa4435507c44f534f6c817 pypy-2.4.0-src.tar.bz2 - 16e9f954e2829fc3961c922d016259f44ac55a0e pypy-2.4.0-src.zip + e2e0bcf8457c0ae5a24f126a60aa921dabfe60fb pypy-2.4.0-src.tar.bz2 b72c3365c23c34ffd35a781fb72d8722e0b7517e pypy-2.4.0-win32.zip aa7f9b41d8bfda16239b629cd1b8dc884c2ad808 pypy-2.4.0-osx64.tar.bz2 17d6cb1818d3d5c4d0a099e1af518a437609f0c1 pypy-2.4.0-sandbox-linux64.tar.bz2 From noreply at buildbot.pypy.org Mon Sep 22 20:04:46 2014 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 22 Sep 2014 20:04:46 +0200 (CEST) Subject: [pypy-commit] pypy.org extradoc: Kill links to the (non-working, removed) sandbox version 2.4.0. Message-ID: <20140922180446.8709F1C072F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r542:fb6771ba1bf0 Date: 2014-09-22 20:04 +0200 http://bitbucket.org/pypy/pypy.org/changeset/fb6771ba1bf0/ Log: Kill links to the (non-working, removed) sandbox version 2.4.0. diff --git a/source/download.txt b/source/download.txt --- a/source/download.txt +++ b/source/download.txt @@ -310,8 +310,6 @@ 907d6fbabc5bcd5bafdcf02a76a8ca33 pypy-2.4.0-win32.zip 6a25a212e7c5121f1f3988c118d05695 pypy-2.4.0-src.tar.bz2 07896c0ac37f82884e021c9a4514f479 pypy-2.4.0-osx64.tar.bz2 - eaba228fa7ba3f7022804900f9c9f885 pypy-2.4.0-sandbox-linux64.tar.bz2 - 9b2aaf173bf424ca12097a13fc58eb0d pypy-2.4.0-sandbox-linux.tar.bz2 a86da5688dfd84e0485239f8991af308 pypy3-2.3.1-linux64.tar.bz2 b0d6a0491e9c9be39d3c314c0823a039 pypy3-2.3.1-linux-armel.tar.bz2 @@ -333,9 +331,6 @@ e2e0bcf8457c0ae5a24f126a60aa921dabfe60fb pypy-2.4.0-src.tar.bz2 b72c3365c23c34ffd35a781fb72d8722e0b7517e pypy-2.4.0-win32.zip aa7f9b41d8bfda16239b629cd1b8dc884c2ad808 pypy-2.4.0-osx64.tar.bz2 - 17d6cb1818d3d5c4d0a099e1af518a437609f0c1 pypy-2.4.0-sandbox-linux64.tar.bz2 - 6a590a4f935b7e5fc95b6a3e783819b5ad2a85ef pypy-2.4.0-sandbox-linux.tar.bz2 - 7276a9e97229e754f66863a0793c59066120ec51 pypy3-2.3.1-linux64.tar.bz2 fb52a30be0fd4c7d8686c98e03595a8b48b11e82 pypy3-2.3.1-linux-armel.tar.bz2 From noreply at buildbot.pypy.org Mon Sep 22 20:06:10 2014 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 22 Sep 2014 20:06:10 +0200 (CEST) Subject: [pypy-commit] pypy.org extradoc: Regenerate .html Message-ID: <20140922180610.8CCA91C072F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r543:a43b044f14c7 Date: 2014-09-22 20:06 +0200 http://bitbucket.org/pypy/pypy.org/changeset/a43b044f14c7/ Log: Regenerate .html diff --git a/download.html b/download.html --- a/download.html +++ b/download.html @@ -270,8 +270,6 @@ 907d6fbabc5bcd5bafdcf02a76a8ca33 pypy-2.4.0-win32.zip 6a25a212e7c5121f1f3988c118d05695 pypy-2.4.0-src.tar.bz2 07896c0ac37f82884e021c9a4514f479 pypy-2.4.0-osx64.tar.bz2 -eaba228fa7ba3f7022804900f9c9f885 pypy-2.4.0-sandbox-linux64.tar.bz2 -9b2aaf173bf424ca12097a13fc58eb0d pypy-2.4.0-sandbox-linux.tar.bz2 a86da5688dfd84e0485239f8991af308 pypy3-2.3.1-linux64.tar.bz2 b0d6a0491e9c9be39d3c314c0823a039 pypy3-2.3.1-linux-armel.tar.bz2 f79f7b5588d2b5a68d2781908bc8f9af pypy3-2.3.1-linux-armhf-raring.tar.bz2 @@ -291,8 +289,6 @@ e2e0bcf8457c0ae5a24f126a60aa921dabfe60fb pypy-2.4.0-src.tar.bz2 b72c3365c23c34ffd35a781fb72d8722e0b7517e pypy-2.4.0-win32.zip aa7f9b41d8bfda16239b629cd1b8dc884c2ad808 pypy-2.4.0-osx64.tar.bz2 -17d6cb1818d3d5c4d0a099e1af518a437609f0c1 pypy-2.4.0-sandbox-linux64.tar.bz2 -6a590a4f935b7e5fc95b6a3e783819b5ad2a85ef pypy-2.4.0-sandbox-linux.tar.bz2 7276a9e97229e754f66863a0793c59066120ec51 pypy3-2.3.1-linux64.tar.bz2 fb52a30be0fd4c7d8686c98e03595a8b48b11e82 pypy3-2.3.1-linux-armel.tar.bz2 0239677fe28a4c22a70853242368456b98ac665a pypy3-2.3.1-linux-armhf-raring.tar.bz2 From noreply at buildbot.pypy.org Mon Sep 22 20:45:44 2014 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 22 Sep 2014 20:45:44 +0200 (CEST) Subject: [pypy-commit] pypy default: update final stages of release process Message-ID: <20140922184544.E69C61C035D@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r73650:d84f89597352 Date: 2014-09-22 21:45 +0300 http://bitbucket.org/pypy/pypy/changeset/d84f89597352/ Log: update final stages of release process diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst --- a/pypy/doc/how-to-release.rst +++ b/pypy/doc/how-to-release.rst @@ -38,14 +38,16 @@ no JIT: windows, linux, os/x sandbox: linux, os/x +* repackage and upload source tar.bz2 to bitbucket and to cobra, as some packagers + prefer a clearly labeled source package * write release announcement pypy/doc/release-x.y(.z).txt the release announcement should contain a direct link to the download page * update pypy.org (under extradoc/pypy.org), rebuild and commit * post announcement on morepypy.blogspot.com -* send announcements to pypy-dev, python-list, +* send announcements to twitter.com, pypy-dev, python-list, python-announce, python-dev ... * add a tag on the pypy/jitviewer repo that corresponds to pypy release * add a tag on the codespeed web site that corresponds to pypy release - +* revise versioning at https://readthedocs.org/projects/pypy From noreply at buildbot.pypy.org Mon Sep 22 23:42:49 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 22 Sep 2014 23:42:49 +0200 (CEST) Subject: [pypy-commit] pypy default: test datetime against both host and lib_pypy Message-ID: <20140922214249.977041C072F@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r73651:4ec1f9c52a23 Date: 2014-09-22 17:41 -0400 http://bitbucket.org/pypy/pypy/changeset/4ec1f9c52a23/ Log: test datetime against both host and lib_pypy diff --git a/pypy/module/test_lib_pypy/test_datetime.py b/pypy/module/test_lib_pypy/test_datetime.py --- a/pypy/module/test_lib_pypy/test_datetime.py +++ b/pypy/module/test_lib_pypy/test_datetime.py @@ -1,211 +1,221 @@ """Additional tests for datetime.""" from __future__ import absolute_import -from lib_pypy import datetime import py -def test_repr(): - print datetime - expected = "datetime.datetime(1, 2, 3, 0, 0)" - assert repr(datetime.datetime(1,2,3)) == expected +class BaseTestDatetime: + def test_repr(self): + print datetime + expected = "datetime.datetime(1, 2, 3, 0, 0)" + assert repr(datetime.datetime(1,2,3)) == expected -def test_attributes(): - for x in [datetime.date.today(), - datetime.time(), - datetime.datetime.utcnow(), - datetime.timedelta(), - datetime.tzinfo()]: - raises(AttributeError, 'x.abc = 1') + def test_attributes(self): + for x in [datetime.date.today(), + datetime.time(), + datetime.datetime.utcnow(), + datetime.timedelta(), + datetime.tzinfo()]: + raises(AttributeError, 'x.abc = 1') -def test_timedelta_init_long(): - td = datetime.timedelta(microseconds=20000000000000000000) - assert td.days == 231481481 - assert td.seconds == 41600 - td = datetime.timedelta(microseconds=20000000000000000000.) - assert td.days == 231481481 - assert td.seconds == 41600 + def test_timedelta_init_long(self): + td = datetime.timedelta(microseconds=20000000000000000000) + assert td.days == 231481481 + assert td.seconds == 41600 + td = datetime.timedelta(microseconds=20000000000000000000.) + assert td.days == 231481481 + assert td.seconds == 41600 -def test_unpickle(): - e = raises(TypeError, datetime.date, '123') - assert e.value.args[0] == 'an integer is required' - e = raises(TypeError, datetime.time, '123') - assert e.value.args[0] == 'an integer is required' - e = raises(TypeError, datetime.datetime, '123') - assert e.value.args[0] == 'an integer is required' + def test_unpickle(self): + e = raises(TypeError, datetime.date, '123') + assert e.value.args[0] == 'an integer is required' + e = raises(TypeError, datetime.time, '123') + assert e.value.args[0] == 'an integer is required' + e = raises(TypeError, datetime.datetime, '123') + assert e.value.args[0] == 'an integer is required' - datetime.time('\x01' * 6, None) - with raises(TypeError) as e: - datetime.time('\x01' * 6, 123) - assert str(e.value) == "bad tzinfo state arg" + datetime.time('\x01' * 6, None) + with raises(TypeError) as e: + datetime.time('\x01' * 6, 123) + assert str(e.value) == "bad tzinfo state arg" - datetime.datetime('\x01' * 10, None) - with raises(TypeError) as e: - datetime.datetime('\x01' * 10, 123) - assert str(e.value) == "bad tzinfo state arg" + datetime.datetime('\x01' * 10, None) + with raises(TypeError) as e: + datetime.datetime('\x01' * 10, 123) + assert str(e.value) == "bad tzinfo state arg" -def test_strptime(): - import time, sys - if sys.version_info < (2, 6): - py.test.skip("needs the _strptime module") + def test_strptime(self): + import time, sys + if sys.version_info < (2, 6): + py.test.skip("needs the _strptime module") - string = '2004-12-01 13:02:47' - format = '%Y-%m-%d %H:%M:%S' - expected = datetime.datetime(*(time.strptime(string, format)[0:6])) - got = datetime.datetime.strptime(string, format) - assert expected == got + string = '2004-12-01 13:02:47' + format = '%Y-%m-%d %H:%M:%S' + expected = datetime.datetime(*(time.strptime(string, format)[0:6])) + got = datetime.datetime.strptime(string, format) + assert expected == got -def test_datetime_rounding(): - b = 0.0000001 - a = 0.9999994 + def test_datetime_rounding(self): + b = 0.0000001 + a = 0.9999994 - assert datetime.datetime.utcfromtimestamp(a).microsecond == 999999 - assert datetime.datetime.utcfromtimestamp(a).second == 0 - a += b - assert datetime.datetime.utcfromtimestamp(a).microsecond == 999999 - assert datetime.datetime.utcfromtimestamp(a).second == 0 - a += b - assert datetime.datetime.utcfromtimestamp(a).microsecond == 0 - assert datetime.datetime.utcfromtimestamp(a).second == 1 + assert datetime.datetime.utcfromtimestamp(a).microsecond == 999999 + assert datetime.datetime.utcfromtimestamp(a).second == 0 + a += b + assert datetime.datetime.utcfromtimestamp(a).microsecond == 999999 + assert datetime.datetime.utcfromtimestamp(a).second == 0 + a += b + assert datetime.datetime.utcfromtimestamp(a).microsecond == 0 + assert datetime.datetime.utcfromtimestamp(a).second == 1 -def test_more_datetime_rounding(): - # this test verified on top of CPython 2.7 (using a plain - # "import datetime" above) - expected_results = { - -1000.0: 'datetime.datetime(1969, 12, 31, 23, 43, 20)', - -999.9999996: 'datetime.datetime(1969, 12, 31, 23, 43, 20)', - -999.4: 'datetime.datetime(1969, 12, 31, 23, 43, 20, 600000)', - -999.0000004: 'datetime.datetime(1969, 12, 31, 23, 43, 21)', - -1.0: 'datetime.datetime(1969, 12, 31, 23, 59, 59)', - -0.9999996: 'datetime.datetime(1969, 12, 31, 23, 59, 59)', - -0.4: 'datetime.datetime(1969, 12, 31, 23, 59, 59, 600000)', - -0.0000004: 'datetime.datetime(1970, 1, 1, 0, 0)', - 0.0: 'datetime.datetime(1970, 1, 1, 0, 0)', - 0.0000004: 'datetime.datetime(1970, 1, 1, 0, 0)', - 0.4: 'datetime.datetime(1970, 1, 1, 0, 0, 0, 400000)', - 0.9999996: 'datetime.datetime(1970, 1, 1, 0, 0, 1)', - 1000.0: 'datetime.datetime(1970, 1, 1, 0, 16, 40)', - 1000.0000004: 'datetime.datetime(1970, 1, 1, 0, 16, 40)', - 1000.4: 'datetime.datetime(1970, 1, 1, 0, 16, 40, 400000)', - 1000.9999996: 'datetime.datetime(1970, 1, 1, 0, 16, 41)', - 1293843661.191: 'datetime.datetime(2011, 1, 1, 1, 1, 1, 191000)', - } - for t in sorted(expected_results): - dt = datetime.datetime.utcfromtimestamp(t) - assert repr(dt) == expected_results[t] + def test_more_datetime_rounding(self): + # this test verified on top of CPython 2.7 (using a plain + # "import datetime" above) + expected_results = { + -1000.0: 'datetime.datetime(1969, 12, 31, 23, 43, 20)', + -999.9999996: 'datetime.datetime(1969, 12, 31, 23, 43, 20)', + -999.4: 'datetime.datetime(1969, 12, 31, 23, 43, 20, 600000)', + -999.0000004: 'datetime.datetime(1969, 12, 31, 23, 43, 21)', + -1.0: 'datetime.datetime(1969, 12, 31, 23, 59, 59)', + -0.9999996: 'datetime.datetime(1969, 12, 31, 23, 59, 59)', + -0.4: 'datetime.datetime(1969, 12, 31, 23, 59, 59, 600000)', + -0.0000004: 'datetime.datetime(1970, 1, 1, 0, 0)', + 0.0: 'datetime.datetime(1970, 1, 1, 0, 0)', + 0.0000004: 'datetime.datetime(1970, 1, 1, 0, 0)', + 0.4: 'datetime.datetime(1970, 1, 1, 0, 0, 0, 400000)', + 0.9999996: 'datetime.datetime(1970, 1, 1, 0, 0, 1)', + 1000.0: 'datetime.datetime(1970, 1, 1, 0, 16, 40)', + 1000.0000004: 'datetime.datetime(1970, 1, 1, 0, 16, 40)', + 1000.4: 'datetime.datetime(1970, 1, 1, 0, 16, 40, 400000)', + 1000.9999996: 'datetime.datetime(1970, 1, 1, 0, 16, 41)', + 1293843661.191: 'datetime.datetime(2011, 1, 1, 1, 1, 1, 191000)', + } + for t in sorted(expected_results): + dt = datetime.datetime.utcfromtimestamp(t) + assert repr(dt) == expected_results[t] -def test_utcfromtimestamp(): - """Confirm that utcfromtimestamp and fromtimestamp give consistent results. + def test_utcfromtimestamp(self): + """Confirm that utcfromtimestamp and fromtimestamp give consistent results. - Based on danchr's test script in https://bugs.pypy.org/issue986 - """ - import os - import time - if os.name == 'nt': - skip("setting os.environ['TZ'] ineffective on windows") - try: - prev_tz = os.environ.get("TZ") - os.environ["TZ"] = "GMT" - time.tzset() - for unused in xrange(100): - now = time.time() - delta = (datetime.datetime.utcfromtimestamp(now) - - datetime.datetime.fromtimestamp(now)) - assert delta.days * 86400 + delta.seconds == 0 - finally: - if prev_tz is None: - del os.environ["TZ"] - else: - os.environ["TZ"] = prev_tz - time.tzset() + Based on danchr's test script in https://bugs.pypy.org/issue986 + """ + import os + import time + if os.name == 'nt': + skip("setting os.environ['TZ'] ineffective on windows") + try: + prev_tz = os.environ.get("TZ") + os.environ["TZ"] = "GMT" + time.tzset() + for unused in xrange(100): + now = time.time() + delta = (datetime.datetime.utcfromtimestamp(now) - + datetime.datetime.fromtimestamp(now)) + assert delta.days * 86400 + delta.seconds == 0 + finally: + if prev_tz is None: + del os.environ["TZ"] + else: + os.environ["TZ"] = prev_tz + time.tzset() -def test_utcfromtimestamp_microsecond(): - dt = datetime.datetime.utcfromtimestamp(0) - assert isinstance(dt.microsecond, int) + def test_utcfromtimestamp_microsecond(self): + dt = datetime.datetime.utcfromtimestamp(0) + assert isinstance(dt.microsecond, int) -def test_default_args(): - with py.test.raises(TypeError): - datetime.datetime() - with py.test.raises(TypeError): - datetime.datetime(10) - with py.test.raises(TypeError): - datetime.datetime(10, 10) - datetime.datetime(10, 10, 10) + def test_default_args(self): + with py.test.raises(TypeError): + datetime.datetime() + with py.test.raises(TypeError): + datetime.datetime(10) + with py.test.raises(TypeError): + datetime.datetime(10, 10) + datetime.datetime(10, 10, 10) -def test_check_arg_types(): - import decimal - class Number: - def __init__(self, value): - self.value = value - def __int__(self): - return self.value + def test_check_arg_types(self): + import decimal + class Number: + def __init__(self, value): + self.value = value + def __int__(self): + return self.value - for xx in [10L, - decimal.Decimal(10), - decimal.Decimal('10.9'), - Number(10), - Number(10L)]: - assert datetime.datetime(10, 10, 10, 10, 10, 10, 10) == \ - datetime.datetime(xx, xx, xx, xx, xx, xx, xx) + for xx in [10L, + decimal.Decimal(10), + decimal.Decimal('10.9'), + Number(10), + Number(10L)]: + assert datetime.datetime(10, 10, 10, 10, 10, 10, 10) == \ + datetime.datetime(xx, xx, xx, xx, xx, xx, xx) - with py.test.raises(TypeError) as e: - datetime.datetime(10, 10, '10') - assert str(e.value) == 'an integer is required' + with py.test.raises(TypeError) as e: + datetime.datetime(10, 10, '10') + assert str(e.value) == 'an integer is required' - f10 = Number(10.9) - with py.test.raises(TypeError) as e: - datetime.datetime(10, 10, f10) - assert str(e.value) == '__int__ method should return an integer' + f10 = Number(10.9) + with py.test.raises(TypeError) as e: + datetime.datetime(10, 10, f10) + assert str(e.value) == '__int__ method should return an integer' - class Float(float): - pass - s10 = Float(10.9) - with py.test.raises(TypeError) as e: - datetime.datetime(10, 10, s10) - assert str(e.value) == 'integer argument expected, got float' + class Float(float): + pass + s10 = Float(10.9) + with py.test.raises(TypeError) as e: + datetime.datetime(10, 10, s10) + assert str(e.value) == 'integer argument expected, got float' - with py.test.raises(TypeError): - datetime.datetime(10., 10, 10) - with py.test.raises(TypeError): - datetime.datetime(10, 10., 10) - with py.test.raises(TypeError): - datetime.datetime(10, 10, 10.) - with py.test.raises(TypeError): - datetime.datetime(10, 10, 10, 10.) - with py.test.raises(TypeError): - datetime.datetime(10, 10, 10, 10, 10.) - with py.test.raises(TypeError): - datetime.datetime(10, 10, 10, 10, 10, 10.) - with py.test.raises(TypeError): - datetime.datetime(10, 10, 10, 10, 10, 10, 10.) + with py.test.raises(TypeError): + datetime.datetime(10., 10, 10) + with py.test.raises(TypeError): + datetime.datetime(10, 10., 10) + with py.test.raises(TypeError): + datetime.datetime(10, 10, 10.) + with py.test.raises(TypeError): + datetime.datetime(10, 10, 10, 10.) + with py.test.raises(TypeError): + datetime.datetime(10, 10, 10, 10, 10.) + with py.test.raises(TypeError): + datetime.datetime(10, 10, 10, 10, 10, 10.) + with py.test.raises(TypeError): + datetime.datetime(10, 10, 10, 10, 10, 10, 10.) -def test_utcnow_microsecond(): - import copy + def test_utcnow_microsecond(self): + import copy - dt = datetime.datetime.utcnow() - assert type(dt.microsecond) is int + dt = datetime.datetime.utcnow() + assert type(dt.microsecond) is int - copy.copy(dt) + copy.copy(dt) -def test_radd(): - class X(object): - def __radd__(self, other): - return "radd" - assert datetime.date(10, 10, 10) + X() == "radd" + def test_radd(self): + class X(object): + def __radd__(self, other): + return "radd" + assert datetime.date(10, 10, 10) + X() == "radd" -def test_raises_if_passed_naive_datetime_and_start_or_end_time_defined(): - class Foo(datetime.tzinfo): - def utcoffset(self, dt): - return datetime.timedelta(0.1) - naive = datetime.datetime(2014, 9, 22) - aware = datetime.datetime(2014, 9, 22, tzinfo=Foo()) - with py.test.raises(TypeError) as e: - naive == aware - assert str(e.value) == "can't compare offset-naive and offset-aware datetimes" - with py.test.raises(TypeError) as e: - naive - aware - assert str(e.value) == "can't subtract offset-naive and offset-aware datetimes" - naive = datetime.time(7, 32, 12) - aware = datetime.time(7, 32, 12, tzinfo=Foo()) - with py.test.raises(TypeError) as e: - naive == aware - assert str(e.value) == "can't compare offset-naive and offset-aware times" + def test_raises_if_passed_naive_datetime_and_start_or_end_time_defined(self): + class Foo(datetime.tzinfo): + def utcoffset(self, dt): + return datetime.timedelta(0.1) + naive = datetime.datetime(2014, 9, 22) + aware = datetime.datetime(2014, 9, 22, tzinfo=Foo()) + with py.test.raises(TypeError) as e: + naive == aware + assert str(e.value) == "can't compare offset-naive and offset-aware datetimes" + with py.test.raises(TypeError) as e: + naive - aware + assert str(e.value) == "can't subtract offset-naive and offset-aware datetimes" + naive = datetime.time(7, 32, 12) + aware = datetime.time(7, 32, 12, tzinfo=Foo()) + with py.test.raises(TypeError) as e: + naive == aware + assert str(e.value) == "can't compare offset-naive and offset-aware times" + +class TestDatetimeCPython(BaseTestDatetime): + def setup_class(cls): + global datetime + import datetime + +class TestDatetimePyPy(BaseTestDatetime): + def setup_class(cls): + global datetime + from lib_pypy import datetime From noreply at buildbot.pypy.org Tue Sep 23 08:59:51 2014 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 23 Sep 2014 08:59:51 +0200 (CEST) Subject: [pypy-commit] pypy default: Update the link (noticed, but for no reason I understand not fixed, by mjacob) Message-ID: <20140923065951.BD53A1C03FC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73652:fff03165885f Date: 2014-09-23 08:59 +0200 http://bitbucket.org/pypy/pypy/changeset/fff03165885f/ Log: Update the link (noticed, but for no reason I understand not fixed, by mjacob) diff --git a/pypy/doc/release-2.4.0.rst b/pypy/doc/release-2.4.0.rst --- a/pypy/doc/release-2.4.0.rst +++ b/pypy/doc/release-2.4.0.rst @@ -105,7 +105,7 @@ * Many issues were resolved_ since the 2.3.1 release on June 8 -.. _`whats-new`: http://doc.pypy.org/en/latest/whatsnew-2.3.1.html +.. _`whats-new`: http://doc.pypy.org/en/latest/whatsnew-2.4.0.html .. _resolved: https://bitbucket.org/pypy/pypy/issues?status=resolved .. _sandbox: http://doc.pypy.org/en/latest/sandbox.html From noreply at buildbot.pypy.org Tue Sep 23 09:01:50 2014 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 23 Sep 2014 09:01:50 +0200 (CEST) Subject: [pypy-commit] pypy release-2.4.x: Update in this branch too Message-ID: <20140923070150.E6CD51C03FC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: release-2.4.x Changeset: r73653:00b7a5a8db36 Date: 2014-09-23 09:01 +0200 http://bitbucket.org/pypy/pypy/changeset/00b7a5a8db36/ Log: Update in this branch too diff --git a/pypy/doc/release-2.4.0.rst b/pypy/doc/release-2.4.0.rst --- a/pypy/doc/release-2.4.0.rst +++ b/pypy/doc/release-2.4.0.rst @@ -103,7 +103,7 @@ * Many issues were resolved_ since the 2.3.1 release on June 8 -.. _`whats-new`: http://doc.pypy.org/en/latest/whatsnew-2.3.1.html +.. _`whats-new`: http://doc.pypy.org/en/latest/whatsnew-2.4.0.html .. _resolved: https://bitbucket.org/pypy/pypy/issues?status=resolved We have further improvements on the way: rpython file handling, From noreply at buildbot.pypy.org Tue Sep 23 10:45:07 2014 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 23 Sep 2014 10:45:07 +0200 (CEST) Subject: [pypy-commit] stmgc c8-new-page-handling: Add a comment Message-ID: <20140923084507.12F021C0605@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c8-new-page-handling Changeset: r1408:fe2e4b498547 Date: 2014-09-23 10:45 +0200 http://bitbucket.org/pypy/stmgc/changeset/fe2e4b498547/ Log: Add a comment diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -197,6 +197,9 @@ dprintf(("-> segment: %d\n", segnum)); char *seg_base = STM_SEGMENT->segment_base; uintptr_t pagenum = ((char*)addr - seg_base) / 4096UL; + /* XXX actual segfault also if the pagenum is out of bounds, + say < END_NURSERY_PAGE. Important for crashing cleanly on + %gs:NULL accesses */ handle_segfault_in_page(pagenum); From noreply at buildbot.pypy.org Tue Sep 23 11:25:29 2014 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 23 Sep 2014 11:25:29 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: merge default Message-ID: <20140923092529.F38701D23FA@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73654:aab99814dec1 Date: 2014-09-23 11:23 +0200 http://bitbucket.org/pypy/pypy/changeset/aab99814dec1/ Log: merge default diff too long, truncating to 2000 out of 3866 lines diff --git a/_pytest/resultlog.py b/_pytest/resultlog.py --- a/_pytest/resultlog.py +++ b/_pytest/resultlog.py @@ -54,15 +54,16 @@ self.logfile = logfile # preferably line buffered def write_log_entry(self, testpath, lettercode, longrepr, sections=None): - py.builtin.print_("%s %s" % (lettercode, testpath), file=self.logfile) + _safeprint("%s %s" % (lettercode, testpath), file=self.logfile) for line in longrepr.splitlines(): - py.builtin.print_(" %s" % line, file=self.logfile) - if sections is not None: + _safeprint(" %s" % line, file=self.logfile) + if sections is not None and ( + lettercode in ('E', 'F')): # to limit the size of logs for title, content in sections: - py.builtin.print_(" ---------- %s ----------" % (title,), - file=self.logfile) + _safeprint(" ---------- %s ----------" % (title,), + file=self.logfile) for line in content.splitlines(): - py.builtin.print_(" %s" % line, file=self.logfile) + _safeprint(" %s" % line, file=self.logfile) def log_outcome(self, report, lettercode, longrepr): testpath = getattr(report, 'nodeid', None) @@ -105,3 +106,8 @@ if path is None: path = "cwd:%s" % py.path.local() self.write_log_entry(path, '!', str(excrepr)) + +def _safeprint(s, file): + if isinstance(s, unicode): + s = s.encode('utf-8') + py.builtin.print_(s, file=file) diff --git a/lib-python/2.7/test/test_mmap.py b/lib-python/2.7/test/test_mmap.py --- a/lib-python/2.7/test/test_mmap.py +++ b/lib-python/2.7/test/test_mmap.py @@ -179,25 +179,27 @@ import sys f = open(TESTFN, "r+b") try: - m = mmap.mmap(f.fileno(), mapsize+1) - except ValueError: - # we do not expect a ValueError on Windows - # CAUTION: This also changes the size of the file on disk, and - # later tests assume that the length hasn't changed. We need to - # repair that. + try: + m = mmap.mmap(f.fileno(), mapsize+1) + except ValueError: + # we do not expect a ValueError on Windows + # CAUTION: This also changes the size of the file on disk, and + # later tests assume that the length hasn't changed. We need to + # repair that. + if sys.platform.startswith('win'): + self.fail("Opening mmap with size+1 should work on Windows.") + else: + # we expect a ValueError on Unix, but not on Windows + if not sys.platform.startswith('win'): + self.fail("Opening mmap with size+1 should raise ValueError.") + m.close() + finally: + f.close() if sys.platform.startswith('win'): - self.fail("Opening mmap with size+1 should work on Windows.") - else: - # we expect a ValueError on Unix, but not on Windows - if not sys.platform.startswith('win'): - self.fail("Opening mmap with size+1 should raise ValueError.") - m.close() - f.close() - if sys.platform.startswith('win'): - # Repair damage from the resizing test. - f = open(TESTFN, 'r+b') - f.truncate(mapsize) - f.close() + # Repair damage from the resizing test. + f = open(TESTFN, 'r+b') + f.truncate(mapsize) + f.close() # Opening mmap with access=ACCESS_WRITE f = open(TESTFN, "r+b") diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py --- a/lib_pypy/_curses.py +++ b/lib_pypy/_curses.py @@ -286,6 +286,13 @@ lib = ffi.verify(""" +#ifdef __APPLE__ +/* the following define is necessary for OS X 10.6+; without it, the + Apple-supplied ncurses.h sets NCURSES_OPAQUE to 1, and then Python + can't get at the WINDOW flags field. */ +#define NCURSES_OPAQUE 0 +#endif + #include #include #include diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -65,9 +65,9 @@ # built documents. # # The short X.Y version. -version = '2.3' +version = '2.4' # The full version, including alpha/beta/rc tags. -release = '2.3.0' +release = '2.4.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pypy/doc/getting-started-python.rst b/pypy/doc/getting-started-python.rst --- a/pypy/doc/getting-started-python.rst +++ b/pypy/doc/getting-started-python.rst @@ -111,6 +111,10 @@ of your choice. Typical example: ``--opt=2`` gives a good (but of course slower) Python interpreter without the JIT. + Consider using PyPy instead of CPython in the above command line, + as it is much faster. (Note that ``rpython`` is a Python 2 program, + not Python 3; you need to run either PyPy 2 or CPython 2.) + .. _`optimization level`: config/opt.html If everything works correctly this will create an executable diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -6,6 +6,7 @@ .. toctree:: + release-2.4.0.rst release-2.3.1.rst release-2.3.0.rst release-2.2.1.rst diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst --- a/pypy/doc/index.rst +++ b/pypy/doc/index.rst @@ -40,7 +40,7 @@ * `FAQ`_: some frequently asked questions. -* `Release 2.3.1`_: the latest official release +* `Release 2.4.0`_: the latest official release * `PyPy Blog`_: news and status info about PyPy @@ -110,7 +110,7 @@ .. _`Getting Started`: getting-started.html .. _`Papers`: extradoc.html .. _`Videos`: video-index.html -.. _`Release 2.3.1`: http://pypy.org/download.html +.. _`Release 2.4.0`: http://pypy.org/download.html .. _`speed.pypy.org`: http://speed.pypy.org .. _`RPython toolchain`: translation.html .. _`potential project ideas`: project-ideas.html diff --git a/pypy/doc/release-2.4.0.rst b/pypy/doc/release-2.4.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-2.4.0.rst @@ -0,0 +1,122 @@ +================================================= +PyPy 2.4 - Snow White +================================================= + +We're pleased to announce PyPy 2.4, which contains significant performance +enhancements and bug fixes. + +You can download the PyPy 2.4.0 release here: + + http://pypy.org/download.html + +We would like to thank our donors for the continued support of the PyPy +project, and for those who donate to our three sub-projects. +We've shown quite a bit of progress, but we're slowly running out of funds. +Please consider donating more, or even better convince your employer to donate, +so we can finish those projects! We would like to also point out that in +September, `the Python Software Foundation`_ will `match funds`_ for +any donations up to $10k! The three sub-projects are: + +* `Py3k`_ (supporting Python 3.x): We have released a Python 3.2.5 compatible version + we call PyPy3 2.3.1, and are working toward a Python 3.3 compatible version + +* `STM`_ (software transactional memory): We have released a first working version, + and continue to try out new promising paths of achieving a fast multithreaded Python + +* `NumPy`_ which requires installation of our fork of upstream numpy, + available `on bitbucket`_ + +.. _`Py3k`: http://pypy.org/py3donate.html +.. _`STM`: http://pypy.org/tmdonate2.html +.. _`NumPy`: http://pypy.org/numpydonate.html +.. _`on bitbucket`: https://www.bitbucket.org/pypy/numpy +.. _`the Python Software Foundation`: https://www.python.org/psf/ +.. _`match funds`: http://morepypy.blogspot.com/2014/09/python-software-foundation-matching.html + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7. It's fast (`pypy 2.4 and cpython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +This release supports **x86** machines on most common operating systems +(Linux 32/64, Mac OS X 64, Windows, and OpenBSD), +as well as newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux. + +While we support 32 bit python on Windows, work on the native Windows 64 +bit python is still stalling, we would welcome a volunteer +to `handle that`_. + +.. _`pypy 2.4 and cpython 2.7.x`: http://speed.pypy.org +.. _`handle that`: http://doc.pypy.org/en/latest/windows.html#what-is-missing-for-a-full-64-bit-translation + +Highlights +========== + +Benchmarks improved after internal enhancements in string and +bytearray handling, and a major rewrite of the GIL handling. This means +that external calls are now a lot faster, especially the CFFI ones. It also +means better performance in a lot of corner cases with handling strings or +bytearrays. The main bugfix is handling of many socket objects in your +program which in the long run used to "leak" memory. + +PyPy now uses Python 2.7.8 standard library. + +We fixed a memory leak in IO in the sandbox_ code + +We welcomed more than 12 new contributors, and conducted two Google +Summer of Code projects, as well as other student projects not +directly related to Summer of Code. + + +Issues reported with our previous release were fixed after reports from users on +our new issue tracker at https://bitbucket.org/pypy/pypy/issues or on IRC at +#pypy. Here is a summary of the user-facing changes; +for more information see `whats-new`_: + +* Reduced internal copying of bytearray operations + +* Tweak the internal structure of StringBuilder to speed up large string + handling, which becomes advantageous on large programs at the cost of slightly + slower small *benchmark* type programs. + +* Boost performance of thread-local variables in both unjitted and jitted code, + this mostly affects errno handling on linux, which makes external calls + faster. + +* Move to a mixed polling and mutex GIL model that make mutlithreaded jitted + code run *much* faster + +* Optimize errno handling in linux (x86 and x86-64 only) + +* Remove ctypes pythonapi and ctypes.PyDLL, which never worked on PyPy + +* Fix performance regression on ufunc(, ) in numpy + +* Classes in the ast module are now distinct from structures used by + the compiler, which simplifies and speeds up translation of our + source code to the PyPy binary interpreter + +* Upgrade stdlib from 2.7.5 to 2.7.8 + +* Win32 now links statically to zlib, expat, bzip, and openssl-1.0.1i. + No more missing DLLs + +* Many issues were resolved_ since the 2.3.1 release on June 8 + +.. _`whats-new`: http://doc.pypy.org/en/latest/whatsnew-2.3.1.html +.. _resolved: https://bitbucket.org/pypy/pypy/issues?status=resolved +.. _sandbox: http://doc.pypy.org/en/latest/sandbox.html + +We have further improvements on the way: rpython file handling, +numpy linalg compatibility, as well +as improved GC and many smaller improvements. + +Please try it out and let us know what you think. We especially welcome +success stories, we know you are using PyPy, please tell us about it! + +Cheers + +The PyPy Team + diff --git a/pypy/doc/release-2.4.rst b/pypy/doc/release-2.4.rst deleted file mode 100644 --- a/pypy/doc/release-2.4.rst +++ /dev/null @@ -1,107 +0,0 @@ -================================================= -PyPy 2.4 - ???????? -================================================= - -We're pleased to announce PyPy 2.4, a significant milestone on it's own right -and the proud parent of our recent PyPy3 and STM releases. - -This release contains several improvements and bugfixes. - -You can download the PyPy 2.4 release here: - - http://pypy.org/download.html - -We would like to thank our donors for the continued support of the PyPy -project, and for those who donate to our three sub-projects. -We've shown quite a bit of progress -but we're slowly running out of funds. -Please consider donating more, or even better convince your employer to donate, -so we can finish those projects! The three sub-projects are: - -* `Py3k`_ (supporting Python 3.x): We have released a Python 3.2.5 compatable version - we call PyPy3 2.3.1, and are working toward a Python 3.3 compatable version - -* `STM`_ (software transactional memory): We have release a first working version, and -continue to try out new promising paths of acheiving a fast multithreaded python - -* `NumPy`_ which requires installation of our fork of upstream numpy, available `on bitbucket`_ - -.. _`Py3k`: http://pypy.org/py3donate.html -.. _`STM`: http://pypy.org/tmdonate2.html -.. _`NumPy`: http://pypy.org/numpydonate.html -.. _`on bitbucket`: https://www.bitbucket.org/pypy/numpy - -What is PyPy? -============= - -PyPy is a very compliant Python interpreter, almost a drop-in replacement for -CPython 2.7. It's fast (`pypy 2.3 and cpython 2.7.x`_ performance comparison; -note that cpython's speed has not changed since 2.7.2) -due to its integrated tracing JIT compiler. - -This release supports x86 machines running Linux 32/64, Mac OS X 64, Windows, -and OpenBSD, -as well as newer ARM hardware (ARMv6 or ARMv7, with VFPv3) running Linux. - -While we support 32 bit python on Windows, work on the native Windows 64 -bit python is still stalling, we would welcome a volunteer -to `handle that`_. - -.. _`pypy 2.3 and cpython 2.7.x`: http://speed.pypy.org -.. _`handle that`: http://doc.pypy.org/en/latest/windows.html#what-is-missing-for-a-full-64-bit-translation - -Highlights -========== - -Benchmarks improved after internal improvements in string and bytearray handling, -and a major rewrite of the GIL handling. Many of these improvements are offshoots -of the STM work. - -We merged in Python's 2.7.8 stdlib in a record time of one week, proving the -maturity of our underlying RPython code base and PyPy interpreter. - -We welcomed more than 12 new contributors, and conducted two Google Summer of Code -projects XXX details? - -Issues reported with our previous release were fixed after reports from users on -our new issue tracker at https://bitbucket.org/pypy/pypy/issues or on IRC at -#pypy. Here is a summary of the user-facing changes; -for more information see `whats-new`_: - -* Reduced internal copying of bytearray operations - -* Tweak the internal structure of StringBuilder to speed up large string -handling, which becomes advantageous on large programs at the cost of slightly -slower small *benchmark* type programs. - -* Boost performance of thread-local variables in both unjitted and jitted code - -* Move to a mixed polling and mutex GIL model that make mutli-threaded jitted - code run *much* faster - -* Optimize errno handling in linux - -* Remove ctypes pythonapi and ctypes.PyDLL, which never worked on PyPy - -* Fix performance regression on ufunc(, ) in numpy - -* Classes in the ast module are now distinct from structures used by the compiler, - which simplifies and speeds up translation of our source code to the PyPy binary - interpreter - -* Upgrade stdlib from 2.7.5 to 2.7.8 - -* - -* Many issues were resolved_ since the 2.3.1 release on June 8 - -.. _`whats-new`: http://doc.pypy.org/en/latest/whatsnew-2.3.1.html -.. _resolved: https://bitbucket.org/pypy/pypy/issues?status=resolved - -Please try it out and let us know what you think. We especially welcome -success stories, we know you are using PyPy, please tell us about it! - -Cheers - -The PyPy Team - diff --git a/pypy/doc/whatsnew-2.4.0.rst b/pypy/doc/whatsnew-2.4.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/whatsnew-2.4.0.rst @@ -0,0 +1,66 @@ +======================= +What's new in PyPy 2.4+ +======================= + +.. this is a revision shortly after release-2.3.x +.. startrev: ca9b7cf02cf4 + +.. branch: fix-bytearray-complexity +Bytearray operations no longer copy the bytearray unnecessarily + +Added support for ``__getitem__``, ``__setitem__``, ``__getslice__``, +``__setslice__``, and ``__len__`` to RPython + +.. branch: stringbuilder2-perf +Give the StringBuilder a more flexible internal structure, with a +chained list of strings instead of just one string. This make it +more efficient when building large strings, e.g. with cStringIO(). + +Also, use systematically jit.conditional_call() instead of regular +branches. This lets the JIT make more linear code, at the cost of +forcing a bit more data (to be passed as arguments to +conditional_calls). I would expect the net result to be a slight +slow-down on some simple benchmarks and a speed-up on bigger +programs. + +.. branch: ec-threadlocal +Change the executioncontext's lookup to be done by reading a thread- +local variable (which is implemented in C using '__thread' if +possible, and pthread_getspecific() otherwise). On Linux x86 and +x86-64, the JIT backend has a special optimization that lets it emit +directly a single MOV from a %gs- or %fs-based address. It seems +actually to give a good boost in performance. + +.. branch: fast-gil +A faster way to handle the GIL, particularly in JIT code. The GIL is +now a composite of two concepts: a global number (it's just set from +1 to 0 and back around CALL_RELEASE_GIL), and a real mutex. If there +are threads waiting to acquire the GIL, one of them is actively +checking the global number every 0.1 ms to 1 ms. Overall, JIT loops +full of external function calls now run a bit faster (if no thread was +started yet), or a *lot* faster (if threads were started already). + +.. branch: jit-get-errno +Optimize the errno handling in the JIT, notably around external +function calls. Linux-only. + +.. branch: disable_pythonapi +Remove non-functioning ctypes.pyhonapi and ctypes.PyDLL, document this +incompatibility with cpython. Recast sys.dllhandle to an int. + +.. branch: scalar-operations +Fix performance regression on ufunc(, ) in numpy. + +.. branch: pytest-25 +Update our copies of py.test and pylib to versions 2.5.2 and 1.4.20, +respectively. + +.. branch: split-ast-classes +Classes in the ast module are now distinct from structures used by the compiler. + +.. branch: stdlib-2.7.8 +Upgrades from 2.7.6 to 2.7.8 + +.. branch: cpybug-seq-radd-rmul +Fix issue #1861 - cpython compatability madness + diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -1,62 +1,8 @@ + ======================= -What's new in PyPy 2.4+ +What's new in PyPy 2.5+ ======================= -.. this is a revision shortly after release-2.3.x -.. startrev: ca9b7cf02cf4 +.. this is a revision shortly after release-2.4.x +.. startrev: 7026746cbb1b -.. branch: fix-bytearray-complexity -Bytearray operations no longer copy the bytearray unnecessarily - -Added support for ``__getitem__``, ``__setitem__``, ``__getslice__``, -``__setslice__``, and ``__len__`` to RPython - -.. branch: stringbuilder2-perf -Give the StringBuilder a more flexible internal structure, with a -chained list of strings instead of just one string. This make it -more efficient when building large strings, e.g. with cStringIO(). - -Also, use systematically jit.conditional_call() instead of regular -branches. This lets the JIT make more linear code, at the cost of -forcing a bit more data (to be passed as arguments to -conditional_calls). I would expect the net result to be a slight -slow-down on some simple benchmarks and a speed-up on bigger -programs. - -.. branch: ec-threadlocal -Change the executioncontext's lookup to be done by reading a thread- -local variable (which is implemented in C using '__thread' if -possible, and pthread_getspecific() otherwise). On Linux x86 and -x86-64, the JIT backend has a special optimization that lets it emit -directly a single MOV from a %gs- or %fs-based address. It seems -actually to give a good boost in performance. - -.. branch: fast-gil -A faster way to handle the GIL, particularly in JIT code. The GIL is -now a composite of two concepts: a global number (it's just set from -1 to 0 and back around CALL_RELEASE_GIL), and a real mutex. If there -are threads waiting to acquire the GIL, one of them is actively -checking the global number every 0.1 ms to 1 ms. Overall, JIT loops -full of external function calls now run a bit faster (if no thread was -started yet), or a *lot* faster (if threads were started already). - -.. branch: jit-get-errno -Optimize the errno handling in the JIT, notably around external -function calls. Linux-only. - -.. branch: disable_pythonapi -Remove non-functioning ctypes.pyhonapi and ctypes.PyDLL, document this -incompatibility with cpython. Recast sys.dllhandle to an int. - -.. branch: scalar-operations -Fix performance regression on ufunc(, ) in numpy. - -.. branch: pytest-25 -Update our copies of py.test and pylib to versions 2.5.2 and 1.4.20, -respectively. - -.. branch: split-ast-classes -Classes in the ast module are now distinct from structures used by the compiler. - -.. branch: stdlib-2.7.8 -Upgrades from 2.7.6 to 2.7.8 diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst --- a/pypy/doc/windows.rst +++ b/pypy/doc/windows.rst @@ -85,10 +85,13 @@ Abridged method (for -Ojit builds using Visual Studio 2008) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Download the versions of all the external packages -from +Download the versions of all the external packages from +https://bitbucket.org/pypy/pypy/downloads/local_2.4.zip +(for 2.4 release and later) or https://bitbucket.org/pypy/pypy/downloads/local.zip -Then expand it into the base directory (base_dir) and modify your environment to reflect this:: +(for pre-2.4 versions) +Then expand it into the base directory (base_dir) and modify your environment +to reflect this:: set PATH=\bin;\tcltk\bin;%PATH% set INCLUDE=\include;\tcltk\include;%INCLUDE% diff --git a/pypy/interpreter/module.py b/pypy/interpreter/module.py --- a/pypy/interpreter/module.py +++ b/pypy/interpreter/module.py @@ -29,6 +29,17 @@ space.w_None) self.startup_called = False + def _cleanup_(self): + """Called by the annotator on prebuilt Module instances. + We don't have many such modules, but for the ones that + show up, remove their __file__ rather than translate it + statically inside the executable.""" + try: + space = self.space + space.delitem(self.w_dict, space.wrap('__file__')) + except OperationError: + pass + def install(self): """NOT_RPYTHON: installs this module into space.builtin_modules""" w_mod = self.space.wrap(self) diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py --- a/pypy/interpreter/pycode.py +++ b/pypy/interpreter/pycode.py @@ -38,18 +38,15 @@ def cpython_code_signature(code): "([list-of-arg-names], vararg-name-or-None, kwarg-name-or-None)." argcount = code.co_argcount + varnames = code.co_varnames assert argcount >= 0 # annotator hint - argnames = list(code.co_varnames[:argcount]) + argnames = list(varnames[:argcount]) if code.co_flags & CO_VARARGS: - varargname = code.co_varnames[argcount] + varargname = varnames[argcount] argcount += 1 else: varargname = None - if code.co_flags & CO_VARKEYWORDS: - kwargname = code.co_varnames[argcount] - argcount += 1 - else: - kwargname = None + kwargname = varnames[argcount] if code.co_flags & CO_VARKEYWORDS else None return Signature(argnames, varargname, kwargname) diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py --- a/pypy/interpreter/pyparser/parsestring.py +++ b/pypy/interpreter/pyparser/parsestring.py @@ -83,12 +83,6 @@ v = PyString_DecodeEscape(space, substr, 'strict', enc) return space.wrap(v) -def hexbyte(val): - result = "%x" % val - if len(result) == 1: - result = "0" + result - return result - def decode_unicode_utf8(space, s, ps, q): # ****The Python 2.7 version, producing UTF-32 escapes**** # String is utf8-encoded, but 'unicode_escape' expects @@ -108,15 +102,14 @@ # instead. lis.append("u005c") if ord(s[ps]) & 0x80: # XXX inefficient - w, ps = decode_utf8(space, s, ps, end, "utf-32-be") - rn = len(w) - assert rn % 4 == 0 - for i in range(0, rn, 4): - lis.append('\\U') - lis.append(hexbyte(ord(w[i]))) - lis.append(hexbyte(ord(w[i+1]))) - lis.append(hexbyte(ord(w[i+2]))) - lis.append(hexbyte(ord(w[i+3]))) + w, ps = decode_utf8(space, s, ps, end) + for c in w: + # The equivalent of %08x, which is not supported by RPython. + # 7 zeroes are enough for the unicode range, and the + # result still fits in 32-bit. + hexa = hex(ord(c) + 0x10000000) + lis.append('\\U0') + lis.append(hexa[3:]) # Skip 0x and the leading 1 else: lis.append(s[ps]) ps += 1 @@ -136,7 +129,7 @@ # note that the C code has a label here. # the logic is the same. if recode_encoding and ord(s[ps]) & 0x80: - w, ps = decode_utf8(space, s, ps, end, recode_encoding) + w, ps = decode_utf8_recode(space, s, ps, end, recode_encoding) # Append bytes to output buffer. builder.append(w) else: @@ -222,14 +215,18 @@ ch >= 'A' and ch <= 'F') -def decode_utf8(space, s, ps, end, encoding): +def decode_utf8(space, s, ps, end): assert ps >= 0 pt = ps # while (s < end && *s != '\\') s++; */ /* inefficient for u".." while ps < end and ord(s[ps]) & 0x80: ps += 1 - w_u = space.wrap(unicodehelper.decode_utf8(space, s[pt:ps])) - w_v = unicodehelper.encode(space, w_u, encoding) + u = unicodehelper.decode_utf8(space, s[pt:ps]) + return u, ps + +def decode_utf8_recode(space, s, ps, end, recode_encoding): + u, ps = decode_utf8(space, s, ps, end) + w_v = unicodehelper.encode(space, space.wrap(u), recode_encoding) v = space.str_w(w_v) return v, ps diff --git a/pypy/interpreter/pyparser/test/test_parsestring.py b/pypy/interpreter/pyparser/test/test_parsestring.py --- a/pypy/interpreter/pyparser/test/test_parsestring.py +++ b/pypy/interpreter/pyparser/test/test_parsestring.py @@ -73,11 +73,11 @@ def test_simple_enc_roundtrip(self): space = self.space - s = "'\x81'" + s = "'\x81\\t'" s = s.decode("koi8-u").encode("utf8") w_ret = parsestring.parsestr(self.space, 'koi8-u', s) ret = space.unwrap(w_ret) - assert ret == eval("# -*- coding: koi8-u -*-\n'\x81'") + assert ret == eval("# -*- coding: koi8-u -*-\n'\x81\\t'") def test_multiline_unicode_strings_with_backslash(self): space = self.space diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py --- a/pypy/interpreter/test/test_app_main.py +++ b/pypy/interpreter/test/test_app_main.py @@ -945,7 +945,7 @@ prefix = udir.join('pathtest').ensure(dir=1) fake_exe = 'bin/pypy-c' if sys.platform == 'win32': - fake_exe += '.exe' + fake_exe = 'pypy-c.exe' fake_exe = prefix.join(fake_exe).ensure(file=1) expected_path = [str(prefix.join(subdir).ensure(dir=1)) for subdir in ('lib_pypy', @@ -985,6 +985,13 @@ assert sys.path == old_sys_path + [self.goal_dir] app_main.setup_bootstrap_path(self.fake_exe) + if not sys.platform == 'win32': + # an existing file is always 'executable' on windows + assert sys.executable == '' # not executable! + assert sys.path == old_sys_path + [self.goal_dir] + + os.chmod(self.fake_exe, 0755) + app_main.setup_bootstrap_path(self.fake_exe) assert sys.executable == self.fake_exe assert self.goal_dir not in sys.path diff --git a/pypy/interpreter/test/test_module.py b/pypy/interpreter/test/test_module.py --- a/pypy/interpreter/test/test_module.py +++ b/pypy/interpreter/test/test_module.py @@ -1,4 +1,5 @@ - +import py +from pypy.interpreter.error import OperationError from pypy.interpreter.module import Module class TestModule: @@ -17,6 +18,18 @@ space.raises_w(space.w_AttributeError, space.delattr, w_m, w('x')) + def test___file__(self, space): + w = space.wrap + m = Module(space, space.wrap('m')) + py.test.raises(OperationError, space.getattr, w(m), w('__file__')) + m._cleanup_() + py.test.raises(OperationError, space.getattr, w(m), w('__file__')) + space.setattr(w(m), w('__file__'), w('m.py')) + space.getattr(w(m), w('__file__')) # does not raise + m._cleanup_() + py.test.raises(OperationError, space.getattr, w(m), w('__file__')) + + class AppTest_ModuleObject: def test_attr(self): m = __import__('__builtin__') diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py --- a/pypy/interpreter/unicodehelper.py +++ b/pypy/interpreter/unicodehelper.py @@ -5,6 +5,7 @@ @specialize.memo() def decode_error_handler(space): + # Fast version of the "strict" errors handler. def raise_unicode_exception_decode(errors, encoding, msg, s, startingpos, endingpos): raise OperationError(space.w_UnicodeDecodeError, @@ -17,6 +18,7 @@ @specialize.memo() def encode_error_handler(space): + # Fast version of the "strict" errors handler. def raise_unicode_exception_encode(errors, encoding, msg, u, startingpos, endingpos): raise OperationError(space.w_UnicodeEncodeError, diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py --- a/pypy/module/_cffi_backend/ctypefunc.py +++ b/pypy/module/_cffi_backend/ctypefunc.py @@ -8,6 +8,7 @@ from rpython.rlib.jit_libffi import (CIF_DESCRIPTION, CIF_DESCRIPTION_P, FFI_TYPE, FFI_TYPE_P, FFI_TYPE_PP, SIZE_OF_FFI_ARG) from rpython.rlib.objectmodel import we_are_translated, instantiate +from rpython.rlib.objectmodel import keepalive_until_here from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from pypy.interpreter.error import OperationError, oefmt @@ -160,6 +161,7 @@ raw_cdata = rffi.cast(rffi.CCHARPP, data)[0] lltype.free(raw_cdata, flavor='raw') lltype.free(buffer, flavor='raw') + keepalive_until_here(args_w) return w_res def get_mustfree_flag(data): diff --git a/pypy/module/_io/interp_stringio.py b/pypy/module/_io/interp_stringio.py --- a/pypy/module/_io/interp_stringio.py +++ b/pypy/module/_io/interp_stringio.py @@ -86,7 +86,7 @@ initval = space.unicode_w(w_initval) size = len(initval) self.resize_buffer(size) - self.buf = [c for c in initval] + self.buf = list(initval) pos = space.getindex_w(w_pos, space.w_TypeError) if pos < 0: raise OperationError(space.w_ValueError, diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py --- a/pypy/module/_ssl/interp_ssl.py +++ b/pypy/module/_ssl/interp_ssl.py @@ -759,17 +759,25 @@ # socket's timeout is in seconds, poll's timeout in ms timeout = int(sock_timeout * 1000 + 0.5) - ready = rpoll.poll(fddict, timeout) + try: + ready = rpoll.poll(fddict, timeout) + except rpoll.PollError, e: + message = e.get_msg() + raise ssl_error(space, message, e.errno) else: if MAX_FD_SIZE is not None and sock_fd >= MAX_FD_SIZE: return SOCKET_TOO_LARGE_FOR_SELECT - if writing: - r, w, e = rpoll.select([], [sock_fd], [], sock_timeout) - ready = w - else: - r, w, e = rpoll.select([sock_fd], [], [], sock_timeout) - ready = r + try: + if writing: + r, w, e = rpoll.select([], [sock_fd], [], sock_timeout) + ready = w + else: + r, w, e = rpoll.select([sock_fd], [], [], sock_timeout) + ready = r + except rpoll.SelectError as e: + message = e.get_msg() + raise ssl_error(space, message, e.errno) if ready: return SOCKET_OPERATION_OK else: diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h --- a/pypy/module/cpyext/include/patchlevel.h +++ b/pypy/module/cpyext/include/patchlevel.h @@ -29,7 +29,7 @@ #define PY_VERSION "2.7.8" /* PyPy version as a string */ -#define PYPY_VERSION "2.4.0-alpha0" +#define PYPY_VERSION "2.5.0-alpha0" /* Subversion Revision number of this file (not of the repository). * Empty since Mercurial migration. */ diff --git a/pypy/module/operator/__init__.py b/pypy/module/operator/__init__.py --- a/pypy/module/operator/__init__.py +++ b/pypy/module/operator/__init__.py @@ -39,7 +39,7 @@ 'irshift', 'isub', 'itruediv', 'ixor', '_length_hint'] interpleveldefs = { - '_compare_digest': 'interp_operator.compare_digest', + '_compare_digest': 'tscmp.compare_digest', } for name in interp_names: diff --git a/pypy/module/operator/app_operator.py b/pypy/module/operator/app_operator.py --- a/pypy/module/operator/app_operator.py +++ b/pypy/module/operator/app_operator.py @@ -4,7 +4,7 @@ This module exports a set of operators as functions. E.g. operator.add(x,y) is equivalent to x+y. ''' -from __pypy__ import builtinify + import types @@ -27,7 +27,7 @@ 'getslice(a, b, c) -- Same as a[b:c].' if not isinstance(start, int) or not isinstance(end, int): raise TypeError("an integer is expected") - return a[start:end] + return a[start:end] __getslice__ = getslice def indexOf(a, b): @@ -37,7 +37,7 @@ if x == b: return index index += 1 - raise ValueError, 'sequence.index(x): x not in sequence' + raise ValueError('sequence.index(x): x not in sequence') def isMappingType(obj,): 'isMappingType(a) -- Return True if a has a mapping type, False otherwise.' @@ -58,9 +58,9 @@ def repeat(obj, num): 'repeat(a, b) -- Return a * b, where a is a sequence, and b is an integer.' if not isinstance(num, (int, long)): - raise TypeError, 'an integer is required' + raise TypeError('an integer is required') if not isSequenceType(obj): - raise TypeError, "non-sequence object can't be repeated" + raise TypeError("non-sequence object can't be repeated") return obj * num @@ -68,59 +68,85 @@ def setslice(a, b, c, d): 'setslice(a, b, c, d) -- Same as a[b:c] = d.' - a[b:c] = d + a[b:c] = d __setslice__ = setslice +def _resolve_attr_chain(chain, obj, idx=0): + obj = getattr(obj, chain[idx]) + if idx + 1 == len(chain): + return obj + else: + return _resolve_attr_chain(chain, obj, idx + 1) + + +class _simple_attrgetter(object): + def __init__(self, attr): + self._attr = attr + + def __call__(self, obj): + return getattr(obj, self._attr) + + +class _single_attrgetter(object): + def __init__(self, attrs): + self._attrs = attrs + + def __call__(self, obj): + return _resolve_attr_chain(self._attrs, obj) + + +class _multi_attrgetter(object): + def __init__(self, attrs): + self._attrs = attrs + + def __call__(self, obj): + return tuple([ + _resolve_attr_chain(attrs, obj) + for attrs in self._attrs + ]) + + def attrgetter(attr, *attrs): + if ( + not isinstance(attr, basestring) or + not all(isinstance(a, basestring) for a in attrs) + ): + def _raise_typeerror(obj): + raise TypeError( + "argument must be a string, not %r" % type(attr).__name__ + ) + return _raise_typeerror if attrs: - getters = [single_attr_getter(a) for a in (attr,) + attrs] - def getter(obj): - return tuple([getter(obj) for getter in getters]) + return _multi_attrgetter([ + a.split(".") for a in [attr] + list(attrs) + ]) + elif "." not in attr: + return _simple_attrgetter(attr) else: - getter = single_attr_getter(attr) - return builtinify(getter) + return _single_attrgetter(attr.split(".")) -def single_attr_getter(attr): - if not isinstance(attr, str): - if not isinstance(attr, unicode): - def _raise_typeerror(obj): - raise TypeError("argument must be a string, not %r" % - (type(attr).__name__,)) - return _raise_typeerror - attr = attr.encode('ascii') - # - def make_getter(name, prevfn=None): - if prevfn is None: - def getter(obj): - return getattr(obj, name) + +class itemgetter(object): + def __init__(self, item, *items): + self._single = not bool(items) + if self._single: + self._idx = item else: - def getter(obj): - return getattr(prevfn(obj), name) - return getter - # - last = 0 - getter = None - while True: - dot = attr.find(".", last) - if dot < 0: break - getter = make_getter(attr[last:dot], getter) - last = dot + 1 - return make_getter(attr[last:], getter) + self._idx = [item] + list(items) + def __call__(self, obj): + if self._single: + return obj[self._idx] + else: + return tuple([obj[i] for i in self._idx]) -def itemgetter(item, *items): - if items: - list_of_indices = [item] + list(items) - def getter(obj): - return tuple([obj[i] for i in list_of_indices]) - else: - def getter(obj): - return obj[item] - return builtinify(getter) +class methodcaller(object): + def __init__(self, method_name, *args, **kwargs): + self._method_name = method_name + self._args = args + self._kwargs = kwargs -def methodcaller(method_name, *args, **kwargs): - def call(obj): - return getattr(obj, method_name)(*args, **kwargs) - return builtinify(call) + def __call__(self, obj): + return getattr(obj, self._method_name)(*self._args, **self._kwargs) diff --git a/pypy/module/operator/interp_operator.py b/pypy/module/operator/interp_operator.py --- a/pypy/module/operator/interp_operator.py +++ b/pypy/module/operator/interp_operator.py @@ -1,6 +1,4 @@ -from rpython.rlib.objectmodel import specialize - -from pypy.interpreter.error import OperationError, oefmt +from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import unwrap_spec @@ -249,33 +247,3 @@ @unwrap_spec(default=int) def _length_hint(space, w_iterable, default): return space.wrap(space.length_hint(w_iterable, default)) - -def compare_digest(space, w_a, w_b): - if ( - space.isinstance_w(w_a, space.w_unicode) and - space.isinstance_w(w_b, space.w_unicode) - ): - return space.wrap(tscmp(space.unicode_w(w_a), space.unicode_w(w_b))) - if ( - space.isinstance_w(w_a, space.w_unicode) or - space.isinstance_w(w_b, space.w_unicode) - ): - raise oefmt( - space.w_TypeError, - "unsupported operand types(s) or combination of types: '%N' and '%N'", - w_a, - w_b, - ) - else: - return space.wrap(tscmp(space.bufferstr_w(w_a), space.bufferstr_w(w_b))) - - - at specialize.argtype(0, 1) -def tscmp(a, b): - len_a = len(a) - len_b = len(b) - length = min(len(a), len(b)) - res = len_a ^ len_b - for i in xrange(length): - res |= ord(a[i]) ^ ord(b[i]) - return res == 0 diff --git a/pypy/module/operator/test/test_operator.py b/pypy/module/operator/test/test_operator.py --- a/pypy/module/operator/test/test_operator.py +++ b/pypy/module/operator/test/test_operator.py @@ -334,3 +334,9 @@ assert operator._compare_digest(a, b) a, b = mybytes(b"foobar"), mybytes(b"foobaz") assert not operator._compare_digest(a, b) + + def test_compare_digest_unicode(self): + import operator + assert operator._compare_digest(u'asd', u'asd') + assert not operator._compare_digest(u'asd', u'qwe') + raises(TypeError, operator._compare_digest, u'asd', b'qwe') diff --git a/pypy/module/operator/test/test_tscmp.py b/pypy/module/operator/test/test_tscmp.py new file mode 100644 --- /dev/null +++ b/pypy/module/operator/test/test_tscmp.py @@ -0,0 +1,28 @@ +from pypy.module.operator.tscmp import pypy_tscmp, pypy_tscmp_wide + +class TestTimingSafeCompare: + tostr = str + tscmp = staticmethod(pypy_tscmp) + + def test_tscmp_neq(self): + assert not self.tscmp(self.tostr('asd'), self.tostr('qwe'), 3, 3) + + def test_tscmp_eq(self): + assert self.tscmp(self.tostr('asd'), self.tostr('asd'), 3, 3) + + def test_tscmp_len(self): + assert self.tscmp(self.tostr('asdp'), self.tostr('asdq'), 3, 3) + + def test_tscmp_nlen(self): + assert not self.tscmp(self.tostr('asd'), self.tostr('asd'), 2, 3) + + +class TestTimingSafeCompareWide(TestTimingSafeCompare): + tostr = unicode + tscmp = staticmethod(pypy_tscmp_wide) + + def test_tscmp_wide_nonascii(self): + a, b = u"\ud808\udf45", u"\ud808\udf45" + assert self.tscmp(a, b, len(a), len(b)) + a, b = u"\ud808\udf45", u"\ud808\udf45 " + assert not self.tscmp(a, b, len(a), len(b)) diff --git a/pypy/module/operator/tscmp.c b/pypy/module/operator/tscmp.c new file mode 100644 --- /dev/null +++ b/pypy/module/operator/tscmp.c @@ -0,0 +1,80 @@ +/* Derived from CPython 3.3.5's operator.c::_tscmp + */ + +#include +#include +#include "tscmp.h" + +int +pypy_tscmp(const char *a, const char *b, long len_a, long len_b) +{ + /* The volatile type declarations make sure that the compiler has no + * chance to optimize and fold the code in any way that may change + * the timing. + */ + volatile long length; + volatile const char *left; + volatile const char *right; + long i; + char result; + + /* loop count depends on length of b */ + length = len_b; + left = NULL; + right = b; + + /* don't use else here to keep the amount of CPU instructions constant, + * volatile forces re-evaluation + * */ + if (len_a == length) { + left = *((volatile const char**)&a); + result = 0; + } + if (len_a != length) { + left = b; + result = 1; + } + + for (i=0; i < length; i++) { + result |= *left++ ^ *right++; + } + + return (result == 0); +} + +int +pypy_tscmp_wide(const wchar_t *a, const wchar_t *b, long len_a, long len_b) +{ + /* The volatile type declarations make sure that the compiler has no + * chance to optimize and fold the code in any way that may change + * the timing. + */ + volatile long length; + volatile const wchar_t *left; + volatile const wchar_t *right; + long i; + wchar_t result; + + /* loop count depends on length of b */ + length = len_b; + left = NULL; + right = b; + + /* don't use else here to keep the amount of CPU instructions constant, + * volatile forces re-evaluation + * */ + if (len_a == length) { + left = *((volatile const wchar_t**)&a); + result = 0; + } + if (len_a != length) { + left = b; + result = 1; + } + + for (i=0; i < length; i++) { + result |= *left++ ^ *right++; + } + + return (result == 0); +} diff --git a/pypy/module/operator/tscmp.h b/pypy/module/operator/tscmp.h new file mode 100644 --- /dev/null +++ b/pypy/module/operator/tscmp.h @@ -0,0 +1,2 @@ +int pypy_tscmp(const char *, const char *, long, long); +int pypy_tscmp_wide(const wchar_t *, const wchar_t *, long, long); diff --git a/pypy/module/operator/tscmp.py b/pypy/module/operator/tscmp.py new file mode 100644 --- /dev/null +++ b/pypy/module/operator/tscmp.py @@ -0,0 +1,73 @@ +""" +Provides _compare_digest method, which is a safe comparing to prevent timing +attacks for the hmac module. +""" +import py + +from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.translator.tool.cbuild import ExternalCompilationInfo + +from pypy.interpreter.error import oefmt + +cwd = py.path.local(__file__).dirpath() +eci = ExternalCompilationInfo( + includes=[cwd.join('tscmp.h')], + include_dirs=[str(cwd)], + separate_module_files=[cwd.join('tscmp.c')], + export_symbols=['pypy_tscmp', 'pypy_tscmp_wide']) + + +def llexternal(*args, **kwargs): + kwargs.setdefault('compilation_info', eci) + kwargs.setdefault('sandboxsafe', True) + return rffi.llexternal(*args, **kwargs) + + +pypy_tscmp = llexternal( + 'pypy_tscmp', + [rffi.CCHARP, rffi.CCHARP, rffi.LONG, rffi.LONG], + rffi.INT) +pypy_tscmp_wide = llexternal( + 'pypy_tscmp_wide', + [rffi.CWCHARP, rffi.CWCHARP, rffi.LONG, rffi.LONG], + rffi.INT) + + +def compare_digest(space, w_a, w_b): + """compare_digest(a, b) -> bool + + Return 'a == b'. This function uses an approach designed to prevent + timing analysis, making it appropriate for cryptography. a and b + must both be of the same type: either str (ASCII only), or any type + that supports the buffer protocol (e.g. bytes). + + Note: If a and b are of different lengths, or if an error occurs, a + timing attack could theoretically reveal information about the types + and lengths of a and b--but not their values. + """ + if (space.isinstance_w(w_a, space.w_unicode) and + space.isinstance_w(w_b, space.w_unicode)): + a = space.unicode_w(w_a) + b = space.unicode_w(w_b) + with rffi.scoped_nonmoving_unicodebuffer(a) as a_buf: + with rffi.scoped_nonmoving_unicodebuffer(b) as b_buf: + result = pypy_tscmp_wide(a_buf, b_buf, len(a), len(b)) + return space.wrap(rffi.cast(lltype.Bool, result)) + return compare_digest_buffer(space, w_a, w_b) + + +def compare_digest_buffer(space, w_a, w_b): + try: + a_buf = w_a.buffer_w(space, space.BUF_SIMPLE) + b_buf = w_b.buffer_w(space, space.BUF_SIMPLE) + except TypeError: + raise oefmt(space.w_TypeError, + "unsupported operand types(s) or combination of types: " + "'%T' and '%T'", w_a, w_b) + + a = a_buf.as_str() + b = b_buf.as_str() + with rffi.scoped_nonmovingbuffer(a) as a_buf: + with rffi.scoped_nonmovingbuffer(b) as b_buf: + result = pypy_tscmp(a_buf, b_buf, len(a), len(b)) + return space.wrap(rffi.cast(lltype.Bool, result)) diff --git a/pypy/module/pypyjit/test_pypy_c/test_cprofile.py b/pypy/module/pypyjit/test_pypy_c/test_cprofile.py --- a/pypy/module/pypyjit/test_pypy_c/test_cprofile.py +++ b/pypy/module/pypyjit/test_pypy_c/test_cprofile.py @@ -1,4 +1,4 @@ -import py, sys +import py, sys, re from pypy.module.pypyjit.test_pypy_c.test_00_model import BaseTestPyPyC class TestCProfile(BaseTestPyPyC): @@ -28,8 +28,18 @@ print loop.ops_by_id(method) # on 32-bit, there is f1=call(read_timestamp); ...; # f2=call(read_timestamp); f3=call(llong_sub,f1,f2) - # but all calls can be special-cased by the backend if supported - if sys.maxint != 2147483647: - assert ' call(' not in repr(loop.ops_by_id(method)) + # but all calls can be special-cased by the backend if + # supported. On 64-bit there is only the two calls to + # read_timestamp. + r = re.compile(r" call[(]ConstClass[(](.+?)[)]") + calls = r.findall(repr(loop.ops_by_id(method))) + if sys.maxint == 2147483647: + assert len(calls) == 6 + else: + assert len(calls) == 2 + for x in calls: + assert ('ll_read_timestamp' in x or 'llong_sub' in x + or 'llong_add' in x) + # assert ' call_may_force(' not in repr(loop.ops_by_id(method)) assert ' cond_call(' in repr(loop.ops_by_id(method)) diff --git a/pypy/module/sys/state.py b/pypy/module/sys/state.py --- a/pypy/module/sys/state.py +++ b/pypy/module/sys/state.py @@ -7,15 +7,15 @@ # ____________________________________________________________ # -class State: - def __init__(self, space): - self.space = space +class State: + def __init__(self, space): + self.space = space self.w_modules = space.newdict(module=True) - self.w_warnoptions = space.newlist([]) self.w_argv = space.newlist([]) - self.setinitialpath(space) + + self.setinitialpath(space) def setinitialpath(self, space): from pypy.module.sys.initpath import compute_stdlib_path @@ -25,10 +25,10 @@ path = compute_stdlib_path(self, srcdir) self.w_path = space.newlist([space.wrap(p) for p in path]) - def get(space): return space.fromcache(State) + class IOState: def __init__(self, space): from pypy.module._file.interp_file import W_File @@ -36,17 +36,17 @@ stdin = W_File(space) stdin.file_fdopen(0, "r", 1) - stdin.name = '' + stdin.w_name = space.wrap('') self.w_stdin = space.wrap(stdin) stdout = W_File(space) stdout.file_fdopen(1, "w", 1) - stdout.name = '' + stdout.w_name = space.wrap('') self.w_stdout = space.wrap(stdout) stderr = W_File(space) stderr.file_fdopen(2, "w", 0) - stderr.name = '' + stderr.w_name = space.wrap('') self.w_stderr = space.wrap(stderr) stdin._when_reading_first_flush(stdout) @@ -54,9 +54,9 @@ def getio(space): return space.fromcache(IOState) + def pypy_getudir(space): """NOT_RPYTHON (should be removed from interpleveldefs before translation)""" from rpython.tool.udir import udir return space.wrap(str(udir)) - diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py --- a/pypy/module/sys/test/test_sysmodule.py +++ b/pypy/module/sys/test/test_sysmodule.py @@ -91,6 +91,10 @@ assert isinstance(sys.__stderr__, file) assert isinstance(sys.__stdin__, file) + #assert sys.__stdin__.name == "" + #assert sys.__stdout__.name == "" + #assert sys.__stderr__.name == "" + if self.appdirect and not isinstance(sys.stdin, file): return diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -10,7 +10,7 @@ #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py CPYTHON_API_VERSION = 1013 #XXX # sync with include/modsupport.h -PYPY_VERSION = (2, 4, 0, "alpha", 0) #XXX # sync patchlevel.h +PYPY_VERSION = (2, 5, 0, "alpha", 0) #XXX # sync patchlevel.h if platform.name == 'msvc': COMPILER_INFO = 'MSC v.%d 32 bit' % (platform.version * 10 + 600) diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_zdistutils.py b/pypy/module/test_lib_pypy/cffi_tests/test_zdistutils.py --- a/pypy/module/test_lib_pypy/cffi_tests/test_zdistutils.py +++ b/pypy/module/test_lib_pypy/cffi_tests/test_zdistutils.py @@ -16,6 +16,10 @@ if distutils.ccompiler.get_default_compiler() == 'msvc': self.lib_m = 'msvcrt' + def teardown_class(self): + if udir.isdir(): + udir.remove() + def test_locate_engine_class(self): cls = _locate_engine_class(FFI(), self.generic) if self.generic: diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_zintegration.py b/pypy/module/test_lib_pypy/cffi_tests/test_zintegration.py --- a/pypy/module/test_lib_pypy/cffi_tests/test_zintegration.py +++ b/pypy/module/test_lib_pypy/cffi_tests/test_zintegration.py @@ -73,50 +73,55 @@ assert not os.path.exists(str(SNIPPET_DIR.join(dirname, 'lextab.py'))) assert not os.path.exists(str(SNIPPET_DIR.join(dirname, 'yacctab.py'))) -def test_infrastructure(): - run_setup_and_program('infrastructure', ''' - import snip_infrastructure - assert snip_infrastructure.func() == 42 - ''') +class TestZIntegration(object): + def teardown_class(self): + if udir.isdir(): + udir.remove() -def test_distutils_module(): - run_setup_and_program("distutils_module", ''' - import snip_basic_verify - p = snip_basic_verify.C.getpwuid(0) - assert snip_basic_verify.ffi.string(p.pw_name) == b"root" - ''') + def test_infrastructure(self): + run_setup_and_program('infrastructure', ''' + import snip_infrastructure + assert snip_infrastructure.func() == 42 + ''') -def test_distutils_package_1(): - run_setup_and_program("distutils_package_1", ''' - import snip_basic_verify1 - p = snip_basic_verify1.C.getpwuid(0) - assert snip_basic_verify1.ffi.string(p.pw_name) == b"root" - ''') + def test_distutils_module(self): + run_setup_and_program("distutils_module", ''' + import snip_basic_verify + p = snip_basic_verify.C.getpwuid(0) + assert snip_basic_verify.ffi.string(p.pw_name) == b"root" + ''') -def test_distutils_package_2(): - run_setup_and_program("distutils_package_2", ''' - import snip_basic_verify2 - p = snip_basic_verify2.C.getpwuid(0) - assert snip_basic_verify2.ffi.string(p.pw_name) == b"root" - ''') + def test_distutils_package_1(self): + run_setup_and_program("distutils_package_1", ''' + import snip_basic_verify1 + p = snip_basic_verify1.C.getpwuid(0) + assert snip_basic_verify1.ffi.string(p.pw_name) == b"root" + ''') -def test_setuptools_module(): - run_setup_and_program("setuptools_module", ''' - import snip_setuptools_verify - p = snip_setuptools_verify.C.getpwuid(0) - assert snip_setuptools_verify.ffi.string(p.pw_name) == b"root" - ''') + def test_distutils_package_2(self): + run_setup_and_program("distutils_package_2", ''' + import snip_basic_verify2 + p = snip_basic_verify2.C.getpwuid(0) + assert snip_basic_verify2.ffi.string(p.pw_name) == b"root" + ''') -def test_setuptools_package_1(): - run_setup_and_program("setuptools_package_1", ''' - import snip_setuptools_verify1 - p = snip_setuptools_verify1.C.getpwuid(0) - assert snip_setuptools_verify1.ffi.string(p.pw_name) == b"root" - ''') + def test_setuptools_module(self): + run_setup_and_program("setuptools_module", ''' + import snip_setuptools_verify + p = snip_setuptools_verify.C.getpwuid(0) + assert snip_setuptools_verify.ffi.string(p.pw_name) == b"root" + ''') -def test_setuptools_package_2(): - run_setup_and_program("setuptools_package_2", ''' - import snip_setuptools_verify2 - p = snip_setuptools_verify2.C.getpwuid(0) - assert snip_setuptools_verify2.ffi.string(p.pw_name) == b"root" - ''') + def test_setuptools_package_1(self): + run_setup_and_program("setuptools_package_1", ''' + import snip_setuptools_verify1 + p = snip_setuptools_verify1.C.getpwuid(0) + assert snip_setuptools_verify1.ffi.string(p.pw_name) == b"root" + ''') + + def test_setuptools_package_2(self): + run_setup_and_program("setuptools_package_2", ''' + import snip_setuptools_verify2 + p = snip_setuptools_verify2.C.getpwuid(0) + assert snip_setuptools_verify2.ffi.string(p.pw_name) == b"root" + ''') diff --git a/pypy/module/thread/test/test_thread.py b/pypy/module/thread/test/test_thread.py --- a/pypy/module/thread/test/test_thread.py +++ b/pypy/module/thread/test/test_thread.py @@ -13,18 +13,26 @@ def f(): lock.acquire() lock.release() + start = thread._count() try: try: for i in range(1000): thread.start_new_thread(f, ()) finally: lock.release() - # wait a bit to allow most threads to finish now - time.sleep(0.5) except (thread.error, MemoryError): cls.w_can_start_many_threads = space.wrap(False) else: cls.w_can_start_many_threads = space.wrap(True) + # wait a bit to allow all threads to finish now + remaining = thread._count() + retries = 0 + while remaining > start: + retries += 1 + if retries == 200: + raise Exception("the test's threads don't stop!") + time.sleep(0.2) + remaining = thread._count() def test_start_new_thread(self): import thread @@ -227,7 +235,7 @@ import signal def f(): - for x in range(5): + for x in range(40): if waiting: thread.interrupt_main() return @@ -236,7 +244,7 @@ def busy_wait(): waiting.append(None) - for x in range(10): + for x in range(50): print 'tick...', x # <-force the GIL to be released, as time.sleep(0.1) # time.sleep doesn't do non-translated waiting.pop() @@ -245,6 +253,8 @@ signal.signal(signal.SIGINT, signal.default_int_handler) for i in range(100): + print + print "loop", i waiting = [] thread.start_new_thread(f, ()) raises(KeyboardInterrupt, busy_wait) diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py --- a/pypy/objspace/descroperation.py +++ b/pypy/objspace/descroperation.py @@ -671,6 +671,7 @@ left, right = specialnames errormsg = "unsupported operand type(s) for %s: '%%N' and '%%N'" % ( symbol.replace('%', '%%'),) + seq_bug_compat = (symbol == '+' or symbol == '*') def binop_impl(space, w_obj1, w_obj2): w_typ1 = space.type(w_obj1) @@ -686,20 +687,16 @@ # __xxx__ and __rxxx__ methods where found by identity. # Note that space.is_w() is potentially not happy if one of them # is None... - if w_left_src is not w_right_src: # XXX - # -- cpython bug compatibility: see objspace/std/test/ - # -- test_unicodeobject.test_str_unicode_concat_overrides. - # -- The following handles "unicode + string subclass" by - # -- pretending that the unicode is a superclass of the - # -- string, thus giving priority to the string subclass' - # -- __radd__() method. The case "string + unicode subclass" - # -- is handled directly by add__String_Unicode(). - if symbol == '+' and space.is_w(w_typ1, space.w_unicode): - w_typ1 = space.w_basestring - # -- end of bug compatibility - if space.is_true(space.issubtype(w_typ2, w_typ1)): - if (w_left_src and w_right_src and - not space.abstract_issubclass_w(w_left_src, w_right_src) and + if w_right_src and (w_left_src is not w_right_src) and w_left_src: + # 'seq_bug_compat' is for cpython bug-to-bug compatibility: + # see objspace/std/test/test_unicodeobject.*concat_overrides + # and objspace/test/test_descrobject.*rmul_overrides. + # For cases like "unicode + string subclass". + if ((seq_bug_compat and w_typ1.flag_sequence_bug_compat + and not w_typ2.flag_sequence_bug_compat) + # the non-bug-compat part is the following check: + or space.is_true(space.issubtype(w_typ2, w_typ1))): + if (not space.abstract_issubclass_w(w_left_src, w_right_src) and not space.abstract_issubclass_w(w_typ1, w_right_src)): w_obj1, w_obj2 = w_obj2, w_obj1 w_left_impl, w_right_impl = w_right_impl, w_left_impl diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py --- a/pypy/objspace/std/bytearrayobject.py +++ b/pypy/objspace/std/bytearrayobject.py @@ -534,7 +534,7 @@ if not e.match(space, space.w_TypeError): raise else: - return [c for c in buf.as_str()] + return list(buf.as_str()) # sequence of bytes w_iter = space.iter(w_source) @@ -1131,6 +1131,7 @@ reverse = interp2app(W_BytearrayObject.descr_reverse, doc=BytearrayDocstrings.reverse.__doc__), ) +W_BytearrayObject.typedef.flag_sequence_bug_compat = True init_signature = Signature(['source', 'encoding', 'errors'], None, None) init_defaults = [None, None, None] diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py --- a/pypy/objspace/std/bytesobject.py +++ b/pypy/objspace/std/bytesobject.py @@ -951,6 +951,7 @@ _formatter_field_name_split = interp2app(W_BytesObject.descr_formatter_field_name_split), ) +W_BytesObject.typedef.flag_sequence_bug_compat = True def string_escape_encode(s, quote): diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py --- a/pypy/objspace/std/listobject.py +++ b/pypy/objspace/std/listobject.py @@ -1874,3 +1874,4 @@ insert = interp2app(W_ListObject.descr_insert), remove = interp2app(W_ListObject.descr_remove), ) +W_ListObject.typedef.flag_sequence_bug_compat = True diff --git a/pypy/objspace/std/stdtypedef.py b/pypy/objspace/std/stdtypedef.py --- a/pypy/objspace/std/stdtypedef.py +++ b/pypy/objspace/std/stdtypedef.py @@ -93,6 +93,8 @@ overridetypedef=overridetypedef) if typedef is not overridetypedef: w_type.w_doc = space.wrap(typedef.doc) + if hasattr(typedef, 'flag_sequence_bug_compat'): + w_type.flag_sequence_bug_compat = typedef.flag_sequence_bug_compat w_type.lazyloaders = lazyloaders return w_type diff --git a/pypy/objspace/std/tupleobject.py b/pypy/objspace/std/tupleobject.py --- a/pypy/objspace/std/tupleobject.py +++ b/pypy/objspace/std/tupleobject.py @@ -244,6 +244,7 @@ count = interp2app(W_AbstractTupleObject.descr_count), index = interp2app(W_AbstractTupleObject.descr_index) ) +W_AbstractTupleObject.typedef.flag_sequence_bug_compat = True class W_TupleObject(W_AbstractTupleObject): diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py --- a/pypy/objspace/std/typeobject.py +++ b/pypy/objspace/std/typeobject.py @@ -67,6 +67,7 @@ _immutable_fields_ = ["flag_heaptype", "flag_cpytype", "flag_abstract?", + "flag_sequence_bug_compat", 'needsdel', 'weakrefable', 'hasdict', @@ -104,6 +105,7 @@ w_self.flag_heaptype = False w_self.flag_cpytype = False w_self.flag_abstract = False + w_self.flag_sequence_bug_compat = False w_self.instancetypedef = overridetypedef if overridetypedef is not None: diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -1068,6 +1068,7 @@ _formatter_field_name_split = interp2app(W_UnicodeObject.descr_formatter_field_name_split), ) +W_UnicodeObject.typedef.flag_sequence_bug_compat = True def _create_list_from_unicode(value): diff --git a/pypy/objspace/test/test_descroperation.py b/pypy/objspace/test/test_descroperation.py --- a/pypy/objspace/test/test_descroperation.py +++ b/pypy/objspace/test/test_descroperation.py @@ -734,6 +734,44 @@ assert X() == 'hello' + def test_sequence_rmul_overrides(self): + class oops(object): + def __rmul__(self, other): + return 42 + def __index__(self): + return 3 + assert '2' * oops() == 42 + assert [2] * oops() == 42 + assert (2,) * oops() == 42 + assert u'2' * oops() == 42 + assert bytearray('2') * oops() == 42 + assert 1000 * oops() == 42 + assert '2'.__mul__(oops()) == '222' + + def test_sequence_rmul_overrides_oldstyle(self): + class oops: + def __rmul__(self, other): + return 42 + def __index__(self): + return 3 + assert '2' * oops() == 42 + assert [2] * oops() == 42 + assert (2,) * oops() == 42 + assert u'2' * oops() == 42 + assert bytearray('2') * oops() == 42 + assert 1000 * oops() == 42 + assert '2'.__mul__(oops()) == '222' + + def test_sequence_radd_overrides(self): + class A1(list): + pass + class A2(list): + def __radd__(self, other): + return 42 + assert [2] + A1([3]) == [2, 3] + assert type([2] + A1([3])) is list + assert [2] + A2([3]) == 42 + class AppTestWithBuiltinShortcut(AppTest_Descroperation): spaceconfig = {'objspace.std.builtinshortcut': True} diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -297,7 +297,13 @@ argparse = imp.load_source('argparse', 'lib-python/2.7/argparse.py') if sys.platform == 'win32': pypy_exe = 'pypy.exe' - license_base = os.path.join(basedir, r'..\..\..\local') # as on buildbot YMMV + for p in [os.path.join(basedir, r'..\..\..\local'), #buildbot + os.path.join(basedir, r'..\local')]: # pypy/doc/windows.rst + if os.path.exists(p): + license_base = p + break + else: + license_base = 'unkown' else: pypy_exe = 'pypy' license_base = '/usr/share/doc' @@ -370,5 +376,21 @@ if __name__ == '__main__': import sys + if sys.platform == 'win32': + # Try to avoid opeing a dialog box if one of the + # subprocesses causes a system error + import ctypes + winapi = ctypes.windll.kernel32 + SetErrorMode = winapi.SetErrorMode + SetErrorMode.argtypes=[ctypes.c_int] + + SEM_FAILCRITICALERRORS = 1 + SEM_NOGPFAULTERRORBOX = 2 + SEM_NOOPENFILEERRORBOX = 0x8000 + flags = SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX | SEM_NOOPENFILEERRORBOX + #Since there is no GetErrorMode, do a double Set + old_mode = SetErrorMode(flags) + SetErrorMode(old_mode | flags) + retval, _ = package(*sys.argv[1:]) sys.exit(retval) diff --git a/pypy/tool/release/test/test_package.py b/pypy/tool/release/test/test_package.py --- a/pypy/tool/release/test/test_package.py +++ b/pypy/tool/release/test/test_package.py @@ -115,15 +115,21 @@ check(pypy, 0755) def test_generate_license(): - from os.path import dirname, abspath, join + from os.path import dirname, abspath, join, exists class Options(object): pass options = Options() basedir = dirname(dirname(dirname(dirname(dirname(abspath(__file__)))))) options.no_tk = False if sys.platform == 'win32': - # as on buildbot YMMV - options.license_base = join(basedir, r'..\..\..\local') + for p in [join(basedir, r'..\..\..\local'), #buildbot + join(basedir, r'..\local')]: # pypy/doc/windows.rst + if exists(p): + license_base = p + break + else: + license_base = 'unkown' + options.license_base = license_base else: options.license_base = '/usr/share/doc' license = package.generate_license(py.path.local(basedir), options) diff --git a/rpython/annotator/builtin.py b/rpython/annotator/builtin.py --- a/rpython/annotator/builtin.py +++ b/rpython/annotator/builtin.py @@ -255,8 +255,13 @@ BUILTIN_ANALYZERS[original] = value - at analyzer_for(getattr(OSError.__init__, 'im_func', OSError.__init__)) -def OSError_init(s_self, *args): + at analyzer_for(getattr(object.__init__, 'im_func', object.__init__)) +def object_init(s_self, *args): + # ignore - mostly used for abstract classes initialization + pass + + at analyzer_for(getattr(EnvironmentError.__init__, 'im_func', EnvironmentError.__init__)) +def EnvironmentError_init(s_self, *args): pass try: @@ -268,11 +273,6 @@ def WindowsError_init(s_self, *args): pass - at analyzer_for(getattr(object.__init__, 'im_func', object.__init__)) -def object_init(s_self, *args): - # ignore - mostly used for abstract classes initialization - pass - @analyzer_for(sys.getdefaultencoding) def conf(): diff --git a/rpython/annotator/classdef.py b/rpython/annotator/classdef.py --- a/rpython/annotator/classdef.py +++ b/rpython/annotator/classdef.py @@ -438,8 +438,10 @@ # ____________________________________________________________ FORCE_ATTRIBUTES_INTO_CLASSES = { - OSError: {'errno': SomeInteger()}, - } + EnvironmentError: {'errno': SomeInteger(), + 'strerror': SomeString(can_be_None=True), + 'filename': SomeString(can_be_None=True)}, +} try: WindowsError @@ -455,4 +457,3 @@ else: FORCE_ATTRIBUTES_INTO_CLASSES[termios.error] = \ {'args': SomeTuple([SomeInteger(), SomeString()])} - diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py --- a/rpython/annotator/unaryop.py +++ b/rpython/annotator/unaryop.py @@ -304,10 +304,10 @@ def hint(self, *args_s): hints = args_s[-1].const if 'maxlength' in hints: - # only for iteration over lists or dicts at the moment, + # only for iteration over lists or dicts or strs at the moment, # not over an iterator object (because it has no known length) s_iterable = args_s[0] - if isinstance(s_iterable, (SomeList, SomeDict)): + if isinstance(s_iterable, (SomeList, SomeDict, SomeString)): self = SomeList(self.listdef) # create a fresh copy self.listdef.resize() self.listdef.listitem.hint_maxlength = True diff --git a/rpython/jit/backend/arm/callbuilder.py b/rpython/jit/backend/arm/callbuilder.py --- a/rpython/jit/backend/arm/callbuilder.py +++ b/rpython/jit/backend/arm/callbuilder.py @@ -92,7 +92,8 @@ self.mc.LDR_ri(r.r7.value, r.r5.value) # change 'rpy_fastgil' to 0 (it should be non-zero right now) - self.mc.DMB() + if self.asm.cpu.cpuinfo.arch_version >= 7: + self.mc.DMB() self.mc.gen_load_int(r.r6.value, fastgil) self.mc.MOV_ri(r.ip.value, 0) self.mc.STR_ri(r.ip.value, r.r6.value) @@ -112,7 +113,8 @@ self.mc.STREX(r.r3.value, r.ip.value, r.r6.value, c=c.EQ) # try to claim the lock self.mc.CMP_ri(r.r3.value, 0, cond=c.EQ) # did this succeed? - self.mc.DMB() + if self.asm.cpu.cpuinfo.arch_version >= 7: + self.mc.DMB() # the success of the lock acquisition is defined by # 'EQ is true', or equivalently by 'r3 == 0'. # diff --git a/rpython/jit/backend/arm/codebuilder.py b/rpython/jit/backend/arm/codebuilder.py --- a/rpython/jit/backend/arm/codebuilder.py +++ b/rpython/jit/backend/arm/codebuilder.py @@ -333,8 +333,13 @@ | (rn & 0xF) << 16) def DMB(self): - # note: 'cond' is only permitted on Thumb here - self.write32(0xf57ff05f) + # ARMv7 only. I guess ARMv6 CPUs cannot be used in symmetric + # multi-processing at all? That would make this instruction unneeded. + # note: 'cond' is only permitted on Thumb here, but don't + # write literally 0xf57ff05f, because it's larger than 31 bits + c = cond.AL + self.write32(c << 28 + | 0x157ff05f) DIV = binary_helper_call('int_div') MOD = binary_helper_call('int_mod') diff --git a/rpython/jit/backend/arm/detect.py b/rpython/jit/backend/arm/detect.py --- a/rpython/jit/backend/arm/detect.py +++ b/rpython/jit/backend/arm/detect.py @@ -38,9 +38,9 @@ try: buf = os.read(fd, 2048) if not buf: + n = 6 # we assume ARMv6 as base case debug_print("Could not detect ARM architecture " "version, assuming", "ARMv%d" % n) - n = 6 # we asume ARMv6 as base case finally: os.close(fd) # "Processor : ARMv%d-compatible processor rev 7 (v6l)" diff --git a/rpython/jit/backend/arm/instructions.py b/rpython/jit/backend/arm/instructions.py --- a/rpython/jit/backend/arm/instructions.py +++ b/rpython/jit/backend/arm/instructions.py @@ -142,6 +142,7 @@ #'VCVT' : {'opc1':0xB, 'opc2':0xE, 'opc3':0x1, 'base': False}, } +# ARMv7 only simd_instructions_3regs = { 'VADD_i64': {'A': 0x8, 'B': 0, 'U': 0}, 'VSUB_i64': {'A': 0x8, 'B': 0, 'U': 1}, diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py --- a/rpython/jit/backend/arm/opassembler.py +++ b/rpython/jit/backend/arm/opassembler.py @@ -1128,6 +1128,8 @@ self.mc.VCVT_int_to_float(res.value, r.svfp_ip.value) return fcond + # the following five instructions are only ARMv7; + # regalloc.py won't call them at all on ARMv6 emit_op_llong_add = gen_emit_float_op('llong_add', 'VADD_i64') emit_op_llong_sub = gen_emit_float_op('llong_sub', 'VSUB_i64') emit_op_llong_and = gen_emit_float_op('llong_and', 'VAND_i64') diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py --- a/rpython/jit/backend/arm/regalloc.py +++ b/rpython/jit/backend/arm/regalloc.py @@ -184,7 +184,7 @@ class Regalloc(BaseRegalloc): - def __init__(self, assembler=None): + def __init__(self, assembler): self.cpu = assembler.cpu From noreply at buildbot.pypy.org Tue Sep 23 11:25:31 2014 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 23 Sep 2014 11:25:31 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: merge Message-ID: <20140923092531.2A6F41D23FA@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73655:d1ca110c9df4 Date: 2014-09-23 11:25 +0200 http://bitbucket.org/pypy/pypy/changeset/d1ca110c9df4/ Log: merge diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -424,7 +424,8 @@ def gen_malloc_nursery(self, size, v_result): """Try to generate or update a CALL_MALLOC_NURSERY. - If that fails, generate a plain CALL_MALLOC_GC instead. + If that succeeds, return True; you still need to write the tid. + If that fails, return False. """ size = self.round_up_for_allocation(size) if not self.gc_ll_descr.can_use_nursery_malloc(size): From noreply at buildbot.pypy.org Tue Sep 23 11:37:32 2014 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 23 Sep 2014 11:37:32 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: zero=True in llgraph backend Message-ID: <20140923093732.AAC991D23FA@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73656:7c1ac1242b23 Date: 2014-09-23 11:37 +0200 http://bitbucket.org/pypy/pypy/changeset/7c1ac1242b23/ Log: zero=True in llgraph backend diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py --- a/rpython/jit/backend/llgraph/runner.py +++ b/rpython/jit/backend/llgraph/runner.py @@ -630,17 +630,17 @@ def bh_new(self, sizedescr): return lltype.cast_opaque_ptr(llmemory.GCREF, - lltype.malloc(sizedescr.S)) + lltype.malloc(sizedescr.S, zero=True)) def bh_new_with_vtable(self, vtable, descr): - result = lltype.malloc(descr.S) + result = lltype.malloc(descr.S, zero=True) result_as_objptr = lltype.cast_pointer(rclass.OBJECTPTR, result) result_as_objptr.typeptr = support.cast_from_int(rclass.CLASSTYPE, vtable) return lltype.cast_opaque_ptr(llmemory.GCREF, result) def bh_new_array(self, length, arraydescr): - array = lltype.malloc(arraydescr.A, length) + array = lltype.malloc(arraydescr.A, length, zero=True) return lltype.cast_opaque_ptr(llmemory.GCREF, array) def bh_clear_array_contents(self, a, descr): From noreply at buildbot.pypy.org Tue Sep 23 11:46:26 2014 From: noreply at buildbot.pypy.org (cfbolz) Date: Tue, 23 Sep 2014 11:46:26 +0200 (CEST) Subject: [pypy-commit] pypy default: make the "store sink" optimization actually do store sinking Message-ID: <20140923094626.777561C0410@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: Changeset: r73657:2cb8ead5710a Date: 2014-09-06 17:37 +0200 http://bitbucket.org/pypy/pypy/changeset/2cb8ead5710a/ Log: make the "store sink" optimization actually do store sinking (so far it only did load forwarding) diff --git a/rpython/translator/backendopt/storesink.py b/rpython/translator/backendopt/storesink.py --- a/rpython/translator/backendopt/storesink.py +++ b/rpython/translator/backendopt/storesink.py @@ -35,8 +35,10 @@ elif op.opname in ['setarrayitem', 'setinteriorfield']: pass elif op.opname == 'setfield': - clear_cache_for(cache, op.args[0].concretetype, - op.args[1].value) + target = op.args[0] + field = op.args[1].value + clear_cache_for(cache, target.concretetype, field) + cache[target, field] = op.args[2] elif has_side_effects(op): cache = {} newops.append(op) diff --git a/rpython/translator/backendopt/test/test_storesink.py b/rpython/translator/backendopt/test/test_storesink.py --- a/rpython/translator/backendopt/test/test_storesink.py +++ b/rpython/translator/backendopt/test/test_storesink.py @@ -42,7 +42,7 @@ a.x = i return a.x - self.check(f, [int], 1) + self.check(f, [int], 0) def test_simple(self): class A(object): @@ -53,7 +53,7 @@ a.x = i return a.x + a.x - self.check(f, [int], 1) + self.check(f, [int], 0) def test_irrelevant_setfield(self): class A(object): @@ -67,7 +67,7 @@ two = a.x return one + two - self.check(f, [int], 1) + self.check(f, [int], 0) def test_relevant_setfield(self): class A(object): @@ -101,7 +101,7 @@ two = a.x return one + two - self.check(f, [int], 1) + self.check(f, [int], 0) def test_subclass(self): class A(object): @@ -119,7 +119,7 @@ two = a.x return one + two - self.check(f, [int], 2) + self.check(f, [int], 1) def test_bug_1(self): class A(object): @@ -133,4 +133,4 @@ return True return n - self.check(f, [int], 1) + self.check(f, [int], 0) From noreply at buildbot.pypy.org Tue Sep 23 11:46:27 2014 From: noreply at buildbot.pypy.org (cfbolz) Date: Tue, 23 Sep 2014 11:46:27 +0200 (CEST) Subject: [pypy-commit] pypy default: put handling of a block into its own function Message-ID: <20140923094627.A8B761C0410@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: Changeset: r73658:25286b94d20e Date: 2014-09-06 17:50 +0200 http://bitbucket.org/pypy/pypy/changeset/25286b94d20e/ Log: put handling of a block into its own function in preparation of a future refactoring diff --git a/rpython/translator/backendopt/storesink.py b/rpython/translator/backendopt/storesink.py --- a/rpython/translator/backendopt/storesink.py +++ b/rpython/translator/backendopt/storesink.py @@ -10,40 +10,49 @@ except AttributeError: return True + def storesink_graph(graph): + added_some_same_as = False + for block in graph.iterblocks(): + newops = [] + cache = {} + + newops, _some_same_as = _storesink_block(block, cache) + added_some_same_as = _some_same_as or added_some_same_as + if block.operations: + block.operations = newops + + if added_some_same_as: + removenoops.remove_same_as(graph) + + +def _storesink_block(block, cache): def clear_cache_for(cache, concretetype, fieldname): for k in cache.keys(): if k[0].concretetype == concretetype and k[1] == fieldname: del cache[k] added_some_same_as = False - - for block in graph.iterblocks(): - newops = [] - cache = {} - for op in block.operations: - if op.opname == 'getfield': - tup = (op.args[0], op.args[1].value) - res = cache.get(tup, None) - if res is not None: - op.opname = 'same_as' - op.args = [res] - added_some_same_as = True - else: - cache[tup] = op.result - elif op.opname in ['setarrayitem', 'setinteriorfield']: - pass - elif op.opname == 'setfield': - target = op.args[0] - field = op.args[1].value - clear_cache_for(cache, target.concretetype, field) - cache[target, field] = op.args[2] - elif has_side_effects(op): - cache = {} - newops.append(op) - if block.operations: - block.operations = newops - - if added_some_same_as: - removenoops.remove_same_as(graph) + newops = [] + for op in block.operations: + if op.opname == 'getfield': + tup = (op.args[0], op.args[1].value) + res = cache.get(tup, None) + if res is not None: + op.opname = 'same_as' + op.args = [res] + added_some_same_as = True + else: + cache[tup] = op.result + elif op.opname in ['setarrayitem', 'setinteriorfield']: + pass + elif op.opname == 'setfield': + target = op.args[0] + field = op.args[1].value + clear_cache_for(cache, target.concretetype, field) + cache[target, field] = op.args[2] + elif has_side_effects(op): + cache = {} + newops.append(op) + return newops, added_some_same_as From noreply at buildbot.pypy.org Tue Sep 23 11:46:28 2014 From: noreply at buildbot.pypy.org (cfbolz) Date: Tue, 23 Sep 2014 11:46:28 +0200 (CEST) Subject: [pypy-commit] pypy default: extend storesink to be super-local, ie to follow links that are not merge links Message-ID: <20140923094628.C42701C0410@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: Changeset: r73659:9f2eaab7ceeb Date: 2014-09-06 18:56 +0200 http://bitbucket.org/pypy/pypy/changeset/9f2eaab7ceeb/ Log: extend storesink to be super-local, ie to follow links that are not merge links diff --git a/rpython/translator/backendopt/storesink.py b/rpython/translator/backendopt/storesink.py --- a/rpython/translator/backendopt/storesink.py +++ b/rpython/translator/backendopt/storesink.py @@ -1,6 +1,8 @@ from rpython.rtyper.lltypesystem.lloperation import llop +from rpython.flowspace.model import mkentrymap, Variable from rpython.translator.backendopt import removenoops +from rpython.translator import simplify def has_side_effects(op): if op.opname == 'debug_assert': @@ -12,22 +14,63 @@ def storesink_graph(graph): + """ remove superfluous getfields. use a super-local method: all non-join + blocks inherit the heap information from their (single) predecessor + """ added_some_same_as = False + entrymap = mkentrymap(graph) - for block in graph.iterblocks(): - newops = [] - cache = {} + # all merge blocks are starting points + todo = [(block, None, None) for (block, prev_blocks) in entrymap.iteritems() + if len(prev_blocks) > 1 or block is graph.startblock] - newops, _some_same_as = _storesink_block(block, cache) - added_some_same_as = _some_same_as or added_some_same_as + visited = 0 + + while todo: + block, cache, inputlink = todo.pop() + visited += 1 + if cache is None: + cache = {} + if block.operations: - block.operations = newops + newops, changed_block = _storesink_block(block, cache, inputlink) + added_some_same_as = changed_block or added_some_same_as + if changed_block: + block.operations = newops + for link in block.exits: + if len(entrymap[link.target]) == 1: + new_cache = _translate_cache(cache, link) + todo.append((link.target, new_cache, link)) + assert visited == len(entrymap) if added_some_same_as: removenoops.remove_same_as(graph) + simplify.transform_dead_op_vars(graph) +def _translate_cache(cache, link): + if link.target.operations == (): # exit or except block: + return {} + block = link.target + local_versions = {var1: var2 for var1, var2 in zip(link.args, block.inputargs)} + def _translate_arg(arg): + if isinstance(arg, Variable): + res = local_versions.get(arg, None) + if res is None: + res = Variable(arg) + res.concretetype = arg.concretetype + link.args.append(arg) + block.inputargs.append(res) + local_versions[arg] = res + return res + else: + return arg + new_cache = {} + for (var, field), res in cache.iteritems(): + if var in local_versions or not isinstance(var, Variable): + new_cache[_translate_arg(var), field] = _translate_arg(res) + return new_cache -def _storesink_block(block, cache): +def _storesink_block(block, cache, inputlink): def clear_cache_for(cache, concretetype, fieldname): for k in cache.keys(): if k[0].concretetype == concretetype and k[1] == fieldname: @@ -53,6 +96,6 @@ clear_cache_for(cache, target.concretetype, field) cache[target, field] = op.args[2] elif has_side_effects(op): - cache = {} + cache.clear() newops.append(op) return newops, added_some_same_as diff --git a/rpython/translator/backendopt/test/test_storesink.py b/rpython/translator/backendopt/test/test_storesink.py --- a/rpython/translator/backendopt/test/test_storesink.py +++ b/rpython/translator/backendopt/test/test_storesink.py @@ -134,3 +134,22 @@ return n self.check(f, [int], 0) + + + def test_cfg_splits(self): + class A(object): + pass + + def f(i): + a = A() + j = i + for i in range(i): + a.x = i + if i: + j = a.x + a.x + else: + j = a.x * 5 + return j + + self.check(f, [int], 0) + From noreply at buildbot.pypy.org Tue Sep 23 11:46:29 2014 From: noreply at buildbot.pypy.org (cfbolz) Date: Tue, 23 Sep 2014 11:46:29 +0200 (CEST) Subject: [pypy-commit] pypy default: up to now, malloc invalidated the cache. fix Message-ID: <20140923094629.E0F481C0410@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: Changeset: r73660:876563a39e5b Date: 2014-09-06 19:11 +0200 http://bitbucket.org/pypy/pypy/changeset/876563a39e5b/ Log: up to now, malloc invalidated the cache. fix diff --git a/rpython/translator/backendopt/storesink.py b/rpython/translator/backendopt/storesink.py --- a/rpython/translator/backendopt/storesink.py +++ b/rpython/translator/backendopt/storesink.py @@ -95,7 +95,7 @@ field = op.args[1].value clear_cache_for(cache, target.concretetype, field) cache[target, field] = op.args[2] - elif has_side_effects(op): + elif has_side_effects(op) and op.opname not in ("malloc", "malloc_varsize"): cache.clear() newops.append(op) return newops, added_some_same_as diff --git a/rpython/translator/backendopt/test/test_storesink.py b/rpython/translator/backendopt/test/test_storesink.py --- a/rpython/translator/backendopt/test/test_storesink.py +++ b/rpython/translator/backendopt/test/test_storesink.py @@ -153,3 +153,17 @@ self.check(f, [int], 0) + def test_malloc_does_not_invalidate(self): + class A(object): + pass + class B(object): + pass + + def f(i): + a = A() + a.x = i + b = B() + return a.x + + self.check(f, [int], 0) + From noreply at buildbot.pypy.org Tue Sep 23 11:46:30 2014 From: noreply at buildbot.pypy.org (cfbolz) Date: Tue, 23 Sep 2014 11:46:30 +0200 (CEST) Subject: [pypy-commit] pypy default: more natural way to express this Message-ID: <20140923094630.F3F041C0410@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: Changeset: r73661:79cdb86a9075 Date: 2014-09-06 20:48 +0200 http://bitbucket.org/pypy/pypy/changeset/79cdb86a9075/ Log: more natural way to express this diff --git a/rpython/translator/backendopt/storesink.py b/rpython/translator/backendopt/storesink.py --- a/rpython/translator/backendopt/storesink.py +++ b/rpython/translator/backendopt/storesink.py @@ -88,14 +88,14 @@ added_some_same_as = True else: cache[tup] = op.result - elif op.opname in ['setarrayitem', 'setinteriorfield']: + elif op.opname in ('setarrayitem', 'setinteriorfield', "malloc", "malloc_varsize"): pass elif op.opname == 'setfield': target = op.args[0] field = op.args[1].value clear_cache_for(cache, target.concretetype, field) cache[target, field] = op.args[2] - elif has_side_effects(op) and op.opname not in ("malloc", "malloc_varsize"): + elif has_side_effects(op): cache.clear() newops.append(op) return newops, added_some_same_as From noreply at buildbot.pypy.org Tue Sep 23 11:46:32 2014 From: noreply at buildbot.pypy.org (cfbolz) Date: Tue, 23 Sep 2014 11:46:32 +0200 (CEST) Subject: [pypy-commit] pypy default: operations are changed in place Message-ID: <20140923094632.1826E1C0410@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: Changeset: r73662:93b1ad2ab1fd Date: 2014-09-06 20:50 +0200 http://bitbucket.org/pypy/pypy/changeset/93b1ad2ab1fd/ Log: operations are changed in place diff --git a/rpython/translator/backendopt/storesink.py b/rpython/translator/backendopt/storesink.py --- a/rpython/translator/backendopt/storesink.py +++ b/rpython/translator/backendopt/storesink.py @@ -33,10 +33,8 @@ cache = {} if block.operations: - newops, changed_block = _storesink_block(block, cache, inputlink) + changed_block = _storesink_block(block, cache, inputlink) added_some_same_as = changed_block or added_some_same_as - if changed_block: - block.operations = newops for link in block.exits: if len(entrymap[link.target]) == 1: new_cache = _translate_cache(cache, link) @@ -77,7 +75,6 @@ del cache[k] added_some_same_as = False - newops = [] for op in block.operations: if op.opname == 'getfield': tup = (op.args[0], op.args[1].value) @@ -97,5 +94,4 @@ cache[target, field] = op.args[2] elif has_side_effects(op): cache.clear() - newops.append(op) - return newops, added_some_same_as + return added_some_same_as From noreply at buildbot.pypy.org Tue Sep 23 12:13:42 2014 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 23 Sep 2014 12:13:42 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: Add a few commented-out lines about a case that crashes but that isn't Message-ID: <20140923101342.2EF791D39A8@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: gc_no_cleanup_nursery Changeset: r73663:eb615ab297d1 Date: 2014-09-23 12:13 +0200 http://bitbucket.org/pypy/pypy/changeset/eb615ab297d1/ Log: Add a few commented-out lines about a case that crashes but that isn't too important diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py --- a/rpython/translator/c/test/test_newgc.py +++ b/rpython/translator/c/test/test_newgc.py @@ -1200,7 +1200,9 @@ lltype.Struct("s1", ("x", lltype.Signed))), ('y', lltype.Signed)) A = lltype.GcArray(lltype.Signed) - + B = lltype.GcStruct("b", ('x', lltype.Signed), + ('y', lltype.Array(lltype.Signed))) + def fn(): s = lltype.malloc(S, zero=True) assert s.x == 0 @@ -1208,6 +1210,11 @@ assert s2.parent.x == 0 a = lltype.malloc(A, 3, zero=True) assert a[2] == 0 + # XXX not supported right now in gctransform/framework.py: + #b = lltype.malloc(B, 3, zero=True) + #assert len(b.y) == 3 + #assert b.x == 0 + #assert b.y[0] == b.y[1] == b.y[2] == 0 return 0 return fn From noreply at buildbot.pypy.org Tue Sep 23 12:39:52 2014 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 23 Sep 2014 12:39:52 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: Add comments. Properize gct_do_malloc_fixedsize_clear(). Message-ID: <20140923103952.C6AB71C03FC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: gc_no_cleanup_nursery Changeset: r73664:840d14a0dfd5 Date: 2014-09-23 12:39 +0200 http://bitbucket.org/pypy/pypy/changeset/840d14a0dfd5/ Log: Add comments. Properize gct_do_malloc_fixedsize_clear(). diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py --- a/rpython/memory/gctransform/framework.py +++ b/rpython/memory/gctransform/framework.py @@ -842,7 +842,7 @@ op = hop.spaceop v_size = op.args[1] c_fixedofs = rmodel.inputconst(lltype.Signed, - rffi.sizeof(lltype.Signed)) + llmemory.sizeof(self.HDR)) c_size = rmodel.inputconst(lltype.Signed, 1) v_a = op.result v_real_size = hop.genop('int_sub', [v_size, c_fixedofs], @@ -867,6 +867,10 @@ # used by the JIT (see rpython.jit.backend.llsupport.gc) self.gct_do_malloc_varsize(hop) if not self.malloc_zero_filled: + # We don't support GcStruct with a nested Array here. But it's + # hard to know if it is one or not... For now we're implicitly + # assuming that the JIT's gc.py know what it is doing and not + # using this llop on GcStructs with nested Arrays. op = hop.spaceop v_size = op.args[1] c_fixedofs = op.args[2] From noreply at buildbot.pypy.org Tue Sep 23 14:31:33 2014 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 23 Sep 2014 14:31:33 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: Expand the code to catch more missing cases and crash with clean errors Message-ID: <20140923123133.BE6F71C0605@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: gc_no_cleanup_nursery Changeset: r73665:b76c81163a3d Date: 2014-09-23 14:31 +0200 http://bitbucket.org/pypy/pypy/changeset/b76c81163a3d/ Log: Expand the code to catch more missing cases and crash with clean errors diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py --- a/rpython/memory/gctransform/framework.py +++ b/rpython/memory/gctransform/framework.py @@ -1222,6 +1222,16 @@ previous_steps + [c_name], everything=everything) continue + if isinstance(FIELD, lltype.Array): + if everything: + raise NotImplementedError( + "%s: Struct-containing-Array with everything=True" + % (TYPE,)) + if gctypelayout.offsets_to_gc_pointers(FIELD.OF): + raise NotImplementedError( + "%s: Struct-containing-Array-with-gc-pointers" + % (TYPE,)) + continue if ((isinstance(FIELD, lltype.Ptr) and FIELD._needsgc()) or everything): c_null = rmodel.inputconst(FIELD, FIELD._defl()) @@ -1230,9 +1240,6 @@ [v] + previous_steps + [c_name, c_null]) else: llops.genop('bare_setfield', [v, c_name, c_null]) - elif (isinstance(FIELD, lltype.Array) and - isinstance(FIELD.OF, lltype.Ptr) and FIELD.OF._needsgc()): - xxx return elif isinstance(TYPE, lltype.Array): From noreply at buildbot.pypy.org Tue Sep 23 14:55:56 2014 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 23 Sep 2014 14:55:56 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: The comment added in 840d14a0dfd5 is wrong, and is the source of the Message-ID: <20140923125556.DE21D1D38CB@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: gc_no_cleanup_nursery Changeset: r73666:c3de4ba09843 Date: 2014-09-23 14:55 +0200 http://bitbucket.org/pypy/pypy/changeset/c3de4ba09843/ Log: The comment added in 840d14a0dfd5 is wrong, and is the source of the crashes of the translated pypy: we get some RPython strings with a bogus uninitialized value for their 'rs_hash'. diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py --- a/rpython/memory/gctransform/framework.py +++ b/rpython/memory/gctransform/framework.py @@ -841,14 +841,13 @@ if not self.malloc_zero_filled: op = hop.spaceop v_size = op.args[1] - c_fixedofs = rmodel.inputconst(lltype.Signed, + c_after_header = rmodel.inputconst(lltype.Signed, llmemory.sizeof(self.HDR)) - c_size = rmodel.inputconst(lltype.Signed, 1) v_a = op.result - v_real_size = hop.genop('int_sub', [v_size, c_fixedofs], - resulttype=lltype.Signed) - self.emit_raw_memclear(hop.llops, v_real_size, c_size, - c_fixedofs, v_a) + v_clear_size = hop.genop('int_sub', [v_size, c_after_header], + resulttype=lltype.Signed) + self.emit_raw_memclear(hop.llops, v_clear_size, None, + c_after_header, v_a) def gct_do_malloc_varsize(self, hop): # used by the JIT (see rpython.jit.backend.llsupport.gc) @@ -867,16 +866,24 @@ # used by the JIT (see rpython.jit.backend.llsupport.gc) self.gct_do_malloc_varsize(hop) if not self.malloc_zero_filled: - # We don't support GcStruct with a nested Array here. But it's - # hard to know if it is one or not... For now we're implicitly - # assuming that the JIT's gc.py know what it is doing and not - # using this llop on GcStructs with nested Arrays. op = hop.spaceop - v_size = op.args[1] - c_fixedofs = op.args[2] - c_size = op.args[3] + v_num_elem = op.args[1] + c_basesize = op.args[2] + c_itemsize = op.args[3] + c_length_ofs = op.args[4] v_a = op.result - self.emit_raw_memclear(hop.llops, v_size, c_size, c_fixedofs, v_a) + # Clear the fixed-size part, which is everything after the + # GC header and before the length field. This might be 0 + # bytes long. + c_after_header = rmodel.inputconst(lltype.Signed, + llmemory.sizeof(self.HDR)) + v_clear_size = hop.genop('int_sub', [c_length_ofs, c_after_header], + resulttype=lltype.Signed) + self.emit_raw_memclear(hop.llops, v_clear_size, None, + c_after_header, v_a) + # Clear the variable-size part + self.emit_raw_memclear(hop.llops, v_num_elem, c_itemsize, + c_basesize, v_a) def gct_get_write_barrier_failing_case(self, hop): op = hop.spaceop @@ -1244,20 +1251,7 @@ return elif isinstance(TYPE, lltype.Array): ITEM = TYPE.OF - if everything: - needs_clearing = True - elif isinstance(ITEM, lltype.Ptr) and ITEM._needsgc(): - needs_clearing = True - elif isinstance(ITEM, lltype.Struct): - for SUBITEM in ITEM._flds.values(): - if isinstance(SUBITEM, lltype.Ptr) and SUBITEM._needsgc(): - needs_clearing = True - break - else: - needs_clearing = False - else: - needs_clearing = False - if needs_clearing: + if everything or gctypelayout.offsets_to_gc_pointers(ITEM): v_size = llops.genop('getarraysize', [v], resulttype=lltype.Signed) c_size = rmodel.inputconst(lltype.Signed, llmemory.sizeof(ITEM)) @@ -1271,8 +1265,11 @@ raise TypeError(TYPE) def emit_raw_memclear(self, llops, v_size, c_size, c_fixedofs, v_a): - v_totalsize = llops.genop('int_mul', [v_size, c_size], - resulttype=lltype.Signed) + if c_size is None: + v_totalsize = v_size + else: + v_totalsize = llops.genop('int_mul', [v_size, c_size], + resulttype=lltype.Signed) v_adr = llops.genop('adr_add', [v_a, c_fixedofs], resulttype=llmemory.Address) llops.genop('raw_memclear', [v_adr, v_totalsize]) From noreply at buildbot.pypy.org Tue Sep 23 16:09:41 2014 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 23 Sep 2014 16:09:41 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: Test and fix: we need to explicitly zero the hash field in the assembler Message-ID: <20140923140941.70A721C0410@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: gc_no_cleanup_nursery Changeset: r73667:bade4d08b1c3 Date: 2014-09-23 16:09 +0200 http://bitbucket.org/pypy/pypy/changeset/bade4d08b1c3/ Log: Test and fix: we need to explicitly zero the hash field in the assembler produced by NEWSTR or NEWUNICODE. diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py --- a/rpython/jit/backend/llsupport/gc.py +++ b/rpython/jit/backend/llsupport/gc.py @@ -38,6 +38,8 @@ def _setup_str(self): self.str_descr = get_array_descr(self, rstr.STR) self.unicode_descr = get_array_descr(self, rstr.UNICODE) + self.str_hash_descr = get_field_descr(self, rstr.STR, 'hash') + self.unicode_hash_descr = get_field_descr(self, rstr.UNICODE, 'hash') def generate_function(self, funcname, func, ARGS, RESULT=llmemory.GCREF): """Generates a variant of malloc with the given name and the given diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -147,14 +147,25 @@ except KeyError: pass - def clear_varsize_gc_fields(self, descr, result, v_length=None): + def clear_varsize_gc_fields(self, kind, descr, result, v_length): if self.gc_ll_descr.malloc_zero_filled: return - if descr.is_array_of_structs() or descr.is_array_of_pointers(): - # for the case of array of structs, this is for correctness only, - # since in practice all GC arrays of structs are allocated - # with malloc(zero=True) - self.handle_clear_array_contents(descr, result, v_length) + if kind == FLAG_ARRAY: + if descr.is_array_of_structs() or descr.is_array_of_pointers(): + # for the case of array of structs, this is for correctness + # only, since in practice all GC arrays of structs are + # allocated with malloc(zero=True) + self.handle_clear_array_contents(descr, result, v_length) + return + if kind == FLAG_STR: + hash_descr = self.gc_ll_descr.str_hash_descr + elif kind == FLAG_UNICODE: + hash_descr = self.gc_ll_descr.unicode_hash_descr + else: + return + op = ResOperation(rop.SETFIELD_GC, [result, self.c_zero], None, + descr=hash_descr) + self.newops.append(op) def handle_new_fixedsize(self, descr, op): assert isinstance(descr, SizeDescr) @@ -185,8 +196,8 @@ # might end up being allocated by malloc_external or some # stuff that initializes GC header fields differently self.gen_initialize_len(op.result, v_length, arraydescr.lendescr) - if kind == FLAG_ARRAY: - self.clear_varsize_gc_fields(op.getdescr(), op.result, v_length) + self.clear_varsize_gc_fields(kind, op.getdescr(), op.result, + v_length) return if (total_size >= 0 and self.gen_malloc_nursery(total_size, op.result)): @@ -204,8 +215,7 @@ self.gen_malloc_unicode(v_length, op.result) else: raise NotImplementedError(op.getopname()) - if kind == FLAG_ARRAY: - self.clear_varsize_gc_fields(op.getdescr(), op.result, v_length) + self.clear_varsize_gc_fields(kind, op.getdescr(), op.result, v_length) def handle_clear_array_contents(self, arraydescr, v_arr, v_length=None): # XXX more work here to reduce or remove the ZERO_ARRAY in some cases diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py --- a/rpython/jit/backend/llsupport/test/test_rewrite.py +++ b/rpython/jit/backend/llsupport/test/test_rewrite.py @@ -69,6 +69,8 @@ unicodedescr = self.gc_ll_descr.unicode_descr strlendescr = strdescr.lendescr unicodelendescr = unicodedescr.lendescr + strhashdescr = self.gc_ll_descr.str_hash_descr + unicodehashdescr = self.gc_ll_descr.unicode_hash_descr casmdescr = JitCellToken() clt = FakeLoopToken() @@ -427,6 +429,7 @@ [i0] p0 = call_malloc_nursery_varsize(1, 1, i0, descr=strdescr) setfield_gc(p0, i0, descr=strlendescr) + setfield_gc(p0, 0, descr=strhashdescr) jump(i0) """) @@ -550,15 +553,19 @@ unicodedescr.basesize + 10 * unicodedescr.itemsize)d) setfield_gc(p0, %(strdescr.tid)d, descr=tiddescr) setfield_gc(p0, 14, descr=strlendescr) + setfield_gc(p0, 0, descr=strhashdescr) p1 = int_add(p0, %(strdescr.basesize + 16 * strdescr.itemsize)d) setfield_gc(p1, %(unicodedescr.tid)d, descr=tiddescr) setfield_gc(p1, 10, descr=unicodelendescr) + setfield_gc(p1, 0, descr=unicodehashdescr) p2 = call_malloc_nursery_varsize(2, %(unicodedescr.itemsize)d, i2,\ descr=unicodedescr) setfield_gc(p2, i2, descr=unicodelendescr) + setfield_gc(p2, 0, descr=unicodehashdescr) p3 = call_malloc_nursery_varsize(1, 1, i2, \ descr=strdescr) setfield_gc(p3, i2, descr=strlendescr) + setfield_gc(p3, 0, descr=strhashdescr) jump() """) @@ -764,6 +771,7 @@ p1 = call_malloc_nursery_varsize(1, 1, i0, \ descr=strdescr) setfield_gc(p1, i0, descr=strlendescr) + setfield_gc(p1, 0, descr=strhashdescr) cond_call_gc_wb(p0, descr=wbdescr) setfield_gc(p0, p1, descr=tzdescr) jump() From noreply at buildbot.pypy.org Tue Sep 23 17:03:15 2014 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 23 Sep 2014 17:03:15 +0200 (CEST) Subject: [pypy-commit] stmgc c8-new-page-handling: tweaks Message-ID: <20140923150315.E74AB1C0410@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c8-new-page-handling Changeset: r1409:4a82e00646d8 Date: 2014-09-23 17:03 +0200 http://bitbucket.org/pypy/stmgc/changeset/4a82e00646d8/ Log: tweaks diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -182,28 +182,36 @@ static void _signal_handler(int sig, siginfo_t *siginfo, void *context) { + int saved_errno = errno; char *addr = siginfo->si_addr; dprintf(("si_addr: %p\n", addr)); - if (addr == NULL || addr < stm_object_pages || addr > stm_object_pages+TOTAL_MEMORY) { - /* actual segfault */ - /* send to GDB (XXX) */ - kill(getpid(), SIGINT); + if (addr == NULL || addr < stm_object_pages || + addr >= stm_object_pages+TOTAL_MEMORY) { + /* actual segfault, unrelated to stmgc */ + fprintf(stderr, "Segmentation fault: accessing %p\n", addr); + abort(); } - /* XXX: should we save 'errno'? */ - int segnum = get_segment_of_linear_address(addr); - OPT_ASSERT(segnum == STM_SEGMENT->segment_num); + if (segnum != STM_SEGMENT->segment_num) { + fprintf(stderr, "Segmentation fault: accessing %p (seg %d) from" + " seg %d\n", addr, STM_SEGMENT->segment_num, segnum); + abort(); + } dprintf(("-> segment: %d\n", segnum)); + char *seg_base = STM_SEGMENT->segment_base; uintptr_t pagenum = ((char*)addr - seg_base) / 4096UL; - /* XXX actual segfault also if the pagenum is out of bounds, - say < END_NURSERY_PAGE. Important for crashing cleanly on - %gs:NULL accesses */ + if (pagenum < END_NURSERY_PAGE) { + fprintf(stderr, "Segmentation fault: accessing %p (seg %d " + "page %lu)\n", addr, segnum, pagenum); + abort(); + } handle_segfault_in_page(pagenum); - return; + errno = saved_errno; + /* now return and retry */ } /* ############# commit log ############# */ @@ -464,6 +472,7 @@ (uintptr_t)obj, (uintptr_t)bk_obj); release_modified_objs_lock(my_segnum); } + /* XXX else... what occurs with bk_obj? */ /* done fiddling with protection and privatization */ release_all_privatization_locks(); From noreply at buildbot.pypy.org Tue Sep 23 18:41:56 2014 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 23 Sep 2014 18:41:56 +0200 (CEST) Subject: [pypy-commit] stmgc c8-new-page-handling: in-progress: break everything :-) The goal is to use backups that are Message-ID: <20140923164156.BC4D11C0410@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c8-new-page-handling Changeset: r1410:61d043d6d6f9 Date: 2014-09-23 18:42 +0200 http://bitbucket.org/pypy/stmgc/changeset/61d043d6d6f9/ Log: in-progress: break everything :-) The goal is to use backups that are longer-lived than one transaction, in order to allow the segfault handler to cleanly reconstruct any missing page in all cases. diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -29,48 +29,6 @@ } -static void _update_obj_from(int from_seg, object_t *obj, uintptr_t only_page) -{ - /* updates 'obj' in our accessible pages from another segment's - page or bk copy. (never touch PROT_NONE memory) - only_page = -1 means update whole obj, only_page=pagenum means only - update memory in page 'pagenum' - */ - /* XXXXXXX: are we sure everything is readable in from_seg??? */ - size_t obj_size; - - dprintf(("_update_obj_from(%d, %p)\n", from_seg, obj)); - assert(get_priv_segment(from_seg)->privatization_lock); - - /* look the obj up in the other segment's modified_old_objects to - get its backup copy: */ - acquire_modified_objs_lock(from_seg); - - wlog_t *item; - struct tree_s *tree = get_priv_segment(from_seg)->modified_old_objects; - TREE_FIND(tree, (uintptr_t)obj, item, goto not_found); - - obj_size = stmcb_size_rounded_up((struct object_s*)item->val); - - memcpy_to_accessible_pages(STM_SEGMENT->segment_num, obj, - (char*)item->val, obj_size, only_page); - - release_modified_objs_lock(from_seg); - return; - - not_found: - /* copy from page directly (obj is unmodified) */ - obj_size = stmcb_size_rounded_up( - (struct object_s*)REAL_ADDRESS(get_segment_base(from_seg), obj)); - - memcpy_to_accessible_pages(STM_SEGMENT->segment_num, obj, - REAL_ADDRESS(get_segment_base(from_seg), obj), - obj_size, only_page); - - release_modified_objs_lock(from_seg); -} - - static void copy_bk_objs_in_page_from(int from_segnum, uintptr_t pagenum) { /* looks at all bk copies of objects overlapping page 'pagenum' and @@ -78,7 +36,8 @@ dprintf(("copy_bk_objs_in_page_from(%d, %lu)\n", from_segnum, pagenum)); acquire_modified_objs_lock(from_segnum); - struct tree_s *tree = get_priv_segment(from_segnum)->modified_old_objects; + abort(); + struct tree_s *tree = NULL; // get_priv_segment(from_segnum)->modified_old_objects; wlog_t *item; TREE_LOOP_FORWARD(tree, item); { object_t *obj = (object_t*)item->addr; @@ -111,15 +70,15 @@ return; while ((cl = cl->next)) { - if ((uintptr_t)cl == -1) + if (cl == (void *)-1) return; OPT_ASSERT(cl->segment_num >= 0 && cl->segment_num < NB_SEGMENTS); object_t *obj; size_t i = 0; - while ((obj = cl->written[i])) { - _update_obj_from(cl->segment_num, obj, pagenum); + while ((obj = cl->written[i].object)) { + abort(); //_update_obj_from(cl->segment_num, obj, pagenum); i++; }; @@ -177,7 +136,7 @@ /* in case page is already newer, validate everything now to have a common revision for all pages */ - _stm_validate(NULL, true); + //_stm_validate(NULL, true); } static void _signal_handler(int sig, siginfo_t *siginfo, void *context) @@ -219,101 +178,111 @@ void _dbg_print_commit_log() { - volatile struct stm_commit_log_entry_s *cl; - cl = (volatile struct stm_commit_log_entry_s *)&commit_log_root; + struct stm_commit_log_entry_s *cl = &commit_log_root; - fprintf(stderr, "root (%p, %d)\n", cl->next, cl->segment_num); + fprintf(stderr, "commit log root (%p, %d)\n", cl->next, cl->segment_num); while ((cl = cl->next)) { - if ((uintptr_t)cl == -1) { - fprintf(stderr, "INEVITABLE\n"); + if (cl == (void *)-1) { + fprintf(stderr, " INEVITABLE\n"); return; } - size_t i = 0; fprintf(stderr, " elem (%p, %d)\n", cl->next, cl->segment_num); - object_t *obj; - while ((obj = cl->written[i])) { - fprintf(stderr, "-> %p\n", obj); - i++; - }; + struct stm_undo_s *undo = cl->written; + struct stm_undo_s *end = undo + cl->written_count; + for (; undo < end; undo++) { + fprintf(stderr, " obj %p, size %d, ofs %lu\n", undo->object, + SLICE_SIZE(undo->slice), SLICE_OFFSET(undo->slice)); + } } } +static void reapply_undo_log(struct stm_undo_s *undo) +{ + /* read the object (or object slice) described by 'undo', and + re-applies it to our current segment. + */ + dprintf(("_update_obj_from_undo(obj=%p, size=%d, ofs=%lu)\n", + undo->object, SLICE_SIZE(undo->slice), SLICE_OFFSET(undo->slice))); -static void _stm_validate(void *free_if_abort, bool locks_acquired) + size_t ofs = SLICE_OFFSET(undo->slice); + size_t size = SLICE_SIZE(undo->slice); + stm_char *slice_start = ((stm_char *)undo->object) + ofs; + stm_char *slice_end = slice_start + size; + + uintptr_t page_start = ((uintptr_t)slice_start) / 4096; + if ((uintptr_t)slice_end <= (page_start + 1) * 4096) { + + /* the object fits inside a single page: fast path */ + if (get_page_status_in(STM_SEGMENT->segment_num, page_start) + == PAGE_NO_ACCESS) { + return; /* ignore the object: it is in a NO_ACCESS page */ + } + + char *src = undo->backup; + char *dst = REAL_ADDRESS(STM_SEGMENT->segment_base, slice_start); + memcpy(dst, src, size); + } + else { + abort(); //XXX + } +} + +static void reset_modified_from_backup_copies(int segment_num); /* forward */ + +static void _stm_validate(void *free_if_abort) { /* go from last known entry in commit log to the most current one and apply all changes done by other transactions. Abort if we read one of the committed objs. */ - if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) { - assert((uintptr_t)STM_PSEGMENT->last_commit_log_entry->next == -1); - if (locks_acquired) - release_all_privatization_locks(); - return; - } - - if (locks_acquired) - assert(all_privatization_locks_acquired()); - else - acquire_all_privatization_locks(); - - volatile struct stm_commit_log_entry_s *cl, *prev_cl; - cl = prev_cl = (volatile struct stm_commit_log_entry_s *) - STM_PSEGMENT->last_commit_log_entry; + struct stm_commit_log_entry_s *cl = STM_PSEGMENT->last_commit_log_entry; + struct stm_commit_log_entry_s *next_cl; + /* Don't check this 'cl'. This entry is already checked */ bool needs_abort = false; - /* Don't check 'cl'. This entry is already checked */ - while ((cl = cl->next)) { - if ((uintptr_t)cl == -1) { + while ((next_cl = cl->next) != NULL) { + if (next_cl == (void *)-1) { /* there is an inevitable transaction running */ - release_all_privatization_locks(); #if STM_TESTS - free(free_if_abort); + if (free_if_abort != (void *)-1) + free(free_if_abort); stm_abort_transaction(); #endif - cl = prev_cl; + abort(); /* XXX non-busy wait here */ _stm_collectable_safe_point(); acquire_all_privatization_locks(); continue; } - prev_cl = cl; + cl = next_cl; - OPT_ASSERT(cl->segment_num >= 0 && cl->segment_num < NB_SEGMENTS); + /*int srcsegnum = cl->segment_num; + OPT_ASSERT(srcsegnum >= 0 && srcsegnum < NB_SEGMENTS);*/ - object_t *obj; - size_t i = 0; - while ((obj = cl->written[i])) { - _update_obj_from(cl->segment_num, obj, -1); + struct stm_undo_s *undo = cl->written; + struct stm_undo_s *end = cl->written + cl->written_count; - if (_stm_was_read(obj)) { - needs_abort = true; + for (; undo < end; undo++) { - /* if we wrote this obj, we need to free its backup and - remove it from modified_old_objects because - we would otherwise overwrite the updated obj on abort */ - acquire_modified_objs_lock(STM_SEGMENT->segment_num); - wlog_t *item; - struct tree_s *tree = STM_PSEGMENT->modified_old_objects; - TREE_FIND(tree, (uintptr_t)obj, item, goto not_found); - - free((void*)item->val); - TREE_FIND_DELETE(tree, item); - - not_found: - /* nothing todo */ - release_modified_objs_lock(STM_SEGMENT->segment_num); + if (_stm_was_read(undo->object)) { + /* first reset all modified objects from the backup + copies as soon as the first conflict is detected; + then we will proceed below to update our segment from + the old (but unmodified) version to the newer version. */ + if (!needs_abort) { + reset_modified_from_backup_copies(STM_SEGMENT->segment_num); + needs_abort = true; + } } - - i++; - }; + reapply_undo_log(undo); + } /* last fully validated entry */ - STM_PSEGMENT->last_commit_log_entry = (struct stm_commit_log_entry_s *)cl; + STM_PSEGMENT->last_commit_log_entry = cl; } - release_all_privatization_locks(); if (needs_abort) { - free(free_if_abort); + if (free_if_abort != (void *)-1) + free(free_if_abort); stm_abort_transaction(); } } @@ -324,69 +293,61 @@ // we don't need the privatization lock, as we are only // reading from modified_old_objs and nobody but us can change it - struct tree_s *tree = STM_PSEGMENT->modified_old_objects; - size_t count = tree_count(tree); - size_t byte_len = sizeof(struct stm_commit_log_entry_s) + (count + 1) * sizeof(object_t*); + struct list_s *list = STM_PSEGMENT->modified_old_objects; + OPT_ASSERT((list_count(list) % 3) == 0); + size_t count = list_count(list) / 3; + size_t byte_len = sizeof(struct stm_commit_log_entry_s) + + count * sizeof(struct stm_undo_s); struct stm_commit_log_entry_s *result = malloc(byte_len); result->next = NULL; result->segment_num = STM_SEGMENT->segment_num; - - int i = 0; - wlog_t *item; - TREE_LOOP_FORWARD(tree, item); { - result->written[i] = (object_t*)item->addr; - i++; - } TREE_LOOP_END; - - OPT_ASSERT(count == i); - result->written[count] = NULL; - + result->written_count = count; + memcpy(result->written, list->items, count * sizeof(struct stm_undo_s)); return result; } -static void _validate_and_add_to_commit_log() +static void _validate_and_attach(struct stm_commit_log_entry_s *new) { - struct stm_commit_log_entry_s *new; - volatile struct stm_commit_log_entry_s **to; + struct stm_commit_log_entry_s *old; + + while (1) { + _stm_validate(/* free_if_abort =*/ new); + + /* try to attach to commit log: */ + old = STM_PSEGMENT->last_commit_log_entry; + if (old->next == NULL && + __sync_bool_compare_and_swap(&old->next, NULL, new)) + break; /* success! */ + } +} + +static void _validate_and_turn_inevitable(void) +{ + _validate_and_attach((struct stm_commit_log_entry_s *)-1); +} + +static void _validate_and_add_to_commit_log(void) +{ + struct stm_commit_log_entry_s *old, *new; new = _create_commit_log_entry(); if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) { - OPT_ASSERT((uintptr_t)STM_PSEGMENT->last_commit_log_entry->next == -1); + old = STM_PSEGMENT->last_commit_log_entry; + OPT_ASSERT(old->next == (void *)-1); - to = &(STM_PSEGMENT->last_commit_log_entry->next); - bool yes = __sync_bool_compare_and_swap(to, (void*)-1, new); + bool yes = __sync_bool_compare_and_swap(&old->next, (void*)-1, new); OPT_ASSERT(yes); - return; } - - /* regular transaction: */ - do { - _stm_validate(new, false); - - /* try attaching to commit log: */ - to = &(STM_PSEGMENT->last_commit_log_entry->next); - } while (!__sync_bool_compare_and_swap(to, NULL, new)); -} - -static void _validate_and_turn_inevitable() -{ - struct stm_commit_log_entry_s *new; - volatile struct stm_commit_log_entry_s **to; - - new = (struct stm_commit_log_entry_s*)-1; - do { - stm_validate(); - - /* try attaching to commit log: */ - to = &(STM_PSEGMENT->last_commit_log_entry->next); - } while (!__sync_bool_compare_and_swap(to, NULL, new)); + else { + _validate_and_attach(new); + } } /* ############# STM ############# */ void stm_validate() { - _stm_validate(NULL, false); + _stm_validate(NULL); } @@ -441,7 +402,7 @@ choosing to make us PRIVATE is harder because then nobody must ever update the shared page in stm_validate() except if it is the sole reader of it. But then we don't actually know which revision the page is at. */ - long i; + int i; for (i = 0; i < NB_SEGMENTS; i++) { if (i == my_segnum) continue; @@ -466,12 +427,13 @@ /* phew, now add the obj to the write-set and register the backup copy. */ /* XXX: possibly slow check; try overflow objs again? */ - if (!tree_contains(STM_PSEGMENT->modified_old_objects, (uintptr_t)obj)) { + abort(); + /*if (!tree_contains(STM_PSEGMENT->modified_old_objects, (uintptr_t)obj)) { acquire_modified_objs_lock(my_segnum); tree_insert(STM_PSEGMENT->modified_old_objects, (uintptr_t)obj, (uintptr_t)bk_obj); release_modified_objs_lock(my_segnum); - } + }*/ /* XXX else... what occurs with bk_obj? */ /* done fiddling with protection and privatization */ @@ -529,7 +491,7 @@ reset_transaction_read_version(); } - assert(tree_is_cleared(STM_PSEGMENT->modified_old_objects)); + assert(list_is_empty(STM_PSEGMENT->modified_old_objects)); assert(list_is_empty(STM_PSEGMENT->objects_pointing_to_nursery)); assert(tree_is_cleared(STM_PSEGMENT->young_outside_nursery)); assert(tree_is_cleared(STM_PSEGMENT->nursery_objects_shadows)); @@ -601,7 +563,8 @@ and clear the tree: */ acquire_modified_objs_lock(STM_SEGMENT->segment_num); - struct tree_s *tree = STM_PSEGMENT->modified_old_objects; + abort(); + struct tree_s *tree = NULL; //XXX STM_PSEGMENT->modified_old_objects; wlog_t *item; TREE_LOOP_FORWARD(tree, item); { object_t *obj = (object_t*)item->addr; @@ -632,7 +595,7 @@ s_mutex_unlock(); } -void reset_modified_from_backup_copies(int segment_num) +static void reset_modified_from_backup_copies(int segment_num) { #pragma push_macro("STM_PSEGMENT") #pragma push_macro("STM_SEGMENT") @@ -641,7 +604,8 @@ acquire_modified_objs_lock(segment_num); struct stm_priv_segment_info_s *pseg = get_priv_segment(segment_num); - struct tree_s *tree = pseg->modified_old_objects; + abort(); + struct tree_s *tree = NULL; //XXX pseg->modified_old_objects; wlog_t *item; TREE_LOOP_FORWARD(tree, item); { object_t *obj = (object_t*)item->addr; diff --git a/c8/stm/core.h b/c8/stm/core.h --- a/c8/stm/core.h +++ b/c8/stm/core.h @@ -51,7 +51,16 @@ struct stm_segment_info_s pub; uint8_t modified_objs_lock; - struct tree_s *modified_old_objects; + + /* All the old objects (older than the current transaction) that + the current transaction attempts to modify. This is used to + track the STM status: these are old objects that where written + to and that will need to be recorded in the commit log. The + list contains three entries for every such object, in the same + format as 'struct stm_undo_s' below. + */ + struct list_s *modified_old_objects; + struct list_s *objects_pointing_to_nursery; struct tree_s *young_outside_nursery; struct tree_s *nursery_objects_shadows; @@ -90,12 +99,24 @@ }; /* Commit Log things */ +struct stm_undo_s { + object_t *object; /* the object that is modified */ + char *backup; /* some backup data (a slice of the original obj) */ + uint64_t slice; /* location and size of this slice (== the whole + object, unless card marking is enabled). The + size is in the lower 2 bytes, and the offset + in the remaining 6 bytes. */ +}; +#define SLICE_OFFSET(slice) ((slice) >> 16) +#define SLICE_SIZE(slice) ((int)((slice) & 0xFFFF)) + struct stm_commit_log_entry_s { - volatile struct stm_commit_log_entry_s *next; + struct stm_commit_log_entry_s *volatile next; int segment_num; - object_t *written[]; /* terminated with a NULL ptr */ + size_t written_count; + struct stm_undo_s written[]; }; -static struct stm_commit_log_entry_s commit_log_root = {NULL, -1}; +static struct stm_commit_log_entry_s commit_log_root = {NULL, -1, 0}; static char *stm_object_pages; @@ -137,7 +158,7 @@ static void abort_data_structures_from_segment_num(int segment_num); static void _signal_handler(int sig, siginfo_t *siginfo, void *context); -static void _stm_validate(void *free_if_abort, bool locks_acquired); +static void _stm_validate(void *free_if_abort); static inline void _duck(void) { /* put a call to _duck() between two instructions that set 0 into diff --git a/c8/stm/misc.c b/c8/stm/misc.c --- a/c8/stm/misc.c +++ b/c8/stm/misc.c @@ -46,8 +46,9 @@ long _stm_count_modified_old_objects(void) { assert(STM_PSEGMENT->modified_old_objects); - assert(tree_count(STM_PSEGMENT->modified_old_objects) < 10000); - return tree_count(STM_PSEGMENT->modified_old_objects); + assert(list_count(STM_PSEGMENT->modified_old_objects) < 30000); + assert((list_count(STM_PSEGMENT->modified_old_objects) % 3) == 0); + return list_count(STM_PSEGMENT->modified_old_objects) / 3; } long _stm_count_objects_pointing_to_nursery(void) @@ -59,8 +60,8 @@ object_t *_stm_enum_modified_old_objects(long index) { - wlog_t* entry = tree_item(STM_PSEGMENT->modified_old_objects, index); - return (object_t*)entry->addr; + return (object_t *)list_item( + STM_PSEGMENT->modified_old_objects, index * 3); } object_t *_stm_enum_objects_pointing_to_nursery(long index) @@ -86,7 +87,7 @@ object_t *_stm_next_last_cl_entry() { if (_last_cl_entry != &commit_log_root) - return _last_cl_entry->written[_last_cl_entry_index++]; + return _last_cl_entry->written[_last_cl_entry_index++].object; return NULL; } #endif diff --git a/c8/stm/setup.c b/c8/stm/setup.c --- a/c8/stm/setup.c +++ b/c8/stm/setup.c @@ -136,7 +136,7 @@ assert(0 <= i && i < 255); /* 255 is WL_VISITED in gcpage.c */ pr->pub.segment_num = i; pr->pub.segment_base = segment_base; - pr->modified_old_objects = tree_create(); + pr->modified_old_objects = list_create(); pr->objects_pointing_to_nursery = list_create(); pr->young_outside_nursery = tree_create(); pr->nursery_objects_shadows = tree_create(); @@ -173,7 +173,7 @@ struct stm_priv_segment_info_s *pr = get_priv_segment(i); assert(list_is_empty(pr->objects_pointing_to_nursery)); list_free(pr->objects_pointing_to_nursery); - tree_free(pr->modified_old_objects); + list_free(pr->modified_old_objects); tree_free(pr->young_outside_nursery); tree_free(pr->nursery_objects_shadows); tree_free(pr->callbacks_on_commit_and_abort[0]); From noreply at buildbot.pypy.org Tue Sep 23 18:50:34 2014 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 23 Sep 2014 18:50:34 +0200 (CEST) Subject: [pypy-commit] stmgc c8-new-page-handling: fix reset_modified_from_backup_copies() Message-ID: <20140923165034.3F1CC1C0410@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c8-new-page-handling Changeset: r1411:f36b2595f248 Date: 2014-09-23 18:50 +0200 http://bitbucket.org/pypy/stmgc/changeset/f36b2595f248/ Log: fix reset_modified_from_backup_copies() diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -601,29 +601,36 @@ #pragma push_macro("STM_SEGMENT") #undef STM_PSEGMENT #undef STM_SEGMENT - acquire_modified_objs_lock(segment_num); + //acquire_modified_objs_lock(segment_num); struct stm_priv_segment_info_s *pseg = get_priv_segment(segment_num); - abort(); - struct tree_s *tree = NULL; //XXX pseg->modified_old_objects; - wlog_t *item; - TREE_LOOP_FORWARD(tree, item); { - object_t *obj = (object_t*)item->addr; - struct object_s* bk_obj = (struct object_s *)item->val; - size_t obj_size; + struct list_s *list = pseg->modified_old_objects; + struct stm_undo_s *undo = (struct stm_undo_s *)list->items; + struct stm_undo_s *end = (struct stm_undo_s *)(list->items + list->count); - obj_size = stmcb_size_rounded_up(bk_obj); + for (; undo < end; undo++) { + object_t *obj = undo->object; + char *dst = REAL_ADDRESS(pseg->pub.segment_base, obj); - memcpy(REAL_ADDRESS(pseg->pub.segment_base, obj), - bk_obj, obj_size); - assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); /* not written */ + memcpy(dst + SLICE_OFFSET(undo->slice), + undo->backup, + SLICE_SIZE(undo->slice)); + free(undo->backup); + } - free(bk_obj); - } TREE_LOOP_END; +#ifndef NDEBUG + /* check that all objects have the GCFLAG_WRITE_BARRIER afterwards */ + undo = (struct stm_undo_s *)list->items; + for (; undo < end; undo++) { + object_t *obj = undo->object; + char *dst = REAL_ADDRESS(pseg->pub.segment_base, obj); + assert(((struct object_s *)dst)->stm_flags & GCFLAG_WRITE_BARRIER); + } +#endif - tree_clear(tree); + list_clear(list); - release_modified_objs_lock(segment_num); + //release_modified_objs_lock(segment_num); #pragma pop_macro("STM_SEGMENT") #pragma pop_macro("STM_PSEGMENT") diff --git a/c8/stm/core.h b/c8/stm/core.h --- a/c8/stm/core.h +++ b/c8/stm/core.h @@ -181,6 +181,7 @@ static inline void acquire_modified_objs_lock(int segnum) { + /* XXX no longer neeeded? */ spinlock_acquire(get_priv_segment(segnum)->modified_objs_lock); } From noreply at buildbot.pypy.org Tue Sep 23 19:03:00 2014 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 23 Sep 2014 19:03:00 +0200 (CEST) Subject: [pypy-commit] stmgc c8-new-page-handling: More tests pass Message-ID: <20140923170300.92B961C10E6@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c8-new-page-handling Changeset: r1412:1aaa7d9300ea Date: 2014-09-23 19:03 +0200 http://bitbucket.org/pypy/stmgc/changeset/1aaa7d9300ea/ Log: More tests pass diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -287,7 +287,7 @@ } } -static struct stm_commit_log_entry_s *_create_commit_log_entry() +static struct stm_commit_log_entry_s *_create_commit_log_entry(void) { /* puts all modified_old_objects in a new commit log entry */ @@ -548,6 +548,19 @@ /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */ } +static void check_all_write_barrier_flags(char *segbase, struct list_s *list) +{ +#ifndef NDEBUG + struct stm_undo_s *undo = (struct stm_undo_s *)list->items; + struct stm_undo_s *end = (struct stm_undo_s *)(list->items + list->count); + for (; undo < end; undo++) { + object_t *obj = undo->object; + char *dst = REAL_ADDRESS(segbase, obj); + assert(((struct object_s *)dst)->stm_flags & GCFLAG_WRITE_BARRIER); + } +#endif +} + void stm_commit_transaction(void) { assert(!_has_mutex()); @@ -557,27 +570,17 @@ dprintf(("> stm_commit_transaction()\n")); minor_collection(1); + /* minor_collection() above should have set again all WRITE_BARRIER flags. + Check that again here for the objects that are about to be copied into + the commit log. */ + check_all_write_barrier_flags(STM_SEGMENT->segment_base, + STM_PSEGMENT->modified_old_objects); + _validate_and_add_to_commit_log(); - /* clear WRITE_BARRIER flags, free all backup copies, - and clear the tree: */ - acquire_modified_objs_lock(STM_SEGMENT->segment_num); - - abort(); - struct tree_s *tree = NULL; //XXX STM_PSEGMENT->modified_old_objects; - wlog_t *item; - TREE_LOOP_FORWARD(tree, item); { - object_t *obj = (object_t*)item->addr; - struct object_s* bk_obj = (struct object_s *)item->val; - free(bk_obj); - obj->stm_flags |= GCFLAG_WRITE_BARRIER; - } TREE_LOOP_END; - tree_clear(tree); - - release_modified_objs_lock(STM_SEGMENT->segment_num); - invoke_and_clear_user_callbacks(0); /* for commit */ + /* XXX do we still need a s_mutex_lock() section here? */ s_mutex_lock(); enter_safe_point_if_requested(); assert(STM_SEGMENT->nursery_end == NURSERY_END); @@ -618,15 +621,8 @@ free(undo->backup); } -#ifndef NDEBUG /* check that all objects have the GCFLAG_WRITE_BARRIER afterwards */ - undo = (struct stm_undo_s *)list->items; - for (; undo < end; undo++) { - object_t *obj = undo->object; - char *dst = REAL_ADDRESS(pseg->pub.segment_base, obj); - assert(((struct object_s *)dst)->stm_flags & GCFLAG_WRITE_BARRIER); - } -#endif + check_all_write_barrier_flags(pseg->pub.segment_base, list); list_clear(list); diff --git a/c8/stm/misc.c b/c8/stm/misc.c --- a/c8/stm/misc.c +++ b/c8/stm/misc.c @@ -70,13 +70,12 @@ STM_PSEGMENT->objects_pointing_to_nursery, index); } -static volatile struct stm_commit_log_entry_s *_last_cl_entry; +static struct stm_commit_log_entry_s *_last_cl_entry; static long _last_cl_entry_index; void _stm_start_enum_last_cl_entry() { _last_cl_entry = &commit_log_root; - volatile struct stm_commit_log_entry_s *cl = (volatile struct stm_commit_log_entry_s *) - &commit_log_root; + struct stm_commit_log_entry_s *cl = &commit_log_root; while ((cl = cl->next)) { _last_cl_entry = cl; @@ -86,8 +85,10 @@ object_t *_stm_next_last_cl_entry() { - if (_last_cl_entry != &commit_log_root) - return _last_cl_entry->written[_last_cl_entry_index++].object; - return NULL; + if (_last_cl_entry == &commit_log_root) + return NULL; + if (_last_cl_entry_index >= _last_cl_entry->written_count) + return NULL; + return _last_cl_entry->written[_last_cl_entry_index++].object; } #endif From noreply at buildbot.pypy.org Tue Sep 23 20:00:50 2014 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 23 Sep 2014 20:00:50 +0200 (CEST) Subject: [pypy-commit] stmgc c8-new-page-handling: in-progress Message-ID: <20140923180050.4EB081D3999@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c8-new-page-handling Changeset: r1413:68c788b03b94 Date: 2014-09-23 20:00 +0200 http://bitbucket.org/pypy/stmgc/changeset/68c788b03b94/ Log: in-progress diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -4,54 +4,28 @@ /* ############# signal handler ############# */ -static void memcpy_to_accessible_pages( - int dst_segnum, object_t *dst_obj, - char *src, size_t len, uintptr_t only_page) -{ - /* XXX: optimize */ - - char *realobj = REAL_ADDRESS(get_segment_base(dst_segnum), dst_obj); - char *dst_end = realobj + len; - uintptr_t loc_addr = (uintptr_t)dst_obj; - - dprintf(("memcpy_to_accessible_pages(%d, %p, %p, %lu, %lu)\n", - dst_segnum, dst_obj, src, len, only_page)); - - while (realobj != dst_end) { - if (get_page_status_in(dst_segnum, loc_addr / 4096UL) != PAGE_NO_ACCESS - && (only_page == -1 || only_page == loc_addr / 4096UL)) { - *realobj = *src; - } - realobj++; - loc_addr++; - src++; - } -} - static void copy_bk_objs_in_page_from(int from_segnum, uintptr_t pagenum) { /* looks at all bk copies of objects overlapping page 'pagenum' and - copies to current segment (never touch PROT_NONE memory). */ + copies the part in 'pagenum' back to the current segment */ dprintf(("copy_bk_objs_in_page_from(%d, %lu)\n", from_segnum, pagenum)); acquire_modified_objs_lock(from_segnum); - abort(); - struct tree_s *tree = NULL; // get_priv_segment(from_segnum)->modified_old_objects; - wlog_t *item; - TREE_LOOP_FORWARD(tree, item); { - object_t *obj = (object_t*)item->addr; - struct object_s* bk_obj = (struct object_s *)item->val; - size_t obj_size = stmcb_size_rounded_up(bk_obj); + struct list_s *list = get_priv_segment(from_segnum)->modified_old_objects; + struct stm_undo_s *undo = (struct stm_undo_s *)list->items; + struct stm_undo_s *end = (struct stm_undo_s *)(list->items + list->count); - if (item->addr < (pagenum + 1) * 4096UL && item->addr + obj_size > pagenum * 4096UL) { - /* XXX: should actually only write to pagenum, but we validate - afterwards anyway and abort in case we had modifications there */ - memcpy_to_accessible_pages(STM_SEGMENT->segment_num, - obj, (char*)bk_obj, obj_size, pagenum); + for (; undo < end; undo++) { + object_t *obj = undo->object; + stm_char *oslice = ((stm_char *)obj) + SLICE_OFFSET(undo->slice); + uintptr_t current_page_num = ((uintptr_t)oslice) / 4096; + + if (current_page_num == pagenum) { + char *dst = REAL_ADDRESS(STM_SEGMENT->segment_base, oslice); + memcpy(dst, undo->backup, SLICE_SIZE(undo->slice)); } - } TREE_LOOP_END; - + } release_modified_objs_lock(from_segnum); } @@ -129,10 +103,12 @@ /* if there were modifications in the page, revert them: */ copy_bk_objs_in_page_from(shared_page_holder, pagenum); - /* if not already newer, update page to our revision */ - update_page_revision_from_to( - pagenum, get_priv_segment(shared_page_holder)->last_commit_log_entry, - STM_PSEGMENT->last_commit_log_entry); + /* Note that we can't really read without careful locking + 'get_priv_segment(shared_page_holder)->last_commit_log_entry'. + Instead, we're just assuming that the current status of the + page is xxxxxxxxxxxxxxx + */ + abort(); /* in case page is already newer, validate everything now to have a common revision for all pages */ @@ -210,21 +186,16 @@ stm_char *slice_end = slice_start + size; uintptr_t page_start = ((uintptr_t)slice_start) / 4096; - if ((uintptr_t)slice_end <= (page_start + 1) * 4096) { + assert((uintptr_t)slice_end <= (page_start + 1) * 4096); - /* the object fits inside a single page: fast path */ - if (get_page_status_in(STM_SEGMENT->segment_num, page_start) + if (get_page_status_in(STM_SEGMENT->segment_num, page_start) == PAGE_NO_ACCESS) { - return; /* ignore the object: it is in a NO_ACCESS page */ - } + return; /* ignore the object: it is in a NO_ACCESS page */ + } - char *src = undo->backup; - char *dst = REAL_ADDRESS(STM_SEGMENT->segment_base, slice_start); - memcpy(dst, src, size); - } - else { - abort(); //XXX - } + char *src = undo->backup; + char *dst = REAL_ADDRESS(STM_SEGMENT->segment_base, slice_start); + memcpy(dst, src, size); } static void reset_modified_from_backup_copies(int segment_num); /* forward */ @@ -342,6 +313,10 @@ else { _validate_and_attach(new); } + + acquire_modified_objs_lock(STM_SEGMENT->segment_num); + list_clear(STM_PSEGMENT->modified_old_objects); + release_modified_objs_lock(STM_SEGMENT->segment_num); } /* ############# STM ############# */ @@ -377,31 +352,38 @@ dprintf(("write_slowpath(%p): sz=%lu, bk=%p\n", obj, obj_size, bk_obj)); retry: /* privatize pages: */ + /* XXX don't always acquire all locks... */ acquire_all_privatization_locks(); uintptr_t page; for (page = first_page; page <= end_page; page++) { /* check if our page is private or we are the only shared-page holder */ - if (get_page_status_in(my_segnum, page) == PAGE_NO_ACCESS) { + switch (get_page_status_in(my_segnum, page)) { + + case PAGE_PRIVATE: + continue; + + case PAGE_NO_ACCESS: /* happens if there is a concurrent WB between us making the backup and acquiring the locks */ release_all_privatization_locks(); volatile char *dummy = REAL_ADDRESS(STM_SEGMENT->segment_base, page * 4096UL); - *dummy = *dummy; /* force segfault */ + *dummy; /* force segfault */ goto retry; + + case PAGE_SHARED: + break; + + default: + assert(0); } - assert(get_page_status_in(my_segnum, page) != PAGE_NO_ACCESS); - - if (get_page_status_in(my_segnum, page) == PAGE_PRIVATE) - continue; - - assert(get_page_status_in(my_segnum, page) == PAGE_SHARED); /* make sure all the others are NO_ACCESS choosing to make us PRIVATE is harder because then nobody must ever update the shared page in stm_validate() except if it is the sole reader of it. But then we don't actually know which revision the page is at. */ + /* XXX this is a temporary solution I suppose */ int i; for (i = 0; i < NB_SEGMENTS; i++) { if (i == my_segnum) @@ -426,15 +408,24 @@ /* phew, now add the obj to the write-set and register the backup copy. */ - /* XXX: possibly slow check; try overflow objs again? */ - abort(); - /*if (!tree_contains(STM_PSEGMENT->modified_old_objects, (uintptr_t)obj)) { - acquire_modified_objs_lock(my_segnum); - tree_insert(STM_PSEGMENT->modified_old_objects, - (uintptr_t)obj, (uintptr_t)bk_obj); - release_modified_objs_lock(my_segnum); - }*/ - /* XXX else... what occurs with bk_obj? */ + /* XXX: we should not be here at all fiddling with page status + if 'obj' is merely an overflow object. FIX ME, likely by copying + the overflow number logic from c7. */ + + assert(first_page == end_page); /* XXX! */ + /* XXX do the case where first_page != end_page in pieces. Maybe also + use mprotect() again to mark pages of the object as read-only, and + only stick it into modified_old_objects page-by-page? Maybe it's + possible to do card-marking that way, too. */ + + uintptr_t slice = obj_size; + assert(SLICE_OFFSET(slice) == 0 && SLICE_SIZE(slice) == obj_size); + + acquire_modified_objs_lock(STM_SEGMENT->segment_num); + STM_PSEGMENT->modified_old_objects = list_append3( + STM_PSEGMENT->modified_old_objects, + (uintptr_t)obj, (uintptr_t)bk_obj, slice); + release_modified_objs_lock(STM_SEGMENT->segment_num); /* done fiddling with protection and privatization */ release_all_privatization_locks(); @@ -604,7 +595,7 @@ #pragma push_macro("STM_SEGMENT") #undef STM_PSEGMENT #undef STM_SEGMENT - //acquire_modified_objs_lock(segment_num); + acquire_modified_objs_lock(segment_num); struct stm_priv_segment_info_s *pseg = get_priv_segment(segment_num); struct list_s *list = pseg->modified_old_objects; @@ -626,7 +617,7 @@ list_clear(list); - //release_modified_objs_lock(segment_num); + release_modified_objs_lock(segment_num); #pragma pop_macro("STM_SEGMENT") #pragma pop_macro("STM_PSEGMENT") diff --git a/c8/stm/core.h b/c8/stm/core.h --- a/c8/stm/core.h +++ b/c8/stm/core.h @@ -102,10 +102,9 @@ struct stm_undo_s { object_t *object; /* the object that is modified */ char *backup; /* some backup data (a slice of the original obj) */ - uint64_t slice; /* location and size of this slice (== the whole - object, unless card marking is enabled). The - size is in the lower 2 bytes, and the offset - in the remaining 6 bytes. */ + uint64_t slice; /* location and size of this slice (cannot cross + pages). The size is in the lower 2 bytes, and + the offset in the remaining 6 bytes. */ }; #define SLICE_OFFSET(slice) ((slice) >> 16) #define SLICE_SIZE(slice) ((int)((slice) & 0xFFFF)) @@ -181,7 +180,6 @@ static inline void acquire_modified_objs_lock(int segnum) { - /* XXX no longer neeeded? */ spinlock_acquire(get_priv_segment(segnum)->modified_objs_lock); } diff --git a/c8/stm/list.h b/c8/stm/list.h --- a/c8/stm/list.h +++ b/c8/stm/list.h @@ -45,6 +45,19 @@ return lst; } +static inline struct list_s *list_append3(struct list_s *lst, uintptr_t item0, + uintptr_t item1, uintptr_t item2) +{ + uintptr_t index = lst->count; + lst->count += 3; + if (UNLIKELY(index + 2 > lst->last_allocated)) + lst = _list_grow(lst, index + 2); + lst->items[index + 0] = item0; + lst->items[index + 1] = item1; + lst->items[index + 2] = item2; + return lst; +} + static inline void list_clear(struct list_s *lst) { From noreply at buildbot.pypy.org Tue Sep 23 20:37:36 2014 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 23 Sep 2014 20:37:36 +0200 (CEST) Subject: [pypy-commit] stmgc c8-new-page-handling: Progress one or two more tests. Message-ID: <20140923183736.4962F1C0410@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c8-new-page-handling Changeset: r1414:d6152847d126 Date: 2014-09-23 20:37 +0200 http://bitbucket.org/pypy/stmgc/changeset/d6152847d126/ Log: Progress one or two more tests. diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -5,17 +5,9 @@ /* ############# signal handler ############# */ -static void copy_bk_objs_in_page_from(int from_segnum, uintptr_t pagenum) +static void _copy_from_undo_log(uintptr_t pagenum, struct stm_undo_s *undo, + struct stm_undo_s *end) { - /* looks at all bk copies of objects overlapping page 'pagenum' and - copies the part in 'pagenum' back to the current segment */ - dprintf(("copy_bk_objs_in_page_from(%d, %lu)\n", from_segnum, pagenum)); - - acquire_modified_objs_lock(from_segnum); - struct list_s *list = get_priv_segment(from_segnum)->modified_old_objects; - struct stm_undo_s *undo = (struct stm_undo_s *)list->items; - struct stm_undo_s *end = (struct stm_undo_s *)(list->items + list->count); - for (; undo < end; undo++) { object_t *obj = undo->object; stm_char *oslice = ((stm_char *)obj) + SLICE_OFFSET(undo->slice); @@ -26,40 +18,57 @@ memcpy(dst, undo->backup, SLICE_SIZE(undo->slice)); } } - release_modified_objs_lock(from_segnum); } -static void update_page_revision_from_to(uintptr_t pagenum, - struct stm_commit_log_entry_s *from, - struct stm_commit_log_entry_s *to) +static void copy_bk_objs_in_page_from(int from_segnum, uintptr_t pagenum) { - /* walk the commit log and update the page 'pagenum' until we reach - the same revision as our segment, or we reach the HEAD. */ + /* looks at all bk copies of objects overlapping page 'pagenum' and + copies the part in 'pagenum' back to the current segment */ + dprintf(("copy_bk_objs_in_page_from(%d, %lu)\n", from_segnum, pagenum)); + + struct list_s *list = get_priv_segment(from_segnum)->modified_old_objects; + struct stm_undo_s *undo = (struct stm_undo_s *)list->items; + struct stm_undo_s *end = (struct stm_undo_s *)(list->items + list->count); + + _copy_from_undo_log(pagenum, undo, end); +} + +static void go_to_the_future(uintptr_t pagenum, + struct stm_commit_log_entry_s *from, + struct stm_commit_log_entry_s *to) +{ + /* walk FORWARDS the commit log and update the page 'pagenum', + initially at revision 'from', until we reach the revision 'to'. */ assert(all_privatization_locks_acquired()); - volatile struct stm_commit_log_entry_s *cl; - cl = (volatile struct stm_commit_log_entry_s *)from; + while (from != to) { + from = from->next; - if (from == to) - return; + struct stm_undo_s *undo = from->written; + struct stm_undo_s *end = from->written + from->written_count; - while ((cl = cl->next)) { - if (cl == (void *)-1) - return; + _copy_from_undo_log(pagenum, undo, end); + } +} - OPT_ASSERT(cl->segment_num >= 0 && cl->segment_num < NB_SEGMENTS); +static void go_to_the_past(uintptr_t pagenum, + struct stm_commit_log_entry_s *from, + struct stm_commit_log_entry_s *to) +{ + /* walk BACKWARDS the commit log and update the page 'pagenum', + initially at revision 'from', until we reach the revision 'to'. */ + assert(all_privatization_locks_acquired()); - object_t *obj; - size_t i = 0; - while ((obj = cl->written[i].object)) { - abort(); //_update_obj_from(cl->segment_num, obj, pagenum); + /* XXXXXXX Recursive algo for now, fix this! */ - i++; - }; + if (from != to) { + struct stm_commit_log_entry_s *cl = from->next; + go_to_the_past(pagenum, cl, to); - /* last fully validated entry */ - if (cl == to) - return; + struct stm_undo_s *undo = cl->written; + struct stm_undo_s *end = cl->written + cl->written_count; + + _copy_from_undo_log(pagenum, undo, end); } } @@ -96,23 +105,48 @@ /* this assert should be true for now... */ assert(shared_ref_count == 1); + /* acquire this lock, so that we know we should get a view + of the page we're about to copy that is consistent with the + backup copies in the other thread's 'modified_old_objects'. */ + acquire_modified_objs_lock(shared_page_holder); + /* make our page private */ page_privatize_in(STM_SEGMENT->segment_num, pagenum); assert(get_page_status_in(my_segnum, pagenum) == PAGE_PRIVATE); - /* if there were modifications in the page, revert them: */ + /* if there were modifications in the page, revert them. */ copy_bk_objs_in_page_from(shared_page_holder, pagenum); - /* Note that we can't really read without careful locking - 'get_priv_segment(shared_page_holder)->last_commit_log_entry'. - Instead, we're just assuming that the current status of the - page is xxxxxxxxxxxxxxx - */ - abort(); + /* we need to go from 'src_version' to 'target_version'. This + might need a walk into the past or the future. */ + struct stm_commit_log_entry_s *src_version, *target_version, *c1, *c2; + src_version = get_priv_segment(shared_page_holder)->last_commit_log_entry; + target_version = STM_PSEGMENT->last_commit_log_entry; - /* in case page is already newer, validate everything now to have a common - revision for all pages */ - //_stm_validate(NULL, true); + c1 = src_version; + c2 = target_version; + while (1) { + if (c1 == target_version) { + go_to_the_future(pagenum, src_version, target_version); + break; + } + if (c2 == src_version) { + go_to_the_past(pagenum, src_version, target_version); + break; + } + c1 = c1->next; + if (c1 == (void *)-1 || c1 == NULL) { + go_to_the_past(pagenum, src_version, target_version); + break; + } + c2 = c2->next; + if (c2 == (void *)-1 || c2 == NULL) { + go_to_the_future(pagenum, src_version, target_version); + break; + } + } + + release_modified_objs_lock(shared_page_holder); } static void _signal_handler(int sig, siginfo_t *siginfo, void *context) @@ -210,6 +244,11 @@ struct stm_commit_log_entry_s *next_cl; /* Don't check this 'cl'. This entry is already checked */ + /* We need this lock to prevent a segfault handler in a different thread + from seeing inconsistent data. It could also be done by carefully + ordering things, but later. */ + acquire_modified_objs_lock(STM_SEGMENT->segment_num); + bool needs_abort = false; while ((next_cl = cl->next) != NULL) { if (next_cl == (void *)-1) { @@ -248,9 +287,12 @@ } /* last fully validated entry */ + STM_PSEGMENT->last_commit_log_entry = cl; } + release_modified_objs_lock(STM_SEGMENT->segment_num); + if (needs_abort) { if (free_if_abort != (void *)-1) free(free_if_abort); @@ -316,6 +358,7 @@ acquire_modified_objs_lock(STM_SEGMENT->segment_num); list_clear(STM_PSEGMENT->modified_old_objects); + STM_PSEGMENT->last_commit_log_entry = new; release_modified_objs_lock(STM_SEGMENT->segment_num); } From noreply at buildbot.pypy.org Tue Sep 23 20:50:45 2014 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 23 Sep 2014 20:50:45 +0200 (CEST) Subject: [pypy-commit] stmgc c8-new-page-handling: lock fixes Message-ID: <20140923185045.721C31C10E6@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c8-new-page-handling Changeset: r1415:5c6e27d8a4f9 Date: 2014-09-23 20:42 +0200 http://bitbucket.org/pypy/stmgc/changeset/5c6e27d8a4f9/ Log: lock fixes diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -147,6 +147,7 @@ } release_modified_objs_lock(shared_page_holder); + release_all_privatization_locks(); } static void _signal_handler(int sig, siginfo_t *siginfo, void *context) @@ -279,7 +280,9 @@ then we will proceed below to update our segment from the old (but unmodified) version to the newer version. */ if (!needs_abort) { + release_modified_objs_lock(STM_SEGMENT->segment_num); reset_modified_from_backup_copies(STM_SEGMENT->segment_num); + acquire_modified_objs_lock(STM_SEGMENT->segment_num); needs_abort = true; } } From noreply at buildbot.pypy.org Tue Sep 23 20:50:46 2014 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 23 Sep 2014 20:50:46 +0200 (CEST) Subject: [pypy-commit] stmgc c8-new-page-handling: Show the content of the undo slices too Message-ID: <20140923185046.AD7FF1C10E6@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c8-new-page-handling Changeset: r1416:9761d0b42cae Date: 2014-09-23 20:50 +0200 http://bitbucket.org/pypy/stmgc/changeset/9761d0b42cae/ Log: Show the content of the undo slices too diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -191,18 +191,22 @@ { struct stm_commit_log_entry_s *cl = &commit_log_root; - fprintf(stderr, "commit log root (%p, %d)\n", cl->next, cl->segment_num); + fprintf(stderr, "commit log:\n"); while ((cl = cl->next)) { if (cl == (void *)-1) { fprintf(stderr, " INEVITABLE\n"); return; } - fprintf(stderr, " elem (%p, %d)\n", cl->next, cl->segment_num); + fprintf(stderr, " entry at %p: seg %d\n", cl, cl->segment_num); struct stm_undo_s *undo = cl->written; struct stm_undo_s *end = undo + cl->written_count; for (; undo < end; undo++) { - fprintf(stderr, " obj %p, size %d, ofs %lu\n", undo->object, + fprintf(stderr, " obj %p, size %d, ofs %lu: ", undo->object, SLICE_SIZE(undo->slice), SLICE_OFFSET(undo->slice)); + long i; + for (i=0; islice); i += 8) + fprintf(stderr, " 0x%016lx", *(long *)(undo->backup + i)); + fprintf(stderr, "\n"); } } } From noreply at buildbot.pypy.org Tue Sep 23 20:54:15 2014 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 23 Sep 2014 20:54:15 +0200 (CEST) Subject: [pypy-commit] stmgc c8-new-page-handling: Confused, but that's enough for tonight Message-ID: <20140923185415.A96381C33E1@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c8-new-page-handling Changeset: r1417:8866dac9e6ab Date: 2014-09-23 20:54 +0200 http://bitbucket.org/pypy/stmgc/changeset/8866dac9e6ab/ Log: Confused, but that's enough for tonight diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -290,6 +290,7 @@ needs_abort = true; } } + /* XXXX Here I'm doing something wrong -- arigo */ reapply_undo_log(undo); } From noreply at buildbot.pypy.org Tue Sep 23 21:15:42 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 23 Sep 2014 21:15:42 +0200 (CEST) Subject: [pypy-commit] pypy default: fix ndarray.swapaxes no-op cases (issue 1872) Message-ID: <20140923191542.52A601C03FC@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r73668:8faa11984f7a Date: 2014-09-23 15:15 -0400 http://bitbucket.org/pypy/pypy/changeset/8faa11984f7a/ Log: fix ndarray.swapaxes no-op cases (issue 1872) diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py --- a/pypy/module/micronumpy/ndarray.py +++ b/pypy/module/micronumpy/ndarray.py @@ -407,7 +407,7 @@ -------- numpy.swapaxes : equivalent function """ - if self.is_scalar(): + if axis1 == axis2 or len(self.get_shape()) <= 1: return self return self.implementation.swapaxes(space, self, axis1, axis2) diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py --- a/pypy/module/micronumpy/test/test_ndarray.py +++ b/pypy/module/micronumpy/test/test_ndarray.py @@ -2020,6 +2020,10 @@ def test_swapaxes(self): from numpypy import array + x = array([]) + assert x.swapaxes(0, 2) is x + x = array([[1, 2]]) + assert x.swapaxes(0, 0) is x # testcases from numpy docstring x = array([[1, 2, 3]]) assert (x.swapaxes(0, 1) == array([[1], [2], [3]])).all() From noreply at buildbot.pypy.org Tue Sep 23 21:29:58 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 23 Sep 2014 21:29:58 +0200 (CEST) Subject: [pypy-commit] pypy default: check values in swapaxes Message-ID: <20140923192958.2B4FA1D2363@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r73669:ae1f561d0a35 Date: 2014-09-23 15:29 -0400 http://bitbucket.org/pypy/pypy/changeset/ae1f561d0a35/ Log: check values in swapaxes diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py --- a/pypy/module/micronumpy/ndarray.py +++ b/pypy/module/micronumpy/ndarray.py @@ -407,8 +407,19 @@ -------- numpy.swapaxes : equivalent function """ - if axis1 == axis2 or len(self.get_shape()) <= 1: + if axis1 == axis2: return self + n = len(self.get_shape()) + if n <= 1: + return self + if axis1 < 0: + axis1 += n + if axis2 < 0: + axis2 += n + if axis1 < 0 or axis1 >= n: + raise oefmt(space.w_ValueError, "bad axis1 argument to swapaxes") + if axis2 < 0 or axis2 >= n: + raise oefmt(space.w_ValueError, "bad axis2 argument to swapaxes") return self.implementation.swapaxes(space, self, axis1, axis2) def descr_nonzero(self, space): diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py --- a/pypy/module/micronumpy/test/test_ndarray.py +++ b/pypy/module/micronumpy/test/test_ndarray.py @@ -2024,6 +2024,10 @@ assert x.swapaxes(0, 2) is x x = array([[1, 2]]) assert x.swapaxes(0, 0) is x + exc = raises(ValueError, x.swapaxes, -3, 0) + assert exc.value.message == "bad axis1 argument to swapaxes" + exc = raises(ValueError, x.swapaxes, 0, 3) + assert exc.value.message == "bad axis2 argument to swapaxes" # testcases from numpy docstring x = array([[1, 2, 3]]) assert (x.swapaxes(0, 1) == array([[1], [2], [3]])).all() From noreply at buildbot.pypy.org Wed Sep 24 10:00:42 2014 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 24 Sep 2014 10:00:42 +0200 (CEST) Subject: [pypy-commit] stmgc c8-new-page-handling: Add comment Message-ID: <20140924080042.D32FD1C33C1@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c8-new-page-handling Changeset: r1418:d041ac365425 Date: 2014-09-24 10:00 +0200 http://bitbucket.org/pypy/stmgc/changeset/d041ac365425/ Log: Add comment diff --git a/c8/stm/core.h b/c8/stm/core.h --- a/c8/stm/core.h +++ b/c8/stm/core.h @@ -109,6 +109,14 @@ #define SLICE_OFFSET(slice) ((slice) >> 16) #define SLICE_SIZE(slice) ((int)((slice) & 0xFFFF)) +/* The model is: we have a global chained list, from 'commit_log_root', + of 'struct stm_commit_log_entry_s' entries. Every one is fully + read-only apart from the 'next' field. Every one stands for one + commit that occurred. It lists the old objects that were modified + in this commit, and their attached "undo logs" --- that is, the + data from 'written[n].backup' is the content of (slices of) the + object as they were *before* that commit occurred. +*/ struct stm_commit_log_entry_s { struct stm_commit_log_entry_s *volatile next; int segment_num; From noreply at buildbot.pypy.org Wed Sep 24 10:12:07 2014 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 24 Sep 2014 10:12:07 +0200 (CEST) Subject: [pypy-commit] pypy shadowstack-again: Close this head Message-ID: <20140924081207.D53021C0250@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: shadowstack-again Changeset: r73670:6c50bbbada5b Date: 2014-09-24 10:11 +0200 http://bitbucket.org/pypy/pypy/changeset/6c50bbbada5b/ Log: Close this head From noreply at buildbot.pypy.org Wed Sep 24 11:33:55 2014 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 24 Sep 2014 11:33:55 +0200 (CEST) Subject: [pypy-commit] stmgc c8-new-page-handling: in-progress Message-ID: <20140924093355.DA3871C03FC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c8-new-page-handling Changeset: r1419:2421dce57e02 Date: 2014-09-24 11:34 +0200 http://bitbucket.org/pypy/stmgc/changeset/2421dce57e02/ Log: in-progress diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -3,23 +3,48 @@ #endif -/* ############# signal handler ############# */ +/* General helper: copies objects into our own segment, from some + source described by a range of 'struct stm_undo_s'. Maybe later + we could specialize this function to avoid the checks in the + inner loop. +*/ +static void import_objects( + int from_segnum, /* or -1: from undo->backup */ + uintptr_t pagenum, /* or -1: "all accessible" */ + struct stm_undo_s *undo, + struct stm_undo_s *end) +{ + char *src_segment_base = (from_segnum >= 0 ? get_segment_base(from_segnum) + : NULL); -static void _copy_from_undo_log(uintptr_t pagenum, struct stm_undo_s *undo, - struct stm_undo_s *end) -{ for (; undo < end; undo++) { object_t *obj = undo->object; stm_char *oslice = ((stm_char *)obj) + SLICE_OFFSET(undo->slice); uintptr_t current_page_num = ((uintptr_t)oslice) / 4096; - if (current_page_num == pagenum) { - char *dst = REAL_ADDRESS(STM_SEGMENT->segment_base, oslice); - memcpy(dst, undo->backup, SLICE_SIZE(undo->slice)); + if (pagenum == -1) { + if (get_page_status_in(STM_SEGMENT->segment_num, + current_page_num) == PAGE_NO_ACCESS) + continue; } + else { + if (current_page_num != pagenum) + continue; + } + + char *src, *dst; + if (src_segment_base != NULL) + src = REAL_ADDRESS(src_segment_base, oslice); + else + src = undo->backup; + dst = REAL_ADDRESS(STM_SEGMENT->segment_base, oslice); + memcpy(dst, src, SLICE_SIZE(undo->slice)); } } + +/* ############# signal handler ############# */ + static void copy_bk_objs_in_page_from(int from_segnum, uintptr_t pagenum) { /* looks at all bk copies of objects overlapping page 'pagenum' and @@ -30,24 +55,27 @@ struct stm_undo_s *undo = (struct stm_undo_s *)list->items; struct stm_undo_s *end = (struct stm_undo_s *)(list->items + list->count); - _copy_from_undo_log(pagenum, undo, end); + import_objects(-1, pagenum, undo, end); } static void go_to_the_future(uintptr_t pagenum, struct stm_commit_log_entry_s *from, struct stm_commit_log_entry_s *to) { + if (from == to) + return; + abort(); // XXX I think it's broken, ignoring the other segment's + // 'modified_old_objects'; but is that reachable anyway? + /* walk FORWARDS the commit log and update the page 'pagenum', initially at revision 'from', until we reach the revision 'to'. */ - assert(all_privatization_locks_acquired()); - while (from != to) { from = from->next; struct stm_undo_s *undo = from->written; struct stm_undo_s *end = from->written + from->written_count; - _copy_from_undo_log(pagenum, undo, end); + import_objects(from->segment_num, pagenum, undo, end); } } @@ -57,10 +85,8 @@ { /* walk BACKWARDS the commit log and update the page 'pagenum', initially at revision 'from', until we reach the revision 'to'. */ - assert(all_privatization_locks_acquired()); /* XXXXXXX Recursive algo for now, fix this! */ - if (from != to) { struct stm_commit_log_entry_s *cl = from->next; go_to_the_past(pagenum, cl, to); @@ -68,7 +94,7 @@ struct stm_undo_s *undo = cl->written; struct stm_undo_s *end = cl->written + cl->written_count; - _copy_from_undo_log(pagenum, undo, end); + import_objects(-1, pagenum, undo, end); } } @@ -123,6 +149,9 @@ src_version = get_priv_segment(shared_page_holder)->last_commit_log_entry; target_version = STM_PSEGMENT->last_commit_log_entry; + release_modified_objs_lock(shared_page_holder); + release_all_privatization_locks(); + c1 = src_version; c2 = target_version; while (1) { @@ -145,9 +174,6 @@ break; } } - - release_modified_objs_lock(shared_page_holder); - release_all_privatization_locks(); } static void _signal_handler(int sig, siginfo_t *siginfo, void *context) @@ -211,32 +237,6 @@ } } -static void reapply_undo_log(struct stm_undo_s *undo) -{ - /* read the object (or object slice) described by 'undo', and - re-applies it to our current segment. - */ - dprintf(("_update_obj_from_undo(obj=%p, size=%d, ofs=%lu)\n", - undo->object, SLICE_SIZE(undo->slice), SLICE_OFFSET(undo->slice))); - - size_t ofs = SLICE_OFFSET(undo->slice); - size_t size = SLICE_SIZE(undo->slice); - stm_char *slice_start = ((stm_char *)undo->object) + ofs; - stm_char *slice_end = slice_start + size; - - uintptr_t page_start = ((uintptr_t)slice_start) / 4096; - assert((uintptr_t)slice_end <= (page_start + 1) * 4096); - - if (get_page_status_in(STM_SEGMENT->segment_num, page_start) - == PAGE_NO_ACCESS) { - return; /* ignore the object: it is in a NO_ACCESS page */ - } - - char *src = undo->backup; - char *dst = REAL_ADDRESS(STM_SEGMENT->segment_base, slice_start); - memcpy(dst, src, size); -} - static void reset_modified_from_backup_copies(int segment_num); /* forward */ static void _stm_validate(void *free_if_abort) @@ -255,6 +255,7 @@ acquire_modified_objs_lock(STM_SEGMENT->segment_num); bool needs_abort = false; + uint64_t segment_copied_from = 0; while ((next_cl = cl->next) != NULL) { if (next_cl == (void *)-1) { /* there is an inevitable transaction running */ @@ -263,7 +264,10 @@ free(free_if_abort); stm_abort_transaction(); #endif - abort(); /* XXX non-busy wait here */ + abort(); /* XXX non-busy wait here + or: XXX do we always need to wait? we should just + break out of this loop and let the caller handle it + if it wants to */ _stm_collectable_safe_point(); acquire_all_privatization_locks(); continue; @@ -273,27 +277,31 @@ /*int srcsegnum = cl->segment_num; OPT_ASSERT(srcsegnum >= 0 && srcsegnum < NB_SEGMENTS);*/ - struct stm_undo_s *undo = cl->written; - struct stm_undo_s *end = cl->written + cl->written_count; - - for (; undo < end; undo++) { - - if (_stm_was_read(undo->object)) { - /* first reset all modified objects from the backup - copies as soon as the first conflict is detected; - then we will proceed below to update our segment from - the old (but unmodified) version to the newer version. */ - if (!needs_abort) { + if (!needs_abort) { + struct stm_undo_s *undo = cl->written; + struct stm_undo_s *end = cl->written + cl->written_count; + for (; undo < end; undo++) { + if (_stm_was_read(undo->object)) { + /* first reset all modified objects from the backup + copies as soon as the first conflict is detected; + then we will proceed below to update our segment from + the old (but unmodified) version to the newer version. + */ release_modified_objs_lock(STM_SEGMENT->segment_num); reset_modified_from_backup_copies(STM_SEGMENT->segment_num); acquire_modified_objs_lock(STM_SEGMENT->segment_num); needs_abort = true; + break; } } - /* XXXX Here I'm doing something wrong -- arigo */ - reapply_undo_log(undo); } + struct stm_undo_s *undo = cl->written; + struct stm_undo_s *end = cl->written + cl->written_count; + + segment_copied_from |= (1UL << cl->segment_num); + import_objects(cl->segment_num, -1, undo, end); + /* last fully validated entry */ STM_PSEGMENT->last_commit_log_entry = cl; @@ -301,6 +309,15 @@ release_modified_objs_lock(STM_SEGMENT->segment_num); + /* XXXXXXXXXXXXXXXX CORRECT LOCKING NEEDED XXXXXXXXXXXXXXXXXXXXXX */ + int segnum; + for (segnum = 0; segment_copied_from != 0; segnum++) { + if (segment_copied_from & (1UL << segnum)) { + segment_copied_from &= ~(1UL << segnum); + copy_bk_objs_in_page_from(segnum, -1); + } + } + if (needs_abort) { if (free_if_abort != (void *)-1) free(free_if_abort); From noreply at buildbot.pypy.org Wed Sep 24 12:45:27 2014 From: noreply at buildbot.pypy.org (mattip) Date: Wed, 24 Sep 2014 12:45:27 +0200 (CEST) Subject: [pypy-commit] benchmarks default: fix for win32 Message-ID: <20140924104527.C24411C3334@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r276:bdd2f0fc83b3 Date: 2014-09-23 21:28 +0300 http://bitbucket.org/pypy/benchmarks/changeset/bdd2f0fc83b3/ Log: fix for win32 diff --git a/benchmarks.py b/benchmarks.py --- a/benchmarks.py +++ b/benchmarks.py @@ -91,7 +91,7 @@ else: iteration_scaling = 1.0 _register_new_bm_twisted(name, 'twisted_' + name, - globals(), bm_env={'PYTHONPATH': ':'.join(TWISTED)}, + globals(), bm_env={'PYTHONPATH': os.pathsep.join(TWISTED)}, iteration_scaling=iteration_scaling) _register_new_bm('spitfire', 'spitfire', globals(), diff --git a/own/pyflate-fast.py b/own/pyflate-fast.py --- a/own/pyflate-fast.py +++ b/own/pyflate-fast.py @@ -645,7 +645,7 @@ def _main(): filename = os.path.join(os.path.dirname(__file__), "interpreter.tar.bz2") - input = open(filename) + input = open(filename, 'rb') field = RBitfield(input) magic = field.readbits(16) diff --git a/unladen_swallow/perf.py b/unladen_swallow/perf.py --- a/unladen_swallow/perf.py +++ b/unladen_swallow/perf.py @@ -969,7 +969,7 @@ def MeasureRietveld(python, options): - PYTHONPATH = ":".join([DJANGO_DIR, + PYTHONPATH = os.pathsep.join([DJANGO_DIR, # These paths are lifted from # lib/google_appengine.appcfg.py. Note that we use # our own version of Django instead of Appengine's. @@ -1406,7 +1406,7 @@ time it took to run the benchmark loop once; mem_usage is a list of memory usage samples in kilobytes. """ - pypath = ":".join([Relative("lib/spambayes"), Relative("lib/lockfile")]) + pypath = os.pathsep.join([Relative("lib/spambayes"), Relative("lib/lockfile")]) bm_path = Relative("performance/bm_spambayes.py") bm_env = {"PYTHONPATH": pypath} return MeasureGeneric(python, options, bm_path, bm_env) From noreply at buildbot.pypy.org Wed Sep 24 14:12:46 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Wed, 24 Sep 2014 14:12:46 +0200 (CEST) Subject: [pypy-commit] stmgc c8-new-page-handling: additional tests Message-ID: <20140924121246.DFDB81C0250@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: c8-new-page-handling Changeset: r1420:1f40964edc8a Date: 2014-09-24 14:12 +0200 http://bitbucket.org/pypy/stmgc/changeset/1f40964edc8a/ Log: additional tests diff --git a/c8/test/test_basic.py b/c8/test/test_basic.py --- a/c8/test/test_basic.py +++ b/c8/test/test_basic.py @@ -110,7 +110,8 @@ # py.test.raises(Conflict, self.switch, 0) # detects rw conflict - def test_read_write_11(self): + @py.test.mark.parametrize("only_bk", [0, 1]) + def test_read_write_11(self, only_bk): # test that stm_validate() and the SEGV-handler # always ensure up-to-date views of pages: lp1 = stm_allocate_old(16) @@ -135,6 +136,10 @@ self.commit_transaction() assert last_commit_log_entries() == [lp1] # commit 'x' # + if only_bk: + self.start_transaction() + stm_set_char(lp1, 'y') # 'x' is only in bk_copy + # # self.switch(2) self.start_transaction() # stm_validate() @@ -145,8 +150,8 @@ # is out-of-date because seg1 committed 'x' # (seg1 hasn't done stm_validate() since) - - def test_read_write_12(self): + @py.test.mark.parametrize("only_bk", [0, 1]) + def test_read_write_12(self, only_bk): # test that stm_validate() and the SEGV-handler # always ensure up-to-date views of pages: lp1 = stm_allocate_old(16) @@ -167,6 +172,10 @@ assert last_commit_log_entries() == [lp1] # '1' is committed # + if only_bk: + self.start_transaction() + stm_set_char(lp1, 'y') # '1' is only in bk_copy + # self.switch(2) self.start_transaction() # stm_validate() res = stm_get_char(lp1) # should be '1' @@ -177,6 +186,45 @@ # therefore is still outdated. py.test.raises(Conflict, self.switch, 0) + @py.test.mark.parametrize("only_bk", [0, 1]) + def test_read_write_13(self, only_bk): + # test that stm_validate() and the SEGV-handler + # always ensure up-to-date views of pages: + lp1 = stm_allocate_old(16) + stm_get_real_address(lp1)[HDR] = 'a' #setchar + # + self.start_transaction() + stm_set_char(lp1, '0') # shared->private + # prot_none in seg: 1,2,3 + # + self.switch(1) + self.start_transaction() + stm_set_char(lp1, '1') + self.switch(2) + self.start_transaction() + # prot_none in seg: 2,3 + # + self.switch(0) + self.commit_transaction() + assert last_commit_log_entries() == [lp1] # commit '0' + # + py.test.raises(Conflict, self.switch, 1) + self.start_transaction() # updates to '0' + stm_set_char(lp1, 'x') + self.commit_transaction() + assert last_commit_log_entries() == [lp1] # commit 'x' + # + if only_bk: + self.start_transaction() + stm_set_char(lp1, 'y') # 'x' is only in bk_copy + # + # + self.switch(2, validate=False) # NO stm_validate + res = stm_get_char(lp1) # SEGV -> should not validate and go back in time -> 'a' + py.test.raises(Conflict, self.commit_transaction) # 'a' is outdated, fail to commit + assert res == 'a' + + def test_commit_fresh_objects(self): self.start_transaction() lp = stm_allocate(16) From noreply at buildbot.pypy.org Wed Sep 24 14:21:24 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Wed, 24 Sep 2014 14:21:24 +0200 (CEST) Subject: [pypy-commit] stmgc c8-new-page-handling: probably fix go_to_the_past Message-ID: <20140924122124.E09DD1C03FC@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: c8-new-page-handling Changeset: r1421:a7f16e60a0c6 Date: 2014-09-24 14:21 +0200 http://bitbucket.org/pypy/stmgc/changeset/a7f16e60a0c6/ Log: probably fix go_to_the_past diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -88,8 +88,8 @@ /* XXXXXXX Recursive algo for now, fix this! */ if (from != to) { - struct stm_commit_log_entry_s *cl = from->next; - go_to_the_past(pagenum, cl, to); + struct stm_commit_log_entry_s *cl = to->next; + go_to_the_past(pagenum, from, cl); struct stm_undo_s *undo = cl->written; struct stm_undo_s *end = cl->written + cl->written_count; From noreply at buildbot.pypy.org Wed Sep 24 14:51:43 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Wed, 24 Sep 2014 14:51:43 +0200 (CEST) Subject: [pypy-commit] stmgc c8-new-page-handling: fix Message-ID: <20140924125143.7FB371C0250@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: c8-new-page-handling Changeset: r1422:4e346ae8add2 Date: 2014-09-24 14:51 +0200 http://bitbucket.org/pypy/stmgc/changeset/4e346ae8add2/ Log: fix diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -64,8 +64,6 @@ { if (from == to) return; - abort(); // XXX I think it's broken, ignoring the other segment's - // 'modified_old_objects'; but is that reachable anyway? /* walk FORWARDS the commit log and update the page 'pagenum', initially at revision 'from', until we reach the revision 'to'. */ @@ -77,6 +75,8 @@ import_objects(from->segment_num, pagenum, undo, end); } + + copy_bk_objs_in_page_from(to->segment_num, pagenum); } static void go_to_the_past(uintptr_t pagenum, diff --git a/c8/test/test_basic.py b/c8/test/test_basic.py --- a/c8/test/test_basic.py +++ b/c8/test/test_basic.py @@ -224,6 +224,35 @@ py.test.raises(Conflict, self.commit_transaction) # 'a' is outdated, fail to commit assert res == 'a' + @py.test.mark.parametrize("only_bk", [0, 1]) + def test_read_write_14(self, only_bk): + lp1 = stm_allocate_old(16) # allocated in seg0 + stm_get_real_address(lp1)[HDR] = 'a' #setchar + # S|P|P|P|P + # + # NO_ACCESS in all segments except seg0 (shared page holder) + # + # + self.switch(2) + self.start_transaction() # stm_validate() + # + self.switch(1) # with private page + self.start_transaction() + stm_set_char(lp1, 'C') + self.commit_transaction() + assert last_commit_log_entries() == [lp1] # commit 'C' + # + if only_bk: + self.start_transaction() + stm_set_char(lp1, 'y') # '1' is only in bk_copy + # + # + self.switch(2, validate=False) + res = stm_get_char(lp1) # should be 'a' + py.test.raises(Conflict, self.commit_transaction) + assert res == 'a' + + def test_commit_fresh_objects(self): self.start_transaction() From noreply at buildbot.pypy.org Wed Sep 24 15:26:23 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Wed, 24 Sep 2014 15:26:23 +0200 (CEST) Subject: [pypy-commit] stmgc c8-new-page-handling: test and fix Message-ID: <20140924132623.B16031C03FC@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: c8-new-page-handling Changeset: r1423:8748c8466dde Date: 2014-09-24 15:26 +0200 http://bitbucket.org/pypy/stmgc/changeset/8748c8466dde/ Log: test and fix diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -74,9 +74,8 @@ struct stm_undo_s *end = from->written + from->written_count; import_objects(from->segment_num, pagenum, undo, end); + copy_bk_objs_in_page_from(from->segment_num, pagenum); } - - copy_bk_objs_in_page_from(to->segment_num, pagenum); } static void go_to_the_past(uintptr_t pagenum, diff --git a/c8/test/test_basic.py b/c8/test/test_basic.py --- a/c8/test/test_basic.py +++ b/c8/test/test_basic.py @@ -252,6 +252,41 @@ py.test.raises(Conflict, self.commit_transaction) assert res == 'a' + def test_read_write_15(self): + lp1 = stm_allocate_old(16) # allocated in seg0 + lp2 = stm_allocate_old(16) # allocated in seg0 + stm_get_real_address(lp1)[HDR] = 'a' #setchar + stm_get_real_address(lp2)[HDR] = 'b' #setchar + # S|P|P|P|P + # + # NO_ACCESS in all segments except seg0 (shared page holder) + # + # all seg at R0 + # + self.start_transaction() + # + self.switch(1) # with private page + self.start_transaction() + stm_set_char(lp2, 'C') + self.commit_transaction() # R1 + assert last_commit_log_entries() == [lp2] # commit 'C' + self.start_transaction() + stm_set_char(lp2, 'c') # R1.1 + # + self.switch(2) + self.start_transaction() + stm_set_char(lp1, 'D') + self.commit_transaction() # R2 + assert last_commit_log_entries() == [lp1] # commit 'D' + self.start_transaction() + stm_set_char(lp1, 'd') # R2.1 + # + self.switch(3) + self.start_transaction() # stm_validate() -> R2 + assert stm_get_char(lp1) == 'D' + assert stm_get_char(lp2) == 'C' + self.commit_transaction() + # def test_commit_fresh_objects(self): From noreply at buildbot.pypy.org Wed Sep 24 15:32:13 2014 From: noreply at buildbot.pypy.org (mattip) Date: Wed, 24 Sep 2014 15:32:13 +0200 (CEST) Subject: [pypy-commit] pypy win32-fixes5: force 4 byte alignment on windows Message-ID: <20140924133213.897D31C0DCA@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: win32-fixes5 Changeset: r73671:b878317fc5f0 Date: 2014-09-24 16:30 +0300 http://bitbucket.org/pypy/pypy/changeset/b878317fc5f0/ Log: force 4 byte alignment on windows diff --git a/rpython/rtyper/tool/rffi_platform.py b/rpython/rtyper/tool/rffi_platform.py --- a/rpython/rtyper/tool/rffi_platform.py +++ b/rpython/rtyper/tool/rffi_platform.py @@ -107,15 +107,18 @@ fields is properly aligned.""" global _memory_alignment if _memory_alignment is None: - S = getstruct('struct memory_alignment_test', """ - struct memory_alignment_test { - double d; - void* p; - }; - """, []) - result = S._hints['align'] - assert result & (result-1) == 0, "not a power of two??" - _memory_alignment = result + if sys.platform == 'win32': + _memory_alignment = 4 + else: + S = getstruct('struct memory_alignment_test', """ + struct memory_alignment_test { + double d; + void* p; + }; + """, []) + result = S._hints['align'] + assert result & (result-1) == 0, "not a power of two??" + _memory_alignment = result return _memory_alignment _memory_alignment = None From noreply at buildbot.pypy.org Wed Sep 24 15:32:16 2014 From: noreply at buildbot.pypy.org (mattip) Date: Wed, 24 Sep 2014 15:32:16 +0200 (CEST) Subject: [pypy-commit] pypy win32-fixes5: merge default into branch Message-ID: <20140924133216.9310C1C0DCA@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: win32-fixes5 Changeset: r73672:6a05a50e294b Date: 2014-09-24 16:31 +0300 http://bitbucket.org/pypy/pypy/changeset/6a05a50e294b/ Log: merge default into branch diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py --- a/lib_pypy/_curses.py +++ b/lib_pypy/_curses.py @@ -286,6 +286,13 @@ lib = ffi.verify(""" +#ifdef __APPLE__ +/* the following define is necessary for OS X 10.6+; without it, the + Apple-supplied ncurses.h sets NCURSES_OPAQUE to 1, and then Python + can't get at the WINDOW flags field. */ +#define NCURSES_OPAQUE 0 +#endif + #include #include #include diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py --- a/lib_pypy/datetime.py +++ b/lib_pypy/datetime.py @@ -1242,7 +1242,7 @@ (other._hour, other._minute, other._second, other._microsecond)) if myoff is None or otoff is None: - raise TypeError("cannot compare naive and aware times") + raise TypeError("can't compare offset-naive and offset-aware times") myhhmm = self._hour * 60 + self._minute - myoff othhmm = other._hour * 60 + other._minute - otoff return _cmp((myhhmm, self._second, self._microsecond), @@ -1838,7 +1838,7 @@ other._hour, other._minute, other._second, other._microsecond)) if myoff is None or otoff is None: - raise TypeError("cannot compare naive and aware datetimes") + raise TypeError("can't compare offset-naive and offset-aware datetimes") # XXX What follows could be done more efficiently... diff = self - other # this will take offsets into account if diff.days < 0: @@ -1885,7 +1885,7 @@ if myoff == otoff: return base if myoff is None or otoff is None: - raise TypeError("cannot mix naive and timezone-aware time") + raise TypeError("can't subtract offset-naive and offset-aware datetimes") return base + timedelta(minutes = otoff-myoff) def __hash__(self): diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst --- a/pypy/doc/how-to-release.rst +++ b/pypy/doc/how-to-release.rst @@ -38,14 +38,16 @@ no JIT: windows, linux, os/x sandbox: linux, os/x +* repackage and upload source tar.bz2 to bitbucket and to cobra, as some packagers + prefer a clearly labeled source package * write release announcement pypy/doc/release-x.y(.z).txt the release announcement should contain a direct link to the download page * update pypy.org (under extradoc/pypy.org), rebuild and commit * post announcement on morepypy.blogspot.com -* send announcements to pypy-dev, python-list, +* send announcements to twitter.com, pypy-dev, python-list, python-announce, python-dev ... * add a tag on the pypy/jitviewer repo that corresponds to pypy release * add a tag on the codespeed web site that corresponds to pypy release - +* revise versioning at https://readthedocs.org/projects/pypy diff --git a/pypy/doc/release-2.4.0.rst b/pypy/doc/release-2.4.0.rst --- a/pypy/doc/release-2.4.0.rst +++ b/pypy/doc/release-2.4.0.rst @@ -5,7 +5,7 @@ We're pleased to announce PyPy 2.4, which contains significant performance enhancements and bug fixes. -You can already download the PyPy 2.4-beta1 pre-release here: +You can download the PyPy 2.4.0 release here: http://pypy.org/download.html @@ -63,6 +63,8 @@ PyPy now uses Python 2.7.8 standard library. +We fixed a memory leak in IO in the sandbox_ code + We welcomed more than 12 new contributors, and conducted two Google Summer of Code projects, as well as other student projects not directly related to Summer of Code. @@ -103,8 +105,9 @@ * Many issues were resolved_ since the 2.3.1 release on June 8 -.. _`whats-new`: http://doc.pypy.org/en/latest/whatsnew-2.3.1.html +.. _`whats-new`: http://doc.pypy.org/en/latest/whatsnew-2.4.0.html .. _resolved: https://bitbucket.org/pypy/pypy/issues?status=resolved +.. _sandbox: http://doc.pypy.org/en/latest/sandbox.html We have further improvements on the way: rpython file handling, numpy linalg compatibility, as well diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py --- a/pypy/interpreter/pycode.py +++ b/pypy/interpreter/pycode.py @@ -38,18 +38,15 @@ def cpython_code_signature(code): "([list-of-arg-names], vararg-name-or-None, kwarg-name-or-None)." argcount = code.co_argcount + varnames = code.co_varnames assert argcount >= 0 # annotator hint - argnames = list(code.co_varnames[:argcount]) + argnames = list(varnames[:argcount]) if code.co_flags & CO_VARARGS: - varargname = code.co_varnames[argcount] + varargname = varnames[argcount] argcount += 1 else: varargname = None - if code.co_flags & CO_VARKEYWORDS: - kwargname = code.co_varnames[argcount] - argcount += 1 - else: - kwargname = None + kwargname = varnames[argcount] if code.co_flags & CO_VARKEYWORDS else None return Signature(argnames, varargname, kwargname) diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py --- a/pypy/interpreter/pyparser/parsestring.py +++ b/pypy/interpreter/pyparser/parsestring.py @@ -83,12 +83,6 @@ v = PyString_DecodeEscape(space, substr, 'strict', enc) return space.wrap(v) -def hexbyte(val): - result = "%x" % val - if len(result) == 1: - result = "0" + result - return result - def decode_unicode_utf8(space, s, ps, q): # ****The Python 2.7 version, producing UTF-32 escapes**** # String is utf8-encoded, but 'unicode_escape' expects @@ -108,15 +102,14 @@ # instead. lis.append("u005c") if ord(s[ps]) & 0x80: # XXX inefficient - w, ps = decode_utf8(space, s, ps, end, "utf-32-be") - rn = len(w) - assert rn % 4 == 0 - for i in range(0, rn, 4): - lis.append('\\U') - lis.append(hexbyte(ord(w[i]))) - lis.append(hexbyte(ord(w[i+1]))) - lis.append(hexbyte(ord(w[i+2]))) - lis.append(hexbyte(ord(w[i+3]))) + w, ps = decode_utf8(space, s, ps, end) + for c in w: + # The equivalent of %08x, which is not supported by RPython. + # 7 zeroes are enough for the unicode range, and the + # result still fits in 32-bit. + hexa = hex(ord(c) + 0x10000000) + lis.append('\\U0') + lis.append(hexa[3:]) # Skip 0x and the leading 1 else: lis.append(s[ps]) ps += 1 @@ -136,7 +129,7 @@ # note that the C code has a label here. # the logic is the same. if recode_encoding and ord(s[ps]) & 0x80: - w, ps = decode_utf8(space, s, ps, end, recode_encoding) + w, ps = decode_utf8_recode(space, s, ps, end, recode_encoding) # Append bytes to output buffer. builder.append(w) else: @@ -222,14 +215,18 @@ ch >= 'A' and ch <= 'F') -def decode_utf8(space, s, ps, end, encoding): +def decode_utf8(space, s, ps, end): assert ps >= 0 pt = ps # while (s < end && *s != '\\') s++; */ /* inefficient for u".." while ps < end and ord(s[ps]) & 0x80: ps += 1 - w_u = space.wrap(unicodehelper.decode_utf8(space, s[pt:ps])) - w_v = unicodehelper.encode(space, w_u, encoding) + u = unicodehelper.decode_utf8(space, s[pt:ps]) + return u, ps + +def decode_utf8_recode(space, s, ps, end, recode_encoding): + u, ps = decode_utf8(space, s, ps, end) + w_v = unicodehelper.encode(space, space.wrap(u), recode_encoding) v = space.str_w(w_v) return v, ps diff --git a/pypy/interpreter/pyparser/test/test_parsestring.py b/pypy/interpreter/pyparser/test/test_parsestring.py --- a/pypy/interpreter/pyparser/test/test_parsestring.py +++ b/pypy/interpreter/pyparser/test/test_parsestring.py @@ -73,11 +73,11 @@ def test_simple_enc_roundtrip(self): space = self.space - s = "'\x81'" + s = "'\x81\\t'" s = s.decode("koi8-u").encode("utf8") w_ret = parsestring.parsestr(self.space, 'koi8-u', s) ret = space.unwrap(w_ret) - assert ret == eval("# -*- coding: koi8-u -*-\n'\x81'") + assert ret == eval("# -*- coding: koi8-u -*-\n'\x81\\t'") def test_multiline_unicode_strings_with_backslash(self): space = self.space diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py --- a/pypy/interpreter/unicodehelper.py +++ b/pypy/interpreter/unicodehelper.py @@ -5,6 +5,7 @@ @specialize.memo() def decode_error_handler(space): + # Fast version of the "strict" errors handler. def raise_unicode_exception_decode(errors, encoding, msg, s, startingpos, endingpos): raise OperationError(space.w_UnicodeDecodeError, @@ -17,6 +18,7 @@ @specialize.memo() def encode_error_handler(space): + # Fast version of the "strict" errors handler. def raise_unicode_exception_encode(errors, encoding, msg, u, startingpos, endingpos): raise OperationError(space.w_UnicodeEncodeError, diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py --- a/pypy/module/_cffi_backend/ctypefunc.py +++ b/pypy/module/_cffi_backend/ctypefunc.py @@ -8,6 +8,7 @@ from rpython.rlib.jit_libffi import (CIF_DESCRIPTION, CIF_DESCRIPTION_P, FFI_TYPE, FFI_TYPE_P, FFI_TYPE_PP, SIZE_OF_FFI_ARG) from rpython.rlib.objectmodel import we_are_translated, instantiate +from rpython.rlib.objectmodel import keepalive_until_here from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from pypy.interpreter.error import OperationError, oefmt @@ -160,6 +161,7 @@ raw_cdata = rffi.cast(rffi.CCHARPP, data)[0] lltype.free(raw_cdata, flavor='raw') lltype.free(buffer, flavor='raw') + keepalive_until_here(args_w) return w_res def get_mustfree_flag(data): diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py --- a/pypy/module/micronumpy/ndarray.py +++ b/pypy/module/micronumpy/ndarray.py @@ -407,8 +407,19 @@ -------- numpy.swapaxes : equivalent function """ - if self.is_scalar(): + if axis1 == axis2: return self + n = len(self.get_shape()) + if n <= 1: + return self + if axis1 < 0: + axis1 += n + if axis2 < 0: + axis2 += n + if axis1 < 0 or axis1 >= n: + raise oefmt(space.w_ValueError, "bad axis1 argument to swapaxes") + if axis2 < 0 or axis2 >= n: + raise oefmt(space.w_ValueError, "bad axis2 argument to swapaxes") return self.implementation.swapaxes(space, self, axis1, axis2) def descr_nonzero(self, space): diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py --- a/pypy/module/micronumpy/test/test_ndarray.py +++ b/pypy/module/micronumpy/test/test_ndarray.py @@ -2020,6 +2020,14 @@ def test_swapaxes(self): from numpypy import array + x = array([]) + assert x.swapaxes(0, 2) is x + x = array([[1, 2]]) + assert x.swapaxes(0, 0) is x + exc = raises(ValueError, x.swapaxes, -3, 0) + assert exc.value.message == "bad axis1 argument to swapaxes" + exc = raises(ValueError, x.swapaxes, 0, 3) + assert exc.value.message == "bad axis2 argument to swapaxes" # testcases from numpy docstring x = array([[1, 2, 3]]) assert (x.swapaxes(0, 1) == array([[1], [2], [3]])).all() diff --git a/pypy/module/operator/app_operator.py b/pypy/module/operator/app_operator.py --- a/pypy/module/operator/app_operator.py +++ b/pypy/module/operator/app_operator.py @@ -4,7 +4,7 @@ This module exports a set of operators as functions. E.g. operator.add(x,y) is equivalent to x+y. ''' -from __pypy__ import builtinify + import types @@ -27,7 +27,7 @@ 'getslice(a, b, c) -- Same as a[b:c].' if not isinstance(start, int) or not isinstance(end, int): raise TypeError("an integer is expected") - return a[start:end] + return a[start:end] __getslice__ = getslice def indexOf(a, b): @@ -37,7 +37,7 @@ if x == b: return index index += 1 - raise ValueError, 'sequence.index(x): x not in sequence' + raise ValueError('sequence.index(x): x not in sequence') def isMappingType(obj,): 'isMappingType(a) -- Return True if a has a mapping type, False otherwise.' @@ -58,9 +58,9 @@ def repeat(obj, num): 'repeat(a, b) -- Return a * b, where a is a sequence, and b is an integer.' if not isinstance(num, (int, long)): - raise TypeError, 'an integer is required' + raise TypeError('an integer is required') if not isSequenceType(obj): - raise TypeError, "non-sequence object can't be repeated" + raise TypeError("non-sequence object can't be repeated") return obj * num @@ -68,59 +68,85 @@ def setslice(a, b, c, d): 'setslice(a, b, c, d) -- Same as a[b:c] = d.' - a[b:c] = d + a[b:c] = d __setslice__ = setslice +def _resolve_attr_chain(chain, obj, idx=0): + obj = getattr(obj, chain[idx]) + if idx + 1 == len(chain): + return obj + else: + return _resolve_attr_chain(chain, obj, idx + 1) + + +class _simple_attrgetter(object): + def __init__(self, attr): + self._attr = attr + + def __call__(self, obj): + return getattr(obj, self._attr) + + +class _single_attrgetter(object): + def __init__(self, attrs): + self._attrs = attrs + + def __call__(self, obj): + return _resolve_attr_chain(self._attrs, obj) + + +class _multi_attrgetter(object): + def __init__(self, attrs): + self._attrs = attrs + + def __call__(self, obj): + return tuple([ + _resolve_attr_chain(attrs, obj) + for attrs in self._attrs + ]) + + def attrgetter(attr, *attrs): + if ( + not isinstance(attr, basestring) or + not all(isinstance(a, basestring) for a in attrs) + ): + def _raise_typeerror(obj): + raise TypeError( + "argument must be a string, not %r" % type(attr).__name__ + ) + return _raise_typeerror if attrs: - getters = [single_attr_getter(a) for a in (attr,) + attrs] - def getter(obj): - return tuple([getter(obj) for getter in getters]) + return _multi_attrgetter([ + a.split(".") for a in [attr] + list(attrs) + ]) + elif "." not in attr: + return _simple_attrgetter(attr) else: - getter = single_attr_getter(attr) - return builtinify(getter) + return _single_attrgetter(attr.split(".")) -def single_attr_getter(attr): - if not isinstance(attr, str): - if not isinstance(attr, unicode): - def _raise_typeerror(obj): - raise TypeError("argument must be a string, not %r" % - (type(attr).__name__,)) - return _raise_typeerror - attr = attr.encode('ascii') - # - def make_getter(name, prevfn=None): - if prevfn is None: - def getter(obj): - return getattr(obj, name) + +class itemgetter(object): + def __init__(self, item, *items): + self._single = not bool(items) + if self._single: + self._idx = item else: - def getter(obj): - return getattr(prevfn(obj), name) - return getter - # - last = 0 - getter = None - while True: - dot = attr.find(".", last) - if dot < 0: break - getter = make_getter(attr[last:dot], getter) - last = dot + 1 - return make_getter(attr[last:], getter) + self._idx = [item] + list(items) + def __call__(self, obj): + if self._single: + return obj[self._idx] + else: + return tuple([obj[i] for i in self._idx]) -def itemgetter(item, *items): - if items: - list_of_indices = [item] + list(items) - def getter(obj): - return tuple([obj[i] for i in list_of_indices]) - else: - def getter(obj): - return obj[item] - return builtinify(getter) +class methodcaller(object): + def __init__(self, method_name, *args, **kwargs): + self._method_name = method_name + self._args = args + self._kwargs = kwargs -def methodcaller(method_name, *args, **kwargs): - def call(obj): - return getattr(obj, method_name)(*args, **kwargs) - return builtinify(call) + def __call__(self, obj): + return getattr(obj, self._method_name)(*self._args, **self._kwargs) diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py --- a/pypy/module/sys/test/test_sysmodule.py +++ b/pypy/module/sys/test/test_sysmodule.py @@ -91,9 +91,9 @@ assert isinstance(sys.__stderr__, file) assert isinstance(sys.__stdin__, file) - assert sys.__stdin__.name == "" - assert sys.__stdout__.name == "" - assert sys.__stderr__.name == "" + #assert sys.__stdin__.name == "" + #assert sys.__stdout__.name == "" + #assert sys.__stderr__.name == "" if self.appdirect and not isinstance(sys.stdin, file): return diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_zdistutils.py b/pypy/module/test_lib_pypy/cffi_tests/test_zdistutils.py --- a/pypy/module/test_lib_pypy/cffi_tests/test_zdistutils.py +++ b/pypy/module/test_lib_pypy/cffi_tests/test_zdistutils.py @@ -16,6 +16,10 @@ if distutils.ccompiler.get_default_compiler() == 'msvc': self.lib_m = 'msvcrt' + def teardown_class(self): + if udir.isdir(): + udir.remove() + def test_locate_engine_class(self): cls = _locate_engine_class(FFI(), self.generic) if self.generic: diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_zintegration.py b/pypy/module/test_lib_pypy/cffi_tests/test_zintegration.py --- a/pypy/module/test_lib_pypy/cffi_tests/test_zintegration.py +++ b/pypy/module/test_lib_pypy/cffi_tests/test_zintegration.py @@ -73,50 +73,55 @@ assert not os.path.exists(str(SNIPPET_DIR.join(dirname, 'lextab.py'))) assert not os.path.exists(str(SNIPPET_DIR.join(dirname, 'yacctab.py'))) -def test_infrastructure(): - run_setup_and_program('infrastructure', ''' - import snip_infrastructure - assert snip_infrastructure.func() == 42 - ''') +class TestZIntegration(object): + def teardown_class(self): + if udir.isdir(): + udir.remove() -def test_distutils_module(): - run_setup_and_program("distutils_module", ''' - import snip_basic_verify - p = snip_basic_verify.C.getpwuid(0) - assert snip_basic_verify.ffi.string(p.pw_name) == b"root" - ''') + def test_infrastructure(self): + run_setup_and_program('infrastructure', ''' + import snip_infrastructure + assert snip_infrastructure.func() == 42 + ''') -def test_distutils_package_1(): - run_setup_and_program("distutils_package_1", ''' - import snip_basic_verify1 - p = snip_basic_verify1.C.getpwuid(0) - assert snip_basic_verify1.ffi.string(p.pw_name) == b"root" - ''') + def test_distutils_module(self): + run_setup_and_program("distutils_module", ''' + import snip_basic_verify + p = snip_basic_verify.C.getpwuid(0) + assert snip_basic_verify.ffi.string(p.pw_name) == b"root" + ''') -def test_distutils_package_2(): - run_setup_and_program("distutils_package_2", ''' - import snip_basic_verify2 - p = snip_basic_verify2.C.getpwuid(0) - assert snip_basic_verify2.ffi.string(p.pw_name) == b"root" - ''') + def test_distutils_package_1(self): + run_setup_and_program("distutils_package_1", ''' + import snip_basic_verify1 + p = snip_basic_verify1.C.getpwuid(0) + assert snip_basic_verify1.ffi.string(p.pw_name) == b"root" + ''') -def test_setuptools_module(): - run_setup_and_program("setuptools_module", ''' - import snip_setuptools_verify - p = snip_setuptools_verify.C.getpwuid(0) - assert snip_setuptools_verify.ffi.string(p.pw_name) == b"root" - ''') + def test_distutils_package_2(self): + run_setup_and_program("distutils_package_2", ''' + import snip_basic_verify2 + p = snip_basic_verify2.C.getpwuid(0) + assert snip_basic_verify2.ffi.string(p.pw_name) == b"root" + ''') -def test_setuptools_package_1(): - run_setup_and_program("setuptools_package_1", ''' - import snip_setuptools_verify1 - p = snip_setuptools_verify1.C.getpwuid(0) - assert snip_setuptools_verify1.ffi.string(p.pw_name) == b"root" - ''') + def test_setuptools_module(self): + run_setup_and_program("setuptools_module", ''' + import snip_setuptools_verify + p = snip_setuptools_verify.C.getpwuid(0) + assert snip_setuptools_verify.ffi.string(p.pw_name) == b"root" + ''') -def test_setuptools_package_2(): - run_setup_and_program("setuptools_package_2", ''' - import snip_setuptools_verify2 - p = snip_setuptools_verify2.C.getpwuid(0) - assert snip_setuptools_verify2.ffi.string(p.pw_name) == b"root" - ''') + def test_setuptools_package_1(self): + run_setup_and_program("setuptools_package_1", ''' + import snip_setuptools_verify1 + p = snip_setuptools_verify1.C.getpwuid(0) + assert snip_setuptools_verify1.ffi.string(p.pw_name) == b"root" + ''') + + def test_setuptools_package_2(self): + run_setup_and_program("setuptools_package_2", ''' + import snip_setuptools_verify2 + p = snip_setuptools_verify2.C.getpwuid(0) + assert snip_setuptools_verify2.ffi.string(p.pw_name) == b"root" + ''') diff --git a/pypy/module/test_lib_pypy/test_datetime.py b/pypy/module/test_lib_pypy/test_datetime.py --- a/pypy/module/test_lib_pypy/test_datetime.py +++ b/pypy/module/test_lib_pypy/test_datetime.py @@ -1,193 +1,221 @@ """Additional tests for datetime.""" from __future__ import absolute_import -from lib_pypy import datetime import py -def test_repr(): - print datetime - expected = "datetime.datetime(1, 2, 3, 0, 0)" - assert repr(datetime.datetime(1,2,3)) == expected +class BaseTestDatetime: + def test_repr(self): + print datetime + expected = "datetime.datetime(1, 2, 3, 0, 0)" + assert repr(datetime.datetime(1,2,3)) == expected -def test_attributes(): - for x in [datetime.date.today(), - datetime.time(), - datetime.datetime.utcnow(), - datetime.timedelta(), - datetime.tzinfo()]: - raises(AttributeError, 'x.abc = 1') + def test_attributes(self): + for x in [datetime.date.today(), + datetime.time(), + datetime.datetime.utcnow(), + datetime.timedelta(), + datetime.tzinfo()]: + raises(AttributeError, 'x.abc = 1') -def test_timedelta_init_long(): - td = datetime.timedelta(microseconds=20000000000000000000) - assert td.days == 231481481 - assert td.seconds == 41600 - td = datetime.timedelta(microseconds=20000000000000000000.) - assert td.days == 231481481 - assert td.seconds == 41600 + def test_timedelta_init_long(self): + td = datetime.timedelta(microseconds=20000000000000000000) + assert td.days == 231481481 + assert td.seconds == 41600 + td = datetime.timedelta(microseconds=20000000000000000000.) + assert td.days == 231481481 + assert td.seconds == 41600 -def test_unpickle(): - e = raises(TypeError, datetime.date, '123') - assert e.value.args[0] == 'an integer is required' - e = raises(TypeError, datetime.time, '123') - assert e.value.args[0] == 'an integer is required' - e = raises(TypeError, datetime.datetime, '123') - assert e.value.args[0] == 'an integer is required' + def test_unpickle(self): + e = raises(TypeError, datetime.date, '123') + assert e.value.args[0] == 'an integer is required' + e = raises(TypeError, datetime.time, '123') + assert e.value.args[0] == 'an integer is required' + e = raises(TypeError, datetime.datetime, '123') + assert e.value.args[0] == 'an integer is required' - datetime.time('\x01' * 6, None) - with raises(TypeError) as e: - datetime.time('\x01' * 6, 123) - assert str(e.value) == "bad tzinfo state arg" + datetime.time('\x01' * 6, None) + with raises(TypeError) as e: + datetime.time('\x01' * 6, 123) + assert str(e.value) == "bad tzinfo state arg" - datetime.datetime('\x01' * 10, None) - with raises(TypeError) as e: - datetime.datetime('\x01' * 10, 123) - assert str(e.value) == "bad tzinfo state arg" + datetime.datetime('\x01' * 10, None) + with raises(TypeError) as e: + datetime.datetime('\x01' * 10, 123) + assert str(e.value) == "bad tzinfo state arg" -def test_strptime(): - import time, sys - if sys.version_info < (2, 6): - py.test.skip("needs the _strptime module") + def test_strptime(self): + import time, sys + if sys.version_info < (2, 6): + py.test.skip("needs the _strptime module") - string = '2004-12-01 13:02:47' - format = '%Y-%m-%d %H:%M:%S' - expected = datetime.datetime(*(time.strptime(string, format)[0:6])) - got = datetime.datetime.strptime(string, format) - assert expected == got + string = '2004-12-01 13:02:47' + format = '%Y-%m-%d %H:%M:%S' + expected = datetime.datetime(*(time.strptime(string, format)[0:6])) + got = datetime.datetime.strptime(string, format) + assert expected == got -def test_datetime_rounding(): - b = 0.0000001 - a = 0.9999994 + def test_datetime_rounding(self): + b = 0.0000001 + a = 0.9999994 - assert datetime.datetime.utcfromtimestamp(a).microsecond == 999999 - assert datetime.datetime.utcfromtimestamp(a).second == 0 - a += b - assert datetime.datetime.utcfromtimestamp(a).microsecond == 999999 - assert datetime.datetime.utcfromtimestamp(a).second == 0 - a += b - assert datetime.datetime.utcfromtimestamp(a).microsecond == 0 - assert datetime.datetime.utcfromtimestamp(a).second == 1 + assert datetime.datetime.utcfromtimestamp(a).microsecond == 999999 + assert datetime.datetime.utcfromtimestamp(a).second == 0 + a += b + assert datetime.datetime.utcfromtimestamp(a).microsecond == 999999 + assert datetime.datetime.utcfromtimestamp(a).second == 0 + a += b + assert datetime.datetime.utcfromtimestamp(a).microsecond == 0 + assert datetime.datetime.utcfromtimestamp(a).second == 1 -def test_more_datetime_rounding(): - # this test verified on top of CPython 2.7 (using a plain - # "import datetime" above) - expected_results = { - -1000.0: 'datetime.datetime(1969, 12, 31, 23, 43, 20)', - -999.9999996: 'datetime.datetime(1969, 12, 31, 23, 43, 20)', - -999.4: 'datetime.datetime(1969, 12, 31, 23, 43, 20, 600000)', - -999.0000004: 'datetime.datetime(1969, 12, 31, 23, 43, 21)', - -1.0: 'datetime.datetime(1969, 12, 31, 23, 59, 59)', - -0.9999996: 'datetime.datetime(1969, 12, 31, 23, 59, 59)', - -0.4: 'datetime.datetime(1969, 12, 31, 23, 59, 59, 600000)', - -0.0000004: 'datetime.datetime(1970, 1, 1, 0, 0)', - 0.0: 'datetime.datetime(1970, 1, 1, 0, 0)', - 0.0000004: 'datetime.datetime(1970, 1, 1, 0, 0)', - 0.4: 'datetime.datetime(1970, 1, 1, 0, 0, 0, 400000)', - 0.9999996: 'datetime.datetime(1970, 1, 1, 0, 0, 1)', - 1000.0: 'datetime.datetime(1970, 1, 1, 0, 16, 40)', - 1000.0000004: 'datetime.datetime(1970, 1, 1, 0, 16, 40)', - 1000.4: 'datetime.datetime(1970, 1, 1, 0, 16, 40, 400000)', - 1000.9999996: 'datetime.datetime(1970, 1, 1, 0, 16, 41)', - 1293843661.191: 'datetime.datetime(2011, 1, 1, 1, 1, 1, 191000)', - } - for t in sorted(expected_results): - dt = datetime.datetime.utcfromtimestamp(t) - assert repr(dt) == expected_results[t] + def test_more_datetime_rounding(self): + # this test verified on top of CPython 2.7 (using a plain + # "import datetime" above) + expected_results = { + -1000.0: 'datetime.datetime(1969, 12, 31, 23, 43, 20)', + -999.9999996: 'datetime.datetime(1969, 12, 31, 23, 43, 20)', + -999.4: 'datetime.datetime(1969, 12, 31, 23, 43, 20, 600000)', + -999.0000004: 'datetime.datetime(1969, 12, 31, 23, 43, 21)', + -1.0: 'datetime.datetime(1969, 12, 31, 23, 59, 59)', + -0.9999996: 'datetime.datetime(1969, 12, 31, 23, 59, 59)', + -0.4: 'datetime.datetime(1969, 12, 31, 23, 59, 59, 600000)', + -0.0000004: 'datetime.datetime(1970, 1, 1, 0, 0)', + 0.0: 'datetime.datetime(1970, 1, 1, 0, 0)', + 0.0000004: 'datetime.datetime(1970, 1, 1, 0, 0)', + 0.4: 'datetime.datetime(1970, 1, 1, 0, 0, 0, 400000)', + 0.9999996: 'datetime.datetime(1970, 1, 1, 0, 0, 1)', + 1000.0: 'datetime.datetime(1970, 1, 1, 0, 16, 40)', + 1000.0000004: 'datetime.datetime(1970, 1, 1, 0, 16, 40)', + 1000.4: 'datetime.datetime(1970, 1, 1, 0, 16, 40, 400000)', + 1000.9999996: 'datetime.datetime(1970, 1, 1, 0, 16, 41)', + 1293843661.191: 'datetime.datetime(2011, 1, 1, 1, 1, 1, 191000)', + } + for t in sorted(expected_results): + dt = datetime.datetime.utcfromtimestamp(t) + assert repr(dt) == expected_results[t] -def test_utcfromtimestamp(): - """Confirm that utcfromtimestamp and fromtimestamp give consistent results. + def test_utcfromtimestamp(self): + """Confirm that utcfromtimestamp and fromtimestamp give consistent results. - Based on danchr's test script in https://bugs.pypy.org/issue986 - """ - import os - import time - if os.name == 'nt': - skip("setting os.environ['TZ'] ineffective on windows") - try: - prev_tz = os.environ.get("TZ") - os.environ["TZ"] = "GMT" - time.tzset() - for unused in xrange(100): - now = time.time() - delta = (datetime.datetime.utcfromtimestamp(now) - - datetime.datetime.fromtimestamp(now)) - assert delta.days * 86400 + delta.seconds == 0 - finally: - if prev_tz is None: - del os.environ["TZ"] - else: - os.environ["TZ"] = prev_tz - time.tzset() + Based on danchr's test script in https://bugs.pypy.org/issue986 + """ + import os + import time + if os.name == 'nt': + skip("setting os.environ['TZ'] ineffective on windows") + try: + prev_tz = os.environ.get("TZ") + os.environ["TZ"] = "GMT" + time.tzset() + for unused in xrange(100): + now = time.time() + delta = (datetime.datetime.utcfromtimestamp(now) - + datetime.datetime.fromtimestamp(now)) + assert delta.days * 86400 + delta.seconds == 0 + finally: + if prev_tz is None: + del os.environ["TZ"] + else: + os.environ["TZ"] = prev_tz + time.tzset() -def test_utcfromtimestamp_microsecond(): - dt = datetime.datetime.utcfromtimestamp(0) - assert isinstance(dt.microsecond, int) + def test_utcfromtimestamp_microsecond(self): + dt = datetime.datetime.utcfromtimestamp(0) + assert isinstance(dt.microsecond, int) -def test_default_args(): - with py.test.raises(TypeError): - datetime.datetime() - with py.test.raises(TypeError): - datetime.datetime(10) - with py.test.raises(TypeError): - datetime.datetime(10, 10) - datetime.datetime(10, 10, 10) + def test_default_args(self): + with py.test.raises(TypeError): + datetime.datetime() + with py.test.raises(TypeError): + datetime.datetime(10) + with py.test.raises(TypeError): + datetime.datetime(10, 10) + datetime.datetime(10, 10, 10) -def test_check_arg_types(): - import decimal - class Number: - def __init__(self, value): - self.value = value - def __int__(self): - return self.value + def test_check_arg_types(self): + import decimal + class Number: + def __init__(self, value): + self.value = value + def __int__(self): + return self.value - for xx in [10L, - decimal.Decimal(10), - decimal.Decimal('10.9'), - Number(10), - Number(10L)]: - assert datetime.datetime(10, 10, 10, 10, 10, 10, 10) == \ - datetime.datetime(xx, xx, xx, xx, xx, xx, xx) + for xx in [10L, + decimal.Decimal(10), + decimal.Decimal('10.9'), + Number(10), + Number(10L)]: + assert datetime.datetime(10, 10, 10, 10, 10, 10, 10) == \ + datetime.datetime(xx, xx, xx, xx, xx, xx, xx) - with py.test.raises(TypeError) as e: - datetime.datetime(10, 10, '10') - assert str(e.value) == 'an integer is required' + with py.test.raises(TypeError) as e: + datetime.datetime(10, 10, '10') + assert str(e.value) == 'an integer is required' - f10 = Number(10.9) - with py.test.raises(TypeError) as e: - datetime.datetime(10, 10, f10) - assert str(e.value) == '__int__ method should return an integer' + f10 = Number(10.9) + with py.test.raises(TypeError) as e: + datetime.datetime(10, 10, f10) + assert str(e.value) == '__int__ method should return an integer' - class Float(float): - pass - s10 = Float(10.9) - with py.test.raises(TypeError) as e: - datetime.datetime(10, 10, s10) - assert str(e.value) == 'integer argument expected, got float' + class Float(float): + pass + s10 = Float(10.9) + with py.test.raises(TypeError) as e: + datetime.datetime(10, 10, s10) + assert str(e.value) == 'integer argument expected, got float' - with py.test.raises(TypeError): - datetime.datetime(10., 10, 10) - with py.test.raises(TypeError): - datetime.datetime(10, 10., 10) - with py.test.raises(TypeError): - datetime.datetime(10, 10, 10.) - with py.test.raises(TypeError): - datetime.datetime(10, 10, 10, 10.) - with py.test.raises(TypeError): - datetime.datetime(10, 10, 10, 10, 10.) - with py.test.raises(TypeError): - datetime.datetime(10, 10, 10, 10, 10, 10.) - with py.test.raises(TypeError): - datetime.datetime(10, 10, 10, 10, 10, 10, 10.) + with py.test.raises(TypeError): + datetime.datetime(10., 10, 10) + with py.test.raises(TypeError): + datetime.datetime(10, 10., 10) + with py.test.raises(TypeError): + datetime.datetime(10, 10, 10.) + with py.test.raises(TypeError): + datetime.datetime(10, 10, 10, 10.) + with py.test.raises(TypeError): + datetime.datetime(10, 10, 10, 10, 10.) + with py.test.raises(TypeError): + datetime.datetime(10, 10, 10, 10, 10, 10.) + with py.test.raises(TypeError): + datetime.datetime(10, 10, 10, 10, 10, 10, 10.) -def test_utcnow_microsecond(): - import copy + def test_utcnow_microsecond(self): + import copy - dt = datetime.datetime.utcnow() - assert type(dt.microsecond) is int + dt = datetime.datetime.utcnow() + assert type(dt.microsecond) is int - copy.copy(dt) + copy.copy(dt) -def test_radd(): - class X(object): - def __radd__(self, other): - return "radd" - assert datetime.date(10, 10, 10) + X() == "radd" + def test_radd(self): + class X(object): + def __radd__(self, other): + return "radd" + assert datetime.date(10, 10, 10) + X() == "radd" + + def test_raises_if_passed_naive_datetime_and_start_or_end_time_defined(self): + class Foo(datetime.tzinfo): + def utcoffset(self, dt): + return datetime.timedelta(0.1) + naive = datetime.datetime(2014, 9, 22) + aware = datetime.datetime(2014, 9, 22, tzinfo=Foo()) + with py.test.raises(TypeError) as e: + naive == aware + assert str(e.value) == "can't compare offset-naive and offset-aware datetimes" + with py.test.raises(TypeError) as e: + naive - aware + assert str(e.value) == "can't subtract offset-naive and offset-aware datetimes" + naive = datetime.time(7, 32, 12) + aware = datetime.time(7, 32, 12, tzinfo=Foo()) + with py.test.raises(TypeError) as e: + naive == aware + assert str(e.value) == "can't compare offset-naive and offset-aware times" + +class TestDatetimeCPython(BaseTestDatetime): + def setup_class(cls): + global datetime + import datetime + +class TestDatetimePyPy(BaseTestDatetime): + def setup_class(cls): + global datetime + from lib_pypy import datetime diff --git a/pypy/sandbox/pypy_interact.py b/pypy/sandbox/pypy_interact.py --- a/pypy/sandbox/pypy_interact.py +++ b/pypy/sandbox/pypy_interact.py @@ -55,7 +55,7 @@ return Dir({ 'bin': Dir({ - 'pypy-c': RealFile(self.executable), + 'pypy-c': RealFile(self.executable, mode=0111), 'lib-python': RealDir(os.path.join(libroot, 'lib-python'), exclude=exclude), 'lib_pypy': RealDir(os.path.join(libroot, 'lib_pypy'), diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -297,7 +297,13 @@ argparse = imp.load_source('argparse', 'lib-python/2.7/argparse.py') if sys.platform == 'win32': pypy_exe = 'pypy.exe' - license_base = os.path.join(basedir, r'..\..\..\local') # as on buildbot YMMV + for p in [os.path.join(basedir, r'..\..\..\local'), #buildbot + os.path.join(basedir, r'..\local')]: # pypy/doc/windows.rst + if os.path.exists(p): + license_base = p + break + else: + license_base = 'unkown' else: pypy_exe = 'pypy' license_base = '/usr/share/doc' @@ -370,5 +376,21 @@ if __name__ == '__main__': import sys + if sys.platform == 'win32': + # Try to avoid opeing a dialog box if one of the + # subprocesses causes a system error + import ctypes + winapi = ctypes.windll.kernel32 + SetErrorMode = winapi.SetErrorMode + SetErrorMode.argtypes=[ctypes.c_int] + + SEM_FAILCRITICALERRORS = 1 + SEM_NOGPFAULTERRORBOX = 2 + SEM_NOOPENFILEERRORBOX = 0x8000 + flags = SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX | SEM_NOOPENFILEERRORBOX + #Since there is no GetErrorMode, do a double Set + old_mode = SetErrorMode(flags) + SetErrorMode(old_mode | flags) + retval, _ = package(*sys.argv[1:]) sys.exit(retval) diff --git a/pypy/tool/release/test/test_package.py b/pypy/tool/release/test/test_package.py --- a/pypy/tool/release/test/test_package.py +++ b/pypy/tool/release/test/test_package.py @@ -115,15 +115,21 @@ check(pypy, 0755) def test_generate_license(): - from os.path import dirname, abspath, join + from os.path import dirname, abspath, join, exists class Options(object): pass options = Options() basedir = dirname(dirname(dirname(dirname(dirname(abspath(__file__)))))) options.no_tk = False if sys.platform == 'win32': - # as on buildbot YMMV - options.license_base = join(basedir, r'..\..\..\local') + for p in [join(basedir, r'..\..\..\local'), #buildbot + join(basedir, r'..\local')]: # pypy/doc/windows.rst + if exists(p): + license_base = p + break + else: + license_base = 'unkown' + options.license_base = license_base else: options.license_base = '/usr/share/doc' license = package.generate_license(py.path.local(basedir), options) diff --git a/rpython/jit/backend/arm/callbuilder.py b/rpython/jit/backend/arm/callbuilder.py --- a/rpython/jit/backend/arm/callbuilder.py +++ b/rpython/jit/backend/arm/callbuilder.py @@ -92,7 +92,8 @@ self.mc.LDR_ri(r.r7.value, r.r5.value) # change 'rpy_fastgil' to 0 (it should be non-zero right now) - self.mc.DMB() + if self.asm.cpu.cpuinfo.arch_version >= 7: + self.mc.DMB() self.mc.gen_load_int(r.r6.value, fastgil) self.mc.MOV_ri(r.ip.value, 0) self.mc.STR_ri(r.ip.value, r.r6.value) @@ -112,7 +113,8 @@ self.mc.STREX(r.r3.value, r.ip.value, r.r6.value, c=c.EQ) # try to claim the lock self.mc.CMP_ri(r.r3.value, 0, cond=c.EQ) # did this succeed? - self.mc.DMB() + if self.asm.cpu.cpuinfo.arch_version >= 7: + self.mc.DMB() # the success of the lock acquisition is defined by # 'EQ is true', or equivalently by 'r3 == 0'. # diff --git a/rpython/jit/backend/arm/codebuilder.py b/rpython/jit/backend/arm/codebuilder.py --- a/rpython/jit/backend/arm/codebuilder.py +++ b/rpython/jit/backend/arm/codebuilder.py @@ -333,6 +333,8 @@ | (rn & 0xF) << 16) def DMB(self): + # ARMv7 only. I guess ARMv6 CPUs cannot be used in symmetric + # multi-processing at all? That would make this instruction unneeded. # note: 'cond' is only permitted on Thumb here, but don't # write literally 0xf57ff05f, because it's larger than 31 bits c = cond.AL diff --git a/rpython/jit/backend/arm/instructions.py b/rpython/jit/backend/arm/instructions.py --- a/rpython/jit/backend/arm/instructions.py +++ b/rpython/jit/backend/arm/instructions.py @@ -142,6 +142,7 @@ #'VCVT' : {'opc1':0xB, 'opc2':0xE, 'opc3':0x1, 'base': False}, } +# ARMv7 only simd_instructions_3regs = { 'VADD_i64': {'A': 0x8, 'B': 0, 'U': 0}, 'VSUB_i64': {'A': 0x8, 'B': 0, 'U': 1}, diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -1,3 +1,4 @@ +from rpython.rlib import rgc from rpython.rlib.rarithmetic import ovfcheck from rpython.rtyper.lltypesystem import llmemory from rpython.jit.metainterp import history @@ -390,8 +391,8 @@ val = op.getarg(0) if val not in self.write_barrier_applied: v = op.getarg(1) - if isinstance(v, BoxPtr) or (isinstance(v, ConstPtr) and - bool(v.value)): # store a non-NULL + if (isinstance(v, BoxPtr) or (isinstance(v, ConstPtr) and + rgc.needs_write_barrier(v.value))): self.gen_write_barrier(val) #op = op.copy_and_change(rop.SETFIELD_RAW) self.newops.append(op) @@ -400,8 +401,8 @@ val = op.getarg(0) if val not in self.write_barrier_applied: v = op.getarg(2) - if isinstance(v, BoxPtr) or (isinstance(v, ConstPtr) and - bool(v.value)): # store a non-NULL + if (isinstance(v, BoxPtr) or (isinstance(v, ConstPtr) and + rgc.needs_write_barrier(v.value))): self.gen_write_barrier_array(val, op.getarg(1)) #op = op.copy_and_change(rop.SET{ARRAYITEM,INTERIORFIELD}_RAW) self.newops.append(op) diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py --- a/rpython/rlib/rgc.py +++ b/rpython/rlib/rgc.py @@ -86,6 +86,14 @@ collect(i) i += 1 +def needs_write_barrier(obj): + """ We need to emit write barrier if the right hand of assignment + is in nursery, used by the JIT for handling set*_gc(Const) + """ + if not obj: + return False + return can_move(obj) + def _heap_stats(): raise NotImplementedError # can't be run directly diff --git a/rpython/rtyper/lltypesystem/lltype.py b/rpython/rtyper/lltypesystem/lltype.py --- a/rpython/rtyper/lltypesystem/lltype.py +++ b/rpython/rtyper/lltypesystem/lltype.py @@ -1,16 +1,17 @@ -from types import NoneType, MethodType import weakref +from types import MethodType, NoneType + +from rpython.annotator.bookkeeper import analyzer_for, immutablevalue from rpython.annotator.model import ( - SomeInteger, SomeBool, SomeObject, AnnotatorError) + AnnotatorError, SomeBool, SomeInteger, SomeObject) +from rpython.rlib.objectmodel import Symbolic from rpython.rlib.rarithmetic import ( - r_int, r_uint, intmask, r_singlefloat, r_ulonglong, r_longlong, - r_longfloat, r_longlonglong, base_int, normalizedinttype, longlongmask, - longlonglongmask, maxint, is_valid_int, is_emulated_long) -from rpython.rlib.objectmodel import Symbolic + base_int, intmask, is_emulated_long, is_valid_int, longlonglongmask, + longlongmask, maxint, normalizedinttype, r_int, r_longfloat, r_longlong, + r_longlonglong, r_singlefloat, r_uint, r_ulonglong) +from rpython.rtyper.extregistry import ExtRegistryEntry +from rpython.tool import leakfinder from rpython.tool.identity_dict import identity_dict -from rpython.tool import leakfinder -from rpython.annotator.bookkeeper import analyzer_for, immutablevalue -from rpython.rtyper.extregistry import ExtRegistryEntry class State(object): pass @@ -313,14 +314,12 @@ except KeyError: return ContainerType.__getattr__(self, name) - def _nofield(self, name): raise AttributeError('struct %s has no field %r' % (self._name, name)) def _names_without_voids(self): - names_without_voids = [name for name in self._names if self._flds[name] is not Void] - return names_without_voids + return [name for name in self._names if self._flds[name] is not Void] def _str_fields_without_voids(self): return ', '.join(['%s: %s' % (name, self._flds[name]) @@ -576,8 +575,10 @@ _gckind = 'raw' def __init__(self, tag, hints={}): - """ if hints['render_structure'] is set, the type is internal and not considered - to come from somewhere else (it should be rendered as a structure) """ + """If hints['render_structure'] is set, the type is internal and + not considered to come from somewhere else (it should be + rendered as a structure) + """ self.tag = tag self.__name__ = tag self.hints = frozendict(hints) @@ -675,7 +676,8 @@ _numbertypes = {int: Number("Signed", int, intmask)} _numbertypes[r_int] = _numbertypes[int] -_numbertypes[r_longlonglong] = Number("SignedLongLongLong", r_longlonglong, longlonglongmask) +_numbertypes[r_longlonglong] = Number("SignedLongLongLong", r_longlonglong, + longlonglongmask) if r_longlong is not r_int: _numbertypes[r_longlong] = Number("SignedLongLong", r_longlong, longlongmask) @@ -702,8 +704,8 @@ UnsignedLongLong = build_number("UnsignedLongLong", r_ulonglong) Float = Primitive("Float", 0.0) # C type 'double' -SingleFloat = Primitive("SingleFloat", r_singlefloat(0.0)) # C type 'float' -LongFloat = Primitive("LongFloat", r_longfloat(0.0)) # C type 'long double' +SingleFloat = Primitive("SingleFloat", r_singlefloat(0.0)) # 'float' +LongFloat = Primitive("LongFloat", r_longfloat(0.0)) # 'long double' r_singlefloat._TYPE = SingleFloat Char = Primitive("Char", '\x00') @@ -876,9 +878,11 @@ @analyzer_for(cast_primitive) def ann_cast_primitive(T, s_v): - from rpython.rtyper.llannotation import annotation_to_lltype, ll_to_annotation + from rpython.rtyper.llannotation import ( + annotation_to_lltype, ll_to_annotation) assert T.is_constant() - return ll_to_annotation(cast_primitive(T.const, annotation_to_lltype(s_v)._defl())) + return ll_to_annotation(cast_primitive(T.const, + annotation_to_lltype(s_v)._defl())) def _cast_whatever(TGT, value): @@ -905,7 +909,8 @@ elif TGT == llmemory.Address and isinstance(ORIG, Ptr): return llmemory.cast_ptr_to_adr(value) elif TGT == Signed and isinstance(ORIG, Ptr) and ORIG.TO._gckind == 'raw': - return llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(value), 'symbolic') + return llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(value), + 'symbolic') raise TypeError("don't know how to cast from %r to %r" % (ORIG, TGT)) @@ -1176,8 +1181,8 @@ except DelayedPointer: return True # assume it's not a delayed null - # _setobj, _getobj and _obj0 are really _internal_ implementations details of _ptr, - # use _obj if necessary instead ! + # _setobj, _getobj and _obj0 are really _internal_ implementations + # details of _ptr, use _obj if necessary instead ! def _setobj(self, pointing_to, solid=False): if pointing_to is None: obj0 = None @@ -1244,12 +1249,12 @@ if T1 == T2: setattr(self._obj, field_name, val) else: - raise TypeError("%r instance field %r:\n" - "expects %r\n" - " got %r" % (self._T, field_name, T1, T2)) + raise TypeError( + "%r instance field %r:\nexpects %r\n got %r" % + (self._T, field_name, T1, T2)) return - raise AttributeError("%r instance has no field %r" % (self._T, - field_name)) + raise AttributeError("%r instance has no field %r" % + (self._T, field_name)) def __getitem__(self, i): # ! can only return basic or ptr ! if isinstance(self._T, (Array, FixedSizeArray)): @@ -1266,7 +1271,8 @@ if isinstance(self._T, (Array, FixedSizeArray)): T1 = self._T.OF if isinstance(T1, ContainerType): - raise TypeError("cannot directly assign to container array items") + raise TypeError("cannot directly assign to container array " + "items") T2 = typeOf(val) if T2 != T1: from rpython.rtyper.lltypesystem import rffi @@ -1316,7 +1322,8 @@ from rpython.rtyper.lltypesystem import rffi if isinstance(self._T, FuncType): if len(args) != len(self._T.ARGS): - raise TypeError("calling %r with wrong argument number: %r" % (self._T, args)) + raise TypeError("calling %r with wrong argument number: %r" % + (self._T, args)) for i, a, ARG in zip(range(len(self._T.ARGS)), args, self._T.ARGS): if typeOf(a) != ARG: # ARG could be Void @@ -1415,11 +1422,13 @@ raise RuntimeError("widening to trash: %r" % self) PARENTTYPE = struc._parent_type if getattr(parent, PARENTTYPE._names[0]) != struc: - raise InvalidCast(CURTYPE, PTRTYPE) # xxx different exception perhaps? + # xxx different exception perhaps? + raise InvalidCast(CURTYPE, PTRTYPE) struc = parent u -= 1 if PARENTTYPE != PTRTYPE.TO: - raise RuntimeError("widening %r inside %r instead of %r" % (CURTYPE, PARENTTYPE, PTRTYPE.TO)) + raise RuntimeError("widening %r inside %r instead of %r" % + (CURTYPE, PARENTTYPE, PTRTYPE.TO)) return _ptr(PTRTYPE, struc, solid=self._solid) def _cast_to_int(self, check=True): @@ -1430,7 +1439,9 @@ return obj # special case for cast_int_to_ptr() results obj = normalizeptr(self, check)._getobj(check) if isinstance(obj, int): - return obj # special case for cast_int_to_ptr() results put into opaques + # special case for cast_int_to_ptr() results put into + # opaques + return obj if getattr(obj, '_read_directly_intval', False): return obj.intval # special case for _llgcopaque result = intmask(obj._getid()) @@ -1468,7 +1479,8 @@ """XXX A nice docstring here""" T = typeOf(val) if isinstance(T, ContainerType): - if self._T._gckind == 'gc' and T._gckind == 'raw' and not isinstance(T, OpaqueType): + if (self._T._gckind == 'gc' and T._gckind == 'raw' and + not isinstance(T, OpaqueType)): val = _interior_ptr(T, self._obj, [offset]) else: val = _ptr(Ptr(T), val, solid=self._solid) @@ -1531,12 +1543,14 @@ setattr(example, s_attr.const, v_lltype._defl()) def call(self, args): - from rpython.rtyper.llannotation import annotation_to_lltype, ll_to_annotation + from rpython.rtyper.llannotation import ( + annotation_to_lltype, ll_to_annotation) args_s, kwds_s = args.unpack() if kwds_s: raise Exception("keyword arguments to call to a low-level fn ptr") info = 'argument to ll function pointer call' - llargs = [annotation_to_lltype(s_arg, info)._defl() for s_arg in args_s] + llargs = [annotation_to_lltype(s_arg, info)._defl() + for s_arg in args_s] v = self.ll_ptrtype._example()(*llargs) return ll_to_annotation(v) @@ -1593,7 +1607,6 @@ return val - assert not '__dict__' in dir(_interior_ptr) class _container(object): @@ -1721,11 +1734,13 @@ __slots__ = ('_hash_cache_', '_compilation_info') - def __new__(self, TYPE, n=None, initialization=None, parent=None, parentindex=None): + def __new__(self, TYPE, n=None, initialization=None, parent=None, + parentindex=None): my_variety = _struct_variety(TYPE._names) return object.__new__(my_variety) - def __init__(self, TYPE, n=None, initialization=None, parent=None, parentindex=None): + def __init__(self, TYPE, n=None, initialization=None, parent=None, + parentindex=None): _parentable.__init__(self, TYPE) if n is not None and TYPE._arrayfld is None: raise TypeError("%r is not variable-sized" % (TYPE,)) @@ -1734,9 +1749,11 @@ first, FIRSTTYPE = TYPE._first_struct() for fld, typ in TYPE._flds.items(): if fld == TYPE._arrayfld: - value = _array(typ, n, initialization=initialization, parent=self, parentindex=fld) + value = _array(typ, n, initialization=initialization, + parent=self, parentindex=fld) else: - value = typ._allocate(initialization=initialization, parent=self, parentindex=fld) + value = typ._allocate(initialization=initialization, + parent=self, parentindex=fld) setattr(self, fld, value) if parent is not None: self._setparentstructure(parent, parentindex) @@ -1795,7 +1812,8 @@ __slots__ = ('items',) - def __init__(self, TYPE, n, initialization=None, parent=None, parentindex=None): + def __init__(self, TYPE, n, initialization=None, parent=None, + parentindex=None): if not is_valid_int(n): raise TypeError("array length must be an int") if n < 0: @@ -1964,7 +1982,8 @@ if not key._was_freed(): newcache[key] = value except RuntimeError: - pass # ignore "accessing subxxx, but already gc-ed parent" + # ignore "accessing subxxx, but already gc-ed parent" + pass if newcache: _subarray._cache[T] = newcache else: @@ -2020,8 +2039,10 @@ attrs.setdefault('_name', '?') attrs.setdefault('_callable', None) self.__dict__.update(attrs) - if '_callable' in attrs and hasattr(attrs['_callable'], '_compilation_info'): - self.__dict__['compilation_info'] = attrs['_callable']._compilation_info + if '_callable' in attrs and hasattr(attrs['_callable'], + '_compilation_info'): + self.__dict__['compilation_info'] = \ + attrs['_callable']._compilation_info def __repr__(self): return '<%s>' % (self,) @@ -2126,8 +2147,8 @@ return _ptr(Ptr(T), o, solid) @analyzer_for(malloc) -def ann_malloc(s_T, s_n=None, s_flavor=None, s_zero=None, s_track_allocation=None, - s_add_memory_pressure=None): +def ann_malloc(s_T, s_n=None, s_flavor=None, s_zero=None, + s_track_allocation=None, s_add_memory_pressure=None): assert (s_n is None or s_n.knowntype == int or issubclass(s_n.knowntype, base_int)) assert s_T.is_constant() @@ -2303,7 +2324,8 @@ @analyzer_for(runtime_type_info) def ann_runtime_type_info(s_p): - assert isinstance(s_p, SomePtr), "runtime_type_info of non-pointer: %r" % s_p + assert isinstance(s_p, SomePtr), \ + "runtime_type_info of non-pointer: %r" % s_p return SomePtr(typeOf(runtime_type_info(s_p.ll_ptrtype._example()))) diff --git a/rpython/rtyper/module/ll_os_stat.py b/rpython/rtyper/module/ll_os_stat.py --- a/rpython/rtyper/module/ll_os_stat.py +++ b/rpython/rtyper/module/ll_os_stat.py @@ -186,7 +186,10 @@ _name_struct_stat = '_stati64' INCLUDES = ['sys/types.h', 'sys/stat.h', 'sys/statvfs.h'] else: - _name_struct_stat = 'stat' + if sys.platform.startswith('linux'): + _name_struct_stat = 'stat64' + else: + _name_struct_stat = 'stat' INCLUDES = ['sys/types.h', 'sys/stat.h', 'sys/statvfs.h', 'unistd.h'] compilation_info = ExternalCompilationInfo( diff --git a/rpython/tool/jitlogparser/parser.py b/rpython/tool/jitlogparser/parser.py --- a/rpython/tool/jitlogparser/parser.py +++ b/rpython/tool/jitlogparser/parser.py @@ -406,9 +406,9 @@ loops = [] cat = extract_category(log, 'jit-log-opt') if not cat: - extract_category(log, 'jit-log-rewritten') + cat = extract_category(log, 'jit-log-rewritten') if not cat: - extract_category(log, 'jit-log-noopt') + cat = extract_category(log, 'jit-log-noopt') for entry in cat: parser = ParserCls(entry, None, {}, 'lltype', None, nonstrict=True) diff --git a/rpython/translator/backendopt/storesink.py b/rpython/translator/backendopt/storesink.py --- a/rpython/translator/backendopt/storesink.py +++ b/rpython/translator/backendopt/storesink.py @@ -1,6 +1,8 @@ from rpython.rtyper.lltypesystem.lloperation import llop +from rpython.flowspace.model import mkentrymap, Variable from rpython.translator.backendopt import removenoops +from rpython.translator import simplify def has_side_effects(op): if op.opname == 'debug_assert': @@ -10,38 +12,86 @@ except AttributeError: return True + def storesink_graph(graph): + """ remove superfluous getfields. use a super-local method: all non-join + blocks inherit the heap information from their (single) predecessor + """ + added_some_same_as = False + entrymap = mkentrymap(graph) + # all merge blocks are starting points + todo = [(block, None, None) for (block, prev_blocks) in entrymap.iteritems() + if len(prev_blocks) > 1 or block is graph.startblock] + + visited = 0 + + while todo: + block, cache, inputlink = todo.pop() + visited += 1 + if cache is None: + cache = {} + + if block.operations: + changed_block = _storesink_block(block, cache, inputlink) + added_some_same_as = changed_block or added_some_same_as + for link in block.exits: + if len(entrymap[link.target]) == 1: + new_cache = _translate_cache(cache, link) + todo.append((link.target, new_cache, link)) + + assert visited == len(entrymap) + if added_some_same_as: + removenoops.remove_same_as(graph) + simplify.transform_dead_op_vars(graph) + +def _translate_cache(cache, link): + if link.target.operations == (): # exit or except block: + return {} + block = link.target + local_versions = {var1: var2 for var1, var2 in zip(link.args, block.inputargs)} + def _translate_arg(arg): + if isinstance(arg, Variable): + res = local_versions.get(arg, None) + if res is None: + res = Variable(arg) + res.concretetype = arg.concretetype + link.args.append(arg) + block.inputargs.append(res) + local_versions[arg] = res + return res + else: + return arg + new_cache = {} + for (var, field), res in cache.iteritems(): + if var in local_versions or not isinstance(var, Variable): + new_cache[_translate_arg(var), field] = _translate_arg(res) + return new_cache + +def _storesink_block(block, cache, inputlink): def clear_cache_for(cache, concretetype, fieldname): for k in cache.keys(): if k[0].concretetype == concretetype and k[1] == fieldname: del cache[k] added_some_same_as = False - - for block in graph.iterblocks(): - newops = [] - cache = {} - for op in block.operations: - if op.opname == 'getfield': - tup = (op.args[0], op.args[1].value) - res = cache.get(tup, None) - if res is not None: - op.opname = 'same_as' - op.args = [res] - added_some_same_as = True - else: - cache[tup] = op.result - elif op.opname in ['setarrayitem', 'setinteriorfield']: - pass - elif op.opname == 'setfield': - clear_cache_for(cache, op.args[0].concretetype, - op.args[1].value) - elif has_side_effects(op): - cache = {} - newops.append(op) - if block.operations: - block.operations = newops - - if added_some_same_as: - removenoops.remove_same_as(graph) + for op in block.operations: + if op.opname == 'getfield': + tup = (op.args[0], op.args[1].value) + res = cache.get(tup, None) + if res is not None: + op.opname = 'same_as' + op.args = [res] + added_some_same_as = True + else: + cache[tup] = op.result + elif op.opname in ('setarrayitem', 'setinteriorfield', "malloc", "malloc_varsize"): + pass + elif op.opname == 'setfield': + target = op.args[0] + field = op.args[1].value + clear_cache_for(cache, target.concretetype, field) + cache[target, field] = op.args[2] + elif has_side_effects(op): + cache.clear() + return added_some_same_as diff --git a/rpython/translator/backendopt/test/test_storesink.py b/rpython/translator/backendopt/test/test_storesink.py --- a/rpython/translator/backendopt/test/test_storesink.py +++ b/rpython/translator/backendopt/test/test_storesink.py @@ -42,7 +42,7 @@ a.x = i return a.x - self.check(f, [int], 1) + self.check(f, [int], 0) def test_simple(self): class A(object): @@ -53,7 +53,7 @@ a.x = i return a.x + a.x - self.check(f, [int], 1) + self.check(f, [int], 0) def test_irrelevant_setfield(self): class A(object): @@ -67,7 +67,7 @@ two = a.x return one + two - self.check(f, [int], 1) + self.check(f, [int], 0) def test_relevant_setfield(self): class A(object): @@ -101,7 +101,7 @@ two = a.x return one + two - self.check(f, [int], 1) + self.check(f, [int], 0) def test_subclass(self): class A(object): @@ -119,7 +119,7 @@ two = a.x return one + two - self.check(f, [int], 2) + self.check(f, [int], 1) def test_bug_1(self): class A(object): @@ -133,4 +133,37 @@ return True return n - self.check(f, [int], 1) + self.check(f, [int], 0) + + + def test_cfg_splits(self): + class A(object): + pass + + def f(i): + a = A() + j = i + for i in range(i): + a.x = i + if i: + j = a.x + a.x + else: + j = a.x * 5 + return j + + self.check(f, [int], 0) + + def test_malloc_does_not_invalidate(self): + class A(object): + pass + class B(object): + pass + + def f(i): + a = A() + a.x = i + b = B() + return a.x + + self.check(f, [int], 0) + diff --git a/rpython/translator/driver.py b/rpython/translator/driver.py --- a/rpython/translator/driver.py +++ b/rpython/translator/driver.py @@ -481,6 +481,14 @@ libname = str(newsoname.dirpath().join('python27.lib')) shutil.copyfile(str(soname.new(ext='lib')), libname) self.log.info("copied: %s" % (libname,)) + # XXX TODO : replace the nonsense above with + # ext_to_copy = ['lib', 'pdb'] + ext_to_copy = ['pdb',] + for ext in ext_to_copy: + name = soname.new(ext=ext) + newname = newexename.new(basename=soname.basename) + shutil.copyfile(str(name), str(newname.new(ext=ext))) + self.log.info("copied: %s" % (newname,)) self.c_entryp = newexename self.log.info('usession directory: %s' % (udir,)) self.log.info("created: %s" % (self.c_entryp,)) diff --git a/rpython/translator/platform/test/test_makefile.py b/rpython/translator/platform/test/test_makefile.py --- a/rpython/translator/platform/test/test_makefile.py +++ b/rpython/translator/platform/test/test_makefile.py @@ -44,6 +44,7 @@ assert res.returncode == 0 def test_900_files(self): + tmpdir = udir.join('test_900_files').ensure(dir=1) txt = '#include \n' for i in range(900): txt += 'int func%03d();\n' % i @@ -52,11 +53,11 @@ txt += ' j += func%03d();\n' % i txt += ' printf("%d\\n", j);\n' txt += ' return 0;};\n' - cfile = udir.join('test_900_files.c') + cfile = tmpdir.join('test_900_files.c') cfile.write(txt) cfiles = [cfile] for i in range(900): - cfile2 = udir.join('implement%03d.c' %i) + cfile2 = tmpdir.join('implement%03d.c' %i) cfile2.write(''' int func%03d() { @@ -64,10 +65,10 @@ } ''' % (i, i)) cfiles.append(cfile2) - mk = self.platform.gen_makefile(cfiles, ExternalCompilationInfo(), path=udir) + mk = self.platform.gen_makefile(cfiles, ExternalCompilationInfo(), path=tmpdir) mk.write() self.platform.execute_makefile(mk) - res = self.platform.execute(udir.join('test_900_files')) + res = self.platform.execute(tmpdir.join('test_900_files')) self.check_res(res, '%d\n' %sum(range(900))) def test_precompiled_headers(self): diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -204,8 +204,9 @@ # must come first, and after the file name all options are ignored. # So please be careful with the order of parameters! ;-) pdb_dir = oname.dirname - args = ['/nologo', '/c'] + compile_args + ['/Fd%s\\' % (pdb_dir,), - '/Fo%s' % (oname,), str(cfile)] + if pdb_dir: + compile_args += ['/Fd%s\\' % (pdb_dir,)] + args = ['/nologo', '/c'] + compile_args + ['/Fo%s' % (oname,), str(cfile)] self._execute_c_compiler(cc, args, oname) return oname @@ -347,7 +348,7 @@ '$(CREATE_PCH) $(INCLUDEDIRS)')) rules.append(('.c.obj', '', '$(CC) /nologo $(CFLAGS) $(CFLAGSEXTRA) $(USE_PCH) ' - '/Fd$(@D)\\ /Fo$@ /c $< $(INCLUDEDIRS)')) + '/Fo$@ /c $< $(INCLUDEDIRS)')) #Do not use precompiled headers for some files #rules.append((r'{..\module_cache}.c{..\module_cache}.obj', '', # '$(CC) /nologo $(CFLAGS) $(CFLAGSEXTRA) /Fo$@ /c $< $(INCLUDEDIRS)')) @@ -362,13 +363,12 @@ target = f[:-1] + 'obj' rules.append((target, f, '$(CC) /nologo $(CFLAGS) $(CFLAGSEXTRA) ' - '/Fd%s\\ /Fo%s /c %s $(INCLUDEDIRS)' %( - os.path.dirname(target), target, f))) + '/Fo%s /c %s $(INCLUDEDIRS)' %(target, f))) else: rules.append(('.c.obj', '', '$(CC) /nologo $(CFLAGS) $(CFLAGSEXTRA) ' - '/Fd$(@D)\\ /Fo$@ /c $< $(INCLUDEDIRS)')) + '/Fo$@ /c $< $(INCLUDEDIRS)')) for args in definitions: @@ -410,7 +410,7 @@ 'int main(int argc, char* argv[]) ' '{ return $(PYPY_MAIN_FUNCTION)(argc, argv); } > $@') m.rule('$(DEFAULT_TARGET)', ['$(TARGET)', 'main.obj'], - ['$(CC_LINK) /nologo main.obj $(SHARED_IMPORT_LIB) /out:$@ /MANIFEST /MANIFESTFILE:$*.manifest', + ['$(CC_LINK) /nologo /debug main.obj $(SHARED_IMPORT_LIB) /out:$@ /MANIFEST /MANIFESTFILE:$*.manifest', 'mt.exe -nologo -manifest $*.manifest -outputresource:$@;1', ]) m.rule('debugmode_$(DEFAULT_TARGET)', ['debugmode_$(TARGET)', 'main.obj'], diff --git a/rpython/translator/sandbox/rsandbox.py b/rpython/translator/sandbox/rsandbox.py --- a/rpython/translator/sandbox/rsandbox.py +++ b/rpython/translator/sandbox/rsandbox.py @@ -60,8 +60,7 @@ def need_more_data(self): buflen = self.buflen - buf = lltype.malloc(rffi.CCHARP.TO, buflen, flavor='raw') - try: + with lltype.scoped_alloc(rffi.CCHARP.TO, buflen) as buf: buflen = rffi.cast(rffi.SIZE_T, buflen) count = ll_read_not_sandboxed(self.fd, buf, buflen) count = rffi.cast(lltype.Signed, count) @@ -69,20 +68,15 @@ raise IOError self.buf += ''.join([buf[i] for i in range(count)]) self.buflen *= 2 - finally: - lltype.free(buf, flavor='raw') def sandboxed_io(buf): STDIN = 0 STDOUT = 1 # send the buffer with the marshalled fnname and input arguments to STDOUT - p = lltype.malloc(rffi.CCHARP.TO, len(buf), flavor='raw') - try: + with lltype.scoped_alloc(rffi.CCHARP.TO, len(buf)) as p: for i in range(len(buf)): p[i] = buf[i] writeall_not_sandboxed(STDOUT, p, len(buf)) - finally: - lltype.free(p, flavor='raw') # build a Loader that will get the answer from STDIN loader = FdLoader(STDIN) # check for errors @@ -108,9 +102,8 @@ @signature(types.str(), returns=types.impossible()) def not_implemented_stub(msg): STDERR = 2 - buf = rffi.str2charp(msg + '\n') - writeall_not_sandboxed(STDERR, buf, len(msg) + 1) - rffi.free_charp(buf) + with rffi.scoped_str2charp(msg + '\n') as buf: + writeall_not_sandboxed(STDERR, buf, len(msg) + 1) raise RuntimeError(msg) # XXX in RPython, the msg is ignored at the moment dump_string = rmarshal.get_marshaller(str) diff --git a/rpython/translator/sandbox/sandlib.py b/rpython/translator/sandbox/sandlib.py --- a/rpython/translator/sandbox/sandlib.py +++ b/rpython/translator/sandbox/sandlib.py @@ -459,6 +459,15 @@ do_ll_os__ll_os_lstat = do_ll_os__ll_os_stat + def do_ll_os__ll_os_access(self, vpathname, mode): + try: + node = self.get_node(vpathname) + except OSError, e: + if e.errno == errno.ENOENT: + return False + raise + return node.access(mode) + def do_ll_os__ll_os_isatty(self, fd): return self.virtual_console_isatty and fd in (0, 1, 2) diff --git a/rpython/translator/sandbox/test/test_vfs.py b/rpython/translator/sandbox/test/test_vfs.py --- a/rpython/translator/sandbox/test/test_vfs.py +++ b/rpython/translator/sandbox/test/test_vfs.py @@ -33,6 +33,8 @@ py.test.raises(OSError, d.join, 'bar') st = d.stat() assert stat.S_ISDIR(st.st_mode) + assert d.access(os.R_OK | os.X_OK) + assert not d.access(os.W_OK) def test_file(): f = File('hello world') @@ -46,6 +48,8 @@ st = f.stat() assert stat.S_ISREG(st.st_mode) assert st.st_size == 11 + assert f.access(os.R_OK) + assert not f.access(os.W_OK) def test_realdir_realfile(): for show_dotfiles in [False, True]: @@ -78,6 +82,7 @@ f = v_test_vfs.join('symlink2') assert stat.S_ISREG(f.stat().st_mode) + assert f.access(os.R_OK) assert f.open().read() == 'secret' else: py.test.raises(OSError, v_test_vfs.join, 'symlink1') diff --git a/rpython/translator/sandbox/vfs.py b/rpython/translator/sandbox/vfs.py --- a/rpython/translator/sandbox/vfs.py +++ b/rpython/translator/sandbox/vfs.py @@ -22,7 +22,7 @@ st_size = self.getsize() st_mode = self.kind st_mode |= stat.S_IWUSR | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH - if self.kind == stat.S_IFDIR: + if stat.S_ISDIR(self.kind): st_mode |= stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH if self.read_only: st_uid = 0 # read-only files are virtually owned by root @@ -37,6 +37,15 @@ (st_mode, st_ino, st_dev, st_nlink, st_uid, st_gid, st_size, st_atime, st_mtime, st_ctime)) + def access(self, mode): + s = self.stat() + e_mode = s.st_mode & stat.S_IRWXO + if UID == s.st_uid: + e_mode |= (s.st_mode & stat.S_IRWXU) >> 6 + if GID == s.st_gid: + e_mode |= (s.st_mode & stat.S_IRWXG) >> 3 + return (e_mode & mode) == mode + def keys(self): raise OSError(errno.ENOTDIR, self) @@ -114,8 +123,9 @@ return cStringIO.StringIO(self.data) class RealFile(File): - def __init__(self, path): + def __init__(self, path, mode=0): self.path = path + self.kind |= mode def __repr__(self): return '' % (self.path,) def getsize(self): diff --git a/rpython/translator/test/test_driver.py b/rpython/translator/test/test_driver.py --- a/rpython/translator/test/test_driver.py +++ b/rpython/translator/test/test_driver.py @@ -55,12 +55,15 @@ src_name = udir.join('src/dydy2.exe') dll_name = udir.join('src/pypy.dll') lib_name = udir.join('src/pypy.lib') + pdb_name = udir.join('src/pypy.pdb') src_name.ensure() src_name.write('exe') dll_name.ensure() dll_name.write('dll') lib_name.ensure() lib_name.write('lib') + pdb_name.ensure() + pdb_name.write('pdb') dst_name.ensure() class CBuilder(object): @@ -75,6 +78,8 @@ assert dst_name.new(purebasename='python27',ext='lib').read() == 'lib' def test_shutil_copy(): + if os.name == 'nt': + py.test.skip('Windows cannot copy or rename to an in-use file') a = udir.join('file_a') b = udir.join('file_a') a.write('hello') From noreply at buildbot.pypy.org Wed Sep 24 15:47:57 2014 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 24 Sep 2014 15:47:57 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: Write this test Message-ID: <20140924134757.A735D1C054C@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: gc_no_cleanup_nursery Changeset: r73673:419bb943f8e2 Date: 2014-09-24 15:46 +0200 http://bitbucket.org/pypy/pypy/changeset/419bb943f8e2/ Log: Write this test diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py --- a/rpython/jit/backend/test/runner_test.py +++ b/rpython/jit/backend/test/runner_test.py @@ -4480,7 +4480,24 @@ assert not s.x def test_zero_ptr_field(self): - XXX # write me! + from rpython.jit.backend.llsupport import symbolic + S = lltype.GcStruct('S', ('x', lltype.Signed), + ('p', llmemory.GCREF), + ('y', lltype.Signed)) + s = lltype.malloc(S) + s.x = -1296321 + s.y = -4398176 + s_ref = lltype.cast_opaque_ptr(llmemory.GCREF, s) + s.p = s_ref + ofs_p, _ = symbolic.get_field_token(S, 'p', False) + # + self.execute_operation(rop.ZERO_PTR_FIELD, [ + BoxPtr(s_ref), ConstInt(ofs_p)], # OK for now to assume that the + 'void') # 2nd argument is a constant + # + assert s.x == -1296321 + assert s.p == lltype.nullptr(llmemory.GCREF.TO) + assert s.y == -4398176 def test_zero_array(self): PAIR = lltype.Struct('PAIR', ('a', lltype.Signed), ('b', lltype.Signed)) From noreply at buildbot.pypy.org Wed Sep 24 15:51:16 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Wed, 24 Sep 2014 15:51:16 +0200 (CEST) Subject: [pypy-commit] stmgc c8-new-page-handling: another issue of go_to_the_future Message-ID: <20140924135116.B47381C03FC@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: c8-new-page-handling Changeset: r1424:d68c60689ca5 Date: 2014-09-24 15:45 +0200 http://bitbucket.org/pypy/stmgc/changeset/d68c60689ca5/ Log: another issue of go_to_the_future diff --git a/c8/test/test_basic.py b/c8/test/test_basic.py --- a/c8/test/test_basic.py +++ b/c8/test/test_basic.py @@ -252,7 +252,8 @@ py.test.raises(Conflict, self.commit_transaction) assert res == 'a' - def test_read_write_15(self): + @py.test.mark.parametrize("only_bk", [0, 1]) + def test_read_write_15(self, only_bk): lp1 = stm_allocate_old(16) # allocated in seg0 lp2 = stm_allocate_old(16) # allocated in seg0 stm_get_real_address(lp1)[HDR] = 'a' #setchar @@ -270,16 +271,18 @@ stm_set_char(lp2, 'C') self.commit_transaction() # R1 assert last_commit_log_entries() == [lp2] # commit 'C' - self.start_transaction() - stm_set_char(lp2, 'c') # R1.1 + if only_bk: + self.start_transaction() + stm_set_char(lp2, 'c') # R1.1 # self.switch(2) self.start_transaction() stm_set_char(lp1, 'D') self.commit_transaction() # R2 assert last_commit_log_entries() == [lp1] # commit 'D' - self.start_transaction() - stm_set_char(lp1, 'd') # R2.1 + if only_bk: + self.start_transaction() + stm_set_char(lp1, 'd') # R2.1 # self.switch(3) self.start_transaction() # stm_validate() -> R2 @@ -289,6 +292,55 @@ # + def test_read_write_16(self): + lp1 = stm_allocate_old(16) # allocated in seg0 + lp2 = stm_allocate_old(16) # allocated in seg0 + stm_get_real_address(lp1)[HDR] = 'a' #setchar + stm_get_real_address(lp2)[HDR] = 'b' #setchar + # S|P|P|P|P + # + # NO_ACCESS in all segments except seg0 (shared page holder) + # + # all seg at R0 + # + self.start_transaction() + # + self.switch(1) # with private page + self.start_transaction() + stm_set_char(lp2, 'C') + self.commit_transaction() # R1 + assert last_commit_log_entries() == [lp2] # commit 'C' + # + self.switch(2) + self.start_transaction() + stm_set_char(lp1, 'D') + self.commit_transaction() # R2 + assert last_commit_log_entries() == [lp1] # commit 'D' + # + self.switch(3) + self.start_transaction() # stm_validate() -> R2 + assert stm_get_char(lp1) == 'D' # R2 + # + self.switch(2) + self.start_transaction() + stm_set_char(lp1, 'I') + self.commit_transaction() # R2 + assert last_commit_log_entries() == [lp1] # commit 'I' + # + self.switch(1) + self.start_transaction() + stm_set_char(lp2, 'H') + self.commit_transaction() # R3 + assert last_commit_log_entries() == [lp2] # commit 'H' + # + self.switch(3, validate=False) # R2 again + assert stm_get_char(lp1) == 'D' # R2 + assert stm_get_char(lp2) == 'C' # R2 + py.test.raises(Conflict, self.commit_transaction) + # + + + def test_commit_fresh_objects(self): self.start_transaction() lp = stm_allocate(16) From noreply at buildbot.pypy.org Wed Sep 24 15:51:17 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Wed, 24 Sep 2014 15:51:17 +0200 (CEST) Subject: [pypy-commit] stmgc c8-new-page-handling: fix go_to_the_future again Message-ID: <20140924135117.C38851C03FC@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: c8-new-page-handling Changeset: r1425:b30add00d261 Date: 2014-09-24 15:51 +0200 http://bitbucket.org/pypy/stmgc/changeset/b30add00d261/ Log: fix go_to_the_future again diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -57,27 +57,6 @@ import_objects(-1, pagenum, undo, end); } - -static void go_to_the_future(uintptr_t pagenum, - struct stm_commit_log_entry_s *from, - struct stm_commit_log_entry_s *to) -{ - if (from == to) - return; - - /* walk FORWARDS the commit log and update the page 'pagenum', - initially at revision 'from', until we reach the revision 'to'. */ - while (from != to) { - from = from->next; - - struct stm_undo_s *undo = from->written; - struct stm_undo_s *end = from->written + from->written_count; - - import_objects(from->segment_num, pagenum, undo, end); - copy_bk_objs_in_page_from(from->segment_num, pagenum); - } -} - static void go_to_the_past(uintptr_t pagenum, struct stm_commit_log_entry_s *from, struct stm_commit_log_entry_s *to) @@ -97,6 +76,50 @@ } } + +static void go_to_the_future(uintptr_t pagenum, + struct stm_commit_log_entry_s *from, + struct stm_commit_log_entry_s *to) +{ + if (from == to) + return; + + /* XXX: specialize. We now go to the HEAD revision, and back again + to where we want. Otherwise, we have to look at backup copies in + the log entries, modified objs, page content and their revisions... + + Same logic as _stm_validate() */ + + /* XXXXXXXXXXXXXXXX CORRECT LOCKING NEEDED XXXXXXXXXXXXXXXXXXXXXX */ + struct stm_commit_log_entry_s *cl = from; + struct stm_commit_log_entry_s *next_cl; + + uint64_t segment_copied_from = 0; + while ((next_cl = cl->next) != NULL) { + if (next_cl == (void *)-1) + break; + + cl = next_cl; + + struct stm_undo_s *undo = cl->written; + struct stm_undo_s *end = cl->written + cl->written_count; + + segment_copied_from |= (1UL << cl->segment_num); + import_objects(cl->segment_num, -1, undo, end); + } + + /* XXXXXXXXXXXXXXXX CORRECT LOCKING NEEDED XXXXXXXXXXXXXXXXXXXXXX */ + int segnum; + for (segnum = 0; segment_copied_from != 0; segnum++) { + if (segment_copied_from & (1UL << segnum)) { + segment_copied_from &= ~(1UL << segnum); + copy_bk_objs_in_page_from(segnum, -1); + } + } + + go_to_the_past(pagenum, cl, to); +} + static void handle_segfault_in_page(uintptr_t pagenum) { /* assumes page 'pagenum' is ACCESS_NONE, privatizes it, From noreply at buildbot.pypy.org Wed Sep 24 16:45:07 2014 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 24 Sep 2014 16:45:07 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: arm: zero_ptr_field Message-ID: <20140924144507.066DC1C054C@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: gc_no_cleanup_nursery Changeset: r73674:712ee4b2bdcf Date: 2014-09-24 14:42 +0000 http://bitbucket.org/pypy/pypy/changeset/712ee4b2bdcf/ Log: arm: zero_ptr_field diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py --- a/rpython/jit/backend/arm/opassembler.py +++ b/rpython/jit/backend/arm/opassembler.py @@ -578,6 +578,7 @@ return fcond emit_op_setfield_raw = emit_op_setfield_gc + emit_op_zero_ptr_field = emit_op_setfield_gc def emit_op_getfield_gc(self, op, arglocs, regalloc, fcond): base_loc, ofs, res, size = arglocs diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py --- a/rpython/jit/backend/arm/regalloc.py +++ b/rpython/jit/backend/arm/regalloc.py @@ -818,8 +818,11 @@ def prepare_op_setfield_gc(self, op, fcond): boxes = op.getarglist() + ofs, size, sign = unpack_fielddescr(op.getdescr()) + return self._prepare_op_setfield(boxes, ofs, size) + + def _prepare_op_setfield(self, boxes, ofs, size): a0, a1 = boxes - ofs, size, sign = unpack_fielddescr(op.getdescr()) base_loc = self.make_sure_var_in_reg(a0, boxes) value_loc = self.make_sure_var_in_reg(a1, boxes) ofs_size = default_imm_size if size < 8 else VMEM_imm_size @@ -832,6 +835,11 @@ prepare_op_setfield_raw = prepare_op_setfield_gc + def prepare_op_zero_ptr_field(self, op, fcond): + a0 = op.getarg(0) + ofs = op.getarg(1).getint() + return self._prepare_op_setfield([a0, ConstInt(0)], ofs, WORD) + def prepare_op_getfield_gc(self, op, fcond): a0 = op.getarg(0) ofs, size, sign = unpack_fielddescr(op.getdescr()) From noreply at buildbot.pypy.org Wed Sep 24 17:53:27 2014 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 24 Sep 2014 17:53:27 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: fight fight fight until the tests start oassinbg Message-ID: <20140924155327.C24CC1C0DCA@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73675:c6ba1b273ac0 Date: 2014-09-24 17:52 +0200 http://bitbucket.org/pypy/pypy/changeset/c6ba1b273ac0/ Log: fight fight fight until the tests start oassinbg diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py --- a/rpython/jit/backend/llgraph/runner.py +++ b/rpython/jit/backend/llgraph/runner.py @@ -165,6 +165,9 @@ def is_array_of_structs(self): return isinstance(self.A.OF, lltype.Struct) + def get_fielddescrs(self): + return self.fielddescrs + def is_item_integer_bounded(self): return getkind(self.A.OF) == 'int' \ and rffi.sizeof(self.A.OF) < symbolic.WORD @@ -225,6 +228,19 @@ 'i': 0, 'f': 0.0} +def fielddescrs_for(cpu, STRUCT, res=None): + if res is None: + res = [] + # order is not relevant, except for tests + for name in STRUCT._names: + FIELD = getattr(STRUCT, name) + if isinstance(FIELD, lltype.Struct): + fielddescrs_for(cpu, FIELD, res) + else: + res.append(cpu.fielddescrof(STRUCT, name)) + return res + + class LLGraphCPU(model.AbstractCPU): from rpython.jit.metainterp.typesystem import llhelper as ts supports_floats = True @@ -392,6 +408,8 @@ except KeyError: descr = ArrayDescr(A) self.descrs[key] = descr + if descr.is_array_of_structs(): + descr.fielddescrs = fielddescrs_for(self, A.OF) return descr def interiorfielddescrof(self, A, fieldname): diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py --- a/rpython/jit/backend/llsupport/descr.py +++ b/rpython/jit/backend/llsupport/descr.py @@ -36,10 +36,10 @@ tid = llop.combine_ushort(lltype.Signed, 0, 0) def __init__(self, size, count_fields_if_immut=-1, - offsets_of_gcfields=None): + fielddescrs=None): self.size = size self.count_fields_if_immut = count_fields_if_immut - self.offsets_of_gcfields = offsets_of_gcfields + self.fielddescrs = fielddescrs def count_fields_if_immutable(self): return self.count_fields_if_immut @@ -60,13 +60,13 @@ except KeyError: size = symbolic.get_size(STRUCT, gccache.translate_support_code) count_fields_if_immut = heaptracker.count_fields_if_immutable(STRUCT) - offsets_of_gcfields = heaptracker.offsets_of_gcfields(gccache, STRUCT) + fielddescrs = heaptracker.fielddescrs(gccache, STRUCT) if heaptracker.has_gcstruct_a_vtable(STRUCT): sizedescr = SizeDescrWithVTable(size, count_fields_if_immut, - offsets_of_gcfields) + fielddescrs) else: sizedescr = SizeDescr(size, count_fields_if_immut, - offsets_of_gcfields) + fielddescrs) gccache.init_size_descr(STRUCT, sizedescr) cache[STRUCT] = sizedescr return sizedescr @@ -192,6 +192,7 @@ lendescr = None flag = '\x00' vinfo = None + fielddescrs = None def __init__(self, basesize, itemsize, lendescr, flag): self.basesize = basesize @@ -215,6 +216,9 @@ return self.flag in (FLAG_SIGNED, FLAG_UNSIGNED) \ and self.itemsize < symbolic.WORD + def get_fielddescrs(self): + return self.fielddescrs + def get_item_integer_min(self): if self.flag == FLAG_UNSIGNED: return intbounds.get_integer_min(True, self.itemsize) @@ -254,6 +258,9 @@ lendescr = get_field_arraylen_descr(gccache, ARRAY_OR_STRUCT) flag = get_type_flag(ARRAY_INSIDE.OF) arraydescr = ArrayDescr(basesize, itemsize, lendescr, flag) + if arraydescr.is_array_of_structs(): + arraydescr.fielddescrs = heaptracker.fielddescrs(gccache, + ARRAY_OR_STRUCT.OF) if ARRAY_OR_STRUCT._gckind == 'gc': gccache.init_array_descr(ARRAY_OR_STRUCT, arraydescr) cache[ARRAY_OR_STRUCT] = arraydescr diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -1,6 +1,6 @@ from rpython.rlib import rgc from rpython.rlib.rarithmetic import ovfcheck -from rpython.rtyper.lltypesystem import llmemory +from rpython.rtyper.lltypesystem import llmemory, lltype from rpython.jit.metainterp import history from rpython.jit.metainterp.history import ConstInt, BoxPtr, ConstPtr, BoxInt from rpython.jit.metainterp.resoperation import ResOperation, rop @@ -39,6 +39,7 @@ _op_malloc_nursery = None _v_last_malloced_nursery = None c_zero = ConstInt(0) + c_null = ConstPtr(lltype.nullptr(llmemory.GCREF.TO)) def __init__(self, gc_ll_descr, cpu): self.gc_ll_descr = gc_ll_descr @@ -137,8 +138,10 @@ except KeyError: d = {} self.delayed_zero_setfields[result] = d - for ofs in descr.offsets_of_gcfields: - d[ofs] = None + for fielddescr in descr.fielddescrs: + if fielddescr.is_pointer_field(): + ofs = self.cpu.unpack_fielddescr(fielddescr) + d[ofs] = None def consider_setfield_gc(self, op): offset = self.cpu.unpack_fielddescr(op.getdescr()) @@ -256,15 +259,15 @@ length_box, descr=descrs.jfi_frame_depth), ResOperation(rop.SETFIELD_GC, [frame, self.c_zero], None, descr=descrs.jf_extra_stack_depth), - ResOperation(rop.SETFIELD_GC, [frame, self.c_zero], + ResOperation(rop.SETFIELD_GC, [frame, self.c_null], None, descr=descrs.jf_savedata), - ResOperation(rop.SETFIELD_GC, [frame, self.c_zero], + ResOperation(rop.SETFIELD_GC, [frame, self.c_null], None, descr=descrs.jf_force_descr), - ResOperation(rop.SETFIELD_GC, [frame, self.c_zero], + ResOperation(rop.SETFIELD_GC, [frame, self.c_null], None, descr=descrs.jf_descr), - ResOperation(rop.SETFIELD_GC, [frame, self.c_zero], + ResOperation(rop.SETFIELD_GC, [frame, self.c_null], None, descr=descrs.jf_guard_exc), - ResOperation(rop.SETFIELD_GC, [frame, self.c_zero], + ResOperation(rop.SETFIELD_GC, [frame, self.c_null], None, descr=descrs.jf_forward), ] self.newops += extra_ops diff --git a/rpython/jit/backend/llsupport/test/test_descr.py b/rpython/jit/backend/llsupport/test/test_descr.py --- a/rpython/jit/backend/llsupport/test/test_descr.py +++ b/rpython/jit/backend/llsupport/test/test_descr.py @@ -19,9 +19,8 @@ assert descr_t.size == symbolic.get_size(T, False) assert descr_s.count_fields_if_immutable() == -1 assert descr_t.count_fields_if_immutable() == -1 - assert descr_t.offsets_of_gcfields == [] - assert descr_s.offsets_of_gcfields == [ - symbolic.get_field_token(S, 'y', False)[0]] + assert descr_t.fielddescrs == [] + assert len(descr_s.fielddescrs) == 2 assert descr_s == get_size_descr(c0, S) assert descr_s != get_size_descr(c1, S) # @@ -32,10 +31,7 @@ PARENT = lltype.Struct('P', ('x', lltype.Ptr(T))) STRUCT = lltype.GcStruct('S', ('parent', PARENT), ('y', lltype.Ptr(T))) descr_struct = get_size_descr(c0, STRUCT) - assert descr_struct.offsets_of_gcfields == [ - symbolic.get_field_token(PARENT, 'x', False)[0], - symbolic.get_field_token(STRUCT, 'y', False)[0], - ] + assert len(descr_struct.fielddescrs) == 2 def test_get_size_descr_immut(): S = lltype.GcStruct('S', hints={'immutable': True}) diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py --- a/rpython/jit/backend/llsupport/test/test_rewrite.py +++ b/rpython/jit/backend/llsupport/test/test_rewrite.py @@ -84,6 +84,11 @@ jfi_frame_depth = framedescrs.jfi_frame_depth jfi_frame_size = framedescrs.jfi_frame_size jf_frame_info = framedescrs.jf_frame_info + jf_savedata = framedescrs.jf_savedata + jf_force_descr = framedescrs.jf_force_descr + jf_descr = framedescrs.jf_descr + jf_guard_exc = framedescrs.jf_guard_exc + jf_forward = framedescrs.jf_forward jf_extra_stack_depth = framedescrs.jf_extra_stack_depth signedframedescr = self.cpu.signedframedescr floatframedescr = self.cpu.floatframedescr @@ -149,7 +154,7 @@ def setup_method(self, meth): class FakeCPU(BaseFakeCPU): def sizeof(self, STRUCT): - return SizeDescrWithVTable(102, offsets_of_gcfields=[]) + return SizeDescrWithVTable(102, fielddescrs=[]) self.cpu = FakeCPU() self.gc_ll_descr = GcLLDescr_boehm(None, None, None) @@ -286,7 +291,7 @@ # class FakeCPU(BaseFakeCPU): def sizeof(self, STRUCT): - descr = SizeDescrWithVTable(104, offsets_of_gcfields=[]) + descr = SizeDescrWithVTable(104, fielddescrs=[]) descr.tid = 9315 return descr self.cpu = FakeCPU() @@ -319,7 +324,7 @@ setfield_gc(p1, 5678, descr=tiddescr) p2 = int_add(p1, %(tdescr.size)d) setfield_gc(p2, 1234, descr=tiddescr) - zero_ptr_field(p1, %(tdescr.offsets_of_gcfields[0])s) + zero_ptr_field(p1, %(tdescr.fielddescrs[1].offset)s) jump() """) @@ -767,7 +772,7 @@ [i0] p0 = call_malloc_nursery(%(tdescr.size)d) setfield_gc(p0, 5678, descr=tiddescr) - zero_ptr_field(p0, %(tdescr.offsets_of_gcfields[0])s) + zero_ptr_field(p0, %(tdescr.fielddescrs[1].offset)s) p1 = call_malloc_nursery_varsize(1, 1, i0, \ descr=strdescr) setfield_gc(p1, i0, descr=strlendescr) @@ -788,7 +793,7 @@ [p1] p0 = call_malloc_nursery(%(tdescr.size)d) setfield_gc(p0, 5678, descr=tiddescr) - zero_ptr_field(p0, %(tdescr.offsets_of_gcfields[0])s) + zero_ptr_field(p0, %(tdescr.fielddescrs[1].offset)s) label(p0, p1) cond_call_gc_wb(p0, descr=wbdescr) setfield_gc(p0, p1, descr=tzdescr) @@ -820,6 +825,11 @@ setfield_gc(p1, 0, descr=tiddescr) i2 = getfield_gc(ConstClass(frame_info), descr=jfi_frame_depth) setfield_gc(p1, 0, descr=jf_extra_stack_depth) + setfield_gc(p1, NULL, descr=jf_savedata) + setfield_gc(p1, NULL, descr=jf_force_descr) + setfield_gc(p1, NULL, descr=jf_descr) + setfield_gc(p1, NULL, descr=jf_guard_exc) + setfield_gc(p1, NULL, descr=jf_forward) setfield_gc(p1, i2, descr=framelendescr) setfield_gc(p1, ConstClass(frame_info), descr=jf_frame_info) setarrayitem_gc(p1, 0, i0, descr=signedframedescr) diff --git a/rpython/jit/codewriter/heaptracker.py b/rpython/jit/codewriter/heaptracker.py --- a/rpython/jit/codewriter/heaptracker.py +++ b/rpython/jit/codewriter/heaptracker.py @@ -126,18 +126,16 @@ vtable = llmemory.cast_ptr_to_adr(vtable) return adr2int(vtable) -def offsets_of_gcfields(gccache, STRUCT, res=None): - from rpython.jit.backend.llsupport import symbolic +def fielddescrs(gccache, STRUCT, res=None): + from rpython.jit.backend.llsupport import descr if res is None: res = [] # order is not relevant, except for tests for name in STRUCT._names: FIELD = getattr(STRUCT, name) - if isinstance(FIELD, lltype.Ptr) and FIELD._needsgc(): - offset, _ = symbolic.get_field_token(STRUCT, name, - gccache.translate_support_code) - res.append(offset) - elif isinstance(FIELD, lltype.Struct): - offsets_of_gcfields(gccache, FIELD, res) + if isinstance(FIELD, lltype.Struct): + fielddescrs(gccache, FIELD, res) + else: + res.append(descr.get_field_descr(gccache, STRUCT, name)) return res diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py --- a/rpython/jit/metainterp/optimizeopt/rewrite.py +++ b/rpython/jit/metainterp/optimizeopt/rewrite.py @@ -502,6 +502,8 @@ descr=arraydescr) self.optimizer.send_extra_operation(newop) val = self.getvalue(resbox) + if val is None: + continue if dest_value.is_virtual(): dest_value.setitem(index + dest_start, val) else: diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py @@ -1135,12 +1135,12 @@ [i1] p1 = new_array(2, descr=arraydescr) setarrayitem_gc(p1, 0, 25, descr=arraydescr) - i2 = getarrayitem_gc(p1, 1, descr=arraydescr) + i2 = getarrayitem_gc(p1, 0, descr=arraydescr) jump(i2) """ expected = """ [i1] - jump(0) + jump(25) """ self.optimize_loop(ops, expected) @@ -2977,6 +2977,7 @@ [p1] p0 = force_token() p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) + setfield_gc(p2, NULL, descr=virtualforceddescr) setfield_gc(p2, p0, descr=virtualtokendescr) escape(p2) setfield_gc(p2, p1, descr=virtualforceddescr) @@ -3009,6 +3010,7 @@ p3 = force_token() # p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) + setfield_gc(p2, NULL, descr=virtualforceddescr) setfield_gc(p2, p3, descr=virtualtokendescr) setfield_gc(p0, p2, descr=nextdescr) # @@ -3048,6 +3050,7 @@ p3 = force_token() # p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) + setfield_gc(p2, NULL, descr=virtualforceddescr) setfield_gc(p2, p3, descr=virtualtokendescr) setfield_gc(p0, p2, descr=nextdescr) # @@ -3124,6 +3127,7 @@ [i1] p3 = force_token() p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) + setfield_gc(p2, NULL, descr=virtualforceddescr) setfield_gc(p2, p3, descr=virtualtokendescr) escape(p2) p1 = new_with_vtable(ConstClass(node_vtable)) @@ -3149,6 +3153,7 @@ [i1, p1] p3 = force_token() p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) + setfield_gc(p2, NULL, descr=virtualforceddescr) setfield_gc(p2, p3, descr=virtualtokendescr) escape(p2) setfield_gc(p2, p1, descr=virtualforceddescr) @@ -5472,6 +5477,22 @@ """ self.optimize_loop(ops, expected) + def test_virtual_clear_array_contents_escape(self): + ops = """ + [] + p0 = new_array(2, descr=arraydescr) + clear_array_contents(p0, descr=arraydescr) + escape(p0) + """ + expected = """ + [] + p0 = new_array(2, descr=arraydescr) + setarrayitem_gc(p0, 0, 0, descr=arraydescr) + setarrayitem_gc(p0, 1, 0, descr=arraydescr) + escape(p0) + """ + self.optimize_loop(ops, expected) + class TestLLtype(BaseTestOptimizeBasic, LLtypeMixin): pass diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -1563,7 +1563,7 @@ [i1] p1 = new_array(2, descr=arraydescr) setarrayitem_gc(p1, 0, 25, descr=arraydescr) - i2 = getarrayitem_gc(p1, 1, descr=arraydescr) + i2 = getarrayitem_gc(p1, 0, descr=arraydescr) jump(i2) """ preamble = """ @@ -3751,6 +3751,7 @@ [p1] p0 = force_token() p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) + setfield_gc(p2, NULL, descr=virtualforceddescr) setfield_gc(p2, p0, descr=virtualtokendescr) escape(p2) setfield_gc(p2, p1, descr=virtualforceddescr) @@ -3783,6 +3784,7 @@ p3 = force_token() # p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) + setfield_gc(p2, NULL, descr=virtualforceddescr) setfield_gc(p2, p3, descr=virtualtokendescr) setfield_gc(p0, p2, descr=nextdescr) # @@ -3822,6 +3824,7 @@ p3 = force_token() # p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) + setfield_gc(p2, NULL, descr=virtualforceddescr) setfield_gc(p2, p3, descr=virtualtokendescr) setfield_gc(p0, p2, descr=nextdescr) # @@ -3909,6 +3912,7 @@ [i1] p3 = force_token() p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) + setfield_gc(p2, NULL, descr=virtualforceddescr) setfield_gc(p2, p3, descr=virtualtokendescr) escape(p2) p1 = new_with_vtable(ConstClass(node_vtable)) @@ -3934,6 +3938,7 @@ [i1, p1] p3 = force_token() p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) + setfield_gc(p2, NULL, descr=virtualforceddescr) setfield_gc(p2, p3, descr=virtualtokendescr) escape(p2) setfield_gc(p2, p1, descr=virtualforceddescr) @@ -4019,6 +4024,7 @@ ops = ''' [] p1 = new_array(3, descr=arraydescr) + clear_array_contents(p1, descr=arraydescr) p2 = new_array(3, descr=arraydescr) setarrayitem_gc(p1, 2, 10, descr=arraydescr) setarrayitem_gc(p2, 2, 13, descr=arraydescr) diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py --- a/rpython/jit/metainterp/optimizeopt/virtualize.py +++ b/rpython/jit/metainterp/optimizeopt/virtualize.py @@ -253,11 +253,13 @@ itemboxes = [] for i in range(self.getlength()): itemvalue = self.get_item_value(i) - itemboxes.append(itemvalue.get_key_box()) + if itemvalue is not None: + itemboxes.append(itemvalue.get_key_box()) visitor.register_virtual_fields(self.keybox, itemboxes) for i in range(self.getlength()): itemvalue = self.get_item_value(i) - itemvalue.visitor_walk_recursive(visitor) + if itemvalue is not None: + itemvalue.visitor_walk_recursive(visitor) class VArrayValue(AbstractVArrayValue): @@ -266,7 +268,7 @@ AbstractVirtualValue.__init__(self, keybox, source_op) self.arraydescr = arraydescr self.constvalue = constvalue - self._items = [self.constvalue] * size + self._items = [None] * size def getlength(self): return len(self._items) @@ -277,6 +279,10 @@ def set_item_value(self, i, newval): self._items[i] = newval + def initialize_with_zeros(self, optimizer): + for i in range(len(self._items)): + self._items[i] = optimizer.new_const_item(self.arraydescr) + def getitem(self, index): res = self._items[index] return res @@ -294,6 +300,10 @@ already_forced[self] = self for index in range(self.getlength()): itemval = self.get_item_value(index) + # XXX should be skip alltogether, but I don't wanna know or + # fight unrolling just yet + if itemval is None: + itemval = self.constvalue itemval = itemval.force_at_end_of_preamble(already_forced, optforce) self.set_item_value(index, itemval) return self @@ -306,11 +316,11 @@ self.box = box = self.source_op.result for index in range(len(self._items)): subvalue = self._items[index] - if subvalue is not self.constvalue: + if subvalue is not None: subbox = subvalue.force_box(optforce) op = ResOperation(rop.SETARRAYITEM_GC, [box, ConstInt(index), subbox], None, - descr=self.arraydescr) + descr=self.arraydescr) optforce.emit_operation(op) @specialize.argtype(1) @@ -334,6 +344,12 @@ assert isinstance(itemvalue, optimizer.OptValue) self._items[index][ofs] = itemvalue + def initialize_with_zeros(self, optimizer): + fielddescrs = self.arraydescr.get_fielddescrs() + for item_d in self._items: + for descr in fielddescrs: + item_d[descr] = optimizer.new_const(descr) + def _really_force(self, optforce): assert self.source_op is not None if not we_are_translated(): @@ -664,6 +680,8 @@ def optimize_CLEAR_ARRAY_CONTENTS(self, op): v = self.getvalue(op.getarg(0)) if v.is_virtual(): + # initialize the items, since we need to store zeros + v.initialize_with_zeros(self.optimizer) return self.emit_operation(op) From noreply at buildbot.pypy.org Wed Sep 24 18:05:02 2014 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 24 Sep 2014 18:05:02 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: ignore void fields Message-ID: <20140924160502.884E11C054C@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73676:e2cff500b06d Date: 2014-09-24 18:04 +0200 http://bitbucket.org/pypy/pypy/changeset/e2cff500b06d/ Log: ignore void fields diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py --- a/rpython/jit/backend/llgraph/runner.py +++ b/rpython/jit/backend/llgraph/runner.py @@ -234,7 +234,9 @@ # order is not relevant, except for tests for name in STRUCT._names: FIELD = getattr(STRUCT, name) - if isinstance(FIELD, lltype.Struct): + if FIELD is lltype.Void: + continue + elif isinstance(FIELD, lltype.Struct): fielddescrs_for(cpu, FIELD, res) else: res.append(cpu.fielddescrof(STRUCT, name)) diff --git a/rpython/jit/codewriter/heaptracker.py b/rpython/jit/codewriter/heaptracker.py --- a/rpython/jit/codewriter/heaptracker.py +++ b/rpython/jit/codewriter/heaptracker.py @@ -134,7 +134,9 @@ # order is not relevant, except for tests for name in STRUCT._names: FIELD = getattr(STRUCT, name) - if isinstance(FIELD, lltype.Struct): + if FIELD is lltype.Void: + continue + elif isinstance(FIELD, lltype.Struct): fielddescrs(gccache, FIELD, res) else: res.append(descr.get_field_descr(gccache, STRUCT, name)) From noreply at buildbot.pypy.org Wed Sep 24 18:13:56 2014 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 24 Sep 2014 18:13:56 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: Enough to pass test_zero_array. Still missing the short-size optimization. Message-ID: <20140924161356.C3A1E1C03FC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: gc_no_cleanup_nursery Changeset: r73677:ec9889b7a5a1 Date: 2014-09-24 16:12 +0000 http://bitbucket.org/pypy/pypy/changeset/ec9889b7a5a1/ Log: Enough to pass test_zero_array. Still missing the short-size optimization. diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py --- a/rpython/jit/backend/arm/opassembler.py +++ b/rpython/jit/backend/arm/opassembler.py @@ -25,7 +25,7 @@ from rpython.jit.backend.llsupport.descr import InteriorFieldDescr from rpython.jit.backend.llsupport.assembler import GuardToken, BaseAssembler from rpython.jit.backend.llsupport.regalloc import get_scale -from rpython.jit.metainterp.history import (Box, AbstractFailDescr, +from rpython.jit.metainterp.history import (Box, AbstractFailDescr, ConstInt, INT, FLOAT, REF) from rpython.jit.metainterp.history import TargetToken from rpython.jit.metainterp.resoperation import rop @@ -1175,3 +1175,76 @@ self.mc.VMOV_cs(r.svfp_ip.value, arg.value) self.mc.VCVT_f32_f64(res.value, r.svfp_ip.value) return fcond + + #from ../x86/regalloc.py:1388 + def emit_op_zero_array(self, op, arglocs, regalloc, fcond): + from rpython.jit.backend.llsupport.descr import unpack_arraydescr + assert len(arglocs) == 0 + itemsize, baseofs, _ = unpack_arraydescr(op.getdescr()) + args = op.getarglist() + base_loc = regalloc.rm.make_sure_var_in_reg(args[0], args) + sibox = args[1] + if isinstance(sibox, ConstInt): + startindex_loc = None + startindex = sibox.getint() + assert startindex >= 0 + else: + startindex_loc = regalloc.rm.make_sure_var_in_reg(sibox, args) + startindex = -1 + length_box = op.getarg(2) + if isinstance(length_box, ConstInt): + constbytes = length_box.getint() * itemsize + else: + constbytes = -1 + if 0 <= constbytes <= 56 and (0): + #valid_addressing_size(itemsize) or + #(isinstance(startindex_loc, ImmedLoc) and + # startindex_loc.value == 0)): + XXX + else: + # base_loc and startindex_loc are in two regs here (or they are + # immediates). Compute the dstaddr_loc, which is the raw + # address that we will pass as first argument to memset(). + # It can be in the same register as either one, but not in + # args[2], because we're still needing the latter. + dstaddr_box = TempBox() + dstaddr_loc = regalloc.rm.force_allocate_reg(dstaddr_box, [args[2]]) + if startindex >= 0: # a constant + ofs = baseofs + startindex * itemsize + reg = base_loc.value + else: + self.mc.gen_load_int(r.ip.value, itemsize) + self.mc.MLA(dstaddr_loc.value, r.ip.value, + startindex_loc.value, base_loc.value) + ofs = baseofs + reg = dstaddr_loc.value + if check_imm_arg(ofs): + self.mc.ADD_ri(dstaddr_loc.value, reg, imm=ofs) + else: + self.mc.gen_load_int(r.ip.value, ofs) + self.mc.ADD_rr(dstaddr_loc.value, reg, r.ip.value) + # + if constbytes >= 0: + length_loc = imm(constbytes) + else: + # load length_loc in a register different than dstaddr_loc + length_loc = regalloc.rm.make_sure_var_in_reg(length_box, + [dstaddr_box]) + if itemsize > 1: + # we need a register that is different from dstaddr_loc, + # but which can be identical to length_loc (as usual, + # only if the length_box is not used by future operations) + bytes_box = TempBox() + bytes_loc = regalloc.rm.force_allocate_reg(bytes_box, + [dstaddr_box]) + self.mc.gen_load_int(r.ip.value, itemsize) + self.mc.MUL(bytes_loc.value, r.ip.value, length_loc.value) + length_box = bytes_box + length_loc = bytes_loc + # + # call memset() + regalloc.before_call() + self.simple_call_no_collect(imm(self.memset_addr), + [dstaddr_loc, imm(0), length_loc]) + regalloc.rm.possibly_free_var(length_box) + regalloc.rm.possibly_free_var(dstaddr_box) diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py --- a/rpython/jit/backend/arm/regalloc.py +++ b/rpython/jit/backend/arm/regalloc.py @@ -996,6 +996,7 @@ prepare_op_copystrcontent = void prepare_op_copyunicodecontent = void + prepare_op_zero_array = void def prepare_op_unicodelen(self, op, fcond): l0 = self.make_sure_var_in_reg(op.getarg(0)) From noreply at buildbot.pypy.org Wed Sep 24 18:13:58 2014 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 24 Sep 2014 18:13:58 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: merge heads Message-ID: <20140924161358.08D751C03FC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: gc_no_cleanup_nursery Changeset: r73678:aec805bce08c Date: 2014-09-24 18:13 +0200 http://bitbucket.org/pypy/pypy/changeset/aec805bce08c/ Log: merge heads diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py --- a/rpython/jit/backend/arm/opassembler.py +++ b/rpython/jit/backend/arm/opassembler.py @@ -25,7 +25,7 @@ from rpython.jit.backend.llsupport.descr import InteriorFieldDescr from rpython.jit.backend.llsupport.assembler import GuardToken, BaseAssembler from rpython.jit.backend.llsupport.regalloc import get_scale -from rpython.jit.metainterp.history import (Box, AbstractFailDescr, +from rpython.jit.metainterp.history import (Box, AbstractFailDescr, ConstInt, INT, FLOAT, REF) from rpython.jit.metainterp.history import TargetToken from rpython.jit.metainterp.resoperation import rop @@ -1175,3 +1175,76 @@ self.mc.VMOV_cs(r.svfp_ip.value, arg.value) self.mc.VCVT_f32_f64(res.value, r.svfp_ip.value) return fcond + + #from ../x86/regalloc.py:1388 + def emit_op_zero_array(self, op, arglocs, regalloc, fcond): + from rpython.jit.backend.llsupport.descr import unpack_arraydescr + assert len(arglocs) == 0 + itemsize, baseofs, _ = unpack_arraydescr(op.getdescr()) + args = op.getarglist() + base_loc = regalloc.rm.make_sure_var_in_reg(args[0], args) + sibox = args[1] + if isinstance(sibox, ConstInt): + startindex_loc = None + startindex = sibox.getint() + assert startindex >= 0 + else: + startindex_loc = regalloc.rm.make_sure_var_in_reg(sibox, args) + startindex = -1 + length_box = op.getarg(2) + if isinstance(length_box, ConstInt): + constbytes = length_box.getint() * itemsize + else: + constbytes = -1 + if 0 <= constbytes <= 56 and (0): + #valid_addressing_size(itemsize) or + #(isinstance(startindex_loc, ImmedLoc) and + # startindex_loc.value == 0)): + XXX + else: + # base_loc and startindex_loc are in two regs here (or they are + # immediates). Compute the dstaddr_loc, which is the raw + # address that we will pass as first argument to memset(). + # It can be in the same register as either one, but not in + # args[2], because we're still needing the latter. + dstaddr_box = TempBox() + dstaddr_loc = regalloc.rm.force_allocate_reg(dstaddr_box, [args[2]]) + if startindex >= 0: # a constant + ofs = baseofs + startindex * itemsize + reg = base_loc.value + else: + self.mc.gen_load_int(r.ip.value, itemsize) + self.mc.MLA(dstaddr_loc.value, r.ip.value, + startindex_loc.value, base_loc.value) + ofs = baseofs + reg = dstaddr_loc.value + if check_imm_arg(ofs): + self.mc.ADD_ri(dstaddr_loc.value, reg, imm=ofs) + else: + self.mc.gen_load_int(r.ip.value, ofs) + self.mc.ADD_rr(dstaddr_loc.value, reg, r.ip.value) + # + if constbytes >= 0: + length_loc = imm(constbytes) + else: + # load length_loc in a register different than dstaddr_loc + length_loc = regalloc.rm.make_sure_var_in_reg(length_box, + [dstaddr_box]) + if itemsize > 1: + # we need a register that is different from dstaddr_loc, + # but which can be identical to length_loc (as usual, + # only if the length_box is not used by future operations) + bytes_box = TempBox() + bytes_loc = regalloc.rm.force_allocate_reg(bytes_box, + [dstaddr_box]) + self.mc.gen_load_int(r.ip.value, itemsize) + self.mc.MUL(bytes_loc.value, r.ip.value, length_loc.value) + length_box = bytes_box + length_loc = bytes_loc + # + # call memset() + regalloc.before_call() + self.simple_call_no_collect(imm(self.memset_addr), + [dstaddr_loc, imm(0), length_loc]) + regalloc.rm.possibly_free_var(length_box) + regalloc.rm.possibly_free_var(dstaddr_box) diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py --- a/rpython/jit/backend/arm/regalloc.py +++ b/rpython/jit/backend/arm/regalloc.py @@ -996,6 +996,7 @@ prepare_op_copystrcontent = void prepare_op_copyunicodecontent = void + prepare_op_zero_array = void def prepare_op_unicodelen(self, op, fcond): l0 = self.make_sure_var_in_reg(op.getarg(0)) From noreply at buildbot.pypy.org Wed Sep 24 18:15:08 2014 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 24 Sep 2014 18:15:08 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: Clarify which cls goes with which argument Message-ID: <20140924161508.B5DFA1C03FC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: gc_no_cleanup_nursery Changeset: r73679:85803d8e9c23 Date: 2014-09-24 18:14 +0200 http://bitbucket.org/pypy/pypy/changeset/85803d8e9c23/ Log: Clarify which cls goes with which argument diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py --- a/rpython/jit/backend/test/runner_test.py +++ b/rpython/jit/backend/test/runner_test.py @@ -4515,10 +4515,8 @@ for cls2 in [ConstInt, BoxInt]: print 'a_int:', a_int print 'of:', OF - print 'start:', start - print 'length:', length - print 'cls1:', cls1.__name__ - print 'cls2:', cls2.__name__ + print 'start:', cls1.__name__, start + print 'length:', cls2.__name__, length for i in range(100): if OF == PAIR: a[i].a = a[i].b = -123456789 From noreply at buildbot.pypy.org Wed Sep 24 18:29:29 2014 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 24 Sep 2014 18:29:29 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: Simplify the entry condition a little bit Message-ID: <20140924162929.85A331C03FC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: gc_no_cleanup_nursery Changeset: r73680:a738fd00ee83 Date: 2014-09-24 18:29 +0200 http://bitbucket.org/pypy/pypy/changeset/a738fd00ee83/ Log: Simplify the entry condition a little bit diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -2376,7 +2376,8 @@ scale = get_scale(itemsize_loc.value) else: assert isinstance(startindex_loc, ImmedLoc) - assert startindex_loc.value == 0 + baseofs += startindex_loc.value * itemsize_loc.value + startindex_loc = imm0 scale = 0 null_reg_cleared = False i = 0 diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -1397,8 +1397,7 @@ constbytes = -1 if 0 <= constbytes <= 16 * 8 and ( valid_addressing_size(itemsize) or - (isinstance(startindex_loc, ImmedLoc) and - startindex_loc.value == 0)): +- isinstance(startindex_loc, ImmedLoc)): if IS_X86_64: null_loc = X86_64_XMM_SCRATCH_REG else: From noreply at buildbot.pypy.org Wed Sep 24 18:51:18 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Wed, 24 Sep 2014 18:51:18 +0200 (CEST) Subject: [pypy-commit] pypy default: fix float mod behavior to match cpython (issue 1873) Message-ID: <20140924165118.1ABB81C03FC@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r73681:42870fecc305 Date: 2014-09-24 09:50 -0700 http://bitbucket.org/pypy/pypy/changeset/42870fecc305/ Log: fix float mod behavior to match cpython (issue 1873) diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py --- a/pypy/objspace/std/floatobject.py +++ b/pypy/objspace/std/floatobject.py @@ -15,6 +15,7 @@ from rpython.rlib.rbigint import rbigint from rpython.rlib import rfloat from rpython.tool.sourcetools import func_with_new_name +from rpython.rtyper.lltypesystem.module.ll_math import math_fmod from pypy.objspace.std.intobject import W_IntObject @@ -360,21 +361,17 @@ y = w_float2.floatval if y == 0.0: raise FailedToImplementArgs(space.w_ZeroDivisionError, space.wrap("float modulo")) - try: - mod = math.fmod(x, y) - except ValueError: - mod = rfloat.NAN + mod = math_fmod(x, y) + if mod: + # ensure the remainder has the same sign as the denominator + if (y < 0.0) != (mod < 0.0): + mod += y else: - if mod: - # ensure the remainder has the same sign as the denominator - if (y < 0.0) != (mod < 0.0): - mod += y - else: - # the remainder is zero, and in the presence of signed zeroes - # fmod returns different results across platforms; ensure - # it has the same sign as the denominator; we'd like to do - # "mod = y * 0.0", but that may get optimized away - mod = copysign(0.0, y) + # the remainder is zero, and in the presence of signed zeroes + # fmod returns different results across platforms; ensure + # it has the same sign as the denominator; we'd like to do + # "mod = y * 0.0", but that may get optimized away + mod = copysign(0.0, y) return W_FloatObject(mod) @@ -383,10 +380,7 @@ y = w_float2.floatval if y == 0.0: raise FailedToImplementArgs(space.w_ZeroDivisionError, space.wrap("float modulo")) - try: - mod = math.fmod(x, y) - except ValueError: - return [W_FloatObject(rfloat.NAN), W_FloatObject(rfloat.NAN)] + mod = math_fmod(x, y) # fmod is typically exact, so vx-mod is *mathematically* an # exact multiple of wx. But this is fp arithmetic, and fp # vx - mod is an approximation; the result is that div may diff --git a/pypy/objspace/std/test/test_floatobject.py b/pypy/objspace/std/test/test_floatobject.py --- a/pypy/objspace/std/test/test_floatobject.py +++ b/pypy/objspace/std/test/test_floatobject.py @@ -794,7 +794,7 @@ raises(ValueError, float.fromhex, "0P") def test_division_edgecases(self): - import math + import math, os # inf inf = float("inf") @@ -803,6 +803,16 @@ x, y = divmod(inf, 3) assert math.isnan(x) assert math.isnan(y) + x, y = divmod(3, inf) + z = 3 % inf + if os.name == 'nt': + assert math.isnan(x) + assert math.isnan(y) + assert math.isnan(z) + else: + assert x == 0 + assert y == 3 + assert z == 3 # divide by 0 raises(ZeroDivisionError, lambda: inf % 0) From noreply at buildbot.pypy.org Wed Sep 24 18:54:29 2014 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 24 Sep 2014 18:54:29 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: Inlined fast-path. Message-ID: <20140924165429.DDDCD1C0DCA@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: gc_no_cleanup_nursery Changeset: r73682:9d73241d63a3 Date: 2014-09-24 16:53 +0000 http://bitbucket.org/pypy/pypy/changeset/9d73241d63a3/ Log: Inlined fast-path. diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py --- a/rpython/jit/backend/arm/opassembler.py +++ b/rpython/jit/backend/arm/opassembler.py @@ -1192,40 +1192,48 @@ startindex_loc = regalloc.rm.make_sure_var_in_reg(sibox, args) startindex = -1 length_box = op.getarg(2) - if isinstance(length_box, ConstInt): - constbytes = length_box.getint() * itemsize + + # base_loc and startindex_loc are in two regs here (or they are + # immediates). Compute the dstaddr_loc, which is the raw + # address that we will pass as first argument to memset(). + # It can be in the same register as either one, but not in + # args[2], because we're still needing the latter. + dstaddr_box = TempBox() + dstaddr_loc = regalloc.rm.force_allocate_reg(dstaddr_box, [args[2]]) + if startindex >= 0: # a constant + ofs = baseofs + startindex * itemsize + reg = base_loc.value else: - constbytes = -1 - if 0 <= constbytes <= 56 and (0): - #valid_addressing_size(itemsize) or - #(isinstance(startindex_loc, ImmedLoc) and - # startindex_loc.value == 0)): - XXX + self.mc.gen_load_int(r.ip.value, itemsize) + self.mc.MLA(dstaddr_loc.value, r.ip.value, + startindex_loc.value, base_loc.value) + ofs = baseofs + reg = dstaddr_loc.value + if check_imm_arg(ofs): + self.mc.ADD_ri(dstaddr_loc.value, reg, imm=ofs) else: - # base_loc and startindex_loc are in two regs here (or they are - # immediates). Compute the dstaddr_loc, which is the raw - # address that we will pass as first argument to memset(). - # It can be in the same register as either one, but not in - # args[2], because we're still needing the latter. - dstaddr_box = TempBox() - dstaddr_loc = regalloc.rm.force_allocate_reg(dstaddr_box, [args[2]]) - if startindex >= 0: # a constant - ofs = baseofs + startindex * itemsize - reg = base_loc.value - else: - self.mc.gen_load_int(r.ip.value, itemsize) - self.mc.MLA(dstaddr_loc.value, r.ip.value, - startindex_loc.value, base_loc.value) - ofs = baseofs - reg = dstaddr_loc.value - if check_imm_arg(ofs): - self.mc.ADD_ri(dstaddr_loc.value, reg, imm=ofs) - else: - self.mc.gen_load_int(r.ip.value, ofs) - self.mc.ADD_rr(dstaddr_loc.value, reg, r.ip.value) - # - if constbytes >= 0: - length_loc = imm(constbytes) + self.mc.gen_load_int(r.ip.value, ofs) + self.mc.ADD_rr(dstaddr_loc.value, reg, r.ip.value) + + if (isinstance(length_box, ConstInt) and + length_box.getint() <= 14 and # same limit as GCC + itemsize in (4, 2, 1)): + # Inline a series of STR operations, starting at 'dstaddr_loc'. + # XXX we could optimize STRB/STRH into STR, but this needs care: + # XXX it only works if startindex_loc is a constant, otherwise + # XXX we'd be doing unaligned accesses + self.mc.gen_load_int(r.ip.value, 0) + for i in range(length_box.getint()): + if itemsize == 4: + self.mc.STR_ri(r.ip.value, dstaddr_loc.value, imm=i*4) + elif itemsize == 2: + self.mc.STRH_ri(r.ip.value, dstaddr_loc.value, imm=i*2) + else: + self.mc.STRB_ri(r.ip.value, dstaddr_loc.value, imm=i*1) + + else: + if isinstance(length_box, ConstInt): + length_loc = imm(length_box.getint() * itemsize) else: # load length_loc in a register different than dstaddr_loc length_loc = regalloc.rm.make_sure_var_in_reg(length_box, @@ -1237,7 +1245,7 @@ bytes_box = TempBox() bytes_loc = regalloc.rm.force_allocate_reg(bytes_box, [dstaddr_box]) - self.mc.gen_load_int(r.ip.value, itemsize) + self.mc.gen_load_int(r.ip.value, itemsize) self.mc.MUL(bytes_loc.value, r.ip.value, length_loc.value) length_box = bytes_box length_loc = bytes_loc @@ -1247,4 +1255,4 @@ self.simple_call_no_collect(imm(self.memset_addr), [dstaddr_loc, imm(0), length_loc]) regalloc.rm.possibly_free_var(length_box) - regalloc.rm.possibly_free_var(dstaddr_box) + regalloc.rm.possibly_free_var(dstaddr_box) From noreply at buildbot.pypy.org Thu Sep 25 04:25:55 2014 From: noreply at buildbot.pypy.org (pjenvey) Date: Thu, 25 Sep 2014 04:25:55 +0200 (CEST) Subject: [pypy-commit] pypy py3k: we really want a unicodebuffer here Message-ID: <20140925022555.087441C0250@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r73683:2e736033bd2b Date: 2014-09-24 19:14 -0700 http://bitbucket.org/pypy/pypy/changeset/2e736033bd2b/ Log: we really want a unicodebuffer here diff --git a/rpython/rtyper/module/ll_win32file.py b/rpython/rtyper/module/ll_win32file.py --- a/rpython/rtyper/module/ll_win32file.py +++ b/rpython/rtyper/module/ll_win32file.py @@ -476,24 +476,24 @@ VOLUME_NAME_DOS = rffi.cast(rwin32.DWORD, win32traits.VOLUME_NAME_DOS) try: - size = win32traits.GetFinalPathNameByHandle( + usize = win32traits.GetFinalPathNameByHandle( hFile, lltype.nullptr(traits.CCHARP.TO), rffi.cast(rwin32.DWORD, 0), VOLUME_NAME_DOS) - if size == 0: + if usize == 0: raise rwin32.lastWindowsError("GetFinalPathNameByHandle") - with lltype.scoped_alloc(traits.CCHARP.TO, size + 1) as target_path: + size = rffi.cast(lltype.Signed, usize) + with rffi.scoped_alloc_unicodebuffer(size + 1) as buf: result = win32traits.GetFinalPathNameByHandle( hFile, - target_path, - size, + buf.raw, + usize, VOLUME_NAME_DOS) if result == 0: raise rwin32.lastWindowsError("GetFinalPathNameByHandle") - return traits.charpsize2str(target_path, - rffi.cast(lltype.Signed, result)) + return buf.str(rffi.cast(lltype.Signed, result)) finally: rwin32.CloseHandle(hFile) From noreply at buildbot.pypy.org Thu Sep 25 04:25:56 2014 From: noreply at buildbot.pypy.org (pjenvey) Date: Thu, 25 Sep 2014 04:25:56 +0200 (CEST) Subject: [pypy-commit] pypy py3k: workaround FuncType pointer calls w/ a non-standard calling convention not Message-ID: <20140925022556.60B5C1C0250@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r73684:eab5254eaafa Date: 2014-09-24 19:24 -0700 http://bitbucket.org/pypy/pypy/changeset/eab5254eaafa/ Log: workaround FuncType pointer calls w/ a non-standard calling convention not actually using said convention (after translation) with a stupid shim for now diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py --- a/pypy/module/posix/__init__.py +++ b/pypy/module/posix/__init__.py @@ -167,8 +167,7 @@ if os.name == 'nt': interpleveldefs.update({ '_getfileinformation': 'interp_posix._getfileinformation', - # XXX: currently broken - #'_getfinalpathname': 'interp_posix._getfinalpathname', + '_getfinalpathname': 'interp_posix._getfinalpathname', }) if hasattr(os, 'chroot'): interpleveldefs['chroot'] = 'interp_posix.chroot' diff --git a/rpython/rtyper/module/ll_win32file.py b/rpython/rtyper/module/ll_win32file.py --- a/rpython/rtyper/module/ll_win32file.py +++ b/rpython/rtyper/module/ll_win32file.py @@ -10,6 +10,17 @@ from rpython.tool.sourcetools import func_renamer from rpython.rlib.objectmodel import specialize +separate_module_source = """\ +DWORD +pypy_GetFinalPathNameByHandle(FARPROC address, HANDLE hFile, + LPTSTR lpszFilePath, DWORD cchFilePath, + DWORD dwFlags) { + DWORD (WINAPI *func)(HANDLE, LPTSTR, DWORD, DWORD); + *(FARPROC*)&func = address; + return func(hFile, lpszFilePath, cchFilePath, dwFlags); +} +""" + def make_win32_traits(traits): from rpython.rlib import rwin32 @@ -21,6 +32,8 @@ class CConfig: _compilation_info_ = ExternalCompilationInfo( includes = ['windows.h', 'winbase.h', 'sys/stat.h'], + separate_module_sources=[separate_module_source], + export_symbols=['pypy_GetFinalPathNameByHandle'] ) WIN32_FIND_DATA = platform.Struct( 'struct _WIN32_FIND_DATA' + suffix, @@ -192,15 +205,15 @@ [traits.CCHARP, traits.CCHARP], rwin32.BOOL) - GETFINALPATHNAMEBYHANDLE_TP = lltype.Ptr(lltype.FuncType( - [rwin32.HANDLE, traits.CCHARP, rwin32.DWORD, rwin32.DWORD], - rwin32.DWORD, abi='FFI_STDCALL')) - # dynamically loaded - GetFinalPathNameByHandle = lltype.nullptr( - GETFINALPATHNAMEBYHANDLE_TP.TO) + GetFinalPathNameByHandle_HANDLE = lltype.nullptr(rffi.VOIDP.TO) + pypy_GetFinalPathNameByHandle = staticmethod(rffi.llexternal( + 'pypy_GetFinalPathNameByHandle', + [rffi.VOIDP, rwin32.HANDLE, rffi.CWCHARP, rwin32.DWORD, rwin32.DWORD], + rwin32.DWORD, compilation_info=CConfig._compilation_info_)) def check_GetFinalPathNameByHandle(self): - if self.GetFinalPathNameByHandle: + if (self.GetFinalPathNameByHandle_HANDLE != + lltype.nullptr(rffi.VOIDP.TO)): return True from rpython.rlib.rdynload import GetModuleHandle, dlsym @@ -210,10 +223,15 @@ except KeyError: return False - self.GetFinalPathNameByHandle = rffi.cast( - Win32Traits.GETFINALPATHNAMEBYHANDLE_TP, func) + self.GetFinalPathNameByHandle_HANDLE = func return True + def GetFinalPathNameByHandle(self, *args): + assert (self.GetFinalPathNameByHandle_HANDLE != + lltype.nullptr(rffi.VOIDP.TO)) + return self.pypy_GetFinalPathNameByHandle( + self.GetFinalPathNameByHandle_HANDLE, *args) + return Win32Traits() #_______________________________________________________________ From noreply at buildbot.pypy.org Thu Sep 25 09:54:39 2014 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 25 Sep 2014 09:54:39 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: wrong descr Message-ID: <20140925075439.932AB1C3363@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73685:8c86202d41ef Date: 2014-09-25 09:52 +0200 http://bitbucket.org/pypy/pypy/changeset/8c86202d41ef/ Log: wrong descr diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py --- a/rpython/jit/backend/llgraph/runner.py +++ b/rpython/jit/backend/llgraph/runner.py @@ -228,18 +228,20 @@ 'i': 0, 'f': 0.0} -def fielddescrs_for(cpu, STRUCT, res=None): +def fielddescrs_for(cpu, A, res=None): if res is None: res = [] + STRUCT = A.OF # order is not relevant, except for tests for name in STRUCT._names: FIELD = getattr(STRUCT, name) if FIELD is lltype.Void: continue elif isinstance(FIELD, lltype.Struct): - fielddescrs_for(cpu, FIELD, res) + assert False + #fielddescrs_for(cpu, FIELD, res) else: - res.append(cpu.fielddescrof(STRUCT, name)) + res.append(cpu.interiorfielddescrof(A, name)) return res @@ -411,7 +413,7 @@ descr = ArrayDescr(A) self.descrs[key] = descr if descr.is_array_of_structs(): - descr.fielddescrs = fielddescrs_for(self, A.OF) + descr.fielddescrs = fielddescrs_for(self, A) return descr def interiorfielddescrof(self, A, fieldname): From noreply at buildbot.pypy.org Thu Sep 25 09:54:41 2014 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 25 Sep 2014 09:54:41 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: merge Message-ID: <20140925075441.207AF1C3363@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73686:c50bae221df1 Date: 2014-09-25 09:54 +0200 http://bitbucket.org/pypy/pypy/changeset/c50bae221df1/ Log: merge diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py --- a/rpython/jit/backend/arm/opassembler.py +++ b/rpython/jit/backend/arm/opassembler.py @@ -25,7 +25,7 @@ from rpython.jit.backend.llsupport.descr import InteriorFieldDescr from rpython.jit.backend.llsupport.assembler import GuardToken, BaseAssembler from rpython.jit.backend.llsupport.regalloc import get_scale -from rpython.jit.metainterp.history import (Box, AbstractFailDescr, +from rpython.jit.metainterp.history import (Box, AbstractFailDescr, ConstInt, INT, FLOAT, REF) from rpython.jit.metainterp.history import TargetToken from rpython.jit.metainterp.resoperation import rop @@ -1175,3 +1175,84 @@ self.mc.VMOV_cs(r.svfp_ip.value, arg.value) self.mc.VCVT_f32_f64(res.value, r.svfp_ip.value) return fcond + + #from ../x86/regalloc.py:1388 + def emit_op_zero_array(self, op, arglocs, regalloc, fcond): + from rpython.jit.backend.llsupport.descr import unpack_arraydescr + assert len(arglocs) == 0 + itemsize, baseofs, _ = unpack_arraydescr(op.getdescr()) + args = op.getarglist() + base_loc = regalloc.rm.make_sure_var_in_reg(args[0], args) + sibox = args[1] + if isinstance(sibox, ConstInt): + startindex_loc = None + startindex = sibox.getint() + assert startindex >= 0 + else: + startindex_loc = regalloc.rm.make_sure_var_in_reg(sibox, args) + startindex = -1 + length_box = op.getarg(2) + + # base_loc and startindex_loc are in two regs here (or they are + # immediates). Compute the dstaddr_loc, which is the raw + # address that we will pass as first argument to memset(). + # It can be in the same register as either one, but not in + # args[2], because we're still needing the latter. + dstaddr_box = TempBox() + dstaddr_loc = regalloc.rm.force_allocate_reg(dstaddr_box, [args[2]]) + if startindex >= 0: # a constant + ofs = baseofs + startindex * itemsize + reg = base_loc.value + else: + self.mc.gen_load_int(r.ip.value, itemsize) + self.mc.MLA(dstaddr_loc.value, r.ip.value, + startindex_loc.value, base_loc.value) + ofs = baseofs + reg = dstaddr_loc.value + if check_imm_arg(ofs): + self.mc.ADD_ri(dstaddr_loc.value, reg, imm=ofs) + else: + self.mc.gen_load_int(r.ip.value, ofs) + self.mc.ADD_rr(dstaddr_loc.value, reg, r.ip.value) + + if (isinstance(length_box, ConstInt) and + length_box.getint() <= 14 and # same limit as GCC + itemsize in (4, 2, 1)): + # Inline a series of STR operations, starting at 'dstaddr_loc'. + # XXX we could optimize STRB/STRH into STR, but this needs care: + # XXX it only works if startindex_loc is a constant, otherwise + # XXX we'd be doing unaligned accesses + self.mc.gen_load_int(r.ip.value, 0) + for i in range(length_box.getint()): + if itemsize == 4: + self.mc.STR_ri(r.ip.value, dstaddr_loc.value, imm=i*4) + elif itemsize == 2: + self.mc.STRH_ri(r.ip.value, dstaddr_loc.value, imm=i*2) + else: + self.mc.STRB_ri(r.ip.value, dstaddr_loc.value, imm=i*1) + + else: + if isinstance(length_box, ConstInt): + length_loc = imm(length_box.getint() * itemsize) + else: + # load length_loc in a register different than dstaddr_loc + length_loc = regalloc.rm.make_sure_var_in_reg(length_box, + [dstaddr_box]) + if itemsize > 1: + # we need a register that is different from dstaddr_loc, + # but which can be identical to length_loc (as usual, + # only if the length_box is not used by future operations) + bytes_box = TempBox() + bytes_loc = regalloc.rm.force_allocate_reg(bytes_box, + [dstaddr_box]) + self.mc.gen_load_int(r.ip.value, itemsize) + self.mc.MUL(bytes_loc.value, r.ip.value, length_loc.value) + length_box = bytes_box + length_loc = bytes_loc + # + # call memset() + regalloc.before_call() + self.simple_call_no_collect(imm(self.memset_addr), + [dstaddr_loc, imm(0), length_loc]) + regalloc.rm.possibly_free_var(length_box) + regalloc.rm.possibly_free_var(dstaddr_box) diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py --- a/rpython/jit/backend/arm/regalloc.py +++ b/rpython/jit/backend/arm/regalloc.py @@ -996,6 +996,7 @@ prepare_op_copystrcontent = void prepare_op_copyunicodecontent = void + prepare_op_zero_array = void def prepare_op_unicodelen(self, op, fcond): l0 = self.make_sure_var_in_reg(op.getarg(0)) diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py --- a/rpython/jit/backend/test/runner_test.py +++ b/rpython/jit/backend/test/runner_test.py @@ -4515,10 +4515,8 @@ for cls2 in [ConstInt, BoxInt]: print 'a_int:', a_int print 'of:', OF - print 'start:', start - print 'length:', length - print 'cls1:', cls1.__name__ - print 'cls2:', cls2.__name__ + print 'start:', cls1.__name__, start + print 'length:', cls2.__name__, length for i in range(100): if OF == PAIR: a[i].a = a[i].b = -123456789 diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -2376,7 +2376,8 @@ scale = get_scale(itemsize_loc.value) else: assert isinstance(startindex_loc, ImmedLoc) - assert startindex_loc.value == 0 + baseofs += startindex_loc.value * itemsize_loc.value + startindex_loc = imm0 scale = 0 null_reg_cleared = False i = 0 diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -1397,8 +1397,7 @@ constbytes = -1 if 0 <= constbytes <= 16 * 8 and ( valid_addressing_size(itemsize) or - (isinstance(startindex_loc, ImmedLoc) and - startindex_loc.value == 0)): +- isinstance(startindex_loc, ImmedLoc)): if IS_X86_64: null_loc = X86_64_XMM_SCRATCH_REG else: From noreply at buildbot.pypy.org Thu Sep 25 10:11:04 2014 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 25 Sep 2014 10:11:04 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: Only emit clear_array_contents for arrays that are allocated with zero=True, Message-ID: <20140925081104.35AFF1C054C@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73687:0157e90c1899 Date: 2014-09-25 10:08 +0200 http://bitbucket.org/pypy/pypy/changeset/0157e90c1899/ Log: Only emit clear_array_contents for arrays that are allocated with zero=True, but with no GC ptrs in them diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py --- a/rpython/jit/codewriter/jtransform.py +++ b/rpython/jit/codewriter/jtransform.py @@ -615,7 +615,14 @@ op1 = SpaceOperation('new_array', [op.args[2], arraydescr], op.result) if op.args[1].value.get('zero', False): - return self.zero_contents([op1], op.result, ARRAY) + # complicated logic here - we only need to emit zero_contents + # in case this is an array of non-gcptrs and non-structs + if isinstance(ARRAY.OF, lltype.Ptr) and ARRAY.OF._needsgc(): + pass + elif isinstance(ARRAY.OF, lltype.Struct): + pass + else: + return self.zero_contents([op1], op.result, ARRAY) return op1 def zero_contents(self, ops, v, TYPE): From noreply at buildbot.pypy.org Thu Sep 25 10:11:05 2014 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 25 Sep 2014 10:11:05 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: Kill some logic, it's unnecessary Message-ID: <20140925081105.71CEB1C054C@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73688:32973aa8ab83 Date: 2014-09-25 10:10 +0200 http://bitbucket.org/pypy/pypy/changeset/32973aa8ab83/ Log: Kill some logic, it's unnecessary diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py --- a/rpython/jit/backend/llgraph/runner.py +++ b/rpython/jit/backend/llgraph/runner.py @@ -165,9 +165,6 @@ def is_array_of_structs(self): return isinstance(self.A.OF, lltype.Struct) - def get_fielddescrs(self): - return self.fielddescrs - def is_item_integer_bounded(self): return getkind(self.A.OF) == 'int' \ and rffi.sizeof(self.A.OF) < symbolic.WORD @@ -228,22 +225,6 @@ 'i': 0, 'f': 0.0} -def fielddescrs_for(cpu, A, res=None): - if res is None: - res = [] - STRUCT = A.OF - # order is not relevant, except for tests - for name in STRUCT._names: - FIELD = getattr(STRUCT, name) - if FIELD is lltype.Void: - continue - elif isinstance(FIELD, lltype.Struct): - assert False - #fielddescrs_for(cpu, FIELD, res) - else: - res.append(cpu.interiorfielddescrof(A, name)) - return res - class LLGraphCPU(model.AbstractCPU): from rpython.jit.metainterp.typesystem import llhelper as ts @@ -412,8 +393,6 @@ except KeyError: descr = ArrayDescr(A) self.descrs[key] = descr - if descr.is_array_of_structs(): - descr.fielddescrs = fielddescrs_for(self, A) return descr def interiorfielddescrof(self, A, fieldname): diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py --- a/rpython/jit/backend/llsupport/descr.py +++ b/rpython/jit/backend/llsupport/descr.py @@ -192,7 +192,6 @@ lendescr = None flag = '\x00' vinfo = None - fielddescrs = None def __init__(self, basesize, itemsize, lendescr, flag): self.basesize = basesize @@ -216,9 +215,6 @@ return self.flag in (FLAG_SIGNED, FLAG_UNSIGNED) \ and self.itemsize < symbolic.WORD - def get_fielddescrs(self): - return self.fielddescrs - def get_item_integer_min(self): if self.flag == FLAG_UNSIGNED: return intbounds.get_integer_min(True, self.itemsize) @@ -258,9 +254,6 @@ lendescr = get_field_arraylen_descr(gccache, ARRAY_OR_STRUCT) flag = get_type_flag(ARRAY_INSIDE.OF) arraydescr = ArrayDescr(basesize, itemsize, lendescr, flag) - if arraydescr.is_array_of_structs(): - arraydescr.fielddescrs = heaptracker.fielddescrs(gccache, - ARRAY_OR_STRUCT.OF) if ARRAY_OR_STRUCT._gckind == 'gc': gccache.init_array_descr(ARRAY_OR_STRUCT, arraydescr) cache[ARRAY_OR_STRUCT] = arraydescr diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py --- a/rpython/jit/metainterp/optimizeopt/virtualize.py +++ b/rpython/jit/metainterp/optimizeopt/virtualize.py @@ -345,10 +345,7 @@ self._items[index][ofs] = itemvalue def initialize_with_zeros(self, optimizer): - fielddescrs = self.arraydescr.get_fielddescrs() - for item_d in self._items: - for descr in fielddescrs: - item_d[descr] = optimizer.new_const(descr) + assert False, "should never happen" def _really_force(self, optforce): assert self.source_op is not None From noreply at buildbot.pypy.org Thu Sep 25 10:14:53 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 25 Sep 2014 10:14:53 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: Try to write down how exactly the NEW_* operations are supposed to clear Message-ID: <20140925081453.1E2861C054C@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: gc_no_cleanup_nursery Changeset: r73689:1559dc36fe9e Date: 2014-09-25 10:14 +0200 http://bitbucket.org/pypy/pypy/changeset/1559dc36fe9e/ Log: Try to write down how exactly the NEW_* operations are supposed to clear (or not) the new objects. Add NEW_ARRAY_CLEAR. diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py --- a/rpython/jit/metainterp/resoperation.py +++ b/rpython/jit/metainterp/resoperation.py @@ -481,11 +481,13 @@ 'GETFIELD_GC/1d', 'GETFIELD_RAW/1d', '_MALLOC_FIRST', - 'NEW/0d', - 'NEW_WITH_VTABLE/1', - 'NEW_ARRAY/1d', - 'NEWSTR/1', - 'NEWUNICODE/1', + 'NEW/0d', #-> GcStruct, gcptrs inside are zeroed (not the rest) + 'NEW_WITH_VTABLE/1', #-> GcStruct with vtable, gcptrs inside are zeroed + 'NEW_ARRAY/1d', #-> GcArray, zeroed if array of gcptr or of structs + # (even if struct has no gcptrs) + 'NEW_ARRAY_CLEAR/1d', #-> GcArray, zeroed always + 'NEWSTR/1', #-> STR, the hash field is zeroed + 'NEWUNICODE/1', #-> UNICODE, the hash field is zeroed '_MALLOC_LAST', 'FORCE_TOKEN/0', 'VIRTUAL_REF/2', # removed before it's passed to the backend From noreply at buildbot.pypy.org Thu Sep 25 10:16:02 2014 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 25 Sep 2014 10:16:02 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: two more setfields from storing zeros Message-ID: <20140925081602.592891C0250@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73690:8d7bbecc3846 Date: 2014-09-25 10:15 +0200 http://bitbucket.org/pypy/pypy/changeset/8d7bbecc3846/ Log: two more setfields from storing zeros diff --git a/rpython/jit/metainterp/test/test_dict.py b/rpython/jit/metainterp/test/test_dict.py --- a/rpython/jit/metainterp/test/test_dict.py +++ b/rpython/jit/metainterp/test/test_dict.py @@ -171,7 +171,7 @@ self.check_resops({'new_array': 2, 'getfield_gc': 2, 'guard_true': 2, 'jump': 1, 'new_with_vtable': 2, 'getinteriorfield_gc': 2, - 'setfield_gc': 6, 'int_gt': 2, 'int_sub': 2, + 'setfield_gc': 8, 'int_gt': 2, 'int_sub': 2, 'call': 10, 'int_and': 2, 'guard_no_exception': 8, 'new': 2, 'guard_false': 2, 'int_is_true': 2}) From noreply at buildbot.pypy.org Thu Sep 25 10:16:03 2014 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 25 Sep 2014 10:16:03 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: merge Message-ID: <20140925081603.8EF751C0250@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73691:aa778d2aaa5d Date: 2014-09-25 10:15 +0200 http://bitbucket.org/pypy/pypy/changeset/aa778d2aaa5d/ Log: merge diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py --- a/rpython/jit/metainterp/resoperation.py +++ b/rpython/jit/metainterp/resoperation.py @@ -481,11 +481,13 @@ 'GETFIELD_GC/1d', 'GETFIELD_RAW/1d', '_MALLOC_FIRST', - 'NEW/0d', - 'NEW_WITH_VTABLE/1', - 'NEW_ARRAY/1d', - 'NEWSTR/1', - 'NEWUNICODE/1', + 'NEW/0d', #-> GcStruct, gcptrs inside are zeroed (not the rest) + 'NEW_WITH_VTABLE/1', #-> GcStruct with vtable, gcptrs inside are zeroed + 'NEW_ARRAY/1d', #-> GcArray, zeroed if array of gcptr or of structs + # (even if struct has no gcptrs) + 'NEW_ARRAY_CLEAR/1d', #-> GcArray, zeroed always + 'NEWSTR/1', #-> STR, the hash field is zeroed + 'NEWUNICODE/1', #-> UNICODE, the hash field is zeroed '_MALLOC_LAST', 'FORCE_TOKEN/0', 'VIRTUAL_REF/2', # removed before it's passed to the backend From noreply at buildbot.pypy.org Thu Sep 25 10:17:15 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 25 Sep 2014 10:17:15 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: Mention that the offset must be constant Message-ID: <20140925081715.0B43C1C0250@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: gc_no_cleanup_nursery Changeset: r73692:fe2dc9a68bb6 Date: 2014-09-25 10:16 +0200 http://bitbucket.org/pypy/pypy/changeset/fe2dc9a68bb6/ Log: Mention that the offset must be constant diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py --- a/rpython/jit/metainterp/resoperation.py +++ b/rpython/jit/metainterp/resoperation.py @@ -507,7 +507,7 @@ 'RAW_STORE/3d', 'SETFIELD_GC/2d', 'ZERO_PTR_FIELD/2', # only emitted by the rewrite, clears a pointer field - # at a given offset, no descr + # at a given constant offset, no descr 'ZERO_ARRAY/3d', # only emitted by the rewrite, clears (part of) an array # [arraygcptr, firstindex, length], descr=ArrayDescr 'SETFIELD_RAW/2d', From noreply at buildbot.pypy.org Thu Sep 25 10:27:33 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 25 Sep 2014 10:27:33 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: NEW_ARRAY should only be used on arrays of primitives. Otherwise, use Message-ID: <20140925082733.E540E1C059C@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: gc_no_cleanup_nursery Changeset: r73693:5a14bf8ac1a6 Date: 2014-09-25 10:27 +0200 http://bitbucket.org/pypy/pypy/changeset/5a14bf8ac1a6/ Log: NEW_ARRAY should only be used on arrays of primitives. Otherwise, use NEW_ARRAY_CLEAR. diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py --- a/rpython/jit/metainterp/resoperation.py +++ b/rpython/jit/metainterp/resoperation.py @@ -483,9 +483,8 @@ '_MALLOC_FIRST', 'NEW/0d', #-> GcStruct, gcptrs inside are zeroed (not the rest) 'NEW_WITH_VTABLE/1', #-> GcStruct with vtable, gcptrs inside are zeroed - 'NEW_ARRAY/1d', #-> GcArray, zeroed if array of gcptr or of structs - # (even if struct has no gcptrs) - 'NEW_ARRAY_CLEAR/1d', #-> GcArray, zeroed always + 'NEW_ARRAY/1d', #-> GcArray, not zeroed. only for arrays of primitives + 'NEW_ARRAY_CLEAR/1d', #-> GcArray, fully zeroed 'NEWSTR/1', #-> STR, the hash field is zeroed 'NEWUNICODE/1', #-> UNICODE, the hash field is zeroed '_MALLOC_LAST', From noreply at buildbot.pypy.org Thu Sep 25 11:12:37 2014 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 25 Sep 2014 11:12:37 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: codewriter support for new_array_clear Message-ID: <20140925091237.0CB4B1C0250@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73694:cef273238737 Date: 2014-09-25 11:11 +0200 http://bitbucket.org/pypy/pypy/changeset/cef273238737/ Log: codewriter support for new_array_clear diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py --- a/rpython/jit/codewriter/jtransform.py +++ b/rpython/jit/codewriter/jtransform.py @@ -612,18 +612,14 @@ # XXX only strings or simple arrays for now ARRAY = op.args[0].value arraydescr = self.cpu.arraydescrof(ARRAY) - op1 = SpaceOperation('new_array', [op.args[2], arraydescr], - op.result) if op.args[1].value.get('zero', False): - # complicated logic here - we only need to emit zero_contents - # in case this is an array of non-gcptrs and non-structs - if isinstance(ARRAY.OF, lltype.Ptr) and ARRAY.OF._needsgc(): - pass - elif isinstance(ARRAY.OF, lltype.Struct): - pass - else: - return self.zero_contents([op1], op.result, ARRAY) - return op1 + opname = 'new_array_clear' + elif ((isinstance(ARRAY.OF, lltype.Ptr) and ARRAY.OF._needsgc()) or + isinstance(ARRAY.OF, lltype.Struct)): + opname = 'new_array_clear' + else: + opname = 'new_array' + return SpaceOperation(opname, [op.args[2], arraydescr], op.result) def zero_contents(self, ops, v, TYPE): if isinstance(TYPE, lltype.Struct): @@ -639,9 +635,7 @@ self.extend_with(ops, self.rewrite_op_setfield(op, override_type=TYPE)) elif isinstance(TYPE, lltype.Array): - arraydescr = self.cpu.arraydescrof(TYPE) - ops.append(SpaceOperation('clear_array_contents', - [v, arraydescr], None)) + assert False # this operation disappeared else: raise TypeError("Expected struct or array, got '%r'", (TYPE,)) if len(ops) == 1: diff --git a/rpython/jit/codewriter/test/test_jtransform.py b/rpython/jit/codewriter/test/test_jtransform.py --- a/rpython/jit/codewriter/test/test_jtransform.py +++ b/rpython/jit/codewriter/test/test_jtransform.py @@ -1061,9 +1061,8 @@ v2 = varoftype(c_A.value) c_flags = Constant({"flavor": "gc", "zero": True}, lltype.Void) op = SpaceOperation('malloc_varsize', [c_A, c_flags, v1], v2) - op1, op2 = Transformer(FakeCPU()).rewrite_operation(op) - assert op1.opname == 'new_array' - assert op2.opname == 'clear_array_contents' + op1 = Transformer(FakeCPU()).rewrite_operation(op) + assert op1.opname == 'new_array_clear' def test_str_concat(): # test that the oopspec is present and correctly transformed diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py --- a/rpython/jit/metainterp/blackhole.py +++ b/rpython/jit/metainterp/blackhole.py @@ -1152,6 +1152,7 @@ @arguments("cpu", "i", "d", returns="r") def bhimpl_new_array(cpu, length, arraydescr): return cpu.bh_new_array(length, arraydescr) + bhimpl_new_array_clear = bhimpl_new_array @arguments("cpu", "r", "i", "d", returns="i") def bhimpl_getarrayitem_gc_i(cpu, array, index, arraydescr): From noreply at buildbot.pypy.org Thu Sep 25 11:39:11 2014 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 25 Sep 2014 11:39:11 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: Kill clear_array_contents and replace with NEW_ARRAY_CLEAR *should* work Message-ID: <20140925093911.821E01C0250@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73695:906a0b752d1f Date: 2014-09-25 11:38 +0200 http://bitbucket.org/pypy/pypy/changeset/906a0b752d1f/ Log: Kill clear_array_contents and replace with NEW_ARRAY_CLEAR *should* work equally well and it's a cleaner concept diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py --- a/rpython/jit/backend/llgraph/runner.py +++ b/rpython/jit/backend/llgraph/runner.py @@ -643,20 +643,7 @@ def bh_new_array(self, length, arraydescr): array = lltype.malloc(arraydescr.A, length, zero=True) return lltype.cast_opaque_ptr(llmemory.GCREF, array) - - def bh_clear_array_contents(self, a, descr): - a = support.cast_arg(lltype.Ptr(descr.A), a) - ITEM = descr.A.OF - array = a._obj - if isinstance(ITEM, lltype.Struct): - for i in xrange(array.getlength()): - for name, FIELD in ITEM._flds.iteritems(): - null = FIELD._defl() - setattr(array.getitem(i), name, null) - else: - null = ITEM._defl() - for i in xrange(array.getlength()): - array.setitem(i, null) + bh_new_array_clear = bh_new_array def bh_classof(self, struct): struct = lltype.cast_opaque_ptr(rclass.OBJECTPTR, struct) diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py --- a/rpython/jit/backend/llsupport/llmodel.py +++ b/rpython/jit/backend/llsupport/llmodel.py @@ -592,14 +592,6 @@ def bh_new(self, sizedescr): return self.gc_ll_descr.gc_malloc(sizedescr) - def bh_clear_array_contents(self, ref, arraydescr): - ofs, size, _ = self.unpack_arraydescr_size(arraydescr) - arraysize = self.bh_arraylen_gc(ref, arraydescr) - totalsize = size * arraysize - adr = rffi.cast(lltype.Signed, ref) + ofs - memset_fn(rffi.cast(llmemory.Address, adr), rffi.cast(rffi.INT, 0), - rffi.cast(rffi.SIZE_T, totalsize)) - def bh_new_with_vtable(self, vtable, sizedescr): res = self.gc_ll_descr.gc_malloc(sizedescr) if self.vtable_offset is not None: diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -66,9 +66,6 @@ continue if op.is_guard(): self.emit_pending_zeros() - elif op.getopnum() == rop.CLEAR_ARRAY_CONTENTS: - self.handle_clear_array_contents(op.getdescr(), op.getarg(0)) - continue elif op.can_malloc(): self.emitting_an_operation_that_can_collect() elif op.getopnum() == rop.LABEL: @@ -117,7 +114,7 @@ [op.result, ConstInt(classint)], None, descr=self.gc_ll_descr.fielddescr_vtable) self.newops.append(op) - elif opnum == rop.NEW_ARRAY: + elif opnum == rop.NEW_ARRAY or opnum == rop.NEW_ARRAY_CLEAR: descr = op.getdescr() assert isinstance(descr, ArrayDescr) self.handle_new_array(descr, op) @@ -150,14 +147,11 @@ except KeyError: pass - def clear_varsize_gc_fields(self, kind, descr, result, v_length): + def clear_varsize_gc_fields(self, kind, descr, result, v_length, opnum): if self.gc_ll_descr.malloc_zero_filled: return if kind == FLAG_ARRAY: - if descr.is_array_of_structs() or descr.is_array_of_pointers(): - # for the case of array of structs, this is for correctness - # only, since in practice all GC arrays of structs are - # allocated with malloc(zero=True) + if opnum == rop.NEW_ARRAY_CLEAR: self.handle_clear_array_contents(descr, result, v_length) return if kind == FLAG_STR: @@ -200,7 +194,7 @@ # stuff that initializes GC header fields differently self.gen_initialize_len(op.result, v_length, arraydescr.lendescr) self.clear_varsize_gc_fields(kind, op.getdescr(), op.result, - v_length) + v_length, op.getopnum()) return if (total_size >= 0 and self.gen_malloc_nursery(total_size, op.result)): @@ -218,7 +212,8 @@ self.gen_malloc_unicode(v_length, op.result) else: raise NotImplementedError(op.getopname()) - self.clear_varsize_gc_fields(kind, op.getdescr(), op.result, v_length) + self.clear_varsize_gc_fields(kind, op.getdescr(), op.result, v_length, + op.getopnum()) def handle_clear_array_contents(self, arraydescr, v_arr, v_length=None): # XXX more work here to reduce or remove the ZERO_ARRAY in some cases diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py --- a/rpython/jit/backend/test/runner_test.py +++ b/rpython/jit/backend/test/runner_test.py @@ -2036,6 +2036,14 @@ 'ref', descr=arraydescr) assert r1.value != r2.value a = lltype.cast_opaque_ptr(lltype.Ptr(A), r1.value) + assert len(a) == 342 + + def test_new_array_clear(self): + A = lltype.GcArray(lltype.Signed) + arraydescr = self.cpu.arraydescrof(A) + r1 = self.execute_operation(rop.NEW_ARRAY_CLEAR, [BoxInt(342)], + 'ref', descr=arraydescr) + a = lltype.cast_opaque_ptr(lltype.Ptr(A), r1.value) assert a[0] == 0 assert len(a) == 342 @@ -4440,24 +4448,6 @@ [boxfloat(12.5)], 'int') assert res.getint() == struct.unpack("I", struct.pack("f", 12.5))[0] - def test_clear_array_contents(self): - from rpython.jit.backend.llsupport.llmodel import AbstractLLCPU - if not isinstance(self.cpu, AbstractLLCPU): - py.test.skip("pointless test on non-asm") - oldval = self.cpu.gc_ll_descr.malloc_zero_filled - self.cpu.gc_ll_descr.malloc_zero_filled = False - try: - A = lltype.GcArray(lltype.Signed) - a = lltype.malloc(A, 3) - a[1] = 13 - descr = self.cpu.arraydescrof(A) - ref = lltype.cast_opaque_ptr(llmemory.GCREF, a) - self.execute_operation(rop.CLEAR_ARRAY_CONTENTS, - [BoxPtr(ref)], 'void', descr=descr) - assert a[1] == 0 - finally: - self.cpu.gc_ll_descr.malloc_zero_filled = oldval - def test_zero_ptr_field(self): T = lltype.GcStruct('T') S = lltype.GcStruct('S', ('x', lltype.Ptr(T))) diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -316,8 +316,7 @@ self.assembler.mc.mark_op(op) self.rm.position = i self.xrm.position = i - if (op.has_no_side_effect() and op.result not in self.longevity - and op.opnum != rop.CLEAR_ARRAY_CONTENTS): + if op.has_no_side_effect() and op.result not in self.longevit: i += 1 self.possibly_free_vars_for_op(op) continue diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py --- a/rpython/jit/metainterp/blackhole.py +++ b/rpython/jit/metainterp/blackhole.py @@ -1346,10 +1346,6 @@ vtable = heaptracker.descr2vtable(cpu, descr) return cpu.bh_new_with_vtable(vtable, descr) - @arguments("cpu", "r", "d") - def bhimpl_clear_array_contents(cpu, ref, descr): - cpu.bh_clear_array_contents(ref, descr) - @arguments("cpu", "r", returns="i") def bhimpl_guard_class(cpu, struct): return cpu.bh_classof(struct) diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py --- a/rpython/jit/metainterp/executor.py +++ b/rpython/jit/metainterp/executor.py @@ -337,7 +337,6 @@ rop.CALL_MALLOC_NURSERY, rop.CALL_MALLOC_NURSERY_VARSIZE, rop.CALL_MALLOC_NURSERY_VARSIZE_FRAME, - rop.CLEAR_ARRAY_CONTENTS, rop.LABEL, ): # list of opcodes never executed by pyjitpl continue diff --git a/rpython/jit/metainterp/optimizeopt/test/test_multilabel.py b/rpython/jit/metainterp/optimizeopt/test/test_multilabel.py --- a/rpython/jit/metainterp/optimizeopt/test/test_multilabel.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_multilabel.py @@ -150,10 +150,10 @@ def test_nonmatching_arraystruct_1(self): ops = """ [p1, f0] - p2 = new_array(3, descr=complexarraydescr) + p2 = new_array_clear(3, descr=complexarraydescr) setinteriorfield_gc(p2, 2, f0, descr=complexrealdescr) label(p2, f0) - p4 = new_array(3, descr=complexarraydescr) + p4 = new_array_clear(3, descr=complexarraydescr) setinteriorfield_gc(p4, 2, f0, descr=compleximagdescr) jump(p4, f0) """ @@ -163,10 +163,10 @@ def test_nonmatching_arraystruct_2(self): ops = """ [p1, f0] - p2 = new_array(3, descr=complexarraydescr) + p2 = new_array_clear(3, descr=complexarraydescr) setinteriorfield_gc(p2, 2, f0, descr=complexrealdescr) label(p2, f0) - p4 = new_array(2, descr=complexarraydescr) + p4 = new_array_clear(2, descr=complexarraydescr) setinteriorfield_gc(p4, 0, f0, descr=complexrealdescr) jump(p4, f0) """ @@ -198,7 +198,7 @@ def test_not_virtual_arraystruct(self): ops = """ [p1] - p3 = new_array(3, descr=complexarraydescr) + p3 = new_array_clear(3, descr=complexarraydescr) label(p3) p4 = escape() jump(p4) diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py @@ -940,7 +940,7 @@ def test_virtual_array_of_struct(self): ops = """ [f0, f1, f2, f3] - p0 = new_array(2, descr=complexarraydescr) + p0 = new_array_clear(2, descr=complexarraydescr) setinteriorfield_gc(p0, 0, f1, descr=compleximagdescr) setinteriorfield_gc(p0, 0, f0, descr=complexrealdescr) setinteriorfield_gc(p0, 1, f3, descr=compleximagdescr) @@ -966,7 +966,7 @@ def test_virtual_array_of_struct_forced(self): ops = """ [f0, f1] - p0 = new_array(1, descr=complexarraydescr) + p0 = new_array_clear(1, descr=complexarraydescr) setinteriorfield_gc(p0, 0, f0, descr=complexrealdescr) setinteriorfield_gc(p0, 0, f1, descr=compleximagdescr) f2 = getinteriorfield_gc(p0, 0, descr=complexrealdescr) @@ -978,7 +978,7 @@ expected = """ [f0, f1] f2 = float_mul(f0, f1) - p0 = new_array(1, descr=complexarraydescr) + p0 = new_array_clear(1, descr=complexarraydescr) setinteriorfield_gc(p0, 0, f1, descr=compleximagdescr) setinteriorfield_gc(p0, 0, f0, descr=complexrealdescr) i0 = escape(f2, p0) @@ -989,7 +989,7 @@ def test_virtual_array_of_struct_len(self): ops = """ [] - p0 = new_array(2, descr=complexarraydescr) + p0 = new_array_clear(2, descr=complexarraydescr) i0 = arraylen_gc(p0) finish(i0) """ @@ -5466,33 +5466,6 @@ """ self.optimize_loop(ops, expected) - def test_virtual_clear_array_contents(self): - ops = """ - [] - p0 = new_array(2, descr=arraydescr) - clear_array_contents(p0, descr=arraydescr) - """ - expected = """ - [] - """ - self.optimize_loop(ops, expected) - - def test_virtual_clear_array_contents_escape(self): - ops = """ - [] - p0 = new_array(2, descr=arraydescr) - clear_array_contents(p0, descr=arraydescr) - escape(p0) - """ - expected = """ - [] - p0 = new_array(2, descr=arraydescr) - setarrayitem_gc(p0, 0, 0, descr=arraydescr) - setarrayitem_gc(p0, 1, 0, descr=arraydescr) - escape(p0) - """ - self.optimize_loop(ops, expected) - class TestLLtype(BaseTestOptimizeBasic, LLtypeMixin): pass diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -4023,8 +4023,7 @@ def test_arraycopy_dest_not_virtual(self): ops = ''' [] - p1 = new_array(3, descr=arraydescr) - clear_array_contents(p1, descr=arraydescr) + p1 = new_array_clear(3, descr=arraydescr) p2 = new_array(3, descr=arraydescr) setarrayitem_gc(p1, 2, 10, descr=arraydescr) setarrayitem_gc(p2, 2, 13, descr=arraydescr) diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py --- a/rpython/jit/metainterp/optimizeopt/virtualize.py +++ b/rpython/jit/metainterp/optimizeopt/virtualize.py @@ -264,11 +264,16 @@ class VArrayValue(AbstractVArrayValue): - def __init__(self, arraydescr, constvalue, size, keybox, source_op=None): + def __init__(self, arraydescr, constvalue, size, keybox, source_op=None, + clear=False): AbstractVirtualValue.__init__(self, keybox, source_op) self.arraydescr = arraydescr self.constvalue = constvalue - self._items = [None] * size + if clear: + self._items = [constvalue] * size + else: + self._items = [None] * size + self.clear = clear def getlength(self): return len(self._items) @@ -279,10 +284,6 @@ def set_item_value(self, i, newval): self._items[i] = newval - def initialize_with_zeros(self, optimizer): - for i in range(len(self._items)): - self._items[i] = optimizer.new_const_item(self.arraydescr) - def getitem(self, index): res = self._items[index] return res @@ -316,7 +317,11 @@ self.box = box = self.source_op.result for index in range(len(self._items)): subvalue = self._items[index] - if subvalue is not None: + if subvalue is None: + continue + if subvalue is self.constvalue and self.clear: + continue + else: subbox = subvalue.force_box(optforce) op = ResOperation(rop.SETARRAYITEM_GC, [box, ConstInt(index), subbox], None, @@ -344,9 +349,6 @@ assert isinstance(itemvalue, optimizer.OptValue) self._items[index][ofs] = itemvalue - def initialize_with_zeros(self, optimizer): - assert False, "should never happen" - def _really_force(self, optforce): assert self.source_op is not None if not we_are_translated(): @@ -497,12 +499,15 @@ self.make_equal_to(box, vvalue) return vvalue - def make_varray(self, arraydescr, size, box, source_op=None): + def make_varray(self, arraydescr, size, box, source_op=None, + clear=False): if arraydescr.is_array_of_structs(): + assert clear vvalue = VArrayStructValue(arraydescr, size, box, source_op) else: constvalue = self.new_const_item(arraydescr) - vvalue = VArrayValue(arraydescr, constvalue, size, box, source_op) + vvalue = VArrayValue(arraydescr, constvalue, size, box, source_op, + clear=clear) self.make_equal_to(box, vvalue) return vvalue @@ -674,13 +679,13 @@ else: self.emit_operation(op) - def optimize_CLEAR_ARRAY_CONTENTS(self, op): - v = self.getvalue(op.getarg(0)) - if v.is_virtual(): - # initialize the items, since we need to store zeros - v.initialize_with_zeros(self.optimizer) - return - self.emit_operation(op) + def optimize_NEW_ARRAY_CLEAR(self, op): + sizebox = self.get_constant_box(op.getarg(0)) + if sizebox is not None: + self.make_varray(op.getdescr(), sizebox.getint(), op.result, op, + clear=True) + else: + self.emit_operation(op) def optimize_CALL(self, op): effectinfo = op.getdescr().get_extra_info() diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -396,10 +396,6 @@ def opimpl_new(self, sizedescr): return self.metainterp.execute_new(sizedescr) - @arguments("box", "descr") - def opimpl_clear_array_contents(self, box, descr): - self.metainterp.execute_and_record(rop.CLEAR_ARRAY_CONTENTS, descr, box) - @arguments("descr") def opimpl_new_with_vtable(self, sizedescr): cpu = self.metainterp.cpu @@ -410,6 +406,10 @@ def opimpl_new_array(self, lengthbox, itemsizedescr): return self.metainterp.execute_new_array(itemsizedescr, lengthbox) + @arguments("box", "descr") + def opimpl_new_array_clear(self, lengthbox, itemsizedescr): + return self.metainterp.execute_new_array_clear(itemsizedescr, lengthbox) + @specialize.arg(1) def _do_getarrayitem_gc_any(self, op, arraybox, indexbox, arraydescr): tobox = self.metainterp.heapcache.getarrayitem( @@ -1917,6 +1917,12 @@ self.heapcache.new_array(resbox, lengthbox) return resbox + def execute_new_array_clear(self, itemsizedescr, lengthbox): + resbox = self.execute_and_record(rop.NEW_ARRAY_CLEAR, itemsizedescr, + lengthbox) + self.heapcache.new_array(resbox, lengthbox) + return resbox + def execute_setfield_gc(self, fielddescr, box, valuebox): self.execute_and_record(rop.SETFIELD_GC, fielddescr, box, valuebox) self.heapcache.setfield(box, valuebox, fielddescr) diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py --- a/rpython/jit/metainterp/resoperation.py +++ b/rpython/jit/metainterp/resoperation.py @@ -493,9 +493,6 @@ 'MARK_OPAQUE_PTR/1b', # this one has no *visible* side effect, since the virtualizable # must be forced, however we need to execute it anyway - 'CLEAR_ARRAY_CONTENTS/1d', - # this one does not *really* have a side effect since it's equivalent - # to array just coming zeroed '_NOSIDEEFFECT_LAST', # ----- end of no_side_effect operations ----- 'INCREMENT_DEBUG_COUNTER/1', diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py --- a/rpython/jit/metainterp/resume.py +++ b/rpython/jit/metainterp/resume.py @@ -981,7 +981,11 @@ def allocate_array(self, length, arraydescr): lengthbox = ConstInt(length) - return self.metainterp.execute_new_array(arraydescr, lengthbox) + if (arraydescr.is_array_of_structs() or + arraydescr.is_array_of_pointers()): + return self.metainterp.execute_new_array_clear(arraydescr, + lengthbox) + return self.metainterp.execute_new_array(arraydescr, lengthbox) def allocate_raw_buffer(self, size): cic = self.metainterp.staticdata.callinfocollection From noreply at buildbot.pypy.org Thu Sep 25 11:44:18 2014 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 25 Sep 2014 11:44:18 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: fix Message-ID: <20140925094418.380721C0250@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73696:dfca3fe04235 Date: 2014-09-25 11:41 +0200 http://bitbucket.org/pypy/pypy/changeset/dfca3fe04235/ Log: fix diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py --- a/rpython/jit/backend/llsupport/test/test_rewrite.py +++ b/rpython/jit/backend/llsupport/test/test_rewrite.py @@ -604,7 +604,7 @@ self.gc_ll_descr.max_size_of_young_obj = 2000 self.check_rewrite(""" [i2, p3] - p1 = new_array(129, descr=cdescr) + p1 = new_array_clear(129, descr=cdescr) call(123456) setarrayitem_gc(p1, i2, p3, descr=cdescr) jump() @@ -626,7 +626,7 @@ self.gc_ll_descr.max_size_of_young_obj = 2000 self.check_rewrite(""" [i2, p3] - p1 = new_array(130, descr=cdescr) + p1 = new_array_clear(130, descr=cdescr) call(123456) setarrayitem_gc(p1, i2, p3, descr=cdescr) jump() @@ -658,7 +658,7 @@ def test_label_makes_size_unknown(self): self.check_rewrite(""" [i2, p3] - p1 = new_array(5, descr=cdescr) + p1 = new_array_clear(5, descr=cdescr) label(p1, i2, p3) setarrayitem_gc(p1, i2, p3, descr=cdescr) jump() @@ -738,7 +738,6 @@ %(cdescr.basesize + 5 * cdescr.itemsize)d) setfield_gc(p0, 8111, descr=tiddescr) setfield_gc(p0, 5, descr=clendescr) - zero_array(p0, 0, 5, descr=cdescr) setarrayitem_gc(p0, i2, p1, descr=cdescr) jump() """) diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -316,7 +316,7 @@ self.assembler.mc.mark_op(op) self.rm.position = i self.xrm.position = i - if op.has_no_side_effect() and op.result not in self.longevit: + if op.has_no_side_effect() and op.result not in self.longevity: i += 1 self.possibly_free_vars_for_op(op) continue diff --git a/rpython/jit/metainterp/test/test_dict.py b/rpython/jit/metainterp/test/test_dict.py --- a/rpython/jit/metainterp/test/test_dict.py +++ b/rpython/jit/metainterp/test/test_dict.py @@ -168,7 +168,7 @@ res = self.meta_interp(f, [100], listops=True) assert res == f(50) - self.check_resops({'new_array': 2, 'getfield_gc': 2, + self.check_resops({'new_array_clear': 2, 'getfield_gc': 2, 'guard_true': 2, 'jump': 1, 'new_with_vtable': 2, 'getinteriorfield_gc': 2, 'setfield_gc': 8, 'int_gt': 2, 'int_sub': 2, From noreply at buildbot.pypy.org Thu Sep 25 12:12:38 2014 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 25 Sep 2014 12:12:38 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: only store gc fielddescrs Message-ID: <20140925101238.20A161D3944@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73697:bd40306ca234 Date: 2014-09-25 12:11 +0200 http://bitbucket.org/pypy/pypy/changeset/bd40306ca234/ Log: only store gc fielddescrs diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py --- a/rpython/jit/backend/llsupport/descr.py +++ b/rpython/jit/backend/llsupport/descr.py @@ -36,10 +36,10 @@ tid = llop.combine_ushort(lltype.Signed, 0, 0) def __init__(self, size, count_fields_if_immut=-1, - fielddescrs=None): + gc_fielddescrs=None): self.size = size self.count_fields_if_immut = count_fields_if_immut - self.fielddescrs = fielddescrs + self.gc_fielddescrs = gc_fielddescrs def count_fields_if_immutable(self): return self.count_fields_if_immut @@ -60,13 +60,13 @@ except KeyError: size = symbolic.get_size(STRUCT, gccache.translate_support_code) count_fields_if_immut = heaptracker.count_fields_if_immutable(STRUCT) - fielddescrs = heaptracker.fielddescrs(gccache, STRUCT) + gc_fielddescrs = heaptracker.gc_fielddescrs(gccache, STRUCT) if heaptracker.has_gcstruct_a_vtable(STRUCT): sizedescr = SizeDescrWithVTable(size, count_fields_if_immut, - fielddescrs) + gc_fielddescrs) else: sizedescr = SizeDescr(size, count_fields_if_immut, - fielddescrs) + gc_fielddescrs) gccache.init_size_descr(STRUCT, sizedescr) cache[STRUCT] = sizedescr return sizedescr diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -135,10 +135,9 @@ except KeyError: d = {} self.delayed_zero_setfields[result] = d - for fielddescr in descr.fielddescrs: - if fielddescr.is_pointer_field(): - ofs = self.cpu.unpack_fielddescr(fielddescr) - d[ofs] = None + for fielddescr in descr.gc_fielddescrs: + ofs = self.cpu.unpack_fielddescr(fielddescr) + d[ofs] = None def consider_setfield_gc(self, op): offset = self.cpu.unpack_fielddescr(op.getdescr()) diff --git a/rpython/jit/backend/llsupport/test/test_descr.py b/rpython/jit/backend/llsupport/test/test_descr.py --- a/rpython/jit/backend/llsupport/test/test_descr.py +++ b/rpython/jit/backend/llsupport/test/test_descr.py @@ -19,8 +19,8 @@ assert descr_t.size == symbolic.get_size(T, False) assert descr_s.count_fields_if_immutable() == -1 assert descr_t.count_fields_if_immutable() == -1 - assert descr_t.fielddescrs == [] - assert len(descr_s.fielddescrs) == 2 + assert descr_t.gc_fielddescrs == [] + assert len(descr_s.gc_fielddescrs) == 1 assert descr_s == get_size_descr(c0, S) assert descr_s != get_size_descr(c1, S) # @@ -31,7 +31,7 @@ PARENT = lltype.Struct('P', ('x', lltype.Ptr(T))) STRUCT = lltype.GcStruct('S', ('parent', PARENT), ('y', lltype.Ptr(T))) descr_struct = get_size_descr(c0, STRUCT) - assert len(descr_struct.fielddescrs) == 2 + assert len(descr_struct.gc_fielddescrs) == 2 def test_get_size_descr_immut(): S = lltype.GcStruct('S', hints={'immutable': True}) diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py --- a/rpython/jit/backend/llsupport/test/test_rewrite.py +++ b/rpython/jit/backend/llsupport/test/test_rewrite.py @@ -154,7 +154,7 @@ def setup_method(self, meth): class FakeCPU(BaseFakeCPU): def sizeof(self, STRUCT): - return SizeDescrWithVTable(102, fielddescrs=[]) + return SizeDescrWithVTable(102, gc_fielddescrs=[]) self.cpu = FakeCPU() self.gc_ll_descr = GcLLDescr_boehm(None, None, None) @@ -291,7 +291,7 @@ # class FakeCPU(BaseFakeCPU): def sizeof(self, STRUCT): - descr = SizeDescrWithVTable(104, fielddescrs=[]) + descr = SizeDescrWithVTable(104, gc_fielddescrs=[]) descr.tid = 9315 return descr self.cpu = FakeCPU() @@ -324,7 +324,7 @@ setfield_gc(p1, 5678, descr=tiddescr) p2 = int_add(p1, %(tdescr.size)d) setfield_gc(p2, 1234, descr=tiddescr) - zero_ptr_field(p1, %(tdescr.fielddescrs[1].offset)s) + zero_ptr_field(p1, %(tdescr.gc_fielddescrs[0].offset)s) jump() """) @@ -771,7 +771,7 @@ [i0] p0 = call_malloc_nursery(%(tdescr.size)d) setfield_gc(p0, 5678, descr=tiddescr) - zero_ptr_field(p0, %(tdescr.fielddescrs[1].offset)s) + zero_ptr_field(p0, %(tdescr.gc_fielddescrs[0].offset)s) p1 = call_malloc_nursery_varsize(1, 1, i0, \ descr=strdescr) setfield_gc(p1, i0, descr=strlendescr) @@ -792,7 +792,7 @@ [p1] p0 = call_malloc_nursery(%(tdescr.size)d) setfield_gc(p0, 5678, descr=tiddescr) - zero_ptr_field(p0, %(tdescr.fielddescrs[1].offset)s) + zero_ptr_field(p0, %(tdescr.gc_fielddescrs[0].offset)s) label(p0, p1) cond_call_gc_wb(p0, descr=wbdescr) setfield_gc(p0, p1, descr=tzdescr) diff --git a/rpython/jit/codewriter/heaptracker.py b/rpython/jit/codewriter/heaptracker.py --- a/rpython/jit/codewriter/heaptracker.py +++ b/rpython/jit/codewriter/heaptracker.py @@ -126,7 +126,7 @@ vtable = llmemory.cast_ptr_to_adr(vtable) return adr2int(vtable) -def fielddescrs(gccache, STRUCT, res=None): +def gc_fielddescrs(gccache, STRUCT, res=None): from rpython.jit.backend.llsupport import descr if res is None: @@ -137,7 +137,7 @@ if FIELD is lltype.Void: continue elif isinstance(FIELD, lltype.Struct): - fielddescrs(gccache, FIELD, res) - else: + gc_fielddescrs(gccache, FIELD, res) + elif isinstance(FIELD, lltype.Ptr) and FIELD._needsgc(): res.append(descr.get_field_descr(gccache, STRUCT, name)) return res From noreply at buildbot.pypy.org Thu Sep 25 12:27:22 2014 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 25 Sep 2014 12:27:22 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: enforce more and skip some tests Message-ID: <20140925102722.27C8B1C12CC@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73698:b61f12a5227e Date: 2014-09-25 12:26 +0200 http://bitbucket.org/pypy/pypy/changeset/b61f12a5227e/ Log: enforce more and skip some tests diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py --- a/rpython/jit/backend/llgraph/runner.py +++ b/rpython/jit/backend/llgraph/runner.py @@ -642,8 +642,12 @@ def bh_new_array(self, length, arraydescr): array = lltype.malloc(arraydescr.A, length, zero=True) + assert getkind(arraydescr.A.OF) != 'ref' # getkind crashes on structs return lltype.cast_opaque_ptr(llmemory.GCREF, array) - bh_new_array_clear = bh_new_array + + def bh_new_array_clear(self, length, arraydescr): + array = lltype.malloc(arraydescr.A, length, zero=True) + return lltype.cast_opaque_ptr(llmemory.GCREF, array) def bh_classof(self, struct): struct = lltype.cast_opaque_ptr(rclass.OBJECTPTR, struct) diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py --- a/rpython/jit/backend/test/runner_test.py +++ b/rpython/jit/backend/test/runner_test.py @@ -4449,6 +4449,10 @@ assert res.getint() == struct.unpack("I", struct.pack("f", 12.5))[0] def test_zero_ptr_field(self): + from rpython.jit.backend.llsupport.llmodel import AbstractLLCPU + + if not isinstance(self.cpu, AbstractLLCPU): + py.test.skip("llgraph can't do zero_ptr_field") T = lltype.GcStruct('T') S = lltype.GcStruct('S', ('x', lltype.Ptr(T))) tdescr = self.cpu.sizeof(T) @@ -4469,7 +4473,12 @@ s = lltype.cast_opaque_ptr(lltype.Ptr(S), ref) assert not s.x - def test_zero_ptr_field(self): + def test_zero_ptr_field_2(self): + from rpython.jit.backend.llsupport.llmodel import AbstractLLCPU + + if not isinstance(self.cpu, AbstractLLCPU): + py.test.skip("llgraph does not do zero_ptr_field") + from rpython.jit.backend.llsupport import symbolic S = lltype.GcStruct('S', ('x', lltype.Signed), ('p', llmemory.GCREF), @@ -4490,6 +4499,11 @@ assert s.y == -4398176 def test_zero_array(self): + from rpython.jit.backend.llsupport.llmodel import AbstractLLCPU + + if not isinstance(self.cpu, AbstractLLCPU): + py.test.skip("llgraph does not do zero_array") + PAIR = lltype.Struct('PAIR', ('a', lltype.Signed), ('b', lltype.Signed)) for OF in [lltype.Signed, rffi.INT, rffi.SHORT, rffi.UCHAR, PAIR]: A = lltype.GcArray(OF) From noreply at buildbot.pypy.org Thu Sep 25 12:29:36 2014 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 25 Sep 2014 12:29:36 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: fix Message-ID: <20140925102936.116981C12CC@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73699:ad3983952de1 Date: 2014-09-25 12:29 +0200 http://bitbucket.org/pypy/pypy/changeset/ad3983952de1/ Log: fix diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py --- a/rpython/jit/metainterp/blackhole.py +++ b/rpython/jit/metainterp/blackhole.py @@ -1152,7 +1152,10 @@ @arguments("cpu", "i", "d", returns="r") def bhimpl_new_array(cpu, length, arraydescr): return cpu.bh_new_array(length, arraydescr) - bhimpl_new_array_clear = bhimpl_new_array + + @arguments("cpu", "i", "d", returns="r") + def bhimpl_new_array_clear(cpu, length, arraydescr): + return cpu.bh_new_array_clear(length, arraydescr) @arguments("cpu", "r", "i", "d", returns="i") def bhimpl_getarrayitem_gc_i(cpu, array, index, arraydescr): From noreply at buildbot.pypy.org Thu Sep 25 12:33:10 2014 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 25 Sep 2014 12:33:10 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: missing case Message-ID: <20140925103310.6309A1D398D@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73700:0cb0f646d4d5 Date: 2014-09-25 12:32 +0200 http://bitbucket.org/pypy/pypy/changeset/0cb0f646d4d5/ Log: missing case diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py --- a/rpython/jit/codewriter/jtransform.py +++ b/rpython/jit/codewriter/jtransform.py @@ -1668,7 +1668,13 @@ v = Variable('new_length') v.concretetype = lltype.Signed ops.append(SpaceOperation('int_force_ge_zero', [v_length], v)) - ops.append(SpaceOperation('new_array', [v, arraydescr], op.result)) + ARRAY = op.result.concretetype.TO + if ((isinstance(ARRAY.OF, lltype.Ptr) and ARRAY.OF._needsgc()) or + isinstance(ARRAY.OF, lltype.Struct)): + opname = 'new_array_clear' + else: + opname = 'new_array' + ops.append(SpaceOperation(opname, [v, arraydescr], op.result)) return ops def do_fixed_list_len(self, op, args, arraydescr): From noreply at buildbot.pypy.org Thu Sep 25 12:35:54 2014 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 25 Sep 2014 12:35:54 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: one more place, well spotted Message-ID: <20140925103554.D08901D398E@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73701:ba0ca08651ff Date: 2014-09-25 12:35 +0200 http://bitbucket.org/pypy/pypy/changeset/ba0ca08651ff/ Log: one more place, well spotted diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -203,7 +203,7 @@ self.gen_boehm_malloc_array(arraydescr, v_length, op.result) else: opnum = op.getopnum() - if opnum == rop.NEW_ARRAY: + if opnum == rop.NEW_ARRAY or opnum == rop.NEW_ARRAY_CLEAR: self.gen_malloc_array(arraydescr, v_length, op.result) elif opnum == rop.NEWSTR: self.gen_malloc_str(v_length, op.result) From noreply at buildbot.pypy.org Thu Sep 25 12:39:59 2014 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 25 Sep 2014 12:39:59 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: minor adjustments Message-ID: <20140925103959.581FF1C0250@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73702:58d8d6eb499c Date: 2014-09-25 12:39 +0200 http://bitbucket.org/pypy/pypy/changeset/58d8d6eb499c/ Log: minor adjustments diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py --- a/rpython/jit/backend/llsupport/llmodel.py +++ b/rpython/jit/backend/llsupport/llmodel.py @@ -608,6 +608,7 @@ def bh_new_array(self, length, arraydescr): return self.gc_ll_descr.gc_malloc_array(length, arraydescr) + bh_new_array_clear = bh_new_array def bh_newstr(self, length): return self.gc_ll_descr.gc_malloc_str(length) diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py --- a/rpython/jit/metainterp/resume.py +++ b/rpython/jit/metainterp/resume.py @@ -1307,6 +1307,9 @@ return self.cpu.bh_new(typedescr) def allocate_array(self, length, arraydescr): + if (arraydescr.is_array_of_structs() or + arraydescr.is_array_of_pointers()): + return self.cpu.bh_new_array_clear(length, arraydescr) return self.cpu.bh_new_array(length, arraydescr) def allocate_string(self, length): From noreply at buildbot.pypy.org Thu Sep 25 13:15:40 2014 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 25 Sep 2014 13:15:40 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: one more fix Message-ID: <20140925111540.DC7241C12CC@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73703:1044dc58b44b Date: 2014-09-25 13:15 +0200 http://bitbucket.org/pypy/pypy/changeset/1044dc58b44b/ Log: one more fix diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py --- a/rpython/jit/metainterp/blackhole.py +++ b/rpython/jit/metainterp/blackhole.py @@ -1008,7 +1008,11 @@ itemsdescr, arraydescr): result = cpu.bh_new(structdescr) cpu.bh_setfield_gc_i(result, length, lengthdescr) - items = cpu.bh_new_array(length, arraydescr) + if (arraydescr.is_array_of_structs() or + arraydescr.is_array_of_pointers()): + items = cpu.bh_new_array_clear(length, arraydescr) + else: + items = cpu.bh_new_array(length, arraydescr) cpu.bh_setfield_gc_r(result, items, itemsdescr) return result @@ -1017,7 +1021,11 @@ itemsdescr, arraydescr): result = cpu.bh_new(structdescr) cpu.bh_setfield_gc_i(result, 0, lengthdescr) - items = cpu.bh_new_array(lengthhint, arraydescr) + if (arraydescr.is_array_of_structs() or + arraydescr.is_array_of_pointers()): + items = cpu.bh_new_array_clear(lengthhint, arraydescr) + else: + items = cpu.bh_new_array(lengthhint, arraydescr) cpu.bh_setfield_gc_r(result, items, itemsdescr) return result diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -513,7 +513,11 @@ itemsdescr, arraydescr): sbox = self.opimpl_new(structdescr) self._opimpl_setfield_gc_any(sbox, sizebox, lengthdescr) - abox = self.opimpl_new_array(sizebox, arraydescr) + if (arraydescr.is_array_of_structs() or + arraydescr.is_array_of_pointers()): + abox = self.opimpl_new_array_clear(sizebox, arraydescr) + else: + abox = self.opimpl_new_array(sizebox, arraydescr) self._opimpl_setfield_gc_any(sbox, abox, itemsdescr) return sbox From noreply at buildbot.pypy.org Thu Sep 25 15:29:26 2014 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 25 Sep 2014 15:29:26 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: boehm zeros memory Message-ID: <20140925132926.E7EDA1D2858@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73704:0fa03688e385 Date: 2014-09-25 15:28 +0200 http://bitbucket.org/pypy/pypy/changeset/0fa03688e385/ Log: boehm zeros memory diff --git a/rpython/memory/gctransform/boehm.py b/rpython/memory/gctransform/boehm.py --- a/rpython/memory/gctransform/boehm.py +++ b/rpython/memory/gctransform/boehm.py @@ -123,6 +123,12 @@ resulttype=llmemory.WeakRefPtr) hop.cast_result(v_wref) + def gct_zero_everything_inside(self, hop): + pass + + def gct_zero_gc_pointers_inside(self, hop): + pass + def gct_weakref_deref(self, hop): v_wref, = hop.spaceop.args v_addr = hop.genop("direct_call", From noreply at buildbot.pypy.org Thu Sep 25 18:48:03 2014 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 25 Sep 2014 18:48:03 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: add an assert Message-ID: <20140925164803.D3BBA1C0250@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73705:f55e2a42f8aa Date: 2014-09-25 18:47 +0200 http://bitbucket.org/pypy/pypy/changeset/f55e2a42f8aa/ Log: add an assert diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -150,6 +150,8 @@ if self.gc_ll_descr.malloc_zero_filled: return if kind == FLAG_ARRAY: + if descr.is_array_of_structs() or descr.is_array_of_pointers(): + assert opnum == rop.NEW_ARRAY_CLEAR if opnum == rop.NEW_ARRAY_CLEAR: self.handle_clear_array_contents(descr, result, v_length) return From noreply at buildbot.pypy.org Thu Sep 25 19:02:32 2014 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 25 Sep 2014 19:02:32 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: nice one armin Message-ID: <20140925170232.B189A1C12CC@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: gc_no_cleanup_nursery Changeset: r73706:ca80e05e36b4 Date: 2014-09-25 19:01 +0200 http://bitbucket.org/pypy/pypy/changeset/ca80e05e36b4/ Log: nice one armin diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -526,7 +526,11 @@ itemsdescr, arraydescr): sbox = self.opimpl_new(structdescr) self._opimpl_setfield_gc_any(sbox, history.CONST_FALSE, lengthdescr) - abox = self.opimpl_new_array(sizehintbox, arraydescr) + if (arraydescr.is_array_of_structs() or + arraydescr.is_array_of_pointers()): + abox = self.opimpl_new_array_clear(sizehintbox, arraydescr) + else: + abox = self.opimpl_new_array(sizehintbox, arraydescr) self._opimpl_setfield_gc_any(sbox, abox, itemsdescr) return sbox From noreply at buildbot.pypy.org Thu Sep 25 19:25:14 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 25 Sep 2014 19:25:14 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: A minor improvement and a passing test Message-ID: <20140925172514.937E71D29DF@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: gc_no_cleanup_nursery Changeset: r73707:3e8238857003 Date: 2014-09-25 19:24 +0200 http://bitbucket.org/pypy/pypy/changeset/3e8238857003/ Log: A minor improvement and a passing test diff --git a/rpython/jit/codewriter/assembler.py b/rpython/jit/codewriter/assembler.py --- a/rpython/jit/codewriter/assembler.py +++ b/rpython/jit/codewriter/assembler.py @@ -291,6 +291,7 @@ 'int_sub', 'jit_merge_point', 'new_array', + 'new_array_clear', 'newstr', 'setarrayitem_gc_i', 'setarrayitem_gc_r', diff --git a/rpython/jit/codewriter/test/test_list.py b/rpython/jit/codewriter/test/test_list.py --- a/rpython/jit/codewriter/test/test_list.py +++ b/rpython/jit/codewriter/test/test_list.py @@ -11,6 +11,7 @@ # ____________________________________________________________ FIXEDLIST = lltype.Ptr(lltype.GcArray(lltype.Signed)) +FIXEDPTRLIST = lltype.Ptr(lltype.GcArray(FIXEDLIST)) VARLIST = lltype.Ptr(lltype.GcStruct('VARLIST', ('length', lltype.Signed), ('items', FIXEDLIST), @@ -100,6 +101,8 @@ builtin_test('newlist', [Constant(5, lltype.Signed), varoftype(lltype.Signed)], FIXEDLIST, NotSupported) + builtin_test('newlist', [], FIXEDPTRLIST, + """new_array_clear $0, -> %r0""") def test_fixed_ll_arraycopy(): builtin_test('list.ll_arraycopy', From noreply at buildbot.pypy.org Thu Sep 25 20:03:39 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 25 Sep 2014 20:03:39 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: Restore the check for is_null() from trunk Message-ID: <20140925180339.4E9881D38EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: gc_no_cleanup_nursery Changeset: r73708:54ddfd15ce4a Date: 2014-09-25 19:57 +0200 http://bitbucket.org/pypy/pypy/changeset/54ddfd15ce4a/ Log: Restore the check for is_null() from trunk diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py --- a/rpython/jit/metainterp/optimizeopt/virtualize.py +++ b/rpython/jit/metainterp/optimizeopt/virtualize.py @@ -319,14 +319,14 @@ subvalue = self._items[index] if subvalue is None: continue - if subvalue is self.constvalue and self.clear: - continue - else: - subbox = subvalue.force_box(optforce) - op = ResOperation(rop.SETARRAYITEM_GC, - [box, ConstInt(index), subbox], None, - descr=self.arraydescr) - optforce.emit_operation(op) + if self.clear: + if subvalue is self.constvalue or subvalue.is_null(): + continue + subbox = subvalue.force_box(optforce) + op = ResOperation(rop.SETARRAYITEM_GC, + [box, ConstInt(index), subbox], None, + descr=self.arraydescr) + optforce.emit_operation(op) @specialize.argtype(1) def _visitor_dispatch_virtual_type(self, visitor): From noreply at buildbot.pypy.org Thu Sep 25 20:03:40 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 25 Sep 2014 20:03:40 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: Tweaks tweaks tweaks: pass around the 'clear' information to resume.py. Message-ID: <20140925180340.914B81D38EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: gc_no_cleanup_nursery Changeset: r73709:2e8918763cd6 Date: 2014-09-25 19:59 +0200 http://bitbucket.org/pypy/pypy/changeset/2e8918763cd6/ Log: Tweaks tweaks tweaks: pass around the 'clear' information to resume.py. Don't pass it around to virtualstate.py, but that might just be ok; see comment added to this file. diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py --- a/rpython/jit/metainterp/optimizeopt/virtualize.py +++ b/rpython/jit/metainterp/optimizeopt/virtualize.py @@ -279,7 +279,14 @@ return len(self._items) def get_item_value(self, i): - return self._items[i] + """Return the i'th item, unless it is 'constvalue' on a 'clear' + array. In that case, return None. The idea is that this + method returns the value that must be set into an array that + was allocated with zero=True if 'clear' is True.""" + subvalue = self._items[i] + if self.clear and subvalue is self.constvalue: + subvalue = None + return subvalue def set_item_value(self, i, newval): self._items[i] = newval @@ -300,7 +307,7 @@ return self already_forced[self] = self for index in range(self.getlength()): - itemval = self.get_item_value(index) + itemval = self._items[index] # XXX should be skip alltogether, but I don't wanna know or # fight unrolling just yet if itemval is None: @@ -330,7 +337,7 @@ @specialize.argtype(1) def _visitor_dispatch_virtual_type(self, visitor): - return visitor.visit_varray(self.arraydescr) + return visitor.visit_varray(self.arraydescr, self.clear) class VArrayStructValue(AbstractVirtualValue): diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py --- a/rpython/jit/metainterp/optimizeopt/virtualstate.py +++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py @@ -563,7 +563,10 @@ def visit_vstruct(self, typedescr, fielddescrs): return VStructStateInfo(typedescr, fielddescrs) - def visit_varray(self, arraydescr): + def visit_varray(self, arraydescr, clear): + # 'clear' is ignored here. I *think* it is correct, because so + # far in force_at_end_of_preamble() we force all array values + # to be non-None, so clearing is not important any more return VArrayStateInfo(arraydescr) def visit_varraystruct(self, arraydescr, fielddescrs): diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py --- a/rpython/jit/metainterp/resume.py +++ b/rpython/jit/metainterp/resume.py @@ -295,8 +295,11 @@ def visit_vstruct(self, typedescr, fielddescrs): return VStructInfo(typedescr, fielddescrs) - def visit_varray(self, arraydescr): - return VArrayInfo(arraydescr) + def visit_varray(self, arraydescr, clear): + if clear: + return VArrayInfoClear(arraydescr) + else: + return VArrayInfoNotClear(arraydescr) def visit_varraystruct(self, arraydescr, fielddescrs): return VArrayStructInfo(arraydescr, fielddescrs) @@ -545,7 +548,7 @@ debug_print("\tvstructinfo", self.typedescr.repr_rpython(), " at ", compute_unique_id(self)) AbstractVirtualStructInfo.debug_prints(self) -class VArrayInfo(AbstractVirtualInfo): +class AbstractVArrayInfo(AbstractVirtualInfo): def __init__(self, arraydescr): self.arraydescr = arraydescr #self.fieldnums = ... @@ -554,7 +557,7 @@ def allocate(self, decoder, index): length = len(self.fieldnums) arraydescr = self.arraydescr - array = decoder.allocate_array(length, arraydescr) + array = decoder.allocate_array(length, arraydescr, self.clear) decoder.virtuals_cache.set_ptr(index, array) # NB. the check for the kind of array elements is moved out of the loop if arraydescr.is_array_of_pointers(): @@ -572,11 +575,19 @@ return array def debug_prints(self): - debug_print("\tvarrayinfo", self.arraydescr, " at ", compute_unique_id(self)) + debug_print("\tvarrayinfo", self.arraydescr, " at ", + compute_unique_id(self), " clear=", self.clear) for i in self.fieldnums: debug_print("\t\t", str(untag(i))) +class VArrayInfoClear(AbstractVArrayInfo): + clear = True + +class VArrayInfoNotClear(AbstractVArrayInfo): + clear = False + + class VAbstractRawInfo(AbstractVirtualInfo): kind = INT is_about_raw = True @@ -637,7 +648,8 @@ @specialize.argtype(1) def allocate(self, decoder, index): - array = decoder.allocate_array(len(self.fielddescrs), self.arraydescr) + array = decoder.allocate_array(len(self.fielddescrs), self.arraydescr, + clear=True) decoder.virtuals_cache.set_ptr(index, array) p = 0 for i in range(len(self.fielddescrs)): @@ -979,13 +991,12 @@ def allocate_struct(self, typedescr): return self.metainterp.execute_new(typedescr) - def allocate_array(self, length, arraydescr): + def allocate_array(self, length, arraydescr, clear): lengthbox = ConstInt(length) - if (arraydescr.is_array_of_structs() or - arraydescr.is_array_of_pointers()): + if clear: return self.metainterp.execute_new_array_clear(arraydescr, lengthbox) - return self.metainterp.execute_new_array(arraydescr, lengthbox) + return self.metainterp.execute_new_array(arraydescr, lengthbox) def allocate_raw_buffer(self, size): cic = self.metainterp.staticdata.callinfocollection @@ -1306,9 +1317,8 @@ def allocate_struct(self, typedescr): return self.cpu.bh_new(typedescr) - def allocate_array(self, length, arraydescr): - if (arraydescr.is_array_of_structs() or - arraydescr.is_array_of_pointers()): + def allocate_array(self, length, arraydescr, clear): + if clear: return self.cpu.bh_new_array_clear(length, arraydescr) return self.cpu.bh_new_array(length, arraydescr) diff --git a/rpython/jit/metainterp/test/test_resume.py b/rpython/jit/metainterp/test/test_resume.py --- a/rpython/jit/metainterp/test/test_resume.py +++ b/rpython/jit/metainterp/test/test_resume.py @@ -354,7 +354,8 @@ return FakeBuiltObject(vtable=known_class) def allocate_struct(self, typedescr): return FakeBuiltObject(typedescr=typedescr) - def allocate_array(self, length, arraydescr): + def allocate_array(self, length, arraydescr, clear): + assert not clear # the only test uses VArrayInfoNotClear return FakeBuiltObject(arraydescr=arraydescr, items=[None]*length) def setfield(self, struct, fieldnum, descr): setattr(struct, descr, fieldnum) @@ -419,7 +420,7 @@ def test_varrayinfo(): arraydescr = FakeArrayDescr() - info = VArrayInfo(arraydescr) + info = VArrayInfoNotClear(arraydescr) info.fieldnums = [tag(456, TAGINT)] reader = FakeResumeDataReader() reader._prepare_virtuals([info]) From noreply at buildbot.pypy.org Thu Sep 25 20:03:41 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 25 Sep 2014 20:03:41 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: Mention two possible future optimizations (at least one looks correct :-) Message-ID: <20140925180341.B31DE1D38EC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: gc_no_cleanup_nursery Changeset: r73710:111a368affe5 Date: 2014-09-25 20:03 +0200 http://bitbucket.org/pypy/pypy/changeset/111a368affe5/ Log: Mention two possible future optimizations (at least one looks correct :-) diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py --- a/rpython/jit/metainterp/optimizeopt/virtualize.py +++ b/rpython/jit/metainterp/optimizeopt/virtualize.py @@ -320,6 +320,11 @@ assert self.source_op is not None if not we_are_translated(): self.source_op.name = 'FORCE ' + self.source_op.name + # XXX two possible optimizations: + # * if source_op is NEW_ARRAY_CLEAR, emit NEW_ARRAY if it's + # immediately followed by SETARRAYITEM_GC into all items (hard?) + # * if source_op is NEW_ARRAY, emit NEW_ARRAY_CLEAR if it's + # followed by setting most items to zero anyway optforce.emit_operation(self.source_op) self.box = box = self.source_op.result for index in range(len(self._items)): From noreply at buildbot.pypy.org Thu Sep 25 20:20:11 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 25 Sep 2014 20:20:11 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: Fix: vstring no longer removes strsetitem(.., 0). This was a rather Message-ID: <20140925182011.A13CF1C054C@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: gc_no_cleanup_nursery Changeset: r73711:deb4d03d1f18 Date: 2014-09-25 20:19 +0200 http://bitbucket.org/pypy/pypy/changeset/deb4d03d1f18/ Log: Fix: vstring no longer removes strsetitem(.., 0). This was a rather pointless anyway. diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py @@ -4791,15 +4791,18 @@ ops = """ [p0] p1 = newstr(4) + strsetitem(p1, 2, 0) setfield_gc(p0, p1, descr=valuedescr) jump(p0) """ - # It used to be the case that this would have a series of - # strsetitem(p1, idx, 0), which was silly because memory is 0 filled - # when allocated. + # This test is slightly bogus: the string is not fully initialized. + # I *think* it is still right to not have a series of extra + # strsetitem(p1, idx, 0). We do preserve the single one from the + # source, though. expected = """ [p0] p1 = newstr(4) + strsetitem(p1, 2, 0) setfield_gc(p0, p1, descr=valuedescr) jump(p0) """ @@ -5115,6 +5118,9 @@ strsetitem(p1, 6, i0) strsetitem(p1, 7, i0) strsetitem(p1, 8, 3) + strsetitem(p1, 9, 0) + strsetitem(p1, 10, 0) + strsetitem(p1, 11, 0) finish(p1) """ self.optimize_strunicode_loop(ops, expected) diff --git a/rpython/jit/metainterp/optimizeopt/vstring.py b/rpython/jit/metainterp/optimizeopt/vstring.py --- a/rpython/jit/metainterp/optimizeopt/vstring.py +++ b/rpython/jit/metainterp/optimizeopt/vstring.py @@ -189,13 +189,11 @@ charvalue = self.getitem(i) if charvalue is not None: charbox = charvalue.force_box(string_optimizer) - if not (isinstance(charbox, Const) and - charbox.same_constant(CONST_0)): - op = ResOperation(mode.STRSETITEM, [targetbox, - offsetbox, - charbox], - None) - string_optimizer.emit_operation(op) + op = ResOperation(mode.STRSETITEM, [targetbox, + offsetbox, + charbox], + None) + string_optimizer.emit_operation(op) offsetbox = _int_add(string_optimizer, offsetbox, CONST_1) return offsetbox From noreply at buildbot.pypy.org Fri Sep 26 08:56:58 2014 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 26 Sep 2014 08:56:58 +0200 (CEST) Subject: [pypy-commit] pypy default: Failing test: untranslated, we get a string index out of range. Message-ID: <20140926065658.C54C61C3339@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73712:19252e1566a1 Date: 2014-09-26 08:55 +0200 http://bitbucket.org/pypy/pypy/changeset/19252e1566a1/ Log: Failing test: untranslated, we get a string index out of range. Translated, it mostly works, because the string happens to be padded with zeroes. diff --git a/pypy/objspace/std/test/test_newformat.py b/pypy/objspace/std/test/test_newformat.py --- a/pypy/objspace/std/test/test_newformat.py +++ b/pypy/objspace/std/test/test_newformat.py @@ -372,6 +372,7 @@ try: assert locale.format('%g', x, grouping=True) == '1,234.57' assert format(x, 'n') == '1,234.57' + assert format(12345678901234, 'n') == '12,345,678,901,234' finally: locale.setlocale(locale.LC_NUMERIC, 'C') From noreply at buildbot.pypy.org Fri Sep 26 08:56:59 2014 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 26 Sep 2014 08:56:59 +0200 (CEST) Subject: [pypy-commit] pypy default: Fix for 19252e1566a1 Message-ID: <20140926065659.ED9791C3339@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73713:8cc70b841989 Date: 2014-09-26 08:56 +0200 http://bitbucket.org/pypy/pypy/changeset/8cc70b841989/ Log: Fix for 19252e1566a1 diff --git a/pypy/objspace/std/newformat.py b/pypy/objspace/std/newformat.py --- a/pypy/objspace/std/newformat.py +++ b/pypy/objspace/std/newformat.py @@ -580,11 +580,11 @@ elif self._thousands_sep: dec = "." thousands = "," - grouping = "\3\0" + grouping = "\3" else: dec = "." thousands = "" - grouping = "\256" + grouping = "\xFF" # special value to mean 'stop' if self.is_unicode: self._loc_dec = dec.decode("ascii") self._loc_thousands = thousands.decode("ascii") @@ -677,14 +677,16 @@ done = False previous = 0 while True: - group = ord(grouping[grouping_state]) - if group > 0: - if group == 256: + if grouping_state >= len(grouping): + group = previous # end of string + else: + # else, get the next value from the string + group = ord(grouping[grouping_state]) + if group == 0xFF: # special value to mean 'stop' break grouping_state += 1 previous = group - else: - group = previous + # final_grouping = min(group, max(left, max(min_width, 1))) n_zeros = max(0, final_grouping - left) n_chars = max(0, min(left, final_grouping)) From noreply at buildbot.pypy.org Fri Sep 26 09:25:06 2014 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 26 Sep 2014 09:25:06 +0200 (CEST) Subject: [pypy-commit] pypy default: A test which segfaults on "pypy py.test -A" Message-ID: <20140926072506.436941C059C@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73714:632cd69be3d3 Date: 2014-09-26 09:17 +0200 http://bitbucket.org/pypy/pypy/changeset/632cd69be3d3/ Log: A test which segfaults on "pypy py.test -A" diff --git a/pypy/module/select/test/test_select.py b/pypy/module/select/test/test_select.py --- a/pypy/module/select/test/test_select.py +++ b/pypy/module/select/test/test_select.py @@ -244,6 +244,18 @@ raises(OverflowError, pollster.modify, 1, -1) raises(OverflowError, pollster.modify, 1, 1 << 64) + def test_resize_list_in_select(self): + import select + class Foo(object): + def fileno(self): + print len(l) + if len(l) < 100: + l.append(Foo()) + return 0 + l = [Foo()] + select.select(l, (), (), 0) + assert map(type, l) == [Foo] * 100 + class AppTestSelectWithPipes(_AppTestSelect): "Use a pipe to get pairs of file descriptors" From noreply at buildbot.pypy.org Fri Sep 26 09:25:07 2014 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 26 Sep 2014 09:25:07 +0200 (CEST) Subject: [pypy-commit] pypy default: Fix 632cd69be3d3 Message-ID: <20140926072507.6D6611C059C@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73715:6e9694e6d3c9 Date: 2014-09-26 09:24 +0200 http://bitbucket.org/pypy/pypy/changeset/6e9694e6d3c9/ Log: Fix 632cd69be3d3 diff --git a/pypy/module/select/interp_select.py b/pypy/module/select/interp_select.py --- a/pypy/module/select/interp_select.py +++ b/pypy/module/select/interp_select.py @@ -173,9 +173,9 @@ On Windows, only sockets are supported; on Unix, all file descriptors. """ - iwtd_w = space.listview(w_iwtd) - owtd_w = space.listview(w_owtd) - ewtd_w = space.listview(w_ewtd) + iwtd_w = space.unpackiterable(w_iwtd) + owtd_w = space.unpackiterable(w_owtd) + ewtd_w = space.unpackiterable(w_ewtd) if space.is_w(w_timeout, space.w_None): timeout = -1.0 diff --git a/pypy/module/select/test/test_select.py b/pypy/module/select/test/test_select.py --- a/pypy/module/select/test/test_select.py +++ b/pypy/module/select/test/test_select.py @@ -254,7 +254,9 @@ return 0 l = [Foo()] select.select(l, (), (), 0) - assert map(type, l) == [Foo] * 100 + assert 1 <= len(l) <= 100 + # ^^^ CPython gives 100, PyPy gives 1. I think both are OK as + # long as there is no crash. class AppTestSelectWithPipes(_AppTestSelect): From noreply at buildbot.pypy.org Fri Sep 26 09:25:54 2014 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 26 Sep 2014 09:25:54 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: hg merge default Message-ID: <20140926072554.38B3C1C059C@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: gc_no_cleanup_nursery Changeset: r73716:bf10e6f91f1a Date: 2014-09-26 09:25 +0200 http://bitbucket.org/pypy/pypy/changeset/bf10e6f91f1a/ Log: hg merge default diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py --- a/lib_pypy/datetime.py +++ b/lib_pypy/datetime.py @@ -1242,7 +1242,7 @@ (other._hour, other._minute, other._second, other._microsecond)) if myoff is None or otoff is None: - raise TypeError("cannot compare naive and aware times") + raise TypeError("can't compare offset-naive and offset-aware times") myhhmm = self._hour * 60 + self._minute - myoff othhmm = other._hour * 60 + other._minute - otoff return _cmp((myhhmm, self._second, self._microsecond), @@ -1838,7 +1838,7 @@ other._hour, other._minute, other._second, other._microsecond)) if myoff is None or otoff is None: - raise TypeError("cannot compare naive and aware datetimes") + raise TypeError("can't compare offset-naive and offset-aware datetimes") # XXX What follows could be done more efficiently... diff = self - other # this will take offsets into account if diff.days < 0: @@ -1885,7 +1885,7 @@ if myoff == otoff: return base if myoff is None or otoff is None: - raise TypeError("cannot mix naive and timezone-aware time") + raise TypeError("can't subtract offset-naive and offset-aware datetimes") return base + timedelta(minutes = otoff-myoff) def __hash__(self): diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst --- a/pypy/doc/how-to-release.rst +++ b/pypy/doc/how-to-release.rst @@ -38,14 +38,16 @@ no JIT: windows, linux, os/x sandbox: linux, os/x +* repackage and upload source tar.bz2 to bitbucket and to cobra, as some packagers + prefer a clearly labeled source package * write release announcement pypy/doc/release-x.y(.z).txt the release announcement should contain a direct link to the download page * update pypy.org (under extradoc/pypy.org), rebuild and commit * post announcement on morepypy.blogspot.com -* send announcements to pypy-dev, python-list, +* send announcements to twitter.com, pypy-dev, python-list, python-announce, python-dev ... * add a tag on the pypy/jitviewer repo that corresponds to pypy release * add a tag on the codespeed web site that corresponds to pypy release - +* revise versioning at https://readthedocs.org/projects/pypy diff --git a/pypy/doc/release-2.4.0.rst b/pypy/doc/release-2.4.0.rst --- a/pypy/doc/release-2.4.0.rst +++ b/pypy/doc/release-2.4.0.rst @@ -105,7 +105,7 @@ * Many issues were resolved_ since the 2.3.1 release on June 8 -.. _`whats-new`: http://doc.pypy.org/en/latest/whatsnew-2.3.1.html +.. _`whats-new`: http://doc.pypy.org/en/latest/whatsnew-2.4.0.html .. _resolved: https://bitbucket.org/pypy/pypy/issues?status=resolved .. _sandbox: http://doc.pypy.org/en/latest/sandbox.html diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py --- a/pypy/module/micronumpy/ndarray.py +++ b/pypy/module/micronumpy/ndarray.py @@ -407,8 +407,19 @@ -------- numpy.swapaxes : equivalent function """ - if self.is_scalar(): + if axis1 == axis2: return self + n = len(self.get_shape()) + if n <= 1: + return self + if axis1 < 0: + axis1 += n + if axis2 < 0: + axis2 += n + if axis1 < 0 or axis1 >= n: + raise oefmt(space.w_ValueError, "bad axis1 argument to swapaxes") + if axis2 < 0 or axis2 >= n: + raise oefmt(space.w_ValueError, "bad axis2 argument to swapaxes") return self.implementation.swapaxes(space, self, axis1, axis2) def descr_nonzero(self, space): diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py --- a/pypy/module/micronumpy/test/test_ndarray.py +++ b/pypy/module/micronumpy/test/test_ndarray.py @@ -2020,6 +2020,14 @@ def test_swapaxes(self): from numpypy import array + x = array([]) + assert x.swapaxes(0, 2) is x + x = array([[1, 2]]) + assert x.swapaxes(0, 0) is x + exc = raises(ValueError, x.swapaxes, -3, 0) + assert exc.value.message == "bad axis1 argument to swapaxes" + exc = raises(ValueError, x.swapaxes, 0, 3) + assert exc.value.message == "bad axis2 argument to swapaxes" # testcases from numpy docstring x = array([[1, 2, 3]]) assert (x.swapaxes(0, 1) == array([[1], [2], [3]])).all() diff --git a/pypy/module/select/interp_select.py b/pypy/module/select/interp_select.py --- a/pypy/module/select/interp_select.py +++ b/pypy/module/select/interp_select.py @@ -173,9 +173,9 @@ On Windows, only sockets are supported; on Unix, all file descriptors. """ - iwtd_w = space.listview(w_iwtd) - owtd_w = space.listview(w_owtd) - ewtd_w = space.listview(w_ewtd) + iwtd_w = space.unpackiterable(w_iwtd) + owtd_w = space.unpackiterable(w_owtd) + ewtd_w = space.unpackiterable(w_ewtd) if space.is_w(w_timeout, space.w_None): timeout = -1.0 diff --git a/pypy/module/select/test/test_select.py b/pypy/module/select/test/test_select.py --- a/pypy/module/select/test/test_select.py +++ b/pypy/module/select/test/test_select.py @@ -244,6 +244,20 @@ raises(OverflowError, pollster.modify, 1, -1) raises(OverflowError, pollster.modify, 1, 1 << 64) + def test_resize_list_in_select(self): + import select + class Foo(object): + def fileno(self): + print len(l) + if len(l) < 100: + l.append(Foo()) + return 0 + l = [Foo()] + select.select(l, (), (), 0) + assert 1 <= len(l) <= 100 + # ^^^ CPython gives 100, PyPy gives 1. I think both are OK as + # long as there is no crash. + class AppTestSelectWithPipes(_AppTestSelect): "Use a pipe to get pairs of file descriptors" diff --git a/pypy/module/test_lib_pypy/test_datetime.py b/pypy/module/test_lib_pypy/test_datetime.py --- a/pypy/module/test_lib_pypy/test_datetime.py +++ b/pypy/module/test_lib_pypy/test_datetime.py @@ -1,193 +1,221 @@ """Additional tests for datetime.""" from __future__ import absolute_import -from lib_pypy import datetime import py -def test_repr(): - print datetime - expected = "datetime.datetime(1, 2, 3, 0, 0)" - assert repr(datetime.datetime(1,2,3)) == expected +class BaseTestDatetime: + def test_repr(self): + print datetime + expected = "datetime.datetime(1, 2, 3, 0, 0)" + assert repr(datetime.datetime(1,2,3)) == expected -def test_attributes(): - for x in [datetime.date.today(), - datetime.time(), - datetime.datetime.utcnow(), - datetime.timedelta(), - datetime.tzinfo()]: - raises(AttributeError, 'x.abc = 1') + def test_attributes(self): + for x in [datetime.date.today(), + datetime.time(), + datetime.datetime.utcnow(), + datetime.timedelta(), + datetime.tzinfo()]: + raises(AttributeError, 'x.abc = 1') -def test_timedelta_init_long(): - td = datetime.timedelta(microseconds=20000000000000000000) - assert td.days == 231481481 - assert td.seconds == 41600 - td = datetime.timedelta(microseconds=20000000000000000000.) - assert td.days == 231481481 - assert td.seconds == 41600 + def test_timedelta_init_long(self): + td = datetime.timedelta(microseconds=20000000000000000000) + assert td.days == 231481481 + assert td.seconds == 41600 + td = datetime.timedelta(microseconds=20000000000000000000.) + assert td.days == 231481481 + assert td.seconds == 41600 -def test_unpickle(): - e = raises(TypeError, datetime.date, '123') - assert e.value.args[0] == 'an integer is required' - e = raises(TypeError, datetime.time, '123') - assert e.value.args[0] == 'an integer is required' - e = raises(TypeError, datetime.datetime, '123') - assert e.value.args[0] == 'an integer is required' + def test_unpickle(self): + e = raises(TypeError, datetime.date, '123') + assert e.value.args[0] == 'an integer is required' + e = raises(TypeError, datetime.time, '123') + assert e.value.args[0] == 'an integer is required' + e = raises(TypeError, datetime.datetime, '123') + assert e.value.args[0] == 'an integer is required' - datetime.time('\x01' * 6, None) - with raises(TypeError) as e: - datetime.time('\x01' * 6, 123) - assert str(e.value) == "bad tzinfo state arg" + datetime.time('\x01' * 6, None) + with raises(TypeError) as e: + datetime.time('\x01' * 6, 123) + assert str(e.value) == "bad tzinfo state arg" - datetime.datetime('\x01' * 10, None) - with raises(TypeError) as e: - datetime.datetime('\x01' * 10, 123) - assert str(e.value) == "bad tzinfo state arg" + datetime.datetime('\x01' * 10, None) + with raises(TypeError) as e: + datetime.datetime('\x01' * 10, 123) + assert str(e.value) == "bad tzinfo state arg" -def test_strptime(): - import time, sys - if sys.version_info < (2, 6): - py.test.skip("needs the _strptime module") + def test_strptime(self): + import time, sys + if sys.version_info < (2, 6): + py.test.skip("needs the _strptime module") - string = '2004-12-01 13:02:47' - format = '%Y-%m-%d %H:%M:%S' - expected = datetime.datetime(*(time.strptime(string, format)[0:6])) - got = datetime.datetime.strptime(string, format) - assert expected == got + string = '2004-12-01 13:02:47' + format = '%Y-%m-%d %H:%M:%S' + expected = datetime.datetime(*(time.strptime(string, format)[0:6])) + got = datetime.datetime.strptime(string, format) + assert expected == got -def test_datetime_rounding(): - b = 0.0000001 - a = 0.9999994 + def test_datetime_rounding(self): + b = 0.0000001 + a = 0.9999994 - assert datetime.datetime.utcfromtimestamp(a).microsecond == 999999 - assert datetime.datetime.utcfromtimestamp(a).second == 0 - a += b - assert datetime.datetime.utcfromtimestamp(a).microsecond == 999999 - assert datetime.datetime.utcfromtimestamp(a).second == 0 - a += b - assert datetime.datetime.utcfromtimestamp(a).microsecond == 0 - assert datetime.datetime.utcfromtimestamp(a).second == 1 + assert datetime.datetime.utcfromtimestamp(a).microsecond == 999999 + assert datetime.datetime.utcfromtimestamp(a).second == 0 + a += b + assert datetime.datetime.utcfromtimestamp(a).microsecond == 999999 + assert datetime.datetime.utcfromtimestamp(a).second == 0 + a += b + assert datetime.datetime.utcfromtimestamp(a).microsecond == 0 + assert datetime.datetime.utcfromtimestamp(a).second == 1 -def test_more_datetime_rounding(): - # this test verified on top of CPython 2.7 (using a plain - # "import datetime" above) - expected_results = { - -1000.0: 'datetime.datetime(1969, 12, 31, 23, 43, 20)', - -999.9999996: 'datetime.datetime(1969, 12, 31, 23, 43, 20)', - -999.4: 'datetime.datetime(1969, 12, 31, 23, 43, 20, 600000)', - -999.0000004: 'datetime.datetime(1969, 12, 31, 23, 43, 21)', - -1.0: 'datetime.datetime(1969, 12, 31, 23, 59, 59)', - -0.9999996: 'datetime.datetime(1969, 12, 31, 23, 59, 59)', - -0.4: 'datetime.datetime(1969, 12, 31, 23, 59, 59, 600000)', - -0.0000004: 'datetime.datetime(1970, 1, 1, 0, 0)', - 0.0: 'datetime.datetime(1970, 1, 1, 0, 0)', - 0.0000004: 'datetime.datetime(1970, 1, 1, 0, 0)', - 0.4: 'datetime.datetime(1970, 1, 1, 0, 0, 0, 400000)', - 0.9999996: 'datetime.datetime(1970, 1, 1, 0, 0, 1)', - 1000.0: 'datetime.datetime(1970, 1, 1, 0, 16, 40)', - 1000.0000004: 'datetime.datetime(1970, 1, 1, 0, 16, 40)', - 1000.4: 'datetime.datetime(1970, 1, 1, 0, 16, 40, 400000)', - 1000.9999996: 'datetime.datetime(1970, 1, 1, 0, 16, 41)', - 1293843661.191: 'datetime.datetime(2011, 1, 1, 1, 1, 1, 191000)', - } - for t in sorted(expected_results): - dt = datetime.datetime.utcfromtimestamp(t) - assert repr(dt) == expected_results[t] + def test_more_datetime_rounding(self): + # this test verified on top of CPython 2.7 (using a plain + # "import datetime" above) + expected_results = { + -1000.0: 'datetime.datetime(1969, 12, 31, 23, 43, 20)', + -999.9999996: 'datetime.datetime(1969, 12, 31, 23, 43, 20)', + -999.4: 'datetime.datetime(1969, 12, 31, 23, 43, 20, 600000)', + -999.0000004: 'datetime.datetime(1969, 12, 31, 23, 43, 21)', + -1.0: 'datetime.datetime(1969, 12, 31, 23, 59, 59)', + -0.9999996: 'datetime.datetime(1969, 12, 31, 23, 59, 59)', + -0.4: 'datetime.datetime(1969, 12, 31, 23, 59, 59, 600000)', + -0.0000004: 'datetime.datetime(1970, 1, 1, 0, 0)', + 0.0: 'datetime.datetime(1970, 1, 1, 0, 0)', + 0.0000004: 'datetime.datetime(1970, 1, 1, 0, 0)', + 0.4: 'datetime.datetime(1970, 1, 1, 0, 0, 0, 400000)', + 0.9999996: 'datetime.datetime(1970, 1, 1, 0, 0, 1)', + 1000.0: 'datetime.datetime(1970, 1, 1, 0, 16, 40)', + 1000.0000004: 'datetime.datetime(1970, 1, 1, 0, 16, 40)', + 1000.4: 'datetime.datetime(1970, 1, 1, 0, 16, 40, 400000)', + 1000.9999996: 'datetime.datetime(1970, 1, 1, 0, 16, 41)', + 1293843661.191: 'datetime.datetime(2011, 1, 1, 1, 1, 1, 191000)', + } + for t in sorted(expected_results): + dt = datetime.datetime.utcfromtimestamp(t) + assert repr(dt) == expected_results[t] -def test_utcfromtimestamp(): - """Confirm that utcfromtimestamp and fromtimestamp give consistent results. + def test_utcfromtimestamp(self): + """Confirm that utcfromtimestamp and fromtimestamp give consistent results. - Based on danchr's test script in https://bugs.pypy.org/issue986 - """ - import os - import time - if os.name == 'nt': - skip("setting os.environ['TZ'] ineffective on windows") - try: - prev_tz = os.environ.get("TZ") - os.environ["TZ"] = "GMT" - time.tzset() - for unused in xrange(100): - now = time.time() - delta = (datetime.datetime.utcfromtimestamp(now) - - datetime.datetime.fromtimestamp(now)) - assert delta.days * 86400 + delta.seconds == 0 - finally: - if prev_tz is None: - del os.environ["TZ"] - else: - os.environ["TZ"] = prev_tz - time.tzset() + Based on danchr's test script in https://bugs.pypy.org/issue986 + """ + import os + import time + if os.name == 'nt': + skip("setting os.environ['TZ'] ineffective on windows") + try: + prev_tz = os.environ.get("TZ") + os.environ["TZ"] = "GMT" + time.tzset() + for unused in xrange(100): + now = time.time() + delta = (datetime.datetime.utcfromtimestamp(now) - + datetime.datetime.fromtimestamp(now)) + assert delta.days * 86400 + delta.seconds == 0 + finally: + if prev_tz is None: + del os.environ["TZ"] + else: + os.environ["TZ"] = prev_tz + time.tzset() -def test_utcfromtimestamp_microsecond(): - dt = datetime.datetime.utcfromtimestamp(0) - assert isinstance(dt.microsecond, int) + def test_utcfromtimestamp_microsecond(self): + dt = datetime.datetime.utcfromtimestamp(0) + assert isinstance(dt.microsecond, int) -def test_default_args(): - with py.test.raises(TypeError): - datetime.datetime() - with py.test.raises(TypeError): - datetime.datetime(10) - with py.test.raises(TypeError): - datetime.datetime(10, 10) - datetime.datetime(10, 10, 10) + def test_default_args(self): + with py.test.raises(TypeError): + datetime.datetime() + with py.test.raises(TypeError): + datetime.datetime(10) + with py.test.raises(TypeError): + datetime.datetime(10, 10) + datetime.datetime(10, 10, 10) -def test_check_arg_types(): - import decimal - class Number: - def __init__(self, value): - self.value = value - def __int__(self): - return self.value + def test_check_arg_types(self): + import decimal + class Number: + def __init__(self, value): + self.value = value + def __int__(self): + return self.value - for xx in [10L, - decimal.Decimal(10), - decimal.Decimal('10.9'), - Number(10), - Number(10L)]: - assert datetime.datetime(10, 10, 10, 10, 10, 10, 10) == \ - datetime.datetime(xx, xx, xx, xx, xx, xx, xx) + for xx in [10L, + decimal.Decimal(10), + decimal.Decimal('10.9'), + Number(10), + Number(10L)]: + assert datetime.datetime(10, 10, 10, 10, 10, 10, 10) == \ + datetime.datetime(xx, xx, xx, xx, xx, xx, xx) - with py.test.raises(TypeError) as e: - datetime.datetime(10, 10, '10') - assert str(e.value) == 'an integer is required' + with py.test.raises(TypeError) as e: + datetime.datetime(10, 10, '10') + assert str(e.value) == 'an integer is required' - f10 = Number(10.9) - with py.test.raises(TypeError) as e: - datetime.datetime(10, 10, f10) - assert str(e.value) == '__int__ method should return an integer' + f10 = Number(10.9) + with py.test.raises(TypeError) as e: + datetime.datetime(10, 10, f10) + assert str(e.value) == '__int__ method should return an integer' - class Float(float): - pass - s10 = Float(10.9) - with py.test.raises(TypeError) as e: - datetime.datetime(10, 10, s10) - assert str(e.value) == 'integer argument expected, got float' + class Float(float): + pass + s10 = Float(10.9) + with py.test.raises(TypeError) as e: + datetime.datetime(10, 10, s10) + assert str(e.value) == 'integer argument expected, got float' - with py.test.raises(TypeError): - datetime.datetime(10., 10, 10) - with py.test.raises(TypeError): - datetime.datetime(10, 10., 10) - with py.test.raises(TypeError): - datetime.datetime(10, 10, 10.) - with py.test.raises(TypeError): - datetime.datetime(10, 10, 10, 10.) - with py.test.raises(TypeError): - datetime.datetime(10, 10, 10, 10, 10.) - with py.test.raises(TypeError): - datetime.datetime(10, 10, 10, 10, 10, 10.) - with py.test.raises(TypeError): - datetime.datetime(10, 10, 10, 10, 10, 10, 10.) + with py.test.raises(TypeError): + datetime.datetime(10., 10, 10) + with py.test.raises(TypeError): + datetime.datetime(10, 10., 10) + with py.test.raises(TypeError): + datetime.datetime(10, 10, 10.) + with py.test.raises(TypeError): + datetime.datetime(10, 10, 10, 10.) + with py.test.raises(TypeError): + datetime.datetime(10, 10, 10, 10, 10.) + with py.test.raises(TypeError): + datetime.datetime(10, 10, 10, 10, 10, 10.) + with py.test.raises(TypeError): + datetime.datetime(10, 10, 10, 10, 10, 10, 10.) -def test_utcnow_microsecond(): - import copy + def test_utcnow_microsecond(self): + import copy - dt = datetime.datetime.utcnow() - assert type(dt.microsecond) is int + dt = datetime.datetime.utcnow() + assert type(dt.microsecond) is int - copy.copy(dt) + copy.copy(dt) -def test_radd(): - class X(object): - def __radd__(self, other): - return "radd" - assert datetime.date(10, 10, 10) + X() == "radd" + def test_radd(self): + class X(object): + def __radd__(self, other): + return "radd" + assert datetime.date(10, 10, 10) + X() == "radd" + + def test_raises_if_passed_naive_datetime_and_start_or_end_time_defined(self): + class Foo(datetime.tzinfo): + def utcoffset(self, dt): + return datetime.timedelta(0.1) + naive = datetime.datetime(2014, 9, 22) + aware = datetime.datetime(2014, 9, 22, tzinfo=Foo()) + with py.test.raises(TypeError) as e: + naive == aware + assert str(e.value) == "can't compare offset-naive and offset-aware datetimes" + with py.test.raises(TypeError) as e: + naive - aware + assert str(e.value) == "can't subtract offset-naive and offset-aware datetimes" + naive = datetime.time(7, 32, 12) + aware = datetime.time(7, 32, 12, tzinfo=Foo()) + with py.test.raises(TypeError) as e: + naive == aware + assert str(e.value) == "can't compare offset-naive and offset-aware times" + +class TestDatetimeCPython(BaseTestDatetime): + def setup_class(cls): + global datetime + import datetime + +class TestDatetimePyPy(BaseTestDatetime): + def setup_class(cls): + global datetime + from lib_pypy import datetime diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py --- a/pypy/objspace/std/floatobject.py +++ b/pypy/objspace/std/floatobject.py @@ -15,6 +15,7 @@ from rpython.rlib.rbigint import rbigint from rpython.rlib import rfloat from rpython.tool.sourcetools import func_with_new_name +from rpython.rtyper.lltypesystem.module.ll_math import math_fmod from pypy.objspace.std.intobject import W_IntObject @@ -360,21 +361,17 @@ y = w_float2.floatval if y == 0.0: raise FailedToImplementArgs(space.w_ZeroDivisionError, space.wrap("float modulo")) - try: - mod = math.fmod(x, y) - except ValueError: - mod = rfloat.NAN + mod = math_fmod(x, y) + if mod: + # ensure the remainder has the same sign as the denominator + if (y < 0.0) != (mod < 0.0): + mod += y else: - if mod: - # ensure the remainder has the same sign as the denominator - if (y < 0.0) != (mod < 0.0): - mod += y - else: - # the remainder is zero, and in the presence of signed zeroes - # fmod returns different results across platforms; ensure - # it has the same sign as the denominator; we'd like to do - # "mod = y * 0.0", but that may get optimized away - mod = copysign(0.0, y) + # the remainder is zero, and in the presence of signed zeroes + # fmod returns different results across platforms; ensure + # it has the same sign as the denominator; we'd like to do + # "mod = y * 0.0", but that may get optimized away + mod = copysign(0.0, y) return W_FloatObject(mod) @@ -383,10 +380,7 @@ y = w_float2.floatval if y == 0.0: raise FailedToImplementArgs(space.w_ZeroDivisionError, space.wrap("float modulo")) - try: - mod = math.fmod(x, y) - except ValueError: - return [W_FloatObject(rfloat.NAN), W_FloatObject(rfloat.NAN)] + mod = math_fmod(x, y) # fmod is typically exact, so vx-mod is *mathematically* an # exact multiple of wx. But this is fp arithmetic, and fp # vx - mod is an approximation; the result is that div may diff --git a/pypy/objspace/std/newformat.py b/pypy/objspace/std/newformat.py --- a/pypy/objspace/std/newformat.py +++ b/pypy/objspace/std/newformat.py @@ -580,11 +580,11 @@ elif self._thousands_sep: dec = "." thousands = "," - grouping = "\3\0" + grouping = "\3" else: dec = "." thousands = "" - grouping = "\256" + grouping = "\xFF" # special value to mean 'stop' if self.is_unicode: self._loc_dec = dec.decode("ascii") self._loc_thousands = thousands.decode("ascii") @@ -677,14 +677,16 @@ done = False previous = 0 while True: - group = ord(grouping[grouping_state]) - if group > 0: - if group == 256: + if grouping_state >= len(grouping): + group = previous # end of string + else: + # else, get the next value from the string + group = ord(grouping[grouping_state]) + if group == 0xFF: # special value to mean 'stop' break grouping_state += 1 previous = group - else: - group = previous + # final_grouping = min(group, max(left, max(min_width, 1))) n_zeros = max(0, final_grouping - left) n_chars = max(0, min(left, final_grouping)) diff --git a/pypy/objspace/std/test/test_floatobject.py b/pypy/objspace/std/test/test_floatobject.py --- a/pypy/objspace/std/test/test_floatobject.py +++ b/pypy/objspace/std/test/test_floatobject.py @@ -794,7 +794,7 @@ raises(ValueError, float.fromhex, "0P") def test_division_edgecases(self): - import math + import math, os # inf inf = float("inf") @@ -803,6 +803,16 @@ x, y = divmod(inf, 3) assert math.isnan(x) assert math.isnan(y) + x, y = divmod(3, inf) + z = 3 % inf + if os.name == 'nt': + assert math.isnan(x) + assert math.isnan(y) + assert math.isnan(z) + else: + assert x == 0 + assert y == 3 + assert z == 3 # divide by 0 raises(ZeroDivisionError, lambda: inf % 0) diff --git a/pypy/objspace/std/test/test_newformat.py b/pypy/objspace/std/test/test_newformat.py --- a/pypy/objspace/std/test/test_newformat.py +++ b/pypy/objspace/std/test/test_newformat.py @@ -372,6 +372,7 @@ try: assert locale.format('%g', x, grouping=True) == '1,234.57' assert format(x, 'n') == '1,234.57' + assert format(12345678901234, 'n') == '12,345,678,901,234' finally: locale.setlocale(locale.LC_NUMERIC, 'C') diff --git a/pypy/sandbox/pypy_interact.py b/pypy/sandbox/pypy_interact.py --- a/pypy/sandbox/pypy_interact.py +++ b/pypy/sandbox/pypy_interact.py @@ -55,7 +55,7 @@ return Dir({ 'bin': Dir({ - 'pypy-c': RealFile(self.executable), + 'pypy-c': RealFile(self.executable, mode=0111), 'lib-python': RealDir(os.path.join(libroot, 'lib-python'), exclude=exclude), 'lib_pypy': RealDir(os.path.join(libroot, 'lib_pypy'), diff --git a/rpython/translator/backendopt/storesink.py b/rpython/translator/backendopt/storesink.py --- a/rpython/translator/backendopt/storesink.py +++ b/rpython/translator/backendopt/storesink.py @@ -1,6 +1,8 @@ from rpython.rtyper.lltypesystem.lloperation import llop +from rpython.flowspace.model import mkentrymap, Variable from rpython.translator.backendopt import removenoops +from rpython.translator import simplify def has_side_effects(op): if op.opname == 'debug_assert': @@ -10,38 +12,86 @@ except AttributeError: return True + def storesink_graph(graph): + """ remove superfluous getfields. use a super-local method: all non-join + blocks inherit the heap information from their (single) predecessor + """ + added_some_same_as = False + entrymap = mkentrymap(graph) + # all merge blocks are starting points + todo = [(block, None, None) for (block, prev_blocks) in entrymap.iteritems() + if len(prev_blocks) > 1 or block is graph.startblock] + + visited = 0 + + while todo: + block, cache, inputlink = todo.pop() + visited += 1 + if cache is None: + cache = {} + + if block.operations: + changed_block = _storesink_block(block, cache, inputlink) + added_some_same_as = changed_block or added_some_same_as + for link in block.exits: + if len(entrymap[link.target]) == 1: + new_cache = _translate_cache(cache, link) + todo.append((link.target, new_cache, link)) + + assert visited == len(entrymap) + if added_some_same_as: + removenoops.remove_same_as(graph) + simplify.transform_dead_op_vars(graph) + +def _translate_cache(cache, link): + if link.target.operations == (): # exit or except block: + return {} + block = link.target + local_versions = {var1: var2 for var1, var2 in zip(link.args, block.inputargs)} + def _translate_arg(arg): + if isinstance(arg, Variable): + res = local_versions.get(arg, None) + if res is None: + res = Variable(arg) + res.concretetype = arg.concretetype + link.args.append(arg) + block.inputargs.append(res) + local_versions[arg] = res + return res + else: + return arg + new_cache = {} + for (var, field), res in cache.iteritems(): + if var in local_versions or not isinstance(var, Variable): + new_cache[_translate_arg(var), field] = _translate_arg(res) + return new_cache + +def _storesink_block(block, cache, inputlink): def clear_cache_for(cache, concretetype, fieldname): for k in cache.keys(): if k[0].concretetype == concretetype and k[1] == fieldname: del cache[k] added_some_same_as = False - - for block in graph.iterblocks(): - newops = [] - cache = {} - for op in block.operations: - if op.opname == 'getfield': - tup = (op.args[0], op.args[1].value) - res = cache.get(tup, None) - if res is not None: - op.opname = 'same_as' - op.args = [res] - added_some_same_as = True - else: - cache[tup] = op.result - elif op.opname in ['setarrayitem', 'setinteriorfield']: - pass - elif op.opname == 'setfield': - clear_cache_for(cache, op.args[0].concretetype, - op.args[1].value) - elif has_side_effects(op): - cache = {} - newops.append(op) - if block.operations: - block.operations = newops - - if added_some_same_as: - removenoops.remove_same_as(graph) + for op in block.operations: + if op.opname == 'getfield': + tup = (op.args[0], op.args[1].value) + res = cache.get(tup, None) + if res is not None: + op.opname = 'same_as' + op.args = [res] + added_some_same_as = True + else: + cache[tup] = op.result + elif op.opname in ('setarrayitem', 'setinteriorfield', "malloc", "malloc_varsize"): + pass + elif op.opname == 'setfield': + target = op.args[0] + field = op.args[1].value + clear_cache_for(cache, target.concretetype, field) + cache[target, field] = op.args[2] + elif has_side_effects(op): + cache.clear() + return added_some_same_as diff --git a/rpython/translator/backendopt/test/test_storesink.py b/rpython/translator/backendopt/test/test_storesink.py --- a/rpython/translator/backendopt/test/test_storesink.py +++ b/rpython/translator/backendopt/test/test_storesink.py @@ -42,7 +42,7 @@ a.x = i return a.x - self.check(f, [int], 1) + self.check(f, [int], 0) def test_simple(self): class A(object): @@ -53,7 +53,7 @@ a.x = i return a.x + a.x - self.check(f, [int], 1) + self.check(f, [int], 0) def test_irrelevant_setfield(self): class A(object): @@ -67,7 +67,7 @@ two = a.x return one + two - self.check(f, [int], 1) + self.check(f, [int], 0) def test_relevant_setfield(self): class A(object): @@ -101,7 +101,7 @@ two = a.x return one + two - self.check(f, [int], 1) + self.check(f, [int], 0) def test_subclass(self): class A(object): @@ -119,7 +119,7 @@ two = a.x return one + two - self.check(f, [int], 2) + self.check(f, [int], 1) def test_bug_1(self): class A(object): @@ -133,4 +133,37 @@ return True return n - self.check(f, [int], 1) + self.check(f, [int], 0) + + + def test_cfg_splits(self): + class A(object): + pass + + def f(i): + a = A() + j = i + for i in range(i): + a.x = i + if i: + j = a.x + a.x + else: + j = a.x * 5 + return j + + self.check(f, [int], 0) + + def test_malloc_does_not_invalidate(self): + class A(object): + pass + class B(object): + pass + + def f(i): + a = A() + a.x = i + b = B() + return a.x + + self.check(f, [int], 0) + diff --git a/rpython/translator/sandbox/sandlib.py b/rpython/translator/sandbox/sandlib.py --- a/rpython/translator/sandbox/sandlib.py +++ b/rpython/translator/sandbox/sandlib.py @@ -459,6 +459,15 @@ do_ll_os__ll_os_lstat = do_ll_os__ll_os_stat + def do_ll_os__ll_os_access(self, vpathname, mode): + try: + node = self.get_node(vpathname) + except OSError, e: + if e.errno == errno.ENOENT: + return False + raise + return node.access(mode) + def do_ll_os__ll_os_isatty(self, fd): return self.virtual_console_isatty and fd in (0, 1, 2) diff --git a/rpython/translator/sandbox/test/test_vfs.py b/rpython/translator/sandbox/test/test_vfs.py --- a/rpython/translator/sandbox/test/test_vfs.py +++ b/rpython/translator/sandbox/test/test_vfs.py @@ -33,6 +33,8 @@ py.test.raises(OSError, d.join, 'bar') st = d.stat() assert stat.S_ISDIR(st.st_mode) + assert d.access(os.R_OK | os.X_OK) + assert not d.access(os.W_OK) def test_file(): f = File('hello world') @@ -46,6 +48,8 @@ st = f.stat() assert stat.S_ISREG(st.st_mode) assert st.st_size == 11 + assert f.access(os.R_OK) + assert not f.access(os.W_OK) def test_realdir_realfile(): for show_dotfiles in [False, True]: @@ -78,6 +82,7 @@ f = v_test_vfs.join('symlink2') assert stat.S_ISREG(f.stat().st_mode) + assert f.access(os.R_OK) assert f.open().read() == 'secret' else: py.test.raises(OSError, v_test_vfs.join, 'symlink1') diff --git a/rpython/translator/sandbox/vfs.py b/rpython/translator/sandbox/vfs.py --- a/rpython/translator/sandbox/vfs.py +++ b/rpython/translator/sandbox/vfs.py @@ -22,7 +22,7 @@ st_size = self.getsize() st_mode = self.kind st_mode |= stat.S_IWUSR | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH - if self.kind == stat.S_IFDIR: + if stat.S_ISDIR(self.kind): st_mode |= stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH if self.read_only: st_uid = 0 # read-only files are virtually owned by root @@ -37,6 +37,15 @@ (st_mode, st_ino, st_dev, st_nlink, st_uid, st_gid, st_size, st_atime, st_mtime, st_ctime)) + def access(self, mode): + s = self.stat() + e_mode = s.st_mode & stat.S_IRWXO + if UID == s.st_uid: + e_mode |= (s.st_mode & stat.S_IRWXU) >> 6 + if GID == s.st_gid: + e_mode |= (s.st_mode & stat.S_IRWXG) >> 3 + return (e_mode & mode) == mode + def keys(self): raise OSError(errno.ENOTDIR, self) @@ -114,8 +123,9 @@ return cStringIO.StringIO(self.data) class RealFile(File): - def __init__(self, path): + def __init__(self, path, mode=0): self.path = path + self.kind |= mode def __repr__(self): return '' % (self.path,) def getsize(self): From noreply at buildbot.pypy.org Fri Sep 26 09:44:55 2014 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 26 Sep 2014 09:44:55 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: Adapt these three tests Message-ID: <20140926074455.7AF971C0250@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: gc_no_cleanup_nursery Changeset: r73717:75ab1d46c8aa Date: 2014-09-26 09:44 +0200 http://bitbucket.org/pypy/pypy/changeset/75ab1d46c8aa/ Log: Adapt these three tests diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py --- a/pypy/module/pypyjit/test_pypy_c/test_call.py +++ b/pypy/module/pypyjit/test_pypy_c/test_call.py @@ -382,12 +382,16 @@ ... p20 = force_token() p22 = new_with_vtable(...) - p24 = new_array(1, descr=) + p24 = new_array_clear(1, descr=) p26 = new_with_vtable(ConstClass(W_ListObject)) {{{ setfield_gc(p0, p20, descr=) + setfield_gc(p22, ConstPtr(null), descr=) + setfield_gc(p22, ConstPtr(null), descr=) setfield_gc(p22, 1, descr=) + setfield_gc(p22, ConstPtr(null), descr=) setfield_gc(p26, ConstPtr(ptr22), descr=) + setfield_gc(p26, ConstPtr(null), descr=) setarrayitem_gc(p24, 0, p26, descr=) setfield_gc(p22, p24, descr=) }}} diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py --- a/pypy/module/pypyjit/test_pypy_c/test_containers.py +++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py @@ -68,10 +68,13 @@ guard_no_exception(descr=...) i12 = call(ConstClass(ll_strhash), p10, descr=) p13 = new(descr=...) - p15 = new_array(8, descr=) + p15 = new_array_clear(8, descr=) setfield_gc(p13, p15, descr=) i17 = call(ConstClass(ll_dict_lookup_trampoline), p13, p10, i12, descr=) + {{{ setfield_gc(p13, 16, descr=) + setfield_gc(p13, 0, descr=) + }}} guard_no_exception(descr=...) p20 = new_with_vtable(ConstClass(W_IntObject)) call(ConstClass(_ll_dict_setitem_lookup_done_trampoline), p13, p10, p20, i12, i17, descr=) diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py --- a/pypy/module/pypyjit/test_pypy_c/test_string.py +++ b/pypy/module/pypyjit/test_pypy_c/test_string.py @@ -110,9 +110,12 @@ i85 = strlen(p80) p86 = new(descr=) p88 = newstr(23) - setfield_gc(..., descr=) - setfield_gc(..., descr=) - setfield_gc(..., descr=) + {{{ + setfield_gc(p86, 0, descr=) + setfield_gc(p86, p88, descr=) + setfield_gc(p86, 23, descr=) + setfield_gc(p86, 23, descr=) + }}} call(ConstClass(ll_append_res0__stringbuilderPtr_rpy_stringPtr), p86, p80, descr=) guard_no_exception(descr=...) i89 = getfield_gc(p86, descr=) From noreply at buildbot.pypy.org Fri Sep 26 10:02:47 2014 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 26 Sep 2014 10:02:47 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: fix Message-ID: <20140926080247.3B9A11C3277@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: gc_no_cleanup_nursery Changeset: r73718:e8b470eb36c2 Date: 2014-09-26 10:02 +0200 http://bitbucket.org/pypy/pypy/changeset/e8b470eb36c2/ Log: fix diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py --- a/rpython/jit/backend/llsupport/test/test_rewrite.py +++ b/rpython/jit/backend/llsupport/test/test_rewrite.py @@ -729,7 +729,7 @@ def test_initialization_store_array(self): self.check_rewrite(""" [p1, i2] - p0 = new_array(5, descr=cdescr) + p0 = new_array_clear(5, descr=cdescr) setarrayitem_gc(p0, i2, p1, descr=cdescr) jump() """, """ @@ -738,6 +738,7 @@ %(cdescr.basesize + 5 * cdescr.itemsize)d) setfield_gc(p0, 8111, descr=tiddescr) setfield_gc(p0, 5, descr=clendescr) + zero_array(p0, 0, 5, descr=cdescr) setarrayitem_gc(p0, i2, p1, descr=cdescr) jump() """) From noreply at buildbot.pypy.org Fri Sep 26 10:10:47 2014 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 26 Sep 2014 10:10:47 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: fix test Message-ID: <20140926081047.6C8881C0250@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: gc_no_cleanup_nursery Changeset: r73719:2c7ffafe9dfd Date: 2014-09-26 10:10 +0200 http://bitbucket.org/pypy/pypy/changeset/2c7ffafe9dfd/ Log: fix test diff --git a/rpython/jit/backend/test/test_ll_random.py b/rpython/jit/backend/test/test_ll_random.py --- a/rpython/jit/backend/test/test_ll_random.py +++ b/rpython/jit/backend/test/test_ll_random.py @@ -251,6 +251,8 @@ class ZeroPtrFieldOperation(test_random.AbstractOperation): def field_descr(self, builder, r): + if getattr(builder.cpu, 'is_llgraph', False): + raise test_random.CannotProduceOperation v, S = builder.get_structptr_var(r, ) names = S._names if names[0] == 'parent': @@ -265,7 +267,7 @@ name = r.choice(choice) descr = builder.cpu.fielddescrof(S, name) return v, descr.offset - + def produce_into(self, builder, r): v, offset = self.field_descr(builder, r) builder.do(self.opnum, [v, ConstInt(offset)], None) @@ -728,7 +730,7 @@ OPERATIONS.append(GetArrayItemOperation(rop.GETARRAYITEM_GC)) OPERATIONS.append(GetArrayItemOperation(rop.GETARRAYITEM_GC)) OPERATIONS.append(SetArrayItemOperation(rop.SETARRAYITEM_GC)) - OPERATIONS.append(NewArrayOperation(rop.NEW_ARRAY)) + OPERATIONS.append(NewArrayOperation(rop.NEW_ARRAY_CLEAR)) OPERATIONS.append(ArrayLenOperation(rop.ARRAYLEN_GC)) OPERATIONS.append(NewStrOperation(rop.NEWSTR)) OPERATIONS.append(NewUnicodeOperation(rop.NEWUNICODE)) From noreply at buildbot.pypy.org Fri Sep 26 10:13:23 2014 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 26 Sep 2014 10:13:23 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: fix for test_lloperation Message-ID: <20140926081323.6289A1C3277@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: gc_no_cleanup_nursery Changeset: r73720:e6791101a58b Date: 2014-09-26 10:12 +0200 http://bitbucket.org/pypy/pypy/changeset/e6791101a58b/ Log: fix for test_lloperation diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py --- a/rpython/rtyper/llinterp.py +++ b/rpython/rtyper/llinterp.py @@ -890,9 +890,12 @@ def op_do_malloc_fixedsize(self): raise NotImplementedError("do_malloc_fixedsize") - + def op_do_malloc_fixedsize_clear(self): + raise NotImplementedError("do_malloc_fixedsize_clear") def op_do_malloc_varsize(self): raise NotImplementedError("do_malloc_varsize") + def op_do_malloc_varsize_clear(self): + raise NotImplementedError("do_malloc_varsize_clear") def op_get_write_barrier_failing_case(self): raise NotImplementedError("get_write_barrier_failing_case") From noreply at buildbot.pypy.org Fri Sep 26 10:17:07 2014 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 26 Sep 2014 10:17:07 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: Fix: there is an extra setfield_gc(.., 0) Message-ID: <20140926081707.A2CFD1C3277@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: gc_no_cleanup_nursery Changeset: r73721:486727982013 Date: 2014-09-26 10:16 +0200 http://bitbucket.org/pypy/pypy/changeset/486727982013/ Log: Fix: there is an extra setfield_gc(.., 0) diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py --- a/pypy/module/micronumpy/test/test_zjit.py +++ b/pypy/module/micronumpy/test/test_zjit.py @@ -617,7 +617,7 @@ 'raw_store': 1, 'same_as': 2, 'setarrayitem_gc': 8, - 'setfield_gc': 21, + 'setfield_gc': 22, }) def define_argsort(): From noreply at buildbot.pypy.org Fri Sep 26 10:34:12 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Fri, 26 Sep 2014 10:34:12 +0200 (CEST) Subject: [pypy-commit] stmgc c8-new-page-handling: introduce revision numbers Message-ID: <20140926083412.875C41C3339@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: c8-new-page-handling Changeset: r1426:fadb937a707d Date: 2014-09-26 09:57 +0200 http://bitbucket.org/pypy/stmgc/changeset/fadb937a707d/ Log: introduce revision numbers diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -61,6 +61,7 @@ struct stm_commit_log_entry_s *from, struct stm_commit_log_entry_s *to) { + assert(from->rev_num >= to->rev_num); /* walk BACKWARDS the commit log and update the page 'pagenum', initially at revision 'from', until we reach the revision 'to'. */ @@ -81,6 +82,7 @@ struct stm_commit_log_entry_s *from, struct stm_commit_log_entry_s *to) { + assert(from->rev_num <= to->rev_num); if (from == to) return; @@ -167,35 +169,18 @@ /* we need to go from 'src_version' to 'target_version'. This might need a walk into the past or the future. */ - struct stm_commit_log_entry_s *src_version, *target_version, *c1, *c2; + struct stm_commit_log_entry_s *src_version, *target_version; src_version = get_priv_segment(shared_page_holder)->last_commit_log_entry; target_version = STM_PSEGMENT->last_commit_log_entry; release_modified_objs_lock(shared_page_holder); release_all_privatization_locks(); - c1 = src_version; - c2 = target_version; - while (1) { - if (c1 == target_version) { - go_to_the_future(pagenum, src_version, target_version); - break; - } - if (c2 == src_version) { - go_to_the_past(pagenum, src_version, target_version); - break; - } - c1 = c1->next; - if (c1 == (void *)-1 || c1 == NULL) { - go_to_the_past(pagenum, src_version, target_version); - break; - } - c2 = c2->next; - if (c2 == (void *)-1 || c2 == NULL) { - go_to_the_future(pagenum, src_version, target_version); - break; - } - } + /* adapt revision of page to our revision: */ + if (src_version->rev_num < target_version->rev_num) + go_to_the_future(pagenum, src_version, target_version); + else if (src_version->rev_num > target_version->rev_num) + go_to_the_past(pagenum, src_version, target_version); } static void _signal_handler(int sig, siginfo_t *siginfo, void *context) @@ -245,7 +230,7 @@ fprintf(stderr, " INEVITABLE\n"); return; } - fprintf(stderr, " entry at %p: seg %d\n", cl, cl->segment_num); + fprintf(stderr, " entry at %p: seg %d, rev %lu\n", cl, cl->segment_num, cl->rev_num); struct stm_undo_s *undo = cl->written; struct stm_undo_s *end = undo + cl->written_count; for (; undo < end; undo++) { @@ -276,6 +261,9 @@ ordering things, but later. */ acquire_modified_objs_lock(STM_SEGMENT->segment_num); + /* XXXXXXXXXX: we shouldn't be able to update pages while someone else copies + from our pages (signal handler / import objs) */ + bool needs_abort = false; uint64_t segment_copied_from = 0; while ((next_cl = cl->next) != NULL) { @@ -290,10 +278,11 @@ or: XXX do we always need to wait? we should just break out of this loop and let the caller handle it if it wants to */ + _stm_collectable_safe_point(); - acquire_all_privatization_locks(); continue; } + assert(next_cl->rev_num > cl->rev_num); cl = next_cl; /*int srcsegnum = cl->segment_num; @@ -362,6 +351,7 @@ result->next = NULL; result->segment_num = STM_SEGMENT->segment_num; + result->rev_num = -1; /* invalid */ result->written_count = count; memcpy(result->written, list->items, count * sizeof(struct stm_undo_s)); return result; @@ -376,9 +366,13 @@ /* try to attach to commit log: */ old = STM_PSEGMENT->last_commit_log_entry; - if (old->next == NULL && - __sync_bool_compare_and_swap(&old->next, NULL, new)) - break; /* success! */ + if (old->next == NULL) { + if ((uintptr_t)new != -1) /* INEVITABLE */ + new->rev_num = old->rev_num + 1; + + if (__sync_bool_compare_and_swap(&old->next, NULL, new)) + break; /* success! */ + } } } @@ -394,6 +388,7 @@ new = _create_commit_log_entry(); if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) { old = STM_PSEGMENT->last_commit_log_entry; + new->rev_num = old->rev_num + 1; OPT_ASSERT(old->next == (void *)-1); bool yes = __sync_bool_compare_and_swap(&old->next, (void*)-1, new); diff --git a/c8/stm/core.h b/c8/stm/core.h --- a/c8/stm/core.h +++ b/c8/stm/core.h @@ -120,10 +120,11 @@ struct stm_commit_log_entry_s { struct stm_commit_log_entry_s *volatile next; int segment_num; + uint64_t rev_num; size_t written_count; struct stm_undo_s written[]; }; -static struct stm_commit_log_entry_s commit_log_root = {NULL, -1, 0}; +static struct stm_commit_log_entry_s commit_log_root = {NULL, -1, 0, 0}; static char *stm_object_pages; From noreply at buildbot.pypy.org Fri Sep 26 10:34:13 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Fri, 26 Sep 2014 10:34:13 +0200 (CEST) Subject: [pypy-commit] stmgc c8-new-page-handling: remove go_to_the_future() Message-ID: <20140926083413.9E9191C3339@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: c8-new-page-handling Changeset: r1427:f8b04d5bb3a2 Date: 2014-09-26 10:34 +0200 http://bitbucket.org/pypy/stmgc/changeset/f8b04d5bb3a2/ Log: remove go_to_the_future() diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -57,6 +57,7 @@ import_objects(-1, pagenum, undo, end); } + static void go_to_the_past(uintptr_t pagenum, struct stm_commit_log_entry_s *from, struct stm_commit_log_entry_s *to) @@ -78,49 +79,6 @@ } -static void go_to_the_future(uintptr_t pagenum, - struct stm_commit_log_entry_s *from, - struct stm_commit_log_entry_s *to) -{ - assert(from->rev_num <= to->rev_num); - if (from == to) - return; - - /* XXX: specialize. We now go to the HEAD revision, and back again - to where we want. Otherwise, we have to look at backup copies in - the log entries, modified objs, page content and their revisions... - - Same logic as _stm_validate() */ - - /* XXXXXXXXXXXXXXXX CORRECT LOCKING NEEDED XXXXXXXXXXXXXXXXXXXXXX */ - struct stm_commit_log_entry_s *cl = from; - struct stm_commit_log_entry_s *next_cl; - - uint64_t segment_copied_from = 0; - while ((next_cl = cl->next) != NULL) { - if (next_cl == (void *)-1) - break; - - cl = next_cl; - - struct stm_undo_s *undo = cl->written; - struct stm_undo_s *end = cl->written + cl->written_count; - - segment_copied_from |= (1UL << cl->segment_num); - import_objects(cl->segment_num, -1, undo, end); - } - - /* XXXXXXXXXXXXXXXX CORRECT LOCKING NEEDED XXXXXXXXXXXXXXXXXXXXXX */ - int segnum; - for (segnum = 0; segment_copied_from != 0; segnum++) { - if (segment_copied_from & (1UL << segnum)) { - segment_copied_from &= ~(1UL << segnum); - copy_bk_objs_in_page_from(segnum, -1); - } - } - - go_to_the_past(pagenum, cl, to); -} static void handle_segfault_in_page(uintptr_t pagenum) { @@ -137,49 +95,63 @@ assert(get_page_status_in(my_segnum, pagenum) == PAGE_NO_ACCESS); - /* find who has the PAGE_SHARED */ + /* find who has the most recent revision of our page */ int shared_page_holder = -1; int shared_ref_count = 0; + int copy_from_segnum = -1; + uint64_t most_recent_rev = 0; for (i = 0; i < NB_SEGMENTS; i++) { if (i == my_segnum) continue; + if (get_page_status_in(i, pagenum) == PAGE_SHARED) { + /* mostly for debugging now: */ shared_page_holder = i; shared_ref_count++; } + + struct stm_commit_log_entry_s *log_entry; + log_entry = get_priv_segment(i)->last_commit_log_entry; + if (get_page_status_in(i, pagenum) != PAGE_NO_ACCESS + && (copy_from_segnum == -1 || log_entry->rev_num > most_recent_rev)) { + copy_from_segnum = i; + most_recent_rev = log_entry->rev_num; + } } - assert(shared_page_holder != -1); + OPT_ASSERT(shared_page_holder != -1); + OPT_ASSERT(copy_from_segnum != -1 && copy_from_segnum != my_segnum); /* XXX: for now, we don't try to get the single shared page. We simply regard it as private for its holder. */ /* this assert should be true for now... */ assert(shared_ref_count == 1); - /* acquire this lock, so that we know we should get a view - of the page we're about to copy that is consistent with the - backup copies in the other thread's 'modified_old_objects'. */ - acquire_modified_objs_lock(shared_page_holder); - /* make our page private */ page_privatize_in(STM_SEGMENT->segment_num, pagenum); assert(get_page_status_in(my_segnum, pagenum) == PAGE_PRIVATE); + acquire_modified_objs_lock(copy_from_segnum); + pagecopy((char*)(get_virt_page_of(my_segnum, pagenum) * 4096UL), + (char*)(get_virt_page_of(copy_from_segnum, pagenum) * 4096UL)); + /* if there were modifications in the page, revert them. */ - copy_bk_objs_in_page_from(shared_page_holder, pagenum); + copy_bk_objs_in_page_from(copy_from_segnum, pagenum); /* we need to go from 'src_version' to 'target_version'. This - might need a walk into the past or the future. */ + might need a walk into the past. */ struct stm_commit_log_entry_s *src_version, *target_version; - src_version = get_priv_segment(shared_page_holder)->last_commit_log_entry; + src_version = get_priv_segment(copy_from_segnum)->last_commit_log_entry; + OPT_ASSERT(src_version->rev_num == most_recent_rev); target_version = STM_PSEGMENT->last_commit_log_entry; - release_modified_objs_lock(shared_page_holder); + release_modified_objs_lock(copy_from_segnum); release_all_privatization_locks(); - /* adapt revision of page to our revision: */ - if (src_version->rev_num < target_version->rev_num) - go_to_the_future(pagenum, src_version, target_version); - else if (src_version->rev_num > target_version->rev_num) + /* adapt revision of page to our revision: + if our rev is higher than the page we copy from, everything + is fine as we never read/modified the page anyway + */ + if (src_version->rev_num > target_version->rev_num) go_to_the_past(pagenum, src_version, target_version); } diff --git a/c8/test/test_basic.py b/c8/test/test_basic.py --- a/c8/test/test_basic.py +++ b/c8/test/test_basic.py @@ -339,6 +339,30 @@ py.test.raises(Conflict, self.commit_transaction) # + def test_read_write_17(self): + lp1 = stm_allocate_old(16) # allocated in seg0 + stm_get_real_address(lp1)[HDR] = 'a' #setchar + # S|P|P|P|P + # + # NO_ACCESS in all segments except seg0 (shared page holder) + # + # all seg at R0 + # + # + self.switch(1) # with private page + self.start_transaction() + stm_set_char(lp1, 'C') + self.commit_transaction() # R1 + assert last_commit_log_entries() == [lp1] # commit 'C' + self.start_transaction() + stm_set_char(lp1, 'c') # bk_copy + # + self.switch(3, validate=False) + self.start_transaction() # validate + assert stm_get_char(lp1) == 'C' # R1 + self.commit_transaction() + # + def test_commit_fresh_objects(self): From noreply at buildbot.pypy.org Fri Sep 26 11:22:43 2014 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 26 Sep 2014 11:22:43 +0200 (CEST) Subject: [pypy-commit] pypy default: Generalize the test to account for PyPy's particular behavior, which is subtly different from CPython's. Message-ID: <20140926092243.E9C981C3340@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73722:4e6dddc6d1ed Date: 2014-09-26 11:22 +0200 http://bitbucket.org/pypy/pypy/changeset/4e6dddc6d1ed/ Log: Generalize the test to account for PyPy's particular behavior, which is subtly different from CPython's. diff --git a/lib-python/2.7/test/test_select.py b/lib-python/2.7/test/test_select.py --- a/lib-python/2.7/test/test_select.py +++ b/lib-python/2.7/test/test_select.py @@ -57,7 +57,12 @@ del a[-1] return sys.__stdout__.fileno() a[:] = [F()] * 10 - self.assertEqual(select.select([], a, []), ([], a[:5], [])) + result = select.select([], a, []) + # CPython: 'a' ends up with 5 items, because each fileno() + # removes an item and at the middle the iteration stops. + # PyPy: 'a' ends up empty, because the iteration is done on + # a copy of the original list: fileno() is called 10 times. + self.assert_(len(result[1]) <= 5) def test_main(): test_support.run_unittest(SelectTestCase) From noreply at buildbot.pypy.org Fri Sep 26 13:55:57 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Fri, 26 Sep 2014 13:55:57 +0200 (CEST) Subject: [pypy-commit] stmgc c8-new-page-handling: do multi-page slicing Message-ID: <20140926115557.580811C33A1@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: c8-new-page-handling Changeset: r1428:3587e00868fd Date: 2014-09-26 13:56 +0200 http://bitbucket.org/pypy/stmgc/changeset/3587e00868fd/ Log: do multi-page slicing diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -32,11 +32,13 @@ continue; } + dprintf(("import slice obj=%p off=%lu pg=%lu\n", + obj, SLICE_OFFSET(undo->slice), current_page_num)); char *src, *dst; if (src_segment_base != NULL) src = REAL_ADDRESS(src_segment_base, oslice); else - src = undo->backup; + src = undo->backup + SLICE_OFFSET(undo->slice); dst = REAL_ADDRESS(STM_SEGMENT->segment_base, oslice); memcpy(dst, src, SLICE_SIZE(undo->slice)); } @@ -469,19 +471,33 @@ if 'obj' is merely an overflow object. FIX ME, likely by copying the overflow number logic from c7. */ - assert(first_page == end_page); /* XXX! */ - /* XXX do the case where first_page != end_page in pieces. Maybe also - use mprotect() again to mark pages of the object as read-only, and - only stick it into modified_old_objects page-by-page? Maybe it's - possible to do card-marking that way, too. */ + acquire_modified_objs_lock(STM_SEGMENT->segment_num); + uintptr_t slice_sz; + uintptr_t in_page_offset = (uintptr_t)obj % 4096UL; + uintptr_t remaining_obj_sz = obj_size; + for (page = first_page; page <= end_page; page++) { + /* XXX Maybe also use mprotect() again to mark pages of the object as read-only, and + only stick it into modified_old_objects page-by-page? Maybe it's + possible to do card-marking that way, too. */ + OPT_ASSERT(remaining_obj_sz); - uintptr_t slice = obj_size; - assert(SLICE_OFFSET(slice) == 0 && SLICE_SIZE(slice) == obj_size); + slice_sz = remaining_obj_sz; + if (in_page_offset + slice_sz > 4096UL) { + /* not over page boundaries */ + slice_sz = 4096UL - in_page_offset; + } - acquire_modified_objs_lock(STM_SEGMENT->segment_num); - STM_PSEGMENT->modified_old_objects = list_append3( - STM_PSEGMENT->modified_old_objects, - (uintptr_t)obj, (uintptr_t)bk_obj, slice); + STM_PSEGMENT->modified_old_objects = list_append3( + STM_PSEGMENT->modified_old_objects, + (uintptr_t)obj, /* obj */ + (uintptr_t)bk_obj, /* bk_addr */ + NEW_SLICE(obj_size - remaining_obj_sz, slice_sz)); + + remaining_obj_sz -= slice_sz; + in_page_offset = (in_page_offset + slice_sz) % 4096UL; /* mostly 0 */ + } + OPT_ASSERT(remaining_obj_sz == 0); + release_modified_objs_lock(STM_SEGMENT->segment_num); /* done fiddling with protection and privatization */ diff --git a/c8/stm/core.h b/c8/stm/core.h --- a/c8/stm/core.h +++ b/c8/stm/core.h @@ -108,6 +108,7 @@ }; #define SLICE_OFFSET(slice) ((slice) >> 16) #define SLICE_SIZE(slice) ((int)((slice) & 0xFFFF)) +#define NEW_SLICE(offset, size) (((uint64_t)(offset)) << 16 | (size)) /* The model is: we have a global chained list, from 'commit_log_root', of 'struct stm_commit_log_entry_s' entries. Every one is fully diff --git a/c8/test/test_basic.py b/c8/test/test_basic.py --- a/c8/test/test_basic.py +++ b/c8/test/test_basic.py @@ -733,6 +733,7 @@ assert self.get_stm_thread_local().last_abort__bytes_in_nursery == 0 def test_abort_in_segfault_handler(self): + py.test.skip("not doing that anymore") lp1 = stm_allocate_old(16) lp2 = stm_allocate_old(16) From noreply at buildbot.pypy.org Fri Sep 26 15:07:57 2014 From: noreply at buildbot.pypy.org (groggi) Date: Fri, 26 Sep 2014 15:07:57 +0200 (CEST) Subject: [pypy-commit] pypy gc-incminimark-pinning: added XXX: incminimark may zero out much more of the nursery than intended Message-ID: <20140926130757.25EDE1C3277@cobra.cs.uni-duesseldorf.de> Author: Gregor Wegberg Branch: gc-incminimark-pinning Changeset: r73723:88df0344d983 Date: 2014-09-26 15:07 +0200 http://bitbucket.org/pypy/pypy/changeset/88df0344d983/ Log: added XXX: incminimark may zero out much more of the nursery than intended diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -1674,6 +1674,10 @@ if prev - self.nursery >= self.nursery_cleanup: nursery_barriers.append(prev) else: + # XXX: length should not be 'self.nursery_cleanup', but rather + # 'self.nursery_cleanup - (prev - self.nursery)'. Otherwise + # in case we end up in this branch we zero out much more space + # overall than in original incminimark. llarena.arena_reset(prev, self.nursery_cleanup, 2) nursery_barriers.append(prev + self.nursery_cleanup) # From noreply at buildbot.pypy.org Fri Sep 26 16:01:44 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Fri, 26 Sep 2014 16:01:44 +0200 (CEST) Subject: [pypy-commit] stmgc c8-new-page-handling: a few fixes for freeing backup copies, not releasing a lock, and inevitable transaction validation Message-ID: <20140926140144.71F4C1C3340@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: c8-new-page-handling Changeset: r1429:504914e27b52 Date: 2014-09-26 14:30 +0200 http://bitbucket.org/pypy/stmgc/changeset/504914e27b52/ Log: a few fixes for freeing backup copies, not releasing a lock, and inevitable transaction validation diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -228,12 +228,18 @@ the committed objs. */ struct stm_commit_log_entry_s *cl = STM_PSEGMENT->last_commit_log_entry; struct stm_commit_log_entry_s *next_cl; + int my_segnum = STM_SEGMENT->segment_num; /* Don't check this 'cl'. This entry is already checked */ + if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) { + assert(cl->next == -1); + return; + } + /* We need this lock to prevent a segfault handler in a different thread from seeing inconsistent data. It could also be done by carefully ordering things, but later. */ - acquire_modified_objs_lock(STM_SEGMENT->segment_num); + acquire_modified_objs_lock(my_segnum); /* XXXXXXXXXX: we shouldn't be able to update pages while someone else copies from our pages (signal handler / import objs) */ @@ -243,6 +249,7 @@ while ((next_cl = cl->next) != NULL) { if (next_cl == (void *)-1) { /* there is an inevitable transaction running */ + release_modified_objs_lock(my_segnum); #if STM_TESTS if (free_if_abort != (void *)-1) free(free_if_abort); @@ -254,6 +261,7 @@ if it wants to */ _stm_collectable_safe_point(); + acquire_modified_objs_lock(my_segnum); continue; } assert(next_cl->rev_num > cl->rev_num); @@ -272,9 +280,9 @@ then we will proceed below to update our segment from the old (but unmodified) version to the newer version. */ - release_modified_objs_lock(STM_SEGMENT->segment_num); - reset_modified_from_backup_copies(STM_SEGMENT->segment_num); - acquire_modified_objs_lock(STM_SEGMENT->segment_num); + release_modified_objs_lock(my_segnum); + reset_modified_from_backup_copies(my_segnum); + acquire_modified_objs_lock(my_segnum); needs_abort = true; break; } @@ -292,7 +300,7 @@ STM_PSEGMENT->last_commit_log_entry = cl; } - release_modified_objs_lock(STM_SEGMENT->segment_num); + release_modified_objs_lock(my_segnum); /* XXXXXXXXXXXXXXXX CORRECT LOCKING NEEDED XXXXXXXXXXXXXXXXXXXXXX */ int segnum; @@ -682,7 +690,11 @@ memcpy(dst + SLICE_OFFSET(undo->slice), undo->backup, SLICE_SIZE(undo->slice)); - free(undo->backup); + + if (SLICE_OFFSET(undo->slice) == 0) { + /* only free bk copy once: */ + free(undo->backup); + } } /* check that all objects have the GCFLAG_WRITE_BARRIER afterwards */ diff --git a/c8/test/test_random.py b/c8/test/test_random.py --- a/c8/test/test_random.py +++ b/c8/test/test_random.py @@ -10,7 +10,7 @@ self.thread_num = 0 def do(self, cmd): - color = "\033[%dm" % (31 + self.thread_num % 6) + color = ">> \033[%dm" % (31 + (self.thread_num + 5) % 6) print >> sys.stderr, color + cmd + "\033[0m" exec cmd in globals(), self.content From noreply at buildbot.pypy.org Fri Sep 26 16:01:45 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Fri, 26 Sep 2014 16:01:45 +0200 (CEST) Subject: [pypy-commit] stmgc c8-new-page-handling: actually free backup *after* copying from it Message-ID: <20140926140145.885171C3340@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: c8-new-page-handling Changeset: r1430:65c9f7a4a27d Date: 2014-09-26 15:30 +0200 http://bitbucket.org/pypy/stmgc/changeset/65c9f7a4a27d/ Log: actually free backup *after* copying from it diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -32,8 +32,9 @@ continue; } - dprintf(("import slice obj=%p off=%lu pg=%lu\n", - obj, SLICE_OFFSET(undo->slice), current_page_num)); + dprintf(("import slice seg=%d obj=%p off=%lu sz=%d pg=%lu\n", + from_segnum, obj, SLICE_OFFSET(undo->slice), + SLICE_SIZE(undo->slice), current_page_num)); char *src, *dst; if (src_segment_base != NULL) src = REAL_ADDRESS(src_segment_base, oslice); @@ -232,7 +233,7 @@ /* Don't check this 'cl'. This entry is already checked */ if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) { - assert(cl->next == -1); + assert(cl->next == (void*)-1); return; } @@ -691,8 +692,9 @@ undo->backup, SLICE_SIZE(undo->slice)); - if (SLICE_OFFSET(undo->slice) == 0) { - /* only free bk copy once: */ + size_t obj_size = stmcb_size_rounded_up(undo->backup); + if (obj_size - SLICE_OFFSET(undo->slice) <= 4096UL) { + /* only free bk copy once (last slice): */ free(undo->backup); } } From noreply at buildbot.pypy.org Fri Sep 26 16:01:46 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Fri, 26 Sep 2014 16:01:46 +0200 (CEST) Subject: [pypy-commit] stmgc c8-new-page-handling: bah, another missing SLICE_OFFSET Message-ID: <20140926140146.8B60E1C3340@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: c8-new-page-handling Changeset: r1431:96346c488f28 Date: 2014-09-26 15:36 +0200 http://bitbucket.org/pypy/stmgc/changeset/96346c488f28/ Log: bah, another missing SLICE_OFFSET diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -689,13 +689,15 @@ char *dst = REAL_ADDRESS(pseg->pub.segment_base, obj); memcpy(dst + SLICE_OFFSET(undo->slice), - undo->backup, + undo->backup + SLICE_OFFSET(undo->slice), SLICE_SIZE(undo->slice)); size_t obj_size = stmcb_size_rounded_up(undo->backup); if (obj_size - SLICE_OFFSET(undo->slice) <= 4096UL) { /* only free bk copy once (last slice): */ free(undo->backup); + dprintf(("reset_modified_from_backup_copies(%d): obj=%p obj_sz=%lu\n", + segment_num, obj, obj_size)); } } From noreply at buildbot.pypy.org Fri Sep 26 16:01:47 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Fri, 26 Sep 2014 16:01:47 +0200 (CEST) Subject: [pypy-commit] stmgc c8-new-page-handling: don't apply undo copies in stm_validate() if we have our own modified version (probably not the whole story yet..) Message-ID: <20140926140147.859341C3340@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: c8-new-page-handling Changeset: r1432:e565123efe0d Date: 2014-09-26 16:01 +0200 http://bitbucket.org/pypy/stmgc/changeset/e565123efe0d/ Log: don't apply undo copies in stm_validate() if we have our own modified version (probably not the whole story yet..) diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -9,7 +9,8 @@ inner loop. */ static void import_objects( - int from_segnum, /* or -1: from undo->backup */ + int from_segnum, /* or -1: from undo->backup, + or -2: from undo->backup if not stm_was_read(obj) */ uintptr_t pagenum, /* or -1: "all accessible" */ struct stm_undo_s *undo, struct stm_undo_s *end) @@ -32,6 +33,9 @@ continue; } + if (from_segnum == -2 && _stm_was_read(obj)) + continue; /* only copy unmodified */ + dprintf(("import slice seg=%d obj=%p off=%lu sz=%d pg=%lu\n", from_segnum, obj, SLICE_OFFSET(undo->slice), SLICE_SIZE(undo->slice), current_page_num)); @@ -48,7 +52,8 @@ /* ############# signal handler ############# */ -static void copy_bk_objs_in_page_from(int from_segnum, uintptr_t pagenum) +static void copy_bk_objs_in_page_from(int from_segnum, uintptr_t pagenum, + bool only_if_not_modified) { /* looks at all bk copies of objects overlapping page 'pagenum' and copies the part in 'pagenum' back to the current segment */ @@ -58,7 +63,8 @@ struct stm_undo_s *undo = (struct stm_undo_s *)list->items; struct stm_undo_s *end = (struct stm_undo_s *)(list->items + list->count); - import_objects(-1, pagenum, undo, end); + import_objects(only_if_not_modified ? -2 : -1, + pagenum, undo, end); } static void go_to_the_past(uintptr_t pagenum, @@ -138,7 +144,7 @@ (char*)(get_virt_page_of(copy_from_segnum, pagenum) * 4096UL)); /* if there were modifications in the page, revert them. */ - copy_bk_objs_in_page_from(copy_from_segnum, pagenum); + copy_bk_objs_in_page_from(copy_from_segnum, pagenum, false); /* we need to go from 'src_version' to 'target_version'. This might need a walk into the past. */ @@ -290,14 +296,15 @@ } } - struct stm_undo_s *undo = cl->written; - struct stm_undo_s *end = cl->written + cl->written_count; + if (cl->written_count) { + struct stm_undo_s *undo = cl->written; + struct stm_undo_s *end = cl->written + cl->written_count; - segment_copied_from |= (1UL << cl->segment_num); - import_objects(cl->segment_num, -1, undo, end); + segment_copied_from |= (1UL << cl->segment_num); + import_objects(cl->segment_num, -1, undo, end); + } /* last fully validated entry */ - STM_PSEGMENT->last_commit_log_entry = cl; } @@ -308,7 +315,10 @@ for (segnum = 0; segment_copied_from != 0; segnum++) { if (segment_copied_from & (1UL << segnum)) { segment_copied_from &= ~(1UL << segnum); - copy_bk_objs_in_page_from(segnum, -1); + /* here we can actually have our own modified version, so + make sure to only copy things that are not modified in our + segment... */ + copy_bk_objs_in_page_from(segnum, -1, true); } } diff --git a/c8/test/test_basic.py b/c8/test/test_basic.py --- a/c8/test/test_basic.py +++ b/c8/test/test_basic.py @@ -763,3 +763,23 @@ # seg1 segfaults, validates, and aborts: py.test.raises(Conflict, stm_get_char, lp2) + + def test_bug(self): + lp_char_5 = stm_allocate_old(384) + + self.start_transaction() # R1 + stm_set_char(lp_char_5, 'i', 384 - 1, False) + stm_set_char(lp_char_5, 'i', HDR, False) + # + # + self.switch(3) + self.start_transaction() # R1 + self.commit_transaction() # R2 + + self.start_transaction() # R2 + stm_set_char(lp_char_5, 'o', 384 - 1, False) # bk_copy + stm_set_char(lp_char_5, 'o', HDR, False) + # + # + self.switch(0) # validate -> R2 + assert stm_get_char(lp_char_5, 384 - 1) == 'i' diff --git a/c8/test/test_random.py b/c8/test/test_random.py --- a/c8/test/test_random.py +++ b/c8/test/test_random.py @@ -497,7 +497,8 @@ def op_switch_thread(ex, global_state, thread_state, new_thread_state=None): if new_thread_state is None: - new_thread_state = global_state.rnd.choice(global_state.thread_states) + new_thread_state = global_state.rnd.choice( + global_state.thread_states + [thread_state] * 3) # more likely not switch if new_thread_state != thread_state: if thread_state.transaction_state: From noreply at buildbot.pypy.org Sat Sep 27 19:16:40 2014 From: noreply at buildbot.pypy.org (pjenvey) Date: Sat, 27 Sep 2014 19:16:40 +0200 (CEST) Subject: [pypy-commit] pypy py3k: fix missing endDFAs for u prefix'd tripled quoted strings Message-ID: <20140927171640.02EC71C073F@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r73724:5f38597ef8a9 Date: 2014-09-26 20:04 -0700 http://bitbucket.org/pypy/pypy/changeset/5f38597ef8a9/ Log: fix missing endDFAs for u prefix'd tripled quoted strings diff --git a/pypy/interpreter/pyparser/pytokenize.py b/pypy/interpreter/pyparser/pytokenize.py --- a/pypy/interpreter/pyparser/pytokenize.py +++ b/pypy/interpreter/pyparser/pytokenize.py @@ -35,8 +35,9 @@ prefix = uniPrefix + rawPrefix endDFAs[prefix + "'''"] = single3DFA endDFAs[prefix + '"""'] = double3DFA -endDFAs["u'''"] = single3DFA -endDFAs['U"""'] = double3DFA +for uniPrefix in ("u", "U"): + endDFAs[uniPrefix + "'''"] = single3DFA + endDFAs[uniPrefix + '"""'] = double3DFA whiteSpaceStatesAccepts = [True] whiteSpaceStates = [{'\t': 0, ' ': 0, '\x0c': 0}] diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py --- a/pypy/interpreter/pyparser/test/test_pyparse.py +++ b/pypy/interpreter/pyparser/test/test_pyparse.py @@ -146,6 +146,12 @@ self.parse('a, *rest, b = 1, 2, 3, 4, 5') self.parse('(a, *rest, b) = 1, 2, 3, 4, 5') + def test_u_triple_quote(self): + self.parse('u""""""') + self.parse('U""""""') + self.parse("u''''''") + self.parse("U''''''") + class TestPythonParserWithSpace: From noreply at buildbot.pypy.org Sat Sep 27 19:23:27 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 27 Sep 2014 19:23:27 +0200 (CEST) Subject: [pypy-commit] stmgc default: Finally getting somewhere with dictionaries in STM. Here is a very Message-ID: <20140927172327.C3AFF1C073F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1433:49cdf03dfaf9 Date: 2014-09-27 19:23 +0200 http://bitbucket.org/pypy/stmgc/changeset/49cdf03dfaf9/ Log: Finally getting somewhere with dictionaries in STM. Here is a very high-level design idea. diff --git a/hashtable/design2.txt b/hashtable/design2.txt new file mode 100644 --- /dev/null +++ b/hashtable/design2.txt @@ -0,0 +1,124 @@ +Goal +====== + +The goal is to have dictionaries where a read-write or write-write +conflict does not cause aborts if they occur with keys that have +different 64-bit hashes. + +(We might prefer the condition to be "on different keys even in case of +hash collision", but that's hard to achieve in general: for Python +dicts, "to be equal" involves calling the __eq__() method of objects.) + +We distinguish between reading a value associated to a key, and only +checking that the key is in the dictionary. It makes a difference if +a concurrent transaction writes to the value. + +Some operations on a dict (particularly len() and __nonzero__()) involve +the global state of the dict, and thus cause conflict with any write +that adds or removes keys. They should not cause conflicts with reads, +or with writes that only change existing values. + +In theory, we might refine this to: a len() only conflicts with a +different transaction whose net effect is to change the length (adding 3 +keys and removing 3 other keys is fine); and a __nonzero__() only +conflicts with a transaction whose net effect is to change the dict from +empty to non-empty or vice-versa. The latter is probably more important +than the former, so we'll ignore the former. + +Iterating over the keys of a dict doesn't have to conflict with other +transactions that only change existing values. Iterating over the +values or the items conflict with other transactions doing any write at +all. + + +Idea +====== + +A dict is implemented used two distinct parts: the committed part, +and the uncommitted one. Each part is optimized differently. + + +Committed part +-------------- + +The committed part uses separate chaining with linked lists. It is an +array of pointers of length some power of two. From the hash, we access +item (hash & (power_of_two - 1)). We get a pointer to some Entry +object, with fields "hash", "key", "value", and "next". The "hash" +field stored in the Entry objects is the full 64-bit hash. The "next" +field might point to more Entry objects. + +This whole structure is only modified during commit, by special code not +subject to the normal STM rules. There is only one writer, the +transaction currently trying to commit; but we need to be careful so that +concurrent reads work as expected. + +For the sequel, the committed part works theoretically like an array of +length 2**64, indexed by the hash, where each item contains zero of more +Entry objects with that hash value. + + +Uncommitted part +---------------- + +For the uncommitted part we can use a hash table similar to the one used +for RPython dicts, with open addressing. We import data from the +committed part to this uncommitted part when needed (at the granularity +of a 64-bit hash value). More precisely, the uncommitted part can be in +one of these states: + +* It can be a freshly created dictionary, with no committed part yet. + That's the easy case: the uncommitted hash table is all we need. + +* Or, we have a committed part, and we have imported from it + zero or more 64-bit hash values. We need to remember which ones. + That includes the imports that yielded zero key/value pairs. For each + imported hash value, we make (zero or more) entries in the uncommitted + part where we copy the key, but where the value is initially missing. + The value is copied lazily, with another lookup that will mark the + Entry object as "read" in the usual STM sense. + +* We may have additionally imported the "emptiness" or "non-emptiness" + of the committed part. + +* Last option: the current transaction is depending on the exact set + of committed keys. We no longer need to remember which ones + individually. This state is equivalent to having imported *all* + possible 64-bit hash values. + + +Commit time +----------- + +At commit time, we need to do these extra steps. The points followed by +(*) need to be done carefully because concurrent threads might be +reading the same data. + +* First, we do the usual STM validation. It will detect read-write + and write-write conflicts on existing values thanks to the read and + write markers xxx + +* We validate the keys: for every imported hash value, we check that + importing it now would give us the same answer as it did previously + (i.e. the committed table has got the same set of keys with this + particular hash as it did previously). + +* For key/value pairs that have been newly added by the current + transaction, the validation above is enough too: to add a key/value + pair, we must have imported all Entries with the same hash anyway. + So at this point we only need to create and attach(*) new Entry + objects for new key/value pairs. + + +xxxxxxxxxxx + + + +* First, we create new Entry objects for all key/value pairs that + are created by the current transaction. + +* First, we create or update the Entry objects for all key/value + pairs that have been modified by the current transaction. We + store the new ones by carefully changing the array of pointers. + +* From noreply at buildbot.pypy.org Sat Sep 27 23:14:34 2014 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 27 Sep 2014 23:14:34 +0200 (CEST) Subject: [pypy-commit] pypy win32-fixes5: os.tmpfile fails for non-admin on win32 Message-ID: <20140927211434.D3E9D1C073F@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: win32-fixes5 Changeset: r73725:c70817364854 Date: 2014-09-27 21:20 +0300 http://bitbucket.org/pypy/pypy/changeset/c70817364854/ Log: os.tmpfile fails for non-admin on win32 diff --git a/rpython/translator/tool/stdoutcapture.py b/rpython/translator/tool/stdoutcapture.py --- a/rpython/translator/tool/stdoutcapture.py +++ b/rpython/translator/tool/stdoutcapture.py @@ -9,7 +9,8 @@ def __init__(self, mixed_out_err = False): "Start capture of the Unix-level stdout and stderr." - if (not hasattr(os, 'tmpfile') or + if (sys.platform == 'win32' or # os.tmpfile fails, cpython issue #2232 + not hasattr(os, 'tmpfile') or not hasattr(os, 'dup') or not hasattr(os, 'dup2') or not hasattr(os, 'fdopen')): From noreply at buildbot.pypy.org Sat Sep 27 23:14:36 2014 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 27 Sep 2014 23:14:36 +0200 (CEST) Subject: [pypy-commit] pypy win32-fixes5: disable profopt on win32 Message-ID: <20140927211436.297281C073F@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: win32-fixes5 Changeset: r73726:7b21b053dd02 Date: 2014-09-27 22:22 +0300 http://bitbucket.org/pypy/pypy/changeset/7b21b053dd02/ Log: disable profopt on win32 diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py --- a/rpython/translator/c/genc.py +++ b/rpython/translator/c/genc.py @@ -302,8 +302,11 @@ def has_profopt(self): profbased = self.getprofbased() - return (profbased and isinstance(profbased, tuple) + retval = (profbased and isinstance(profbased, tuple) and profbased[0] is ProfOpt) + if retval and self.translator.platform.name == 'msvc': + raise ValueError('Cannot do profile based optimization on MSVC,' + 'it is not supported in free compiler version') def getentrypointptr(self): # XXX check that the entrypoint has the correct diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py --- a/rpython/translator/c/test/test_standalone.py +++ b/rpython/translator/c/test/test_standalone.py @@ -187,6 +187,8 @@ assert map(float, data.split()) == [0.0, 0.0] def test_profopt(self): + if sys.platform == 'win32': + py.test.skip("no profopt on win32") def add(a,b): return a + b - b + b - b + b - b + b - b + b - b + b - b + b def entry_point(argv): From noreply at buildbot.pypy.org Sat Sep 27 23:14:37 2014 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 27 Sep 2014 23:14:37 +0200 (CEST) Subject: [pypy-commit] pypy win32-fixes5: on windows NamedTemporaryFile cannot be reopened by gdb.execute while open in caller Message-ID: <20140927211437.58D721C073F@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: win32-fixes5 Changeset: r73727:5ca6de0f2fa5 Date: 2014-09-27 23:04 +0300 http://bitbucket.org/pypy/pypy/changeset/5ca6de0f2fa5/ Log: on windows NamedTemporaryFile cannot be reopened by gdb.execute while open in caller diff --git a/pypy/tool/gdb_pypy.py b/pypy/tool/gdb_pypy.py --- a/pypy/tool/gdb_pypy.py +++ b/pypy/tool/gdb_pypy.py @@ -123,11 +123,15 @@ vname = 'pypy_g_rpython_memory_gctypelayout_GCData.gcd_inst_typeids_z' length = int(self.gdb.parse_and_eval('*(long*)%s' % vname)) vstart = '(char*)(((long*)%s)+1)' % vname - with tempfile.NamedTemporaryFile('rb') as fobj: + fname = tempfile.mktemp() + try: self.gdb.execute('dump binary memory %s %s %s+%d' % - (fobj.name, vstart, vstart, length)) - data = fobj.read() - return TypeIdsMap(zlib.decompress(data).splitlines(True), self.gdb) + (fname, vstart, vstart, length)) + with open(fname, 'rt') as fobj: + data = fobj.read() + return TypeIdsMap(zlib.decompress(data).splitlines(True), self.gdb) + finally: + os.remove(fname) class TypeIdsMap(object): From noreply at buildbot.pypy.org Sat Sep 27 23:14:38 2014 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 27 Sep 2014 23:14:38 +0200 (CEST) Subject: [pypy-commit] pypy win32-fixes5: account for rounding error in st_mtime resolution Message-ID: <20140927211438.8612B1C073F@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: win32-fixes5 Changeset: r73728:857bafb8905f Date: 2014-09-27 23:14 +0300 http://bitbucket.org/pypy/pypy/changeset/857bafb8905f/ Log: account for rounding error in st_mtime resolution diff --git a/rpython/rtyper/module/test/test_ll_os_stat.py b/rpython/rtyper/module/test/test_ll_os_stat.py --- a/rpython/rtyper/module/test/test_ll_os_stat.py +++ b/rpython/rtyper/module/test/test_ll_os_stat.py @@ -22,10 +22,10 @@ stat = ll_os_stat.make_win32_stat_impl('stat', ll_os.StringTraits()) wstat = ll_os_stat.make_win32_stat_impl('stat', ll_os.UnicodeTraits()) def check(f): - # msec resolution + # msec resolution, +- rounding error expected = int(os.stat(f).st_mtime*1000) - assert int(stat(f).st_mtime*1000) == expected - assert int(wstat(unicode(f)).st_mtime*1000) == expected + assert abs(int(stat(f).st_mtime*1000) - expected) < 2 + assert abs(int(wstat(unicode(f)).st_mtime*1000) - expected) < 2 check('c:/') check(os.environ['TEMP']) From noreply at buildbot.pypy.org Sat Sep 27 23:35:38 2014 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 27 Sep 2014 23:35:38 +0200 (CEST) Subject: [pypy-commit] pypy ufuncapi: merge default into branch Message-ID: <20140927213538.2F2411C073F@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: ufuncapi Changeset: r73729:2cfa83babbb9 Date: 2014-09-28 00:25 +0300 http://bitbucket.org/pypy/pypy/changeset/2cfa83babbb9/ Log: merge default into branch diff too long, truncating to 2000 out of 67964 lines diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -35,280 +35,290 @@ the beginning of each file) the files in the 'pypy' directory are each copyrighted by one or more of the following people and organizations: - Armin Rigo - Maciej Fijalkowski - Carl Friedrich Bolz - Antonio Cuni - Amaury Forgeot d'Arc - Samuele Pedroni - Alex Gaynor - Michael Hudson - David Schneider - Matti Picus - Brian Kearns - Philip Jenvey - Holger Krekel - Christian Tismer - Hakan Ardo - Benjamin Peterson - Manuel Jacob - Anders Chrigstrom - Eric van Riet Paap - Wim Lavrijsen - Ronan Lamy - Richard Emslie - Alexander Schremmer - Dan Villiom Podlaski Christiansen - Lukas Diekmann - Sven Hager - Anders Lehmann - Aurelien Campeas - Niklaus Haldimann - Camillo Bruni - Laura Creighton - Toon Verwaest - Remi Meier - Leonardo Santagada - Seo Sanghyeon - Romain Guillebert - Justin Peel - Ronny Pfannschmidt - David Edelsohn - Anders Hammarquist - Jakub Gustak - Guido Wesdorp - Lawrence Oluyede - Bartosz Skowron - Daniel Roberts - Niko Matsakis - Adrien Di Mascio - Alexander Hesse - Ludovic Aubry - Jacob Hallen - Jason Creighton - Alex Martelli - Michal Bendowski - Jan de Mooij - stian - Michael Foord - Stephan Diehl - Stefan Schwarzer - Valentino Volonghi - Tomek Meka - Patrick Maupin - Bob Ippolito - Bruno Gola - Jean-Paul Calderone - Timo Paulssen - Squeaky - Alexandre Fayolle - Simon Burton - Marius Gedminas - John Witulski - Konstantin Lopuhin - Greg Price - Dario Bertini - Mark Pearse - Simon Cross - Andreas Stührk - Jean-Philippe St. Pierre - Guido van Rossum - Pavel Vinogradov - Paweł Piotr Przeradowski - Paul deGrandis - Ilya Osadchiy - Tobias Oberstein - Adrian Kuhn - Boris Feigin - Stefano Rivera - tav - Taavi Burns - Georg Brandl - Bert Freudenberg - Stian Andreassen - Laurence Tratt - Wanja Saatkamp - Ivan Sichmann Freitas - Gerald Klix - Mike Blume - Oscar Nierstrasz - Stefan H. Muller - Jeremy Thurgood - Gregor Wegberg - Rami Chowdhury - Tobias Pape - Edd Barrett - David Malcolm - Eugene Oden - Henry Mason - Preston Timmons - Jeff Terrace - David Ripton - Dusty Phillips - Lukas Renggli - Guenter Jantzen - Ned Batchelder - Amit Regmi - Ben Young - Nicolas Chauvat - Andrew Durdin - Andrew Chambers - Michael Schneider - Nicholas Riley - Jason Chu - Igor Trindade Oliveira - Rocco Moretti - Gintautas Miliauskas - Michael Twomey - Lucian Branescu Mihaila - Tim Felgentreff - Tyler Wade - Gabriel Lavoie - Olivier Dormond - Jared Grubb - Karl Bartel - Brian Dorsey - Victor Stinner - Andrews Medina - Stuart Williams - Jasper Schulz - Christian Hudon - Toby Watson - Antoine Pitrou - Aaron Iles - Michael Cheng - Justas Sadzevicius - Mikael Schönenberg - Gasper Zejn - Neil Shepperd - Elmo Mäntynen - Jonathan David Riehl - Stanislaw Halik - Anders Qvist - Chirag Jadwani - Beatrice During - Alex Perry - Vincent Legoll - Alan McIntyre - Alexander Sedov - Corbin Simpson - Christopher Pope - wenzhuman - Christian Tismer - Marc Abramowitz - Dan Stromberg - Stefano Parmesan - Alexis Daboville - Jens-Uwe Mager - Carl Meyer - Karl Ramm - Pieter Zieschang - Gabriel - Lukas Vacek - Andrew Dalke - Sylvain Thenault - Nathan Taylor - Vladimir Kryachko - Jacek Generowicz - Alejandro J. Cura - Jacob Oscarson - Travis Francis Athougies - Ryan Gonzalez - Kristjan Valur Jonsson - Sebastian Pawluś - Neil Blakey-Milner - anatoly techtonik - Lutz Paelike - Lucio Torre - Lars Wassermann - Henrik Vendelbo - Dan Buch - Miguel de Val Borro - Artur Lisiecki - Sergey Kishchenko - Ignas Mikalajunas - Christoph Gerum - Martin Blais - Lene Wagner - Tomo Cocoa - roberto at goyle - Yury V. Zaytsev - Anna Katrina Dominguez - William Leslie - Bobby Impollonia - timo at eistee.fritz.box - Andrew Thompson - Ben Darnell - Roberto De Ioris - Juan Francisco Cantero Hurtado - Godefroid Chappelle - Joshua Gilbert - Dan Colish - Christopher Armstrong - Michael Hudson-Doyle - Anders Sigfridsson - Yasir Suhail - rafalgalczynski at gmail.com - Floris Bruynooghe - Laurens Van Houtven - Akira Li - Gustavo Niemeyer - Stephan Busemann - Rafał Gałczyński - Yusei Tahara - Christian Muirhead - James Lan - shoma hosaka - Daniel Neuh?user - Matthew Miller - Buck Golemon - Konrad Delong - Dinu Gherman - Chris Lambacher - coolbutuseless at gmail.com - Rodrigo Araújo - w31rd0 - Jim Baker - James Robert - Armin Ronacher - Brett Cannon - yrttyr - aliceinwire - OlivierBlanvillain - Zooko Wilcox-O Hearn - Tomer Chachamu - Christopher Groskopf - Asmo Soinio - Stefan Marr - jiaaro - opassembler.py - Antony Lee - Jim Hunziker - Markus Unterwaditzer - Even Wiik Thomassen - jbs - soareschen - Kurt Griffiths - Mike Bayer - Flavio Percoco - Kristoffer Kleine - yasirs - Michael Chermside - Anna Ravencroft - Julien Phalip - Dan Loewenherz + Armin Rigo + Maciej Fijalkowski + Carl Friedrich Bolz + Antonio Cuni + Amaury Forgeot d'Arc + Samuele Pedroni + Alex Gaynor + Michael Hudson + David Schneider + Matti Picus + Brian Kearns + Philip Jenvey + Holger Krekel + Christian Tismer + Hakan Ardo + Benjamin Peterson + Manuel Jacob + Anders Chrigstrom + Eric van Riet Paap + Ronan Lamy + Wim Lavrijsen + Richard Emslie + Alexander Schremmer + Dan Villiom Podlaski Christiansen + Lukas Diekmann + Sven Hager + Anders Lehmann + Aurelien Campeas + Niklaus Haldimann + Remi Meier + Camillo Bruni + Laura Creighton + Toon Verwaest + Leonardo Santagada + Seo Sanghyeon + Romain Guillebert + Justin Peel + Ronny Pfannschmidt + David Edelsohn + Anders Hammarquist + Jakub Gustak + Guido Wesdorp + Lawrence Oluyede + Bartosz Skowron + Gregor Wegberg + Daniel Roberts + Niko Matsakis + Adrien Di Mascio + Alexander Hesse + Ludovic Aubry + Jacob Hallen + Jason Creighton + Alex Martelli + Michal Bendowski + Jan de Mooij + stian + Michael Foord + Stephan Diehl + Tyler Wade + Stefan Schwarzer + Valentino Volonghi + Tomek Meka + Patrick Maupin + Bob Ippolito + Bruno Gola + Jean-Paul Calderone + Timo Paulssen + Squeaky + Alexandre Fayolle + Simon Burton + Marius Gedminas + Martin Matusiak + Konstantin Lopuhin + John Witulski + Wenzhu Man + Greg Price + Dario Bertini + Mark Pearse + Simon Cross + Ivan Sichmann Freitas + Andreas Stührk + Jean-Philippe St. Pierre + Guido van Rossum + Pavel Vinogradov + Stefano Rivera + Paweł Piotr Przeradowski + Paul deGrandis + Ilya Osadchiy + Tobias Oberstein + Adrian Kuhn + Boris Feigin + tav + Taavi Burns + Georg Brandl + Laurence Tratt + Bert Freudenberg + Stian Andreassen + Wanja Saatkamp + Gerald Klix + Mike Blume + Oscar Nierstrasz + Stefan H. Muller + Edd Barrett + Jeremy Thurgood + Rami Chowdhury + Tobias Pape + David Malcolm + Eugene Oden + Henry Mason + Vasily Kuznetsov + Preston Timmons + Jeff Terrace + David Ripton + Dusty Phillips + Lukas Renggli + Guenter Jantzen + Ned Batchelder + Amit Regmi + Ben Young + Nicolas Chauvat + Andrew Durdin + Andrew Chambers + Michael Schneider + Nicholas Riley + Jason Chu + Igor Trindade Oliveira + Tim Felgentreff + Rocco Moretti + Gintautas Miliauskas + Michael Twomey + Lucian Branescu Mihaila + Gabriel Lavoie + Olivier Dormond + Jared Grubb + Karl Bartel + Brian Dorsey + Victor Stinner + Andrews Medina + Stuart Williams + Jasper Schulz + Christian Hudon + Toby Watson + Antoine Pitrou + Aaron Iles + Michael Cheng + Justas Sadzevicius + Gasper Zejn + anatoly techtonik + Neil Shepperd + Mikael Schönenberg + Elmo M?ntynen + Jonathan David Riehl + Stanislaw Halik + Anders Qvist + Corbin Simpson + Chirag Jadwani + Beatrice During + Alex Perry + Vincent Legoll + Alan McIntyre + Alexander Sedov + Christopher Pope + Christian Tismer + Marc Abramowitz + Dan Stromberg + Stefano Parmesan + Alexis Daboville + Jens-Uwe Mager + Carl Meyer + Karl Ramm + Pieter Zieschang + Sebastian Pawluś + Gabriel + Lukas Vacek + Andrew Dalke + Sylvain Thenault + Nathan Taylor + Vladimir Kryachko + Arjun Naik + Attila Gobi + Jacek Generowicz + Alejandro J. Cura + Jacob Oscarson + Travis Francis Athougies + Ryan Gonzalez + Ian Foote + Kristjan Valur Jonsson + Neil Blakey-Milner + Lutz Paelike + Lucio Torre + Lars Wassermann + Valentina Mukhamedzhanova + Henrik Vendelbo + Dan Buch + Miguel de Val Borro + Artur Lisiecki + Sergey Kishchenko + Yichao Yu + Ignas Mikalajunas + Christoph Gerum + Martin Blais + Lene Wagner + Tomo Cocoa + roberto at goyle + Yury V. Zaytsev + Anna Katrina Dominguez + William Leslie + Bobby Impollonia + timo at eistee.fritz.box + Andrew Thompson + Yusei Tahara + Ben Darnell + Roberto De Ioris + Juan Francisco Cantero Hurtado + Godefroid Chappelle + Joshua Gilbert + Dan Colish + Christopher Armstrong + Michael Hudson-Doyle + Anders Sigfridsson + Yasir Suhail + Jason Michalski + rafalgalczynski at gmail.com + Floris Bruynooghe + Laurens Van Houtven + Akira Li + Gustavo Niemeyer + Stephan Busemann + Rafał Gałczyński + Christian Muirhead + James Lan + shoma hosaka + Daniel Neuh?user + Matthew Miller + Buck Golemon + Konrad Delong + Dinu Gherman + Chris Lambacher + coolbutuseless at gmail.com + Rodrigo Araújo + Jim Baker + James Robert + Armin Ronacher + Brett Cannon + yrttyr + aliceinwire + OlivierBlanvillain + Zooko Wilcox-O Hearn + Tomer Chachamu + Christopher Groskopf + Asmo Soinio + Stefan Marr + jiaaro + Mads Kiilerich + opassembler.py + Antony Lee + Jim Hunziker + Markus Unterwaditzer + Even Wiik Thomassen + jbs + soareschen + Kurt Griffiths + Mike Bayer + Matthew Miller + Flavio Percoco + Kristoffer Kleine + yasirs + Michael Chermside + Anna Ravencroft + Dan Crosta + Julien Phalip + Dan Loewenherz - Heinrich-Heine University, Germany - Open End AB (formerly AB Strakt), Sweden - merlinux GmbH, Germany - tismerysoft GmbH, Germany - Logilab Paris, France - DFKI GmbH, Germany - Impara, Germany - Change Maker, Sweden - University of California Berkeley, USA - Google Inc. - King's College London + Heinrich-Heine University, Germany + Open End AB (formerly AB Strakt), Sweden + merlinux GmbH, Germany + tismerysoft GmbH, Germany + Logilab Paris, France + DFKI GmbH, Germany + Impara, Germany + Change Maker, Sweden + University of California Berkeley, USA + Google Inc. + King's College London The PyPy Logo as used by http://speed.pypy.org and others was created by Samuel Reis and is distributed on terms of Creative Commons Share Alike @@ -354,6 +364,6 @@ See the License for the specific language governing permissions and limitations under the License. -Detailled license information is contained in the NOTICE file in the +Detailed license information is contained in the NOTICE file in the directory. diff --git a/_pytest/README-BEFORE-UPDATING b/_pytest/README-BEFORE-UPDATING new file mode 100644 --- /dev/null +++ b/_pytest/README-BEFORE-UPDATING @@ -0,0 +1,17 @@ +This is PyPy's code of the pytest lib. We don't expect to upgrade it +very often, but once we do: + + WARNING! + + WE HAVE MADE A FEW TWEAKS HERE! + +Please be sure that you don't just copy the newer version from +upstream without checking the few changes that we did. This +can be done like this: + + cd + hg log . -v | less + +then search for all " _pytest/" in that list to know which are the +relevant checkins. (Look for the checkins that only edit one +or two files in this directory.) diff --git a/_pytest/__init__.py b/_pytest/__init__.py --- a/_pytest/__init__.py +++ b/_pytest/__init__.py @@ -1,2 +1,2 @@ # -__version__ = '2.2.4.dev2' +__version__ = '2.5.2' diff --git a/_pytest/_argcomplete.py b/_pytest/_argcomplete.py new file mode 100644 --- /dev/null +++ b/_pytest/_argcomplete.py @@ -0,0 +1,104 @@ + +"""allow bash-completion for argparse with argcomplete if installed +needs argcomplete>=0.5.6 for python 3.2/3.3 (older versions fail +to find the magic string, so _ARGCOMPLETE env. var is never set, and +this does not need special code. + +argcomplete does not support python 2.5 (although the changes for that +are minor). + +Function try_argcomplete(parser) should be called directly before +the call to ArgumentParser.parse_args(). + +The filescompleter is what you normally would use on the positional +arguments specification, in order to get "dirname/" after "dirn" +instead of the default "dirname ": + + optparser.add_argument(Config._file_or_dir, nargs='*' + ).completer=filescompleter + +Other, application specific, completers should go in the file +doing the add_argument calls as they need to be specified as .completer +attributes as well. (If argcomplete is not installed, the function the +attribute points to will not be used). + +SPEEDUP +======= +The generic argcomplete script for bash-completion +(/etc/bash_completion.d/python-argcomplete.sh ) +uses a python program to determine startup script generated by pip. +You can speed up completion somewhat by changing this script to include + # PYTHON_ARGCOMPLETE_OK +so the the python-argcomplete-check-easy-install-script does not +need to be called to find the entry point of the code and see if that is +marked with PYTHON_ARGCOMPLETE_OK + +INSTALL/DEBUGGING +================= +To include this support in another application that has setup.py generated +scripts: +- add the line: + # PYTHON_ARGCOMPLETE_OK + near the top of the main python entry point +- include in the file calling parse_args(): + from _argcomplete import try_argcomplete, filescompleter + , call try_argcomplete just before parse_args(), and optionally add + filescompleter to the positional arguments' add_argument() +If things do not work right away: +- switch on argcomplete debugging with (also helpful when doing custom + completers): + export _ARC_DEBUG=1 +- run: + python-argcomplete-check-easy-install-script $(which appname) + echo $? + will echo 0 if the magic line has been found, 1 if not +- sometimes it helps to find early on errors using: + _ARGCOMPLETE=1 _ARC_DEBUG=1 appname + which should throw a KeyError: 'COMPLINE' (which is properly set by the + global argcomplete script). +""" + +import sys +import os +from glob import glob + +class FastFilesCompleter: + 'Fast file completer class' + def __init__(self, directories=True): + self.directories = directories + + def __call__(self, prefix, **kwargs): + """only called on non option completions""" + if os.path.sep in prefix[1:]: # + prefix_dir = len(os.path.dirname(prefix) + os.path.sep) + else: + prefix_dir = 0 + completion = [] + globbed = [] + if '*' not in prefix and '?' not in prefix: + if prefix[-1] == os.path.sep: # we are on unix, otherwise no bash + globbed.extend(glob(prefix + '.*')) + prefix += '*' + globbed.extend(glob(prefix)) + for x in sorted(globbed): + if os.path.isdir(x): + x += '/' + # append stripping the prefix (like bash, not like compgen) + completion.append(x[prefix_dir:]) + return completion + +if os.environ.get('_ARGCOMPLETE'): + # argcomplete 0.5.6 is not compatible with python 2.5.6: print/with/format + if sys.version_info[:2] < (2, 6): + sys.exit(1) + try: + import argcomplete.completers + except ImportError: + sys.exit(-1) + filescompleter = FastFilesCompleter() + + def try_argcomplete(parser): + argcomplete.autocomplete(parser) +else: + def try_argcomplete(parser): pass + filescompleter = None diff --git a/_pytest/assertion/__init__.py b/_pytest/assertion/__init__.py --- a/_pytest/assertion/__init__.py +++ b/_pytest/assertion/__init__.py @@ -3,7 +3,6 @@ """ import py import sys -import pytest from _pytest.monkeypatch import monkeypatch from _pytest.assertion import util @@ -19,8 +18,8 @@ to provide assert expression information. """) group.addoption('--no-assert', action="store_true", default=False, dest="noassert", help="DEPRECATED equivalent to --assert=plain") - group.addoption('--nomagic', action="store_true", default=False, - dest="nomagic", help="DEPRECATED equivalent to --assert=plain") + group.addoption('--nomagic', '--no-magic', action="store_true", + default=False, help="DEPRECATED equivalent to --assert=plain") class AssertionState: """State for the assertion plugin.""" @@ -35,22 +34,25 @@ mode = "plain" if mode == "rewrite": try: - import ast + import ast # noqa except ImportError: mode = "reinterp" else: - if sys.platform.startswith('java'): + # Both Jython and CPython 2.6.0 have AST bugs that make the + # assertion rewriting hook malfunction. + if (sys.platform.startswith('java') or + sys.version_info[:3] == (2, 6, 0)): mode = "reinterp" if mode != "plain": _load_modules(mode) m = monkeypatch() config._cleanup.append(m.undo) m.setattr(py.builtin.builtins, 'AssertionError', - reinterpret.AssertionError) + reinterpret.AssertionError) # noqa hook = None if mode == "rewrite": - hook = rewrite.AssertionRewritingHook() - sys.meta_path.append(hook) + hook = rewrite.AssertionRewritingHook() # noqa + sys.meta_path.insert(0, hook) warn_about_missing_assertion(mode) config._assertstate = AssertionState(config, mode) config._assertstate.hook = hook @@ -73,9 +75,16 @@ def callbinrepr(op, left, right): hook_result = item.ihook.pytest_assertrepr_compare( config=item.config, op=op, left=left, right=right) + for new_expl in hook_result: if new_expl: - res = '\n~'.join(new_expl) + # Don't include pageloads of data unless we are very + # verbose (-vv) + if (sum(len(p) for p in new_expl[1:]) > 80*8 + and item.config.option.verbose < 2): + new_expl[1:] = [py.builtin._totext( + 'Detailed information truncated, use "-vv" to show')] + res = py.builtin._totext('\n~').join(new_expl) if item.config.getvalue("assertmode") == "rewrite": # The result will be fed back a python % formatting # operation, which will fail if there are extraneous @@ -95,9 +104,9 @@ def _load_modules(mode): """Lazily import assertion related code.""" global rewrite, reinterpret - from _pytest.assertion import reinterpret + from _pytest.assertion import reinterpret # noqa if mode == "rewrite": - from _pytest.assertion import rewrite + from _pytest.assertion import rewrite # noqa def warn_about_missing_assertion(mode): try: diff --git a/_pytest/assertion/newinterpret.py b/_pytest/assertion/newinterpret.py --- a/_pytest/assertion/newinterpret.py +++ b/_pytest/assertion/newinterpret.py @@ -11,7 +11,7 @@ from _pytest.assertion.reinterpret import BuiltinAssertionError -if sys.platform.startswith("java") and sys.version_info < (2, 5, 2): +if sys.platform.startswith("java"): # See http://bugs.jython.org/issue1497 _exprs = ("BoolOp", "BinOp", "UnaryOp", "Lambda", "IfExp", "Dict", "ListComp", "GeneratorExp", "Yield", "Compare", "Call", diff --git a/_pytest/assertion/oldinterpret.py b/_pytest/assertion/oldinterpret.py --- a/_pytest/assertion/oldinterpret.py +++ b/_pytest/assertion/oldinterpret.py @@ -526,10 +526,13 @@ # example: def f(): return 5 + def g(): return 3 + def h(x): return 'never' + check("f() * g() == 5") check("not f()") check("not (f() and g() or 0)") diff --git a/_pytest/assertion/reinterpret.py b/_pytest/assertion/reinterpret.py --- a/_pytest/assertion/reinterpret.py +++ b/_pytest/assertion/reinterpret.py @@ -1,18 +1,26 @@ import sys import py from _pytest.assertion.util import BuiltinAssertionError +u = py.builtin._totext + class AssertionError(BuiltinAssertionError): def __init__(self, *args): BuiltinAssertionError.__init__(self, *args) if args: + # on Python2.6 we get len(args)==2 for: assert 0, (x,y) + # on Python2.7 and above we always get len(args) == 1 + # with args[0] being the (x,y) tuple. + if len(args) > 1: + toprint = args + else: + toprint = args[0] try: - self.msg = str(args[0]) - except py.builtin._sysex: - raise - except: - self.msg = "<[broken __repr__] %s at %0xd>" %( - args[0].__class__, id(args[0])) + self.msg = u(toprint) + except Exception: + self.msg = u( + "<[broken __repr__] %s at %0xd>" + % (toprint.__class__, id(toprint))) else: f = py.code.Frame(sys._getframe(1)) try: @@ -44,4 +52,3 @@ from _pytest.assertion.newinterpret import interpret as reinterpret else: reinterpret = reinterpret_old - diff --git a/_pytest/assertion/rewrite.py b/_pytest/assertion/rewrite.py --- a/_pytest/assertion/rewrite.py +++ b/_pytest/assertion/rewrite.py @@ -6,6 +6,7 @@ import imp import marshal import os +import re import struct import sys import types @@ -14,13 +15,7 @@ from _pytest.assertion import util -# Windows gives ENOENT in places *nix gives ENOTDIR. -if sys.platform.startswith("win"): - PATH_COMPONENT_NOT_DIR = errno.ENOENT -else: - PATH_COMPONENT_NOT_DIR = errno.ENOTDIR - -# py.test caches rewritten pycs in __pycache__. +# pytest caches rewritten pycs in __pycache__. if hasattr(imp, "get_tag"): PYTEST_TAG = imp.get_tag() + "-PYTEST" else: @@ -34,17 +29,19 @@ PYTEST_TAG = "%s-%s%s-PYTEST" % (impl, ver[0], ver[1]) del ver, impl -PYC_EXT = ".py" + "c" if __debug__ else "o" +PYC_EXT = ".py" + (__debug__ and "c" or "o") PYC_TAIL = "." + PYTEST_TAG + PYC_EXT REWRITE_NEWLINES = sys.version_info[:2] != (2, 7) and sys.version_info < (3, 2) +ASCII_IS_DEFAULT_ENCODING = sys.version_info[0] < 3 class AssertionRewritingHook(object): - """Import hook which rewrites asserts.""" + """PEP302 Import hook which rewrites asserts.""" def __init__(self): self.session = None self.modules = {} + self._register_with_pkg_resources() def set_session(self, session): self.fnpats = session.config.getini("python_files") @@ -59,8 +56,12 @@ names = name.rsplit(".", 1) lastname = names[-1] pth = None - if path is not None and len(path) == 1: - pth = path[0] + if path is not None: + # Starting with Python 3.3, path is a _NamespacePath(), which + # causes problems if not converted to list. + path = list(path) + if len(path) == 1: + pth = path[0] if pth is None: try: fd, fn, desc = imp.find_module(lastname, path) @@ -95,12 +96,13 @@ finally: self.session = sess else: - state.trace("matched test file (was specified on cmdline): %r" % (fn,)) + state.trace("matched test file (was specified on cmdline): %r" % + (fn,)) # The requested module looks like a test file, so rewrite it. This is # the most magical part of the process: load the source, rewrite the # asserts, and load the rewritten source. We also cache the rewritten # module code in a special pyc. We must be aware of the possibility of - # concurrent py.test processes rewriting and loading pycs. To avoid + # concurrent pytest processes rewriting and loading pycs. To avoid # tricky race conditions, we maintain the following invariant: The # cached pyc is always a complete, valid pyc. Operations on it must be # atomic. POSIX's atomic rename comes in handy. @@ -116,19 +118,19 @@ # common case) or it's blocked by a non-dir node. In the # latter case, we'll ignore it in _write_pyc. pass - elif e == PATH_COMPONENT_NOT_DIR: + elif e in [errno.ENOENT, errno.ENOTDIR]: # One of the path components was not a directory, likely # because we're in a zip file. write = False elif e == errno.EACCES: - state.trace("read only directory: %r" % (fn_pypath.dirname,)) + state.trace("read only directory: %r" % fn_pypath.dirname) write = False else: raise cache_name = fn_pypath.basename[:-3] + PYC_TAIL pyc = os.path.join(cache_dir, cache_name) - # Notice that even if we're in a read-only directory, I'm going to check - # for a cached pyc. This may not be optimal... + # Notice that even if we're in a read-only directory, I'm going + # to check for a cached pyc. This may not be optimal... co = _read_pyc(fn_pypath, pyc) if co is None: state.trace("rewriting %r" % (fn,)) @@ -153,27 +155,59 @@ mod.__file__ = co.co_filename # Normally, this attribute is 3.2+. mod.__cached__ = pyc + mod.__loader__ = self py.builtin.exec_(co, mod.__dict__) except: del sys.modules[name] raise return sys.modules[name] -def _write_pyc(co, source_path, pyc): - # Technically, we don't have to have the same pyc format as (C)Python, since - # these "pycs" should never be seen by builtin import. However, there's - # little reason deviate, and I hope sometime to be able to use - # imp.load_compiled to load them. (See the comment in load_module above.) + + + def is_package(self, name): + try: + fd, fn, desc = imp.find_module(name) + except ImportError: + return False + if fd is not None: + fd.close() + tp = desc[2] + return tp == imp.PKG_DIRECTORY + + @classmethod + def _register_with_pkg_resources(cls): + """ + Ensure package resources can be loaded from this loader. May be called + multiple times, as the operation is idempotent. + """ + try: + import pkg_resources + # access an attribute in case a deferred importer is present + pkg_resources.__name__ + except ImportError: + return + + # Since pytest tests are always located in the file system, the + # DefaultProvider is appropriate. + pkg_resources.register_loader_type(cls, pkg_resources.DefaultProvider) + + +def _write_pyc(state, co, source_path, pyc): + # Technically, we don't have to have the same pyc format as + # (C)Python, since these "pycs" should never be seen by builtin + # import. However, there's little reason deviate, and I hope + # sometime to be able to use imp.load_compiled to load them. (See + # the comment in load_module above.) mtime = int(source_path.mtime()) try: fp = open(pyc, "wb") except IOError: err = sys.exc_info()[1].errno - if err == PATH_COMPONENT_NOT_DIR: - # This happens when we get a EEXIST in find_module creating the - # __pycache__ directory and __pycache__ is by some non-dir node. - return False - raise + state.trace("error writing pyc file at %s: errno=%s" %(pyc, err)) + # we ignore any failure to write the cache file + # there are many reasons, permission-denied, __pycache__ being a + # file etc. + return False try: fp.write(imp.get_magic()) fp.write(struct.pack(">", - ast.Add : "+", - ast.Sub : "-", - ast.Mult : "*", - ast.Div : "/", - ast.FloorDiv : "//", - ast.Mod : "%", - ast.Eq : "==", - ast.NotEq : "!=", - ast.Lt : "<", - ast.LtE : "<=", - ast.Gt : ">", - ast.GtE : ">=", - ast.Pow : "**", - ast.Is : "is", - ast.IsNot : "is not", - ast.In : "in", - ast.NotIn : "not in" + ast.BitOr: "|", + ast.BitXor: "^", + ast.BitAnd: "&", + ast.LShift: "<<", + ast.RShift: ">>", + ast.Add: "+", + ast.Sub: "-", + ast.Mult: "*", + ast.Div: "/", + ast.FloorDiv: "//", + ast.Mod: "%%", # escaped for string formatting + ast.Eq: "==", + ast.NotEq: "!=", + ast.Lt: "<", + ast.LtE: "<=", + ast.Gt: ">", + ast.GtE: ">=", + ast.Pow: "**", + ast.Is: "is", + ast.IsNot: "is not", + ast.In: "in", + ast.NotIn: "not in" } @@ -341,7 +408,7 @@ lineno = 0 for item in mod.body: if (expect_docstring and isinstance(item, ast.Expr) and - isinstance(item.value, ast.Str)): + isinstance(item.value, ast.Str)): doc = item.value.s if "PYTEST_DONT_REWRITE" in doc: # The module has disabled assertion rewriting. @@ -462,7 +529,8 @@ body.append(raise_) # Clear temporary variables by setting them to None. if self.variables: - variables = [ast.Name(name, ast.Store()) for name in self.variables] + variables = [ast.Name(name, ast.Store()) + for name in self.variables] clear = ast.Assign(variables, ast.Name("None", ast.Load())) self.statements.append(clear) # Fix line numbers. @@ -471,11 +539,12 @@ return self.statements def visit_Name(self, name): - # Check if the name is local or not. + # Display the repr of the name if it's a local variable or + # _should_repr_global_name() thinks it's acceptable. locs = ast.Call(self.builtin("locals"), [], [], None, None) - globs = ast.Call(self.builtin("globals"), [], [], None, None) - ops = [ast.In(), ast.IsNot()] - test = ast.Compare(ast.Str(name.id), ops, [locs, globs]) + inlocs = ast.Compare(ast.Str(name.id), [ast.In()], [locs]) + dorepr = self.helper("should_repr_global_name", name) + test = ast.BoolOp(ast.Or(), [inlocs, dorepr]) expr = ast.IfExp(test, self.display(name), ast.Str(name.id)) return name, self.explanation_param(expr) @@ -492,7 +561,8 @@ for i, v in enumerate(boolop.values): if i: fail_inner = [] - self.on_failure.append(ast.If(cond, fail_inner, [])) + # cond is set in a prior loop iteration below + self.on_failure.append(ast.If(cond, fail_inner, [])) # noqa self.on_failure = fail_inner self.push_format_context() res, expl = self.visit(v) @@ -548,7 +618,8 @@ new_kwarg, expl = self.visit(call.kwargs) arg_expls.append("**" + expl) expl = "%s(%s)" % (func_expl, ', '.join(arg_expls)) - new_call = ast.Call(new_func, new_args, new_kwargs, new_star, new_kwarg) + new_call = ast.Call(new_func, new_args, new_kwargs, + new_star, new_kwarg) res = self.assign(new_call) res_expl = self.explanation_param(self.display(res)) outer_expl = "%s\n{%s = %s\n}" % (res_expl, res_expl, expl) @@ -584,7 +655,7 @@ res_expr = ast.Compare(left_res, [op], [next_res]) self.statements.append(ast.Assign([store_names[i]], res_expr)) left_res, left_expl = next_res, next_expl - # Use py.code._reprcompare if that's available. + # Use pytest.assertion.util._reprcompare if that's available. expl_call = self.helper("call_reprcompare", ast.Tuple(syms, ast.Load()), ast.Tuple(load_names, ast.Load()), diff --git a/_pytest/assertion/util.py b/_pytest/assertion/util.py --- a/_pytest/assertion/util.py +++ b/_pytest/assertion/util.py @@ -1,8 +1,13 @@ """Utilities for assertion debugging""" import py +try: + from collections import Sequence +except ImportError: + Sequence = list BuiltinAssertionError = py.builtin.builtins.AssertionError +u = py.builtin._totext # The _reprcompare attribute on the util module is used by the new assertion # interpretation code and assertion rewriter to detect this plugin was @@ -10,6 +15,7 @@ # DebugInterpreter. _reprcompare = None + def format_explanation(explanation): """This formats an explanation @@ -20,7 +26,18 @@ for when one explanation needs to span multiple lines, e.g. when displaying diffs. """ - # simplify 'assert False where False = ...' + explanation = _collapse_false(explanation) + lines = _split_explanation(explanation) + result = _format_lines(lines) + return u('\n').join(result) + + +def _collapse_false(explanation): + """Collapse expansions of False + + So this strips out any "assert False\n{where False = ...\n}" + blocks. + """ where = 0 while True: start = where = explanation.find("False\n{False = ", where) @@ -42,28 +59,48 @@ explanation = (explanation[:start] + explanation[start+15:end-1] + explanation[end+1:]) where -= 17 - raw_lines = (explanation or '').split('\n') - # escape newlines not followed by {, } and ~ + return explanation + + +def _split_explanation(explanation): + """Return a list of individual lines in the explanation + + This will return a list of lines split on '\n{', '\n}' and '\n~'. + Any other newlines will be escaped and appear in the line as the + literal '\n' characters. + """ + raw_lines = (explanation or u('')).split('\n') lines = [raw_lines[0]] for l in raw_lines[1:]: if l.startswith('{') or l.startswith('}') or l.startswith('~'): lines.append(l) else: lines[-1] += '\\n' + l + return lines + +def _format_lines(lines): + """Format the individual lines + + This will replace the '{', '}' and '~' characters of our mini + formatting language with the proper 'where ...', 'and ...' and ' + + ...' text, taking care of indentation along the way. + + Return a list of formatted lines. + """ result = lines[:1] stack = [0] stackcnt = [0] for line in lines[1:]: if line.startswith('{'): if stackcnt[-1]: - s = 'and ' + s = u('and ') else: - s = 'where ' + s = u('where ') stack.append(len(result)) stackcnt[-1] += 1 stackcnt.append(0) - result.append(' +' + ' '*(len(stack)-1) + s + line[1:]) + result.append(u(' +') + u(' ')*(len(stack)-1) + s + line[1:]) elif line.startswith('}'): assert line.startswith('}') stack.pop() @@ -71,9 +108,9 @@ result[stack[-1]] += line[1:] else: assert line.startswith('~') - result.append(' '*len(stack) + line[1:]) + result.append(u(' ')*len(stack) + line[1:]) assert len(stack) == 1 - return '\n'.join(result) + return result # Provide basestring in python3 @@ -83,132 +120,163 @@ basestring = str -def assertrepr_compare(op, left, right): - """return specialised explanations for some operators/operands""" - width = 80 - 15 - len(op) - 2 # 15 chars indentation, 1 space around op +def assertrepr_compare(config, op, left, right): + """Return specialised explanations for some operators/operands""" + width = 80 - 15 - len(op) - 2 # 15 chars indentation, 1 space around op left_repr = py.io.saferepr(left, maxsize=int(width/2)) right_repr = py.io.saferepr(right, maxsize=width-len(left_repr)) - summary = '%s %s %s' % (left_repr, op, right_repr) + summary = u('%s %s %s') % (left_repr, op, right_repr) - issequence = lambda x: isinstance(x, (list, tuple)) + issequence = lambda x: (isinstance(x, (list, tuple, Sequence)) + and not isinstance(x, basestring)) istext = lambda x: isinstance(x, basestring) isdict = lambda x: isinstance(x, dict) - isset = lambda x: isinstance(x, set) + isset = lambda x: isinstance(x, (set, frozenset)) + verbose = config.getoption('verbose') explanation = None try: if op == '==': if istext(left) and istext(right): - explanation = _diff_text(left, right) + explanation = _diff_text(left, right, verbose) elif issequence(left) and issequence(right): - explanation = _compare_eq_sequence(left, right) + explanation = _compare_eq_sequence(left, right, verbose) elif isset(left) and isset(right): - explanation = _compare_eq_set(left, right) + explanation = _compare_eq_set(left, right, verbose) elif isdict(left) and isdict(right): - explanation = _diff_text(py.std.pprint.pformat(left), - py.std.pprint.pformat(right)) + explanation = _compare_eq_dict(left, right, verbose) elif op == 'not in': if istext(left) and istext(right): - explanation = _notin_text(left, right) - except py.builtin._sysex: - raise - except: + explanation = _notin_text(left, right, verbose) + except Exception: excinfo = py.code.ExceptionInfo() - explanation = ['(pytest_assertion plugin: representation of ' - 'details failed. Probably an object has a faulty __repr__.)', - str(excinfo) - ] - + explanation = [ + u('(pytest_assertion plugin: representation of details failed. ' + 'Probably an object has a faulty __repr__.)'), + u(excinfo)] if not explanation: return None - # Don't include pageloads of data, should be configurable - if len(''.join(explanation)) > 80*8: - explanation = ['Detailed information too verbose, truncated'] - return [summary] + explanation -def _diff_text(left, right): - """Return the explanation for the diff between text +def _diff_text(left, right, verbose=False): + """Return the explanation for the diff between text or bytes - This will skip leading and trailing characters which are - identical to keep the diff minimal. + Unless --verbose is used this will skip leading and trailing + characters which are identical to keep the diff minimal. + + If the input are bytes they will be safely converted to text. """ explanation = [] - i = 0 # just in case left or right has zero length - for i in range(min(len(left), len(right))): - if left[i] != right[i]: - break - if i > 42: - i -= 10 # Provide some context - explanation = ['Skipping %s identical ' - 'leading characters in diff' % i] - left = left[i:] - right = right[i:] - if len(left) == len(right): - for i in range(len(left)): - if left[-i] != right[-i]: + if isinstance(left, py.builtin.bytes): + left = u(repr(left)[1:-1]).replace(r'\n', '\n') + if isinstance(right, py.builtin.bytes): + right = u(repr(right)[1:-1]).replace(r'\n', '\n') + if not verbose: + i = 0 # just in case left or right has zero length + for i in range(min(len(left), len(right))): + if left[i] != right[i]: break if i > 42: - i -= 10 # Provide some context - explanation += ['Skipping %s identical ' - 'trailing characters in diff' % i] - left = left[:-i] - right = right[:-i] + i -= 10 # Provide some context + explanation = [u('Skipping %s identical leading ' + 'characters in diff, use -v to show') % i] + left = left[i:] + right = right[i:] + if len(left) == len(right): + for i in range(len(left)): + if left[-i] != right[-i]: + break + if i > 42: + i -= 10 # Provide some context + explanation += [u('Skipping %s identical trailing ' + 'characters in diff, use -v to show') % i] + left = left[:-i] + right = right[:-i] explanation += [line.strip('\n') for line in py.std.difflib.ndiff(left.splitlines(), right.splitlines())] return explanation -def _compare_eq_sequence(left, right): +def _compare_eq_sequence(left, right, verbose=False): explanation = [] for i in range(min(len(left), len(right))): if left[i] != right[i]: - explanation += ['At index %s diff: %r != %r' % - (i, left[i], right[i])] + explanation += [u('At index %s diff: %r != %r') + % (i, left[i], right[i])] break if len(left) > len(right): - explanation += ['Left contains more items, ' - 'first extra item: %s' % py.io.saferepr(left[len(right)],)] + explanation += [u('Left contains more items, first extra item: %s') + % py.io.saferepr(left[len(right)],)] elif len(left) < len(right): - explanation += ['Right contains more items, ' - 'first extra item: %s' % py.io.saferepr(right[len(left)],)] - return explanation # + _diff_text(py.std.pprint.pformat(left), - # py.std.pprint.pformat(right)) + explanation += [ + u('Right contains more items, first extra item: %s') % + py.io.saferepr(right[len(left)],)] + return explanation # + _diff_text(py.std.pprint.pformat(left), + # py.std.pprint.pformat(right)) -def _compare_eq_set(left, right): +def _compare_eq_set(left, right, verbose=False): explanation = [] diff_left = left - right diff_right = right - left if diff_left: - explanation.append('Extra items in the left set:') + explanation.append(u('Extra items in the left set:')) for item in diff_left: explanation.append(py.io.saferepr(item)) if diff_right: - explanation.append('Extra items in the right set:') + explanation.append(u('Extra items in the right set:')) for item in diff_right: explanation.append(py.io.saferepr(item)) return explanation -def _notin_text(term, text): +def _compare_eq_dict(left, right, verbose=False): + explanation = [] + common = set(left).intersection(set(right)) + same = dict((k, left[k]) for k in common if left[k] == right[k]) + if same and not verbose: + explanation += [u('Omitting %s identical items, use -v to show') % + len(same)] + elif same: + explanation += [u('Common items:')] + explanation += py.std.pprint.pformat(same).splitlines() + diff = set(k for k in common if left[k] != right[k]) + if diff: + explanation += [u('Differing items:')] + for k in diff: + explanation += [py.io.saferepr({k: left[k]}) + ' != ' + + py.io.saferepr({k: right[k]})] + extra_left = set(left) - set(right) + if extra_left: + explanation.append(u('Left contains more items:')) + explanation.extend(py.std.pprint.pformat( + dict((k, left[k]) for k in extra_left)).splitlines()) + extra_right = set(right) - set(left) + if extra_right: + explanation.append(u('Right contains more items:')) + explanation.extend(py.std.pprint.pformat( + dict((k, right[k]) for k in extra_right)).splitlines()) + return explanation + + +def _notin_text(term, text, verbose=False): index = text.find(term) head = text[:index] tail = text[index+len(term):] correct_text = head + tail - diff = _diff_text(correct_text, text) - newdiff = ['%s is contained here:' % py.io.saferepr(term, maxsize=42)] + diff = _diff_text(correct_text, text, verbose) + newdiff = [u('%s is contained here:') % py.io.saferepr(term, maxsize=42)] for line in diff: - if line.startswith('Skipping'): + if line.startswith(u('Skipping')): continue - if line.startswith('- '): + if line.startswith(u('- ')): continue - if line.startswith('+ '): - newdiff.append(' ' + line[2:]) + if line.startswith(u('+ ')): + newdiff.append(u(' ') + line[2:]) else: newdiff.append(line) return newdiff diff --git a/_pytest/capture.py b/_pytest/capture.py --- a/_pytest/capture.py +++ b/_pytest/capture.py @@ -1,43 +1,114 @@ -""" per-test stdout/stderr capturing mechanisms, ``capsys`` and ``capfd`` function arguments. """ +""" + per-test stdout/stderr capturing mechanisms, + ``capsys`` and ``capfd`` function arguments. +""" +# note: py.io capture was where copied from +# pylib 1.4.20.dev2 (rev 13d9af95547e) +import sys +import os +import tempfile -import pytest, py -import os +import py +import pytest + +try: + from io import StringIO +except ImportError: + from StringIO import StringIO + +try: + from io import BytesIO +except ImportError: + class BytesIO(StringIO): + def write(self, data): + if isinstance(data, unicode): + raise TypeError("not a byte value: %r" % (data,)) + StringIO.write(self, data) + +if sys.version_info < (3, 0): + class TextIO(StringIO): + def write(self, data): + if not isinstance(data, unicode): + enc = getattr(self, '_encoding', 'UTF-8') + data = unicode(data, enc, 'replace') + StringIO.write(self, data) +else: + TextIO = StringIO + + +patchsysdict = {0: 'stdin', 1: 'stdout', 2: 'stderr'} + def pytest_addoption(parser): group = parser.getgroup("general") - group._addoption('--capture', action="store", default=None, - metavar="method", type="choice", choices=['fd', 'sys', 'no'], + group._addoption( + '--capture', action="store", default=None, + metavar="method", choices=['fd', 'sys', 'no'], help="per-test capturing method: one of fd (default)|sys|no.") - group._addoption('-s', action="store_const", const="no", dest="capture", + group._addoption( + '-s', action="store_const", const="no", dest="capture", help="shortcut for --capture=no.") + @pytest.mark.tryfirst -def pytest_cmdline_parse(pluginmanager, args): - # we want to perform capturing already for plugin/conftest loading - if '-s' in args or "--capture=no" in args: - method = "no" - elif hasattr(os, 'dup') and '--capture=sys' not in args: +def pytest_load_initial_conftests(early_config, parser, args, __multicall__): + ns = parser.parse_known_args(args) + method = ns.capture + if not method: method = "fd" - else: + if method == "fd" and not hasattr(os, "dup"): method = "sys" capman = CaptureManager(method) - pluginmanager.register(capman, "capturemanager") + early_config.pluginmanager.register(capman, "capturemanager") + + # make sure that capturemanager is properly reset at final shutdown + def teardown(): + try: + capman.reset_capturings() + except ValueError: + pass + + early_config.pluginmanager.add_shutdown(teardown) + + # make sure logging does not raise exceptions at the end + def silence_logging_at_shutdown(): + if "logging" in sys.modules: + sys.modules["logging"].raiseExceptions = False + early_config.pluginmanager.add_shutdown(silence_logging_at_shutdown) + + # finally trigger conftest loading but while capturing (issue93) + capman.resumecapture() + try: + try: + return __multicall__.execute() + finally: + out, err = capman.suspendcapture() + except: + sys.stdout.write(out) + sys.stderr.write(err) + raise + def addouterr(rep, outerr): for secname, content in zip(["out", "err"], outerr): if content: rep.sections.append(("Captured std%s" % secname, content)) + class NoCapture: def startall(self): pass + def resume(self): pass + def reset(self): pass + def suspend(self): return "", "" + class CaptureManager: def __init__(self, defaultmethod=None): self._method2capture = {} @@ -45,21 +116,23 @@ def _maketempfile(self): f = py.std.tempfile.TemporaryFile() - newf = py.io.dupfile(f, encoding="UTF-8") + newf = dupfile(f, encoding="UTF-8") f.close() return newf def _makestringio(self): - return py.io.TextIO() + return TextIO() def _getcapture(self, method): if method == "fd": - return py.io.StdCaptureFD(now=False, - out=self._maketempfile(), err=self._maketempfile() + return StdCaptureFD( + out=self._maketempfile(), + err=self._maketempfile(), ) elif method == "sys": - return py.io.StdCapture(now=False, - out=self._makestringio(), err=self._makestringio() + return StdCapture( + out=self._makestringio(), + err=self._makestringio(), ) elif method == "no": return NoCapture() @@ -74,23 +147,24 @@ method = config._conftest.rget("option_capture", path=fspath) except KeyError: method = "fd" - if method == "fd" and not hasattr(os, 'dup'): # e.g. jython + if method == "fd" and not hasattr(os, 'dup'): # e.g. jython method = "sys" return method def reset_capturings(self): - for name, cap in self._method2capture.items(): + for cap in self._method2capture.values(): cap.reset() def resumecapture_item(self, item): method = self._getmethod(item.config, item.fspath) if not hasattr(item, 'outerr'): - item.outerr = ('', '') # we accumulate outerr on the item + item.outerr = ('', '') # we accumulate outerr on the item return self.resumecapture(method) def resumecapture(self, method=None): if hasattr(self, '_capturing'): - raise ValueError("cannot resume, already capturing with %r" % + raise ValueError( + "cannot resume, already capturing with %r" % (self._capturing,)) if method is None: method = self._defaultmethod @@ -119,30 +193,29 @@ return "", "" def activate_funcargs(self, pyfuncitem): - if not hasattr(pyfuncitem, 'funcargs'): - return - assert not hasattr(self, '_capturing_funcargs') - self._capturing_funcargs = capturing_funcargs = [] - for name, capfuncarg in pyfuncitem.funcargs.items(): - if name in ('capsys', 'capfd'): - capturing_funcargs.append(capfuncarg) - capfuncarg._start() + funcargs = getattr(pyfuncitem, "funcargs", None) + if funcargs is not None: + for name, capfuncarg in funcargs.items(): + if name in ('capsys', 'capfd'): + assert not hasattr(self, '_capturing_funcarg') + self._capturing_funcarg = capfuncarg + capfuncarg._start() def deactivate_funcargs(self): - capturing_funcargs = getattr(self, '_capturing_funcargs', None) - if capturing_funcargs is not None: - while capturing_funcargs: - capfuncarg = capturing_funcargs.pop() - capfuncarg._finalize() - del self._capturing_funcargs + capturing_funcarg = getattr(self, '_capturing_funcarg', None) + if capturing_funcarg: + outerr = capturing_funcarg._finalize() + del self._capturing_funcarg + return outerr def pytest_make_collect_report(self, __multicall__, collector): method = self._getmethod(collector.config, collector.fspath) try: self.resumecapture(method) except ValueError: - return # recursive collect, XXX refactor capturing - # to allow for more lightweight recursive capturing + # recursive collect, XXX refactor capturing + # to allow for more lightweight recursive capturing + return try: rep = __multicall__.execute() finally: @@ -169,46 +242,371 @@ @pytest.mark.tryfirst def pytest_runtest_makereport(self, __multicall__, item, call): - self.deactivate_funcargs() + funcarg_outerr = self.deactivate_funcargs() rep = __multicall__.execute() outerr = self.suspendcapture(item) - if not rep.passed: - addouterr(rep, outerr) + if funcarg_outerr is not None: + outerr = (outerr[0] + funcarg_outerr[0], + outerr[1] + funcarg_outerr[1]) + addouterr(rep, outerr) if not rep.passed or rep.when == "teardown": outerr = ('', '') item.outerr = outerr return rep +error_capsysfderror = "cannot use capsys and capfd at the same time" + + def pytest_funcarg__capsys(request): """enables capturing of writes to sys.stdout/sys.stderr and makes captured output available via ``capsys.readouterr()`` method calls which return a ``(out, err)`` tuple. """ - return CaptureFuncarg(py.io.StdCapture) + if "capfd" in request._funcargs: + raise request.raiseerror(error_capsysfderror) + return CaptureFixture(StdCapture) + def pytest_funcarg__capfd(request): """enables capturing of writes to file descriptors 1 and 2 and makes captured output available via ``capsys.readouterr()`` method calls which return a ``(out, err)`` tuple. """ + if "capsys" in request._funcargs: + request.raiseerror(error_capsysfderror) if not hasattr(os, 'dup'): - py.test.skip("capfd funcarg needs os.dup") - return CaptureFuncarg(py.io.StdCaptureFD) + pytest.skip("capfd funcarg needs os.dup") + return CaptureFixture(StdCaptureFD) -class CaptureFuncarg: + +class CaptureFixture: def __init__(self, captureclass): - self.capture = captureclass(now=False) + self._capture = captureclass() def _start(self): - self.capture.startall() + self._capture.startall() def _finalize(self): - if hasattr(self, 'capture'): - self.capture.reset() - del self.capture + if hasattr(self, '_capture'): + outerr = self._outerr = self._capture.reset() + del self._capture + return outerr def readouterr(self): - return self.capture.readouterr() + try: + return self._capture.readouterr() + except AttributeError: + return self._outerr def close(self): self._finalize() + + +class FDCapture: + """ Capture IO to/from a given os-level filedescriptor. """ + + def __init__(self, targetfd, tmpfile=None, patchsys=False): + """ save targetfd descriptor, and open a new + temporary file there. If no tmpfile is + specified a tempfile.Tempfile() will be opened + in text mode. + """ + self.targetfd = targetfd + if tmpfile is None and targetfd != 0: + f = tempfile.TemporaryFile('wb+') + tmpfile = dupfile(f, encoding="UTF-8") + f.close() + self.tmpfile = tmpfile + self._savefd = os.dup(self.targetfd) + if patchsys: + self._oldsys = getattr(sys, patchsysdict[targetfd]) + + def start(self): + try: + os.fstat(self._savefd) + except OSError: + raise ValueError( + "saved filedescriptor not valid, " + "did you call start() twice?") + if self.targetfd == 0 and not self.tmpfile: + fd = os.open(os.devnull, os.O_RDONLY) + os.dup2(fd, 0) + os.close(fd) + if hasattr(self, '_oldsys'): + setattr(sys, patchsysdict[self.targetfd], DontReadFromInput()) + else: + os.dup2(self.tmpfile.fileno(), self.targetfd) + if hasattr(self, '_oldsys'): + setattr(sys, patchsysdict[self.targetfd], self.tmpfile) + + def done(self): + """ unpatch and clean up, returns the self.tmpfile (file object) + """ + os.dup2(self._savefd, self.targetfd) + os.close(self._savefd) + if self.targetfd != 0: + self.tmpfile.seek(0) + if hasattr(self, '_oldsys'): + setattr(sys, patchsysdict[self.targetfd], self._oldsys) + return self.tmpfile + + def writeorg(self, data): + """ write a string to the original file descriptor + """ + tempfp = tempfile.TemporaryFile() + try: + os.dup2(self._savefd, tempfp.fileno()) + tempfp.write(data) + finally: + tempfp.close() + + +def dupfile(f, mode=None, buffering=0, raising=False, encoding=None): + """ return a new open file object that's a duplicate of f + + mode is duplicated if not given, 'buffering' controls + buffer size (defaulting to no buffering) and 'raising' + defines whether an exception is raised when an incompatible + file object is passed in (if raising is False, the file + object itself will be returned) + """ + try: + fd = f.fileno() + mode = mode or f.mode + except AttributeError: + if raising: + raise + return f + newfd = os.dup(fd) + if sys.version_info >= (3, 0): + if encoding is not None: + mode = mode.replace("b", "") + buffering = True + return os.fdopen(newfd, mode, buffering, encoding, closefd=True) + else: + f = os.fdopen(newfd, mode, buffering) + if encoding is not None: + return EncodedFile(f, encoding) + return f + + +class EncodedFile(object): + def __init__(self, _stream, encoding): + self._stream = _stream + self.encoding = encoding + From noreply at buildbot.pypy.org Sun Sep 28 00:31:14 2014 From: noreply at buildbot.pypy.org (mattip) Date: Sun, 28 Sep 2014 00:31:14 +0200 (CEST) Subject: [pypy-commit] pypy default: move test to more appropriate class Message-ID: <20140927223114.BB2691C101E@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r73730:742921282679 Date: 2014-09-28 01:26 +0300 http://bitbucket.org/pypy/pypy/changeset/742921282679/ Log: move test to more appropriate class diff --git a/pypy/module/select/test/test_select.py b/pypy/module/select/test/test_select.py --- a/pypy/module/select/test/test_select.py +++ b/pypy/module/select/test/test_select.py @@ -244,20 +244,6 @@ raises(OverflowError, pollster.modify, 1, -1) raises(OverflowError, pollster.modify, 1, 1 << 64) - def test_resize_list_in_select(self): - import select - class Foo(object): - def fileno(self): - print len(l) - if len(l) < 100: - l.append(Foo()) - return 0 - l = [Foo()] - select.select(l, (), (), 0) - assert 1 <= len(l) <= 100 - # ^^^ CPython gives 100, PyPy gives 1. I think both are OK as - # long as there is no crash. - class AppTestSelectWithPipes(_AppTestSelect): "Use a pipe to get pairs of file descriptors" @@ -318,6 +304,20 @@ for fd in rfds: os.close(fd) + def test_resize_list_in_select(self): + import select + class Foo(object): + def fileno(self): + print len(l) + if len(l) < 100: + l.append(Foo()) + return 0 + l = [Foo()] + select.select(l, (), (), 0) + assert 1 <= len(l) <= 100 + # ^^^ CPython gives 100, PyPy gives 1. I think both are OK as + # long as there is no crash. + class AppTestSelectWithSockets(_AppTestSelect): """Same tests with connected sockets. From noreply at buildbot.pypy.org Sun Sep 28 08:39:24 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 28 Sep 2014 08:39:24 +0200 (CEST) Subject: [pypy-commit] pypy default: Oups, wrong test. Fix it and be more precise: check exactly each Message-ID: <20140928063924.97AE41C02ED@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73731:688d9d295fc2 Date: 2014-09-28 08:39 +0200 http://bitbucket.org/pypy/pypy/changeset/688d9d295fc2/ Log: Oups, wrong test. Fix it and be more precise: check exactly each implementation. diff --git a/lib-python/2.7/test/test_select.py b/lib-python/2.7/test/test_select.py --- a/lib-python/2.7/test/test_select.py +++ b/lib-python/2.7/test/test_select.py @@ -62,7 +62,12 @@ # removes an item and at the middle the iteration stops. # PyPy: 'a' ends up empty, because the iteration is done on # a copy of the original list: fileno() is called 10 times. - self.assert_(len(result[1]) <= 5) + if test_support.check_impl_detail(cpython=True): + self.assertEqual(len(result[1]), 5) + self.assertEqual(len(a), 5) + if test_support.check_impl_detail(pypy=True): + self.assertEqual(len(result[1]), 10) + self.assertEqual(len(a), 0) def test_main(): test_support.run_unittest(SelectTestCase) From noreply at buildbot.pypy.org Sun Sep 28 08:43:05 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 28 Sep 2014 08:43:05 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: hg merge default Message-ID: <20140928064305.BF90A1C02ED@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: gc_no_cleanup_nursery Changeset: r73732:6e9e4f255563 Date: 2014-09-28 08:42 +0200 http://bitbucket.org/pypy/pypy/changeset/6e9e4f255563/ Log: hg merge default diff --git a/lib-python/2.7/test/test_select.py b/lib-python/2.7/test/test_select.py --- a/lib-python/2.7/test/test_select.py +++ b/lib-python/2.7/test/test_select.py @@ -57,7 +57,17 @@ del a[-1] return sys.__stdout__.fileno() a[:] = [F()] * 10 - self.assertEqual(select.select([], a, []), ([], a[:5], [])) + result = select.select([], a, []) + # CPython: 'a' ends up with 5 items, because each fileno() + # removes an item and at the middle the iteration stops. + # PyPy: 'a' ends up empty, because the iteration is done on + # a copy of the original list: fileno() is called 10 times. + if test_support.check_impl_detail(cpython=True): + self.assertEqual(len(result[1]), 5) + self.assertEqual(len(a), 5) + if test_support.check_impl_detail(pypy=True): + self.assertEqual(len(result[1]), 10) + self.assertEqual(len(a), 0) def test_main(): test_support.run_unittest(SelectTestCase) diff --git a/pypy/module/select/test/test_select.py b/pypy/module/select/test/test_select.py --- a/pypy/module/select/test/test_select.py +++ b/pypy/module/select/test/test_select.py @@ -244,20 +244,6 @@ raises(OverflowError, pollster.modify, 1, -1) raises(OverflowError, pollster.modify, 1, 1 << 64) - def test_resize_list_in_select(self): - import select - class Foo(object): - def fileno(self): - print len(l) - if len(l) < 100: - l.append(Foo()) - return 0 - l = [Foo()] - select.select(l, (), (), 0) - assert 1 <= len(l) <= 100 - # ^^^ CPython gives 100, PyPy gives 1. I think both are OK as - # long as there is no crash. - class AppTestSelectWithPipes(_AppTestSelect): "Use a pipe to get pairs of file descriptors" @@ -318,6 +304,20 @@ for fd in rfds: os.close(fd) + def test_resize_list_in_select(self): + import select + class Foo(object): + def fileno(self): + print len(l) + if len(l) < 100: + l.append(Foo()) + return 0 + l = [Foo()] + select.select(l, (), (), 0) + assert 1 <= len(l) <= 100 + # ^^^ CPython gives 100, PyPy gives 1. I think both are OK as + # long as there is no crash. + class AppTestSelectWithSockets(_AppTestSelect): """Same tests with connected sockets. From noreply at buildbot.pypy.org Sun Sep 28 08:59:52 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 28 Sep 2014 08:59:52 +0200 (CEST) Subject: [pypy-commit] pypy default: Random attempt at fixing this test which sometimes fails under OS/X. Message-ID: <20140928065952.19AFF1C02ED@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73733:c5efcd807854 Date: 2014-09-28 08:59 +0200 http://bitbucket.org/pypy/pypy/changeset/c5efcd807854/ Log: Random attempt at fixing this test which sometimes fails under OS/X. Maybe it's the kernel that isn't ready to give back the data we pushed into the local socket, if we use a delay of 0. So while we didn't get all data back, use a delay of 5 seconds. diff --git a/pypy/module/select/test/test_select.py b/pypy/module/select/test/test_select.py --- a/pypy/module/select/test/test_select.py +++ b/pypy/module/select/test/test_select.py @@ -85,17 +85,18 @@ assert owtd == [writeend] total_out += writeend.send(b'x' * 512) total_in = 0 - while True: - iwtd, owtd, ewtd = select.select([readend], [], [], 0) + while total_in < total_out: + iwtd, owtd, ewtd = select.select([readend], [], [], 5) assert owtd == ewtd == [] - if iwtd == []: - break - assert iwtd == [readend] + assert iwtd == [readend] # there is more expected data = readend.recv(4096) assert len(data) > 0 assert data == b'x' * len(data) total_in += len(data) assert total_in == total_out + iwtd, owtd, ewtd = select.select([readend], [], [], 0) + assert owtd == ewtd == [] + assert iwtd == [] # there is not more expected finally: writeend.close() readend.close() From noreply at buildbot.pypy.org Sun Sep 28 09:10:36 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 28 Sep 2014 09:10:36 +0200 (CEST) Subject: [pypy-commit] pypy default: SetValueEx(REG_DWORD): support None and longs. Message-ID: <20140928071036.BAE101C038C@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73734:2962561471e0 Date: 2014-09-28 09:10 +0200 http://bitbucket.org/pypy/pypy/changeset/2962561471e0/ Log: SetValueEx(REG_DWORD): support None and longs. diff --git a/pypy/module/_winreg/interp_winreg.py b/pypy/module/_winreg/interp_winreg.py --- a/pypy/module/_winreg/interp_winreg.py +++ b/pypy/module/_winreg/interp_winreg.py @@ -266,10 +266,16 @@ buf = None if typ == rwinreg.REG_DWORD: - if space.isinstance_w(w_value, space.w_int): + if space.is_none(w_value) or ( + space.isinstance_w(w_value, space.w_int) or + space.isinstance_w(w_value, space.w_long)): + if space.is_none(w_value): + value = r_uint(0) + else: + value = space.c_uint_w(w_value) buflen = rffi.sizeof(rwin32.DWORD) buf1 = lltype.malloc(rffi.CArray(rwin32.DWORD), 1, flavor='raw') - buf1[0] = space.uint_w(w_value) + buf1[0] = value buf = rffi.cast(rffi.CCHARP, buf1) elif typ == rwinreg.REG_SZ or typ == rwinreg.REG_EXPAND_SZ: From noreply at buildbot.pypy.org Sun Sep 28 09:15:16 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 28 Sep 2014 09:15:16 +0200 (CEST) Subject: [pypy-commit] pypy default: test for 2962561471e0 Message-ID: <20140928071516.D98661C038C@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73735:83acad02ae05 Date: 2014-09-28 09:14 +0200 http://bitbucket.org/pypy/pypy/changeset/83acad02ae05/ Log: test for 2962561471e0 diff --git a/pypy/module/_winreg/test/test_winreg.py b/pypy/module/_winreg/test/test_winreg.py --- a/pypy/module/_winreg/test/test_winreg.py +++ b/pypy/module/_winreg/test/test_winreg.py @@ -40,7 +40,7 @@ cls.w_tmpfilename = space.wrap(str(udir.join('winreg-temp'))) test_data = [ - ("Int Value", 45, _winreg.REG_DWORD), + ("Int Value", 0xFEDCBA98, _winreg.REG_DWORD), ("Str Value", "A string Value", _winreg.REG_SZ), ("Unicode Value", u"A unicode Value", _winreg.REG_SZ), ("Str Expand", "The path is %path%", _winreg.REG_EXPAND_SZ), @@ -137,9 +137,11 @@ assert 0, "Did not raise" def test_SetValueEx(self): - from _winreg import CreateKey, SetValueEx, REG_BINARY + from _winreg import CreateKey, SetValueEx, REG_BINARY, REG_DWORD key = CreateKey(self.root_key, self.test_key_name) sub_key = CreateKey(key, "sub_key") + SetValueEx(sub_key, 'Int Value', 0, REG_DWORD, None) + SetValueEx(sub_key, 'Int Value', 0, REG_DWORD, 45) for name, value, type in self.test_data: SetValueEx(sub_key, name, 0, type, value) exc = raises(TypeError, SetValueEx, sub_key, 'test_name', None, From noreply at buildbot.pypy.org Sun Sep 28 16:52:26 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 28 Sep 2014 16:52:26 +0200 (CEST) Subject: [pypy-commit] stmgc default: Uh, sorry, 'design.txt' is the one I meant to commit... Message-ID: <20140928145226.D33FF1C0250@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1434:57c2c6d8f2ef Date: 2014-09-28 16:52 +0200 http://bitbucket.org/pypy/stmgc/changeset/57c2c6d8f2ef/ Log: Uh, sorry, 'design.txt' is the one I meant to commit... diff --git a/hashtable/design.txt b/hashtable/design.txt new file mode 100644 --- /dev/null +++ b/hashtable/design.txt @@ -0,0 +1,72 @@ +Goal +====== + +The goal is to have dictionaries where a read-write or write-write +conflict does not cause aborts if they occur with keys that have +different 64-bit hashes. + +(We might prefer the condition to be "on different keys even in case of +hash collision", but that's hard to achieve in general: for Python +dicts, "to be equal" involves calling the __eq__() method of objects.) + +We distinguish between reading a value associated to a key, and only +checking that the key is in the dictionary. It makes a difference if +a concurrent transaction writes to the value. + +Some operations on a dict (particularly len() and __nonzero__()) involve +the global state of the dict, and thus cause conflict with any write +that adds or removes keys. They should not cause conflicts with reads, +or with writes that only change existing values. + +In theory, we might refine this to: a len() only conflicts with a +different transaction whose net effect is to change the length (adding 3 +keys and removing 3 other keys is fine); and a __nonzero__() only +conflicts with a transaction whose net effect is to change the dict from +empty to non-empty or vice-versa. + +Iterating over the keys of a dict doesn't have to conflict with other +transactions that only change existing values. Iterating over the +values or the items conflict with other transactions doing any write at +all. + + +Model +======= + +We can use the following idea to give a theoretical model of the +above: + +Let H = {0, ... 2**64-1} be the set of possible hashes. A dictionary is +an array of length 2**64, where each item contains a "line" of zero or +more key/value pairs. We have STM read and write markers as follows: + +* for every key/value pair, we have two markers (a read and a write) on + the "value"; + +* for every line (i.e. for every possible hash value), we also have two + markers (a read and a write) on the line itself. + +Then: + +* Reading or writing the value associated with an existing key accesses + the read marker of the line, and the read or write marker of that + particular value. + +* Checking for the presence of a key only accesses the read marker of + the line. + +* Creating a new key accesses the write marker of the line (the write + marker of the newly added value is not relevant then, because other + transactions won't be able to access the line anyway). + +* Deleting a key also accesses the write marker of the line. (We cannot + do it by pretending the write the value NULL, so accessing only the + write marker of the value, because then it wouldn't conflict with + another transaction that checks for the presence of the key by + accessing only the read marker of the line.) + +* Global operations, like getting the list of keys, work by mass-marking + all the lines in H (all 2**64 of them, so obviously it needs to be + special-cased in the implementation). More precisely, len(), keys(), + etc., sets all the lines' read markers; clear() sets all the lines' + write markers. diff --git a/hashtable/design2.txt b/hashtable/design2.txt deleted file mode 100644 --- a/hashtable/design2.txt +++ /dev/null @@ -1,124 +0,0 @@ -Goal -====== - -The goal is to have dictionaries where a read-write or write-write -conflict does not cause aborts if they occur with keys that have -different 64-bit hashes. - -(We might prefer the condition to be "on different keys even in case of -hash collision", but that's hard to achieve in general: for Python -dicts, "to be equal" involves calling the __eq__() method of objects.) - -We distinguish between reading a value associated to a key, and only -checking that the key is in the dictionary. It makes a difference if -a concurrent transaction writes to the value. - -Some operations on a dict (particularly len() and __nonzero__()) involve -the global state of the dict, and thus cause conflict with any write -that adds or removes keys. They should not cause conflicts with reads, -or with writes that only change existing values. - -In theory, we might refine this to: a len() only conflicts with a -different transaction whose net effect is to change the length (adding 3 -keys and removing 3 other keys is fine); and a __nonzero__() only -conflicts with a transaction whose net effect is to change the dict from -empty to non-empty or vice-versa. The latter is probably more important -than the former, so we'll ignore the former. - -Iterating over the keys of a dict doesn't have to conflict with other -transactions that only change existing values. Iterating over the -values or the items conflict with other transactions doing any write at -all. - - -Idea -====== - -A dict is implemented used two distinct parts: the committed part, -and the uncommitted one. Each part is optimized differently. - - -Committed part --------------- - -The committed part uses separate chaining with linked lists. It is an -array of pointers of length some power of two. From the hash, we access -item (hash & (power_of_two - 1)). We get a pointer to some Entry -object, with fields "hash", "key", "value", and "next". The "hash" -field stored in the Entry objects is the full 64-bit hash. The "next" -field might point to more Entry objects. - -This whole structure is only modified during commit, by special code not -subject to the normal STM rules. There is only one writer, the -transaction currently trying to commit; but we need to be careful so that -concurrent reads work as expected. - -For the sequel, the committed part works theoretically like an array of -length 2**64, indexed by the hash, where each item contains zero of more -Entry objects with that hash value. - - -Uncommitted part ----------------- - -For the uncommitted part we can use a hash table similar to the one used -for RPython dicts, with open addressing. We import data from the -committed part to this uncommitted part when needed (at the granularity -of a 64-bit hash value). More precisely, the uncommitted part can be in -one of these states: - -* It can be a freshly created dictionary, with no committed part yet. - That's the easy case: the uncommitted hash table is all we need. - -* Or, we have a committed part, and we have imported from it - zero or more 64-bit hash values. We need to remember which ones. - That includes the imports that yielded zero key/value pairs. For each - imported hash value, we make (zero or more) entries in the uncommitted - part where we copy the key, but where the value is initially missing. - The value is copied lazily, with another lookup that will mark the - Entry object as "read" in the usual STM sense. - -* We may have additionally imported the "emptiness" or "non-emptiness" - of the committed part. - -* Last option: the current transaction is depending on the exact set - of committed keys. We no longer need to remember which ones - individually. This state is equivalent to having imported *all* - possible 64-bit hash values. - - -Commit time ------------ - -At commit time, we need to do these extra steps. The points followed by -(*) need to be done carefully because concurrent threads might be -reading the same data. - -* First, we do the usual STM validation. It will detect read-write - and write-write conflicts on existing values thanks to the read and - write markers xxx - -* We validate the keys: for every imported hash value, we check that - importing it now would give us the same answer as it did previously - (i.e. the committed table has got the same set of keys with this - particular hash as it did previously). - -* For key/value pairs that have been newly added by the current - transaction, the validation above is enough too: to add a key/value - pair, we must have imported all Entries with the same hash anyway. - So at this point we only need to create and attach(*) new Entry - objects for new key/value pairs. - - -xxxxxxxxxxx - - - -* First, we create new Entry objects for all key/value pairs that - are created by the current transaction. - -* First, we create or update the Entry objects for all key/value - pairs that have been modified by the current transaction. We - store the new ones by carefully changing the array of pointers. - -* From noreply at buildbot.pypy.org Sun Sep 28 17:01:28 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 28 Sep 2014 17:01:28 +0200 (CEST) Subject: [pypy-commit] stmgc default: update Message-ID: <20140928150128.9EDBD1C0250@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1435:cae98ae2308a Date: 2014-09-28 16:56 +0200 http://bitbucket.org/pypy/stmgc/changeset/cae98ae2308a/ Log: update diff --git a/hashtable/design.txt b/hashtable/design.txt --- a/hashtable/design.txt +++ b/hashtable/design.txt @@ -68,5 +68,7 @@ * Global operations, like getting the list of keys, work by mass-marking all the lines in H (all 2**64 of them, so obviously it needs to be special-cased in the implementation). More precisely, len(), keys(), - etc., sets all the lines' read markers; clear() sets all the lines' - write markers. + clear(), etc., set all the lines' read markers; clear() additionally + sets all the non-empty lines' write markers (so that it doesn't conflict + with another transaction checking that some key is really not in the + dict). From noreply at buildbot.pypy.org Sun Sep 28 18:16:55 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 28 Sep 2014 18:16:55 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: Translated, lldebug builds occasionally crash on 32-bit when seeing Message-ID: <20140928161655.282851C02ED@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: gc_no_cleanup_nursery Changeset: r73736:e1ccb206cbe8 Date: 2014-09-28 18:16 +0200 http://bitbucket.org/pypy/pypy/changeset/e1ccb206cbe8/ Log: Translated, lldebug builds occasionally crash on 32-bit when seeing the ">> 32". This is not an issue in this particular case, but fix it anyway. diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py --- a/rpython/jit/metainterp/optimizeopt/intbounds.py +++ b/rpython/jit/metainterp/optimizeopt/intbounds.py @@ -24,6 +24,8 @@ return (1 << ((byte_size << 3) - 1)) - 1 +IS_64_BIT = sys.maxint > 2**32 + def next_pow2_m1(n): """Calculate next power of 2 greater than n minus one.""" n |= n >> 1 @@ -31,7 +33,8 @@ n |= n >> 4 n |= n >> 8 n |= n >> 16 - n |= n >> 32 + if IS_64_BIT: + n |= n >> 32 return n From noreply at buildbot.pypy.org Sun Sep 28 19:09:09 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 28 Sep 2014 19:09:09 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: fix Message-ID: <20140928170909.930481C0250@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: gc_no_cleanup_nursery Changeset: r73737:bf3432da552c Date: 2014-09-28 18:45 +0200 http://bitbucket.org/pypy/pypy/changeset/bf3432da552c/ Log: fix diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py --- a/rpython/jit/metainterp/optimizeopt/virtualize.py +++ b/rpython/jit/metainterp/optimizeopt/virtualize.py @@ -254,7 +254,10 @@ for i in range(self.getlength()): itemvalue = self.get_item_value(i) if itemvalue is not None: - itemboxes.append(itemvalue.get_key_box()) + box = itemvalue.get_key_box() + else: + box = None + itemboxes.append(box) visitor.register_virtual_fields(self.keybox, itemboxes) for i in range(self.getlength()): itemvalue = self.get_item_value(i) @@ -280,9 +283,10 @@ def get_item_value(self, i): """Return the i'th item, unless it is 'constvalue' on a 'clear' - array. In that case, return None. The idea is that this - method returns the value that must be set into an array that - was allocated with zero=True if 'clear' is True.""" + array. In that case (or if the i'th item is already None), + return None. The idea is that this method returns the value + that must be set into an array that was allocated "correctly", + i.e. if 'clear' is True, that means with zero=True.""" subvalue = self._items[i] if self.clear and subvalue is self.constvalue: subvalue = None diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py --- a/rpython/jit/metainterp/resume.py +++ b/rpython/jit/metainterp/resume.py @@ -562,16 +562,19 @@ # NB. the check for the kind of array elements is moved out of the loop if arraydescr.is_array_of_pointers(): for i in range(length): - decoder.setarrayitem_ref(array, i, self.fieldnums[i], - arraydescr) + num = self.fieldnums[i] + if not tagged_eq(num, UNINITIALIZED): + decoder.setarrayitem_ref(array, i, num, arraydescr) elif arraydescr.is_array_of_floats(): for i in range(length): - decoder.setarrayitem_float(array, i, self.fieldnums[i], - arraydescr) + num = self.fieldnums[i] + if not tagged_eq(num, UNINITIALIZED): + decoder.setarrayitem_float(array, i, num, arraydescr) else: for i in range(length): - decoder.setarrayitem_int(array, i, self.fieldnums[i], - arraydescr) + num = self.fieldnums[i] + if not tagged_eq(num, UNINITIALIZED): + decoder.setarrayitem_int(array, i, num, arraydescr) return array def debug_prints(self): From noreply at buildbot.pypy.org Sun Sep 28 19:09:10 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 28 Sep 2014 19:09:10 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: A failing test (couldn't get one without the extra check in Message-ID: <20140928170910.C47411C0250@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: gc_no_cleanup_nursery Changeset: r73738:bd4b98bd22c8 Date: 2014-09-28 18:54 +0200 http://bitbucket.org/pypy/pypy/changeset/bd4b98bd22c8/ Log: A failing test (couldn't get one without the extra check in get_item_value(), but something similar is what occurs in a translated pypy) diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py --- a/rpython/jit/metainterp/optimizeopt/virtualize.py +++ b/rpython/jit/metainterp/optimizeopt/virtualize.py @@ -284,7 +284,8 @@ method returns the value that must be set into an array that was allocated with zero=True if 'clear' is True.""" subvalue = self._items[i] - if self.clear and subvalue is self.constvalue: + if self.clear and (subvalue is self.constvalue or + subvalue.is_null()): subvalue = None return subvalue diff --git a/rpython/jit/metainterp/test/test_virtual.py b/rpython/jit/metainterp/test/test_virtual.py --- a/rpython/jit/metainterp/test/test_virtual.py +++ b/rpython/jit/metainterp/test/test_virtual.py @@ -707,6 +707,25 @@ return node[0] + node[1] assert self.meta_interp(f, [40]) == f(40) + def test_virtual_array_with_nulls(self): + class Foo: + pass + myjitdriver = JitDriver(greens=[], reds=['n', 'node']) + def f(n): + node = [None, Foo()] + while n > 0: + myjitdriver.can_enter_jit(n=n, node=node) + myjitdriver.jit_merge_point(n=n, node=node) + newnode = [None] * 2 + if (n >> 3) & 1: + newnode[1] = node[1] + else: + newnode[1] = node[1] + node = newnode + n -= 1 + return 42 + assert self.meta_interp(f, [40]) == 42 + def test_this_doesnt_force1(self): mydriver = JitDriver(reds=['i', 'j'], greens=[]) def f(): From noreply at buildbot.pypy.org Sun Sep 28 19:09:11 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 28 Sep 2014 19:09:11 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: Merge heads, and fix the test Message-ID: <20140928170911.E96301C0250@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: gc_no_cleanup_nursery Changeset: r73739:baa552df4768 Date: 2014-09-28 19:04 +0200 http://bitbucket.org/pypy/pypy/changeset/baa552df4768/ Log: Merge heads, and fix the test diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py --- a/rpython/jit/metainterp/optimizeopt/optimizer.py +++ b/rpython/jit/metainterp/optimizeopt/optimizer.py @@ -257,6 +257,9 @@ def setinteriorfield(self, index, ofs, value): raise NotImplementedError + def get_missing_null_value(self): + raise NotImplementedError # only for VArrayValue + class ConstantValue(OptValue): def __init__(self, box): diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py --- a/rpython/jit/metainterp/optimizeopt/virtualize.py +++ b/rpython/jit/metainterp/optimizeopt/virtualize.py @@ -254,7 +254,10 @@ for i in range(self.getlength()): itemvalue = self.get_item_value(i) if itemvalue is not None: - itemboxes.append(itemvalue.get_key_box()) + box = itemvalue.get_key_box() + else: + box = None + itemboxes.append(box) visitor.register_virtual_fields(self.keybox, itemboxes) for i in range(self.getlength()): itemvalue = self.get_item_value(i) @@ -278,11 +281,15 @@ def getlength(self): return len(self._items) + def get_missing_null_value(self): + return self.constvalue + def get_item_value(self, i): """Return the i'th item, unless it is 'constvalue' on a 'clear' - array. In that case, return None. The idea is that this - method returns the value that must be set into an array that - was allocated with zero=True if 'clear' is True.""" + array. In that case (or if the i'th item is already None), + return None. The idea is that this method returns the value + that must be set into an array that was allocated "correctly", + i.e. if 'clear' is True, that means with zero=True.""" subvalue = self._items[i] if self.clear and (subvalue is self.constvalue or subvalue.is_null()): diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py --- a/rpython/jit/metainterp/optimizeopt/virtualstate.py +++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py @@ -535,9 +535,14 @@ self.info[box] = info = value.visitor_dispatch_virtual_type(self) if value.is_virtual(): flds = self.fieldboxes[box] - info.fieldstate = [self.state(b) for b in flds] + info.fieldstate = [self.state_or_none(b, value) for b in flds] return info + def state_or_none(self, box, value): + if box is None: + box = value.get_missing_null_value().box + return self.state(box) + def get_virtual_state(self, jump_args): self.optimizer.force_at_end_of_preamble() already_forced = {} diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py --- a/rpython/jit/metainterp/resume.py +++ b/rpython/jit/metainterp/resume.py @@ -562,16 +562,19 @@ # NB. the check for the kind of array elements is moved out of the loop if arraydescr.is_array_of_pointers(): for i in range(length): - decoder.setarrayitem_ref(array, i, self.fieldnums[i], - arraydescr) + num = self.fieldnums[i] + if not tagged_eq(num, UNINITIALIZED): + decoder.setarrayitem_ref(array, i, num, arraydescr) elif arraydescr.is_array_of_floats(): for i in range(length): - decoder.setarrayitem_float(array, i, self.fieldnums[i], - arraydescr) + num = self.fieldnums[i] + if not tagged_eq(num, UNINITIALIZED): + decoder.setarrayitem_float(array, i, num, arraydescr) else: for i in range(length): - decoder.setarrayitem_int(array, i, self.fieldnums[i], - arraydescr) + num = self.fieldnums[i] + if not tagged_eq(num, UNINITIALIZED): + decoder.setarrayitem_int(array, i, num, arraydescr) return array def debug_prints(self): From noreply at buildbot.pypy.org Sun Sep 28 20:47:36 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 28 Sep 2014 20:47:36 +0200 (CEST) Subject: [pypy-commit] pypy gc_no_cleanup_nursery: Close branch, ready for merge Message-ID: <20140928184736.BD2AE1C0250@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: gc_no_cleanup_nursery Changeset: r73740:0e3b6b49f5e1 Date: 2014-09-28 20:41 +0200 http://bitbucket.org/pypy/pypy/changeset/0e3b6b49f5e1/ Log: Close branch, ready for merge From noreply at buildbot.pypy.org Sun Sep 28 20:47:40 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 28 Sep 2014 20:47:40 +0200 (CEST) Subject: [pypy-commit] pypy default: hg merge gc_no_cleanup_nursery Message-ID: <20140928184740.5132A1C0250@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73741:9e2f7a37c1e2 Date: 2014-09-28 20:46 +0200 http://bitbucket.org/pypy/pypy/changeset/9e2f7a37c1e2/ Log: hg merge gc_no_cleanup_nursery Merge the 'gc_no_cleanup_nursery' branch, started by from Wenzhu Man (SoC'14) and then done by fijal. It removes the clearing of the nursery. The drawback is that new objects are not automatically filled with zeros any longer, which needs some care, mostly for GC references (which the GC tries to follow, so they must not contain garbage). The benefit is a quite large speed-up; I've heard about 10%, but we'll see more precisely on the benchmarks. diff too long, truncating to 2000 out of 4325 lines diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py --- a/pypy/module/micronumpy/test/test_zjit.py +++ b/pypy/module/micronumpy/test/test_zjit.py @@ -617,7 +617,7 @@ 'raw_store': 1, 'same_as': 2, 'setarrayitem_gc': 8, - 'setfield_gc': 21, + 'setfield_gc': 22, }) def define_argsort(): diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py --- a/pypy/module/pypyjit/test_pypy_c/test_call.py +++ b/pypy/module/pypyjit/test_pypy_c/test_call.py @@ -382,12 +382,16 @@ ... p20 = force_token() p22 = new_with_vtable(...) - p24 = new_array(1, descr=) + p24 = new_array_clear(1, descr=) p26 = new_with_vtable(ConstClass(W_ListObject)) {{{ setfield_gc(p0, p20, descr=) + setfield_gc(p22, ConstPtr(null), descr=) + setfield_gc(p22, ConstPtr(null), descr=) setfield_gc(p22, 1, descr=) + setfield_gc(p22, ConstPtr(null), descr=) setfield_gc(p26, ConstPtr(ptr22), descr=) + setfield_gc(p26, ConstPtr(null), descr=) setarrayitem_gc(p24, 0, p26, descr=) setfield_gc(p22, p24, descr=) }}} diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py --- a/pypy/module/pypyjit/test_pypy_c/test_containers.py +++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py @@ -68,10 +68,13 @@ guard_no_exception(descr=...) i12 = call(ConstClass(ll_strhash), p10, descr=) p13 = new(descr=...) - p15 = new_array(8, descr=) + p15 = new_array_clear(8, descr=) setfield_gc(p13, p15, descr=) i17 = call(ConstClass(ll_dict_lookup_trampoline), p13, p10, i12, descr=) + {{{ setfield_gc(p13, 16, descr=) + setfield_gc(p13, 0, descr=) + }}} guard_no_exception(descr=...) p20 = new_with_vtable(ConstClass(W_IntObject)) call(ConstClass(_ll_dict_setitem_lookup_done_trampoline), p13, p10, p20, i12, i17, descr=) diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py --- a/pypy/module/pypyjit/test_pypy_c/test_string.py +++ b/pypy/module/pypyjit/test_pypy_c/test_string.py @@ -110,9 +110,12 @@ i85 = strlen(p80) p86 = new(descr=) p88 = newstr(23) - setfield_gc(..., descr=) - setfield_gc(..., descr=) - setfield_gc(..., descr=) + {{{ + setfield_gc(p86, 0, descr=) + setfield_gc(p86, p88, descr=) + setfield_gc(p86, 23, descr=) + setfield_gc(p86, 23, descr=) + }}} call(ConstClass(ll_append_res0__stringbuilderPtr_rpy_stringPtr), p86, p80, descr=) guard_no_exception(descr=...) i89 = getfield_gc(p86, descr=) diff --git a/pypy/tool/pypyjit_child.py b/pypy/tool/pypyjit_child.py --- a/pypy/tool/pypyjit_child.py +++ b/pypy/tool/pypyjit_child.py @@ -10,10 +10,6 @@ graph = loc['graph'] interp.malloc_check = False - def returns_null(T, *args, **kwds): - return lltype.nullptr(T) - interp.heap.malloc_nonmovable = returns_null # XXX - from rpython.jit.backend.llgraph.runner import LLGraphCPU #LLtypeCPU.supports_floats = False # for now apply_jit(interp, graph, LLGraphCPU) diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py --- a/rpython/jit/backend/arm/opassembler.py +++ b/rpython/jit/backend/arm/opassembler.py @@ -25,7 +25,7 @@ from rpython.jit.backend.llsupport.descr import InteriorFieldDescr from rpython.jit.backend.llsupport.assembler import GuardToken, BaseAssembler from rpython.jit.backend.llsupport.regalloc import get_scale -from rpython.jit.metainterp.history import (Box, AbstractFailDescr, +from rpython.jit.metainterp.history import (Box, AbstractFailDescr, ConstInt, INT, FLOAT, REF) from rpython.jit.metainterp.history import TargetToken from rpython.jit.metainterp.resoperation import rop @@ -578,6 +578,7 @@ return fcond emit_op_setfield_raw = emit_op_setfield_gc + emit_op_zero_ptr_field = emit_op_setfield_gc def emit_op_getfield_gc(self, op, arglocs, regalloc, fcond): base_loc, ofs, res, size = arglocs @@ -1174,3 +1175,84 @@ self.mc.VMOV_cs(r.svfp_ip.value, arg.value) self.mc.VCVT_f32_f64(res.value, r.svfp_ip.value) return fcond + + #from ../x86/regalloc.py:1388 + def emit_op_zero_array(self, op, arglocs, regalloc, fcond): + from rpython.jit.backend.llsupport.descr import unpack_arraydescr + assert len(arglocs) == 0 + itemsize, baseofs, _ = unpack_arraydescr(op.getdescr()) + args = op.getarglist() + base_loc = regalloc.rm.make_sure_var_in_reg(args[0], args) + sibox = args[1] + if isinstance(sibox, ConstInt): + startindex_loc = None + startindex = sibox.getint() + assert startindex >= 0 + else: + startindex_loc = regalloc.rm.make_sure_var_in_reg(sibox, args) + startindex = -1 + length_box = op.getarg(2) + + # base_loc and startindex_loc are in two regs here (or they are + # immediates). Compute the dstaddr_loc, which is the raw + # address that we will pass as first argument to memset(). + # It can be in the same register as either one, but not in + # args[2], because we're still needing the latter. + dstaddr_box = TempBox() + dstaddr_loc = regalloc.rm.force_allocate_reg(dstaddr_box, [args[2]]) + if startindex >= 0: # a constant + ofs = baseofs + startindex * itemsize + reg = base_loc.value + else: + self.mc.gen_load_int(r.ip.value, itemsize) + self.mc.MLA(dstaddr_loc.value, r.ip.value, + startindex_loc.value, base_loc.value) + ofs = baseofs + reg = dstaddr_loc.value + if check_imm_arg(ofs): + self.mc.ADD_ri(dstaddr_loc.value, reg, imm=ofs) + else: + self.mc.gen_load_int(r.ip.value, ofs) + self.mc.ADD_rr(dstaddr_loc.value, reg, r.ip.value) + + if (isinstance(length_box, ConstInt) and + length_box.getint() <= 14 and # same limit as GCC + itemsize in (4, 2, 1)): + # Inline a series of STR operations, starting at 'dstaddr_loc'. + # XXX we could optimize STRB/STRH into STR, but this needs care: + # XXX it only works if startindex_loc is a constant, otherwise + # XXX we'd be doing unaligned accesses + self.mc.gen_load_int(r.ip.value, 0) + for i in range(length_box.getint()): + if itemsize == 4: + self.mc.STR_ri(r.ip.value, dstaddr_loc.value, imm=i*4) + elif itemsize == 2: + self.mc.STRH_ri(r.ip.value, dstaddr_loc.value, imm=i*2) + else: + self.mc.STRB_ri(r.ip.value, dstaddr_loc.value, imm=i*1) + + else: + if isinstance(length_box, ConstInt): + length_loc = imm(length_box.getint() * itemsize) + else: + # load length_loc in a register different than dstaddr_loc + length_loc = regalloc.rm.make_sure_var_in_reg(length_box, + [dstaddr_box]) + if itemsize > 1: + # we need a register that is different from dstaddr_loc, + # but which can be identical to length_loc (as usual, + # only if the length_box is not used by future operations) + bytes_box = TempBox() + bytes_loc = regalloc.rm.force_allocate_reg(bytes_box, + [dstaddr_box]) + self.mc.gen_load_int(r.ip.value, itemsize) + self.mc.MUL(bytes_loc.value, r.ip.value, length_loc.value) + length_box = bytes_box + length_loc = bytes_loc + # + # call memset() + regalloc.before_call() + self.simple_call_no_collect(imm(self.memset_addr), + [dstaddr_loc, imm(0), length_loc]) + regalloc.rm.possibly_free_var(length_box) + regalloc.rm.possibly_free_var(dstaddr_box) diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py --- a/rpython/jit/backend/arm/regalloc.py +++ b/rpython/jit/backend/arm/regalloc.py @@ -818,8 +818,11 @@ def prepare_op_setfield_gc(self, op, fcond): boxes = op.getarglist() + ofs, size, sign = unpack_fielddescr(op.getdescr()) + return self._prepare_op_setfield(boxes, ofs, size) + + def _prepare_op_setfield(self, boxes, ofs, size): a0, a1 = boxes - ofs, size, sign = unpack_fielddescr(op.getdescr()) base_loc = self.make_sure_var_in_reg(a0, boxes) value_loc = self.make_sure_var_in_reg(a1, boxes) ofs_size = default_imm_size if size < 8 else VMEM_imm_size @@ -832,6 +835,11 @@ prepare_op_setfield_raw = prepare_op_setfield_gc + def prepare_op_zero_ptr_field(self, op, fcond): + a0 = op.getarg(0) + ofs = op.getarg(1).getint() + return self._prepare_op_setfield([a0, ConstInt(0)], ofs, WORD) + def prepare_op_getfield_gc(self, op, fcond): a0 = op.getarg(0) ofs, size, sign = unpack_fielddescr(op.getdescr()) @@ -988,6 +996,7 @@ prepare_op_copystrcontent = void prepare_op_copyunicodecontent = void + prepare_op_zero_array = void def prepare_op_unicodelen(self, op, fcond): l0 = self.make_sure_var_in_reg(op.getarg(0)) diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py --- a/rpython/jit/backend/llgraph/runner.py +++ b/rpython/jit/backend/llgraph/runner.py @@ -225,6 +225,7 @@ 'i': 0, 'f': 0.0} + class LLGraphCPU(model.AbstractCPU): from rpython.jit.metainterp.typesystem import llhelper as ts supports_floats = True @@ -641,6 +642,11 @@ def bh_new_array(self, length, arraydescr): array = lltype.malloc(arraydescr.A, length, zero=True) + assert getkind(arraydescr.A.OF) != 'ref' # getkind crashes on structs + return lltype.cast_opaque_ptr(llmemory.GCREF, array) + + def bh_new_array_clear(self, length, arraydescr): + array = lltype.malloc(arraydescr.A, length, zero=True) return lltype.cast_opaque_ptr(llmemory.GCREF, array) def bh_classof(self, struct): diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py --- a/rpython/jit/backend/llsupport/assembler.py +++ b/rpython/jit/backend/llsupport/assembler.py @@ -1,5 +1,5 @@ from rpython.jit.backend.llsupport import jitframe -from rpython.jit.backend.llsupport.memcpy import memcpy_fn +from rpython.jit.backend.llsupport.memcpy import memcpy_fn, memset_fn from rpython.jit.backend.llsupport.symbolic import WORD from rpython.jit.metainterp.history import (INT, REF, FLOAT, JitCellToken, ConstInt, BoxInt, AbstractFailDescr) @@ -63,6 +63,7 @@ def __init__(self, cpu, translate_support_code=False): self.cpu = cpu self.memcpy_addr = 0 + self.memset_addr = 0 self.rtyper = cpu.rtyper self._debug = False @@ -79,6 +80,7 @@ else: self.gc_size_of_header = WORD # for tests self.memcpy_addr = self.cpu.cast_ptr_to_int(memcpy_fn) + self.memset_addr = self.cpu.cast_ptr_to_int(memset_fn) self._build_failure_recovery(False, withfloats=False) self._build_failure_recovery(True, withfloats=False) self._build_wb_slowpath(False) diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py --- a/rpython/jit/backend/llsupport/descr.py +++ b/rpython/jit/backend/llsupport/descr.py @@ -35,9 +35,11 @@ size = 0 # help translation tid = llop.combine_ushort(lltype.Signed, 0, 0) - def __init__(self, size, count_fields_if_immut=-1): + def __init__(self, size, count_fields_if_immut=-1, + gc_fielddescrs=None): self.size = size self.count_fields_if_immut = count_fields_if_immut + self.gc_fielddescrs = gc_fielddescrs def count_fields_if_immutable(self): return self.count_fields_if_immut @@ -58,10 +60,13 @@ except KeyError: size = symbolic.get_size(STRUCT, gccache.translate_support_code) count_fields_if_immut = heaptracker.count_fields_if_immutable(STRUCT) + gc_fielddescrs = heaptracker.gc_fielddescrs(gccache, STRUCT) if heaptracker.has_gcstruct_a_vtable(STRUCT): - sizedescr = SizeDescrWithVTable(size, count_fields_if_immut) + sizedescr = SizeDescrWithVTable(size, count_fields_if_immut, + gc_fielddescrs) else: - sizedescr = SizeDescr(size, count_fields_if_immut) + sizedescr = SizeDescr(size, count_fields_if_immut, + gc_fielddescrs) gccache.init_size_descr(STRUCT, sizedescr) cache[STRUCT] = sizedescr return sizedescr @@ -95,6 +100,9 @@ self.field_size = field_size self.flag = flag + def __repr__(self): + return 'FieldDescr<%s>' % (self.name,) + def is_pointer_field(self): return self.flag == FLAG_POINTER diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py --- a/rpython/jit/backend/llsupport/gc.py +++ b/rpython/jit/backend/llsupport/gc.py @@ -18,10 +18,12 @@ from rpython.jit.backend.llsupport.descr import get_call_descr from rpython.jit.backend.llsupport.rewrite import GcRewriterAssembler from rpython.memory.gctransform import asmgcroot +from rpython.jit.codewriter.effectinfo import EffectInfo # ____________________________________________________________ class GcLLDescription(GcCache): + malloc_zero_filled = True def __init__(self, gcdescr, translator=None, rtyper=None): GcCache.__init__(self, translator is not None, rtyper) @@ -36,6 +38,8 @@ def _setup_str(self): self.str_descr = get_array_descr(self, rstr.STR) self.unicode_descr = get_array_descr(self, rstr.UNICODE) + self.str_hash_descr = get_field_descr(self, rstr.STR, 'hash') + self.unicode_hash_descr = get_field_descr(self, rstr.UNICODE, 'hash') def generate_function(self, funcname, func, ARGS, RESULT=llmemory.GCREF): """Generates a variant of malloc with the given name and the given @@ -118,7 +122,8 @@ descrs = JitFrameDescrs() descrs.arraydescr = cpu.arraydescrof(jitframe.JITFRAME) for name in ['jf_descr', 'jf_guard_exc', 'jf_force_descr', - 'jf_frame_info', 'jf_gcmap', 'jf_extra_stack_depth']: + 'jf_frame_info', 'jf_gcmap', 'jf_extra_stack_depth', + 'jf_savedata', 'jf_forward']: setattr(descrs, name, cpu.fielddescrof(jitframe.JITFRAME, name)) descrs.jfi_frame_size = cpu.fielddescrof(jitframe.JITFRAMEINFO, 'jfi_frame_size') @@ -377,6 +382,7 @@ from rpython.memory.gcheader import GCHeaderBuilder self.GCClass = self.layoutbuilder.GCClass self.moving_gc = self.GCClass.moving_gc + self.malloc_zero_filled = self.GCClass.malloc_zero_filled self.HDRPTR = lltype.Ptr(self.GCClass.HDR) self.gcheaderbuilder = GCHeaderBuilder(self.HDRPTR.TO) self.max_size_of_young_obj = self.GCClass.JIT_max_size_of_young_obj() @@ -410,9 +416,9 @@ if self.DEBUG: self._random_usage_of_xmm_registers() type_id = rffi.cast(llgroup.HALFWORD, 0) # missing here - return llop1.do_malloc_fixedsize_clear(llmemory.GCREF, - type_id, size, - False, False, False) + return llop1.do_malloc_fixedsize(llmemory.GCREF, + type_id, size, + False, False, False) self.generate_function('malloc_nursery', malloc_nursery_slowpath, [lltype.Signed]) @@ -455,7 +461,7 @@ def malloc_str(length): type_id = llop.extract_ushort(llgroup.HALFWORD, str_type_id) - return llop1.do_malloc_varsize_clear( + return llop1.do_malloc_varsize( llmemory.GCREF, type_id, length, str_basesize, str_itemsize, str_ofs_length) @@ -464,7 +470,7 @@ def malloc_unicode(length): type_id = llop.extract_ushort(llgroup.HALFWORD, unicode_type_id) - return llop1.do_malloc_varsize_clear( + return llop1.do_malloc_varsize( llmemory.GCREF, type_id, length, unicode_basesize, unicode_itemsize, unicode_ofs_length) diff --git a/rpython/jit/backend/llsupport/jitframe.py b/rpython/jit/backend/llsupport/jitframe.py --- a/rpython/jit/backend/llsupport/jitframe.py +++ b/rpython/jit/backend/llsupport/jitframe.py @@ -45,8 +45,9 @@ # detailed explanation how it is on your architecture def jitframe_allocate(frame_info): - frame = lltype.malloc(JITFRAME, frame_info.jfi_frame_depth, zero=True) + frame = lltype.malloc(JITFRAME, frame_info.jfi_frame_depth) frame.jf_frame_info = frame_info + frame.jf_extra_stack_depth = 0 return frame def jitframe_resolve(frame): diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py --- a/rpython/jit/backend/llsupport/llmodel.py +++ b/rpython/jit/backend/llsupport/llmodel.py @@ -14,6 +14,7 @@ get_call_descr, get_interiorfield_descr, FieldDescr, ArrayDescr, CallDescr, InteriorFieldDescr, FLAG_POINTER, FLAG_FLOAT) +from rpython.jit.backend.llsupport.memcpy import memset_fn from rpython.jit.backend.llsupport.asmmemmgr import AsmMemoryManager from rpython.rlib.unroll import unrolling_iterable @@ -607,6 +608,7 @@ def bh_new_array(self, length, arraydescr): return self.gc_ll_descr.gc_malloc_array(length, arraydescr) + bh_new_array_clear = bh_new_array def bh_newstr(self, length): return self.gc_ll_descr.gc_malloc_str(length) diff --git a/rpython/jit/backend/llsupport/memcpy.py b/rpython/jit/backend/llsupport/memcpy.py --- a/rpython/jit/backend/llsupport/memcpy.py +++ b/rpython/jit/backend/llsupport/memcpy.py @@ -3,3 +3,6 @@ memcpy_fn = rffi.llexternal('memcpy', [llmemory.Address, llmemory.Address, rffi.SIZE_T], lltype.Void, sandboxsafe=True, _nowrapper=True) +memset_fn = rffi.llexternal('memset', [llmemory.Address, rffi.INT, + rffi.SIZE_T], lltype.Void, + sandboxsafe=True, _nowrapper=True) diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -1,12 +1,13 @@ from rpython.rlib import rgc from rpython.rlib.rarithmetic import ovfcheck -from rpython.rtyper.lltypesystem import llmemory +from rpython.rtyper.lltypesystem import llmemory, lltype from rpython.jit.metainterp import history -from rpython.jit.metainterp.history import ConstInt, BoxPtr, ConstPtr +from rpython.jit.metainterp.history import ConstInt, BoxPtr, ConstPtr, BoxInt from rpython.jit.metainterp.resoperation import ResOperation, rop from rpython.jit.codewriter import heaptracker from rpython.jit.backend.llsupport.symbolic import WORD -from rpython.jit.backend.llsupport.descr import SizeDescr, ArrayDescr +from rpython.jit.backend.llsupport.descr import SizeDescr, ArrayDescr,\ + FLAG_POINTER from rpython.jit.metainterp.history import JitCellToken FLAG_ARRAY = 0 @@ -38,6 +39,7 @@ _op_malloc_nursery = None _v_last_malloced_nursery = None c_zero = ConstInt(0) + c_null = ConstPtr(lltype.nullptr(llmemory.GCREF.TO)) def __init__(self, gc_ll_descr, cpu): self.gc_ll_descr = gc_ll_descr @@ -45,6 +47,8 @@ self.newops = [] self.known_lengths = {} self.write_barrier_applied = {} + self.delayed_zero_setfields = {} + self.delayed_zero_setarrayitems = {} def rewrite(self, operations): # we can only remember one malloc since the next malloc can possibly @@ -60,6 +64,8 @@ if op.is_malloc(): self.handle_malloc_operation(op) continue + if op.is_guard(): + self.emit_pending_zeros() elif op.can_malloc(): self.emitting_an_operation_that_can_collect() elif op.getopnum() == rop.LABEL: @@ -68,6 +74,7 @@ # ---------- write barriers ---------- if self.gc_ll_descr.write_barrier_descr is not None: if op.getopnum() == rop.SETFIELD_GC: + self.consider_setfield_gc(op) self.handle_write_barrier_setfield(op) continue if op.getopnum() == rop.SETINTERIORFIELD_GC: @@ -76,10 +83,18 @@ if op.getopnum() == rop.SETARRAYITEM_GC: self.handle_write_barrier_setarrayitem(op) continue + else: + # this is dead code, but in case we have a gc that does + # not have a write barrier and does not zero memory, we would + # need to clal it + if op.getopnum() == rop.SETFIELD_GC: + self.consider_setfield_gc(op) # ---------- call assembler ----------- if op.getopnum() == rop.CALL_ASSEMBLER: self.handle_call_assembler(op) continue + if op.getopnum() == rop.JUMP or op.getopnum() == rop.FINISH: + self.emit_pending_zeros() # self.newops.append(op) return self.newops @@ -99,7 +114,7 @@ [op.result, ConstInt(classint)], None, descr=self.gc_ll_descr.fielddescr_vtable) self.newops.append(op) - elif opnum == rop.NEW_ARRAY: + elif opnum == rop.NEW_ARRAY or opnum == rop.NEW_ARRAY_CLEAR: descr = op.getdescr() assert isinstance(descr, ArrayDescr) self.handle_new_array(descr, op) @@ -112,6 +127,44 @@ else: raise NotImplementedError(op.getopname()) + def clear_gc_fields(self, descr, result): + if self.gc_ll_descr.malloc_zero_filled: + return + try: + d = self.delayed_zero_setfields[result] + except KeyError: + d = {} + self.delayed_zero_setfields[result] = d + for fielddescr in descr.gc_fielddescrs: + ofs = self.cpu.unpack_fielddescr(fielddescr) + d[ofs] = None + + def consider_setfield_gc(self, op): + offset = self.cpu.unpack_fielddescr(op.getdescr()) + try: + del self.delayed_zero_setfields[op.getarg(0)][offset] + except KeyError: + pass + + def clear_varsize_gc_fields(self, kind, descr, result, v_length, opnum): + if self.gc_ll_descr.malloc_zero_filled: + return + if kind == FLAG_ARRAY: + if descr.is_array_of_structs() or descr.is_array_of_pointers(): + assert opnum == rop.NEW_ARRAY_CLEAR + if opnum == rop.NEW_ARRAY_CLEAR: + self.handle_clear_array_contents(descr, result, v_length) + return + if kind == FLAG_STR: + hash_descr = self.gc_ll_descr.str_hash_descr + elif kind == FLAG_UNICODE: + hash_descr = self.gc_ll_descr.unicode_hash_descr + else: + return + op = ResOperation(rop.SETFIELD_GC, [result, self.c_zero], None, + descr=hash_descr) + self.newops.append(op) + def handle_new_fixedsize(self, descr, op): assert isinstance(descr, SizeDescr) size = descr.size @@ -119,6 +172,7 @@ self.gen_initialize_tid(op.result, descr.tid) else: self.gen_malloc_fixedsize(size, descr.tid, op.result) + self.clear_gc_fields(descr, op.result) def handle_new_array(self, arraydescr, op, kind=FLAG_ARRAY): v_length = op.getarg(0) @@ -140,6 +194,8 @@ # might end up being allocated by malloc_external or some # stuff that initializes GC header fields differently self.gen_initialize_len(op.result, v_length, arraydescr.lendescr) + self.clear_varsize_gc_fields(kind, op.getdescr(), op.result, + v_length, op.getopnum()) return if (total_size >= 0 and self.gen_malloc_nursery(total_size, op.result)): @@ -149,7 +205,7 @@ self.gen_boehm_malloc_array(arraydescr, v_length, op.result) else: opnum = op.getopnum() - if opnum == rop.NEW_ARRAY: + if opnum == rop.NEW_ARRAY or opnum == rop.NEW_ARRAY_CLEAR: self.gen_malloc_array(arraydescr, v_length, op.result) elif opnum == rop.NEWSTR: self.gen_malloc_str(v_length, op.result) @@ -157,6 +213,21 @@ self.gen_malloc_unicode(v_length, op.result) else: raise NotImplementedError(op.getopname()) + self.clear_varsize_gc_fields(kind, op.getdescr(), op.result, v_length, + op.getopnum()) + + def handle_clear_array_contents(self, arraydescr, v_arr, v_length=None): + # XXX more work here to reduce or remove the ZERO_ARRAY in some cases + if v_length is None: + v_length = BoxInt() + o = ResOperation(rop.ARRAYLEN_GC, [v_arr], v_length, + descr=arraydescr) + self.newops.append(o) + elif isinstance(v_length, ConstInt) and v_length.getint() == 0: + return + o = ResOperation(rop.ZERO_ARRAY, [v_arr, self.c_zero, v_length], None, + descr=arraydescr) + self.newops.append(o) def gen_malloc_frame(self, frame_info, frame, size_box): descrs = self.gc_ll_descr.getframedescrs(self.cpu) @@ -177,10 +248,25 @@ self.gen_malloc_nursery_varsize_frame(size_box, frame) self.gen_initialize_tid(frame, descrs.arraydescr.tid) length_box = history.BoxInt() - op1 = ResOperation(rop.GETFIELD_GC, [history.ConstInt(frame_info)], - length_box, - descr=descrs.jfi_frame_depth) - self.newops.append(op1) + # we need to explicitely zero all the gc fields, because + # of the unusal malloc pattern + extra_ops = [ + ResOperation(rop.GETFIELD_GC, [history.ConstInt(frame_info)], + length_box, descr=descrs.jfi_frame_depth), + ResOperation(rop.SETFIELD_GC, [frame, self.c_zero], + None, descr=descrs.jf_extra_stack_depth), + ResOperation(rop.SETFIELD_GC, [frame, self.c_null], + None, descr=descrs.jf_savedata), + ResOperation(rop.SETFIELD_GC, [frame, self.c_null], + None, descr=descrs.jf_force_descr), + ResOperation(rop.SETFIELD_GC, [frame, self.c_null], + None, descr=descrs.jf_descr), + ResOperation(rop.SETFIELD_GC, [frame, self.c_null], + None, descr=descrs.jf_guard_exc), + ResOperation(rop.SETFIELD_GC, [frame, self.c_null], + None, descr=descrs.jf_forward), + ] + self.newops += extra_ops self.gen_initialize_len(frame, length_box, descrs.arraydescr.lendescr) @@ -225,8 +311,17 @@ # forgets the previous MALLOC_NURSERY, if any; and empty the # set 'write_barrier_applied', so that future SETFIELDs will generate # a write barrier as usual. + # it also writes down all the pending zero ptr fields self._op_malloc_nursery = None self.write_barrier_applied.clear() + self.emit_pending_zeros() + + def emit_pending_zeros(self): + for v, d in self.delayed_zero_setfields.iteritems(): + for ofs in d.iterkeys(): + op = ResOperation(rop.ZERO_PTR_FIELD, [v, ConstInt(ofs)], None) + self.newops.append(op) + self.delayed_zero_setfields.clear() def _gen_call_malloc_gc(self, args, v_result, descr): """Generate a CALL_MALLOC_GC with the given args.""" @@ -338,7 +433,8 @@ def gen_malloc_nursery(self, size, v_result): """Try to generate or update a CALL_MALLOC_NURSERY. - If that fails, generate a plain CALL_MALLOC_GC instead. + If that succeeds, return True; you still need to write the tid. + If that fails, return False. """ size = self.round_up_for_allocation(size) if not self.gc_ll_descr.can_use_nursery_malloc(size): diff --git a/rpython/jit/backend/llsupport/test/test_descr.py b/rpython/jit/backend/llsupport/test/test_descr.py --- a/rpython/jit/backend/llsupport/test/test_descr.py +++ b/rpython/jit/backend/llsupport/test/test_descr.py @@ -19,6 +19,8 @@ assert descr_t.size == symbolic.get_size(T, False) assert descr_s.count_fields_if_immutable() == -1 assert descr_t.count_fields_if_immutable() == -1 + assert descr_t.gc_fielddescrs == [] + assert len(descr_s.gc_fielddescrs) == 1 assert descr_s == get_size_descr(c0, S) assert descr_s != get_size_descr(c1, S) # @@ -26,6 +28,11 @@ assert isinstance(descr_s.size, Symbolic) assert descr_s.count_fields_if_immutable() == -1 + PARENT = lltype.Struct('P', ('x', lltype.Ptr(T))) + STRUCT = lltype.GcStruct('S', ('parent', PARENT), ('y', lltype.Ptr(T))) + descr_struct = get_size_descr(c0, STRUCT) + assert len(descr_struct.gc_fielddescrs) == 2 + def test_get_size_descr_immut(): S = lltype.GcStruct('S', hints={'immutable': True}) T = lltype.GcStruct('T', ('parent', S), diff --git a/rpython/jit/backend/llsupport/test/test_gc.py b/rpython/jit/backend/llsupport/test/test_gc.py --- a/rpython/jit/backend/llsupport/test/test_gc.py +++ b/rpython/jit/backend/llsupport/test/test_gc.py @@ -59,7 +59,7 @@ x += self.gcheaderbuilder.size_gc_header return x, tid - def do_malloc_fixedsize_clear(self, RESTYPE, type_id, size, + def do_malloc_fixedsize(self, RESTYPE, type_id, size, has_finalizer, has_light_finalizer, contains_weakptr): assert not contains_weakptr @@ -70,7 +70,9 @@ self.record.append(("fixedsize", repr(size), tid, p)) return p - def do_malloc_varsize_clear(self, RESTYPE, type_id, length, size, + do_malloc_fixedsize_clear = do_malloc_fixedsize + + def do_malloc_varsize(self, RESTYPE, type_id, length, size, itemsize, offset_to_length): p, tid = self._malloc(type_id, size + itemsize * length) (p + offset_to_length).signed[0] = length @@ -80,6 +82,8 @@ repr(offset_to_length), p)) return p + do_malloc_varsize_clear = do_malloc_varsize + def _write_barrier_failing_case(self, adr_struct): self.record.append(('barrier', adr_struct)) diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py --- a/rpython/jit/backend/llsupport/test/test_rewrite.py +++ b/rpython/jit/backend/llsupport/test/test_rewrite.py @@ -69,6 +69,8 @@ unicodedescr = self.gc_ll_descr.unicode_descr strlendescr = strdescr.lendescr unicodelendescr = unicodedescr.lendescr + strhashdescr = self.gc_ll_descr.str_hash_descr + unicodehashdescr = self.gc_ll_descr.unicode_hash_descr casmdescr = JitCellToken() clt = FakeLoopToken() @@ -82,10 +84,15 @@ jfi_frame_depth = framedescrs.jfi_frame_depth jfi_frame_size = framedescrs.jfi_frame_size jf_frame_info = framedescrs.jf_frame_info + jf_savedata = framedescrs.jf_savedata + jf_force_descr = framedescrs.jf_force_descr + jf_descr = framedescrs.jf_descr + jf_guard_exc = framedescrs.jf_guard_exc + jf_forward = framedescrs.jf_forward + jf_extra_stack_depth = framedescrs.jf_extra_stack_depth signedframedescr = self.cpu.signedframedescr floatframedescr = self.cpu.floatframedescr casmdescr.compiled_loop_token = clt - tzdescr = None # noone cares # namespace.update(locals()) # @@ -123,6 +130,9 @@ def unpack_arraydescr_size(self, d): return 0, d.itemsize, 0 + def unpack_fielddescr(self, d): + return d.offset + def arraydescrof(self, ARRAY): try: return self._cache[ARRAY] @@ -144,7 +154,7 @@ def setup_method(self, meth): class FakeCPU(BaseFakeCPU): def sizeof(self, STRUCT): - return SizeDescrWithVTable(102) + return SizeDescrWithVTable(102, gc_fielddescrs=[]) self.cpu = FakeCPU() self.gc_ll_descr = GcLLDescr_boehm(None, None, None) @@ -277,10 +287,11 @@ really_not_translated=True) self.gc_ll_descr.write_barrier_descr.has_write_barrier_from_array = ( lambda cpu: True) + self.gc_ll_descr.malloc_zero_filled = False # class FakeCPU(BaseFakeCPU): def sizeof(self, STRUCT): - descr = SizeDescrWithVTable(104) + descr = SizeDescrWithVTable(104, gc_fielddescrs=[]) descr.tid = 9315 return descr self.cpu = FakeCPU() @@ -313,6 +324,7 @@ setfield_gc(p1, 5678, descr=tiddescr) p2 = int_add(p1, %(tdescr.size)d) setfield_gc(p2, 1234, descr=tiddescr) + zero_ptr_field(p1, %(tdescr.gc_fielddescrs[0].offset)s) jump() """) @@ -422,6 +434,7 @@ [i0] p0 = call_malloc_nursery_varsize(1, 1, i0, descr=strdescr) setfield_gc(p0, i0, descr=strlendescr) + setfield_gc(p0, 0, descr=strhashdescr) jump(i0) """) @@ -545,15 +558,19 @@ unicodedescr.basesize + 10 * unicodedescr.itemsize)d) setfield_gc(p0, %(strdescr.tid)d, descr=tiddescr) setfield_gc(p0, 14, descr=strlendescr) + setfield_gc(p0, 0, descr=strhashdescr) p1 = int_add(p0, %(strdescr.basesize + 16 * strdescr.itemsize)d) setfield_gc(p1, %(unicodedescr.tid)d, descr=tiddescr) setfield_gc(p1, 10, descr=unicodelendescr) + setfield_gc(p1, 0, descr=unicodehashdescr) p2 = call_malloc_nursery_varsize(2, %(unicodedescr.itemsize)d, i2,\ descr=unicodedescr) setfield_gc(p2, i2, descr=unicodelendescr) + setfield_gc(p2, 0, descr=unicodehashdescr) p3 = call_malloc_nursery_varsize(1, 1, i2, \ descr=strdescr) setfield_gc(p3, i2, descr=strlendescr) + setfield_gc(p3, 0, descr=strhashdescr) jump() """) @@ -587,7 +604,7 @@ self.gc_ll_descr.max_size_of_young_obj = 2000 self.check_rewrite(""" [i2, p3] - p1 = new_array(129, descr=cdescr) + p1 = new_array_clear(129, descr=cdescr) call(123456) setarrayitem_gc(p1, i2, p3, descr=cdescr) jump() @@ -597,6 +614,7 @@ %(cdescr.basesize + 129 * cdescr.itemsize)d) setfield_gc(p1, 8111, descr=tiddescr) setfield_gc(p1, 129, descr=clendescr) + zero_array(p1, 0, 129, descr=cdescr) call(123456) cond_call_gc_wb(p1, descr=wbdescr) setarrayitem_gc(p1, i2, p3, descr=cdescr) @@ -608,7 +626,7 @@ self.gc_ll_descr.max_size_of_young_obj = 2000 self.check_rewrite(""" [i2, p3] - p1 = new_array(130, descr=cdescr) + p1 = new_array_clear(130, descr=cdescr) call(123456) setarrayitem_gc(p1, i2, p3, descr=cdescr) jump() @@ -618,6 +636,7 @@ %(cdescr.basesize + 130 * cdescr.itemsize)d) setfield_gc(p1, 8111, descr=tiddescr) setfield_gc(p1, 130, descr=clendescr) + zero_array(p1, 0, 130, descr=cdescr) call(123456) cond_call_gc_wb_array(p1, i2, descr=wbdescr) setarrayitem_gc(p1, i2, p3, descr=cdescr) @@ -639,7 +658,7 @@ def test_label_makes_size_unknown(self): self.check_rewrite(""" [i2, p3] - p1 = new_array(5, descr=cdescr) + p1 = new_array_clear(5, descr=cdescr) label(p1, i2, p3) setarrayitem_gc(p1, i2, p3, descr=cdescr) jump() @@ -649,6 +668,7 @@ %(cdescr.basesize + 5 * cdescr.itemsize)d) setfield_gc(p1, 8111, descr=tiddescr) setfield_gc(p1, 5, descr=clendescr) + zero_array(p1, 0, 5, descr=cdescr) label(p1, i2, p3) cond_call_gc_wb_array(p1, i2, descr=wbdescr) setarrayitem_gc(p1, i2, p3, descr=cdescr) @@ -709,7 +729,7 @@ def test_initialization_store_array(self): self.check_rewrite(""" [p1, i2] - p0 = new_array(5, descr=cdescr) + p0 = new_array_clear(5, descr=cdescr) setarrayitem_gc(p0, i2, p1, descr=cdescr) jump() """, """ @@ -718,6 +738,7 @@ %(cdescr.basesize + 5 * cdescr.itemsize)d) setfield_gc(p0, 8111, descr=tiddescr) setfield_gc(p0, 5, descr=clendescr) + zero_array(p0, 0, 5, descr=cdescr) setarrayitem_gc(p0, i2, p1, descr=cdescr) jump() """) @@ -751,9 +772,11 @@ [i0] p0 = call_malloc_nursery(%(tdescr.size)d) setfield_gc(p0, 5678, descr=tiddescr) + zero_ptr_field(p0, %(tdescr.gc_fielddescrs[0].offset)s) p1 = call_malloc_nursery_varsize(1, 1, i0, \ descr=strdescr) setfield_gc(p1, i0, descr=strlendescr) + setfield_gc(p1, 0, descr=strhashdescr) cond_call_gc_wb(p0, descr=wbdescr) setfield_gc(p0, p1, descr=tzdescr) jump() @@ -770,6 +793,7 @@ [p1] p0 = call_malloc_nursery(%(tdescr.size)d) setfield_gc(p0, 5678, descr=tiddescr) + zero_ptr_field(p0, %(tdescr.gc_fielddescrs[0].offset)s) label(p0, p1) cond_call_gc_wb(p0, descr=wbdescr) setfield_gc(p0, p1, descr=tzdescr) @@ -800,6 +824,12 @@ p1 = call_malloc_nursery_varsize_frame(i1) setfield_gc(p1, 0, descr=tiddescr) i2 = getfield_gc(ConstClass(frame_info), descr=jfi_frame_depth) + setfield_gc(p1, 0, descr=jf_extra_stack_depth) + setfield_gc(p1, NULL, descr=jf_savedata) + setfield_gc(p1, NULL, descr=jf_force_descr) + setfield_gc(p1, NULL, descr=jf_descr) + setfield_gc(p1, NULL, descr=jf_guard_exc) + setfield_gc(p1, NULL, descr=jf_forward) setfield_gc(p1, i2, descr=framelendescr) setfield_gc(p1, ConstClass(frame_info), descr=jf_frame_info) setarrayitem_gc(p1, 0, i0, descr=signedframedescr) diff --git a/rpython/jit/backend/llsupport/test/zrpy_gc_test.py b/rpython/jit/backend/llsupport/test/zrpy_gc_test.py --- a/rpython/jit/backend/llsupport/test/zrpy_gc_test.py +++ b/rpython/jit/backend/llsupport/test/zrpy_gc_test.py @@ -223,7 +223,7 @@ ## return None, f, None def define_compile_framework_1(cls): - # a moving GC. Supports malloc_varsize_nonmovable. Simple test, works + # a moving GC. Simple test, works # without write_barriers and root stack enumeration. def f(n, x, *args): y = X() diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py --- a/rpython/jit/backend/test/runner_test.py +++ b/rpython/jit/backend/test/runner_test.py @@ -2036,6 +2036,14 @@ 'ref', descr=arraydescr) assert r1.value != r2.value a = lltype.cast_opaque_ptr(lltype.Ptr(A), r1.value) + assert len(a) == 342 + + def test_new_array_clear(self): + A = lltype.GcArray(lltype.Signed) + arraydescr = self.cpu.arraydescrof(A) + r1 = self.execute_operation(rop.NEW_ARRAY_CLEAR, [BoxInt(342)], + 'ref', descr=arraydescr) + a = lltype.cast_opaque_ptr(lltype.Ptr(A), r1.value) assert a[0] == 0 assert len(a) == 342 @@ -4272,9 +4280,6 @@ fail = self.cpu.get_latest_descr(deadframe) assert fail.identifier == 23 assert self.cpu.get_int_value(deadframe, 0) == 42 - # make sure that force reads the registers from a zeroed piece of - # memory - assert values[0] == 0 def test_compile_bridge_while_running(self): def func(): @@ -4442,3 +4447,99 @@ res = self.execute_operation(rop.CAST_FLOAT_TO_SINGLEFLOAT, [boxfloat(12.5)], 'int') assert res.getint() == struct.unpack("I", struct.pack("f", 12.5))[0] + + def test_zero_ptr_field(self): + from rpython.jit.backend.llsupport.llmodel import AbstractLLCPU + + if not isinstance(self.cpu, AbstractLLCPU): + py.test.skip("llgraph can't do zero_ptr_field") + T = lltype.GcStruct('T') + S = lltype.GcStruct('S', ('x', lltype.Ptr(T))) + tdescr = self.cpu.sizeof(T) + sdescr = self.cpu.sizeof(S) + fielddescr = self.cpu.fielddescrof(S, 'x') + loop = parse(""" + [] + p0 = new(descr=tdescr) + p1 = new(descr=sdescr) + setfield_gc(p1, p0, descr=fielddescr) + zero_ptr_field(p1, %d) + finish(p1) + """ % fielddescr.offset, namespace=locals()) + looptoken = JitCellToken() + self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) + deadframe = self.cpu.execute_token(looptoken) + ref = self.cpu.get_ref_value(deadframe, 0) + s = lltype.cast_opaque_ptr(lltype.Ptr(S), ref) + assert not s.x + + def test_zero_ptr_field_2(self): + from rpython.jit.backend.llsupport.llmodel import AbstractLLCPU + + if not isinstance(self.cpu, AbstractLLCPU): + py.test.skip("llgraph does not do zero_ptr_field") + + from rpython.jit.backend.llsupport import symbolic + S = lltype.GcStruct('S', ('x', lltype.Signed), + ('p', llmemory.GCREF), + ('y', lltype.Signed)) + s = lltype.malloc(S) + s.x = -1296321 + s.y = -4398176 + s_ref = lltype.cast_opaque_ptr(llmemory.GCREF, s) + s.p = s_ref + ofs_p, _ = symbolic.get_field_token(S, 'p', False) + # + self.execute_operation(rop.ZERO_PTR_FIELD, [ + BoxPtr(s_ref), ConstInt(ofs_p)], # OK for now to assume that the + 'void') # 2nd argument is a constant + # + assert s.x == -1296321 + assert s.p == lltype.nullptr(llmemory.GCREF.TO) + assert s.y == -4398176 + + def test_zero_array(self): + from rpython.jit.backend.llsupport.llmodel import AbstractLLCPU + + if not isinstance(self.cpu, AbstractLLCPU): + py.test.skip("llgraph does not do zero_array") + + PAIR = lltype.Struct('PAIR', ('a', lltype.Signed), ('b', lltype.Signed)) + for OF in [lltype.Signed, rffi.INT, rffi.SHORT, rffi.UCHAR, PAIR]: + A = lltype.GcArray(OF) + arraydescr = self.cpu.arraydescrof(A) + a = lltype.malloc(A, 100) + addr = llmemory.cast_ptr_to_adr(a) + a_int = heaptracker.adr2int(addr) + a_ref = lltype.cast_opaque_ptr(llmemory.GCREF, a) + for (start, length) in [(0, 100), (49, 49), (1, 98), + (15, 9), (10, 10), (47, 0), + (0, 4)]: + for cls1 in [ConstInt, BoxInt]: + for cls2 in [ConstInt, BoxInt]: + print 'a_int:', a_int + print 'of:', OF + print 'start:', cls1.__name__, start + print 'length:', cls2.__name__, length + for i in range(100): + if OF == PAIR: + a[i].a = a[i].b = -123456789 + else: + a[i] = rffi.cast(OF, -123456789) + startbox = cls1(start) + lengthbox = cls2(length) + if cls1 == cls2 and start == length: + lengthbox = startbox # same box! + self.execute_operation(rop.ZERO_ARRAY, + [BoxPtr(a_ref), + startbox, + lengthbox], + 'void', descr=arraydescr) + assert len(a) == 100 + for i in range(100): + val = (0 if start <= i < start + length + else -123456789) + if OF == PAIR: + assert a[i].a == a[i].b == val + else: + assert a[i] == rffi.cast(OF, val) diff --git a/rpython/jit/backend/test/test_ll_random.py b/rpython/jit/backend/test/test_ll_random.py --- a/rpython/jit/backend/test/test_ll_random.py +++ b/rpython/jit/backend/test/test_ll_random.py @@ -95,7 +95,10 @@ fields.append(('parent', rclass.OBJECT)) kwds['hints'] = {'vtable': with_vtable._obj} for i in range(r.randrange(1, 5)): - TYPE = self.get_random_primitive_type(r) + if r.random() < 0.1: + TYPE = llmemory.GCREF + else: + TYPE = self.get_random_primitive_type(r) fields.append(('f%d' % i, TYPE)) S = type('S%d' % self.counter, *fields, **kwds) self.counter += 1 @@ -246,13 +249,43 @@ op = ResOperation(self.opnum, [v, c_vtable2], None) return op, False +class ZeroPtrFieldOperation(test_random.AbstractOperation): + def field_descr(self, builder, r): + if getattr(builder.cpu, 'is_llgraph', False): + raise test_random.CannotProduceOperation + v, S = builder.get_structptr_var(r, ) + names = S._names + if names[0] == 'parent': + names = names[1:] + choice = [] + for name in names: + FIELD = getattr(S, name) + if isinstance(FIELD, lltype.Ptr) and FIELD._needsgc(): + choice.append(name) + if not choice: + raise test_random.CannotProduceOperation + name = r.choice(choice) + descr = builder.cpu.fielddescrof(S, name) + return v, descr.offset + + def produce_into(self, builder, r): + v, offset = self.field_descr(builder, r) + builder.do(self.opnum, [v, ConstInt(offset)], None) + class GetFieldOperation(test_random.AbstractOperation): def field_descr(self, builder, r): v, S = builder.get_structptr_var(r, ) names = S._names if names[0] == 'parent': names = names[1:] - name = r.choice(names) + choice = [] + for name in names: + FIELD = getattr(S, name) + if not isinstance(FIELD, lltype.Ptr): + choice.append(name) + if not choice: + raise test_random.CannotProduceOperation + name = r.choice(choice) descr = builder.cpu.fielddescrof(S, name) descr._random_info = 'cpu.fielddescrof(..., %r)' % (name,) descr._random_type = S @@ -274,7 +307,14 @@ array_of_structs=True) array = v.getref(lltype.Ptr(A)) v_index = builder.get_index(len(array), r) - name = r.choice(A.OF._names) + choice = [] + for name in A.OF._names: + FIELD = getattr(A.OF, name) + if not isinstance(FIELD, lltype.Ptr): + choice.append(name) + if not choice: + raise test_random.CannotProduceOperation + name = r.choice(choice) descr = builder.cpu.interiorfielddescrof(A, name) descr._random_info = 'cpu.interiorfielddescrof(..., %r)' % (name,) descr._random_type = A @@ -682,6 +722,7 @@ OPERATIONS.append(GetFieldOperation(rop.GETFIELD_GC)) OPERATIONS.append(GetInteriorFieldOperation(rop.GETINTERIORFIELD_GC)) OPERATIONS.append(SetFieldOperation(rop.SETFIELD_GC)) + OPERATIONS.append(ZeroPtrFieldOperation(rop.ZERO_PTR_FIELD)) OPERATIONS.append(SetInteriorFieldOperation(rop.SETINTERIORFIELD_GC)) OPERATIONS.append(NewOperation(rop.NEW)) OPERATIONS.append(NewOperation(rop.NEW_WITH_VTABLE)) @@ -689,7 +730,7 @@ OPERATIONS.append(GetArrayItemOperation(rop.GETARRAYITEM_GC)) OPERATIONS.append(GetArrayItemOperation(rop.GETARRAYITEM_GC)) OPERATIONS.append(SetArrayItemOperation(rop.SETARRAYITEM_GC)) - OPERATIONS.append(NewArrayOperation(rop.NEW_ARRAY)) + OPERATIONS.append(NewArrayOperation(rop.NEW_ARRAY_CLEAR)) OPERATIONS.append(ArrayLenOperation(rop.ARRAYLEN_GC)) OPERATIONS.append(NewStrOperation(rop.NEWSTR)) OPERATIONS.append(NewUnicodeOperation(rop.NEWUNICODE)) diff --git a/rpython/jit/backend/test/test_random.py b/rpython/jit/backend/test/test_random.py --- a/rpython/jit/backend/test/test_random.py +++ b/rpython/jit/backend/test/test_random.py @@ -52,10 +52,13 @@ def do(self, opnum, argboxes, descr=None): self.fakemetainterp._got_exc = None - v_result = execute_nonspec(self.cpu, self.fakemetainterp, - opnum, argboxes, descr) - if isinstance(v_result, Const): - v_result = v_result.clonebox() + if opnum == rop.ZERO_PTR_FIELD: + v_result = None + else: + v_result = execute_nonspec(self.cpu, self.fakemetainterp, + opnum, argboxes, descr) + if isinstance(v_result, Const): + v_result = v_result.clonebox() self.loop.operations.append(ResOperation(opnum, argboxes, v_result, descr)) return v_result diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -1486,6 +1486,8 @@ dest_addr = AddressLoc(base_loc, ofs_loc) self.save_into_mem(dest_addr, value_loc, size_loc) + genop_discard_zero_ptr_field = genop_discard_setfield_gc + def genop_discard_setinteriorfield_gc(self, op, arglocs): (base_loc, ofs_loc, itemsize_loc, fieldsize_loc, index_loc, temp_loc, value_loc) = arglocs @@ -2361,6 +2363,43 @@ elif IS_X86_64: mc.MOVSX32_rj(loc.value, addr) # memory read, sign-extend + def genop_discard_zero_array(self, op, arglocs): + (base_loc, startindex_loc, bytes_loc, + itemsize_loc, baseofs_loc, null_loc) = arglocs + assert isinstance(bytes_loc, ImmedLoc) + assert isinstance(itemsize_loc, ImmedLoc) + assert isinstance(baseofs_loc, ImmedLoc) + assert isinstance(null_loc, RegLoc) and null_loc.is_xmm + baseofs = baseofs_loc.value + nbytes = bytes_loc.value + if valid_addressing_size(itemsize_loc.value): + scale = get_scale(itemsize_loc.value) + else: + assert isinstance(startindex_loc, ImmedLoc) + baseofs += startindex_loc.value * itemsize_loc.value + startindex_loc = imm0 + scale = 0 + null_reg_cleared = False + i = 0 + while i < nbytes: + addr = addr_add(base_loc, startindex_loc, baseofs + i, scale) + current = nbytes - i + if current >= 16: + current = 16 + if not null_reg_cleared: + self.mc.XORPS_xx(null_loc.value, null_loc.value) + null_reg_cleared = True + self.mc.MOVUPS(addr, null_loc) + else: + if current >= WORD: + current = WORD + elif current >= 4: + current = 4 + elif current >= 2: + current = 2 + self.save_into_mem(addr, imm0, imm(current)) + i += current + genop_discard_list = [Assembler386.not_implemented_op_discard] * rop._LAST genop_list = [Assembler386.not_implemented_op] * rop._LAST diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -8,7 +8,8 @@ unpack_arraydescr, unpack_fielddescr, unpack_interiorfielddescr) from rpython.jit.backend.llsupport.gcmap import allocate_gcmap from rpython.jit.backend.llsupport.regalloc import (FrameManager, BaseRegalloc, - RegisterManager, TempBox, compute_vars_longevity, is_comparison_or_ovf_op) + RegisterManager, TempBox, compute_vars_longevity, is_comparison_or_ovf_op, + valid_addressing_size) from rpython.jit.backend.x86 import rx86 from rpython.jit.backend.x86.arch import (WORD, JITFRAME_FIXED_SIZE, IS_X86_32, IS_X86_64) @@ -958,6 +959,13 @@ need_lower_byte=need_lower_byte) self.perform_discard(op, [base_loc, ofs_loc, size_loc, value_loc]) + def consider_zero_ptr_field(self, op): + ofs_loc = imm(op.getarg(1).getint()) + size_loc = imm(WORD) + base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), []) + value_loc = imm(0) + self.perform_discard(op, [base_loc, ofs_loc, size_loc, value_loc]) + consider_setfield_raw = consider_setfield_gc def consider_setinteriorfield_gc(self, op): @@ -1376,6 +1384,70 @@ def consider_keepalive(self, op): pass + def consider_zero_array(self, op): + itemsize, baseofs, _ = unpack_arraydescr(op.getdescr()) + args = op.getarglist() + base_loc = self.rm.make_sure_var_in_reg(args[0], args) + startindex_loc = self.rm.make_sure_var_in_reg(args[1], args) + length_box = op.getarg(2) + if isinstance(length_box, ConstInt): + constbytes = length_box.getint() * itemsize + else: + constbytes = -1 + if 0 <= constbytes <= 16 * 8 and ( + valid_addressing_size(itemsize) or +- isinstance(startindex_loc, ImmedLoc)): + if IS_X86_64: + null_loc = X86_64_XMM_SCRATCH_REG + else: + null_box = TempBox() + null_loc = self.xrm.force_allocate_reg(null_box) + self.xrm.possibly_free_var(null_box) + self.perform_discard(op, [base_loc, startindex_loc, + imm(constbytes), imm(itemsize), + imm(baseofs), null_loc]) + else: + # base_loc and startindex_loc are in two regs here (or they are + # immediates). Compute the dstaddr_loc, which is the raw + # address that we will pass as first argument to memset(). + # It can be in the same register as either one, but not in + # args[2], because we're still needing the latter. + dstaddr_box = TempBox() + dstaddr_loc = self.rm.force_allocate_reg(dstaddr_box, [args[2]]) + itemsize_loc = imm(itemsize) + dst_addr = self.assembler._get_interiorfield_addr( + dstaddr_loc, startindex_loc, itemsize_loc, + base_loc, imm(baseofs)) + self.assembler.mc.LEA(dstaddr_loc, dst_addr) + # + if constbytes >= 0: + length_loc = imm(constbytes) + else: + # load length_loc in a register different than dstaddr_loc + length_loc = self.rm.make_sure_var_in_reg(length_box, + [dstaddr_box]) + if itemsize > 1: + # we need a register that is different from dstaddr_loc, + # but which can be identical to length_loc (as usual, + # only if the length_box is not used by future operations) + bytes_box = TempBox() + bytes_loc = self.rm.force_allocate_reg(bytes_box, + [dstaddr_box]) + b_adr = self.assembler._get_interiorfield_addr( + bytes_loc, length_loc, itemsize_loc, imm0, imm0) + self.assembler.mc.LEA(bytes_loc, b_adr) + length_box = bytes_box + length_loc = bytes_loc + # + # call memset() + self.rm.before_call() + self.xrm.before_call() + self.assembler.simple_call_no_collect( + imm(self.assembler.memset_addr), + [dstaddr_loc, imm0, length_loc]) + self.rm.possibly_free_var(length_box) + self.rm.possibly_free_var(dstaddr_box) + def not_implemented_op(self, op): not_implemented("not implemented operation: %s" % op.getopname()) diff --git a/rpython/jit/backend/x86/regloc.py b/rpython/jit/backend/x86/regloc.py --- a/rpython/jit/backend/x86/regloc.py +++ b/rpython/jit/backend/x86/regloc.py @@ -664,6 +664,7 @@ MOVDQ = _binaryop('MOVDQ') MOVD32 = _binaryop('MOVD32') + MOVUPS = _binaryop('MOVUPS') CALL = _relative_unaryop('CALL') JMP = _relative_unaryop('JMP') diff --git a/rpython/jit/backend/x86/rx86.py b/rpython/jit/backend/x86/rx86.py --- a/rpython/jit/backend/x86/rx86.py +++ b/rpython/jit/backend/x86/rx86.py @@ -634,6 +634,9 @@ MOVD32_xs = xmminsn('\x66', rex_nw, '\x0F\x6E', register(1, 8), stack_sp(2)) PSRAD_xi = xmminsn('\x66', rex_nw, '\x0F\x72', register(1), '\xE0', immediate(2, 'b')) + MOVUPS_mx = xmminsn(rex_nw, '\x0F\x11', register(2, 8), mem_reg_plus_const(1)) + MOVUPS_jx = xmminsn(rex_nw, '\x0F\x11', register(2, 8), abs_(1)) + MOVUPS_ax = xmminsn(rex_nw, '\x0F\x11', register(2, 8), mem_reg_plus_scaled_reg_plus_const(1)) # ------------------------------------------------------------ @@ -764,6 +767,7 @@ define_modrm_modes('DIVSD_x*', ['\xF2', rex_nw, '\x0F\x5E', register(1, 8)], regtype='XMM') define_modrm_modes('UCOMISD_x*', ['\x66', rex_nw, '\x0F\x2E', register(1, 8)], regtype='XMM') define_modrm_modes('XORPD_x*', ['\x66', rex_nw, '\x0F\x57', register(1, 8)], regtype='XMM') +define_modrm_modes('XORPS_x*', [rex_nw, '\x0F\x57', register(1, 8)], regtype='XMM') define_modrm_modes('ANDPD_x*', ['\x66', rex_nw, '\x0F\x54', register(1, 8)], regtype='XMM') def define_pxmm_insn(insnname_template, insn_char): diff --git a/rpython/jit/codewriter/assembler.py b/rpython/jit/codewriter/assembler.py --- a/rpython/jit/codewriter/assembler.py +++ b/rpython/jit/codewriter/assembler.py @@ -291,6 +291,7 @@ 'int_sub', 'jit_merge_point', 'new_array', + 'new_array_clear', 'newstr', 'setarrayitem_gc_i', 'setarrayitem_gc_r', diff --git a/rpython/jit/codewriter/codewriter.py b/rpython/jit/codewriter/codewriter.py --- a/rpython/jit/codewriter/codewriter.py +++ b/rpython/jit/codewriter/codewriter.py @@ -13,7 +13,7 @@ class CodeWriter(object): callcontrol = None # for tests - debug = False + debug = True def __init__(self, cpu=None, jitdrivers_sd=[]): self.cpu = cpu diff --git a/rpython/jit/codewriter/heaptracker.py b/rpython/jit/codewriter/heaptracker.py --- a/rpython/jit/codewriter/heaptracker.py +++ b/rpython/jit/codewriter/heaptracker.py @@ -125,3 +125,19 @@ vtable = descr.as_vtable_size_descr()._corresponding_vtable vtable = llmemory.cast_ptr_to_adr(vtable) return adr2int(vtable) + +def gc_fielddescrs(gccache, STRUCT, res=None): + from rpython.jit.backend.llsupport import descr + + if res is None: + res = [] + # order is not relevant, except for tests + for name in STRUCT._names: + FIELD = getattr(STRUCT, name) + if FIELD is lltype.Void: + continue + elif isinstance(FIELD, lltype.Struct): + gc_fielddescrs(gccache, FIELD, res) + elif isinstance(FIELD, lltype.Ptr) and FIELD._needsgc(): + res.append(descr.get_field_descr(gccache, STRUCT, name)) + return res diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py --- a/rpython/jit/codewriter/jtransform.py +++ b/rpython/jit/codewriter/jtransform.py @@ -612,8 +612,43 @@ # XXX only strings or simple arrays for now ARRAY = op.args[0].value arraydescr = self.cpu.arraydescrof(ARRAY) - return SpaceOperation('new_array', [op.args[2], arraydescr], - op.result) + if op.args[1].value.get('zero', False): + opname = 'new_array_clear' + elif ((isinstance(ARRAY.OF, lltype.Ptr) and ARRAY.OF._needsgc()) or + isinstance(ARRAY.OF, lltype.Struct)): + opname = 'new_array_clear' + else: + opname = 'new_array' + return SpaceOperation(opname, [op.args[2], arraydescr], op.result) + + def zero_contents(self, ops, v, TYPE): + if isinstance(TYPE, lltype.Struct): + for name, FIELD in TYPE._flds.iteritems(): + if isinstance(FIELD, lltype.Struct): + # substruct + self.zero_contents(ops, v, FIELD) + else: + c_name = Constant(name, lltype.Void) + c_null = Constant(FIELD._defl(), FIELD) + op = SpaceOperation('setfield', [v, c_name, c_null], + None) + self.extend_with(ops, self.rewrite_op_setfield(op, + override_type=TYPE)) + elif isinstance(TYPE, lltype.Array): + assert False # this operation disappeared + else: + raise TypeError("Expected struct or array, got '%r'", (TYPE,)) + if len(ops) == 1: + return ops[0] + return ops + + def extend_with(self, l, ops): + if ops is None: + return + if isinstance(ops, list): + l.extend(ops) + else: + l.append(ops) def rewrite_op_free(self, op): d = op.args[1].value.copy() @@ -759,13 +794,17 @@ op1] return op1 - def rewrite_op_setfield(self, op): + def rewrite_op_setfield(self, op, override_type=None): if self.is_typeptr_getset(op): # ignore the operation completely -- instead, it's done by 'new' return # turn the flow graph 'setfield' operation into our own version [v_inst, c_fieldname, v_value] = op.args RESULT = v_value.concretetype + if override_type is not None: + TYPE = override_type + else: + TYPE = v_inst.concretetype.TO if RESULT is lltype.Void: return # check for virtualizable @@ -775,10 +814,12 @@ return [SpaceOperation('-live-', [], None), SpaceOperation('setfield_vable_%s' % kind, [v_inst, v_value, descr], None)] - self.check_field_access(v_inst.concretetype.TO) - argname = getattr(v_inst.concretetype.TO, '_gckind', 'gc') - descr = self.cpu.fielddescrof(v_inst.concretetype.TO, - c_fieldname.value) + self.check_field_access(TYPE) + if override_type: + argname = 'gc' + else: + argname = getattr(TYPE, '_gckind', 'gc') + descr = self.cpu.fielddescrof(TYPE, c_fieldname.value) kind = getkind(RESULT)[0] if argname == 'raw' and kind == 'r': raise Exception("setfield_raw_r not supported") @@ -860,7 +901,10 @@ if op.args[1].value['flavor'] == 'raw': return self._rewrite_raw_malloc(op, 'raw_malloc_fixedsize', []) # - assert op.args[1].value == {'flavor': 'gc'} + if op.args[1].value.get('zero', False): + zero = True + else: + zero = False STRUCT = op.args[0].value vtable = heaptracker.get_vtable_for_gcstruct(self.cpu, STRUCT) if vtable: @@ -881,7 +925,25 @@ else: opname = 'new' sizedescr = self.cpu.sizeof(STRUCT) - return SpaceOperation(opname, [sizedescr], op.result) + op1 = SpaceOperation(opname, [sizedescr], op.result) + if zero: + return self.zero_contents([op1], op.result, STRUCT) + return op1 + + def _has_gcptrs_in(self, STRUCT): + if isinstance(STRUCT, lltype.Array): + ITEM = STRUCT.OF + if isinstance(ITEM, lltype.Struct): + STRUCT = ITEM + else: + return isinstance(ITEM, lltype.Ptr) and ITEM._needsgc() + for FIELD in STRUCT._flds.values(): + if isinstance(FIELD, lltype.Ptr) and FIELD._needsgc(): + return True + elif isinstance(FIELD, lltype.Struct): + if self._has_gcptrs_in(FIELD): + return True + return False def rewrite_op_getinteriorarraysize(self, op): # only supports strings and unicodes @@ -1606,7 +1668,13 @@ v = Variable('new_length') v.concretetype = lltype.Signed ops.append(SpaceOperation('int_force_ge_zero', [v_length], v)) - ops.append(SpaceOperation('new_array', [v, arraydescr], op.result)) + ARRAY = op.result.concretetype.TO + if ((isinstance(ARRAY.OF, lltype.Ptr) and ARRAY.OF._needsgc()) or + isinstance(ARRAY.OF, lltype.Struct)): + opname = 'new_array_clear' + else: + opname = 'new_array' + ops.append(SpaceOperation(opname, [v, arraydescr], op.result)) return ops def do_fixed_list_len(self, op, args, arraydescr): diff --git a/rpython/jit/codewriter/test/test_jtransform.py b/rpython/jit/codewriter/test/test_jtransform.py --- a/rpython/jit/codewriter/test/test_jtransform.py +++ b/rpython/jit/codewriter/test/test_jtransform.py @@ -529,6 +529,35 @@ assert op1.opname == 'new' assert op1.args == [('sizedescr', S)] +def test_malloc_new_zero_2(): + S = lltype.GcStruct('S', ('x', lltype.Signed)) + v = varoftype(lltype.Ptr(S)) + op = SpaceOperation('malloc', [Constant(S, lltype.Void), + Constant({'flavor': 'gc', + 'zero': True}, lltype.Void)], v) + op1, op2 = Transformer(FakeCPU()).rewrite_operation(op) + assert op1.opname == 'new' + assert op1.args == [('sizedescr', S)] + assert op2.opname == 'setfield_gc_i' + assert op2.args[0] == v + +def test_malloc_new_zero_nested(): + S0 = lltype.GcStruct('S0') + S = lltype.Struct('S', ('x', lltype.Ptr(S0))) + S2 = lltype.GcStruct('S2', ('parent', S), + ('xx', lltype.Ptr(S0))) + v = varoftype(lltype.Ptr(S2)) + op = SpaceOperation('malloc', [Constant(S2, lltype.Void), + Constant({'flavor': 'gc', + 'zero': True}, lltype.Void)], v) + op1, op2, op3 = Transformer(FakeCPU()).rewrite_operation(op) + assert op1.opname == 'new' + assert op1.args == [('sizedescr', S2)] + assert op2.opname == 'setfield_gc_r' + assert op2.args[0] == v + assert op3.opname == 'setfield_gc_r' + assert op3.args[0] == v + def test_malloc_new_with_vtable(): vtable = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True) S = lltype.GcStruct('S', ('parent', rclass.OBJECT)) @@ -1026,6 +1055,15 @@ assert op1.args == [v1] assert op1.result == v2 +def test_malloc_varsize_zero(): + c_A = Constant(lltype.GcArray(lltype.Signed), lltype.Void) + v1 = varoftype(lltype.Signed) + v2 = varoftype(c_A.value) + c_flags = Constant({"flavor": "gc", "zero": True}, lltype.Void) + op = SpaceOperation('malloc_varsize', [c_A, c_flags, v1], v2) + op1 = Transformer(FakeCPU()).rewrite_operation(op) + assert op1.opname == 'new_array_clear' + def test_str_concat(): # test that the oopspec is present and correctly transformed PSTR = lltype.Ptr(rstr.STR) diff --git a/rpython/jit/codewriter/test/test_list.py b/rpython/jit/codewriter/test/test_list.py --- a/rpython/jit/codewriter/test/test_list.py +++ b/rpython/jit/codewriter/test/test_list.py @@ -11,6 +11,7 @@ # ____________________________________________________________ FIXEDLIST = lltype.Ptr(lltype.GcArray(lltype.Signed)) +FIXEDPTRLIST = lltype.Ptr(lltype.GcArray(FIXEDLIST)) VARLIST = lltype.Ptr(lltype.GcStruct('VARLIST', ('length', lltype.Signed), ('items', FIXEDLIST), @@ -100,6 +101,8 @@ builtin_test('newlist', [Constant(5, lltype.Signed), varoftype(lltype.Signed)], FIXEDLIST, NotSupported) + builtin_test('newlist', [], FIXEDPTRLIST, + """new_array_clear $0, -> %r0""") def test_fixed_ll_arraycopy(): builtin_test('list.ll_arraycopy', diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py --- a/rpython/jit/metainterp/blackhole.py +++ b/rpython/jit/metainterp/blackhole.py @@ -1008,7 +1008,11 @@ itemsdescr, arraydescr): result = cpu.bh_new(structdescr) cpu.bh_setfield_gc_i(result, length, lengthdescr) - items = cpu.bh_new_array(length, arraydescr) + if (arraydescr.is_array_of_structs() or + arraydescr.is_array_of_pointers()): + items = cpu.bh_new_array_clear(length, arraydescr) + else: + items = cpu.bh_new_array(length, arraydescr) cpu.bh_setfield_gc_r(result, items, itemsdescr) return result @@ -1017,7 +1021,11 @@ itemsdescr, arraydescr): result = cpu.bh_new(structdescr) cpu.bh_setfield_gc_i(result, 0, lengthdescr) - items = cpu.bh_new_array(lengthhint, arraydescr) + if (arraydescr.is_array_of_structs() or + arraydescr.is_array_of_pointers()): + items = cpu.bh_new_array_clear(lengthhint, arraydescr) + else: + items = cpu.bh_new_array(lengthhint, arraydescr) cpu.bh_setfield_gc_r(result, items, itemsdescr) return result @@ -1153,6 +1161,10 @@ def bhimpl_new_array(cpu, length, arraydescr): return cpu.bh_new_array(length, arraydescr) + @arguments("cpu", "i", "d", returns="r") + def bhimpl_new_array_clear(cpu, length, arraydescr): + return cpu.bh_new_array_clear(length, arraydescr) + @arguments("cpu", "r", "i", "d", returns="i") def bhimpl_getarrayitem_gc_i(cpu, array, index, arraydescr): return cpu.bh_getarrayitem_gc_i(array, index, arraydescr) diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py --- a/rpython/jit/metainterp/executor.py +++ b/rpython/jit/metainterp/executor.py @@ -325,6 +325,8 @@ rop.INCREMENT_DEBUG_COUNTER, rop.COND_CALL_GC_WB, rop.COND_CALL_GC_WB_ARRAY, + rop.ZERO_PTR_FIELD, + rop.ZERO_ARRAY, rop.DEBUG_MERGE_POINT, rop.JIT_DEBUG, rop.SETARRAYITEM_RAW, diff --git a/rpython/jit/metainterp/gc.py b/rpython/jit/metainterp/gc.py --- a/rpython/jit/metainterp/gc.py +++ b/rpython/jit/metainterp/gc.py @@ -26,7 +26,7 @@ malloc_zero_filled = True class GC_incminimark(GcDescription): - malloc_zero_filled = True + malloc_zero_filled = False def get_description(config): diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py --- a/rpython/jit/metainterp/optimizeopt/intbounds.py +++ b/rpython/jit/metainterp/optimizeopt/intbounds.py @@ -24,6 +24,8 @@ return (1 << ((byte_size << 3) - 1)) - 1 +IS_64_BIT = sys.maxint > 2**32 + def next_pow2_m1(n): """Calculate next power of 2 greater than n minus one.""" n |= n >> 1 @@ -31,7 +33,8 @@ n |= n >> 4 n |= n >> 8 n |= n >> 16 - n |= n >> 32 + if IS_64_BIT: + n |= n >> 32 return n diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py --- a/rpython/jit/metainterp/optimizeopt/optimizer.py +++ b/rpython/jit/metainterp/optimizeopt/optimizer.py @@ -257,6 +257,9 @@ def setinteriorfield(self, index, ofs, value): raise NotImplementedError + def get_missing_null_value(self): + raise NotImplementedError # only for VArrayValue + class ConstantValue(OptValue): def __init__(self, box): diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py --- a/rpython/jit/metainterp/optimizeopt/rewrite.py +++ b/rpython/jit/metainterp/optimizeopt/rewrite.py @@ -502,6 +502,8 @@ descr=arraydescr) self.optimizer.send_extra_operation(newop) val = self.getvalue(resbox) + if val is None: + continue if dest_value.is_virtual(): dest_value.setitem(index + dest_start, val) else: diff --git a/rpython/jit/metainterp/optimizeopt/test/test_multilabel.py b/rpython/jit/metainterp/optimizeopt/test/test_multilabel.py --- a/rpython/jit/metainterp/optimizeopt/test/test_multilabel.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_multilabel.py @@ -150,10 +150,10 @@ def test_nonmatching_arraystruct_1(self): ops = """ [p1, f0] - p2 = new_array(3, descr=complexarraydescr) + p2 = new_array_clear(3, descr=complexarraydescr) setinteriorfield_gc(p2, 2, f0, descr=complexrealdescr) label(p2, f0) - p4 = new_array(3, descr=complexarraydescr) + p4 = new_array_clear(3, descr=complexarraydescr) setinteriorfield_gc(p4, 2, f0, descr=compleximagdescr) jump(p4, f0) """ @@ -163,10 +163,10 @@ def test_nonmatching_arraystruct_2(self): ops = """ [p1, f0] - p2 = new_array(3, descr=complexarraydescr) + p2 = new_array_clear(3, descr=complexarraydescr) setinteriorfield_gc(p2, 2, f0, descr=complexrealdescr) label(p2, f0) - p4 = new_array(2, descr=complexarraydescr) + p4 = new_array_clear(2, descr=complexarraydescr) setinteriorfield_gc(p4, 0, f0, descr=complexrealdescr) jump(p4, f0) """ @@ -198,7 +198,7 @@ def test_not_virtual_arraystruct(self): ops = """ [p1] - p3 = new_array(3, descr=complexarraydescr) + p3 = new_array_clear(3, descr=complexarraydescr) label(p3) p4 = escape() jump(p4) diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py @@ -940,7 +940,7 @@ def test_virtual_array_of_struct(self): ops = """ [f0, f1, f2, f3] - p0 = new_array(2, descr=complexarraydescr) + p0 = new_array_clear(2, descr=complexarraydescr) setinteriorfield_gc(p0, 0, f1, descr=compleximagdescr) setinteriorfield_gc(p0, 0, f0, descr=complexrealdescr) setinteriorfield_gc(p0, 1, f3, descr=compleximagdescr) @@ -966,7 +966,7 @@ def test_virtual_array_of_struct_forced(self): ops = """ [f0, f1] - p0 = new_array(1, descr=complexarraydescr) + p0 = new_array_clear(1, descr=complexarraydescr) setinteriorfield_gc(p0, 0, f0, descr=complexrealdescr) setinteriorfield_gc(p0, 0, f1, descr=compleximagdescr) f2 = getinteriorfield_gc(p0, 0, descr=complexrealdescr) @@ -978,7 +978,7 @@ expected = """ [f0, f1] f2 = float_mul(f0, f1) - p0 = new_array(1, descr=complexarraydescr) + p0 = new_array_clear(1, descr=complexarraydescr) setinteriorfield_gc(p0, 0, f1, descr=compleximagdescr) setinteriorfield_gc(p0, 0, f0, descr=complexrealdescr) i0 = escape(f2, p0) @@ -989,7 +989,7 @@ def test_virtual_array_of_struct_len(self): ops = """ [] - p0 = new_array(2, descr=complexarraydescr) + p0 = new_array_clear(2, descr=complexarraydescr) i0 = arraylen_gc(p0) finish(i0) """ @@ -1056,7 +1056,7 @@ """ self.optimize_loop(ops, expected) - def test_nonvirtual_dont_write_null_fields_on_force(self): + def test_nonvirtual_write_null_fields_on_force(self): ops = """ [i] p1 = new_with_vtable(ConstClass(node_vtable)) @@ -1070,6 +1070,7 @@ expected = """ [i] p1 = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(p1, 0, descr=valuedescr) escape(p1) i2 = getfield_gc(p1, descr=valuedescr) jump(i2) @@ -1134,12 +1135,12 @@ [i1] p1 = new_array(2, descr=arraydescr) setarrayitem_gc(p1, 0, 25, descr=arraydescr) - i2 = getarrayitem_gc(p1, 1, descr=arraydescr) + i2 = getarrayitem_gc(p1, 0, descr=arraydescr) jump(i2) """ expected = """ [i1] - jump(0) + jump(25) """ self.optimize_loop(ops, expected) @@ -1176,7 +1177,7 @@ """ self.optimize_loop(ops, expected) - def test_nonvirtual_array_dont_write_null_fields_on_force(self): + def test_nonvirtual_array_write_null_fields_on_force(self): ops = """ [i1] p1 = new_array(5, descr=arraydescr) @@ -1189,6 +1190,7 @@ [i1] p1 = new_array(5, descr=arraydescr) setarrayitem_gc(p1, 0, i1, descr=arraydescr) + setarrayitem_gc(p1, 1, 0, descr=arraydescr) escape(p1) jump(i1) """ @@ -2975,6 +2977,7 @@ [p1] p0 = force_token() p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) + setfield_gc(p2, NULL, descr=virtualforceddescr) setfield_gc(p2, p0, descr=virtualtokendescr) escape(p2) setfield_gc(p2, p1, descr=virtualforceddescr) @@ -3007,6 +3010,7 @@ p3 = force_token() # p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) + setfield_gc(p2, NULL, descr=virtualforceddescr) setfield_gc(p2, p3, descr=virtualtokendescr) setfield_gc(p0, p2, descr=nextdescr) # @@ -3046,6 +3050,7 @@ p3 = force_token() # p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) + setfield_gc(p2, NULL, descr=virtualforceddescr) setfield_gc(p2, p3, descr=virtualtokendescr) setfield_gc(p0, p2, descr=nextdescr) # @@ -3122,6 +3127,7 @@ [i1] p3 = force_token() p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) + setfield_gc(p2, NULL, descr=virtualforceddescr) setfield_gc(p2, p3, descr=virtualtokendescr) escape(p2) p1 = new_with_vtable(ConstClass(node_vtable)) @@ -3147,6 +3153,7 @@ [i1, p1] p3 = force_token() p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) + setfield_gc(p2, NULL, descr=virtualforceddescr) setfield_gc(p2, p3, descr=virtualtokendescr) escape(p2) setfield_gc(p2, p1, descr=virtualforceddescr) @@ -4784,15 +4791,18 @@ ops = """ [p0] p1 = newstr(4) + strsetitem(p1, 2, 0) setfield_gc(p0, p1, descr=valuedescr) jump(p0) """ - # It used to be the case that this would have a series of - # strsetitem(p1, idx, 0), which was silly because memory is 0 filled - # when allocated. + # This test is slightly bogus: the string is not fully initialized. + # I *think* it is still right to not have a series of extra + # strsetitem(p1, idx, 0). We do preserve the single one from the + # source, though. expected = """ [p0] p1 = newstr(4) + strsetitem(p1, 2, 0) setfield_gc(p0, p1, descr=valuedescr) jump(p0) """ @@ -5108,6 +5118,9 @@ strsetitem(p1, 6, i0) strsetitem(p1, 7, i0) strsetitem(p1, 8, 3) + strsetitem(p1, 9, 0) + strsetitem(p1, 10, 0) + strsetitem(p1, 11, 0) finish(p1) """ self.optimize_strunicode_loop(ops, expected) diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -1397,7 +1397,7 @@ """ self.optimize_loop(ops, expected) - def test_nonvirtual_dont_write_null_fields_on_force(self): + def test_nonvirtual_write_null_fields_on_force(self): ops = """ [i] p1 = new_with_vtable(ConstClass(node_vtable)) @@ -1411,6 +1411,7 @@ expected = """ [i] p1 = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(p1, 0, descr=valuedescr) escape(p1) i2 = getfield_gc(p1, descr=valuedescr) jump(i2) @@ -1562,7 +1563,7 @@ [i1] p1 = new_array(2, descr=arraydescr) setarrayitem_gc(p1, 0, 25, descr=arraydescr) - i2 = getarrayitem_gc(p1, 1, descr=arraydescr) + i2 = getarrayitem_gc(p1, 0, descr=arraydescr) jump(i2) """ preamble = """ @@ -1608,7 +1609,7 @@ """ self.optimize_loop(ops, expected) - def test_nonvirtual_array_dont_write_null_fields_on_force(self): + def test_nonvirtual_array_write_null_fields_on_force(self): ops = """ [i1] p1 = new_array(5, descr=arraydescr) @@ -1621,6 +1622,7 @@ [i1] p1 = new_array(5, descr=arraydescr) setarrayitem_gc(p1, 0, i1, descr=arraydescr) + setarrayitem_gc(p1, 1, 0, descr=arraydescr) escape(p1) jump(i1) """ @@ -3749,6 +3751,7 @@ [p1] p0 = force_token() p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) + setfield_gc(p2, NULL, descr=virtualforceddescr) setfield_gc(p2, p0, descr=virtualtokendescr) escape(p2) setfield_gc(p2, p1, descr=virtualforceddescr) @@ -3781,6 +3784,7 @@ p3 = force_token() # p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) + setfield_gc(p2, NULL, descr=virtualforceddescr) setfield_gc(p2, p3, descr=virtualtokendescr) setfield_gc(p0, p2, descr=nextdescr) # @@ -3820,6 +3824,7 @@ p3 = force_token() # p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) + setfield_gc(p2, NULL, descr=virtualforceddescr) setfield_gc(p2, p3, descr=virtualtokendescr) setfield_gc(p0, p2, descr=nextdescr) # @@ -3907,6 +3912,7 @@ From noreply at buildbot.pypy.org Sun Sep 28 22:46:12 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 28 Sep 2014 22:46:12 +0200 (CEST) Subject: [pypy-commit] pypy default: Test and fix (for 64-bit only): the nonnull() method of ConstFloat Message-ID: <20140928204612.A2C841C0250@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73742:03e137fd14b3 Date: 2014-09-28 22:45 +0200 http://bitbucket.org/pypy/pypy/changeset/03e137fd14b3/ Log: Test and fix (for 64-bit only): the nonnull() method of ConstFloat and BoxFloat was implemented with "self.value != 0.0", which returns false for both 0.0 and -0.0. But the real intent of this method is to check if the value is null in the sense that a pattern of zeros is a correct initialization for the value. diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py --- a/rpython/jit/metainterp/history.py +++ b/rpython/jit/metainterp/history.py @@ -285,7 +285,7 @@ return False def nonnull(self): - return self.value != longlong.ZEROF + return bool(longlong.extract_bits(self.value)) def _getrepr_(self): return self.getfloat() @@ -460,7 +460,7 @@ return longlong.gethash(self.value) def nonnull(self): - return self.value != longlong.ZEROF + return bool(longlong.extract_bits(self.value)) def _getrepr_(self): return self.getfloat() diff --git a/rpython/jit/metainterp/test/test_history.py b/rpython/jit/metainterp/test/test_history.py --- a/rpython/jit/metainterp/test/test_history.py +++ b/rpython/jit/metainterp/test/test_history.py @@ -59,6 +59,20 @@ assert not c5.same_constant(c2) assert not c5.same_constant(c4) +def test_float_nonnull(): + c1 = Const._new(0.0) + c2 = Const._new(1.0) + c3 = Const._new(INFINITY) + c4 = Const._new(-INFINITY) + c5 = Const._new(NAN) + c6 = Const._new(-0.0) + assert not c1.nonnull() + assert c2.nonnull() + assert c3.nonnull() + assert c4.nonnull() + assert c5.nonnull() + assert c6.nonnull() + class TestZTranslated(StandaloneTests): def test_ztranslated_same_constant_float(self): From noreply at buildbot.pypy.org Mon Sep 29 10:27:03 2014 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 29 Sep 2014 10:27:03 +0200 (CEST) Subject: [pypy-commit] pypy default: Translation fix Message-ID: <20140929082703.08DE71C02ED@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73743:987a3cad8bfd Date: 2014-09-29 10:26 +0200 http://bitbucket.org/pypy/pypy/changeset/987a3cad8bfd/ Log: Translation fix diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py --- a/rpython/jit/backend/arm/assembler.py +++ b/rpython/jit/backend/arm/assembler.py @@ -931,6 +931,7 @@ guard, fcond) fcond = asm_operations_with_guard[opnum](self, op, guard, arglocs, regalloc, fcond) + assert fcond is not None regalloc.next_instruction() regalloc.possibly_free_vars_for_op(guard) regalloc.possibly_free_vars(guard.getfailargs()) @@ -941,6 +942,7 @@ if arglocs is not None: fcond = asm_operations[opnum](self, op, arglocs, regalloc, fcond) + assert fcond is not None if op.is_guard(): regalloc.possibly_free_vars(op.getfailargs()) if op.result: diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py --- a/rpython/jit/backend/arm/opassembler.py +++ b/rpython/jit/backend/arm/opassembler.py @@ -1256,3 +1256,4 @@ [dstaddr_loc, imm(0), length_loc]) regalloc.rm.possibly_free_var(length_box) regalloc.rm.possibly_free_var(dstaddr_box) + return fcond From noreply at buildbot.pypy.org Mon Sep 29 11:17:34 2014 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 29 Sep 2014 11:17:34 +0200 (CEST) Subject: [pypy-commit] pypy default: Optimize ZERO_ARRAY(const) followed by some number of SETARRAYITEM_GCs. Message-ID: <20140929091734.BBA491C0250@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73744:1083964c3070 Date: 2014-09-29 11:16 +0200 http://bitbucket.org/pypy/pypy/changeset/1083964c3070/ Log: Optimize ZERO_ARRAY(const) followed by some number of SETARRAYITEM_GCs. diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py --- a/rpython/jit/backend/arm/opassembler.py +++ b/rpython/jit/backend/arm/opassembler.py @@ -1180,6 +1180,9 @@ def emit_op_zero_array(self, op, arglocs, regalloc, fcond): from rpython.jit.backend.llsupport.descr import unpack_arraydescr assert len(arglocs) == 0 + length_box = op.getarg(2) + if isinstance(length_box, ConstInt) and length_box.getint() == 0: + return fcond # nothing to do itemsize, baseofs, _ = unpack_arraydescr(op.getdescr()) args = op.getarglist() base_loc = regalloc.rm.make_sure_var_in_reg(args[0], args) @@ -1191,7 +1194,6 @@ else: startindex_loc = regalloc.rm.make_sure_var_in_reg(sibox, args) startindex = -1 - length_box = op.getarg(2) # base_loc and startindex_loc are in two regs here (or they are # immediates). Compute the dstaddr_loc, which is the raw diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -48,7 +48,8 @@ self.known_lengths = {} self.write_barrier_applied = {} self.delayed_zero_setfields = {} - self.delayed_zero_setarrayitems = {} + self.last_zero_arrays = [] + self.setarrayitems_occurred = {} # {box: {set-of-indexes}} def rewrite(self, operations): # we can only remember one malloc since the next malloc can possibly @@ -81,6 +82,7 @@ self.handle_write_barrier_setinteriorfield(op) continue if op.getopnum() == rop.SETARRAYITEM_GC: + self.consider_setarrayitem_gc(op) self.handle_write_barrier_setarrayitem(op) continue else: @@ -89,6 +91,8 @@ # need to clal it if op.getopnum() == rop.SETFIELD_GC: self.consider_setfield_gc(op) + elif op.getopnum() == rop.SETARRAYITEM_GC: + self.consider_setarrayitem_gc(op) # ---------- call assembler ----------- if op.getopnum() == rop.CALL_ASSEMBLER: self.handle_call_assembler(op) @@ -146,6 +150,16 @@ except KeyError: pass + def consider_setarrayitem_gc(self, op): + array_box = op.getarg(0) + index_box = op.getarg(1) + if isinstance(array_box, BoxPtr) and isinstance(index_box, ConstInt): + try: + intset = self.setarrayitems_occurred[array_box] + except KeyError: + intset = self.setarrayitems_occurred[array_box] = {} + intset[index_box.getint()] = None + def clear_varsize_gc_fields(self, kind, descr, result, v_length, opnum): if self.gc_ll_descr.malloc_zero_filled: return @@ -216,18 +230,18 @@ self.clear_varsize_gc_fields(kind, op.getdescr(), op.result, v_length, op.getopnum()) - def handle_clear_array_contents(self, arraydescr, v_arr, v_length=None): - # XXX more work here to reduce or remove the ZERO_ARRAY in some cases - if v_length is None: - v_length = BoxInt() - o = ResOperation(rop.ARRAYLEN_GC, [v_arr], v_length, - descr=arraydescr) - self.newops.append(o) - elif isinstance(v_length, ConstInt) and v_length.getint() == 0: + def handle_clear_array_contents(self, arraydescr, v_arr, v_length): + assert v_length is not None + if isinstance(v_length, ConstInt) and v_length.getint() == 0: return + # the ZERO_ARRAY operation will be optimized according to what + # SETARRAYITEM_GC we see before the next allocation operation. + # See emit_pending_zeros(). o = ResOperation(rop.ZERO_ARRAY, [v_arr, self.c_zero, v_length], None, descr=arraydescr) self.newops.append(o) + if isinstance(v_length, ConstInt): + self.last_zero_arrays.append(o) def gen_malloc_frame(self, frame_info, frame, size_box): descrs = self.gc_ll_descr.getframedescrs(self.cpu) @@ -317,6 +331,31 @@ self.emit_pending_zeros() def emit_pending_zeros(self): + # First, try to rewrite the existing ZERO_ARRAY operations from + # the 'last_zero_arrays' list. Note that these operation objects + # are also already in 'newops', which is the point. + for op in self.last_zero_arrays: + assert op.getopnum() == rop.ZERO_ARRAY + box = op.getarg(0) + try: + intset = self.setarrayitems_occurred[box] + except KeyError: + continue + assert op.getarg(1).getint() == 0 # always 'start=0' initially + start = 0 + while start in intset: + start += 1 + op.setarg(1, ConstInt(start)) + stop = op.getarg(2).getint() + assert start <= stop + while stop > start and (stop - 1) in intset: + stop -= 1 + op.setarg(2, ConstInt(stop - start)) + # ^^ may be ConstInt(0); then the operation becomes a no-op + del self.last_zero_arrays[:] + self.setarrayitems_occurred.clear() + # + # Then write the ZERO_PTR_FIELDs that are still pending for v, d in self.delayed_zero_setfields.iteritems(): for ofs in d.iterkeys(): op = ResOperation(rop.ZERO_PTR_FIELD, [v, ConstInt(ofs)], None) diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py --- a/rpython/jit/backend/llsupport/test/test_rewrite.py +++ b/rpython/jit/backend/llsupport/test/test_rewrite.py @@ -743,6 +743,163 @@ jump() """) + def test_zero_array_reduced_left(self): + self.check_rewrite(""" + [p1, p2] + p0 = new_array_clear(5, descr=cdescr) + setarrayitem_gc(p0, 1, p1, descr=cdescr) + setarrayitem_gc(p0, 0, p2, descr=cdescr) + jump() + """, """ + [p1, p2] + p0 = call_malloc_nursery( \ + %(cdescr.basesize + 5 * cdescr.itemsize)d) + setfield_gc(p0, 8111, descr=tiddescr) + setfield_gc(p0, 5, descr=clendescr) + zero_array(p0, 2, 3, descr=cdescr) + setarrayitem_gc(p0, 1, p1, descr=cdescr) + setarrayitem_gc(p0, 0, p2, descr=cdescr) + jump() + """) + + def test_zero_array_reduced_right(self): + self.check_rewrite(""" + [p1, p2] + p0 = new_array_clear(5, descr=cdescr) + setarrayitem_gc(p0, 3, p1, descr=cdescr) + setarrayitem_gc(p0, 4, p2, descr=cdescr) + jump() + """, """ + [p1, p2] + p0 = call_malloc_nursery( \ + %(cdescr.basesize + 5 * cdescr.itemsize)d) + setfield_gc(p0, 8111, descr=tiddescr) + setfield_gc(p0, 5, descr=clendescr) + zero_array(p0, 0, 3, descr=cdescr) + setarrayitem_gc(p0, 3, p1, descr=cdescr) + setarrayitem_gc(p0, 4, p2, descr=cdescr) + jump() + """) + + def test_zero_array_not_reduced_at_all(self): + self.check_rewrite(""" + [p1, p2] + p0 = new_array_clear(5, descr=cdescr) + setarrayitem_gc(p0, 3, p1, descr=cdescr) + setarrayitem_gc(p0, 2, p2, descr=cdescr) + setarrayitem_gc(p0, 1, p2, descr=cdescr) + jump() + """, """ + [p1, p2] + p0 = call_malloc_nursery( \ + %(cdescr.basesize + 5 * cdescr.itemsize)d) + setfield_gc(p0, 8111, descr=tiddescr) + setfield_gc(p0, 5, descr=clendescr) + zero_array(p0, 0, 5, descr=cdescr) + setarrayitem_gc(p0, 3, p1, descr=cdescr) + setarrayitem_gc(p0, 2, p2, descr=cdescr) + setarrayitem_gc(p0, 1, p2, descr=cdescr) + jump() + """) + + def test_zero_array_reduced_completely(self): + self.check_rewrite(""" + [p1, p2] + p0 = new_array_clear(5, descr=cdescr) + setarrayitem_gc(p0, 3, p1, descr=cdescr) + setarrayitem_gc(p0, 4, p2, descr=cdescr) + setarrayitem_gc(p0, 0, p1, descr=cdescr) + setarrayitem_gc(p0, 2, p2, descr=cdescr) + setarrayitem_gc(p0, 1, p2, descr=cdescr) + jump() + """, """ + [p1, p2] + p0 = call_malloc_nursery( \ + %(cdescr.basesize + 5 * cdescr.itemsize)d) + setfield_gc(p0, 8111, descr=tiddescr) + setfield_gc(p0, 5, descr=clendescr) + zero_array(p0, 5, 0, descr=cdescr) + setarrayitem_gc(p0, 3, p1, descr=cdescr) + setarrayitem_gc(p0, 4, p2, descr=cdescr) + setarrayitem_gc(p0, 0, p1, descr=cdescr) + setarrayitem_gc(p0, 2, p2, descr=cdescr) + setarrayitem_gc(p0, 1, p2, descr=cdescr) + jump() + """) + + def test_zero_array_reduced_left_with_call(self): + self.check_rewrite(""" + [p1, p2] + p0 = new_array_clear(5, descr=cdescr) + setarrayitem_gc(p0, 0, p1, descr=cdescr) + call(321321) + setarrayitem_gc(p0, 1, p2, descr=cdescr) + jump() + """, """ + [p1, p2] + p0 = call_malloc_nursery( \ + %(cdescr.basesize + 5 * cdescr.itemsize)d) + setfield_gc(p0, 8111, descr=tiddescr) + setfield_gc(p0, 5, descr=clendescr) + zero_array(p0, 1, 4, descr=cdescr) + setarrayitem_gc(p0, 0, p1, descr=cdescr) + call(321321) + cond_call_gc_wb(p0, descr=wbdescr) + setarrayitem_gc(p0, 1, p2, descr=cdescr) + jump() + """) + + def test_zero_array_reduced_left_with_label(self): + self.check_rewrite(""" + [p1, p2] + p0 = new_array_clear(5, descr=cdescr) + setarrayitem_gc(p0, 0, p1, descr=cdescr) + label(p0, p2) + setarrayitem_gc(p0, 1, p2, descr=cdescr) + jump() + """, """ + [p1, p2] + p0 = call_malloc_nursery( \ + %(cdescr.basesize + 5 * cdescr.itemsize)d) + setfield_gc(p0, 8111, descr=tiddescr) + setfield_gc(p0, 5, descr=clendescr) + zero_array(p0, 1, 4, descr=cdescr) + setarrayitem_gc(p0, 0, p1, descr=cdescr) + label(p0, p2) + cond_call_gc_wb_array(p0, 1, descr=wbdescr) + setarrayitem_gc(p0, 1, p2, descr=cdescr) + jump() + """) + + def test_zero_array_varsize(self): + self.check_rewrite(""" + [p1, p2, i3] + p0 = new_array_clear(i3, descr=bdescr) + jump() + """, """ + [p1, p2, i3] + p0 = call_malloc_nursery_varsize(0, 1, i3, descr=bdescr) + setfield_gc(p0, i3, descr=blendescr) + zero_array(p0, 0, i3, descr=bdescr) + jump() + """) + + def test_zero_array_varsize_cannot_reduce(self): + self.check_rewrite(""" + [p1, p2, i3] + p0 = new_array_clear(i3, descr=bdescr) + setarrayitem_gc(p0, 0, p1, descr=bdescr) + jump() + """, """ + [p1, p2, i3] + p0 = call_malloc_nursery_varsize(0, 1, i3, descr=bdescr) + setfield_gc(p0, i3, descr=blendescr) + zero_array(p0, 0, i3, descr=bdescr) + cond_call_gc_wb_array(p0, 0, descr=wbdescr) + setarrayitem_gc(p0, 0, p1, descr=bdescr) + jump() + """) + def test_initialization_store_potentially_large_array(self): # the write barrier cannot be omitted, because we might get # an array with cards and the GC assumes that the write diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -1386,14 +1386,16 @@ def consider_zero_array(self, op): itemsize, baseofs, _ = unpack_arraydescr(op.getdescr()) + length_box = op.getarg(2) + if isinstance(length_box, ConstInt): + constbytes = length_box.getint() * itemsize + if constbytes == 0: + return # nothing to do + else: + constbytes = -1 args = op.getarglist() base_loc = self.rm.make_sure_var_in_reg(args[0], args) startindex_loc = self.rm.make_sure_var_in_reg(args[1], args) - length_box = op.getarg(2) - if isinstance(length_box, ConstInt): - constbytes = length_box.getint() * itemsize - else: - constbytes = -1 if 0 <= constbytes <= 16 * 8 and ( valid_addressing_size(itemsize) or - isinstance(startindex_loc, ImmedLoc)): From noreply at buildbot.pypy.org Mon Sep 29 16:08:56 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Mon, 29 Sep 2014 16:08:56 +0200 (CEST) Subject: [pypy-commit] stmgc c8-new-page-handling: actually *do* apply undo copies if we abort Message-ID: <20140929140856.47A821D23D6@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: c8-new-page-handling Changeset: r1436:14164f0fc6a8 Date: 2014-09-29 16:09 +0200 http://bitbucket.org/pypy/stmgc/changeset/14164f0fc6a8/ Log: actually *do* apply undo copies if we abort diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -156,6 +156,8 @@ release_modified_objs_lock(copy_from_segnum); release_all_privatization_locks(); + dprintf(("handle_segfault_in_page: rev %lu to rev %lu\n", + src_version->rev_num, target_version->rev_num)); /* adapt revision of page to our revision: if our rev is higher than the page we copy from, everything is fine as we never read/modified the page anyway @@ -217,9 +219,9 @@ for (; undo < end; undo++) { fprintf(stderr, " obj %p, size %d, ofs %lu: ", undo->object, SLICE_SIZE(undo->slice), SLICE_OFFSET(undo->slice)); - long i; - for (i=0; islice); i += 8) - fprintf(stderr, " 0x%016lx", *(long *)(undo->backup + i)); + /* long i; */ + /* for (i=0; islice); i += 8) */ + /* fprintf(stderr, " 0x%016lx", *(long *)(undo->backup + i)); */ fprintf(stderr, "\n"); } } @@ -258,9 +260,8 @@ /* there is an inevitable transaction running */ release_modified_objs_lock(my_segnum); #if STM_TESTS - if (free_if_abort != (void *)-1) - free(free_if_abort); - stm_abort_transaction(); + needs_abort = true; + goto complete_work_and_possibly_abort; #endif abort(); /* XXX non-busy wait here or: XXX do we always need to wait? we should just @@ -310,6 +311,9 @@ release_modified_objs_lock(my_segnum); +#if STM_TESTS + complete_work_and_possibly_abort:; +#endif /* XXXXXXXXXXXXXXXX CORRECT LOCKING NEEDED XXXXXXXXXXXXXXXXXXXXXX */ int segnum; for (segnum = 0; segment_copied_from != 0; segnum++) { @@ -317,8 +321,10 @@ segment_copied_from &= ~(1UL << segnum); /* here we can actually have our own modified version, so make sure to only copy things that are not modified in our - segment... */ - copy_bk_objs_in_page_from(segnum, -1, true); + segment... (if we do not abort) */ + copy_bk_objs_in_page_from( + segnum, -1, /* any page */ + !needs_abort); /* if we abort, we still want to copy everything */ } } diff --git a/c8/test/test_basic.py b/c8/test/test_basic.py --- a/c8/test/test_basic.py +++ b/c8/test/test_basic.py @@ -783,3 +783,30 @@ # self.switch(0) # validate -> R2 assert stm_get_char(lp_char_5, 384 - 1) == 'i' + + def test_bug2(self): + lp_char_5 = stm_allocate_old(384) + + self.start_transaction() # R1 + stm_set_char(lp_char_5, 'i', 384 - 1, False) + stm_set_char(lp_char_5, 'i', HDR, False) + # + self.switch(1) + self.start_transaction() + # + # + self.switch(3) + self.start_transaction() # R1 + stm_set_char(lp_char_5, 'o', 384 - 1, False) # bk_copy + stm_set_char(lp_char_5, 'o', HDR, False) + self.commit_transaction() # R2 + + self.start_transaction() # R2 + stm_set_char(lp_char_5, 'r', 384 - 1, False) # bk_copy + stm_set_char(lp_char_5, 'r', HDR, False) + # + py.test.raises(Conflict, self.switch, 0) # abort modified objs + # + self.switch(1) # validate -> R2 + + assert stm_get_char(lp_char_5, 384 - 1) == 'o' diff --git a/c8/test/test_random.py b/c8/test/test_random.py --- a/c8/test/test_random.py +++ b/c8/test/test_random.py @@ -298,6 +298,7 @@ confl_set = other_trs.read_set & trs.write_set if confl_set: + assert not other_trs.inevitable # trs wins! other_trs.set_must_abort(objs_in_conflict=confl_set) assert not trs.check_must_abort() @@ -506,14 +507,16 @@ ex.do('#') # trs = new_thread_state.transaction_state - if trs is not None and not trs.inevitable: - if global_state.is_inevitable_transaction_running(): - trs.set_must_abort() - conflicts = trs is not None and trs.check_must_abort() + if trs and not trs.check_must_abort(): + global_state.check_if_can_become_inevitable(trs) + conflicts = trs and trs.check_must_abort() ex.thread_num = new_thread_state.num # + if conflicts: + ex.do("# objs_in_conflict=%s" % trs.objs_in_conflict) ex.do(raising_call(conflicts, "self.switch", new_thread_state.num)) + if conflicts: new_thread_state.abort_transaction() elif trs: From noreply at buildbot.pypy.org Mon Sep 29 16:42:34 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Mon, 29 Sep 2014 16:42:34 +0200 (CEST) Subject: [pypy-commit] stmgc c8-new-page-handling: we must not execute the STM part of the WB twice and make backup copies more than once Message-ID: <20140929144234.944331D23CD@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: c8-new-page-handling Changeset: r1437:4e523f10a999 Date: 2014-09-29 16:42 +0200 http://bitbucket.org/pypy/stmgc/changeset/4e523f10a999/ Log: we must not execute the STM part of the WB twice and make backup copies more than once diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -429,7 +429,22 @@ /* add to read set: */ stm_read(obj); - /* create backup copy (this may cause several page faults XXX): */ + /* XXX: add overflow number again? n^2 algorithm ahead... */ + struct list_s *list = STM_PSEGMENT->modified_old_objects; + int i, c = list_count(list); + for (i = 0; i < c; i += 3) { + if (list->items[i] == (uintptr_t)obj) { + /* already executed WB once in this transaction. do GC + part again: */ + obj->stm_flags &= ~GCFLAG_WRITE_BARRIER; + LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, obj); + return; + } + } + + /* create backup copy (this may cause several page faults + XXX: do backup later and maybe allow for having NO_ACCESS + pages around anyway (kind of card marking)): */ struct object_s *bk_obj = malloc(obj_size); memcpy(bk_obj, realobj, obj_size); @@ -468,7 +483,6 @@ update the shared page in stm_validate() except if it is the sole reader of it. But then we don't actually know which revision the page is at. */ /* XXX this is a temporary solution I suppose */ - int i; for (i = 0; i < NB_SEGMENTS; i++) { if (i == my_segnum) continue; @@ -708,12 +722,14 @@ undo->backup + SLICE_OFFSET(undo->slice), SLICE_SIZE(undo->slice)); - size_t obj_size = stmcb_size_rounded_up(undo->backup); + size_t obj_size = stmcb_size_rounded_up((struct object_s*)undo->backup); + dprintf(("reset_modified_from_backup_copies(%d): obj=%p off=%lu bk=%p obj_sz=%lu\n", + segment_num, obj, SLICE_OFFSET(undo->slice), undo->backup, obj_size)); + if (obj_size - SLICE_OFFSET(undo->slice) <= 4096UL) { /* only free bk copy once (last slice): */ free(undo->backup); - dprintf(("reset_modified_from_backup_copies(%d): obj=%p obj_sz=%lu\n", - segment_num, obj, obj_size)); + dprintf(("-> free(%p)\n", undo->backup)); } } diff --git a/c8/test/test_basic.py b/c8/test/test_basic.py --- a/c8/test/test_basic.py +++ b/c8/test/test_basic.py @@ -810,3 +810,20 @@ self.switch(1) # validate -> R2 assert stm_get_char(lp_char_5, 384 - 1) == 'o' + + def test_repeated_wb(self): + lp_char_5 = stm_allocate_old(384) + + self.start_transaction() + stm_set_char(lp_char_5, 'i', 384 - 1, False) + stm_set_char(lp_char_5, 'i', HDR, False) + + stm_minor_collect() + + stm_set_char(lp_char_5, 'j', 384 - 1, False) + stm_set_char(lp_char_5, 'j', HDR, False) + + self.abort_transaction() + + self.check_char_everywhere(lp_char_5, '\0', offset=HDR) + self.check_char_everywhere(lp_char_5, '\0', offset=384-1) From noreply at buildbot.pypy.org Mon Sep 29 22:21:59 2014 From: noreply at buildbot.pypy.org (rguillebert) Date: Mon, 29 Sep 2014 22:21:59 +0200 (CEST) Subject: [pypy-commit] extradoc extradoc: Add the beginning of my PyCon IE talk Message-ID: <20140929202200.0121A1D238B@cobra.cs.uni-duesseldorf.de> Author: Romain Guillebert Branch: extradoc Changeset: r5414:8e809924b51f Date: 2014-09-24 17:11 +0200 http://bitbucket.org/pypy/extradoc/changeset/8e809924b51f/ Log: Add the beginning of my PyCon IE talk diff --git a/talk/pyconie2014/Makefile b/talk/pyconie2014/Makefile new file mode 100644 --- /dev/null +++ b/talk/pyconie2014/Makefile @@ -0,0 +1,18 @@ +# you can find rst2beamer.py here: +# https://bitbucket.org/antocuni/env/raw/default/bin/rst2beamer.py + +# WARNING: to work, it needs this patch for docutils +# https://sourceforge.net/tracker/?func=detail&atid=422032&aid=1459707&group_id=38414 + +talk.pdf: talk.rst author.latex stylesheet.latex + python `which rst2beamer.py` --stylesheet=stylesheet.latex --documentoptions=14pt talk.rst talk.latex || exit + #/home/antocuni/.virtualenvs/rst2beamer/bin/python `which rst2beamer.py` --stylesheet=stylesheet.latex --documentoptions=14pt talk.rst talk.latex || exit + sed 's/\\date{}/\\input{author.latex}/' -i talk.latex || exit + #sed 's/\\maketitle/\\input{title.latex}/' -i talk.latex || exit + pdflatex talk.latex || exit + +view: talk.pdf + evince talk.pdf & + +xpdf: talk.pdf + xpdf talk.pdf & diff --git a/talk/pyconie2014/author.latex b/talk/pyconie2014/author.latex new file mode 100644 --- /dev/null +++ b/talk/pyconie2014/author.latex @@ -0,0 +1,9 @@ +\definecolor{rrblitbackground}{rgb}{0.0, 0.0, 0.0} + +\title[PyPy : A fast Python Virtual Machine]{PyPy : A fast Python Virtual Machine} +\author[rguillebert] +{Romain Guillebert\\ +\includegraphics[width=80px]{../img/py-web-new.png}} + +\institute{Pycon IE} +\date{October 12th, 2014} diff --git a/talk/pyconie2014/beamerdefs.txt b/talk/pyconie2014/beamerdefs.txt new file mode 100644 --- /dev/null +++ b/talk/pyconie2014/beamerdefs.txt @@ -0,0 +1,108 @@ +.. colors +.. =========================== + +.. role:: green +.. role:: red + + +.. general useful commands +.. =========================== + +.. |pause| raw:: latex + + \pause + +.. |small| raw:: latex + + {\small + +.. |end_small| raw:: latex + + } + +.. |scriptsize| raw:: latex + + {\scriptsize + +.. |end_scriptsize| raw:: latex + + } + +.. |strike<| raw:: latex + + \sout{ + +.. closed bracket +.. =========================== + +.. |>| raw:: latex + + } + + +.. example block +.. =========================== + +.. |example<| raw:: latex + + \begin{exampleblock}{ + + +.. |end_example| raw:: latex + + \end{exampleblock} + + + +.. alert block +.. =========================== + +.. |alert<| raw:: latex + + \begin{alertblock}{ + + +.. |end_alert| raw:: latex + + \end{alertblock} + + + +.. columns +.. =========================== + +.. |column1| raw:: latex + + \begin{columns} + \begin{column}{0.45\textwidth} + +.. |column2| raw:: latex + + \end{column} + \begin{column}{0.45\textwidth} + + +.. |end_columns| raw:: latex + + \end{column} + \end{columns} + + + +.. |snake| image:: ../../img/py-web-new.png + :scale: 15% + + + +.. nested blocks +.. =========================== + +.. |nested| raw:: latex + + \begin{columns} + \begin{column}{0.85\textwidth} + +.. |end_nested| raw:: latex + + \end{column} + \end{columns} diff --git a/talk/pyconie2014/speed.png b/talk/pyconie2014/speed.png new file mode 100644 index 0000000000000000000000000000000000000000..4640c76f8a665af1c414dc4c4ca22be3bd8ff360 GIT binary patch [cut] diff --git a/talk/pyconie2014/stylesheet.latex b/talk/pyconie2014/stylesheet.latex new file mode 100644 --- /dev/null +++ b/talk/pyconie2014/stylesheet.latex @@ -0,0 +1,9 @@ +\setbeamercovered{transparent} +\setbeamertemplate{navigation symbols}{} + +\definecolor{darkgreen}{rgb}{0, 0.5, 0.0} +\newcommand{\docutilsrolegreen}[1]{\color{darkgreen}#1\normalcolor} +\newcommand{\docutilsrolered}[1]{\color{red}#1\normalcolor} + +\newcommand{\green}[1]{\color{darkgreen}#1\normalcolor} +\newcommand{\red}[1]{\color{red}#1\normalcolor} diff --git a/talk/pyconie2014/talk.rst b/talk/pyconie2014/talk.rst new file mode 100644 --- /dev/null +++ b/talk/pyconie2014/talk.rst @@ -0,0 +1,34 @@ +PyPy : A fast Python Virtual Machine +==================================== + +Me +-- + +- rguillebert on twitter and irc + +- PyPy contributor since 2011 + +- NumPyPy contributor + +- Software consultant (hire me !) + +- Feel free to interrupt + +Introduction +------------ + +- "PyPy is a fast, compliant alternative implementation of the Python language" + +- Aims to reach the best performance possible without changing the syntax or semantics + +- Supports x86, x86_64, ARM + +- Production ready + +- MIT Licensed + +Speed +----- + +.. image:: speed.png + :scale: 37% From noreply at buildbot.pypy.org Mon Sep 29 22:22:01 2014 From: noreply at buildbot.pypy.org (rguillebert) Date: Mon, 29 Sep 2014 22:22:01 +0200 (CEST) Subject: [pypy-commit] extradoc extradoc: Add more slides Message-ID: <20140929202201.21C2C1D238B@cobra.cs.uni-duesseldorf.de> Author: Romain Guillebert Branch: extradoc Changeset: r5415:1d748d9bdbc8 Date: 2014-09-29 22:20 +0200 http://bitbucket.org/pypy/extradoc/changeset/1d748d9bdbc8/ Log: Add more slides diff --git a/talk/pyconie2014/talk.rst b/talk/pyconie2014/talk.rst --- a/talk/pyconie2014/talk.rst +++ b/talk/pyconie2014/talk.rst @@ -1,3 +1,5 @@ +.. include:: beamerdefs.tx + PyPy : A fast Python Virtual Machine ==================================== @@ -32,3 +34,43 @@ .. image:: speed.png :scale: 37% + +Speed +----- + +- Automatically generated tracing just-in-time compiler + +- Generates efficient machine code based on runtime observations + +- Removes overhead when unnecessary + +- But these Python features remain available (pdb) + +RPython +------- + +- Subset of Python + +- Made for writting virtual machines + +- Takes care of garbage collection and JIT compilation + +- A VM written in RPython doesn't have to know about the garbage collector + +- Minimal help from the VM is needed in order to have an efficient JIT (a few annotations) + +Demo +---- + +- Real-time edge detection + +How +--- + +- Removes boxing, integer objects become machine integers + +- Specializes trace on types, helps speed-up method lookup + +- If the type of the object is different from the type in the trace, go back to the interpreter : "guard failure" + +- If a guard fails too many times, optimize the trace for the other types frequently encountered From noreply at buildbot.pypy.org Mon Sep 29 22:22:02 2014 From: noreply at buildbot.pypy.org (rguillebert) Date: Mon, 29 Sep 2014 22:22:02 +0200 (CEST) Subject: [pypy-commit] extradoc extradoc: Add myself Message-ID: <20140929202202.424F71D238B@cobra.cs.uni-duesseldorf.de> Author: Romain Guillebert Branch: extradoc Changeset: r5416:2cf1024f1867 Date: 2014-09-29 22:21 +0200 http://bitbucket.org/pypy/extradoc/changeset/2cf1024f1867/ Log: Add myself diff --git a/sprintinfo/warsaw-2014/people.txt b/sprintinfo/warsaw-2014/people.txt --- a/sprintinfo/warsaw-2014/people.txt +++ b/sprintinfo/warsaw-2014/people.txt @@ -10,4 +10,5 @@ Name Arrive/Depart Accomodation ==================== ============== ======================= Armin Rigo 20/10-2X/10 with fijal? +Romain Guillebert 19/10-26-10 ??? ==================== ============== ======================= From noreply at buildbot.pypy.org Mon Sep 29 23:45:55 2014 From: noreply at buildbot.pypy.org (mjacob) Date: Mon, 29 Sep 2014 23:45:55 +0200 (CEST) Subject: [pypy-commit] extradoc extradoc: Add myself to the people coming to the Warsaw sprint. Message-ID: <20140929214555.2249C1D23D6@cobra.cs.uni-duesseldorf.de> Author: Manuel Jacob Branch: extradoc Changeset: r5417:8e3e7fcb1471 Date: 2014-09-29 23:45 +0200 http://bitbucket.org/pypy/extradoc/changeset/8e3e7fcb1471/ Log: Add myself to the people coming to the Warsaw sprint. diff --git a/sprintinfo/warsaw-2014/people.txt b/sprintinfo/warsaw-2014/people.txt --- a/sprintinfo/warsaw-2014/people.txt +++ b/sprintinfo/warsaw-2014/people.txt @@ -11,4 +11,5 @@ ==================== ============== ======================= Armin Rigo 20/10-2X/10 with fijal? Romain Guillebert 19/10-26-10 ??? +Manuel Jacob 20/10-26/10 ? (shared hotel room?) ==================== ============== ======================= From noreply at buildbot.pypy.org Tue Sep 30 10:15:49 2014 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 30 Sep 2014 10:15:49 +0200 (CEST) Subject: [pypy-commit] extradoc extradoc: updates Message-ID: <20140930081549.7672B1C06C0@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r5418:8014405740a0 Date: 2014-09-30 10:15 +0200 http://bitbucket.org/pypy/extradoc/changeset/8014405740a0/ Log: updates diff --git a/sprintinfo/warsaw-2014/announcement.txt b/sprintinfo/warsaw-2014/announcement.txt --- a/sprintinfo/warsaw-2014/announcement.txt +++ b/sprintinfo/warsaw-2014/announcement.txt @@ -38,8 +38,8 @@ ------------ The sprint will happen within a room of Warsaw University. The -address is Pasteura 5 (which is a form of "Pasteur street"), room 550. -The person of contact is Maciej Fijalkowski. +address is Pasteura 5 (which is a form of "Pasteur street"), dept. of +Physics, room 450. The person of contact is Maciej Fijalkowski. -------------- diff --git a/sprintinfo/warsaw-2014/people.txt b/sprintinfo/warsaw-2014/people.txt --- a/sprintinfo/warsaw-2014/people.txt +++ b/sprintinfo/warsaw-2014/people.txt @@ -12,4 +12,5 @@ Armin Rigo 20/10-2X/10 with fijal? Romain Guillebert 19/10-26-10 ??? Manuel Jacob 20/10-26/10 ? (shared hotel room?) +Kostia Lopuhin ==================== ============== ======================= From noreply at buildbot.pypy.org Tue Sep 30 10:24:02 2014 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 30 Sep 2014 10:24:02 +0200 (CEST) Subject: [pypy-commit] extradoc extradoc: add myself Message-ID: <20140930082402.37EDD1C319E@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: extradoc Changeset: r5419:e6201b6ed27d Date: 2014-09-30 10:24 +0200 http://bitbucket.org/pypy/extradoc/changeset/e6201b6ed27d/ Log: add myself diff --git a/sprintinfo/warsaw-2014/people.txt b/sprintinfo/warsaw-2014/people.txt --- a/sprintinfo/warsaw-2014/people.txt +++ b/sprintinfo/warsaw-2014/people.txt @@ -9,7 +9,8 @@ ==================== ============== ======================= Name Arrive/Depart Accomodation ==================== ============== ======================= -Armin Rigo 20/10-2X/10 with fijal? +Armin Rigo 20/10-2X/10 with fijal +Maciej Fijalkowski 20/10-30/10 private Romain Guillebert 19/10-26-10 ??? Manuel Jacob 20/10-26/10 ? (shared hotel room?) Kostia Lopuhin From noreply at buildbot.pypy.org Tue Sep 30 14:34:22 2014 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 30 Sep 2014 14:34:22 +0200 (CEST) Subject: [pypy-commit] pypy default: Fixes: Message-ID: <20140930123422.370801D28CF@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73745:b422496d47be Date: 2014-09-30 14:21 +0200 http://bitbucket.org/pypy/pypy/changeset/b422496d47be/ Log: Fixes: * don't zero out large external mallocs for no reason * fix gc.malloc(), used only for testing, to respect the 'zero' flag diff --git a/rpython/memory/gc/base.py b/rpython/memory/gc/base.py --- a/rpython/memory/gc/base.py +++ b/rpython/memory/gc/base.py @@ -127,17 +127,10 @@ return self.get_size(obj) def malloc(self, typeid, length=0, zero=False): - """For testing. The interface used by the gctransformer is + """NOT_RPYTHON + For testing. The interface used by the gctransformer is the four malloc_[fixed,var]size[_clear]() functions. """ - #TODO:check if the zero flag is unuseful now. If so, remove it - - # Rules about fallbacks in case of missing malloc methods: - # * malloc_fixedsize_clear() and malloc_varsize_clear() are mandatory - # * malloc_fixedsize() and malloc_varsize() fallback to the above - # XXX: as of r49360, gctransformer.framework never inserts calls - # to malloc_varsize(), but always uses malloc_varsize_clear() - size = self.fixed_size(typeid) needs_finalizer = bool(self.getfinalizer(typeid)) finalizer_is_light = bool(self.getlightfinalizer(typeid)) @@ -154,6 +147,7 @@ malloc_varsize = self.malloc_varsize ref = malloc_varsize(typeid, length, size, itemsize, offset_to_length) + size += itemsize * length else: if self.malloc_zero_filled: malloc_fixedsize = self.malloc_fixedsize_clear @@ -163,7 +157,10 @@ finalizer_is_light, contains_weakptr) # lots of cast and reverse-cast around... - return llmemory.cast_ptr_to_adr(ref) + ref = llmemory.cast_ptr_to_adr(ref) + if zero and not self.malloc_zero_filled: + llmemory.raw_memclear(ref, size) + return ref def id(self, ptr): return lltype.cast_ptr_to_int(ptr) diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -431,7 +431,7 @@ def _alloc_nursery(self): # the start of the nursery: we actually allocate a bit more for # the nursery than really needed, to simplify pointer arithmetic - # in malloc_fixedsize_clear(). The few extra pages are never used + # in malloc_fixedsize(). The few extra pages are never used # anyway so it doesn't even counct. nursery = llarena.arena_malloc(self._nursery_memory_size(), 0) if not nursery: @@ -613,7 +613,7 @@ # 'minimal_size_in_nursery' ll_assert(raw_malloc_usage(totalsize) >= raw_malloc_usage(self.minimal_size_in_nursery), - "malloc_varsize_clear(): totalsize < minimalsize") + "malloc_varsize(): totalsize < minimalsize") # # Get the memory from the nursery. If there is not enough space # there, do a collect first. @@ -687,7 +687,7 @@ raw_malloc(), possibly as an object with card marking enabled, if it has gc pointers in its var-sized part. 'length' should be specified as 0 if the object is not varsized. The returned - object is fully initialized and zero-filled.""" + object is fully initialized, but not zero-filled.""" # # Here we really need a valid 'typeid', not 0 (as the JIT might # try to send us if there is still a bug). @@ -731,9 +731,8 @@ self.small_request_threshold, "rounding up made totalsize > small_request_threshold") # - # Allocate from the ArenaCollection and clear the memory returned. + # Allocate from the ArenaCollection. Don't clear it. result = self.ac.malloc(totalsize) - llmemory.raw_memclear(result, totalsize) # # An object allocated from ArenaCollection is always old, even # if 'can_make_young'. The interesting case of 'can_make_young' @@ -779,19 +778,20 @@ # Allocate the object using arena_malloc(), which we assume here # is just the same as raw_malloc(), but allows the extra # flexibility of saying that we have extra words in the header. - # The memory returned is cleared by a raw_memclear(). - arena = llarena.arena_malloc(allocsize, 2) + # The memory returned is not cleared. + arena = llarena.arena_malloc(allocsize, 0) if not arena: raise MemoryError("cannot allocate large object") # - # Reserve the card mark bits as a list of single bytes - # (the loop is empty in C). + # Reserve the card mark bits as a list of single bytes, + # and clear these bytes. i = 0 while i < cardheadersize: llarena.arena_reserve(arena + i, llmemory.sizeof(lltype.Char)) + arena.char[i] = '\x00' i += 1 # - # Reserve the actual object. (This is also a no-op in C). + # Reserve the actual object. (This is a no-op in C). result = arena + cardheadersize llarena.arena_reserve(result, totalsize) # From noreply at buildbot.pypy.org Tue Sep 30 14:34:23 2014 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 30 Sep 2014 14:34:23 +0200 (CEST) Subject: [pypy-commit] pypy default: Test fix Message-ID: <20140930123423.8A39E1D28CF@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73746:90df6dfc9c89 Date: 2014-09-30 14:32 +0200 http://bitbucket.org/pypy/pypy/changeset/90df6dfc9c89/ Log: Test fix diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -787,8 +787,9 @@ # and clear these bytes. i = 0 while i < cardheadersize: - llarena.arena_reserve(arena + i, llmemory.sizeof(lltype.Char)) - arena.char[i] = '\x00' + p = arena + i + llarena.arena_reserve(p, llmemory.sizeof(lltype.Char)) + p.char[0] = '\x00' i += 1 # # Reserve the actual object. (This is a no-op in C). From noreply at buildbot.pypy.org Tue Sep 30 17:19:45 2014 From: noreply at buildbot.pypy.org (Raemi) Date: Tue, 30 Sep 2014 17:19:45 +0200 (CEST) Subject: [pypy-commit] stmgc c8-new-page-handling: introduce GCFLAG_WB_EXECUTED to check if we already executed the write barrier Message-ID: <20140930151945.D3C731C094F@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: c8-new-page-handling Changeset: r1438:b4042a2af79d Date: 2014-09-30 17:20 +0200 http://bitbucket.org/pypy/stmgc/changeset/b4042a2af79d/ Log: introduce GCFLAG_WB_EXECUTED to check if we already executed the write barrier in the same transaction. Alternatively, we could introduce write-locks again or some other mechanism. diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -10,7 +10,7 @@ */ static void import_objects( int from_segnum, /* or -1: from undo->backup, - or -2: from undo->backup if not stm_was_read(obj) */ + or -2: from undo->backup if not modified */ uintptr_t pagenum, /* or -1: "all accessible" */ struct stm_undo_s *undo, struct stm_undo_s *end) @@ -33,8 +33,16 @@ continue; } - if (from_segnum == -2 && _stm_was_read(obj)) + if (from_segnum == -2 && _stm_was_read(obj) && (obj->stm_flags & GCFLAG_WB_EXECUTED)) { + /* called from stm_validate(): + > if not was_read(), we certainly didn't modify + > if not WB_EXECUTED, we may have read from the obj in a different page but + did not modify it (should not occur right now, but future proof!) + only the WB_EXECUTED alone is not enough, since we may have imported from a + segment's private page (which had the flag set) */ + assert(IMPLY(_stm_was_read(obj), (obj->stm_flags & GCFLAG_WB_EXECUTED))); /* for now */ continue; /* only copy unmodified */ + } dprintf(("import slice seg=%d obj=%p off=%lu sz=%d pg=%lu\n", from_segnum, obj, SLICE_OFFSET(undo->slice), @@ -46,6 +54,11 @@ src = undo->backup + SLICE_OFFSET(undo->slice); dst = REAL_ADDRESS(STM_SEGMENT->segment_base, oslice); memcpy(dst, src, SLICE_SIZE(undo->slice)); + + if (src_segment_base == NULL) { + /* backups never should have WB_EXECUTED */ + assert(!(obj->stm_flags & GCFLAG_WB_EXECUTED)); + } } } @@ -57,7 +70,8 @@ { /* looks at all bk copies of objects overlapping page 'pagenum' and copies the part in 'pagenum' back to the current segment */ - dprintf(("copy_bk_objs_in_page_from(%d, %lu)\n", from_segnum, pagenum)); + dprintf(("copy_bk_objs_in_page_from(%d, %lu, %d)\n", + from_segnum, pagenum, only_if_not_modified)); struct list_s *list = get_priv_segment(from_segnum)->modified_old_objects; struct stm_undo_s *undo = (struct stm_undo_s *)list->items; @@ -429,17 +443,13 @@ /* add to read set: */ stm_read(obj); - /* XXX: add overflow number again? n^2 algorithm ahead... */ - struct list_s *list = STM_PSEGMENT->modified_old_objects; - int i, c = list_count(list); - for (i = 0; i < c; i += 3) { - if (list->items[i] == (uintptr_t)obj) { - /* already executed WB once in this transaction. do GC - part again: */ - obj->stm_flags &= ~GCFLAG_WRITE_BARRIER; - LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, obj); - return; - } + if (obj->stm_flags & GCFLAG_WB_EXECUTED) { + /* already executed WB once in this transaction. do GC + part again: */ + dprintf(("write_slowpath-fast(%p)\n", obj)); + obj->stm_flags &= ~GCFLAG_WRITE_BARRIER; + LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, obj); + return; } /* create backup copy (this may cause several page faults @@ -447,6 +457,7 @@ pages around anyway (kind of card marking)): */ struct object_s *bk_obj = malloc(obj_size); memcpy(bk_obj, realobj, obj_size); + assert(!(bk_obj->stm_flags & GCFLAG_WB_EXECUTED)); dprintf(("write_slowpath(%p): sz=%lu, bk=%p\n", obj, obj_size, bk_obj)); retry: @@ -483,6 +494,7 @@ update the shared page in stm_validate() except if it is the sole reader of it. But then we don't actually know which revision the page is at. */ /* XXX this is a temporary solution I suppose */ + int i; for (i = 0; i < NB_SEGMENTS; i++) { if (i == my_segnum) continue; @@ -498,12 +510,6 @@ /* all pages are either private or we were the first to write to a shared page and therefore got it as our private one */ - /* remove the WRITE_BARRIER flag */ - obj->stm_flags &= ~GCFLAG_WRITE_BARRIER; - - /* also add it to the GC list for minor collections */ - LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, obj); - /* phew, now add the obj to the write-set and register the backup copy. */ /* XXX: we should not be here at all fiddling with page status @@ -538,9 +544,15 @@ OPT_ASSERT(remaining_obj_sz == 0); release_modified_objs_lock(STM_SEGMENT->segment_num); - /* done fiddling with protection and privatization */ release_all_privatization_locks(); + + /* remove the WRITE_BARRIER flag and add WB_EXECUTED */ + obj->stm_flags &= ~GCFLAG_WRITE_BARRIER; + obj->stm_flags |= GCFLAG_WB_EXECUTED; + + /* also add it to the GC list for minor collections */ + LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, obj); } static void reset_transaction_read_version(void) @@ -563,6 +575,18 @@ STM_SEGMENT->transaction_read_version = 1; } +static void reset_wb_executed_flags(void) +{ + struct list_s *list = STM_PSEGMENT->modified_old_objects; + struct stm_undo_s *undo = (struct stm_undo_s *)list->items; + struct stm_undo_s *end = (struct stm_undo_s *)(list->items + list->count); + + for (; undo < end; undo++) { + object_t *obj = undo->object; + obj->stm_flags &= ~GCFLAG_WB_EXECUTED; + } +} + static void _stm_start_transaction(stm_thread_local_t *tl) { @@ -660,6 +684,7 @@ object_t *obj = undo->object; char *dst = REAL_ADDRESS(segbase, obj); assert(((struct object_s *)dst)->stm_flags & GCFLAG_WRITE_BARRIER); + assert(!(((struct object_s *)dst)->stm_flags & GCFLAG_WB_EXECUTED)); } #endif } @@ -673,6 +698,8 @@ dprintf(("> stm_commit_transaction()\n")); minor_collection(1); + reset_wb_executed_flags(); + /* minor_collection() above should have set again all WRITE_BARRIER flags. Check that again here for the objects that are about to be copied into the commit log. */ @@ -739,7 +766,6 @@ list_clear(list); release_modified_objs_lock(segment_num); - #pragma pop_macro("STM_SEGMENT") #pragma pop_macro("STM_PSEGMENT") } diff --git a/c8/stm/core.h b/c8/stm/core.h --- a/c8/stm/core.h +++ b/c8/stm/core.h @@ -37,9 +37,11 @@ enum /* stm_flags */ { GCFLAG_WRITE_BARRIER = _STM_GCFLAG_WRITE_BARRIER, GCFLAG_HAS_SHADOW = 0x02, + GCFLAG_WB_EXECUTED = 0x04, }; + /************************************************************/ diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c --- a/c8/stm/nursery.c +++ b/c8/stm/nursery.c @@ -164,7 +164,9 @@ struct list_s *lst = STM_PSEGMENT->objects_pointing_to_nursery; while (!list_is_empty(lst)) { - object_t *obj = (object_t *)list_pop_item(lst);; + object_t *obj = (object_t *)list_pop_item(lst); + + assert(!_is_in_nursery(obj)); _collect_now(obj); diff --git a/c8/test/test_random.py b/c8/test/test_random.py --- a/c8/test/test_random.py +++ b/c8/test/test_random.py @@ -8,10 +8,12 @@ def __init__(self, test): self.content = {'self': test} self.thread_num = 0 + self.executed = [] def do(self, cmd): color = ">> \033[%dm" % (31 + (self.thread_num + 5) % 6) print >> sys.stderr, color + cmd + "\033[0m" + self.executed.append(cmd) exec cmd in globals(), self.content @@ -579,7 +581,7 @@ op_minor_collect, #op_major_collect, ] - for _ in range(500): + for _ in range(1000): # make sure we are in a transaction: curr_thread = op_switch_thread(ex, global_state, curr_thread) @@ -616,6 +618,6 @@ test_fun.__name__ = 'test_random_%d' % seed return test_fun - for _seed in range(5000, 5200): + for _seed in range(5000, 5400): _fn = _make_fun(_seed) locals()[_fn.__name__] = _fn From noreply at buildbot.pypy.org Tue Sep 30 18:51:52 2014 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 30 Sep 2014 18:51:52 +0200 (CEST) Subject: [pypy-commit] pypy default: Remove the crazy copying of license files from the current system. Message-ID: <20140930165152.775311C356F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r73747:97f975585866 Date: 2014-09-30 18:51 +0200 http://bitbucket.org/pypy/pypy/changeset/97f975585866/ Log: Remove the crazy copying of license files from the current system. Instead, write some static blurb of text that I hope is better anyway. diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -367,3 +367,43 @@ Detailed license information is contained in the NOTICE file in the directory. + +Licenses and Acknowledgements for Incorporated Software +======================================================= + +This section is an incomplete, but growing list of licenses and +acknowledgements for third-party software incorporated in the PyPy +distribution. + +License for 'Tcl/Tk' +-------------------- + +This copy of PyPy contains library code that may, when used, result in +the Tcl/Tk library to be loaded. PyPy also includes code that may be +regarded as being a copy of some parts of the Tcl/Tk header files. +You may see a copy of the License for Tcl/Tk in the file +`lib_pypy/_tkinter/license.terms` included here. + +License for 'bzip2' +------------------- + +This copy of PyPy may be linked (dynamically or statically) with the +bzip2 library. You may see a copy of the License for bzip2/libbzip2 at + + http://www.bzip.org/1.0.5/bzip2-manual-1.0.5.html + +License for 'openssl' +--------------------- + +This copy of PyPy may be linked (dynamically or statically) with the +openssl library. You may see a copy of the License for OpenSSL at + + https://www.openssl.org/source/license.html + +License for 'gdbm' +------------------ + +The gdbm module includes code from gdbm.h, which is distributed under +the terms of the GPL license version 2 or any later version. Thus the +gdbm module, provided in the file lib_pypy/gdbm.py, is redistributed +under the terms of the GPL license as well. diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -49,6 +49,16 @@ os.system("chmod -R a+rX %s" % dirname) os.system("chmod -R g-w %s" % dirname) + +# +# Some crazy nonsense (imho) about including automatically the license +# of various libraries as they happen to be on this system. This is +# strange because most of these libraries are linked to dynamically, +# and so at runtime might end up with a different version. I (arigo) +# killed this logic and wrote some general info (which I hope is more +# sensible anyway) into our ../../../LICENSE file. +# +''' sep_template = "\nThis copy of PyPy includes a copy of %s, which is licensed under the following terms:\n\n" def generate_license(basedir, options): @@ -95,6 +105,7 @@ # Do something for gdbm, which is GPL txt += gdbm_bit return txt +''' def create_cffi_import_libraries(pypy_c, options): modules = ['_sqlite3'] @@ -216,19 +227,19 @@ for file in ['_testcapimodule.c', '_ctypes_test.c']: shutil.copyfile(str(basedir.join('lib_pypy', file)), str(pypydir.join('lib_pypy', file))) - try: + if 0: # disabled license = generate_license(basedir, options) with open(str(pypydir.join('LICENSE')), 'w') as LICENSE: LICENSE.write(license) - except: - # Non-fatal error, use original LICENCE file - import traceback;traceback.print_exc() + else: + # Use original LICENCE file + #import traceback;traceback.print_exc() base_file = str(basedir.join('LICENSE')) with open(base_file) as fid: license = fid.read() with open(str(pypydir.join('LICENSE')), 'w') as LICENSE: LICENSE.write(license) - retval = -1 + #retval = -1 # spdir = pypydir.ensure('site-packages', dir=True) shutil.copy(str(basedir.join('site-packages', 'README')), str(spdir)) @@ -321,7 +332,8 @@ parser.add_argument('--archive-name', dest='name', type=str, default='', help='pypy-VER-PLATFORM') parser.add_argument('--license_base', type=str, default=license_base, - help='where to start looking for third party upstream licensing info') + #help='where to start looking for third party upstream licensing info') + help='(ignored)') parser.add_argument('--builddir', type=str, default='', help='tmp dir for packaging') parser.add_argument('--targetdir', type=str, default='', @@ -356,24 +368,6 @@ return create_package(basedir, options) -third_party_header = '''\n\nLicenses and Acknowledgements for Incorporated Software -======================================================= - -This section is an incomplete, but growing list of licenses and acknowledgements -for third-party software incorporated in the PyPy distribution. - -''' - -gdbm_bit = '''gdbm ----- - -The gdbm module includes code from gdbm.h, which is distributed under the terms -of the GPL license version 2 or any later version. Thus the gdbm module, provided in -the file lib_pypy/gdbm.py, is redistributed under the terms of the GPL license as -well. -''' - - if __name__ == '__main__': import sys if sys.platform == 'win32':