Building from source

Branch: Changeset: r69962:66662b92489d Date: 2014-03-14 12:16 -0700 http://bitbucket.org/pypy/pypy/changeset/66662b92489d/ Log: py3k compat: avoid cmp diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py --- a/lib_pypy/_sqlite3.py +++ b/lib_pypy/_sqlite3.py @@ -778,7 +778,7 @@ try: ret = callback(text1, text2) assert isinstance(ret, (int, long)) - return cmp(ret, 0) + return (ret > 0) - (ret < 0) except Exception: return 0 From noreply at buildbot.pypy.org Fri Mar 14 20:37:53 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Fri, 14 Mar 2014 20:37:53 +0100 (CET) Subject: [pypy-commit] pypy default: still use cmp builtin on py2, only apply manual case for py3 Message-ID: <20140314193753.35F781C3058@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r69963:ddbc70485efb Date: 2014-03-14 15:36 -0400 http://bitbucket.org/pypy/pypy/changeset/ddbc70485efb/ Log: still use cmp builtin on py2, only apply manual case for py3 diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py --- a/lib_pypy/_sqlite3.py +++ b/lib_pypy/_sqlite3.py @@ -38,6 +38,7 @@ if sys.version_info[0] >= 3: StandardError = Exception + cmp = lambda x, y: (x > y) - (x < y) long = int xrange = range basestring = unicode = str @@ -778,7 +779,7 @@ try: ret = callback(text1, text2) assert isinstance(ret, (int, long)) - return (ret > 0) - (ret < 0) + return cmp(ret, 0) except Exception: return 0 From noreply at buildbot.pypy.org Fri Mar 14 22:02:27 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Fri, 14 Mar 2014 22:02:27 +0100 (CET) Subject: [pypy-commit] pypy default: cleanup Message-ID: <20140314210227.9D27F1C06AD@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r69964:d5b78e64b2f0 Date: 2014-03-14 16:58 -0400 http://bitbucket.org/pypy/pypy/changeset/d5b78e64b2f0/ Log: cleanup diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -1,14 +1,15 @@ import py from rpython.rlib.objectmodel import instantiate +from rpython.jit.metainterp import compile, resume +from rpython.jit.metainterp.history import AbstractDescr, ConstInt, BoxInt, TreeLoop +from rpython.jit.metainterp.optimize import InvalidLoop +from rpython.jit.metainterp.optimizeopt import build_opt_chain from rpython.jit.metainterp.optimizeopt.test.test_util import ( LLtypeMixin, BaseTest, convert_old_style_to_targets) -from rpython.jit.metainterp.optimizeopt import build_opt_chain -from rpython.jit.metainterp.optimize import InvalidLoop -from rpython.jit.metainterp.history import AbstractDescr, ConstInt, BoxInt -from rpython.jit.metainterp.history import TreeLoop -from rpython.jit.metainterp import compile, resume +from rpython.jit.metainterp.optimizeopt.test.test_optimizebasic import \ + FakeMetaInterpStaticData from rpython.jit.metainterp.resoperation import rop, opname, oparity -from rpython.jit.metainterp.optimizeopt.test.test_optimizebasic import FakeMetaInterpStaticData + def test_build_opt_chain(): def check(chain, expected_names): @@ -40,7 +41,6 @@ class BaseTestWithUnroll(BaseTest): - enable_opts = "intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll" def optimize_loop(self, ops, expected, expected_preamble=None, @@ -93,8 +93,8 @@ def raises(self, e, fn, *args): return py.test.raises(e, fn, *args).value + class OptimizeOptTest(BaseTestWithUnroll): - def setup_method(self, meth=None): class FailDescr(compile.ResumeGuardDescr): oparse = None @@ -130,7 +130,6 @@ self.namespace.pop('fdescr', None) self.namespace.pop('fdescr2', None) - def test_simple(self): ops = """ [] @@ -974,7 +973,6 @@ """ self.optimize_loop(ops, expected, preamble) - # ---------- def test_virtual_1(self): @@ -1252,7 +1250,6 @@ """ self.optimize_loop(ops, expected, preamble) - def test_virtual_constant_isnonnull(self): ops = """ [i0] @@ -2789,8 +2786,7 @@ p2 = new_with_vtable(ConstClass(node_vtable)) jump(p2) """ - self.raises(InvalidLoop, self.optimize_loop, - ops, "crash!") + self.raises(InvalidLoop, self.optimize_loop, ops, "crash!") def test_invalid_loop_2(self): ops = """ @@ -2801,8 +2797,7 @@ escape(p2) # prevent it from staying Virtual jump(p2) """ - self.raises(InvalidLoop, self.optimize_loop, - ops, "crash!") + self.raises(InvalidLoop, self.optimize_loop, ops, "crash!") def test_invalid_loop_3(self): ops = """ @@ -2824,8 +2819,7 @@ guard_value(p2, ConstPtr(myptr)) [] jump(p2) """ - exc = self.raises(InvalidLoop, self.optimize_loop, - ops, "crash!") + exc = self.raises(InvalidLoop, self.optimize_loop, ops, "crash!") if exc: assert "node" in exc.msg @@ -3151,7 +3145,6 @@ """ self.optimize_loop(ops, expected) - def test_int_and_or_with_zero(self): ops = """ [i0, i1] @@ -5107,7 +5100,6 @@ """ self.optimize_loop(ops, expected) - def test_division_nonneg(self): py.test.skip("harder") # this is how an app-level division turns into right now @@ -5444,7 +5436,6 @@ """ self.optimize_loop(ops, ops, ops) - def test_mul_ovf(self): ops = """ [i0, i1] @@ -5591,7 +5582,6 @@ def is_integer_bounded(self): return False - for n in ('inst_w_seq', 'inst_index', 'inst_w_list', 'inst_length', 'inst_start', 'inst_step'): self.namespace[n] = FakeDescr(n) @@ -5847,7 +5837,7 @@ self.optimize_loop(ops, optops, preamble) # check with replacing 'str' with 'unicode' everywhere def r(s): - return s.replace('str','unicode').replace('s"', 'u"') + return s.replace('str', 'unicode').replace('s"', 'u"') self.optimize_loop(r(ops), r(optops), r(preamble)) def test_newstr_1(self): @@ -6277,7 +6267,7 @@ if isinstance(value, calldescrtype): extra = value.get_extra_info() if (extra and isinstance(extra, effectinfotype) and - extra.oopspecindex == oopspecindex): + extra.oopspecindex == oopspecindex): # returns 0 for 'func' in this test return value, 0 raise AssertionError("not found: oopspecindex=%d" % @@ -7395,7 +7385,6 @@ """ self.optimize_loop(ops, expected, expected_short=short) - def test_loopinvariant_constant_strgetitem(self): ops = """ [p0] @@ -7454,7 +7443,7 @@ """ self.optimize_loop(ops, expected, expected_short=short) - def test_propagate_virtual_arryalen(self): + def test_propagate_virtual_arraylen(self): ops = """ [p0] p404 = new_array(2, descr=arraydescr) @@ -7831,7 +7820,6 @@ """ self.optimize_loop(ops, expected) - def test_setarrayitem_followed_by_arraycopy(self): ops = """ [p1, p2] @@ -8124,7 +8112,6 @@ """ self.optimize_loop(ops, expected) - def test_issue1080_infinitie_loop_simple(self): ops = """ [p69] @@ -8149,8 +8136,7 @@ guard_value(p1, ConstPtr(myptr)) [] jump(p1) """ - self.raises(InvalidLoop, self.optimize_loop, - ops, ops) + self.raises(InvalidLoop, self.optimize_loop, ops, ops) def test_licm_boxed_opaque_getitem(self): ops = """ @@ -8225,8 +8211,7 @@ guard_value(p1, ConstPtr(myptr)) [] jump(p1) """ - self.raises(InvalidLoop, self.optimize_loop, - ops, ops) + self.raises(InvalidLoop, self.optimize_loop, ops, ops) def test_cond_call_with_a_constant(self): ops = """ @@ -8263,6 +8248,6 @@ """ self.optimize_loop(ops, ops) + class TestLLtype(OptimizeOptTest, LLtypeMixin): pass - From noreply at buildbot.pypy.org Fri Mar 14 22:39:37 2014 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 14 Mar 2014 22:39:37 +0100 (CET) Subject: [pypy-commit] stmgc default: Add ((always_inline)) on the three major places that *must* be inlined Message-ID: <20140314213937.156311C0124@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1015:240faf17faf5 Date: 2014-03-14 22:39 +0100 http://bitbucket.org/pypy/stmgc/changeset/240faf17faf5/ Log: Add ((always_inline)) on the three major places that *must* be inlined diff --git a/c7/stmgc.h b/c7/stmgc.h --- a/c7/stmgc.h +++ b/c7/stmgc.h @@ -147,6 +147,7 @@ stm_write() is called, or immediately after getting the object from stm_allocate(), as long as the rules above are respected. */ +__attribute__((always_inline)) static inline void stm_read(object_t *obj) { ((stm_read_marker_t *)(((uintptr_t)obj) >> 4))->rm = @@ -158,6 +159,7 @@ the next one, then stm_write() needs to be called again. It is not necessary to call it immediately after stm_allocate(). */ +__attribute__((always_inline)) static inline void stm_write(object_t *obj) { if (UNLIKELY((obj->stm_flags & _STM_GCFLAG_WRITE_BARRIER) != 0)) @@ -177,6 +179,7 @@ of 8 and at least 16. In the fast-path, this is inlined to just a few assembler instructions. */ +__attribute__((always_inline)) static inline object_t *stm_allocate(ssize_t size_rounded_up) { OPT_ASSERT(size_rounded_up >= 16); diff --git a/gil-c7/stmgc.h b/gil-c7/stmgc.h --- a/gil-c7/stmgc.h +++ b/gil-c7/stmgc.h @@ -52,6 +52,7 @@ object_t *_stm_allocate_slowpath(ssize_t); object_t *stm_allocate_weakref(ssize_t size_rounded_up); +__attribute__((always_inline)) inline static object_t *stm_allocate(ssize_t size_rounded_up) { OPT_ASSERT(size_rounded_up >= 16); OPT_ASSERT((size_rounded_up & 7) == 0); @@ -101,6 +102,7 @@ void _stm_write_slowpath(object_t *); +__attribute__((always_inline)) inline static void stm_write(object_t *ob) { if (UNLIKELY(ob->gil_flags & _STM_GCFLAG_WRITE_BARRIER)) _stm_write_slowpath(ob); From noreply at buildbot.pypy.org Sat Mar 15 01:48:16 2014 From: noreply at buildbot.pypy.org (pjenvey) Date: Sat, 15 Mar 2014 01:48:16 +0100 (CET) Subject: [pypy-commit] pypy stdlib-3.2.5: merge vendor/stdlib Message-ID: <20140315004816.EA9A61C3058@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: stdlib-3.2.5 Changeset: r69965:cbb08b3b1f21 Date: 2014-03-14 17:40 -0700 http://bitbucket.org/pypy/pypy/changeset/cbb08b3b1f21/ Log: merge vendor/stdlib diff too long, truncating to 2000 out of 27211 lines diff --git a/lib-python/3/__future__.py b/lib-python/3/__future__.py --- a/lib-python/3/__future__.py +++ b/lib-python/3/__future__.py @@ -114,7 +114,7 @@ CO_FUTURE_DIVISION) absolute_import = _Feature((2, 5, 0, "alpha", 1), - (2, 7, 0, "alpha", 0), + (3, 0, 0, "alpha", 0), CO_FUTURE_ABSOLUTE_IMPORT) with_statement = _Feature((2, 5, 0, "alpha", 1), diff --git a/lib-python/3/_abcoll.py b/lib-python/3/_abcoll.py --- a/lib-python/3/_abcoll.py +++ b/lib-python/3/_abcoll.py @@ -184,12 +184,12 @@ def __gt__(self, other): if not isinstance(other, Set): return NotImplemented - return other < self + return other.__lt__(self) def __ge__(self, other): if not isinstance(other, Set): return NotImplemented - return other <= self + return other.__le__(self) def __eq__(self, other): if not isinstance(other, Set): diff --git a/lib-python/3/_osx_support.py b/lib-python/3/_osx_support.py new file mode 100644 --- /dev/null +++ b/lib-python/3/_osx_support.py @@ -0,0 +1,488 @@ +"""Shared OS X support functions.""" + +import os +import re +import sys + +__all__ = [ + 'compiler_fixup', + 'customize_config_vars', + 'customize_compiler', + 'get_platform_osx', +] + +# configuration variables that may contain universal build flags, +# like "-arch" or "-isdkroot", that may need customization for +# the user environment +_UNIVERSAL_CONFIG_VARS = ('CFLAGS', 'LDFLAGS', 'CPPFLAGS', 'BASECFLAGS', + 'BLDSHARED', 'LDSHARED', 'CC', 'CXX', + 'PY_CFLAGS', 'PY_LDFLAGS', 'PY_CPPFLAGS', + 'PY_CORE_CFLAGS') + +# configuration variables that may contain compiler calls +_COMPILER_CONFIG_VARS = ('BLDSHARED', 'LDSHARED', 'CC', 'CXX') + +# prefix added to original configuration variable names +_INITPRE = '_OSX_SUPPORT_INITIAL_' + + +def _find_executable(executable, path=None): + """Tries to find 'executable' in the directories listed in 'path'. + + A string listing directories separated by 'os.pathsep'; defaults to + os.environ['PATH']. Returns the complete filename or None if not found. + """ + if path is None: + path = os.environ['PATH'] + + paths = path.split(os.pathsep) + base, ext = os.path.splitext(executable) + + if (sys.platform == 'win32' or os.name == 'os2') and (ext != '.exe'): + executable = executable + '.exe' + + if not os.path.isfile(executable): + for p in paths: + f = os.path.join(p, executable) + if os.path.isfile(f): + # the file exists, we have a shot at spawn working + return f + return None + else: + return executable + + +def _read_output(commandstring): + """Output from succesful command execution or None""" + # Similar to os.popen(commandstring, "r").read(), + # but without actually using os.popen because that + # function is not usable during python bootstrap. + # tempfile is also not available then. + import contextlib + try: + import tempfile + fp = tempfile.NamedTemporaryFile() + except ImportError: + fp = open("/tmp/_osx_support.%s"%( + os.getpid(),), "w+b") + + with contextlib.closing(fp) as fp: + cmd = "%s 2>/dev/null >'%s'" % (commandstring, fp.name) + return fp.read().decode('utf-8').strip() if not os.system(cmd) else None + + +def _find_build_tool(toolname): + """Find a build tool on current path or using xcrun""" + return (_find_executable(toolname) + or _read_output("/usr/bin/xcrun -find %s" % (toolname,)) + or '' + ) + +_SYSTEM_VERSION = None + +def _get_system_version(): + """Return the OS X system version as a string""" + # Reading this plist is a documented way to get the system + # version (see the documentation for the Gestalt Manager) + # We avoid using platform.mac_ver to avoid possible bootstrap issues during + # the build of Python itself (distutils is used to build standard library + # extensions). + + global _SYSTEM_VERSION + + if _SYSTEM_VERSION is None: + _SYSTEM_VERSION = '' + try: + f = open('/System/Library/CoreServices/SystemVersion.plist') + except IOError: + # We're on a plain darwin box, fall back to the default + # behaviour. + pass + else: + try: + m = re.search(r'ProductUserVisibleVersion\s*' + r'(.*?)', f.read()) + finally: + f.close() + if m is not None: + _SYSTEM_VERSION = '.'.join(m.group(1).split('.')[:2]) + # else: fall back to the default behaviour + + return _SYSTEM_VERSION + +def _remove_original_values(_config_vars): + """Remove original unmodified values for testing""" + # This is needed for higher-level cross-platform tests of get_platform. + for k in list(_config_vars): + if k.startswith(_INITPRE): + del _config_vars[k] + +def _save_modified_value(_config_vars, cv, newvalue): + """Save modified and original unmodified value of configuration var""" + + oldvalue = _config_vars.get(cv, '') + if (oldvalue != newvalue) and (_INITPRE + cv not in _config_vars): + _config_vars[_INITPRE + cv] = oldvalue + _config_vars[cv] = newvalue + +def _supports_universal_builds(): + """Returns True if universal builds are supported on this system""" + # As an approximation, we assume that if we are running on 10.4 or above, + # then we are running with an Xcode environment that supports universal + # builds, in particular -isysroot and -arch arguments to the compiler. This + # is in support of allowing 10.4 universal builds to run on 10.3.x systems. + + osx_version = _get_system_version() + if osx_version: + try: + osx_version = tuple(int(i) for i in osx_version.split('.')) + except ValueError: + osx_version = '' + return bool(osx_version >= (10, 4)) if osx_version else False + + +def _find_appropriate_compiler(_config_vars): + """Find appropriate C compiler for extension module builds""" + + # Issue #13590: + # The OSX location for the compiler varies between OSX + # (or rather Xcode) releases. With older releases (up-to 10.5) + # the compiler is in /usr/bin, with newer releases the compiler + # can only be found inside Xcode.app if the "Command Line Tools" + # are not installed. + # + # Futhermore, the compiler that can be used varies between + # Xcode releases. Upto Xcode 4 it was possible to use 'gcc-4.2' + # as the compiler, after that 'clang' should be used because + # gcc-4.2 is either not present, or a copy of 'llvm-gcc' that + # miscompiles Python. + + # skip checks if the compiler was overriden with a CC env variable + if 'CC' in os.environ: + return _config_vars + + # The CC config var might contain additional arguments. + # Ignore them while searching. + cc = oldcc = _config_vars['CC'].split()[0] + if not _find_executable(cc): + # Compiler is not found on the shell search PATH. + # Now search for clang, first on PATH (if the Command LIne + # Tools have been installed in / or if the user has provided + # another location via CC). If not found, try using xcrun + # to find an uninstalled clang (within a selected Xcode). + + # NOTE: Cannot use subprocess here because of bootstrap + # issues when building Python itself (and os.popen is + # implemented on top of subprocess and is therefore not + # usable as well) + + cc = _find_build_tool('clang') + + elif os.path.basename(cc).startswith('gcc'): + # Compiler is GCC, check if it is LLVM-GCC + data = _read_output("'%s' --version" + % (cc.replace("'", "'\"'\"'"),)) + if 'llvm-gcc' in data: + # Found LLVM-GCC, fall back to clang + cc = _find_build_tool('clang') + + if not cc: + raise SystemError( + "Cannot locate working compiler") + + if cc != oldcc: + # Found a replacement compiler. + # Modify config vars using new compiler, if not already explictly + # overriden by an env variable, preserving additional arguments. + for cv in _COMPILER_CONFIG_VARS: + if cv in _config_vars and cv not in os.environ: + cv_split = _config_vars[cv].split() + cv_split[0] = cc if cv != 'CXX' else cc + '++' + _save_modified_value(_config_vars, cv, ' '.join(cv_split)) + + return _config_vars + + +def _remove_universal_flags(_config_vars): + """Remove all universal build arguments from config vars""" + + for cv in _UNIVERSAL_CONFIG_VARS: + # Do not alter a config var explicitly overriden by env var + if cv in _config_vars and cv not in os.environ: + flags = _config_vars[cv] + flags = re.sub('-arch\s+\w+\s', ' ', flags, re.ASCII) + flags = re.sub('-isysroot [^ \t]*', ' ', flags) + _save_modified_value(_config_vars, cv, flags) + + return _config_vars + + +def _remove_unsupported_archs(_config_vars): + """Remove any unsupported archs from config vars""" + # Different Xcode releases support different sets for '-arch' + # flags. In particular, Xcode 4.x no longer supports the + # PPC architectures. + # + # This code automatically removes '-arch ppc' and '-arch ppc64' + # when these are not supported. That makes it possible to + # build extensions on OSX 10.7 and later with the prebuilt + # 32-bit installer on the python.org website. + + # skip checks if the compiler was overriden with a CC env variable + if 'CC' in os.environ: + return _config_vars + + if re.search('-arch\s+ppc', _config_vars['CFLAGS']) is not None: + # NOTE: Cannot use subprocess here because of bootstrap + # issues when building Python itself + status = os.system("'%s' -arch ppc -x c /dev/null 2>/dev/null"%( + _config_vars['CC'].replace("'", "'\"'\"'"),)) + # The Apple compiler drivers return status 255 if no PPC + if (status >> 8) == 255: + # Compiler doesn't support PPC, remove the related + # '-arch' flags if not explicitly overridden by an + # environment variable + for cv in _UNIVERSAL_CONFIG_VARS: + if cv in _config_vars and cv not in os.environ: + flags = _config_vars[cv] + flags = re.sub('-arch\s+ppc\w*\s', ' ', flags) + _save_modified_value(_config_vars, cv, flags) + + return _config_vars + + +def _override_all_archs(_config_vars): + """Allow override of all archs with ARCHFLAGS env var""" + # NOTE: This name was introduced by Apple in OSX 10.5 and + # is used by several scripting languages distributed with + # that OS release. + if 'ARCHFLAGS' in os.environ: + arch = os.environ['ARCHFLAGS'] + for cv in _UNIVERSAL_CONFIG_VARS: + if cv in _config_vars and '-arch' in _config_vars[cv]: + flags = _config_vars[cv] + flags = re.sub('-arch\s+\w+\s', ' ', flags) + flags = flags + ' ' + arch + _save_modified_value(_config_vars, cv, flags) + + return _config_vars + + +def _check_for_unavailable_sdk(_config_vars): + """Remove references to any SDKs not available""" + # If we're on OSX 10.5 or later and the user tries to + # compile an extension using an SDK that is not present + # on the current machine it is better to not use an SDK + # than to fail. This is particularly important with + # the standalong Command Line Tools alternative to a + # full-blown Xcode install since the CLT packages do not + # provide SDKs. If the SDK is not present, it is assumed + # that the header files and dev libs have been installed + # to /usr and /System/Library by either a standalone CLT + # package or the CLT component within Xcode. + cflags = _config_vars.get('CFLAGS', '') + m = re.search(r'-isysroot\s+(\S+)', cflags) + if m is not None: + sdk = m.group(1) + if not os.path.exists(sdk): + for cv in _UNIVERSAL_CONFIG_VARS: + # Do not alter a config var explicitly overriden by env var + if cv in _config_vars and cv not in os.environ: + flags = _config_vars[cv] + flags = re.sub(r'-isysroot\s+\S+(?:\s|$)', ' ', flags) + _save_modified_value(_config_vars, cv, flags) + + return _config_vars + + +def compiler_fixup(compiler_so, cc_args): + """ + This function will strip '-isysroot PATH' and '-arch ARCH' from the + compile flags if the user has specified one them in extra_compile_flags. + + This is needed because '-arch ARCH' adds another architecture to the + build, without a way to remove an architecture. Furthermore GCC will + barf if multiple '-isysroot' arguments are present. + """ + stripArch = stripSysroot = False + + compiler_so = list(compiler_so) + + if not _supports_universal_builds(): + # OSX before 10.4.0, these don't support -arch and -isysroot at + # all. + stripArch = stripSysroot = True + else: + stripArch = '-arch' in cc_args + stripSysroot = '-isysroot' in cc_args + + if stripArch or 'ARCHFLAGS' in os.environ: + while True: + try: + index = compiler_so.index('-arch') + # Strip this argument and the next one: + del compiler_so[index:index+2] + except ValueError: + break + + if 'ARCHFLAGS' in os.environ and not stripArch: + # User specified different -arch flags in the environ, + # see also distutils.sysconfig + compiler_so = compiler_so + os.environ['ARCHFLAGS'].split() + + if stripSysroot: + while True: + try: + index = compiler_so.index('-isysroot') + # Strip this argument and the next one: + del compiler_so[index:index+2] + except ValueError: + break + + # Check if the SDK that is used during compilation actually exists, + # the universal build requires the usage of a universal SDK and not all + # users have that installed by default. + sysroot = None + if '-isysroot' in cc_args: + idx = cc_args.index('-isysroot') + sysroot = cc_args[idx+1] + elif '-isysroot' in compiler_so: + idx = compiler_so.index('-isysroot') + sysroot = compiler_so[idx+1] + + if sysroot and not os.path.isdir(sysroot): + from distutils import log + log.warn("Compiling with an SDK that doesn't seem to exist: %s", + sysroot) + log.warn("Please check your Xcode installation") + + return compiler_so + + +def customize_config_vars(_config_vars): + """Customize Python build configuration variables. + + Called internally from sysconfig with a mutable mapping + containing name/value pairs parsed from the configured + makefile used to build this interpreter. Returns + the mapping updated as needed to reflect the environment + in which the interpreter is running; in the case of + a Python from a binary installer, the installed + environment may be very different from the build + environment, i.e. different OS levels, different + built tools, different available CPU architectures. + + This customization is performed whenever + distutils.sysconfig.get_config_vars() is first + called. It may be used in environments where no + compilers are present, i.e. when installing pure + Python dists. Customization of compiler paths + and detection of unavailable archs is deferred + until the first extention module build is + requested (in distutils.sysconfig.customize_compiler). + + Currently called from distutils.sysconfig + """ + + if not _supports_universal_builds(): + # On Mac OS X before 10.4, check if -arch and -isysroot + # are in CFLAGS or LDFLAGS and remove them if they are. + # This is needed when building extensions on a 10.3 system + # using a universal build of python. + _remove_universal_flags(_config_vars) + + # Allow user to override all archs with ARCHFLAGS env var + _override_all_archs(_config_vars) + + # Remove references to sdks that are not found + _check_for_unavailable_sdk(_config_vars) + + return _config_vars + + +def customize_compiler(_config_vars): + """Customize compiler path and configuration variables. + + This customization is performed when the first + extension module build is requested + in distutils.sysconfig.customize_compiler). + """ + + # Find a compiler to use for extension module builds + _find_appropriate_compiler(_config_vars) + + # Remove ppc arch flags if not supported here + _remove_unsupported_archs(_config_vars) + + # Allow user to override all archs with ARCHFLAGS env var + _override_all_archs(_config_vars) + + return _config_vars + + +def get_platform_osx(_config_vars, osname, release, machine): + """Filter values for get_platform()""" + # called from get_platform() in sysconfig and distutils.util + # + # For our purposes, we'll assume that the system version from + # distutils' perspective is what MACOSX_DEPLOYMENT_TARGET is set + # to. This makes the compatibility story a bit more sane because the + # machine is going to compile and link as if it were + # MACOSX_DEPLOYMENT_TARGET. + + macver = _config_vars.get('MACOSX_DEPLOYMENT_TARGET', '') + macrelease = _get_system_version() or macver + macver = macver or macrelease + + if macver: + release = macver + osname = "macosx" + + # Use the original CFLAGS value, if available, so that we + # return the same machine type for the platform string. + # Otherwise, distutils may consider this a cross-compiling + # case and disallow installs. + cflags = _config_vars.get(_INITPRE+'CFLAGS', + _config_vars.get('CFLAGS', '')) + if ((macrelease + '.') >= '10.4.' and + '-arch' in cflags.strip()): + # The universal build will build fat binaries, but not on + # systems before 10.4 + + machine = 'fat' + + archs = re.findall('-arch\s+(\S+)', cflags) + archs = tuple(sorted(set(archs))) + + if len(archs) == 1: + machine = archs[0] + elif archs == ('i386', 'ppc'): + machine = 'fat' + elif archs == ('i386', 'x86_64'): + machine = 'intel' + elif archs == ('i386', 'ppc', 'x86_64'): + machine = 'fat3' + elif archs == ('ppc64', 'x86_64'): + machine = 'fat64' + elif archs == ('i386', 'ppc', 'ppc64', 'x86_64'): + machine = 'universal' + else: + raise ValueError( + "Don't know machine value for archs=%r" % (archs,)) + + elif machine == 'i386': + # On OSX the machine type returned by uname is always the + # 32-bit variant, even if the executable architecture is + # the 64-bit variant + if sys.maxsize >= 2**32: + machine = 'x86_64' + + elif machine in ('PowerPC', 'Power_Macintosh'): + # Pick a sane name for the PPC architecture. + # See 'i386' case + if sys.maxsize >= 2**32: + machine = 'ppc64' + else: + machine = 'ppc' + + return (osname, release, machine) diff --git a/lib-python/3/_pyio.py b/lib-python/3/_pyio.py --- a/lib-python/3/_pyio.py +++ b/lib-python/3/_pyio.py @@ -298,7 +298,7 @@ def seek(self, pos, whence=0): """Change stream position. - Change the stream position to byte offset offset. offset is + Change the stream position to byte offset pos. Argument pos is interpreted relative to the position indicated by whence. Values for whence are ints: @@ -889,12 +889,18 @@ return pos def readable(self): + if self.closed: + raise ValueError("I/O operation on closed file.") return True def writable(self): + if self.closed: + raise ValueError("I/O operation on closed file.") return True def seekable(self): + if self.closed: + raise ValueError("I/O operation on closed file.") return True @@ -1567,6 +1573,8 @@ return self._buffer def seekable(self): + if self.closed: + raise ValueError("I/O operation on closed file.") return self._seekable def readable(self): diff --git a/lib-python/3/_strptime.py b/lib-python/3/_strptime.py --- a/lib-python/3/_strptime.py +++ b/lib-python/3/_strptime.py @@ -339,7 +339,7 @@ raise ValueError("unconverted data remains: %s" % data_string[found.end():]) - year = 1900 + year = None month = day = 1 hour = minute = second = fraction = 0 tz = -1 @@ -444,6 +444,12 @@ else: tz = value break + leap_year_fix = False + if year is None and month == 2 and day == 29: + year = 1904 # 1904 is first leap year of 20th century + leap_year_fix = True + elif year is None: + year = 1900 # If we know the week of the year and what day of that week, we can figure # out the Julian day of the year. if julian == -1 and week_of_year != -1 and weekday != -1: @@ -472,6 +478,12 @@ else: gmtoff = None + if leap_year_fix: + # the caller didn't supply a year but asked for Feb 29th. We couldn't + # use the default of 1900 for computations. We set it back to ensure + # that February 29th is smaller than March 1st. + year = 1900 + return (year, month, day, hour, minute, second, weekday, julian, tz, gmtoff, tzname), fraction diff --git a/lib-python/3/_weakrefset.py b/lib-python/3/_weakrefset.py --- a/lib-python/3/_weakrefset.py +++ b/lib-python/3/_weakrefset.py @@ -63,7 +63,7 @@ yield item def __len__(self): - return sum(x() is not None for x in self.data) + return len(self.data) - len(self._pending_removals) def __contains__(self, item): try: @@ -114,36 +114,21 @@ def update(self, other): if self._pending_removals: self._commit_removals() - if isinstance(other, self.__class__): - self.data.update(other.data) - else: - for element in other: - self.add(element) + for element in other: + self.add(element) def __ior__(self, other): self.update(other) return self - # Helper functions for simple delegating methods. - def _apply(self, other, method): - if not isinstance(other, self.__class__): - other = self.__class__(other) - newdata = method(other.data) - newset = self.__class__() - newset.data = newdata + def difference(self, other): + newset = self.copy() + newset.difference_update(other) return newset - - def difference(self, other): - return self._apply(other, self.data.difference) __sub__ = difference def difference_update(self, other): - if self._pending_removals: - self._commit_removals() - if self is other: - self.data.clear() - else: - self.data.difference_update(ref(item) for item in other) + self.__isub__(other) def __isub__(self, other): if self._pending_removals: self._commit_removals() @@ -154,13 +139,11 @@ return self def intersection(self, other): - return self._apply(other, self.data.intersection) + return self.__class__(item for item in other if item in self) __and__ = intersection def intersection_update(self, other): - if self._pending_removals: - self._commit_removals() - self.data.intersection_update(ref(item) for item in other) + self.__iand__(other) def __iand__(self, other): if self._pending_removals: self._commit_removals() @@ -169,17 +152,17 @@ def issubset(self, other): return self.data.issubset(ref(item) for item in other) - __lt__ = issubset + __le__ = issubset - def __le__(self, other): - return self.data <= set(ref(item) for item in other) + def __lt__(self, other): + return self.data < set(ref(item) for item in other) def issuperset(self, other): return self.data.issuperset(ref(item) for item in other) - __gt__ = issuperset + __ge__ = issuperset - def __ge__(self, other): - return self.data >= set(ref(item) for item in other) + def __gt__(self, other): + return self.data > set(ref(item) for item in other) def __eq__(self, other): if not isinstance(other, self.__class__): @@ -187,27 +170,24 @@ return self.data == set(ref(item) for item in other) def symmetric_difference(self, other): - return self._apply(other, self.data.symmetric_difference) + newset = self.copy() + newset.symmetric_difference_update(other) + return newset __xor__ = symmetric_difference def symmetric_difference_update(self, other): - if self._pending_removals: - self._commit_removals() - if self is other: - self.data.clear() - else: - self.data.symmetric_difference_update(ref(item) for item in other) + self.__ixor__(other) def __ixor__(self, other): if self._pending_removals: self._commit_removals() if self is other: self.data.clear() else: - self.data.symmetric_difference_update(ref(item) for item in other) + self.data.symmetric_difference_update(ref(item, self._remove) for item in other) return self def union(self, other): - return self._apply(other, self.data.union) + return self.__class__(e for s in (self, other) for e in s) __or__ = union def isdisjoint(self, other): diff --git a/lib-python/3/aifc.py b/lib-python/3/aifc.py --- a/lib-python/3/aifc.py +++ b/lib-python/3/aifc.py @@ -692,7 +692,9 @@ self._patchheader() def close(self): - if self._file: + if self._file is None: + return + try: self._ensure_header_written(0) if self._datawritten & 1: # quick pad to even size @@ -703,10 +705,12 @@ self._datalength != self._datawritten or \ self._marklength: self._patchheader() + finally: # Prevent ref cycles self._convert = None - self._file.close() + f = self._file self._file = None + f.close() # # Internal methods. diff --git a/lib-python/3/argparse.py b/lib-python/3/argparse.py --- a/lib-python/3/argparse.py +++ b/lib-python/3/argparse.py @@ -736,10 +736,10 @@ - default -- The value to be produced if the option is not specified. - - type -- The type which the command-line arguments should be converted - to, should be one of 'string', 'int', 'float', 'complex' or a - callable object that accepts a single string argument. If None, - 'string' is assumed. + - type -- A callable that accepts a single string argument, and + returns the converted value. The standard Python types str, int, + float, and complex are useful examples of such callables. If None, + str is used. - choices -- A container of values that should be allowed. If not None, after a command-line argument has been converted to the appropriate @@ -1701,9 +1701,12 @@ return args def parse_known_args(self, args=None, namespace=None): - # args default to the system args if args is None: + # args default to the system args args = _sys.argv[1:] + else: + # make sure that args are mutable + args = list(args) # default Namespace built from parser defaults if namespace is None: @@ -1714,10 +1717,7 @@ if action.dest is not SUPPRESS: if not hasattr(namespace, action.dest): if action.default is not SUPPRESS: - default = action.default - if isinstance(action.default, str): - default = self._get_value(action, default) - setattr(namespace, action.dest, default) + setattr(namespace, action.dest, action.default) # add any parser defaults that aren't present for dest in self._defaults: @@ -1957,12 +1957,23 @@ if positionals: self.error(_('too few arguments')) - # make sure all required actions were present + # make sure all required actions were present, and convert defaults. for action in self._actions: - if action.required: - if action not in seen_actions: + if action not in seen_actions: + if action.required: name = _get_action_name(action) self.error(_('argument %s is required') % name) + else: + # Convert action default now instead of doing it before + # parsing arguments to avoid calling convert functions + # twice (which may fail) if the argument was given, but + # only if it was defined already in the namespace + if (action.default is not None and + isinstance(action.default, str) and + hasattr(namespace, action.dest) and + action.default is getattr(namespace, action.dest)): + setattr(namespace, action.dest, + self._get_value(action, action.default)) # make sure all required groups had one option present for group in self._mutually_exclusive_groups: @@ -1988,7 +1999,7 @@ for arg_string in arg_strings: # for regular arguments, just add them back into the list - if arg_string[0] not in self.fromfile_prefix_chars: + if not arg_string or arg_string[0] not in self.fromfile_prefix_chars: new_arg_strings.append(arg_string) # replace arguments referencing files with the file content @@ -2198,9 +2209,12 @@ # Value conversion methods # ======================== def _get_values(self, action, arg_strings): - # for everything but PARSER args, strip out '--' + # for everything but PARSER, REMAINDER args, strip out first '--' if action.nargs not in [PARSER, REMAINDER]: - arg_strings = [s for s in arg_strings if s != '--'] + try: + arg_strings.remove('--') + except ValueError: + pass # optional argument produces a default when not present if not arg_strings and action.nargs == OPTIONAL: diff --git a/lib-python/3/asyncore.py b/lib-python/3/asyncore.py --- a/lib-python/3/asyncore.py +++ b/lib-python/3/asyncore.py @@ -225,6 +225,7 @@ debug = False connected = False accepting = False + connecting = False closing = False addr = None ignore_log_types = frozenset(['warning']) @@ -248,7 +249,7 @@ try: self.addr = sock.getpeername() except socket.error as err: - if err.args[0] == ENOTCONN: + if err.args[0] in (ENOTCONN, EINVAL): # To handle the case where we got an unconnected # socket. self.connected = False @@ -342,9 +343,11 @@ def connect(self, address): self.connected = False + self.connecting = True err = self.socket.connect_ex(address) if err in (EINPROGRESS, EALREADY, EWOULDBLOCK) \ or err == EINVAL and os.name in ('nt', 'ce'): + self.addr = address return if err in (0, EISCONN): self.addr = address @@ -390,7 +393,7 @@ else: return data except socket.error as why: - # winsock sometimes throws ENOTCONN + # winsock sometimes raises ENOTCONN if why.args[0] in _DISCONNECTED: self.handle_close() return b'' @@ -400,6 +403,7 @@ def close(self): self.connected = False self.accepting = False + self.connecting = False self.del_channel() try: self.socket.close() @@ -438,7 +442,8 @@ # sockets that are connected self.handle_accept() elif not self.connected: - self.handle_connect_event() + if self.connecting: + self.handle_connect_event() self.handle_read() else: self.handle_read() @@ -449,6 +454,7 @@ raise socket.error(err, _strerror(err)) self.handle_connect() self.connected = True + self.connecting = False def handle_write_event(self): if self.accepting: @@ -457,12 +463,8 @@ return if not self.connected: - #check for errors - err = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) - if err != 0: - raise socket.error(err, _strerror(err)) - - self.handle_connect_event() + if self.connecting: + self.handle_connect_event() self.handle_write() def handle_expt_event(self): diff --git a/lib-python/3/bdb.py b/lib-python/3/bdb.py --- a/lib-python/3/bdb.py +++ b/lib-python/3/bdb.py @@ -22,6 +22,7 @@ self.skip = set(skip) if skip else None self.breaks = {} self.fncache = {} + self.frame_returning = None def canonic(self, filename): if filename == "<" + filename[1:-1] + ">": @@ -80,7 +81,11 @@ def dispatch_return(self, frame, arg): if self.stop_here(frame) or frame == self.returnframe: - self.user_return(frame, arg) + try: + self.frame_returning = frame + self.user_return(frame, arg) + finally: + self.frame_returning = None if self.quitting: raise BdbQuit return self.trace_dispatch @@ -186,6 +191,14 @@ def set_step(self): """Stop after one line of code.""" + # Issue #13183: pdb skips frames after hitting a breakpoint and running + # step commands. + # Restore the trace function in the caller (that may not have been set + # for performance reasons) when returning from the current frame. + if self.frame_returning: + caller_frame = self.frame_returning.f_back + if caller_frame and not caller_frame.f_trace: + caller_frame.f_trace = self.trace_dispatch self._set_stopinfo(None, None) def set_next(self, frame): diff --git a/lib-python/3/calendar.py b/lib-python/3/calendar.py --- a/lib-python/3/calendar.py +++ b/lib-python/3/calendar.py @@ -161,7 +161,11 @@ oneday = datetime.timedelta(days=1) while True: yield date - date += oneday + try: + date += oneday + except OverflowError: + # Adding one day could fail after datetime.MAXYEAR + break if date.month != month and date.weekday() == self.firstweekday: break diff --git a/lib-python/3/cgi.py b/lib-python/3/cgi.py --- a/lib-python/3/cgi.py +++ b/lib-python/3/cgi.py @@ -214,17 +214,17 @@ """ import http.client - boundary = "" + boundary = b"" if 'boundary' in pdict: boundary = pdict['boundary'] if not valid_boundary(boundary): raise ValueError('Invalid boundary in multipart form: %r' % (boundary,)) - nextpart = "--" + boundary - lastpart = "--" + boundary + "--" + nextpart = b"--" + boundary + lastpart = b"--" + boundary + b"--" partdict = {} - terminator = "" + terminator = b"" while terminator != lastpart: bytes = -1 @@ -243,7 +243,7 @@ raise ValueError('Maximum content length exceeded') data = fp.read(bytes) else: - data = "" + data = b"" # Read lines until end of part. lines = [] while 1: @@ -251,7 +251,7 @@ if not line: terminator = lastpart # End outer loop break - if line.startswith("--"): + if line.startswith(b"--"): terminator = line.rstrip() if terminator in (nextpart, lastpart): break @@ -263,12 +263,12 @@ if lines: # Strip final line terminator line = lines[-1] - if line[-2:] == "\r\n": + if line[-2:] == b"\r\n": line = line[:-2] - elif line[-1:] == "\n": + elif line[-1:] == b"\n": line = line[:-1] lines[-1] = line - data = "".join(lines) + data = b"".join(lines) line = headers['content-disposition'] if not line: continue diff --git a/lib-python/3/cgitb.py b/lib-python/3/cgitb.py --- a/lib-python/3/cgitb.py +++ b/lib-python/3/cgitb.py @@ -293,14 +293,19 @@ if self.logdir is not None: suffix = ['.txt', '.html'][self.format=="html"] (fd, path) = tempfile.mkstemp(suffix=suffix, dir=self.logdir) + try: file = os.fdopen(fd, 'w') file.write(doc) file.close() - msg = '

%s contains the description of this error.' % path + msg = '%s contains the description of this error.' % path except: - msg = '

Tried to save traceback to %s, but failed.' % path - self.file.write(msg + '\n') + msg = 'Tried to save traceback to %s, but failed.' % path + + if self.format == 'html': + self.file.write('

\n' % msg) + else: + self.file.write(msg + '\n') try: self.file.flush() except: pass diff --git a/lib-python/3/collections.py b/lib-python/3/collections.py --- a/lib-python/3/collections.py +++ b/lib-python/3/collections.py @@ -281,6 +281,10 @@ 'Return self as a plain tuple. Used by copy and pickle.' return tuple(self) + def __getstate__(self): + 'Exclude the OrderedDict from pickling' + return None + {field_defs} ''' diff --git a/lib-python/3/concurrent/futures/_base.py b/lib-python/3/concurrent/futures/_base.py --- a/lib-python/3/concurrent/futures/_base.py +++ b/lib-python/3/concurrent/futures/_base.py @@ -112,12 +112,14 @@ def __init__(self, num_pending_calls, stop_on_exception): self.num_pending_calls = num_pending_calls self.stop_on_exception = stop_on_exception + self.lock = threading.Lock() super().__init__() def _decrement_pending_calls(self): - self.num_pending_calls -= 1 - if not self.num_pending_calls: - self.event.set() + with self.lock: + self.num_pending_calls -= 1 + if not self.num_pending_calls: + self.event.set() def add_result(self, future): super().add_result(future) @@ -517,7 +519,7 @@ """Returns a iterator equivalent to map(fn, iter). Args: - fn: A callable that will take take as many arguments as there are + fn: A callable that will take as many arguments as there are passed iterables. timeout: The maximum number of seconds to wait. If None, then there is no limit on the wait time. diff --git a/lib-python/3/configparser.py b/lib-python/3/configparser.py --- a/lib-python/3/configparser.py +++ b/lib-python/3/configparser.py @@ -99,10 +99,9 @@ yes, on for True). Returns False or True. items(section=_UNSET, raw=False, vars=None) - If section is given, return a list of tuples with (section_name, - section_proxy) for each section, including DEFAULTSECT. Otherwise, - return a list of tuples with (name, value) for each option - in the section. + If section is given, return a list of tuples with (name, value) for + each option in the section. Otherwise, return a list of tuples with + (section_name, section_proxy) for each section, including DEFAULTSECT. remove_section(section) Remove the given file section and all its options. @@ -852,6 +851,19 @@ value_getter = lambda option: d[option] return [(option, value_getter(option)) for option in d.keys()] + def popitem(self): + """Remove a section from the parser and return it as + a (section_name, section_proxy) tuple. If no section is present, raise + KeyError. + + The section DEFAULT is never returned because it cannot be removed. + """ + for key in self.sections(): + value = self[key] + del self[key] + return key, value + raise KeyError + def optionxform(self, optionstr): return optionstr.lower() @@ -947,7 +959,8 @@ # XXX this is not atomic if read_dict fails at any point. Then again, # no update method in configparser is atomic in this implementation. - self.remove_section(key) + if key in self._sections: + self._sections[key].clear() self.read_dict({key: value}) def __delitem__(self, key): diff --git a/lib-python/3/ctypes/test/test_bitfields.py b/lib-python/3/ctypes/test/test_bitfields.py --- a/lib-python/3/ctypes/test/test_bitfields.py +++ b/lib-python/3/ctypes/test/test_bitfields.py @@ -246,5 +246,25 @@ _anonymous_ = ["_"] _fields_ = [("_", X)] + @unittest.skipUnless(hasattr(ctypes, "c_uint32"), "c_int32 is required") + def test_uint32(self): + class X(Structure): + _fields_ = [("a", c_uint32, 32)] + x = X() + x.a = 10 + self.assertEqual(x.a, 10) + x.a = 0xFDCBA987 + self.assertEqual(x.a, 0xFDCBA987) + + @unittest.skipUnless(hasattr(ctypes, "c_uint64"), "c_int64 is required") + def test_uint64(self): + class X(Structure): + _fields_ = [("a", c_uint64, 64)] + x = X() + x.a = 10 + self.assertEqual(x.a, 10) + x.a = 0xFEDCBA9876543211 + self.assertEqual(x.a, 0xFEDCBA9876543211) + if __name__ == "__main__": unittest.main() diff --git a/lib-python/3/ctypes/test/test_numbers.py b/lib-python/3/ctypes/test/test_numbers.py --- a/lib-python/3/ctypes/test/test_numbers.py +++ b/lib-python/3/ctypes/test/test_numbers.py @@ -220,6 +220,16 @@ # probably be changed: self.assertRaises(TypeError, c_int, c_long(42)) + def test_float_overflow(self): + import sys + big_int = int(sys.float_info.max) * 2 + for t in float_types + [c_longdouble]: + self.assertRaises(OverflowError, t, big_int) + if (hasattr(t, "__ctype_be__")): + self.assertRaises(OverflowError, t.__ctype_be__, big_int) + if (hasattr(t, "__ctype_le__")): + self.assertRaises(OverflowError, t.__ctype_le__, big_int) + ## def test_perf(self): ## check_perf() diff --git a/lib-python/3/ctypes/test/test_returnfuncptrs.py b/lib-python/3/ctypes/test/test_returnfuncptrs.py --- a/lib-python/3/ctypes/test/test_returnfuncptrs.py +++ b/lib-python/3/ctypes/test/test_returnfuncptrs.py @@ -1,5 +1,6 @@ import unittest from ctypes import * +import os import _ctypes_test @@ -33,5 +34,34 @@ self.assertRaises(ArgumentError, strchr, b"abcdef", 3.0) self.assertRaises(TypeError, strchr, b"abcdef") + def test_from_dll(self): + dll = CDLL(_ctypes_test.__file__) + # _CFuncPtr instances are now callable with a tuple argument + # which denotes a function name and a dll: + strchr = CFUNCTYPE(c_char_p, c_char_p, c_char)(("my_strchr", dll)) + self.assertTrue(strchr(b"abcdef", b"b"), "bcdef") + self.assertEqual(strchr(b"abcdef", b"x"), None) + self.assertRaises(ArgumentError, strchr, b"abcdef", 3.0) + self.assertRaises(TypeError, strchr, b"abcdef") + + # Issue 6083: Reference counting bug + def test_from_dll_refcount(self): + class BadSequence(tuple): + def __getitem__(self, key): + if key == 0: + return "my_strchr" + if key == 1: + return CDLL(_ctypes_test.__file__) + raise IndexError + + # _CFuncPtr instances are now callable with a tuple argument + # which denotes a function name and a dll: + strchr = CFUNCTYPE(c_char_p, c_char_p, c_char)( + BadSequence(("my_strchr", CDLL(_ctypes_test.__file__)))) + self.assertTrue(strchr(b"abcdef", b"b"), "bcdef") + self.assertEqual(strchr(b"abcdef", b"x"), None) + self.assertRaises(ArgumentError, strchr, b"abcdef", 3.0) + self.assertRaises(TypeError, strchr, b"abcdef") + if __name__ == "__main__": unittest.main() diff --git a/lib-python/3/ctypes/test/test_structures.py b/lib-python/3/ctypes/test/test_structures.py --- a/lib-python/3/ctypes/test/test_structures.py +++ b/lib-python/3/ctypes/test/test_structures.py @@ -1,6 +1,7 @@ import unittest from ctypes import * from struct import calcsize +import _testcapi class SubclassesTest(unittest.TestCase): def test_subclass(self): @@ -199,6 +200,14 @@ "_pack_": -1} self.assertRaises(ValueError, type(Structure), "X", (Structure,), d) + # Issue 15989 + d = {"_fields_": [("a", c_byte)], + "_pack_": _testcapi.INT_MAX + 1} + self.assertRaises(ValueError, type(Structure), "X", (Structure,), d) + d = {"_fields_": [("a", c_byte)], + "_pack_": _testcapi.UINT_MAX + 2} + self.assertRaises(ValueError, type(Structure), "X", (Structure,), d) + def test_initializers(self): class Person(Structure): _fields_ = [("name", c_char*6), diff --git a/lib-python/3/ctypes/test/test_win32.py b/lib-python/3/ctypes/test/test_win32.py --- a/lib-python/3/ctypes/test/test_win32.py +++ b/lib-python/3/ctypes/test/test_win32.py @@ -3,6 +3,7 @@ from ctypes import * from ctypes.test import is_resource_enabled import unittest, sys +from test import support import _ctypes_test @@ -60,7 +61,9 @@ def test_COMError(self): from _ctypes import COMError - self.assertEqual(COMError.__doc__, "Raised when a COM method call failed.") + if support.HAVE_DOCSTRINGS: + self.assertEqual(COMError.__doc__, + "Raised when a COM method call failed.") ex = COMError(-1, "text", ("details",)) self.assertEqual(ex.hresult, -1) diff --git a/lib-python/3/curses/__init__.py b/lib-python/3/curses/__init__.py --- a/lib-python/3/curses/__init__.py +++ b/lib-python/3/curses/__init__.py @@ -5,7 +5,7 @@ import curses from curses import textpad - curses.initwin() + curses.initscr() ... """ diff --git a/lib-python/3/decimal.py b/lib-python/3/decimal.py --- a/lib-python/3/decimal.py +++ b/lib-python/3/decimal.py @@ -1555,7 +1555,13 @@ def __float__(self): """Float representation.""" - return float(str(self)) + if self._isnan(): + if self.is_snan(): + raise ValueError("Cannot convert signaling NaN to float") + s = "-nan" if self._sign else "nan" + else: + s = str(self) + return float(s) def __int__(self): """Converts self to an int, truncating if necessary.""" diff --git a/lib-python/3/distutils/__init__.py b/lib-python/3/distutils/__init__.py --- a/lib-python/3/distutils/__init__.py +++ b/lib-python/3/distutils/__init__.py @@ -13,5 +13,5 @@ # Updated automatically by the Python release process. # #--start constants-- -__version__ = "3.2.3" +__version__ = "3.2.5" #--end constants-- diff --git a/lib-python/3/distutils/command/bdist_rpm.py b/lib-python/3/distutils/command/bdist_rpm.py --- a/lib-python/3/distutils/command/bdist_rpm.py +++ b/lib-python/3/distutils/command/bdist_rpm.py @@ -3,7 +3,7 @@ Implements the Distutils 'bdist_rpm' command (create RPM source and binary distributions).""" -import sys, os +import subprocess, sys, os from distutils.core import Command from distutils.debug import DEBUG from distutils.util import get_platform @@ -190,7 +190,7 @@ if self.fix_python: self.python = sys.executable else: - self.python = "python" + self.python = "python3" elif self.fix_python: raise DistutilsOptionError( "--python and --fix-python are mutually exclusive options") @@ -320,6 +320,7 @@ rpm_cmd.append('-bb') else: rpm_cmd.append('-ba') + rpm_cmd.extend(['--define', '__python %s' % self.python]) if self.rpm3_mode: rpm_cmd.extend(['--define', '_topdir %s' % os.path.abspath(self.rpm_base)]) @@ -405,6 +406,21 @@ 'Summary: ' + self.distribution.get_description(), ] + # Workaround for #14443 which affects some RPM based systems such as + # RHEL6 (and probably derivatives) + vendor_hook = subprocess.getoutput('rpm --eval %{__os_install_post}') + # Generate a potential replacement value for __os_install_post (whilst + # normalizing the whitespace to simplify the test for whether the + # invocation of brp-python-bytecompile passes in __python): + vendor_hook = '\n'.join([' %s \\' % line.strip() + for line in vendor_hook.splitlines()]) + problem = "brp-python-bytecompile \\\n" + fixed = "brp-python-bytecompile %{__python} \\\n" + fixed_hook = vendor_hook.replace(problem, fixed) + if fixed_hook != vendor_hook: + spec_file.append('# Workaround for http://bugs.python.org/issue14443') + spec_file.append('%define __os_install_post ' + fixed_hook + '\n') + # put locale summaries into spec file # XXX not supported for now (hard to put a dictionary # in a config file -- arg!) diff --git a/lib-python/3/distutils/command/build_ext.py b/lib-python/3/distutils/command/build_ext.py --- a/lib-python/3/distutils/command/build_ext.py +++ b/lib-python/3/distutils/command/build_ext.py @@ -682,13 +682,13 @@ # CPython too. The point is that on PyPy with cpyext, the # config var 'SO' is just ".so" but we want to return # ".pypy-VERSION.so" instead. - so_ext = _get_c_extension_suffix() - if so_ext is None: - so_ext = get_config_var('SO') # fall-back + ext_suffix = _get_c_extension_suffix() + if ext_suffix is None: + ext_suffix = get_config_var('EXT_SUFFIX') # fall-back # extensions in debug_mode are named 'module_d.pyd' under windows if os.name == 'nt' and self.debug: - so_ext = '_d.pyd' - return os.path.join(*ext_path) + so_ext + ext_suffix = '_d.pyd' + return os.path.join(*ext_path) + ext_suffix def get_export_symbols(self, ext): """Return the list of symbols that a shared extension has to diff --git a/lib-python/3/distutils/command/check.py b/lib-python/3/distutils/command/check.py --- a/lib-python/3/distutils/command/check.py +++ b/lib-python/3/distutils/command/check.py @@ -23,6 +23,9 @@ def system_message(self, level, message, *children, **kwargs): self.messages.append((level, message, children, kwargs)) + return nodes.system_message(message, level=level, + type=self.levels[level], + *children, **kwargs) HAS_DOCUTILS = True except Exception: diff --git a/lib-python/3/distutils/command/install.py b/lib-python/3/distutils/command/install.py --- a/lib-python/3/distutils/command/install.py +++ b/lib-python/3/distutils/command/install.py @@ -285,8 +285,8 @@ if self.user and (self.prefix or self.exec_prefix or self.home or self.install_base or self.install_platbase): - raise DistutilsOptionError("can't combine user with with prefix/" - "exec_prefix/home or install_(plat)base") + raise DistutilsOptionError("can't combine user with prefix, " + "exec_prefix/home, or install_(plat)base") # Next, stuff that's wrong (or dubious) only on certain platforms. if os.name != "posix": diff --git a/lib-python/3/distutils/command/upload.py b/lib-python/3/distutils/command/upload.py --- a/lib-python/3/distutils/command/upload.py +++ b/lib-python/3/distutils/command/upload.py @@ -125,7 +125,7 @@ if self.sign: data['gpg_signature'] = (os.path.basename(filename) + ".asc", - open(filename+".asc").read()) + open(filename+".asc", "rb").read()) # set up the authentication user_pass = (self.username + ":" + self.password).encode('ascii') diff --git a/lib-python/3/distutils/config.py b/lib-python/3/distutils/config.py --- a/lib-python/3/distutils/config.py +++ b/lib-python/3/distutils/config.py @@ -4,7 +4,6 @@ that uses .pypirc in the distutils.command package. """ import os -import sys from configparser import ConfigParser from distutils.cmd import Command @@ -43,16 +42,8 @@ def _store_pypirc(self, username, password): """Creates a default .pypirc file.""" rc = self._get_rc_file() - f = open(rc, 'w') - try: + with os.fdopen(os.open(rc, os.O_CREAT | os.O_WRONLY, 0o600), 'w') as f: f.write(DEFAULT_PYPIRC % (username, password)) - finally: - f.close() - try: - os.chmod(rc, 0o600) - except OSError: - # should do something better here - pass def _read_pypirc(self): """Reads the .pypirc file.""" diff --git a/lib-python/3/distutils/dir_util.py b/lib-python/3/distutils/dir_util.py --- a/lib-python/3/distutils/dir_util.py +++ b/lib-python/3/distutils/dir_util.py @@ -141,6 +141,10 @@ src_name = os.path.join(src, n) dst_name = os.path.join(dst, n) + if n.startswith('.nfs'): + # skip NFS rename files + continue + if preserve_symlinks and os.path.islink(src_name): link_dest = os.readlink(src_name) if verbose >= 1: diff --git a/lib-python/3/distutils/sysconfig.py b/lib-python/3/distutils/sysconfig.py --- a/lib-python/3/distutils/sysconfig.py +++ b/lib-python/3/distutils/sysconfig.py @@ -23,6 +23,3 @@ from distutils.sysconfig_cpython import * from distutils.sysconfig_cpython import _config_vars # needed by setuptools from distutils.sysconfig_cpython import _variable_rx # read_setup_file() - -_USE_CLANG = None - diff --git a/lib-python/3/distutils/sysconfig_cpython.py b/lib-python/3/distutils/sysconfig_cpython.py --- a/lib-python/3/distutils/sysconfig_cpython.py +++ b/lib-python/3/distutils/sysconfig_cpython.py @@ -146,7 +146,7 @@ "I don't know where Python installs its library " "on platform '%s'" % os.name) -_USE_CLANG = None + def customize_compiler(compiler): """Do any platform-specific customization of a CCompiler instance. @@ -155,42 +155,28 @@ varies across Unices and is stored in Python's Makefile. """ if compiler.compiler_type == "unix": - (cc, cxx, opt, cflags, ccshared, ldshared, so_ext, ar, ar_flags) = \ + if sys.platform == "darwin": + # Perform first-time customization of compiler-related + # config vars on OS X now that we know we need a compiler. + # This is primarily to support Pythons from binary + # installers. The kind and paths to build tools on + # the user system may vary significantly from the system + # that Python itself was built on. Also the user OS + # version and build tools may not support the same set + # of CPU architectures for universal builds. + global _config_vars + if not _config_vars.get('CUSTOMIZED_OSX_COMPILER', ''): + import _osx_support + _osx_support.customize_compiler(_config_vars) + _config_vars['CUSTOMIZED_OSX_COMPILER'] = 'True' + + (cc, cxx, opt, cflags, ccshared, ldshared, shlib_suffix, ar, ar_flags) = \ get_config_vars('CC', 'CXX', 'OPT', 'CFLAGS', - 'CCSHARED', 'LDSHARED', 'SO', 'AR', 'ARFLAGS') + 'CCSHARED', 'LDSHARED', 'SHLIB_SUFFIX', 'AR', 'ARFLAGS') newcc = None if 'CC' in os.environ: - newcc = os.environ['CC'] - elif sys.platform == 'darwin' and cc == 'gcc-4.2': - # Issue #13590: - # Since Apple removed gcc-4.2 in Xcode 4.2, we can no - # longer assume it is available for extension module builds. - # If Python was built with gcc-4.2, check first to see if - # it is available on this system; if not, try to use clang - # instead unless the caller explicitly set CC. - global _USE_CLANG - if _USE_CLANG is None: - from distutils import log - from subprocess import Popen, PIPE - p = Popen("! type gcc-4.2 && type clang && exit 2", - shell=True, stdout=PIPE, stderr=PIPE) - p.wait() - if p.returncode == 2: - _USE_CLANG = True - log.warn("gcc-4.2 not found, using clang instead") - else: - _USE_CLANG = False - if _USE_CLANG: - newcc = 'clang' - if newcc: - # On OS X, if CC is overridden, use that as the default - # command for LDSHARED as well - if (sys.platform == 'darwin' - and 'LDSHARED' not in os.environ - and ldshared.startswith(cc)): - ldshared = newcc + ldshared[len(cc):] - cc = newcc + cc = os.environ['CC'] if 'CXX' in os.environ: cxx = os.environ['CXX'] if 'LDSHARED' in os.environ: @@ -225,7 +211,7 @@ linker_exe=cc, archiver=archiver) - compiler.shared_lib_extension = so_ext + compiler.shared_lib_extension = shlib_suffix def get_config_h_filename(): @@ -480,6 +466,7 @@ g['INCLUDEPY'] = get_python_inc(plat_specific=0) g['SO'] = '.pyd' + g['EXT_SUFFIX'] = '.pyd' g['EXE'] = ".exe" g['VERSION'] = get_python_version().replace(".", "") g['BINDIR'] = os.path.dirname(os.path.abspath(sys.executable)) @@ -499,6 +486,7 @@ g['INCLUDEPY'] = get_python_inc(plat_specific=0) g['SO'] = '.pyd' + g['EXT_SUFFIX'] = '.pyd' g['EXE'] = ".exe" global _config_vars @@ -543,43 +531,11 @@ srcdir = os.path.join(base, _config_vars['srcdir']) _config_vars['srcdir'] = os.path.normpath(srcdir) + # OS X platforms require special customization to handle + # multi-architecture, multi-os-version installers if sys.platform == 'darwin': - kernel_version = os.uname()[2] # Kernel version (8.4.3) - major_version = int(kernel_version.split('.')[0]) - - if major_version < 8: - # On Mac OS X before 10.4, check if -arch and -isysroot - # are in CFLAGS or LDFLAGS and remove them if they are. - # This is needed when building extensions on a 10.3 system - # using a universal build of python. - for key in ('LDFLAGS', 'BASECFLAGS', - # a number of derived variables. These need to be - # patched up as well. - 'CFLAGS', 'PY_CFLAGS', 'BLDSHARED'): - flags = _config_vars[key] - flags = re.sub('-arch\s+\w+\s', ' ', flags, re.ASCII) - flags = re.sub('-isysroot [^ \t]*', ' ', flags) - _config_vars[key] = flags - - else: - - # Allow the user to override the architecture flags using - # an environment variable. - # NOTE: This name was introduced by Apple in OSX 10.5 and - # is used by several scripting languages distributed with - # that OS release. - - if 'ARCHFLAGS' in os.environ: - arch = os.environ['ARCHFLAGS'] - for key in ('LDFLAGS', 'BASECFLAGS', - # a number of derived variables. These need to be - # patched up as well. - 'CFLAGS', 'PY_CFLAGS', 'BLDSHARED'): - - flags = _config_vars[key] - flags = re.sub('-arch\s+\w+\s', ' ', flags) - flags = flags + ' ' + arch - _config_vars[key] = flags + import _osx_support + _osx_support.customize_config_vars(_config_vars) if args: vals = [] diff --git a/lib-python/3/distutils/tests/test_bdist_dumb.py b/lib-python/3/distutils/tests/test_bdist_dumb.py --- a/lib-python/3/distutils/tests/test_bdist_dumb.py +++ b/lib-python/3/distutils/tests/test_bdist_dumb.py @@ -88,9 +88,9 @@ fp.close() contents = sorted(os.path.basename(fn) for fn in contents) - wanted = ['foo-0.1-py%s.%s.egg-info' % sys.version_info[:2], - 'foo.%s.pyc' % imp.get_tag(), - 'foo.py'] + wanted = ['foo-0.1-py%s.%s.egg-info' % sys.version_info[:2], 'foo.py'] + if not sys.dont_write_bytecode: + wanted.append('foo.%s.pyc' % imp.get_tag()) self.assertEqual(contents, sorted(wanted)) def test_suite(): diff --git a/lib-python/3/distutils/tests/test_bdist_msi.py b/lib-python/3/distutils/tests/test_bdist_msi.py --- a/lib-python/3/distutils/tests/test_bdist_msi.py +++ b/lib-python/3/distutils/tests/test_bdist_msi.py @@ -1,12 +1,11 @@ """Tests for distutils.command.bdist_msi.""" +import sys import unittest -import sys - from test.support import run_unittest - from distutils.tests import support - at unittest.skipUnless(sys.platform=="win32", "These tests are only for win32") + + at unittest.skipUnless(sys.platform == 'win32', 'these tests require Windows') class BDistMSITestCase(support.TempdirManager, support.LoggingSilencer, unittest.TestCase): @@ -14,10 +13,11 @@ def test_minimal(self): # minimal test XXX need more tests from distutils.command.bdist_msi import bdist_msi - pkg_pth, dist = self.create_dist() + project_dir, dist = self.create_dist() cmd = bdist_msi(dist) cmd.ensure_finalized() + def test_suite(): return unittest.makeSuite(BDistMSITestCase) diff --git a/lib-python/3/distutils/tests/test_build_ext.py b/lib-python/3/distutils/tests/test_build_ext.py --- a/lib-python/3/distutils/tests/test_build_ext.py +++ b/lib-python/3/distutils/tests/test_build_ext.py @@ -73,8 +73,9 @@ self.assertEqual(xx.foo(2, 5), 7) self.assertEqual(xx.foo(13,15), 28) self.assertEqual(xx.new().demo(), None) - doc = 'This is a template module just for instruction.' - self.assertEqual(xx.__doc__, doc) + if support.HAVE_DOCSTRINGS: + doc = 'This is a template module just for instruction.' + self.assertEqual(xx.__doc__, doc) self.assertTrue(isinstance(xx.Null(), xx.Null)) self.assertTrue(isinstance(xx.Str(), xx.Str)) @@ -317,8 +318,8 @@ finally: os.chdir(old_wd) self.assertTrue(os.path.exists(so_file)) - so_ext = sysconfig.get_config_var('SO') - self.assertTrue(so_file.endswith(so_ext)) + ext_suffix = sysconfig.get_config_var('EXT_SUFFIX') + self.assertTrue(so_file.endswith(ext_suffix)) so_dir = os.path.dirname(so_file) self.assertEqual(so_dir, other_tmp_dir) @@ -327,7 +328,7 @@ cmd.run() so_file = cmd.get_outputs()[0] self.assertTrue(os.path.exists(so_file)) - self.assertTrue(so_file.endswith(so_ext)) + self.assertTrue(so_file.endswith(ext_suffix)) so_dir = os.path.dirname(so_file) self.assertEqual(so_dir, cmd.build_lib) @@ -354,7 +355,7 @@ self.assertEqual(lastdir, 'bar') def test_ext_fullpath(self): - ext = sysconfig.get_config_vars()['SO'] + ext = sysconfig.get_config_var('EXT_SUFFIX') # building lxml.etree inplace #etree_c = os.path.join(self.tmp_dir, 'lxml.etree.c') #etree_ext = Extension('lxml.etree', [etree_c]) diff --git a/lib-python/3/distutils/tests/test_dir_util.py b/lib-python/3/distutils/tests/test_dir_util.py --- a/lib-python/3/distutils/tests/test_dir_util.py +++ b/lib-python/3/distutils/tests/test_dir_util.py @@ -76,7 +76,6 @@ remove_tree(self.root_target, verbose=0) - def test_copy_tree_verbosity(self): mkpath(self.target, verbose=0) @@ -88,11 +87,8 @@ mkpath(self.target, verbose=0) a_file = os.path.join(self.target, 'ok.txt') - f = open(a_file, 'w') - try: + with open(a_file, 'w') as f: f.write('some content') - finally: - f.close() wanted = ['copying %s -> %s' % (a_file, self.target2)] copy_tree(self.target, self.target2, verbose=1) @@ -101,6 +97,21 @@ remove_tree(self.root_target, verbose=0) remove_tree(self.target2, verbose=0) + def test_copy_tree_skips_nfs_temp_files(self): + mkpath(self.target, verbose=0) + + a_file = os.path.join(self.target, 'ok.txt') + nfs_file = os.path.join(self.target, '.nfs123abc') + for f in a_file, nfs_file: + with open(f, 'w') as fh: + fh.write('some content') + + copy_tree(self.target, self.target2) + self.assertEqual(os.listdir(self.target2), ['ok.txt']) + + remove_tree(self.root_target, verbose=0) + remove_tree(self.target2, verbose=0) + def test_ensure_relative(self): if os.sep == '/': self.assertEqual(ensure_relative('/home/foo'), 'home/foo') diff --git a/lib-python/3/distutils/tests/test_install.py b/lib-python/3/distutils/tests/test_install.py --- a/lib-python/3/distutils/tests/test_install.py +++ b/lib-python/3/distutils/tests/test_install.py @@ -23,7 +23,7 @@ def _make_ext_name(modname): if os.name == 'nt' and sys.executable.endswith('_d.exe'): modname += '_d' - return modname + sysconfig.get_config_var('SO') + return modname + sysconfig.get_config_var('EXT_SUFFIX') class InstallTestCase(support.TempdirManager, @@ -165,7 +165,7 @@ cmd.home = 'home' self.assertRaises(DistutilsOptionError, cmd.finalize_options) - # can't combine user with with prefix/exec_prefix/home or + # can't combine user with prefix/exec_prefix/home or # install_(plat)base cmd.prefix = None cmd.user = 'user' diff --git a/lib-python/3/distutils/tests/test_msvc9compiler.py b/lib-python/3/distutils/tests/test_msvc9compiler.py --- a/lib-python/3/distutils/tests/test_msvc9compiler.py +++ b/lib-python/3/distutils/tests/test_msvc9compiler.py @@ -104,7 +104,7 @@ unittest.TestCase): def test_no_compiler(self): - # makes sure query_vcvarsall throws + # makes sure query_vcvarsall raises # a DistutilsPlatformError if the compiler # is not found from distutils.msvc9compiler import query_vcvarsall diff --git a/lib-python/3/distutils/tests/test_register.py b/lib-python/3/distutils/tests/test_register.py --- a/lib-python/3/distutils/tests/test_register.py +++ b/lib-python/3/distutils/tests/test_register.py @@ -1,5 +1,4 @@ """Tests for distutils.command.register.""" -import sys import os import unittest import getpass @@ -10,11 +9,14 @@ from distutils.command import register as register_module from distutils.command.register import register -from distutils.core import Distribution from distutils.errors import DistutilsSetupError -from distutils.tests import support -from distutils.tests.test_config import PYPIRC, PyPIRCCommandTestCase +from distutils.tests.test_config import PyPIRCCommandTestCase + +try: + import docutils +except ImportError: + docutils = None PYPIRC_NOPASSWORD = """\ [distutils] @@ -193,6 +195,7 @@ self.assertEqual(headers['Content-length'], '290') self.assertTrue((b'tarek') in req.data) + @unittest.skipUnless(docutils is not None, 'needs docutils') def test_strict(self): # testing the script option # when on, the register command stops if @@ -205,13 +208,6 @@ cmd.strict = 1 self.assertRaises(DistutilsSetupError, cmd.run) - # we don't test the reSt feature if docutils - # is not installed - try: - import docutils - except ImportError: - return - # metadata are OK but long_description is broken metadata = {'url': 'xxx', 'author': 'xxx', 'author_email': 'éxéxé', @@ -265,6 +261,22 @@ finally: del register_module.input + @unittest.skipUnless(docutils is not None, 'needs docutils') + def test_register_invalid_long_description(self): + description = ':funkie:`str`' # mimic Sphinx-specific markup + metadata = {'url': 'xxx', 'author': 'xxx', + 'author_email': 'xxx', + 'name': 'xxx', 'version': 'xxx', + 'long_description': description} + cmd = self._get_cmd(metadata) + cmd.ensure_finalized() + cmd.strict = True + inputs = Inputs('2', 'tarek', 'tarek at ziade.org') + register_module.input = inputs + self.addCleanup(delattr, register_module, 'input') + + self.assertRaises(DistutilsSetupError, cmd.run) + def test_check_metadata_deprecated(self): # makes sure make_metadata is deprecated cmd = self._get_cmd() diff --git a/lib-python/3/distutils/tests/test_sdist.py b/lib-python/3/distutils/tests/test_sdist.py --- a/lib-python/3/distutils/tests/test_sdist.py +++ b/lib-python/3/distutils/tests/test_sdist.py @@ -6,6 +6,7 @@ import zipfile from os.path import join from textwrap import dedent +from test.support import captured_stdout, check_warnings, run_unittest try: import zlib @@ -13,7 +14,6 @@ except ImportError: ZLIB_SUPPORT = False -from test.support import captured_stdout, check_warnings, run_unittest from distutils.command.sdist import sdist, show_formats from distutils.core import Distribution @@ -83,9 +83,8 @@ @unittest.skipUnless(ZLIB_SUPPORT, 'Need zlib support to run') def test_prune_file_list(self): - # this test creates a package with some vcs dirs in it - # and launch sdist to make sure they get pruned - # on all systems + # this test creates a project with some VCS dirs and an NFS rename + # file, then launches sdist to check they get pruned on all systems # creating VCS directories with some files in them os.mkdir(join(self.tmp_dir, 'somecode', '.svn')) @@ -99,6 +98,8 @@ self.write_file((self.tmp_dir, 'somecode', '.git', 'ok'), 'xxx') + self.write_file((self.tmp_dir, 'somecode', '.nfs0001'), 'xxx') + # now building a sdist dist, cmd = self.get_cmd() @@ -326,6 +327,7 @@ # filling data_files by pointing files in package_data dist.package_data = {'somecode': ['*.txt']} self.write_file((self.tmp_dir, 'somecode', 'doc.txt'), '#') + cmd.formats = ['gztar'] cmd.ensure_finalized() cmd.run() diff --git a/lib-python/3/distutils/tests/test_sysconfig.py b/lib-python/3/distutils/tests/test_sysconfig.py --- a/lib-python/3/distutils/tests/test_sysconfig.py +++ b/lib-python/3/distutils/tests/test_sysconfig.py @@ -102,7 +102,27 @@ import sysconfig as global_sysconfig self.assertEqual(global_sysconfig.get_config_var('CFLAGS'), sysconfig.get_config_var('CFLAGS')) self.assertEqual(global_sysconfig.get_config_var('LDFLAGS'), sysconfig.get_config_var('LDFLAGS')) - self.assertEqual(global_sysconfig.get_config_var('LDSHARED'),sysconfig.get_config_var('LDSHARED')) + + @unittest.skipIf(sysconfig.get_config_var('CUSTOMIZED_OSX_COMPILER'),'compiler flags customized') + def test_sysconfig_compiler_vars(self): + # On OS X, binary installers support extension module building on + # various levels of the operating system with differing Xcode + # configurations. This requires customization of some of the + # compiler configuration directives to suit the environment on + # the installed machine. Some of these customizations may require + # running external programs and, so, are deferred until needed by + # the first extension module build. With Python 3.3, only + # the Distutils version of sysconfig is used for extension module + # builds, which happens earlier in the Distutils tests. This may + # cause the following tests to fail since no tests have caused + # the global version of sysconfig to call the customization yet. + # The solution for now is to simply skip this test in this case. + # The longer-term solution is to only have one version of sysconfig. + + import sysconfig as global_sysconfig + if sysconfig.get_config_var('CUSTOMIZED_OSX_COMPILER'): + return + self.assertEqual(global_sysconfig.get_config_var('LDSHARED'), sysconfig.get_config_var('LDSHARED')) self.assertEqual(global_sysconfig.get_config_var('CC'), sysconfig.get_config_var('CC')) diff --git a/lib-python/3/distutils/tests/test_util.py b/lib-python/3/distutils/tests/test_util.py --- a/lib-python/3/distutils/tests/test_util.py +++ b/lib-python/3/distutils/tests/test_util.py @@ -13,6 +13,7 @@ from distutils.sysconfig import get_config_vars from distutils import sysconfig from distutils.tests import support +import _osx_support class UtilTestCase(support.EnvironGuard, unittest.TestCase): @@ -92,6 +93,7 @@ ('Darwin Kernel Version 8.11.1: ' 'Wed Oct 10 18:23:28 PDT 2007; ' 'root:xnu-792.25.20~1/RELEASE_I386'), 'i386')) + _osx_support._remove_original_values(get_config_vars()) get_config_vars()['MACOSX_DEPLOYMENT_TARGET'] = '10.3' From noreply at buildbot.pypy.org Sat Mar 15 01:48:18 2014 From: noreply at buildbot.pypy.org (pjenvey) Date: Sat, 15 Mar 2014 01:48:18 +0100 (CET) Subject: [pypy-commit] pypy stdlib-3.2.5: bump Message-ID: <20140315004818.4026B1C3058@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: stdlib-3.2.5 Changeset: r69966:9c9dbbb65ba8 Date: 2014-03-14 17:46 -0700 http://bitbucket.org/pypy/pypy/changeset/9c9dbbb65ba8/ Log: bump diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h --- a/pypy/module/cpyext/include/patchlevel.h +++ b/pypy/module/cpyext/include/patchlevel.h @@ -26,7 +26,7 @@ #define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "3.2.3" +#define PY_VERSION "3.2.5" /* PyPy version as a string */ #define PYPY_VERSION "2.3.0-alpha0" diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -7,7 +7,7 @@ from pypy.interpreter import gateway #XXX # the release serial 42 is not in range(16) -CPYTHON_VERSION = (3, 2, 3, "final", 0) +CPYTHON_VERSION = (3, 2, 5, "final", 0) #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py CPYTHON_API_VERSION = 1013 #XXX # sync with include/modsupport.h From noreply at buildbot.pypy.org Sat Mar 15 07:54:04 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 15 Mar 2014 07:54:04 +0100 (CET) Subject: [pypy-commit] cffi default: Use the word 'Note' to make it sound less ominous Message-ID: <20140315065404.11E9D1C03A3@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1481:ddbfc6f0300b Date: 2014-03-15 07:53 +0100 http://bitbucket.org/cffi/cffi/changeset/ddbfc6f0300b/ Log: Use the word 'Note' to make it sound less ominous diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -43,9 +43,9 @@ def ask_supports_thread(): if sys.platform == "darwin": - sys.stderr.write("OS/X: confusion between 'cc' versus 'gcc'") - sys.stderr.write(" (see issue 123)\n") - sys.stderr.write("will not use '__thread' in the C code\n") + sys.stderr.write("Note: will not use '__thread' in the C code\n") + sys.stderr.write("This is for OS/X-specific reasons: confusion " + "between 'cc' versus 'gcc' (see issue 123)\n") return import distutils.errors from distutils.ccompiler import new_compiler @@ -53,8 +53,8 @@ try: compiler.compile(['c/check__thread.c']) except distutils.errors.CompileError: - sys.stderr.write("the above error message can be safely ignored;\n") - sys.stderr.write("will not use '__thread' in the C code\n") + sys.stderr.write("Note: will not use '__thread' in the C code\n") + sys.stderr.write("The above error message can be safely ignored\n") else: define_macros.append(('USE__THREAD', None)) try: From noreply at buildbot.pypy.org Sat Mar 15 08:22:36 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 15 Mar 2014 08:22:36 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: hg merge default Message-ID: <20140315072236.8E05E1C0483@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r69967:fbedfb1494f1 Date: 2014-03-15 08:21 +0100 http://bitbucket.org/pypy/pypy/changeset/fbedfb1494f1/ Log: hg merge default diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py --- a/lib_pypy/_sqlite3.py +++ b/lib_pypy/_sqlite3.py @@ -38,6 +38,7 @@ if sys.version_info[0] >= 3: StandardError = Exception + cmp = lambda x, y: (x > y) - (x < y) long = int xrange = range basestring = unicode = str diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py --- a/pypy/module/cpyext/test/test_cpyext.py +++ b/pypy/module/cpyext/test/test_cpyext.py @@ -64,6 +64,8 @@ kwds["libraries"] = [api_library] # '%s' undefined; assuming extern returning int kwds["compile_extra"] = ["/we4013"] + # prevent linking with python27.lib + kwds["compile_extra"].append("/DPy_BUILD_CORE") elif sys.platform == 'darwin': kwds["link_files"] = [str(api_library + '.dylib')] else: diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py --- a/rpython/jit/metainterp/optimizeopt/heap.py +++ b/rpython/jit/metainterp/optimizeopt/heap.py @@ -93,6 +93,11 @@ # possible aliasing). self.clear() self._lazy_setfield = None + if optheap.postponed_op: + for a in op.getarglist(): + if a is optheap.postponed_op.result: + optheap.emit_postponed_op() + break optheap.next_optimization.propagate_forward(op) if not can_cache: return @@ -179,6 +184,9 @@ def flush(self): self.force_all_lazy_setfields_and_arrayitems() + self.emit_postponed_op() + + def emit_postponed_op(self): if self.postponed_op: postponed_op = self.postponed_op self.postponed_op = None @@ -227,10 +235,7 @@ def emit_operation(self, op): self.emitting_operation(op) - if self.postponed_op: - postponed_op = self.postponed_op - self.postponed_op = None - self.next_optimization.propagate_forward(postponed_op) + self.emit_postponed_op() if (op.is_comparison() or op.getopnum() == rop.CALL_MAY_FORCE or op.is_ovf()): self.postponed_op = op diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py @@ -1660,6 +1660,16 @@ """ self.optimize_loop(ops, ops) + def test_setfield_int_eq_result(self): + # test that the setfield_gc does not end up before int_eq + ops = """ + [p1, i1, i2] + i3 = int_eq(i1, i2) + setfield_gc(p1, i3, descr=valuedescr) + jump(p1, i1, i2) + """ + self.optimize_loop(ops, ops) + def test_duplicate_setfield_aliasing(self): # a case where aliasing issues (and not enough cleverness) mean # that we fail to remove any setfield_gc @@ -5433,7 +5443,6 @@ jump(i0) """ self.optimize_loop(ops, expected) - - + class TestLLtype(BaseTestOptimizeBasic, LLtypeMixin): pass diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -1,14 +1,15 @@ import py from rpython.rlib.objectmodel import instantiate +from rpython.jit.metainterp import compile, resume +from rpython.jit.metainterp.history import AbstractDescr, ConstInt, BoxInt, TreeLoop +from rpython.jit.metainterp.optimize import InvalidLoop +from rpython.jit.metainterp.optimizeopt import build_opt_chain from rpython.jit.metainterp.optimizeopt.test.test_util import ( LLtypeMixin, BaseTest, convert_old_style_to_targets) -from rpython.jit.metainterp.optimizeopt import build_opt_chain -from rpython.jit.metainterp.optimize import InvalidLoop -from rpython.jit.metainterp.history import AbstractDescr, ConstInt, BoxInt -from rpython.jit.metainterp.history import TreeLoop -from rpython.jit.metainterp import compile, resume +from rpython.jit.metainterp.optimizeopt.test.test_optimizebasic import \ + FakeMetaInterpStaticData from rpython.jit.metainterp.resoperation import rop, opname, oparity -from rpython.jit.metainterp.optimizeopt.test.test_optimizebasic import FakeMetaInterpStaticData + def test_build_opt_chain(): def check(chain, expected_names): @@ -40,7 +41,6 @@ class BaseTestWithUnroll(BaseTest): - enable_opts = "intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll" def optimize_loop(self, ops, expected, expected_preamble=None, @@ -93,8 +93,8 @@ def raises(self, e, fn, *args): return py.test.raises(e, fn, *args).value + class OptimizeOptTest(BaseTestWithUnroll): - def setup_method(self, meth=None): class FailDescr(compile.ResumeGuardDescr): oparse = None @@ -130,7 +130,6 @@ self.namespace.pop('fdescr', None) self.namespace.pop('fdescr2', None) - def test_simple(self): ops = """ [] @@ -974,7 +973,6 @@ """ self.optimize_loop(ops, expected, preamble) - # ---------- def test_virtual_1(self): @@ -1252,7 +1250,6 @@ """ self.optimize_loop(ops, expected, preamble) - def test_virtual_constant_isnonnull(self): ops = """ [i0] @@ -2789,8 +2786,7 @@ p2 = new_with_vtable(ConstClass(node_vtable)) jump(p2) """ - self.raises(InvalidLoop, self.optimize_loop, - ops, "crash!") + self.raises(InvalidLoop, self.optimize_loop, ops, "crash!") def test_invalid_loop_2(self): ops = """ @@ -2801,8 +2797,7 @@ escape(p2) # prevent it from staying Virtual jump(p2) """ - self.raises(InvalidLoop, self.optimize_loop, - ops, "crash!") + self.raises(InvalidLoop, self.optimize_loop, ops, "crash!") def test_invalid_loop_3(self): ops = """ @@ -2824,8 +2819,7 @@ guard_value(p2, ConstPtr(myptr)) [] jump(p2) """ - exc = self.raises(InvalidLoop, self.optimize_loop, - ops, "crash!") + exc = self.raises(InvalidLoop, self.optimize_loop, ops, "crash!") if exc: assert "node" in exc.msg @@ -3151,7 +3145,6 @@ """ self.optimize_loop(ops, expected) - def test_int_and_or_with_zero(self): ops = """ [i0, i1] @@ -5107,7 +5100,6 @@ """ self.optimize_loop(ops, expected) - def test_division_nonneg(self): py.test.skip("harder") # this is how an app-level division turns into right now @@ -5444,7 +5436,6 @@ """ self.optimize_loop(ops, ops, ops) - def test_mul_ovf(self): ops = """ [i0, i1] @@ -5591,7 +5582,6 @@ def is_integer_bounded(self): return False - for n in ('inst_w_seq', 'inst_index', 'inst_w_list', 'inst_length', 'inst_start', 'inst_step'): self.namespace[n] = FakeDescr(n) @@ -5847,7 +5837,7 @@ self.optimize_loop(ops, optops, preamble) # check with replacing 'str' with 'unicode' everywhere def r(s): - return s.replace('str','unicode').replace('s"', 'u"') + return s.replace('str', 'unicode').replace('s"', 'u"') self.optimize_loop(r(ops), r(optops), r(preamble)) def test_newstr_1(self): @@ -6277,7 +6267,7 @@ if isinstance(value, calldescrtype): extra = value.get_extra_info() if (extra and isinstance(extra, effectinfotype) and - extra.oopspecindex == oopspecindex): + extra.oopspecindex == oopspecindex): # returns 0 for 'func' in this test return value, 0 raise AssertionError("not found: oopspecindex=%d" % @@ -7395,7 +7385,6 @@ """ self.optimize_loop(ops, expected, expected_short=short) - def test_loopinvariant_constant_strgetitem(self): ops = """ [p0] @@ -7454,7 +7443,7 @@ """ self.optimize_loop(ops, expected, expected_short=short) - def test_propagate_virtual_arryalen(self): + def test_propagate_virtual_arraylen(self): ops = """ [p0] p404 = new_array(2, descr=arraydescr) @@ -7831,7 +7820,6 @@ """ self.optimize_loop(ops, expected) - def test_setarrayitem_followed_by_arraycopy(self): ops = """ [p1, p2] @@ -8124,7 +8112,6 @@ """ self.optimize_loop(ops, expected) - def test_issue1080_infinitie_loop_simple(self): ops = """ [p69] @@ -8149,8 +8136,7 @@ guard_value(p1, ConstPtr(myptr)) [] jump(p1) """ - self.raises(InvalidLoop, self.optimize_loop, - ops, ops) + self.raises(InvalidLoop, self.optimize_loop, ops, ops) def test_licm_boxed_opaque_getitem(self): ops = """ @@ -8225,8 +8211,7 @@ guard_value(p1, ConstPtr(myptr)) [] jump(p1) """ - self.raises(InvalidLoop, self.optimize_loop, - ops, ops) + self.raises(InvalidLoop, self.optimize_loop, ops, ops) def test_cond_call_with_a_constant(self): ops = """ @@ -8253,6 +8238,16 @@ """ self.optimize_loop(ops, expected) + def test_hippyvm_unroll_bug(self): + ops = """ + [p0, i1, i2] + i3 = int_add(i1, 1) + i4 = int_eq(i3, i2) + setfield_gc(p0, i4, descr=valuedescr) + jump(p0, i3, i2) + """ + self.optimize_loop(ops, ops) + + class TestLLtype(OptimizeOptTest, LLtypeMixin): pass - diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -369,32 +369,21 @@ for rule in rules: m.rule(*rule) - objects = ' $(OBJECTS)' - create_obj_response_file = [] - if len(' '.join(rel_ofiles)) > 4000: - # cmd.exe has a limit of ~4000 characters before a command line is too long. - # Use a response file instead, at the cost of making the Makefile very ugly. - for i in range(len(rel_ofiles) - 1): - create_obj_response_file.append('echo %s >> obj_names.rsp' % \ - rel_ofiles[i]) - # use cmd /c for the last one so that the file is flushed - create_obj_response_file.append('cmd /c echo %s >> obj_names.rsp' % \ - rel_ofiles[-1]) - objects = ' @obj_names.rsp' if self.version < 80: m.rule('$(TARGET)', '$(OBJECTS)', - create_obj_response_file + [\ - '$(CC_LINK) /nologo $(LDFLAGS) $(LDFLAGSEXTRA)' + objects + ' /out:$@ $(LIBDIRS) $(LIBS)', + [ '$(CC_LINK) /nologo $(LDFLAGS) $(LDFLAGSEXTRA) /out:$@' +\ + ' $(LIBDIRS) $(LIBS) @<<\n$(OBJECTS)\n<<', ]) else: m.rule('$(TARGET)', '$(OBJECTS)', - create_obj_response_file + [\ - '$(CC_LINK) /nologo $(LDFLAGS) $(LDFLAGSEXTRA)' + objects + ' $(LINKFILES) /out:$@ $(LIBDIRS) $(LIBS) /MANIFEST /MANIFESTFILE:$*.manifest', + [ '$(CC_LINK) /nologo $(LDFLAGS) $(LDFLAGSEXTRA)' + \ + ' $(LINKFILES) /out:$@ $(LIBDIRS) $(LIBS) /MANIFEST' + \ + ' /MANIFESTFILE:$*.manifest @<<\n$(OBJECTS)\n<<', 'mt.exe -nologo -manifest $*.manifest -outputresource:$@;1', ]) m.rule('debugmode_$(TARGET)', '$(OBJECTS)', - create_obj_response_file + [\ - '$(CC_LINK) /nologo /DEBUG $(LDFLAGS) $(LDFLAGSEXTRA)' + objects + ' $(LINKFILES) /out:$@ $(LIBDIRS) $(LIBS)', + [ '$(CC_LINK) /nologo /DEBUG $(LDFLAGS) $(LDFLAGSEXTRA)' + \ + ' $(LINKFILES) /out:$@ $(LIBDIRS) $(LIBS) @<<\n$(OBJECTS)\n<<', ]) if shared: From noreply at buildbot.pypy.org Sat Mar 15 08:30:55 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 15 Mar 2014 08:30:55 +0100 (CET) Subject: [pypy-commit] stmgc default: Should not have an effect, but better safe than (rarely) sorry Message-ID: <20140315073055.E664E1C06AD@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1016:7f6e7192dd38 Date: 2014-03-15 08:30 +0100 http://bitbucket.org/pypy/stmgc/changeset/7f6e7192dd38/ Log: Should not have an effect, but better safe than (rarely) sorry diff --git a/c7/stm/contention.c b/c7/stm/contention.c --- a/c7/stm/contention.c +++ b/c7/stm/contention.c @@ -172,8 +172,14 @@ kind! The shadowstack may not be correct here. It should not end in a deadlock, because the target thread is, in principle, guaranteed to call abort_with_mutex() - very soon. + very soon. Just to be on the safe side, make it really + impossible for the target thread to later enter the same + cond_wait(C_ABORTED) (and thus wait, possibly for us, + ending in a deadlock): check again must_abort() first. */ + if (must_abort()) + abort_with_mutex(); + dprintf(("contention: wait C_ABORTED...\n")); cond_wait(C_ABORTED); dprintf(("contention: done\n")); From noreply at buildbot.pypy.org Sat Mar 15 09:18:48 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 15 Mar 2014 09:18:48 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: import stmgc/7f6e7192dd38 Message-ID: <20140315081848.B3C451C011F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r69968:1b664888133d Date: 2014-03-15 09:18 +0100 http://bitbucket.org/pypy/pypy/changeset/1b664888133d/ Log: import stmgc/7f6e7192dd38 diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision --- a/rpython/translator/stm/src_stm/revision +++ b/rpython/translator/stm/src_stm/revision @@ -1,1 +1,1 @@ -3c39b8d8e184 +7f6e7192dd38 diff --git a/rpython/translator/stm/src_stm/stm/contention.c b/rpython/translator/stm/src_stm/stm/contention.c --- a/rpython/translator/stm/src_stm/stm/contention.c +++ b/rpython/translator/stm/src_stm/stm/contention.c @@ -173,8 +173,14 @@ kind! The shadowstack may not be correct here. It should not end in a deadlock, because the target thread is, in principle, guaranteed to call abort_with_mutex() - very soon. + very soon. Just to be on the safe side, make it really + impossible for the target thread to later enter the same + cond_wait(C_ABORTED) (and thus wait, possibly for us, + ending in a deadlock): check again must_abort() first. */ + if (must_abort()) + abort_with_mutex(); + dprintf(("contention: wait C_ABORTED...\n")); cond_wait(C_ABORTED); dprintf(("contention: done\n")); diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c --- a/rpython/translator/stm/src_stm/stm/core.c +++ b/rpython/translator/stm/src_stm/stm/core.c @@ -12,7 +12,7 @@ void _stm_write_slowpath(object_t *obj) { - assert(_running_transaction()); + assert(_seems_to_be_running_transaction()); assert(!_is_young(obj)); /* is this an object from the same transaction, outside the nursery? */ diff --git a/rpython/translator/stm/src_stm/stm/core.h b/rpython/translator/stm/src_stm/stm/core.h --- a/rpython/translator/stm/src_stm/stm/core.h +++ b/rpython/translator/stm/src_stm/stm/core.h @@ -205,7 +205,7 @@ } static bool _is_tl_registered(stm_thread_local_t *tl); -static bool _running_transaction(void); +static bool _seems_to_be_running_transaction(void); static void teardown_core(void); static void abort_with_mutex(void) __attribute__((noreturn)); diff --git a/rpython/translator/stm/src_stm/stm/sync.c b/rpython/translator/stm/src_stm/stm/sync.c --- a/rpython/translator/stm/src_stm/stm/sync.c +++ b/rpython/translator/stm/src_stm/stm/sync.c @@ -214,7 +214,7 @@ } __attribute__((unused)) -static bool _running_transaction(void) +static bool _seems_to_be_running_transaction(void) { return (STM_SEGMENT->running_thread != NULL); } @@ -302,6 +302,7 @@ static void enter_safe_point_if_requested(void) { + assert(_seems_to_be_running_transaction()); assert(_has_mutex()); while (1) { if (must_abort()) diff --git a/rpython/translator/stm/src_stm/stmgc.h b/rpython/translator/stm/src_stm/stmgc.h --- a/rpython/translator/stm/src_stm/stmgc.h +++ b/rpython/translator/stm/src_stm/stmgc.h @@ -148,6 +148,7 @@ stm_write() is called, or immediately after getting the object from stm_allocate(), as long as the rules above are respected. */ +__attribute__((always_inline)) static inline void stm_read(object_t *obj) { ((stm_read_marker_t *)(((uintptr_t)obj) >> 4))->rm = @@ -159,6 +160,7 @@ the next one, then stm_write() needs to be called again. It is not necessary to call it immediately after stm_allocate(). */ +__attribute__((always_inline)) static inline void stm_write(object_t *obj) { if (UNLIKELY((obj->stm_flags & _STM_GCFLAG_WRITE_BARRIER) != 0)) @@ -178,6 +180,7 @@ of 8 and at least 16. In the fast-path, this is inlined to just a few assembler instructions. */ +__attribute__((always_inline)) static inline object_t *stm_allocate(ssize_t size_rounded_up) { OPT_ASSERT(size_rounded_up >= 16); From noreply at buildbot.pypy.org Sat Mar 15 09:26:21 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 15 Mar 2014 09:26:21 +0100 (CET) Subject: [pypy-commit] pypy default: Fix "return 0" when the return type is some pointer Message-ID: <20140315082621.B1E8E1C31BA@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r69969:4b77b625bdcc Date: 2014-03-15 09:24 +0100 http://bitbucket.org/pypy/pypy/changeset/4b77b625bdcc/ Log: Fix "return 0" when the return type is some pointer diff --git a/pypy/module/cpyext/pystate.py b/pypy/module/cpyext/pystate.py --- a/pypy/module/cpyext/pystate.py +++ b/pypy/module/cpyext/pystate.py @@ -213,7 +213,7 @@ if rffi.aroundstate.after: # After external call is before entering Python rffi.aroundstate.after() - return 0 + return lltype.nullptr(PyGILState_STATE.TO) @cpython_api([PyGILState_STATE], lltype.Void) def PyGILState_Release(space, state): From noreply at buildbot.pypy.org Sat Mar 15 09:26:23 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 15 Mar 2014 09:26:23 +0100 (CET) Subject: [pypy-commit] pypy default: merge heads Message-ID: <20140315082623.162FB1C31BA@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r69970:3d19219df4dd Date: 2014-03-15 09:25 +0100 http://bitbucket.org/pypy/pypy/changeset/3d19219df4dd/ Log: merge heads diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py --- a/lib_pypy/_sqlite3.py +++ b/lib_pypy/_sqlite3.py @@ -38,6 +38,7 @@ if sys.version_info[0] >= 3: StandardError = Exception + cmp = lambda x, y: (x > y) - (x < y) long = int xrange = range basestring = unicode = str diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py --- a/pypy/module/cpyext/test/test_cpyext.py +++ b/pypy/module/cpyext/test/test_cpyext.py @@ -64,6 +64,8 @@ kwds["libraries"] = [api_library] # '%s' undefined; assuming extern returning int kwds["compile_extra"] = ["/we4013"] + # prevent linking with python27.lib + kwds["compile_extra"].append("/DPy_BUILD_CORE") elif sys.platform == 'darwin': kwds["link_files"] = [str(api_library + '.dylib')] else: diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py --- a/pypy/module/micronumpy/concrete.py +++ b/pypy/module/micronumpy/concrete.py @@ -368,13 +368,10 @@ class ConcreteArray(ConcreteArrayNotOwning): def __init__(self, shape, dtype, order, strides, backstrides, storage=lltype.nullptr(RAW_STORAGE)): - null_storage = lltype.nullptr(RAW_STORAGE) + if storage == lltype.nullptr(RAW_STORAGE): + storage = dtype.itemtype.malloc(support.product(shape) * dtype.elsize) ConcreteArrayNotOwning.__init__(self, shape, dtype, order, strides, backstrides, - null_storage) - if storage == lltype.nullptr(RAW_STORAGE): - self.storage = dtype.itemtype.malloc(self.size) - else: - self.storage = storage + storage) def __del__(self): free_raw_storage(self.storage, track_allocation=False) diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py --- a/rpython/jit/metainterp/optimizeopt/heap.py +++ b/rpython/jit/metainterp/optimizeopt/heap.py @@ -93,6 +93,11 @@ # possible aliasing). self.clear() self._lazy_setfield = None + if optheap.postponed_op: + for a in op.getarglist(): + if a is optheap.postponed_op.result: + optheap.emit_postponed_op() + break optheap.next_optimization.propagate_forward(op) if not can_cache: return @@ -179,6 +184,9 @@ def flush(self): self.force_all_lazy_setfields_and_arrayitems() + self.emit_postponed_op() + + def emit_postponed_op(self): if self.postponed_op: postponed_op = self.postponed_op self.postponed_op = None @@ -227,10 +235,7 @@ def emit_operation(self, op): self.emitting_operation(op) - if self.postponed_op: - postponed_op = self.postponed_op - self.postponed_op = None - self.next_optimization.propagate_forward(postponed_op) + self.emit_postponed_op() if (op.is_comparison() or op.getopnum() == rop.CALL_MAY_FORCE or op.is_ovf()): self.postponed_op = op diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py @@ -1660,6 +1660,16 @@ """ self.optimize_loop(ops, ops) + def test_setfield_int_eq_result(self): + # test that the setfield_gc does not end up before int_eq + ops = """ + [p1, i1, i2] + i3 = int_eq(i1, i2) + setfield_gc(p1, i3, descr=valuedescr) + jump(p1, i1, i2) + """ + self.optimize_loop(ops, ops) + def test_duplicate_setfield_aliasing(self): # a case where aliasing issues (and not enough cleverness) mean # that we fail to remove any setfield_gc @@ -5433,7 +5443,6 @@ jump(i0) """ self.optimize_loop(ops, expected) - - + class TestLLtype(BaseTestOptimizeBasic, LLtypeMixin): pass diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -1,14 +1,15 @@ import py from rpython.rlib.objectmodel import instantiate +from rpython.jit.metainterp import compile, resume +from rpython.jit.metainterp.history import AbstractDescr, ConstInt, BoxInt, TreeLoop +from rpython.jit.metainterp.optimize import InvalidLoop +from rpython.jit.metainterp.optimizeopt import build_opt_chain from rpython.jit.metainterp.optimizeopt.test.test_util import ( LLtypeMixin, BaseTest, convert_old_style_to_targets) -from rpython.jit.metainterp.optimizeopt import build_opt_chain -from rpython.jit.metainterp.optimize import InvalidLoop -from rpython.jit.metainterp.history import AbstractDescr, ConstInt, BoxInt -from rpython.jit.metainterp.history import TreeLoop -from rpython.jit.metainterp import compile, resume +from rpython.jit.metainterp.optimizeopt.test.test_optimizebasic import \ + FakeMetaInterpStaticData from rpython.jit.metainterp.resoperation import rop, opname, oparity -from rpython.jit.metainterp.optimizeopt.test.test_optimizebasic import FakeMetaInterpStaticData + def test_build_opt_chain(): def check(chain, expected_names): @@ -40,7 +41,6 @@ class BaseTestWithUnroll(BaseTest): - enable_opts = "intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll" def optimize_loop(self, ops, expected, expected_preamble=None, @@ -93,8 +93,8 @@ def raises(self, e, fn, *args): return py.test.raises(e, fn, *args).value + class OptimizeOptTest(BaseTestWithUnroll): - def setup_method(self, meth=None): class FailDescr(compile.ResumeGuardDescr): oparse = None @@ -130,7 +130,6 @@ self.namespace.pop('fdescr', None) self.namespace.pop('fdescr2', None) - def test_simple(self): ops = """ [] @@ -974,7 +973,6 @@ """ self.optimize_loop(ops, expected, preamble) - # ---------- def test_virtual_1(self): @@ -1252,7 +1250,6 @@ """ self.optimize_loop(ops, expected, preamble) - def test_virtual_constant_isnonnull(self): ops = """ [i0] @@ -2789,8 +2786,7 @@ p2 = new_with_vtable(ConstClass(node_vtable)) jump(p2) """ - self.raises(InvalidLoop, self.optimize_loop, - ops, "crash!") + self.raises(InvalidLoop, self.optimize_loop, ops, "crash!") def test_invalid_loop_2(self): ops = """ @@ -2801,8 +2797,7 @@ escape(p2) # prevent it from staying Virtual jump(p2) """ - self.raises(InvalidLoop, self.optimize_loop, - ops, "crash!") + self.raises(InvalidLoop, self.optimize_loop, ops, "crash!") def test_invalid_loop_3(self): ops = """ @@ -2824,8 +2819,7 @@ guard_value(p2, ConstPtr(myptr)) [] jump(p2) """ - exc = self.raises(InvalidLoop, self.optimize_loop, - ops, "crash!") + exc = self.raises(InvalidLoop, self.optimize_loop, ops, "crash!") if exc: assert "node" in exc.msg @@ -3151,7 +3145,6 @@ """ self.optimize_loop(ops, expected) - def test_int_and_or_with_zero(self): ops = """ [i0, i1] @@ -5107,7 +5100,6 @@ """ self.optimize_loop(ops, expected) - def test_division_nonneg(self): py.test.skip("harder") # this is how an app-level division turns into right now @@ -5444,7 +5436,6 @@ """ self.optimize_loop(ops, ops, ops) - def test_mul_ovf(self): ops = """ [i0, i1] @@ -5591,7 +5582,6 @@ def is_integer_bounded(self): return False - for n in ('inst_w_seq', 'inst_index', 'inst_w_list', 'inst_length', 'inst_start', 'inst_step'): self.namespace[n] = FakeDescr(n) @@ -5847,7 +5837,7 @@ self.optimize_loop(ops, optops, preamble) # check with replacing 'str' with 'unicode' everywhere def r(s): - return s.replace('str','unicode').replace('s"', 'u"') + return s.replace('str', 'unicode').replace('s"', 'u"') self.optimize_loop(r(ops), r(optops), r(preamble)) def test_newstr_1(self): @@ -6277,7 +6267,7 @@ if isinstance(value, calldescrtype): extra = value.get_extra_info() if (extra and isinstance(extra, effectinfotype) and - extra.oopspecindex == oopspecindex): + extra.oopspecindex == oopspecindex): # returns 0 for 'func' in this test return value, 0 raise AssertionError("not found: oopspecindex=%d" % @@ -7395,7 +7385,6 @@ """ self.optimize_loop(ops, expected, expected_short=short) - def test_loopinvariant_constant_strgetitem(self): ops = """ [p0] @@ -7454,7 +7443,7 @@ """ self.optimize_loop(ops, expected, expected_short=short) - def test_propagate_virtual_arryalen(self): + def test_propagate_virtual_arraylen(self): ops = """ [p0] p404 = new_array(2, descr=arraydescr) @@ -7831,7 +7820,6 @@ """ self.optimize_loop(ops, expected) - def test_setarrayitem_followed_by_arraycopy(self): ops = """ [p1, p2] @@ -8124,7 +8112,6 @@ """ self.optimize_loop(ops, expected) - def test_issue1080_infinitie_loop_simple(self): ops = """ [p69] @@ -8149,8 +8136,7 @@ guard_value(p1, ConstPtr(myptr)) [] jump(p1) """ - self.raises(InvalidLoop, self.optimize_loop, - ops, ops) + self.raises(InvalidLoop, self.optimize_loop, ops, ops) def test_licm_boxed_opaque_getitem(self): ops = """ @@ -8225,8 +8211,7 @@ guard_value(p1, ConstPtr(myptr)) [] jump(p1) """ - self.raises(InvalidLoop, self.optimize_loop, - ops, ops) + self.raises(InvalidLoop, self.optimize_loop, ops, ops) def test_cond_call_with_a_constant(self): ops = """ @@ -8253,6 +8238,16 @@ """ self.optimize_loop(ops, expected) + def test_hippyvm_unroll_bug(self): + ops = """ + [p0, i1, i2] + i3 = int_add(i1, 1) + i4 = int_eq(i3, i2) + setfield_gc(p0, i4, descr=valuedescr) + jump(p0, i3, i2) + """ + self.optimize_loop(ops, ops) + + class TestLLtype(OptimizeOptTest, LLtypeMixin): pass - diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -369,32 +369,21 @@ for rule in rules: m.rule(*rule) - objects = ' $(OBJECTS)' - create_obj_response_file = [] - if len(' '.join(rel_ofiles)) > 4000: - # cmd.exe has a limit of ~4000 characters before a command line is too long. - # Use a response file instead, at the cost of making the Makefile very ugly. - for i in range(len(rel_ofiles) - 1): - create_obj_response_file.append('echo %s >> obj_names.rsp' % \ - rel_ofiles[i]) - # use cmd /c for the last one so that the file is flushed - create_obj_response_file.append('cmd /c echo %s >> obj_names.rsp' % \ - rel_ofiles[-1]) - objects = ' @obj_names.rsp' if self.version < 80: m.rule('$(TARGET)', '$(OBJECTS)', - create_obj_response_file + [\ - '$(CC_LINK) /nologo $(LDFLAGS) $(LDFLAGSEXTRA)' + objects + ' /out:$@ $(LIBDIRS) $(LIBS)', + [ '$(CC_LINK) /nologo $(LDFLAGS) $(LDFLAGSEXTRA) /out:$@' +\ + ' $(LIBDIRS) $(LIBS) @<<\n$(OBJECTS)\n<<', ]) else: m.rule('$(TARGET)', '$(OBJECTS)', - create_obj_response_file + [\ - '$(CC_LINK) /nologo $(LDFLAGS) $(LDFLAGSEXTRA)' + objects + ' $(LINKFILES) /out:$@ $(LIBDIRS) $(LIBS) /MANIFEST /MANIFESTFILE:$*.manifest', + [ '$(CC_LINK) /nologo $(LDFLAGS) $(LDFLAGSEXTRA)' + \ + ' $(LINKFILES) /out:$@ $(LIBDIRS) $(LIBS) /MANIFEST' + \ + ' /MANIFESTFILE:$*.manifest @<<\n$(OBJECTS)\n<<', 'mt.exe -nologo -manifest $*.manifest -outputresource:$@;1', ]) m.rule('debugmode_$(TARGET)', '$(OBJECTS)', - create_obj_response_file + [\ - '$(CC_LINK) /nologo /DEBUG $(LDFLAGS) $(LDFLAGSEXTRA)' + objects + ' $(LINKFILES) /out:$@ $(LIBDIRS) $(LIBS)', + [ '$(CC_LINK) /nologo /DEBUG $(LDFLAGS) $(LDFLAGSEXTRA)' + \ + ' $(LINKFILES) /out:$@ $(LIBDIRS) $(LIBS) @<<\n$(OBJECTS)\n<<', ]) if shared: From noreply at buildbot.pypy.org Sat Mar 15 09:27:34 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 15 Mar 2014 09:27:34 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: hg merge default Message-ID: <20140315082734.048401C31BA@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r69971:6983332fc29c Date: 2014-03-15 09:27 +0100 http://bitbucket.org/pypy/pypy/changeset/6983332fc29c/ Log: hg merge default diff --git a/pypy/module/cpyext/pystate.py b/pypy/module/cpyext/pystate.py --- a/pypy/module/cpyext/pystate.py +++ b/pypy/module/cpyext/pystate.py @@ -213,7 +213,7 @@ if rffi.aroundstate.after: # After external call is before entering Python rffi.aroundstate.after() - return 0 + return lltype.nullptr(PyGILState_STATE.TO) @cpython_api([PyGILState_STATE], lltype.Void) def PyGILState_Release(space, state): From noreply at buildbot.pypy.org Sat Mar 15 11:11:21 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 15 Mar 2014 11:11:21 +0100 (CET) Subject: [pypy-commit] stmgc default: shadowstack: improve the aliasing analysis in the C code. Message-ID: <20140315101121.CF6BA1C011F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1017:ddbc16971682 Date: 2014-03-15 11:11 +0100 http://bitbucket.org/pypy/stmgc/changeset/ddbc16971682/ Log: shadowstack: improve the aliasing analysis in the C code. diff --git a/c7/stm/core.h b/c7/stm/core.h --- a/c7/stm/core.h +++ b/c7/stm/core.h @@ -150,7 +150,7 @@ /* In case of abort, we restore the 'shadowstack' field and the 'thread_local_obj' field. */ - object_t **shadowstack_at_start_of_transaction; + struct stm_shadowentry_s *shadowstack_at_start_of_transaction; object_t *threadlocal_at_start_of_transaction; /* For debugging */ diff --git a/c7/stm/gcpage.c b/c7/stm/gcpage.c --- a/c7/stm/gcpage.c +++ b/c7/stm/gcpage.c @@ -397,11 +397,11 @@ if (must_be_zero == (segment_base == get_segment_base(0))) { - object_t **current = tl->shadowstack; - object_t **base = tl->shadowstack_base; + struct stm_shadowentry_s *current = tl->shadowstack; + struct stm_shadowentry_s *base = tl->shadowstack_base; while (current-- != base) { - assert(*current != (object_t *)-1); - mark_visit_object(*current, segment_base); + assert(current->ss != (object_t *)-1); + mark_visit_object(current->ss, segment_base); } mark_visit_object(tl->thread_local_obj, segment_base); } diff --git a/c7/stm/nursery.c b/c7/stm/nursery.c --- a/c7/stm/nursery.c +++ b/c7/stm/nursery.c @@ -158,11 +158,11 @@ static void collect_roots_in_nursery(void) { stm_thread_local_t *tl = STM_SEGMENT->running_thread; - object_t **current = tl->shadowstack; - object_t **base = tl->shadowstack_base; + struct stm_shadowentry_s *current = tl->shadowstack; + struct stm_shadowentry_s *base = tl->shadowstack_base; while (current-- != base) { - assert(*current != (object_t *)-1); - minor_trace_if_young(current); + assert(current->ss != (object_t *)-1); + minor_trace_if_young(¤t->ss); } minor_trace_if_young(&tl->thread_local_obj); } diff --git a/c7/stm/setup.c b/c7/stm/setup.c --- a/c7/stm/setup.c +++ b/c7/stm/setup.c @@ -118,7 +118,8 @@ void _init_shadow_stack(stm_thread_local_t *tl) { - object_t **s = (object_t **)malloc(SHADOW_STACK_SIZE * sizeof(object_t *)); + struct stm_shadowentry_s *s = (struct stm_shadowentry_s *) + malloc(SHADOW_STACK_SIZE * sizeof(struct stm_shadowentry_s)); assert(s); tl->shadowstack = s; tl->shadowstack_base = s; diff --git a/c7/stmgc.h b/c7/stmgc.h --- a/c7/stmgc.h +++ b/c7/stmgc.h @@ -48,9 +48,15 @@ }; #define STM_SEGMENT ((stm_segment_info_t *)4352) +struct stm_shadowentry_s { + /* Like stm_read_marker_s, this is a struct to enable better + aliasing analysis in the C code. */ + object_t *ss; +}; + typedef struct stm_thread_local_s { /* every thread should handle the shadow stack itself */ - object_t **shadowstack, **shadowstack_base; + struct stm_shadowentry_s *shadowstack, *shadowstack_base; /* a generic optional thread-local object */ object_t *thread_local_obj; /* in case this thread runs a transaction that aborts, @@ -218,9 +224,9 @@ /* Push and pop roots from/to the shadow stack. Only allowed inside transaction. */ -#define STM_PUSH_ROOT(tl, p) (*((tl).shadowstack++) = (object_t *)(p)) -#define STM_POP_ROOT(tl, p) ((p) = (typeof(p))*(--(tl).shadowstack)) -#define STM_POP_ROOT_RET(tl) (*(--(tl).shadowstack)) +#define STM_PUSH_ROOT(tl, p) ((tl).shadowstack++->ss = (object_t *)(p)) +#define STM_POP_ROOT(tl, p) ((p) = (typeof(p))((--(tl).shadowstack)->ss)) +#define STM_POP_ROOT_RET(tl) ((--(tl).shadowstack)->ss) /* Every thread needs to have a corresponding stm_thread_local_t diff --git a/c7/test/support.py b/c7/test/support.py --- a/c7/test/support.py +++ b/c7/test/support.py @@ -12,8 +12,12 @@ #define _STM_FAST_ALLOC ... #define _STM_GCFLAG_WRITE_BARRIER ... +struct stm_shadowentry_s { + object_t *ss; +}; + typedef struct { - object_t **shadowstack, **shadowstack_base; + struct stm_shadowentry_s *shadowstack, *shadowstack_base; object_t *thread_local_obj; char *mem_clear_on_abort; size_t mem_bytes_to_clear_on_abort; @@ -389,7 +393,7 @@ def _allocate_thread_local(): tl = ffi.new("stm_thread_local_t *") - ss = ffi.new("object_t *[]", SHADOWSTACK_LENGTH) + ss = ffi.new("struct stm_shadowentry_s[]", SHADOWSTACK_LENGTH) _keepalive[tl] = ss tl.shadowstack = ss tl.shadowstack_base = ss @@ -462,7 +466,7 @@ tl = self.tls[self.current_thread] curlength = tl.shadowstack - tl.shadowstack_base assert 0 <= curlength < SHADOWSTACK_LENGTH - tl.shadowstack[0] = ffi.cast("object_t *", o) + tl.shadowstack[0].ss = ffi.cast("object_t *", o) tl.shadowstack += 1 def pop_root(self): @@ -472,7 +476,7 @@ raise EmptyStack assert 0 < curlength <= SHADOWSTACK_LENGTH tl.shadowstack -= 1 - return ffi.cast("object_t *", tl.shadowstack[0]) + return ffi.cast("object_t *", tl.shadowstack[0].ss) def push_root_no_gc(self): "Pushes an invalid object, to crash in case the GC is called" From noreply at buildbot.pypy.org Sat Mar 15 11:20:34 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 15 Mar 2014 11:20:34 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: import stmgc/ddbc16971682 Message-ID: <20140315102034.47F421C011F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r69972:1d0b870195e7 Date: 2014-03-15 11:11 +0100 http://bitbucket.org/pypy/pypy/changeset/1d0b870195e7/ Log: import stmgc/ddbc16971682 diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision --- a/rpython/translator/stm/src_stm/revision +++ b/rpython/translator/stm/src_stm/revision @@ -1,1 +1,1 @@ -7f6e7192dd38 +ddbc16971682 diff --git a/rpython/translator/stm/src_stm/stm/core.h b/rpython/translator/stm/src_stm/stm/core.h --- a/rpython/translator/stm/src_stm/stm/core.h +++ b/rpython/translator/stm/src_stm/stm/core.h @@ -151,7 +151,7 @@ /* In case of abort, we restore the 'shadowstack' field and the 'thread_local_obj' field. */ - object_t **shadowstack_at_start_of_transaction; + struct stm_shadowentry_s *shadowstack_at_start_of_transaction; object_t *threadlocal_at_start_of_transaction; /* For debugging */ diff --git a/rpython/translator/stm/src_stm/stm/gcpage.c b/rpython/translator/stm/src_stm/stm/gcpage.c --- a/rpython/translator/stm/src_stm/stm/gcpage.c +++ b/rpython/translator/stm/src_stm/stm/gcpage.c @@ -398,11 +398,11 @@ if (must_be_zero == (segment_base == get_segment_base(0))) { - object_t **current = tl->shadowstack; - object_t **base = tl->shadowstack_base; + struct stm_shadowentry_s *current = tl->shadowstack; + struct stm_shadowentry_s *base = tl->shadowstack_base; while (current-- != base) { - assert(*current != (object_t *)-1); - mark_visit_object(*current, segment_base); + assert(current->ss != (object_t *)-1); + mark_visit_object(current->ss, segment_base); } mark_visit_object(tl->thread_local_obj, segment_base); } diff --git a/rpython/translator/stm/src_stm/stm/nursery.c b/rpython/translator/stm/src_stm/stm/nursery.c --- a/rpython/translator/stm/src_stm/stm/nursery.c +++ b/rpython/translator/stm/src_stm/stm/nursery.c @@ -159,11 +159,11 @@ static void collect_roots_in_nursery(void) { stm_thread_local_t *tl = STM_SEGMENT->running_thread; - object_t **current = tl->shadowstack; - object_t **base = tl->shadowstack_base; + struct stm_shadowentry_s *current = tl->shadowstack; + struct stm_shadowentry_s *base = tl->shadowstack_base; while (current-- != base) { - assert(*current != (object_t *)-1); - minor_trace_if_young(current); + assert(current->ss != (object_t *)-1); + minor_trace_if_young(¤t->ss); } minor_trace_if_young(&tl->thread_local_obj); } diff --git a/rpython/translator/stm/src_stm/stm/setup.c b/rpython/translator/stm/src_stm/stm/setup.c --- a/rpython/translator/stm/src_stm/stm/setup.c +++ b/rpython/translator/stm/src_stm/stm/setup.c @@ -119,7 +119,8 @@ void _init_shadow_stack(stm_thread_local_t *tl) { - object_t **s = (object_t **)malloc(SHADOW_STACK_SIZE * sizeof(object_t *)); + struct stm_shadowentry_s *s = (struct stm_shadowentry_s *) + malloc(SHADOW_STACK_SIZE * sizeof(struct stm_shadowentry_s)); assert(s); tl->shadowstack = s; tl->shadowstack_base = s; diff --git a/rpython/translator/stm/src_stm/stmgc.h b/rpython/translator/stm/src_stm/stmgc.h --- a/rpython/translator/stm/src_stm/stmgc.h +++ b/rpython/translator/stm/src_stm/stmgc.h @@ -49,9 +49,15 @@ }; #define STM_SEGMENT ((stm_segment_info_t *)4352) +struct stm_shadowentry_s { + /* Like stm_read_marker_s, this is a struct to enable better + aliasing analysis in the C code. */ + object_t *ss; +}; + typedef struct stm_thread_local_s { /* every thread should handle the shadow stack itself */ - object_t **shadowstack, **shadowstack_base; + struct stm_shadowentry_s *shadowstack, *shadowstack_base; /* a generic optional thread-local object */ object_t *thread_local_obj; /* in case this thread runs a transaction that aborts, @@ -219,9 +225,9 @@ /* Push and pop roots from/to the shadow stack. Only allowed inside transaction. */ -#define STM_PUSH_ROOT(tl, p) (*((tl).shadowstack++) = (object_t *)(p)) -#define STM_POP_ROOT(tl, p) ((p) = (typeof(p))*(--(tl).shadowstack)) -#define STM_POP_ROOT_RET(tl) (*(--(tl).shadowstack)) +#define STM_PUSH_ROOT(tl, p) ((tl).shadowstack++->ss = (object_t *)(p)) +#define STM_POP_ROOT(tl, p) ((p) = (typeof(p))((--(tl).shadowstack)->ss)) +#define STM_POP_ROOT_RET(tl) ((--(tl).shadowstack)->ss) /* Every thread needs to have a corresponding stm_thread_local_t diff --git a/rpython/translator/stm/test/richards.py b/rpython/translator/stm/test/richards.py --- a/rpython/translator/stm/test/richards.py +++ b/rpython/translator/stm/test/richards.py @@ -433,7 +433,7 @@ if __name__ == '__main__': import sys - max_num_threads = 5 + max_num_threads = 2 if len(sys.argv) > 1: iterations = int(sys.argv[1]) if len(sys.argv) > 2: From noreply at buildbot.pypy.org Sat Mar 15 11:21:24 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 15 Mar 2014 11:21:24 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Produce a breakdown of where the slow-down occurs Message-ID: <20140315102124.1C2CA1C011F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r69973:e96443dab32f Date: 2014-03-15 11:20 +0100 http://bitbucket.org/pypy/pypy/changeset/e96443dab32f/ Log: Produce a breakdown of where the slow-down occurs diff --git a/TODO b/TODO --- a/TODO +++ b/TODO @@ -1,3 +1,32 @@ + +1b664888133d (March 15, 2014): the overhead of a non-JIT STM, when +compared with a non-JIT plain PyPy, is measured to be 54% in a +single-threaded trivial benchmark. A tentative break-down of this figure: + +* 15% are from stm_write() on writing non-GC pointers into GC objects + +* 7% are from stm_write() on writing a constant GC pointer (likely null) + (in a regular pypy, this doesn't emit a write_barrier) + +* 14% are from stm_read() + +* 3% (in this benchmark) is just a slower startup time + +* 6% where removed soon afterwards by ddbc16971682 + +* the rest: ~9% from unknown other places (may include accessing + prebuilt GC objects, which requires an indirection) + + + + + +=============================================================================== +=========== the rest is from stmgc-c4 =================================== +=============================================================================== + + + ------------------------------------------------------------ POSSIBLE BUG: From noreply at buildbot.pypy.org Sat Mar 15 11:26:58 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 15 Mar 2014 11:26:58 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Update Message-ID: <20140315102658.24AD31C011F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r69974:05e94e7b9235 Date: 2014-03-15 11:26 +0100 http://bitbucket.org/pypy/pypy/changeset/05e94e7b9235/ Log: Update diff --git a/TODO b/TODO --- a/TODO +++ b/TODO @@ -17,6 +17,9 @@ * the rest: ~9% from unknown other places (may include accessing prebuilt GC objects, which requires an indirection) +UPDATE: with ddbc16971682 the figure seems to be: only 38% slower. +Assuming that all other points stayed at the same overhead, it would +perfectly explain the slow-down. From noreply at buildbot.pypy.org Sat Mar 15 12:08:11 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 15 Mar 2014 12:08:11 +0100 (CET) Subject: [pypy-commit] extradoc extradoc: Add a blog draft Message-ID: <20140315110811.553F51C0124@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r5163:61f1eb9baa3a Date: 2014-03-15 12:08 +0100 http://bitbucket.org/pypy/extradoc/changeset/61f1eb9baa3a/ Log: Add a blog draft diff --git a/blog/draft/stm-mar2014.txt b/blog/draft/stm-mar2014.txt new file mode 100644 --- /dev/null +++ b/blog/draft/stm-mar2014.txt @@ -0,0 +1,29 @@ +Hi all, + +Here is one +of the first full PyPy's compiled with the new StmGC-c7 +library. It has no JIT so far, but it runs some small +single-threaded benchmarks by taking around 40% more time than a +corresponding non-STM, no-JIT version of PyPy. It scales --- up to two +threads only, which is the hard-coded maximum so far in the c7 code. +But the scaling looks perfect in these small benchmarks without +conflict: starting two threads each running a copy of the benchmark +takes almost exactly the same amount of total time, simply using two +cores. + +Feel free to try it! It is not actually useful so far, because it is +limited to two cores and CPython is something like 2.5x faster. One of +the important next steps is to re-enable the JIT. Based on our current +understanding the "40%" figure, we can probably reduce it with +enough efforts; but also, the JIT should be able to easily produce +machine code that suffers a bit less than the interpreter from these +effects. This seems to mean that we're looking at 20%-ish slow-downs +for the future PyPy-STM-JIT. + +Interesting times :-) + + +Armin (as well as Remi for the work) From noreply at buildbot.pypy.org Sat Mar 15 17:28:10 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 15 Mar 2014 17:28:10 +0100 (CET) Subject: [pypy-commit] stmgc c7-more-segments: A branch in which to try to support more than 2 segments Message-ID: <20140315162810.76E451C0124@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c7-more-segments Changeset: r1018:ce2390ac80bf Date: 2014-03-15 17:01 +0100 http://bitbucket.org/pypy/stmgc/changeset/ce2390ac80bf/ Log: A branch in which to try to support more than 2 segments From noreply at buildbot.pypy.org Sat Mar 15 17:28:11 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 15 Mar 2014 17:28:11 +0100 (CET) Subject: [pypy-commit] stmgc c7-more-segments: Kill unused stuff Message-ID: <20140315162811.967EF1C0124@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c7-more-segments Changeset: r1019:1eef635359f7 Date: 2014-03-15 17:01 +0100 http://bitbucket.org/pypy/stmgc/changeset/1eef635359f7/ Log: Kill unused stuff diff --git a/c7/stm/core.c b/c7/stm/core.c --- a/c7/stm/core.c +++ b/c7/stm/core.c @@ -59,7 +59,7 @@ the common case. Otherwise, we need to compute it based on its location and size. */ if ((obj->stm_flags & GCFLAG_SMALL_UNIFORM) != 0) { - pages_privatize(first_page, 1, true); + pages_privatize(first_page, 1); } else { char *realobj; @@ -73,7 +73,7 @@ /* that's the page *following* the last page with the object */ end_page = (((uintptr_t)obj) + obj_size + 4095) / 4096UL; - pages_privatize(first_page, end_page - first_page, true); + pages_privatize(first_page, end_page - first_page); } } else if (write_locks[lock_idx] == lock_num) { diff --git a/c7/stm/pages.c b/c7/stm/pages.c --- a/c7/stm/pages.c +++ b/c7/stm/pages.c @@ -133,7 +133,7 @@ } #endif -static void privatize_range(uintptr_t pagenum, uintptr_t count, bool full) +static void privatize_range(uintptr_t pagenum, uintptr_t count) { ssize_t pgoff1 = pagenum; ssize_t pgoff2 = pagenum + NB_PAGES; @@ -146,29 +146,16 @@ memset(flag_page_private + pagenum, REMAPPING_PAGE, count); d_remap_file_pages(localpg, count * 4096, pgoff2); uintptr_t i; - if (full) { - for (i = 0; i < count; i++) { - pagecopy(localpg + 4096 * i, otherpg + 4096 * i); - } - } - else { - pagecopy(localpg, otherpg); - if (count > 1) - pagecopy(localpg + 4096 * (count-1), otherpg + 4096 * (count-1)); + for (i = 0; i < count; i++) { + pagecopy(localpg + 4096 * i, otherpg + 4096 * i); } write_fence(); memset(flag_page_private + pagenum, PRIVATE_PAGE, count); increment_total_allocated(4096 * count); } -static void _pages_privatize(uintptr_t pagenum, uintptr_t count, bool full) +static void _pages_privatize(uintptr_t pagenum, uintptr_t count) { - /* narrow the range of pages to privatize from the end: */ - while (flag_page_private[pagenum + count - 1] == PRIVATE_PAGE) { - if (!--count) - return; - } - mutex_pages_lock(); uintptr_t page_start_range = pagenum; @@ -179,7 +166,7 @@ if (prev == PRIVATE_PAGE) { if (pagenum > page_start_range) { privatize_range(page_start_range, - pagenum - page_start_range, full); + pagenum - page_start_range); } page_start_range = pagenum + 1; } @@ -190,7 +177,7 @@ if (pagenum > page_start_range) { privatize_range(page_start_range, - pagenum - page_start_range, full); + pagenum - page_start_range); } mutex_pages_unlock(); diff --git a/c7/stm/pages.h b/c7/stm/pages.h --- a/c7/stm/pages.h +++ b/c7/stm/pages.h @@ -21,9 +21,8 @@ static uint8_t flag_page_private[NB_PAGES]; -static void _pages_privatize(uintptr_t pagenum, uintptr_t count, bool full); +static void _pages_privatize(uintptr_t pagenum, uintptr_t count); static void pages_initialize_shared(uintptr_t pagenum, uintptr_t count); -//static void pages_make_shared_again(uintptr_t pagenum, uintptr_t count); static void mutex_pages_lock(void); static void mutex_pages_unlock(void); @@ -32,8 +31,7 @@ static void force_major_collection_request(void); static void reset_major_collection_requested(void); -inline static void pages_privatize(uintptr_t pagenum, uintptr_t count, - bool full) { +inline static void pages_privatize(uintptr_t pagenum, uintptr_t count) { /* This is written a bit carefully so that a call with a constant count == 1 will turn this loop into just one "if". */ while (flag_page_private[pagenum] == PRIVATE_PAGE) { @@ -42,7 +40,5 @@ } pagenum++; } - _pages_privatize(pagenum, count, full); + _pages_privatize(pagenum, count); } - -/* static bool is_fully_in_shared_pages(object_t *obj); */ From noreply at buildbot.pypy.org Sat Mar 15 17:28:12 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 15 Mar 2014 17:28:12 +0100 (CET) Subject: [pypy-commit] stmgc c7-more-segments: Disable this too Message-ID: <20140315162812.A763F1C0124@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c7-more-segments Changeset: r1020:f8c3c7960895 Date: 2014-03-15 17:22 +0100 http://bitbucket.org/pypy/stmgc/changeset/f8c3c7960895/ Log: Disable this too diff --git a/c7/stm/gcpage.c b/c7/stm/gcpage.c --- a/c7/stm/gcpage.c +++ b/c7/stm/gcpage.c @@ -193,6 +193,7 @@ /************************************************************/ +#if 0 static inline void mark_single_flag_private(uintptr_t pagenum) { if (flag_page_private[pagenum] == PRIVATE_PAGE) { @@ -314,6 +315,7 @@ (uninitialized_page_stop - stm_object_pages) / 4096UL, NB_PAGES); } +#endif /************************************************************/ @@ -346,11 +348,13 @@ while (1) { +#if 0 /* first, if we're not seeing segment 0, we must change the flags in flag_page_private[] from PRIVATE_PAGE to SEGMENT1_PAGE, which will mean "can't re-share" */ if (segment_base != stm_object_pages && RESHARE_PAGES) mark_flag_page_private(obj, segment_base); +#endif /* trace into the object (the version from 'segment_base') */ struct object_s *realobj = @@ -549,8 +553,10 @@ /* sweeping */ mutex_pages_lock(); +#if 0 if (RESHARE_PAGES) major_reshare_pages(); +#endif sweep_large_objects(); //sweep_uniform_pages(); mutex_pages_unlock(); diff --git a/c7/stm/pages.h b/c7/stm/pages.h --- a/c7/stm/pages.h +++ b/c7/stm/pages.h @@ -13,10 +13,6 @@ /* Page is private for each segment. */ PRIVATE_PAGE, - - /* gcpage.c: page contains objects that have been traced in the - segment > 0 */ - SEGMENT1_PAGE, }; static uint8_t flag_page_private[NB_PAGES]; From noreply at buildbot.pypy.org Sat Mar 15 17:28:13 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 15 Mar 2014 17:28:13 +0100 (CET) Subject: [pypy-commit] stmgc default: Fix for a remaining inevitable transaction Message-ID: <20140315162813.BC90B1C0124@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1021:2203a869efb3 Date: 2014-03-15 17:24 +0100 http://bitbucket.org/pypy/stmgc/changeset/2203a869efb3/ Log: Fix for a remaining inevitable transaction diff --git a/c7/test/support.py b/c7/test/support.py --- a/c7/test/support.py +++ b/c7/test/support.py @@ -413,7 +413,10 @@ if lib._stm_in_transaction(tl): if self.current_thread != n: self.switch(n) - self.abort_transaction() + if lib.stm_is_inevitable(): + self.commit_transaction() # must succeed! + else: + self.abort_transaction() for tl in self.tls: lib.stm_unregister_thread_local(tl) lib.stm_teardown() From noreply at buildbot.pypy.org Sat Mar 15 17:28:14 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 15 Mar 2014 17:28:14 +0100 (CET) Subject: [pypy-commit] stmgc default: Skip this test for now Message-ID: <20140315162814.C4E441C0124@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1022:6b3a3103ccc4 Date: 2014-03-15 17:26 +0100 http://bitbucket.org/pypy/stmgc/changeset/6b3a3103ccc4/ Log: Skip this test for now diff --git a/c7/test/test_weakref.py b/c7/test/test_weakref.py --- a/c7/test/test_weakref.py +++ b/c7/test/test_weakref.py @@ -146,6 +146,7 @@ class TestIsolation(BaseTest): def test_not_break(self): + py.test.xfail("known to fail") lpold = stm_allocate_old_refs(1) self.start_transaction() From noreply at buildbot.pypy.org Sat Mar 15 17:48:12 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 15 Mar 2014 17:48:12 +0100 (CET) Subject: [pypy-commit] extradoc extradoc: Add a paragraph about how to try it Message-ID: <20140315164812.4E86E1C0124@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r5164:5fab61cd53f6 Date: 2014-03-15 17:48 +0100 http://bitbucket.org/pypy/extradoc/changeset/5fab61cd53f6/ Log: Add a paragraph about how to try it diff --git a/blog/draft/stm-mar2014.txt b/blog/draft/stm-mar2014.txt --- a/blog/draft/stm-mar2014.txt +++ b/blog/draft/stm-mar2014.txt @@ -17,7 +17,7 @@ limited to two cores and CPython is something like 2.5x faster. One of the important next steps is to re-enable the JIT. Based on our current -understanding the "40%" figure, we can probably reduce it with +understanding of the "40%" figure, we can probably reduce it with enough efforts; but also, the JIT should be able to easily produce machine code that suffers a bit less than the interpreter from these effects. This seems to mean that we're looking at 20%-ish slow-downs @@ -25,5 +25,15 @@ Interesting times :-) +For reference, this is what you get by downloading the +PyPy binary linked above: a Linux 64 binary (Ubuntu 12.04) that +should behave mostly like a regular PyPy. (One main missing feature is +that destructors are never called.) It uses two cores, but obviously +only if the Python program you run is multithreaded. The only new +built-in feature is with __pypy__.thread.atomic: this gives +you a way to enforce that a block of code runs "atomically", which means +without any operation from any other thread randomly interleaved. -Armin (as well as Remi for the work) + +Armin & Remi From noreply at buildbot.pypy.org Sat Mar 15 17:50:04 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 15 Mar 2014 17:50:04 +0100 (CET) Subject: [pypy-commit] extradoc extradoc: A goodbye-like sentence Message-ID: <20140315165004.01D491C0124@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r5165:c352d811ef4a Date: 2014-03-15 17:49 +0100 http://bitbucket.org/pypy/extradoc/changeset/c352d811ef4a/ Log: A goodbye-like sentence diff --git a/blog/draft/stm-mar2014.txt b/blog/draft/stm-mar2014.txt --- a/blog/draft/stm-mar2014.txt +++ b/blog/draft/stm-mar2014.txt @@ -35,5 +35,7 @@ you a way to enforce that a block of code runs "atomically", which means without any operation from any other thread randomly interleaved. +Stay tuned for more! + Armin & Remi From noreply at buildbot.pypy.org Sat Mar 15 18:00:21 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 15 Mar 2014 18:00:21 +0100 (CET) Subject: [pypy-commit] extradoc extradoc: Yet another small paragraph Message-ID: <20140315170021.0E1371C0124@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r5166:f2fa60f6ed2b Date: 2014-03-15 18:00 +0100 http://bitbucket.org/pypy/extradoc/changeset/f2fa60f6ed2b/ Log: Yet another small paragraph diff --git a/blog/draft/stm-mar2014.txt b/blog/draft/stm-mar2014.txt --- a/blog/draft/stm-mar2014.txt +++ b/blog/draft/stm-mar2014.txt @@ -35,6 +35,11 @@ you a way to enforce that a block of code runs "atomically", which means without any operation from any other thread randomly interleaved. +If you want to translate it yourself, you need a trunk version of clang +with three patches applied. That's the number of bugs that we couldn't +find workarounds for, not the total number of bugs we found by (ab)using +the address_space feature... + Stay tuned for more! From noreply at buildbot.pypy.org Sat Mar 15 18:18:25 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 15 Mar 2014 18:18:25 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Mention multiple stm_writes in a row Message-ID: <20140315171825.5EA481C011F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r69975:9cbbb042ad60 Date: 2014-03-15 18:17 +0100 http://bitbucket.org/pypy/pypy/changeset/9cbbb042ad60/ Log: Mention multiple stm_writes in a row diff --git a/TODO b/TODO --- a/TODO +++ b/TODO @@ -21,6 +21,14 @@ Assuming that all other points stayed at the same overhead, it would perfectly explain the slow-down. +------------------------------------------------------------ + +clang doesn't optimize multiple stm_write() in a row (unlike GCC). +Optimize them manually... + +------------------------------------------------------------ + + From noreply at buildbot.pypy.org Sat Mar 15 18:42:16 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 15 Mar 2014 18:42:16 +0100 (CET) Subject: [pypy-commit] stmgc c7-more-segments: Write a long comment about the new model I'm aiming for. Message-ID: <20140315174216.C6B6B1C011F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c7-more-segments Changeset: r1023:fa8bfb4eb410 Date: 2014-03-15 18:42 +0100 http://bitbucket.org/pypy/stmgc/changeset/fa8bfb4eb410/ Log: Write a long comment about the new model I'm aiming for. diff --git a/c7/stm/pages.h b/c7/stm/pages.h --- a/c7/stm/pages.h +++ b/c7/stm/pages.h @@ -1,21 +1,32 @@ -enum /* flag_page_private */ { - /* The page is not in use. Assume that each segment sees its own copy. */ - FREE_PAGE=0, +/* For every page, 'num_segments_sharing_page' stores a number that + counts the number of segments that share the page. If 0, the page is + not used so far. - /* The page is shared by all segments. Each segment sees the same - physical page (the one that is within the segment 0 mmap address). */ - SHARED_PAGE, + When the page is first initialized, 'num_segments_sharing_page' is + set to NB_SEGMENTS. When later a segment wants a private copy, it + looks first in its own 'private_page_mapping' tree, which maps shared + pages to private copies. If not found, then it proceeds like this: - /* For only one range of pages at a time, around the call to - remap_file_pages() that un-shares the pages (SHARED -> PRIVATE). */ - REMAPPING_PAGE, + If 'num_segments_sharing_page' is greater than 1, then it is + decremented and a private copy of the page is made. - /* Page is private for each segment. */ - PRIVATE_PAGE, -}; + If 'num_segments_sharing_page' is equal to 1, then we know we are the + last segment that sees this "shared" copy, and so it is actually not + shared with anybody else --- i.e. it is private already. -static uint8_t flag_page_private[NB_PAGES]; + The shared copy of a page is stored in the mmap at the file offset + corresponding to the segment 0 offset (with all other segments + remapping to the segment 0 offset). Private copies are made in the + offset from segment 1 (and if full, more segments afterwards), + picking file offsets that are simply the next free ones. This is + probably good for long-term memory usage: a major collection looks + for pages that are no-longer-used private copies of some shared page, + and discard them, remapping the address to the shared page. The + pages thus freed are recorded into a free list, and can be reused as + the private copies of the following (unrelated) pages. +*/ +static uint8_t num_segments_sharing_page[NB_PAGES]; static void _pages_privatize(uintptr_t pagenum, uintptr_t count); static void pages_initialize_shared(uintptr_t pagenum, uintptr_t count); From noreply at buildbot.pypy.org Sat Mar 15 19:01:24 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 15 Mar 2014 19:01:24 +0100 (CET) Subject: [pypy-commit] stmgc c7-more-segments: Complete Message-ID: <20140315180124.8F0CB1C0124@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c7-more-segments Changeset: r1024:4ab1f6857a09 Date: 2014-03-15 18:54 +0100 http://bitbucket.org/pypy/stmgc/changeset/4ab1f6857a09/ Log: Complete diff --git a/c7/stm/pages.h b/c7/stm/pages.h --- a/c7/stm/pages.h +++ b/c7/stm/pages.h @@ -15,16 +15,31 @@ last segment that sees this "shared" copy, and so it is actually not shared with anybody else --- i.e. it is private already. + (This means that 'num_segments_sharing_page' is basically just an + optimization. Without it, we might need 'NB_SEGMENTS + 1' copies of + the same data; with it, we can bound the number to 'NB_SEGMENTS'. + This is probably important if NB_SEGMENTS is very small.) + The shared copy of a page is stored in the mmap at the file offset corresponding to the segment 0 offset (with all other segments remapping to the segment 0 offset). Private copies are made in the offset from segment 1 (and if full, more segments afterwards), picking file offsets that are simply the next free ones. This is probably good for long-term memory usage: a major collection looks - for pages that are no-longer-used private copies of some shared page, + for pages that are no-longer-used private copies of some shared page(*), and discard them, remapping the address to the shared page. The pages thus freed are recorded into a free list, and can be reused as the private copies of the following (unrelated) pages. + + (*) an additional subtlety here is that the shared page should not + contain uncommitted changes; if 'num_segments_sharing_page' is 1 this + can occur. + + Note that this page manipulation logic is independent from actually + tracking which objects are uncommitted, which occurs at the level of + segment-relative offsets; and propagating changes during commit, + which is done by copying objects (not pages) to the same offset + relative to a different segment. */ static uint8_t num_segments_sharing_page[NB_PAGES]; From noreply at buildbot.pypy.org Sat Mar 15 19:01:25 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 15 Mar 2014 19:01:25 +0100 (CET) Subject: [pypy-commit] stmgc c7-more-segments: Kill again 'num_segments_sharing_page' for now; it's just an additional complication. Message-ID: <20140315180125.BBF5B1C0124@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c7-more-segments Changeset: r1025:49e2782f8a0b Date: 2014-03-15 19:01 +0100 http://bitbucket.org/pypy/stmgc/changeset/49e2782f8a0b/ Log: Kill again 'num_segments_sharing_page' for now; it's just an additional complication. diff --git a/c7/stm/pages.h b/c7/stm/pages.h --- a/c7/stm/pages.h +++ b/c7/stm/pages.h @@ -1,39 +1,19 @@ -/* For every page, 'num_segments_sharing_page' stores a number that - counts the number of segments that share the page. If 0, the page is - not used so far. - - When the page is first initialized, 'num_segments_sharing_page' is - set to NB_SEGMENTS. When later a segment wants a private copy, it - looks first in its own 'private_page_mapping' tree, which maps shared - pages to private copies. If not found, then it proceeds like this: - - If 'num_segments_sharing_page' is greater than 1, then it is - decremented and a private copy of the page is made. - - If 'num_segments_sharing_page' is equal to 1, then we know we are the - last segment that sees this "shared" copy, and so it is actually not - shared with anybody else --- i.e. it is private already. - - (This means that 'num_segments_sharing_page' is basically just an - optimization. Without it, we might need 'NB_SEGMENTS + 1' copies of - the same data; with it, we can bound the number to 'NB_SEGMENTS'. - This is probably important if NB_SEGMENTS is very small.) +/* This handles pages of objects outside the nursery. Every page + has a "shared copy" and zero or more "private copies". The shared copy of a page is stored in the mmap at the file offset corresponding to the segment 0 offset (with all other segments remapping to the segment 0 offset). Private copies are made in the offset from segment 1 (and if full, more segments afterwards), - picking file offsets that are simply the next free ones. This is - probably good for long-term memory usage: a major collection looks - for pages that are no-longer-used private copies of some shared page(*), - and discard them, remapping the address to the shared page. The - pages thus freed are recorded into a free list, and can be reused as - the private copies of the following (unrelated) pages. + picking file offsets that are simply the next free ones. Each + segment maintains a tree 'private_page_mapping', which maps shared + pages to private copies. - (*) an additional subtlety here is that the shared page should not - contain uncommitted changes; if 'num_segments_sharing_page' is 1 this - can occur. + A major collection looks for pages that are no-longer-used private + copies, and discard them, remapping the address to the shared page. + The pages thus freed are recorded into a free list, and can be reused + as the private copies of the following (unrelated) pages. Note that this page manipulation logic is independent from actually tracking which objects are uncommitted, which occurs at the level of @@ -41,7 +21,6 @@ which is done by copying objects (not pages) to the same offset relative to a different segment. */ -static uint8_t num_segments_sharing_page[NB_PAGES]; static void _pages_privatize(uintptr_t pagenum, uintptr_t count); static void pages_initialize_shared(uintptr_t pagenum, uintptr_t count); From noreply at buildbot.pypy.org Sun Mar 16 00:03:43 2014 From: noreply at buildbot.pypy.org (mattip) Date: Sun, 16 Mar 2014 00:03:43 +0100 (CET) Subject: [pypy-commit] pypy default: a better fix for ignoring pragma commane(lib...) on MSVC Message-ID: <20140315230343.145051C03A3@cobra.cs.uni-duesseldorf.de> Author: Matti Picus Branch: Changeset: r69976:b03b791c1e26 Date: 2014-03-16 00:50 +0200 http://bitbucket.org/pypy/pypy/changeset/b03b791c1e26/ Log: a better fix for ignoring pragma commane(lib...) on MSVC diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py --- a/pypy/module/cpyext/test/test_cpyext.py +++ b/pypy/module/cpyext/test/test_cpyext.py @@ -65,7 +65,7 @@ # '%s' undefined; assuming extern returning int kwds["compile_extra"] = ["/we4013"] # prevent linking with python27.lib - kwds["compile_extra"].append("/DPy_BUILD_CORE") + kwds["link_extra"]=["/NODEFAULTLIB:Python27.lib"] elif sys.platform == 'darwin': kwds["link_files"] = [str(api_library + '.dylib')] else: From noreply at buildbot.pypy.org Sun Mar 16 00:03:44 2014 From: noreply at buildbot.pypy.org (mattip) Date: Sun, 16 Mar 2014 00:03:44 +0100 (CET) Subject: [pypy-commit] pypy default: a more generic extern declaration Message-ID: <20140315230344.6E3041C03A3@cobra.cs.uni-duesseldorf.de> Author: Matti Picus Branch: Changeset: r69977:0e3e28702d34 Date: 2014-03-16 00:55 +0200 http://bitbucket.org/pypy/pypy/changeset/0e3e28702d34/ Log: a more generic extern declaration diff --git a/pypy/module/cpyext/include/numpy/arrayobject.h b/pypy/module/cpyext/include/numpy/arrayobject.h --- a/pypy/module/cpyext/include/numpy/arrayobject.h +++ b/pypy/module/cpyext/include/numpy/arrayobject.h @@ -20,7 +20,7 @@ #define PyArrayObject PyObject #define PyArray_Descr PyObject -extern PyTypeObject PyArray_Type; +PyAPI_DATA(PyTypeObject) PyArray_Type; typedef unsigned char npy_bool; typedef unsigned char npy_uint8; From noreply at buildbot.pypy.org Sun Mar 16 01:51:56 2014 From: noreply at buildbot.pypy.org (pjenvey) Date: Sun, 16 Mar 2014 01:51:56 +0100 (CET) Subject: [pypy-commit] pypy default: don't hardcode the version Message-ID: <20140316005156.DDF821C0124@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: Changeset: r69978:ad48ce2a9283 Date: 2014-03-15 17:51 -0700 http://bitbucket.org/pypy/pypy/changeset/ad48ce2a9283/ Log: don't hardcode the version diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py --- a/pypy/module/cpyext/test/test_cpyext.py +++ b/pypy/module/cpyext/test/test_cpyext.py @@ -64,8 +64,10 @@ kwds["libraries"] = [api_library] # '%s' undefined; assuming extern returning int kwds["compile_extra"] = ["/we4013"] - # prevent linking with python27.lib - kwds["link_extra"]=["/NODEFAULTLIB:Python27.lib"] + # prevent linking with PythonXX.lib + w_maj, w_min = space.fixedview(space.sys.get('version_info'), 5)[:2] + kwds["link_extra"] = ["/NODEFAULTLIB:Python%d%d.lib" % + (space.int_w(w_maj), space.int_w(w_min))] elif sys.platform == 'darwin': kwds["link_files"] = [str(api_library + '.dylib')] else: From noreply at buildbot.pypy.org Sun Mar 16 07:49:14 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 16 Mar 2014 07:49:14 +0100 (CET) Subject: [pypy-commit] stmgc default: Oups (for tests only) Message-ID: <20140316064914.65D0C1C00B9@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1026:13df2f3e0e20 Date: 2014-03-16 07:49 +0100 http://bitbucket.org/pypy/stmgc/changeset/13df2f3e0e20/ Log: Oups (for tests only) diff --git a/c7/stm/sync.c b/c7/stm/sync.c --- a/c7/stm/sync.c +++ b/c7/stm/sync.c @@ -60,7 +60,7 @@ stm_fatalerror("cond destroy: %m\n"); } - memset(&sync_ctl, 0, sizeof(sync_ctl.in_use)); + memset(&sync_ctl, 0, sizeof(sync_ctl)); } #ifndef NDEBUG From noreply at buildbot.pypy.org Sun Mar 16 09:29:56 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 16 Mar 2014 09:29:56 +0100 (CET) Subject: [pypy-commit] stmgc c7-more-segments: in-progress Message-ID: <20140316082956.7B5001C0670@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c7-more-segments Changeset: r1027:67d4e6e71904 Date: 2014-03-16 09:29 +0100 http://bitbucket.org/pypy/stmgc/changeset/67d4e6e71904/ Log: in-progress diff --git a/c7/stm/core.c b/c7/stm/core.c --- a/c7/stm/core.c +++ b/c7/stm/core.c @@ -29,6 +29,8 @@ safepoints that may be issued in write_write_contention_management(). */ stm_read(obj); + /* XXX XXX XXX make the logic of write-locking objects optional! */ + /* claim the write-lock for this object. In case we're running the same transaction since a long while, the object can be already in 'modified_old_objects' (but, because it had GCFLAG_WRITE_BARRIER, @@ -59,12 +61,12 @@ the common case. Otherwise, we need to compute it based on its location and size. */ if ((obj->stm_flags & GCFLAG_SMALL_UNIFORM) != 0) { - pages_privatize(first_page, 1); + page_privatize(first_page); } else { char *realobj; size_t obj_size; - uintptr_t end_page; + uintptr_t i, end_page; /* get the size of the object */ realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj); @@ -73,7 +75,9 @@ /* that's the page *following* the last page with the object */ end_page = (((uintptr_t)obj) + obj_size + 4095) / 4096UL; - pages_privatize(first_page, end_page - first_page); + for (i = first_page; i < end_page; i++) { + page_privatize(i); + } } } else if (write_locks[lock_idx] == lock_num) { @@ -108,7 +112,7 @@ /* for sanity, check that all other segment copies of this object still have the flag */ long i; - for (i = 0; i < NB_SEGMENTS; i++) { + for (i = 1; i <= NB_SEGMENTS; i++) { if (i != STM_SEGMENT->segment_num) assert(((struct object_s *)REAL_ADDRESS(get_segment_base(i), obj)) ->stm_flags & GCFLAG_WRITE_BARRIER); @@ -193,43 +197,51 @@ /************************************************************/ -#if NB_SEGMENTS != 2 -# error "The logic in the functions below only works with two segments" -#endif static bool detect_write_read_conflicts(void) { - long remote_num = 1 - STM_SEGMENT->segment_num; - char *remote_base = get_segment_base(remote_num); - uint8_t remote_version = get_segment(remote_num)->transaction_read_version; + /* Detect conflicts of the form: we want to commit a write to an object, + but the same object was also read in a different thread. + */ + long i; + for (i = 1; i <= NB_SEGMENTS; i++) { - if (get_priv_segment(remote_num)->transaction_state == TS_NONE) - return false; /* no need to check */ + if (i == STM_SEGMENT->segment_num) + continue; - if (is_aborting_now(remote_num)) - return false; /* no need to check: is pending immediate abort */ + if (get_priv_segment(i)->transaction_state == TS_NONE) + continue; /* no need to check */ - LIST_FOREACH_R( - STM_PSEGMENT->modified_old_objects, - object_t * /*item*/, - ({ - if (was_read_remote(remote_base, item, remote_version)) { - /* A write-read conflict! */ - write_read_contention_management(remote_num); + if (is_aborting_now(i)) + continue; /* no need to check: is pending immediate abort */ - /* If we reach this point, we didn't abort, but maybe we - had to wait for the other thread to commit. If we - did, then we have to restart committing from our call - to synchronize_all_threads(). */ - return true; - } - })); + char *remote_base = get_segment_base(i); + uint8_t remote_version = get_segment(i)->transaction_read_version; + + LIST_FOREACH_R( + STM_PSEGMENT->modified_old_objects, + object_t * /*item*/, + ({ + if (was_read_remote(remote_base, item, remote_version)) { + /* A write-read conflict! */ + write_read_contention_management(i); + + /* If we reach this point, we didn't abort, but maybe we + had to wait for the other thread to commit. If we + did, then we have to restart committing from our call + to synchronize_all_threads(). */ + return true; + } + })); + } return false; } static void synchronize_overflow_object_now(object_t *obj) { + abort();//XXX +#if 0 assert(!_is_young(obj)); assert((obj->stm_flags & GCFLAG_SMALL_UNIFORM) == 0); assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); @@ -264,6 +276,7 @@ long i; char *src = REAL_ADDRESS(STM_SEGMENT->segment_base, start); for (i = 0; i < NB_SEGMENTS; i++) { + abort();//XXX if (i != STM_SEGMENT->segment_num) { char *dst = REAL_ADDRESS(get_segment_base(i), start); memcpy(dst, src, copy_size); @@ -273,6 +286,68 @@ start = (start + 4096) & ~4095; } while (first_page++ < last_page); +#endif +} + +static void synchronize_object_now(object_t *obj) +{ + /* Assume that the version of 'obj' in the shared pages is up-to-date. + Assume also that the version in our own private page is up-to-date. + This function updates the private page of other threads. + */ + assert(!_is_young(obj)); + assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); + + uintptr_t start = (uintptr_t)obj; + uintptr_t first_page = start / 4096UL; + long i; + + if (obj->stm_flags & GCFLAG_SMALL_UNIFORM) { + abort();//XXX WRITE THE FAST CASE + } + else { + char *realobj = REAL_ADDRESS(stm_object_pages, obj); + ssize_t obj_size = stmcb_size_rounded_up((struct object_s *)realobj); + assert(obj_size >= 16); + uintptr_t end = start + obj_size; + uintptr_t last_page = (end - 1) / 4096UL; + + for (; first_page <= last_page; first_page++) { + + for (i = 1; i < NB_SEGMENTS; i++) { + + if (i == STM_SEGMENT->segment_num) + continue; + + if (!is_private_page(i, first_page)) + continue; + + /* The page is a PRIVATE_PAGE. We need to diffuse this + fragment of object from the shared page to this private + page. */ + + uintptr_t copy_size; + if (first_page == last_page) { + /* this is the final fragment */ + copy_size = end - start; + } + else { + /* this is a non-final fragment, going up to the + page's end */ + copy_size = 4096 - (start & 4095); + } + + /* double-check that the result fits in one page */ + assert(copy_size > 0); + assert(copy_size + (start & 4095) <= 4096); + + char *src = REAL_ADDRESS(stm_object_pages, start); + char *dst = REAL_ADDRESS(get_segment_base(i), start); + memcpy(dst, src, copy_size); + } + start = (start + 4096) & ~4095; + } + } } static void push_overflow_objects_from_privatized_pages(void) @@ -286,22 +361,12 @@ static void push_modified_to_other_segments(void) { - long remote_num = 1 - STM_SEGMENT->segment_num; char *local_base = STM_SEGMENT->segment_base; - char *remote_base = get_segment_base(remote_num); - bool remote_active = - (get_priv_segment(remote_num)->transaction_state != TS_NONE && - get_segment(remote_num)->nursery_end != NSE_SIGABORT); LIST_FOREACH_R( STM_PSEGMENT->modified_old_objects, object_t * /*item*/, ({ - if (remote_active) { - assert(!was_read_remote(remote_base, item, - get_segment(remote_num)->transaction_read_version)); - } - /* clear the write-lock (note that this runs with all other threads paused, so no need to be careful about ordering) */ uintptr_t lock_idx = (((uintptr_t)item) >> 4) - WRITELOCK_START; @@ -313,11 +378,14 @@ minor_collection() */ assert((item->stm_flags & GCFLAG_WRITE_BARRIER) != 0); - /* copy the modified object to the other segment */ + /* copy the modified object to the shared copy */ char *src = REAL_ADDRESS(local_base, item); - char *dst = REAL_ADDRESS(remote_base, item); + char *dst = REAL_ADDRESS(stm_object_pages, item); ssize_t size = stmcb_size_rounded_up((struct object_s *)src); memcpy(dst, src, size); + + /* copy the object to the other private pages as needed */ + synchronize_object_now(item); })); list_clear(STM_PSEGMENT->modified_old_objects); diff --git a/c7/stm/core.h b/c7/stm/core.h --- a/c7/stm/core.h +++ b/c7/stm/core.h @@ -20,7 +20,7 @@ #define MAP_PAGES_FLAGS (MAP_SHARED | MAP_ANONYMOUS | MAP_NORESERVE) #define NB_NURSERY_PAGES (STM_GC_NURSERY/4) -#define TOTAL_MEMORY (NB_PAGES * 4096UL * NB_SEGMENTS) +#define TOTAL_MEMORY (NB_PAGES * 4096UL * (1 + NB_SEGMENTS)) #define READMARKER_END ((NB_PAGES * 4096UL) >> 4) #define FIRST_OBJECT_PAGE ((READMARKER_END + 4095) / 4096UL) #define FIRST_NURSERY_PAGE FIRST_OBJECT_PAGE @@ -75,6 +75,13 @@ struct stm_priv_segment_info_s { struct stm_segment_info_s pub; + /* Dict whose keys are shared page numbers, and whose values are + the corresponding private page number. */ + struct tree_s *private_page_mapping; + + /* Head of a free list of private pages. */ + uintptr_t private_free_page_num; + /* List of old objects (older than the current transaction) that the current transaction attempts to modify. This is used to track the STM status: they are old objects that where written to and @@ -178,10 +185,6 @@ static char *stm_object_pages; static stm_thread_local_t *stm_all_thread_locals = NULL; -#ifdef STM_TESTS -static char *stm_other_pages; -#endif - static uint8_t write_locks[WRITELOCK_END - WRITELOCK_START]; diff --git a/c7/stm/misc.c b/c7/stm/misc.c --- a/c7/stm/misc.c +++ b/c7/stm/misc.c @@ -43,7 +43,7 @@ #ifdef STM_TESTS uint8_t _stm_get_page_flag(uintptr_t index) { - return flag_page_private[index]; + abort();//XXX } long _stm_count_modified_old_objects(void) diff --git a/c7/stm/nursery.c b/c7/stm/nursery.c --- a/c7/stm/nursery.c +++ b/c7/stm/nursery.c @@ -26,7 +26,7 @@ _stm_nursery_start = NURSERY_START; long i; - for (i = 0; i < NB_SEGMENTS; i++) { + for (i = 1; i <= NB_SEGMENTS; i++) { get_segment(i)->nursery_current = (stm_char *)NURSERY_START; get_segment(i)->nursery_end = NURSERY_END; } @@ -378,7 +378,7 @@ _stm_nursery_start = NURSERY_END - free_count; long i; - for (i = 0; i < NB_SEGMENTS; i++) { + for (i = 1; i <= NB_SEGMENTS; i++) { if ((uintptr_t)get_segment(i)->nursery_current < _stm_nursery_start) get_segment(i)->nursery_current = (stm_char *)_stm_nursery_start; } @@ -411,7 +411,7 @@ int original_num = STM_SEGMENT->segment_num; long i; - for (i = 0; i < NB_SEGMENTS; i++) { + for (i = 1; i <= NB_SEGMENTS; i++) { struct stm_priv_segment_info_s *pseg = get_priv_segment(i); if (MINOR_NOTHING_TO_DO(pseg)) /*TS_NONE segments have NOTHING_TO_DO*/ continue; diff --git a/c7/stm/pages.c b/c7/stm/pages.c --- a/c7/stm/pages.c +++ b/c7/stm/pages.c @@ -102,85 +102,44 @@ segment 0. */ uintptr_t i; assert(_has_mutex_pages()); - for (i = 1; i < NB_SEGMENTS; i++) { + for (i = 1; i <= NB_SEGMENTS; i++) { char *segment_base = get_segment_base(i); d_remap_file_pages(segment_base + pagenum * 4096UL, count * 4096UL, pagenum); } - for (i = 0; i < count; i++) - flag_page_private[pagenum + i] = SHARED_PAGE; } -#if 0 -static void pages_make_shared_again(uintptr_t pagenum, uintptr_t count) +static void page_privatize(uintptr_t pagenum) { - /* Same as pages_initialize_shared(), but tries hard to minimize the - total number of pages that remap_file_pages() must handle, by - fragmenting calls as much as possible (the overhead of one system - call appears smaller as the overhead per page). */ - uintptr_t start, i = 0; - while (i < count) { - if (flag_page_private[pagenum + (i++)] == SHARED_PAGE) - continue; - start = i; /* first index of a private page */ - while (1) { - i++; - if (i == count || flag_page_private[pagenum + i] == SHARED_PAGE) - break; - } - pages_initialize_shared(pagenum + start, i - start); + wlog_t *item; + TREE_FIND(*STM_PSEGMENT->private_page_mapping, pagenum, item, + goto not_found); + + /* the page is already privatized */ + return; + + not_found:; + /* look up the next free page */ + uintptr_t free_page_num = STM_PSEGMENT->private_free_page_num; + + /* "mount" it in the segment */ + char *new_page = STM_SEGMENT->segment_base + pagenum * 4096UL; + d_remap_file_pages(new_page, 4096, + NB_PAGES * STM_SEGMENT->segment_num + free_page_num); + increment_total_allocated(4096); + + /* update private_free_page_num */ + uintptr_t future_page = *(uintptr_t *)new_page; + if (future_page == 0) { + future_page = free_page_num + 1; } -} -#endif + STM_PSEGMENT->private_free_page_num = future_page; -static void privatize_range(uintptr_t pagenum, uintptr_t count) -{ - ssize_t pgoff1 = pagenum; - ssize_t pgoff2 = pagenum + NB_PAGES; - ssize_t localpgoff = pgoff1 + NB_PAGES * STM_SEGMENT->segment_num; - ssize_t otherpgoff = pgoff1 + NB_PAGES * (1 - STM_SEGMENT->segment_num); + /* copy the content from the shared (segment 0) source */ + pagecopy(new_page, stm_object_pages + pagenum * 4096UL); - void *localpg = stm_object_pages + localpgoff * 4096UL; - void *otherpg = stm_object_pages + otherpgoff * 4096UL; - - memset(flag_page_private + pagenum, REMAPPING_PAGE, count); - d_remap_file_pages(localpg, count * 4096, pgoff2); - uintptr_t i; - for (i = 0; i < count; i++) { - pagecopy(localpg + 4096 * i, otherpg + 4096 * i); - } - write_fence(); - memset(flag_page_private + pagenum, PRIVATE_PAGE, count); - increment_total_allocated(4096 * count); -} - -static void _pages_privatize(uintptr_t pagenum, uintptr_t count) -{ - mutex_pages_lock(); - - uintptr_t page_start_range = pagenum; - uintptr_t pagestop = pagenum + count; - - for (; pagenum < pagestop; pagenum++) { - uint8_t prev = flag_page_private[pagenum]; - if (prev == PRIVATE_PAGE) { - if (pagenum > page_start_range) { - privatize_range(page_start_range, - pagenum - page_start_range); - } - page_start_range = pagenum + 1; - } - else { - assert(prev == SHARED_PAGE); - } - } - - if (pagenum > page_start_range) { - privatize_range(page_start_range, - pagenum - page_start_range); - } - - mutex_pages_unlock(); + /* update private_page_mapping */ + tree_insert(STM_PSEGMENT->private_page_mapping, pagenum, free_page_num); } #if 0 diff --git a/c7/stm/pages.h b/c7/stm/pages.h --- a/c7/stm/pages.h +++ b/c7/stm/pages.h @@ -4,8 +4,8 @@ The shared copy of a page is stored in the mmap at the file offset corresponding to the segment 0 offset (with all other segments - remapping to the segment 0 offset). Private copies are made in the - offset from segment 1 (and if full, more segments afterwards), + remapping to the segment 0 offset). Private copies for segment N are + made in the offset from segment N (for 1 <= N <= NB_SEGMENTS), picking file offsets that are simply the next free ones. Each segment maintains a tree 'private_page_mapping', which maps shared pages to private copies. @@ -15,14 +15,14 @@ The pages thus freed are recorded into a free list, and can be reused as the private copies of the following (unrelated) pages. - Note that this page manipulation logic is independent from actually - tracking which objects are uncommitted, which occurs at the level of - segment-relative offsets; and propagating changes during commit, - which is done by copying objects (not pages) to the same offset - relative to a different segment. + Note that this page manipulation logic uses remap_file_pages() to + fully hide its execution cost behind the CPU's memory management unit. + It should not be confused with the logic of tracking which objects + are old-and-committed, old-but-modified, overflow objects, and so on + (which works at the object granularity, not the page granularity). */ -static void _pages_privatize(uintptr_t pagenum, uintptr_t count); +static void page_privatize(uintptr_t pagenum); static void pages_initialize_shared(uintptr_t pagenum, uintptr_t count); static void mutex_pages_lock(void); @@ -32,14 +32,8 @@ static void force_major_collection_request(void); static void reset_major_collection_requested(void); -inline static void pages_privatize(uintptr_t pagenum, uintptr_t count) { - /* This is written a bit carefully so that a call with a constant - count == 1 will turn this loop into just one "if". */ - while (flag_page_private[pagenum] == PRIVATE_PAGE) { - if (!--count) { - return; - } - pagenum++; - } - _pages_privatize(pagenum, count); +static inline bool is_private_page(long segnum, uintptr_t pagenum) +{ + return tree_contains(get_priv_segment(segnum)->private_page_mapping, + pagenum); } diff --git a/c7/stm/setup.c b/c7/stm/setup.c --- a/c7/stm/setup.c +++ b/c7/stm/setup.c @@ -26,12 +26,15 @@ if (stm_object_pages == MAP_FAILED) stm_fatalerror("initial stm_object_pages mmap() failed: %m\n"); + /* The segment 0 is not used to run transactions, but to contain the + shared copy of the pages. We mprotect all pages before so that + accesses fail, up to and including the pages corresponding to the + nurseries of the other segments. */ + mprotect(stm_object_pages, END_NURSERY_PAGE * 4096UL, PROT_NONE); + long i; - for (i = 0; i < NB_SEGMENTS; i++) { + for (i = 1; i <= NB_SEGMENTS; i++) { char *segment_base = get_segment_base(i); -#ifdef STM_TESTS - stm_other_pages = segment_base; -#endif /* In each segment, the first page is where TLPREFIX'ed NULL accesses land. We mprotect it so that accesses fail. */ @@ -39,7 +42,7 @@ /* Fill the TLS page (page 1) with 0xDC, for debugging */ memset(REAL_ADDRESS(segment_base, 4096), 0xDC, 4096); - /* Make a "hole" at STM_PSEGMENT */ + /* Make a "hole" at STM_PSEGMENT (which includes STM_SEGMENT) */ memset(REAL_ADDRESS(segment_base, STM_PSEGMENT), 0, sizeof(*STM_PSEGMENT)); @@ -49,11 +52,14 @@ (FIRST_READMARKER_PAGE - 2) * 4096UL, PROT_NONE); + /* Initialize STM_PSEGMENT */ struct stm_priv_segment_info_s *pr = get_priv_segment(i); - assert(i + 1 < 255); /* 255 is WL_VISITED in gcpage.c */ - pr->write_lock_num = i + 1; + assert(1 <= i && i < 255); /* 255 is WL_VISITED in gcpage.c */ + pr->write_lock_num = i; pr->pub.segment_num = i; pr->pub.segment_base = segment_base; + pr->private_page_mapping = tree_create(); + pr->private_free_page_num = END_NURSERY_PAGE; pr->objects_pointing_to_nursery = NULL; pr->large_overflow_objects = NULL; pr->modified_old_objects = list_create(); @@ -62,7 +68,7 @@ pr->young_outside_nursery = tree_create(); pr->nursery_objects_shadows = tree_create(); pr->callbacks_on_abort = tree_create(); - pr->overflow_number = GCFLAG_OVERFLOW_NUMBER_bit0 * (i + 1); + pr->overflow_number = GCFLAG_OVERFLOW_NUMBER_bit0 * i; highest_overflow_number = pr->overflow_number; } @@ -73,10 +79,6 @@ STM_SEGMENT->transaction_read_version never contains zero, so a null read marker means "not read" whatever the current transaction_read_version is. - - The creation markers are initially zero, which is correct: - it means "objects of this line of 256 bytes have not been - allocated by the current transaction." */ setup_sync(); @@ -92,7 +94,7 @@ assert(!_has_mutex()); long i; - for (i = 0; i < NB_SEGMENTS; i++) { + for (i = 1; i <= NB_SEGMENTS; i++) { struct stm_priv_segment_info_s *pr = get_priv_segment(i); assert(pr->objects_pointing_to_nursery == NULL); assert(pr->large_overflow_objects == NULL); @@ -107,8 +109,6 @@ munmap(stm_object_pages, TOTAL_MEMORY); stm_object_pages = NULL; - memset(flag_page_private, 0, sizeof(flag_page_private)); - teardown_core(); teardown_sync(); teardown_gcpage(); @@ -146,14 +146,14 @@ tl->prev = stm_all_thread_locals->prev; stm_all_thread_locals->prev->next = tl; stm_all_thread_locals->prev = tl; - num = tl->prev->associated_segment_num + 1; + num = tl->prev->associated_segment_num; } tl->thread_local_obj = NULL; /* assign numbers consecutively, but that's for tests; we could also assign the same number to all of them and they would get their own numbers automatically. */ - num = num % NB_SEGMENTS; + num = (num % NB_SEGMENTS) + 1; tl->associated_segment_num = num; _init_shadow_stack(tl); set_gs_register(get_segment_base(num)); diff --git a/c7/stm/sync.c b/c7/stm/sync.c --- a/c7/stm/sync.c +++ b/c7/stm/sync.c @@ -30,7 +30,7 @@ pthread_mutex_t global_mutex; pthread_cond_t cond[_C_TOTAL]; /* some additional pieces of global state follow */ - uint8_t in_use[NB_SEGMENTS]; /* 1 if running a pthread */ + uint8_t in_use1[NB_SEGMENTS]; /* 1 if running a pthread */ uint64_t global_time; }; char reserved[192]; @@ -60,7 +60,7 @@ stm_fatalerror("cond destroy: %m\n"); } - memset(&sync_ctl, 0, sizeof(sync_ctl.in_use)); + memset(&sync_ctl, 0, sizeof(sync_ctl)); } #ifndef NDEBUG @@ -124,12 +124,12 @@ { long i; restart: - for (i = 0; i < NB_SEGMENTS; i++) { + for (i = 1; i <= NB_SEGMENTS; i++) { if (get_priv_segment(i)->transaction_state == TS_INEVITABLE) { if (can_abort) { /* handle this case like a contention: it will either abort us (not the other thread, which is inevitable), - or for a while. If we go past this call, then we + or wait for a while. If we go past this call, then we waited; in this case we have to re-check if no other thread is inevitable. */ inevitable_contention_management(i); @@ -152,7 +152,7 @@ assert(_is_tl_registered(tl)); int num = tl->associated_segment_num; - if (sync_ctl.in_use[num] == 0) { + if (sync_ctl.in_use1[num - 1] == 0) { /* fast-path: we can get the same segment number than the one we had before. The value stored in GS is still valid. */ #ifdef STM_TESTS @@ -165,10 +165,10 @@ } /* Look for the next free segment. If there is none, wait for the condition variable. */ - int i; - for (i = 0; i < NB_SEGMENTS; i++) { - num = (num + 1) % NB_SEGMENTS; - if (sync_ctl.in_use[num] == 0) { + int retries; + for (retries = 0; retries < NB_SEGMENTS; retries++) { + num = (num % NB_SEGMENTS) + 1; + if (sync_ctl.in_use1[num - 1] == 0) { /* we're getting 'num', a different number. */ dprintf(("acquired different segment: %d->%d\n", tl->associated_segment_num, num)); tl->associated_segment_num = num; @@ -184,7 +184,7 @@ return false; got_num: - sync_ctl.in_use[num] = 1; + sync_ctl.in_use1[num - 1] = 1; assert(STM_SEGMENT->segment_num == num); assert(STM_SEGMENT->running_thread == NULL); STM_SEGMENT->running_thread = tl; @@ -208,8 +208,8 @@ assert(STM_SEGMENT->running_thread == tl); STM_SEGMENT->running_thread = NULL; - assert(sync_ctl.in_use[tl->associated_segment_num] == 1); - sync_ctl.in_use[tl->associated_segment_num] = 0; + assert(sync_ctl.in_use1[tl->associated_segment_num - 1] == 1); + sync_ctl.in_use1[tl->associated_segment_num - 1] = 0; } __attribute__((unused)) @@ -221,7 +221,7 @@ bool _stm_in_transaction(stm_thread_local_t *tl) { int num = tl->associated_segment_num; - assert(num < NB_SEGMENTS); + assert(1 <= num && num <= NB_SEGMENTS); return get_segment(num)->running_thread == tl; } @@ -262,7 +262,7 @@ assert((_safe_points_requested = true, 1)); long i; - for (i = 0; i < NB_SEGMENTS; i++) { + for (i = 1; i <= NB_SEGMENTS; i++) { if (get_segment(i)->nursery_end == NURSERY_END) get_segment(i)->nursery_end = NSE_SIGPAUSE; } @@ -276,7 +276,7 @@ long result = 0; int my_num = STM_SEGMENT->segment_num; - for (i = 0; i < NB_SEGMENTS; i++) { + for (i = 1; i <= NB_SEGMENTS; i++) { if (i != my_num && get_priv_segment(i)->safe_point == SP_RUNNING) { assert(get_segment(i)->nursery_end <= _STM_NSE_SIGNAL_MAX); result++; @@ -291,7 +291,7 @@ assert((_safe_points_requested = false, 1)); long i; - for (i = 0; i < NB_SEGMENTS; i++) { + for (i = 1; i <= NB_SEGMENTS; i++) { assert(get_segment(i)->nursery_end != NURSERY_END); if (get_segment(i)->nursery_end == NSE_SIGPAUSE) get_segment(i)->nursery_end = NURSERY_END; diff --git a/c7/stm/weakref.c b/c7/stm/weakref.c --- a/c7/stm/weakref.c +++ b/c7/stm/weakref.c @@ -29,6 +29,8 @@ static void _set_weakref_in_all_segments(object_t *weakref, object_t *value) { + abort();//XXX +#if 0 ssize_t size = 16; stm_char *point_to_loc = (stm_char*)WEAKREF_PTR(weakref, size); @@ -44,6 +46,7 @@ else { *WEAKREF_PTR(weakref, size) = value; } +#endif } /***** Minor collection *****/ From noreply at buildbot.pypy.org Sun Mar 16 09:46:32 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 16 Mar 2014 09:46:32 +0100 (CET) Subject: [pypy-commit] stmgc c7-more-segments: in-progress Message-ID: <20140316084632.903921C00B9@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c7-more-segments Changeset: r1028:c06abb7b21ca Date: 2014-03-16 09:46 +0100 http://bitbucket.org/pypy/stmgc/changeset/c06abb7b21ca/ Log: in-progress diff --git a/c7/stm/core.c b/c7/stm/core.c --- a/c7/stm/core.c +++ b/c7/stm/core.c @@ -238,26 +238,30 @@ return false; } -static void synchronize_overflow_object_now(object_t *obj) +static void synchronize_object_now(object_t *obj, bool assume_local_private) { - abort();//XXX -#if 0 + /* Copy around the version of 'obj' that lives in our own segment. + It is first copied into the shared pages, and then into other + segments' own private pages. + */ assert(!_is_young(obj)); - assert((obj->stm_flags & GCFLAG_SMALL_UNIFORM) == 0); assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); - char *realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj); - ssize_t obj_size = stmcb_size_rounded_up((struct object_s *)realobj); - assert(obj_size >= 16); uintptr_t start = (uintptr_t)obj; - uintptr_t end = start + obj_size; uintptr_t first_page = start / 4096UL; - uintptr_t last_page = (end - 1) / 4096UL; - do { - if (flag_page_private[first_page] != SHARED_PAGE) { - /* The page is a PRIVATE_PAGE. We need to diffuse this fragment - of our object from our own segment to all other segments. */ + if (obj->stm_flags & GCFLAG_SMALL_UNIFORM) { + abort();//XXX WRITE THE FAST CASE + } + else { + char *realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj); + ssize_t obj_size = stmcb_size_rounded_up((struct object_s *)realobj); + assert(obj_size >= 16); + uintptr_t end = start + obj_size; + uintptr_t last_page = (end - 1) / 4096UL; + long i, myself = STM_SEGMENT->segment_num; + + for (; first_page <= last_page; first_page++) { uintptr_t copy_size; if (first_page == last_page) { @@ -265,86 +269,44 @@ copy_size = end - start; } else { - /* this is a non-final fragment, going up to the page's end */ + /* this is a non-final fragment, going up to the + page's end */ copy_size = 4096 - (start & 4095); } - /* double-check that the result fits in one page */ assert(copy_size > 0); assert(copy_size + (start & 4095) <= 4096); - long i; - char *src = REAL_ADDRESS(STM_SEGMENT->segment_base, start); - for (i = 0; i < NB_SEGMENTS; i++) { - abort();//XXX - if (i != STM_SEGMENT->segment_num) { - char *dst = REAL_ADDRESS(get_segment_base(i), start); + /* First copy the object into the shared page, if needed */ + assert(IMPLY(assume_local_private, + is_private_page(myself, first_page))); + + if (assume_local_private || is_private_page(myself, first_page)) { + char *src = REAL_ADDRESS(STM_SEGMENT->segment_base, start); + char *dst = REAL_ADDRESS(stm_object_pages, start); + if (copy_size == 4096) + pagecopy(dst, src); + else memcpy(dst, src, copy_size); - } } - } - - start = (start + 4096) & ~4095; - } while (first_page++ < last_page); -#endif -} - -static void synchronize_object_now(object_t *obj) -{ - /* Assume that the version of 'obj' in the shared pages is up-to-date. - Assume also that the version in our own private page is up-to-date. - This function updates the private page of other threads. - */ - assert(!_is_young(obj)); - assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); - - uintptr_t start = (uintptr_t)obj; - uintptr_t first_page = start / 4096UL; - long i; - - if (obj->stm_flags & GCFLAG_SMALL_UNIFORM) { - abort();//XXX WRITE THE FAST CASE - } - else { - char *realobj = REAL_ADDRESS(stm_object_pages, obj); - ssize_t obj_size = stmcb_size_rounded_up((struct object_s *)realobj); - assert(obj_size >= 16); - uintptr_t end = start + obj_size; - uintptr_t last_page = (end - 1) / 4096UL; - - for (; first_page <= last_page; first_page++) { for (i = 1; i < NB_SEGMENTS; i++) { - - if (i == STM_SEGMENT->segment_num) + if (i == myself) continue; - if (!is_private_page(i, first_page)) continue; - /* The page is a PRIVATE_PAGE. We need to diffuse this + /* The page is a private page. We need to diffuse this fragment of object from the shared page to this private page. */ - - uintptr_t copy_size; - if (first_page == last_page) { - /* this is the final fragment */ - copy_size = end - start; - } - else { - /* this is a non-final fragment, going up to the - page's end */ - copy_size = 4096 - (start & 4095); - } - - /* double-check that the result fits in one page */ - assert(copy_size > 0); - assert(copy_size + (start & 4095) <= 4096); - char *src = REAL_ADDRESS(stm_object_pages, start); char *dst = REAL_ADDRESS(get_segment_base(i), start); - memcpy(dst, src, copy_size); + if (copy_size == 4096) + pagecopy(dst, src); + else + memcpy(dst, src, copy_size); } + start = (start + 4096) & ~4095; } } @@ -356,13 +318,11 @@ return; LIST_FOREACH_R(STM_PSEGMENT->large_overflow_objects, object_t *, - synchronize_overflow_object_now(item)); + synchronize_object_now(item, false)); } static void push_modified_to_other_segments(void) { - char *local_base = STM_SEGMENT->segment_base; - LIST_FOREACH_R( STM_PSEGMENT->modified_old_objects, object_t * /*item*/, @@ -378,14 +338,8 @@ minor_collection() */ assert((item->stm_flags & GCFLAG_WRITE_BARRIER) != 0); - /* copy the modified object to the shared copy */ - char *src = REAL_ADDRESS(local_base, item); - char *dst = REAL_ADDRESS(stm_object_pages, item); - ssize_t size = stmcb_size_rounded_up((struct object_s *)src); - memcpy(dst, src, size); - /* copy the object to the other private pages as needed */ - synchronize_object_now(item); + synchronize_object_now(item, true); })); list_clear(STM_PSEGMENT->modified_old_objects); diff --git a/c7/stm/core.h b/c7/stm/core.h --- a/c7/stm/core.h +++ b/c7/stm/core.h @@ -231,4 +231,4 @@ asm("/* workaround for llvm bug */"); } -static void synchronize_overflow_object_now(object_t *obj); +static void synchronize_object_now(object_t *obj, bool assume_local_private); diff --git a/c7/stm/nursery.c b/c7/stm/nursery.c --- a/c7/stm/nursery.c +++ b/c7/stm/nursery.c @@ -199,7 +199,7 @@ content); or add the object to 'large_overflow_objects'. */ if (STM_PSEGMENT->minor_collect_will_commit_now) - synchronize_overflow_object_now(obj); + synchronize_object_now(obj, false); else LIST_APPEND(STM_PSEGMENT->large_overflow_objects, obj); } From noreply at buildbot.pypy.org Sun Mar 16 10:00:55 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 16 Mar 2014 10:00:55 +0100 (CET) Subject: [pypy-commit] stmgc c7-more-segments: fix Message-ID: <20140316090055.7FD431C0670@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c7-more-segments Changeset: r1029:8469a9bdaccd Date: 2014-03-16 09:49 +0100 http://bitbucket.org/pypy/stmgc/changeset/8469a9bdaccd/ Log: fix diff --git a/c7/stm/contention.c b/c7/stm/contention.c --- a/c7/stm/contention.c +++ b/c7/stm/contention.c @@ -235,7 +235,7 @@ uint8_t prev_owner = ((volatile uint8_t *)write_locks)[lock_idx]; if (prev_owner != 0 && prev_owner != STM_PSEGMENT->write_lock_num) { - uint8_t other_segment_num = prev_owner - 1; + uint8_t other_segment_num = prev_owner; assert(get_priv_segment(other_segment_num)->write_lock_num == prev_owner); From noreply at buildbot.pypy.org Sun Mar 16 10:00:56 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 16 Mar 2014 10:00:56 +0100 (CET) Subject: [pypy-commit] stmgc c7-more-segments: Fix many tests Message-ID: <20140316090056.B1A931C0670@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c7-more-segments Changeset: r1030:1d43c3e1a2ff Date: 2014-03-16 10:00 +0100 http://bitbucket.org/pypy/stmgc/changeset/1d43c3e1a2ff/ Log: Fix many tests diff --git a/c7/stm/gcpage.c b/c7/stm/gcpage.c --- a/c7/stm/gcpage.c +++ b/c7/stm/gcpage.c @@ -533,6 +533,8 @@ dprintf((" .----- major collection -----------------------\n")); assert(_has_mutex()); + if (0) { // XXX TEMP + /* first, force a minor collection in each of the other segments */ major_do_minor_collections(); @@ -566,6 +568,7 @@ dprintf((" | used after collection: %ld\n", (long)pages_ctl.total_allocated)); + } dprintf((" `----------------------------------------------\n")); reset_major_collection_requested(); diff --git a/c7/stm/misc.c b/c7/stm/misc.c --- a/c7/stm/misc.c +++ b/c7/stm/misc.c @@ -41,9 +41,11 @@ } #ifdef STM_TESTS -uint8_t _stm_get_page_flag(uintptr_t index) +uintptr_t _stm_get_private_page(uintptr_t pagenum) { - abort();//XXX + wlog_t *item; + TREE_FIND(*STM_PSEGMENT->private_page_mapping, pagenum, item, return 0); + return item->val; } long _stm_count_modified_old_objects(void) diff --git a/c7/stm/weakref.c b/c7/stm/weakref.c --- a/c7/stm/weakref.c +++ b/c7/stm/weakref.c @@ -118,7 +118,7 @@ static void stm_visit_old_weakrefs(void) { long i; - for (i = 0; i < NB_SEGMENTS; i++) { + for (i = 1; i <= NB_SEGMENTS; i++) { struct stm_priv_segment_info_s *pseg = get_priv_segment(i); struct list_s *lst; diff --git a/c7/stmgc.h b/c7/stmgc.h --- a/c7/stmgc.h +++ b/c7/stmgc.h @@ -87,7 +87,7 @@ #include bool _stm_was_read(object_t *obj); bool _stm_was_written(object_t *obj); -uint8_t _stm_get_page_flag(uintptr_t index); +uintptr_t _stm_get_private_page(uintptr_t pagenum); bool _stm_in_transaction(stm_thread_local_t *tl); char *_stm_get_segment_base(long index); void _stm_test_switch(stm_thread_local_t *tl); diff --git a/c7/test/support.py b/c7/test/support.py --- a/c7/test/support.py +++ b/c7/test/support.py @@ -46,7 +46,7 @@ char *_stm_get_segment_base(long index); bool _stm_in_transaction(stm_thread_local_t *tl); void _stm_test_switch(stm_thread_local_t *tl); -uint8_t _stm_get_page_flag(uintptr_t index); +uintptr_t _stm_get_private_page(uintptr_t pagenum); int _stm_get_flags(object_t *obj); void _stm_start_transaction(stm_thread_local_t *tl, stm_jmpbuf_t *jmpbuf); @@ -98,8 +98,6 @@ GC_N_SMALL_REQUESTS = 36 # from gcpage.c -SHARED_PAGE = 1 # from pages.h -PRIVATE_PAGE = 3 # from pages.h LARGE_MALLOC_OVERHEAD = 16 # from largemalloc.h lib = ffi.verify(''' @@ -361,8 +359,8 @@ def stm_major_collect(): lib.stm_collect(1) -def stm_get_page_flag(pagenum): - return lib._stm_get_page_flag(pagenum) +def stm_get_private_page(pagenum): + return lib._stm_get_private_page(pagenum) def stm_get_obj_size(o): return lib.stmcb_size_rounded_up(stm_get_real_address(o)) diff --git a/c7/test/test_gcpage.py b/c7/test/test_gcpage.py --- a/c7/test/test_gcpage.py +++ b/c7/test/test_gcpage.py @@ -24,8 +24,8 @@ new = self.pop_root() assert len(stm_get_obj_pages(new)) == 2 - assert ([stm_get_page_flag(p) for p in stm_get_obj_pages(new)] - == [SHARED_PAGE]*2) + assert ([stm_get_private_page(p) for p in stm_get_obj_pages(new)] + == [0, 0]) assert not is_in_nursery(new) stm_write(new) @@ -33,11 +33,11 @@ # now proceed to write into the object in a new transaction self.start_transaction() - assert ([stm_get_page_flag(p) for p in stm_get_obj_pages(new)] - == [SHARED_PAGE]*2) + assert ([stm_get_private_page(p) for p in stm_get_obj_pages(new)] + == [0, 0]) stm_write(new) - assert ([stm_get_page_flag(p) for p in stm_get_obj_pages(new)] - == [PRIVATE_PAGE]*2) + assert ([bool(stm_get_private_page(p)) for p in stm_get_obj_pages(new)] + == [True, True]) # write to 2nd page of object!! wnew = stm_get_real_address(new) @@ -52,8 +52,8 @@ self.switch(0) self.abort_transaction() - assert ([stm_get_page_flag(p) for p in stm_get_obj_pages(new)] - == [PRIVATE_PAGE]*2) + assert ([bool(stm_get_private_page(p)) for p in stm_get_obj_pages(new)] + == [True, True]) def test_partial_alloced_pages(self): self.start_transaction() @@ -62,14 +62,14 @@ stm_minor_collect() new = self.pop_root() - assert stm_get_page_flag(stm_get_obj_pages(new)[0]) == SHARED_PAGE + assert stm_get_private_page(stm_get_obj_pages(new)[0]) == 0 assert stm_get_flags(new) & GCFLAG_WRITE_BARRIER stm_write(new) assert not (stm_get_flags(new) & GCFLAG_WRITE_BARRIER) self.commit_transaction() - assert stm_get_page_flag(stm_get_obj_pages(new)[0]) == SHARED_PAGE + assert stm_get_private_page(stm_get_obj_pages(new)[0]) == 0 assert stm_get_flags(new) & GCFLAG_WRITE_BARRIER self.start_transaction() @@ -78,7 +78,7 @@ stm_minor_collect() newer = self.pop_root() # 'new' is still in shared_page and committed - assert stm_get_page_flag(stm_get_obj_pages(new)[0]) == SHARED_PAGE + assert stm_get_private_page(stm_get_obj_pages(new)[0]) == 0 assert stm_get_flags(new) & GCFLAG_WRITE_BARRIER # 'newer' is now part of the SHARED page with 'new', but # uncommitted, so no privatization has to take place: @@ -86,10 +86,10 @@ assert stm_get_flags(newer) & GCFLAG_WRITE_BARRIER stm_write(newer) # does not privatize assert not (stm_get_flags(newer) & GCFLAG_WRITE_BARRIER) - assert stm_get_page_flag(stm_get_obj_pages(newer)[0]) == SHARED_PAGE + assert stm_get_private_page(stm_get_obj_pages(newer)[0]) == 0 self.commit_transaction() - assert stm_get_page_flag(stm_get_obj_pages(newer)[0]) == SHARED_PAGE + assert stm_get_private_page(stm_get_obj_pages(newer)[0]) == 0 assert stm_get_flags(newer) & GCFLAG_WRITE_BARRIER def test_major_collection(self): From noreply at buildbot.pypy.org Sun Mar 16 10:38:48 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 16 Mar 2014 10:38:48 +0100 (CET) Subject: [pypy-commit] stmgc c7-more-segments: Don't blink. This fixes a bug ('<' instead of '<=') in the middle Message-ID: <20140316093848.AF6F21C0670@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c7-more-segments Changeset: r1031:aef2a1cd2a7c Date: 2014-03-16 10:38 +0100 http://bitbucket.org/pypy/stmgc/changeset/aef2a1cd2a7c/ Log: Don't blink. This fixes a bug ('<' instead of '<=') in the middle of a ton of extra asserts needed to find it. diff --git a/c7/stm/core.c b/c7/stm/core.c --- a/c7/stm/core.c +++ b/c7/stm/core.c @@ -13,6 +13,7 @@ { assert(_seems_to_be_running_transaction()); assert(!_is_young(obj)); + assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); /* is this an object from the same transaction, outside the nursery? */ if ((obj->stm_flags & -GCFLAG_OVERFLOW_NUMBER_bit0) == @@ -104,15 +105,24 @@ LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, obj); } + /* check that we really have a private page */ + assert(_stm_get_private_page(((uintptr_t)obj) / 4096)); + + /* check that so far all copies of the object have the flag */ + long i; + for (i = 0; i <= NB_SEGMENTS; i++) { + assert(((struct object_s *)REAL_ADDRESS(get_segment_base(i), obj)) + ->stm_flags & GCFLAG_WRITE_BARRIER); + } + /* add the write-barrier-already-called flag ONLY if we succeeded in getting the write-lock */ assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); obj->stm_flags &= ~GCFLAG_WRITE_BARRIER; /* for sanity, check that all other segment copies of this object - still have the flag */ - long i; - for (i = 1; i <= NB_SEGMENTS; i++) { + still have the flag (including the shared copy) */ + for (i = 0; i <= NB_SEGMENTS; i++) { if (i != STM_SEGMENT->segment_num) assert(((struct object_s *)REAL_ADDRESS(get_segment_base(i), obj)) ->stm_flags & GCFLAG_WRITE_BARRIER); @@ -281,30 +291,36 @@ assert(IMPLY(assume_local_private, is_private_page(myself, first_page))); + char *src = REAL_ADDRESS(STM_SEGMENT->segment_base, start); + char *dst = REAL_ADDRESS(stm_object_pages, start); if (assume_local_private || is_private_page(myself, first_page)) { - char *src = REAL_ADDRESS(STM_SEGMENT->segment_base, start); - char *dst = REAL_ADDRESS(stm_object_pages, start); if (copy_size == 4096) pagecopy(dst, src); else memcpy(dst, src, copy_size); } + else { + assert(memcmp(dst, src, copy_size) == 0); /* same page */ + } - for (i = 1; i < NB_SEGMENTS; i++) { + for (i = 1; i <= NB_SEGMENTS; i++) { if (i == myself) continue; - if (!is_private_page(i, first_page)) - continue; - /* The page is a private page. We need to diffuse this - fragment of object from the shared page to this private - page. */ - char *src = REAL_ADDRESS(stm_object_pages, start); - char *dst = REAL_ADDRESS(get_segment_base(i), start); - if (copy_size == 4096) - pagecopy(dst, src); - else - memcpy(dst, src, copy_size); + src = REAL_ADDRESS(stm_object_pages, start); + dst = REAL_ADDRESS(get_segment_base(i), start); + if (is_private_page(i, first_page)) { + /* The page is a private page. We need to diffuse this + fragment of object from the shared page to this private + page. */ + if (copy_size == 4096) + pagecopy(dst, src); + else + memcpy(dst, src, copy_size); + } + else { + assert(memcmp(dst, src, copy_size) == 0); /* same page */ + } } start = (start + 4096) & ~4095; @@ -338,7 +354,8 @@ minor_collection() */ assert((item->stm_flags & GCFLAG_WRITE_BARRIER) != 0); - /* copy the object to the other private pages as needed */ + /* copy the object to the shared page, and to the other + private pages as needed */ synchronize_object_now(item, true); })); diff --git a/c7/stm/fprintcolor.h b/c7/stm/fprintcolor.h --- a/c7/stm/fprintcolor.h +++ b/c7/stm/fprintcolor.h @@ -9,7 +9,7 @@ #define dprintf(args) threadcolor_printf args static inline int dprintfcolor(void) { - return 31 + STM_SEGMENT->segment_num % 6; + return 31 + (STM_SEGMENT->segment_num + 5) % 6; } static int threadcolor_printf(const char *format, ...) From noreply at buildbot.pypy.org Sun Mar 16 10:40:27 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 16 Mar 2014 10:40:27 +0100 (CET) Subject: [pypy-commit] stmgc c7-more-segments: Fix for non-STM_TESTS cases Message-ID: <20140316094027.394761C00B9@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c7-more-segments Changeset: r1032:4dc5acd4c8e0 Date: 2014-03-16 10:40 +0100 http://bitbucket.org/pypy/stmgc/changeset/4dc5acd4c8e0/ Log: Fix for non-STM_TESTS cases diff --git a/c7/stm/core.c b/c7/stm/core.c --- a/c7/stm/core.c +++ b/c7/stm/core.c @@ -106,7 +106,8 @@ } /* check that we really have a private page */ - assert(_stm_get_private_page(((uintptr_t)obj) / 4096)); + assert(tree_contains(STM_PSEGMENT->private_page_mapping, + ((uintptr_t)obj) / 4096)); /* check that so far all copies of the object have the flag */ long i; From noreply at buildbot.pypy.org Sun Mar 16 10:57:06 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 16 Mar 2014 10:57:06 +0100 (CET) Subject: [pypy-commit] stmgc c7-more-segments: Fix major GC. Message-ID: <20140316095706.E11451C0670@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c7-more-segments Changeset: r1033:ece9046d8595 Date: 2014-03-16 10:56 +0100 http://bitbucket.org/pypy/stmgc/changeset/ece9046d8595/ Log: Fix major GC. diff --git a/c7/stm/gcpage.c b/c7/stm/gcpage.c --- a/c7/stm/gcpage.c +++ b/c7/stm/gcpage.c @@ -325,11 +325,6 @@ /* takes a normal pointer to a thread-local pointer to an object */ object_t *obj = *pobj; - if (obj == NULL || mark_visited_test_and_set(obj)) - return; /* already visited this object */ - - LIST_APPEND(mark_objects_to_trace, obj); - /* Note: this obj might be visited already, but from a different segment. We ignore this case and skip re-visiting the object anyway. The idea is that such an object is old (not from the @@ -340,6 +335,10 @@ segments and only needs visiting once. (It may actually be in a shared page, or maybe not.) */ + if (obj == NULL || mark_visited_test_and_set(obj)) + return; /* already visited this object */ + + LIST_APPEND(mark_objects_to_trace, obj); } static void mark_trace(object_t *obj, char *segment_base) @@ -347,15 +346,6 @@ assert(list_is_empty(mark_objects_to_trace)); while (1) { - -#if 0 - /* first, if we're not seeing segment 0, we must change the - flags in flag_page_private[] from PRIVATE_PAGE to - SEGMENT1_PAGE, which will mean "can't re-share" */ - if (segment_base != stm_object_pages && RESHARE_PAGES) - mark_flag_page_private(obj, segment_base); -#endif - /* trace into the object (the version from 'segment_base') */ struct object_s *realobj = (struct object_s *)REAL_ADDRESS(segment_base, obj); @@ -377,45 +367,33 @@ static void mark_visit_from_roots(void) { - if (testing_prebuilt_objs != NULL) { LIST_FOREACH_R(testing_prebuilt_objs, object_t * /*item*/, - mark_visit_object(item, get_segment_base(0))); + mark_visit_object(item, stm_object_pages)); } - /* Do the following twice, so that we trace first the objects from - segment 0, and then all others. XXX This is a hack to make it - more likely that we'll be able to re-share pages. */ + stm_thread_local_t *tl = stm_all_thread_locals; + do { + /* If 'tl' is currently running, its 'associated_segment_num' + field is the segment number that contains the correct + version of its overflowed objects. If not, then the + field is still some correct segment number, and it doesn't + matter which one we pick. */ + char *segment_base = get_segment_base(tl->associated_segment_num); - int must_be_zero; - for (must_be_zero = 1; must_be_zero >= 0; must_be_zero--) { + struct stm_shadowentry_s *current = tl->shadowstack; + struct stm_shadowentry_s *base = tl->shadowstack_base; + while (current-- != base) { + assert(current->ss != (object_t *)-1); + mark_visit_object(current->ss, segment_base); + } + mark_visit_object(tl->thread_local_obj, segment_base); - stm_thread_local_t *tl = stm_all_thread_locals; - do { - /* If 'tl' is currently running, its 'associated_segment_num' - field is the segment number that contains the correct - version of its overflowed objects. If not, then the - field is still some correct segment number, and it doesn't - matter which one we pick. */ - char *segment_base = get_segment_base(tl->associated_segment_num); - - if (must_be_zero == (segment_base == get_segment_base(0))) { - - struct stm_shadowentry_s *current = tl->shadowstack; - struct stm_shadowentry_s *base = tl->shadowstack_base; - while (current-- != base) { - assert(current->ss != (object_t *)-1); - mark_visit_object(current->ss, segment_base); - } - mark_visit_object(tl->thread_local_obj, segment_base); - } - - tl = tl->next; - } while (tl != stm_all_thread_locals); - } + tl = tl->next; + } while (tl != stm_all_thread_locals); long i; - for (i = 0; i < NB_SEGMENTS; i++) { + for (i = 1; i <= NB_SEGMENTS; i++) { if (get_priv_segment(i)->transaction_state != TS_NONE) mark_visit_object( get_priv_segment(i)->threadlocal_at_start_of_transaction, @@ -426,20 +404,21 @@ static void mark_visit_from_modified_objects(void) { /* The modified objects are the ones that may exist in two different - versions: one in the segment that modified it, and another in - all other segments. */ + versions: one in the segment that modified it, and another in all + other segments. (It can also be more than two if we don't have + eager write locking.) + */ long i; - for (i = 0; i < NB_SEGMENTS; i++) { - char *base1 = get_segment_base(i); /* two different segments */ - char *base2 = get_segment_base(!i); + for (i = 1; i <= NB_SEGMENTS; i++) { + char *base = get_segment_base(i); LIST_FOREACH_R( get_priv_segment(i)->modified_old_objects, object_t * /*item*/, ({ mark_visited_test_and_set(item); - mark_trace(item, base1); - mark_trace(item, base2); + mark_trace(item, stm_object_pages); /* shared version */ + mark_trace(item, base); /* private version */ })); } } @@ -447,7 +426,7 @@ static void clean_up_segment_lists(void) { long i; - for (i = 0; i < NB_SEGMENTS; i++) { + for (i = 1; i <= NB_SEGMENTS; i++) { struct stm_priv_segment_info_s *pseg = get_priv_segment(i); struct list_s *lst; @@ -513,7 +492,7 @@ { /* restore the write locks on the modified objects */ long i; - for (i = 0; i < NB_SEGMENTS; i++) { + for (i = 1; i <= NB_SEGMENTS; i++) { struct stm_priv_segment_info_s *pseg = get_priv_segment(i); LIST_FOREACH_R( @@ -533,8 +512,6 @@ dprintf((" .----- major collection -----------------------\n")); assert(_has_mutex()); - if (0) { // XXX TEMP - /* first, force a minor collection in each of the other segments */ major_do_minor_collections(); @@ -548,7 +525,8 @@ LIST_FREE(mark_objects_to_trace); /* weakrefs: */ - stm_visit_old_weakrefs(); + if (0)//XXX + stm_visit_old_weakrefs(); /* cleanup */ clean_up_segment_lists(); @@ -568,7 +546,6 @@ dprintf((" | used after collection: %ld\n", (long)pages_ctl.total_allocated)); - } dprintf((" `----------------------------------------------\n")); reset_major_collection_requested(); From noreply at buildbot.pypy.org Sun Mar 16 11:02:45 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 16 Mar 2014 11:02:45 +0100 (CET) Subject: [pypy-commit] stmgc c7-more-segments: Fix (and simplify the code a bit) Message-ID: <20140316100245.9B0B31C357E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c7-more-segments Changeset: r1034:88810db654ce Date: 2014-03-16 11:00 +0100 http://bitbucket.org/pypy/stmgc/changeset/88810db654ce/ Log: Fix (and simplify the code a bit) diff --git a/c7/stm/weakref.c b/c7/stm/weakref.c --- a/c7/stm/weakref.c +++ b/c7/stm/weakref.c @@ -29,29 +29,21 @@ static void _set_weakref_in_all_segments(object_t *weakref, object_t *value) { - abort();//XXX -#if 0 ssize_t size = 16; stm_char *point_to_loc = (stm_char*)WEAKREF_PTR(weakref, size); - if (flag_page_private[(uintptr_t)point_to_loc / 4096UL] == PRIVATE_PAGE) { - long i; - for (i = 0; i < NB_SEGMENTS; i++) { - char *base = get_segment_base(i); /* two different segments */ - object_t ** ref_loc = (object_t **)REAL_ADDRESS(base, point_to_loc); - *ref_loc = value; - } + long i; + for (i = 1; i <= NB_SEGMENTS; i++) { + char *base = get_segment_base(i); + object_t ** ref_loc = (object_t **)REAL_ADDRESS(base, point_to_loc); + *ref_loc = value; } - else { - *WEAKREF_PTR(weakref, size) = value; - } -#endif } /***** Minor collection *****/ -static void stm_move_young_weakrefs() +static void stm_move_young_weakrefs(void) { /* The code relies on the fact that no weakref can be an old object weakly pointing to a young object. Indeed, weakrefs are immutable From noreply at buildbot.pypy.org Sun Mar 16 11:02:46 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 16 Mar 2014 11:02:46 +0100 (CET) Subject: [pypy-commit] stmgc c7-more-segments: hg merge default Message-ID: <20140316100246.A5D051C357E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c7-more-segments Changeset: r1035:6587c5783a11 Date: 2014-03-16 11:00 +0100 http://bitbucket.org/pypy/stmgc/changeset/6587c5783a11/ Log: hg merge default diff --git a/c7/test/support.py b/c7/test/support.py --- a/c7/test/support.py +++ b/c7/test/support.py @@ -411,7 +411,10 @@ if lib._stm_in_transaction(tl): if self.current_thread != n: self.switch(n) - self.abort_transaction() + if lib.stm_is_inevitable(): + self.commit_transaction() # must succeed! + else: + self.abort_transaction() for tl in self.tls: lib.stm_unregister_thread_local(tl) lib.stm_teardown() diff --git a/c7/test/test_weakref.py b/c7/test/test_weakref.py --- a/c7/test/test_weakref.py +++ b/c7/test/test_weakref.py @@ -146,6 +146,7 @@ class TestIsolation(BaseTest): def test_not_break(self): + py.test.xfail("known to fail") lpold = stm_allocate_old_refs(1) self.start_transaction() From noreply at buildbot.pypy.org Sun Mar 16 11:02:47 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 16 Mar 2014 11:02:47 +0100 (CET) Subject: [pypy-commit] stmgc c7-more-segments: Fixes Message-ID: <20140316100247.B28531C357E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c7-more-segments Changeset: r1036:93b0deba3f1f Date: 2014-03-16 11:02 +0100 http://bitbucket.org/pypy/stmgc/changeset/93b0deba3f1f/ Log: Fixes diff --git a/c7/stm/gcpage.c b/c7/stm/gcpage.c --- a/c7/stm/gcpage.c +++ b/c7/stm/gcpage.c @@ -525,8 +525,7 @@ LIST_FREE(mark_objects_to_trace); /* weakrefs: */ - if (0)//XXX - stm_visit_old_weakrefs(); + stm_visit_old_weakrefs(); /* cleanup */ clean_up_segment_lists(); diff --git a/c7/test/test_weakref.py b/c7/test/test_weakref.py --- a/c7/test/test_weakref.py +++ b/c7/test/test_weakref.py @@ -279,7 +279,7 @@ stm_write(lp0) # privatize page with weakref in it too - assert stm_get_page_flag(stm_get_obj_pages(lp1)[0]) == PRIVATE_PAGE + assert stm_get_private_page(stm_get_obj_pages(lp1)[0]) != 0 assert stm_get_weakref(lp1) == lp0 self.commit_transaction() From noreply at buildbot.pypy.org Sun Mar 16 11:55:42 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 16 Mar 2014 11:55:42 +0100 (CET) Subject: [pypy-commit] stmgc c7-more-segments: Trying to use three threads. Buggy still. Message-ID: <20140316105542.DCE301C00B9@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c7-more-segments Changeset: r1037:d9d666092944 Date: 2014-03-16 11:51 +0100 http://bitbucket.org/pypy/stmgc/changeset/d9d666092944/ Log: Trying to use three threads. Buggy still. diff --git a/c7/demo/demo2.c b/c7/demo/demo2.c --- a/c7/demo/demo2.c +++ b/c7/demo/demo2.c @@ -6,6 +6,7 @@ #include "stmgc.h" +#define NTHREADS 3 #define LIST_LENGTH 2000 #define BUNCH 100 @@ -223,7 +224,7 @@ int main(void) { - int status; + int status, i; status = sem_init(&done, 0, 0); assert(status == 0); @@ -233,11 +234,13 @@ setup_list(); - newthread(demo2, (void*)1); - newthread(demo2, (void*)2); + for (i = 1; i <= NTHREADS; i++) { + newthread(demo2, (void*)(uintptr_t)i); + } - status = sem_wait(&done); assert(status == 0); - status = sem_wait(&done); assert(status == 0); + for (i = 1; i <= NTHREADS; i++) { + status = sem_wait(&done); assert(status == 0); + } final_check(); diff --git a/c7/stm/core.c b/c7/stm/core.c --- a/c7/stm/core.c +++ b/c7/stm/core.c @@ -529,6 +529,7 @@ static void abort_with_mutex(void) { + assert(_has_mutex()); dprintf(("~~~ ABORT\n")); switch (STM_PSEGMENT->transaction_state) { diff --git a/c7/stm/core.h b/c7/stm/core.h --- a/c7/stm/core.h +++ b/c7/stm/core.h @@ -15,7 +15,7 @@ #define NB_PAGES (1500*256) // 1500MB -#define NB_SEGMENTS 2 +#define NB_SEGMENTS 3 #define NB_SEGMENTS_MAX 240 /* don't increase NB_SEGMENTS past this */ #define MAP_PAGES_FLAGS (MAP_SHARED | MAP_ANONYMOUS | MAP_NORESERVE) #define NB_NURSERY_PAGES (STM_GC_NURSERY/4) diff --git a/c7/stm/sync.c b/c7/stm/sync.c --- a/c7/stm/sync.c +++ b/c7/stm/sync.c @@ -260,6 +260,7 @@ { assert(_safe_points_requested == false); assert((_safe_points_requested = true, 1)); + assert(_has_mutex()); long i; for (i = 1; i <= NB_SEGMENTS; i++) { From noreply at buildbot.pypy.org Sun Mar 16 11:55:44 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 16 Mar 2014 11:55:44 +0100 (CET) Subject: [pypy-commit] stmgc c7-more-segments: The error was a too-strict assertion. Message-ID: <20140316105544.123221C00B9@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c7-more-segments Changeset: r1038:b65a48fadcbd Date: 2014-03-16 11:55 +0100 http://bitbucket.org/pypy/stmgc/changeset/b65a48fadcbd/ Log: The error was a too-strict assertion. diff --git a/c7/stm/sync.c b/c7/stm/sync.c --- a/c7/stm/sync.c +++ b/c7/stm/sync.c @@ -293,7 +293,12 @@ long i; for (i = 1; i <= NB_SEGMENTS; i++) { - assert(get_segment(i)->nursery_end != NURSERY_END); + /* note: the only possible way to concurrently change the value + of 'nursery_end' is with an abort done while we wait for + C_AT_SAFE_POINT. It's fine because the next transaction + should not start. */ + assert(get_segment(i)->nursery_end != NURSERY_END || + get_priv_segment(i)->transaction_state == TS_NONE); if (get_segment(i)->nursery_end == NSE_SIGPAUSE) get_segment(i)->nursery_end = NURSERY_END; } From noreply at buildbot.pypy.org Sun Mar 16 12:04:50 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 16 Mar 2014 12:04:50 +0100 (CET) Subject: [pypy-commit] stmgc c7-more-segments: Test the maximum number of segments, in test_random. Message-ID: <20140316110450.DBC3B1C011F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c7-more-segments Changeset: r1039:0454581ce4a8 Date: 2014-03-16 12:04 +0100 http://bitbucket.org/pypy/stmgc/changeset/0454581ce4a8/ Log: Test the maximum number of segments, in test_random. diff --git a/c7/stm/core.h b/c7/stm/core.h --- a/c7/stm/core.h +++ b/c7/stm/core.h @@ -15,7 +15,7 @@ #define NB_PAGES (1500*256) // 1500MB -#define NB_SEGMENTS 3 +#define NB_SEGMENTS STM_NB_SEGMENTS #define NB_SEGMENTS_MAX 240 /* don't increase NB_SEGMENTS past this */ #define MAP_PAGES_FLAGS (MAP_SHARED | MAP_ANONYMOUS | MAP_NORESERVE) #define NB_NURSERY_PAGES (STM_GC_NURSERY/4) diff --git a/c7/stmgc.h b/c7/stmgc.h --- a/c7/stmgc.h +++ b/c7/stmgc.h @@ -127,6 +127,12 @@ /* ==================== PUBLIC API ==================== */ +/* Number of segments (i.e. how many threads can be executed in + parallel, in maximum). +*/ +#define STM_NB_SEGMENTS 4 + + /* Structure of objects -------------------- diff --git a/c7/test/support.py b/c7/test/support.py --- a/c7/test/support.py +++ b/c7/test/support.py @@ -9,6 +9,7 @@ typedef ... object_t; typedef ... stm_jmpbuf_t; #define SIZEOF_MYOBJ ... +#define STM_NB_SEGMENTS ... #define _STM_FAST_ALLOC ... #define _STM_GCFLAG_WRITE_BARRIER ... @@ -260,6 +261,7 @@ HDR = lib.SIZEOF_MYOBJ assert HDR == 8 GCFLAG_WRITE_BARRIER = lib._STM_GCFLAG_WRITE_BARRIER +NB_SEGMENTS = lib.STM_NB_SEGMENTS class Conflict(Exception): @@ -400,10 +402,11 @@ class BaseTest(object): + NB_THREADS = 2 def setup_method(self, meth): lib.stm_setup() - self.tls = [_allocate_thread_local(), _allocate_thread_local()] + self.tls = [_allocate_thread_local() for i in range(self.NB_THREADS)] self.current_thread = 0 def teardown_method(self, meth): diff --git a/c7/test/test_random.py b/c7/test/test_random.py --- a/c7/test/test_random.py +++ b/c7/test/test_random.py @@ -533,12 +533,13 @@ class TestRandom(BaseTest): + NB_THREADS = NB_SEGMENTS def test_fixed_16_bytes_objects(self, seed=1010): rnd = random.Random(seed) N_OBJECTS = 3 - N_THREADS = 2 + N_THREADS = self.NB_THREADS ex = Exec(self) ex.do("################################################################\n"*10) ex.do('# initialization') From noreply at buildbot.pypy.org Sun Mar 16 12:25:17 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 16 Mar 2014 12:25:17 +0100 (CET) Subject: [pypy-commit] stmgc c7-more-segments: More debugging info Message-ID: <20140316112517.3EAB01C011F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c7-more-segments Changeset: r1040:b38942a05903 Date: 2014-03-16 12:24 +0100 http://bitbucket.org/pypy/stmgc/changeset/b38942a05903/ Log: More debugging info diff --git a/c7/test/test_random.py b/c7/test/test_random.py --- a/c7/test/test_random.py +++ b/c7/test/test_random.py @@ -54,7 +54,7 @@ e.g. maintains read/write sets. The state will be discarded on abort or pushed to other threads""" - def __init__(self, start_time): + def __init__(self, start_time, thread_num=None): self.read_set = set() self.write_set = set() self.values = {} @@ -63,6 +63,7 @@ self.objs_in_conflict = set() self.inevitable = False self.created_in_this_transaction = set() + self.thread_num = thread_num def get_old_modified(self): # returns only the ones that are modified and not from @@ -74,6 +75,8 @@ if objs_in_conflict is not None: self.objs_in_conflict |= objs_in_conflict self._must_abort = True + color = "\033[%dm" % (31 + self.thread_num % 6) + print >> sys.stderr, color + "# must abort: %r\033[0m" % (objs_in_conflict,) def check_must_abort(self): return self._must_abort @@ -180,10 +183,10 @@ r, int(ffi.cast("uintptr_t", ex.content[r])), stm_get_obj_size(ex.content[r]))) - def start_transaction(self): + def start_transaction(self, thread_num): assert self.transaction_state is None start_time = self.global_state.inc_and_get_global_time() - trs = TransactionState(start_time) + trs = TransactionState(start_time, thread_num) trs.update_from_committed( self.global_state.committed_transaction_state) self.transaction_state = trs @@ -305,7 +308,7 @@ def op_start_transaction(ex, global_state, thread_state): - thread_state.start_transaction() + thread_state.start_transaction(ex.thread_num) # ex.do('self.start_transaction()') thread_state.reload_roots(ex) From noreply at buildbot.pypy.org Sun Mar 16 12:25:18 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 16 Mar 2014 12:25:18 +0100 (CET) Subject: [pypy-commit] stmgc c7-more-segments: Fix Message-ID: <20140316112518.471181C011F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c7-more-segments Changeset: r1041:e37e0bb66492 Date: 2014-03-16 12:24 +0100 http://bitbucket.org/pypy/stmgc/changeset/e37e0bb66492/ Log: Fix diff --git a/c7/test/test_random.py b/c7/test/test_random.py --- a/c7/test/test_random.py +++ b/c7/test/test_random.py @@ -294,6 +294,8 @@ if confl_set: contention_management(trs, other_trs, objs_in_conflict=confl_set) + if trs.check_must_abort(): + break if trs.check_must_abort(): self.ex.do('# write-read conflict: %s' % From noreply at buildbot.pypy.org Sun Mar 16 12:32:41 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 16 Mar 2014 12:32:41 +0100 (CET) Subject: [pypy-commit] stmgc default: Fix the test from llvm's current point of view Message-ID: <20140316113241.455B41C0670@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1042:d7e384cd1ea9 Date: 2014-03-16 12:32 +0100 http://bitbucket.org/pypy/stmgc/changeset/d7e384cd1ea9/ Log: Fix the test from llvm's current point of view diff --git a/c7/demo/demo_random.c b/c7/demo/demo_random.c --- a/c7/demo/demo_random.c +++ b/c7/demo/demo_random.c @@ -385,7 +385,7 @@ for (i = 0; i < PREBUILT_ROOTS; i++) { void* new_templ = malloc(sizeof(struct node_s)); memcpy(new_templ, &prebuilt_template, sizeof(struct node_s)); - prebuilt_roots[i] = stm_setup_prebuilt((objptr_t)new_templ); + prebuilt_roots[i] = stm_setup_prebuilt((objptr_t)(long)new_templ); if (i % 2 == 0) { int hash = i + 5; From noreply at buildbot.pypy.org Sun Mar 16 12:34:13 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 16 Mar 2014 12:34:13 +0100 (CET) Subject: [pypy-commit] stmgc c7-more-segments: Use by default the value of STM_NB_SEGMENTS. Message-ID: <20140316113413.E84781C0670@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c7-more-segments Changeset: r1043:883134c1db24 Date: 2014-03-16 12:33 +0100 http://bitbucket.org/pypy/stmgc/changeset/883134c1db24/ Log: Use by default the value of STM_NB_SEGMENTS. diff --git a/duhton/duhton.c b/duhton/duhton.c --- a/duhton/duhton.c +++ b/duhton/duhton.c @@ -7,7 +7,7 @@ char *filename = NULL; int interactive = 1; int i; - int num_threads = DEFAULT_NUM_THREADS; + int num_threads = STM_NB_SEGMENTS; for (i = 1; i < argc; ++i) { if (strcmp(argv[i], "--help") == 0) { diff --git a/duhton/duhton.h b/duhton/duhton.h --- a/duhton/duhton.h +++ b/duhton/duhton.h @@ -13,8 +13,6 @@ #endif -#define DEFAULT_NUM_THREADS 2 - extern __thread stm_thread_local_t stm_thread_local; struct DuObject_s { From noreply at buildbot.pypy.org Sun Mar 16 12:34:15 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 16 Mar 2014 12:34:15 +0100 (CET) Subject: [pypy-commit] stmgc c7-more-segments: hg merge default Message-ID: <20140316113415.03F5D1C0670@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c7-more-segments Changeset: r1044:898bd6b4224f Date: 2014-03-16 12:33 +0100 http://bitbucket.org/pypy/stmgc/changeset/898bd6b4224f/ Log: hg merge default diff --git a/c7/demo/demo_random.c b/c7/demo/demo_random.c --- a/c7/demo/demo_random.c +++ b/c7/demo/demo_random.c @@ -385,7 +385,7 @@ for (i = 0; i < PREBUILT_ROOTS; i++) { void* new_templ = malloc(sizeof(struct node_s)); memcpy(new_templ, &prebuilt_template, sizeof(struct node_s)); - prebuilt_roots[i] = stm_setup_prebuilt((objptr_t)new_templ); + prebuilt_roots[i] = stm_setup_prebuilt((objptr_t)(long)new_templ); if (i % 2 == 0) { int hash = i + 5; From noreply at buildbot.pypy.org Sun Mar 16 12:46:04 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 16 Mar 2014 12:46:04 +0100 (CET) Subject: [pypy-commit] stmgc c7-more-segments: Comment Message-ID: <20140316114605.012B61C011F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c7-more-segments Changeset: r1045:a04c4c49dedd Date: 2014-03-16 12:45 +0100 http://bitbucket.org/pypy/stmgc/changeset/a04c4c49dedd/ Log: Comment diff --git a/c7/stm/pages.c b/c7/stm/pages.c --- a/c7/stm/pages.c +++ b/c7/stm/pages.c @@ -122,7 +122,9 @@ /* look up the next free page */ uintptr_t free_page_num = STM_PSEGMENT->private_free_page_num; - /* "mount" it in the segment */ + /* "mount" it in the segment + (XXX later we should again attempt to group together many calls to + d_remap_file_pages() in succession) */ char *new_page = STM_SEGMENT->segment_base + pagenum * 4096UL; d_remap_file_pages(new_page, 4096, NB_PAGES * STM_SEGMENT->segment_num + free_page_num); From noreply at buildbot.pypy.org Sun Mar 16 12:46:06 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 16 Mar 2014 12:46:06 +0100 (CET) Subject: [pypy-commit] stmgc c7-more-segments: Proper fix. Now demo_random passes at least once :-) Message-ID: <20140316114606.264B71C011F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c7-more-segments Changeset: r1046:d9c43a29891e Date: 2014-03-16 12:45 +0100 http://bitbucket.org/pypy/stmgc/changeset/d9c43a29891e/ Log: Proper fix. Now demo_random passes at least once :-) diff --git a/c7/stm/core.c b/c7/stm/core.c --- a/c7/stm/core.c +++ b/c7/stm/core.c @@ -555,8 +555,11 @@ /* invoke the callbacks */ invoke_and_clear_callbacks_on_abort(); - if (STM_SEGMENT->nursery_end == NSE_SIGABORT) - STM_SEGMENT->nursery_end = NURSERY_END; /* done aborting */ + if (STM_SEGMENT->nursery_end == NSE_SIGABORT) { + /* done aborting */ + STM_SEGMENT->nursery_end = pause_signalled ? NSE_SIGPAUSE + : NURSERY_END; + } _finish_transaction(); /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */ diff --git a/c7/stm/sync.c b/c7/stm/sync.c --- a/c7/stm/sync.c +++ b/c7/stm/sync.c @@ -267,6 +267,8 @@ if (get_segment(i)->nursery_end == NURSERY_END) get_segment(i)->nursery_end = NSE_SIGPAUSE; } + assert(!pause_signalled); + pause_signalled = true; } static inline long count_other_threads_sp_running(void) @@ -288,17 +290,14 @@ static void remove_requests_for_safe_point(void) { + assert(pause_signalled); + pause_signalled = false; assert(_safe_points_requested == true); assert((_safe_points_requested = false, 1)); long i; for (i = 1; i <= NB_SEGMENTS; i++) { - /* note: the only possible way to concurrently change the value - of 'nursery_end' is with an abort done while we wait for - C_AT_SAFE_POINT. It's fine because the next transaction - should not start. */ - assert(get_segment(i)->nursery_end != NURSERY_END || - get_priv_segment(i)->transaction_state == TS_NONE); + assert(get_segment(i)->nursery_end != NURSERY_END); if (get_segment(i)->nursery_end == NSE_SIGPAUSE) get_segment(i)->nursery_end = NURSERY_END; } diff --git a/c7/stm/sync.h b/c7/stm/sync.h --- a/c7/stm/sync.h +++ b/c7/stm/sync.h @@ -30,3 +30,5 @@ static void wait_for_end_of_inevitable_transaction(bool can_abort); static void synchronize_all_threads(void); + +static bool pause_signalled; From noreply at buildbot.pypy.org Sun Mar 16 13:32:15 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 16 Mar 2014 13:32:15 +0100 (CET) Subject: [pypy-commit] stmgc c7-more-segments: Messy check, needs to be verified... Message-ID: <20140316123215.989CE1D27C0@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c7-more-segments Changeset: r1047:3716ced45a0b Date: 2014-03-16 13:00 +0100 http://bitbucket.org/pypy/stmgc/changeset/3716ced45a0b/ Log: Messy check, needs to be verified... diff --git a/c7/stm/core.c b/c7/stm/core.c --- a/c7/stm/core.c +++ b/c7/stm/core.c @@ -109,14 +109,23 @@ assert(tree_contains(STM_PSEGMENT->private_page_mapping, ((uintptr_t)obj) / 4096)); - /* check that so far all copies of the object have the flag */ + /* check that so far all copies of the object have the flag + (a bit messy because it's possible that we read a page in + the middle of privatization by another thread) */ +#ifndef NDEBUG long i; + long busy_loop = 1000000000; for (i = 0; i <= NB_SEGMENTS; i++) { - assert(((struct object_s *)REAL_ADDRESS(get_segment_base(i), obj)) - ->stm_flags & GCFLAG_WRITE_BARRIER); + while (!(((struct object_s *)REAL_ADDRESS(get_segment_base(i), obj)) + ->stm_flags & GCFLAG_WRITE_BARRIER)) { + spin_loop(); + if (!--busy_loop) + stm_fatalerror("missing GCFLAG_WRITE_BARRIER"); + } } +#endif - /* add the write-barrier-already-called flag ONLY if we succeeded in + /* remove GCFLAG_WRITE_BARRIER, but only if we succeeded in getting the write-lock */ assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); obj->stm_flags &= ~GCFLAG_WRITE_BARRIER; From noreply at buildbot.pypy.org Sun Mar 16 13:32:16 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 16 Mar 2014 13:32:16 +0100 (CET) Subject: [pypy-commit] stmgc c7-more-segments: Tweak: we need after all to have the mutex_pages_lock around Message-ID: <20140316123216.D88701D27C0@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c7-more-segments Changeset: r1048:1edbabf86da8 Date: 2014-03-16 13:32 +0100 http://bitbucket.org/pypy/stmgc/changeset/1edbabf86da8/ Log: Tweak: we need after all to have the mutex_pages_lock around remappings and around synchronize_object_now(). diff --git a/c7/stm/core.c b/c7/stm/core.c --- a/c7/stm/core.c +++ b/c7/stm/core.c @@ -112,8 +112,8 @@ /* check that so far all copies of the object have the flag (a bit messy because it's possible that we read a page in the middle of privatization by another thread) */ + long i; #ifndef NDEBUG - long i; long busy_loop = 1000000000; for (i = 0; i <= NB_SEGMENTS; i++) { while (!(((struct object_s *)REAL_ADDRESS(get_segment_base(i), obj)) @@ -263,7 +263,10 @@ /* Copy around the version of 'obj' that lives in our own segment. It is first copied into the shared pages, and then into other segments' own private pages. + + This must be called with the mutex_pages_lock! */ + assert(_has_mutex_pages()); assert(!_is_young(obj)); assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); @@ -417,10 +420,12 @@ major_collection_now_at_safe_point(); /* synchronize overflow objects living in privatized pages */ + mutex_pages_lock(); push_overflow_objects_from_privatized_pages(); /* synchronize modified old objects to other threads */ push_modified_to_other_segments(); + mutex_pages_unlock(); /* update 'overflow_number' if needed */ if (STM_PSEGMENT->overflow_number_has_been_used) { diff --git a/c7/stm/misc.c b/c7/stm/misc.c --- a/c7/stm/misc.c +++ b/c7/stm/misc.c @@ -81,4 +81,14 @@ mutex_pages_unlock(); return result; } + +void _stm_mutex_pages_lock(void) +{ + mutex_pages_lock(); +} + +void _stm_mutex_pages_unlock(void) +{ + mutex_pages_unlock(); +} #endif diff --git a/c7/stm/nursery.c b/c7/stm/nursery.c --- a/c7/stm/nursery.c +++ b/c7/stm/nursery.c @@ -198,8 +198,11 @@ WRITE_BARRIER flag and traced into it to fix its content); or add the object to 'large_overflow_objects'. */ - if (STM_PSEGMENT->minor_collect_will_commit_now) + if (STM_PSEGMENT->minor_collect_will_commit_now) { + mutex_pages_lock(); synchronize_object_now(obj, false); + mutex_pages_unlock(); + } else LIST_APPEND(STM_PSEGMENT->large_overflow_objects, obj); } diff --git a/c7/stm/pages.c b/c7/stm/pages.c --- a/c7/stm/pages.c +++ b/c7/stm/pages.c @@ -39,7 +39,6 @@ __sync_lock_release(&pages_ctl.mutex_pages); } -__attribute__((unused)) static bool _has_mutex_pages(void) { return pages_ctl.mutex_pages != 0; @@ -47,6 +46,7 @@ static uint64_t increment_total_allocated(ssize_t add_or_remove) { + assert(_has_mutex_pages()); pages_ctl.total_allocated += add_or_remove; if (pages_ctl.total_allocated >= pages_ctl.total_allocated_bound) @@ -119,6 +119,10 @@ return; not_found:; + /* lock, to prevent concurrent threads from looking up my own + 'private_page_mapping' in parallel */ + mutex_pages_lock(); + /* look up the next free page */ uintptr_t free_page_num = STM_PSEGMENT->private_free_page_num; @@ -142,6 +146,8 @@ /* update private_page_mapping */ tree_insert(STM_PSEGMENT->private_page_mapping, pagenum, free_page_num); + + mutex_pages_unlock(); } #if 0 diff --git a/c7/stm/pages.h b/c7/stm/pages.h --- a/c7/stm/pages.h +++ b/c7/stm/pages.h @@ -27,6 +27,7 @@ static void mutex_pages_lock(void); static void mutex_pages_unlock(void); +static bool _has_mutex_pages(void) __attribute__((unused)); static uint64_t increment_total_allocated(ssize_t add_or_remove); static bool is_major_collection_requested(void); static void force_major_collection_request(void); diff --git a/c7/stmgc.h b/c7/stmgc.h --- a/c7/stmgc.h +++ b/c7/stmgc.h @@ -107,6 +107,8 @@ object_t *_stm_enum_modified_old_objects(long index); object_t *_stm_enum_objects_pointing_to_nursery(long index); uint64_t _stm_total_allocated(void); +void _stm_mutex_pages_lock(void); +void _stm_mutex_pages_unlock(void); #endif #define _STM_GCFLAG_WRITE_BARRIER 0x01 diff --git a/c7/test/support.py b/c7/test/support.py --- a/c7/test/support.py +++ b/c7/test/support.py @@ -88,6 +88,8 @@ void stm_collect(long level); uint64_t _stm_total_allocated(void); +void _stm_mutex_pages_lock(void); +void _stm_mutex_pages_unlock(void); long stm_identityhash(object_t *obj); long stm_id(object_t *obj); diff --git a/c7/test/test_largemalloc.py b/c7/test/test_largemalloc.py --- a/c7/test/test_largemalloc.py +++ b/c7/test/test_largemalloc.py @@ -14,6 +14,7 @@ lib.memset(self.rawmem, 0xcd, self.size) lib._stm_largemalloc_init_arena(self.rawmem, self.size) + lib._stm_mutex_pages_lock() # for this file def test_simple(self): d1 = lib._stm_large_malloc(7000) From noreply at buildbot.pypy.org Sun Mar 16 14:34:18 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 16 Mar 2014 14:34:18 +0100 (CET) Subject: [pypy-commit] stmgc c7-more-segments: Refactor after discussion with Remi: don't use a tree to record page Message-ID: <20140316133418.792171C0670@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c7-more-segments Changeset: r1049:9eed4e26972b Date: 2014-03-16 14:34 +0100 http://bitbucket.org/pypy/stmgc/changeset/9eed4e26972b/ Log: Refactor after discussion with Remi: don't use a tree to record page remappings, and only do trivial remappings (page N is mapped to either to segment 0's page N, or to itself). Use a simple array of bits. diff --git a/c7/stm/core.c b/c7/stm/core.c --- a/c7/stm/core.c +++ b/c7/stm/core.c @@ -8,6 +8,28 @@ memset(write_locks, 0, sizeof(write_locks)); } +static void check_flag_write_barrier(object_t *obj) +{ + /* check that all copies of the object, apart from mine, have the + GCFLAG_WRITE_BARRIER. (a bit messy because it's possible that we + read a page in the middle of privatization by another thread) + */ +#ifndef NDEBUG + long i; + struct object_s *o1; + for (i = 0; i <= NB_SEGMENTS; i++) { + if (i == STM_SEGMENT->segment_num) + continue; + o1 = (struct object_s *)REAL_ADDRESS(get_segment_base(i), obj); + if (!(o1->stm_flags & GCFLAG_WRITE_BARRIER)) { + mutex_pages_lock(); /* try again... */ + if (!(o1->stm_flags & GCFLAG_WRITE_BARRIER)) + stm_fatalerror("missing GCFLAG_WRITE_BARRIER"); + mutex_pages_unlock(); + } + } +#endif +} void _stm_write_slowpath(object_t *obj) { @@ -106,37 +128,20 @@ } /* check that we really have a private page */ - assert(tree_contains(STM_PSEGMENT->private_page_mapping, - ((uintptr_t)obj) / 4096)); + assert(is_private_page(STM_SEGMENT->segment_num, + ((uintptr_t)obj) / 4096)); - /* check that so far all copies of the object have the flag - (a bit messy because it's possible that we read a page in - the middle of privatization by another thread) */ - long i; -#ifndef NDEBUG - long busy_loop = 1000000000; - for (i = 0; i <= NB_SEGMENTS; i++) { - while (!(((struct object_s *)REAL_ADDRESS(get_segment_base(i), obj)) - ->stm_flags & GCFLAG_WRITE_BARRIER)) { - spin_loop(); - if (!--busy_loop) - stm_fatalerror("missing GCFLAG_WRITE_BARRIER"); - } - } -#endif + /* check that so far all copies of the object have the flag */ + check_flag_write_barrier(obj); /* remove GCFLAG_WRITE_BARRIER, but only if we succeeded in getting the write-lock */ assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); obj->stm_flags &= ~GCFLAG_WRITE_BARRIER; - /* for sanity, check that all other segment copies of this object - still have the flag (including the shared copy) */ - for (i = 0; i <= NB_SEGMENTS; i++) { - if (i != STM_SEGMENT->segment_num) - assert(((struct object_s *)REAL_ADDRESS(get_segment_base(i), obj)) - ->stm_flags & GCFLAG_WRITE_BARRIER); - } + /* for sanity, check again that all other segment copies of this + object still have the flag (so privatization worked) */ + check_flag_write_barrier(obj); } static void reset_transaction_read_version(void) diff --git a/c7/stm/core.h b/c7/stm/core.h --- a/c7/stm/core.h +++ b/c7/stm/core.h @@ -75,13 +75,6 @@ struct stm_priv_segment_info_s { struct stm_segment_info_s pub; - /* Dict whose keys are shared page numbers, and whose values are - the corresponding private page number. */ - struct tree_s *private_page_mapping; - - /* Head of a free list of private pages. */ - uintptr_t private_free_page_num; - /* List of old objects (older than the current transaction) that the current transaction attempts to modify. This is used to track the STM status: they are old objects that where written to and diff --git a/c7/stm/misc.c b/c7/stm/misc.c --- a/c7/stm/misc.c +++ b/c7/stm/misc.c @@ -43,9 +43,8 @@ #ifdef STM_TESTS uintptr_t _stm_get_private_page(uintptr_t pagenum) { - wlog_t *item; - TREE_FIND(*STM_PSEGMENT->private_page_mapping, pagenum, item, return 0); - return item->val; + /* xxx returns 0 or 1 now */ + return is_private_page(STM_SEGMENT->segment_num, pagenum); } long _stm_count_modified_old_objects(void) diff --git a/c7/stm/pages.c b/c7/stm/pages.c --- a/c7/stm/pages.c +++ b/c7/stm/pages.c @@ -25,6 +25,7 @@ static void teardown_pages(void) { memset(&pages_ctl, 0, sizeof(pages_ctl)); + memset(pages_privatized, 0, sizeof(pages_privatized)); } static void mutex_pages_lock(void) @@ -111,41 +112,30 @@ static void page_privatize(uintptr_t pagenum) { - wlog_t *item; - TREE_FIND(*STM_PSEGMENT->private_page_mapping, pagenum, item, - goto not_found); + if (is_private_page(STM_SEGMENT->segment_num, pagenum)) { + /* the page is already privatized */ + return; + } - /* the page is already privatized */ - return; - - not_found:; - /* lock, to prevent concurrent threads from looking up my own - 'private_page_mapping' in parallel */ + /* lock, to prevent concurrent threads from looking up this thread's + 'pages_privatized' bits in parallel */ mutex_pages_lock(); - /* look up the next free page */ - uintptr_t free_page_num = STM_PSEGMENT->private_free_page_num; - - /* "mount" it in the segment - (XXX later we should again attempt to group together many calls to - d_remap_file_pages() in succession) */ - char *new_page = STM_SEGMENT->segment_base + pagenum * 4096UL; - d_remap_file_pages(new_page, 4096, - NB_PAGES * STM_SEGMENT->segment_num + free_page_num); + /* "unmaps" the page to make the address space location correspond + again to its underlying file offset (XXX later we should again + attempt to group together many calls to d_remap_file_pages() in + succession) */ + uintptr_t pagenum_in_file = NB_PAGES * STM_SEGMENT->segment_num + pagenum; + char *new_page = stm_object_pages + pagenum_in_file * 4096UL; + d_remap_file_pages(new_page, 4096, pagenum_in_file); increment_total_allocated(4096); - /* update private_free_page_num */ - uintptr_t future_page = *(uintptr_t *)new_page; - if (future_page == 0) { - future_page = free_page_num + 1; - } - STM_PSEGMENT->private_free_page_num = future_page; - /* copy the content from the shared (segment 0) source */ pagecopy(new_page, stm_object_pages + pagenum * 4096UL); - /* update private_page_mapping */ - tree_insert(STM_PSEGMENT->private_page_mapping, pagenum, free_page_num); + /* add this thread's 'pages_privatized' bit */ + uint64_t bitmask = 1UL << (STM_SEGMENT->segment_num - 1); + pages_privatized[pagenum - PAGE_FLAG_START].by_segment |= bitmask; mutex_pages_unlock(); } diff --git a/c7/stm/pages.h b/c7/stm/pages.h --- a/c7/stm/pages.h +++ b/c7/stm/pages.h @@ -3,17 +3,11 @@ has a "shared copy" and zero or more "private copies". The shared copy of a page is stored in the mmap at the file offset - corresponding to the segment 0 offset (with all other segments - remapping to the segment 0 offset). Private copies for segment N are - made in the offset from segment N (for 1 <= N <= NB_SEGMENTS), - picking file offsets that are simply the next free ones. Each - segment maintains a tree 'private_page_mapping', which maps shared - pages to private copies. - - A major collection looks for pages that are no-longer-used private - copies, and discard them, remapping the address to the shared page. - The pages thus freed are recorded into a free list, and can be reused - as the private copies of the following (unrelated) pages. + corresponding to the segment 0 offset. Initially, accessing a page + from segment N remaps to segment 0. If the page is turned private, + then we "un-remap" it to its initial location. The 'pages_privatized' + global array records if a page is currently mapped to segment 0 + (shared page) or to its natural location (private page). Note that this page manipulation logic uses remap_file_pages() to fully hide its execution cost behind the CPU's memory management unit. @@ -22,6 +16,25 @@ (which works at the object granularity, not the page granularity). */ +#define PAGE_FLAG_START END_NURSERY_PAGE +#define PAGE_FLAG_END NB_PAGES + +struct page_shared_s { +#if NB_SEGMENTS <= 8 + uint8_t by_segment; +#elif NB_SEGMENTS <= 16 + uint16_t by_segment; +#elif NB_SEGMENTS <= 32 + uint32_t by_segment; +#elif NB_SEGMENTS <= 64 + uint64_t by_segment; +#else +# error "NB_SEGMENTS > 64 not supported right now" +#endif +}; + +static struct page_shared_s pages_privatized[PAGE_FLAG_END - PAGE_FLAG_START]; + static void page_privatize(uintptr_t pagenum); static void pages_initialize_shared(uintptr_t pagenum, uintptr_t count); @@ -35,6 +48,7 @@ static inline bool is_private_page(long segnum, uintptr_t pagenum) { - return tree_contains(get_priv_segment(segnum)->private_page_mapping, - pagenum); + assert(pagenum >= PAGE_FLAG_START); + uint64_t bitmask = 1UL << (segnum - 1); + return (pages_privatized[pagenum - PAGE_FLAG_START].by_segment & bitmask); } diff --git a/c7/stm/setup.c b/c7/stm/setup.c --- a/c7/stm/setup.c +++ b/c7/stm/setup.c @@ -58,8 +58,6 @@ pr->write_lock_num = i; pr->pub.segment_num = i; pr->pub.segment_base = segment_base; - pr->private_page_mapping = tree_create(); - pr->private_free_page_num = END_NURSERY_PAGE; pr->objects_pointing_to_nursery = NULL; pr->large_overflow_objects = NULL; pr->modified_old_objects = list_create(); From noreply at buildbot.pypy.org Sun Mar 16 14:38:21 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 16 Mar 2014 14:38:21 +0100 (CET) Subject: [pypy-commit] stmgc default: Base the diffs on llvm r201645 rather than the slightly more recent revision Message-ID: <20140316133821.90D1C1C0670@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1050:23c28d61b36e Date: 2014-03-16 14:38 +0100 http://bitbucket.org/pypy/stmgc/changeset/23c28d61b36e/ Log: Base the diffs on llvm r201645 rather than the slightly more recent revision which seems to have the bug that it doesn't produce gdb information. diff --git a/c7/llvmfix/addrspacecast-in-constant.diff b/c7/llvmfix/addrspacecast-in-constant.diff --- a/c7/llvmfix/addrspacecast-in-constant.diff +++ b/c7/llvmfix/addrspacecast-in-constant.diff @@ -1,8 +1,8 @@ Index: lib/CodeGen/AsmPrinter/AsmPrinter.cpp =================================================================== ---- lib/CodeGen/AsmPrinter/AsmPrinter.cpp (revision 203791) -+++ lib/CodeGen/AsmPrinter/AsmPrinter.cpp (working copy) -@@ -1529,6 +1529,8 @@ +--- lib/CodeGen/AsmPrinter/AsmPrinter.cpp (revision 201645) ++++ lib/CodeGen/AsmPrinter/AsmPrinter.cpp (working copy) +@@ -1617,6 +1617,8 @@ Ctx); } diff --git a/c7/llvmfix/no-introduce-bogus-cast-in-combine.diff b/c7/llvmfix/no-introduce-bogus-cast-in-combine.diff --- a/c7/llvmfix/no-introduce-bogus-cast-in-combine.diff +++ b/c7/llvmfix/no-introduce-bogus-cast-in-combine.diff @@ -1,6 +1,6 @@ Index: lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp =================================================================== ---- lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp (revision 203791) +--- lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp (revision 201645) +++ lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp (working copy) @@ -299,6 +299,17 @@ From noreply at buildbot.pypy.org Sun Mar 16 17:22:49 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 16 Mar 2014 17:22:49 +0100 (CET) Subject: [pypy-commit] stmgc c7-more-segments: Resharing pages, still buggy, in-progress Message-ID: <20140316162249.0516C1D287B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c7-more-segments Changeset: r1051:234b6a0ba9a5 Date: 2014-03-16 17:22 +0100 http://bitbucket.org/pypy/stmgc/changeset/234b6a0ba9a5/ Log: Resharing pages, still buggy, in-progress diff --git a/c7/stm/core.c b/c7/stm/core.c --- a/c7/stm/core.c +++ b/c7/stm/core.c @@ -263,7 +263,62 @@ return false; } -static void synchronize_object_now(object_t *obj, bool assume_local_private) +static void copy_object_to_shared(object_t *obj, int source_segment_num) +{ + /* Only used by major GC. XXX There is a lot of code duplication + with synchronize_object_now() but I don't completely see how to + improve... + */ + assert(_has_mutex_pages()); + assert(!_is_young(obj)); + + uintptr_t start = (uintptr_t)obj; + uintptr_t first_page = start / 4096UL; + struct object_s *realobj = (struct object_s *) + REAL_ADDRESS(get_segment(source_segment_num)->segment_base, obj); + + if (realobj->stm_flags & GCFLAG_SMALL_UNIFORM) { + abort();//XXX WRITE THE FAST CASE + } + else { + ssize_t obj_size = stmcb_size_rounded_up(realobj); + assert(obj_size >= 16); + uintptr_t end = start + obj_size; + uintptr_t last_page = (end - 1) / 4096UL; + + for (; first_page <= last_page; first_page++) { + + /* Copy the object into the shared page, if needed */ + if (is_private_page(source_segment_num, first_page)) { + + uintptr_t copy_size; + if (first_page == last_page) { + /* this is the final fragment */ + copy_size = end - start; + } + else { + /* this is a non-final fragment, going up to the + page's end */ + copy_size = 4096 - (start & 4095); + } + /* double-check that the result fits in one page */ + assert(copy_size > 0); + assert(copy_size + (start & 4095) <= 4096); + + char *src = REAL_ADDRESS(STM_SEGMENT->segment_base, start); + char *dst = REAL_ADDRESS(stm_object_pages, start); + if (copy_size == 4096) + pagecopy(dst, src); + else + memcpy(dst, src, copy_size); + } + + start = (start + 4096) & ~4095; + } + } +} + +static void synchronize_object_now(object_t *obj) { /* Copy around the version of 'obj' that lives in our own segment. It is first copied into the shared pages, and then into other @@ -306,12 +361,9 @@ assert(copy_size + (start & 4095) <= 4096); /* First copy the object into the shared page, if needed */ - assert(IMPLY(assume_local_private, - is_private_page(myself, first_page))); - char *src = REAL_ADDRESS(STM_SEGMENT->segment_base, start); char *dst = REAL_ADDRESS(stm_object_pages, start); - if (assume_local_private || is_private_page(myself, first_page)) { + if (is_private_page(myself, first_page)) { if (copy_size == 4096) pagecopy(dst, src); else @@ -352,7 +404,7 @@ return; LIST_FOREACH_R(STM_PSEGMENT->large_overflow_objects, object_t *, - synchronize_object_now(item, false)); + synchronize_object_now(item)); } static void push_modified_to_other_segments(void) @@ -374,7 +426,7 @@ /* copy the object to the shared page, and to the other private pages as needed */ - synchronize_object_now(item, true); + synchronize_object_now(item); })); list_clear(STM_PSEGMENT->modified_old_objects); diff --git a/c7/stm/core.h b/c7/stm/core.h --- a/c7/stm/core.h +++ b/c7/stm/core.h @@ -224,4 +224,5 @@ asm("/* workaround for llvm bug */"); } -static void synchronize_object_now(object_t *obj, bool assume_local_private); +static void copy_object_to_shared(object_t *obj, int source_segment_num); +static void synchronize_object_now(object_t *obj); diff --git a/c7/stm/gcpage.c b/c7/stm/gcpage.c --- a/c7/stm/gcpage.c +++ b/c7/stm/gcpage.c @@ -192,130 +192,129 @@ /************************************************************/ +static uintptr_t object_last_page(object_t *obj) +{ + uintptr_t lastbyte; + struct object_s *realobj = + (struct object_s *)REAL_ADDRESS(stm_object_pages, obj); -#if 0 -static inline void mark_single_flag_private(uintptr_t pagenum) -{ - if (flag_page_private[pagenum] == PRIVATE_PAGE) { - assert(pagenum >= END_NURSERY_PAGE); - assert(pagenum < NB_PAGES); - flag_page_private[pagenum] = SEGMENT1_PAGE; + if (realobj->stm_flags & GCFLAG_SMALL_UNIFORM) { + lastbyte = (uintptr_t)obj; } else { - assert(flag_page_private[pagenum] == SHARED_PAGE || - flag_page_private[pagenum] == SEGMENT1_PAGE); + /* get the size of the object */ + size_t obj_size = stmcb_size_rounded_up(realobj); + + /* that's the last byte within the object */ + lastbyte = ((uintptr_t)obj) + obj_size - 1; } + return lastbyte / 4096UL; } -static inline void mark_flag_page_private(object_t *obj, char *segment_base) +/* A macro that expands to: run the 'expression' for every page that + touches objects in the 'modified_old_objects' list. +*/ +#define BITOP(expression) \ + LIST_FOREACH_R( \ + get_priv_segment(segment_num)->modified_old_objects, \ + object_t * /* item */, \ + ({ \ + struct page_shared_s *ps; \ + uintptr_t pagenum = ((uintptr_t)item) / 4096UL; \ + uintptr_t count = object_last_page(item) - pagenum; \ + ps = &pages_privatized[pagenum - PAGE_FLAG_START]; \ + do { \ + expression; \ + ps++; \ + } while (count--); \ + })); + +static void major_hide_private_bits_for_modified_objects(long segment_num) { - uintptr_t first_page = ((uintptr_t)obj) / 4096UL; - - if (LIKELY((obj->stm_flags & GCFLAG_SMALL_UNIFORM) != 0)) { - mark_single_flag_private(first_page); - } - else { - char *realobj; - size_t obj_size; - uintptr_t end_page; - - /* get the size of the object */ - realobj = REAL_ADDRESS(segment_base, obj); - obj_size = stmcb_size_rounded_up((struct object_s *)realobj); - - /* that's the page *following* the last page with the object */ - end_page = (((uintptr_t)obj) + obj_size + 4095) / 4096UL; - - while (first_page < end_page) - mark_single_flag_private(first_page++); - } + uint64_t negativebitmask = ~(1 << (segment_num - 1)); +#ifndef NDEBUG + BITOP(assert((ps->by_segment & negativebitmask) != ps->by_segment)); +#endif + BITOP(ps->by_segment &= negativebitmask); } -static void major_reshare_pages_range(uintptr_t first_page, uintptr_t end_page) +static void major_restore_private_bits_for_modified_objects(long segment_num) { - uintptr_t i; - for (i = first_page; i < end_page; i++) { + uint64_t positivebitmask = 1 << (segment_num - 1); + BITOP(ps->by_segment |= positivebitmask); +} - switch (flag_page_private[i]) { - - case SEGMENT1_PAGE: - /* this page stays private after major collection */ - flag_page_private[i] = PRIVATE_PAGE; - break; - - case PRIVATE_PAGE:; - /* this page becomes shared again. No object in it was - traced belonging to a segment other than 0. - - XXX This is maybe a too-strict condition, but the more - general condition "all traced objects belong to the same - segment" has problems with large objects in segments > 0. - More precisely: we'd need to keep in the shared page the - content of the objects (from segment > 0), but also the - largemalloc's chunk data (stored in segment 0). - */ -#if NB_SEGMENTS != 2 -# error "limited to NB_SEGMENTS == 2" -#endif - char *ppage0 = get_segment_base(0) + i * 4096; - char *ppage1 = get_segment_base(1) + i * 4096; - - /* two cases for mapping pages to file-pages (fpages): - - (0->0, 1->1) - - (0->1, 1->0) - Distinguish which case it is by hacking a lot */ - - // 0->0,1->1 or 0->1,1->0 - /* map page 1 to fpage 0: */ - d_remap_file_pages(ppage1, 4096, i); - // 0->0,1->0 or 0->1,1->0 - - char oldvalue0 = *ppage0; - char oldvalue1 = *ppage1; - asm("":::"memory"); - *ppage0 = 1 + oldvalue1; - asm("":::"memory"); - char newvalue1 = *ppage1; - asm("":::"memory"); - *ppage0 = oldvalue0; - /* if we are in 0->0,1->0, old and new are different: - In this case we are done. We keep the largemalloc - data structure and objects of ppage0/fpage0 */ - if (oldvalue1 == newvalue1) { - // 0->1,1->0 - /* ppage0/fpage1 has the data structure that we want - in ppage1/fpage0, so we copy it */ - pagecopy(ppage1, ppage0); // copy from page0 to page1, - // i.e. from the underlying memory seg1 to seg0 - d_remap_file_pages(ppage0, 4096, i); - // 0->0,1->0 - } - flag_page_private[i] = SHARED_PAGE; - - increment_total_allocated(-4096 * (NB_SEGMENTS-1)); - break; - - case SHARED_PAGE: - break; /* stay shared */ - - default: - assert(!"unexpected flag_page_private"); - } - } -} +#undef BITOP static void major_reshare_pages(void) { /* re-share pages if possible. Each re-sharing decreases total_allocated by 4096. */ - major_reshare_pages_range( - END_NURSERY_PAGE, /* not the nursery! */ - (uninitialized_page_start - stm_object_pages) / 4096UL); - major_reshare_pages_range( - (uninitialized_page_stop - stm_object_pages) / 4096UL, - NB_PAGES); + + long i; + mutex_pages_lock(); + for (i = 1; i <= NB_SEGMENTS; i++) { + + /* For each segment, push the current overflow objects from + private pages to the corresponding shared pages, if necessary. + */ + struct list_s *lst = get_priv_segment(i)->large_overflow_objects; + if (lst != NULL) { + LIST_FOREACH_R(lst, object_t *, copy_object_to_shared(item, i)); + } + + /* The 'modified_old_objects' list gives the list of objects + whose pages need to remain private. We temporarily remove + these bits from 'pages_privatized', so that these pages will + be skipped by the loop below. + */ + major_hide_private_bits_for_modified_objects(i); + } + + /* Now loop over all pages that are still in 'pages_privatized', + and re-share them. + */ + uintptr_t pagenum, endpagenum; + struct page_shared_s ps; + pagenum = END_NURSERY_PAGE; /* starts after the nursery */ + endpagenum = (uninitialized_page_start - stm_object_pages) / 4096UL; + + while (1) { + if (UNLIKELY(pagenum == endpagenum)) { + /* we reach this point usually twice, because there are + more pages after 'uninitialized_page_stop' */ + if (endpagenum == NB_PAGES) + break; /* done */ + pagenum = (uninitialized_page_stop - stm_object_pages) / 4096UL; + endpagenum = NB_PAGES; + if (pagenum == endpagenum) + break; /* no pages in the 2nd section, so done too */ + } + + ps = pages_privatized[pagenum - PAGE_FLAG_START]; + if (ps.by_segment != 0) { + pages_privatized[pagenum - PAGE_FLAG_START].by_segment = 0; + + long j; + for (j = 0; j < NB_SEGMENTS; j++) { + if (ps.by_segment & (1 << j)) { + /* Page 'pagenum' is private in segment 'j + 1'. Reshare */ + page_reshare(j + 1, pagenum); + } + } + } + pagenum++; + } + + /* Done. Now 'pages_privatized' should be entirely zeroes. Restore + the previously-hidden bits + */ + for (i = 1; i <= NB_SEGMENTS; i++) { + major_restore_private_bits_for_modified_objects(i); + } + mutex_pages_unlock(); } -#endif + /************************************************************/ @@ -472,7 +471,9 @@ static void sweep_large_objects(void) { + mutex_pages_lock(); _stm_largemalloc_sweep(); + mutex_pages_unlock(); } static void clean_write_locks(void) @@ -488,7 +489,7 @@ memset(write_locks + lock2_idx, 0, sizeof(write_locks) - lock2_idx); } -static void major_set_write_locks(void) +static void major_restore_write_locks(void) { /* restore the write locks on the modified objects */ long i; @@ -518,6 +519,10 @@ dprintf((" | used before collection: %ld\n", (long)pages_ctl.total_allocated)); + /* reshare pages */ + if (RESHARE_PAGES) + major_reshare_pages(); + /* marking */ LIST_CREATE(mark_objects_to_trace); mark_visit_from_modified_objects(); @@ -531,17 +536,11 @@ clean_up_segment_lists(); /* sweeping */ - mutex_pages_lock(); -#if 0 - if (RESHARE_PAGES) - major_reshare_pages(); -#endif sweep_large_objects(); //sweep_uniform_pages(); - mutex_pages_unlock(); clean_write_locks(); - major_set_write_locks(); + major_restore_write_locks(); dprintf((" | used after collection: %ld\n", (long)pages_ctl.total_allocated)); diff --git a/c7/stm/gcpage.h b/c7/stm/gcpage.h --- a/c7/stm/gcpage.h +++ b/c7/stm/gcpage.h @@ -15,7 +15,7 @@ #define GC_MAJOR_COLLECT 1.82 /* re-share pages after major collections (1 or 0) */ -#define RESHARE_PAGES 0 +#define RESHARE_PAGES 1 diff --git a/c7/stm/nursery.c b/c7/stm/nursery.c --- a/c7/stm/nursery.c +++ b/c7/stm/nursery.c @@ -200,7 +200,7 @@ */ if (STM_PSEGMENT->minor_collect_will_commit_now) { mutex_pages_lock(); - synchronize_object_now(obj, false); + synchronize_object_now(obj); mutex_pages_unlock(); } else diff --git a/c7/stm/pages.c b/c7/stm/pages.c --- a/c7/stm/pages.c +++ b/c7/stm/pages.c @@ -140,6 +140,33 @@ mutex_pages_unlock(); } +static void page_reshare(long segment_num, uintptr_t pagenum) +{ + char *segment_base = get_segment(segment_num)->segment_base; + +#if 0 /* disabled: the private page that we are removing is + typically missing the inter-object information from + largemalloc.c */ + long i, errors=0; + uint64_t *p = (uint64_t *)(stm_object_pages + pagenum * 4096UL); + uint64_t *q = (uint64_t *)(segment_base + pagenum * 4096UL); + for (i = 0; i < 4096 / 8; i++) { + if (p[i] != q[i]) { + fprintf(stderr, "%p: 0x%lx\t\t%p: 0x%lx\n", + &p[i], p[i], &q[i], q[i]); + errors++; + } + } + assert(!errors); +#endif + + madvise(segment_base + pagenum * 4096UL, 4096, MADV_DONTNEED); + d_remap_file_pages(segment_base + pagenum * 4096UL, + 4096, pagenum); + increment_total_allocated(-4096); +} + + #if 0 static bool is_fully_in_shared_pages(object_t *obj) { diff --git a/c7/stm/pages.h b/c7/stm/pages.h --- a/c7/stm/pages.h +++ b/c7/stm/pages.h @@ -35,8 +35,9 @@ static struct page_shared_s pages_privatized[PAGE_FLAG_END - PAGE_FLAG_START]; +static void pages_initialize_shared(uintptr_t pagenum, uintptr_t count); static void page_privatize(uintptr_t pagenum); -static void pages_initialize_shared(uintptr_t pagenum, uintptr_t count); +static void page_reshare(long segment_num, uintptr_t pagenum); static void mutex_pages_lock(void); static void mutex_pages_unlock(void); diff --git a/c7/test/test_gcpage.py b/c7/test/test_gcpage.py --- a/c7/test/test_gcpage.py +++ b/c7/test/test_gcpage.py @@ -202,7 +202,6 @@ # self.start_transaction() stm_major_collect() - py.test.skip("XXX implement me") assert lib._stm_total_allocated() == 5000 + LMO # shared again def test_reshare_if_no_longer_modified_1(self): From noreply at buildbot.pypy.org Sun Mar 16 17:55:49 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 16 Mar 2014 17:55:49 +0100 (CET) Subject: [pypy-commit] stmgc c7-more-segments: Bug fixes Message-ID: <20140316165549.7686F1D28FF@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c7-more-segments Changeset: r1052:885ed3b0f6ee Date: 2014-03-16 17:55 +0100 http://bitbucket.org/pypy/stmgc/changeset/885ed3b0f6ee/ Log: Bug fixes diff --git a/c7/stm/core.c b/c7/stm/core.c --- a/c7/stm/core.c +++ b/c7/stm/core.c @@ -272,10 +272,11 @@ assert(_has_mutex_pages()); assert(!_is_young(obj)); + char *segment_base = get_segment(source_segment_num)->segment_base; uintptr_t start = (uintptr_t)obj; uintptr_t first_page = start / 4096UL; struct object_s *realobj = (struct object_s *) - REAL_ADDRESS(get_segment(source_segment_num)->segment_base, obj); + REAL_ADDRESS(segment_base, obj); if (realobj->stm_flags & GCFLAG_SMALL_UNIFORM) { abort();//XXX WRITE THE FAST CASE @@ -305,7 +306,7 @@ assert(copy_size > 0); assert(copy_size + (start & 4095) <= 4096); - char *src = REAL_ADDRESS(STM_SEGMENT->segment_base, start); + char *src = REAL_ADDRESS(segment_base, start); char *dst = REAL_ADDRESS(stm_object_pages, start); if (copy_size == 4096) pagecopy(dst, src); diff --git a/c7/stm/gcpage.c b/c7/stm/gcpage.c --- a/c7/stm/gcpage.c +++ b/c7/stm/gcpage.c @@ -253,29 +253,31 @@ long i; mutex_pages_lock(); + for (i = 1; i <= NB_SEGMENTS; i++) { + /* The 'modified_old_objects' list gives the list of objects + whose pages need to remain private. We temporarily remove + these bits from 'pages_privatized', so that these pages will + be skipped by the loop below (and by copy_object_to_shared()). + */ + major_hide_private_bits_for_modified_objects(i); /* For each segment, push the current overflow objects from - private pages to the corresponding shared pages, if necessary. + private pages to the corresponding shared pages, if + necessary. The pages that we will re-share must contain this + data; otherwise, it would exist only in the private pages, + and get lost in the loop below. */ struct list_s *lst = get_priv_segment(i)->large_overflow_objects; if (lst != NULL) { LIST_FOREACH_R(lst, object_t *, copy_object_to_shared(item, i)); } - - /* The 'modified_old_objects' list gives the list of objects - whose pages need to remain private. We temporarily remove - these bits from 'pages_privatized', so that these pages will - be skipped by the loop below. - */ - major_hide_private_bits_for_modified_objects(i); } /* Now loop over all pages that are still in 'pages_privatized', and re-share them. */ uintptr_t pagenum, endpagenum; - struct page_shared_s ps; pagenum = END_NURSERY_PAGE; /* starts after the nursery */ endpagenum = (uninitialized_page_start - stm_object_pages) / 4096UL; @@ -291,18 +293,7 @@ break; /* no pages in the 2nd section, so done too */ } - ps = pages_privatized[pagenum - PAGE_FLAG_START]; - if (ps.by_segment != 0) { - pages_privatized[pagenum - PAGE_FLAG_START].by_segment = 0; - - long j; - for (j = 0; j < NB_SEGMENTS; j++) { - if (ps.by_segment & (1 << j)) { - /* Page 'pagenum' is private in segment 'j + 1'. Reshare */ - page_reshare(j + 1, pagenum); - } - } - } + page_check_and_reshare(pagenum); pagenum++; } diff --git a/c7/stm/pages.c b/c7/stm/pages.c --- a/c7/stm/pages.c +++ b/c7/stm/pages.c @@ -140,30 +140,24 @@ mutex_pages_unlock(); } -static void page_reshare(long segment_num, uintptr_t pagenum) +static void page_reshare(uintptr_t pagenum) { - char *segment_base = get_segment(segment_num)->segment_base; + struct page_shared_s ps = pages_privatized[pagenum - PAGE_FLAG_START]; + pages_privatized[pagenum - PAGE_FLAG_START].by_segment = 0; -#if 0 /* disabled: the private page that we are removing is - typically missing the inter-object information from - largemalloc.c */ - long i, errors=0; - uint64_t *p = (uint64_t *)(stm_object_pages + pagenum * 4096UL); - uint64_t *q = (uint64_t *)(segment_base + pagenum * 4096UL); - for (i = 0; i < 4096 / 8; i++) { - if (p[i] != q[i]) { - fprintf(stderr, "%p: 0x%lx\t\t%p: 0x%lx\n", - &p[i], p[i], &q[i], q[i]); - errors++; + long j, total = 0; + for (j = 0; j < NB_SEGMENTS; j++) { + if (ps.by_segment & (1 << j)) { + /* Page 'pagenum' is private in segment 'j + 1'. Reshare */ + char *segment_base = stm_object_pages + NB_PAGES * 4096UL * (j+1); + + madvise(segment_base + pagenum * 4096UL, 4096, MADV_DONTNEED); + d_remap_file_pages(segment_base + pagenum * 4096UL, + 4096, pagenum); + total -= 4096; } } - assert(!errors); -#endif - - madvise(segment_base + pagenum * 4096UL, 4096, MADV_DONTNEED); - d_remap_file_pages(segment_base + pagenum * 4096UL, - 4096, pagenum); - increment_total_allocated(-4096); + increment_total_allocated(total); } diff --git a/c7/stm/pages.h b/c7/stm/pages.h --- a/c7/stm/pages.h +++ b/c7/stm/pages.h @@ -37,7 +37,7 @@ static void pages_initialize_shared(uintptr_t pagenum, uintptr_t count); static void page_privatize(uintptr_t pagenum); -static void page_reshare(long segment_num, uintptr_t pagenum); +static void page_reshare(uintptr_t pagenum); static void mutex_pages_lock(void); static void mutex_pages_unlock(void); @@ -53,3 +53,9 @@ uint64_t bitmask = 1UL << (segnum - 1); return (pages_privatized[pagenum - PAGE_FLAG_START].by_segment & bitmask); } + +static inline void page_check_and_reshare(uintptr_t pagenum) +{ + if (pages_privatized[pagenum - PAGE_FLAG_START].by_segment != 0) + page_reshare(pagenum); +} From noreply at buildbot.pypy.org Sun Mar 16 18:11:13 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 16 Mar 2014 18:11:13 +0100 (CET) Subject: [pypy-commit] stmgc c7-more-segments: Comment Message-ID: <20140316171113.7D8961C011F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c7-more-segments Changeset: r1053:4885b404222a Date: 2014-03-16 18:11 +0100 http://bitbucket.org/pypy/stmgc/changeset/4885b404222a/ Log: Comment diff --git a/c7/stmgc.h b/c7/stmgc.h --- a/c7/stmgc.h +++ b/c7/stmgc.h @@ -129,8 +129,10 @@ /* ==================== PUBLIC API ==================== */ -/* Number of segments (i.e. how many threads can be executed in - parallel, in maximum). +/* Number of segments (i.e. how many transactions can be executed in + parallel, in maximum). If you try to start transactions in more + threads than the number of segments, it will block, waiting for the + next segment to become free. */ #define STM_NB_SEGMENTS 4 From noreply at buildbot.pypy.org Sun Mar 16 18:29:49 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 16 Mar 2014 18:29:49 +0100 (CET) Subject: [pypy-commit] stmgc c7-more-segments: Forgot about get_segment_base() Message-ID: <20140316172949.712BB1C011F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c7-more-segments Changeset: r1054:c7ed97e2f504 Date: 2014-03-16 18:29 +0100 http://bitbucket.org/pypy/stmgc/changeset/c7ed97e2f504/ Log: Forgot about get_segment_base() diff --git a/c7/stm/core.c b/c7/stm/core.c --- a/c7/stm/core.c +++ b/c7/stm/core.c @@ -272,7 +272,7 @@ assert(_has_mutex_pages()); assert(!_is_young(obj)); - char *segment_base = get_segment(source_segment_num)->segment_base; + char *segment_base = get_segment_base(source_segment_num); uintptr_t start = (uintptr_t)obj; uintptr_t first_page = start / 4096UL; struct object_s *realobj = (struct object_s *) diff --git a/c7/stm/pages.c b/c7/stm/pages.c --- a/c7/stm/pages.c +++ b/c7/stm/pages.c @@ -149,7 +149,7 @@ for (j = 0; j < NB_SEGMENTS; j++) { if (ps.by_segment & (1 << j)) { /* Page 'pagenum' is private in segment 'j + 1'. Reshare */ - char *segment_base = stm_object_pages + NB_PAGES * 4096UL * (j+1); + char *segment_base = get_segment_base(j + 1); madvise(segment_base + pagenum * 4096UL, 4096, MADV_DONTNEED); d_remap_file_pages(segment_base + pagenum * 4096UL, From noreply at buildbot.pypy.org Sun Mar 16 18:33:38 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 16 Mar 2014 18:33:38 +0100 (CET) Subject: [pypy-commit] stmgc c7-more-segments: It worked fine by chance: "!segment_num" was always 0, which was the correct value. Message-ID: <20140316173338.A03311C011F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c7-more-segments Changeset: r1055:b23d72e6aa6b Date: 2014-03-16 18:33 +0100 http://bitbucket.org/pypy/stmgc/changeset/b23d72e6aa6b/ Log: It worked fine by chance: "!segment_num" was always 0, which was the correct value. diff --git a/c7/stm/core.c b/c7/stm/core.c --- a/c7/stm/core.c +++ b/c7/stm/core.c @@ -518,19 +518,17 @@ static void reset_modified_from_other_segments(int segment_num) { - /* pull the right versions from other threads in order + /* pull the right versions from segment 0 in order to reset our pages as part of an abort. Note that this function is also sometimes called from contention.c to clean up the state of a different thread, when we would really like it to be aborted now and it is suspended at a safe-point. - */ struct stm_priv_segment_info_s *pseg = get_priv_segment(segment_num); - long remote_num = !segment_num; char *local_base = get_segment_base(segment_num); - char *remote_base = get_segment_base(remote_num); + char *remote_base = get_segment_base(0); LIST_FOREACH_R( pseg->modified_old_objects, From noreply at buildbot.pypy.org Sun Mar 16 19:46:28 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 16 Mar 2014 19:46:28 +0100 (CET) Subject: [pypy-commit] stmgc default: merge heads Message-ID: <20140316184628.C57E61C011F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1057:526f7325d475 Date: 2014-03-16 19:45 +0100 http://bitbucket.org/pypy/stmgc/changeset/526f7325d475/ Log: merge heads diff --git a/c7/llvmfix/addrspacecast-in-constant.diff b/c7/llvmfix/addrspacecast-in-constant.diff --- a/c7/llvmfix/addrspacecast-in-constant.diff +++ b/c7/llvmfix/addrspacecast-in-constant.diff @@ -1,8 +1,8 @@ Index: lib/CodeGen/AsmPrinter/AsmPrinter.cpp =================================================================== ---- lib/CodeGen/AsmPrinter/AsmPrinter.cpp (revision 203791) -+++ lib/CodeGen/AsmPrinter/AsmPrinter.cpp (working copy) -@@ -1529,6 +1529,8 @@ +--- lib/CodeGen/AsmPrinter/AsmPrinter.cpp (revision 201645) ++++ lib/CodeGen/AsmPrinter/AsmPrinter.cpp (working copy) +@@ -1617,6 +1617,8 @@ Ctx); } diff --git a/c7/llvmfix/no-introduce-bogus-cast-in-combine.diff b/c7/llvmfix/no-introduce-bogus-cast-in-combine.diff --- a/c7/llvmfix/no-introduce-bogus-cast-in-combine.diff +++ b/c7/llvmfix/no-introduce-bogus-cast-in-combine.diff @@ -1,6 +1,6 @@ Index: lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp =================================================================== ---- lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp (revision 203791) +--- lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp (revision 201645) +++ lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp (working copy) @@ -299,6 +299,17 @@ From noreply at buildbot.pypy.org Sun Mar 16 19:46:27 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 16 Mar 2014 19:46:27 +0100 (CET) Subject: [pypy-commit] stmgc default: Merge c7-more-segments. It seems to work, including in a pypy. There Message-ID: <20140316184627.B53771C011F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1056:1e46b1ad2e26 Date: 2014-03-16 19:45 +0100 http://bitbucket.org/pypy/stmgc/changeset/1e46b1ad2e26/ Log: Merge c7-more-segments. It seems to work, including in a pypy. There is a very small but measurable overhead when compared to the previous version, probably because it makes a bit more copies for now, but I think it's ok. diff --git a/c7/demo/demo2.c b/c7/demo/demo2.c --- a/c7/demo/demo2.c +++ b/c7/demo/demo2.c @@ -6,6 +6,7 @@ #include "stmgc.h" +#define NTHREADS 3 #define LIST_LENGTH 2000 #define BUNCH 100 @@ -223,7 +224,7 @@ int main(void) { - int status; + int status, i; status = sem_init(&done, 0, 0); assert(status == 0); @@ -233,11 +234,13 @@ setup_list(); - newthread(demo2, (void*)1); - newthread(demo2, (void*)2); + for (i = 1; i <= NTHREADS; i++) { + newthread(demo2, (void*)(uintptr_t)i); + } - status = sem_wait(&done); assert(status == 0); - status = sem_wait(&done); assert(status == 0); + for (i = 1; i <= NTHREADS; i++) { + status = sem_wait(&done); assert(status == 0); + } final_check(); diff --git a/c7/demo/demo_random.c b/c7/demo/demo_random.c --- a/c7/demo/demo_random.c +++ b/c7/demo/demo_random.c @@ -385,7 +385,7 @@ for (i = 0; i < PREBUILT_ROOTS; i++) { void* new_templ = malloc(sizeof(struct node_s)); memcpy(new_templ, &prebuilt_template, sizeof(struct node_s)); - prebuilt_roots[i] = stm_setup_prebuilt((objptr_t)new_templ); + prebuilt_roots[i] = stm_setup_prebuilt((objptr_t)(long)new_templ); if (i % 2 == 0) { int hash = i + 5; diff --git a/c7/stm/contention.c b/c7/stm/contention.c --- a/c7/stm/contention.c +++ b/c7/stm/contention.c @@ -235,7 +235,7 @@ uint8_t prev_owner = ((volatile uint8_t *)write_locks)[lock_idx]; if (prev_owner != 0 && prev_owner != STM_PSEGMENT->write_lock_num) { - uint8_t other_segment_num = prev_owner - 1; + uint8_t other_segment_num = prev_owner; assert(get_priv_segment(other_segment_num)->write_lock_num == prev_owner); diff --git a/c7/stm/core.c b/c7/stm/core.c --- a/c7/stm/core.c +++ b/c7/stm/core.c @@ -8,11 +8,34 @@ memset(write_locks, 0, sizeof(write_locks)); } +static void check_flag_write_barrier(object_t *obj) +{ + /* check that all copies of the object, apart from mine, have the + GCFLAG_WRITE_BARRIER. (a bit messy because it's possible that we + read a page in the middle of privatization by another thread) + */ +#ifndef NDEBUG + long i; + struct object_s *o1; + for (i = 0; i <= NB_SEGMENTS; i++) { + if (i == STM_SEGMENT->segment_num) + continue; + o1 = (struct object_s *)REAL_ADDRESS(get_segment_base(i), obj); + if (!(o1->stm_flags & GCFLAG_WRITE_BARRIER)) { + mutex_pages_lock(); /* try again... */ + if (!(o1->stm_flags & GCFLAG_WRITE_BARRIER)) + stm_fatalerror("missing GCFLAG_WRITE_BARRIER"); + mutex_pages_unlock(); + } + } +#endif +} void _stm_write_slowpath(object_t *obj) { assert(_seems_to_be_running_transaction()); assert(!_is_young(obj)); + assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); /* is this an object from the same transaction, outside the nursery? */ if ((obj->stm_flags & -GCFLAG_OVERFLOW_NUMBER_bit0) == @@ -29,6 +52,8 @@ safepoints that may be issued in write_write_contention_management(). */ stm_read(obj); + /* XXX XXX XXX make the logic of write-locking objects optional! */ + /* claim the write-lock for this object. In case we're running the same transaction since a long while, the object can be already in 'modified_old_objects' (but, because it had GCFLAG_WRITE_BARRIER, @@ -59,12 +84,12 @@ the common case. Otherwise, we need to compute it based on its location and size. */ if ((obj->stm_flags & GCFLAG_SMALL_UNIFORM) != 0) { - pages_privatize(first_page, 1, true); + page_privatize(first_page); } else { char *realobj; size_t obj_size; - uintptr_t end_page; + uintptr_t i, end_page; /* get the size of the object */ realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj); @@ -73,7 +98,9 @@ /* that's the page *following* the last page with the object */ end_page = (((uintptr_t)obj) + obj_size + 4095) / 4096UL; - pages_privatize(first_page, end_page - first_page, true); + for (i = first_page; i < end_page; i++) { + page_privatize(i); + } } } else if (write_locks[lock_idx] == lock_num) { @@ -100,19 +127,21 @@ LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, obj); } - /* add the write-barrier-already-called flag ONLY if we succeeded in + /* check that we really have a private page */ + assert(is_private_page(STM_SEGMENT->segment_num, + ((uintptr_t)obj) / 4096)); + + /* check that so far all copies of the object have the flag */ + check_flag_write_barrier(obj); + + /* remove GCFLAG_WRITE_BARRIER, but only if we succeeded in getting the write-lock */ assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); obj->stm_flags &= ~GCFLAG_WRITE_BARRIER; - /* for sanity, check that all other segment copies of this object - still have the flag */ - long i; - for (i = 0; i < NB_SEGMENTS; i++) { - if (i != STM_SEGMENT->segment_num) - assert(((struct object_s *)REAL_ADDRESS(get_segment_base(i), obj)) - ->stm_flags & GCFLAG_WRITE_BARRIER); - } + /* for sanity, check again that all other segment copies of this + object still have the flag (so privatization worked) */ + check_flag_write_barrier(obj); } static void reset_transaction_read_version(void) @@ -193,59 +222,130 @@ /************************************************************/ -#if NB_SEGMENTS != 2 -# error "The logic in the functions below only works with two segments" -#endif static bool detect_write_read_conflicts(void) { - long remote_num = 1 - STM_SEGMENT->segment_num; - char *remote_base = get_segment_base(remote_num); - uint8_t remote_version = get_segment(remote_num)->transaction_read_version; + /* Detect conflicts of the form: we want to commit a write to an object, + but the same object was also read in a different thread. + */ + long i; + for (i = 1; i <= NB_SEGMENTS; i++) { - if (get_priv_segment(remote_num)->transaction_state == TS_NONE) - return false; /* no need to check */ + if (i == STM_SEGMENT->segment_num) + continue; - if (is_aborting_now(remote_num)) - return false; /* no need to check: is pending immediate abort */ + if (get_priv_segment(i)->transaction_state == TS_NONE) + continue; /* no need to check */ - LIST_FOREACH_R( - STM_PSEGMENT->modified_old_objects, - object_t * /*item*/, - ({ - if (was_read_remote(remote_base, item, remote_version)) { - /* A write-read conflict! */ - write_read_contention_management(remote_num); + if (is_aborting_now(i)) + continue; /* no need to check: is pending immediate abort */ - /* If we reach this point, we didn't abort, but maybe we - had to wait for the other thread to commit. If we - did, then we have to restart committing from our call - to synchronize_all_threads(). */ - return true; - } - })); + char *remote_base = get_segment_base(i); + uint8_t remote_version = get_segment(i)->transaction_read_version; + + LIST_FOREACH_R( + STM_PSEGMENT->modified_old_objects, + object_t * /*item*/, + ({ + if (was_read_remote(remote_base, item, remote_version)) { + /* A write-read conflict! */ + write_read_contention_management(i); + + /* If we reach this point, we didn't abort, but maybe we + had to wait for the other thread to commit. If we + did, then we have to restart committing from our call + to synchronize_all_threads(). */ + return true; + } + })); + } return false; } -static void synchronize_overflow_object_now(object_t *obj) +static void copy_object_to_shared(object_t *obj, int source_segment_num) { + /* Only used by major GC. XXX There is a lot of code duplication + with synchronize_object_now() but I don't completely see how to + improve... + */ + assert(_has_mutex_pages()); assert(!_is_young(obj)); - assert((obj->stm_flags & GCFLAG_SMALL_UNIFORM) == 0); + + char *segment_base = get_segment_base(source_segment_num); + uintptr_t start = (uintptr_t)obj; + uintptr_t first_page = start / 4096UL; + struct object_s *realobj = (struct object_s *) + REAL_ADDRESS(segment_base, obj); + + if (realobj->stm_flags & GCFLAG_SMALL_UNIFORM) { + abort();//XXX WRITE THE FAST CASE + } + else { + ssize_t obj_size = stmcb_size_rounded_up(realobj); + assert(obj_size >= 16); + uintptr_t end = start + obj_size; + uintptr_t last_page = (end - 1) / 4096UL; + + for (; first_page <= last_page; first_page++) { + + /* Copy the object into the shared page, if needed */ + if (is_private_page(source_segment_num, first_page)) { + + uintptr_t copy_size; + if (first_page == last_page) { + /* this is the final fragment */ + copy_size = end - start; + } + else { + /* this is a non-final fragment, going up to the + page's end */ + copy_size = 4096 - (start & 4095); + } + /* double-check that the result fits in one page */ + assert(copy_size > 0); + assert(copy_size + (start & 4095) <= 4096); + + char *src = REAL_ADDRESS(segment_base, start); + char *dst = REAL_ADDRESS(stm_object_pages, start); + if (copy_size == 4096) + pagecopy(dst, src); + else + memcpy(dst, src, copy_size); + } + + start = (start + 4096) & ~4095; + } + } +} + +static void synchronize_object_now(object_t *obj) +{ + /* Copy around the version of 'obj' that lives in our own segment. + It is first copied into the shared pages, and then into other + segments' own private pages. + + This must be called with the mutex_pages_lock! + */ + assert(_has_mutex_pages()); + assert(!_is_young(obj)); assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); - char *realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj); - ssize_t obj_size = stmcb_size_rounded_up((struct object_s *)realobj); - assert(obj_size >= 16); uintptr_t start = (uintptr_t)obj; - uintptr_t end = start + obj_size; uintptr_t first_page = start / 4096UL; - uintptr_t last_page = (end - 1) / 4096UL; - do { - if (flag_page_private[first_page] != SHARED_PAGE) { - /* The page is a PRIVATE_PAGE. We need to diffuse this fragment - of our object from our own segment to all other segments. */ + if (obj->stm_flags & GCFLAG_SMALL_UNIFORM) { + abort();//XXX WRITE THE FAST CASE + } + else { + char *realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj); + ssize_t obj_size = stmcb_size_rounded_up((struct object_s *)realobj); + assert(obj_size >= 16); + uintptr_t end = start + obj_size; + uintptr_t last_page = (end - 1) / 4096UL; + long i, myself = STM_SEGMENT->segment_num; + + for (; first_page <= last_page; first_page++) { uintptr_t copy_size; if (first_page == last_page) { @@ -253,26 +353,50 @@ copy_size = end - start; } else { - /* this is a non-final fragment, going up to the page's end */ + /* this is a non-final fragment, going up to the + page's end */ copy_size = 4096 - (start & 4095); } - /* double-check that the result fits in one page */ assert(copy_size > 0); assert(copy_size + (start & 4095) <= 4096); - long i; + /* First copy the object into the shared page, if needed */ char *src = REAL_ADDRESS(STM_SEGMENT->segment_base, start); - for (i = 0; i < NB_SEGMENTS; i++) { - if (i != STM_SEGMENT->segment_num) { - char *dst = REAL_ADDRESS(get_segment_base(i), start); + char *dst = REAL_ADDRESS(stm_object_pages, start); + if (is_private_page(myself, first_page)) { + if (copy_size == 4096) + pagecopy(dst, src); + else memcpy(dst, src, copy_size); + } + else { + assert(memcmp(dst, src, copy_size) == 0); /* same page */ + } + + for (i = 1; i <= NB_SEGMENTS; i++) { + if (i == myself) + continue; + + src = REAL_ADDRESS(stm_object_pages, start); + dst = REAL_ADDRESS(get_segment_base(i), start); + if (is_private_page(i, first_page)) { + /* The page is a private page. We need to diffuse this + fragment of object from the shared page to this private + page. */ + if (copy_size == 4096) + pagecopy(dst, src); + else + memcpy(dst, src, copy_size); + } + else { + assert(memcmp(dst, src, copy_size) == 0); /* same page */ } } + + start = (start + 4096) & ~4095; } - - start = (start + 4096) & ~4095; - } while (first_page++ < last_page); + } } static void push_overflow_objects_from_privatized_pages(void) @@ -281,27 +405,15 @@ return; LIST_FOREACH_R(STM_PSEGMENT->large_overflow_objects, object_t *, - synchronize_overflow_object_now(item)); + synchronize_object_now(item)); } static void push_modified_to_other_segments(void) { - long remote_num = 1 - STM_SEGMENT->segment_num; - char *local_base = STM_SEGMENT->segment_base; - char *remote_base = get_segment_base(remote_num); - bool remote_active = - (get_priv_segment(remote_num)->transaction_state != TS_NONE && - get_segment(remote_num)->nursery_end != NSE_SIGABORT); - LIST_FOREACH_R( STM_PSEGMENT->modified_old_objects, object_t * /*item*/, ({ - if (remote_active) { - assert(!was_read_remote(remote_base, item, - get_segment(remote_num)->transaction_read_version)); - } - /* clear the write-lock (note that this runs with all other threads paused, so no need to be careful about ordering) */ uintptr_t lock_idx = (((uintptr_t)item) >> 4) - WRITELOCK_START; @@ -313,11 +425,9 @@ minor_collection() */ assert((item->stm_flags & GCFLAG_WRITE_BARRIER) != 0); - /* copy the modified object to the other segment */ - char *src = REAL_ADDRESS(local_base, item); - char *dst = REAL_ADDRESS(remote_base, item); - ssize_t size = stmcb_size_rounded_up((struct object_s *)src); - memcpy(dst, src, size); + /* copy the object to the shared page, and to the other + private pages as needed */ + synchronize_object_now(item); })); list_clear(STM_PSEGMENT->modified_old_objects); @@ -368,10 +478,12 @@ major_collection_now_at_safe_point(); /* synchronize overflow objects living in privatized pages */ + mutex_pages_lock(); push_overflow_objects_from_privatized_pages(); /* synchronize modified old objects to other threads */ push_modified_to_other_segments(); + mutex_pages_unlock(); /* update 'overflow_number' if needed */ if (STM_PSEGMENT->overflow_number_has_been_used) { @@ -406,19 +518,17 @@ static void reset_modified_from_other_segments(int segment_num) { - /* pull the right versions from other threads in order + /* pull the right versions from segment 0 in order to reset our pages as part of an abort. Note that this function is also sometimes called from contention.c to clean up the state of a different thread, when we would really like it to be aborted now and it is suspended at a safe-point. - */ struct stm_priv_segment_info_s *pseg = get_priv_segment(segment_num); - long remote_num = !segment_num; char *local_base = get_segment_base(segment_num); - char *remote_base = get_segment_base(remote_num); + char *remote_base = get_segment_base(0); LIST_FOREACH_R( pseg->modified_old_objects, @@ -489,6 +599,7 @@ static void abort_with_mutex(void) { + assert(_has_mutex()); dprintf(("~~~ ABORT\n")); switch (STM_PSEGMENT->transaction_state) { @@ -514,8 +625,11 @@ /* invoke the callbacks */ invoke_and_clear_callbacks_on_abort(); - if (STM_SEGMENT->nursery_end == NSE_SIGABORT) - STM_SEGMENT->nursery_end = NURSERY_END; /* done aborting */ + if (STM_SEGMENT->nursery_end == NSE_SIGABORT) { + /* done aborting */ + STM_SEGMENT->nursery_end = pause_signalled ? NSE_SIGPAUSE + : NURSERY_END; + } _finish_transaction(); /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */ diff --git a/c7/stm/core.h b/c7/stm/core.h --- a/c7/stm/core.h +++ b/c7/stm/core.h @@ -15,12 +15,12 @@ #define NB_PAGES (1500*256) // 1500MB -#define NB_SEGMENTS 2 +#define NB_SEGMENTS STM_NB_SEGMENTS #define NB_SEGMENTS_MAX 240 /* don't increase NB_SEGMENTS past this */ #define MAP_PAGES_FLAGS (MAP_SHARED | MAP_ANONYMOUS | MAP_NORESERVE) #define NB_NURSERY_PAGES (STM_GC_NURSERY/4) -#define TOTAL_MEMORY (NB_PAGES * 4096UL * NB_SEGMENTS) +#define TOTAL_MEMORY (NB_PAGES * 4096UL * (1 + NB_SEGMENTS)) #define READMARKER_END ((NB_PAGES * 4096UL) >> 4) #define FIRST_OBJECT_PAGE ((READMARKER_END + 4095) / 4096UL) #define FIRST_NURSERY_PAGE FIRST_OBJECT_PAGE @@ -178,10 +178,6 @@ static char *stm_object_pages; static stm_thread_local_t *stm_all_thread_locals = NULL; -#ifdef STM_TESTS -static char *stm_other_pages; -#endif - static uint8_t write_locks[WRITELOCK_END - WRITELOCK_START]; @@ -228,4 +224,5 @@ asm("/* workaround for llvm bug */"); } -static void synchronize_overflow_object_now(object_t *obj); +static void copy_object_to_shared(object_t *obj, int source_segment_num); +static void synchronize_object_now(object_t *obj); diff --git a/c7/stm/fprintcolor.h b/c7/stm/fprintcolor.h --- a/c7/stm/fprintcolor.h +++ b/c7/stm/fprintcolor.h @@ -9,7 +9,7 @@ #define dprintf(args) threadcolor_printf args static inline int dprintfcolor(void) { - return 31 + STM_SEGMENT->segment_num % 6; + return 31 + (STM_SEGMENT->segment_num + 5) % 6; } static int threadcolor_printf(const char *format, ...) diff --git a/c7/stm/gcpage.c b/c7/stm/gcpage.c --- a/c7/stm/gcpage.c +++ b/c7/stm/gcpage.c @@ -192,129 +192,121 @@ /************************************************************/ +static uintptr_t object_last_page(object_t *obj) +{ + uintptr_t lastbyte; + struct object_s *realobj = + (struct object_s *)REAL_ADDRESS(stm_object_pages, obj); -static inline void mark_single_flag_private(uintptr_t pagenum) -{ - if (flag_page_private[pagenum] == PRIVATE_PAGE) { - assert(pagenum >= END_NURSERY_PAGE); - assert(pagenum < NB_PAGES); - flag_page_private[pagenum] = SEGMENT1_PAGE; + if (realobj->stm_flags & GCFLAG_SMALL_UNIFORM) { + lastbyte = (uintptr_t)obj; } else { - assert(flag_page_private[pagenum] == SHARED_PAGE || - flag_page_private[pagenum] == SEGMENT1_PAGE); + /* get the size of the object */ + size_t obj_size = stmcb_size_rounded_up(realobj); + + /* that's the last byte within the object */ + lastbyte = ((uintptr_t)obj) + obj_size - 1; } + return lastbyte / 4096UL; } -static inline void mark_flag_page_private(object_t *obj, char *segment_base) +/* A macro that expands to: run the 'expression' for every page that + touches objects in the 'modified_old_objects' list. +*/ +#define BITOP(expression) \ + LIST_FOREACH_R( \ + get_priv_segment(segment_num)->modified_old_objects, \ + object_t * /* item */, \ + ({ \ + struct page_shared_s *ps; \ + uintptr_t pagenum = ((uintptr_t)item) / 4096UL; \ + uintptr_t count = object_last_page(item) - pagenum; \ + ps = &pages_privatized[pagenum - PAGE_FLAG_START]; \ + do { \ + expression; \ + ps++; \ + } while (count--); \ + })); + +static void major_hide_private_bits_for_modified_objects(long segment_num) { - uintptr_t first_page = ((uintptr_t)obj) / 4096UL; - - if (LIKELY((obj->stm_flags & GCFLAG_SMALL_UNIFORM) != 0)) { - mark_single_flag_private(first_page); - } - else { - char *realobj; - size_t obj_size; - uintptr_t end_page; - - /* get the size of the object */ - realobj = REAL_ADDRESS(segment_base, obj); - obj_size = stmcb_size_rounded_up((struct object_s *)realobj); - - /* that's the page *following* the last page with the object */ - end_page = (((uintptr_t)obj) + obj_size + 4095) / 4096UL; - - while (first_page < end_page) - mark_single_flag_private(first_page++); - } + uint64_t negativebitmask = ~(1 << (segment_num - 1)); +#ifndef NDEBUG + BITOP(assert((ps->by_segment & negativebitmask) != ps->by_segment)); +#endif + BITOP(ps->by_segment &= negativebitmask); } -static void major_reshare_pages_range(uintptr_t first_page, uintptr_t end_page) +static void major_restore_private_bits_for_modified_objects(long segment_num) { - uintptr_t i; - for (i = first_page; i < end_page; i++) { + uint64_t positivebitmask = 1 << (segment_num - 1); + BITOP(ps->by_segment |= positivebitmask); +} - switch (flag_page_private[i]) { - - case SEGMENT1_PAGE: - /* this page stays private after major collection */ - flag_page_private[i] = PRIVATE_PAGE; - break; - - case PRIVATE_PAGE:; - /* this page becomes shared again. No object in it was - traced belonging to a segment other than 0. - - XXX This is maybe a too-strict condition, but the more - general condition "all traced objects belong to the same - segment" has problems with large objects in segments > 0. - More precisely: we'd need to keep in the shared page the - content of the objects (from segment > 0), but also the - largemalloc's chunk data (stored in segment 0). - */ -#if NB_SEGMENTS != 2 -# error "limited to NB_SEGMENTS == 2" -#endif - char *ppage0 = get_segment_base(0) + i * 4096; - char *ppage1 = get_segment_base(1) + i * 4096; - - /* two cases for mapping pages to file-pages (fpages): - - (0->0, 1->1) - - (0->1, 1->0) - Distinguish which case it is by hacking a lot */ - - // 0->0,1->1 or 0->1,1->0 - /* map page 1 to fpage 0: */ - d_remap_file_pages(ppage1, 4096, i); - // 0->0,1->0 or 0->1,1->0 - - char oldvalue0 = *ppage0; - char oldvalue1 = *ppage1; - asm("":::"memory"); - *ppage0 = 1 + oldvalue1; - asm("":::"memory"); - char newvalue1 = *ppage1; - asm("":::"memory"); - *ppage0 = oldvalue0; - /* if we are in 0->0,1->0, old and new are different: - In this case we are done. We keep the largemalloc - data structure and objects of ppage0/fpage0 */ - if (oldvalue1 == newvalue1) { - // 0->1,1->0 - /* ppage0/fpage1 has the data structure that we want - in ppage1/fpage0, so we copy it */ - pagecopy(ppage1, ppage0); // copy from page0 to page1, - // i.e. from the underlying memory seg1 to seg0 - d_remap_file_pages(ppage0, 4096, i); - // 0->0,1->0 - } - flag_page_private[i] = SHARED_PAGE; - - increment_total_allocated(-4096 * (NB_SEGMENTS-1)); - break; - - case SHARED_PAGE: - break; /* stay shared */ - - default: - assert(!"unexpected flag_page_private"); - } - } -} +#undef BITOP static void major_reshare_pages(void) { /* re-share pages if possible. Each re-sharing decreases total_allocated by 4096. */ - major_reshare_pages_range( - END_NURSERY_PAGE, /* not the nursery! */ - (uninitialized_page_start - stm_object_pages) / 4096UL); - major_reshare_pages_range( - (uninitialized_page_stop - stm_object_pages) / 4096UL, - NB_PAGES); + + long i; + mutex_pages_lock(); + + for (i = 1; i <= NB_SEGMENTS; i++) { + /* The 'modified_old_objects' list gives the list of objects + whose pages need to remain private. We temporarily remove + these bits from 'pages_privatized', so that these pages will + be skipped by the loop below (and by copy_object_to_shared()). + */ + major_hide_private_bits_for_modified_objects(i); + + /* For each segment, push the current overflow objects from + private pages to the corresponding shared pages, if + necessary. The pages that we will re-share must contain this + data; otherwise, it would exist only in the private pages, + and get lost in the loop below. + */ + struct list_s *lst = get_priv_segment(i)->large_overflow_objects; + if (lst != NULL) { + LIST_FOREACH_R(lst, object_t *, copy_object_to_shared(item, i)); + } + } + + /* Now loop over all pages that are still in 'pages_privatized', + and re-share them. + */ + uintptr_t pagenum, endpagenum; + pagenum = END_NURSERY_PAGE; /* starts after the nursery */ + endpagenum = (uninitialized_page_start - stm_object_pages) / 4096UL; + + while (1) { + if (UNLIKELY(pagenum == endpagenum)) { + /* we reach this point usually twice, because there are + more pages after 'uninitialized_page_stop' */ + if (endpagenum == NB_PAGES) + break; /* done */ + pagenum = (uninitialized_page_stop - stm_object_pages) / 4096UL; + endpagenum = NB_PAGES; + if (pagenum == endpagenum) + break; /* no pages in the 2nd section, so done too */ + } + + page_check_and_reshare(pagenum); + pagenum++; + } + + /* Done. Now 'pages_privatized' should be entirely zeroes. Restore + the previously-hidden bits + */ + for (i = 1; i <= NB_SEGMENTS; i++) { + major_restore_private_bits_for_modified_objects(i); + } + mutex_pages_unlock(); } + /************************************************************/ @@ -323,11 +315,6 @@ /* takes a normal pointer to a thread-local pointer to an object */ object_t *obj = *pobj; - if (obj == NULL || mark_visited_test_and_set(obj)) - return; /* already visited this object */ - - LIST_APPEND(mark_objects_to_trace, obj); - /* Note: this obj might be visited already, but from a different segment. We ignore this case and skip re-visiting the object anyway. The idea is that such an object is old (not from the @@ -338,6 +325,10 @@ segments and only needs visiting once. (It may actually be in a shared page, or maybe not.) */ + if (obj == NULL || mark_visited_test_and_set(obj)) + return; /* already visited this object */ + + LIST_APPEND(mark_objects_to_trace, obj); } static void mark_trace(object_t *obj, char *segment_base) @@ -345,13 +336,6 @@ assert(list_is_empty(mark_objects_to_trace)); while (1) { - - /* first, if we're not seeing segment 0, we must change the - flags in flag_page_private[] from PRIVATE_PAGE to - SEGMENT1_PAGE, which will mean "can't re-share" */ - if (segment_base != stm_object_pages && RESHARE_PAGES) - mark_flag_page_private(obj, segment_base); - /* trace into the object (the version from 'segment_base') */ struct object_s *realobj = (struct object_s *)REAL_ADDRESS(segment_base, obj); @@ -373,45 +357,33 @@ static void mark_visit_from_roots(void) { - if (testing_prebuilt_objs != NULL) { LIST_FOREACH_R(testing_prebuilt_objs, object_t * /*item*/, - mark_visit_object(item, get_segment_base(0))); + mark_visit_object(item, stm_object_pages)); } - /* Do the following twice, so that we trace first the objects from - segment 0, and then all others. XXX This is a hack to make it - more likely that we'll be able to re-share pages. */ + stm_thread_local_t *tl = stm_all_thread_locals; + do { + /* If 'tl' is currently running, its 'associated_segment_num' + field is the segment number that contains the correct + version of its overflowed objects. If not, then the + field is still some correct segment number, and it doesn't + matter which one we pick. */ + char *segment_base = get_segment_base(tl->associated_segment_num); - int must_be_zero; - for (must_be_zero = 1; must_be_zero >= 0; must_be_zero--) { + struct stm_shadowentry_s *current = tl->shadowstack; + struct stm_shadowentry_s *base = tl->shadowstack_base; + while (current-- != base) { + assert(current->ss != (object_t *)-1); + mark_visit_object(current->ss, segment_base); + } + mark_visit_object(tl->thread_local_obj, segment_base); - stm_thread_local_t *tl = stm_all_thread_locals; - do { - /* If 'tl' is currently running, its 'associated_segment_num' - field is the segment number that contains the correct - version of its overflowed objects. If not, then the - field is still some correct segment number, and it doesn't - matter which one we pick. */ - char *segment_base = get_segment_base(tl->associated_segment_num); - - if (must_be_zero == (segment_base == get_segment_base(0))) { - - struct stm_shadowentry_s *current = tl->shadowstack; - struct stm_shadowentry_s *base = tl->shadowstack_base; - while (current-- != base) { - assert(current->ss != (object_t *)-1); - mark_visit_object(current->ss, segment_base); - } - mark_visit_object(tl->thread_local_obj, segment_base); - } - - tl = tl->next; - } while (tl != stm_all_thread_locals); - } + tl = tl->next; + } while (tl != stm_all_thread_locals); long i; - for (i = 0; i < NB_SEGMENTS; i++) { + for (i = 1; i <= NB_SEGMENTS; i++) { if (get_priv_segment(i)->transaction_state != TS_NONE) mark_visit_object( get_priv_segment(i)->threadlocal_at_start_of_transaction, @@ -422,20 +394,21 @@ static void mark_visit_from_modified_objects(void) { /* The modified objects are the ones that may exist in two different - versions: one in the segment that modified it, and another in - all other segments. */ + versions: one in the segment that modified it, and another in all + other segments. (It can also be more than two if we don't have + eager write locking.) + */ long i; - for (i = 0; i < NB_SEGMENTS; i++) { - char *base1 = get_segment_base(i); /* two different segments */ - char *base2 = get_segment_base(!i); + for (i = 1; i <= NB_SEGMENTS; i++) { + char *base = get_segment_base(i); LIST_FOREACH_R( get_priv_segment(i)->modified_old_objects, object_t * /*item*/, ({ mark_visited_test_and_set(item); - mark_trace(item, base1); - mark_trace(item, base2); + mark_trace(item, stm_object_pages); /* shared version */ + mark_trace(item, base); /* private version */ })); } } @@ -443,7 +416,7 @@ static void clean_up_segment_lists(void) { long i; - for (i = 0; i < NB_SEGMENTS; i++) { + for (i = 1; i <= NB_SEGMENTS; i++) { struct stm_priv_segment_info_s *pseg = get_priv_segment(i); struct list_s *lst; @@ -489,7 +462,9 @@ static void sweep_large_objects(void) { + mutex_pages_lock(); _stm_largemalloc_sweep(); + mutex_pages_unlock(); } static void clean_write_locks(void) @@ -505,11 +480,11 @@ memset(write_locks + lock2_idx, 0, sizeof(write_locks) - lock2_idx); } -static void major_set_write_locks(void) +static void major_restore_write_locks(void) { /* restore the write locks on the modified objects */ long i; - for (i = 0; i < NB_SEGMENTS; i++) { + for (i = 1; i <= NB_SEGMENTS; i++) { struct stm_priv_segment_info_s *pseg = get_priv_segment(i); LIST_FOREACH_R( @@ -535,6 +510,10 @@ dprintf((" | used before collection: %ld\n", (long)pages_ctl.total_allocated)); + /* reshare pages */ + if (RESHARE_PAGES) + major_reshare_pages(); + /* marking */ LIST_CREATE(mark_objects_to_trace); mark_visit_from_modified_objects(); @@ -548,15 +527,11 @@ clean_up_segment_lists(); /* sweeping */ - mutex_pages_lock(); - if (RESHARE_PAGES) - major_reshare_pages(); sweep_large_objects(); //sweep_uniform_pages(); - mutex_pages_unlock(); clean_write_locks(); - major_set_write_locks(); + major_restore_write_locks(); dprintf((" | used after collection: %ld\n", (long)pages_ctl.total_allocated)); diff --git a/c7/stm/gcpage.h b/c7/stm/gcpage.h --- a/c7/stm/gcpage.h +++ b/c7/stm/gcpage.h @@ -15,7 +15,7 @@ #define GC_MAJOR_COLLECT 1.82 /* re-share pages after major collections (1 or 0) */ -#define RESHARE_PAGES 0 +#define RESHARE_PAGES 1 diff --git a/c7/stm/misc.c b/c7/stm/misc.c --- a/c7/stm/misc.c +++ b/c7/stm/misc.c @@ -41,9 +41,10 @@ } #ifdef STM_TESTS -uint8_t _stm_get_page_flag(uintptr_t index) +uintptr_t _stm_get_private_page(uintptr_t pagenum) { - return flag_page_private[index]; + /* xxx returns 0 or 1 now */ + return is_private_page(STM_SEGMENT->segment_num, pagenum); } long _stm_count_modified_old_objects(void) @@ -79,4 +80,14 @@ mutex_pages_unlock(); return result; } + +void _stm_mutex_pages_lock(void) +{ + mutex_pages_lock(); +} + +void _stm_mutex_pages_unlock(void) +{ + mutex_pages_unlock(); +} #endif diff --git a/c7/stm/nursery.c b/c7/stm/nursery.c --- a/c7/stm/nursery.c +++ b/c7/stm/nursery.c @@ -26,7 +26,7 @@ _stm_nursery_start = NURSERY_START; long i; - for (i = 0; i < NB_SEGMENTS; i++) { + for (i = 1; i <= NB_SEGMENTS; i++) { get_segment(i)->nursery_current = (stm_char *)NURSERY_START; get_segment(i)->nursery_end = NURSERY_END; } @@ -198,8 +198,11 @@ WRITE_BARRIER flag and traced into it to fix its content); or add the object to 'large_overflow_objects'. */ - if (STM_PSEGMENT->minor_collect_will_commit_now) - synchronize_overflow_object_now(obj); + if (STM_PSEGMENT->minor_collect_will_commit_now) { + mutex_pages_lock(); + synchronize_object_now(obj); + mutex_pages_unlock(); + } else LIST_APPEND(STM_PSEGMENT->large_overflow_objects, obj); } @@ -378,7 +381,7 @@ _stm_nursery_start = NURSERY_END - free_count; long i; - for (i = 0; i < NB_SEGMENTS; i++) { + for (i = 1; i <= NB_SEGMENTS; i++) { if ((uintptr_t)get_segment(i)->nursery_current < _stm_nursery_start) get_segment(i)->nursery_current = (stm_char *)_stm_nursery_start; } @@ -411,7 +414,7 @@ int original_num = STM_SEGMENT->segment_num; long i; - for (i = 0; i < NB_SEGMENTS; i++) { + for (i = 1; i <= NB_SEGMENTS; i++) { struct stm_priv_segment_info_s *pseg = get_priv_segment(i); if (MINOR_NOTHING_TO_DO(pseg)) /*TS_NONE segments have NOTHING_TO_DO*/ continue; diff --git a/c7/stm/pages.c b/c7/stm/pages.c --- a/c7/stm/pages.c +++ b/c7/stm/pages.c @@ -25,6 +25,7 @@ static void teardown_pages(void) { memset(&pages_ctl, 0, sizeof(pages_ctl)); + memset(pages_privatized, 0, sizeof(pages_privatized)); } static void mutex_pages_lock(void) @@ -39,7 +40,6 @@ __sync_lock_release(&pages_ctl.mutex_pages); } -__attribute__((unused)) static bool _has_mutex_pages(void) { return pages_ctl.mutex_pages != 0; @@ -47,6 +47,7 @@ static uint64_t increment_total_allocated(ssize_t add_or_remove) { + assert(_has_mutex_pages()); pages_ctl.total_allocated += add_or_remove; if (pages_ctl.total_allocated >= pages_ctl.total_allocated_bound) @@ -102,100 +103,64 @@ segment 0. */ uintptr_t i; assert(_has_mutex_pages()); - for (i = 1; i < NB_SEGMENTS; i++) { + for (i = 1; i <= NB_SEGMENTS; i++) { char *segment_base = get_segment_base(i); d_remap_file_pages(segment_base + pagenum * 4096UL, count * 4096UL, pagenum); } - for (i = 0; i < count; i++) - flag_page_private[pagenum + i] = SHARED_PAGE; } -#if 0 -static void pages_make_shared_again(uintptr_t pagenum, uintptr_t count) +static void page_privatize(uintptr_t pagenum) { - /* Same as pages_initialize_shared(), but tries hard to minimize the - total number of pages that remap_file_pages() must handle, by - fragmenting calls as much as possible (the overhead of one system - call appears smaller as the overhead per page). */ - uintptr_t start, i = 0; - while (i < count) { - if (flag_page_private[pagenum + (i++)] == SHARED_PAGE) - continue; - start = i; /* first index of a private page */ - while (1) { - i++; - if (i == count || flag_page_private[pagenum + i] == SHARED_PAGE) - break; - } - pages_initialize_shared(pagenum + start, i - start); - } -} -#endif - -static void privatize_range(uintptr_t pagenum, uintptr_t count, bool full) -{ - ssize_t pgoff1 = pagenum; - ssize_t pgoff2 = pagenum + NB_PAGES; - ssize_t localpgoff = pgoff1 + NB_PAGES * STM_SEGMENT->segment_num; - ssize_t otherpgoff = pgoff1 + NB_PAGES * (1 - STM_SEGMENT->segment_num); - - void *localpg = stm_object_pages + localpgoff * 4096UL; - void *otherpg = stm_object_pages + otherpgoff * 4096UL; - - memset(flag_page_private + pagenum, REMAPPING_PAGE, count); - d_remap_file_pages(localpg, count * 4096, pgoff2); - uintptr_t i; - if (full) { - for (i = 0; i < count; i++) { - pagecopy(localpg + 4096 * i, otherpg + 4096 * i); - } - } - else { - pagecopy(localpg, otherpg); - if (count > 1) - pagecopy(localpg + 4096 * (count-1), otherpg + 4096 * (count-1)); - } - write_fence(); - memset(flag_page_private + pagenum, PRIVATE_PAGE, count); - increment_total_allocated(4096 * count); -} - -static void _pages_privatize(uintptr_t pagenum, uintptr_t count, bool full) -{ - /* narrow the range of pages to privatize from the end: */ - while (flag_page_private[pagenum + count - 1] == PRIVATE_PAGE) { - if (!--count) - return; + if (is_private_page(STM_SEGMENT->segment_num, pagenum)) { + /* the page is already privatized */ + return; } + /* lock, to prevent concurrent threads from looking up this thread's + 'pages_privatized' bits in parallel */ mutex_pages_lock(); - uintptr_t page_start_range = pagenum; - uintptr_t pagestop = pagenum + count; + /* "unmaps" the page to make the address space location correspond + again to its underlying file offset (XXX later we should again + attempt to group together many calls to d_remap_file_pages() in + succession) */ + uintptr_t pagenum_in_file = NB_PAGES * STM_SEGMENT->segment_num + pagenum; + char *new_page = stm_object_pages + pagenum_in_file * 4096UL; + d_remap_file_pages(new_page, 4096, pagenum_in_file); + increment_total_allocated(4096); - for (; pagenum < pagestop; pagenum++) { - uint8_t prev = flag_page_private[pagenum]; - if (prev == PRIVATE_PAGE) { - if (pagenum > page_start_range) { - privatize_range(page_start_range, - pagenum - page_start_range, full); - } - page_start_range = pagenum + 1; - } - else { - assert(prev == SHARED_PAGE); - } - } + /* copy the content from the shared (segment 0) source */ + pagecopy(new_page, stm_object_pages + pagenum * 4096UL); - if (pagenum > page_start_range) { - privatize_range(page_start_range, - pagenum - page_start_range, full); - } + /* add this thread's 'pages_privatized' bit */ + uint64_t bitmask = 1UL << (STM_SEGMENT->segment_num - 1); + pages_privatized[pagenum - PAGE_FLAG_START].by_segment |= bitmask; mutex_pages_unlock(); } +static void page_reshare(uintptr_t pagenum) +{ + struct page_shared_s ps = pages_privatized[pagenum - PAGE_FLAG_START]; + pages_privatized[pagenum - PAGE_FLAG_START].by_segment = 0; + + long j, total = 0; + for (j = 0; j < NB_SEGMENTS; j++) { + if (ps.by_segment & (1 << j)) { + /* Page 'pagenum' is private in segment 'j + 1'. Reshare */ + char *segment_base = get_segment_base(j + 1); + + madvise(segment_base + pagenum * 4096UL, 4096, MADV_DONTNEED); + d_remap_file_pages(segment_base + pagenum * 4096UL, + 4096, pagenum); + total -= 4096; + } + } + increment_total_allocated(total); +} + + #if 0 static bool is_fully_in_shared_pages(object_t *obj) { diff --git a/c7/stm/pages.h b/c7/stm/pages.h --- a/c7/stm/pages.h +++ b/c7/stm/pages.h @@ -1,48 +1,61 @@ -enum /* flag_page_private */ { - /* The page is not in use. Assume that each segment sees its own copy. */ - FREE_PAGE=0, +/* This handles pages of objects outside the nursery. Every page + has a "shared copy" and zero or more "private copies". - /* The page is shared by all segments. Each segment sees the same - physical page (the one that is within the segment 0 mmap address). */ - SHARED_PAGE, + The shared copy of a page is stored in the mmap at the file offset + corresponding to the segment 0 offset. Initially, accessing a page + from segment N remaps to segment 0. If the page is turned private, + then we "un-remap" it to its initial location. The 'pages_privatized' + global array records if a page is currently mapped to segment 0 + (shared page) or to its natural location (private page). - /* For only one range of pages at a time, around the call to - remap_file_pages() that un-shares the pages (SHARED -> PRIVATE). */ - REMAPPING_PAGE, + Note that this page manipulation logic uses remap_file_pages() to + fully hide its execution cost behind the CPU's memory management unit. + It should not be confused with the logic of tracking which objects + are old-and-committed, old-but-modified, overflow objects, and so on + (which works at the object granularity, not the page granularity). +*/ - /* Page is private for each segment. */ - PRIVATE_PAGE, +#define PAGE_FLAG_START END_NURSERY_PAGE +#define PAGE_FLAG_END NB_PAGES - /* gcpage.c: page contains objects that have been traced in the - segment > 0 */ - SEGMENT1_PAGE, +struct page_shared_s { +#if NB_SEGMENTS <= 8 + uint8_t by_segment; +#elif NB_SEGMENTS <= 16 + uint16_t by_segment; +#elif NB_SEGMENTS <= 32 + uint32_t by_segment; +#elif NB_SEGMENTS <= 64 + uint64_t by_segment; +#else +# error "NB_SEGMENTS > 64 not supported right now" +#endif }; -static uint8_t flag_page_private[NB_PAGES]; +static struct page_shared_s pages_privatized[PAGE_FLAG_END - PAGE_FLAG_START]; -static void _pages_privatize(uintptr_t pagenum, uintptr_t count, bool full); static void pages_initialize_shared(uintptr_t pagenum, uintptr_t count); -//static void pages_make_shared_again(uintptr_t pagenum, uintptr_t count); +static void page_privatize(uintptr_t pagenum); +static void page_reshare(uintptr_t pagenum); static void mutex_pages_lock(void); static void mutex_pages_unlock(void); +static bool _has_mutex_pages(void) __attribute__((unused)); static uint64_t increment_total_allocated(ssize_t add_or_remove); static bool is_major_collection_requested(void); static void force_major_collection_request(void); static void reset_major_collection_requested(void); -inline static void pages_privatize(uintptr_t pagenum, uintptr_t count, - bool full) { - /* This is written a bit carefully so that a call with a constant - count == 1 will turn this loop into just one "if". */ - while (flag_page_private[pagenum] == PRIVATE_PAGE) { - if (!--count) { - return; - } - pagenum++; - } - _pages_privatize(pagenum, count, full); +static inline bool is_private_page(long segnum, uintptr_t pagenum) +{ + assert(pagenum >= PAGE_FLAG_START); + uint64_t bitmask = 1UL << (segnum - 1); + return (pages_privatized[pagenum - PAGE_FLAG_START].by_segment & bitmask); } -/* static bool is_fully_in_shared_pages(object_t *obj); */ +static inline void page_check_and_reshare(uintptr_t pagenum) +{ + if (pages_privatized[pagenum - PAGE_FLAG_START].by_segment != 0) + page_reshare(pagenum); +} diff --git a/c7/stm/setup.c b/c7/stm/setup.c --- a/c7/stm/setup.c +++ b/c7/stm/setup.c @@ -26,12 +26,15 @@ if (stm_object_pages == MAP_FAILED) stm_fatalerror("initial stm_object_pages mmap() failed: %m\n"); + /* The segment 0 is not used to run transactions, but to contain the + shared copy of the pages. We mprotect all pages before so that + accesses fail, up to and including the pages corresponding to the + nurseries of the other segments. */ + mprotect(stm_object_pages, END_NURSERY_PAGE * 4096UL, PROT_NONE); + long i; - for (i = 0; i < NB_SEGMENTS; i++) { + for (i = 1; i <= NB_SEGMENTS; i++) { char *segment_base = get_segment_base(i); -#ifdef STM_TESTS - stm_other_pages = segment_base; -#endif /* In each segment, the first page is where TLPREFIX'ed NULL accesses land. We mprotect it so that accesses fail. */ @@ -39,7 +42,7 @@ /* Fill the TLS page (page 1) with 0xDC, for debugging */ memset(REAL_ADDRESS(segment_base, 4096), 0xDC, 4096); - /* Make a "hole" at STM_PSEGMENT */ + /* Make a "hole" at STM_PSEGMENT (which includes STM_SEGMENT) */ memset(REAL_ADDRESS(segment_base, STM_PSEGMENT), 0, sizeof(*STM_PSEGMENT)); @@ -49,9 +52,10 @@ (FIRST_READMARKER_PAGE - 2) * 4096UL, PROT_NONE); + /* Initialize STM_PSEGMENT */ struct stm_priv_segment_info_s *pr = get_priv_segment(i); - assert(i + 1 < 255); /* 255 is WL_VISITED in gcpage.c */ - pr->write_lock_num = i + 1; + assert(1 <= i && i < 255); /* 255 is WL_VISITED in gcpage.c */ + pr->write_lock_num = i; pr->pub.segment_num = i; pr->pub.segment_base = segment_base; pr->objects_pointing_to_nursery = NULL; @@ -62,7 +66,7 @@ pr->young_outside_nursery = tree_create(); pr->nursery_objects_shadows = tree_create(); pr->callbacks_on_abort = tree_create(); - pr->overflow_number = GCFLAG_OVERFLOW_NUMBER_bit0 * (i + 1); + pr->overflow_number = GCFLAG_OVERFLOW_NUMBER_bit0 * i; highest_overflow_number = pr->overflow_number; } @@ -73,10 +77,6 @@ STM_SEGMENT->transaction_read_version never contains zero, so a null read marker means "not read" whatever the current transaction_read_version is. - - The creation markers are initially zero, which is correct: - it means "objects of this line of 256 bytes have not been - allocated by the current transaction." */ setup_sync(); @@ -92,7 +92,7 @@ assert(!_has_mutex()); long i; - for (i = 0; i < NB_SEGMENTS; i++) { + for (i = 1; i <= NB_SEGMENTS; i++) { struct stm_priv_segment_info_s *pr = get_priv_segment(i); assert(pr->objects_pointing_to_nursery == NULL); assert(pr->large_overflow_objects == NULL); @@ -107,8 +107,6 @@ munmap(stm_object_pages, TOTAL_MEMORY); stm_object_pages = NULL; - memset(flag_page_private, 0, sizeof(flag_page_private)); - teardown_core(); teardown_sync(); teardown_gcpage(); @@ -146,14 +144,14 @@ tl->prev = stm_all_thread_locals->prev; stm_all_thread_locals->prev->next = tl; stm_all_thread_locals->prev = tl; - num = tl->prev->associated_segment_num + 1; + num = tl->prev->associated_segment_num; } tl->thread_local_obj = NULL; /* assign numbers consecutively, but that's for tests; we could also assign the same number to all of them and they would get their own numbers automatically. */ - num = num % NB_SEGMENTS; + num = (num % NB_SEGMENTS) + 1; tl->associated_segment_num = num; _init_shadow_stack(tl); set_gs_register(get_segment_base(num)); diff --git a/c7/stm/sync.c b/c7/stm/sync.c --- a/c7/stm/sync.c +++ b/c7/stm/sync.c @@ -30,7 +30,7 @@ pthread_mutex_t global_mutex; pthread_cond_t cond[_C_TOTAL]; /* some additional pieces of global state follow */ - uint8_t in_use[NB_SEGMENTS]; /* 1 if running a pthread */ + uint8_t in_use1[NB_SEGMENTS]; /* 1 if running a pthread */ uint64_t global_time; }; char reserved[192]; @@ -124,12 +124,12 @@ { long i; restart: - for (i = 0; i < NB_SEGMENTS; i++) { + for (i = 1; i <= NB_SEGMENTS; i++) { if (get_priv_segment(i)->transaction_state == TS_INEVITABLE) { if (can_abort) { /* handle this case like a contention: it will either abort us (not the other thread, which is inevitable), - or for a while. If we go past this call, then we + or wait for a while. If we go past this call, then we waited; in this case we have to re-check if no other thread is inevitable. */ inevitable_contention_management(i); @@ -152,7 +152,7 @@ assert(_is_tl_registered(tl)); int num = tl->associated_segment_num; - if (sync_ctl.in_use[num] == 0) { + if (sync_ctl.in_use1[num - 1] == 0) { /* fast-path: we can get the same segment number than the one we had before. The value stored in GS is still valid. */ #ifdef STM_TESTS @@ -165,10 +165,10 @@ } /* Look for the next free segment. If there is none, wait for the condition variable. */ - int i; - for (i = 0; i < NB_SEGMENTS; i++) { - num = (num + 1) % NB_SEGMENTS; - if (sync_ctl.in_use[num] == 0) { + int retries; + for (retries = 0; retries < NB_SEGMENTS; retries++) { + num = (num % NB_SEGMENTS) + 1; + if (sync_ctl.in_use1[num - 1] == 0) { /* we're getting 'num', a different number. */ dprintf(("acquired different segment: %d->%d\n", tl->associated_segment_num, num)); tl->associated_segment_num = num; @@ -184,7 +184,7 @@ return false; got_num: - sync_ctl.in_use[num] = 1; + sync_ctl.in_use1[num - 1] = 1; assert(STM_SEGMENT->segment_num == num); assert(STM_SEGMENT->running_thread == NULL); STM_SEGMENT->running_thread = tl; @@ -208,8 +208,8 @@ assert(STM_SEGMENT->running_thread == tl); STM_SEGMENT->running_thread = NULL; - assert(sync_ctl.in_use[tl->associated_segment_num] == 1); - sync_ctl.in_use[tl->associated_segment_num] = 0; + assert(sync_ctl.in_use1[tl->associated_segment_num - 1] == 1); + sync_ctl.in_use1[tl->associated_segment_num - 1] = 0; } __attribute__((unused)) @@ -221,7 +221,7 @@ bool _stm_in_transaction(stm_thread_local_t *tl) { int num = tl->associated_segment_num; - assert(num < NB_SEGMENTS); + assert(1 <= num && num <= NB_SEGMENTS); return get_segment(num)->running_thread == tl; } @@ -260,12 +260,15 @@ { assert(_safe_points_requested == false); assert((_safe_points_requested = true, 1)); + assert(_has_mutex()); long i; - for (i = 0; i < NB_SEGMENTS; i++) { + for (i = 1; i <= NB_SEGMENTS; i++) { if (get_segment(i)->nursery_end == NURSERY_END) get_segment(i)->nursery_end = NSE_SIGPAUSE; } + assert(!pause_signalled); + pause_signalled = true; } static inline long count_other_threads_sp_running(void) @@ -276,7 +279,7 @@ long result = 0; int my_num = STM_SEGMENT->segment_num; - for (i = 0; i < NB_SEGMENTS; i++) { + for (i = 1; i <= NB_SEGMENTS; i++) { if (i != my_num && get_priv_segment(i)->safe_point == SP_RUNNING) { assert(get_segment(i)->nursery_end <= _STM_NSE_SIGNAL_MAX); result++; @@ -287,11 +290,13 @@ static void remove_requests_for_safe_point(void) { + assert(pause_signalled); + pause_signalled = false; assert(_safe_points_requested == true); assert((_safe_points_requested = false, 1)); long i; - for (i = 0; i < NB_SEGMENTS; i++) { + for (i = 1; i <= NB_SEGMENTS; i++) { assert(get_segment(i)->nursery_end != NURSERY_END); if (get_segment(i)->nursery_end == NSE_SIGPAUSE) get_segment(i)->nursery_end = NURSERY_END; diff --git a/c7/stm/sync.h b/c7/stm/sync.h --- a/c7/stm/sync.h +++ b/c7/stm/sync.h @@ -30,3 +30,5 @@ static void wait_for_end_of_inevitable_transaction(bool can_abort); static void synchronize_all_threads(void); + +static bool pause_signalled; diff --git a/c7/stm/weakref.c b/c7/stm/weakref.c --- a/c7/stm/weakref.c +++ b/c7/stm/weakref.c @@ -32,23 +32,18 @@ ssize_t size = 16; stm_char *point_to_loc = (stm_char*)WEAKREF_PTR(weakref, size); - if (flag_page_private[(uintptr_t)point_to_loc / 4096UL] == PRIVATE_PAGE) { - long i; - for (i = 0; i < NB_SEGMENTS; i++) { - char *base = get_segment_base(i); /* two different segments */ - object_t ** ref_loc = (object_t **)REAL_ADDRESS(base, point_to_loc); - *ref_loc = value; - } - } - else { - *WEAKREF_PTR(weakref, size) = value; + long i; + for (i = 1; i <= NB_SEGMENTS; i++) { + char *base = get_segment_base(i); + object_t ** ref_loc = (object_t **)REAL_ADDRESS(base, point_to_loc); + *ref_loc = value; } } /***** Minor collection *****/ -static void stm_move_young_weakrefs() +static void stm_move_young_weakrefs(void) { /* The code relies on the fact that no weakref can be an old object weakly pointing to a young object. Indeed, weakrefs are immutable @@ -115,7 +110,7 @@ static void stm_visit_old_weakrefs(void) { long i; - for (i = 0; i < NB_SEGMENTS; i++) { + for (i = 1; i <= NB_SEGMENTS; i++) { struct stm_priv_segment_info_s *pseg = get_priv_segment(i); struct list_s *lst; diff --git a/c7/stmgc.h b/c7/stmgc.h --- a/c7/stmgc.h +++ b/c7/stmgc.h @@ -87,7 +87,7 @@ #include bool _stm_was_read(object_t *obj); bool _stm_was_written(object_t *obj); -uint8_t _stm_get_page_flag(uintptr_t index); +uintptr_t _stm_get_private_page(uintptr_t pagenum); bool _stm_in_transaction(stm_thread_local_t *tl); char *_stm_get_segment_base(long index); void _stm_test_switch(stm_thread_local_t *tl); @@ -107,6 +107,8 @@ object_t *_stm_enum_modified_old_objects(long index); object_t *_stm_enum_objects_pointing_to_nursery(long index); uint64_t _stm_total_allocated(void); +void _stm_mutex_pages_lock(void); +void _stm_mutex_pages_unlock(void); #endif #define _STM_GCFLAG_WRITE_BARRIER 0x01 @@ -127,6 +129,14 @@ /* ==================== PUBLIC API ==================== */ +/* Number of segments (i.e. how many transactions can be executed in + parallel, in maximum). If you try to start transactions in more + threads than the number of segments, it will block, waiting for the + next segment to become free. +*/ +#define STM_NB_SEGMENTS 4 + + /* Structure of objects -------------------- diff --git a/c7/test/support.py b/c7/test/support.py --- a/c7/test/support.py +++ b/c7/test/support.py @@ -9,6 +9,7 @@ typedef ... object_t; typedef ... stm_jmpbuf_t; #define SIZEOF_MYOBJ ... +#define STM_NB_SEGMENTS ... #define _STM_FAST_ALLOC ... #define _STM_GCFLAG_WRITE_BARRIER ... @@ -46,7 +47,7 @@ char *_stm_get_segment_base(long index); bool _stm_in_transaction(stm_thread_local_t *tl); void _stm_test_switch(stm_thread_local_t *tl); -uint8_t _stm_get_page_flag(uintptr_t index); +uintptr_t _stm_get_private_page(uintptr_t pagenum); int _stm_get_flags(object_t *obj); void _stm_start_transaction(stm_thread_local_t *tl, stm_jmpbuf_t *jmpbuf); @@ -87,6 +88,8 @@ void stm_collect(long level); uint64_t _stm_total_allocated(void); +void _stm_mutex_pages_lock(void); +void _stm_mutex_pages_unlock(void); long stm_identityhash(object_t *obj); long stm_id(object_t *obj); @@ -98,8 +101,6 @@ GC_N_SMALL_REQUESTS = 36 # from gcpage.c -SHARED_PAGE = 1 # from pages.h -PRIVATE_PAGE = 3 # from pages.h LARGE_MALLOC_OVERHEAD = 16 # from largemalloc.h lib = ffi.verify(''' @@ -262,6 +263,7 @@ HDR = lib.SIZEOF_MYOBJ assert HDR == 8 GCFLAG_WRITE_BARRIER = lib._STM_GCFLAG_WRITE_BARRIER +NB_SEGMENTS = lib.STM_NB_SEGMENTS class Conflict(Exception): @@ -361,8 +363,8 @@ def stm_major_collect(): lib.stm_collect(1) -def stm_get_page_flag(pagenum): - return lib._stm_get_page_flag(pagenum) +def stm_get_private_page(pagenum): + return lib._stm_get_private_page(pagenum) def stm_get_obj_size(o): return lib.stmcb_size_rounded_up(stm_get_real_address(o)) @@ -402,10 +404,11 @@ class BaseTest(object): + NB_THREADS = 2 def setup_method(self, meth): lib.stm_setup() - self.tls = [_allocate_thread_local(), _allocate_thread_local()] + self.tls = [_allocate_thread_local() for i in range(self.NB_THREADS)] self.current_thread = 0 def teardown_method(self, meth): diff --git a/c7/test/test_gcpage.py b/c7/test/test_gcpage.py --- a/c7/test/test_gcpage.py +++ b/c7/test/test_gcpage.py @@ -24,8 +24,8 @@ new = self.pop_root() assert len(stm_get_obj_pages(new)) == 2 - assert ([stm_get_page_flag(p) for p in stm_get_obj_pages(new)] - == [SHARED_PAGE]*2) + assert ([stm_get_private_page(p) for p in stm_get_obj_pages(new)] + == [0, 0]) assert not is_in_nursery(new) stm_write(new) @@ -33,11 +33,11 @@ # now proceed to write into the object in a new transaction self.start_transaction() - assert ([stm_get_page_flag(p) for p in stm_get_obj_pages(new)] - == [SHARED_PAGE]*2) + assert ([stm_get_private_page(p) for p in stm_get_obj_pages(new)] + == [0, 0]) stm_write(new) - assert ([stm_get_page_flag(p) for p in stm_get_obj_pages(new)] - == [PRIVATE_PAGE]*2) + assert ([bool(stm_get_private_page(p)) for p in stm_get_obj_pages(new)] + == [True, True]) # write to 2nd page of object!! wnew = stm_get_real_address(new) @@ -52,8 +52,8 @@ self.switch(0) self.abort_transaction() - assert ([stm_get_page_flag(p) for p in stm_get_obj_pages(new)] - == [PRIVATE_PAGE]*2) + assert ([bool(stm_get_private_page(p)) for p in stm_get_obj_pages(new)] + == [True, True]) def test_partial_alloced_pages(self): self.start_transaction() @@ -62,14 +62,14 @@ stm_minor_collect() new = self.pop_root() - assert stm_get_page_flag(stm_get_obj_pages(new)[0]) == SHARED_PAGE + assert stm_get_private_page(stm_get_obj_pages(new)[0]) == 0 assert stm_get_flags(new) & GCFLAG_WRITE_BARRIER stm_write(new) assert not (stm_get_flags(new) & GCFLAG_WRITE_BARRIER) self.commit_transaction() - assert stm_get_page_flag(stm_get_obj_pages(new)[0]) == SHARED_PAGE + assert stm_get_private_page(stm_get_obj_pages(new)[0]) == 0 assert stm_get_flags(new) & GCFLAG_WRITE_BARRIER self.start_transaction() @@ -78,7 +78,7 @@ stm_minor_collect() newer = self.pop_root() # 'new' is still in shared_page and committed - assert stm_get_page_flag(stm_get_obj_pages(new)[0]) == SHARED_PAGE + assert stm_get_private_page(stm_get_obj_pages(new)[0]) == 0 assert stm_get_flags(new) & GCFLAG_WRITE_BARRIER # 'newer' is now part of the SHARED page with 'new', but # uncommitted, so no privatization has to take place: @@ -86,10 +86,10 @@ assert stm_get_flags(newer) & GCFLAG_WRITE_BARRIER stm_write(newer) # does not privatize assert not (stm_get_flags(newer) & GCFLAG_WRITE_BARRIER) - assert stm_get_page_flag(stm_get_obj_pages(newer)[0]) == SHARED_PAGE + assert stm_get_private_page(stm_get_obj_pages(newer)[0]) == 0 self.commit_transaction() - assert stm_get_page_flag(stm_get_obj_pages(newer)[0]) == SHARED_PAGE + assert stm_get_private_page(stm_get_obj_pages(newer)[0]) == 0 assert stm_get_flags(newer) & GCFLAG_WRITE_BARRIER def test_major_collection(self): @@ -202,7 +202,6 @@ # self.start_transaction() stm_major_collect() - py.test.skip("XXX implement me") assert lib._stm_total_allocated() == 5000 + LMO # shared again def test_reshare_if_no_longer_modified_1(self): diff --git a/c7/test/test_largemalloc.py b/c7/test/test_largemalloc.py --- a/c7/test/test_largemalloc.py +++ b/c7/test/test_largemalloc.py @@ -14,6 +14,7 @@ lib.memset(self.rawmem, 0xcd, self.size) lib._stm_largemalloc_init_arena(self.rawmem, self.size) + lib._stm_mutex_pages_lock() # for this file def test_simple(self): d1 = lib._stm_large_malloc(7000) diff --git a/c7/test/test_random.py b/c7/test/test_random.py --- a/c7/test/test_random.py +++ b/c7/test/test_random.py @@ -54,7 +54,7 @@ e.g. maintains read/write sets. The state will be discarded on abort or pushed to other threads""" - def __init__(self, start_time): + def __init__(self, start_time, thread_num=None): self.read_set = set() self.write_set = set() self.values = {} @@ -63,6 +63,7 @@ self.objs_in_conflict = set() self.inevitable = False self.created_in_this_transaction = set() + self.thread_num = thread_num def get_old_modified(self): # returns only the ones that are modified and not from @@ -74,6 +75,8 @@ if objs_in_conflict is not None: self.objs_in_conflict |= objs_in_conflict self._must_abort = True + color = "\033[%dm" % (31 + self.thread_num % 6) + print >> sys.stderr, color + "# must abort: %r\033[0m" % (objs_in_conflict,) def check_must_abort(self): return self._must_abort @@ -180,10 +183,10 @@ r, int(ffi.cast("uintptr_t", ex.content[r])), stm_get_obj_size(ex.content[r]))) - def start_transaction(self): + def start_transaction(self, thread_num): assert self.transaction_state is None start_time = self.global_state.inc_and_get_global_time() - trs = TransactionState(start_time) + trs = TransactionState(start_time, thread_num) trs.update_from_committed( self.global_state.committed_transaction_state) self.transaction_state = trs @@ -291,6 +294,8 @@ if confl_set: contention_management(trs, other_trs, objs_in_conflict=confl_set) + if trs.check_must_abort(): + break if trs.check_must_abort(): self.ex.do('# write-read conflict: %s' % @@ -305,7 +310,7 @@ def op_start_transaction(ex, global_state, thread_state): - thread_state.start_transaction() + thread_state.start_transaction(ex.thread_num) # ex.do('self.start_transaction()') thread_state.reload_roots(ex) @@ -533,12 +538,13 @@ class TestRandom(BaseTest): + NB_THREADS = NB_SEGMENTS def test_fixed_16_bytes_objects(self, seed=1010): rnd = random.Random(seed) N_OBJECTS = 3 - N_THREADS = 2 + N_THREADS = self.NB_THREADS ex = Exec(self) ex.do("################################################################\n"*10) ex.do('# initialization') diff --git a/c7/test/test_weakref.py b/c7/test/test_weakref.py --- a/c7/test/test_weakref.py +++ b/c7/test/test_weakref.py @@ -279,7 +279,7 @@ stm_write(lp0) # privatize page with weakref in it too - assert stm_get_page_flag(stm_get_obj_pages(lp1)[0]) == PRIVATE_PAGE + assert stm_get_private_page(stm_get_obj_pages(lp1)[0]) != 0 assert stm_get_weakref(lp1) == lp0 self.commit_transaction() diff --git a/duhton/duhton.c b/duhton/duhton.c --- a/duhton/duhton.c +++ b/duhton/duhton.c @@ -7,7 +7,7 @@ char *filename = NULL; int interactive = 1; int i; - int num_threads = DEFAULT_NUM_THREADS; + int num_threads = STM_NB_SEGMENTS; for (i = 1; i < argc; ++i) { if (strcmp(argv[i], "--help") == 0) { diff --git a/duhton/duhton.h b/duhton/duhton.h --- a/duhton/duhton.h +++ b/duhton/duhton.h @@ -13,8 +13,6 @@ #endif -#define DEFAULT_NUM_THREADS 2 - extern __thread stm_thread_local_t stm_thread_local; struct DuObject_s { From noreply at buildbot.pypy.org Sun Mar 16 19:47:52 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 16 Mar 2014 19:47:52 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: import stmgc/885ed3b0f6ee Message-ID: <20140316184752.C2C931C011F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r69979:c561ed442ef2 Date: 2014-03-16 18:04 +0100 http://bitbucket.org/pypy/pypy/changeset/c561ed442ef2/ Log: import stmgc/885ed3b0f6ee diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision --- a/rpython/translator/stm/src_stm/revision +++ b/rpython/translator/stm/src_stm/revision @@ -1,1 +1,1 @@ -ddbc16971682 +885ed3b0f6ee diff --git a/rpython/translator/stm/src_stm/stm/contention.c b/rpython/translator/stm/src_stm/stm/contention.c --- a/rpython/translator/stm/src_stm/stm/contention.c +++ b/rpython/translator/stm/src_stm/stm/contention.c @@ -236,7 +236,7 @@ uint8_t prev_owner = ((volatile uint8_t *)write_locks)[lock_idx]; if (prev_owner != 0 && prev_owner != STM_PSEGMENT->write_lock_num) { - uint8_t other_segment_num = prev_owner - 1; + uint8_t other_segment_num = prev_owner; assert(get_priv_segment(other_segment_num)->write_lock_num == prev_owner); diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c --- a/rpython/translator/stm/src_stm/stm/core.c +++ b/rpython/translator/stm/src_stm/stm/core.c @@ -9,11 +9,34 @@ memset(write_locks, 0, sizeof(write_locks)); } +static void check_flag_write_barrier(object_t *obj) +{ + /* check that all copies of the object, apart from mine, have the + GCFLAG_WRITE_BARRIER. (a bit messy because it's possible that we + read a page in the middle of privatization by another thread) + */ +#ifndef NDEBUG + long i; + struct object_s *o1; + for (i = 0; i <= NB_SEGMENTS; i++) { + if (i == STM_SEGMENT->segment_num) + continue; + o1 = (struct object_s *)REAL_ADDRESS(get_segment_base(i), obj); + if (!(o1->stm_flags & GCFLAG_WRITE_BARRIER)) { + mutex_pages_lock(); /* try again... */ + if (!(o1->stm_flags & GCFLAG_WRITE_BARRIER)) + stm_fatalerror("missing GCFLAG_WRITE_BARRIER"); + mutex_pages_unlock(); + } + } +#endif +} void _stm_write_slowpath(object_t *obj) { assert(_seems_to_be_running_transaction()); assert(!_is_young(obj)); + assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); /* is this an object from the same transaction, outside the nursery? */ if ((obj->stm_flags & -GCFLAG_OVERFLOW_NUMBER_bit0) == @@ -30,6 +53,8 @@ safepoints that may be issued in write_write_contention_management(). */ stm_read(obj); + /* XXX XXX XXX make the logic of write-locking objects optional! */ + /* claim the write-lock for this object. In case we're running the same transaction since a long while, the object can be already in 'modified_old_objects' (but, because it had GCFLAG_WRITE_BARRIER, @@ -60,12 +85,12 @@ the common case. Otherwise, we need to compute it based on its location and size. */ if ((obj->stm_flags & GCFLAG_SMALL_UNIFORM) != 0) { - pages_privatize(first_page, 1, true); + page_privatize(first_page); } else { char *realobj; size_t obj_size; - uintptr_t end_page; + uintptr_t i, end_page; /* get the size of the object */ realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj); @@ -74,7 +99,9 @@ /* that's the page *following* the last page with the object */ end_page = (((uintptr_t)obj) + obj_size + 4095) / 4096UL; - pages_privatize(first_page, end_page - first_page, true); + for (i = first_page; i < end_page; i++) { + page_privatize(i); + } } } else if (write_locks[lock_idx] == lock_num) { @@ -101,19 +128,21 @@ LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, obj); } - /* add the write-barrier-already-called flag ONLY if we succeeded in + /* check that we really have a private page */ + assert(is_private_page(STM_SEGMENT->segment_num, + ((uintptr_t)obj) / 4096)); + + /* check that so far all copies of the object have the flag */ + check_flag_write_barrier(obj); + + /* remove GCFLAG_WRITE_BARRIER, but only if we succeeded in getting the write-lock */ assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); obj->stm_flags &= ~GCFLAG_WRITE_BARRIER; - /* for sanity, check that all other segment copies of this object - still have the flag */ - long i; - for (i = 0; i < NB_SEGMENTS; i++) { - if (i != STM_SEGMENT->segment_num) - assert(((struct object_s *)REAL_ADDRESS(get_segment_base(i), obj)) - ->stm_flags & GCFLAG_WRITE_BARRIER); - } + /* for sanity, check again that all other segment copies of this + object still have the flag (so privatization worked) */ + check_flag_write_barrier(obj); } static void reset_transaction_read_version(void) @@ -194,59 +223,130 @@ /************************************************************/ -#if NB_SEGMENTS != 2 -# error "The logic in the functions below only works with two segments" -#endif static bool detect_write_read_conflicts(void) { - long remote_num = 1 - STM_SEGMENT->segment_num; - char *remote_base = get_segment_base(remote_num); - uint8_t remote_version = get_segment(remote_num)->transaction_read_version; + /* Detect conflicts of the form: we want to commit a write to an object, + but the same object was also read in a different thread. + */ + long i; + for (i = 1; i <= NB_SEGMENTS; i++) { - if (get_priv_segment(remote_num)->transaction_state == TS_NONE) - return false; /* no need to check */ + if (i == STM_SEGMENT->segment_num) + continue; - if (is_aborting_now(remote_num)) - return false; /* no need to check: is pending immediate abort */ + if (get_priv_segment(i)->transaction_state == TS_NONE) + continue; /* no need to check */ - LIST_FOREACH_R( - STM_PSEGMENT->modified_old_objects, - object_t * /*item*/, - ({ - if (was_read_remote(remote_base, item, remote_version)) { - /* A write-read conflict! */ - write_read_contention_management(remote_num); + if (is_aborting_now(i)) + continue; /* no need to check: is pending immediate abort */ - /* If we reach this point, we didn't abort, but maybe we - had to wait for the other thread to commit. If we - did, then we have to restart committing from our call - to synchronize_all_threads(). */ - return true; - } - })); + char *remote_base = get_segment_base(i); + uint8_t remote_version = get_segment(i)->transaction_read_version; + + LIST_FOREACH_R( + STM_PSEGMENT->modified_old_objects, + object_t * /*item*/, + ({ + if (was_read_remote(remote_base, item, remote_version)) { + /* A write-read conflict! */ + write_read_contention_management(i); + + /* If we reach this point, we didn't abort, but maybe we + had to wait for the other thread to commit. If we + did, then we have to restart committing from our call + to synchronize_all_threads(). */ + return true; + } + })); + } return false; } -static void synchronize_overflow_object_now(object_t *obj) +static void copy_object_to_shared(object_t *obj, int source_segment_num) { + /* Only used by major GC. XXX There is a lot of code duplication + with synchronize_object_now() but I don't completely see how to + improve... + */ + assert(_has_mutex_pages()); assert(!_is_young(obj)); - assert((obj->stm_flags & GCFLAG_SMALL_UNIFORM) == 0); + + char *segment_base = get_segment(source_segment_num)->segment_base; + uintptr_t start = (uintptr_t)obj; + uintptr_t first_page = start / 4096UL; + struct object_s *realobj = (struct object_s *) + REAL_ADDRESS(segment_base, obj); + + if (realobj->stm_flags & GCFLAG_SMALL_UNIFORM) { + abort();//XXX WRITE THE FAST CASE + } + else { + ssize_t obj_size = stmcb_size_rounded_up(realobj); + assert(obj_size >= 16); + uintptr_t end = start + obj_size; + uintptr_t last_page = (end - 1) / 4096UL; + + for (; first_page <= last_page; first_page++) { + + /* Copy the object into the shared page, if needed */ + if (is_private_page(source_segment_num, first_page)) { + + uintptr_t copy_size; + if (first_page == last_page) { + /* this is the final fragment */ + copy_size = end - start; + } + else { + /* this is a non-final fragment, going up to the + page's end */ + copy_size = 4096 - (start & 4095); + } + /* double-check that the result fits in one page */ + assert(copy_size > 0); + assert(copy_size + (start & 4095) <= 4096); + + char *src = REAL_ADDRESS(segment_base, start); + char *dst = REAL_ADDRESS(stm_object_pages, start); + if (copy_size == 4096) + pagecopy(dst, src); + else + memcpy(dst, src, copy_size); + } + + start = (start + 4096) & ~4095; + } + } +} + +static void synchronize_object_now(object_t *obj) +{ + /* Copy around the version of 'obj' that lives in our own segment. + It is first copied into the shared pages, and then into other + segments' own private pages. + + This must be called with the mutex_pages_lock! + */ + assert(_has_mutex_pages()); + assert(!_is_young(obj)); assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); - char *realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj); - ssize_t obj_size = stmcb_size_rounded_up((struct object_s *)realobj); - assert(obj_size >= 16); uintptr_t start = (uintptr_t)obj; - uintptr_t end = start + obj_size; uintptr_t first_page = start / 4096UL; - uintptr_t last_page = (end - 1) / 4096UL; - do { - if (flag_page_private[first_page] != SHARED_PAGE) { - /* The page is a PRIVATE_PAGE. We need to diffuse this fragment - of our object from our own segment to all other segments. */ + if (obj->stm_flags & GCFLAG_SMALL_UNIFORM) { + abort();//XXX WRITE THE FAST CASE + } + else { + char *realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj); + ssize_t obj_size = stmcb_size_rounded_up((struct object_s *)realobj); + assert(obj_size >= 16); + uintptr_t end = start + obj_size; + uintptr_t last_page = (end - 1) / 4096UL; + long i, myself = STM_SEGMENT->segment_num; + + for (; first_page <= last_page; first_page++) { uintptr_t copy_size; if (first_page == last_page) { @@ -254,26 +354,50 @@ copy_size = end - start; } else { - /* this is a non-final fragment, going up to the page's end */ + /* this is a non-final fragment, going up to the + page's end */ copy_size = 4096 - (start & 4095); } - /* double-check that the result fits in one page */ assert(copy_size > 0); assert(copy_size + (start & 4095) <= 4096); - long i; + /* First copy the object into the shared page, if needed */ char *src = REAL_ADDRESS(STM_SEGMENT->segment_base, start); - for (i = 0; i < NB_SEGMENTS; i++) { - if (i != STM_SEGMENT->segment_num) { - char *dst = REAL_ADDRESS(get_segment_base(i), start); + char *dst = REAL_ADDRESS(stm_object_pages, start); + if (is_private_page(myself, first_page)) { + if (copy_size == 4096) + pagecopy(dst, src); + else memcpy(dst, src, copy_size); + } + else { + assert(memcmp(dst, src, copy_size) == 0); /* same page */ + } + + for (i = 1; i <= NB_SEGMENTS; i++) { + if (i == myself) + continue; + + src = REAL_ADDRESS(stm_object_pages, start); + dst = REAL_ADDRESS(get_segment_base(i), start); + if (is_private_page(i, first_page)) { + /* The page is a private page. We need to diffuse this + fragment of object from the shared page to this private + page. */ + if (copy_size == 4096) + pagecopy(dst, src); + else + memcpy(dst, src, copy_size); + } + else { + assert(memcmp(dst, src, copy_size) == 0); /* same page */ } } + + start = (start + 4096) & ~4095; } - - start = (start + 4096) & ~4095; - } while (first_page++ < last_page); + } } static void push_overflow_objects_from_privatized_pages(void) @@ -282,27 +406,15 @@ return; LIST_FOREACH_R(STM_PSEGMENT->large_overflow_objects, object_t *, - synchronize_overflow_object_now(item)); + synchronize_object_now(item)); } static void push_modified_to_other_segments(void) { - long remote_num = 1 - STM_SEGMENT->segment_num; - char *local_base = STM_SEGMENT->segment_base; - char *remote_base = get_segment_base(remote_num); - bool remote_active = - (get_priv_segment(remote_num)->transaction_state != TS_NONE && - get_segment(remote_num)->nursery_end != NSE_SIGABORT); - LIST_FOREACH_R( STM_PSEGMENT->modified_old_objects, object_t * /*item*/, ({ - if (remote_active) { - assert(!was_read_remote(remote_base, item, - get_segment(remote_num)->transaction_read_version)); - } - /* clear the write-lock (note that this runs with all other threads paused, so no need to be careful about ordering) */ uintptr_t lock_idx = (((uintptr_t)item) >> 4) - WRITELOCK_START; @@ -314,11 +426,9 @@ minor_collection() */ assert((item->stm_flags & GCFLAG_WRITE_BARRIER) != 0); - /* copy the modified object to the other segment */ - char *src = REAL_ADDRESS(local_base, item); - char *dst = REAL_ADDRESS(remote_base, item); - ssize_t size = stmcb_size_rounded_up((struct object_s *)src); - memcpy(dst, src, size); + /* copy the object to the shared page, and to the other + private pages as needed */ + synchronize_object_now(item); })); list_clear(STM_PSEGMENT->modified_old_objects); @@ -369,10 +479,12 @@ major_collection_now_at_safe_point(); /* synchronize overflow objects living in privatized pages */ + mutex_pages_lock(); push_overflow_objects_from_privatized_pages(); /* synchronize modified old objects to other threads */ push_modified_to_other_segments(); + mutex_pages_unlock(); /* update 'overflow_number' if needed */ if (STM_PSEGMENT->overflow_number_has_been_used) { @@ -490,6 +602,7 @@ static void abort_with_mutex(void) { + assert(_has_mutex()); dprintf(("~~~ ABORT\n")); switch (STM_PSEGMENT->transaction_state) { @@ -515,8 +628,11 @@ /* invoke the callbacks */ invoke_and_clear_callbacks_on_abort(); - if (STM_SEGMENT->nursery_end == NSE_SIGABORT) - STM_SEGMENT->nursery_end = NURSERY_END; /* done aborting */ + if (STM_SEGMENT->nursery_end == NSE_SIGABORT) { + /* done aborting */ + STM_SEGMENT->nursery_end = pause_signalled ? NSE_SIGPAUSE + : NURSERY_END; + } _finish_transaction(); /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */ diff --git a/rpython/translator/stm/src_stm/stm/core.h b/rpython/translator/stm/src_stm/stm/core.h --- a/rpython/translator/stm/src_stm/stm/core.h +++ b/rpython/translator/stm/src_stm/stm/core.h @@ -16,12 +16,12 @@ #define NB_PAGES (1500*256) // 1500MB -#define NB_SEGMENTS 2 +#define NB_SEGMENTS STM_NB_SEGMENTS #define NB_SEGMENTS_MAX 240 /* don't increase NB_SEGMENTS past this */ #define MAP_PAGES_FLAGS (MAP_SHARED | MAP_ANONYMOUS | MAP_NORESERVE) #define NB_NURSERY_PAGES (STM_GC_NURSERY/4) -#define TOTAL_MEMORY (NB_PAGES * 4096UL * NB_SEGMENTS) +#define TOTAL_MEMORY (NB_PAGES * 4096UL * (1 + NB_SEGMENTS)) #define READMARKER_END ((NB_PAGES * 4096UL) >> 4) #define FIRST_OBJECT_PAGE ((READMARKER_END + 4095) / 4096UL) #define FIRST_NURSERY_PAGE FIRST_OBJECT_PAGE @@ -179,10 +179,6 @@ static char *stm_object_pages; static stm_thread_local_t *stm_all_thread_locals = NULL; -#ifdef STM_TESTS -static char *stm_other_pages; -#endif - static uint8_t write_locks[WRITELOCK_END - WRITELOCK_START]; @@ -229,4 +225,5 @@ asm("/* workaround for llvm bug */"); } -static void synchronize_overflow_object_now(object_t *obj); +static void copy_object_to_shared(object_t *obj, int source_segment_num); +static void synchronize_object_now(object_t *obj); diff --git a/rpython/translator/stm/src_stm/stm/fprintcolor.h b/rpython/translator/stm/src_stm/stm/fprintcolor.h --- a/rpython/translator/stm/src_stm/stm/fprintcolor.h +++ b/rpython/translator/stm/src_stm/stm/fprintcolor.h @@ -10,7 +10,7 @@ #define dprintf(args) threadcolor_printf args static inline int dprintfcolor(void) { - return 31 + STM_SEGMENT->segment_num % 6; + return 31 + (STM_SEGMENT->segment_num + 5) % 6; } static int threadcolor_printf(const char *format, ...) diff --git a/rpython/translator/stm/src_stm/stm/gcpage.c b/rpython/translator/stm/src_stm/stm/gcpage.c --- a/rpython/translator/stm/src_stm/stm/gcpage.c +++ b/rpython/translator/stm/src_stm/stm/gcpage.c @@ -193,129 +193,121 @@ /************************************************************/ +static uintptr_t object_last_page(object_t *obj) +{ + uintptr_t lastbyte; + struct object_s *realobj = + (struct object_s *)REAL_ADDRESS(stm_object_pages, obj); -static inline void mark_single_flag_private(uintptr_t pagenum) -{ - if (flag_page_private[pagenum] == PRIVATE_PAGE) { - assert(pagenum >= END_NURSERY_PAGE); - assert(pagenum < NB_PAGES); - flag_page_private[pagenum] = SEGMENT1_PAGE; + if (realobj->stm_flags & GCFLAG_SMALL_UNIFORM) { + lastbyte = (uintptr_t)obj; } else { - assert(flag_page_private[pagenum] == SHARED_PAGE || - flag_page_private[pagenum] == SEGMENT1_PAGE); + /* get the size of the object */ + size_t obj_size = stmcb_size_rounded_up(realobj); + + /* that's the last byte within the object */ + lastbyte = ((uintptr_t)obj) + obj_size - 1; } + return lastbyte / 4096UL; } -static inline void mark_flag_page_private(object_t *obj, char *segment_base) +/* A macro that expands to: run the 'expression' for every page that + touches objects in the 'modified_old_objects' list. +*/ +#define BITOP(expression) \ + LIST_FOREACH_R( \ + get_priv_segment(segment_num)->modified_old_objects, \ + object_t * /* item */, \ + ({ \ + struct page_shared_s *ps; \ + uintptr_t pagenum = ((uintptr_t)item) / 4096UL; \ + uintptr_t count = object_last_page(item) - pagenum; \ + ps = &pages_privatized[pagenum - PAGE_FLAG_START]; \ + do { \ + expression; \ + ps++; \ + } while (count--); \ + })); + +static void major_hide_private_bits_for_modified_objects(long segment_num) { - uintptr_t first_page = ((uintptr_t)obj) / 4096UL; - - if (LIKELY((obj->stm_flags & GCFLAG_SMALL_UNIFORM) != 0)) { - mark_single_flag_private(first_page); - } - else { - char *realobj; - size_t obj_size; - uintptr_t end_page; - - /* get the size of the object */ - realobj = REAL_ADDRESS(segment_base, obj); - obj_size = stmcb_size_rounded_up((struct object_s *)realobj); - - /* that's the page *following* the last page with the object */ - end_page = (((uintptr_t)obj) + obj_size + 4095) / 4096UL; - - while (first_page < end_page) - mark_single_flag_private(first_page++); - } + uint64_t negativebitmask = ~(1 << (segment_num - 1)); +#ifndef NDEBUG + BITOP(assert((ps->by_segment & negativebitmask) != ps->by_segment)); +#endif + BITOP(ps->by_segment &= negativebitmask); } -static void major_reshare_pages_range(uintptr_t first_page, uintptr_t end_page) +static void major_restore_private_bits_for_modified_objects(long segment_num) { - uintptr_t i; - for (i = first_page; i < end_page; i++) { + uint64_t positivebitmask = 1 << (segment_num - 1); + BITOP(ps->by_segment |= positivebitmask); +} - switch (flag_page_private[i]) { - - case SEGMENT1_PAGE: - /* this page stays private after major collection */ - flag_page_private[i] = PRIVATE_PAGE; - break; - - case PRIVATE_PAGE:; - /* this page becomes shared again. No object in it was - traced belonging to a segment other than 0. - - XXX This is maybe a too-strict condition, but the more - general condition "all traced objects belong to the same - segment" has problems with large objects in segments > 0. - More precisely: we'd need to keep in the shared page the - content of the objects (from segment > 0), but also the - largemalloc's chunk data (stored in segment 0). - */ -#if NB_SEGMENTS != 2 -# error "limited to NB_SEGMENTS == 2" -#endif - char *ppage0 = get_segment_base(0) + i * 4096; - char *ppage1 = get_segment_base(1) + i * 4096; - - /* two cases for mapping pages to file-pages (fpages): - - (0->0, 1->1) - - (0->1, 1->0) - Distinguish which case it is by hacking a lot */ - - // 0->0,1->1 or 0->1,1->0 - /* map page 1 to fpage 0: */ - d_remap_file_pages(ppage1, 4096, i); - // 0->0,1->0 or 0->1,1->0 - - char oldvalue0 = *ppage0; - char oldvalue1 = *ppage1; - asm("":::"memory"); - *ppage0 = 1 + oldvalue1; - asm("":::"memory"); - char newvalue1 = *ppage1; - asm("":::"memory"); - *ppage0 = oldvalue0; - /* if we are in 0->0,1->0, old and new are different: - In this case we are done. We keep the largemalloc - data structure and objects of ppage0/fpage0 */ - if (oldvalue1 == newvalue1) { - // 0->1,1->0 - /* ppage0/fpage1 has the data structure that we want - in ppage1/fpage0, so we copy it */ - pagecopy(ppage1, ppage0); // copy from page0 to page1, - // i.e. from the underlying memory seg1 to seg0 - d_remap_file_pages(ppage0, 4096, i); - // 0->0,1->0 - } - flag_page_private[i] = SHARED_PAGE; - - increment_total_allocated(-4096 * (NB_SEGMENTS-1)); - break; - - case SHARED_PAGE: - break; /* stay shared */ - - default: - assert(!"unexpected flag_page_private"); - } - } -} +#undef BITOP static void major_reshare_pages(void) { /* re-share pages if possible. Each re-sharing decreases total_allocated by 4096. */ - major_reshare_pages_range( - END_NURSERY_PAGE, /* not the nursery! */ - (uninitialized_page_start - stm_object_pages) / 4096UL); - major_reshare_pages_range( - (uninitialized_page_stop - stm_object_pages) / 4096UL, - NB_PAGES); + + long i; + mutex_pages_lock(); + + for (i = 1; i <= NB_SEGMENTS; i++) { + /* The 'modified_old_objects' list gives the list of objects + whose pages need to remain private. We temporarily remove + these bits from 'pages_privatized', so that these pages will + be skipped by the loop below (and by copy_object_to_shared()). + */ + major_hide_private_bits_for_modified_objects(i); + + /* For each segment, push the current overflow objects from + private pages to the corresponding shared pages, if + necessary. The pages that we will re-share must contain this + data; otherwise, it would exist only in the private pages, + and get lost in the loop below. + */ + struct list_s *lst = get_priv_segment(i)->large_overflow_objects; + if (lst != NULL) { + LIST_FOREACH_R(lst, object_t *, copy_object_to_shared(item, i)); + } + } + + /* Now loop over all pages that are still in 'pages_privatized', + and re-share them. + */ + uintptr_t pagenum, endpagenum; + pagenum = END_NURSERY_PAGE; /* starts after the nursery */ + endpagenum = (uninitialized_page_start - stm_object_pages) / 4096UL; + + while (1) { + if (UNLIKELY(pagenum == endpagenum)) { + /* we reach this point usually twice, because there are + more pages after 'uninitialized_page_stop' */ + if (endpagenum == NB_PAGES) + break; /* done */ + pagenum = (uninitialized_page_stop - stm_object_pages) / 4096UL; + endpagenum = NB_PAGES; + if (pagenum == endpagenum) + break; /* no pages in the 2nd section, so done too */ + } + + page_check_and_reshare(pagenum); + pagenum++; + } + + /* Done. Now 'pages_privatized' should be entirely zeroes. Restore + the previously-hidden bits + */ + for (i = 1; i <= NB_SEGMENTS; i++) { + major_restore_private_bits_for_modified_objects(i); + } + mutex_pages_unlock(); } + /************************************************************/ @@ -324,11 +316,6 @@ /* takes a normal pointer to a thread-local pointer to an object */ object_t *obj = *pobj; - if (obj == NULL || mark_visited_test_and_set(obj)) - return; /* already visited this object */ - - LIST_APPEND(mark_objects_to_trace, obj); - /* Note: this obj might be visited already, but from a different segment. We ignore this case and skip re-visiting the object anyway. The idea is that such an object is old (not from the @@ -339,6 +326,10 @@ segments and only needs visiting once. (It may actually be in a shared page, or maybe not.) */ + if (obj == NULL || mark_visited_test_and_set(obj)) + return; /* already visited this object */ + + LIST_APPEND(mark_objects_to_trace, obj); } static void mark_trace(object_t *obj, char *segment_base) @@ -346,13 +337,6 @@ assert(list_is_empty(mark_objects_to_trace)); while (1) { - - /* first, if we're not seeing segment 0, we must change the - flags in flag_page_private[] from PRIVATE_PAGE to - SEGMENT1_PAGE, which will mean "can't re-share" */ - if (segment_base != stm_object_pages && RESHARE_PAGES) - mark_flag_page_private(obj, segment_base); - /* trace into the object (the version from 'segment_base') */ struct object_s *realobj = (struct object_s *)REAL_ADDRESS(segment_base, obj); @@ -374,45 +358,33 @@ static void mark_visit_from_roots(void) { - if (testing_prebuilt_objs != NULL) { LIST_FOREACH_R(testing_prebuilt_objs, object_t * /*item*/, - mark_visit_object(item, get_segment_base(0))); + mark_visit_object(item, stm_object_pages)); } - /* Do the following twice, so that we trace first the objects from - segment 0, and then all others. XXX This is a hack to make it - more likely that we'll be able to re-share pages. */ + stm_thread_local_t *tl = stm_all_thread_locals; + do { + /* If 'tl' is currently running, its 'associated_segment_num' + field is the segment number that contains the correct + version of its overflowed objects. If not, then the + field is still some correct segment number, and it doesn't + matter which one we pick. */ + char *segment_base = get_segment_base(tl->associated_segment_num); - int must_be_zero; - for (must_be_zero = 1; must_be_zero >= 0; must_be_zero--) { + struct stm_shadowentry_s *current = tl->shadowstack; + struct stm_shadowentry_s *base = tl->shadowstack_base; + while (current-- != base) { + assert(current->ss != (object_t *)-1); + mark_visit_object(current->ss, segment_base); + } + mark_visit_object(tl->thread_local_obj, segment_base); - stm_thread_local_t *tl = stm_all_thread_locals; - do { - /* If 'tl' is currently running, its 'associated_segment_num' - field is the segment number that contains the correct - version of its overflowed objects. If not, then the - field is still some correct segment number, and it doesn't - matter which one we pick. */ - char *segment_base = get_segment_base(tl->associated_segment_num); - - if (must_be_zero == (segment_base == get_segment_base(0))) { - - struct stm_shadowentry_s *current = tl->shadowstack; - struct stm_shadowentry_s *base = tl->shadowstack_base; - while (current-- != base) { - assert(current->ss != (object_t *)-1); - mark_visit_object(current->ss, segment_base); - } - mark_visit_object(tl->thread_local_obj, segment_base); - } - - tl = tl->next; - } while (tl != stm_all_thread_locals); - } + tl = tl->next; + } while (tl != stm_all_thread_locals); long i; - for (i = 0; i < NB_SEGMENTS; i++) { + for (i = 1; i <= NB_SEGMENTS; i++) { if (get_priv_segment(i)->transaction_state != TS_NONE) mark_visit_object( get_priv_segment(i)->threadlocal_at_start_of_transaction, @@ -423,20 +395,21 @@ static void mark_visit_from_modified_objects(void) { /* The modified objects are the ones that may exist in two different - versions: one in the segment that modified it, and another in - all other segments. */ + versions: one in the segment that modified it, and another in all + other segments. (It can also be more than two if we don't have + eager write locking.) + */ long i; - for (i = 0; i < NB_SEGMENTS; i++) { - char *base1 = get_segment_base(i); /* two different segments */ - char *base2 = get_segment_base(!i); + for (i = 1; i <= NB_SEGMENTS; i++) { + char *base = get_segment_base(i); LIST_FOREACH_R( get_priv_segment(i)->modified_old_objects, object_t * /*item*/, ({ mark_visited_test_and_set(item); - mark_trace(item, base1); - mark_trace(item, base2); + mark_trace(item, stm_object_pages); /* shared version */ + mark_trace(item, base); /* private version */ })); } } @@ -444,7 +417,7 @@ static void clean_up_segment_lists(void) { long i; - for (i = 0; i < NB_SEGMENTS; i++) { + for (i = 1; i <= NB_SEGMENTS; i++) { struct stm_priv_segment_info_s *pseg = get_priv_segment(i); struct list_s *lst; @@ -490,7 +463,9 @@ static void sweep_large_objects(void) { + mutex_pages_lock(); _stm_largemalloc_sweep(); + mutex_pages_unlock(); } static void clean_write_locks(void) @@ -506,11 +481,11 @@ memset(write_locks + lock2_idx, 0, sizeof(write_locks) - lock2_idx); } -static void major_set_write_locks(void) +static void major_restore_write_locks(void) { /* restore the write locks on the modified objects */ long i; - for (i = 0; i < NB_SEGMENTS; i++) { + for (i = 1; i <= NB_SEGMENTS; i++) { struct stm_priv_segment_info_s *pseg = get_priv_segment(i); LIST_FOREACH_R( @@ -536,6 +511,10 @@ dprintf((" | used before collection: %ld\n", (long)pages_ctl.total_allocated)); + /* reshare pages */ + if (RESHARE_PAGES) + major_reshare_pages(); + /* marking */ LIST_CREATE(mark_objects_to_trace); mark_visit_from_modified_objects(); @@ -549,15 +528,11 @@ clean_up_segment_lists(); /* sweeping */ - mutex_pages_lock(); - if (RESHARE_PAGES) - major_reshare_pages(); sweep_large_objects(); //sweep_uniform_pages(); - mutex_pages_unlock(); clean_write_locks(); - major_set_write_locks(); + major_restore_write_locks(); dprintf((" | used after collection: %ld\n", (long)pages_ctl.total_allocated)); diff --git a/rpython/translator/stm/src_stm/stm/gcpage.h b/rpython/translator/stm/src_stm/stm/gcpage.h --- a/rpython/translator/stm/src_stm/stm/gcpage.h +++ b/rpython/translator/stm/src_stm/stm/gcpage.h @@ -16,7 +16,7 @@ #define GC_MAJOR_COLLECT 1.82 /* re-share pages after major collections (1 or 0) */ -#define RESHARE_PAGES 0 +#define RESHARE_PAGES 1 diff --git a/rpython/translator/stm/src_stm/stm/misc.c b/rpython/translator/stm/src_stm/stm/misc.c --- a/rpython/translator/stm/src_stm/stm/misc.c +++ b/rpython/translator/stm/src_stm/stm/misc.c @@ -42,9 +42,10 @@ } #ifdef STM_TESTS -uint8_t _stm_get_page_flag(uintptr_t index) +uintptr_t _stm_get_private_page(uintptr_t pagenum) { - return flag_page_private[index]; + /* xxx returns 0 or 1 now */ + return is_private_page(STM_SEGMENT->segment_num, pagenum); } long _stm_count_modified_old_objects(void) @@ -80,4 +81,14 @@ mutex_pages_unlock(); return result; } + +void _stm_mutex_pages_lock(void) +{ + mutex_pages_lock(); +} + +void _stm_mutex_pages_unlock(void) +{ + mutex_pages_unlock(); +} #endif diff --git a/rpython/translator/stm/src_stm/stm/nursery.c b/rpython/translator/stm/src_stm/stm/nursery.c --- a/rpython/translator/stm/src_stm/stm/nursery.c +++ b/rpython/translator/stm/src_stm/stm/nursery.c @@ -27,7 +27,7 @@ _stm_nursery_start = NURSERY_START; long i; - for (i = 0; i < NB_SEGMENTS; i++) { + for (i = 1; i <= NB_SEGMENTS; i++) { get_segment(i)->nursery_current = (stm_char *)NURSERY_START; get_segment(i)->nursery_end = NURSERY_END; } @@ -199,8 +199,11 @@ WRITE_BARRIER flag and traced into it to fix its content); or add the object to 'large_overflow_objects'. */ - if (STM_PSEGMENT->minor_collect_will_commit_now) - synchronize_overflow_object_now(obj); + if (STM_PSEGMENT->minor_collect_will_commit_now) { + mutex_pages_lock(); + synchronize_object_now(obj); + mutex_pages_unlock(); + } else LIST_APPEND(STM_PSEGMENT->large_overflow_objects, obj); } @@ -379,7 +382,7 @@ _stm_nursery_start = NURSERY_END - free_count; long i; - for (i = 0; i < NB_SEGMENTS; i++) { + for (i = 1; i <= NB_SEGMENTS; i++) { if ((uintptr_t)get_segment(i)->nursery_current < _stm_nursery_start) get_segment(i)->nursery_current = (stm_char *)_stm_nursery_start; } @@ -412,7 +415,7 @@ int original_num = STM_SEGMENT->segment_num; long i; - for (i = 0; i < NB_SEGMENTS; i++) { + for (i = 1; i <= NB_SEGMENTS; i++) { struct stm_priv_segment_info_s *pseg = get_priv_segment(i); if (MINOR_NOTHING_TO_DO(pseg)) /*TS_NONE segments have NOTHING_TO_DO*/ continue; diff --git a/rpython/translator/stm/src_stm/stm/pages.c b/rpython/translator/stm/src_stm/stm/pages.c --- a/rpython/translator/stm/src_stm/stm/pages.c +++ b/rpython/translator/stm/src_stm/stm/pages.c @@ -26,6 +26,7 @@ static void teardown_pages(void) { memset(&pages_ctl, 0, sizeof(pages_ctl)); + memset(pages_privatized, 0, sizeof(pages_privatized)); } static void mutex_pages_lock(void) @@ -40,7 +41,6 @@ __sync_lock_release(&pages_ctl.mutex_pages); } -__attribute__((unused)) static bool _has_mutex_pages(void) { return pages_ctl.mutex_pages != 0; @@ -48,6 +48,7 @@ static uint64_t increment_total_allocated(ssize_t add_or_remove) { + assert(_has_mutex_pages()); pages_ctl.total_allocated += add_or_remove; if (pages_ctl.total_allocated >= pages_ctl.total_allocated_bound) @@ -103,100 +104,64 @@ segment 0. */ uintptr_t i; assert(_has_mutex_pages()); - for (i = 1; i < NB_SEGMENTS; i++) { + for (i = 1; i <= NB_SEGMENTS; i++) { char *segment_base = get_segment_base(i); d_remap_file_pages(segment_base + pagenum * 4096UL, count * 4096UL, pagenum); } - for (i = 0; i < count; i++) - flag_page_private[pagenum + i] = SHARED_PAGE; } -#if 0 -static void pages_make_shared_again(uintptr_t pagenum, uintptr_t count) +static void page_privatize(uintptr_t pagenum) { - /* Same as pages_initialize_shared(), but tries hard to minimize the - total number of pages that remap_file_pages() must handle, by - fragmenting calls as much as possible (the overhead of one system - call appears smaller as the overhead per page). */ - uintptr_t start, i = 0; - while (i < count) { - if (flag_page_private[pagenum + (i++)] == SHARED_PAGE) - continue; - start = i; /* first index of a private page */ - while (1) { - i++; - if (i == count || flag_page_private[pagenum + i] == SHARED_PAGE) - break; - } - pages_initialize_shared(pagenum + start, i - start); - } -} -#endif - -static void privatize_range(uintptr_t pagenum, uintptr_t count, bool full) -{ - ssize_t pgoff1 = pagenum; - ssize_t pgoff2 = pagenum + NB_PAGES; - ssize_t localpgoff = pgoff1 + NB_PAGES * STM_SEGMENT->segment_num; - ssize_t otherpgoff = pgoff1 + NB_PAGES * (1 - STM_SEGMENT->segment_num); - - void *localpg = stm_object_pages + localpgoff * 4096UL; - void *otherpg = stm_object_pages + otherpgoff * 4096UL; - - memset(flag_page_private + pagenum, REMAPPING_PAGE, count); - d_remap_file_pages(localpg, count * 4096, pgoff2); - uintptr_t i; - if (full) { - for (i = 0; i < count; i++) { - pagecopy(localpg + 4096 * i, otherpg + 4096 * i); - } - } - else { - pagecopy(localpg, otherpg); - if (count > 1) - pagecopy(localpg + 4096 * (count-1), otherpg + 4096 * (count-1)); - } - write_fence(); - memset(flag_page_private + pagenum, PRIVATE_PAGE, count); - increment_total_allocated(4096 * count); -} - -static void _pages_privatize(uintptr_t pagenum, uintptr_t count, bool full) -{ - /* narrow the range of pages to privatize from the end: */ - while (flag_page_private[pagenum + count - 1] == PRIVATE_PAGE) { - if (!--count) - return; + if (is_private_page(STM_SEGMENT->segment_num, pagenum)) { + /* the page is already privatized */ + return; } + /* lock, to prevent concurrent threads from looking up this thread's + 'pages_privatized' bits in parallel */ mutex_pages_lock(); - uintptr_t page_start_range = pagenum; - uintptr_t pagestop = pagenum + count; + /* "unmaps" the page to make the address space location correspond + again to its underlying file offset (XXX later we should again + attempt to group together many calls to d_remap_file_pages() in + succession) */ + uintptr_t pagenum_in_file = NB_PAGES * STM_SEGMENT->segment_num + pagenum; + char *new_page = stm_object_pages + pagenum_in_file * 4096UL; + d_remap_file_pages(new_page, 4096, pagenum_in_file); + increment_total_allocated(4096); - for (; pagenum < pagestop; pagenum++) { - uint8_t prev = flag_page_private[pagenum]; - if (prev == PRIVATE_PAGE) { - if (pagenum > page_start_range) { - privatize_range(page_start_range, - pagenum - page_start_range, full); - } - page_start_range = pagenum + 1; - } - else { - assert(prev == SHARED_PAGE); - } - } + /* copy the content from the shared (segment 0) source */ + pagecopy(new_page, stm_object_pages + pagenum * 4096UL); - if (pagenum > page_start_range) { - privatize_range(page_start_range, - pagenum - page_start_range, full); - } + /* add this thread's 'pages_privatized' bit */ + uint64_t bitmask = 1UL << (STM_SEGMENT->segment_num - 1); + pages_privatized[pagenum - PAGE_FLAG_START].by_segment |= bitmask; mutex_pages_unlock(); } +static void page_reshare(uintptr_t pagenum) +{ + struct page_shared_s ps = pages_privatized[pagenum - PAGE_FLAG_START]; + pages_privatized[pagenum - PAGE_FLAG_START].by_segment = 0; + + long j, total = 0; + for (j = 0; j < NB_SEGMENTS; j++) { + if (ps.by_segment & (1 << j)) { + /* Page 'pagenum' is private in segment 'j + 1'. Reshare */ + char *segment_base = stm_object_pages + NB_PAGES * 4096UL * (j+1); + + madvise(segment_base + pagenum * 4096UL, 4096, MADV_DONTNEED); + d_remap_file_pages(segment_base + pagenum * 4096UL, + 4096, pagenum); + total -= 4096; + } + } + increment_total_allocated(total); +} + + #if 0 static bool is_fully_in_shared_pages(object_t *obj) { diff --git a/rpython/translator/stm/src_stm/stm/pages.h b/rpython/translator/stm/src_stm/stm/pages.h --- a/rpython/translator/stm/src_stm/stm/pages.h +++ b/rpython/translator/stm/src_stm/stm/pages.h @@ -1,49 +1,62 @@ /* Imported by rpython/translator/stm/import_stmgc.py */ -enum /* flag_page_private */ { - /* The page is not in use. Assume that each segment sees its own copy. */ - FREE_PAGE=0, +/* This handles pages of objects outside the nursery. Every page + has a "shared copy" and zero or more "private copies". - /* The page is shared by all segments. Each segment sees the same - physical page (the one that is within the segment 0 mmap address). */ - SHARED_PAGE, + The shared copy of a page is stored in the mmap at the file offset + corresponding to the segment 0 offset. Initially, accessing a page + from segment N remaps to segment 0. If the page is turned private, + then we "un-remap" it to its initial location. The 'pages_privatized' + global array records if a page is currently mapped to segment 0 + (shared page) or to its natural location (private page). - /* For only one range of pages at a time, around the call to - remap_file_pages() that un-shares the pages (SHARED -> PRIVATE). */ - REMAPPING_PAGE, + Note that this page manipulation logic uses remap_file_pages() to + fully hide its execution cost behind the CPU's memory management unit. + It should not be confused with the logic of tracking which objects + are old-and-committed, old-but-modified, overflow objects, and so on + (which works at the object granularity, not the page granularity). +*/ - /* Page is private for each segment. */ - PRIVATE_PAGE, +#define PAGE_FLAG_START END_NURSERY_PAGE +#define PAGE_FLAG_END NB_PAGES - /* gcpage.c: page contains objects that have been traced in the - segment > 0 */ - SEGMENT1_PAGE, +struct page_shared_s { +#if NB_SEGMENTS <= 8 + uint8_t by_segment; +#elif NB_SEGMENTS <= 16 + uint16_t by_segment; +#elif NB_SEGMENTS <= 32 + uint32_t by_segment; +#elif NB_SEGMENTS <= 64 + uint64_t by_segment; +#else +# error "NB_SEGMENTS > 64 not supported right now" +#endif }; -static uint8_t flag_page_private[NB_PAGES]; +static struct page_shared_s pages_privatized[PAGE_FLAG_END - PAGE_FLAG_START]; -static void _pages_privatize(uintptr_t pagenum, uintptr_t count, bool full); static void pages_initialize_shared(uintptr_t pagenum, uintptr_t count); -//static void pages_make_shared_again(uintptr_t pagenum, uintptr_t count); +static void page_privatize(uintptr_t pagenum); +static void page_reshare(uintptr_t pagenum); static void mutex_pages_lock(void); static void mutex_pages_unlock(void); +static bool _has_mutex_pages(void) __attribute__((unused)); static uint64_t increment_total_allocated(ssize_t add_or_remove); static bool is_major_collection_requested(void); static void force_major_collection_request(void); static void reset_major_collection_requested(void); -inline static void pages_privatize(uintptr_t pagenum, uintptr_t count, - bool full) { - /* This is written a bit carefully so that a call with a constant - count == 1 will turn this loop into just one "if". */ - while (flag_page_private[pagenum] == PRIVATE_PAGE) { - if (!--count) { - return; - } - pagenum++; - } - _pages_privatize(pagenum, count, full); +static inline bool is_private_page(long segnum, uintptr_t pagenum) +{ + assert(pagenum >= PAGE_FLAG_START); + uint64_t bitmask = 1UL << (segnum - 1); + return (pages_privatized[pagenum - PAGE_FLAG_START].by_segment & bitmask); } -/* static bool is_fully_in_shared_pages(object_t *obj); */ +static inline void page_check_and_reshare(uintptr_t pagenum) +{ + if (pages_privatized[pagenum - PAGE_FLAG_START].by_segment != 0) + page_reshare(pagenum); +} diff --git a/rpython/translator/stm/src_stm/stm/setup.c b/rpython/translator/stm/src_stm/stm/setup.c --- a/rpython/translator/stm/src_stm/stm/setup.c +++ b/rpython/translator/stm/src_stm/stm/setup.c @@ -27,12 +27,15 @@ if (stm_object_pages == MAP_FAILED) stm_fatalerror("initial stm_object_pages mmap() failed: %m\n"); + /* The segment 0 is not used to run transactions, but to contain the + shared copy of the pages. We mprotect all pages before so that + accesses fail, up to and including the pages corresponding to the + nurseries of the other segments. */ + mprotect(stm_object_pages, END_NURSERY_PAGE * 4096UL, PROT_NONE); + long i; - for (i = 0; i < NB_SEGMENTS; i++) { + for (i = 1; i <= NB_SEGMENTS; i++) { char *segment_base = get_segment_base(i); -#ifdef STM_TESTS - stm_other_pages = segment_base; -#endif /* In each segment, the first page is where TLPREFIX'ed NULL accesses land. We mprotect it so that accesses fail. */ @@ -40,7 +43,7 @@ /* Fill the TLS page (page 1) with 0xDC, for debugging */ memset(REAL_ADDRESS(segment_base, 4096), 0xDC, 4096); - /* Make a "hole" at STM_PSEGMENT */ + /* Make a "hole" at STM_PSEGMENT (which includes STM_SEGMENT) */ memset(REAL_ADDRESS(segment_base, STM_PSEGMENT), 0, sizeof(*STM_PSEGMENT)); @@ -50,9 +53,10 @@ (FIRST_READMARKER_PAGE - 2) * 4096UL, PROT_NONE); + /* Initialize STM_PSEGMENT */ struct stm_priv_segment_info_s *pr = get_priv_segment(i); - assert(i + 1 < 255); /* 255 is WL_VISITED in gcpage.c */ - pr->write_lock_num = i + 1; + assert(1 <= i && i < 255); /* 255 is WL_VISITED in gcpage.c */ + pr->write_lock_num = i; pr->pub.segment_num = i; pr->pub.segment_base = segment_base; pr->objects_pointing_to_nursery = NULL; @@ -63,7 +67,7 @@ pr->young_outside_nursery = tree_create(); pr->nursery_objects_shadows = tree_create(); pr->callbacks_on_abort = tree_create(); - pr->overflow_number = GCFLAG_OVERFLOW_NUMBER_bit0 * (i + 1); + pr->overflow_number = GCFLAG_OVERFLOW_NUMBER_bit0 * i; highest_overflow_number = pr->overflow_number; } @@ -74,10 +78,6 @@ STM_SEGMENT->transaction_read_version never contains zero, so a null read marker means "not read" whatever the current transaction_read_version is. - - The creation markers are initially zero, which is correct: - it means "objects of this line of 256 bytes have not been - allocated by the current transaction." */ setup_sync(); @@ -93,7 +93,7 @@ assert(!_has_mutex()); long i; - for (i = 0; i < NB_SEGMENTS; i++) { + for (i = 1; i <= NB_SEGMENTS; i++) { struct stm_priv_segment_info_s *pr = get_priv_segment(i); assert(pr->objects_pointing_to_nursery == NULL); assert(pr->large_overflow_objects == NULL); @@ -108,8 +108,6 @@ munmap(stm_object_pages, TOTAL_MEMORY); stm_object_pages = NULL; - memset(flag_page_private, 0, sizeof(flag_page_private)); - teardown_core(); teardown_sync(); teardown_gcpage(); @@ -147,14 +145,14 @@ tl->prev = stm_all_thread_locals->prev; stm_all_thread_locals->prev->next = tl; stm_all_thread_locals->prev = tl; - num = tl->prev->associated_segment_num + 1; + num = tl->prev->associated_segment_num; } tl->thread_local_obj = NULL; /* assign numbers consecutively, but that's for tests; we could also assign the same number to all of them and they would get their own numbers automatically. */ - num = num % NB_SEGMENTS; + num = (num % NB_SEGMENTS) + 1; tl->associated_segment_num = num; _init_shadow_stack(tl); set_gs_register(get_segment_base(num)); diff --git a/rpython/translator/stm/src_stm/stm/sync.c b/rpython/translator/stm/src_stm/stm/sync.c --- a/rpython/translator/stm/src_stm/stm/sync.c +++ b/rpython/translator/stm/src_stm/stm/sync.c @@ -31,7 +31,7 @@ pthread_mutex_t global_mutex; pthread_cond_t cond[_C_TOTAL]; /* some additional pieces of global state follow */ - uint8_t in_use[NB_SEGMENTS]; /* 1 if running a pthread */ + uint8_t in_use1[NB_SEGMENTS]; /* 1 if running a pthread */ uint64_t global_time; }; char reserved[192]; @@ -61,7 +61,7 @@ stm_fatalerror("cond destroy: %m\n"); } - memset(&sync_ctl, 0, sizeof(sync_ctl.in_use)); + memset(&sync_ctl, 0, sizeof(sync_ctl)); } #ifndef NDEBUG @@ -125,12 +125,12 @@ { long i; restart: - for (i = 0; i < NB_SEGMENTS; i++) { + for (i = 1; i <= NB_SEGMENTS; i++) { if (get_priv_segment(i)->transaction_state == TS_INEVITABLE) { if (can_abort) { /* handle this case like a contention: it will either abort us (not the other thread, which is inevitable), - or for a while. If we go past this call, then we + or wait for a while. If we go past this call, then we waited; in this case we have to re-check if no other thread is inevitable. */ inevitable_contention_management(i); @@ -153,7 +153,7 @@ assert(_is_tl_registered(tl)); int num = tl->associated_segment_num; - if (sync_ctl.in_use[num] == 0) { + if (sync_ctl.in_use1[num - 1] == 0) { /* fast-path: we can get the same segment number than the one we had before. The value stored in GS is still valid. */ #ifdef STM_TESTS @@ -166,10 +166,10 @@ } /* Look for the next free segment. If there is none, wait for the condition variable. */ - int i; - for (i = 0; i < NB_SEGMENTS; i++) { - num = (num + 1) % NB_SEGMENTS; - if (sync_ctl.in_use[num] == 0) { + int retries; + for (retries = 0; retries < NB_SEGMENTS; retries++) { + num = (num % NB_SEGMENTS) + 1; + if (sync_ctl.in_use1[num - 1] == 0) { /* we're getting 'num', a different number. */ dprintf(("acquired different segment: %d->%d\n", tl->associated_segment_num, num)); tl->associated_segment_num = num; @@ -185,7 +185,7 @@ return false; got_num: - sync_ctl.in_use[num] = 1; + sync_ctl.in_use1[num - 1] = 1; assert(STM_SEGMENT->segment_num == num); assert(STM_SEGMENT->running_thread == NULL); STM_SEGMENT->running_thread = tl; @@ -209,8 +209,8 @@ assert(STM_SEGMENT->running_thread == tl); STM_SEGMENT->running_thread = NULL; - assert(sync_ctl.in_use[tl->associated_segment_num] == 1); - sync_ctl.in_use[tl->associated_segment_num] = 0; + assert(sync_ctl.in_use1[tl->associated_segment_num - 1] == 1); + sync_ctl.in_use1[tl->associated_segment_num - 1] = 0; } __attribute__((unused)) @@ -222,7 +222,7 @@ bool _stm_in_transaction(stm_thread_local_t *tl) { int num = tl->associated_segment_num; - assert(num < NB_SEGMENTS); + assert(1 <= num && num <= NB_SEGMENTS); return get_segment(num)->running_thread == tl; } @@ -261,12 +261,15 @@ { assert(_safe_points_requested == false); assert((_safe_points_requested = true, 1)); + assert(_has_mutex()); long i; - for (i = 0; i < NB_SEGMENTS; i++) { + for (i = 1; i <= NB_SEGMENTS; i++) { if (get_segment(i)->nursery_end == NURSERY_END) get_segment(i)->nursery_end = NSE_SIGPAUSE; } + assert(!pause_signalled); + pause_signalled = true; } static inline long count_other_threads_sp_running(void) @@ -277,7 +280,7 @@ long result = 0; int my_num = STM_SEGMENT->segment_num; - for (i = 0; i < NB_SEGMENTS; i++) { + for (i = 1; i <= NB_SEGMENTS; i++) { if (i != my_num && get_priv_segment(i)->safe_point == SP_RUNNING) { assert(get_segment(i)->nursery_end <= _STM_NSE_SIGNAL_MAX); result++; @@ -288,11 +291,13 @@ static void remove_requests_for_safe_point(void) { + assert(pause_signalled); + pause_signalled = false; assert(_safe_points_requested == true); assert((_safe_points_requested = false, 1)); long i; - for (i = 0; i < NB_SEGMENTS; i++) { + for (i = 1; i <= NB_SEGMENTS; i++) { assert(get_segment(i)->nursery_end != NURSERY_END); if (get_segment(i)->nursery_end == NSE_SIGPAUSE) get_segment(i)->nursery_end = NURSERY_END; diff --git a/rpython/translator/stm/src_stm/stm/sync.h b/rpython/translator/stm/src_stm/stm/sync.h --- a/rpython/translator/stm/src_stm/stm/sync.h +++ b/rpython/translator/stm/src_stm/stm/sync.h @@ -31,3 +31,5 @@ static void wait_for_end_of_inevitable_transaction(bool can_abort); static void synchronize_all_threads(void); + +static bool pause_signalled; diff --git a/rpython/translator/stm/src_stm/stm/weakref.c b/rpython/translator/stm/src_stm/stm/weakref.c --- a/rpython/translator/stm/src_stm/stm/weakref.c +++ b/rpython/translator/stm/src_stm/stm/weakref.c @@ -33,23 +33,18 @@ ssize_t size = 16; stm_char *point_to_loc = (stm_char*)WEAKREF_PTR(weakref, size); - if (flag_page_private[(uintptr_t)point_to_loc / 4096UL] == PRIVATE_PAGE) { - long i; - for (i = 0; i < NB_SEGMENTS; i++) { - char *base = get_segment_base(i); /* two different segments */ - object_t ** ref_loc = (object_t **)REAL_ADDRESS(base, point_to_loc); - *ref_loc = value; - } - } - else { - *WEAKREF_PTR(weakref, size) = value; + long i; + for (i = 1; i <= NB_SEGMENTS; i++) { + char *base = get_segment_base(i); + object_t ** ref_loc = (object_t **)REAL_ADDRESS(base, point_to_loc); + *ref_loc = value; } } /***** Minor collection *****/ -static void stm_move_young_weakrefs() +static void stm_move_young_weakrefs(void) { /* The code relies on the fact that no weakref can be an old object weakly pointing to a young object. Indeed, weakrefs are immutable @@ -116,7 +111,7 @@ static void stm_visit_old_weakrefs(void) { long i; - for (i = 0; i < NB_SEGMENTS; i++) { + for (i = 1; i <= NB_SEGMENTS; i++) { struct stm_priv_segment_info_s *pseg = get_priv_segment(i); struct list_s *lst; diff --git a/rpython/translator/stm/src_stm/stmgc.h b/rpython/translator/stm/src_stm/stmgc.h --- a/rpython/translator/stm/src_stm/stmgc.h +++ b/rpython/translator/stm/src_stm/stmgc.h @@ -88,7 +88,7 @@ #include bool _stm_was_read(object_t *obj); bool _stm_was_written(object_t *obj); -uint8_t _stm_get_page_flag(uintptr_t index); +uintptr_t _stm_get_private_page(uintptr_t pagenum); bool _stm_in_transaction(stm_thread_local_t *tl); char *_stm_get_segment_base(long index); void _stm_test_switch(stm_thread_local_t *tl); @@ -108,6 +108,8 @@ object_t *_stm_enum_modified_old_objects(long index); object_t *_stm_enum_objects_pointing_to_nursery(long index); uint64_t _stm_total_allocated(void); +void _stm_mutex_pages_lock(void); +void _stm_mutex_pages_unlock(void); #endif #define _STM_GCFLAG_WRITE_BARRIER 0x01 @@ -128,6 +130,12 @@ /* ==================== PUBLIC API ==================== */ +/* Number of segments (i.e. how many threads can be executed in + parallel, in maximum). +*/ +#define STM_NB_SEGMENTS 4 + + /* Structure of objects -------------------- From noreply at buildbot.pypy.org Sun Mar 16 20:59:09 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 16 Mar 2014 20:59:09 +0100 (CET) Subject: [pypy-commit] stmgc default: Make this comment and reality take a step toward each other Message-ID: <20140316195909.97D651C011F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1058:543aeede2318 Date: 2014-03-16 20:58 +0100 http://bitbucket.org/pypy/stmgc/changeset/543aeede2318/ Log: Make this comment and reality take a step toward each other diff --git a/c7/demo/Makefile b/c7/demo/Makefile --- a/c7/demo/Makefile +++ b/c7/demo/Makefile @@ -18,13 +18,13 @@ C_FILES = ../stmgc.c ../stm/*.c -# note that 'build' is optimized but still contains all asserts +# note that 'build' is partially optimized but still contains all asserts debug-%: %.c ${H_FILES} ${C_FILES} clang -I.. -pthread -DSTM_DEBUGPRINT -DSTM_GC_NURSERY=128 -g -O0 \ $< -o debug-$* -Wall -Werror ../stmgc.c build-%: %.c ${H_FILES} ${C_FILES} - clang -I.. -pthread -DSTM_GC_NURSERY=128 -g -O0 \ + clang -I.. -pthread -DSTM_GC_NURSERY=128 -g -O1 \ $< -o build-$* -Wall -Werror ../stmgc.c release-%: %.c ${H_FILES} ${C_FILES} From noreply at buildbot.pypy.org Sun Mar 16 21:18:02 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 16 Mar 2014 21:18:02 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: os.fork()! Message-ID: <20140316201802.DB0401C011F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r69980:4891028503f5 Date: 2014-03-16 21:17 +0100 http://bitbucket.org/pypy/pypy/changeset/4891028503f5/ Log: os.fork()! diff --git a/TODO b/TODO --- a/TODO +++ b/TODO @@ -28,6 +28,10 @@ ------------------------------------------------------------ +os.fork()! + +------------------------------------------------------------ + From noreply at buildbot.pypy.org Sun Mar 16 22:40:22 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Sun, 16 Mar 2014 22:40:22 +0100 (CET) Subject: [pypy-commit] pypy default: cleanups Message-ID: <20140316214022.A49D01C011F@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r69981:eb5cbd7cbd8b Date: 2014-03-16 13:14 -0700 http://bitbucket.org/pypy/pypy/changeset/eb5cbd7cbd8b/ Log: cleanups diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py --- a/rpython/jit/codewriter/jtransform.py +++ b/rpython/jit/codewriter/jtransform.py @@ -547,20 +547,18 @@ track_allocation = d.pop('track_allocation', True) if d: raise UnsupportedMallocFlags(d) - TYPE = op.args[0].value if zero: name += '_zero' if add_memory_pressure: name += '_add_memory_pressure' if not track_allocation: name += '_no_track_allocation' + TYPE = op.args[0].value op1 = self.prepare_builtin_call(op, name, args, (TYPE,), TYPE) - if name == 'raw_malloc_varsize': - ITEMTYPE = op.args[0].value.OF - if ITEMTYPE == lltype.Char: - return self._handle_oopspec_call(op1, args, - EffectInfo.OS_RAW_MALLOC_VARSIZE_CHAR, - EffectInfo.EF_CAN_RAISE) + if name == 'raw_malloc_varsize' and TYPE.OF == lltype.Char: + return self._handle_oopspec_call(op1, args, + EffectInfo.OS_RAW_MALLOC_VARSIZE_CHAR, + EffectInfo.EF_CAN_RAISE) return self.rewrite_op_direct_call(op1) def rewrite_op_malloc_varsize(self, op): diff --git a/rpython/jit/metainterp/optimizeopt/test/test_rawbuffer.py b/rpython/jit/metainterp/optimizeopt/test/test_rawbuffer.py --- a/rpython/jit/metainterp/optimizeopt/test/test_rawbuffer.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_rawbuffer.py @@ -17,8 +17,7 @@ ( 4, 2, 'descr2', 'two'), ( 8, 4, 'descr3', 'three'), (12, 2, 'descr4', 'four'), - ] - # + ] def test_write_value_update(): buf = RawBuffer(FakeCPU()) @@ -28,7 +27,7 @@ assert buf._get_memory() == [ ( 0, 4, 'descr', 'ONE'), ( 4, 2, 'descr', 'two'), - ] + ] def test_write_value_invalid_length(): buf = RawBuffer(FakeCPU()) @@ -38,7 +37,6 @@ with py.test.raises(InvalidRawWrite): buf.write_value(0, 4, 'descr2', 'two') - def test_write_value_overlapping_next(): buf = RawBuffer(FakeCPU()) buf.write_value(0, 4, 'descr', 'one') diff --git a/rpython/jit/metainterp/test/test_rawmem.py b/rpython/jit/metainterp/test/test_rawmem.py --- a/rpython/jit/metainterp/test/test_rawmem.py +++ b/rpython/jit/metainterp/test/test_rawmem.py @@ -1,7 +1,8 @@ from rpython.jit.metainterp.test.support import LLJitMixin from rpython.rtyper.lltypesystem import lltype, rffi from rpython.rlib.rawstorage import (alloc_raw_storage, raw_storage_setitem, - free_raw_storage, raw_storage_getitem) + free_raw_storage, raw_storage_getitem) + class RawMemTests(object): def test_cast_void_ptr(self): From noreply at buildbot.pypy.org Sun Mar 16 22:40:23 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Sun, 16 Mar 2014 22:40:23 +0100 (CET) Subject: [pypy-commit] pypy virtual-raw-store-load: optimize raw_load/raw_store like arrayitem_raw Message-ID: <20140316214023.D60EC1C011F@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: virtual-raw-store-load Changeset: r69982:e165cc881e89 Date: 2014-03-16 14:03 -0700 http://bitbucket.org/pypy/pypy/changeset/e165cc881e89/ Log: optimize raw_load/raw_store like arrayitem_raw diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -1894,6 +1894,23 @@ """ self.optimize_loop(ops, expected) + def test_virtual_raw_store_load(self): + ops = """ + [i1] + i0 = call('malloc', 10, descr=raw_malloc_descr) + raw_store(i0, 0, i1, descr=rawarraydescr) + i2 = raw_load(i0, 0, descr=rawarraydescr) + i3 = int_add(i1, i2) + call('free', i0, descr=raw_free_descr) + jump(i3) + """ + expected = """ + [i1] + i2 = int_add(i1, i1) + jump(i2) + """ + self.optimize_loop(ops, expected) + def test_duplicate_getfield_1(self): ops = """ [p1, p2] diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py --- a/rpython/jit/metainterp/optimizeopt/virtualize.py +++ b/rpython/jit/metainterp/optimizeopt/virtualize.py @@ -788,6 +788,8 @@ value.ensure_nonnull() self.emit_operation(op) + optimize_RAW_LOAD = optimize_GETARRAYITEM_RAW + def optimize_SETARRAYITEM_RAW(self, op): value = self.getvalue(op.getarg(0)) if value.is_virtual(): @@ -805,6 +807,8 @@ value.ensure_nonnull() self.emit_operation(op) + optimize_RAW_STORE = optimize_SETARRAYITEM_RAW + def optimize_GETINTERIORFIELD_GC(self, op): value = self.getvalue(op.getarg(0)) if value.is_virtual(): From noreply at buildbot.pypy.org Sun Mar 16 22:40:25 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Sun, 16 Mar 2014 22:40:25 +0100 (CET) Subject: [pypy-commit] pypy virtual-raw-store-load: raw_load/raw_store take offset not index Message-ID: <20140316214025.177881C011F@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: virtual-raw-store-load Changeset: r69983:a56510bcbcc8 Date: 2014-03-16 14:30 -0700 http://bitbucket.org/pypy/pypy/changeset/a56510bcbcc8/ Log: raw_load/raw_store take offset not index diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -1894,7 +1894,7 @@ """ self.optimize_loop(ops, expected) - def test_virtual_raw_store_load(self): + def test_virtual_raw_store_raw_load(self): ops = """ [i1] i0 = call('malloc', 10, descr=raw_malloc_descr) @@ -1911,6 +1911,23 @@ """ self.optimize_loop(ops, expected) + def test_virtual_raw_store_getarrayitem_raw(self): + ops = """ + [f1] + i0 = call('malloc', 16, descr=raw_malloc_descr) + raw_store(i0, 8, f1, descr=rawarraydescr_float) + f2 = getarrayitem_raw(i0, 1, descr=rawarraydescr_float) + f3 = float_add(f1, f2) + call('free', i0, descr=raw_free_descr) + jump(f3) + """ + expected = """ + [f1] + f2 = float_add(f1, f1) + jump(f2) + """ + self.optimize_loop(ops, expected) + def test_duplicate_getfield_1(self): ops = """ [p1, p2] diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py --- a/rpython/jit/metainterp/optimizeopt/test/test_util.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py @@ -226,6 +226,8 @@ hints={'nolength': True})) rawarraydescr_char = cpu.arraydescrof(lltype.Array(lltype.Char, hints={'nolength': True})) + rawarraydescr_float = cpu.arraydescrof(lltype.Array(lltype.Float, + hints={'nolength': True})) fc_array = lltype.GcArray( lltype.Struct( diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py --- a/rpython/jit/metainterp/optimizeopt/virtualize.py +++ b/rpython/jit/metainterp/optimizeopt/virtualize.py @@ -788,8 +788,6 @@ value.ensure_nonnull() self.emit_operation(op) - optimize_RAW_LOAD = optimize_GETARRAYITEM_RAW - def optimize_SETARRAYITEM_RAW(self, op): value = self.getvalue(op.getarg(0)) if value.is_virtual(): @@ -807,7 +805,46 @@ value.ensure_nonnull() self.emit_operation(op) - optimize_RAW_STORE = optimize_SETARRAYITEM_RAW + def _unpack_raw_load_store_op(self, op, offsetbox): + offset = offsetbox.getint() + cpu = self.optimizer.cpu + descr = op.getdescr() + itemsize = cpu.unpack_arraydescr_size(descr)[1] + return offset, itemsize, descr + + def optimize_RAW_LOAD(self, op): + value = self.getvalue(op.getarg(0)) + if value.is_virtual(): + offsetbox = self.get_constant_box(op.getarg(1)) + if offsetbox is not None: + offset, itemsize, descr = self._unpack_raw_load_store_op(op, offsetbox) + try: + itemvalue = value.getitem_raw(offset, itemsize, descr) + self.make_equal_to(op.result, itemvalue) + except InvalidRawOperation: + box = value.force_box(self) + op.setarg(0, box) + self.emit_operation(op) + return + value.ensure_nonnull() + self.emit_operation(op) + + def optimize_RAW_STORE(self, op): + value = self.getvalue(op.getarg(0)) + if value.is_virtual(): + offsetbox = self.get_constant_box(op.getarg(1)) + if offsetbox is not None: + offset, itemsize, descr = self._unpack_raw_load_store_op(op, offsetbox) + itemvalue = self.getvalue(op.getarg(2)) + try: + value.setitem_raw(offset, itemsize, descr, itemvalue) + except InvalidRawOperation: + box = value.force_box(self) + op.setarg(0, box) + self.emit_operation(op) + return + value.ensure_nonnull() + self.emit_operation(op) def optimize_GETINTERIORFIELD_GC(self, op): value = self.getvalue(op.getarg(0)) From noreply at buildbot.pypy.org Sun Mar 16 22:40:26 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Sun, 16 Mar 2014 22:40:26 +0100 (CET) Subject: [pypy-commit] pypy virtual-raw-store-load: cleanup Message-ID: <20140316214026.4E6391C011F@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: virtual-raw-store-load Changeset: r69984:37d2c78f5076 Date: 2014-03-16 14:39 -0700 http://bitbucket.org/pypy/pypy/changeset/37d2c78f5076/ Log: cleanup diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py --- a/rpython/jit/metainterp/optimizeopt/virtualize.py +++ b/rpython/jit/metainterp/optimizeopt/virtualize.py @@ -779,11 +779,12 @@ offset, itemsize, descr = self._unpack_arrayitem_raw_op(op, indexbox) try: itemvalue = value.getitem_raw(offset, itemsize, descr) - self.make_equal_to(op.result, itemvalue) except InvalidRawOperation: box = value.force_box(self) op.setarg(0, box) self.emit_operation(op) + else: + self.make_equal_to(op.result, itemvalue) return value.ensure_nonnull() self.emit_operation(op) @@ -820,11 +821,12 @@ offset, itemsize, descr = self._unpack_raw_load_store_op(op, offsetbox) try: itemvalue = value.getitem_raw(offset, itemsize, descr) - self.make_equal_to(op.result, itemvalue) except InvalidRawOperation: box = value.force_box(self) op.setarg(0, box) self.emit_operation(op) + else: + self.make_equal_to(op.result, itemvalue) return value.ensure_nonnull() self.emit_operation(op) From noreply at buildbot.pypy.org Sun Mar 16 23:20:19 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Sun, 16 Mar 2014 23:20:19 +0100 (CET) Subject: [pypy-commit] pypy virtual-raw-store-load: close branch for merging Message-ID: <20140316222019.D83031C0185@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: virtual-raw-store-load Changeset: r69985:82624ea198f6 Date: 2014-03-16 18:11 -0400 http://bitbucket.org/pypy/pypy/changeset/82624ea198f6/ Log: close branch for merging From noreply at buildbot.pypy.org Sun Mar 16 23:20:21 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Sun, 16 Mar 2014 23:20:21 +0100 (CET) Subject: [pypy-commit] pypy default: merge virtual-raw-store-load Message-ID: <20140316222021.2A2D41C0185@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r69986:9fca5c72c344 Date: 2014-03-16 18:11 -0400 http://bitbucket.org/pypy/pypy/changeset/9fca5c72c344/ Log: merge virtual-raw-store-load diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -1894,6 +1894,40 @@ """ self.optimize_loop(ops, expected) + def test_virtual_raw_store_raw_load(self): + ops = """ + [i1] + i0 = call('malloc', 10, descr=raw_malloc_descr) + raw_store(i0, 0, i1, descr=rawarraydescr) + i2 = raw_load(i0, 0, descr=rawarraydescr) + i3 = int_add(i1, i2) + call('free', i0, descr=raw_free_descr) + jump(i3) + """ + expected = """ + [i1] + i2 = int_add(i1, i1) + jump(i2) + """ + self.optimize_loop(ops, expected) + + def test_virtual_raw_store_getarrayitem_raw(self): + ops = """ + [f1] + i0 = call('malloc', 16, descr=raw_malloc_descr) + raw_store(i0, 8, f1, descr=rawarraydescr_float) + f2 = getarrayitem_raw(i0, 1, descr=rawarraydescr_float) + f3 = float_add(f1, f2) + call('free', i0, descr=raw_free_descr) + jump(f3) + """ + expected = """ + [f1] + f2 = float_add(f1, f1) + jump(f2) + """ + self.optimize_loop(ops, expected) + def test_duplicate_getfield_1(self): ops = """ [p1, p2] diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py --- a/rpython/jit/metainterp/optimizeopt/test/test_util.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py @@ -226,6 +226,8 @@ hints={'nolength': True})) rawarraydescr_char = cpu.arraydescrof(lltype.Array(lltype.Char, hints={'nolength': True})) + rawarraydescr_float = cpu.arraydescrof(lltype.Array(lltype.Float, + hints={'nolength': True})) fc_array = lltype.GcArray( lltype.Struct( diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py --- a/rpython/jit/metainterp/optimizeopt/virtualize.py +++ b/rpython/jit/metainterp/optimizeopt/virtualize.py @@ -779,11 +779,12 @@ offset, itemsize, descr = self._unpack_arrayitem_raw_op(op, indexbox) try: itemvalue = value.getitem_raw(offset, itemsize, descr) - self.make_equal_to(op.result, itemvalue) except InvalidRawOperation: box = value.force_box(self) op.setarg(0, box) self.emit_operation(op) + else: + self.make_equal_to(op.result, itemvalue) return value.ensure_nonnull() self.emit_operation(op) @@ -805,6 +806,48 @@ value.ensure_nonnull() self.emit_operation(op) + def _unpack_raw_load_store_op(self, op, offsetbox): + offset = offsetbox.getint() + cpu = self.optimizer.cpu + descr = op.getdescr() + itemsize = cpu.unpack_arraydescr_size(descr)[1] + return offset, itemsize, descr + + def optimize_RAW_LOAD(self, op): + value = self.getvalue(op.getarg(0)) + if value.is_virtual(): + offsetbox = self.get_constant_box(op.getarg(1)) + if offsetbox is not None: + offset, itemsize, descr = self._unpack_raw_load_store_op(op, offsetbox) + try: + itemvalue = value.getitem_raw(offset, itemsize, descr) + except InvalidRawOperation: + box = value.force_box(self) + op.setarg(0, box) + self.emit_operation(op) + else: + self.make_equal_to(op.result, itemvalue) + return + value.ensure_nonnull() + self.emit_operation(op) + + def optimize_RAW_STORE(self, op): + value = self.getvalue(op.getarg(0)) + if value.is_virtual(): + offsetbox = self.get_constant_box(op.getarg(1)) + if offsetbox is not None: + offset, itemsize, descr = self._unpack_raw_load_store_op(op, offsetbox) + itemvalue = self.getvalue(op.getarg(2)) + try: + value.setitem_raw(offset, itemsize, descr, itemvalue) + except InvalidRawOperation: + box = value.force_box(self) + op.setarg(0, box) + self.emit_operation(op) + return + value.ensure_nonnull() + self.emit_operation(op) + def optimize_GETINTERIORFIELD_GC(self, op): value = self.getvalue(op.getarg(0)) if value.is_virtual(): From noreply at buildbot.pypy.org Sun Mar 16 23:20:22 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Sun, 16 Mar 2014 23:20:22 +0100 (CET) Subject: [pypy-commit] pypy default: fix whatsnew Message-ID: <20140316222022.63E651C0185@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r69987:4428f691da85 Date: 2014-03-16 18:12 -0400 http://bitbucket.org/pypy/pypy/changeset/4428f691da85/ Log: fix whatsnew diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -105,3 +105,6 @@ .. branch: stdlib-2.7.6 Update stdlib to v2.7.6 + +.. branch: virtual-raw-store-load +Support for virtualizing raw_store/raw_load operations From noreply at buildbot.pypy.org Sun Mar 16 23:20:23 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Sun, 16 Mar 2014 23:20:23 +0100 (CET) Subject: [pypy-commit] pypy default: fix test_zjit Message-ID: <20140316222023.94D751C0185@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r69988:297062a586c6 Date: 2014-03-16 18:16 -0400 http://bitbucket.org/pypy/pypy/changeset/297062a586c6/ Log: fix test_zjit diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py --- a/pypy/module/micronumpy/test/test_zjit.py +++ b/pypy/module/micronumpy/test/test_zjit.py @@ -526,7 +526,7 @@ 'guard_class': 4, 'guard_false': 2, 'guard_no_exception': 3, - 'guard_nonnull': 8, + 'guard_nonnull': 12, 'guard_nonnull_class': 4, 'guard_not_invalidated': 2, 'guard_true': 9, From noreply at buildbot.pypy.org Mon Mar 17 00:03:39 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 17 Mar 2014 00:03:39 +0100 (CET) Subject: [pypy-commit] pypy default: skipped tests for alloc_raw_storage optimization Message-ID: <20140316230339.0F83B1D289D@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r69989:6bfb88a291c0 Date: 2014-03-16 19:02 -0400 http://bitbucket.org/pypy/pypy/changeset/6bfb88a291c0/ Log: skipped tests for alloc_raw_storage optimization diff --git a/rpython/jit/metainterp/test/test_rawmem.py b/rpython/jit/metainterp/test/test_rawmem.py --- a/rpython/jit/metainterp/test/test_rawmem.py +++ b/rpython/jit/metainterp/test/test_rawmem.py @@ -1,3 +1,4 @@ +import pytest from rpython.jit.metainterp.test.support import LLJitMixin from rpython.rtyper.lltypesystem import lltype, rffi from rpython.rlib.rawstorage import (alloc_raw_storage, raw_storage_setitem, @@ -45,6 +46,10 @@ self.check_operations_history({'call': 2, 'guard_no_exception': 1, 'raw_store': 1, 'raw_load': 1, 'finish': 1}) + pytest.skip("XXX alloc_raw_storage doesn't generate virtualizable raw buffer") + self.metainterp.staticdata.stats.check_resops( + {'guard_no_exception': 1, 'finish': 1}, + omit_finish=False) def test_raw_storage_float(self): def f(): @@ -58,6 +63,10 @@ self.check_operations_history({'call': 2, 'guard_no_exception': 1, 'raw_store': 1, 'raw_load': 1, 'finish': 1}) + pytest.skip("XXX alloc_raw_storage doesn't generate virtualizable raw buffer") + self.metainterp.staticdata.stats.check_resops( + {'guard_no_exception': 1, 'finish': 1}, + omit_finish=False) def test_raw_storage_byte(self): def f(): @@ -71,6 +80,10 @@ self.check_operations_history({'call': 2, 'guard_no_exception': 1, 'raw_store': 1, 'raw_load': 1, 'finish': 1}) + pytest.skip("XXX alloc_raw_storage doesn't generate virtualizable raw buffer") + self.metainterp.staticdata.stats.check_resops( + {'guard_no_exception': 1, 'finish': 1}, + omit_finish=False) class TestRawMem(RawMemTests, LLJitMixin): From noreply at buildbot.pypy.org Mon Mar 17 00:21:19 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 17 Mar 2014 00:21:19 +0100 (CET) Subject: [pypy-commit] pypy default: also test alloc_raw_storage with options Message-ID: <20140316232119.423D91C011F@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r69990:c25774fb2035 Date: 2014-03-16 19:17 -0400 http://bitbucket.org/pypy/pypy/changeset/c25774fb2035/ Log: also test alloc_raw_storage with options diff --git a/rpython/jit/metainterp/test/test_rawmem.py b/rpython/jit/metainterp/test/test_rawmem.py --- a/rpython/jit/metainterp/test/test_rawmem.py +++ b/rpython/jit/metainterp/test/test_rawmem.py @@ -85,6 +85,23 @@ {'guard_no_exception': 1, 'finish': 1}, omit_finish=False) + def test_raw_storage_options(self): + def f(): + p = alloc_raw_storage(15, track_allocation=False, zero=True) + raw_storage_setitem(p, 3, 24) + res = raw_storage_getitem(lltype.Signed, p, 3) + free_raw_storage(p, track_allocation=False) + return res + res = self.interp_operations(f, []) + assert res == 24 + self.check_operations_history({'call': 2, 'guard_no_exception': 1, + 'raw_store': 1, 'raw_load': 1, + 'finish': 1}) + pytest.skip("XXX alloc_raw_storage doesn't generate virtualizable raw buffer") + self.metainterp.staticdata.stats.check_resops( + {'guard_no_exception': 1, 'finish': 1}, + omit_finish=False) + class TestRawMem(RawMemTests, LLJitMixin): From noreply at buildbot.pypy.org Mon Mar 17 00:52:46 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 17 Mar 2014 00:52:46 +0100 (CET) Subject: [pypy-commit] pypy default: fix test_raw_malloc_resume Message-ID: <20140316235246.17F041C011F@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r69991:ff8af8813b5e Date: 2014-03-16 19:51 -0400 http://bitbucket.org/pypy/pypy/changeset/ff8af8813b5e/ Log: fix test_raw_malloc_resume diff --git a/rpython/jit/metainterp/test/test_virtual.py b/rpython/jit/metainterp/test/test_virtual.py --- a/rpython/jit/metainterp/test/test_virtual.py +++ b/rpython/jit/metainterp/test/test_virtual.py @@ -1150,7 +1150,8 @@ res = self.meta_interp(f, [10]) assert res == 55 self.check_trace_count(1) - self.check_resops(setarrayitem_raw=0, getarrayitem_raw=0) + self.check_resops(getarrayitem_raw=0, setarrayitem_raw=0, + raw_load=0, raw_store=0) def test_raw_malloc_resume(self): mydriver = JitDriver(greens=[], reds = 'auto') @@ -1171,8 +1172,9 @@ assert f(10) == 4000+55 res = self.meta_interp(f, [10]) assert res == 4000+55 - # the getarrayitem_raw is in the bridge - self.check_resops(getarrayitem_raw=1, setarrayitem_raw=0) + self.check_trace_count(2) + self.check_resops(getarrayitem_raw=0, setarrayitem_raw=0, + raw_load=0, raw_store=0) def test_raw_malloc_no_virtualstate(self): mydriver = JitDriver(greens=[], reds = 'auto') From noreply at buildbot.pypy.org Mon Mar 17 02:01:19 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 17 Mar 2014 02:01:19 +0100 (CET) Subject: [pypy-commit] pypy default: cleanup Message-ID: <20140317010119.C930C1C0670@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r69992:23bf2ee9e818 Date: 2014-03-16 20:43 -0400 http://bitbucket.org/pypy/pypy/changeset/23bf2ee9e818/ Log: cleanup diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py --- a/rpython/jit/codewriter/jtransform.py +++ b/rpython/jit/codewriter/jtransform.py @@ -835,8 +835,8 @@ RESULT = lltype.Ptr(STRUCT) assert RESULT == op.result.concretetype return self._do_builtin_call(op, 'alloc_with_del', [], - extra = (RESULT, vtable), - extrakey = STRUCT) + extra=(RESULT, vtable), + extrakey=STRUCT) heaptracker.register_known_gctype(self.cpu, vtable, STRUCT) opname = 'new_with_vtable' else: @@ -1235,7 +1235,7 @@ op1 = self.prepare_builtin_call(op, "llong_%s", args) op2 = self._handle_oopspec_call(op1, args, EffectInfo.OS_LLONG_%s, - EffectInfo.EF_ELIDABLE_CANNOT_RAISE) + EffectInfo.EF_ELIDABLE_CANNOT_RAISE) if %r == "TO_INT": assert op2.result.concretetype == lltype.Signed return op2 @@ -1267,7 +1267,7 @@ op1 = self.prepare_builtin_call(op, "ullong_%s", args) op2 = self._handle_oopspec_call(op1, args, EffectInfo.OS_LLONG_%s, - EffectInfo.EF_ELIDABLE_CANNOT_RAISE) + EffectInfo.EF_ELIDABLE_CANNOT_RAISE) return op2 ''' % (_op, _oopspec.lower(), _oopspec)).compile() From noreply at buildbot.pypy.org Mon Mar 17 02:29:41 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 17 Mar 2014 02:29:41 +0100 (CET) Subject: [pypy-commit] pypy default: ensure the guard_no_exception is removed after making a virtual raw buffer Message-ID: <20140317012941.CA2C01C357E@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r69993:3dfe9fa95df6 Date: 2014-03-16 21:22 -0400 http://bitbucket.org/pypy/pypy/changeset/3dfe9fa95df6/ Log: ensure the guard_no_exception is removed after making a virtual raw buffer diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py --- a/rpython/jit/metainterp/optimizeopt/virtualize.py +++ b/rpython/jit/metainterp/optimizeopt/virtualize.py @@ -705,6 +705,7 @@ return size = sizebox.value self.make_virtual_raw_memory(size, op.result, op) + self.last_emitted_operation = REMOVED def do_RAW_FREE(self, op): value = self.getvalue(op.getarg(1)) diff --git a/rpython/jit/metainterp/test/test_rawmem.py b/rpython/jit/metainterp/test/test_rawmem.py --- a/rpython/jit/metainterp/test/test_rawmem.py +++ b/rpython/jit/metainterp/test/test_rawmem.py @@ -47,9 +47,7 @@ 'raw_store': 1, 'raw_load': 1, 'finish': 1}) pytest.skip("XXX alloc_raw_storage doesn't generate virtualizable raw buffer") - self.metainterp.staticdata.stats.check_resops( - {'guard_no_exception': 1, 'finish': 1}, - omit_finish=False) + self.metainterp.staticdata.stats.check_resops({'finish': 1}, omit_finish=False) def test_raw_storage_float(self): def f(): @@ -64,9 +62,7 @@ 'raw_store': 1, 'raw_load': 1, 'finish': 1}) pytest.skip("XXX alloc_raw_storage doesn't generate virtualizable raw buffer") - self.metainterp.staticdata.stats.check_resops( - {'guard_no_exception': 1, 'finish': 1}, - omit_finish=False) + self.metainterp.staticdata.stats.check_resops({'finish': 1}, omit_finish=False) def test_raw_storage_byte(self): def f(): @@ -81,9 +77,7 @@ 'raw_store': 1, 'raw_load': 1, 'finish': 1}) pytest.skip("XXX alloc_raw_storage doesn't generate virtualizable raw buffer") - self.metainterp.staticdata.stats.check_resops( - {'guard_no_exception': 1, 'finish': 1}, - omit_finish=False) + self.metainterp.staticdata.stats.check_resops({'finish': 1}, omit_finish=False) def test_raw_storage_options(self): def f(): @@ -98,9 +92,7 @@ 'raw_store': 1, 'raw_load': 1, 'finish': 1}) pytest.skip("XXX alloc_raw_storage doesn't generate virtualizable raw buffer") - self.metainterp.staticdata.stats.check_resops( - {'guard_no_exception': 1, 'finish': 1}, - omit_finish=False) + self.metainterp.staticdata.stats.check_resops({'finish': 1}, omit_finish=False) class TestRawMem(RawMemTests, LLJitMixin): diff --git a/rpython/jit/metainterp/test/test_virtual.py b/rpython/jit/metainterp/test/test_virtual.py --- a/rpython/jit/metainterp/test/test_virtual.py +++ b/rpython/jit/metainterp/test/test_virtual.py @@ -1150,8 +1150,7 @@ res = self.meta_interp(f, [10]) assert res == 55 self.check_trace_count(1) - self.check_resops(getarrayitem_raw=0, setarrayitem_raw=0, - raw_load=0, raw_store=0) + self.check_resops({'guard_true': 2, 'int_add': 4, 'int_lt': 2, 'jump': 1}) def test_raw_malloc_resume(self): mydriver = JitDriver(greens=[], reds = 'auto') @@ -1173,8 +1172,9 @@ res = self.meta_interp(f, [10]) assert res == 4000+55 self.check_trace_count(2) - self.check_resops(getarrayitem_raw=0, setarrayitem_raw=0, - raw_load=0, raw_store=0) + self.check_resops({'guard_false': 2, 'guard_true': 5, + 'int_add': 8, 'int_gt': 3, 'int_lt': 4, 'int_mul': 2, + 'jump': 2}) def test_raw_malloc_no_virtualstate(self): mydriver = JitDriver(greens=[], reds = 'auto') From noreply at buildbot.pypy.org Mon Mar 17 03:53:07 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 17 Mar 2014 03:53:07 +0100 (CET) Subject: [pypy-commit] pypy default: include guard_no_exception after raw malloc calls in tests Message-ID: <20140317025307.895881C0185@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r69994:299fc1410b29 Date: 2014-03-16 22:51 -0400 http://bitbucket.org/pypy/pypy/changeset/299fc1410b29/ Log: include guard_no_exception after raw malloc calls in tests diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -1729,6 +1729,7 @@ ops = """ [i1] i2 = call('malloc', 10, descr=raw_malloc_descr) + guard_no_exception() [] setarrayitem_raw(i2, 0, i1, descr=rawarraydescr) i3 = getarrayitem_raw(i2, 0, descr=rawarraydescr) call('free', i2, descr=raw_free_descr) @@ -1744,6 +1745,7 @@ ops = """ [i1] i2 = call('malloc', 10, descr=raw_malloc_descr) + guard_no_exception() [] setarrayitem_raw(i2, 0, i1, descr=rawarraydescr_char) setarrayitem_raw(i2, 2, 456, descr=rawarraydescr_char) setarrayitem_raw(i2, 1, 123, descr=rawarraydescr_char) @@ -1756,6 +1758,7 @@ [i1] label('foo') i2 = call('malloc', 10, descr=raw_malloc_descr) + guard_no_exception() [] setarrayitem_raw(i2, 0, i1, descr=rawarraydescr_char) i3 = int_add(i2, 1) setarrayitem_raw(i3, 0, 123, descr=rawarraydescr_char) @@ -1771,6 +1774,7 @@ ops = """ [i1] i2 = call('malloc', 10, descr=raw_malloc_descr) + guard_no_exception() [] setarrayitem_raw(i2, 0, i1, descr=rawarraydescr) label('foo') # we expect the buffer to be forced *after* the label setarrayitem_raw(i2, 2, 456, descr=rawarraydescr_char) # overlap! @@ -1781,6 +1785,7 @@ [i1] label('foo') i2 = call('malloc', 10, descr=raw_malloc_descr) + guard_no_exception() [] setarrayitem_raw(i2, 0, i1, descr=rawarraydescr) setarrayitem_raw(i2, 2, 456, descr=rawarraydescr_char) call('free', i2, descr=raw_free_descr) @@ -1792,6 +1797,7 @@ ops = """ [i1] i2 = call('malloc', 10, descr=raw_malloc_descr) + guard_no_exception() [] setarrayitem_raw(i2, 0, i1, descr=rawarraydescr) label('foo') # we expect the buffer to be forced *after* the label i3 = getarrayitem_raw(i2, 0, descr=rawarraydescr_char) @@ -1802,6 +1808,7 @@ [i1] label('foo') i2 = call('malloc', 10, descr=raw_malloc_descr) + guard_no_exception() [] setarrayitem_raw(i2, 0, i1, descr=rawarraydescr) i3 = getarrayitem_raw(i2, 0, descr=rawarraydescr_char) call('free', i2, descr=raw_free_descr) @@ -1813,6 +1820,7 @@ ops = """ [i0, i1] i2 = call('malloc', 10, descr=raw_malloc_descr) + guard_no_exception() [] setarrayitem_raw(i2, 0, 42, descr=rawarraydescr_char) i3 = int_add(i2, 1) # get a slice of the original buffer setarrayitem_raw(i3, 0, 4242, descr=rawarraydescr) # write to the slice @@ -1832,6 +1840,7 @@ ops = """ [i0, i1] i2 = call('malloc', 10, descr=raw_malloc_descr) + guard_no_exception() [] i3 = int_add(i2, 1) # get a slice of the original buffer i4 = int_add(i3, 1) # get a slice of a slice setarrayitem_raw(i4, 0, i1, descr=rawarraydescr_char) # write to the slice @@ -1849,6 +1858,7 @@ ops = """ [i0, i1] i2 = call('malloc', 10, descr=raw_malloc_descr) + guard_no_exception() [] setarrayitem_raw(i2, 0, 42, descr=rawarraydescr_char) i3 = int_add(i2, 1) # get a slice of the original buffer setarrayitem_raw(i3, 4, 4242, descr=rawarraydescr_char) # write to the slice @@ -1861,6 +1871,7 @@ label('foo') # these ops are generated by VirtualRawBufferValue._really_force i2 = call('malloc', 10, descr=raw_malloc_descr) + guard_no_exception() [] setarrayitem_raw(i2, 0, 42, descr=rawarraydescr_char) i3 = int_add(i2, 5) # 1+4*sizeof(char) setarrayitem_raw(i3, 0, 4242, descr=rawarraydescr_char) @@ -1878,6 +1889,7 @@ i2 = int_add(i1, 1) call('free', i0, descr=raw_free_descr) i3 = call('malloc', 10, descr=raw_malloc_descr) + guard_no_exception() [] setarrayitem_raw(i3, 0, i2, descr=rawarraydescr) label('foo') jump(i3) @@ -1889,6 +1901,7 @@ call('free', i0, descr=raw_free_descr) label('foo') i3 = call('malloc', 10, descr=raw_malloc_descr) + guard_no_exception() [] setarrayitem_raw(i3, 0, i2, descr=rawarraydescr) jump(i3) """ @@ -1898,6 +1911,7 @@ ops = """ [i1] i0 = call('malloc', 10, descr=raw_malloc_descr) + guard_no_exception() [] raw_store(i0, 0, i1, descr=rawarraydescr) i2 = raw_load(i0, 0, descr=rawarraydescr) i3 = int_add(i1, i2) @@ -1915,6 +1929,7 @@ ops = """ [f1] i0 = call('malloc', 16, descr=raw_malloc_descr) + guard_no_exception() [] raw_store(i0, 8, f1, descr=rawarraydescr_float) f2 = getarrayitem_raw(i0, 1, descr=rawarraydescr_float) f3 = float_add(f1, f2) From noreply at buildbot.pypy.org Mon Mar 17 04:09:31 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 17 Mar 2014 04:09:31 +0100 (CET) Subject: [pypy-commit] pypy default: set oopspecindex for all versions of raw_malloc_varsize_char and raw_free Message-ID: <20140317030931.A18281C011F@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r69995:d026840bbc03 Date: 2014-03-16 23:06 -0400 http://bitbucket.org/pypy/pypy/changeset/d026840bbc03/ Log: set oopspecindex for all versions of raw_malloc_varsize_char and raw_free diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py --- a/rpython/jit/codewriter/jtransform.py +++ b/rpython/jit/codewriter/jtransform.py @@ -555,7 +555,7 @@ name += '_no_track_allocation' TYPE = op.args[0].value op1 = self.prepare_builtin_call(op, name, args, (TYPE,), TYPE) - if name == 'raw_malloc_varsize' and TYPE.OF == lltype.Char: + if name.startswith('raw_malloc_varsize') and TYPE.OF == lltype.Char: return self._handle_oopspec_call(op1, args, EffectInfo.OS_RAW_MALLOC_VARSIZE_CHAR, EffectInfo.EF_CAN_RAISE) @@ -589,7 +589,7 @@ name += '_no_track_allocation' op1 = self.prepare_builtin_call(op, name, [op.args[0]], (STRUCT,), STRUCT) - if name == 'raw_free': + if name.startswith('raw_free'): return self._handle_oopspec_call(op1, [op.args[0]], EffectInfo.OS_RAW_FREE, EffectInfo.EF_CANNOT_RAISE) diff --git a/rpython/jit/codewriter/test/test_jtransform.py b/rpython/jit/codewriter/test/test_jtransform.py --- a/rpython/jit/codewriter/test/test_jtransform.py +++ b/rpython/jit/codewriter/test/test_jtransform.py @@ -60,7 +60,7 @@ class FakeResidualCallControl: def guess_call_kind(self, op): return 'residual' - def getcalldescr(self, op, **kwds): + def getcalldescr(self, op, oopspecindex=None, extraeffect=None): return 'calldescr' def calldescr_canraise(self, calldescr): return True diff --git a/rpython/jit/metainterp/test/test_rawmem.py b/rpython/jit/metainterp/test/test_rawmem.py --- a/rpython/jit/metainterp/test/test_rawmem.py +++ b/rpython/jit/metainterp/test/test_rawmem.py @@ -1,4 +1,3 @@ -import pytest from rpython.jit.metainterp.test.support import LLJitMixin from rpython.rtyper.lltypesystem import lltype, rffi from rpython.rlib.rawstorage import (alloc_raw_storage, raw_storage_setitem, @@ -46,7 +45,6 @@ self.check_operations_history({'call': 2, 'guard_no_exception': 1, 'raw_store': 1, 'raw_load': 1, 'finish': 1}) - pytest.skip("XXX alloc_raw_storage doesn't generate virtualizable raw buffer") self.metainterp.staticdata.stats.check_resops({'finish': 1}, omit_finish=False) def test_raw_storage_float(self): @@ -61,7 +59,6 @@ self.check_operations_history({'call': 2, 'guard_no_exception': 1, 'raw_store': 1, 'raw_load': 1, 'finish': 1}) - pytest.skip("XXX alloc_raw_storage doesn't generate virtualizable raw buffer") self.metainterp.staticdata.stats.check_resops({'finish': 1}, omit_finish=False) def test_raw_storage_byte(self): @@ -76,7 +73,6 @@ self.check_operations_history({'call': 2, 'guard_no_exception': 1, 'raw_store': 1, 'raw_load': 1, 'finish': 1}) - pytest.skip("XXX alloc_raw_storage doesn't generate virtualizable raw buffer") self.metainterp.staticdata.stats.check_resops({'finish': 1}, omit_finish=False) def test_raw_storage_options(self): @@ -91,7 +87,6 @@ self.check_operations_history({'call': 2, 'guard_no_exception': 1, 'raw_store': 1, 'raw_load': 1, 'finish': 1}) - pytest.skip("XXX alloc_raw_storage doesn't generate virtualizable raw buffer") self.metainterp.staticdata.stats.check_resops({'finish': 1}, omit_finish=False) From noreply at buildbot.pypy.org Mon Mar 17 04:23:46 2014 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Mon, 17 Mar 2014 04:23:46 +0100 (CET) Subject: [pypy-commit] pypy default: Support creating subclasses of _socket (who knows why) Message-ID: <20140317032346.BB49B1C0185@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: Changeset: r69996:f777f3fcf74a Date: 2014-03-16 20:21 -0700 http://bitbucket.org/pypy/pypy/changeset/f777f3fcf74a/ Log: Support creating subclasses of _socket (who knows why) diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py --- a/pypy/module/_socket/interp_socket.py +++ b/pypy/module/_socket/interp_socket.py @@ -536,12 +536,9 @@ @unwrap_spec(family=int, type=int, proto=int) def newsocket(space, w_subtype, family=AF_INET, type=SOCK_STREAM, proto=0): - # XXX If we want to support subclassing the socket type we will need - # something along these lines. But allocate_instance is only defined - # on the standard object space, so this is not really correct. - #sock = space.allocate_instance(W_RSocket, w_subtype) - #Socket.__init__(sock, space, fd, family, type, proto) + sock = space.allocate_instance(W_RSocket, w_subtype) try: + W_RSocket.__init__(sock, family, type, proto) sock = W_RSocket(family, type, proto) except SocketError, e: raise converted_error(space, e) From noreply at buildbot.pypy.org Mon Mar 17 04:23:48 2014 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Mon, 17 Mar 2014 04:23:48 +0100 (CET) Subject: [pypy-commit] pypy default: merged upstream Message-ID: <20140317032348.4CF471C0185@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: Changeset: r69997:039dd41e5153 Date: 2014-03-16 20:23 -0700 http://bitbucket.org/pypy/pypy/changeset/039dd41e5153/ Log: merged upstream diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -105,3 +105,6 @@ .. branch: stdlib-2.7.6 Update stdlib to v2.7.6 + +.. branch: virtual-raw-store-load +Support for virtualizing raw_store/raw_load operations diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py --- a/pypy/module/micronumpy/test/test_zjit.py +++ b/pypy/module/micronumpy/test/test_zjit.py @@ -526,7 +526,7 @@ 'guard_class': 4, 'guard_false': 2, 'guard_no_exception': 3, - 'guard_nonnull': 8, + 'guard_nonnull': 12, 'guard_nonnull_class': 4, 'guard_not_invalidated': 2, 'guard_true': 9, diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py --- a/rpython/jit/codewriter/jtransform.py +++ b/rpython/jit/codewriter/jtransform.py @@ -547,20 +547,18 @@ track_allocation = d.pop('track_allocation', True) if d: raise UnsupportedMallocFlags(d) - TYPE = op.args[0].value if zero: name += '_zero' if add_memory_pressure: name += '_add_memory_pressure' if not track_allocation: name += '_no_track_allocation' + TYPE = op.args[0].value op1 = self.prepare_builtin_call(op, name, args, (TYPE,), TYPE) - if name == 'raw_malloc_varsize': - ITEMTYPE = op.args[0].value.OF - if ITEMTYPE == lltype.Char: - return self._handle_oopspec_call(op1, args, - EffectInfo.OS_RAW_MALLOC_VARSIZE_CHAR, - EffectInfo.EF_CAN_RAISE) + if name.startswith('raw_malloc_varsize') and TYPE.OF == lltype.Char: + return self._handle_oopspec_call(op1, args, + EffectInfo.OS_RAW_MALLOC_VARSIZE_CHAR, + EffectInfo.EF_CAN_RAISE) return self.rewrite_op_direct_call(op1) def rewrite_op_malloc_varsize(self, op): @@ -591,7 +589,7 @@ name += '_no_track_allocation' op1 = self.prepare_builtin_call(op, name, [op.args[0]], (STRUCT,), STRUCT) - if name == 'raw_free': + if name.startswith('raw_free'): return self._handle_oopspec_call(op1, [op.args[0]], EffectInfo.OS_RAW_FREE, EffectInfo.EF_CANNOT_RAISE) @@ -837,8 +835,8 @@ RESULT = lltype.Ptr(STRUCT) assert RESULT == op.result.concretetype return self._do_builtin_call(op, 'alloc_with_del', [], - extra = (RESULT, vtable), - extrakey = STRUCT) + extra=(RESULT, vtable), + extrakey=STRUCT) heaptracker.register_known_gctype(self.cpu, vtable, STRUCT) opname = 'new_with_vtable' else: @@ -1237,7 +1235,7 @@ op1 = self.prepare_builtin_call(op, "llong_%s", args) op2 = self._handle_oopspec_call(op1, args, EffectInfo.OS_LLONG_%s, - EffectInfo.EF_ELIDABLE_CANNOT_RAISE) + EffectInfo.EF_ELIDABLE_CANNOT_RAISE) if %r == "TO_INT": assert op2.result.concretetype == lltype.Signed return op2 @@ -1269,7 +1267,7 @@ op1 = self.prepare_builtin_call(op, "ullong_%s", args) op2 = self._handle_oopspec_call(op1, args, EffectInfo.OS_LLONG_%s, - EffectInfo.EF_ELIDABLE_CANNOT_RAISE) + EffectInfo.EF_ELIDABLE_CANNOT_RAISE) return op2 ''' % (_op, _oopspec.lower(), _oopspec)).compile() diff --git a/rpython/jit/codewriter/test/test_jtransform.py b/rpython/jit/codewriter/test/test_jtransform.py --- a/rpython/jit/codewriter/test/test_jtransform.py +++ b/rpython/jit/codewriter/test/test_jtransform.py @@ -60,7 +60,7 @@ class FakeResidualCallControl: def guess_call_kind(self, op): return 'residual' - def getcalldescr(self, op, **kwds): + def getcalldescr(self, op, oopspecindex=None, extraeffect=None): return 'calldescr' def calldescr_canraise(self, calldescr): return True diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -1729,6 +1729,7 @@ ops = """ [i1] i2 = call('malloc', 10, descr=raw_malloc_descr) + guard_no_exception() [] setarrayitem_raw(i2, 0, i1, descr=rawarraydescr) i3 = getarrayitem_raw(i2, 0, descr=rawarraydescr) call('free', i2, descr=raw_free_descr) @@ -1744,6 +1745,7 @@ ops = """ [i1] i2 = call('malloc', 10, descr=raw_malloc_descr) + guard_no_exception() [] setarrayitem_raw(i2, 0, i1, descr=rawarraydescr_char) setarrayitem_raw(i2, 2, 456, descr=rawarraydescr_char) setarrayitem_raw(i2, 1, 123, descr=rawarraydescr_char) @@ -1756,6 +1758,7 @@ [i1] label('foo') i2 = call('malloc', 10, descr=raw_malloc_descr) + guard_no_exception() [] setarrayitem_raw(i2, 0, i1, descr=rawarraydescr_char) i3 = int_add(i2, 1) setarrayitem_raw(i3, 0, 123, descr=rawarraydescr_char) @@ -1771,6 +1774,7 @@ ops = """ [i1] i2 = call('malloc', 10, descr=raw_malloc_descr) + guard_no_exception() [] setarrayitem_raw(i2, 0, i1, descr=rawarraydescr) label('foo') # we expect the buffer to be forced *after* the label setarrayitem_raw(i2, 2, 456, descr=rawarraydescr_char) # overlap! @@ -1781,6 +1785,7 @@ [i1] label('foo') i2 = call('malloc', 10, descr=raw_malloc_descr) + guard_no_exception() [] setarrayitem_raw(i2, 0, i1, descr=rawarraydescr) setarrayitem_raw(i2, 2, 456, descr=rawarraydescr_char) call('free', i2, descr=raw_free_descr) @@ -1792,6 +1797,7 @@ ops = """ [i1] i2 = call('malloc', 10, descr=raw_malloc_descr) + guard_no_exception() [] setarrayitem_raw(i2, 0, i1, descr=rawarraydescr) label('foo') # we expect the buffer to be forced *after* the label i3 = getarrayitem_raw(i2, 0, descr=rawarraydescr_char) @@ -1802,6 +1808,7 @@ [i1] label('foo') i2 = call('malloc', 10, descr=raw_malloc_descr) + guard_no_exception() [] setarrayitem_raw(i2, 0, i1, descr=rawarraydescr) i3 = getarrayitem_raw(i2, 0, descr=rawarraydescr_char) call('free', i2, descr=raw_free_descr) @@ -1813,6 +1820,7 @@ ops = """ [i0, i1] i2 = call('malloc', 10, descr=raw_malloc_descr) + guard_no_exception() [] setarrayitem_raw(i2, 0, 42, descr=rawarraydescr_char) i3 = int_add(i2, 1) # get a slice of the original buffer setarrayitem_raw(i3, 0, 4242, descr=rawarraydescr) # write to the slice @@ -1832,6 +1840,7 @@ ops = """ [i0, i1] i2 = call('malloc', 10, descr=raw_malloc_descr) + guard_no_exception() [] i3 = int_add(i2, 1) # get a slice of the original buffer i4 = int_add(i3, 1) # get a slice of a slice setarrayitem_raw(i4, 0, i1, descr=rawarraydescr_char) # write to the slice @@ -1849,6 +1858,7 @@ ops = """ [i0, i1] i2 = call('malloc', 10, descr=raw_malloc_descr) + guard_no_exception() [] setarrayitem_raw(i2, 0, 42, descr=rawarraydescr_char) i3 = int_add(i2, 1) # get a slice of the original buffer setarrayitem_raw(i3, 4, 4242, descr=rawarraydescr_char) # write to the slice @@ -1861,6 +1871,7 @@ label('foo') # these ops are generated by VirtualRawBufferValue._really_force i2 = call('malloc', 10, descr=raw_malloc_descr) + guard_no_exception() [] setarrayitem_raw(i2, 0, 42, descr=rawarraydescr_char) i3 = int_add(i2, 5) # 1+4*sizeof(char) setarrayitem_raw(i3, 0, 4242, descr=rawarraydescr_char) @@ -1878,6 +1889,7 @@ i2 = int_add(i1, 1) call('free', i0, descr=raw_free_descr) i3 = call('malloc', 10, descr=raw_malloc_descr) + guard_no_exception() [] setarrayitem_raw(i3, 0, i2, descr=rawarraydescr) label('foo') jump(i3) @@ -1889,11 +1901,48 @@ call('free', i0, descr=raw_free_descr) label('foo') i3 = call('malloc', 10, descr=raw_malloc_descr) + guard_no_exception() [] setarrayitem_raw(i3, 0, i2, descr=rawarraydescr) jump(i3) """ self.optimize_loop(ops, expected) + def test_virtual_raw_store_raw_load(self): + ops = """ + [i1] + i0 = call('malloc', 10, descr=raw_malloc_descr) + guard_no_exception() [] + raw_store(i0, 0, i1, descr=rawarraydescr) + i2 = raw_load(i0, 0, descr=rawarraydescr) + i3 = int_add(i1, i2) + call('free', i0, descr=raw_free_descr) + jump(i3) + """ + expected = """ + [i1] + i2 = int_add(i1, i1) + jump(i2) + """ + self.optimize_loop(ops, expected) + + def test_virtual_raw_store_getarrayitem_raw(self): + ops = """ + [f1] + i0 = call('malloc', 16, descr=raw_malloc_descr) + guard_no_exception() [] + raw_store(i0, 8, f1, descr=rawarraydescr_float) + f2 = getarrayitem_raw(i0, 1, descr=rawarraydescr_float) + f3 = float_add(f1, f2) + call('free', i0, descr=raw_free_descr) + jump(f3) + """ + expected = """ + [f1] + f2 = float_add(f1, f1) + jump(f2) + """ + self.optimize_loop(ops, expected) + def test_duplicate_getfield_1(self): ops = """ [p1, p2] diff --git a/rpython/jit/metainterp/optimizeopt/test/test_rawbuffer.py b/rpython/jit/metainterp/optimizeopt/test/test_rawbuffer.py --- a/rpython/jit/metainterp/optimizeopt/test/test_rawbuffer.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_rawbuffer.py @@ -17,8 +17,7 @@ ( 4, 2, 'descr2', 'two'), ( 8, 4, 'descr3', 'three'), (12, 2, 'descr4', 'four'), - ] - # + ] def test_write_value_update(): buf = RawBuffer(FakeCPU()) @@ -28,7 +27,7 @@ assert buf._get_memory() == [ ( 0, 4, 'descr', 'ONE'), ( 4, 2, 'descr', 'two'), - ] + ] def test_write_value_invalid_length(): buf = RawBuffer(FakeCPU()) @@ -38,7 +37,6 @@ with py.test.raises(InvalidRawWrite): buf.write_value(0, 4, 'descr2', 'two') - def test_write_value_overlapping_next(): buf = RawBuffer(FakeCPU()) buf.write_value(0, 4, 'descr', 'one') diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py --- a/rpython/jit/metainterp/optimizeopt/test/test_util.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py @@ -226,6 +226,8 @@ hints={'nolength': True})) rawarraydescr_char = cpu.arraydescrof(lltype.Array(lltype.Char, hints={'nolength': True})) + rawarraydescr_float = cpu.arraydescrof(lltype.Array(lltype.Float, + hints={'nolength': True})) fc_array = lltype.GcArray( lltype.Struct( diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py --- a/rpython/jit/metainterp/optimizeopt/virtualize.py +++ b/rpython/jit/metainterp/optimizeopt/virtualize.py @@ -705,6 +705,7 @@ return size = sizebox.value self.make_virtual_raw_memory(size, op.result, op) + self.last_emitted_operation = REMOVED def do_RAW_FREE(self, op): value = self.getvalue(op.getarg(1)) @@ -779,11 +780,12 @@ offset, itemsize, descr = self._unpack_arrayitem_raw_op(op, indexbox) try: itemvalue = value.getitem_raw(offset, itemsize, descr) - self.make_equal_to(op.result, itemvalue) except InvalidRawOperation: box = value.force_box(self) op.setarg(0, box) self.emit_operation(op) + else: + self.make_equal_to(op.result, itemvalue) return value.ensure_nonnull() self.emit_operation(op) @@ -805,6 +807,48 @@ value.ensure_nonnull() self.emit_operation(op) + def _unpack_raw_load_store_op(self, op, offsetbox): + offset = offsetbox.getint() + cpu = self.optimizer.cpu + descr = op.getdescr() + itemsize = cpu.unpack_arraydescr_size(descr)[1] + return offset, itemsize, descr + + def optimize_RAW_LOAD(self, op): + value = self.getvalue(op.getarg(0)) + if value.is_virtual(): + offsetbox = self.get_constant_box(op.getarg(1)) + if offsetbox is not None: + offset, itemsize, descr = self._unpack_raw_load_store_op(op, offsetbox) + try: + itemvalue = value.getitem_raw(offset, itemsize, descr) + except InvalidRawOperation: + box = value.force_box(self) + op.setarg(0, box) + self.emit_operation(op) + else: + self.make_equal_to(op.result, itemvalue) + return + value.ensure_nonnull() + self.emit_operation(op) + + def optimize_RAW_STORE(self, op): + value = self.getvalue(op.getarg(0)) + if value.is_virtual(): + offsetbox = self.get_constant_box(op.getarg(1)) + if offsetbox is not None: + offset, itemsize, descr = self._unpack_raw_load_store_op(op, offsetbox) + itemvalue = self.getvalue(op.getarg(2)) + try: + value.setitem_raw(offset, itemsize, descr, itemvalue) + except InvalidRawOperation: + box = value.force_box(self) + op.setarg(0, box) + self.emit_operation(op) + return + value.ensure_nonnull() + self.emit_operation(op) + def optimize_GETINTERIORFIELD_GC(self, op): value = self.getvalue(op.getarg(0)) if value.is_virtual(): diff --git a/rpython/jit/metainterp/test/test_rawmem.py b/rpython/jit/metainterp/test/test_rawmem.py --- a/rpython/jit/metainterp/test/test_rawmem.py +++ b/rpython/jit/metainterp/test/test_rawmem.py @@ -1,7 +1,8 @@ from rpython.jit.metainterp.test.support import LLJitMixin from rpython.rtyper.lltypesystem import lltype, rffi from rpython.rlib.rawstorage import (alloc_raw_storage, raw_storage_setitem, - free_raw_storage, raw_storage_getitem) + free_raw_storage, raw_storage_getitem) + class RawMemTests(object): def test_cast_void_ptr(self): @@ -44,6 +45,7 @@ self.check_operations_history({'call': 2, 'guard_no_exception': 1, 'raw_store': 1, 'raw_load': 1, 'finish': 1}) + self.metainterp.staticdata.stats.check_resops({'finish': 1}, omit_finish=False) def test_raw_storage_float(self): def f(): @@ -57,6 +59,7 @@ self.check_operations_history({'call': 2, 'guard_no_exception': 1, 'raw_store': 1, 'raw_load': 1, 'finish': 1}) + self.metainterp.staticdata.stats.check_resops({'finish': 1}, omit_finish=False) def test_raw_storage_byte(self): def f(): @@ -70,6 +73,21 @@ self.check_operations_history({'call': 2, 'guard_no_exception': 1, 'raw_store': 1, 'raw_load': 1, 'finish': 1}) + self.metainterp.staticdata.stats.check_resops({'finish': 1}, omit_finish=False) + + def test_raw_storage_options(self): + def f(): + p = alloc_raw_storage(15, track_allocation=False, zero=True) + raw_storage_setitem(p, 3, 24) + res = raw_storage_getitem(lltype.Signed, p, 3) + free_raw_storage(p, track_allocation=False) + return res + res = self.interp_operations(f, []) + assert res == 24 + self.check_operations_history({'call': 2, 'guard_no_exception': 1, + 'raw_store': 1, 'raw_load': 1, + 'finish': 1}) + self.metainterp.staticdata.stats.check_resops({'finish': 1}, omit_finish=False) class TestRawMem(RawMemTests, LLJitMixin): diff --git a/rpython/jit/metainterp/test/test_virtual.py b/rpython/jit/metainterp/test/test_virtual.py --- a/rpython/jit/metainterp/test/test_virtual.py +++ b/rpython/jit/metainterp/test/test_virtual.py @@ -1150,7 +1150,7 @@ res = self.meta_interp(f, [10]) assert res == 55 self.check_trace_count(1) - self.check_resops(setarrayitem_raw=0, getarrayitem_raw=0) + self.check_resops({'guard_true': 2, 'int_add': 4, 'int_lt': 2, 'jump': 1}) def test_raw_malloc_resume(self): mydriver = JitDriver(greens=[], reds = 'auto') @@ -1171,8 +1171,10 @@ assert f(10) == 4000+55 res = self.meta_interp(f, [10]) assert res == 4000+55 - # the getarrayitem_raw is in the bridge - self.check_resops(getarrayitem_raw=1, setarrayitem_raw=0) + self.check_trace_count(2) + self.check_resops({'guard_false': 2, 'guard_true': 5, + 'int_add': 8, 'int_gt': 3, 'int_lt': 4, 'int_mul': 2, + 'jump': 2}) def test_raw_malloc_no_virtualstate(self): mydriver = JitDriver(greens=[], reds = 'auto') From noreply at buildbot.pypy.org Mon Mar 17 04:31:39 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 17 Mar 2014 04:31:39 +0100 (CET) Subject: [pypy-commit] pypy default: fix f777f3fcf74a, add socket subclass test Message-ID: <20140317033139.6CCA41C0670@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r69998:10525853b510 Date: 2014-03-16 23:30 -0400 http://bitbucket.org/pypy/pypy/changeset/10525853b510/ Log: fix f777f3fcf74a, add socket subclass test diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py --- a/pypy/module/_socket/interp_socket.py +++ b/pypy/module/_socket/interp_socket.py @@ -539,7 +539,6 @@ sock = space.allocate_instance(W_RSocket, w_subtype) try: W_RSocket.__init__(sock, family, type, proto) - sock = W_RSocket(family, type, proto) except SocketError, e: raise converted_error(space, e) return space.wrap(sock) diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py --- a/pypy/module/_socket/test/test_sock_app.py +++ b/pypy/module/_socket/test/test_sock_app.py @@ -399,7 +399,7 @@ name = s.getpeername() # Will raise socket.error if not connected assert name[1] == 80 s.close() - + def test_socket_connect_ex(self): import _socket s = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM, 0) @@ -474,6 +474,13 @@ import socket s = socket.socket() + def test_subclass(self): + from _socket import socket + class MySock(socket): + blah = 123 + s = MySock() + assert s.blah == 123 + def test_getsetsockopt(self): import _socket as socket import struct From noreply at buildbot.pypy.org Mon Mar 17 04:43:48 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 17 Mar 2014 04:43:48 +0100 (CET) Subject: [pypy-commit] pypy default: set socket buffer size so untranslated tests finish in reasonable time Message-ID: <20140317034348.E52471C11A4@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r69999:cc839a8262d5 Date: 2014-03-16 23:42 -0400 http://bitbucket.org/pypy/pypy/changeset/cc839a8262d5/ Log: set socket buffer size so untranslated tests finish in reasonable time diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py --- a/pypy/module/_socket/test/test_sock_app.py +++ b/pypy/module/_socket/test/test_sock_app.py @@ -582,11 +582,11 @@ class AppTestSocketTCP: + HOST = 'localhost' + def setup_class(cls): cls.space = space - HOST = 'localhost' - def setup_method(self, method): w_HOST = space.wrap(self.HOST) self.w_serv = space.appexec([w_socket, w_HOST], @@ -596,6 +596,7 @@ serv.listen(1) return serv ''') + def teardown_method(self, method): if hasattr(self, 'w_serv'): space.appexec([self.w_serv], '(serv): serv.close()') @@ -616,7 +617,7 @@ raises(error, raise_error) def test_recv_send_timeout(self): - from _socket import socket, timeout + from _socket import socket, timeout, SOL_SOCKET, SO_RCVBUF, SO_SNDBUF cli = socket() cli.connect(self.serv.getsockname()) t, addr = self.serv.accept() @@ -636,6 +637,9 @@ assert count is None buf = t.recv(1) assert buf == '?' + # speed up filling the buffers + t.setsockopt(SOL_SOCKET, SO_RCVBUF, 4096) + cli.setsockopt(SOL_SOCKET, SO_SNDBUF, 4096) # test send() timeout count = 0 try: @@ -663,7 +667,7 @@ conn, addr = self.serv.accept() buf = buffer(MSG) conn.send(buf) - buf = array.array('c', ' '*1024) + buf = array.array('c', ' ' * 1024) nbytes = cli.recv_into(buf) assert nbytes == len(MSG) msg = buf.tostring()[:len(MSG)] @@ -678,7 +682,7 @@ conn, addr = self.serv.accept() buf = buffer(MSG) conn.send(buf) - buf = array.array('c', ' '*1024) + buf = array.array('c', ' ' * 1024) nbytes, addr = cli.recvfrom_into(buf) assert nbytes == len(MSG) msg = buf.tostring()[:len(MSG)] @@ -689,6 +693,7 @@ cli = socket.socket(socket.AF_INET, socket.SOCK_STREAM) assert cli.family == socket.AF_INET + class AppTestErrno: def setup_class(cls): cls.space = space From noreply at buildbot.pypy.org Mon Mar 17 05:30:44 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 17 Mar 2014 05:30:44 +0100 (CET) Subject: [pypy-commit] pypy default: update test_flat_setitem Message-ID: <20140317043044.690571C0185@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r70000:40755c7bd64c Date: 2014-03-17 00:28 -0400 http://bitbucket.org/pypy/pypy/changeset/40755c7bd64c/ Log: update test_flat_setitem diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py --- a/pypy/module/micronumpy/test/test_zjit.py +++ b/pypy/module/micronumpy/test/test_zjit.py @@ -480,23 +480,19 @@ def test_flat_setitem(self): result = self.run("flat_setitem") assert result == 1.0 - py.test.skip("don't run for now") self.check_trace_count(1) - # XXX not ideal, but hey, let's ignore it for now - self.check_simple_loop({'raw_load': 1, - 'raw_store': 1, - 'int_lt': 1, - 'int_gt': 1, - 'int_add': 4, - 'guard_true': 2, - 'arraylen_gc': 2, - 'jump': 1, - 'int_sub': 1, - # XXX bad part - 'int_and': 1, - 'int_mod': 1, - 'int_rshift': 1, - }) + self.check_simple_loop({ + 'call': 2, + 'getfield_gc': 2, + 'guard_no_exception': 2, + 'guard_not_invalidated': 1, + 'guard_true': 1, + 'int_gt': 1, + 'int_sub': 1, + 'jump': 1, + 'raw_load': 1, + 'raw_store': 1, + }) def define_dot(): return """ @@ -509,6 +505,7 @@ def test_dot(self): result = self.run("dot") assert result == 184 + self.check_trace_count(3) self.check_simple_loop({'float_add': 1, 'float_mul': 1, 'guard_not_invalidated': 1, From noreply at buildbot.pypy.org Mon Mar 17 05:54:41 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 17 Mar 2014 05:54:41 +0100 (CET) Subject: [pypy-commit] pypy default: fix socket exception type for overflowing port/flowinfo Message-ID: <20140317045442.005A41C0670@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r70001:13377182ed09 Date: 2014-03-17 00:53 -0400 http://bitbucket.org/pypy/pypy/changeset/13377182ed09/ Log: fix socket exception type for overflowing port/flowinfo diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py --- a/pypy/module/_socket/interp_socket.py +++ b/pypy/module/_socket/interp_socket.py @@ -65,10 +65,7 @@ else: flowinfo = 0 if len(pieces_w) > 3: scope_id = space.uint_w(pieces_w[3]) else: scope_id = 0 - if flowinfo < 0 or flowinfo > 0xfffff: - raise OperationError(space.w_OverflowError, space.wrap( - "flowinfo must be 0-1048575.")) - flowinfo = rffi.cast(lltype.Unsigned, flowinfo) + flowinfo = make_unsigned_flowinfo(space, flowinfo) a = addr.lock(_c.sockaddr_in6) rffi.setintfield(a, 'c_sin6_port', rsocket.htons(port)) rffi.setintfield(a, 'c_sin6_flowinfo', rsocket.htonl(flowinfo)) @@ -97,10 +94,7 @@ else: flowinfo = 0 if len(pieces_w) > 3: scope_id = space.uint_w(pieces_w[3]) else: scope_id = 0 - if flowinfo < 0 or flowinfo > 0xfffff: - raise OperationError(space.w_OverflowError, space.wrap( - "flowinfo must be 0-1048575.")) - flowinfo = rffi.cast(lltype.Unsigned, flowinfo) + flowinfo = make_unsigned_flowinfo(space, flowinfo) return rsocket.INET6Address(host, port, flowinfo, scope_id) if rsocket.HAS_AF_UNIX and family == rsocket.AF_UNIX: return rsocket.UNIXAddress(space.str_w(w_address)) @@ -112,10 +106,16 @@ # XXX Hack to seperate rpython and pypy def make_ushort_port(space, port): if port < 0 or port > 0xffff: - raise OperationError(space.w_ValueError, space.wrap( + raise OperationError(space.w_OverflowError, space.wrap( "port must be 0-65535.")) return rffi.cast(rffi.USHORT, port) +def make_unsigned_flowinfo(space, flowinfo): + if flowinfo < 0 or flowinfo > 0xfffff: + raise OperationError(space.w_OverflowError, space.wrap( + "flowinfo must be 0-1048575.")) + return rffi.cast(lltype.Unsigned, flowinfo) + # XXX Hack to seperate rpython and pypy def ipaddr_from_object(space, w_sockaddr): host = space.str_w(space.getitem(w_sockaddr, space.wrap(0))) diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py --- a/pypy/module/_socket/test/test_sock_app.py +++ b/pypy/module/_socket/test/test_sock_app.py @@ -425,8 +425,13 @@ def test_bigport(self): import _socket s = _socket.socket() - raises(ValueError, s.connect, ("localhost", 1000000)) - raises(ValueError, s.connect, ("localhost", -1)) + exc = raises(OverflowError, s.connect, ("localhost", -1)) + assert "port must be 0-65535." in str(exc.value) + exc = raises(OverflowError, s.connect, ("localhost", 1000000)) + assert "port must be 0-65535." in str(exc.value) + s = _socket.socket(_socket.AF_INET6) + exc = raises(OverflowError, s.connect, ("::1", 1234, 1048576)) + assert "flowinfo must be 0-1048575." in str(exc.value) def test_NtoH(self): import sys From noreply at buildbot.pypy.org Mon Mar 17 06:16:41 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 17 Mar 2014 06:16:41 +0100 (CET) Subject: [pypy-commit] pypy default: fix test_pypy_c after d026840bbc03 Message-ID: <20140317051641.86F6D1C011F@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r70002:d4e0f726ebc2 Date: 2014-03-16 22:15 -0700 http://bitbucket.org/pypy/pypy/changeset/d4e0f726ebc2/ Log: fix test_pypy_c after d026840bbc03 diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py --- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py +++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py @@ -341,7 +341,7 @@ guard_value(p166, ConstPtr(ptr72), descr=...) p167 = call(ConstClass(_ll_0_alloc_with_del___), descr=) guard_no_exception(descr=...) - i168 = call(ConstClass(_ll_1_raw_malloc_varsize__Signed), 6, descr=) + i168 = call(ConstClass(_ll_1_raw_malloc_varsize__Signed), 6, descr=) setfield_gc(p167, 0, descr=) setfield_gc(p167, ConstPtr(ptr86), descr=) guard_no_exception(descr=...) From noreply at buildbot.pypy.org Mon Mar 17 07:52:13 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 17 Mar 2014 07:52:13 +0100 (CET) Subject: [pypy-commit] pypy default: fix virtual raw malloc for all constant args Message-ID: <20140317065213.411BB1C0166@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r70003:b81700f91370 Date: 2014-03-17 02:05 -0400 http://bitbucket.org/pypy/pypy/changeset/b81700f91370/ Log: fix virtual raw malloc for all constant args diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -1725,7 +1725,7 @@ # We cannot track virtuals that survive for more than two iterations. self.optimize_loop(ops, expected, preamble) - def test_virtual_raw_malloc(self): + def test_virtual_raw_malloc_basic(self): ops = """ [i1] i2 = call('malloc', 10, descr=raw_malloc_descr) @@ -1741,6 +1741,23 @@ """ self.optimize_loop(ops, expected) + def test_virtual_raw_malloc_const(self): + ops = """ + [i1] + i5 = int_mul(10, 1) + i2 = call('malloc', i5, descr=raw_malloc_descr) + guard_no_exception() [] + setarrayitem_raw(i2, 0, i1, descr=rawarraydescr) + i3 = getarrayitem_raw(i2, 0, descr=rawarraydescr) + call('free', i2, descr=raw_free_descr) + jump(i3) + """ + expected = """ + [i1] + jump(i1) + """ + self.optimize_loop(ops, expected) + def test_virtual_raw_malloc_force(self): ops = """ [i1] diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py --- a/rpython/jit/metainterp/optimizeopt/virtualize.py +++ b/rpython/jit/metainterp/optimizeopt/virtualize.py @@ -699,12 +699,11 @@ self.emit_operation(op) def do_RAW_MALLOC_VARSIZE_CHAR(self, op): - sizebox = op.getarg(1) - if not isinstance(sizebox, ConstInt): + sizebox = self.get_constant_box(op.getarg(1)) + if sizebox is None: self.emit_operation(op) return - size = sizebox.value - self.make_virtual_raw_memory(size, op.result, op) + self.make_virtual_raw_memory(sizebox.getint(), op.result, op) self.last_emitted_operation = REMOVED def do_RAW_FREE(self, op): From noreply at buildbot.pypy.org Mon Mar 17 07:52:14 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 17 Mar 2014 07:52:14 +0100 (CET) Subject: [pypy-commit] pypy default: emit_operation should handle this Message-ID: <20140317065214.AAA061C0166@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r70004:9b585f45af76 Date: 2014-03-17 02:19 -0400 http://bitbucket.org/pypy/pypy/changeset/9b585f45af76/ Log: emit_operation should handle this diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py --- a/rpython/jit/metainterp/optimizeopt/virtualize.py +++ b/rpython/jit/metainterp/optimizeopt/virtualize.py @@ -674,11 +674,6 @@ def optimize_NEW_ARRAY(self, op): sizebox = self.get_constant_box(op.getarg(0)) if sizebox is not None: - # if the original 'op' did not have a ConstInt as argument, - # build a new one with the ConstInt argument - if not isinstance(op.getarg(0), ConstInt): - op = ResOperation(rop.NEW_ARRAY, [sizebox], op.result, - descr=op.getdescr()) self.make_varray(op.getdescr(), sizebox.getint(), op.result, op) else: self.getvalue(op.result).ensure_nonnull() From noreply at buildbot.pypy.org Mon Mar 17 07:52:15 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 17 Mar 2014 07:52:15 +0100 (CET) Subject: [pypy-commit] pypy default: cleanup Message-ID: <20140317065215.EDF761C0166@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r70005:0e26ea0abf69 Date: 2014-03-17 02:45 -0400 http://bitbucket.org/pypy/pypy/changeset/0e26ea0abf69/ Log: cleanup diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -605,9 +605,9 @@ i1 = ptr_eq(p0, NULL) guard_false(i1) [] i2 = ptr_ne(NULL, p0) - guard_true(i0) [] + guard_true(i2) [] i3 = ptr_eq(NULL, p0) - guard_false(i1) [] + guard_false(i3) [] guard_nonnull(p0) [] jump(p0) """ @@ -622,6 +622,30 @@ """ self.optimize_loop(ops, expected, preamble) + def test_nonnull_2(self): + ops = """ + [] + p0 = new_array(5, descr=arraydescr) # forces p0 != NULL + i0 = ptr_ne(p0, NULL) + guard_true(i0) [] + i1 = ptr_eq(p0, NULL) + guard_false(i1) [] + i2 = ptr_ne(NULL, p0) + guard_true(i2) [] + i3 = ptr_eq(NULL, p0) + guard_false(i3) [] + guard_nonnull(p0) [] + escape(p0) + jump() + """ + expected = """ + [] + p0 = new_array(5, descr=arraydescr) + escape(p0) + jump() + """ + self.optimize_loop(ops, expected) + def test_const_guard_value(self): ops = """ [] diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py --- a/rpython/jit/metainterp/optimizeopt/virtualize.py +++ b/rpython/jit/metainterp/optimizeopt/virtualize.py @@ -676,7 +676,6 @@ if sizebox is not None: self.make_varray(op.getdescr(), sizebox.getint(), op.result, op) else: - self.getvalue(op.result).ensure_nonnull() self.emit_operation(op) def optimize_CALL(self, op): From noreply at buildbot.pypy.org Mon Mar 17 10:48:27 2014 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 17 Mar 2014 10:48:27 +0100 (CET) Subject: [pypy-commit] pypy improve-consecutive-dict-lookups: start working on optimizing consecutive dict lookups Message-ID: <20140317094827.4EA6E1C35F0@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: improve-consecutive-dict-lookups Changeset: r70006:f6665773ece4 Date: 2014-03-14 17:33 +0200 http://bitbucket.org/pypy/pypy/changeset/f6665773ece4/ Log: start working on optimizing consecutive dict lookups diff --git a/rpython/jit/codewriter/effectinfo.py b/rpython/jit/codewriter/effectinfo.py --- a/rpython/jit/codewriter/effectinfo.py +++ b/rpython/jit/codewriter/effectinfo.py @@ -21,6 +21,7 @@ OS_ARRAYCOPY = 1 # "list.ll_arraycopy" OS_STR2UNICODE = 2 # "str.str2unicode" OS_SHRINK_ARRAY = 3 # rgc.ll_shrink_array + OS_DICT_LOOKUP = 4 # ll_dict_lookup # OS_STR_CONCAT = 22 # "stroruni.concat" OS_STR_SLICE = 23 # "stroruni.slice" @@ -88,7 +89,7 @@ # for debugging: _OS_CANRAISE = set([ OS_NONE, OS_STR2UNICODE, OS_LIBFFI_CALL, OS_RAW_MALLOC_VARSIZE_CHAR, - OS_JIT_FORCE_VIRTUAL, OS_SHRINK_ARRAY, + OS_JIT_FORCE_VIRTUAL, OS_SHRINK_ARRAY, OS_DICT_LOOKUP, ]) def __new__(cls, readonly_descrs_fields, readonly_descrs_arrays, diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py --- a/rpython/jit/codewriter/jtransform.py +++ b/rpython/jit/codewriter/jtransform.py @@ -403,6 +403,8 @@ prepare = self._handle_math_sqrt_call elif oopspec_name.startswith('rgc.'): prepare = self._handle_rgc_call + elif oopspec_name == 'dict.lookup': + prepare = self._handle_dict_lookup_call else: prepare = self.prepare_builtin_call try: @@ -1848,6 +1850,10 @@ return self._handle_oopspec_call(op, args, EffectInfo.OS_MATH_SQRT, EffectInfo.EF_ELIDABLE_CANNOT_RAISE) + def _handle_dict_lookup_call(self, op, oopspec_name, args): + return self._handle_oopspec_call(op, args, EffectInfo.OS_DICT_LOOKUP, + EffectInfo.EF_CAN_RAISE) + def _handle_rgc_call(self, op, oopspec_name, args): if oopspec_name == 'rgc.ll_shrink_array': return self._handle_oopspec_call(op, args, EffectInfo.OS_SHRINK_ARRAY, EffectInfo.EF_CAN_RAISE) diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py --- a/rpython/jit/metainterp/optimizeopt/heap.py +++ b/rpython/jit/metainterp/optimizeopt/heap.py @@ -1,5 +1,6 @@ import os +from rpython.jit.codewriter.effectinfo import EffectInfo from rpython.jit.metainterp.history import Const from rpython.jit.metainterp.jitexc import JitException from rpython.jit.metainterp.optimizeopt.optimizer import Optimization, MODE_ARRAY, LEVEL_KNOWNCLASS @@ -168,6 +169,7 @@ self.cached_fields = {} # cached array items: {array descr: {index: CachedField}} self.cached_arrayitems = {} + self.cached_dict_reads = {} # self._lazy_setfields_and_arrayitems = [] self._remove_guard_not_invalidated = False @@ -277,6 +279,20 @@ self.force_all_lazy_setfields_and_arrayitems() self.clean_caches() + def optimize_CALL(self, op): + # dispatch based on 'oopspecindex' to a method that handles + # specifically the given oopspec call. For non-oopspec calls, + # oopspecindex is just zero. + effectinfo = op.getdescr().get_extra_info() + oopspecindex = effectinfo.oopspecindex + if oopspecindex == EffectInfo.OS_DICT_LOOKUP: + if self._optimize_CALL_DICT_LOOKUP(op): + return + self.emit_operation(op) + + def _optimize_CALL_DICT_LOOKUP(self, op): + xxx + def force_from_effectinfo(self, effectinfo): # XXX we can get the wrong complexity here, if the lists # XXX stored on effectinfo are large diff --git a/rpython/jit/metainterp/test/test_dict.py b/rpython/jit/metainterp/test/test_dict.py --- a/rpython/jit/metainterp/test/test_dict.py +++ b/rpython/jit/metainterp/test/test_dict.py @@ -193,6 +193,21 @@ self.check_simple_loop({'int_sub': 1, 'int_gt': 1, 'guard_true': 1, 'jump': 1}) + def test_dict_two_lookups(self): + driver = JitDriver(greens = [], reds = 'auto') + d = {'a': 3, 'b': 4} + indexes = ['a', 'b'] + + def f(n): + s = 0 + while n > 0: + driver.jit_merge_point() + s += d[indexes[n & 1]] + s += d[indexes[n & 1]] + n -= 1 + return s + + self.meta_interp(f, [10]) class TestLLtype(DictTests, LLJitMixin): pass diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py --- a/rpython/rtyper/lltypesystem/rdict.py +++ b/rpython/rtyper/lltypesystem/rdict.py @@ -569,6 +569,7 @@ PERTURB_SHIFT = 5 @jit.look_inside_iff(lambda d, key, hash: jit.isvirtual(d) and jit.isconstant(key)) + at jit.oopspec('dict.lookup(d, key, hash)') def ll_dict_lookup(d, key, hash): entries = d.entries ENTRIES = lltype.typeOf(entries).TO From noreply at buildbot.pypy.org Mon Mar 17 10:48:28 2014 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 17 Mar 2014 10:48:28 +0100 (CET) Subject: [pypy-commit] pypy improve-consecutive-dict-lookups: pass the first test Message-ID: <20140317094828.811D31C35F0@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: improve-consecutive-dict-lookups Changeset: r70007:ae17d4126ade Date: 2014-03-17 11:46 +0200 http://bitbucket.org/pypy/pypy/changeset/ae17d4126ade/ Log: pass the first test diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py --- a/rpython/jit/metainterp/optimizeopt/heap.py +++ b/rpython/jit/metainterp/optimizeopt/heap.py @@ -1,6 +1,7 @@ import os from rpython.jit.codewriter.effectinfo import EffectInfo +from rpython.jit.metainterp.optimizeopt.util import args_dict from rpython.jit.metainterp.history import Const from rpython.jit.metainterp.jitexc import JitException from rpython.jit.metainterp.optimizeopt.optimizer import Optimization, MODE_ARRAY, LEVEL_KNOWNCLASS @@ -169,6 +170,7 @@ self.cached_fields = {} # cached array items: {array descr: {index: CachedField}} self.cached_arrayitems = {} + # cached dict items: {dict descr: {(optval, index): box-or-const}} self.cached_dict_reads = {} # self._lazy_setfields_and_arrayitems = [] @@ -177,9 +179,11 @@ self.postponed_op = None def force_at_end_of_preamble(self): + self.cached_dict_reads.clear() self.force_all_lazy_setfields_and_arrayitems() def flush(self): + self.cached_dict_reads.clear() self.force_all_lazy_setfields_and_arrayitems() if self.postponed_op: postponed_op = self.postponed_op @@ -208,6 +212,7 @@ del self._lazy_setfields_and_arrayitems[:] self.cached_fields.clear() self.cached_arrayitems.clear() + self.cached_dict_reads.clear() def field_cache(self, descr): try: @@ -291,7 +296,22 @@ self.emit_operation(op) def _optimize_CALL_DICT_LOOKUP(self, op): - xxx + args = self.optimizer.make_args_key(op) + descr = op.getdescr() + res_v = self.getvalue(op.result) + if descr in self.cached_dict_reads: + d = self.cached_dict_reads[descr] + else: + d = args_dict() + try: + res_v = d[args] + self.optimizer.make_equal_to(op.result, res_v, True) + res = True + except KeyError: + d[args] = res_v + res = False + self.cached_dict_reads[descr] = d + return res def force_from_effectinfo(self, effectinfo): # XXX we can get the wrong complexity here, if the lists From noreply at buildbot.pypy.org Mon Mar 17 10:55:20 2014 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 17 Mar 2014 10:55:20 +0100 (CET) Subject: [pypy-commit] pypy improve-consecutive-dict-lookups: fix guard_exception and write a failing test Message-ID: <20140317095520.84E1B1C362D@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: improve-consecutive-dict-lookups Changeset: r70008:0fdf94cb8da0 Date: 2014-03-17 11:54 +0200 http://bitbucket.org/pypy/pypy/changeset/0fdf94cb8da0/ Log: fix guard_exception and write a failing test diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py --- a/rpython/jit/metainterp/optimizeopt/heap.py +++ b/rpython/jit/metainterp/optimizeopt/heap.py @@ -177,6 +177,7 @@ self._remove_guard_not_invalidated = False self._seen_guard_not_invalidated = False self.postponed_op = None + self.remove_next_guard = False def force_at_end_of_preamble(self): self.cached_dict_reads.clear() @@ -306,6 +307,7 @@ try: res_v = d[args] self.optimizer.make_equal_to(op.result, res_v, True) + self.remove_next_guard = True res = True except KeyError: d[args] = res_v @@ -313,6 +315,14 @@ self.cached_dict_reads[descr] = d return res + def optimize_GUARD_NO_EXCEPTION(self, op): + if self.remove_next_guard: + self.remove_next_guard = False + return + self.emit_operation(op) + + optimize_GUARD_EXCEPTION = optimize_GUARD_NO_EXCEPTION + def force_from_effectinfo(self, effectinfo): # XXX we can get the wrong complexity here, if the lists # XXX stored on effectinfo are large diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py @@ -5434,6 +5434,21 @@ """ self.optimize_loop(ops, expected) + def test_consecutive_getinteriorfields(self): + py.test.skip("we want this to pass") + ops = """ + [p0, i0] + i1 = getinteriorfield_gc(p0, i0, descr=valuedescr) + i2 = getinteriorfield_gc(p0, i0, descr=valuedescr) + jump(i1, i2) + """ + expected = """ + [p0, i0] + i1 = getinteriorfield_gc(p0, i0, descr=valuedescr) + jump(i1, i1) + """ + self.optimize_loop(ops, expected) + class TestLLtype(BaseTestOptimizeBasic, LLtypeMixin): pass diff --git a/rpython/jit/metainterp/test/test_dict.py b/rpython/jit/metainterp/test/test_dict.py --- a/rpython/jit/metainterp/test/test_dict.py +++ b/rpython/jit/metainterp/test/test_dict.py @@ -208,6 +208,8 @@ return s self.meta_interp(f, [10]) + self.check_simple_loop(call=1, getinteriorfield_gc=1, + guard_no_exception=1) class TestLLtype(DictTests, LLJitMixin): pass From noreply at buildbot.pypy.org Mon Mar 17 12:31:56 2014 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 17 Mar 2014 12:31:56 +0100 (CET) Subject: [pypy-commit] pypy improve-consecutive-dict-lookups: A failing test (for stupid reasons) Message-ID: <20140317113156.099261D2824@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: improve-consecutive-dict-lookups Changeset: r70009:657f9e5f56a7 Date: 2014-03-17 13:31 +0200 http://bitbucket.org/pypy/pypy/changeset/657f9e5f56a7/ Log: A failing test (for stupid reasons) diff --git a/rpython/jit/metainterp/test/test_dict.py b/rpython/jit/metainterp/test/test_dict.py --- a/rpython/jit/metainterp/test/test_dict.py +++ b/rpython/jit/metainterp/test/test_dict.py @@ -208,6 +208,27 @@ return s self.meta_interp(f, [10]) + # XXX should be one getinteriorfield_gc + self.check_simple_loop(call=1, getinteriorfield_gc=2, + guard_no_exception=1) + + def test_dict_insert_invalidates_caches(self): + driver = JitDriver(greens = [], reds = 'auto') + d = {'a': 3, 'b': 4} + indexes = ['a', 'b'] + + def f(n): + s = 0 + while n > 0: + driver.jit_merge_point() + index = indexes[n & 1] + s += d[index] + d['aa'] = 13 # this will invalidate the index + s += d[index] + n -= 1 + return s + + self.meta_interp(f, [10]) self.check_simple_loop(call=1, getinteriorfield_gc=1, guard_no_exception=1) From noreply at buildbot.pypy.org Mon Mar 17 13:35:52 2014 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 17 Mar 2014 13:35:52 +0100 (CET) Subject: [pypy-commit] pypy improve-consecutive-dict-lookups: fix and write more tests Message-ID: <20140317123552.F04F81C357E@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: improve-consecutive-dict-lookups Changeset: r70010:b828e827b47f Date: 2014-03-17 14:35 +0200 http://bitbucket.org/pypy/pypy/changeset/b828e827b47f/ Log: fix and write more tests diff --git a/rpython/jit/codewriter/call.py b/rpython/jit/codewriter/call.py --- a/rpython/jit/codewriter/call.py +++ b/rpython/jit/codewriter/call.py @@ -178,7 +178,7 @@ return (fnaddr, calldescr) def getcalldescr(self, op, oopspecindex=EffectInfo.OS_NONE, - extraeffect=None): + extraeffect=None, extradescr=None): """Return the calldescr that describes all calls done by 'op'. This returns a calldescr that we can put in the corresponding call operation in the calling jitcode. It gets an effectinfo @@ -259,6 +259,7 @@ effectinfo = effectinfo_from_writeanalyze( self.readwrite_analyzer.analyze(op, self.seen), self.cpu, extraeffect, oopspecindex, can_invalidate, call_release_gil_target, + extradescr, ) # assert effectinfo is not None diff --git a/rpython/jit/codewriter/effectinfo.py b/rpython/jit/codewriter/effectinfo.py --- a/rpython/jit/codewriter/effectinfo.py +++ b/rpython/jit/codewriter/effectinfo.py @@ -97,7 +97,8 @@ extraeffect=EF_CAN_RAISE, oopspecindex=OS_NONE, can_invalidate=False, - call_release_gil_target=llmemory.NULL): + call_release_gil_target=llmemory.NULL, + extradescr=None): key = (frozenset_or_none(readonly_descrs_fields), frozenset_or_none(readonly_descrs_arrays), frozenset_or_none(write_descrs_fields), @@ -133,6 +134,7 @@ result.extraeffect = extraeffect result.can_invalidate = can_invalidate result.oopspecindex = oopspecindex + result.extradescr = extradescr result.call_release_gil_target = call_release_gil_target if result.check_can_raise(): assert oopspecindex in cls._OS_CANRAISE @@ -173,7 +175,8 @@ extraeffect=EffectInfo.EF_CAN_RAISE, oopspecindex=EffectInfo.OS_NONE, can_invalidate=False, - call_release_gil_target=llmemory.NULL): + call_release_gil_target=llmemory.NULL, + extradescr=None): from rpython.translator.backendopt.writeanalyze import top_set if effects is top_set or extraeffect == EffectInfo.EF_RANDOM_EFFECTS: readonly_descrs_fields = None @@ -222,7 +225,8 @@ extraeffect, oopspecindex, can_invalidate, - call_release_gil_target) + call_release_gil_target, + extradescr) def consider_struct(TYPE, fieldname): if fieldType(TYPE, fieldname) is lltype.Void: diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py --- a/rpython/jit/codewriter/jtransform.py +++ b/rpython/jit/codewriter/jtransform.py @@ -1684,9 +1684,11 @@ # ---------- # Strings and Unicodes. - def _handle_oopspec_call(self, op, args, oopspecindex, extraeffect=None): + def _handle_oopspec_call(self, op, args, oopspecindex, extraeffect=None, + extradescr=None): calldescr = self.callcontrol.getcalldescr(op, oopspecindex, - extraeffect) + extraeffect, + extradescr=extradescr) if extraeffect is not None: assert (is_test_calldescr(calldescr) # for tests or calldescr.get_extra_info().extraeffect == extraeffect) @@ -1851,8 +1853,11 @@ EffectInfo.EF_ELIDABLE_CANNOT_RAISE) def _handle_dict_lookup_call(self, op, oopspec_name, args): + extradescr = self.cpu.fielddescrof(op.args[1].concretetype.TO, + 'entries') return self._handle_oopspec_call(op, args, EffectInfo.OS_DICT_LOOKUP, - EffectInfo.EF_CAN_RAISE) + EffectInfo.EF_CAN_RAISE, + extradescr=extradescr) def _handle_rgc_call(self, op, oopspec_name, args): if oopspec_name == 'rgc.ll_shrink_array': diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py --- a/rpython/jit/metainterp/optimizeopt/heap.py +++ b/rpython/jit/metainterp/optimizeopt/heap.py @@ -298,7 +298,7 @@ def _optimize_CALL_DICT_LOOKUP(self, op): args = self.optimizer.make_args_key(op) - descr = op.getdescr() + descr = op.getdescr().extrainfo.extradescr res_v = self.getvalue(op.result) if descr in self.cached_dict_reads: d = self.cached_dict_reads[descr] @@ -331,6 +331,10 @@ for arraydescr in effectinfo.readonly_descrs_arrays: self.force_lazy_setarrayitem(arraydescr) for fielddescr in effectinfo.write_descrs_fields: + try: + del self.cached_dict_reads[fielddescr] + except KeyError: + pass self.force_lazy_setfield(fielddescr, can_cache=False) for arraydescr in effectinfo.write_descrs_arrays: self.force_lazy_setarrayitem(arraydescr, can_cache=False) diff --git a/rpython/jit/metainterp/test/test_dict.py b/rpython/jit/metainterp/test/test_dict.py --- a/rpython/jit/metainterp/test/test_dict.py +++ b/rpython/jit/metainterp/test/test_dict.py @@ -214,23 +214,42 @@ def test_dict_insert_invalidates_caches(self): driver = JitDriver(greens = [], reds = 'auto') - d = {'a': 3, 'b': 4} - indexes = ['a', 'b'] + indexes = ['aa', 'b', 'cc'] def f(n): + d = {'aa': 3, 'b': 4, 'cc': 5} s = 0 while n > 0: driver.jit_merge_point() index = indexes[n & 1] s += d[index] - d['aa'] = 13 # this will invalidate the index + d['aa'] += 1 # this will invalidate the index s += d[index] n -= 1 return s - self.meta_interp(f, [10]) - self.check_simple_loop(call=1, getinteriorfield_gc=1, - guard_no_exception=1) + res = self.meta_interp(f, [10]) + assert res == f(10) + self.check_simple_loop(call=5) + + def test_dict_double_lookup_2(self): + driver = JitDriver(greens = [], reds = 'auto') + indexes = ['aa', 'b', 'cc'] + + def f(n): + d = {'aa': 3, 'b': 4, 'cc': 5} + s = 0 + while n > 0: + driver.jit_merge_point() + index = indexes[n & 1] + s += d[index] + d[index] += 1 + n -= 1 + return s + + res = self.meta_interp(f, [10]) + assert res == f(10) + self.check_simple_loop(call=3) class TestLLtype(DictTests, LLJitMixin): pass From noreply at buildbot.pypy.org Mon Mar 17 13:38:37 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 17 Mar 2014 13:38:37 +0100 (CET) Subject: [pypy-commit] pypy default: fix test_pypy_c/test_ffi.py Message-ID: <20140317123837.698131C357E@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r70011:3da8cfa97734 Date: 2014-03-17 08:37 -0400 http://bitbucket.org/pypy/pypy/changeset/3da8cfa97734/ Log: fix test_pypy_c/test_ffi.py diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py --- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py +++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py @@ -342,12 +342,10 @@ p167 = call(ConstClass(_ll_0_alloc_with_del___), descr=) guard_no_exception(descr=...) i168 = call(ConstClass(_ll_1_raw_malloc_varsize__Signed), 6, descr=) - setfield_gc(p167, 0, descr=) - setfield_gc(p167, ConstPtr(ptr86), descr=) - guard_no_exception(descr=...) i169 = int_add(i168, i97) i170 = int_sub(i160, i106) setfield_gc(p167, i168, descr=) + setfield_gc(p167, ConstPtr(null), descr=) setfield_gc(p167, ConstPtr(ptr89), descr=) i171 = uint_gt(i170, i108) guard_false(i171, descr=...) From noreply at buildbot.pypy.org Mon Mar 17 14:11:35 2014 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 17 Mar 2014 14:11:35 +0100 (CET) Subject: [pypy-commit] pypy improve-consecutive-dict-lookups: fix for ordereddicts too Message-ID: <20140317131135.11DF81C11A4@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: improve-consecutive-dict-lookups Changeset: r70012:cc34a5b32a04 Date: 2014-03-17 15:10 +0200 http://bitbucket.org/pypy/pypy/changeset/cc34a5b32a04/ Log: fix for ordereddicts too diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py --- a/rpython/jit/codewriter/jtransform.py +++ b/rpython/jit/codewriter/jtransform.py @@ -403,7 +403,8 @@ prepare = self._handle_math_sqrt_call elif oopspec_name.startswith('rgc.'): prepare = self._handle_rgc_call - elif oopspec_name == 'dict.lookup': + elif oopspec_name.endswith('dict.lookup'): + # also ordereddict.lookup prepare = self._handle_dict_lookup_call else: prepare = self.prepare_builtin_call diff --git a/rpython/jit/metainterp/test/test_dict.py b/rpython/jit/metainterp/test/test_dict.py --- a/rpython/jit/metainterp/test/test_dict.py +++ b/rpython/jit/metainterp/test/test_dict.py @@ -212,6 +212,28 @@ self.check_simple_loop(call=1, getinteriorfield_gc=2, guard_no_exception=1) + + def test_ordered_dict_two_lookups(self): + driver = JitDriver(greens = [], reds = 'auto') + d = OrderedDict() + d['a'] = 3 + d['b'] = 4 + indexes = ['a', 'b'] + + def f(n): + s = 0 + while n > 0: + driver.jit_merge_point() + s += d[indexes[n & 1]] + s += d[indexes[n & 1]] + n -= 1 + return s + + self.meta_interp(f, [10]) + # XXX should be one getinteriorfield_gc + self.check_simple_loop(call=1, getinteriorfield_gc=2, + guard_no_exception=1) + def test_dict_insert_invalidates_caches(self): driver = JitDriver(greens = [], reds = 'auto') indexes = ['aa', 'b', 'cc'] diff --git a/rpython/rtyper/lltypesystem/rbuilder.py b/rpython/rtyper/lltypesystem/rbuilder.py --- a/rpython/rtyper/lltypesystem/rbuilder.py +++ b/rpython/rtyper/lltypesystem/rbuilder.py @@ -85,6 +85,7 @@ ll_builder.used = needed @staticmethod + @enforceargs(None, lltype.Char) def ll_append_char(ll_builder, char): if ll_builder.used == ll_builder.allocated: ll_builder.grow(ll_builder, 1) diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py --- a/rpython/rtyper/lltypesystem/rordereddict.py +++ b/rpython/rtyper/lltypesystem/rordereddict.py @@ -709,6 +709,7 @@ @jit.look_inside_iff(lambda d, key, hash, store_flag, T: jit.isvirtual(d) and jit.isconstant(key)) + at jit.oopspec('ordereddict.lookup(d, key, hash, store_flag, T)') def ll_dict_lookup(d, key, hash, store_flag, T): INDEXES = _ll_ptr_to_array_of(T) entries = d.entries From noreply at buildbot.pypy.org Mon Mar 17 14:27:23 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 17 Mar 2014 14:27:23 +0100 (CET) Subject: [pypy-commit] pypy default: virtual raw buffers force using raw_store rather than int_add + setarrayitem_raw Message-ID: <20140317132723.1AEA21C362D@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r70013:5da6957bd86b Date: 2014-03-17 09:20 -0400 http://bitbucket.org/pypy/pypy/changeset/5da6957bd86b/ Log: virtual raw buffers force using raw_store rather than int_add + setarrayitem_raw diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -1785,11 +1785,12 @@ def test_virtual_raw_malloc_force(self): ops = """ [i1] - i2 = call('malloc', 10, descr=raw_malloc_descr) + i2 = call('malloc', 20, descr=raw_malloc_descr) guard_no_exception() [] setarrayitem_raw(i2, 0, i1, descr=rawarraydescr_char) setarrayitem_raw(i2, 2, 456, descr=rawarraydescr_char) setarrayitem_raw(i2, 1, 123, descr=rawarraydescr_char) + setarrayitem_raw(i2, 1, 789, descr=rawarraydescr_float) label('foo') # we expect the buffer to be forced *after* the label escape(i2) call('free', i2, descr=raw_free_descr) @@ -1798,13 +1799,12 @@ expected = """ [i1] label('foo') - i2 = call('malloc', 10, descr=raw_malloc_descr) + i2 = call('malloc', 20, descr=raw_malloc_descr) guard_no_exception() [] - setarrayitem_raw(i2, 0, i1, descr=rawarraydescr_char) - i3 = int_add(i2, 1) - setarrayitem_raw(i3, 0, 123, descr=rawarraydescr_char) - i4 = int_add(i2, 2) - setarrayitem_raw(i4, 0, 456, descr=rawarraydescr_char) + raw_store(i2, 0, i1, descr=rawarraydescr_char) + raw_store(i2, 1, 123, descr=rawarraydescr_char) + raw_store(i2, 2, 456, descr=rawarraydescr_char) + raw_store(i2, 8, 789, descr=rawarraydescr_float) escape(i2) call('free', i2, descr=raw_free_descr) jump(i1) @@ -1827,7 +1827,7 @@ label('foo') i2 = call('malloc', 10, descr=raw_malloc_descr) guard_no_exception() [] - setarrayitem_raw(i2, 0, i1, descr=rawarraydescr) + raw_store(i2, 0, i1, descr=rawarraydescr) setarrayitem_raw(i2, 2, 456, descr=rawarraydescr_char) call('free', i2, descr=raw_free_descr) jump(i1) @@ -1850,7 +1850,7 @@ label('foo') i2 = call('malloc', 10, descr=raw_malloc_descr) guard_no_exception() [] - setarrayitem_raw(i2, 0, i1, descr=rawarraydescr) + raw_store(i2, 0, i1, descr=rawarraydescr) i3 = getarrayitem_raw(i2, 0, descr=rawarraydescr_char) call('free', i2, descr=raw_free_descr) jump(i1) @@ -1913,9 +1913,8 @@ # these ops are generated by VirtualRawBufferValue._really_force i2 = call('malloc', 10, descr=raw_malloc_descr) guard_no_exception() [] - setarrayitem_raw(i2, 0, 42, descr=rawarraydescr_char) - i3 = int_add(i2, 5) # 1+4*sizeof(char) - setarrayitem_raw(i3, 0, 4242, descr=rawarraydescr_char) + raw_store(i2, 0, 42, descr=rawarraydescr_char) + raw_store(i2, 5, 4242, descr=rawarraydescr_char) # this is generated by VirtualRawSliceValue._really_force i4 = int_add(i2, 1) escape(i4) @@ -1943,7 +1942,7 @@ label('foo') i3 = call('malloc', 10, descr=raw_malloc_descr) guard_no_exception() [] - setarrayitem_raw(i3, 0, i2, descr=rawarraydescr) + raw_store(i3, 0, i2, descr=rawarraydescr) jump(i3) """ self.optimize_loop(ops, expected) diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py --- a/rpython/jit/metainterp/optimizeopt/virtualize.py +++ b/rpython/jit/metainterp/optimizeopt/virtualize.py @@ -432,22 +432,13 @@ optforce.emit_operation(self.source_op) self.box = self.source_op.result for i in range(len(self.buffer.offsets)): - # get a pointer to self.box+offset + # write the value offset = self.buffer.offsets[i] - if offset == 0: - arraybox = self.box - else: - arraybox = BoxInt() - op = ResOperation(rop.INT_ADD, - [self.box, ConstInt(offset)], arraybox) - optforce.emit_operation(op) - # - # write the value descr = self.buffer.descrs[i] itemvalue = self.buffer.values[i] itembox = itemvalue.force_box(optforce) - op = ResOperation(rop.SETARRAYITEM_RAW, - [arraybox, ConstInt(0), itembox], None, + op = ResOperation(rop.RAW_STORE, + [self.box, ConstInt(offset), itembox], None, descr=descr) optforce.emit_operation(op) diff --git a/rpython/jit/metainterp/test/test_virtual.py b/rpython/jit/metainterp/test/test_virtual.py --- a/rpython/jit/metainterp/test/test_virtual.py +++ b/rpython/jit/metainterp/test/test_virtual.py @@ -1196,7 +1196,7 @@ assert res == 45 # make sure that the raw buffer is *not* virtualized because we do not # support virtualstate - self.check_resops(getarrayitem_raw=2, setarrayitem_raw=2) + self.check_resops(getarrayitem_raw=2, raw_store=2) def test_raw_malloc_only_chars(self): mydriver = JitDriver(greens=[], reds = 'auto') From noreply at buildbot.pypy.org Mon Mar 17 14:31:55 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 17 Mar 2014 14:31:55 +0100 (CET) Subject: [pypy-commit] pypy default: passing test for virtualization of setarrayitem_raw followed by raw_load Message-ID: <20140317133155.CB8821C362D@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r70014:2b667d34e6cd Date: 2014-03-17 09:30 -0400 http://bitbucket.org/pypy/pypy/changeset/2b667d34e6cd/ Log: passing test for virtualization of setarrayitem_raw followed by raw_load diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -1983,6 +1983,24 @@ """ self.optimize_loop(ops, expected) + def test_virtual_setarrayitem_raw_raw_load(self): + ops = """ + [f1] + i0 = call('malloc', 16, descr=raw_malloc_descr) + guard_no_exception() [] + setarrayitem_raw(i0, 1, f1, descr=rawarraydescr_float) + f2 = raw_load(i0, 8, descr=rawarraydescr_float) + f3 = float_add(f1, f2) + call('free', i0, descr=raw_free_descr) + jump(f3) + """ + expected = """ + [f1] + f2 = float_add(f1, f1) + jump(f2) + """ + self.optimize_loop(ops, expected) + def test_duplicate_getfield_1(self): ops = """ [p1, p2] From noreply at buildbot.pypy.org Mon Mar 17 15:03:58 2014 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 17 Mar 2014 15:03:58 +0100 (CET) Subject: [pypy-commit] pypy improve-consecutive-dict-lookups: fixes Message-ID: <20140317140358.6E4591C11A4@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: improve-consecutive-dict-lookups Changeset: r70015:b6d875f4bbaf Date: 2014-03-17 16:00 +0200 http://bitbucket.org/pypy/pypy/changeset/b6d875f4bbaf/ Log: fixes diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py --- a/rpython/jit/metainterp/optimizeopt/heap.py +++ b/rpython/jit/metainterp/optimizeopt/heap.py @@ -298,7 +298,7 @@ def _optimize_CALL_DICT_LOOKUP(self, op): args = self.optimizer.make_args_key(op) - descr = op.getdescr().extrainfo.extradescr + descr = op.getdescr().get_extra_info().extradescr res_v = self.getvalue(op.result) if descr in self.cached_dict_reads: d = self.cached_dict_reads[descr] @@ -306,7 +306,7 @@ d = args_dict() try: res_v = d[args] - self.optimizer.make_equal_to(op.result, res_v, True) + self.make_equal_to(op.result, res_v) self.remove_next_guard = True res = True except KeyError: diff --git a/rpython/rtyper/lltypesystem/rbuilder.py b/rpython/rtyper/lltypesystem/rbuilder.py --- a/rpython/rtyper/lltypesystem/rbuilder.py +++ b/rpython/rtyper/lltypesystem/rbuilder.py @@ -85,14 +85,6 @@ ll_builder.used = needed @staticmethod - @enforceargs(None, lltype.Char) - def ll_append_char(ll_builder, char): - if ll_builder.used == ll_builder.allocated: - ll_builder.grow(ll_builder, 1) - ll_builder.buf.chars[ll_builder.used] = char - ll_builder.used += 1 - - @staticmethod def ll_append_slice(ll_builder, ll_str, start, end): needed = end - start used = ll_builder.used @@ -114,6 +106,7 @@ ll_builder.used = used @staticmethod + @enforceargs(None, None, int) def ll_append_charpsize(ll_builder, charp, size): used = ll_builder.used if used + size > ll_builder.allocated: @@ -139,6 +132,14 @@ return ll_builder != nullptr(cls.lowleveltype.TO) class StringBuilderRepr(BaseStringBuilderRepr): + @staticmethod + @enforceargs(None, lltype.Char) + def ll_append_char(ll_builder, char): + if ll_builder.used == ll_builder.allocated: + ll_builder.grow(ll_builder, 1) + ll_builder.buf.chars[ll_builder.used] = char + ll_builder.used += 1 + lowleveltype = lltype.Ptr(STRINGBUILDER) basetp = STR mallocfn = staticmethod(rstr.mallocstr) @@ -149,6 +150,15 @@ ) class UnicodeBuilderRepr(BaseStringBuilderRepr): + + @staticmethod + @enforceargs(None, lltype.UniChar) + def ll_append_char(ll_builder, char): + if ll_builder.used == ll_builder.allocated: + ll_builder.grow(ll_builder, 1) + ll_builder.buf.chars[ll_builder.used] = char + ll_builder.used += 1 + lowleveltype = lltype.Ptr(UNICODEBUILDER) basetp = UNICODE mallocfn = staticmethod(rstr.mallocunicode) diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py --- a/rpython/rtyper/lltypesystem/rstr.py +++ b/rpython/rtyper/lltypesystem/rstr.py @@ -4,7 +4,7 @@ from rpython.rlib import jit, types from rpython.rlib.debug import ll_assert from rpython.rlib.objectmodel import (malloc_zero_filled, we_are_translated, - _hash_string, keepalive_until_here, specialize) + _hash_string, keepalive_until_here, specialize, enforceargs) from rpython.rlib.signature import signature from rpython.rlib.rarithmetic import ovfcheck from rpython.rtyper.error import TyperError @@ -108,6 +108,7 @@ copy_string_to_raw = func_with_new_name(copy_string_to_raw, 'copy_%s_to_raw' % name) @jit.dont_look_inside + @enforceargs(None, None, None, int) def copy_raw_to_string(ptrsrc, dst, dststart, length): # xxx Warning: same note as above apply: don't do this at home assert length >= 0 From noreply at buildbot.pypy.org Mon Mar 17 15:51:07 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 17 Mar 2014 15:51:07 +0100 (CET) Subject: [pypy-commit] pypy default: cleanup repr of resops with descr but no args Message-ID: <20140317145107.D81AA1D27BF@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r70016:084cf842f3c1 Date: 2014-03-17 10:49 -0400 http://bitbucket.org/pypy/pypy/changeset/084cf842f3c1/ Log: cleanup repr of resops with descr but no args diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py --- a/rpython/jit/metainterp/resoperation.py +++ b/rpython/jit/metainterp/resoperation.py @@ -119,10 +119,11 @@ descr = self.getdescr() if descr is None or we_are_translated(): return '%s%s%s(%s)' % (prefix, sres, self.getopname(), - ', '.join([str(a) for a in args])) + ', '.join([str(a) for a in args])) else: - return '%s%s%s(%s, descr=%r)' % (prefix, sres, self.getopname(), - ', '.join([str(a) for a in args]), descr) + return '%s%s%s(%s)' % (prefix, sres, self.getopname(), + ', '.join([str(a) for a in args] + + ['descr=%r' % descr])) def getopname(self): try: diff --git a/rpython/jit/metainterp/test/test_resoperation.py b/rpython/jit/metainterp/test/test_resoperation.py --- a/rpython/jit/metainterp/test/test_resoperation.py +++ b/rpython/jit/metainterp/test/test_resoperation.py @@ -1,6 +1,7 @@ import py +import re from rpython.jit.metainterp import resoperation as rop -from rpython.jit.metainterp.history import AbstractDescr +from rpython.jit.metainterp.history import AbstractDescr, AbstractFailDescr def test_arity_mixins(): cases = [ @@ -55,12 +56,18 @@ op = rop.ResOperation(rop.rop.INT_ADD, ['a', 'b'], 'c') assert op.getarglist() == ['a', 'b'] assert op.result == 'c' + assert repr(op) == "c = int_add(a, b)" mydescr = AbstractDescr() op = rop.ResOperation(rop.rop.CALL, ['a', 'b'], 'c', descr=mydescr) assert op.getarglist() == ['a', 'b'] assert op.result == 'c' assert op.getdescr() is mydescr + assert re.match("c = call$a, b, descr=<.+>$$", repr(op)) + + mydescr = AbstractFailDescr() + op = rop.ResOperation(rop.rop.GUARD_NO_EXCEPTION, [], None, descr=mydescr) + assert re.match("guard_no_exception$descr=<.+>$$", repr(op)) def test_can_malloc(): mydescr = AbstractDescr() From noreply at buildbot.pypy.org Mon Mar 17 15:52:14 2014 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 17 Mar 2014 15:52:14 +0100 (CET) Subject: [pypy-commit] pypy improve-consecutive-dict-lookups: maybe fix translation Message-ID: <20140317145214.65AA31D27BF@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: improve-consecutive-dict-lookups Changeset: r70017:7bef0e7352f9 Date: 2014-03-17 16:51 +0200 http://bitbucket.org/pypy/pypy/changeset/7bef0e7352f9/ Log: maybe fix translation diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py --- a/rpython/jit/metainterp/optimizeopt/heap.py +++ b/rpython/jit/metainterp/optimizeopt/heap.py @@ -1,7 +1,7 @@ import os from rpython.jit.codewriter.effectinfo import EffectInfo -from rpython.jit.metainterp.optimizeopt.util import args_dict +from rpython.jit.metainterp.optimizeopt.util import args_dict_value from rpython.jit.metainterp.history import Const from rpython.jit.metainterp.jitexc import JitException from rpython.jit.metainterp.optimizeopt.optimizer import Optimization, MODE_ARRAY, LEVEL_KNOWNCLASS @@ -303,7 +303,7 @@ if descr in self.cached_dict_reads: d = self.cached_dict_reads[descr] else: - d = args_dict() + d = args_dict_value() try: res_v = d[args] self.make_equal_to(op.result, res_v) diff --git a/rpython/jit/metainterp/optimizeopt/util.py b/rpython/jit/metainterp/optimizeopt/util.py --- a/rpython/jit/metainterp/optimizeopt/util.py +++ b/rpython/jit/metainterp/optimizeopt/util.py @@ -92,6 +92,7 @@ # ____________________________________________________________ +def _new_args_dict(): def args_eq(args1, args2): make_sure_not_resized(args1) make_sure_not_resized(args2) @@ -124,6 +125,9 @@ def args_dict_box(): return r_dict(args_eq, args_hash) +def args_dict_value(): + return r_dict(args_eq, args_hash) + # ____________________________________________________________ diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py --- a/rpython/rtyper/lltypesystem/rstr.py +++ b/rpython/rtyper/lltypesystem/rstr.py @@ -108,7 +108,6 @@ copy_string_to_raw = func_with_new_name(copy_string_to_raw, 'copy_%s_to_raw' % name) @jit.dont_look_inside - @enforceargs(None, None, None, int) def copy_raw_to_string(ptrsrc, dst, dststart, length): # xxx Warning: same note as above apply: don't do this at home assert length >= 0 @@ -120,9 +119,10 @@ llmemory.raw_memcopy(srcbuf, dst, llmemory.sizeof(CHAR_TP) * length) # end of "no GC" section keepalive_until_here(dst) - copy_raw_to_string._always_inline_ = True copy_raw_to_string = func_with_new_name(copy_raw_to_string, 'copy_raw_to_%s' % name) + copy_raw_to_string._always_inline_ = True + copy_raw_to_string._annenforceargs_ = (None, None, int, int) return copy_string_to_raw, copy_raw_to_string, copy_string_contents From noreply at buildbot.pypy.org Mon Mar 17 16:01:22 2014 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 17 Mar 2014 16:01:22 +0100 (CET) Subject: [pypy-commit] pypy improve-consecutive-dict-lookups: ups Message-ID: <20140317150122.09B681D27C0@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: improve-consecutive-dict-lookups Changeset: r70018:eef1a9f78543 Date: 2014-03-17 17:00 +0200 http://bitbucket.org/pypy/pypy/changeset/eef1a9f78543/ Log: ups diff --git a/rpython/jit/metainterp/optimizeopt/util.py b/rpython/jit/metainterp/optimizeopt/util.py --- a/rpython/jit/metainterp/optimizeopt/util.py +++ b/rpython/jit/metainterp/optimizeopt/util.py @@ -92,7 +92,6 @@ # ____________________________________________________________ -def _new_args_dict(): def args_eq(args1, args2): make_sure_not_resized(args1) make_sure_not_resized(args2) From noreply at buildbot.pypy.org Mon Mar 17 16:27:58 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 17 Mar 2014 16:27:58 +0100 (CET) Subject: [pypy-commit] pypy default: unskip this test Message-ID: <20140317152758.8F4BA1C0166@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r70019:10739383a275 Date: 2014-03-17 11:26 -0400 http://bitbucket.org/pypy/pypy/changeset/10739383a275/ Log: unskip this test diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py --- a/rpython/annotator/test/test_annrpython.py +++ b/rpython/annotator/test/test_annrpython.py @@ -2872,7 +2872,6 @@ py.test.raises(Exception, a.build_types, fun, [s_nonneg, int]) def test_sig_bug(self): - py.test.skip("_annenforceargs_ does not work for default arguments") def g(x, y=5): return y == 5 g._annenforceargs_ = (int, int) @@ -2880,7 +2879,8 @@ return g(x) a = self.RPythonAnnotator() s = a.build_types(fun, [int]) - assert not s.is_constant() + assert s.knowntype is bool + assert s.is_constant() def test_sig_list(self): def g(buf): From noreply at buildbot.pypy.org Mon Mar 17 19:23:21 2014 From: noreply at buildbot.pypy.org (cfbolz) Date: Mon, 17 Mar 2014 19:23:21 +0100 (CET) Subject: [pypy-commit] pypy improve-consecutive-dict-lookups: I think this expresses the logic much more clearly Message-ID: <20140317182321.551661C12F3@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: improve-consecutive-dict-lookups Changeset: r70020:325e14172b33 Date: 2014-03-17 19:22 +0100 http://bitbucket.org/pypy/pypy/changeset/325e14172b33/ Log: I think this expresses the logic much more clearly diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py --- a/rpython/jit/metainterp/optimizeopt/heap.py +++ b/rpython/jit/metainterp/optimizeopt/heap.py @@ -299,20 +299,20 @@ def _optimize_CALL_DICT_LOOKUP(self, op): args = self.optimizer.make_args_key(op) descr = op.getdescr().get_extra_info().extradescr - res_v = self.getvalue(op.result) if descr in self.cached_dict_reads: d = self.cached_dict_reads[descr] else: d = args_dict_value() + self.cached_dict_reads[descr] = d try: res_v = d[args] + except KeyError: + d[args] = self.getvalue(op.result) + res = False + else: self.make_equal_to(op.result, res_v) self.remove_next_guard = True res = True - except KeyError: - d[args] = res_v - res = False - self.cached_dict_reads[descr] = d return res def optimize_GUARD_NO_EXCEPTION(self, op): From noreply at buildbot.pypy.org Mon Mar 17 21:07:53 2014 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 17 Mar 2014 21:07:53 +0100 (CET) Subject: [pypy-commit] pypy default: win32 fix Message-ID: <20140317200753.596791C00B9@cobra.cs.uni-duesseldorf.de> Author: Matti Picus Branch: Changeset: r70021:a3b81c345e03 Date: 2014-03-17 20:53 +0200 http://bitbucket.org/pypy/pypy/changeset/a3b81c345e03/ Log: win32 fix diff --git a/pypy/module/_multiprocessing/test/test_connection.py b/pypy/module/_multiprocessing/test/test_connection.py --- a/pypy/module/_multiprocessing/test/test_connection.py +++ b/pypy/module/_multiprocessing/test/test_connection.py @@ -9,8 +9,12 @@ class AppTestBufferTooShort: spaceconfig = {'usemodules': ['_multiprocessing', 'thread', 'signal', - 'itertools', 'select', 'fcntl', 'struct', - 'binascii']} + 'itertools', 'select', 'struct', 'binascii']} + if sys.platform == 'win32': + spaceconfig['usemodules'].append('_rawffi') + else: + spaceconfig['usemodules'].append('fcntl') + def setup_class(cls): if cls.runappdirect: @@ -100,9 +104,12 @@ spaceconfig = { "usemodules": [ '_multiprocessing', 'thread', 'signal', 'struct', 'array', - 'itertools', '_socket', 'binascii', 'select', 'fcntl', - ] + 'itertools', '_socket', 'binascii', 'select' ] } + if sys.platform == 'win32': + spaceconfig['usemodules'].append('_rawffi') + else: + spaceconfig['usemodules'].append('fcntl') def setup_class(cls): cls.w_connections = cls.space.newlist([]) From noreply at buildbot.pypy.org Mon Mar 17 21:07:54 2014 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 17 Mar 2014 21:07:54 +0100 (CET) Subject: [pypy-commit] pypy default: win32 fix Message-ID: <20140317200754.847371C00B9@cobra.cs.uni-duesseldorf.de> Author: Matti Picus Branch: Changeset: r70022:2057b7ec2690 Date: 2014-03-17 22:06 +0200 http://bitbucket.org/pypy/pypy/changeset/2057b7ec2690/ Log: win32 fix diff --git a/pypy/module/_multiprocessing/test/test_connection.py b/pypy/module/_multiprocessing/test/test_connection.py --- a/pypy/module/_multiprocessing/test/test_connection.py +++ b/pypy/module/_multiprocessing/test/test_connection.py @@ -79,6 +79,8 @@ 'itertools', '_socket', 'binascii', ] } + if sys.platform == 'win32': + spaceconfig['usemodules'].append('_rawffi') def setup_class(cls): if sys.platform != "win32": @@ -90,7 +92,6 @@ # just for multiprocessing to import correctly on Windows w_modules = space.sys.get('modules') space.setitem(w_modules, space.wrap('msvcrt'), space.sys) - space.setitem(w_modules, space.wrap('_subprocess'), space.sys) else: import _multiprocessing From noreply at buildbot.pypy.org Mon Mar 17 21:19:20 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 17 Mar 2014 21:19:20 +0100 (CET) Subject: [pypy-commit] pypy default: cleanups Message-ID: <20140317201920.3CA721C00B9@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r70023:c3ed9ebe70f4 Date: 2014-03-17 11:56 -0400 http://bitbucket.org/pypy/pypy/changeset/c3ed9ebe70f4/ Log: cleanups diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py @@ -1661,7 +1661,7 @@ self.optimize_loop(ops, ops) def test_setfield_int_eq_result(self): - # test that the setfield_gc does not end up before int_eq + # test that the setfield_gc does not end up before int_eq ops = """ [p1, i1, i2] i3 = int_eq(i1, i2) @@ -5443,6 +5443,7 @@ jump(i0) """ self.optimize_loop(ops, expected) - + + class TestLLtype(BaseTestOptimizeBasic, LLtypeMixin): pass diff --git a/rpython/jit/metainterp/optimizeopt/util.py b/rpython/jit/metainterp/optimizeopt/util.py --- a/rpython/jit/metainterp/optimizeopt/util.py +++ b/rpython/jit/metainterp/optimizeopt/util.py @@ -1,7 +1,7 @@ import itertools import py -from rpython.rlib.objectmodel import r_dict, compute_identity_hash +from rpython.rlib.objectmodel import r_dict, compute_identity_hash, specialize from rpython.rlib.rarithmetic import intmask from rpython.rlib.unroll import unrolling_iterable from rpython.jit.metainterp import resoperation @@ -118,13 +118,10 @@ res = intmask((1000003 * res) ^ y) return res + at specialize.call_location() def args_dict(): return r_dict(args_eq, args_hash) -def args_dict_box(): - return r_dict(args_eq, args_hash) - - # ____________________________________________________________ def equaloplists(oplist1, oplist2, strict_fail_args=True, remap={}, @@ -183,4 +180,3 @@ assert False assert len(oplist1) == len(oplist2) return True - diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py --- a/rpython/jit/metainterp/optimizeopt/virtualize.py +++ b/rpython/jit/metainterp/optimizeopt/virtualize.py @@ -648,7 +648,6 @@ def optimize_SETFIELD_GC(self, op): value = self.getvalue(op.getarg(0)) - if value.is_virtual(): fieldvalue = self.getvalue(op.getarg(1)) value.setfield(op.getdescr(), fieldvalue) diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -11,7 +11,7 @@ ConstFloat, Box, TargetToken) from rpython.jit.metainterp.jitprof import EmptyProfiler from rpython.jit.metainterp.logger import Logger -from rpython.jit.metainterp.optimizeopt.util import args_dict_box +from rpython.jit.metainterp.optimizeopt.util import args_dict from rpython.jit.metainterp.resoperation import rop from rpython.rlib import nonconst, rstack from rpython.rlib.debug import debug_start, debug_stop, debug_print, make_sure_not_resized @@ -1656,7 +1656,7 @@ self.forced_virtualizable = None self.partial_trace = None self.retracing_from = -1 - self.call_pure_results = args_dict_box() + self.call_pure_results = args_dict() self.heapcache = HeapCache() self.call_ids = [] @@ -1784,16 +1784,14 @@ moreargs = [box] + extraargs else: moreargs = list(extraargs) - metainterp_sd = self.staticdata if opnum == rop.GUARD_NOT_FORCED or opnum == rop.GUARD_NOT_FORCED_2: - resumedescr = compile.ResumeGuardForcedDescr(metainterp_sd, + resumedescr = compile.ResumeGuardForcedDescr(self.staticdata, self.jitdriver_sd) elif opnum == rop.GUARD_NOT_INVALIDATED: resumedescr = compile.ResumeGuardNotInvalidated() else: resumedescr = compile.ResumeGuardDescr() - guard_op = self.history.record(opnum, moreargs, None, - descr=resumedescr) + guard_op = self.history.record(opnum, moreargs, None, descr=resumedescr) self.capture_resumedata(resumedescr, resumepc) self.staticdata.profiler.count_ops(opnum, Counters.GUARDS) # count From noreply at buildbot.pypy.org Mon Mar 17 21:19:21 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 17 Mar 2014 21:19:21 +0100 (CET) Subject: [pypy-commit] pypy default: guard_no_exception isn't currently produced here Message-ID: <20140317201921.876771C00B9@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r70024:256ecf7cf106 Date: 2014-03-17 14:57 -0400 http://bitbucket.org/pypy/pypy/changeset/256ecf7cf106/ Log: guard_no_exception isn't currently produced here diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -1800,7 +1800,7 @@ [i1] label('foo') i2 = call('malloc', 20, descr=raw_malloc_descr) - guard_no_exception() [] + #guard_no_exception() [] # XXX should appear raw_store(i2, 0, i1, descr=rawarraydescr_char) raw_store(i2, 1, 123, descr=rawarraydescr_char) raw_store(i2, 2, 456, descr=rawarraydescr_char) @@ -1826,7 +1826,7 @@ [i1] label('foo') i2 = call('malloc', 10, descr=raw_malloc_descr) - guard_no_exception() [] + #guard_no_exception() [] # XXX should appear raw_store(i2, 0, i1, descr=rawarraydescr) setarrayitem_raw(i2, 2, 456, descr=rawarraydescr_char) call('free', i2, descr=raw_free_descr) @@ -1849,7 +1849,7 @@ [i1] label('foo') i2 = call('malloc', 10, descr=raw_malloc_descr) - guard_no_exception() [] + #guard_no_exception() [] # XXX should appear raw_store(i2, 0, i1, descr=rawarraydescr) i3 = getarrayitem_raw(i2, 0, descr=rawarraydescr_char) call('free', i2, descr=raw_free_descr) @@ -1912,7 +1912,7 @@ label('foo') # these ops are generated by VirtualRawBufferValue._really_force i2 = call('malloc', 10, descr=raw_malloc_descr) - guard_no_exception() [] + #guard_no_exception() [] # XXX should appear raw_store(i2, 0, 42, descr=rawarraydescr_char) raw_store(i2, 5, 4242, descr=rawarraydescr_char) # this is generated by VirtualRawSliceValue._really_force @@ -1941,7 +1941,7 @@ call('free', i0, descr=raw_free_descr) label('foo') i3 = call('malloc', 10, descr=raw_malloc_descr) - guard_no_exception() [] + #guard_no_exception() [] # XXX should appear raw_store(i3, 0, i2, descr=rawarraydescr) jump(i3) """ From noreply at buildbot.pypy.org Mon Mar 17 21:19:22 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 17 Mar 2014 21:19:22 +0100 (CET) Subject: [pypy-commit] pypy default: merge heads Message-ID: <20140317201922.B44CB1C00B9@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r70025:d3abcc897136 Date: 2014-03-17 16:18 -0400 http://bitbucket.org/pypy/pypy/changeset/d3abcc897136/ Log: merge heads diff --git a/pypy/module/_multiprocessing/test/test_connection.py b/pypy/module/_multiprocessing/test/test_connection.py --- a/pypy/module/_multiprocessing/test/test_connection.py +++ b/pypy/module/_multiprocessing/test/test_connection.py @@ -9,8 +9,12 @@ class AppTestBufferTooShort: spaceconfig = {'usemodules': ['_multiprocessing', 'thread', 'signal', - 'itertools', 'select', 'fcntl', 'struct', - 'binascii']} + 'itertools', 'select', 'struct', 'binascii']} + if sys.platform == 'win32': + spaceconfig['usemodules'].append('_rawffi') + else: + spaceconfig['usemodules'].append('fcntl') + def setup_class(cls): if cls.runappdirect: @@ -75,6 +79,8 @@ 'itertools', '_socket', 'binascii', ] } + if sys.platform == 'win32': + spaceconfig['usemodules'].append('_rawffi') def setup_class(cls): if sys.platform != "win32": @@ -86,7 +92,6 @@ # just for multiprocessing to import correctly on Windows w_modules = space.sys.get('modules') space.setitem(w_modules, space.wrap('msvcrt'), space.sys) - space.setitem(w_modules, space.wrap('_subprocess'), space.sys) else: import _multiprocessing @@ -100,9 +105,12 @@ spaceconfig = { "usemodules": [ '_multiprocessing', 'thread', 'signal', 'struct', 'array', - 'itertools', '_socket', 'binascii', 'select', 'fcntl', - ] + 'itertools', '_socket', 'binascii', 'select' ] } + if sys.platform == 'win32': + spaceconfig['usemodules'].append('_rawffi') + else: + spaceconfig['usemodules'].append('fcntl') def setup_class(cls): cls.w_connections = cls.space.newlist([]) From noreply at buildbot.pypy.org Mon Mar 17 21:36:19 2014 From: noreply at buildbot.pypy.org (pjenvey) Date: Mon, 17 Mar 2014 21:36:19 +0100 (CET) Subject: [pypy-commit] pypy py3k: update comment Message-ID: <20140317203619.D1ADE1C00B9@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r70026:523ca391770e Date: 2014-03-17 13:31 -0700 http://bitbucket.org/pypy/pypy/changeset/523ca391770e/ Log: update comment diff --git a/lib-python/3/test/test_aifc.py b/lib-python/3/test/test_aifc.py --- a/lib-python/3/test/test_aifc.py +++ b/lib-python/3/test/test_aifc.py @@ -72,7 +72,7 @@ self.assertEqual(f.getparams(), fout.getparams()) self.assertEqual(f.readframes(5), fout.readframes(5)) - @impl_detail("PyPy has no audioop module yet", pypy=False) + @impl_detail("PyPy has no audioop.lin2ulaw yet", pypy=False) def test_compress(self): f = self.f = aifc.open(self.sndfilepath) fout = self.fout = aifc.open(TESTFN, 'wb') From noreply at buildbot.pypy.org Mon Mar 17 21:36:21 2014 From: noreply at buildbot.pypy.org (pjenvey) Date: Mon, 17 Mar 2014 21:36:21 +0100 (CET) Subject: [pypy-commit] pypy stdlib-3.2.5: add missing test entries Message-ID: <20140317203621.38F471C00B9@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: stdlib-3.2.5 Changeset: r70027:9bce0cf32d0f Date: 2014-03-17 13:34 -0700 http://bitbucket.org/pypy/pypy/changeset/9bce0cf32d0f/ Log: add missing test entries diff --git a/lib-python/conftest.py b/lib-python/conftest.py --- a/lib-python/conftest.py +++ b/lib-python/conftest.py @@ -98,7 +98,7 @@ RegrTest('test___all__.py', core=True), RegrTest('test___future__.py', core=True), RegrTest('test__locale.py', usemodules='_locale'), - #RegrTest('test__osx_support.py'), + RegrTest('test__osx_support.py'), RegrTest('test_abc.py'), RegrTest('test_abstract_numbers.py'), RegrTest('test_aifc.py'), @@ -199,6 +199,7 @@ RegrTest('test_extcall.py', core=True), RegrTest('test_fcntl.py', usemodules='fcntl'), RegrTest('test_file.py', usemodules="posix", core=True), + RegrTest('test_file_eintr.py'), RegrTest('test_filecmp.py', core=True), RegrTest('test_fileinput.py', core=True), RegrTest('test_fileio.py'), @@ -404,7 +405,7 @@ RegrTest('test_timeout.py'), RegrTest('test_tk.py'), RegrTest('test_tokenize.py'), - #RegrTest('test_tools.py'), + RegrTest('test_tools.py'), RegrTest('test_trace.py'), RegrTest('test_traceback.py', core=True), RegrTest('test_ttk_guionly.py'), From noreply at buildbot.pypy.org Mon Mar 17 21:39:51 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 17 Mar 2014 21:39:51 +0100 (CET) Subject: [pypy-commit] pypy improve-consecutive-dict-lookups: merge default Message-ID: <20140317203951.3DCD01C00B9@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: improve-consecutive-dict-lookups Changeset: r70028:44f6695da32e Date: 2014-03-17 14:58 -0400 http://bitbucket.org/pypy/pypy/changeset/44f6695da32e/ Log: merge default diff too long, truncating to 2000 out of 2035 lines diff --git a/_pytest/resultlog.py b/_pytest/resultlog.py --- a/_pytest/resultlog.py +++ b/_pytest/resultlog.py @@ -51,16 +51,22 @@ self.config = config self.logfile = logfile # preferably line buffered - def write_log_entry(self, testpath, lettercode, longrepr): + def write_log_entry(self, testpath, lettercode, longrepr, sections=[]): py.builtin.print_("%s %s" % (lettercode, testpath), file=self.logfile) for line in longrepr.splitlines(): py.builtin.print_(" %s" % line, file=self.logfile) + for key, text in sections: + py.builtin.print_(" ", file=self.logfile) + py.builtin.print_(" -------------------- %s --------------------" + % key.rstrip(), file=self.logfile) + py.builtin.print_(" %s" % (text.rstrip().replace('\n', '\n '),), + file=self.logfile) def log_outcome(self, report, lettercode, longrepr): testpath = getattr(report, 'nodeid', None) if testpath is None: testpath = report.fspath - self.write_log_entry(testpath, lettercode, longrepr) + self.write_log_entry(testpath, lettercode, longrepr, report.sections) def pytest_runtest_logreport(self, report): if report.when != "call" and report.passed: diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py --- a/lib_pypy/_sqlite3.py +++ b/lib_pypy/_sqlite3.py @@ -38,6 +38,7 @@ if sys.version_info[0] >= 3: StandardError = Exception + cmp = lambda x, y: (x > y) - (x < y) long = int xrange = range basestring = unicode = str diff --git a/lib_pypy/audioop.py b/lib_pypy/audioop.py --- a/lib_pypy/audioop.py +++ b/lib_pypy/audioop.py @@ -350,7 +350,7 @@ r_sample = getsample(cp, size, i + 1) sample = (l_sample * fac1) + (r_sample * fac2) - sample = clip(sample) + sample = int(clip(sample)) _put_sample(result, size, i // 2, sample) @@ -501,7 +501,7 @@ # slice off extra bytes trim_index = (out_i * bytes_per_frame) - len(retval) - retval = _buffer(retval)[:trim_index] + retval = retval[:trim_index] return (retval, (d, tuple(samps))) diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst --- a/pypy/doc/faq.rst +++ b/pypy/doc/faq.rst @@ -187,7 +187,7 @@ No, we found no way of doing that. The JIT generates machine code containing a large number of constant addresses --- constant at the time -the machine code is written. The vast majority is probably not at all +the machine code is generated. The vast majority is probably not at all constants that you find in the executable, with a nice link name. E.g. the addresses of Python classes are used all the time, but Python classes don't come statically from the executable; they are created anew @@ -212,12 +212,16 @@ garbage collection, implementation of various things like arbitrarily long integers, etc. -Currently, we have preliminary versions of a JavaScript interpreter -(Leonardo Santagada as his Summer of PyPy project), a `Prolog interpreter`_ -(Carl Friedrich Bolz as his Bachelor thesis), and a `SmallTalk interpreter`_ +Currently, we have `Topaz`_, a Ruby interpreter; `Hippy`_, a PHP +interpreter; preliminary versions of a `JavaScript interpreter`_ +(Leonardo Santagada as his Summer of PyPy project); a `Prolog interpreter`_ +(Carl Friedrich Bolz as his Bachelor thesis); and a `SmallTalk interpreter`_ (produced during a sprint). On the `PyPy bitbucket page`_ there is also a Scheme and an Io implementation; both of these are unfinished at the moment. +.. _`Topaz`: http://topazruby.com/ +.. _`Hippy`: http://morepypy.blogspot.ch/2012/07/hello-everyone.html +.. _`JavaScript interpreter`: https://bitbucket.org/pypy/lang-js/ .. _`Prolog interpreter`: https://bitbucket.org/cfbolz/pyrolog/ .. _`SmallTalk interpreter`: http://dx.doi.org/10.1007/978-3-540-89275-5_7 .. _`PyPy bitbucket page`: https://bitbucket.org/pypy/ diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -105,3 +105,6 @@ .. branch: stdlib-2.7.6 Update stdlib to v2.7.6 + +.. branch: virtual-raw-store-load +Support for virtualizing raw_store/raw_load operations diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py --- a/pypy/module/_codecs/test/test_codecs.py +++ b/pypy/module/_codecs/test/test_codecs.py @@ -330,12 +330,12 @@ raises(UnicodeDecodeError, decode, r"\U00110000") assert decode(r"\U00110000", "ignore") == (u"", 10) assert decode(r"\U00110000", "replace") == (u"\ufffd", 10) - exc = raises(UnicodeDecodeError, unicode_escape_decode, "\u1z32z3", 'strict') - assert str(exc.value) == "'unicodeescape' codec can't decode bytes in position 0-2: truncated \uXXXX escape" - exc = raises(UnicodeDecodeError, raw_unicode_escape_decode, "\u1z32z3", 'strict') - assert str(exc.value) == "'rawunicodeescape' codec can't decode bytes in position 0-2: truncated \uXXXX" - exc = raises(UnicodeDecodeError, raw_unicode_escape_decode, "\U1z32z3", 'strict') - assert str(exc.value) == "'rawunicodeescape' codec can't decode bytes in position 0-2: truncated \uXXXX" + exc = raises(UnicodeDecodeError, unicode_escape_decode, b"\u1z32z3", 'strict') + assert str(exc.value) == r"'unicodeescape' codec can't decode bytes in position 0-2: truncated \uXXXX escape" + exc = raises(UnicodeDecodeError, raw_unicode_escape_decode, b"\u1z32z3", 'strict') + assert str(exc.value) == r"'rawunicodeescape' codec can't decode bytes in position 0-2: truncated \uXXXX" + exc = raises(UnicodeDecodeError, raw_unicode_escape_decode, b"\U1z32z3", 'strict') + assert str(exc.value) == r"'rawunicodeescape' codec can't decode bytes in position 0-2: truncated \uXXXX" def test_escape_encode(self): assert '"'.encode('string_escape') == '"' @@ -596,7 +596,7 @@ l = [u"<%d>" % ord(exc.object[pos]) for pos in xrange(exc.start, exc.end)] return (u"[%s]" % u"".join(l), exc.end) codecs.register_error("test.handler1", handler1) - assert "\\u3042\u3xxx".decode("unicode-escape", "test.handler1") == \ + assert b"\\u3042\u3xxx".decode("unicode-escape", "test.handler1") == \ u"\u3042[<92><117><51>]xxx" def test_encode_error_bad_handler(self): @@ -649,22 +649,22 @@ def test_utf7_errors(self): import codecs tests = [ - ('a\xffb', u'a\ufffdb'), - ('a+IK', u'a\ufffd'), - ('a+IK-b', u'a\ufffdb'), - ('a+IK,b', u'a\ufffdb'), - ('a+IKx', u'a\u20ac\ufffd'), - ('a+IKx-b', u'a\u20ac\ufffdb'), - ('a+IKwgr', u'a\u20ac\ufffd'), - ('a+IKwgr-b', u'a\u20ac\ufffdb'), - ('a+IKwgr,', u'a\u20ac\ufffd'), - ('a+IKwgr,-b', u'a\u20ac\ufffd-b'), - ('a+IKwgrB', u'a\u20ac\u20ac\ufffd'), - ('a+IKwgrB-b', u'a\u20ac\u20ac\ufffdb'), - ('a+/,+IKw-b', u'a\ufffd\u20acb'), - ('a+//,+IKw-b', u'a\ufffd\u20acb'), - ('a+///,+IKw-b', u'a\uffff\ufffd\u20acb'), - ('a+////,+IKw-b', u'a\uffff\ufffd\u20acb'), + (b'a\xffb', u'a\ufffdb'), + (b'a+IK', u'a\ufffd'), + (b'a+IK-b', u'a\ufffdb'), + (b'a+IK,b', u'a\ufffdb'), + (b'a+IKx', u'a\u20ac\ufffd'), + (b'a+IKx-b', u'a\u20ac\ufffdb'), + (b'a+IKwgr', u'a\u20ac\ufffd'), + (b'a+IKwgr-b', u'a\u20ac\ufffdb'), + (b'a+IKwgr,', u'a\u20ac\ufffd'), + (b'a+IKwgr,-b', u'a\u20ac\ufffd-b'), + (b'a+IKwgrB', u'a\u20ac\u20ac\ufffd'), + (b'a+IKwgrB-b', u'a\u20ac\u20ac\ufffdb'), + (b'a+/,+IKw-b', u'a\ufffd\u20acb'), + (b'a+//,+IKw-b', u'a\ufffd\u20acb'), + (b'a+///,+IKw-b', u'a\uffff\ufffd\u20acb'), + (b'a+////,+IKw-b', u'a\uffff\ufffd\u20acb'), ] for raw, expected in tests: raises(UnicodeDecodeError, codecs.utf_7_decode, raw, 'strict', True) diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py --- a/pypy/module/_socket/interp_socket.py +++ b/pypy/module/_socket/interp_socket.py @@ -65,10 +65,7 @@ else: flowinfo = 0 if len(pieces_w) > 3: scope_id = space.uint_w(pieces_w[3]) else: scope_id = 0 - if flowinfo < 0 or flowinfo > 0xfffff: - raise OperationError(space.w_OverflowError, space.wrap( - "flowinfo must be 0-1048575.")) - flowinfo = rffi.cast(lltype.Unsigned, flowinfo) + flowinfo = make_unsigned_flowinfo(space, flowinfo) a = addr.lock(_c.sockaddr_in6) rffi.setintfield(a, 'c_sin6_port', rsocket.htons(port)) rffi.setintfield(a, 'c_sin6_flowinfo', rsocket.htonl(flowinfo)) @@ -97,10 +94,7 @@ else: flowinfo = 0 if len(pieces_w) > 3: scope_id = space.uint_w(pieces_w[3]) else: scope_id = 0 - if flowinfo < 0 or flowinfo > 0xfffff: - raise OperationError(space.w_OverflowError, space.wrap( - "flowinfo must be 0-1048575.")) - flowinfo = rffi.cast(lltype.Unsigned, flowinfo) + flowinfo = make_unsigned_flowinfo(space, flowinfo) return rsocket.INET6Address(host, port, flowinfo, scope_id) if rsocket.HAS_AF_UNIX and family == rsocket.AF_UNIX: return rsocket.UNIXAddress(space.str_w(w_address)) @@ -112,10 +106,16 @@ # XXX Hack to seperate rpython and pypy def make_ushort_port(space, port): if port < 0 or port > 0xffff: - raise OperationError(space.w_ValueError, space.wrap( + raise OperationError(space.w_OverflowError, space.wrap( "port must be 0-65535.")) return rffi.cast(rffi.USHORT, port) +def make_unsigned_flowinfo(space, flowinfo): + if flowinfo < 0 or flowinfo > 0xfffff: + raise OperationError(space.w_OverflowError, space.wrap( + "flowinfo must be 0-1048575.")) + return rffi.cast(lltype.Unsigned, flowinfo) + # XXX Hack to seperate rpython and pypy def ipaddr_from_object(space, w_sockaddr): host = space.str_w(space.getitem(w_sockaddr, space.wrap(0))) @@ -536,13 +536,9 @@ @unwrap_spec(family=int, type=int, proto=int) def newsocket(space, w_subtype, family=AF_INET, type=SOCK_STREAM, proto=0): - # XXX If we want to support subclassing the socket type we will need - # something along these lines. But allocate_instance is only defined - # on the standard object space, so this is not really correct. - #sock = space.allocate_instance(W_RSocket, w_subtype) - #Socket.__init__(sock, space, fd, family, type, proto) + sock = space.allocate_instance(W_RSocket, w_subtype) try: - sock = W_RSocket(family, type, proto) + W_RSocket.__init__(sock, family, type, proto) except SocketError, e: raise converted_error(space, e) return space.wrap(sock) diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py --- a/pypy/module/_socket/test/test_sock_app.py +++ b/pypy/module/_socket/test/test_sock_app.py @@ -399,7 +399,7 @@ name = s.getpeername() # Will raise socket.error if not connected assert name[1] == 80 s.close() - + def test_socket_connect_ex(self): import _socket s = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM, 0) @@ -425,8 +425,13 @@ def test_bigport(self): import _socket s = _socket.socket() - raises(ValueError, s.connect, ("localhost", 1000000)) - raises(ValueError, s.connect, ("localhost", -1)) + exc = raises(OverflowError, s.connect, ("localhost", -1)) + assert "port must be 0-65535." in str(exc.value) + exc = raises(OverflowError, s.connect, ("localhost", 1000000)) + assert "port must be 0-65535." in str(exc.value) + s = _socket.socket(_socket.AF_INET6) + exc = raises(OverflowError, s.connect, ("::1", 1234, 1048576)) + assert "flowinfo must be 0-1048575." in str(exc.value) def test_NtoH(self): import sys @@ -474,6 +479,13 @@ import socket s = socket.socket() + def test_subclass(self): + from _socket import socket + class MySock(socket): + blah = 123 + s = MySock() + assert s.blah == 123 + def test_getsetsockopt(self): import _socket as socket import struct @@ -575,11 +587,11 @@ class AppTestSocketTCP: + HOST = 'localhost' + def setup_class(cls): cls.space = space - HOST = 'localhost' - def setup_method(self, method): w_HOST = space.wrap(self.HOST) self.w_serv = space.appexec([w_socket, w_HOST], @@ -589,6 +601,7 @@ serv.listen(1) return serv ''') + def teardown_method(self, method): if hasattr(self, 'w_serv'): space.appexec([self.w_serv], '(serv): serv.close()') @@ -609,7 +622,7 @@ raises(error, raise_error) def test_recv_send_timeout(self): - from _socket import socket, timeout + from _socket import socket, timeout, SOL_SOCKET, SO_RCVBUF, SO_SNDBUF cli = socket() cli.connect(self.serv.getsockname()) t, addr = self.serv.accept() @@ -629,6 +642,9 @@ assert count is None buf = t.recv(1) assert buf == '?' + # speed up filling the buffers + t.setsockopt(SOL_SOCKET, SO_RCVBUF, 4096) + cli.setsockopt(SOL_SOCKET, SO_SNDBUF, 4096) # test send() timeout count = 0 try: @@ -656,7 +672,7 @@ conn, addr = self.serv.accept() buf = buffer(MSG) conn.send(buf) - buf = array.array('c', ' '*1024) + buf = array.array('c', ' ' * 1024) nbytes = cli.recv_into(buf) assert nbytes == len(MSG) msg = buf.tostring()[:len(MSG)] @@ -671,7 +687,7 @@ conn, addr = self.serv.accept() buf = buffer(MSG) conn.send(buf) - buf = array.array('c', ' '*1024) + buf = array.array('c', ' ' * 1024) nbytes, addr = cli.recvfrom_into(buf) assert nbytes == len(MSG) msg = buf.tostring()[:len(MSG)] @@ -682,6 +698,7 @@ cli = socket.socket(socket.AF_INET, socket.SOCK_STREAM) assert cli.family == socket.AF_INET + class AppTestErrno: def setup_class(cls): cls.space = space diff --git a/pypy/module/cpyext/include/numpy/arrayobject.h b/pypy/module/cpyext/include/numpy/arrayobject.h --- a/pypy/module/cpyext/include/numpy/arrayobject.h +++ b/pypy/module/cpyext/include/numpy/arrayobject.h @@ -20,7 +20,7 @@ #define PyArrayObject PyObject #define PyArray_Descr PyObject -extern PyTypeObject PyArray_Type; +PyAPI_DATA(PyTypeObject) PyArray_Type; typedef unsigned char npy_bool; typedef unsigned char npy_uint8; diff --git a/pypy/module/cpyext/pystate.py b/pypy/module/cpyext/pystate.py --- a/pypy/module/cpyext/pystate.py +++ b/pypy/module/cpyext/pystate.py @@ -213,7 +213,7 @@ if rffi.aroundstate.after: # After external call is before entering Python rffi.aroundstate.after() - return 0 + return lltype.nullptr(PyGILState_STATE.TO) @cpython_api([PyGILState_STATE], lltype.Void) def PyGILState_Release(space, state): diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py --- a/pypy/module/cpyext/test/test_cpyext.py +++ b/pypy/module/cpyext/test/test_cpyext.py @@ -64,6 +64,10 @@ kwds["libraries"] = [api_library] # '%s' undefined; assuming extern returning int kwds["compile_extra"] = ["/we4013"] + # prevent linking with PythonXX.lib + w_maj, w_min = space.fixedview(space.sys.get('version_info'), 5)[:2] + kwds["link_extra"] = ["/NODEFAULTLIB:Python%d%d.lib" % + (space.int_w(w_maj), space.int_w(w_min))] elif sys.platform == 'darwin': kwds["link_files"] = [str(api_library + '.dylib')] else: diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -14,12 +14,12 @@ assert 'foo' in sys.modules assert "copy" in dir(module.fooType) obj = module.new() - print obj.foo + print(obj.foo) assert obj.foo == 42 - print "Obj has type", type(obj) + print("Obj has type", type(obj)) assert type(obj) is module.fooType - print "type of obj has type", type(type(obj)) - print "type of type of obj has type", type(type(type(obj))) + print("type of obj has type", type(type(obj))) + print("type of type of obj has type", type(type(type(obj)))) assert module.fooType.__doc__ == "foo is for testing." def test_typeobject_method_descriptor(self): @@ -36,7 +36,7 @@ assert repr(module.fooType.__call__) == "" assert obj2(foo=1, bar=2) == dict(foo=1, bar=2) - print obj.foo + print(obj.foo) assert obj.foo == 42 assert obj.int_member == obj.foo @@ -592,5 +592,5 @@ def test_tp_new_in_subclass_of_type(self): skip("BROKEN") module = self.import_module(name='foo3') - print 'calling module.Type()...' + print('calling module.Type()...') module.Type("X", (object,), {}) diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py --- a/pypy/module/micronumpy/__init__.py +++ b/pypy/module/micronumpy/__init__.py @@ -6,23 +6,26 @@ interpleveldefs = { 'ndarray': 'ndarray.W_NDimArray', 'dtype': 'descriptor.W_Dtype', + 'flatiter': 'flatiter.W_FlatIterator', + '_reconstruct' : 'ndarray._reconstruct', + 'scalar' : 'ctors.build_scalar', 'array': 'ctors.array', 'zeros': 'ctors.zeros', 'empty': 'ctors.zeros', 'empty_like': 'ctors.empty_like', - '_reconstruct' : 'ndarray._reconstruct', - 'scalar' : 'ctors.build_scalar', + 'fromstring': 'ctors.fromstring', + + 'concatenate': 'arrayops.concatenate', + 'count_nonzero': 'arrayops.count_nonzero', 'dot': 'arrayops.dot', - 'fromstring': 'ctors.fromstring', - 'flatiter': 'flatiter.W_FlatIterator', - 'concatenate': 'arrayops.concatenate', 'where': 'arrayops.where', - 'count_nonzero': 'arrayops.count_nonzero', 'set_string_function': 'appbridge.set_string_function', 'typeinfo': 'descriptor.get_dtype_cache(space).w_typeinfo', } + for c in ['MAXDIMS', 'CLIP', 'WRAP', 'RAISE']: + interpleveldefs[c] = 'space.wrap(constants.%s)' % c class UMathModule(MixedModule): diff --git a/pypy/module/micronumpy/base.py b/pypy/module/micronumpy/base.py --- a/pypy/module/micronumpy/base.py +++ b/pypy/module/micronumpy/base.py @@ -58,10 +58,10 @@ elif owning: # Will free storage when GCd impl = concrete.ConcreteArray(shape, dtype, order, strides, - backstrides, storage=storage) + backstrides, storage=storage) else: impl = concrete.ConcreteArrayNotOwning(shape, dtype, order, strides, - backstrides, storage) + backstrides, storage) if w_subtype: w_ret = space.allocate_instance(W_NDimArray, w_subtype) W_NDimArray.__init__(w_ret, impl) diff --git a/pypy/module/micronumpy/boxes.py b/pypy/module/micronumpy/boxes.py --- a/pypy/module/micronumpy/boxes.py +++ b/pypy/module/micronumpy/boxes.py @@ -87,7 +87,8 @@ value[0] = self.value builder = StringBuilder() - builder.append_charpsize(rffi.cast(rffi.CCHARP, value), rffi.sizeof(lltype.typeOf(self.value))) + builder.append_charpsize(rffi.cast(rffi.CCHARP, value), + rffi.sizeof(lltype.typeOf(self.value))) ret = builder.build() lltype.free(value, flavor="raw") @@ -117,7 +118,8 @@ value[1] = self.imag builder = StringBuilder() - builder.append_charpsize(rffi.cast(rffi.CCHARP, value), rffi.sizeof(lltype.typeOf(self.real)) * 2) + builder.append_charpsize(rffi.cast(rffi.CCHARP, value), + rffi.sizeof(lltype.typeOf(self.real)) * 2) ret = builder.build() lltype.free(value, flavor="raw") @@ -186,27 +188,27 @@ dtype = self.get_dtype(space) return space.wrap(dtype.itemtype.bool(self)) + def _unaryop_impl(ufunc_name): + def impl(self, space, w_out=None): + from pypy.module.micronumpy import ufuncs + return getattr(ufuncs.get(space), ufunc_name).call( + space, [self, w_out]) + return func_with_new_name(impl, "unaryop_%s_impl" % ufunc_name) + def _binop_impl(ufunc_name): def impl(self, space, w_other, w_out=None): from pypy.module.micronumpy import ufuncs - return getattr(ufuncs.get(space), ufunc_name).call(space, - [self, w_other, w_out]) + return getattr(ufuncs.get(space), ufunc_name).call( + space, [self, w_other, w_out]) return func_with_new_name(impl, "binop_%s_impl" % ufunc_name) def _binop_right_impl(ufunc_name): def impl(self, space, w_other, w_out=None): from pypy.module.micronumpy import ufuncs - return getattr(ufuncs.get(space), ufunc_name).call(space, - [w_other, self, w_out]) + return getattr(ufuncs.get(space), ufunc_name).call( + space, [w_other, self, w_out]) return func_with_new_name(impl, "binop_right_%s_impl" % ufunc_name) - def _unaryop_impl(ufunc_name): - def impl(self, space, w_out=None): - from pypy.module.micronumpy import ufuncs - return getattr(ufuncs.get(space), ufunc_name).call(space, - [self, w_out]) - return func_with_new_name(impl, "unaryop_%s_impl" % ufunc_name) - descr_add = _binop_impl("add") descr_sub = _binop_impl("subtract") descr_mul = _binop_impl("multiply") diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py --- a/pypy/module/micronumpy/concrete.py +++ b/pypy/module/micronumpy/concrete.py @@ -368,13 +368,10 @@ class ConcreteArray(ConcreteArrayNotOwning): def __init__(self, shape, dtype, order, strides, backstrides, storage=lltype.nullptr(RAW_STORAGE)): - null_storage = lltype.nullptr(RAW_STORAGE) + if storage == lltype.nullptr(RAW_STORAGE): + storage = dtype.itemtype.malloc(support.product(shape) * dtype.elsize) ConcreteArrayNotOwning.__init__(self, shape, dtype, order, strides, backstrides, - null_storage) - if storage == lltype.nullptr(RAW_STORAGE): - self.storage = dtype.itemtype.malloc(self.size) - else: - self.storage = storage + storage) def __del__(self): free_raw_storage(self.storage, track_allocation=False) diff --git a/pypy/module/micronumpy/constants.py b/pypy/module/micronumpy/constants.py --- a/pypy/module/micronumpy/constants.py +++ b/pypy/module/micronumpy/constants.py @@ -1,3 +1,5 @@ +MAXDIMS = 32 + BOOL = 0 BYTE = 1 UBYTE = 2 diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py --- a/pypy/module/micronumpy/ctors.py +++ b/pypy/module/micronumpy/ctors.py @@ -5,7 +5,6 @@ from pypy.module.micronumpy import descriptor, loop, ufuncs from pypy.module.micronumpy.base import W_NDimArray, convert_to_array from pypy.module.micronumpy.converters import shape_converter -from pypy.module.micronumpy.strides import find_shape_and_elems def build_scalar(space, w_dtype, w_state): @@ -27,6 +26,8 @@ @unwrap_spec(ndmin=int, copy=bool, subok=bool) def array(space, w_object, w_dtype=None, copy=True, w_order=None, subok=False, ndmin=0): + from pypy.module.micronumpy import strides + # for anything that isn't already an array, try __array__ method first if not isinstance(w_object, W_NDimArray): w___array__ = space.lookup(w_object, "__array__") @@ -68,12 +69,9 @@ return w_ret # not an array or incorrect dtype - shape, elems_w = find_shape_and_elems(space, w_object, dtype) + shape, elems_w = strides.find_shape_and_elems(space, w_object, dtype) if dtype is None or (dtype.is_str_or_unicode() and dtype.elsize < 1): - for w_elem in elems_w: - if isinstance(w_elem, W_NDimArray) and w_elem.is_scalar(): - w_elem = w_elem.get_scalar_value() - dtype = ufuncs.find_dtype_for_scalar(space, w_elem, dtype) + dtype = strides.find_dtype_for_seq(space, elems_w, dtype) if dtype is None: dtype = descriptor.get_dtype_cache(space).w_float64dtype elif dtype.is_str_or_unicode() and dtype.elsize < 1: @@ -83,10 +81,10 @@ if ndmin > len(shape): shape = [1] * (ndmin - len(shape)) + shape w_arr = W_NDimArray.from_shape(space, shape, dtype, order=order) - arr_iter = w_arr.create_iter() - for w_elem in elems_w: - arr_iter.setitem(dtype.coerce(space, w_elem)) - arr_iter.next() + if len(elems_w) == 1: + w_arr.set_scalar_value(dtype.coerce(space, elems_w[0])) + else: + loop.assign(space, w_arr, elems_w) return w_arr diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py --- a/pypy/module/micronumpy/loop.py +++ b/pypy/module/micronumpy/loop.py @@ -164,6 +164,13 @@ arr_iter.setitem(box) arr_iter.next() +def assign(space, arr, seq): + arr_iter = arr.create_iter() + arr_dtype = arr.get_dtype() + for item in seq: + arr_iter.setitem(arr_dtype.coerce(space, item)) + arr_iter.next() + where_driver = jit.JitDriver(name='numpy_where', greens = ['shapelen', 'dtype', 'arr_dtype'], reds = 'auto') diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py --- a/pypy/module/micronumpy/ndarray.py +++ b/pypy/module/micronumpy/ndarray.py @@ -1045,6 +1045,26 @@ value = self.get_scalar_value() return space.float(value) + def descr_hex(self, space): + if self.get_size() != 1: + raise oefmt(space.w_TypeError, + "only length-1 arrays can be converted to Python scalars") + if not self.get_dtype().is_int(): + raise oefmt(space.w_TypeError, + "don't know how to convert scalar number to hex") + value = self.get_scalar_value() + return space.hex(value) + + def descr_oct(self, space): + if self.get_size() != 1: + raise oefmt(space.w_TypeError, + "only length-1 arrays can be converted to Python scalars") + if not self.get_dtype().is_int(): + raise oefmt(space.w_TypeError, + "don't know how to convert scalar number to oct") + value = self.get_scalar_value() + return space.oct(value) + def descr_index(self, space): if self.get_size() != 1 or \ not self.get_dtype().is_int() or self.get_dtype().is_bool(): @@ -1237,6 +1257,8 @@ __int__ = interp2app(W_NDimArray.descr_int), __long__ = interp2app(W_NDimArray.descr_long), __float__ = interp2app(W_NDimArray.descr_float), + __hex__ = interp2app(W_NDimArray.descr_hex), + __oct__ = interp2app(W_NDimArray.descr_oct), __buffer__ = interp2app(W_NDimArray.descr_get_data), __index__ = interp2app(W_NDimArray.descr_index), diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py --- a/pypy/module/micronumpy/strides.py +++ b/pypy/module/micronumpy/strides.py @@ -181,6 +181,10 @@ return [], [w_iterable] if isinstance(w_iterable, W_NDimArray) and w_iterable.is_scalar(): return [], [w_iterable] + return _find_shape_and_elems(space, w_iterable, is_rec_type) + + +def _find_shape_and_elems(space, w_iterable, is_rec_type): shape = [space.len_w(w_iterable)] batch = space.listview(w_iterable) while True: @@ -210,6 +214,25 @@ batch = new_batch +def find_dtype_for_seq(space, elems_w, dtype): + from pypy.module.micronumpy.ufuncs import find_dtype_for_scalar + if len(elems_w) == 1: + w_elem = elems_w[0] + if isinstance(w_elem, W_NDimArray) and w_elem.is_scalar(): + w_elem = w_elem.get_scalar_value() + return find_dtype_for_scalar(space, w_elem, dtype) + return _find_dtype_for_seq(space, elems_w, dtype) + + +def _find_dtype_for_seq(space, elems_w, dtype): + from pypy.module.micronumpy.ufuncs import find_dtype_for_scalar + for w_elem in elems_w: + if isinstance(w_elem, W_NDimArray) and w_elem.is_scalar(): + w_elem = w_elem.get_scalar_value() + dtype = find_dtype_for_scalar(space, w_elem, dtype) + return dtype + + def to_coords(space, shape, size, order, w_item_or_slice): '''Returns a start coord, step, and length. ''' diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py --- a/pypy/module/micronumpy/test/test_ndarray.py +++ b/pypy/module/micronumpy/test/test_ndarray.py @@ -246,6 +246,13 @@ return CustomIntObject(value) + def test_constants(self): + import numpy as np + assert np.MAXDIMS is 32 + assert np.CLIP is 0 + assert np.WRAP is 1 + assert np.RAISE is 2 + def test_ndarray(self): from numpy import ndarray, array, dtype, flatiter @@ -2276,6 +2283,30 @@ exc = raises(TypeError, "float(np.array([1.5, 2.5]))") assert exc.value[0] == 'only length-1 arrays can be converted to Python scalars' + def test__hex__(self): + import numpy as np + assert hex(np.array(True)) == '0x1' + assert hex(np.array(15)) == '0xf' + assert hex(np.array([15])) == '0xf' + exc = raises(TypeError, "hex(np.array(1.5))") + assert str(exc.value) == "don't know how to convert scalar number to hex" + exc = raises(TypeError, "hex(np.array('15'))") + assert str(exc.value) == "don't know how to convert scalar number to hex" + exc = raises(TypeError, "hex(np.array([1, 2]))") + assert str(exc.value) == "only length-1 arrays can be converted to Python scalars" + + def test__oct__(self): + import numpy as np + assert oct(np.array(True)) == '01' + assert oct(np.array(15)) == '017' + assert oct(np.array([15])) == '017' + exc = raises(TypeError, "oct(np.array(1.5))") + assert str(exc.value) == "don't know how to convert scalar number to oct" + exc = raises(TypeError, "oct(np.array('15'))") + assert str(exc.value) == "don't know how to convert scalar number to oct" + exc = raises(TypeError, "oct(np.array([1, 2]))") + assert str(exc.value) == "only length-1 arrays can be converted to Python scalars" + def test__reduce__(self): from numpypy import array, dtype from cPickle import loads, dumps diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py --- a/pypy/module/micronumpy/test/test_zjit.py +++ b/pypy/module/micronumpy/test/test_zjit.py @@ -480,23 +480,19 @@ def test_flat_setitem(self): result = self.run("flat_setitem") assert result == 1.0 - py.test.skip("don't run for now") self.check_trace_count(1) - # XXX not ideal, but hey, let's ignore it for now - self.check_simple_loop({'raw_load': 1, - 'raw_store': 1, - 'int_lt': 1, - 'int_gt': 1, - 'int_add': 4, - 'guard_true': 2, - 'arraylen_gc': 2, - 'jump': 1, - 'int_sub': 1, - # XXX bad part - 'int_and': 1, - 'int_mod': 1, - 'int_rshift': 1, - }) + self.check_simple_loop({ + 'call': 2, + 'getfield_gc': 2, + 'guard_no_exception': 2, + 'guard_not_invalidated': 1, + 'guard_true': 1, + 'int_gt': 1, + 'int_sub': 1, + 'jump': 1, + 'raw_load': 1, + 'raw_store': 1, + }) def define_dot(): return """ @@ -509,6 +505,7 @@ def test_dot(self): result = self.run("dot") assert result == 184 + self.check_trace_count(3) self.check_simple_loop({'float_add': 1, 'float_mul': 1, 'guard_not_invalidated': 1, @@ -526,7 +523,7 @@ 'guard_class': 4, 'guard_false': 2, 'guard_no_exception': 3, - 'guard_nonnull': 8, + 'guard_nonnull': 12, 'guard_nonnull_class': 4, 'guard_not_invalidated': 2, 'guard_true': 9, diff --git a/pypy/module/pypyjit/test/test_ztranslation.py b/pypy/module/pypyjit/test/test_ztranslation.py --- a/pypy/module/pypyjit/test/test_ztranslation.py +++ b/pypy/module/pypyjit/test/test_ztranslation.py @@ -1,5 +1,5 @@ +from pypy.objspace.fake.checkmodule import checkmodule -from pypy.objspace.fake.checkmodule import checkmodule def test_pypyjit_translates(): checkmodule('pypyjit') diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py --- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py +++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py @@ -7,13 +7,13 @@ from rpython.tool.udir import udir from rpython.tool import logparser from rpython.jit.tool.jitoutput import parse_prof -from pypy.module.pypyjit.test_pypy_c.model import (Log, find_ids_range, - find_ids, - OpMatcher, InvalidMatch) +from pypy.module.pypyjit.test_pypy_c.model import \ + Log, find_ids_range, find_ids, OpMatcher, InvalidMatch + class BaseTestPyPyC(object): log_string = 'jit-log-opt,jit-log-noopt,jit-log-virtualstate,jit-summary' - + def setup_class(cls): if '__pypy__' not in sys.builtin_module_names: py.test.skip("must run this test with pypy") @@ -98,7 +98,6 @@ class TestLog(object): - def test_find_ids_range(self): def f(): a = 0 # ID: myline @@ -127,7 +126,6 @@ class TestOpMatcher_(object): - def match(self, src1, src2, **kwds): from rpython.tool.jitlogparser.parser import SimpleParser loop = SimpleParser.parse_from_input(src1) @@ -347,7 +345,6 @@ class TestRunPyPyC(BaseTestPyPyC): - def test_run_function(self): def f(a, b): return a+b @@ -385,7 +382,7 @@ assert len(loops) == 1 assert loops[0].filename == self.filepath assert len([op for op in loops[0].allops() if op.name == 'label']) == 0 - assert len([op for op in loops[0].allops() if op.name == 'guard_nonnull_class']) == 0 + assert len([op for op in loops[0].allops() if op.name == 'guard_nonnull_class']) == 0 # loops = log.loops_by_filename(self.filepath, is_entry_bridge=True) assert len(loops) == 1 @@ -454,7 +451,6 @@ # ops = loop.ops_by_id('foo', opcode='INPLACE_SUBTRACT') assert log.opnames(ops) == ['int_sub_ovf', 'guard_no_overflow'] - def test_inlined_function(self): def f(): diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py --- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py +++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py @@ -341,13 +341,11 @@ guard_value(p166, ConstPtr(ptr72), descr=...) p167 = call(ConstClass(_ll_0_alloc_with_del___), descr=) guard_no_exception(descr=...) - i168 = call(ConstClass(_ll_1_raw_malloc_varsize__Signed), 6, descr=) - setfield_gc(p167, 0, descr=) - setfield_gc(p167, ConstPtr(ptr86), descr=) - guard_no_exception(descr=...) + i168 = call(ConstClass(_ll_1_raw_malloc_varsize__Signed), 6, descr=) i169 = int_add(i168, i97) i170 = int_sub(i160, i106) setfield_gc(p167, i168, descr=) + setfield_gc(p167, ConstPtr(null), descr=) setfield_gc(p167, ConstPtr(ptr89), descr=) i171 = uint_gt(i170, i108) guard_false(i171, descr=...) diff --git a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py --- a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py +++ b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py @@ -43,6 +43,7 @@ log = self.run(main, []) assert log.result == 0 loop, = log.loops_by_filename(self.filepath) + skip('used to pass on 69421-f3e717c94913') assert loop.match(""" i81 = int_lt(i76, 300) guard_true(i81, descr=...) diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py --- a/rpython/annotator/test/test_annrpython.py +++ b/rpython/annotator/test/test_annrpython.py @@ -2872,7 +2872,6 @@ py.test.raises(Exception, a.build_types, fun, [s_nonneg, int]) def test_sig_bug(self): - py.test.skip("_annenforceargs_ does not work for default arguments") def g(x, y=5): return y == 5 g._annenforceargs_ = (int, int) @@ -2880,7 +2879,8 @@ return g(x) a = self.RPythonAnnotator() s = a.build_types(fun, [int]) - assert not s.is_constant() + assert s.knowntype is bool + assert s.is_constant() def test_sig_list(self): def g(buf): diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py --- a/rpython/jit/codewriter/jtransform.py +++ b/rpython/jit/codewriter/jtransform.py @@ -550,20 +550,18 @@ track_allocation = d.pop('track_allocation', True) if d: raise UnsupportedMallocFlags(d) - TYPE = op.args[0].value if zero: name += '_zero' if add_memory_pressure: name += '_add_memory_pressure' if not track_allocation: name += '_no_track_allocation' + TYPE = op.args[0].value op1 = self.prepare_builtin_call(op, name, args, (TYPE,), TYPE) - if name == 'raw_malloc_varsize': - ITEMTYPE = op.args[0].value.OF - if ITEMTYPE == lltype.Char: - return self._handle_oopspec_call(op1, args, - EffectInfo.OS_RAW_MALLOC_VARSIZE_CHAR, - EffectInfo.EF_CAN_RAISE) + if name.startswith('raw_malloc_varsize') and TYPE.OF == lltype.Char: + return self._handle_oopspec_call(op1, args, + EffectInfo.OS_RAW_MALLOC_VARSIZE_CHAR, + EffectInfo.EF_CAN_RAISE) return self.rewrite_op_direct_call(op1) def rewrite_op_malloc_varsize(self, op): @@ -594,7 +592,7 @@ name += '_no_track_allocation' op1 = self.prepare_builtin_call(op, name, [op.args[0]], (STRUCT,), STRUCT) - if name == 'raw_free': + if name.startswith('raw_free'): return self._handle_oopspec_call(op1, [op.args[0]], EffectInfo.OS_RAW_FREE, EffectInfo.EF_CANNOT_RAISE) @@ -840,8 +838,8 @@ RESULT = lltype.Ptr(STRUCT) assert RESULT == op.result.concretetype return self._do_builtin_call(op, 'alloc_with_del', [], - extra = (RESULT, vtable), - extrakey = STRUCT) + extra=(RESULT, vtable), + extrakey=STRUCT) heaptracker.register_known_gctype(self.cpu, vtable, STRUCT) opname = 'new_with_vtable' else: @@ -1240,7 +1238,7 @@ op1 = self.prepare_builtin_call(op, "llong_%s", args) op2 = self._handle_oopspec_call(op1, args, EffectInfo.OS_LLONG_%s, - EffectInfo.EF_ELIDABLE_CANNOT_RAISE) + EffectInfo.EF_ELIDABLE_CANNOT_RAISE) if %r == "TO_INT": assert op2.result.concretetype == lltype.Signed return op2 @@ -1272,7 +1270,7 @@ op1 = self.prepare_builtin_call(op, "ullong_%s", args) op2 = self._handle_oopspec_call(op1, args, EffectInfo.OS_LLONG_%s, - EffectInfo.EF_ELIDABLE_CANNOT_RAISE) + EffectInfo.EF_ELIDABLE_CANNOT_RAISE) return op2 ''' % (_op, _oopspec.lower(), _oopspec)).compile() diff --git a/rpython/jit/codewriter/test/test_jtransform.py b/rpython/jit/codewriter/test/test_jtransform.py --- a/rpython/jit/codewriter/test/test_jtransform.py +++ b/rpython/jit/codewriter/test/test_jtransform.py @@ -60,7 +60,7 @@ class FakeResidualCallControl: def guess_call_kind(self, op): return 'residual' - def getcalldescr(self, op, **kwds): + def getcalldescr(self, op, oopspecindex=None, extraeffect=None): return 'calldescr' def calldescr_canraise(self, calldescr): return True diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py --- a/rpython/jit/metainterp/optimizeopt/heap.py +++ b/rpython/jit/metainterp/optimizeopt/heap.py @@ -1,7 +1,7 @@ import os from rpython.jit.codewriter.effectinfo import EffectInfo -from rpython.jit.metainterp.optimizeopt.util import args_dict_value +from rpython.jit.metainterp.optimizeopt.util import args_dict from rpython.jit.metainterp.history import Const from rpython.jit.metainterp.jitexc import JitException from rpython.jit.metainterp.optimizeopt.optimizer import Optimization, MODE_ARRAY, LEVEL_KNOWNCLASS @@ -95,6 +95,11 @@ # possible aliasing). self.clear() self._lazy_setfield = None + if optheap.postponed_op: + for a in op.getarglist(): + if a is optheap.postponed_op.result: + optheap.emit_postponed_op() + break optheap.next_optimization.propagate_forward(op) if not can_cache: return @@ -186,6 +191,9 @@ def flush(self): self.cached_dict_reads.clear() self.force_all_lazy_setfields_and_arrayitems() + self.emit_postponed_op() + + def emit_postponed_op(self): if self.postponed_op: postponed_op = self.postponed_op self.postponed_op = None @@ -235,10 +243,7 @@ def emit_operation(self, op): self.emitting_operation(op) - if self.postponed_op: - postponed_op = self.postponed_op - self.postponed_op = None - self.next_optimization.propagate_forward(postponed_op) + self.emit_postponed_op() if (op.is_comparison() or op.getopnum() == rop.CALL_MAY_FORCE or op.is_ovf()): self.postponed_op = op @@ -302,7 +307,7 @@ if descr in self.cached_dict_reads: d = self.cached_dict_reads[descr] else: - d = args_dict_value() + d = args_dict() self.cached_dict_reads[descr] = d try: res_v = d[args] diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py @@ -1660,6 +1660,16 @@ """ self.optimize_loop(ops, ops) + def test_setfield_int_eq_result(self): + # test that the setfield_gc does not end up before int_eq + ops = """ + [p1, i1, i2] + i3 = int_eq(i1, i2) + setfield_gc(p1, i3, descr=valuedescr) + jump(p1, i1, i2) + """ + self.optimize_loop(ops, ops) + def test_duplicate_setfield_aliasing(self): # a case where aliasing issues (and not enough cleverness) mean # that we fail to remove any setfield_gc diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -1,14 +1,15 @@ import py from rpython.rlib.objectmodel import instantiate +from rpython.jit.metainterp import compile, resume +from rpython.jit.metainterp.history import AbstractDescr, ConstInt, BoxInt, TreeLoop +from rpython.jit.metainterp.optimize import InvalidLoop +from rpython.jit.metainterp.optimizeopt import build_opt_chain from rpython.jit.metainterp.optimizeopt.test.test_util import ( LLtypeMixin, BaseTest, convert_old_style_to_targets) -from rpython.jit.metainterp.optimizeopt import build_opt_chain -from rpython.jit.metainterp.optimize import InvalidLoop -from rpython.jit.metainterp.history import AbstractDescr, ConstInt, BoxInt -from rpython.jit.metainterp.history import TreeLoop -from rpython.jit.metainterp import compile, resume +from rpython.jit.metainterp.optimizeopt.test.test_optimizebasic import \ + FakeMetaInterpStaticData from rpython.jit.metainterp.resoperation import rop, opname, oparity -from rpython.jit.metainterp.optimizeopt.test.test_optimizebasic import FakeMetaInterpStaticData + def test_build_opt_chain(): def check(chain, expected_names): @@ -40,7 +41,6 @@ class BaseTestWithUnroll(BaseTest): - enable_opts = "intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll" def optimize_loop(self, ops, expected, expected_preamble=None, @@ -93,8 +93,8 @@ def raises(self, e, fn, *args): return py.test.raises(e, fn, *args).value + class OptimizeOptTest(BaseTestWithUnroll): - def setup_method(self, meth=None): class FailDescr(compile.ResumeGuardDescr): oparse = None @@ -130,7 +130,6 @@ self.namespace.pop('fdescr', None) self.namespace.pop('fdescr2', None) - def test_simple(self): ops = """ [] @@ -606,9 +605,9 @@ i1 = ptr_eq(p0, NULL) guard_false(i1) [] i2 = ptr_ne(NULL, p0) - guard_true(i0) [] + guard_true(i2) [] i3 = ptr_eq(NULL, p0) - guard_false(i1) [] + guard_false(i3) [] guard_nonnull(p0) [] jump(p0) """ @@ -623,6 +622,30 @@ """ self.optimize_loop(ops, expected, preamble) + def test_nonnull_2(self): + ops = """ + [] + p0 = new_array(5, descr=arraydescr) # forces p0 != NULL + i0 = ptr_ne(p0, NULL) + guard_true(i0) [] + i1 = ptr_eq(p0, NULL) + guard_false(i1) [] + i2 = ptr_ne(NULL, p0) + guard_true(i2) [] + i3 = ptr_eq(NULL, p0) + guard_false(i3) [] + guard_nonnull(p0) [] + escape(p0) + jump() + """ + expected = """ + [] + p0 = new_array(5, descr=arraydescr) + escape(p0) + jump() + """ + self.optimize_loop(ops, expected) + def test_const_guard_value(self): ops = """ [] @@ -974,7 +997,6 @@ """ self.optimize_loop(ops, expected, preamble) - # ---------- def test_virtual_1(self): @@ -1252,7 +1274,6 @@ """ self.optimize_loop(ops, expected, preamble) - def test_virtual_constant_isnonnull(self): ops = """ [i0] @@ -1728,10 +1749,11 @@ # We cannot track virtuals that survive for more than two iterations. self.optimize_loop(ops, expected, preamble) - def test_virtual_raw_malloc(self): + def test_virtual_raw_malloc_basic(self): ops = """ [i1] i2 = call('malloc', 10, descr=raw_malloc_descr) + guard_no_exception() [] setarrayitem_raw(i2, 0, i1, descr=rawarraydescr) i3 = getarrayitem_raw(i2, 0, descr=rawarraydescr) call('free', i2, descr=raw_free_descr) @@ -1743,13 +1765,32 @@ """ self.optimize_loop(ops, expected) + def test_virtual_raw_malloc_const(self): + ops = """ + [i1] + i5 = int_mul(10, 1) + i2 = call('malloc', i5, descr=raw_malloc_descr) + guard_no_exception() [] + setarrayitem_raw(i2, 0, i1, descr=rawarraydescr) + i3 = getarrayitem_raw(i2, 0, descr=rawarraydescr) + call('free', i2, descr=raw_free_descr) + jump(i3) + """ + expected = """ + [i1] + jump(i1) + """ + self.optimize_loop(ops, expected) + def test_virtual_raw_malloc_force(self): ops = """ [i1] - i2 = call('malloc', 10, descr=raw_malloc_descr) + i2 = call('malloc', 20, descr=raw_malloc_descr) + guard_no_exception() [] setarrayitem_raw(i2, 0, i1, descr=rawarraydescr_char) setarrayitem_raw(i2, 2, 456, descr=rawarraydescr_char) setarrayitem_raw(i2, 1, 123, descr=rawarraydescr_char) + setarrayitem_raw(i2, 1, 789, descr=rawarraydescr_float) label('foo') # we expect the buffer to be forced *after* the label escape(i2) call('free', i2, descr=raw_free_descr) @@ -1758,12 +1799,12 @@ expected = """ [i1] label('foo') - i2 = call('malloc', 10, descr=raw_malloc_descr) - setarrayitem_raw(i2, 0, i1, descr=rawarraydescr_char) - i3 = int_add(i2, 1) - setarrayitem_raw(i3, 0, 123, descr=rawarraydescr_char) - i4 = int_add(i2, 2) - setarrayitem_raw(i4, 0, 456, descr=rawarraydescr_char) + i2 = call('malloc', 20, descr=raw_malloc_descr) + #guard_no_exception() [] # XXX should appear + raw_store(i2, 0, i1, descr=rawarraydescr_char) + raw_store(i2, 1, 123, descr=rawarraydescr_char) + raw_store(i2, 2, 456, descr=rawarraydescr_char) + raw_store(i2, 8, 789, descr=rawarraydescr_float) escape(i2) call('free', i2, descr=raw_free_descr) jump(i1) @@ -1774,6 +1815,7 @@ ops = """ [i1] i2 = call('malloc', 10, descr=raw_malloc_descr) + guard_no_exception() [] setarrayitem_raw(i2, 0, i1, descr=rawarraydescr) label('foo') # we expect the buffer to be forced *after* the label setarrayitem_raw(i2, 2, 456, descr=rawarraydescr_char) # overlap! @@ -1784,7 +1826,8 @@ [i1] label('foo') i2 = call('malloc', 10, descr=raw_malloc_descr) - setarrayitem_raw(i2, 0, i1, descr=rawarraydescr) + #guard_no_exception() [] # XXX should appear + raw_store(i2, 0, i1, descr=rawarraydescr) setarrayitem_raw(i2, 2, 456, descr=rawarraydescr_char) call('free', i2, descr=raw_free_descr) jump(i1) @@ -1795,6 +1838,7 @@ ops = """ [i1] i2 = call('malloc', 10, descr=raw_malloc_descr) + guard_no_exception() [] setarrayitem_raw(i2, 0, i1, descr=rawarraydescr) label('foo') # we expect the buffer to be forced *after* the label i3 = getarrayitem_raw(i2, 0, descr=rawarraydescr_char) @@ -1805,7 +1849,8 @@ [i1] label('foo') i2 = call('malloc', 10, descr=raw_malloc_descr) - setarrayitem_raw(i2, 0, i1, descr=rawarraydescr) + #guard_no_exception() [] # XXX should appear + raw_store(i2, 0, i1, descr=rawarraydescr) i3 = getarrayitem_raw(i2, 0, descr=rawarraydescr_char) call('free', i2, descr=raw_free_descr) jump(i1) @@ -1816,6 +1861,7 @@ ops = """ [i0, i1] i2 = call('malloc', 10, descr=raw_malloc_descr) + guard_no_exception() [] setarrayitem_raw(i2, 0, 42, descr=rawarraydescr_char) i3 = int_add(i2, 1) # get a slice of the original buffer setarrayitem_raw(i3, 0, 4242, descr=rawarraydescr) # write to the slice @@ -1835,6 +1881,7 @@ ops = """ [i0, i1] i2 = call('malloc', 10, descr=raw_malloc_descr) + guard_no_exception() [] i3 = int_add(i2, 1) # get a slice of the original buffer i4 = int_add(i3, 1) # get a slice of a slice setarrayitem_raw(i4, 0, i1, descr=rawarraydescr_char) # write to the slice @@ -1852,6 +1899,7 @@ ops = """ [i0, i1] i2 = call('malloc', 10, descr=raw_malloc_descr) + guard_no_exception() [] setarrayitem_raw(i2, 0, 42, descr=rawarraydescr_char) i3 = int_add(i2, 1) # get a slice of the original buffer setarrayitem_raw(i3, 4, 4242, descr=rawarraydescr_char) # write to the slice @@ -1864,9 +1912,9 @@ label('foo') # these ops are generated by VirtualRawBufferValue._really_force i2 = call('malloc', 10, descr=raw_malloc_descr) - setarrayitem_raw(i2, 0, 42, descr=rawarraydescr_char) - i3 = int_add(i2, 5) # 1+4*sizeof(char) - setarrayitem_raw(i3, 0, 4242, descr=rawarraydescr_char) + #guard_no_exception() [] # XXX should appear + raw_store(i2, 0, 42, descr=rawarraydescr_char) + raw_store(i2, 5, 4242, descr=rawarraydescr_char) # this is generated by VirtualRawSliceValue._really_force i4 = int_add(i2, 1) escape(i4) @@ -1881,6 +1929,7 @@ i2 = int_add(i1, 1) call('free', i0, descr=raw_free_descr) i3 = call('malloc', 10, descr=raw_malloc_descr) + guard_no_exception() [] setarrayitem_raw(i3, 0, i2, descr=rawarraydescr) label('foo') jump(i3) @@ -1892,11 +1941,66 @@ call('free', i0, descr=raw_free_descr) label('foo') i3 = call('malloc', 10, descr=raw_malloc_descr) - setarrayitem_raw(i3, 0, i2, descr=rawarraydescr) + #guard_no_exception() [] # XXX should appear + raw_store(i3, 0, i2, descr=rawarraydescr) jump(i3) """ self.optimize_loop(ops, expected) + def test_virtual_raw_store_raw_load(self): + ops = """ + [i1] + i0 = call('malloc', 10, descr=raw_malloc_descr) + guard_no_exception() [] + raw_store(i0, 0, i1, descr=rawarraydescr) + i2 = raw_load(i0, 0, descr=rawarraydescr) + i3 = int_add(i1, i2) + call('free', i0, descr=raw_free_descr) + jump(i3) + """ + expected = """ + [i1] + i2 = int_add(i1, i1) + jump(i2) + """ + self.optimize_loop(ops, expected) + + def test_virtual_raw_store_getarrayitem_raw(self): + ops = """ + [f1] + i0 = call('malloc', 16, descr=raw_malloc_descr) + guard_no_exception() [] + raw_store(i0, 8, f1, descr=rawarraydescr_float) + f2 = getarrayitem_raw(i0, 1, descr=rawarraydescr_float) + f3 = float_add(f1, f2) + call('free', i0, descr=raw_free_descr) + jump(f3) + """ + expected = """ + [f1] + f2 = float_add(f1, f1) + jump(f2) + """ + self.optimize_loop(ops, expected) + + def test_virtual_setarrayitem_raw_raw_load(self): + ops = """ + [f1] + i0 = call('malloc', 16, descr=raw_malloc_descr) + guard_no_exception() [] + setarrayitem_raw(i0, 1, f1, descr=rawarraydescr_float) + f2 = raw_load(i0, 8, descr=rawarraydescr_float) + f3 = float_add(f1, f2) + call('free', i0, descr=raw_free_descr) + jump(f3) + """ + expected = """ + [f1] + f2 = float_add(f1, f1) + jump(f2) + """ + self.optimize_loop(ops, expected) + def test_duplicate_getfield_1(self): ops = """ [p1, p2] @@ -2789,8 +2893,7 @@ p2 = new_with_vtable(ConstClass(node_vtable)) jump(p2) """ - self.raises(InvalidLoop, self.optimize_loop, - ops, "crash!") + self.raises(InvalidLoop, self.optimize_loop, ops, "crash!") def test_invalid_loop_2(self): ops = """ @@ -2801,8 +2904,7 @@ escape(p2) # prevent it from staying Virtual jump(p2) """ - self.raises(InvalidLoop, self.optimize_loop, - ops, "crash!") + self.raises(InvalidLoop, self.optimize_loop, ops, "crash!") def test_invalid_loop_3(self): ops = """ @@ -2824,8 +2926,7 @@ guard_value(p2, ConstPtr(myptr)) [] jump(p2) """ - exc = self.raises(InvalidLoop, self.optimize_loop, - ops, "crash!") + exc = self.raises(InvalidLoop, self.optimize_loop, ops, "crash!") if exc: assert "node" in exc.msg @@ -3151,7 +3252,6 @@ """ self.optimize_loop(ops, expected) - def test_int_and_or_with_zero(self): ops = """ [i0, i1] @@ -5107,7 +5207,6 @@ """ self.optimize_loop(ops, expected) - def test_division_nonneg(self): py.test.skip("harder") # this is how an app-level division turns into right now @@ -5444,7 +5543,6 @@ """ self.optimize_loop(ops, ops, ops) - def test_mul_ovf(self): ops = """ [i0, i1] @@ -5591,7 +5689,6 @@ def is_integer_bounded(self): return False - for n in ('inst_w_seq', 'inst_index', 'inst_w_list', 'inst_length', 'inst_start', 'inst_step'): self.namespace[n] = FakeDescr(n) @@ -5847,7 +5944,7 @@ self.optimize_loop(ops, optops, preamble) # check with replacing 'str' with 'unicode' everywhere def r(s): - return s.replace('str','unicode').replace('s"', 'u"') + return s.replace('str', 'unicode').replace('s"', 'u"') self.optimize_loop(r(ops), r(optops), r(preamble)) def test_newstr_1(self): @@ -6277,7 +6374,7 @@ if isinstance(value, calldescrtype): extra = value.get_extra_info() if (extra and isinstance(extra, effectinfotype) and - extra.oopspecindex == oopspecindex): + extra.oopspecindex == oopspecindex): # returns 0 for 'func' in this test return value, 0 raise AssertionError("not found: oopspecindex=%d" % @@ -7395,7 +7492,6 @@ """ self.optimize_loop(ops, expected, expected_short=short) - def test_loopinvariant_constant_strgetitem(self): ops = """ [p0] @@ -7454,7 +7550,7 @@ """ self.optimize_loop(ops, expected, expected_short=short) - def test_propagate_virtual_arryalen(self): + def test_propagate_virtual_arraylen(self): ops = """ [p0] p404 = new_array(2, descr=arraydescr) @@ -7831,7 +7927,6 @@ """ self.optimize_loop(ops, expected) - def test_setarrayitem_followed_by_arraycopy(self): ops = """ [p1, p2] @@ -8124,7 +8219,6 @@ """ self.optimize_loop(ops, expected) - def test_issue1080_infinitie_loop_simple(self): ops = """ [p69] @@ -8149,8 +8243,7 @@ guard_value(p1, ConstPtr(myptr)) [] jump(p1) """ - self.raises(InvalidLoop, self.optimize_loop, - ops, ops) + self.raises(InvalidLoop, self.optimize_loop, ops, ops) def test_licm_boxed_opaque_getitem(self): ops = """ @@ -8225,8 +8318,7 @@ guard_value(p1, ConstPtr(myptr)) [] jump(p1) """ - self.raises(InvalidLoop, self.optimize_loop, - ops, ops) + self.raises(InvalidLoop, self.optimize_loop, ops, ops) def test_cond_call_with_a_constant(self): ops = """ @@ -8253,6 +8345,16 @@ """ self.optimize_loop(ops, expected) + def test_hippyvm_unroll_bug(self): + ops = """ + [p0, i1, i2] + i3 = int_add(i1, 1) + i4 = int_eq(i3, i2) + setfield_gc(p0, i4, descr=valuedescr) + jump(p0, i3, i2) + """ + self.optimize_loop(ops, ops) + + class TestLLtype(OptimizeOptTest, LLtypeMixin): pass - diff --git a/rpython/jit/metainterp/optimizeopt/test/test_rawbuffer.py b/rpython/jit/metainterp/optimizeopt/test/test_rawbuffer.py --- a/rpython/jit/metainterp/optimizeopt/test/test_rawbuffer.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_rawbuffer.py @@ -17,8 +17,7 @@ ( 4, 2, 'descr2', 'two'), ( 8, 4, 'descr3', 'three'), (12, 2, 'descr4', 'four'), - ] - # + ] def test_write_value_update(): buf = RawBuffer(FakeCPU()) @@ -28,7 +27,7 @@ assert buf._get_memory() == [ ( 0, 4, 'descr', 'ONE'), ( 4, 2, 'descr', 'two'), - ] + ] def test_write_value_invalid_length(): buf = RawBuffer(FakeCPU()) @@ -38,7 +37,6 @@ with py.test.raises(InvalidRawWrite): buf.write_value(0, 4, 'descr2', 'two') - def test_write_value_overlapping_next(): buf = RawBuffer(FakeCPU()) buf.write_value(0, 4, 'descr', 'one') diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py --- a/rpython/jit/metainterp/optimizeopt/test/test_util.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py @@ -226,6 +226,8 @@ hints={'nolength': True})) rawarraydescr_char = cpu.arraydescrof(lltype.Array(lltype.Char, hints={'nolength': True})) + rawarraydescr_float = cpu.arraydescrof(lltype.Array(lltype.Float, + hints={'nolength': True})) fc_array = lltype.GcArray( lltype.Struct( diff --git a/rpython/jit/metainterp/optimizeopt/util.py b/rpython/jit/metainterp/optimizeopt/util.py --- a/rpython/jit/metainterp/optimizeopt/util.py +++ b/rpython/jit/metainterp/optimizeopt/util.py @@ -1,7 +1,7 @@ import itertools import py -from rpython.rlib.objectmodel import r_dict, compute_identity_hash +from rpython.rlib.objectmodel import r_dict, compute_identity_hash, specialize from rpython.rlib.rarithmetic import intmask from rpython.rlib.unroll import unrolling_iterable from rpython.jit.metainterp import resoperation @@ -118,16 +118,10 @@ res = intmask((1000003 * res) ^ y) return res + at specialize.call_location() def args_dict(): return r_dict(args_eq, args_hash) -def args_dict_box(): - return r_dict(args_eq, args_hash) - -def args_dict_value(): - return r_dict(args_eq, args_hash) - - # ____________________________________________________________ def equaloplists(oplist1, oplist2, strict_fail_args=True, remap={}, @@ -186,4 +180,3 @@ assert False assert len(oplist1) == len(oplist2) return True - diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py --- a/rpython/jit/metainterp/optimizeopt/virtualize.py +++ b/rpython/jit/metainterp/optimizeopt/virtualize.py @@ -432,22 +432,13 @@ optforce.emit_operation(self.source_op) self.box = self.source_op.result for i in range(len(self.buffer.offsets)): - # get a pointer to self.box+offset + # write the value offset = self.buffer.offsets[i] - if offset == 0: - arraybox = self.box - else: - arraybox = BoxInt() - op = ResOperation(rop.INT_ADD, - [self.box, ConstInt(offset)], arraybox) - optforce.emit_operation(op) - # - # write the value descr = self.buffer.descrs[i] itemvalue = self.buffer.values[i] itembox = itemvalue.force_box(optforce) - op = ResOperation(rop.SETARRAYITEM_RAW, - [arraybox, ConstInt(0), itembox], None, + op = ResOperation(rop.RAW_STORE, + [self.box, ConstInt(offset), itembox], None, descr=descr) optforce.emit_operation(op) @@ -657,7 +648,6 @@ def optimize_SETFIELD_GC(self, op): value = self.getvalue(op.getarg(0)) - if value.is_virtual(): fieldvalue = self.getvalue(op.getarg(1)) value.setfield(op.getdescr(), fieldvalue) @@ -674,14 +664,8 @@ def optimize_NEW_ARRAY(self, op): sizebox = self.get_constant_box(op.getarg(0)) if sizebox is not None: - # if the original 'op' did not have a ConstInt as argument, - # build a new one with the ConstInt argument - if not isinstance(op.getarg(0), ConstInt): - op = ResOperation(rop.NEW_ARRAY, [sizebox], op.result, - descr=op.getdescr()) self.make_varray(op.getdescr(), sizebox.getint(), op.result, op) else: - self.getvalue(op.result).ensure_nonnull() self.emit_operation(op) def optimize_CALL(self, op): @@ -699,12 +683,12 @@ self.emit_operation(op) def do_RAW_MALLOC_VARSIZE_CHAR(self, op): - sizebox = op.getarg(1) - if not isinstance(sizebox, ConstInt): + sizebox = self.get_constant_box(op.getarg(1)) + if sizebox is None: self.emit_operation(op) return - size = sizebox.value - self.make_virtual_raw_memory(size, op.result, op) + self.make_virtual_raw_memory(sizebox.getint(), op.result, op) + self.last_emitted_operation = REMOVED def do_RAW_FREE(self, op): value = self.getvalue(op.getarg(1)) @@ -779,11 +763,12 @@ offset, itemsize, descr = self._unpack_arrayitem_raw_op(op, indexbox) try: itemvalue = value.getitem_raw(offset, itemsize, descr) - self.make_equal_to(op.result, itemvalue) except InvalidRawOperation: box = value.force_box(self) op.setarg(0, box) self.emit_operation(op) + else: + self.make_equal_to(op.result, itemvalue) return value.ensure_nonnull() self.emit_operation(op) @@ -805,6 +790,48 @@ value.ensure_nonnull() self.emit_operation(op) + def _unpack_raw_load_store_op(self, op, offsetbox): + offset = offsetbox.getint() + cpu = self.optimizer.cpu + descr = op.getdescr() + itemsize = cpu.unpack_arraydescr_size(descr)[1] + return offset, itemsize, descr + + def optimize_RAW_LOAD(self, op): + value = self.getvalue(op.getarg(0)) + if value.is_virtual(): + offsetbox = self.get_constant_box(op.getarg(1)) + if offsetbox is not None: + offset, itemsize, descr = self._unpack_raw_load_store_op(op, offsetbox) + try: + itemvalue = value.getitem_raw(offset, itemsize, descr) + except InvalidRawOperation: + box = value.force_box(self) + op.setarg(0, box) + self.emit_operation(op) + else: + self.make_equal_to(op.result, itemvalue) + return + value.ensure_nonnull() + self.emit_operation(op) + + def optimize_RAW_STORE(self, op): + value = self.getvalue(op.getarg(0)) + if value.is_virtual(): + offsetbox = self.get_constant_box(op.getarg(1)) + if offsetbox is not None: + offset, itemsize, descr = self._unpack_raw_load_store_op(op, offsetbox) + itemvalue = self.getvalue(op.getarg(2)) + try: + value.setitem_raw(offset, itemsize, descr, itemvalue) + except InvalidRawOperation: + box = value.force_box(self) + op.setarg(0, box) + self.emit_operation(op) + return + value.ensure_nonnull() + self.emit_operation(op) + def optimize_GETINTERIORFIELD_GC(self, op): value = self.getvalue(op.getarg(0)) if value.is_virtual(): diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -11,7 +11,7 @@ ConstFloat, Box, TargetToken) from rpython.jit.metainterp.jitprof import EmptyProfiler from rpython.jit.metainterp.logger import Logger -from rpython.jit.metainterp.optimizeopt.util import args_dict_box +from rpython.jit.metainterp.optimizeopt.util import args_dict from rpython.jit.metainterp.resoperation import rop from rpython.rlib import nonconst, rstack from rpython.rlib.debug import debug_start, debug_stop, debug_print, make_sure_not_resized @@ -1656,7 +1656,7 @@ self.forced_virtualizable = None self.partial_trace = None self.retracing_from = -1 - self.call_pure_results = args_dict_box() + self.call_pure_results = args_dict() self.heapcache = HeapCache() self.call_ids = [] @@ -1784,16 +1784,14 @@ moreargs = [box] + extraargs else: moreargs = list(extraargs) - metainterp_sd = self.staticdata if opnum == rop.GUARD_NOT_FORCED or opnum == rop.GUARD_NOT_FORCED_2: - resumedescr = compile.ResumeGuardForcedDescr(metainterp_sd, + resumedescr = compile.ResumeGuardForcedDescr(self.staticdata, self.jitdriver_sd) elif opnum == rop.GUARD_NOT_INVALIDATED: resumedescr = compile.ResumeGuardNotInvalidated() else: resumedescr = compile.ResumeGuardDescr() - guard_op = self.history.record(opnum, moreargs, None, - descr=resumedescr) + guard_op = self.history.record(opnum, moreargs, None, descr=resumedescr) self.capture_resumedata(resumedescr, resumepc) self.staticdata.profiler.count_ops(opnum, Counters.GUARDS) # count diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py --- a/rpython/jit/metainterp/resoperation.py +++ b/rpython/jit/metainterp/resoperation.py @@ -119,10 +119,11 @@ descr = self.getdescr() if descr is None or we_are_translated(): return '%s%s%s(%s)' % (prefix, sres, self.getopname(), - ', '.join([str(a) for a in args])) + ', '.join([str(a) for a in args])) else: - return '%s%s%s(%s, descr=%r)' % (prefix, sres, self.getopname(), - ', '.join([str(a) for a in args]), descr) + return '%s%s%s(%s)' % (prefix, sres, self.getopname(), + ', '.join([str(a) for a in args] + + ['descr=%r' % descr])) def getopname(self): try: diff --git a/rpython/jit/metainterp/test/test_rawmem.py b/rpython/jit/metainterp/test/test_rawmem.py --- a/rpython/jit/metainterp/test/test_rawmem.py +++ b/rpython/jit/metainterp/test/test_rawmem.py @@ -1,7 +1,8 @@ from rpython.jit.metainterp.test.support import LLJitMixin from rpython.rtyper.lltypesystem import lltype, rffi from rpython.rlib.rawstorage import (alloc_raw_storage, raw_storage_setitem, - free_raw_storage, raw_storage_getitem) + free_raw_storage, raw_storage_getitem) + class RawMemTests(object): def test_cast_void_ptr(self): @@ -44,6 +45,7 @@ self.check_operations_history({'call': 2, 'guard_no_exception': 1, 'raw_store': 1, 'raw_load': 1, 'finish': 1}) + self.metainterp.staticdata.stats.check_resops({'finish': 1}, omit_finish=False) def test_raw_storage_float(self): def f(): @@ -57,6 +59,7 @@ self.check_operations_history({'call': 2, 'guard_no_exception': 1, 'raw_store': 1, 'raw_load': 1, 'finish': 1}) + self.metainterp.staticdata.stats.check_resops({'finish': 1}, omit_finish=False) def test_raw_storage_byte(self): def f(): @@ -70,6 +73,21 @@ self.check_operations_history({'call': 2, 'guard_no_exception': 1, 'raw_store': 1, 'raw_load': 1, 'finish': 1}) + self.metainterp.staticdata.stats.check_resops({'finish': 1}, omit_finish=False) + + def test_raw_storage_options(self): + def f(): + p = alloc_raw_storage(15, track_allocation=False, zero=True) + raw_storage_setitem(p, 3, 24) + res = raw_storage_getitem(lltype.Signed, p, 3) + free_raw_storage(p, track_allocation=False) + return res + res = self.interp_operations(f, []) + assert res == 24 + self.check_operations_history({'call': 2, 'guard_no_exception': 1, + 'raw_store': 1, 'raw_load': 1, + 'finish': 1}) + self.metainterp.staticdata.stats.check_resops({'finish': 1}, omit_finish=False) class TestRawMem(RawMemTests, LLJitMixin): diff --git a/rpython/jit/metainterp/test/test_resoperation.py b/rpython/jit/metainterp/test/test_resoperation.py --- a/rpython/jit/metainterp/test/test_resoperation.py +++ b/rpython/jit/metainterp/test/test_resoperation.py @@ -1,6 +1,7 @@ import py +import re from rpython.jit.metainterp import resoperation as rop -from rpython.jit.metainterp.history import AbstractDescr +from rpython.jit.metainterp.history import AbstractDescr, AbstractFailDescr def test_arity_mixins(): cases = [ @@ -55,12 +56,18 @@ op = rop.ResOperation(rop.rop.INT_ADD, ['a', 'b'], 'c') assert op.getarglist() == ['a', 'b'] assert op.result == 'c' + assert repr(op) == "c = int_add(a, b)" mydescr = AbstractDescr() op = rop.ResOperation(rop.rop.CALL, ['a', 'b'], 'c', descr=mydescr) assert op.getarglist() == ['a', 'b'] assert op.result == 'c' assert op.getdescr() is mydescr + assert re.match("c = call$a, b, descr=<.+>$$", repr(op)) + + mydescr = AbstractFailDescr() + op = rop.ResOperation(rop.rop.GUARD_NO_EXCEPTION, [], None, descr=mydescr) + assert re.match("guard_no_exception$descr=<.+>$$", repr(op)) def test_can_malloc(): mydescr = AbstractDescr() diff --git a/rpython/jit/metainterp/test/test_virtual.py b/rpython/jit/metainterp/test/test_virtual.py --- a/rpython/jit/metainterp/test/test_virtual.py +++ b/rpython/jit/metainterp/test/test_virtual.py @@ -1150,7 +1150,7 @@ res = self.meta_interp(f, [10]) assert res == 55 self.check_trace_count(1) - self.check_resops(setarrayitem_raw=0, getarrayitem_raw=0) + self.check_resops({'guard_true': 2, 'int_add': 4, 'int_lt': 2, 'jump': 1}) def test_raw_malloc_resume(self): mydriver = JitDriver(greens=[], reds = 'auto') @@ -1171,8 +1171,10 @@ assert f(10) == 4000+55 res = self.meta_interp(f, [10]) assert res == 4000+55 - # the getarrayitem_raw is in the bridge - self.check_resops(getarrayitem_raw=1, setarrayitem_raw=0) + self.check_trace_count(2) + self.check_resops({'guard_false': 2, 'guard_true': 5, + 'int_add': 8, 'int_gt': 3, 'int_lt': 4, 'int_mul': 2, + 'jump': 2}) def test_raw_malloc_no_virtualstate(self): mydriver = JitDriver(greens=[], reds = 'auto') @@ -1194,7 +1196,7 @@ assert res == 45 # make sure that the raw buffer is *not* virtualized because we do not # support virtualstate - self.check_resops(getarrayitem_raw=2, setarrayitem_raw=2) + self.check_resops(getarrayitem_raw=2, raw_store=2) def test_raw_malloc_only_chars(self): mydriver = JitDriver(greens=[], reds = 'auto') diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py --- a/rpython/rlib/rfile.py +++ b/rpython/rlib/rfile.py @@ -132,8 +132,8 @@ The actual return value may be determined with os.WEXITSTATUS. """ + res = 0 ll_f = self.ll_file - res = 0 if ll_f: # double close is allowed self.ll_file = lltype.nullptr(FILE) diff --git a/rpython/rlib/test/test_rfile.py b/rpython/rlib/test/test_rfile.py --- a/rpython/rlib/test/test_rfile.py +++ b/rpython/rlib/test/test_rfile.py @@ -1,9 +1,9 @@ - import os, sys, py from rpython.rtyper.test.tool import BaseRtypingTest from rpython.tool.udir import udir from rpython.rlib import rfile + class TestFile(BaseRtypingTest): def setup_class(cls): cls.tmpdir = udir.join('test_rfile') @@ -208,6 +208,7 @@ assert s == "%s\n" % printval assert os.WEXITSTATUS(r) == retval + class TestPopenR(BaseRtypingTest): def setup_class(cls): if sys.platform == 'win32': diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py --- a/rpython/rtyper/lltypesystem/rstr.py +++ b/rpython/rtyper/lltypesystem/rstr.py @@ -108,6 +108,8 @@ copy_string_to_raw = func_with_new_name(copy_string_to_raw, 'copy_%s_to_raw' % name) @jit.dont_look_inside + @signature(types.any(), types.any(), types.int(), types.int(), + returns=types.none()) def copy_raw_to_string(ptrsrc, dst, dststart, length): # xxx Warning: same note as above apply: don't do this at home assert length >= 0 diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -369,32 +369,21 @@ for rule in rules: m.rule(*rule) - objects = ' $(OBJECTS)' From noreply at buildbot.pypy.org Mon Mar 17 21:39:52 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 17 Mar 2014 21:39:52 +0100 (CET) Subject: [pypy-commit] pypy improve-consecutive-dict-lookups: cleanup/reduce diff with default Message-ID: <20140317203952.98AC21C00B9@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: improve-consecutive-dict-lookups Changeset: r70029:0f464df7cb9f Date: 2014-03-17 15:41 -0400 http://bitbucket.org/pypy/pypy/changeset/0f464df7cb9f/ Log: cleanup/reduce diff with default diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py --- a/rpython/jit/metainterp/optimizeopt/heap.py +++ b/rpython/jit/metainterp/optimizeopt/heap.py @@ -302,13 +302,12 @@ self.emit_operation(op) def _optimize_CALL_DICT_LOOKUP(self, op): - args = self.optimizer.make_args_key(op) descr = op.getdescr().get_extra_info().extradescr if descr in self.cached_dict_reads: d = self.cached_dict_reads[descr] else: - d = args_dict() - self.cached_dict_reads[descr] = d + d = self.cached_dict_reads[descr] = args_dict() + args = self.optimizer.make_args_key(op) try: res_v = d[args] except KeyError: diff --git a/rpython/jit/metainterp/test/test_dict.py b/rpython/jit/metainterp/test/test_dict.py --- a/rpython/jit/metainterp/test/test_dict.py +++ b/rpython/jit/metainterp/test/test_dict.py @@ -212,7 +212,6 @@ self.check_simple_loop(call=1, getinteriorfield_gc=2, guard_no_exception=1) - def test_ordered_dict_two_lookups(self): driver = JitDriver(greens = [], reds = 'auto') d = OrderedDict() @@ -273,6 +272,7 @@ assert res == f(10) self.check_simple_loop(call=3) + class TestLLtype(DictTests, LLJitMixin): pass diff --git a/rpython/rtyper/lltypesystem/rbuilder.py b/rpython/rtyper/lltypesystem/rbuilder.py --- a/rpython/rtyper/lltypesystem/rbuilder.py +++ b/rpython/rtyper/lltypesystem/rbuilder.py @@ -85,6 +85,13 @@ ll_builder.used = needed @staticmethod + def ll_append_char(ll_builder, char): + if ll_builder.used == ll_builder.allocated: + ll_builder.grow(ll_builder, 1) + ll_builder.buf.chars[ll_builder.used] = char + ll_builder.used += 1 + + @staticmethod def ll_append_slice(ll_builder, ll_str, start, end): needed = end - start used = ll_builder.used @@ -106,7 +113,6 @@ ll_builder.used = used @staticmethod - @enforceargs(None, None, int) def ll_append_charpsize(ll_builder, charp, size): used = ll_builder.used if used + size > ll_builder.allocated: @@ -132,14 +138,6 @@ return ll_builder != nullptr(cls.lowleveltype.TO) class StringBuilderRepr(BaseStringBuilderRepr): - @staticmethod - @enforceargs(None, lltype.Char) - def ll_append_char(ll_builder, char): - if ll_builder.used == ll_builder.allocated: - ll_builder.grow(ll_builder, 1) - ll_builder.buf.chars[ll_builder.used] = char - ll_builder.used += 1 - lowleveltype = lltype.Ptr(STRINGBUILDER) basetp = STR mallocfn = staticmethod(rstr.mallocstr) @@ -150,15 +148,6 @@ ) class UnicodeBuilderRepr(BaseStringBuilderRepr): - - @staticmethod - @enforceargs(None, lltype.UniChar) - def ll_append_char(ll_builder, char): - if ll_builder.used == ll_builder.allocated: - ll_builder.grow(ll_builder, 1) - ll_builder.buf.chars[ll_builder.used] = char - ll_builder.used += 1 - lowleveltype = lltype.Ptr(UNICODEBUILDER) basetp = UNICODE mallocfn = staticmethod(rstr.mallocunicode) diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py --- a/rpython/rtyper/lltypesystem/rstr.py +++ b/rpython/rtyper/lltypesystem/rstr.py @@ -4,7 +4,7 @@ from rpython.rlib import jit, types from rpython.rlib.debug import ll_assert from rpython.rlib.objectmodel import (malloc_zero_filled, we_are_translated, - _hash_string, keepalive_until_here, specialize, enforceargs) + _hash_string, keepalive_until_here, specialize) from rpython.rlib.signature import signature from rpython.rlib.rarithmetic import ovfcheck from rpython.rtyper.error import TyperError @@ -121,10 +121,9 @@ llmemory.raw_memcopy(srcbuf, dst, llmemory.sizeof(CHAR_TP) * length) # end of "no GC" section keepalive_until_here(dst) + copy_raw_to_string._always_inline_ = True copy_raw_to_string = func_with_new_name(copy_raw_to_string, 'copy_raw_to_%s' % name) - copy_raw_to_string._always_inline_ = True - copy_raw_to_string._annenforceargs_ = (None, None, int, int) return copy_string_to_raw, copy_raw_to_string, copy_string_contents From noreply at buildbot.pypy.org Mon Mar 17 21:39:53 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 17 Mar 2014 21:39:53 +0100 (CET) Subject: [pypy-commit] pypy improve-consecutive-dict-lookups: filter guards like the other optimizers Message-ID: <20140317203953.DB5301C00B9@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: improve-consecutive-dict-lookups Changeset: r70030:ecaad25d95ac Date: 2014-03-17 15:19 -0400 http://bitbucket.org/pypy/pypy/changeset/ecaad25d95ac/ Log: filter guards like the other optimizers diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py --- a/rpython/jit/metainterp/optimizeopt/heap.py +++ b/rpython/jit/metainterp/optimizeopt/heap.py @@ -4,7 +4,7 @@ from rpython.jit.metainterp.optimizeopt.util import args_dict from rpython.jit.metainterp.history import Const from rpython.jit.metainterp.jitexc import JitException -from rpython.jit.metainterp.optimizeopt.optimizer import Optimization, MODE_ARRAY, LEVEL_KNOWNCLASS +from rpython.jit.metainterp.optimizeopt.optimizer import Optimization, MODE_ARRAY, LEVEL_KNOWNCLASS, REMOVED from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method from rpython.jit.metainterp.resoperation import rop, ResOperation from rpython.rlib.objectmodel import we_are_translated @@ -182,7 +182,6 @@ self._remove_guard_not_invalidated = False self._seen_guard_not_invalidated = False self.postponed_op = None - self.remove_next_guard = False def force_at_end_of_preamble(self): self.cached_dict_reads.clear() @@ -312,16 +311,14 @@ res_v = d[args] except KeyError: d[args] = self.getvalue(op.result) - res = False + return False else: self.make_equal_to(op.result, res_v) - self.remove_next_guard = True - res = True - return res + self.last_emitted_operation = REMOVED + return True def optimize_GUARD_NO_EXCEPTION(self, op): - if self.remove_next_guard: - self.remove_next_guard = False + if self.last_emitted_operation is REMOVED: return self.emit_operation(op) From noreply at buildbot.pypy.org Mon Mar 17 22:23:26 2014 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 17 Mar 2014 22:23:26 +0100 (CET) Subject: [pypy-commit] pypy default: allow pytineractive to use ssl and thread modules on win32 Message-ID: <20140317212326.E99451C11A4@cobra.cs.uni-duesseldorf.de> Author: Matti Picus Branch: Changeset: r70031:9a94ce726040 Date: 2014-03-17 23:22 +0200 http://bitbucket.org/pypy/pypy/changeset/9a94ce726040/ Log: allow pytineractive to use ssl and thread modules on win32 diff --git a/pypy/module/_ssl/thread_lock.py b/pypy/module/_ssl/thread_lock.py --- a/pypy/module/_ssl/thread_lock.py +++ b/pypy/module/_ssl/thread_lock.py @@ -1,6 +1,7 @@ from rpython.rlib.ropenssl import * from rpython.rtyper.lltypesystem import lltype, rffi from rpython.translator.tool.cbuild import ExternalCompilationInfo +import sys # CRYPTO_set_locking_callback: # @@ -64,11 +65,17 @@ from rpython.rlib import rthread +libraries = [] +if sys.platform == 'win32': + # XXX Not needed for mingw32... + libraries = ['libeay32', 'user32', 'advapi32', 'gdi32'] + eci = rthread.eci.merge(ExternalCompilationInfo( separate_module_sources=[separate_module_source], post_include_bits=[ "int _PyPy_SSL_SetupThreads(void);"], export_symbols=['_PyPy_SSL_SetupThreads'], + libraries = libraries, )) _PyPy_SSL_SetupThreads = rffi.llexternal('_PyPy_SSL_SetupThreads', From noreply at buildbot.pypy.org Mon Mar 17 22:45:21 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 17 Mar 2014 22:45:21 +0100 (CET) Subject: [pypy-commit] pypy default: cleanup Message-ID: <20140317214521.D53B11C11A4@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r70032:8984f0d53705 Date: 2014-03-17 17:35 -0400 http://bitbucket.org/pypy/pypy/changeset/8984f0d53705/ Log: cleanup diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py --- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py +++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py @@ -207,14 +207,13 @@ ... """) ops = loop.ops_by_id('cfficall') - assert 'raw_malloc' not in str(ops) - assert 'raw_free' not in str(ops) - assert 'getarrayitem_raw' not in log.opnames(ops) - assert 'setarrayitem_raw' not in log.opnames(ops) + for name in ['raw_malloc', 'raw_free']: + assert name not in str(ops) + for name in ['raw_load', 'raw_store', 'getarrayitem_raw', 'setarrayitem_raw']: + assert name not in log.opnames(ops) # so far just check that call_release_gil() is produced. # later, also check that the arguments to call_release_gil() # are constants - # are constants, and that the numerous raw_mallocs are removed def test_cffi_call_guard_not_forced_fails(self): # this is the test_pypy_c equivalent of From noreply at buildbot.pypy.org Mon Mar 17 22:45:23 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 17 Mar 2014 22:45:23 +0100 (CET) Subject: [pypy-commit] pypy default: get libraries from ropenssl Message-ID: <20140317214523.2A4801C11A4@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r70033:56b6b496d480 Date: 2014-03-17 17:44 -0400 http://bitbucket.org/pypy/pypy/changeset/56b6b496d480/ Log: get libraries from ropenssl diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py --- a/pypy/module/_ssl/test/test_ssl.py +++ b/pypy/module/_ssl/test/test_ssl.py @@ -1,5 +1,5 @@ class AppTestSSL: - spaceconfig = dict(usemodules=('_ssl', '_socket')) + spaceconfig = dict(usemodules=('_ssl', '_socket', 'thread')) def setup_class(cls): import os diff --git a/pypy/module/_ssl/thread_lock.py b/pypy/module/_ssl/thread_lock.py --- a/pypy/module/_ssl/thread_lock.py +++ b/pypy/module/_ssl/thread_lock.py @@ -1,7 +1,7 @@ -from rpython.rlib.ropenssl import * +from rpython.rlib import rthread +from rpython.rlib.ropenssl import libraries from rpython.rtyper.lltypesystem import lltype, rffi from rpython.translator.tool.cbuild import ExternalCompilationInfo -import sys # CRYPTO_set_locking_callback: # @@ -23,7 +23,6 @@ # without caring about the GIL. separate_module_source = """ - #include static unsigned int _ssl_locks_count = 0; @@ -63,13 +62,6 @@ } """ -from rpython.rlib import rthread - -libraries = [] -if sys.platform == 'win32': - # XXX Not needed for mingw32... - libraries = ['libeay32', 'user32', 'advapi32', 'gdi32'] - eci = rthread.eci.merge(ExternalCompilationInfo( separate_module_sources=[separate_module_source], post_include_bits=[ From noreply at buildbot.pypy.org Mon Mar 17 23:19:03 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 17 Mar 2014 23:19:03 +0100 (CET) Subject: [pypy-commit] pypy default: cleanup Message-ID: <20140317221903.5796C1C12F3@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r70034:8475b0021a83 Date: 2014-03-17 18:07 -0400 http://bitbucket.org/pypy/pypy/changeset/8475b0021a83/ Log: cleanup diff --git a/rpython/jit/metainterp/test/test_virtual.py b/rpython/jit/metainterp/test/test_virtual.py --- a/rpython/jit/metainterp/test/test_virtual.py +++ b/rpython/jit/metainterp/test/test_virtual.py @@ -7,12 +7,13 @@ from rpython.rtyper.lltypesystem.lloperation import llop from rpython.jit.codewriter import heaptracker + class VirtualTests: def _freeze_(self): return True def test_virtualized1(self): - myjitdriver = JitDriver(greens = [], reds = ['n', 'node']) + myjitdriver = JitDriver(greens=[], reds=['n', 'node']) def f(n): node = self._new() node.value = 0 @@ -33,9 +34,8 @@ self.check_resops(new_with_vtable=0, setfield_gc=0, getfield_gc=2, new=0) - def test_virtualized2(self): - myjitdriver = JitDriver(greens = [], reds = ['n', 'node1', 'node2']) + myjitdriver = JitDriver(greens=[], reds=['n', 'node1', 'node2']) def f(n): node1 = self._new() node1.value = 0 @@ -59,7 +59,7 @@ def test_virtualized_circular1(self): class MyNode(): pass - myjitdriver = JitDriver(greens = [], reds = ['n', 'node']) + myjitdriver = JitDriver(greens=[], reds=['n', 'node']) def f(n): node = MyNode() node.value = 0 @@ -79,11 +79,11 @@ res = self.meta_interp(f, [10]) assert res == 55 * 10 self.check_trace_count(1) - self.check_resops(new_with_vtable=0, setfield_gc=0, - getfield_gc=3, new=0) + self.check_resops(new_with_vtable=0, setfield_gc=0, getfield_gc=3, + new=0) def test_virtualized_float(self): - myjitdriver = JitDriver(greens = [], reds = ['n', 'node']) + myjitdriver = JitDriver(greens=[], reds=['n', 'node']) def f(n): node = self._new() node.floatval = 0.0 @@ -100,7 +100,7 @@ self.check_resops(new=0, float_add=1) def test_virtualized_float2(self): - myjitdriver = JitDriver(greens = [], reds = ['n', 'node']) + myjitdriver = JitDriver(greens=[], reds=['n', 'node']) def f(n): node = self._new() node.floatval = 0.0 @@ -117,9 +117,8 @@ self.check_trace_count(1) self.check_resops(new=0, float_add=2) - def test_virtualized_2(self): - myjitdriver = JitDriver(greens = [], reds = ['n', 'node']) + myjitdriver = JitDriver(greens=[], reds=['n', 'node']) def f(n): node = self._new() node.value = 0 @@ -144,7 +143,7 @@ new=0) def test_nonvirtual_obj_delays_loop(self): - myjitdriver = JitDriver(greens = [], reds = ['n', 'node']) + myjitdriver = JitDriver(greens=[], reds=['n', 'node']) node0 = self._new() node0.value = 10 def f(n): @@ -161,11 +160,11 @@ res = self.meta_interp(f, [500]) assert res == 640 self.check_trace_count(1) - self.check_resops(new_with_vtable=0, setfield_gc=0, - getfield_gc=1, new=0) + self.check_resops(new_with_vtable=0, setfield_gc=0, getfield_gc=1, + new=0) def test_two_loops_with_virtual(self): - myjitdriver = JitDriver(greens = [], reds = ['n', 'node']) + myjitdriver = JitDriver(greens=[], reds=['n', 'node']) def f(n): node = self._new() node.value = 0 @@ -185,12 +184,11 @@ res = self.meta_interp(f, [18]) assert res == f(18) self.check_trace_count(2) - self.check_resops(new_with_vtable=0, setfield_gc=0, - getfield_gc=2, new=0) - + self.check_resops(new_with_vtable=0, setfield_gc=0, getfield_gc=2, + new=0) def test_two_loops_with_escaping_virtual(self): - myjitdriver = JitDriver(greens = [], reds = ['n', 'node']) + myjitdriver = JitDriver(greens=[], reds=['n', 'node']) def externfn(node): llop.debug_print(lltype.Void, compute_unique_id(node), node.value, node.extra) @@ -218,7 +216,7 @@ self.check_resops(int_mul=0, call=1) def test_two_virtuals(self): - myjitdriver = JitDriver(greens = [], reds = ['n', 'prev']) + myjitdriver = JitDriver(greens=[], reds=['n', 'prev']) class Foo(object): def __init__(self, x, y): self.x = x @@ -241,7 +239,7 @@ self.check_resops(new_with_vtable=0, new=0) def test_specialied_bridge(self): - myjitdriver = JitDriver(greens = [], reds = ['y', 'x', 'res']) + myjitdriver = JitDriver(greens=[], reds=['y', 'x', 'res']) class A: def __init__(self, val): self.val = val @@ -253,7 +251,7 @@ myjitdriver.can_enter_jit(y=y, x=x, res=res) myjitdriver.jit_merge_point(y=y, x=x, res=res) res = res.binop(A(y)) - if y<7: + if y < 7: res = x x = A(1) y -= 1 @@ -267,7 +265,7 @@ assert res == g(6, 14) def test_both_virtual_and_field_variable(self): - myjitdriver = JitDriver(greens = [], reds = ['n']) + myjitdriver = JitDriver(greens=[], reds=['n']) class Foo(object): pass def f(n): @@ -286,7 +284,7 @@ self.check_resops(new_with_vtable=0, new=0) def test_immutable_constant_getfield(self): - myjitdriver = JitDriver(greens = ['stufflist'], reds = ['n', 'i']) + myjitdriver = JitDriver(greens=['stufflist'], reds=['n', 'i']) class Stuff(object): _immutable_ = True @@ -312,7 +310,7 @@ self.check_resops(getfield_gc=0) def test_escapes(self): - myjitdriver = JitDriver(greens = [], reds = ['n', 'parent']) + myjitdriver = JitDriver(greens=[], reds=['n', 'parent']) class Parent(object): def __init__(self, node): @@ -341,7 +339,7 @@ self.check_resops(**{self._new_op: 1}) def test_virtual_on_virtual(self): - myjitdriver = JitDriver(greens = [], reds = ['n', 'parent']) + myjitdriver = JitDriver(greens=[], reds=['n', 'parent']) class Node(object): def __init__(self, f): @@ -371,7 +369,7 @@ self.check_resops(new=0, new_with_vtable=0) def test_bridge_from_interpreter(self): - mydriver = JitDriver(reds = ['n', 'f'], greens = []) + mydriver = JitDriver(reds=['n', 'f'], greens=[]) def f(n): f = self._new() @@ -397,7 +395,7 @@ self.check_enter_count(2) def test_new_virtual_member_in_bridge(self): - myjitdriver = JitDriver(greens = [], reds = ['n', 'sa', 'node']) + myjitdriver = JitDriver(greens=[], reds=['n', 'sa', 'node']) def f(n): node = self._new() node.value = 1 @@ -406,19 +404,19 @@ while n > 0: myjitdriver.can_enter_jit(n=n, sa=sa, node=node) myjitdriver.jit_merge_point(n=n, sa=sa, node=node) - if n&30 > 0: + if n & 30 > 0: sa += node.value next = self._new() next.value = n node = next - if n<10: + if n < 10: node.extra = sa n -= 1 return node.extra assert self.meta_interp(f, [20]) == f(20) def test_constant_virtual1(self): - myjitdriver = JitDriver(greens = [], reds = ['n', 'sa', 'node']) + myjitdriver = JitDriver(greens=[], reds=['n', 'sa', 'node']) def f(n): node = self._new() node.value = 1 @@ -426,11 +424,11 @@ while n > 0: myjitdriver.can_enter_jit(n=n, sa=sa, node=node) myjitdriver.jit_merge_point(n=n, sa=sa, node=node) - if n>20: + if n > 20: next = self._new() next.value = 2 node = next - elif n>10: + elif n > 10: next = self._new() next.value = 3 node = next @@ -440,7 +438,7 @@ assert self.meta_interp(f, [30]) == f(30) def test_constant_virtual2(self): - myjitdriver = JitDriver(greens = [], reds = ['n', 'sa', 'node']) + myjitdriver = JitDriver(greens=[], reds=['n', 'sa', 'node']) def f(n): node = self._new() node.value = 1 @@ -449,7 +447,7 @@ myjitdriver.can_enter_jit(n=n, sa=sa, node=node) myjitdriver.jit_merge_point(n=n, sa=sa, node=node) sa += node.value - if n&15 > 7: + if n & 15 > 7: next = self._new() next.value = 2 node = next @@ -465,7 +463,7 @@ class RefNode(object): def __init__(self, ref): self.ref = ref - myjitdriver = JitDriver(greens = [], reds = ['n', 'sa', 'node1', 'node2']) + myjitdriver = JitDriver(greens=[], reds=['n', 'sa', 'node1', 'node2']) def f(n): node1 = self._new() node1.value = 1 @@ -474,7 +472,7 @@ while n > 0: myjitdriver.can_enter_jit(n=n, sa=sa, node1=node1, node2=node2) myjitdriver.jit_merge_point(n=n, sa=sa, node1=node1, node2=node2) - if n>10: + if n > 10: next = self._new() next.value = 2 node1 = next @@ -496,7 +494,7 @@ class RefNode(object): def __init__(self, ref): self.ref = ref - myjitdriver = JitDriver(greens = [], reds = ['n', 'sa', 'node1', 'node2']) + myjitdriver = JitDriver(greens=[], reds=['n', 'sa', 'node1', 'node2']) def f(n): node1 = self._new() node1.value = 1 @@ -505,7 +503,7 @@ while n > 0: myjitdriver.can_enter_jit(n=n, sa=sa, node1=node1, node2=node2) myjitdriver.jit_merge_point(n=n, sa=sa, node1=node1, node2=node2) - if n>10: + if n > 10: next = self._new() next.value = node1.value + 2 node1 = next @@ -522,7 +520,7 @@ class RefNode(object): def __init__(self, ref): self.ref = ref - myjitdriver = JitDriver(greens = [], reds = ['n', 'sa', 'node1', 'node2']) + myjitdriver = JitDriver(greens=[], reds=['n', 'sa', 'node1', 'node2']) def f(n): node1 = self._new() node1.value = 1 @@ -533,7 +531,7 @@ myjitdriver.jit_merge_point(n=n, sa=sa, node1=node1, node2=node2) node2.ref.value += n sa += node1.value - if n>10: + if n > 10: next = self._new() next.value = node1.value + 1 node1 = next @@ -544,7 +542,7 @@ assert self.meta_interp(f, [20]) == f(20) def test_dual_counter(self): - myjitdriver = JitDriver(greens = [], reds = ['n', 's', 'node1', 'node2']) + myjitdriver = JitDriver(greens=[], reds=['n', 's', 'node1', 'node2']) def f(n, s): node1 = self._new() node1.value = 1 @@ -553,7 +551,7 @@ while n > 0: myjitdriver.can_enter_jit(n=n, s=s, node1=node1, node2=node2) myjitdriver.jit_merge_point(n=n, s=s, node1=node1, node2=node2) - if (n>>s) & 1: + if (n >> s) & 1: next = self._new() next.value = node1.value + 1 node1 = next @@ -570,7 +568,7 @@ self.check_resops(new=0, new_with_vtable=0) def test_single_virtual_forced_in_bridge(self): - myjitdriver = JitDriver(greens = [], reds = ['n', 's', 'node']) + myjitdriver = JitDriver(greens=[], reds=['n', 's', 'node']) def externfn(node): node.value *= 2 def f(n, s): @@ -582,7 +580,7 @@ next = self._new() next.value = node.value + 1 node = next - if (n>>s) & 1: + if (n >> s) & 1: externfn(node) n -= 1 return node.value @@ -592,7 +590,7 @@ assert res == f(40, 3) def test_forced_virtual_assigned_in_bridge(self): - myjitdriver = JitDriver(greens = [], reds = ['n', 's', 'node', 'node2']) + myjitdriver = JitDriver(greens=[], reds=['n', 's', 'node', 'node2']) def externfn(node): node.value += 1 def f(n, s): @@ -606,7 +604,7 @@ next = self._new() next.value = node.value + 1 node = next - if (n>>s) & 1: + if (n >> s) & 1: node2.value += node.value node = node2 externfn(node) @@ -620,7 +618,7 @@ self.check_trace_count(3) def test_forced_virtual_assigned_different_class_in_bridge(self): - myjitdriver = JitDriver(greens = [], reds = ['n', 's', 'node', 'node2']) + myjitdriver = JitDriver(greens=[], reds=['n', 's', 'node', 'node2']) def externfn(node): node.value += 1 class A(object): @@ -636,7 +634,7 @@ while n > 0: myjitdriver.can_enter_jit(n=n, s=s, node=node, node2=node2) myjitdriver.jit_merge_point(n=n, s=s, node=node, node2=node2) - if (n>>s) & 1: + if (n >> s) & 1: node2.value += node.value node = node2 else: @@ -659,7 +657,7 @@ assert res == g2(48, 3) def test_empty_virtual_with_bridge(self): - myjitdriver = JitDriver(greens = [], reds = ['n', 's', 'sa', 'node']) + myjitdriver = JitDriver(greens=[], reds=['n', 's', 'sa', 'node']) def f(n, s): node = self._new() sa = 0 @@ -668,7 +666,7 @@ myjitdriver.jit_merge_point(n=n, s=s, sa=sa, node=node) next = self._new() node = next - if (n>>s) & 1: + if (n >> s) & 1: sa += 1 else: sa += 2 @@ -680,13 +678,13 @@ assert res == f(40, 3) def test_virtual_array_bridge(self): - myjitdriver = JitDriver(greens = [], reds = ['n', 'node']) + myjitdriver = JitDriver(greens=[], reds=['n', 'node']) def f(n): node = [42, 42] while n > 0: myjitdriver.can_enter_jit(n=n, node=node) myjitdriver.jit_merge_point(n=n, node=node) - if (n>>3) & 1: + if (n >> 3) & 1: node = [node[0], node[1] + n] else: node = [node[0] + n, node[1]] @@ -695,13 +693,13 @@ assert self.meta_interp(f, [40]) == f(40) def test_virtual_array_different_bridge(self): - myjitdriver = JitDriver(greens = [], reds = ['n', 'node']) + myjitdriver = JitDriver(greens=[], reds=['n', 'node']) def f(n): node = [42, 42] while n > 0: myjitdriver.can_enter_jit(n=n, node=node) myjitdriver.jit_merge_point(n=n, node=node) - if (n>>3) & 1: + if (n >> 3) & 1: node = [node[0], node[1] + n] else: node = [node[0] + n, node[-1], node[0] + node[1]] @@ -710,7 +708,7 @@ assert self.meta_interp(f, [40]) == f(40) def FIXME_why_does_this_force(self): - mydriver = JitDriver(reds = ['i', 'j'], greens = []) + mydriver = JitDriver(reds=['i', 'j'], greens=[]) def f(): i = self._new() i.value = 0 @@ -726,7 +724,7 @@ assert self.meta_interp(f, []) == 20 def FIXME_why_does_this_force2(self): - mydriver = JitDriver(reds = ['i', 'j'], greens = []) + mydriver = JitDriver(reds=['i', 'j'], greens=[]) def f(): i = self._new() i.value = 0 @@ -746,7 +744,7 @@ assert self.meta_interp(f, []) == 20 def test_virtual_skipped_by_bridge(self): - myjitdriver = JitDriver(greens = [], reds = ['n', 'm', 'i', 'x']) + myjitdriver = JitDriver(greens=[], reds=['n', 'm', 'i', 'x']) def f(n, m): x = self._new() x.value = 0 @@ -754,7 +752,7 @@ while i < n: myjitdriver.can_enter_jit(n=n, m=m, i=i, x=x) myjitdriver.jit_merge_point(n=n, m=m, i=i, x=x) - if i&m != m: + if i & m != m: newx = self._new() newx.value = x.value + i x = newx @@ -764,7 +762,7 @@ assert res == f(0x1F, 0x11) def test_duplicated_virtual(self): - myjitdriver = JitDriver(greens = [], reds = ['n', 'node1', 'node2']) + myjitdriver = JitDriver(greens=[], reds=['n', 'node1', 'node2']) def f(n): node1 = self._new() node1.value = 0 @@ -782,13 +780,11 @@ assert res == f(10) self.check_resops(new_with_vtable=0, new=0) - - def test_retrace_not_matching_bridge(self): @dont_look_inside def external(node): return node.value + 1 - myjitdriver = JitDriver(greens = [], reds = ['n', 'i', 'node', 'node2']) + myjitdriver = JitDriver(greens=[], reds=['n', 'i', 'node', 'node2']) class A(): def new(self): return A() @@ -825,7 +821,7 @@ @dont_look_inside def external(node): return node.value + 1 - myjitdriver = JitDriver(greens = [], reds = ['n', 'i', 'node', 'node2', 's']) + myjitdriver = JitDriver(greens=[], reds=['n', 'i', 'node', 'node2', 's']) class A(): def new(self): return A() @@ -866,7 +862,7 @@ bytecode = "iajb+JI" def get_printable_location(i): return "%d: %s" % (i, bytecode[i]) - myjitdriver = JitDriver(greens = ['pc'], reds = ['n', 'sa', 'i', 'j'], + myjitdriver = JitDriver(greens=['pc'], reds=['n', 'sa', 'i', 'j'], get_printable_location=get_printable_location) def f(n): pc = sa = 0 @@ -907,7 +903,7 @@ class Int(object): def __init__(self, val): self.val = val - myjitdriver = JitDriver(greens = ['pc'], reds = ['n', 'sa', 'i', 'j']) + myjitdriver = JitDriver(greens=['pc'], reds=['n', 'sa', 'i', 'j']) bytecode = "iajb+JI" def f(n): pc = sa = 0 @@ -947,10 +943,10 @@ self.check_aborted_count(0) self.check_target_token_count(3) + class VirtualMiscTests: - def test_multiple_equal_virtuals(self): - mydriver = JitDriver(reds = ['i'], greens = []) + mydriver = JitDriver(reds=['i'], greens=[]) class A: pass def f(): @@ -971,7 +967,7 @@ class A(object): def __init__(self, x): self.x = x - mydriver = JitDriver(reds = ['n'], greens = []) + mydriver = JitDriver(reds=['n'], greens=[]) global_a = A(0) def g(a): @@ -996,7 +992,7 @@ class A(object): def __init__(self, x): self.x = x - mydriver = JitDriver(reds = ['n', 'tot'], greens = []) + mydriver = JitDriver(reds=['n', 'tot'], greens=[]) def f(n): tot = 0 @@ -1017,7 +1013,7 @@ assert r == expected def test_arraycopy_disappears(self): - mydriver = JitDriver(reds = ['i'], greens = []) + mydriver = JitDriver(reds=['i'], greens=[]) def f(): i = 0 while i < 10: @@ -1033,7 +1029,7 @@ self.check_resops(new_array=0) def test_virtual_streq_bug(self): - mydriver = JitDriver(reds = ['i', 's', 'a'], greens = []) + mydriver = JitDriver(reds=['i', 's', 'a'], greens=[]) class A(object): def __init__(self, state): @@ -1060,7 +1056,7 @@ assert res == f() def test_getfield_gc_pure_nobug(self): - mydriver = JitDriver(reds = ['i', 's', 'a'], greens = []) + mydriver = JitDriver(reds=['i', 's', 'a'], greens=[]) class A(object): _immutable_fields_ = ['foo'] @@ -1088,7 +1084,7 @@ assert res == f() def test_virtual_attribute_pure_function(self): - mydriver = JitDriver(reds = ['i', 'sa', 'n', 'node'], greens = []) + mydriver = JitDriver(reds=['i', 'sa', 'n', 'node'], greens=[]) class A(object): def __init__(self, v1, v2): self.v1 = v1 @@ -1110,7 +1106,7 @@ assert res == f(16) def test_virtual_loop_invariant_getitem(self): - mydriver = JitDriver(reds = ['i', 'sa', 'n', 'node1', 'node2'], greens = []) + mydriver = JitDriver(reds=['i', 'sa', 'n', 'node1', 'node2'], greens=[]) class A(object): def __init__(self, v1, v2): self.v1 = v1 @@ -1134,7 +1130,7 @@ self.check_resops(getfield_gc=7) def test_raw_malloc(self): - mydriver = JitDriver(greens=[], reds = 'auto') + mydriver = JitDriver(greens=[], reds='auto') def f(n): i = 0 res = 0 @@ -1153,7 +1149,7 @@ self.check_resops({'guard_true': 2, 'int_add': 4, 'int_lt': 2, 'jump': 1}) def test_raw_malloc_resume(self): - mydriver = JitDriver(greens=[], reds = 'auto') + mydriver = JitDriver(greens=[], reds='auto') def f(n): i = 0 res = 0 @@ -1177,7 +1173,7 @@ 'jump': 2}) def test_raw_malloc_no_virtualstate(self): - mydriver = JitDriver(greens=[], reds = 'auto') + mydriver = JitDriver(greens=[], reds='auto') def f(n): res = 0 buffer = lltype.malloc(rffi.CCHARP.TO, 1, flavor='raw') @@ -1199,7 +1195,7 @@ self.check_resops(getarrayitem_raw=2, raw_store=2) def test_raw_malloc_only_chars(self): - mydriver = JitDriver(greens=[], reds = 'auto') + mydriver = JitDriver(greens=[], reds='auto') def f(n): i = 0 res = 0 @@ -1218,7 +1214,6 @@ self.check_trace_count(1) self.check_resops(setarrayitem_raw=2, getarrayitem_raw=4) - # ____________________________________________________________ # Run 1: all the tests instantiate a real RPython class @@ -1241,12 +1236,12 @@ class Subclass(MyClass): pass - myjitdriver = JitDriver(greens = [], reds = ['n', 'res']) + myjitdriver = JitDriver(greens=[], reds=['n', 'res']) def f(n): res = 0 node = MyClass() - node.value = n # so that the annotator doesn't think that value is constant - node.value2 = n # ditto + node.value = n # so that the annotator doesn't think that value is constant + node.value2 = n # ditto while n > 0: myjitdriver.can_enter_jit(n=n, res=res) myjitdriver.jit_merge_point(n=n, res=res) @@ -1277,7 +1272,6 @@ def _new(): return lltype.malloc(NODE) - # ____________________________________________________________ # Run 3: all the tests use lltype.malloc to make a NODE2 # (same as Run 2 but it is part of the OBJECT hierarchy) From noreply at buildbot.pypy.org Mon Mar 17 23:19:04 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 17 Mar 2014 23:19:04 +0100 (CET) Subject: [pypy-commit] pypy default: enable these two tests Message-ID: <20140317221904.95E8E1C12F3@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r70035:049d8d5677f9 Date: 2014-03-17 18:18 -0400 http://bitbucket.org/pypy/pypy/changeset/049d8d5677f9/ Log: enable these two tests diff --git a/rpython/jit/metainterp/test/test_virtual.py b/rpython/jit/metainterp/test/test_virtual.py --- a/rpython/jit/metainterp/test/test_virtual.py +++ b/rpython/jit/metainterp/test/test_virtual.py @@ -707,7 +707,7 @@ return node[0] + node[1] assert self.meta_interp(f, [40]) == f(40) - def FIXME_why_does_this_force(self): + def test_this_doesnt_force1(self): mydriver = JitDriver(reds=['i', 'j'], greens=[]) def f(): i = self._new() @@ -722,8 +722,9 @@ j = nxt return i.value + j.value assert self.meta_interp(f, []) == 20 + self.check_resops(new_with_vtable=0, new=0) - def FIXME_why_does_this_force2(self): + def test_this_doesnt_force2(self): mydriver = JitDriver(reds=['i', 'j'], greens=[]) def f(): i = self._new() @@ -742,6 +743,7 @@ i = j return i.value + j.value assert self.meta_interp(f, []) == 20 + self.check_resops(new_with_vtable=0, new=0) def test_virtual_skipped_by_bridge(self): myjitdriver = JitDriver(greens=[], reds=['n', 'm', 'i', 'x']) From noreply at buildbot.pypy.org Tue Mar 18 02:35:31 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 18 Mar 2014 02:35:31 +0100 (CET) Subject: [pypy-commit] pypy default: win32 doesn't have unistd.h Message-ID: <20140318013531.538E61C3273@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r70036:3cec52588761 Date: 2014-03-17 18:17 -0700 http://bitbucket.org/pypy/pypy/changeset/3cec52588761/ Log: win32 doesn't have unistd.h diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py --- a/rpython/rlib/rfile.py +++ b/rpython/rlib/rfile.py @@ -11,7 +11,10 @@ from rpython.rlib import rposix from rpython.rlib.rstring import StringBuilder -eci = ExternalCompilationInfo(includes=['stdio.h', 'unistd.h', 'sys/types.h']) +includes = ['stdio.h', 'sys/types.h'] +if os.name == "posix": + includes += ['unistd.h'] +eci = ExternalCompilationInfo(includes=includes) def llexternal(*args, **kwargs): return rffi.llexternal(*args, compilation_info=eci, **kwargs) From noreply at buildbot.pypy.org Tue Mar 18 02:36:06 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 18 Mar 2014 02:36:06 +0100 (CET) Subject: [pypy-commit] pypy default: skip this test on older cpythons Message-ID: <20140318013606.B34DF1C3273@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r70037:67fa0a7737b8 Date: 2014-03-17 21:34 -0400 http://bitbucket.org/pypy/pypy/changeset/67fa0a7737b8/ Log: skip this test on older cpythons diff --git a/pypy/module/_file/test/test_file_extra.py b/pypy/module/_file/test/test_file_extra.py --- a/pypy/module/_file/test/test_file_extra.py +++ b/pypy/module/_file/test/test_file_extra.py @@ -200,6 +200,10 @@ assert f.closed == True def test_repr(self): + import sys + if '__pypy__' not in sys.builtin_module_names and \ + sys.version_info < (2, 7, 4): + skip("see cpython issue14161") assert repr(self.file).startswith( " Author: Brian Kearns Branch: Changeset: r70038:51832197b83d Date: 2014-03-17 18:44 -0700 http://bitbucket.org/pypy/pypy/changeset/51832197b83d/ Log: fix test_stat_exception on win32 diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -178,17 +178,9 @@ import sys import errno for fn in [self.posix.stat, self.posix.lstat]: - try: - fn("nonexistentdir/nonexistentfile") - except OSError, e: - assert e.errno == errno.ENOENT - assert e.filename == "nonexistentdir/nonexistentfile" - # On Windows, when the parent directory does not exist, - # the winerror is 3 (cannot find the path specified) - # instead of 2 (cannot find the file specified) - if sys.platform == 'win32': - assert isinstance(e, WindowsError) - assert e.winerror == 3 + exc = raises(OSError, fn, "nonexistentdir/nonexistentfile") + assert exc.value.errno == errno.ENOENT + assert exc.value.filename == "nonexistentdir/nonexistentfile" if hasattr(__import__(os.name), "statvfs"): def test_statvfs(self): From noreply at buildbot.pypy.org Tue Mar 18 02:59:04 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 18 Mar 2014 02:59:04 +0100 (CET) Subject: [pypy-commit] pypy default: cleanup Message-ID: <20140318015904.4EE4A1C357E@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r70039:05ec203821cc Date: 2014-03-17 21:57 -0400 http://bitbucket.org/pypy/pypy/changeset/05ec203821cc/ Log: cleanup diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py --- a/rpython/rtyper/module/ll_os.py +++ b/rpython/rtyper/module/ll_os.py @@ -397,10 +397,10 @@ includes = ['sys/time.h'] else: includes = ['time.h'] + eci = ExternalCompilationInfo(includes=includes) + class CConfig: - _compilation_info_ = ExternalCompilationInfo( - includes=includes - ) + _compilation_info_ = eci HAVE_UTIMES = platform.Has('utimes') config = platform.configure(CConfig) @@ -409,21 +409,14 @@ if config['HAVE_UTIMES']: class CConfig: - if not _WIN32: - _compilation_info_ = ExternalCompilationInfo( - includes = includes - ) - else: - _compilation_info_ = ExternalCompilationInfo( - includes = ['time.h'] - ) + _compilation_info_ = eci TIMEVAL = platform.Struct('struct timeval', [('tv_sec', rffi.LONG), ('tv_usec', rffi.LONG)]) config = platform.configure(CConfig) TIMEVAL = config['TIMEVAL'] TIMEVAL2P = rffi.CArrayPtr(TIMEVAL) os_utimes = self.llexternal('utimes', [rffi.CCHARP, TIMEVAL2P], - rffi.INT, compilation_info=CConfig._compilation_info_) + rffi.INT, compilation_info=eci) def os_utime_platform(path, actime, modtime): import math From noreply at buildbot.pypy.org Tue Mar 18 03:09:01 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 18 Mar 2014 03:09:01 +0100 (CET) Subject: [pypy-commit] pypy default: skip pwd test_ztranslation on non-posix Message-ID: <20140318020901.2D5A01C35F0@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r70040:34c8e179633d Date: 2014-03-17 19:08 -0700 http://bitbucket.org/pypy/pypy/changeset/34c8e179633d/ Log: skip pwd test_ztranslation on non-posix diff --git a/pypy/module/pwd/test/test_pwd.py b/pypy/module/pwd/test/test_pwd.py --- a/pypy/module/pwd/test/test_pwd.py +++ b/pypy/module/pwd/test/test_pwd.py @@ -1,8 +1,8 @@ import os -import py +import pytest if os.name != 'posix': - py.test.skip('pwd module only available on unix') + pytest.skip('pwd module only available on unix') class AppTestPwd: spaceconfig = dict(usemodules=['pwd']) diff --git a/pypy/module/pwd/test/test_ztranslation.py b/pypy/module/pwd/test/test_ztranslation.py --- a/pypy/module/pwd/test/test_ztranslation.py +++ b/pypy/module/pwd/test/test_ztranslation.py @@ -1,5 +1,9 @@ +import os +import pytest from pypy.objspace.fake.checkmodule import checkmodule +if os.name != 'posix': + pytest.skip('pwd module only available on unix') def test_checkmodule(): checkmodule('pwd') From noreply at buildbot.pypy.org Tue Mar 18 07:01:01 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 18 Mar 2014 07:01:01 +0100 (CET) Subject: [pypy-commit] pypy default: cleanup Message-ID: <20140318060101.BD0E41C0A66@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r70041:16c7d08815c0 Date: 2014-03-18 01:58 -0400 http://bitbucket.org/pypy/pypy/changeset/16c7d08815c0/ Log: cleanup diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py --- a/pypy/module/imp/importing.py +++ b/pypy/module/imp/importing.py @@ -247,12 +247,10 @@ fromlist_w = None rel_modulename = None - if (level != 0 and - w_globals is not None and - space.isinstance_w(w_globals, space.w_dict)): - - rel_modulename, rel_level = _get_relative_name(space, modulename, level, w_globals) - + if (level != 0 and w_globals is not None and + space.isinstance_w(w_globals, space.w_dict)): + rel_modulename, rel_level = _get_relative_name(space, modulename, level, + w_globals) if rel_modulename: # if no level was set, ignore import errors, and # fall back to absolute import at the end of the @@ -601,7 +599,7 @@ try: if find_info.modtype == PY_SOURCE: load_source_module( - space, w_modulename, w_mod, + space, w_modulename, w_mod, find_info.filename, find_info.stream.readall(), find_info.stream.try_to_find_file_descriptor()) return w_mod diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py --- a/pypy/module/imp/test/test_import.py +++ b/pypy/module/imp/test/test_import.py @@ -174,7 +174,6 @@ import notapackage import warnings - warnings.simplefilter('error', ImportWarning) try: raises(ImportWarning, imp) @@ -412,7 +411,7 @@ def test_future_relative_import_level_1(self): from pkg import relative_c assert relative_c.inpackage == 1 - + def test_future_relative_import_level_2(self): from pkg.pkg1 import relative_d assert relative_d.inpackage == 1 @@ -670,10 +669,7 @@ import imp import pkg import os - - info = ('.py', 'r', imp.PY_SOURCE) pathname = os.path.join(os.path.dirname(pkg.__file__), 'a.py') - module = imp.load_module('a', open(pathname), 'invalid_path_name', ('.py', 'r', imp.PY_SOURCE)) assert module.__name__ == 'a' @@ -1128,7 +1124,7 @@ def load_module(self, name): sys.modules[name] = sys return sys - + def importer_for_path(path): if path == "xxx": return Importer() diff --git a/pypy/module/marshal/__init__.py b/pypy/module/marshal/__init__.py --- a/pypy/module/marshal/__init__.py +++ b/pypy/module/marshal/__init__.py @@ -8,7 +8,7 @@ appleveldefs = { } - + interpleveldefs = { 'dump' : 'interp_marshal.dump', 'dumps' : 'interp_marshal.dumps', From noreply at buildbot.pypy.org Tue Mar 18 07:41:56 2014 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 18 Mar 2014 07:41:56 +0100 (CET) Subject: [pypy-commit] pypy improve-consecutive-dict-lookups: fix Message-ID: <20140318064156.1F4651C00B9@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: improve-consecutive-dict-lookups Changeset: r70042:06f75ff60246 Date: 2014-03-18 08:41 +0200 http://bitbucket.org/pypy/pypy/changeset/06f75ff60246/ Log: fix diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py --- a/rpython/jit/codewriter/jtransform.py +++ b/rpython/jit/codewriter/jtransform.py @@ -1855,7 +1855,6 @@ extradescr = self.cpu.fielddescrof(op.args[1].concretetype.TO, 'entries') return self._handle_oopspec_call(op, args, EffectInfo.OS_DICT_LOOKUP, - EffectInfo.EF_CAN_RAISE, extradescr=extradescr) def _handle_rgc_call(self, op, oopspec_name, args): From noreply at buildbot.pypy.org Tue Mar 18 08:07:58 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 18 Mar 2014 08:07:58 +0100 (CET) Subject: [pypy-commit] pypy default: only look in sys.modules if force_init=False Message-ID: <20140318070758.9130F1C0362@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r70043:310dcc241b1f Date: 2014-03-18 03:01 -0400 http://bitbucket.org/pypy/pypy/changeset/310dcc241b1f/ Log: only look in sys.modules if force_init=False diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -435,14 +435,12 @@ def getbuiltinmodule(self, name, force_init=False): w_name = self.wrap(name) w_modules = self.sys.get('modules') - try: - w_mod = self.getitem(w_modules, w_name) - except OperationError, e: - if not e.match(self, self.w_KeyError): - raise - else: - if not force_init: - return w_mod + if not force_init: + try: + return self.getitem(w_modules, w_name) + except OperationError, e: + if not e.match(self, self.w_KeyError): + raise # If the module is a builtin but not yet imported, # retrieve it and initialize it @@ -453,13 +451,13 @@ "getbuiltinmodule() called with non-builtin module %s", name) else: - # Add the module to sys.modules - self.setitem(w_modules, w_name, w_mod) - # And initialize it from pypy.interpreter.module import Module if isinstance(w_mod, Module): w_mod.init(self) + + # Add the module to sys.modules + self.setitem(w_modules, w_name, w_mod) return w_mod def get_builtinmodule_to_install(self): diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py --- a/pypy/module/imp/importing.py +++ b/pypy/module/imp/importing.py @@ -574,6 +574,7 @@ def load_module(space, w_modulename, find_info, reuse=False): if find_info is None: return + if find_info.w_loader: return space.call_method(find_info.w_loader, "load_module", w_modulename) From noreply at buildbot.pypy.org Tue Mar 18 08:35:08 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 18 Mar 2014 08:35:08 +0100 (CET) Subject: [pypy-commit] pypy default: fix reimport/reload of builtin modules (issue1514) Message-ID: <20140318073508.F1CCB1D2540@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r70044:0ecfb7242213 Date: 2014-03-18 03:33 -0400 http://bitbucket.org/pypy/pypy/changeset/0ecfb7242213/ Log: fix reimport/reload of builtin modules (issue1514) diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -432,10 +432,11 @@ return name - def getbuiltinmodule(self, name, force_init=False): + def getbuiltinmodule(self, name, force_init=False, reuse=True): w_name = self.wrap(name) w_modules = self.sys.get('modules') if not force_init: + assert reuse is True try: return self.getitem(w_modules, w_name) except OperationError, e: @@ -454,6 +455,8 @@ # And initialize it from pypy.interpreter.module import Module if isinstance(w_mod, Module): + if not reuse: + w_mod = type(w_mod)(self, w_name) w_mod.init(self) # Add the module to sys.modules diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py --- a/pypy/module/imp/importing.py +++ b/pypy/module/imp/importing.py @@ -579,7 +579,7 @@ return space.call_method(find_info.w_loader, "load_module", w_modulename) if find_info.modtype == C_BUILTIN: - return space.getbuiltinmodule(find_info.filename, force_init=True) + return space.getbuiltinmodule(find_info.filename, force_init=True, reuse=reuse) if find_info.modtype in (PY_SOURCE, PY_COMPILED, C_EXTENSION, PKG_DIRECTORY): w_mod = None diff --git a/pypy/module/imp/test/test_app.py b/pypy/module/imp/test/test_app.py --- a/pypy/module/imp/test/test_app.py +++ b/pypy/module/imp/test/test_app.py @@ -203,7 +203,6 @@ def test_builtin_reimport(self): # from https://bugs.pypy.org/issue1514 - skip("fix me") import sys, marshal old = marshal.loads @@ -223,7 +222,6 @@ # taken from https://bugs.pypy.org/issue1514, with extra cases # that show a difference with CPython: we can get on CPython # several module objects for the same built-in module :-( - skip("several built-in module objects: not supported by pypy") import sys, marshal old = marshal.loads diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py --- a/pypy/module/imp/test/test_import.py +++ b/pypy/module/imp/test/test_import.py @@ -578,7 +578,6 @@ assert hasattr(time, 'clock') def test_reimport_builtin_simple_case_2(self): - skip("fix me") import sys, time time.foo = "bar" del sys.modules['time'] @@ -586,7 +585,6 @@ assert not hasattr(time, 'foo') def test_reimport_builtin(self): - skip("fix me") import sys, time oldpath = sys.path time.tzset = "" From noreply at buildbot.pypy.org Tue Mar 18 08:40:21 2014 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 18 Mar 2014 08:40:21 +0100 (CET) Subject: [pypy-commit] stmgc gc-small-uniform: A branch to redo small uniform allocations Message-ID: <20140318074021.16A321C00B9@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: gc-small-uniform Changeset: r1059:f4dcc59b09e6 Date: 2014-03-17 11:18 +0100 http://bitbucket.org/pypy/stmgc/changeset/f4dcc59b09e6/ Log: A branch to redo small uniform allocations From noreply at buildbot.pypy.org Tue Mar 18 08:40:22 2014 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 18 Mar 2014 08:40:22 +0100 (CET) Subject: [pypy-commit] stmgc gc-small-uniform: Get started Message-ID: <20140318074022.422A01C00B9@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: gc-small-uniform Changeset: r1060:f1e60e1cb9cf Date: 2014-03-17 11:19 +0100 http://bitbucket.org/pypy/stmgc/changeset/f1e60e1cb9cf/ Log: Get started diff --git a/c7/stm/core.c b/c7/stm/core.c --- a/c7/stm/core.c +++ b/c7/stm/core.c @@ -83,7 +83,7 @@ (outside the nursery), then it fits into one page. This is the common case. Otherwise, we need to compute it based on its location and size. */ - if ((obj->stm_flags & GCFLAG_SMALL_UNIFORM) != 0) { + if (is_small_uniform(obj)) { page_privatize(first_page); } else { @@ -272,16 +272,16 @@ assert(_has_mutex_pages()); assert(!_is_young(obj)); - char *segment_base = get_segment_base(source_segment_num); uintptr_t start = (uintptr_t)obj; uintptr_t first_page = start / 4096UL; - struct object_s *realobj = (struct object_s *) - REAL_ADDRESS(segment_base, obj); - if (realobj->stm_flags & GCFLAG_SMALL_UNIFORM) { + if (is_small_uniform(obj)) { abort();//XXX WRITE THE FAST CASE } else { + char *segment_base = get_segment_base(source_segment_num); + struct object_s *realobj = (struct object_s *) + REAL_ADDRESS(segment_base, obj); ssize_t obj_size = stmcb_size_rounded_up(realobj); assert(obj_size >= 16); uintptr_t end = start + obj_size; @@ -334,7 +334,7 @@ uintptr_t start = (uintptr_t)obj; uintptr_t first_page = start / 4096UL; - if (obj->stm_flags & GCFLAG_SMALL_UNIFORM) { + if (is_small_uniform(obj)) { abort();//XXX WRITE THE FAST CASE } else { @@ -488,6 +488,9 @@ /* update 'overflow_number' if needed */ if (STM_PSEGMENT->overflow_number_has_been_used) { highest_overflow_number += GCFLAG_OVERFLOW_NUMBER_bit0; + /* Note that the overflow number cannot be entirely 1 bits; + this prevents stm_flags from ever containing the value -1, + which might be confused with GCWORD_MOVED. */ assert(highest_overflow_number != /* XXX else, overflow! */ (uint32_t)-GCFLAG_OVERFLOW_NUMBER_bit0); STM_PSEGMENT->overflow_number = highest_overflow_number; diff --git a/c7/stm/core.h b/c7/stm/core.h --- a/c7/stm/core.h +++ b/c7/stm/core.h @@ -41,18 +41,13 @@ */ GCFLAG_WRITE_BARRIER = _STM_GCFLAG_WRITE_BARRIER, - /* This flag is set by gcpage.c for all objects living in - uniformly-sized pages of small objects. - */ - GCFLAG_SMALL_UNIFORM = 0x02, - /* The following flag is set on nursery objects of which we asked the id or the identityhash. It means that a space of the size of the object has already been allocated in the nonmovable part. The same flag is abused to mark prebuilt objects whose hash has been taken during translation and is statically recorded just after the object. */ - GCFLAG_HAS_SHADOW = 0x04, + GCFLAG_HAS_SHADOW = 0x2, /* All remaining bits of the 32-bit 'stm_flags' field are taken by the "overflow number". This is a number that identifies the @@ -61,7 +56,7 @@ current transaction that have been flushed out of the nursery, which occurs if the same transaction allocates too many objects. */ - GCFLAG_OVERFLOW_NUMBER_bit0 = 0x8 /* must be last */ + GCFLAG_OVERFLOW_NUMBER_bit0 = 0x4 /* must be last */ }; @@ -157,6 +152,9 @@ #ifndef NDEBUG pthread_t running_pthread; #endif + + /* This is for smallmalloc.c */ + struct small_malloc_data_s small_malloc_data; }; enum /* safe_point */ { diff --git a/c7/stm/gcpage.c b/c7/stm/gcpage.c --- a/c7/stm/gcpage.c +++ b/c7/stm/gcpage.c @@ -19,8 +19,6 @@ static void teardown_gcpage(void) { - memset(small_alloc, 0, sizeof(small_alloc)); - free_uniform_pages = NULL; LIST_FREE(testing_prebuilt_objs); if (tree_prebuilt_objs != NULL) { tree_free(tree_prebuilt_objs); @@ -29,52 +27,11 @@ } -#define GCPAGE_NUM_PAGES 20 - static void setup_N_pages(char *pages_addr, uint64_t num) { pages_initialize_shared((pages_addr - stm_object_pages) / 4096UL, num); } -static void grab_more_free_pages_for_small_allocations(void) -{ - /* grab N (= GCPAGE_NUM_PAGES) pages out of the top addresses */ - uintptr_t decrease_by = GCPAGE_NUM_PAGES * 4096; - if (uninitialized_page_stop - uninitialized_page_start <= decrease_by) - goto out_of_memory; - - uninitialized_page_stop -= decrease_by; - - if (!_stm_largemalloc_resize_arena(uninitialized_page_stop - - uninitialized_page_start)) - goto out_of_memory; - - setup_N_pages(uninitialized_page_start, GCPAGE_NUM_PAGES); - - char *p = uninitialized_page_start; - long i; - for (i = 0; i < 16; i++) { - *(char **)p = free_uniform_pages; - free_uniform_pages = p; - } - return; - - out_of_memory: - stm_fatalerror("out of memory!\n"); /* XXX */ -} - -static char *_allocate_small_slowpath(uint64_t size) -{ - /* not thread-safe! Use only when holding the mutex */ - assert(_has_mutex()); - - if (free_uniform_pages == NULL) - grab_more_free_pages_for_small_allocations(); - - abort();//... -} - - static char *allocate_outside_nursery_large(uint64_t size) { /* thread-safe: use the lock of pages.c to prevent any remapping @@ -195,14 +152,14 @@ static uintptr_t object_last_page(object_t *obj) { uintptr_t lastbyte; - struct object_s *realobj = - (struct object_s *)REAL_ADDRESS(stm_object_pages, obj); - if (realobj->stm_flags & GCFLAG_SMALL_UNIFORM) { + if (is_small_uniform(obj)) { lastbyte = (uintptr_t)obj; } else { /* get the size of the object */ + struct object_s *realobj = + (struct object_s *)REAL_ADDRESS(stm_object_pages, obj); size_t obj_size = stmcb_size_rounded_up(realobj); /* that's the last byte within the object */ diff --git a/c7/stm/gcpage.h b/c7/stm/gcpage.h --- a/c7/stm/gcpage.h +++ b/c7/stm/gcpage.h @@ -1,19 +1,12 @@ -/* Outside the nursery, we are taking from the highest addresses - complete pages, one at a time, which uniformly contain objects of - size "8 * N" for some N in range(2, GC_N_SMALL_REQUESTS). We are - taking from the lowest addresses "large" objects, which are at least - 288 bytes long, allocated by largemalloc.c. The limit is the same - as used in PyPy's default GC. -*/ - -#define GC_N_SMALL_REQUESTS 36 - -/* More parameters fished directly from PyPy's default GC +/* Some parameters fished directly from PyPy's default GC XXX document me */ #define GC_MIN (NB_NURSERY_PAGES * 4096 * 8) #define GC_MAJOR_COLLECT 1.82 +/* Granularity when grabbing more unused pages: take 50 at a time */ +#define GCPAGE_NUM_PAGES 50 + /* re-share pages after major collections (1 or 0) */ #define RESHARE_PAGES 1 @@ -23,16 +16,6 @@ static char *uninitialized_page_stop; -struct small_alloc_s { - char *next_object; /* the next address we will return, or NULL */ - char *range_last; /* if equal to next_object: next_object starts with - a next pointer; if greater: last item of a - contiguous range of unallocated objs */ -}; - -static struct small_alloc_s small_alloc[GC_N_SMALL_REQUESTS]; -static char *free_uniform_pages; - static void setup_gcpage(void); static void teardown_gcpage(void); static char *allocate_outside_nursery_large(uint64_t size); @@ -40,29 +23,4 @@ static void major_collection_if_requested(void); static void major_collection_now_at_safe_point(void); static bool largemalloc_keep_object_at(char *data); /* for largemalloc.c */ - - -static char *_allocate_small_slowpath(uint64_t size); - -static inline char *allocate_outside_nursery_small(uint64_t size) -{ - uint64_t index = size / 8; - OPT_ASSERT(2 <= index); - OPT_ASSERT(index < GC_N_SMALL_REQUESTS); - - char *result = small_alloc[index].next_object; - if (result == NULL) - return _allocate_small_slowpath(size); - - char *following; - if (small_alloc[index].range_last == result) { - following = ((char **)result)[0]; - small_alloc[index].range_last = ((char **)result)[1]; - } - else { - following = result + size; - } - small_alloc[index].next_object = following; - - return result; -} +static void setup_N_pages(char *pages_addr, uint64_t num); diff --git a/c7/stm/setup.c b/c7/stm/setup.c --- a/c7/stm/setup.c +++ b/c7/stm/setup.c @@ -111,6 +111,7 @@ teardown_sync(); teardown_gcpage(); teardown_nursery(); + teardown_smallmalloc(); teardown_pages(); } diff --git a/c7/stm/smallmalloc.c b/c7/stm/smallmalloc.c new file mode 100644 --- /dev/null +++ b/c7/stm/smallmalloc.c @@ -0,0 +1,128 @@ +#ifndef _STM_CORE_H_ +# error "must be compiled via stmgc.c" +#endif + + +static void teardown_smallmalloc(void) +{ + memset(small_page_lists, 0, sizeof(small_page_lists)); + assert(free_uniform_pages == NULL); + first_small_uniform_loc = (uintptr_t) -1; +} + +static void grab_more_free_pages_for_small_allocations(void) +{ + /* Grab GCPAGE_NUM_PAGES pages out of the top addresses. Use the + lock of pages.c to prevent any remapping from occurring under our + feet. + */ + mutex_pages_lock(); + + if (free_uniform_pages == NULL) { + + uintptr_t decrease_by = GCPAGE_NUM_PAGES * 4096; + if (uninitialized_page_stop - uninitialized_page_start < decrease_by) + goto out_of_memory; + + uninitialized_page_stop -= decrease_by; + first_small_uniform_loc = (uintptr_t)uninitialized_page_stop; + + char *base = stm_object_pages + END_NURSERY_PAGE * 4096UL; + if (!_stm_largemalloc_resize_arena(uninitialized_page_stop - base)) + goto out_of_memory; + + setup_N_pages(uninitialized_page_stop, GCPAGE_NUM_PAGES); + + char *p = uninitialized_page_stop; + long i; + for (i = 0; i < GCPAGE_NUM_PAGES; i++) { + ((struct small_page_list_s *)p)->nextpage = free_uniform_pages; + free_uniform_pages = (struct small_page_list_s *)p; + p += 4096; + } + } + + mutex_pages_unlock(); + return; + + out_of_memory: + stm_fatalerror("out of memory!\n"); /* XXX */ +} + +static char *_allocate_small_slowpath(uint64_t size) +{ + /* First try to grab the next page from the global 'small_page_list' + */ + long n = size / 8; + struct small_page_list_s *smallpage; + struct small_free_loc_s *TLPREFIX *fl = + &STM_PSEGMENT->small_malloc_data.loc_free[n]; + assert(*fl == NULL); + + retry: + smallpage = small_page_lists[n]; + if (smallpage != NULL) { + if (UNLIKELY(!__sync_bool_compare_and_swap(&small_page_lists[n], + smallpage, + smallpage->nextpage))) + goto retry; + + /* Succeeded: we have a page in 'smallpage' */ + *fl = smallpage->header.next; + return (char *)smallpage; + } + + /* There is no more page waiting. Maybe we can pick one from + free_uniform_pages. + */ + smallpage = free_uniform_pages; + if (smallpage != NULL) { + if (UNLIKELY(!__sync_bool_compare_and_swap(&free_uniform_pages, + smallpage, + smallpage->nextpage))) + goto retry; + + /* Succeeded: we have a page in 'smallpage', which is not + initialized so far, apart from the 'nextpage' field read + above. Initialize it. + */ + assert(!(((uintptr_t)smallpage) & 4095)); + struct small_free_loc_s *p, *following = NULL; + + /* Initialize all slots from the second one to the last one to + contain a chained list */ + uintptr_t i = size; + while (i <= 4096 - size) { + p = (struct small_free_loc_s *)(((char *)smallpage) + i); + p->next = following; + following = p; + i += size; + } + + /* The first slot is immediately returned */ + *fl = following; + return (char *)smallpage; + } + + /* Not a single free page left. Grab some more free pges and retry. */ + grab_more_free_pages_for_small_allocations(); + goto retry; +} + +__attribute__((always_inline)) +static inline char *allocate_outside_nursery_small(uint64_t size) +{ + OPT_ASSERT((size & 7) == 0); + OPT_ASSERT(16 <= size && size < 8 * GC_N_SMALL_REQUESTS); + + struct small_free_loc_s *TLPREFIX *fl = + &STM_PSEGMENT->small_malloc_data.loc_free[size / 8]; + + struct small_free_loc_s *result = *fl; + + if (UNLIKELY(result == NULL)) + return _allocate_small_slowpath(size); + + *fl = result->next; + return (char *)result; +} diff --git a/c7/stm/smallmalloc.h b/c7/stm/smallmalloc.h new file mode 100644 --- /dev/null +++ b/c7/stm/smallmalloc.h @@ -0,0 +1,65 @@ + +/* Outside the nursery, we are taking from the highest addresses + complete pages, one at a time, which uniformly contain objects of + size "8 * N" for some N in range(2, GC_N_SMALL_REQUESTS). We are + taking from the lowest addresses "large" objects, which are at least + 288 bytes long, allocated by largemalloc.c. The limit is the same + as used in PyPy's default GC. +*/ + +#define GC_N_SMALL_REQUESTS 36 + + +struct small_free_loc_s { + struct small_free_loc_s *next; +}; + +struct small_page_list_s { + /* A chained list of locations within the same page which are + free. */ + struct small_free_loc_s header; + + /* A chained list of all small pages containing objects of + a given small size, and that have at least one free object. */ + struct small_page_list_s *nextpage; + + /* This structure is only two words, so it always fits inside one + free slot inside the page. */ +}; + + +/* For every size from 16 bytes to 8*(GC_N_SMALL_REQUESTS-1), this is + a list of pages that contain objects of that size and have at least + one free location. Additionally, the item 0 in the following list + is a chained list of fully-free pages (which can be reused for a + different size than the one they originally contained). +*/ +static struct small_page_list_s *small_page_lists[GC_N_SMALL_REQUESTS]; + +#define free_uniform_pages (small_page_lists[0]) + + +/* For is_small_uniform(). */ +static uintptr_t first_small_uniform_loc = (uintptr_t) -1; + + +/* This is a definition for 'STM_PSEGMENT->small_malloc_data'. Each + segment grabs one page at a time from the global list, and then + requests for data are answered locally. +*/ +struct small_malloc_data_s { + struct small_free_loc_s *loc_free[GC_N_SMALL_REQUESTS]; +}; + + +/* Functions + */ +static inline char *allocate_outside_nursery_small(uint64_t size) + __attribute__((always_inline)); + +static char *_allocate_small_slowpath(uint64_t size); +static void teardown_smallmalloc(void); + +static inline bool is_small_uniform(object_t *obj) { + return ((uintptr_t)obj) >= first_small_uniform_loc; +} diff --git a/c7/stmgc.c b/c7/stmgc.c --- a/c7/stmgc.c +++ b/c7/stmgc.c @@ -2,6 +2,7 @@ #include "stmgc.h" #include "stm/atomic.h" #include "stm/list.h" +#include "stm/smallmalloc.h" #include "stm/core.h" #include "stm/pagecopy.h" #include "stm/pages.h" @@ -21,6 +22,7 @@ #include "stm/prebuilt.c" #include "stm/gcpage.c" #include "stm/largemalloc.c" +#include "stm/smallmalloc.c" #include "stm/nursery.c" #include "stm/sync.c" #include "stm/setup.c" From noreply at buildbot.pypy.org Tue Mar 18 08:40:23 2014 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 18 Mar 2014 08:40:23 +0100 (CET) Subject: [pypy-commit] stmgc gc-small-uniform: in-progress Message-ID: <20140318074023.6DD041C00B9@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: gc-small-uniform Changeset: r1061:34fbe565894a Date: 2014-03-18 08:38 +0100 http://bitbucket.org/pypy/stmgc/changeset/34fbe565894a/ Log: in-progress diff --git a/c7/stm/core.c b/c7/stm/core.c --- a/c7/stm/core.c +++ b/c7/stm/core.c @@ -84,6 +84,7 @@ the common case. Otherwise, we need to compute it based on its location and size. */ if (is_small_uniform(obj)) { + abort(); page_privatize(first_page); } else { diff --git a/c7/stm/nursery.c b/c7/stm/nursery.c --- a/c7/stm/nursery.c +++ b/c7/stm/nursery.c @@ -97,7 +97,7 @@ obj->stm_flags &= ~GCFLAG_HAS_SHADOW; realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj); size = stmcb_size_rounded_up((struct object_s *)realobj); - goto copy_large_object; + goto handle_large_object; } } /* We need to make a copy of this object. It goes either in @@ -107,25 +107,27 @@ realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj); size = stmcb_size_rounded_up((struct object_s *)realobj); - if (1 /*size >= GC_N_SMALL_REQUESTS*8*/) { + if (size >= GC_N_SMALL_REQUESTS) { /* case 1: object is not small enough. Ask gcpage.c for an allocation via largemalloc. */ char *allocated = allocate_outside_nursery_large(size); nobj = (object_t *)(allocated - stm_object_pages); - /* Copy the object */ - copy_large_object:; - char *realnobj = REAL_ADDRESS(STM_SEGMENT->segment_base, nobj); - memcpy(realnobj, realobj, size); - + handle_large_object: nobj_sync_now = ((uintptr_t)nobj) | FLAG_SYNC_LARGE; } else { /* case "small enough" */ - abort(); //... + char *allocated = allocate_outside_nursery_small(size); + nobj = (object_t *)(allocated - stm_object_pages); + nobj_sync_now = (uintptr_t)nobj; } + /* Copy the object */ + char *realnobj = REAL_ADDRESS(STM_SEGMENT->segment_base, nobj); + memcpy(realnobj, realobj, size); + /* Done copying the object. */ //dprintf(("\t\t\t\t\t%p -> %p\n", obj, nobj)); pforwarded_array[0] = GCWORD_MOVED; @@ -153,6 +155,7 @@ /* Must trace the object later */ LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, nobj_sync_now); + assert(nobj_sync_now == ((uintptr_t)nobj | is_small_uniform(nobj))); } static void collect_roots_in_nursery(void) diff --git a/c7/stm/smallmalloc.c b/c7/stm/smallmalloc.c --- a/c7/stm/smallmalloc.c +++ b/c7/stm/smallmalloc.c @@ -25,7 +25,7 @@ goto out_of_memory; uninitialized_page_stop -= decrease_by; - first_small_uniform_loc = (uintptr_t)uninitialized_page_stop; + first_small_uniform_loc = uninitialized_page_stop - stm_object_pages; char *base = stm_object_pages + END_NURSERY_PAGE * 4096UL; if (!_stm_largemalloc_resize_arena(uninitialized_page_stop - base)) @@ -51,8 +51,6 @@ static char *_allocate_small_slowpath(uint64_t size) { - /* First try to grab the next page from the global 'small_page_list' - */ long n = size / 8; struct small_page_list_s *smallpage; struct small_free_loc_s *TLPREFIX *fl = @@ -60,6 +58,8 @@ assert(*fl == NULL); retry: + /* First try to grab the next page from the global 'small_page_list' + */ smallpage = small_page_lists[n]; if (smallpage != NULL) { if (UNLIKELY(!__sync_bool_compare_and_swap(&small_page_lists[n], @@ -72,8 +72,8 @@ return (char *)smallpage; } - /* There is no more page waiting. Maybe we can pick one from - free_uniform_pages. + /* There is no more page waiting for the correct size of objects. + Maybe we can pick one from free_uniform_pages. */ smallpage = free_uniform_pages; if (smallpage != NULL) { @@ -104,7 +104,8 @@ return (char *)smallpage; } - /* Not a single free page left. Grab some more free pges and retry. */ + /* Not a single free page left. Grab some more free pages and retry. + */ grab_more_free_pages_for_small_allocations(); goto retry; } From noreply at buildbot.pypy.org Tue Mar 18 08:49:01 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 18 Mar 2014 08:49:01 +0100 (CET) Subject: [pypy-commit] pypy default: Backed out changeset 0ecfb7242213 Message-ID: <20140318074901.E4A6A1D26D4@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r70045:7c6999457664 Date: 2014-03-18 03:48 -0400 http://bitbucket.org/pypy/pypy/changeset/7c6999457664/ Log: Backed out changeset 0ecfb7242213 diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -432,11 +432,10 @@ return name - def getbuiltinmodule(self, name, force_init=False, reuse=True): + def getbuiltinmodule(self, name, force_init=False): w_name = self.wrap(name) w_modules = self.sys.get('modules') if not force_init: - assert reuse is True try: return self.getitem(w_modules, w_name) except OperationError, e: @@ -455,8 +454,6 @@ # And initialize it from pypy.interpreter.module import Module if isinstance(w_mod, Module): - if not reuse: - w_mod = type(w_mod)(self, w_name) w_mod.init(self) # Add the module to sys.modules diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py --- a/pypy/module/imp/importing.py +++ b/pypy/module/imp/importing.py @@ -579,7 +579,7 @@ return space.call_method(find_info.w_loader, "load_module", w_modulename) if find_info.modtype == C_BUILTIN: - return space.getbuiltinmodule(find_info.filename, force_init=True, reuse=reuse) + return space.getbuiltinmodule(find_info.filename, force_init=True) if find_info.modtype in (PY_SOURCE, PY_COMPILED, C_EXTENSION, PKG_DIRECTORY): w_mod = None diff --git a/pypy/module/imp/test/test_app.py b/pypy/module/imp/test/test_app.py --- a/pypy/module/imp/test/test_app.py +++ b/pypy/module/imp/test/test_app.py @@ -203,6 +203,7 @@ def test_builtin_reimport(self): # from https://bugs.pypy.org/issue1514 + skip("fix me") import sys, marshal old = marshal.loads @@ -222,6 +223,7 @@ # taken from https://bugs.pypy.org/issue1514, with extra cases # that show a difference with CPython: we can get on CPython # several module objects for the same built-in module :-( + skip("several built-in module objects: not supported by pypy") import sys, marshal old = marshal.loads diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py --- a/pypy/module/imp/test/test_import.py +++ b/pypy/module/imp/test/test_import.py @@ -578,6 +578,7 @@ assert hasattr(time, 'clock') def test_reimport_builtin_simple_case_2(self): + skip("fix me") import sys, time time.foo = "bar" del sys.modules['time'] @@ -585,6 +586,7 @@ assert not hasattr(time, 'foo') def test_reimport_builtin(self): + skip("fix me") import sys, time oldpath = sys.path time.tzset = "" From noreply at buildbot.pypy.org Tue Mar 18 08:49:29 2014 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 18 Mar 2014 08:49:29 +0100 (CET) Subject: [pypy-commit] pypy elidable-canfold-exception: Start a branch to constant-fold some errors in @elidable. Begin with Message-ID: <20140318074929.CA2CB1D26D4@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: elidable-canfold-exception Changeset: r70046:83468ba5580d Date: 2014-03-18 09:48 +0200 http://bitbucket.org/pypy/pypy/changeset/83468ba5580d/ Log: Start a branch to constant-fold some errors in @elidable. Begin with changing all @elidable to @elidable() diff --git a/pypy/interpreter/signature.py b/pypy/interpreter/signature.py --- a/pypy/interpreter/signature.py +++ b/pypy/interpreter/signature.py @@ -10,7 +10,7 @@ self.varargname = varargname self.kwargname = kwargname - @jit.elidable + @jit.elidable() def find_argname(self, name): try: return self.argnames.index(name) @@ -69,4 +69,4 @@ return self.varargname if i == 2: return self.kwargname - raise IndexError \ No newline at end of file + raise IndexError diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py --- a/pypy/module/_cffi_backend/ctypestruct.py +++ b/pypy/module/_cffi_backend/ctypestruct.py @@ -135,7 +135,7 @@ msg = "list or tuple or dict" raise self._convert_error(msg, w_ob) - @jit.elidable + @jit.elidable() def _getcfield_const(self, attr): return self.fields_dict[attr] diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py --- a/pypy/module/_codecs/interp_codecs.py +++ b/pypy/module/_codecs/interp_codecs.py @@ -100,7 +100,7 @@ def get_codec_from_cache(self, key): return self._get_codec_with_version(key, self.version) - @jit.elidable + @jit.elidable() def _get_codec_with_version(self, key, version): return self.codec_search_cache.get(key, None) diff --git a/pypy/module/_lsprof/interp_lsprof.py b/pypy/module/_lsprof/interp_lsprof.py --- a/pypy/module/_lsprof/interp_lsprof.py +++ b/pypy/module/_lsprof/interp_lsprof.py @@ -150,7 +150,7 @@ factor * float(self.ll_it), w_sublist) return space.wrap(w_se) - @jit.elidable + @jit.elidable() def _get_or_make_subentry(self, entry, make=True): try: return self.calls[entry] @@ -302,7 +302,7 @@ c_setup_profiling() space.getexecutioncontext().setllprofile(lsprof_call, space.wrap(self)) - @jit.elidable + @jit.elidable() def _get_or_make_entry(self, f_code, make=True): try: return self.data[f_code] @@ -313,7 +313,7 @@ return entry raise - @jit.elidable + @jit.elidable() def _get_or_make_builtin_entry(self, key, make=True): try: return self.builtin_data[key] diff --git a/pypy/module/_rawffi/alt/interp_ffitype.py b/pypy/module/_rawffi/alt/interp_ffitype.py --- a/pypy/module/_rawffi/alt/interp_ffitype.py +++ b/pypy/module/_rawffi/alt/interp_ffitype.py @@ -17,7 +17,7 @@ self.w_pointer_to = w_pointer_to self.set_ffitype(ffitype) - @jit.elidable + @jit.elidable() def get_ffitype(self): if not self._ffitype: raise ValueError("Operation not permitted on an incomplete type") diff --git a/pypy/module/cppyy/capi/loadable_capi.py b/pypy/module/cppyy/capi/loadable_capi.py --- a/pypy/module/cppyy/capi/loadable_capi.py +++ b/pypy/module/cppyy/capi/loadable_capi.py @@ -342,10 +342,10 @@ return _cdata_to_ptr(space, call_capi(space, 'allocate_function_args', [_Arg(l=size)])) def c_deallocate_function_args(space, cargs): call_capi(space, 'deallocate_function_args', [_Arg(vp=cargs)]) - at jit.elidable + at jit.elidable() def c_function_arg_sizeof(space): return _cdata_to_size_t(space, call_capi(space, 'function_arg_sizeof', [])) - at jit.elidable + at jit.elidable() def c_function_arg_typeoffset(space): return _cdata_to_size_t(space, call_capi(space, 'function_arg_typeoffset', [])) diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py --- a/pypy/module/imp/importing.py +++ b/pypy/module/imp/importing.py @@ -121,7 +121,7 @@ def check_sys_modules_w(space, modulename): return space.finditem_str(space.sys.get('modules'), modulename) - at jit.elidable + at jit.elidable() def _get_dot_position(str, n): # return the index in str of the '.' such that there are n '.'-separated # strings after it diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py --- a/pypy/objspace/std/mapdict.py +++ b/pypy/objspace/std/mapdict.py @@ -43,7 +43,7 @@ else: return obj._mapdict_read_storage(attr.storageindex) - @jit.elidable + @jit.elidable() def _pure_mapdict_read_storage(self, obj, storageindex): return obj._mapdict_read_storage(storageindex) @@ -68,7 +68,7 @@ else: return self._find_map_attr_indirection(selector) - @jit.elidable + @jit.elidable() def _find_map_attr_jit_pure(self, name, index): return self._find_map_attr_indirection((name, index)) @@ -128,14 +128,14 @@ def set_terminator(self, obj, terminator): raise NotImplementedError("abstract base class") - @jit.elidable + @jit.elidable() def size_estimate(self): return self._size_estimate >> NUM_DIGITS def search(self, attrtype): return None - @jit.elidable + @jit.elidable() def _get_new_attr(self, name, index): selector = name, index cache = self.cache_attrs diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py --- a/pypy/objspace/std/stringmethods.py +++ b/pypy/objspace/std/stringmethods.py @@ -585,7 +585,7 @@ return self return self._new(self.title(selfval)) - @jit.elidable + @jit.elidable() def title(self, value): builder = self._builder(len(value)) previous_is_cased = False diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py --- a/pypy/objspace/std/typeobject.py +++ b/pypy/objspace/std/typeobject.py @@ -259,7 +259,7 @@ return w_value return w_value - @elidable + @elidable() def _pure_getdictvalue_no_unwrapping(w_self, space, version_tag, attr): return w_self._getdictvalue_no_unwrapping(space, attr) @@ -371,7 +371,7 @@ w_class, w_value = w_self._pure_lookup_where_with_method_cache(name, version_tag) return w_class, unwrap_cell(space, w_value) - @elidable + @elidable() def _pure_lookup_where_with_method_cache(w_self, name, version_tag): space = w_self.space cache = space.fromcache(MethodCache) diff --git a/rpython/jit/backend/llsupport/test/zrpy_gc_test.py b/rpython/jit/backend/llsupport/test/zrpy_gc_test.py --- a/rpython/jit/backend/llsupport/test/zrpy_gc_test.py +++ b/rpython/jit/backend/llsupport/test/zrpy_gc_test.py @@ -634,7 +634,7 @@ self.run('compile_framework_external_exception_handling') def define_compile_framework_bug1(self): - @elidable + @elidable() def nonmoving(): x = X(1) for i in range(7): diff --git a/rpython/jit/codewriter/support.py b/rpython/jit/codewriter/support.py --- a/rpython/jit/codewriter/support.py +++ b/rpython/jit/codewriter/support.py @@ -216,12 +216,12 @@ _ll_3_list_resize_hint_really = rlist_ll._ll_list_resize_hint_really - at elidable + at elidable() def _ll_1_gc_identityhash(x): return lltype.identityhash(x) -# the following function should not be "@elidable": I can think of +# the following function should not be "@elidable()": I can think of # a corner case in which id(const) is constant-folded, and then 'const' # disappears and is collected too early (possibly causing another object # with the same id() to appear). diff --git a/rpython/jit/codewriter/test/test_effectinfo.py b/rpython/jit/codewriter/test/test_effectinfo.py --- a/rpython/jit/codewriter/test/test_effectinfo.py +++ b/rpython/jit/codewriter/test/test_effectinfo.py @@ -122,7 +122,7 @@ class B(A): x = 2 - @jit.elidable + @jit.elidable() def g(cls): return cls() diff --git a/rpython/jit/codewriter/test/test_policy.py b/rpython/jit/codewriter/test/test_policy.py --- a/rpython/jit/codewriter/test/test_policy.py +++ b/rpython/jit/codewriter/test/test_policy.py @@ -52,7 +52,7 @@ assert not policy.look_inside_graph(graph) def test_elidable(): - @jit.elidable + @jit.elidable() def g(x): return x + 2 graph = support.getgraph(g, [5]) diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py --- a/rpython/jit/metainterp/test/test_ajit.py +++ b/rpython/jit/metainterp/test/test_ajit.py @@ -413,7 +413,7 @@ call=0, call_pure=0) def test_residual_call_elidable_1(self): - @elidable + @elidable() def externfn(x, y): return x * y def f(n): @@ -426,7 +426,7 @@ def test_residual_call_elidable_2(self): myjitdriver = JitDriver(greens = [], reds = ['n']) - @elidable + @elidable() def externfn(x): return x - 1 def f(n): @@ -443,7 +443,7 @@ def test_constfold_call_elidable(self): myjitdriver = JitDriver(greens = ['m'], reds = ['n']) - @elidable + @elidable() def externfn(x): return x - 3 def f(n, m): @@ -459,7 +459,7 @@ def test_constfold_call_elidable_2(self): myjitdriver = JitDriver(greens = ['m'], reds = ['n']) - @elidable + @elidable() def externfn(x): return x - 3 class V: @@ -484,7 +484,7 @@ self.x = x v1 = V(1) v2 = V(2) - @elidable + @elidable() def externfn(x): if x: return v1 @@ -504,7 +504,7 @@ def test_elidable_raising(self): myjitdriver = JitDriver(greens = ['m'], reds = ['n']) - @elidable + @elidable() def externfn(x): if x <= 0: raise ValueError @@ -530,7 +530,7 @@ def test_elidable_raising_2(self): myjitdriver = JitDriver(greens = ['m'], reds = ['n']) - @elidable + @elidable() def externfn(x): if x <= 0: raise ValueError diff --git a/rpython/jit/metainterp/test/test_call.py b/rpython/jit/metainterp/test/test_call.py --- a/rpython/jit/metainterp/test/test_call.py +++ b/rpython/jit/metainterp/test/test_call.py @@ -28,7 +28,7 @@ def test_call_elidable_none(self): d = {} - @jit.elidable + @jit.elidable() def f(a): return d.get(a, None) diff --git a/rpython/jit/metainterp/test/test_jitprof.py b/rpython/jit/metainterp/test/test_jitprof.py --- a/rpython/jit/metainterp/test/test_jitprof.py +++ b/rpython/jit/metainterp/test/test_jitprof.py @@ -77,7 +77,7 @@ assert profiler.calls == 1 def test_blackhole_pure(self): - @elidable + @elidable() def g(n): return n+1 diff --git a/rpython/jit/metainterp/test/test_send.py b/rpython/jit/metainterp/test/test_send.py --- a/rpython/jit/metainterp/test/test_send.py +++ b/rpython/jit/metainterp/test/test_send.py @@ -596,7 +596,7 @@ def test_constfold_pure_oosend(self): myjitdriver = JitDriver(greens=[], reds = ['i', 'obj']) class A: - @elidable + @elidable() def foo(self): return 42 def fn(n, i): diff --git a/rpython/jit/metainterp/test/test_tracingopts.py b/rpython/jit/metainterp/test/test_tracingopts.py --- a/rpython/jit/metainterp/test/test_tracingopts.py +++ b/rpython/jit/metainterp/test/test_tracingopts.py @@ -411,7 +411,7 @@ a1.y = 6 a2 = A() a2.y = 13 - @jit.elidable + @jit.elidable() def f(b): return b + 1 def fn(n): diff --git a/rpython/jit/tl/tlc.py b/rpython/jit/tl/tlc.py --- a/rpython/jit/tl/tlc.py +++ b/rpython/jit/tl/tlc.py @@ -72,7 +72,7 @@ classes = [] # [(descr, cls), ...] - @elidable + @elidable() def get(key): for descr, cls in Class.classes: if key.attributes == descr.attributes and\ diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py --- a/rpython/rlib/jit.py +++ b/rpython/rlib/jit.py @@ -1,4 +1,5 @@ import sys +import types import py @@ -11,7 +12,7 @@ DEBUG_ELIDABLE_FUNCTIONS = False -def elidable(func): +def elidable(canfolderror=None): """ Decorate a function as "trace-elidable". Usually this means simply that the function is constant-foldable, i.e. is pure and has no side-effects. @@ -31,22 +32,31 @@ Most importantly it doesn't mean that an elidable function has no observable side effect, but those side effects are idempotent (ie caching). If a particular call to this function ends up raising an exception, then it - is handled like a normal function call (this decorator is ignored). + is handled like a normal function call (this decorator is ignored), unless + canfolderror is specified (which is an exception class or a tuple). + In this case, if it raises specific exception listed there, it's also + constant folded away. """ - if DEBUG_ELIDABLE_FUNCTIONS: - cache = {} - oldfunc = func - def func(*args): - result = oldfunc(*args) # if it raises, no caching - try: - oldresult = cache.setdefault(args, result) - except TypeError: - pass # unhashable args - else: - assert oldresult == result - return result - func._elidable_function_ = True - return func + if isinstance(canfolderror, types.FunctionType): + raise Exception("@elidable was changed to a full " + "decorator, use @elidable()") + def decorator(func): + if DEBUG_ELIDABLE_FUNCTIONS: + cache = {} + oldfunc = func + def func(*args): + result = oldfunc(*args) # if it raises, no caching + try: + oldresult = cache.setdefault(args, result) + except TypeError: + pass # unhashable args + else: + assert oldresult == result + return result + func._elidable_function_ = True + if canfolderror is not None: + func._elidable_exceptions_ = canfolderror + return func def purefunction(*args, **kwargs): import warnings diff --git a/rpython/rlib/jit_libffi.py b/rpython/rlib/jit_libffi.py --- a/rpython/rlib/jit_libffi.py +++ b/rpython/rlib/jit_libffi.py @@ -245,7 +245,7 @@ del cls._import @staticmethod - @jit.elidable + @jit.elidable() def getkind(ffi_type): """Returns 'v' for void, 'f' for float, 'i' for signed integer, 'u' for unsigned integer, 'S' for singlefloat, 'L' for long long @@ -281,7 +281,7 @@ return '?' @staticmethod - @jit.elidable + @jit.elidable() def is_struct(ffi_type): return rffi.getintfield(ffi_type, 'c_type') == FFI_TYPE_STRUCT diff --git a/rpython/rlib/libffi.py b/rpython/rlib/libffi.py --- a/rpython/rlib/libffi.py +++ b/rpython/rlib/libffi.py @@ -45,7 +45,7 @@ del cls._import @staticmethod - @jit.elidable + @jit.elidable() def getkind(ffi_type): """Returns 'v' for void, 'f' for float, 'i' for signed integer, and 'u' for unsigned integer. @@ -79,7 +79,7 @@ raise KeyError @staticmethod - @jit.elidable + @jit.elidable() def is_struct(ffi_type): return intmask(ffi_type.c_type) == FFI_TYPE_STRUCT diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py --- a/rpython/rlib/rbigint.py +++ b/rpython/rlib/rbigint.py @@ -169,7 +169,7 @@ numdigits._always_inline_ = True @staticmethod - @jit.elidable + @jit.elidable() def fromint(intval): # This function is marked as pure, so you must not call it and # then modify the result. @@ -192,7 +192,7 @@ return rbigint([_store_digit(ival & MASK)], sign, 1) @staticmethod - @jit.elidable + @jit.elidable() def frombool(b): # You must not call this function and then modify the result. if b: @@ -205,7 +205,7 @@ return rbigint(*args_from_long(l)) @staticmethod - @jit.elidable + @jit.elidable() def fromfloat(dval): """ Create a new bigint object from a float """ # This function is not marked as pure because it can raise @@ -216,7 +216,7 @@ return rbigint._fromfloat_finite(dval) @staticmethod - @jit.elidable + @jit.elidable() def _fromfloat_finite(dval): sign = 1 if dval < 0.0: @@ -238,7 +238,7 @@ return v @staticmethod - @jit.elidable + @jit.elidable() @specialize.argtype(0) def fromrarith_int(i): # This function is marked as pure, so you must not call it and @@ -246,14 +246,14 @@ return rbigint(*args_from_rarith_int(i)) @staticmethod - @jit.elidable + @jit.elidable() def fromdecimalstr(s): # This function is marked as elidable, so you must not call it and # then modify the result. return _decimalstr_to_bigint(s) @staticmethod - @jit.elidable + @jit.elidable() def fromstr(s, base=0): """As string_to_int(), but ignores an optional 'l' or 'L' suffix and returns an rbigint.""" @@ -271,7 +271,7 @@ return parse_digit_string(parser) @staticmethod - @jit.elidable + @jit.elidable() def frombytes(s, byteorder, signed): if byteorder not in ('big', 'little'): raise InvalidEndiannessError() @@ -311,7 +311,7 @@ result._normalize() return result - @jit.elidable + @jit.elidable() def tobytes(self, nbytes, byteorder, signed): if byteorder not in ('big', 'little'): raise InvalidEndiannessError() @@ -384,7 +384,7 @@ digits = ''.join([digits[i] for i in range(length-1, -1, -1)]) return digits - @jit.elidable + @jit.elidable() def toint(self): """ Get an integer from a bigint object. @@ -400,7 +400,7 @@ raise OverflowError return intmask(intmask(x) * sign) - @jit.elidable + @jit.elidable() def tolonglong(self): return _AsLongLong(self) @@ -408,13 +408,13 @@ def tobool(self): return self.sign != 0 - @jit.elidable + @jit.elidable() def touint(self): if self.sign == -1: raise ValueError("cannot convert negative integer to unsigned int") return self._touint_helper() - @jit.elidable + @jit.elidable() def _touint_helper(self): x = r_uint(0) i = self.numdigits() - 1 @@ -427,32 +427,32 @@ i -= 1 return x - @jit.elidable + @jit.elidable() def toulonglong(self): if self.sign == -1: raise ValueError("cannot convert negative integer to unsigned int") return _AsULonglong_ignore_sign(self) - @jit.elidable + @jit.elidable() def uintmask(self): return _AsUInt_mask(self) - @jit.elidable + @jit.elidable() def ulonglongmask(self): """Return r_ulonglong(self), truncating.""" return _AsULonglong_mask(self) - @jit.elidable + @jit.elidable() def tofloat(self): return _AsDouble(self) - @jit.elidable + @jit.elidable() def format(self, digits, prefix='', suffix=''): # 'digits' is a string whose length is the base to use, # and where each character is the corresponding digit. return _format(self, digits, prefix, suffix) - @jit.elidable + @jit.elidable() def repr(self): try: x = self.toint() @@ -460,7 +460,7 @@ return self.format(BASE10, suffix="L") return str(x) + "L" - @jit.elidable + @jit.elidable() def str(self): try: x = self.toint() @@ -468,7 +468,7 @@ return self.format(BASE10) return str(x) - @jit.elidable + @jit.elidable() def eq(self, other): if (self.sign != other.sign or self.numdigits() != other.numdigits()): @@ -486,7 +486,7 @@ def ne(self, other): return not self.eq(other) - @jit.elidable + @jit.elidable() def lt(self, other): if self.sign > other.sign: return False @@ -533,11 +533,11 @@ def ge(self, other): return not self.lt(other) - @jit.elidable + @jit.elidable() def hash(self): return _hash(self) - @jit.elidable + @jit.elidable() def add(self, other): if self.sign == 0: return other @@ -550,7 +550,7 @@ result.sign *= other.sign return result - @jit.elidable + @jit.elidable() def sub(self, other): if other.sign == 0: return self @@ -563,7 +563,7 @@ result.sign *= self.sign return result - @jit.elidable + @jit.elidable() def mul(self, b): asize = self.numdigits() bsize = b.numdigits() @@ -608,12 +608,12 @@ result.sign = a.sign * b.sign return result - @jit.elidable + @jit.elidable() def truediv(self, other): div = _bigint_true_divide(self, other) return div - @jit.elidable + @jit.elidable() def floordiv(self, other): if self.sign == 1 and other.numdigits() == 1 and other.sign == 1: digit = other.digit(0) @@ -634,7 +634,7 @@ def div(self, other): return self.floordiv(other) - @jit.elidable + @jit.elidable() def mod(self, other): if self.sign == 0: return NULLRBIGINT @@ -671,7 +671,7 @@ mod = mod.add(other) return mod - @jit.elidable + @jit.elidable() def divmod(v, w): """ The / and % operators are now defined in terms of divmod(). @@ -697,7 +697,7 @@ div = div.sub(ONERBIGINT) return div, mod - @jit.elidable + @jit.elidable() def pow(a, b, c=None): negativeOutput = False # if x<0 return negative output @@ -837,17 +837,17 @@ z = z.sub(c) return z - @jit.elidable + @jit.elidable() def neg(self): return rbigint(self._digits, -self.sign, self.size) - @jit.elidable + @jit.elidable() def abs(self): if self.sign != -1: return self return rbigint(self._digits, 1, self.size) - @jit.elidable + @jit.elidable() def invert(self): #Implement ~x as -(x + 1) if self.sign == 0: return ONENEGATIVERBIGINT @@ -856,7 +856,7 @@ ret.sign = -ret.sign return ret - @jit.elidable + @jit.elidable() def lshift(self, int_other): if int_other < 0: raise ValueError("negative shift count") @@ -893,7 +893,7 @@ return z lshift._always_inline_ = True # It's so fast that it's always benefitial. - @jit.elidable + @jit.elidable() def lqshift(self, int_other): " A quicker one with much less checks, int_other is valid and for the most part constant." assert int_other > 0 @@ -913,7 +913,7 @@ return z lqshift._always_inline_ = True # It's so fast that it's always benefitial. - @jit.elidable + @jit.elidable() def rshift(self, int_other, dont_invert=False): if int_other < 0: raise ValueError("negative shift count") @@ -943,7 +943,7 @@ return z rshift._always_inline_ = 'try' # It's so fast that it's always benefitial. - @jit.elidable + @jit.elidable() def abs_rshift_and_mask(self, bigshiftcount, mask): assert isinstance(bigshiftcount, r_ulonglong) assert mask >= 0 @@ -992,30 +992,30 @@ z._normalize() return z - @jit.elidable + @jit.elidable() def and_(self, other): return _bitwise(self, '&', other) - @jit.elidable + @jit.elidable() def xor(self, other): return _bitwise(self, '^', other) - @jit.elidable + @jit.elidable() def or_(self, other): return _bitwise(self, '|', other) - @jit.elidable + @jit.elidable() def oct(self): if self.sign == 0: return '0L' else: return _format(self, BASE8, '0', 'L') - @jit.elidable + @jit.elidable() def hex(self): return _format(self, BASE16, '0x', 'L') - @jit.elidable + @jit.elidable() def log(self, base): # base is supposed to be positive or 0.0, which means we use e if base == 10.0: @@ -1050,7 +1050,7 @@ _normalize._always_inline_ = True - @jit.elidable + @jit.elidable() def bit_length(self): i = self.numdigits() if i == 1 and self._digits[0] == NULLDIGIT: diff --git a/rpython/rlib/rstring.py b/rpython/rlib/rstring.py --- a/rpython/rlib/rstring.py +++ b/rpython/rlib/rstring.py @@ -160,7 +160,7 @@ @specialize.argtype(0) - at jit.elidable + at jit.elidable() def replace(input, sub, by, maxsplit=-1): if isinstance(input, str): assert isinstance(sub, str) @@ -236,7 +236,7 @@ return start, end @specialize.argtype(0) - at jit.elidable + at jit.elidable() def startswith(u_self, prefix, start=0, end=sys.maxint): length = len(u_self) start, end = _normalize_start_end(length, start, end) @@ -249,7 +249,7 @@ return True @specialize.argtype(0) - at jit.elidable + at jit.elidable() def endswith(u_self, suffix, start=0, end=sys.maxint): length = len(u_self) start, end = _normalize_start_end(length, start, end) diff --git a/rpython/rtyper/lltypesystem/ll_str.py b/rpython/rtyper/lltypesystem/ll_str.py --- a/rpython/rtyper/lltypesystem/ll_str.py +++ b/rpython/rtyper/lltypesystem/ll_str.py @@ -10,7 +10,7 @@ else: return r_uint(i) - at jit.elidable + at jit.elidable() def ll_int2dec(val): from rpython.rtyper.lltypesystem.rstr import mallocstr @@ -44,7 +44,7 @@ for i in range(16): hex_chars[i] = "%x" % i - at jit.elidable + at jit.elidable() def ll_int2hex(i, addPrefix): from rpython.rtyper.lltypesystem.rstr import mallocstr temp = malloc(CHAR_ARRAY, 20) @@ -81,7 +81,7 @@ j += 1 return result - at jit.elidable + at jit.elidable() def ll_int2oct(i, addPrefix): from rpython.rtyper.lltypesystem.rstr import mallocstr if i == 0: diff --git a/rpython/rtyper/lltypesystem/module/ll_math.py b/rpython/rtyper/lltypesystem/module/ll_math.py --- a/rpython/rtyper/lltypesystem/module/ll_math.py +++ b/rpython/rtyper/lltypesystem/module/ll_math.py @@ -71,7 +71,7 @@ math_sin = llexternal('sin', [rffi.DOUBLE], rffi.DOUBLE, elidable_function=True) math_cos = llexternal('cos', [rffi.DOUBLE], rffi.DOUBLE, elidable_function=True) - at jit.elidable + at jit.elidable() def sqrt_nonneg(x): return math_sqrt(x) sqrt_nonneg.oopspec = "math.sqrt_nonneg(x)" diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py --- a/rpython/rtyper/lltypesystem/rstr.py +++ b/rpython/rtyper/lltypesystem/rstr.py @@ -199,7 +199,7 @@ self.ll = LLHelpers self.malloc = mallocunicode - @jit.elidable + @jit.elidable() def ll_str(self, s): # XXX crazy that this is here, but I don't want to break # rmodel logic @@ -214,14 +214,14 @@ result.chars[i] = cast_primitive(Char, c) return result - @jit.elidable + @jit.elidable() def ll_unicode(self, s): if s: return s else: return self.ll.ll_constant_unicode(u'None') - @jit.elidable + @jit.elidable() def ll_encode_latin1(self, s): length = len(s.chars) result = mallocstr(length) @@ -266,7 +266,7 @@ class LLHelpers(AbstractLLHelpers): from rpython.rtyper.annlowlevel import llstr, llunicode - @jit.elidable + @jit.elidable() def ll_str_mul(s, times): if times < 0: times = 0 @@ -288,7 +288,7 @@ i += j return newstr - @jit.elidable + @jit.elidable() def ll_char_mul(ch, times): if typeOf(ch) is Char: malloc = mallocstr @@ -343,7 +343,7 @@ b.chars[i] = str.chars[i] return b - @jit.elidable + @jit.elidable() def ll_strhash(s): # unlike CPython, there is no reason to avoid to return -1 # but our malloc initializes the memory to zero, so we use zero as the @@ -364,7 +364,7 @@ def ll_strfasthash(s): return s.hash # assumes that the hash is already computed - @jit.elidable + @jit.elidable() def ll_strconcat(s1, s2): len1 = s1.length() len2 = s2.length() @@ -384,7 +384,7 @@ return newstr ll_strconcat.oopspec = 'stroruni.concat(s1, s2)' - @jit.elidable + @jit.elidable() def ll_strip(s, ch, left, right): s_len = len(s.chars) if s_len == 0: @@ -404,7 +404,7 @@ s.copy_contents(s, result, lpos, 0, r_len) return result - @jit.elidable + @jit.elidable() def ll_strip_default(s, left, right): s_len = len(s.chars) if s_len == 0: @@ -424,7 +424,7 @@ s.copy_contents(s, result, lpos, 0, r_len) return result - @jit.elidable + @jit.elidable() def ll_strip_multiple(s, s2, left, right): s_len = len(s.chars) if s_len == 0: @@ -444,7 +444,7 @@ s.copy_contents(s, result, lpos, 0, r_len) return result - @jit.elidable + @jit.elidable() def ll_upper(s): s_chars = s.chars s_len = len(s_chars) @@ -458,7 +458,7 @@ i += 1 return result - @jit.elidable + @jit.elidable() def ll_lower(s): s_chars = s.chars s_len = len(s_chars) @@ -505,7 +505,7 @@ i += 1 return result - @jit.elidable + @jit.elidable() def ll_strcmp(s1, s2): if not s1 and not s2: return True @@ -528,7 +528,7 @@ i += 1 return len1 - len2 - @jit.elidable + @jit.elidable() def ll_streq(s1, s2): if s1 == s2: # also if both are NULLs return True @@ -548,7 +548,7 @@ return True ll_streq.oopspec = 'stroruni.equal(s1, s2)' - @jit.elidable + @jit.elidable() def ll_startswith(s1, s2): len1 = len(s1.chars) len2 = len(s2.chars) @@ -569,7 +569,7 @@ return False return s.chars[0] == ch - @jit.elidable + @jit.elidable() def ll_endswith(s1, s2): len1 = len(s1.chars) len2 = len(s2.chars) @@ -591,7 +591,7 @@ return False return s.chars[len(s.chars) - 1] == ch - @jit.elidable + @jit.elidable() @signature(types.any(), types.any(), types.int(), types.int(), returns=types.int()) def ll_find_char(s, ch, start, end): i = start @@ -603,7 +603,7 @@ i += 1 return -1 - @jit.elidable + @jit.elidable() def ll_rfind_char(s, ch, start, end): if end > len(s.chars): end = len(s.chars) @@ -614,7 +614,7 @@ return i return -1 - @jit.elidable + @jit.elidable() def ll_count_char(s, ch, start, end): count = 0 i = start @@ -676,7 +676,7 @@ res = 0 return res - @jit.elidable + @jit.elidable() def ll_search(s1, s2, start, end, mode): count = 0 n = end - start @@ -818,7 +818,7 @@ @jit.oopspec('stroruni.slice(s1, start, stop)') @signature(types.any(), types.int(), types.int(), returns=types.any()) - @jit.elidable + @jit.elidable() def _ll_stringslice(s1, start, stop): lgt = stop - start assert start >= 0 @@ -980,7 +980,7 @@ item.copy_contents(s, item, 0, 0, prev_pos) return res - @jit.elidable + @jit.elidable() def ll_replace_chr_chr(s, c1, c2): length = len(s.chars) newstr = s.malloc(length) @@ -995,7 +995,7 @@ j += 1 return newstr - @jit.elidable + @jit.elidable() def ll_contains(s, c): chars = s.chars strlen = len(chars) @@ -1006,7 +1006,7 @@ i += 1 return False - @jit.elidable + @jit.elidable() def ll_int(s, base): if not 2 <= base <= 36: raise ValueError diff --git a/rpython/rtyper/rfloat.py b/rpython/rtyper/rfloat.py --- a/rpython/rtyper/rfloat.py +++ b/rpython/rtyper/rfloat.py @@ -136,7 +136,7 @@ hop.exception_cannot_occur() return vlist[0] - @jit.elidable + @jit.elidable() def ll_str(self, f): return llstr(formatd(f, 'f', 6)) diff --git a/rpython/rtyper/rint.py b/rpython/rtyper/rint.py --- a/rpython/rtyper/rint.py +++ b/rpython/rtyper/rint.py @@ -366,7 +366,7 @@ hop.exception_cannot_occur() return vlist[0] - @jit.elidable + @jit.elidable() def ll_str(self, i): from rpython.rtyper.lltypesystem.ll_str import ll_int2dec return ll_int2dec(i) diff --git a/rpython/rtyper/rstr.py b/rpython/rtyper/rstr.py --- a/rpython/rtyper/rstr.py +++ b/rpython/rtyper/rstr.py @@ -20,7 +20,7 @@ self.rstr_decode_utf_8 = func_with_new_name(str_decode_utf_8_impl, 'rstr_decode_utf_8_impl') - @jit.elidable + @jit.elidable() def ll_decode_utf8(self, llvalue): from rpython.rtyper.annlowlevel import hlstr value = hlstr(llvalue) @@ -70,7 +70,7 @@ def rtype_method_lower(self, hop): raise TypeError("Cannot do tolower on unicode string") - @jit.elidable + @jit.elidable() def ll_encode_utf8(self, ll_s): from rpython.rtyper.annlowlevel import hlunicode s = hlunicode(ll_s) diff --git a/rpython/tool/error.py b/rpython/tool/error.py --- a/rpython/tool/error.py +++ b/rpython/tool/error.py @@ -160,7 +160,7 @@ pdb_plus_show.start(tb) - at jit.elidable + at jit.elidable() def offset2lineno(c, stopat): tab = c.co_lnotab line = c.co_firstlineno From noreply at buildbot.pypy.org Tue Mar 18 09:32:49 2014 From: noreply at buildbot.pypy.org (cfbolz) Date: Tue, 18 Mar 2014 09:32:49 +0100 (CET) Subject: [pypy-commit] pypy elidable-canfold-exception: make the old @elidable still usable, with a warning Message-ID: <20140318083249.551191C0166@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: elidable-canfold-exception Changeset: r70047:4ce5abc5856b Date: 2014-03-18 09:32 +0100 http://bitbucket.org/pypy/pypy/changeset/4ce5abc5856b/ Log: make the old @elidable still usable, with a warning diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py --- a/rpython/rlib/jit.py +++ b/rpython/rlib/jit.py @@ -1,5 +1,6 @@ import sys import types +import warnings import py @@ -12,7 +13,7 @@ DEBUG_ELIDABLE_FUNCTIONS = False -def elidable(canfolderror=None): +def elidable(*args, **kwargs): """ Decorate a function as "trace-elidable". Usually this means simply that the function is constant-foldable, i.e. is pure and has no side-effects. @@ -37,9 +38,7 @@ In this case, if it raises specific exception listed there, it's also constant folded away. """ - if isinstance(canfolderror, types.FunctionType): - raise Exception("@elidable was changed to a full " - "decorator, use @elidable()") + def decorator(func): if DEBUG_ELIDABLE_FUNCTIONS: cache = {} @@ -58,6 +57,17 @@ func._elidable_exceptions_ = canfolderror return func + if len(args) == 1: + # non-ported + assert len(kwargs) == 0 + warnings.warn("@elidable is deprecated, use @elidable() instead", stacklevel=2) + canfolderror = None + return decorator(args[0]) + else: + assert len(args) == 0 + canfolderror = kwargs.get('canfolderror', None) + return decorator + def purefunction(*args, **kwargs): import warnings warnings.warn("purefunction is deprecated, use elidable instead", DeprecationWarning) @@ -133,7 +143,7 @@ function """ def decorator(func): - elidable(func) + elidable()(func) args = _get_args(func) argstring = ", ".join(args) code = ["def f(%s):\n" % (argstring, )] diff --git a/rpython/rlib/test/test_jit.py b/rpython/rlib/test/test_jit.py --- a/rpython/rlib/test/test_jit.py +++ b/rpython/rlib/test/test_jit.py @@ -3,7 +3,7 @@ from rpython.conftest import option from rpython.annotator.model import UnionError from rpython.rlib.jit import (hint, we_are_jitted, JitDriver, elidable_promote, - JitHintError, oopspec, isconstant, conditional_call) + JitHintError, oopspec, isconstant, conditional_call, elidable) from rpython.rlib.rarithmetic import r_uint from rpython.rtyper.test.tool import BaseRtypingTest from rpython.rtyper.lltypesystem import lltype @@ -91,6 +91,27 @@ myjitdriver = JitDriver(greens=['n'], reds=[]) py.test.raises(JitHintError, fn, 100) +def test_elidable(): + import warnings + # deprecated version + if 1:#with warnings.catch_warnings(record=True) as w: + #assert not w + @elidable + def f(): + pass + assert f._elidable_function_ + #assert "@elidable()" in w[0].message[0] + + # proper version + @elidable() + def f(): + pass + assert f._elidable_function_ + @elidable(canfolderror=ValueError) + def f(): + pass + assert f._elidable_function_ + class TestJIT(BaseRtypingTest): def test_hint(self): def f(): From noreply at buildbot.pypy.org Tue Mar 18 09:34:44 2014 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 18 Mar 2014 09:34:44 +0100 (CET) Subject: [pypy-commit] stmgc default: Add a README for the llvmfix directory Message-ID: <20140318083444.DD2591C0166@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1062:57aa8f49c843 Date: 2014-03-18 09:32 +0100 http://bitbucket.org/pypy/stmgc/changeset/57aa8f49c843/ Log: Add a README for the llvmfix directory diff --git a/c7/llvmfix/README.txt b/c7/llvmfix/README.txt new file mode 100644 --- /dev/null +++ b/c7/llvmfix/README.txt @@ -0,0 +1,24 @@ + +no-introduce-bogus-cast-in-combine.diff + + This is just fixes for a couple of bugs. + + +no-memset-creation-with-addrspace.diff + + This is a workaround for the fact that llvm.memset doesn't support + the address_space 256. It's a workaround, because it also prevents + some useful optimizations: for example replacing "x->a = 0; x->b = + 0;" with a single larger zeroing instruction. In other words, it + crashes only if an unpatched llvm introduce llvm.memset *and* this + memset remains as a real function call in the end. + + +addrspacecast-in-constant.diff + + This is a workaround for (what we believe to be) clang producing + incorrectly the addrspacecast operation for this kind of code: + + static __attribute__((address_space(256))) long a = 42; + struct s1 { void *a; }; + struct s1 fofo = { (void *)(long)&a }; From noreply at buildbot.pypy.org Tue Mar 18 09:36:28 2014 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 18 Mar 2014 09:36:28 +0100 (CET) Subject: [pypy-commit] stmgc default: Change the example to not rely on globals in strange address spaces Message-ID: <20140318083628.231C41C0166@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1063:4574e37f7ae6 Date: 2014-03-18 09:34 +0100 http://bitbucket.org/pypy/stmgc/changeset/4574e37f7ae6/ Log: Change the example to not rely on globals in strange address spaces diff --git a/c7/llvmfix/README.txt b/c7/llvmfix/README.txt --- a/c7/llvmfix/README.txt +++ b/c7/llvmfix/README.txt @@ -19,6 +19,6 @@ This is a workaround for (what we believe to be) clang producing incorrectly the addrspacecast operation for this kind of code: - static __attribute__((address_space(256))) long a = 42; - struct s1 { void *a; }; - struct s1 fofo = { (void *)(long)&a }; + static int a = 42; + struct s1 { void __attribute__((address_space(256))) *a; }; + struct s1 fofo = { (void __attribute__((address_space(256))) *)(long)&a }; From noreply at buildbot.pypy.org Tue Mar 18 09:40:47 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 18 Mar 2014 09:40:47 +0100 (CET) Subject: [pypy-commit] pypy elidable-canfold-exception: cleaner? no need to unpack args ourself Message-ID: <20140318084047.F41CF1C00B9@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: elidable-canfold-exception Changeset: r70048:ca1d94c202fc Date: 2014-03-18 04:39 -0400 http://bitbucket.org/pypy/pypy/changeset/ca1d94c202fc/ Log: cleaner? no need to unpack args ourself diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py --- a/rpython/rlib/jit.py +++ b/rpython/rlib/jit.py @@ -1,5 +1,4 @@ import sys -import types import warnings import py @@ -13,7 +12,7 @@ DEBUG_ELIDABLE_FUNCTIONS = False -def elidable(*args, **kwargs): +def elidable(canfolderror=None): """ Decorate a function as "trace-elidable". Usually this means simply that the function is constant-foldable, i.e. is pure and has no side-effects. @@ -38,7 +37,6 @@ In this case, if it raises specific exception listed there, it's also constant folded away. """ - def decorator(func): if DEBUG_ELIDABLE_FUNCTIONS: cache = {} @@ -56,17 +54,17 @@ if canfolderror is not None: func._elidable_exceptions_ = canfolderror return func + return decorator +_elidable = elidable +def elidable(*args, **kwargs): if len(args) == 1: - # non-ported assert len(kwargs) == 0 warnings.warn("@elidable is deprecated, use @elidable() instead", stacklevel=2) - canfolderror = None - return decorator(args[0]) + return _elidable()(args[0]) else: assert len(args) == 0 - canfolderror = kwargs.get('canfolderror', None) - return decorator + return _elidable(**kwargs) def purefunction(*args, **kwargs): import warnings From noreply at buildbot.pypy.org Tue Mar 18 10:08:53 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 18 Mar 2014 10:08:53 +0100 (CET) Subject: [pypy-commit] pypy elidable-canfold-exception: could also take this approach Message-ID: <20140318090853.725DB1D2540@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: elidable-canfold-exception Changeset: r70049:f73abcc35c33 Date: 2014-03-18 05:07 -0400 http://bitbucket.org/pypy/pypy/changeset/f73abcc35c33/ Log: could also take this approach diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py --- a/rpython/rlib/jit.py +++ b/rpython/rlib/jit.py @@ -1,4 +1,6 @@ +from __future__ import absolute_import import sys +import types import warnings import py @@ -58,16 +60,13 @@ _elidable = elidable def elidable(*args, **kwargs): - if len(args) == 1: + if len(args) == 1 and isinstance(args[0], types.FunctionType): assert len(kwargs) == 0 warnings.warn("@elidable is deprecated, use @elidable() instead", stacklevel=2) return _elidable()(args[0]) - else: - assert len(args) == 0 - return _elidable(**kwargs) + return _elidable(*args, **kwargs) def purefunction(*args, **kwargs): - import warnings warnings.warn("purefunction is deprecated, use elidable instead", DeprecationWarning) return elidable(*args, **kwargs) @@ -160,7 +159,6 @@ return decorator def purefunction_promote(*args, **kwargs): - import warnings warnings.warn("purefunction_promote is deprecated, use elidable_promote instead", DeprecationWarning) return elidable_promote(*args, **kwargs) diff --git a/rpython/rlib/test/test_jit.py b/rpython/rlib/test/test_jit.py --- a/rpython/rlib/test/test_jit.py +++ b/rpython/rlib/test/test_jit.py @@ -94,19 +94,25 @@ def test_elidable(): import warnings # deprecated version - if 1:#with warnings.catch_warnings(record=True) as w: - #assert not w + with warnings.catch_warnings(record=True) as w: + assert not w @elidable def f(): pass assert f._elidable_function_ - #assert "@elidable()" in w[0].message[0] + assert "@elidable()" in w[0].message[0] # proper version @elidable() def f(): pass assert f._elidable_function_ + + @elidable(ValueError) + def f(): + pass + assert f._elidable_function_ + @elidable(canfolderror=ValueError) def f(): pass From noreply at buildbot.pypy.org Tue Mar 18 10:31:29 2014 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 18 Mar 2014 10:31:29 +0100 (CET) Subject: [pypy-commit] stmgc c7-fork: A branch to add fork() support Message-ID: <20140318093129.1A8EE1D26D4@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c7-fork Changeset: r1064:1df308a509dd Date: 2014-03-18 10:29 +0100 http://bitbucket.org/pypy/stmgc/changeset/1df308a509dd/ Log: A branch to add fork() support From noreply at buildbot.pypy.org Tue Mar 18 10:31:30 2014 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 18 Mar 2014 10:31:30 +0100 (CET) Subject: [pypy-commit] stmgc c7-fork: in-progress, maybe, who knows Message-ID: <20140318093130.38ED71D26D4@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c7-fork Changeset: r1065:f467a519f7b6 Date: 2014-03-18 10:29 +0100 http://bitbucket.org/pypy/stmgc/changeset/f467a519f7b6/ Log: in-progress, maybe, who knows diff --git a/c7/stm/forksupport.c b/c7/stm/forksupport.c new file mode 100644 --- /dev/null +++ b/c7/stm/forksupport.c @@ -0,0 +1,94 @@ +#ifndef _STM_CORE_H_ +# error "must be compiled via stmgc.c" +#endif + + +/* XXX this is currently not doing copy-on-write, but simply forces a + copy of all shared pages as soon as fork() is called. */ + + +static char *fork_big_copy; + + +static void forksupport_prepare(void) +{ + if (stm_object_pages == NULL) + return; + + /* This silently assumes that fork() is not called from transactions. + It's hard to check though... + */ + s_mutex_lock(); + + synchronize_all_threads(); + + mutex_pages_lock(); + + char *big_copy = setup_mmap("stmgc's fork support"); + + uintptr_t pagenum, endpagenum; + pagenum = END_NURSERY_PAGE; /* starts after the nursery */ + endpagenum = (uninitialized_page_start - stm_object_pages) / 4096UL; + + while (1) { + if (UNLIKELY(pagenum == endpagenum)) { + /* we reach this point usually twice, because there are + more pages after 'uninitialized_page_stop' */ + if (endpagenum == NB_PAGES) + break; /* done */ + pagenum = (uninitialized_page_stop - stm_object_pages) / 4096UL; + endpagenum = NB_PAGES; + if (pagenum == endpagenum) + break; /* no pages in the 2nd section, so done too */ + } + + pagecopy(big_copy + pagenum * 4096UL, + stm_object_pages + pagenum * 4096UL); + pagenum++; + } + + assert(fork_big_copy == NULL); + fork_big_copy = big_copy; +} + +static void forksupport_parent(void) +{ + if (stm_object_pages == NULL) + return; + + assert(fork_big_copy != NULL); + munmap(fork_big_copy, TOTAL_MEMORY); + stm_object_pages = NULL; + + mutex_pages_unlock(); + s_mutex_unlock(); +} + +static void forksupport_child(void) +{ + if (stm_object_pages == NULL) + return; + + mremap(fork_big_copy, TOTAL_MEMORY, TOTAL_MEMORY, + MREMAP_MAYMOVE | MREMAP_FIXED, + stm_object_pages); + + ...; reset carefully a much bigger part of the state here :-((( + memset(pages_privatized, 0, sizeof(pages_privatized)); + + mutex_pages_unlock(); + s_mutex_unlock(); +} + + +static void setup_forksupport(void) +{ + static bool fork_support_ready = false; + + if (!fork_support_ready) { + int res = pthread_atfork(forksupport_prepare, forksupport_parent, + forksupport_child); + assert(res == 0); + fork_support_ready = true; + } +} diff --git a/c7/stm/forksupport.h b/c7/stm/forksupport.h new file mode 100644 --- /dev/null +++ b/c7/stm/forksupport.h @@ -0,0 +1,2 @@ + +static void setup_forksupport(void); diff --git a/c7/stm/pages.h b/c7/stm/pages.h --- a/c7/stm/pages.h +++ b/c7/stm/pages.h @@ -39,6 +39,7 @@ static void page_privatize(uintptr_t pagenum); static void page_reshare(uintptr_t pagenum); +/* Note: don't ever do "mutex_pages_lock(); mutex_lock()" in that order */ static void mutex_pages_lock(void); static void mutex_pages_unlock(void); static bool _has_mutex_pages(void) __attribute__((unused)); diff --git a/c7/stm/setup.c b/c7/stm/setup.c --- a/c7/stm/setup.c +++ b/c7/stm/setup.c @@ -3,6 +3,38 @@ #endif +static char *setup_mmap(char *reason) +{ + char *result = mmap(NULL, TOTAL_MEMORY, + PROT_READ | PROT_WRITE, + MAP_PAGES_FLAGS, -1, 0); + if (result == MAP_FAILED) + stm_fatalerror("%s failed: %m\n", reason); + + /* The segment 0 is not used to run transactions, but contains the + shared copy of the pages. We mprotect all pages before so that + accesses fail, up to and including the pages corresponding to the + nurseries of the other segments. */ + mprotect(result, END_NURSERY_PAGE * 4096UL, PROT_NONE); + + long i; + for (i = 1; i <= NB_SEGMENTS; i++) { + char *segment_base = result + i * (NB_PAGES * 4096UL); + + /* In each segment, the first page is where TLPREFIX'ed + NULL accesses land. We mprotect it so that accesses fail. */ + mprotect(segment_base, 4096, PROT_NONE); + + /* Pages in range(2, FIRST_READMARKER_PAGE) are never used */ + if (FIRST_READMARKER_PAGE > 2) + mprotect(segment_base + 8192, + (FIRST_READMARKER_PAGE - 2) * 4096UL, + PROT_NONE); + } + + return result; +} + void stm_setup(void) { /* Check that some values are acceptable */ @@ -20,38 +52,18 @@ (FIRST_READMARKER_PAGE * 4096UL)); assert(_STM_FAST_ALLOC <= NB_NURSERY_PAGES * 4096); - stm_object_pages = mmap(NULL, TOTAL_MEMORY, - PROT_READ | PROT_WRITE, - MAP_PAGES_FLAGS, -1, 0); - if (stm_object_pages == MAP_FAILED) - stm_fatalerror("initial stm_object_pages mmap() failed: %m\n"); - - /* The segment 0 is not used to run transactions, but to contain the - shared copy of the pages. We mprotect all pages before so that - accesses fail, up to and including the pages corresponding to the - nurseries of the other segments. */ - mprotect(stm_object_pages, END_NURSERY_PAGE * 4096UL, PROT_NONE); + stm_object_pages = setup_mmap("initial stm_object_pages mmap()"); long i; for (i = 1; i <= NB_SEGMENTS; i++) { char *segment_base = get_segment_base(i); - /* In each segment, the first page is where TLPREFIX'ed - NULL accesses land. We mprotect it so that accesses fail. */ - mprotect(segment_base, 4096, PROT_NONE); - /* Fill the TLS page (page 1) with 0xDC, for debugging */ memset(REAL_ADDRESS(segment_base, 4096), 0xDC, 4096); /* Make a "hole" at STM_PSEGMENT (which includes STM_SEGMENT) */ memset(REAL_ADDRESS(segment_base, STM_PSEGMENT), 0, sizeof(*STM_PSEGMENT)); - /* Pages in range(2, FIRST_READMARKER_PAGE) are never used */ - if (FIRST_READMARKER_PAGE > 2) - mprotect(segment_base + 8192, - (FIRST_READMARKER_PAGE - 2) * 4096UL, - PROT_NONE); - /* Initialize STM_PSEGMENT */ struct stm_priv_segment_info_s *pr = get_priv_segment(i); assert(1 <= i && i < 255); /* 255 is WL_VISITED in gcpage.c */ @@ -83,6 +95,7 @@ setup_nursery(); setup_gcpage(); setup_pages(); + setup_forksupport(); } void stm_teardown(void) diff --git a/c7/stmgc.c b/c7/stmgc.c --- a/c7/stmgc.c +++ b/c7/stmgc.c @@ -13,6 +13,7 @@ #include "stm/extra.h" #include "stm/fprintcolor.h" #include "stm/weakref.h" +#include "stm/forksupport.h" #include "stm/misc.c" #include "stm/list.c" @@ -30,3 +31,4 @@ #include "stm/extra.c" #include "stm/fprintcolor.c" #include "stm/weakref.c" +#include "stm/forksupport.c" From noreply at buildbot.pypy.org Tue Mar 18 11:52:20 2014 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 18 Mar 2014 11:52:20 +0100 (CET) Subject: [pypy-commit] stmgc c7-fork: Hopefully finish fork support Message-ID: <20140318105220.5FBD51D26D3@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c7-fork Changeset: r1066:c07cd6a30719 Date: 2014-03-18 11:50 +0100 http://bitbucket.org/pypy/stmgc/changeset/c07cd6a30719/ Log: Hopefully finish fork support diff --git a/c7/stm/forksupport.c b/c7/stm/forksupport.c --- a/c7/stm/forksupport.c +++ b/c7/stm/forksupport.c @@ -7,7 +7,11 @@ copy of all shared pages as soon as fork() is called. */ -static char *fork_big_copy; +static char *fork_big_copy = NULL; + +static char *setup_mmap(char *reason); /* forward, in setup.c */ +static void do_or_redo_setup_after_fork(void); /* forward, in setup.c */ +static void do_or_redo_teardown_after_fork(void); /* forward, in setup.c */ static void forksupport_prepare(void) @@ -58,7 +62,7 @@ assert(fork_big_copy != NULL); munmap(fork_big_copy, TOTAL_MEMORY); - stm_object_pages = NULL; + fork_big_copy = NULL; mutex_pages_unlock(); s_mutex_unlock(); @@ -69,15 +73,23 @@ if (stm_object_pages == NULL) return; + /* xxx the stm_thread_local_t belonging to other threads just leak. + Note that stm_all_thread_locals is preserved across a + stm_teardown/stm_setup sequence. */ + + mutex_pages_unlock(); + s_mutex_unlock(); + + do_or_redo_teardown_after_fork(); + + assert(fork_big_copy != NULL); + assert(stm_object_pages != NULL); mremap(fork_big_copy, TOTAL_MEMORY, TOTAL_MEMORY, MREMAP_MAYMOVE | MREMAP_FIXED, stm_object_pages); + fork_big_copy = NULL; - ...; reset carefully a much bigger part of the state here :-((( - memset(pages_privatized, 0, sizeof(pages_privatized)); - - mutex_pages_unlock(); - s_mutex_unlock(); + do_or_redo_setup_after_fork(); } @@ -88,7 +100,8 @@ if (!fork_support_ready) { int res = pthread_atfork(forksupport_prepare, forksupport_parent, forksupport_child); - assert(res == 0); + if (res != 0) + stm_fatalerror("pthread_atfork() failed: %m"); fork_support_ready = true; } } diff --git a/c7/stm/forksupport.h b/c7/stm/forksupport.h deleted file mode 100644 --- a/c7/stm/forksupport.h +++ /dev/null @@ -1,2 +0,0 @@ - -static void setup_forksupport(void); diff --git a/c7/stm/nursery.c b/c7/stm/nursery.c --- a/c7/stm/nursery.c +++ b/c7/stm/nursery.c @@ -32,10 +32,6 @@ } } -static void teardown_nursery(void) -{ -} - static inline bool _is_in_nursery(object_t *obj) { assert((uintptr_t)obj >= NURSERY_START); diff --git a/c7/stm/pages.c b/c7/stm/pages.c --- a/c7/stm/pages.c +++ b/c7/stm/pages.c @@ -25,6 +25,10 @@ static void teardown_pages(void) { memset(&pages_ctl, 0, sizeof(pages_ctl)); +} + +static void teardown_pages_1(void) +{ memset(pages_privatized, 0, sizeof(pages_privatized)); } diff --git a/c7/stm/setup.c b/c7/stm/setup.c --- a/c7/stm/setup.c +++ b/c7/stm/setup.c @@ -35,25 +35,8 @@ return result; } -void stm_setup(void) +static void do_or_redo_setup_after_fork(void) { - /* Check that some values are acceptable */ - assert(NB_SEGMENTS <= NB_SEGMENTS_MAX); - assert(4096 <= ((uintptr_t)STM_SEGMENT)); - assert((uintptr_t)STM_SEGMENT == (uintptr_t)STM_PSEGMENT); - assert(((uintptr_t)STM_PSEGMENT) + sizeof(*STM_PSEGMENT) <= 8192); - assert(2 <= FIRST_READMARKER_PAGE); - assert(FIRST_READMARKER_PAGE * 4096UL <= READMARKER_START); - assert(READMARKER_START < READMARKER_END); - assert(READMARKER_END <= 4096UL * FIRST_OBJECT_PAGE); - assert(FIRST_OBJECT_PAGE < NB_PAGES); - assert((NB_PAGES * 4096UL) >> 8 <= (FIRST_OBJECT_PAGE * 4096UL) >> 4); - assert((END_NURSERY_PAGE * 4096UL) >> 8 <= - (FIRST_READMARKER_PAGE * 4096UL)); - assert(_STM_FAST_ALLOC <= NB_NURSERY_PAGES * 4096); - - stm_object_pages = setup_mmap("initial stm_object_pages mmap()"); - long i; for (i = 1; i <= NB_SEGMENTS; i++) { char *segment_base = get_segment_base(i); @@ -90,20 +73,38 @@ so a null read marker means "not read" whatever the current transaction_read_version is. */ + setup_nursery(); +} + +void stm_setup(void) +{ + /* Check that some values are acceptable */ + assert(NB_SEGMENTS <= NB_SEGMENTS_MAX); + assert(4096 <= ((uintptr_t)STM_SEGMENT)); + assert((uintptr_t)STM_SEGMENT == (uintptr_t)STM_PSEGMENT); + assert(((uintptr_t)STM_PSEGMENT) + sizeof(*STM_PSEGMENT) <= 8192); + assert(2 <= FIRST_READMARKER_PAGE); + assert(FIRST_READMARKER_PAGE * 4096UL <= READMARKER_START); + assert(READMARKER_START < READMARKER_END); + assert(READMARKER_END <= 4096UL * FIRST_OBJECT_PAGE); + assert(FIRST_OBJECT_PAGE < NB_PAGES); + assert((NB_PAGES * 4096UL) >> 8 <= (FIRST_OBJECT_PAGE * 4096UL) >> 4); + assert((END_NURSERY_PAGE * 4096UL) >> 8 <= + (FIRST_READMARKER_PAGE * 4096UL)); + assert(_STM_FAST_ALLOC <= NB_NURSERY_PAGES * 4096); + + stm_object_pages = setup_mmap("initial stm_object_pages mmap()"); + + do_or_redo_setup_after_fork(); setup_sync(); - setup_nursery(); setup_gcpage(); setup_pages(); setup_forksupport(); } -void stm_teardown(void) +static void do_or_redo_teardown_after_fork(void) { - /* This function is called during testing, but normal programs don't - need to call it. */ - assert(!_has_mutex()); - long i; for (i = 1; i <= NB_SEGMENTS; i++) { struct stm_priv_segment_info_s *pr = get_priv_segment(i); @@ -117,13 +118,24 @@ tree_free(pr->callbacks_on_abort); } + teardown_core(); + teardown_sync_1(); + teardown_pages_1(); +} + +void stm_teardown(void) +{ + /* This function is called during testing, but normal programs don't + need to call it. */ + assert(!_has_mutex()); + + do_or_redo_teardown_after_fork(); + munmap(stm_object_pages, TOTAL_MEMORY); stm_object_pages = NULL; - teardown_core(); teardown_sync(); teardown_gcpage(); - teardown_nursery(); teardown_pages(); } @@ -174,6 +186,7 @@ void stm_unregister_thread_local(stm_thread_local_t *tl) { s_mutex_lock(); + assert(tl->prev != NULL); assert(tl->next != NULL); _done_shadow_stack(tl); if (tl == stm_all_thread_locals) { diff --git a/c7/stm/sync.c b/c7/stm/sync.c --- a/c7/stm/sync.c +++ b/c7/stm/sync.c @@ -59,7 +59,10 @@ if (pthread_cond_destroy(&sync_ctl.cond[i]) != 0) stm_fatalerror("cond destroy: %m\n"); } +} +static void teardown_sync_1(void) +{ memset(&sync_ctl, 0, sizeof(sync_ctl)); } diff --git a/c7/stmgc.c b/c7/stmgc.c --- a/c7/stmgc.c +++ b/c7/stmgc.c @@ -13,7 +13,6 @@ #include "stm/extra.h" #include "stm/fprintcolor.h" #include "stm/weakref.h" -#include "stm/forksupport.h" #include "stm/misc.c" #include "stm/list.c" @@ -24,6 +23,7 @@ #include "stm/largemalloc.c" #include "stm/nursery.c" #include "stm/sync.c" +#include "stm/forksupport.c" #include "stm/setup.c" #include "stm/hash_id.c" #include "stm/core.c" @@ -31,4 +31,3 @@ #include "stm/extra.c" #include "stm/fprintcolor.c" #include "stm/weakref.c" -#include "stm/forksupport.c" From noreply at buildbot.pypy.org Tue Mar 18 12:46:42 2014 From: noreply at buildbot.pypy.org (cfbolz) Date: Tue, 18 Mar 2014 12:46:42 +0100 (CET) Subject: [pypy-commit] lang-smalltalk default: make better names for the generated elidable funcs Message-ID: <20140318114642.1A61F1C0166@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: Changeset: r657:a9926a6cd014 Date: 2014-03-18 12:46 +0100 http://bitbucket.org/pypy/lang-smalltalk/changeset/a9926a6cd014/ Log: make better names for the generated elidable funcs diff --git a/spyvm/shadow.py b/spyvm/shadow.py --- a/spyvm/shadow.py +++ b/spyvm/shadow.py @@ -5,6 +5,8 @@ from rpython.rlib import rarithmetic, jit from rpython.rlib.objectmodel import import_from_mixin from rpython.rlib.debug import make_sure_not_resized + elidable_func.func_name = "elidable_" + func.func_name + meth.func_name = "elidable_meth_" + func.func_name class AbstractShadow(object): """A shadow is an optional extra bit of information that From noreply at buildbot.pypy.org Tue Mar 18 13:19:11 2014 From: noreply at buildbot.pypy.org (cfbolz) Date: Tue, 18 Mar 2014 13:19:11 +0100 (CET) Subject: [pypy-commit] lang-smalltalk default: gah, fix Message-ID: <20140318121911.49FCD1D2808@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: Changeset: r659:c3530864395f Date: 2014-03-18 13:19 +0100 http://bitbucket.org/pypy/lang-smalltalk/changeset/c3530864395f/ Log: gah, fix diff --git a/spyvm/shadow.py b/spyvm/shadow.py --- a/spyvm/shadow.py +++ b/spyvm/shadow.py @@ -5,8 +5,6 @@ from rpython.rlib import rarithmetic, jit from rpython.rlib.objectmodel import import_from_mixin from rpython.rlib.debug import make_sure_not_resized - elidable_func.func_name = "elidable_" + func.func_name - meth.func_name = "elidable_meth_" + func.func_name class AbstractShadow(object): """A shadow is an optional extra bit of information that diff --git a/spyvm/version.py b/spyvm/version.py --- a/spyvm/version.py +++ b/spyvm/version.py @@ -8,6 +8,8 @@ return func(self, *args) def meth(self, *args): return elidable_func(self, self.version, *args) + elidable_func.func_name = "elidable_" + func.func_name + meth.func_name = "elidable_meth_" + func.func_name return meth # In addition to marking the decorated function as "pure", both the receiver From noreply at buildbot.pypy.org Tue Mar 18 14:04:13 2014 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 18 Mar 2014 14:04:13 +0100 (CET) Subject: [pypy-commit] stmgc default: Don't put "\n" at the end of stm_fatalerror() messages (as it was Message-ID: <20140318130413.424BF1D2971@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1067:8aa49554dd50 Date: 2014-03-18 14:03 +0100 http://bitbucket.org/pypy/stmgc/changeset/8aa49554dd50/ Log: Don't put "\n" at the end of stm_fatalerror() messages (as it was already forgotten occasionally) diff --git a/c7/stm/core.c b/c7/stm/core.c --- a/c7/stm/core.c +++ b/c7/stm/core.c @@ -157,7 +157,7 @@ MAP_FIXED | MAP_PAGES_FLAGS, -1, 0) != readmarkers) { /* fall-back */ #if STM_TESTS - stm_fatalerror("reset_transaction_read_version: %m\n"); + stm_fatalerror("reset_transaction_read_version: %m"); #endif memset(readmarkers, 0, NB_READMARKER_PAGES * 4096UL); } diff --git a/c7/stm/fprintcolor.c b/c7/stm/fprintcolor.c --- a/c7/stm/fprintcolor.c +++ b/c7/stm/fprintcolor.c @@ -41,6 +41,7 @@ va_start(ap, format); vfprintf(stderr, format, ap); + fprintf(stderr, "\n"); va_end(ap); abort(); diff --git a/c7/stm/gcpage.c b/c7/stm/gcpage.c --- a/c7/stm/gcpage.c +++ b/c7/stm/gcpage.c @@ -60,7 +60,7 @@ return; out_of_memory: - stm_fatalerror("out of memory!\n"); /* XXX */ + stm_fatalerror("out of memory!"); /* XXX */ } static char *_allocate_small_slowpath(uint64_t size) @@ -85,7 +85,7 @@ /* Allocate the object with largemalloc.c from the lower addresses. */ char *addr = _stm_large_malloc(size); if (addr == NULL) - stm_fatalerror("not enough memory!\n"); + stm_fatalerror("not enough memory!"); if (addr + size > uninitialized_page_start) { uintptr_t npages; @@ -93,7 +93,7 @@ npages += GCPAGE_NUM_PAGES; if (uninitialized_page_stop - uninitialized_page_start < npages * 4096UL) { - stm_fatalerror("out of memory!\n"); /* XXX */ + stm_fatalerror("out of memory!"); /* XXX */ } setup_N_pages(uninitialized_page_start, npages); uninitialized_page_start += npages * 4096UL; diff --git a/c7/stm/list.c b/c7/stm/list.c --- a/c7/stm/list.c +++ b/c7/stm/list.c @@ -12,7 +12,7 @@ uintptr_t initial_allocation = 32; struct list_s *lst = malloc(LIST_SETSIZE(initial_allocation)); if (lst == NULL) - stm_fatalerror("out of memory in list_create\n"); /* XXX */ + stm_fatalerror("out of memory in list_create"); /* XXX */ lst->count = 0; lst->last_allocated = initial_allocation - 1; @@ -24,7 +24,7 @@ nalloc = LIST_OVERCNT(nalloc); lst = realloc(lst, LIST_SETSIZE(nalloc)); if (lst == NULL) - stm_fatalerror("out of memory in _list_grow\n"); /* XXX */ + stm_fatalerror("out of memory in _list_grow"); /* XXX */ lst->last_allocated = nalloc - 1; return lst; @@ -93,7 +93,7 @@ //fprintf(stderr, "growth: %ld\n", newalloc); char *newitems = malloc(newalloc); if (newitems == NULL) { - stm_fatalerror("out of memory!\n"); /* XXX */ + stm_fatalerror("out of memory!"); /* XXX */ } newtree.raw_start = newitems; newtree.raw_current = newitems; diff --git a/c7/stm/pages.c b/c7/stm/pages.c --- a/c7/stm/pages.c +++ b/c7/stm/pages.c @@ -93,7 +93,7 @@ int res = remap_file_pages(addr, size, 0, pgoff, 0); if (UNLIKELY(res < 0)) - stm_fatalerror("remap_file_pages: %m\n"); + stm_fatalerror("remap_file_pages: %m"); } static void pages_initialize_shared(uintptr_t pagenum, uintptr_t count) diff --git a/c7/stm/setup.c b/c7/stm/setup.c --- a/c7/stm/setup.c +++ b/c7/stm/setup.c @@ -24,7 +24,7 @@ PROT_READ | PROT_WRITE, MAP_PAGES_FLAGS, -1, 0); if (stm_object_pages == MAP_FAILED) - stm_fatalerror("initial stm_object_pages mmap() failed: %m\n"); + stm_fatalerror("initial stm_object_pages mmap() failed: %m"); /* The segment 0 is not used to run transactions, but to contain the shared copy of the pages. We mprotect all pages before so that diff --git a/c7/stm/sync.c b/c7/stm/sync.c --- a/c7/stm/sync.c +++ b/c7/stm/sync.c @@ -40,24 +40,24 @@ static void setup_sync(void) { if (pthread_mutex_init(&sync_ctl.global_mutex, NULL) != 0) - stm_fatalerror("mutex initialization: %m\n"); + stm_fatalerror("mutex initialization: %m"); long i; for (i = 0; i < _C_TOTAL; i++) { if (pthread_cond_init(&sync_ctl.cond[i], NULL) != 0) - stm_fatalerror("cond initialization: %m\n"); + stm_fatalerror("cond initialization: %m"); } } static void teardown_sync(void) { if (pthread_mutex_destroy(&sync_ctl.global_mutex) != 0) - stm_fatalerror("mutex destroy: %m\n"); + stm_fatalerror("mutex destroy: %m"); long i; for (i = 0; i < _C_TOTAL; i++) { if (pthread_cond_destroy(&sync_ctl.cond[i]) != 0) - stm_fatalerror("cond destroy: %m\n"); + stm_fatalerror("cond destroy: %m"); } memset(&sync_ctl, 0, sizeof(sync_ctl)); @@ -74,14 +74,14 @@ static void set_gs_register(char *value) { if (UNLIKELY(syscall(SYS_arch_prctl, ARCH_SET_GS, (uint64_t)value) != 0)) - stm_fatalerror("syscall(arch_prctl, ARCH_SET_GS): %m\n"); + stm_fatalerror("syscall(arch_prctl, ARCH_SET_GS): %m"); } static inline void s_mutex_lock(void) { assert(!_has_mutex_here); if (UNLIKELY(pthread_mutex_lock(&sync_ctl.global_mutex) != 0)) - stm_fatalerror("pthread_mutex_lock: %m\n"); + stm_fatalerror("pthread_mutex_lock: %m"); assert((_has_mutex_here = true, 1)); } @@ -89,32 +89,32 @@ { assert(_has_mutex_here); if (UNLIKELY(pthread_mutex_unlock(&sync_ctl.global_mutex) != 0)) - stm_fatalerror("pthread_mutex_unlock: %m\n"); + stm_fatalerror("pthread_mutex_unlock: %m"); assert((_has_mutex_here = false, 1)); } static inline void cond_wait(enum cond_type_e ctype) { #ifdef STM_NO_COND_WAIT - stm_fatalerror("*** cond_wait/%d called!\n", (int)ctype); + stm_fatalerror("*** cond_wait/%d called!", (int)ctype); #endif assert(_has_mutex_here); if (UNLIKELY(pthread_cond_wait(&sync_ctl.cond[ctype], &sync_ctl.global_mutex) != 0)) - stm_fatalerror("pthread_cond_wait/%d: %m\n", (int)ctype); + stm_fatalerror("pthread_cond_wait/%d: %m", (int)ctype); } static inline void cond_signal(enum cond_type_e ctype) { if (UNLIKELY(pthread_cond_signal(&sync_ctl.cond[ctype]) != 0)) - stm_fatalerror("pthread_cond_signal/%d: %m\n", (int)ctype); + stm_fatalerror("pthread_cond_signal/%d: %m", (int)ctype); } static inline void cond_broadcast(enum cond_type_e ctype) { if (UNLIKELY(pthread_cond_broadcast(&sync_ctl.cond[ctype]) != 0)) - stm_fatalerror("pthread_cond_broadcast/%d: %m\n", (int)ctype); + stm_fatalerror("pthread_cond_broadcast/%d: %m", (int)ctype); } /************************************************************/ From noreply at buildbot.pypy.org Tue Mar 18 14:04:14 2014 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 18 Mar 2014 14:04:14 +0100 (CET) Subject: [pypy-commit] stmgc c7-fork: hg merge default Message-ID: <20140318130414.689901D2972@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c7-fork Changeset: r1068:91b34e65d842 Date: 2014-03-18 14:04 +0100 http://bitbucket.org/pypy/stmgc/changeset/91b34e65d842/ Log: hg merge default diff --git a/c7/stm/core.c b/c7/stm/core.c --- a/c7/stm/core.c +++ b/c7/stm/core.c @@ -157,7 +157,7 @@ MAP_FIXED | MAP_PAGES_FLAGS, -1, 0) != readmarkers) { /* fall-back */ #if STM_TESTS - stm_fatalerror("reset_transaction_read_version: %m\n"); + stm_fatalerror("reset_transaction_read_version: %m"); #endif memset(readmarkers, 0, NB_READMARKER_PAGES * 4096UL); } diff --git a/c7/stm/fprintcolor.c b/c7/stm/fprintcolor.c --- a/c7/stm/fprintcolor.c +++ b/c7/stm/fprintcolor.c @@ -41,6 +41,7 @@ va_start(ap, format); vfprintf(stderr, format, ap); + fprintf(stderr, "\n"); va_end(ap); abort(); diff --git a/c7/stm/gcpage.c b/c7/stm/gcpage.c --- a/c7/stm/gcpage.c +++ b/c7/stm/gcpage.c @@ -60,7 +60,7 @@ return; out_of_memory: - stm_fatalerror("out of memory!\n"); /* XXX */ + stm_fatalerror("out of memory!"); /* XXX */ } static char *_allocate_small_slowpath(uint64_t size) @@ -85,7 +85,7 @@ /* Allocate the object with largemalloc.c from the lower addresses. */ char *addr = _stm_large_malloc(size); if (addr == NULL) - stm_fatalerror("not enough memory!\n"); + stm_fatalerror("not enough memory!"); if (addr + size > uninitialized_page_start) { uintptr_t npages; @@ -93,7 +93,7 @@ npages += GCPAGE_NUM_PAGES; if (uninitialized_page_stop - uninitialized_page_start < npages * 4096UL) { - stm_fatalerror("out of memory!\n"); /* XXX */ + stm_fatalerror("out of memory!"); /* XXX */ } setup_N_pages(uninitialized_page_start, npages); uninitialized_page_start += npages * 4096UL; diff --git a/c7/stm/list.c b/c7/stm/list.c --- a/c7/stm/list.c +++ b/c7/stm/list.c @@ -12,7 +12,7 @@ uintptr_t initial_allocation = 32; struct list_s *lst = malloc(LIST_SETSIZE(initial_allocation)); if (lst == NULL) - stm_fatalerror("out of memory in list_create\n"); /* XXX */ + stm_fatalerror("out of memory in list_create"); /* XXX */ lst->count = 0; lst->last_allocated = initial_allocation - 1; @@ -24,7 +24,7 @@ nalloc = LIST_OVERCNT(nalloc); lst = realloc(lst, LIST_SETSIZE(nalloc)); if (lst == NULL) - stm_fatalerror("out of memory in _list_grow\n"); /* XXX */ + stm_fatalerror("out of memory in _list_grow"); /* XXX */ lst->last_allocated = nalloc - 1; return lst; @@ -93,7 +93,7 @@ //fprintf(stderr, "growth: %ld\n", newalloc); char *newitems = malloc(newalloc); if (newitems == NULL) { - stm_fatalerror("out of memory!\n"); /* XXX */ + stm_fatalerror("out of memory!"); /* XXX */ } newtree.raw_start = newitems; newtree.raw_current = newitems; diff --git a/c7/stm/pages.c b/c7/stm/pages.c --- a/c7/stm/pages.c +++ b/c7/stm/pages.c @@ -97,7 +97,7 @@ int res = remap_file_pages(addr, size, 0, pgoff, 0); if (UNLIKELY(res < 0)) - stm_fatalerror("remap_file_pages: %m\n"); + stm_fatalerror("remap_file_pages: %m"); } static void pages_initialize_shared(uintptr_t pagenum, uintptr_t count) diff --git a/c7/stm/sync.c b/c7/stm/sync.c --- a/c7/stm/sync.c +++ b/c7/stm/sync.c @@ -40,24 +40,24 @@ static void setup_sync(void) { if (pthread_mutex_init(&sync_ctl.global_mutex, NULL) != 0) - stm_fatalerror("mutex initialization: %m\n"); + stm_fatalerror("mutex initialization: %m"); long i; for (i = 0; i < _C_TOTAL; i++) { if (pthread_cond_init(&sync_ctl.cond[i], NULL) != 0) - stm_fatalerror("cond initialization: %m\n"); + stm_fatalerror("cond initialization: %m"); } } static void teardown_sync(void) { if (pthread_mutex_destroy(&sync_ctl.global_mutex) != 0) - stm_fatalerror("mutex destroy: %m\n"); + stm_fatalerror("mutex destroy: %m"); long i; for (i = 0; i < _C_TOTAL; i++) { if (pthread_cond_destroy(&sync_ctl.cond[i]) != 0) - stm_fatalerror("cond destroy: %m\n"); + stm_fatalerror("cond destroy: %m"); } } @@ -77,14 +77,14 @@ static void set_gs_register(char *value) { if (UNLIKELY(syscall(SYS_arch_prctl, ARCH_SET_GS, (uint64_t)value) != 0)) - stm_fatalerror("syscall(arch_prctl, ARCH_SET_GS): %m\n"); + stm_fatalerror("syscall(arch_prctl, ARCH_SET_GS): %m"); } static inline void s_mutex_lock(void) { assert(!_has_mutex_here); if (UNLIKELY(pthread_mutex_lock(&sync_ctl.global_mutex) != 0)) - stm_fatalerror("pthread_mutex_lock: %m\n"); + stm_fatalerror("pthread_mutex_lock: %m"); assert((_has_mutex_here = true, 1)); } @@ -92,32 +92,32 @@ { assert(_has_mutex_here); if (UNLIKELY(pthread_mutex_unlock(&sync_ctl.global_mutex) != 0)) - stm_fatalerror("pthread_mutex_unlock: %m\n"); + stm_fatalerror("pthread_mutex_unlock: %m"); assert((_has_mutex_here = false, 1)); } static inline void cond_wait(enum cond_type_e ctype) { #ifdef STM_NO_COND_WAIT - stm_fatalerror("*** cond_wait/%d called!\n", (int)ctype); + stm_fatalerror("*** cond_wait/%d called!", (int)ctype); #endif assert(_has_mutex_here); if (UNLIKELY(pthread_cond_wait(&sync_ctl.cond[ctype], &sync_ctl.global_mutex) != 0)) - stm_fatalerror("pthread_cond_wait/%d: %m\n", (int)ctype); + stm_fatalerror("pthread_cond_wait/%d: %m", (int)ctype); } static inline void cond_signal(enum cond_type_e ctype) { if (UNLIKELY(pthread_cond_signal(&sync_ctl.cond[ctype]) != 0)) - stm_fatalerror("pthread_cond_signal/%d: %m\n", (int)ctype); + stm_fatalerror("pthread_cond_signal/%d: %m", (int)ctype); } static inline void cond_broadcast(enum cond_type_e ctype) { if (UNLIKELY(pthread_cond_broadcast(&sync_ctl.cond[ctype]) != 0)) - stm_fatalerror("pthread_cond_broadcast/%d: %m\n", (int)ctype); + stm_fatalerror("pthread_cond_broadcast/%d: %m", (int)ctype); } /************************************************************/ From noreply at buildbot.pypy.org Tue Mar 18 14:06:18 2014 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 18 Mar 2014 14:06:18 +0100 (CET) Subject: [pypy-commit] stmgc c7-fork: progress Message-ID: <20140318130618.41FFC1D297A@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c7-fork Changeset: r1069:a1db46a027b5 Date: 2014-03-18 14:06 +0100 http://bitbucket.org/pypy/stmgc/changeset/a1db46a027b5/ Log: progress diff --git a/c7/stm/forksupport.c b/c7/stm/forksupport.c --- a/c7/stm/forksupport.c +++ b/c7/stm/forksupport.c @@ -8,10 +8,12 @@ static char *fork_big_copy = NULL; +static stm_thread_local_t *fork_this_tl; static char *setup_mmap(char *reason); /* forward, in setup.c */ static void do_or_redo_setup_after_fork(void); /* forward, in setup.c */ static void do_or_redo_teardown_after_fork(void); /* forward, in setup.c */ +static pthread_t *_get_cpth(stm_thread_local_t *);/* forward, in setup.c */ static void forksupport_prepare(void) @@ -19,15 +21,35 @@ if (stm_object_pages == NULL) return; - /* This silently assumes that fork() is not called from transactions. - It's hard to check though... - */ + /* This assumes that fork() is not called from transactions. + So far we attempt to check this by walking all stm_thread_local_t, + marking the one from the current thread, and verifying that it's not + running a transaction. This assumes that the stm_thread_local_t is just + a __thread variable, so never changes threads. + */ s_mutex_lock(); synchronize_all_threads(); mutex_pages_lock(); + fork_this_tl = NULL; + stm_thread_local_t *tl = stm_all_thread_locals; + do { + if (pthread_equal(*_get_cpth(tl), pthread_self())) { + if (_stm_in_transaction(tl)) + stm_fatalerror("fork(): cannot be used inside a transaction"); + if (fork_this_tl != NULL) + stm_fatalerror("fork(): found several stm_thread_local_t" + " from the same thread"); + fork_this_tl = tl; + } + tl = tl->next; + } while (tl != stm_all_thread_locals); + + if (fork_this_tl == NULL) + stm_fatalerror("fork(): found no stm_thread_local_t from this thread"); + char *big_copy = setup_mmap("stmgc's fork support"); uintptr_t pagenum, endpagenum; @@ -80,6 +102,15 @@ mutex_pages_unlock(); s_mutex_unlock(); + stm_thread_local_t *tl = stm_all_thread_locals; + do { + stm_thread_local_t *nexttl = tl->next; + if (tl != fork_this_tl) { + stm_unregister_thread_local(tl); + } + tl = nexttl; + } while (tl != stm_all_thread_locals); + do_or_redo_teardown_after_fork(); assert(fork_big_copy != NULL); diff --git a/c7/stm/setup.c b/c7/stm/setup.c --- a/c7/stm/setup.c +++ b/c7/stm/setup.c @@ -139,7 +139,7 @@ teardown_pages(); } -void _init_shadow_stack(stm_thread_local_t *tl) +static void _init_shadow_stack(stm_thread_local_t *tl) { struct stm_shadowentry_s *s = (struct stm_shadowentry_s *) malloc(SHADOW_STACK_SIZE * sizeof(struct stm_shadowentry_s)); @@ -148,13 +148,18 @@ tl->shadowstack_base = s; } -void _done_shadow_stack(stm_thread_local_t *tl) +static void _done_shadow_stack(stm_thread_local_t *tl) { free(tl->shadowstack_base); tl->shadowstack = NULL; tl->shadowstack_base = NULL; } +static pthread_t *_get_cpth(stm_thread_local_t *tl) +{ + assert(sizeof(pthread_t) <= sizeof(tl->creating_pthread)); + return (pthread_t *)(tl->creating_pthread); +} void stm_register_thread_local(stm_thread_local_t *tl) { @@ -178,6 +183,7 @@ numbers automatically. */ num = (num % NB_SEGMENTS) + 1; tl->associated_segment_num = num; + *_get_cpth(tl) = pthread_self(); _init_shadow_stack(tl); set_gs_register(get_segment_base(num)); s_mutex_unlock(); diff --git a/c7/stmgc.h b/c7/stmgc.h --- a/c7/stmgc.h +++ b/c7/stmgc.h @@ -69,6 +69,7 @@ /* the next fields are handled internally by the library */ int associated_segment_num; struct stm_thread_local_s *prev, *next; + long creating_pthread[4]; } stm_thread_local_t; /* this should use llvm's coldcc calling convention, From noreply at buildbot.pypy.org Tue Mar 18 15:31:41 2014 From: noreply at buildbot.pypy.org (timfel) Date: Tue, 18 Mar 2014 15:31:41 +0100 (CET) Subject: [pypy-commit] lang-smalltalk strategies: change jit.py so we can just run code Message-ID: <20140318143141.B523A1C0A66@cobra.cs.uni-duesseldorf.de> Author: Tim Felgentreff Branch: strategies Changeset: r660:ae6d13449b51 Date: 2014-03-18 13:10 +0100 http://bitbucket.org/pypy/lang-smalltalk/changeset/ae6d13449b51/ Log: change jit.py so we can just run code diff --git a/spyvm/test/jit.py b/spyvm/test/jit.py --- a/spyvm/test/jit.py +++ b/spyvm/test/jit.py @@ -3,6 +3,10 @@ # # view jit. # +Code = """ +(1 to: 2000) asOrderedCollection +""" + import sys, os from rpython import conftest @@ -52,15 +56,34 @@ class TestLLtype(LLJitMixin): def test_miniloop(self): - + import time from spyvm import objspace space = objspace.ObjSpace() image = create_testimage(space) interp = interpreter.Interpreter(space, image) - w_selector = interp.perform(space.wrap_string('loopTest2'), "asSymbol") + + selector = "codeTest%d" % int(time.time()) + try: + w_result = interp.perform( + interp.space.w_SmallInteger, + "compile:classified:notifying:", + space.wrap_string("%s\r\n%s" % (selector, Code)), + space.wrap_string("spy-run-code"), + space.w_nil + ) + except interpreter.ReturnFromTopLevel, e: + print e.object + return 1 + except error.Exit, e: + print e.msg + return 1 + + w_selector = interp.perform(space.wrap_string(selector), "asSymbol") assert isinstance(w_selector, model.W_BytesObject) def interp_w(): - interp.perform(model.W_SmallInteger(1000), w_selector) + interp.perform(space.wrap_int(0), w_selector) + self.meta_interp(interp_w, [], listcomp=True, listops=True, backendopt=True, inline=True) - self.meta_interp(interp_w, [], listcomp=True, listops=True, backendopt=True, inline=True) +if __name__ == "__main__": + TestLLtype().test_miniloop() From noreply at buildbot.pypy.org Tue Mar 18 15:31:42 2014 From: noreply at buildbot.pypy.org (timfel) Date: Tue, 18 Mar 2014 15:31:42 +0100 (CET) Subject: [pypy-commit] lang-smalltalk strategies: merge default Message-ID: <20140318143142.DB6411C0A66@cobra.cs.uni-duesseldorf.de> Author: Tim Felgentreff Branch: strategies Changeset: r661:20795bd19094 Date: 2014-03-18 13:11 +0100 http://bitbucket.org/pypy/lang-smalltalk/changeset/20795bd19094/ Log: merge default diff --git a/spyvm/shadow.py b/spyvm/shadow.py --- a/spyvm/shadow.py +++ b/spyvm/shadow.py @@ -5,6 +5,8 @@ from rpython.rlib import rarithmetic, jit from rpython.rlib.objectmodel import import_from_mixin from rpython.rlib.debug import make_sure_not_resized + elidable_func.func_name = "elidable_" + func.func_name + meth.func_name = "elidable_meth_" + func.func_name class AbstractShadow(object): """A shadow is an optional extra bit of information that From noreply at buildbot.pypy.org Tue Mar 18 15:31:44 2014 From: noreply at buildbot.pypy.org (timfel) Date: Tue, 18 Mar 2014 15:31:44 +0100 (CET) Subject: [pypy-commit] lang-smalltalk strategies: merge tip Message-ID: <20140318143144.113121C0A66@cobra.cs.uni-duesseldorf.de> Author: Tim Felgentreff Branch: strategies Changeset: r662:28eef032af8c Date: 2014-03-18 15:28 +0100 http://bitbucket.org/pypy/lang-smalltalk/changeset/28eef032af8c/ Log: merge tip diff --git a/spyvm/shadow.py b/spyvm/shadow.py --- a/spyvm/shadow.py +++ b/spyvm/shadow.py @@ -5,8 +5,6 @@ from rpython.rlib import rarithmetic, jit from rpython.rlib.objectmodel import import_from_mixin from rpython.rlib.debug import make_sure_not_resized - elidable_func.func_name = "elidable_" + func.func_name - meth.func_name = "elidable_meth_" + func.func_name class AbstractShadow(object): """A shadow is an optional extra bit of information that diff --git a/spyvm/version.py b/spyvm/version.py --- a/spyvm/version.py +++ b/spyvm/version.py @@ -8,6 +8,8 @@ return func(self, *args) def meth(self, *args): return elidable_func(self, self.version, *args) + elidable_func.func_name = "elidable_" + func.func_name + meth.func_name = "elidable_meth_" + func.func_name return meth # In addition to marking the decorated function as "pure", both the receiver From noreply at buildbot.pypy.org Tue Mar 18 15:31:45 2014 From: noreply at buildbot.pypy.org (timfel) Date: Tue, 18 Mar 2014 15:31:45 +0100 (CET) Subject: [pypy-commit] lang-smalltalk strategies: merge upstream Message-ID: <20140318143145.2D6EA1C0A66@cobra.cs.uni-duesseldorf.de> Author: Tim Felgentreff Branch: strategies Changeset: r663:072f88561376 Date: 2014-03-18 15:29 +0100 http://bitbucket.org/pypy/lang-smalltalk/changeset/072f88561376/ Log: merge upstream diff --git a/spyvm/model.py b/spyvm/model.py --- a/spyvm/model.py +++ b/spyvm/model.py @@ -485,11 +485,15 @@ class W_AbstractPointersObject(W_AbstractObjectWithClassReference): """Common object.""" - _attrs_ = ['shadow', 'version'] - import_from_mixin(version.VersionMixin) + _attrs_ = ['shadow'] shadow = None # Default value + def changed(self): + # This is called whenever an instance variable is changed on the receiver. + # Was used with a version variable before. Left here in case it might be usefull in the future. + pass + @jit.unroll_safe def __init__(self, space, w_class, size): """Create new object with size = fixed + variable size.""" @@ -537,7 +541,6 @@ assert self.shadow is None or self.shadow is shadow self.shadow = shadow - @elidable_for_version def _get_shadow(self): return self.shadow @@ -680,11 +683,9 @@ def set_storage(self, storage): self._storage = storage - @elidable_for_version def get_storage(self): return self._storage - @elidable_for_version def get_strategy(self): return self.strategy From noreply at buildbot.pypy.org Tue Mar 18 15:43:20 2014 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 18 Mar 2014 15:43:20 +0100 (CET) Subject: [pypy-commit] stmgc c7-fork: in-progress Message-ID: <20140318144320.5B4421C0166@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c7-fork Changeset: r1070:bef9be522fb9 Date: 2014-03-18 15:43 +0100 http://bitbucket.org/pypy/stmgc/changeset/bef9be522fb9/ Log: in-progress diff --git a/c7/demo/demo_random.c b/c7/demo/demo_random.c --- a/c7/demo/demo_random.c +++ b/c7/demo/demo_random.c @@ -4,20 +4,24 @@ #include #include #include +#include +#include #include "stmgc.h" #define NUMTHREADS 3 -#define STEPS_PER_THREAD 500 -#define THREAD_STARTS 1000 // how many restarts of threads +#define STEPS_PER_THREAD 300 +#define THREAD_STARTS 300 // how many restarts of threads #define PREBUILT_ROOTS 3 #define MAXROOTS 1000 +#define FORKS 3 // SUPPORT struct node_s; typedef TLPREFIX struct node_s node_t; typedef node_t* nodeptr_t; typedef object_t* objptr_t; +int num_forked_children = 0; struct node_s { struct object_s hdr; @@ -337,6 +341,22 @@ push_roots(); stm_commit_transaction(); + if (arg) { + printf("========== FORK =========\n"); + arg = NULL; + pid_t child = fork(); + printf("=== in process %d thread %lx, fork() returned %d\n", + (int)getpid(), (long)pthread_self(), (int)child); + if (child == -1) { + fprintf(stderr, "fork() error: %m\n"); + abort(); + } + if (child != 0) + num_forked_children++; + else + num_forked_children = 0; + } + td.num_roots_at_transaction_start = td.num_roots; if (get_rand(100) < 98) { @@ -427,8 +447,24 @@ assert(status == 0); printf("thread finished\n"); if (thread_starts) { + long forkbase = NUMTHREADS * THREAD_STARTS / (FORKS + 1); + long _fork = (thread_starts % forkbase) == 0; thread_starts--; - newthread(demo_random, NULL); + newthread(demo_random, (void *)_fork); + } + } + + for (i = 0; i < num_forked_children; i++) { + pid_t child = wait(&status); + if (child == -1) + perror("wait"); + printf("From %d: child %d terminated with exit status %d\n", + (int)getpid(), (int)child, status); + if (WIFEXITED(status) && WEXITSTATUS(status) == 0) + ; + else { + printf("*** error from the child ***\n"); + return 1; } } diff --git a/c7/stm/forksupport.c b/c7/stm/forksupport.c --- a/c7/stm/forksupport.c +++ b/c7/stm/forksupport.c @@ -28,10 +28,9 @@ a __thread variable, so never changes threads. */ s_mutex_lock(); + mutex_pages_lock(); - synchronize_all_threads(); - - mutex_pages_lock(); + dprintf(("forksupport_prepare: synchronized all threads\n")); fork_this_tl = NULL; stm_thread_local_t *tl = stm_all_thread_locals; @@ -50,11 +49,29 @@ if (fork_this_tl == NULL) stm_fatalerror("fork(): found no stm_thread_local_t from this thread"); + /* Make a new mmap at some other address, but of the same size as + the standard mmap at stm_object_pages + */ char *big_copy = setup_mmap("stmgc's fork support"); + /* Copy each of the segment infos into the new mmap + */ + long i; + for (i = 1; i <= NB_SEGMENTS; i++) { + struct stm_priv_segment_info_s *src = get_priv_segment(i); + char *dst = big_copy + (((char *)src) - stm_object_pages); + *(struct stm_priv_segment_info_s *)dst = *src; + } + + /* Copy all the data from the two ranges of objects (large, small) + into the new mmap --- but only the shared objects + */ uintptr_t pagenum, endpagenum; pagenum = END_NURSERY_PAGE; /* starts after the nursery */ endpagenum = (uninitialized_page_start - stm_object_pages) / 4096UL; + if (endpagenum < NB_PAGES) + endpagenum++; /* the next page too, because it might contain + data from largemalloc */ while (1) { if (UNLIKELY(pagenum == endpagenum)) { @@ -63,9 +80,9 @@ if (endpagenum == NB_PAGES) break; /* done */ pagenum = (uninitialized_page_stop - stm_object_pages) / 4096UL; + pagenum--; /* the prev page too, because it does contain + data from largemalloc */ endpagenum = NB_PAGES; - if (pagenum == endpagenum) - break; /* no pages in the 2nd section, so done too */ } pagecopy(big_copy + pagenum * 4096UL, @@ -82,10 +99,15 @@ if (stm_object_pages == NULL) return; + /* In the parent, after fork(), we can simply forget about the big copy + that we made for the child. + */ assert(fork_big_copy != NULL); munmap(fork_big_copy, TOTAL_MEMORY); fork_big_copy = NULL; + dprintf(("forksupport_parent: continuing to run\n")); + mutex_pages_unlock(); s_mutex_unlock(); } @@ -95,32 +117,62 @@ if (stm_object_pages == NULL) return; - /* xxx the stm_thread_local_t belonging to other threads just leak. - Note that stm_all_thread_locals is preserved across a - stm_teardown/stm_setup sequence. */ - + /* In the child, first unregister all other stm_thread_local_t, + mostly as a way to free the memory used by the shadowstacks + */ mutex_pages_unlock(); s_mutex_unlock(); - stm_thread_local_t *tl = stm_all_thread_locals; - do { - stm_thread_local_t *nexttl = tl->next; - if (tl != fork_this_tl) { - stm_unregister_thread_local(tl); - } - tl = nexttl; - } while (tl != stm_all_thread_locals); + assert(fork_this_tl != NULL); + while (stm_all_thread_locals->next != stm_all_thread_locals) { + if (stm_all_thread_locals == fork_this_tl) + stm_unregister_thread_local(stm_all_thread_locals->next); + else + stm_unregister_thread_local(stm_all_thread_locals); + } + assert(stm_all_thread_locals == fork_this_tl); - do_or_redo_teardown_after_fork(); + /* Restore a few things in the child: the new pthread_self(), and + the %gs register (although I suppose it should be preserved by + fork()) + */ + *_get_cpth(fork_this_tl) = pthread_self(); + set_gs_register(get_segment_base(fork_this_tl->associated_segment_num)); + fork_this_tl = NULL; + /* Move the copy of the mmap over the old one, overwriting it + and thus freeing the old mapping in this process + */ assert(fork_big_copy != NULL); assert(stm_object_pages != NULL); - mremap(fork_big_copy, TOTAL_MEMORY, TOTAL_MEMORY, - MREMAP_MAYMOVE | MREMAP_FIXED, - stm_object_pages); + void *res = mremap(fork_big_copy, TOTAL_MEMORY, TOTAL_MEMORY, + MREMAP_MAYMOVE | MREMAP_FIXED, + stm_object_pages); + if (res != stm_object_pages) + stm_fatalerror("after fork: mremap failed: %m"); fork_big_copy = NULL; + /* Call a subset of stm_teardown() / stm_setup() to free and + recreate the necessary data in all segments, and to clean up some + of the global data like the big arrays that don't make sense any + more. We keep other things like the smallmalloc and largemalloc + internal state. + */ + do_or_redo_teardown_after_fork(); do_or_redo_setup_after_fork(); + + /* Make all pages shared again. + */ + mutex_pages_lock(); + uintptr_t start = END_NURSERY_PAGE; + uintptr_t stop = (uninitialized_page_start - stm_object_pages) / 4096UL; + pages_initialize_shared(start, stop - start); + start = (uninitialized_page_stop - stm_object_pages) / 4096UL; + stop = NB_PAGES; + pages_initialize_shared(start, stop - start); + mutex_pages_unlock(); + + dprintf(("forksupport_child: running one thread now\n")); } diff --git a/c7/stm/fprintcolor.c b/c7/stm/fprintcolor.c --- a/c7/stm/fprintcolor.c +++ b/c7/stm/fprintcolor.c @@ -8,8 +8,8 @@ char buffer[2048]; va_list ap; int result; - int size = (int)sprintf(buffer, "\033[%dm[%lx] ", dprintfcolor(), - (long)pthread_self()); + int size = (int)sprintf(buffer, "\033[%dm[%d,%lx] ", dprintfcolor(), + (int)getpid(), (long)pthread_self()); assert(size >= 0); va_start(ap, format); diff --git a/c7/stm/largemalloc.c b/c7/stm/largemalloc.c --- a/c7/stm/largemalloc.c +++ b/c7/stm/largemalloc.c @@ -273,8 +273,10 @@ /* unlink the following chunk */ mscan->d.next->prev = mscan->d.prev; mscan->d.prev->next = mscan->d.next; - assert((mscan->prev_size = (size_t)-258, 1)); /* 0xfffffffffffffefe */ - assert((mscan->size = (size_t)-515, 1)); /* 0xfffffffffffffdfd */ +#ifndef NDEBUG + mscan->prev_size = (size_t)-258; /* 0xfffffffffffffefe */ + mscan->size = (size_t)-515; /* 0xfffffffffffffdfd */ +#endif /* merge the two chunks */ assert(fsize == fscan->prev_size); diff --git a/c7/stm/pages.c b/c7/stm/pages.c --- a/c7/stm/pages.c +++ b/c7/stm/pages.c @@ -107,6 +107,8 @@ segment 0. */ uintptr_t i; assert(_has_mutex_pages()); + if (count == 0) + return; for (i = 1; i <= NB_SEGMENTS; i++) { char *segment_base = get_segment_base(i); d_remap_file_pages(segment_base + pagenum * 4096UL, diff --git a/c7/stm/setup.c b/c7/stm/setup.c --- a/c7/stm/setup.c +++ b/c7/stm/setup.c @@ -11,41 +11,36 @@ if (result == MAP_FAILED) stm_fatalerror("%s failed: %m\n", reason); + return result; +} + +static void do_or_redo_setup_after_fork(void) +{ /* The segment 0 is not used to run transactions, but contains the shared copy of the pages. We mprotect all pages before so that accesses fail, up to and including the pages corresponding to the nurseries of the other segments. */ - mprotect(result, END_NURSERY_PAGE * 4096UL, PROT_NONE); + mprotect(stm_object_pages, END_NURSERY_PAGE * 4096UL, PROT_NONE); long i; for (i = 1; i <= NB_SEGMENTS; i++) { - char *segment_base = result + i * (NB_PAGES * 4096UL); + char *segment_base = get_segment_base(i); /* In each segment, the first page is where TLPREFIX'ed NULL accesses land. We mprotect it so that accesses fail. */ mprotect(segment_base, 4096, PROT_NONE); + /* Fill the TLS page (page 1) with 0xDC, for debugging */ + memset(REAL_ADDRESS(segment_base, 4096), 0xDC, 4096); + /* Make a "hole" at STM_PSEGMENT (which includes STM_SEGMENT) */ + memset(REAL_ADDRESS(segment_base, STM_PSEGMENT), 0, + sizeof(*STM_PSEGMENT)); + /* Pages in range(2, FIRST_READMARKER_PAGE) are never used */ if (FIRST_READMARKER_PAGE > 2) mprotect(segment_base + 8192, (FIRST_READMARKER_PAGE - 2) * 4096UL, PROT_NONE); - } - - return result; -} - -static void do_or_redo_setup_after_fork(void) -{ - long i; - for (i = 1; i <= NB_SEGMENTS; i++) { - char *segment_base = get_segment_base(i); - - /* Fill the TLS page (page 1) with 0xDC, for debugging */ - memset(REAL_ADDRESS(segment_base, 4096), 0xDC, 4096); - /* Make a "hole" at STM_PSEGMENT (which includes STM_SEGMENT) */ - memset(REAL_ADDRESS(segment_base, STM_PSEGMENT), 0, - sizeof(*STM_PSEGMENT)); /* Initialize STM_PSEGMENT */ struct stm_priv_segment_info_s *pr = get_priv_segment(i); @@ -108,8 +103,8 @@ long i; for (i = 1; i <= NB_SEGMENTS; i++) { struct stm_priv_segment_info_s *pr = get_priv_segment(i); - assert(pr->objects_pointing_to_nursery == NULL); - assert(pr->large_overflow_objects == NULL); + LIST_FREE(pr->objects_pointing_to_nursery); + LIST_FREE(pr->large_overflow_objects); list_free(pr->modified_old_objects); list_free(pr->young_weakrefs); list_free(pr->old_weakrefs); From noreply at buildbot.pypy.org Tue Mar 18 15:46:40 2014 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 18 Mar 2014 15:46:40 +0100 (CET) Subject: [pypy-commit] stmgc c7-fork: Reset this state after fork too Message-ID: <20140318144640.1C9D91C0166@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c7-fork Changeset: r1071:2c4acbf96895 Date: 2014-03-18 15:46 +0100 http://bitbucket.org/pypy/stmgc/changeset/2c4acbf96895/ Log: Reset this state after fork too diff --git a/c7/stm/sync.c b/c7/stm/sync.c --- a/c7/stm/sync.c +++ b/c7/stm/sync.c @@ -36,6 +36,10 @@ char reserved[192]; } sync_ctl __attribute__((aligned(64))); +#ifndef NDEBUG +static bool _safe_points_requested = false; +#endif + static void setup_sync(void) { @@ -64,6 +68,10 @@ static void teardown_sync_1(void) { memset(&sync_ctl, 0, sizeof(sync_ctl)); +#ifndef NDEBUG + _safe_points_requested = false; +#endif + pause_signalled = false; } #ifndef NDEBUG @@ -255,10 +263,6 @@ /************************************************************/ -#ifndef NDEBUG -static bool _safe_points_requested = false; -#endif - static void signal_everybody_to_pause_running(void) { assert(_safe_points_requested == false); From noreply at buildbot.pypy.org Tue Mar 18 16:48:22 2014 From: noreply at buildbot.pypy.org (Manuel Jacob) Date: Tue, 18 Mar 2014 16:48:22 +0100 (CET) Subject: [pypy-commit] pypy refactor-buffer-api: Close to-be-merged branch. Message-ID: <20140318154822.9BB6A1D26D3@cobra.cs.uni-duesseldorf.de> Author: Manuel Jacob Branch: refactor-buffer-api Changeset: r70052:0ae760fa2735 Date: 2014-03-18 16:21 +0100 http://bitbucket.org/pypy/pypy/changeset/0ae760fa2735/ Log: Close to-be-merged branch. From noreply at buildbot.pypy.org Tue Mar 18 16:48:24 2014 From: noreply at buildbot.pypy.org (Manuel Jacob) Date: Tue, 18 Mar 2014 16:48:24 +0100 (CET) Subject: [pypy-commit] pypy default: hg merge refactor-buffer-api Message-ID: <20140318154824.A82851D26D3@cobra.cs.uni-duesseldorf.de> Author: Manuel Jacob Branch: Changeset: r70053:5cff90211004 Date: 2014-03-18 16:41 +0100 http://bitbucket.org/pypy/pypy/changeset/5cff90211004/ Log: hg merge refactor-buffer-api Separate the interp-level buffer API from the buffer type exposed to app-level. The `Buffer` class is now used by `W_MemoryView` and `W_Buffer`, which is not present in Python 3. Previously `W_Buffer` was an alias to `Buffer`, which was wrappable itself. diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -194,6 +194,15 @@ def immutable_unique_id(self, space): return None + def buffer_w(self, space): + w_impl = space.lookup(self, '__buffer__') + if w_impl is not None: + w_result = space.get_and_call_function(w_impl, self) + from pypy.module.__builtin__.interp_memoryview import W_Buffer + if isinstance(w_result, W_Buffer): + return w_result.buf + self._typed_unwrap_error(space, "buffer") + def str_w(self, space): self._typed_unwrap_error(space, "string") @@ -1314,10 +1323,7 @@ 'to unsigned int')) def buffer_w(self, w_obj): - # returns a Buffer instance - from pypy.interpreter.buffer import Buffer - w_buffer = self.buffer(w_obj) - return self.interp_w(Buffer, w_buffer) + return w_obj.buffer_w(self) def rwbuffer_w(self, w_obj): # returns a RWBuffer instance @@ -1677,7 +1683,6 @@ ('set', 'set', 3, ['__set__']), ('delete', 'delete', 2, ['__delete__']), ('userdel', 'del', 1, ['__del__']), - ('buffer', 'buffer', 1, ['__buffer__']), # see buffer.py ] ObjSpace.BuiltinModuleTable = [ diff --git a/pypy/interpreter/buffer.py b/pypy/interpreter/buffer.py --- a/pypy/interpreter/buffer.py +++ b/pypy/interpreter/buffer.py @@ -1,30 +1,12 @@ """ Buffer protocol support. """ +from pypy.interpreter.error import OperationError +from rpython.rlib.objectmodel import import_from_mixin -# The implementation of the buffer protocol. The basic idea is that we -# can ask any app-level object for a 'buffer' view on it, by calling its -# __buffer__() special method. It should return a wrapped instance of a -# subclass of the Buffer class defined below. Note that __buffer__() is -# a PyPy-only extension to the Python language, made necessary by the -# fact that it's not natural in PyPy to hack an interp-level-only -# interface. -# In normal usage, the convenience method space.buffer_w() should be -# used to get directly a Buffer instance. Doing so also gives you for -# free the typecheck that __buffer__() really returned a wrapped Buffer. - -import operator -from pypy.interpreter.baseobjspace import W_Root -from pypy.interpreter.typedef import TypeDef -from pypy.interpreter.gateway import interp2app, unwrap_spec -from pypy.interpreter.error import OperationError -from rpython.rlib.objectmodel import compute_hash, import_from_mixin -from rpython.rlib.rstring import StringBuilder - - -class Buffer(W_Root): - """Abstract base class for memory views.""" +class Buffer(object): + """Abstract base class for buffers.""" __slots__ = () # no extra slot here @@ -47,94 +29,12 @@ def get_raw_address(self): raise ValueError("no raw buffer") + def is_writable(self): return False - # __________ app-level support __________ - - def descr_len(self, space): - return space.wrap(self.getlength()) - - def descr_getitem(self, space, w_index): - start, stop, step, size = space.decode_index4(w_index, self.getlength()) - if step == 0: # index only - return space.wrap(self.getitem(start)) - res = self.getslice(start, stop, step, size) - return space.wrap(res) - - @unwrap_spec(newstring='bufferstr') - def descr_setitem(self, space, w_index, newstring): - if not isinstance(self, RWBuffer): - raise OperationError(space.w_TypeError, - space.wrap("buffer is read-only")) - start, stop, step, size = space.decode_index4(w_index, self.getlength()) - if step == 0: # index only - if len(newstring) != 1: - msg = 'buffer[index]=x: x must be a single character' - raise OperationError(space.w_TypeError, space.wrap(msg)) - char = newstring[0] # annotator hint - self.setitem(start, char) - elif step == 1: - if len(newstring) != size: - msg = "right operand length must match slice length" - raise OperationError(space.w_ValueError, space.wrap(msg)) - self.setslice(start, newstring) - else: - raise OperationError(space.w_ValueError, - space.wrap("buffer object does not support" - " slicing with a step")) - - def descr__buffer__(self, space): - return space.wrap(self) - - def descr_str(self, space): - return space.wrap(self.as_str()) - - @unwrap_spec(other='bufferstr') - def descr_add(self, space, other): - return space.wrap(self.as_str() + other) - - def _make_descr__cmp(name): - def descr__cmp(self, space, w_other): - if not isinstance(w_other, Buffer): - return space.w_NotImplemented - # xxx not the most efficient implementation - str1 = self.as_str() - str2 = w_other.as_str() - return space.wrap(getattr(operator, name)(str1, str2)) - descr__cmp.func_name = name - return descr__cmp - - descr_eq = _make_descr__cmp('eq') - descr_ne = _make_descr__cmp('ne') - descr_lt = _make_descr__cmp('lt') - descr_le = _make_descr__cmp('le') - descr_gt = _make_descr__cmp('gt') - descr_ge = _make_descr__cmp('ge') - - def descr_hash(self, space): - return space.wrap(compute_hash(self.as_str())) - - def descr_mul(self, space, w_times): - # xxx not the most efficient implementation - w_string = space.wrap(self.as_str()) - # use the __mul__ method instead of space.mul() so that we - # return NotImplemented instead of raising a TypeError - return space.call_method(w_string, '__mul__', w_times) - - def descr_repr(self, space): - if isinstance(self, RWBuffer): - info = 'read-write buffer' - else: - info = 'read-only buffer' - addrstring = self.getaddrstring(space) - - return space.wrap("<%s for 0x%s, size %d>" % - (info, addrstring, self.getlength())) - - class RWBuffer(Buffer): - """Abstract base class for read-write memory views.""" + """Abstract base class for read-write buffers.""" __slots__ = () # no extra slot here @@ -151,72 +51,6 @@ self.setitem(start + i, string[i]) - at unwrap_spec(offset=int, size=int) -def descr_buffer__new__(space, w_subtype, w_object, offset=0, size=-1): - # w_subtype can only be exactly 'buffer' for now - if not space.is_w(w_subtype, space.gettypefor(Buffer)): - raise OperationError(space.w_TypeError, - space.wrap("argument 1 must be 'buffer'")) - - if space.isinstance_w(w_object, space.w_unicode): - # unicode objects support the old buffer interface - # but not the new buffer interface (change in python 2.7) - from rpython.rlib.rstruct.unichar import pack_unichar, UNICODE_SIZE - unistr = space.unicode_w(w_object) - builder = StringBuilder(len(unistr) * UNICODE_SIZE) - for unich in unistr: - pack_unichar(unich, builder) - from pypy.interpreter.buffer import StringBuffer - w_buffer = space.wrap(StringBuffer(builder.build())) - else: - w_buffer = space.buffer(w_object) - - buffer = space.interp_w(Buffer, w_buffer) # type-check - if offset == 0 and size == -1: - return w_buffer - # handle buffer slices - if offset < 0: - raise OperationError(space.w_ValueError, - space.wrap("offset must be zero or positive")) - if size < -1: - raise OperationError(space.w_ValueError, - space.wrap("size must be zero or positive")) - if isinstance(buffer, RWBuffer): - buffer = RWSubBuffer(buffer, offset, size) - else: - buffer = SubBuffer(buffer, offset, size) - return space.wrap(buffer) - - -Buffer.typedef = TypeDef( - "buffer", - __doc__ = """\ -buffer(object [, offset[, size]]) - -Create a new buffer object which references the given object. -The buffer will reference a slice of the target object from the -start of the object (or at the specified offset). The slice will -extend to the end of the target object (or with the specified size). -""", - __new__ = interp2app(descr_buffer__new__), - __len__ = interp2app(Buffer.descr_len), - __getitem__ = interp2app(Buffer.descr_getitem), - __setitem__ = interp2app(Buffer.descr_setitem), - __buffer__ = interp2app(Buffer.descr__buffer__), - __str__ = interp2app(Buffer.descr_str), - __add__ = interp2app(Buffer.descr_add), - __eq__ = interp2app(Buffer.descr_eq), - __ne__ = interp2app(Buffer.descr_ne), - __lt__ = interp2app(Buffer.descr_lt), - __le__ = interp2app(Buffer.descr_le), - __gt__ = interp2app(Buffer.descr_gt), - __ge__ = interp2app(Buffer.descr_ge), - __hash__ = interp2app(Buffer.descr_hash), - __mul__ = interp2app(Buffer.descr_mul), - __rmul__ = interp2app(Buffer.descr_mul), - __repr__ = interp2app(Buffer.descr_repr), -) -Buffer.typedef.acceptable_as_base_class = False # ____________________________________________________________ diff --git a/pypy/interpreter/test/test_buffer.py b/pypy/interpreter/test/test_buffer.py --- a/pypy/interpreter/test/test_buffer.py +++ b/pypy/interpreter/test/test_buffer.py @@ -1,5 +1,5 @@ import py -from pypy.interpreter.buffer import Buffer +from pypy.module.__builtin__.interp_memoryview import W_Buffer from rpython.tool.udir import udir testdir = udir.ensure('test_buffer', dir=1) @@ -11,19 +11,17 @@ space = self.space w_hello = space.wrap('hello world') buf = space.buffer_w(w_hello) - assert isinstance(buf, Buffer) assert buf.getlength() == 11 assert buf.as_str() == 'hello world' assert buf.getslice(1, 6, 1, 5) == 'ello ' - assert space.buffer_w(space.wrap(buf)) is buf + assert space.buffer_w(W_Buffer(buf)) is buf assert space.bufferstr_w(w_hello) == 'hello world' - assert space.bufferstr_w(space.buffer(w_hello)) == 'hello world' + assert space.bufferstr_w(W_Buffer(space.buffer_w(w_hello))) == 'hello world' space.raises_w(space.w_TypeError, space.buffer_w, space.wrap(5)) - space.raises_w(space.w_TypeError, space.buffer, space.wrap(5)) def test_file_write(self): space = self.space - w_buffer = space.buffer(space.wrap('hello world')) + w_buffer = W_Buffer(space.buffer_w(space.wrap('hello world'))) filename = str(testdir.join('test_file_write')) space.appexec([w_buffer, space.wrap(filename)], """(buffer, filename): f = open(filename, 'wb') diff --git a/pypy/module/__builtin__/interp_memoryview.py b/pypy/module/__builtin__/interp_memoryview.py --- a/pypy/module/__builtin__/interp_memoryview.py +++ b/pypy/module/__builtin__/interp_memoryview.py @@ -1,14 +1,168 @@ """ Implementation of the 'buffer' and 'memoryview' types. """ +import operator + +from pypy.interpreter import buffer from pypy.interpreter.baseobjspace import W_Root -from pypy.interpreter import buffer +from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.typedef import TypeDef, GetSetProperty -from pypy.interpreter.error import OperationError -import operator +from rpython.rlib.objectmodel import compute_hash +from rpython.rlib.rstring import StringBuilder -W_Buffer = buffer.Buffer # actually implemented in pypy.interpreter.buffer + +def _buffer_setitem(space, buf, w_index, newstring): + start, stop, step, size = space.decode_index4(w_index, buf.getlength()) + if step == 0: # index only + if len(newstring) != 1: + msg = 'buffer[index]=x: x must be a single character' + raise OperationError(space.w_TypeError, space.wrap(msg)) + char = newstring[0] # annotator hint + buf.setitem(start, char) + elif step == 1: + if len(newstring) != size: + msg = "right operand length must match slice length" + raise OperationError(space.w_ValueError, space.wrap(msg)) + buf.setslice(start, newstring) + else: + raise OperationError(space.w_ValueError, + space.wrap("buffer object does not support" + " slicing with a step")) + + +class W_Buffer(W_Root): + """Implement the built-in 'buffer' type as a thin wrapper around + an interp-level buffer. + """ + + def __init__(self, buf): + self.buf = buf + + def buffer_w(self, space): + return self.buf + + @staticmethod + @unwrap_spec(offset=int, size=int) + def descr_new_buffer(space, w_subtype, w_object, offset=0, size=-1): + if space.isinstance_w(w_object, space.w_unicode): + # unicode objects support the old buffer interface + # but not the new buffer interface (change in python 2.7) + from rpython.rlib.rstruct.unichar import pack_unichar, UNICODE_SIZE + unistr = space.unicode_w(w_object) + builder = StringBuilder(len(unistr) * UNICODE_SIZE) + for unich in unistr: + pack_unichar(unich, builder) + from pypy.interpreter.buffer import StringBuffer + buf = StringBuffer(builder.build()) + else: + buf = space.buffer_w(w_object) + + if offset == 0 and size == -1: + return W_Buffer(buf) + # handle buffer slices + if offset < 0: + raise OperationError(space.w_ValueError, + space.wrap("offset must be zero or positive")) + if size < -1: + raise OperationError(space.w_ValueError, + space.wrap("size must be zero or positive")) + if isinstance(buf, buffer.RWBuffer): + buf = buffer.RWSubBuffer(buf, offset, size) + else: + buf = buffer.SubBuffer(buf, offset, size) + return W_Buffer(buf) + + def descr_len(self, space): + return space.wrap(self.buf.getlength()) + + def descr_getitem(self, space, w_index): + start, stop, step, size = space.decode_index4(w_index, self.buf.getlength()) + if step == 0: # index only + return space.wrap(self.buf.getitem(start)) + res = self.buf.getslice(start, stop, step, size) + return space.wrap(res) + + @unwrap_spec(newstring='bufferstr') + def descr_setitem(self, space, w_index, newstring): + if not isinstance(self.buf, buffer.RWBuffer): + raise OperationError(space.w_TypeError, + space.wrap("buffer is read-only")) + _buffer_setitem(space, self.buf, w_index, newstring) + + def descr_str(self, space): + return space.wrap(self.buf.as_str()) + + @unwrap_spec(other='bufferstr') + def descr_add(self, space, other): + return space.wrap(self.buf.as_str() + other) + + def _make_descr__cmp(name): + def descr__cmp(self, space, w_other): + if not isinstance(w_other, W_Buffer): + return space.w_NotImplemented + # xxx not the most efficient implementation + str1 = self.buf.as_str() + str2 = w_other.buf.as_str() + return space.wrap(getattr(operator, name)(str1, str2)) + descr__cmp.func_name = name + return descr__cmp + + descr_eq = _make_descr__cmp('eq') + descr_ne = _make_descr__cmp('ne') + descr_lt = _make_descr__cmp('lt') + descr_le = _make_descr__cmp('le') + descr_gt = _make_descr__cmp('gt') + descr_ge = _make_descr__cmp('ge') + + def descr_hash(self, space): + return space.wrap(compute_hash(self.buf.as_str())) + + def descr_mul(self, space, w_times): + # xxx not the most efficient implementation + w_string = space.wrap(self.buf.as_str()) + # use the __mul__ method instead of space.mul() so that we + # return NotImplemented instead of raising a TypeError + return space.call_method(w_string, '__mul__', w_times) + + def descr_repr(self, space): + if isinstance(self.buf, buffer.RWBuffer): + info = 'read-write buffer' + else: + info = 'read-only buffer' + addrstring = self.getaddrstring(space) + + return space.wrap("<%s for 0x%s, size %d>" % + (info, addrstring, self.buf.getlength())) + +W_Buffer.typedef = TypeDef( + "buffer", + __doc__ = """\ +buffer(object [, offset[, size]]) + +Create a new buffer object which references the given object. +The buffer will reference a slice of the target object from the +start of the object (or at the specified offset). The slice will +extend to the end of the target object (or with the specified size). +""", + __new__ = interp2app(W_Buffer.descr_new_buffer), + __len__ = interp2app(W_Buffer.descr_len), + __getitem__ = interp2app(W_Buffer.descr_getitem), + __setitem__ = interp2app(W_Buffer.descr_setitem), + __str__ = interp2app(W_Buffer.descr_str), + __add__ = interp2app(W_Buffer.descr_add), + __eq__ = interp2app(W_Buffer.descr_eq), + __ne__ = interp2app(W_Buffer.descr_ne), + __lt__ = interp2app(W_Buffer.descr_lt), + __le__ = interp2app(W_Buffer.descr_le), + __gt__ = interp2app(W_Buffer.descr_gt), + __ge__ = interp2app(W_Buffer.descr_ge), + __hash__ = interp2app(W_Buffer.descr_hash), + __mul__ = interp2app(W_Buffer.descr_mul), + __rmul__ = interp2app(W_Buffer.descr_mul), + __repr__ = interp2app(W_Buffer.descr_repr), +) +W_Buffer.typedef.acceptable_as_base_class = False class W_MemoryView(W_Root): @@ -17,9 +171,16 @@ """ def __init__(self, buf): - assert isinstance(buf, buffer.Buffer) self.buf = buf + def buffer_w(self, space): + return self.buf + + @staticmethod + def descr_new_memoryview(space, w_subtype, w_object): + w_memoryview = W_MemoryView(space.buffer_w(w_object)) + return w_memoryview + def _make_descr__cmp(name): def descr__cmp(self, space, w_other): if isinstance(w_other, W_MemoryView): @@ -29,14 +190,14 @@ return space.wrap(getattr(operator, name)(str1, str2)) try: - w_buf = space.buffer(w_other) + buf = space.buffer_w(w_other) except OperationError, e: if not e.match(space, space.w_TypeError): raise return space.w_NotImplemented else: str1 = self.as_str() - str2 = space.buffer_w(w_buf).as_str() + str2 = buf.as_str() return space.wrap(getattr(operator, name)(str1, str2)) descr__cmp.func_name = name return descr__cmp @@ -102,15 +263,13 @@ @unwrap_spec(newstring='bufferstr') def descr_setitem(self, space, w_index, newstring): - buf = self.buf - if isinstance(buf, buffer.RWBuffer): - buf.descr_setitem(space, w_index, newstring) - else: + if not isinstance(self.buf, buffer.RWBuffer): raise OperationError(space.w_TypeError, space.wrap("cannot modify read-only memory")) + _buffer_setitem(space, self.buf, w_index, newstring) def descr_len(self, space): - return self.buf.descr_len(space) + return space.wrap(self.buf.getlength()) def w_get_format(self, space): return space.wrap("B") @@ -134,18 +293,12 @@ # I've never seen anyone filling this field return space.w_None - -def descr_new(space, w_subtype, w_object): - memoryview = W_MemoryView(space.buffer(w_object)) - return space.wrap(memoryview) - W_MemoryView.typedef = TypeDef( "memoryview", __doc__ = """\ Create a new memoryview object which references the given object. """, - __new__ = interp2app(descr_new), - __buffer__ = interp2app(W_MemoryView.descr_buffer), + __new__ = interp2app(W_MemoryView.descr_new_memoryview), __eq__ = interp2app(W_MemoryView.descr_eq), __ge__ = interp2app(W_MemoryView.descr_ge), __getitem__ = interp2app(W_MemoryView.descr_getitem), diff --git a/pypy/module/__pypy__/bytebuffer.py b/pypy/module/__pypy__/bytebuffer.py --- a/pypy/module/__pypy__/bytebuffer.py +++ b/pypy/module/__pypy__/bytebuffer.py @@ -4,6 +4,7 @@ from pypy.interpreter.buffer import RWBuffer from pypy.interpreter.gateway import unwrap_spec +from pypy.module.__builtin__.interp_memoryview import W_Buffer class ByteBuffer(RWBuffer): @@ -23,4 +24,4 @@ @unwrap_spec(length=int) def bytebuffer(space, length): - return space.wrap(ByteBuffer(length)) + return W_Buffer(ByteBuffer(length)) diff --git a/pypy/module/_cffi_backend/cbuffer.py b/pypy/module/_cffi_backend/cbuffer.py --- a/pypy/module/_cffi_backend/cbuffer.py +++ b/pypy/module/_cffi_backend/cbuffer.py @@ -4,6 +4,7 @@ from pypy.interpreter.gateway import unwrap_spec, interp2app from pypy.interpreter.typedef import TypeDef, make_weakref_descr from pypy.module._cffi_backend import cdataobj, ctypeptr, ctypearray +from pypy.module.__builtin__.interp_memoryview import W_Buffer from rpython.rtyper.annlowlevel import llstr from rpython.rtyper.lltypesystem import rffi @@ -39,38 +40,19 @@ copy_string_to_raw(llstr(string), raw_cdata, 0, len(string)) -class MiniBuffer(W_Root): - # a different subclass of W_Root for the MiniBuffer, because we - # want a slightly different (simplified) API at the level of Python. +# Override the typedef to narrow down the interface that's exposed to app-level +class MiniBuffer(W_Buffer): def __init__(self, buffer, keepalive=None): - self.buffer = buffer + W_Buffer.__init__(self, buffer) self.keepalive = keepalive - def descr_len(self, space): - return self.buffer.descr_len(space) - - def descr_getitem(self, space, w_index): - return self.buffer.descr_getitem(space, w_index) - - @unwrap_spec(newstring='bufferstr') - def descr_setitem(self, space, w_index, newstring): - self.buffer.descr_setitem(space, w_index, newstring) - - def descr__buffer__(self, space): - return self.buffer.descr__buffer__(space) - - def descr_str(self, space): - return space.wrap(self.buffer.as_str()) - - MiniBuffer.typedef = TypeDef( "buffer", __module__ = "_cffi_backend", __len__ = interp2app(MiniBuffer.descr_len), __getitem__ = interp2app(MiniBuffer.descr_getitem), __setitem__ = interp2app(MiniBuffer.descr_setitem), - __buffer__ = interp2app(MiniBuffer.descr__buffer__), __weakref__ = make_weakref_descr(MiniBuffer), __str__ = interp2app(MiniBuffer.descr_str), ) diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py --- a/pypy/module/_io/interp_bufferedio.py +++ b/pypy/module/_io/interp_bufferedio.py @@ -5,6 +5,7 @@ TypeDef, GetSetProperty, generic_new_descr, interp_attrproperty_w) from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault from pypy.interpreter.buffer import RWBuffer +from pypy.module.__builtin__.interp_memoryview import W_Buffer from rpython.rlib.rstring import StringBuilder from rpython.rlib.rarithmetic import r_longlong, intmask from rpython.rlib import rposix @@ -511,7 +512,7 @@ def _raw_read(self, space, buffer, start, length): length = intmask(length) - w_buf = space.wrap(RawBuffer(buffer, start, length)) + w_buf = W_Buffer(RawBuffer(buffer, start, length)) while True: try: w_size = space.call_method(self.w_raw, "readinto", w_buf) diff --git a/pypy/module/_rawffi/array.py b/pypy/module/_rawffi/array.py --- a/pypy/module/_rawffi/array.py +++ b/pypy/module/_rawffi/array.py @@ -207,7 +207,6 @@ __setitem__ = interp2app(W_ArrayInstance.descr_setitem), __getitem__ = interp2app(W_ArrayInstance.descr_getitem), __len__ = interp2app(W_ArrayInstance.getlength), - __buffer__ = interp2app(W_ArrayInstance.descr_buffer), buffer = GetSetProperty(W_ArrayInstance.getbuffer), shape = interp_attrproperty('shape', W_ArrayInstance), free = interp2app(W_ArrayInstance.free), @@ -232,7 +231,6 @@ __setitem__ = interp2app(W_ArrayInstance.descr_setitem), __getitem__ = interp2app(W_ArrayInstance.descr_getitem), __len__ = interp2app(W_ArrayInstance.getlength), - __buffer__ = interp2app(W_ArrayInstance.descr_buffer), buffer = GetSetProperty(W_ArrayInstance.getbuffer), shape = interp_attrproperty('shape', W_ArrayInstance), byptr = interp2app(W_ArrayInstance.byptr), diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py --- a/pypy/module/_rawffi/interp_rawffi.py +++ b/pypy/module/_rawffi/interp_rawffi.py @@ -352,9 +352,9 @@ lltype.free(self.ll_buffer, flavor='raw') self.ll_buffer = lltype.nullptr(rffi.VOIDP.TO) - def descr_buffer(self, space): + def buffer_w(self, space): from pypy.module._rawffi.buffer import RawFFIBuffer - return space.wrap(RawFFIBuffer(self)) + return RawFFIBuffer(self) def getrawsize(self): raise NotImplementedError("abstract base class") diff --git a/pypy/module/_rawffi/structure.py b/pypy/module/_rawffi/structure.py --- a/pypy/module/_rawffi/structure.py +++ b/pypy/module/_rawffi/structure.py @@ -364,7 +364,6 @@ __repr__ = interp2app(W_StructureInstance.descr_repr), __getattr__ = interp2app(W_StructureInstance.getattr), __setattr__ = interp2app(W_StructureInstance.setattr), - __buffer__ = interp2app(W_StructureInstance.descr_buffer), buffer = GetSetProperty(W_StructureInstance.getbuffer), free = interp2app(W_StructureInstance.free), shape = interp_attrproperty('shape', W_StructureInstance), @@ -387,7 +386,6 @@ __repr__ = interp2app(W_StructureInstance.descr_repr), __getattr__ = interp2app(W_StructureInstance.getattr), __setattr__ = interp2app(W_StructureInstance.setattr), - __buffer__ = interp2app(W_StructureInstance.descr_buffer), buffer = GetSetProperty(W_StructureInstance.getbuffer), shape = interp_attrproperty('shape', W_StructureInstance), byptr = interp2app(W_StructureInstance.byptr), diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py --- a/pypy/module/array/interp_array.py +++ b/pypy/module/array/interp_array.py @@ -132,6 +132,9 @@ self.len = 0 self.allocated = 0 + def buffer_w(self, space): + return ArrayBuffer(self) + def descr_append(self, space, w_x): """ append(x) @@ -462,9 +465,6 @@ # Misc methods - def descr_buffer(self, space): - return space.wrap(ArrayBuffer(self)) - def descr_repr(self, space): if self.len == 0: return space.wrap("array('%s')" % self.typecode) @@ -508,7 +508,6 @@ __radd__ = interp2app(W_ArrayBase.descr_radd), __rmul__ = interp2app(W_ArrayBase.descr_rmul), - __buffer__ = interp2app(W_ArrayBase.descr_buffer), __repr__ = interp2app(W_ArrayBase.descr_repr), itemsize = GetSetProperty(descr_itemsize), diff --git a/pypy/module/cpyext/bufferobject.py b/pypy/module/cpyext/bufferobject.py --- a/pypy/module/cpyext/bufferobject.py +++ b/pypy/module/cpyext/bufferobject.py @@ -3,8 +3,9 @@ cpython_api, Py_ssize_t, cpython_struct, bootstrap_function, PyObjectFields, PyObject) from pypy.module.cpyext.pyobject import make_typedescr, Py_DecRef, make_ref -from pypy.interpreter.buffer import Buffer, StringBuffer, SubBuffer +from pypy.interpreter.buffer import StringBuffer, SubBuffer from pypy.interpreter.error import OperationError +from pypy.module.__builtin__.interp_memoryview import W_Buffer from pypy.module.array.interp_array import ArrayBuffer @@ -24,7 +25,7 @@ @bootstrap_function def init_bufferobject(space): "Type description of PyBufferObject" - make_typedescr(space.gettypefor(Buffer).instancetypedef, + make_typedescr(space.gettypefor(W_Buffer).instancetypedef, basestruct=PyBufferObject.TO, attach=buffer_attach, dealloc=buffer_dealloc, @@ -39,23 +40,26 @@ rffi.setintfield(py_buf, 'c_b_readonly', 1) rffi.setintfield(py_buf, 'c_b_hash', -1) - if isinstance(w_obj, SubBuffer): - py_buf.c_b_offset = w_obj.offset - w_obj = w_obj.buffer + assert isinstance(w_obj, W_Buffer) + buf = w_obj.buf - # If w_obj already allocated a fixed buffer, use it, and keep a - # reference to w_obj. + if isinstance(buf, SubBuffer): + py_buf.c_b_offset = buf.offset + buf = buf.buffer + + # If buf already allocated a fixed buffer, use it, and keep a + # reference to buf. # Otherwise, b_base stays NULL, and we own the b_ptr. - if isinstance(w_obj, StringBuffer): + if isinstance(buf, StringBuffer): py_buf.c_b_base = lltype.nullptr(PyObject.TO) - py_buf.c_b_ptr = rffi.cast(rffi.VOIDP, rffi.str2charp(w_obj.value)) - py_buf.c_b_size = w_obj.getlength() - elif isinstance(w_obj, ArrayBuffer): - w_base = w_obj.array + py_buf.c_b_ptr = rffi.cast(rffi.VOIDP, rffi.str2charp(buf.value)) + py_buf.c_b_size = buf.getlength() + elif isinstance(buf, ArrayBuffer): + w_base = buf.array py_buf.c_b_base = make_ref(space, w_base) - py_buf.c_b_ptr = rffi.cast(rffi.VOIDP, w_obj.array._charbuf_start()) - py_buf.c_b_size = w_obj.getlength() + py_buf.c_b_ptr = rffi.cast(rffi.VOIDP, buf.array._charbuf_start()) + py_buf.c_b_size = buf.getlength() else: raise OperationError(space.w_NotImplementedError, space.wrap( "buffer flavor not supported")) diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -14,8 +14,9 @@ from pypy.module.cpyext.pyobject import from_ref from pypy.module.cpyext.pyerrors import PyErr_Occurred from pypy.module.cpyext.state import State +from pypy.module.__builtin__.interp_memoryview import W_Buffer from pypy.interpreter.error import OperationError, oefmt -from pypy.interpreter.buffer import Buffer as W_Buffer +from pypy.interpreter.buffer import Buffer from pypy.interpreter.argument import Arguments from rpython.rlib.unroll import unrolling_iterable from rpython.rlib.objectmodel import specialize @@ -228,7 +229,7 @@ check_num_args(space, w_args, 0) return space.wrap(generic_cpy_call(space, func_target, w_self)) -class CPyBuffer(W_Buffer): +class CPyBuffer(Buffer): # Similar to Py_buffer def __init__(self, ptr, size, w_obj): @@ -249,7 +250,7 @@ size = generic_cpy_call(space, func_target, w_self, index, ptr) if size < 0: space.fromcache(State).check_and_raise_exception(always=True) - return space.wrap(CPyBuffer(ptr[0], size, w_self)) + return W_Buffer(CPyBuffer(ptr[0], size, w_self)) def get_richcmp_func(OP_CONST): def inner(space, w_self, w_args, func): diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -28,7 +28,7 @@ PyNumberMethods, PyMappingMethods, PySequenceMethods, PyBufferProcs) from pypy.module.cpyext.slotdefs import ( slotdefs_for_tp_slots, slotdefs_for_wrappers, get_slot_tp_function) -from pypy.interpreter.buffer import Buffer +from pypy.module.__builtin__.interp_memoryview import W_Buffer from pypy.interpreter.error import OperationError from rpython.rlib.rstring import rsplit from rpython.rlib.objectmodel import specialize @@ -509,7 +509,7 @@ # buffer protocol if space.is_w(w_type, space.w_str): setup_string_buffer_procs(space, pto) - if space.is_w(w_type, space.gettypefor(Buffer)): + if space.is_w(w_type, space.gettypefor(W_Buffer)): setup_buffer_buffer_procs(space, pto) pto.c_tp_free = llhelper(PyObject_Del.api_func.functype, diff --git a/pypy/module/micronumpy/boxes.py b/pypy/module/micronumpy/boxes.py --- a/pypy/module/micronumpy/boxes.py +++ b/pypy/module/micronumpy/boxes.py @@ -342,8 +342,8 @@ def descr_copy(self, space): return self.convert_to(space, self.get_dtype(space)) - def descr_buffer(self, space): - return self.descr_ravel(space).descr_get_data(space) + def buffer_w(self, space): + return self.descr_ravel(space).buffer_w(space) def descr_byteswap(self, space): return self.get_dtype(space).itemtype.byteswap(self) @@ -553,7 +553,6 @@ __nonzero__ = interp2app(W_GenericBox.descr_nonzero), __oct__ = interp2app(W_GenericBox.descr_oct), __hex__ = interp2app(W_GenericBox.descr_hex), - __buffer__ = interp2app(W_GenericBox.descr_buffer), __add__ = interp2app(W_GenericBox.descr_add), __sub__ = interp2app(W_GenericBox.descr_sub), diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py --- a/pypy/module/micronumpy/ndarray.py +++ b/pypy/module/micronumpy/ndarray.py @@ -7,6 +7,7 @@ from rpython.rlib.rawstorage import RAW_STORAGE_PTR from rpython.rtyper.lltypesystem import rffi from rpython.tool.sourcetools import func_with_new_name +from pypy.module.__builtin__.interp_memoryview import W_Buffer from pypy.module.micronumpy import descriptor, ufuncs, boxes, arrayops, loop, \ support, constants as NPY from pypy.module.micronumpy.appbridge import get_appbridge_cache @@ -602,8 +603,11 @@ raise OperationError(space.w_NotImplementedError, space.wrap( "ctypes not implemented yet")) + def buffer_w(self, space): + return self.implementation.get_buffer(space) + def descr_get_data(self, space): - return self.implementation.get_buffer(space) + return W_Buffer(self.buffer_w(space)) @unwrap_spec(offset=int, axis1=int, axis2=int) def descr_diagonal(self, space, offset=0, axis1=0, axis2=1): @@ -1259,7 +1263,6 @@ __float__ = interp2app(W_NDimArray.descr_float), __hex__ = interp2app(W_NDimArray.descr_hex), __oct__ = interp2app(W_NDimArray.descr_oct), - __buffer__ = interp2app(W_NDimArray.descr_get_data), __index__ = interp2app(W_NDimArray.descr_index), __pos__ = interp2app(W_NDimArray.descr_pos), diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py --- a/pypy/module/mmap/interp_mmap.py +++ b/pypy/module/mmap/interp_mmap.py @@ -17,6 +17,10 @@ self.space = space self.mmap = mmap_obj + def buffer_w(self, space): + self.check_valid() + return MMapBuffer(self.space, self.mmap) + def close(self): self.mmap.close() @@ -196,10 +200,6 @@ self.mmap.setitem(start, value[i]) start += step - def descr_buffer(self): - self.check_valid() - return self.space.wrap(MMapBuffer(self.space, self.mmap)) - if rmmap._POSIX: @unwrap_spec(fileno=int, length=int, flags=int, @@ -256,7 +256,6 @@ __len__ = interp2app(W_MMap.__len__), __getitem__ = interp2app(W_MMap.descr_getitem), __setitem__ = interp2app(W_MMap.descr_setitem), - __buffer__ = interp2app(W_MMap.descr_buffer), ) constants = rmmap.constants diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py --- a/pypy/module/pyexpat/interp_pyexpat.py +++ b/pypy/module/pyexpat/interp_pyexpat.py @@ -429,7 +429,7 @@ self.handlers = [None] * NB_HANDLERS - self.buffer_w = None + self.buffer_w_ = None self.buffer_size = 8192 self.buffer_used = 0 self.w_character_data_handler = None @@ -541,7 +541,7 @@ def buffer_string(self, space, w_string, length): ll_length = rffi.cast(lltype.Signed, length) - if self.buffer_w is not None: + if self.buffer_w_ is not None: if self.buffer_used + ll_length > self.buffer_size: self.flush_character_buffer(space) # handler might have changed; drop the rest on the floor @@ -549,11 +549,11 @@ if self.w_character_data_handler is None: return True if ll_length <= self.buffer_size: - self.buffer_w.append(w_string) + self.buffer_w_.append(w_string) self.buffer_used += ll_length return True else: - self.buffer_w = [] + self.buffer_w_ = [] self.buffer_used = 0 return False @@ -685,12 +685,12 @@ return space.wrap(parser) def flush_character_buffer(self, space): - if not self.buffer_w: + if not self.buffer_w_: return w_data = space.call_function( space.getattr(space.wrap(''), space.wrap('join')), - space.newlist(self.buffer_w)) - self.buffer_w = [] + space.newlist(self.buffer_w_)) + self.buffer_w_ = [] self.buffer_used = 0 if self.w_character_data_handler: @@ -735,14 +735,14 @@ self.buffer_size = value def get_buffer_text(self, space): - return space.wrap(self.buffer_w is not None) + return space.wrap(self.buffer_w_ is not None) def set_buffer_text(self, space, w_value): if space.is_true(w_value): - self.buffer_w = [] + self.buffer_w_ = [] self.buffer_used = 0 else: self.flush_character_buffer(space) - self.buffer_w = None + self.buffer_w_ = None def get_intern(self, space): if self.w_intern: diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py --- a/pypy/objspace/fake/objspace.py +++ b/pypy/objspace/fake/objspace.py @@ -292,6 +292,11 @@ ec._py_repr = None return ec + def buffer_w(self, w_obj): + from pypy.interpreter.buffer import Buffer + is_root(w_obj) + return Buffer() + def unicode_from_object(self, w_obj): return w_some_obj() diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py --- a/pypy/objspace/std/bytearrayobject.py +++ b/pypy/objspace/std/bytearrayobject.py @@ -27,6 +27,9 @@ """representation for debugging purposes""" return "%s(%s)" % (w_self.__class__.__name__, ''.join(w_self.data)) + def buffer_w(w_self, space): + return BytearrayBuffer(w_self.data) + def _new(self, value): return W_BytearrayObject(_make_data(value)) @@ -295,9 +298,6 @@ def descr_iter(self, space): return space.newseqiter(self) - def descr_buffer(self, space): - return BytearrayBuffer(self.data) - def descr_inplace_add(self, space, w_other): if isinstance(w_other, W_BytearrayObject): self.data += w_other.data @@ -1011,7 +1011,6 @@ __init__ = interp2app(W_BytearrayObject.descr_init, doc=BytearrayDocstrings.__init__.__doc__), - __buffer__ = interp2app(W_BytearrayObject.descr_buffer), __iadd__ = interp2app(W_BytearrayObject.descr_inplace_add, doc=BytearrayDocstrings.__iadd__.__doc__), diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py --- a/pypy/objspace/std/bytesobject.py +++ b/pypy/objspace/std/bytesobject.py @@ -424,9 +424,6 @@ of the specified width. The string S is never truncated. """ - def descr_buffer(self, space): - pass - class W_BytesObject(W_AbstractBytesObject): import_from_mixin(StringMethods) @@ -445,6 +442,9 @@ def str_w(self, space): return self._value + def buffer_w(w_self, space): + return StringBuffer(w_self._value) + def listview_bytes(self): return _create_list_from_bytes(self._value) @@ -567,9 +567,6 @@ def descr_mod(self, space, w_values): return mod_format(space, self, w_values, do_unicode=False) - def descr_buffer(self, space): - return space.wrap(StringBuffer(self._value)) - def descr_eq(self, space, w_other): if space.config.objspace.std.withstrbuf: from pypy.objspace.std.strbufobject import W_StringBufferObject @@ -853,7 +850,6 @@ format = interpindirect2app(W_BytesObject.descr_format), __format__ = interpindirect2app(W_BytesObject.descr__format__), __mod__ = interpindirect2app(W_BytesObject.descr_mod), - __buffer__ = interpindirect2app(W_AbstractBytesObject.descr_buffer), __getnewargs__ = interpindirect2app( W_AbstractBytesObject.descr_getnewargs), _formatter_parser = interp2app(W_BytesObject.descr_formatter_parser), diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py --- a/pypy/objspace/std/intobject.py +++ b/pypy/objspace/std/intobject.py @@ -696,7 +696,7 @@ else: # If object supports the buffer interface try: - w_buffer = space.buffer(w_value) + buf = space.buffer_w(w_value) except OperationError as e: if not e.match(space, space.w_TypeError): raise @@ -704,7 +704,6 @@ "int() argument must be a string or a number, " "not '%T'", w_value) else: - buf = space.interp_w(Buffer, w_buffer) value, w_longval = _string_to_int_or_long(space, w_value, buf.as_str()) ok = True diff --git a/pypy/objspace/std/longobject.py b/pypy/objspace/std/longobject.py --- a/pypy/objspace/std/longobject.py +++ b/pypy/objspace/std/longobject.py @@ -513,7 +513,7 @@ unicode_to_decimal_w(space, w_value)) else: try: - w_buffer = space.buffer(w_value) + buf = space.buffer_w(w_value) except OperationError, e: if not e.match(space, space.w_TypeError): raise @@ -521,7 +521,6 @@ "long() argument must be a string or a number, " "not '%T'", w_value) else: - buf = space.interp_w(Buffer, w_buffer) return _string_to_w_long(space, w_longtype, w_value, buf.as_str()) else: diff --git a/pypy/objspace/std/strbufobject.py b/pypy/objspace/std/strbufobject.py --- a/pypy/objspace/std/strbufobject.py +++ b/pypy/objspace/std/strbufobject.py @@ -2,12 +2,12 @@ import py -from pypy.objspace.std.basestringtype import basestring_typedef -from pypy.objspace.std.bytesobject import W_AbstractBytesObject, W_BytesObject -from pypy.objspace.std.stdtypedef import StdTypeDef +from pypy.objspace.std.bytesobject import (W_AbstractBytesObject, + W_BytesObject, StringBuffer) from pypy.interpreter.gateway import interp2app, unwrap_spec from rpython.rlib.rstring import StringBuilder + class W_StringBufferObject(W_AbstractBytesObject): w_str = None @@ -36,6 +36,9 @@ def str_w(self, space): return self.force() + def buffer_w(self, space): + return StringBuffer(self.force()) + def descr_len(self, space): return space.wrap(self.length) From noreply at buildbot.pypy.org Tue Mar 18 16:48:25 2014 From: noreply at buildbot.pypy.org (Manuel Jacob) Date: Tue, 18 Mar 2014 16:48:25 +0100 (CET) Subject: [pypy-commit] pypy default: Document branch. Message-ID: <20140318154825.DB0AC1D26D3@cobra.cs.uni-duesseldorf.de> Author: Manuel Jacob Branch: Changeset: r70054:d80cbba4def2 Date: 2014-03-18 16:43 +0100 http://bitbucket.org/pypy/pypy/changeset/d80cbba4def2/ Log: Document branch. diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -108,3 +108,9 @@ .. branch: virtual-raw-store-load Support for virtualizing raw_store/raw_load operations + +.. branch: refactor-buffer-api +Separate the interp-level buffer API from the buffer type exposed to +app-level. The `Buffer` class is now used by `W_MemoryView` and +`W_Buffer`, which is not present in Python 3. Previously `W_Buffer` was +an alias to `Buffer`, which was wrappable itself. From noreply at buildbot.pypy.org Tue Mar 18 16:48:27 2014 From: noreply at buildbot.pypy.org (Manuel Jacob) Date: Tue, 18 Mar 2014 16:48:27 +0100 (CET) Subject: [pypy-commit] pypy default: Change misleading comments: calling these wrappers 'thin' is a bit overstated. Message-ID: <20140318154827.126081D26D3@cobra.cs.uni-duesseldorf.de> Author: Manuel Jacob Branch: Changeset: r70055:a90697f1fab2 Date: 2014-03-18 16:47 +0100 http://bitbucket.org/pypy/pypy/changeset/a90697f1fab2/ Log: Change misleading comments: calling these wrappers 'thin' is a bit overstated. diff --git a/pypy/module/__builtin__/interp_memoryview.py b/pypy/module/__builtin__/interp_memoryview.py --- a/pypy/module/__builtin__/interp_memoryview.py +++ b/pypy/module/__builtin__/interp_memoryview.py @@ -32,7 +32,7 @@ class W_Buffer(W_Root): - """Implement the built-in 'buffer' type as a thin wrapper around + """Implement the built-in 'buffer' type as a wrapper around an interp-level buffer. """ @@ -166,7 +166,7 @@ class W_MemoryView(W_Root): - """Implement the built-in 'memoryview' type as a thin wrapper around + """Implement the built-in 'memoryview' type as a wrapper around an interp-level buffer. """ From noreply at buildbot.pypy.org Tue Mar 18 18:57:23 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 18 Mar 2014 18:57:23 +0100 (CET) Subject: [pypy-commit] pypy default: cleanup buffer changes Message-ID: <20140318175723.5E27D1C0A66@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r70056:56dd1868523c Date: 2014-03-18 13:45 -0400 http://bitbucket.org/pypy/pypy/changeset/56dd1868523c/ Log: cleanup buffer changes diff --git a/pypy/module/__builtin__/interp_memoryview.py b/pypy/module/__builtin__/interp_memoryview.py --- a/pypy/module/__builtin__/interp_memoryview.py +++ b/pypy/module/__builtin__/interp_memoryview.py @@ -44,7 +44,7 @@ @staticmethod @unwrap_spec(offset=int, size=int) - def descr_new_buffer(space, w_subtype, w_object, offset=0, size=-1): + def descr_new(space, w_subtype, w_object, offset=0, size=-1): if space.isinstance_w(w_object, space.w_unicode): # unicode objects support the old buffer interface # but not the new buffer interface (change in python 2.7) @@ -145,7 +145,7 @@ start of the object (or at the specified offset). The slice will extend to the end of the target object (or with the specified size). """, - __new__ = interp2app(W_Buffer.descr_new_buffer), + __new__ = interp2app(W_Buffer.descr_new), __len__ = interp2app(W_Buffer.descr_len), __getitem__ = interp2app(W_Buffer.descr_getitem), __setitem__ = interp2app(W_Buffer.descr_setitem), @@ -177,7 +177,7 @@ return self.buf @staticmethod - def descr_new_memoryview(space, w_subtype, w_object): + def descr_new(space, w_subtype, w_object): w_memoryview = W_MemoryView(space.buffer_w(w_object)) return w_memoryview @@ -298,7 +298,7 @@ __doc__ = """\ Create a new memoryview object which references the given object. """, - __new__ = interp2app(W_MemoryView.descr_new_memoryview), + __new__ = interp2app(W_MemoryView.descr_new), __eq__ = interp2app(W_MemoryView.descr_eq), __ge__ = interp2app(W_MemoryView.descr_ge), __getitem__ = interp2app(W_MemoryView.descr_getitem), diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py --- a/pypy/module/pyexpat/interp_pyexpat.py +++ b/pypy/module/pyexpat/interp_pyexpat.py @@ -429,7 +429,7 @@ self.handlers = [None] * NB_HANDLERS - self.buffer_w_ = None + self.buffer = None self.buffer_size = 8192 self.buffer_used = 0 self.w_character_data_handler = None @@ -541,7 +541,7 @@ def buffer_string(self, space, w_string, length): ll_length = rffi.cast(lltype.Signed, length) - if self.buffer_w_ is not None: + if self.buffer is not None: if self.buffer_used + ll_length > self.buffer_size: self.flush_character_buffer(space) # handler might have changed; drop the rest on the floor @@ -549,11 +549,11 @@ if self.w_character_data_handler is None: return True if ll_length <= self.buffer_size: - self.buffer_w_.append(w_string) + self.buffer.append(w_string) self.buffer_used += ll_length return True else: - self.buffer_w_ = [] + self.buffer = [] self.buffer_used = 0 return False @@ -685,12 +685,12 @@ return space.wrap(parser) def flush_character_buffer(self, space): - if not self.buffer_w_: + if not self.buffer: return w_data = space.call_function( space.getattr(space.wrap(''), space.wrap('join')), - space.newlist(self.buffer_w_)) - self.buffer_w_ = [] + space.newlist(self.buffer)) + self.buffer = [] self.buffer_used = 0 if self.w_character_data_handler: @@ -735,14 +735,14 @@ self.buffer_size = value def get_buffer_text(self, space): - return space.wrap(self.buffer_w_ is not None) + return space.wrap(self.buffer is not None) def set_buffer_text(self, space, w_value): if space.is_true(w_value): - self.buffer_w_ = [] + self.buffer = [] self.buffer_used = 0 else: self.flush_character_buffer(space) - self.buffer_w_ = None + self.buffer = None def get_intern(self, space): if self.w_intern: From noreply at buildbot.pypy.org Tue Mar 18 18:57:24 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 18 Mar 2014 18:57:24 +0100 (CET) Subject: [pypy-commit] pypy default: provide space.w_buffer Message-ID: <20140318175724.960C61C0A66@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r70057:bf7b7094d768 Date: 2014-03-18 13:47 -0400 http://bitbucket.org/pypy/pypy/changeset/bf7b7094d768/ Log: provide space.w_buffer diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -198,8 +198,7 @@ w_impl = space.lookup(self, '__buffer__') if w_impl is not None: w_result = space.get_and_call_function(w_impl, self) - from pypy.module.__builtin__.interp_memoryview import W_Buffer - if isinstance(w_result, W_Buffer): + if space.isinstance_w(w_result, space.w_buffer): return w_result.buf self._typed_unwrap_error(space, "buffer") diff --git a/pypy/module/cpyext/bufferobject.py b/pypy/module/cpyext/bufferobject.py --- a/pypy/module/cpyext/bufferobject.py +++ b/pypy/module/cpyext/bufferobject.py @@ -25,7 +25,7 @@ @bootstrap_function def init_bufferobject(space): "Type description of PyBufferObject" - make_typedescr(space.gettypefor(W_Buffer).instancetypedef, + make_typedescr(space.w_buffer.instancetypedef, basestruct=PyBufferObject.TO, attach=buffer_attach, dealloc=buffer_dealloc, diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -509,7 +509,7 @@ # buffer protocol if space.is_w(w_type, space.w_str): setup_string_buffer_procs(space, pto) - if space.is_w(w_type, space.gettypefor(W_Buffer)): + if space.is_w(w_type, space.w_buffer): setup_buffer_buffer_procs(space, pto) pto.c_tp_free = llhelper(PyObject_Del.api_func.functype, diff --git a/pypy/objspace/std/model.py b/pypy/objspace/std/model.py --- a/pypy/objspace/std/model.py +++ b/pypy/objspace/std/model.py @@ -63,6 +63,7 @@ from pypy.objspace.std import unicodeobject from pypy.objspace.std import dictproxyobject from pypy.objspace.std import proxyobject + from pypy.module.__builtin__.interp_memoryview import W_Buffer import pypy.objspace.std.default # register a few catch-all multimethods import pypy.objspace.std.marshal_impl # install marshal multimethods @@ -82,6 +83,7 @@ self.pythontypes.append(intobject.W_IntObject.typedef) self.pythontypes.append(boolobject.W_BoolObject.typedef) self.pythontypes.append(longobject.W_LongObject.typedef) + self.pythontypes.append(W_Buffer.typedef) # the set of implementation types self.typeorder = { From noreply at buildbot.pypy.org Tue Mar 18 19:25:27 2014 From: noreply at buildbot.pypy.org (Manuel Jacob) Date: Tue, 18 Mar 2014 19:25:27 +0100 (CET) Subject: [pypy-commit] pypy default: Organize imports. Message-ID: <20140318182527.CBC0B1C00B9@cobra.cs.uni-duesseldorf.de> Author: Manuel Jacob Branch: Changeset: r70058:2d7d9db55899 Date: 2014-03-18 19:24 +0100 http://bitbucket.org/pypy/pypy/changeset/2d7d9db55899/ Log: Organize imports. diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -1,40 +1,36 @@ import os -import sys +from rpython.rlib import jit +from rpython.rlib.objectmodel import specialize +from rpython.rlib.rstring import rsplit +from rpython.rtyper.annlowlevel import llhelper from rpython.rtyper.lltypesystem import rffi, lltype -from rpython.rtyper.annlowlevel import llhelper + from pypy.interpreter.baseobjspace import W_Root, DescrMismatch -from pypy.objspace.std.typeobject import W_TypeObject, find_best_base +from pypy.interpreter.error import OperationError from pypy.interpreter.typedef import GetSetProperty +from pypy.module.__builtin__.abstractinst import abstract_issubclass_w +from pypy.module.cpyext import structmemberdefs from pypy.module.cpyext.api import ( cpython_api, cpython_struct, bootstrap_function, Py_ssize_t, Py_ssize_tP, generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING, Py_TPFLAGS_HEAPTYPE, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL, - Py_TPFLAGS_HAVE_GETCHARBUFFER, - build_type_checkers, PyObjectFields) + Py_TPFLAGS_HAVE_GETCHARBUFFER, build_type_checkers) +from pypy.module.cpyext.methodobject import ( + PyDescr_NewWrapper, PyCFunction_NewEx, PyCFunction_typedef) +from pypy.module.cpyext.modsupport import convert_method_defs from pypy.module.cpyext.pyobject import ( PyObject, make_ref, create_ref, from_ref, get_typedescr, make_typedescr, - track_reference, RefcountState, borrow_from) -from pypy.interpreter.module import Module -from pypy.module.cpyext import structmemberdefs -from pypy.module.cpyext.modsupport import convert_method_defs + track_reference, RefcountState, borrow_from, Py_DecRef) +from pypy.module.cpyext.slotdefs import ( + slotdefs_for_tp_slots, slotdefs_for_wrappers, get_slot_tp_function) from pypy.module.cpyext.state import State -from pypy.module.cpyext.methodobject import ( - PyDescr_NewWrapper, PyCFunction_NewEx, PyCFunction_typedef) -from pypy.module.cpyext.pyobject import Py_IncRef, Py_DecRef, _Py_Dealloc from pypy.module.cpyext.structmember import PyMember_GetOne, PyMember_SetOne from pypy.module.cpyext.typeobjectdefs import ( PyTypeObjectPtr, PyTypeObject, PyGetSetDef, PyMemberDef, newfunc, PyNumberMethods, PyMappingMethods, PySequenceMethods, PyBufferProcs) -from pypy.module.cpyext.slotdefs import ( - slotdefs_for_tp_slots, slotdefs_for_wrappers, get_slot_tp_function) -from pypy.module.__builtin__.interp_memoryview import W_Buffer -from pypy.interpreter.error import OperationError -from rpython.rlib.rstring import rsplit -from rpython.rlib.objectmodel import specialize -from pypy.module.__builtin__.abstractinst import abstract_issubclass_w -from pypy.module.__builtin__.interp_classobj import W_ClassObject -from rpython.rlib import jit +from pypy.objspace.std.typeobject import W_TypeObject, find_best_base + WARN_ABOUT_MISSING_SLOT_FUNCTIONS = False @@ -471,7 +467,6 @@ def type_alloc(space, w_metatype): - size = rffi.sizeof(PyHeapTypeObject) metatype = rffi.cast(PyTypeObjectPtr, make_ref(space, w_metatype)) # Don't increase refcount for non-heaptypes if metatype: From noreply at buildbot.pypy.org Tue Mar 18 19:37:42 2014 From: noreply at buildbot.pypy.org (pjenvey) Date: Tue, 18 Mar 2014 19:37:42 +0100 (CET) Subject: [pypy-commit] pypy default: make test_load_error more resilient by improving the leak detection Message-ID: <20140318183742.B04651C00B9@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: Changeset: r70059:f777d0ca3f24 Date: 2014-03-18 11:36 -0700 http://bitbucket.org/pypy/pypy/changeset/f777d0ca3f24/ Log: make test_load_error more resilient by improving the leak detection diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py --- a/pypy/module/cpyext/test/test_cpyext.py +++ b/pypy/module/cpyext/test/test_cpyext.py @@ -183,6 +183,19 @@ from rpython.rlib.clibffi import get_libc_name cls.w_libc = cls.space.wrap(get_libc_name()) + def setup_method(self, meth): + freeze_refcnts(self) + + def teardown_method(self, meth): + self.cleanup_references(self.space) + # XXX: like AppTestCpythonExtensionBase.teardown_method: + # find out how to disable check_and_print_leaks() if the + # test failed + assert not self.check_and_print_leaks(), ( + "Test leaks or loses object(s). You should also check if " + "the test actually passed in the first place; if it failed " + "it is likely to reach this place.") + def test_load_error(self): import cpyext raises(ImportError, cpyext.load_module, "missing.file", "foo") @@ -358,13 +371,12 @@ for name in self.imported_module_names: self.unimport_module(name) self.cleanup_references(self.space) - if self.check_and_print_leaks(): - assert False, ( - "Test leaks or loses object(s). You should also check if " - "the test actually passed in the first place; if it failed " - "it is likely to reach this place.") - # XXX find out how to disable check_and_print_leaks() if the - # XXX test failed... + # XXX: find out how to disable check_and_print_leaks() if the + # test failed... + assert not self.check_and_print_leaks(), ( + "Test leaks or loses object(s). You should also check if " + "the test actually passed in the first place; if it failed " + "it is likely to reach this place.") class AppTestCpythonExtension(AppTestCpythonExtensionBase): From noreply at buildbot.pypy.org Tue Mar 18 19:48:41 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 18 Mar 2014 19:48:41 +0100 (CET) Subject: [pypy-commit] pypy default: fix translation Message-ID: <20140318184841.677C61C00B9@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r70060:131563225cdf Date: 2014-03-18 14:48 -0400 http://bitbucket.org/pypy/pypy/changeset/131563225cdf/ Log: fix translation diff --git a/pypy/module/__builtin__/interp_memoryview.py b/pypy/module/__builtin__/interp_memoryview.py --- a/pypy/module/__builtin__/interp_memoryview.py +++ b/pypy/module/__builtin__/interp_memoryview.py @@ -42,7 +42,6 @@ def buffer_w(self, space): return self.buf - @staticmethod @unwrap_spec(offset=int, size=int) def descr_new(space, w_subtype, w_object, offset=0, size=-1): if space.isinstance_w(w_object, space.w_unicode): @@ -145,7 +144,7 @@ start of the object (or at the specified offset). The slice will extend to the end of the target object (or with the specified size). """, - __new__ = interp2app(W_Buffer.descr_new), + __new__ = interp2app(W_Buffer.descr_new.im_func), __len__ = interp2app(W_Buffer.descr_len), __getitem__ = interp2app(W_Buffer.descr_getitem), __setitem__ = interp2app(W_Buffer.descr_setitem), @@ -176,7 +175,6 @@ def buffer_w(self, space): return self.buf - @staticmethod def descr_new(space, w_subtype, w_object): w_memoryview = W_MemoryView(space.buffer_w(w_object)) return w_memoryview @@ -298,7 +296,7 @@ __doc__ = """\ Create a new memoryview object which references the given object. """, - __new__ = interp2app(W_MemoryView.descr_new), + __new__ = interp2app(W_MemoryView.descr_new.im_func), __eq__ = interp2app(W_MemoryView.descr_eq), __ge__ = interp2app(W_MemoryView.descr_ge), __getitem__ = interp2app(W_MemoryView.descr_getitem), From noreply at buildbot.pypy.org Tue Mar 18 19:57:06 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 18 Mar 2014 19:57:06 +0100 (CET) Subject: [pypy-commit] pypy default: seems these do need to be unique, even in separate classes Message-ID: <20140318185706.AFD861D26D3@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r70061:9d175a0294eb Date: 2014-03-18 14:56 -0400 http://bitbucket.org/pypy/pypy/changeset/9d175a0294eb/ Log: seems these do need to be unique, even in separate classes diff --git a/pypy/module/__builtin__/interp_memoryview.py b/pypy/module/__builtin__/interp_memoryview.py --- a/pypy/module/__builtin__/interp_memoryview.py +++ b/pypy/module/__builtin__/interp_memoryview.py @@ -42,8 +42,9 @@ def buffer_w(self, space): return self.buf + @staticmethod @unwrap_spec(offset=int, size=int) - def descr_new(space, w_subtype, w_object, offset=0, size=-1): + def descr_new_buffer(space, w_subtype, w_object, offset=0, size=-1): if space.isinstance_w(w_object, space.w_unicode): # unicode objects support the old buffer interface # but not the new buffer interface (change in python 2.7) @@ -144,7 +145,7 @@ start of the object (or at the specified offset). The slice will extend to the end of the target object (or with the specified size). """, - __new__ = interp2app(W_Buffer.descr_new.im_func), + __new__ = interp2app(W_Buffer.descr_new_buffer), __len__ = interp2app(W_Buffer.descr_len), __getitem__ = interp2app(W_Buffer.descr_getitem), __setitem__ = interp2app(W_Buffer.descr_setitem), @@ -175,9 +176,9 @@ def buffer_w(self, space): return self.buf - def descr_new(space, w_subtype, w_object): - w_memoryview = W_MemoryView(space.buffer_w(w_object)) - return w_memoryview + @staticmethod + def descr_new_memoryview(space, w_subtype, w_object): + return W_MemoryView(space.buffer_w(w_object)) def _make_descr__cmp(name): def descr__cmp(self, space, w_other): @@ -296,7 +297,7 @@ __doc__ = """\ Create a new memoryview object which references the given object. """, - __new__ = interp2app(W_MemoryView.descr_new.im_func), + __new__ = interp2app(W_MemoryView.descr_new_memoryview), __eq__ = interp2app(W_MemoryView.descr_eq), __ge__ = interp2app(W_MemoryView.descr_ge), __getitem__ = interp2app(W_MemoryView.descr_getitem), From noreply at buildbot.pypy.org Tue Mar 18 20:25:34 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 18 Mar 2014 20:25:34 +0100 (CET) Subject: [pypy-commit] pypy default: test/fix buffer init Message-ID: <20140318192534.697C71D2824@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r70062:3c1786c06a69 Date: 2014-03-18 15:24 -0400 http://bitbucket.org/pypy/pypy/changeset/3c1786c06a69/ Log: test/fix buffer init diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -195,11 +195,6 @@ return None def buffer_w(self, space): - w_impl = space.lookup(self, '__buffer__') - if w_impl is not None: - w_result = space.get_and_call_function(w_impl, self) - if space.isinstance_w(w_result, space.w_buffer): - return w_result.buf self._typed_unwrap_error(space, "buffer") def str_w(self, space): diff --git a/pypy/module/__builtin__/test/test_buffer.py b/pypy/module/__builtin__/test/test_buffer.py --- a/pypy/module/__builtin__/test/test_buffer.py +++ b/pypy/module/__builtin__/test/test_buffer.py @@ -4,6 +4,12 @@ class AppTestBuffer: spaceconfig = dict(usemodules=['array']) + def test_init(self): + class A(object): + def __buffer__(self): + return buffer('123') + raises(TypeError, buffer, A()) + def test_unicode_buffer(self): import sys b = buffer(u"ab") From noreply at buildbot.pypy.org Tue Mar 18 22:27:22 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 18 Mar 2014 22:27:22 +0100 (CET) Subject: [pypy-commit] pypy default: more buffer cleanups Message-ID: <20140318212722.04DA91C0166@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r70063:922c7ee9e9d2 Date: 2014-03-18 17:15 -0400 http://bitbucket.org/pypy/pypy/changeset/922c7ee9e9d2/ Log: more buffer cleanups diff --git a/lib-python/2.7/test/test_memoryview.py b/lib-python/2.7/test/test_memoryview.py --- a/lib-python/2.7/test/test_memoryview.py +++ b/lib-python/2.7/test/test_memoryview.py @@ -171,7 +171,7 @@ # very inconsisten on CPython. In PyPy, memoryview supports # the buffer interface, and thus the following comparison # succeeds. See also the comment in - # pypy.modules.__builtin__.interp_memoryview.W_MemoryView.descr_buffer + # pypy.objspace.std.memoryview.W_MemoryView.descr_buffer # # Comparison with objects which don't support the buffer API self.assertFalse(m == u"abcdef", "%s %s" % (self, tp)) diff --git a/pypy/interpreter/test/test_buffer.py b/pypy/interpreter/test/test_buffer.py --- a/pypy/interpreter/test/test_buffer.py +++ b/pypy/interpreter/test/test_buffer.py @@ -1,12 +1,10 @@ import py -from pypy.module.__builtin__.interp_memoryview import W_Buffer from rpython.tool.udir import udir testdir = udir.ensure('test_buffer', dir=1) class TestBuffer: - def test_buffer_w(self): space = self.space w_hello = space.wrap('hello world') @@ -14,14 +12,14 @@ assert buf.getlength() == 11 assert buf.as_str() == 'hello world' assert buf.getslice(1, 6, 1, 5) == 'ello ' - assert space.buffer_w(W_Buffer(buf)) is buf + assert space.buffer_w(space.newbuffer(buf)) is buf assert space.bufferstr_w(w_hello) == 'hello world' - assert space.bufferstr_w(W_Buffer(space.buffer_w(w_hello))) == 'hello world' + assert space.bufferstr_w(space.newbuffer(space.buffer_w(w_hello))) == 'hello world' space.raises_w(space.w_TypeError, space.buffer_w, space.wrap(5)) def test_file_write(self): space = self.space - w_buffer = W_Buffer(space.buffer_w(space.wrap('hello world'))) + w_buffer = space.newbuffer(space.buffer_w(space.wrap('hello world'))) filename = str(testdir.join('test_file_write')) space.appexec([w_buffer, space.wrap(filename)], """(buffer, filename): f = open(filename, 'wb') @@ -42,4 +40,4 @@ space.bufferstr_w, space.wrap(u'\xe9')) -# Note: some app-level tests for buffer are in module/__builtin__/test/. +# Note: some app-level tests for buffer are in objspace/std/test/test_memoryview.py. diff --git a/pypy/module/__builtin__/__init__.py b/pypy/module/__builtin__/__init__.py --- a/pypy/module/__builtin__/__init__.py +++ b/pypy/module/__builtin__/__init__.py @@ -33,16 +33,11 @@ interpleveldefs = { # constants + '__debug__' : '(space.w_True)', # XXX 'None' : '(space.w_None)', 'False' : '(space.w_False)', 'True' : '(space.w_True)', - '__debug__' : '(space.w_True)', # XXX - 'type' : '(space.w_type)', - 'object' : '(space.w_object)', 'bytes' : '(space.w_str)', - 'unicode' : '(space.w_unicode)', - 'buffer' : 'interp_memoryview.W_Buffer', - 'memoryview' : 'interp_memoryview.W_MemoryView', 'file' : 'state.get(space).w_file', 'open' : 'state.get(space).w_file', diff --git a/pypy/module/__builtin__/test/test_builtin.py b/pypy/module/__builtin__/test/test_builtin.py --- a/pypy/module/__builtin__/test/test_builtin.py +++ b/pypy/module/__builtin__/test/test_builtin.py @@ -24,6 +24,17 @@ else: cls.w_safe_runtimerror = cls.space.wrap(sys.version_info < (2, 6)) + def test_builtin_names(self): + import __builtin__ + assert __builtin__.None is None + assert __builtin__.False is False + assert __builtin__.True is True + + assert __builtin__.buffer is buffer + assert __builtin__.bytes is str + assert __builtin__.dict is dict + assert __builtin__.memoryview is memoryview + def test_bytes_alias(self): assert bytes is str assert isinstance(eval("b'hi'"), str) diff --git a/pypy/module/__pypy__/bytebuffer.py b/pypy/module/__pypy__/bytebuffer.py --- a/pypy/module/__pypy__/bytebuffer.py +++ b/pypy/module/__pypy__/bytebuffer.py @@ -4,11 +4,9 @@ from pypy.interpreter.buffer import RWBuffer from pypy.interpreter.gateway import unwrap_spec -from pypy.module.__builtin__.interp_memoryview import W_Buffer class ByteBuffer(RWBuffer): - def __init__(self, len): self.data = ['\x00'] * len @@ -24,4 +22,4 @@ @unwrap_spec(length=int) def bytebuffer(space, length): - return W_Buffer(ByteBuffer(length)) + return space.newbuffer(ByteBuffer(length)) diff --git a/pypy/module/_cffi_backend/cbuffer.py b/pypy/module/_cffi_backend/cbuffer.py --- a/pypy/module/_cffi_backend/cbuffer.py +++ b/pypy/module/_cffi_backend/cbuffer.py @@ -4,7 +4,7 @@ from pypy.interpreter.gateway import unwrap_spec, interp2app from pypy.interpreter.typedef import TypeDef, make_weakref_descr from pypy.module._cffi_backend import cdataobj, ctypeptr, ctypearray -from pypy.module.__builtin__.interp_memoryview import W_Buffer +from pypy.objspace.std.memoryview import W_Buffer from rpython.rtyper.annlowlevel import llstr from rpython.rtyper.lltypesystem import rffi diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py --- a/pypy/module/_io/interp_bufferedio.py +++ b/pypy/module/_io/interp_bufferedio.py @@ -5,7 +5,6 @@ TypeDef, GetSetProperty, generic_new_descr, interp_attrproperty_w) from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault from pypy.interpreter.buffer import RWBuffer -from pypy.module.__builtin__.interp_memoryview import W_Buffer from rpython.rlib.rstring import StringBuilder from rpython.rlib.rarithmetic import r_longlong, intmask from rpython.rlib import rposix @@ -512,7 +511,7 @@ def _raw_read(self, space, buffer, start, length): length = intmask(length) - w_buf = W_Buffer(RawBuffer(buffer, start, length)) + w_buf = space.newbuffer(RawBuffer(buffer, start, length)) while True: try: w_size = space.call_method(self.w_raw, "readinto", w_buf) diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -22,10 +22,10 @@ from pypy.interpreter.nestedscope import Cell from pypy.interpreter.module import Module from pypy.interpreter.function import StaticMethod +from pypy.objspace.std.memoryview import W_MemoryView from pypy.objspace.std.sliceobject import W_SliceObject from pypy.module.__builtin__.descriptor import W_Property from pypy.module.__builtin__.interp_classobj import W_ClassObject -from pypy.module.__builtin__.interp_memoryview import W_MemoryView from pypy.module.micronumpy.base import W_NDimArray from rpython.rlib.entrypoint import entrypoint_lowlevel from rpython.rlib.rposix import is_valid_fd, validate_fd diff --git a/pypy/module/cpyext/bufferobject.py b/pypy/module/cpyext/bufferobject.py --- a/pypy/module/cpyext/bufferobject.py +++ b/pypy/module/cpyext/bufferobject.py @@ -1,12 +1,12 @@ from rpython.rtyper.lltypesystem import rffi, lltype +from pypy.interpreter.buffer import StringBuffer, SubBuffer +from pypy.interpreter.error import OperationError from pypy.module.cpyext.api import ( cpython_api, Py_ssize_t, cpython_struct, bootstrap_function, PyObjectFields, PyObject) from pypy.module.cpyext.pyobject import make_typedescr, Py_DecRef, make_ref -from pypy.interpreter.buffer import StringBuffer, SubBuffer -from pypy.interpreter.error import OperationError -from pypy.module.__builtin__.interp_memoryview import W_Buffer from pypy.module.array.interp_array import ArrayBuffer +from pypy.objspace.std.memoryview import W_Buffer PyBufferObjectStruct = lltype.ForwardReference() diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -14,7 +14,6 @@ from pypy.module.cpyext.pyobject import from_ref from pypy.module.cpyext.pyerrors import PyErr_Occurred from pypy.module.cpyext.state import State -from pypy.module.__builtin__.interp_memoryview import W_Buffer from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter.buffer import Buffer from pypy.interpreter.argument import Arguments @@ -250,7 +249,7 @@ size = generic_cpy_call(space, func_target, w_self, index, ptr) if size < 0: space.fromcache(State).check_and_raise_exception(always=True) - return W_Buffer(CPyBuffer(ptr[0], size, w_self)) + return space.newbuffer(CPyBuffer(ptr[0], size, w_self)) def get_richcmp_func(OP_CONST): def inner(space, w_self, w_args, func): diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py --- a/pypy/module/micronumpy/ndarray.py +++ b/pypy/module/micronumpy/ndarray.py @@ -7,7 +7,6 @@ from rpython.rlib.rawstorage import RAW_STORAGE_PTR from rpython.rtyper.lltypesystem import rffi from rpython.tool.sourcetools import func_with_new_name -from pypy.module.__builtin__.interp_memoryview import W_Buffer from pypy.module.micronumpy import descriptor, ufuncs, boxes, arrayops, loop, \ support, constants as NPY from pypy.module.micronumpy.appbridge import get_appbridge_cache @@ -607,7 +606,7 @@ return self.implementation.get_buffer(space) def descr_get_data(self, space): - return W_Buffer(self.buffer_w(space)) + return space.newbuffer(self.buffer_w(space)) @unwrap_spec(offset=int, axis1=int, axis2=int) def descr_diagonal(self, space, offset=0, axis1=0, axis2=1): diff --git a/pypy/module/__builtin__/interp_memoryview.py b/pypy/objspace/std/memoryview.py rename from pypy/module/__builtin__/interp_memoryview.py rename to pypy/objspace/std/memoryview.py --- a/pypy/module/__builtin__/interp_memoryview.py +++ b/pypy/objspace/std/memoryview.py @@ -37,6 +37,7 @@ """ def __init__(self, buf): + assert isinstance(buf, buffer.Buffer) self.buf = buf def buffer_w(self, space): @@ -171,6 +172,7 @@ """ def __init__(self, buf): + assert isinstance(buf, buffer.Buffer) self.buf = buf def buffer_w(self, space): diff --git a/pypy/objspace/std/model.py b/pypy/objspace/std/model.py --- a/pypy/objspace/std/model.py +++ b/pypy/objspace/std/model.py @@ -63,7 +63,7 @@ from pypy.objspace.std import unicodeobject from pypy.objspace.std import dictproxyobject from pypy.objspace.std import proxyobject - from pypy.module.__builtin__.interp_memoryview import W_Buffer + from pypy.objspace.std.memoryview import W_Buffer, W_MemoryView import pypy.objspace.std.default # register a few catch-all multimethods import pypy.objspace.std.marshal_impl # install marshal multimethods @@ -84,6 +84,7 @@ self.pythontypes.append(boolobject.W_BoolObject.typedef) self.pythontypes.append(longobject.W_LongObject.typedef) self.pythontypes.append(W_Buffer.typedef) + self.pythontypes.append(W_MemoryView.typedef) # the set of implementation types self.typeorder = { diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py --- a/pypy/objspace/std/objspace.py +++ b/pypy/objspace/std/objspace.py @@ -24,6 +24,7 @@ from pypy.objspace.std.iterobject import W_AbstractSeqIterObject from pypy.objspace.std.listobject import W_ListObject from pypy.objspace.std.longobject import W_LongObject, newlong +from pypy.objspace.std.memoryview import W_Buffer from pypy.objspace.std.noneobject import W_NoneObject from pypy.objspace.std.objectobject import W_ObjectObject from pypy.objspace.std.iterobject import W_SeqIterObject @@ -314,6 +315,9 @@ def newseqiter(self, w_obj): return W_SeqIterObject(w_obj) + def newbuffer(self, w_obj): + return W_Buffer(w_obj) + def type(self, w_obj): jit.promote(w_obj.__class__) return w_obj.getclass(self) diff --git a/pypy/module/__builtin__/test/test_buffer.py b/pypy/objspace/std/test/test_memoryview.py rename from pypy/module/__builtin__/test/test_buffer.py rename to pypy/objspace/std/test/test_memoryview.py --- a/pypy/module/__builtin__/test/test_buffer.py +++ b/pypy/objspace/std/test/test_memoryview.py @@ -1,6 +1,3 @@ -"""Tests some behaviour of the buffer type that is not tested in -lib-python/2.5.2/test/test_types.py where the stdlib buffer tests live.""" - class AppTestBuffer: spaceconfig = dict(usemodules=['array']) From noreply at buildbot.pypy.org Tue Mar 18 22:32:50 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 18 Mar 2014 22:32:50 +0100 (CET) Subject: [pypy-commit] pypy default: fake objspace needs newbuffer also Message-ID: <20140318213250.588641C0166@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r70064:95ca8c340008 Date: 2014-03-18 17:32 -0400 http://bitbucket.org/pypy/pypy/changeset/95ca8c340008/ Log: fake objspace needs newbuffer also diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py --- a/pypy/objspace/fake/objspace.py +++ b/pypy/objspace/fake/objspace.py @@ -168,6 +168,9 @@ def newseqiter(self, x): return w_some_obj() + def newbuffer(self, x): + return w_some_obj() + def marshal_w(self, w_obj): "NOT_RPYTHON" raise NotImplementedError From noreply at buildbot.pypy.org Tue Mar 18 22:51:29 2014 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 18 Mar 2014 22:51:29 +0100 (CET) Subject: [pypy-commit] buildbot default: add SalsaSalsa_own Message-ID: <20140318215129.781B21D2808@cobra.cs.uni-duesseldorf.de> Author: Matti Picus Branch: Changeset: r913:8fd3d5a743e3 Date: 2014-03-18 23:51 +0200 http://bitbucket.org/pypy/buildbot/changeset/8fd3d5a743e3/ Log: add SalsaSalsa_own diff --git a/bot2/pypybuildbot/master.py b/bot2/pypybuildbot/master.py --- a/bot2/pypybuildbot/master.py +++ b/bot2/pypybuildbot/master.py @@ -385,7 +385,7 @@ 'category' : 'mac64', }, {"name": WIN32, - "slavenames": ["aurora", "SalsaSalsa"], + "slavenames": ["aurora", "SalsaSalsa_own"], "builddir": WIN32, "factory": pypyOwnTestFactoryWin, "locks": [WinSlaveLock.access('counting')], From noreply at buildbot.pypy.org Tue Mar 18 22:55:33 2014 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 18 Mar 2014 22:55:33 +0100 (CET) Subject: [pypy-commit] buildbot default: Backed out changeset: 8fd3d5a743e3 Message-ID: <20140318215533.688041D2818@cobra.cs.uni-duesseldorf.de> Author: Matti Picus Branch: Changeset: r914:aaf1be5bce44 Date: 2014-03-18 23:55 +0200 http://bitbucket.org/pypy/buildbot/changeset/aaf1be5bce44/ Log: Backed out changeset: 8fd3d5a743e3 diff --git a/bot2/pypybuildbot/master.py b/bot2/pypybuildbot/master.py --- a/bot2/pypybuildbot/master.py +++ b/bot2/pypybuildbot/master.py @@ -385,7 +385,7 @@ 'category' : 'mac64', }, {"name": WIN32, - "slavenames": ["aurora", "SalsaSalsa_own"], + "slavenames": ["aurora", "SalsaSalsa"], "builddir": WIN32, "factory": pypyOwnTestFactoryWin, "locks": [WinSlaveLock.access('counting')], From noreply at buildbot.pypy.org Tue Mar 18 23:08:32 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 18 Mar 2014 23:08:32 +0100 (CET) Subject: [pypy-commit] pypy default: kill unused StringLikeBuffer Message-ID: <20140318220832.D1A901D2818@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r70065:b0ba9ef7bb1d Date: 2014-03-18 18:07 -0400 http://bitbucket.org/pypy/pypy/changeset/b0ba9ef7bb1d/ Log: kill unused StringLikeBuffer diff --git a/pypy/interpreter/buffer.py b/pypy/interpreter/buffer.py --- a/pypy/interpreter/buffer.py +++ b/pypy/interpreter/buffer.py @@ -7,8 +7,7 @@ class Buffer(object): """Abstract base class for buffers.""" - - __slots__ = () # no extra slot here + __slots__ = [] def getlength(self): raise NotImplementedError @@ -29,14 +28,13 @@ def get_raw_address(self): raise ValueError("no raw buffer") - def is_writable(self): return False + class RWBuffer(Buffer): """Abstract base class for read-write buffers.""" - - __slots__ = () # no extra slot here + __slots__ = [] def is_writable(self): return True @@ -51,10 +49,8 @@ self.setitem(start + i, string[i]) - -# ____________________________________________________________ - class StringBuffer(Buffer): + __slots__ = ['value'] def __init__(self, value): self.value = value @@ -76,42 +72,11 @@ return self.value[start:stop] return "".join([self.value[start + i*step] for i in xrange(size)]) - -class StringLikeBuffer(Buffer): - """For app-level objects that already have a string-like interface - with __len__ and a __getitem__ that returns characters or (with - slicing) substrings.""" - # XXX this is inefficient, it should only be used temporarily - - def __init__(self, space, w_obj): - self.space = space - self.w_obj = w_obj - - def getlength(self): - space = self.space - return space.len_w(self.w_obj) - - def getitem(self, index): - space = self.space - s = space.str_w(space.getitem(self.w_obj, space.wrap(index))) - if len(s) != 1: - raise OperationError(space.w_ValueError, - space.wrap("character expected, got string")) - char = s[0] # annotator hint - return char - - def getslice(self, start, stop, step, size): - space = self.space - if step != 1: - raise OperationError(space.w_ValueError, space.wrap( - "buffer object does not support slicing with a step")) - s = space.str_w(space.getslice(self.w_obj, space.wrap(start), - space.wrap(stop))) - return s - # ____________________________________________________________ class SubBufferMixin(object): + _attrs_ = ['buffer', 'offset', 'size'] + def __init__(self, buffer, offset, size): self.buffer = buffer self.offset = offset @@ -135,9 +100,11 @@ # out of bounds return self.buffer.getslice(self.offset + start, self.offset + stop, step, size) + class SubBuffer(Buffer): import_from_mixin(SubBufferMixin) + class RWSubBuffer(RWBuffer): import_from_mixin(SubBufferMixin) From noreply at buildbot.pypy.org Tue Mar 18 23:13:42 2014 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 18 Mar 2014 23:13:42 +0100 (CET) Subject: [pypy-commit] pypy default: fix compilation error that caused msvc to follow mingw32 compile path, with no difference in final error mapping Message-ID: <20140318221342.AF3021C0166@cobra.cs.uni-duesseldorf.de> Author: Matti Picus Branch: Changeset: r70066:f2d5c5bf7675 Date: 2014-03-18 23:01 +0200 http://bitbucket.org/pypy/pypy/changeset/f2d5c5bf7675/ Log: fix compilation error that caused msvc to follow mingw32 compile path, with no difference in final error mapping diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py --- a/rpython/rlib/rwin32.py +++ b/rpython/rlib/rwin32.py @@ -169,6 +169,7 @@ cfile = udir.join('dosmaperr.c') cfile.write(r''' #include + #include #include #ifdef __GNUC__ #define _dosmaperr mingw_dosmaperr From noreply at buildbot.pypy.org Tue Mar 18 23:22:21 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 18 Mar 2014 23:22:21 +0100 (CET) Subject: [pypy-commit] pypy default: sanity check on platform when swallowing this exception Message-ID: <20140318222221.3A92D1C0166@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r70067:90f48765ab97 Date: 2014-03-18 15:21 -0700 http://bitbucket.org/pypy/pypy/changeset/90f48765ab97/ Log: sanity check on platform when swallowing this exception diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py --- a/rpython/rlib/rwin32.py +++ b/rpython/rlib/rwin32.py @@ -170,7 +170,7 @@ cfile.write(r''' #include #include - #include + #include #ifdef __GNUC__ #define _dosmaperr mingw_dosmaperr #endif @@ -197,6 +197,7 @@ standalone=True) except (CompilationError, WindowsError): # Fallback for the mingw32 compiler + assert static_platform.name == 'mingw32' errors = { 2: 2, 3: 2, 4: 24, 5: 13, 6: 9, 7: 12, 8: 12, 9: 12, 10: 7, 11: 8, 15: 2, 16: 13, 17: 18, 18: 2, 19: 13, 20: 13, 21: 13, From noreply at buildbot.pypy.org Tue Mar 18 23:24:17 2014 From: noreply at buildbot.pypy.org (pjenvey) Date: Tue, 18 Mar 2014 23:24:17 +0100 (CET) Subject: [pypy-commit] pypy py3k: merge default Message-ID: <20140318222417.B44D81C0166@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r70068:82be71463fe1 Date: 2014-03-18 15:15 -0700 http://bitbucket.org/pypy/pypy/changeset/82be71463fe1/ Log: merge default diff too long, truncating to 2000 out of 3534 lines diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py --- a/lib_pypy/_sqlite3.py +++ b/lib_pypy/_sqlite3.py @@ -38,6 +38,7 @@ if sys.version_info[0] >= 3: StandardError = Exception + cmp = lambda x, y: (x > y) - (x < y) long = int xrange = range basestring = unicode = str diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -105,3 +105,12 @@ .. branch: stdlib-2.7.6 Update stdlib to v2.7.6 + +.. branch: virtual-raw-store-load +Support for virtualizing raw_store/raw_load operations + +.. branch: refactor-buffer-api +Separate the interp-level buffer API from the buffer type exposed to +app-level. The `Buffer` class is now used by `W_MemoryView` and +`W_Buffer`, which is not present in Python 3. Previously `W_Buffer` was +an alias to `Buffer`, which was wrappable itself. diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -193,6 +193,14 @@ def immutable_unique_id(self, space): return None + def buffer_w(self, space): + w_impl = space.lookup(self, '__buffer__') + if w_impl is not None: + w_result = space.get_and_call_function(w_impl, self) + if space.isinstance_w(w_result, space.w_buffer): + return w_result.buf + self._typed_unwrap_error(space, "buffer") + def bytes_w(self, space): self._typed_unwrap_error(space, "bytes") @@ -436,14 +444,12 @@ def getbuiltinmodule(self, name, force_init=False): w_name = self.wrap(name) w_modules = self.sys.get('modules') - try: - w_mod = self.getitem(w_modules, w_name) - except OperationError, e: - if not e.match(self, self.w_KeyError): - raise - else: - if not force_init: - return w_mod + if not force_init: + try: + return self.getitem(w_modules, w_name) + except OperationError, e: + if not e.match(self, self.w_KeyError): + raise # If the module is a builtin but not yet imported, # retrieve it and initialize it @@ -454,13 +460,13 @@ "getbuiltinmodule() called with non-builtin module %s", name) else: - # Add the module to sys.modules - self.setitem(w_modules, w_name, w_mod) - # And initialize it from pypy.interpreter.module import Module if isinstance(w_mod, Module): w_mod.init(self) + + # Add the module to sys.modules + self.setitem(w_modules, w_name, w_mod) return w_mod def get_builtinmodule_to_install(self): @@ -1307,10 +1313,7 @@ 'to unsigned int')) def buffer_w(self, w_obj): - # returns a Buffer instance - from pypy.interpreter.buffer import Buffer - w_buffer = self.buffer(w_obj) - return self.interp_w(Buffer, w_buffer) + return w_obj.buffer_w(self) def rwbuffer_w(self, w_obj): # returns a RWBuffer instance @@ -1727,7 +1730,6 @@ ('set', 'set', 3, ['__set__']), ('delete', 'delete', 2, ['__delete__']), ('userdel', 'del', 1, ['__del__']), - ('buffer', 'buffer', 1, ['__buffer__']), # see buffer.py ] ObjSpace.BuiltinModuleTable = [ diff --git a/pypy/interpreter/buffer.py b/pypy/interpreter/buffer.py --- a/pypy/interpreter/buffer.py +++ b/pypy/interpreter/buffer.py @@ -1,30 +1,12 @@ """ Buffer protocol support. """ +from pypy.interpreter.error import OperationError +from rpython.rlib.objectmodel import import_from_mixin -# The implementation of the buffer protocol. The basic idea is that we -# can ask any app-level object for a 'buffer' view on it, by calling its -# __buffer__() special method. It should return a wrapped instance of a -# subclass of the Buffer class defined below. Note that __buffer__() is -# a PyPy-only extension to the Python language, made necessary by the -# fact that it's not natural in PyPy to hack an interp-level-only -# interface. -# In normal usage, the convenience method space.buffer_w() should be -# used to get directly a Buffer instance. Doing so also gives you for -# free the typecheck that __buffer__() really returned a wrapped Buffer. - -import operator -from pypy.interpreter.baseobjspace import W_Root -from pypy.interpreter.typedef import TypeDef -from pypy.interpreter.gateway import interp2app, unwrap_spec -from pypy.interpreter.error import OperationError -from rpython.rlib.objectmodel import compute_hash, import_from_mixin -from rpython.rlib.rstring import StringBuilder - - -class Buffer(W_Root): - """Abstract base class for memory views.""" +class Buffer(object): + """Abstract base class for buffers.""" __slots__ = () # no extra slot here @@ -47,94 +29,12 @@ def get_raw_address(self): raise ValueError("no raw buffer") + def is_writable(self): return False - # __________ app-level support __________ - - def descr_len(self, space): - return space.wrap(self.getlength()) - - def descr_getitem(self, space, w_index): - start, stop, step, size = space.decode_index4(w_index, self.getlength()) - if step == 0: # index only - return space.wrapbytes(self.getitem(start)) - res = self.getslice(start, stop, step, size) - return space.wrapbytes(res) - - @unwrap_spec(newstring='bufferstr') - def descr_setitem(self, space, w_index, newstring): - if not isinstance(self, RWBuffer): - raise OperationError(space.w_TypeError, - space.wrap("buffer is read-only")) - start, stop, step, size = space.decode_index4(w_index, self.getlength()) - if step == 0: # index only - if len(newstring) != 1: - msg = 'buffer[index]=x: x must be a single character' - raise OperationError(space.w_TypeError, space.wrap(msg)) - char = newstring[0] # annotator hint - self.setitem(start, char) - elif step == 1: - if len(newstring) != size: - msg = "right operand length must match slice length" - raise OperationError(space.w_ValueError, space.wrap(msg)) - self.setslice(start, newstring) - else: - raise OperationError(space.w_ValueError, - space.wrap("buffer object does not support" - " slicing with a step")) - - def descr__buffer__(self, space): - return space.wrap(self) - - def descr_str(self, space): - return space.wrap(self.as_str()) - - @unwrap_spec(other='bufferstr') - def descr_add(self, space, other): - return space.wrapbytes(self.as_str() + other) - - def _make_descr__cmp(name): - def descr__cmp(self, space, w_other): - if not isinstance(w_other, Buffer): - return space.w_NotImplemented - # xxx not the most efficient implementation - str1 = self.as_str() - str2 = w_other.as_str() - return space.wrap(getattr(operator, name)(str1, str2)) - descr__cmp.func_name = name - return descr__cmp - - descr_eq = _make_descr__cmp('eq') - descr_ne = _make_descr__cmp('ne') - descr_lt = _make_descr__cmp('lt') - descr_le = _make_descr__cmp('le') - descr_gt = _make_descr__cmp('gt') - descr_ge = _make_descr__cmp('ge') - - def descr_hash(self, space): - return space.wrap(compute_hash(self.as_str())) - - def descr_mul(self, space, w_times): - # xxx not the most efficient implementation - w_string = space.wrapbytes(self.as_str()) - # use the __mul__ method instead of space.mul() so that we - # return NotImplemented instead of raising a TypeError - return space.call_method(w_string, '__mul__', w_times) - - def descr_repr(self, space): - if isinstance(self, RWBuffer): - info = 'read-write buffer' - else: - info = 'read-only buffer' - addrstring = self.getaddrstring(space) - - return space.wrap("<%s for 0x%s, size %d>" % - (info, addrstring, self.getlength())) - - class RWBuffer(Buffer): - """Abstract base class for read-write memory views.""" + """Abstract base class for read-write buffers.""" __slots__ = () # no extra slot here @@ -151,72 +51,6 @@ self.setitem(start + i, string[i]) - at unwrap_spec(offset=int, size=int) -def descr_buffer__new__(space, w_subtype, w_object, offset=0, size=-1): - # w_subtype can only be exactly 'buffer' for now - if not space.is_w(w_subtype, space.gettypefor(Buffer)): - raise OperationError(space.w_TypeError, - space.wrap("argument 1 must be 'buffer'")) - - if space.isinstance_w(w_object, space.w_unicode): - # unicode objects support the old buffer interface - # but not the new buffer interface (change in python 2.7) - from rpython.rlib.rstruct.unichar import pack_unichar, UNICODE_SIZE - unistr = space.unicode_w(w_object) - builder = StringBuilder(len(unistr) * UNICODE_SIZE) - for unich in unistr: - pack_unichar(unich, builder) - from pypy.interpreter.buffer import StringBuffer - w_buffer = space.wrap(StringBuffer(builder.build())) - else: - w_buffer = space.buffer(w_object) - - buffer = space.interp_w(Buffer, w_buffer) # type-check - if offset == 0 and size == -1: - return w_buffer - # handle buffer slices - if offset < 0: - raise OperationError(space.w_ValueError, - space.wrap("offset must be zero or positive")) - if size < -1: - raise OperationError(space.w_ValueError, - space.wrap("size must be zero or positive")) - if isinstance(buffer, RWBuffer): - buffer = RWSubBuffer(buffer, offset, size) - else: - buffer = SubBuffer(buffer, offset, size) - return space.wrap(buffer) - - -Buffer.typedef = TypeDef( - "buffer", - __doc__ = """\ -buffer(object [, offset[, size]]) - -Create a new buffer object which references the given object. -The buffer will reference a slice of the target object from the -start of the object (or at the specified offset). The slice will -extend to the end of the target object (or with the specified size). -""", - __new__ = interp2app(descr_buffer__new__), - __len__ = interp2app(Buffer.descr_len), - __getitem__ = interp2app(Buffer.descr_getitem), - __setitem__ = interp2app(Buffer.descr_setitem), - __buffer__ = interp2app(Buffer.descr__buffer__), - __str__ = interp2app(Buffer.descr_str), - __add__ = interp2app(Buffer.descr_add), - __eq__ = interp2app(Buffer.descr_eq), - __ne__ = interp2app(Buffer.descr_ne), - __lt__ = interp2app(Buffer.descr_lt), - __le__ = interp2app(Buffer.descr_le), - __gt__ = interp2app(Buffer.descr_gt), - __ge__ = interp2app(Buffer.descr_ge), - __hash__ = interp2app(Buffer.descr_hash), - __mul__ = interp2app(Buffer.descr_mul), - __rmul__ = interp2app(Buffer.descr_mul), - __repr__ = interp2app(Buffer.descr_repr), -) -Buffer.typedef.acceptable_as_base_class = False # ____________________________________________________________ diff --git a/pypy/interpreter/test/test_buffer.py b/pypy/interpreter/test/test_buffer.py --- a/pypy/interpreter/test/test_buffer.py +++ b/pypy/interpreter/test/test_buffer.py @@ -1,5 +1,5 @@ import py -from pypy.interpreter.buffer import Buffer +from pypy.module.__builtin__.interp_memoryview import W_Buffer from rpython.tool.udir import udir testdir = udir.ensure('test_buffer', dir=1) @@ -11,13 +11,12 @@ space = self.space w_hello = space.wrapbytes('hello world') buf = space.buffer_w(w_hello) - assert isinstance(buf, Buffer) assert buf.getlength() == 11 assert buf.as_str() == 'hello world' assert buf.getslice(1, 6, 1, 5) == 'ello ' - assert space.buffer_w(space.wrap(buf)) is buf + assert space.buffer_w(W_Buffer(buf)) is buf assert space.bufferstr_w(w_hello) == 'hello world' - assert space.bufferstr_w(space.buffer(w_hello)) == 'hello world' + assert space.bufferstr_w(W_Buffer(space.buffer_w(w_hello))) == 'hello world' space.raises_w(space.w_TypeError, space.buffer_w, space.wrap(5)) e = space.raises_w(space.w_TypeError, space.buffer, space.wrap(5)) message = space.unwrap(e.value.get_w_value(space)) @@ -25,7 +24,7 @@ def test_file_write(self): space = self.space - w_buffer = space.buffer(space.wrapbytes('hello world')) + w_buffer = W_Buffer(space.buffer_w(space.wrapbytes('hello world'))) filename = str(testdir.join('test_file_write')) space.appexec([w_buffer, space.wrap(filename)], """(buffer, filename): f = open(filename, 'wb') diff --git a/pypy/module/__builtin__/interp_memoryview.py b/pypy/module/__builtin__/interp_memoryview.py --- a/pypy/module/__builtin__/interp_memoryview.py +++ b/pypy/module/__builtin__/interp_memoryview.py @@ -1,22 +1,194 @@ """ Implementation of the 'buffer' and 'memoryview' types. """ +import operator + +from pypy.interpreter import buffer from pypy.interpreter.baseobjspace import W_Root -from pypy.interpreter import buffer +from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.typedef import TypeDef, GetSetProperty -from pypy.interpreter.error import OperationError -import operator +from rpython.rlib.objectmodel import compute_hash +from rpython.rlib.rstring import StringBuilder -class W_MemoryView(W_Root): - """Implement the built-in 'memoryview' type as a thin wrapper around + +def _buffer_setitem(space, buf, w_index, newstring): + start, stop, step, size = space.decode_index4(w_index, buf.getlength()) + if step == 0: # index only + if len(newstring) != 1: + msg = 'buffer[index]=x: x must be a single character' + raise OperationError(space.w_TypeError, space.wrap(msg)) + char = newstring[0] # annotator hint + buf.setitem(start, char) + elif step == 1: + if len(newstring) != size: + msg = "right operand length must match slice length" + raise OperationError(space.w_ValueError, space.wrap(msg)) + buf.setslice(start, newstring) + else: + raise OperationError(space.w_ValueError, + space.wrap("buffer object does not support" + " slicing with a step")) + + +class W_Buffer(W_Root): + """Implement the built-in 'buffer' type as a wrapper around an interp-level buffer. """ def __init__(self, buf): - assert isinstance(buf, buffer.Buffer) self.buf = buf + def buffer_w(self, space): + return self.buf + + @staticmethod + @unwrap_spec(offset=int, size=int) + def descr_new(space, w_subtype, w_object, offset=0, size=-1): + if space.isinstance_w(w_object, space.w_unicode): + # unicode objects support the old buffer interface + # but not the new buffer interface (change in python 2.7) + from rpython.rlib.rstruct.unichar import pack_unichar, UNICODE_SIZE + unistr = space.unicode_w(w_object) + builder = StringBuilder(len(unistr) * UNICODE_SIZE) + for unich in unistr: + pack_unichar(unich, builder) + from pypy.interpreter.buffer import StringBuffer + buf = StringBuffer(builder.build()) + else: + buf = space.buffer_w(w_object) + + if offset == 0 and size == -1: + return W_Buffer(buf) + # handle buffer slices + if offset < 0: + raise OperationError(space.w_ValueError, + space.wrap("offset must be zero or positive")) + if size < -1: + raise OperationError(space.w_ValueError, + space.wrap("size must be zero or positive")) + if isinstance(buf, buffer.RWBuffer): + buf = buffer.RWSubBuffer(buf, offset, size) + else: + buf = buffer.SubBuffer(buf, offset, size) + return W_Buffer(buf) + + def descr_len(self, space): + return space.wrap(self.buf.getlength()) + + def descr_getitem(self, space, w_index): + start, stop, step, size = space.decode_index4(w_index, self.buf.getlength()) + if step == 0: # index only + return space.wrapbytes(self.buf.getitem(start)) + res = self.buf.getslice(start, stop, step, size) + return space.wrapbytes(res) + + @unwrap_spec(newstring='bufferstr') + def descr_setitem(self, space, w_index, newstring): + if not isinstance(self.buf, buffer.RWBuffer): + raise OperationError(space.w_TypeError, + space.wrap("buffer is read-only")) + _buffer_setitem(space, self.buf, w_index, newstring) + + def descr_str(self, space): + return space.wrap(self.buf.as_str()) + + @unwrap_spec(other='bufferstr') + def descr_add(self, space, other): + return space.wrapbytes(self.buf.as_str() + other) + + def _make_descr__cmp(name): + def descr__cmp(self, space, w_other): + if not isinstance(w_other, W_Buffer): + return space.w_NotImplemented + # xxx not the most efficient implementation + str1 = self.buf.as_str() + str2 = w_other.buf.as_str() + return space.wrap(getattr(operator, name)(str1, str2)) + descr__cmp.func_name = name + return descr__cmp + + descr_eq = _make_descr__cmp('eq') + descr_ne = _make_descr__cmp('ne') + descr_lt = _make_descr__cmp('lt') + descr_le = _make_descr__cmp('le') + descr_gt = _make_descr__cmp('gt') + descr_ge = _make_descr__cmp('ge') + + def descr_hash(self, space): + return space.wrap(compute_hash(self.buf.as_str())) + + def descr_mul(self, space, w_times): + # xxx not the most efficient implementation + w_string = space.wrapbytes(self.buf.as_str()) + # use the __mul__ method instead of space.mul() so that we + # return NotImplemented instead of raising a TypeError + return space.call_method(w_string, '__mul__', w_times) + + def descr_repr(self, space): + if isinstance(self.buf, buffer.RWBuffer): + info = 'read-write buffer' + else: + info = 'read-only buffer' + addrstring = self.getaddrstring(space) + + return space.wrap("<%s for 0x%s, size %d>" % + (info, addrstring, self.buf.getlength())) + +W_Buffer.typedef = TypeDef( + "buffer", + __doc__ = """\ +buffer(object [, offset[, size]]) + +Create a new buffer object which references the given object. +The buffer will reference a slice of the target object from the +start of the object (or at the specified offset). The slice will +extend to the end of the target object (or with the specified size). +""", + __new__ = interp2app(W_Buffer.descr_new), + __len__ = interp2app(W_Buffer.descr_len), + __getitem__ = interp2app(W_Buffer.descr_getitem), + __setitem__ = interp2app(W_Buffer.descr_setitem), + __str__ = interp2app(W_Buffer.descr_str), + __add__ = interp2app(W_Buffer.descr_add), + __eq__ = interp2app(W_Buffer.descr_eq), + __ne__ = interp2app(W_Buffer.descr_ne), + __lt__ = interp2app(W_Buffer.descr_lt), + __le__ = interp2app(W_Buffer.descr_le), + __gt__ = interp2app(W_Buffer.descr_gt), + __ge__ = interp2app(W_Buffer.descr_ge), + __hash__ = interp2app(W_Buffer.descr_hash), + __mul__ = interp2app(W_Buffer.descr_mul), + __rmul__ = interp2app(W_Buffer.descr_mul), + __repr__ = interp2app(W_Buffer.descr_repr), +) +W_Buffer.typedef.acceptable_as_base_class = False + + +class W_MemoryView(W_Root): + """Implement the built-in 'memoryview' type as a wrapper around + an interp-level buffer. + """ + + def __init__(self, buf): + self.buf = buf + + def buffer_w(self, space): + """Note that memoryview() is very inconsistent in CPython: it + does not support the buffer interface but does support the new + buffer interface: as a result, it is possible to pass memoryview + to e.g. socket.send() but not to file.write(). For simplicity + and consistency, in PyPy memoryview DOES support buffer(), which + means that it is accepted in more places than CPython. + """ + self._check_released(space) + return self.buf + + @staticmethod + def descr_new(space, w_subtype, w_object): + w_memoryview = W_MemoryView(space.buffer_w(w_object)) + return w_memoryview + def _make_descr__cmp(name): def descr__cmp(self, space, w_other): if self.buf is None: @@ -28,14 +200,14 @@ return space.wrap(getattr(operator, name)(str1, str2)) try: - w_buf = space.buffer(w_other) + buf = space.buffer_w(w_other) except OperationError, e: if not e.match(space, space.w_TypeError): raise return space.w_NotImplemented else: str1 = self.as_str() - str2 = space.buffer_w(w_buf).as_str() + str2 = buf.as_str() return space.wrap(getattr(operator, name)(str1, str2)) descr__cmp.func_name = name return descr__cmp @@ -62,18 +234,6 @@ buf = buffer.SubBuffer(buf, start, size) return W_MemoryView(buf) - def descr_buffer(self, space): - """ - Note that memoryview() is very inconsistent in CPython: it does not - support the buffer interface but does support the new buffer - interface: as a result, it is possible to pass memoryview to - e.g. socket.send() but not to file.write(). For simplicity and - consistency, in PyPy memoryview DOES support buffer(), which means - that it is accepted in more places than CPython. - """ - self._check_released(space) - return space.wrap(self.buf) - def descr_tobytes(self, space): self._check_released(space) return space.wrapbytes(self.as_str()) @@ -102,16 +262,14 @@ @unwrap_spec(newstring='bufferstr') def descr_setitem(self, space, w_index, newstring): self._check_released(space) - buf = self.buf - if isinstance(buf, buffer.RWBuffer): - buf.descr_setitem(space, w_index, newstring) - else: + if not isinstance(self.buf, buffer.RWBuffer): raise OperationError(space.w_TypeError, space.wrap("cannot modify read-only memory")) + _buffer_setitem(space, self.buf, w_index, newstring) def descr_len(self, space): self._check_released(space) - return self.buf.descr_len(space) + return space.wrap(self.buf.getlength()) def w_get_format(self, space): self._check_released(space) @@ -165,17 +323,12 @@ return space.w_None -def descr_new(space, w_subtype, w_object): - memoryview = W_MemoryView(space.buffer(w_object)) - return space.wrap(memoryview) - W_MemoryView.typedef = TypeDef( "memoryview", __doc__ = """\ Create a new memoryview object which references the given object. """, - __new__ = interp2app(descr_new), - __buffer__ = interp2app(W_MemoryView.descr_buffer), + __new__ = interp2app(W_MemoryView.descr_new), __eq__ = interp2app(W_MemoryView.descr_eq), __getitem__ = interp2app(W_MemoryView.descr_getitem), __len__ = interp2app(W_MemoryView.descr_len), diff --git a/pypy/module/__pypy__/bytebuffer.py b/pypy/module/__pypy__/bytebuffer.py --- a/pypy/module/__pypy__/bytebuffer.py +++ b/pypy/module/__pypy__/bytebuffer.py @@ -4,6 +4,7 @@ from pypy.interpreter.buffer import RWBuffer from pypy.interpreter.gateway import unwrap_spec +from pypy.module.__builtin__.interp_memoryview import W_Buffer class ByteBuffer(RWBuffer): @@ -23,4 +24,4 @@ @unwrap_spec(length=int) def bytebuffer(space, length): - return space.wrap(ByteBuffer(length)) + return W_Buffer(ByteBuffer(length)) diff --git a/pypy/module/_cffi_backend/cbuffer.py b/pypy/module/_cffi_backend/cbuffer.py --- a/pypy/module/_cffi_backend/cbuffer.py +++ b/pypy/module/_cffi_backend/cbuffer.py @@ -4,6 +4,7 @@ from pypy.interpreter.gateway import unwrap_spec, interp2app from pypy.interpreter.typedef import TypeDef, make_weakref_descr from pypy.module._cffi_backend import cdataobj, ctypeptr, ctypearray +from pypy.module.__builtin__.interp_memoryview import W_Buffer from rpython.rtyper.annlowlevel import llstr from rpython.rtyper.lltypesystem import rffi @@ -39,35 +40,19 @@ copy_string_to_raw(llstr(string), raw_cdata, 0, len(string)) -class MiniBuffer(W_Root): - # a different subclass of W_Root for the MiniBuffer, because we - # want a slightly different (simplified) API at the level of Python. +# Override the typedef to narrow down the interface that's exposed to app-level +class MiniBuffer(W_Buffer): def __init__(self, buffer, keepalive=None): - self.buffer = buffer + W_Buffer.__init__(self, buffer) self.keepalive = keepalive - def descr_len(self, space): - return self.buffer.descr_len(space) - - def descr_getitem(self, space, w_index): - return self.buffer.descr_getitem(space, w_index) - - @unwrap_spec(newstring='bufferstr') - def descr_setitem(self, space, w_index, newstring): - self.buffer.descr_setitem(space, w_index, newstring) - - def descr__buffer__(self, space): - return self.buffer.descr__buffer__(space) - - MiniBuffer.typedef = TypeDef( "buffer", __module__ = "_cffi_backend", __len__ = interp2app(MiniBuffer.descr_len), __getitem__ = interp2app(MiniBuffer.descr_getitem), __setitem__ = interp2app(MiniBuffer.descr_setitem), - __buffer__ = interp2app(MiniBuffer.descr__buffer__), __weakref__ = make_weakref_descr(MiniBuffer), ) MiniBuffer.typedef.acceptable_as_base_class = False diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py --- a/pypy/module/_io/interp_bufferedio.py +++ b/pypy/module/_io/interp_bufferedio.py @@ -5,6 +5,7 @@ TypeDef, GetSetProperty, generic_new_descr, interp_attrproperty_w) from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault from pypy.interpreter.buffer import RWBuffer +from pypy.module.__builtin__.interp_memoryview import W_Buffer from rpython.rlib.rstring import StringBuilder from rpython.rlib.rarithmetic import r_longlong, intmask from rpython.rlib import rposix @@ -521,7 +522,7 @@ def _raw_read(self, space, buffer, start, length): length = intmask(length) - w_buf = space.wrap(RawBuffer(buffer, start, length)) + w_buf = W_Buffer(RawBuffer(buffer, start, length)) while True: try: w_size = space.call_method(self.w_raw, "readinto", w_buf) diff --git a/pypy/module/_multiprocessing/test/test_connection.py b/pypy/module/_multiprocessing/test/test_connection.py --- a/pypy/module/_multiprocessing/test/test_connection.py +++ b/pypy/module/_multiprocessing/test/test_connection.py @@ -9,8 +9,12 @@ class AppTestBufferTooShort: spaceconfig = {'usemodules': ['_multiprocessing', 'thread', 'signal', - 'select', 'fcntl', 'struct', - 'binascii']} + 'select', 'struct', 'binascii']} + if sys.platform == 'win32': + spaceconfig['usemodules'].append('_rawffi') + else: + spaceconfig['usemodules'].append('fcntl') + def setup_class(cls): if cls.runappdirect: @@ -75,6 +79,8 @@ 'itertools', '_socket', 'binascii', ] } + if sys.platform == 'win32': + spaceconfig['usemodules'].append('_rawffi') def setup_class(cls): if sys.platform != "win32": @@ -86,7 +92,6 @@ # just for multiprocessing to import correctly on Windows w_modules = space.sys.get('modules') space.setitem(w_modules, space.wrap('msvcrt'), space.sys) - space.setitem(w_modules, space.wrap('_subprocess'), space.sys) else: import _multiprocessing @@ -100,9 +105,12 @@ spaceconfig = { "usemodules": [ '_multiprocessing', 'thread', 'signal', 'struct', 'array', - '_socket', 'binascii', 'select', 'fcntl', - ] + '_socket', 'binascii', 'select' ] } + if sys.platform == 'win32': + spaceconfig['usemodules'].append('_rawffi') + else: + spaceconfig['usemodules'].append('fcntl') def setup_class(cls): cls.w_connections = cls.space.newlist([]) diff --git a/pypy/module/_rawffi/array.py b/pypy/module/_rawffi/array.py --- a/pypy/module/_rawffi/array.py +++ b/pypy/module/_rawffi/array.py @@ -207,7 +207,6 @@ __setitem__ = interp2app(W_ArrayInstance.descr_setitem), __getitem__ = interp2app(W_ArrayInstance.descr_getitem), __len__ = interp2app(W_ArrayInstance.getlength), - __buffer__ = interp2app(W_ArrayInstance.descr_buffer), buffer = GetSetProperty(W_ArrayInstance.getbuffer), shape = interp_attrproperty('shape', W_ArrayInstance), free = interp2app(W_ArrayInstance.free), @@ -232,7 +231,6 @@ __setitem__ = interp2app(W_ArrayInstance.descr_setitem), __getitem__ = interp2app(W_ArrayInstance.descr_getitem), __len__ = interp2app(W_ArrayInstance.getlength), - __buffer__ = interp2app(W_ArrayInstance.descr_buffer), buffer = GetSetProperty(W_ArrayInstance.getbuffer), shape = interp_attrproperty('shape', W_ArrayInstance), byptr = interp2app(W_ArrayInstance.byptr), diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py --- a/pypy/module/_rawffi/interp_rawffi.py +++ b/pypy/module/_rawffi/interp_rawffi.py @@ -359,9 +359,9 @@ lltype.free(self.ll_buffer, flavor='raw') self.ll_buffer = lltype.nullptr(rffi.VOIDP.TO) - def descr_buffer(self, space): + def buffer_w(self, space): from pypy.module._rawffi.buffer import RawFFIBuffer - return space.wrap(RawFFIBuffer(self)) + return RawFFIBuffer(self) def getrawsize(self): raise NotImplementedError("abstract base class") diff --git a/pypy/module/_rawffi/structure.py b/pypy/module/_rawffi/structure.py --- a/pypy/module/_rawffi/structure.py +++ b/pypy/module/_rawffi/structure.py @@ -364,7 +364,6 @@ __repr__ = interp2app(W_StructureInstance.descr_repr), __getattr__ = interp2app(W_StructureInstance.getattr), __setattr__ = interp2app(W_StructureInstance.setattr), - __buffer__ = interp2app(W_StructureInstance.descr_buffer), buffer = GetSetProperty(W_StructureInstance.getbuffer), free = interp2app(W_StructureInstance.free), shape = interp_attrproperty('shape', W_StructureInstance), @@ -387,7 +386,6 @@ __repr__ = interp2app(W_StructureInstance.descr_repr), __getattr__ = interp2app(W_StructureInstance.getattr), __setattr__ = interp2app(W_StructureInstance.setattr), - __buffer__ = interp2app(W_StructureInstance.descr_buffer), buffer = GetSetProperty(W_StructureInstance.getbuffer), shape = interp_attrproperty('shape', W_StructureInstance), byptr = interp2app(W_StructureInstance.byptr), diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py --- a/pypy/module/_socket/interp_socket.py +++ b/pypy/module/_socket/interp_socket.py @@ -69,10 +69,7 @@ else: flowinfo = 0 if len(pieces_w) > 3: scope_id = space.uint_w(pieces_w[3]) else: scope_id = 0 - if flowinfo < 0 or flowinfo > 0xfffff: - raise OperationError(space.w_OverflowError, space.wrap( - "flowinfo must be 0-1048575.")) - flowinfo = rffi.cast(lltype.Unsigned, flowinfo) + flowinfo = make_unsigned_flowinfo(space, flowinfo) a = addr.lock(_c.sockaddr_in6) rffi.setintfield(a, 'c_sin6_port', rsocket.htons(port)) rffi.setintfield(a, 'c_sin6_flowinfo', rsocket.htonl(flowinfo)) @@ -101,10 +98,7 @@ else: flowinfo = 0 if len(pieces_w) > 3: scope_id = space.uint_w(pieces_w[3]) else: scope_id = 0 - if flowinfo < 0 or flowinfo > 0xfffff: - raise OperationError(space.w_OverflowError, space.wrap( - "flowinfo must be 0-1048575.")) - flowinfo = rffi.cast(lltype.Unsigned, flowinfo) + flowinfo = make_unsigned_flowinfo(space, flowinfo) return rsocket.INET6Address(host, port, flowinfo, scope_id) if rsocket.HAS_AF_UNIX and family == rsocket.AF_UNIX: return rsocket.UNIXAddress(space.str_w(w_address)) @@ -116,10 +110,16 @@ # XXX Hack to seperate rpython and pypy def make_ushort_port(space, port): if port < 0 or port > 0xffff: - raise OperationError(space.w_ValueError, space.wrap( + raise OperationError(space.w_OverflowError, space.wrap( "port must be 0-65535.")) return rffi.cast(rffi.USHORT, port) +def make_unsigned_flowinfo(space, flowinfo): + if flowinfo < 0 or flowinfo > 0xfffff: + raise OperationError(space.w_OverflowError, space.wrap( + "flowinfo must be 0-1048575.")) + return rffi.cast(lltype.Unsigned, flowinfo) + # XXX Hack to seperate rpython and pypy def ipaddr_from_object(space, w_sockaddr): host = space.str_w(space.getitem(w_sockaddr, space.wrap(0))) diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py --- a/pypy/module/_socket/test/test_sock_app.py +++ b/pypy/module/_socket/test/test_sock_app.py @@ -391,7 +391,7 @@ name = s.getpeername() # Will raise socket.error if not connected assert name[1] == 80 s.close() - + def test_socket_connect_ex(self): import _socket s = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM, 0) @@ -417,8 +417,13 @@ def test_bigport(self): import _socket s = _socket.socket() - raises(ValueError, s.connect, ("localhost", 1000000)) - raises(ValueError, s.connect, ("localhost", -1)) + exc = raises(OverflowError, s.connect, ("localhost", -1)) + assert "port must be 0-65535." in str(exc.value) + exc = raises(OverflowError, s.connect, ("localhost", 1000000)) + assert "port must be 0-65535." in str(exc.value) + s = _socket.socket(_socket.AF_INET6) + exc = raises(OverflowError, s.connect, ("::1", 1234, 1048576)) + assert "flowinfo must be 0-1048575." in str(exc.value) def test_NtoH(self): import sys @@ -466,6 +471,13 @@ import socket s = socket.socket() + def test_subclass(self): + from _socket import socket + class MySock(socket): + blah = 123 + s = MySock() + assert s.blah == 123 + def test_getsetsockopt(self): import _socket as socket import struct @@ -587,11 +599,11 @@ class AppTestSocketTCP: + HOST = 'localhost' + def setup_class(cls): cls.space = space - HOST = 'localhost' - def setup_method(self, method): w_HOST = space.wrap(self.HOST) self.w_serv = space.appexec([w_HOST], @@ -602,6 +614,7 @@ serv.listen(1) return serv ''') + def teardown_method(self, method): if hasattr(self, 'w_serv'): space.appexec([self.w_serv], '(serv): serv.close()') @@ -622,7 +635,7 @@ raises(error, raise_error) def test_recv_send_timeout(self): - from _socket import socket, timeout + from _socket import socket, timeout, SOL_SOCKET, SO_RCVBUF, SO_SNDBUF cli = socket() cli.connect(self.serv.getsockname()) fileno, addr = self.serv._accept() @@ -643,6 +656,9 @@ assert count is None buf = t.recv(1) assert buf == b'?' + # speed up filling the buffers + t.setsockopt(SOL_SOCKET, SO_RCVBUF, 4096) + cli.setsockopt(SOL_SOCKET, SO_SNDBUF, 4096) # test send() timeout count = 0 try: @@ -671,7 +687,7 @@ conn = socket.socket(fileno=fileno) buf = memoryview(MSG) conn.send(buf) - buf = array.array('b', b' '*1024) + buf = array.array('b', b' ' * 1024) nbytes = cli.recv_into(buf) assert nbytes == len(MSG) msg = buf.tobytes()[:len(MSG)] @@ -687,7 +703,7 @@ conn = socket.socket(fileno=fileno) buf = memoryview(MSG) conn.send(buf) - buf = array.array('b', b' '*1024) + buf = array.array('b', b' ' * 1024) nbytes, addr = cli.recvfrom_into(buf) assert nbytes == len(MSG) msg = buf.tobytes()[:len(MSG)] @@ -698,6 +714,7 @@ cli = socket.socket(socket.AF_INET, socket.SOCK_STREAM) assert cli.family == socket.AF_INET + class AppTestErrno: def setup_class(cls): cls.space = space diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py --- a/pypy/module/_ssl/test/test_ssl.py +++ b/pypy/module/_ssl/test/test_ssl.py @@ -1,7 +1,7 @@ from rpython.tool.udir import udir class AppTestSSL: - spaceconfig = dict(usemodules=('_ssl', '_socket', 'binascii')) + spaceconfig = dict(usemodules=('_ssl', '_socket', 'binascii', 'thread')) def setup_class(cls): import os diff --git a/pypy/module/_ssl/thread_lock.py b/pypy/module/_ssl/thread_lock.py --- a/pypy/module/_ssl/thread_lock.py +++ b/pypy/module/_ssl/thread_lock.py @@ -1,4 +1,5 @@ -from rpython.rlib.ropenssl import * +from rpython.rlib import rthread +from rpython.rlib.ropenssl import libraries from rpython.rtyper.lltypesystem import lltype, rffi from rpython.translator.tool.cbuild import ExternalCompilationInfo @@ -22,7 +23,6 @@ # without caring about the GIL. separate_module_source = """ - #include static unsigned int _ssl_locks_count = 0; @@ -62,13 +62,12 @@ } """ -from rpython.rlib import rthread - eci = rthread.eci.merge(ExternalCompilationInfo( separate_module_sources=[separate_module_source], post_include_bits=[ "int _PyPy_SSL_SetupThreads(void);"], export_symbols=['_PyPy_SSL_SetupThreads'], + libraries = libraries, )) _PyPy_SSL_SetupThreads = rffi.llexternal('_PyPy_SSL_SetupThreads', diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py --- a/pypy/module/array/interp_array.py +++ b/pypy/module/array/interp_array.py @@ -135,6 +135,9 @@ self.len = 0 self.allocated = 0 + def buffer_w(self, space): + return ArrayBuffer(self) + def descr_append(self, space, w_x): """ append(x) @@ -505,9 +508,6 @@ def descr_iter(self, space): return space.wrap(ArrayIterator(self)) - def descr_buffer(self, space): - return space.wrap(ArrayBuffer(self)) - def descr_repr(self, space): if self.len == 0: return space.wrap("array('%s')" % self.typecode) @@ -544,7 +544,6 @@ __radd__ = interp2app(W_ArrayBase.descr_radd), __rmul__ = interp2app(W_ArrayBase.descr_rmul), - __buffer__ = interp2app(W_ArrayBase.descr_buffer), __iter__ = interp2app(W_ArrayBase.descr_iter), __repr__ = interp2app(W_ArrayBase.descr_repr), diff --git a/pypy/module/cpyext/include/numpy/arrayobject.h b/pypy/module/cpyext/include/numpy/arrayobject.h --- a/pypy/module/cpyext/include/numpy/arrayobject.h +++ b/pypy/module/cpyext/include/numpy/arrayobject.h @@ -20,7 +20,7 @@ #define PyArrayObject PyObject #define PyArray_Descr PyObject -extern PyTypeObject PyArray_Type; +PyAPI_DATA(PyTypeObject) PyArray_Type; typedef unsigned char npy_bool; typedef unsigned char npy_uint8; diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py --- a/pypy/module/cpyext/memoryobject.py +++ b/pypy/module/cpyext/memoryobject.py @@ -20,4 +20,4 @@ raise OperationError(space.w_ValueError, space.wrap(msg)) w_obj = from_ref(space, view.c_obj) buf = CBuffer(space, view.c_buf, view.c_len, w_obj) - return space.wrap(W_MemoryView(space.wrap(buf))) + return space.wrap(W_MemoryView(buf)) diff --git a/pypy/module/cpyext/pystate.py b/pypy/module/cpyext/pystate.py --- a/pypy/module/cpyext/pystate.py +++ b/pypy/module/cpyext/pystate.py @@ -217,7 +217,7 @@ if rffi.aroundstate.after: # After external call is before entering Python rffi.aroundstate.after() - return 0 + return lltype.nullptr(PyGILState_STATE.TO) @cpython_api([PyGILState_STATE], lltype.Void) def PyGILState_Release(space, state): diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -14,8 +14,9 @@ from pypy.module.cpyext.pyobject import from_ref from pypy.module.cpyext.pyerrors import PyErr_Occurred from pypy.module.cpyext.state import State +from pypy.module.__builtin__.interp_memoryview import W_Buffer from pypy.interpreter.error import OperationError, oefmt -from pypy.interpreter.buffer import Buffer as W_Buffer +from pypy.interpreter.buffer import Buffer from pypy.interpreter.argument import Arguments from rpython.rlib.unroll import unrolling_iterable from rpython.rlib.objectmodel import specialize @@ -228,7 +229,7 @@ check_num_args(space, w_args, 0) return space.wrap(generic_cpy_call(space, func_target, w_self)) -class CPyBuffer(W_Buffer): +class CPyBuffer(Buffer): # Similar to Py_buffer def __init__(self, ptr, size, w_obj): @@ -249,7 +250,7 @@ ret = generic_cpy_call(space, func_target, w_self, view, flags) if rffi.cast(lltype.Signed, ret) == -1: space.fromcache(State).check_and_raise_exception(always=True) - return space.wrap(CPyBuffer(view.c_buf, view.c_len, w_self)) + return W_Buffer(CPyBuffer(view.c_buf, view.c_len, w_self)) def get_richcmp_func(OP_CONST): def inner(space, w_self, w_args, func): diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py --- a/pypy/module/cpyext/test/test_cpyext.py +++ b/pypy/module/cpyext/test/test_cpyext.py @@ -64,6 +64,10 @@ kwds["libraries"] = [api_library] # '%s' undefined; assuming extern returning int kwds["compile_extra"] = ["/we4013"] + # prevent linking with PythonXX.lib + w_maj, w_min = space.fixedview(space.sys.get('version_info'), 5)[:2] + kwds["link_extra"] = ["/NODEFAULTLIB:Python%d%d.lib" % + (space.int_w(w_maj), space.int_w(w_min))] elif sys.platform == 'darwin': kwds["link_files"] = [str(api_library + '.dylib')] else: @@ -181,6 +185,19 @@ state = cls.space.fromcache(RefcountState) state.non_heaptypes_w[:] = [] + def setup_method(self, meth): + freeze_refcnts(self) + + def teardown_method(self, meth): + self.cleanup_references(self.space) + # XXX: like AppTestCpythonExtensionBase.teardown_method: + # find out how to disable check_and_print_leaks() if the + # test failed + assert not self.check_and_print_leaks(), ( + "Test leaks or loses object(s). You should also check if " + "the test actually passed in the first place; if it failed " + "it is likely to reach this place.") + def test_load_error(self): import cpyext raises(ImportError, cpyext.load_module, "missing.file", "foo") @@ -368,13 +385,12 @@ for name in self.imported_module_names: self.unimport_module(name) self.cleanup_references(self.space) - if self.check_and_print_leaks(): - assert False, ( - "Test leaks or loses object(s). You should also check if " - "the test actually passed in the first place; if it failed " - "it is likely to reach this place.") - # XXX find out how to disable check_and_print_leaks() if the - # XXX test failed... + # XXX: find out how to disable check_and_print_leaks() if the + # test failed... + assert not self.check_and_print_leaks(), ( + "Test leaks or loses object(s). You should also check if " + "the test actually passed in the first place; if it failed " + "it is likely to reach this place.") class AppTestCpythonExtension(AppTestCpythonExtensionBase): diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -1,38 +1,36 @@ import os -import sys +from rpython.rlib import jit +from rpython.rlib.objectmodel import specialize +from rpython.rlib.rstring import rsplit +from rpython.rtyper.annlowlevel import llhelper from rpython.rtyper.lltypesystem import rffi, lltype -from rpython.rtyper.annlowlevel import llhelper + from pypy.interpreter.baseobjspace import W_Root, DescrMismatch -from pypy.objspace.std.typeobject import W_TypeObject, find_best_base +from pypy.interpreter.error import OperationError from pypy.interpreter.typedef import GetSetProperty +from pypy.module.__builtin__.abstractinst import abstract_issubclass_w +from pypy.module.cpyext import structmemberdefs from pypy.module.cpyext.api import ( cpython_api, cpython_struct, bootstrap_function, Py_ssize_t, Py_ssize_tP, generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING, Py_TPFLAGS_HEAPTYPE, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL, - Py_TPFLAGS_HAVE_GETCHARBUFFER, - build_type_checkers, PyObjectFields, Py_buffer) + Py_TPFLAGS_HAVE_GETCHARBUFFER, build_type_checkers, Py_buffer) +from pypy.module.cpyext.methodobject import ( + PyDescr_NewWrapper, PyCFunction_NewEx, PyCFunction_typedef) +from pypy.module.cpyext.modsupport import convert_method_defs from pypy.module.cpyext.pyobject import ( PyObject, make_ref, create_ref, from_ref, get_typedescr, make_typedescr, - track_reference, RefcountState, borrow_from) -from pypy.interpreter.module import Module -from pypy.module.cpyext import structmemberdefs -from pypy.module.cpyext.modsupport import convert_method_defs + track_reference, RefcountState, borrow_from, Py_DecRef) +from pypy.module.cpyext.slotdefs import ( + slotdefs_for_tp_slots, slotdefs_for_wrappers, get_slot_tp_function) from pypy.module.cpyext.state import State -from pypy.module.cpyext.methodobject import ( - PyDescr_NewWrapper, PyCFunction_NewEx, PyCFunction_typedef) -from pypy.module.cpyext.pyobject import Py_IncRef, Py_DecRef, _Py_Dealloc from pypy.module.cpyext.structmember import PyMember_GetOne, PyMember_SetOne from pypy.module.cpyext.typeobjectdefs import ( PyTypeObjectPtr, PyTypeObject, PyGetSetDef, PyMemberDef, newfunc, PyNumberMethods, PyMappingMethods, PySequenceMethods, PyBufferProcs) -from pypy.module.cpyext.slotdefs import ( - slotdefs_for_tp_slots, slotdefs_for_wrappers, get_slot_tp_function) -from pypy.interpreter.error import OperationError -from rpython.rlib.rstring import rsplit -from rpython.rlib.objectmodel import specialize -from pypy.module.__builtin__.abstractinst import abstract_issubclass_w -from rpython.rlib import jit +from pypy.objspace.std.typeobject import W_TypeObject, find_best_base + WARN_ABOUT_MISSING_SLOT_FUNCTIONS = False @@ -421,7 +419,6 @@ def type_alloc(space, w_metatype): - size = rffi.sizeof(PyHeapTypeObject) metatype = rffi.cast(PyTypeObjectPtr, make_ref(space, w_metatype)) # Don't increase refcount for non-heaptypes if metatype: diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py --- a/pypy/module/imp/importing.py +++ b/pypy/module/imp/importing.py @@ -243,12 +243,10 @@ fromlist_w = None rel_modulename = None - if (level != 0 and - w_globals is not None and - space.isinstance_w(w_globals, space.w_dict)): - - rel_modulename, rel_level = _get_relative_name(space, modulename, level, w_globals) - + if (level != 0 and w_globals is not None and + space.isinstance_w(w_globals, space.w_dict)): + rel_modulename, rel_level = _get_relative_name(space, modulename, level, + w_globals) if rel_modulename: # if no level was set, ignore import errors, and # fall back to absolute import at the end of the @@ -582,6 +580,7 @@ def load_module(space, w_modulename, find_info, reuse=False): if find_info is None: return + if find_info.w_loader: return space.call_method(find_info.w_loader, "load_module", w_modulename) @@ -607,7 +606,7 @@ try: if find_info.modtype == PY_SOURCE: load_source_module( - space, w_modulename, w_mod, + space, w_modulename, w_mod, find_info.filename, find_info.stream.readall(), find_info.stream.try_to_find_file_descriptor()) return w_mod diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py --- a/pypy/module/imp/test/test_import.py +++ b/pypy/module/imp/test/test_import.py @@ -232,7 +232,6 @@ import _warnings def simplefilter(action, category): _warnings.filters.insert(0, (action, None, category, None, 0)) - simplefilter('error', ImportWarning) try: raises(ImportWarning, imp) @@ -467,7 +466,7 @@ def test_future_relative_import_level_1(self): from pkg import relative_c assert relative_c.inpackage == 1 - + def test_future_relative_import_level_2(self): from pkg.pkg1 import relative_d assert relative_d.inpackage == 1 @@ -751,10 +750,7 @@ import imp import pkg import os - - info = ('.py', 'r', imp.PY_SOURCE) pathname = os.path.join(os.path.dirname(pkg.__file__), 'a.py') - module = imp.load_module('a', open(pathname), 'invalid_path_name', ('.py', 'r', imp.PY_SOURCE)) assert module.__name__ == 'a' @@ -1284,7 +1280,7 @@ def load_module(self, name): sys.modules[name] = sys return sys - + def importer_for_path(path): if path == "xxx": return Importer() diff --git a/pypy/module/marshal/__init__.py b/pypy/module/marshal/__init__.py --- a/pypy/module/marshal/__init__.py +++ b/pypy/module/marshal/__init__.py @@ -8,7 +8,7 @@ appleveldefs = { } - + interpleveldefs = { 'dump' : 'interp_marshal.dump', 'dumps' : 'interp_marshal.dumps', diff --git a/pypy/module/micronumpy/boxes.py b/pypy/module/micronumpy/boxes.py --- a/pypy/module/micronumpy/boxes.py +++ b/pypy/module/micronumpy/boxes.py @@ -335,8 +335,8 @@ def descr_copy(self, space): return self.convert_to(space, self.get_dtype(space)) - def descr_buffer(self, space): - return self.descr_ravel(space).descr_get_data(space) + def buffer_w(self, space): + return self.descr_ravel(space).buffer_w(space) def descr_byteswap(self, space): return self.get_dtype(space).itemtype.byteswap(self) @@ -547,7 +547,6 @@ __bool__ = interp2app(W_GenericBox.descr_nonzero), __oct__ = interp2app(W_GenericBox.descr_oct), __hex__ = interp2app(W_GenericBox.descr_hex), - __buffer__ = interp2app(W_GenericBox.descr_buffer), __add__ = interp2app(W_GenericBox.descr_add), __sub__ = interp2app(W_GenericBox.descr_sub), diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py --- a/pypy/module/micronumpy/ndarray.py +++ b/pypy/module/micronumpy/ndarray.py @@ -7,6 +7,7 @@ from rpython.rlib.rawstorage import RAW_STORAGE_PTR from rpython.rtyper.lltypesystem import rffi from rpython.tool.sourcetools import func_with_new_name +from pypy.module.__builtin__.interp_memoryview import W_Buffer from pypy.module.micronumpy import descriptor, ufuncs, boxes, arrayops, loop, \ support, constants as NPY from pypy.module.micronumpy.appbridge import get_appbridge_cache @@ -602,8 +603,11 @@ raise OperationError(space.w_NotImplementedError, space.wrap( "ctypes not implemented yet")) + def buffer_w(self, space): + return self.implementation.get_buffer(space) + def descr_get_data(self, space): - return self.implementation.get_buffer(space) + return W_Buffer(self.buffer_w(space)) @unwrap_spec(offset=int, axis1=int, axis2=int) def descr_diagonal(self, space, offset=0, axis1=0, axis2=1): @@ -1248,7 +1252,6 @@ __float__ = interp2app(W_NDimArray.descr_float), __hex__ = interp2app(W_NDimArray.descr_hex), __oct__ = interp2app(W_NDimArray.descr_oct), - __buffer__ = interp2app(W_NDimArray.descr_get_data), __index__ = interp2app(W_NDimArray.descr_index), __pos__ = interp2app(W_NDimArray.descr_pos), diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py --- a/pypy/module/micronumpy/test/test_zjit.py +++ b/pypy/module/micronumpy/test/test_zjit.py @@ -480,23 +480,19 @@ def test_flat_setitem(self): result = self.run("flat_setitem") assert result == 1.0 - py.test.skip("don't run for now") self.check_trace_count(1) - # XXX not ideal, but hey, let's ignore it for now - self.check_simple_loop({'raw_load': 1, - 'raw_store': 1, - 'int_lt': 1, - 'int_gt': 1, - 'int_add': 4, - 'guard_true': 2, - 'arraylen_gc': 2, - 'jump': 1, - 'int_sub': 1, - # XXX bad part - 'int_and': 1, - 'int_mod': 1, - 'int_rshift': 1, - }) + self.check_simple_loop({ + 'call': 2, + 'getfield_gc': 2, + 'guard_no_exception': 2, + 'guard_not_invalidated': 1, + 'guard_true': 1, + 'int_gt': 1, + 'int_sub': 1, + 'jump': 1, + 'raw_load': 1, + 'raw_store': 1, + }) def define_dot(): return """ @@ -509,6 +505,7 @@ def test_dot(self): result = self.run("dot") assert result == 184 + self.check_trace_count(3) self.check_simple_loop({'float_add': 1, 'float_mul': 1, 'guard_not_invalidated': 1, @@ -526,7 +523,7 @@ 'guard_class': 4, 'guard_false': 2, 'guard_no_exception': 3, - 'guard_nonnull': 8, + 'guard_nonnull': 12, 'guard_nonnull_class': 4, 'guard_not_invalidated': 2, 'guard_true': 9, diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py --- a/pypy/module/mmap/interp_mmap.py +++ b/pypy/module/mmap/interp_mmap.py @@ -17,6 +17,10 @@ self.space = space self.mmap = mmap_obj + def buffer_w(self, space): + self.check_valid() + return MMapBuffer(self.space, self.mmap) + def close(self): self.mmap.close() @@ -202,10 +206,6 @@ self.mmap.setitem(start, value[i]) start += step - def descr_buffer(self): - self.check_valid() - return self.space.wrap(MMapBuffer(self.space, self.mmap)) - def descr_enter(self, space): self.check_valid() return space.wrap(self) @@ -270,7 +270,6 @@ __len__ = interp2app(W_MMap.__len__), __getitem__ = interp2app(W_MMap.descr_getitem), __setitem__ = interp2app(W_MMap.descr_setitem), - __buffer__ = interp2app(W_MMap.descr_buffer), __enter__ = interp2app(W_MMap.descr_enter), __exit__ = interp2app(W_MMap.descr_exit), diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -177,17 +177,9 @@ import sys import errno for fn in [self.posix.stat, self.posix.lstat]: - try: - fn("nonexistentdir/nonexistentfile") - except OSError as e: - assert e.errno == errno.ENOENT - assert e.filename == "nonexistentdir/nonexistentfile" - # On Windows, when the parent directory does not exist, - # the winerror is 3 (cannot find the path specified) - # instead of 2 (cannot find the file specified) - if sys.platform == 'win32': - assert isinstance(e, WindowsError) - assert e.winerror == 3 + exc = raises(OSError, fn, "nonexistentdir/nonexistentfile") + assert exc.value.errno == errno.ENOENT + assert exc.value.filename == "nonexistentdir/nonexistentfile" if hasattr(__import__(os.name), "statvfs"): def test_statvfs(self): diff --git a/pypy/module/pwd/test/test_pwd.py b/pypy/module/pwd/test/test_pwd.py --- a/pypy/module/pwd/test/test_pwd.py +++ b/pypy/module/pwd/test/test_pwd.py @@ -1,8 +1,8 @@ import os -import py +import pytest if os.name != 'posix': - py.test.skip('pwd module only available on unix') + pytest.skip('pwd module only available on unix') class AppTestPwd: spaceconfig = dict(usemodules=['pwd']) diff --git a/pypy/module/pwd/test/test_ztranslation.py b/pypy/module/pwd/test/test_ztranslation.py --- a/pypy/module/pwd/test/test_ztranslation.py +++ b/pypy/module/pwd/test/test_ztranslation.py @@ -1,5 +1,9 @@ +import os +import pytest from pypy.objspace.fake.checkmodule import checkmodule +if os.name != 'posix': + pytest.skip('pwd module only available on unix') def test_checkmodule(): checkmodule('pwd') diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py --- a/pypy/module/pyexpat/interp_pyexpat.py +++ b/pypy/module/pyexpat/interp_pyexpat.py @@ -428,7 +428,7 @@ self.handlers = [None] * NB_HANDLERS - self.buffer_w = None + self.buffer = None self.buffer_size = 8192 self.buffer_used = 0 self.w_character_data_handler = None @@ -537,7 +537,7 @@ def buffer_string(self, space, w_string, length): ll_length = rffi.cast(lltype.Signed, length) - if self.buffer_w is not None: + if self.buffer is not None: if self.buffer_used + ll_length > self.buffer_size: self.flush_character_buffer(space) # handler might have changed; drop the rest on the floor @@ -545,11 +545,11 @@ if self.w_character_data_handler is None: return True if ll_length <= self.buffer_size: - self.buffer_w.append(w_string) + self.buffer.append(w_string) self.buffer_used += ll_length return True else: - self.buffer_w = [] + self.buffer = [] self.buffer_used = 0 return False @@ -680,12 +680,12 @@ return space.wrap(parser) def flush_character_buffer(self, space): - if not self.buffer_w: + if not self.buffer: return w_data = space.call_function( space.getattr(space.wrap(''), space.wrap('join')), - space.newlist(self.buffer_w)) - self.buffer_w = [] + space.newlist(self.buffer)) + self.buffer = [] self.buffer_used = 0 if self.w_character_data_handler: @@ -730,14 +730,14 @@ self.buffer_size = value def get_buffer_text(self, space): - return space.wrap(self.buffer_w is not None) + return space.wrap(self.buffer is not None) def set_buffer_text(self, space, w_value): if space.is_true(w_value): - self.buffer_w = [] + self.buffer = [] self.buffer_used = 0 else: self.flush_character_buffer(space) - self.buffer_w = None + self.buffer = None def get_intern(self, space): if self.w_intern: diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py --- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py +++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py @@ -207,14 +207,13 @@ ... """) ops = loop.ops_by_id('cfficall') - assert 'raw_malloc' not in str(ops) - assert 'raw_free' not in str(ops) - assert 'getarrayitem_raw' not in log.opnames(ops) - assert 'setarrayitem_raw' not in log.opnames(ops) + for name in ['raw_malloc', 'raw_free']: + assert name not in str(ops) + for name in ['raw_load', 'raw_store', 'getarrayitem_raw', 'setarrayitem_raw']: + assert name not in log.opnames(ops) # so far just check that call_release_gil() is produced. # later, also check that the arguments to call_release_gil() # are constants - # are constants, and that the numerous raw_mallocs are removed def test_cffi_call_guard_not_forced_fails(self): # this is the test_pypy_c equivalent of @@ -341,13 +340,11 @@ guard_value(p166, ConstPtr(ptr72), descr=...) p167 = call(ConstClass(_ll_0_alloc_with_del___), descr=) guard_no_exception(descr=...) - i168 = call(ConstClass(_ll_1_raw_malloc_varsize__Signed), 6, descr=) - setfield_gc(p167, 0, descr=) - setfield_gc(p167, ConstPtr(ptr86), descr=) - guard_no_exception(descr=...) + i168 = call(ConstClass(_ll_1_raw_malloc_varsize__Signed), 6, descr=) i169 = int_add(i168, i97) i170 = int_sub(i160, i106) setfield_gc(p167, i168, descr=) + setfield_gc(p167, ConstPtr(null), descr=) setfield_gc(p167, ConstPtr(ptr89), descr=) i171 = uint_gt(i170, i108) guard_false(i171, descr=...) diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py --- a/pypy/objspace/fake/objspace.py +++ b/pypy/objspace/fake/objspace.py @@ -302,6 +302,11 @@ ec._py_repr = None return ec + def buffer_w(self, w_obj): + from pypy.interpreter.buffer import Buffer + is_root(w_obj) + return Buffer() + def unicode_from_object(self, w_obj): return w_some_obj() diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py --- a/pypy/objspace/std/bytearrayobject.py +++ b/pypy/objspace/std/bytearrayobject.py @@ -28,6 +28,9 @@ """representation for debugging purposes""" return "%s(%s)" % (w_self.__class__.__name__, ''.join(w_self.data)) + def buffer_w(w_self, space): + return BytearrayBuffer(w_self.data) + def _new(self, value): return W_BytearrayObject(_make_data(value)) @@ -240,9 +243,6 @@ raise return space.newbool(res) - def descr_buffer(self, space): - return BytearrayBuffer(self.data) - def descr_inplace_add(self, space, w_other): if isinstance(w_other, W_BytearrayObject): self.data += w_other.data @@ -941,7 +941,6 @@ __init__ = interp2app(W_BytearrayObject.descr_init, doc=BytearrayDocstrings.__init__.__doc__), - __buffer__ = interp2app(W_BytearrayObject.descr_buffer), __iadd__ = interp2app(W_BytearrayObject.descr_inplace_add, doc=BytearrayDocstrings.__iadd__.__doc__), diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py --- a/pypy/objspace/std/bytesobject.py +++ b/pypy/objspace/std/bytesobject.py @@ -379,9 +379,6 @@ of the specified width. The string S is never truncated. """ - def descr_buffer(self, space): - pass - class W_BytesObject(W_AbstractBytesObject): import_from_mixin(StringMethods) @@ -401,6 +398,9 @@ def bytes_w(w_self, space): return w_self._value + def buffer_w(w_self, space): + return StringBuffer(w_self._value) + def listview_bytes(self): return _create_list_from_bytes(self._value) @@ -521,9 +521,6 @@ x = compute_hash(self._value) return space.wrap(x) - def descr_buffer(self, space): - return space.wrap(StringBuffer(self._value)) - def descr_eq(self, space, w_other): if space.config.objspace.std.withstrbuf: from pypy.objspace.std.strbufobject import W_StringBufferObject @@ -837,7 +834,6 @@ upper = interpindirect2app(W_AbstractBytesObject.descr_upper), zfill = interpindirect2app(W_AbstractBytesObject.descr_zfill), - __buffer__ = interpindirect2app(W_AbstractBytesObject.descr_buffer), __getnewargs__ = interpindirect2app( W_AbstractBytesObject.descr_getnewargs), diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py --- a/pypy/objspace/std/intobject.py +++ b/pypy/objspace/std/intobject.py @@ -889,7 +889,7 @@ else: # If object supports the buffer interface try: - w_buffer = space.buffer(w_value) + buf = space.buffer_w(w_value) except OperationError as e: if not e.match(space, space.w_TypeError): raise @@ -897,7 +897,6 @@ "int() argument must be a string or a number, " "not '%T'", w_value) else: - buf = space.interp_w(Buffer, w_buffer) return _string_to_int_or_long(space, w_inttype, w_value, buf.as_str()) else: diff --git a/pypy/objspace/std/model.py b/pypy/objspace/std/model.py --- a/pypy/objspace/std/model.py +++ b/pypy/objspace/std/model.py @@ -61,6 +61,7 @@ from pypy.objspace.std import unicodeobject from pypy.objspace.std import dictproxyobject from pypy.objspace.std import proxyobject + from pypy.module.__builtin__.interp_memoryview import W_Buffer import pypy.objspace.std.default # register a few catch-all multimethods import pypy.objspace.std.marshal_impl # install marshal multimethods @@ -79,6 +80,7 @@ self.pythontypes.append(intobject.W_IntObject.typedef) self.pythontypes.append(boolobject.W_BoolObject.typedef) self.pythontypes.append(longobject.W_LongObject.typedef) + self.pythontypes.append(W_Buffer.typedef) # the set of implementation types self.typeorder = { diff --git a/pypy/objspace/std/strbufobject.py b/pypy/objspace/std/strbufobject.py --- a/pypy/objspace/std/strbufobject.py +++ b/pypy/objspace/std/strbufobject.py @@ -2,11 +2,12 @@ import py -from pypy.objspace.std.bytesobject import W_AbstractBytesObject, W_BytesObject -from pypy.objspace.std.stdtypedef import StdTypeDef +from pypy.objspace.std.bytesobject import (W_AbstractBytesObject, + W_BytesObject, StringBuffer) from pypy.interpreter.gateway import interp2app, unwrap_spec from rpython.rlib.rstring import StringBuilder + class W_StringBufferObject(W_AbstractBytesObject): w_str = None @@ -35,6 +36,9 @@ def bytes_w(self, space): return self.force() + def buffer_w(self, space): + return StringBuffer(self.force()) + def descr_len(self, space): return space.wrap(self.length) diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py --- a/rpython/annotator/test/test_annrpython.py +++ b/rpython/annotator/test/test_annrpython.py @@ -2872,7 +2872,6 @@ py.test.raises(Exception, a.build_types, fun, [s_nonneg, int]) def test_sig_bug(self): - py.test.skip("_annenforceargs_ does not work for default arguments") def g(x, y=5): return y == 5 g._annenforceargs_ = (int, int) @@ -2880,7 +2879,8 @@ return g(x) a = self.RPythonAnnotator() s = a.build_types(fun, [int]) - assert not s.is_constant() + assert s.knowntype is bool + assert s.is_constant() def test_sig_list(self): def g(buf): diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py --- a/rpython/jit/codewriter/jtransform.py +++ b/rpython/jit/codewriter/jtransform.py @@ -547,20 +547,18 @@ track_allocation = d.pop('track_allocation', True) if d: raise UnsupportedMallocFlags(d) - TYPE = op.args[0].value if zero: name += '_zero' if add_memory_pressure: name += '_add_memory_pressure' if not track_allocation: name += '_no_track_allocation' + TYPE = op.args[0].value op1 = self.prepare_builtin_call(op, name, args, (TYPE,), TYPE) - if name == 'raw_malloc_varsize': - ITEMTYPE = op.args[0].value.OF - if ITEMTYPE == lltype.Char: - return self._handle_oopspec_call(op1, args, - EffectInfo.OS_RAW_MALLOC_VARSIZE_CHAR, - EffectInfo.EF_CAN_RAISE) + if name.startswith('raw_malloc_varsize') and TYPE.OF == lltype.Char: + return self._handle_oopspec_call(op1, args, + EffectInfo.OS_RAW_MALLOC_VARSIZE_CHAR, + EffectInfo.EF_CAN_RAISE) return self.rewrite_op_direct_call(op1) def rewrite_op_malloc_varsize(self, op): @@ -591,7 +589,7 @@ name += '_no_track_allocation' op1 = self.prepare_builtin_call(op, name, [op.args[0]], (STRUCT,), STRUCT) - if name == 'raw_free': + if name.startswith('raw_free'): return self._handle_oopspec_call(op1, [op.args[0]], EffectInfo.OS_RAW_FREE, EffectInfo.EF_CANNOT_RAISE) @@ -837,8 +835,8 @@ RESULT = lltype.Ptr(STRUCT) assert RESULT == op.result.concretetype return self._do_builtin_call(op, 'alloc_with_del', [], - extra = (RESULT, vtable), - extrakey = STRUCT) + extra=(RESULT, vtable), + extrakey=STRUCT) heaptracker.register_known_gctype(self.cpu, vtable, STRUCT) opname = 'new_with_vtable' else: @@ -1237,7 +1235,7 @@ op1 = self.prepare_builtin_call(op, "llong_%s", args) op2 = self._handle_oopspec_call(op1, args, EffectInfo.OS_LLONG_%s, - EffectInfo.EF_ELIDABLE_CANNOT_RAISE) + EffectInfo.EF_ELIDABLE_CANNOT_RAISE) if %r == "TO_INT": assert op2.result.concretetype == lltype.Signed return op2 @@ -1269,7 +1267,7 @@ op1 = self.prepare_builtin_call(op, "ullong_%s", args) op2 = self._handle_oopspec_call(op1, args, EffectInfo.OS_LLONG_%s, - EffectInfo.EF_ELIDABLE_CANNOT_RAISE) + EffectInfo.EF_ELIDABLE_CANNOT_RAISE) return op2 ''' % (_op, _oopspec.lower(), _oopspec)).compile() diff --git a/rpython/jit/codewriter/test/test_jtransform.py b/rpython/jit/codewriter/test/test_jtransform.py --- a/rpython/jit/codewriter/test/test_jtransform.py +++ b/rpython/jit/codewriter/test/test_jtransform.py @@ -60,7 +60,7 @@ class FakeResidualCallControl: def guess_call_kind(self, op): return 'residual' - def getcalldescr(self, op, **kwds): + def getcalldescr(self, op, oopspecindex=None, extraeffect=None): return 'calldescr' def calldescr_canraise(self, calldescr): return True diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py --- a/rpython/jit/metainterp/optimizeopt/heap.py +++ b/rpython/jit/metainterp/optimizeopt/heap.py @@ -93,6 +93,11 @@ # possible aliasing). self.clear() self._lazy_setfield = None + if optheap.postponed_op: + for a in op.getarglist(): + if a is optheap.postponed_op.result: + optheap.emit_postponed_op() + break optheap.next_optimization.propagate_forward(op) if not can_cache: return @@ -179,6 +184,9 @@ def flush(self): self.force_all_lazy_setfields_and_arrayitems() + self.emit_postponed_op() + + def emit_postponed_op(self): if self.postponed_op: postponed_op = self.postponed_op self.postponed_op = None @@ -227,10 +235,7 @@ def emit_operation(self, op): self.emitting_operation(op) - if self.postponed_op: - postponed_op = self.postponed_op - self.postponed_op = None - self.next_optimization.propagate_forward(postponed_op) + self.emit_postponed_op() if (op.is_comparison() or op.getopnum() == rop.CALL_MAY_FORCE or op.is_ovf()): self.postponed_op = op diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py @@ -1660,6 +1660,16 @@ """ self.optimize_loop(ops, ops) + def test_setfield_int_eq_result(self): + # test that the setfield_gc does not end up before int_eq + ops = """ + [p1, i1, i2] + i3 = int_eq(i1, i2) + setfield_gc(p1, i3, descr=valuedescr) + jump(p1, i1, i2) + """ + self.optimize_loop(ops, ops) + def test_duplicate_setfield_aliasing(self): # a case where aliasing issues (and not enough cleverness) mean # that we fail to remove any setfield_gc diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -1,14 +1,15 @@ import py from rpython.rlib.objectmodel import instantiate +from rpython.jit.metainterp import compile, resume +from rpython.jit.metainterp.history import AbstractDescr, ConstInt, BoxInt, TreeLoop +from rpython.jit.metainterp.optimize import InvalidLoop +from rpython.jit.metainterp.optimizeopt import build_opt_chain from rpython.jit.metainterp.optimizeopt.test.test_util import ( LLtypeMixin, BaseTest, convert_old_style_to_targets) -from rpython.jit.metainterp.optimizeopt import build_opt_chain -from rpython.jit.metainterp.optimize import InvalidLoop -from rpython.jit.metainterp.history import AbstractDescr, ConstInt, BoxInt -from rpython.jit.metainterp.history import TreeLoop -from rpython.jit.metainterp import compile, resume +from rpython.jit.metainterp.optimizeopt.test.test_optimizebasic import \ + FakeMetaInterpStaticData from rpython.jit.metainterp.resoperation import rop, opname, oparity -from rpython.jit.metainterp.optimizeopt.test.test_optimizebasic import FakeMetaInterpStaticData + def test_build_opt_chain(): def check(chain, expected_names): @@ -40,7 +41,6 @@ class BaseTestWithUnroll(BaseTest): - enable_opts = "intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll" def optimize_loop(self, ops, expected, expected_preamble=None, @@ -93,8 +93,8 @@ def raises(self, e, fn, *args): return py.test.raises(e, fn, *args).value + class OptimizeOptTest(BaseTestWithUnroll): - def setup_method(self, meth=None): class FailDescr(compile.ResumeGuardDescr): oparse = None @@ -130,7 +130,6 @@ self.namespace.pop('fdescr', None) self.namespace.pop('fdescr2', None) - def test_simple(self): ops = """ [] @@ -606,9 +605,9 @@ i1 = ptr_eq(p0, NULL) guard_false(i1) [] i2 = ptr_ne(NULL, p0) - guard_true(i0) [] + guard_true(i2) [] i3 = ptr_eq(NULL, p0) - guard_false(i1) [] + guard_false(i3) [] guard_nonnull(p0) [] jump(p0) """ @@ -623,6 +622,30 @@ """ self.optimize_loop(ops, expected, preamble) + def test_nonnull_2(self): From noreply at buildbot.pypy.org Tue Mar 18 23:24:19 2014 From: noreply at buildbot.pypy.org (pjenvey) Date: Tue, 18 Mar 2014 23:24:19 +0100 (CET) Subject: [pypy-commit] pypy py3k: merge default Message-ID: <20140318222419.299ED1C0166@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r70069:fb381945ef16 Date: 2014-03-18 15:23 -0700 http://bitbucket.org/pypy/pypy/changeset/fb381945ef16/ Log: merge default diff --git a/lib-python/2.7/test/test_memoryview.py b/lib-python/2.7/test/test_memoryview.py --- a/lib-python/2.7/test/test_memoryview.py +++ b/lib-python/2.7/test/test_memoryview.py @@ -171,7 +171,7 @@ # very inconsisten on CPython. In PyPy, memoryview supports # the buffer interface, and thus the following comparison # succeeds. See also the comment in - # pypy.modules.__builtin__.interp_memoryview.W_MemoryView.descr_buffer + # pypy.objspace.std.memoryview.W_MemoryView.descr_buffer # # Comparison with objects which don't support the buffer API self.assertFalse(m == u"abcdef", "%s %s" % (self, tp)) diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -194,11 +194,6 @@ return None def buffer_w(self, space): - w_impl = space.lookup(self, '__buffer__') - if w_impl is not None: - w_result = space.get_and_call_function(w_impl, self) - if space.isinstance_w(w_result, space.w_buffer): - return w_result.buf self._typed_unwrap_error(space, "buffer") def bytes_w(self, space): diff --git a/pypy/interpreter/buffer.py b/pypy/interpreter/buffer.py --- a/pypy/interpreter/buffer.py +++ b/pypy/interpreter/buffer.py @@ -7,8 +7,7 @@ class Buffer(object): """Abstract base class for buffers.""" - - __slots__ = () # no extra slot here + __slots__ = [] def getlength(self): raise NotImplementedError @@ -29,14 +28,13 @@ def get_raw_address(self): raise ValueError("no raw buffer") - def is_writable(self): return False + class RWBuffer(Buffer): """Abstract base class for read-write buffers.""" - - __slots__ = () # no extra slot here + __slots__ = [] def is_writable(self): return True @@ -51,10 +49,8 @@ self.setitem(start + i, string[i]) - -# ____________________________________________________________ - class StringBuffer(Buffer): + __slots__ = ['value'] def __init__(self, value): self.value = value @@ -76,48 +72,11 @@ return self.value[start:stop] return "".join([self.value[start + i*step] for i in xrange(size)]) - -class StringLikeBuffer(Buffer): - """For app-level objects that already have a string-like interface - with __len__ and a __getitem__ that returns characters or (with - slicing) substrings.""" - # XXX this is inefficient, it should only be used temporarily - - def __init__(self, space, w_obj): - self.space = space - self.w_obj = w_obj - - def getlength(self): - space = self.space - return space.len_w(self.w_obj) - - def getitem(self, index): - space = self.space - w_value = space.getitem(self.w_obj, space.wrap(index)) - try: - return chr(space.int_w(w_value)) - except OperationError as e: - if not e.match(space, space.w_TypeError): - raise - s = space.bytes_w(w_value) - if len(s) != 1: - raise OperationError(space.w_ValueError, - space.wrap("single byte expected, got string")) - char = s[0] # annotator hint - return char - - def getslice(self, start, stop, step, size): - space = self.space - if step != 1: - raise OperationError(space.w_ValueError, space.wrap( - "buffer object does not support slicing with a step")) - s = space.str_w(space.getslice(self.w_obj, space.wrap(start), - space.wrap(stop))) - return s - # ____________________________________________________________ class SubBufferMixin(object): + _attrs_ = ['buffer', 'offset', 'size'] + def __init__(self, buffer, offset, size): self.buffer = buffer self.offset = offset @@ -141,9 +100,11 @@ # out of bounds return self.buffer.getslice(self.offset + start, self.offset + stop, step, size) + class SubBuffer(Buffer): import_from_mixin(SubBufferMixin) + class RWSubBuffer(RWBuffer): import_from_mixin(SubBufferMixin) diff --git a/pypy/interpreter/test/test_buffer.py b/pypy/interpreter/test/test_buffer.py --- a/pypy/interpreter/test/test_buffer.py +++ b/pypy/interpreter/test/test_buffer.py @@ -1,12 +1,10 @@ import py -from pypy.module.__builtin__.interp_memoryview import W_Buffer from rpython.tool.udir import udir testdir = udir.ensure('test_buffer', dir=1) class TestBuffer: - def test_buffer_w(self): space = self.space w_hello = space.wrapbytes('hello world') @@ -14,9 +12,9 @@ assert buf.getlength() == 11 assert buf.as_str() == 'hello world' assert buf.getslice(1, 6, 1, 5) == 'ello ' - assert space.buffer_w(W_Buffer(buf)) is buf + assert space.buffer_w(space.newbuffer(buf)) is buf assert space.bufferstr_w(w_hello) == 'hello world' - assert space.bufferstr_w(W_Buffer(space.buffer_w(w_hello))) == 'hello world' + assert space.bufferstr_w(space.newbuffer(space.buffer_w(w_hello))) == 'hello world' space.raises_w(space.w_TypeError, space.buffer_w, space.wrap(5)) e = space.raises_w(space.w_TypeError, space.buffer, space.wrap(5)) message = space.unwrap(e.value.get_w_value(space)) @@ -24,7 +22,7 @@ def test_file_write(self): space = self.space - w_buffer = W_Buffer(space.buffer_w(space.wrapbytes('hello world'))) + w_buffer = space.newbuffer(space.buffer_w(space.wrapbytes('hello world'))) filename = str(testdir.join('test_file_write')) space.appexec([w_buffer, space.wrap(filename)], """(buffer, filename): f = open(filename, 'wb') @@ -36,4 +34,4 @@ f.close() assert data == 'hello world' -# Note: some app-level tests for buffer are in module/__builtin__/test/. +# Note: some app-level tests for buffer are in objspace/std/test/test_memoryview.py. diff --git a/pypy/module/__builtin__/__init__.py b/pypy/module/__builtin__/__init__.py --- a/pypy/module/__builtin__/__init__.py +++ b/pypy/module/__builtin__/__init__.py @@ -29,14 +29,10 @@ interpleveldefs = { # constants + '__debug__' : '(space.w_True)', # XXX 'None' : '(space.w_None)', 'False' : '(space.w_False)', 'True' : '(space.w_True)', - '__debug__' : '(space.w_True)', # XXX - 'type' : '(space.w_type)', - 'object' : '(space.w_object)', - 'memoryview' : 'interp_memoryview.W_MemoryView', - 'open' : 'state.get(space).w_open', # interp-level function definitions diff --git a/pypy/module/__builtin__/test/test_buffer.py b/pypy/module/__builtin__/test/test_buffer.py deleted file mode 100644 --- a/pypy/module/__builtin__/test/test_buffer.py +++ /dev/null @@ -1,122 +0,0 @@ -"""Tests some behaviour of the buffer type that is not tested in -lib-python/2.5.2/test/test_types.py where the stdlib buffer tests live.""" - -class AppTestMemoryView: - spaceconfig = dict(usemodules=['array']) - - def test_basic(self): - v = memoryview(b"abc") - assert v.tobytes() == b"abc" - assert len(v) == 3 - assert v[0] == b'a' - assert list(v) == [b'a', b'b', b'c'] - assert v.tolist() == [97, 98, 99] - assert v[1] == b"b" - assert v[-1] == b"c" - raises(TypeError, "v[1] = 'x'") - assert v.readonly is True - w = v[1:234] - assert isinstance(w, memoryview) - assert len(w) == 2 - - def test_array_buffer(self): - import array - b = memoryview(array.array("B", [1, 2, 3])) - assert len(b) == 3 - assert b[0:3] == b"\x01\x02\x03" - - def test_nonzero(self): - assert memoryview(b'\x00') - assert not memoryview(b'') - import array - assert memoryview(array.array("B", [0])) - assert not memoryview(array.array("B", [])) - - def test_bytes(self): - assert bytes(memoryview(b'hello')) == b'hello' - - def test_repr(self): - assert repr(memoryview(b'hello')).startswith(' memoryview(b'ab')") - raises(TypeError, "memoryview(b'ab') >= memoryview(b'ab')") - raises(TypeError, "memoryview(b'ab') < memoryview(b'abc')") - raises(TypeError, "memoryview(b'ab') <= memoryview(b'ab')") - raises(TypeError, "memoryview(b'ab') > memoryview(b'aa')") - raises(TypeError, "memoryview(b'ab') >= memoryview(b'ab')") - - def test_hash(self): - raises(TypeError, "hash(memoryview(b'hello'))") - - def test_getitem_only_ints(self): - class MyInt(object): - def __init__(self, x): - self.x = x - - def __int__(self): - return self.x - - buf = memoryview(b'hello world') - raises(TypeError, "buf[MyInt(0)]") - raises(TypeError, "buf[MyInt(0):MyInt(5)]") - - def test_rw(self): - data = bytearray(b'abcefg') - v = memoryview(data) - assert v.readonly is False - v[0] = b'z' - assert data == bytearray(eval("b'zbcefg'")) - v[1:4] = b'123' - assert data == bytearray(eval("b'z123fg'")) - raises((ValueError, TypeError), "v[2] = 'spam'") - - def test_memoryview_attrs(self): - v = memoryview(b"a"*100) - assert v.format == "B" - assert v.itemsize == 1 - assert v.shape == (100,) - assert v.ndim == 1 - assert v.strides == (1,) - - def test_suboffsets(self): - v = memoryview(b"a"*100) - assert v.suboffsets == None - - def test_release(self): - v = memoryview(b"a"*100) - v.release() - raises(ValueError, len, v) - raises(ValueError, v.tolist) - raises(ValueError, v.tobytes) - raises(ValueError, "v[0]") - raises(ValueError, "v[0] = b'a'") - raises(ValueError, "v.format") - raises(ValueError, "v.itemsize") - raises(ValueError, "v.ndim") - raises(ValueError, "v.readonly") - raises(ValueError, "v.shape") - raises(ValueError, "v.strides") - raises(ValueError, "v.suboffsets") - raises(ValueError, "with v as cm: pass") - raises(ValueError, "memoryview(v)") - assert v == v - assert v != memoryview(b"a"*100) - assert v != b"a"*100 - assert "released memory" in repr(v) - - def test_context_manager(self): - v = memoryview(b"a"*100) - with v as cm: - assert cm is v - assert "released memory" in repr(v) diff --git a/pypy/module/__builtin__/test/test_builtin.py b/pypy/module/__builtin__/test/test_builtin.py --- a/pypy/module/__builtin__/test/test_builtin.py +++ b/pypy/module/__builtin__/test/test_builtin.py @@ -25,6 +25,17 @@ else: cls.w_safe_runtimerror = cls.space.wrap(sys.version_info < (2, 6)) + def test_builtin_names(self): + import __builtin__ + assert __builtin__.None is None + assert __builtin__.False is False + assert __builtin__.True is True + + assert __builtin__.buffer is buffer + assert __builtin__.bytes is str + assert __builtin__.dict is dict + assert __builtin__.memoryview is memoryview + def test_bytes_alias(self): assert bytes is not str assert isinstance(eval("b'hi'"), bytes) diff --git a/pypy/module/__pypy__/bytebuffer.py b/pypy/module/__pypy__/bytebuffer.py --- a/pypy/module/__pypy__/bytebuffer.py +++ b/pypy/module/__pypy__/bytebuffer.py @@ -4,11 +4,9 @@ from pypy.interpreter.buffer import RWBuffer from pypy.interpreter.gateway import unwrap_spec -from pypy.module.__builtin__.interp_memoryview import W_Buffer class ByteBuffer(RWBuffer): - def __init__(self, len): self.data = ['\x00'] * len @@ -24,4 +22,4 @@ @unwrap_spec(length=int) def bytebuffer(space, length): - return W_Buffer(ByteBuffer(length)) + return space.newbuffer(ByteBuffer(length)) diff --git a/pypy/module/_cffi_backend/cbuffer.py b/pypy/module/_cffi_backend/cbuffer.py --- a/pypy/module/_cffi_backend/cbuffer.py +++ b/pypy/module/_cffi_backend/cbuffer.py @@ -4,7 +4,7 @@ from pypy.interpreter.gateway import unwrap_spec, interp2app from pypy.interpreter.typedef import TypeDef, make_weakref_descr from pypy.module._cffi_backend import cdataobj, ctypeptr, ctypearray -from pypy.module.__builtin__.interp_memoryview import W_Buffer +from pypy.objspace.std.memoryview import W_Buffer from rpython.rtyper.annlowlevel import llstr from rpython.rtyper.lltypesystem import rffi diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py --- a/pypy/module/_io/interp_bufferedio.py +++ b/pypy/module/_io/interp_bufferedio.py @@ -5,7 +5,6 @@ TypeDef, GetSetProperty, generic_new_descr, interp_attrproperty_w) from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault from pypy.interpreter.buffer import RWBuffer -from pypy.module.__builtin__.interp_memoryview import W_Buffer from rpython.rlib.rstring import StringBuilder from rpython.rlib.rarithmetic import r_longlong, intmask from rpython.rlib import rposix @@ -522,7 +521,7 @@ def _raw_read(self, space, buffer, start, length): length = intmask(length) - w_buf = W_Buffer(RawBuffer(buffer, start, length)) + w_buf = space.newbuffer(RawBuffer(buffer, start, length)) while True: try: w_size = space.call_method(self.w_raw, "readinto", w_buf) diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -22,9 +22,9 @@ from pypy.interpreter.nestedscope import Cell from pypy.interpreter.module import Module from pypy.interpreter.function import StaticMethod +from pypy.objspace.std.memoryview import W_MemoryView from pypy.objspace.std.sliceobject import W_SliceObject from pypy.module.__builtin__.descriptor import W_Property -from pypy.module.__builtin__.interp_memoryview import W_MemoryView from pypy.module.micronumpy.base import W_NDimArray from rpython.rlib.entrypoint import entrypoint_lowlevel from rpython.rlib.rposix import is_valid_fd, validate_fd diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -14,7 +14,6 @@ from pypy.module.cpyext.pyobject import from_ref from pypy.module.cpyext.pyerrors import PyErr_Occurred from pypy.module.cpyext.state import State -from pypy.module.__builtin__.interp_memoryview import W_Buffer from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter.buffer import Buffer from pypy.interpreter.argument import Arguments @@ -250,7 +249,7 @@ ret = generic_cpy_call(space, func_target, w_self, view, flags) if rffi.cast(lltype.Signed, ret) == -1: space.fromcache(State).check_and_raise_exception(always=True) - return W_Buffer(CPyBuffer(view.c_buf, view.c_len, w_self)) + return space.newbuffer(CPyBuffer(view.c_buf, view.c_len, w_self)) def get_richcmp_func(OP_CONST): def inner(space, w_self, w_args, func): diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py --- a/pypy/module/micronumpy/ndarray.py +++ b/pypy/module/micronumpy/ndarray.py @@ -7,7 +7,6 @@ from rpython.rlib.rawstorage import RAW_STORAGE_PTR from rpython.rtyper.lltypesystem import rffi from rpython.tool.sourcetools import func_with_new_name -from pypy.module.__builtin__.interp_memoryview import W_Buffer from pypy.module.micronumpy import descriptor, ufuncs, boxes, arrayops, loop, \ support, constants as NPY from pypy.module.micronumpy.appbridge import get_appbridge_cache @@ -607,7 +606,7 @@ return self.implementation.get_buffer(space) def descr_get_data(self, space): - return W_Buffer(self.buffer_w(space)) + return space.newbuffer(self.buffer_w(space)) @unwrap_spec(offset=int, axis1=int, axis2=int) def descr_diagonal(self, space, offset=0, axis1=0, axis2=1): diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py --- a/pypy/objspace/fake/objspace.py +++ b/pypy/objspace/fake/objspace.py @@ -175,6 +175,9 @@ def newseqiter(self, x): return w_some_obj() + def newbuffer(self, x): + return w_some_obj() + def marshal_w(self, w_obj): "NOT_RPYTHON" raise NotImplementedError diff --git a/pypy/module/__builtin__/interp_memoryview.py b/pypy/objspace/std/memoryview.py rename from pypy/module/__builtin__/interp_memoryview.py rename to pypy/objspace/std/memoryview.py --- a/pypy/module/__builtin__/interp_memoryview.py +++ b/pypy/objspace/std/memoryview.py @@ -37,6 +37,7 @@ """ def __init__(self, buf): + assert isinstance(buf, buffer.Buffer) self.buf = buf def buffer_w(self, space): @@ -44,7 +45,7 @@ @staticmethod @unwrap_spec(offset=int, size=int) - def descr_new(space, w_subtype, w_object, offset=0, size=-1): + def descr_new_buffer(space, w_subtype, w_object, offset=0, size=-1): if space.isinstance_w(w_object, space.w_unicode): # unicode objects support the old buffer interface # but not the new buffer interface (change in python 2.7) @@ -145,7 +146,7 @@ start of the object (or at the specified offset). The slice will extend to the end of the target object (or with the specified size). """, - __new__ = interp2app(W_Buffer.descr_new), + __new__ = interp2app(W_Buffer.descr_new_buffer), __len__ = interp2app(W_Buffer.descr_len), __getitem__ = interp2app(W_Buffer.descr_getitem), __setitem__ = interp2app(W_Buffer.descr_setitem), @@ -171,6 +172,7 @@ """ def __init__(self, buf): + assert isinstance(buf, buffer.Buffer) self.buf = buf def buffer_w(self, space): @@ -185,9 +187,8 @@ return self.buf @staticmethod - def descr_new(space, w_subtype, w_object): - w_memoryview = W_MemoryView(space.buffer_w(w_object)) - return w_memoryview + def descr_new_memoryview(space, w_subtype, w_object): + return W_MemoryView(space.buffer_w(w_object)) def _make_descr__cmp(name): def descr__cmp(self, space, w_other): @@ -328,7 +329,7 @@ __doc__ = """\ Create a new memoryview object which references the given object. """, - __new__ = interp2app(W_MemoryView.descr_new), + __new__ = interp2app(W_MemoryView.descr_new_memoryview), __eq__ = interp2app(W_MemoryView.descr_eq), __getitem__ = interp2app(W_MemoryView.descr_getitem), __len__ = interp2app(W_MemoryView.descr_len), diff --git a/pypy/objspace/std/model.py b/pypy/objspace/std/model.py --- a/pypy/objspace/std/model.py +++ b/pypy/objspace/std/model.py @@ -61,7 +61,7 @@ from pypy.objspace.std import unicodeobject from pypy.objspace.std import dictproxyobject from pypy.objspace.std import proxyobject - from pypy.module.__builtin__.interp_memoryview import W_Buffer + from pypy.objspace.std.memoryview import W_Buffer, W_MemoryView import pypy.objspace.std.default # register a few catch-all multimethods import pypy.objspace.std.marshal_impl # install marshal multimethods @@ -81,6 +81,7 @@ self.pythontypes.append(boolobject.W_BoolObject.typedef) self.pythontypes.append(longobject.W_LongObject.typedef) self.pythontypes.append(W_Buffer.typedef) + self.pythontypes.append(W_MemoryView.typedef) # the set of implementation types self.typeorder = { diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py --- a/pypy/objspace/std/objspace.py +++ b/pypy/objspace/std/objspace.py @@ -25,6 +25,7 @@ from pypy.objspace.std.iterobject import W_AbstractSeqIterObject from pypy.objspace.std.listobject import W_ListObject from pypy.objspace.std.longobject import W_LongObject, newlong +from pypy.objspace.std.memoryview import W_Buffer from pypy.objspace.std.noneobject import W_NoneObject from pypy.objspace.std.objectobject import W_ObjectObject from pypy.objspace.std.iterobject import W_SeqIterObject @@ -339,6 +340,9 @@ def newseqiter(self, w_obj): return W_SeqIterObject(w_obj) + def newbuffer(self, w_obj): + return W_Buffer(w_obj) + def type(self, w_obj): jit.promote(w_obj.__class__) return w_obj.getclass(self) diff --git a/pypy/objspace/std/test/test_memoryview.py b/pypy/objspace/std/test/test_memoryview.py new file mode 100644 --- /dev/null +++ b/pypy/objspace/std/test/test_memoryview.py @@ -0,0 +1,119 @@ +class AppTestMemoryView: + spaceconfig = dict(usemodules=['array']) + + def test_basic(self): + v = memoryview(b"abc") + assert v.tobytes() == b"abc" + assert len(v) == 3 + assert v[0] == b'a' + assert list(v) == [b'a', b'b', b'c'] + assert v.tolist() == [97, 98, 99] + assert v[1] == b"b" + assert v[-1] == b"c" + raises(TypeError, "v[1] = 'x'") + assert v.readonly is True + w = v[1:234] + assert isinstance(w, memoryview) + assert len(w) == 2 + + def test_array_buffer(self): + import array + b = memoryview(array.array("B", [1, 2, 3])) + assert len(b) == 3 + assert b[0:3] == b"\x01\x02\x03" + + def test_nonzero(self): + assert memoryview(b'\x00') + assert not memoryview(b'') + import array + assert memoryview(array.array("B", [0])) + assert not memoryview(array.array("B", [])) + + def test_bytes(self): + assert bytes(memoryview(b'hello')) == b'hello' + + def test_repr(self): + assert repr(memoryview(b'hello')).startswith(' memoryview(b'ab')") + raises(TypeError, "memoryview(b'ab') >= memoryview(b'ab')") + raises(TypeError, "memoryview(b'ab') < memoryview(b'abc')") + raises(TypeError, "memoryview(b'ab') <= memoryview(b'ab')") + raises(TypeError, "memoryview(b'ab') > memoryview(b'aa')") + raises(TypeError, "memoryview(b'ab') >= memoryview(b'ab')") + + def test_hash(self): + raises(TypeError, "hash(memoryview(b'hello'))") + + def test_getitem_only_ints(self): + class MyInt(object): + def __init__(self, x): + self.x = x + + def __int__(self): + return self.x + + buf = memoryview(b'hello world') + raises(TypeError, "buf[MyInt(0)]") + raises(TypeError, "buf[MyInt(0):MyInt(5)]") + + def test_rw(self): + data = bytearray(b'abcefg') + v = memoryview(data) + assert v.readonly is False + v[0] = b'z' + assert data == bytearray(eval("b'zbcefg'")) + v[1:4] = b'123' + assert data == bytearray(eval("b'z123fg'")) + raises((ValueError, TypeError), "v[2] = 'spam'") + + def test_memoryview_attrs(self): + v = memoryview(b"a"*100) + assert v.format == "B" + assert v.itemsize == 1 + assert v.shape == (100,) + assert v.ndim == 1 + assert v.strides == (1,) + + def test_suboffsets(self): + v = memoryview(b"a"*100) + assert v.suboffsets == None + + def test_release(self): + v = memoryview(b"a"*100) + v.release() + raises(ValueError, len, v) + raises(ValueError, v.tolist) + raises(ValueError, v.tobytes) + raises(ValueError, "v[0]") + raises(ValueError, "v[0] = b'a'") + raises(ValueError, "v.format") + raises(ValueError, "v.itemsize") + raises(ValueError, "v.ndim") + raises(ValueError, "v.readonly") + raises(ValueError, "v.shape") + raises(ValueError, "v.strides") + raises(ValueError, "v.suboffsets") + raises(ValueError, "with v as cm: pass") + raises(ValueError, "memoryview(v)") + assert v == v + assert v != memoryview(b"a"*100) + assert v != b"a"*100 + assert "released memory" in repr(v) + + def test_context_manager(self): + v = memoryview(b"a"*100) + with v as cm: + assert cm is v + assert "released memory" in repr(v) diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py --- a/rpython/rlib/rwin32.py +++ b/rpython/rlib/rwin32.py @@ -169,6 +169,7 @@ cfile = udir.join('dosmaperr.c') cfile.write(r''' #include + #include #include #ifdef __GNUC__ #define _dosmaperr mingw_dosmaperr From noreply at buildbot.pypy.org Wed Mar 19 01:49:58 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Wed, 19 Mar 2014 01:49:58 +0100 (CET) Subject: [pypy-commit] pypy default: cleanups Message-ID: <20140319004958.B1D8F1C0166@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r70070:d88001e37970 Date: 2014-03-18 19:45 -0400 http://bitbucket.org/pypy/pypy/changeset/d88001e37970/ Log: cleanups diff --git a/pypy/interpreter/buffer.py b/pypy/interpreter/buffer.py --- a/pypy/interpreter/buffer.py +++ b/pypy/interpreter/buffer.py @@ -71,8 +71,8 @@ assert 0 <= start <= stop return self.value[start:stop] return "".join([self.value[start + i*step] for i in xrange(size)]) +# ____________________________________________________________ -# ____________________________________________________________ class SubBufferMixin(object): _attrs_ = ['buffer', 'offset', 'size'] @@ -98,7 +98,8 @@ if start == stop: return '' # otherwise, adding self.offset might make them # out of bounds - return self.buffer.getslice(self.offset + start, self.offset + stop, step, size) + return self.buffer.getslice(self.offset + start, self.offset + stop, + step, size) class SubBuffer(Buffer): diff --git a/pypy/module/array/test/test_array.py b/pypy/module/array/test/test_array.py --- a/pypy/module/array/test/test_array.py +++ b/pypy/module/array/test/test_array.py @@ -1,25 +1,8 @@ import sys -import py -import py.test - - -## class AppTestSimpleArray: -## spaceconfig = dict(usemodules=('array',)) -## def setup_class(cls): -## cls.w_simple_array = cls.space.appexec([], """(): -## import array -## return array.simple_array -## """) - -## def test_simple(self): -## a = self.simple_array(10) -## a[5] = 7.42 -## assert a[5] == 7.42 +import pytest class BaseArrayTests: - - def test_ctor(self): assert len(self.array('c')) == 0 assert len(self.array('i')) == 0 @@ -563,7 +546,6 @@ assert not a > 2*a assert not a >= 2*a - def test_reduce(self): import pickle a = self.array('i', [1, 2, 3]) @@ -794,7 +776,6 @@ assert img[3, 25] == 3 * 9 - def test_override_from(self): class mya(self.array): def fromlist(self, lst): @@ -879,41 +860,41 @@ def test_assign_object_with_special_methods(self): from array import array - + class Num(object): def __float__(self): return 5.25 - + def __int__(self): return 7 - + class NotNum(object): pass - + class Silly(object): def __float__(self): return None - + def __int__(self): - return None + return None class OldNum: def __float__(self): return 6.25 - + def __int__(self): return 8 - + class OldNotNum: pass - + class OldSilly: def __float__(self): return None - + def __int__(self): return None - + for tc in 'bBhHiIlL': a = array(tc, [0]) raises(TypeError, a.__setitem__, 0, 1.0) @@ -931,7 +912,7 @@ a = array(tc, [0]) a[0] = 1.0 a[0] = 1 - a[0] = Num() + a[0] = Num() assert a[0] == 5.25 raises(TypeError, a.__setitem__, NotNum()) a[0] = OldNum() @@ -939,24 +920,23 @@ raises(TypeError, a.__setitem__, OldNotNum()) raises(TypeError, a.__setitem__, Silly()) raises(TypeError, a.__setitem__, OldSilly()) - + a = array('c', 'hi') a[0] = 'b' assert a[0] == 'b' - + a = array('u', u'hi') a[0] = u'b' assert a[0] == u'b' - + class TestCPythonsOwnArray(BaseArrayTests): - def setup_class(cls): import array cls.array = array.array import struct cls.struct = struct - cls.tempfile = str(py.test.ensuretemp('array').join('tmpfile')) + cls.tempfile = str(pytest.ensuretemp('array').join('tmpfile')) cls.maxint = sys.maxint @@ -969,7 +949,7 @@ return array.array """) cls.w_tempfile = cls.space.wrap( - str(py.test.ensuretemp('array').join('tmpfile'))) + str(pytest.ensuretemp('array').join('tmpfile'))) cls.w_maxint = cls.space.wrap(sys.maxint) def test_buffer_info(self): @@ -1036,11 +1016,11 @@ def test_getitem_only_ints(self): class MyInt(object): - def __init__(self, x): - self.x = x + def __init__(self, x): + self.x = x - def __int__(self): - return self.x + def __int__(self): + return self.x a = self.array('i', [1, 2, 3, 4, 5, 6]) raises(TypeError, "a[MyInt(0)]") @@ -1050,4 +1030,3 @@ class AppTestArrayBuiltinShortcut(AppTestArray): spaceconfig = AppTestArray.spaceconfig.copy() spaceconfig['objspace.std.builtinshortcut'] = True - diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py --- a/pypy/module/cpyext/test/test_arraymodule.py +++ b/pypy/module/cpyext/test/test_arraymodule.py @@ -1,7 +1,5 @@ from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase -import py -import sys class AppTestArrayModule(AppTestCpythonExtensionBase): enable_leak_checking = False @@ -21,7 +19,7 @@ module = self.import_module(name='array') arr = module.array('i', [1,2,3]) sum = 0 - for i in arr: + for i in arr: sum += i assert sum == 6 @@ -60,4 +58,3 @@ '\x02\0\0\0' '\x03\0\0\0' '\x04\0\0\0') - From noreply at buildbot.pypy.org Wed Mar 19 01:50:00 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Wed, 19 Mar 2014 01:50:00 +0100 (CET) Subject: [pypy-commit] pypy default: test/fix marshal behavior wrt buffers Message-ID: <20140319005000.1E6B31C0166@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r70071:07f40b57d13c Date: 2014-03-18 20:35 -0400 http://bitbucket.org/pypy/pypy/changeset/07f40b57d13c/ Log: test/fix marshal behavior wrt buffers diff --git a/pypy/module/marshal/interp_marshal.py b/pypy/module/marshal/interp_marshal.py --- a/pypy/module/marshal/interp_marshal.py +++ b/pypy/module/marshal/interp_marshal.py @@ -217,11 +217,6 @@ self.space.marshal_w(w_obj, self) def dump_w_obj(self, w_obj): - space = self.space - if (space.type(w_obj).is_heaptype() and - space.lookup(w_obj, "__buffer__") is None): - w_err = space.wrap("only builtins can be marshaled") - raise OperationError(space.w_ValueError, w_err) try: self.put_w_obj(w_obj) except rstackovf.StackOverflow: diff --git a/pypy/module/marshal/test/test_marshal.py b/pypy/module/marshal/test/test_marshal.py --- a/pypy/module/marshal/test/test_marshal.py +++ b/pypy/module/marshal/test/test_marshal.py @@ -2,6 +2,8 @@ class AppTestMarshal: + spaceconfig = {'usemodules': ['array']} + def setup_class(cls): tmpfile = udir.join('AppTestMarshal.tmp') cls.w_tmpfile = cls.space.wrap(str(tmpfile)) @@ -173,7 +175,15 @@ for cls in types: class subtype(cls): pass - raises(ValueError, marshal.dumps, subtype) + exc = raises(ValueError, marshal.dumps, subtype) + assert str(exc.value) == 'unmarshallable object' + + def test_valid_subtypes(self): + import marshal + from array import array + class subtype(array): + pass + assert marshal.dumps(subtype('c', 'test')) == marshal.dumps(array('c', 'test')) def test_bad_typecode(self): import marshal @@ -182,7 +192,8 @@ class AppTestSmallLong(AppTestMarshal): - spaceconfig = {"objspace.std.withsmalllong": True} + spaceconfig = AppTestMarshal.spaceconfig.copy() + spaceconfig["objspace.std.withsmalllong"] = True def test_smalllong(self): import __pypy__ From noreply at buildbot.pypy.org Wed Mar 19 01:50:01 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Wed, 19 Mar 2014 01:50:01 +0100 (CET) Subject: [pypy-commit] pypy default: fix buffer_w on objects defining pypy's app-level buffer interface Message-ID: <20140319005001.86ED61C0166@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r70072:66fcaf1343c6 Date: 2014-03-18 20:41 -0400 http://bitbucket.org/pypy/pypy/changeset/66fcaf1343c6/ Log: fix buffer_w on objects defining pypy's app-level buffer interface diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -195,6 +195,11 @@ return None def buffer_w(self, space): + w_impl = space.lookup(self, '__buffer__') + if w_impl is not None: + w_result = space.get_and_call_function(w_impl, self) + if space.isinstance_w(w_result, space.w_buffer): + return w_result.buffer_w(space) self._typed_unwrap_error(space, "buffer") def str_w(self, space): diff --git a/pypy/objspace/std/test/test_memoryview.py b/pypy/objspace/std/test/test_memoryview.py --- a/pypy/objspace/std/test/test_memoryview.py +++ b/pypy/objspace/std/test/test_memoryview.py @@ -2,10 +2,14 @@ spaceconfig = dict(usemodules=['array']) def test_init(self): + import sys class A(object): def __buffer__(self): return buffer('123') - raises(TypeError, buffer, A()) + if '__pypy__' not in sys.builtin_module_names: + raises(TypeError, buffer, A()) + else: + assert buffer(A()) == buffer('123') def test_unicode_buffer(self): import sys From noreply at buildbot.pypy.org Wed Mar 19 02:50:20 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Wed, 19 Mar 2014 02:50:20 +0100 (CET) Subject: [pypy-commit] pypy default: unused imports/variables Message-ID: <20140319015020.5665C1C0166@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r70073:4de7fd2dd15f Date: 2014-03-18 21:34 -0400 http://bitbucket.org/pypy/pypy/changeset/4de7fd2dd15f/ Log: unused imports/variables diff --git a/pypy/objspace/std/callmethod.py b/pypy/objspace/std/callmethod.py --- a/pypy/objspace/std/callmethod.py +++ b/pypy/objspace/std/callmethod.py @@ -11,7 +11,6 @@ """ from pypy.interpreter import function -from pypy.objspace.descroperation import object_getattribute from rpython.rlib import jit from pypy.objspace.std.mapdict import LOOKUP_METHOD_mapdict, \ LOOKUP_METHOD_mapdict_fill_cache_method @@ -36,7 +35,6 @@ if space.config.objspace.std.withmapdict and not jit.we_are_jitted(): # mapdict has an extra-fast version of this function - from pypy.objspace.std.mapdict import LOOKUP_METHOD_mapdict if LOOKUP_METHOD_mapdict(f, nameindex, w_obj): return @@ -79,7 +77,7 @@ n_kwargs = (oparg >> 8) & 0xff w_self = f.peekvalue(n_args + (2 * n_kwargs)) n = n_args + (w_self is not None) - + if not n_kwargs: w_callable = f.peekvalue(n_args + (2 * n_kwargs) + 1) try: @@ -98,7 +96,7 @@ key = f.space.str_w(w_key) keywords[n_kwargs] = key keywords_w[n_kwargs] = w_value - + arguments = f.popvalues(n) # includes w_self if it is not None args = f.argument_factory(arguments, keywords, keywords_w, None, None) if w_self is None: diff --git a/pypy/objspace/std/celldict.py b/pypy/objspace/std/celldict.py --- a/pypy/objspace/std/celldict.py +++ b/pypy/objspace/std/celldict.py @@ -140,7 +140,7 @@ for key, cell in iterator()] def clear(self, w_dict): - iterator = self.unerase(w_dict.dstorage).clear() + self.unerase(w_dict.dstorage).clear() self.mutated() def popitem(self, w_dict): diff --git a/pypy/objspace/std/complexobject.py b/pypy/objspace/std/complexobject.py --- a/pypy/objspace/std/complexobject.py +++ b/pypy/objspace/std/complexobject.py @@ -1,4 +1,5 @@ -from pypy.interpreter import gateway +import math + from pypy.interpreter.error import OperationError from pypy.objspace.std import newformat from pypy.objspace.std.intobject import W_IntObject @@ -12,8 +13,6 @@ from rpython.rlib import jit, rcomplex from rpython.rlib.rarithmetic import intmask, r_ulonglong -import math - class W_AbstractComplexObject(W_Object): __slots__ = () diff --git a/pypy/objspace/std/complextype.py b/pypy/objspace/std/complextype.py --- a/pypy/objspace/std/complextype.py +++ b/pypy/objspace/std/complextype.py @@ -1,7 +1,6 @@ from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault from pypy.objspace.std.register_all import register_all -from pypy.objspace.std.noneobject import W_NoneObject from pypy.objspace.std.stdtypedef import GetSetProperty, StdTypeDef from pypy.objspace.std.stdtypedef import StdObjSpaceMultiMethod from rpython.rlib.rfloat import string_to_float diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py --- a/pypy/objspace/std/dictmultiobject.py +++ b/pypy/objspace/std/dictmultiobject.py @@ -1184,7 +1184,7 @@ w_clone.setup_iterator() # spool until we have the same pos while w_clone.pos < self.pos: - w_obj = w_clone.next_entry() + w_clone.next_entry() w_clone.pos += 1 stuff = [w_clone.next_entry() for i in range(w_clone.pos, w_clone.len)] w_res = space.newlist(stuff) diff --git a/pypy/objspace/std/dictproxyobject.py b/pypy/objspace/std/dictproxyobject.py --- a/pypy/objspace/std/dictproxyobject.py +++ b/pypy/objspace/std/dictproxyobject.py @@ -1,7 +1,6 @@ -from pypy.objspace.std.model import registerimplementation, W_Object -from pypy.objspace.std.register_all import register_all -from pypy.objspace.std.dictmultiobject import W_DictMultiObject, create_iterator_classes -from pypy.objspace.std.dictmultiobject import DictStrategy +#from pypy.objspace.std.model import registerimplementation, W_Object +#from pypy.objspace.std.register_all import register_all +from pypy.objspace.std.dictmultiobject import DictStrategy, create_iterator_classes from pypy.objspace.std.typeobject import unwrap_cell from pypy.interpreter.error import OperationError, oefmt @@ -9,7 +8,6 @@ class DictProxyStrategy(DictStrategy): - erase, unerase = rerased.new_erasing_pair("dictproxy") erase = staticmethod(erase) unerase = staticmethod(unerase) @@ -62,7 +60,6 @@ w_type.dict_w[key] = w_value def setdefault(self, w_dict, w_key, w_default): - space = self.space w_result = self.getitem(w_dict, w_key) if w_result is not None: return w_result diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py --- a/pypy/objspace/std/floatobject.py +++ b/pypy/objspace/std/floatobject.py @@ -1,12 +1,12 @@ +import math import operator from pypy.interpreter.error import OperationError, oefmt -from pypy.objspace.std import model, newformat +from pypy.objspace.std import newformat from pypy.objspace.std.floattype import float_typedef, W_AbstractFloatObject from pypy.objspace.std.multimethod import FailedToImplementArgs from pypy.objspace.std.model import registerimplementation, W_Object from pypy.objspace.std.register_all import register_all -from pypy.objspace.std.noneobject import W_NoneObject from pypy.objspace.std.longobject import W_LongObject from rpython.rlib.rarithmetic import ovfcheck_float_to_int, intmask, LONG_BIT from rpython.rlib.rfloat import ( @@ -16,8 +16,6 @@ from rpython.rlib import rfloat from rpython.tool.sourcetools import func_with_new_name - -import math from pypy.objspace.std.intobject import W_IntObject diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py --- a/pypy/objspace/std/intobject.py +++ b/pypy/objspace/std/intobject.py @@ -4,12 +4,11 @@ translation this module uses rarithmetic.ovfcheck to explicitly check for overflows, something CPython does not do anymore. """ - import operator import sys from rpython.rlib import jit -from rpython.rlib.objectmodel import instantiate, import_from_mixin, specialize +from rpython.rlib.objectmodel import instantiate from rpython.rlib.rarithmetic import ( LONG_BIT, is_valid_int, ovfcheck, r_longlong, r_uint, string_to_int) from rpython.rlib.rbigint import rbigint @@ -20,7 +19,6 @@ from pypy.interpreter import typedef from pypy.interpreter.baseobjspace import W_Root -from pypy.interpreter.buffer import Buffer from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter.gateway import WrappedDefault, interp2app, unwrap_spec from pypy.objspace.std import newformat @@ -28,12 +26,10 @@ BINARY_OPS, CMP_OPS, COMMUTATIVE_OPS, IDTAG_INT) from pypy.objspace.std.stdtypedef import StdTypeDef - SENTINEL = object() class W_AbstractIntObject(W_Root): - __slots__ = () def is_w(self, space, w_other): @@ -706,7 +702,6 @@ else: value, w_longval = _string_to_int_or_long(space, w_value, buf.as_str()) - ok = True else: base = space.int_w(w_base) diff --git a/pypy/objspace/std/longobject.py b/pypy/objspace/std/longobject.py --- a/pypy/objspace/std/longobject.py +++ b/pypy/objspace/std/longobject.py @@ -9,7 +9,6 @@ from pypy.interpreter import typedef from pypy.interpreter.baseobjspace import W_Root -from pypy.interpreter.buffer import Buffer from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter.gateway import ( WrappedDefault, interp2app, interpindirect2app, unwrap_spec) diff --git a/pypy/objspace/std/marshal_impl.py b/pypy/objspace/std/marshal_impl.py --- a/pypy/objspace/std/marshal_impl.py +++ b/pypy/objspace/std/marshal_impl.py @@ -10,7 +10,7 @@ from pypy.interpreter.error import OperationError from pypy.objspace.std.register_all import register_all -from rpython.rlib.rarithmetic import LONG_BIT, r_longlong, r_uint, intmask +from rpython.rlib.rarithmetic import LONG_BIT, r_longlong, r_uint from pypy.objspace.std import model from pypy.objspace.std.dictmultiobject import W_DictMultiObject from pypy.interpreter.special import Ellipsis @@ -201,7 +201,6 @@ register(TYPE_BINARY_COMPLEX, unmarshal_Complex_bin) def marshal_w__Long(space, w_long, m): - from rpython.rlib.rbigint import rbigint from rpython.rlib.rarithmetic import r_ulonglong m.start(TYPE_LONG) SHIFT = 15 @@ -375,7 +374,6 @@ lng = u.atom_lng(tc) res = [None] * lng idx = 0 - space = u.space while idx < lng: res[idx] = unmarshal_str(u) idx += 1 diff --git a/pypy/objspace/std/multimethod.py b/pypy/objspace/std/multimethod.py --- a/pypy/objspace/std/multimethod.py +++ b/pypy/objspace/std/multimethod.py @@ -866,7 +866,6 @@ entryarray = CompressedArray(null_entry) indexarray = self.mrdtable.indexarray lst = self.mrdtable.list_of_types - indexline = [] def compress(typesprefix, typesnum): if len(typesprefix) == self.multimethod.arity: diff --git a/pypy/objspace/std/newformat.py b/pypy/objspace/std/newformat.py --- a/pypy/objspace/std/newformat.py +++ b/pypy/objspace/std/newformat.py @@ -461,7 +461,6 @@ if not got_align: self._align = "=" i += 1 - start_i = i self._width, i = _parse_int(self.space, spec, i, length) if length != i and spec[i] == ",": self._thousands_sep = True @@ -576,7 +575,6 @@ return space.wrap(self._pad(string)) def _get_locale(self, tp): - space = self.space if tp == "n": dec, thousands, grouping = rlocale.numeric_formatting() elif self._thousands_sep: @@ -673,12 +671,10 @@ grouping = self._loc_grouping min_width = spec.n_min_width grouping_state = 0 - count = 0 left = spec.n_digits n_ts = len(self._loc_thousands) need_separator = False done = False - groupings = len(grouping) previous = 0 while True: group = ord(grouping[grouping_state]) diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py --- a/pypy/objspace/std/objspace.py +++ b/pypy/objspace/std/objspace.py @@ -306,7 +306,6 @@ strdict=strdict, kwargs=kwargs) def newset(self): - from pypy.objspace.std.setobject import newset return W_SetObject(self, None) def newslice(self, w_start, w_end, w_step): diff --git a/pypy/objspace/std/proxy_helpers.py b/pypy/objspace/std/proxy_helpers.py --- a/pypy/objspace/std/proxy_helpers.py +++ b/pypy/objspace/std/proxy_helpers.py @@ -1,10 +1,8 @@ - """ Some transparent helpers, put here because of cyclic imports """ -from pypy.objspace.std.model import W_ANY, W_Object -from pypy.interpreter import baseobjspace +from pypy.objspace.std.model import W_ANY from pypy.interpreter.argument import Arguments from rpython.tool.sourcetools import func_with_new_name @@ -24,7 +22,7 @@ def function(space, w_transparent_list, __args__): args = __args__.prepend(space.wrap(op_name)) return space.call_args(w_transparent_list.w_controller, args) - + function = func_with_new_name(function, mm.name) mm.register(function, type_) @@ -32,14 +30,14 @@ def function(space, w_transparent_list, *args_w): args = Arguments(space, [space.wrap(op_name)] + list(args_w[:-1]) + args_w[-1]) return space.call_args(w_transparent_list.w_controller, args) - + function = func_with_new_name(function, mm.name) mm.register(function, type_, *([W_ANY] * (mm.arity - 1))) def install_mm_trampoline(type_, mm, is_local): classname = type_.__name__[2:] mm_name, op_name = create_mm_names(classname, mm, is_local) - + if ['__args__'] == mm.argnames_after: return install_general_args_trampoline(type_, mm, is_local, op_name) if ['args_w'] == mm.argnames_after: @@ -58,10 +56,10 @@ """ if mm.arity != 2: return False - + if len(mm.specialnames) != 2: return False - + # search over the signatures for signature in mm.signatures(): if signature == (type_.original, type_.original): @@ -69,21 +67,21 @@ return False def install_mm_special(type_, mm, is_local): - classname = type_.__name__[2:] + #classname = type_.__name__[2:] #mm_name, op_name = create_mm_names(classname, mm, is_local) - + def function(space, w_any, w_transparent_list): retval = space.call_function(w_transparent_list.w_controller, space.wrap(mm.specialnames[1]), w_any) return retval - + function = func_with_new_name(function, mm.specialnames[0]) - + mm.register(function, type_.typedef.any, type_) def register_type(type_): from pypy.objspace.std.stdtypedef import multimethods_defined_on - + for mm, is_local in multimethods_defined_on(type_.original): if not mm.name.startswith('__'): install_mm_trampoline(type_, mm, is_local) diff --git a/pypy/objspace/std/proxyobject.py b/pypy/objspace/std/proxyobject.py --- a/pypy/objspace/std/proxyobject.py +++ b/pypy/objspace/std/proxyobject.py @@ -1,15 +1,8 @@ - """ transparent list implementation """ - -from pypy.objspace.std.model import W_Object from pypy.interpreter.error import OperationError from pypy.interpreter import baseobjspace -#class W_Transparent(W_Object): -# def __init__(self, w_controller): -# self.controller = w_controller - def transparent_class(name, BaseCls): class W_Transparent(BaseCls): @@ -72,25 +65,3 @@ return W_Transparent W_Transparent = transparent_class('W_Transparent', baseobjspace.W_Root) -#W_TransparentObject = transparent_class('W_TransparentObject', W_Object) - -#from pypy.objspace.std.objecttype import object_typedef -#W_TransparentObject.typedef = object_typedef - -from pypy.interpreter.typedef import Function, GeneratorIterator, PyTraceback, \ - PyFrame, PyCode - -class W_TransparentFunction(W_Transparent): - typedef = Function.typedef - -class W_TransparentTraceback(W_Transparent): - typedef = PyTraceback.typedef - -class W_TransparentCode(W_Transparent): - typedef = PyCode.typedef - -class W_TransparentFrame(W_Transparent): - typedef = PyFrame.typedef - -class W_TransparentGenerator(W_Transparent): - typedef = GeneratorIterator.typedef diff --git a/pypy/objspace/std/slicetype.py b/pypy/objspace/std/slicetype.py --- a/pypy/objspace/std/slicetype.py +++ b/pypy/objspace/std/slicetype.py @@ -1,4 +1,4 @@ -from pypy.interpreter import baseobjspace, gateway +from pypy.interpreter import gateway from pypy.interpreter.typedef import GetSetProperty from pypy.objspace.std.stdtypedef import StdTypeDef, SMM from pypy.objspace.std.register_all import register_all diff --git a/pypy/objspace/std/stdtypedef.py b/pypy/objspace/std/stdtypedef.py --- a/pypy/objspace/std/stdtypedef.py +++ b/pypy/objspace/std/stdtypedef.py @@ -1,8 +1,7 @@ -from pypy.interpreter import gateway, baseobjspace, argument +from pypy.interpreter import gateway, baseobjspace from pypy.interpreter.error import OperationError, oefmt -from pypy.interpreter.typedef import TypeDef, GetSetProperty, Member -from pypy.interpreter.typedef import descr_get_dict, descr_set_dict -from pypy.interpreter.typedef import descr_del_dict +from pypy.interpreter.typedef import TypeDef, GetSetProperty, \ + descr_get_dict, descr_set_dict, descr_del_dict from pypy.interpreter.baseobjspace import SpaceCache from pypy.objspace.std import model from pypy.objspace.std.model import StdObjSpaceMultiMethod diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py --- a/pypy/objspace/std/stringmethods.py +++ b/pypy/objspace/std/stringmethods.py @@ -437,7 +437,6 @@ def descr_split(self, space, w_sep=None, maxsplit=-1): res = [] value = self._val(space) - length = len(value) if space.is_none(w_sep): res = split(value, maxsplit=maxsplit) return self._newlist_unwrapped(space, res) diff --git a/pypy/objspace/std/transparent.py b/pypy/objspace/std/transparent.py --- a/pypy/objspace/std/transparent.py +++ b/pypy/objspace/std/transparent.py @@ -1,14 +1,30 @@ - """ transparent.py - Several transparent proxy helpers """ - from pypy.interpreter import gateway from pypy.interpreter.error import OperationError, oefmt -from pypy.objspace.std.proxyobject import * +from pypy.interpreter.typedef import Function, GeneratorIterator, PyTraceback, \ + PyFrame, PyCode +from pypy.objspace.std.proxyobject import W_Transparent from pypy.objspace.std.typeobject import W_TypeObject -from rpython.rlib.objectmodel import r_dict from rpython.rlib.unroll import unrolling_iterable + +class W_TransparentFunction(W_Transparent): + typedef = Function.typedef + +class W_TransparentTraceback(W_Transparent): + typedef = PyTraceback.typedef + +class W_TransparentCode(W_Transparent): + typedef = PyCode.typedef + +class W_TransparentFrame(W_Transparent): + typedef = PyFrame.typedef + +class W_TransparentGenerator(W_Transparent): + typedef = GeneratorIterator.typedef + + class TypeCache(object): def __init__(self): self.cache = [] @@ -28,13 +44,10 @@ space.wrap(app_proxy_controller)) - def proxy(space, w_type, w_controller): """tproxy(typ, controller) -> obj Return something that looks like it is of type typ. Its behaviour is completely controlled by the controller.""" - from pypy.interpreter.typedef import Function, PyTraceback, PyFrame, \ - PyCode, GeneratorIterator if not space.is_true(space.callable(w_controller)): raise OperationError(space.w_TypeError, space.wrap("controller should be function")) diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py --- a/pypy/objspace/std/typeobject.py +++ b/pypy/objspace/std/typeobject.py @@ -2,12 +2,12 @@ from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter.function import Function, StaticMethod -from pypy.interpreter.typedef import weakref_descr, GetSetProperty,\ +from pypy.interpreter.typedef import weakref_descr, GetSetProperty, Member, \ descr_get_dict from pypy.interpreter.astcompiler.misc import mangle from pypy.objspace.std.model import W_Object from pypy.objspace.std.register_all import register_all -from pypy.objspace.std.stdtypedef import std_dict_descr, issubtypedef, Member +from pypy.objspace.std.stdtypedef import std_dict_descr, issubtypedef from pypy.objspace.std.stdtypedef import StdTypeDef from rpython.rlib.jit import (promote, elidable_promote, we_are_jitted, From noreply at buildbot.pypy.org Wed Mar 19 03:15:50 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Wed, 19 Mar 2014 03:15:50 +0100 (CET) Subject: [pypy-commit] pypy default: more unused Message-ID: <20140319021550.D0DB11C0166@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r70074:851618b4e09d Date: 2014-03-18 22:10 -0400 http://bitbucket.org/pypy/pypy/changeset/851618b4e09d/ Log: more unused diff --git a/pypy/interpreter/buffer.py b/pypy/interpreter/buffer.py --- a/pypy/interpreter/buffer.py +++ b/pypy/interpreter/buffer.py @@ -1,7 +1,6 @@ """ Buffer protocol support. """ -from pypy.interpreter.error import OperationError from rpython.rlib.objectmodel import import_from_mixin diff --git a/pypy/module/__pypy__/interp_builders.py b/pypy/module/__pypy__/interp_builders.py --- a/pypy/module/__pypy__/interp_builders.py +++ b/pypy/module/__pypy__/interp_builders.py @@ -4,7 +4,6 @@ from pypy.interpreter.typedef import TypeDef from rpython.rlib.rstring import UnicodeBuilder, StringBuilder from rpython.tool.sourcetools import func_with_new_name -from rpython.rlib import jit def create_builder(name, strtype, builder_cls): diff --git a/pypy/module/_cffi_backend/cbuffer.py b/pypy/module/_cffi_backend/cbuffer.py --- a/pypy/module/_cffi_backend/cbuffer.py +++ b/pypy/module/_cffi_backend/cbuffer.py @@ -1,4 +1,3 @@ -from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.buffer import RWBuffer from pypy.interpreter.error import oefmt from pypy.interpreter.gateway import unwrap_spec, interp2app diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py --- a/pypy/module/_cffi_backend/ctypestruct.py +++ b/pypy/module/_cffi_backend/ctypestruct.py @@ -80,7 +80,6 @@ return (cfield.ctype, cfield.offset) def _copy_from_same(self, cdata, w_ob): - space = self.space if isinstance(w_ob, cdataobj.W_CData): if w_ob.ctype is self and self.size >= 0: misc._raw_memcopy(w_ob._cdata, cdata, self.size) diff --git a/pypy/module/_cffi_backend/handle.py b/pypy/module/_cffi_backend/handle.py --- a/pypy/module/_cffi_backend/handle.py +++ b/pypy/module/_cffi_backend/handle.py @@ -1,4 +1,3 @@ -import weakref from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter.gateway import unwrap_spec from pypy.module._cffi_backend import ctypeobj, ctypeptr, cdataobj diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py --- a/pypy/module/_cffi_backend/misc.py +++ b/pypy/module/_cffi_backend/misc.py @@ -4,7 +4,7 @@ from rpython.rlib import jit from rpython.rlib.objectmodel import keepalive_until_here, specialize -from rpython.rlib.rarithmetic import r_uint, r_ulonglong, is_signed_integer_type +from rpython.rlib.rarithmetic import r_uint, r_ulonglong from rpython.rlib.unroll import unrolling_iterable from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from rpython.translator.tool.cbuild import ExternalCompilationInfo diff --git a/pypy/module/_file/__init__.py b/pypy/module/_file/__init__.py --- a/pypy/module/_file/__init__.py +++ b/pypy/module/_file/__init__.py @@ -1,7 +1,6 @@ - # Package initialisation from pypy.interpreter.mixedmodule import MixedModule -import sys + class Module(MixedModule): appleveldefs = { diff --git a/pypy/module/_minimal_curses/fficurses.py b/pypy/module/_minimal_curses/fficurses.py --- a/pypy/module/_minimal_curses/fficurses.py +++ b/pypy/module/_minimal_curses/fficurses.py @@ -1,4 +1,3 @@ - """ The ffi for rpython, need to be imported for side effects """ @@ -8,8 +7,6 @@ from rpython.rtyper.extfunc import register_external from pypy.module._minimal_curses import interp_curses from rpython.translator.tool.cbuild import ExternalCompilationInfo -from sys import platform -import os.path # We cannot trust ncurses5-config, it's broken in various ways in # various versions. For example it might not list -ltinfo even though diff --git a/pypy/module/_pickle_support/maker.py b/pypy/module/_pickle_support/maker.py --- a/pypy/module/_pickle_support/maker.py +++ b/pypy/module/_pickle_support/maker.py @@ -3,7 +3,6 @@ from pypy.interpreter.pycode import PyCode from pypy.interpreter.function import Function, Method from pypy.interpreter.module import Module -from pypy.interpreter.pyframe import PyFrame from pypy.interpreter.pytraceback import PyTraceback from pypy.interpreter.generator import GeneratorIteratorWithDel from rpython.rlib.objectmodel import instantiate diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py --- a/pypy/module/_pypyjson/interp_decoder.py +++ b/pypy/module/_pypyjson/interp_decoder.py @@ -1,13 +1,10 @@ import sys -import math from rpython.rlib.rstring import StringBuilder from rpython.rlib.objectmodel import specialize from rpython.rlib import rfloat, runicode from rpython.rtyper.lltypesystem import lltype, rffi from pypy.interpreter.error import OperationError, oefmt -from pypy.interpreter.gateway import unwrap_spec from pypy.interpreter import unicodehelper -from rpython.rtyper.annlowlevel import llstr, hlunicode OVF_DIGITS = len(str(sys.maxint)) @@ -30,7 +27,7 @@ Internally it's implemented at the level of low-level helpers, to avoid the extra copy we would need if we take the actual slice first. - + No bound checking is done, use carefully. """ from rpython.rtyper.annlowlevel import llstr, hlunicode @@ -226,7 +223,6 @@ def decode_array(self, i): w_list = self.space.newlist([]) start = i - count = 0 i = self.skip_whitespace(start) if self.ll_chars[i] == ']': self.pos = i+1 diff --git a/pypy/module/_rawffi/callback.py b/pypy/module/_rawffi/callback.py --- a/pypy/module/_rawffi/callback.py +++ b/pypy/module/_rawffi/callback.py @@ -2,7 +2,7 @@ from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.typedef import TypeDef, GetSetProperty from rpython.rtyper.lltypesystem import lltype, rffi -from pypy.module._rawffi.interp_rawffi import read_ptr, write_ptr +from pypy.module._rawffi.interp_rawffi import write_ptr from pypy.module._rawffi.structure import W_Structure from pypy.module._rawffi.interp_rawffi import (W_DataInstance, letter2tp, unwrap_value, unpack_argshapes, got_libffi_error) diff --git a/pypy/module/_rawffi/structure.py b/pypy/module/_rawffi/structure.py --- a/pypy/module/_rawffi/structure.py +++ b/pypy/module/_rawffi/structure.py @@ -15,7 +15,7 @@ from pypy.module._rawffi.interp_rawffi import size_alignment from pypy.module._rawffi.interp_rawffi import read_ptr, write_ptr from rpython.rlib import clibffi, rgc -from rpython.rlib.rarithmetic import intmask, signedtype, widen, r_uint, \ +from rpython.rlib.rarithmetic import intmask, signedtype, r_uint, \ r_ulonglong from rpython.rtyper.lltypesystem import lltype, rffi diff --git a/pypy/module/_rawffi/tracker.py b/pypy/module/_rawffi/tracker.py --- a/pypy/module/_rawffi/tracker.py +++ b/pypy/module/_rawffi/tracker.py @@ -1,4 +1,3 @@ - """ The file that keeps track about freed/kept-alive objects allocated by _rawffi. Used for debugging ctypes """ diff --git a/pypy/module/rctime/interp_time.py b/pypy/module/rctime/interp_time.py --- a/pypy/module/rctime/interp_time.py +++ b/pypy/module/rctime/interp_time.py @@ -3,7 +3,7 @@ from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter.gateway import unwrap_spec from rpython.rtyper.lltypesystem import lltype -from rpython.rlib.rarithmetic import ovfcheck_float_to_int, intmask +from rpython.rlib.rarithmetic import intmask from rpython.rlib import rposix from rpython.translator.tool.cbuild import ExternalCompilationInfo import os diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -2,7 +2,6 @@ Version numbers exposed by PyPy through the 'sys' module. """ import os -import re from rpython.translator.platform import platform from pypy.interpreter import gateway diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py --- a/rpython/rlib/rfile.py +++ b/rpython/rlib/rfile.py @@ -4,12 +4,12 @@ """ import os +from rpython.rlib import rposix +from rpython.rlib.rarithmetic import intmask +from rpython.rlib.rstring import StringBuilder from rpython.rtyper.lltypesystem import rffi, lltype from rpython.rtyper.tool import rffi_platform as platform from rpython.translator.tool.cbuild import ExternalCompilationInfo -from rpython.rlib.rarithmetic import r_uint, intmask -from rpython.rlib import rposix -from rpython.rlib.rstring import StringBuilder includes = ['stdio.h', 'sys/types.h'] if os.name == "posix": From noreply at buildbot.pypy.org Wed Mar 19 04:03:31 2014 From: noreply at buildbot.pypy.org (pjenvey) Date: Wed, 19 Mar 2014 04:03:31 +0100 (CET) Subject: [pypy-commit] pypy py3k: fix/cleanup imports Message-ID: <20140319030331.73FF11D2812@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r70075:9b55b0c3aacc Date: 2014-03-18 15:30 -0700 http://bitbucket.org/pypy/pypy/changeset/9b55b0c3aacc/ Log: fix/cleanup imports diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py --- a/pypy/module/cpyext/memoryobject.py +++ b/pypy/module/cpyext/memoryobject.py @@ -1,9 +1,10 @@ -from pypy.interpreter.error import OperationError -from pypy.module.cpyext.api import cpython_api, Py_buffer +from rpython.rtyper.lltypesystem import lltype + +from pypy.interpreter.error import oefmt +from pypy.module.cpyext.api import Py_buffer, cpython_api from pypy.module.cpyext.pyobject import PyObject, from_ref from pypy.module.cpyext.buffer import CBuffer -from rpython.rtyper.lltypesystem import rffi, lltype -from pypy.module.__builtin__.interp_memoryview import W_MemoryView +from pypy.objspace.std.memoryview import W_MemoryView @cpython_api([PyObject], PyObject) def PyMemoryView_FromObject(space, w_obj): @@ -16,8 +17,9 @@ means you shouldn't try to call PyBuffer_Release() yourself: it will be done on deallocation of the memoryview object.""" if not view.c_buf: - msg = "cannot make memory view from a buffer with a NULL data pointer" - raise OperationError(space.w_ValueError, space.wrap(msg)) + raise oefmt(space.w_ValueError, + "cannot make memory view from a buffer with a NULL data " + "pointer") w_obj = from_ref(space, view.c_obj) buf = CBuffer(space, view.c_buf, view.c_len, w_obj) return space.wrap(W_MemoryView(buf)) From noreply at buildbot.pypy.org Wed Mar 19 04:03:32 2014 From: noreply at buildbot.pypy.org (pjenvey) Date: Wed, 19 Mar 2014 04:03:32 +0100 (CET) Subject: [pypy-commit] pypy py3k: fix translation Message-ID: <20140319030332.DECD81D2812@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r70076:4656f3c15efb Date: 2014-03-18 20:02 -0700 http://bitbucket.org/pypy/pypy/changeset/4656f3c15efb/ Log: fix translation diff --git a/pypy/module/_io/interp_bytesio.py b/pypy/module/_io/interp_bytesio.py --- a/pypy/module/_io/interp_bytesio.py +++ b/pypy/module/_io/interp_bytesio.py @@ -7,6 +7,7 @@ from rpython.rlib.rarithmetic import r_longlong from pypy.module._io.interp_bufferedio import W_BufferedIOBase from pypy.module._io.interp_iobase import convert_size +from pypy.objspace.std.memoryview import W_MemoryView import sys @@ -121,7 +122,7 @@ return space.wrap(size) def getbuffer_w(self, space): - return space.wrap(BytesIOBuffer(self)) + return space.wrap(W_MemoryView(BytesIOBuffer(self))) def getvalue_w(self, space): self._check_closed(space) diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py --- a/pypy/module/cpyext/buffer.py +++ b/pypy/module/cpyext/buffer.py @@ -37,5 +37,4 @@ class CBuffer(CBufferMixin, buffer.Buffer): def __del__(self): - self.enqueue_for_destruction(self.space, CBufferMixin.destructor, - 'internal __del__ of ') + CBufferMixin.destructor(self) From noreply at buildbot.pypy.org Wed Mar 19 05:25:28 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Wed, 19 Mar 2014 05:25:28 +0100 (CET) Subject: [pypy-commit] pypy default: fix test_marshal Message-ID: <20140319042528.B26181D2808@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r70077:e9a9d862dd67 Date: 2014-03-19 00:24 -0400 http://bitbucket.org/pypy/pypy/changeset/e9a9d862dd67/ Log: fix test_marshal diff --git a/pypy/module/marshal/interp_marshal.py b/pypy/module/marshal/interp_marshal.py --- a/pypy/module/marshal/interp_marshal.py +++ b/pypy/module/marshal/interp_marshal.py @@ -7,7 +7,7 @@ Py_MARSHAL_VERSION = 2 - at unwrap_spec(w_version = WrappedDefault(Py_MARSHAL_VERSION)) + at unwrap_spec(w_version=WrappedDefault(Py_MARSHAL_VERSION)) def dump(space, w_data, w_f, w_version): """Write the 'data' object into the open file 'f'.""" # special case real files for performance @@ -24,7 +24,7 @@ finally: writer.finished() - at unwrap_spec(w_version = WrappedDefault(Py_MARSHAL_VERSION)) + at unwrap_spec(w_version=WrappedDefault(Py_MARSHAL_VERSION)) def dumps(space, w_data, w_version): """Return the string that would have been written to a file by dump(data, file).""" @@ -217,6 +217,16 @@ self.space.marshal_w(w_obj, self) def dump_w_obj(self, w_obj): + space = self.space + if space.type(w_obj).is_heaptype(): + try: + buf = space.buffer_w(w_obj) + except OperationError as e: + if not e.match(space, space.w_TypeError): + raise + self.raise_exc("unmarshallable object") + else: + w_obj = space.newbuffer(buf) try: self.put_w_obj(w_obj) except rstackovf.StackOverflow: diff --git a/pypy/module/marshal/test/test_marshal.py b/pypy/module/marshal/test/test_marshal.py --- a/pypy/module/marshal/test/test_marshal.py +++ b/pypy/module/marshal/test/test_marshal.py @@ -173,10 +173,13 @@ import marshal types = (float, complex, int, long, tuple, list, dict, set, frozenset) for cls in types: + print cls class subtype(cls): pass exc = raises(ValueError, marshal.dumps, subtype) assert str(exc.value) == 'unmarshallable object' + exc = raises(ValueError, marshal.dumps, subtype()) + assert str(exc.value) == 'unmarshallable object' def test_valid_subtypes(self): import marshal From noreply at buildbot.pypy.org Wed Mar 19 05:58:00 2014 From: noreply at buildbot.pypy.org (pjenvey) Date: Wed, 19 Mar 2014 05:58:00 +0100 (CET) Subject: [pypy-commit] pypy py3k: kill __builtin__.buffer, adjust test Message-ID: <20140319045800.EA9081C0166@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r70078:0f937ae8b835 Date: 2014-03-18 20:08 -0700 http://bitbucket.org/pypy/pypy/changeset/0f937ae8b835/ Log: kill __builtin__.buffer, adjust test diff --git a/pypy/module/__builtin__/test/test_builtin.py b/pypy/module/__builtin__/test/test_builtin.py --- a/pypy/module/__builtin__/test/test_builtin.py +++ b/pypy/module/__builtin__/test/test_builtin.py @@ -26,15 +26,11 @@ cls.w_safe_runtimerror = cls.space.wrap(sys.version_info < (2, 6)) def test_builtin_names(self): - import __builtin__ - assert __builtin__.None is None - assert __builtin__.False is False - assert __builtin__.True is True - - assert __builtin__.buffer is buffer - assert __builtin__.bytes is str + import builtins as __builtin__ + assert __builtin__.bytes is bytes assert __builtin__.dict is dict assert __builtin__.memoryview is memoryview + assert not hasattr(__builtin__, 'buffer') def test_bytes_alias(self): assert bytes is not str diff --git a/pypy/objspace/std/model.py b/pypy/objspace/std/model.py --- a/pypy/objspace/std/model.py +++ b/pypy/objspace/std/model.py @@ -61,7 +61,7 @@ from pypy.objspace.std import unicodeobject from pypy.objspace.std import dictproxyobject from pypy.objspace.std import proxyobject - from pypy.objspace.std.memoryview import W_Buffer, W_MemoryView + from pypy.objspace.std.memoryview import W_MemoryView import pypy.objspace.std.default # register a few catch-all multimethods import pypy.objspace.std.marshal_impl # install marshal multimethods @@ -80,7 +80,6 @@ self.pythontypes.append(intobject.W_IntObject.typedef) self.pythontypes.append(boolobject.W_BoolObject.typedef) self.pythontypes.append(longobject.W_LongObject.typedef) - self.pythontypes.append(W_Buffer.typedef) self.pythontypes.append(W_MemoryView.typedef) # the set of implementation types From noreply at buildbot.pypy.org Wed Mar 19 05:58:02 2014 From: noreply at buildbot.pypy.org (pjenvey) Date: Wed, 19 Mar 2014 05:58:02 +0100 (CET) Subject: [pypy-commit] pypy py3k: adapt to buffer refactor: not everything exposes a __buffer__ now Message-ID: <20140319045802.534A91C0166@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r70079:aaba1a61bc19 Date: 2014-03-18 21:18 -0700 http://bitbucket.org/pypy/pypy/changeset/aaba1a61bc19/ Log: adapt to buffer refactor: not everything exposes a __buffer__ now diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py --- a/pypy/module/array/interp_array.py +++ b/pypy/module/array/interp_array.py @@ -40,16 +40,19 @@ if len(__args__.arguments_w) > 0: w_initializer = __args__.arguments_w[0] - if space.lookup(w_initializer, '__buffer__') is not None: - if isinstance(w_initializer, W_ArrayBase): - a.extend(w_initializer, True) - else: - a.descr_frombytes(space, - space.bufferstr_w(w_initializer)) + if isinstance(w_initializer, W_ArrayBase): + a.extend(w_initializer, True) elif space.type(w_initializer) is space.w_list: a.descr_fromlist(space, w_initializer) else: - a.extend(w_initializer, True) + try: + buf = space.bufferstr_w(w_initializer) + except OperationError as e: + if not e.match(space, space.w_TypeError): + raise + a.extend(w_initializer, True) + else: + a.descr_frombytes(space, buf) break else: msg = 'bad typecode (must be b, B, u, h, H, i, I, l, L, f or d)' From noreply at buildbot.pypy.org Wed Mar 19 05:58:03 2014 From: noreply at buildbot.pypy.org (pjenvey) Date: Wed, 19 Mar 2014 05:58:03 +0100 (CET) Subject: [pypy-commit] pypy py3k: kill W_Buffer Message-ID: <20140319045803.B52B71C0166@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r70080:415ae19bb4d1 Date: 2014-03-18 21:56 -0700 http://bitbucket.org/pypy/pypy/changeset/415ae19bb4d1/ Log: kill W_Buffer diff --git a/pypy/objspace/std/memoryview.py b/pypy/objspace/std/memoryview.py --- a/pypy/objspace/std/memoryview.py +++ b/pypy/objspace/std/memoryview.py @@ -31,141 +31,6 @@ " slicing with a step")) -class W_Buffer(W_Root): - """Implement the built-in 'buffer' type as a wrapper around - an interp-level buffer. - """ - - def __init__(self, buf): - assert isinstance(buf, buffer.Buffer) - self.buf = buf - - def buffer_w(self, space): - return self.buf - - @staticmethod - @unwrap_spec(offset=int, size=int) - def descr_new_buffer(space, w_subtype, w_object, offset=0, size=-1): - if space.isinstance_w(w_object, space.w_unicode): - # unicode objects support the old buffer interface - # but not the new buffer interface (change in python 2.7) - from rpython.rlib.rstruct.unichar import pack_unichar, UNICODE_SIZE - unistr = space.unicode_w(w_object) - builder = StringBuilder(len(unistr) * UNICODE_SIZE) - for unich in unistr: - pack_unichar(unich, builder) - from pypy.interpreter.buffer import StringBuffer - buf = StringBuffer(builder.build()) - else: - buf = space.buffer_w(w_object) - - if offset == 0 and size == -1: - return W_Buffer(buf) - # handle buffer slices - if offset < 0: - raise OperationError(space.w_ValueError, - space.wrap("offset must be zero or positive")) - if size < -1: - raise OperationError(space.w_ValueError, - space.wrap("size must be zero or positive")) - if isinstance(buf, buffer.RWBuffer): - buf = buffer.RWSubBuffer(buf, offset, size) - else: - buf = buffer.SubBuffer(buf, offset, size) - return W_Buffer(buf) - - def descr_len(self, space): - return space.wrap(self.buf.getlength()) - - def descr_getitem(self, space, w_index): - start, stop, step, size = space.decode_index4(w_index, self.buf.getlength()) - if step == 0: # index only - return space.wrapbytes(self.buf.getitem(start)) - res = self.buf.getslice(start, stop, step, size) - return space.wrapbytes(res) - - @unwrap_spec(newstring='bufferstr') - def descr_setitem(self, space, w_index, newstring): - if not isinstance(self.buf, buffer.RWBuffer): - raise OperationError(space.w_TypeError, - space.wrap("buffer is read-only")) - _buffer_setitem(space, self.buf, w_index, newstring) - - def descr_str(self, space): - return space.wrap(self.buf.as_str()) - - @unwrap_spec(other='bufferstr') - def descr_add(self, space, other): - return space.wrapbytes(self.buf.as_str() + other) - - def _make_descr__cmp(name): - def descr__cmp(self, space, w_other): - if not isinstance(w_other, W_Buffer): - return space.w_NotImplemented - # xxx not the most efficient implementation - str1 = self.buf.as_str() - str2 = w_other.buf.as_str() - return space.wrap(getattr(operator, name)(str1, str2)) - descr__cmp.func_name = name - return descr__cmp - - descr_eq = _make_descr__cmp('eq') - descr_ne = _make_descr__cmp('ne') - descr_lt = _make_descr__cmp('lt') - descr_le = _make_descr__cmp('le') - descr_gt = _make_descr__cmp('gt') - descr_ge = _make_descr__cmp('ge') - - def descr_hash(self, space): - return space.wrap(compute_hash(self.buf.as_str())) - - def descr_mul(self, space, w_times): - # xxx not the most efficient implementation - w_string = space.wrapbytes(self.buf.as_str()) - # use the __mul__ method instead of space.mul() so that we - # return NotImplemented instead of raising a TypeError - return space.call_method(w_string, '__mul__', w_times) - - def descr_repr(self, space): - if isinstance(self.buf, buffer.RWBuffer): - info = 'read-write buffer' - else: - info = 'read-only buffer' - addrstring = self.getaddrstring(space) - - return space.wrap("<%s for 0x%s, size %d>" % - (info, addrstring, self.buf.getlength())) - -W_Buffer.typedef = TypeDef( - "buffer", - __doc__ = """\ -buffer(object [, offset[, size]]) - -Create a new buffer object which references the given object. -The buffer will reference a slice of the target object from the -start of the object (or at the specified offset). The slice will -extend to the end of the target object (or with the specified size). -""", - __new__ = interp2app(W_Buffer.descr_new_buffer), - __len__ = interp2app(W_Buffer.descr_len), - __getitem__ = interp2app(W_Buffer.descr_getitem), - __setitem__ = interp2app(W_Buffer.descr_setitem), - __str__ = interp2app(W_Buffer.descr_str), - __add__ = interp2app(W_Buffer.descr_add), - __eq__ = interp2app(W_Buffer.descr_eq), - __ne__ = interp2app(W_Buffer.descr_ne), - __lt__ = interp2app(W_Buffer.descr_lt), - __le__ = interp2app(W_Buffer.descr_le), - __gt__ = interp2app(W_Buffer.descr_gt), - __ge__ = interp2app(W_Buffer.descr_ge), - __hash__ = interp2app(W_Buffer.descr_hash), - __mul__ = interp2app(W_Buffer.descr_mul), - __rmul__ = interp2app(W_Buffer.descr_mul), - __repr__ = interp2app(W_Buffer.descr_repr), -) -W_Buffer.typedef.acceptable_as_base_class = False - - class W_MemoryView(W_Root): """Implement the built-in 'memoryview' type as a wrapper around an interp-level buffer. diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py --- a/pypy/objspace/std/objspace.py +++ b/pypy/objspace/std/objspace.py @@ -25,7 +25,7 @@ from pypy.objspace.std.iterobject import W_AbstractSeqIterObject from pypy.objspace.std.listobject import W_ListObject from pypy.objspace.std.longobject import W_LongObject, newlong -from pypy.objspace.std.memoryview import W_Buffer +from pypy.objspace.std.memoryview import W_MemoryView from pypy.objspace.std.noneobject import W_NoneObject from pypy.objspace.std.objectobject import W_ObjectObject from pypy.objspace.std.iterobject import W_SeqIterObject @@ -341,7 +341,7 @@ return W_SeqIterObject(w_obj) def newbuffer(self, w_obj): - return W_Buffer(w_obj) + return W_MemoryView(w_obj) def type(self, w_obj): jit.promote(w_obj.__class__) From noreply at buildbot.pypy.org Wed Mar 19 05:58:05 2014 From: noreply at buildbot.pypy.org (pjenvey) Date: Wed, 19 Mar 2014 05:58:05 +0100 (CET) Subject: [pypy-commit] pypy py3k: try to get away away with this simply subclassing MemoryView Message-ID: <20140319045805.173641C0166@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r70081:8b3f43ab9322 Date: 2014-03-18 21:57 -0700 http://bitbucket.org/pypy/pypy/changeset/8b3f43ab9322/ Log: try to get away away with this simply subclassing MemoryView diff --git a/pypy/module/_cffi_backend/cbuffer.py b/pypy/module/_cffi_backend/cbuffer.py --- a/pypy/module/_cffi_backend/cbuffer.py +++ b/pypy/module/_cffi_backend/cbuffer.py @@ -4,7 +4,7 @@ from pypy.interpreter.gateway import unwrap_spec, interp2app from pypy.interpreter.typedef import TypeDef, make_weakref_descr from pypy.module._cffi_backend import cdataobj, ctypeptr, ctypearray -from pypy.objspace.std.memoryview import W_Buffer +from pypy.objspace.std.memoryview import W_MemoryView from rpython.rtyper.annlowlevel import llstr from rpython.rtyper.lltypesystem import rffi @@ -42,9 +42,9 @@ # Override the typedef to narrow down the interface that's exposed to app-level -class MiniBuffer(W_Buffer): +class MiniBuffer(W_MemoryView): def __init__(self, buffer, keepalive=None): - W_Buffer.__init__(self, buffer) + W_MemoryView.__init__(self, buffer) self.keepalive = keepalive MiniBuffer.typedef = TypeDef( From noreply at buildbot.pypy.org Wed Mar 19 06:11:33 2014 From: noreply at buildbot.pypy.org (pjenvey) Date: Wed, 19 Mar 2014 06:11:33 +0100 (CET) Subject: [pypy-commit] pypy py3k: merge default Message-ID: <20140319051133.BA69D1C0166@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r70082:1cf13b13b4a1 Date: 2014-03-18 22:08 -0700 http://bitbucket.org/pypy/pypy/changeset/1cf13b13b4a1/ Log: merge default diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -194,6 +194,11 @@ return None def buffer_w(self, space): + w_impl = space.lookup(self, '__buffer__') + if w_impl is not None: + w_result = space.get_and_call_function(w_impl, self) + if space.isinstance_w(w_result, space.w_buffer): + return w_result.buffer_w(space) self._typed_unwrap_error(space, "buffer") def bytes_w(self, space): diff --git a/pypy/interpreter/buffer.py b/pypy/interpreter/buffer.py --- a/pypy/interpreter/buffer.py +++ b/pypy/interpreter/buffer.py @@ -1,7 +1,6 @@ """ Buffer protocol support. """ -from pypy.interpreter.error import OperationError from rpython.rlib.objectmodel import import_from_mixin @@ -71,8 +70,8 @@ assert 0 <= start <= stop return self.value[start:stop] return "".join([self.value[start + i*step] for i in xrange(size)]) +# ____________________________________________________________ -# ____________________________________________________________ class SubBufferMixin(object): _attrs_ = ['buffer', 'offset', 'size'] @@ -98,7 +97,8 @@ if start == stop: return '' # otherwise, adding self.offset might make them # out of bounds - return self.buffer.getslice(self.offset + start, self.offset + stop, step, size) + return self.buffer.getslice(self.offset + start, self.offset + stop, + step, size) class SubBuffer(Buffer): diff --git a/pypy/module/__pypy__/interp_builders.py b/pypy/module/__pypy__/interp_builders.py --- a/pypy/module/__pypy__/interp_builders.py +++ b/pypy/module/__pypy__/interp_builders.py @@ -4,7 +4,6 @@ from pypy.interpreter.typedef import TypeDef from rpython.rlib.rstring import UnicodeBuilder, StringBuilder from rpython.tool.sourcetools import func_with_new_name -from rpython.rlib import jit def create_builder(name, strtype, builder_cls): diff --git a/pypy/module/_cffi_backend/cbuffer.py b/pypy/module/_cffi_backend/cbuffer.py --- a/pypy/module/_cffi_backend/cbuffer.py +++ b/pypy/module/_cffi_backend/cbuffer.py @@ -1,4 +1,3 @@ -from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.buffer import RWBuffer from pypy.interpreter.error import oefmt from pypy.interpreter.gateway import unwrap_spec, interp2app diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py --- a/pypy/module/_cffi_backend/ctypestruct.py +++ b/pypy/module/_cffi_backend/ctypestruct.py @@ -80,7 +80,6 @@ return (cfield.ctype, cfield.offset) def _copy_from_same(self, cdata, w_ob): - space = self.space if isinstance(w_ob, cdataobj.W_CData): if w_ob.ctype is self and self.size >= 0: misc._raw_memcopy(w_ob._cdata, cdata, self.size) diff --git a/pypy/module/_cffi_backend/handle.py b/pypy/module/_cffi_backend/handle.py --- a/pypy/module/_cffi_backend/handle.py +++ b/pypy/module/_cffi_backend/handle.py @@ -1,4 +1,3 @@ -import weakref from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter.gateway import unwrap_spec from pypy.module._cffi_backend import ctypeobj, ctypeptr, cdataobj diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py --- a/pypy/module/_cffi_backend/misc.py +++ b/pypy/module/_cffi_backend/misc.py @@ -4,7 +4,7 @@ from rpython.rlib import jit from rpython.rlib.objectmodel import keepalive_until_here, specialize -from rpython.rlib.rarithmetic import r_uint, r_ulonglong, is_signed_integer_type +from rpython.rlib.rarithmetic import r_uint, r_ulonglong from rpython.rlib.unroll import unrolling_iterable from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from rpython.translator.tool.cbuild import ExternalCompilationInfo diff --git a/pypy/module/_minimal_curses/fficurses.py b/pypy/module/_minimal_curses/fficurses.py --- a/pypy/module/_minimal_curses/fficurses.py +++ b/pypy/module/_minimal_curses/fficurses.py @@ -1,4 +1,3 @@ - """ The ffi for rpython, need to be imported for side effects """ @@ -8,8 +7,6 @@ from rpython.rtyper.extfunc import register_external from pypy.module._minimal_curses import interp_curses from rpython.translator.tool.cbuild import ExternalCompilationInfo -from sys import platform -import os.path # We cannot trust ncurses5-config, it's broken in various ways in # various versions. For example it might not list -ltinfo even though diff --git a/pypy/module/_pickle_support/maker.py b/pypy/module/_pickle_support/maker.py --- a/pypy/module/_pickle_support/maker.py +++ b/pypy/module/_pickle_support/maker.py @@ -3,7 +3,6 @@ from pypy.interpreter.pycode import PyCode from pypy.interpreter.function import Function, Method from pypy.interpreter.module import Module -from pypy.interpreter.pyframe import PyFrame from pypy.interpreter.pytraceback import PyTraceback from pypy.interpreter.generator import GeneratorIteratorWithDel from rpython.rlib.objectmodel import instantiate diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py --- a/pypy/module/_pypyjson/interp_decoder.py +++ b/pypy/module/_pypyjson/interp_decoder.py @@ -1,13 +1,10 @@ import sys -import math from rpython.rlib.rstring import StringBuilder from rpython.rlib.objectmodel import specialize from rpython.rlib import rfloat, runicode from rpython.rtyper.lltypesystem import lltype, rffi from pypy.interpreter.error import OperationError, oefmt -from pypy.interpreter.gateway import unwrap_spec from pypy.interpreter import unicodehelper -from rpython.rtyper.annlowlevel import llstr, hlunicode OVF_DIGITS = len(str(sys.maxint)) @@ -30,7 +27,7 @@ Internally it's implemented at the level of low-level helpers, to avoid the extra copy we would need if we take the actual slice first. - + No bound checking is done, use carefully. """ from rpython.rtyper.annlowlevel import llstr, hlunicode @@ -226,7 +223,6 @@ def decode_array(self, i): w_list = self.space.newlist([]) start = i - count = 0 i = self.skip_whitespace(start) if self.ll_chars[i] == ']': self.pos = i+1 diff --git a/pypy/module/_rawffi/callback.py b/pypy/module/_rawffi/callback.py --- a/pypy/module/_rawffi/callback.py +++ b/pypy/module/_rawffi/callback.py @@ -2,7 +2,7 @@ from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.typedef import TypeDef, GetSetProperty from rpython.rtyper.lltypesystem import lltype, rffi -from pypy.module._rawffi.interp_rawffi import read_ptr, write_ptr +from pypy.module._rawffi.interp_rawffi import write_ptr from pypy.module._rawffi.structure import W_Structure from pypy.module._rawffi.interp_rawffi import (W_DataInstance, letter2tp, unwrap_value, unpack_argshapes, got_libffi_error) diff --git a/pypy/module/_rawffi/structure.py b/pypy/module/_rawffi/structure.py --- a/pypy/module/_rawffi/structure.py +++ b/pypy/module/_rawffi/structure.py @@ -15,7 +15,7 @@ from pypy.module._rawffi.interp_rawffi import size_alignment from pypy.module._rawffi.interp_rawffi import read_ptr, write_ptr from rpython.rlib import clibffi, rgc -from rpython.rlib.rarithmetic import intmask, signedtype, widen, r_uint, \ +from rpython.rlib.rarithmetic import intmask, signedtype, r_uint, \ r_ulonglong from rpython.rtyper.lltypesystem import lltype, rffi diff --git a/pypy/module/_rawffi/tracker.py b/pypy/module/_rawffi/tracker.py --- a/pypy/module/_rawffi/tracker.py +++ b/pypy/module/_rawffi/tracker.py @@ -1,4 +1,3 @@ - """ The file that keeps track about freed/kept-alive objects allocated by _rawffi. Used for debugging ctypes """ diff --git a/pypy/module/array/test/test_array.py b/pypy/module/array/test/test_array.py --- a/pypy/module/array/test/test_array.py +++ b/pypy/module/array/test/test_array.py @@ -1,25 +1,8 @@ import sys -import py -import py.test - - -## class AppTestSimpleArray: -## spaceconfig = dict(usemodules=('array',)) -## def setup_class(cls): -## cls.w_simple_array = cls.space.appexec([], """(): -## import array -## return array.simple_array -## """) - -## def test_simple(self): -## a = self.simple_array(10) -## a[5] = 7.42 -## assert a[5] == 7.42 +import pytest class BaseArrayTests: - - def test_ctor(self): assert len(self.array('i')) == 0 @@ -545,7 +528,6 @@ assert not a > 2*a assert not a >= 2*a - def test_reduce(self): import pickle a = self.array('i', [1, 2, 3]) @@ -779,7 +761,6 @@ assert img[3, 25] == 3 * 9 - def test_override_from(self): class mya(self.array): def fromlist(self, lst): @@ -862,41 +843,41 @@ def test_assign_object_with_special_methods(self): from array import array - + class Num(object): def __float__(self): return 5.25 - + def __int__(self): return 7 - + class NotNum(object): pass - + class Silly(object): def __float__(self): return None - + def __int__(self): - return None + return None class OldNum: def __float__(self): return 6.25 - + def __int__(self): return 8 - + class OldNotNum: pass - + class OldSilly: def __float__(self): return None - + def __int__(self): return None - + for tc in 'bBhHiIlL': a = array(tc, [0]) raises(TypeError, a.__setitem__, 0, 1.0) @@ -914,7 +895,7 @@ a = array(tc, [0]) a[0] = 1.0 a[0] = 1 - a[0] = Num() + a[0] = Num() assert a[0] == 5.25 raises(TypeError, a.__setitem__, NotNum()) a[0] = OldNum() @@ -922,11 +903,15 @@ raises(TypeError, a.__setitem__, OldNotNum()) raises(TypeError, a.__setitem__, Silly()) raises(TypeError, a.__setitem__, OldSilly()) - + a = array('u', 'hi') a[0] = 'b' assert a[0] == 'b' - + + a = array('u', u'hi') + a[0] = u'b' + assert a[0] == u'b' + def test_bytearray(self): a = self.array('u', 'hi') b = self.array('u') @@ -940,15 +925,13 @@ assert repr(a) == "array('u', {!r})".format(s) assert eval(repr(a), {'array': self.array}) == a - class DontTestCPythonsOwnArray(BaseArrayTests): - def setup_class(cls): import array cls.array = array.array import struct cls.struct = struct - cls.tempfile = str(py.test.ensuretemp('array').join('tmpfile')) + cls.tempfile = str(pytest.ensuretemp('array').join('tmpfile')) cls.maxint = sys.maxint @@ -961,7 +944,7 @@ return array.array """) cls.w_tempfile = cls.space.wrap( - str(py.test.ensuretemp('array').join('tmpfile'))) + str(pytest.ensuretemp('array').join('tmpfile'))) cls.w_maxint = cls.space.wrap(sys.maxint) def test_buffer_info(self): @@ -1028,11 +1011,11 @@ def test_getitem_only_ints(self): class MyInt(object): - def __init__(self, x): - self.x = x + def __init__(self, x): + self.x = x - def __int__(self): - return self.x + def __int__(self): + return self.x a = self.array('i', [1, 2, 3, 4, 5, 6]) raises(TypeError, "a[MyInt(0)]") diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py --- a/pypy/module/cpyext/test/test_arraymodule.py +++ b/pypy/module/cpyext/test/test_arraymodule.py @@ -1,7 +1,5 @@ from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase -import py -import sys class AppTestArrayModule(AppTestCpythonExtensionBase): enable_leak_checking = False @@ -21,7 +19,7 @@ module = self.import_module(name='array') arr = module.array('i', [1,2,3]) sum = 0 - for i in arr: + for i in arr: sum += i assert sum == 6 @@ -60,4 +58,3 @@ b'\x02\0\0\0' b'\x03\0\0\0' b'\x04\0\0\0') - diff --git a/pypy/module/marshal/interp_marshal.py b/pypy/module/marshal/interp_marshal.py --- a/pypy/module/marshal/interp_marshal.py +++ b/pypy/module/marshal/interp_marshal.py @@ -6,7 +6,7 @@ Py_MARSHAL_VERSION = 2 - at unwrap_spec(w_version = WrappedDefault(Py_MARSHAL_VERSION)) + at unwrap_spec(w_version=WrappedDefault(Py_MARSHAL_VERSION)) def dump(space, w_data, w_f, w_version): """Write the 'data' object into the open file 'f'.""" # XXX: before py3k, we special-cased W_File to use a more performant @@ -22,7 +22,7 @@ finally: writer.finished() - at unwrap_spec(w_version = WrappedDefault(Py_MARSHAL_VERSION)) + at unwrap_spec(w_version=WrappedDefault(Py_MARSHAL_VERSION)) def dumps(space, w_data, w_version): """Return the string that would have been written to a file by dump(data, file).""" @@ -221,10 +221,15 @@ def dump_w_obj(self, w_obj): space = self.space - if (space.type(w_obj).is_heaptype() and - space.lookup(w_obj, "__buffer__") is None): - w_err = space.wrap("only builtins can be marshaled") - raise OperationError(space.w_ValueError, w_err) + if space.type(w_obj).is_heaptype(): + try: + buf = space.buffer_w(w_obj) + except OperationError as e: + if not e.match(space, space.w_TypeError): + raise + self.raise_exc("unmarshallable object") + else: + w_obj = space.newbuffer(buf) try: self.put_w_obj(w_obj) except rstackovf.StackOverflow: diff --git a/pypy/module/marshal/test/test_marshal.py b/pypy/module/marshal/test/test_marshal.py --- a/pypy/module/marshal/test/test_marshal.py +++ b/pypy/module/marshal/test/test_marshal.py @@ -2,6 +2,8 @@ class AppTestMarshal: + spaceconfig = {'usemodules': ['array']} + def setup_class(cls): tmpfile = udir.join('AppTestMarshal.tmp') cls.w_tmpfile = cls.space.wrap(str(tmpfile)) @@ -171,9 +173,20 @@ import marshal types = (float, complex, int, tuple, list, dict, set, frozenset) for cls in types: + print cls class subtype(cls): pass - raises(ValueError, marshal.dumps, subtype) + exc = raises(ValueError, marshal.dumps, subtype) + assert str(exc.value) == 'unmarshallable object' + exc = raises(ValueError, marshal.dumps, subtype()) + assert str(exc.value) == 'unmarshallable object' + + def test_valid_subtypes(self): + import marshal + from array import array + class subtype(array): + pass + assert marshal.dumps(subtype('c', 'test')) == marshal.dumps(array('c', 'test')) def test_bad_typecode(self): import marshal @@ -188,7 +201,8 @@ class AppTestSmallLong(AppTestMarshal): - spaceconfig = {"objspace.std.withsmalllong": True} + spaceconfig = AppTestMarshal.spaceconfig.copy() + spaceconfig["objspace.std.withsmalllong"] = True def setup_class(cls): from pypy.interpreter import gateway diff --git a/pypy/module/rctime/interp_time.py b/pypy/module/rctime/interp_time.py --- a/pypy/module/rctime/interp_time.py +++ b/pypy/module/rctime/interp_time.py @@ -3,7 +3,7 @@ from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter.gateway import unwrap_spec from rpython.rtyper.lltypesystem import lltype -from rpython.rlib.rarithmetic import ovfcheck_float_to_int, intmask +from rpython.rlib.rarithmetic import intmask from rpython.rlib import rposix from rpython.translator.tool.cbuild import ExternalCompilationInfo import os diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -2,7 +2,6 @@ Version numbers exposed by PyPy through the 'sys' module. """ import os -import re from rpython.translator.platform import platform from pypy.interpreter import gateway diff --git a/pypy/objspace/std/callmethod.py b/pypy/objspace/std/callmethod.py --- a/pypy/objspace/std/callmethod.py +++ b/pypy/objspace/std/callmethod.py @@ -11,7 +11,6 @@ """ from pypy.interpreter import function -from pypy.objspace.descroperation import object_getattribute from rpython.rlib import jit from pypy.objspace.std.mapdict import LOOKUP_METHOD_mapdict, \ LOOKUP_METHOD_mapdict_fill_cache_method @@ -36,7 +35,6 @@ if space.config.objspace.std.withmapdict and not jit.we_are_jitted(): # mapdict has an extra-fast version of this function - from pypy.objspace.std.mapdict import LOOKUP_METHOD_mapdict if LOOKUP_METHOD_mapdict(f, nameindex, w_obj): return @@ -79,7 +77,7 @@ n_kwargs = (oparg >> 8) & 0xff w_self = f.peekvalue(n_args + (2 * n_kwargs)) n = n_args + (w_self is not None) - + if not n_kwargs: w_callable = f.peekvalue(n_args + (2 * n_kwargs) + 1) try: @@ -98,7 +96,7 @@ key = f.space.str_w(w_key) keywords[n_kwargs] = key keywords_w[n_kwargs] = w_value - + arguments = f.popvalues(n) # includes w_self if it is not None args = f.argument_factory(arguments, keywords, keywords_w, None, None) if w_self is None: diff --git a/pypy/objspace/std/celldict.py b/pypy/objspace/std/celldict.py --- a/pypy/objspace/std/celldict.py +++ b/pypy/objspace/std/celldict.py @@ -140,7 +140,7 @@ for key, cell in iterator()] def clear(self, w_dict): - iterator = self.unerase(w_dict.dstorage).clear() + self.unerase(w_dict.dstorage).clear() self.mutated() def popitem(self, w_dict): diff --git a/pypy/objspace/std/complexobject.py b/pypy/objspace/std/complexobject.py --- a/pypy/objspace/std/complexobject.py +++ b/pypy/objspace/std/complexobject.py @@ -1,4 +1,5 @@ -from pypy.interpreter import gateway +import math + from pypy.interpreter.error import OperationError from pypy.objspace.std import newformat from pypy.objspace.std.intobject import W_IntObject @@ -12,8 +13,6 @@ from rpython.rlib import jit, rcomplex from rpython.rlib.rarithmetic import intmask, r_ulonglong -import math - HASH_IMAG = 1000003 diff --git a/pypy/objspace/std/complextype.py b/pypy/objspace/std/complextype.py --- a/pypy/objspace/std/complextype.py +++ b/pypy/objspace/std/complextype.py @@ -1,7 +1,6 @@ from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault from pypy.objspace.std.register_all import register_all -from pypy.objspace.std.noneobject import W_NoneObject from pypy.objspace.std.stdtypedef import GetSetProperty, StdTypeDef from pypy.objspace.std.stdtypedef import StdObjSpaceMultiMethod from pypy.objspace.std.unicodeobject import unicode_to_decimal_w diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py --- a/pypy/objspace/std/dictmultiobject.py +++ b/pypy/objspace/std/dictmultiobject.py @@ -1092,7 +1092,7 @@ w_clone.setup_iterator() # spool until we have the same pos while w_clone.pos < self.pos: - w_obj = w_clone.next_entry() + w_clone.next_entry() w_clone.pos += 1 stuff = [w_clone.next_entry() for i in range(w_clone.pos, w_clone.len)] w_res = space.newlist(stuff) diff --git a/pypy/objspace/std/dictproxyobject.py b/pypy/objspace/std/dictproxyobject.py --- a/pypy/objspace/std/dictproxyobject.py +++ b/pypy/objspace/std/dictproxyobject.py @@ -1,7 +1,6 @@ -from pypy.objspace.std.model import registerimplementation, W_Object -from pypy.objspace.std.register_all import register_all -from pypy.objspace.std.dictmultiobject import W_DictMultiObject, create_iterator_classes -from pypy.objspace.std.dictmultiobject import DictStrategy +#from pypy.objspace.std.model import registerimplementation, W_Object +#from pypy.objspace.std.register_all import register_all +from pypy.objspace.std.dictmultiobject import DictStrategy, create_iterator_classes from pypy.objspace.std.typeobject import unwrap_cell from pypy.interpreter.error import OperationError, oefmt @@ -9,7 +8,6 @@ class DictProxyStrategy(DictStrategy): - erase, unerase = rerased.new_erasing_pair("dictproxy") erase = staticmethod(erase) unerase = staticmethod(unerase) @@ -52,7 +50,6 @@ w_type.dict_w[key] = w_value def setdefault(self, w_dict, w_key, w_default): - space = self.space w_result = self.getitem(w_dict, w_key) if w_result is not None: return w_result diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py --- a/pypy/objspace/std/floatobject.py +++ b/pypy/objspace/std/floatobject.py @@ -1,14 +1,15 @@ +import math import operator from pypy.interpreter.error import OperationError, oefmt -from pypy.objspace.std import model, newformat +from pypy.objspace.std import newformat from pypy.objspace.std.floattype import float_typedef, W_AbstractFloatObject from pypy.objspace.std.intobject import HASH_BITS, HASH_MODULUS from pypy.objspace.std.multimethod import FailedToImplementArgs from pypy.objspace.std.model import registerimplementation, W_Object from pypy.objspace.std.register_all import register_all +from pypy.objspace.std.longobject import W_LongObject, newlong_from_float from pypy.objspace.std.noneobject import W_NoneObject -from pypy.objspace.std.longobject import W_LongObject, newlong_from_float from rpython.rlib.rarithmetic import ( LONG_BIT, intmask, ovfcheck_float_to_int, r_uint) from rpython.rlib.rfloat import ( @@ -18,8 +19,6 @@ from rpython.rlib import rfloat from rpython.tool.sourcetools import func_with_new_name - -import math from pypy.objspace.std.intobject import W_IntObject HASH_INF = 314159 diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py --- a/pypy/objspace/std/intobject.py +++ b/pypy/objspace/std/intobject.py @@ -5,12 +5,11 @@ translation this module uses rarithmetic.ovfcheck to explicitly check for overflows, something CPython does not do anymore. """ - import operator import sys from rpython.rlib import jit -from rpython.rlib.objectmodel import instantiate, import_from_mixin, specialize +from rpython.rlib.objectmodel import instantiate from rpython.rlib.rarithmetic import ( LONG_BIT, intmask, is_valid_int, ovfcheck, r_longlong, r_uint, string_to_int) @@ -23,7 +22,6 @@ from pypy.interpreter import typedef from pypy.interpreter.baseobjspace import W_Root -from pypy.interpreter.buffer import Buffer from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter.gateway import ( WrappedDefault, applevel, interp2app, interpindirect2app, unwrap_spec) @@ -32,7 +30,6 @@ BINARY_OPS, CMP_OPS, COMMUTATIVE_OPS, IDTAG_INT) from pypy.objspace.std.stdtypedef import StdTypeDef - SENTINEL = object() HASH_BITS = 61 if sys.maxsize > 2 ** 31 - 1 else 31 @@ -40,7 +37,6 @@ class W_AbstractIntObject(W_Root): - __slots__ = () def is_w(self, space, w_other): diff --git a/pypy/objspace/std/marshal_impl.py b/pypy/objspace/std/marshal_impl.py --- a/pypy/objspace/std/marshal_impl.py +++ b/pypy/objspace/std/marshal_impl.py @@ -10,7 +10,7 @@ from pypy.interpreter.error import OperationError from pypy.objspace.std.register_all import register_all -from rpython.rlib.rarithmetic import LONG_BIT, r_longlong, r_uint, intmask +from rpython.rlib.rarithmetic import LONG_BIT, r_longlong, r_uint from pypy.objspace.std import model from pypy.objspace.std.dictmultiobject import W_DictMultiObject from pypy.interpreter.special import Ellipsis @@ -200,7 +200,6 @@ register(TYPE_BINARY_COMPLEX, unmarshal_Complex_bin) def marshal_w__Long(space, w_long, m): - from rpython.rlib.rbigint import rbigint from rpython.rlib.rarithmetic import r_ulonglong m.start(TYPE_LONG) SHIFT = 15 @@ -359,7 +358,6 @@ lng = u.atom_lng(tc) res = [None] * lng idx = 0 - space = u.space while idx < lng: res[idx] = unmarshal_str(u) idx += 1 diff --git a/pypy/objspace/std/multimethod.py b/pypy/objspace/std/multimethod.py --- a/pypy/objspace/std/multimethod.py +++ b/pypy/objspace/std/multimethod.py @@ -866,7 +866,6 @@ entryarray = CompressedArray(null_entry) indexarray = self.mrdtable.indexarray lst = self.mrdtable.list_of_types - indexline = [] def compress(typesprefix, typesnum): if len(typesprefix) == self.multimethod.arity: diff --git a/pypy/objspace/std/newformat.py b/pypy/objspace/std/newformat.py --- a/pypy/objspace/std/newformat.py +++ b/pypy/objspace/std/newformat.py @@ -466,7 +466,6 @@ if not got_align: self._align = "=" i += 1 - start_i = i self._width, i = _parse_int(self.space, spec, i, length) if length != i and spec[i] == ",": self._thousands_sep = True @@ -584,7 +583,6 @@ return space.wrap(self._pad(string)) def _get_locale(self, tp): - space = self.space if tp == "n": dec, thousands, grouping = rlocale.numeric_formatting() elif self._thousands_sep: @@ -681,12 +679,10 @@ grouping = self._loc_grouping min_width = spec.n_min_width grouping_state = 0 - count = 0 left = spec.n_digits n_ts = len(self._loc_thousands) need_separator = False done = False - groupings = len(grouping) previous = 0 while True: group = ord(grouping[grouping_state]) diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py --- a/pypy/objspace/std/objspace.py +++ b/pypy/objspace/std/objspace.py @@ -331,7 +331,6 @@ strdict=strdict, kwargs=kwargs) def newset(self): - from pypy.objspace.std.setobject import newset return W_SetObject(self, None) def newslice(self, w_start, w_end, w_step): diff --git a/pypy/objspace/std/proxy_helpers.py b/pypy/objspace/std/proxy_helpers.py --- a/pypy/objspace/std/proxy_helpers.py +++ b/pypy/objspace/std/proxy_helpers.py @@ -1,10 +1,8 @@ - """ Some transparent helpers, put here because of cyclic imports """ -from pypy.objspace.std.model import W_ANY, W_Object -from pypy.interpreter import baseobjspace +from pypy.objspace.std.model import W_ANY from pypy.interpreter.argument import Arguments from rpython.tool.sourcetools import func_with_new_name @@ -24,7 +22,7 @@ def function(space, w_transparent_list, __args__): args = __args__.prepend(space.wrap(op_name)) return space.call_args(w_transparent_list.w_controller, args) - + function = func_with_new_name(function, mm.name) mm.register(function, type_) @@ -32,14 +30,14 @@ def function(space, w_transparent_list, *args_w): args = Arguments(space, [space.wrap(op_name)] + list(args_w[:-1]) + args_w[-1]) return space.call_args(w_transparent_list.w_controller, args) - + function = func_with_new_name(function, mm.name) mm.register(function, type_, *([W_ANY] * (mm.arity - 1))) def install_mm_trampoline(type_, mm, is_local): classname = type_.__name__[2:] mm_name, op_name = create_mm_names(classname, mm, is_local) - + if ['__args__'] == mm.argnames_after: return install_general_args_trampoline(type_, mm, is_local, op_name) if ['args_w'] == mm.argnames_after: @@ -58,10 +56,10 @@ """ if mm.arity != 2: return False - + if len(mm.specialnames) != 2: return False - + # search over the signatures for signature in mm.signatures(): if signature == (type_.original, type_.original): @@ -69,21 +67,21 @@ return False def install_mm_special(type_, mm, is_local): - classname = type_.__name__[2:] + #classname = type_.__name__[2:] #mm_name, op_name = create_mm_names(classname, mm, is_local) - + def function(space, w_any, w_transparent_list): retval = space.call_function(w_transparent_list.w_controller, space.wrap(mm.specialnames[1]), w_any) return retval - + function = func_with_new_name(function, mm.specialnames[0]) - + mm.register(function, type_.typedef.any, type_) def register_type(type_): from pypy.objspace.std.stdtypedef import multimethods_defined_on - + for mm, is_local in multimethods_defined_on(type_.original): if not mm.name.startswith('__'): install_mm_trampoline(type_, mm, is_local) diff --git a/pypy/objspace/std/proxyobject.py b/pypy/objspace/std/proxyobject.py --- a/pypy/objspace/std/proxyobject.py +++ b/pypy/objspace/std/proxyobject.py @@ -1,15 +1,8 @@ - """ transparent list implementation """ - -from pypy.objspace.std.model import W_Object from pypy.interpreter.error import OperationError from pypy.interpreter import baseobjspace -#class W_Transparent(W_Object): -# def __init__(self, w_controller): -# self.controller = w_controller - def transparent_class(name, BaseCls): class W_Transparent(BaseCls): @@ -72,25 +65,3 @@ return W_Transparent W_Transparent = transparent_class('W_Transparent', baseobjspace.W_Root) -#W_TransparentObject = transparent_class('W_TransparentObject', W_Object) - -#from pypy.objspace.std.objecttype import object_typedef -#W_TransparentObject.typedef = object_typedef - -from pypy.interpreter.typedef import Function, GeneratorIterator, PyTraceback, \ - PyFrame, PyCode - -class W_TransparentFunction(W_Transparent): - typedef = Function.typedef - -class W_TransparentTraceback(W_Transparent): - typedef = PyTraceback.typedef - -class W_TransparentCode(W_Transparent): - typedef = PyCode.typedef - -class W_TransparentFrame(W_Transparent): - typedef = PyFrame.typedef - -class W_TransparentGenerator(W_Transparent): - typedef = GeneratorIterator.typedef diff --git a/pypy/objspace/std/slicetype.py b/pypy/objspace/std/slicetype.py --- a/pypy/objspace/std/slicetype.py +++ b/pypy/objspace/std/slicetype.py @@ -1,4 +1,4 @@ -from pypy.interpreter import baseobjspace, gateway +from pypy.interpreter import gateway from pypy.interpreter.typedef import GetSetProperty from pypy.objspace.std.stdtypedef import StdTypeDef, SMM from pypy.objspace.std.register_all import register_all diff --git a/pypy/objspace/std/stdtypedef.py b/pypy/objspace/std/stdtypedef.py --- a/pypy/objspace/std/stdtypedef.py +++ b/pypy/objspace/std/stdtypedef.py @@ -1,8 +1,7 @@ -from pypy.interpreter import gateway, baseobjspace, argument +from pypy.interpreter import gateway, baseobjspace from pypy.interpreter.error import OperationError, oefmt -from pypy.interpreter.typedef import TypeDef, GetSetProperty, Member -from pypy.interpreter.typedef import descr_get_dict, descr_set_dict -from pypy.interpreter.typedef import descr_del_dict +from pypy.interpreter.typedef import TypeDef, GetSetProperty, \ + descr_get_dict, descr_set_dict, descr_del_dict from pypy.interpreter.baseobjspace import SpaceCache from pypy.objspace.std import model from pypy.objspace.std.model import StdObjSpaceMultiMethod diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py --- a/pypy/objspace/std/stringmethods.py +++ b/pypy/objspace/std/stringmethods.py @@ -446,7 +446,6 @@ def descr_split(self, space, w_sep=None, maxsplit=-1): res = [] value = self._val(space) - length = len(value) if space.is_none(w_sep): res = split(value, maxsplit=maxsplit) return self._newlist_unwrapped(space, res) diff --git a/pypy/objspace/std/transparent.py b/pypy/objspace/std/transparent.py --- a/pypy/objspace/std/transparent.py +++ b/pypy/objspace/std/transparent.py @@ -1,14 +1,30 @@ - """ transparent.py - Several transparent proxy helpers """ - from pypy.interpreter import gateway from pypy.interpreter.error import OperationError, oefmt -from pypy.objspace.std.proxyobject import * +from pypy.interpreter.typedef import Function, GeneratorIterator, PyTraceback, \ + PyFrame, PyCode +from pypy.objspace.std.proxyobject import W_Transparent from pypy.objspace.std.typeobject import W_TypeObject -from rpython.rlib.objectmodel import r_dict from rpython.rlib.unroll import unrolling_iterable + +class W_TransparentFunction(W_Transparent): + typedef = Function.typedef + +class W_TransparentTraceback(W_Transparent): + typedef = PyTraceback.typedef + +class W_TransparentCode(W_Transparent): + typedef = PyCode.typedef + +class W_TransparentFrame(W_Transparent): + typedef = PyFrame.typedef + +class W_TransparentGenerator(W_Transparent): + typedef = GeneratorIterator.typedef + + class TypeCache(object): def __init__(self): self.cache = [] @@ -28,13 +44,10 @@ space.wrap(app_proxy_controller)) - def proxy(space, w_type, w_controller): """tproxy(typ, controller) -> obj Return something that looks like it is of type typ. Its behaviour is completely controlled by the controller.""" - from pypy.interpreter.typedef import Function, PyTraceback, PyFrame, \ - PyCode, GeneratorIterator if not space.is_true(space.callable(w_controller)): raise OperationError(space.w_TypeError, space.wrap("controller should be function")) diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py --- a/pypy/objspace/std/typeobject.py +++ b/pypy/objspace/std/typeobject.py @@ -2,12 +2,12 @@ from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter.function import Function, StaticMethod -from pypy.interpreter.typedef import weakref_descr, GetSetProperty,\ +from pypy.interpreter.typedef import weakref_descr, GetSetProperty, Member, \ descr_get_dict from pypy.interpreter.astcompiler.misc import mangle from pypy.objspace.std.model import W_Object from pypy.objspace.std.register_all import register_all -from pypy.objspace.std.stdtypedef import std_dict_descr, issubtypedef, Member +from pypy.objspace.std.stdtypedef import std_dict_descr, issubtypedef from pypy.objspace.std.stdtypedef import StdTypeDef from rpython.rlib.jit import (promote, elidable_promote, we_are_jitted, diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py --- a/rpython/rlib/rfile.py +++ b/rpython/rlib/rfile.py @@ -4,12 +4,12 @@ """ import os +from rpython.rlib import rposix +from rpython.rlib.rarithmetic import intmask +from rpython.rlib.rstring import StringBuilder from rpython.rtyper.lltypesystem import rffi, lltype from rpython.rtyper.tool import rffi_platform as platform from rpython.translator.tool.cbuild import ExternalCompilationInfo -from rpython.rlib.rarithmetic import r_uint, intmask -from rpython.rlib import rposix -from rpython.rlib.rstring import StringBuilder includes = ['stdio.h', 'sys/types.h'] if os.name == "posix": diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py --- a/rpython/rlib/rwin32.py +++ b/rpython/rlib/rwin32.py @@ -170,7 +170,7 @@ cfile.write(r''' #include #include - #include + #include #ifdef __GNUC__ #define _dosmaperr mingw_dosmaperr #endif @@ -197,6 +197,7 @@ standalone=True) except (CompilationError, WindowsError): # Fallback for the mingw32 compiler + assert static_platform.name == 'mingw32' errors = { 2: 2, 3: 2, 4: 24, 5: 13, 6: 9, 7: 12, 8: 12, 9: 12, 10: 7, 11: 8, 15: 2, 16: 13, 17: 18, 18: 2, 19: 13, 20: 13, 21: 13, From noreply at buildbot.pypy.org Wed Mar 19 07:26:20 2014 From: noreply at buildbot.pypy.org (Manuel Jacob) Date: Wed, 19 Mar 2014 07:26:20 +0100 (CET) Subject: [pypy-commit] pypy default: Kill W_MemoryView.descr_buffer() and move the comment to W_MemoryView.buffer_w(). Message-ID: <20140319062620.72D6F1D26D3@cobra.cs.uni-duesseldorf.de> Author: Manuel Jacob Branch: Changeset: r70083:f2302918495d Date: 2014-03-19 07:25 +0100 http://bitbucket.org/pypy/pypy/changeset/f2302918495d/ Log: Kill W_MemoryView.descr_buffer() and move the comment to W_MemoryView.buffer_w(). diff --git a/pypy/objspace/std/memoryview.py b/pypy/objspace/std/memoryview.py --- a/pypy/objspace/std/memoryview.py +++ b/pypy/objspace/std/memoryview.py @@ -176,6 +176,14 @@ self.buf = buf def buffer_w(self, space): + """ + Note that memoryview() is very inconsistent in CPython: it does not + support the buffer interface but does support the new buffer + interface: as a result, it is possible to pass memoryview to + e.g. socket.send() but not to file.write(). For simplicity and + consistency, in PyPy memoryview DOES support buffer(), which means + that it is accepted in more places than CPython. + """ return self.buf @staticmethod @@ -229,17 +237,6 @@ buf = buffer.SubBuffer(buf, start, size) return W_MemoryView(buf) - def descr_buffer(self, space): - """ - Note that memoryview() is very inconsistent in CPython: it does not - support the buffer interface but does support the new buffer - interface: as a result, it is possible to pass memoryview to - e.g. socket.send() but not to file.write(). For simplicity and - consistency, in PyPy memoryview DOES support buffer(), which means - that it is accepted in more places than CPython. - """ - return space.wrap(self.buf) - def descr_tobytes(self, space): return space.wrap(self.as_str()) From noreply at buildbot.pypy.org Wed Mar 19 07:28:15 2014 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 19 Mar 2014 07:28:15 +0100 (CET) Subject: [pypy-commit] stmgc c7-fork: intermediate Message-ID: <20140319062815.CD4D91D26D3@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c7-fork Changeset: r1072:5b46d0dcbbfd Date: 2014-03-19 07:28 +0100 http://bitbucket.org/pypy/stmgc/changeset/5b46d0dcbbfd/ Log: intermediate diff --git a/c7/demo/demo_random.c b/c7/demo/demo_random.c --- a/c7/demo/demo_random.c +++ b/c7/demo/demo_random.c @@ -14,7 +14,7 @@ #define THREAD_STARTS 300 // how many restarts of threads #define PREBUILT_ROOTS 3 #define MAXROOTS 1000 -#define FORKS 3 +#define FORKS 1 // SUPPORT struct node_s; @@ -339,9 +339,22 @@ if (p == (objptr_t)-1) { push_roots(); - stm_commit_transaction(); - if (arg) { + if (arg == NULL) { /* common case */ + stm_commit_transaction(); + td.num_roots_at_transaction_start = td.num_roots; + if (get_rand(100) < 98) { + STM_START_TRANSACTION(&stm_thread_local, here); + } else { + stm_start_inevitable_transaction(&stm_thread_local); + } + td.num_roots = td.num_roots_at_transaction_start; + p = NULL; + pop_roots(); + reload_roots(); + } + else { + /* run a fork() inside the transaction */ printf("========== FORK =========\n"); arg = NULL; pid_t child = fork(); @@ -355,19 +368,10 @@ num_forked_children++; else num_forked_children = 0; + + pop_roots(); + p = NULL; } - - td.num_roots_at_transaction_start = td.num_roots; - - if (get_rand(100) < 98) { - STM_START_TRANSACTION(&stm_thread_local, here); - } else { - stm_start_inevitable_transaction(&stm_thread_local); - } - td.num_roots = td.num_roots_at_transaction_start; - p = NULL; - pop_roots(); - reload_roots(); } } stm_commit_transaction(); diff --git a/c7/stm/core.c b/c7/stm/core.c --- a/c7/stm/core.c +++ b/c7/stm/core.c @@ -164,11 +164,13 @@ STM_SEGMENT->transaction_read_version = 1; } -void _stm_start_transaction(stm_thread_local_t *tl, stm_jmpbuf_t *jmpbuf) +void _stm_start_transaction(stm_thread_local_t *tl, stm_jmpbuf_t *jmpbuf, + int already_got_the_lock) { - assert(!_stm_in_transaction(tl)); - - s_mutex_lock(); + if (!already_got_the_lock) { + assert(!_stm_in_transaction(tl)); + s_mutex_lock(); + } retry: if (jmpbuf == NULL) { @@ -447,7 +449,7 @@ /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */ } -void stm_commit_transaction(void) +void _stm_commit_transaction(int keep_the_lock_at_the_end) { assert(!_has_mutex()); assert(STM_PSEGMENT->safe_point == SP_RUNNING); @@ -506,7 +508,8 @@ _finish_transaction(); /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */ - s_mutex_unlock(); + if (!keep_the_lock_at_the_end) + s_mutex_unlock(); } void stm_abort_transaction(void) diff --git a/c7/stm/forksupport.c b/c7/stm/forksupport.c --- a/c7/stm/forksupport.c +++ b/c7/stm/forksupport.c @@ -9,6 +9,7 @@ static char *fork_big_copy = NULL; static stm_thread_local_t *fork_this_tl; +static bool fork_was_in_transaction; static char *setup_mmap(char *reason); /* forward, in setup.c */ static void do_or_redo_setup_after_fork(void); /* forward, in setup.c */ @@ -21,23 +22,19 @@ if (stm_object_pages == NULL) return; - /* This assumes that fork() is not called from transactions. - So far we attempt to check this by walking all stm_thread_local_t, + /* So far we attempt to check this by walking all stm_thread_local_t, marking the one from the current thread, and verifying that it's not running a transaction. This assumes that the stm_thread_local_t is just a __thread variable, so never changes threads. */ s_mutex_lock(); - mutex_pages_lock(); - dprintf(("forksupport_prepare: synchronized all threads\n")); + dprintf(("forksupport_prepare\n")); fork_this_tl = NULL; stm_thread_local_t *tl = stm_all_thread_locals; do { if (pthread_equal(*_get_cpth(tl), pthread_self())) { - if (_stm_in_transaction(tl)) - stm_fatalerror("fork(): cannot be used inside a transaction"); if (fork_this_tl != NULL) stm_fatalerror("fork(): found several stm_thread_local_t" " from the same thread"); @@ -48,6 +45,24 @@ if (fork_this_tl == NULL) stm_fatalerror("fork(): found no stm_thread_local_t from this thread"); + s_mutex_unlock(); + + /* Run a commit without releasing the mutex at the end; if necessary, + actually start a dummy inevitable transaction for this + */ + fork_was_in_transaction = _stm_in_transaction(fork_this_tl); + if (!fork_was_in_transaction) + stm_start_inevitable_transaction(fork_this_tl); + _stm_commit_transaction(/*keep_the_lock_at_the_end =*/ 1); + + printf("fork_was_in_transaction: %d\n" + "fork_this_tl->associated_segment_num: %d\n", + (int)fork_was_in_transaction, + (int)fork_this_tl->associated_segment_num); + + /* Note that the commit can still fail and abort, which should be fine */ + + mutex_pages_lock(); /* Make a new mmap at some other address, but of the same size as the standard mmap at stm_object_pages @@ -92,6 +107,8 @@ assert(fork_big_copy == NULL); fork_big_copy = big_copy; + + assert(_has_mutex()); } static void forksupport_parent(void) @@ -99,6 +116,9 @@ if (stm_object_pages == NULL) return; + assert(_is_tl_registered(fork_this_tl)); + assert(_has_mutex()); + /* In the parent, after fork(), we can simply forget about the big copy that we made for the child. */ @@ -109,37 +129,32 @@ dprintf(("forksupport_parent: continuing to run\n")); mutex_pages_unlock(); - s_mutex_unlock(); + + printf("AFTER: fork_was_in_transaction: %d\n" + "fork_this_tl->associated_segment_num: %d\n", + (int)fork_was_in_transaction, + (int)fork_this_tl->associated_segment_num); + + if (fork_was_in_transaction) { + _stm_start_transaction(fork_this_tl, NULL, + /*already_got_the_lock =*/ 1); + } + else { + s_mutex_unlock(); + } } static void forksupport_child(void) { if (stm_object_pages == NULL) return; + abort(); - /* In the child, first unregister all other stm_thread_local_t, - mostly as a way to free the memory used by the shadowstacks - */ + /* this new process contains no other thread, so we can + just release these locks early */ mutex_pages_unlock(); s_mutex_unlock(); - assert(fork_this_tl != NULL); - while (stm_all_thread_locals->next != stm_all_thread_locals) { - if (stm_all_thread_locals == fork_this_tl) - stm_unregister_thread_local(stm_all_thread_locals->next); - else - stm_unregister_thread_local(stm_all_thread_locals); - } - assert(stm_all_thread_locals == fork_this_tl); - - /* Restore a few things in the child: the new pthread_self(), and - the %gs register (although I suppose it should be preserved by - fork()) - */ - *_get_cpth(fork_this_tl) = pthread_self(); - set_gs_register(get_segment_base(fork_this_tl->associated_segment_num)); - fork_this_tl = NULL; - /* Move the copy of the mmap over the old one, overwriting it and thus freeing the old mapping in this process */ @@ -152,6 +167,24 @@ stm_fatalerror("after fork: mremap failed: %m"); fork_big_copy = NULL; + /* Unregister all other stm_thread_local_t, mostly as a way to free + the memory used by the shadowstacks + */ + assert(fork_this_tl != NULL); + while (stm_all_thread_locals->next != stm_all_thread_locals) { + if (stm_all_thread_locals == fork_this_tl) + stm_unregister_thread_local(stm_all_thread_locals->next); + else + stm_unregister_thread_local(stm_all_thread_locals); + } + assert(stm_all_thread_locals == fork_this_tl); + + /* Restore a few things: the new pthread_self(), and the %gs + register (although I suppose it should be preserved by fork()) + */ + *_get_cpth(fork_this_tl) = pthread_self(); + set_gs_register(get_segment_base(fork_this_tl->associated_segment_num)); + /* Call a subset of stm_teardown() / stm_setup() to free and recreate the necessary data in all segments, and to clean up some of the global data like the big arrays that don't make sense any @@ -172,6 +205,11 @@ pages_initialize_shared(start, stop - start); mutex_pages_unlock(); + /* Now restart the transaction if needed + */ + if (fork_was_in_transaction) + stm_start_inevitable_transaction(fork_this_tl); + dprintf(("forksupport_child: running one thread now\n")); } diff --git a/c7/stmgc.h b/c7/stmgc.h --- a/c7/stmgc.h +++ b/c7/stmgc.h @@ -78,7 +78,8 @@ object_t *_stm_allocate_slowpath(ssize_t); object_t *_stm_allocate_external(ssize_t); void _stm_become_inevitable(const char*); -void _stm_start_transaction(stm_thread_local_t *, stm_jmpbuf_t *); +void _stm_start_transaction(stm_thread_local_t *, stm_jmpbuf_t *, int); +void _stm_commit_transaction(int); void _stm_collectable_safe_point(void); /* for tests, but also used in duhton: */ @@ -257,17 +258,19 @@ stm_jmpbuf_t). */ #define STM_START_TRANSACTION(tl, jmpbuf) ({ \ while (__builtin_setjmp(jmpbuf) == 1) { /*redo setjmp*/ } \ - _stm_start_transaction(tl, &jmpbuf); \ + _stm_start_transaction(tl, &jmpbuf, 0); \ }) /* Start an inevitable transaction, if it's going to return from the current function immediately. */ static inline void stm_start_inevitable_transaction(stm_thread_local_t *tl) { - _stm_start_transaction(tl, NULL); + _stm_start_transaction(tl, NULL, 0); } /* Commit a transaction. */ -void stm_commit_transaction(void); +static inline void stm_commit_transaction(void) { + _stm_commit_transaction(0); +} /* Abort the currently running transaction. */ void stm_abort_transaction(void) __attribute__((noreturn)); From noreply at buildbot.pypy.org Wed Mar 19 07:34:20 2014 From: noreply at buildbot.pypy.org (Manuel Jacob) Date: Wed, 19 Mar 2014 07:34:20 +0100 (CET) Subject: [pypy-commit] pypy py3k: Fix translation. Message-ID: <20140319063420.9B0F21D2800@cobra.cs.uni-duesseldorf.de> Author: Manuel Jacob Branch: py3k Changeset: r70084:34feb354a1da Date: 2014-03-19 07:33 +0100 http://bitbucket.org/pypy/pypy/changeset/34feb354a1da/ Log: Fix translation. diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -197,7 +197,7 @@ w_impl = space.lookup(self, '__buffer__') if w_impl is not None: w_result = space.get_and_call_function(w_impl, self) - if space.isinstance_w(w_result, space.w_buffer): + if space.isinstance_w(w_result, space.w_memoryview): return w_result.buffer_w(space) self._typed_unwrap_error(space, "buffer") From noreply at buildbot.pypy.org Wed Mar 19 08:48:46 2014 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 19 Mar 2014 08:48:46 +0100 (CET) Subject: [pypy-commit] stmgc c7-fork: ... Message-ID: <20140319074846.D33A61C01DE@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c7-fork Changeset: r1073:769b3113998d Date: 2014-03-19 08:48 +0100 http://bitbucket.org/pypy/stmgc/changeset/769b3113998d/ Log: ... diff --git a/c7/stm/forksupport.c b/c7/stm/forksupport.c --- a/c7/stm/forksupport.c +++ b/c7/stm/forksupport.c @@ -4,7 +4,7 @@ /* XXX this is currently not doing copy-on-write, but simply forces a - copy of all shared pages as soon as fork() is called. */ + copy of all pages as soon as fork() is called. */ static char *fork_big_copy = NULL; @@ -12,8 +12,6 @@ static bool fork_was_in_transaction; static char *setup_mmap(char *reason); /* forward, in setup.c */ -static void do_or_redo_setup_after_fork(void); /* forward, in setup.c */ -static void do_or_redo_teardown_after_fork(void); /* forward, in setup.c */ static pthread_t *_get_cpth(stm_thread_local_t *);/* forward, in setup.c */ @@ -31,37 +29,34 @@ dprintf(("forksupport_prepare\n")); - fork_this_tl = NULL; + stm_thread_local_t *this_tl = NULL; stm_thread_local_t *tl = stm_all_thread_locals; do { if (pthread_equal(*_get_cpth(tl), pthread_self())) { - if (fork_this_tl != NULL) + if (this_tl != NULL) stm_fatalerror("fork(): found several stm_thread_local_t" " from the same thread"); - fork_this_tl = tl; + this_tl = tl; } tl = tl->next; } while (tl != stm_all_thread_locals); - if (fork_this_tl == NULL) + if (this_tl == NULL) stm_fatalerror("fork(): found no stm_thread_local_t from this thread"); s_mutex_unlock(); - /* Run a commit without releasing the mutex at the end; if necessary, - actually start a dummy inevitable transaction for this - */ - fork_was_in_transaction = _stm_in_transaction(fork_this_tl); - if (!fork_was_in_transaction) - stm_start_inevitable_transaction(fork_this_tl); - _stm_commit_transaction(/*keep_the_lock_at_the_end =*/ 1); + bool was_in_transaction = _stm_in_transaction(this_tl); + if (was_in_transaction) { + stm_become_inevitable("fork"); + /* Note that the line above can still fail and abort, which should + be fine */ + } + else { + stm_start_inevitable_transaction(this_tl); + } - printf("fork_was_in_transaction: %d\n" - "fork_this_tl->associated_segment_num: %d\n", - (int)fork_was_in_transaction, - (int)fork_this_tl->associated_segment_num); - - /* Note that the commit can still fail and abort, which should be fine */ - + s_mutex_lock(); + synchronize_all_threads(); mutex_pages_lock(); /* Make a new mmap at some other address, but of the same size as @@ -79,7 +74,7 @@ } /* Copy all the data from the two ranges of objects (large, small) - into the new mmap --- but only the shared objects + into the new mmap */ uintptr_t pagenum, endpagenum; pagenum = END_NURSERY_PAGE; /* starts after the nursery */ @@ -100,15 +95,32 @@ endpagenum = NB_PAGES; } - pagecopy(big_copy + pagenum * 4096UL, - stm_object_pages + pagenum * 4096UL); + char *src = stm_object_pages + pagenum * 4096UL; + char *dst = big_copy + pagenum * 4096UL; + pagecopy(dst, src); + + struct page_shared_s ps = pages_privatized[pagenum - PAGE_FLAG_START]; + if (ps.by_segment != 0) { + long j; + for (j = 0; j < NB_SEGMENTS; j++) { + src += NB_PAGES * 4096UL; + dst += NB_PAGES * 4096UL; + if (ps.by_segment & (1 << j)) { + pagecopy(dst, src); + } + } + } pagenum++; } assert(fork_big_copy == NULL); fork_big_copy = big_copy; + fork_this_tl = this_tl; + fork_was_in_transaction = was_in_transaction; assert(_has_mutex()); + printf("forksupport_prepare: from %p %p\n", fork_this_tl, + fork_this_tl->creating_pthread[0]); } static void forksupport_parent(void) @@ -116,8 +128,10 @@ if (stm_object_pages == NULL) return; + printf("forksupport_parent: continuing to run %p %p\n", fork_this_tl, + fork_this_tl->creating_pthread[0]); + assert(_has_mutex()); assert(_is_tl_registered(fork_this_tl)); - assert(_has_mutex()); /* In the parent, after fork(), we can simply forget about the big copy that we made for the child. @@ -125,22 +139,33 @@ assert(fork_big_copy != NULL); munmap(fork_big_copy, TOTAL_MEMORY); fork_big_copy = NULL; + bool was_in_transaction = fork_was_in_transaction; + + mutex_pages_unlock(); + s_mutex_unlock(); + + if (!was_in_transaction) { + stm_commit_transaction(); + } dprintf(("forksupport_parent: continuing to run\n")); +} - mutex_pages_unlock(); +static void fork_abort_thread(long i) +{ + struct stm_priv_segment_info_s *pr = get_priv_segment(i); + dprintf(("forksupport_child: abort in seg%ld\n", i)); + assert(pr->pub.running_thread->associated_segment_num == i); + assert(pr->transaction_state == TS_REGULAR); + set_gs_register(get_segment_base(i)); - printf("AFTER: fork_was_in_transaction: %d\n" - "fork_this_tl->associated_segment_num: %d\n", - (int)fork_was_in_transaction, - (int)fork_this_tl->associated_segment_num); - - if (fork_was_in_transaction) { - _stm_start_transaction(fork_this_tl, NULL, - /*already_got_the_lock =*/ 1); - } - else { - s_mutex_unlock(); + stm_jmpbuf_t jmpbuf; + if (__builtin_setjmp(jmpbuf) == 0) { + pr->pub.jmpbuf_ptr = &jmpbuf; +#ifndef NDEBUG + pr->running_pthread = pthread_self(); +#endif + stm_abort_transaction(); } } @@ -148,7 +173,6 @@ { if (stm_object_pages == NULL) return; - abort(); /* this new process contains no other thread, so we can just release these locks early */ @@ -170,7 +194,6 @@ /* Unregister all other stm_thread_local_t, mostly as a way to free the memory used by the shadowstacks */ - assert(fork_this_tl != NULL); while (stm_all_thread_locals->next != stm_all_thread_locals) { if (stm_all_thread_locals == fork_this_tl) stm_unregister_thread_local(stm_all_thread_locals->next); @@ -179,37 +202,58 @@ } assert(stm_all_thread_locals == fork_this_tl); - /* Restore a few things: the new pthread_self(), and the %gs - register (although I suppose it should be preserved by fork()) - */ - *_get_cpth(fork_this_tl) = pthread_self(); - set_gs_register(get_segment_base(fork_this_tl->associated_segment_num)); - - /* Call a subset of stm_teardown() / stm_setup() to free and - recreate the necessary data in all segments, and to clean up some - of the global data like the big arrays that don't make sense any - more. We keep other things like the smallmalloc and largemalloc - internal state. - */ - do_or_redo_teardown_after_fork(); - do_or_redo_setup_after_fork(); - /* Make all pages shared again. */ - mutex_pages_lock(); - uintptr_t start = END_NURSERY_PAGE; - uintptr_t stop = (uninitialized_page_start - stm_object_pages) / 4096UL; - pages_initialize_shared(start, stop - start); - start = (uninitialized_page_stop - stm_object_pages) / 4096UL; - stop = NB_PAGES; - pages_initialize_shared(start, stop - start); - mutex_pages_unlock(); + uintptr_t pagenum, endpagenum; + pagenum = END_NURSERY_PAGE; /* starts after the nursery */ + endpagenum = (uninitialized_page_start - stm_object_pages) / 4096UL; - /* Now restart the transaction if needed + while (1) { + if (UNLIKELY(pagenum == endpagenum)) { + /* we reach this point usually twice, because there are + more pages after 'uninitialized_page_stop' */ + if (endpagenum == NB_PAGES) + break; /* done */ + pagenum = (uninitialized_page_stop - stm_object_pages) / 4096UL; + endpagenum = NB_PAGES; + if (endpagenum == NB_PAGES) + break; /* done */ + } + + struct page_shared_s ps = pages_privatized[pagenum - PAGE_FLAG_START]; + long j; + for (j = 0; j < NB_SEGMENTS; j++) { + if (!(ps.by_segment & (1 << j))) { + _page_do_reshare(j + 1, pagenum); + } + } + pagenum++; + } + + /* Force the interruption of other running segments */ - if (fork_was_in_transaction) - stm_start_inevitable_transaction(fork_this_tl); + long i; + for (i = 1; i <= NB_SEGMENTS; i++) { + struct stm_priv_segment_info_s *pr = get_priv_segment(i); + if (pr->pub.running_thread != NULL && + pr->pub.running_thread != fork_this_tl) { + fork_abort_thread(i); + } + } + /* Restore a few things: the new pthread_self(), and the %gs + register */ + int segnum = fork_this_tl->associated_segment_num; + assert(1 <= segnum && segnum <= NB_SEGMENTS); + *_get_cpth(fork_this_tl) = pthread_self(); + set_gs_register(get_segment_base(segnum)); + assert(STM_SEGMENT->segment_num == segnum); + + if (!fork_was_in_transaction) { + stm_commit_transaction(); + } + + /* Done */ dprintf(("forksupport_child: running one thread now\n")); } diff --git a/c7/stm/pages.c b/c7/stm/pages.c --- a/c7/stm/pages.c +++ b/c7/stm/pages.c @@ -146,6 +146,13 @@ mutex_pages_unlock(); } +static void _page_do_reshare(long segnum, uintptr_t pagenum) +{ + char *segment_base = get_segment_base(segnum); + d_remap_file_pages(segment_base + pagenum * 4096UL, + 4096, pagenum); +} + static void page_reshare(uintptr_t pagenum) { struct page_shared_s ps = pages_privatized[pagenum - PAGE_FLAG_START]; diff --git a/c7/stm/pages.h b/c7/stm/pages.h --- a/c7/stm/pages.h +++ b/c7/stm/pages.h @@ -38,6 +38,7 @@ static void pages_initialize_shared(uintptr_t pagenum, uintptr_t count); static void page_privatize(uintptr_t pagenum); static void page_reshare(uintptr_t pagenum); +static void _page_do_reshare(long segnum, uintptr_t pagenum); /* Note: don't ever do "mutex_pages_lock(); mutex_lock()" in that order */ static void mutex_pages_lock(void); diff --git a/c7/stmgc.h b/c7/stmgc.h --- a/c7/stmgc.h +++ b/c7/stmgc.h @@ -69,7 +69,7 @@ /* the next fields are handled internally by the library */ int associated_segment_num; struct stm_thread_local_s *prev, *next; - long creating_pthread[4]; + void *creating_pthread[2]; } stm_thread_local_t; /* this should use llvm's coldcc calling convention, From noreply at buildbot.pypy.org Wed Mar 19 08:59:40 2014 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 19 Mar 2014 08:59:40 +0100 (CET) Subject: [pypy-commit] stmgc c7-fork: Found out I was missing copying this Message-ID: <20140319075940.781271C01DE@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c7-fork Changeset: r1074:d4ff527cda67 Date: 2014-03-19 08:59 +0100 http://bitbucket.org/pypy/stmgc/changeset/d4ff527cda67/ Log: Found out I was missing copying this diff --git a/c7/stm/forksupport.c b/c7/stm/forksupport.c --- a/c7/stm/forksupport.c +++ b/c7/stm/forksupport.c @@ -15,6 +15,17 @@ static pthread_t *_get_cpth(stm_thread_local_t *);/* forward, in setup.c */ +static bool page_is_null(char *p) +{ + long *q = (long *)p; + long i; + for (i = 0; i < 4096 / sizeof(long); i++) + if (q[i] != 0) + return false; + return true; +} + + static void forksupport_prepare(void) { if (stm_object_pages == NULL) @@ -64,13 +75,25 @@ */ char *big_copy = setup_mmap("stmgc's fork support"); - /* Copy each of the segment infos into the new mmap + /* Copy each of the segment infos into the new mmap, nurseries, + and associated read markers */ long i; for (i = 1; i <= NB_SEGMENTS; i++) { - struct stm_priv_segment_info_s *src = get_priv_segment(i); - char *dst = big_copy + (((char *)src) - stm_object_pages); - *(struct stm_priv_segment_info_s *)dst = *src; + char *src, *dst; + struct stm_priv_segment_info_s *psrc = get_priv_segment(i); + dst = big_copy + (((char *)psrc) - stm_object_pages); + *(struct stm_priv_segment_info_s *)dst = *psrc; + + src = get_segment_base(i) + FIRST_READMARKER_PAGE * 4096UL; + dst = big_copy + (src - stm_object_pages); + long j; + for (j = 0; j < END_NURSERY_PAGE - FIRST_READMARKER_PAGE; j++) { + if (!page_is_null(src)) + pagecopy(dst, src); + src += 4096; + dst += 4096; + } } /* Copy all the data from the two ranges of objects (large, small) From noreply at buildbot.pypy.org Wed Mar 19 09:10:43 2014 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 19 Mar 2014 09:10:43 +0100 (CET) Subject: [pypy-commit] stmgc c7-fork: Seems to work now; clean up intermediate attempts Message-ID: <20140319081043.539B01C01DE@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c7-fork Changeset: r1075:75893b92af4e Date: 2014-03-19 09:10 +0100 http://bitbucket.org/pypy/stmgc/changeset/75893b92af4e/ Log: Seems to work now; clean up intermediate attempts diff --git a/c7/demo/demo_random.c b/c7/demo/demo_random.c --- a/c7/demo/demo_random.c +++ b/c7/demo/demo_random.c @@ -10,11 +10,11 @@ #include "stmgc.h" #define NUMTHREADS 3 -#define STEPS_PER_THREAD 300 -#define THREAD_STARTS 300 // how many restarts of threads +#define STEPS_PER_THREAD 500 +#define THREAD_STARTS 1000 // how many restarts of threads #define PREBUILT_ROOTS 3 #define MAXROOTS 1000 -#define FORKS 1 +#define FORKS 3 // SUPPORT struct node_s; diff --git a/c7/stm/core.c b/c7/stm/core.c --- a/c7/stm/core.c +++ b/c7/stm/core.c @@ -164,13 +164,11 @@ STM_SEGMENT->transaction_read_version = 1; } -void _stm_start_transaction(stm_thread_local_t *tl, stm_jmpbuf_t *jmpbuf, - int already_got_the_lock) +void _stm_start_transaction(stm_thread_local_t *tl, stm_jmpbuf_t *jmpbuf) { - if (!already_got_the_lock) { - assert(!_stm_in_transaction(tl)); - s_mutex_lock(); - } + assert(!_stm_in_transaction(tl)); + + s_mutex_lock(); retry: if (jmpbuf == NULL) { @@ -449,7 +447,7 @@ /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */ } -void _stm_commit_transaction(int keep_the_lock_at_the_end) +void stm_commit_transaction(void) { assert(!_has_mutex()); assert(STM_PSEGMENT->safe_point == SP_RUNNING); @@ -508,8 +506,7 @@ _finish_transaction(); /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */ - if (!keep_the_lock_at_the_end) - s_mutex_unlock(); + s_mutex_unlock(); } void stm_abort_transaction(void) diff --git a/c7/stm/pages.c b/c7/stm/pages.c --- a/c7/stm/pages.c +++ b/c7/stm/pages.c @@ -25,10 +25,6 @@ static void teardown_pages(void) { memset(&pages_ctl, 0, sizeof(pages_ctl)); -} - -static void teardown_pages_1(void) -{ memset(pages_privatized, 0, sizeof(pages_privatized)); } diff --git a/c7/stm/setup.c b/c7/stm/setup.c --- a/c7/stm/setup.c +++ b/c7/stm/setup.c @@ -14,8 +14,25 @@ return result; } -static void do_or_redo_setup_after_fork(void) +void stm_setup(void) { + /* Check that some values are acceptable */ + assert(NB_SEGMENTS <= NB_SEGMENTS_MAX); + assert(4096 <= ((uintptr_t)STM_SEGMENT)); + assert((uintptr_t)STM_SEGMENT == (uintptr_t)STM_PSEGMENT); + assert(((uintptr_t)STM_PSEGMENT) + sizeof(*STM_PSEGMENT) <= 8192); + assert(2 <= FIRST_READMARKER_PAGE); + assert(FIRST_READMARKER_PAGE * 4096UL <= READMARKER_START); + assert(READMARKER_START < READMARKER_END); + assert(READMARKER_END <= 4096UL * FIRST_OBJECT_PAGE); + assert(FIRST_OBJECT_PAGE < NB_PAGES); + assert((NB_PAGES * 4096UL) >> 8 <= (FIRST_OBJECT_PAGE * 4096UL) >> 4); + assert((END_NURSERY_PAGE * 4096UL) >> 8 <= + (FIRST_READMARKER_PAGE * 4096UL)); + assert(_STM_FAST_ALLOC <= NB_NURSERY_PAGES * 4096); + + stm_object_pages = setup_mmap("initial stm_object_pages mmap()"); + /* The segment 0 is not used to run transactions, but contains the shared copy of the pages. We mprotect all pages before so that accesses fail, up to and including the pages corresponding to the @@ -68,43 +85,25 @@ so a null read marker means "not read" whatever the current transaction_read_version is. */ - setup_nursery(); -} - -void stm_setup(void) -{ - /* Check that some values are acceptable */ - assert(NB_SEGMENTS <= NB_SEGMENTS_MAX); - assert(4096 <= ((uintptr_t)STM_SEGMENT)); - assert((uintptr_t)STM_SEGMENT == (uintptr_t)STM_PSEGMENT); - assert(((uintptr_t)STM_PSEGMENT) + sizeof(*STM_PSEGMENT) <= 8192); - assert(2 <= FIRST_READMARKER_PAGE); - assert(FIRST_READMARKER_PAGE * 4096UL <= READMARKER_START); - assert(READMARKER_START < READMARKER_END); - assert(READMARKER_END <= 4096UL * FIRST_OBJECT_PAGE); - assert(FIRST_OBJECT_PAGE < NB_PAGES); - assert((NB_PAGES * 4096UL) >> 8 <= (FIRST_OBJECT_PAGE * 4096UL) >> 4); - assert((END_NURSERY_PAGE * 4096UL) >> 8 <= - (FIRST_READMARKER_PAGE * 4096UL)); - assert(_STM_FAST_ALLOC <= NB_NURSERY_PAGES * 4096); - - stm_object_pages = setup_mmap("initial stm_object_pages mmap()"); - - do_or_redo_setup_after_fork(); setup_sync(); + setup_nursery(); setup_gcpage(); setup_pages(); setup_forksupport(); } -static void do_or_redo_teardown_after_fork(void) +void stm_teardown(void) { + /* This function is called during testing, but normal programs don't + need to call it. */ + assert(!_has_mutex()); + long i; for (i = 1; i <= NB_SEGMENTS; i++) { struct stm_priv_segment_info_s *pr = get_priv_segment(i); - LIST_FREE(pr->objects_pointing_to_nursery); - LIST_FREE(pr->large_overflow_objects); + assert(pr->objects_pointing_to_nursery == NULL); + assert(pr->large_overflow_objects == NULL); list_free(pr->modified_old_objects); list_free(pr->young_weakrefs); list_free(pr->old_weakrefs); @@ -113,22 +112,10 @@ tree_free(pr->callbacks_on_abort); } - teardown_core(); - teardown_sync_1(); - teardown_pages_1(); -} - -void stm_teardown(void) -{ - /* This function is called during testing, but normal programs don't - need to call it. */ - assert(!_has_mutex()); - - do_or_redo_teardown_after_fork(); - munmap(stm_object_pages, TOTAL_MEMORY); stm_object_pages = NULL; + teardown_core(); teardown_sync(); teardown_gcpage(); teardown_pages(); diff --git a/c7/stm/sync.c b/c7/stm/sync.c --- a/c7/stm/sync.c +++ b/c7/stm/sync.c @@ -36,10 +36,6 @@ char reserved[192]; } sync_ctl __attribute__((aligned(64))); -#ifndef NDEBUG -static bool _safe_points_requested = false; -#endif - static void setup_sync(void) { @@ -63,15 +59,8 @@ if (pthread_cond_destroy(&sync_ctl.cond[i]) != 0) stm_fatalerror("cond destroy: %m"); } -} -static void teardown_sync_1(void) -{ memset(&sync_ctl, 0, sizeof(sync_ctl)); -#ifndef NDEBUG - _safe_points_requested = false; -#endif - pause_signalled = false; } #ifndef NDEBUG @@ -263,6 +252,10 @@ /************************************************************/ +#ifndef NDEBUG +static bool _safe_points_requested = false; +#endif + static void signal_everybody_to_pause_running(void) { assert(_safe_points_requested == false); diff --git a/c7/stmgc.h b/c7/stmgc.h --- a/c7/stmgc.h +++ b/c7/stmgc.h @@ -78,8 +78,7 @@ object_t *_stm_allocate_slowpath(ssize_t); object_t *_stm_allocate_external(ssize_t); void _stm_become_inevitable(const char*); -void _stm_start_transaction(stm_thread_local_t *, stm_jmpbuf_t *, int); -void _stm_commit_transaction(int); +void _stm_start_transaction(stm_thread_local_t *, stm_jmpbuf_t *); void _stm_collectable_safe_point(void); /* for tests, but also used in duhton: */ @@ -258,19 +257,17 @@ stm_jmpbuf_t). */ #define STM_START_TRANSACTION(tl, jmpbuf) ({ \ while (__builtin_setjmp(jmpbuf) == 1) { /*redo setjmp*/ } \ - _stm_start_transaction(tl, &jmpbuf, 0); \ + _stm_start_transaction(tl, &jmpbuf); \ }) /* Start an inevitable transaction, if it's going to return from the current function immediately. */ static inline void stm_start_inevitable_transaction(stm_thread_local_t *tl) { - _stm_start_transaction(tl, NULL, 0); + _stm_start_transaction(tl, NULL); } /* Commit a transaction. */ -static inline void stm_commit_transaction(void) { - _stm_commit_transaction(0); -} +void stm_commit_transaction(void); /* Abort the currently running transaction. */ void stm_abort_transaction(void) __attribute__((noreturn)); From noreply at buildbot.pypy.org Wed Mar 19 09:17:22 2014 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 19 Mar 2014 09:17:22 +0100 (CET) Subject: [pypy-commit] stmgc c7-fork: Re-hide these printfs Message-ID: <20140319081722.901911C01DE@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c7-fork Changeset: r1076:62a1be4e540f Date: 2014-03-19 09:17 +0100 http://bitbucket.org/pypy/stmgc/changeset/62a1be4e540f/ Log: Re-hide these printfs diff --git a/c7/stm/forksupport.c b/c7/stm/forksupport.c --- a/c7/stm/forksupport.c +++ b/c7/stm/forksupport.c @@ -142,8 +142,8 @@ fork_was_in_transaction = was_in_transaction; assert(_has_mutex()); - printf("forksupport_prepare: from %p %p\n", fork_this_tl, - fork_this_tl->creating_pthread[0]); + dprintf(("forksupport_prepare: from %p %p\n", fork_this_tl, + fork_this_tl->creating_pthread[0])); } static void forksupport_parent(void) @@ -151,8 +151,8 @@ if (stm_object_pages == NULL) return; - printf("forksupport_parent: continuing to run %p %p\n", fork_this_tl, - fork_this_tl->creating_pthread[0]); + dprintf(("forksupport_parent: continuing to run %p %p\n", fork_this_tl, + fork_this_tl->creating_pthread[0])); assert(_has_mutex()); assert(_is_tl_registered(fork_this_tl)); From noreply at buildbot.pypy.org Wed Mar 19 09:23:20 2014 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 19 Mar 2014 09:23:20 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: import stmgc/75893b92af4e (c7-fork) Message-ID: <20140319082320.3357C1C01DE@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r70085:22b83e351878 Date: 2014-03-19 09:22 +0100 http://bitbucket.org/pypy/pypy/changeset/22b83e351878/ Log: import stmgc/75893b92af4e (c7-fork) diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision --- a/rpython/translator/stm/src_stm/revision +++ b/rpython/translator/stm/src_stm/revision @@ -1,1 +1,1 @@ -885ed3b0f6ee +75893b92af4e diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c --- a/rpython/translator/stm/src_stm/stm/core.c +++ b/rpython/translator/stm/src_stm/stm/core.c @@ -158,7 +158,7 @@ MAP_FIXED | MAP_PAGES_FLAGS, -1, 0) != readmarkers) { /* fall-back */ #if STM_TESTS - stm_fatalerror("reset_transaction_read_version: %m\n"); + stm_fatalerror("reset_transaction_read_version: %m"); #endif memset(readmarkers, 0, NB_READMARKER_PAGES * 4096UL); } @@ -273,7 +273,7 @@ assert(_has_mutex_pages()); assert(!_is_young(obj)); - char *segment_base = get_segment(source_segment_num)->segment_base; + char *segment_base = get_segment_base(source_segment_num); uintptr_t start = (uintptr_t)obj; uintptr_t first_page = start / 4096UL; struct object_s *realobj = (struct object_s *) @@ -519,19 +519,17 @@ static void reset_modified_from_other_segments(int segment_num) { - /* pull the right versions from other threads in order + /* pull the right versions from segment 0 in order to reset our pages as part of an abort. Note that this function is also sometimes called from contention.c to clean up the state of a different thread, when we would really like it to be aborted now and it is suspended at a safe-point. - */ struct stm_priv_segment_info_s *pseg = get_priv_segment(segment_num); - long remote_num = !segment_num; char *local_base = get_segment_base(segment_num); - char *remote_base = get_segment_base(remote_num); + char *remote_base = get_segment_base(0); LIST_FOREACH_R( pseg->modified_old_objects, diff --git a/rpython/translator/stm/src_stm/stm/fprintcolor.c b/rpython/translator/stm/src_stm/stm/fprintcolor.c --- a/rpython/translator/stm/src_stm/stm/fprintcolor.c +++ b/rpython/translator/stm/src_stm/stm/fprintcolor.c @@ -9,8 +9,8 @@ char buffer[2048]; va_list ap; int result; - int size = (int)sprintf(buffer, "\033[%dm[%lx] ", dprintfcolor(), - (long)pthread_self()); + int size = (int)sprintf(buffer, "\033[%dm[%d,%lx] ", dprintfcolor(), + (int)getpid(), (long)pthread_self()); assert(size >= 0); va_start(ap, format); @@ -42,6 +42,7 @@ va_start(ap, format); vfprintf(stderr, format, ap); + fprintf(stderr, "\n"); va_end(ap); abort(); diff --git a/rpython/translator/stm/src_stm/stm/gcpage.c b/rpython/translator/stm/src_stm/stm/gcpage.c --- a/rpython/translator/stm/src_stm/stm/gcpage.c +++ b/rpython/translator/stm/src_stm/stm/gcpage.c @@ -61,7 +61,7 @@ return; out_of_memory: - stm_fatalerror("out of memory!\n"); /* XXX */ + stm_fatalerror("out of memory!"); /* XXX */ } static char *_allocate_small_slowpath(uint64_t size) @@ -86,7 +86,7 @@ /* Allocate the object with largemalloc.c from the lower addresses. */ char *addr = _stm_large_malloc(size); if (addr == NULL) - stm_fatalerror("not enough memory!\n"); + stm_fatalerror("not enough memory!"); if (addr + size > uninitialized_page_start) { uintptr_t npages; @@ -94,7 +94,7 @@ npages += GCPAGE_NUM_PAGES; if (uninitialized_page_stop - uninitialized_page_start < npages * 4096UL) { - stm_fatalerror("out of memory!\n"); /* XXX */ + stm_fatalerror("out of memory!"); /* XXX */ } setup_N_pages(uninitialized_page_start, npages); uninitialized_page_start += npages * 4096UL; diff --git a/rpython/translator/stm/src_stm/stm/largemalloc.c b/rpython/translator/stm/src_stm/stm/largemalloc.c --- a/rpython/translator/stm/src_stm/stm/largemalloc.c +++ b/rpython/translator/stm/src_stm/stm/largemalloc.c @@ -274,8 +274,10 @@ /* unlink the following chunk */ mscan->d.next->prev = mscan->d.prev; mscan->d.prev->next = mscan->d.next; - assert((mscan->prev_size = (size_t)-258, 1)); /* 0xfffffffffffffefe */ - assert((mscan->size = (size_t)-515, 1)); /* 0xfffffffffffffdfd */ +#ifndef NDEBUG + mscan->prev_size = (size_t)-258; /* 0xfffffffffffffefe */ + mscan->size = (size_t)-515; /* 0xfffffffffffffdfd */ +#endif /* merge the two chunks */ assert(fsize == fscan->prev_size); diff --git a/rpython/translator/stm/src_stm/stm/list.c b/rpython/translator/stm/src_stm/stm/list.c --- a/rpython/translator/stm/src_stm/stm/list.c +++ b/rpython/translator/stm/src_stm/stm/list.c @@ -13,7 +13,7 @@ uintptr_t initial_allocation = 32; struct list_s *lst = malloc(LIST_SETSIZE(initial_allocation)); if (lst == NULL) - stm_fatalerror("out of memory in list_create\n"); /* XXX */ + stm_fatalerror("out of memory in list_create"); /* XXX */ lst->count = 0; lst->last_allocated = initial_allocation - 1; @@ -25,7 +25,7 @@ nalloc = LIST_OVERCNT(nalloc); lst = realloc(lst, LIST_SETSIZE(nalloc)); if (lst == NULL) - stm_fatalerror("out of memory in _list_grow\n"); /* XXX */ + stm_fatalerror("out of memory in _list_grow"); /* XXX */ lst->last_allocated = nalloc - 1; return lst; @@ -94,7 +94,7 @@ //fprintf(stderr, "growth: %ld\n", newalloc); char *newitems = malloc(newalloc); if (newitems == NULL) { - stm_fatalerror("out of memory!\n"); /* XXX */ + stm_fatalerror("out of memory!"); /* XXX */ } newtree.raw_start = newitems; newtree.raw_current = newitems; diff --git a/rpython/translator/stm/src_stm/stm/nursery.c b/rpython/translator/stm/src_stm/stm/nursery.c --- a/rpython/translator/stm/src_stm/stm/nursery.c +++ b/rpython/translator/stm/src_stm/stm/nursery.c @@ -33,10 +33,6 @@ } } -static void teardown_nursery(void) -{ -} - static inline bool _is_in_nursery(object_t *obj) { assert((uintptr_t)obj >= NURSERY_START); diff --git a/rpython/translator/stm/src_stm/stm/pages.c b/rpython/translator/stm/src_stm/stm/pages.c --- a/rpython/translator/stm/src_stm/stm/pages.c +++ b/rpython/translator/stm/src_stm/stm/pages.c @@ -94,7 +94,7 @@ int res = remap_file_pages(addr, size, 0, pgoff, 0); if (UNLIKELY(res < 0)) - stm_fatalerror("remap_file_pages: %m\n"); + stm_fatalerror("remap_file_pages: %m"); } static void pages_initialize_shared(uintptr_t pagenum, uintptr_t count) @@ -104,6 +104,8 @@ segment 0. */ uintptr_t i; assert(_has_mutex_pages()); + if (count == 0) + return; for (i = 1; i <= NB_SEGMENTS; i++) { char *segment_base = get_segment_base(i); d_remap_file_pages(segment_base + pagenum * 4096UL, @@ -141,6 +143,13 @@ mutex_pages_unlock(); } +static void _page_do_reshare(long segnum, uintptr_t pagenum) +{ + char *segment_base = get_segment_base(segnum); + d_remap_file_pages(segment_base + pagenum * 4096UL, + 4096, pagenum); +} + static void page_reshare(uintptr_t pagenum) { struct page_shared_s ps = pages_privatized[pagenum - PAGE_FLAG_START]; @@ -150,7 +159,7 @@ for (j = 0; j < NB_SEGMENTS; j++) { if (ps.by_segment & (1 << j)) { /* Page 'pagenum' is private in segment 'j + 1'. Reshare */ - char *segment_base = stm_object_pages + NB_PAGES * 4096UL * (j+1); + char *segment_base = get_segment_base(j + 1); madvise(segment_base + pagenum * 4096UL, 4096, MADV_DONTNEED); d_remap_file_pages(segment_base + pagenum * 4096UL, diff --git a/rpython/translator/stm/src_stm/stm/pages.h b/rpython/translator/stm/src_stm/stm/pages.h --- a/rpython/translator/stm/src_stm/stm/pages.h +++ b/rpython/translator/stm/src_stm/stm/pages.h @@ -39,7 +39,9 @@ static void pages_initialize_shared(uintptr_t pagenum, uintptr_t count); static void page_privatize(uintptr_t pagenum); static void page_reshare(uintptr_t pagenum); +static void _page_do_reshare(long segnum, uintptr_t pagenum); +/* Note: don't ever do "mutex_pages_lock(); mutex_lock()" in that order */ static void mutex_pages_lock(void); static void mutex_pages_unlock(void); static bool _has_mutex_pages(void) __attribute__((unused)); diff --git a/rpython/translator/stm/src_stm/stm/setup.c b/rpython/translator/stm/src_stm/stm/setup.c --- a/rpython/translator/stm/src_stm/stm/setup.c +++ b/rpython/translator/stm/src_stm/stm/setup.c @@ -4,6 +4,17 @@ #endif +static char *setup_mmap(char *reason) +{ + char *result = mmap(NULL, TOTAL_MEMORY, + PROT_READ | PROT_WRITE, + MAP_PAGES_FLAGS, -1, 0); + if (result == MAP_FAILED) + stm_fatalerror("%s failed: %m\n", reason); + + return result; +} + void stm_setup(void) { /* Check that some values are acceptable */ @@ -21,13 +32,9 @@ (FIRST_READMARKER_PAGE * 4096UL)); assert(_STM_FAST_ALLOC <= NB_NURSERY_PAGES * 4096); - stm_object_pages = mmap(NULL, TOTAL_MEMORY, - PROT_READ | PROT_WRITE, - MAP_PAGES_FLAGS, -1, 0); - if (stm_object_pages == MAP_FAILED) - stm_fatalerror("initial stm_object_pages mmap() failed: %m\n"); + stm_object_pages = setup_mmap("initial stm_object_pages mmap()"); - /* The segment 0 is not used to run transactions, but to contain the + /* The segment 0 is not used to run transactions, but contains the shared copy of the pages. We mprotect all pages before so that accesses fail, up to and including the pages corresponding to the nurseries of the other segments. */ @@ -84,6 +91,7 @@ setup_nursery(); setup_gcpage(); setup_pages(); + setup_forksupport(); } void stm_teardown(void) @@ -111,11 +119,10 @@ teardown_core(); teardown_sync(); teardown_gcpage(); - teardown_nursery(); teardown_pages(); } -void _init_shadow_stack(stm_thread_local_t *tl) +static void _init_shadow_stack(stm_thread_local_t *tl) { struct stm_shadowentry_s *s = (struct stm_shadowentry_s *) malloc(SHADOW_STACK_SIZE * sizeof(struct stm_shadowentry_s)); @@ -124,13 +131,18 @@ tl->shadowstack_base = s; } -void _done_shadow_stack(stm_thread_local_t *tl) +static void _done_shadow_stack(stm_thread_local_t *tl) { free(tl->shadowstack_base); tl->shadowstack = NULL; tl->shadowstack_base = NULL; } +static pthread_t *_get_cpth(stm_thread_local_t *tl) +{ + assert(sizeof(pthread_t) <= sizeof(tl->creating_pthread)); + return (pthread_t *)(tl->creating_pthread); +} void stm_register_thread_local(stm_thread_local_t *tl) { @@ -154,6 +166,7 @@ numbers automatically. */ num = (num % NB_SEGMENTS) + 1; tl->associated_segment_num = num; + *_get_cpth(tl) = pthread_self(); _init_shadow_stack(tl); set_gs_register(get_segment_base(num)); s_mutex_unlock(); @@ -162,6 +175,7 @@ void stm_unregister_thread_local(stm_thread_local_t *tl) { s_mutex_lock(); + assert(tl->prev != NULL); assert(tl->next != NULL); _done_shadow_stack(tl); if (tl == stm_all_thread_locals) { diff --git a/rpython/translator/stm/src_stm/stm/sync.c b/rpython/translator/stm/src_stm/stm/sync.c --- a/rpython/translator/stm/src_stm/stm/sync.c +++ b/rpython/translator/stm/src_stm/stm/sync.c @@ -41,24 +41,24 @@ static void setup_sync(void) { if (pthread_mutex_init(&sync_ctl.global_mutex, NULL) != 0) - stm_fatalerror("mutex initialization: %m\n"); + stm_fatalerror("mutex initialization: %m"); long i; for (i = 0; i < _C_TOTAL; i++) { if (pthread_cond_init(&sync_ctl.cond[i], NULL) != 0) - stm_fatalerror("cond initialization: %m\n"); + stm_fatalerror("cond initialization: %m"); } } static void teardown_sync(void) { if (pthread_mutex_destroy(&sync_ctl.global_mutex) != 0) - stm_fatalerror("mutex destroy: %m\n"); + stm_fatalerror("mutex destroy: %m"); long i; for (i = 0; i < _C_TOTAL; i++) { if (pthread_cond_destroy(&sync_ctl.cond[i]) != 0) - stm_fatalerror("cond destroy: %m\n"); + stm_fatalerror("cond destroy: %m"); } memset(&sync_ctl, 0, sizeof(sync_ctl)); @@ -75,14 +75,14 @@ static void set_gs_register(char *value) { if (UNLIKELY(syscall(SYS_arch_prctl, ARCH_SET_GS, (uint64_t)value) != 0)) - stm_fatalerror("syscall(arch_prctl, ARCH_SET_GS): %m\n"); + stm_fatalerror("syscall(arch_prctl, ARCH_SET_GS): %m"); } static inline void s_mutex_lock(void) { assert(!_has_mutex_here); if (UNLIKELY(pthread_mutex_lock(&sync_ctl.global_mutex) != 0)) - stm_fatalerror("pthread_mutex_lock: %m\n"); + stm_fatalerror("pthread_mutex_lock: %m"); assert((_has_mutex_here = true, 1)); } @@ -90,32 +90,32 @@ { assert(_has_mutex_here); if (UNLIKELY(pthread_mutex_unlock(&sync_ctl.global_mutex) != 0)) - stm_fatalerror("pthread_mutex_unlock: %m\n"); + stm_fatalerror("pthread_mutex_unlock: %m"); assert((_has_mutex_here = false, 1)); } static inline void cond_wait(enum cond_type_e ctype) { #ifdef STM_NO_COND_WAIT - stm_fatalerror("*** cond_wait/%d called!\n", (int)ctype); + stm_fatalerror("*** cond_wait/%d called!", (int)ctype); #endif assert(_has_mutex_here); if (UNLIKELY(pthread_cond_wait(&sync_ctl.cond[ctype], &sync_ctl.global_mutex) != 0)) - stm_fatalerror("pthread_cond_wait/%d: %m\n", (int)ctype); + stm_fatalerror("pthread_cond_wait/%d: %m", (int)ctype); } static inline void cond_signal(enum cond_type_e ctype) { if (UNLIKELY(pthread_cond_signal(&sync_ctl.cond[ctype]) != 0)) - stm_fatalerror("pthread_cond_signal/%d: %m\n", (int)ctype); + stm_fatalerror("pthread_cond_signal/%d: %m", (int)ctype); } static inline void cond_broadcast(enum cond_type_e ctype) { if (UNLIKELY(pthread_cond_broadcast(&sync_ctl.cond[ctype]) != 0)) - stm_fatalerror("pthread_cond_broadcast/%d: %m\n", (int)ctype); + stm_fatalerror("pthread_cond_broadcast/%d: %m", (int)ctype); } /************************************************************/ diff --git a/rpython/translator/stm/src_stm/stmgc.c b/rpython/translator/stm/src_stm/stmgc.c --- a/rpython/translator/stm/src_stm/stmgc.c +++ b/rpython/translator/stm/src_stm/stmgc.c @@ -24,6 +24,7 @@ #include "stm/largemalloc.c" #include "stm/nursery.c" #include "stm/sync.c" +#include "stm/forksupport.c" #include "stm/setup.c" #include "stm/hash_id.c" #include "stm/core.c" diff --git a/rpython/translator/stm/src_stm/stmgc.h b/rpython/translator/stm/src_stm/stmgc.h --- a/rpython/translator/stm/src_stm/stmgc.h +++ b/rpython/translator/stm/src_stm/stmgc.h @@ -70,6 +70,7 @@ /* the next fields are handled internally by the library */ int associated_segment_num; struct stm_thread_local_s *prev, *next; + void *creating_pthread[2]; } stm_thread_local_t; /* this should use llvm's coldcc calling convention, @@ -130,8 +131,10 @@ /* ==================== PUBLIC API ==================== */ -/* Number of segments (i.e. how many threads can be executed in - parallel, in maximum). +/* Number of segments (i.e. how many transactions can be executed in + parallel, in maximum). If you try to start transactions in more + threads than the number of segments, it will block, waiting for the + next segment to become free. */ #define STM_NB_SEGMENTS 4 From noreply at buildbot.pypy.org Wed Mar 19 09:23:21 2014 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 19 Mar 2014 09:23:21 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Convenience Message-ID: <20140319082321.E3CB21C01DE@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r70086:9b2f37131a9b Date: 2014-03-19 09:22 +0100 http://bitbucket.org/pypy/pypy/changeset/9b2f37131a9b/ Log: Convenience diff --git a/rpython/translator/stm/import_stmgc.py b/rpython/translator/stm/import_stmgc.py --- a/rpython/translator/stm/import_stmgc.py +++ b/rpython/translator/stm/import_stmgc.py @@ -37,6 +37,7 @@ path.chmod(0444) # stmgc_dest.join('revision').write('%s\n' % rev) + print rev if __name__ == '__main__': if len(sys.argv) != 2: From noreply at buildbot.pypy.org Wed Mar 19 10:14:55 2014 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 19 Mar 2014 10:14:55 +0100 (CET) Subject: [pypy-commit] cffi default: (Lisandro Dalcin, part of issue 141) Message-ID: <20140319091455.61EF91D2511@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1482:56f591248800 Date: 2014-03-19 10:14 +0100 http://bitbucket.org/cffi/cffi/changeset/56f591248800/ Log: (Lisandro Dalcin, part of issue 141) Check for "__thread" by asking distutils specifically for the same compiler as the one it will use later. diff --git a/c/check__thread.c b/c/check__thread.c deleted file mode 100644 --- a/c/check__thread.c +++ /dev/null @@ -1,1 +0,0 @@ -__thread int some_threadlocal_variable_42; diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -42,25 +42,14 @@ resultlist[:] = res def ask_supports_thread(): - if sys.platform == "darwin": - sys.stderr.write("Note: will not use '__thread' in the C code\n") - sys.stderr.write("This is for OS/X-specific reasons: confusion " - "between 'cc' versus 'gcc' (see issue 123)\n") - return - import distutils.errors - from distutils.ccompiler import new_compiler - compiler = new_compiler(force=1) - try: - compiler.compile(['c/check__thread.c']) - except distutils.errors.CompileError: + from distutils.core import Distribution + config = Distribution().get_command_obj('config') + ok = config.try_compile('__thread int some_threadlocal_variable_42;') + if ok: + define_macros.append(('USE__THREAD', None)) + else: sys.stderr.write("Note: will not use '__thread' in the C code\n") sys.stderr.write("The above error message can be safely ignored\n") - else: - define_macros.append(('USE__THREAD', None)) - try: - os.unlink('c/check__thread.o') - except OSError: - pass def use_pkg_config(): _ask_pkg_config(include_dirs, '--cflags-only-I', '-I', sysroot=True) From noreply at buildbot.pypy.org Wed Mar 19 10:21:09 2014 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 19 Mar 2014 10:21:09 +0100 (CET) Subject: [pypy-commit] stmgc c7-fork: Close branch ready for merging Message-ID: <20140319092109.7DC7A1D2511@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c7-fork Changeset: r1077:ede4b4739ad4 Date: 2014-03-19 10:20 +0100 http://bitbucket.org/pypy/stmgc/changeset/ede4b4739ad4/ Log: Close branch ready for merging From noreply at buildbot.pypy.org Wed Mar 19 10:21:11 2014 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 19 Mar 2014 10:21:11 +0100 (CET) Subject: [pypy-commit] stmgc default: hg merge c7-fork Message-ID: <20140319092111.0F49F1D2511@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1078:0e7ff5304cac Date: 2014-03-19 10:20 +0100 http://bitbucket.org/pypy/stmgc/changeset/0e7ff5304cac/ Log: hg merge c7-fork Add pthread_atfork() to manually de-share the mapping, mounted with MAP_SHARED. diff --git a/c7/demo/demo_random.c b/c7/demo/demo_random.c --- a/c7/demo/demo_random.c +++ b/c7/demo/demo_random.c @@ -4,6 +4,8 @@ #include #include #include +#include +#include #include "stmgc.h" @@ -12,12 +14,14 @@ #define THREAD_STARTS 1000 // how many restarts of threads #define PREBUILT_ROOTS 3 #define MAXROOTS 1000 +#define FORKS 3 // SUPPORT struct node_s; typedef TLPREFIX struct node_s node_t; typedef node_t* nodeptr_t; typedef object_t* objptr_t; +int num_forked_children = 0; struct node_s { struct object_s hdr; @@ -335,19 +339,39 @@ if (p == (objptr_t)-1) { push_roots(); - stm_commit_transaction(); - td.num_roots_at_transaction_start = td.num_roots; + if (arg == NULL) { /* common case */ + stm_commit_transaction(); + td.num_roots_at_transaction_start = td.num_roots; + if (get_rand(100) < 98) { + STM_START_TRANSACTION(&stm_thread_local, here); + } else { + stm_start_inevitable_transaction(&stm_thread_local); + } + td.num_roots = td.num_roots_at_transaction_start; + p = NULL; + pop_roots(); + reload_roots(); + } + else { + /* run a fork() inside the transaction */ + printf("========== FORK =========\n"); + arg = NULL; + pid_t child = fork(); + printf("=== in process %d thread %lx, fork() returned %d\n", + (int)getpid(), (long)pthread_self(), (int)child); + if (child == -1) { + fprintf(stderr, "fork() error: %m\n"); + abort(); + } + if (child != 0) + num_forked_children++; + else + num_forked_children = 0; - if (get_rand(100) < 98) { - STM_START_TRANSACTION(&stm_thread_local, here); - } else { - stm_start_inevitable_transaction(&stm_thread_local); + pop_roots(); + p = NULL; } - td.num_roots = td.num_roots_at_transaction_start; - p = NULL; - pop_roots(); - reload_roots(); } } stm_commit_transaction(); @@ -427,8 +451,24 @@ assert(status == 0); printf("thread finished\n"); if (thread_starts) { + long forkbase = NUMTHREADS * THREAD_STARTS / (FORKS + 1); + long _fork = (thread_starts % forkbase) == 0; thread_starts--; - newthread(demo_random, NULL); + newthread(demo_random, (void *)_fork); + } + } + + for (i = 0; i < num_forked_children; i++) { + pid_t child = wait(&status); + if (child == -1) + perror("wait"); + printf("From %d: child %d terminated with exit status %d\n", + (int)getpid(), (int)child, status); + if (WIFEXITED(status) && WEXITSTATUS(status) == 0) + ; + else { + printf("*** error from the child ***\n"); + return 1; } } diff --git a/c7/stm/forksupport.c b/c7/stm/forksupport.c new file mode 100644 --- /dev/null +++ b/c7/stm/forksupport.c @@ -0,0 +1,295 @@ +#ifndef _STM_CORE_H_ +# error "must be compiled via stmgc.c" +#endif + + +/* XXX this is currently not doing copy-on-write, but simply forces a + copy of all pages as soon as fork() is called. */ + + +static char *fork_big_copy = NULL; +static stm_thread_local_t *fork_this_tl; +static bool fork_was_in_transaction; + +static char *setup_mmap(char *reason); /* forward, in setup.c */ +static pthread_t *_get_cpth(stm_thread_local_t *);/* forward, in setup.c */ + + +static bool page_is_null(char *p) +{ + long *q = (long *)p; + long i; + for (i = 0; i < 4096 / sizeof(long); i++) + if (q[i] != 0) + return false; + return true; +} + + +static void forksupport_prepare(void) +{ + if (stm_object_pages == NULL) + return; + + /* So far we attempt to check this by walking all stm_thread_local_t, + marking the one from the current thread, and verifying that it's not + running a transaction. This assumes that the stm_thread_local_t is just + a __thread variable, so never changes threads. + */ + s_mutex_lock(); + + dprintf(("forksupport_prepare\n")); + + stm_thread_local_t *this_tl = NULL; + stm_thread_local_t *tl = stm_all_thread_locals; + do { + if (pthread_equal(*_get_cpth(tl), pthread_self())) { + if (this_tl != NULL) + stm_fatalerror("fork(): found several stm_thread_local_t" + " from the same thread"); + this_tl = tl; + } + tl = tl->next; + } while (tl != stm_all_thread_locals); + + if (this_tl == NULL) + stm_fatalerror("fork(): found no stm_thread_local_t from this thread"); + s_mutex_unlock(); + + bool was_in_transaction = _stm_in_transaction(this_tl); + if (was_in_transaction) { + stm_become_inevitable("fork"); + /* Note that the line above can still fail and abort, which should + be fine */ + } + else { + stm_start_inevitable_transaction(this_tl); + } + + s_mutex_lock(); + synchronize_all_threads(); + mutex_pages_lock(); + + /* Make a new mmap at some other address, but of the same size as + the standard mmap at stm_object_pages + */ + char *big_copy = setup_mmap("stmgc's fork support"); + + /* Copy each of the segment infos into the new mmap, nurseries, + and associated read markers + */ + long i; + for (i = 1; i <= NB_SEGMENTS; i++) { + char *src, *dst; + struct stm_priv_segment_info_s *psrc = get_priv_segment(i); + dst = big_copy + (((char *)psrc) - stm_object_pages); + *(struct stm_priv_segment_info_s *)dst = *psrc; + + src = get_segment_base(i) + FIRST_READMARKER_PAGE * 4096UL; + dst = big_copy + (src - stm_object_pages); + long j; + for (j = 0; j < END_NURSERY_PAGE - FIRST_READMARKER_PAGE; j++) { + if (!page_is_null(src)) + pagecopy(dst, src); + src += 4096; + dst += 4096; + } + } + + /* Copy all the data from the two ranges of objects (large, small) + into the new mmap + */ + uintptr_t pagenum, endpagenum; + pagenum = END_NURSERY_PAGE; /* starts after the nursery */ + endpagenum = (uninitialized_page_start - stm_object_pages) / 4096UL; + if (endpagenum < NB_PAGES) + endpagenum++; /* the next page too, because it might contain + data from largemalloc */ + + while (1) { + if (UNLIKELY(pagenum == endpagenum)) { + /* we reach this point usually twice, because there are + more pages after 'uninitialized_page_stop' */ + if (endpagenum == NB_PAGES) + break; /* done */ + pagenum = (uninitialized_page_stop - stm_object_pages) / 4096UL; + pagenum--; /* the prev page too, because it does contain + data from largemalloc */ + endpagenum = NB_PAGES; + } + + char *src = stm_object_pages + pagenum * 4096UL; + char *dst = big_copy + pagenum * 4096UL; + pagecopy(dst, src); + + struct page_shared_s ps = pages_privatized[pagenum - PAGE_FLAG_START]; + if (ps.by_segment != 0) { + long j; + for (j = 0; j < NB_SEGMENTS; j++) { + src += NB_PAGES * 4096UL; + dst += NB_PAGES * 4096UL; + if (ps.by_segment & (1 << j)) { + pagecopy(dst, src); + } + } + } + pagenum++; + } + + assert(fork_big_copy == NULL); + fork_big_copy = big_copy; + fork_this_tl = this_tl; + fork_was_in_transaction = was_in_transaction; + + assert(_has_mutex()); + dprintf(("forksupport_prepare: from %p %p\n", fork_this_tl, + fork_this_tl->creating_pthread[0])); +} + +static void forksupport_parent(void) +{ + if (stm_object_pages == NULL) + return; + + dprintf(("forksupport_parent: continuing to run %p %p\n", fork_this_tl, + fork_this_tl->creating_pthread[0])); + assert(_has_mutex()); + assert(_is_tl_registered(fork_this_tl)); + + /* In the parent, after fork(), we can simply forget about the big copy + that we made for the child. + */ + assert(fork_big_copy != NULL); + munmap(fork_big_copy, TOTAL_MEMORY); + fork_big_copy = NULL; + bool was_in_transaction = fork_was_in_transaction; + + mutex_pages_unlock(); + s_mutex_unlock(); + + if (!was_in_transaction) { + stm_commit_transaction(); + } + + dprintf(("forksupport_parent: continuing to run\n")); +} + +static void fork_abort_thread(long i) +{ + struct stm_priv_segment_info_s *pr = get_priv_segment(i); + dprintf(("forksupport_child: abort in seg%ld\n", i)); + assert(pr->pub.running_thread->associated_segment_num == i); + assert(pr->transaction_state == TS_REGULAR); + set_gs_register(get_segment_base(i)); + + stm_jmpbuf_t jmpbuf; + if (__builtin_setjmp(jmpbuf) == 0) { + pr->pub.jmpbuf_ptr = &jmpbuf; +#ifndef NDEBUG + pr->running_pthread = pthread_self(); +#endif + stm_abort_transaction(); + } +} + +static void forksupport_child(void) +{ + if (stm_object_pages == NULL) + return; + + /* this new process contains no other thread, so we can + just release these locks early */ + mutex_pages_unlock(); + s_mutex_unlock(); + + /* Move the copy of the mmap over the old one, overwriting it + and thus freeing the old mapping in this process + */ + assert(fork_big_copy != NULL); + assert(stm_object_pages != NULL); + void *res = mremap(fork_big_copy, TOTAL_MEMORY, TOTAL_MEMORY, + MREMAP_MAYMOVE | MREMAP_FIXED, + stm_object_pages); + if (res != stm_object_pages) + stm_fatalerror("after fork: mremap failed: %m"); + fork_big_copy = NULL; + + /* Unregister all other stm_thread_local_t, mostly as a way to free + the memory used by the shadowstacks + */ + while (stm_all_thread_locals->next != stm_all_thread_locals) { + if (stm_all_thread_locals == fork_this_tl) + stm_unregister_thread_local(stm_all_thread_locals->next); + else + stm_unregister_thread_local(stm_all_thread_locals); + } + assert(stm_all_thread_locals == fork_this_tl); + + /* Make all pages shared again. + */ + uintptr_t pagenum, endpagenum; + pagenum = END_NURSERY_PAGE; /* starts after the nursery */ + endpagenum = (uninitialized_page_start - stm_object_pages) / 4096UL; + + while (1) { + if (UNLIKELY(pagenum == endpagenum)) { + /* we reach this point usually twice, because there are + more pages after 'uninitialized_page_stop' */ + if (endpagenum == NB_PAGES) + break; /* done */ + pagenum = (uninitialized_page_stop - stm_object_pages) / 4096UL; + endpagenum = NB_PAGES; + if (endpagenum == NB_PAGES) + break; /* done */ + } + + struct page_shared_s ps = pages_privatized[pagenum - PAGE_FLAG_START]; + long j; + for (j = 0; j < NB_SEGMENTS; j++) { + if (!(ps.by_segment & (1 << j))) { + _page_do_reshare(j + 1, pagenum); + } + } + pagenum++; + } + + /* Force the interruption of other running segments + */ + long i; + for (i = 1; i <= NB_SEGMENTS; i++) { + struct stm_priv_segment_info_s *pr = get_priv_segment(i); + if (pr->pub.running_thread != NULL && + pr->pub.running_thread != fork_this_tl) { + fork_abort_thread(i); + } + } + + /* Restore a few things: the new pthread_self(), and the %gs + register */ + int segnum = fork_this_tl->associated_segment_num; + assert(1 <= segnum && segnum <= NB_SEGMENTS); + *_get_cpth(fork_this_tl) = pthread_self(); + set_gs_register(get_segment_base(segnum)); + assert(STM_SEGMENT->segment_num == segnum); + + if (!fork_was_in_transaction) { + stm_commit_transaction(); + } + + /* Done */ + dprintf(("forksupport_child: running one thread now\n")); +} + + +static void setup_forksupport(void) +{ + static bool fork_support_ready = false; + + if (!fork_support_ready) { + int res = pthread_atfork(forksupport_prepare, forksupport_parent, + forksupport_child); + if (res != 0) + stm_fatalerror("pthread_atfork() failed: %m"); + fork_support_ready = true; + } +} diff --git a/c7/stm/fprintcolor.c b/c7/stm/fprintcolor.c --- a/c7/stm/fprintcolor.c +++ b/c7/stm/fprintcolor.c @@ -8,8 +8,8 @@ char buffer[2048]; va_list ap; int result; - int size = (int)sprintf(buffer, "\033[%dm[%lx] ", dprintfcolor(), - (long)pthread_self()); + int size = (int)sprintf(buffer, "\033[%dm[%d,%lx] ", dprintfcolor(), + (int)getpid(), (long)pthread_self()); assert(size >= 0); va_start(ap, format); diff --git a/c7/stm/largemalloc.c b/c7/stm/largemalloc.c --- a/c7/stm/largemalloc.c +++ b/c7/stm/largemalloc.c @@ -273,8 +273,10 @@ /* unlink the following chunk */ mscan->d.next->prev = mscan->d.prev; mscan->d.prev->next = mscan->d.next; - assert((mscan->prev_size = (size_t)-258, 1)); /* 0xfffffffffffffefe */ - assert((mscan->size = (size_t)-515, 1)); /* 0xfffffffffffffdfd */ +#ifndef NDEBUG + mscan->prev_size = (size_t)-258; /* 0xfffffffffffffefe */ + mscan->size = (size_t)-515; /* 0xfffffffffffffdfd */ +#endif /* merge the two chunks */ assert(fsize == fscan->prev_size); diff --git a/c7/stm/nursery.c b/c7/stm/nursery.c --- a/c7/stm/nursery.c +++ b/c7/stm/nursery.c @@ -32,10 +32,6 @@ } } -static void teardown_nursery(void) -{ -} - static inline bool _is_in_nursery(object_t *obj) { assert((uintptr_t)obj >= NURSERY_START); diff --git a/c7/stm/pages.c b/c7/stm/pages.c --- a/c7/stm/pages.c +++ b/c7/stm/pages.c @@ -103,6 +103,8 @@ segment 0. */ uintptr_t i; assert(_has_mutex_pages()); + if (count == 0) + return; for (i = 1; i <= NB_SEGMENTS; i++) { char *segment_base = get_segment_base(i); d_remap_file_pages(segment_base + pagenum * 4096UL, @@ -140,6 +142,13 @@ mutex_pages_unlock(); } +static void _page_do_reshare(long segnum, uintptr_t pagenum) +{ + char *segment_base = get_segment_base(segnum); + d_remap_file_pages(segment_base + pagenum * 4096UL, + 4096, pagenum); +} + static void page_reshare(uintptr_t pagenum) { struct page_shared_s ps = pages_privatized[pagenum - PAGE_FLAG_START]; diff --git a/c7/stm/pages.h b/c7/stm/pages.h --- a/c7/stm/pages.h +++ b/c7/stm/pages.h @@ -38,7 +38,9 @@ static void pages_initialize_shared(uintptr_t pagenum, uintptr_t count); static void page_privatize(uintptr_t pagenum); static void page_reshare(uintptr_t pagenum); +static void _page_do_reshare(long segnum, uintptr_t pagenum); +/* Note: don't ever do "mutex_pages_lock(); mutex_lock()" in that order */ static void mutex_pages_lock(void); static void mutex_pages_unlock(void); static bool _has_mutex_pages(void) __attribute__((unused)); diff --git a/c7/stm/setup.c b/c7/stm/setup.c --- a/c7/stm/setup.c +++ b/c7/stm/setup.c @@ -3,6 +3,17 @@ #endif +static char *setup_mmap(char *reason) +{ + char *result = mmap(NULL, TOTAL_MEMORY, + PROT_READ | PROT_WRITE, + MAP_PAGES_FLAGS, -1, 0); + if (result == MAP_FAILED) + stm_fatalerror("%s failed: %m\n", reason); + + return result; +} + void stm_setup(void) { /* Check that some values are acceptable */ @@ -20,13 +31,9 @@ (FIRST_READMARKER_PAGE * 4096UL)); assert(_STM_FAST_ALLOC <= NB_NURSERY_PAGES * 4096); - stm_object_pages = mmap(NULL, TOTAL_MEMORY, - PROT_READ | PROT_WRITE, - MAP_PAGES_FLAGS, -1, 0); - if (stm_object_pages == MAP_FAILED) - stm_fatalerror("initial stm_object_pages mmap() failed: %m"); + stm_object_pages = setup_mmap("initial stm_object_pages mmap()"); - /* The segment 0 is not used to run transactions, but to contain the + /* The segment 0 is not used to run transactions, but contains the shared copy of the pages. We mprotect all pages before so that accesses fail, up to and including the pages corresponding to the nurseries of the other segments. */ @@ -83,6 +90,7 @@ setup_nursery(); setup_gcpage(); setup_pages(); + setup_forksupport(); } void stm_teardown(void) @@ -110,11 +118,10 @@ teardown_core(); teardown_sync(); teardown_gcpage(); - teardown_nursery(); teardown_pages(); } -void _init_shadow_stack(stm_thread_local_t *tl) +static void _init_shadow_stack(stm_thread_local_t *tl) { struct stm_shadowentry_s *s = (struct stm_shadowentry_s *) malloc(SHADOW_STACK_SIZE * sizeof(struct stm_shadowentry_s)); @@ -123,13 +130,18 @@ tl->shadowstack_base = s; } -void _done_shadow_stack(stm_thread_local_t *tl) +static void _done_shadow_stack(stm_thread_local_t *tl) { free(tl->shadowstack_base); tl->shadowstack = NULL; tl->shadowstack_base = NULL; } +static pthread_t *_get_cpth(stm_thread_local_t *tl) +{ + assert(sizeof(pthread_t) <= sizeof(tl->creating_pthread)); + return (pthread_t *)(tl->creating_pthread); +} void stm_register_thread_local(stm_thread_local_t *tl) { @@ -153,6 +165,7 @@ numbers automatically. */ num = (num % NB_SEGMENTS) + 1; tl->associated_segment_num = num; + *_get_cpth(tl) = pthread_self(); _init_shadow_stack(tl); set_gs_register(get_segment_base(num)); s_mutex_unlock(); @@ -161,6 +174,7 @@ void stm_unregister_thread_local(stm_thread_local_t *tl) { s_mutex_lock(); + assert(tl->prev != NULL); assert(tl->next != NULL); _done_shadow_stack(tl); if (tl == stm_all_thread_locals) { diff --git a/c7/stmgc.c b/c7/stmgc.c --- a/c7/stmgc.c +++ b/c7/stmgc.c @@ -23,6 +23,7 @@ #include "stm/largemalloc.c" #include "stm/nursery.c" #include "stm/sync.c" +#include "stm/forksupport.c" #include "stm/setup.c" #include "stm/hash_id.c" #include "stm/core.c" diff --git a/c7/stmgc.h b/c7/stmgc.h --- a/c7/stmgc.h +++ b/c7/stmgc.h @@ -69,6 +69,7 @@ /* the next fields are handled internally by the library */ int associated_segment_num; struct stm_thread_local_s *prev, *next; + void *creating_pthread[2]; } stm_thread_local_t; /* this should use llvm's coldcc calling convention, From noreply at buildbot.pypy.org Wed Mar 19 10:51:24 2014 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 19 Mar 2014 10:51:24 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: tweaks Message-ID: <20140319095124.6CDC21C01DE@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r70087:9846f4e476df Date: 2014-03-19 10:50 +0100 http://bitbucket.org/pypy/pypy/changeset/9846f4e476df/ Log: tweaks diff --git a/rpython/memory/gctransform/stmframework.py b/rpython/memory/gctransform/stmframework.py --- a/rpython/memory/gctransform/stmframework.py +++ b/rpython/memory/gctransform/stmframework.py @@ -71,7 +71,12 @@ v_struct = hop.spaceop.args[0] assert opname in ('setfield', 'setarrayitem', 'setinteriorfield', 'raw_store') - if var_needsgc(v_struct) and hop.spaceop not in self.clean_sets: + if not var_needsgc(v_struct): + if (var_needsgc(hop.spaceop.args[-1]) and + 'is_excdata' not in hop.spaceop.args[0].concretetype.TO._hints): + raise Exception("%s: GC pointer written into a non-GC location" + % (hop.spaceop,)) + elif hop.spaceop not in self.clean_sets: if self.in_stm_ignored: # detect if we're inside a 'stm_ignored' block and in # that case don't call stm_write(). This only works for diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py --- a/rpython/translator/c/genc.py +++ b/rpython/translator/c/genc.py @@ -906,6 +906,13 @@ **cur = stm_setup_prebuilt(**cur); } } + +void pypy_stm_register_thread_local(void) +{ + stm_register_thread_local(&stm_thread_local); + stm_thread_local.mem_clear_on_abort = &pypy_g_ExcData; + stm_thread_local.mem_bytes_to_clear_on_abort = sizeof(pypy_g_ExcData); +} ''' def commondefs(defines): diff --git a/rpython/translator/exceptiontransform.py b/rpython/translator/exceptiontransform.py --- a/rpython/translator/exceptiontransform.py +++ b/rpython/translator/exceptiontransform.py @@ -464,7 +464,8 @@ ('exc_type', self.lltype_of_exception_type), ('exc_value', self.lltype_of_exception_value), hints={'stm_thread_local': True, - 'stm_dont_track_raw_accesses':True}) + 'stm_dont_track_raw_accesses':True, + 'is_excdata': True}) self.EXCDATA = EXCDATA exc_data = lltype.malloc(EXCDATA, immortal=True) diff --git a/rpython/translator/stm/funcgen.py b/rpython/translator/stm/funcgen.py --- a/rpython/translator/stm/funcgen.py +++ b/rpython/translator/stm/funcgen.py @@ -44,10 +44,7 @@ def stm_register_thread_local(funcgen, op): - return ( - 'stm_register_thread_local(&stm_thread_local);\n\t' - 'stm_thread_local.mem_clear_on_abort = &pypy_g_ExcData;\n\t' - 'stm_thread_local.mem_bytes_to_clear_on_abort = sizeof(pypy_g_ExcData);') + return 'pypy_stm_register_thread_local();' def stm_unregister_thread_local(funcgen, op): return 'stm_unregister_thread_local(&stm_thread_local);' diff --git a/rpython/translator/stm/src_stm/stmgcintf.c b/rpython/translator/stm/src_stm/stmgcintf.c --- a/rpython/translator/stm/src_stm/stmgcintf.c +++ b/rpython/translator/stm/src_stm/stmgcintf.c @@ -48,7 +48,7 @@ void pypy_stm_setup(void) { stm_setup(); - stm_register_thread_local(&stm_thread_local); + pypy_stm_register_thread_local(); pypy_stm_ready_atomic = 1; pypy_stm_set_transaction_length(100); pypy_stm_start_inevitable_if_not_atomic(); @@ -59,7 +59,7 @@ if (pypy_stm_ready_atomic == 0) { /* first time we see this thread */ int e = errno; - stm_register_thread_local(&stm_thread_local); + pypy_stm_register_thread_local(); errno = e; pypy_stm_ready_atomic = 1; pypy_stm_start_inevitable_if_not_atomic(); diff --git a/rpython/translator/stm/src_stm/stmgcintf.h b/rpython/translator/stm/src_stm/stmgcintf.h --- a/rpython/translator/stm/src_stm/stmgcintf.h +++ b/rpython/translator/stm/src_stm/stmgcintf.h @@ -13,7 +13,8 @@ extern __thread uintptr_t pypy_stm_nursery_low_fill_mark; void pypy_stm_setup(void); -void pypy_stm_setup_prebuilt(void); /* generated into stm_prebuilt.c */ +void pypy_stm_setup_prebuilt(void); /* generated into stm_prebuilt.c */ +void pypy_stm_register_thread_local(void); /* generated into stm_prebuilt.c */ static inline void pypy_stm_commit_if_not_atomic(void) { int e = errno; From noreply at buildbot.pypy.org Wed Mar 19 11:06:15 2014 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 19 Mar 2014 11:06:15 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Oups, forgot to add this. Message-ID: <20140319100615.8CEEA1C01DE@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r70088:b6ee514a4d42 Date: 2014-03-19 11:05 +0100 http://bitbucket.org/pypy/pypy/changeset/b6ee514a4d42/ Log: Oups, forgot to add this. diff --git a/rpython/translator/stm/src_stm/stm/forksupport.c b/rpython/translator/stm/src_stm/stm/forksupport.c new file mode 100644 --- /dev/null +++ b/rpython/translator/stm/src_stm/stm/forksupport.c @@ -0,0 +1,296 @@ +/* Imported by rpython/translator/stm/import_stmgc.py */ +#ifndef _STM_CORE_H_ +# error "must be compiled via stmgc.c" +#endif + + +/* XXX this is currently not doing copy-on-write, but simply forces a + copy of all pages as soon as fork() is called. */ + + +static char *fork_big_copy = NULL; +static stm_thread_local_t *fork_this_tl; +static bool fork_was_in_transaction; + +static char *setup_mmap(char *reason); /* forward, in setup.c */ +static pthread_t *_get_cpth(stm_thread_local_t *);/* forward, in setup.c */ + + +static bool page_is_null(char *p) +{ + long *q = (long *)p; + long i; + for (i = 0; i < 4096 / sizeof(long); i++) + if (q[i] != 0) + return false; + return true; +} + + +static void forksupport_prepare(void) +{ + if (stm_object_pages == NULL) + return; + + /* So far we attempt to check this by walking all stm_thread_local_t, + marking the one from the current thread, and verifying that it's not + running a transaction. This assumes that the stm_thread_local_t is just + a __thread variable, so never changes threads. + */ + s_mutex_lock(); + + dprintf(("forksupport_prepare\n")); + + stm_thread_local_t *this_tl = NULL; + stm_thread_local_t *tl = stm_all_thread_locals; + do { + if (pthread_equal(*_get_cpth(tl), pthread_self())) { + if (this_tl != NULL) + stm_fatalerror("fork(): found several stm_thread_local_t" + " from the same thread"); + this_tl = tl; + } + tl = tl->next; + } while (tl != stm_all_thread_locals); + + if (this_tl == NULL) + stm_fatalerror("fork(): found no stm_thread_local_t from this thread"); + s_mutex_unlock(); + + bool was_in_transaction = _stm_in_transaction(this_tl); + if (was_in_transaction) { + stm_become_inevitable("fork"); + /* Note that the line above can still fail and abort, which should + be fine */ + } + else { + stm_start_inevitable_transaction(this_tl); + } + + s_mutex_lock(); + synchronize_all_threads(); + mutex_pages_lock(); + + /* Make a new mmap at some other address, but of the same size as + the standard mmap at stm_object_pages + */ + char *big_copy = setup_mmap("stmgc's fork support"); + + /* Copy each of the segment infos into the new mmap, nurseries, + and associated read markers + */ + long i; + for (i = 1; i <= NB_SEGMENTS; i++) { + char *src, *dst; + struct stm_priv_segment_info_s *psrc = get_priv_segment(i); + dst = big_copy + (((char *)psrc) - stm_object_pages); + *(struct stm_priv_segment_info_s *)dst = *psrc; + + src = get_segment_base(i) + FIRST_READMARKER_PAGE * 4096UL; + dst = big_copy + (src - stm_object_pages); + long j; + for (j = 0; j < END_NURSERY_PAGE - FIRST_READMARKER_PAGE; j++) { + if (!page_is_null(src)) + pagecopy(dst, src); + src += 4096; + dst += 4096; + } + } + + /* Copy all the data from the two ranges of objects (large, small) + into the new mmap + */ + uintptr_t pagenum, endpagenum; + pagenum = END_NURSERY_PAGE; /* starts after the nursery */ + endpagenum = (uninitialized_page_start - stm_object_pages) / 4096UL; + if (endpagenum < NB_PAGES) + endpagenum++; /* the next page too, because it might contain + data from largemalloc */ + + while (1) { + if (UNLIKELY(pagenum == endpagenum)) { + /* we reach this point usually twice, because there are + more pages after 'uninitialized_page_stop' */ + if (endpagenum == NB_PAGES) + break; /* done */ + pagenum = (uninitialized_page_stop - stm_object_pages) / 4096UL; + pagenum--; /* the prev page too, because it does contain + data from largemalloc */ + endpagenum = NB_PAGES; + } + + char *src = stm_object_pages + pagenum * 4096UL; + char *dst = big_copy + pagenum * 4096UL; + pagecopy(dst, src); + + struct page_shared_s ps = pages_privatized[pagenum - PAGE_FLAG_START]; + if (ps.by_segment != 0) { + long j; + for (j = 0; j < NB_SEGMENTS; j++) { + src += NB_PAGES * 4096UL; + dst += NB_PAGES * 4096UL; + if (ps.by_segment & (1 << j)) { + pagecopy(dst, src); + } + } + } + pagenum++; + } + + assert(fork_big_copy == NULL); + fork_big_copy = big_copy; + fork_this_tl = this_tl; + fork_was_in_transaction = was_in_transaction; + + assert(_has_mutex()); + printf("forksupport_prepare: from %p %p\n", fork_this_tl, + fork_this_tl->creating_pthread[0]); +} + +static void forksupport_parent(void) +{ + if (stm_object_pages == NULL) + return; + + printf("forksupport_parent: continuing to run %p %p\n", fork_this_tl, + fork_this_tl->creating_pthread[0]); + assert(_has_mutex()); + assert(_is_tl_registered(fork_this_tl)); + + /* In the parent, after fork(), we can simply forget about the big copy + that we made for the child. + */ + assert(fork_big_copy != NULL); + munmap(fork_big_copy, TOTAL_MEMORY); + fork_big_copy = NULL; + bool was_in_transaction = fork_was_in_transaction; + + mutex_pages_unlock(); + s_mutex_unlock(); + + if (!was_in_transaction) { + stm_commit_transaction(); + } + + dprintf(("forksupport_parent: continuing to run\n")); +} + +static void fork_abort_thread(long i) +{ + struct stm_priv_segment_info_s *pr = get_priv_segment(i); + dprintf(("forksupport_child: abort in seg%ld\n", i)); + assert(pr->pub.running_thread->associated_segment_num == i); + assert(pr->transaction_state == TS_REGULAR); + set_gs_register(get_segment_base(i)); + + stm_jmpbuf_t jmpbuf; + if (__builtin_setjmp(jmpbuf) == 0) { + pr->pub.jmpbuf_ptr = &jmpbuf; +#ifndef NDEBUG + pr->running_pthread = pthread_self(); +#endif + stm_abort_transaction(); + } +} + +static void forksupport_child(void) +{ + if (stm_object_pages == NULL) + return; + + /* this new process contains no other thread, so we can + just release these locks early */ + mutex_pages_unlock(); + s_mutex_unlock(); + + /* Move the copy of the mmap over the old one, overwriting it + and thus freeing the old mapping in this process + */ + assert(fork_big_copy != NULL); + assert(stm_object_pages != NULL); + void *res = mremap(fork_big_copy, TOTAL_MEMORY, TOTAL_MEMORY, + MREMAP_MAYMOVE | MREMAP_FIXED, + stm_object_pages); + if (res != stm_object_pages) + stm_fatalerror("after fork: mremap failed: %m"); + fork_big_copy = NULL; + + /* Unregister all other stm_thread_local_t, mostly as a way to free + the memory used by the shadowstacks + */ + while (stm_all_thread_locals->next != stm_all_thread_locals) { + if (stm_all_thread_locals == fork_this_tl) + stm_unregister_thread_local(stm_all_thread_locals->next); + else + stm_unregister_thread_local(stm_all_thread_locals); + } + assert(stm_all_thread_locals == fork_this_tl); + + /* Make all pages shared again. + */ + uintptr_t pagenum, endpagenum; + pagenum = END_NURSERY_PAGE; /* starts after the nursery */ + endpagenum = (uninitialized_page_start - stm_object_pages) / 4096UL; + + while (1) { + if (UNLIKELY(pagenum == endpagenum)) { + /* we reach this point usually twice, because there are + more pages after 'uninitialized_page_stop' */ + if (endpagenum == NB_PAGES) + break; /* done */ + pagenum = (uninitialized_page_stop - stm_object_pages) / 4096UL; + endpagenum = NB_PAGES; + if (endpagenum == NB_PAGES) + break; /* done */ + } + + struct page_shared_s ps = pages_privatized[pagenum - PAGE_FLAG_START]; + long j; + for (j = 0; j < NB_SEGMENTS; j++) { + if (!(ps.by_segment & (1 << j))) { + _page_do_reshare(j + 1, pagenum); + } + } + pagenum++; + } + + /* Force the interruption of other running segments + */ + long i; + for (i = 1; i <= NB_SEGMENTS; i++) { + struct stm_priv_segment_info_s *pr = get_priv_segment(i); + if (pr->pub.running_thread != NULL && + pr->pub.running_thread != fork_this_tl) { + fork_abort_thread(i); + } + } + + /* Restore a few things: the new pthread_self(), and the %gs + register */ + int segnum = fork_this_tl->associated_segment_num; + assert(1 <= segnum && segnum <= NB_SEGMENTS); + *_get_cpth(fork_this_tl) = pthread_self(); + set_gs_register(get_segment_base(segnum)); + assert(STM_SEGMENT->segment_num == segnum); + + if (!fork_was_in_transaction) { + stm_commit_transaction(); + } + + /* Done */ + dprintf(("forksupport_child: running one thread now\n")); +} + + +static void setup_forksupport(void) +{ + static bool fork_support_ready = false; + + if (!fork_support_ready) { + int res = pthread_atfork(forksupport_prepare, forksupport_parent, + forksupport_child); + if (res != 0) + stm_fatalerror("pthread_atfork() failed: %m"); + fork_support_ready = true; + } +} From noreply at buildbot.pypy.org Wed Mar 19 12:06:09 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Wed, 19 Mar 2014 12:06:09 +0100 (CET) Subject: [pypy-commit] pypy default: fix reload/reimport of builtin modules (issue1514) (patch adapted from yamt) Message-ID: <20140319110609.D86EE1C1578@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r70089:6002c93c0cc0 Date: 2014-03-19 07:01 -0400 http://bitbucket.org/pypy/pypy/changeset/6002c93c0cc0/ Log: fix reload/reimport of builtin modules (issue1514) (patch adapted from yamt) diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -440,10 +440,11 @@ return name - def getbuiltinmodule(self, name, force_init=False): + def getbuiltinmodule(self, name, force_init=False, reuse=True): w_name = self.wrap(name) w_modules = self.sys.get('modules') if not force_init: + assert reuse is True try: return self.getitem(w_modules, w_name) except OperationError, e: @@ -459,10 +460,17 @@ "getbuiltinmodule() called with non-builtin module %s", name) else: - # And initialize it + # Initialize the module from pypy.interpreter.module import Module if isinstance(w_mod, Module): - w_mod.init(self) + if not reuse and w_mod.startup_called: + # Create a copy of the module + w_new = self.wrap(Module(self, w_name)) + self.call_method(w_new.getdict(self), 'update', + w_mod.w_initialdict) + w_mod = w_new + else: + w_mod.init(self) # Add the module to sys.modules self.setitem(w_modules, w_name, w_mod) diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py --- a/pypy/module/imp/importing.py +++ b/pypy/module/imp/importing.py @@ -579,7 +579,8 @@ return space.call_method(find_info.w_loader, "load_module", w_modulename) if find_info.modtype == C_BUILTIN: - return space.getbuiltinmodule(find_info.filename, force_init=True) + return space.getbuiltinmodule(find_info.filename, force_init=True, + reuse=reuse) if find_info.modtype in (PY_SOURCE, PY_COMPILED, C_EXTENSION, PKG_DIRECTORY): w_mod = None diff --git a/pypy/module/imp/test/test_app.py b/pypy/module/imp/test/test_app.py --- a/pypy/module/imp/test/test_app.py +++ b/pypy/module/imp/test/test_app.py @@ -203,7 +203,6 @@ def test_builtin_reimport(self): # from https://bugs.pypy.org/issue1514 - skip("fix me") import sys, marshal old = marshal.loads @@ -223,7 +222,6 @@ # taken from https://bugs.pypy.org/issue1514, with extra cases # that show a difference with CPython: we can get on CPython # several module objects for the same built-in module :-( - skip("several built-in module objects: not supported by pypy") import sys, marshal old = marshal.loads diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py --- a/pypy/module/imp/test/test_import.py +++ b/pypy/module/imp/test/test_import.py @@ -578,7 +578,6 @@ assert hasattr(time, 'clock') def test_reimport_builtin_simple_case_2(self): - skip("fix me") import sys, time time.foo = "bar" del sys.modules['time'] @@ -586,7 +585,6 @@ assert not hasattr(time, 'foo') def test_reimport_builtin(self): - skip("fix me") import sys, time oldpath = sys.path time.tzset = "" From noreply at buildbot.pypy.org Wed Mar 19 12:29:19 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Wed, 19 Mar 2014 12:29:19 +0100 (CET) Subject: [pypy-commit] pypy default: fix last commit Message-ID: <20140319112920.019381C00B9@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r70090:4e0378f68161 Date: 2014-03-19 07:27 -0400 http://bitbucket.org/pypy/pypy/changeset/4e0378f68161/ Log: fix last commit diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -465,6 +465,7 @@ if isinstance(w_mod, Module): if not reuse and w_mod.startup_called: # Create a copy of the module + w_mod.getdict(self) # unlazy w_initialdict w_new = self.wrap(Module(self, w_name)) self.call_method(w_new.getdict(self), 'update', w_mod.w_initialdict) From noreply at buildbot.pypy.org Wed Mar 19 12:57:14 2014 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 19 Mar 2014 12:57:14 +0100 (CET) Subject: [pypy-commit] stmgc default: update the TODO list Message-ID: <20140319115714.E96951C1578@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1079:6937d94d24cc Date: 2014-03-19 12:57 +0100 http://bitbucket.org/pypy/stmgc/changeset/6937d94d24cc/ Log: update the TODO list diff --git a/c7/TODO b/c7/TODO --- a/c7/TODO +++ b/c7/TODO @@ -1,15 +1,14 @@ -known-working revision: 5e4ec1af0e0c +known-working revision: 75893b92af4e - use small uniform gcpages - write barrier for big arrays -- weakrefs - finalizers - the highest_overflow_number can overflow after 2**30 non-collect-time minor collections -- re-enable the buggy RESHARE_PAGES=1, probably with a better impl +- fork() is done by copying the whole mmap non-lazily; improve. From noreply at buildbot.pypy.org Wed Mar 19 17:04:12 2014 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 19 Mar 2014 17:04:12 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: update Message-ID: <20140319160412.0BCD21D2824@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r70092:ee693a309114 Date: 2014-03-19 17:03 +0100 http://bitbucket.org/pypy/pypy/changeset/ee693a309114/ Log: update diff --git a/TODO b/TODO --- a/TODO +++ b/TODO @@ -28,7 +28,9 @@ ------------------------------------------------------------ -os.fork()! +reintroduce 'stm_ignored', disabled in 4294a7789103 for causing +a rare bug in ll_strhash() shown by http://bpaste.net/show/190868/ +failing after a few minutes (or just running translate.py) ------------------------------------------------------------ From noreply at buildbot.pypy.org Wed Mar 19 17:04:10 2014 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 19 Mar 2014 17:04:10 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Temporary: disable stm_ignored Message-ID: <20140319160410.9EB241D2822@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r70091:4294a7789103 Date: 2014-03-19 16:20 +0100 http://bitbucket.org/pypy/pypy/changeset/4294a7789103/ Log: Temporary: disable stm_ignored diff --git a/rpython/memory/gctransform/stmframework.py b/rpython/memory/gctransform/stmframework.py --- a/rpython/memory/gctransform/stmframework.py +++ b/rpython/memory/gctransform/stmframework.py @@ -53,13 +53,13 @@ "unbalanced stm_ignore_start/stm_ignore_stop in block") def gct_stm_ignored_start(self, hop): - assert not self.in_stm_ignored - self.in_stm_ignored = True + #assert not self.in_stm_ignored + #self.in_stm_ignored = True self.default(hop) def gct_stm_ignored_stop(self, hop): - assert self.in_stm_ignored - self.in_stm_ignored = False + #assert self.in_stm_ignored + #self.in_stm_ignored = False self.default(hop) def var_needs_set_transform(self, var): diff --git a/rpython/translator/stm/readbarrier.py b/rpython/translator/stm/readbarrier.py --- a/rpython/translator/stm/readbarrier.py +++ b/rpython/translator/stm/readbarrier.py @@ -76,11 +76,11 @@ [op.args[0]], v_none)) transformer.read_barrier_counts += 1 elif op.opname == 'stm_ignored_start': - assert stm_ignored == False - stm_ignored = True + pass #assert stm_ignored == False + #stm_ignored = True elif op.opname == 'stm_ignored_stop': - assert stm_ignored == True - stm_ignored = False + pass #assert stm_ignored == True + #stm_ignored = False newops.append(op) assert stm_ignored == False block.operations = newops From noreply at buildbot.pypy.org Wed Mar 19 17:34:12 2014 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 19 Mar 2014 17:34:12 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Add a "debug_stm" makefile target to compile with STM_DEBUGPRINTs Message-ID: <20140319163412.28A071D2824@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r70093:ead53269de59 Date: 2014-03-19 17:33 +0100 http://bitbucket.org/pypy/pypy/changeset/ead53269de59/ Log: Add a "debug_stm" makefile target to compile with STM_DEBUGPRINTs diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py --- a/rpython/translator/c/genc.py +++ b/rpython/translator/c/genc.py @@ -461,6 +461,7 @@ ('llsafer', '', '$(MAKE) CFLAGS="-O2 -DRPY_LL_ASSERT" $(TARGET)'), ('lldebug', '', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -DRPY_ASSERT -DRPY_LL_ASSERT -DRPY_STM_ASSERT" debug_target'), ('lldebug0','', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -O0 -DRPY_ASSERT -DRPY_LL_ASSERT -DRPY_STM_ASSERT" debug_target'), + ('debug_stm','', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -O0 -DRPY_ASSERT -DRPY_LL_ASSERT -DRPY_STM_ASSERT -DSTM_DEBUGPRINT" debug_target'), ('profile', '', '$(MAKE) CFLAGS="-g -O1 -pg $(CFLAGS) -fno-omit-frame-pointer" LDFLAGS="-pg $(LDFLAGS)" $(TARGET)'), ] if self.has_profopt(): From noreply at buildbot.pypy.org Wed Mar 19 17:41:58 2014 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 19 Mar 2014 17:41:58 +0100 (CET) Subject: [pypy-commit] pypy default: Yet another attempt to fix these C compiler warnings: don't Message-ID: <20140319164158.3089D1C00B9@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r70094:381eb668a116 Date: 2014-03-19 17:41 +0100 http://bitbucket.org/pypy/pypy/changeset/381eb668a116/ Log: Yet another attempt to fix these C compiler warnings: don't use an enum at all, just declare the type as 'int'. diff --git a/pypy/module/cpyext/include/pystate.h b/pypy/module/cpyext/include/pystate.h --- a/pypy/module/cpyext/include/pystate.h +++ b/pypy/module/cpyext/include/pystate.h @@ -21,9 +21,8 @@ #define Py_END_ALLOW_THREADS PyEval_RestoreThread(_save); \ } -typedef - enum {PyGILState_LOCKED, PyGILState_UNLOCKED} - PyGILState_STATE; +enum {PyGILState_LOCKED, PyGILState_UNLOCKED}; +typedef int PyGILState_STATE; #define PyThreadState_GET() PyThreadState_Get() diff --git a/pypy/module/cpyext/pystate.py b/pypy/module/cpyext/pystate.py --- a/pypy/module/cpyext/pystate.py +++ b/pypy/module/cpyext/pystate.py @@ -204,16 +204,14 @@ # Before external call is after running Python rffi.aroundstate.before() -PyGILState_STATE = rffi.COpaquePtr('PyGILState_STATE', - typedef='PyGILState_STATE', - compilation_info=CConfig._compilation_info_) +PyGILState_STATE = rffi.INT @cpython_api([], PyGILState_STATE, error=CANNOT_FAIL) def PyGILState_Ensure(space): if rffi.aroundstate.after: # After external call is before entering Python rffi.aroundstate.after() - return lltype.nullptr(PyGILState_STATE.TO) + return rffi.cast(PyGILState_STATE, 0) @cpython_api([PyGILState_STATE], lltype.Void) def PyGILState_Release(space, state): From noreply at buildbot.pypy.org Wed Mar 19 18:13:41 2014 From: noreply at buildbot.pypy.org (pjenvey) Date: Wed, 19 Mar 2014 18:13:41 +0100 (CET) Subject: [pypy-commit] pypy default: print as a statement is bad (py3k compat) Message-ID: <20140319171341.C301E1C01DE@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: Changeset: r70095:c6058cfbd0c1 Date: 2014-03-19 10:12 -0700 http://bitbucket.org/pypy/pypy/changeset/c6058cfbd0c1/ Log: print as a statement is bad (py3k compat) diff --git a/pypy/module/marshal/test/test_marshal.py b/pypy/module/marshal/test/test_marshal.py --- a/pypy/module/marshal/test/test_marshal.py +++ b/pypy/module/marshal/test/test_marshal.py @@ -11,7 +11,7 @@ def w_marshal_check(self, case): import marshal, StringIO s = marshal.dumps(case) - print repr(s) + print(repr(s)) x = marshal.loads(s) assert x == case and type(x) is type(case) f = StringIO.StringIO() @@ -173,7 +173,7 @@ import marshal types = (float, complex, int, long, tuple, list, dict, set, frozenset) for cls in types: - print cls + print(cls) class subtype(cls): pass exc = raises(ValueError, marshal.dumps, subtype) From noreply at buildbot.pypy.org Wed Mar 19 20:37:58 2014 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 19 Mar 2014 20:37:58 +0100 (CET) Subject: [pypy-commit] jitviewer default: Backed out changeset e0b82d426d01 - it actually does not run on cpython Message-ID: <20140319193758.CC68F1D2543@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r257:8488aa916ea4 Date: 2014-03-19 21:37 +0200 http://bitbucket.org/pypy/jitviewer/changeset/8488aa916ea4/ Log: Backed out changeset e0b82d426d01 - it actually does not run on cpython diff --git a/bin/jitviewer.py b/bin/jitviewer.py --- a/bin/jitviewer.py +++ b/bin/jitviewer.py @@ -6,5 +6,10 @@ pythonpath = os.path.dirname(os.path.dirname(script_path)) sys.path.append(pythonpath) +# Check we are running with PyPy first. +if not '__pypy__' in sys.builtin_module_names: + from _jitviewer.misc import failout + failout("jitviewer must be run with PyPy") + from _jitviewer.app import main main(sys.argv) From noreply at buildbot.pypy.org Wed Mar 19 20:55:32 2014 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 19 Mar 2014 20:55:32 +0100 (CET) Subject: [pypy-commit] pypy elidable-canfold-exception: failing test Message-ID: <20140319195532.C89A51C0962@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: elidable-canfold-exception Changeset: r70096:361433cd94a2 Date: 2014-03-18 14:16 +0200 http://bitbucket.org/pypy/pypy/changeset/361433cd94a2/ Log: failing test diff --git a/rpython/jit/metainterp/test/test_call.py b/rpython/jit/metainterp/test/test_call.py --- a/rpython/jit/metainterp/test/test_call.py +++ b/rpython/jit/metainterp/test/test_call.py @@ -44,6 +44,25 @@ self.meta_interp(main, [10]) + def test_call_elidable_raise(self): + @jit.elidable(canfolderror=(ValueError,)) + def f(a): + raise ValueError + + driver = jit.JitDriver(greens = [], reds = 'auto') + + def main(n): + while n >= 0: + driver.jit_merge_point() + try: + f(1) + except ValueError: + pass + n -= 1 + + self.meta_interp(main, [10]) + self.check_simple_loop(call=0) + def test_cond_call(self): def f(l, n): l.append(n) From noreply at buildbot.pypy.org Wed Mar 19 20:55:34 2014 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 19 Mar 2014 20:55:34 +0100 (CET) Subject: [pypy-commit] pypy improve-consecutive-dict-lookups: implement interiorfieldsupport for writeanalyzer Message-ID: <20140319195534.3BD3E1C0962@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: improve-consecutive-dict-lookups Changeset: r70097:3bf5dd780599 Date: 2014-03-19 20:28 +0200 http://bitbucket.org/pypy/pypy/changeset/3bf5dd780599/ Log: implement interiorfieldsupport for writeanalyzer diff --git a/rpython/translator/backendopt/test/test_writeanalyze.py b/rpython/translator/backendopt/test/test_writeanalyze.py --- a/rpython/translator/backendopt/test/test_writeanalyze.py +++ b/rpython/translator/backendopt/test/test_writeanalyze.py @@ -353,3 +353,23 @@ result = wa.analyze(fgraph.startblock.operations[-1]) assert list(result) == [("struct", lltype.Ptr(S), "x")] + + def test_interiorfield(self): + A = lltype.GcArray(lltype.Struct('x', ('x', lltype.Signed), + ('y', lltype.Signed))) + + def g(x): + a = lltype.malloc(A, 1) + a[0].y = 3 + return f(a, x) + + def f(a, x): + a[0].x = x + return a[0].y + + t, wa = self.translate(g, [int]) + ggraph = graphof(t, g) + result = wa.analyze(ggraph.startblock.operations[-1]) + res = list(result) + assert ('readinteriorfield', lltype.Ptr(A), 'y') in res + assert ('interiorfield', lltype.Ptr(A), 'x') in res diff --git a/rpython/translator/backendopt/writeanalyze.py b/rpython/translator/backendopt/writeanalyze.py --- a/rpython/translator/backendopt/writeanalyze.py +++ b/rpython/translator/backendopt/writeanalyze.py @@ -45,11 +45,18 @@ elif op.opname == "setarrayitem": if graphinfo is None or not graphinfo.is_fresh_malloc(op.args[0]): return self._array_result(op.args[0].concretetype) + elif op.opname == "setinteriorfield": + if graphinfo is None or not graphinfo.is_fresh_malloc(op.args[0]): + return self._interiorfield_result(op.args[0].concretetype, + op.args[2].value) return empty_set def _array_result(self, TYPE): return frozenset([("array", TYPE)]) + def _interiorfield_result(self, TYPE, fieldname): + return frozenset([("interiorfield", TYPE, fieldname)]) + def compute_graph_info(self, graph): return FreshMallocs(graph) @@ -99,4 +106,8 @@ elif op.opname == "getarrayitem": return frozenset([ ("readarray", op.args[0].concretetype)]) + elif op.opname == "getinteriorfield": + return frozenset([ + ("readinteriorfield", op.args[0].concretetype, + op.args[2].value)]) return WriteAnalyzer.analyze_simple_operation(self, op, graphinfo) From noreply at buildbot.pypy.org Wed Mar 19 20:55:35 2014 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 19 Mar 2014 20:55:35 +0100 (CET) Subject: [pypy-commit] pypy improve-consecutive-dict-lookups: sort out the issue with interiorfielddescrs (I think) and dicts Message-ID: <20140319195535.8C6741C0962@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: improve-consecutive-dict-lookups Changeset: r70098:2cf24861ad15 Date: 2014-03-19 21:48 +0200 http://bitbucket.org/pypy/pypy/changeset/2cf24861ad15/ Log: sort out the issue with interiorfielddescrs (I think) and dicts diff --git a/rpython/jit/codewriter/effectinfo.py b/rpython/jit/codewriter/effectinfo.py --- a/rpython/jit/codewriter/effectinfo.py +++ b/rpython/jit/codewriter/effectinfo.py @@ -93,12 +93,14 @@ ]) def __new__(cls, readonly_descrs_fields, readonly_descrs_arrays, + readonly_descrs_interiorfields, write_descrs_fields, write_descrs_arrays, + write_descrs_interiorfields, extraeffect=EF_CAN_RAISE, oopspecindex=OS_NONE, can_invalidate=False, call_release_gil_target=llmemory.NULL, - extradescr=None): + extradescrs=None): key = (frozenset_or_none(readonly_descrs_fields), frozenset_or_none(readonly_descrs_arrays), frozenset_or_none(write_descrs_fields), @@ -123,18 +125,21 @@ result = object.__new__(cls) result.readonly_descrs_fields = readonly_descrs_fields result.readonly_descrs_arrays = readonly_descrs_arrays + result.readonly_descrs_interiorfields = readonly_descrs_interiorfields if extraeffect == EffectInfo.EF_LOOPINVARIANT or \ extraeffect == EffectInfo.EF_ELIDABLE_CANNOT_RAISE or \ extraeffect == EffectInfo.EF_ELIDABLE_CAN_RAISE: result.write_descrs_fields = [] result.write_descrs_arrays = [] + result.write_descrs_interiorfields = [] else: result.write_descrs_fields = write_descrs_fields result.write_descrs_arrays = write_descrs_arrays + result.write_descrs_interiorfields = write_descrs_interiorfields result.extraeffect = extraeffect result.can_invalidate = can_invalidate result.oopspecindex = oopspecindex - result.extradescr = extradescr + result.extradescrs = extradescrs result.call_release_gil_target = call_release_gil_target if result.check_can_raise(): assert oopspecindex in cls._OS_CANRAISE @@ -166,7 +171,7 @@ return None return frozenset(x) -EffectInfo.MOST_GENERAL = EffectInfo(None, None, None, None, +EffectInfo.MOST_GENERAL = EffectInfo(None, None, None, None, None, None, EffectInfo.EF_RANDOM_EFFECTS, can_invalidate=True) @@ -181,14 +186,18 @@ if effects is top_set or extraeffect == EffectInfo.EF_RANDOM_EFFECTS: readonly_descrs_fields = None readonly_descrs_arrays = None + readonly_descrs_interiorfields = None write_descrs_fields = None write_descrs_arrays = None + write_descrs_interiorfields = None extraeffect = EffectInfo.EF_RANDOM_EFFECTS else: readonly_descrs_fields = [] readonly_descrs_arrays = [] + readonly_descrs_interiorfields = [] write_descrs_fields = [] write_descrs_arrays = [] + write_descrs_interiorfields = [] def add_struct(descrs_fields, (_, T, fieldname)): T = deref(T) @@ -202,6 +211,17 @@ descr = cpu.arraydescrof(ARRAY) descrs_arrays.append(descr) + def add_interiorfield(descrs_interiorfields, (_, T, fieldname)): + T = deref(T) + if not isinstance(T, lltype.Array): + return # let's not consider structs for now + if not consider_array(T): + return + if getattr(T.OF, fieldname) is lltype.Void: + return + descr = cpu.interiorfielddescrof(T, fieldname) + descrs_interiorfields.append(descr) + for tup in effects: if tup[0] == "struct": add_struct(write_descrs_fields, tup) @@ -209,6 +229,12 @@ tupw = ("struct",) + tup[1:] if tupw not in effects: add_struct(readonly_descrs_fields, tup) + elif tup[0] == "interiorfield": + add_interiorfield(write_descrs_interiorfields, tup) + elif tup[0] == "readinteriorfield": + tupw = ('interiorfield',) + tup[1:] + if tupw not in effects: + add_interiorfield(readonly_descrs_interiorfields, tup) elif tup[0] == "array": add_array(write_descrs_arrays, tup) elif tup[0] == "readarray": @@ -220,8 +246,10 @@ # return EffectInfo(readonly_descrs_fields, readonly_descrs_arrays, + readonly_descrs_interiorfields, write_descrs_fields, write_descrs_arrays, + write_descrs_interiorfields, extraeffect, oopspecindex, can_invalidate, diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py --- a/rpython/jit/codewriter/jtransform.py +++ b/rpython/jit/codewriter/jtransform.py @@ -1852,10 +1852,12 @@ EffectInfo.EF_ELIDABLE_CANNOT_RAISE) def _handle_dict_lookup_call(self, op, oopspec_name, args): - extradescr = self.cpu.fielddescrof(op.args[1].concretetype.TO, - 'entries') + extradescr1 = self.cpu.fielddescrof(op.args[1].concretetype.TO, + 'entries') + extradescr2 = self.cpu.interiorfielddescrof( + op.args[1].concretetype.TO.entries.TO, 'key') return self._handle_oopspec_call(op, args, EffectInfo.OS_DICT_LOOKUP, - extradescr=extradescr) + extradescr=[extradescr1, extradescr2]) def _handle_rgc_call(self, op, oopspec_name, args): if oopspec_name == 'rgc.ll_shrink_array': diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py --- a/rpython/jit/metainterp/optimizeopt/heap.py +++ b/rpython/jit/metainterp/optimizeopt/heap.py @@ -177,6 +177,8 @@ self.cached_arrayitems = {} # cached dict items: {dict descr: {(optval, index): box-or-const}} self.cached_dict_reads = {} + # cache of corresponding array descrs + self.corresponding_array_descrs = {} # self._lazy_setfields_and_arrayitems = [] self._remove_guard_not_invalidated = False @@ -185,10 +187,12 @@ def force_at_end_of_preamble(self): self.cached_dict_reads.clear() + self.corresponding_array_descrs.clear() self.force_all_lazy_setfields_and_arrayitems() def flush(self): self.cached_dict_reads.clear() + self.corresponding_array_descrs.clear() self.force_all_lazy_setfields_and_arrayitems() self.emit_postponed_op() @@ -301,11 +305,14 @@ self.emit_operation(op) def _optimize_CALL_DICT_LOOKUP(self, op): - descr = op.getdescr().get_extra_info().extradescr - if descr in self.cached_dict_reads: - d = self.cached_dict_reads[descr] + descrs = op.getdescr().get_extra_info().extradescrs + descr1 = descrs[0] + descr2 = descrs[1] + if descr1 in self.cached_dict_reads: + d = self.cached_dict_reads[descr1] else: - d = self.cached_dict_reads[descr] = args_dict() + d = self.cached_dict_reads[descr1] = args_dict() + self.corresponding_array_descrs[descr2] = descr1 args = self.optimizer.make_args_key(op) try: res_v = d[args] @@ -339,6 +346,13 @@ self.force_lazy_setfield(fielddescr, can_cache=False) for arraydescr in effectinfo.write_descrs_arrays: self.force_lazy_setarrayitem(arraydescr, can_cache=False) + for descr in effectinfo.write_descrs_interiorfields: + if descr in self.corresponding_array_descrs: + dictdescr = self.corresponding_array_descrs.pop(descr) + try: + del self.cached_dict_reads[dictdescr] + except KeyError: + pass # someone did it already if effectinfo.check_forces_virtual_or_virtualizable(): vrefinfo = self.optimizer.metainterp_sd.virtualref_info self.force_lazy_setfield(vrefinfo.descr_forced) diff --git a/rpython/jit/metainterp/test/test_dict.py b/rpython/jit/metainterp/test/test_dict.py --- a/rpython/jit/metainterp/test/test_dict.py +++ b/rpython/jit/metainterp/test/test_dict.py @@ -253,6 +253,28 @@ assert res == f(10) self.check_simple_loop(call=5) + def test_dict_array_write_invalidates_caches(self): + driver = JitDriver(greens = [], reds = 'auto') + indexes = ['aa', 'b', 'cc'] + + def f(n): + d = {'aa': 3, 'b': 4, 'cc': 5} + s = 0 + while n > 0: + driver.jit_merge_point() + index = indexes[n & 1] + s += d[index] + del d['cc'] + s += d[index] + d['cc'] = 3 + n -= 1 + return s + + exp = f(10) + res = self.meta_interp(f, [10]) + assert res == exp + self.check_simple_loop(call=7) + def test_dict_double_lookup_2(self): driver = JitDriver(greens = [], reds = 'auto') indexes = ['aa', 'b', 'cc'] diff --git a/rpython/translator/backendopt/writeanalyze.py b/rpython/translator/backendopt/writeanalyze.py --- a/rpython/translator/backendopt/writeanalyze.py +++ b/rpython/translator/backendopt/writeanalyze.py @@ -1,4 +1,4 @@ -from rpython.flowspace.model import Variable +from rpython.flowspace.model import Variable, Constant from rpython.translator.backendopt import graphanalyze top_set = object() @@ -37,6 +37,12 @@ return top_set return result1.union(result2) + def _getinteriorname(self, op): + if (isinstance(op.args[1], Constant) and + isinstance(op.args[1].value, str)): + return op.args[1].value + return op.args[2].value + def analyze_simple_operation(self, op, graphinfo): if op.opname == "setfield": if graphinfo is None or not graphinfo.is_fresh_malloc(op.args[0]): @@ -47,8 +53,8 @@ return self._array_result(op.args[0].concretetype) elif op.opname == "setinteriorfield": if graphinfo is None or not graphinfo.is_fresh_malloc(op.args[0]): - return self._interiorfield_result(op.args[0].concretetype, - op.args[2].value) + name = self._getinteriorname(op) + return self._interiorfield_result(op.args[0].concretetype, name) return empty_set def _array_result(self, TYPE): @@ -107,7 +113,7 @@ return frozenset([ ("readarray", op.args[0].concretetype)]) elif op.opname == "getinteriorfield": - return frozenset([ - ("readinteriorfield", op.args[0].concretetype, - op.args[2].value)]) + name = self._getinteriorname(op) + return frozenset([("readinteriorfield", op.args[0].concretetype, + name)]) return WriteAnalyzer.analyze_simple_operation(self, op, graphinfo) From noreply at buildbot.pypy.org Wed Mar 19 21:31:32 2014 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 19 Mar 2014 21:31:32 +0100 (CET) Subject: [pypy-commit] pypy improve-consecutive-dict-lookups: fix Message-ID: <20140319203132.0F04A1D284F@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: improve-consecutive-dict-lookups Changeset: r70099:498fe7267231 Date: 2014-03-19 22:30 +0200 http://bitbucket.org/pypy/pypy/changeset/498fe7267231/ Log: fix diff --git a/rpython/jit/metainterp/virtualizable.py b/rpython/jit/metainterp/virtualizable.py --- a/rpython/jit/metainterp/virtualizable.py +++ b/rpython/jit/metainterp/virtualizable.py @@ -302,7 +302,7 @@ self.clear_vable_ptr = self.warmrunnerdesc.helper_func( FUNCPTR, self.clear_vable_token) FUNC = FUNCPTR.TO - ei = EffectInfo([], [], [], [], EffectInfo.EF_CANNOT_RAISE, + ei = EffectInfo([], [], [], [], [], [], EffectInfo.EF_CANNOT_RAISE, can_invalidate=False, oopspecindex=EffectInfo.OS_JIT_FORCE_VIRTUALIZABLE) From noreply at buildbot.pypy.org Wed Mar 19 21:37:42 2014 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 19 Mar 2014 21:37:42 +0100 (CET) Subject: [pypy-commit] pypy improve-consecutive-dict-lookups: fix some tests Message-ID: <20140319203742.4467D1D288F@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: improve-consecutive-dict-lookups Changeset: r70100:89278beb6037 Date: 2014-03-19 22:37 +0200 http://bitbucket.org/pypy/pypy/changeset/89278beb6037/ Log: fix some tests diff --git a/rpython/jit/codewriter/test/test_jtransform.py b/rpython/jit/codewriter/test/test_jtransform.py --- a/rpython/jit/codewriter/test/test_jtransform.py +++ b/rpython/jit/codewriter/test/test_jtransform.py @@ -60,7 +60,8 @@ class FakeResidualCallControl: def guess_call_kind(self, op): return 'residual' - def getcalldescr(self, op, oopspecindex=None, extraeffect=None): + def getcalldescr(self, op, oopspecindex=None, extraeffect=None, + extradescr=None): return 'calldescr' def calldescr_canraise(self, calldescr): return True @@ -117,7 +118,8 @@ self.callinfocollection = FakeCallInfoCollection() def guess_call_kind(self, op): return 'builtin' - def getcalldescr(self, op, oopspecindex=None, extraeffect=None): + def getcalldescr(self, op, oopspecindex=None, extraeffect=None, + extradescr=None): assert oopspecindex is not None # in this test EI = effectinfo.EffectInfo if oopspecindex != EI.OS_ARRAYCOPY: diff --git a/rpython/jit/codewriter/test/test_list.py b/rpython/jit/codewriter/test/test_list.py --- a/rpython/jit/codewriter/test/test_list.py +++ b/rpython/jit/codewriter/test/test_list.py @@ -37,7 +37,8 @@ class FakeCallControl: class getcalldescr(AbstractDescr): - def __init__(self, op, oopspecindex=0, extraeffect=None): + def __init__(self, op, oopspecindex=0, extraeffect=None, + extradescr=None): self.op = op self.oopspecindex = oopspecindex def __repr__(self): diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py --- a/rpython/jit/metainterp/optimizeopt/test/test_util.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py @@ -181,28 +181,29 @@ plaincalldescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, EffectInfo.MOST_GENERAL) nonwritedescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, - EffectInfo([], [], [], [])) + EffectInfo([], [], [], [], [], [])) writeadescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, - EffectInfo([], [], [adescr], [])) + EffectInfo([], [], [adescr], [], [], [])) writearraydescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, - EffectInfo([], [], [adescr], [arraydescr])) + EffectInfo([], [], [adescr], [arraydescr], + [], [])) readadescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, - EffectInfo([adescr], [], [], [])) + EffectInfo([adescr], [], [], [], [], [])) mayforcevirtdescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, - EffectInfo([nextdescr], [], [], [], + EffectInfo([nextdescr], [], [], [], [], [], EffectInfo.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE, can_invalidate=True)) arraycopydescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, - EffectInfo([], [arraydescr], [], [arraydescr], + EffectInfo([], [arraydescr], [], [arraydescr], [], [], EffectInfo.EF_CANNOT_RAISE, oopspecindex=EffectInfo.OS_ARRAYCOPY)) raw_malloc_descr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, - EffectInfo([], [], [], [], + EffectInfo([], [], [], [], [], [], EffectInfo.EF_CAN_RAISE, oopspecindex=EffectInfo.OS_RAW_MALLOC_VARSIZE_CHAR)) raw_free_descr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, - EffectInfo([], [], [], [], + EffectInfo([], [], [], [], [], [], EffectInfo.EF_CANNOT_RAISE, oopspecindex=EffectInfo.OS_RAW_FREE)) @@ -251,17 +252,18 @@ _oopspecindex = getattr(EffectInfo, _os) locals()[_name] = \ cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, - EffectInfo([], [], [], [], EffectInfo.EF_CANNOT_RAISE, + EffectInfo([], [], [], [], [], [], EffectInfo.EF_CANNOT_RAISE, oopspecindex=_oopspecindex)) # _oopspecindex = getattr(EffectInfo, _os.replace('STR', 'UNI')) locals()[_name.replace('str', 'unicode')] = \ cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, - EffectInfo([], [], [], [], EffectInfo.EF_CANNOT_RAISE, + EffectInfo([], [], [], [], [], [], EffectInfo.EF_CANNOT_RAISE, oopspecindex=_oopspecindex)) s2u_descr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, - EffectInfo([], [], [], [], oopspecindex=EffectInfo.OS_STR2UNICODE)) + EffectInfo([], [], [], [], [], [], + oopspecindex=EffectInfo.OS_STR2UNICODE)) # class LoopToken(AbstractDescr): @@ -277,7 +279,7 @@ virtualtokendescr = vrefinfo.descr_virtual_token virtualforceddescr = vrefinfo.descr_forced FUNC = lltype.FuncType([], lltype.Void) - ei = EffectInfo([], [], [], [], EffectInfo.EF_CANNOT_RAISE, + ei = EffectInfo([], [], [], [], [], [], EffectInfo.EF_CANNOT_RAISE, can_invalidate=False, oopspecindex=EffectInfo.OS_JIT_FORCE_VIRTUALIZABLE) clear_vable = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, ei) From noreply at buildbot.pypy.org Wed Mar 19 21:43:59 2014 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 19 Mar 2014 21:43:59 +0100 (CET) Subject: [pypy-commit] pypy improve-consecutive-dict-lookups: two more fixes Message-ID: <20140319204359.DB85D1D28C3@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: improve-consecutive-dict-lookups Changeset: r70101:20b546accb72 Date: 2014-03-19 22:43 +0200 http://bitbucket.org/pypy/pypy/changeset/20b546accb72/ Log: two more fixes diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py --- a/rpython/jit/metainterp/optimizeopt/test/test_util.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py @@ -183,10 +183,10 @@ nonwritedescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, EffectInfo([], [], [], [], [], [])) writeadescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, - EffectInfo([], [], [adescr], [], [], [])) + EffectInfo([], [], [], [adescr], [], [])) writearraydescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, - EffectInfo([], [], [adescr], [arraydescr], - [], [])) + EffectInfo([], [], [], [adescr], [arraydescr], + [])) readadescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, EffectInfo([adescr], [], [], [], [], [])) mayforcevirtdescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, @@ -194,7 +194,7 @@ EffectInfo.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE, can_invalidate=True)) arraycopydescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, - EffectInfo([], [arraydescr], [], [arraydescr], [], [], + EffectInfo([], [arraydescr], [], [], [arraydescr], [], EffectInfo.EF_CANNOT_RAISE, oopspecindex=EffectInfo.OS_ARRAYCOPY)) From noreply at buildbot.pypy.org Wed Mar 19 23:15:57 2014 From: noreply at buildbot.pypy.org (mattip) Date: Wed, 19 Mar 2014 23:15:57 +0100 (CET) Subject: [pypy-commit] pypy default: win32 compatability Message-ID: <20140319221557.5C8221C00B9@cobra.cs.uni-duesseldorf.de> Author: Matti Picus Branch: Changeset: r70102:cbeca026046f Date: 2014-03-20 00:15 +0200 http://bitbucket.org/pypy/pypy/changeset/cbeca026046f/ Log: win32 compatability diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py --- a/rpython/rlib/rfile.py +++ b/rpython/rlib/rfile.py @@ -14,6 +14,11 @@ includes = ['stdio.h', 'sys/types.h'] if os.name == "posix": includes += ['unistd.h'] + ftruncate = 'ftruncate' + fileno = 'fileno' +else: + ftruncate = '_chsize' + fileno = '_fileno' eci = ExternalCompilationInfo(includes=includes) def llexternal(*args, **kwargs): @@ -41,10 +46,10 @@ c_fseek = llexternal('fseek', [lltype.Ptr(FILE), rffi.LONG, rffi.INT], rffi.INT) c_tmpfile = llexternal('tmpfile', [], lltype.Ptr(FILE)) -c_fileno = llexternal('fileno', [lltype.Ptr(FILE)], rffi.INT) +c_fileno = llexternal(fileno, [lltype.Ptr(FILE)], rffi.INT) c_ftell = llexternal('ftell', [lltype.Ptr(FILE)], rffi.LONG) c_fflush = llexternal('fflush', [lltype.Ptr(FILE)], rffi.INT) -c_ftruncate = llexternal('ftruncate', [rffi.INT, OFF_T], rffi.INT, macro=True) +c_ftruncate = llexternal(ftruncate, [rffi.INT, OFF_T], rffi.INT, macro=True) c_fgets = llexternal('fgets', [rffi.CCHARP, rffi.INT, lltype.Ptr(FILE)], rffi.CCHARP) From noreply at buildbot.pypy.org Thu Mar 20 08:27:40 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 20 Mar 2014 08:27:40 +0100 (CET) Subject: [pypy-commit] stmgc default: This contention management policy is much better. Document the Message-ID: <20140320072740.9D4FD1C13AD@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1080:9b94866e95da Date: 2014-03-20 08:27 +0100 http://bitbucket.org/pypy/stmgc/changeset/9b94866e95da/ Log: This contention management policy is much better. Document the problem with the previous one. diff --git a/c7/stm/contention.c b/c7/stm/contention.c --- a/c7/stm/contention.c +++ b/c7/stm/contention.c @@ -69,6 +69,14 @@ __attribute__((unused)) static void cm_always_wait_for_other_thread(struct contmgr_s *cm) { + /* we tried this contention management, but it seems to have + very bad cases: if thread 1 always reads an object in every + transaction, and thread 2 wants to write this object just + once, then thread 2 will pause when it tries to commit; + it will wait until thread 1 committed; but by the time + thread 2 resumes again, thread 1 has already started the + next transaction and read the object again. + */ cm_abort_the_younger(cm); cm->try_sleep = true; } @@ -110,7 +118,7 @@ #ifdef STM_TESTS cm_abort_the_younger(&contmgr); #else - cm_always_wait_for_other_thread(&contmgr); + cm_pause_if_younger(&contmgr); #endif /* Fix the choices that are found incorrect due to TS_INEVITABLE From noreply at buildbot.pypy.org Thu Mar 20 08:30:38 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 20 Mar 2014 08:30:38 +0100 (CET) Subject: [pypy-commit] stmgc default: Print a warning to stderr Message-ID: <20140320073038.7AFDF1C1578@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1081:0e5239ae07f2 Date: 2014-03-20 08:30 +0100 http://bitbucket.org/pypy/stmgc/changeset/0e5239ae07f2/ Log: Print a warning to stderr diff --git a/c7/stm/forksupport.c b/c7/stm/forksupport.c --- a/c7/stm/forksupport.c +++ b/c7/stm/forksupport.c @@ -39,6 +39,7 @@ s_mutex_lock(); dprintf(("forksupport_prepare\n")); + fprintf(stderr, "[forking: for now, this operation can take some time]\n"); stm_thread_local_t *this_tl = NULL; stm_thread_local_t *tl = stm_all_thread_locals; From noreply at buildbot.pypy.org Thu Mar 20 08:31:52 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 20 Mar 2014 08:31:52 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: import stmgc/0e5239ae07f2 Message-ID: <20140320073152.2261C1C1578@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r70103:7ace040eff96 Date: 2014-03-20 08:31 +0100 http://bitbucket.org/pypy/pypy/changeset/7ace040eff96/ Log: import stmgc/0e5239ae07f2 diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision --- a/rpython/translator/stm/src_stm/revision +++ b/rpython/translator/stm/src_stm/revision @@ -1,1 +1,1 @@ -75893b92af4e +0e5239ae07f2 diff --git a/rpython/translator/stm/src_stm/stm/contention.c b/rpython/translator/stm/src_stm/stm/contention.c --- a/rpython/translator/stm/src_stm/stm/contention.c +++ b/rpython/translator/stm/src_stm/stm/contention.c @@ -70,6 +70,14 @@ __attribute__((unused)) static void cm_always_wait_for_other_thread(struct contmgr_s *cm) { + /* we tried this contention management, but it seems to have + very bad cases: if thread 1 always reads an object in every + transaction, and thread 2 wants to write this object just + once, then thread 2 will pause when it tries to commit; + it will wait until thread 1 committed; but by the time + thread 2 resumes again, thread 1 has already started the + next transaction and read the object again. + */ cm_abort_the_younger(cm); cm->try_sleep = true; } @@ -111,7 +119,7 @@ #ifdef STM_TESTS cm_abort_the_younger(&contmgr); #else - cm_always_wait_for_other_thread(&contmgr); + cm_pause_if_younger(&contmgr); #endif /* Fix the choices that are found incorrect due to TS_INEVITABLE diff --git a/rpython/translator/stm/src_stm/stm/forksupport.c b/rpython/translator/stm/src_stm/stm/forksupport.c --- a/rpython/translator/stm/src_stm/stm/forksupport.c +++ b/rpython/translator/stm/src_stm/stm/forksupport.c @@ -40,6 +40,7 @@ s_mutex_lock(); dprintf(("forksupport_prepare\n")); + fprintf(stderr, "[forking: for now, this operation can take some time]\n"); stm_thread_local_t *this_tl = NULL; stm_thread_local_t *tl = stm_all_thread_locals; @@ -143,8 +144,8 @@ fork_was_in_transaction = was_in_transaction; assert(_has_mutex()); - printf("forksupport_prepare: from %p %p\n", fork_this_tl, - fork_this_tl->creating_pthread[0]); + dprintf(("forksupport_prepare: from %p %p\n", fork_this_tl, + fork_this_tl->creating_pthread[0])); } static void forksupport_parent(void) @@ -152,8 +153,8 @@ if (stm_object_pages == NULL) return; - printf("forksupport_parent: continuing to run %p %p\n", fork_this_tl, - fork_this_tl->creating_pthread[0]); + dprintf(("forksupport_parent: continuing to run %p %p\n", fork_this_tl, + fork_this_tl->creating_pthread[0])); assert(_has_mutex()); assert(_is_tl_registered(fork_this_tl)); From noreply at buildbot.pypy.org Thu Mar 20 10:30:58 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 20 Mar 2014 10:30:58 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Don't produce stm_write() for writes of a Void value. Message-ID: <20140320093058.A0E931C08F3@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r70104:c0537b11889f Date: 2014-03-20 10:30 +0100 http://bitbucket.org/pypy/pypy/changeset/c0537b11889f/ Log: Don't produce stm_write() for writes of a Void value. diff --git a/rpython/memory/gctransform/stmframework.py b/rpython/memory/gctransform/stmframework.py --- a/rpython/memory/gctransform/stmframework.py +++ b/rpython/memory/gctransform/stmframework.py @@ -71,7 +71,9 @@ v_struct = hop.spaceop.args[0] assert opname in ('setfield', 'setarrayitem', 'setinteriorfield', 'raw_store') - if not var_needsgc(v_struct): + if hop.spaceop.args[-1].concretetype == lltype.Void: + pass # ignore setfields of a Void type + elif not var_needsgc(v_struct): if (var_needsgc(hop.spaceop.args[-1]) and 'is_excdata' not in hop.spaceop.args[0].concretetype.TO._hints): raise Exception("%s: GC pointer written into a non-GC location" From noreply at buildbot.pypy.org Thu Mar 20 11:11:11 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 20 Mar 2014 11:11:11 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Revert a change from c4 Message-ID: <20140320101111.B518D1C01DE@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r70105:d41fc1add33b Date: 2014-03-20 10:41 +0100 http://bitbucket.org/pypy/pypy/changeset/d41fc1add33b/ Log: Revert a change from c4 diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py --- a/rpython/rlib/rgc.py +++ b/rpython/rlib/rgc.py @@ -93,22 +93,11 @@ def _make_sure_does_not_move(p): """'p' is a non-null GC object. This (tries to) make sure that the object does not move any more, by forcing collections if needed. - It may return a different addr! Warning: should ideally only be used with the minimark GC, and only on objects that are already a bit old, so have a chance to be already non-movable.""" if not we_are_translated(): - if isinstance(p, _GcRef): - return cast_gcref_to_int(p) - else: - from rpython.rtyper.lltypesystem import rffi - return rffi.cast(lltype.Signed, p) - - if stm_is_enabled(): - from rpython.rtyper.lltypesystem.lloperation import llop - res = llop.stm_allocate_nonmovable_int_adr(lltype.Signed, p) - return res - + return i = 0 while can_move(p): if i > 6: @@ -116,8 +105,6 @@ collect(i) i += 1 - return 0 - def _heap_stats(): raise NotImplementedError # can't be run directly diff --git a/rpython/rlib/test/test_rgc.py b/rpython/rlib/test/test_rgc.py --- a/rpython/rlib/test/test_rgc.py +++ b/rpython/rlib/test/test_rgc.py @@ -228,5 +228,3 @@ x1 = X() n = rgc.get_rpy_memory_usage(rgc.cast_instance_to_gcref(x1)) assert n >= 8 and n <= 64 - - From noreply at buildbot.pypy.org Thu Mar 20 11:11:13 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 20 Mar 2014 11:11:13 +0100 (CET) Subject: [pypy-commit] pypy default: tweaks; reducing the diff with stm Message-ID: <20140320101113.1BC6D1C01DE@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r70106:37af0c2ee363 Date: 2014-03-20 11:06 +0100 http://bitbucket.org/pypy/pypy/changeset/37af0c2ee363/ Log: tweaks; reducing the diff with stm diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -287,7 +287,6 @@ cast_instance_to_gcref(self.cpu.propagate_exception_descr)) ofs = self.cpu.get_ofs_of_frame_field('jf_descr') self.mc.MOV(RawEbpLoc(ofs), imm(propagate_exception_descr)) - self.mc.MOV_rr(eax.value, ebp.value) # self._call_footer() rawstart = self.mc.materialize(self.cpu.asmmemmgr, []) @@ -761,6 +760,9 @@ # def _call_footer(self): + # the return value is the jitframe + self.mc.MOV_rr(eax.value, ebp.value) + gcrootmap = self.cpu.gc_ll_descr.gcrootmap if gcrootmap and gcrootmap.is_shadow_stack: self._call_footer_shadowstack(gcrootmap) @@ -1827,8 +1829,6 @@ mc.MOV_br(ofs2, eax.value) mc.POP(eax) mc.MOV_br(ofs, eax.value) - # the return value is the jitframe - mc.MOV_rr(eax.value, ebp.value) self._call_footer() rawstart = mc.materialize(self.cpu.asmmemmgr, []) @@ -1861,7 +1861,6 @@ # keep that one and kill all the others ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') self.mc.MOV_bi(ofs, 0) - self.mc.MOV_rr(eax.value, ebp.value) # exit function self._call_footer() From noreply at buildbot.pypy.org Thu Mar 20 11:11:14 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 20 Mar 2014 11:11:14 +0100 (CET) Subject: [pypy-commit] pypy default: simplify Message-ID: <20140320101114.60E471C01DE@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r70107:c548a82ba9d3 Date: 2014-03-20 11:07 +0100 http://bitbucket.org/pypy/pypy/changeset/c548a82ba9d3/ Log: simplify diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -1825,10 +1825,8 @@ # did just above. ofs = self.cpu.get_ofs_of_frame_field('jf_descr') ofs2 = self.cpu.get_ofs_of_frame_field('jf_gcmap') - mc.POP(eax) - mc.MOV_br(ofs2, eax.value) - mc.POP(eax) - mc.MOV_br(ofs, eax.value) + mc.POP_b(ofs2) + mc.POP_b(ofs) self._call_footer() rawstart = mc.materialize(self.cpu.asmmemmgr, []) From noreply at buildbot.pypy.org Thu Mar 20 11:12:45 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 20 Mar 2014 11:12:45 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Revert changes to 'make_sure_does_not_move(faildescr)' Message-ID: <20140320101245.C44411C01DE@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r70108:a68436ce38ec Date: 2014-03-20 11:12 +0100 http://bitbucket.org/pypy/pypy/changeset/a68436ce38ec/ Log: Revert changes to 'make_sure_does_not_move(faildescr)' diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py --- a/rpython/jit/backend/arm/assembler.py +++ b/rpython/jit/backend/arm/assembler.py @@ -623,7 +623,7 @@ ops_offset = self.mc.ops_offset if logger is not None: - logger.log_loop(inputargs, operations, looptoken.number, "rewritten", + logger.log_loop(inputargs, operations, 0, "rewritten", name=loopname, ops_offset=ops_offset) self.teardown() diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py --- a/rpython/jit/backend/arm/regalloc.py +++ b/rpython/jit/backend/arm/regalloc.py @@ -630,10 +630,10 @@ # the frame is in fp, but we have to point where in the frame is # the potential argument to FINISH descr = op.getdescr() - fail_descr = rgc.cast_instance_to_gcref(descr) + fail_descr = cast_instance_to_gcref(descr) # we know it does not move, but well - fail_descr = rgc._make_sure_does_not_move(fail_descr) - fail_descr = rgc.cast_gcref_to_int(fail_descr) + rgc._make_sure_does_not_move(fail_descr) + fail_descr = rffi.cast(lltype.Signed, fail_descr) if op.numargs() == 1: loc = self.make_sure_var_in_reg(op.getarg(0)) locs = [loc, imm(fail_descr)] diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py --- a/rpython/jit/backend/llsupport/assembler.py +++ b/rpython/jit/backend/llsupport/assembler.py @@ -182,6 +182,8 @@ break exc = guardtok.exc target = self.failure_recovery_code[exc + 2 * withfloats] + fail_descr = cast_instance_to_gcref(guardtok.faildescr) + fail_descr = rffi.cast(lltype.Signed, fail_descr) base_ofs = self.cpu.get_baseofs_of_frame_field() positions = [0] * len(guardtok.fail_locs) for i, loc in enumerate(guardtok.fail_locs): @@ -204,8 +206,6 @@ guardtok.faildescr.rd_locs = positions # we want the descr to keep alive guardtok.faildescr.rd_loop_token = self.current_clt - fail_descr = rgc.cast_instance_to_gcref(guardtok.faildescr) - fail_descr = rgc._make_sure_does_not_move(fail_descr) return fail_descr, target def call_assembler(self, op, guard_op, argloc, vloc, result_loc, tmploc): @@ -236,8 +236,9 @@ else: raise AssertionError(kind) - gcref = rgc.cast_instance_to_gcref(value) - value = rgc._make_sure_does_not_move(gcref) + gcref = cast_instance_to_gcref(value) + rgc._make_sure_does_not_move(gcref) + value = rffi.cast(lltype.Signed, gcref) je_location = self._call_assembler_check_descr(value, tmploc) # # Path A: use assembler_helper_adr diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py --- a/rpython/jit/backend/llsupport/gc.py +++ b/rpython/jit/backend/llsupport/gc.py @@ -5,8 +5,7 @@ from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, rclass, rstr from rpython.rtyper.lltypesystem import llgroup from rpython.rtyper.lltypesystem.lloperation import llop -from rpython.rtyper.annlowlevel import (llhelper, cast_instance_to_gcref, - cast_base_ptr_to_instance) +from rpython.rtyper.annlowlevel import llhelper, cast_instance_to_gcref from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.jit.codewriter import heaptracker from rpython.jit.metainterp.history import ConstPtr, AbstractDescr @@ -97,17 +96,12 @@ for i in range(op.numargs()): v = op.getarg(i) if isinstance(v, ConstPtr) and bool(v.value): - v.imm_value = rgc._make_sure_does_not_move(v.value) - # XXX: fix for stm, record imm_values and unregister - # them again (below too): - gcrefs_output_list.append(v.value) - - if self.stm: - return # for descr, we do it on the fly in assembler.py + p = v.value + rgc._make_sure_does_not_move(p) + gcrefs_output_list.append(p) if op.is_guard() or op.getopnum() == rop.FINISH: # the only ops with descrs that get recorded in a trace - descr = op.getdescr() - llref = rgc.cast_instance_to_gcref(descr) + llref = cast_instance_to_gcref(op.getdescr()) rgc._make_sure_does_not_move(llref) gcrefs_output_list.append(llref) diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -341,8 +341,8 @@ self._store_and_reset_exception(self.mc, eax) ofs = self.cpu.get_ofs_of_frame_field('jf_guard_exc') self.mc.MOV_br(ofs, eax.value) - propagate_exception_descr = rgc._make_sure_does_not_move( - rgc.cast_instance_to_gcref(self.cpu.propagate_exception_descr)) + propagate_exception_descr = rffi.cast(lltype.Signed, + cast_instance_to_gcref(self.cpu.propagate_exception_descr)) ofs = self.cpu.get_ofs_of_frame_field('jf_descr') self.mc.MOV(RawEbpLoc(ofs), imm(propagate_exception_descr)) # @@ -2298,10 +2298,10 @@ cb.emit() def _store_force_index(self, guard_op): - faildescr = rgc._make_sure_does_not_move( - rgc.cast_instance_to_gcref(guard_op.getdescr())) + faildescr = guard_op.getdescr() ofs = self.cpu.get_ofs_of_frame_field('jf_force_descr') - self.mc.MOV(raw_stack(ofs), imm(faildescr)) + self.mc.MOV(raw_stack(ofs), imm(rffi.cast(lltype.Signed, + cast_instance_to_gcref(faildescr)))) def _emit_guard_not_forced(self, guard_token): ofs = self.cpu.get_ofs_of_frame_field('jf_descr') diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -369,9 +369,10 @@ # the frame is in ebp, but we have to point where in the frame is # the potential argument to FINISH descr = op.getdescr() - fail_descr = rgc.cast_instance_to_gcref(descr) + fail_descr = cast_instance_to_gcref(descr) # we know it does not move, but well - fail_descr = rgc._make_sure_does_not_move(fail_descr) + rgc._make_sure_does_not_move(fail_descr) + fail_descr = rffi.cast(lltype.Signed, fail_descr) if op.numargs() == 1: loc = self.make_sure_var_in_reg(op.getarg(0)) locs = [loc, imm(fail_descr)] From noreply at buildbot.pypy.org Thu Mar 20 11:17:16 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 20 Mar 2014 11:17:16 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: stm_ptr_eq dies Message-ID: <20140320101716.76EC11C08F3@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r70109:1f090ff1b51f Date: 2014-03-20 11:16 +0100 http://bitbucket.org/pypy/pypy/changeset/1f090ff1b51f/ Log: stm_ptr_eq dies diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py --- a/rpython/jit/backend/llsupport/gc.py +++ b/rpython/jit/backend/llsupport/gc.py @@ -706,21 +706,9 @@ if self.stm: # XXX remove the indirections in the following calls from rpython.rlib import rstm - def stm_allocate_nonmovable_int_adr(obj): - return llop1.stm_allocate_nonmovable_int_adr( - lltype.Signed, obj) - self.generate_function('stm_allocate_nonmovable_int_adr', - stm_allocate_nonmovable_int_adr, - [llmemory.GCREF], RESULT=lltype.Signed) self.generate_function('stm_try_inevitable', rstm.become_inevitable, [], RESULT=lltype.Void) - def ptr_eq(x, y): return x == y - def ptr_ne(x, y): return x != y - self.generate_function('stm_ptr_eq', ptr_eq, [llmemory.GCREF] * 2, - RESULT=lltype.Bool) - self.generate_function('stm_ptr_ne', ptr_ne, [llmemory.GCREF] * 2, - RESULT=lltype.Bool) def _bh_malloc(self, sizedescr): from rpython.memory.gctypelayout import check_typeid diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -1253,58 +1253,6 @@ self.mc.LEA_rm(result_loc.value, (loc.value, delta)) return genop_binary_or_lea - - def genop_ptr_eq(self, op, arglocs, result_loc): - if not self.cpu.gc_ll_descr.stm: - self.genop_int_eq(op, arglocs, result_loc) - return - assert self.cpu.gc_ll_descr.stm - rl = result_loc.lowest8bits() - self._stm_ptr_eq_fastpath(self.mc, arglocs, result_loc) - self.mc.SET_ir(rx86.Conditions['NZ'], rl.value) - self.mc.MOVZX8_rr(result_loc.value, rl.value) - - def genop_ptr_ne(self, op, arglocs, result_loc): - if not self.cpu.gc_ll_descr.stm: - self.genop_int_ne(op, arglocs, result_loc) - return - assert self.cpu.gc_ll_descr.stm - rl = result_loc.lowest8bits() - self._stm_ptr_eq_fastpath(self.mc, arglocs, result_loc) - self.mc.SET_ir(rx86.Conditions['Z'], rl.value) - self.mc.MOVZX8_rr(result_loc.value, rl.value) - - def genop_guard_ptr_eq(self, op, guard_op, guard_token, - arglocs, result_loc): - if not self.cpu.gc_ll_descr.stm: - self.genop_guard_int_eq(op, guard_op, guard_token, - arglocs, result_loc) - return - assert self.cpu.gc_ll_descr.stm - guard_opnum = guard_op.getopnum() - self._stm_ptr_eq_fastpath(self.mc, arglocs, result_loc) - # p1==p2 -> NZ - if guard_opnum == rop.GUARD_FALSE: - # jump to failure-code if ptrs are equal - self.implement_guard(guard_token, "NZ") - else: - # jump to failure-code if ptrs are not equal - self.implement_guard(guard_token, "Z") - - def genop_guard_ptr_ne(self, op, guard_op, guard_token, - arglocs, result_loc): - if not self.cpu.gc_ll_descr.stm: - self.genop_guard_int_ne(op, guard_op, guard_token, - arglocs, result_loc) - return - assert self.cpu.gc_ll_descr.stm - guard_opnum = guard_op.getopnum() - self._stm_ptr_eq_fastpath(self.mc, arglocs, result_loc) - if guard_opnum == rop.GUARD_FALSE: - self.implement_guard(guard_token, "Z") - else: - self.implement_guard(guard_token, "NZ") - def _cmpop(cond, rev_cond): def genop_cmp(self, op, arglocs, result_loc): rl = result_loc.lowest8bits() @@ -1453,8 +1401,8 @@ genop_int_ne = _cmpop("NE", "NE") genop_int_gt = _cmpop("G", "L") genop_int_ge = _cmpop("GE", "LE") - genop_instance_ptr_eq = genop_ptr_eq - genop_instance_ptr_ne = genop_ptr_ne + genop_ptr_eq = genop_instance_ptr_eq = genop_int_eq + genop_ptr_ne = genop_instance_ptr_ne = genop_int_ne genop_float_lt = _cmpop_float('B', 'A') genop_float_le = _cmpop_float('BE', 'AE') @@ -1474,8 +1422,8 @@ genop_guard_int_ne = _cmpop_guard("NE", "NE", "E", "E") genop_guard_int_gt = _cmpop_guard("G", "L", "LE", "GE") genop_guard_int_ge = _cmpop_guard("GE", "LE", "L", "G") - genop_guard_instance_ptr_eq = genop_guard_ptr_eq - genop_guard_instance_ptr_ne = genop_guard_ptr_ne + genop_guard_ptr_eq = genop_guard_instance_ptr_eq = genop_guard_int_eq + genop_guard_ptr_ne = genop_guard_instance_ptr_ne = genop_guard_int_ne genop_guard_uint_gt = _cmpop_guard("A", "B", "BE", "AE") genop_guard_uint_lt = _cmpop_guard("B", "A", "AE", "BE") From noreply at buildbot.pypy.org Thu Mar 20 11:24:11 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 20 Mar 2014 11:24:11 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: sorry :-/ cond_call_stm_b, stm_set_revision_gc, and more stm_ptr_eq all die Message-ID: <20140320102411.2A4551C08F3@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r70110:26e595d6e017 Date: 2014-03-20 11:23 +0100 http://bitbucket.org/pypy/pypy/changeset/26e595d6e017/ Log: sorry :-/ cond_call_stm_b, stm_set_revision_gc, and more stm_ptr_eq all die diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py --- a/rpython/jit/backend/llgraph/runner.py +++ b/rpython/jit/backend/llgraph/runner.py @@ -1029,9 +1029,6 @@ def execute_cond_call_gc_wb_array(self, descr, a, b): py.test.skip("cond_call_gc_wb_array not supported") - def execute_cond_call_stm_b(self, descr, a): - py.test.skip("cond_call_stm_b not supported") - def execute_stm_transaction_break(self, _, really_wanted): pass diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -2374,257 +2374,7 @@ self.mc.overwrite(jmp_location - 1, chr(offset)) # ------------------- END CALL ASSEMBLER ----------------------- - def _stm_ptr_eq_fastpath(self, mc, arglocs, result_loc): - assert self.cpu.gc_ll_descr.stm - assert self.ptr_eq_slowpath is not None - a_base = arglocs[0] - b_base = arglocs[1] - if isinstance(a_base, ImmedLoc): - # make sure there is a non-immed as the first - # argument to mc.CMP(). (2 immeds are caught below) - a_base, b_base = b_base, a_base - - # - # FASTPATH - # - # a == b -> SET NZ - if isinstance(a_base, ImmedLoc) and isinstance(b_base, ImmedLoc): - if a_base.getint() == b_base.getint(): - mc.MOV_ri(X86_64_SCRATCH_REG.value, 1) - mc.TEST(X86_64_SCRATCH_REG, X86_64_SCRATCH_REG) # NZ flag - mc.JMP_l8(0) - j_ok1 = mc.get_relative_pos() - else: - j_ok1 = 0 - else: - mc.CMP(a_base, b_base) - # reverse flags: if p1==p2, set NZ - sl = X86_64_SCRATCH_REG.lowest8bits() - mc.SET_ir(rx86.Conditions['Z'], sl.value) - mc.TEST8_rr(sl.value, sl.value) - mc.J_il8(rx86.Conditions['NZ'], 0) - j_ok1 = mc.get_relative_pos() - skip = False - # a == 0 || b == 0 -> SET Z - j_ok2 = 0 - if isinstance(a_base, ImmedLoc): - if a_base.getint() == 0: - # set Z flag: - mc.XOR(X86_64_SCRATCH_REG, X86_64_SCRATCH_REG) - skip = True - else: - mc.CMP(a_base, imm(0)) - mc.J_il8(rx86.Conditions['Z'], 0) - j_ok2 = mc.get_relative_pos() - # - j_ok3 = 0 - if not skip: - if isinstance(b_base, ImmedLoc): - if b_base.getint() == 0: - # set Z flag: - mc.XOR(X86_64_SCRATCH_REG, X86_64_SCRATCH_REG) - skip = True - else: - mc.CMP(b_base, imm(0)) - mc.J_il8(rx86.Conditions['Z'], 0) - j_ok3 = mc.get_relative_pos() - # a.type != b.type - # XXX: todo, if it ever happens.. - - # - # SLOWPATH - # - if not skip: - mc.PUSH(b_base) - mc.PUSH(a_base) - func = self.ptr_eq_slowpath - mc.CALL(imm(func)) - # result still on stack - mc.POP_r(X86_64_SCRATCH_REG.value) - # _Bool return type only sets lower 8 bits of return value - sl = X86_64_SCRATCH_REG.lowest8bits() - mc.TEST8_rr(sl.value, sl.value) - # - # END SLOWPATH - # - - # OK: flags already set - if j_ok1: - offset = mc.get_relative_pos() - j_ok1 - assert 0 <= offset <= 127 - mc.overwrite(j_ok1 - 1, chr(offset)) - if j_ok2: - offset = mc.get_relative_pos() - j_ok2 - assert 0 <= offset <= 127 - mc.overwrite(j_ok2 - 1, chr(offset)) - if j_ok3: - offset = mc.get_relative_pos() - j_ok3 - assert 0 <= offset <= 127 - mc.overwrite(j_ok3 - 1, chr(offset)) - - def _get_stm_private_rev_num_addr(self): - return self._get_stm_tl(rstm.get_adr_of_private_rev_num()) - - def _get_stm_read_barrier_cache_addr(self): - return self._get_stm_tl(rstm.get_adr_of_read_barrier_cache()) - - def _stm_barrier_fastpath(self, mc, descr, arglocs, is_frame=False, - align_stack=False): - assert self.cpu.gc_ll_descr.stm - if we_are_translated(): - # tests use a a mock class, but translation needs it - assert isinstance(descr, STMBarrierDescr) - assert descr.returns_modified_object - loc_base = arglocs[0] - assert isinstance(loc_base, RegLoc) - - helper_num = 0 - if is_frame: - helper_num = 4 - elif self._regalloc is not None and self._regalloc.xrm.reg_bindings: - helper_num += 2 - # - # FASTPATH: - # do slowpath IF: - # A2W: - # (obj->h_revision != stm_private_rev_num) - # || (obj->h_tid & GCFLAG_WRITE_BARRIER) != 0) - # V2W: - # (obj->h_tid & GCFLAG_WRITE_BARRIER) != 0) - # A2V: - # (obj->h_revision != stm_private_rev_num) - # A2R: - # (obj->h_revision != stm_private_rev_num) - # && (FXCACHE_AT(obj) != obj))) - # Q2R: - # (obj->h_tid & (GCFLAG_PUBLIC_TO_PRIVATE | GCFLAG_MOVED) != 0) - # A2I: - # (obj->h_tid & GCFLAG_STUB) - if IS_X86_32: # XXX: todo - todo() - jz_location = 0 - jz_location2 = 0 - jnz_location = 0 - # compare h_revision with stm_private_rev_num - if descr.stmcat in ['A2W', 'A2R', 'A2V']: - rn = self._get_stm_private_rev_num_addr() - if we_are_translated(): - # during tests, _get_stm_private_rev_num_addr returns - # an absolute address, not a tl-offset - self._tl_segment_if_stm(mc) - mc.MOV_rj(X86_64_SCRATCH_REG.value, rn) - else: # testing: - mc.MOV(X86_64_SCRATCH_REG, heap(rn)) - - if loc_base == ebp: - mc.CMP_rb(X86_64_SCRATCH_REG.value, StmGC.H_REVISION) - else: - mc.CMP(X86_64_SCRATCH_REG, mem(loc_base, StmGC.H_REVISION)) - # - if descr.stmcat in ('A2R', 'A2V'): - # jump to end if h_rev==priv_rev - mc.J_il8(rx86.Conditions['Z'], 0) # patched below - jz_location = mc.get_relative_pos() - else: # A2W - # jump to slowpath if h_rev!=priv_rev - mc.J_il8(rx86.Conditions['NZ'], 0) # patched below - jnz_location = mc.get_relative_pos() - # - # FXCACHE_AT(obj) != obj - if descr.stmcat == 'A2R': - # calculate: temp = obj & FX_MASK - assert StmGC.FX_MASK == 65535 - assert not is_frame - mc.MOVZX16(X86_64_SCRATCH_REG, loc_base) - # calculate: rbc + temp == obj - rbc = self._get_stm_read_barrier_cache_addr() - if we_are_translated(): - # during tests, _get_stm_rbca returns - # an absolute address, not a tl-offset - self._tl_segment_if_stm(mc) - mc.ADD_rj(X86_64_SCRATCH_REG.value, rbc) - else: # testing: - mc.PUSH_r(eax.value) - mc.MOV_ri(eax.value, rbc) - mc.MOV_rm(eax.value, (eax.value, 0)) - mc.ADD(X86_64_SCRATCH_REG, eax) - mc.POP_r(eax.value) - mc.CMP_rm(loc_base.value, (X86_64_SCRATCH_REG.value, 0)) - mc.J_il8(rx86.Conditions['Z'], 0) # patched below - jz_location2 = mc.get_relative_pos() - # - # check flags: - if descr.stmcat in ['A2W', 'V2W', 'Q2R', 'A2I']: - flags = 0 - off = 0 - if descr.stmcat in ['A2W', 'V2W']: - # obj->h_tid & GCFLAG_WRITE_BARRIER) != 0 - flags = StmGC.GCFLAG_WRITE_BARRIER - elif descr.stmcat == 'Q2R': - # obj->h_tid & PUBLIC_TO_PRIVATE|MOVED - flags = StmGC.GCFLAG_PUBLIC_TO_PRIVATE | StmGC.GCFLAG_MOVED - elif descr.stmcat == 'A2I': - # obj->h_tid & STUB - flags = StmGC.GCFLAG_STUB - - assert IS_X86_64 - if (flags >> 32) > 0 and (flags >> 40) == 0: - flags = flags >> 32 - off = 4 - elif (flags >> 40) > 0 and (flags >> 48) == 0: - flags = flags >> 40 - off = 5 - # - if loc_base == ebp: - mc.TEST8_bi(StmGC.H_TID + off, flags) - else: - mc.TEST8_mi((loc_base.value, StmGC.H_TID + off), flags) - - mc.J_il8(rx86.Conditions['Z'], 0) # patched below - jz_location = mc.get_relative_pos() - # if flags not set, jump to end - # jump target slowpath: - if jnz_location: - offset = mc.get_relative_pos() - jnz_location - assert 0 < offset <= 127 - mc.overwrite(jnz_location - 1, chr(offset)) - # - # SLOWPATH_START - # - if not is_frame: - mc.PUSH(loc_base) - elif is_frame and align_stack: - # ||retadr| - mc.SUB_ri(esp.value, 16 - WORD) # erase the return address - # ||retadr|...|| - func = descr.get_b_slowpath(helper_num) - assert func != 0 - mc.CALL(imm(func)) - # get result: - if is_frame: - # result already written back to ebp - assert loc_base is ebp - else: - # result where argument was: - mc.POP_r(loc_base.value) - - if is_frame and align_stack: - mc.ADD_ri(esp.value, 16 - WORD) # erase the return address - # - # SLOWPATH_END - # - # jump target end: - offset = mc.get_relative_pos() - jz_location - assert 0 < offset <= 127 - mc.overwrite(jz_location - 1, chr(offset)) - if descr.stmcat == 'A2R':#isinstance(descr, STMReadBarrierDescr): - offset = mc.get_relative_pos() - jz_location2 - assert 0 < offset <= 127 - mc.overwrite(jz_location2 - 1, chr(offset)) - - - def _write_barrier_fastpath(self, mc, descr, arglocs, array=False, is_frame=False, align_stack=False): # Write code equivalent to write_barrier() in the GC: it checks @@ -2750,9 +2500,6 @@ self._write_barrier_fastpath(self.mc, op.getdescr(), arglocs, array=True) - def genop_discard_cond_call_stm_b(self, op, arglocs): - self._stm_barrier_fastpath(self.mc, op.getdescr(), arglocs) - def not_implemented_op_discard(self, op, arglocs): not_implemented("not implemented operation: %s" % op.getopname()) @@ -3114,21 +2861,6 @@ assert isinstance(reg, RegLoc) self.mc.MOV_rr(reg.value, ebp.value) - def genop_discard_stm_set_revision_gc(self, op, arglocs): - base_loc, ofs_loc, size_loc = arglocs - assert isinstance(size_loc, ImmedLoc) - mc = self.mc - - if IS_X86_32: - todo() - - rn = self._get_stm_private_rev_num_addr() - self._tl_segment_if_stm(mc) - mc.MOV_rj(X86_64_SCRATCH_REG.value, rn) - - dest_addr = AddressLoc(base_loc, ofs_loc) - mc.MOV(dest_addr, X86_64_SCRATCH_REG) - def genop_guard_stm_transaction_break(self, op, guard_op, guard_token, arglocs, result_loc): assert self.cpu.gc_ll_descr.stm diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -801,21 +801,6 @@ for i in range(N)] self.perform_discard(op, arglocs) - def consider_cond_call_stm_b(self, op): - assert op.result is None - # we force all arguments in a reg (unless they are Consts), - # because it will be needed anyway by the following setfield_gc - # or setarrayitem_gc. It avoids loading it twice from the memory. - arg = op.getarg(0) - argloc = self.rm.make_sure_var_in_reg(arg) - self.perform_discard(op, [argloc]) - - # if 'arg' is in two locations (once in argloc and once spilled - # on the frame), we need to ensure that both locations are - # updated with the possibly changed reference. - self.rm.update_spill_loc_if_necessary(arg, argloc) - - consider_cond_call_gc_wb_array = consider_cond_call_gc_wb def consider_cond_call(self, op): @@ -1265,17 +1250,7 @@ if isinstance(loc, FrameLoc): self.fm.hint_frame_pos[box] = self.fm.get_loc_index(loc) - - def consider_stm_set_revision_gc(self, op): - ofs, size, _ = unpack_fielddescr(op.getdescr()) - ofs_loc = imm(ofs) - size_loc = imm(size) - assert isinstance(size_loc, ImmedLoc) - args = op.getarglist() - base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args) - self.perform_discard(op, [base_loc, ofs_loc, size_loc]) - def consider_stm_transaction_break(self, op, guard_op): # # only save regs for the should_break_transaction call diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py --- a/rpython/jit/metainterp/executor.py +++ b/rpython/jit/metainterp/executor.py @@ -335,7 +335,6 @@ rop.INCREMENT_DEBUG_COUNTER, rop.COND_CALL_GC_WB, rop.COND_CALL_GC_WB_ARRAY, - rop.COND_CALL_STM_B, rop.DEBUG_MERGE_POINT, rop.JIT_DEBUG, rop.SETARRAYITEM_RAW, @@ -347,7 +346,6 @@ rop.CALL_MALLOC_NURSERY_VARSIZE, rop.CALL_MALLOC_NURSERY_VARSIZE_FRAME, rop.LABEL, - rop.STM_SET_REVISION_GC, ): # list of opcodes never executed by pyjitpl continue raise AssertionError("missing %r" % (key,)) diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py --- a/rpython/jit/metainterp/resoperation.py +++ b/rpython/jit/metainterp/resoperation.py @@ -499,7 +499,6 @@ 'SETFIELD_RAW/2d', 'STRSETITEM/3', 'UNICODESETITEM/3', - 'COND_CALL_STM_B/1d', # objptr (write/read barrier) 'COND_CALL_GC_WB/1d', # [objptr] (for the write barrier) 'COND_CALL_GC_WB_ARRAY/2d', # [objptr, arrayindex] (write barr. for array) 'DEBUG_MERGE_POINT/*', # debugging only @@ -511,7 +510,6 @@ 'RECORD_KNOWN_CLASS/2', # [objptr, clsptr] 'KEEPALIVE/1', 'STM_TRANSACTION_BREAK/1', - 'STM_SET_REVISION_GC/1d', # not really GC, writes raw to the header '_CANRAISE_FIRST', # ----- start of can_raise operations ----- '_CALL_FIRST', From noreply at buildbot.pypy.org Thu Mar 20 11:32:23 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 20 Mar 2014 11:32:23 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: fix tests here Message-ID: <20140320103223.B04531D24ED@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r70111:60e88988d828 Date: 2014-03-20 11:31 +0100 http://bitbucket.org/pypy/pypy/changeset/60e88988d828/ Log: fix tests here diff --git a/rpython/jit/codewriter/test/test_call.py b/rpython/jit/codewriter/test/test_call.py --- a/rpython/jit/codewriter/test/test_call.py +++ b/rpython/jit/codewriter/test/test_call.py @@ -207,7 +207,7 @@ T = rffi.CArrayPtr(rffi.TIME_T) external = rffi.llexternal("time", [T], rffi.TIME_T, _nowrapper=True, - threadsafe=False, + releasegil=False, transactionsafe=transactionsafe) @jit.dont_look_inside diff --git a/rpython/jit/codewriter/test/test_jtransform.py b/rpython/jit/codewriter/test/test_jtransform.py --- a/rpython/jit/codewriter/test/test_jtransform.py +++ b/rpython/jit/codewriter/test/test_jtransform.py @@ -921,6 +921,7 @@ assert block.exits[0].args == [v1] def test_jit_stm_transaction_break_point(): + py.test.skip("XXX?") op = SpaceOperation('jit_stm_transaction_break_point', [Constant(1, lltype.Signed)], lltype.Void) tr = Transformer() From noreply at buildbot.pypy.org Thu Mar 20 13:16:01 2014 From: noreply at buildbot.pypy.org (anton_gulenko) Date: Thu, 20 Mar 2014 13:16:01 +0100 (CET) Subject: [pypy-commit] lang-smalltalk strategies-tagging: Refactoring to remove rerased and use two separate variables for wrapped and unwrapped storage. Message-ID: <20140320121601.D7D741C08F3@cobra.cs.uni-duesseldorf.de> Author: Anton Gulenko Branch: strategies-tagging Changeset: r669:f63aa764c7bf Date: 2014-03-20 11:39 +0100 http://bitbucket.org/pypy/lang-smalltalk/changeset/f63aa764c7bf/ Log: Refactoring to remove rerased and use two separate variables for wrapped and unwrapped storage. diff --git a/spyvm/model.py b/spyvm/model.py --- a/spyvm/model.py +++ b/spyvm/model.py @@ -653,16 +653,15 @@ strategy_stats = StrategyStatistics() class W_PointersObject(W_AbstractPointersObject): - _attrs_ = ['_storage', 'strategy'] + _attrs_ = ['_size', '_storage', 'int_storage', 'strategy'] @jit.unroll_safe def __init__(self, space, w_class, size): from spyvm.strategies import strategy_of_size """Create new object with size = fixed + variable size.""" W_AbstractPointersObject.__init__(self, space, w_class, size) - # TODO - setting strategy/storage is useless if fillin() will be called afterwards. self.strategy = strategy_of_size(self.s_class, size) - self.set_storage(self.strategy.initial_storage(space, size)) + self.set_storage(space, size) self.log_strategy_operation("Initialized") def log_strategy_operation(self, op, old_strategy=None): @@ -680,11 +679,12 @@ if strategy_stats.do_log: strategy_stats.log_operation(op, new_strategy_tag, old_strategy_tag, classname, size) - def set_storage(self, storage): - self._storage = storage - - def get_storage(self): - return self._storage + def set_storage(self, space, size): + self._size = size + if self.strategy.uses_int_storage: + self.int_storage = self.strategy.initial_int_storage(space, size) + else: + self._storage = self.strategy.initial_storage(space, size) def get_strategy(self): return self.strategy @@ -692,7 +692,11 @@ def fillin_pointers(self, space, collection): from spyvm.strategies import strategy_for_list self.strategy = strategy_for_list(self.s_class, collection) - self.set_storage(self.strategy.storage_for_list(space, collection)) + self._size = len(collection) + if self.strategy.uses_int_storage: + self.int_storage = self.strategy.int_storage_for_list(space, collection) + else: + self._storage = self.strategy.storage_for_list(space, collection) def fillin(self, space, g_self): W_AbstractPointersObject.fillin(self, space, g_self) @@ -701,10 +705,12 @@ def switch_strategy(self, space, new_strategy): assert self.strategy != new_strategy - new_storage = new_strategy.copy_storage_from(space, self, reuse_storage=True) + if new_strategy.uses_int_storage: + self.int_storage = new_strategy.copy_int_storage_from(space, self, reuse_storage=True) + else: + self._storage = new_strategy.copy_storage_from(space, self, reuse_storage=True) old_strategy = self.strategy self.strategy = new_strategy - self.set_storage(new_storage) self.log_strategy_operation("Switched", old_strategy) def store_with_new_strategy(self, space, new_strategy, n0, w_val): @@ -735,15 +741,15 @@ return self.get_strategy().store(space, self, n0, w_value) def basic_size(self): - return self.get_strategy().size_of(self) + return self._size def become(self, w_other): if not isinstance(w_other, W_PointersObject): return False self.strategy, w_other.strategy = w_other.strategy, self.strategy - self_storage = self._storage - self.set_storage(w_other._storage) - w_other.set_storage(self_storage) + self._size, w_other._size = w_other._size, self._size + self._storage, w_other._storage = w_other._storage, self._storage + self.int_storage, w_other.int_storage = w_other.int_storage, self.int_storage return W_AbstractPointersObject.become(self, w_other) @jit.unroll_safe diff --git a/spyvm/objspace.py b/spyvm/objspace.py --- a/spyvm/objspace.py +++ b/spyvm/objspace.py @@ -165,7 +165,7 @@ from spyvm.strategies import ListStorageStrategy w_nil.space = self w_nil.strategy = ListStorageStrategy.singleton - w_nil.set_storage(w_nil.strategy.initial_storage(self, 0)) + w_nil.set_storage(self, 0) w_nil.s_class = self.classtable['w_UndefinedObject'].as_class_get_penumbra(self) return w_nil w_nil = self.w_nil = patch_nil(model.w_nil) diff --git a/spyvm/shadow.py b/spyvm/shadow.py --- a/spyvm/shadow.py +++ b/spyvm/shadow.py @@ -432,7 +432,7 @@ self.copy_from_w_self(i) except error.SenderChainManipulation, e: assert e.s_context == self - w_self.set_storage(w_self.strategy.initial_storage(self.space, 0)) + w_self.set_storage(self.space, 0) # def detach_shadow(self): # w_self = self.w_self() diff --git a/spyvm/strategies.py b/spyvm/strategies.py --- a/spyvm/strategies.py +++ b/spyvm/strategies.py @@ -14,6 +14,7 @@ _attrs_ = [] _settled_ = True strategy_tag = 'abstract' + uses_int_storage = False def __init__(self): pass @@ -31,14 +32,23 @@ def initial_storage(self, space, size): raise NotImplementedError("Abstract base class") + def initial_int_storage(self, space, size): + raise NotImplementedError("Abstract base class") def storage_for_list(self, space, collection): raise NotImplementedError("Abstract base class") + def int_storage_for_list(self, space, collection): + raise NotImplementedError("Abstract base class") + def copy_int_storage_from(self, space, w_obj, reuse_storage=False): + old_strategy = w_obj.strategy + if old_strategy == self and reuse_storage: + return w_obj.int_storage + else: + # This can be overridden and optimized (reuse_storage flag, less temporary storage) + return self.int_storage_for_list(space, w_obj.fetch_all(space)) def copy_storage_from(self, space, w_obj, reuse_storage=False): old_strategy = w_obj.strategy if old_strategy == self and reuse_storage: - return w_obj.get_storage() - if isinstance(old_strategy, AllNilStorageStrategy): - return self.initial_storage(space, old_strategy.size_of(w_obj)) + return w_obj._storage else: # This can be overridden and optimized (reuse_storage flag, less temporary storage) return self.storage_for_list(space, w_obj.fetch_all(space)) @@ -51,22 +61,24 @@ class BasicStorageStrategyMixin(object): # Concrete class must implement: unerase + def size_of(self, w_obj): + if self.uses_int_storage: + return len(self.int_storage(w_obj)) + else: + return len(self.storage(w_obj)) + def int_storage(self, w_obj): + return w_obj.int_storage def storage(self, w_obj): - return self.unerase(w_obj.get_storage()) - -# This is a container for an int-value to be used with a rerased-pair -class SizeStorage(object): - _attrs_ = ['size'] - _settled_ = True - def __init__(self, size): - self.size = size + return w_obj._storage + def erase(self, a): return a + def unerase(self, a): return a # this is the typical "initial" storage strategy, for when every slot # in a var-sized object is still nil. No storage is allocated except for # holding the size of the object. class AllNilStorageStrategy(AbstractStorageStrategy): __metaclass__ = SingletonMeta - erase, unerase = rerased.new_static_erasing_pair("all-nil-strategy") + # erase, unerase = rerased.new_static_erasing_pair("all-nil-strategy") import_from_mixin(BasicStorageStrategyMixin) strategy_tag = 'allnil' @@ -82,21 +94,19 @@ return w_obj.store_with_new_strategy(space, TaggingSmallIntegerStorageStrategy.singleton, n0, w_val) return w_obj.store_with_new_strategy(space, ListStorageStrategy.singleton, n0, w_val) - def size_of(self, w_obj): - return self.storage(w_obj).size def initial_storage(self, space, size): - return self.erase(SizeStorage(size)) + return [] def storage_for_list(self, space, collection): - return self.erase(SizeStorage(len(collection))) + return [] def copy_storage_from(self, space, w_obj, reuse_storage=False): - return self.erase(SizeStorage(w_obj.basic_size())) + return [] # This is the regular storage strategy that does not result in any # optimizations but can handle every case. Applicable for both # fixed-sized and var-sized objects. class ListStorageStrategy(AbstractStorageStrategy): __metaclass__ = SingletonMeta - erase, unerase = rerased.new_static_erasing_pair("list-storage-strategy") + # erase, unerase = rerased.new_static_erasing_pair("list-storage-strategy") import_from_mixin(BasicStorageStrategyMixin) strategy_tag = 'list' @@ -105,8 +115,6 @@ def store(self, space, w_obj, n0, w_val): # TODO enable generalization by maintaining a counter of elements that are nil. self.storage(w_obj)[n0] = w_val - def size_of(self, w_obj): - return len(self.storage(w_obj)) def erased_list(self, list): make_sure_not_resized(list) return self.erase(list) @@ -117,147 +125,13 @@ def copy_storage_from(self, space, w_obj, reuse_storage=False): length = w_obj.basic_size() return self.erased_list([w_obj.strategy.fetch(space, w_obj, i) for i in range(length)]) - -class DenseStorage(object): - # Subclass must provide attribute: default_element - _immutable_fields_ = ['arr'] - _attrs_ = ['arr', '_from', '_to'] - _settled_ = True - - def __init__(self, _from, _to, size): - self._from = _from # first used index ("inclusive") - self._to = _to # first unused index ("exclusive") - self.arr = [self.default_element] * size - make_sure_not_resized(self.arr) - -class DenseStorageStrategyMixin(object): - # Concrete class must implement: storage, erase, do_fetch, do_store, sparse_strategy - # Concrete class must provide attributes: storage_type (subclass of DenseStorage) - - def fetch(self, space, w_obj, n0): - store = self.storage(w_obj) - if n0 < store._from or n0 >= store._to: - return model.w_nil - return self.do_fetch(space, store.arr, n0) - def store(self, space, w_obj, n0, w_val): - store = self.storage(w_obj) - if not self.can_contain_object(w_val): - if w_val == model.w_nil: - if store._to - 1 == n0: # Optimize Collection >> remove: - store._to = store._to - 1 - elif n0 < store._from or store._to <= n0: - pass # Storing nil to an already-nil position - elif store._from == n0: - store._from = store._from + 1 - else: - # Deletion from the middle of the storage. Deoptimize to sparse storage. - return w_obj.store_with_new_strategy(space, self.sparse_strategy().singleton, n0, w_val) - if store._from == store._to: - # Deleted last element. Generelize to AllNilStorage. - w_obj.switch_strategy(space, AllNilStorageStrategy.singleton) - return - else: - # Storing a non-int - dehomogenize to ListStorage - return w_obj.store_with_new_strategy(space, ListStorageStrategy.singleton, n0, w_val) - if n0 == store._to: # Optimize Collection >> add: - store._to = store._to+1 - elif store._from <= n0 and n0 < store._to: - pass - elif n0 == store._from - 1: # It's ok if this wraps around. - store._from = store._from-1 - else: - if store._from == store._to: - # Initial store to non-zero position. - store._from = n0 - store._to = n0+1 - else: - # Store to a non-dense position. Deoptimize to sparse storage. - return w_obj.store_with_new_strategy(space, self.sparse_strategy().singleton, n0, w_val) - # It is a dense store, so finally store the unwrapped value. - self.do_store(space, store.arr, n0, w_val) - def initial_storage(self, space, size): - return self.erase(self.storage_type(0, 0, size)) - def storage_for_list(self, space, collection): - _from = 0 - while _from < len(collection) and collection[_from] == model.w_nil: - _from = _from+1 - _to = _from - while _to < len(collection) and collection[_to] != model.w_nil: - _to = _to+1 - store = self.storage_type(_from, _to, len(collection)) - for i in range(_from, _to): - self.do_store(space, store.arr, i, collection[i]) - return self.erase(store) - -class SparseStorage(object): - _immutable_fields_ = ['arr', 'nil_flags'] - _attrs_ = ['arr', 'nil_flags'] - _settled_ = True - - def __init__(self, arr, nil_flags): - self.arr = arr - self.nil_flags = nil_flags - make_sure_not_resized(self.arr) - make_sure_not_resized(self.nil_flags) - -class SparseStorageStrategyMixin(object): - # Concrete class must implement: storage, erase, do_fetch, do_store, dense_strategy - # Concrete class must provide attributes: storage_type (Subclass of SparseStorage) - - def fetch(self, space, w_obj, n0): - store = self.storage(w_obj) - if store.nil_flags[n0]: - return model.w_nil - return self.do_fetch(space, store.arr, n0) - def store(self, space, w_obj, n0, w_val): - store = self.storage(w_obj) - if not self.can_contain_object(w_val): - if w_val == model.w_nil: - # TODO - generelize to AllNilStorage by maintaining a counter of nil-elements - store.nil_flags[n0] = True - return - else: - # Storing a wrong type - dehomogenize to ListStorage - return w_obj.store_with_new_strategy(space, ListStorageStrategy.singleton, n0, w_val) - store.nil_flags[n0] = False - self.do_store(space, store.arr, n0, w_val) - def storage_for_size(self, size): - # TODO -- for inlining strategy, the size must be extended!! - # size = size * self.slots_per_object() - return self.storage_type([self.storage_type.default_element] * size, [True] * size) - def initial_storage(self, space, size): - return self.erase(self.storage_for_size(size)) - def storage_for_list(self, space, collection): - length = len(collection) - store = self.storage_for_size(length) - for i in range(length): - if collection[i] != model.w_nil: - store.nil_flags[i] = False - self.do_store(space, store.arr, i, collection[i]) - return self.erase(store) - def copy_storage_from(self, space, w_obj, reuse_storage=False): - old_strategy = w_obj.strategy - if isinstance(old_strategy, self.dense_strategy()): - # Optimized transition from dense to sparse strategy - store = old_strategy.storage(w_obj) - return self.erase(self.copy_from_dense_storage(store, reuse_storage)) - else: - return AbstractStorageStrategy.copy_storage_from(self, space, w_obj, reuse_storage) - def copy_from_dense_storage(self, store, reuse_storage): - # TODO possible optimization: compare len(arr) with _to-_from, use smaller iteration size - nil_flags = [True] * len(store.arr) - for i in range(store._from, store._to): - nil_flags[i] = False - arr = store.arr - if not reuse_storage: - arr = [x for x in arr] - return self.storage_type(arr, nil_flags) class TaggingSmallIntegerStorageStrategy(AbstractStorageStrategy): __metaclass__ = SingletonMeta strategy_tag = 'tagging-small-int' - erase, unerase = rerased.new_static_erasing_pair("tagging-small-integer-strategry") + # erase, unerase = rerased.new_static_erasing_pair("tagging-small-integer-strategry") import_from_mixin(BasicStorageStrategyMixin) + uses_int_storage = True @staticmethod def wrap(val): @@ -266,9 +140,6 @@ def unwrap(val): return val >> 1 @staticmethod - def is_nil(val): - return (val & 1) == 1 - @staticmethod def can_contain(w_val): return isinstance(w_val, model.W_SmallInteger) # TODO - use just a single value to represent nil (max_int-1) @@ -280,35 +151,32 @@ return True def fetch(self, space, w_obj, n0): - val = self.storage(w_obj)[n0] - if (self.is_nil(val)): + val = self.int_storage(w_obj)[n0] + if val == self.nil_value: return space.w_nil else: return space.wrap_int(self.unwrap(val)) def store(self, space, w_obj, n0, w_val): - store = self.storage(w_obj) + store = self.int_storage(w_obj) if self.can_contain(w_val): store[n0] = self.wrap(space.unwrap_int(w_val)) else: - if w_val == model.w_nil: + if w_val == space.w_nil: # TODO - generelize to AllNilStorage by maintaining a counter of nil-elements store[n0] = self.nil_value else: # Storing a wrong type - dehomogenize to ListStorage return w_obj.store_with_new_strategy(space, ListStorageStrategy.singleton, n0, w_val) - def size_of(self, w_obj): - return len(self.storage(w_obj)) - - def initial_storage(self, space, size): + def initial_int_storage(self, space, size): return self.erase([self.nil_value] * size) - def storage_for_list(self, space, collection): + def int_storage_for_list(self, space, collection): length = len(collection) store = [self.nil_value] * length for i in range(length): - if collection[i] != model.w_nil: + if collection[i] != space.w_nil: store[i] = self.wrap(space.unwrap_int(collection[i])) return self.erase(store) @@ -324,8 +192,8 @@ def strategy_for_list(s_containing_class, vars): if s_containing_class is None: - # This is a weird and rare special case for w_nil - return ListStorageStrategy.singleton + # This is a weird and rare special case for w_nil + return ListStorageStrategy.singleton try: is_variable = s_containing_class.isvariable() except AttributeError: diff --git a/spyvm/test/test_primitives.py b/spyvm/test/test_primitives.py --- a/spyvm/test/test_primitives.py +++ b/spyvm/test/test_primitives.py @@ -16,7 +16,7 @@ def __init__(self, stack): size = 6 + len(stack) + 6 self.strategy = strategies.ListStorageStrategy.singleton - self.set_storage(self.strategy.initial_storage(space, size)) + self.set_storage(space, size) self.store_all(space, [None] * 6 + stack + [space.w_nil] * 6) s_self = self.as_blockcontext_get_shadow() s_self.init_stack_and_temps() From noreply at buildbot.pypy.org Thu Mar 20 13:16:03 2014 From: noreply at buildbot.pypy.org (anton_gulenko) Date: Thu, 20 Mar 2014 13:16:03 +0100 (CET) Subject: [pypy-commit] lang-smalltalk strategies-tagging: Renamed set_storage to initialize_storage and _storage to list_storage. Message-ID: <20140320121603.0F4671C08F3@cobra.cs.uni-duesseldorf.de> Author: Anton Gulenko Branch: strategies-tagging Changeset: r670:dcaa168e8fe8 Date: 2014-03-20 11:53 +0100 http://bitbucket.org/pypy/lang-smalltalk/changeset/dcaa168e8fe8/ Log: Renamed set_storage to initialize_storage and _storage to list_storage. Moved the special casing based on uses_int_storage into the strategy base class. diff --git a/spyvm/model.py b/spyvm/model.py --- a/spyvm/model.py +++ b/spyvm/model.py @@ -653,7 +653,7 @@ strategy_stats = StrategyStatistics() class W_PointersObject(W_AbstractPointersObject): - _attrs_ = ['_size', '_storage', 'int_storage', 'strategy'] + _attrs_ = ['_size', 'list_storage', 'int_storage', 'strategy'] @jit.unroll_safe def __init__(self, space, w_class, size): @@ -661,7 +661,7 @@ """Create new object with size = fixed + variable size.""" W_AbstractPointersObject.__init__(self, space, w_class, size) self.strategy = strategy_of_size(self.s_class, size) - self.set_storage(space, size) + self.initialize_storage(space, size) self.log_strategy_operation("Initialized") def log_strategy_operation(self, op, old_strategy=None): @@ -679,24 +679,15 @@ if strategy_stats.do_log: strategy_stats.log_operation(op, new_strategy_tag, old_strategy_tag, classname, size) - def set_storage(self, space, size): + def initialize_storage(self, space, size): self._size = size - if self.strategy.uses_int_storage: - self.int_storage = self.strategy.initial_int_storage(space, size) - else: - self._storage = self.strategy.initial_storage(space, size) - - def get_strategy(self): - return self.strategy + self.strategy.set_initial_storage(space, self, size) def fillin_pointers(self, space, collection): from spyvm.strategies import strategy_for_list self.strategy = strategy_for_list(self.s_class, collection) self._size = len(collection) - if self.strategy.uses_int_storage: - self.int_storage = self.strategy.int_storage_for_list(space, collection) - else: - self._storage = self.strategy.storage_for_list(space, collection) + self.strategy.set_storage_for_list(space, self, collection) def fillin(self, space, g_self): W_AbstractPointersObject.fillin(self, space, g_self) @@ -705,10 +696,7 @@ def switch_strategy(self, space, new_strategy): assert self.strategy != new_strategy - if new_strategy.uses_int_storage: - self.int_storage = new_strategy.copy_int_storage_from(space, self, reuse_storage=True) - else: - self._storage = new_strategy.copy_storage_from(space, self, reuse_storage=True) + new_strategy.set_storage_copied_from(space, self, self, reuse_storage=True) old_strategy = self.strategy self.strategy = new_strategy self.log_strategy_operation("Switched", old_strategy) @@ -735,10 +723,10 @@ i = i+1 def _fetch(self, space, n0): - return self.get_strategy().fetch(space, self, n0) + return self.strategy.fetch(space, self, n0) def _store(self, space, n0, w_value): - return self.get_strategy().store(space, self, n0, w_value) + return self.strategy.store(space, self, n0, w_value) def basic_size(self): return self._size @@ -748,7 +736,7 @@ return False self.strategy, w_other.strategy = w_other.strategy, self.strategy self._size, w_other._size = w_other._size, self._size - self._storage, w_other._storage = w_other._storage, self._storage + self.list_storage, w_other.list_storage = w_other.list_storage, self.list_storage self.int_storage, w_other.int_storage = w_other.int_storage, self.int_storage return W_AbstractPointersObject.become(self, w_other) diff --git a/spyvm/objspace.py b/spyvm/objspace.py --- a/spyvm/objspace.py +++ b/spyvm/objspace.py @@ -165,7 +165,7 @@ from spyvm.strategies import ListStorageStrategy w_nil.space = self w_nil.strategy = ListStorageStrategy.singleton - w_nil.set_storage(self, 0) + w_nil.initialize_storage(self, 0) w_nil.s_class = self.classtable['w_UndefinedObject'].as_class_get_penumbra(self) return w_nil w_nil = self.w_nil = patch_nil(model.w_nil) diff --git a/spyvm/shadow.py b/spyvm/shadow.py --- a/spyvm/shadow.py +++ b/spyvm/shadow.py @@ -432,7 +432,7 @@ self.copy_from_w_self(i) except error.SenderChainManipulation, e: assert e.s_context == self - w_self.set_storage(self.space, 0) + w_self.initialize_storage(self.space, 0) # def detach_shadow(self): # w_self = self.w_self() diff --git a/spyvm/strategies.py b/spyvm/strategies.py --- a/spyvm/strategies.py +++ b/spyvm/strategies.py @@ -1,8 +1,6 @@ from spyvm import model, shadow -from rpython.rlib import rerased -from rpython.rlib import objectmodel, jit, signature -from rpython.rlib.listsort import TimSort +from rpython.rlib import rerased, objectmodel, jit, signature from rpython.rlib.objectmodel import import_from_mixin from rpython.rlib.debug import make_sure_not_resized @@ -23,6 +21,24 @@ # If not, the space-parameter can be passed in as None (probably). return False + def set_initial_storage(self, space, w_obj, size): + if self.uses_int_storage: + w_obj.int_storage = self.initial_int_storage(space, size) + else: + w_obj.list_storage = self.initial_storage(space, size) + + def set_storage_for_list(self, space, w_obj, collection): + if self.uses_int_storage: + w_obj.int_storage = self.int_storage_for_list(space, collection) + else: + w_obj.list_storage = self.storage_for_list(space, collection) + + def set_storage_copied_from(self, space, w_obj, w_source_obj, reuse_storage=False): + if self.uses_int_storage: + w_obj.int_storage = self.int_storage_for_list(space, collection) + else: + w_obj.list_storage = self.storage_for_list(space, collection) + def fetch(self, space, w_obj, n0): raise NotImplementedError("Abstract base class") def store(self, space, w_obj, n0, w_val): @@ -48,7 +64,7 @@ def copy_storage_from(self, space, w_obj, reuse_storage=False): old_strategy = w_obj.strategy if old_strategy == self and reuse_storage: - return w_obj._storage + return w_obj.list_storage else: # This can be overridden and optimized (reuse_storage flag, less temporary storage) return self.storage_for_list(space, w_obj.fetch_all(space)) @@ -69,7 +85,7 @@ def int_storage(self, w_obj): return w_obj.int_storage def storage(self, w_obj): - return w_obj._storage + return w_obj.list_storage def erase(self, a): return a def unerase(self, a): return a @@ -128,7 +144,7 @@ class TaggingSmallIntegerStorageStrategy(AbstractStorageStrategy): __metaclass__ = SingletonMeta - strategy_tag = 'tagging-small-int' + strategy_tag = 'tagging' # erase, unerase = rerased.new_static_erasing_pair("tagging-small-integer-strategry") import_from_mixin(BasicStorageStrategyMixin) uses_int_storage = True diff --git a/spyvm/test/test_primitives.py b/spyvm/test/test_primitives.py --- a/spyvm/test/test_primitives.py +++ b/spyvm/test/test_primitives.py @@ -16,7 +16,7 @@ def __init__(self, stack): size = 6 + len(stack) + 6 self.strategy = strategies.ListStorageStrategy.singleton - self.set_storage(space, size) + self.initialize_storage(space, size) self.store_all(space, [None] * 6 + stack + [space.w_nil] * 6) s_self = self.as_blockcontext_get_shadow() s_self.init_stack_and_temps() From noreply at buildbot.pypy.org Thu Mar 20 13:16:04 2014 From: noreply at buildbot.pypy.org (anton_gulenko) Date: Thu, 20 Mar 2014 13:16:04 +0100 (CET) Subject: [pypy-commit] lang-smalltalk strategies-tagging: Fixed refactoring. Message-ID: <20140320121604.3AEC41C08F3@cobra.cs.uni-duesseldorf.de> Author: Anton Gulenko Branch: strategies-tagging Changeset: r671:3b7a82b8f4e1 Date: 2014-03-20 11:55 +0100 http://bitbucket.org/pypy/lang-smalltalk/changeset/3b7a82b8f4e1/ Log: Fixed refactoring. diff --git a/spyvm/strategies.py b/spyvm/strategies.py --- a/spyvm/strategies.py +++ b/spyvm/strategies.py @@ -35,9 +35,9 @@ def set_storage_copied_from(self, space, w_obj, w_source_obj, reuse_storage=False): if self.uses_int_storage: - w_obj.int_storage = self.int_storage_for_list(space, collection) + w_obj.int_storage = self.copy_int_storage_from(space, w_source_obj, reuse_storage) else: - w_obj.list_storage = self.storage_for_list(space, collection) + w_obj.list_storage = self.copy_storage_from(space, w_source_obj, reuse_storage) def fetch(self, space, w_obj, n0): raise NotImplementedError("Abstract base class") @@ -54,14 +54,14 @@ raise NotImplementedError("Abstract base class") def int_storage_for_list(self, space, collection): raise NotImplementedError("Abstract base class") - def copy_int_storage_from(self, space, w_obj, reuse_storage=False): + def copy_int_storage_from(self, space, w_obj, reuse_storage): old_strategy = w_obj.strategy if old_strategy == self and reuse_storage: return w_obj.int_storage else: # This can be overridden and optimized (reuse_storage flag, less temporary storage) return self.int_storage_for_list(space, w_obj.fetch_all(space)) - def copy_storage_from(self, space, w_obj, reuse_storage=False): + def copy_storage_from(self, space, w_obj, reuse_storage): old_strategy = w_obj.strategy if old_strategy == self and reuse_storage: return w_obj.list_storage From noreply at buildbot.pypy.org Thu Mar 20 13:16:05 2014 From: noreply at buildbot.pypy.org (anton_gulenko) Date: Thu, 20 Mar 2014 13:16:05 +0100 (CET) Subject: [pypy-commit] lang-smalltalk strategies-tagging: Replaced uses_int_storage special casing in favor of subclassing. Message-ID: <20140320121605.7F2731C08F3@cobra.cs.uni-duesseldorf.de> Author: Anton Gulenko Branch: strategies-tagging Changeset: r672:fd9669dc8f6d Date: 2014-03-20 12:17 +0100 http://bitbucket.org/pypy/lang-smalltalk/changeset/fd9669dc8f6d/ Log: Replaced uses_int_storage special casing in favor of subclassing. Removed size_of from the strategy interface. Removed make_sure_not_resized. Turned needs_objspace into an attribute. Removed unused imports. diff --git a/spyvm/model.py b/spyvm/model.py --- a/spyvm/model.py +++ b/spyvm/model.py @@ -1268,7 +1268,7 @@ if isinstance(w_candidate, W_PointersObject): c_shadow = w_candidate._get_shadow() if c_shadow is None and w_candidate.size() >= 2: - if not w_candidate.strategy.needs_objspace(): + if not w_candidate.strategy.needs_objspace: # We can fetch without having an object space at hand. # XXX How to get an object space from a CompiledMethodShadow, anyways? w_class = w_candidate._fetch(None, 1) diff --git a/spyvm/strategies.py b/spyvm/strategies.py --- a/spyvm/strategies.py +++ b/spyvm/strategies.py @@ -1,8 +1,7 @@ from spyvm import model, shadow -from rpython.rlib import rerased, objectmodel, jit, signature +from rpython.rlib import rerased from rpython.rlib.objectmodel import import_from_mixin -from rpython.rlib.debug import make_sure_not_resized # Disables all optimized strategies, for debugging. only_list_storage = False @@ -12,55 +11,36 @@ _attrs_ = [] _settled_ = True strategy_tag = 'abstract' - uses_int_storage = False - - def __init__(self): - pass - def needs_objspace(self): - # Return True, if fetch/store operations use the space parameter. - # If not, the space-parameter can be passed in as None (probably). - return False + needs_objspace = False def set_initial_storage(self, space, w_obj, size): - if self.uses_int_storage: - w_obj.int_storage = self.initial_int_storage(space, size) - else: - w_obj.list_storage = self.initial_storage(space, size) - + raise NotImplementedError("Abstract base class") def set_storage_for_list(self, space, w_obj, collection): - if self.uses_int_storage: - w_obj.int_storage = self.int_storage_for_list(space, collection) - else: - w_obj.list_storage = self.storage_for_list(space, collection) - + raise NotImplementedError("Abstract base class") def set_storage_copied_from(self, space, w_obj, w_source_obj, reuse_storage=False): - if self.uses_int_storage: - w_obj.int_storage = self.copy_int_storage_from(space, w_source_obj, reuse_storage) - else: - w_obj.list_storage = self.copy_storage_from(space, w_source_obj, reuse_storage) + raise NotImplementedError("Abstract base class") def fetch(self, space, w_obj, n0): raise NotImplementedError("Abstract base class") def store(self, space, w_obj, n0, w_val): raise NotImplementedError("Abstract base class") - def size_of(self, w_obj): - raise NotImplementedError("Abstract base class") + +class AbstractListStorageStrategy(AbstractStorageStrategy): + strategy_tag = 'abstract-list' + + def storage(self, w_obj): + return w_obj.list_storage + def set_initial_storage(self, space, w_obj, size): + w_obj.list_storage = self.initial_storage(space, size) + def set_storage_for_list(self, space, w_obj, collection): + w_obj.list_storage = self.storage_for_list(space, collection) + def set_storage_copied_from(self, space, w_obj, w_source_obj, reuse_storage=False): + w_obj.list_storage = self.copy_storage_from(space, w_source_obj, reuse_storage) def initial_storage(self, space, size): raise NotImplementedError("Abstract base class") - def initial_int_storage(self, space, size): - raise NotImplementedError("Abstract base class") def storage_for_list(self, space, collection): raise NotImplementedError("Abstract base class") - def int_storage_for_list(self, space, collection): - raise NotImplementedError("Abstract base class") - def copy_int_storage_from(self, space, w_obj, reuse_storage): - old_strategy = w_obj.strategy - if old_strategy == self and reuse_storage: - return w_obj.int_storage - else: - # This can be overridden and optimized (reuse_storage flag, less temporary storage) - return self.int_storage_for_list(space, w_obj.fetch_all(space)) def copy_storage_from(self, space, w_obj, reuse_storage): old_strategy = w_obj.strategy if old_strategy == self and reuse_storage: @@ -68,6 +48,30 @@ else: # This can be overridden and optimized (reuse_storage flag, less temporary storage) return self.storage_for_list(space, w_obj.fetch_all(space)) + +class AbstractIntStorageStrategy(AbstractStorageStrategy): + strategy_tag = 'abstract-int' + + def storage(self, w_obj): + return w_obj.int_storage + def set_initial_storage(self, space, w_obj, size): + w_obj.int_storage = self.initial_storage(space, size) + def set_storage_for_list(self, space, w_obj, collection): + w_obj.int_storage = self.storage_for_list(space, collection) + def set_storage_copied_from(self, space, w_obj, w_source_obj, reuse_storage=False): + w_obj.int_storage = self.copy_storage_from(space, w_source_obj, reuse_storage) + + def initial_storage(self, space, size): + raise NotImplementedError("Abstract base class") + def storage_for_list(self, space, collection): + raise NotImplementedError("Abstract base class") + def copy_storage_from(self, space, w_obj, reuse_storage): + old_strategy = w_obj.strategy + if old_strategy == self and reuse_storage: + return w_obj.int_storage + else: + # This can be overridden and optimized (reuse_storage flag, less temporary storage) + return self.int_storage_for_list(space, w_obj.fetch_all(space)) class SingletonMeta(type): def __new__(cls, name, bases, dct): @@ -76,25 +80,15 @@ return result class BasicStorageStrategyMixin(object): - # Concrete class must implement: unerase - def size_of(self, w_obj): - if self.uses_int_storage: - return len(self.int_storage(w_obj)) - else: - return len(self.storage(w_obj)) - def int_storage(self, w_obj): - return w_obj.int_storage - def storage(self, w_obj): - return w_obj.list_storage def erase(self, a): return a def unerase(self, a): return a + # erase, unerase = rerased.new_static_erasing_pair(self.strategy_tag) # this is the typical "initial" storage strategy, for when every slot -# in a var-sized object is still nil. No storage is allocated except for -# holding the size of the object. +# in an object is still nil. No storage is allocated. class AllNilStorageStrategy(AbstractStorageStrategy): __metaclass__ = SingletonMeta - # erase, unerase = rerased.new_static_erasing_pair("all-nil-strategy") + # erase, unerase = rerased.new_static_erasing_pair("allnil-strategy") import_from_mixin(BasicStorageStrategyMixin) strategy_tag = 'allnil' @@ -110,19 +104,19 @@ return w_obj.store_with_new_strategy(space, TaggingSmallIntegerStorageStrategy.singleton, n0, w_val) return w_obj.store_with_new_strategy(space, ListStorageStrategy.singleton, n0, w_val) - def initial_storage(self, space, size): - return [] - def storage_for_list(self, space, collection): - return [] - def copy_storage_from(self, space, w_obj, reuse_storage=False): - return [] + def set_initial_storage(self, space, w_obj, size): + pass + def set_storage_for_list(self, space, w_obj, collection): + pass + def set_storage_copied_from(self, space, w_obj, w_source_obj, reuse_storage=False): + pass # This is the regular storage strategy that does not result in any # optimizations but can handle every case. Applicable for both # fixed-sized and var-sized objects. -class ListStorageStrategy(AbstractStorageStrategy): +class ListStorageStrategy(AbstractListStorageStrategy): __metaclass__ = SingletonMeta - # erase, unerase = rerased.new_static_erasing_pair("list-storage-strategy") + # erase, unerase = rerased.new_static_erasing_pair("list-strategy") import_from_mixin(BasicStorageStrategyMixin) strategy_tag = 'list' @@ -131,23 +125,20 @@ def store(self, space, w_obj, n0, w_val): # TODO enable generalization by maintaining a counter of elements that are nil. self.storage(w_obj)[n0] = w_val - def erased_list(self, list): - make_sure_not_resized(list) - return self.erase(list) def initial_storage(self, space, size): - return self.erased_list([model.w_nil] * size) + return [model.w_nil] * size def storage_for_list(self, space, collection): - return self.erased_list([x for x in collection]) + return [x for x in collection] def copy_storage_from(self, space, w_obj, reuse_storage=False): length = w_obj.basic_size() - return self.erased_list([w_obj.strategy.fetch(space, w_obj, i) for i in range(length)]) + return [w_obj.strategy.fetch(space, w_obj, i) for i in range(length)] -class TaggingSmallIntegerStorageStrategy(AbstractStorageStrategy): +class TaggingSmallIntegerStorageStrategy(AbstractIntStorageStrategy): __metaclass__ = SingletonMeta - strategy_tag = 'tagging' - # erase, unerase = rerased.new_static_erasing_pair("tagging-small-integer-strategry") + # erase, unerase = rerased.new_static_erasing_pair("tagging-smallint-strategry") import_from_mixin(BasicStorageStrategyMixin) - uses_int_storage = True + strategy_tag = 'tagging-smallint' + needs_objspace = True @staticmethod def wrap(val): @@ -163,18 +154,15 @@ # also store W_LargePositiveInteger1Word? nil_value = 1 - def needs_objspace(self): - return True - def fetch(self, space, w_obj, n0): - val = self.int_storage(w_obj)[n0] + val = self.storage(w_obj)[n0] if val == self.nil_value: return space.w_nil else: return space.wrap_int(self.unwrap(val)) def store(self, space, w_obj, n0, w_val): - store = self.int_storage(w_obj) + store = self.storage(w_obj) if self.can_contain(w_val): store[n0] = self.wrap(space.unwrap_int(w_val)) else: @@ -186,7 +174,7 @@ return w_obj.store_with_new_strategy(space, ListStorageStrategy.singleton, n0, w_val) def initial_int_storage(self, space, size): - return self.erase([self.nil_value] * size) + return [self.nil_value] * size def int_storage_for_list(self, space, collection): length = len(collection) @@ -194,7 +182,7 @@ for i in range(length): if collection[i] != space.w_nil: store[i] = self.wrap(space.unwrap_int(collection[i])) - return self.erase(store) + return store def strategy_of_size(s_containing_class, size): if s_containing_class is None: From noreply at buildbot.pypy.org Thu Mar 20 13:16:06 2014 From: noreply at buildbot.pypy.org (anton_gulenko) Date: Thu, 20 Mar 2014 13:16:06 +0100 (CET) Subject: [pypy-commit] lang-smalltalk strategies-tagging: Fixing tests. Message-ID: <20140320121606.AB4161C08F3@cobra.cs.uni-duesseldorf.de> Author: Anton Gulenko Branch: strategies-tagging Changeset: r673:b1593fffd1e7 Date: 2014-03-20 12:54 +0100 http://bitbucket.org/pypy/lang-smalltalk/changeset/b1593fffd1e7/ Log: Fixing tests. diff --git a/spyvm/model.py b/spyvm/model.py --- a/spyvm/model.py +++ b/spyvm/model.py @@ -654,7 +654,9 @@ class W_PointersObject(W_AbstractPointersObject): _attrs_ = ['_size', 'list_storage', 'int_storage', 'strategy'] - + list_storage = None + int_storage = None + @jit.unroll_safe def __init__(self, space, w_class, size): from spyvm.strategies import strategy_of_size diff --git a/spyvm/strategies.py b/spyvm/strategies.py --- a/spyvm/strategies.py +++ b/spyvm/strategies.py @@ -44,7 +44,7 @@ def copy_storage_from(self, space, w_obj, reuse_storage): old_strategy = w_obj.strategy if old_strategy == self and reuse_storage: - return w_obj.list_storage + return self.storage(w_obj) else: # This can be overridden and optimized (reuse_storage flag, less temporary storage) return self.storage_for_list(space, w_obj.fetch_all(space)) @@ -68,10 +68,10 @@ def copy_storage_from(self, space, w_obj, reuse_storage): old_strategy = w_obj.strategy if old_strategy == self and reuse_storage: - return w_obj.int_storage + return self.storage(w_obj) else: # This can be overridden and optimized (reuse_storage flag, less temporary storage) - return self.int_storage_for_list(space, w_obj.fetch_all(space)) + return self.storage_for_list(space, w_obj.fetch_all(space)) class SingletonMeta(type): def __new__(cls, name, bases, dct): @@ -173,10 +173,10 @@ # Storing a wrong type - dehomogenize to ListStorage return w_obj.store_with_new_strategy(space, ListStorageStrategy.singleton, n0, w_val) - def initial_int_storage(self, space, size): + def initial_storage(self, space, size): return [self.nil_value] * size - def int_storage_for_list(self, space, collection): + def storage_for_list(self, space, collection): length = len(collection) store = [self.nil_value] * length for i in range(length): diff --git a/spyvm/test/test_primitives.py b/spyvm/test/test_primitives.py --- a/spyvm/test/test_primitives.py +++ b/spyvm/test/test_primitives.py @@ -18,15 +18,17 @@ self.strategy = strategies.ListStorageStrategy.singleton self.initialize_storage(space, size) self.store_all(space, [None] * 6 + stack + [space.w_nil] * 6) - s_self = self.as_blockcontext_get_shadow() + import pdb; pdb.set_trace() + s_self = self.as_blockcontext_get_shadow(space) s_self.init_stack_and_temps() s_self.reset_stack() s_self.push_all(stack) s_self.store_expected_argument_count(0) self.s_class = space.w_MethodContext.as_class_get_shadow(space) - def as_blockcontext_get_shadow(self): - self.shadow = shadow.BlockContextShadow(space, self) + def as_blockcontext_get_shadow(self, space): + if not self.shadow: + self.shadow = shadow.BlockContextShadow(space, self) return self.shadow def wrap(x): From noreply at buildbot.pypy.org Thu Mar 20 13:16:07 2014 From: noreply at buildbot.pypy.org (anton_gulenko) Date: Thu, 20 Mar 2014 13:16:07 +0100 (CET) Subject: [pypy-commit] lang-smalltalk strategies-tagging: Updated strategies tests. Message-ID: <20140320121607.C12951C08F3@cobra.cs.uni-duesseldorf.de> Author: Anton Gulenko Branch: strategies-tagging Changeset: r674:f5c765f96240 Date: 2014-03-20 13:14 +0100 http://bitbucket.org/pypy/lang-smalltalk/changeset/f5c765f96240/ Log: Updated strategies tests. Fixed other tests. diff --git a/spyvm/shadow.py b/spyvm/shadow.py --- a/spyvm/shadow.py +++ b/spyvm/shadow.py @@ -757,16 +757,17 @@ def __init__(self, space, w_self=None, w_home=None, argcnt=0, initialip=0): self = jit.hint(self, access_directly=True, fresh_virtualizable=True) - contextsize = w_home.as_methodcontext_get_shadow(space).myblocksize() creating_w_self = w_self is None if creating_w_self: + contextsize = w_home.as_methodcontext_get_shadow(space).myblocksize() w_self = model.W_PointersObject(space, space.w_BlockContext, contextsize) ContextPartShadow.__init__(self, space, w_self) if creating_w_self: w_self.store_shadow(self) self.store_expected_argument_count(argcnt) self.store_initialip(initialip) - self.store_w_home(w_home) + if w_home: + self.store_w_home(w_home) self.store_pc(initialip) self.init_stack_and_temps() diff --git a/spyvm/test/test_primitives.py b/spyvm/test/test_primitives.py --- a/spyvm/test/test_primitives.py +++ b/spyvm/test/test_primitives.py @@ -14,11 +14,11 @@ class MockFrame(model.W_PointersObject): def __init__(self, stack): + self.space = space size = 6 + len(stack) + 6 self.strategy = strategies.ListStorageStrategy.singleton self.initialize_storage(space, size) self.store_all(space, [None] * 6 + stack + [space.w_nil] * 6) - import pdb; pdb.set_trace() s_self = self.as_blockcontext_get_shadow(space) s_self.init_stack_and_temps() s_self.reset_stack() diff --git a/spyvm/test/test_shadow.py b/spyvm/test/test_shadow.py --- a/spyvm/test/test_shadow.py +++ b/spyvm/test/test_shadow.py @@ -155,7 +155,7 @@ assert s_object.s_home() == s_object def assert_contains_nils(w_obj): - for i in range(w_obj.strategy.size_of(w_obj)): + for i in range(w_obj.basic_size()): assert model.w_nil == w_obj.strategy.fetch(i, space, w_obj) def test_attach_mc(): diff --git a/spyvm/test/test_strategies.py b/spyvm/test/test_strategies.py --- a/spyvm/test/test_strategies.py +++ b/spyvm/test/test_strategies.py @@ -7,10 +7,8 @@ # Fieldtypes have a separate test file space, interp = tools.setup_module(tools, filename='bootstrapped.image') - class_Array = space.classtable["w_Array"] - def arr(size): return model.W_PointersObject(space, class_Array, size) @@ -19,21 +17,16 @@ a.store(space, 0, arr(1)) return a -def dense_arr(size): +def tagging_arr(size): a = arr(size) a.store(space, 0, space.wrap_int(12)) return a -def dense_arr_odd(size): +def tagging_arr_odd(size): a = arr(size) a.store(space, 2, space.wrap_int(12)) return a -def sparse_arr(size): - a = dense_arr(size) - a.store(space, 2, space.wrap_int(20)) - return a - def check_arr(arr, expected): for i in range(arr.basic_size()): if expected[i] == w_nil: @@ -85,55 +78,27 @@ a.store(space, 1, arr(1)) assert a.basic_size() == 5 -# ====== Dense and Sparse *SmallInteger-StorageStrategy +# ====== Tagging SmallInteger StorageStrategy -def test_AllNil_to_Dense(): - a = dense_arr(5) - assert isinstance(a.strategy, strategies.DenseSmallIntegerStorageStrategy) +def test_AllNil_to_Int(): + a = tagging_arr(5) + assert isinstance(a.strategy, strategies.TaggingSmallIntegerStorageStrategy) check_arr(a, [12, w_nil, w_nil, w_nil, w_nil]) -def test_Dense_store(): - a = dense_arr(5) +def test_Tagging_store(): + a = tagging_arr(5) a.store(space, 1, space.wrap_int(20)) a.store(space, 2, space.wrap_int(20)) - assert isinstance(a.strategy, strategies.DenseSmallIntegerStorageStrategy) + assert isinstance(a.strategy, strategies.TaggingSmallIntegerStorageStrategy) check_arr(a, [12, 20, 20, w_nil, w_nil]) -def test_Dense_overwrite_middle(): - a = dense_arr(5) - a.store(space, 1, space.wrap_int(20)) - a.store(space, 2, space.wrap_int(20)) - a.store(space, 1, space.wrap_int(30)) - check_arr(a, [12, 30, 20, w_nil, w_nil]) - -def test_Dense_overwrite_first(): - a = dense_arr(5) - a.store(space, 1, space.wrap_int(20)) - a.store(space, 2, space.wrap_int(20)) - a.store(space, 0, space.wrap_int(30)) - check_arr(a, [30, 20, 20, w_nil, w_nil]) - -def test_Dense_overwrite_last(): - a = dense_arr(5) - a.store(space, 1, space.wrap_int(20)) - a.store(space, 2, space.wrap_int(20)) - a.store(space, 2, space.wrap_int(30)) - check_arr(a, [12, 20, 30, w_nil, w_nil]) - -def test_Dense_odd(): - a = dense_arr_odd(5) - assert isinstance(a.strategy, strategies.DenseSmallIntegerStorageStrategy) +def test_Tagging_store_nil_to_nil(): + a = tagging_arr_odd(5) + a.store(space, 1, w_nil) check_arr(a, [w_nil, w_nil, 12, w_nil, w_nil]) - -def test_Dense_odd_store(): - a = dense_arr_odd(5) - a.store(space, 1, space.wrap_int(20)) - a.store(space, 3, space.wrap_int(40)) - a.store(space, 4, space.wrap_int(30)) - check_arr(a, [w_nil, 20, 12, 40, 30]) - -def test_Dense_odd_overwrite(): - a = dense_arr_odd(5) + +def test_Tagging_delete(): + a = tagging_arr_odd(5) a.store(space, 1, space.wrap_int(1)) a.store(space, 3, space.wrap_int(2)) a.store(space, 2, space.wrap_int(100)) @@ -141,67 +106,13 @@ a.store(space, 3, space.wrap_int(300)) check_arr(a, [w_nil, 200, 100, 300, w_nil]) -def test_Dense_store_nil_to_nil(): - a = dense_arr_odd(5) - a.store(space, 1, w_nil) - check_arr(a, [w_nil, w_nil, 12, w_nil, w_nil]) - -def test_Dense_delete(): - a = dense_arr_odd(5) - a.store(space, 1, space.wrap_int(1)) - a.store(space, 3, space.wrap_int(2)) - a.store(space, 2, space.wrap_int(100)) - a.store(space, 1, space.wrap_int(200)) - a.store(space, 3, space.wrap_int(300)) - check_arr(a, [w_nil, 200, 100, 300, w_nil]) - -def test_Dense_delete_first(): - a = dense_arr_odd(5) +def test_Tagging_delete_first(): + a = tagging_arr_odd(5) a.store(space, 1, space.wrap_int(1)) a.store(space, 1, w_nil) check_arr(a, [w_nil, w_nil, 12, w_nil, w_nil]) -def test_Dense_delete_last(): - a = dense_arr_odd(5) - a.store(space, 1, space.wrap_int(1)) - a.store(space, 2, w_nil) - check_arr(a, [w_nil, 1, w_nil, w_nil, w_nil]) - -def test_Dense_to_AllNil(): - a = dense_arr_odd(5) - a.store(space, 2, w_nil) - assert isinstance(a.strategy, strategies.AllNilStorageStrategy) - -def test_Dense_to_List(): - a = dense_arr_odd(5) +def test_Tagging_to_List(): + a = tagging_arr_odd(5) a.store(space, 1, arr(1)) assert isinstance(a.strategy, strategies.ListStorageStrategy) - -def test_Dense_to_Sparse_by_deleting(): - a = dense_arr_odd(5) - a.store(space, 1, space.wrap_int(10)) - a.store(space, 3, space.wrap_int(20)) - a.store(space, 2, w_nil) - assert isinstance(a.strategy, strategies.SparseSmallIntegerStorageStrategy) - check_arr(a, [w_nil, 10, w_nil, 20, w_nil]) - -def test_Dense_to_Sparse_by_storing(): - a = dense_arr_odd(5) - a.store(space, 4, space.wrap_int(10)) - assert isinstance(a.strategy, strategies.SparseSmallIntegerStorageStrategy) - check_arr(a, [w_nil, w_nil, 12, w_nil, 10]) - -def test_Sparse_store_nil(): - a = sparse_arr(5) - a.store(space, 2, w_nil) - check_arr(a, [12, w_nil, w_nil, w_nil, w_nil]) - -def test_Sparse_store(): - a = sparse_arr(5) - a.store(space, 4, space.wrap_int(100)) - check_arr(a, [12, w_nil, 20, w_nil, 100]) - -def test_Sparse_to_List(): - a = sparse_arr(5) - a.store(space, 4, arr(5)) - assert isinstance(a.strategy, strategies.ListStorageStrategy) From noreply at buildbot.pypy.org Thu Mar 20 13:20:59 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 20 Mar 2014 13:20:59 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Found out and explain why we get rare crashes in pypy if we Message-ID: <20140320122059.473081C02AF@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r70112:3ada12cdea0e Date: 2014-03-20 13:20 +0100 http://bitbucket.org/pypy/pypy/changeset/3ada12cdea0e/ Log: Found out and explain why we get rare crashes in pypy if we enable this 'stm_ignored'. diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py --- a/rpython/rtyper/lltypesystem/rstr.py +++ b/rpython/rtyper/lltypesystem/rstr.py @@ -377,14 +377,18 @@ # special non-computed-yet value. if not s: return 0 - with stm_ignored: - x = s.hash + #with stm_ignored: + x = s.hash if x == 0: x = _hash_string(s.chars) if x == 0: x = 29872897 - with stm_ignored: - s.hash = x + # XXX STM note: we would like this write to be stm-ignored, + # but we can't, because ll_strfasthash() might later miss + # the written value and return 0 again (rarely). Think + # again later about the best option. + #with stm_ignored: + s.hash = x return x def ll_length(s): From noreply at buildbot.pypy.org Thu Mar 20 15:37:49 2014 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 20 Mar 2014 15:37:49 +0100 (CET) Subject: [pypy-commit] pypy improve-consecutive-dict-lookups: close to be merged branch Message-ID: <20140320143749.1B0F21C1578@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: improve-consecutive-dict-lookups Changeset: r70113:f0136fc09640 Date: 2014-03-20 16:36 +0200 http://bitbucket.org/pypy/pypy/changeset/f0136fc09640/ Log: close to be merged branch diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py --- a/rpython/jit/backend/test/runner_test.py +++ b/rpython/jit/backend/test/runner_test.py @@ -3736,7 +3736,7 @@ assert False, 'should not be called' from rpython.jit.codewriter.effectinfo import EffectInfo - effectinfo = EffectInfo([], [], [], [], EffectInfo.EF_CANNOT_RAISE, EffectInfo.OS_MATH_SQRT) + effectinfo = EffectInfo([], [], [], [], [], [], EffectInfo.EF_CANNOT_RAISE, EffectInfo.OS_MATH_SQRT) FPTR = self.Ptr(self.FuncType([lltype.Float], lltype.Float)) func_ptr = llhelper(FPTR, math_sqrt) FUNC = deref(FPTR) From noreply at buildbot.pypy.org Thu Mar 20 15:37:51 2014 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 20 Mar 2014 15:37:51 +0100 (CET) Subject: [pypy-commit] pypy default: Merge "improve-consecutive-dict-lookups". This branch makes it possible Message-ID: <20140320143751.1E8531C1578@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r70114:0679ba1c1143 Date: 2014-03-20 16:37 +0200 http://bitbucket.org/pypy/pypy/changeset/0679ba1c1143/ Log: Merge "improve-consecutive-dict-lookups". This branch makes it possible to cache consecutive dict lookups (for both normal and ordered dicts) diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py --- a/rpython/jit/backend/test/runner_test.py +++ b/rpython/jit/backend/test/runner_test.py @@ -3736,7 +3736,7 @@ assert False, 'should not be called' from rpython.jit.codewriter.effectinfo import EffectInfo - effectinfo = EffectInfo([], [], [], [], EffectInfo.EF_CANNOT_RAISE, EffectInfo.OS_MATH_SQRT) + effectinfo = EffectInfo([], [], [], [], [], [], EffectInfo.EF_CANNOT_RAISE, EffectInfo.OS_MATH_SQRT) FPTR = self.Ptr(self.FuncType([lltype.Float], lltype.Float)) func_ptr = llhelper(FPTR, math_sqrt) FUNC = deref(FPTR) diff --git a/rpython/jit/codewriter/call.py b/rpython/jit/codewriter/call.py --- a/rpython/jit/codewriter/call.py +++ b/rpython/jit/codewriter/call.py @@ -178,7 +178,7 @@ return (fnaddr, calldescr) def getcalldescr(self, op, oopspecindex=EffectInfo.OS_NONE, - extraeffect=None): + extraeffect=None, extradescr=None): """Return the calldescr that describes all calls done by 'op'. This returns a calldescr that we can put in the corresponding call operation in the calling jitcode. It gets an effectinfo @@ -259,6 +259,7 @@ effectinfo = effectinfo_from_writeanalyze( self.readwrite_analyzer.analyze(op, self.seen), self.cpu, extraeffect, oopspecindex, can_invalidate, call_release_gil_target, + extradescr, ) # assert effectinfo is not None diff --git a/rpython/jit/codewriter/effectinfo.py b/rpython/jit/codewriter/effectinfo.py --- a/rpython/jit/codewriter/effectinfo.py +++ b/rpython/jit/codewriter/effectinfo.py @@ -21,6 +21,7 @@ OS_ARRAYCOPY = 1 # "list.ll_arraycopy" OS_STR2UNICODE = 2 # "str.str2unicode" OS_SHRINK_ARRAY = 3 # rgc.ll_shrink_array + OS_DICT_LOOKUP = 4 # ll_dict_lookup # OS_STR_CONCAT = 22 # "stroruni.concat" OS_STR_SLICE = 23 # "stroruni.slice" @@ -88,15 +89,18 @@ # for debugging: _OS_CANRAISE = set([ OS_NONE, OS_STR2UNICODE, OS_LIBFFI_CALL, OS_RAW_MALLOC_VARSIZE_CHAR, - OS_JIT_FORCE_VIRTUAL, OS_SHRINK_ARRAY, + OS_JIT_FORCE_VIRTUAL, OS_SHRINK_ARRAY, OS_DICT_LOOKUP, ]) def __new__(cls, readonly_descrs_fields, readonly_descrs_arrays, + readonly_descrs_interiorfields, write_descrs_fields, write_descrs_arrays, + write_descrs_interiorfields, extraeffect=EF_CAN_RAISE, oopspecindex=OS_NONE, can_invalidate=False, - call_release_gil_target=llmemory.NULL): + call_release_gil_target=llmemory.NULL, + extradescrs=None): key = (frozenset_or_none(readonly_descrs_fields), frozenset_or_none(readonly_descrs_arrays), frozenset_or_none(write_descrs_fields), @@ -121,17 +125,21 @@ result = object.__new__(cls) result.readonly_descrs_fields = readonly_descrs_fields result.readonly_descrs_arrays = readonly_descrs_arrays + result.readonly_descrs_interiorfields = readonly_descrs_interiorfields if extraeffect == EffectInfo.EF_LOOPINVARIANT or \ extraeffect == EffectInfo.EF_ELIDABLE_CANNOT_RAISE or \ extraeffect == EffectInfo.EF_ELIDABLE_CAN_RAISE: result.write_descrs_fields = [] result.write_descrs_arrays = [] + result.write_descrs_interiorfields = [] else: result.write_descrs_fields = write_descrs_fields result.write_descrs_arrays = write_descrs_arrays + result.write_descrs_interiorfields = write_descrs_interiorfields result.extraeffect = extraeffect result.can_invalidate = can_invalidate result.oopspecindex = oopspecindex + result.extradescrs = extradescrs result.call_release_gil_target = call_release_gil_target if result.check_can_raise(): assert oopspecindex in cls._OS_CANRAISE @@ -163,7 +171,7 @@ return None return frozenset(x) -EffectInfo.MOST_GENERAL = EffectInfo(None, None, None, None, +EffectInfo.MOST_GENERAL = EffectInfo(None, None, None, None, None, None, EffectInfo.EF_RANDOM_EFFECTS, can_invalidate=True) @@ -172,19 +180,24 @@ extraeffect=EffectInfo.EF_CAN_RAISE, oopspecindex=EffectInfo.OS_NONE, can_invalidate=False, - call_release_gil_target=llmemory.NULL): + call_release_gil_target=llmemory.NULL, + extradescr=None): from rpython.translator.backendopt.writeanalyze import top_set if effects is top_set or extraeffect == EffectInfo.EF_RANDOM_EFFECTS: readonly_descrs_fields = None readonly_descrs_arrays = None + readonly_descrs_interiorfields = None write_descrs_fields = None write_descrs_arrays = None + write_descrs_interiorfields = None extraeffect = EffectInfo.EF_RANDOM_EFFECTS else: readonly_descrs_fields = [] readonly_descrs_arrays = [] + readonly_descrs_interiorfields = [] write_descrs_fields = [] write_descrs_arrays = [] + write_descrs_interiorfields = [] def add_struct(descrs_fields, (_, T, fieldname)): T = deref(T) @@ -198,6 +211,17 @@ descr = cpu.arraydescrof(ARRAY) descrs_arrays.append(descr) + def add_interiorfield(descrs_interiorfields, (_, T, fieldname)): + T = deref(T) + if not isinstance(T, lltype.Array): + return # let's not consider structs for now + if not consider_array(T): + return + if getattr(T.OF, fieldname) is lltype.Void: + return + descr = cpu.interiorfielddescrof(T, fieldname) + descrs_interiorfields.append(descr) + for tup in effects: if tup[0] == "struct": add_struct(write_descrs_fields, tup) @@ -205,6 +229,12 @@ tupw = ("struct",) + tup[1:] if tupw not in effects: add_struct(readonly_descrs_fields, tup) + elif tup[0] == "interiorfield": + add_interiorfield(write_descrs_interiorfields, tup) + elif tup[0] == "readinteriorfield": + tupw = ('interiorfield',) + tup[1:] + if tupw not in effects: + add_interiorfield(readonly_descrs_interiorfields, tup) elif tup[0] == "array": add_array(write_descrs_arrays, tup) elif tup[0] == "readarray": @@ -216,12 +246,15 @@ # return EffectInfo(readonly_descrs_fields, readonly_descrs_arrays, + readonly_descrs_interiorfields, write_descrs_fields, write_descrs_arrays, + write_descrs_interiorfields, extraeffect, oopspecindex, can_invalidate, - call_release_gil_target) + call_release_gil_target, + extradescr) def consider_struct(TYPE, fieldname): if fieldType(TYPE, fieldname) is lltype.Void: diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py --- a/rpython/jit/codewriter/jtransform.py +++ b/rpython/jit/codewriter/jtransform.py @@ -403,6 +403,9 @@ prepare = self._handle_math_sqrt_call elif oopspec_name.startswith('rgc.'): prepare = self._handle_rgc_call + elif oopspec_name.endswith('dict.lookup'): + # also ordereddict.lookup + prepare = self._handle_dict_lookup_call else: prepare = self.prepare_builtin_call try: @@ -1680,9 +1683,11 @@ # ---------- # Strings and Unicodes. - def _handle_oopspec_call(self, op, args, oopspecindex, extraeffect=None): + def _handle_oopspec_call(self, op, args, oopspecindex, extraeffect=None, + extradescr=None): calldescr = self.callcontrol.getcalldescr(op, oopspecindex, - extraeffect) + extraeffect, + extradescr=extradescr) if extraeffect is not None: assert (is_test_calldescr(calldescr) # for tests or calldescr.get_extra_info().extraeffect == extraeffect) @@ -1846,6 +1851,14 @@ return self._handle_oopspec_call(op, args, EffectInfo.OS_MATH_SQRT, EffectInfo.EF_ELIDABLE_CANNOT_RAISE) + def _handle_dict_lookup_call(self, op, oopspec_name, args): + extradescr1 = self.cpu.fielddescrof(op.args[1].concretetype.TO, + 'entries') + extradescr2 = self.cpu.interiorfielddescrof( + op.args[1].concretetype.TO.entries.TO, 'key') + return self._handle_oopspec_call(op, args, EffectInfo.OS_DICT_LOOKUP, + extradescr=[extradescr1, extradescr2]) + def _handle_rgc_call(self, op, oopspec_name, args): if oopspec_name == 'rgc.ll_shrink_array': return self._handle_oopspec_call(op, args, EffectInfo.OS_SHRINK_ARRAY, EffectInfo.EF_CAN_RAISE) diff --git a/rpython/jit/codewriter/test/test_jtransform.py b/rpython/jit/codewriter/test/test_jtransform.py --- a/rpython/jit/codewriter/test/test_jtransform.py +++ b/rpython/jit/codewriter/test/test_jtransform.py @@ -60,7 +60,8 @@ class FakeResidualCallControl: def guess_call_kind(self, op): return 'residual' - def getcalldescr(self, op, oopspecindex=None, extraeffect=None): + def getcalldescr(self, op, oopspecindex=None, extraeffect=None, + extradescr=None): return 'calldescr' def calldescr_canraise(self, calldescr): return True @@ -117,7 +118,8 @@ self.callinfocollection = FakeCallInfoCollection() def guess_call_kind(self, op): return 'builtin' - def getcalldescr(self, op, oopspecindex=None, extraeffect=None): + def getcalldescr(self, op, oopspecindex=None, extraeffect=None, + extradescr=None): assert oopspecindex is not None # in this test EI = effectinfo.EffectInfo if oopspecindex != EI.OS_ARRAYCOPY: diff --git a/rpython/jit/codewriter/test/test_list.py b/rpython/jit/codewriter/test/test_list.py --- a/rpython/jit/codewriter/test/test_list.py +++ b/rpython/jit/codewriter/test/test_list.py @@ -37,7 +37,8 @@ class FakeCallControl: class getcalldescr(AbstractDescr): - def __init__(self, op, oopspecindex=0, extraeffect=None): + def __init__(self, op, oopspecindex=0, extraeffect=None, + extradescr=None): self.op = op self.oopspecindex = oopspecindex def __repr__(self): diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py --- a/rpython/jit/metainterp/optimizeopt/heap.py +++ b/rpython/jit/metainterp/optimizeopt/heap.py @@ -1,8 +1,10 @@ import os +from rpython.jit.codewriter.effectinfo import EffectInfo +from rpython.jit.metainterp.optimizeopt.util import args_dict from rpython.jit.metainterp.history import Const from rpython.jit.metainterp.jitexc import JitException -from rpython.jit.metainterp.optimizeopt.optimizer import Optimization, MODE_ARRAY, LEVEL_KNOWNCLASS +from rpython.jit.metainterp.optimizeopt.optimizer import Optimization, MODE_ARRAY, LEVEL_KNOWNCLASS, REMOVED from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method from rpython.jit.metainterp.resoperation import rop, ResOperation from rpython.rlib.objectmodel import we_are_translated @@ -173,6 +175,10 @@ self.cached_fields = {} # cached array items: {array descr: {index: CachedField}} self.cached_arrayitems = {} + # cached dict items: {dict descr: {(optval, index): box-or-const}} + self.cached_dict_reads = {} + # cache of corresponding array descrs + self.corresponding_array_descrs = {} # self._lazy_setfields_and_arrayitems = [] self._remove_guard_not_invalidated = False @@ -180,9 +186,13 @@ self.postponed_op = None def force_at_end_of_preamble(self): + self.cached_dict_reads.clear() + self.corresponding_array_descrs.clear() self.force_all_lazy_setfields_and_arrayitems() def flush(self): + self.cached_dict_reads.clear() + self.corresponding_array_descrs.clear() self.force_all_lazy_setfields_and_arrayitems() self.emit_postponed_op() @@ -214,6 +224,7 @@ del self._lazy_setfields_and_arrayitems[:] self.cached_fields.clear() self.cached_arrayitems.clear() + self.cached_dict_reads.clear() def field_cache(self, descr): try: @@ -282,6 +293,44 @@ self.force_all_lazy_setfields_and_arrayitems() self.clean_caches() + def optimize_CALL(self, op): + # dispatch based on 'oopspecindex' to a method that handles + # specifically the given oopspec call. For non-oopspec calls, + # oopspecindex is just zero. + effectinfo = op.getdescr().get_extra_info() + oopspecindex = effectinfo.oopspecindex + if oopspecindex == EffectInfo.OS_DICT_LOOKUP: + if self._optimize_CALL_DICT_LOOKUP(op): + return + self.emit_operation(op) + + def _optimize_CALL_DICT_LOOKUP(self, op): + descrs = op.getdescr().get_extra_info().extradescrs + descr1 = descrs[0] + descr2 = descrs[1] + if descr1 in self.cached_dict_reads: + d = self.cached_dict_reads[descr1] + else: + d = self.cached_dict_reads[descr1] = args_dict() + self.corresponding_array_descrs[descr2] = descr1 + args = self.optimizer.make_args_key(op) + try: + res_v = d[args] + except KeyError: + d[args] = self.getvalue(op.result) + return False + else: + self.make_equal_to(op.result, res_v) + self.last_emitted_operation = REMOVED + return True + + def optimize_GUARD_NO_EXCEPTION(self, op): + if self.last_emitted_operation is REMOVED: + return + self.emit_operation(op) + + optimize_GUARD_EXCEPTION = optimize_GUARD_NO_EXCEPTION + def force_from_effectinfo(self, effectinfo): # XXX we can get the wrong complexity here, if the lists # XXX stored on effectinfo are large @@ -290,9 +339,20 @@ for arraydescr in effectinfo.readonly_descrs_arrays: self.force_lazy_setarrayitem(arraydescr) for fielddescr in effectinfo.write_descrs_fields: + try: + del self.cached_dict_reads[fielddescr] + except KeyError: + pass self.force_lazy_setfield(fielddescr, can_cache=False) for arraydescr in effectinfo.write_descrs_arrays: self.force_lazy_setarrayitem(arraydescr, can_cache=False) + for descr in effectinfo.write_descrs_interiorfields: + if descr in self.corresponding_array_descrs: + dictdescr = self.corresponding_array_descrs.pop(descr) + try: + del self.cached_dict_reads[dictdescr] + except KeyError: + pass # someone did it already if effectinfo.check_forces_virtual_or_virtualizable(): vrefinfo = self.optimizer.metainterp_sd.virtualref_info self.force_lazy_setfield(vrefinfo.descr_forced) diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py @@ -5444,6 +5444,21 @@ """ self.optimize_loop(ops, expected) + def test_consecutive_getinteriorfields(self): + py.test.skip("we want this to pass") + ops = """ + [p0, i0] + i1 = getinteriorfield_gc(p0, i0, descr=valuedescr) + i2 = getinteriorfield_gc(p0, i0, descr=valuedescr) + jump(i1, i2) + """ + expected = """ + [p0, i0] + i1 = getinteriorfield_gc(p0, i0, descr=valuedescr) + jump(i1, i1) + """ + self.optimize_loop(ops, expected) + class TestLLtype(BaseTestOptimizeBasic, LLtypeMixin): pass diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py --- a/rpython/jit/metainterp/optimizeopt/test/test_util.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py @@ -181,28 +181,29 @@ plaincalldescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, EffectInfo.MOST_GENERAL) nonwritedescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, - EffectInfo([], [], [], [])) + EffectInfo([], [], [], [], [], [])) writeadescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, - EffectInfo([], [], [adescr], [])) + EffectInfo([], [], [], [adescr], [], [])) writearraydescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, - EffectInfo([], [], [adescr], [arraydescr])) + EffectInfo([], [], [], [adescr], [arraydescr], + [])) readadescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, - EffectInfo([adescr], [], [], [])) + EffectInfo([adescr], [], [], [], [], [])) mayforcevirtdescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, - EffectInfo([nextdescr], [], [], [], + EffectInfo([nextdescr], [], [], [], [], [], EffectInfo.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE, can_invalidate=True)) arraycopydescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, - EffectInfo([], [arraydescr], [], [arraydescr], + EffectInfo([], [arraydescr], [], [], [arraydescr], [], EffectInfo.EF_CANNOT_RAISE, oopspecindex=EffectInfo.OS_ARRAYCOPY)) raw_malloc_descr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, - EffectInfo([], [], [], [], + EffectInfo([], [], [], [], [], [], EffectInfo.EF_CAN_RAISE, oopspecindex=EffectInfo.OS_RAW_MALLOC_VARSIZE_CHAR)) raw_free_descr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, - EffectInfo([], [], [], [], + EffectInfo([], [], [], [], [], [], EffectInfo.EF_CANNOT_RAISE, oopspecindex=EffectInfo.OS_RAW_FREE)) @@ -251,17 +252,18 @@ _oopspecindex = getattr(EffectInfo, _os) locals()[_name] = \ cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, - EffectInfo([], [], [], [], EffectInfo.EF_CANNOT_RAISE, + EffectInfo([], [], [], [], [], [], EffectInfo.EF_CANNOT_RAISE, oopspecindex=_oopspecindex)) # _oopspecindex = getattr(EffectInfo, _os.replace('STR', 'UNI')) locals()[_name.replace('str', 'unicode')] = \ cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, - EffectInfo([], [], [], [], EffectInfo.EF_CANNOT_RAISE, + EffectInfo([], [], [], [], [], [], EffectInfo.EF_CANNOT_RAISE, oopspecindex=_oopspecindex)) s2u_descr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, - EffectInfo([], [], [], [], oopspecindex=EffectInfo.OS_STR2UNICODE)) + EffectInfo([], [], [], [], [], [], + oopspecindex=EffectInfo.OS_STR2UNICODE)) # class LoopToken(AbstractDescr): @@ -277,7 +279,7 @@ virtualtokendescr = vrefinfo.descr_virtual_token virtualforceddescr = vrefinfo.descr_forced FUNC = lltype.FuncType([], lltype.Void) - ei = EffectInfo([], [], [], [], EffectInfo.EF_CANNOT_RAISE, + ei = EffectInfo([], [], [], [], [], [], EffectInfo.EF_CANNOT_RAISE, can_invalidate=False, oopspecindex=EffectInfo.OS_JIT_FORCE_VIRTUALIZABLE) clear_vable = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, ei) diff --git a/rpython/jit/metainterp/test/test_dict.py b/rpython/jit/metainterp/test/test_dict.py --- a/rpython/jit/metainterp/test/test_dict.py +++ b/rpython/jit/metainterp/test/test_dict.py @@ -193,6 +193,107 @@ self.check_simple_loop({'int_sub': 1, 'int_gt': 1, 'guard_true': 1, 'jump': 1}) + def test_dict_two_lookups(self): + driver = JitDriver(greens = [], reds = 'auto') + d = {'a': 3, 'b': 4} + indexes = ['a', 'b'] + + def f(n): + s = 0 + while n > 0: + driver.jit_merge_point() + s += d[indexes[n & 1]] + s += d[indexes[n & 1]] + n -= 1 + return s + + self.meta_interp(f, [10]) + # XXX should be one getinteriorfield_gc + self.check_simple_loop(call=1, getinteriorfield_gc=2, + guard_no_exception=1) + + def test_ordered_dict_two_lookups(self): + driver = JitDriver(greens = [], reds = 'auto') + d = OrderedDict() + d['a'] = 3 + d['b'] = 4 + indexes = ['a', 'b'] + + def f(n): + s = 0 + while n > 0: + driver.jit_merge_point() + s += d[indexes[n & 1]] + s += d[indexes[n & 1]] + n -= 1 + return s + + self.meta_interp(f, [10]) + # XXX should be one getinteriorfield_gc + self.check_simple_loop(call=1, getinteriorfield_gc=2, + guard_no_exception=1) + + def test_dict_insert_invalidates_caches(self): + driver = JitDriver(greens = [], reds = 'auto') + indexes = ['aa', 'b', 'cc'] + + def f(n): + d = {'aa': 3, 'b': 4, 'cc': 5} + s = 0 + while n > 0: + driver.jit_merge_point() + index = indexes[n & 1] + s += d[index] + d['aa'] += 1 # this will invalidate the index + s += d[index] + n -= 1 + return s + + res = self.meta_interp(f, [10]) + assert res == f(10) + self.check_simple_loop(call=5) + + def test_dict_array_write_invalidates_caches(self): + driver = JitDriver(greens = [], reds = 'auto') + indexes = ['aa', 'b', 'cc'] + + def f(n): + d = {'aa': 3, 'b': 4, 'cc': 5} + s = 0 + while n > 0: + driver.jit_merge_point() + index = indexes[n & 1] + s += d[index] + del d['cc'] + s += d[index] + d['cc'] = 3 + n -= 1 + return s + + exp = f(10) + res = self.meta_interp(f, [10]) + assert res == exp + self.check_simple_loop(call=7) + + def test_dict_double_lookup_2(self): + driver = JitDriver(greens = [], reds = 'auto') + indexes = ['aa', 'b', 'cc'] + + def f(n): + d = {'aa': 3, 'b': 4, 'cc': 5} + s = 0 + while n > 0: + driver.jit_merge_point() + index = indexes[n & 1] + s += d[index] + d[index] += 1 + n -= 1 + return s + + res = self.meta_interp(f, [10]) + assert res == f(10) + self.check_simple_loop(call=3) + class TestLLtype(DictTests, LLJitMixin): pass diff --git a/rpython/jit/metainterp/virtualizable.py b/rpython/jit/metainterp/virtualizable.py --- a/rpython/jit/metainterp/virtualizable.py +++ b/rpython/jit/metainterp/virtualizable.py @@ -302,7 +302,7 @@ self.clear_vable_ptr = self.warmrunnerdesc.helper_func( FUNCPTR, self.clear_vable_token) FUNC = FUNCPTR.TO - ei = EffectInfo([], [], [], [], EffectInfo.EF_CANNOT_RAISE, + ei = EffectInfo([], [], [], [], [], [], EffectInfo.EF_CANNOT_RAISE, can_invalidate=False, oopspecindex=EffectInfo.OS_JIT_FORCE_VIRTUALIZABLE) diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py --- a/rpython/rtyper/lltypesystem/rdict.py +++ b/rpython/rtyper/lltypesystem/rdict.py @@ -569,6 +569,7 @@ PERTURB_SHIFT = 5 @jit.look_inside_iff(lambda d, key, hash: jit.isvirtual(d) and jit.isconstant(key)) + at jit.oopspec('dict.lookup(d, key, hash)') def ll_dict_lookup(d, key, hash): entries = d.entries ENTRIES = lltype.typeOf(entries).TO diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py --- a/rpython/rtyper/lltypesystem/rordereddict.py +++ b/rpython/rtyper/lltypesystem/rordereddict.py @@ -709,6 +709,7 @@ @jit.look_inside_iff(lambda d, key, hash, store_flag, T: jit.isvirtual(d) and jit.isconstant(key)) + at jit.oopspec('ordereddict.lookup(d, key, hash, store_flag, T)') def ll_dict_lookup(d, key, hash, store_flag, T): INDEXES = _ll_ptr_to_array_of(T) entries = d.entries diff --git a/rpython/translator/backendopt/test/test_writeanalyze.py b/rpython/translator/backendopt/test/test_writeanalyze.py --- a/rpython/translator/backendopt/test/test_writeanalyze.py +++ b/rpython/translator/backendopt/test/test_writeanalyze.py @@ -353,3 +353,23 @@ result = wa.analyze(fgraph.startblock.operations[-1]) assert list(result) == [("struct", lltype.Ptr(S), "x")] + + def test_interiorfield(self): + A = lltype.GcArray(lltype.Struct('x', ('x', lltype.Signed), + ('y', lltype.Signed))) + + def g(x): + a = lltype.malloc(A, 1) + a[0].y = 3 + return f(a, x) + + def f(a, x): + a[0].x = x + return a[0].y + + t, wa = self.translate(g, [int]) + ggraph = graphof(t, g) + result = wa.analyze(ggraph.startblock.operations[-1]) + res = list(result) + assert ('readinteriorfield', lltype.Ptr(A), 'y') in res + assert ('interiorfield', lltype.Ptr(A), 'x') in res diff --git a/rpython/translator/backendopt/writeanalyze.py b/rpython/translator/backendopt/writeanalyze.py --- a/rpython/translator/backendopt/writeanalyze.py +++ b/rpython/translator/backendopt/writeanalyze.py @@ -1,4 +1,4 @@ -from rpython.flowspace.model import Variable +from rpython.flowspace.model import Variable, Constant from rpython.translator.backendopt import graphanalyze top_set = object() @@ -37,6 +37,12 @@ return top_set return result1.union(result2) + def _getinteriorname(self, op): + if (isinstance(op.args[1], Constant) and + isinstance(op.args[1].value, str)): + return op.args[1].value + return op.args[2].value + def analyze_simple_operation(self, op, graphinfo): if op.opname == "setfield": if graphinfo is None or not graphinfo.is_fresh_malloc(op.args[0]): @@ -45,11 +51,18 @@ elif op.opname == "setarrayitem": if graphinfo is None or not graphinfo.is_fresh_malloc(op.args[0]): return self._array_result(op.args[0].concretetype) + elif op.opname == "setinteriorfield": + if graphinfo is None or not graphinfo.is_fresh_malloc(op.args[0]): + name = self._getinteriorname(op) + return self._interiorfield_result(op.args[0].concretetype, name) return empty_set def _array_result(self, TYPE): return frozenset([("array", TYPE)]) + def _interiorfield_result(self, TYPE, fieldname): + return frozenset([("interiorfield", TYPE, fieldname)]) + def compute_graph_info(self, graph): return FreshMallocs(graph) @@ -99,4 +112,8 @@ elif op.opname == "getarrayitem": return frozenset([ ("readarray", op.args[0].concretetype)]) + elif op.opname == "getinteriorfield": + name = self._getinteriorname(op) + return frozenset([("readinteriorfield", op.args[0].concretetype, + name)]) return WriteAnalyzer.analyze_simple_operation(self, op, graphinfo) From noreply at buildbot.pypy.org Thu Mar 20 15:51:44 2014 From: noreply at buildbot.pypy.org (Manuel Jacob) Date: Thu, 20 Mar 2014 15:51:44 +0100 (CET) Subject: [pypy-commit] pypy default: Remove trailing whitespace introduced by the merge of improve-consecutive-dict-lookups. Message-ID: <20140320145144.21C021C1578@cobra.cs.uni-duesseldorf.de> Author: Manuel Jacob Branch: Changeset: r70115:4c6239cc82ff Date: 2014-03-20 15:50 +0100 http://bitbucket.org/pypy/pypy/changeset/4c6239cc82ff/ Log: Remove trailing whitespace introduced by the merge of improve- consecutive-dict-lookups. diff --git a/rpython/translator/backendopt/test/test_writeanalyze.py b/rpython/translator/backendopt/test/test_writeanalyze.py --- a/rpython/translator/backendopt/test/test_writeanalyze.py +++ b/rpython/translator/backendopt/test/test_writeanalyze.py @@ -362,7 +362,7 @@ a = lltype.malloc(A, 1) a[0].y = 3 return f(a, x) - + def f(a, x): a[0].x = x return a[0].y From noreply at buildbot.pypy.org Thu Mar 20 15:56:55 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 20 Mar 2014 15:56:55 +0100 (CET) Subject: [pypy-commit] pypy default: Kill some tests that are broken (they write a word at a random location in Message-ID: <20140320145655.914021D2864@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r70116:23f63b53184e Date: 2014-03-20 15:55 +0100 http://bitbucket.org/pypy/pypy/changeset/23f63b53184e/ Log: Kill some tests that are broken (they write a word at a random location in memory). Their usage is unclear. diff --git a/rpython/jit/backend/x86/test/test_assembler.py b/rpython/jit/backend/x86/test/test_assembler.py --- a/rpython/jit/backend/x86/test/test_assembler.py +++ b/rpython/jit/backend/x86/test/test_assembler.py @@ -55,9 +55,7 @@ asm = cpu.assembler asm.setup_once() asm.setup(looptoken) - self.fm = X86FrameManager(0) - self.xrm = X86XMMRegisterManager(None, frame_manager=self.fm, - assembler=asm) + self.xrm = X86XMMRegisterManager(None, assembler=asm) callback(asm) asm.mc.RET() rawstart = asm.materialize_loop(looptoken) @@ -75,29 +73,6 @@ res = self.do_test(callback) assert res == 42 - def test_push_stack(self): - def callback(asm): - loc = self.fm.frame_pos(5, INT) - asm.mc.SUB_ri(esp.value, 64) - asm.mov(imm(42), loc) - asm.regalloc_push(loc) - asm.regalloc_pop(eax) - asm.mc.ADD_ri(esp.value, 64) - res = self.do_test(callback) - assert res == 42 - - def test_pop_stack(self): - def callback(asm): - loc = self.fm.frame_pos(5, INT) - asm.mc.SUB_ri(esp.value, 64) - asm.mov(imm(42), edx) - asm.regalloc_push(edx) - asm.regalloc_pop(loc) - asm.mov(loc, eax) - asm.mc.ADD_ri(esp.value, 64) - res = self.do_test(callback) - assert res == 42 - def test_simple_xmm(self): def callback(asm): c = ConstFloat(longlong.getfloatstorage(-42.5)) @@ -108,33 +83,3 @@ asm.mc.CVTTSD2SI(eax, xmm0) res = self.do_test(callback) assert res == -42 - - def test_push_stack_xmm(self): - def callback(asm): - c = ConstFloat(longlong.getfloatstorage(-42.5)) - loc = self.xrm.convert_to_imm(c) - loc2 = self.fm.frame_pos(4, FLOAT) - asm.mc.SUB_ri(esp.value, 64) - asm.mov(loc, xmm5) - asm.mov(xmm5, loc2) - asm.regalloc_push(loc2) - asm.regalloc_pop(xmm0) - asm.mc.ADD_ri(esp.value, 64) - asm.mc.CVTTSD2SI(eax, xmm0) - res = self.do_test(callback) - assert res == -42 - - def test_pop_stack_xmm(self): - def callback(asm): - c = ConstFloat(longlong.getfloatstorage(-42.5)) - loc = self.xrm.convert_to_imm(c) - loc2 = self.fm.frame_pos(4, FLOAT) - asm.mc.SUB_ri(esp.value, 64) - asm.mov(loc, xmm5) - asm.regalloc_push(xmm5) - asm.regalloc_pop(loc2) - asm.mov(loc2, xmm0) - asm.mc.ADD_ri(esp.value, 64) - asm.mc.CVTTSD2SI(eax, xmm0) - res = self.do_test(callback) - assert res == -42 From noreply at buildbot.pypy.org Thu Mar 20 16:00:34 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 20 Mar 2014 16:00:34 +0100 (CET) Subject: [pypy-commit] pypy default: A test specifically for 76b06820d08b. Message-ID: <20140320150034.6E04C1D2888@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r70117:e30ec8f6a495 Date: 2014-03-20 15:59 +0100 http://bitbucket.org/pypy/pypy/changeset/e30ec8f6a495/ Log: A test specifically for 76b06820d08b. diff --git a/rpython/jit/backend/x86/test/test_assembler.py b/rpython/jit/backend/x86/test/test_assembler.py --- a/rpython/jit/backend/x86/test/test_assembler.py +++ b/rpython/jit/backend/x86/test/test_assembler.py @@ -83,3 +83,9 @@ asm.mc.CVTTSD2SI(eax, xmm0) res = self.do_test(callback) assert res == -42 + + def test_xmm_pushes_8_bytes(self): + def callback(asm): + asm.regalloc_push(xmm5) + asm.mc.ADD(esp, imm(8)) + self.do_test(callback) From noreply at buildbot.pypy.org Thu Mar 20 16:23:17 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 20 Mar 2014 16:23:17 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: in-progress Message-ID: <20140320152317.4250F1D2902@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r70118:19982e263d93 Date: 2014-03-20 16:22 +0100 http://bitbucket.org/pypy/pypy/changeset/19982e263d93/ Log: in-progress diff --git a/rpython/jit/backend/x86/stmtlocal.py b/rpython/jit/backend/x86/stmtlocal.py --- a/rpython/jit/backend/x86/stmtlocal.py +++ b/rpython/jit/backend/x86/stmtlocal.py @@ -11,7 +11,7 @@ eci = ExternalCompilationInfo(post_include_bits=[''' static long pypy__threadlocal_base(void) { - /* XXX ONLY LINUX WITH GCC FOR NOW XXX */ + /* XXX ONLY LINUX WITH GCC/CLANG FOR NOW XXX */ long result; asm("%s" : "=r"(result)); return result; @@ -23,8 +23,7 @@ 'pypy__threadlocal_base', [], lltype.Signed, compilation_info=eci, - threadsafe=False, - transactionsafe=True) + _nowrapper=True) def tl_segment_prefix(mc): @@ -47,9 +46,3 @@ 'stm_invalidate_jmp_buf', [llmemory.Address], lltype.Void, sandboxsafe=True, _nowrapper=True, transactionsafe=True) -stm_pointer_equal_fn = rffi.llexternal( - 'stm_pointer_equal', - [llmemory.Address, llmemory.Address], lltype.Bool, - sandboxsafe=True, _nowrapper=True, transactionsafe=True) - - diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py --- a/rpython/jit/metainterp/compile.py +++ b/rpython/jit/metainterp/compile.py @@ -502,9 +502,6 @@ rd_virtuals = None rd_pendingfields = lltype.nullptr(PENDINGFIELDSP.TO) - rd_stm_busy = False # same as CNT_BUSY_FLAG, in a different field, - # only for stm - status = r_uint(0) ST_BUSY_FLAG = 0x01 # if set, busy tracing from the guard @@ -573,30 +570,6 @@ _trace_and_compile_from_bridge._dont_inline_ = True def must_compile(self, deadframe, metainterp_sd, jitdriver_sd): - if rgc.stm_is_enabled(): - method = self.must_compile_stm - else: - method = self.must_compile_nonstm - return method(deadframe, metainterp_sd, jitdriver_sd) - - def must_compile_stm(self, deadframe, metainterp_sd, jitdriver_sd): - XXX # fix me - trace_eagerness = jitdriver_sd.warmstate.trace_eagerness - with stm_ignored: - approx_counter = self._counter + 1 - self._counter = approx_counter - - # The call to guard_already_patched is necessary because it is - # possible that the current transaction didn't see the - # patched JMP yet, but already sees rd_stm_busy as False (because - # the patching is in raw-memory). - # Thus it may try to compile a trace too and also patch the assembler. - # However, this would trigger the assertion in - # x86.assembler.patch_jump_for_descr. - return (approx_counter >= trace_eagerness and not self.rd_stm_busy - and not metainterp_sd.cpu.guard_already_patched(self)) - - def must_compile_nonstm(self, deadframe, metainterp_sd, jitdriver_sd): jitcounter = metainterp_sd.warmrunnerdesc.jitcounter # if self.status & (self.ST_BUSY_FLAG | self.ST_TYPE_MASK) == 0: @@ -638,23 +611,27 @@ intval * 1442968193) # increment = jitdriver_sd.warmstate.increment_trace_eagerness - return jitcounter.tick(hash, increment) + result = jitcounter.tick(hash, increment) + if rgc.stm_is_enabled(): + # The call to guard_already_patched is necessary because it is + # possible that the current transaction didn't see the + # patched JMP yet, but already sees the ST_BUSY_FLAG as 0 (because + # the patching is in raw-memory). + # Thus it may try to compile a trace too and also patch the assembler. + # However, this would trigger the assertion in + # x86.assembler.patch_jump_for_descr. + result = result and not metainterp_sd.cpu.guard_already_patched(self) + return result def start_compiling(self): # start tracing and compiling from this guard. - if rgc.stm_is_enabled(): - self.rd_stm_busy = True - else: - self.status |= self.ST_BUSY_FLAG + self.status |= self.ST_BUSY_FLAG def done_compiling(self): # done tracing and compiling from this guard. Note that if the # bridge has not been successfully compiled, the jitcounter for # it was reset to 0 already by jitcounter.tick() and not # incremented at all as long as ST_BUSY_FLAG was set. - if rgc.stm_is_enabled(): - XXX # review - self.rd_stm_busy = False self.status &= ~self.ST_BUSY_FLAG def compile_and_attach(self, metainterp, new_loop): diff --git a/rpython/jit/metainterp/counter.py b/rpython/jit/metainterp/counter.py --- a/rpython/jit/metainterp/counter.py +++ b/rpython/jit/metainterp/counter.py @@ -10,7 +10,10 @@ # keep in sync with the C code in pypy__decay_jit_counters ENTRY = lltype.Struct('timetable_entry', ('times', lltype.FixedSizeArray(rffi.FLOAT, 5)), - ('subhashes', lltype.FixedSizeArray(rffi.USHORT, 5))) + ('subhashes', lltype.FixedSizeArray(rffi.USHORT, 5)), + hints={'stm_dont_track_raw_accesses': True}) +ENTRY_ARRAY = lltype.Array(ENTRY, hints={'nolength': True, + 'stm_dont_track_raw_accesses': True}) class JitCounter: @@ -29,7 +32,7 @@ # and we're getting a 32-bytes-long entry; then this entry # contains 5 possible ways, each occupying 6 bytes: 4 bytes # for a float, and the 2 lowest bytes from the original hash. - self.timetable = lltype.malloc(rffi.CArray(ENTRY), self.size, + self.timetable = lltype.malloc(ENTRY_ARRAY, self.size, flavor='raw', zero=True, track_allocation=False) self._nexthash = r_uint(0) @@ -205,7 +208,8 @@ pypy__decay_jit_counters = rffi.llexternal( "pypy__decay_jit_counters", [rffi.CCHARP, lltype.Float, lltype.Signed], - lltype.Void, compilation_info=eci, _nowrapper=True, sandboxsafe=True) + lltype.Void, compilation_info=eci, _nowrapper=True, sandboxsafe=True, + transactionsafe=True) # ____________________________________________________________ diff --git a/rpython/jit/metainterp/optimizeopt/__init__.py b/rpython/jit/metainterp/optimizeopt/__init__.py --- a/rpython/jit/metainterp/optimizeopt/__init__.py +++ b/rpython/jit/metainterp/optimizeopt/__init__.py @@ -35,7 +35,7 @@ def build_opt_chain(metainterp_sd, enable_opts): optimizations = [] unroll = 'unroll' in enable_opts # 'enable_opts' is normally a dict - if config.translation.stm: + if metainterp_sd.config.translation.stm: optimizations.append(OptSTM()) for name, opt in unroll_all_opts: diff --git a/rpython/jit/metainterp/test/test_warmstate.py b/rpython/jit/metainterp/test/test_warmstate.py --- a/rpython/jit/metainterp/test/test_warmstate.py +++ b/rpython/jit/metainterp/test/test_warmstate.py @@ -4,7 +4,6 @@ from rpython.jit.metainterp.warmstate import wrap, unwrap, specialize_value from rpython.jit.metainterp.warmstate import equal_whatever, hash_whatever from rpython.jit.metainterp.warmstate import WarmEnterState -from rpython.jit.metainterp.warmstate import MODE_HAVE_PROC, MODE_TRACING from rpython.jit.metainterp.history import BoxInt, BoxFloat, BoxPtr from rpython.jit.metainterp.history import ConstInt, ConstFloat, ConstPtr from rpython.jit.metainterp.counter import DeterministicJitCounter diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py --- a/rpython/jit/metainterp/warmstate.py +++ b/rpython/jit/metainterp/warmstate.py @@ -1,7 +1,3 @@ -XXX -this file here is from default. It's missing some changes related to stm -that were conflicting (see the stmgc-c4 branch). We need to review carefully... -XXX import sys import weakref From noreply at buildbot.pypy.org Thu Mar 20 16:50:15 2014 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 20 Mar 2014 16:50:15 +0100 (CET) Subject: [pypy-commit] pypy default: fix test_whatsnew Message-ID: <20140320155015.B4CDF1C1413@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r70119:ae422bbeeb59 Date: 2014-03-20 17:49 +0200 http://bitbucket.org/pypy/pypy/changeset/ae422bbeeb59/ Log: fix test_whatsnew diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -114,3 +114,6 @@ app-level. The `Buffer` class is now used by `W_MemoryView` and `W_Buffer`, which is not present in Python 3. Previously `W_Buffer` was an alias to `Buffer`, which was wrappable itself. + +.. branch: improve-consecutive-dict-lookups +Improve the situation when dict lookups of the same key are performed in a chain From noreply at buildbot.pypy.org Thu Mar 20 16:54:26 2014 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 20 Mar 2014 16:54:26 +0100 (CET) Subject: [pypy-commit] pypy default: mention dict differencies Message-ID: <20140320155426.A25EA1C02AF@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r70120:eb1389e607fa Date: 2014-03-20 17:53 +0200 http://bitbucket.org/pypy/pypy/changeset/eb1389e607fa/ Log: mention dict differencies diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -292,6 +292,10 @@ depending on the compiler settings, the default of 768KB is enough for about 1400 calls. +* since the implementation of dictionary is different, the exact number + which ``__hash__`` and ``__eq__`` are called is different. Since CPython + does not give any specific guarantees either, don't rely on it. + * assignment to ``__class__`` is limited to the cases where it works on CPython 2.5. On CPython 2.6 and 2.7 it works in a bit more cases, which are not supported by PyPy so far. (If needed, From noreply at buildbot.pypy.org Thu Mar 20 16:58:15 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 20 Mar 2014 16:58:15 +0100 (CET) Subject: [pypy-commit] pypy default: Try to settle on this order of arguments, which is the same one as the Message-ID: <20140320155815.E78551C1413@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r70121:43b3b5bb76fa Date: 2014-03-20 16:57 +0100 http://bitbucket.org/pypy/pypy/changeset/43b3b5bb76fa/ Log: Try to settle on this order of arguments, which is the same one as the one on compile_loop / compile_bridge. diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -434,8 +434,8 @@ self.wb_slowpath[withcards + 2 * withfloats] = rawstart @rgc.no_release_gil - def assemble_loop(self, logger, loopname, inputargs, operations, looptoken, - log): + def assemble_loop(self, inputargs, operations, looptoken, log, + loopname, logger): '''adds the following attributes to looptoken: _ll_function_addr (address of the generated func, as an int) _ll_loop_code (debug: addr of the start of the ResOps) @@ -514,8 +514,8 @@ size_excluding_failure_stuff - looppos) @rgc.no_release_gil - def assemble_bridge(self, logger, faildescr, inputargs, operations, - original_loop_token, log): + def assemble_bridge(self, faildescr, inputargs, operations, + original_loop_token, log, logger): if not we_are_translated(): # Arguments should be unique assert len(set(inputargs)) == len(inputargs) diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py --- a/rpython/jit/backend/x86/runner.py +++ b/rpython/jit/backend/x86/runner.py @@ -93,16 +93,15 @@ def compile_loop(self, inputargs, operations, looptoken, log=True, name='', logger=None): - return self.assembler.assemble_loop(logger, name, inputargs, operations, - looptoken, log=log) + return self.assembler.assemble_loop(inputargs, operations, looptoken, log, + name, logger) def compile_bridge(self, faildescr, inputargs, operations, original_loop_token, log=True, logger=None): clt = original_loop_token.compiled_loop_token clt.compiling_a_bridge() - return self.assembler.assemble_bridge(logger, faildescr, inputargs, - operations, - original_loop_token, log=log) + return self.assembler.assemble_bridge(faildescr, inputargs, operations, + original_loop_token, log, logger) def clear_latest_values(self, count): setitem = self.assembler.fail_boxes_ptr.setitem From noreply at buildbot.pypy.org Thu Mar 20 18:27:02 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 20 Mar 2014 18:27:02 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: in-progress Message-ID: <20140320172702.AAD2A1C02AF@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r70122:a82acece79d2 Date: 2014-03-20 17:08 +0100 http://bitbucket.org/pypy/pypy/changeset/a82acece79d2/ Log: in-progress diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -685,8 +685,8 @@ @rgc.no_release_gil - def assemble_loop(self, loopname, inputargs, operations, looptoken, log, - logger=None): + def assemble_loop(self, inputargs, operations, looptoken, log, + loopname, logger): '''adds the following attributes to looptoken: _ll_function_addr (address of the generated func, as an int) _ll_loop_code (debug: addr of the start of the ResOps) @@ -767,7 +767,7 @@ @rgc.no_release_gil def assemble_bridge(self, faildescr, inputargs, operations, - original_loop_token, log, logger=None): + original_loop_token, log, logger): if not we_are_translated(): # Arguments should be unique assert len(set(inputargs)) == len(inputargs) diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -11,7 +11,7 @@ RegisterManager, TempBox, compute_vars_longevity, is_comparison_or_ovf_op) from rpython.jit.backend.x86 import rx86 from rpython.jit.backend.x86.arch import (WORD, JITFRAME_FIXED_SIZE, IS_X86_32, - IS_X86_64, FRAME_FIXED_SIZE) + IS_X86_64) from rpython.jit.backend.x86.jump import remap_frame_layout_mixed from rpython.jit.backend.x86.regloc import (FrameLoc, RegLoc, ConstFloatLoc, FloatImmedLoc, ImmedLoc, imm, imm0, imm1, ecx, eax, edx, ebx, esi, edi, @@ -26,7 +26,7 @@ from rpython.rlib import rgc from rpython.rlib.objectmodel import we_are_translated from rpython.rlib.rarithmetic import r_longlong, r_uint -from rpython.rtyper.lltypesystem.lloperation import llop +from rpython.rtyper.annlowlevel import cast_instance_to_gcref from rpython.rtyper.lltypesystem import lltype, rffi, rstr from rpython.rtyper.lltypesystem.lloperation import llop @@ -45,11 +45,9 @@ if isinstance(c, ConstInt): return imm(c.value) elif isinstance(c, ConstPtr): - # if we_are_translated() and c.value and rgc.can_move(c.value): - # not_implemented("convert_to_imm: ConstPtr needs special care") - if c.value and not c.imm_value: + if we_are_translated() and c.value and rgc.can_move(c.value): not_implemented("convert_to_imm: ConstPtr needs special care") - return imm(c.get_imm_value()) + return imm(rffi.cast(lltype.Signed, c.value)) else: not_implemented("convert_to_imm: got a %s" % c) diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py --- a/rpython/jit/backend/x86/runner.py +++ b/rpython/jit/backend/x86/runner.py @@ -94,16 +94,15 @@ def compile_loop(self, inputargs, operations, looptoken, log=True, name='', logger=None): - return self.assembler.assemble_loop(logger, name, inputargs, operations, - looptoken, log=log) + return self.assembler.assemble_loop(inputargs, operations, looptoken, log, + name, logger) def compile_bridge(self, faildescr, inputargs, operations, original_loop_token, log=True, logger=None): clt = original_loop_token.compiled_loop_token clt.compiling_a_bridge() - return self.assembler.assemble_bridge(logger, faildescr, inputargs, - operations, - original_loop_token, log=log) + return self.assembler.assemble_bridge(faildescr, inputargs, operations, + original_loop_token, log, logger) def guard_already_patched(self, faildescr): # only needed for STM so far diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py --- a/rpython/jit/metainterp/compile.py +++ b/rpython/jit/metainterp/compile.py @@ -1,7 +1,7 @@ import weakref from rpython.rtyper.lltypesystem import lltype, llmemory from rpython.rtyper.annlowlevel import cast_instance_to_gcref -from rpython.rlib.objectmodel import we_are_translated, stm_ignored +from rpython.rlib.objectmodel import we_are_translated from rpython.rlib.debug import debug_start, debug_stop, debug_print from rpython.rlib.rarithmetic import r_uint, intmask from rpython.rlib import rstack, rgc @@ -394,8 +394,8 @@ metainterp_sd.profiler.start_backend() debug_start("jit-backend") try: - asminfo = do_compile_bridge(metainterp_sd, faildescr, - inputargs, operations, + asminfo = do_compile_bridge(metainterp_sd, faildescr, inputargs, + operations, original_loop_token) finally: debug_stop("jit-backend") @@ -526,8 +526,6 @@ self.status = hash & self.ST_SHIFT_MASK def make_a_counter_per_value(self, guard_value_op): - if rgc.stm_is_enabled(): - return # XXX don't use the special counters in stm mode for now assert guard_value_op.getopnum() == rop.GUARD_VALUE box = guard_value_op.getarg(0) try: diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py --- a/rpython/jit/metainterp/history.py +++ b/rpython/jit/metainterp/history.py @@ -9,7 +9,6 @@ from rpython.jit.metainterp.resoperation import ResOperation, rop from rpython.jit.codewriter import heaptracker, longlong from rpython.rlib.objectmodel import compute_identity_hash -from rpython.rlib import rgc import weakref # ____________________________________________________________ @@ -298,24 +297,17 @@ class ConstPtr(Const): type = REF value = lltype.nullptr(llmemory.GCREF.TO) - imm_value = 0 - _attrs_ = ('value', 'imm_value',) + _attrs_ = ('value',) def __init__(self, value): assert lltype.typeOf(value) == llmemory.GCREF self.value = value - self.imm_value = 0 def clonebox(self): return BoxPtr(self.value) nonconstbox = clonebox - def get_imm_value(self): - # imm_value set if needed: - assert (not self.value) or self.imm_value - return self.imm_value - def getref_base(self): return self.value diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -18,7 +18,7 @@ from rpython.rlib.jit import Counters from rpython.rlib.objectmodel import we_are_translated, specialize from rpython.rlib.unroll import unrolling_iterable -from rpython.rtyper.lltypesystem import lltype, rclass, rffi, llmemory +from rpython.rtyper.lltypesystem import lltype, rclass, rffi # ____________________________________________________________ @@ -201,6 +201,7 @@ @arguments("int") def opimpl_stm_should_break_transaction(self, if_there_is_no_other): + from rpython.rtyper.lltypesystem import llmemory val = bool(if_there_is_no_other) mi = self.metainterp if val: @@ -2749,6 +2750,7 @@ # if the codewriter didn't produce any OS_LIBFFI_CALL at all. assert self.staticdata.has_libffi_call # + from rpython.rtyper.lltypesystem import llmemory from rpython.rlib.jit_libffi import CIF_DESCRIPTION_P from rpython.jit.backend.llsupport.ffisupport import get_arg_descr # From noreply at buildbot.pypy.org Thu Mar 20 18:27:04 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 20 Mar 2014 18:27:04 +0100 (CET) Subject: [pypy-commit] pypy default: From stm: to increment debug counters, replace the three instructions Message-ID: <20140320172704.282F51C02AF@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r70123:ab65db3705bd Date: 2014-03-20 18:20 +0100 http://bitbucket.org/pypy/pypy/changeset/ab65db3705bd/ Log: From stm: to increment debug counters, replace the three instructions with a single one. diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py --- a/rpython/jit/backend/llsupport/assembler.py +++ b/rpython/jit/backend/llsupport/assembler.py @@ -15,7 +15,7 @@ DEBUG_COUNTER = lltype.Struct('DEBUG_COUNTER', # 'b'ridge, 'l'abel or # 'e'ntry point - ('i', lltype.Signed), + ('i', lltype.Signed), # first field, at offset 0 ('type', lltype.Char), ('number', lltype.Signed) ) @@ -64,7 +64,6 @@ self.cpu = cpu self.memcpy_addr = 0 self.rtyper = cpu.rtyper - self.debug_counter_descr = cpu.fielddescrof(DEBUG_COUNTER, 'i') self._debug = False def setup_once(self): @@ -265,14 +264,8 @@ def _append_debugging_code(self, operations, tp, number, token): counter = self._register_counter(tp, number, token) c_adr = ConstInt(rffi.cast(lltype.Signed, counter)) - box = BoxInt() - box2 = BoxInt() - ops = [ResOperation(rop.GETFIELD_RAW, [c_adr], - box, descr=self.debug_counter_descr), - ResOperation(rop.INT_ADD, [box, ConstInt(1)], box2), - ResOperation(rop.SETFIELD_RAW, [c_adr, box2], - None, descr=self.debug_counter_descr)] - operations.extend(ops) + operations.append( + ResOperation(rop.INCREMENT_DEBUG_COUNTER, [c_adr], None)) def _register_counter(self, tp, number, token): # YYY very minor leak -- we need the counters to stay alive diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py --- a/rpython/jit/backend/test/runner_test.py +++ b/rpython/jit/backend/test/runner_test.py @@ -4338,3 +4338,12 @@ assert rffi.cast(lltype.Signed, a[0]) == -7654 assert rffi.cast(lltype.Signed, a[1]) == 777 lltype.free(a, flavor='raw') + + def test_increment_debug_counter(self): + foo = lltype.malloc(rffi.CArray(lltype.Signed), 1, flavor='raw') + foo[0] = 1789200 + self.execute_operation(rop.INCREMENT_DEBUG_COUNTER, + [ConstInt(rffi.cast(lltype.Signed, foo))], + 'void') + assert foo[0] == 1789201 + lltype.free(foo, flavor='raw') diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -1469,6 +1469,14 @@ ofs_loc) self.load_from_mem(resloc, src_addr, fieldsize_loc, sign_loc) + def genop_discard_increment_debug_counter(self, op, arglocs): + # The argument should be an immediate address. This should + # generate code equivalent to a GETFIELD_RAW, an ADD(1), and a + # SETFIELD_RAW. Here we use the direct from-memory-to-memory + # increment operation of x86. + base_loc, = arglocs + self.mc.INC(mem(base_loc, 0)) + def genop_discard_setfield_gc(self, op, arglocs): base_loc, ofs_loc, size_loc, value_loc = arglocs assert isinstance(size_loc, ImmedLoc) diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -1003,6 +1003,10 @@ consider_getfield_raw_pure = consider_getfield_gc consider_getfield_gc_pure = consider_getfield_gc + def consider_increment_debug_counter(self, op): + base_loc = self.loc(op.getarg(0)) + self.perform_discard(op, [base_loc]) + def consider_getarrayitem_gc(self, op): itemsize, ofs, sign = unpack_arraydescr(op.getdescr()) args = op.getarglist() diff --git a/rpython/jit/backend/x86/regloc.py b/rpython/jit/backend/x86/regloc.py --- a/rpython/jit/backend/x86/regloc.py +++ b/rpython/jit/backend/x86/regloc.py @@ -488,12 +488,22 @@ for possible_code in unrolling_location_codes: if code == possible_code: val = getattr(loc, "value_" + possible_code)() - if self.WORD == 8 and possible_code == 'i' and not rx86.fits_in_32bits(val): - self._load_scratch(val) + # Faking out of certain operations for x86_64 + fits32 = rx86.fits_in_32bits + if possible_code == 'i' and not fits32(val): + self._load_scratch(val) # for 'PUSH(imm)' _rx86_getattr(self, name + "_r")(X86_64_SCRATCH_REG.value) - else: - methname = name + "_" + possible_code - _rx86_getattr(self, methname)(val) + return + if possible_code == 'j' and not fits32(val): + val = self._addr_as_reg_offset(val) + _rx86_getattr(self, name + "_m")(val) + return + if possible_code == 'm' and not fits32(val[1]): + val = self._fix_static_offset_64_m(val) + if possible_code == 'a' and not fits32(val[3]): + val = self._fix_static_offset_64_a(val) + methname = name + "_" + possible_code + _rx86_getattr(self, methname)(val) return func_with_new_name(INSN, "INSN_" + name) @@ -600,6 +610,7 @@ TEST8 = _binaryop('TEST8') BTS = _binaryop('BTS') + INC = _unaryop('INC') ADD = _binaryop('ADD') SUB = _binaryop('SUB') IMUL = _binaryop('IMUL') diff --git a/rpython/jit/backend/x86/rx86.py b/rpython/jit/backend/x86/rx86.py --- a/rpython/jit/backend/x86/rx86.py +++ b/rpython/jit/backend/x86/rx86.py @@ -470,6 +470,9 @@ # ------------------------------ Arithmetic ------------------------------ + INC_m = insn(rex_w, '\xFF', orbyte(0), mem_reg_plus_const(1)) + INC_j = insn(rex_w, '\xFF', orbyte(0), abs_(1)) + ADD_ri,ADD_rr,ADD_rb,_,_,ADD_rm,ADD_rj,_,_ = common_modes(0) OR_ri, OR_rr, OR_rb, _,_,OR_rm, OR_rj, _,_ = common_modes(1) AND_ri,AND_rr,AND_rb,_,_,AND_rm,AND_rj,_,_ = common_modes(4) diff --git a/rpython/jit/backend/x86/test/test_regloc.py b/rpython/jit/backend/x86/test/test_regloc.py --- a/rpython/jit/backend/x86/test/test_regloc.py +++ b/rpython/jit/backend/x86/test/test_regloc.py @@ -373,3 +373,56 @@ '\x59' ) assert cb.getvalue() == expected_instructions + + # ------------------------------------------------------------ + + def test_push_immed64(self): + immed = 0x0123456789ABCDEF + cb = LocationCodeBuilder64() + cb.PUSH(imm(immed)) + # + expected_instructions = ( + # mov r11, 0x0123456789ABCDEF + '\x49\xBB\xEF\xCD\xAB\x89\x67\x45\x23\x01' + # push r11 + '\x41\x53' + ) + assert cb.getvalue() == expected_instructions + + def test_inc_64bit_address_1(self): + base_addr = 0x0123456789ABCDEF + cb = LocationCodeBuilder64() + cb.INC(AddressLoc(ImmedLoc(0), ImmedLoc(0), 0, base_addr)) + # this case is a INC_j + # + expected_instructions = ( + # mov r11, 0x0123456789ABCDEF + '\x49\xBB\xEF\xCD\xAB\x89\x67\x45\x23\x01' + # inc [r11] + '\x49\xFF\x03' + ) + assert cb.getvalue() == expected_instructions + + def test_inc_64bit_address_2(self): + py.test.skip("there is no unary instruction INSN_a so far") + base_addr = 0x0123456789ABCDEF + cb = LocationCodeBuilder64() + cb.INC(AddressLoc(ImmedLoc(0), edx, 3, base_addr)) + # this case would be a INC_a + xxx + + def test_inc_64bit_address_3(self): + base_addr = 0x0123456789ABCDEF + cb = LocationCodeBuilder64() + cb.INC(AddressLoc(eax, ImmedLoc(0), 0, base_addr)) + # this case is a INC_m + # + expected_instructions = ( + # mov r11, 0x0123456789ABCDEF + '\x49\xBB\xEF\xCD\xAB\x89\x67\x45\x23\x01' + # lea r11, [rax+r11] + '\x4E\x8D\x1C\x18' + # inc [r11] + '\x49\xFF\x03' + ) + assert cb.getvalue() == expected_instructions diff --git a/rpython/jit/backend/x86/test/test_runner.py b/rpython/jit/backend/x86/test/test_runner.py --- a/rpython/jit/backend/x86/test/test_runner.py +++ b/rpython/jit/backend/x86/test/test_runner.py @@ -427,8 +427,8 @@ debug._log = None # assert ops_offset is looptoken._x86_ops_offset - # 2*(getfield_raw/int_add/setfield_raw) + ops + None - assert len(ops_offset) == 2*3 + len(operations) + 1 + # 2*increment_debug_counter + ops + None + assert len(ops_offset) == 2 + len(operations) + 1 assert (ops_offset[operations[0]] <= ops_offset[operations[1]] <= ops_offset[operations[2]] <= diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py --- a/rpython/jit/metainterp/executor.py +++ b/rpython/jit/metainterp/executor.py @@ -332,6 +332,7 @@ continue if value in (rop.FORCE_TOKEN, rop.CALL_ASSEMBLER, + rop.INCREMENT_DEBUG_COUNTER, rop.COND_CALL_GC_WB, rop.COND_CALL_GC_WB_ARRAY, rop.DEBUG_MERGE_POINT, diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py --- a/rpython/jit/metainterp/resoperation.py +++ b/rpython/jit/metainterp/resoperation.py @@ -494,6 +494,7 @@ # must be forced, however we need to execute it anyway '_NOSIDEEFFECT_LAST', # ----- end of no_side_effect operations ----- + 'INCREMENT_DEBUG_COUNTER/1', 'SETARRAYITEM_GC/3d', 'SETARRAYITEM_RAW/3d', 'SETINTERIORFIELD_GC/3d', From noreply at buildbot.pypy.org Thu Mar 20 18:27:05 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 20 Mar 2014 18:27:05 +0100 (CET) Subject: [pypy-commit] pypy default: The strict minimum for ARM is this, I think Message-ID: <20140320172705.6DEA51C02AF@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r70124:aaf62bccd9c3 Date: 2014-03-20 18:26 +0100 http://bitbucket.org/pypy/pypy/changeset/aaf62bccd9c3/ Log: The strict minimum for ARM is this, I think diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py --- a/rpython/jit/backend/arm/opassembler.py +++ b/rpython/jit/backend/arm/opassembler.py @@ -583,6 +583,10 @@ emit_op_getfield_raw_pure = emit_op_getfield_gc emit_op_getfield_gc_pure = emit_op_getfield_gc + def emit_op_increment_debug_counter(self, op, arglocs, regalloc, fcond): + # XXX implement me + return fcond + def emit_op_getinteriorfield_gc(self, op, arglocs, regalloc, fcond): (base_loc, index_loc, res_loc, ofs_loc, ofs, itemsize, fieldsize) = arglocs diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py --- a/rpython/jit/backend/arm/regalloc.py +++ b/rpython/jit/backend/arm/regalloc.py @@ -849,6 +849,10 @@ prepare_op_getfield_raw_pure = prepare_op_getfield_gc prepare_op_getfield_gc_pure = prepare_op_getfield_gc + def prepare_op_increment_debug_counter(self, op, fcond): + # XXX implement me + return [] + def prepare_op_getinteriorfield_gc(self, op, fcond): t = unpack_interiorfielddescr(op.getdescr()) ofs, itemsize, fieldsize, sign = t From noreply at buildbot.pypy.org Thu Mar 20 18:46:05 2014 From: noreply at buildbot.pypy.org (pjenvey) Date: Thu, 20 Mar 2014 18:46:05 +0100 (CET) Subject: [pypy-commit] pypy default: backport 9813b8a36001 from py3k: cache UnsupportedOperation but this time to Message-ID: <20140320174605.B9E8F1C1413@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: Changeset: r70125:f15f9a524509 Date: 2014-03-20 10:42 -0700 http://bitbucket.org/pypy/pypy/changeset/f15f9a524509/ Log: backport 9813b8a36001 from py3k: cache UnsupportedOperation but this time to avoid potential module reloading issues diff --git a/pypy/module/_io/__init__.py b/pypy/module/_io/__init__.py --- a/pypy/module/_io/__init__.py +++ b/pypy/module/_io/__init__.py @@ -8,6 +8,8 @@ interpleveldefs = { 'DEFAULT_BUFFER_SIZE': 'space.wrap(interp_iobase.DEFAULT_BUFFER_SIZE)', 'BlockingIOError': 'interp_io.W_BlockingIOError', + 'UnsupportedOperation': + 'space.fromcache(interp_io.Cache).w_unsupportedoperation', '_IOBase': 'interp_iobase.W_IOBase', '_RawIOBase': 'interp_iobase.W_RawIOBase', '_BufferedIOBase': 'interp_bufferedio.W_BufferedIOBase', @@ -26,16 +28,6 @@ 'IncrementalNewlineDecoder': 'interp_textio.W_IncrementalNewlineDecoder', } - def init(self, space): - MixedModule.init(self, space) - w_UnsupportedOperation = space.call_function( - space.w_type, - space.wrap('UnsupportedOperation'), - space.newtuple([space.w_ValueError, space.w_IOError]), - space.newdict()) - space.setattr(self, space.wrap('UnsupportedOperation'), - w_UnsupportedOperation) - def shutdown(self, space): # at shutdown, flush all open streams. Ignore I/O errors. from pypy.module._io.interp_iobase import get_autoflusher diff --git a/pypy/module/_io/interp_io.py b/pypy/module/_io/interp_io.py --- a/pypy/module/_io/interp_io.py +++ b/pypy/module/_io/interp_io.py @@ -10,6 +10,12 @@ from rpython.rtyper.module.ll_os_stat import STAT_FIELD_TYPES +class Cache: + def __init__(self, space): + self.w_unsupportedoperation = space.new_exception_class( + "io.UnsupportedOperation", + space.newtuple([space.w_ValueError, space.w_IOError])) + class W_BlockingIOError(W_IOError): def __init__(self, space): W_IOError.__init__(self, space) From noreply at buildbot.pypy.org Thu Mar 20 19:46:44 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 20 Mar 2014 19:46:44 +0100 (CET) Subject: [pypy-commit] pypy default: Fix for the test in llgraph/test/ Message-ID: <20140320184644.4224E1C02AF@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r70126:c6230cd996a0 Date: 2014-03-20 19:42 +0100 http://bitbucket.org/pypy/pypy/changeset/c6230cd996a0/ Log: Fix for the test in llgraph/test/ diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py --- a/rpython/jit/backend/llgraph/runner.py +++ b/rpython/jit/backend/llgraph/runner.py @@ -553,6 +553,10 @@ else: return self.bh_raw_load_i(struct, offset, descr) + def bh_increment_debug_counter(self, addr): + p = rffi.cast(rffi.CArrayPtr(lltype.Signed), addr) + p[0] += 1 + def unpack_arraydescr_size(self, arraydescr): from rpython.jit.backend.llsupport.symbolic import get_array_token from rpython.jit.backend.llsupport.descr import get_type_flag, FLAG_SIGNED From noreply at buildbot.pypy.org Thu Mar 20 20:11:04 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 20 Mar 2014 20:11:04 +0100 (CET) Subject: [pypy-commit] pypy default: fix builtin reimport/reload (test_zipimport_deflated failure) Message-ID: <20140320191104.3097D1C02AF@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r70127:f29413fd44b7 Date: 2014-03-20 15:09 -0400 http://bitbucket.org/pypy/pypy/changeset/f29413fd44b7/ Log: fix builtin reimport/reload (test_zipimport_deflated failure) diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -465,13 +465,8 @@ if isinstance(w_mod, Module): if not reuse and w_mod.startup_called: # Create a copy of the module - w_mod.getdict(self) # unlazy w_initialdict - w_new = self.wrap(Module(self, w_name)) - self.call_method(w_new.getdict(self), 'update', - w_mod.w_initialdict) - w_mod = w_new - else: - w_mod.init(self) + w_mod = self.wrap(w_mod.__class__(self, w_name)) + w_mod.init(self) # Add the module to sys.modules self.setitem(w_modules, w_name, w_mod) From noreply at buildbot.pypy.org Thu Mar 20 20:11:54 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 20 Mar 2014 20:11:54 +0100 (CET) Subject: [pypy-commit] pypy default: Fix x86/test/test_z* Message-ID: <20140320191154.49E871C02AF@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r70128:855c8b15eee6 Date: 2014-03-20 20:11 +0100 http://bitbucket.org/pypy/pypy/changeset/855c8b15eee6/ Log: Fix x86/test/test_z* diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py --- a/rpython/jit/metainterp/optimizeopt/heap.py +++ b/rpython/jit/metainterp/optimizeopt/heap.py @@ -306,6 +306,7 @@ def _optimize_CALL_DICT_LOOKUP(self, op): descrs = op.getdescr().get_extra_info().extradescrs + assert descrs # translation hint descr1 = descrs[0] descr2 = descrs[1] if descr1 in self.cached_dict_reads: From noreply at buildbot.pypy.org Thu Mar 20 20:24:01 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 20 Mar 2014 20:24:01 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Remove this Message-ID: <20140320192401.6498E1C02AF@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r70129:d272e010c27b Date: 2014-03-20 20:13 +0100 http://bitbucket.org/pypy/pypy/changeset/d272e010c27b/ Log: Remove this diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py --- a/rpython/jit/backend/llgraph/runner.py +++ b/rpython/jit/backend/llgraph/runner.py @@ -1032,9 +1032,6 @@ def execute_stm_transaction_break(self, _, really_wanted): pass - def execute_increment_debug_counter(self, descr, a): - pass - def execute_keepalive(self, descr, x): pass From noreply at buildbot.pypy.org Thu Mar 20 20:24:05 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 20 Mar 2014 20:24:05 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: hg merge default Message-ID: <20140320192405.D02721C02AF@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r70130:5d12511d4ee3 Date: 2014-03-20 20:22 +0100 http://bitbucket.org/pypy/pypy/changeset/5d12511d4ee3/ Log: hg merge default diff too long, truncating to 2000 out of 6372 lines diff --git a/lib-python/2.7/test/test_memoryview.py b/lib-python/2.7/test/test_memoryview.py --- a/lib-python/2.7/test/test_memoryview.py +++ b/lib-python/2.7/test/test_memoryview.py @@ -171,7 +171,7 @@ # very inconsisten on CPython. In PyPy, memoryview supports # the buffer interface, and thus the following comparison # succeeds. See also the comment in - # pypy.modules.__builtin__.interp_memoryview.W_MemoryView.descr_buffer + # pypy.objspace.std.memoryview.W_MemoryView.descr_buffer # # Comparison with objects which don't support the buffer API self.assertFalse(m == u"abcdef", "%s %s" % (self, tp)) diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -292,6 +292,10 @@ depending on the compiler settings, the default of 768KB is enough for about 1400 calls. +* since the implementation of dictionary is different, the exact number + which ``__hash__`` and ``__eq__`` are called is different. Since CPython + does not give any specific guarantees either, don't rely on it. + * assignment to ``__class__`` is limited to the cases where it works on CPython 2.5. On CPython 2.6 and 2.7 it works in a bit more cases, which are not supported by PyPy so far. (If needed, diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -105,3 +105,15 @@ .. branch: stdlib-2.7.6 Update stdlib to v2.7.6 + +.. branch: virtual-raw-store-load +Support for virtualizing raw_store/raw_load operations + +.. branch: refactor-buffer-api +Separate the interp-level buffer API from the buffer type exposed to +app-level. The `Buffer` class is now used by `W_MemoryView` and +`W_Buffer`, which is not present in Python 3. Previously `W_Buffer` was +an alias to `Buffer`, which was wrappable itself. + +.. branch: improve-consecutive-dict-lookups +Improve the situation when dict lookups of the same key are performed in a chain diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -194,6 +194,14 @@ def immutable_unique_id(self, space): return None + def buffer_w(self, space): + w_impl = space.lookup(self, '__buffer__') + if w_impl is not None: + w_result = space.get_and_call_function(w_impl, self) + if space.isinstance_w(w_result, space.w_buffer): + return w_result.buffer_w(space) + self._typed_unwrap_error(space, "buffer") + def str_w(self, space): self._typed_unwrap_error(space, "string") @@ -432,17 +440,16 @@ return name - def getbuiltinmodule(self, name, force_init=False): + def getbuiltinmodule(self, name, force_init=False, reuse=True): w_name = self.wrap(name) w_modules = self.sys.get('modules') - try: - w_mod = self.getitem(w_modules, w_name) - except OperationError, e: - if not e.match(self, self.w_KeyError): - raise - else: - if not force_init: - return w_mod + if not force_init: + assert reuse is True + try: + return self.getitem(w_modules, w_name) + except OperationError, e: + if not e.match(self, self.w_KeyError): + raise # If the module is a builtin but not yet imported, # retrieve it and initialize it @@ -453,13 +460,16 @@ "getbuiltinmodule() called with non-builtin module %s", name) else: + # Initialize the module + from pypy.interpreter.module import Module + if isinstance(w_mod, Module): + if not reuse and w_mod.startup_called: + # Create a copy of the module + w_mod = self.wrap(w_mod.__class__(self, w_name)) + w_mod.init(self) + # Add the module to sys.modules self.setitem(w_modules, w_name, w_mod) - - # And initialize it - from pypy.interpreter.module import Module - if isinstance(w_mod, Module): - w_mod.init(self) return w_mod def get_builtinmodule_to_install(self): @@ -1320,10 +1330,7 @@ 'to unsigned int')) def buffer_w(self, w_obj): - # returns a Buffer instance - from pypy.interpreter.buffer import Buffer - w_buffer = self.buffer(w_obj) - return self.interp_w(Buffer, w_buffer) + return w_obj.buffer_w(self) def rwbuffer_w(self, w_obj): # returns a RWBuffer instance @@ -1683,7 +1690,6 @@ ('set', 'set', 3, ['__set__']), ('delete', 'delete', 2, ['__delete__']), ('userdel', 'del', 1, ['__del__']), - ('buffer', 'buffer', 1, ['__buffer__']), # see buffer.py ] ObjSpace.BuiltinModuleTable = [ diff --git a/pypy/interpreter/buffer.py b/pypy/interpreter/buffer.py --- a/pypy/interpreter/buffer.py +++ b/pypy/interpreter/buffer.py @@ -1,32 +1,12 @@ """ Buffer protocol support. """ +from rpython.rlib.objectmodel import import_from_mixin -# The implementation of the buffer protocol. The basic idea is that we -# can ask any app-level object for a 'buffer' view on it, by calling its -# __buffer__() special method. It should return a wrapped instance of a -# subclass of the Buffer class defined below. Note that __buffer__() is -# a PyPy-only extension to the Python language, made necessary by the -# fact that it's not natural in PyPy to hack an interp-level-only -# interface. -# In normal usage, the convenience method space.buffer_w() should be -# used to get directly a Buffer instance. Doing so also gives you for -# free the typecheck that __buffer__() really returned a wrapped Buffer. - -import operator -from pypy.interpreter.baseobjspace import W_Root -from pypy.interpreter.typedef import TypeDef -from pypy.interpreter.gateway import interp2app, unwrap_spec -from pypy.interpreter.error import OperationError -from rpython.rlib.objectmodel import compute_hash, import_from_mixin -from rpython.rlib.rstring import StringBuilder - - -class Buffer(W_Root): - """Abstract base class for memory views.""" - - __slots__ = () # no extra slot here +class Buffer(object): + """Abstract base class for buffers.""" + __slots__ = [] def getlength(self): raise NotImplementedError @@ -50,93 +30,10 @@ def is_writable(self): return False - # __________ app-level support __________ - - def descr_len(self, space): - return space.wrap(self.getlength()) - - def descr_getitem(self, space, w_index): - start, stop, step, size = space.decode_index4(w_index, self.getlength()) - if step == 0: # index only - return space.wrap(self.getitem(start)) - res = self.getslice(start, stop, step, size) - return space.wrap(res) - - @unwrap_spec(newstring='bufferstr') - def descr_setitem(self, space, w_index, newstring): - if not isinstance(self, RWBuffer): - raise OperationError(space.w_TypeError, - space.wrap("buffer is read-only")) - start, stop, step, size = space.decode_index4(w_index, self.getlength()) - if step == 0: # index only - if len(newstring) != 1: - msg = 'buffer[index]=x: x must be a single character' - raise OperationError(space.w_TypeError, space.wrap(msg)) - char = newstring[0] # annotator hint - self.setitem(start, char) - elif step == 1: - if len(newstring) != size: - msg = "right operand length must match slice length" - raise OperationError(space.w_ValueError, space.wrap(msg)) - self.setslice(start, newstring) - else: - raise OperationError(space.w_ValueError, - space.wrap("buffer object does not support" - " slicing with a step")) - - def descr__buffer__(self, space): - return space.wrap(self) - - def descr_str(self, space): - return space.wrap(self.as_str()) - - @unwrap_spec(other='bufferstr') - def descr_add(self, space, other): - return space.wrap(self.as_str() + other) - - def _make_descr__cmp(name): - def descr__cmp(self, space, w_other): - if not isinstance(w_other, Buffer): - return space.w_NotImplemented - # xxx not the most efficient implementation - str1 = self.as_str() - str2 = w_other.as_str() - return space.wrap(getattr(operator, name)(str1, str2)) - descr__cmp.func_name = name - return descr__cmp - - descr_eq = _make_descr__cmp('eq') - descr_ne = _make_descr__cmp('ne') - descr_lt = _make_descr__cmp('lt') - descr_le = _make_descr__cmp('le') - descr_gt = _make_descr__cmp('gt') - descr_ge = _make_descr__cmp('ge') - - def descr_hash(self, space): - return space.wrap(compute_hash(self.as_str())) - - def descr_mul(self, space, w_times): - # xxx not the most efficient implementation - w_string = space.wrap(self.as_str()) - # use the __mul__ method instead of space.mul() so that we - # return NotImplemented instead of raising a TypeError - return space.call_method(w_string, '__mul__', w_times) - - def descr_repr(self, space): - if isinstance(self, RWBuffer): - info = 'read-write buffer' - else: - info = 'read-only buffer' - addrstring = self.getaddrstring(space) - - return space.wrap("<%s for 0x%s, size %d>" % - (info, addrstring, self.getlength())) - class RWBuffer(Buffer): - """Abstract base class for read-write memory views.""" - - __slots__ = () # no extra slot here + """Abstract base class for read-write buffers.""" + __slots__ = [] def is_writable(self): return True @@ -151,76 +48,8 @@ self.setitem(start + i, string[i]) - at unwrap_spec(offset=int, size=int) -def descr_buffer__new__(space, w_subtype, w_object, offset=0, size=-1): - # w_subtype can only be exactly 'buffer' for now - if not space.is_w(w_subtype, space.gettypefor(Buffer)): - raise OperationError(space.w_TypeError, - space.wrap("argument 1 must be 'buffer'")) - - if space.isinstance_w(w_object, space.w_unicode): - # unicode objects support the old buffer interface - # but not the new buffer interface (change in python 2.7) - from rpython.rlib.rstruct.unichar import pack_unichar, UNICODE_SIZE - unistr = space.unicode_w(w_object) - builder = StringBuilder(len(unistr) * UNICODE_SIZE) - for unich in unistr: - pack_unichar(unich, builder) - from pypy.interpreter.buffer import StringBuffer - w_buffer = space.wrap(StringBuffer(builder.build())) - else: - w_buffer = space.buffer(w_object) - - buffer = space.interp_w(Buffer, w_buffer) # type-check - if offset == 0 and size == -1: - return w_buffer - # handle buffer slices - if offset < 0: - raise OperationError(space.w_ValueError, - space.wrap("offset must be zero or positive")) - if size < -1: - raise OperationError(space.w_ValueError, - space.wrap("size must be zero or positive")) - if isinstance(buffer, RWBuffer): - buffer = RWSubBuffer(buffer, offset, size) - else: - buffer = SubBuffer(buffer, offset, size) - return space.wrap(buffer) - - -Buffer.typedef = TypeDef( - "buffer", - __doc__ = """\ -buffer(object [, offset[, size]]) - -Create a new buffer object which references the given object. -The buffer will reference a slice of the target object from the -start of the object (or at the specified offset). The slice will -extend to the end of the target object (or with the specified size). -""", - __new__ = interp2app(descr_buffer__new__), - __len__ = interp2app(Buffer.descr_len), - __getitem__ = interp2app(Buffer.descr_getitem), - __setitem__ = interp2app(Buffer.descr_setitem), - __buffer__ = interp2app(Buffer.descr__buffer__), - __str__ = interp2app(Buffer.descr_str), - __add__ = interp2app(Buffer.descr_add), - __eq__ = interp2app(Buffer.descr_eq), - __ne__ = interp2app(Buffer.descr_ne), - __lt__ = interp2app(Buffer.descr_lt), - __le__ = interp2app(Buffer.descr_le), - __gt__ = interp2app(Buffer.descr_gt), - __ge__ = interp2app(Buffer.descr_ge), - __hash__ = interp2app(Buffer.descr_hash), - __mul__ = interp2app(Buffer.descr_mul), - __rmul__ = interp2app(Buffer.descr_mul), - __repr__ = interp2app(Buffer.descr_repr), -) -Buffer.typedef.acceptable_as_base_class = False - -# ____________________________________________________________ - class StringBuffer(Buffer): + __slots__ = ['value'] def __init__(self, value): self.value = value @@ -241,43 +70,12 @@ assert 0 <= start <= stop return self.value[start:stop] return "".join([self.value[start + i*step] for i in xrange(size)]) - - -class StringLikeBuffer(Buffer): - """For app-level objects that already have a string-like interface - with __len__ and a __getitem__ that returns characters or (with - slicing) substrings.""" - # XXX this is inefficient, it should only be used temporarily - - def __init__(self, space, w_obj): - self.space = space - self.w_obj = w_obj - - def getlength(self): - space = self.space - return space.len_w(self.w_obj) - - def getitem(self, index): - space = self.space - s = space.str_w(space.getitem(self.w_obj, space.wrap(index))) - if len(s) != 1: - raise OperationError(space.w_ValueError, - space.wrap("character expected, got string")) - char = s[0] # annotator hint - return char - - def getslice(self, start, stop, step, size): - space = self.space - if step != 1: - raise OperationError(space.w_ValueError, space.wrap( - "buffer object does not support slicing with a step")) - s = space.str_w(space.getslice(self.w_obj, space.wrap(start), - space.wrap(stop))) - return s - # ____________________________________________________________ + class SubBufferMixin(object): + _attrs_ = ['buffer', 'offset', 'size'] + def __init__(self, buffer, offset, size): self.buffer = buffer self.offset = offset @@ -299,11 +97,14 @@ if start == stop: return '' # otherwise, adding self.offset might make them # out of bounds - return self.buffer.getslice(self.offset + start, self.offset + stop, step, size) + return self.buffer.getslice(self.offset + start, self.offset + stop, + step, size) + class SubBuffer(Buffer): import_from_mixin(SubBufferMixin) + class RWSubBuffer(RWBuffer): import_from_mixin(SubBufferMixin) diff --git a/pypy/interpreter/test/test_buffer.py b/pypy/interpreter/test/test_buffer.py --- a/pypy/interpreter/test/test_buffer.py +++ b/pypy/interpreter/test/test_buffer.py @@ -1,29 +1,25 @@ import py -from pypy.interpreter.buffer import Buffer from rpython.tool.udir import udir testdir = udir.ensure('test_buffer', dir=1) class TestBuffer: - def test_buffer_w(self): space = self.space w_hello = space.wrap('hello world') buf = space.buffer_w(w_hello) - assert isinstance(buf, Buffer) assert buf.getlength() == 11 assert buf.as_str() == 'hello world' assert buf.getslice(1, 6, 1, 5) == 'ello ' - assert space.buffer_w(space.wrap(buf)) is buf + assert space.buffer_w(space.newbuffer(buf)) is buf assert space.bufferstr_w(w_hello) == 'hello world' - assert space.bufferstr_w(space.buffer(w_hello)) == 'hello world' + assert space.bufferstr_w(space.newbuffer(space.buffer_w(w_hello))) == 'hello world' space.raises_w(space.w_TypeError, space.buffer_w, space.wrap(5)) - space.raises_w(space.w_TypeError, space.buffer, space.wrap(5)) def test_file_write(self): space = self.space - w_buffer = space.buffer(space.wrap('hello world')) + w_buffer = space.newbuffer(space.buffer_w(space.wrap('hello world'))) filename = str(testdir.join('test_file_write')) space.appexec([w_buffer, space.wrap(filename)], """(buffer, filename): f = open(filename, 'wb') @@ -44,4 +40,4 @@ space.bufferstr_w, space.wrap(u'\xe9')) -# Note: some app-level tests for buffer are in module/__builtin__/test/. +# Note: some app-level tests for buffer are in objspace/std/test/test_memoryview.py. diff --git a/pypy/module/__builtin__/__init__.py b/pypy/module/__builtin__/__init__.py --- a/pypy/module/__builtin__/__init__.py +++ b/pypy/module/__builtin__/__init__.py @@ -33,16 +33,11 @@ interpleveldefs = { # constants + '__debug__' : '(space.w_True)', # XXX 'None' : '(space.w_None)', 'False' : '(space.w_False)', 'True' : '(space.w_True)', - '__debug__' : '(space.w_True)', # XXX - 'type' : '(space.w_type)', - 'object' : '(space.w_object)', 'bytes' : '(space.w_str)', - 'unicode' : '(space.w_unicode)', - 'buffer' : 'interp_memoryview.W_Buffer', - 'memoryview' : 'interp_memoryview.W_MemoryView', 'file' : 'state.get(space).w_file', 'open' : 'state.get(space).w_file', diff --git a/pypy/module/__builtin__/interp_memoryview.py b/pypy/module/__builtin__/interp_memoryview.py deleted file mode 100644 --- a/pypy/module/__builtin__/interp_memoryview.py +++ /dev/null @@ -1,168 +0,0 @@ -""" -Implementation of the 'buffer' and 'memoryview' types. -""" -from pypy.interpreter.baseobjspace import W_Root -from pypy.interpreter import buffer -from pypy.interpreter.gateway import interp2app, unwrap_spec -from pypy.interpreter.typedef import TypeDef, GetSetProperty -from pypy.interpreter.error import OperationError -import operator - -W_Buffer = buffer.Buffer # actually implemented in pypy.interpreter.buffer - - -class W_MemoryView(W_Root): - """Implement the built-in 'memoryview' type as a thin wrapper around - an interp-level buffer. - """ - - def __init__(self, buf): - assert isinstance(buf, buffer.Buffer) - self.buf = buf - - def _make_descr__cmp(name): - def descr__cmp(self, space, w_other): - if isinstance(w_other, W_MemoryView): - # xxx not the most efficient implementation - str1 = self.as_str() - str2 = w_other.as_str() - return space.wrap(getattr(operator, name)(str1, str2)) - - try: - w_buf = space.buffer(w_other) - except OperationError, e: - if not e.match(space, space.w_TypeError): - raise - return space.w_NotImplemented - else: - str1 = self.as_str() - str2 = space.buffer_w(w_buf).as_str() - return space.wrap(getattr(operator, name)(str1, str2)) - descr__cmp.func_name = name - return descr__cmp - - descr_eq = _make_descr__cmp('eq') - descr_ne = _make_descr__cmp('ne') - descr_lt = _make_descr__cmp('lt') - descr_le = _make_descr__cmp('le') - descr_gt = _make_descr__cmp('gt') - descr_ge = _make_descr__cmp('ge') - - def as_str(self): - return self.buf.as_str() - - def getlength(self): - return self.buf.getlength() - - def getslice(self, start, stop): - if start < 0: - start = 0 - size = stop - start - if size < 0: - size = 0 - buf = self.buf - if isinstance(buf, buffer.RWBuffer): - buf = buffer.RWSubBuffer(buf, start, size) - else: - buf = buffer.SubBuffer(buf, start, size) - return W_MemoryView(buf) - - def descr_buffer(self, space): - """ - Note that memoryview() is very inconsistent in CPython: it does not - support the buffer interface but does support the new buffer - interface: as a result, it is possible to pass memoryview to - e.g. socket.send() but not to file.write(). For simplicity and - consistency, in PyPy memoryview DOES support buffer(), which means - that it is accepted in more places than CPython. - """ - return space.wrap(self.buf) - - def descr_tobytes(self, space): - return space.wrap(self.as_str()) - - def descr_tolist(self, space): - buf = self.buf - result = [] - for i in range(buf.getlength()): - result.append(space.wrap(ord(buf.getitem(i)))) - return space.newlist(result) - - def descr_getitem(self, space, w_index): - start, stop, step = space.decode_index(w_index, self.getlength()) - if step == 0: # index only - return space.wrap(self.buf.getitem(start)) - elif step == 1: - res = self.getslice(start, stop) - return space.wrap(res) - else: - raise OperationError(space.w_ValueError, - space.wrap("memoryview object does not support" - " slicing with a step")) - - @unwrap_spec(newstring='bufferstr') - def descr_setitem(self, space, w_index, newstring): - buf = self.buf - if isinstance(buf, buffer.RWBuffer): - buf.descr_setitem(space, w_index, newstring) - else: - raise OperationError(space.w_TypeError, - space.wrap("cannot modify read-only memory")) - - def descr_len(self, space): - return self.buf.descr_len(space) - - def w_get_format(self, space): - return space.wrap("B") - - def w_get_itemsize(self, space): - return space.wrap(1) - - def w_get_ndim(self, space): - return space.wrap(1) - - def w_is_readonly(self, space): - return space.wrap(not isinstance(self.buf, buffer.RWBuffer)) - - def w_get_shape(self, space): - return space.newtuple([space.wrap(self.getlength())]) - - def w_get_strides(self, space): - return space.newtuple([space.wrap(1)]) - - def w_get_suboffsets(self, space): - # I've never seen anyone filling this field - return space.w_None - - -def descr_new(space, w_subtype, w_object): - memoryview = W_MemoryView(space.buffer(w_object)) - return space.wrap(memoryview) - -W_MemoryView.typedef = TypeDef( - "memoryview", - __doc__ = """\ -Create a new memoryview object which references the given object. -""", - __new__ = interp2app(descr_new), - __buffer__ = interp2app(W_MemoryView.descr_buffer), - __eq__ = interp2app(W_MemoryView.descr_eq), - __ge__ = interp2app(W_MemoryView.descr_ge), - __getitem__ = interp2app(W_MemoryView.descr_getitem), - __gt__ = interp2app(W_MemoryView.descr_gt), - __le__ = interp2app(W_MemoryView.descr_le), - __len__ = interp2app(W_MemoryView.descr_len), - __lt__ = interp2app(W_MemoryView.descr_lt), - __ne__ = interp2app(W_MemoryView.descr_ne), - __setitem__ = interp2app(W_MemoryView.descr_setitem), - tobytes = interp2app(W_MemoryView.descr_tobytes), - tolist = interp2app(W_MemoryView.descr_tolist), - format = GetSetProperty(W_MemoryView.w_get_format), - itemsize = GetSetProperty(W_MemoryView.w_get_itemsize), - ndim = GetSetProperty(W_MemoryView.w_get_ndim), - readonly = GetSetProperty(W_MemoryView.w_is_readonly), - shape = GetSetProperty(W_MemoryView.w_get_shape), - strides = GetSetProperty(W_MemoryView.w_get_strides), - suboffsets = GetSetProperty(W_MemoryView.w_get_suboffsets), - ) -W_MemoryView.typedef.acceptable_as_base_class = False diff --git a/pypy/module/__builtin__/test/test_buffer.py b/pypy/module/__builtin__/test/test_buffer.py deleted file mode 100644 --- a/pypy/module/__builtin__/test/test_buffer.py +++ /dev/null @@ -1,229 +0,0 @@ -"""Tests some behaviour of the buffer type that is not tested in -lib-python/2.5.2/test/test_types.py where the stdlib buffer tests live.""" - -class AppTestBuffer: - spaceconfig = dict(usemodules=['array']) - - def test_unicode_buffer(self): - import sys - b = buffer(u"ab") - if sys.maxunicode == 65535: # UCS2 build - assert len(b) == 4 - if sys.byteorder == "big": - assert b[0:4] == "\x00a\x00b" - else: - assert b[0:4] == "a\x00b\x00" - else: # UCS4 build - assert len(b) == 8 - if sys.byteorder == "big": - assert b[0:8] == "\x00\x00\x00a\x00\x00\x00b" - else: - assert b[0:8] == "a\x00\x00\x00b\x00\x00\x00" - - def test_array_buffer(self): - import array - b = buffer(array.array("B", [1, 2, 3])) - assert len(b) == 3 - assert b[0:3] == "\x01\x02\x03" - - def test_nonzero(self): - assert buffer('\x00') - assert not buffer('') - import array - assert buffer(array.array("B", [0])) - assert not buffer(array.array("B", [])) - - def test_str(self): - assert str(buffer('hello')) == 'hello' - - def test_repr(self): - # from 2.5.2 lib tests - assert repr(buffer('hello')).startswith(' buffer('ab')) - assert buffer('ab') >= buffer('ab') - assert buffer('ab') != buffer('abc') - assert buffer('ab') < buffer('abc') - assert buffer('ab') <= buffer('ab') - assert buffer('ab') > buffer('aa') - assert buffer('ab') >= buffer('ab') - - def test_hash(self): - assert hash(buffer('hello')) == hash('hello') - - def test_mul(self): - assert buffer('ab') * 5 == 'ababababab' - assert buffer('ab') * (-2) == '' - assert 5 * buffer('ab') == 'ababababab' - assert (-2) * buffer('ab') == '' - - def test_offset_size(self): - b = buffer('hello world', 6) - assert len(b) == 5 - assert b[0] == 'w' - assert b[:] == 'world' - raises(IndexError, 'b[5]') - b = buffer(b, 2) - assert len(b) == 3 - assert b[0] == 'r' - assert b[:] == 'rld' - raises(IndexError, 'b[3]') - b = buffer('hello world', 1, 8) - assert len(b) == 8 - assert b[0] == 'e' - assert b[:] == 'ello wor' - raises(IndexError, 'b[8]') - b = buffer(b, 2, 3) - assert len(b) == 3 - assert b[2] == ' ' - assert b[:] == 'lo ' - raises(IndexError, 'b[3]') - b = buffer('hello world', 55) - assert len(b) == 0 - assert b[:] == '' - b = buffer('hello world', 6, 999) - assert len(b) == 5 - assert b[:] == 'world' - - raises(ValueError, buffer, "abc", -1) - raises(ValueError, buffer, "abc", 0, -2) - - def test_rw_offset_size(self): - import array - - a = array.array("c", 'hello world') - b = buffer(a, 6) - assert len(b) == 5 - assert b[0] == 'w' - assert b[:] == 'world' - raises(IndexError, 'b[5]') - b[0] = 'W' - assert str(b) == 'World' - assert a.tostring() == 'hello World' - b[:] = '12345' - assert a.tostring() == 'hello 12345' - raises(IndexError, 'b[5] = "."') - b[4:2] = '' - assert a.tostring() == 'hello 12345' - - b = buffer(b, 2) - assert len(b) == 3 - assert b[0] == '3' - assert b[:] == '345' - raises(IndexError, 'b[3]') - b[1] = 'X' - assert a.tostring() == 'hello 123X5' - raises(IndexError, 'b[3] = "."') - - a = array.array("c", 'hello world') - b = buffer(a, 1, 8) - assert len(b) == 8 - assert b[0] == 'e' - assert b[:] == 'ello wor' - raises(IndexError, 'b[8]') - b[0] = 'E' - assert str(b) == 'Ello wor' - assert a.tostring() == 'hEllo world' - b[:] = '12345678' - assert a.tostring() == 'h12345678ld' - raises(IndexError, 'b[8] = "."') - - b = buffer(b, 2, 3) - assert len(b) == 3 - assert b[2] == '5' - assert b[:] == '345' - raises(IndexError, 'b[3]') - b[1] = 'X' - assert a.tostring() == 'h123X5678ld' - raises(IndexError, 'b[3] = "."') - - b = buffer(a, 55) - assert len(b) == 0 - assert b[:] == '' - b = buffer(a, 6, 999) - assert len(b) == 5 - assert b[:] == '678ld' - - raises(ValueError, buffer, a, -1) - raises(ValueError, buffer, a, 0, -2) - - def test_slice(self): - # Test extended slicing by comparing with list slicing. - s = "".join(chr(c) for c in list(range(255, -1, -1))) - b = buffer(s) - indices = (0, None, 1, 3, 19, 300, -1, -2, -31, -300) - for start in indices: - for stop in indices: - # Skip step 0 (invalid) - for step in indices[1:]: - assert b[start:stop:step] == s[start:stop:step] - - def test_getitem_only_ints(self): - class MyInt(object): - def __init__(self, x): - self.x = x - - def __int__(self): - return self.x - - buf = buffer('hello world') - raises(TypeError, "buf[MyInt(0)]") - raises(TypeError, "buf[MyInt(0):MyInt(5)]") - - -class AppTestMemoryView: - def test_basic(self): - v = memoryview("abc") - assert v.tobytes() == "abc" - assert len(v) == 3 - assert list(v) == ['a', 'b', 'c'] - assert v.tolist() == [97, 98, 99] - assert v[1] == "b" - assert v[-1] == "c" - raises(TypeError, "v[1] = 'x'") - assert v.readonly is True - w = v[1:234] - assert isinstance(w, memoryview) - assert len(w) == 2 - - def test_rw(self): - data = bytearray('abcefg') - v = memoryview(data) - assert v.readonly is False - v[0] = 'z' - assert data == bytearray(eval("b'zbcefg'")) - v[1:4] = '123' - assert data == bytearray(eval("b'z123fg'")) - raises((ValueError, TypeError), "v[2] = 'spam'") - - def test_memoryview_attrs(self): - v = memoryview("a"*100) - assert v.format == "B" - assert v.itemsize == 1 - assert v.shape == (100,) - assert v.ndim == 1 - assert v.strides == (1,) - - def test_suboffsets(self): - v = memoryview("a"*100) - assert v.suboffsets == None - v = memoryview(buffer("a"*100, 2)) - assert v.shape == (98,) - assert v.suboffsets == None - - def test_compare(self): - assert memoryview("abc") == "abc" - assert memoryview("abc") == bytearray("abc") - assert memoryview("abc") != 3 diff --git a/pypy/module/__builtin__/test/test_builtin.py b/pypy/module/__builtin__/test/test_builtin.py --- a/pypy/module/__builtin__/test/test_builtin.py +++ b/pypy/module/__builtin__/test/test_builtin.py @@ -24,6 +24,17 @@ else: cls.w_safe_runtimerror = cls.space.wrap(sys.version_info < (2, 6)) + def test_builtin_names(self): + import __builtin__ + assert __builtin__.None is None + assert __builtin__.False is False + assert __builtin__.True is True + + assert __builtin__.buffer is buffer + assert __builtin__.bytes is str + assert __builtin__.dict is dict + assert __builtin__.memoryview is memoryview + def test_bytes_alias(self): assert bytes is str assert isinstance(eval("b'hi'"), str) diff --git a/pypy/module/__pypy__/bytebuffer.py b/pypy/module/__pypy__/bytebuffer.py --- a/pypy/module/__pypy__/bytebuffer.py +++ b/pypy/module/__pypy__/bytebuffer.py @@ -7,7 +7,6 @@ class ByteBuffer(RWBuffer): - def __init__(self, len): self.data = ['\x00'] * len @@ -23,4 +22,4 @@ @unwrap_spec(length=int) def bytebuffer(space, length): - return space.wrap(ByteBuffer(length)) + return space.newbuffer(ByteBuffer(length)) diff --git a/pypy/module/__pypy__/interp_builders.py b/pypy/module/__pypy__/interp_builders.py --- a/pypy/module/__pypy__/interp_builders.py +++ b/pypy/module/__pypy__/interp_builders.py @@ -4,7 +4,6 @@ from pypy.interpreter.typedef import TypeDef from rpython.rlib.rstring import UnicodeBuilder, StringBuilder from rpython.tool.sourcetools import func_with_new_name -from rpython.rlib import jit def create_builder(name, strtype, builder_cls): diff --git a/pypy/module/_cffi_backend/cbuffer.py b/pypy/module/_cffi_backend/cbuffer.py --- a/pypy/module/_cffi_backend/cbuffer.py +++ b/pypy/module/_cffi_backend/cbuffer.py @@ -1,9 +1,9 @@ -from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.buffer import RWBuffer from pypy.interpreter.error import oefmt from pypy.interpreter.gateway import unwrap_spec, interp2app from pypy.interpreter.typedef import TypeDef, make_weakref_descr from pypy.module._cffi_backend import cdataobj, ctypeptr, ctypearray +from pypy.objspace.std.memoryview import W_Buffer from rpython.rtyper.annlowlevel import llstr from rpython.rtyper.lltypesystem import rffi @@ -39,38 +39,19 @@ copy_string_to_raw(llstr(string), raw_cdata, 0, len(string)) -class MiniBuffer(W_Root): - # a different subclass of W_Root for the MiniBuffer, because we - # want a slightly different (simplified) API at the level of Python. +# Override the typedef to narrow down the interface that's exposed to app-level +class MiniBuffer(W_Buffer): def __init__(self, buffer, keepalive=None): - self.buffer = buffer + W_Buffer.__init__(self, buffer) self.keepalive = keepalive - def descr_len(self, space): - return self.buffer.descr_len(space) - - def descr_getitem(self, space, w_index): - return self.buffer.descr_getitem(space, w_index) - - @unwrap_spec(newstring='bufferstr') - def descr_setitem(self, space, w_index, newstring): - self.buffer.descr_setitem(space, w_index, newstring) - - def descr__buffer__(self, space): - return self.buffer.descr__buffer__(space) - - def descr_str(self, space): - return space.wrap(self.buffer.as_str()) - - MiniBuffer.typedef = TypeDef( "buffer", __module__ = "_cffi_backend", __len__ = interp2app(MiniBuffer.descr_len), __getitem__ = interp2app(MiniBuffer.descr_getitem), __setitem__ = interp2app(MiniBuffer.descr_setitem), - __buffer__ = interp2app(MiniBuffer.descr__buffer__), __weakref__ = make_weakref_descr(MiniBuffer), __str__ = interp2app(MiniBuffer.descr_str), ) diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py --- a/pypy/module/_cffi_backend/ctypestruct.py +++ b/pypy/module/_cffi_backend/ctypestruct.py @@ -80,7 +80,6 @@ return (cfield.ctype, cfield.offset) def _copy_from_same(self, cdata, w_ob): - space = self.space if isinstance(w_ob, cdataobj.W_CData): if w_ob.ctype is self and self.size >= 0: misc._raw_memcopy(w_ob._cdata, cdata, self.size) diff --git a/pypy/module/_cffi_backend/handle.py b/pypy/module/_cffi_backend/handle.py --- a/pypy/module/_cffi_backend/handle.py +++ b/pypy/module/_cffi_backend/handle.py @@ -1,4 +1,3 @@ -import weakref from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter.gateway import unwrap_spec from pypy.module._cffi_backend import ctypeobj, ctypeptr, cdataobj diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py --- a/pypy/module/_cffi_backend/misc.py +++ b/pypy/module/_cffi_backend/misc.py @@ -4,7 +4,7 @@ from rpython.rlib import jit from rpython.rlib.objectmodel import keepalive_until_here, specialize -from rpython.rlib.rarithmetic import r_uint, r_ulonglong, is_signed_integer_type +from rpython.rlib.rarithmetic import r_uint, r_ulonglong from rpython.rlib.unroll import unrolling_iterable from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from rpython.translator.tool.cbuild import ExternalCompilationInfo diff --git a/pypy/module/_file/__init__.py b/pypy/module/_file/__init__.py --- a/pypy/module/_file/__init__.py +++ b/pypy/module/_file/__init__.py @@ -1,7 +1,6 @@ - # Package initialisation from pypy.interpreter.mixedmodule import MixedModule -import sys + class Module(MixedModule): appleveldefs = { diff --git a/pypy/module/_file/test/test_file_extra.py b/pypy/module/_file/test/test_file_extra.py --- a/pypy/module/_file/test/test_file_extra.py +++ b/pypy/module/_file/test/test_file_extra.py @@ -200,6 +200,10 @@ assert f.closed == True def test_repr(self): + import sys + if '__pypy__' not in sys.builtin_module_names and \ + sys.version_info < (2, 7, 4): + skip("see cpython issue14161") assert repr(self.file).startswith( " 3: scope_id = space.uint_w(pieces_w[3]) else: scope_id = 0 - if flowinfo < 0 or flowinfo > 0xfffff: - raise OperationError(space.w_OverflowError, space.wrap( - "flowinfo must be 0-1048575.")) - flowinfo = rffi.cast(lltype.Unsigned, flowinfo) + flowinfo = make_unsigned_flowinfo(space, flowinfo) a = addr.lock(_c.sockaddr_in6) rffi.setintfield(a, 'c_sin6_port', rsocket.htons(port)) rffi.setintfield(a, 'c_sin6_flowinfo', rsocket.htonl(flowinfo)) @@ -97,10 +94,7 @@ else: flowinfo = 0 if len(pieces_w) > 3: scope_id = space.uint_w(pieces_w[3]) else: scope_id = 0 - if flowinfo < 0 or flowinfo > 0xfffff: - raise OperationError(space.w_OverflowError, space.wrap( - "flowinfo must be 0-1048575.")) - flowinfo = rffi.cast(lltype.Unsigned, flowinfo) + flowinfo = make_unsigned_flowinfo(space, flowinfo) return rsocket.INET6Address(host, port, flowinfo, scope_id) if rsocket.HAS_AF_UNIX and family == rsocket.AF_UNIX: return rsocket.UNIXAddress(space.str_w(w_address)) @@ -112,10 +106,16 @@ # XXX Hack to seperate rpython and pypy def make_ushort_port(space, port): if port < 0 or port > 0xffff: - raise OperationError(space.w_ValueError, space.wrap( + raise OperationError(space.w_OverflowError, space.wrap( "port must be 0-65535.")) return rffi.cast(rffi.USHORT, port) +def make_unsigned_flowinfo(space, flowinfo): + if flowinfo < 0 or flowinfo > 0xfffff: + raise OperationError(space.w_OverflowError, space.wrap( + "flowinfo must be 0-1048575.")) + return rffi.cast(lltype.Unsigned, flowinfo) + # XXX Hack to seperate rpython and pypy def ipaddr_from_object(space, w_sockaddr): host = space.str_w(space.getitem(w_sockaddr, space.wrap(0))) @@ -536,13 +536,9 @@ @unwrap_spec(family=int, type=int, proto=int) def newsocket(space, w_subtype, family=AF_INET, type=SOCK_STREAM, proto=0): - # XXX If we want to support subclassing the socket type we will need - # something along these lines. But allocate_instance is only defined - # on the standard object space, so this is not really correct. - #sock = space.allocate_instance(W_RSocket, w_subtype) - #Socket.__init__(sock, space, fd, family, type, proto) + sock = space.allocate_instance(W_RSocket, w_subtype) try: - sock = W_RSocket(family, type, proto) + W_RSocket.__init__(sock, family, type, proto) except SocketError, e: raise converted_error(space, e) return space.wrap(sock) diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py --- a/pypy/module/_socket/test/test_sock_app.py +++ b/pypy/module/_socket/test/test_sock_app.py @@ -399,7 +399,7 @@ name = s.getpeername() # Will raise socket.error if not connected assert name[1] == 80 s.close() - + def test_socket_connect_ex(self): import _socket s = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM, 0) @@ -425,8 +425,13 @@ def test_bigport(self): import _socket s = _socket.socket() - raises(ValueError, s.connect, ("localhost", 1000000)) - raises(ValueError, s.connect, ("localhost", -1)) + exc = raises(OverflowError, s.connect, ("localhost", -1)) + assert "port must be 0-65535." in str(exc.value) + exc = raises(OverflowError, s.connect, ("localhost", 1000000)) + assert "port must be 0-65535." in str(exc.value) + s = _socket.socket(_socket.AF_INET6) + exc = raises(OverflowError, s.connect, ("::1", 1234, 1048576)) + assert "flowinfo must be 0-1048575." in str(exc.value) def test_NtoH(self): import sys @@ -474,6 +479,13 @@ import socket s = socket.socket() + def test_subclass(self): + from _socket import socket + class MySock(socket): + blah = 123 + s = MySock() + assert s.blah == 123 + def test_getsetsockopt(self): import _socket as socket import struct @@ -575,11 +587,11 @@ class AppTestSocketTCP: + HOST = 'localhost' + def setup_class(cls): cls.space = space - HOST = 'localhost' - def setup_method(self, method): w_HOST = space.wrap(self.HOST) self.w_serv = space.appexec([w_socket, w_HOST], @@ -589,6 +601,7 @@ serv.listen(1) return serv ''') + def teardown_method(self, method): if hasattr(self, 'w_serv'): space.appexec([self.w_serv], '(serv): serv.close()') @@ -609,7 +622,7 @@ raises(error, raise_error) def test_recv_send_timeout(self): - from _socket import socket, timeout + from _socket import socket, timeout, SOL_SOCKET, SO_RCVBUF, SO_SNDBUF cli = socket() cli.connect(self.serv.getsockname()) t, addr = self.serv.accept() @@ -629,6 +642,9 @@ assert count is None buf = t.recv(1) assert buf == '?' + # speed up filling the buffers + t.setsockopt(SOL_SOCKET, SO_RCVBUF, 4096) + cli.setsockopt(SOL_SOCKET, SO_SNDBUF, 4096) # test send() timeout count = 0 try: @@ -656,7 +672,7 @@ conn, addr = self.serv.accept() buf = buffer(MSG) conn.send(buf) - buf = array.array('c', ' '*1024) + buf = array.array('c', ' ' * 1024) nbytes = cli.recv_into(buf) assert nbytes == len(MSG) msg = buf.tostring()[:len(MSG)] @@ -671,7 +687,7 @@ conn, addr = self.serv.accept() buf = buffer(MSG) conn.send(buf) - buf = array.array('c', ' '*1024) + buf = array.array('c', ' ' * 1024) nbytes, addr = cli.recvfrom_into(buf) assert nbytes == len(MSG) msg = buf.tostring()[:len(MSG)] @@ -682,6 +698,7 @@ cli = socket.socket(socket.AF_INET, socket.SOCK_STREAM) assert cli.family == socket.AF_INET + class AppTestErrno: def setup_class(cls): cls.space = space diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py --- a/pypy/module/_ssl/test/test_ssl.py +++ b/pypy/module/_ssl/test/test_ssl.py @@ -1,5 +1,5 @@ class AppTestSSL: - spaceconfig = dict(usemodules=('_ssl', '_socket')) + spaceconfig = dict(usemodules=('_ssl', '_socket', 'thread')) def setup_class(cls): import os diff --git a/pypy/module/_ssl/thread_lock.py b/pypy/module/_ssl/thread_lock.py --- a/pypy/module/_ssl/thread_lock.py +++ b/pypy/module/_ssl/thread_lock.py @@ -1,4 +1,5 @@ -from rpython.rlib.ropenssl import * +from rpython.rlib import rthread +from rpython.rlib.ropenssl import libraries from rpython.rtyper.lltypesystem import lltype, rffi from rpython.translator.tool.cbuild import ExternalCompilationInfo @@ -22,7 +23,6 @@ # without caring about the GIL. separate_module_source = """ - #include static unsigned int _ssl_locks_count = 0; @@ -62,13 +62,12 @@ } """ -from rpython.rlib import rthread - eci = rthread.eci.merge(ExternalCompilationInfo( separate_module_sources=[separate_module_source], post_include_bits=[ "int _PyPy_SSL_SetupThreads(void);"], export_symbols=['_PyPy_SSL_SetupThreads'], + libraries = libraries, )) _PyPy_SSL_SetupThreads = rffi.llexternal('_PyPy_SSL_SetupThreads', diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py --- a/pypy/module/array/interp_array.py +++ b/pypy/module/array/interp_array.py @@ -132,6 +132,9 @@ self.len = 0 self.allocated = 0 + def buffer_w(self, space): + return ArrayBuffer(self) + def descr_append(self, space, w_x): """ append(x) @@ -462,9 +465,6 @@ # Misc methods - def descr_buffer(self, space): - return space.wrap(ArrayBuffer(self)) - def descr_repr(self, space): if self.len == 0: return space.wrap("array('%s')" % self.typecode) @@ -508,7 +508,6 @@ __radd__ = interp2app(W_ArrayBase.descr_radd), __rmul__ = interp2app(W_ArrayBase.descr_rmul), - __buffer__ = interp2app(W_ArrayBase.descr_buffer), __repr__ = interp2app(W_ArrayBase.descr_repr), itemsize = GetSetProperty(descr_itemsize), diff --git a/pypy/module/array/test/test_array.py b/pypy/module/array/test/test_array.py --- a/pypy/module/array/test/test_array.py +++ b/pypy/module/array/test/test_array.py @@ -1,25 +1,8 @@ import sys -import py -import py.test - - -## class AppTestSimpleArray: -## spaceconfig = dict(usemodules=('array',)) -## def setup_class(cls): -## cls.w_simple_array = cls.space.appexec([], """(): -## import array -## return array.simple_array -## """) - -## def test_simple(self): -## a = self.simple_array(10) -## a[5] = 7.42 -## assert a[5] == 7.42 +import pytest class BaseArrayTests: - - def test_ctor(self): assert len(self.array('c')) == 0 assert len(self.array('i')) == 0 @@ -563,7 +546,6 @@ assert not a > 2*a assert not a >= 2*a - def test_reduce(self): import pickle a = self.array('i', [1, 2, 3]) @@ -794,7 +776,6 @@ assert img[3, 25] == 3 * 9 - def test_override_from(self): class mya(self.array): def fromlist(self, lst): @@ -879,41 +860,41 @@ def test_assign_object_with_special_methods(self): from array import array - + class Num(object): def __float__(self): return 5.25 - + def __int__(self): return 7 - + class NotNum(object): pass - + class Silly(object): def __float__(self): return None - + def __int__(self): - return None + return None class OldNum: def __float__(self): return 6.25 - + def __int__(self): return 8 - + class OldNotNum: pass - + class OldSilly: def __float__(self): return None - + def __int__(self): return None - + for tc in 'bBhHiIlL': a = array(tc, [0]) raises(TypeError, a.__setitem__, 0, 1.0) @@ -931,7 +912,7 @@ a = array(tc, [0]) a[0] = 1.0 a[0] = 1 - a[0] = Num() + a[0] = Num() assert a[0] == 5.25 raises(TypeError, a.__setitem__, NotNum()) a[0] = OldNum() @@ -939,24 +920,23 @@ raises(TypeError, a.__setitem__, OldNotNum()) raises(TypeError, a.__setitem__, Silly()) raises(TypeError, a.__setitem__, OldSilly()) - + a = array('c', 'hi') a[0] = 'b' assert a[0] == 'b' - + a = array('u', u'hi') a[0] = u'b' assert a[0] == u'b' - + class TestCPythonsOwnArray(BaseArrayTests): - def setup_class(cls): import array cls.array = array.array import struct cls.struct = struct - cls.tempfile = str(py.test.ensuretemp('array').join('tmpfile')) + cls.tempfile = str(pytest.ensuretemp('array').join('tmpfile')) cls.maxint = sys.maxint @@ -969,7 +949,7 @@ return array.array """) cls.w_tempfile = cls.space.wrap( - str(py.test.ensuretemp('array').join('tmpfile'))) + str(pytest.ensuretemp('array').join('tmpfile'))) cls.w_maxint = cls.space.wrap(sys.maxint) def test_buffer_info(self): @@ -1036,11 +1016,11 @@ def test_getitem_only_ints(self): class MyInt(object): - def __init__(self, x): - self.x = x + def __init__(self, x): + self.x = x - def __int__(self): - return self.x + def __int__(self): + return self.x a = self.array('i', [1, 2, 3, 4, 5, 6]) raises(TypeError, "a[MyInt(0)]") @@ -1050,4 +1030,3 @@ class AppTestArrayBuiltinShortcut(AppTestArray): spaceconfig = AppTestArray.spaceconfig.copy() spaceconfig['objspace.std.builtinshortcut'] = True - diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -22,10 +22,10 @@ from pypy.interpreter.nestedscope import Cell from pypy.interpreter.module import Module from pypy.interpreter.function import StaticMethod +from pypy.objspace.std.memoryview import W_MemoryView from pypy.objspace.std.sliceobject import W_SliceObject from pypy.module.__builtin__.descriptor import W_Property from pypy.module.__builtin__.interp_classobj import W_ClassObject -from pypy.module.__builtin__.interp_memoryview import W_MemoryView from pypy.module.micronumpy.base import W_NDimArray from rpython.rlib.entrypoint import entrypoint_lowlevel from rpython.rlib.rposix import is_valid_fd, validate_fd diff --git a/pypy/module/cpyext/bufferobject.py b/pypy/module/cpyext/bufferobject.py --- a/pypy/module/cpyext/bufferobject.py +++ b/pypy/module/cpyext/bufferobject.py @@ -1,11 +1,12 @@ from rpython.rtyper.lltypesystem import rffi, lltype +from pypy.interpreter.buffer import StringBuffer, SubBuffer +from pypy.interpreter.error import OperationError from pypy.module.cpyext.api import ( cpython_api, Py_ssize_t, cpython_struct, bootstrap_function, PyObjectFields, PyObject) from pypy.module.cpyext.pyobject import make_typedescr, Py_DecRef, make_ref -from pypy.interpreter.buffer import Buffer, StringBuffer, SubBuffer -from pypy.interpreter.error import OperationError from pypy.module.array.interp_array import ArrayBuffer +from pypy.objspace.std.memoryview import W_Buffer PyBufferObjectStruct = lltype.ForwardReference() @@ -24,7 +25,7 @@ @bootstrap_function def init_bufferobject(space): "Type description of PyBufferObject" - make_typedescr(space.gettypefor(Buffer).instancetypedef, + make_typedescr(space.w_buffer.instancetypedef, basestruct=PyBufferObject.TO, attach=buffer_attach, dealloc=buffer_dealloc, @@ -39,23 +40,26 @@ rffi.setintfield(py_buf, 'c_b_readonly', 1) rffi.setintfield(py_buf, 'c_b_hash', -1) - if isinstance(w_obj, SubBuffer): - py_buf.c_b_offset = w_obj.offset - w_obj = w_obj.buffer + assert isinstance(w_obj, W_Buffer) + buf = w_obj.buf - # If w_obj already allocated a fixed buffer, use it, and keep a - # reference to w_obj. + if isinstance(buf, SubBuffer): + py_buf.c_b_offset = buf.offset + buf = buf.buffer + + # If buf already allocated a fixed buffer, use it, and keep a + # reference to buf. # Otherwise, b_base stays NULL, and we own the b_ptr. - if isinstance(w_obj, StringBuffer): + if isinstance(buf, StringBuffer): py_buf.c_b_base = lltype.nullptr(PyObject.TO) - py_buf.c_b_ptr = rffi.cast(rffi.VOIDP, rffi.str2charp(w_obj.value)) - py_buf.c_b_size = w_obj.getlength() - elif isinstance(w_obj, ArrayBuffer): - w_base = w_obj.array + py_buf.c_b_ptr = rffi.cast(rffi.VOIDP, rffi.str2charp(buf.value)) + py_buf.c_b_size = buf.getlength() + elif isinstance(buf, ArrayBuffer): + w_base = buf.array py_buf.c_b_base = make_ref(space, w_base) - py_buf.c_b_ptr = rffi.cast(rffi.VOIDP, w_obj.array._charbuf_start()) - py_buf.c_b_size = w_obj.getlength() + py_buf.c_b_ptr = rffi.cast(rffi.VOIDP, buf.array._charbuf_start()) + py_buf.c_b_size = buf.getlength() else: raise OperationError(space.w_NotImplementedError, space.wrap( "buffer flavor not supported")) diff --git a/pypy/module/cpyext/include/numpy/arrayobject.h b/pypy/module/cpyext/include/numpy/arrayobject.h --- a/pypy/module/cpyext/include/numpy/arrayobject.h +++ b/pypy/module/cpyext/include/numpy/arrayobject.h @@ -20,7 +20,7 @@ #define PyArrayObject PyObject #define PyArray_Descr PyObject -extern PyTypeObject PyArray_Type; +PyAPI_DATA(PyTypeObject) PyArray_Type; typedef unsigned char npy_bool; typedef unsigned char npy_uint8; diff --git a/pypy/module/cpyext/include/pystate.h b/pypy/module/cpyext/include/pystate.h --- a/pypy/module/cpyext/include/pystate.h +++ b/pypy/module/cpyext/include/pystate.h @@ -21,9 +21,8 @@ #define Py_END_ALLOW_THREADS PyEval_RestoreThread(_save); \ } -typedef - enum {PyGILState_LOCKED, PyGILState_UNLOCKED} - PyGILState_STATE; +enum {PyGILState_LOCKED, PyGILState_UNLOCKED}; +typedef int PyGILState_STATE; #define PyThreadState_GET() PyThreadState_Get() diff --git a/pypy/module/cpyext/pystate.py b/pypy/module/cpyext/pystate.py --- a/pypy/module/cpyext/pystate.py +++ b/pypy/module/cpyext/pystate.py @@ -204,16 +204,14 @@ # Before external call is after running Python rffi.aroundstate.before() -PyGILState_STATE = rffi.COpaquePtr('PyGILState_STATE', - typedef='PyGILState_STATE', - compilation_info=CConfig._compilation_info_) +PyGILState_STATE = rffi.INT @cpython_api([], PyGILState_STATE, error=CANNOT_FAIL) def PyGILState_Ensure(space): if rffi.aroundstate.after: # After external call is before entering Python rffi.aroundstate.after() - return lltype.nullptr(PyGILState_STATE.TO) + return rffi.cast(PyGILState_STATE, 0) @cpython_api([PyGILState_STATE], lltype.Void) def PyGILState_Release(space, state): diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -15,7 +15,7 @@ from pypy.module.cpyext.pyerrors import PyErr_Occurred from pypy.module.cpyext.state import State from pypy.interpreter.error import OperationError, oefmt -from pypy.interpreter.buffer import Buffer as W_Buffer +from pypy.interpreter.buffer import Buffer from pypy.interpreter.argument import Arguments from rpython.rlib.unroll import unrolling_iterable from rpython.rlib.objectmodel import specialize @@ -228,7 +228,7 @@ check_num_args(space, w_args, 0) return space.wrap(generic_cpy_call(space, func_target, w_self)) -class CPyBuffer(W_Buffer): +class CPyBuffer(Buffer): # Similar to Py_buffer def __init__(self, ptr, size, w_obj): @@ -249,7 +249,7 @@ size = generic_cpy_call(space, func_target, w_self, index, ptr) if size < 0: space.fromcache(State).check_and_raise_exception(always=True) - return space.wrap(CPyBuffer(ptr[0], size, w_self)) + return space.newbuffer(CPyBuffer(ptr[0], size, w_self)) def get_richcmp_func(OP_CONST): def inner(space, w_self, w_args, func): diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py --- a/pypy/module/cpyext/test/test_arraymodule.py +++ b/pypy/module/cpyext/test/test_arraymodule.py @@ -1,7 +1,5 @@ from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase -import py -import sys class AppTestArrayModule(AppTestCpythonExtensionBase): enable_leak_checking = False @@ -21,7 +19,7 @@ module = self.import_module(name='array') arr = module.array('i', [1,2,3]) sum = 0 - for i in arr: + for i in arr: sum += i assert sum == 6 @@ -60,4 +58,3 @@ '\x02\0\0\0' '\x03\0\0\0' '\x04\0\0\0') - diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py --- a/pypy/module/cpyext/test/test_cpyext.py +++ b/pypy/module/cpyext/test/test_cpyext.py @@ -64,8 +64,10 @@ kwds["libraries"] = [api_library] # '%s' undefined; assuming extern returning int kwds["compile_extra"] = ["/we4013"] - # prevent linking with python27.lib - kwds["compile_extra"].append("/DPy_BUILD_CORE") + # prevent linking with PythonXX.lib + w_maj, w_min = space.fixedview(space.sys.get('version_info'), 5)[:2] + kwds["link_extra"] = ["/NODEFAULTLIB:Python%d%d.lib" % + (space.int_w(w_maj), space.int_w(w_min))] elif sys.platform == 'darwin': kwds["link_files"] = [str(api_library + '.dylib')] else: @@ -181,6 +183,19 @@ from rpython.rlib.clibffi import get_libc_name cls.w_libc = cls.space.wrap(get_libc_name()) + def setup_method(self, meth): + freeze_refcnts(self) + + def teardown_method(self, meth): + self.cleanup_references(self.space) + # XXX: like AppTestCpythonExtensionBase.teardown_method: + # find out how to disable check_and_print_leaks() if the + # test failed + assert not self.check_and_print_leaks(), ( + "Test leaks or loses object(s). You should also check if " + "the test actually passed in the first place; if it failed " + "it is likely to reach this place.") + def test_load_error(self): import cpyext raises(ImportError, cpyext.load_module, "missing.file", "foo") @@ -356,13 +371,12 @@ for name in self.imported_module_names: self.unimport_module(name) self.cleanup_references(self.space) - if self.check_and_print_leaks(): - assert False, ( - "Test leaks or loses object(s). You should also check if " - "the test actually passed in the first place; if it failed " - "it is likely to reach this place.") - # XXX find out how to disable check_and_print_leaks() if the - # XXX test failed... + # XXX: find out how to disable check_and_print_leaks() if the + # test failed... + assert not self.check_and_print_leaks(), ( + "Test leaks or loses object(s). You should also check if " + "the test actually passed in the first place; if it failed " + "it is likely to reach this place.") class AppTestCpythonExtension(AppTestCpythonExtensionBase): diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -1,40 +1,36 @@ import os -import sys +from rpython.rlib import jit +from rpython.rlib.objectmodel import specialize +from rpython.rlib.rstring import rsplit +from rpython.rtyper.annlowlevel import llhelper from rpython.rtyper.lltypesystem import rffi, lltype -from rpython.rtyper.annlowlevel import llhelper + from pypy.interpreter.baseobjspace import W_Root, DescrMismatch -from pypy.objspace.std.typeobject import W_TypeObject, find_best_base +from pypy.interpreter.error import OperationError from pypy.interpreter.typedef import GetSetProperty +from pypy.module.__builtin__.abstractinst import abstract_issubclass_w +from pypy.module.cpyext import structmemberdefs from pypy.module.cpyext.api import ( cpython_api, cpython_struct, bootstrap_function, Py_ssize_t, Py_ssize_tP, generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING, Py_TPFLAGS_HEAPTYPE, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL, - Py_TPFLAGS_HAVE_GETCHARBUFFER, - build_type_checkers, PyObjectFields) + Py_TPFLAGS_HAVE_GETCHARBUFFER, build_type_checkers) +from pypy.module.cpyext.methodobject import ( + PyDescr_NewWrapper, PyCFunction_NewEx, PyCFunction_typedef) +from pypy.module.cpyext.modsupport import convert_method_defs from pypy.module.cpyext.pyobject import ( PyObject, make_ref, create_ref, from_ref, get_typedescr, make_typedescr, - track_reference, RefcountState, borrow_from) -from pypy.interpreter.module import Module -from pypy.module.cpyext import structmemberdefs -from pypy.module.cpyext.modsupport import convert_method_defs + track_reference, RefcountState, borrow_from, Py_DecRef) +from pypy.module.cpyext.slotdefs import ( + slotdefs_for_tp_slots, slotdefs_for_wrappers, get_slot_tp_function) from pypy.module.cpyext.state import State -from pypy.module.cpyext.methodobject import ( - PyDescr_NewWrapper, PyCFunction_NewEx, PyCFunction_typedef) -from pypy.module.cpyext.pyobject import Py_IncRef, Py_DecRef, _Py_Dealloc from pypy.module.cpyext.structmember import PyMember_GetOne, PyMember_SetOne from pypy.module.cpyext.typeobjectdefs import ( PyTypeObjectPtr, PyTypeObject, PyGetSetDef, PyMemberDef, newfunc, PyNumberMethods, PyMappingMethods, PySequenceMethods, PyBufferProcs) -from pypy.module.cpyext.slotdefs import ( - slotdefs_for_tp_slots, slotdefs_for_wrappers, get_slot_tp_function) -from pypy.interpreter.buffer import Buffer -from pypy.interpreter.error import OperationError -from rpython.rlib.rstring import rsplit -from rpython.rlib.objectmodel import specialize -from pypy.module.__builtin__.abstractinst import abstract_issubclass_w -from pypy.module.__builtin__.interp_classobj import W_ClassObject -from rpython.rlib import jit +from pypy.objspace.std.typeobject import W_TypeObject, find_best_base From noreply at buildbot.pypy.org Thu Mar 20 20:26:16 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 20 Mar 2014 20:26:16 +0100 (CET) Subject: [pypy-commit] pypy default: backout 6002c93c0cc0 until properly fixed Message-ID: <20140320192616.09F8F1C02AF@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r70131:c402f888f629 Date: 2014-03-20 15:25 -0400 http://bitbucket.org/pypy/pypy/changeset/c402f888f629/ Log: backout 6002c93c0cc0 until properly fixed diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -440,11 +440,10 @@ return name - def getbuiltinmodule(self, name, force_init=False, reuse=True): + def getbuiltinmodule(self, name, force_init=False): w_name = self.wrap(name) w_modules = self.sys.get('modules') if not force_init: - assert reuse is True try: return self.getitem(w_modules, w_name) except OperationError, e: @@ -463,9 +462,6 @@ # Initialize the module from pypy.interpreter.module import Module if isinstance(w_mod, Module): - if not reuse and w_mod.startup_called: - # Create a copy of the module - w_mod = self.wrap(w_mod.__class__(self, w_name)) w_mod.init(self) # Add the module to sys.modules diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py --- a/pypy/module/imp/importing.py +++ b/pypy/module/imp/importing.py @@ -579,8 +579,7 @@ return space.call_method(find_info.w_loader, "load_module", w_modulename) if find_info.modtype == C_BUILTIN: - return space.getbuiltinmodule(find_info.filename, force_init=True, - reuse=reuse) + return space.getbuiltinmodule(find_info.filename, force_init=True) if find_info.modtype in (PY_SOURCE, PY_COMPILED, C_EXTENSION, PKG_DIRECTORY): w_mod = None diff --git a/pypy/module/imp/test/test_app.py b/pypy/module/imp/test/test_app.py --- a/pypy/module/imp/test/test_app.py +++ b/pypy/module/imp/test/test_app.py @@ -203,6 +203,7 @@ def test_builtin_reimport(self): # from https://bugs.pypy.org/issue1514 + skip("fix me") import sys, marshal old = marshal.loads @@ -222,6 +223,7 @@ # taken from https://bugs.pypy.org/issue1514, with extra cases # that show a difference with CPython: we can get on CPython # several module objects for the same built-in module :-( + skip("several built-in module objects: not supported by pypy") import sys, marshal old = marshal.loads diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py --- a/pypy/module/imp/test/test_import.py +++ b/pypy/module/imp/test/test_import.py @@ -578,6 +578,7 @@ assert hasattr(time, 'clock') def test_reimport_builtin_simple_case_2(self): + skip("fix me") import sys, time time.foo = "bar" del sys.modules['time'] @@ -585,6 +586,7 @@ assert not hasattr(time, 'foo') def test_reimport_builtin(self): + skip("fix me") import sys, time oldpath = sys.path time.tzset = "" From noreply at buildbot.pypy.org Thu Mar 20 20:27:21 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 20 Mar 2014 20:27:21 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: hg merge default Message-ID: <20140320192721.7B21A1C02AF@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r70132:7cdeff21eeec Date: 2014-03-20 20:26 +0100 http://bitbucket.org/pypy/pypy/changeset/7cdeff21eeec/ Log: hg merge default diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -440,11 +440,10 @@ return name - def getbuiltinmodule(self, name, force_init=False, reuse=True): + def getbuiltinmodule(self, name, force_init=False): w_name = self.wrap(name) w_modules = self.sys.get('modules') if not force_init: - assert reuse is True try: return self.getitem(w_modules, w_name) except OperationError, e: @@ -463,9 +462,6 @@ # Initialize the module from pypy.interpreter.module import Module if isinstance(w_mod, Module): - if not reuse and w_mod.startup_called: - # Create a copy of the module - w_mod = self.wrap(w_mod.__class__(self, w_name)) w_mod.init(self) # Add the module to sys.modules diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py --- a/pypy/module/imp/importing.py +++ b/pypy/module/imp/importing.py @@ -579,8 +579,7 @@ return space.call_method(find_info.w_loader, "load_module", w_modulename) if find_info.modtype == C_BUILTIN: - return space.getbuiltinmodule(find_info.filename, force_init=True, - reuse=reuse) + return space.getbuiltinmodule(find_info.filename, force_init=True) if find_info.modtype in (PY_SOURCE, PY_COMPILED, C_EXTENSION, PKG_DIRECTORY): w_mod = None diff --git a/pypy/module/imp/test/test_app.py b/pypy/module/imp/test/test_app.py --- a/pypy/module/imp/test/test_app.py +++ b/pypy/module/imp/test/test_app.py @@ -203,6 +203,7 @@ def test_builtin_reimport(self): # from https://bugs.pypy.org/issue1514 + skip("fix me") import sys, marshal old = marshal.loads @@ -222,6 +223,7 @@ # taken from https://bugs.pypy.org/issue1514, with extra cases # that show a difference with CPython: we can get on CPython # several module objects for the same built-in module :-( + skip("several built-in module objects: not supported by pypy") import sys, marshal old = marshal.loads diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py --- a/pypy/module/imp/test/test_import.py +++ b/pypy/module/imp/test/test_import.py @@ -578,6 +578,7 @@ assert hasattr(time, 'clock') def test_reimport_builtin_simple_case_2(self): + skip("fix me") import sys, time time.foo = "bar" del sys.modules['time'] @@ -585,6 +586,7 @@ assert not hasattr(time, 'foo') def test_reimport_builtin(self): + skip("fix me") import sys, time oldpath = sys.path time.tzset = "" From noreply at buildbot.pypy.org Thu Mar 20 20:59:12 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 20 Mar 2014 20:59:12 +0100 (CET) Subject: [pypy-commit] pypy default: simplify Message-ID: <20140320195912.D15D61C1413@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r70133:2d49948e8eba Date: 2014-03-20 15:57 -0400 http://bitbucket.org/pypy/pypy/changeset/2d49948e8eba/ Log: simplify diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py --- a/rpython/jit/metainterp/optimizeopt/heap.py +++ b/rpython/jit/metainterp/optimizeopt/heap.py @@ -308,12 +308,11 @@ descrs = op.getdescr().get_extra_info().extradescrs assert descrs # translation hint descr1 = descrs[0] - descr2 = descrs[1] - if descr1 in self.cached_dict_reads: + try: d = self.cached_dict_reads[descr1] - else: + except KeyError: d = self.cached_dict_reads[descr1] = args_dict() - self.corresponding_array_descrs[descr2] = descr1 + self.corresponding_array_descrs[descrs[1]] = descr1 args = self.optimizer.make_args_key(op) try: res_v = d[args] From noreply at buildbot.pypy.org Fri Mar 21 00:02:31 2014 From: noreply at buildbot.pypy.org (pjenvey) Date: Fri, 21 Mar 2014 00:02:31 +0100 (CET) Subject: [pypy-commit] pypy py3k: merge default Message-ID: <20140320230231.5C1E51D2866@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r70134:6ff661c8a2b0 Date: 2014-03-19 16:56 -0700 http://bitbucket.org/pypy/pypy/changeset/6ff661c8a2b0/ Log: merge default diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -441,10 +441,11 @@ return name - def getbuiltinmodule(self, name, force_init=False): + def getbuiltinmodule(self, name, force_init=False, reuse=True): w_name = self.wrap(name) w_modules = self.sys.get('modules') if not force_init: + assert reuse is True try: return self.getitem(w_modules, w_name) except OperationError, e: @@ -460,10 +461,18 @@ "getbuiltinmodule() called with non-builtin module %s", name) else: - # And initialize it + # Initialize the module from pypy.interpreter.module import Module if isinstance(w_mod, Module): - w_mod.init(self) + if not reuse and w_mod.startup_called: + # Create a copy of the module + w_mod.getdict(self) # unlazy w_initialdict + w_new = self.wrap(Module(self, w_name)) + self.call_method(w_new.getdict(self), 'update', + w_mod.w_initialdict) + w_mod = w_new + else: + w_mod.init(self) # Add the module to sys.modules self.setitem(w_modules, w_name, w_mod) diff --git a/pypy/module/cpyext/include/pystate.h b/pypy/module/cpyext/include/pystate.h --- a/pypy/module/cpyext/include/pystate.h +++ b/pypy/module/cpyext/include/pystate.h @@ -21,9 +21,8 @@ #define Py_END_ALLOW_THREADS PyEval_RestoreThread(_save); \ } -typedef - enum {PyGILState_LOCKED, PyGILState_UNLOCKED} - PyGILState_STATE; +enum {PyGILState_LOCKED, PyGILState_UNLOCKED}; +typedef int PyGILState_STATE; #define PyThreadState_GET() PyThreadState_Get() diff --git a/pypy/module/cpyext/pystate.py b/pypy/module/cpyext/pystate.py --- a/pypy/module/cpyext/pystate.py +++ b/pypy/module/cpyext/pystate.py @@ -208,16 +208,14 @@ # Before external call is after running Python rffi.aroundstate.before() -PyGILState_STATE = rffi.COpaquePtr('PyGILState_STATE', - typedef='PyGILState_STATE', - compilation_info=CConfig._compilation_info_) +PyGILState_STATE = rffi.INT @cpython_api([], PyGILState_STATE, error=CANNOT_FAIL) def PyGILState_Ensure(space): if rffi.aroundstate.after: # After external call is before entering Python rffi.aroundstate.after() - return lltype.nullptr(PyGILState_STATE.TO) + return rffi.cast(PyGILState_STATE, 0) @cpython_api([PyGILState_STATE], lltype.Void) def PyGILState_Release(space, state): diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py --- a/pypy/module/imp/importing.py +++ b/pypy/module/imp/importing.py @@ -585,7 +585,8 @@ return space.call_method(find_info.w_loader, "load_module", w_modulename) if find_info.modtype == C_BUILTIN: - return space.getbuiltinmodule(find_info.filename, force_init=True) + return space.getbuiltinmodule(find_info.filename, force_init=True, + reuse=reuse) if find_info.modtype in (PY_SOURCE, PY_COMPILED, C_EXTENSION, PKG_DIRECTORY): w_mod = None diff --git a/pypy/module/imp/test/test_app.py b/pypy/module/imp/test/test_app.py --- a/pypy/module/imp/test/test_app.py +++ b/pypy/module/imp/test/test_app.py @@ -219,7 +219,6 @@ def test_builtin_reimport(self): # from https://bugs.pypy.org/issue1514 - skip("fix me") import sys, marshal old = marshal.loads @@ -239,7 +238,6 @@ # taken from https://bugs.pypy.org/issue1514, with extra cases # that show a difference with CPython: we can get on CPython # several module objects for the same built-in module :-( - skip("several built-in module objects: not supported by pypy") import sys, marshal old = marshal.loads diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py --- a/pypy/module/imp/test/test_import.py +++ b/pypy/module/imp/test/test_import.py @@ -652,7 +652,6 @@ assert hasattr(time, 'clock') def test_reimport_builtin_simple_case_2(self): - skip("fix me") import sys, time time.foo = "bar" del sys.modules['time'] @@ -660,7 +659,6 @@ assert not hasattr(time, 'foo') def test_reimport_builtin(self): - skip("fix me") import imp, sys, time oldpath = sys.path time.tzset = "" diff --git a/pypy/module/marshal/test/test_marshal.py b/pypy/module/marshal/test/test_marshal.py --- a/pypy/module/marshal/test/test_marshal.py +++ b/pypy/module/marshal/test/test_marshal.py @@ -173,7 +173,7 @@ import marshal types = (float, complex, int, tuple, list, dict, set, frozenset) for cls in types: - print cls + print(cls) class subtype(cls): pass exc = raises(ValueError, marshal.dumps, subtype) diff --git a/pypy/objspace/std/memoryview.py b/pypy/objspace/std/memoryview.py --- a/pypy/objspace/std/memoryview.py +++ b/pypy/objspace/std/memoryview.py @@ -41,14 +41,14 @@ self.buf = buf def buffer_w(self, space): - """Note that memoryview() is very inconsistent in CPython: it - does not support the buffer interface but does support the new - buffer interface: as a result, it is possible to pass memoryview - to e.g. socket.send() but not to file.write(). For simplicity - and consistency, in PyPy memoryview DOES support buffer(), which - means that it is accepted in more places than CPython. """ - self._check_released(space) + Note that memoryview() is very inconsistent in CPython: it does not + support the buffer interface but does support the new buffer + interface: as a result, it is possible to pass memoryview to + e.g. socket.send() but not to file.write(). For simplicity and + consistency, in PyPy memoryview DOES support buffer(), which means + that it is accepted in more places than CPython. + """ return self.buf @staticmethod From noreply at buildbot.pypy.org Fri Mar 21 00:02:33 2014 From: noreply at buildbot.pypy.org (pjenvey) Date: Fri, 21 Mar 2014 00:02:33 +0100 (CET) Subject: [pypy-commit] pypy py3k: merge default Message-ID: <20140320230233.92BA41D2866@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r70135:706e3a4c1ffa Date: 2014-03-20 15:41 -0700 http://bitbucket.org/pypy/pypy/changeset/706e3a4c1ffa/ Log: merge default diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -292,6 +292,10 @@ depending on the compiler settings, the default of 768KB is enough for about 1400 calls. +* since the implementation of dictionary is different, the exact number + which ``__hash__`` and ``__eq__`` are called is different. Since CPython + does not give any specific guarantees either, don't rely on it. + * assignment to ``__class__`` is limited to the cases where it works on CPython 2.5. On CPython 2.6 and 2.7 it works in a bit more cases, which are not supported by PyPy so far. (If needed, diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -114,3 +114,6 @@ app-level. The `Buffer` class is now used by `W_MemoryView` and `W_Buffer`, which is not present in Python 3. Previously `W_Buffer` was an alias to `Buffer`, which was wrappable itself. + +.. branch: improve-consecutive-dict-lookups +Improve the situation when dict lookups of the same key are performed in a chain diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -441,11 +441,10 @@ return name - def getbuiltinmodule(self, name, force_init=False, reuse=True): + def getbuiltinmodule(self, name, force_init=False): w_name = self.wrap(name) w_modules = self.sys.get('modules') if not force_init: - assert reuse is True try: return self.getitem(w_modules, w_name) except OperationError, e: @@ -464,15 +463,7 @@ # Initialize the module from pypy.interpreter.module import Module if isinstance(w_mod, Module): - if not reuse and w_mod.startup_called: - # Create a copy of the module - w_mod.getdict(self) # unlazy w_initialdict - w_new = self.wrap(Module(self, w_name)) - self.call_method(w_new.getdict(self), 'update', - w_mod.w_initialdict) - w_mod = w_new - else: - w_mod.init(self) + w_mod.init(self) # Add the module to sys.modules self.setitem(w_modules, w_name, w_mod) diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py --- a/pypy/module/imp/importing.py +++ b/pypy/module/imp/importing.py @@ -585,8 +585,7 @@ return space.call_method(find_info.w_loader, "load_module", w_modulename) if find_info.modtype == C_BUILTIN: - return space.getbuiltinmodule(find_info.filename, force_init=True, - reuse=reuse) + return space.getbuiltinmodule(find_info.filename, force_init=True) if find_info.modtype in (PY_SOURCE, PY_COMPILED, C_EXTENSION, PKG_DIRECTORY): w_mod = None diff --git a/pypy/module/imp/test/test_app.py b/pypy/module/imp/test/test_app.py --- a/pypy/module/imp/test/test_app.py +++ b/pypy/module/imp/test/test_app.py @@ -219,6 +219,7 @@ def test_builtin_reimport(self): # from https://bugs.pypy.org/issue1514 + skip("fix me") import sys, marshal old = marshal.loads @@ -238,6 +239,7 @@ # taken from https://bugs.pypy.org/issue1514, with extra cases # that show a difference with CPython: we can get on CPython # several module objects for the same built-in module :-( + skip("several built-in module objects: not supported by pypy") import sys, marshal old = marshal.loads diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py --- a/pypy/module/imp/test/test_import.py +++ b/pypy/module/imp/test/test_import.py @@ -652,6 +652,7 @@ assert hasattr(time, 'clock') def test_reimport_builtin_simple_case_2(self): + skip("fix me") import sys, time time.foo = "bar" del sys.modules['time'] @@ -660,6 +661,7 @@ def test_reimport_builtin(self): import imp, sys, time + skip("fix me") oldpath = sys.path time.tzset = "" diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py --- a/rpython/jit/backend/arm/opassembler.py +++ b/rpython/jit/backend/arm/opassembler.py @@ -583,6 +583,10 @@ emit_op_getfield_raw_pure = emit_op_getfield_gc emit_op_getfield_gc_pure = emit_op_getfield_gc + def emit_op_increment_debug_counter(self, op, arglocs, regalloc, fcond): + # XXX implement me + return fcond + def emit_op_getinteriorfield_gc(self, op, arglocs, regalloc, fcond): (base_loc, index_loc, res_loc, ofs_loc, ofs, itemsize, fieldsize) = arglocs diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py --- a/rpython/jit/backend/arm/regalloc.py +++ b/rpython/jit/backend/arm/regalloc.py @@ -849,6 +849,10 @@ prepare_op_getfield_raw_pure = prepare_op_getfield_gc prepare_op_getfield_gc_pure = prepare_op_getfield_gc + def prepare_op_increment_debug_counter(self, op, fcond): + # XXX implement me + return [] + def prepare_op_getinteriorfield_gc(self, op, fcond): t = unpack_interiorfielddescr(op.getdescr()) ofs, itemsize, fieldsize, sign = t diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py --- a/rpython/jit/backend/llgraph/runner.py +++ b/rpython/jit/backend/llgraph/runner.py @@ -553,6 +553,10 @@ else: return self.bh_raw_load_i(struct, offset, descr) + def bh_increment_debug_counter(self, addr): + p = rffi.cast(rffi.CArrayPtr(lltype.Signed), addr) + p[0] += 1 + def unpack_arraydescr_size(self, arraydescr): from rpython.jit.backend.llsupport.symbolic import get_array_token from rpython.jit.backend.llsupport.descr import get_type_flag, FLAG_SIGNED diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py --- a/rpython/jit/backend/llsupport/assembler.py +++ b/rpython/jit/backend/llsupport/assembler.py @@ -15,7 +15,7 @@ DEBUG_COUNTER = lltype.Struct('DEBUG_COUNTER', # 'b'ridge, 'l'abel or # 'e'ntry point - ('i', lltype.Signed), + ('i', lltype.Signed), # first field, at offset 0 ('type', lltype.Char), ('number', lltype.Signed) ) @@ -64,7 +64,6 @@ self.cpu = cpu self.memcpy_addr = 0 self.rtyper = cpu.rtyper - self.debug_counter_descr = cpu.fielddescrof(DEBUG_COUNTER, 'i') self._debug = False def setup_once(self): @@ -265,14 +264,8 @@ def _append_debugging_code(self, operations, tp, number, token): counter = self._register_counter(tp, number, token) c_adr = ConstInt(rffi.cast(lltype.Signed, counter)) - box = BoxInt() - box2 = BoxInt() - ops = [ResOperation(rop.GETFIELD_RAW, [c_adr], - box, descr=self.debug_counter_descr), - ResOperation(rop.INT_ADD, [box, ConstInt(1)], box2), - ResOperation(rop.SETFIELD_RAW, [c_adr, box2], - None, descr=self.debug_counter_descr)] - operations.extend(ops) + operations.append( + ResOperation(rop.INCREMENT_DEBUG_COUNTER, [c_adr], None)) def _register_counter(self, tp, number, token): # YYY very minor leak -- we need the counters to stay alive diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py --- a/rpython/jit/backend/test/runner_test.py +++ b/rpython/jit/backend/test/runner_test.py @@ -3736,7 +3736,7 @@ assert False, 'should not be called' from rpython.jit.codewriter.effectinfo import EffectInfo - effectinfo = EffectInfo([], [], [], [], EffectInfo.EF_CANNOT_RAISE, EffectInfo.OS_MATH_SQRT) + effectinfo = EffectInfo([], [], [], [], [], [], EffectInfo.EF_CANNOT_RAISE, EffectInfo.OS_MATH_SQRT) FPTR = self.Ptr(self.FuncType([lltype.Float], lltype.Float)) func_ptr = llhelper(FPTR, math_sqrt) FUNC = deref(FPTR) @@ -4338,3 +4338,12 @@ assert rffi.cast(lltype.Signed, a[0]) == -7654 assert rffi.cast(lltype.Signed, a[1]) == 777 lltype.free(a, flavor='raw') + + def test_increment_debug_counter(self): + foo = lltype.malloc(rffi.CArray(lltype.Signed), 1, flavor='raw') + foo[0] = 1789200 + self.execute_operation(rop.INCREMENT_DEBUG_COUNTER, + [ConstInt(rffi.cast(lltype.Signed, foo))], + 'void') + assert foo[0] == 1789201 + lltype.free(foo, flavor='raw') diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -287,7 +287,6 @@ cast_instance_to_gcref(self.cpu.propagate_exception_descr)) ofs = self.cpu.get_ofs_of_frame_field('jf_descr') self.mc.MOV(RawEbpLoc(ofs), imm(propagate_exception_descr)) - self.mc.MOV_rr(eax.value, ebp.value) # self._call_footer() rawstart = self.mc.materialize(self.cpu.asmmemmgr, []) @@ -435,8 +434,8 @@ self.wb_slowpath[withcards + 2 * withfloats] = rawstart @rgc.no_release_gil - def assemble_loop(self, logger, loopname, inputargs, operations, looptoken, - log): + def assemble_loop(self, inputargs, operations, looptoken, log, + loopname, logger): '''adds the following attributes to looptoken: _ll_function_addr (address of the generated func, as an int) _ll_loop_code (debug: addr of the start of the ResOps) @@ -515,8 +514,8 @@ size_excluding_failure_stuff - looppos) @rgc.no_release_gil - def assemble_bridge(self, logger, faildescr, inputargs, operations, - original_loop_token, log): + def assemble_bridge(self, faildescr, inputargs, operations, + original_loop_token, log, logger): if not we_are_translated(): # Arguments should be unique assert len(set(inputargs)) == len(inputargs) @@ -761,6 +760,9 @@ # def _call_footer(self): + # the return value is the jitframe + self.mc.MOV_rr(eax.value, ebp.value) + gcrootmap = self.cpu.gc_ll_descr.gcrootmap if gcrootmap and gcrootmap.is_shadow_stack: self._call_footer_shadowstack(gcrootmap) @@ -1467,6 +1469,14 @@ ofs_loc) self.load_from_mem(resloc, src_addr, fieldsize_loc, sign_loc) + def genop_discard_increment_debug_counter(self, op, arglocs): + # The argument should be an immediate address. This should + # generate code equivalent to a GETFIELD_RAW, an ADD(1), and a + # SETFIELD_RAW. Here we use the direct from-memory-to-memory + # increment operation of x86. + base_loc, = arglocs + self.mc.INC(mem(base_loc, 0)) + def genop_discard_setfield_gc(self, op, arglocs): base_loc, ofs_loc, size_loc, value_loc = arglocs assert isinstance(size_loc, ImmedLoc) @@ -1823,12 +1833,8 @@ # did just above. ofs = self.cpu.get_ofs_of_frame_field('jf_descr') ofs2 = self.cpu.get_ofs_of_frame_field('jf_gcmap') - mc.POP(eax) - mc.MOV_br(ofs2, eax.value) - mc.POP(eax) - mc.MOV_br(ofs, eax.value) - # the return value is the jitframe - mc.MOV_rr(eax.value, ebp.value) + mc.POP_b(ofs2) + mc.POP_b(ofs) self._call_footer() rawstart = mc.materialize(self.cpu.asmmemmgr, []) @@ -1861,7 +1867,6 @@ # keep that one and kill all the others ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') self.mc.MOV_bi(ofs, 0) - self.mc.MOV_rr(eax.value, ebp.value) # exit function self._call_footer() diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -1003,6 +1003,10 @@ consider_getfield_raw_pure = consider_getfield_gc consider_getfield_gc_pure = consider_getfield_gc + def consider_increment_debug_counter(self, op): + base_loc = self.loc(op.getarg(0)) + self.perform_discard(op, [base_loc]) + def consider_getarrayitem_gc(self, op): itemsize, ofs, sign = unpack_arraydescr(op.getdescr()) args = op.getarglist() diff --git a/rpython/jit/backend/x86/regloc.py b/rpython/jit/backend/x86/regloc.py --- a/rpython/jit/backend/x86/regloc.py +++ b/rpython/jit/backend/x86/regloc.py @@ -488,12 +488,22 @@ for possible_code in unrolling_location_codes: if code == possible_code: val = getattr(loc, "value_" + possible_code)() - if self.WORD == 8 and possible_code == 'i' and not rx86.fits_in_32bits(val): - self._load_scratch(val) + # Faking out of certain operations for x86_64 + fits32 = rx86.fits_in_32bits + if possible_code == 'i' and not fits32(val): + self._load_scratch(val) # for 'PUSH(imm)' _rx86_getattr(self, name + "_r")(X86_64_SCRATCH_REG.value) - else: - methname = name + "_" + possible_code - _rx86_getattr(self, methname)(val) + return + if possible_code == 'j' and not fits32(val): + val = self._addr_as_reg_offset(val) + _rx86_getattr(self, name + "_m")(val) + return + if possible_code == 'm' and not fits32(val[1]): + val = self._fix_static_offset_64_m(val) + if possible_code == 'a' and not fits32(val[3]): + val = self._fix_static_offset_64_a(val) + methname = name + "_" + possible_code + _rx86_getattr(self, methname)(val) return func_with_new_name(INSN, "INSN_" + name) @@ -600,6 +610,7 @@ TEST8 = _binaryop('TEST8') BTS = _binaryop('BTS') + INC = _unaryop('INC') ADD = _binaryop('ADD') SUB = _binaryop('SUB') IMUL = _binaryop('IMUL') diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py --- a/rpython/jit/backend/x86/runner.py +++ b/rpython/jit/backend/x86/runner.py @@ -93,16 +93,15 @@ def compile_loop(self, inputargs, operations, looptoken, log=True, name='', logger=None): - return self.assembler.assemble_loop(logger, name, inputargs, operations, - looptoken, log=log) + return self.assembler.assemble_loop(inputargs, operations, looptoken, log, + name, logger) def compile_bridge(self, faildescr, inputargs, operations, original_loop_token, log=True, logger=None): clt = original_loop_token.compiled_loop_token clt.compiling_a_bridge() - return self.assembler.assemble_bridge(logger, faildescr, inputargs, - operations, - original_loop_token, log=log) + return self.assembler.assemble_bridge(faildescr, inputargs, operations, + original_loop_token, log, logger) def clear_latest_values(self, count): setitem = self.assembler.fail_boxes_ptr.setitem diff --git a/rpython/jit/backend/x86/rx86.py b/rpython/jit/backend/x86/rx86.py --- a/rpython/jit/backend/x86/rx86.py +++ b/rpython/jit/backend/x86/rx86.py @@ -470,6 +470,9 @@ # ------------------------------ Arithmetic ------------------------------ + INC_m = insn(rex_w, '\xFF', orbyte(0), mem_reg_plus_const(1)) + INC_j = insn(rex_w, '\xFF', orbyte(0), abs_(1)) + ADD_ri,ADD_rr,ADD_rb,_,_,ADD_rm,ADD_rj,_,_ = common_modes(0) OR_ri, OR_rr, OR_rb, _,_,OR_rm, OR_rj, _,_ = common_modes(1) AND_ri,AND_rr,AND_rb,_,_,AND_rm,AND_rj,_,_ = common_modes(4) diff --git a/rpython/jit/backend/x86/test/test_assembler.py b/rpython/jit/backend/x86/test/test_assembler.py --- a/rpython/jit/backend/x86/test/test_assembler.py +++ b/rpython/jit/backend/x86/test/test_assembler.py @@ -55,9 +55,7 @@ asm = cpu.assembler asm.setup_once() asm.setup(looptoken) - self.fm = X86FrameManager(0) - self.xrm = X86XMMRegisterManager(None, frame_manager=self.fm, - assembler=asm) + self.xrm = X86XMMRegisterManager(None, assembler=asm) callback(asm) asm.mc.RET() rawstart = asm.materialize_loop(looptoken) @@ -75,29 +73,6 @@ res = self.do_test(callback) assert res == 42 - def test_push_stack(self): - def callback(asm): - loc = self.fm.frame_pos(5, INT) - asm.mc.SUB_ri(esp.value, 64) - asm.mov(imm(42), loc) - asm.regalloc_push(loc) - asm.regalloc_pop(eax) - asm.mc.ADD_ri(esp.value, 64) - res = self.do_test(callback) - assert res == 42 - - def test_pop_stack(self): - def callback(asm): - loc = self.fm.frame_pos(5, INT) - asm.mc.SUB_ri(esp.value, 64) - asm.mov(imm(42), edx) - asm.regalloc_push(edx) - asm.regalloc_pop(loc) - asm.mov(loc, eax) - asm.mc.ADD_ri(esp.value, 64) - res = self.do_test(callback) - assert res == 42 - def test_simple_xmm(self): def callback(asm): c = ConstFloat(longlong.getfloatstorage(-42.5)) @@ -109,32 +84,8 @@ res = self.do_test(callback) assert res == -42 - def test_push_stack_xmm(self): + def test_xmm_pushes_8_bytes(self): def callback(asm): - c = ConstFloat(longlong.getfloatstorage(-42.5)) - loc = self.xrm.convert_to_imm(c) - loc2 = self.fm.frame_pos(4, FLOAT) - asm.mc.SUB_ri(esp.value, 64) - asm.mov(loc, xmm5) - asm.mov(xmm5, loc2) - asm.regalloc_push(loc2) - asm.regalloc_pop(xmm0) - asm.mc.ADD_ri(esp.value, 64) - asm.mc.CVTTSD2SI(eax, xmm0) - res = self.do_test(callback) - assert res == -42 - - def test_pop_stack_xmm(self): - def callback(asm): - c = ConstFloat(longlong.getfloatstorage(-42.5)) - loc = self.xrm.convert_to_imm(c) - loc2 = self.fm.frame_pos(4, FLOAT) - asm.mc.SUB_ri(esp.value, 64) - asm.mov(loc, xmm5) asm.regalloc_push(xmm5) - asm.regalloc_pop(loc2) - asm.mov(loc2, xmm0) - asm.mc.ADD_ri(esp.value, 64) - asm.mc.CVTTSD2SI(eax, xmm0) - res = self.do_test(callback) - assert res == -42 + asm.mc.ADD(esp, imm(8)) + self.do_test(callback) diff --git a/rpython/jit/backend/x86/test/test_regloc.py b/rpython/jit/backend/x86/test/test_regloc.py --- a/rpython/jit/backend/x86/test/test_regloc.py +++ b/rpython/jit/backend/x86/test/test_regloc.py @@ -373,3 +373,56 @@ '\x59' ) assert cb.getvalue() == expected_instructions + + # ------------------------------------------------------------ + + def test_push_immed64(self): + immed = 0x0123456789ABCDEF + cb = LocationCodeBuilder64() + cb.PUSH(imm(immed)) + # + expected_instructions = ( + # mov r11, 0x0123456789ABCDEF + '\x49\xBB\xEF\xCD\xAB\x89\x67\x45\x23\x01' + # push r11 + '\x41\x53' + ) + assert cb.getvalue() == expected_instructions + + def test_inc_64bit_address_1(self): + base_addr = 0x0123456789ABCDEF + cb = LocationCodeBuilder64() + cb.INC(AddressLoc(ImmedLoc(0), ImmedLoc(0), 0, base_addr)) + # this case is a INC_j + # + expected_instructions = ( + # mov r11, 0x0123456789ABCDEF + '\x49\xBB\xEF\xCD\xAB\x89\x67\x45\x23\x01' + # inc [r11] + '\x49\xFF\x03' + ) + assert cb.getvalue() == expected_instructions + + def test_inc_64bit_address_2(self): + py.test.skip("there is no unary instruction INSN_a so far") + base_addr = 0x0123456789ABCDEF + cb = LocationCodeBuilder64() + cb.INC(AddressLoc(ImmedLoc(0), edx, 3, base_addr)) + # this case would be a INC_a + xxx + + def test_inc_64bit_address_3(self): + base_addr = 0x0123456789ABCDEF + cb = LocationCodeBuilder64() + cb.INC(AddressLoc(eax, ImmedLoc(0), 0, base_addr)) + # this case is a INC_m + # + expected_instructions = ( + # mov r11, 0x0123456789ABCDEF + '\x49\xBB\xEF\xCD\xAB\x89\x67\x45\x23\x01' + # lea r11, [rax+r11] + '\x4E\x8D\x1C\x18' + # inc [r11] + '\x49\xFF\x03' + ) + assert cb.getvalue() == expected_instructions diff --git a/rpython/jit/backend/x86/test/test_runner.py b/rpython/jit/backend/x86/test/test_runner.py --- a/rpython/jit/backend/x86/test/test_runner.py +++ b/rpython/jit/backend/x86/test/test_runner.py @@ -427,8 +427,8 @@ debug._log = None # assert ops_offset is looptoken._x86_ops_offset - # 2*(getfield_raw/int_add/setfield_raw) + ops + None - assert len(ops_offset) == 2*3 + len(operations) + 1 + # 2*increment_debug_counter + ops + None + assert len(ops_offset) == 2 + len(operations) + 1 assert (ops_offset[operations[0]] <= ops_offset[operations[1]] <= ops_offset[operations[2]] <= diff --git a/rpython/jit/codewriter/call.py b/rpython/jit/codewriter/call.py --- a/rpython/jit/codewriter/call.py +++ b/rpython/jit/codewriter/call.py @@ -178,7 +178,7 @@ return (fnaddr, calldescr) def getcalldescr(self, op, oopspecindex=EffectInfo.OS_NONE, - extraeffect=None): + extraeffect=None, extradescr=None): """Return the calldescr that describes all calls done by 'op'. This returns a calldescr that we can put in the corresponding call operation in the calling jitcode. It gets an effectinfo @@ -259,6 +259,7 @@ effectinfo = effectinfo_from_writeanalyze( self.readwrite_analyzer.analyze(op, self.seen), self.cpu, extraeffect, oopspecindex, can_invalidate, call_release_gil_target, + extradescr, ) # assert effectinfo is not None diff --git a/rpython/jit/codewriter/effectinfo.py b/rpython/jit/codewriter/effectinfo.py --- a/rpython/jit/codewriter/effectinfo.py +++ b/rpython/jit/codewriter/effectinfo.py @@ -21,6 +21,7 @@ OS_ARRAYCOPY = 1 # "list.ll_arraycopy" OS_STR2UNICODE = 2 # "str.str2unicode" OS_SHRINK_ARRAY = 3 # rgc.ll_shrink_array + OS_DICT_LOOKUP = 4 # ll_dict_lookup # OS_STR_CONCAT = 22 # "stroruni.concat" OS_STR_SLICE = 23 # "stroruni.slice" @@ -88,15 +89,18 @@ # for debugging: _OS_CANRAISE = set([ OS_NONE, OS_STR2UNICODE, OS_LIBFFI_CALL, OS_RAW_MALLOC_VARSIZE_CHAR, - OS_JIT_FORCE_VIRTUAL, OS_SHRINK_ARRAY, + OS_JIT_FORCE_VIRTUAL, OS_SHRINK_ARRAY, OS_DICT_LOOKUP, ]) def __new__(cls, readonly_descrs_fields, readonly_descrs_arrays, + readonly_descrs_interiorfields, write_descrs_fields, write_descrs_arrays, + write_descrs_interiorfields, extraeffect=EF_CAN_RAISE, oopspecindex=OS_NONE, can_invalidate=False, - call_release_gil_target=llmemory.NULL): + call_release_gil_target=llmemory.NULL, + extradescrs=None): key = (frozenset_or_none(readonly_descrs_fields), frozenset_or_none(readonly_descrs_arrays), frozenset_or_none(write_descrs_fields), @@ -121,17 +125,21 @@ result = object.__new__(cls) result.readonly_descrs_fields = readonly_descrs_fields result.readonly_descrs_arrays = readonly_descrs_arrays + result.readonly_descrs_interiorfields = readonly_descrs_interiorfields if extraeffect == EffectInfo.EF_LOOPINVARIANT or \ extraeffect == EffectInfo.EF_ELIDABLE_CANNOT_RAISE or \ extraeffect == EffectInfo.EF_ELIDABLE_CAN_RAISE: result.write_descrs_fields = [] result.write_descrs_arrays = [] + result.write_descrs_interiorfields = [] else: result.write_descrs_fields = write_descrs_fields result.write_descrs_arrays = write_descrs_arrays + result.write_descrs_interiorfields = write_descrs_interiorfields result.extraeffect = extraeffect result.can_invalidate = can_invalidate result.oopspecindex = oopspecindex + result.extradescrs = extradescrs result.call_release_gil_target = call_release_gil_target if result.check_can_raise(): assert oopspecindex in cls._OS_CANRAISE @@ -163,7 +171,7 @@ return None return frozenset(x) -EffectInfo.MOST_GENERAL = EffectInfo(None, None, None, None, +EffectInfo.MOST_GENERAL = EffectInfo(None, None, None, None, None, None, EffectInfo.EF_RANDOM_EFFECTS, can_invalidate=True) @@ -172,19 +180,24 @@ extraeffect=EffectInfo.EF_CAN_RAISE, oopspecindex=EffectInfo.OS_NONE, can_invalidate=False, - call_release_gil_target=llmemory.NULL): + call_release_gil_target=llmemory.NULL, + extradescr=None): from rpython.translator.backendopt.writeanalyze import top_set if effects is top_set or extraeffect == EffectInfo.EF_RANDOM_EFFECTS: readonly_descrs_fields = None readonly_descrs_arrays = None + readonly_descrs_interiorfields = None write_descrs_fields = None write_descrs_arrays = None + write_descrs_interiorfields = None extraeffect = EffectInfo.EF_RANDOM_EFFECTS else: readonly_descrs_fields = [] readonly_descrs_arrays = [] + readonly_descrs_interiorfields = [] write_descrs_fields = [] write_descrs_arrays = [] + write_descrs_interiorfields = [] def add_struct(descrs_fields, (_, T, fieldname)): T = deref(T) @@ -198,6 +211,17 @@ descr = cpu.arraydescrof(ARRAY) descrs_arrays.append(descr) + def add_interiorfield(descrs_interiorfields, (_, T, fieldname)): + T = deref(T) + if not isinstance(T, lltype.Array): + return # let's not consider structs for now + if not consider_array(T): + return + if getattr(T.OF, fieldname) is lltype.Void: + return + descr = cpu.interiorfielddescrof(T, fieldname) + descrs_interiorfields.append(descr) + for tup in effects: if tup[0] == "struct": add_struct(write_descrs_fields, tup) @@ -205,6 +229,12 @@ tupw = ("struct",) + tup[1:] if tupw not in effects: add_struct(readonly_descrs_fields, tup) + elif tup[0] == "interiorfield": + add_interiorfield(write_descrs_interiorfields, tup) + elif tup[0] == "readinteriorfield": + tupw = ('interiorfield',) + tup[1:] + if tupw not in effects: + add_interiorfield(readonly_descrs_interiorfields, tup) elif tup[0] == "array": add_array(write_descrs_arrays, tup) elif tup[0] == "readarray": @@ -216,12 +246,15 @@ # return EffectInfo(readonly_descrs_fields, readonly_descrs_arrays, + readonly_descrs_interiorfields, write_descrs_fields, write_descrs_arrays, + write_descrs_interiorfields, extraeffect, oopspecindex, can_invalidate, - call_release_gil_target) + call_release_gil_target, + extradescr) def consider_struct(TYPE, fieldname): if fieldType(TYPE, fieldname) is lltype.Void: diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py --- a/rpython/jit/codewriter/jtransform.py +++ b/rpython/jit/codewriter/jtransform.py @@ -403,6 +403,9 @@ prepare = self._handle_math_sqrt_call elif oopspec_name.startswith('rgc.'): prepare = self._handle_rgc_call + elif oopspec_name.endswith('dict.lookup'): + # also ordereddict.lookup + prepare = self._handle_dict_lookup_call else: prepare = self.prepare_builtin_call try: @@ -1680,9 +1683,11 @@ # ---------- # Strings and Unicodes. - def _handle_oopspec_call(self, op, args, oopspecindex, extraeffect=None): + def _handle_oopspec_call(self, op, args, oopspecindex, extraeffect=None, + extradescr=None): calldescr = self.callcontrol.getcalldescr(op, oopspecindex, - extraeffect) + extraeffect, + extradescr=extradescr) if extraeffect is not None: assert (is_test_calldescr(calldescr) # for tests or calldescr.get_extra_info().extraeffect == extraeffect) @@ -1846,6 +1851,14 @@ return self._handle_oopspec_call(op, args, EffectInfo.OS_MATH_SQRT, EffectInfo.EF_ELIDABLE_CANNOT_RAISE) + def _handle_dict_lookup_call(self, op, oopspec_name, args): + extradescr1 = self.cpu.fielddescrof(op.args[1].concretetype.TO, + 'entries') + extradescr2 = self.cpu.interiorfielddescrof( + op.args[1].concretetype.TO.entries.TO, 'key') + return self._handle_oopspec_call(op, args, EffectInfo.OS_DICT_LOOKUP, + extradescr=[extradescr1, extradescr2]) + def _handle_rgc_call(self, op, oopspec_name, args): if oopspec_name == 'rgc.ll_shrink_array': return self._handle_oopspec_call(op, args, EffectInfo.OS_SHRINK_ARRAY, EffectInfo.EF_CAN_RAISE) diff --git a/rpython/jit/codewriter/test/test_jtransform.py b/rpython/jit/codewriter/test/test_jtransform.py --- a/rpython/jit/codewriter/test/test_jtransform.py +++ b/rpython/jit/codewriter/test/test_jtransform.py @@ -60,7 +60,8 @@ class FakeResidualCallControl: def guess_call_kind(self, op): return 'residual' - def getcalldescr(self, op, oopspecindex=None, extraeffect=None): + def getcalldescr(self, op, oopspecindex=None, extraeffect=None, + extradescr=None): return 'calldescr' def calldescr_canraise(self, calldescr): return True @@ -117,7 +118,8 @@ self.callinfocollection = FakeCallInfoCollection() def guess_call_kind(self, op): return 'builtin' - def getcalldescr(self, op, oopspecindex=None, extraeffect=None): + def getcalldescr(self, op, oopspecindex=None, extraeffect=None, + extradescr=None): assert oopspecindex is not None # in this test EI = effectinfo.EffectInfo if oopspecindex != EI.OS_ARRAYCOPY: diff --git a/rpython/jit/codewriter/test/test_list.py b/rpython/jit/codewriter/test/test_list.py --- a/rpython/jit/codewriter/test/test_list.py +++ b/rpython/jit/codewriter/test/test_list.py @@ -37,7 +37,8 @@ class FakeCallControl: class getcalldescr(AbstractDescr): - def __init__(self, op, oopspecindex=0, extraeffect=None): + def __init__(self, op, oopspecindex=0, extraeffect=None, + extradescr=None): self.op = op self.oopspecindex = oopspecindex def __repr__(self): diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py --- a/rpython/jit/metainterp/executor.py +++ b/rpython/jit/metainterp/executor.py @@ -332,6 +332,7 @@ continue if value in (rop.FORCE_TOKEN, rop.CALL_ASSEMBLER, + rop.INCREMENT_DEBUG_COUNTER, rop.COND_CALL_GC_WB, rop.COND_CALL_GC_WB_ARRAY, rop.DEBUG_MERGE_POINT, diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py --- a/rpython/jit/metainterp/optimizeopt/heap.py +++ b/rpython/jit/metainterp/optimizeopt/heap.py @@ -1,8 +1,10 @@ import os +from rpython.jit.codewriter.effectinfo import EffectInfo +from rpython.jit.metainterp.optimizeopt.util import args_dict from rpython.jit.metainterp.history import Const from rpython.jit.metainterp.jitexc import JitException -from rpython.jit.metainterp.optimizeopt.optimizer import Optimization, MODE_ARRAY, LEVEL_KNOWNCLASS +from rpython.jit.metainterp.optimizeopt.optimizer import Optimization, MODE_ARRAY, LEVEL_KNOWNCLASS, REMOVED from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method from rpython.jit.metainterp.resoperation import rop, ResOperation from rpython.rlib.objectmodel import we_are_translated @@ -173,6 +175,10 @@ self.cached_fields = {} # cached array items: {array descr: {index: CachedField}} self.cached_arrayitems = {} + # cached dict items: {dict descr: {(optval, index): box-or-const}} + self.cached_dict_reads = {} + # cache of corresponding array descrs + self.corresponding_array_descrs = {} # self._lazy_setfields_and_arrayitems = [] self._remove_guard_not_invalidated = False @@ -180,9 +186,13 @@ self.postponed_op = None def force_at_end_of_preamble(self): + self.cached_dict_reads.clear() + self.corresponding_array_descrs.clear() self.force_all_lazy_setfields_and_arrayitems() def flush(self): + self.cached_dict_reads.clear() + self.corresponding_array_descrs.clear() self.force_all_lazy_setfields_and_arrayitems() self.emit_postponed_op() @@ -214,6 +224,7 @@ del self._lazy_setfields_and_arrayitems[:] self.cached_fields.clear() self.cached_arrayitems.clear() + self.cached_dict_reads.clear() def field_cache(self, descr): try: @@ -282,6 +293,44 @@ self.force_all_lazy_setfields_and_arrayitems() self.clean_caches() + def optimize_CALL(self, op): + # dispatch based on 'oopspecindex' to a method that handles + # specifically the given oopspec call. For non-oopspec calls, + # oopspecindex is just zero. + effectinfo = op.getdescr().get_extra_info() + oopspecindex = effectinfo.oopspecindex + if oopspecindex == EffectInfo.OS_DICT_LOOKUP: + if self._optimize_CALL_DICT_LOOKUP(op): + return + self.emit_operation(op) + + def _optimize_CALL_DICT_LOOKUP(self, op): + descrs = op.getdescr().get_extra_info().extradescrs + assert descrs # translation hint + descr1 = descrs[0] + try: + d = self.cached_dict_reads[descr1] + except KeyError: + d = self.cached_dict_reads[descr1] = args_dict() + self.corresponding_array_descrs[descrs[1]] = descr1 + args = self.optimizer.make_args_key(op) + try: + res_v = d[args] + except KeyError: + d[args] = self.getvalue(op.result) + return False + else: + self.make_equal_to(op.result, res_v) + self.last_emitted_operation = REMOVED + return True + + def optimize_GUARD_NO_EXCEPTION(self, op): + if self.last_emitted_operation is REMOVED: + return + self.emit_operation(op) + + optimize_GUARD_EXCEPTION = optimize_GUARD_NO_EXCEPTION + def force_from_effectinfo(self, effectinfo): # XXX we can get the wrong complexity here, if the lists # XXX stored on effectinfo are large @@ -290,9 +339,20 @@ for arraydescr in effectinfo.readonly_descrs_arrays: self.force_lazy_setarrayitem(arraydescr) for fielddescr in effectinfo.write_descrs_fields: + try: + del self.cached_dict_reads[fielddescr] + except KeyError: + pass self.force_lazy_setfield(fielddescr, can_cache=False) for arraydescr in effectinfo.write_descrs_arrays: self.force_lazy_setarrayitem(arraydescr, can_cache=False) + for descr in effectinfo.write_descrs_interiorfields: + if descr in self.corresponding_array_descrs: + dictdescr = self.corresponding_array_descrs.pop(descr) + try: + del self.cached_dict_reads[dictdescr] + except KeyError: + pass # someone did it already if effectinfo.check_forces_virtual_or_virtualizable(): vrefinfo = self.optimizer.metainterp_sd.virtualref_info self.force_lazy_setfield(vrefinfo.descr_forced) diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py @@ -5444,6 +5444,21 @@ """ self.optimize_loop(ops, expected) + def test_consecutive_getinteriorfields(self): + py.test.skip("we want this to pass") + ops = """ + [p0, i0] + i1 = getinteriorfield_gc(p0, i0, descr=valuedescr) + i2 = getinteriorfield_gc(p0, i0, descr=valuedescr) + jump(i1, i2) + """ + expected = """ + [p0, i0] + i1 = getinteriorfield_gc(p0, i0, descr=valuedescr) + jump(i1, i1) + """ + self.optimize_loop(ops, expected) + class TestLLtype(BaseTestOptimizeBasic, LLtypeMixin): pass diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py --- a/rpython/jit/metainterp/optimizeopt/test/test_util.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py @@ -181,28 +181,29 @@ plaincalldescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, EffectInfo.MOST_GENERAL) nonwritedescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, - EffectInfo([], [], [], [])) + EffectInfo([], [], [], [], [], [])) writeadescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, - EffectInfo([], [], [adescr], [])) + EffectInfo([], [], [], [adescr], [], [])) writearraydescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, - EffectInfo([], [], [adescr], [arraydescr])) + EffectInfo([], [], [], [adescr], [arraydescr], + [])) readadescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, - EffectInfo([adescr], [], [], [])) + EffectInfo([adescr], [], [], [], [], [])) mayforcevirtdescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, - EffectInfo([nextdescr], [], [], [], + EffectInfo([nextdescr], [], [], [], [], [], EffectInfo.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE, can_invalidate=True)) arraycopydescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, - EffectInfo([], [arraydescr], [], [arraydescr], + EffectInfo([], [arraydescr], [], [], [arraydescr], [], EffectInfo.EF_CANNOT_RAISE, oopspecindex=EffectInfo.OS_ARRAYCOPY)) raw_malloc_descr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, - EffectInfo([], [], [], [], + EffectInfo([], [], [], [], [], [], EffectInfo.EF_CAN_RAISE, oopspecindex=EffectInfo.OS_RAW_MALLOC_VARSIZE_CHAR)) raw_free_descr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, - EffectInfo([], [], [], [], + EffectInfo([], [], [], [], [], [], EffectInfo.EF_CANNOT_RAISE, oopspecindex=EffectInfo.OS_RAW_FREE)) @@ -251,17 +252,18 @@ _oopspecindex = getattr(EffectInfo, _os) locals()[_name] = \ cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, - EffectInfo([], [], [], [], EffectInfo.EF_CANNOT_RAISE, + EffectInfo([], [], [], [], [], [], EffectInfo.EF_CANNOT_RAISE, oopspecindex=_oopspecindex)) # _oopspecindex = getattr(EffectInfo, _os.replace('STR', 'UNI')) locals()[_name.replace('str', 'unicode')] = \ cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, - EffectInfo([], [], [], [], EffectInfo.EF_CANNOT_RAISE, + EffectInfo([], [], [], [], [], [], EffectInfo.EF_CANNOT_RAISE, oopspecindex=_oopspecindex)) s2u_descr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, - EffectInfo([], [], [], [], oopspecindex=EffectInfo.OS_STR2UNICODE)) + EffectInfo([], [], [], [], [], [], + oopspecindex=EffectInfo.OS_STR2UNICODE)) # class LoopToken(AbstractDescr): @@ -277,7 +279,7 @@ virtualtokendescr = vrefinfo.descr_virtual_token virtualforceddescr = vrefinfo.descr_forced FUNC = lltype.FuncType([], lltype.Void) - ei = EffectInfo([], [], [], [], EffectInfo.EF_CANNOT_RAISE, + ei = EffectInfo([], [], [], [], [], [], EffectInfo.EF_CANNOT_RAISE, can_invalidate=False, oopspecindex=EffectInfo.OS_JIT_FORCE_VIRTUALIZABLE) clear_vable = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, ei) diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py --- a/rpython/jit/metainterp/resoperation.py +++ b/rpython/jit/metainterp/resoperation.py @@ -494,6 +494,7 @@ # must be forced, however we need to execute it anyway '_NOSIDEEFFECT_LAST', # ----- end of no_side_effect operations ----- + 'INCREMENT_DEBUG_COUNTER/1', 'SETARRAYITEM_GC/3d', 'SETARRAYITEM_RAW/3d', 'SETINTERIORFIELD_GC/3d', diff --git a/rpython/jit/metainterp/test/test_dict.py b/rpython/jit/metainterp/test/test_dict.py --- a/rpython/jit/metainterp/test/test_dict.py +++ b/rpython/jit/metainterp/test/test_dict.py @@ -193,6 +193,107 @@ self.check_simple_loop({'int_sub': 1, 'int_gt': 1, 'guard_true': 1, 'jump': 1}) + def test_dict_two_lookups(self): + driver = JitDriver(greens = [], reds = 'auto') + d = {'a': 3, 'b': 4} + indexes = ['a', 'b'] + + def f(n): + s = 0 + while n > 0: + driver.jit_merge_point() + s += d[indexes[n & 1]] + s += d[indexes[n & 1]] + n -= 1 + return s + + self.meta_interp(f, [10]) + # XXX should be one getinteriorfield_gc + self.check_simple_loop(call=1, getinteriorfield_gc=2, + guard_no_exception=1) + + def test_ordered_dict_two_lookups(self): + driver = JitDriver(greens = [], reds = 'auto') + d = OrderedDict() + d['a'] = 3 + d['b'] = 4 + indexes = ['a', 'b'] + + def f(n): + s = 0 + while n > 0: + driver.jit_merge_point() + s += d[indexes[n & 1]] + s += d[indexes[n & 1]] + n -= 1 + return s + + self.meta_interp(f, [10]) + # XXX should be one getinteriorfield_gc + self.check_simple_loop(call=1, getinteriorfield_gc=2, + guard_no_exception=1) + + def test_dict_insert_invalidates_caches(self): + driver = JitDriver(greens = [], reds = 'auto') + indexes = ['aa', 'b', 'cc'] + + def f(n): + d = {'aa': 3, 'b': 4, 'cc': 5} + s = 0 + while n > 0: + driver.jit_merge_point() + index = indexes[n & 1] + s += d[index] + d['aa'] += 1 # this will invalidate the index + s += d[index] + n -= 1 + return s + + res = self.meta_interp(f, [10]) + assert res == f(10) + self.check_simple_loop(call=5) + + def test_dict_array_write_invalidates_caches(self): + driver = JitDriver(greens = [], reds = 'auto') + indexes = ['aa', 'b', 'cc'] + + def f(n): + d = {'aa': 3, 'b': 4, 'cc': 5} + s = 0 + while n > 0: + driver.jit_merge_point() + index = indexes[n & 1] + s += d[index] + del d['cc'] + s += d[index] + d['cc'] = 3 + n -= 1 + return s + + exp = f(10) + res = self.meta_interp(f, [10]) + assert res == exp + self.check_simple_loop(call=7) + + def test_dict_double_lookup_2(self): + driver = JitDriver(greens = [], reds = 'auto') + indexes = ['aa', 'b', 'cc'] + + def f(n): + d = {'aa': 3, 'b': 4, 'cc': 5} + s = 0 + while n > 0: + driver.jit_merge_point() + index = indexes[n & 1] + s += d[index] + d[index] += 1 + n -= 1 + return s + + res = self.meta_interp(f, [10]) + assert res == f(10) + self.check_simple_loop(call=3) + class TestLLtype(DictTests, LLJitMixin): pass diff --git a/rpython/jit/metainterp/virtualizable.py b/rpython/jit/metainterp/virtualizable.py --- a/rpython/jit/metainterp/virtualizable.py +++ b/rpython/jit/metainterp/virtualizable.py @@ -302,7 +302,7 @@ self.clear_vable_ptr = self.warmrunnerdesc.helper_func( FUNCPTR, self.clear_vable_token) FUNC = FUNCPTR.TO - ei = EffectInfo([], [], [], [], EffectInfo.EF_CANNOT_RAISE, + ei = EffectInfo([], [], [], [], [], [], EffectInfo.EF_CANNOT_RAISE, can_invalidate=False, oopspecindex=EffectInfo.OS_JIT_FORCE_VIRTUALIZABLE) diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py --- a/rpython/rlib/rfile.py +++ b/rpython/rlib/rfile.py @@ -14,6 +14,11 @@ includes = ['stdio.h', 'sys/types.h'] if os.name == "posix": includes += ['unistd.h'] + ftruncate = 'ftruncate' + fileno = 'fileno' +else: + ftruncate = '_chsize' + fileno = '_fileno' eci = ExternalCompilationInfo(includes=includes) def llexternal(*args, **kwargs): @@ -41,10 +46,10 @@ c_fseek = llexternal('fseek', [lltype.Ptr(FILE), rffi.LONG, rffi.INT], rffi.INT) c_tmpfile = llexternal('tmpfile', [], lltype.Ptr(FILE)) -c_fileno = llexternal('fileno', [lltype.Ptr(FILE)], rffi.INT) +c_fileno = llexternal(fileno, [lltype.Ptr(FILE)], rffi.INT) c_ftell = llexternal('ftell', [lltype.Ptr(FILE)], rffi.LONG) c_fflush = llexternal('fflush', [lltype.Ptr(FILE)], rffi.INT) -c_ftruncate = llexternal('ftruncate', [rffi.INT, OFF_T], rffi.INT, macro=True) +c_ftruncate = llexternal(ftruncate, [rffi.INT, OFF_T], rffi.INT, macro=True) c_fgets = llexternal('fgets', [rffi.CCHARP, rffi.INT, lltype.Ptr(FILE)], rffi.CCHARP) diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py --- a/rpython/rtyper/lltypesystem/rdict.py +++ b/rpython/rtyper/lltypesystem/rdict.py @@ -569,6 +569,7 @@ PERTURB_SHIFT = 5 @jit.look_inside_iff(lambda d, key, hash: jit.isvirtual(d) and jit.isconstant(key)) + at jit.oopspec('dict.lookup(d, key, hash)') def ll_dict_lookup(d, key, hash): entries = d.entries ENTRIES = lltype.typeOf(entries).TO diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py --- a/rpython/rtyper/lltypesystem/rordereddict.py +++ b/rpython/rtyper/lltypesystem/rordereddict.py @@ -709,6 +709,7 @@ @jit.look_inside_iff(lambda d, key, hash, store_flag, T: jit.isvirtual(d) and jit.isconstant(key)) + at jit.oopspec('ordereddict.lookup(d, key, hash, store_flag, T)') def ll_dict_lookup(d, key, hash, store_flag, T): INDEXES = _ll_ptr_to_array_of(T) entries = d.entries diff --git a/rpython/translator/backendopt/test/test_writeanalyze.py b/rpython/translator/backendopt/test/test_writeanalyze.py --- a/rpython/translator/backendopt/test/test_writeanalyze.py +++ b/rpython/translator/backendopt/test/test_writeanalyze.py @@ -353,3 +353,23 @@ result = wa.analyze(fgraph.startblock.operations[-1]) assert list(result) == [("struct", lltype.Ptr(S), "x")] + + def test_interiorfield(self): + A = lltype.GcArray(lltype.Struct('x', ('x', lltype.Signed), + ('y', lltype.Signed))) + + def g(x): + a = lltype.malloc(A, 1) + a[0].y = 3 + return f(a, x) + + def f(a, x): + a[0].x = x + return a[0].y + + t, wa = self.translate(g, [int]) + ggraph = graphof(t, g) + result = wa.analyze(ggraph.startblock.operations[-1]) + res = list(result) + assert ('readinteriorfield', lltype.Ptr(A), 'y') in res + assert ('interiorfield', lltype.Ptr(A), 'x') in res diff --git a/rpython/translator/backendopt/writeanalyze.py b/rpython/translator/backendopt/writeanalyze.py --- a/rpython/translator/backendopt/writeanalyze.py +++ b/rpython/translator/backendopt/writeanalyze.py @@ -1,4 +1,4 @@ -from rpython.flowspace.model import Variable +from rpython.flowspace.model import Variable, Constant from rpython.translator.backendopt import graphanalyze top_set = object() @@ -37,6 +37,12 @@ return top_set return result1.union(result2) + def _getinteriorname(self, op): + if (isinstance(op.args[1], Constant) and + isinstance(op.args[1].value, str)): + return op.args[1].value + return op.args[2].value + def analyze_simple_operation(self, op, graphinfo): if op.opname == "setfield": if graphinfo is None or not graphinfo.is_fresh_malloc(op.args[0]): @@ -45,11 +51,18 @@ elif op.opname == "setarrayitem": if graphinfo is None or not graphinfo.is_fresh_malloc(op.args[0]): return self._array_result(op.args[0].concretetype) + elif op.opname == "setinteriorfield": + if graphinfo is None or not graphinfo.is_fresh_malloc(op.args[0]): + name = self._getinteriorname(op) + return self._interiorfield_result(op.args[0].concretetype, name) return empty_set def _array_result(self, TYPE): return frozenset([("array", TYPE)]) + def _interiorfield_result(self, TYPE, fieldname): + return frozenset([("interiorfield", TYPE, fieldname)]) + def compute_graph_info(self, graph): return FreshMallocs(graph) @@ -99,4 +112,8 @@ elif op.opname == "getarrayitem": return frozenset([ ("readarray", op.args[0].concretetype)]) + elif op.opname == "getinteriorfield": + name = self._getinteriorname(op) + return frozenset([("readinteriorfield", op.args[0].concretetype, + name)]) return WriteAnalyzer.analyze_simple_operation(self, op, graphinfo) From noreply at buildbot.pypy.org Fri Mar 21 00:02:34 2014 From: noreply at buildbot.pypy.org (pjenvey) Date: Fri, 21 Mar 2014 00:02:34 +0100 (CET) Subject: [pypy-commit] pypy py3k: skip for now: requires the not NotImplemented audioop.lin2ulaw Message-ID: <20140320230234.EE87F1D2866@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r70136:eed6c50a88fc Date: 2014-03-20 16:00 -0700 http://bitbucket.org/pypy/pypy/changeset/eed6c50a88fc/ Log: skip for now: requires the not NotImplemented audioop.lin2ulaw diff --git a/lib-python/3/test/test_sunau.py b/lib-python/3/test/test_sunau.py --- a/lib-python/3/test/test_sunau.py +++ b/lib-python/3/test/test_sunau.py @@ -1,4 +1,4 @@ -from test.support import run_unittest, TESTFN +from test.support import run_unittest, TESTFN, impl_detail import unittest import os @@ -41,6 +41,7 @@ self.assertEqual(self.f.readframes(nframes), output) self.f.close() + @impl_detail(pypy=False) def test_ulaw(self): self.f = sunau.open(TESTFN, 'w') self.f.setnchannels(nchannels) From noreply at buildbot.pypy.org Fri Mar 21 00:02:36 2014 From: noreply at buildbot.pypy.org (pjenvey) Date: Fri, 21 Mar 2014 00:02:36 +0100 (CET) Subject: [pypy-commit] pypy py3k: adapt to buffer refactoring Message-ID: <20140320230236.363E21D2866@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r70137:f890d933e737 Date: 2014-03-20 16:00 -0700 http://bitbucket.org/pypy/pypy/changeset/f890d933e737/ Log: adapt to buffer refactoring diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py --- a/lib_pypy/_ctypes/basics.py +++ b/lib_pypy/_ctypes/basics.py @@ -136,7 +136,7 @@ return self.value def __buffer__(self): - return self._buffer.__buffer__() + return memoryview(self._buffer) def _get_b_base(self): try: From noreply at buildbot.pypy.org Fri Mar 21 00:02:37 2014 From: noreply at buildbot.pypy.org (pjenvey) Date: Fri, 21 Mar 2014 00:02:37 +0100 (CET) Subject: [pypy-commit] pypy py3k: add a TODO comment Message-ID: <20140320230237.9B7451D2866@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r70138:5f6d9d6505b2 Date: 2014-03-20 16:01 -0700 http://bitbucket.org/pypy/pypy/changeset/5f6d9d6505b2/ Log: add a TODO comment diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py --- a/pypy/objspace/std/typeobject.py +++ b/pypy/objspace/std/typeobject.py @@ -781,6 +781,8 @@ return space.newlist(w_type.get_subclasses()) def descr___prepare__(space, __args__): + # XXX: space.newdict(strdict=True)? (XXX: which should be + # UnicodeDictStrategy but is currently BytesDictStrategy) return space.newdict() # ____________________________________________________________ From noreply at buildbot.pypy.org Fri Mar 21 01:34:35 2014 From: noreply at buildbot.pypy.org (bdkearns) Date: Fri, 21 Mar 2014 01:34:35 +0100 (CET) Subject: [pypy-commit] pypy default: fix test_flatten/test_longlong on 32bit Message-ID: <20140321003435.52DB41C02AF@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r70139:d5dd9d0ba5c4 Date: 2014-03-21 00:33 +0000 http://bitbucket.org/pypy/pypy/changeset/d5dd9d0ba5c4/ Log: fix test_flatten/test_longlong on 32bit diff --git a/rpython/jit/codewriter/test/test_flatten.py b/rpython/jit/codewriter/test/test_flatten.py --- a/rpython/jit/codewriter/test/test_flatten.py +++ b/rpython/jit/codewriter/test/test_flatten.py @@ -73,7 +73,7 @@ def guess_call_kind(self, op): return 'residual' def getcalldescr(self, op, oopspecindex=EffectInfo.OS_NONE, - extraeffect=None): + extraeffect=None, extradescr=None): try: name = op.args[0].value._obj._name if 'cannot_raise' in name or name.startswith('cast_'): diff --git a/rpython/jit/codewriter/test/test_longlong.py b/rpython/jit/codewriter/test/test_longlong.py --- a/rpython/jit/codewriter/test/test_longlong.py +++ b/rpython/jit/codewriter/test/test_longlong.py @@ -17,7 +17,7 @@ class FakeBuiltinCallControl: def guess_call_kind(self, op): return 'builtin' - def getcalldescr(self, op, oopspecindex=None, extraeffect=None): + def getcalldescr(self, op, oopspecindex=None, extraeffect=None, extradescr=None): assert oopspecindex is not None # in this test return 'calldescr-%d' % oopspecindex def calldescr_canraise(self, calldescr): From noreply at buildbot.pypy.org Fri Mar 21 07:55:52 2014 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 21 Mar 2014 07:55:52 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Un-comment-out the implementation of stm_ignored, even if right now it's not used Message-ID: <20140321065552.66EF91C029E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r70140:5e13c2ff77d6 Date: 2014-03-21 07:48 +0100 http://bitbucket.org/pypy/pypy/changeset/5e13c2ff77d6/ Log: Un-comment-out the implementation of stm_ignored, even if right now it's not used diff --git a/rpython/memory/gctransform/stmframework.py b/rpython/memory/gctransform/stmframework.py --- a/rpython/memory/gctransform/stmframework.py +++ b/rpython/memory/gctransform/stmframework.py @@ -53,13 +53,13 @@ "unbalanced stm_ignore_start/stm_ignore_stop in block") def gct_stm_ignored_start(self, hop): - #assert not self.in_stm_ignored - #self.in_stm_ignored = True + assert not self.in_stm_ignored + self.in_stm_ignored = True self.default(hop) def gct_stm_ignored_stop(self, hop): - #assert self.in_stm_ignored - #self.in_stm_ignored = False + assert self.in_stm_ignored + self.in_stm_ignored = False self.default(hop) def var_needs_set_transform(self, var): diff --git a/rpython/translator/stm/readbarrier.py b/rpython/translator/stm/readbarrier.py --- a/rpython/translator/stm/readbarrier.py +++ b/rpython/translator/stm/readbarrier.py @@ -76,11 +76,11 @@ [op.args[0]], v_none)) transformer.read_barrier_counts += 1 elif op.opname == 'stm_ignored_start': - pass #assert stm_ignored == False - #stm_ignored = True + assert stm_ignored == False + stm_ignored = True elif op.opname == 'stm_ignored_stop': - pass #assert stm_ignored == True - #stm_ignored = False + assert stm_ignored == True + stm_ignored = False newops.append(op) assert stm_ignored == False block.operations = newops From noreply at buildbot.pypy.org Fri Mar 21 07:55:53 2014 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 21 Mar 2014 07:55:53 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Skip or remove the remaining tests Message-ID: <20140321065553.CF4321C029E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r70141:53a2a9ecde3e Date: 2014-03-21 07:54 +0100 http://bitbucket.org/pypy/pypy/changeset/53a2a9ecde3e/ Log: Skip or remove the remaining tests diff --git a/rpython/translator/stm/test/test_ztranslated.py b/rpython/translator/stm/test/test_ztranslated.py --- a/rpython/translator/stm/test/test_ztranslated.py +++ b/rpython/translator/stm/test/test_ztranslated.py @@ -1,3 +1,4 @@ +import py from rpython.rlib import rstm, rgc, objectmodel from rpython.rlib.debug import debug_print from rpython.rtyper.lltypesystem import lltype, rffi @@ -190,6 +191,7 @@ assert '12\n12\n' in data, "got: %r" % (data,) def test_prebuilt_nongc(self): + py.test.skip("stmframework: GC pointer written into a non-GC location") def check(foobar, retry_counter): return 0 # do nothing from rpython.rtyper.lltypesystem import lltype @@ -380,24 +382,6 @@ assert match assert int(match.group(1)) < 20 - def test_gc_writebarrier(self): - class X(object): - pass - prebuilt = X() - prebuilt.foo = 42 - - def main(argv): - llop.gc_writebarrier(lltype.Void, prebuilt) - debug_print(objectmodel.current_object_addr_as_int(prebuilt)) - prebuilt.foo = 43 - debug_print(objectmodel.current_object_addr_as_int(prebuilt)) - return 0 - - t, cbuilder = self.compile(main) - data, dataerr = cbuilder.cmdexec('', err=True) - lines = dataerr.split('\n') - assert lines[0] == lines[1] - def test_dtoa(self): def main(argv): a = len(argv) * 0.2 From noreply at buildbot.pypy.org Fri Mar 21 09:40:22 2014 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 21 Mar 2014 09:40:22 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Remove this bogus setting; always use 10000 as the default for Message-ID: <20140321084022.B912B1C02CF@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r70142:b5920bc6dd6c Date: 2014-03-21 09:39 +0100 http://bitbucket.org/pypy/pypy/changeset/b5920bc6dd6c/ Log: Remove this bogus setting; always use 10000 as the default for sys.setcheckinterval(). Use floating-point fractions internally. diff --git a/pypy/module/thread/stm.py b/pypy/module/thread/stm.py --- a/pypy/module/thread/stm.py +++ b/pypy/module/thread/stm.py @@ -63,10 +63,6 @@ """NOT_RPYTHON: set up a mechanism to send to the C code the value set by space.actionflag.setcheckinterval().""" # - # Set the default checkinterval to 200000, found by exploration to - # be a good default value. XXX do some more in-depth tests - space.actionflag.setcheckinterval(200000) - # def setcheckinterval_callback(): self.configure_transaction_length(space) # @@ -110,7 +106,7 @@ def configure_transaction_length(self, space): if self.threads_running: interval = space.actionflag.getcheckinterval() - rstm.set_transaction_length(interval) + rstm.set_transaction_length(interval / 10000.0) # ____________________________________________________________ diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py --- a/rpython/rlib/rstm.py +++ b/rpython/rlib/rstm.py @@ -61,8 +61,8 @@ llop.stm_should_break_transaction(lltype.Bool)) @dont_look_inside -def set_transaction_length(length): - llop.stm_set_transaction_length(lltype.Void, length) +def set_transaction_length(fraction): + llop.stm_set_transaction_length(lltype.Void, float(fraction)) @dont_look_inside def increment_atomic(): diff --git a/rpython/translator/stm/src_stm/stmgcintf.c b/rpython/translator/stm/src_stm/stmgcintf.c --- a/rpython/translator/stm/src_stm/stmgcintf.c +++ b/rpython/translator/stm/src_stm/stmgcintf.c @@ -35,11 +35,11 @@ static long pypy_transaction_length; -void pypy_stm_set_transaction_length(long percentage) +void pypy_stm_set_transaction_length(double fraction) { /* the value '100' means 'use the default'. Other values are interpreted proportionally, up to some maximum. */ - long low_fill_mark = LOW_FILL_MARK * percentage / 100; + long low_fill_mark = (long)(LOW_FILL_MARK * fraction); if (low_fill_mark > NURSERY_SIZE / 2) low_fill_mark = NURSERY_SIZE / 2; pypy_transaction_length = low_fill_mark; @@ -50,7 +50,7 @@ stm_setup(); pypy_stm_register_thread_local(); pypy_stm_ready_atomic = 1; - pypy_stm_set_transaction_length(100); + pypy_stm_set_transaction_length(1.0); pypy_stm_start_inevitable_if_not_atomic(); } diff --git a/rpython/translator/stm/src_stm/stmgcintf.h b/rpython/translator/stm/src_stm/stmgcintf.h --- a/rpython/translator/stm/src_stm/stmgcintf.h +++ b/rpython/translator/stm/src_stm/stmgcintf.h @@ -46,7 +46,7 @@ } long pypy_stm_enter_callback_call(void); void pypy_stm_leave_callback_call(long); -void pypy_stm_set_transaction_length(long); +void pypy_stm_set_transaction_length(double); void pypy_stm_perform_transaction(object_t *, int(object_t *, int)); static inline int pypy_stm_should_break_transaction(void) diff --git a/rpython/translator/stm/test/test_ztranslated.py b/rpython/translator/stm/test/test_ztranslated.py --- a/rpython/translator/stm/test/test_ztranslated.py +++ b/rpython/translator/stm/test/test_ztranslated.py @@ -98,7 +98,7 @@ def test_set_transaction_length(self): def entry_point(argv): - rstm.set_transaction_length(123) + rstm.set_transaction_length(0.123) return 0 t, cbuilder = self.compile(entry_point) cbuilder.cmdexec('') From noreply at buildbot.pypy.org Fri Mar 21 11:20:18 2014 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 21 Mar 2014 11:20:18 +0100 (CET) Subject: [pypy-commit] pypy default: add more debugging info for InvalidLoops Message-ID: <20140321102019.010BE1C02EA@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r70143:ca27e8ab9f84 Date: 2014-03-21 12:18 +0200 http://bitbucket.org/pypy/pypy/changeset/ca27e8ab9f84/ Log: add more debugging info for InvalidLoops diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py --- a/rpython/jit/metainterp/optimizeopt/rewrite.py +++ b/rpython/jit/metainterp/optimizeopt/rewrite.py @@ -242,8 +242,9 @@ box = value.box assert isinstance(box, Const) if not box.same_constant(constbox): - raise InvalidLoop('A GUARD_{VALUE,TRUE,FALSE} was proven to' + - 'always fail') + r = self.optimizer.metainterp_sd.logger_ops.repr_of_resop(op) + raise InvalidLoop('A GUARD_{VALUE,TRUE,FALSE} (%s) was proven ' + 'to always fail' % r) return if emit_operation: self.emit_operation(op) @@ -255,7 +256,9 @@ if value.is_null(): return elif value.is_nonnull(): - raise InvalidLoop('A GUARD_ISNULL was proven to always fail') + r = self.optimizer.metainterp_sd.logger_ops.repr_of_resop(op) + raise InvalidLoop('A GUARD_ISNULL (%s) was proven to always fail' + % r) self.emit_operation(op) value.make_constant(self.optimizer.cpu.ts.CONST_NULL) @@ -264,7 +267,9 @@ if value.is_nonnull(): return elif value.is_null(): - raise InvalidLoop('A GUARD_NONNULL was proven to always fail') + r = self.optimizer.metainterp_sd.logger_ops.repr_of_resop(op) + raise InvalidLoop('A GUARD_NONNULL (%s) was proven to always fail' + % r) self.emit_operation(op) value.make_nonnull(op) @@ -292,7 +297,8 @@ assert previous_classbox is not None assert expected_classbox is not None if not previous_classbox.same_constant(expected_classbox): - raise InvalidLoop('A GUARD_VALUE was proven to always fail') + r = self.optimizer.metainterp_sd.logger_ops.repr_of_resop(op) + raise InvalidLoop('A GUARD_VALUE (%s) was proven to always fail' % r) op = old_guard_op.copy_and_change(rop.GUARD_VALUE, args = [old_guard_op.getarg(0), op.getarg(1)]) self.optimizer.replaces_guard[op] = old_guard_op @@ -333,7 +339,9 @@ if realclassbox is not None: if realclassbox.same_constant(expectedclassbox): return - raise InvalidLoop('A GUARD_CLASS was proven to always fail') + r = self.optimizer.metainterp_sd.logger_ops.repr_of_resop(op) + raise InvalidLoop('A GUARD_CLASS (%s) was proven to always fail' + % r) if value.last_guard: # there already has been a guard_nonnull or guard_class or # guard_nonnull_class on this value. @@ -356,8 +364,9 @@ def optimize_GUARD_NONNULL_CLASS(self, op): value = self.getvalue(op.getarg(0)) if value.is_null(): - raise InvalidLoop('A GUARD_NONNULL_CLASS was proven to always ' + - 'fail') + r = self.optimizer.metainterp_sd.logger_ops.repr_of_resop(op) + raise InvalidLoop('A GUARD_NONNULL_CLASS (%s) was proven to ' + 'always fail' % r) self.optimize_GUARD_CLASS(op) def optimize_CALL_LOOPINVARIANT(self, op): From noreply at buildbot.pypy.org Fri Mar 21 11:20:21 2014 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 21 Mar 2014 11:20:21 +0100 (CET) Subject: [pypy-commit] pypy default: merge Message-ID: <20140321102021.207161C02EA@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r70144:a4dca0f2cdfe Date: 2014-03-21 12:19 +0200 http://bitbucket.org/pypy/pypy/changeset/a4dca0f2cdfe/ Log: merge diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -440,11 +440,10 @@ return name - def getbuiltinmodule(self, name, force_init=False, reuse=True): + def getbuiltinmodule(self, name, force_init=False): w_name = self.wrap(name) w_modules = self.sys.get('modules') if not force_init: - assert reuse is True try: return self.getitem(w_modules, w_name) except OperationError, e: @@ -463,15 +462,7 @@ # Initialize the module from pypy.interpreter.module import Module if isinstance(w_mod, Module): - if not reuse and w_mod.startup_called: - # Create a copy of the module - w_mod.getdict(self) # unlazy w_initialdict - w_new = self.wrap(Module(self, w_name)) - self.call_method(w_new.getdict(self), 'update', - w_mod.w_initialdict) - w_mod = w_new - else: - w_mod.init(self) + w_mod.init(self) # Add the module to sys.modules self.setitem(w_modules, w_name, w_mod) diff --git a/pypy/module/_io/__init__.py b/pypy/module/_io/__init__.py --- a/pypy/module/_io/__init__.py +++ b/pypy/module/_io/__init__.py @@ -8,6 +8,8 @@ interpleveldefs = { 'DEFAULT_BUFFER_SIZE': 'space.wrap(interp_iobase.DEFAULT_BUFFER_SIZE)', 'BlockingIOError': 'interp_io.W_BlockingIOError', + 'UnsupportedOperation': + 'space.fromcache(interp_io.Cache).w_unsupportedoperation', '_IOBase': 'interp_iobase.W_IOBase', '_RawIOBase': 'interp_iobase.W_RawIOBase', '_BufferedIOBase': 'interp_bufferedio.W_BufferedIOBase', @@ -26,16 +28,6 @@ 'IncrementalNewlineDecoder': 'interp_textio.W_IncrementalNewlineDecoder', } - def init(self, space): - MixedModule.init(self, space) - w_UnsupportedOperation = space.call_function( - space.w_type, - space.wrap('UnsupportedOperation'), - space.newtuple([space.w_ValueError, space.w_IOError]), - space.newdict()) - space.setattr(self, space.wrap('UnsupportedOperation'), - w_UnsupportedOperation) - def shutdown(self, space): # at shutdown, flush all open streams. Ignore I/O errors. from pypy.module._io.interp_iobase import get_autoflusher diff --git a/pypy/module/_io/interp_io.py b/pypy/module/_io/interp_io.py --- a/pypy/module/_io/interp_io.py +++ b/pypy/module/_io/interp_io.py @@ -10,6 +10,12 @@ from rpython.rtyper.module.ll_os_stat import STAT_FIELD_TYPES +class Cache: + def __init__(self, space): + self.w_unsupportedoperation = space.new_exception_class( + "io.UnsupportedOperation", + space.newtuple([space.w_ValueError, space.w_IOError])) + class W_BlockingIOError(W_IOError): def __init__(self, space): W_IOError.__init__(self, space) diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py --- a/pypy/module/imp/importing.py +++ b/pypy/module/imp/importing.py @@ -579,8 +579,7 @@ return space.call_method(find_info.w_loader, "load_module", w_modulename) if find_info.modtype == C_BUILTIN: - return space.getbuiltinmodule(find_info.filename, force_init=True, - reuse=reuse) + return space.getbuiltinmodule(find_info.filename, force_init=True) if find_info.modtype in (PY_SOURCE, PY_COMPILED, C_EXTENSION, PKG_DIRECTORY): w_mod = None diff --git a/pypy/module/imp/test/test_app.py b/pypy/module/imp/test/test_app.py --- a/pypy/module/imp/test/test_app.py +++ b/pypy/module/imp/test/test_app.py @@ -203,6 +203,7 @@ def test_builtin_reimport(self): # from https://bugs.pypy.org/issue1514 + skip("fix me") import sys, marshal old = marshal.loads @@ -222,6 +223,7 @@ # taken from https://bugs.pypy.org/issue1514, with extra cases # that show a difference with CPython: we can get on CPython # several module objects for the same built-in module :-( + skip("several built-in module objects: not supported by pypy") import sys, marshal old = marshal.loads diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py --- a/pypy/module/imp/test/test_import.py +++ b/pypy/module/imp/test/test_import.py @@ -578,6 +578,7 @@ assert hasattr(time, 'clock') def test_reimport_builtin_simple_case_2(self): + skip("fix me") import sys, time time.foo = "bar" del sys.modules['time'] @@ -585,6 +586,7 @@ assert not hasattr(time, 'foo') def test_reimport_builtin(self): + skip("fix me") import sys, time oldpath = sys.path time.tzset = "" diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py --- a/rpython/jit/backend/arm/opassembler.py +++ b/rpython/jit/backend/arm/opassembler.py @@ -583,6 +583,10 @@ emit_op_getfield_raw_pure = emit_op_getfield_gc emit_op_getfield_gc_pure = emit_op_getfield_gc + def emit_op_increment_debug_counter(self, op, arglocs, regalloc, fcond): + # XXX implement me + return fcond + def emit_op_getinteriorfield_gc(self, op, arglocs, regalloc, fcond): (base_loc, index_loc, res_loc, ofs_loc, ofs, itemsize, fieldsize) = arglocs diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py --- a/rpython/jit/backend/arm/regalloc.py +++ b/rpython/jit/backend/arm/regalloc.py @@ -849,6 +849,10 @@ prepare_op_getfield_raw_pure = prepare_op_getfield_gc prepare_op_getfield_gc_pure = prepare_op_getfield_gc + def prepare_op_increment_debug_counter(self, op, fcond): + # XXX implement me + return [] + def prepare_op_getinteriorfield_gc(self, op, fcond): t = unpack_interiorfielddescr(op.getdescr()) ofs, itemsize, fieldsize, sign = t diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py --- a/rpython/jit/backend/llgraph/runner.py +++ b/rpython/jit/backend/llgraph/runner.py @@ -553,6 +553,10 @@ else: return self.bh_raw_load_i(struct, offset, descr) + def bh_increment_debug_counter(self, addr): + p = rffi.cast(rffi.CArrayPtr(lltype.Signed), addr) + p[0] += 1 + def unpack_arraydescr_size(self, arraydescr): from rpython.jit.backend.llsupport.symbolic import get_array_token from rpython.jit.backend.llsupport.descr import get_type_flag, FLAG_SIGNED diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py --- a/rpython/jit/backend/llsupport/assembler.py +++ b/rpython/jit/backend/llsupport/assembler.py @@ -15,7 +15,7 @@ DEBUG_COUNTER = lltype.Struct('DEBUG_COUNTER', # 'b'ridge, 'l'abel or # 'e'ntry point - ('i', lltype.Signed), + ('i', lltype.Signed), # first field, at offset 0 ('type', lltype.Char), ('number', lltype.Signed) ) @@ -64,7 +64,6 @@ self.cpu = cpu self.memcpy_addr = 0 self.rtyper = cpu.rtyper - self.debug_counter_descr = cpu.fielddescrof(DEBUG_COUNTER, 'i') self._debug = False def setup_once(self): @@ -265,14 +264,8 @@ def _append_debugging_code(self, operations, tp, number, token): counter = self._register_counter(tp, number, token) c_adr = ConstInt(rffi.cast(lltype.Signed, counter)) - box = BoxInt() - box2 = BoxInt() - ops = [ResOperation(rop.GETFIELD_RAW, [c_adr], - box, descr=self.debug_counter_descr), - ResOperation(rop.INT_ADD, [box, ConstInt(1)], box2), - ResOperation(rop.SETFIELD_RAW, [c_adr, box2], - None, descr=self.debug_counter_descr)] - operations.extend(ops) + operations.append( + ResOperation(rop.INCREMENT_DEBUG_COUNTER, [c_adr], None)) def _register_counter(self, tp, number, token): # YYY very minor leak -- we need the counters to stay alive diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py --- a/rpython/jit/backend/test/runner_test.py +++ b/rpython/jit/backend/test/runner_test.py @@ -4338,3 +4338,12 @@ assert rffi.cast(lltype.Signed, a[0]) == -7654 assert rffi.cast(lltype.Signed, a[1]) == 777 lltype.free(a, flavor='raw') + + def test_increment_debug_counter(self): + foo = lltype.malloc(rffi.CArray(lltype.Signed), 1, flavor='raw') + foo[0] = 1789200 + self.execute_operation(rop.INCREMENT_DEBUG_COUNTER, + [ConstInt(rffi.cast(lltype.Signed, foo))], + 'void') + assert foo[0] == 1789201 + lltype.free(foo, flavor='raw') diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -434,8 +434,8 @@ self.wb_slowpath[withcards + 2 * withfloats] = rawstart @rgc.no_release_gil - def assemble_loop(self, logger, loopname, inputargs, operations, looptoken, - log): + def assemble_loop(self, inputargs, operations, looptoken, log, + loopname, logger): '''adds the following attributes to looptoken: _ll_function_addr (address of the generated func, as an int) _ll_loop_code (debug: addr of the start of the ResOps) @@ -514,8 +514,8 @@ size_excluding_failure_stuff - looppos) @rgc.no_release_gil - def assemble_bridge(self, logger, faildescr, inputargs, operations, - original_loop_token, log): + def assemble_bridge(self, faildescr, inputargs, operations, + original_loop_token, log, logger): if not we_are_translated(): # Arguments should be unique assert len(set(inputargs)) == len(inputargs) @@ -1469,6 +1469,14 @@ ofs_loc) self.load_from_mem(resloc, src_addr, fieldsize_loc, sign_loc) + def genop_discard_increment_debug_counter(self, op, arglocs): + # The argument should be an immediate address. This should + # generate code equivalent to a GETFIELD_RAW, an ADD(1), and a + # SETFIELD_RAW. Here we use the direct from-memory-to-memory + # increment operation of x86. + base_loc, = arglocs + self.mc.INC(mem(base_loc, 0)) + def genop_discard_setfield_gc(self, op, arglocs): base_loc, ofs_loc, size_loc, value_loc = arglocs assert isinstance(size_loc, ImmedLoc) diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -1003,6 +1003,10 @@ consider_getfield_raw_pure = consider_getfield_gc consider_getfield_gc_pure = consider_getfield_gc + def consider_increment_debug_counter(self, op): + base_loc = self.loc(op.getarg(0)) + self.perform_discard(op, [base_loc]) + def consider_getarrayitem_gc(self, op): itemsize, ofs, sign = unpack_arraydescr(op.getdescr()) args = op.getarglist() diff --git a/rpython/jit/backend/x86/regloc.py b/rpython/jit/backend/x86/regloc.py --- a/rpython/jit/backend/x86/regloc.py +++ b/rpython/jit/backend/x86/regloc.py @@ -488,12 +488,22 @@ for possible_code in unrolling_location_codes: if code == possible_code: val = getattr(loc, "value_" + possible_code)() - if self.WORD == 8 and possible_code == 'i' and not rx86.fits_in_32bits(val): - self._load_scratch(val) + # Faking out of certain operations for x86_64 + fits32 = rx86.fits_in_32bits + if possible_code == 'i' and not fits32(val): + self._load_scratch(val) # for 'PUSH(imm)' _rx86_getattr(self, name + "_r")(X86_64_SCRATCH_REG.value) - else: - methname = name + "_" + possible_code - _rx86_getattr(self, methname)(val) + return + if possible_code == 'j' and not fits32(val): + val = self._addr_as_reg_offset(val) + _rx86_getattr(self, name + "_m")(val) + return + if possible_code == 'm' and not fits32(val[1]): + val = self._fix_static_offset_64_m(val) + if possible_code == 'a' and not fits32(val[3]): + val = self._fix_static_offset_64_a(val) + methname = name + "_" + possible_code + _rx86_getattr(self, methname)(val) return func_with_new_name(INSN, "INSN_" + name) @@ -600,6 +610,7 @@ TEST8 = _binaryop('TEST8') BTS = _binaryop('BTS') + INC = _unaryop('INC') ADD = _binaryop('ADD') SUB = _binaryop('SUB') IMUL = _binaryop('IMUL') diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py --- a/rpython/jit/backend/x86/runner.py +++ b/rpython/jit/backend/x86/runner.py @@ -93,16 +93,15 @@ def compile_loop(self, inputargs, operations, looptoken, log=True, name='', logger=None): - return self.assembler.assemble_loop(logger, name, inputargs, operations, - looptoken, log=log) + return self.assembler.assemble_loop(inputargs, operations, looptoken, log, + name, logger) def compile_bridge(self, faildescr, inputargs, operations, original_loop_token, log=True, logger=None): clt = original_loop_token.compiled_loop_token clt.compiling_a_bridge() - return self.assembler.assemble_bridge(logger, faildescr, inputargs, - operations, - original_loop_token, log=log) + return self.assembler.assemble_bridge(faildescr, inputargs, operations, + original_loop_token, log, logger) def clear_latest_values(self, count): setitem = self.assembler.fail_boxes_ptr.setitem diff --git a/rpython/jit/backend/x86/rx86.py b/rpython/jit/backend/x86/rx86.py --- a/rpython/jit/backend/x86/rx86.py +++ b/rpython/jit/backend/x86/rx86.py @@ -470,6 +470,9 @@ # ------------------------------ Arithmetic ------------------------------ + INC_m = insn(rex_w, '\xFF', orbyte(0), mem_reg_plus_const(1)) + INC_j = insn(rex_w, '\xFF', orbyte(0), abs_(1)) + ADD_ri,ADD_rr,ADD_rb,_,_,ADD_rm,ADD_rj,_,_ = common_modes(0) OR_ri, OR_rr, OR_rb, _,_,OR_rm, OR_rj, _,_ = common_modes(1) AND_ri,AND_rr,AND_rb,_,_,AND_rm,AND_rj,_,_ = common_modes(4) diff --git a/rpython/jit/backend/x86/test/test_regloc.py b/rpython/jit/backend/x86/test/test_regloc.py --- a/rpython/jit/backend/x86/test/test_regloc.py +++ b/rpython/jit/backend/x86/test/test_regloc.py @@ -373,3 +373,56 @@ '\x59' ) assert cb.getvalue() == expected_instructions + + # ------------------------------------------------------------ + + def test_push_immed64(self): + immed = 0x0123456789ABCDEF + cb = LocationCodeBuilder64() + cb.PUSH(imm(immed)) + # + expected_instructions = ( + # mov r11, 0x0123456789ABCDEF + '\x49\xBB\xEF\xCD\xAB\x89\x67\x45\x23\x01' + # push r11 + '\x41\x53' + ) + assert cb.getvalue() == expected_instructions + + def test_inc_64bit_address_1(self): + base_addr = 0x0123456789ABCDEF + cb = LocationCodeBuilder64() + cb.INC(AddressLoc(ImmedLoc(0), ImmedLoc(0), 0, base_addr)) + # this case is a INC_j + # + expected_instructions = ( + # mov r11, 0x0123456789ABCDEF + '\x49\xBB\xEF\xCD\xAB\x89\x67\x45\x23\x01' + # inc [r11] + '\x49\xFF\x03' + ) + assert cb.getvalue() == expected_instructions + + def test_inc_64bit_address_2(self): + py.test.skip("there is no unary instruction INSN_a so far") + base_addr = 0x0123456789ABCDEF + cb = LocationCodeBuilder64() + cb.INC(AddressLoc(ImmedLoc(0), edx, 3, base_addr)) + # this case would be a INC_a + xxx + + def test_inc_64bit_address_3(self): + base_addr = 0x0123456789ABCDEF + cb = LocationCodeBuilder64() + cb.INC(AddressLoc(eax, ImmedLoc(0), 0, base_addr)) + # this case is a INC_m + # + expected_instructions = ( + # mov r11, 0x0123456789ABCDEF + '\x49\xBB\xEF\xCD\xAB\x89\x67\x45\x23\x01' + # lea r11, [rax+r11] + '\x4E\x8D\x1C\x18' + # inc [r11] + '\x49\xFF\x03' + ) + assert cb.getvalue() == expected_instructions diff --git a/rpython/jit/backend/x86/test/test_runner.py b/rpython/jit/backend/x86/test/test_runner.py --- a/rpython/jit/backend/x86/test/test_runner.py +++ b/rpython/jit/backend/x86/test/test_runner.py @@ -427,8 +427,8 @@ debug._log = None # assert ops_offset is looptoken._x86_ops_offset - # 2*(getfield_raw/int_add/setfield_raw) + ops + None - assert len(ops_offset) == 2*3 + len(operations) + 1 + # 2*increment_debug_counter + ops + None + assert len(ops_offset) == 2 + len(operations) + 1 assert (ops_offset[operations[0]] <= ops_offset[operations[1]] <= ops_offset[operations[2]] <= diff --git a/rpython/jit/codewriter/test/test_flatten.py b/rpython/jit/codewriter/test/test_flatten.py --- a/rpython/jit/codewriter/test/test_flatten.py +++ b/rpython/jit/codewriter/test/test_flatten.py @@ -73,7 +73,7 @@ def guess_call_kind(self, op): return 'residual' def getcalldescr(self, op, oopspecindex=EffectInfo.OS_NONE, - extraeffect=None): + extraeffect=None, extradescr=None): try: name = op.args[0].value._obj._name if 'cannot_raise' in name or name.startswith('cast_'): diff --git a/rpython/jit/codewriter/test/test_longlong.py b/rpython/jit/codewriter/test/test_longlong.py --- a/rpython/jit/codewriter/test/test_longlong.py +++ b/rpython/jit/codewriter/test/test_longlong.py @@ -17,7 +17,7 @@ class FakeBuiltinCallControl: def guess_call_kind(self, op): return 'builtin' - def getcalldescr(self, op, oopspecindex=None, extraeffect=None): + def getcalldescr(self, op, oopspecindex=None, extraeffect=None, extradescr=None): assert oopspecindex is not None # in this test return 'calldescr-%d' % oopspecindex def calldescr_canraise(self, calldescr): diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py --- a/rpython/jit/metainterp/executor.py +++ b/rpython/jit/metainterp/executor.py @@ -332,6 +332,7 @@ continue if value in (rop.FORCE_TOKEN, rop.CALL_ASSEMBLER, + rop.INCREMENT_DEBUG_COUNTER, rop.COND_CALL_GC_WB, rop.COND_CALL_GC_WB_ARRAY, rop.DEBUG_MERGE_POINT, diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py --- a/rpython/jit/metainterp/optimizeopt/heap.py +++ b/rpython/jit/metainterp/optimizeopt/heap.py @@ -306,13 +306,13 @@ def _optimize_CALL_DICT_LOOKUP(self, op): descrs = op.getdescr().get_extra_info().extradescrs + assert descrs # translation hint descr1 = descrs[0] - descr2 = descrs[1] - if descr1 in self.cached_dict_reads: + try: d = self.cached_dict_reads[descr1] - else: + except KeyError: d = self.cached_dict_reads[descr1] = args_dict() - self.corresponding_array_descrs[descr2] = descr1 + self.corresponding_array_descrs[descrs[1]] = descr1 args = self.optimizer.make_args_key(op) try: res_v = d[args] diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py --- a/rpython/jit/metainterp/resoperation.py +++ b/rpython/jit/metainterp/resoperation.py @@ -494,6 +494,7 @@ # must be forced, however we need to execute it anyway '_NOSIDEEFFECT_LAST', # ----- end of no_side_effect operations ----- + 'INCREMENT_DEBUG_COUNTER/1', 'SETARRAYITEM_GC/3d', 'SETARRAYITEM_RAW/3d', 'SETINTERIORFIELD_GC/3d', From noreply at buildbot.pypy.org Fri Mar 21 11:27:28 2014 From: noreply at buildbot.pypy.org (anton_gulenko) Date: Fri, 21 Mar 2014 11:27:28 +0100 (CET) Subject: [pypy-commit] lang-smalltalk strategies-tagging: Added configurable flag to enable/disable usage of a rerased pair. Will compare performance. Message-ID: <20140321102728.A0C031C02EA@cobra.cs.uni-duesseldorf.de> Author: Anton Gulenko Branch: strategies-tagging Changeset: r675:d878eb7b148c Date: 2014-03-20 14:55 +0100 http://bitbucket.org/pypy/lang-smalltalk/changeset/d878eb7b148c/ Log: Added configurable flag to enable/disable usage of a rerased pair. Will compare performance. diff --git a/spyvm/strategies.py b/spyvm/strategies.py --- a/spyvm/strategies.py +++ b/spyvm/strategies.py @@ -1,5 +1,6 @@ + +import sys from spyvm import model, shadow - from rpython.rlib import rerased from rpython.rlib.objectmodel import import_from_mixin @@ -53,13 +54,13 @@ strategy_tag = 'abstract-int' def storage(self, w_obj): - return w_obj.int_storage + return self.unerase(w_obj.int_storage) def set_initial_storage(self, space, w_obj, size): - w_obj.int_storage = self.initial_storage(space, size) + w_obj.int_storage = self.erase(self.initial_storage(space, size)) def set_storage_for_list(self, space, w_obj, collection): - w_obj.int_storage = self.storage_for_list(space, collection) + w_obj.int_storage = self.erase(self.storage_for_list(space, collection)) def set_storage_copied_from(self, space, w_obj, w_source_obj, reuse_storage=False): - w_obj.int_storage = self.copy_storage_from(space, w_source_obj, reuse_storage) + w_obj.int_storage = self.erase(self.copy_storage_from(space, w_source_obj, reuse_storage)) def initial_storage(self, space, size): raise NotImplementedError("Abstract base class") @@ -79,18 +80,20 @@ result.singleton = result() return result -class BasicStorageStrategyMixin(object): - def erase(self, a): return a - def unerase(self, a): return a - # erase, unerase = rerased.new_static_erasing_pair(self.strategy_tag) +use_rerased = False +def setup_rerased_pair(): + locals = sys._getframe(1).f_locals + if use_rerased: + locals["erase"], locals["unerase"] = rerased.new_static_erasing_pair("strategy-%s" % locals["strategy_tag"]) + else: + locals["erase"], locals["unerase"] = lambda self, x: x, lambda self, x: x # this is the typical "initial" storage strategy, for when every slot # in an object is still nil. No storage is allocated. class AllNilStorageStrategy(AbstractStorageStrategy): __metaclass__ = SingletonMeta - # erase, unerase = rerased.new_static_erasing_pair("allnil-strategy") - import_from_mixin(BasicStorageStrategyMixin) strategy_tag = 'allnil' + setup_rerased_pair() def fetch(self, space, w_obj, n0): return model.w_nil @@ -116,9 +119,8 @@ # fixed-sized and var-sized objects. class ListStorageStrategy(AbstractListStorageStrategy): __metaclass__ = SingletonMeta - # erase, unerase = rerased.new_static_erasing_pair("list-strategy") - import_from_mixin(BasicStorageStrategyMixin) strategy_tag = 'list' + setup_rerased_pair() def fetch(self, space, w_obj, n0): return self.storage(w_obj)[n0] @@ -135,9 +137,8 @@ class TaggingSmallIntegerStorageStrategy(AbstractIntStorageStrategy): __metaclass__ = SingletonMeta - # erase, unerase = rerased.new_static_erasing_pair("tagging-smallint-strategry") - import_from_mixin(BasicStorageStrategyMixin) strategy_tag = 'tagging-smallint' + setup_rerased_pair() needs_objspace = True @staticmethod From noreply at buildbot.pypy.org Fri Mar 21 11:27:29 2014 From: noreply at buildbot.pypy.org (anton_gulenko) Date: Fri, 21 Mar 2014 11:27:29 +0100 (CET) Subject: [pypy-commit] lang-smalltalk strategies-tagging: Added comment Message-ID: <20140321102729.BCF711C02EA@cobra.cs.uni-duesseldorf.de> Author: Anton Gulenko Branch: strategies-tagging Changeset: r676:30ce07241962 Date: 2014-03-20 14:59 +0100 http://bitbucket.org/pypy/lang-smalltalk/changeset/30ce07241962/ Log: Added comment diff --git a/spyvm/strategies.py b/spyvm/strategies.py --- a/spyvm/strategies.py +++ b/spyvm/strategies.py @@ -82,11 +82,12 @@ use_rerased = False def setup_rerased_pair(): - locals = sys._getframe(1).f_locals + # Small piece of metaprogramming stolen from rpython.rlib.objectmodel.import_from_mixin + cls = sys._getframe(1).f_locals if use_rerased: - locals["erase"], locals["unerase"] = rerased.new_static_erasing_pair("strategy-%s" % locals["strategy_tag"]) + cls["erase"], cls["unerase"] = rerased.new_static_erasing_pair("strategy-%s" % cls["strategy_tag"]) else: - locals["erase"], locals["unerase"] = lambda self, x: x, lambda self, x: x + cls["erase"], cls["unerase"] = lambda self, x: x, lambda self, x: x # this is the typical "initial" storage strategy, for when every slot # in an object is still nil. No storage is allocated. From noreply at buildbot.pypy.org Fri Mar 21 11:27:30 2014 From: noreply at buildbot.pypy.org (anton_gulenko) Date: Fri, 21 Mar 2014 11:27:30 +0100 (CET) Subject: [pypy-commit] lang-smalltalk strategies-tagging: Fixed the configurable rerased thing. Message-ID: <20140321102730.D6ACB1C02EA@cobra.cs.uni-duesseldorf.de> Author: Anton Gulenko Branch: strategies-tagging Changeset: r677:9aca5f6a189d Date: 2014-03-20 15:33 +0100 http://bitbucket.org/pypy/lang-smalltalk/changeset/9aca5f6a189d/ Log: Fixed the configurable rerased thing. diff --git a/spyvm/model.py b/spyvm/model.py --- a/spyvm/model.py +++ b/spyvm/model.py @@ -654,8 +654,6 @@ class W_PointersObject(W_AbstractPointersObject): _attrs_ = ['_size', 'list_storage', 'int_storage', 'strategy'] - list_storage = None - int_storage = None @jit.unroll_safe def __init__(self, space, w_class, size): @@ -738,8 +736,26 @@ return False self.strategy, w_other.strategy = w_other.strategy, self.strategy self._size, w_other._size = w_other._size, self._size - self.list_storage, w_other.list_storage = w_other.list_storage, self.list_storage - self.int_storage, w_other.int_storage = w_other.int_storage, self.int_storage + + # Unfortunately, the following is necessary to work both with RPYTHON and in interpreted mode. + # Rpython cannot handle list_storage = None in combination with a rerased pair. + + if hasattr(self, 'list_storage'): + if hasattr(w_other, 'list_storage'): + self.list_storage, w_other.list_storage = w_other.list_storage, self.list_storage + else: + w_other.list_storage = self.list_storage + elif hasattr(w_other, 'list_storage'): + self.list_storage = w_other.list_storage + + if hasattr(self, 'int_storage'): + if hasattr(w_other, 'int_storage'): + self.int_storage, w_other.int_storage = w_other.int_storage, self.int_storage + else: + w_other.int_storage = self.int_storage + elif hasattr(w_other, 'int_storage'): + self.int_storage = w_other.int_storage + return W_AbstractPointersObject.become(self, w_other) @jit.unroll_safe diff --git a/spyvm/strategies.py b/spyvm/strategies.py --- a/spyvm/strategies.py +++ b/spyvm/strategies.py @@ -30,13 +30,13 @@ strategy_tag = 'abstract-list' def storage(self, w_obj): - return w_obj.list_storage + return self.unerase(w_obj.list_storage) def set_initial_storage(self, space, w_obj, size): - w_obj.list_storage = self.initial_storage(space, size) + w_obj.list_storage = self.erase(self.initial_storage(space, size)) def set_storage_for_list(self, space, w_obj, collection): - w_obj.list_storage = self.storage_for_list(space, collection) + w_obj.list_storage = self.erase(self.storage_for_list(space, collection)) def set_storage_copied_from(self, space, w_obj, w_source_obj, reuse_storage=False): - w_obj.list_storage = self.copy_storage_from(space, w_source_obj, reuse_storage) + w_obj.list_storage = self.erase(self.copy_storage_from(space, w_source_obj, reuse_storage)) def initial_storage(self, space, size): raise NotImplementedError("Abstract base class") From noreply at buildbot.pypy.org Fri Mar 21 11:27:32 2014 From: noreply at buildbot.pypy.org (anton_gulenko) Date: Fri, 21 Mar 2014 11:27:32 +0100 (CET) Subject: [pypy-commit] lang-smalltalk strategies-tagging: Fix did not work, had to use we_are_translated(). Message-ID: <20140321102732.062441C02EA@cobra.cs.uni-duesseldorf.de> Author: Anton Gulenko Branch: strategies-tagging Changeset: r678:4aa0466bbb5a Date: 2014-03-20 16:14 +0100 http://bitbucket.org/pypy/lang-smalltalk/changeset/4aa0466bbb5a/ Log: Fix did not work, had to use we_are_translated(). Also removed obsolete pieces of fieldtype functionality. diff --git a/spyvm/model.py b/spyvm/model.py --- a/spyvm/model.py +++ b/spyvm/model.py @@ -22,7 +22,7 @@ from rpython.rlib.rarithmetic import intmask, r_uint, r_int from rpython.rlib.debug import make_sure_not_resized from rpython.tool.pairtype import extendabletype -from rpython.rlib.objectmodel import instantiate, compute_hash, import_from_mixin +from rpython.rlib.objectmodel import instantiate, compute_hash, import_from_mixin, we_are_translated from rpython.rtyper.lltypesystem import lltype, rffi from rsdl import RSDL, RSDL_helper @@ -143,10 +143,6 @@ def unwrap_uint(self, space): raise error.UnwrappingError("Got unexpected class in unwrap_uint") - def fieldtype(self): - from spyvm.strategies import obj - return obj - def is_array_object(self): return False @@ -221,10 +217,6 @@ def clone(self, space): return self - def fieldtype(self): - from spyvm.strategies import SInt - return SInt - class W_AbstractObjectWithIdentityHash(W_Object): """Object with explicit hash (ie all except small ints and floats).""" @@ -327,10 +319,6 @@ def invariant(self): return isinstance(self.value, int) - def fieldtype(self): - from spyvm.strategies import LPI - return LPI - def is_array_object(self): return True @@ -422,10 +410,6 @@ def size(self): return 2 - def fieldtype(self): - from spyvm.strategies import flt - return flt - @signature.finishsigs class W_AbstractObjectWithClassReference(W_AbstractObjectWithIdentityHash): """Objects with arbitrary class (ie not CompiledMethod, SmallInteger or @@ -654,6 +638,9 @@ class W_PointersObject(W_AbstractPointersObject): _attrs_ = ['_size', 'list_storage', 'int_storage', 'strategy'] + if not we_are_translated(): + list_storage = None + int_storage = None @jit.unroll_safe def __init__(self, space, w_class, size): @@ -736,26 +723,8 @@ return False self.strategy, w_other.strategy = w_other.strategy, self.strategy self._size, w_other._size = w_other._size, self._size - - # Unfortunately, the following is necessary to work both with RPYTHON and in interpreted mode. - # Rpython cannot handle list_storage = None in combination with a rerased pair. - - if hasattr(self, 'list_storage'): - if hasattr(w_other, 'list_storage'): - self.list_storage, w_other.list_storage = w_other.list_storage, self.list_storage - else: - w_other.list_storage = self.list_storage - elif hasattr(w_other, 'list_storage'): - self.list_storage = w_other.list_storage - - if hasattr(self, 'int_storage'): - if hasattr(w_other, 'int_storage'): - self.int_storage, w_other.int_storage = w_other.int_storage, self.int_storage - else: - w_other.int_storage = self.int_storage - elif hasattr(w_other, 'int_storage'): - self.int_storage = w_other.int_storage - + self.list_storage, w_other.list_storage = w_other.list_storage, self.list_storage + self.int_storage, w_other.int_storage = w_other.int_storage, self.int_storage return W_AbstractPointersObject.become(self, w_other) @jit.unroll_safe @@ -766,13 +735,9 @@ self.log_strategy_operation("Cloned") return w_result - def fieldtype(self): - from spyvm.strategies import obj - return obj - class W_WeakPointersObject(W_AbstractPointersObject): _attrs_ = ['_weakvars'] - + @jit.unroll_safe def __init__(self, space, w_class, size): W_AbstractPointersObject.__init__(self, space, w_class, size) diff --git a/spyvm/test/test_strategies.py b/spyvm/test/test_strategies.py --- a/spyvm/test/test_strategies.py +++ b/spyvm/test/test_strategies.py @@ -4,8 +4,6 @@ from spyvm.test import test_miniimage as tools from spyvm.error import WrapperException, FatalError -# Fieldtypes have a separate test file - space, interp = tools.setup_module(tools, filename='bootstrapped.image') class_Array = space.classtable["w_Array"] From noreply at buildbot.pypy.org Fri Mar 21 11:27:33 2014 From: noreply at buildbot.pypy.org (anton_gulenko) Date: Fri, 21 Mar 2014 11:27:33 +0100 (CET) Subject: [pypy-commit] lang-smalltalk strategies-tagging: Removed rerased functionality completely, since it is only hindering the trace optimizer. Message-ID: <20140321102733.125FE1C02EA@cobra.cs.uni-duesseldorf.de> Author: Anton Gulenko Branch: strategies-tagging Changeset: r679:106d0c112fc1 Date: 2014-03-20 16:23 +0100 http://bitbucket.org/pypy/lang-smalltalk/changeset/106d0c112fc1/ Log: Removed rerased functionality completely, since it is only hindering the trace optimizer. diff --git a/spyvm/strategies.py b/spyvm/strategies.py --- a/spyvm/strategies.py +++ b/spyvm/strategies.py @@ -1,7 +1,6 @@ import sys from spyvm import model, shadow -from rpython.rlib import rerased from rpython.rlib.objectmodel import import_from_mixin # Disables all optimized strategies, for debugging. @@ -30,13 +29,13 @@ strategy_tag = 'abstract-list' def storage(self, w_obj): - return self.unerase(w_obj.list_storage) + return w_obj.list_storage def set_initial_storage(self, space, w_obj, size): - w_obj.list_storage = self.erase(self.initial_storage(space, size)) + w_obj.list_storage = self.initial_storage(space, size) def set_storage_for_list(self, space, w_obj, collection): - w_obj.list_storage = self.erase(self.storage_for_list(space, collection)) + w_obj.list_storage = self.storage_for_list(space, collection) def set_storage_copied_from(self, space, w_obj, w_source_obj, reuse_storage=False): - w_obj.list_storage = self.erase(self.copy_storage_from(space, w_source_obj, reuse_storage)) + w_obj.list_storage = self.copy_storage_from(space, w_source_obj, reuse_storage) def initial_storage(self, space, size): raise NotImplementedError("Abstract base class") @@ -54,13 +53,13 @@ strategy_tag = 'abstract-int' def storage(self, w_obj): - return self.unerase(w_obj.int_storage) + return w_obj.int_storage def set_initial_storage(self, space, w_obj, size): - w_obj.int_storage = self.erase(self.initial_storage(space, size)) + w_obj.int_storage = self.initial_storage(space, size) def set_storage_for_list(self, space, w_obj, collection): - w_obj.int_storage = self.erase(self.storage_for_list(space, collection)) + w_obj.int_storage = self.storage_for_list(space, collection) def set_storage_copied_from(self, space, w_obj, w_source_obj, reuse_storage=False): - w_obj.int_storage = self.erase(self.copy_storage_from(space, w_source_obj, reuse_storage)) + w_obj.int_storage = self.copy_storage_from(space, w_source_obj, reuse_storage) def initial_storage(self, space, size): raise NotImplementedError("Abstract base class") @@ -80,21 +79,11 @@ result.singleton = result() return result -use_rerased = False -def setup_rerased_pair(): - # Small piece of metaprogramming stolen from rpython.rlib.objectmodel.import_from_mixin - cls = sys._getframe(1).f_locals - if use_rerased: - cls["erase"], cls["unerase"] = rerased.new_static_erasing_pair("strategy-%s" % cls["strategy_tag"]) - else: - cls["erase"], cls["unerase"] = lambda self, x: x, lambda self, x: x - # this is the typical "initial" storage strategy, for when every slot # in an object is still nil. No storage is allocated. class AllNilStorageStrategy(AbstractStorageStrategy): __metaclass__ = SingletonMeta strategy_tag = 'allnil' - setup_rerased_pair() def fetch(self, space, w_obj, n0): return model.w_nil @@ -121,7 +110,6 @@ class ListStorageStrategy(AbstractListStorageStrategy): __metaclass__ = SingletonMeta strategy_tag = 'list' - setup_rerased_pair() def fetch(self, space, w_obj, n0): return self.storage(w_obj)[n0] @@ -139,7 +127,6 @@ class TaggingSmallIntegerStorageStrategy(AbstractIntStorageStrategy): __metaclass__ = SingletonMeta strategy_tag = 'tagging-smallint' - setup_rerased_pair() needs_objspace = True @staticmethod From noreply at buildbot.pypy.org Fri Mar 21 11:27:34 2014 From: noreply at buildbot.pypy.org (anton_gulenko) Date: Fri, 21 Mar 2014 11:27:34 +0100 (CET) Subject: [pypy-commit] lang-smalltalk strategies-tagging: Generalized TaggingStrategy to ValueOrNilStrategy, added strategy for floats. Message-ID: <20140321102734.31ED71C02EA@cobra.cs.uni-duesseldorf.de> Author: Anton Gulenko Branch: strategies-tagging Changeset: r680:74e7f1232b64 Date: 2014-03-20 17:37 +0100 http://bitbucket.org/pypy/lang-smalltalk/changeset/74e7f1232b64/ Log: Generalized TaggingStrategy to ValueOrNilStrategy, added strategy for floats. Small renamings and refactorings. diff --git a/spyvm/model.py b/spyvm/model.py --- a/spyvm/model.py +++ b/spyvm/model.py @@ -644,10 +644,10 @@ @jit.unroll_safe def __init__(self, space, w_class, size): - from spyvm.strategies import strategy_of_size + from spyvm.strategies import empty_strategy """Create new object with size = fixed + variable size.""" W_AbstractPointersObject.__init__(self, space, w_class, size) - self.strategy = strategy_of_size(self.s_class, size) + self.strategy = empty_strategy(self.s_class) self.initialize_storage(space, size) self.log_strategy_operation("Initialized") diff --git a/spyvm/strategies.py b/spyvm/strategies.py --- a/spyvm/strategies.py +++ b/spyvm/strategies.py @@ -1,7 +1,8 @@ import sys -from spyvm import model, shadow +from spyvm import model, shadow, constants from rpython.rlib.objectmodel import import_from_mixin +from rpython.rlib.rfloat import string_to_float # Disables all optimized strategies, for debugging. only_list_storage = False @@ -20,9 +21,17 @@ def set_storage_copied_from(self, space, w_obj, w_source_obj, reuse_storage=False): raise NotImplementedError("Abstract base class") + def store(self, space, w_obj, n0, w_val): + if self.can_contain(space, w_val): + self.do_store(space, w_obj, n0, w_val) + new_strategy = find_strategy_for_object(w_val) + return w_obj.store_with_new_strategy(space, new_strategy, n0, w_val) + + def can_contain(self, space, w_val): + raise NotImplementedError("Abstract base class") def fetch(self, space, w_obj, n0): raise NotImplementedError("Abstract base class") - def store(self, space, w_obj, n0, w_val): + def do_store(self, space, w_obj, n0, w_val): raise NotImplementedError("Abstract base class") class AbstractListStorageStrategy(AbstractStorageStrategy): @@ -85,17 +94,12 @@ __metaclass__ = SingletonMeta strategy_tag = 'allnil' + def can_contain(self, space, w_obj): + return w_obj == model.w_nil def fetch(self, space, w_obj, n0): return model.w_nil - - def store(self, space, w_obj, n0, w_val): - # This is an important moment, where we decide where to go on the first non-nil store. - if w_val == model.w_nil: - return - if not only_list_storage: - if TaggingSmallIntegerStorageStrategy.can_contain(w_val): - return w_obj.store_with_new_strategy(space, TaggingSmallIntegerStorageStrategy.singleton, n0, w_val) - return w_obj.store_with_new_strategy(space, ListStorageStrategy.singleton, n0, w_val) + def do_store(self, space, w_obj, n0, w_val): + pass def set_initial_storage(self, space, w_obj, size): pass @@ -111,9 +115,11 @@ __metaclass__ = SingletonMeta strategy_tag = 'list' + def can_contain(self, space, w_val): + return True def fetch(self, space, w_obj, n0): return self.storage(w_obj)[n0] - def store(self, space, w_obj, n0, w_val): + def do_store(self, space, w_obj, n0, w_val): # TODO enable generalization by maintaining a counter of elements that are nil. self.storage(w_obj)[n0] = w_val def initial_storage(self, space, size): @@ -124,56 +130,90 @@ length = w_obj.basic_size() return [w_obj.strategy.fetch(space, w_obj, i) for i in range(length)] -class TaggingSmallIntegerStorageStrategy(AbstractIntStorageStrategy): - __metaclass__ = SingletonMeta - strategy_tag = 'tagging-smallint' +class AbstractValueOrNilStorageStrategy(AbstractIntStorageStrategy): needs_objspace = True + strategy_tag = 'abstract-valueOrNil' - @staticmethod - def wrap(val): - return val << 1 - @staticmethod - def unwrap(val): - return val >> 1 - @staticmethod - def can_contain(w_val): - return isinstance(w_val, model.W_SmallInteger) - # TODO - use just a single value to represent nil (max_int-1) - # Then, turn wrap/unwrap into noops - # also store W_LargePositiveInteger1Word? - nil_value = 1 + def can_contain(self, space, w_val): + return w_val == model.w_nil or (isinstance(w_val, self.wrapper_class) and self.unwrap(space, w_val) != self.nil_value) def fetch(self, space, w_obj, n0): val = self.storage(w_obj)[n0] if val == self.nil_value: return space.w_nil else: - return space.wrap_int(self.unwrap(val)) + return self.wrap(space, val) - def store(self, space, w_obj, n0, w_val): + def do_store(self, space, w_obj, n0, w_val): store = self.storage(w_obj) - if self.can_contain(w_val): - store[n0] = self.wrap(space.unwrap_int(w_val)) + if w_val == space.w_nil: + store[n0] = self.nil_value else: - if w_val == space.w_nil: - # TODO - generelize to AllNilStorage by maintaining a counter of nil-elements - store[n0] = self.nil_value - else: - # Storing a wrong type - dehomogenize to ListStorage - return w_obj.store_with_new_strategy(space, ListStorageStrategy.singleton, n0, w_val) - + store[n0] = self.unwrap(space, w_val) + def initial_storage(self, space, size): return [self.nil_value] * size - + def storage_for_list(self, space, collection): length = len(collection) - store = [self.nil_value] * length + store = self.initial_storage(length) for i in range(length): if collection[i] != space.w_nil: - store[i] = self.wrap(space.unwrap_int(collection[i])) + store[i] = self.unwrap(space, collection[i]) return store -def strategy_of_size(s_containing_class, size): +class SmallIntegerOrNilStorageStrategy(AbstractValueOrNilStorageStrategy): + __metaclass__ = SingletonMeta + strategy_tag = 'float-orNil' + nil_value = constants.MAXINT + wrapper_class = model.W_SmallInteger + def wrap(self, space, val): return space.wrap_int(val) + def unwrap(self, space, w_val): return space.unwrap_int(w_val) + +class FloatOrNilStorageStrategy(AbstractValueOrNilStorageStrategy): + __metaclass__ = SingletonMeta + strategy_tag = 'smallint-orNil' + nil_value = string_to_float("-nan") + wrapper_class = model.W_Float + def wrap(self, space, val): return space.wrap_float(val) + def unwrap(self, space, w_val): return space.unwrap_float(w_val) + +def find_strategy_for_object(space, var): + return find_strategy_for_objects(space, [var]) + +def find_strategy_for_objects(space, vars): + if only_list_storage: + ListStorageStrategy.singleton + + specialized_strategies = 3 + all_nil_can_handle = True + small_int_can_handle = True + float_can_handle = True + for w_obj in vars: + if all_nil_can_handle and not AllNilStorageStrategy.singleton.can_contain(space, w_obj): + all_nil_can_handle = False + specialized_strategies = specialized_strategies - 1 + if small_int_can_handle and not SmallIntegerOrNilStorageStrategy.singleton.can_contain(space, w_obj): + small_int_can_handle = False + specialized_strategies = specialized_strategies - 1 + if float_can_handle and not FloatOrNilStorageStrategy.singleton.can_contain(space, w_obj): + float_can_handle = False + specialized_strategies = specialized_strategies - 1 + + if specialized_strategies <= 0: + return ListStorageStrategy.singleton + + if all_nil_can_handle: + return AllNilStorageStrategy.singleton + if small_int_can_handle: + return SmallIntegerOrNilStorageStrategy.singleton + if float_can_handle: + return FloatOrNilStorageStrategy.singleton + + # If this happens, please look for a bug in the code above. + assert False, "No strategy could be found for list %r" % vars + +def empty_strategy(s_containing_class): if s_containing_class is None: # This is a weird and rare special case for w_nil return ListStorageStrategy.singleton @@ -194,19 +234,7 @@ # Ths class object shadows are not yet synchronized. return ListStorageStrategy.singleton - if not is_variable or only_list_storage: + if is_variable: + return find_strategy_for_objects(s_containing_class.space, vars) + else: return ListStorageStrategy.singleton - - is_all_nils = True - for w_obj in vars: - if w_obj != model.w_nil: - is_all_nils = False - if not TaggingSmallIntegerStorageStrategy.can_contain(w_obj): - # TODO -- here we can still optimize if there is only - # one single type in the collection. - return ListStorageStrategy.singleton - if is_all_nils: - return AllNilStorageStrategy.singleton - else: - return TaggingSmallIntegerStorageStrategy.singleton - \ No newline at end of file From noreply at buildbot.pypy.org Fri Mar 21 11:27:35 2014 From: noreply at buildbot.pypy.org (anton_gulenko) Date: Fri, 21 Mar 2014 11:27:35 +0100 (CET) Subject: [pypy-commit] lang-smalltalk strategies-tagging: Fixes Message-ID: <20140321102735.54FBF1C02EA@cobra.cs.uni-duesseldorf.de> Author: Anton Gulenko Branch: strategies-tagging Changeset: r681:bb282722e3e4 Date: 2014-03-20 17:41 +0100 http://bitbucket.org/pypy/lang-smalltalk/changeset/bb282722e3e4/ Log: Fixes diff --git a/spyvm/strategies.py b/spyvm/strategies.py --- a/spyvm/strategies.py +++ b/spyvm/strategies.py @@ -23,8 +23,8 @@ def store(self, space, w_obj, n0, w_val): if self.can_contain(space, w_val): - self.do_store(space, w_obj, n0, w_val) - new_strategy = find_strategy_for_object(w_val) + return self.do_store(space, w_obj, n0, w_val) + new_strategy = find_strategy_for_object(space, w_val) return w_obj.store_with_new_strategy(space, new_strategy, n0, w_val) def can_contain(self, space, w_val): @@ -146,7 +146,7 @@ def do_store(self, space, w_obj, n0, w_val): store = self.storage(w_obj) - if w_val == space.w_nil: + if w_val == model.w_nil: store[n0] = self.nil_value else: store[n0] = self.unwrap(space, w_val) @@ -156,9 +156,9 @@ def storage_for_list(self, space, collection): length = len(collection) - store = self.initial_storage(length) + store = self.initial_storage(space, length) for i in range(length): - if collection[i] != space.w_nil: + if collection[i] != model.w_nil: store[i] = self.unwrap(space, collection[i]) return store From noreply at buildbot.pypy.org Fri Mar 21 11:27:36 2014 From: noreply at buildbot.pypy.org (anton_gulenko) Date: Fri, 21 Mar 2014 11:27:36 +0100 (CET) Subject: [pypy-commit] lang-smalltalk strategies-tagging: Using a list of floats to store either float or int values. Message-ID: <20140321102736.7CF601C02EA@cobra.cs.uni-duesseldorf.de> Author: Anton Gulenko Branch: strategies-tagging Changeset: r682:efb3ddfef529 Date: 2014-03-20 19:14 +0100 http://bitbucket.org/pypy/lang-smalltalk/changeset/efb3ddfef529/ Log: Using a list of floats to store either float or int values. diff --git a/spyvm/strategies.py b/spyvm/strategies.py --- a/spyvm/strategies.py +++ b/spyvm/strategies.py @@ -1,6 +1,8 @@ -import sys +import sys, math from spyvm import model, shadow, constants +from rpython.rlib import longlong2float, rarithmetic +from rpython.rtyper.lltypesystem import rffi, lltype from rpython.rlib.objectmodel import import_from_mixin from rpython.rlib.rfloat import string_to_float @@ -133,13 +135,21 @@ class AbstractValueOrNilStorageStrategy(AbstractIntStorageStrategy): needs_objspace = True strategy_tag = 'abstract-valueOrNil' + # TODO -- use another value... something like max_float? + nil_value = string_to_float("nan") + + def is_nil_value(self, val): + # return val == self.nil_value + return math.isnan(val) def can_contain(self, space, w_val): - return w_val == model.w_nil or (isinstance(w_val, self.wrapper_class) and self.unwrap(space, w_val) != self.nil_value) + return w_val == model.w_nil or \ + (isinstance(w_val, self.wrapper_class) \ + and not self.is_nil_value(self.unwrap(space, w_val))) def fetch(self, space, w_obj, n0): val = self.storage(w_obj)[n0] - if val == self.nil_value: + if self.is_nil_value(val): return space.w_nil else: return self.wrap(space, val) @@ -147,7 +157,7 @@ def do_store(self, space, w_obj, n0, w_val): store = self.storage(w_obj) if w_val == model.w_nil: - store[n0] = self.nil_value + store[n0] = self.nil_value else: store[n0] = self.unwrap(space, w_val) @@ -162,21 +172,32 @@ store[i] = self.unwrap(space, collection[i]) return store +def _int_to_float(int_val): + return longlong2float.longlong2float(rffi.cast(lltype.SignedLongLong, int_val)) + class SmallIntegerOrNilStorageStrategy(AbstractValueOrNilStorageStrategy): __metaclass__ = SingletonMeta - strategy_tag = 'float-orNil' - nil_value = constants.MAXINT + strategy_tag = 'smallint-orNil' wrapper_class = model.W_SmallInteger - def wrap(self, space, val): return space.wrap_int(val) - def unwrap(self, space, w_val): return space.unwrap_int(w_val) + + def wrap(self, space, val): + int_val = rarithmetic.intmask(longlong2float.float2longlong(val)) + return space.wrap_int(int_val) + def unwrap(self, space, w_val): + assert isinstance(w_val, model.W_SmallInteger) + int_val = space.unwrap_int(w_val) + return _int_to_float(int_val) class FloatOrNilStorageStrategy(AbstractValueOrNilStorageStrategy): __metaclass__ = SingletonMeta - strategy_tag = 'smallint-orNil' - nil_value = string_to_float("-nan") + strategy_tag = 'float-orNil' wrapper_class = model.W_Float - def wrap(self, space, val): return space.wrap_float(val) - def unwrap(self, space, w_val): return space.unwrap_float(w_val) + + def wrap(self, space, val): + return space.wrap_float(val) + def unwrap(self, space, w_val): + assert isinstance(w_val, model.W_Float) + return space.unwrap_float(w_val) def find_strategy_for_object(space, var): return find_strategy_for_objects(space, [var]) @@ -211,7 +232,7 @@ return FloatOrNilStorageStrategy.singleton # If this happens, please look for a bug in the code above. - assert False, "No strategy could be found for list %r" % vars + assert False, "No strategy could be found for list..." def empty_strategy(s_containing_class): if s_containing_class is None: From noreply at buildbot.pypy.org Fri Mar 21 11:27:37 2014 From: noreply at buildbot.pypy.org (anton_gulenko) Date: Fri, 21 Mar 2014 11:27:37 +0100 (CET) Subject: [pypy-commit] lang-smalltalk strategies-tagging: Fixed test_strategies.py, added tests for FloatStrategy. Message-ID: <20140321102737.8D73B1C02EA@cobra.cs.uni-duesseldorf.de> Author: Anton Gulenko Branch: strategies-tagging Changeset: r683:818054edd715 Date: 2014-03-20 19:36 +0100 http://bitbucket.org/pypy/lang-smalltalk/changeset/818054edd715/ Log: Fixed test_strategies.py, added tests for FloatStrategy. diff --git a/spyvm/strategies.py b/spyvm/strategies.py --- a/spyvm/strategies.py +++ b/spyvm/strategies.py @@ -26,7 +26,7 @@ def store(self, space, w_obj, n0, w_val): if self.can_contain(space, w_val): return self.do_store(space, w_obj, n0, w_val) - new_strategy = find_strategy_for_object(space, w_val) + new_strategy = find_strategy_for_objects(space, [w_val]) return w_obj.store_with_new_strategy(space, new_strategy, n0, w_val) def can_contain(self, space, w_val): @@ -199,9 +199,6 @@ assert isinstance(w_val, model.W_Float) return space.unwrap_float(w_val) -def find_strategy_for_object(space, var): - return find_strategy_for_objects(space, [var]) - def find_strategy_for_objects(space, vars): if only_list_storage: ListStorageStrategy.singleton diff --git a/spyvm/test/test_strategies.py b/spyvm/test/test_strategies.py --- a/spyvm/test/test_strategies.py +++ b/spyvm/test/test_strategies.py @@ -15,24 +15,29 @@ a.store(space, 0, arr(1)) return a -def tagging_arr(size): +def int_arr(size): a = arr(size) a.store(space, 0, space.wrap_int(12)) return a -def tagging_arr_odd(size): +def float_arr(size): a = arr(size) - a.store(space, 2, space.wrap_int(12)) + a.store(space, 0, space.wrap_float(1.2)) return a def check_arr(arr, expected): for i in range(arr.basic_size()): + w_val = arr.fetch(space, i) if expected[i] == w_nil: - assert arr.fetch(space, i) == w_nil - else: - w_val = arr.fetch(space, i) + assert w_val == w_nil + elif isinstance(expected[i], int): assert isinstance(w_val, model.W_SmallInteger) assert space.unwrap_int(w_val) == expected[i] + elif isinstance(expected[i], float): + assert isinstance(w_val, model.W_Float) + assert space.unwrap_float(w_val) == expected[i] + else: + assert False, "Unexpected array of expected values." # ====== AllNil StorageStrategy @@ -76,41 +81,82 @@ a.store(space, 1, arr(1)) assert a.basic_size() == 5 -# ====== Tagging SmallInteger StorageStrategy +# ====== SmallIntegerOrNil StorageStrategy def test_AllNil_to_Int(): - a = tagging_arr(5) - assert isinstance(a.strategy, strategies.TaggingSmallIntegerStorageStrategy) + a = int_arr(5) + assert isinstance(a.strategy, strategies.SmallIntegerOrNilStorageStrategy) check_arr(a, [12, w_nil, w_nil, w_nil, w_nil]) -def test_Tagging_store(): - a = tagging_arr(5) +def test_SmallInt_store(): + a = int_arr(5) a.store(space, 1, space.wrap_int(20)) a.store(space, 2, space.wrap_int(20)) - assert isinstance(a.strategy, strategies.TaggingSmallIntegerStorageStrategy) + assert isinstance(a.strategy, strategies.SmallIntegerOrNilStorageStrategy) check_arr(a, [12, 20, 20, w_nil, w_nil]) -def test_Tagging_store_nil_to_nil(): - a = tagging_arr_odd(5) +def test_SmallInt_store_nil_to_nil(): + a = int_arr(5) a.store(space, 1, w_nil) - check_arr(a, [w_nil, w_nil, 12, w_nil, w_nil]) + check_arr(a, [12, w_nil, w_nil, w_nil, w_nil]) -def test_Tagging_delete(): - a = tagging_arr_odd(5) +def test_SmallInt_overwrite(): + a = int_arr(5) a.store(space, 1, space.wrap_int(1)) a.store(space, 3, space.wrap_int(2)) - a.store(space, 2, space.wrap_int(100)) + a.store(space, 0, space.wrap_int(100)) a.store(space, 1, space.wrap_int(200)) a.store(space, 3, space.wrap_int(300)) - check_arr(a, [w_nil, 200, 100, 300, w_nil]) + check_arr(a, [100, 200, w_nil, 300, w_nil]) -def test_Tagging_delete_first(): - a = tagging_arr_odd(5) +def test_SmallInt_delete(): + a = int_arr(5) a.store(space, 1, space.wrap_int(1)) a.store(space, 1, w_nil) - check_arr(a, [w_nil, w_nil, 12, w_nil, w_nil]) + check_arr(a, [12, w_nil, w_nil, w_nil, w_nil]) -def test_Tagging_to_List(): - a = tagging_arr_odd(5) +def test_SmallInt_to_List(): + a = int_arr(5) a.store(space, 1, arr(1)) assert isinstance(a.strategy, strategies.ListStorageStrategy) + +# ====== FloatOrNil StorageStrategy + +def test_AllNil_to_Float(): + a = float_arr(5) + assert isinstance(a.strategy, strategies.FloatOrNilStorageStrategy) + check_arr(a, [1.2, w_nil, w_nil, w_nil, w_nil]) + +def test_Float_store(): + a = float_arr(5) + a.store(space, 1, space.wrap_float(20.0)) + a.store(space, 2, space.wrap_float(20.0)) + assert isinstance(a.strategy, strategies.FloatOrNilStorageStrategy) + check_arr(a, [1.2, 20.0, 20.0, w_nil, w_nil]) + +def test_Float_store_nil_to_nil(): + a = float_arr(5) + a.store(space, 1, w_nil) + check_arr(a, [1.2, w_nil, w_nil, w_nil, w_nil]) + +def test_Float_overwrite(): + a = float_arr(5) + a.store(space, 1, space.wrap_float(1.0)) + a.store(space, 3, space.wrap_float(2.0)) + a.store(space, 0, space.wrap_float(100.0)) + a.store(space, 1, space.wrap_float(200.0)) + a.store(space, 3, space.wrap_float(300.0)) + check_arr(a, [100.0, 200.0, w_nil, 300.0, w_nil]) + +def test_Float_delete(): + a = float_arr(5) + a.store(space, 1, space.wrap_float(1)) + a.store(space, 1, w_nil) + check_arr(a, [1.2, w_nil, w_nil, w_nil, w_nil]) + +def test_Float_to_List(): + a = float_arr(5) + a.store(space, 1, arr(1)) + assert isinstance(a.strategy, strategies.ListStorageStrategy) + + \ No newline at end of file From noreply at buildbot.pypy.org Fri Mar 21 11:27:38 2014 From: noreply at buildbot.pypy.org (anton_gulenko) Date: Fri, 21 Mar 2014 11:27:38 +0100 (CET) Subject: [pypy-commit] lang-smalltalk strategies-tagging: Using an artificial float value as sentinel for nil in a collection. Message-ID: <20140321102738.A645F1C02EA@cobra.cs.uni-duesseldorf.de> Author: Anton Gulenko Branch: strategies-tagging Changeset: r684:91e7b60922ab Date: 2014-03-21 11:27 +0100 http://bitbucket.org/pypy/lang-smalltalk/changeset/91e7b60922ab/ Log: Using an artificial float value as sentinel for nil in a collection. diff --git a/spyvm/strategies.py b/spyvm/strategies.py --- a/spyvm/strategies.py +++ b/spyvm/strategies.py @@ -2,6 +2,7 @@ import sys, math from spyvm import model, shadow, constants from rpython.rlib import longlong2float, rarithmetic +from rpython.rlib.rstruct.runpack import runpack from rpython.rtyper.lltypesystem import rffi, lltype from rpython.rlib.objectmodel import import_from_mixin from rpython.rlib.rfloat import string_to_float @@ -136,11 +137,11 @@ needs_objspace = True strategy_tag = 'abstract-valueOrNil' # TODO -- use another value... something like max_float? - nil_value = string_to_float("nan") + nil_value = runpack("\x10\x00\x00\x00\x00\x00\xf8\x7f") + nil_value_longlong = long2floatfloat.float2longlong(nil_value) def is_nil_value(self, val): - # return val == self.nil_value - return math.isnan(val) + return long2floatfloat.float2longlong(val) == self.nil_value_longlong def can_contain(self, space, w_val): return w_val == model.w_nil or \ From noreply at buildbot.pypy.org Fri Mar 21 11:53:06 2014 From: noreply at buildbot.pypy.org (anton_gulenko) Date: Fri, 21 Mar 2014 11:53:06 +0100 (CET) Subject: [pypy-commit] lang-smalltalk strategies-tagging: Fix Message-ID: <20140321105306.E2AF41C02EA@cobra.cs.uni-duesseldorf.de> Author: Anton Gulenko Branch: strategies-tagging Changeset: r685:0b31925287c6 Date: 2014-03-21 11:28 +0100 http://bitbucket.org/pypy/lang-smalltalk/changeset/0b31925287c6/ Log: Fix diff --git a/spyvm/strategies.py b/spyvm/strategies.py --- a/spyvm/strategies.py +++ b/spyvm/strategies.py @@ -137,7 +137,7 @@ needs_objspace = True strategy_tag = 'abstract-valueOrNil' # TODO -- use another value... something like max_float? - nil_value = runpack("\x10\x00\x00\x00\x00\x00\xf8\x7f") + nil_value = runpack("d", "\x10\x00\x00\x00\x00\x00\xf8\x7f") nil_value_longlong = long2floatfloat.float2longlong(nil_value) def is_nil_value(self, val): From noreply at buildbot.pypy.org Fri Mar 21 11:53:08 2014 From: noreply at buildbot.pypy.org (anton_gulenko) Date: Fri, 21 Mar 2014 11:53:08 +0100 (CET) Subject: [pypy-commit] lang-smalltalk strategies-tagging: Fixed typo Message-ID: <20140321105308.10B631C02EA@cobra.cs.uni-duesseldorf.de> Author: Anton Gulenko Branch: strategies-tagging Changeset: r686:615f424d5684 Date: 2014-03-21 11:30 +0100 http://bitbucket.org/pypy/lang-smalltalk/changeset/615f424d5684/ Log: Fixed typo diff --git a/spyvm/strategies.py b/spyvm/strategies.py --- a/spyvm/strategies.py +++ b/spyvm/strategies.py @@ -138,10 +138,10 @@ strategy_tag = 'abstract-valueOrNil' # TODO -- use another value... something like max_float? nil_value = runpack("d", "\x10\x00\x00\x00\x00\x00\xf8\x7f") - nil_value_longlong = long2floatfloat.float2longlong(nil_value) + nil_value_longlong = longlong2float.float2longlong(nil_value) def is_nil_value(self, val): - return long2floatfloat.float2longlong(val) == self.nil_value_longlong + return longlong2float.float2longlong(val) == self.nil_value_longlong def can_contain(self, space, w_val): return w_val == model.w_nil or \ From noreply at buildbot.pypy.org Fri Mar 21 11:53:09 2014 From: noreply at buildbot.pypy.org (anton_gulenko) Date: Fri, 21 Mar 2014 11:53:09 +0100 (CET) Subject: [pypy-commit] lang-smalltalk strategies-tagging: Fixed a test, marked a failing test as skipped.. Message-ID: <20140321105309.228B51C02EA@cobra.cs.uni-duesseldorf.de> Author: Anton Gulenko Branch: strategies-tagging Changeset: r687:f13ab96da4bb Date: 2014-03-21 11:52 +0100 http://bitbucket.org/pypy/lang-smalltalk/changeset/f13ab96da4bb/ Log: Fixed a test, marked a failing test as skipped.. diff --git a/spyvm/test/test_model.py b/spyvm/test/test_model.py --- a/spyvm/test/test_model.py +++ b/spyvm/test/test_model.py @@ -353,6 +353,7 @@ assert target.getword(0) == 0xffff0100 assert target.getword(1) == 0x7fff8000 + at py.test.mark.skipif("'This test must be fixed!'") def test_display_bitmap(): # XXX: Patch SDLDisplay -> get_pixelbuffer() to circumvent # double-free bug diff --git a/spyvm/test/test_strategies.py b/spyvm/test/test_strategies.py --- a/spyvm/test/test_strategies.py +++ b/spyvm/test/test_strategies.py @@ -150,7 +150,7 @@ def test_Float_delete(): a = float_arr(5) - a.store(space, 1, space.wrap_float(1)) + a.store(space, 1, space.wrap_float(1.0)) a.store(space, 1, w_nil) check_arr(a, [1.2, w_nil, w_nil, w_nil, w_nil]) From noreply at buildbot.pypy.org Fri Mar 21 12:07:30 2014 From: noreply at buildbot.pypy.org (anton_gulenko) Date: Fri, 21 Mar 2014 12:07:30 +0100 (CET) Subject: [pypy-commit] lang-smalltalk strategies-tagging: Fixed bug in strategies implementation and added test. Message-ID: <20140321110730.CA7931C13AD@cobra.cs.uni-duesseldorf.de> Author: Anton Gulenko Branch: strategies-tagging Changeset: r688:98f32dad569e Date: 2014-03-21 12:07 +0100 http://bitbucket.org/pypy/lang-smalltalk/changeset/98f32dad569e/ Log: Fixed bug in strategies implementation and added test. diff --git a/spyvm/strategies.py b/spyvm/strategies.py --- a/spyvm/strategies.py +++ b/spyvm/strategies.py @@ -27,9 +27,11 @@ def store(self, space, w_obj, n0, w_val): if self.can_contain(space, w_val): return self.do_store(space, w_obj, n0, w_val) - new_strategy = find_strategy_for_objects(space, [w_val]) + new_strategy = self.generelized_strategy_for(space, w_val) return w_obj.store_with_new_strategy(space, new_strategy, n0, w_val) + def generelized_strategy_for(self, space, w_val): + raise NotImplementedError("Abstract base class") def can_contain(self, space, w_val): raise NotImplementedError("Abstract base class") def fetch(self, space, w_obj, n0): @@ -73,6 +75,8 @@ def set_storage_copied_from(self, space, w_obj, w_source_obj, reuse_storage=False): w_obj.int_storage = self.copy_storage_from(space, w_source_obj, reuse_storage) + def generelized_strategy_for(self, space, w_val): + return ListStorageStrategy.singleton def initial_storage(self, space, size): raise NotImplementedError("Abstract base class") def storage_for_list(self, space, collection): @@ -104,6 +108,8 @@ def do_store(self, space, w_obj, n0, w_val): pass + def generelized_strategy_for(self, space, w_val): + return find_strategy_for_objects(space, [w_val]) def set_initial_storage(self, space, w_obj, size): pass def set_storage_for_list(self, space, w_obj, collection): diff --git a/spyvm/test/test_strategies.py b/spyvm/test/test_strategies.py --- a/spyvm/test/test_strategies.py +++ b/spyvm/test/test_strategies.py @@ -120,6 +120,12 @@ a.store(space, 1, arr(1)) assert isinstance(a.strategy, strategies.ListStorageStrategy) +def test_SmallInt_store_Float_to_List(): + a = int_arr(5) + a.store(space, 1, space.wrap_float(2.2)) + assert isinstance(a.strategy, strategies.ListStorageStrategy) + check_arr(a, [12, 2.2, w_nil, w_nil, w_nil]) + # ====== FloatOrNil StorageStrategy def test_AllNil_to_Float(): @@ -159,4 +165,9 @@ a.store(space, 1, arr(1)) assert isinstance(a.strategy, strategies.ListStorageStrategy) +def test_Float_store_SmallInt_to_List(): + a = float_arr(5) + a.store(space, 1, space.wrap_int(2)) + assert isinstance(a.strategy, strategies.ListStorageStrategy) + check_arr(a, [1.2, 2, w_nil, w_nil, w_nil]) \ No newline at end of file From noreply at buildbot.pypy.org Fri Mar 21 12:12:47 2014 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 21 Mar 2014 12:12:47 +0100 (CET) Subject: [pypy-commit] pypy resume-refactor: some missing commit Message-ID: <20140321111247.33C611C029E@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: resume-refactor Changeset: r70145:8f7d82dde2ed Date: 2014-03-21 12:35 +0200 http://bitbucket.org/pypy/pypy/changeset/8f7d82dde2ed/ Log: some missing commit diff --git a/TODO b/TODO --- a/TODO +++ b/TODO @@ -7,4 +7,4 @@ * do escape analysis in the resume/optimizer.py -* make_a_counter_per_value got screwed, but a bit no clue what it does +* reimplement make_a_counter_per_value diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py --- a/rpython/jit/metainterp/compile.py +++ b/rpython/jit/metainterp/compile.py @@ -498,9 +498,8 @@ self.guard_opnum = guard_op.getopnum() def make_a_counter_per_value(self, guard_value_op): + return # XXXX assert guard_value_op.getopnum() == rop.GUARD_VALUE - # XXX I have no clue what exactly it does, but we killed failargs - # so i is always 0 now box = guard_value_op.getarg(0) i = 0 # used to be i = guard_value_op.getfailargs().index(box) @@ -557,6 +556,7 @@ else: # we have a GUARD_VALUE that fails. Make a _counters instance # (only now, when the guard is actually failing at least once), # and use it to record some statistics about the failing values. + assert 0 # XXX this should be dead code until we fix it index = self._counter & self.CNT_BASE_MASK typetag = self._counter & self.CNT_TYPE_MASK counters = self._counters diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py --- a/rpython/jit/metainterp/optimizeopt/optimizer.py +++ b/rpython/jit/metainterp/optimizeopt/optimizer.py @@ -568,10 +568,6 @@ raise ValueError, "invalid optimization" self.seen_results[op.result] = None self._newoperations.append(op) - if (self.delayed_resume_put is not None and - self.delayed_resume_put.getarg(0) is op.result): - self._newoperations.append(self.delayed_resume_put) - self.delayed_resume_put = None def replace_op(self, old_op, new_op): # XXX: Do we want to cache indexes to prevent search? diff --git a/rpython/jit/metainterp/optimizeopt/resume.py b/rpython/jit/metainterp/optimizeopt/resume.py --- a/rpython/jit/metainterp/optimizeopt/resume.py +++ b/rpython/jit/metainterp/optimizeopt/resume.py @@ -1,6 +1,7 @@ from rpython.jit.metainterp.optimizeopt import optimizer from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method +from rpython.jit.metainterp.history import Const """ All of this directly emit the ops, without calling emit_operation (they also don't have boxes except a resume_put) @@ -8,9 +9,12 @@ class OptResume(optimizer.Optimization): def optimize_RESUME_PUT(self, op): - if op.getarg(0) in self.optimizer.producer: + arg = op.getarg(0) + if (isinstance(arg, Const) or arg in self.optimizer.producer or + arg in self.optimizer.loop.inputargs): self.optimizer.resumebuilder.resume_put(op) else: + xxx self.optimizer.delayed_resume_put = op # otherwise we did not emit the operation just yet diff --git a/rpython/jit/resume/optimizer.py b/rpython/jit/resume/optimizer.py --- a/rpython/jit/resume/optimizer.py +++ b/rpython/jit/resume/optimizer.py @@ -49,6 +49,7 @@ no = op.getarg(2).getint() self.framestack[op.getarg(1).getint()].values[no] = value else: + XXX self.opt.emit_operation(op) def new_virtual_with_vtable(self, box, vtable, vvalue): diff --git a/rpython/jit/resume/test/support.py b/rpython/jit/resume/test/support.py new file mode 100644 --- /dev/null +++ b/rpython/jit/resume/test/support.py @@ -0,0 +1,5 @@ + +class MockStaticData(object): + def __init__(self, jitcodes, descrs): + self.alljitcodes = jitcodes + self.opcode_descrs = descrs From noreply at buildbot.pypy.org Fri Mar 21 12:12:48 2014 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 21 Mar 2014 12:12:48 +0100 (CET) Subject: [pypy-commit] pypy default: fix Message-ID: <20140321111248.885A71C029E@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r70146:345c69c621c5 Date: 2014-03-21 13:11 +0200 http://bitbucket.org/pypy/pypy/changeset/345c69c621c5/ Log: fix diff --git a/rpython/jit/metainterp/logger.py b/rpython/jit/metainterp/logger.py --- a/rpython/jit/metainterp/logger.py +++ b/rpython/jit/metainterp/logger.py @@ -72,6 +72,9 @@ def _make_log_operations(self): return LogOperations(self.metainterp_sd, self.guard_number) + def repr_of_resop(self, op): + return LogOperations(self.metainterp_sd, self.guard_number).repr_of_resop(op) + class LogOperations(object): """ From noreply at buildbot.pypy.org Fri Mar 21 14:26:19 2014 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 21 Mar 2014 14:26:19 +0100 (CET) Subject: [pypy-commit] pypy default: A passing test Message-ID: <20140321132619.CF43F1C02AF@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r70147:9b2113d7bb18 Date: 2014-03-21 14:25 +0100 http://bitbucket.org/pypy/pypy/changeset/9b2113d7bb18/ Log: A passing test diff --git a/rpython/jit/metainterp/test/test_dict.py b/rpython/jit/metainterp/test/test_dict.py --- a/rpython/jit/metainterp/test/test_dict.py +++ b/rpython/jit/metainterp/test/test_dict.py @@ -294,6 +294,35 @@ assert res == f(10) self.check_simple_loop(call=3) + def test_dict_eq_can_release_gil(self): + from rpython.rtyper.lltypesystem import lltype, rffi + if type(self.newdict()) is not dict: + py.test.skip("this is an r_dict test") + T = rffi.CArrayPtr(rffi.TIME_T) + external = rffi.llexternal("time", [T], rffi.TIME_T, releasegil=True) + myjitdriver = JitDriver(greens = [], reds = ['total', 'dct']) + def key(x): + return x % 2 + def eq(x, y): + external(lltype.nullptr(T.TO)) + return (x % 2) == (y % 2) + + def f(n): + dct = objectmodel.r_dict(eq, key) + total = n + x = 44444 + while total: + myjitdriver.jit_merge_point(total=total, dct=dct) + dct[total] = total + x = dct[total] + total -= 1 + return len(dct) + x + + res = self.meta_interp(f, [10], listops=True) + assert res == 2 + 1 + self.check_simple_loop(call_may_force=2, # ll_dict_lookup_trampoline + call=1) # ll_dict_setitem_lookup_done_trampoline + class TestLLtype(DictTests, LLJitMixin): pass From noreply at buildbot.pypy.org Fri Mar 21 14:28:52 2014 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 21 Mar 2014 14:28:52 +0100 (CET) Subject: [pypy-commit] pypy default: Improve the test, showing that the calls to eq() are not optimized Message-ID: <20140321132852.AC3101C029E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r70148:f6f44dc30721 Date: 2014-03-21 14:28 +0100 http://bitbucket.org/pypy/pypy/changeset/f6f44dc30721/ Log: Improve the test, showing that the calls to eq() are not optimized diff --git a/rpython/jit/metainterp/test/test_dict.py b/rpython/jit/metainterp/test/test_dict.py --- a/rpython/jit/metainterp/test/test_dict.py +++ b/rpython/jit/metainterp/test/test_dict.py @@ -311,16 +311,20 @@ dct = objectmodel.r_dict(eq, key) total = n x = 44444 + y = 55555 + z = 66666 while total: myjitdriver.jit_merge_point(total=total, dct=dct) dct[total] = total x = dct[total] + y = dct[total] + z = dct[total] total -= 1 - return len(dct) + x + return len(dct) + x + y + z res = self.meta_interp(f, [10], listops=True) - assert res == 2 + 1 - self.check_simple_loop(call_may_force=2, # ll_dict_lookup_trampoline + assert res == 2 + 1 + 1 + 1 + self.check_simple_loop(call_may_force=4, # ll_dict_lookup_trampoline call=1) # ll_dict_setitem_lookup_done_trampoline From noreply at buildbot.pypy.org Sat Mar 22 00:51:50 2014 From: noreply at buildbot.pypy.org (pjenvey) Date: Sat, 22 Mar 2014 00:51:50 +0100 (CET) Subject: [pypy-commit] pypy py3k: MiniBuffer's API actually differs from MemoryView, fix Message-ID: <20140321235150.A5EA31C029E@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r70149:4f39e0698984 Date: 2014-03-21 16:32 -0700 http://bitbucket.org/pypy/pypy/changeset/4f39e0698984/ Log: MiniBuffer's API actually differs from MemoryView, fix diff --git a/pypy/module/_cffi_backend/cbuffer.py b/pypy/module/_cffi_backend/cbuffer.py --- a/pypy/module/_cffi_backend/cbuffer.py +++ b/pypy/module/_cffi_backend/cbuffer.py @@ -1,9 +1,10 @@ +from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.buffer import RWBuffer from pypy.interpreter.error import oefmt from pypy.interpreter.gateway import unwrap_spec, interp2app from pypy.interpreter.typedef import TypeDef, make_weakref_descr from pypy.module._cffi_backend import cdataobj, ctypeptr, ctypearray -from pypy.objspace.std.memoryview import W_MemoryView +from pypy.objspace.std.memoryview import _buffer_setitem from rpython.rtyper.annlowlevel import llstr from rpython.rtyper.lltypesystem import rffi @@ -41,11 +42,30 @@ # Override the typedef to narrow down the interface that's exposed to app-level -class MiniBuffer(W_MemoryView): +class MiniBuffer(W_Root): def __init__(self, buffer, keepalive=None): - W_MemoryView.__init__(self, buffer) + self.buffer = buffer self.keepalive = keepalive + def buffer_w(self, space): + return self.buffer + + def descr_len(self, space): + return space.wrap(self.buffer.getlength()) + + def descr_getitem(self, space, w_index): + start, stop, step, size = space.decode_index4(w_index, + self.buffer.getlength()) + if step == 0: + return space.wrapbytes(self.buffer.getitem(start)) + res = self.buffer.getslice(start, stop, step, size) + return space.wrapbytes(res) + + @unwrap_spec(newstring='bufferstr') + def descr_setitem(self, space, w_index, newstring): + _buffer_setitem(space, self.buffer, w_index, newstring) + + MiniBuffer.typedef = TypeDef( "buffer", __module__ = "_cffi_backend", From noreply at buildbot.pypy.org Sat Mar 22 00:51:52 2014 From: noreply at buildbot.pypy.org (pjenvey) Date: Sat, 22 Mar 2014 00:51:52 +0100 (CET) Subject: [pypy-commit] pypy py3k: restore our buffer related error message Message-ID: <20140321235152.2AE121C029E@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r70150:707e60600dbe Date: 2014-03-21 16:48 -0700 http://bitbucket.org/pypy/pypy/changeset/707e60600dbe/ Log: restore our buffer related error message diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -199,7 +199,8 @@ w_result = space.get_and_call_function(w_impl, self) if space.isinstance_w(w_result, space.w_memoryview): return w_result.buffer_w(space) - self._typed_unwrap_error(space, "buffer") + raise oefmt(space.w_TypeError, + "'%T' does not support the buffer interface", self) def bytes_w(self, space): self._typed_unwrap_error(space, "bytes") diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py --- a/pypy/module/_socket/test/test_sock_app.py +++ b/pypy/module/_socket/test/test_sock_app.py @@ -536,7 +536,8 @@ skip("GAIError - probably no connection: %s" % str(ex.args)) assert s.send(memoryview(b'')) == 0 assert s.sendall(memoryview(b'')) is None - raises(TypeError, s.send, '') + exc = raises(TypeError, s.send, '') + assert str(exc.value) == "'str' does not support the buffer interface" raises(TypeError, s.sendall, '') s.close() s = _socket.socket(_socket.AF_INET, _socket.SOCK_DGRAM, 0) From noreply at buildbot.pypy.org Sat Mar 22 00:51:53 2014 From: noreply at buildbot.pypy.org (pjenvey) Date: Sat, 22 Mar 2014 00:51:53 +0100 (CET) Subject: [pypy-commit] pypy py3k: this now lives in baseobjspace after the buffer refactor Message-ID: <20140321235153.80C971C029E@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r70151:144c447e88ff Date: 2014-03-21 16:48 -0700 http://bitbucket.org/pypy/pypy/changeset/144c447e88ff/ Log: this now lives in baseobjspace after the buffer refactor diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py --- a/pypy/objspace/descroperation.py +++ b/pypy/objspace/descroperation.py @@ -473,13 +473,6 @@ else: return space.isinstance(w_inst, w_type) - def buffer(space, w_obj): - w_impl = space.lookup(w_obj, '__buffer__') - if w_impl is None: - raise oefmt(space.w_TypeError, - "'%T' does not support the buffer interface", w_obj) - return space.get_and_call_function(w_impl, w_obj) - # helpers From noreply at buildbot.pypy.org Sat Mar 22 08:07:51 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 22 Mar 2014 08:07:51 +0100 (CET) Subject: [pypy-commit] pypy default: Untested: implement increment_debug_counter for ARM Message-ID: <20140322070751.E378A1D2848@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r70152:53058f410b21 Date: 2014-03-22 08:06 +0100 http://bitbucket.org/pypy/pypy/changeset/53058f410b21/ Log: Untested: implement increment_debug_counter for ARM diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py --- a/rpython/jit/backend/arm/opassembler.py +++ b/rpython/jit/backend/arm/opassembler.py @@ -584,7 +584,10 @@ emit_op_getfield_gc_pure = emit_op_getfield_gc def emit_op_increment_debug_counter(self, op, arglocs, regalloc, fcond): - # XXX implement me + base_loc, value_loc = arglocs + self.mc.LDR_ri(value_loc.value, base_loc.value, 0, cond=fcond) + self.mc.ADD_ri(value_loc.value, value_loc.value, 1, cond=fcond) + self.mc.STR_ri(value_loc.value, base_loc.value, 0, cond=fcond) return fcond def emit_op_getinteriorfield_gc(self, op, arglocs, regalloc, fcond): diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py --- a/rpython/jit/backend/arm/regalloc.py +++ b/rpython/jit/backend/arm/regalloc.py @@ -850,8 +850,12 @@ prepare_op_getfield_gc_pure = prepare_op_getfield_gc def prepare_op_increment_debug_counter(self, op, fcond): - # XXX implement me - return [] + boxes = op.getarglist() + a0, = boxes + base_loc = self.make_sure_var_in_reg(a0, boxes) + value_loc = self.get_scratch_reg(INT, boxes) + self.free_temp_vars() + return [base_loc, value_loc] def prepare_op_getinteriorfield_gc(self, op, fcond): t = unpack_interiorfielddescr(op.getdescr()) From noreply at buildbot.pypy.org Sat Mar 22 08:10:31 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 22 Mar 2014 08:10:31 +0100 (CET) Subject: [pypy-commit] pypy default: fix test Message-ID: <20140322071031.1B3091C0290@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r70153:87d94ae4f90e Date: 2014-03-22 08:09 +0100 http://bitbucket.org/pypy/pypy/changeset/87d94ae4f90e/ Log: fix test diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py --- a/rpython/jit/metainterp/optimizeopt/test/test_util.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py @@ -319,6 +319,9 @@ def log_loop(*args): pass + class logger_ops: + repr_of_resop = repr + class warmrunnerdesc: class memory_manager: retrace_limit = 5 From noreply at buildbot.pypy.org Sat Mar 22 08:21:09 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 22 Mar 2014 08:21:09 +0100 (CET) Subject: [pypy-commit] pypy default: Fix test Message-ID: <20140322072109.447561C0290@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r70154:aaf1ce9e21d0 Date: 2014-03-22 08:20 +0100 http://bitbucket.org/pypy/pypy/changeset/aaf1ce9e21d0/ Log: Fix test diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py --- a/pypy/module/pypyjit/test_pypy_c/test_containers.py +++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py @@ -70,14 +70,14 @@ p13 = new(descr=...) p15 = new_array(8, descr=) setfield_gc(p13, p15, descr=) - i17 = call(ConstClass(ll_dict_lookup_trampoline), p13, p10, i12, descr=) + i17 = call(ConstClass(ll_dict_lookup_trampoline), p13, p10, i12, descr=) setfield_gc(p13, 16, descr=) guard_no_exception(descr=...) p20 = new_with_vtable(ConstClass(W_IntObject)) call(ConstClass(_ll_dict_setitem_lookup_done_trampoline), p13, p10, p20, i12, i17, descr=) setfield_gc(p20, i5, descr=) guard_no_exception(descr=...) - i23 = call(ConstClass(ll_dict_lookup_trampoline), p13, p10, i12, descr=) + i23 = call(ConstClass(ll_dict_lookup_trampoline), p13, p10, i12, descr=) guard_no_exception(descr=...) i26 = int_and(i23, .*) i27 = int_is_true(i26) From noreply at buildbot.pypy.org Sat Mar 22 08:30:22 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 22 Mar 2014 08:30:22 +0100 (CET) Subject: [pypy-commit] pypy default: Fix test. Message-ID: <20140322073022.681831C0290@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r70155:b03753b1bf5e Date: 2014-03-22 08:29 +0100 http://bitbucket.org/pypy/pypy/changeset/b03753b1bf5e/ Log: Fix test. A nice side-effect of the improve-consecutive-dict-lookups branch: multiple loads or stores of attributes in an oldstyle class instance are folded. diff --git a/pypy/module/pypyjit/test_pypy_c/test_instance.py b/pypy/module/pypyjit/test_pypy_c/test_instance.py --- a/pypy/module/pypyjit/test_pypy_c/test_instance.py +++ b/pypy/module/pypyjit/test_pypy_c/test_instance.py @@ -141,15 +141,16 @@ i = 0 b = B(1) while i < 100: - b.x - v = b.x # ID: loadattr + v = b.x # ID: loadattr1 + v = b.x # ID: loadattr2 i += v return i log = self.run(main, [], threshold=80) loop, = log.loops_by_filename(self.filepath) - assert loop.match_by_id('loadattr', + assert loop.match_by_id('loadattr1', ''' + guard_not_invalidated(descr=...) i19 = call(ConstClass(ll_dict_lookup), _, _, _, descr=...) guard_no_exception(descr=...) i21 = int_and(i19, _) @@ -161,6 +162,7 @@ i29 = int_is_true(i28) guard_true(i29, descr=...) ''') + assert loop.match_by_id('loadattr2', "") # completely folded away def test_python_contains(self): def main(): From noreply at buildbot.pypy.org Sat Mar 22 08:55:32 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 22 Mar 2014 08:55:32 +0100 (CET) Subject: [pypy-commit] pypy default: oups Message-ID: <20140322075532.C93DD1C02AF@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r70156:0ee9349b856b Date: 2014-03-22 08:54 +0100 http://bitbucket.org/pypy/pypy/changeset/0ee9349b856b/ Log: oups diff --git a/rpython/jit/codewriter/effectinfo.py b/rpython/jit/codewriter/effectinfo.py --- a/rpython/jit/codewriter/effectinfo.py +++ b/rpython/jit/codewriter/effectinfo.py @@ -103,8 +103,10 @@ extradescrs=None): key = (frozenset_or_none(readonly_descrs_fields), frozenset_or_none(readonly_descrs_arrays), + frozenset_or_none(readonly_descrs_interiorfields), frozenset_or_none(write_descrs_fields), frozenset_or_none(write_descrs_arrays), + frozenset_or_none(write_descrs_interiorfields), extraeffect, oopspecindex, can_invalidate) From noreply at buildbot.pypy.org Sat Mar 22 09:18:15 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 22 Mar 2014 09:18:15 +0100 (CET) Subject: [pypy-commit] pypy default: Test and fix for the issue of dicts occasionally not updating their Message-ID: <20140322081815.EAAA31C0290@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r70157:7e2b44e79080 Date: 2014-03-22 09:17 +0100 http://bitbucket.org/pypy/pypy/changeset/7e2b44e79080/ Log: Test and fix for the issue of dicts occasionally not updating their cached status on a "del". The fix is to record the whole array-of- struct in the 'write_descrs_arrays' as soon as we write in one interior field, which is probably a good idea. This removes the last usage of effectinfo.write_descrs_interiorfields, which will no longer be translated right now -- but I'm leaving it in as the way forward. diff --git a/rpython/jit/codewriter/effectinfo.py b/rpython/jit/codewriter/effectinfo.py --- a/rpython/jit/codewriter/effectinfo.py +++ b/rpython/jit/codewriter/effectinfo.py @@ -224,6 +224,18 @@ descr = cpu.interiorfielddescrof(T, fieldname) descrs_interiorfields.append(descr) + # a read or a write to an interiorfield, inside an array of + # structs, is additionally recorded as a read or write of + # the array itself + extraef = set() + for tup in effects: + if tup[0] == "interiorfield" or tup[0] == "readinteriorfield": + T = deref(tup[1]) + if isinstance(T, lltype.Array) and consider_array(T): + extraef.add((tup[0].replace("interiorfield", "array"), + tup[1])) + effects |= extraef + for tup in effects: if tup[0] == "struct": add_struct(write_descrs_fields, tup) diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py --- a/rpython/jit/codewriter/jtransform.py +++ b/rpython/jit/codewriter/jtransform.py @@ -1854,8 +1854,7 @@ def _handle_dict_lookup_call(self, op, oopspec_name, args): extradescr1 = self.cpu.fielddescrof(op.args[1].concretetype.TO, 'entries') - extradescr2 = self.cpu.interiorfielddescrof( - op.args[1].concretetype.TO.entries.TO, 'key') + extradescr2 = self.cpu.arraydescrof(op.args[1].concretetype.TO.entries.TO) return self._handle_oopspec_call(op, args, EffectInfo.OS_DICT_LOOKUP, extradescr=[extradescr1, extradescr2]) diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py --- a/rpython/jit/metainterp/optimizeopt/heap.py +++ b/rpython/jit/metainterp/optimizeopt/heap.py @@ -177,7 +177,7 @@ self.cached_arrayitems = {} # cached dict items: {dict descr: {(optval, index): box-or-const}} self.cached_dict_reads = {} - # cache of corresponding array descrs + # cache of corresponding {array descrs: dict 'entries' field descr} self.corresponding_array_descrs = {} # self._lazy_setfields_and_arrayitems = [] @@ -346,9 +346,8 @@ self.force_lazy_setfield(fielddescr, can_cache=False) for arraydescr in effectinfo.write_descrs_arrays: self.force_lazy_setarrayitem(arraydescr, can_cache=False) - for descr in effectinfo.write_descrs_interiorfields: - if descr in self.corresponding_array_descrs: - dictdescr = self.corresponding_array_descrs.pop(descr) + if arraydescr in self.corresponding_array_descrs: + dictdescr = self.corresponding_array_descrs.pop(arraydescr) try: del self.cached_dict_reads[dictdescr] except KeyError: diff --git a/rpython/jit/metainterp/test/test_dict.py b/rpython/jit/metainterp/test/test_dict.py --- a/rpython/jit/metainterp/test/test_dict.py +++ b/rpython/jit/metainterp/test/test_dict.py @@ -327,6 +327,21 @@ self.check_simple_loop(call_may_force=4, # ll_dict_lookup_trampoline call=1) # ll_dict_setitem_lookup_done_trampoline + def test_bug42(self): + myjitdriver = JitDriver(greens = [], reds = 'auto') + def f(n): + mdict = {0: None, 1: None, 2: None, 3: None, 4: None, + 5: None, 6: None, 7: None, 8: None, 9: None} + while n > 0: + myjitdriver.jit_merge_point() + n -= 1 + if n in mdict: + del mdict[n] + if n in mdict: + raise Exception + self.meta_interp(f, [10]) + self.check_simple_loop(call_may_force=0, call=3) + class TestLLtype(DictTests, LLJitMixin): pass From noreply at buildbot.pypy.org Sat Mar 22 11:20:53 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 22 Mar 2014 11:20:53 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: in-progress Message-ID: <20140322102053.A03B41C073C@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r70158:005b73ce47d4 Date: 2014-03-22 11:20 +0100 http://bitbucket.org/pypy/pypy/changeset/005b73ce47d4/ Log: in-progress diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py --- a/rpython/jit/backend/llsupport/gc.py +++ b/rpython/jit/backend/llsupport/gc.py @@ -107,7 +107,8 @@ def rewrite_assembler(self, cpu, operations, gcrefs_output_list): if not self.stm: - from rpython.jit.backend.llsupport.rewrite import GcRewriterAssembler + from rpython.jit.backend.llsupport import rewrite + GcRewriterAssembler = rewrite.GcRewriterAssembler else: from rpython.jit.backend.llsupport import stmrewrite GcRewriterAssembler = stmrewrite.GcStmRewriterAssembler diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -52,7 +52,6 @@ # barriers. We do this on each "basic block" of operations, which in # this case means between CALLs or unknown-size mallocs. # - # SYNC with stmrewrite.py! for op in operations: if op.getopnum() == rop.DEBUG_MERGE_POINT: continue @@ -60,7 +59,7 @@ if op.is_malloc(): self.handle_malloc_operation(op) continue - elif op.is_call(): + elif op.can_malloc(): self.emitting_an_operation_that_can_collect() elif op.getopnum() == rop.LABEL: self.emitting_an_operation_that_can_collect() @@ -80,10 +79,13 @@ if op.getopnum() == rop.CALL_ASSEMBLER: self.handle_call_assembler(op) continue - # - self.newops.append(op) + # ---------- fallback case (overwritten in stmrewrite) ----------- + self.other_operation(op) return self.newops + def other_operation(self, op): + self.newops.append(op) + # ---------- def handle_malloc_operation(self, op): @@ -158,8 +160,7 @@ else: raise NotImplementedError(op.getopname()) - def gen_malloc_frame(self, frame_info, frame): - size_box = history.BoxInt() + def gen_malloc_frame(self, frame_info, frame, size_box): descrs = self.gc_ll_descr.getframedescrs(self.cpu) if self.gc_ll_descr.kind == 'boehm' or self.gc_ll_descr.stm: op0 = ResOperation(rop.GETFIELD_GC, [history.ConstInt(frame_info)], @@ -176,8 +177,8 @@ size_box, descr=descrs.jfi_frame_size) self.newops.append(op0) - self.gen_malloc_nursery_varsize_frame(size_box, frame, - descrs.arraydescr.tid) + self.gen_malloc_nursery_varsize_frame(size_box, frame) + self.gen_initialize_tid(frame, descrs.arraydescr.tid) length_box = history.BoxInt() op1 = ResOperation(rop.GETFIELD_GC, [history.ConstInt(frame_info)], length_box, @@ -192,8 +193,9 @@ assert isinstance(loop_token, history.JitCellToken) jfi = loop_token.compiled_loop_token.frame_info llfi = heaptracker.adr2int(llmemory.cast_ptr_to_adr(jfi)) + size_box = history.BoxInt() frame = history.BoxPtr() - self.gen_malloc_frame(llfi, frame) + self.gen_malloc_frame(llfi, frame, size_box) op2 = ResOperation(rop.SETFIELD_GC, [frame, history.ConstInt(llfi)], None, descr=descrs.jf_frame_info) self.newops.append(op2) @@ -322,7 +324,7 @@ self.write_barrier_applied[v_result] = None return True - def gen_malloc_nursery_varsize_frame(self, sizebox, v_result, tid): + def gen_malloc_nursery_varsize_frame(self, sizebox, v_result): """ Generate CALL_MALLOC_NURSERY_VARSIZE_FRAME """ self.emitting_an_operation_that_can_collect() @@ -333,8 +335,6 @@ self.newops.append(op) self.write_barrier_applied[v_result] = None - self.gen_initialize_tid(v_result, tid) - def gen_malloc_nursery(self, size, v_result): """Try to generate or update a CALL_MALLOC_NURSERY. If that fails, generate a plain CALL_MALLOC_GC instead. @@ -392,8 +392,8 @@ v = op.getarg(1) if isinstance(v, BoxPtr) or (isinstance(v, ConstPtr) and bool(v.value)): # store a non-NULL - op = self.prepare_write_barrier(op, rop.SETFIELD_RAW) - self.gen_write_barrier(op.getarg(0)) + self.gen_write_barrier(val) + #op = op.copy_and_change(rop.SETFIELD_RAW) self.newops.append(op) def handle_write_barrier_setinteriorfield(self, op): @@ -402,8 +402,8 @@ v = op.getarg(2) if isinstance(v, BoxPtr) or (isinstance(v, ConstPtr) and bool(v.value)): # store a non-NULL - op = self.prepare_write_barrier(op, rop.SETINTERIORFIELD_RAW) - self.gen_write_barrier(op.getarg(0)) + self.gen_write_barrier(val) + #op = op.copy_and_change(rop.SETINTERIORFIELD_RAW) self.newops.append(op) def handle_write_barrier_setarrayitem(self, op): @@ -412,22 +412,10 @@ v = op.getarg(2) if isinstance(v, BoxPtr) or (isinstance(v, ConstPtr) and bool(v.value)): # store a non-NULL - op = self.prepare_write_barrier(op, rop.SETARRAYITEM_RAW) - self.gen_write_barrier_array(op.getarg(0), - op.getarg(1)) + self.gen_write_barrier_array(val, op.getarg(1)) + #op = op.copy_and_change(rop.SETARRAYITEM_RAW) self.newops.append(op) - def prepare_write_barrier(self, op, newopnum): - write_barrier_descr = self.gc_ll_descr.write_barrier_descr - if (write_barrier_descr.returns_modified_object and - isinstance(op.getarg(0), ConstPtr)): - args = op.getarglist() - v_box = BoxPtr() - self.newops.append(ResOperation(rop.SAME_AS, [args[0]], v_box)) - args[0] = v_box - return op.copy_and_change(opnum=newopnum, args=args) - return op - def gen_write_barrier(self, v_base): write_barrier_descr = self.gc_ll_descr.write_barrier_descr args = [v_base] @@ -437,11 +425,10 @@ def gen_write_barrier_array(self, v_base, v_index): write_barrier_descr = self.gc_ll_descr.write_barrier_descr - if write_barrier_descr.has_barrier_from_array(self.cpu): + if write_barrier_descr.has_write_barrier_from_array(self.cpu): # If we know statically the length of 'v', and it is not too # big, then produce a regular write_barrier. If it's unknown or # too big, produce instead a write_barrier_from_array. - assert not write_barrier_descr.returns_modified_object LARGE = 130 length = self.known_lengths.get(v_base, LARGE) if length >= LARGE: diff --git a/rpython/jit/backend/llsupport/stmrewrite.py b/rpython/jit/backend/llsupport/stmrewrite.py --- a/rpython/jit/backend/llsupport/stmrewrite.py +++ b/rpython/jit/backend/llsupport/stmrewrite.py @@ -9,32 +9,6 @@ debug_print) from rpython.jit.codewriter.effectinfo import EffectInfo -### XXX: -### we changed some 'x2I' barriers to 'x2R' since -### obj initialization may happen in 2 different transactions. -### check and fix this assumption - - -# -# STM Support -# ----------- -# -# Any SETFIELD_GC, SETARRAYITEM_GC, SETINTERIORFIELD_GC must be done on a -# W object. The operation that forces an object p1 to be W is -# COND_CALL_STM_B(p1, descr=x2Wdescr), for x in 'AIQRVWZ'. This -# COND_CALL_STM_B is a bit special because if p1 is not W, it *replaces* -# its value with the W copy (by changing the register's value and -# patching the stack location if any). It's still conceptually the same -# object, but the pointer is different. -# -# The case of GETFIELD_GC & friends is similar, excepted that it goes to -# a R or L object (at first, always a R object). -# -# The name "x2y" of write barriers is called the *category* or "cat". -# - - - class GcStmRewriterAssembler(GcRewriterAssembler): # This class performs the same rewrites as its base class, @@ -42,143 +16,83 @@ def __init__(self, *args): GcRewriterAssembler.__init__(self, *args) - self.known_category = {} # variable: letter (R, W, ...) self.always_inevitable = False - - def rewrite(self, operations): - debug_start("jit-stmrewrite-ops") - # overridden method from parent class - # - for op in operations: - opnum = op.getopnum() - if not we_are_translated(): - # only possible in tests: - if opnum in (rop.COND_CALL_STM_B, - -124): # FORCE_SPILL + def other_operation(self, op): + opnum = op.getopnum() + if opnum == rop.INCREMENT_DEBUG_COUNTER: + self.newops.append(op) + return + # ---------- transaction breaks ---------- + if opnum == rop.STM_TRANSACTION_BREAK: + # XXX redo! + #self.emitting_an_operation_that_can_collect() + #self.next_op_may_be_in_new_transaction() + #self.newops.append(op) + return + # ---------- pure operations, guards ---------- + if op.is_always_pure() or op.is_guard() or op.is_ovf(): + self.newops.append(op) + return + # ---------- non-pure getfields ---------- + if opnum in (rop.GETFIELD_GC, rop.GETARRAYITEM_GC, + rop.GETINTERIORFIELD_GC): + self.handle_getfields(op) + return + # ---------- calls ---------- + if op.is_call(): + if opnum == rop.CALL_RELEASE_GIL: + # self.fallback_inevitable(op) + # is done by assembler._release_gil_shadowstack() + self.newops.append(op) + elif opnum == rop.CALL_ASSEMBLER: + assert 0 # case handled by the parent class + else: + # only insert become_inevitable if calling a + # non-transactionsafe and non-releasegil function + descr = op.getdescr() + assert not descr or isinstance(descr, CallDescr) + + if not descr or not descr.get_extra_info() \ + or descr.get_extra_info().call_needs_inevitable(): + self.fallback_inevitable(op) + else: self.newops.append(op) - continue - if opnum in (rop.INCREMENT_DEBUG_COUNTER, - rop.DEBUG_MERGE_POINT): - self.newops.append(op) + return + # ---------- copystrcontent ---------- + if opnum in (rop.COPYSTRCONTENT, rop.COPYUNICODECONTENT): + self.handle_copystrcontent(op) + continue + XXX + # ---------- raw getfields and setfields ---------- + if opnum in (rop.GETFIELD_RAW, rop.SETFIELD_RAW): + if self.maybe_handle_raw_accesses(op): continue - # ---------- transaction breaks ---------- - if opnum == rop.STM_TRANSACTION_BREAK: - self.emitting_an_operation_that_can_collect() - self.next_op_may_be_in_new_transaction() - self.newops.append(op) - continue - # ---------- ptr_eq ---------- - if opnum in (rop.PTR_EQ, rop.INSTANCE_PTR_EQ, - rop.PTR_NE, rop.INSTANCE_PTR_NE): - self.newops.append(op) - continue - # ---------- guard_class ---------- - if opnum == rop.GUARD_CLASS: - assert self.cpu.vtable_offset is None - # requires gcremovetypeptr translation option - # uses h_tid which doesn't need a read-barrier - self.newops.append(op) - continue - # ---------- pure operations needing read-barrier ---------- - if opnum in (rop.GETFIELD_GC_PURE, - rop.GETARRAYITEM_GC_PURE, - rop.ARRAYLEN_GC, rop.STRGETITEM, - rop.UNICODEGETITEM, rop.STRLEN, - rop.UNICODELEN): - # e.g. getting inst_intval of a W_IntObject that is - # currently only a stub needs to first resolve to a - # real object - # XXX: 'I' enough? - self.handle_category_operations(op, 'R') - continue - # ---------- pure operations, guards ---------- - if op.is_always_pure() or op.is_guard() or op.is_ovf(): - self.newops.append(op) - continue - # ---------- getfields ---------- - if opnum in (rop.GETFIELD_GC, rop.GETARRAYITEM_GC, - rop.GETINTERIORFIELD_GC): - self.handle_getfields(op) - continue - # ---------- setfields ---------- - if opnum in (rop.SETFIELD_GC, rop.SETINTERIORFIELD_GC, - rop.SETARRAYITEM_GC, rop.STRSETITEM, - rop.UNICODESETITEM): - self.handle_setfields(op) - continue - # ---------- mallocs ---------- - if op.is_malloc(): - self.handle_malloc_operation(op) - continue - # ---------- calls ---------- - if op.is_call(): - if opnum == rop.CALL and op.getdescr(): - d = op.getdescr() - assert isinstance(d, CallDescr) - ei = d.get_extra_info() - if ei and ei.oopspecindex == EffectInfo.OS_JIT_STM_SHOULD_BREAK_TRANSACTION: - self.newops.append(op) - continue - - self.emitting_an_operation_that_can_collect() - self.next_op_may_be_in_new_transaction() - - if opnum == rop.CALL_RELEASE_GIL: - # self.fallback_inevitable(op) - # is done by assembler._release_gil_shadowstack() - self.newops.append(op) - elif opnum == rop.CALL_ASSEMBLER: - self.handle_call_assembler(op) - else: - # only insert become_inevitable if calling a - # non-transactionsafe and non-releasegil function - descr = op.getdescr() - assert not descr or isinstance(descr, CallDescr) - - if not descr or not descr.get_extra_info() \ - or descr.get_extra_info().call_needs_inevitable(): - self.fallback_inevitable(op) - else: - self.newops.append(op) - continue - # ---------- copystrcontent ---------- - if opnum in (rop.COPYSTRCONTENT, rop.COPYUNICODECONTENT): - self.handle_copystrcontent(op) - continue - # ---------- raw getfields and setfields ---------- - if opnum in (rop.GETFIELD_RAW, rop.SETFIELD_RAW): - if self.maybe_handle_raw_accesses(op): - continue - # ---------- labels ---------- - if opnum == rop.LABEL: - self.emitting_an_operation_that_can_collect() - self.next_op_may_be_in_new_transaction() - - self.newops.append(op) - continue - # ---------- jumps ---------- - if opnum == rop.JUMP: - self.newops.append(op) - continue - # ---------- finish, other ignored ops ---------- - if opnum in (rop.FINISH, rop.FORCE_TOKEN, - rop.READ_TIMESTAMP, rop.MARK_OPAQUE_PTR, - rop.JIT_DEBUG, rop.KEEPALIVE, - rop.QUASIIMMUT_FIELD, rop.RECORD_KNOWN_CLASS, - ): - self.newops.append(op) - continue - # ---------- fall-back ---------- - # Check that none of the ops handled here can_collect - # or cause a transaction break. This is not done by - # the fallback here - self.fallback_inevitable(op) - debug_print("fallback for", op.repr()) - # - - debug_stop("jit-stmrewrite-ops") - return self.newops + # ---------- labels ---------- + if opnum == rop.LABEL: + self.emitting_an_operation_that_can_collect() + self.next_op_may_be_in_new_transaction() + + self.newops.append(op) + continue + # ---------- jumps ---------- + if opnum == rop.JUMP: + self.newops.append(op) + continue + # ---------- finish, other ignored ops ---------- + if opnum in (rop.FINISH, rop.FORCE_TOKEN, + rop.READ_TIMESTAMP, rop.MARK_OPAQUE_PTR, + rop.JIT_DEBUG, rop.KEEPALIVE, + rop.QUASIIMMUT_FIELD, rop.RECORD_KNOWN_CLASS, + ): + self.newops.append(op) + continue + # ---------- fall-back ---------- + # Check that none of the ops handled here can_collect + # or cause a transaction break. This is not done by + # the fallback here + self.fallback_inevitable(op) + debug_print("fallback for", op.repr()) def emitting_an_operation_that_can_collect(self): GcRewriterAssembler.emitting_an_operation_that_can_collect(self) From noreply at buildbot.pypy.org Sat Mar 22 12:04:19 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 22 Mar 2014 12:04:19 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Revert some more changes Message-ID: <20140322110419.AB76E1D253B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r70159:f7dc1aca6874 Date: 2014-03-22 11:43 +0100 http://bitbucket.org/pypy/pypy/changeset/f7dc1aca6874/ Log: Revert some more changes diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py --- a/rpython/jit/backend/llsupport/gc.py +++ b/rpython/jit/backend/llsupport/gc.py @@ -280,60 +280,16 @@ def get_root_stack_top_addr(self): rst_addr = llop.gc_adr_of_root_stack_top(llmemory.Address) return rffi.cast(lltype.Signed, rst_addr) - - -class BarrierDescr(AbstractDescr): + +class WriteBarrierDescr(AbstractDescr): def __init__(self, gc_ll_descr): self.llop1 = gc_ll_descr.llop1 - - self.returns_modified_object = False - self.gcheaderbuilder = gc_ll_descr.gcheaderbuilder - self.HDRPTR = gc_ll_descr.HDRPTR - self.b_slowpath = [0, 0, 0, 0, 0] - - def repr_of_descr(self): - raise NotImplementedError - - def __repr(self): - raise NotImplementedError - - def get_b_slowpath(self, num): - return self.b_slowpath[num] - - def set_b_slowpath(self, num, addr): - self.b_slowpath[num] = addr - - def get_barrier_funcptr(self, returns_modified_object): - raise NotImplementedError - - def get_barrier_fn(self, cpu, returns_modified_object): - # must pass in 'self.returns_modified_object', to make sure that - # the callers are fixed for this case - funcptr = self.get_barrier_funcptr(returns_modified_object) - funcaddr = llmemory.cast_ptr_to_adr(funcptr) - return cpu.cast_adr_to_int(funcaddr) - - def get_barrier_from_array_fn(self, cpu): - # returns a function with arguments [array, index, newvalue] - llop1 = self.llop1 - funcptr = llop1.get_write_barrier_from_array_failing_case( - self.FUNCPTR) - funcaddr = llmemory.cast_ptr_to_adr(funcptr) - return cpu.cast_adr_to_int(funcaddr) # this may return 0 - - def has_barrier_from_array(self, cpu): - return self.get_barrier_from_array_fn(cpu) != 0 - - - -class WriteBarrierDescr(BarrierDescr): - def __init__(self, gc_ll_descr): - BarrierDescr.__init__(self, gc_ll_descr) + self.WB_FUNCPTR = gc_ll_descr.WB_FUNCPTR self.fielddescr_tid = gc_ll_descr.fielddescr_tid - self.FUNCPTR = lltype.Ptr(lltype.FuncType( - [llmemory.Address], lltype.Void)) - + # GCClass = gc_ll_descr.GCClass + if GCClass is None: # for tests + return self.jit_wb_if_flag = GCClass.JIT_WB_IF_FLAG self.jit_wb_if_flag_byteofs, self.jit_wb_if_flag_singlebyte = ( self.extract_flag_byte(self.jit_wb_if_flag)) @@ -351,17 +307,9 @@ else: self.jit_wb_cards_set = 0 - def repr_of_descr(self): - return 'wbdescr' - - def __repr__(self): - return '' % (self.repr_of_descr(),) - def extract_flag_byte(self, flag_word): # if convenient for the backend, we compute the info about # the flag as (byte-offset, single-byte-flag). - if flag_word == 0: - return (0, 0) import struct value = struct.pack(lltype.SignedFmt, flag_word) assert value.count('\x00') == len(value) - 1 # only one byte is != 0 @@ -369,143 +317,24 @@ while value[i] == '\x00': i += 1 return (i, struct.unpack('b', value[i])[0]) - def get_barrier_funcptr(self, returns_modified_object): - assert not returns_modified_object - FUNCTYPE = self.FUNCPTR - return self.llop1.get_write_barrier_failing_case(FUNCTYPE) + def get_write_barrier_fn(self, cpu): + llop1 = self.llop1 + funcptr = llop1.get_write_barrier_failing_case(self.WB_FUNCPTR) + funcaddr = llmemory.cast_ptr_to_adr(funcptr) + return cpu.cast_adr_to_int(funcaddr) - @specialize.arg(2) - def _do_barrier(self, gcref_struct, returns_modified_object): - assert self.returns_modified_object == returns_modified_object - hdr_addr = llmemory.cast_ptr_to_adr(gcref_struct) - hdr_addr -= self.gcheaderbuilder.size_gc_header - hdr = llmemory.cast_adr_to_ptr(hdr_addr, self.HDRPTR) - if self.jit_wb_if_flag == 0 or hdr.tid & self.jit_wb_if_flag: - # get a pointer to the 'remember_young_pointer' function from - # the GC, and call it immediately - funcptr = self.get_barrier_funcptr(returns_modified_object) - funcptr(llmemory.cast_ptr_to_adr(gcref_struct)) + def get_write_barrier_from_array_fn(self, cpu): + # returns a function with arguments [array, index, newvalue] + llop1 = self.llop1 + funcptr = llop1.get_write_barrier_from_array_failing_case( + self.WB_FUNCPTR) + funcaddr = llmemory.cast_ptr_to_adr(funcptr) + return cpu.cast_adr_to_int(funcaddr) # this may return 0 - -class STMBarrierDescr(BarrierDescr): - def __init__(self, gc_ll_descr, stmcat, cfunc_name): - BarrierDescr.__init__(self, gc_ll_descr) - self.stmcat = stmcat - self.returns_modified_object = True - self.B_FUNCPTR_MOD = lltype.Ptr(lltype.FuncType( - [llmemory.Address], llmemory.Address)) + def has_write_barrier_from_array(self, cpu): + return self.get_write_barrier_from_array_fn(cpu) != 0 - self.b_failing_case_ptr = rffi.llexternal( - cfunc_name, - self.B_FUNCPTR_MOD.TO.ARGS, - self.B_FUNCPTR_MOD.TO.RESULT, - sandboxsafe=True, - _nowrapper=True) - def repr_of_descr(self): - return self.stmcat - - def __repr__(self): - return '' % (self.repr_of_descr(),) - - def get_barrier_funcptr(self, returns_modified_object): - assert returns_modified_object - return self.b_failing_case_ptr - - @specialize.arg(2) - def _do_barrier(self, gcref_struct, returns_modified_object): - raise NotImplementedError("implement in subclasses!") - assert self.returns_modified_object == returns_modified_object - funcptr = self.get_barrier_funcptr(returns_modified_object) - res = funcptr(llmemory.cast_ptr_to_adr(gcref_struct)) - return llmemory.cast_adr_to_ptr(res, llmemory.GCREF) - - -class STMReadBarrierDescr(STMBarrierDescr): - def __init__(self, gc_ll_descr, stmcat): - assert stmcat in ['A2R', 'Q2R', 'A2I'] - func = {'A2R': 'stm_DirectReadBarrier', - 'Q2R': 'stm_RepeatReadBarrier', - 'A2I': 'stm_ImmutReadBarrier', - } - - STMBarrierDescr.__init__(self, gc_ll_descr, stmcat, - func[stmcat]) - - @specialize.arg(2) - def _do_barrier(self, gcref_struct, returns_modified_object): - assert returns_modified_object - from rpython.memory.gc.stmgc import StmGC - objadr = llmemory.cast_ptr_to_adr(gcref_struct) - objhdr = rffi.cast(StmGC.GCHDRP, gcref_struct) - - if self.stmcat == 'A2R': - # if h_revision == privat_rev of transaction - priv_rev = self.llop1.stm_get_adr_of_private_rev_num(rffi.SIGNEDP) - if objhdr.h_revision == priv_rev[0]: - return gcref_struct - - # readcache[obj] == obj - read_cache = self.llop1.stm_get_adr_of_read_barrier_cache(rffi.SIGNEDP) - objint = llmemory.cast_adr_to_int(objadr) - assert WORD == 8, "check for 32bit compatibility" - index = (objint & StmGC.FX_MASK) / WORD - CP = lltype.Ptr(rffi.CArray(lltype.Signed)) - rcp = rffi.cast(CP, read_cache[0]) - if rcp[index] == objint: - return gcref_struct - elif self.stmcat == 'Q2R': - # is GCFLAG_PUBLIC_TO_PRIVATE or GCFLAG_MOVED set? - if not (objhdr.h_tid & - (StmGC.GCFLAG_PUBLIC_TO_PRIVATE | StmGC.GCFLAG_MOVED)): - # no. - return gcref_struct - else: # A2I - # GCFLAG_STUB set? - if not (objhdr.h_tid & StmGC.GCFLAG_STUB): - return gcref_struct - - funcptr = self.get_barrier_funcptr(returns_modified_object) - res = funcptr(objadr) - return llmemory.cast_adr_to_ptr(res, llmemory.GCREF) - - -class STMWriteBarrierDescr(STMBarrierDescr): - def __init__(self, gc_ll_descr, stmcat): - assert stmcat in ['A2W', 'V2W', 'A2V'] - func = {'A2W':'stm_WriteBarrier', - 'V2W':'stm_RepeatWriteBarrier', - 'A2V':'stm_WriteBarrier', - } - - STMBarrierDescr.__init__(self, gc_ll_descr, stmcat, - func[stmcat]) - - - @specialize.arg(2) - def _do_barrier(self, gcref_struct, returns_modified_object): - assert returns_modified_object - from rpython.memory.gc.stmgc import StmGC - objadr = llmemory.cast_ptr_to_adr(gcref_struct) - objhdr = rffi.cast(StmGC.GCHDRP, gcref_struct) - - # for A2W, we check h_revision and WRITE_BARRIER - # for V2W, we only check WRITE_BARRIER - # for A2V, we only check h_revision - - # if it is a repeated WB or h_revision == privat_rev of transaction - priv_rev = self.llop1.stm_get_adr_of_private_rev_num(rffi.SIGNEDP) - if self.stmcat == 'V2W' or objhdr.h_revision == priv_rev[0]: - # also WRITE_BARRIER not set? - if (self.stmcat == 'A2V' - or not (objhdr.h_tid & StmGC.GCFLAG_WRITE_BARRIER)): - return gcref_struct - - funcptr = self.get_barrier_funcptr(returns_modified_object) - res = funcptr(objadr) - return llmemory.cast_adr_to_ptr(res, llmemory.GCREF) - - class GcLLDescr_framework(GcLLDescription): DEBUG = False # forced to True by x86/test/test_zrpy_gc.py kind = 'framework' @@ -551,7 +380,6 @@ def _initialize_for_tests(self): self.layoutbuilder = None self.fielddescr_tid = AbstractDescr() - self.fielddescr_rev = AbstractDescr() self.max_size_of_young_obj = 1000 self.GCClass = None self.gcheaderbuilder = None @@ -592,27 +420,17 @@ def _setup_tid(self): if not self.stm: self.fielddescr_tid = get_field_descr(self, self.GCClass.HDR, 'tid') - self.fielddescr_rev = None else: + xxxxxxxx self.fielddescr_tid = get_field_descr(self, self.GCClass.GCHDR, 'h_tid') - self.fielddescr_rev = get_field_descr(self, self.GCClass.GCHDR, - 'h_revision') - frame_tid = self.layoutbuilder.get_type_id(jitframe.JITFRAME) self.translator._jit2gc['frame_tid'] = frame_tid def _setup_write_barrier(self): - if self.stm: - self.A2Rdescr = STMReadBarrierDescr(self, 'A2R') - self.A2Idescr = STMReadBarrierDescr(self, 'A2I') - self.Q2Rdescr = STMReadBarrierDescr(self, 'Q2R') - self.A2Wdescr = STMWriteBarrierDescr(self, 'A2W') - self.A2Vdescr = STMWriteBarrierDescr(self, 'A2V') - self.V2Wdescr = STMWriteBarrierDescr(self, 'V2W') - self.write_barrier_descr = "wbdescr: do not use" - else: - self.write_barrier_descr = WriteBarrierDescr(self) + self.WB_FUNCPTR = lltype.Ptr(lltype.FuncType( + [llmemory.Address], lltype.Void)) + self.write_barrier_descr = WriteBarrierDescr(self) def _make_functions(self, really_not_translated): from rpython.memory.gctypelayout import check_typeid @@ -775,15 +593,12 @@ hdr.tid = tid def can_use_nursery_malloc(self, size): - return (self.max_size_of_young_obj is not None and - size < self.max_size_of_young_obj) + return size < self.max_size_of_young_obj def has_write_barrier_class(self): return WriteBarrierDescr def get_malloc_slowpath_addr(self): - if self.max_size_of_young_obj is None: # stm - return None return self.get_malloc_fn_addr('malloc_nursery') def get_malloc_slowpath_array_addr(self): diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -59,7 +59,7 @@ if op.is_malloc(): self.handle_malloc_operation(op) continue - elif op.can_malloc(): + elif op.is_call(): self.emitting_an_operation_that_can_collect() elif op.getopnum() == rop.LABEL: self.emitting_an_operation_that_can_collect() diff --git a/rpython/jit/backend/llsupport/stmrewrite.py b/rpython/jit/backend/llsupport/stmrewrite.py --- a/rpython/jit/backend/llsupport/stmrewrite.py +++ b/rpython/jit/backend/llsupport/stmrewrite.py @@ -25,10 +25,9 @@ return # ---------- transaction breaks ---------- if opnum == rop.STM_TRANSACTION_BREAK: - # XXX redo! - #self.emitting_an_operation_that_can_collect() - #self.next_op_may_be_in_new_transaction() - #self.newops.append(op) + self.emitting_an_operation_that_can_collect() + self.next_op_may_be_in_new_transaction() + self.newops.append(op) return # ---------- pure operations, guards ---------- if op.is_always_pure() or op.is_guard() or op.is_ovf(): @@ -41,6 +40,17 @@ return # ---------- calls ---------- if op.is_call(): + if opnum == rop.CALL and op.getdescr(): + d = op.getdescr() + assert isinstance(d, CallDescr) + ei = d.get_extra_info() + if ei and (ei.oopspecindex == + EffectInfo.OS_JIT_STM_SHOULD_BREAK_TRANSACTION): + self.newops.append(op) + return + # + self.next_op_may_be_in_new_transaction() + # if opnum == rop.CALL_RELEASE_GIL: # self.fallback_inevitable(op) # is done by assembler._release_gil_shadowstack() @@ -59,125 +69,41 @@ else: self.newops.append(op) return + # ---------- setters for pure fields ---------- + if opnum in (rop.STRSETITEM, rop.UNICODESETITEM): + self.handle_setters_for_pure_fields(op) + return # ---------- copystrcontent ---------- if opnum in (rop.COPYSTRCONTENT, rop.COPYUNICODECONTENT): self.handle_copystrcontent(op) - continue - XXX + return # ---------- raw getfields and setfields ---------- if opnum in (rop.GETFIELD_RAW, rop.SETFIELD_RAW): if self.maybe_handle_raw_accesses(op): - continue + return # ---------- labels ---------- if opnum == rop.LABEL: - self.emitting_an_operation_that_can_collect() + # note that the parent class also clears some things on a LABEL self.next_op_may_be_in_new_transaction() - self.newops.append(op) - continue - # ---------- jumps ---------- - if opnum == rop.JUMP: + return + # ---------- jumps, finish, other ignored ops ---------- + if opnum in (rop.JUMP, rop.FINISH, rop.FORCE_TOKEN, + rop.READ_TIMESTAMP, rop.MARK_OPAQUE_PTR, + rop.JIT_DEBUG, rop.KEEPALIVE, + rop.QUASIIMMUT_FIELD, rop.RECORD_KNOWN_CLASS, + ): self.newops.append(op) - continue - # ---------- finish, other ignored ops ---------- - if opnum in (rop.FINISH, rop.FORCE_TOKEN, - rop.READ_TIMESTAMP, rop.MARK_OPAQUE_PTR, - rop.JIT_DEBUG, rop.KEEPALIVE, - rop.QUASIIMMUT_FIELD, rop.RECORD_KNOWN_CLASS, - ): - self.newops.append(op) - continue + return # ---------- fall-back ---------- - # Check that none of the ops handled here can_collect - # or cause a transaction break. This is not done by - # the fallback here + # Check that none of the ops handled here can collect. + # This is not done by the fallback here + assert not op.is_call() and not op.is_malloc() self.fallback_inevitable(op) - debug_print("fallback for", op.repr()) - def emitting_an_operation_that_can_collect(self): - GcRewriterAssembler.emitting_an_operation_that_can_collect(self) - self.invalidate_write_categories() - def next_op_may_be_in_new_transaction(self): - self.known_lengths.clear() # XXX: check if really necessary or - # just for labels - self.known_category.clear() self.always_inevitable = False - def invalidate_write_categories(self): - for v, c in self.known_category.items(): - if c == 'W': - self.known_category[v] = 'V' - - def invalidate_read_categories(self, reason): - # XXX: needs aliasing info to be better - # XXX: move to optimizeopt to only invalidate same typed vars? - for v, c in self.known_category.items(): - if c == 'R': - self.known_category[v] = 'Q' - - - def get_barrier_descr(self, from_cat, to_cat): - # compare with translator.stm.funcgen.stm_barrier - # XXX: specialize more with info of IMMUTABLE and NOPTR - if from_cat >= to_cat: - return None - - gc = self.gc_ll_descr - if to_cat == 'W': - if from_cat >= 'V': - return gc.V2Wdescr - return gc.A2Wdescr - elif to_cat == 'V': - return gc.A2Vdescr - elif to_cat == 'R': - if from_cat >= 'Q': - return gc.Q2Rdescr - return gc.A2Rdescr - elif to_cat == 'I': - return gc.A2Idescr - - def gen_initialize_tid(self, v_newgcobj, tid): - GcRewriterAssembler.gen_initialize_tid(self, v_newgcobj, tid) - if self.gc_ll_descr.fielddescr_rev is not None: - op = ResOperation(rop.STM_SET_REVISION_GC, [v_newgcobj,], None, - descr=self.gc_ll_descr.fielddescr_rev) - self.newops.append(op) - - def gen_write_barrier(self, v): - raise NotImplementedError - - def gen_barrier(self, v_base, target_category): - v_base = self.unconstifyptr(v_base) - assert isinstance(v_base, BoxPtr) - source_category = self.known_category.get(v_base, 'A') - write_barrier_descr = self.get_barrier_descr(source_category, - target_category) - if write_barrier_descr is None: - return v_base # no barrier needed - - if target_category in ('W', 'V'): - # if *any* of the readable vars is the same object, - # it must repeat the read_barrier now - self.invalidate_read_categories(v_base) - - args = [v_base,] - op = rop.COND_CALL_STM_B - self.newops.append(ResOperation(op, args, None, - descr=write_barrier_descr)) - - self.known_category[v_base] = target_category - return v_base - - def unconstifyptr(self, v): - if isinstance(v, ConstPtr): - v_in = v - v_out = BoxPtr() - self.newops.append(ResOperation(rop.SAME_AS, [v_in], v_out)) - v = v_out - assert isinstance(v, BoxPtr) - return v - def handle_getfields(self, op): opnum = op.getopnum() descr = op.getdescr() @@ -254,6 +180,7 @@ self._do_stm_call('stm_try_inevitable', [], None) self.always_inevitable = True self.newops.append(op) + debug_print("fallback for", op.repr()) def _is_null(self, box): return isinstance(box, ConstPtr) and not box.value diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py --- a/rpython/jit/backend/llsupport/test/test_rewrite.py +++ b/rpython/jit/backend/llsupport/test/test_rewrite.py @@ -71,7 +71,6 @@ register_known_gctype(self.cpu, o_vtable, O) # tiddescr = self.gc_ll_descr.fielddescr_tid - revdescr = self.gc_ll_descr.fielddescr_rev wbdescr = self.gc_ll_descr.write_barrier_descr WORD = globals()['WORD'] # @@ -103,11 +102,9 @@ namespace[funcname + '_descr'] = getattr(self.gc_ll_descr, '%s_descr' % funcname) # - ops = parse(frm_operations, namespace=namespace, - invent_fail_descr=False) + ops = parse(frm_operations, namespace=namespace) expected = parse(to_operations % Evaluator(namespace), - namespace=namespace, - invent_fail_descr=False) + namespace=namespace) operations = self.gc_ll_descr.rewrite_assembler(self.cpu, ops.operations, []) @@ -287,7 +284,7 @@ gcdescr = get_description(config_) self.gc_ll_descr = GcLLDescr_framework(gcdescr, None, None, None, really_not_translated=True) - self.gc_ll_descr.write_barrier_descr.has_barrier_from_array = ( + self.gc_ll_descr.write_barrier_descr.has_write_barrier_from_array = ( lambda cpu: True) # class FakeCPU(BaseFakeCPU): @@ -579,7 +576,7 @@ """) def test_write_barrier_before_array_without_from_array(self): - self.gc_ll_descr.write_barrier_descr.has_barrier_from_array = ( + self.gc_ll_descr.write_barrier_descr.has_write_barrier_from_array = ( lambda cpu: False) self.check_rewrite(""" [p1, i2, p3] From noreply at buildbot.pypy.org Sat Mar 22 12:04:21 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 22 Mar 2014 12:04:21 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: read barriers Message-ID: <20140322110421.0CD0D1D253B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r70160:8c9539ab0df6 Date: 2014-03-22 12:03 +0100 http://bitbucket.org/pypy/pypy/changeset/8c9539ab0df6/ Log: read barriers diff --git a/rpython/jit/backend/llsupport/stmrewrite.py b/rpython/jit/backend/llsupport/stmrewrite.py --- a/rpython/jit/backend/llsupport/stmrewrite.py +++ b/rpython/jit/backend/llsupport/stmrewrite.py @@ -17,6 +17,7 @@ def __init__(self, *args): GcRewriterAssembler.__init__(self, *args) self.always_inevitable = False + self.read_barrier_applied = {} def other_operation(self, op): opnum = op.getopnum() @@ -103,28 +104,23 @@ def next_op_may_be_in_new_transaction(self): self.always_inevitable = False + self.read_barrier_applied.clear() def handle_getfields(self, op): - opnum = op.getopnum() - descr = op.getdescr() - target_category = 'R' - # XXX: review: - # if opnum == rop.GETFIELD_GC: - # assert isinstance(descr, FieldDescr) - # if descr.is_immutable(): - # target_category = 'I' - # elif opnum == rop.GETINTERIORFIELD_GC: - # assert isinstance(descr, InteriorFieldDescr) - # if descr.is_immutable(): - # target_category = 'I' - # elif opnum == rop.GETARRAYITEM_GC: - # assert isinstance(descr, ArrayDescr) - # if descr.is_immutable(): - # target_category = 'I' - - self.handle_category_operations(op, target_category) + # XXX missing optimitations: the placement of stm_read should + # ideally be delayed for a bit longer after the getfields; if we + # group together several stm_reads then we can save one + # instruction; if delayed over a cond_call_gc_wb then we can + # omit the stm_read completely; ... + self.newops.append(op) + v_ptr = op.getarg(0) + if (v_ptr not in self.read_barrier_applied and + v_ptr not in self.write_barrier_applied): + op1 = ResOperation(rop.STM_READ, [v_ptr], None) + self.newops.append(op1) + self.read_barrier_applied[v_ptr] = None - + def handle_setfields(self, op): opnum = op.getopnum() descr = op.getdescr() @@ -174,7 +170,6 @@ self.newops.append(op1) def fallback_inevitable(self, op): - self.known_category.clear() if not self.always_inevitable: self.emitting_an_operation_that_can_collect() self._do_stm_call('stm_try_inevitable', [], None) diff --git a/rpython/jit/backend/llsupport/test/test_stmrewrite.py b/rpython/jit/backend/llsupport/test/test_stmrewrite.py --- a/rpython/jit/backend/llsupport/test/test_stmrewrite.py +++ b/rpython/jit/backend/llsupport/test/test_stmrewrite.py @@ -20,6 +20,10 @@ words.append('CALL_MALLOC_GC') words.append('COND_CALL_GC_WB') words.append('COND_CALL_GC_WB_ARRAY') + # these are pure, and can be done without any read barrier + words.append('ARRAYLEN_GC') + words.append('GETFIELD_GC_PURE') + words.append('GETARRAYITEM_GC_PURE') # words = set(words) missing = [] @@ -85,7 +89,7 @@ call(123, descr=cd) jump() """ % ("$INEV" if inev else "",), cd=calldescr) - + def test_rewrite_one_setfield_gc(self): self.check_rewrite(""" [p1, p2] @@ -93,9 +97,8 @@ jump() """, """ [p1, p2] - cond_call_stm_b(p1, descr=A2Wdescr) + cond_call_gc_wb(p1, descr=wbdescr) setfield_gc(p1, p2, descr=tzdescr) - jump() """) @@ -108,13 +111,42 @@ jump() """, """ [p1, p2] - p3 = same_as(ConstPtr(t)) - cond_call_stm_b(p3, descr=A2Wdescr) - setfield_gc(p3, p2, descr=tzdescr) - + cond_call_gc_wb(ConstPtr(t), descr=wbdescr) + setfield_gc(ConstPtr(t), p2, descr=tzdescr) jump() """, t=NULL) + def test_rewrite_one_getfield_gc(self): + self.check_rewrite(""" + [p1] + p2 = getfield_gc(p1, descr=tzdescr) + jump() + """, """ + [p1] + p2 = getfield_gc(p1, descr=tzdescr) + stm_read(p1) + jump() + """) + + def test_rewrite_several_getfield_gc(self): + self.check_rewrite(""" + [p1, p2] + p3 = getfield_gc(p1, descr=tzdescr) + p4 = getfield_gc(p1, descr=tzdescr) + p5 = getfield_gc(p2, descr=tzdescr) + p6 = getfield_gc(p1, descr=tzdescr) + jump() + """, """ + [p1, p2] + p3 = getfield_gc(p1, descr=tzdescr) + stm_read(p1) + p4 = getfield_gc(p1, descr=tzdescr) + p5 = getfield_gc(p2, descr=tzdescr) + stm_read(p2) + p6 = getfield_gc(p1, descr=tzdescr) + jump() + """) + def test_invalidate_read_status_after_write_to_constptr(self): TP = lltype.GcArray(lltype.Signed) NULL = lltype.cast_opaque_ptr(llmemory.GCREF, lltype.nullptr(TP)) diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py --- a/rpython/jit/metainterp/executor.py +++ b/rpython/jit/metainterp/executor.py @@ -346,6 +346,7 @@ rop.CALL_MALLOC_NURSERY_VARSIZE, rop.CALL_MALLOC_NURSERY_VARSIZE_FRAME, rop.LABEL, + rop.STM_READ, ): # list of opcodes never executed by pyjitpl continue raise AssertionError("missing %r" % (key,)) diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py --- a/rpython/jit/metainterp/resoperation.py +++ b/rpython/jit/metainterp/resoperation.py @@ -511,6 +511,7 @@ 'RECORD_KNOWN_CLASS/2', # [objptr, clsptr] 'KEEPALIVE/1', 'STM_TRANSACTION_BREAK/1', + 'STM_READ/1', '_CANRAISE_FIRST', # ----- start of can_raise operations ----- '_CALL_FIRST', From noreply at buildbot.pypy.org Sat Mar 22 12:12:39 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 22 Mar 2014 12:12:39 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Fixes Message-ID: <20140322111239.0E0621D253B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r70161:c3d7dd930444 Date: 2014-03-22 12:12 +0100 http://bitbucket.org/pypy/pypy/changeset/c3d7dd930444/ Log: Fixes diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -386,34 +386,32 @@ # ---------- + def must_apply_write_barrier(self, val, v): + if val not in self.write_barrier_applied: + if isinstance(v, BoxPtr) or (isinstance(v, ConstPtr) and + bool(v.value)): # store a non-NULL + return True + return False + def handle_write_barrier_setfield(self, op): val = op.getarg(0) - if val not in self.write_barrier_applied: - v = op.getarg(1) - if isinstance(v, BoxPtr) or (isinstance(v, ConstPtr) and - bool(v.value)): # store a non-NULL - self.gen_write_barrier(val) - #op = op.copy_and_change(rop.SETFIELD_RAW) + if self.must_apply_write_barrier(val, op.getarg(1)): + self.gen_write_barrier(val) + #op = op.copy_and_change(rop.SETFIELD_RAW) self.newops.append(op) def handle_write_barrier_setinteriorfield(self, op): val = op.getarg(0) - if val not in self.write_barrier_applied: - v = op.getarg(2) - if isinstance(v, BoxPtr) or (isinstance(v, ConstPtr) and - bool(v.value)): # store a non-NULL - self.gen_write_barrier(val) - #op = op.copy_and_change(rop.SETINTERIORFIELD_RAW) + if self.must_apply_write_barrier(val, op.getarg(2)): + self.gen_write_barrier(val) + #op = op.copy_and_change(rop.SETINTERIORFIELD_RAW) self.newops.append(op) def handle_write_barrier_setarrayitem(self, op): val = op.getarg(0) - if val not in self.write_barrier_applied: - v = op.getarg(2) - if isinstance(v, BoxPtr) or (isinstance(v, ConstPtr) and - bool(v.value)): # store a non-NULL - self.gen_write_barrier_array(val, op.getarg(1)) - #op = op.copy_and_change(rop.SETARRAYITEM_RAW) + if self.must_apply_write_barrier(val, op.getarg(2)): + self.gen_write_barrier_array(val, op.getarg(1)) + #op = op.copy_and_change(rop.SETARRAYITEM_RAW) self.newops.append(op) def gen_write_barrier(self, v_base): diff --git a/rpython/jit/backend/llsupport/stmrewrite.py b/rpython/jit/backend/llsupport/stmrewrite.py --- a/rpython/jit/backend/llsupport/stmrewrite.py +++ b/rpython/jit/backend/llsupport/stmrewrite.py @@ -121,6 +121,10 @@ self.read_barrier_applied[v_ptr] = None + def must_apply_write_barrier(self, val, v): + return val not in self.write_barrier_applied + + def handle_setfields(self, op): opnum = op.getopnum() descr = op.getdescr() diff --git a/rpython/jit/backend/llsupport/test/test_stmrewrite.py b/rpython/jit/backend/llsupport/test/test_stmrewrite.py --- a/rpython/jit/backend/llsupport/test/test_stmrewrite.py +++ b/rpython/jit/backend/llsupport/test/test_stmrewrite.py @@ -102,6 +102,18 @@ jump() """) + def test_rewrite_one_setfield_gc_i(self): + self.check_rewrite(""" + [p1, i2] + setfield_gc(p1, i2, descr=tzdescr) + jump() + """, """ + [p1, i2] + cond_call_gc_wb(p1, descr=wbdescr) + setfield_gc(p1, i2, descr=tzdescr) + jump() + """) + def test_rewrite_setfield_gc_const(self): TP = lltype.GcArray(lltype.Signed) NULL = lltype.cast_opaque_ptr(llmemory.GCREF, lltype.nullptr(TP)) @@ -147,6 +159,20 @@ jump() """) + def test_rewrite_getfield_after_setfield(self): + self.check_rewrite(""" + [p1, i2] + setfield_gc(p1, i2, descr=tydescr) + p3 = getfield_gc(p1, descr=tzdescr) + jump(p3) + """, """ + [p1, i2] + cond_call_gc_wb(p1, descr=wbdescr) + setfield_gc(p1, i2, descr=tydescr) + p3 = getfield_gc(p1, descr=tzdescr) + jump(p3) + """) + def test_invalidate_read_status_after_write_to_constptr(self): TP = lltype.GcArray(lltype.Signed) NULL = lltype.cast_opaque_ptr(llmemory.GCREF, lltype.nullptr(TP)) @@ -162,9 +188,9 @@ [p0] p1 = same_as(ConstPtr(t)) p2 = same_as(ConstPtr(t)) - cond_call_stm_b(p1, descr=A2Rdescr) p3 = getfield_gc(p1, descr=tzdescr) - cond_call_stm_b(p2, descr=A2Wdescr) + stm_read(p1) + cond_call_gc_wb(p2, descr=wbdescr) setfield_gc(p2, p0, descr=tzdescr) cond_call_stm_b(p1, descr=Q2Rdescr) p4 = getfield_gc(p1, descr=tzdescr) From noreply at buildbot.pypy.org Sat Mar 22 12:17:55 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 22 Mar 2014 12:17:55 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: progress Message-ID: <20140322111755.CFE8C1D253B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r70162:1226e7465baf Date: 2014-03-22 12:17 +0100 http://bitbucket.org/pypy/pypy/changeset/1226e7465baf/ Log: progress diff --git a/rpython/jit/backend/llsupport/stmrewrite.py b/rpython/jit/backend/llsupport/stmrewrite.py --- a/rpython/jit/backend/llsupport/stmrewrite.py +++ b/rpython/jit/backend/llsupport/stmrewrite.py @@ -125,38 +125,8 @@ return val not in self.write_barrier_applied - def handle_setfields(self, op): - opnum = op.getopnum() - descr = op.getdescr() - target_category = 'W' - if opnum == rop.SETFIELD_GC: - assert isinstance(descr, FieldDescr) - if not descr.is_pointer_field(): - target_category = 'V' - elif opnum == rop.SETINTERIORFIELD_GC: - assert isinstance(descr, InteriorFieldDescr) - if not descr.is_pointer_field(): - target_category = 'V' - elif opnum == rop.SETARRAYITEM_GC: - assert isinstance(descr, ArrayDescr) - if not descr.is_array_of_pointers(): - target_category = 'V' - elif opnum in (rop.STRSETITEM, rop.UNICODESETITEM): - target_category = 'V' - - self.handle_category_operations(op, target_category) - - - def handle_category_operations(self, op, target_category): - lst = op.getarglist() - lst[0] = self.gen_barrier(lst[0], target_category) - self.newops.append(op.copy_and_change(op.getopnum(), args=lst)) - - def handle_malloc_operation(self, op): - GcRewriterAssembler.handle_malloc_operation(self, op) - self.known_category[op.result] = 'W' - def handle_copystrcontent(self, op): + xxxxxxxx # first, a write barrier on the target string lst = op.getarglist() lst[1] = self.gen_barrier(lst[1], 'W') @@ -181,10 +151,8 @@ self.newops.append(op) debug_print("fallback for", op.repr()) - def _is_null(self, box): - return isinstance(box, ConstPtr) and not box.value - def maybe_handle_raw_accesses(self, op): + xxxxx from rpython.jit.backend.llsupport.descr import FieldDescr descr = op.getdescr() assert isinstance(descr, FieldDescr) diff --git a/rpython/jit/backend/llsupport/test/test_stmrewrite.py b/rpython/jit/backend/llsupport/test/test_stmrewrite.py --- a/rpython/jit/backend/llsupport/test/test_stmrewrite.py +++ b/rpython/jit/backend/llsupport/test/test_stmrewrite.py @@ -24,6 +24,10 @@ words.append('ARRAYLEN_GC') words.append('GETFIELD_GC_PURE') words.append('GETARRAYITEM_GC_PURE') + # these are handled by rewrite.py + words.append('SETFIELD_GC') + words.append('SETARRAYITEM_GC') + words.append('SETINTERIORFIELD_GC') # words = set(words) missing = [] @@ -173,108 +177,25 @@ jump(p3) """) - def test_invalidate_read_status_after_write_to_constptr(self): + def test_mixed_case(self): TP = lltype.GcArray(lltype.Signed) NULL = lltype.cast_opaque_ptr(llmemory.GCREF, lltype.nullptr(TP)) self.check_rewrite(""" - [p0] - p1 = same_as(ConstPtr(t)) - p2 = same_as(ConstPtr(t)) + [p0, p1, p2] p3 = getfield_gc(p1, descr=tzdescr) setfield_gc(p2, p0, descr=tzdescr) p4 = getfield_gc(p1, descr=tzdescr) jump() """, """ - [p0] - p1 = same_as(ConstPtr(t)) - p2 = same_as(ConstPtr(t)) + [p0, p1, p2] p3 = getfield_gc(p1, descr=tzdescr) stm_read(p1) cond_call_gc_wb(p2, descr=wbdescr) setfield_gc(p2, p0, descr=tzdescr) - cond_call_stm_b(p1, descr=Q2Rdescr) p4 = getfield_gc(p1, descr=tzdescr) - jump() """, t=NULL) - def test_invalidate_read_status_after_write(self): - self.check_rewrite(""" - [p0] - p1 = same_as(p0) - p2 = same_as(p0) - p4 = getfield_gc(p1, descr=tzdescr) - setfield_gc(p2, p0, descr=tzdescr) - p5 = getfield_gc(p1, descr=tzdescr) - jump() - """, """ - [p0] - p1 = same_as(p0) - p2 = same_as(p0) - cond_call_stm_b(p1, descr=A2Rdescr) - p4 = getfield_gc(p1, descr=tzdescr) - cond_call_stm_b(p2, descr=A2Wdescr) - setfield_gc(p2, p0, descr=tzdescr) - cond_call_stm_b(p1, descr=Q2Rdescr) - p5 = getfield_gc(p1, descr=tzdescr) - - jump() - """) - - def test_invalidate_read_status_after_write_to_field(self): - self.check_rewrite(""" - [p0] - p1 = getfield_gc(p0, descr=tzdescr) - p2 = getfield_gc(p0, descr=tzdescr) - p3 = getfield_gc(p1, descr=tzdescr) - setfield_gc(p2, p0, descr=tzdescr) - p4 = getfield_gc(p1, descr=tzdescr) - jump() - """, """ - [p0] - cond_call_stm_b(p0, descr=A2Rdescr) - p1 = getfield_gc(p0, descr=tzdescr) - p2 = getfield_gc(p0, descr=tzdescr) - cond_call_stm_b(p1, descr=A2Rdescr) - p3 = getfield_gc(p1, descr=tzdescr) - cond_call_stm_b(p2, descr=A2Wdescr) - setfield_gc(p2, p0, descr=tzdescr) - cond_call_stm_b(p1, descr=Q2Rdescr) - p4 = getfield_gc(p1, descr=tzdescr) - - jump() - """) - - def test_invalidate_read_status_after_write_array_interior(self): - ops = [('getarrayitem_gc', 'adescr'), - ('getinteriorfield_gc', 'intzdescr')] - original = """ - [p0, i1, i2] - p1 = %s(p0, i1, descr=%s) - p2 = %s(p0, i2, descr=%s) - p3 = getfield_gc(p1, descr=tzdescr) - setfield_gc(p2, p0, descr=tzdescr) - p4 = getfield_gc(p1, descr=tzdescr) - jump() - """ - rewritten = """ - [p0, i1, i2] - cond_call_stm_b(p0, descr=A2Rdescr) - p1 = %s(p0, i1, descr=%s) - p2 = %s(p0, i2, descr=%s) - cond_call_stm_b(p1, descr=A2Rdescr) - p3 = getfield_gc(p1, descr=tzdescr) - cond_call_stm_b(p2, descr=A2Wdescr) - setfield_gc(p2, p0, descr=tzdescr) - cond_call_stm_b(p1, descr=Q2Rdescr) - p4 = getfield_gc(p1, descr=tzdescr) - - jump() - """ - for op, descr in ops: - self.check_rewrite(original % (op, descr, op, descr), - rewritten % (op, descr, op, descr)) - def test_rewrite_write_barrier_after_malloc(self): self.check_rewrite(""" [p1, p3] @@ -284,18 +205,16 @@ jump(p2) """, """ [p1, p3] - cond_call_stm_b(p3, descr=A2Wdescr) + cond_call_gc_wb(p3, descr=wbdescr) setfield_gc(p3, p1, descr=tzdescr) p2 = call_malloc_nursery(%(tdescr.size)d) setfield_gc(p2, %(tdescr.tid)d, descr=tiddescr) - stm_set_revision_gc(p2, descr=revdescr) - cond_call_stm_b(p3, descr=V2Wdescr) + cond_call_gc_wb(p3, descr=wbdescr) setfield_gc(p3, p1, descr=tzdescr) - jump(p2) """) - def test_rewrite_read_barrier_after_malloc(self): + def test_rewrite_no_read_barrier_after_malloc(self): self.check_rewrite(""" [p1] p2 = getfield_gc(p1, descr=tzdescr) @@ -304,16 +223,14 @@ jump(p2) """, """ [p1] - cond_call_stm_b(p1, descr=A2Rdescr) p2 = getfield_gc(p1, descr=tzdescr) + stm_read(p1) p3 = call_malloc_nursery(%(tdescr.size)d) setfield_gc(p3, %(tdescr.tid)d, descr=tiddescr) - stm_set_revision_gc(p3, descr=revdescr) p4 = getfield_gc(p1, descr=tzdescr) - jump(p2) """) - + def test_rewrite_setfield_gc_on_local(self): self.check_rewrite(""" [p1] From noreply at buildbot.pypy.org Sat Mar 22 12:39:19 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 22 Mar 2014 12:39:19 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: progress Message-ID: <20140322113919.58E0F1C0290@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r70163:cf31cfe0189e Date: 2014-03-22 12:38 +0100 http://bitbucket.org/pypy/pypy/changeset/cf31cfe0189e/ Log: progress diff --git a/rpython/jit/backend/llsupport/stmrewrite.py b/rpython/jit/backend/llsupport/stmrewrite.py --- a/rpython/jit/backend/llsupport/stmrewrite.py +++ b/rpython/jit/backend/llsupport/stmrewrite.py @@ -121,7 +121,7 @@ self.read_barrier_applied[v_ptr] = None - def must_apply_write_barrier(self, val, v): + def must_apply_write_barrier(self, val, v=None): return val not in self.write_barrier_applied @@ -152,7 +152,6 @@ debug_print("fallback for", op.repr()) def maybe_handle_raw_accesses(self, op): - xxxxx from rpython.jit.backend.llsupport.descr import FieldDescr descr = op.getdescr() assert isinstance(descr, FieldDescr) @@ -160,3 +159,9 @@ self.newops.append(op) return True return False + + def handle_setters_for_pure_fields(self, op): + val = op.getarg(0) + if self.must_apply_write_barrier(val): + self.gen_write_barrier(val) + self.newops.append(op) diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py --- a/rpython/jit/backend/llsupport/test/test_rewrite.py +++ b/rpython/jit/backend/llsupport/test/test_rewrite.py @@ -102,9 +102,11 @@ namespace[funcname + '_descr'] = getattr(self.gc_ll_descr, '%s_descr' % funcname) # - ops = parse(frm_operations, namespace=namespace) + ops = parse(frm_operations, namespace=namespace, + invent_fail_descr=False) expected = parse(to_operations % Evaluator(namespace), - namespace=namespace) + namespace=namespace, + invent_fail_descr=False) operations = self.gc_ll_descr.rewrite_assembler(self.cpu, ops.operations, []) diff --git a/rpython/jit/backend/llsupport/test/test_stmrewrite.py b/rpython/jit/backend/llsupport/test/test_stmrewrite.py --- a/rpython/jit/backend/llsupport/test/test_stmrewrite.py +++ b/rpython/jit/backend/llsupport/test/test_stmrewrite.py @@ -50,6 +50,8 @@ gcdescr = get_description(config_) self.gc_ll_descr = GcLLDescr_framework(gcdescr, None, None, None, really_not_translated=True) + self.gc_ll_descr.write_barrier_descr.has_write_barrier_from_array = ( + lambda cpu: False) # for now # class FakeCPU(BaseFakeCPU): def sizeof(self, STRUCT): @@ -241,12 +243,24 @@ [p1] p2 = call_malloc_nursery(%(tdescr.size)d) setfield_gc(p2, %(tdescr.tid)d, descr=tiddescr) - stm_set_revision_gc(p2, descr=revdescr) setfield_gc(p2, p1, descr=tzdescr) - jump(p2) """) + def test_rewrite_getfield_gc_on_local(self): + self.check_rewrite(""" + [] + p2 = new(descr=tdescr) + p1 = getfield_gc(p2, descr=tzdescr) + jump(p1) + """, """ + [] + p2 = call_malloc_nursery(%(tdescr.size)d) + setfield_gc(p2, %(tdescr.tid)d, descr=tiddescr) + p1 = getfield_gc(p2, descr=tzdescr) + jump(p1) + """) + def test_rewrite_unrelated_setfield_gcs(self): self.check_rewrite(""" [p1, p2, p3, p4] @@ -255,11 +269,10 @@ jump() """, """ [p1, p2, p3, p4] - cond_call_stm_b(p1, descr=A2Wdescr) + cond_call_gc_wb(p1, descr=wbdescr) setfield_gc(p1, p2, descr=tzdescr) - cond_call_stm_b(p3, descr=A2Wdescr) + cond_call_gc_wb(p3, descr=wbdescr) setfield_gc(p3, p4, descr=tzdescr) - jump() """) @@ -271,10 +284,9 @@ jump() """, """ [p1, p2, i3] - cond_call_stm_b(p1, descr=A2Wdescr) + cond_call_gc_wb(p1, descr=wbdescr) setfield_gc(p1, p2, descr=tzdescr) setfield_gc(p1, i3, descr=tydescr) - jump() """) @@ -283,16 +295,15 @@ [p1, p2, i3] setfield_gc(p1, p2, descr=tzdescr) label(p1, i3) - setfield_gc(p1, i3, descr=tydescr) # noptr + setfield_gc(p1, i3, descr=tydescr) jump(p1) """, """ [p1, p2, i3] - cond_call_stm_b(p1, descr=A2Wdescr) + cond_call_gc_wb(p1, descr=wbdescr) setfield_gc(p1, p2, descr=tzdescr) label(p1, i3) - cond_call_stm_b(p1, descr=A2Vdescr) # noptr + cond_call_gc_wb(p1, descr=wbdescr) setfield_gc(p1, i3, descr=tydescr) - jump(p1) """) @@ -319,8 +330,9 @@ "jit_debug(i1, i2)", "keepalive(i1)", "i3 = int_sub_ovf(i1, i2)", # is_ovf operations + "increment_debug_counter(i1)", ] - for op in oplist: + for op in oplist: testcase = """ [i1, i2, p1, p2, f1] %s @@ -328,19 +340,6 @@ """ % op self.check_rewrite(testcase, testcase) - def test_rewrite_getfield_gc(self): - self.check_rewrite(""" - [p1] - p2 = getfield_gc(p1, descr=tzdescr) - jump(p2) - """, """ - [p1] - cond_call_stm_b(p1, descr=A2Rdescr) - p2 = getfield_gc(p1, descr=tzdescr) - - jump(p2) - """) - def test_rewrite_getfield_gc_const(self): TP = lltype.GcArray(lltype.Signed) NULL = lltype.cast_opaque_ptr(llmemory.GCREF, lltype.nullptr(TP)) @@ -350,10 +349,8 @@ jump(p2) """, """ [p1] - p3 = same_as(ConstPtr(t)) - cond_call_stm_b(p3, descr=A2Rdescr) - p2 = getfield_gc(p3, descr=tzdescr) - + p2 = getfield_gc(ConstPtr(t), descr=tzdescr) + stm_read(ConstPtr(t)) jump(p2) """, t=NULL) # XXX could do better: G2Rdescr @@ -365,9 +362,8 @@ jump(i3) """, """ [p1, i2] - cond_call_stm_b(p1, descr=A2Rdescr) i3 = getarrayitem_gc(p1, i2, descr=adescr) - + stm_read(p1) jump(i3) """) @@ -378,27 +374,11 @@ jump(i3) """, """ [p1, i2] - cond_call_stm_b(p1, descr=A2Rdescr) i3 = getinteriorfield_gc(p1, i2, descr=intzdescr) - + stm_read(p1) jump(i3) """) - def test_rewrite_several_getfield_gcs(self): - self.check_rewrite(""" - [p1] - p2 = getfield_gc(p1, descr=tzdescr) - i2 = getfield_gc(p1, descr=tydescr) - jump(p2, i2) - """, """ - [p1] - cond_call_stm_b(p1, descr=A2Rdescr) - p2 = getfield_gc(p1, descr=tzdescr) - i2 = getfield_gc(p1, descr=tydescr) - - jump(p2, i2) - """) - def test_rewrite_unrelated_getfield_gcs(self): self.check_rewrite(""" [p1] @@ -407,11 +387,10 @@ jump(p2, i2) """, """ [p1] - cond_call_stm_b(p1, descr=A2Rdescr) p2 = getfield_gc(p1, descr=tzdescr) - cond_call_stm_b(p2, descr=A2Rdescr) + stm_read(p1) i2 = getfield_gc(p2, descr=tydescr) - + stm_read(p2) jump(p2, i2) """) @@ -426,46 +405,15 @@ jump(p1) """, """ [p1] - cond_call_stm_b(p1, descr=A2Rdescr) i1 = getfield_gc(p1, descr=tydescr) + stm_read(p1) i2 = int_add(i1, 1) - cond_call_stm_b(p1, descr=A2Vdescr) + cond_call_gc_wb(p1, descr=wbdescr) setfield_gc(p1, i2, descr=tydescr) jump(p1) """) - def test_setfield_followed_by_getfield(self): - self.check_rewrite(""" - [p1] - setfield_gc(p1, 123, descr=tydescr) # noptr - p2 = getfield_gc(p1, descr=tzdescr) - jump(p2) - """, """ - [p1] - cond_call_stm_b(p1, descr=A2Vdescr) - setfield_gc(p1, 123, descr=tydescr) - p2 = getfield_gc(p1, descr=tzdescr) - - jump(p2) - """) - - def test_rewrite_getfield_gc_on_local_2(self): - self.check_rewrite(""" - [p0] - p1 = new(descr=tdescr) - p2 = getfield_gc(p1, descr=tzdescr) - jump(p2) - """, """ - [p0] - p1 = call_malloc_nursery(%(tdescr.size)d) - setfield_gc(p1, %(tdescr.tid)d, descr=tiddescr) - stm_set_revision_gc(p1, descr=revdescr) - p2 = getfield_gc(p1, descr=tzdescr) - - jump(p2) - """) - def test_rewrite_getfield_gc_on_future_local_after_call(self): # XXX could detect CALLs that cannot interrupt the transaction # and/or could use the L category @@ -484,12 +432,11 @@ jump(p2) """, """ [p1] - cond_call_stm_b(p1, descr=A2Rdescr) p2 = getfield_gc(p1, descr=tzdescr) + stm_read(p1) call(p2, descr=calldescr1) - cond_call_stm_b(p1, descr=A2Vdescr) + cond_call_gc_wb(p1, descr=wbdescr) setfield_gc(p1, 5, descr=tydescr) - jump(p2) """, calldescr1=calldescr1) @@ -567,9 +514,9 @@ jump() """, """ [p1, i1, p2, p3, i3, p4] - cond_call_stm_b(p1, descr=A2Vdescr) + cond_call_gc_wb(p1, descr=wbdescr) setarrayitem_gc(p1, i1, p2, descr=adescr) - cond_call_stm_b(p3, descr=A2Vdescr) + cond_call_gc_wb(p3, descr=wbdescr) setarrayitem_gc(p3, i3, p4, descr=adescr) jump() @@ -584,7 +531,7 @@ jump() """, """ [p1, p2, i2, p3, i3] - cond_call_stm_b(p1, descr=A2Vdescr) + cond_call_gc_wb(p1, descr=wbdescr) setarrayitem_gc(p1, i2, p2, descr=adescr) i4 = read_timestamp() setarrayitem_gc(p1, i3, p3, descr=adescr) @@ -601,7 +548,7 @@ jump() """, """ [p1, p2, i2, p3, i3] - cond_call_stm_b(p1, descr=A2Wdescr) + cond_call_gc_wb(p1, descr=wbdescr) setinteriorfield_gc(p1, i2, p2, descr=intzdescr) i4 = read_timestamp() setinteriorfield_gc(p1, i3, p3, descr=intzdescr) @@ -617,14 +564,28 @@ jump() """, """ [p1, i2, i3] - cond_call_stm_b(p1, descr=A2Vdescr) + cond_call_gc_wb(p1, descr=wbdescr) strsetitem(p1, i2, i3) unicodesetitem(p1, i2, i3) jump() """) - # py.test.skip("XXX not really right: should instead be an assert " - # "that p1 is already a W") + + def test_rewrite_strsetitem_unicodesetitem_on_fresh_malloc(self): + self.check_rewrite(""" + [i2, i3] + p1 = newstr(i3) + strsetitem(p1, i2, i3) + unicodesetitem(p1, i2, i3) + jump() + """, """ + [i2, i3] + p1 = call_malloc_nursery_varsize(1, 1, i3, descr=strdescr) + setfield_gc(p1, i3, descr=strlendescr) + strsetitem(p1, i2, i3) + unicodesetitem(p1, i2, i3) + jump() + """) def test_rewrite_strgetitem_unicodegetitem(self): self.check_rewrite(""" @@ -634,13 +595,10 @@ jump() """, """ [p1, i2, i3] - cond_call_stm_b(p1, descr=A2Rdescr) i4=strgetitem(p1, i2) i5=unicodegetitem(p1, i2) - jump() """) - def test_call_release_gil(self): T = rffi.CArrayPtr(rffi.TIME_T) From noreply at buildbot.pypy.org Sat Mar 22 12:43:38 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 22 Mar 2014 12:43:38 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: progress Message-ID: <20140322114338.74E7D1C0290@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r70164:59318684cd56 Date: 2014-03-22 12:42 +0100 http://bitbucket.org/pypy/pypy/changeset/59318684cd56/ Log: progress diff --git a/rpython/jit/backend/llsupport/stmrewrite.py b/rpython/jit/backend/llsupport/stmrewrite.py --- a/rpython/jit/backend/llsupport/stmrewrite.py +++ b/rpython/jit/backend/llsupport/stmrewrite.py @@ -72,11 +72,11 @@ return # ---------- setters for pure fields ---------- if opnum in (rop.STRSETITEM, rop.UNICODESETITEM): - self.handle_setters_for_pure_fields(op) + self.handle_setters_for_pure_fields(op, 0) return # ---------- copystrcontent ---------- if opnum in (rop.COPYSTRCONTENT, rop.COPYUNICODECONTENT): - self.handle_copystrcontent(op) + self.handle_setters_for_pure_fields(op, 1) return # ---------- raw getfields and setfields ---------- if opnum in (rop.GETFIELD_RAW, rop.SETFIELD_RAW): @@ -160,8 +160,8 @@ return True return False - def handle_setters_for_pure_fields(self, op): - val = op.getarg(0) + def handle_setters_for_pure_fields(self, op, targetindex): + val = op.getarg(targetindex) if self.must_apply_write_barrier(val): self.gen_write_barrier(val) self.newops.append(op) diff --git a/rpython/jit/backend/llsupport/test/test_stmrewrite.py b/rpython/jit/backend/llsupport/test/test_stmrewrite.py --- a/rpython/jit/backend/llsupport/test/test_stmrewrite.py +++ b/rpython/jit/backend/llsupport/test/test_stmrewrite.py @@ -612,17 +612,17 @@ jump(i2, p7) """, """ [i1, i2, i3, p7] - cond_call_stm_b(p7, descr=A2Vdescr) + cond_call_gc_wb(p7, descr=wbdescr) setfield_gc(p7, 10, descr=tydescr) call_release_gil(123, descr=calldescr2) guard_not_forced() [] - cond_call_stm_b(p7, descr=A2Vdescr) + cond_call_gc_wb(p7, descr=wbdescr) setfield_gc(p7, 20, descr=tydescr) jump(i2, p7) """, calldescr2=calldescr2) - + def test_fallback_to_inevitable(self): T = rffi.CArrayPtr(rffi.TIME_T) calldescr2 = get_call_descr(self.gc_ll_descr, [T], rffi.TIME_T) @@ -641,27 +641,39 @@ jump(i2, p7) """ % op, """ [i1, i2, i3, p7] - cond_call_stm_b(p7, descr=A2Vdescr) + cond_call_gc_wb(p7, descr=wbdescr) setfield_gc(p7, 10, descr=tydescr) $INEV %s - cond_call_stm_b(p7, descr=A2Vdescr) + cond_call_gc_wb(p7, descr=wbdescr) setfield_gc(p7, 20, descr=tydescr) jump(i2, p7) """ % op, calldescr2=calldescr2) - def test_copystrcontent(self): + def test_copystrcontent_new(self): + self.check_rewrite(""" + [p1, i1, i2, i3] + p2 = newstr(i3) + copystrcontent(p1, p2, i1, i2, i3) + jump() + """, """ + [p1, i1, i2, i3] + p2 = call_malloc_nursery_varsize(1, 1, i3, descr=strdescr) + setfield_gc(p2, i3, descr=strlendescr) + copystrcontent(p1, p2, i1, i2, i3) + jump() + """) + + def test_copystrcontent_old(self): self.check_rewrite(""" [p1, p2, i1, i2, i3] copystrcontent(p1, p2, i1, i2, i3) jump() """, """ [p1, p2, i1, i2, i3] - cond_call_stm_b(p2, descr=A2Wdescr) - cond_call_stm_b(p1, descr=A2Rdescr) + cond_call_gc_wb(p2, descr=wbdescr) copystrcontent(p1, p2, i1, i2, i3) - jump() """) From noreply at buildbot.pypy.org Sat Mar 22 12:58:34 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 22 Mar 2014 12:58:34 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: progress Message-ID: <20140322115834.9028F1C0290@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r70165:829d58f318f1 Date: 2014-03-22 12:57 +0100 http://bitbucket.org/pypy/pypy/changeset/829d58f318f1/ Log: progress diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -160,9 +160,10 @@ else: raise NotImplementedError(op.getopname()) - def gen_malloc_frame(self, frame_info, frame, size_box): + def gen_malloc_frame(self, frame_info, frame): descrs = self.gc_ll_descr.getframedescrs(self.cpu) - if self.gc_ll_descr.kind == 'boehm' or self.gc_ll_descr.stm: + if self.gc_ll_descr.kind == 'boehm': + size_box = history.BoxInt() op0 = ResOperation(rop.GETFIELD_GC, [history.ConstInt(frame_info)], size_box, descr=descrs.jfi_frame_depth) @@ -170,9 +171,9 @@ op1 = ResOperation(rop.NEW_ARRAY, [size_box], frame, descr=descrs.arraydescr) self.handle_new_array(descrs.arraydescr, op1) - else: + elif not self.gc_ll_descr.stm: # we read size in bytes here, not the length - # jfi_frame_size not set in STM! + size_box = history.BoxInt() op0 = ResOperation(rop.GETFIELD_GC, [history.ConstInt(frame_info)], size_box, descr=descrs.jfi_frame_size) @@ -186,6 +187,17 @@ self.newops.append(op1) self.gen_initialize_len(frame, length_box, descrs.arraydescr.lendescr) + else: + # jfi_frame_size not set in STM! + length_box = history.BoxInt() + op0 = ResOperation(rop.GETFIELD_GC, [history.ConstInt(frame_info)], + length_box, + descr=descrs.jfi_frame_depth) + self.newops.append(op0) + self.gen_malloc_nursery_varsize_frame(length_box, frame) + self.gen_initialize_tid(frame, descrs.arraydescr.tid) + self.gen_initialize_len(frame, length_box, + descrs.arraydescr.lendescr) def handle_call_assembler(self, op): descrs = self.gc_ll_descr.getframedescrs(self.cpu) @@ -193,9 +205,8 @@ assert isinstance(loop_token, history.JitCellToken) jfi = loop_token.compiled_loop_token.frame_info llfi = heaptracker.adr2int(llmemory.cast_ptr_to_adr(jfi)) - size_box = history.BoxInt() frame = history.BoxPtr() - self.gen_malloc_frame(llfi, frame, size_box) + self.gen_malloc_frame(llfi, frame) op2 = ResOperation(rop.SETFIELD_GC, [frame, history.ConstInt(llfi)], None, descr=descrs.jf_frame_info) self.newops.append(op2) @@ -329,7 +340,7 @@ """ self.emitting_an_operation_that_can_collect() op = ResOperation(rop.CALL_MALLOC_NURSERY_VARSIZE_FRAME, - [sizebox], + [sizebox], # if STM, this is actually lengthbox! v_result) self.newops.append(op) diff --git a/rpython/jit/backend/llsupport/test/test_stmrewrite.py b/rpython/jit/backend/llsupport/test/test_stmrewrite.py --- a/rpython/jit/backend/llsupport/test/test_stmrewrite.py +++ b/rpython/jit/backend/llsupport/test/test_stmrewrite.py @@ -723,32 +723,28 @@ jump(p1) """ % (op, guard), """ [p1] - cond_call_stm_b(p1, descr=A2Vdescr) + cond_call_gc_wb(p1, descr=wbdescr) setfield_gc(p1, 10, descr=tydescr) %s %s %s - cond_call_stm_b(p1, descr=A2Vdescr) + cond_call_gc_wb(p1, descr=wbdescr) setfield_gc(p1, 20, descr=tydescr) jump(p1) """ % (op, guard, tr_break), calldescr2=calldescr2) def test_call_assembler(self): - py.test.skip("XXX: works, but somehow the test doesn't") - self.check_rewrite(""" [i0, f0] i2 = call_assembler(i0, f0, descr=casmdescr) guard_not_forced()[] """, """ [i0, f0] - i1 = getfield_gc(ConstClass(frame_info), descr=jfi_frame_size) + i1 = getfield_gc(ConstClass(frame_info), descr=jfi_frame_depth) p1 = call_malloc_nursery_varsize_frame(i1) setfield_gc(p1, 0, descr=tiddescr) - stm_set_revision_gc(p1, descr=revdescr) - i2 = getfield_gc(ConstClass(frame_info), descr=jfi_frame_depth) - setfield_gc(p1, i2, descr=framelendescr) + setfield_gc(p1, i1, descr=framelendescr) setfield_gc(p1, ConstClass(frame_info), descr=jf_frame_info) setarrayitem_gc(p1, 0, i0, descr=signedframedescr) setarrayitem_gc(p1, 1, f0, descr=floatframedescr) @@ -817,11 +813,8 @@ jump(i1) """) - def test_ptr_eq_other_direct_cases(self): - py.test.skip("can also keep ptr_eq if both args are L or W, " - "or if one arg is freshly malloced") + # ----------- tests copied from rewrite.py ------------- - # ----------- tests copied from rewrite.py ------------- def test_rewrite_assembler_new_to_malloc(self): self.check_rewrite(""" [p1] @@ -830,7 +823,6 @@ [p1] p0 = call_malloc_nursery(%(sdescr.size)d) setfield_gc(p0, 1234, descr=tiddescr) - stm_set_revision_gc(p0, descr=revdescr) """) def test_rewrite_assembler_new3_to_malloc(self): @@ -844,13 +836,10 @@ p0 = call_malloc_nursery( \ %(sdescr.size + tdescr.size + sdescr.size)d) setfield_gc(p0, 1234, descr=tiddescr) - stm_set_revision_gc(p0, descr=revdescr) p1 = int_add(p0, %(sdescr.size)d) setfield_gc(p1, 5678, descr=tiddescr) - stm_set_revision_gc(p1, descr=revdescr) p2 = int_add(p1, %(tdescr.size)d) setfield_gc(p2, 1234, descr=tiddescr) - stm_set_revision_gc(p2, descr=revdescr) """) def test_rewrite_assembler_new_array_fixed_to_malloc(self): @@ -862,7 +851,6 @@ p0 = call_malloc_nursery( \ %(adescr.basesize + 10 * adescr.itemsize)d) setfield_gc(p0, 4321, descr=tiddescr) - stm_set_revision_gc(p0, descr=revdescr) setfield_gc(p0, 10, descr=alendescr) """) @@ -877,10 +865,8 @@ %(sdescr.size + \ adescr.basesize + 10 * adescr.itemsize)d) setfield_gc(p0, 1234, descr=tiddescr) - stm_set_revision_gc(p0, descr=revdescr) p1 = int_add(p0, %(sdescr.size)d) setfield_gc(p1, 4321, descr=tiddescr) - stm_set_revision_gc(p1, descr=revdescr) setfield_gc(p1, 10, descr=alendescr) """) @@ -892,7 +878,6 @@ [] p0 = call_malloc_nursery(%(bdescr.basesize + 8)d) setfield_gc(p0, 8765, descr=tiddescr) - stm_set_revision_gc(p0, descr=revdescr) setfield_gc(p0, 6, descr=blendescr) """) @@ -907,19 +892,15 @@ [] p0 = call_malloc_nursery(%(4 * (bdescr.basesize + 8))d) setfield_gc(p0, 8765, descr=tiddescr) - stm_set_revision_gc(p0, descr=revdescr) setfield_gc(p0, 5, descr=blendescr) p1 = int_add(p0, %(bdescr.basesize + 8)d) setfield_gc(p1, 8765, descr=tiddescr) - stm_set_revision_gc(p1, descr=revdescr) setfield_gc(p1, 5, descr=blendescr) p2 = int_add(p1, %(bdescr.basesize + 8)d) setfield_gc(p2, 8765, descr=tiddescr) - stm_set_revision_gc(p2, descr=revdescr) setfield_gc(p2, 5, descr=blendescr) p3 = int_add(p2, %(bdescr.basesize + 8)d) setfield_gc(p3, 8765, descr=tiddescr) - stm_set_revision_gc(p3, descr=revdescr) setfield_gc(p3, 5, descr=blendescr) """) @@ -932,10 +913,8 @@ [] p0 = call_malloc_nursery(%(4*WORD)d) setfield_gc(p0, 9000, descr=tiddescr) - stm_set_revision_gc(p0, descr=revdescr) p1 = int_add(p0, %(2*WORD)d) setfield_gc(p1, 9000, descr=tiddescr) - stm_set_revision_gc(p1, descr=revdescr) """) def test_rewrite_assembler_variable_size(self): @@ -1015,16 +994,13 @@ p0 = call_malloc_nursery( \ %(2 * (bdescr.basesize + 104))d) setfield_gc(p0, 8765, descr=tiddescr) - stm_set_revision_gc(p0, descr=revdescr) setfield_gc(p0, 101, descr=blendescr) p1 = int_add(p0, %(bdescr.basesize + 104)d) setfield_gc(p1, 8765, descr=tiddescr) - stm_set_revision_gc(p1, descr=revdescr) setfield_gc(p1, 102, descr=blendescr) p2 = call_malloc_nursery( \ %(bdescr.basesize + 104)d) setfield_gc(p2, 8765, descr=tiddescr) - stm_set_revision_gc(p2, descr=revdescr) setfield_gc(p2, 103, descr=blendescr) """) @@ -1051,7 +1027,6 @@ [p1] p0 = call_malloc_nursery(104) # rounded up setfield_gc(p0, 9315, descr=tiddescr) - stm_set_revision_gc(p0, descr=revdescr) setfield_gc(p0, ConstClass(o_vtable), descr=vtable_descr) """) @@ -1080,11 +1055,9 @@ %(strdescr.basesize + 16 * strdescr.itemsize + \ unicodedescr.basesize + 10 * unicodedescr.itemsize)d) setfield_gc(p0, %(strdescr.tid)d, descr=tiddescr) - stm_set_revision_gc(p0, descr=revdescr) setfield_gc(p0, 14, descr=strlendescr) p1 = int_add(p0, %(strdescr.basesize + 16 * strdescr.itemsize)d) setfield_gc(p1, %(unicodedescr.tid)d, descr=tiddescr) - stm_set_revision_gc(p1, descr=revdescr) setfield_gc(p1, 10, descr=unicodelendescr) p2 = call_malloc_nursery_varsize(2, 4, i2, \ descr=unicodedescr) @@ -1105,10 +1078,9 @@ p1 = call_malloc_nursery( \ %(cdescr.basesize + 5 * cdescr.itemsize)d) setfield_gc(p1, 8111, descr=tiddescr) - stm_set_revision_gc(p1, descr=revdescr) setfield_gc(p1, 5, descr=clendescr) label(p1, i2, p3) - cond_call_stm_b(p1, descr=A2Wdescr) + cond_call_gc_wb(p1, descr=wbdescr) setarrayitem_gc(p1, i2, p3, descr=cdescr) """) @@ -1132,11 +1104,9 @@ p0 = call_malloc_nursery( \ %(2 * (bdescr.basesize + 8))d) setfield_gc(p0, 8765, descr=tiddescr) - stm_set_revision_gc(p0, descr=revdescr) setfield_gc(p0, 5, descr=blendescr) p1 = int_add(p0, %(bdescr.basesize + 8)d) setfield_gc(p1, 8765, descr=tiddescr) - stm_set_revision_gc(p1, descr=revdescr) setfield_gc(p1, 5, descr=blendescr) stm_transaction_break(1) @@ -1144,7 +1114,6 @@ p2 = call_malloc_nursery( \ %(bdescr.basesize + 8)d) setfield_gc(p2, 8765, descr=tiddescr) - stm_set_revision_gc(p2, descr=revdescr) setfield_gc(p2, 5, descr=blendescr) """, calldescr2=calldescr2) diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -865,6 +865,7 @@ self.rm.possibly_free_var(tmp_box) # if gc_ll_descr.stm: + xxxxxx self.assembler.malloc_cond_varsize_frame_stm(sizeloc, gcmap) else: self.assembler.malloc_cond_varsize_frame( From noreply at buildbot.pypy.org Sat Mar 22 13:05:10 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 22 Mar 2014 13:05:10 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: progress Message-ID: <20140322120510.7207F1C0483@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r70166:388415f649a5 Date: 2014-03-22 13:04 +0100 http://bitbucket.org/pypy/pypy/changeset/388415f649a5/ Log: progress diff --git a/rpython/jit/backend/llsupport/test/test_stmrewrite.py b/rpython/jit/backend/llsupport/test/test_stmrewrite.py --- a/rpython/jit/backend/llsupport/test/test_stmrewrite.py +++ b/rpython/jit/backend/llsupport/test/test_stmrewrite.py @@ -1133,98 +1133,23 @@ V = lltype.GcArray(('z', lltype.Ptr(S)), hints=imm_hint) vdescr = get_array_descr(self.gc_ll_descr, V) vdescr.tid = 1233 - vzdescr = get_interiorfield_descr(self.gc_ll_descr, V, 'z') + #vzdescr = get_interiorfield_descr(self.gc_ll_descr, V, 'z') - # XXX: "A2Idescr" if imm_hint else "A2Rdescr" - barr = "A2Rdescr" if imm_hint else "A2Rdescr" + if imm_hint: + d = {'comment': '#', 'pure': '_pure'} + else: + d = {'comment': '', 'pure': ''} + self.check_rewrite(""" [p1, p3, i1, p4] - p2 = getfield_gc(p1, descr=uxdescr) - i3 = getinteriorfield_gc(p3, i1, descr=vzdescr) - i4 = getarrayitem_gc(p4, i3, descr=vdescr) + p2 = getfield_gc%(pure)s(p1, descr=uxdescr) + i4 = getarrayitem_gc%(pure)s(p4, i1, descr=vdescr) jump(p2) - """, """ + """ % d, """ [p1, p3, i1, p4] - cond_call_stm_b(p1, descr=%s) - p2 = getfield_gc(p1, descr=uxdescr) - cond_call_stm_b(p3, descr=%s) - i3 = getinteriorfield_gc(p3, i1, descr=vzdescr) - cond_call_stm_b(p4, descr=%s) - i4 = getarrayitem_gc(p4, i3, descr=vdescr) - + p2 = getfield_gc%(pure)s(p1, descr=uxdescr) + %(comment)s stm_read(p1) + i4 = getarrayitem_gc%(pure)s(p4, i1, descr=vdescr) + %(comment)s stm_read(p4) jump(p2) - """ % (barr, barr, barr), uxdescr=uxdescr, - vzdescr=vzdescr, vdescr=vdescr) - - def test_noptr_setfields(self): - S = lltype.GcStruct('S') - U = lltype.GcStruct('U', - ('x', lltype.Signed), - ('y', lltype.Ptr(S))) - udescr = get_size_descr(self.gc_ll_descr, U) - udescr.tid = 2123 - uxdescr = get_field_descr(self.gc_ll_descr, U, 'x') - #uydescr = get_field_descr(self.gc_ll_descr, U, 'y') - - V = lltype.GcArray(('z', lltype.Signed)) - vdescr = get_array_descr(self.gc_ll_descr, V) - vdescr.tid = 1233 - vzdescr = get_interiorfield_descr(self.gc_ll_descr, V, 'z') - - self.check_rewrite(""" - [p1, p3, i1, p4] - setfield_gc(p1, 1, descr=uxdescr) - setinteriorfield_gc(p3, i1, 1, descr=vzdescr) - setarrayitem_gc(p4, i1, 1, descr=vdescr) - jump(p3) - """, """ - [p1, p3, i1, p4] - cond_call_stm_b(p1, descr=A2Vdescr) - setfield_gc(p1, 1, descr=uxdescr) - cond_call_stm_b(p3, descr=A2Vdescr) - setinteriorfield_gc(p3, i1, 1, descr=vzdescr) - cond_call_stm_b(p4, descr=A2Vdescr) - setarrayitem_gc(p4, i1, 1, descr=vdescr) - - jump(p3) - """, uxdescr=uxdescr, vzdescr=vzdescr, vdescr=vdescr) - - def test_weaken_previous_barrier(self): - class fakeextrainfo: - oopspecindex=0 - def call_needs_inevitable(self): - return False - T = rffi.CArrayPtr(rffi.TIME_T) - calldescr2 = get_call_descr(self.gc_ll_descr, [T], rffi.TIME_T, - fakeextrainfo()) - - # True: weaken previous barrier - # False: do not weaken - ops = [("stm_transaction_break(1)", False), - ("call(123, descr=cd)", False), - ("label()", False), - ("i2 = int_add(i1, 1)", True) - ] - for op, weaken in ops: - b1 = ("cond_call_stm_b(p1, descr=A2Vdescr)" if weaken - else "cond_call_stm_b(p1, descr=A2Rdescr)") - b2 = ("" if weaken - else "cond_call_stm_b(p1, descr=A2Vdescr)") - self.check_rewrite(""" - [p1, i3] - i1 = getfield_gc(p1, descr=tydescr) # noptr - %s - setfield_gc(p1, i3, descr=tydescr) # noptr - jump(p1) - """ % (op,), """ - [p1, i3] - %s - i1 = getfield_gc(p1, descr=tydescr) - %s - %s - setfield_gc(p1, i3, descr=tydescr) - - jump(p1) - """ % (b1, op, b2), cd=calldescr2) - - + """ % d, uxdescr=uxdescr, vdescr=vdescr) From noreply at buildbot.pypy.org Sat Mar 22 17:18:09 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 22 Mar 2014 17:18:09 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: in-progress Message-ID: <20140322161809.072D91C0290@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r70167:813983a8da84 Date: 2014-03-22 16:11 +0100 http://bitbucket.org/pypy/pypy/changeset/813983a8da84/ Log: in-progress diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py --- a/rpython/jit/backend/llsupport/assembler.py +++ b/rpython/jit/backend/llsupport/assembler.py @@ -79,34 +79,23 @@ else: self.gc_size_of_header = WORD # for tests self.memcpy_addr = self.cpu.cast_ptr_to_int(memcpy_fn) - if gc_ll_descr.stm: - descrs = [gc_ll_descr.A2Rdescr, gc_ll_descr.Q2Rdescr, - gc_ll_descr.A2Idescr, gc_ll_descr.A2Vdescr, - gc_ll_descr.A2Wdescr, gc_ll_descr.V2Wdescr] - else: - descrs = [gc_ll_descr.write_barrier_descr] - for d in descrs: - self._build_b_slowpath(d, False) - self._build_b_slowpath(d, True) - self._build_b_slowpath(d, False, for_frame=True) # building the barriers needs to happen before these: self._build_failure_recovery(False, withfloats=False) self._build_failure_recovery(True, withfloats=False) + self._build_wb_slowpath(False) + self._build_wb_slowpath(True) + self._build_wb_slowpath(False, for_frame=True) # only for stm: if gc_ll_descr.stm: - self._build_ptr_eq_slowpath() self._build_stm_longjmp_callback() self.stm_transaction_break_path = self._build_stm_transaction_break_path() - else: - self.ptr_eq_slowpath = None # only one of those self.build_frame_realloc_slowpath() if self.cpu.supports_floats: self._build_failure_recovery(False, withfloats=True) self._build_failure_recovery(True, withfloats=True) - for d in descrs: - self._build_b_slowpath(d, False, withfloats=True) - self._build_b_slowpath(d, True, withfloats=True) + self._build_wb_slowpath(False, withfloats=True) + self._build_wb_slowpath(True, withfloats=True) self._build_propagate_exception_path() if gc_ll_descr.get_malloc_slowpath_addr() is not None: @@ -346,12 +335,14 @@ next.prev = prev @staticmethod + @rgc.no_collect def _release_gil_shadowstack(): before = rffi.aroundstate.before if before: before() @staticmethod + @rgc.no_collect def _reacquire_gil_shadowstack(): after = rffi.aroundstate.after if after: diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py --- a/rpython/jit/backend/llsupport/descr.py +++ b/rpython/jit/backend/llsupport/descr.py @@ -160,6 +160,11 @@ cachedict[fieldname] = fielddescr return fielddescr +def build_stm_tid_field_descr(): + from rpython.rlib import rstm + return FieldDescr('tid', rstm.tid_offset, rffi.sizeof(rstm.TID), + get_type_flag(rstm.TID), False, True) + def get_type_flag(TYPE): if isinstance(TYPE, lltype.Ptr): if TYPE.TO._gckind == 'gc': diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py --- a/rpython/jit/backend/llsupport/gc.py +++ b/rpython/jit/backend/llsupport/gc.py @@ -16,6 +16,7 @@ from rpython.jit.backend.llsupport.descr import GcCache, get_field_descr from rpython.jit.backend.llsupport.descr import get_array_descr from rpython.jit.backend.llsupport.descr import get_call_descr +from rpython.jit.backend.llsupport.descr import build_stm_tid_field_descr from rpython.memory.gctransform import asmgcroot # ____________________________________________________________ @@ -418,12 +419,12 @@ assert self.GCClass.inline_simple_malloc_varsize def _setup_tid(self): + "NOT_RPYTHON" if not self.stm: - self.fielddescr_tid = get_field_descr(self, self.GCClass.HDR, 'tid') + self.fielddescr_tid = get_field_descr(self, self.GCClass.HDR, + 'tid') else: - xxxxxxxx - self.fielddescr_tid = get_field_descr(self, self.GCClass.GCHDR, - 'h_tid') + self.fielddescr_tid = build_stm_tid_field_descr() frame_tid = self.layoutbuilder.get_type_id(jitframe.JITFRAME) self.translator._jit2gc['frame_tid'] = frame_tid diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py --- a/rpython/jit/backend/llsupport/llmodel.py +++ b/rpython/jit/backend/llsupport/llmodel.py @@ -39,8 +39,6 @@ if translator and translator.config.translation.gcremovetypeptr: self.vtable_offset = None else: - assert not self.gc_ll_descr.stm, """doesn't work in stm - because it would need a read barrier when reading typeptr""" self.vtable_offset, _ = symbolic.get_field_token(rclass.OBJECT, 'typeptr', translate_support_code) diff --git a/rpython/jit/backend/llsupport/regalloc.py b/rpython/jit/backend/llsupport/regalloc.py --- a/rpython/jit/backend/llsupport/regalloc.py +++ b/rpython/jit/backend/llsupport/regalloc.py @@ -396,12 +396,12 @@ self.reg_bindings[v] = loc return loc - def update_spill_loc_if_necessary(self, var, current_loc): - """if variable var is in two locations (spilled and current_loc), - update spilled location with current_loc""" - spill_loc = self.frame_manager.get(var) - if spill_loc: - self.assembler.regalloc_mov(current_loc, spill_loc) + #def update_spill_loc_if_necessary(self, var, current_loc): + # """if variable var is in two locations (spilled and current_loc), + # update spilled location with current_loc""" + # spill_loc = self.frame_manager.get(var) + # if spill_loc: + # self.assembler.regalloc_mov(current_loc, spill_loc) def _spill_var(self, v, forbidden_vars, selected_reg, need_lower_byte=False): diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -256,8 +256,7 @@ mallocs. (For all I know this latter case never occurs in practice, but better safe than sorry.) """ - if self.gc_ll_descr.fielddescr_tid is not None \ - or self.gc_ll_descr.stm: # framework GC + if self.gc_ll_descr.fielddescr_tid is not None: assert (size & (WORD-1)) == 0, "size not aligned?" addr = self.gc_ll_descr.get_malloc_fn_addr('malloc_big_fixedsize') args = [ConstInt(addr), ConstInt(size), ConstInt(typeid)] diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -36,8 +36,6 @@ from rpython.rlib.objectmodel import compute_unique_id from rpython.jit.backend.x86 import stmtlocal from rpython.rlib import rstm -from rpython.memory.gc.stmgc import StmGC -from rpython.jit.backend.llsupport.gc import STMBarrierDescr class Assembler386(BaseAssembler): @@ -55,6 +53,7 @@ self.float_const_abs_addr = 0 self.malloc_slowpath = 0 self.malloc_slowpath_varsize = 0 + self.wb_slowpath = [0, 0, 0, 0, 0] self.setup_failure_recovery() self.datablockwrapper = None self.stack_check_slowpath = 0 @@ -423,101 +422,17 @@ rawstart = mc.materialize(self.cpu.asmmemmgr, []) self.stack_check_slowpath = rawstart - - def _build_ptr_eq_slowpath(self): - cpu = self.cpu - assert cpu.gc_ll_descr.stm - # - # SYNCHRONIZE WITH extra.c'S IMPLEMENTATION! - # - # This builds a helper function called from the slow path of - # ptr_eq/ne. It must save all registers, and optionally - # all XMM registers. It takes two values pushed on the stack, - # even on X86_64. It must restore stack alignment accordingly. - mc = codebuf.MachineCodeBlockWrapper() - # - # we want 2 registers: - mc.PUSH_r(esi.value) - mc.PUSH_r(edi.value) - # - # get arguments: ||val2|val1||retaddr|esi||edi| - mc.MOV_rs(esi.value, 3 * WORD) - mc.MOV_rs(edi.value, 4 * WORD) - # - # the fastpath checks if val1==val2 or any of them is NULL - # thus, we only have to get to their h_original - # if they are *not* PREBUILT_ORIGINALS - # - flag = StmGC.GCFLAG_PREBUILT_ORIGINAL - assert (flag >> 32) > 0 and (flag >> 40) == 0 - flag = flag >> 32 - off = 4 - # if !(val1->h_original), leave EDI as is - mc.MOV_rm(X86_64_SCRATCH_REG.value, (edi.value, StmGC.H_ORIGINAL)) - mc.TEST_rr(X86_64_SCRATCH_REG.value, X86_64_SCRATCH_REG.value) - mc.J_il8(rx86.Conditions['Z'], 0) - z1_location = mc.get_relative_pos() - # if val1->h_tid & PREBUILT_ORIGINAL, take h_original - mc.TEST8_mi((edi.value, StmGC.H_TID + off), flag) - mc.CMOVE_rr(edi.value, X86_64_SCRATCH_REG.value) - # - # Do the same for val2=ESI - offset = mc.get_relative_pos() - z1_location - assert 0 < offset <= 127 - mc.overwrite(z1_location - 1, chr(offset)) - # if !(val2->h_original), leave ESI as is - mc.MOV_rm(X86_64_SCRATCH_REG.value, (esi.value, StmGC.H_ORIGINAL)) - mc.TEST_rr(X86_64_SCRATCH_REG.value, X86_64_SCRATCH_REG.value) - mc.J_il8(rx86.Conditions['Z'], 0) - z2_location = mc.get_relative_pos() - # if val2->h_tid & PREBUILT_ORIGINAL, take h_original - mc.TEST8_mi((esi.value, StmGC.H_TID + off), flag) - mc.CMOVE_rr(esi.value, X86_64_SCRATCH_REG.value) - # - # COMPARE - offset = mc.get_relative_pos() - z2_location - assert 0 < offset <= 127 - mc.overwrite(z2_location - 1, chr(offset)) - # - mc.CMP_rr(edi.value, esi.value) - sl = X86_64_SCRATCH_REG.lowest8bits() - mc.SET_ir(rx86.Conditions['Z'], sl.value) - # mov result to val2 on stack - # ||val2|val1||retaddr|esi||edi| - mc.MOV_sr(4 * WORD, X86_64_SCRATCH_REG.value) - # - # Restore everything: - mc.POP_r(edi.value) - mc.POP_r(esi.value) - # ||result|val1|retaddr| - # - # - # only remove one arg: - mc.RET16_i(1 * WORD) - - rawstart = mc.materialize(self.cpu.asmmemmgr, []) - self.ptr_eq_slowpath = rawstart - - - def _build_b_slowpath(self, descr, withcards, withfloats=False, - for_frame=False): - is_stm = self.cpu.gc_ll_descr.stm + def _build_wb_slowpath(self, withcards, withfloats=False, for_frame=False): + descr = self.cpu.gc_ll_descr.write_barrier_descr exc0, exc1 = None, None if descr is None: return - - if is_stm and withcards: - return - if not withcards: - func = descr.get_barrier_fn(self.cpu, - returns_modified_object=is_stm) - assert func is not None + func = descr.get_write_barrier_fn(self.cpu) else: - assert not is_stm if descr.jit_wb_cards_set == 0: return - func = descr.get_barrier_from_array_fn(self.cpu) + func = descr.get_write_barrier_from_array_fn(self.cpu) if func == 0: return # @@ -529,53 +444,20 @@ mc = codebuf.MachineCodeBlockWrapper() # if not for_frame: - if descr.stmcat in ['A2W', 'A2V']: - # slow fastpath - # check if PRIV_FROM_PROT is set, but not - # WRITE_BARRIER - mc.MOV_rs(X86_64_SCRATCH_REG.value, WORD) - - flag = StmGC.GCFLAG_WRITE_BARRIER >> 32 - off = 4 - assert 0 < flag < 256 - mc.TEST8_mi((X86_64_SCRATCH_REG.value, off), flag) - mc.J_il8(rx86.Conditions['NZ'], 0) - jz1 = mc.get_relative_pos() - # if flag set, jump over the next check & RET - - flag = StmGC.GCFLAG_PRIVATE_FROM_PROTECTED >> 40 - off = 5 - assert 0 < flag < 256 - mc.TEST8_mi((X86_64_SCRATCH_REG.value, off), flag) - mc.J_il8(rx86.Conditions['Z'], 0) - jz2 = mc.get_relative_pos() - # if PRIV_F_PROT, RET - mc.RET() - mc.overwrite(jz2 - 1, chr(mc.get_relative_pos() - jz2)) - mc.overwrite(jz1 - 1, chr(mc.get_relative_pos() - jz1)) - self._push_all_regs_to_frame(mc, [], withfloats, callee_only=True) if IS_X86_32: # we have 2 extra words on stack for retval and we pass 1 extra # arg, so we need to substract 2 words - # ||val|retadr| mc.SUB_ri(esp.value, 2 * WORD) - # ||val|retadr|x|x|| mc.MOV_rs(eax.value, 3 * WORD) # 2 + 1 mc.MOV_sr(0, eax.value) - # ||val|retadr|x|val|| else: - # ||val|retadr|| mc.MOV_rs(edi.value, WORD) else: - # ||retadr| # we have one word to align mc.SUB_ri(esp.value, 7 * WORD) # align and reserve some space - # ||retadr|x||x|x||x|x||x|x|| mc.MOV_sr(WORD, eax.value) # save for later - # ||retadr|x||x|x||x|x||rax|x|| mc.MOVSD_sx(3 * WORD, xmm0.value) - # ||retadr|x||x|x||xmm0|x||rax|x|| if IS_X86_32: mc.MOV_sr(4 * WORD, edx.value) mc.MOV_sr(0, ebp.value) @@ -592,63 +474,41 @@ self._store_and_reset_exception(mc, exc0, exc1) mc.CALL(imm(func)) - - if descr.returns_modified_object: - # new addr in eax, save to now unused arg - if for_frame: - # ||retadr|x||x|x||xmm0|x||rax|x|| - # directly move to rbp - mc.MOV_rr(ebp.value, eax.value) - elif IS_X86_32: - mc.MOV_sr(3 * WORD, eax.value) - # ||val|retadr|x|val|| - # -> ||result|retaddr|x|val|| - else: - mc.MOV_sr(WORD, eax.value) - # ||val|retadr|| -> ||result|retadr|| - + # if withcards: # A final TEST8 before the RET, for the caller. Careful to # not follow this instruction with another one that changes # the status of the CPU flags! - assert not is_stm and not descr.returns_modified_object if IS_X86_32: - mc.MOV_rs(eax.value, 3 * WORD) + mc.MOV_rs(eax.value, 3*WORD) else: mc.MOV_rs(eax.value, WORD) mc.TEST8(addr_add_const(eax, descr.jit_wb_if_flag_byteofs), imm(-0x80)) # + if not for_frame: if IS_X86_32: # ADD touches CPU flags mc.LEA_rs(esp.value, 2 * WORD) self._pop_all_regs_from_frame(mc, [], withfloats, callee_only=True) - - if descr.returns_modified_object: - # preserve argument which now holds the result - mc.RET() - else: - mc.RET16_i(WORD) + mc.RET16_i(WORD) else: if IS_X86_32: - mc.MOV_rs(edx.value, 5 * WORD) - # ||retadr|x||x|x||xmm0|x||rax|x|| + mc.MOV_rs(edx.value, 4 * WORD) mc.MOVSD_xs(xmm0.value, 3 * WORD) mc.MOV_rs(eax.value, WORD) # restore self._restore_exception(mc, exc0, exc1) mc.MOV(exc0, RawEspLoc(WORD * 5, REF)) mc.MOV(exc1, RawEspLoc(WORD * 6, INT)) - mc.LEA_rs(esp.value, 7 * WORD) - # retval already in ebp mc.RET() rawstart = mc.materialize(self.cpu.asmmemmgr, []) if for_frame: - descr.set_b_slowpath(4, rawstart) + self.wb_slowpath[4] = rawstart else: - descr.set_b_slowpath(withcards + 2 * withfloats, rawstart) + self.wb_slowpath[withcards + 2 * withfloats] = rawstart def _build_stm_longjmp_callback(self): @@ -2598,134 +2458,7 @@ offset = mc.get_relative_pos() - jmp2_adr assert 0 < offset <= 127 mc.overwrite(jmp2_adr-1, chr(offset)) - - def malloc_cond_stm(self, size, gcmap): - assert self.cpu.gc_ll_descr.stm - assert size & (WORD-1) == 0 # must be correctly aligned - mc = self.mc - # load nursery_current and nursery_nextlimit - nc = self._get_stm_tl(rstm.get_nursery_current_adr()) - self._tl_segment_if_stm(mc) - mc.MOV_rj(eax.value, nc) - # - mc.LEA_rm(edi.value, (eax.value, size)) - # - # eax=nursery_current, edi=nursery_current+size - self._cond_allocate_in_nursery_or_slowpath(mc, gcmap) - def malloc_cond_varsize_frame_stm(self, sizeloc, gcmap): - assert self.cpu.gc_ll_descr.stm - mc = self.mc - if sizeloc is eax: - self.mc.MOV(edi, sizeloc) - sizeloc = edi - - nc = self._get_stm_tl(rstm.get_nursery_current_adr()) - self._tl_segment_if_stm(mc) - mc.MOV_rj(eax.value, nc) - - if sizeloc is edi: - self.mc.ADD_rr(edi.value, eax.value) - else: - self.mc.LEA_ra(edi.value, (eax.value, sizeloc.value, 0, 0)) - # - # eax=nursery_current, edi=nursery_current+size - self._cond_allocate_in_nursery_or_slowpath(mc, gcmap) - - def malloc_cond_varsize_stm(self, kind, lengthloc, itemsize, - maxlength, gcmap, arraydescr): - assert self.cpu.gc_ll_descr.stm - from rpython.jit.backend.llsupport.descr import ArrayDescr - assert isinstance(arraydescr, ArrayDescr) - - mc = self.mc - nc = self._get_stm_tl(rstm.get_nursery_current_adr()) - nnl = self._get_stm_tl(rstm.get_nursery_nextlimit_adr()) - - # lengthloc is the length of the array, which we must not modify! - assert lengthloc is not eax and lengthloc is not edi - if isinstance(lengthloc, RegLoc): - varsizeloc = lengthloc - else: - mc.MOV(edi, lengthloc) - varsizeloc = edi - - mc.CMP(varsizeloc, imm(maxlength)) - mc.J_il8(rx86.Conditions['A'], 0) # patched later - jmp_adr0 = mc.get_relative_pos() - - self._tl_segment_if_stm(mc) - mc.MOV_rj(eax.value, nc) - - if valid_addressing_size(itemsize): - shift = get_scale(itemsize) - else: - shift = self._imul_const_scaled(mc, edi.value, - varsizeloc.value, itemsize) - varsizeloc = edi - # now varsizeloc is a register != eax. The size of - # the variable part of the array is (varsizeloc << shift) - assert arraydescr.basesize >= self.gc_minimal_size_in_nursery - constsize = arraydescr.basesize + self.gc_size_of_header - force_realignment = (itemsize % WORD) != 0 - if force_realignment: - constsize += WORD - 1 - mc.LEA_ra(edi.value, (eax.value, varsizeloc.value, shift, - constsize)) - if force_realignment: - mc.AND_ri(edi.value, ~(WORD - 1)) - # now edi contains the total size in bytes, rounded up to a multiple - # of WORD, plus nursery_free_adr - self._tl_segment_if_stm(mc) - mc.CMP_rj(edi.value, nnl) - mc.J_il8(rx86.Conditions['NA'], 0) # patched later - jmp_adr1 = mc.get_relative_pos() - # - # == SLOWPATH == - offset = mc.get_relative_pos() - jmp_adr0 - assert 0 < offset <= 127 - mc.overwrite(jmp_adr0-1, chr(offset)) - # save the gcmap - self.push_gcmap(mc, gcmap, mov=True) # mov into RawEspLoc(0) - if kind == rewrite.FLAG_ARRAY: - mc.MOV_si(WORD, itemsize) - mc.MOV(edi, lengthloc) - mc.MOV_ri(eax.value, arraydescr.tid) - addr = self.malloc_slowpath_varsize - else: - if kind == rewrite.FLAG_STR: - addr = self.malloc_slowpath_str - else: - assert kind == rewrite.FLAG_UNICODE - addr = self.malloc_slowpath_unicode - mc.MOV(edi, lengthloc) - mc.CALL(imm(addr)) - mc.JMP_l8(0) # jump to done, patched later - jmp_location = mc.get_relative_pos() - # - # == FASTPATH == - offset = mc.get_relative_pos() - jmp_adr1 - assert 0 < offset <= 127 - mc.overwrite(jmp_adr1-1, chr(offset)) - # - # set stm_nursery_current - self._tl_segment_if_stm(mc) - mc.MOV_jr(nc, edi.value) - # - # write down the tid - mc.MOV(mem(eax, 0), imm(arraydescr.tid)) - # also set private_rev_num: - rn = self._get_stm_private_rev_num_addr() - self._tl_segment_if_stm(mc) - mc.MOV_rj(X86_64_SCRATCH_REG.value, rn) - mc.MOV(mem(eax, StmGC.H_REVISION), X86_64_SCRATCH_REG) - # - # == END == - offset = mc.get_relative_pos() - jmp_location - assert 0 < offset <= 127 - mc.overwrite(jmp_location - 1, chr(offset)) - - def malloc_cond(self, nursery_free_adr, nursery_top_adr, size, gcmap): assert not self.cpu.gc_ll_descr.stm assert size & (WORD-1) == 0 # must be correctly aligned @@ -2742,15 +2475,20 @@ self.mc.overwrite(jmp_adr-1, chr(offset)) self.mc.MOV(heap(nursery_free_adr), edi) - def malloc_cond_varsize_frame(self, nursery_free_adr, nursery_top_adr, sizeloc, gcmap): - assert not self.cpu.gc_ll_descr.stm + # 'sizeloc' is the size in bytes if not STM; and the length of + # the array to allocate if STM if sizeloc is eax: self.mc.MOV(edi, sizeloc) sizeloc = edi self.mc.MOV(eax, heap(nursery_free_adr)) - if sizeloc is edi: + if self.cpu.gc_ll_descr.stm: + constsize = self.cpu.get_baseofs_of_frame_field() + shift = get_scale(WORD) + self.mc.LEA_ra(edi.value, (eax.value, sizeloc.value, shift, + constsize)) + elif sizeloc is edi: self.mc.ADD_rr(edi.value, eax.value) else: self.mc.LEA_ra(edi.value, (eax.value, sizeloc.value, 0, 0)) @@ -2914,7 +2652,29 @@ self._emit_guard_not_forced(guard_token) - + def genop_discard_stm_read(self, op, arglocs): + assert IS_X86_64, "needed for X86_64_SCRATCH_REG" + mc = self.mc + rm8reg = X86_64_SCRATCH_REG.value | BYTE_REG_FLAG + xxxxxx #load STM_SEGMENT->transaction_read_version into rm8reg + loc_src, loc_tmp = arglocs + if tmp_loc is None: + assert isinstance(loc_src, ImmedLoc) + assert loc_src.value > 0 + mem = loc_src.value >> 4 + assert rx86.fits_in_32bits(mem) + tl_segment_prefix(mc) + mc.MOV8_jr(mem, rm8reg) + else: + assert isinstance(loc_tmp, RegLoc) + if isinstance(loc_src, ImmedLoc): + mc.MOV_ri(loc_tmp.value, loc_src.value >> 4) + else: + if loc_tmp is not loc_src: + mc.MOV(loc_tmp, loc_src) + mc.SHR_ri(loc_tmp.value, 4) + tl_segment_prefix(mc) + mc.MOV8_mr((loc_tmp.value, 0), rm8reg) genop_discard_list = [Assembler386.not_implemented_op_discard] * rop._LAST diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -864,14 +864,10 @@ gcmap = self.get_gcmap([eax, edi]) # allocate the gcmap *before* self.rm.possibly_free_var(tmp_box) # - if gc_ll_descr.stm: - xxxxxx - self.assembler.malloc_cond_varsize_frame_stm(sizeloc, gcmap) - else: - self.assembler.malloc_cond_varsize_frame( - gc_ll_descr.get_nursery_free_addr(), - gc_ll_descr.get_nursery_top_addr(), - sizeloc, gcmap) + self.assembler.malloc_cond_varsize_frame( + gc_ll_descr.get_nursery_free_addr(), + gc_ll_descr.get_nursery_top_addr(), + sizeloc, gcmap) def consider_call_malloc_nursery_varsize(self, op): gc_ll_descr = self.assembler.cpu.gc_ll_descr @@ -1369,6 +1365,20 @@ def consider_keepalive(self, op): pass + def consider_stm_read(self, op): + loc_src = self.loc(op.getarg(0)) + self.possibly_free_vars_for_op(op) + # this will get in 'loc_tmp' a register that is the same as + # 'loc_src' if the op.getarg(0) is freed now + if (isinstance(loc_src, ImmedLoc) and + rx86.fits_in_32bits(loc_src.value >> 4)): + loc_tmp = None + else: + tmpxvar = TempBox() + loc_tmp = self.rm.force_allocate_reg(tmpxvar) + self.rm.possibly_free_var(tmpxvar) + self.perform_discard(op, [loc_src, loc_tmp]) + def not_implemented_op(self, op): not_implemented("not implemented operation: %s" % op.getopname()) diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py --- a/rpython/rlib/rstm.py +++ b/rpython/rlib/rstm.py @@ -4,6 +4,11 @@ from rpython.rtyper.extregistry import ExtRegistryEntry from rpython.rlib.jit import dont_look_inside + +TID = rffi.UINT +tid_offset = CDefinedIntSymbolic('offsetof(struct rpyobj_s, tid)', default=4) + + @dont_look_inside def get_nursery_current_adr(): addr = llop.stm_get_adr_of_nursery_current(llmemory.Address) From noreply at buildbot.pypy.org Sat Mar 22 17:18:10 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 22 Mar 2014 17:18:10 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: in-progress Message-ID: <20140322161810.5E8BD1C0290@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r70168:c851d6840147 Date: 2014-03-22 17:15 +0100 http://bitbucket.org/pypy/pypy/changeset/c851d6840147/ Log: in-progress diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py --- a/rpython/jit/backend/llsupport/gc.py +++ b/rpython/jit/backend/llsupport/gc.py @@ -564,12 +564,20 @@ self.for_test_only.x = x0 + x1 + x2 + x3 def get_nursery_free_addr(self): - nurs_addr = llop.gc_adr_of_nursery_free(llmemory.Address) - return rffi.cast(lltype.Signed, nurs_addr) + if self.stm: + from rpython.rlib import rstm + return rstm.adr_nursery_free + else: + nurs_addr = llop.gc_adr_of_nursery_free(llmemory.Address) + return rffi.cast(lltype.Signed, nurs_addr) def get_nursery_top_addr(self): - nurs_top_addr = llop.gc_adr_of_nursery_top(llmemory.Address) - return rffi.cast(lltype.Signed, nurs_top_addr) + if self.stm: + from rpython.rlib import rstm + return rstm.adr_nursery_top + else: + nurs_top_addr = llop.gc_adr_of_nursery_top(llmemory.Address) + return rffi.cast(lltype.Signed, nurs_top_addr) def initialize(self): pass diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -73,7 +73,7 @@ if WORD == 8: self.pending_memoryerror_trampoline_from = [] self.error_trampoline_64 = 0 - self.mc = codebuf.MachineCodeBlockWrapper() + self.mc = codebuf.MachineCodeBlockWrapper(self.cpu) #assert self.datablockwrapper is None --- but obscure case # possible, e.g. getting MemoryError and continuing allblocks = self.get_asmmemmgr_blocks(looptoken) @@ -112,7 +112,7 @@ mc.MOV_bi(extra_ofs, value) def build_frame_realloc_slowpath(self): - mc = codebuf.MachineCodeBlockWrapper() + mc = codebuf.MachineCodeBlockWrapper(self.cpu) self._push_all_regs_to_frame(mc, [], self.cpu.supports_floats) # this is the gcmap stored by push_gcmap(mov=True) in _check_stack_frame mc.MOV_rs(ecx.value, WORD) @@ -156,7 +156,7 @@ """ This builds a general call slowpath, for whatever call happens to come. """ - mc = codebuf.MachineCodeBlockWrapper() + mc = codebuf.MachineCodeBlockWrapper(self.cpu) # copy registers to the frame, with the exception of the # 'cond_call_register_arguments' and eax, because these have already # been saved by the caller. Note that this is not symmetrical: @@ -196,7 +196,7 @@ This function does not have to preserve registers. It expects all registers to be saved in the caller. """ - mc = codebuf.MachineCodeBlockWrapper() + mc = codebuf.MachineCodeBlockWrapper(self.cpu) # store the gc pattern ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') mc.MOV_rs(ecx.value, WORD) @@ -250,7 +250,7 @@ This function must preserve all registers apart from eax and edi. """ assert kind in ['fixed', 'str', 'unicode', 'var'] - mc = codebuf.MachineCodeBlockWrapper() + mc = codebuf.MachineCodeBlockWrapper(self.cpu) self._push_all_regs_to_frame(mc, [eax, edi], self.cpu.supports_floats) # store the gc pattern ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') @@ -305,14 +305,9 @@ self._reload_frame_if_necessary(mc, align_stack=True) self.set_extra_stack_depth(mc, 0) self._pop_all_regs_from_frame(mc, [eax, edi], self.cpu.supports_floats) - if self.cpu.gc_ll_descr.stm: - # load nursery_current into EDI - nc = self._get_stm_tl(rstm.get_nursery_current_adr()) - self._tl_segment_if_stm(mc) - mc.MOV_rj(edi.value, nc) - else: - nursery_free_adr = self.cpu.gc_ll_descr.get_nursery_free_addr() - mc.MOV(edi, heap(nursery_free_adr)) # load this in EDI + nursery_free_adr = self.cpu.gc_ll_descr.get_nursery_free_addr() + mc.SEGC7() + mc.MOV(edi, heap(nursery_free_adr)) # load this in EDI # clear the gc pattern mc.MOV_bi(ofs, 0) mc.RET() @@ -333,7 +328,7 @@ if not self.cpu.propagate_exception_descr: return # not supported (for tests, or non-translated) # - self.mc = codebuf.MachineCodeBlockWrapper() + self.mc = codebuf.MachineCodeBlockWrapper(self.cpu) # # read and reset the current exception @@ -350,22 +345,6 @@ self.propagate_exception_path = rawstart self.mc = None - def _get_stm_tl(self, adr): - """Makes 'adr' relative to threadlocal-base if we run in STM. - Before using such a relative address, call _tl_segment_if_stm().""" - if self.cpu.gc_ll_descr.stm and we_are_translated(): - # only for STM and not during tests - result = adr - stmtlocal.threadlocal_base() - assert rx86.fits_in_32bits(result) - return result - return adr - - def _tl_segment_if_stm(self, mc): - """Insert segment prefix for thread-local memory if we run - in STM and not during testing.""" - if self.cpu.gc_ll_descr.stm and we_are_translated(): - stmtlocal.tl_segment_prefix(mc) - def _build_stack_check_slowpath(self): if self.cpu.gc_ll_descr.stm: return # XXX no stack check on STM for now @@ -381,7 +360,7 @@ # | my own retaddr | <-- esp # +---------------------+ # - mc = codebuf.MachineCodeBlockWrapper() + mc = codebuf.MachineCodeBlockWrapper(self.cpu) # if IS_X86_64: # on the x86_64, we have to save all the registers that may @@ -402,8 +381,8 @@ else: mc.ADD_ri(esp.value, WORD) # - ea = self._get_stm_tl(self.cpu.pos_exception()) - self._tl_segment_if_stm(mc) + ea = mc.in_tl_segment(self.cpu.pos_exception()) + mc.SEGTL() mc.MOV(eax, heap(ea)) mc.TEST_rr(eax.value, eax.value) mc.J_il8(rx86.Conditions['NZ'], 0) @@ -441,7 +420,7 @@ # all XMM registers. It takes a single argument just pushed # on the stack even on X86_64. It must restore stack alignment # accordingly. - mc = codebuf.MachineCodeBlockWrapper() + mc = codebuf.MachineCodeBlockWrapper(self.cpu) # if not for_frame: self._push_all_regs_to_frame(mc, [], withfloats, callee_only=True) @@ -518,7 +497,7 @@ # # make the stm_longjmp_callback() function, with signature # void (*longjmp_callback)(void *stm_resume_buffer) - mc = codebuf.MachineCodeBlockWrapper() + mc = codebuf.MachineCodeBlockWrapper(self.cpu) # # 'edi' contains the stm resume buffer, so the new stack # location that we have to enforce is 'edi - FRAME_FIXED_SIZE * WORD'. @@ -529,9 +508,9 @@ # # restore the shadowstack pointer from stm_resume_buffer[1] gcrootmap = self.cpu.gc_ll_descr.gcrootmap - rst = self._get_stm_tl(gcrootmap.get_root_stack_top_addr()) + rst = mc.in_tl_segment(gcrootmap.get_root_stack_top_addr()) mc.MOV_rs(eax.value, (FRAME_FIXED_SIZE + 1) * WORD) - self._tl_segment_if_stm(mc) + mc.SEGTL() mc.MOV_jr(rst, eax.value) # # must restore 'ebp' from its saved value in the shadowstack @@ -702,7 +681,7 @@ assert rx86.fits_in_32bits(relative_target) # if not tok.is_guard_not_invalidated: - mc = codebuf.MachineCodeBlockWrapper() + mc = codebuf.MachineCodeBlockWrapper(self.cpu) mc.writeimm32(relative_target) mc.copy_to_raw_memory(addr) else: @@ -727,7 +706,7 @@ if WORD == 8: for pos_after_jz in self.pending_memoryerror_trampoline_from: assert self.error_trampoline_64 != 0 # only if non-empty - mc = codebuf.MachineCodeBlockWrapper() + mc = codebuf.MachineCodeBlockWrapper(self.cpu) mc.writeimm32(self.error_trampoline_64 - pos_after_jz) mc.copy_to_raw_memory(rawstart + pos_after_jz - 4) @@ -786,7 +765,7 @@ self.frame_depth_to_patch.append(ofs2) def _patch_frame_depth(self, adr, allocated_depth): - mc = codebuf.MachineCodeBlockWrapper() + mc = codebuf.MachineCodeBlockWrapper(self.cpu) mc.writeimm32(allocated_depth) mc.copy_to_raw_memory(adr) @@ -811,7 +790,7 @@ # that. Otherwise, leave the original rel32 to the recovery stub in # place, but clobber the recovery stub with a jump to the real # target. - mc = codebuf.MachineCodeBlockWrapper() + mc = codebuf.MachineCodeBlockWrapper(self.cpu) if rx86.fits_in_32bits(offset): mc.writeimm32(offset) mc.copy_to_raw_memory(adr_jump_offset) @@ -894,7 +873,7 @@ fn = stmtlocal.stm_invalidate_jmp_buf_fn self.mc.CALL(imm(self.cpu.cast_ptr_to_int(fn))) # there could have been a collection in invalidate_jmp_buf() - self._reload_frame_if_necessary(self.mc) + self._reload_frame_if_necessary(self.mc, wb=False) # the return value is the jitframe self.mc.MOV_rr(eax.value, ebp.value) @@ -916,9 +895,9 @@ that gives the address of the stack top. If this integer doesn't fit in 32 bits, it will be loaded in r11. """ - rst = self._get_stm_tl(gcrootmap.get_root_stack_top_addr()) + rst = mc.in_tl_segment(gcrootmap.get_root_stack_top_addr()) if rx86.fits_in_32bits(rst): - self._tl_segment_if_stm(mc) + mc.SEGTL() mc.MOV_rj(ebx.value, rst) # MOV ebx, [rootstacktop] else: mc.MOV_ri(X86_64_SCRATCH_REG.value, rst) # MOV r11, rootstacktop @@ -934,9 +913,9 @@ rst = self._load_shadowstack_top_in_ebx(self.mc, gcrootmap) self.mc.MOV_mr((ebx.value, 0), ebp.value) # MOV [ebx], ebp self.mc.ADD_ri(ebx.value, WORD) - + if rx86.fits_in_32bits(rst): - self._tl_segment_if_stm(self.mc) + self.mc.SEGTL() self.mc.MOV_jr(rst, ebx.value) # MOV [rootstacktop], ebx else: # The integer 'rst' doesn't fit in 32 bits, so we know that @@ -946,9 +925,9 @@ ebx.value) # MOV [r11], ebx def _call_footer_shadowstack(self, gcrootmap): - rst = self._get_stm_tl(gcrootmap.get_root_stack_top_addr()) + rst = self.mc.in_tl_segment(gcrootmap.get_root_stack_top_addr()) if rx86.fits_in_32bits(rst): - self._tl_segment_if_stm(self.mc) + self.mc.SEGTL() self.mc.SUB_ji8(rst, WORD) # SUB [rootstacktop], WORD else: self.mc.MOV_ri(ebx.value, rst) # MOV ebx, rootstacktop @@ -968,7 +947,7 @@ baseofs = self.cpu.get_baseofs_of_frame_field() newlooptoken.compiled_loop_token.update_frame_info( oldlooptoken.compiled_loop_token, baseofs) - mc = codebuf.MachineCodeBlockWrapper() + mc = codebuf.MachineCodeBlockWrapper(self.cpu) mc.JMP(imm(target)) if WORD == 4: # keep in sync with prepare_loop() assert mc.get_relative_pos() == 5 @@ -1215,25 +1194,17 @@ cb = callbuilder.CallBuilder(self, fnloc, arglocs) cb.emit_no_collect() - def _reload_frame_if_necessary(self, mc, align_stack=False): + def _reload_frame_if_necessary(self, mc, align_stack=False, wb=True): gc_ll_descr = self.cpu.gc_ll_descr gcrootmap = gc_ll_descr.gcrootmap if gcrootmap and gcrootmap.is_shadow_stack: - rst = self._get_stm_tl(gcrootmap.get_root_stack_top_addr()) - self._tl_segment_if_stm(mc) + rst = mc.in_tl_segment(gcrootmap.get_root_stack_top_addr()) + mc.SEGTL() mc.MOV(ecx, heap(rst)) mc.MOV(ebp, mem(ecx, -WORD)) # - if gcrootmap and gcrootmap.is_stm: - if not hasattr(gc_ll_descr, 'A2Wdescr'): - raise Exception("unreachable code") - wbdescr = gc_ll_descr.A2Wdescr - self._stm_barrier_fastpath(mc, wbdescr, [ebp], is_frame=True, - align_stack=align_stack) - return - # wbdescr = gc_ll_descr.write_barrier_descr - if gcrootmap and wbdescr: + if gcrootmap and wbdescr and wb: # frame never uses card marking, so we enforce this is not # an array self._write_barrier_fastpath(mc, wbdescr, [ebp], array=False, @@ -1745,8 +1716,8 @@ def genop_guard_guard_no_exception(self, ign_1, guard_op, guard_token, locs, ign_2): - ea = self._get_stm_tl(self.cpu.pos_exception()) - self._tl_segment_if_stm(self.mc) + ea = self.mc.in_tl_segment(self.cpu.pos_exception()) + self.mc.SEGTL() self.mc.CMP(heap(ea), imm0) self.implement_guard(guard_token, 'NZ') @@ -1760,8 +1731,8 @@ locs, resloc): loc = locs[0] loc1 = locs[1] - ea = self._get_stm_tl(self.cpu.pos_exception()) - self._tl_segment_if_stm(self.mc) + ea = self.mc.in_tl_segment(self.cpu.pos_exception()) + self.mc.SEGTL() self.mc.MOV(loc1, heap(ea)) self.mc.CMP(loc1, loc) self.implement_guard(guard_token, 'NE') @@ -1772,42 +1743,43 @@ """ Resest the exception. If excvalloc is None, then store it on the frame in jf_guard_exc """ - eva = self._get_stm_tl(self.cpu.pos_exc_value()) - ea = self._get_stm_tl(self.cpu.pos_exception()) + eva = mc.in_tl_segment(self.cpu.pos_exc_value()) + ea = mc.in_tl_segment(self.cpu.pos_exception()) # - self._tl_segment_if_stm(mc) if excvalloc is not None: assert excvalloc.is_core_reg() + mc.SEGTL() mc.MOV(excvalloc, heap(eva)) elif tmploc is not None: # if both are None, just ignore ofs = self.cpu.get_ofs_of_frame_field('jf_guard_exc') + mc.SEGTL() mc.MOV(tmploc, heap(eva)) mc.MOV(RawEbpLoc(ofs), tmploc) # if exctploc is not None: assert exctploc.is_core_reg() - self._tl_segment_if_stm(mc) + mc.SEGTL() mc.MOV(exctploc, heap(ea)) # - self._tl_segment_if_stm(mc) + mc.SEGTL() mc.MOV(heap(ea), imm0) - self._tl_segment_if_stm(mc) + mc.SEGTL() mc.MOV(heap(eva), imm0) def _restore_exception(self, mc, excvalloc, exctploc, tmploc=None): - eva = self._get_stm_tl(self.cpu.pos_exc_value()) - ea = self._get_stm_tl(self.cpu.pos_exception()) + eva = mc.in_tl_segment(self.cpu.pos_exc_value()) + ea = mc.in_tl_segment(self.cpu.pos_exception()) if excvalloc is not None: - self._tl_segment_if_stm(mc) + mc.SEGTL() mc.MOV(heap(eva), excvalloc) else: assert tmploc is not None ofs = self.cpu.get_ofs_of_frame_field('jf_guard_exc') mc.MOV(tmploc, RawEbpLoc(ofs)) mc.MOV_bi(ofs, 0) - self._tl_segment_if_stm(mc) + mc.SEGTL() mc.MOV(heap(eva), tmploc) - self._tl_segment_if_stm(mc) + mc.SEGTL() mc.MOV(heap(ea), exctploc) def _gen_guard_overflow(self, guard_op, guard_token): @@ -2000,20 +1972,20 @@ mc.MOVSD_xb(i, (ofs + i * coeff) * WORD + base_ofs) def _build_failure_recovery(self, exc, withfloats=False): - mc = codebuf.MachineCodeBlockWrapper() + mc = codebuf.MachineCodeBlockWrapper(self.cpu) self.mc = mc self._push_all_regs_to_frame(mc, [], withfloats) if exc: # We might have an exception pending. Load it into ebx... - eva = self._get_stm_tl(self.cpu.pos_exc_value()) - ea = self._get_stm_tl(self.cpu.pos_exception()) - self._tl_segment_if_stm(mc) + eva = mc.in_tl_segment(self.cpu.pos_exc_value()) + ea = mc.in_tl_segment(self.cpu.pos_exception()) + mc.SEGTL() mc.MOV(ebx, heap(eva)) - self._tl_segment_if_stm(mc) + mc.SEGTL() mc.MOV(heap(ea), imm0) - self._tl_segment_if_stm(mc) + mc.SEGTL() mc.MOV(heap(eva), imm0) # ...and save ebx into 'jf_guard_exc' offset = self.cpu.get_ofs_of_frame_field('jf_guard_exc') @@ -2424,46 +2396,13 @@ # XXX if the next operation is a GUARD_NO_EXCEPTION, we should # somehow jump over it too in the fast path - - def _cond_allocate_in_nursery_or_slowpath(self, mc, gcmap): - # needed for slowpath: - # eax = nursery_current - # edi = nursery_current + size - # - # cmp nursery_current+size > nursery_nextlimit - nnl = self._get_stm_tl(rstm.get_nursery_nextlimit_adr()) - self._tl_segment_if_stm(mc) - mc.CMP_rj(edi.value, nnl) - mc.J_il8(rx86.Conditions['NA'], 0) # patched later - jmp_adr = mc.get_relative_pos() - # - # == SLOWPATH == - # save the gcmap - self.push_gcmap(mc, gcmap, mov=True) - mc.CALL(imm(self.malloc_slowpath)) - mc.JMP_l8(0) # XXX: is JMP over 1 instr good? - jmp2_adr = mc.get_relative_pos() - # - # == FASTPATH == - offset = mc.get_relative_pos() - jmp_adr - assert 0 < offset <= 127 - mc.overwrite(jmp_adr-1, chr(offset)) - # - # stm_nursery_current = stm_nursery_current+size - nc = self._get_stm_tl(rstm.get_nursery_current_adr()) - self._tl_segment_if_stm(mc) - mc.MOV_jr(nc, edi.value) - # - # END - offset = mc.get_relative_pos() - jmp2_adr - assert 0 < offset <= 127 - mc.overwrite(jmp2_adr-1, chr(offset)) def malloc_cond(self, nursery_free_adr, nursery_top_adr, size, gcmap): - assert not self.cpu.gc_ll_descr.stm assert size & (WORD-1) == 0 # must be correctly aligned + self.mc.SEGC7() self.mc.MOV(eax, heap(nursery_free_adr)) self.mc.LEA_rm(edi.value, (eax.value, size)) + self.mc.SEGC7() self.mc.CMP(edi, heap(nursery_top_adr)) self.mc.J_il8(rx86.Conditions['NA'], 0) # patched later jmp_adr = self.mc.get_relative_pos() @@ -2473,6 +2412,7 @@ offset = self.mc.get_relative_pos() - jmp_adr assert 0 < offset <= 127 self.mc.overwrite(jmp_adr-1, chr(offset)) + self.mc.SEGC7() self.mc.MOV(heap(nursery_free_adr), edi) def malloc_cond_varsize_frame(self, nursery_free_adr, nursery_top_adr, @@ -2482,6 +2422,7 @@ if sizeloc is eax: self.mc.MOV(edi, sizeloc) sizeloc = edi + self.mc.SEGC7() self.mc.MOV(eax, heap(nursery_free_adr)) if self.cpu.gc_ll_descr.stm: constsize = self.cpu.get_baseofs_of_frame_field() @@ -2492,6 +2433,7 @@ self.mc.ADD_rr(edi.value, eax.value) else: self.mc.LEA_ra(edi.value, (eax.value, sizeloc.value, 0, 0)) + self.mc.SEGC7() self.mc.CMP(edi, heap(nursery_top_adr)) self.mc.J_il8(rx86.Conditions['NA'], 0) # patched later jmp_adr = self.mc.get_relative_pos() @@ -2501,6 +2443,7 @@ offset = self.mc.get_relative_pos() - jmp_adr assert 0 < offset <= 127 self.mc.overwrite(jmp_adr-1, chr(offset)) + self.mc.SEGC7() self.mc.MOV(heap(nursery_free_adr), edi) def malloc_cond_varsize(self, kind, nursery_free_adr, nursery_top_adr, @@ -2522,6 +2465,7 @@ self.mc.J_il8(rx86.Conditions['A'], 0) # patched later jmp_adr0 = self.mc.get_relative_pos() + self.mc.SEGC7() self.mc.MOV(eax, heap(nursery_free_adr)) if valid_addressing_size(itemsize): shift = get_scale(itemsize) @@ -2542,6 +2486,7 @@ self.mc.AND_ri(edi.value, ~(WORD - 1)) # now edi contains the total size in bytes, rounded up to a multiple # of WORD, plus nursery_free_adr + self.mc.SEGC7() self.mc.CMP(edi, heap(nursery_top_adr)) self.mc.J_il8(rx86.Conditions['NA'], 0) # patched later jmp_adr1 = self.mc.get_relative_pos() @@ -2573,6 +2518,7 @@ # write down the tid, but not if it's the result of the CALL self.mc.MOV(mem(eax, 0), imm(arraydescr.tid)) # while we're at it, this line is not needed if we've done the CALL + self.mc.SEGC7() self.mc.MOV(heap(nursery_free_adr), edi) # offset = self.mc.get_relative_pos() - jmp_location @@ -2655,16 +2601,18 @@ def genop_discard_stm_read(self, op, arglocs): assert IS_X86_64, "needed for X86_64_SCRATCH_REG" mc = self.mc - rm8reg = X86_64_SCRATCH_REG.value | BYTE_REG_FLAG - xxxxxx #load STM_SEGMENT->transaction_read_version into rm8reg + rmreg = X86_64_SCRATCH_REG.value + mc.SEGC7() + mc.MOVZX8_rj(rmreg, rstm.adr_transaction_read_version) + # loc_src, loc_tmp = arglocs if tmp_loc is None: assert isinstance(loc_src, ImmedLoc) assert loc_src.value > 0 mem = loc_src.value >> 4 assert rx86.fits_in_32bits(mem) - tl_segment_prefix(mc) - mc.MOV8_jr(mem, rm8reg) + mc.SEGC7() + mc.MOV8_jr(mem, rmreg | rx86.BYTE_REG_FLAG) else: assert isinstance(loc_tmp, RegLoc) if isinstance(loc_src, ImmedLoc): @@ -2673,8 +2621,8 @@ if loc_tmp is not loc_src: mc.MOV(loc_tmp, loc_src) mc.SHR_ri(loc_tmp.value, 4) - tl_segment_prefix(mc) - mc.MOV8_mr((loc_tmp.value, 0), rm8reg) + mc.SEGC7() + mc.MOV8_mr((loc_tmp.value, 0), rmreg | rx86.BYTE_REG_FLAG) genop_discard_list = [Assembler386.not_implemented_op_discard] * rop._LAST diff --git a/rpython/jit/backend/x86/codebuf.py b/rpython/jit/backend/x86/codebuf.py --- a/rpython/jit/backend/x86/codebuf.py +++ b/rpython/jit/backend/x86/codebuf.py @@ -1,5 +1,6 @@ from rpython.rtyper.lltypesystem import lltype, rffi from rpython.rlib.rarithmetic import intmask +from rpython.rlib.objectmodel import we_are_translated from rpython.rlib.debug import debug_start, debug_print, debug_stop from rpython.rlib.debug import have_debug_prints from rpython.jit.backend.llsupport.asmmemmgr import BlockBuilderMixin @@ -21,7 +22,8 @@ class MachineCodeBlockWrapper(BlockBuilderMixin, LocationCodeBuilder, codebuilder_cls): - def __init__(self): + def __init__(self, cpu): + self.stm = cpu.gc_ll_descr.stm self.init_block_builder() # a list of relative positions; for each position p, the bytes # at [p-4:p] encode an absolute address that will need to be @@ -52,3 +54,30 @@ adr[0] = intmask(adr[0] - p) valgrind.discard_translations(addr, self.get_relative_pos()) self._dump(addr, "jit-backend-dump", backend_name) + + def in_tl_segment(self, adr): + """Makes 'adr' relative to threadlocal-base if we run in STM. + Before using such a relative address, call SEGTL().""" + if self.stm and we_are_translated(): + # only for STM and not during tests + from rpython.jit.backend.x86 import stmtlocal, rx86 + result = adr - stmtlocal.threadlocal_base() + assert rx86.fits_in_32bits(result) + return result + return adr + + def SEGTL(self): + """Insert segment prefix for thread-local memory if we run + in STM and not during testing. This is used to access thread-local + data structures like the struct stm_thread_local_s.""" + if self.stm and we_are_translated(): + from rpython.jit.backend.x86 import stmtlocal + stmtlocal.tl_segment_prefix(self) + + def SEGC7(self): + """Insert segment prefix for the stmgc-c7 segment of memory + if we run in STM and not during testing. This is used to access + any GC object, or things in the STM_SEGMENT structure.""" + if self.stm and we_are_translated(): + from rpython.jit.backend.x86 import stmtlocal + stmtlocal.c7_segment_prefix(self) diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -839,13 +839,10 @@ gcmap = self.get_gcmap([eax, edi]) # allocate the gcmap *before* self.rm.possibly_free_var(tmp_box) # - if gc_ll_descr.stm: - self.assembler.malloc_cond_stm(size, gcmap) - else: - self.assembler.malloc_cond( - gc_ll_descr.get_nursery_free_addr(), - gc_ll_descr.get_nursery_top_addr(), - size, gcmap) + self.assembler.malloc_cond( + gc_ll_descr.get_nursery_free_addr(), + gc_ll_descr.get_nursery_top_addr(), + size, gcmap) def consider_call_malloc_nursery_varsize_frame(self, op): gc_ll_descr = self.assembler.cpu.gc_ll_descr @@ -893,16 +890,11 @@ # itemsize = op.getarg(1).getint() maxlength = (gc_ll_descr.max_size_of_young_obj - WORD * 2) / itemsize - if gc_ll_descr.stm: - self.assembler.malloc_cond_varsize_stm( - op.getarg(0).getint(), - lengthloc, itemsize, maxlength, gcmap, arraydescr) - else: - self.assembler.malloc_cond_varsize( - op.getarg(0).getint(), - gc_ll_descr.get_nursery_free_addr(), - gc_ll_descr.get_nursery_top_addr(), - lengthloc, itemsize, maxlength, gcmap, arraydescr) + self.assembler.malloc_cond_varsize( + op.getarg(0).getint(), + gc_ll_descr.get_nursery_free_addr(), + gc_ll_descr.get_nursery_top_addr(), + lengthloc, itemsize, maxlength, gcmap, arraydescr) def get_gcmap(self, forbidden_regs=[], noregs=False): frame_depth = self.fm.get_frame_depth() diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py --- a/rpython/jit/backend/x86/runner.py +++ b/rpython/jit/backend/x86/runner.py @@ -131,7 +131,7 @@ rstm.stop_all_other_threads() for addr, tgt in looptoken.compiled_loop_token.invalidate_positions: - mc = codebuf.MachineCodeBlockWrapper() + mc = codebuf.MachineCodeBlockWrapper(self) mc.JMP_l(tgt) assert mc.get_relative_pos() == 5 # [JMP] [tgt 4 bytes] mc.copy_to_raw_memory(addr - 1) diff --git a/rpython/jit/backend/x86/stmtlocal.py b/rpython/jit/backend/x86/stmtlocal.py --- a/rpython/jit/backend/x86/stmtlocal.py +++ b/rpython/jit/backend/x86/stmtlocal.py @@ -32,6 +32,10 @@ else: mc.writechar('\x64') # %fs: +def c7_segment_prefix(mc): + assert WORD == 8 + mc.writechar('\x65') # %gs: + # special STM functions called directly by the JIT backend stm_should_break_transaction_fn = rffi.llexternal( diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -1546,7 +1546,7 @@ _callable=lambda : False) FUNC = lltype.typeOf(self.stm_should_break_transaction).TO - ei = EffectInfo([], [], [], [], + ei = EffectInfo([], [], [], [], [], [], EffectInfo.EF_CANNOT_RAISE, oopspecindex=EffectInfo.OS_JIT_STM_SHOULD_BREAK_TRANSACTION, can_invalidate=False) diff --git a/rpython/memory/gc/stmgc.py b/rpython/memory/gc/stmgc.py --- a/rpython/memory/gc/stmgc.py +++ b/rpython/memory/gc/stmgc.py @@ -33,7 +33,9 @@ VISIT_FPTR = lltype.Ptr(lltype.FuncType([llmemory.Address], lltype.Void)) - minimal_size_in_nursery = llmemory.sizeof(HDR) + JIT_WB_IF_FLAG = 0x01 # value of _STM_GCFLAG_WRITE_BARRIER + stm_fast_alloc = 66*1024 # value of _STM_FAST_ALLOC in stmgc.h + minimal_size_in_nursery = 16 # hard-coded lower limit TRANSLATION_PARAMS = { } @@ -97,7 +99,7 @@ @classmethod def JIT_max_size_of_young_obj(cls): - return cls.GC_NURSERY_SECTION + return cls.stm_fast_alloc @classmethod def JIT_minimal_size_in_nursery(cls): diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py --- a/rpython/memory/gctransform/framework.py +++ b/rpython/memory/gctransform/framework.py @@ -773,6 +773,7 @@ v_typeid], resultvar=op.result) def _gc_adr_of_gc_attr(self, hop, attrname): + assert not self.translator.config.translation.stm if getattr(self.gcdata.gc, attrname, None) is None: raise NotImplementedError("gc_adr_of_%s only for generational gcs" % (attrname,)) @@ -780,7 +781,6 @@ ofs = llmemory.offsetof(self.c_const_gc.concretetype.TO, 'inst_' + attrname) c_ofs = rmodel.inputconst(lltype.Signed, ofs) - assert not self.translator.config.translation.stm, "XXX" v_gc_adr = hop.genop('cast_ptr_to_adr', [self.c_const_gc], resulttype=llmemory.Address) hop.genop('adr_add', [v_gc_adr, c_ofs], resultvar=op.result) diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py --- a/rpython/rlib/rstm.py +++ b/rpython/rlib/rstm.py @@ -1,4 +1,5 @@ from rpython.rlib.objectmodel import we_are_translated, specialize +from rpython.rlib.objectmodel import CDefinedIntSymbolic from rpython.rtyper.lltypesystem import lltype, rffi, rstr, llmemory from rpython.rtyper.lltypesystem.lloperation import llop from rpython.rtyper.extregistry import ExtRegistryEntry @@ -6,39 +7,17 @@ TID = rffi.UINT -tid_offset = CDefinedIntSymbolic('offsetof(struct rpyobj_s, tid)', default=4) +tid_offset = CDefinedIntSymbolic('offsetof(struct rpyobj_s, tid)') +adr_nursery_free = CDefinedIntSymbolic('(long)(&STM_SEGMENT->nursery_current)') +adr_nursery_top = CDefinedIntSymbolic('(long)(&STM_SEGMENT->nursery_end)') +adr_transaction_read_version = ( + CDefinedIntSymbolic('(long)(&STM_SEGMENT->transaction_read_version)')) - at dont_look_inside -def get_nursery_current_adr(): - addr = llop.stm_get_adr_of_nursery_current(llmemory.Address) - return rffi.cast(lltype.Signed, addr) - - at dont_look_inside -def get_nursery_nextlimit_adr(): - addr = llop.stm_get_adr_of_nursery_nextlimit(llmemory.Address) - return rffi.cast(lltype.Signed, addr) - - at dont_look_inside -def get_active_adr(): - addr = llop.stm_get_adr_of_active(llmemory.Address) - return rffi.cast(lltype.Signed, addr) - - at dont_look_inside -def get_adr_of_private_rev_num(): - addr = llop.stm_get_adr_of_private_rev_num(llmemory.Address) - return rffi.cast(lltype.Signed, addr) - - at dont_look_inside -def get_adr_of_read_barrier_cache(): - addr = llop.stm_get_adr_of_read_barrier_cache(llmemory.Address) - return rffi.cast(lltype.Signed, addr) - def jit_stm_transaction_break_point(): if we_are_translated(): llop.jit_stm_transaction_break_point(lltype.Void) - def jit_stm_should_break_transaction(if_there_is_no_other): # if_there_is_no_other means that we use this point only # if there is no other break point in the trace. @@ -47,20 +26,11 @@ return llop.jit_stm_should_break_transaction(lltype.Bool, if_there_is_no_other) - @dont_look_inside def become_inevitable(): llop.stm_become_inevitable(lltype.Void) @dont_look_inside -def stop_all_other_threads(): - llop.stm_stop_all_other_threads(lltype.Void) - - at dont_look_inside -def partial_commit_and_resume_other_threads(): - llop.stm_partial_commit_and_resume_other_threads(lltype.Void) - - at dont_look_inside def should_break_transaction(): return we_are_translated() and ( llop.stm_should_break_transaction(lltype.Bool)) From noreply at buildbot.pypy.org Sat Mar 22 17:47:58 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 22 Mar 2014 17:47:58 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Try to be more careful: the jit backend must now emit GETFIELD_GC with Message-ID: <20140322164758.56A6E1D253B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r70169:d1db1d32b975 Date: 2014-03-22 17:45 +0100 http://bitbucket.org/pypy/pypy/changeset/d1db1d32b975/ Log: Try to be more careful: the jit backend must now emit GETFIELD_GC with a SEGC7 prefix, but not GETFIELD_RAW. diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -164,7 +164,7 @@ descrs = self.gc_ll_descr.getframedescrs(self.cpu) if self.gc_ll_descr.kind == 'boehm': size_box = history.BoxInt() - op0 = ResOperation(rop.GETFIELD_GC, [history.ConstInt(frame_info)], + op0 = ResOperation(rop.GETFIELD_RAW,[history.ConstInt(frame_info)], size_box, descr=descrs.jfi_frame_depth) self.newops.append(op0) @@ -174,14 +174,14 @@ elif not self.gc_ll_descr.stm: # we read size in bytes here, not the length size_box = history.BoxInt() - op0 = ResOperation(rop.GETFIELD_GC, [history.ConstInt(frame_info)], + op0 = ResOperation(rop.GETFIELD_RAW,[history.ConstInt(frame_info)], size_box, descr=descrs.jfi_frame_size) self.newops.append(op0) self.gen_malloc_nursery_varsize_frame(size_box, frame) self.gen_initialize_tid(frame, descrs.arraydescr.tid) length_box = history.BoxInt() - op1 = ResOperation(rop.GETFIELD_GC, [history.ConstInt(frame_info)], + op1 = ResOperation(rop.GETFIELD_RAW,[history.ConstInt(frame_info)], length_box, descr=descrs.jfi_frame_depth) self.newops.append(op1) @@ -190,7 +190,7 @@ else: # jfi_frame_size not set in STM! length_box = history.BoxInt() - op0 = ResOperation(rop.GETFIELD_GC, [history.ConstInt(frame_info)], + op0 = ResOperation(rop.GETFIELD_RAW,[history.ConstInt(frame_info)], length_box, descr=descrs.jfi_frame_depth) self.newops.append(op0) @@ -407,21 +407,18 @@ val = op.getarg(0) if self.must_apply_write_barrier(val, op.getarg(1)): self.gen_write_barrier(val) - #op = op.copy_and_change(rop.SETFIELD_RAW) self.newops.append(op) def handle_write_barrier_setinteriorfield(self, op): val = op.getarg(0) if self.must_apply_write_barrier(val, op.getarg(2)): self.gen_write_barrier(val) - #op = op.copy_and_change(rop.SETINTERIORFIELD_RAW) self.newops.append(op) def handle_write_barrier_setarrayitem(self, op): val = op.getarg(0) if self.must_apply_write_barrier(val, op.getarg(2)): self.gen_write_barrier_array(val, op.getarg(1)) - #op = op.copy_and_change(rop.SETARRAYITEM_RAW) self.newops.append(op) def gen_write_barrier(self, v_base): diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py --- a/rpython/jit/backend/llsupport/test/test_rewrite.py +++ b/rpython/jit/backend/llsupport/test/test_rewrite.py @@ -786,10 +786,10 @@ i2 = call_assembler(i0, f0, descr=casmdescr) """, """ [i0, f0] - i1 = getfield_gc(ConstClass(frame_info), descr=jfi_frame_size) + i1 = getfield_raw(ConstClass(frame_info), descr=jfi_frame_size) p1 = call_malloc_nursery_varsize_frame(i1) setfield_gc(p1, 0, descr=tiddescr) - i2 = getfield_gc(ConstClass(frame_info), descr=jfi_frame_depth) + i2 = getfield_raw(ConstClass(frame_info), descr=jfi_frame_depth) setfield_gc(p1, i2, descr=framelendescr) setfield_gc(p1, ConstClass(frame_info), descr=jf_frame_info) setarrayitem_gc(p1, 0, i0, descr=signedframedescr) diff --git a/rpython/jit/backend/llsupport/test/test_stmrewrite.py b/rpython/jit/backend/llsupport/test/test_stmrewrite.py --- a/rpython/jit/backend/llsupport/test/test_stmrewrite.py +++ b/rpython/jit/backend/llsupport/test/test_stmrewrite.py @@ -741,7 +741,7 @@ guard_not_forced()[] """, """ [i0, f0] - i1 = getfield_gc(ConstClass(frame_info), descr=jfi_frame_depth) + i1 = getfield_raw(ConstClass(frame_info), descr=jfi_frame_depth) p1 = call_malloc_nursery_varsize_frame(i1) setfield_gc(p1, 0, descr=tiddescr) setfield_gc(p1, i1, descr=framelendescr) diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -1475,10 +1475,11 @@ # ---------- - def load_from_mem(self, resloc, source_addr, size_loc, sign_loc): + def load_from_mem(self, resloc, source_addr, size_loc, sign_loc, op): assert isinstance(resloc, RegLoc) size = size_loc.value sign = sign_loc.value + self.mc.SEGC7_if_gc(op) if resloc.is_xmm: self.mc.MOVSD(resloc, source_addr) elif size == WORD: @@ -1503,6 +1504,7 @@ def save_into_mem(self, dest_addr, value_loc, size_loc): size = size_loc.value + self.mc.SEGC7_if_gc(op) if isinstance(value_loc, RegLoc) and value_loc.is_xmm: self.mc.MOVSD(dest_addr, value_loc) elif size == 1: @@ -1517,6 +1519,7 @@ else: assert isinstance(value_loc, FloatImmedLoc) self.mc.MOV(dest_addr, value_loc.low_part_loc()) + self.mc.SEGC7_if_gc(op) self.mc.MOV(dest_addr.add_offset(4), value_loc.high_part_loc()) else: not_implemented("save_into_mem size = %d" % size) @@ -1525,7 +1528,7 @@ base_loc, ofs_loc, size_loc, sign_loc = arglocs assert isinstance(size_loc, ImmedLoc) source_addr = AddressLoc(base_loc, ofs_loc) - self.load_from_mem(resloc, source_addr, size_loc, sign_loc) + self.load_from_mem(resloc, source_addr, size_loc, sign_loc, op) genop_getfield_raw = genop_getfield_gc genop_getfield_raw_pure = genop_getfield_gc @@ -1537,7 +1540,7 @@ assert isinstance(size_loc, ImmedLoc) scale = get_scale(size_loc.value) src_addr = addr_add(base_loc, ofs_loc, ofs.value, scale) - self.load_from_mem(resloc, src_addr, size_loc, sign_loc) + self.load_from_mem(resloc, src_addr, size_loc, sign_loc, op) genop_getarrayitem_gc_pure = genop_getarrayitem_gc genop_getarrayitem_raw = genop_getarrayitem_gc @@ -1547,7 +1550,7 @@ base_loc, ofs_loc, size_loc, ofs, sign_loc = arglocs assert isinstance(ofs, ImmedLoc) src_addr = addr_add(base_loc, ofs_loc, ofs.value, 0) - self.load_from_mem(resloc, src_addr, size_loc, sign_loc) + self.load_from_mem(resloc, src_addr, size_loc, sign_loc, op) def _imul_const_scaled(self, mc, targetreg, sourcereg, itemsize): """Produce one operation to do roughly @@ -1600,7 +1603,7 @@ src_addr = self._get_interiorfield_addr(temp_loc, index_loc, itemsize_loc, base_loc, ofs_loc) - self.load_from_mem(resloc, src_addr, fieldsize_loc, sign_loc) + self.load_from_mem(resloc, src_addr, fieldsize_loc, sign_loc, op) def genop_discard_increment_debug_counter(self, op, arglocs): # The argument should be an immediate address. This should @@ -1614,7 +1617,7 @@ base_loc, ofs_loc, size_loc, value_loc = arglocs assert isinstance(size_loc, ImmedLoc) dest_addr = AddressLoc(base_loc, ofs_loc) - self.save_into_mem(dest_addr, value_loc, size_loc) + self.save_into_mem(dest_addr, value_loc, size_loc, op) def genop_discard_setinteriorfield_gc(self, op, arglocs): (base_loc, ofs_loc, itemsize_loc, fieldsize_loc, @@ -1622,7 +1625,7 @@ dest_addr = self._get_interiorfield_addr(temp_loc, index_loc, itemsize_loc, base_loc, ofs_loc) - self.save_into_mem(dest_addr, value_loc, fieldsize_loc) + self.save_into_mem(dest_addr, value_loc, fieldsize_loc, op) genop_discard_setinteriorfield_raw = genop_discard_setinteriorfield_gc @@ -1632,13 +1635,13 @@ assert isinstance(size_loc, ImmedLoc) scale = get_scale(size_loc.value) dest_addr = AddressLoc(base_loc, ofs_loc, scale, baseofs.value) - self.save_into_mem(dest_addr, value_loc, size_loc) + self.save_into_mem(dest_addr, value_loc, size_loc, op) def genop_discard_raw_store(self, op, arglocs): base_loc, ofs_loc, value_loc, size_loc, baseofs = arglocs assert isinstance(baseofs, ImmedLoc) dest_addr = AddressLoc(base_loc, ofs_loc, 0, baseofs.value) - self.save_into_mem(dest_addr, value_loc, size_loc) + self.save_into_mem(dest_addr, value_loc, size_loc, op) def genop_discard_strsetitem(self, op, arglocs): base_loc, ofs_loc, val_loc = arglocs @@ -2013,7 +2016,7 @@ size = WORD * 2 else: size = WORD - self.save_into_mem(raw_stack(base_ofs), return_val, imm(size)) + self.save_into_mem(raw_stack(base_ofs), return_val, imm(size), op) else: [fail_descr_loc] = arglocs ofs = self.cpu.get_ofs_of_frame_field('jf_descr') diff --git a/rpython/jit/backend/x86/codebuf.py b/rpython/jit/backend/x86/codebuf.py --- a/rpython/jit/backend/x86/codebuf.py +++ b/rpython/jit/backend/x86/codebuf.py @@ -81,3 +81,20 @@ if self.stm and we_are_translated(): from rpython.jit.backend.x86 import stmtlocal stmtlocal.c7_segment_prefix(self) + + def SEGC7_if_gc(self, op): + if self.stm and we_are_translated(): + from rpython.jit.backend.x86 import stmtlocal + from rpython.jit.metainterp.resoperation import rop + # + opnum = op.getopnum() + if opnum in (rop.GETFIELD_GC, + rop.GETFIELD_GC_PURE, + rop.GETARRAYITEM_GC, + rop.GETARRAYITEM_GC_PURE, + rop.GETINTERIORFIELD_GC, + rop.SETFIELD_GC, + rop.SETARRAYITEM_GC, + rop.SETINTERIORFIELD_GC, + ): + stmtlocal.c7_segment_prefix(self) From noreply at buildbot.pypy.org Sat Mar 22 18:31:42 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 22 Mar 2014 18:31:42 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Add these temporarily, until we figure out how exactly we need it Message-ID: <20140322173142.E3EAA1C0483@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r70170:ca4bf01af754 Date: 2014-03-22 18:29 +0100 http://bitbucket.org/pypy/pypy/changeset/ca4bf01af754/ Log: Add these temporarily, until we figure out how exactly we need it diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py --- a/rpython/rlib/rstm.py +++ b/rpython/rlib/rstm.py @@ -6,6 +6,10 @@ from rpython.rlib.jit import dont_look_inside +def stop_all_other_threads(): "XXX" +def partial_commit_and_resume_other_threads(): "XXX" + + TID = rffi.UINT tid_offset = CDefinedIntSymbolic('offsetof(struct rpyobj_s, tid)') adr_nursery_free = CDefinedIntSymbolic('(long)(&STM_SEGMENT->nursery_current)') From noreply at buildbot.pypy.org Sat Mar 22 18:46:07 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 22 Mar 2014 18:46:07 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: fixes Message-ID: <20140322174607.45A271C073C@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r70171:03eba8c7dd62 Date: 2014-03-22 18:43 +0100 http://bitbucket.org/pypy/pypy/changeset/03eba8c7dd62/ Log: fixes diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -1476,33 +1476,36 @@ # ---------- def load_from_mem(self, resloc, source_addr, size_loc, sign_loc, op): - assert isinstance(resloc, RegLoc) size = size_loc.value sign = sign_loc.value self.mc.SEGC7_if_gc(op) + self.generate_one_mov_with_extension(resloc, source_addr, size, sign) + + def generate_one_mov_with_extension(self, resloc, srcloc, size, sign): + assert isinstance(resloc, RegLoc) if resloc.is_xmm: - self.mc.MOVSD(resloc, source_addr) + self.mc.MOVSD(resloc, srcloc) elif size == WORD: - self.mc.MOV(resloc, source_addr) + self.mc.MOV(resloc, srcloc) elif size == 1: if sign: - self.mc.MOVSX8(resloc, source_addr) + self.mc.MOVSX8(resloc, srcloc) else: - self.mc.MOVZX8(resloc, source_addr) + self.mc.MOVZX8(resloc, srcloc) elif size == 2: if sign: - self.mc.MOVSX16(resloc, source_addr) + self.mc.MOVSX16(resloc, srcloc) else: - self.mc.MOVZX16(resloc, source_addr) + self.mc.MOVZX16(resloc, srcloc) elif IS_X86_64 and size == 4: if sign: - self.mc.MOVSX32(resloc, source_addr) + self.mc.MOVSX32(resloc, srcloc) else: - self.mc.MOV32(resloc, source_addr) # zero-extending + self.mc.MOV32(resloc, srcloc) # zero-extending else: not_implemented("load_from_mem size = %d" % size) - def save_into_mem(self, dest_addr, value_loc, size_loc): + def save_into_mem(self, dest_addr, value_loc, size_loc, op): size = size_loc.value self.mc.SEGC7_if_gc(op) if isinstance(value_loc, RegLoc) and value_loc.is_xmm: @@ -2156,15 +2159,6 @@ def _call_assembler_check_descr(self, value, tmploc): ofs = self.cpu.get_ofs_of_frame_field('jf_descr') - - if self.cpu.gc_ll_descr.stm: - # value is non-moving, but jf_descr may have a changed - # descr -> different copy - self._stm_ptr_eq_fastpath(self.mc, [mem(eax, ofs), imm(value)], - tmploc) - self.mc.J_il8(rx86.Conditions['NZ'], 0) - return self.mc.get_relative_pos() - self.mc.CMP(mem(eax, ofs), imm(value)) # patched later self.mc.J_il8(rx86.Conditions['E'], 0) # goto B if we get 'done_with_this_frame' @@ -2602,14 +2596,15 @@ self._emit_guard_not_forced(guard_token) def genop_discard_stm_read(self, op, arglocs): - assert IS_X86_64, "needed for X86_64_SCRATCH_REG" + if not IS_X86_64: + todo() # "needed for X86_64_SCRATCH_REG" mc = self.mc rmreg = X86_64_SCRATCH_REG.value mc.SEGC7() mc.MOVZX8_rj(rmreg, rstm.adr_transaction_read_version) # loc_src, loc_tmp = arglocs - if tmp_loc is None: + if loc_tmp is None: assert isinstance(loc_src, ImmedLoc) assert loc_src.value > 0 mem = loc_src.value >> 4 diff --git a/rpython/jit/backend/x86/callbuilder.py b/rpython/jit/backend/x86/callbuilder.py --- a/rpython/jit/backend/x86/callbuilder.py +++ b/rpython/jit/backend/x86/callbuilder.py @@ -68,7 +68,7 @@ """Overridden in CallBuilder32 and CallBuilder64""" if self.ressize == 0: return # void result - # use the code in load_from_mem to do the zero- or sign-extension + # use the code in self.asm to do the zero- or sign-extension srcloc = self.tmpresloc if srcloc is None: if self.restype == FLOAT: @@ -79,8 +79,8 @@ return # no need for any MOV if self.ressize == 1 and isinstance(srcloc, RegLoc): srcloc = srcloc.lowest8bits() - self.asm.load_from_mem(self.resloc, srcloc, - imm(self.ressize), imm(self.ressign)) + self.asm.generate_one_mov_with_extension(self.resloc, srcloc, + self.ressize, self.ressign) def push_gcmap(self): # we push *now* the gcmap, describing the status of GC registers From noreply at buildbot.pypy.org Sat Mar 22 20:01:19 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 22 Mar 2014 20:01:19 +0100 (CET) Subject: [pypy-commit] stmgc default: stm_become_globally_unique_transaction() Message-ID: <20140322190119.CA7A21D290B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1082:a5f0a9669efe Date: 2014-03-22 19:59 +0100 http://bitbucket.org/pypy/stmgc/changeset/a5f0a9669efe/ Log: stm_become_globally_unique_transaction() diff --git a/c7/stm/core.c b/c7/stm/core.c --- a/c7/stm/core.c +++ b/c7/stm/core.c @@ -461,7 +461,7 @@ /* force all other threads to be paused. They will unpause automatically when we are done here, i.e. at mutex_unlock(). Important: we should not call cond_wait() in the meantime. */ - synchronize_all_threads(); + synchronize_all_threads(STOP_OTHERS_UNTIL_MUTEX_UNLOCK); /* detect conflicts */ if (detect_write_read_conflicts()) @@ -500,6 +500,8 @@ if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) { /* wake up one thread in wait_for_end_of_inevitable_transaction() */ cond_signal(C_INEVITABLE); + if (globally_unique_transaction) + committed_globally_unique_transaction(); } /* done */ @@ -672,3 +674,12 @@ s_mutex_unlock(); } + +void stm_become_globally_unique_transaction(const char *msg) +{ + stm_become_inevitable(msg); /* may still abort */ + + s_mutex_lock(); + synchronize_all_threads(STOP_OTHERS_AND_BECOME_GLOBALLY_UNIQUE); + s_mutex_unlock(); +} diff --git a/c7/stm/forksupport.c b/c7/stm/forksupport.c --- a/c7/stm/forksupport.c +++ b/c7/stm/forksupport.c @@ -68,7 +68,7 @@ } s_mutex_lock(); - synchronize_all_threads(); + synchronize_all_threads(STOP_OTHERS_UNTIL_MUTEX_UNLOCK); mutex_pages_lock(); /* Make a new mmap at some other address, but of the same size as diff --git a/c7/stm/gcpage.c b/c7/stm/gcpage.c --- a/c7/stm/gcpage.c +++ b/c7/stm/gcpage.c @@ -134,7 +134,7 @@ if (is_major_collection_requested()) { /* if still true */ - synchronize_all_threads(); + synchronize_all_threads(STOP_OTHERS_UNTIL_MUTEX_UNLOCK); if (is_major_collection_requested()) { /* if *still* true */ major_collection_now_at_safe_point(); diff --git a/c7/stm/sync.c b/c7/stm/sync.c --- a/c7/stm/sync.c +++ b/c7/stm/sync.c @@ -319,7 +319,9 @@ /* If we are requested to enter a safe-point, we cannot proceed now. Wait until the safe-point request is removed for us. */ - +#ifdef STM_TESTS + abort_with_mutex(); +#endif cond_signal(C_AT_SAFE_POINT); STM_PSEGMENT->safe_point = SP_WAIT_FOR_C_REQUEST_REMOVED; cond_wait(C_REQUEST_REMOVED); @@ -327,7 +329,7 @@ } } -static void synchronize_all_threads(void) +static void synchronize_all_threads(enum sync_type_e sync_type) { enter_safe_point_if_requested(); @@ -335,7 +337,13 @@ why: if several threads call this function, the first one that goes past this point will set the "request safe point" on all other threads; then none of the other threads will go past the - enter_safe_point_if_requested() above. */ + enter_safe_point_if_requested() above. + */ + if (UNLIKELY(globally_unique_transaction)) { + assert(count_other_threads_sp_running() == 0); + return; + } + signal_everybody_to_pause_running(); /* If some other threads are SP_RUNNING, we cannot proceed now. @@ -352,6 +360,13 @@ } } + if (UNLIKELY(sync_type == STOP_OTHERS_AND_BECOME_GLOBALLY_UNIQUE)) { + globally_unique_transaction = true; + assert(STM_SEGMENT->nursery_end == NSE_SIGPAUSE); + STM_SEGMENT->nursery_end = NURSERY_END; + return; /* don't remove the requests for safe-points in this case */ + } + /* Remove the requests for safe-points now. In principle we should remove it later, when the caller is done, but this is equivalent as long as we hold the mutex. @@ -359,6 +374,15 @@ remove_requests_for_safe_point(); /* => C_REQUEST_REMOVED */ } +static void committed_globally_unique_transaction(void) +{ + assert(globally_unique_transaction); + assert(STM_SEGMENT->nursery_end == NURSERY_END); + STM_SEGMENT->nursery_end = NSE_SIGPAUSE; + globally_unique_transaction = false; + remove_requests_for_safe_point(); +} + void _stm_collectable_safe_point(void) { /* If 'nursery_end' was set to NSE_SIGxxx by another thread, diff --git a/c7/stm/sync.h b/c7/stm/sync.h --- a/c7/stm/sync.h +++ b/c7/stm/sync.h @@ -29,6 +29,12 @@ static void release_thread_segment(stm_thread_local_t *tl); static void wait_for_end_of_inevitable_transaction(bool can_abort); -static void synchronize_all_threads(void); -static bool pause_signalled; +enum sync_type_e { + STOP_OTHERS_UNTIL_MUTEX_UNLOCK, + STOP_OTHERS_AND_BECOME_GLOBALLY_UNIQUE, +}; +static void synchronize_all_threads(enum sync_type_e sync_type); +static void committed_globally_unique_transaction(void); + +static bool pause_signalled, globally_unique_transaction; diff --git a/c7/stmgc.h b/c7/stmgc.h --- a/c7/stmgc.h +++ b/c7/stmgc.h @@ -325,6 +325,14 @@ void stm_call_on_abort(stm_thread_local_t *, void *key, void callback(void *)); +/* Similar to stm_become_inevitable(), but additionally suspend all + other threads. A very heavy-handed way to make sure that no other + transaction is running concurrently. Avoid as much as possible. + Other transactions will continue running only after this transaction + commits. */ +void stm_become_globally_unique_transaction(const char *msg); + + /* ==================== END ==================== */ #endif diff --git a/c7/test/support.py b/c7/test/support.py --- a/c7/test/support.py +++ b/c7/test/support.py @@ -54,6 +54,7 @@ bool _check_commit_transaction(void); bool _check_abort_transaction(void); bool _check_become_inevitable(void); +bool _check_become_globally_unique_transaction(void); int stm_is_inevitable(void); void _set_type_id(object_t *obj, uint32_t h); @@ -161,6 +162,10 @@ CHECKED(stm_become_inevitable("TEST")); } +bool _check_become_globally_unique_transaction() { + CHECKED(stm_become_globally_unique_transaction("TESTGUT")); +} + #undef CHECKED @@ -357,6 +362,10 @@ if lib._check_become_inevitable(): raise Conflict() +def stm_become_globally_unique_transaction(): + if lib._check_become_globally_unique_transaction(): + raise Conflict() + def stm_minor_collect(): lib.stm_collect(0) @@ -412,6 +421,10 @@ self.current_thread = 0 def teardown_method(self, meth): + tl = self.tls[self.current_thread] + if lib._stm_in_transaction(tl) and lib.stm_is_inevitable(): + self.commit_transaction() # must succeed! + # for n, tl in enumerate(self.tls): if lib._stm_in_transaction(tl): if self.current_thread != n: @@ -420,6 +433,7 @@ self.commit_transaction() # must succeed! else: self.abort_transaction() + # for tl in self.tls: lib.stm_unregister_thread_local(tl) lib.stm_teardown() diff --git a/c7/test/test_extra.py b/c7/test/test_extra.py --- a/c7/test/test_extra.py +++ b/c7/test/test_extra.py @@ -80,3 +80,13 @@ self.start_transaction() self.abort_transaction() assert seen == [] + + def test_stm_become_globally_unique_transaction(self): + self.start_transaction() + # + self.switch(1) + self.start_transaction() + lib._check_become_globally_unique_transaction() + assert lib.stm_is_inevitable() + # + py.test.raises(Conflict, self.switch, 0) From noreply at buildbot.pypy.org Sat Mar 22 20:05:45 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 22 Mar 2014 20:05:45 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: import stmgc/a5f0a9669efe Message-ID: <20140322190545.50CB31D290C@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r70172:1d78acd6d99b Date: 2014-03-22 20:01 +0100 http://bitbucket.org/pypy/pypy/changeset/1d78acd6d99b/ Log: import stmgc/a5f0a9669efe diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision --- a/rpython/translator/stm/src_stm/revision +++ b/rpython/translator/stm/src_stm/revision @@ -1,1 +1,1 @@ -0e5239ae07f2 +a5f0a9669efe diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c --- a/rpython/translator/stm/src_stm/stm/core.c +++ b/rpython/translator/stm/src_stm/stm/core.c @@ -462,7 +462,7 @@ /* force all other threads to be paused. They will unpause automatically when we are done here, i.e. at mutex_unlock(). Important: we should not call cond_wait() in the meantime. */ - synchronize_all_threads(); + synchronize_all_threads(STOP_OTHERS_UNTIL_MUTEX_UNLOCK); /* detect conflicts */ if (detect_write_read_conflicts()) @@ -501,6 +501,8 @@ if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) { /* wake up one thread in wait_for_end_of_inevitable_transaction() */ cond_signal(C_INEVITABLE); + if (globally_unique_transaction) + committed_globally_unique_transaction(); } /* done */ @@ -673,3 +675,12 @@ s_mutex_unlock(); } + +void stm_become_globally_unique_transaction(const char *msg) +{ + stm_become_inevitable(msg); /* may still abort */ + + s_mutex_lock(); + synchronize_all_threads(STOP_OTHERS_AND_BECOME_GLOBALLY_UNIQUE); + s_mutex_unlock(); +} diff --git a/rpython/translator/stm/src_stm/stm/forksupport.c b/rpython/translator/stm/src_stm/stm/forksupport.c --- a/rpython/translator/stm/src_stm/stm/forksupport.c +++ b/rpython/translator/stm/src_stm/stm/forksupport.c @@ -69,7 +69,7 @@ } s_mutex_lock(); - synchronize_all_threads(); + synchronize_all_threads(STOP_OTHERS_UNTIL_MUTEX_UNLOCK); mutex_pages_lock(); /* Make a new mmap at some other address, but of the same size as diff --git a/rpython/translator/stm/src_stm/stm/gcpage.c b/rpython/translator/stm/src_stm/stm/gcpage.c --- a/rpython/translator/stm/src_stm/stm/gcpage.c +++ b/rpython/translator/stm/src_stm/stm/gcpage.c @@ -135,7 +135,7 @@ if (is_major_collection_requested()) { /* if still true */ - synchronize_all_threads(); + synchronize_all_threads(STOP_OTHERS_UNTIL_MUTEX_UNLOCK); if (is_major_collection_requested()) { /* if *still* true */ major_collection_now_at_safe_point(); diff --git a/rpython/translator/stm/src_stm/stm/sync.c b/rpython/translator/stm/src_stm/stm/sync.c --- a/rpython/translator/stm/src_stm/stm/sync.c +++ b/rpython/translator/stm/src_stm/stm/sync.c @@ -320,7 +320,9 @@ /* If we are requested to enter a safe-point, we cannot proceed now. Wait until the safe-point request is removed for us. */ - +#ifdef STM_TESTS + abort_with_mutex(); +#endif cond_signal(C_AT_SAFE_POINT); STM_PSEGMENT->safe_point = SP_WAIT_FOR_C_REQUEST_REMOVED; cond_wait(C_REQUEST_REMOVED); @@ -328,7 +330,7 @@ } } -static void synchronize_all_threads(void) +static void synchronize_all_threads(enum sync_type_e sync_type) { enter_safe_point_if_requested(); @@ -336,7 +338,13 @@ why: if several threads call this function, the first one that goes past this point will set the "request safe point" on all other threads; then none of the other threads will go past the - enter_safe_point_if_requested() above. */ + enter_safe_point_if_requested() above. + */ + if (UNLIKELY(globally_unique_transaction)) { + assert(count_other_threads_sp_running() == 0); + return; + } + signal_everybody_to_pause_running(); /* If some other threads are SP_RUNNING, we cannot proceed now. @@ -353,6 +361,13 @@ } } + if (UNLIKELY(sync_type == STOP_OTHERS_AND_BECOME_GLOBALLY_UNIQUE)) { + globally_unique_transaction = true; + assert(STM_SEGMENT->nursery_end == NSE_SIGPAUSE); + STM_SEGMENT->nursery_end = NURSERY_END; + return; /* don't remove the requests for safe-points in this case */ + } + /* Remove the requests for safe-points now. In principle we should remove it later, when the caller is done, but this is equivalent as long as we hold the mutex. @@ -360,6 +375,15 @@ remove_requests_for_safe_point(); /* => C_REQUEST_REMOVED */ } +static void committed_globally_unique_transaction(void) +{ + assert(globally_unique_transaction); + assert(STM_SEGMENT->nursery_end == NURSERY_END); + STM_SEGMENT->nursery_end = NSE_SIGPAUSE; + globally_unique_transaction = false; + remove_requests_for_safe_point(); +} + void _stm_collectable_safe_point(void) { /* If 'nursery_end' was set to NSE_SIGxxx by another thread, diff --git a/rpython/translator/stm/src_stm/stm/sync.h b/rpython/translator/stm/src_stm/stm/sync.h --- a/rpython/translator/stm/src_stm/stm/sync.h +++ b/rpython/translator/stm/src_stm/stm/sync.h @@ -30,6 +30,12 @@ static void release_thread_segment(stm_thread_local_t *tl); static void wait_for_end_of_inevitable_transaction(bool can_abort); -static void synchronize_all_threads(void); -static bool pause_signalled; +enum sync_type_e { + STOP_OTHERS_UNTIL_MUTEX_UNLOCK, + STOP_OTHERS_AND_BECOME_GLOBALLY_UNIQUE, +}; +static void synchronize_all_threads(enum sync_type_e sync_type); +static void committed_globally_unique_transaction(void); + +static bool pause_signalled, globally_unique_transaction; diff --git a/rpython/translator/stm/src_stm/stmgc.h b/rpython/translator/stm/src_stm/stmgc.h --- a/rpython/translator/stm/src_stm/stmgc.h +++ b/rpython/translator/stm/src_stm/stmgc.h @@ -326,6 +326,14 @@ void stm_call_on_abort(stm_thread_local_t *, void *key, void callback(void *)); +/* Similar to stm_become_inevitable(), but additionally suspend all + other threads. A very heavy-handed way to make sure that no other + transaction is running concurrently. Avoid as much as possible. + Other transactions will continue running only after this transaction + commits. */ +void stm_become_globally_unique_transaction(const char *msg); + + /* ==================== END ==================== */ #endif From noreply at buildbot.pypy.org Sat Mar 22 20:05:47 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 22 Mar 2014 20:05:47 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Use stm_become_globally_unique_transaction here Message-ID: <20140322190547.0BE811D290C@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r70173:95d7a38c9d56 Date: 2014-03-22 20:03 +0100 http://bitbucket.org/pypy/pypy/changeset/95d7a38c9d56/ Log: Use stm_become_globally_unique_transaction here diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py --- a/rpython/rlib/rstm.py +++ b/rpython/rlib/rstm.py @@ -6,10 +6,6 @@ from rpython.rlib.jit import dont_look_inside -def stop_all_other_threads(): "XXX" -def partial_commit_and_resume_other_threads(): "XXX" - - TID = rffi.UINT tid_offset = CDefinedIntSymbolic('offsetof(struct rpyobj_s, tid)') adr_nursery_free = CDefinedIntSymbolic('(long)(&STM_SEGMENT->nursery_current)') @@ -35,6 +31,13 @@ llop.stm_become_inevitable(lltype.Void) @dont_look_inside +def stop_all_other_threads(): + llop.stm_become_globally_unique_transaction(lltype.Void) + +def partial_commit_and_resume_other_threads(): + pass # for now + + at dont_look_inside def should_break_transaction(): return we_are_translated() and ( llop.stm_should_break_transaction(lltype.Bool)) diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py --- a/rpython/rtyper/lltypesystem/lloperation.py +++ b/rpython/rtyper/lltypesystem/lloperation.py @@ -426,6 +426,7 @@ 'stm_identityhash': LLOp(canfold=True), 'stm_addr_get_tid': LLOp(canfold=True), 'stm_become_inevitable': LLOp(canmallocgc=True), + 'stm_become_globally_unique_transaction': LLOp(canmallocgc=True), 'stm_push_root': LLOp(), 'stm_pop_root_into': LLOp(), 'stm_commit_if_not_atomic': LLOp(canmallocgc=True), diff --git a/rpython/translator/stm/funcgen.py b/rpython/translator/stm/funcgen.py --- a/rpython/translator/stm/funcgen.py +++ b/rpython/translator/stm/funcgen.py @@ -129,6 +129,9 @@ string_literal = c_string_constant(info) return 'stm_become_inevitable(%s);' % (string_literal,) +def stm_become_globally_unique_transaction(funcgen, op): + return 'stm_become_globally_unique_transaction("for the JIT");' + def stm_push_root(funcgen, op): arg0 = funcgen.expr(op.args[0]) return 'STM_PUSH_ROOT(stm_thread_local, %s);' % (arg0,) From noreply at buildbot.pypy.org Sat Mar 22 20:13:04 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 22 Mar 2014 20:13:04 +0100 (CET) Subject: [pypy-commit] stmgc default: Test stm_become_globally_unique_transaction here, it fails Message-ID: <20140322191304.6D3541C02AF@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1083:badf9ae726c2 Date: 2014-03-22 20:11 +0100 http://bitbucket.org/pypy/stmgc/changeset/badf9ae726c2/ Log: Test stm_become_globally_unique_transaction here, it fails diff --git a/c7/demo/demo_random.c b/c7/demo/demo_random.c --- a/c7/demo/demo_random.c +++ b/c7/demo/demo_random.c @@ -288,6 +288,11 @@ stm_become_inevitable("please"); pop_roots(); return NULL; + } else if (get_rand(360) == 1) { + fprintf(stdout, "GUT"); + push_roots(); + stm_become_globally_unique_transaction("really"); + pop_roots(); } return p; } From noreply at buildbot.pypy.org Sat Mar 22 20:20:54 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 22 Mar 2014 20:20:54 +0100 (CET) Subject: [pypy-commit] stmgc default: Can't sleep if we're inevitable! Message-ID: <20140322192054.9DC171C02AF@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1084:378758606915 Date: 2014-03-22 20:18 +0100 http://bitbucket.org/pypy/stmgc/changeset/378758606915/ Log: Can't sleep if we're inevitable! diff --git a/c7/stm/contention.c b/c7/stm/contention.c --- a/c7/stm/contention.c +++ b/c7/stm/contention.c @@ -130,6 +130,7 @@ else if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) { assert(contmgr.other_pseg->transaction_state != TS_INEVITABLE); contmgr.abort_other = true; + contmgr.try_sleep = false; } else if (contmgr.other_pseg->transaction_state == TS_INEVITABLE) { contmgr.abort_other = false; From noreply at buildbot.pypy.org Sat Mar 22 20:43:57 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 22 Mar 2014 20:43:57 +0100 (CET) Subject: [pypy-commit] stmgc default: Fix the test (don't expect variables like 'arg' to remain sensible Message-ID: <20140322194357.CFFB01C02AF@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1085:a33130d9f35c Date: 2014-03-22 20:42 +0100 http://bitbucket.org/pypy/stmgc/changeset/a33130d9f35c/ Log: Fix the test (don't expect variables like 'arg' to remain sensible across a setjmp! They need to be volatile) diff --git a/c7/demo/demo_random.c b/c7/demo/demo_random.c --- a/c7/demo/demo_random.c +++ b/c7/demo/demo_random.c @@ -288,11 +288,12 @@ stm_become_inevitable("please"); pop_roots(); return NULL; - } else if (get_rand(360) == 1) { - fprintf(stdout, "GUT"); + } else if (get_rand(240) == 1) { push_roots(); stm_become_globally_unique_transaction("really"); + fprintf(stderr, "[GUT/%d]", (int)STM_SEGMENT->segment_num); pop_roots(); + return NULL; } return p; } @@ -324,8 +325,9 @@ setup_thread(); - objptr_t p = NULL; + objptr_t p; stm_jmpbuf_t here; + volatile int call_fork = (arg != NULL); STM_START_TRANSACTION(&stm_thread_local, here); assert(td.num_roots >= td.num_roots_at_transaction_start); @@ -345,7 +347,7 @@ if (p == (objptr_t)-1) { push_roots(); - if (arg == NULL) { /* common case */ + if (call_fork == 0) { /* common case */ stm_commit_transaction(); td.num_roots_at_transaction_start = td.num_roots; if (get_rand(100) < 98) { @@ -361,7 +363,7 @@ else { /* run a fork() inside the transaction */ printf("========== FORK =========\n"); - arg = NULL; + call_fork = 0; pid_t child = fork(); printf("=== in process %d thread %lx, fork() returned %d\n", (int)getpid(), (long)pthread_self(), (int)child); diff --git a/c7/stm/core.c b/c7/stm/core.c --- a/c7/stm/core.c +++ b/c7/stm/core.c @@ -191,9 +191,9 @@ STM_PSEGMENT->shadowstack_at_start_of_transaction = tl->shadowstack; STM_PSEGMENT->threadlocal_at_start_of_transaction = tl->thread_local_obj; + enter_safe_point_if_requested(); dprintf(("start_transaction\n")); - enter_safe_point_if_requested(); s_mutex_unlock(); /* Now running the SP_RUNNING start. We can set our From noreply at buildbot.pypy.org Sat Mar 22 20:45:39 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 22 Mar 2014 20:45:39 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: import stmgc/a33130d9f35c Message-ID: <20140322194539.EFB441C0483@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r70174:12e6d39bdeea Date: 2014-03-22 20:43 +0100 http://bitbucket.org/pypy/pypy/changeset/12e6d39bdeea/ Log: import stmgc/a33130d9f35c diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision --- a/rpython/translator/stm/src_stm/revision +++ b/rpython/translator/stm/src_stm/revision @@ -1,1 +1,1 @@ -a5f0a9669efe +a33130d9f35c diff --git a/rpython/translator/stm/src_stm/stm/contention.c b/rpython/translator/stm/src_stm/stm/contention.c --- a/rpython/translator/stm/src_stm/stm/contention.c +++ b/rpython/translator/stm/src_stm/stm/contention.c @@ -131,6 +131,7 @@ else if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) { assert(contmgr.other_pseg->transaction_state != TS_INEVITABLE); contmgr.abort_other = true; + contmgr.try_sleep = false; } else if (contmgr.other_pseg->transaction_state == TS_INEVITABLE) { contmgr.abort_other = false; diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c --- a/rpython/translator/stm/src_stm/stm/core.c +++ b/rpython/translator/stm/src_stm/stm/core.c @@ -192,9 +192,9 @@ STM_PSEGMENT->shadowstack_at_start_of_transaction = tl->shadowstack; STM_PSEGMENT->threadlocal_at_start_of_transaction = tl->thread_local_obj; + enter_safe_point_if_requested(); dprintf(("start_transaction\n")); - enter_safe_point_if_requested(); s_mutex_unlock(); /* Now running the SP_RUNNING start. We can set our From noreply at buildbot.pypy.org Sat Mar 22 23:10:31 2014 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 22 Mar 2014 23:10:31 +0100 (CET) Subject: [pypy-commit] pypy win32-fixes4: clean up win32 test failures Message-ID: <20140322221031.217171C0290@cobra.cs.uni-duesseldorf.de> Author: Matti Picus Branch: win32-fixes4 Changeset: r70175:ed924db837d6 Date: 2014-03-22 23:13 +0200 http://bitbucket.org/pypy/pypy/changeset/ed924db837d6/ Log: clean up win32 test failures From noreply at buildbot.pypy.org Sat Mar 22 23:10:32 2014 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 22 Mar 2014 23:10:32 +0100 (CET) Subject: [pypy-commit] pypy win32-fixes4: add MSVC macros Message-ID: <20140322221032.660E91C0290@cobra.cs.uni-duesseldorf.de> Author: Matti Picus Branch: win32-fixes4 Changeset: r70176:d75ef8250857 Date: 2014-03-22 23:14 +0200 http://bitbucket.org/pypy/pypy/changeset/d75ef8250857/ Log: add MSVC macros diff --git a/rpython/jit/backend/detect_cpu.py b/rpython/jit/backend/detect_cpu.py --- a/rpython/jit/backend/detect_cpu.py +++ b/rpython/jit/backend/detect_cpu.py @@ -20,10 +20,11 @@ def detect_model_from_c_compiler(): # based on http://sourceforge.net/p/predef/wiki/Architectures/ + # and http://msdn.microsoft.com/en-us/library/b0084kay.aspx mapping = { - MODEL_X86_64: ['__amd64__', '__amd64', '__x86_64__', '__x86_64'], - MODEL_ARM: ['__arm__', '__thumb__'], - MODEL_X86: ['i386', '__i386', '__i386__', '__i686__'], + MODEL_X86_64: ['__amd64__', '__amd64', '__x86_64__', '__x86_64', '_M_X64', '_M_AMD64'], + MODEL_ARM: ['__arm__', '__thumb__','_M_ARM_EP'], + MODEL_X86: ['i386', '__i386', '__i386__', '__i686__','_M_IX86'], MODEL_PPC_64: ['__powerpc64__'], } for k, v in mapping.iteritems(): From noreply at buildbot.pypy.org Sat Mar 22 23:10:33 2014 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 22 Mar 2014 23:10:33 +0100 (CET) Subject: [pypy-commit] pypy win32-fixes4: unicode on win32 is 2 bytes Message-ID: <20140322221033.970661C0290@cobra.cs.uni-duesseldorf.de> Author: Matti Picus Branch: win32-fixes4 Changeset: r70177:c900402751c9 Date: 2014-03-22 23:21 +0200 http://bitbucket.org/pypy/pypy/changeset/c900402751c9/ Log: unicode on win32 is 2 bytes diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py --- a/rpython/jit/backend/llsupport/test/test_rewrite.py +++ b/rpython/jit/backend/llsupport/test/test_rewrite.py @@ -528,6 +528,11 @@ """) def test_rewrite_assembler_newstr_newunicode(self): + import sys + if sys.platform == 'win32': + unicode_size = 2 + else: + unicode_size = 4 self.check_rewrite(""" [i2] p0 = newstr(14) @@ -538,21 +543,21 @@ """, """ [i2] p0 = call_malloc_nursery( \ - %(strdescr.basesize + 16 * strdescr.itemsize + \ + %%(strdescr.basesize + 16 * strdescr.itemsize + \ unicodedescr.basesize + 10 * unicodedescr.itemsize)d) - setfield_gc(p0, %(strdescr.tid)d, descr=tiddescr) + setfield_gc(p0, %%(strdescr.tid)d, descr=tiddescr) setfield_gc(p0, 14, descr=strlendescr) - p1 = int_add(p0, %(strdescr.basesize + 16 * strdescr.itemsize)d) - setfield_gc(p1, %(unicodedescr.tid)d, descr=tiddescr) + p1 = int_add(p0, %%(strdescr.basesize + 16 * strdescr.itemsize)d) + setfield_gc(p1, %%(unicodedescr.tid)d, descr=tiddescr) setfield_gc(p1, 10, descr=unicodelendescr) - p2 = call_malloc_nursery_varsize(2, 4, i2, \ + p2 = call_malloc_nursery_varsize(2, %d, i2, \ descr=unicodedescr) setfield_gc(p2, i2, descr=unicodelendescr) p3 = call_malloc_nursery_varsize(1, 1, i2, \ descr=strdescr) setfield_gc(p3, i2, descr=strlendescr) jump() - """) + """ % unicode_size) def test_write_barrier_before_setfield_gc(self): self.check_rewrite(""" From noreply at buildbot.pypy.org Sat Mar 22 23:10:34 2014 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 22 Mar 2014 23:10:34 +0100 (CET) Subject: [pypy-commit] pypy win32-fixes4: fix compilation, only to discover testing gil release on extern function call fails Message-ID: <20140322221034.ED1A21C0290@cobra.cs.uni-duesseldorf.de> Author: Matti Picus Branch: win32-fixes4 Changeset: r70178:c1c9972e5a4e Date: 2014-03-22 23:42 +0200 http://bitbucket.org/pypy/pypy/changeset/c1c9972e5a4e/ Log: fix compilation, only to discover testing gil release on extern function call fails diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py --- a/rpython/jit/metainterp/test/test_ajit.py +++ b/rpython/jit/metainterp/test/test_ajit.py @@ -14,6 +14,7 @@ from rpython.rlib.longlong2float import float2longlong, longlong2float from rpython.rlib.rarithmetic import ovfcheck, is_valid_int from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.translator.tool.cbuild import ExternalCompilationInfo class BasicTests: @@ -3228,11 +3229,12 @@ self.check_resops(arraylen_gc=2) def test_release_gil_flush_heap_cache(self): + eci = ExternalCompilationInfo() if sys.platform == "win32": - py.test.skip("needs 'time'") + eci = ExternalCompilationInfo(libraries=["msvcrt"]) T = rffi.CArrayPtr(rffi.TIME_T) - external = rffi.llexternal("time", [T], rffi.TIME_T, releasegil=True) + external = rffi.llexternal("time", [T], rffi.TIME_T, releasegil=True, compilation_info=eci) # Not a real lock, has all the same properties with respect to GIL # release though, so good for this test. class Lock(object): @@ -3920,10 +3922,13 @@ self.interp_operations(f, []) def test_external_call(self): + eci = ExternalCompilationInfo() + if sys.platform == "win32": + eci = ExternalCompilationInfo(libraries=["msvcrt"]) from rpython.rlib.objectmodel import invoke_around_extcall T = rffi.CArrayPtr(rffi.TIME_T) - external = rffi.llexternal("time", [T], rffi.TIME_T) + external = rffi.llexternal("time", [T], rffi.TIME_T, compilation_info=eci) class Oups(Exception): pass From noreply at buildbot.pypy.org Sat Mar 22 23:10:36 2014 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 22 Mar 2014 23:10:36 +0100 (CET) Subject: [pypy-commit] pypy win32-fixes4: ansi C Message-ID: <20140322221036.349A81C0290@cobra.cs.uni-duesseldorf.de> Author: Matti Picus Branch: win32-fixes4 Changeset: r70179:feb18385325c Date: 2014-03-22 23:56 +0200 http://bitbucket.org/pypy/pypy/changeset/feb18385325c/ Log: ansi C diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test/test_eval.py --- a/pypy/module/cpyext/test/test_eval.py +++ b/pypy/module/cpyext/test/test_eval.py @@ -312,8 +312,9 @@ ("get_flags", "METH_NOARGS", """ PyCompilerFlags flags; + int result; flags.cf_flags = 0; - int result = PyEval_MergeCompilerFlags(&flags); + result = PyEval_MergeCompilerFlags(&flags); return Py_BuildValue("ii", result, flags.cf_flags); """), ]) From noreply at buildbot.pypy.org Sat Mar 22 23:10:37 2014 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 22 Mar 2014 23:10:37 +0100 (CET) Subject: [pypy-commit] pypy win32-fixes4: cpython compatibility: did this test ever pass with -A ? Message-ID: <20140322221037.7F2E21C0290@cobra.cs.uni-duesseldorf.de> Author: Matti Picus Branch: win32-fixes4 Changeset: r70180:0c785cb5b007 Date: 2014-03-23 00:08 +0200 http://bitbucket.org/pypy/pypy/changeset/0c785cb5b007/ Log: cpython compatibility: did this test ever pass with -A ? diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py --- a/pypy/module/_codecs/test/test_codecs.py +++ b/pypy/module/_codecs/test/test_codecs.py @@ -695,7 +695,7 @@ if sys.platform != 'win32': return assert u'test'.encode('mbcs') == 'test' - assert u'caf\xe9'.encode('mbcs') == 'caf\xe9' + assert u'caf\xe9'.encode('mbcs') == '?' assert u'\u040a'.encode('mbcs') == '?' # some cyrillic letter assert 'cafx\e9'.decode('mbcs') == u'cafx\e9' From noreply at buildbot.pypy.org Sun Mar 23 08:14:21 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 23 Mar 2014 08:14:21 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Silence warnings Message-ID: <20140323071421.C645A1C0290@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r70181:bd87e99d44f6 Date: 2014-03-23 08:13 +0100 http://bitbucket.org/pypy/pypy/changeset/bd87e99d44f6/ Log: Silence warnings diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py --- a/rpython/translator/c/genc.py +++ b/rpython/translator/c/genc.py @@ -911,7 +911,7 @@ void pypy_stm_register_thread_local(void) { stm_register_thread_local(&stm_thread_local); - stm_thread_local.mem_clear_on_abort = &pypy_g_ExcData; + stm_thread_local.mem_clear_on_abort = (char *)&pypy_g_ExcData; stm_thread_local.mem_bytes_to_clear_on_abort = sizeof(pypy_g_ExcData); } ''' diff --git a/rpython/translator/stm/src_stm/stmgcintf.c b/rpython/translator/stm/src_stm/stmgcintf.c --- a/rpython/translator/stm/src_stm/stmgcintf.c +++ b/rpython/translator/stm/src_stm/stmgcintf.c @@ -95,7 +95,8 @@ stm_jmpbuf_t jmpbuf; long volatile v_counter = 0; #ifndef NDEBUG - object_t **volatile old_shadowstack = stm_thread_local.shadowstack; + struct stm_shadowentry_s *volatile old_shadowstack = + stm_thread_local.shadowstack; #endif STM_PUSH_ROOT(stm_thread_local, arg); From noreply at buildbot.pypy.org Sun Mar 23 08:24:55 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 23 Mar 2014 08:24:55 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Un-delete this test (see 53a2a9ecde3e) and make it pass Message-ID: <20140323072455.6D9EB1C0290@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo

Installing more modules

Installing NumPy

Building from source