[pypy-commit] pypy py3k: hg merge default
mjacob
noreply at buildbot.pypy.org
Sat Aug 22 01:22:29 CEST 2015
Author: Manuel Jacob <me at manueljacob.de>
Branch: py3k
Changeset: r79128:61296bb0341b
Date: 2015-08-22 01:22 +0200
http://bitbucket.org/pypy/pypy/changeset/61296bb0341b/
Log: hg merge default
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
Metadata-Version: 1.1
Name: cffi
-Version: 1.2.0
+Version: 1.2.1
Summary: Foreign Function Interface for Python calling C code.
Home-page: http://cffi.readthedocs.org
Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
from .api import FFI, CDefError, FFIError
from .ffiplatform import VerificationError, VerificationMissing
-__version__ = "1.2.0"
-__version_info__ = (1, 2, 0)
+__version__ = "1.2.1"
+__version_info__ = (1, 2, 1)
# The verifier module file names are based on the CRC32 of a string that
# contains the following version number. It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -46,7 +46,7 @@
# endif
#else
# include <stdint.h>
-# if (defined (__SVR4) && defined (__sun)) || defined(_AIX)
+# if (defined (__SVR4) && defined (__sun)) || defined(_AIX) || defined(__hpux)
# include <alloca.h>
# endif
#endif
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -15,9 +15,11 @@
except ImportError:
lock = None
-_r_comment = re.compile(r"/\*.*?\*/|//.*?$", re.DOTALL | re.MULTILINE)
-_r_define = re.compile(r"^\s*#\s*define\s+([A-Za-z_][A-Za-z_0-9]*)\s+(.*?)$",
- re.MULTILINE)
+_r_comment = re.compile(r"/\*.*?\*/|//([^\n\\]|\\.)*?$",
+ re.DOTALL | re.MULTILINE)
+_r_define = re.compile(r"^\s*#\s*define\s+([A-Za-z_][A-Za-z_0-9]*)"
+ r"\b((?:[^\n\\]|\\.)*?)$",
+ re.DOTALL | re.MULTILINE)
_r_partial_enum = re.compile(r"=\s*\.\.\.\s*[,}]|\.\.\.\s*\}")
_r_enum_dotdotdot = re.compile(r"__dotdotdot\d+__$")
_r_partial_array = re.compile(r"\[\s*\.\.\.\s*\]")
@@ -39,6 +41,7 @@
macros = {}
for match in _r_define.finditer(csource):
macroname, macrovalue = match.groups()
+ macrovalue = macrovalue.replace('\\\n', '').strip()
macros[macroname] = macrovalue
csource = _r_define.sub('', csource)
# Replace "[...]" with "[__dotdotdotarray__]"
@@ -423,13 +426,10 @@
raise api.CDefError(
"%s: a function with only '(...)' as argument"
" is not correct C" % (funcname or 'in expression'))
- elif (len(params) == 1 and
- isinstance(params[0].type, pycparser.c_ast.TypeDecl) and
- isinstance(params[0].type.type, pycparser.c_ast.IdentifierType)
- and list(params[0].type.type.names) == ['void']):
- del params[0]
args = [self._as_func_arg(self._get_type(argdeclnode.type))
for argdeclnode in params]
+ if not ellipsis and args == [model.void_type]:
+ args = []
result = self._get_type(typenode.type)
return model.RawFunctionType(tuple(args), result, ellipsis)
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -4,11 +4,6 @@
VERSION = "0x2601"
-try:
- int_type = (int, long)
-except NameError: # Python 3
- int_type = int
-
class GlobalExpr:
def __init__(self, name, address, type_op, size=0, check_value=0):
diff --git a/lib_pypy/cffi/setuptools_ext.py b/lib_pypy/cffi/setuptools_ext.py
--- a/lib_pypy/cffi/setuptools_ext.py
+++ b/lib_pypy/cffi/setuptools_ext.py
@@ -81,10 +81,16 @@
allsources.extend(kwds.pop('sources', []))
ext = Extension(name=module_name, sources=allsources, **kwds)
- def make_mod(tmpdir):
+ def make_mod(tmpdir, pre_run=None):
c_file = os.path.join(tmpdir, module_name + source_extension)
log.info("generating cffi module %r" % c_file)
mkpath(tmpdir)
+ # a setuptools-only, API-only hook: called with the "ext" and "ffi"
+ # arguments just before we turn the ffi into C code. To use it,
+ # subclass the 'distutils.command.build_ext.build_ext' class and
+ # add a method 'def pre_run(self, ext, ffi)'.
+ if pre_run is not None:
+ pre_run(ext, ffi)
updated = recompiler.make_c_source(ffi, module_name, source, c_file)
if not updated:
log.info("already up-to-date")
@@ -98,7 +104,8 @@
class build_ext_make_mod(base_class):
def run(self):
if ext.sources[0] == '$PLACEHOLDER':
- ext.sources[0] = make_mod(self.build_temp)
+ pre_run = getattr(self, 'pre_run', None)
+ ext.sources[0] = make_mod(self.build_temp, pre_run)
base_class.run(self)
dist.cmdclass['build_ext'] = build_ext_make_mod
# NB. multiple runs here will create multiple 'build_ext_make_mod'
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,7 @@
.. toctree::
whatsnew-head.rst
+ whatsnew-2.6.1.rst
whatsnew-2.6.0.rst
whatsnew-2.5.1.rst
whatsnew-2.5.0.rst
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-2.6.1.rst
copy from pypy/doc/whatsnew-head.rst
copy to pypy/doc/whatsnew-2.6.1.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-2.6.1.rst
@@ -1,6 +1,6 @@
-=======================
-What's new in PyPy 2.6+
-=======================
+========================
+What's new in PyPy 2.6.1
+========================
.. this is a revision shortly after release-2.6.0
.. startrev: 91904d5c5188
@@ -32,7 +32,10 @@
``lst[0]`` is still *not* the float ``42.0`` but the integer ``42``.)
.. branch: cffi-callback-onerror
+Part of cffi 1.2.
+
.. branch: cffi-new-allocator
+Part of cffi 1.2.
.. branch: unicode-dtype
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -2,68 +2,6 @@
What's new in PyPy 2.6+
=======================
-.. this is a revision shortly after release-2.6.0
-.. startrev: 91904d5c5188
+.. this is a revision shortly after release-2.6.1
+.. startrev: 83ebc73d4fcb
-.. branch: use_min_scalar
-Correctly resolve the output dtype of ufunc(array, scalar) calls.
-
-.. branch: stdlib-2.7.10
-
-Update stdlib to version 2.7.10
-
-.. branch: issue2062
-
-.. branch: disable-unroll-for-short-loops
-The JIT no longer performs loop unrolling if the loop compiles to too much code.
-
-.. branch: run-create_cffi_imports
-
-Build cffi import libraries as part of translation by monkey-patching an
-additional task into translation
-
-.. branch: int-float-list-strategy
-
-Use a compact strategy for Python lists that mix integers and floats,
-at least if the integers fit inside 32 bits. These lists are now
-stored as an array of floats, like lists that contain only floats; the
-difference is that integers are stored as tagged NaNs. (This should
-have no visible effect! After ``lst = [42, 42.5]``, the value of
-``lst[0]`` is still *not* the float ``42.0`` but the integer ``42``.)
-
-.. branch: cffi-callback-onerror
-.. branch: cffi-new-allocator
-
-.. branch: unicode-dtype
-
-Partial implementation of unicode dtype and unicode scalars.
-
-.. branch: dtypes-compatability
-
-Improve compatibility with numpy dtypes; handle offsets to create unions,
-fix str() and repr(), allow specifying itemsize, metadata and titles, add flags,
-allow subclassing dtype
-
-.. branch: indexing
-
-Refactor array indexing to support ellipses.
-
-.. branch: numpy-docstrings
-
-Allow the docstrings of built-in numpy objects to be set at run-time.
-
-.. branch: nditer-revisited
-
-Implement nditer 'buffered' flag and fix some edge cases
-
-.. branch: ufunc-reduce
-
-Allow multiple axes in ufunc.reduce()
-
-.. branch: fix-tinylang-goals
-
-Update tinylang goals to match current rpython
-
-.. branch: vmprof-review
-
-Clean up of vmprof, notably to handle correctly multiple threads
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -2,7 +2,7 @@
from pypy.interpreter.mixedmodule import MixedModule
from rpython.rlib import rdynload
-VERSION = "1.2.0"
+VERSION = "1.2.1"
class Module(MixedModule):
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -3427,4 +3427,4 @@
def test_version():
# this test is here mostly for PyPy
- assert __version__ == "1.2.0"
+ assert __version__ == "1.2.1"
diff --git a/pypy/module/_vmprof/test/test__vmprof.py b/pypy/module/_vmprof/test/test__vmprof.py
--- a/pypy/module/_vmprof/test/test__vmprof.py
+++ b/pypy/module/_vmprof/test/test__vmprof.py
@@ -21,11 +21,12 @@
i = 0
count = 0
i += 5 * WORD # header
- assert s[i] == 4
- i += 1 # marker
- assert s[i] == 4
- i += 1 # length
- i += len('pypy')
+ assert s[i ] == 5 # MARKER_HEADER
+ assert s[i + 1] == 0 # 0
+ assert s[i + 2] == 1 # VERSION_THREAD_ID
+ assert s[i + 3] == 4 # len('pypy')
+ assert s[i + 4: i + 8] == 'pypy'
+ i += 8
while i < len(s):
if s[i] == 3:
break
diff --git a/pypy/module/_vmprof/test/test_direct.py b/pypy/module/_vmprof/test/test_direct.py
--- a/pypy/module/_vmprof/test/test_direct.py
+++ b/pypy/module/_vmprof/test/test_direct.py
@@ -42,7 +42,7 @@
}
-""" + open(str(srcdir.join("rvmprof_get_custom_offset.h"))).read())
+""" + open(str(srcdir.join("vmprof_get_custom_offset.h"))).read())
class TestDirect(object):
def test_infrastructure(self):
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -364,6 +364,8 @@
PyObject *ht_name, *ht_slots;
} PyHeapTypeObject;
+#define PyObject_Bytes PyObject_Str
+
/* Flag bits for printing: */
#define Py_PRINT_RAW 1 /* No string quotes etc. */
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
@@ -160,6 +160,35 @@
assert func.name == 'sin'
assert func.BType == '<func (<double>, <double>), <double>, False>'
+def test_remove_line_continuation_comments():
+ ffi = FFI(backend=FakeBackend())
+ ffi.cdef("""
+ double // blah \\
+ more comments
+ x(void);
+ double // blah\\\\
+ y(void);
+ double // blah\\ \
+ etc
+ z(void);
+ """)
+ m = ffi.dlopen(lib_m)
+ m.x
+ m.y
+ m.z
+
+def test_line_continuation_in_defines():
+ ffi = FFI(backend=FakeBackend())
+ ffi.cdef("""
+ #define ABC\\
+ 42
+ #define BCD \\
+ 43
+ """)
+ m = ffi.dlopen(lib_m)
+ assert m.ABC == 42
+ assert m.BCD == 43
+
def test_define_not_supported_for_now():
ffi = FFI(backend=FakeBackend())
e = py.test.raises(CDefError, ffi.cdef, '#define FOO "blah"')
@@ -238,6 +267,13 @@
ffi = FFI()
ffi.cdef("typedef _Bool bool; void f(bool);")
+def test_void_renamed_as_only_arg():
+ ffi = FFI()
+ ffi.cdef("typedef void void_t1;"
+ "typedef void_t1 void_t;"
+ "typedef int (*func_t)(void_t);")
+ assert ffi.typeof("func_t").args == ()
+
def test_win_common_types():
from cffi.commontypes import COMMON_TYPES, _CACHE
from cffi.commontypes import win_common_types, resolve_common_type
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -25,6 +25,9 @@
if 1: # test the .cpp mode too
kwds.setdefault('source_extension', '.cpp')
source = 'extern "C" {\n%s\n}' % (source,)
+ else:
+ kwds['extra_compile_args'] = (kwds.get('extra_compile_args', []) +
+ ['-Werror'])
return recompiler._verify(ffi, module_name, source, *args, **kwds)
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py
@@ -318,15 +318,32 @@
import cffi
ffi = cffi.FFI()
ffi.set_source("pack3.mymod", "/*code would be here*/")
+ ffi._hi_there = 42
""")
with open("setup.py", "w") as f:
- f.write("""if 1:
+ f.write("from __future__ import print_function\n"
+ """if 1:
from setuptools import setup
+ from distutils.command.build_ext import build_ext
+ import os
+
+ class TestBuildExt(build_ext):
+ def pre_run(self, ext, ffi):
+ print('_make_setuptools_api: in pre_run:', end=" ")
+ assert ffi._hi_there == 42
+ assert ext.name == "pack3.mymod"
+ fn = os.path.join(os.path.dirname(self.build_lib),
+ '..', 'see_me')
+ print('creating %r' % (fn,))
+ open(fn, 'w').close()
+
setup(name='example1',
version='0.1',
packages=['pack3'],
package_dir={'': 'src1'},
- cffi_modules=["src1/pack3/_build.py:ffi"])
+ cffi_modules=["src1/pack3/_build.py:ffi"],
+ cmdclass={'build_ext': TestBuildExt},
+ )
""")
@chdir_to_tmp
@@ -335,6 +352,7 @@
self.run(["setup.py", "build"])
self.check_produced_files({'setup.py': None,
'build': '?',
+ 'see_me': None,
'src1': {'pack3': {'__init__.py': None,
'_build.py': None}}})
@@ -344,6 +362,7 @@
self.run(["setup.py", "build_ext", "-i"])
self.check_produced_files({'setup.py': None,
'build': '?',
+ 'see_me': None,
'src1': {'pack3': {'__init__.py': None,
'_build.py': None,
'mymod.SO': None}}})
diff --git a/rpython/flowspace/objspace.py b/rpython/flowspace/objspace.py
--- a/rpython/flowspace/objspace.py
+++ b/rpython/flowspace/objspace.py
@@ -13,6 +13,11 @@
def _assert_rpythonic(func):
"""Raise ValueError if ``func`` is obviously not RPython"""
+ try:
+ func.func_code.co_cellvars
+ except AttributeError:
+ raise ValueError("%r is not RPython: it is likely an unexpected "
+ "built-in function or type" % (func,))
if func.func_doc and func.func_doc.lstrip().startswith('NOT_RPYTHON'):
raise ValueError("%r is tagged as NOT_RPYTHON" % (func,))
if func.func_code.co_cellvars:
diff --git a/rpython/flowspace/test/test_objspace.py b/rpython/flowspace/test/test_objspace.py
--- a/rpython/flowspace/test/test_objspace.py
+++ b/rpython/flowspace/test/test_objspace.py
@@ -1363,6 +1363,15 @@
simplify_graph(graph)
assert self.all_operations(graph) == {'bool': 1, 'inplace_add': 1}
+ def test_unexpected_builtin_function(self):
+ import itertools
+ e = py.test.raises(ValueError, build_flow, itertools.permutations)
+ assert ' is not RPython:' in str(e.value)
+ e = py.test.raises(ValueError, build_flow, itertools.tee)
+ assert ' is not RPython:' in str(e.value)
+ e = py.test.raises(ValueError, build_flow, Exception.__init__)
+ assert ' is not RPython:' in str(e.value)
+
DATA = {'x': 5,
'y': 6}
diff --git a/rpython/jit/backend/arm/runner.py b/rpython/jit/backend/arm/runner.py
--- a/rpython/jit/backend/arm/runner.py
+++ b/rpython/jit/backend/arm/runner.py
@@ -64,12 +64,6 @@
operations,
original_loop_token, log=log)
- def clear_latest_values(self, count):
- setitem = self.assembler.fail_boxes_ptr.setitem
- null = lltype.nullptr(llmemory.GCREF.TO)
- for index in range(count):
- setitem(index, null)
-
def cast_ptr_to_int(x):
adr = llmemory.cast_ptr_to_adr(x)
return CPU_ARM.cast_adr_to_int(adr)
diff --git a/rpython/jit/backend/llsupport/regalloc.py b/rpython/jit/backend/llsupport/regalloc.py
--- a/rpython/jit/backend/llsupport/regalloc.py
+++ b/rpython/jit/backend/llsupport/regalloc.py
@@ -636,8 +636,7 @@
assert isinstance(box, Box)
loc = self.fm.get_new_loc(box)
locs.append(loc.value - base_ofs)
- if looptoken.compiled_loop_token is not None:
- # for tests
+ if looptoken.compiled_loop_token is not None: # <- for tests
looptoken.compiled_loop_token._ll_initial_locs = locs
def can_merge_with_next_guard(self, op, i, operations):
diff --git a/rpython/jit/backend/llsupport/src/codemap.c b/rpython/jit/backend/llsupport/src/codemap.c
--- a/rpython/jit/backend/llsupport/src/codemap.c
+++ b/rpython/jit/backend/llsupport/src/codemap.c
@@ -6,9 +6,9 @@
#endif
#ifdef RPYTHON_VMPROF
-RPY_EXTERN void rpython_vmprof_ignore_signals(int ignored);
+RPY_EXTERN void vmprof_ignore_signals(int ignored);
static void pypy_codemap_invalid_set(int ignored) {
- rpython_vmprof_ignore_signals(ignored);
+ vmprof_ignore_signals(ignored);
}
#else
static void pypy_codemap_invalid_set(int ignored) {
diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py
--- a/rpython/jit/backend/x86/runner.py
+++ b/rpython/jit/backend/x86/runner.py
@@ -100,12 +100,6 @@
return self.assembler.assemble_bridge(faildescr, inputargs, operations,
original_loop_token, log, logger)
- def clear_latest_values(self, count):
- setitem = self.assembler.fail_boxes_ptr.setitem
- null = lltype.nullptr(llmemory.GCREF.TO)
- for index in range(count):
- setitem(index, null)
-
def cast_ptr_to_int(x):
adr = llmemory.cast_ptr_to_adr(x)
return CPU386.cast_adr_to_int(adr)
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -268,6 +268,7 @@
y -= 1
return res
res = self.meta_interp(f, [6, sys.maxint, 48])
+ self.check_trace_count(6)
assert res == f(6, sys.maxint, 48)
def test_loop_invariant_mul_bridge_ovf2(self):
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -1087,6 +1087,16 @@
"""
assert type(value) is cls
+def ll_record_exact_class(ll_value, ll_cls):
+ from rpython.rlib.debug import ll_assert
+ from rpython.rtyper.lltypesystem.lloperation import llop
+ from rpython.rtyper.lltypesystem import lltype
+ from rpython.rtyper.rclass import ll_type
+ ll_assert(ll_value == lltype.nullptr(lltype.typeOf(ll_value).TO), "record_exact_class called with None argument")
+ ll_assert(ll_type(ll_value) is ll_cls, "record_exact_class called with invalid arguments")
+ llop.jit_record_exact_class(lltype.Void, ll_value, ll_cls)
+
+
class Entry(ExtRegistryEntry):
_about_ = record_exact_class
@@ -1100,12 +1110,10 @@
from rpython.rtyper import rclass
classrepr = rclass.get_type_repr(hop.rtyper)
-
- hop.exception_cannot_occur()
v_inst = hop.inputarg(hop.args_r[0], arg=0)
v_cls = hop.inputarg(classrepr, arg=1)
- return hop.genop('jit_record_exact_class', [v_inst, v_cls],
- resulttype=lltype.Void)
+ hop.exception_is_here()
+ return hop.gendirectcall(ll_record_exact_class, v_inst, v_cls)
def _jit_conditional_call(condition, function, *args):
pass
diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -40,24 +40,20 @@
**eci_kwds))
- vmprof_init = rffi.llexternal("rpython_vmprof_init", [rffi.INT], rffi.CCHARP,
- compilation_info=eci)
- vmprof_enable = rffi.llexternal("rpython_vmprof_enable", [rffi.LONG], rffi.INT,
+ vmprof_init = rffi.llexternal("vmprof_init",
+ [rffi.INT, rffi.DOUBLE, rffi.CCHARP],
+ rffi.CCHARP, compilation_info=eci)
+ vmprof_enable = rffi.llexternal("vmprof_enable", [], rffi.INT,
compilation_info=eci,
save_err=rffi.RFFI_SAVE_ERRNO)
- vmprof_disable = rffi.llexternal("rpython_vmprof_disable", [], rffi.INT,
+ vmprof_disable = rffi.llexternal("vmprof_disable", [], rffi.INT,
compilation_info=eci,
save_err=rffi.RFFI_SAVE_ERRNO)
- vmprof_write_buf = rffi.llexternal("rpython_vmprof_write_buf",
- [rffi.CCHARP, rffi.LONG],
- lltype.Void, compilation_info=eci)
-
- ## vmprof_register_virtual_function = rffi.llexternal(
- ## "vmprof_register_virtual_function",
- ## [rffi.CCHARP, rffi.VOIDP, rffi.VOIDP], lltype.Void,
- ## compilation_info=eci, _nowrapper=True)
-
- vmprof_ignore_signals = rffi.llexternal("rpython_vmprof_ignore_signals",
+ vmprof_register_virtual_function = rffi.llexternal(
+ "vmprof_register_virtual_function",
+ [rffi.CCHARP, rffi.LONG, rffi.INT],
+ rffi.INT, compilation_info=eci)
+ vmprof_ignore_signals = rffi.llexternal("vmprof_ignore_signals",
[rffi.INT], lltype.Void,
compilation_info=eci)
return CInterface(locals())
@@ -83,10 +79,20 @@
cont_name = 'rpyvmprof_f_%s_%s' % (name, token)
tramp_name = 'rpyvmprof_t_%s_%s' % (name, token)
+ orig_tramp_name = tramp_name
func.c_name = cont_name
func._dont_inline_ = True
+ if sys.platform == 'darwin':
+ # according to internet "At the time UNIX was written in 1974...."
+ # "... all C functions are prefixed with _"
+ cont_name = '_' + cont_name
+ tramp_name = '_' + tramp_name
+ PLT = ""
+ else:
+ PLT = "@PLT"
+
assert detect_cpu.autodetect().startswith(detect_cpu.MODEL_X86_64), (
"rvmprof only supports x86-64 CPUs for now")
@@ -111,17 +117,15 @@
target.write("""\
\t.text
\t.globl\t%(tramp_name)s
-\t.type\t%(tramp_name)s, @function
%(tramp_name)s:
\t.cfi_startproc
\tpushq\t%(reg)s
\t.cfi_def_cfa_offset 16
-\tcall %(cont_name)s at PLT
+\tcall %(cont_name)s%(PLT)s
\taddq\t$8, %%rsp
\t.cfi_def_cfa_offset 8
\tret
\t.cfi_endproc
-\t.size\t%(tramp_name)s, .-%(tramp_name)s
""" % locals())
def tok2cname(tok):
@@ -133,7 +137,7 @@
header = 'RPY_EXTERN %s %s(%s);\n' % (
tok2cname(restok),
- tramp_name,
+ orig_tramp_name,
', '.join([tok2cname(tok) for tok in token] + ['long']))
header += """\
@@ -147,7 +151,7 @@
#endif
#define VMPROF_ADDR_OF_TRAMPOLINE cmp_%s
}
-""" % (tramp_name, tramp_name, tramp_name)
+""" % (tramp_name, orig_tramp_name, tramp_name)
eci = ExternalCompilationInfo(
post_include_bits = [header],
@@ -155,7 +159,7 @@
)
return rffi.llexternal(
- tramp_name,
+ orig_tramp_name,
[token2lltype(tok) for tok in token] + [lltype.Signed],
token2lltype(restok),
compilation_info=eci,
diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -1,14 +1,12 @@
import sys, os
from rpython.rlib.objectmodel import specialize, we_are_translated
-from rpython.rlib.rstring import StringBuilder
from rpython.rlib import jit, rgc, rposix
from rpython.rlib.rvmprof import cintf
from rpython.rtyper.annlowlevel import cast_instance_to_gcref
from rpython.rtyper.annlowlevel import cast_base_ptr_to_instance
from rpython.rtyper.lltypesystem import rffi
-MAX_CODES = 8000 - 255
-MAX_FUNC_NAME = 255
+MAX_FUNC_NAME = 1023
# ____________________________________________________________
@@ -34,8 +32,6 @@
def _cleanup_(self):
self.is_enabled = False
- self.fileno = -1
- self._current_codes = None
@specialize.argtype(1)
def register_code(self, code, full_name_func):
@@ -102,18 +98,13 @@
assert fileno >= 0
if self.is_enabled:
raise VMProfError("vmprof is already enabled")
- if not (1e-6 <= interval < 1.0):
- raise VMProfError("bad value for 'interval'")
- interval_usec = int(interval * 1000000.0)
- p_error = self.cintf.vmprof_init(fileno)
+ p_error = self.cintf.vmprof_init(fileno, interval, "pypy")
if p_error:
raise VMProfError(rffi.charp2str(p_error))
- self.fileno = fileno
- self._write_header(interval_usec)
self._gather_all_code_objs()
- res = self.cintf.vmprof_enable(interval_usec)
+ res = self.cintf.vmprof_enable()
if res < 0:
raise VMProfError(os.strerror(rposix.get_saved_errno()))
self.is_enabled = True
@@ -125,9 +116,6 @@
if not self.is_enabled:
raise VMProfError("vmprof is not enabled")
self.is_enabled = False
- if self._current_codes is not None:
- self._flush_codes()
- self.fileno = -1
res = self.cintf.vmprof_disable()
if res < 0:
raise VMProfError(os.strerror(rposix.get_saved_errno()))
@@ -136,48 +124,8 @@
assert name.count(':') == 3 and len(name) <= MAX_FUNC_NAME, (
"the name must be 'class:func_name:func_line:filename' "
"and at most %d characters; got '%s'" % (MAX_FUNC_NAME, name))
- b = self._current_codes
- if b is None:
- b = self._current_codes = StringBuilder()
- b.append('\x02')
- _write_long_to_string_builder(uid, b)
- _write_long_to_string_builder(len(name), b)
- b.append(name)
- if b.getlength() >= MAX_CODES:
- self._flush_codes()
-
- def _flush_codes(self):
- buf = self._current_codes.build()
- self._current_codes = None
- self.cintf.vmprof_write_buf(buf, len(buf))
- # NOTE: keep in mind that vmprof_write_buf() can only write
- # a maximum of 8184 bytes. This should be guaranteed here because:
- assert MAX_CODES + 17 + MAX_FUNC_NAME <= 8184
-
- def _write_header(self, interval_usec):
- b = StringBuilder()
- _write_long_to_string_builder(0, b)
- _write_long_to_string_builder(3, b)
- _write_long_to_string_builder(0, b)
- _write_long_to_string_builder(interval_usec, b)
- _write_long_to_string_builder(0, b)
- b.append('\x04') # interp name
- b.append(chr(len('pypy')))
- b.append('pypy')
- buf = b.build()
- self.cintf.vmprof_write_buf(buf, len(buf))
-
-
-def _write_long_to_string_builder(l, b):
- b.append(chr(l & 0xff))
- b.append(chr((l >> 8) & 0xff))
- b.append(chr((l >> 16) & 0xff))
- b.append(chr((l >> 24) & 0xff))
- if sys.maxint > 2147483647:
- b.append(chr((l >> 32) & 0xff))
- b.append(chr((l >> 40) & 0xff))
- b.append(chr((l >> 48) & 0xff))
- b.append(chr((l >> 56) & 0xff))
+ if self.cintf.vmprof_register_virtual_function(name, uid, 500000) < 0:
+ raise VMProfError("vmprof buffers full! disk full or too slow")
def vmprof_execute_code(name, get_code_fn, result_class=None):
diff --git a/rpython/rlib/rvmprof/src/rvmprof.c b/rpython/rlib/rvmprof/src/rvmprof.c
--- a/rpython/rlib/rvmprof/src/rvmprof.c
+++ b/rpython/rlib/rvmprof/src/rvmprof.c
@@ -1,22 +1,3 @@
-/* VMPROF
- *
- * statistical sampling profiler specifically designed to profile programs
- * which run on a Virtual Machine and/or bytecode interpreter, such as Python,
- * etc.
- *
- * The logic to dump the C stack traces is partly stolen from the code in
- * gperftools.
- * The file "getpc.h" has been entirely copied from gperftools.
- *
- * Tested only on gcc, linux, x86_64.
- *
- * Copyright (C) 2014-2015
- * Antonio Cuni - anto.cuni at gmail.com
- * Maciej Fijalkowski - fijall at gmail.com
- * Armin Rigo - arigo at tunes.org
- *
- */
-
#define _GNU_SOURCE 1
@@ -39,431 +20,4 @@
#endif
-#include <dlfcn.h>
-#include <assert.h>
-#include <pthread.h>
-#include <sys/time.h>
-#include <errno.h>
-#include <unistd.h>
-#include <stdio.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include "rvmprof_getpc.h"
-#include "rvmprof_unwind.h"
-#include "rvmprof_mt.h"
-
-
-/************************************************************/
-
-// functions copied from libunwind using dlopen
-
-static int (*unw_get_reg)(unw_cursor_t*, int, unw_word_t*) = NULL;
-static int (*unw_step)(unw_cursor_t*) = NULL;
-static int (*unw_init_local)(unw_cursor_t *, unw_context_t *) = NULL;
-static int (*unw_get_proc_info)(unw_cursor_t *, unw_proc_info_t *) = NULL;
-
-static int profile_file = -1;
-
-
-RPY_EXTERN
-char *rpython_vmprof_init(int fd)
-{
- if (!unw_get_reg) {
- void *libhandle;
-
- if (!(libhandle = dlopen("libunwind.so", RTLD_LAZY | RTLD_LOCAL)))
- goto error;
- if (!(unw_get_reg = dlsym(libhandle, "_ULx86_64_get_reg")))
- goto error;
- if (!(unw_get_proc_info = dlsym(libhandle, "_ULx86_64_get_proc_info")))
- goto error;
- if (!(unw_init_local = dlsym(libhandle, "_ULx86_64_init_local")))
- goto error;
- if (!(unw_step = dlsym(libhandle, "_ULx86_64_step")))
- goto error;
- }
- if (prepare_concurrent_bufs() < 0)
- return "out of memory";
-
- assert(fd >= 0);
- profile_file = fd;
- return NULL;
-
- error:
- return dlerror();
-}
-
-/************************************************************/
-
-/* value: last bit is 1 if signals must be ignored; all other bits
- are a counter for how many threads are currently in a signal handler */
-static long volatile signal_handler_value = 1;
-
-RPY_EXTERN
-void rpython_vmprof_ignore_signals(int ignored)
-{
- if (!ignored) {
- __sync_fetch_and_and(&signal_handler_value, ~1L);
- }
- else {
- /* set the last bit, and wait until concurrently-running signal
- handlers finish */
- while (__sync_or_and_fetch(&signal_handler_value, 1L) != 1L) {
- usleep(1);
- }
- }
-}
-
-
-/* *************************************************************
- * functions to write a profile file compatible with gperftools
- * *************************************************************
- */
-
-#define MAX_FUNC_NAME 128
-#define MAX_STACK_DEPTH \
- ((SINGLE_BUF_SIZE - sizeof(struct prof_stacktrace_s)) / sizeof(void *))
-
-#define MARKER_STACKTRACE '\x01'
-#define MARKER_VIRTUAL_IP '\x02'
-#define MARKER_TRAILER '\x03'
-
-struct prof_stacktrace_s {
- char padding[sizeof(long) - 1];
- char marker;
- long count, depth;
- void *stack[];
-};
-
-static long profile_interval_usec = 0;
-static char atfork_hook_installed = 0;
-
-
-/* ******************************************************
- * libunwind workaround for process JIT frames correctly
- * ******************************************************
- */
-
-#include "rvmprof_get_custom_offset.h"
-
-typedef struct {
- void* _unused1;
- void* _unused2;
- void* sp;
- void* ip;
- void* _unused3[sizeof(unw_cursor_t)/sizeof(void*) - 4];
-} vmprof_hacked_unw_cursor_t;
-
-static int vmprof_unw_step(unw_cursor_t *cp, int first_run)
-{
- void* ip;
- void* sp;
- ptrdiff_t sp_offset;
- unw_get_reg (cp, UNW_REG_IP, (unw_word_t*)&ip);
- unw_get_reg (cp, UNW_REG_SP, (unw_word_t*)&sp);
- if (!first_run) {
- // make sure we're pointing to the CALL and not to the first
- // instruction after. If the callee adjusts the stack for us
- // it's not safe to be at the instruction after
- ip -= 1;
- }
- sp_offset = vmprof_unw_get_custom_offset(ip, cp);
-
- if (sp_offset == -1) {
- // it means that the ip is NOT in JITted code, so we can use the
- // stardard unw_step
- return unw_step(cp);
- }
- else {
- // this is a horrible hack to manually walk the stack frame, by
- // setting the IP and SP in the cursor
- vmprof_hacked_unw_cursor_t *cp2 = (vmprof_hacked_unw_cursor_t*)cp;
- void* bp = (void*)sp + sp_offset;
- cp2->sp = bp;
- bp -= sizeof(void*);
- cp2->ip = ((void**)bp)[0];
- // the ret is on the top of the stack minus WORD
- return 1;
- }
-}
-
-
-/* *************************************************************
- * functions to dump the stack trace
- * *************************************************************
- */
-
-static int get_stack_trace(void** result, int max_depth, ucontext_t *ucontext)
-{
- void *ip;
- int n = 0;
- unw_cursor_t cursor;
- unw_context_t uc = *ucontext;
-
- int ret = unw_init_local(&cursor, &uc);
- assert(ret >= 0);
- (void)ret;
-
- while (n < max_depth) {
- if (unw_get_reg(&cursor, UNW_REG_IP, (unw_word_t *) &ip) < 0) {
- break;
- }
-
- unw_proc_info_t pip;
- unw_get_proc_info(&cursor, &pip);
-
- /* if n==0, it means that the signal handler interrupted us while we
- were in the trampoline, so we are not executing (yet) the real main
- loop function; just skip it */
- if (VMPROF_ADDR_OF_TRAMPOLINE((void*)pip.start_ip) && n > 0) {
- // found main loop stack frame
- void* sp;
- unw_get_reg(&cursor, UNW_REG_SP, (unw_word_t *) &sp);
- void *arg_addr = (char*)sp /* + mainloop_sp_offset */;
- void **arg_ptr = (void**)arg_addr;
- /* if (mainloop_get_virtual_ip) {
- ip = mainloop_get_virtual_ip(*arg_ptr);
- } else { */
- ip = *arg_ptr;
- }
-
- int first_run = (n == 0);
- result[n++] = ip;
- n = vmprof_write_header_for_jit_addr(result, n, ip, max_depth);
- if (vmprof_unw_step(&cursor, first_run) <= 0)
- break;
- }
- return n;
-}
-
-
-/* *************************************************************
- * the signal handler
- * *************************************************************
- */
-
-static void sigprof_handler(int sig_nr, siginfo_t* info, void *ucontext)
-{
- long val = __sync_fetch_and_add(&signal_handler_value, 2L);
-
- if ((val & 1) == 0) {
- int saved_errno = errno;
- int fd = profile_file;
- assert(fd >= 0);
-
- struct profbuf_s *p = reserve_buffer(fd);
- if (p == NULL) {
- /* ignore this signal: there are no free buffers right now */
- }
- else {
- int depth;
- struct prof_stacktrace_s *st = (struct prof_stacktrace_s *)p->data;
- st->marker = MARKER_STACKTRACE;
- st->count = 1;
- st->stack[0] = GetPC((ucontext_t*)ucontext);
- depth = get_stack_trace(st->stack+1, MAX_STACK_DEPTH-1, ucontext);
- depth++; // To account for pc value in stack[0];
- st->depth = depth;
- p->data_offset = offsetof(struct prof_stacktrace_s, marker);
- p->data_size = (depth * sizeof(void *) +
- sizeof(struct prof_stacktrace_s) -
- offsetof(struct prof_stacktrace_s, marker));
- commit_buffer(fd, p);
- }
-
- errno = saved_errno;
- }
-
- __sync_sub_and_fetch(&signal_handler_value, 2L);
-}
-
-
-/* *************************************************************
- * the setup and teardown functions
- * *************************************************************
- */
-
-static int install_sigprof_handler(void)
-{
- struct sigaction sa;
- memset(&sa, 0, sizeof(sa));
- sa.sa_sigaction = sigprof_handler;
- sa.sa_flags = SA_RESTART | SA_SIGINFO;
- if (sigemptyset(&sa.sa_mask) == -1 ||
- sigaction(SIGPROF, &sa, NULL) == -1)
- return -1;
- return 0;
-}
-
-static int remove_sigprof_handler(void)
-{
- if (signal(SIGPROF, SIG_DFL) == SIG_ERR)
- return -1;
- return 0;
-}
-
-static int install_sigprof_timer(void)
-{
- static struct itimerval timer;
- timer.it_interval.tv_sec = 0;
- timer.it_interval.tv_usec = profile_interval_usec;
- timer.it_value = timer.it_interval;
- if (setitimer(ITIMER_PROF, &timer, NULL) != 0)
- return -1;
- return 0;
-}
-
-static int remove_sigprof_timer(void) {
- static struct itimerval timer;
- timer.it_interval.tv_sec = 0;
- timer.it_interval.tv_usec = 0;
- timer.it_value.tv_sec = 0;
- timer.it_value.tv_usec = 0;
- if (setitimer(ITIMER_PROF, &timer, NULL) != 0)
- return -1;
- return 0;
-}
-
-static void atfork_disable_timer(void) {
- if (profile_interval_usec > 0) {
- remove_sigprof_timer();
- }
-}
-
-static void atfork_enable_timer(void) {
- if (profile_interval_usec > 0) {
- install_sigprof_timer();
- }
-}
-
-static int install_pthread_atfork_hooks(void) {
- /* this is needed to prevent the problems described there:
- - http://code.google.com/p/gperftools/issues/detail?id=278
- - http://lists.debian.org/debian-glibc/2010/03/msg00161.html
-
- TL;DR: if the RSS of the process is large enough, the clone() syscall
- will be interrupted by the SIGPROF before it can complete, then
- retried, interrupted again and so on, in an endless loop. The
- solution is to disable the timer around the fork, and re-enable it
- only inside the parent.
- */
- if (atfork_hook_installed)
- return 0;
- int ret = pthread_atfork(atfork_disable_timer, atfork_enable_timer, NULL);
- if (ret != 0)
- return -1;
- atfork_hook_installed = 1;
- return 0;
-}
-
-RPY_EXTERN
-int rpython_vmprof_enable(long interval_usec)
-{
- assert(profile_file >= 0);
- assert(interval_usec > 0);
- profile_interval_usec = interval_usec;
-
- if (install_pthread_atfork_hooks() == -1)
- goto error;
- if (install_sigprof_handler() == -1)
- goto error;
- if (install_sigprof_timer() == -1)
- goto error;
- rpython_vmprof_ignore_signals(0);
- return 0;
-
- error:
- profile_file = -1;
- profile_interval_usec = 0;
- return -1;
-}
-
-static int _write_all(const void *buf, size_t bufsize)
-{
- while (bufsize > 0) {
- ssize_t count = write(profile_file, buf, bufsize);
- if (count <= 0)
- return -1; /* failed */
- buf += count;
- bufsize -= count;
- }
- return 0;
-}
-
-static int close_profile(void)
-{
- char buf[4096];
- ssize_t size;
- unsigned char marker = MARKER_TRAILER;
-
- if (_write_all(&marker, 1) < 0)
- return -1;
-
-#ifdef __linux__
- // copy /proc/self/maps to the end of the profile file
- int srcfd = open("/proc/self/maps", O_RDONLY);
- if (srcfd < 0)
- return -1;
-
- while ((size = read(srcfd, buf, sizeof buf)) > 0) {
- if (_write_all(buf, size) < 0) {
- close(srcfd);
- return -1;
- }
- }
- close(srcfd);
-#else
- // freebsd and mac
- sprintf(buf, "procstat -v %d", getpid());
- FILE *srcf = popen(buf, "r");
- if (!srcf)
- return -1;
-
- while ((size = fread(buf, 1, sizeof buf, src))) {
- if (_write_all(buf, size) < 0) {
- pclose(srcf);
- return -1;
- }
- }
- pclose(srcf);
-#endif
-
- /* don't close() the file descriptor from here */
- profile_file = -1;
- return 0;
-}
-
-RPY_EXTERN
-int rpython_vmprof_disable(void)
-{
- rpython_vmprof_ignore_signals(1);
- profile_interval_usec = 0;
-
- if (remove_sigprof_timer() == -1)
- return -1;
- if (remove_sigprof_handler() == -1)
- return -1;
- if (shutdown_concurrent_bufs(profile_file) < 0)
- return -1;
- return close_profile();
-}
-
-RPY_EXTERN
-void rpython_vmprof_write_buf(char *buf, long size)
-{
- struct profbuf_s *p;
-
- while ((p = reserve_buffer(profile_file)) == NULL) {
- /* spin loop waiting for a buffer to be ready; should almost never
- be the case */
- usleep(1);
- }
-
- if (size > SINGLE_BUF_SIZE)
- size = SINGLE_BUF_SIZE;
- memcpy(p->data, buf, size);
- p->data_size = size;
-
- commit_buffer(profile_file, p);
-}
+#include "vmprof_main.h"
diff --git a/rpython/rlib/rvmprof/src/rvmprof.h b/rpython/rlib/rvmprof/src/rvmprof.h
--- a/rpython/rlib/rvmprof/src/rvmprof.h
+++ b/rpython/rlib/rvmprof/src/rvmprof.h
@@ -1,6 +1,6 @@
-RPY_EXTERN char *rpython_vmprof_init(int);
-RPY_EXTERN void rpython_vmprof_ignore_signals(int);
-RPY_EXTERN int rpython_vmprof_enable(long);
-RPY_EXTERN int rpython_vmprof_disable(void);
-RPY_EXTERN void rpython_vmprof_write_buf(char *, long);
+RPY_EXTERN char *vmprof_init(int, double, char *);
+RPY_EXTERN void vmprof_ignore_signals(int);
+RPY_EXTERN int vmprof_enable(void);
+RPY_EXTERN int vmprof_disable(void);
+RPY_EXTERN int vmprof_register_virtual_function(char *, long, int);
diff --git a/rpython/rlib/rvmprof/src/rvmprof_config.h b/rpython/rlib/rvmprof/src/vmprof_config.h
rename from rpython/rlib/rvmprof/src/rvmprof_config.h
rename to rpython/rlib/rvmprof/src/vmprof_config.h
diff --git a/rpython/rlib/rvmprof/src/rvmprof_get_custom_offset.h b/rpython/rlib/rvmprof/src/vmprof_get_custom_offset.h
rename from rpython/rlib/rvmprof/src/rvmprof_get_custom_offset.h
rename to rpython/rlib/rvmprof/src/vmprof_get_custom_offset.h
--- a/rpython/rlib/rvmprof/src/rvmprof_get_custom_offset.h
+++ b/rpython/rlib/rvmprof/src/vmprof_get_custom_offset.h
@@ -7,12 +7,69 @@
#endif
+#ifdef CPYTHON_GET_CUSTOM_OFFSET
+static void *tramp_start, *tramp_end;
+#endif
+
+
static ptrdiff_t vmprof_unw_get_custom_offset(void* ip, void *cp) {
-#ifdef PYPY_JIT_CODEMAP
+
+#if defined(PYPY_JIT_CODEMAP)
+
intptr_t ip_l = (intptr_t)ip;
return pypy_jit_stack_depth_at_loc(ip_l);
+
+#elif defined(CPYTHON_GET_CUSTOM_OFFSET)
+
+ if (ip >= tramp_start && ip <= tramp_end) {
+ // XXX the return value is wrong for all the places before push and
+ // after pop, fix
+ void *bp;
+ void *sp;
+
+ /* This is a stage2 trampoline created by hotpatch:
+
+ push %rbx
+ push %rbp
+ mov %rsp,%rbp
+ and $0xfffffffffffffff0,%rsp // make sure the stack is aligned
+ movabs $0x7ffff687bb10,%rbx
+ callq *%rbx
+ leaveq
+ pop %rbx
+ retq
+
+ the stack layout is like this:
+
+ +-----------+ high addresses
+ | ret addr |
+ +-----------+
+ | saved rbx | start of the function frame
+ +-----------+
+ | saved rbp |
+ +-----------+
+ | ........ | <-- rbp
+ +-----------+ low addresses
+
+ So, the trampoline frame starts at rbp+16, and the return address,
+ is at rbp+24. The vmprof API requires us to return the offset of
+ the frame relative to sp, hence we have this weird computation.
+
+ XXX (antocuni): I think we could change the API to return directly
+ the frame address instead of the offset; however, this require a
+ change in the PyPy code too
+ */
+
+ unw_get_reg (cp, UNW_REG_SP, (unw_word_t*)&sp);
+ unw_get_reg (cp, UNW_X86_64_RBP, (unw_word_t*)&bp);
+ return bp+16+8-sp;
+ }
+ return -1;
+
#else
+
return -1;
+
#endif
}
diff --git a/rpython/rlib/rvmprof/src/rvmprof_getpc.h b/rpython/rlib/rvmprof/src/vmprof_getpc.h
rename from rpython/rlib/rvmprof/src/rvmprof_getpc.h
rename to rpython/rlib/rvmprof/src/vmprof_getpc.h
--- a/rpython/rlib/rvmprof/src/rvmprof_getpc.h
+++ b/rpython/rlib/rvmprof/src/vmprof_getpc.h
@@ -44,7 +44,7 @@
#ifndef BASE_GETPC_H_
#define BASE_GETPC_H_
-#include "rvmprof_config.h"
+#include "vmprof_config.h"
// On many linux systems, we may need _GNU_SOURCE to get access to
// the defined constants that define the register we want to see (eg
@@ -53,7 +53,9 @@
// If #define _GNU_SOURCE causes problems, this might work instead.
// It will cause problems for FreeBSD though!, because it turns off
// the needed __BSD_VISIBLE.
-//#define _XOPEN_SOURCE 500
+#ifdef __APPLE__
+#define _XOPEN_SOURCE 500
+#endif
#include <string.h> // for memcmp
#if defined(HAVE_SYS_UCONTEXT_H)
@@ -179,7 +181,11 @@
// configure.ac (or set it manually in your config.h).
#else
void* GetPC(ucontext_t *signal_ucontext) {
+#ifdef __APPLE__
+ return (void*)(signal_ucontext->uc_mcontext->__ss.__rip);
+#else
return (void*)signal_ucontext->PC_FROM_UCONTEXT; // defined in config.h
+#endif
}
#endif
diff --git a/rpython/rlib/rvmprof/src/rvmprof.c b/rpython/rlib/rvmprof/src/vmprof_main.h
copy from rpython/rlib/rvmprof/src/rvmprof.c
copy to rpython/rlib/rvmprof/src/vmprof_main.h
--- a/rpython/rlib/rvmprof/src/rvmprof.c
+++ b/rpython/rlib/rvmprof/src/vmprof_main.h
@@ -19,26 +19,6 @@
#define _GNU_SOURCE 1
-
-#ifdef RPYTHON_LL2CTYPES
- /* only for testing: ll2ctypes sets RPY_EXTERN from the command-line */
-# ifndef RPY_EXTERN
-# define RPY_EXTERN RPY_EXPORTED
-# endif
-# define RPY_EXPORTED extern __attribute__((visibility("default")))
-# define VMPROF_ADDR_OF_TRAMPOLINE(addr) 0
-
-#else
-
-# include "common_header.h"
-# include "rvmprof.h"
-# ifndef VMPROF_ADDR_OF_TRAMPOLINE
-# error "RPython program using rvmprof, but not calling vmprof_execute_code()"
-# endif
-
-#endif
-
-
#include <dlfcn.h>
#include <assert.h>
#include <pthread.h>
@@ -47,11 +27,12 @@
#include <unistd.h>
#include <stdio.h>
#include <sys/types.h>
+#include <signal.h>
#include <sys/stat.h>
#include <fcntl.h>
-#include "rvmprof_getpc.h"
-#include "rvmprof_unwind.h"
-#include "rvmprof_mt.h"
+#include "vmprof_getpc.h"
+#include "vmprof_unwind.h"
+#include "vmprof_mt.h"
/************************************************************/
@@ -64,23 +45,40 @@
static int (*unw_get_proc_info)(unw_cursor_t *, unw_proc_info_t *) = NULL;
static int profile_file = -1;
+static long prepare_interval_usec;
+static struct profbuf_s *volatile current_codes;
+static void *(*mainloop_get_virtual_ip)(char *) = 0;
+static int opened_profile(char *interp_name);
+static void flush_codes(void);
+
+#ifdef __APPLE__
+#define UNWIND_NAME "/usr/lib/system/libunwind.dylib"
+#define UNW_PREFIX "unw"
+#else
+#define UNWIND_NAME "libunwind.so"
+#define UNW_PREFIX "_ULx86_64"
+#endif
RPY_EXTERN
-char *rpython_vmprof_init(int fd)
+char *vmprof_init(int fd, double interval, char *interp_name)
{
+ if (interval < 1e-6 || interval >= 1.0)
+ return "bad value for 'interval'";
+ prepare_interval_usec = (int)(interval * 1000000.0);
+
if (!unw_get_reg) {
void *libhandle;
- if (!(libhandle = dlopen("libunwind.so", RTLD_LAZY | RTLD_LOCAL)))
+ if (!(libhandle = dlopen(UNWIND_NAME, RTLD_LAZY | RTLD_LOCAL)))
goto error;
- if (!(unw_get_reg = dlsym(libhandle, "_ULx86_64_get_reg")))
+ if (!(unw_get_reg = dlsym(libhandle, UNW_PREFIX "_get_reg")))
goto error;
- if (!(unw_get_proc_info = dlsym(libhandle, "_ULx86_64_get_proc_info")))
+ if (!(unw_get_proc_info = dlsym(libhandle, UNW_PREFIX "_get_proc_info")))
goto error;
- if (!(unw_init_local = dlsym(libhandle, "_ULx86_64_init_local")))
+ if (!(unw_init_local = dlsym(libhandle, UNW_PREFIX "_init_local")))
goto error;
- if (!(unw_step = dlsym(libhandle, "_ULx86_64_step")))
+ if (!(unw_step = dlsym(libhandle, UNW_PREFIX "_step")))
goto error;
}
if (prepare_concurrent_bufs() < 0)
@@ -88,6 +86,10 @@
assert(fd >= 0);
profile_file = fd;
+ if (opened_profile(interp_name) < 0) {
+ profile_file = -1;
+ return strerror(errno);
+ }
return NULL;
error:
@@ -101,7 +103,7 @@
static long volatile signal_handler_value = 1;
RPY_EXTERN
-void rpython_vmprof_ignore_signals(int ignored)
+void vmprof_ignore_signals(int ignored)
{
if (!ignored) {
__sync_fetch_and_and(&signal_handler_value, ~1L);
@@ -128,6 +130,11 @@
#define MARKER_STACKTRACE '\x01'
#define MARKER_VIRTUAL_IP '\x02'
#define MARKER_TRAILER '\x03'
+#define MARKER_INTERP_NAME '\x04' /* deprecated */
+#define MARKER_HEADER '\x05'
+
+#define VERSION_BASE '\x00'
+#define VERSION_THREAD_ID '\x01'
struct prof_stacktrace_s {
char padding[sizeof(long) - 1];
@@ -145,7 +152,7 @@
* ******************************************************
*/
-#include "rvmprof_get_custom_offset.h"
+#include "vmprof_get_custom_offset.h"
typedef struct {
void* _unused1;
@@ -220,12 +227,10 @@
// found main loop stack frame
void* sp;
unw_get_reg(&cursor, UNW_REG_SP, (unw_word_t *) &sp);
- void *arg_addr = (char*)sp /* + mainloop_sp_offset */;
- void **arg_ptr = (void**)arg_addr;
- /* if (mainloop_get_virtual_ip) {
- ip = mainloop_get_virtual_ip(*arg_ptr);
- } else { */
- ip = *arg_ptr;
+ if (mainloop_get_virtual_ip)
+ ip = mainloop_get_virtual_ip((char *)sp);
+ else
+ ip = *(void **)sp;
}
int first_run = (n == 0);
@@ -237,6 +242,23 @@
return n;
}
+static void *get_current_thread_id(void)
+{
+ /* xxx This function is a hack on two fronts:
+
+ - It assumes that pthread_self() is async-signal-safe. This
+ should be true on Linux. I hope it is also true elsewhere.
+
+ - It abuses pthread_self() by assuming it just returns an
+ integer. According to comments in CPython's source code, the
+ platforms where it is not the case are rare nowadays.
+
+ An alternative would be to try to look if the information is
+ available in the ucontext_t in the caller.
+ */
+ return (void *)pthread_self();
+}
+
/* *************************************************************
* the signal handler
@@ -262,9 +284,10 @@
st->marker = MARKER_STACKTRACE;
st->count = 1;
st->stack[0] = GetPC((ucontext_t*)ucontext);
- depth = get_stack_trace(st->stack+1, MAX_STACK_DEPTH-1, ucontext);
+ depth = get_stack_trace(st->stack+1, MAX_STACK_DEPTH-2, ucontext);
depth++; // To account for pc value in stack[0];
st->depth = depth;
+ st->stack[depth++] = get_current_thread_id();
p->data_offset = offsetof(struct prof_stacktrace_s, marker);
p->data_size = (depth * sizeof(void *) +
sizeof(struct prof_stacktrace_s) -
@@ -358,11 +381,11 @@
}
RPY_EXTERN
-int rpython_vmprof_enable(long interval_usec)
+int vmprof_enable(void)
{
assert(profile_file >= 0);
- assert(interval_usec > 0);
- profile_interval_usec = interval_usec;
+ assert(prepare_interval_usec > 0);
+ profile_interval_usec = prepare_interval_usec;
if (install_pthread_atfork_hooks() == -1)
goto error;
@@ -370,7 +393,7 @@
goto error;
if (install_sigprof_timer() == -1)
goto error;
- rpython_vmprof_ignore_signals(0);
+ vmprof_ignore_signals(0);
return 0;
error:
@@ -391,6 +414,29 @@
return 0;
}
+static int opened_profile(char *interp_name)
+{
+ struct {
+ long hdr[5];
+ char interp_name[259];
+ } header;
+
+ size_t namelen = strnlen(interp_name, 255);
+ current_codes = NULL;
+
+ header.hdr[0] = 0;
+ header.hdr[1] = 3;
+ header.hdr[2] = 0;
+ header.hdr[3] = prepare_interval_usec;
+ header.hdr[4] = 0;
+ header.interp_name[0] = MARKER_HEADER;
+ header.interp_name[1] = '\x00';
+ header.interp_name[2] = VERSION_THREAD_ID;
+ header.interp_name[3] = namelen;
+ memcpy(&header.interp_name[4], interp_name, namelen);
+ return _write_all(&header, 5 * sizeof(long) + 4 + namelen);
+}
+
static int close_profile(void)
{
char buf[4096];
@@ -415,12 +461,16 @@
close(srcfd);
#else
// freebsd and mac
+#if defined(__APPLE__)
+ sprintf(buf, "vmmap %d", getpid());
+#else
sprintf(buf, "procstat -v %d", getpid());
+#endif
FILE *srcf = popen(buf, "r");
if (!srcf)
return -1;
- while ((size = fread(buf, 1, sizeof buf, src))) {
+ while ((size = fread(buf, 1, sizeof buf, srcf))) {
if (_write_all(buf, size) < 0) {
pclose(srcf);
return -1;
@@ -435,35 +485,85 @@
}
RPY_EXTERN
-int rpython_vmprof_disable(void)
+int vmprof_disable(void)
{
- rpython_vmprof_ignore_signals(1);
+ vmprof_ignore_signals(1);
profile_interval_usec = 0;
if (remove_sigprof_timer() == -1)
return -1;
if (remove_sigprof_handler() == -1)
return -1;
+ flush_codes();
if (shutdown_concurrent_bufs(profile_file) < 0)
return -1;
return close_profile();
}
RPY_EXTERN
-void rpython_vmprof_write_buf(char *buf, long size)
+int vmprof_register_virtual_function(char *code_name, long code_uid,
+ int auto_retry)
{
+ long namelen = strnlen(code_name, 1023);
+ long blocklen = 1 + 2 * sizeof(long) + namelen;
struct profbuf_s *p;
+ char *t;
- while ((p = reserve_buffer(profile_file)) == NULL) {
- /* spin loop waiting for a buffer to be ready; should almost never
- be the case */
- usleep(1);
+ retry:
+ p = current_codes;
+ if (p != NULL) {
+ if (__sync_bool_compare_and_swap(¤t_codes, p, NULL)) {
+ /* grabbed 'current_codes': we will append the current block
+ to it if it contains enough room */
+ size_t freesize = SINGLE_BUF_SIZE - p->data_size;
+ if (freesize < blocklen) {
+ /* full: flush it */
+ commit_buffer(profile_file, p);
+ p = NULL;
+ }
+ }
+ else {
+ /* compare-and-swap failed, don't try again */
+ p = NULL;
+ }
}
- if (size > SINGLE_BUF_SIZE)
- size = SINGLE_BUF_SIZE;
- memcpy(p->data, buf, size);
- p->data_size = size;
+ if (p == NULL) {
+ p = reserve_buffer(profile_file);
+ if (p == NULL) {
+ /* can't get a free block; should almost never be the
+ case. Spin loop if allowed, or return a failure code
+ if not (e.g. we're in a signal handler) */
+ if (auto_retry > 0) {
+ auto_retry--;
+ usleep(1);
+ goto retry;
+ }
+ return -1;
+ }
+ }
- commit_buffer(profile_file, p);
+ t = p->data + p->data_size;
+ p->data_size += blocklen;
+ assert(p->data_size <= SINGLE_BUF_SIZE);
+ *t++ = MARKER_VIRTUAL_IP;
+ memcpy(t, &code_uid, sizeof(long)); t += sizeof(long);
+ memcpy(t, &namelen, sizeof(long)); t += sizeof(long);
+ memcpy(t, code_name, namelen);
+
+ /* try to reattach 'p' to 'current_codes' */
+ if (!__sync_bool_compare_and_swap(¤t_codes, NULL, p)) {
+ /* failed, flush it */
+ commit_buffer(profile_file, p);
+ }
+ return 0;
}
+
+static void flush_codes(void)
+{
+ struct profbuf_s *p = current_codes;
+ if (p != NULL) {
+ current_codes = NULL;
+ commit_buffer(profile_file, p);
+ }
+}
diff --git a/rpython/rlib/rvmprof/src/rvmprof_mt.h b/rpython/rlib/rvmprof/src/vmprof_mt.h
rename from rpython/rlib/rvmprof/src/rvmprof_mt.h
rename to rpython/rlib/rvmprof/src/vmprof_mt.h
--- a/rpython/rlib/rvmprof/src/rvmprof_mt.h
+++ b/rpython/rlib/rvmprof/src/vmprof_mt.h
@@ -66,7 +66,7 @@
unprepare_concurrent_bufs();
profbuf_all_buffers = mmap(NULL, sizeof(struct profbuf_s) * MAX_NUM_BUFFERS,
PROT_READ | PROT_WRITE,
- MAP_PRIVATE | MAP_ANONYMOUS,
+ MAP_PRIVATE | MAP_ANON,
-1, 0);
if (profbuf_all_buffers == MAP_FAILED) {
profbuf_all_buffers = NULL;
@@ -190,10 +190,17 @@
}
}
+static void cancel_buffer(struct profbuf_s *buf)
+{
+ long i = buf - profbuf_all_buffers;
+ assert(profbuf_state[i] == PROFBUF_FILLING);
+ profbuf_state[i] = PROFBUF_UNUSED;
+}
+
static int shutdown_concurrent_bufs(int fd)
{
/* no signal handler can be running concurrently here, because we
- already did rpython_vmprof_ignore_signals(1) */
+ already did vmprof_ignore_signals(1) */
assert(profbuf_write_lock == 0);
profbuf_write_lock = 2;
diff --git a/rpython/rlib/rvmprof/src/rvmprof_unwind.h b/rpython/rlib/rvmprof/src/vmprof_unwind.h
rename from rpython/rlib/rvmprof/src/rvmprof_unwind.h
rename to rpython/rlib/rvmprof/src/vmprof_unwind.h
diff --git a/rpython/rlib/rvmprof/test/test_ztranslation.py b/rpython/rlib/rvmprof/test/test_ztranslation.py
--- a/rpython/rlib/rvmprof/test/test_ztranslation.py
+++ b/rpython/rlib/rvmprof/test/test_ztranslation.py
@@ -1,4 +1,7 @@
-import time, os, sys, py
+import time, os, sys
+if __name__ == '__main__':
+ sys.path += ['../../../..'] # for subprocess in test_interpreted
+import py
from rpython.tool.udir import udir
from rpython.rlib import rvmprof
from rpython.translator.c.test.test_genc import compile
@@ -57,7 +60,8 @@
def test_interpreted():
# takes forever if the Python process is already big...
import subprocess
- subprocess.check_call([sys.executable, __file__])
+ subprocess.check_call([sys.executable, os.path.basename(__file__)],
+ cwd=(os.path.dirname(__file__) or '.'))
def test_compiled():
fn = compile(main, [], gcpolicy="minimark")
diff --git a/rpython/rtyper/rclass.py b/rpython/rtyper/rclass.py
--- a/rpython/rtyper/rclass.py
+++ b/rpython/rtyper/rclass.py
@@ -616,26 +616,33 @@
while rbase.classdef is not None:
immutable_fields.update(rbase.immutable_field_set)
rbase = rbase.rbase
- self._parse_field_list(immutable_fields, accessor)
+ self._parse_field_list(immutable_fields, accessor, hints)
- def _parse_field_list(self, fields, accessor):
+ def _parse_field_list(self, fields, accessor, hints):
ranking = {}
for name in fields:
+ quasi = False
if name.endswith('?[*]'): # a quasi-immutable field pointing to
name = name[:-4] # an immutable array
rank = IR_QUASIIMMUTABLE_ARRAY
+ quasi = True
elif name.endswith('[*]'): # for virtualizables' lists
name = name[:-3]
rank = IR_IMMUTABLE_ARRAY
elif name.endswith('?'): # a quasi-immutable field
name = name[:-1]
rank = IR_QUASIIMMUTABLE
+ quasi = True
else: # a regular immutable/green field
rank = IR_IMMUTABLE
try:
mangled_name, r = self._get_field(name)
except KeyError:
continue
+ if quasi and hints.get("immutable"):
+ raise TyperError(
+ "can't have _immutable_ = True and a quasi-immutable field "
+ "%s in class %s" % (name, self.classdef))
ranking[mangled_name] = rank
accessor.initialize(self.object_type, ranking)
return ranking
diff --git a/rpython/rtyper/rvirtualizable.py b/rpython/rtyper/rvirtualizable.py
--- a/rpython/rtyper/rvirtualizable.py
+++ b/rpython/rtyper/rvirtualizable.py
@@ -38,8 +38,8 @@
else:
self._super()._setup_repr(hints = hints)
c_vfields = self.classdef.classdesc.classdict['_virtualizable_']
- self.my_redirected_fields = self._parse_field_list(c_vfields.value,
- self.accessor)
+ self.my_redirected_fields = self._parse_field_list(
+ c_vfields.value, self.accessor, hints)
else:
self._super()._setup_repr()
# ootype needs my_redirected_fields even for subclass. lltype does
diff --git a/rpython/rtyper/test/test_rclass.py b/rpython/rtyper/test/test_rclass.py
--- a/rpython/rtyper/test/test_rclass.py
+++ b/rpython/rtyper/test/test_rclass.py
@@ -943,6 +943,19 @@
found.append(op.args[1].value)
assert found == ['mutate_a', 'mutate_a', 'mutate_b']
+ def test_quasi_immutable_clashes_with_immutable(self):
+ from rpython.jit.metainterp.typesystem import deref
+ class A(object):
+ _immutable_ = True
+ _immutable_fields_ = ['a?']
+ def f():
+ a = A()
+ a.x = 42
+ a.a = 142
+ return A()
+ with py.test.raises(TyperError):
+ self.gengraph(f, [])
+
def test_quasi_immutable_array(self):
from rpython.jit.metainterp.typesystem import deref
class A(object):
diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py
--- a/rpython/translator/platform/posix.py
+++ b/rpython/translator/platform/posix.py
@@ -181,6 +181,7 @@
('all', '$(DEFAULT_TARGET)', []),
('$(TARGET)', '$(OBJECTS)', '$(CC_LINK) $(LDFLAGSEXTRA) -o $@ $(OBJECTS) $(LIBDIRS) $(LIBS) $(LINKFILES) $(LDFLAGS)'),
('%.o', '%.c', '$(CC) $(CFLAGS) $(CFLAGSEXTRA) -o $@ -c $< $(INCLUDEDIRS)'),
+ ('%.o', '%.s', '$(CC) $(CFLAGS) $(CFLAGSEXTRA) -o $@ -c $< $(INCLUDEDIRS)'),
('%.o', '%.cxx', '$(CXX) $(CFLAGS) $(CFLAGSEXTRA) -o $@ -c $< $(INCLUDEDIRS)'),
]
More information about the pypy-commit
mailing list