From pypy.commits at gmail.com Sat Oct 1 04:16:38 2016 From: pypy.commits at gmail.com (arigo) Date: Sat, 01 Oct 2016 01:16:38 -0700 (PDT) Subject: [pypy-commit] pypy default: sys.path issues (thanks matti) Message-ID: <57ef70e6.e2cdc20a.2c914.f542@mx.google.com> Author: Armin Rigo Branch: Changeset: r87482:6642fe40f5f4 Date: 2016-10-01 10:16 +0200 http://bitbucket.org/pypy/pypy/changeset/6642fe40f5f4/ Log: sys.path issues (thanks matti) diff --git a/pypy/tool/build_cffi_imports.py b/pypy/tool/build_cffi_imports.py --- a/pypy/tool/build_cffi_imports.py +++ b/pypy/tool/build_cffi_imports.py @@ -1,5 +1,4 @@ -import sys, shutil -from rpython.tool.runsubprocess import run_subprocess +import sys, shutil, os class MissingDependenciesError(Exception): pass @@ -18,6 +17,8 @@ } def create_cffi_import_libraries(pypy_c, options, basedir): + from rpython.tool.runsubprocess import run_subprocess + shutil.rmtree(str(basedir.join('lib_pypy', '__pycache__')), ignore_errors=True) failures = [] @@ -42,11 +43,16 @@ return failures if __name__ == '__main__': - import py, os if '__pypy__' not in sys.builtin_module_names: print >> sys.stderr, 'Call with a pypy interpreter' sys.exit(1) + tool_dir = os.path.dirname(os.path.abspath(sys.argv[0])) + base_dir = os.path.dirname(os.path.dirname(tool_dir)) + sys.path.insert(0, base_dir) + + import py + class Options(object): pass From pypy.commits at gmail.com Sat Oct 1 04:37:05 2016 From: pypy.commits at gmail.com (arigo) Date: Sat, 01 Oct 2016 01:37:05 -0700 (PDT) Subject: [pypy-commit] pypy default: Move clock_{gettime, getres, settime} entirely to rpython/rlib/rtime, with tests Message-ID: <57ef75b1.862f1c0a.6f95c.6c86@mx.google.com> Author: Armin Rigo Branch: Changeset: r87483:fbace1f687b0 Date: 2016-10-01 10:25 +0200 http://bitbucket.org/pypy/pypy/changeset/fbace1f687b0/ Log: Move clock_{gettime,getres,settime} entirely to rpython/rlib/rtime, with tests diff --git a/rpython/rlib/rtime.py b/rpython/rlib/rtime.py --- a/rpython/rlib/rtime.py +++ b/rpython/rlib/rtime.py @@ -67,10 +67,17 @@ includes=['time.h'], libraries=libraries ) + _NO_MISSING_RT = rffi_platform.Has('printf("%d", clock_gettime(0, 0))') TIMESPEC = rffi_platform.Struct('struct timespec', [('tv_sec', rffi.LONG), ('tv_nsec', rffi.LONG)]) -constant_names = ['RUSAGE_SELF', 'EINTR', 'CLOCK_PROCESS_CPUTIME_ID'] +constant_names = ['RUSAGE_SELF', 'EINTR', + 'CLOCK_REALTIME', + 'CLOCK_MONOTONIC', + 'CLOCK_MONOTONIC_RAW', + 'CLOCK_PROCESS_CPUTIME_ID', + 'CLOCK_THREAD_CPUTIME_ID', +] for const in constant_names: setattr(CConfig, const, rffi_platform.DefinedConstantInteger(const)) defs_names = ['GETTIMEOFDAY_NO_TZ'] @@ -158,16 +165,33 @@ divisor = 0.0 counter_start = 0 state = State() -elif CLOCK_PROCESS_CPUTIME_ID is not None: + +HAS_CLOCK_GETTIME = (CLOCK_MONOTONIC is not None) +if HAS_CLOCK_GETTIME: # Linux and other POSIX systems with clock_gettime() + # TIMESPEC: globals().update(rffi_platform.configure(CConfigForClockGetTime)) - TIMESPEC = TIMESPEC - CLOCK_PROCESS_CPUTIME_ID = CLOCK_PROCESS_CPUTIME_ID - eci_with_lrt = eci.merge(ExternalCompilationInfo(libraries=['rt'])) + # do we need to add -lrt? + eciclock = CConfigForClockGetTime._compilation_info_ + if not _NO_MISSING_RT: + eciclock = eciclock.merge(ExternalCompilationInfo(libraries=['rt'])) + # the functions: + c_clock_getres = external("clock_getres", + [lltype.Signed, lltype.Ptr(TIMESPEC)], + rffi.INT, releasegil=False, + save_err=rffi.RFFI_SAVE_ERRNO, + compilation_info=eciclock) c_clock_gettime = external('clock_gettime', [lltype.Signed, lltype.Ptr(TIMESPEC)], rffi.INT, releasegil=False, - compilation_info=eci_with_lrt) + save_err=rffi.RFFI_SAVE_ERRNO, + compilation_info=eciclock) + c_clock_settime = external('clock_settime', + [lltype.Signed, lltype.Ptr(TIMESPEC)], + rffi.INT, releasegil=False, + save_err=rffi.RFFI_SAVE_ERRNO, + compilation_info=eciclock) + if need_rusage: RUSAGE = RUSAGE RUSAGE_SELF = RUSAGE_SELF or 0 @@ -191,18 +215,16 @@ def clock(): if _WIN32: return win_perf_counter() - elif CLOCK_PROCESS_CPUTIME_ID is not None: + elif HAS_CLOCK_GETTIME and CLOCK_PROCESS_CPUTIME_ID is not None: with lltype.scoped_alloc(TIMESPEC) as a: - c_clock_gettime(CLOCK_PROCESS_CPUTIME_ID, a) - result = (float(rffi.getintfield(a, 'c_tv_sec')) + - float(rffi.getintfield(a, 'c_tv_nsec')) * 0.000000001) - return result - else: - with lltype.scoped_alloc(RUSAGE) as a: - c_getrusage(RUSAGE_SELF, a) - result = (decode_timeval(a.c_ru_utime) + - decode_timeval(a.c_ru_stime)) - return result + if c_clock_gettime(CLOCK_PROCESS_CPUTIME_ID, a) == 0: + return (float(rffi.getintfield(a, 'c_tv_sec')) + + float(rffi.getintfield(a, 'c_tv_nsec')) * 0.000000001) + with lltype.scoped_alloc(RUSAGE) as a: + c_getrusage(RUSAGE_SELF, a) + result = (decode_timeval(a.c_ru_utime) + + decode_timeval(a.c_ru_stime)) + return result # _______________________________________________________________ # time.sleep() diff --git a/rpython/rlib/test/test_rtime.py b/rpython/rlib/test/test_rtime.py --- a/rpython/rlib/test/test_rtime.py +++ b/rpython/rlib/test/test_rtime.py @@ -1,7 +1,9 @@ from rpython.rtyper.test.tool import BaseRtypingTest +from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.rlib import rtime -import time, sys +import time, sys, py class TestTime(BaseRtypingTest): def test_time_time(self): @@ -53,3 +55,27 @@ t1 = time.time() assert t0 <= t1 assert t1 - t0 >= 0.15 + + def test_clock_gettime(self): + if not rtime.HAS_CLOCK_GETTIME: + py.test.skip("no clock_gettime()") + lst = [] + for i in range(50): + with lltype.scoped_alloc(rtime.TIMESPEC) as a1: + res = rtime.c_clock_gettime(rtime.CLOCK_MONOTONIC, a1) + assert res == 0 + t = (float(rffi.getintfield(a1, 'c_tv_sec')) + + float(rffi.getintfield(a1, 'c_tv_nsec')) * 0.000000001) + lst.append(t) + assert lst == sorted(lst) + + def test_clock_getres(self): + if not rtime.HAS_CLOCK_GETTIME: + py.test.skip("no clock_gettime()") + lst = [] + with lltype.scoped_alloc(rtime.TIMESPEC) as a1: + res = rtime.c_clock_getres(rtime.CLOCK_MONOTONIC, a1) + assert res == 0 + t = (float(rffi.getintfield(a1, 'c_tv_sec')) + + float(rffi.getintfield(a1, 'c_tv_nsec')) * 0.000000001) + assert 0.0 < t <= 1.0 From pypy.commits at gmail.com Sat Oct 1 04:37:09 2016 From: pypy.commits at gmail.com (arigo) Date: Sat, 01 Oct 2016 01:37:09 -0700 (PDT) Subject: [pypy-commit] pypy default: Use rtime.c_clock_gettime() here Message-ID: <57ef75b5.0aa71c0a.c61d8.aa3d@mx.google.com> Author: Armin Rigo Branch: Changeset: r87484:d5e85ca41a7f Date: 2016-10-01 10:36 +0200 http://bitbucket.org/pypy/pypy/changeset/d5e85ca41a7f/ Log: Use rtime.c_clock_gettime() here diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py --- a/pypy/module/__pypy__/__init__.py +++ b/pypy/module/__pypy__/__init__.py @@ -2,6 +2,8 @@ from pypy.interpreter.mixedmodule import MixedModule from pypy.module.imp.importing import get_pyc_magic +from rpython.rlib import rtime + class BuildersModule(MixedModule): appleveldefs = {} @@ -14,16 +16,15 @@ class TimeModule(MixedModule): appleveldefs = {} interpleveldefs = {} - if sys.platform.startswith("linux") or 'bsd' in sys.platform: - from pypy.module.__pypy__ import interp_time + if rtime.HAS_CLOCK_GETTIME: interpleveldefs["clock_gettime"] = "interp_time.clock_gettime" interpleveldefs["clock_getres"] = "interp_time.clock_getres" for name in [ "CLOCK_REALTIME", "CLOCK_MONOTONIC", "CLOCK_MONOTONIC_RAW", "CLOCK_PROCESS_CPUTIME_ID", "CLOCK_THREAD_CPUTIME_ID" ]: - if getattr(interp_time, name) is not None: - interpleveldefs[name] = "space.wrap(interp_time.%s)" % name + if getattr(rtime, name) is not None: + interpleveldefs[name] = "space.wrap(%d)" % getattr(rtime, name) class ThreadModule(MixedModule): diff --git a/pypy/module/__pypy__/interp_time.py b/pypy/module/__pypy__/interp_time.py --- a/pypy/module/__pypy__/interp_time.py +++ b/pypy/module/__pypy__/interp_time.py @@ -4,71 +4,28 @@ from pypy.interpreter.error import exception_from_saved_errno from pypy.interpreter.gateway import unwrap_spec from rpython.rtyper.lltypesystem import rffi, lltype -from rpython.rtyper.tool import rffi_platform -from rpython.translator.tool.cbuild import ExternalCompilationInfo +from rpython.rlib import rtime +from rpython.rlib.rtime import HAS_CLOCK_GETTIME -if sys.platform == 'linux2': - libraries = ["rt"] -else: - libraries = [] - - -class CConfig: - _compilation_info_ = ExternalCompilationInfo( - includes=["time.h"], - libraries=libraries, - ) - - HAS_CLOCK_GETTIME = rffi_platform.Has('clock_gettime') - - CLOCK_REALTIME = rffi_platform.DefinedConstantInteger("CLOCK_REALTIME") - CLOCK_MONOTONIC = rffi_platform.DefinedConstantInteger("CLOCK_MONOTONIC") - CLOCK_MONOTONIC_RAW = rffi_platform.DefinedConstantInteger("CLOCK_MONOTONIC_RAW") - CLOCK_PROCESS_CPUTIME_ID = rffi_platform.DefinedConstantInteger("CLOCK_PROCESS_CPUTIME_ID") - CLOCK_THREAD_CPUTIME_ID = rffi_platform.DefinedConstantInteger("CLOCK_THREAD_CPUTIME_ID") - -cconfig = rffi_platform.configure(CConfig) - -HAS_CLOCK_GETTIME = cconfig["HAS_CLOCK_GETTIME"] - -CLOCK_REALTIME = cconfig["CLOCK_REALTIME"] -CLOCK_MONOTONIC = cconfig["CLOCK_MONOTONIC"] -CLOCK_MONOTONIC_RAW = cconfig["CLOCK_MONOTONIC_RAW"] -CLOCK_PROCESS_CPUTIME_ID = cconfig["CLOCK_PROCESS_CPUTIME_ID"] -CLOCK_THREAD_CPUTIME_ID = cconfig["CLOCK_THREAD_CPUTIME_ID"] if HAS_CLOCK_GETTIME: - #redo it for timespec - CConfig.TIMESPEC = rffi_platform.Struct("struct timespec", [ - ("tv_sec", rffi.TIME_T), - ("tv_nsec", rffi.LONG), - ]) - cconfig = rffi_platform.configure(CConfig) - TIMESPEC = cconfig['TIMESPEC'] - - c_clock_gettime = rffi.llexternal("clock_gettime", - [lltype.Signed, lltype.Ptr(TIMESPEC)], rffi.INT, - compilation_info=CConfig._compilation_info_, releasegil=False, - save_err=rffi.RFFI_SAVE_ERRNO - ) - c_clock_getres = rffi.llexternal("clock_getres", - [lltype.Signed, lltype.Ptr(TIMESPEC)], rffi.INT, - compilation_info=CConfig._compilation_info_, releasegil=False, - save_err=rffi.RFFI_SAVE_ERRNO - ) @unwrap_spec(clk_id="c_int") def clock_gettime(space, clk_id): - with lltype.scoped_alloc(TIMESPEC) as tp: - ret = c_clock_gettime(clk_id, tp) + with lltype.scoped_alloc(rtime.TIMESPEC) as tp: + ret = rtime.c_clock_gettime(clk_id, tp) if ret != 0: raise exception_from_saved_errno(space, space.w_IOError) - return space.wrap(int(tp.c_tv_sec) + 1e-9 * int(tp.c_tv_nsec)) + t = (float(rffi.getintfield(tp, 'c_tv_sec')) + + float(rffi.getintfield(tp, 'c_tv_nsec')) * 0.000000001) + return space.wrap(t) @unwrap_spec(clk_id="c_int") def clock_getres(space, clk_id): - with lltype.scoped_alloc(TIMESPEC) as tp: - ret = c_clock_getres(clk_id, tp) + with lltype.scoped_alloc(rtime.TIMESPEC) as tp: + ret = rtime.c_clock_getres(clk_id, tp) if ret != 0: raise exception_from_saved_errno(space, space.w_IOError) - return space.wrap(int(tp.c_tv_sec) + 1e-9 * int(tp.c_tv_nsec)) + t = (float(rffi.getintfield(tp, 'c_tv_sec')) + + float(rffi.getintfield(tp, 'c_tv_nsec')) * 0.000000001) + return space.wrap(t) diff --git a/rpython/rlib/rtime.py b/rpython/rlib/rtime.py --- a/rpython/rlib/rtime.py +++ b/rpython/rlib/rtime.py @@ -191,6 +191,10 @@ rffi.INT, releasegil=False, save_err=rffi.RFFI_SAVE_ERRNO, compilation_info=eciclock) + # Note: there is no higher-level functions here to access + # clock_gettime(). The issue is that we'd need a way that keeps + # nanosecond precision, depending on the usage, so we can't have a + # nice function that returns the time as a float. if need_rusage: RUSAGE = RUSAGE From pypy.commits at gmail.com Sat Oct 1 04:38:27 2016 From: pypy.commits at gmail.com (arigo) Date: Sat, 01 Oct 2016 01:38:27 -0700 (PDT) Subject: [pypy-commit] pypy py3k: hg merge default Message-ID: <57ef7603.c4f6c20a.96550.f891@mx.google.com> Author: Armin Rigo Branch: py3k Changeset: r87485:863ec7aea4f4 Date: 2016-10-01 10:37 +0200 http://bitbucket.org/pypy/pypy/changeset/863ec7aea4f4/ Log: hg merge default diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py --- a/pypy/module/__pypy__/__init__.py +++ b/pypy/module/__pypy__/__init__.py @@ -2,6 +2,8 @@ from pypy.interpreter.mixedmodule import MixedModule from pypy.module.imp.importing import get_pyc_magic +from rpython.rlib import rtime + class BuildersModule(MixedModule): appleveldefs = {} @@ -14,16 +16,15 @@ class TimeModule(MixedModule): appleveldefs = {} interpleveldefs = {} - if sys.platform.startswith("linux") or 'bsd' in sys.platform: - from pypy.module.__pypy__ import interp_time + if rtime.HAS_CLOCK_GETTIME: interpleveldefs["clock_gettime"] = "interp_time.clock_gettime" interpleveldefs["clock_getres"] = "interp_time.clock_getres" for name in [ "CLOCK_REALTIME", "CLOCK_MONOTONIC", "CLOCK_MONOTONIC_RAW", "CLOCK_PROCESS_CPUTIME_ID", "CLOCK_THREAD_CPUTIME_ID" ]: - if getattr(interp_time, name) is not None: - interpleveldefs[name] = "space.wrap(interp_time.%s)" % name + if getattr(rtime, name) is not None: + interpleveldefs[name] = "space.wrap(%d)" % getattr(rtime, name) class ThreadModule(MixedModule): diff --git a/pypy/module/__pypy__/interp_time.py b/pypy/module/__pypy__/interp_time.py --- a/pypy/module/__pypy__/interp_time.py +++ b/pypy/module/__pypy__/interp_time.py @@ -4,71 +4,28 @@ from pypy.interpreter.error import exception_from_saved_errno from pypy.interpreter.gateway import unwrap_spec from rpython.rtyper.lltypesystem import rffi, lltype -from rpython.rtyper.tool import rffi_platform -from rpython.translator.tool.cbuild import ExternalCompilationInfo +from rpython.rlib import rtime +from rpython.rlib.rtime import HAS_CLOCK_GETTIME -if sys.platform == 'linux2': - libraries = ["rt"] -else: - libraries = [] - - -class CConfig: - _compilation_info_ = ExternalCompilationInfo( - includes=["time.h"], - libraries=libraries, - ) - - HAS_CLOCK_GETTIME = rffi_platform.Has('clock_gettime') - - CLOCK_REALTIME = rffi_platform.DefinedConstantInteger("CLOCK_REALTIME") - CLOCK_MONOTONIC = rffi_platform.DefinedConstantInteger("CLOCK_MONOTONIC") - CLOCK_MONOTONIC_RAW = rffi_platform.DefinedConstantInteger("CLOCK_MONOTONIC_RAW") - CLOCK_PROCESS_CPUTIME_ID = rffi_platform.DefinedConstantInteger("CLOCK_PROCESS_CPUTIME_ID") - CLOCK_THREAD_CPUTIME_ID = rffi_platform.DefinedConstantInteger("CLOCK_THREAD_CPUTIME_ID") - -cconfig = rffi_platform.configure(CConfig) - -HAS_CLOCK_GETTIME = cconfig["HAS_CLOCK_GETTIME"] - -CLOCK_REALTIME = cconfig["CLOCK_REALTIME"] -CLOCK_MONOTONIC = cconfig["CLOCK_MONOTONIC"] -CLOCK_MONOTONIC_RAW = cconfig["CLOCK_MONOTONIC_RAW"] -CLOCK_PROCESS_CPUTIME_ID = cconfig["CLOCK_PROCESS_CPUTIME_ID"] -CLOCK_THREAD_CPUTIME_ID = cconfig["CLOCK_THREAD_CPUTIME_ID"] if HAS_CLOCK_GETTIME: - #redo it for timespec - CConfig.TIMESPEC = rffi_platform.Struct("struct timespec", [ - ("tv_sec", rffi.TIME_T), - ("tv_nsec", rffi.LONG), - ]) - cconfig = rffi_platform.configure(CConfig) - TIMESPEC = cconfig['TIMESPEC'] - - c_clock_gettime = rffi.llexternal("clock_gettime", - [lltype.Signed, lltype.Ptr(TIMESPEC)], rffi.INT, - compilation_info=CConfig._compilation_info_, releasegil=False, - save_err=rffi.RFFI_SAVE_ERRNO - ) - c_clock_getres = rffi.llexternal("clock_getres", - [lltype.Signed, lltype.Ptr(TIMESPEC)], rffi.INT, - compilation_info=CConfig._compilation_info_, releasegil=False, - save_err=rffi.RFFI_SAVE_ERRNO - ) @unwrap_spec(clk_id="c_int") def clock_gettime(space, clk_id): - with lltype.scoped_alloc(TIMESPEC) as tp: - ret = c_clock_gettime(clk_id, tp) + with lltype.scoped_alloc(rtime.TIMESPEC) as tp: + ret = rtime.c_clock_gettime(clk_id, tp) if ret != 0: raise exception_from_saved_errno(space, space.w_IOError) - return space.wrap(int(tp.c_tv_sec) + 1e-9 * int(tp.c_tv_nsec)) + t = (float(rffi.getintfield(tp, 'c_tv_sec')) + + float(rffi.getintfield(tp, 'c_tv_nsec')) * 0.000000001) + return space.wrap(t) @unwrap_spec(clk_id="c_int") def clock_getres(space, clk_id): - with lltype.scoped_alloc(TIMESPEC) as tp: - ret = c_clock_getres(clk_id, tp) + with lltype.scoped_alloc(rtime.TIMESPEC) as tp: + ret = rtime.c_clock_getres(clk_id, tp) if ret != 0: raise exception_from_saved_errno(space, space.w_IOError) - return space.wrap(int(tp.c_tv_sec) + 1e-9 * int(tp.c_tv_nsec)) + t = (float(rffi.getintfield(tp, 'c_tv_sec')) + + float(rffi.getintfield(tp, 'c_tv_nsec')) * 0.000000001) + return space.wrap(t) diff --git a/pypy/tool/build_cffi_imports.py b/pypy/tool/build_cffi_imports.py --- a/pypy/tool/build_cffi_imports.py +++ b/pypy/tool/build_cffi_imports.py @@ -1,5 +1,4 @@ -import sys, shutil -from rpython.tool.runsubprocess import run_subprocess +import sys, shutil, os class MissingDependenciesError(Exception): pass @@ -20,6 +19,8 @@ } def create_cffi_import_libraries(pypy_c, options, basedir): + from rpython.tool.runsubprocess import run_subprocess + shutil.rmtree(str(basedir.join('lib_pypy', '__pycache__')), ignore_errors=True) failures = [] @@ -44,11 +45,16 @@ return failures if __name__ == '__main__': - import py, os if '__pypy__' not in sys.builtin_module_names: print >> sys.stderr, 'Call with a pypy interpreter' sys.exit(1) + tool_dir = os.path.dirname(os.path.abspath(sys.argv[0])) + base_dir = os.path.dirname(os.path.dirname(tool_dir)) + sys.path.insert(0, base_dir) + + import py + class Options(object): pass diff --git a/rpython/rlib/rtime.py b/rpython/rlib/rtime.py --- a/rpython/rlib/rtime.py +++ b/rpython/rlib/rtime.py @@ -67,10 +67,17 @@ includes=['time.h'], libraries=libraries ) + _NO_MISSING_RT = rffi_platform.Has('printf("%d", clock_gettime(0, 0))') TIMESPEC = rffi_platform.Struct('struct timespec', [('tv_sec', rffi.LONG), ('tv_nsec', rffi.LONG)]) -constant_names = ['RUSAGE_SELF', 'EINTR', 'CLOCK_PROCESS_CPUTIME_ID'] +constant_names = ['RUSAGE_SELF', 'EINTR', + 'CLOCK_REALTIME', + 'CLOCK_MONOTONIC', + 'CLOCK_MONOTONIC_RAW', + 'CLOCK_PROCESS_CPUTIME_ID', + 'CLOCK_THREAD_CPUTIME_ID', +] for const in constant_names: setattr(CConfig, const, rffi_platform.DefinedConstantInteger(const)) defs_names = ['GETTIMEOFDAY_NO_TZ'] @@ -158,16 +165,37 @@ divisor = 0.0 counter_start = 0 state = State() -elif CLOCK_PROCESS_CPUTIME_ID is not None: + +HAS_CLOCK_GETTIME = (CLOCK_MONOTONIC is not None) +if HAS_CLOCK_GETTIME: # Linux and other POSIX systems with clock_gettime() + # TIMESPEC: globals().update(rffi_platform.configure(CConfigForClockGetTime)) - TIMESPEC = TIMESPEC - CLOCK_PROCESS_CPUTIME_ID = CLOCK_PROCESS_CPUTIME_ID - eci_with_lrt = eci.merge(ExternalCompilationInfo(libraries=['rt'])) + # do we need to add -lrt? + eciclock = CConfigForClockGetTime._compilation_info_ + if not _NO_MISSING_RT: + eciclock = eciclock.merge(ExternalCompilationInfo(libraries=['rt'])) + # the functions: + c_clock_getres = external("clock_getres", + [lltype.Signed, lltype.Ptr(TIMESPEC)], + rffi.INT, releasegil=False, + save_err=rffi.RFFI_SAVE_ERRNO, + compilation_info=eciclock) c_clock_gettime = external('clock_gettime', [lltype.Signed, lltype.Ptr(TIMESPEC)], rffi.INT, releasegil=False, - compilation_info=eci_with_lrt) + save_err=rffi.RFFI_SAVE_ERRNO, + compilation_info=eciclock) + c_clock_settime = external('clock_settime', + [lltype.Signed, lltype.Ptr(TIMESPEC)], + rffi.INT, releasegil=False, + save_err=rffi.RFFI_SAVE_ERRNO, + compilation_info=eciclock) + # Note: there is no higher-level functions here to access + # clock_gettime(). The issue is that we'd need a way that keeps + # nanosecond precision, depending on the usage, so we can't have a + # nice function that returns the time as a float. + if need_rusage: RUSAGE = RUSAGE RUSAGE_SELF = RUSAGE_SELF or 0 @@ -191,18 +219,16 @@ def clock(): if _WIN32: return win_perf_counter() - elif CLOCK_PROCESS_CPUTIME_ID is not None: + elif HAS_CLOCK_GETTIME and CLOCK_PROCESS_CPUTIME_ID is not None: with lltype.scoped_alloc(TIMESPEC) as a: - c_clock_gettime(CLOCK_PROCESS_CPUTIME_ID, a) - result = (float(rffi.getintfield(a, 'c_tv_sec')) + - float(rffi.getintfield(a, 'c_tv_nsec')) * 0.000000001) - return result - else: - with lltype.scoped_alloc(RUSAGE) as a: - c_getrusage(RUSAGE_SELF, a) - result = (decode_timeval(a.c_ru_utime) + - decode_timeval(a.c_ru_stime)) - return result + if c_clock_gettime(CLOCK_PROCESS_CPUTIME_ID, a) == 0: + return (float(rffi.getintfield(a, 'c_tv_sec')) + + float(rffi.getintfield(a, 'c_tv_nsec')) * 0.000000001) + with lltype.scoped_alloc(RUSAGE) as a: + c_getrusage(RUSAGE_SELF, a) + result = (decode_timeval(a.c_ru_utime) + + decode_timeval(a.c_ru_stime)) + return result # _______________________________________________________________ # time.sleep() diff --git a/rpython/rlib/test/test_rtime.py b/rpython/rlib/test/test_rtime.py --- a/rpython/rlib/test/test_rtime.py +++ b/rpython/rlib/test/test_rtime.py @@ -1,7 +1,9 @@ from rpython.rtyper.test.tool import BaseRtypingTest +from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.rlib import rtime -import time, sys +import time, sys, py class TestTime(BaseRtypingTest): def test_time_time(self): @@ -53,3 +55,27 @@ t1 = time.time() assert t0 <= t1 assert t1 - t0 >= 0.15 + + def test_clock_gettime(self): + if not rtime.HAS_CLOCK_GETTIME: + py.test.skip("no clock_gettime()") + lst = [] + for i in range(50): + with lltype.scoped_alloc(rtime.TIMESPEC) as a1: + res = rtime.c_clock_gettime(rtime.CLOCK_MONOTONIC, a1) + assert res == 0 + t = (float(rffi.getintfield(a1, 'c_tv_sec')) + + float(rffi.getintfield(a1, 'c_tv_nsec')) * 0.000000001) + lst.append(t) + assert lst == sorted(lst) + + def test_clock_getres(self): + if not rtime.HAS_CLOCK_GETTIME: + py.test.skip("no clock_gettime()") + lst = [] + with lltype.scoped_alloc(rtime.TIMESPEC) as a1: + res = rtime.c_clock_getres(rtime.CLOCK_MONOTONIC, a1) + assert res == 0 + t = (float(rffi.getintfield(a1, 'c_tv_sec')) + + float(rffi.getintfield(a1, 'c_tv_nsec')) * 0.000000001) + assert 0.0 < t <= 1.0 From pypy.commits at gmail.com Sat Oct 1 04:46:43 2016 From: pypy.commits at gmail.com (arigo) Date: Sat, 01 Oct 2016 01:46:43 -0700 (PDT) Subject: [pypy-commit] pypy default: add some more CLOCK_xxx constants that exist on some platforms Message-ID: <57ef77f3.57981c0a.62791.afb6@mx.google.com> Author: Armin Rigo Branch: Changeset: r87486:8f678b4a73b3 Date: 2016-10-01 10:42 +0200 http://bitbucket.org/pypy/pypy/changeset/8f678b4a73b3/ Log: add some more CLOCK_xxx constants that exist on some platforms diff --git a/rpython/rlib/rtime.py b/rpython/rlib/rtime.py --- a/rpython/rlib/rtime.py +++ b/rpython/rlib/rtime.py @@ -73,10 +73,15 @@ constant_names = ['RUSAGE_SELF', 'EINTR', 'CLOCK_REALTIME', + 'CLOCK_REALTIME_COARSE', 'CLOCK_MONOTONIC', + 'CLOCK_MONOTONIC_COARSE', 'CLOCK_MONOTONIC_RAW', + 'CLOCK_BOOTTIME', 'CLOCK_PROCESS_CPUTIME_ID', 'CLOCK_THREAD_CPUTIME_ID', + 'CLOCK_HIGHRES', + 'CLOCK_PROF', ] for const in constant_names: setattr(CConfig, const, rffi_platform.DefinedConstantInteger(const)) From pypy.commits at gmail.com Sat Oct 1 04:46:45 2016 From: pypy.commits at gmail.com (arigo) Date: Sat, 01 Oct 2016 01:46:45 -0700 (PDT) Subject: [pypy-commit] pypy default: Expose all defined 'CLOCK_' names from rtime.py Message-ID: <57ef77f5.0cce1c0a.95f46.a94d@mx.google.com> Author: Armin Rigo Branch: Changeset: r87487:2d371cd33c0f Date: 2016-10-01 10:46 +0200 http://bitbucket.org/pypy/pypy/changeset/2d371cd33c0f/ Log: Expose all defined 'CLOCK_' names from rtime.py diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py --- a/pypy/module/__pypy__/__init__.py +++ b/pypy/module/__pypy__/__init__.py @@ -19,12 +19,8 @@ if rtime.HAS_CLOCK_GETTIME: interpleveldefs["clock_gettime"] = "interp_time.clock_gettime" interpleveldefs["clock_getres"] = "interp_time.clock_getres" - for name in [ - "CLOCK_REALTIME", "CLOCK_MONOTONIC", "CLOCK_MONOTONIC_RAW", - "CLOCK_PROCESS_CPUTIME_ID", "CLOCK_THREAD_CPUTIME_ID" - ]: - if getattr(rtime, name) is not None: - interpleveldefs[name] = "space.wrap(%d)" % getattr(rtime, name) + for name in rtime.ALL_DEFINED_CLOCKS: + interpleveldefs[name] = "space.wrap(%d)" % getattr(rtime, name) class ThreadModule(MixedModule): diff --git a/rpython/rlib/rtime.py b/rpython/rlib/rtime.py --- a/rpython/rlib/rtime.py +++ b/rpython/rlib/rtime.py @@ -200,6 +200,9 @@ # clock_gettime(). The issue is that we'd need a way that keeps # nanosecond precision, depending on the usage, so we can't have a # nice function that returns the time as a float. + ALL_DEFINED_CLOCKS = [const for const in constant_names + if const.startswith('CLOCK_') + and globals()[const] is not None] if need_rusage: RUSAGE = RUSAGE From pypy.commits at gmail.com Sat Oct 1 04:56:03 2016 From: pypy.commits at gmail.com (arigo) Date: Sat, 01 Oct 2016 01:56:03 -0700 (PDT) Subject: [pypy-commit] pypy py3k: hg merge default Message-ID: <57ef7a23.8f081c0a.598b3.af8d@mx.google.com> Author: Armin Rigo Branch: py3k Changeset: r87488:160c8ded58c9 Date: 2016-10-01 10:46 +0200 http://bitbucket.org/pypy/pypy/changeset/160c8ded58c9/ Log: hg merge default diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py --- a/pypy/module/__pypy__/__init__.py +++ b/pypy/module/__pypy__/__init__.py @@ -19,12 +19,8 @@ if rtime.HAS_CLOCK_GETTIME: interpleveldefs["clock_gettime"] = "interp_time.clock_gettime" interpleveldefs["clock_getres"] = "interp_time.clock_getres" - for name in [ - "CLOCK_REALTIME", "CLOCK_MONOTONIC", "CLOCK_MONOTONIC_RAW", - "CLOCK_PROCESS_CPUTIME_ID", "CLOCK_THREAD_CPUTIME_ID" - ]: - if getattr(rtime, name) is not None: - interpleveldefs[name] = "space.wrap(%d)" % getattr(rtime, name) + for name in rtime.ALL_DEFINED_CLOCKS: + interpleveldefs[name] = "space.wrap(%d)" % getattr(rtime, name) class ThreadModule(MixedModule): diff --git a/rpython/rlib/rtime.py b/rpython/rlib/rtime.py --- a/rpython/rlib/rtime.py +++ b/rpython/rlib/rtime.py @@ -73,10 +73,15 @@ constant_names = ['RUSAGE_SELF', 'EINTR', 'CLOCK_REALTIME', + 'CLOCK_REALTIME_COARSE', 'CLOCK_MONOTONIC', + 'CLOCK_MONOTONIC_COARSE', 'CLOCK_MONOTONIC_RAW', + 'CLOCK_BOOTTIME', 'CLOCK_PROCESS_CPUTIME_ID', 'CLOCK_THREAD_CPUTIME_ID', + 'CLOCK_HIGHRES', + 'CLOCK_PROF', ] for const in constant_names: setattr(CConfig, const, rffi_platform.DefinedConstantInteger(const)) @@ -195,6 +200,9 @@ # clock_gettime(). The issue is that we'd need a way that keeps # nanosecond precision, depending on the usage, so we can't have a # nice function that returns the time as a float. + ALL_DEFINED_CLOCKS = [const for const in constant_names + if const.startswith('CLOCK_') + and globals()[const] is not None] if need_rusage: RUSAGE = RUSAGE From pypy.commits at gmail.com Sat Oct 1 04:56:05 2016 From: pypy.commits at gmail.com (arigo) Date: Sat, 01 Oct 2016 01:56:05 -0700 (PDT) Subject: [pypy-commit] pypy py3k: Use rtime.clock_{gettime,settime,getres} Message-ID: <57ef7a25.0558c20a.e174a.06c8@mx.google.com> Author: Armin Rigo Branch: py3k Changeset: r87489:1b443a0430fd Date: 2016-10-01 10:55 +0200 http://bitbucket.org/pypy/pypy/changeset/1b443a0430fd/ Log: Use rtime.clock_{gettime,settime,getres} diff --git a/pypy/module/time/__init__.py b/pypy/module/time/__init__.py --- a/pypy/module/time/__init__.py +++ b/pypy/module/time/__init__.py @@ -1,7 +1,7 @@ from pypy.interpreter.mixedmodule import MixedModule -from .interp_time import (CLOCK_CONSTANTS, HAS_CLOCK_GETTIME, cConfig, - HAS_MONOTONIC) +from .interp_time import HAS_MONOTONIC +from rpython.rlib import rtime import os _WIN = os.name == "nt" @@ -24,20 +24,18 @@ 'process_time': 'interp_time.process_time', } - if HAS_CLOCK_GETTIME: + if rtime.HAS_CLOCK_GETTIME: interpleveldefs['clock_gettime'] = 'interp_time.clock_gettime' interpleveldefs['clock_settime'] = 'interp_time.clock_settime' interpleveldefs['clock_getres'] = 'interp_time.clock_getres' + for constant in rtime.ALL_DEFINED_CLOCKS: + interpleveldefs[constant] = 'space.wrap(%d)' % ( + getattr(rtime, constant),) if HAS_MONOTONIC: interpleveldefs['monotonic'] = 'interp_time.monotonic' if os.name == "posix": interpleveldefs['tzset'] = 'interp_time.tzset' - for constant in CLOCK_CONSTANTS: - value = getattr(cConfig, constant) - if value is not None: - interpleveldefs[constant] = 'space.wrap(interp_time.cConfig.%s)' % constant - appleveldefs = { 'struct_time': 'app_time.struct_time', '__doc__': 'app_time.__doc__', diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py --- a/pypy/module/time/interp_time.py +++ b/pypy/module/time/interp_time.py @@ -160,15 +160,6 @@ ) CLOCKS_PER_SEC = platform.ConstantInteger("CLOCKS_PER_SEC") has_gettimeofday = platform.Has('gettimeofday') - has_clock_gettime = platform.Has('clock_gettime') - CLOCK_PROF = platform.DefinedConstantInteger('CLOCK_PROF') - -CLOCK_CONSTANTS = ['CLOCK_HIGHRES', 'CLOCK_MONOTONIC', 'CLOCK_MONOTONIC_RAW', - 'CLOCK_PROCESS_CPUTIME_ID', 'CLOCK_REALTIME', - 'CLOCK_THREAD_CPUTIME_ID'] - -for constant in CLOCK_CONSTANTS: - setattr(CConfig, constant, platform.DefinedConstantInteger(constant)) if _POSIX: calling_conv = 'c' @@ -227,9 +218,9 @@ timeval = cConfig.timeval CLOCKS_PER_SEC = cConfig.CLOCKS_PER_SEC -HAS_CLOCK_GETTIME = cConfig.has_clock_gettime -HAS_CLOCK_HIGHRES = cConfig.CLOCK_HIGHRES is not None -HAS_CLOCK_MONOTONIC = cConfig.CLOCK_MONOTONIC is not None +HAS_CLOCK_GETTIME = rtime.HAS_CLOCK_GETTIME +HAS_CLOCK_HIGHRES = rtime.CLOCK_HIGHRES is not None +HAS_CLOCK_MONOTONIC = rtime.CLOCK_MONOTONIC is not None HAS_MONOTONIC = (_WIN or _MACOSX or (HAS_CLOCK_GETTIME and (HAS_CLOCK_HIGHRES or HAS_CLOCK_MONOTONIC))) tm = cConfig.tm @@ -316,12 +307,7 @@ save_err=rffi.RFFI_SAVE_ERRNO) if HAS_CLOCK_GETTIME: from rpython.rlib.rtime import TIMESPEC, c_clock_gettime - c_clock_settime = external('clock_settime', - [lltype.Signed, lltype.Ptr(TIMESPEC)], rffi.INT, - save_err=rffi.RFFI_SAVE_ERRNO) - c_clock_getres = external('clock_getres', - [lltype.Signed, lltype.Ptr(TIMESPEC)], rffi.INT, - save_err=rffi.RFFI_SAVE_ERRNO) + from rpython.rlib.rtime import c_clock_settime, c_clock_getres if _POSIX: c_tzset = external('tzset', [], lltype.Void) if _WIN: @@ -623,11 +609,11 @@ Fractions of a second may be present if the system clock provides them.""" if HAS_CLOCK_GETTIME: with lltype.scoped_alloc(TIMESPEC) as timespec: - ret = c_clock_gettime(cConfig.CLOCK_REALTIME, timespec) + ret = c_clock_gettime(rtime.CLOCK_REALTIME, timespec) if ret == 0: if w_info is not None: with lltype.scoped_alloc(TIMESPEC) as tsres: - ret = c_clock_getres(cConfig.CLOCK_REALTIME, tsres) + ret = c_clock_getres(rtime.CLOCK_REALTIME, tsres) if ret == 0: res = _timespec_to_seconds(tsres) else: @@ -903,11 +889,11 @@ else: assert _POSIX def monotonic(space, w_info=None): - if cConfig.CLOCK_HIGHRES is not None: - clk_id = cConfig.CLOCK_HIGHRES + if rtime.CLOCK_HIGHRES is not None: + clk_id = rtime.CLOCK_HIGHRES implementation = "clock_gettime(CLOCK_HIGHRES)" else: - clk_id = cConfig.CLOCK_MONOTONIC + clk_id = rtime.CLOCK_MONOTONIC implementation = "clock_gettime(CLOCK_MONOTONIC)" w_result = clock_gettime(space, clk_id) if w_info is not None: @@ -994,13 +980,13 @@ def process_time(space, w_info=None): if HAS_CLOCK_GETTIME and ( - cConfig.CLOCK_PROF is not None or - cConfig.CLOCK_PROCESS_CPUTIME_ID is not None): - if cConfig.CLOCK_PROF is not None: - clk_id = cConfig.CLOCK_PROF + rtime.CLOCK_PROF is not None or + rtime.CLOCK_PROCESS_CPUTIME_ID is not None): + if rtime.CLOCK_PROF is not None: + clk_id = rtime.CLOCK_PROF implementation = "clock_gettime(CLOCK_PROF)" else: - clk_id = cConfig.CLOCK_PROCESS_CPUTIME_ID + clk_id = rtime.CLOCK_PROCESS_CPUTIME_ID implementation = "clock_gettime(CLOCK_PROCESS_CPUTIME_ID)" with lltype.scoped_alloc(TIMESPEC) as timespec: ret = c_clock_gettime(clk_id, timespec) From pypy.commits at gmail.com Sat Oct 1 05:16:50 2016 From: pypy.commits at gmail.com (arigo) Date: Sat, 01 Oct 2016 02:16:50 -0700 (PDT) Subject: [pypy-commit] pypy faulthandler: A branch to backport py3k-faulthandler Message-ID: <57ef7f02.0145c20a.1aa82.0fd3@mx.google.com> Author: Armin Rigo Branch: faulthandler Changeset: r87490:7e361495eaca Date: 2016-10-01 10:58 +0200 http://bitbucket.org/pypy/pypy/changeset/7e361495eaca/ Log: A branch to backport py3k-faulthandler From pypy.commits at gmail.com Sat Oct 1 05:16:52 2016 From: pypy.commits at gmail.com (arigo) Date: Sat, 01 Oct 2016 02:16:52 -0700 (PDT) Subject: [pypy-commit] pypy faulthandler: Manual graft of the py3k-faulthandler branch Message-ID: <57ef7f04.94a81c0a.eb70.2fe8@mx.google.com> Author: Armin Rigo Branch: faulthandler Changeset: r87491:15ef8db2a0e7 Date: 2016-10-01 11:16 +0200 http://bitbucket.org/pypy/pypy/changeset/15ef8db2a0e7/ Log: Manual graft of the py3k-faulthandler branch diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py --- a/pypy/config/pypyoption.py +++ b/pypy/config/pypyoption.py @@ -43,6 +43,7 @@ try: if detect_cpu.autodetect().startswith('x86'): working_modules.add('_vmprof') + working_modules.add('faulthandler') except detect_cpu.ProcessorAutodetectError: pass @@ -89,6 +90,7 @@ ('objspace.usemodules.thread', True)], 'cpyext': [('objspace.usemodules.array', True)], 'cppyy': [('objspace.usemodules.cpyext', True)], + 'faulthandler': [('objspace.usemodules._vmprof', True)], } module_suggests = { # the reason you want _rawffi is for ctypes, which @@ -114,7 +116,8 @@ "_hashlib" : ["pypy.module._ssl.interp_ssl"], "_minimal_curses": ["pypy.module._minimal_curses.fficurses"], "_continuation": ["rpython.rlib.rstacklet"], - "_vmprof" : ["pypy.module._vmprof.interp_vmprof"], + "_vmprof" : ["pypy.module._vmprof.interp_vmprof"], + "faulthandler" : ["pypy.module._vmprof.interp_vmprof"], } def get_module_validator(modname): diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py --- a/pypy/interpreter/app_main.py +++ b/pypy/interpreter/app_main.py @@ -33,6 +33,7 @@ --info : print translation information about this PyPy executable -X track-resources : track the creation of files and sockets and display a warning if they are not closed explicitly +-X faulthandler : attempt to display tracebacks when PyPy crashes """ # Missing vs CPython: PYTHONHOME, PYTHONCASEOK USAGE2 = """ @@ -233,12 +234,22 @@ import pypyjit pypyjit.set_param(jitparam) +def run_faulthandler(): + if 'faulthandler' in sys.builtin_module_names: + import faulthandler + try: + faulthandler.enable(2) # manually set to stderr + except ValueError: + pass # ignore "2 is not a valid file descriptor" + def set_runtime_options(options, Xparam, *args): if Xparam == 'track-resources': sys.pypy_set_track_resources(True) + elif Xparam == 'faulthandler': + run_faulthandler() else: print >> sys.stderr, 'usage: %s -X [options]' % (get_sys_executable(),) - print >> sys.stderr, '[options] can be: track-resources' + print >> sys.stderr, '[options] can be: track-resources, faulthandler' raise SystemExit class CommandLineError(Exception): @@ -527,6 +538,9 @@ print >> sys.stderr, ( "Warning: pypy does not implement py3k warnings") + if os.getenv('PYTHONFAULTHANDLER'): + run_faulthandler() + ## if not we_are_translated(): ## for key in sorted(options): ## print '%40s: %s' % (key, options[key]) diff --git a/pypy/module/faulthandler/__init__.py b/pypy/module/faulthandler/__init__.py new file mode 100644 --- /dev/null +++ b/pypy/module/faulthandler/__init__.py @@ -0,0 +1,38 @@ +import sys +from pypy.interpreter.mixedmodule import MixedModule + + +class Module(MixedModule): + appleveldefs = { + } + + interpleveldefs = { + 'enable': 'handler.enable', + 'disable': 'handler.disable', + 'is_enabled': 'handler.is_enabled', +# 'register': 'interp_faulthandler.register', +# + 'dump_traceback': 'handler.dump_traceback', +# + '_read_null': 'handler.read_null', + '_sigsegv': 'handler.sigsegv', + '_sigfpe': 'handler.sigfpe', + '_sigabrt': 'handler.sigabrt', + '_stack_overflow': 'handler.stack_overflow', + } + + def setup_after_space_initialization(self): + """NOT_RPYTHON""" + if self.space.config.translation.thread: + self.extra_interpdef('dump_traceback_later', + 'handler.dump_traceback_later') + self.extra_interpdef('cancel_dump_traceback_later', + 'handler.cancel_dump_traceback_later') + if sys.platform != 'win32': + self.extra_interpdef('register', 'handler.register') + self.extra_interpdef('unregister', 'handler.unregister') + + def shutdown(self, space): + from pypy.module.faulthandler import handler + handler.finish(space) + MixedModule.shutdown(self, space) diff --git a/pypy/module/faulthandler/cintf.py b/pypy/module/faulthandler/cintf.py new file mode 100644 --- /dev/null +++ b/pypy/module/faulthandler/cintf.py @@ -0,0 +1,99 @@ +import py +from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, rstr +from rpython.translator import cdir +from rpython.translator.tool.cbuild import ExternalCompilationInfo + + +cwd = py.path.local(__file__).dirpath() +eci = ExternalCompilationInfo( + includes=[cwd.join('faulthandler.h')], + include_dirs=[str(cwd), cdir], + separate_module_files=[cwd.join('faulthandler.c')]) + +eci_later = eci.merge(ExternalCompilationInfo( + pre_include_bits=['#define PYPY_FAULTHANDLER_LATER\n'])) +eci_user = eci.merge(ExternalCompilationInfo( + pre_include_bits=['#define PYPY_FAULTHANDLER_USER\n'])) + +def direct_llexternal(*args, **kwargs): + kwargs.setdefault('_nowrapper', True) + kwargs.setdefault('compilation_info', eci) + return rffi.llexternal(*args, **kwargs) + + +DUMP_CALLBACK = lltype.Ptr(lltype.FuncType( + [rffi.INT, rffi.SIGNEDP, lltype.Signed], lltype.Void)) + +pypy_faulthandler_setup = direct_llexternal( + 'pypy_faulthandler_setup', [DUMP_CALLBACK], rffi.CCHARP) + +pypy_faulthandler_teardown = direct_llexternal( + 'pypy_faulthandler_teardown', [], lltype.Void) + +pypy_faulthandler_enable = direct_llexternal( + 'pypy_faulthandler_enable', [rffi.INT, rffi.INT], rffi.CCHARP) + +pypy_faulthandler_disable = direct_llexternal( + 'pypy_faulthandler_disable', [], lltype.Void) + +pypy_faulthandler_is_enabled = direct_llexternal( + 'pypy_faulthandler_is_enabled', [], rffi.INT) + +pypy_faulthandler_write = direct_llexternal( + 'pypy_faulthandler_write', [rffi.INT, rffi.CCHARP], lltype.Void) + +pypy_faulthandler_write_int = direct_llexternal( + 'pypy_faulthandler_write_int', [rffi.INT, lltype.Signed], lltype.Void) + +pypy_faulthandler_dump_traceback = direct_llexternal( + 'pypy_faulthandler_dump_traceback', + [rffi.INT, rffi.INT, llmemory.Address], lltype.Void) + +pypy_faulthandler_dump_traceback_later = direct_llexternal( + 'pypy_faulthandler_dump_traceback_later', + [rffi.LONGLONG, rffi.INT, rffi.INT, rffi.INT], rffi.CCHARP, + compilation_info=eci_later) + +pypy_faulthandler_cancel_dump_traceback_later = direct_llexternal( + 'pypy_faulthandler_cancel_dump_traceback_later', [], lltype.Void) + +pypy_faulthandler_check_signum = direct_llexternal( + 'pypy_faulthandler_check_signum', + [rffi.LONG], rffi.INT, + compilation_info=eci_user) + +pypy_faulthandler_register = direct_llexternal( + 'pypy_faulthandler_register', + [rffi.INT, rffi.INT, rffi.INT, rffi.INT], rffi.CCHARP, + compilation_info=eci_user) + +pypy_faulthandler_unregister = direct_llexternal( + 'pypy_faulthandler_unregister', + [rffi.INT], rffi.INT, + compilation_info=eci_user) + + +# for tests... + +pypy_faulthandler_read_null = direct_llexternal( + 'pypy_faulthandler_read_null', [], lltype.Void) + +pypy_faulthandler_read_null_releasegil = direct_llexternal( + 'pypy_faulthandler_read_null', [], lltype.Void, + _nowrapper=False, releasegil=True) + +pypy_faulthandler_sigsegv = direct_llexternal( + 'pypy_faulthandler_sigsegv', [], lltype.Void) + +pypy_faulthandler_sigsegv_releasegil = direct_llexternal( + 'pypy_faulthandler_sigsegv', [], lltype.Void, + _nowrapper=False, releasegil=True) + +pypy_faulthandler_sigfpe = direct_llexternal( + 'pypy_faulthandler_sigfpe', [], lltype.Void) + +pypy_faulthandler_sigabrt = direct_llexternal( + 'pypy_faulthandler_sigabrt', [], lltype.Void) + +pypy_faulthandler_stackoverflow = direct_llexternal( + 'pypy_faulthandler_stackoverflow', [lltype.Float], lltype.Float) diff --git a/pypy/module/faulthandler/dumper.py b/pypy/module/faulthandler/dumper.py new file mode 100644 --- /dev/null +++ b/pypy/module/faulthandler/dumper.py @@ -0,0 +1,54 @@ +from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.rlib import rgc +from rpython.rlib.rvmprof import traceback + +from pypy.interpreter.pycode import PyCode +from pypy.module.faulthandler.cintf import pypy_faulthandler_write +from pypy.module.faulthandler.cintf import pypy_faulthandler_write_int + + +MAX_STRING_LENGTH = 500 + +global_buf = lltype.malloc(rffi.CCHARP.TO, MAX_STRING_LENGTH, flavor='raw', + immortal=True, zero=True) + +def _dump(fd, s): + assert isinstance(s, str) + l = len(s) + if l >= MAX_STRING_LENGTH: + l = MAX_STRING_LENGTH - 1 + i = 0 + while i < l: + global_buf[i] = s[i] + i += 1 + global_buf[l] = '\x00' + pypy_faulthandler_write(fd, global_buf) + +def _dump_int(fd, i): + pypy_faulthandler_write_int(fd, i) + + +def dump_code(pycode, loc, fd): + if pycode is None: + _dump(fd, " File ???") + else: + _dump(fd, ' File "') + _dump(fd, pycode.co_filename) + _dump(fd, '" in ') + _dump(fd, pycode.co_name) + _dump(fd, ", from line ") + _dump_int(fd, pycode.co_firstlineno) + if loc == traceback.LOC_JITTED: + _dump(fd, " [jitted]") + elif loc == traceback.LOC_JITTED_INLINED: + _dump(fd, " [jit inlined]") + _dump(fd, "\n") + + + at rgc.no_collect +def _dump_callback(fd, array_p, array_length): + """We are as careful as we can reasonably be here (i.e. not 100%, + but hopefully close enough). In particular, this is written as + RPython but shouldn't allocate anything. + """ + traceback.walk_traceback(PyCode, dump_code, fd, array_p, array_length) diff --git a/pypy/module/faulthandler/faulthandler.c b/pypy/module/faulthandler/faulthandler.c new file mode 100644 --- /dev/null +++ b/pypy/module/faulthandler/faulthandler.c @@ -0,0 +1,679 @@ +#include "faulthandler.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef RPYTHON_LL2CTYPES +# include "../../../rpython/rlib/rvmprof/src/rvmprof.h" +#else +# include "common_header.h" +# include "structdef.h" +# include "rvmprof.h" +#endif +#include "src/threadlocal.h" + +#define MAX_FRAME_DEPTH 100 +#define FRAME_DEPTH_N RVMPROF_TRACEBACK_ESTIMATE_N(MAX_FRAME_DEPTH) + + +typedef struct sigaction _Py_sighandler_t; + +typedef struct { + const int signum; + volatile int enabled; + const char* name; + _Py_sighandler_t previous; +} fault_handler_t; + +static struct { + int initialized; + int enabled; + volatile int fd, all_threads; + volatile pypy_faulthandler_cb_t dump_traceback; +} fatal_error; + +static stack_t stack; + + +static fault_handler_t faulthandler_handlers[] = { +#ifdef SIGBUS + {SIGBUS, 0, "Bus error", }, +#endif +#ifdef SIGILL + {SIGILL, 0, "Illegal instruction", }, +#endif + {SIGFPE, 0, "Floating point exception", }, + {SIGABRT, 0, "Aborted", }, + /* define SIGSEGV at the end to make it the default choice if searching the + handler fails in faulthandler_fatal_error() */ + {SIGSEGV, 0, "Segmentation fault", } +}; +static const int faulthandler_nsignals = + sizeof(faulthandler_handlers) / sizeof(fault_handler_t); + +RPY_EXTERN +void pypy_faulthandler_write(int fd, const char *str) +{ + (void)write(fd, str, strlen(str)); +} + +RPY_EXTERN +void pypy_faulthandler_write_int(int fd, long value) +{ + char buf[48]; + sprintf(buf, "%ld", value); + pypy_faulthandler_write(fd, buf); +} + + +RPY_EXTERN +void pypy_faulthandler_dump_traceback(int fd, int all_threads, + void *ucontext) +{ + pypy_faulthandler_cb_t fn; + intptr_t array_p[FRAME_DEPTH_N], array_length; + + fn = fatal_error.dump_traceback; + if (!fn) + return; + +#ifndef RPYTHON_LL2CTYPES + if (all_threads && _RPython_ThreadLocals_AcquireTimeout(10000) == 0) { + /* This is known not to be perfectly safe against segfaults if we + don't hold the GIL ourselves. Too bad. I suspect that CPython + has issues there too. + */ + struct pypy_threadlocal_s *my, *p; + int blankline = 0; + char buf[40]; + + my = (struct pypy_threadlocal_s *)_RPy_ThreadLocals_Get(); + p = _RPython_ThreadLocals_Head(); + p = _RPython_ThreadLocals_Enum(p); + while (p != NULL) { + if (blankline) + pypy_faulthandler_write(fd, "\n"); + blankline = 1; + + pypy_faulthandler_write(fd, my == p ? "Current thread" : "Thread"); + sprintf(buf, " 0x%lx", (unsigned long)p->thread_ident); + pypy_faulthandler_write(fd, buf); + pypy_faulthandler_write(fd, " (most recent call first):\n"); + + array_length = vmprof_get_traceback(p->vmprof_tl_stack, + my == p ? ucontext : NULL, + array_p, FRAME_DEPTH_N); + fn(fd, array_p, array_length); + + p = _RPython_ThreadLocals_Enum(p); + } + _RPython_ThreadLocals_Release(); + } + else { + pypy_faulthandler_write(fd, "Stack (most recent call first):\n"); + array_length = vmprof_get_traceback(NULL, ucontext, + array_p, FRAME_DEPTH_N); + fn(fd, array_p, array_length); + } +#else + pypy_faulthandler_write(fd, "(no traceback when untranslated)\n"); +#endif +} + +static void +faulthandler_dump_traceback(int fd, int all_threads, void *ucontext) +{ + static volatile int reentrant = 0; + + if (reentrant) + return; + reentrant = 1; + pypy_faulthandler_dump_traceback(fd, all_threads, ucontext); + reentrant = 0; +} + + +/************************************************************/ + + +#ifdef PYPY_FAULTHANDLER_LATER +#include "src/thread.h" +static struct { + int fd; + long long microseconds; + int repeat, exit; + /* The main thread always holds this lock. It is only released when + faulthandler_thread() is interrupted before this thread exits, or at + Python exit. */ + struct RPyOpaque_ThreadLock cancel_event; + /* released by child thread when joined */ + struct RPyOpaque_ThreadLock running; +} thread_later; + +static void faulthandler_thread(void) +{ +#ifndef _WIN32 + /* we don't want to receive any signal */ + sigset_t set; + sigfillset(&set); + pthread_sigmask(SIG_SETMASK, &set, NULL); +#endif + + RPyLockStatus st; + char buf[64]; + unsigned long hour, minutes, seconds, fraction; + long long t; + + do { + st = RPyThreadAcquireLockTimed(&thread_later.cancel_event, + thread_later.microseconds, 0); + if (st == RPY_LOCK_ACQUIRED) { + RPyThreadReleaseLock(&thread_later.cancel_event); + break; + } + /* Timeout => dump traceback */ + assert(st == RPY_LOCK_FAILURE); + + /* getting to know which thread holds the GIL is not as simple + * as in CPython, so for now we don't */ + + t = thread_later.microseconds; + fraction = (unsigned long)(t % 1000000); + t /= 1000000; + seconds = (unsigned long)(t % 60); + t /= 60; + minutes = (unsigned long)(t % 60); + t /= 60; + hour = (unsigned long)t; + if (fraction == 0) + sprintf(buf, "Timeout (%lu:%02lu:%02lu)!\n", + hour, minutes, seconds); + else + sprintf(buf, "Timeout (%lu:%02lu:%02lu.%06lu)!\n", + hour, minutes, seconds, fraction); + + pypy_faulthandler_write(thread_later.fd, buf); + pypy_faulthandler_dump_traceback(thread_later.fd, 1, NULL); + + if (thread_later.exit) + _exit(1); + } while (thread_later.repeat); + + /* The only way out */ + RPyThreadReleaseLock(&thread_later.running); +} + +RPY_EXTERN +char *pypy_faulthandler_dump_traceback_later(long long microseconds, int repeat, + int fd, int exit) +{ + pypy_faulthandler_cancel_dump_traceback_later(); + + thread_later.fd = fd; + thread_later.microseconds = microseconds; + thread_later.repeat = repeat; + thread_later.exit = exit; + + RPyThreadAcquireLock(&thread_later.running, 1); + + if (RPyThreadStart(&faulthandler_thread) == -1) { + RPyThreadReleaseLock(&thread_later.running); + return "unable to start watchdog thread"; + } + return NULL; +} +#endif /* PYPY_FAULTHANDLER_LATER */ + +RPY_EXTERN +void pypy_faulthandler_cancel_dump_traceback_later(void) +{ +#ifdef PYPY_FAULTHANDLER_LATER + /* Notify cancellation */ + RPyThreadReleaseLock(&thread_later.cancel_event); + + /* Wait for thread to join (or does nothing if no thread is running) */ + RPyThreadAcquireLock(&thread_later.running, 1); + RPyThreadReleaseLock(&thread_later.running); + + /* The main thread should always hold the cancel_event lock */ + RPyThreadAcquireLock(&thread_later.cancel_event, 1); +#endif /* PYPY_FAULTHANDLER_LATER */ +} + + +/************************************************************/ + + +#ifdef PYPY_FAULTHANDLER_USER +typedef struct { + int enabled; + int fd; + int all_threads; + int chain; + _Py_sighandler_t previous; +} user_signal_t; + +static user_signal_t *user_signals; + +#ifndef NSIG +# if defined(_NSIG) +# define NSIG _NSIG /* For BSD/SysV */ +# elif defined(_SIGMAX) +# define NSIG (_SIGMAX + 1) /* For QNX */ +# elif defined(SIGMAX) +# define NSIG (SIGMAX + 1) /* For djgpp */ +# else +# define NSIG 64 /* Use a reasonable default value */ +# endif +#endif + +static void faulthandler_user(int signum, siginfo_t *info, void *ucontext); + +static int +faulthandler_register(int signum, int chain, _Py_sighandler_t *p_previous) +{ + struct sigaction action; + action.sa_handler = faulthandler_user; + sigemptyset(&action.sa_mask); + /* if the signal is received while the kernel is executing a system + call, try to restart the system call instead of interrupting it and + return EINTR. */ + action.sa_flags = SA_RESTART | SA_SIGINFO; + if (chain) { + /* do not prevent the signal from being received from within its + own signal handler */ + action.sa_flags = SA_NODEFER; + } + if (stack.ss_sp != NULL) { + /* Call the signal handler on an alternate signal stack + provided by sigaltstack() */ + action.sa_flags |= SA_ONSTACK; + } + return sigaction(signum, &action, p_previous); +} + +static void faulthandler_user(int signum, siginfo_t *info, void *ucontext) +{ + int save_errno; + user_signal_t *user = &user_signals[signum]; + + if (!user->enabled) + return; + + save_errno = errno; + faulthandler_dump_traceback(user->fd, user->all_threads, ucontext); + + if (user->chain) { + (void)sigaction(signum, &user->previous, NULL); + errno = save_errno; + + /* call the previous signal handler */ + raise(signum); + + save_errno = errno; + (void)faulthandler_register(signum, user->chain, NULL); + } + + errno = save_errno; +} + +RPY_EXTERN +int pypy_faulthandler_check_signum(long signum) +{ + unsigned int i; + + for (i = 0; i < faulthandler_nsignals; i++) { + if (faulthandler_handlers[i].signum == signum) { + return -1; + } + } + if (signum < 1 || NSIG <= signum) { + return -2; + } + return 0; +} + +RPY_EXTERN +char *pypy_faulthandler_register(int signum, int fd, int all_threads, int chain) +{ + user_signal_t *user; + _Py_sighandler_t previous; + int err; + + if (user_signals == NULL) { + user_signals = malloc(NSIG * sizeof(user_signal_t)); + if (user_signals == NULL) + return "out of memory"; + memset(user_signals, 0, NSIG * sizeof(user_signal_t)); + } + + user = &user_signals[signum]; + user->fd = fd; + user->all_threads = all_threads; + user->chain = chain; + + if (!user->enabled) { + err = faulthandler_register(signum, chain, &previous); + if (err) + return strerror(errno); + + user->previous = previous; + user->enabled = 1; + } + return NULL; +} + +RPY_EXTERN +int pypy_faulthandler_unregister(int signum) +{ + user_signal_t *user; + + if (user_signals == NULL) + return 0; + + user = &user_signals[signum]; + if (user->enabled) { + user->enabled = 0; + (void)sigaction(signum, &user->previous, NULL); + user->fd = -1; + return 1; + } + else + return 0; +} +#endif /* PYPY_FAULTHANDLER_USER */ + + +/************************************************************/ + + +/* Handler for SIGSEGV, SIGFPE, SIGABRT, SIGBUS and SIGILL signals. + + Display the current Python traceback, restore the previous handler and call + the previous handler. + + On Windows, don't explicitly call the previous handler, because the Windows + signal handler would not be called (for an unknown reason). The execution of + the program continues at faulthandler_fatal_error() exit, but the same + instruction will raise the same fault (signal), and so the previous handler + will be called. + + This function is signal-safe and should only call signal-safe functions. */ + +static void +faulthandler_fatal_error(int signum, siginfo_t *info, void *ucontext) +{ + int fd = fatal_error.fd; + int i; + fault_handler_t *handler = NULL; + int save_errno = errno; + + for (i = 0; i < faulthandler_nsignals; i++) { + handler = &faulthandler_handlers[i]; + if (handler->signum == signum) + break; + } + /* If not found, we use the SIGSEGV handler (the last one in the list) */ + + /* restore the previous handler */ + if (handler->enabled) { + (void)sigaction(signum, &handler->previous, NULL); + handler->enabled = 0; + } + + pypy_faulthandler_write(fd, "Fatal Python error: "); + pypy_faulthandler_write(fd, handler->name); + pypy_faulthandler_write(fd, "\n\n"); + + faulthandler_dump_traceback(fd, fatal_error.all_threads, ucontext); + + errno = save_errno; +#ifdef MS_WINDOWS + if (signum == SIGSEGV) { + /* don't explicitly call the previous handler for SIGSEGV in this signal + handler, because the Windows signal handler would not be called */ + return; + } +#endif + /* call the previous signal handler: it is called immediately if we use + sigaction() thanks to SA_NODEFER flag, otherwise it is deferred */ + raise(signum); +} + + +RPY_EXTERN +char *pypy_faulthandler_setup(pypy_faulthandler_cb_t dump_callback) +{ + if (fatal_error.initialized) + return NULL; + assert(!fatal_error.enabled); + fatal_error.dump_traceback = dump_callback; + + /* Try to allocate an alternate stack for faulthandler() signal handler to + * be able to allocate memory on the stack, even on a stack overflow. If it + * fails, ignore the error. */ + stack.ss_flags = 0; + stack.ss_size = SIGSTKSZ; + stack.ss_sp = malloc(stack.ss_size); + if (stack.ss_sp != NULL) { + int err = sigaltstack(&stack, NULL); + if (err) { + free(stack.ss_sp); + stack.ss_sp = NULL; + } + } + +#ifdef PYPY_FAULTHANDLER_LATER + if (!RPyThreadLockInit(&thread_later.cancel_event) || + !RPyThreadLockInit(&thread_later.running)) + return "failed to initialize locks"; + RPyThreadAcquireLock(&thread_later.cancel_event, 1); +#endif + + fatal_error.fd = -1; + fatal_error.initialized = 1; + + return NULL; +} + +RPY_EXTERN +void pypy_faulthandler_teardown(void) +{ + if (fatal_error.initialized) { + +#ifdef PYPY_FAULTHANDLER_LATER + pypy_faulthandler_cancel_dump_traceback_later(); + RPyThreadReleaseLock(&thread_later.cancel_event); + RPyOpaqueDealloc_ThreadLock(&thread_later.running); + RPyOpaqueDealloc_ThreadLock(&thread_later.cancel_event); +#endif + +#ifdef PYPY_FAULTHANDLER_USER + int signum; + for (signum = 0; signum < NSIG; signum++) + pypy_faulthandler_unregister(signum); + /* don't free 'user_signals', the gain is very minor and it can + lead to rare crashes if another thread is still busy */ +#endif + + pypy_faulthandler_disable(); + fatal_error.initialized = 0; + if (stack.ss_sp) { + stack.ss_flags = SS_DISABLE; + sigaltstack(&stack, NULL); + free(stack.ss_sp); + stack.ss_sp = NULL; + } + } +} + +RPY_EXTERN +char *pypy_faulthandler_enable(int fd, int all_threads) +{ + /* Install the handler for fatal signals, faulthandler_fatal_error(). */ + int i; + fatal_error.fd = fd; + fatal_error.all_threads = all_threads; + + if (!fatal_error.enabled) { + fatal_error.enabled = 1; + + for (i = 0; i < faulthandler_nsignals; i++) { + int err; + struct sigaction action; + fault_handler_t *handler = &faulthandler_handlers[i]; + + action.sa_sigaction = faulthandler_fatal_error; + sigemptyset(&action.sa_mask); + /* Do not prevent the signal from being received from within + its own signal handler */ + action.sa_flags = SA_NODEFER | SA_SIGINFO; + if (stack.ss_sp != NULL) { + /* Call the signal handler on an alternate signal stack + provided by sigaltstack() */ + action.sa_flags |= SA_ONSTACK; + } + err = sigaction(handler->signum, &action, &handler->previous); + if (err) { + return strerror(errno); + } + handler->enabled = 1; + } + } + return NULL; +} + +RPY_EXTERN +void pypy_faulthandler_disable(void) +{ + int i; + if (fatal_error.enabled) { + fatal_error.enabled = 0; + for (i = 0; i < faulthandler_nsignals; i++) { + fault_handler_t *handler = &faulthandler_handlers[i]; + if (!handler->enabled) + continue; + (void)sigaction(handler->signum, &handler->previous, NULL); + handler->enabled = 0; + } + } + fatal_error.fd = -1; +} + +RPY_EXTERN +int pypy_faulthandler_is_enabled(void) +{ + return fatal_error.enabled; +} + + +/************************************************************/ + + +/* for tests... */ + +static void +faulthandler_suppress_crash_report(void) +{ +#ifdef MS_WINDOWS + UINT mode; + + /* Configure Windows to not display the Windows Error Reporting dialog */ + mode = SetErrorMode(SEM_NOGPFAULTERRORBOX); + SetErrorMode(mode | SEM_NOGPFAULTERRORBOX); +#endif + +#ifndef MS_WINDOWS + struct rlimit rl; + + /* Disable creation of core dump */ + if (getrlimit(RLIMIT_CORE, &rl) != 0) { + rl.rlim_cur = 0; + setrlimit(RLIMIT_CORE, &rl); + } +#endif + +#ifdef _MSC_VER + /* Visual Studio: configure abort() to not display an error message nor + open a popup asking to report the fault. */ + _set_abort_behavior(0, _WRITE_ABORT_MSG | _CALL_REPORTFAULT); +#endif +} + +RPY_EXTERN +int pypy_faulthandler_read_null(void) +{ + int *volatile x; + + faulthandler_suppress_crash_report(); + x = NULL; + return *x; +} + +RPY_EXTERN +void pypy_faulthandler_sigsegv(void) +{ + faulthandler_suppress_crash_report(); +#if defined(MS_WINDOWS) + /* For SIGSEGV, faulthandler_fatal_error() restores the previous signal + handler and then gives back the execution flow to the program (without + explicitly calling the previous error handler). In a normal case, the + SIGSEGV was raised by the kernel because of a fault, and so if the + program retries to execute the same instruction, the fault will be + raised again. + + Here the fault is simulated by a fake SIGSEGV signal raised by the + application. We have to raise SIGSEGV at lease twice: once for + faulthandler_fatal_error(), and one more time for the previous signal + handler. */ + while(1) + raise(SIGSEGV); +#else + raise(SIGSEGV); +#endif +} + +RPY_EXTERN +int pypy_faulthandler_sigfpe(void) +{ + /* Do an integer division by zero: raise a SIGFPE on Intel CPU, but not on + PowerPC. Use volatile to disable compile-time optimizations. */ + volatile int x = 1, y = 0, z; + faulthandler_suppress_crash_report(); + z = x / y; + /* If the division by zero didn't raise a SIGFPE (e.g. on PowerPC), + raise it manually. */ + raise(SIGFPE); + /* This line is never reached, but we pretend to make something with z + to silence a compiler warning. */ + return z; +} + +RPY_EXTERN +void pypy_faulthandler_sigabrt(void) +{ + faulthandler_suppress_crash_report(); + abort(); +} + +static double fh_stack_overflow(double levels) +{ + if (levels > 2.5) { + return (sqrt(fh_stack_overflow(levels - 1.0)) + + fh_stack_overflow(levels * 1e-10)); + } + return 1e100 + levels; +} + +RPY_EXTERN +double pypy_faulthandler_stackoverflow(double levels) +{ + faulthandler_suppress_crash_report(); + return fh_stack_overflow(levels); +} diff --git a/pypy/module/faulthandler/faulthandler.h b/pypy/module/faulthandler/faulthandler.h new file mode 100644 --- /dev/null +++ b/pypy/module/faulthandler/faulthandler.h @@ -0,0 +1,40 @@ +#ifndef PYPY_FAULTHANDLER_H +#define PYPY_FAULTHANDLER_H + +#include "src/precommondefs.h" +#include + + +typedef void (*pypy_faulthandler_cb_t)(int fd, intptr_t *array_p, + intptr_t length); + +RPY_EXTERN char *pypy_faulthandler_setup(pypy_faulthandler_cb_t dump_callback); +RPY_EXTERN void pypy_faulthandler_teardown(void); + +RPY_EXTERN char *pypy_faulthandler_enable(int fd, int all_threads); +RPY_EXTERN void pypy_faulthandler_disable(void); +RPY_EXTERN int pypy_faulthandler_is_enabled(void); + +RPY_EXTERN void pypy_faulthandler_write(int fd, const char *str); +RPY_EXTERN void pypy_faulthandler_write_int(int fd, long value); + +RPY_EXTERN void pypy_faulthandler_dump_traceback(int fd, int all_threads, + void *ucontext); + +RPY_EXTERN char *pypy_faulthandler_dump_traceback_later( + long long microseconds, int repeat, int fd, int exit); +RPY_EXTERN void pypy_faulthandler_cancel_dump_traceback_later(void); + +RPY_EXTERN int pypy_faulthandler_check_signum(long signum); +RPY_EXTERN char *pypy_faulthandler_register(int, int, int, int); +RPY_EXTERN int pypy_faulthandler_unregister(int signum); + + +RPY_EXTERN int pypy_faulthandler_read_null(void); +RPY_EXTERN void pypy_faulthandler_sigsegv(void); +RPY_EXTERN int pypy_faulthandler_sigfpe(void); +RPY_EXTERN void pypy_faulthandler_sigabrt(void); +RPY_EXTERN double pypy_faulthandler_stackoverflow(double); + + +#endif /* PYPY_FAULTHANDLER_H */ diff --git a/pypy/module/faulthandler/handler.py b/pypy/module/faulthandler/handler.py new file mode 100644 --- /dev/null +++ b/pypy/module/faulthandler/handler.py @@ -0,0 +1,209 @@ +import os +from rpython.rtyper.lltypesystem import lltype, llmemory, rffi +from rpython.rlib.rposix import is_valid_fd +from rpython.rlib.rarithmetic import widen, ovfcheck_float_to_longlong +from rpython.rlib.objectmodel import keepalive_until_here +from rpython.rtyper.annlowlevel import llhelper + +from pypy.interpreter.error import OperationError, oefmt +from pypy.interpreter.error import exception_from_saved_errno +from pypy.interpreter.gateway import unwrap_spec +from pypy.module.faulthandler import cintf, dumper + + +class Handler(object): + def __init__(self, space): + "NOT_RPYTHON" + self.space = space + self._cleanup_() + + def _cleanup_(self): + self.fatal_error_w_file = None + self.dump_traceback_later_w_file = None + self.user_w_files = None + + def check_err(self, p_err): + if p_err: + raise oefmt(self.space.w_RuntimeError, 'faulthandler: %8', + rffi.charp2str(p_err)) + + def get_fileno_and_file(self, w_file): + space = self.space + if space.is_none(w_file): + w_file = space.sys.get('stderr') + if space.is_none(w_file): + raise oefmt(space.w_RuntimeError, "sys.stderr is None") + elif space.isinstance_w(w_file, space.w_int): + fd = space.c_int_w(w_file) + if fd < 0 or not is_valid_fd(fd): + raise oefmt(space.w_ValueError, + "file is not a valid file descriptor") + return fd, None + + fd = space.c_int_w(space.call_method(w_file, 'fileno')) + try: + space.call_method(w_file, 'flush') + except OperationError as e: + if e.async(space): + raise + pass # ignore flush() error + return fd, w_file + + def setup(self): + dump_callback = llhelper(cintf.DUMP_CALLBACK, dumper._dump_callback) + self.check_err(cintf.pypy_faulthandler_setup(dump_callback)) + + def enable(self, w_file, all_threads): + fileno, w_file = self.get_fileno_and_file(w_file) + self.setup() + self.fatal_error_w_file = w_file + self.check_err(cintf.pypy_faulthandler_enable( + rffi.cast(rffi.INT, fileno), + rffi.cast(rffi.INT, all_threads))) + + def disable(self): + cintf.pypy_faulthandler_disable() + self.fatal_error_w_file = None + + def is_enabled(self): + return bool(widen(cintf.pypy_faulthandler_is_enabled())) + + def dump_traceback(self, w_file, all_threads): + fileno, w_file = self.get_fileno_and_file(w_file) + self.setup() + cintf.pypy_faulthandler_dump_traceback( + rffi.cast(rffi.INT, fileno), + rffi.cast(rffi.INT, all_threads), + llmemory.NULL) + keepalive_until_here(w_file) + + def dump_traceback_later(self, timeout, repeat, w_file, exit): + space = self.space + timeout *= 1e6 + try: + microseconds = ovfcheck_float_to_longlong(timeout) + except OverflowError: + raise oefmt(space.w_OverflowError, "timeout value is too large") + if microseconds <= 0: + raise oefmt(space.w_ValueError, "timeout must be greater than 0") + fileno, w_file = self.get_fileno_and_file(w_file) + self.setup() + self.check_err(cintf.pypy_faulthandler_dump_traceback_later( + rffi.cast(rffi.LONGLONG, microseconds), + rffi.cast(rffi.INT, repeat), + rffi.cast(rffi.INT, fileno), + rffi.cast(rffi.INT, exit))) + self.dump_traceback_later_w_file = w_file + + def cancel_dump_traceback_later(self): + cintf.pypy_faulthandler_cancel_dump_traceback_later() + self.dump_traceback_later_w_file = None + + def check_signum(self, signum): + err = rffi.cast(lltype.Signed, + cintf.pypy_faulthandler_check_signum(signum)) + if err < 0: + space = self.space + if err == -1: + raise oefmt(space.w_RuntimeError, + "signal %d cannot be registered, " + "use enable() instead", signum) + else: + raise oefmt(space.w_ValueError, "signal number out of range") + + def register(self, signum, w_file, all_threads, chain): + self.check_signum(signum) + fileno, w_file = self.get_fileno_and_file(w_file) + self.setup() + self.check_err(cintf.pypy_faulthandler_register( + rffi.cast(rffi.INT, signum), + rffi.cast(rffi.INT, fileno), + rffi.cast(rffi.INT, all_threads), + rffi.cast(rffi.INT, chain))) + if self.user_w_files is None: + self.user_w_files = {} + self.user_w_files[signum] = w_file + + def unregister(self, signum): + self.check_signum(signum) + change = cintf.pypy_faulthandler_unregister( + rffi.cast(rffi.INT, signum)) + if self.user_w_files is not None: + self.user_w_files.pop(signum, None) + return rffi.cast(lltype.Signed, change) == 1 + + def finish(self): + cintf.pypy_faulthandler_teardown() + self._cleanup_() + + +def finish(space): + "Finalize the faulthandler logic (called from shutdown())" + space.fromcache(Handler).finish() + + + at unwrap_spec(all_threads=int) +def enable(space, w_file=None, all_threads=0): + "enable(file=sys.stderr, all_threads=True): enable the fault handler" + space.fromcache(Handler).enable(w_file, all_threads) + +def disable(space): + "disable(): disable the fault handler" + space.fromcache(Handler).disable() + +def is_enabled(space): + "is_enabled()->bool: check if the handler is enabled" + return space.wrap(space.fromcache(Handler).is_enabled()) + + at unwrap_spec(all_threads=int) +def dump_traceback(space, w_file=None, all_threads=0): + """dump the traceback of the current thread into file + including all threads if all_threads is True""" + space.fromcache(Handler).dump_traceback(w_file, all_threads) + + at unwrap_spec(timeout=float, repeat=int, exit=int) +def dump_traceback_later(space, timeout, repeat=0, w_file=None, exit=0): + """dump the traceback of all threads in timeout seconds, + or each timeout seconds if repeat is True. If exit is True, + call _exit(1) which is not safe.""" + space.fromcache(Handler).dump_traceback_later(timeout, repeat, w_file, exit) + +def cancel_dump_traceback_later(space): + """cancel the previous call to dump_traceback_later().""" + space.fromcache(Handler).cancel_dump_traceback_later() + + at unwrap_spec(signum=int, all_threads=int, chain=int) +def register(space, signum, w_file=None, all_threads=1, chain=0): + space.fromcache(Handler).register(signum, w_file, all_threads, chain) + + at unwrap_spec(signum=int) +def unregister(space, signum): + return space.wrap(space.fromcache(Handler).unregister(signum)) + + +# for tests... + + at unwrap_spec(release_gil=int) +def read_null(space, release_gil=0): + if release_gil: + cintf.pypy_faulthandler_read_null_releasegil() + else: + cintf.pypy_faulthandler_read_null() + + at unwrap_spec(release_gil=int) +def sigsegv(space, release_gil=0): + if release_gil: + cintf.pypy_faulthandler_sigsegv_releasegil() + else: + cintf.pypy_faulthandler_sigsegv() + +def sigfpe(space): + cintf.pypy_faulthandler_sigfpe() + +def sigabrt(space): + cintf.pypy_faulthandler_sigabrt() + + at unwrap_spec(levels=int) +def stack_overflow(space, levels=100000000): + levels = float(levels) + return space.wrap(cintf.pypy_faulthandler_stackoverflow(levels)) diff --git a/pypy/module/faulthandler/test/test_faulthander.py b/pypy/module/faulthandler/test/test_faulthander.py new file mode 100644 --- /dev/null +++ b/pypy/module/faulthandler/test/test_faulthander.py @@ -0,0 +1,18 @@ + +class AppTestFaultHandler: + spaceconfig = { + "usemodules": ["faulthandler", "_vmprof"] + } + + def test_enable(self): + import faulthandler, sys + faulthandler.enable() + assert faulthandler.is_enabled() is True + faulthandler.enable(file=sys.stderr, all_threads=True) + faulthandler.disable() + assert faulthandler.is_enabled() is False + + def test_dump_traceback(self): + import faulthandler, sys + faulthandler.dump_traceback() + faulthandler.dump_traceback(file=sys.stderr, all_threads=True) diff --git a/pypy/module/faulthandler/test/test_ztranslation.py b/pypy/module/faulthandler/test/test_ztranslation.py new file mode 100644 --- /dev/null +++ b/pypy/module/faulthandler/test/test_ztranslation.py @@ -0,0 +1,4 @@ +from pypy.objspace.fake.checkmodule import checkmodule + +def test_faulthandler_translates(): + checkmodule('faulthandler') diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py --- a/rpython/rlib/rvmprof/cintf.py +++ b/rpython/rlib/rvmprof/cintf.py @@ -56,6 +56,11 @@ [rffi.INT], lltype.Void, compilation_info=eci, _nowrapper=True) + vmprof_get_traceback = rffi.llexternal("vmprof_get_traceback", + [PVMPROFSTACK, llmemory.Address, + rffi.SIGNEDP, lltype.Signed], + lltype.Signed, compilation_info=eci, + _nowrapper=True) return CInterface(locals()) @@ -154,3 +159,9 @@ def restore_rvmprof_stack(x): vmprof_tl_stack.setraw(x) + +# +# traceback support + +def get_rvmprof_stack(): + return vmprof_tl_stack.get_or_make_raw() diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py --- a/rpython/rlib/rvmprof/rvmprof.py +++ b/rpython/rlib/rvmprof/rvmprof.py @@ -1,6 +1,6 @@ import sys, os from rpython.rlib.objectmodel import specialize, we_are_translated -from rpython.rlib import jit, rposix +from rpython.rlib import jit, rposix, rgc from rpython.rlib.rvmprof import cintf from rpython.rtyper.annlowlevel import cast_instance_to_gcref from rpython.rtyper.annlowlevel import cast_base_ptr_to_instance diff --git a/rpython/rlib/rvmprof/src/rvmprof.h b/rpython/rlib/rvmprof/src/rvmprof.h --- a/rpython/rlib/rvmprof/src/rvmprof.h +++ b/rpython/rlib/rvmprof/src/rvmprof.h @@ -1,3 +1,4 @@ +#include RPY_EXTERN char *vmprof_init(int, double, char *); RPY_EXTERN void vmprof_ignore_signals(int); @@ -8,3 +9,6 @@ RPY_EXTERN int vmprof_stack_append(void*, long); RPY_EXTERN long vmprof_stack_pop(void*); RPY_EXTERN void vmprof_stack_free(void*); +RPY_EXTERN intptr_t vmprof_get_traceback(void *, void *, intptr_t*, intptr_t); + +#define RVMPROF_TRACEBACK_ESTIMATE_N(num_entries) (2 * (num_entries) + 4) diff --git a/rpython/rlib/rvmprof/src/vmprof_common.h b/rpython/rlib/rvmprof/src/vmprof_common.h --- a/rpython/rlib/rvmprof/src/vmprof_common.h +++ b/rpython/rlib/rvmprof/src/vmprof_common.h @@ -128,3 +128,15 @@ return 0; } #endif + +RPY_EXTERN +intptr_t vmprof_get_traceback(void *stack, void *ucontext, + intptr_t *result_p, intptr_t result_length) +{ + int n; + intptr_t pc = ucontext ? GetPC((ucontext_t *)ucontext) : 0; + if (stack == NULL) + stack = get_vmprof_stack(); + n = get_stack_trace(stack, result_p, result_length - 2, pc); + return (intptr_t)n; +} diff --git a/rpython/rlib/rvmprof/test/test_traceback.py b/rpython/rlib/rvmprof/test/test_traceback.py new file mode 100644 --- /dev/null +++ b/rpython/rlib/rvmprof/test/test_traceback.py @@ -0,0 +1,113 @@ +import re +from rpython.rlib import rvmprof, jit +from rpython.rlib.rvmprof import traceback +from rpython.translator.interactive import Translation +from rpython.rtyper.lltypesystem import lltype + + +def test_direct(): + class MyCode: + pass + def get_name(mycode): + raise NotImplementedError + rvmprof.register_code_object_class(MyCode, get_name) + # + @rvmprof.vmprof_execute_code("mycode", lambda code, level: code, + _hack_update_stack_untranslated=True) + def mainloop(code, level): + if level > 0: + mainloop(code, level - 1) + else: + p, length = traceback.traceback(20) + traceback.walk_traceback(MyCode, my_callback, 42, p, length) + lltype.free(p, flavor='raw') + # + seen = [] + def my_callback(code, loc, arg): + seen.append((code, loc, arg)) + # + code1 = MyCode() + rvmprof.register_code(code1, "foo") + mainloop(code1, 2) + # + assert seen == [(code1, traceback.LOC_INTERPRETED, 42), + (code1, traceback.LOC_INTERPRETED, 42), + (code1, traceback.LOC_INTERPRETED, 42)] + +def test_compiled(): + class MyCode: + pass + def get_name(mycode): + raise NotImplementedError + rvmprof.register_code_object_class(MyCode, get_name) + + @rvmprof.vmprof_execute_code("mycode", lambda code, level: code) + def mainloop(code, level): + if level > 0: + mainloop(code, level - 1) + else: + p, length = traceback.traceback(20) + traceback.walk_traceback(MyCode, my_callback, 42, p, length) + lltype.free(p, flavor='raw') + + def my_callback(code, loc, arg): + print code, loc, arg + return 0 + + def f(argv): + code1 = MyCode() + rvmprof.register_code(code1, "foo") + mainloop(code1, 2) + return 0 + + t = Translation(f, None, gc="boehm") + t.compile_c() + stdout = t.driver.cbuilder.cmdexec('') + r = re.compile("[<]MyCode object at 0x([0-9a-f]+)[>] 0 42\n") + got = r.findall(stdout) + assert got == [got[0]] * 3 + +def test_jitted(): + class MyCode: + pass + def get_name(mycode): + raise NotImplementedError + rvmprof.register_code_object_class(MyCode, get_name) + + jitdriver = jit.JitDriver(greens=['code'], reds='auto', + is_recursive=True, + get_unique_id=lambda code: rvmprof.get_unique_id(code)) + + @rvmprof.vmprof_execute_code("mycode", lambda code, level, total_i: code) + def mainloop(code, level, total_i): + i = 20 + while i > 0: + jitdriver.jit_merge_point(code=code) + i -= 1 + if level > 0: + mainloop(code, level - 1, total_i + i) + if level == 0 and total_i == 0: + p, length = traceback.traceback(20) + traceback.walk_traceback(MyCode, my_callback, 42, p, length) + lltype.free(p, flavor='raw') + + def my_callback(code, loc, arg): + print code, loc, arg + return 0 + + def f(argv): + jit.set_param(jitdriver, "inlining", 0) + code1 = MyCode() + rvmprof.register_code(code1, "foo") + mainloop(code1, 2, 0) + return 0 + + t = Translation(f, None, gc="boehm") + t.rtype() + t.driver.pyjitpl_lltype() + t.compile_c() + stdout = t.driver.cbuilder.cmdexec('') + r = re.compile("[<]MyCode object at 0x([0-9a-f]+)[>] (\d) 42\n") + got = r.findall(stdout) + addr = got[0][0] + assert got == [(addr, '1'), (addr, '1'), (addr, '0')] diff --git a/rpython/rlib/rvmprof/traceback.py b/rpython/rlib/rvmprof/traceback.py new file mode 100644 --- /dev/null +++ b/rpython/rlib/rvmprof/traceback.py @@ -0,0 +1,65 @@ +""" +Semi-public interface to gather and print a raw traceback, e.g. +from the faulthandler module. +""" + +from rpython.rlib.rvmprof import cintf, rvmprof +from rpython.rlib.objectmodel import specialize +from rpython.rtyper.lltypesystem import lltype, llmemory, rffi + + +def traceback(estimate_number_of_entries): + """Build and return a vmprof-like traceback, as a pair (array_p, + array_length). The caller must free array_p. Not for signal handlers: + for these, call vmprof_get_traceback() from C code. + """ + _cintf = rvmprof._get_vmprof().cintf + size = estimate_number_of_entries * 2 + 4 + stack = cintf.get_rvmprof_stack() + array_p = lltype.malloc(rffi.SIGNEDP.TO, size, flavor='raw') + NULL = llmemory.NULL + array_length = _cintf.vmprof_get_traceback(stack, NULL, array_p, size) + return (array_p, array_length) + + +LOC_INTERPRETED = 0 +LOC_JITTED = 1 +LOC_JITTED_INLINED = 2 + + + at specialize.arg(0, 1) +def _traceback_one(CodeClass, callback, arg, code_id, loc): + found_code = None + if code_id != 0: + all_code_wrefs = CodeClass._vmprof_weak_list.get_all_handles() + i = len(all_code_wrefs) - 1 + while i >= 0: + code = all_code_wrefs[i]() + if code is not None and code._vmprof_unique_id == code_id: + found_code = code + break + i -= 1 + callback(found_code, loc, arg) + + at specialize.arg(0, 1) +def walk_traceback(CodeClass, callback, arg, array_p, array_length): + """Invoke 'callback(code_obj, loc, arg)' for every traceback entry. + 'code_obj' may be None if it can't be determined. 'loc' is one + of the LOC_xxx constants. + """ + i = 0 + while i < array_length - 1: + tag = array_p[i] + tagged_value = array_p[i + 1] + if tag == rvmprof.VMPROF_CODE_TAG: + loc = LOC_INTERPRETED + _traceback_one(CodeClass, callback, arg, tagged_value, loc) + elif tag == rvmprof.VMPROF_JITTED_TAG: + if i + 2 >= array_length: # skip last entry, can't determine if + break # it's LOC_JITTED_INLINED or LOC_JITTED + if array_p[i + 2] == rvmprof.VMPROF_JITTED_TAG: + loc = LOC_JITTED_INLINED + else: + loc = LOC_JITTED + _traceback_one(CodeClass, callback, arg, tagged_value, loc) + i += 2 diff --git a/rpython/translator/c/src/threadlocal.c b/rpython/translator/c/src/threadlocal.c --- a/rpython/translator/c/src/threadlocal.c +++ b/rpython/translator/c/src/threadlocal.c @@ -14,12 +14,22 @@ static int check_valid(void); +int _RPython_ThreadLocals_AcquireTimeout(int max_wait_iterations) { + while (1) { + long old_value = pypy_lock_test_and_set(&pypy_threadlocal_lock, 1); + if (old_value == 0) + break; + /* busy loop */ + if (max_wait_iterations == 0) + return -1; + if (max_wait_iterations > 0) + --max_wait_iterations; + } + assert(check_valid()); + return 0; +} void _RPython_ThreadLocals_Acquire(void) { - long old_value; - do { - old_value = pypy_lock_test_and_set(&pypy_threadlocal_lock, 1); - } while (old_value != 0); /* busy loop */ - assert(check_valid()); + _RPython_ThreadLocals_AcquireTimeout(-1); } void _RPython_ThreadLocals_Release(void) { assert(check_valid()); @@ -60,11 +70,7 @@ { /* assume that at most one pypy_threadlocal_s survived, the current one */ struct pypy_threadlocal_s *cur; -#ifdef USE___THREAD - cur = &pypy_threadlocal; -#else cur = (struct pypy_threadlocal_s *)_RPy_ThreadLocals_Get(); -#endif if (cur && cur->ready == 42) { cur->next = cur->prev = &linkedlist_head; linkedlist_head.next = linkedlist_head.prev = cur; diff --git a/rpython/translator/c/src/threadlocal.h b/rpython/translator/c/src/threadlocal.h --- a/rpython/translator/c/src/threadlocal.h +++ b/rpython/translator/c/src/threadlocal.h @@ -21,6 +21,7 @@ RPY_EXTERN void _RPython_ThreadLocals_Acquire(void); RPY_EXTERN void _RPython_ThreadLocals_Release(void); +RPY_EXTERN int _RPython_ThreadLocals_AcquireTimeout(int max_wait_iterations); /* Must acquire/release the thread-local lock around a series of calls to the following function */ @@ -48,14 +49,14 @@ #define OP_THREADLOCALREF_ADDR(r) \ do { \ - r = (char *)&pypy_threadlocal; \ + r = (void *)&pypy_threadlocal; \ if (pypy_threadlocal.ready != 42) \ r = _RPython_ThreadLocals_Build(); \ } while (0) #define _OP_THREADLOCALREF_ADDR_SIGHANDLER(r) \ do { \ - r = (char *)&pypy_threadlocal; \ + r = (void *)&pypy_threadlocal; \ if (pypy_threadlocal.ready != 42) \ r = NULL; \ } while (0) @@ -66,6 +67,8 @@ #define RPY_THREADLOCALREF_GET(FIELD) pypy_threadlocal.FIELD +#define _RPy_ThreadLocals_Get() (&pypy_threadlocal) + /* ------------------------------------------------------------ */ #else @@ -89,14 +92,14 @@ #define OP_THREADLOCALREF_ADDR(r) \ do { \ - r = (char *)_RPy_ThreadLocals_Get(); \ + r = (void *)_RPy_ThreadLocals_Get(); \ if (!r) \ r = _RPython_ThreadLocals_Build(); \ } while (0) #define _OP_THREADLOCALREF_ADDR_SIGHANDLER(r) \ do { \ - r = (char *)_RPy_ThreadLocals_Get(); \ + r = (void *)_RPy_ThreadLocals_Get(); \ } while (0) #define RPY_THREADLOCALREF_ENSURE() \ From pypy.commits at gmail.com Sat Oct 1 05:54:40 2016 From: pypy.commits at gmail.com (arigo) Date: Sat, 01 Oct 2016 02:54:40 -0700 (PDT) Subject: [pypy-commit] pypy faulthandler: fix Message-ID: <57ef87e0.6286c20a.ab23e.0cbc@mx.google.com> Author: Armin Rigo Branch: faulthandler Changeset: r87492:fbb4fa5a6d49 Date: 2016-10-01 11:17 +0200 http://bitbucket.org/pypy/pypy/changeset/fbb4fa5a6d49/ Log: fix diff --git a/pypy/module/faulthandler/handler.py b/pypy/module/faulthandler/handler.py --- a/pypy/module/faulthandler/handler.py +++ b/pypy/module/faulthandler/handler.py @@ -24,7 +24,7 @@ def check_err(self, p_err): if p_err: - raise oefmt(self.space.w_RuntimeError, 'faulthandler: %8', + raise oefmt(self.space.w_RuntimeError, 'faulthandler: %s', rffi.charp2str(p_err)) def get_fileno_and_file(self, w_file): From pypy.commits at gmail.com Sat Oct 1 05:59:10 2016 From: pypy.commits at gmail.com (arigo) Date: Sat, 01 Oct 2016 02:59:10 -0700 (PDT) Subject: [pypy-commit] pypy default: hg merge faulthandler Message-ID: <57ef88ee.c22e1c0a.331af.c0f2@mx.google.com> Author: Armin Rigo Branch: Changeset: r87493:931bb35a83bf Date: 2016-10-01 11:58 +0200 http://bitbucket.org/pypy/pypy/changeset/931bb35a83bf/ Log: hg merge faulthandler Port the 'faulthandler' module to PyPy default. This module is standard in Python 3.3 but can also be installed from CPython >= 2.6 from PyPI. diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py --- a/pypy/config/pypyoption.py +++ b/pypy/config/pypyoption.py @@ -43,6 +43,7 @@ try: if detect_cpu.autodetect().startswith('x86'): working_modules.add('_vmprof') + working_modules.add('faulthandler') except detect_cpu.ProcessorAutodetectError: pass @@ -89,6 +90,7 @@ ('objspace.usemodules.thread', True)], 'cpyext': [('objspace.usemodules.array', True)], 'cppyy': [('objspace.usemodules.cpyext', True)], + 'faulthandler': [('objspace.usemodules._vmprof', True)], } module_suggests = { # the reason you want _rawffi is for ctypes, which @@ -114,7 +116,8 @@ "_hashlib" : ["pypy.module._ssl.interp_ssl"], "_minimal_curses": ["pypy.module._minimal_curses.fficurses"], "_continuation": ["rpython.rlib.rstacklet"], - "_vmprof" : ["pypy.module._vmprof.interp_vmprof"], + "_vmprof" : ["pypy.module._vmprof.interp_vmprof"], + "faulthandler" : ["pypy.module._vmprof.interp_vmprof"], } def get_module_validator(modname): diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py --- a/pypy/interpreter/app_main.py +++ b/pypy/interpreter/app_main.py @@ -33,6 +33,7 @@ --info : print translation information about this PyPy executable -X track-resources : track the creation of files and sockets and display a warning if they are not closed explicitly +-X faulthandler : attempt to display tracebacks when PyPy crashes """ # Missing vs CPython: PYTHONHOME, PYTHONCASEOK USAGE2 = """ @@ -233,12 +234,22 @@ import pypyjit pypyjit.set_param(jitparam) +def run_faulthandler(): + if 'faulthandler' in sys.builtin_module_names: + import faulthandler + try: + faulthandler.enable(2) # manually set to stderr + except ValueError: + pass # ignore "2 is not a valid file descriptor" + def set_runtime_options(options, Xparam, *args): if Xparam == 'track-resources': sys.pypy_set_track_resources(True) + elif Xparam == 'faulthandler': + run_faulthandler() else: print >> sys.stderr, 'usage: %s -X [options]' % (get_sys_executable(),) - print >> sys.stderr, '[options] can be: track-resources' + print >> sys.stderr, '[options] can be: track-resources, faulthandler' raise SystemExit class CommandLineError(Exception): @@ -527,6 +538,9 @@ print >> sys.stderr, ( "Warning: pypy does not implement py3k warnings") + if os.getenv('PYTHONFAULTHANDLER'): + run_faulthandler() + ## if not we_are_translated(): ## for key in sorted(options): ## print '%40s: %s' % (key, options[key]) diff --git a/pypy/module/faulthandler/__init__.py b/pypy/module/faulthandler/__init__.py new file mode 100644 --- /dev/null +++ b/pypy/module/faulthandler/__init__.py @@ -0,0 +1,38 @@ +import sys +from pypy.interpreter.mixedmodule import MixedModule + + +class Module(MixedModule): + appleveldefs = { + } + + interpleveldefs = { + 'enable': 'handler.enable', + 'disable': 'handler.disable', + 'is_enabled': 'handler.is_enabled', +# 'register': 'interp_faulthandler.register', +# + 'dump_traceback': 'handler.dump_traceback', +# + '_read_null': 'handler.read_null', + '_sigsegv': 'handler.sigsegv', + '_sigfpe': 'handler.sigfpe', + '_sigabrt': 'handler.sigabrt', + '_stack_overflow': 'handler.stack_overflow', + } + + def setup_after_space_initialization(self): + """NOT_RPYTHON""" + if self.space.config.translation.thread: + self.extra_interpdef('dump_traceback_later', + 'handler.dump_traceback_later') + self.extra_interpdef('cancel_dump_traceback_later', + 'handler.cancel_dump_traceback_later') + if sys.platform != 'win32': + self.extra_interpdef('register', 'handler.register') + self.extra_interpdef('unregister', 'handler.unregister') + + def shutdown(self, space): + from pypy.module.faulthandler import handler + handler.finish(space) + MixedModule.shutdown(self, space) diff --git a/pypy/module/faulthandler/cintf.py b/pypy/module/faulthandler/cintf.py new file mode 100644 --- /dev/null +++ b/pypy/module/faulthandler/cintf.py @@ -0,0 +1,99 @@ +import py +from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, rstr +from rpython.translator import cdir +from rpython.translator.tool.cbuild import ExternalCompilationInfo + + +cwd = py.path.local(__file__).dirpath() +eci = ExternalCompilationInfo( + includes=[cwd.join('faulthandler.h')], + include_dirs=[str(cwd), cdir], + separate_module_files=[cwd.join('faulthandler.c')]) + +eci_later = eci.merge(ExternalCompilationInfo( + pre_include_bits=['#define PYPY_FAULTHANDLER_LATER\n'])) +eci_user = eci.merge(ExternalCompilationInfo( + pre_include_bits=['#define PYPY_FAULTHANDLER_USER\n'])) + +def direct_llexternal(*args, **kwargs): + kwargs.setdefault('_nowrapper', True) + kwargs.setdefault('compilation_info', eci) + return rffi.llexternal(*args, **kwargs) + + +DUMP_CALLBACK = lltype.Ptr(lltype.FuncType( + [rffi.INT, rffi.SIGNEDP, lltype.Signed], lltype.Void)) + +pypy_faulthandler_setup = direct_llexternal( + 'pypy_faulthandler_setup', [DUMP_CALLBACK], rffi.CCHARP) + +pypy_faulthandler_teardown = direct_llexternal( + 'pypy_faulthandler_teardown', [], lltype.Void) + +pypy_faulthandler_enable = direct_llexternal( + 'pypy_faulthandler_enable', [rffi.INT, rffi.INT], rffi.CCHARP) + +pypy_faulthandler_disable = direct_llexternal( + 'pypy_faulthandler_disable', [], lltype.Void) + +pypy_faulthandler_is_enabled = direct_llexternal( + 'pypy_faulthandler_is_enabled', [], rffi.INT) + +pypy_faulthandler_write = direct_llexternal( + 'pypy_faulthandler_write', [rffi.INT, rffi.CCHARP], lltype.Void) + +pypy_faulthandler_write_int = direct_llexternal( + 'pypy_faulthandler_write_int', [rffi.INT, lltype.Signed], lltype.Void) + +pypy_faulthandler_dump_traceback = direct_llexternal( + 'pypy_faulthandler_dump_traceback', + [rffi.INT, rffi.INT, llmemory.Address], lltype.Void) + +pypy_faulthandler_dump_traceback_later = direct_llexternal( + 'pypy_faulthandler_dump_traceback_later', + [rffi.LONGLONG, rffi.INT, rffi.INT, rffi.INT], rffi.CCHARP, + compilation_info=eci_later) + +pypy_faulthandler_cancel_dump_traceback_later = direct_llexternal( + 'pypy_faulthandler_cancel_dump_traceback_later', [], lltype.Void) + +pypy_faulthandler_check_signum = direct_llexternal( + 'pypy_faulthandler_check_signum', + [rffi.LONG], rffi.INT, + compilation_info=eci_user) + +pypy_faulthandler_register = direct_llexternal( + 'pypy_faulthandler_register', + [rffi.INT, rffi.INT, rffi.INT, rffi.INT], rffi.CCHARP, + compilation_info=eci_user) + +pypy_faulthandler_unregister = direct_llexternal( + 'pypy_faulthandler_unregister', + [rffi.INT], rffi.INT, + compilation_info=eci_user) + + +# for tests... + +pypy_faulthandler_read_null = direct_llexternal( + 'pypy_faulthandler_read_null', [], lltype.Void) + +pypy_faulthandler_read_null_releasegil = direct_llexternal( + 'pypy_faulthandler_read_null', [], lltype.Void, + _nowrapper=False, releasegil=True) + +pypy_faulthandler_sigsegv = direct_llexternal( + 'pypy_faulthandler_sigsegv', [], lltype.Void) + +pypy_faulthandler_sigsegv_releasegil = direct_llexternal( + 'pypy_faulthandler_sigsegv', [], lltype.Void, + _nowrapper=False, releasegil=True) + +pypy_faulthandler_sigfpe = direct_llexternal( + 'pypy_faulthandler_sigfpe', [], lltype.Void) + +pypy_faulthandler_sigabrt = direct_llexternal( + 'pypy_faulthandler_sigabrt', [], lltype.Void) + +pypy_faulthandler_stackoverflow = direct_llexternal( + 'pypy_faulthandler_stackoverflow', [lltype.Float], lltype.Float) diff --git a/pypy/module/faulthandler/dumper.py b/pypy/module/faulthandler/dumper.py new file mode 100644 --- /dev/null +++ b/pypy/module/faulthandler/dumper.py @@ -0,0 +1,54 @@ +from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.rlib import rgc +from rpython.rlib.rvmprof import traceback + +from pypy.interpreter.pycode import PyCode +from pypy.module.faulthandler.cintf import pypy_faulthandler_write +from pypy.module.faulthandler.cintf import pypy_faulthandler_write_int + + +MAX_STRING_LENGTH = 500 + +global_buf = lltype.malloc(rffi.CCHARP.TO, MAX_STRING_LENGTH, flavor='raw', + immortal=True, zero=True) + +def _dump(fd, s): + assert isinstance(s, str) + l = len(s) + if l >= MAX_STRING_LENGTH: + l = MAX_STRING_LENGTH - 1 + i = 0 + while i < l: + global_buf[i] = s[i] + i += 1 + global_buf[l] = '\x00' + pypy_faulthandler_write(fd, global_buf) + +def _dump_int(fd, i): + pypy_faulthandler_write_int(fd, i) + + +def dump_code(pycode, loc, fd): + if pycode is None: + _dump(fd, " File ???") + else: + _dump(fd, ' File "') + _dump(fd, pycode.co_filename) + _dump(fd, '" in ') + _dump(fd, pycode.co_name) + _dump(fd, ", from line ") + _dump_int(fd, pycode.co_firstlineno) + if loc == traceback.LOC_JITTED: + _dump(fd, " [jitted]") + elif loc == traceback.LOC_JITTED_INLINED: + _dump(fd, " [jit inlined]") + _dump(fd, "\n") + + + at rgc.no_collect +def _dump_callback(fd, array_p, array_length): + """We are as careful as we can reasonably be here (i.e. not 100%, + but hopefully close enough). In particular, this is written as + RPython but shouldn't allocate anything. + """ + traceback.walk_traceback(PyCode, dump_code, fd, array_p, array_length) diff --git a/pypy/module/faulthandler/faulthandler.c b/pypy/module/faulthandler/faulthandler.c new file mode 100644 --- /dev/null +++ b/pypy/module/faulthandler/faulthandler.c @@ -0,0 +1,679 @@ +#include "faulthandler.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef RPYTHON_LL2CTYPES +# include "../../../rpython/rlib/rvmprof/src/rvmprof.h" +#else +# include "common_header.h" +# include "structdef.h" +# include "rvmprof.h" +#endif +#include "src/threadlocal.h" + +#define MAX_FRAME_DEPTH 100 +#define FRAME_DEPTH_N RVMPROF_TRACEBACK_ESTIMATE_N(MAX_FRAME_DEPTH) + + +typedef struct sigaction _Py_sighandler_t; + +typedef struct { + const int signum; + volatile int enabled; + const char* name; + _Py_sighandler_t previous; +} fault_handler_t; + +static struct { + int initialized; + int enabled; + volatile int fd, all_threads; + volatile pypy_faulthandler_cb_t dump_traceback; +} fatal_error; + +static stack_t stack; + + +static fault_handler_t faulthandler_handlers[] = { +#ifdef SIGBUS + {SIGBUS, 0, "Bus error", }, +#endif +#ifdef SIGILL + {SIGILL, 0, "Illegal instruction", }, +#endif + {SIGFPE, 0, "Floating point exception", }, + {SIGABRT, 0, "Aborted", }, + /* define SIGSEGV at the end to make it the default choice if searching the + handler fails in faulthandler_fatal_error() */ + {SIGSEGV, 0, "Segmentation fault", } +}; +static const int faulthandler_nsignals = + sizeof(faulthandler_handlers) / sizeof(fault_handler_t); + +RPY_EXTERN +void pypy_faulthandler_write(int fd, const char *str) +{ + (void)write(fd, str, strlen(str)); +} + +RPY_EXTERN +void pypy_faulthandler_write_int(int fd, long value) +{ + char buf[48]; + sprintf(buf, "%ld", value); + pypy_faulthandler_write(fd, buf); +} + + +RPY_EXTERN +void pypy_faulthandler_dump_traceback(int fd, int all_threads, + void *ucontext) +{ + pypy_faulthandler_cb_t fn; + intptr_t array_p[FRAME_DEPTH_N], array_length; + + fn = fatal_error.dump_traceback; + if (!fn) + return; + +#ifndef RPYTHON_LL2CTYPES + if (all_threads && _RPython_ThreadLocals_AcquireTimeout(10000) == 0) { + /* This is known not to be perfectly safe against segfaults if we + don't hold the GIL ourselves. Too bad. I suspect that CPython + has issues there too. + */ + struct pypy_threadlocal_s *my, *p; + int blankline = 0; + char buf[40]; + + my = (struct pypy_threadlocal_s *)_RPy_ThreadLocals_Get(); + p = _RPython_ThreadLocals_Head(); + p = _RPython_ThreadLocals_Enum(p); + while (p != NULL) { + if (blankline) + pypy_faulthandler_write(fd, "\n"); + blankline = 1; + + pypy_faulthandler_write(fd, my == p ? "Current thread" : "Thread"); + sprintf(buf, " 0x%lx", (unsigned long)p->thread_ident); + pypy_faulthandler_write(fd, buf); + pypy_faulthandler_write(fd, " (most recent call first):\n"); + + array_length = vmprof_get_traceback(p->vmprof_tl_stack, + my == p ? ucontext : NULL, + array_p, FRAME_DEPTH_N); + fn(fd, array_p, array_length); + + p = _RPython_ThreadLocals_Enum(p); + } + _RPython_ThreadLocals_Release(); + } + else { + pypy_faulthandler_write(fd, "Stack (most recent call first):\n"); + array_length = vmprof_get_traceback(NULL, ucontext, + array_p, FRAME_DEPTH_N); + fn(fd, array_p, array_length); + } +#else + pypy_faulthandler_write(fd, "(no traceback when untranslated)\n"); +#endif +} + +static void +faulthandler_dump_traceback(int fd, int all_threads, void *ucontext) +{ + static volatile int reentrant = 0; + + if (reentrant) + return; + reentrant = 1; + pypy_faulthandler_dump_traceback(fd, all_threads, ucontext); + reentrant = 0; +} + + +/************************************************************/ + + +#ifdef PYPY_FAULTHANDLER_LATER +#include "src/thread.h" +static struct { + int fd; + long long microseconds; + int repeat, exit; + /* The main thread always holds this lock. It is only released when + faulthandler_thread() is interrupted before this thread exits, or at + Python exit. */ + struct RPyOpaque_ThreadLock cancel_event; + /* released by child thread when joined */ + struct RPyOpaque_ThreadLock running; +} thread_later; + +static void faulthandler_thread(void) +{ +#ifndef _WIN32 + /* we don't want to receive any signal */ + sigset_t set; + sigfillset(&set); + pthread_sigmask(SIG_SETMASK, &set, NULL); +#endif + + RPyLockStatus st; + char buf[64]; + unsigned long hour, minutes, seconds, fraction; + long long t; + + do { + st = RPyThreadAcquireLockTimed(&thread_later.cancel_event, + thread_later.microseconds, 0); + if (st == RPY_LOCK_ACQUIRED) { + RPyThreadReleaseLock(&thread_later.cancel_event); + break; + } + /* Timeout => dump traceback */ + assert(st == RPY_LOCK_FAILURE); + + /* getting to know which thread holds the GIL is not as simple + * as in CPython, so for now we don't */ + + t = thread_later.microseconds; + fraction = (unsigned long)(t % 1000000); + t /= 1000000; + seconds = (unsigned long)(t % 60); + t /= 60; + minutes = (unsigned long)(t % 60); + t /= 60; + hour = (unsigned long)t; + if (fraction == 0) + sprintf(buf, "Timeout (%lu:%02lu:%02lu)!\n", + hour, minutes, seconds); + else + sprintf(buf, "Timeout (%lu:%02lu:%02lu.%06lu)!\n", + hour, minutes, seconds, fraction); + + pypy_faulthandler_write(thread_later.fd, buf); + pypy_faulthandler_dump_traceback(thread_later.fd, 1, NULL); + + if (thread_later.exit) + _exit(1); + } while (thread_later.repeat); + + /* The only way out */ + RPyThreadReleaseLock(&thread_later.running); +} + +RPY_EXTERN +char *pypy_faulthandler_dump_traceback_later(long long microseconds, int repeat, + int fd, int exit) +{ + pypy_faulthandler_cancel_dump_traceback_later(); + + thread_later.fd = fd; + thread_later.microseconds = microseconds; + thread_later.repeat = repeat; + thread_later.exit = exit; + + RPyThreadAcquireLock(&thread_later.running, 1); + + if (RPyThreadStart(&faulthandler_thread) == -1) { + RPyThreadReleaseLock(&thread_later.running); + return "unable to start watchdog thread"; + } + return NULL; +} +#endif /* PYPY_FAULTHANDLER_LATER */ + +RPY_EXTERN +void pypy_faulthandler_cancel_dump_traceback_later(void) +{ +#ifdef PYPY_FAULTHANDLER_LATER + /* Notify cancellation */ + RPyThreadReleaseLock(&thread_later.cancel_event); + + /* Wait for thread to join (or does nothing if no thread is running) */ + RPyThreadAcquireLock(&thread_later.running, 1); + RPyThreadReleaseLock(&thread_later.running); + + /* The main thread should always hold the cancel_event lock */ + RPyThreadAcquireLock(&thread_later.cancel_event, 1); +#endif /* PYPY_FAULTHANDLER_LATER */ +} + + +/************************************************************/ + + +#ifdef PYPY_FAULTHANDLER_USER +typedef struct { + int enabled; + int fd; + int all_threads; + int chain; + _Py_sighandler_t previous; +} user_signal_t; + +static user_signal_t *user_signals; + +#ifndef NSIG +# if defined(_NSIG) +# define NSIG _NSIG /* For BSD/SysV */ +# elif defined(_SIGMAX) +# define NSIG (_SIGMAX + 1) /* For QNX */ +# elif defined(SIGMAX) +# define NSIG (SIGMAX + 1) /* For djgpp */ +# else +# define NSIG 64 /* Use a reasonable default value */ +# endif +#endif + +static void faulthandler_user(int signum, siginfo_t *info, void *ucontext); + +static int +faulthandler_register(int signum, int chain, _Py_sighandler_t *p_previous) +{ + struct sigaction action; + action.sa_handler = faulthandler_user; + sigemptyset(&action.sa_mask); + /* if the signal is received while the kernel is executing a system + call, try to restart the system call instead of interrupting it and + return EINTR. */ + action.sa_flags = SA_RESTART | SA_SIGINFO; + if (chain) { + /* do not prevent the signal from being received from within its + own signal handler */ + action.sa_flags = SA_NODEFER; + } + if (stack.ss_sp != NULL) { + /* Call the signal handler on an alternate signal stack + provided by sigaltstack() */ + action.sa_flags |= SA_ONSTACK; + } + return sigaction(signum, &action, p_previous); +} + +static void faulthandler_user(int signum, siginfo_t *info, void *ucontext) +{ + int save_errno; + user_signal_t *user = &user_signals[signum]; + + if (!user->enabled) + return; + + save_errno = errno; + faulthandler_dump_traceback(user->fd, user->all_threads, ucontext); + + if (user->chain) { + (void)sigaction(signum, &user->previous, NULL); + errno = save_errno; + + /* call the previous signal handler */ + raise(signum); + + save_errno = errno; + (void)faulthandler_register(signum, user->chain, NULL); + } + + errno = save_errno; +} + +RPY_EXTERN +int pypy_faulthandler_check_signum(long signum) +{ + unsigned int i; + + for (i = 0; i < faulthandler_nsignals; i++) { + if (faulthandler_handlers[i].signum == signum) { + return -1; + } + } + if (signum < 1 || NSIG <= signum) { + return -2; + } + return 0; +} + +RPY_EXTERN +char *pypy_faulthandler_register(int signum, int fd, int all_threads, int chain) +{ + user_signal_t *user; + _Py_sighandler_t previous; + int err; + + if (user_signals == NULL) { + user_signals = malloc(NSIG * sizeof(user_signal_t)); + if (user_signals == NULL) + return "out of memory"; + memset(user_signals, 0, NSIG * sizeof(user_signal_t)); + } + + user = &user_signals[signum]; + user->fd = fd; + user->all_threads = all_threads; + user->chain = chain; + + if (!user->enabled) { + err = faulthandler_register(signum, chain, &previous); + if (err) + return strerror(errno); + + user->previous = previous; + user->enabled = 1; + } + return NULL; +} + +RPY_EXTERN +int pypy_faulthandler_unregister(int signum) +{ + user_signal_t *user; + + if (user_signals == NULL) + return 0; + + user = &user_signals[signum]; + if (user->enabled) { + user->enabled = 0; + (void)sigaction(signum, &user->previous, NULL); + user->fd = -1; + return 1; + } + else + return 0; +} +#endif /* PYPY_FAULTHANDLER_USER */ + + +/************************************************************/ + + +/* Handler for SIGSEGV, SIGFPE, SIGABRT, SIGBUS and SIGILL signals. + + Display the current Python traceback, restore the previous handler and call + the previous handler. + + On Windows, don't explicitly call the previous handler, because the Windows + signal handler would not be called (for an unknown reason). The execution of + the program continues at faulthandler_fatal_error() exit, but the same + instruction will raise the same fault (signal), and so the previous handler + will be called. + + This function is signal-safe and should only call signal-safe functions. */ + +static void +faulthandler_fatal_error(int signum, siginfo_t *info, void *ucontext) +{ + int fd = fatal_error.fd; + int i; + fault_handler_t *handler = NULL; + int save_errno = errno; + + for (i = 0; i < faulthandler_nsignals; i++) { + handler = &faulthandler_handlers[i]; + if (handler->signum == signum) + break; + } + /* If not found, we use the SIGSEGV handler (the last one in the list) */ + + /* restore the previous handler */ + if (handler->enabled) { + (void)sigaction(signum, &handler->previous, NULL); + handler->enabled = 0; + } + + pypy_faulthandler_write(fd, "Fatal Python error: "); + pypy_faulthandler_write(fd, handler->name); + pypy_faulthandler_write(fd, "\n\n"); + + faulthandler_dump_traceback(fd, fatal_error.all_threads, ucontext); + + errno = save_errno; +#ifdef MS_WINDOWS + if (signum == SIGSEGV) { + /* don't explicitly call the previous handler for SIGSEGV in this signal + handler, because the Windows signal handler would not be called */ + return; + } +#endif + /* call the previous signal handler: it is called immediately if we use + sigaction() thanks to SA_NODEFER flag, otherwise it is deferred */ + raise(signum); +} + + +RPY_EXTERN +char *pypy_faulthandler_setup(pypy_faulthandler_cb_t dump_callback) +{ + if (fatal_error.initialized) + return NULL; + assert(!fatal_error.enabled); + fatal_error.dump_traceback = dump_callback; + + /* Try to allocate an alternate stack for faulthandler() signal handler to + * be able to allocate memory on the stack, even on a stack overflow. If it + * fails, ignore the error. */ + stack.ss_flags = 0; + stack.ss_size = SIGSTKSZ; + stack.ss_sp = malloc(stack.ss_size); + if (stack.ss_sp != NULL) { + int err = sigaltstack(&stack, NULL); + if (err) { + free(stack.ss_sp); + stack.ss_sp = NULL; + } + } + +#ifdef PYPY_FAULTHANDLER_LATER + if (!RPyThreadLockInit(&thread_later.cancel_event) || + !RPyThreadLockInit(&thread_later.running)) + return "failed to initialize locks"; + RPyThreadAcquireLock(&thread_later.cancel_event, 1); +#endif + + fatal_error.fd = -1; + fatal_error.initialized = 1; + + return NULL; +} + +RPY_EXTERN +void pypy_faulthandler_teardown(void) +{ + if (fatal_error.initialized) { + +#ifdef PYPY_FAULTHANDLER_LATER + pypy_faulthandler_cancel_dump_traceback_later(); + RPyThreadReleaseLock(&thread_later.cancel_event); + RPyOpaqueDealloc_ThreadLock(&thread_later.running); + RPyOpaqueDealloc_ThreadLock(&thread_later.cancel_event); +#endif + +#ifdef PYPY_FAULTHANDLER_USER + int signum; + for (signum = 0; signum < NSIG; signum++) + pypy_faulthandler_unregister(signum); + /* don't free 'user_signals', the gain is very minor and it can + lead to rare crashes if another thread is still busy */ +#endif + + pypy_faulthandler_disable(); + fatal_error.initialized = 0; + if (stack.ss_sp) { + stack.ss_flags = SS_DISABLE; + sigaltstack(&stack, NULL); + free(stack.ss_sp); + stack.ss_sp = NULL; + } + } +} + +RPY_EXTERN +char *pypy_faulthandler_enable(int fd, int all_threads) +{ + /* Install the handler for fatal signals, faulthandler_fatal_error(). */ + int i; + fatal_error.fd = fd; + fatal_error.all_threads = all_threads; + + if (!fatal_error.enabled) { + fatal_error.enabled = 1; + + for (i = 0; i < faulthandler_nsignals; i++) { + int err; + struct sigaction action; + fault_handler_t *handler = &faulthandler_handlers[i]; + + action.sa_sigaction = faulthandler_fatal_error; + sigemptyset(&action.sa_mask); + /* Do not prevent the signal from being received from within + its own signal handler */ + action.sa_flags = SA_NODEFER | SA_SIGINFO; + if (stack.ss_sp != NULL) { + /* Call the signal handler on an alternate signal stack + provided by sigaltstack() */ + action.sa_flags |= SA_ONSTACK; + } + err = sigaction(handler->signum, &action, &handler->previous); + if (err) { + return strerror(errno); + } + handler->enabled = 1; + } + } + return NULL; +} + +RPY_EXTERN +void pypy_faulthandler_disable(void) +{ + int i; + if (fatal_error.enabled) { + fatal_error.enabled = 0; + for (i = 0; i < faulthandler_nsignals; i++) { + fault_handler_t *handler = &faulthandler_handlers[i]; + if (!handler->enabled) + continue; + (void)sigaction(handler->signum, &handler->previous, NULL); + handler->enabled = 0; + } + } + fatal_error.fd = -1; +} + +RPY_EXTERN +int pypy_faulthandler_is_enabled(void) +{ + return fatal_error.enabled; +} + + +/************************************************************/ + + +/* for tests... */ + +static void +faulthandler_suppress_crash_report(void) +{ +#ifdef MS_WINDOWS + UINT mode; + + /* Configure Windows to not display the Windows Error Reporting dialog */ + mode = SetErrorMode(SEM_NOGPFAULTERRORBOX); + SetErrorMode(mode | SEM_NOGPFAULTERRORBOX); +#endif + +#ifndef MS_WINDOWS + struct rlimit rl; + + /* Disable creation of core dump */ + if (getrlimit(RLIMIT_CORE, &rl) != 0) { + rl.rlim_cur = 0; + setrlimit(RLIMIT_CORE, &rl); + } +#endif + +#ifdef _MSC_VER + /* Visual Studio: configure abort() to not display an error message nor + open a popup asking to report the fault. */ + _set_abort_behavior(0, _WRITE_ABORT_MSG | _CALL_REPORTFAULT); +#endif +} + +RPY_EXTERN +int pypy_faulthandler_read_null(void) +{ + int *volatile x; + + faulthandler_suppress_crash_report(); + x = NULL; + return *x; +} + +RPY_EXTERN +void pypy_faulthandler_sigsegv(void) +{ + faulthandler_suppress_crash_report(); +#if defined(MS_WINDOWS) + /* For SIGSEGV, faulthandler_fatal_error() restores the previous signal + handler and then gives back the execution flow to the program (without + explicitly calling the previous error handler). In a normal case, the + SIGSEGV was raised by the kernel because of a fault, and so if the + program retries to execute the same instruction, the fault will be + raised again. + + Here the fault is simulated by a fake SIGSEGV signal raised by the + application. We have to raise SIGSEGV at lease twice: once for + faulthandler_fatal_error(), and one more time for the previous signal + handler. */ + while(1) + raise(SIGSEGV); +#else + raise(SIGSEGV); +#endif +} + +RPY_EXTERN +int pypy_faulthandler_sigfpe(void) +{ + /* Do an integer division by zero: raise a SIGFPE on Intel CPU, but not on + PowerPC. Use volatile to disable compile-time optimizations. */ + volatile int x = 1, y = 0, z; + faulthandler_suppress_crash_report(); + z = x / y; + /* If the division by zero didn't raise a SIGFPE (e.g. on PowerPC), + raise it manually. */ + raise(SIGFPE); + /* This line is never reached, but we pretend to make something with z + to silence a compiler warning. */ + return z; +} + +RPY_EXTERN +void pypy_faulthandler_sigabrt(void) +{ + faulthandler_suppress_crash_report(); + abort(); +} + +static double fh_stack_overflow(double levels) +{ + if (levels > 2.5) { + return (sqrt(fh_stack_overflow(levels - 1.0)) + + fh_stack_overflow(levels * 1e-10)); + } + return 1e100 + levels; +} + +RPY_EXTERN +double pypy_faulthandler_stackoverflow(double levels) +{ + faulthandler_suppress_crash_report(); + return fh_stack_overflow(levels); +} diff --git a/pypy/module/faulthandler/faulthandler.h b/pypy/module/faulthandler/faulthandler.h new file mode 100644 --- /dev/null +++ b/pypy/module/faulthandler/faulthandler.h @@ -0,0 +1,40 @@ +#ifndef PYPY_FAULTHANDLER_H +#define PYPY_FAULTHANDLER_H + +#include "src/precommondefs.h" +#include + + +typedef void (*pypy_faulthandler_cb_t)(int fd, intptr_t *array_p, + intptr_t length); + +RPY_EXTERN char *pypy_faulthandler_setup(pypy_faulthandler_cb_t dump_callback); +RPY_EXTERN void pypy_faulthandler_teardown(void); + +RPY_EXTERN char *pypy_faulthandler_enable(int fd, int all_threads); +RPY_EXTERN void pypy_faulthandler_disable(void); +RPY_EXTERN int pypy_faulthandler_is_enabled(void); + +RPY_EXTERN void pypy_faulthandler_write(int fd, const char *str); +RPY_EXTERN void pypy_faulthandler_write_int(int fd, long value); + +RPY_EXTERN void pypy_faulthandler_dump_traceback(int fd, int all_threads, + void *ucontext); + +RPY_EXTERN char *pypy_faulthandler_dump_traceback_later( + long long microseconds, int repeat, int fd, int exit); +RPY_EXTERN void pypy_faulthandler_cancel_dump_traceback_later(void); + +RPY_EXTERN int pypy_faulthandler_check_signum(long signum); +RPY_EXTERN char *pypy_faulthandler_register(int, int, int, int); +RPY_EXTERN int pypy_faulthandler_unregister(int signum); + + +RPY_EXTERN int pypy_faulthandler_read_null(void); +RPY_EXTERN void pypy_faulthandler_sigsegv(void); +RPY_EXTERN int pypy_faulthandler_sigfpe(void); +RPY_EXTERN void pypy_faulthandler_sigabrt(void); +RPY_EXTERN double pypy_faulthandler_stackoverflow(double); + + +#endif /* PYPY_FAULTHANDLER_H */ diff --git a/pypy/module/faulthandler/handler.py b/pypy/module/faulthandler/handler.py new file mode 100644 --- /dev/null +++ b/pypy/module/faulthandler/handler.py @@ -0,0 +1,209 @@ +import os +from rpython.rtyper.lltypesystem import lltype, llmemory, rffi +from rpython.rlib.rposix import is_valid_fd +from rpython.rlib.rarithmetic import widen, ovfcheck_float_to_longlong +from rpython.rlib.objectmodel import keepalive_until_here +from rpython.rtyper.annlowlevel import llhelper + +from pypy.interpreter.error import OperationError, oefmt +from pypy.interpreter.error import exception_from_saved_errno +from pypy.interpreter.gateway import unwrap_spec +from pypy.module.faulthandler import cintf, dumper + + +class Handler(object): + def __init__(self, space): + "NOT_RPYTHON" + self.space = space + self._cleanup_() + + def _cleanup_(self): + self.fatal_error_w_file = None + self.dump_traceback_later_w_file = None + self.user_w_files = None + + def check_err(self, p_err): + if p_err: + raise oefmt(self.space.w_RuntimeError, 'faulthandler: %s', + rffi.charp2str(p_err)) + + def get_fileno_and_file(self, w_file): + space = self.space + if space.is_none(w_file): + w_file = space.sys.get('stderr') + if space.is_none(w_file): + raise oefmt(space.w_RuntimeError, "sys.stderr is None") + elif space.isinstance_w(w_file, space.w_int): + fd = space.c_int_w(w_file) + if fd < 0 or not is_valid_fd(fd): + raise oefmt(space.w_ValueError, + "file is not a valid file descriptor") + return fd, None + + fd = space.c_int_w(space.call_method(w_file, 'fileno')) + try: + space.call_method(w_file, 'flush') + except OperationError as e: + if e.async(space): + raise + pass # ignore flush() error + return fd, w_file + + def setup(self): + dump_callback = llhelper(cintf.DUMP_CALLBACK, dumper._dump_callback) + self.check_err(cintf.pypy_faulthandler_setup(dump_callback)) + + def enable(self, w_file, all_threads): + fileno, w_file = self.get_fileno_and_file(w_file) + self.setup() + self.fatal_error_w_file = w_file + self.check_err(cintf.pypy_faulthandler_enable( + rffi.cast(rffi.INT, fileno), + rffi.cast(rffi.INT, all_threads))) + + def disable(self): + cintf.pypy_faulthandler_disable() + self.fatal_error_w_file = None + + def is_enabled(self): + return bool(widen(cintf.pypy_faulthandler_is_enabled())) + + def dump_traceback(self, w_file, all_threads): + fileno, w_file = self.get_fileno_and_file(w_file) + self.setup() + cintf.pypy_faulthandler_dump_traceback( + rffi.cast(rffi.INT, fileno), + rffi.cast(rffi.INT, all_threads), + llmemory.NULL) + keepalive_until_here(w_file) + + def dump_traceback_later(self, timeout, repeat, w_file, exit): + space = self.space + timeout *= 1e6 + try: + microseconds = ovfcheck_float_to_longlong(timeout) + except OverflowError: + raise oefmt(space.w_OverflowError, "timeout value is too large") + if microseconds <= 0: + raise oefmt(space.w_ValueError, "timeout must be greater than 0") + fileno, w_file = self.get_fileno_and_file(w_file) + self.setup() + self.check_err(cintf.pypy_faulthandler_dump_traceback_later( + rffi.cast(rffi.LONGLONG, microseconds), + rffi.cast(rffi.INT, repeat), + rffi.cast(rffi.INT, fileno), + rffi.cast(rffi.INT, exit))) + self.dump_traceback_later_w_file = w_file + + def cancel_dump_traceback_later(self): + cintf.pypy_faulthandler_cancel_dump_traceback_later() + self.dump_traceback_later_w_file = None + + def check_signum(self, signum): + err = rffi.cast(lltype.Signed, + cintf.pypy_faulthandler_check_signum(signum)) + if err < 0: + space = self.space + if err == -1: + raise oefmt(space.w_RuntimeError, + "signal %d cannot be registered, " + "use enable() instead", signum) + else: + raise oefmt(space.w_ValueError, "signal number out of range") + + def register(self, signum, w_file, all_threads, chain): + self.check_signum(signum) + fileno, w_file = self.get_fileno_and_file(w_file) + self.setup() + self.check_err(cintf.pypy_faulthandler_register( + rffi.cast(rffi.INT, signum), + rffi.cast(rffi.INT, fileno), + rffi.cast(rffi.INT, all_threads), + rffi.cast(rffi.INT, chain))) + if self.user_w_files is None: + self.user_w_files = {} + self.user_w_files[signum] = w_file + + def unregister(self, signum): + self.check_signum(signum) + change = cintf.pypy_faulthandler_unregister( + rffi.cast(rffi.INT, signum)) + if self.user_w_files is not None: + self.user_w_files.pop(signum, None) + return rffi.cast(lltype.Signed, change) == 1 + + def finish(self): + cintf.pypy_faulthandler_teardown() + self._cleanup_() + + +def finish(space): + "Finalize the faulthandler logic (called from shutdown())" + space.fromcache(Handler).finish() + + + at unwrap_spec(all_threads=int) +def enable(space, w_file=None, all_threads=0): + "enable(file=sys.stderr, all_threads=True): enable the fault handler" + space.fromcache(Handler).enable(w_file, all_threads) + +def disable(space): + "disable(): disable the fault handler" + space.fromcache(Handler).disable() + +def is_enabled(space): + "is_enabled()->bool: check if the handler is enabled" + return space.wrap(space.fromcache(Handler).is_enabled()) + + at unwrap_spec(all_threads=int) +def dump_traceback(space, w_file=None, all_threads=0): + """dump the traceback of the current thread into file + including all threads if all_threads is True""" + space.fromcache(Handler).dump_traceback(w_file, all_threads) + + at unwrap_spec(timeout=float, repeat=int, exit=int) +def dump_traceback_later(space, timeout, repeat=0, w_file=None, exit=0): + """dump the traceback of all threads in timeout seconds, + or each timeout seconds if repeat is True. If exit is True, + call _exit(1) which is not safe.""" + space.fromcache(Handler).dump_traceback_later(timeout, repeat, w_file, exit) + +def cancel_dump_traceback_later(space): + """cancel the previous call to dump_traceback_later().""" + space.fromcache(Handler).cancel_dump_traceback_later() + + at unwrap_spec(signum=int, all_threads=int, chain=int) +def register(space, signum, w_file=None, all_threads=1, chain=0): + space.fromcache(Handler).register(signum, w_file, all_threads, chain) + + at unwrap_spec(signum=int) +def unregister(space, signum): + return space.wrap(space.fromcache(Handler).unregister(signum)) + + +# for tests... + + at unwrap_spec(release_gil=int) +def read_null(space, release_gil=0): + if release_gil: + cintf.pypy_faulthandler_read_null_releasegil() + else: + cintf.pypy_faulthandler_read_null() + + at unwrap_spec(release_gil=int) +def sigsegv(space, release_gil=0): + if release_gil: + cintf.pypy_faulthandler_sigsegv_releasegil() + else: + cintf.pypy_faulthandler_sigsegv() + +def sigfpe(space): + cintf.pypy_faulthandler_sigfpe() + +def sigabrt(space): + cintf.pypy_faulthandler_sigabrt() + + at unwrap_spec(levels=int) +def stack_overflow(space, levels=100000000): + levels = float(levels) + return space.wrap(cintf.pypy_faulthandler_stackoverflow(levels)) diff --git a/pypy/module/faulthandler/test/test_faulthander.py b/pypy/module/faulthandler/test/test_faulthander.py new file mode 100644 --- /dev/null +++ b/pypy/module/faulthandler/test/test_faulthander.py @@ -0,0 +1,18 @@ + +class AppTestFaultHandler: + spaceconfig = { + "usemodules": ["faulthandler", "_vmprof"] + } + + def test_enable(self): + import faulthandler, sys + faulthandler.enable() + assert faulthandler.is_enabled() is True + faulthandler.enable(file=sys.stderr, all_threads=True) + faulthandler.disable() + assert faulthandler.is_enabled() is False + + def test_dump_traceback(self): + import faulthandler, sys + faulthandler.dump_traceback() + faulthandler.dump_traceback(file=sys.stderr, all_threads=True) diff --git a/pypy/module/faulthandler/test/test_ztranslation.py b/pypy/module/faulthandler/test/test_ztranslation.py new file mode 100644 --- /dev/null +++ b/pypy/module/faulthandler/test/test_ztranslation.py @@ -0,0 +1,4 @@ +from pypy.objspace.fake.checkmodule import checkmodule + +def test_faulthandler_translates(): + checkmodule('faulthandler') diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py --- a/rpython/rlib/rvmprof/cintf.py +++ b/rpython/rlib/rvmprof/cintf.py @@ -56,6 +56,11 @@ [rffi.INT], lltype.Void, compilation_info=eci, _nowrapper=True) + vmprof_get_traceback = rffi.llexternal("vmprof_get_traceback", + [PVMPROFSTACK, llmemory.Address, + rffi.SIGNEDP, lltype.Signed], + lltype.Signed, compilation_info=eci, + _nowrapper=True) return CInterface(locals()) @@ -154,3 +159,9 @@ def restore_rvmprof_stack(x): vmprof_tl_stack.setraw(x) + +# +# traceback support + +def get_rvmprof_stack(): + return vmprof_tl_stack.get_or_make_raw() diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py --- a/rpython/rlib/rvmprof/rvmprof.py +++ b/rpython/rlib/rvmprof/rvmprof.py @@ -1,6 +1,6 @@ import sys, os from rpython.rlib.objectmodel import specialize, we_are_translated -from rpython.rlib import jit, rposix +from rpython.rlib import jit, rposix, rgc from rpython.rlib.rvmprof import cintf from rpython.rtyper.annlowlevel import cast_instance_to_gcref from rpython.rtyper.annlowlevel import cast_base_ptr_to_instance diff --git a/rpython/rlib/rvmprof/src/rvmprof.h b/rpython/rlib/rvmprof/src/rvmprof.h --- a/rpython/rlib/rvmprof/src/rvmprof.h +++ b/rpython/rlib/rvmprof/src/rvmprof.h @@ -1,3 +1,4 @@ +#include RPY_EXTERN char *vmprof_init(int, double, char *); RPY_EXTERN void vmprof_ignore_signals(int); @@ -8,3 +9,6 @@ RPY_EXTERN int vmprof_stack_append(void*, long); RPY_EXTERN long vmprof_stack_pop(void*); RPY_EXTERN void vmprof_stack_free(void*); +RPY_EXTERN intptr_t vmprof_get_traceback(void *, void *, intptr_t*, intptr_t); + +#define RVMPROF_TRACEBACK_ESTIMATE_N(num_entries) (2 * (num_entries) + 4) diff --git a/rpython/rlib/rvmprof/src/vmprof_common.h b/rpython/rlib/rvmprof/src/vmprof_common.h --- a/rpython/rlib/rvmprof/src/vmprof_common.h +++ b/rpython/rlib/rvmprof/src/vmprof_common.h @@ -128,3 +128,15 @@ return 0; } #endif + +RPY_EXTERN +intptr_t vmprof_get_traceback(void *stack, void *ucontext, + intptr_t *result_p, intptr_t result_length) +{ + int n; + intptr_t pc = ucontext ? GetPC((ucontext_t *)ucontext) : 0; + if (stack == NULL) + stack = get_vmprof_stack(); + n = get_stack_trace(stack, result_p, result_length - 2, pc); + return (intptr_t)n; +} diff --git a/rpython/rlib/rvmprof/test/test_traceback.py b/rpython/rlib/rvmprof/test/test_traceback.py new file mode 100644 --- /dev/null +++ b/rpython/rlib/rvmprof/test/test_traceback.py @@ -0,0 +1,113 @@ +import re +from rpython.rlib import rvmprof, jit +from rpython.rlib.rvmprof import traceback +from rpython.translator.interactive import Translation +from rpython.rtyper.lltypesystem import lltype + + +def test_direct(): + class MyCode: + pass + def get_name(mycode): + raise NotImplementedError + rvmprof.register_code_object_class(MyCode, get_name) + # + @rvmprof.vmprof_execute_code("mycode", lambda code, level: code, + _hack_update_stack_untranslated=True) + def mainloop(code, level): + if level > 0: + mainloop(code, level - 1) + else: + p, length = traceback.traceback(20) + traceback.walk_traceback(MyCode, my_callback, 42, p, length) + lltype.free(p, flavor='raw') + # + seen = [] + def my_callback(code, loc, arg): + seen.append((code, loc, arg)) + # + code1 = MyCode() + rvmprof.register_code(code1, "foo") + mainloop(code1, 2) + # + assert seen == [(code1, traceback.LOC_INTERPRETED, 42), + (code1, traceback.LOC_INTERPRETED, 42), + (code1, traceback.LOC_INTERPRETED, 42)] + +def test_compiled(): + class MyCode: + pass + def get_name(mycode): + raise NotImplementedError + rvmprof.register_code_object_class(MyCode, get_name) + + @rvmprof.vmprof_execute_code("mycode", lambda code, level: code) + def mainloop(code, level): + if level > 0: + mainloop(code, level - 1) + else: + p, length = traceback.traceback(20) + traceback.walk_traceback(MyCode, my_callback, 42, p, length) + lltype.free(p, flavor='raw') + + def my_callback(code, loc, arg): + print code, loc, arg + return 0 + + def f(argv): + code1 = MyCode() + rvmprof.register_code(code1, "foo") + mainloop(code1, 2) + return 0 + + t = Translation(f, None, gc="boehm") + t.compile_c() + stdout = t.driver.cbuilder.cmdexec('') + r = re.compile("[<]MyCode object at 0x([0-9a-f]+)[>] 0 42\n") + got = r.findall(stdout) + assert got == [got[0]] * 3 + +def test_jitted(): + class MyCode: + pass + def get_name(mycode): + raise NotImplementedError + rvmprof.register_code_object_class(MyCode, get_name) + + jitdriver = jit.JitDriver(greens=['code'], reds='auto', + is_recursive=True, + get_unique_id=lambda code: rvmprof.get_unique_id(code)) + + @rvmprof.vmprof_execute_code("mycode", lambda code, level, total_i: code) + def mainloop(code, level, total_i): + i = 20 + while i > 0: + jitdriver.jit_merge_point(code=code) + i -= 1 + if level > 0: + mainloop(code, level - 1, total_i + i) + if level == 0 and total_i == 0: + p, length = traceback.traceback(20) + traceback.walk_traceback(MyCode, my_callback, 42, p, length) + lltype.free(p, flavor='raw') + + def my_callback(code, loc, arg): + print code, loc, arg + return 0 + + def f(argv): + jit.set_param(jitdriver, "inlining", 0) + code1 = MyCode() + rvmprof.register_code(code1, "foo") + mainloop(code1, 2, 0) + return 0 + + t = Translation(f, None, gc="boehm") + t.rtype() + t.driver.pyjitpl_lltype() + t.compile_c() + stdout = t.driver.cbuilder.cmdexec('') + r = re.compile("[<]MyCode object at 0x([0-9a-f]+)[>] (\d) 42\n") + got = r.findall(stdout) + addr = got[0][0] + assert got == [(addr, '1'), (addr, '1'), (addr, '0')] diff --git a/rpython/rlib/rvmprof/traceback.py b/rpython/rlib/rvmprof/traceback.py new file mode 100644 --- /dev/null +++ b/rpython/rlib/rvmprof/traceback.py @@ -0,0 +1,65 @@ +""" +Semi-public interface to gather and print a raw traceback, e.g. +from the faulthandler module. +""" + +from rpython.rlib.rvmprof import cintf, rvmprof +from rpython.rlib.objectmodel import specialize +from rpython.rtyper.lltypesystem import lltype, llmemory, rffi + + +def traceback(estimate_number_of_entries): + """Build and return a vmprof-like traceback, as a pair (array_p, + array_length). The caller must free array_p. Not for signal handlers: + for these, call vmprof_get_traceback() from C code. + """ + _cintf = rvmprof._get_vmprof().cintf + size = estimate_number_of_entries * 2 + 4 + stack = cintf.get_rvmprof_stack() + array_p = lltype.malloc(rffi.SIGNEDP.TO, size, flavor='raw') + NULL = llmemory.NULL + array_length = _cintf.vmprof_get_traceback(stack, NULL, array_p, size) + return (array_p, array_length) + + +LOC_INTERPRETED = 0 +LOC_JITTED = 1 +LOC_JITTED_INLINED = 2 + + + at specialize.arg(0, 1) +def _traceback_one(CodeClass, callback, arg, code_id, loc): + found_code = None + if code_id != 0: + all_code_wrefs = CodeClass._vmprof_weak_list.get_all_handles() + i = len(all_code_wrefs) - 1 + while i >= 0: + code = all_code_wrefs[i]() + if code is not None and code._vmprof_unique_id == code_id: + found_code = code + break + i -= 1 + callback(found_code, loc, arg) + + at specialize.arg(0, 1) +def walk_traceback(CodeClass, callback, arg, array_p, array_length): + """Invoke 'callback(code_obj, loc, arg)' for every traceback entry. + 'code_obj' may be None if it can't be determined. 'loc' is one + of the LOC_xxx constants. + """ + i = 0 + while i < array_length - 1: + tag = array_p[i] + tagged_value = array_p[i + 1] + if tag == rvmprof.VMPROF_CODE_TAG: + loc = LOC_INTERPRETED + _traceback_one(CodeClass, callback, arg, tagged_value, loc) + elif tag == rvmprof.VMPROF_JITTED_TAG: + if i + 2 >= array_length: # skip last entry, can't determine if + break # it's LOC_JITTED_INLINED or LOC_JITTED + if array_p[i + 2] == rvmprof.VMPROF_JITTED_TAG: + loc = LOC_JITTED_INLINED + else: + loc = LOC_JITTED + _traceback_one(CodeClass, callback, arg, tagged_value, loc) + i += 2 diff --git a/rpython/translator/c/src/threadlocal.c b/rpython/translator/c/src/threadlocal.c --- a/rpython/translator/c/src/threadlocal.c +++ b/rpython/translator/c/src/threadlocal.c @@ -14,12 +14,22 @@ static int check_valid(void); +int _RPython_ThreadLocals_AcquireTimeout(int max_wait_iterations) { + while (1) { + long old_value = pypy_lock_test_and_set(&pypy_threadlocal_lock, 1); + if (old_value == 0) + break; + /* busy loop */ + if (max_wait_iterations == 0) + return -1; + if (max_wait_iterations > 0) + --max_wait_iterations; + } + assert(check_valid()); + return 0; +} void _RPython_ThreadLocals_Acquire(void) { - long old_value; - do { - old_value = pypy_lock_test_and_set(&pypy_threadlocal_lock, 1); - } while (old_value != 0); /* busy loop */ - assert(check_valid()); + _RPython_ThreadLocals_AcquireTimeout(-1); } void _RPython_ThreadLocals_Release(void) { assert(check_valid()); @@ -60,11 +70,7 @@ { /* assume that at most one pypy_threadlocal_s survived, the current one */ struct pypy_threadlocal_s *cur; -#ifdef USE___THREAD - cur = &pypy_threadlocal; -#else cur = (struct pypy_threadlocal_s *)_RPy_ThreadLocals_Get(); -#endif if (cur && cur->ready == 42) { cur->next = cur->prev = &linkedlist_head; linkedlist_head.next = linkedlist_head.prev = cur; diff --git a/rpython/translator/c/src/threadlocal.h b/rpython/translator/c/src/threadlocal.h --- a/rpython/translator/c/src/threadlocal.h +++ b/rpython/translator/c/src/threadlocal.h @@ -21,6 +21,7 @@ RPY_EXTERN void _RPython_ThreadLocals_Acquire(void); RPY_EXTERN void _RPython_ThreadLocals_Release(void); +RPY_EXTERN int _RPython_ThreadLocals_AcquireTimeout(int max_wait_iterations); /* Must acquire/release the thread-local lock around a series of calls to the following function */ @@ -48,14 +49,14 @@ #define OP_THREADLOCALREF_ADDR(r) \ do { \ - r = (char *)&pypy_threadlocal; \ + r = (void *)&pypy_threadlocal; \ if (pypy_threadlocal.ready != 42) \ r = _RPython_ThreadLocals_Build(); \ } while (0) #define _OP_THREADLOCALREF_ADDR_SIGHANDLER(r) \ do { \ - r = (char *)&pypy_threadlocal; \ + r = (void *)&pypy_threadlocal; \ if (pypy_threadlocal.ready != 42) \ r = NULL; \ } while (0) @@ -66,6 +67,8 @@ #define RPY_THREADLOCALREF_GET(FIELD) pypy_threadlocal.FIELD +#define _RPy_ThreadLocals_Get() (&pypy_threadlocal) + /* ------------------------------------------------------------ */ #else @@ -89,14 +92,14 @@ #define OP_THREADLOCALREF_ADDR(r) \ do { \ - r = (char *)_RPy_ThreadLocals_Get(); \ + r = (void *)_RPy_ThreadLocals_Get(); \ if (!r) \ r = _RPython_ThreadLocals_Build(); \ } while (0) #define _OP_THREADLOCALREF_ADDR_SIGHANDLER(r) \ do { \ - r = (char *)_RPy_ThreadLocals_Get(); \ + r = (void *)_RPy_ThreadLocals_Get(); \ } while (0) #define RPY_THREADLOCALREF_ENSURE() \ From pypy.commits at gmail.com Sat Oct 1 06:21:04 2016 From: pypy.commits at gmail.com (arigo) Date: Sat, 01 Oct 2016 03:21:04 -0700 (PDT) Subject: [pypy-commit] pypy py3k-faulthandler: hg merge py3k Message-ID: <57ef8e10.442d1c0a.ac30d.cb92@mx.google.com> Author: Armin Rigo Branch: py3k-faulthandler Changeset: r87494:62275cf72b32 Date: 2016-10-01 12:15 +0200 http://bitbucket.org/pypy/pypy/changeset/62275cf72b32/ Log: hg merge py3k diff too long, truncating to 2000 out of 3262 lines diff --git a/Makefile b/Makefile --- a/Makefile +++ b/Makefile @@ -40,4 +40,4 @@ # http://lists.gnu.org/archive/html/help-make/2010-08/msg00106.html cffi_imports: pypy-c - PYTHONPATH=. ./pypy-c pypy/tool/build_cffi_imports.py + PYTHONPATH=. ./pypy-c pypy/tool/build_cffi_imports.py || /bin/true diff --git a/lib-python/3/imp.py b/lib-python/3/imp.py --- a/lib-python/3/imp.py +++ b/lib-python/3/imp.py @@ -73,7 +73,7 @@ elif os.path.isdir(path): raise ImportError('existing directory', path=path) - def find_module(self, fullname, path=None): + def find_module(self, fullname): """Always returns None.""" return None diff --git a/lib_pypy/_pypy_winbase_build.py b/lib_pypy/_pypy_winbase_build.py --- a/lib_pypy/_pypy_winbase_build.py +++ b/lib_pypy/_pypy_winbase_build.py @@ -76,9 +76,9 @@ HANDLE WINAPI GetCurrentProcess(void); BOOL WINAPI DuplicateHandle(HANDLE, HANDLE, HANDLE, LPHANDLE, DWORD, BOOL, DWORD); -BOOL WINAPI CreateProcessA(char *, char *, void *, - void *, BOOL, DWORD, char *, - char *, LPSTARTUPINFO, LPPROCESS_INFORMATION); +BOOL WINAPI CreateProcessW(wchar_t *, wchar_t *, void *, + void *, BOOL, DWORD, wchar_t *, + wchar_t *, LPSTARTUPINFO, LPPROCESS_INFORMATION); DWORD WINAPI WaitForSingleObject(HANDLE, DWORD); BOOL WINAPI GetExitCodeProcess(HANDLE, LPDWORD); BOOL WINAPI TerminateProcess(HANDLE, UINT); diff --git a/lib_pypy/_pypy_winbase_cffi.py b/lib_pypy/_pypy_winbase_cffi.py --- a/lib_pypy/_pypy_winbase_cffi.py +++ b/lib_pypy/_pypy_winbase_cffi.py @@ -3,8 +3,8 @@ ffi = _cffi_backend.FFI('_pypy_winbase_cffi', _version = 0x2601, - _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x50\x03\x00\x00\x13\x11\x00\x00\x53\x03\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x13\x11\x00\x00\x13\x11\x00\x00\x4F\x03\x00\x00\x4E\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x03\x00\x00\x1F\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x18\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x1F\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x18\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x02\x0F\x00\x00\x42\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\x42\x0D\x00\x00\x00\x0F\x00\x00\x42\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x02\x0F\x00\x00\x00\x09\x00\x00\x01\x09\x00\x00\x02\x01\x00\x00\x52\x03\x00\x00\x04\x01\x00\x00\x00\x01', - _globals = (b'\x00\x00\x24\x23CloseHandle',0,b'\x00\x00\x1E\x23CreatePipe',0,b'\x00\x00\x12\x23CreateProcessA',0,b'\x00\x00\x2F\x23DuplicateHandle',0,b'\x00\x00\x4C\x23GetCurrentProcess',0,b'\x00\x00\x2B\x23GetExitCodeProcess',0,b'\x00\x00\x49\x23GetStdHandle',0,b'\x00\x00\x3F\x23GetVersion',0,b'\x00\x00\x27\x23TerminateProcess',0,b'\x00\x00\x3B\x23WaitForSingleObject',0,b'\x00\x00\x38\x23_get_osfhandle',0,b'\x00\x00\x10\x23_getch',0,b'\x00\x00\x10\x23_getche',0,b'\x00\x00\x44\x23_getwch',0,b'\x00\x00\x44\x23_getwche',0,b'\x00\x00\x10\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\x46\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\x41\x23_ungetwch',0), - _struct_unions = ((b'\x00\x00\x00\x4E\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x15\x11hProcess',b'\x00\x00\x15\x11hThread',b'\x00\x00\x18\x11dwProcessId',b'\x00\x00\x18\x11dwThreadId'),(b'\x00\x00\x00\x4F\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x18\x11cb',b'\x00\x00\x13\x11lpReserved',b'\x00\x00\x13\x11lpDesktop',b'\x00\x00\x13\x11lpTitle',b'\x00\x00\x18\x11dwX',b'\x00\x00\x18\x11dwY',b'\x00\x00\x18\x11dwXSize',b'\x00\x00\x18\x11dwYSize',b'\x00\x00\x18\x11dwXCountChars',b'\x00\x00\x18\x11dwYCountChars',b'\x00\x00\x18\x11dwFillAttribute',b'\x00\x00\x18\x11dwFlags',b'\x00\x00\x42\x11wShowWindow',b'\x00\x00\x42\x11cbReserved2',b'\x00\x00\x51\x11lpReserved2',b'\x00\x00\x15\x11hStdInput',b'\x00\x00\x15\x11hStdOutput',b'\x00\x00\x15\x11hStdError')), - _typenames = (b'\x00\x00\x00\x1CLPPROCESS_INFORMATION',b'\x00\x00\x00\x1BLPSTARTUPINFO',b'\x00\x00\x00\x4EPROCESS_INFORMATION',b'\x00\x00\x00\x4FSTARTUPINFO',b'\x00\x00\x00\x42wint_t'), + _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x15\x03\x00\x00\x13\x11\x00\x00\x59\x03\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x16\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x13\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x4C\x03\x00\x00\x2D\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x2D\x11\x00\x00\x2D\x11\x00\x00\x54\x03\x00\x00\x53\x03\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x16\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x16\x0D\x00\x00\x15\x11\x00\x00\x2D\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x16\x0D\x00\x00\x02\x0F\x00\x00\x47\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\x47\x0D\x00\x00\x00\x0F\x00\x00\x47\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x02\x0F\x00\x00\x00\x09\x00\x00\x01\x09\x00\x00\x56\x03\x00\x00\x02\x01\x00\x00\x58\x03\x00\x00\x04\x01\x00\x00\x00\x01', + _globals = (b'\x00\x00\x18\x23CloseHandle',0,b'\x00\x00\x12\x23CreatePipe',0,b'\x00\x00\x2C\x23CreateProcessW',0,b'\x00\x00\x23\x23DuplicateHandle',0,b'\x00\x00\x51\x23GetCurrentProcess',0,b'\x00\x00\x1F\x23GetExitCodeProcess',0,b'\x00\x00\x3F\x23GetModuleFileNameW',0,b'\x00\x00\x4E\x23GetStdHandle',0,b'\x00\x00\x44\x23GetVersion',0,b'\x00\x00\x1B\x23TerminateProcess',0,b'\x00\x00\x3B\x23WaitForSingleObject',0,b'\x00\x00\x38\x23_get_osfhandle',0,b'\x00\x00\x10\x23_getch',0,b'\x00\x00\x10\x23_getche',0,b'\x00\x00\x49\x23_getwch',0,b'\x00\x00\x49\x23_getwche',0,b'\x00\x00\x10\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\x4B\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\x46\x23_ungetwch',0), + _struct_unions = ((b'\x00\x00\x00\x53\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x15\x11hProcess',b'\x00\x00\x15\x11hThread',b'\x00\x00\x16\x11dwProcessId',b'\x00\x00\x16\x11dwThreadId'),(b'\x00\x00\x00\x54\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x16\x11cb',b'\x00\x00\x55\x11lpReserved',b'\x00\x00\x55\x11lpDesktop',b'\x00\x00\x55\x11lpTitle',b'\x00\x00\x16\x11dwX',b'\x00\x00\x16\x11dwY',b'\x00\x00\x16\x11dwXSize',b'\x00\x00\x16\x11dwYSize',b'\x00\x00\x16\x11dwXCountChars',b'\x00\x00\x16\x11dwYCountChars',b'\x00\x00\x16\x11dwFillAttribute',b'\x00\x00\x16\x11dwFlags',b'\x00\x00\x47\x11wShowWindow',b'\x00\x00\x47\x11cbReserved2',b'\x00\x00\x57\x11lpReserved2',b'\x00\x00\x15\x11hStdInput',b'\x00\x00\x15\x11hStdOutput',b'\x00\x00\x15\x11hStdError')), + _typenames = (b'\x00\x00\x00\x36LPPROCESS_INFORMATION',b'\x00\x00\x00\x35LPSTARTUPINFO',b'\x00\x00\x00\x53PROCESS_INFORMATION',b'\x00\x00\x00\x54STARTUPINFO',b'\x00\x00\x00\x47wint_t'), ) diff --git a/lib_pypy/_winapi.py b/lib_pypy/_winapi.py --- a/lib_pypy/_winapi.py +++ b/lib_pypy/_winapi.py @@ -22,35 +22,13 @@ code, message = _ffi.getwinerror() raise WindowsError(code, message) -_INVALID_HANDLE_VALUE = _ffi.cast("HANDLE", -1) +def _int2handle(val): + return _ffi.cast("HANDLE", val) -class _handle(object): - def __init__(self, c_handle): - # 'c_handle' is a cffi cdata of type HANDLE, which is basically 'void *' - self.c_handle = c_handle - if int(self) != -1: - self.c_handle = _ffi.gc(self.c_handle, _kernel32.CloseHandle) +def _handle2int(handle): + return int(_ffi.cast("intptr_t", handle)) - def __int__(self): - return int(_ffi.cast("intptr_t", self.c_handle)) - - def __repr__(self): - return '<_subprocess.handle %d at 0x%x>' % (int(self), id(self)) - - def Detach(self): - h = int(self) - if h != -1: - c_handle = self.c_handle - self.c_handle = _INVALID_HANDLE_VALUE - _ffi.gc(c_handle, None) - return h - - def Close(self): - if int(self) != -1: - c_handle = self.c_handle - self.c_handle = _INVALID_HANDLE_VALUE - _ffi.gc(c_handle, None) - _kernel32.CloseHandle(c_handle) +_INVALID_HANDLE_VALUE = _int2handle(-1) def CreatePipe(attributes, size): handles = _ffi.new("HANDLE[2]") @@ -60,32 +38,32 @@ if not res: raise _WinError() - return _handle(handles[0]), _handle(handles[1]) + return _handle2int(handles[0]), _handle2int(handles[1]) def GetCurrentProcess(): - return _handle(_kernel32.GetCurrentProcess()) + return _handle2int(_kernel32.GetCurrentProcess()) def DuplicateHandle(source_process, source, target_process, access, inherit, options=0): # CPython: the first three arguments are expected to be integers target = _ffi.new("HANDLE[1]") res = _kernel32.DuplicateHandle( - _ffi.cast("HANDLE", source_process), - _ffi.cast("HANDLE", source), - _ffi.cast("HANDLE", target_process), + _int2handle(source_process), + _int2handle(source), + _int2handle(target_process), target, access, inherit, options) if not res: raise _WinError() - return _handle(target[0]) + return _handle2int(target[0]) -def _z(input): +def _Z(input): if input is None: return _ffi.NULL - if isinstance(input, basestring): - return str(input) - raise TypeError("string/unicode/None expected, got %r" % ( + if isinstance(input, str): + return input + raise TypeError("str or None expected, got %r" % ( type(input).__name__,)) def CreateProcess(name, command_line, process_attr, thread_attr, @@ -94,13 +72,14 @@ if startup_info is not None: si.dwFlags = startup_info.dwFlags si.wShowWindow = startup_info.wShowWindow - # CPython: these three handles are expected to be _handle objects + # CPython: these three handles are expected to be + # subprocess.Handle (int) objects if startup_info.hStdInput: - si.hStdInput = startup_info.hStdInput.c_handle + si.hStdInput = _int2handle(startup_info.hStdInput) if startup_info.hStdOutput: - si.hStdOutput = startup_info.hStdOutput.c_handle + si.hStdOutput = _int2handle(startup_info.hStdOutput) if startup_info.hStdError: - si.hStdError = startup_info.hStdError.c_handle + si.hStdError = _int2handle(startup_info.hStdError) pi = _ffi.new("PROCESS_INFORMATION *") flags |= CREATE_UNICODE_ENVIRONMENT @@ -113,19 +92,21 @@ else: envbuf = _ffi.NULL - res = _kernel32.CreateProcessA(_z(name), _z(command_line), _ffi.NULL, + res = _kernel32.CreateProcessW(_Z(name), _Z(command_line), _ffi.NULL, _ffi.NULL, inherit, flags, envbuf, - _z(start_dir), si, pi) + _Z(start_dir), si, pi) if not res: raise _WinError() - return _handle(pi.hProcess), _handle(pi.hThread), pi.dwProcessId, pi.dwThreadId + return (_handle2int(pi.hProcess), + _handle2int(pi.hThread), + pi.dwProcessId, + pi.dwThreadId) def WaitForSingleObject(handle, milliseconds): # CPython: the first argument is expected to be an integer. - res = _kernel32.WaitForSingleObject(_ffi.cast("HANDLE", handle), - milliseconds) + res = _kernel32.WaitForSingleObject(_int2handle(handle), milliseconds) if res < 0: raise _WinError() @@ -135,7 +116,7 @@ # CPython: the first argument is expected to be an integer. code = _ffi.new("DWORD[1]") - res = _kernel32.GetExitCodeProcess(_ffi.cast("HANDLE", handle), code) + res = _kernel32.GetExitCodeProcess(_int2handle(handle), code) if not res: raise _WinError() @@ -145,7 +126,7 @@ def TerminateProcess(handle, exitcode): # CPython: the first argument is expected to be an integer. # The second argument is silently wrapped in a UINT. - res = _kernel32.TerminateProcess(_ffi.cast("HANDLE", handle), + res = _kernel32.TerminateProcess(_int2handle(handle), _ffi.cast("UINT", exitcode)) if not res: @@ -158,19 +139,17 @@ if not res: return None else: - # note: returns integer, not handle object - return int(_ffi.cast("intptr_t", res)) + return _handle2int(res) def CloseHandle(handle): - res = _kernel32.CloseHandle(_ffi.cast("HANDLE", handle)) + res = _kernel32.CloseHandle(_int2handle(handle)) if not res: raise _WinError() def GetModuleFileName(module): buf = _ffi.new("wchar_t[]", _MAX_PATH) - res = _kernel32.GetModuleFileNameW(_ffi.cast("HANDLE", module), - buf, _MAX_PATH) + res = _kernel32.GetModuleFileNameW(_int2handle(module), buf, _MAX_PATH) if not res: raise _WinError() diff --git a/lib_pypy/stackless.py b/lib_pypy/stackless.py --- a/lib_pypy/stackless.py +++ b/lib_pypy/stackless.py @@ -535,7 +535,7 @@ _main_tasklet = coroutine.getcurrent() _main_tasklet.__class__ = tasklet # XXX HAAAAAAAAAAAAAAAAAAAAACK _last_task = _main_tasklet - tasklet._init.__func__(_main_tasklet, label='main') + tasklet._init(_main_tasklet, label='main') _squeue = deque() _scheduler_append(_main_tasklet) diff --git a/pypy/doc/embedding.rst b/pypy/doc/embedding.rst --- a/pypy/doc/embedding.rst +++ b/pypy/doc/embedding.rst @@ -34,9 +34,11 @@ This function searches the PyPy standard library starting from the given "PyPy home directory". The arguments are: - * ``home``: NULL terminated path to an executable inside the pypy directory + * ``home``: path to an executable inside the pypy directory (can be a .so name, can be made up). Used to look up the standard - library, and is also set as ``sys.executable``. + library, and is also set as ``sys.executable``. From PyPy 5.5, you can + just say NULL here, as long as the ``libpypy-c.so/dylib/dll`` is itself + inside this directory. * ``verbose``: if non-zero, it will print error messages to stderr @@ -82,18 +84,14 @@ Note that this API is a lot more minimal than say CPython C API, so at first it's obvious to think that you can't do much. However, the trick is to do -all the logic in Python and expose it via `cffi`_ callbacks. Let's assume -we're on linux and pypy is installed in ``/opt/pypy`` (with -subdirectories like ``lib-python`` and ``lib_pypy``), and with the -library in ``/opt/pypy/bin/libpypy-c.so``. (It doesn't need to be -installed; you can also replace these paths with a local extract of the -installation tarballs, or with your local checkout of pypy.) We write a -little C program: +all the logic in Python and expose it via `cffi`_ callbacks. +We write a little C program: .. code-block:: c #include "PyPy.h" #include + #include static char source[] = "print 'hello from pypy'"; @@ -102,9 +100,9 @@ int res; rpython_startup_code(); - /* note: in the path /opt/pypy/x, the final x is ignored and - replaced with lib-python and lib_pypy. */ - res = pypy_setup_home("/opt/pypy/x", 1); + /* Before PyPy 5.5, you may need to say e.g. "/opt/pypy/bin" instead + * of NULL. */ + res = pypy_setup_home(NULL, 1); if (res) { printf("Error setting pypy home!\n"); return 1; @@ -123,11 +121,6 @@ $ LD_LIBRARY_PATH=/opt/pypy/bin ./x hello from pypy -.. note:: If the compilation fails because of missing PyPy.h header file, - you are running PyPy <= 2.2.1. Get it here__. - -.. __: https://bitbucket.org/pypy/pypy/raw/c4cd6eca9358066571500ac82aaacfdaa3889e8c/include/PyPy.h - On OSX it is necessary to set the rpath of the binary if one wants to link to it, with a command like:: @@ -181,6 +174,7 @@ /* C example */ #include "PyPy.h" #include + #include struct API { double (*add_numbers)(double x, double y); @@ -196,7 +190,7 @@ int res; rpython_startup_code(); - res = pypy_setup_home("/opt/pypy/x", 1); + res = pypy_setup_home(NULL, 1); if (res) { fprintf(stderr, "Error setting pypy home!\n"); return -1; @@ -237,6 +231,8 @@ Finding pypy_home ----------------- +**You can usually skip this section if you are running PyPy >= 5.5.** + The function pypy_setup_home() takes as first parameter the path to a file from which it can deduce the location of the standard library. More precisely, it tries to remove final components until it finds diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -27,3 +27,28 @@ JIT residual calls: if the called function starts with a fast-path like "if x.foo != 0: return x.foo", then inline the check before doing the CALL. For now, string hashing is about the only case. + +.. branch: search-path-from-libpypy + +The compiled pypy now looks for its lib-python/lib_pypy path starting +from the location of the *libpypy-c* instead of the executable. This is +arguably more consistent, and also it is what occurs anyway if you're +embedding pypy. Linux distribution packagers, take note! At a minimum, +the ``libpypy-c.so`` must really be inside the path containing +``lib-python`` and ``lib_pypy``. Of course, you can put a symlink to it +from somewhere else. You no longer have to do the same with the +``pypy`` executable, as long as it finds its ``libpypy-c.so`` library. + +.. branch: _warning + +CPython allows warning.warn(('something', 1), Warning), on PyPy this +produced a "expected a readable buffer object" error. Test and fix. + +.. branch: stricter-strip + +CPython rejects 'a'.strip(buffer(' ')); only None, str or unicode are +allowed as arguments. Test and fix for str and unicode + +.. branch: test-cpyext + +Refactor cpyext testing to be more pypy3-friendly. diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py --- a/pypy/goal/targetpypystandalone.py +++ b/pypy/goal/targetpypystandalone.py @@ -96,17 +96,18 @@ def pypy_setup_home(ll_home, verbose): from pypy.module.sys.initpath import pypy_find_stdlib verbose = rffi.cast(lltype.Signed, verbose) - if ll_home: + if ll_home and ord(ll_home[0]): home1 = rffi.charp2str(ll_home) home = os.path.join(home1, 'x') # <- so that 'll_home' can be # directly the root directory else: - home = home1 = pypydir + home1 = "pypy's shared library location" + home = '*' w_path = pypy_find_stdlib(space, home) if space.is_none(w_path): if verbose: debug("pypy_setup_home: directories 'lib-python' and 'lib_pypy'" - " not found in '%s' or in any parent directory" % home1) + " not found in %s or in any parent directory" % home1) return rffi.cast(rffi.INT, 1) space.startup() space.appexec([w_path], """(path): @@ -306,37 +307,20 @@ # HACKHACKHACK # ugly hack to modify target goal from compile_* to build_cffi_imports # this should probably get cleaned up and merged with driver.create_exe + from rpython.tool.runsubprocess import run_subprocess from rpython.translator.driver import taskdef import types - class Options(object): - pass - - - def mkexename(name): - if sys.platform == 'win32': - name = name.new(ext='exe') - return name - compile_goal, = driver.backend_select_goals(['compile']) @taskdef([compile_goal], "Create cffi bindings for modules") def task_build_cffi_imports(self): - from pypy.tool.build_cffi_imports import create_cffi_import_libraries ''' Use cffi to compile cffi interfaces to modules''' - exename = mkexename(driver.compute_exe_name()) - basedir = exename - while not basedir.join('include').exists(): - _basedir = basedir.dirpath() - if _basedir == basedir: - raise ValueError('interpreter %s not inside pypy repo', - str(exename)) - basedir = _basedir - modules = self.config.objspace.usemodules.getpaths() - options = Options() - # XXX possibly adapt options using modules - failures = create_cffi_import_libraries(exename, options, basedir) - # if failures, they were already printed - print >> sys.stderr, str(exename),'successfully built (errors, if any, while building the above modules are ignored)' + filename = os.path.join(pypydir, 'tool', 'build_cffi_imports.py') + status, out, err = run_subprocess(str(driver.compute_exe_name()), + [filename]) + sys.stdout.write(out) + sys.stderr.write(err) + # otherwise, ignore errors driver.task_build_cffi_imports = types.MethodType(task_build_cffi_imports, driver) driver.tasks['build_cffi_imports'] = driver.task_build_cffi_imports, [compile_goal] driver.default_goal = 'build_cffi_imports' diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -379,12 +379,12 @@ class BufferInterfaceNotFound(Exception): pass + at specialize.memo() def wrappable_class_name(Class): try: return Class.typedef.name except AttributeError: return 'internal subclass of %s' % (Class.__name__,) -wrappable_class_name._annspecialcase_ = 'specialize:memo' class CannotHaveLock(Exception): """Raised by space.allocate_lock() if we're translating.""" @@ -558,16 +558,6 @@ 'parser', 'fcntl', '_codecs', 'binascii' ] - # These modules are treated like CPython treats built-in modules, - # i.e. they always shadow any xx.py. The other modules are treated - # like CPython treats extension modules, and are loaded in sys.path - # order by the fake entry '.../lib_pypy/__extensions__'. - MODULES_THAT_ALWAYS_SHADOW = dict.fromkeys([ - '__builtin__', '__pypy__', '_ast', '_codecs', '_sre', '_warnings', - '_weakref', 'errno', '__exceptions__', 'gc', 'imp', 'marshal', - 'posix', 'nt', 'pwd', 'signal', 'sys', 'thread', 'zipimport', - ], None) - def make_builtins(self): "NOT_RPYTHON: only for initializing the space." @@ -846,12 +836,13 @@ u = s.decode('utf-8') return self.interned_strings.get(u) is not None + @specialize.arg(1) def descr_self_interp_w(self, RequiredClass, w_obj): if not isinstance(w_obj, RequiredClass): raise DescrMismatch() return w_obj - descr_self_interp_w._annspecialcase_ = 'specialize:arg(1)' + @specialize.arg(1) def interp_w(self, RequiredClass, w_obj, can_be_None=False): """ Unwrap w_obj, checking that it is an instance of the required internal @@ -866,7 +857,6 @@ wrappable_class_name(RequiredClass), w_obj.getclass(self)) return w_obj - interp_w._annspecialcase_ = 'specialize:arg(1)' def unpackiterable(self, w_iterable, expected_length=-1): """Unpack an iterable into a real (interpreter-level) list. @@ -1280,6 +1270,7 @@ self.setitem(w_globals, w_key, self.wrap(self.builtin)) return statement.exec_code(self, w_globals, w_locals) + @specialize.arg(2) def appexec(self, posargs_w, source): """ return value from executing given source at applevel. EXPERIMENTAL. The source must look like @@ -1291,7 +1282,6 @@ w_func = self.fromcache(AppExecCache).getorbuild(source) args = Arguments(self, list(posargs_w)) return self.call_args(w_func, args) - appexec._annspecialcase_ = 'specialize:arg(2)' def _next_or_none(self, w_it): try: @@ -1301,6 +1291,7 @@ raise return None + @specialize.arg(3) def compare_by_iteration(self, w_iterable1, w_iterable2, op): w_it1 = self.iter(w_iterable1) w_it2 = self.iter(w_iterable2) @@ -1323,7 +1314,6 @@ if op == 'gt': return self.gt(w_x1, w_x2) if op == 'ge': return self.ge(w_x1, w_x2) assert False, "bad value for op" - compare_by_iteration._annspecialcase_ = 'specialize:arg(3)' def decode_index(self, w_index_or_slice, seqlength): """Helper for custom sequence implementations diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py --- a/pypy/interpreter/error.py +++ b/pypy/interpreter/error.py @@ -582,3 +582,16 @@ if module: space.setattr(w_exc, space.wrap("__module__"), space.wrap(module)) return w_exc + +def new_import_error(space, w_msg, w_name, w_path): + """Create a new instance of ImportError. + + The result corresponds to ImportError(msg, name=name, path=path) + """ + return space.appexec( + [w_msg, w_name, w_path], """(msg, name, path): + return ImportError(msg, name=name, path=path)""") + +def raise_import_error(space, w_msg, w_name, w_path): + w_exc = new_import_error(space, w_msg, w_name, w_path) + raise OperationError(space.w_ImportError, w_exc) diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py --- a/pypy/interpreter/executioncontext.py +++ b/pypy/interpreter/executioncontext.py @@ -1,7 +1,7 @@ import sys from pypy.interpreter.error import OperationError, get_cleared_operation_error from rpython.rlib.unroll import unrolling_iterable -from rpython.rlib import jit, rgc +from rpython.rlib import jit, rgc, objectmodel TICK_COUNTER_STEP = 100 @@ -130,6 +130,7 @@ if self.gettrace() is not None: self._trace(frame, 'return', w_retval) + @objectmodel.always_inline def bytecode_trace(self, frame, decr_by=TICK_COUNTER_STEP): "Trace function called before each bytecode." # this is split into a fast path and a slower path that is @@ -138,7 +139,6 @@ actionflag = self.space.actionflag if actionflag.decrement_ticker(decr_by) < 0: actionflag.action_dispatcher(self, frame) # slow path - bytecode_trace._always_inline_ = True def _run_finalizers_now(self): # Tests only: run the actions now, to ensure that the @@ -146,6 +146,7 @@ # pypy.tool.pytest.apptest. self.space.actionflag.action_dispatcher(self, None) + @objectmodel.always_inline def bytecode_only_trace(self, frame): """ Like bytecode_trace() but doesn't invoke any other events besides the @@ -155,7 +156,6 @@ self.gettrace() is None): return self.run_trace_func(frame) - bytecode_only_trace._always_inline_ = True @jit.unroll_safe def run_trace_func(self, frame): @@ -202,13 +202,13 @@ d.instr_prev_plus_one = frame.last_instr + 1 + @objectmodel.try_inline def bytecode_trace_after_exception(self, frame): "Like bytecode_trace(), but without increasing the ticker." actionflag = self.space.actionflag self.bytecode_only_trace(frame) if actionflag.get_ticker() < 0: actionflag.action_dispatcher(self, frame) # slow path - bytecode_trace_after_exception._always_inline_ = 'try' # NB. this function is not inlined right now. backendopt.inline would # need some improvements to handle this case, but it's not really an # issue @@ -452,6 +452,7 @@ periodic_actions = unrolling_iterable(self._periodic_actions) @jit.unroll_safe + @objectmodel.dont_inline def action_dispatcher(ec, frame): # periodic actions (first reset the bytecode counter) self.reset_ticker(self.checkinterval_scaled) @@ -473,7 +474,6 @@ action._fired = False action.perform(ec, frame) - action_dispatcher._dont_inline_ = True self.action_dispatcher = action_dispatcher diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py --- a/pypy/interpreter/pycode.py +++ b/pypy/interpreter/pycode.py @@ -106,6 +106,7 @@ self.co_varnames = varnames self.co_freevars = freevars self.co_cellvars = cellvars + assert isinstance(filename, str) rstring.check_str0(filename) self.co_filename = filename self.co_name = name diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -6,7 +6,8 @@ from rpython.rlib import jit, rstackovf, rstring from rpython.rlib.debug import check_nonneg -from rpython.rlib.objectmodel import we_are_translated +from rpython.rlib.objectmodel import (we_are_translated, always_inline, + dont_inline) from rpython.rlib.rarithmetic import r_uint, intmask from rpython.tool.sourcetools import func_with_new_name @@ -453,20 +454,20 @@ # of oparg failed to produce an integer which is annotated as non-neg check_nonneg(oparg) + @always_inline def LOAD_FAST(self, varindex, next_instr): # access a local variable directly w_value = self.locals_cells_stack_w[varindex] if w_value is None: self._load_fast_failed(varindex) self.pushvalue(w_value) - LOAD_FAST._always_inline_ = True + @dont_inline def _load_fast_failed(self, varindex): varname = self.getlocalvarname(varindex) raise oefmt(self.space.w_UnboundLocalError, "local variable '%s' referenced before assignment", varname) - _load_fast_failed._dont_inline_ = True def LOAD_CONST(self, constindex, next_instr): w_const = self.getconstant_w(constindex) @@ -847,26 +848,26 @@ "name %R is not defined", w_varname) self.pushvalue(w_value) + @always_inline def _load_global(self, varname): w_value = self.space.finditem_str(self.get_w_globals(), varname) if w_value is None: # not in the globals, now look in the built-ins w_value = self.get_builtin().getdictvalue(self.space, varname) return w_value - _load_global._always_inline_ = True + @dont_inline def _load_global_failed(self, w_varname): raise oefmt(self.space.w_NameError, "global name %R is not defined", w_varname) - _load_global_failed._dont_inline_ = True + @always_inline def LOAD_GLOBAL(self, nameindex, next_instr): w_varname = self.getname_w(nameindex) w_value = self._load_global(self.space.identifier_w(w_varname)) if w_value is None: self._load_global_failed(w_varname) self.pushvalue(w_value) - LOAD_GLOBAL._always_inline_ = True def DELETE_FAST(self, varindex, next_instr): if self.locals_cells_stack_w[varindex] is None: @@ -900,6 +901,7 @@ self.pushvalue(space.newlist([], sizehint=length_hint)) self.pushvalue(last_val) + @always_inline def LOAD_ATTR(self, nameindex, next_instr): "obj.attributename" w_obj = self.popvalue() @@ -910,7 +912,6 @@ w_attributename = self.getname_w(nameindex) w_value = self.space.getattr(w_obj, w_attributename) self.pushvalue(w_value) - LOAD_ATTR._always_inline_ = True @jit.unroll_safe def cmp_exc_match(self, w_1, w_2): diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py --- a/pypy/interpreter/test/test_app_main.py +++ b/pypy/interpreter/test/test_app_main.py @@ -17,8 +17,7 @@ def get_python3(): if PYTHON3: return PYTHON3 - import py.test - py.test.fail("Test requires %r (not found in PATH) or a PYTHON3 " + py.test.skip("Test requires %r (not found in PATH) or a PYTHON3 " "environment variable set" % (LOOK_FOR_PYTHON3,)) _counter = 0 @@ -51,7 +50,8 @@ # return relative path for testing purposes return py.path.local().bestrelpath(pdir) -def pytest_funcarg__demo_script(request): + at py.test.fixture +def demo_script(): return getscript(""" print('hello') print('Name:', __name__) @@ -64,7 +64,8 @@ myvalue = 6*7 """) -def pytest_funcarg__crashing_demo_script(request): + at py.test.fixture +def crashing_demo_script(): return getscript(""" print('Hello2') myvalue2 = 11 @@ -1089,23 +1090,32 @@ old_sys_path = sys.path[:] old_cwd = os.getcwd() - sys.path.append(self.goal_dir) # make sure cwd does not contain a stdlib if self.tmp_dir.startswith(self.trunkdir): skip('TMPDIR is inside the PyPy source') - os.chdir(self.tmp_dir) + sys.path.append(self.goal_dir) tmp_pypy_c = os.path.join(self.tmp_dir, 'pypy-c') try: + os.chdir(self.tmp_dir) + + # If we are running PyPy with a libpypy-c, the following + # lines find the stdlib anyway. Otherwise, it is not found. + expected_found = ( + getattr(sys, 'pypy_translation_info', {}) + .get('translation.shared')) + import app_main - app_main.setup_bootstrap_path(tmp_pypy_c) # stdlib not found + app_main.setup_bootstrap_path(tmp_pypy_c) assert sys.executable == '' - assert sys.path == old_sys_path + [self.goal_dir] + if not expected_found: + assert sys.path == old_sys_path + [self.goal_dir] app_main.setup_bootstrap_path(self.fake_exe) if not sys.platform == 'win32': # an existing file is always 'executable' on windows assert sys.executable == '' # not executable! - assert sys.path == old_sys_path + [self.goal_dir] + if not expected_found: + assert sys.path == old_sys_path + [self.goal_dir] os.chmod(self.fake_exe, 0o755) app_main.setup_bootstrap_path(self.fake_exe) @@ -1116,7 +1126,8 @@ if newpath[0].endswith('__extensions__'): newpath = newpath[1:] # we get at least 'expected_path', and maybe more (e.g.plat-linux2) - assert newpath[:len(self.expected_path)] == self.expected_path + if not expected_found: + assert newpath[:len(self.expected_path)] == self.expected_path finally: sys.path[:] = old_sys_path os.chdir(old_cwd) diff --git a/pypy/interpreter/test/test_error.py b/pypy/interpreter/test/test_error.py --- a/pypy/interpreter/test/test_error.py +++ b/pypy/interpreter/test/test_error.py @@ -3,7 +3,7 @@ import py, os, errno from pypy.interpreter.error import ( OperationError, decompose_valuefmt, get_operrcls2, new_exception_class, - oefmt, wrap_oserror) + oefmt, wrap_oserror, new_import_error) def test_decompose_valuefmt(): @@ -154,3 +154,8 @@ assert operr.match(space, space.w_ValueError) assert operr.match(space, space.w_TypeError) +def test_import_error(space): + w_exc = new_import_error( + space, space.wrap(u'msg'), space.wrap(u'name'), space.wrap(u'path')) + assert space.getattr(w_exc, space.wrap(u'name')).unwrap(space) == u'name' + assert space.getattr(w_exc, space.wrap(u'path')).unwrap(space) == u'path' diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py --- a/pypy/interpreter/typedef.py +++ b/pypy/interpreter/typedef.py @@ -109,6 +109,7 @@ # we need two subclasses of the app-level type, one to add mapdict, and then one # to add del to not slow down the GC. + at specialize.memo() def get_unique_interplevel_subclass(space, cls): "NOT_RPYTHON: initialization-time only" assert cls.typedef.acceptable_as_base_class @@ -119,7 +120,6 @@ assert cls not in _unique_subclass_cache _unique_subclass_cache[cls] = subcls return subcls -get_unique_interplevel_subclass._annspecialcase_ = "specialize:memo" _unique_subclass_cache = {} def _getusercls(cls, reallywantdict=False): diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py --- a/pypy/module/__pypy__/__init__.py +++ b/pypy/module/__pypy__/__init__.py @@ -2,6 +2,8 @@ from pypy.interpreter.mixedmodule import MixedModule from pypy.module.imp.importing import get_pyc_magic +from rpython.rlib import rtime + class BuildersModule(MixedModule): appleveldefs = {} @@ -14,16 +16,11 @@ class TimeModule(MixedModule): appleveldefs = {} interpleveldefs = {} - if sys.platform.startswith("linux") or 'bsd' in sys.platform: - from pypy.module.__pypy__ import interp_time + if rtime.HAS_CLOCK_GETTIME: interpleveldefs["clock_gettime"] = "interp_time.clock_gettime" interpleveldefs["clock_getres"] = "interp_time.clock_getres" - for name in [ - "CLOCK_REALTIME", "CLOCK_MONOTONIC", "CLOCK_MONOTONIC_RAW", - "CLOCK_PROCESS_CPUTIME_ID", "CLOCK_THREAD_CPUTIME_ID" - ]: - if getattr(interp_time, name) is not None: - interpleveldefs[name] = "space.wrap(interp_time.%s)" % name + for name in rtime.ALL_DEFINED_CLOCKS: + interpleveldefs[name] = "space.wrap(%d)" % getattr(rtime, name) class ThreadModule(MixedModule): diff --git a/pypy/module/__pypy__/interp_time.py b/pypy/module/__pypy__/interp_time.py --- a/pypy/module/__pypy__/interp_time.py +++ b/pypy/module/__pypy__/interp_time.py @@ -4,71 +4,28 @@ from pypy.interpreter.error import exception_from_saved_errno from pypy.interpreter.gateway import unwrap_spec from rpython.rtyper.lltypesystem import rffi, lltype -from rpython.rtyper.tool import rffi_platform -from rpython.translator.tool.cbuild import ExternalCompilationInfo +from rpython.rlib import rtime +from rpython.rlib.rtime import HAS_CLOCK_GETTIME -if sys.platform == 'linux2': - libraries = ["rt"] -else: - libraries = [] - - -class CConfig: - _compilation_info_ = ExternalCompilationInfo( - includes=["time.h"], - libraries=libraries, - ) - - HAS_CLOCK_GETTIME = rffi_platform.Has('clock_gettime') - - CLOCK_REALTIME = rffi_platform.DefinedConstantInteger("CLOCK_REALTIME") - CLOCK_MONOTONIC = rffi_platform.DefinedConstantInteger("CLOCK_MONOTONIC") - CLOCK_MONOTONIC_RAW = rffi_platform.DefinedConstantInteger("CLOCK_MONOTONIC_RAW") - CLOCK_PROCESS_CPUTIME_ID = rffi_platform.DefinedConstantInteger("CLOCK_PROCESS_CPUTIME_ID") - CLOCK_THREAD_CPUTIME_ID = rffi_platform.DefinedConstantInteger("CLOCK_THREAD_CPUTIME_ID") - -cconfig = rffi_platform.configure(CConfig) - -HAS_CLOCK_GETTIME = cconfig["HAS_CLOCK_GETTIME"] - -CLOCK_REALTIME = cconfig["CLOCK_REALTIME"] -CLOCK_MONOTONIC = cconfig["CLOCK_MONOTONIC"] -CLOCK_MONOTONIC_RAW = cconfig["CLOCK_MONOTONIC_RAW"] -CLOCK_PROCESS_CPUTIME_ID = cconfig["CLOCK_PROCESS_CPUTIME_ID"] -CLOCK_THREAD_CPUTIME_ID = cconfig["CLOCK_THREAD_CPUTIME_ID"] if HAS_CLOCK_GETTIME: - #redo it for timespec - CConfig.TIMESPEC = rffi_platform.Struct("struct timespec", [ - ("tv_sec", rffi.TIME_T), - ("tv_nsec", rffi.LONG), - ]) - cconfig = rffi_platform.configure(CConfig) - TIMESPEC = cconfig['TIMESPEC'] - - c_clock_gettime = rffi.llexternal("clock_gettime", - [lltype.Signed, lltype.Ptr(TIMESPEC)], rffi.INT, - compilation_info=CConfig._compilation_info_, releasegil=False, - save_err=rffi.RFFI_SAVE_ERRNO - ) - c_clock_getres = rffi.llexternal("clock_getres", - [lltype.Signed, lltype.Ptr(TIMESPEC)], rffi.INT, - compilation_info=CConfig._compilation_info_, releasegil=False, - save_err=rffi.RFFI_SAVE_ERRNO - ) @unwrap_spec(clk_id="c_int") def clock_gettime(space, clk_id): - with lltype.scoped_alloc(TIMESPEC) as tp: - ret = c_clock_gettime(clk_id, tp) + with lltype.scoped_alloc(rtime.TIMESPEC) as tp: + ret = rtime.c_clock_gettime(clk_id, tp) if ret != 0: raise exception_from_saved_errno(space, space.w_IOError) - return space.wrap(int(tp.c_tv_sec) + 1e-9 * int(tp.c_tv_nsec)) + t = (float(rffi.getintfield(tp, 'c_tv_sec')) + + float(rffi.getintfield(tp, 'c_tv_nsec')) * 0.000000001) + return space.wrap(t) @unwrap_spec(clk_id="c_int") def clock_getres(space, clk_id): - with lltype.scoped_alloc(TIMESPEC) as tp: - ret = c_clock_getres(clk_id, tp) + with lltype.scoped_alloc(rtime.TIMESPEC) as tp: + ret = rtime.c_clock_getres(clk_id, tp) if ret != 0: raise exception_from_saved_errno(space, space.w_IOError) - return space.wrap(int(tp.c_tv_sec) + 1e-9 * int(tp.c_tv_nsec)) + t = (float(rffi.getintfield(tp, 'c_tv_sec')) + + float(rffi.getintfield(tp, 'c_tv_nsec')) * 0.000000001) + return space.wrap(t) diff --git a/pypy/module/_cffi_backend/embedding.py b/pypy/module/_cffi_backend/embedding.py --- a/pypy/module/_cffi_backend/embedding.py +++ b/pypy/module/_cffi_backend/embedding.py @@ -112,29 +112,7 @@ #define _WIN32_WINNT 0x0501 #include -#define CFFI_INIT_HOME_PATH_MAX _MAX_PATH static void _cffi_init(void); -static void _cffi_init_error(const char *msg, const char *extra); - -static int _cffi_init_home(char *output_home_path) -{ - HMODULE hModule = 0; - DWORD res; - - GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | - GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, - (LPCTSTR)&_cffi_init, &hModule); - - if (hModule == 0 ) { - _cffi_init_error("GetModuleHandleEx() failed", ""); - return -1; - } - res = GetModuleFileName(hModule, output_home_path, CFFI_INIT_HOME_PATH_MAX); - if (res >= CFFI_INIT_HOME_PATH_MAX) { - return -1; - } - return 0; -} static void _cffi_init_once(void) { @@ -155,28 +133,9 @@ else: do_includes = r""" -#include #include -#define CFFI_INIT_HOME_PATH_MAX PATH_MAX static void _cffi_init(void); -static void _cffi_init_error(const char *msg, const char *extra); - -static int _cffi_init_home(char *output_home_path) -{ - Dl_info info; - dlerror(); /* reset */ - if (dladdr(&_cffi_init, &info) == 0) { - _cffi_init_error("dladdr() failed: ", dlerror()); - return -1; - } - if (realpath(info.dli_fname, output_home_path) == NULL) { - perror("realpath() failed"); - _cffi_init_error("realpath() failed", ""); - return -1; - } - return 0; -} static void _cffi_init_once(void) { @@ -201,14 +160,10 @@ static void _cffi_init(void) { - char home[CFFI_INIT_HOME_PATH_MAX + 1]; - rpython_startup_code(); RPyGilAllocate(); - if (_cffi_init_home(home) != 0) - return; - if (pypy_setup_home(home, 1) != 0) { + if (pypy_setup_home(NULL, 1) != 0) { _cffi_init_error("pypy_setup_home() failed", ""); return; } diff --git a/pypy/module/_collections/interp_deque.py b/pypy/module/_collections/interp_deque.py --- a/pypy/module/_collections/interp_deque.py +++ b/pypy/module/_collections/interp_deque.py @@ -6,6 +6,7 @@ from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.error import OperationError, oefmt from rpython.rlib.debug import check_nonneg +from rpython.rlib.objectmodel import specialize # A `dequeobject` is composed of a doubly-linked list of `block` nodes. @@ -316,12 +317,12 @@ w_currently_in_repr = ec._py_repr = space.newdict() return dequerepr(space, w_currently_in_repr, space.wrap(self)) + @specialize.arg(2) def compare(self, w_other, op): space = self.space if not isinstance(w_other, W_Deque): return space.w_NotImplemented return space.compare_by_iteration(space.wrap(self), w_other, op) - compare._annspecialcase_ = 'specialize:arg(2)' def lt(self, w_other): return self.compare(w_other, 'lt') diff --git a/pypy/module/_csv/interp_reader.py b/pypy/module/_csv/interp_reader.py --- a/pypy/module/_csv/interp_reader.py +++ b/pypy/module/_csv/interp_reader.py @@ -1,4 +1,5 @@ from rpython.rlib.rstring import UnicodeBuilder +from rpython.rlib import objectmodel from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import unwrap_spec @@ -24,13 +25,13 @@ def iter_w(self): return self.space.wrap(self) + @objectmodel.dont_inline def error(self, msg): space = self.space msg = u'line %d: %s' % (self.line_num, msg) w_module = space.getbuiltinmodule('_csv') w_error = space.getattr(w_module, space.wrap('Error')) raise OperationError(w_error, space.wrap(msg)) - error._dont_inline_ = True def add_char(self, field_builder, c): assert field_builder is not None diff --git a/pypy/module/_csv/interp_writer.py b/pypy/module/_csv/interp_writer.py --- a/pypy/module/_csv/interp_writer.py +++ b/pypy/module/_csv/interp_writer.py @@ -1,4 +1,5 @@ from rpython.rlib.rstring import UnicodeBuilder +from rpython.rlib import objectmodel from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.error import OperationError from pypy.interpreter.typedef import TypeDef, interp2app @@ -21,12 +22,12 @@ special += dialect.quotechar self.special_characters = special + @objectmodel.dont_inline def error(self, msg): space = self.space w_module = space.getbuiltinmodule('_csv') w_error = space.getattr(w_module, space.wrap('Error')) raise OperationError(w_error, space.wrap(msg)) - error._dont_inline_ = True def writerow(self, w_fields): """Construct and write a CSV record from a sequence of fields. diff --git a/pypy/module/_lsprof/interp_lsprof.py b/pypy/module/_lsprof/interp_lsprof.py --- a/pypy/module/_lsprof/interp_lsprof.py +++ b/pypy/module/_lsprof/interp_lsprof.py @@ -7,7 +7,7 @@ from pypy.interpreter.typedef import (TypeDef, GetSetProperty, interp_attrproperty) from rpython.rlib import jit -from rpython.rlib.objectmodel import we_are_translated +from rpython.rlib.objectmodel import we_are_translated, always_inline from rpython.rlib.rtimer import read_timestamp, _is_64_bit from rpython.rtyper.lltypesystem import rffi, lltype from rpython.translator.tool.cbuild import ExternalCompilationInfo @@ -258,7 +258,7 @@ return w_frame.wrap_string(space) return w_frame # actually a PyCode object - + at always_inline def prepare_spec(space, w_arg): if isinstance(w_arg, Method): return (w_arg.w_function, space.type(w_arg.w_instance)) @@ -266,8 +266,6 @@ return (w_arg, None) else: return (None, space.type(w_arg)) -prepare_spec._always_inline_ = True - def lsprof_call(space, w_self, frame, event, w_arg): assert isinstance(w_self, W_Profiler) diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py --- a/pypy/module/_pypyjson/interp_decoder.py +++ b/pypy/module/_pypyjson/interp_decoder.py @@ -1,6 +1,6 @@ import sys from rpython.rlib.rstring import StringBuilder -from rpython.rlib.objectmodel import specialize +from rpython.rlib.objectmodel import specialize, always_inline from rpython.rlib import rfloat, runicode from rpython.rtyper.lltypesystem import lltype, rffi from pypy.interpreter.error import oefmt @@ -189,6 +189,7 @@ self.pos = i return self.space.call_function(self.space.w_int, self.space.wrap(s)) + @always_inline def parse_integer(self, i): "Parse a decimal number with an optional minus sign" sign = 1 @@ -219,7 +220,6 @@ # overflowed ovf_maybe = (count >= OVF_DIGITS) return i, ovf_maybe, sign * intval - parse_integer._always_inline_ = True def decode_array(self, i): w_list = self.space.newlist([]) diff --git a/pypy/module/_pypyjson/targetjson.py b/pypy/module/_pypyjson/targetjson.py --- a/pypy/module/_pypyjson/targetjson.py +++ b/pypy/module/_pypyjson/targetjson.py @@ -6,6 +6,7 @@ import time from pypy.interpreter.error import OperationError from pypy.module._pypyjson.interp_decoder import loads +from rpython.rlib.objectmodel import specialize, dont_inline ## MSG = open('msg.json').read() @@ -68,11 +69,11 @@ assert isinstance(w_x, W_String) return w_x.strval + @dont_inline def call_method(self, obj, name, arg): assert name == 'append' assert isinstance(obj, W_List) obj.listval.append(arg) - call_method._dont_inline_ = True def call_function(self, w_func, *args_w): return self.w_None # XXX @@ -91,6 +92,7 @@ def wrapfloat(self, x): return W_Float(x) + @specialize.argtype(1) def wrap(self, x): if isinstance(x, int): return W_Int(x) @@ -100,7 +102,6 @@ ## assert False else: return W_Unicode(unicode(x)) - wrap._annspecialcase_ = "specialize:argtype(1)" fakespace = FakeSpace() diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py --- a/pypy/module/_rawffi/interp_rawffi.py +++ b/pypy/module/_rawffi/interp_rawffi.py @@ -11,6 +11,7 @@ from rpython.rtyper.lltypesystem import lltype, rffi from rpython.rtyper.tool import rffi_platform from rpython.rlib.unroll import unrolling_iterable +from rpython.rlib.objectmodel import specialize import rpython.rlib.rposix as rposix _MS_WINDOWS = os.name == "nt" @@ -262,6 +263,7 @@ _ARM = rffi_platform.getdefined('__arm__', '') + at specialize.arg(2) def read_ptr(ptr, ofs, TP): T = lltype.Ptr(rffi.CArray(TP)) for c in unroll_letters_for_floats: @@ -281,8 +283,8 @@ return ptr_val else: return rffi.cast(T, ptr)[ofs] -read_ptr._annspecialcase_ = 'specialize:arg(2)' + at specialize.argtype(2) def write_ptr(ptr, ofs, value): TP = lltype.typeOf(value) T = lltype.Ptr(rffi.CArray(TP)) @@ -303,7 +305,6 @@ return else: rffi.cast(T, ptr)[ofs] = value -write_ptr._annspecialcase_ = 'specialize:argtype(2)' def segfault_exception(space, reason): w_mod = space.getbuiltinmodule("_rawffi") @@ -375,11 +376,12 @@ def getrawsize(self): raise NotImplementedError("abstract base class") + at specialize.arg(0) def unwrap_truncate_int(TP, space, w_arg): return rffi.cast(TP, space.bigint_w(w_arg).ulonglongmask()) -unwrap_truncate_int._annspecialcase_ = 'specialize:arg(0)' + at specialize.arg(1) def unwrap_value(space, push_func, add_arg, argdesc, letter, w_arg): if letter in TYPEMAP_PTR_LETTERS: # check for NULL ptr @@ -423,10 +425,10 @@ return else: raise oefmt(space.w_TypeError, "cannot directly write value") -unwrap_value._annspecialcase_ = 'specialize:arg(1)' ll_typemap_iter = unrolling_iterable(LL_TYPEMAP.items()) + at specialize.arg(1) def wrap_value(space, func, add_arg, argdesc, letter): for c, ll_type in ll_typemap_iter: if letter == c: @@ -440,7 +442,6 @@ else: return space.wrap(func(add_arg, argdesc, ll_type)) raise oefmt(space.w_TypeError, "cannot directly read value") -wrap_value._annspecialcase_ = 'specialize:arg(1)' NARROW_INTEGER_TYPES = 'cbhiBIH?' diff --git a/pypy/module/_rawffi/structure.py b/pypy/module/_rawffi/structure.py --- a/pypy/module/_rawffi/structure.py +++ b/pypy/module/_rawffi/structure.py @@ -18,6 +18,7 @@ from rpython.rlib.rarithmetic import intmask, signedtype, r_uint, \ r_ulonglong from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.rlib.objectmodel import specialize import sys IS_BIG_ENDIAN = sys.byteorder == 'big' @@ -284,6 +285,7 @@ def NUM_BITS(x): return x >> 16 + at specialize.arg(1) def BIT_MASK(x, ll_t): if ll_t is lltype.SignedLongLong or ll_t is lltype.UnsignedLongLong: one = r_ulonglong(1) @@ -291,8 +293,8 @@ one = r_uint(1) # to avoid left shift by x == sizeof(ll_t) return (((one << (x - 1)) - 1) << 1) + 1 -BIT_MASK._annspecialcase_ = 'specialize:arg(1)' + at specialize.argtype(2) def push_field(self, num, value): ptr = rffi.ptradd(self.ll_buffer, self.shape.ll_positions[num]) TP = lltype.typeOf(value) @@ -313,8 +315,8 @@ value = rffi.cast(TP, current) break write_ptr(ptr, 0, value) -push_field._annspecialcase_ = 'specialize:argtype(2)' + at specialize.arg(2) def cast_pos(self, i, ll_t): pos = rffi.ptradd(self.ll_buffer, self.shape.ll_positions[i]) value = read_ptr(pos, 0, ll_t) @@ -337,7 +339,6 @@ value = rffi.cast(ll_t, value) break return value -cast_pos._annspecialcase_ = 'specialize:arg(2)' class W_StructureInstance(W_DataInstance): def __init__(self, space, shape, address): diff --git a/pypy/module/_warnings/interp_warnings.py b/pypy/module/_warnings/interp_warnings.py --- a/pypy/module/_warnings/interp_warnings.py +++ b/pypy/module/_warnings/interp_warnings.py @@ -248,6 +248,10 @@ if space.isinstance_w(w_message, space.w_Warning): w_text = space.str(w_message) w_category = space.type(w_message) + elif (not space.isinstance_w(w_message, space.w_unicode) or + not space.isinstance_w(w_message, space.w_str)): + w_text = space.str(w_message) + w_message = space.call_function(w_category, w_message) else: w_text = w_message w_message = space.call_function(w_category, w_message) diff --git a/pypy/module/_warnings/test/test_warnings.py b/pypy/module/_warnings/test/test_warnings.py --- a/pypy/module/_warnings/test/test_warnings.py +++ b/pypy/module/_warnings/test/test_warnings.py @@ -16,6 +16,7 @@ import _warnings _warnings.warn("some message", DeprecationWarning) _warnings.warn("some message", Warning) + _warnings.warn(("some message",1), Warning) def test_lineno(self): import warnings, _warnings, sys @@ -45,7 +46,10 @@ def test_show_source_line(self): import warnings import sys, io - from test.warning_tests import inner + try: + from test.warning_tests import inner + except ImportError: + skip('no test, -A on cpython?') # With showarning() missing, make sure that output is okay. del warnings.showwarning diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -20,7 +20,7 @@ from rpython.tool.udir import udir from rpython.translator import platform from pypy.module.cpyext.state import State -from pypy.interpreter.error import OperationError, oefmt +from pypy.interpreter.error import OperationError, oefmt, raise_import_error from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.gateway import unwrap_spec from pypy.interpreter.nestedscope import Cell @@ -982,7 +982,7 @@ py_fatalerror = rffi.llexternal('%s_FatalError' % prefix, [CONST_STRING], lltype.Void, compilation_info=eci) - _reinit_tls = rffi.llexternal('%sThread_ReInitTLS' % prefix, [], + _reinit_tls = rffi.llexternal('%sThread_ReInitTLS' % prefix, [], lltype.Void, compilation_info=eci) def reinit_tls(space): _reinit_tls() @@ -1523,9 +1523,10 @@ finally: lltype.free(ll_libname, flavor='raw') except rdynload.DLOpenError as e: - raise oefmt(space.w_ImportError, - "unable to load extension module '%s': %s", - path, e.msg) + w_name = space.newunicode(name.decode('ascii')) + w_path = space.wrap_fsdecoded(path) + raise raise_import_error(space, + space.wrap_fsdecoded(e.msg), w_name, w_path) look_for = None # if space.config.objspace.usemodules._cffi_backend: @@ -1555,9 +1556,12 @@ look_for += ' or ' + also_look_for else: look_for = also_look_for - # - raise oefmt(space.w_ImportError, - "function %s not found in library %s", look_for, path) + msg = u"function %s not found in library %s" % ( + unicode(look_for), space.unicode_w(space.wrap_fsdecoded(path))) + w_name = space.newunicode(name.decode('ascii')) + w_path = space.wrap_fsdecoded(path) + raise_import_error(space, space.newunicode(msg), w_name, w_path) + initfunctype = lltype.Ptr(lltype.FuncType([], PyObject)) diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -214,7 +214,9 @@ i = space.int_w(space.index(args_w[0])) j = space.int_w(space.index(args_w[1])) w_y = args_w[2] - return space.wrap(generic_cpy_call(space, func_target, w_self, i, j, w_y)) + res = generic_cpy_call(space, func_target, w_self, i, j, w_y) + if rffi.cast(lltype.Signed, res) == -1: + space.fromcache(State).check_and_raise_exception(always=True) def wrap_lenfunc(space, w_self, w_args, func): func_len = rffi.cast(lenfunc, func) @@ -296,7 +298,10 @@ def wrap_hashfunc(space, w_self, w_args, func): func_target = rffi.cast(hashfunc, func) check_num_args(space, w_args, 0) - return space.wrap(generic_cpy_call(space, func_target, w_self)) + res = generic_cpy_call(space, func_target, w_self) + if res == -1: + space.fromcache(State).check_and_raise_exception(always=True) + return space.wrap(res) class CPyBuffer(Buffer): # Similar to Py_buffer diff --git a/pypy/module/cpyext/test/conftest.py b/pypy/module/cpyext/test/conftest.py --- a/pypy/module/cpyext/test/conftest.py +++ b/pypy/module/cpyext/test/conftest.py @@ -2,6 +2,8 @@ import pytest def pytest_configure(config): + if config.option.runappdirect: + return from pypy.tool.pytest.objspace import gettestobjspace # For some reason (probably a ll2ctypes cache issue on linux64) # it's necessary to run "import time" at least once before any diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py --- a/pypy/module/cpyext/test/test_arraymodule.py +++ b/pypy/module/cpyext/test/test_arraymodule.py @@ -49,6 +49,7 @@ assert arr.tolist() == [1, 21, 22, 23, 4] del arr[slice(1, 3)] assert arr.tolist() == [1, 23, 4] + raises(TypeError, 'arr[slice(1, 3)] = "abc"') def test_buffer(self): import sys diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py --- a/pypy/module/cpyext/test/test_cpyext.py +++ b/pypy/module/cpyext/test/test_cpyext.py @@ -5,7 +5,7 @@ import py, pytest from pypy import pypydir -from pypy.interpreter import gateway +from pypy.interpreter.gateway import unwrap_spec, interp2app from rpython.rtyper.lltypesystem import lltype, ll2ctypes from rpython.translator.gensupp import uniquemodulename from rpython.tool.udir import udir @@ -18,6 +18,7 @@ from .support import c_compile +HERE = py.path.local(pypydir) / 'module' / 'cpyext' / 'test' only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" @api.cpython_api([], api.PyObject) @@ -42,19 +43,6 @@ files.append(filename) return files -def create_so(modname, include_dirs, source_strings=None, source_files=None, - compile_extra=None, link_extra=None, libraries=None): - dirname = (udir/uniquemodulename('module')).ensure(dir=1) - if source_strings: - assert not source_files - files = convert_sources_to_files(source_strings, dirname) - source_files = files - soname = c_compile(source_files, outputfilename=str(dirname/modname), - compile_extra=compile_extra, link_extra=link_extra, - include_dirs=include_dirs, - libraries=libraries) - return soname - class SystemCompilationInfo(object): """Bundles all the generic information required to compile extensions. @@ -68,6 +56,96 @@ self.extra_libs = extra_libs self.ext = ext + def compile_extension_module(self, name, include_dirs=None, + source_files=None, source_strings=None): + """ + Build an extension module and return the filename of the resulting + native code file. + + name is the name of the module, possibly including dots if it is a + module inside a package. + + Any extra keyword arguments are passed on to ExternalCompilationInfo to + build the module (so specify your source with one of those). + """ + include_dirs = include_dirs or [] + modname = name.split('.')[-1] + dirname = (udir/uniquemodulename('module')).ensure(dir=1) + if source_strings: + assert not source_files + files = convert_sources_to_files(source_strings, dirname) + source_files = files + soname = c_compile(source_files, outputfilename=str(dirname/modname), + compile_extra=self.compile_extra, + link_extra=self.link_extra, + include_dirs=self.include_extra + include_dirs, + libraries=self.extra_libs) + pydname = soname.new(purebasename=modname, ext=self.ext) + soname.rename(pydname) + return str(pydname) + + def import_module(self, name, init=None, body='', filename=None, + include_dirs=None, PY_SSIZE_T_CLEAN=False): + """ + init specifies the overall template of the module. + + if init is None, the module source will be loaded from a file in this + test directory, give a name given by the filename parameter. + + if filename is None, the module name will be used to construct the + filename. + """ + name = name.encode() + if body or init: + body = body.encode() + if init is None: + init = "return PyModule_Create(&moduledef);" + else: + init = init.encode() + if init is not None: + code = make_source(name, init, body, PY_SSIZE_T_CLEAN) + kwds = dict(source_strings=[code]) + else: + assert not PY_SSIZE_T_CLEAN + if filename is None: + filename = name + filename = HERE / (filename + ".c") + kwds = dict(source_files=[filename]) + mod = self.compile_extension_module( + name, include_dirs=include_dirs, **kwds) + return self.load_module(mod, name) + + def import_extension(self, modname, functions, prologue="", + include_dirs=None, more_init="", PY_SSIZE_T_CLEAN=False): + body = prologue + make_methods(functions, modname) + init = """PyObject *mod = PyModule_Create(&moduledef);""" + if more_init: + init += more_init + init += "\nreturn mod;" + return self.import_module( + name=modname, init=init, body=body, include_dirs=include_dirs, + PY_SSIZE_T_CLEAN=PY_SSIZE_T_CLEAN) + + +class ExtensionCompiler(SystemCompilationInfo): + """Extension compiler for appdirect mode""" + def load_module(space, mod, name): + import imp + return imp.load_dynamic(name, mod) + +class SpaceCompiler(SystemCompilationInfo): + """Extension compiler for regular (untranslated PyPy) mode""" + def __init__(self, space, *args, **kwargs): + self.space = space + SystemCompilationInfo.__init__(self, *args, **kwargs) + + def load_module(self, mod, name): + space = self.space + api.load_extension_module(space, mod, name) + return space.getitem( + space.sys.get('modules'), space.wrap(name)) + + def get_cpyext_info(space): from pypy.module.imp.importing import get_so_extension state = space.fromcache(State) @@ -88,7 +166,7 @@ link_extra = ["-g"] else: compile_extra = link_extra = None - return SystemCompilationInfo( + return SpaceCompiler(space, include_extra=api.include_dirs, compile_extra=compile_extra, link_extra=link_extra, @@ -96,30 +174,6 @@ ext=get_so_extension(space)) -def compile_extension_module(sys_info, modname, include_dirs=[], - source_files=None, source_strings=None): - """ - Build an extension module and return the filename of the resulting native - code file. - - modname is the name of the module, possibly including dots if it is a module - inside a package. - - Any extra keyword arguments are passed on to ExternalCompilationInfo to - build the module (so specify your source with one of those). - """ - modname = modname.split('.')[-1] - soname = create_so(modname, - include_dirs=sys_info.include_extra + include_dirs, - source_files=source_files, - source_strings=source_strings, - compile_extra=sys_info.compile_extra, - link_extra=sys_info.link_extra, - libraries=sys_info.extra_libs) - pydname = soname.new(purebasename=modname, ext=sys_info.ext) - soname.rename(pydname) - return str(pydname) - def get_so_suffix(): from imp import get_suffixes, C_EXTENSION for suffix, mode, typ in get_suffixes(): @@ -142,12 +196,58 @@ "-O0", "-g", "-Werror=implicit-function-declaration", "-fPIC"] link_extra = None ext = get_so_suffix() - return SystemCompilationInfo( + return ExtensionCompiler( include_extra=[get_python_inc()], compile_extra=compile_extra, link_extra=link_extra, ext=get_so_suffix()) +def make_methods(functions, modname): + methods_table = [] + codes = [] + for funcname, flags, code in functions: + cfuncname = "%s_%s" % (modname, funcname) + methods_table.append("{\"%s\", %s, %s}," % + (funcname, cfuncname, flags)) + func_code = """ + static PyObject* %s(PyObject* self, PyObject* args) + { + %s + } + """ % (cfuncname, code) + codes.append(func_code) + + body = "\n".join(codes) + """ + static PyMethodDef methods[] = { + %(methods)s + { NULL } + }; + static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + "%(modname)s", /* m_name */ + NULL, /* m_doc */ + -1, /* m_size */ + methods, /* m_methods */ + }; + """ % dict(methods='\n'.join(methods_table), modname=modname) + return body + +def make_source(name, init, body, PY_SSIZE_T_CLEAN): + code = """ + %(PY_SSIZE_T_CLEAN)s + #include + + %(body)s + + PyMODINIT_FUNC + PyInit_%(name)s(void) { + %(init)s + } + """ % dict(name=name, init=init, body=body, + PY_SSIZE_T_CLEAN='#define PY_SSIZE_T_CLEAN' + if PY_SSIZE_T_CLEAN else '') + return code + def freeze_refcnts(self): rawrefcount._dont_free_any_more() @@ -159,22 +259,6 @@ #state.print_refcounts() self.frozen_ll2callocations = set(ll2ctypes.ALLOCATED.values()) -class FakeSpace(object): - """Like TinyObjSpace, but different""" - def __init__(self, config): - self.config = config - - def passthrough(self, arg): - return arg - listview = passthrough - str_w = passthrough - - def unwrap(self, args): - try: - return args.str_w(None) - except: - return args - class LeakCheckingTest(object): """Base class for all cpyext tests.""" spaceconfig = dict(usemodules=['cpyext', 'thread', '_rawffi', 'array', @@ -267,9 +351,12 @@ state.non_heaptypes_w[:] = [] def setup_method(self, meth): - freeze_refcnts(self) + if not self.runappdirect: + freeze_refcnts(self) def teardown_method(self, meth): + if self.runappdirect: + return self.cleanup_references(self.space) # XXX: like AppTestCpythonExtensionBase.teardown_method: # find out how to disable check_and_print_leaks() if the @@ -295,21 +382,42 @@ skip("Windows Python >= 2.6 only") assert isinstance(sys.dllhandle, int) + +def _unwrap_include_dirs(space, w_include_dirs): + if w_include_dirs is None: + return None + else: + return [space.str_w(s) for s in space.listview(w_include_dirs)] + +def debug_collect(space): + rawrefcount._collect() + class AppTestCpythonExtensionBase(LeakCheckingTest): def setup_class(cls): space = cls.space - space.getbuiltinmodule("cpyext") - # 'import os' to warm up reference counts - w_import = space.builtin.getdictvalue(space, '__import__') - space.call_function(w_import, space.wrap("os")) - #state = cls.space.fromcache(RefcountState) ZZZ - #state.non_heaptypes_w[:] = [] + cls.w_here = space.wrap(str(HERE)) if not cls.runappdirect: cls.w_runappdirect = space.wrap(cls.runappdirect) + space.getbuiltinmodule("cpyext") + # 'import os' to warm up reference counts + w_import = space.builtin.getdictvalue(space, '__import__') + space.call_function(w_import, space.wrap("os")) + #state = cls.space.fromcache(RefcountState) ZZZ + #state.non_heaptypes_w[:] = [] + cls.w_debug_collect = space.wrap(interp2app(debug_collect)) + + def record_imported_module(self, name): + """ + Record a module imported in a test so that it can be cleaned up in + teardown before the check for leaks is done. + + name gives the name of the module in the space's sys.modules. + """ + self.imported_module_names.append(name) def setup_method(self, func): - @gateway.unwrap_spec(name=str) + @unwrap_spec(name=str) def compile_module(space, name, w_source_files=None, w_source_strings=None): @@ -324,178 +432,68 @@ source_strings = space.listview_bytes(w_source_strings) else: source_strings = None - pydname = compile_extension_module( - self.sys_info, name, + pydname = self.sys_info.compile_extension_module( + name, source_files=source_files, source_strings=source_strings) + + # hackish, but tests calling compile_module() always end up + # importing the result + self.record_imported_module(name) + return space.wrap(pydname) - @gateway.unwrap_spec(name=str, init='str_or_None', body=str, - load_it=bool, filename='str_or_None', - PY_SSIZE_T_CLEAN=bool) - def import_module(space, name, init=None, body='', load_it=True, + @unwrap_spec(name=str, init='str_or_None', body=str, + filename='str_or_None', PY_SSIZE_T_CLEAN=bool) + def import_module(space, name, init=None, body='', filename=None, w_include_dirs=None, PY_SSIZE_T_CLEAN=False): - """ - init specifies the overall template of the module. + include_dirs = _unwrap_include_dirs(space, w_include_dirs) + w_result = self.sys_info.import_module( + name, init, body, filename, include_dirs, PY_SSIZE_T_CLEAN) + self.record_imported_module(name) + return w_result - if init is None, the module source will be loaded from a file in this - test direcory, give a name given by the filename parameter. - if filename is None, the module name will be used to construct the - filename. - """ - name = name.encode() - if body or init: - body = body.encode() - if init is None: - init = "return PyModule_Create(&moduledef);" - else: - init = init.encode() - if w_include_dirs is None: - include_dirs = [] - else: - include_dirs = [space.str_w(s) for s in space.listview(w_include_dirs)] - if init is not None: - code = """ - %(PY_SSIZE_T_CLEAN)s - #include + @unwrap_spec(mod=str, name=str) + def load_module(space, mod, name): + return self.sys_info.load_module(mod, name) - %(body)s - - PyMODINIT_FUNC - PyInit_%(name)s(void) { - %(init)s - } - """ % dict(name=name, init=init, body=body, - PY_SSIZE_T_CLEAN='#define PY_SSIZE_T_CLEAN' - if PY_SSIZE_T_CLEAN else '') - kwds = dict(source_strings=[code]) - else: - assert not PY_SSIZE_T_CLEAN - if filename is None: - filename = name - filename = py.path.local(pypydir) / 'module' \ - / 'cpyext'/ 'test' / (filename + ".c") - kwds = dict(source_files=[filename]) - mod = compile_extension_module(self.sys_info, name, - include_dirs=include_dirs, **kwds) - - if load_it: - if self.runappdirect: - import imp - return imp.load_dynamic(name, mod) - else: - api.load_extension_module(space, mod, name) - self.imported_module_names.append(name) - return space.getitem( - space.sys.get('modules'), - space.wrap(name)) - else: - path = os.path.dirname(mod) - if self.runappdirect: - return path - else: - return space.wrap(path) - - @gateway.unwrap_spec(mod=str, name=str) - def reimport_module(space, mod, name): - if self.runappdirect: - import imp - return imp.load_dynamic(name, mod) - else: - api.load_extension_module(space, mod, name) - return space.getitem( - space.sys.get('modules'), - space.wrap(name)) - - @gateway.unwrap_spec(modname=str, prologue=str, + @unwrap_spec(modname=str, prologue=str, more_init=str, PY_SSIZE_T_CLEAN=bool) def import_extension(space, modname, w_functions, prologue="", w_include_dirs=None, more_init="", PY_SSIZE_T_CLEAN=False): functions = space.unwrap(w_functions) - methods_table = [] - codes = [] - for funcname, flags, code in functions: - cfuncname = "%s_%s" % (modname, funcname) - methods_table.append("{\"%s\", %s, %s}," % - (funcname, cfuncname, flags)) - func_code = """ - static PyObject* %s(PyObject* self, PyObject* args) - { - %s - } - """ % (cfuncname, code) - codes.append(func_code) - - body = prologue + "\n".join(codes) + """ - static PyMethodDef methods[] = { - %(methods)s - { NULL } - }; - static struct PyModuleDef moduledef = { - PyModuleDef_HEAD_INIT, - "%(modname)s", /* m_name */ - NULL, /* m_doc */ - -1, /* m_size */ - methods, /* m_methods */ - }; - """ % dict(methods='\n'.join(methods_table), modname=modname) - init = """PyObject *mod = PyModule_Create(&moduledef);""" - if more_init: - init += more_init - init += "\nreturn mod;" - return import_module(space, name=modname, init=init, body=body, - w_include_dirs=w_include_dirs, - PY_SSIZE_T_CLEAN=PY_SSIZE_T_CLEAN) - - @gateway.unwrap_spec(name=str) - def record_imported_module(name): - """ - Record a module imported in a test so that it can be cleaned up in - teardown before the check for leaks is done. - - name gives the name of the module in the space's sys.modules. - """ - self.imported_module_names.append(name) - - def debug_collect(space): - rawrefcount._collect() + include_dirs = _unwrap_include_dirs(space, w_include_dirs) + w_result = self.sys_info.import_extension( + modname, functions, prologue, include_dirs, more_init, + PY_SSIZE_T_CLEAN) + self.record_imported_module(modname) + return w_result # A list of modules which the test caused to be imported (in # self.space). These will be cleaned up automatically in teardown. self.imported_module_names = [] if self.runappdirect: - fake = FakeSpace(self.space.config) - def interp2app(func): - def run(*args, **kwargs): - for k in kwargs.keys(): - if k not in func.unwrap_spec and not k.startswith('w_'): - v = kwargs.pop(k) - kwargs['w_' + k] = v - return func(fake, *args, **kwargs) - return run - def wrap(func): - return func self.sys_info = get_sys_info_app() + self.compile_module = self.sys_info.compile_extension_module + self.load_module = self.sys_info.load_module + self.import_module = self.sys_info.import_module + self.import_extension = self.sys_info.import_extension else: - interp2app = gateway.interp2app wrap = self.space.wrap self.sys_info = get_cpyext_info(self.space) - self.w_compile_module = wrap(interp2app(compile_module)) - self.w_import_module = wrap(interp2app(import_module)) - self.w_reimport_module = wrap(interp2app(reimport_module)) - self.w_import_extension = wrap(interp2app(import_extension)) - self.w_record_imported_module = wrap(interp2app(record_imported_module)) - self.w_here = wrap(str(py.path.local(pypydir)) + '/module/cpyext/test/') - self.w_debug_collect = wrap(interp2app(debug_collect)) + self.w_compile_module = wrap(interp2app(compile_module)) + self.w_load_module = wrap(interp2app(load_module)) + self.w_import_module = wrap(interp2app(import_module)) + self.w_import_extension = wrap(interp2app(import_extension)) - # create the file lock before we count allocations - self.space.call_method(self.space.sys.get("stdout"), "flush") + # create the file lock before we count allocations + self.space.call_method(self.space.sys.get("stdout"), "flush") - freeze_refcnts(self) - #self.check_and_print_leaks() + freeze_refcnts(self) + #self.check_and_print_leaks() def unimport_module(self, name): """ @@ -506,6 +504,8 @@ self.space.delitem(w_modules, w_name) def teardown_method(self, func): + if self.runappdirect: + return for name in self.imported_module_names: self.unimport_module(name) self.cleanup_references(self.space) @@ -628,19 +628,15 @@ From pypy.commits at gmail.com Sat Oct 1 06:21:08 2016 From: pypy.commits at gmail.com (arigo) Date: Sat, 01 Oct 2016 03:21:08 -0700 (PDT) Subject: [pypy-commit] pypy py3k-faulthandler: ready to merge Message-ID: <57ef8e14.8d951c0a.bea7a.e9ec@mx.google.com> Author: Armin Rigo Branch: py3k-faulthandler Changeset: r87496:66638a75ce5c Date: 2016-10-01 12:20 +0200 http://bitbucket.org/pypy/pypy/changeset/66638a75ce5c/ Log: ready to merge From pypy.commits at gmail.com Sat Oct 1 06:21:10 2016 From: pypy.commits at gmail.com (arigo) Date: Sat, 01 Oct 2016 03:21:10 -0700 (PDT) Subject: [pypy-commit] pypy py3k: hg merge py3k-faulthandler Message-ID: <57ef8e16.657bc20a.33cf9.1d96@mx.google.com> Author: Armin Rigo Branch: py3k Changeset: r87497:0863ca6a99cc Date: 2016-10-01 12:20 +0200 http://bitbucket.org/pypy/pypy/changeset/0863ca6a99cc/ Log: hg merge py3k-faulthandler Merge the py3k version of faulthandler. diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py --- a/pypy/config/pypyoption.py +++ b/pypy/config/pypyoption.py @@ -40,13 +40,14 @@ "binascii", "_multiprocessing", '_warnings', "_collections", "_multibytecodec", "_continuation", "_cffi_backend", "_csv", "_pypyjson", "_posixsubprocess", # "cppyy", "micronumpy" - "faulthandler", "_jitlog", + "_jitlog", ]) from rpython.jit.backend import detect_cpu try: if detect_cpu.autodetect().startswith('x86'): working_modules.add('_vmprof') + working_modules.add('faulthandler') except detect_cpu.ProcessorAutodetectError: pass @@ -93,6 +94,7 @@ ('objspace.usemodules.thread', True)], 'cpyext': [('objspace.usemodules.array', True)], 'cppyy': [('objspace.usemodules.cpyext', True)], + 'faulthandler': [('objspace.usemodules._vmprof', True)], } module_suggests = { # the reason you want _rawffi is for ctypes, which @@ -118,7 +120,8 @@ "_hashlib" : ["pypy.module._ssl.interp_ssl"], "_minimal_curses": ["pypy.module._minimal_curses.fficurses"], "_continuation": ["rpython.rlib.rstacklet"], - "_vmprof" : ["pypy.module._vmprof.interp_vmprof"], + "_vmprof" : ["pypy.module._vmprof.interp_vmprof"], + "faulthandler" : ["pypy.module._vmprof.interp_vmprof"], "_lzma" : ["pypy.module._lzma.interp_lzma"], } diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py --- a/pypy/interpreter/app_main.py +++ b/pypy/interpreter/app_main.py @@ -31,6 +31,7 @@ arg ...: arguments passed to program in sys.argv[1:] PyPy options and arguments: --info : print translation information about this PyPy executable +-X faulthandler: attempt to display tracebacks when PyPy crashes """ # Missing vs CPython: PYTHONHOME USAGE2 = """ @@ -518,6 +519,14 @@ sys._xoptions = dict(x.split('=', 1) if '=' in x else (x, True) for x in options['_xoptions']) + if 'faulthandler' in sys.builtin_module_names: + if 'faulthandler' in sys._xoptions or os.getenv('PYTHONFAULTHANDLER'): + import faulthandler + try: + faulthandler.enable(2) # manually set to stderr + except ValueError: + pass # ignore "2 is not a valid file descriptor" + ## if not we_are_translated(): ## for key in sorted(options): ## print '%40s: %s' % (key, options[key]) diff --git a/pypy/module/faulthandler/__init__.py b/pypy/module/faulthandler/__init__.py --- a/pypy/module/faulthandler/__init__.py +++ b/pypy/module/faulthandler/__init__.py @@ -1,22 +1,38 @@ +import sys from pypy.interpreter.mixedmodule import MixedModule + class Module(MixedModule): appleveldefs = { } interpleveldefs = { - 'enable': 'interp_faulthandler.enable', - 'disable': 'interp_faulthandler.disable', - 'is_enabled': 'interp_faulthandler.is_enabled', - 'register': 'interp_faulthandler.register', + 'enable': 'handler.enable', + 'disable': 'handler.disable', + 'is_enabled': 'handler.is_enabled', +# 'register': 'interp_faulthandler.register', +# + 'dump_traceback': 'handler.dump_traceback', +# + '_read_null': 'handler.read_null', + '_sigsegv': 'handler.sigsegv', + '_sigfpe': 'handler.sigfpe', + '_sigabrt': 'handler.sigabrt', + '_stack_overflow': 'handler.stack_overflow', + } - 'dump_traceback': 'interp_faulthandler.dump_traceback', + def setup_after_space_initialization(self): + """NOT_RPYTHON""" + if self.space.config.translation.thread: + self.extra_interpdef('dump_traceback_later', + 'handler.dump_traceback_later') + self.extra_interpdef('cancel_dump_traceback_later', + 'handler.cancel_dump_traceback_later') + if sys.platform != 'win32': + self.extra_interpdef('register', 'handler.register') + self.extra_interpdef('unregister', 'handler.unregister') - '_read_null': 'interp_faulthandler.read_null', - '_sigsegv': 'interp_faulthandler.sigsegv', - '_sigfpe': 'interp_faulthandler.sigfpe', - '_sigabrt': 'interp_faulthandler.sigabrt', - #'_sigbus': 'interp_faulthandler.sigbus', - #'_sigill': 'interp_faulthandler.sigill', - '_fatal_error': 'interp_faulthandler.fatal_error', - } + def shutdown(self, space): + from pypy.module.faulthandler import handler + handler.finish(space) + MixedModule.shutdown(self, space) diff --git a/pypy/module/faulthandler/cintf.py b/pypy/module/faulthandler/cintf.py new file mode 100644 --- /dev/null +++ b/pypy/module/faulthandler/cintf.py @@ -0,0 +1,99 @@ +import py +from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, rstr +from rpython.translator import cdir +from rpython.translator.tool.cbuild import ExternalCompilationInfo + + +cwd = py.path.local(__file__).dirpath() +eci = ExternalCompilationInfo( + includes=[cwd.join('faulthandler.h')], + include_dirs=[str(cwd), cdir], + separate_module_files=[cwd.join('faulthandler.c')]) + +eci_later = eci.merge(ExternalCompilationInfo( + pre_include_bits=['#define PYPY_FAULTHANDLER_LATER\n'])) +eci_user = eci.merge(ExternalCompilationInfo( + pre_include_bits=['#define PYPY_FAULTHANDLER_USER\n'])) + +def direct_llexternal(*args, **kwargs): + kwargs.setdefault('_nowrapper', True) + kwargs.setdefault('compilation_info', eci) + return rffi.llexternal(*args, **kwargs) + + +DUMP_CALLBACK = lltype.Ptr(lltype.FuncType( + [rffi.INT, rffi.SIGNEDP, lltype.Signed], lltype.Void)) + +pypy_faulthandler_setup = direct_llexternal( + 'pypy_faulthandler_setup', [DUMP_CALLBACK], rffi.CCHARP) + +pypy_faulthandler_teardown = direct_llexternal( + 'pypy_faulthandler_teardown', [], lltype.Void) + +pypy_faulthandler_enable = direct_llexternal( + 'pypy_faulthandler_enable', [rffi.INT, rffi.INT], rffi.CCHARP) + +pypy_faulthandler_disable = direct_llexternal( + 'pypy_faulthandler_disable', [], lltype.Void) + +pypy_faulthandler_is_enabled = direct_llexternal( + 'pypy_faulthandler_is_enabled', [], rffi.INT) + +pypy_faulthandler_write = direct_llexternal( + 'pypy_faulthandler_write', [rffi.INT, rffi.CCHARP], lltype.Void) + +pypy_faulthandler_write_int = direct_llexternal( + 'pypy_faulthandler_write_int', [rffi.INT, lltype.Signed], lltype.Void) + +pypy_faulthandler_dump_traceback = direct_llexternal( + 'pypy_faulthandler_dump_traceback', + [rffi.INT, rffi.INT, llmemory.Address], lltype.Void) + +pypy_faulthandler_dump_traceback_later = direct_llexternal( + 'pypy_faulthandler_dump_traceback_later', + [rffi.LONGLONG, rffi.INT, rffi.INT, rffi.INT], rffi.CCHARP, + compilation_info=eci_later) + +pypy_faulthandler_cancel_dump_traceback_later = direct_llexternal( + 'pypy_faulthandler_cancel_dump_traceback_later', [], lltype.Void) + +pypy_faulthandler_check_signum = direct_llexternal( + 'pypy_faulthandler_check_signum', + [rffi.LONG], rffi.INT, + compilation_info=eci_user) + +pypy_faulthandler_register = direct_llexternal( + 'pypy_faulthandler_register', + [rffi.INT, rffi.INT, rffi.INT, rffi.INT], rffi.CCHARP, + compilation_info=eci_user) + +pypy_faulthandler_unregister = direct_llexternal( + 'pypy_faulthandler_unregister', + [rffi.INT], rffi.INT, + compilation_info=eci_user) + + +# for tests... + +pypy_faulthandler_read_null = direct_llexternal( + 'pypy_faulthandler_read_null', [], lltype.Void) + +pypy_faulthandler_read_null_releasegil = direct_llexternal( + 'pypy_faulthandler_read_null', [], lltype.Void, + _nowrapper=False, releasegil=True) + +pypy_faulthandler_sigsegv = direct_llexternal( + 'pypy_faulthandler_sigsegv', [], lltype.Void) + +pypy_faulthandler_sigsegv_releasegil = direct_llexternal( + 'pypy_faulthandler_sigsegv', [], lltype.Void, + _nowrapper=False, releasegil=True) + +pypy_faulthandler_sigfpe = direct_llexternal( + 'pypy_faulthandler_sigfpe', [], lltype.Void) + +pypy_faulthandler_sigabrt = direct_llexternal( + 'pypy_faulthandler_sigabrt', [], lltype.Void) + +pypy_faulthandler_stackoverflow = direct_llexternal( + 'pypy_faulthandler_stackoverflow', [lltype.Float], lltype.Float) diff --git a/pypy/module/faulthandler/dumper.py b/pypy/module/faulthandler/dumper.py new file mode 100644 --- /dev/null +++ b/pypy/module/faulthandler/dumper.py @@ -0,0 +1,54 @@ +from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.rlib import rgc +from rpython.rlib.rvmprof import traceback + +from pypy.interpreter.pycode import PyCode +from pypy.module.faulthandler.cintf import pypy_faulthandler_write +from pypy.module.faulthandler.cintf import pypy_faulthandler_write_int + + +MAX_STRING_LENGTH = 500 + +global_buf = lltype.malloc(rffi.CCHARP.TO, MAX_STRING_LENGTH, flavor='raw', + immortal=True, zero=True) + +def _dump(fd, s): + assert isinstance(s, str) + l = len(s) + if l >= MAX_STRING_LENGTH: + l = MAX_STRING_LENGTH - 1 + i = 0 + while i < l: + global_buf[i] = s[i] + i += 1 + global_buf[l] = '\x00' + pypy_faulthandler_write(fd, global_buf) + +def _dump_int(fd, i): + pypy_faulthandler_write_int(fd, i) + + +def dump_code(pycode, loc, fd): + if pycode is None: + _dump(fd, " File ???") + else: + _dump(fd, ' File "') + _dump(fd, pycode.co_filename) + _dump(fd, '" in ') + _dump(fd, pycode.co_name) + _dump(fd, ", from line ") + _dump_int(fd, pycode.co_firstlineno) + if loc == traceback.LOC_JITTED: + _dump(fd, " [jitted]") + elif loc == traceback.LOC_JITTED_INLINED: + _dump(fd, " [jit inlined]") + _dump(fd, "\n") + + + at rgc.no_collect +def _dump_callback(fd, array_p, array_length): + """We are as careful as we can reasonably be here (i.e. not 100%, + but hopefully close enough). In particular, this is written as + RPython but shouldn't allocate anything. + """ + traceback.walk_traceback(PyCode, dump_code, fd, array_p, array_length) diff --git a/pypy/module/faulthandler/faulthandler.c b/pypy/module/faulthandler/faulthandler.c --- a/pypy/module/faulthandler/faulthandler.c +++ b/pypy/module/faulthandler/faulthandler.c @@ -1,20 +1,625 @@ +#include "faulthandler.h" #include -#include "faulthandler.h" +#include +#include +#include +#include +#include +#include +#include +#include -int -pypy_faulthandler_read_null(void) +#ifdef RPYTHON_LL2CTYPES +# include "../../../rpython/rlib/rvmprof/src/rvmprof.h" +#else +# include "common_header.h" +# include "structdef.h" +# include "rvmprof.h" +#endif +#include "src/threadlocal.h" + +#define MAX_FRAME_DEPTH 100 +#define FRAME_DEPTH_N RVMPROF_TRACEBACK_ESTIMATE_N(MAX_FRAME_DEPTH) + + +typedef struct sigaction _Py_sighandler_t; + +typedef struct { + const int signum; + volatile int enabled; + const char* name; + _Py_sighandler_t previous; +} fault_handler_t; + +static struct { + int initialized; + int enabled; + volatile int fd, all_threads; + volatile pypy_faulthandler_cb_t dump_traceback; +} fatal_error; + +static stack_t stack; + + +static fault_handler_t faulthandler_handlers[] = { +#ifdef SIGBUS + {SIGBUS, 0, "Bus error", }, +#endif +#ifdef SIGILL + {SIGILL, 0, "Illegal instruction", }, +#endif + {SIGFPE, 0, "Floating point exception", }, + {SIGABRT, 0, "Aborted", }, + /* define SIGSEGV at the end to make it the default choice if searching the + handler fails in faulthandler_fatal_error() */ + {SIGSEGV, 0, "Segmentation fault", } +}; +static const int faulthandler_nsignals = + sizeof(faulthandler_handlers) / sizeof(fault_handler_t); + +RPY_EXTERN +void pypy_faulthandler_write(int fd, const char *str) { - volatile int *x; - volatile int y; - - x = NULL; - y = *x; - return y; + (void)write(fd, str, strlen(str)); } -void -pypy_faulthandler_sigsegv(void) +RPY_EXTERN +void pypy_faulthandler_write_int(int fd, long value) { + char buf[48]; + sprintf(buf, "%ld", value); + pypy_faulthandler_write(fd, buf); +} + + +RPY_EXTERN +void pypy_faulthandler_dump_traceback(int fd, int all_threads, + void *ucontext) +{ + pypy_faulthandler_cb_t fn; + intptr_t array_p[FRAME_DEPTH_N], array_length; + + fn = fatal_error.dump_traceback; + if (!fn) + return; + +#ifndef RPYTHON_LL2CTYPES + if (all_threads && _RPython_ThreadLocals_AcquireTimeout(10000) == 0) { + /* This is known not to be perfectly safe against segfaults if we + don't hold the GIL ourselves. Too bad. I suspect that CPython + has issues there too. + */ + struct pypy_threadlocal_s *my, *p; + int blankline = 0; + char buf[40]; + + my = (struct pypy_threadlocal_s *)_RPy_ThreadLocals_Get(); + p = _RPython_ThreadLocals_Head(); + p = _RPython_ThreadLocals_Enum(p); + while (p != NULL) { + if (blankline) + pypy_faulthandler_write(fd, "\n"); + blankline = 1; + + pypy_faulthandler_write(fd, my == p ? "Current thread" : "Thread"); + sprintf(buf, " 0x%lx", (unsigned long)p->thread_ident); + pypy_faulthandler_write(fd, buf); + pypy_faulthandler_write(fd, " (most recent call first):\n"); + + array_length = vmprof_get_traceback(p->vmprof_tl_stack, + my == p ? ucontext : NULL, + array_p, FRAME_DEPTH_N); + fn(fd, array_p, array_length); + + p = _RPython_ThreadLocals_Enum(p); + } + _RPython_ThreadLocals_Release(); + } + else { + pypy_faulthandler_write(fd, "Stack (most recent call first):\n"); + array_length = vmprof_get_traceback(NULL, ucontext, + array_p, FRAME_DEPTH_N); + fn(fd, array_p, array_length); + } +#else + pypy_faulthandler_write(fd, "(no traceback when untranslated)\n"); +#endif +} + +static void +faulthandler_dump_traceback(int fd, int all_threads, void *ucontext) +{ + static volatile int reentrant = 0; + + if (reentrant) + return; + reentrant = 1; + pypy_faulthandler_dump_traceback(fd, all_threads, ucontext); + reentrant = 0; +} + + +/************************************************************/ + + +#ifdef PYPY_FAULTHANDLER_LATER +#include "src/thread.h" +static struct { + int fd; + long long microseconds; + int repeat, exit; + /* The main thread always holds this lock. It is only released when + faulthandler_thread() is interrupted before this thread exits, or at + Python exit. */ + struct RPyOpaque_ThreadLock cancel_event; + /* released by child thread when joined */ + struct RPyOpaque_ThreadLock running; +} thread_later; + +static void faulthandler_thread(void) +{ +#ifndef _WIN32 + /* we don't want to receive any signal */ + sigset_t set; + sigfillset(&set); + pthread_sigmask(SIG_SETMASK, &set, NULL); +#endif + + RPyLockStatus st; + char buf[64]; + unsigned long hour, minutes, seconds, fraction; + long long t; + + do { + st = RPyThreadAcquireLockTimed(&thread_later.cancel_event, + thread_later.microseconds, 0); + if (st == RPY_LOCK_ACQUIRED) { + RPyThreadReleaseLock(&thread_later.cancel_event); + break; + } + /* Timeout => dump traceback */ + assert(st == RPY_LOCK_FAILURE); + + /* getting to know which thread holds the GIL is not as simple + * as in CPython, so for now we don't */ + + t = thread_later.microseconds; + fraction = (unsigned long)(t % 1000000); + t /= 1000000; + seconds = (unsigned long)(t % 60); + t /= 60; + minutes = (unsigned long)(t % 60); + t /= 60; + hour = (unsigned long)t; + if (fraction == 0) + sprintf(buf, "Timeout (%lu:%02lu:%02lu)!\n", + hour, minutes, seconds); + else + sprintf(buf, "Timeout (%lu:%02lu:%02lu.%06lu)!\n", + hour, minutes, seconds, fraction); + + pypy_faulthandler_write(thread_later.fd, buf); + pypy_faulthandler_dump_traceback(thread_later.fd, 1, NULL); + + if (thread_later.exit) + _exit(1); + } while (thread_later.repeat); + + /* The only way out */ + RPyThreadReleaseLock(&thread_later.running); +} + +RPY_EXTERN +char *pypy_faulthandler_dump_traceback_later(long long microseconds, int repeat, + int fd, int exit) +{ + pypy_faulthandler_cancel_dump_traceback_later(); + + thread_later.fd = fd; + thread_later.microseconds = microseconds; + thread_later.repeat = repeat; + thread_later.exit = exit; + + RPyThreadAcquireLock(&thread_later.running, 1); + + if (RPyThreadStart(&faulthandler_thread) == -1) { + RPyThreadReleaseLock(&thread_later.running); + return "unable to start watchdog thread"; + } + return NULL; +} +#endif /* PYPY_FAULTHANDLER_LATER */ + +RPY_EXTERN +void pypy_faulthandler_cancel_dump_traceback_later(void) +{ +#ifdef PYPY_FAULTHANDLER_LATER + /* Notify cancellation */ + RPyThreadReleaseLock(&thread_later.cancel_event); + + /* Wait for thread to join (or does nothing if no thread is running) */ + RPyThreadAcquireLock(&thread_later.running, 1); + RPyThreadReleaseLock(&thread_later.running); + + /* The main thread should always hold the cancel_event lock */ + RPyThreadAcquireLock(&thread_later.cancel_event, 1); +#endif /* PYPY_FAULTHANDLER_LATER */ +} + + +/************************************************************/ + + +#ifdef PYPY_FAULTHANDLER_USER +typedef struct { + int enabled; + int fd; + int all_threads; + int chain; + _Py_sighandler_t previous; +} user_signal_t; + +static user_signal_t *user_signals; + +#ifndef NSIG +# if defined(_NSIG) +# define NSIG _NSIG /* For BSD/SysV */ +# elif defined(_SIGMAX) +# define NSIG (_SIGMAX + 1) /* For QNX */ +# elif defined(SIGMAX) +# define NSIG (SIGMAX + 1) /* For djgpp */ +# else +# define NSIG 64 /* Use a reasonable default value */ +# endif +#endif + +static void faulthandler_user(int signum, siginfo_t *info, void *ucontext); + +static int +faulthandler_register(int signum, int chain, _Py_sighandler_t *p_previous) +{ + struct sigaction action; + action.sa_handler = faulthandler_user; + sigemptyset(&action.sa_mask); + /* if the signal is received while the kernel is executing a system + call, try to restart the system call instead of interrupting it and + return EINTR. */ + action.sa_flags = SA_RESTART | SA_SIGINFO; + if (chain) { + /* do not prevent the signal from being received from within its + own signal handler */ + action.sa_flags = SA_NODEFER; + } + if (stack.ss_sp != NULL) { + /* Call the signal handler on an alternate signal stack + provided by sigaltstack() */ + action.sa_flags |= SA_ONSTACK; + } + return sigaction(signum, &action, p_previous); +} + +static void faulthandler_user(int signum, siginfo_t *info, void *ucontext) +{ + int save_errno; + user_signal_t *user = &user_signals[signum]; + + if (!user->enabled) + return; + + save_errno = errno; + faulthandler_dump_traceback(user->fd, user->all_threads, ucontext); + + if (user->chain) { + (void)sigaction(signum, &user->previous, NULL); + errno = save_errno; + + /* call the previous signal handler */ + raise(signum); + + save_errno = errno; + (void)faulthandler_register(signum, user->chain, NULL); + } + + errno = save_errno; +} + +RPY_EXTERN +int pypy_faulthandler_check_signum(long signum) +{ + unsigned int i; + + for (i = 0; i < faulthandler_nsignals; i++) { + if (faulthandler_handlers[i].signum == signum) { + return -1; + } + } + if (signum < 1 || NSIG <= signum) { + return -2; + } + return 0; +} + +RPY_EXTERN +char *pypy_faulthandler_register(int signum, int fd, int all_threads, int chain) +{ + user_signal_t *user; + _Py_sighandler_t previous; + int err; + + if (user_signals == NULL) { + user_signals = malloc(NSIG * sizeof(user_signal_t)); + if (user_signals == NULL) + return "out of memory"; + memset(user_signals, 0, NSIG * sizeof(user_signal_t)); + } + + user = &user_signals[signum]; + user->fd = fd; + user->all_threads = all_threads; + user->chain = chain; + + if (!user->enabled) { + err = faulthandler_register(signum, chain, &previous); + if (err) + return strerror(errno); + + user->previous = previous; + user->enabled = 1; + } + return NULL; +} + +RPY_EXTERN +int pypy_faulthandler_unregister(int signum) +{ + user_signal_t *user; + + if (user_signals == NULL) + return 0; + + user = &user_signals[signum]; + if (user->enabled) { + user->enabled = 0; + (void)sigaction(signum, &user->previous, NULL); + user->fd = -1; + return 1; + } + else + return 0; +} +#endif /* PYPY_FAULTHANDLER_USER */ + + +/************************************************************/ + + +/* Handler for SIGSEGV, SIGFPE, SIGABRT, SIGBUS and SIGILL signals. + + Display the current Python traceback, restore the previous handler and call + the previous handler. + + On Windows, don't explicitly call the previous handler, because the Windows + signal handler would not be called (for an unknown reason). The execution of + the program continues at faulthandler_fatal_error() exit, but the same + instruction will raise the same fault (signal), and so the previous handler + will be called. + + This function is signal-safe and should only call signal-safe functions. */ + +static void +faulthandler_fatal_error(int signum, siginfo_t *info, void *ucontext) +{ + int fd = fatal_error.fd; + int i; + fault_handler_t *handler = NULL; + int save_errno = errno; + + for (i = 0; i < faulthandler_nsignals; i++) { + handler = &faulthandler_handlers[i]; + if (handler->signum == signum) + break; + } + /* If not found, we use the SIGSEGV handler (the last one in the list) */ + + /* restore the previous handler */ + if (handler->enabled) { + (void)sigaction(signum, &handler->previous, NULL); + handler->enabled = 0; + } + + pypy_faulthandler_write(fd, "Fatal Python error: "); + pypy_faulthandler_write(fd, handler->name); + pypy_faulthandler_write(fd, "\n\n"); + + faulthandler_dump_traceback(fd, fatal_error.all_threads, ucontext); + + errno = save_errno; +#ifdef MS_WINDOWS + if (signum == SIGSEGV) { + /* don't explicitly call the previous handler for SIGSEGV in this signal + handler, because the Windows signal handler would not be called */ + return; + } +#endif + /* call the previous signal handler: it is called immediately if we use + sigaction() thanks to SA_NODEFER flag, otherwise it is deferred */ + raise(signum); +} + + +RPY_EXTERN +char *pypy_faulthandler_setup(pypy_faulthandler_cb_t dump_callback) +{ + if (fatal_error.initialized) + return NULL; + assert(!fatal_error.enabled); + fatal_error.dump_traceback = dump_callback; + + /* Try to allocate an alternate stack for faulthandler() signal handler to + * be able to allocate memory on the stack, even on a stack overflow. If it + * fails, ignore the error. */ + stack.ss_flags = 0; + stack.ss_size = SIGSTKSZ; + stack.ss_sp = malloc(stack.ss_size); + if (stack.ss_sp != NULL) { + int err = sigaltstack(&stack, NULL); + if (err) { + free(stack.ss_sp); + stack.ss_sp = NULL; + } + } + +#ifdef PYPY_FAULTHANDLER_LATER + if (!RPyThreadLockInit(&thread_later.cancel_event) || + !RPyThreadLockInit(&thread_later.running)) + return "failed to initialize locks"; + RPyThreadAcquireLock(&thread_later.cancel_event, 1); +#endif + + fatal_error.fd = -1; + fatal_error.initialized = 1; + + return NULL; +} + +RPY_EXTERN +void pypy_faulthandler_teardown(void) +{ + if (fatal_error.initialized) { + +#ifdef PYPY_FAULTHANDLER_LATER + pypy_faulthandler_cancel_dump_traceback_later(); + RPyThreadReleaseLock(&thread_later.cancel_event); + RPyOpaqueDealloc_ThreadLock(&thread_later.running); + RPyOpaqueDealloc_ThreadLock(&thread_later.cancel_event); +#endif + +#ifdef PYPY_FAULTHANDLER_USER + int signum; + for (signum = 0; signum < NSIG; signum++) + pypy_faulthandler_unregister(signum); + /* don't free 'user_signals', the gain is very minor and it can + lead to rare crashes if another thread is still busy */ +#endif + + pypy_faulthandler_disable(); + fatal_error.initialized = 0; + if (stack.ss_sp) { + stack.ss_flags = SS_DISABLE; + sigaltstack(&stack, NULL); + free(stack.ss_sp); + stack.ss_sp = NULL; + } + } +} + +RPY_EXTERN +char *pypy_faulthandler_enable(int fd, int all_threads) +{ + /* Install the handler for fatal signals, faulthandler_fatal_error(). */ + int i; + fatal_error.fd = fd; + fatal_error.all_threads = all_threads; + + if (!fatal_error.enabled) { + fatal_error.enabled = 1; + + for (i = 0; i < faulthandler_nsignals; i++) { + int err; + struct sigaction action; + fault_handler_t *handler = &faulthandler_handlers[i]; + + action.sa_sigaction = faulthandler_fatal_error; + sigemptyset(&action.sa_mask); + /* Do not prevent the signal from being received from within + its own signal handler */ + action.sa_flags = SA_NODEFER | SA_SIGINFO; + if (stack.ss_sp != NULL) { + /* Call the signal handler on an alternate signal stack + provided by sigaltstack() */ + action.sa_flags |= SA_ONSTACK; + } + err = sigaction(handler->signum, &action, &handler->previous); + if (err) { + return strerror(errno); + } + handler->enabled = 1; + } + } + return NULL; +} + +RPY_EXTERN +void pypy_faulthandler_disable(void) +{ + int i; + if (fatal_error.enabled) { + fatal_error.enabled = 0; + for (i = 0; i < faulthandler_nsignals; i++) { + fault_handler_t *handler = &faulthandler_handlers[i]; + if (!handler->enabled) + continue; + (void)sigaction(handler->signum, &handler->previous, NULL); + handler->enabled = 0; + } + } + fatal_error.fd = -1; +} + +RPY_EXTERN +int pypy_faulthandler_is_enabled(void) +{ + return fatal_error.enabled; +} + + +/************************************************************/ + + +/* for tests... */ + +static void +faulthandler_suppress_crash_report(void) +{ +#ifdef MS_WINDOWS + UINT mode; + + /* Configure Windows to not display the Windows Error Reporting dialog */ + mode = SetErrorMode(SEM_NOGPFAULTERRORBOX); + SetErrorMode(mode | SEM_NOGPFAULTERRORBOX); +#endif + +#ifndef MS_WINDOWS + struct rlimit rl; + + /* Disable creation of core dump */ + if (getrlimit(RLIMIT_CORE, &rl) != 0) { + rl.rlim_cur = 0; + setrlimit(RLIMIT_CORE, &rl); + } +#endif + +#ifdef _MSC_VER + /* Visual Studio: configure abort() to not display an error message nor + open a popup asking to report the fault. */ + _set_abort_behavior(0, _WRITE_ABORT_MSG | _CALL_REPORTFAULT); +#endif +} + +RPY_EXTERN +int pypy_faulthandler_read_null(void) +{ + int *volatile x; + + faulthandler_suppress_crash_report(); + x = NULL; + return *x; +} + +RPY_EXTERN +void pypy_faulthandler_sigsegv(void) +{ + faulthandler_suppress_crash_report(); #if defined(MS_WINDOWS) /* For SIGSEGV, faulthandler_fatal_error() restores the previous signal handler and then gives back the execution flow to the program (without @@ -34,12 +639,13 @@ #endif } -int -pypy_faulthandler_sigfpe(void) +RPY_EXTERN +int pypy_faulthandler_sigfpe(void) { /* Do an integer division by zero: raise a SIGFPE on Intel CPU, but not on PowerPC. Use volatile to disable compile-time optimizations. */ volatile int x = 1, y = 0, z; + faulthandler_suppress_crash_report(); z = x / y; /* If the division by zero didn't raise a SIGFPE (e.g. on PowerPC), raise it manually. */ @@ -49,29 +655,25 @@ return z; } -void -pypy_faulthandler_sigabrt() +RPY_EXTERN +void pypy_faulthandler_sigabrt(void) { -#ifdef _MSC_VER - /* Visual Studio: configure abort() to not display an error message nor - open a popup asking to report the fault. */ - _set_abort_behavior(0, _WRITE_ABORT_MSG | _CALL_REPORTFAULT); -#endif + faulthandler_suppress_crash_report(); abort(); } -#ifdef SIGBUS -void -pypy_faulthandler_sigbus(void) +static double fh_stack_overflow(double levels) { - raise(SIGBUS); + if (levels > 2.5) { + return (sqrt(fh_stack_overflow(levels - 1.0)) + + fh_stack_overflow(levels * 1e-10)); + } + return 1e100 + levels; } -#endif -#ifdef SIGILL -void -pypy_faulthandler_sigill(void) +RPY_EXTERN +double pypy_faulthandler_stackoverflow(double levels) { - raise(SIGILL); + faulthandler_suppress_crash_report(); + return fh_stack_overflow(levels); } -#endif diff --git a/pypy/module/faulthandler/faulthandler.h b/pypy/module/faulthandler/faulthandler.h --- a/pypy/module/faulthandler/faulthandler.h +++ b/pypy/module/faulthandler/faulthandler.h @@ -1,19 +1,40 @@ #ifndef PYPY_FAULTHANDLER_H #define PYPY_FAULTHANDLER_H -#include #include "src/precommondefs.h" +#include + + +typedef void (*pypy_faulthandler_cb_t)(int fd, intptr_t *array_p, + intptr_t length); + +RPY_EXTERN char *pypy_faulthandler_setup(pypy_faulthandler_cb_t dump_callback); +RPY_EXTERN void pypy_faulthandler_teardown(void); + +RPY_EXTERN char *pypy_faulthandler_enable(int fd, int all_threads); +RPY_EXTERN void pypy_faulthandler_disable(void); +RPY_EXTERN int pypy_faulthandler_is_enabled(void); + +RPY_EXTERN void pypy_faulthandler_write(int fd, const char *str); +RPY_EXTERN void pypy_faulthandler_write_int(int fd, long value); + +RPY_EXTERN void pypy_faulthandler_dump_traceback(int fd, int all_threads, + void *ucontext); + +RPY_EXTERN char *pypy_faulthandler_dump_traceback_later( + long long microseconds, int repeat, int fd, int exit); +RPY_EXTERN void pypy_faulthandler_cancel_dump_traceback_later(void); + +RPY_EXTERN int pypy_faulthandler_check_signum(long signum); +RPY_EXTERN char *pypy_faulthandler_register(int, int, int, int); +RPY_EXTERN int pypy_faulthandler_unregister(int signum); + RPY_EXTERN int pypy_faulthandler_read_null(void); RPY_EXTERN void pypy_faulthandler_sigsegv(void); RPY_EXTERN int pypy_faulthandler_sigfpe(void); -RPY_EXTERN void pypy_faulthandler_sigabrt(); -#ifdef SIGBUS -RPY_EXTERN void pypy_faulthandler_sigbus(void); -#endif +RPY_EXTERN void pypy_faulthandler_sigabrt(void); +RPY_EXTERN double pypy_faulthandler_stackoverflow(double); -#ifdef SIGILL -RPY_EXTERN void pypy_faulthandler_sigill(void); -#endif #endif /* PYPY_FAULTHANDLER_H */ diff --git a/pypy/module/faulthandler/handler.py b/pypy/module/faulthandler/handler.py new file mode 100644 --- /dev/null +++ b/pypy/module/faulthandler/handler.py @@ -0,0 +1,209 @@ +import os +from rpython.rtyper.lltypesystem import lltype, llmemory, rffi +from rpython.rlib.rposix import is_valid_fd +from rpython.rlib.rarithmetic import widen, ovfcheck_float_to_longlong +from rpython.rlib.objectmodel import keepalive_until_here +from rpython.rtyper.annlowlevel import llhelper + +from pypy.interpreter.error import OperationError, oefmt +from pypy.interpreter.error import exception_from_saved_errno +from pypy.interpreter.gateway import unwrap_spec +from pypy.module.faulthandler import cintf, dumper + + +class Handler(object): + def __init__(self, space): + "NOT_RPYTHON" + self.space = space + self._cleanup_() + + def _cleanup_(self): + self.fatal_error_w_file = None + self.dump_traceback_later_w_file = None + self.user_w_files = None + + def check_err(self, p_err): + if p_err: + raise oefmt(self.space.w_RuntimeError, 'faulthandler: %8', + rffi.charp2str(p_err)) + + def get_fileno_and_file(self, w_file): + space = self.space + if space.is_none(w_file): + w_file = space.sys.get('stderr') + if space.is_none(w_file): + raise oefmt(space.w_RuntimeError, "sys.stderr is None") + elif space.isinstance_w(w_file, space.w_int): + fd = space.c_int_w(w_file) + if fd < 0 or not is_valid_fd(fd): + raise oefmt(space.w_ValueError, + "file is not a valid file descriptor") + return fd, None + + fd = space.c_int_w(space.call_method(w_file, 'fileno')) + try: + space.call_method(w_file, 'flush') + except OperationError as e: + if e.async(space): + raise + pass # ignore flush() error + return fd, w_file + + def setup(self): + dump_callback = llhelper(cintf.DUMP_CALLBACK, dumper._dump_callback) + self.check_err(cintf.pypy_faulthandler_setup(dump_callback)) + + def enable(self, w_file, all_threads): + fileno, w_file = self.get_fileno_and_file(w_file) + self.setup() + self.fatal_error_w_file = w_file + self.check_err(cintf.pypy_faulthandler_enable( + rffi.cast(rffi.INT, fileno), + rffi.cast(rffi.INT, all_threads))) + + def disable(self): + cintf.pypy_faulthandler_disable() + self.fatal_error_w_file = None + + def is_enabled(self): + return bool(widen(cintf.pypy_faulthandler_is_enabled())) + + def dump_traceback(self, w_file, all_threads): + fileno, w_file = self.get_fileno_and_file(w_file) + self.setup() + cintf.pypy_faulthandler_dump_traceback( + rffi.cast(rffi.INT, fileno), + rffi.cast(rffi.INT, all_threads), + llmemory.NULL) + keepalive_until_here(w_file) + + def dump_traceback_later(self, timeout, repeat, w_file, exit): + space = self.space + timeout *= 1e6 + try: + microseconds = ovfcheck_float_to_longlong(timeout) + except OverflowError: + raise oefmt(space.w_OverflowError, "timeout value is too large") + if microseconds <= 0: + raise oefmt(space.w_ValueError, "timeout must be greater than 0") + fileno, w_file = self.get_fileno_and_file(w_file) + self.setup() + self.check_err(cintf.pypy_faulthandler_dump_traceback_later( + rffi.cast(rffi.LONGLONG, microseconds), + rffi.cast(rffi.INT, repeat), + rffi.cast(rffi.INT, fileno), + rffi.cast(rffi.INT, exit))) + self.dump_traceback_later_w_file = w_file + + def cancel_dump_traceback_later(self): + cintf.pypy_faulthandler_cancel_dump_traceback_later() + self.dump_traceback_later_w_file = None + + def check_signum(self, signum): + err = rffi.cast(lltype.Signed, + cintf.pypy_faulthandler_check_signum(signum)) + if err < 0: + space = self.space + if err == -1: + raise oefmt(space.w_RuntimeError, + "signal %d cannot be registered, " + "use enable() instead", signum) + else: + raise oefmt(space.w_ValueError, "signal number out of range") + + def register(self, signum, w_file, all_threads, chain): + self.check_signum(signum) + fileno, w_file = self.get_fileno_and_file(w_file) + self.setup() + self.check_err(cintf.pypy_faulthandler_register( + rffi.cast(rffi.INT, signum), + rffi.cast(rffi.INT, fileno), + rffi.cast(rffi.INT, all_threads), + rffi.cast(rffi.INT, chain))) + if self.user_w_files is None: + self.user_w_files = {} + self.user_w_files[signum] = w_file + + def unregister(self, signum): + self.check_signum(signum) + change = cintf.pypy_faulthandler_unregister( + rffi.cast(rffi.INT, signum)) + if self.user_w_files is not None: + self.user_w_files.pop(signum, None) + return rffi.cast(lltype.Signed, change) == 1 + + def finish(self): + cintf.pypy_faulthandler_teardown() + self._cleanup_() + + +def finish(space): + "Finalize the faulthandler logic (called from shutdown())" + space.fromcache(Handler).finish() + + + at unwrap_spec(all_threads=int) +def enable(space, w_file=None, all_threads=0): + "enable(file=sys.stderr, all_threads=True): enable the fault handler" + space.fromcache(Handler).enable(w_file, all_threads) + +def disable(space): + "disable(): disable the fault handler" + space.fromcache(Handler).disable() + +def is_enabled(space): + "is_enabled()->bool: check if the handler is enabled" + return space.wrap(space.fromcache(Handler).is_enabled()) + + at unwrap_spec(all_threads=int) +def dump_traceback(space, w_file=None, all_threads=0): + """dump the traceback of the current thread into file + including all threads if all_threads is True""" + space.fromcache(Handler).dump_traceback(w_file, all_threads) + + at unwrap_spec(timeout=float, repeat=int, exit=int) +def dump_traceback_later(space, timeout, repeat=0, w_file=None, exit=0): + """dump the traceback of all threads in timeout seconds, + or each timeout seconds if repeat is True. If exit is True, + call _exit(1) which is not safe.""" + space.fromcache(Handler).dump_traceback_later(timeout, repeat, w_file, exit) + +def cancel_dump_traceback_later(space): + """cancel the previous call to dump_traceback_later().""" + space.fromcache(Handler).cancel_dump_traceback_later() + + at unwrap_spec(signum=int, all_threads=int, chain=int) +def register(space, signum, w_file=None, all_threads=1, chain=0): + space.fromcache(Handler).register(signum, w_file, all_threads, chain) + + at unwrap_spec(signum=int) +def unregister(space, signum): + return space.wrap(space.fromcache(Handler).unregister(signum)) + + +# for tests... + + at unwrap_spec(release_gil=int) +def read_null(space, release_gil=0): + if release_gil: + cintf.pypy_faulthandler_read_null_releasegil() + else: + cintf.pypy_faulthandler_read_null() + + at unwrap_spec(release_gil=int) +def sigsegv(space, release_gil=0): + if release_gil: + cintf.pypy_faulthandler_sigsegv_releasegil() + else: + cintf.pypy_faulthandler_sigsegv() + +def sigfpe(space): + cintf.pypy_faulthandler_sigfpe() + +def sigabrt(space): + cintf.pypy_faulthandler_sigabrt() + + at unwrap_spec(levels=int) +def stack_overflow(space, levels=100000000): + levels = float(levels) + return space.wrap(cintf.pypy_faulthandler_stackoverflow(levels)) diff --git a/pypy/module/faulthandler/interp_faulthandler.py b/pypy/module/faulthandler/interp_faulthandler.py deleted file mode 100644 --- a/pypy/module/faulthandler/interp_faulthandler.py +++ /dev/null @@ -1,127 +0,0 @@ -import os -import py - -from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault -from rpython.rtyper.lltypesystem import lltype, rffi -from rpython.translator import cdir -from rpython.translator.tool.cbuild import ExternalCompilationInfo -from pypy.interpreter.error import OperationError, oefmt - -MAX_NTHREADS = 100 - -cwd = py.path.local(__file__).dirpath() -eci = ExternalCompilationInfo( - includes=[cwd.join('faulthandler.h')], - include_dirs=[str(cwd), cdir], - separate_module_files=[cwd.join('faulthandler.c')]) - -def llexternal(*args, **kwargs): - kwargs.setdefault('releasegil', False) - kwargs.setdefault('compilation_info', eci) - return rffi.llexternal(*args, **kwargs) - -pypy_faulthandler_read_null = llexternal( - 'pypy_faulthandler_read_null', [], lltype.Void) -pypy_faulthandler_read_null_nogil = llexternal( - 'pypy_faulthandler_read_null', [], lltype.Void, - releasegil=True) -pypy_faulthandler_sigsegv = llexternal( - 'pypy_faulthandler_sigsegv', [], lltype.Void) -pypy_faulthandler_sigfpe = llexternal( - 'pypy_faulthandler_sigfpe', [], lltype.Void) -pypy_faulthandler_sigabrt = llexternal( - 'pypy_faulthandler_sigabrt', [], lltype.Void) -pypy_faulthandler_sigbus = llexternal( - 'pypy_faulthandler_sigbus', [], lltype.Void) -pypy_faulthandler_sigill = llexternal( - 'pypy_faulthandler_sigill', [], lltype.Void) - -class FatalErrorState(object): - def __init__(self, space): - self.enabled = False - self.all_threads = True - - at unwrap_spec(w_file=WrappedDefault(None), - w_all_threads=WrappedDefault(True)) -def enable(space, w_file, w_all_threads): - state = space.fromcache(FatalErrorState) - state.enabled = True - state.all_threads = bool(space.int_w(w_all_threads)) - -def disable(space): - state = space.fromcache(FatalErrorState) - state.enabled = False - -def is_enabled(space): - return space.wrap(space.fromcache(FatalErrorState).enabled) - -def register(space, __args__): - pass - - - at unwrap_spec(w_file=WrappedDefault(None), - w_all_threads=WrappedDefault(True)) -def dump_traceback(space, w_file, w_all_threads): - current_ec = space.getexecutioncontext() - if space.int_w(w_all_threads): - ecs = space.threadlocals.getallvalues() - else: - ecs = {0: current_ec} - - if space.is_none(w_file): - w_file = space.sys.get('stderr') - fd = space.c_filedescriptor_w(w_file) - - nthreads = 0 - for thread_ident, ec in ecs.items(): - if nthreads: - os.write(fd, "\n") - if nthreads >= MAX_NTHREADS: - os.write(fd, "...\n") - break - if ec is current_ec: - os.write(fd, "Current thread 0x%x:\n" % thread_ident) - else: - os.write(fd, "Thread 0x%x:\n" % thread_ident) - - frame = ec.gettopframe() - while frame: - code = frame.pycode - lineno = frame.get_last_lineno() - if code: - os.write(fd, " File \"%s\", line %s in %s\n" % ( - code.co_filename, lineno, code.co_name)) - else: - os.write(fd, " File ???, line %s in ???\n" % ( - lineno,)) - - frame = frame.f_backref() - - - at unwrap_spec(w_release_gil=WrappedDefault(False)) -def read_null(space, w_release_gil): - if space.is_true(w_release_gil): - pypy_faulthandler_read_null_nogil() - else: - pypy_faulthandler_read_null() - -def sigsegv(): - pypy_faulthandler_sigsegv() - -def sigfpe(): - pypy_faulthandler_sigfpe() - -def sigabrt(): - pypy_faulthandler_sigabrt() - -#def sigbus(): -# pypy_faulthandler_sigbus() - -#def sigill(): -# pypy_faulthandler_sigill() - - at unwrap_spec(msg=str) -def fatal_error(space, msg): - os.write(2, "Fatal Python error: %s\n" % msg); - dump_traceback(space, space.wrap(None), space.wrap(True)) - raise RuntimeError(msg) diff --git a/pypy/module/faulthandler/test/test_faulthander.py b/pypy/module/faulthandler/test/test_faulthander.py --- a/pypy/module/faulthandler/test/test_faulthander.py +++ b/pypy/module/faulthandler/test/test_faulthander.py @@ -1,12 +1,7 @@ -from pypy.module.faulthandler import interp_faulthandler - -class TestFaultHandler: - def test_fatal_error(self, space): - raises(RuntimeError, interp_faulthandler.fatal_error, space, "Message") class AppTestFaultHandler: spaceconfig = { - "usemodules": ["faulthandler"] + "usemodules": ["faulthandler", "_vmprof"] } def test_enable(self): diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py --- a/pypy/module/sys/__init__.py +++ b/pypy/module/sys/__init__.py @@ -136,6 +136,7 @@ space.setitem(self.w_dict, space.wrap('thread_info'), thread_info) def setup_after_space_initialization(self): + "NOT_RPYTHON" space = self.space if not space.config.translating: diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py --- a/rpython/rlib/rvmprof/cintf.py +++ b/rpython/rlib/rvmprof/cintf.py @@ -56,6 +56,11 @@ [rffi.INT], lltype.Void, compilation_info=eci, _nowrapper=True) + vmprof_get_traceback = rffi.llexternal("vmprof_get_traceback", + [PVMPROFSTACK, llmemory.Address, + rffi.SIGNEDP, lltype.Signed], + lltype.Signed, compilation_info=eci, + _nowrapper=True) return CInterface(locals()) @@ -154,3 +159,9 @@ def restore_rvmprof_stack(x): vmprof_tl_stack.setraw(x) + +# +# traceback support + +def get_rvmprof_stack(): + return vmprof_tl_stack.get_or_make_raw() diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py --- a/rpython/rlib/rvmprof/rvmprof.py +++ b/rpython/rlib/rvmprof/rvmprof.py @@ -1,6 +1,6 @@ import sys, os from rpython.rlib.objectmodel import specialize, we_are_translated -from rpython.rlib import jit, rposix +from rpython.rlib import jit, rposix, rgc from rpython.rlib.rvmprof import cintf from rpython.rtyper.annlowlevel import cast_instance_to_gcref from rpython.rtyper.annlowlevel import cast_base_ptr_to_instance diff --git a/rpython/rlib/rvmprof/src/rvmprof.h b/rpython/rlib/rvmprof/src/rvmprof.h --- a/rpython/rlib/rvmprof/src/rvmprof.h +++ b/rpython/rlib/rvmprof/src/rvmprof.h @@ -1,3 +1,4 @@ +#include RPY_EXTERN char *vmprof_init(int, double, char *); RPY_EXTERN void vmprof_ignore_signals(int); @@ -8,3 +9,6 @@ RPY_EXTERN int vmprof_stack_append(void*, long); RPY_EXTERN long vmprof_stack_pop(void*); RPY_EXTERN void vmprof_stack_free(void*); +RPY_EXTERN intptr_t vmprof_get_traceback(void *, void *, intptr_t*, intptr_t); + +#define RVMPROF_TRACEBACK_ESTIMATE_N(num_entries) (2 * (num_entries) + 4) diff --git a/rpython/rlib/rvmprof/src/vmprof_common.h b/rpython/rlib/rvmprof/src/vmprof_common.h --- a/rpython/rlib/rvmprof/src/vmprof_common.h +++ b/rpython/rlib/rvmprof/src/vmprof_common.h @@ -128,3 +128,15 @@ return 0; } #endif + +RPY_EXTERN +intptr_t vmprof_get_traceback(void *stack, void *ucontext, + intptr_t *result_p, intptr_t result_length) +{ + int n; + intptr_t pc = ucontext ? GetPC((ucontext_t *)ucontext) : 0; + if (stack == NULL) + stack = get_vmprof_stack(); + n = get_stack_trace(stack, result_p, result_length - 2, pc); + return (intptr_t)n; +} diff --git a/rpython/rlib/rvmprof/test/test_traceback.py b/rpython/rlib/rvmprof/test/test_traceback.py new file mode 100644 --- /dev/null +++ b/rpython/rlib/rvmprof/test/test_traceback.py @@ -0,0 +1,113 @@ +import re +from rpython.rlib import rvmprof, jit +from rpython.rlib.rvmprof import traceback +from rpython.translator.interactive import Translation +from rpython.rtyper.lltypesystem import lltype + + +def test_direct(): + class MyCode: + pass + def get_name(mycode): + raise NotImplementedError + rvmprof.register_code_object_class(MyCode, get_name) + # + @rvmprof.vmprof_execute_code("mycode", lambda code, level: code, + _hack_update_stack_untranslated=True) + def mainloop(code, level): + if level > 0: + mainloop(code, level - 1) + else: + p, length = traceback.traceback(20) + traceback.walk_traceback(MyCode, my_callback, 42, p, length) + lltype.free(p, flavor='raw') + # + seen = [] + def my_callback(code, loc, arg): + seen.append((code, loc, arg)) + # + code1 = MyCode() + rvmprof.register_code(code1, "foo") + mainloop(code1, 2) + # + assert seen == [(code1, traceback.LOC_INTERPRETED, 42), + (code1, traceback.LOC_INTERPRETED, 42), + (code1, traceback.LOC_INTERPRETED, 42)] + +def test_compiled(): + class MyCode: + pass + def get_name(mycode): + raise NotImplementedError + rvmprof.register_code_object_class(MyCode, get_name) + + @rvmprof.vmprof_execute_code("mycode", lambda code, level: code) + def mainloop(code, level): + if level > 0: + mainloop(code, level - 1) + else: + p, length = traceback.traceback(20) + traceback.walk_traceback(MyCode, my_callback, 42, p, length) + lltype.free(p, flavor='raw') + + def my_callback(code, loc, arg): + print code, loc, arg + return 0 + + def f(argv): + code1 = MyCode() + rvmprof.register_code(code1, "foo") + mainloop(code1, 2) + return 0 + + t = Translation(f, None, gc="boehm") + t.compile_c() + stdout = t.driver.cbuilder.cmdexec('') + r = re.compile("[<]MyCode object at 0x([0-9a-f]+)[>] 0 42\n") + got = r.findall(stdout) + assert got == [got[0]] * 3 + +def test_jitted(): + class MyCode: + pass + def get_name(mycode): + raise NotImplementedError + rvmprof.register_code_object_class(MyCode, get_name) + + jitdriver = jit.JitDriver(greens=['code'], reds='auto', + is_recursive=True, + get_unique_id=lambda code: rvmprof.get_unique_id(code)) + + @rvmprof.vmprof_execute_code("mycode", lambda code, level, total_i: code) + def mainloop(code, level, total_i): + i = 20 + while i > 0: + jitdriver.jit_merge_point(code=code) + i -= 1 + if level > 0: + mainloop(code, level - 1, total_i + i) + if level == 0 and total_i == 0: + p, length = traceback.traceback(20) + traceback.walk_traceback(MyCode, my_callback, 42, p, length) + lltype.free(p, flavor='raw') + + def my_callback(code, loc, arg): + print code, loc, arg + return 0 + + def f(argv): + jit.set_param(jitdriver, "inlining", 0) + code1 = MyCode() + rvmprof.register_code(code1, "foo") + mainloop(code1, 2, 0) + return 0 + + t = Translation(f, None, gc="boehm") + t.rtype() + t.driver.pyjitpl_lltype() + t.compile_c() + stdout = t.driver.cbuilder.cmdexec('') + r = re.compile("[<]MyCode object at 0x([0-9a-f]+)[>] (\d) 42\n") + got = r.findall(stdout) + addr = got[0][0] + assert got == [(addr, '1'), (addr, '1'), (addr, '0')] diff --git a/rpython/rlib/rvmprof/traceback.py b/rpython/rlib/rvmprof/traceback.py new file mode 100644 --- /dev/null +++ b/rpython/rlib/rvmprof/traceback.py @@ -0,0 +1,65 @@ +""" +Semi-public interface to gather and print a raw traceback, e.g. +from the faulthandler module. +""" + +from rpython.rlib.rvmprof import cintf, rvmprof +from rpython.rlib.objectmodel import specialize +from rpython.rtyper.lltypesystem import lltype, llmemory, rffi + + +def traceback(estimate_number_of_entries): + """Build and return a vmprof-like traceback, as a pair (array_p, + array_length). The caller must free array_p. Not for signal handlers: + for these, call vmprof_get_traceback() from C code. + """ + _cintf = rvmprof._get_vmprof().cintf + size = estimate_number_of_entries * 2 + 4 + stack = cintf.get_rvmprof_stack() + array_p = lltype.malloc(rffi.SIGNEDP.TO, size, flavor='raw') + NULL = llmemory.NULL + array_length = _cintf.vmprof_get_traceback(stack, NULL, array_p, size) + return (array_p, array_length) + + +LOC_INTERPRETED = 0 +LOC_JITTED = 1 +LOC_JITTED_INLINED = 2 + + + at specialize.arg(0, 1) +def _traceback_one(CodeClass, callback, arg, code_id, loc): + found_code = None + if code_id != 0: + all_code_wrefs = CodeClass._vmprof_weak_list.get_all_handles() + i = len(all_code_wrefs) - 1 + while i >= 0: + code = all_code_wrefs[i]() + if code is not None and code._vmprof_unique_id == code_id: + found_code = code + break + i -= 1 + callback(found_code, loc, arg) + + at specialize.arg(0, 1) +def walk_traceback(CodeClass, callback, arg, array_p, array_length): + """Invoke 'callback(code_obj, loc, arg)' for every traceback entry. + 'code_obj' may be None if it can't be determined. 'loc' is one + of the LOC_xxx constants. + """ + i = 0 + while i < array_length - 1: + tag = array_p[i] + tagged_value = array_p[i + 1] + if tag == rvmprof.VMPROF_CODE_TAG: + loc = LOC_INTERPRETED + _traceback_one(CodeClass, callback, arg, tagged_value, loc) + elif tag == rvmprof.VMPROF_JITTED_TAG: + if i + 2 >= array_length: # skip last entry, can't determine if + break # it's LOC_JITTED_INLINED or LOC_JITTED + if array_p[i + 2] == rvmprof.VMPROF_JITTED_TAG: + loc = LOC_JITTED_INLINED + else: + loc = LOC_JITTED + _traceback_one(CodeClass, callback, arg, tagged_value, loc) + i += 2 diff --git a/rpython/translator/c/src/threadlocal.c b/rpython/translator/c/src/threadlocal.c --- a/rpython/translator/c/src/threadlocal.c +++ b/rpython/translator/c/src/threadlocal.c @@ -14,12 +14,22 @@ static int check_valid(void); +int _RPython_ThreadLocals_AcquireTimeout(int max_wait_iterations) { + while (1) { + long old_value = pypy_lock_test_and_set(&pypy_threadlocal_lock, 1); + if (old_value == 0) + break; + /* busy loop */ + if (max_wait_iterations == 0) + return -1; + if (max_wait_iterations > 0) + --max_wait_iterations; + } + assert(check_valid()); + return 0; +} void _RPython_ThreadLocals_Acquire(void) { - long old_value; - do { - old_value = pypy_lock_test_and_set(&pypy_threadlocal_lock, 1); - } while (old_value != 0); /* busy loop */ - assert(check_valid()); + _RPython_ThreadLocals_AcquireTimeout(-1); } void _RPython_ThreadLocals_Release(void) { assert(check_valid()); @@ -60,11 +70,7 @@ { /* assume that at most one pypy_threadlocal_s survived, the current one */ struct pypy_threadlocal_s *cur; -#ifdef USE___THREAD - cur = &pypy_threadlocal; -#else cur = (struct pypy_threadlocal_s *)_RPy_ThreadLocals_Get(); -#endif if (cur && cur->ready == 42) { cur->next = cur->prev = &linkedlist_head; linkedlist_head.next = linkedlist_head.prev = cur; diff --git a/rpython/translator/c/src/threadlocal.h b/rpython/translator/c/src/threadlocal.h --- a/rpython/translator/c/src/threadlocal.h +++ b/rpython/translator/c/src/threadlocal.h @@ -21,6 +21,7 @@ RPY_EXTERN void _RPython_ThreadLocals_Acquire(void); RPY_EXTERN void _RPython_ThreadLocals_Release(void); +RPY_EXTERN int _RPython_ThreadLocals_AcquireTimeout(int max_wait_iterations); /* Must acquire/release the thread-local lock around a series of calls to the following function */ @@ -48,14 +49,14 @@ #define OP_THREADLOCALREF_ADDR(r) \ do { \ - r = (char *)&pypy_threadlocal; \ + r = (void *)&pypy_threadlocal; \ if (pypy_threadlocal.ready != 42) \ r = _RPython_ThreadLocals_Build(); \ } while (0) #define _OP_THREADLOCALREF_ADDR_SIGHANDLER(r) \ do { \ - r = (char *)&pypy_threadlocal; \ + r = (void *)&pypy_threadlocal; \ if (pypy_threadlocal.ready != 42) \ r = NULL; \ } while (0) @@ -66,6 +67,8 @@ #define RPY_THREADLOCALREF_GET(FIELD) pypy_threadlocal.FIELD +#define _RPy_ThreadLocals_Get() (&pypy_threadlocal) + /* ------------------------------------------------------------ */ #else @@ -89,14 +92,14 @@ #define OP_THREADLOCALREF_ADDR(r) \ do { \ - r = (char *)_RPy_ThreadLocals_Get(); \ + r = (void *)_RPy_ThreadLocals_Get(); \ if (!r) \ r = _RPython_ThreadLocals_Build(); \ } while (0) #define _OP_THREADLOCALREF_ADDR_SIGHANDLER(r) \ do { \ - r = (char *)_RPy_ThreadLocals_Get(); \ + r = (void *)_RPy_ThreadLocals_Get(); \ } while (0) #define RPY_THREADLOCALREF_ENSURE() \ From pypy.commits at gmail.com Sat Oct 1 06:21:05 2016 From: pypy.commits at gmail.com (arigo) Date: Sat, 01 Oct 2016 03:21:05 -0700 (PDT) Subject: [pypy-commit] pypy py3k-faulthandler: hg merge default Message-ID: <57ef8e11.c186c20a.d3485.2468@mx.google.com> Author: Armin Rigo Branch: py3k-faulthandler Changeset: r87495:4cc420d0f7f8 Date: 2016-10-01 12:19 +0200 http://bitbucket.org/pypy/pypy/changeset/4cc420d0f7f8/ Log: hg merge default diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py --- a/pypy/interpreter/app_main.py +++ b/pypy/interpreter/app_main.py @@ -31,6 +31,7 @@ arg ...: arguments passed to program in sys.argv[1:] PyPy options and arguments: --info : print translation information about this PyPy executable +-X faulthandler: attempt to display tracebacks when PyPy crashes """ # Missing vs CPython: PYTHONHOME USAGE2 = """ From pypy.commits at gmail.com Sat Oct 1 06:22:38 2016 From: pypy.commits at gmail.com (arigo) Date: Sat, 01 Oct 2016 03:22:38 -0700 (PDT) Subject: [pypy-commit] pypy py3k: hg merge default Message-ID: <57ef8e6e.04141c0a.65c54.c877@mx.google.com> Author: Armin Rigo Branch: py3k Changeset: r87498:aaa04894afde Date: 2016-10-01 12:22 +0200 http://bitbucket.org/pypy/pypy/changeset/aaa04894afde/ Log: hg merge default From pypy.commits at gmail.com Sat Oct 1 09:47:57 2016 From: pypy.commits at gmail.com (arigo) Date: Sat, 01 Oct 2016 06:47:57 -0700 (PDT) Subject: [pypy-commit] pypy default: Fix test for arm Message-ID: <57efbe8d.0d8d1c0a.f33a2.1874@mx.google.com> Author: Armin Rigo Branch: Changeset: r87499:87c5d21350cd Date: 2016-10-01 15:46 +0200 http://bitbucket.org/pypy/pypy/changeset/87c5d21350cd/ Log: Fix test for arm diff --git a/rpython/jit/backend/llsupport/test/test_gc_integration.py b/rpython/jit/backend/llsupport/test/test_gc_integration.py --- a/rpython/jit/backend/llsupport/test/test_gc_integration.py +++ b/rpython/jit/backend/llsupport/test/test_gc_integration.py @@ -323,13 +323,10 @@ def test_malloc_slowpath(self): def check(frame): expected_size = 1 - idx = 0 fixed_size = self.cpu.JITFRAME_FIXED_SIZE if self.cpu.backend_name.startswith('arm'): # jitframe fixed part is larger here expected_size = 2 - idx = 1 - fixed_size -= 32 if self.cpu.backend_name.startswith('zarch') or \ self.cpu.backend_name.startswith('ppc'): # the allocation always allocates the register @@ -342,7 +339,10 @@ # registers (p0 and p1 are moved away when doing p2, but not # spilled, just moved to different registers) bits = [n for n in range(fixed_size) - if frame.jf_gcmap[idx] & (1< 1: + bits += [n for n in range(32, fixed_size) + if frame.jf_gcmap[1] & (1<<(n - 32))] assert len(bits) == 2 self.cpu = self.getcpu(check) From pypy.commits at gmail.com Sat Oct 1 13:26:45 2016 From: pypy.commits at gmail.com (amauryfa) Date: Sat, 01 Oct 2016 10:26:45 -0700 (PDT) Subject: [pypy-commit] pypy py35-getbuiltin: Good: we don't update sys.modules anymore in baseobjspace.py Message-ID: <57eff1d5.516d1c0a.cc065.5a6d@mx.google.com> Author: Amaury Forgeot d'Arc Branch: py35-getbuiltin Changeset: r87500:9bd1f653a930 Date: 2016-10-01 19:25 +0200 http://bitbucket.org/pypy/pypy/changeset/9bd1f653a930/ Log: Good: we don't update sys.modules anymore in baseobjspace.py Add a workaround for a limitation of importlib._bootstrap, which does not correctly supports builtin submodules. diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -644,10 +644,8 @@ if self.config.objspace.usemodules.cpyext: from pypy.module.cpyext.state import State self.fromcache(State).build_api(self) - w_modules = self.sys.get('modules') for name in ('sys', '_imp', '_frozen_importlib', 'builtins'): mod = self.getbuiltinmodule(name) - self.setitem(w_modules, mod.w_name, mod) for mod in self.builtin_modules.values(): mod.setup_after_space_initialization() diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py --- a/pypy/module/imp/importing.py +++ b/pypy/module/imp/importing.py @@ -10,6 +10,7 @@ from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter.baseobjspace import W_Root, CannotHaveLock from pypy.interpreter.eval import Code +from pypy.interpreter.mixedmodule import MixedModule from pypy.interpreter.pycode import PyCode from rpython.rlib import streamio, jit from rpython.rlib.streamio import StreamErrors @@ -80,7 +81,11 @@ lock.acquire_lock() if modulename in space.builtin_modules: - return space.getbuiltinmodule(modulename) + w_mod = space.getbuiltinmodule(modulename) + space.setitem(space.sys.get('modules'), + space.wrap(modulename), w_mod) + install_mixed_submodules(space, w_mod) + return w_mod ec = space.getexecutioncontext() with open(os.path.join(lib_pypy, modulename + '.py')) as fp: @@ -98,6 +103,22 @@ return w_mod +def install_mixed_submodules(space, w_mod): + # The pure-Python importlib does not support builtin submodules: + # when doing e.g. 'import pyexpat.errors': + # - to be considered a package, pyexpact.__path__ has to be set + # - the BuiltinImporter requires that pyexpat.__path__ is None + # - but at startup, _spec_from_module() tries a list(module.__path__)... + # + # In pyexpat.c, CPython explicitly adds the submodules to sys.path. + if not isinstance(w_mod, MixedModule): + return + for w_submodule in w_mod.submodules_w: + space.setitem(space.sys.get('modules'), + w_submodule.w_name, w_submodule) + install_mixed_submodules(space, w_submodule) + + class _WIN32Path(object): def __init__(self, path): self.path = path diff --git a/pypy/module/imp/interp_imp.py b/pypy/module/imp/interp_imp.py --- a/pypy/module/imp/interp_imp.py +++ b/pypy/module/imp/interp_imp.py @@ -71,7 +71,8 @@ return space.getbuiltinmodule(name, force_init=True, reuse=reuse) def exec_builtin(space, w_mod): - return # Until we really support ModuleDef + # Not a lot to do until we really support ModuleDef + importing.install_mixed_submodules(space, w_mod) def init_frozen(space, w_name): return None From pypy.commits at gmail.com Sat Oct 1 14:33:40 2016 From: pypy.commits at gmail.com (rlamy) Date: Sat, 01 Oct 2016 11:33:40 -0700 (PDT) Subject: [pypy-commit] pypy test-cpyext: Move support code to pypy.tool.cpyext.extbuild Message-ID: <57f00184.e97bc20a.32f11.cc0b@mx.google.com> Author: Ronan Lamy Branch: test-cpyext Changeset: r87501:53117faa3adf Date: 2016-10-01 19:32 +0100 http://bitbucket.org/pypy/pypy/changeset/53117faa3adf/ Log: Move support code to pypy.tool.cpyext.extbuild (This a step towards making it possible to import and run this code under Python3) diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py --- a/pypy/module/cpyext/test/test_cpyext.py +++ b/pypy/module/cpyext/test/test_cpyext.py @@ -1,14 +1,12 @@ import sys import weakref -import os -import py, pytest +import pytest -from pypy import pypydir +from pypy.tool.cpyext.extbuild import ( + SystemCompilationInfo, HERE, get_sys_info_app) from pypy.interpreter.gateway import unwrap_spec, interp2app from rpython.rtyper.lltypesystem import lltype, ll2ctypes -from rpython.translator.gensupp import uniquemodulename -from rpython.tool.udir import udir from pypy.module.cpyext import api from pypy.module.cpyext.state import State from pypy.module.cpyext.pyobject import Py_DecRef @@ -16,9 +14,6 @@ from rpython.tool import leakfinder from rpython.rlib import rawrefcount -from .support import c_compile - -HERE = py.path.local(pypydir) / 'module' / 'cpyext' / 'test' only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" @api.cpython_api([], api.PyObject) @@ -34,96 +29,6 @@ assert 'PyModule_Check' in api.FUNCTIONS assert api.FUNCTIONS['PyModule_Check'].argtypes == [api.PyObject] -def convert_sources_to_files(sources, dirname): - files = [] - for i, source in enumerate(sources): - filename = dirname / ('source_%d.c' % i) - with filename.open('w') as f: - f.write(str(source)) - files.append(filename) - return files - -class SystemCompilationInfo(object): - """Bundles all the generic information required to compile extensions. - - Note: here, 'system' means OS + target interpreter + test config + ... - """ - def __init__(self, include_extra=None, compile_extra=None, link_extra=None, - extra_libs=None, ext=None): - self.include_extra = include_extra or [] - self.compile_extra = compile_extra - self.link_extra = link_extra - self.extra_libs = extra_libs - self.ext = ext - - def compile_extension_module(self, name, include_dirs=None, - source_files=None, source_strings=None): - """ - Build an extension module and return the filename of the resulting - native code file. - - name is the name of the module, possibly including dots if it is a - module inside a package. - - Any extra keyword arguments are passed on to ExternalCompilationInfo to - build the module (so specify your source with one of those). - """ - include_dirs = include_dirs or [] - modname = name.split('.')[-1] - dirname = (udir/uniquemodulename('module')).ensure(dir=1) - if source_strings: - assert not source_files - files = convert_sources_to_files(source_strings, dirname) - source_files = files - soname = c_compile(source_files, outputfilename=str(dirname/modname), - compile_extra=self.compile_extra, - link_extra=self.link_extra, - include_dirs=self.include_extra + include_dirs, - libraries=self.extra_libs) - pydname = soname.new(purebasename=modname, ext=self.ext) - soname.rename(pydname) - return str(pydname) - - def import_module(self, name, init=None, body='', filename=None, - include_dirs=None, PY_SSIZE_T_CLEAN=False): - """ - init specifies the overall template of the module. - - if init is None, the module source will be loaded from a file in this - test directory, give a name given by the filename parameter. - - if filename is None, the module name will be used to construct the - filename. - """ - if init is not None: - code = make_source(name, init, body, PY_SSIZE_T_CLEAN) - kwds = dict(source_strings=[code]) - else: - assert not PY_SSIZE_T_CLEAN - if filename is None: - filename = name - filename = HERE / (filename + ".c") - kwds = dict(source_files=[filename]) - mod = self.compile_extension_module( - name, include_dirs=include_dirs, **kwds) - return self.load_module(mod, name) - - def import_extension(self, modname, functions, prologue="", - include_dirs=None, more_init="", PY_SSIZE_T_CLEAN=False): - body = prologue + make_methods(functions, modname) - init = """Py_InitModule("%s", methods);""" % (modname,) - if more_init: - init += more_init - return self.import_module( - name=modname, init=init, body=body, include_dirs=include_dirs, - PY_SSIZE_T_CLEAN=PY_SSIZE_T_CLEAN) - - -class ExtensionCompiler(SystemCompilationInfo): - """Extension compiler for appdirect mode""" - def load_module(space, mod, name): - import imp - return imp.load_dynamic(name, mod) class SpaceCompiler(SystemCompilationInfo): """Extension compiler for regular (untranslated PyPy) mode""" @@ -166,83 +71,6 @@ ext=get_so_extension(space)) -def get_so_suffix(): - from imp import get_suffixes, C_EXTENSION - for suffix, mode, typ in get_suffixes(): - if typ == C_EXTENSION: - return suffix - else: - raise RuntimeError("This interpreter does not define a filename " - "suffix for C extensions!") - -def get_sys_info_app(): - from distutils.sysconfig import get_python_inc - if sys.platform == 'win32': - compile_extra = ["/we4013"] - link_extra = ["/LIBPATH:" + os.path.join(sys.exec_prefix, 'libs')] - elif sys.platform == 'darwin': - compile_extra = link_extra = None - pass - elif sys.platform.startswith('linux'): - compile_extra = [ - "-O0", "-g", "-Werror=implicit-function-declaration", "-fPIC"] - link_extra = None - ext = get_so_suffix() - return ExtensionCompiler( - include_extra=[get_python_inc()], - compile_extra=compile_extra, - link_extra=link_extra, - ext=get_so_suffix()) - -def make_methods(functions, modname): - methods_table = [] - codes = [] - for funcname, flags, code in functions: - cfuncname = "%s_%s" % (modname, funcname) - methods_table.append( - "{\"%s\", %s, %s}," % (funcname, cfuncname, flags)) - func_code = """ - static PyObject* %s(PyObject* self, PyObject* args) - { - %s - } - """ % (cfuncname, code) - codes.append(func_code) - - body = "\n".join(codes) + """ - static PyMethodDef methods[] = { - %s - { NULL } - }; - """ % ('\n'.join(methods_table),) - return body - -def make_source(name, init, body, PY_SSIZE_T_CLEAN): - code = """ - %(PY_SSIZE_T_CLEAN)s - #include - /* fix for cpython 2.7 Python.h if running tests with -A - since pypy compiles with -fvisibility-hidden */ - #undef PyMODINIT_FUNC - #ifdef __GNUC__ - # define RPY_EXPORTED extern __attribute__((visibility("default"))) - #else - # define RPY_EXPORTED extern __declspec(dllexport) - #endif - #define PyMODINIT_FUNC RPY_EXPORTED void - - %(body)s - - PyMODINIT_FUNC - init%(name)s(void) { - %(init)s - } - """ % dict(name=name, init=init, body=body, - PY_SSIZE_T_CLEAN='#define PY_SSIZE_T_CLEAN' - if PY_SSIZE_T_CLEAN else '') - return code - - def freeze_refcnts(self): rawrefcount._dont_free_any_more() return #ZZZ diff --git a/pypy/module/cpyext/test/test_pyfile.py b/pypy/module/cpyext/test/test_pyfile.py --- a/pypy/module/cpyext/test/test_pyfile.py +++ b/pypy/module/cpyext/test/test_pyfile.py @@ -156,4 +156,3 @@ t_py = fid.tell() assert t_c == t_py, 'after a fread, c level ftell(fp) %d but PyFile.tell() %d' % (t_c, t_py) - diff --git a/pypy/module/cpyext/test/support.py b/pypy/tool/cpyext/extbuild.py rename from pypy/module/cpyext/test/support.py rename to pypy/tool/cpyext/extbuild.py --- a/pypy/module/cpyext/test/support.py +++ b/pypy/tool/cpyext/extbuild.py @@ -1,12 +1,157 @@ import os +import sys import py -from sys import platform +from pypy import pypydir +from rpython.tool.udir import udir +from rpython.translator.gensupp import uniquemodulename if os.name != 'nt': so_ext = 'so' else: so_ext = 'dll' +HERE = py.path.local(pypydir) / 'module' / 'cpyext' / 'test' + +class SystemCompilationInfo(object): + """Bundles all the generic information required to compile extensions. + + Note: here, 'system' means OS + target interpreter + test config + ... + """ + def __init__(self, include_extra=None, compile_extra=None, link_extra=None, + extra_libs=None, ext=None): + self.include_extra = include_extra or [] + self.compile_extra = compile_extra + self.link_extra = link_extra + self.extra_libs = extra_libs + self.ext = ext + + def compile_extension_module(self, name, include_dirs=None, + source_files=None, source_strings=None): + """ + Build an extension module and return the filename of the resulting + native code file. + + name is the name of the module, possibly including dots if it is a + module inside a package. + + Any extra keyword arguments are passed on to ExternalCompilationInfo to + build the module (so specify your source with one of those). + """ + include_dirs = include_dirs or [] + modname = name.split('.')[-1] + dirname = (udir/uniquemodulename('module')).ensure(dir=1) + if source_strings: + assert not source_files + files = convert_sources_to_files(source_strings, dirname) + source_files = files + soname = c_compile(source_files, outputfilename=str(dirname/modname), + compile_extra=self.compile_extra, + link_extra=self.link_extra, + include_dirs=self.include_extra + include_dirs, + libraries=self.extra_libs) + pydname = soname.new(purebasename=modname, ext=self.ext) + soname.rename(pydname) + return str(pydname) + + def import_module(self, name, init=None, body='', filename=None, + include_dirs=None, PY_SSIZE_T_CLEAN=False): + """ + init specifies the overall template of the module. + + if init is None, the module source will be loaded from a file in this + test directory, give a name given by the filename parameter. + + if filename is None, the module name will be used to construct the + filename. + """ + if init is not None: + code = make_source(name, init, body, PY_SSIZE_T_CLEAN) + kwds = dict(source_strings=[code]) + else: + assert not PY_SSIZE_T_CLEAN + if filename is None: + filename = name + filename = HERE / (filename + ".c") + kwds = dict(source_files=[filename]) + mod = self.compile_extension_module( + name, include_dirs=include_dirs, **kwds) + return self.load_module(mod, name) + + def import_extension(self, modname, functions, prologue="", + include_dirs=None, more_init="", PY_SSIZE_T_CLEAN=False): + body = prologue + make_methods(functions, modname) + init = """Py_InitModule("%s", methods);""" % (modname,) + if more_init: + init += more_init + return self.import_module( + name=modname, init=init, body=body, include_dirs=include_dirs, + PY_SSIZE_T_CLEAN=PY_SSIZE_T_CLEAN) + +class ExtensionCompiler(SystemCompilationInfo): + """Extension compiler for appdirect mode""" + def load_module(space, mod, name): + import imp + return imp.load_dynamic(name, mod) + +def convert_sources_to_files(sources, dirname): + files = [] + for i, source in enumerate(sources): + filename = dirname / ('source_%d.c' % i) + with filename.open('w') as f: + f.write(str(source)) + files.append(filename) + return files + + +def make_methods(functions, modname): + methods_table = [] + codes = [] + for funcname, flags, code in functions: + cfuncname = "%s_%s" % (modname, funcname) + methods_table.append( + "{\"%s\", %s, %s}," % (funcname, cfuncname, flags)) + func_code = """ + static PyObject* %s(PyObject* self, PyObject* args) + { + %s + } + """ % (cfuncname, code) + codes.append(func_code) + + body = "\n".join(codes) + """ + static PyMethodDef methods[] = { + %s + { NULL } + }; + """ % ('\n'.join(methods_table),) + return body + +def make_source(name, init, body, PY_SSIZE_T_CLEAN): + code = """ + %(PY_SSIZE_T_CLEAN)s + #include + /* fix for cpython 2.7 Python.h if running tests with -A + since pypy compiles with -fvisibility-hidden */ + #undef PyMODINIT_FUNC + #ifdef __GNUC__ + # define RPY_EXPORTED extern __attribute__((visibility("default"))) + #else + # define RPY_EXPORTED extern __declspec(dllexport) + #endif + #define PyMODINIT_FUNC RPY_EXPORTED void + + %(body)s + + PyMODINIT_FUNC + init%(name)s(void) { + %(init)s + } + """ % dict(name=name, init=init, body=body, + PY_SSIZE_T_CLEAN='#define PY_SSIZE_T_CLEAN' + if PY_SSIZE_T_CLEAN else '') + return code + + def c_compile(cfilenames, outputfilename, compile_extra=None, link_extra=None, include_dirs=None, libraries=None, library_dirs=None): @@ -15,9 +160,9 @@ include_dirs = include_dirs or [] libraries = libraries or [] library_dirs = library_dirs or [] - if platform == 'win32': + if sys.platform == 'win32': link_extra = link_extra + ['/DEBUG'] # generate .pdb file - if platform == 'darwin': + if sys.platform == 'darwin': # support Fink & Darwinports for s in ('/sw/', '/opt/local/'): if (s + 'include' not in include_dirs @@ -66,3 +211,30 @@ libraries=libraries, extra_preargs=link_extra, library_dirs=library_dirs) + +def get_so_suffix(): + from imp import get_suffixes, C_EXTENSION + for suffix, mode, typ in get_suffixes(): + if typ == C_EXTENSION: + return suffix + else: + raise RuntimeError("This interpreter does not define a filename " + "suffix for C extensions!") + +def get_sys_info_app(): + from distutils.sysconfig import get_python_inc + if sys.platform == 'win32': + compile_extra = ["/we4013"] + link_extra = ["/LIBPATH:" + os.path.join(sys.exec_prefix, 'libs')] + elif sys.platform == 'darwin': + compile_extra = link_extra = None + pass + elif sys.platform.startswith('linux'): + compile_extra = [ + "-O0", "-g", "-Werror=implicit-function-declaration", "-fPIC"] + link_extra = None + return ExtensionCompiler( + include_extra=[get_python_inc()], + compile_extra=compile_extra, + link_extra=link_extra, + ext=get_so_suffix()) From pypy.commits at gmail.com Sat Oct 1 15:11:05 2016 From: pypy.commits at gmail.com (amauryfa) Date: Sat, 01 Oct 2016 12:11:05 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: With the dict unpacking syntax {1:2, **dict, 3:4} ast.Dict.keys can contain null values. Message-ID: <57f00a49.8d951c0a.bea7a.a027@mx.google.com> Author: Amaury Forgeot d'Arc Branch: py3.5 Changeset: r87502:a45e290ae6ee Date: 2016-10-01 21:10 +0200 http://bitbucket.org/pypy/pypy/changeset/a45e290ae6ee/ Log: With the dict unpacking syntax {1:2, **dict, 3:4} ast.Dict.keys can contain null values. Fix a segfault in test_ast. diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py --- a/pypy/interpreter/astcompiler/ast.py +++ b/pypy/interpreter/astcompiler/ast.py @@ -1963,7 +1963,7 @@ if self.keys is None: keys_w = [] else: - keys_w = [node.to_object(space) for node in self.keys] # expr + keys_w = [node.to_object(space) if node is not None else space.w_None for node in self.keys] # expr w_keys = space.newlist(keys_w) space.setattr(w_node, space.wrap('keys'), w_keys) if self.values is None: diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py --- a/pypy/interpreter/astcompiler/tools/asdl_py.py +++ b/pypy/interpreter/astcompiler/tools/asdl_py.py @@ -136,9 +136,13 @@ return "space.wrap(%s)" % (value,) else: wrapper = "%s.to_object(space)" % (value,) - # XXX: kw_defaults, unlike other sequences, allows None - # values - if field.opt or field.name == 'kw_defaults': + allow_none = field.opt + # Some sequences allow None values: + # - arguments.kw_defaults (for mandatory kw-only arguments) + # - Dict.keys (for **nested_dict elements) + if field.name in ('kw_defaults', 'keys'): + allow_none = True + if allow_none: wrapper += " if %s is not None else space.w_None" % (value,) return wrapper diff --git a/pypy/module/_ast/test/test_ast.py b/pypy/module/_ast/test/test_ast.py --- a/pypy/module/_ast/test/test_ast.py +++ b/pypy/module/_ast/test/test_ast.py @@ -478,3 +478,6 @@ comp = ast.Compare(left, [ast.In()], [ast.Num("blah")]) _expr(comp, "non-numeric", exc=TypeError) + def test_dict_unpacking(self): + self.get_ast("{**{1:2}, 2:3}") + From pypy.commits at gmail.com Sat Oct 1 15:14:47 2016 From: pypy.commits at gmail.com (mattip) Date: Sat, 01 Oct 2016 12:14:47 -0700 (PDT) Subject: [pypy-commit] pypy buffer-interface2: from the docs of memoryview, "If ndim is 0, strides and suboffsets must be NULL" Message-ID: <57f00b27.902d1c0a.8aabe.865c@mx.google.com> Author: Matti Picus Branch: buffer-interface2 Changeset: r87503:1112dcf14c96 Date: 2016-10-01 22:13 +0300 http://bitbucket.org/pypy/pypy/changeset/1112dcf14c96/ Log: from the docs of memoryview, "If ndim is 0, strides and suboffsets must be NULL" diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py --- a/pypy/module/cpyext/memoryobject.py +++ b/pypy/module/cpyext/memoryobject.py @@ -36,15 +36,13 @@ view.c_obj = make_ref(space, w_obj) return fill_Py_buffer(space, buf, view) -def fill_Py_buffer(space, buf, view): +def fill_Py_buffer(space, buf, view): # c_buf, c_obj have been filled in ndim = buf.getndim() view.c_len = buf.getlength() view.c_itemsize = buf.getitemsize() rffi.setintfield(view, 'c_ndim', ndim) view.c_format = rffi.cast(rffi.CCHARP, view.c__format) - view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape) - view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides) fmt = buf.getformat() n = Py_MAX_FMT - 1 # NULL terminated buffer if len(fmt) > n: @@ -54,12 +52,18 @@ n = len(fmt) for i in range(n): view.c_format[i] = fmt[i] - view.c_format[n] = '\x00' - shape = buf.getshape() - strides = buf.getstrides() - for i in range(ndim): - view.c_shape[i] = shape[i] - view.c_strides[i] = strides[i] + view.c_format[n] = '\x00' + if ndim > 0: + view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape) + view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides) + shape = buf.getshape() + strides = buf.getstrides() + for i in range(ndim): + view.c_shape[i] = shape[i] + view.c_strides[i] = strides[i] + else: + view.c_shape = lltype.nullptr(Py_ssize_tP.TO) + view.c_strides = lltype.nullptr(Py_ssize_tP.TO) view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO) view.c_internal = lltype.nullptr(rffi.VOIDP.TO) return 0 @@ -150,6 +154,6 @@ view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp(space.str_w(w_s), track_allocation=False)) rffi.setintfield(view, 'c_readonly', 1) isstr = True - fill_Py_buffer(space, w_obj.buf, view) + fill_Py_buffer(space, w_obj.buf, view) return view diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py --- a/pypy/objspace/std/memoryobject.py +++ b/pypy/objspace/std/memoryobject.py @@ -131,9 +131,13 @@ return space.newbool(bool(self.buf.readonly)) def w_get_shape(self, space): + if self.buf.getndim() == 0: + return space.w_None return space.newtuple([space.wrap(x) for x in self.buf.getshape()]) def w_get_strides(self, space): + if self.buf.getndim() == 0: + return space.w_None return space.newtuple([space.wrap(x) for x in self.buf.getstrides()]) def w_get_suboffsets(self, space): From pypy.commits at gmail.com Sat Oct 1 15:19:20 2016 From: pypy.commits at gmail.com (pjenvey) Date: Sat, 01 Oct 2016 12:19:20 -0700 (PDT) Subject: [pypy-commit] pypy vendor/stdlib: update the 2.7 stdlib to 2.7.12 Message-ID: <57f00c38.02c11c0a.927a0.8538@mx.google.com> Author: Philip Jenvey Branch: vendor/stdlib Changeset: r87504:e6575c58d3aa Date: 2016-10-01 11:14 -0700 http://bitbucket.org/pypy/pypy/changeset/e6575c58d3aa/ Log: update the 2.7 stdlib to 2.7.12 diff too long, truncating to 2000 out of 26235 lines diff --git a/lib-python/2.7/BaseHTTPServer.py b/lib-python/2.7/BaseHTTPServer.py --- a/lib-python/2.7/BaseHTTPServer.py +++ b/lib-python/2.7/BaseHTTPServer.py @@ -362,14 +362,25 @@ message = short explain = long self.log_error("code %d, message %s", code, message) - # using _quote_html to prevent Cross Site Scripting attacks (see bug #1100201) - content = (self.error_message_format % - {'code': code, 'message': _quote_html(message), 'explain': explain}) self.send_response(code, message) - self.send_header("Content-Type", self.error_content_type) self.send_header('Connection', 'close') + + # Message body is omitted for cases described in: + # - RFC7230: 3.3. 1xx, 204(No Content), 304(Not Modified) + # - RFC7231: 6.3.6. 205(Reset Content) + content = None + if code >= 200 and code not in (204, 205, 304): + # HTML encode to prevent Cross Site Scripting attacks + # (see bug #1100201) + content = (self.error_message_format % { + 'code': code, + 'message': _quote_html(message), + 'explain': explain + }) + self.send_header("Content-Type", self.error_content_type) self.end_headers() - if self.command != 'HEAD' and code >= 200 and code not in (204, 304): + + if self.command != 'HEAD' and content: self.wfile.write(content) error_message_format = DEFAULT_ERROR_MESSAGE diff --git a/lib-python/2.7/Cookie.py b/lib-python/2.7/Cookie.py --- a/lib-python/2.7/Cookie.py +++ b/lib-python/2.7/Cookie.py @@ -190,7 +190,7 @@ Backwards Compatibility ----------------------- -In order to keep compatibilty with earlier versions of Cookie.py, +In order to keep compatibility with earlier versions of Cookie.py, it is still possible to use Cookie.Cookie() to create a Cookie. In fact, this simply returns a SmartCookie. diff --git a/lib-python/2.7/SimpleHTTPServer.py b/lib-python/2.7/SimpleHTTPServer.py --- a/lib-python/2.7/SimpleHTTPServer.py +++ b/lib-python/2.7/SimpleHTTPServer.py @@ -167,9 +167,9 @@ words = filter(None, words) path = os.getcwd() for word in words: - drive, word = os.path.splitdrive(word) - head, word = os.path.split(word) - if word in (os.curdir, os.pardir): continue + if os.path.dirname(word) or word in (os.curdir, os.pardir): + # Ignore components that are not a simple file/directory name + continue path = os.path.join(path, word) if trailing_slash: path += '/' diff --git a/lib-python/2.7/SocketServer.py b/lib-python/2.7/SocketServer.py --- a/lib-python/2.7/SocketServer.py +++ b/lib-python/2.7/SocketServer.py @@ -121,11 +121,6 @@ # Author of the BaseServer patch: Luke Kenneth Casson Leighton -# XXX Warning! -# There is a test suite for this module, but it cannot be run by the -# standard regression test. -# To run it manually, run Lib/test/test_socketserver.py. - __version__ = "0.4" @@ -296,6 +291,8 @@ except: self.handle_error(request, client_address) self.shutdown_request(request) + else: + self.shutdown_request(request) def handle_timeout(self): """Called if no new request arrives within self.timeout. @@ -642,7 +639,7 @@ client address as self.client_address, and the server (in case it needs access to per-server information) as self.server. Since a separate instance is created for each request, the handle() method - can define arbitrary other instance variariables. + can define other arbitrary instance variables. """ @@ -710,7 +707,7 @@ try: self.wfile.flush() except socket.error: - # An final socket error may have occurred here, such as + # A final socket error may have occurred here, such as # the local error ECONNABORTED. pass self.wfile.close() @@ -719,9 +716,6 @@ class DatagramRequestHandler(BaseRequestHandler): - # XXX Regrettably, I cannot get this working on Linux; - # s.recvfrom() doesn't return a meaningful client address. - """Define self.rfile and self.wfile for datagram sockets.""" def setup(self): diff --git a/lib-python/2.7/_LWPCookieJar.py b/lib-python/2.7/_LWPCookieJar.py --- a/lib-python/2.7/_LWPCookieJar.py +++ b/lib-python/2.7/_LWPCookieJar.py @@ -49,7 +49,7 @@ class LWPCookieJar(FileCookieJar): """ The LWPCookieJar saves a sequence of "Set-Cookie3" lines. - "Set-Cookie3" is the format used by the libwww-perl libary, not known + "Set-Cookie3" is the format used by the libwww-perl library, not known to be compatible with any browser, but which is easy to read and doesn't lose information about RFC 2965 cookies. diff --git a/lib-python/2.7/_osx_support.py b/lib-python/2.7/_osx_support.py --- a/lib-python/2.7/_osx_support.py +++ b/lib-python/2.7/_osx_support.py @@ -151,13 +151,13 @@ # can only be found inside Xcode.app if the "Command Line Tools" # are not installed. # - # Futhermore, the compiler that can be used varies between + # Furthermore, the compiler that can be used varies between # Xcode releases. Up to Xcode 4 it was possible to use 'gcc-4.2' # as the compiler, after that 'clang' should be used because # gcc-4.2 is either not present, or a copy of 'llvm-gcc' that # miscompiles Python. - # skip checks if the compiler was overriden with a CC env variable + # skip checks if the compiler was overridden with a CC env variable if 'CC' in os.environ: return _config_vars @@ -193,7 +193,7 @@ if cc != oldcc: # Found a replacement compiler. # Modify config vars using new compiler, if not already explicitly - # overriden by an env variable, preserving additional arguments. + # overridden by an env variable, preserving additional arguments. for cv in _COMPILER_CONFIG_VARS: if cv in _config_vars and cv not in os.environ: cv_split = _config_vars[cv].split() @@ -207,7 +207,7 @@ """Remove all universal build arguments from config vars""" for cv in _UNIVERSAL_CONFIG_VARS: - # Do not alter a config var explicitly overriden by env var + # Do not alter a config var explicitly overridden by env var if cv in _config_vars and cv not in os.environ: flags = _config_vars[cv] flags = re.sub('-arch\s+\w+\s', ' ', flags) @@ -228,7 +228,7 @@ # build extensions on OSX 10.7 and later with the prebuilt # 32-bit installer on the python.org website. - # skip checks if the compiler was overriden with a CC env variable + # skip checks if the compiler was overridden with a CC env variable if 'CC' in os.environ: return _config_vars @@ -244,7 +244,7 @@ # across Xcode and compiler versions, there is no reliable way # to be sure why it failed. Assume here it was due to lack of # PPC support and remove the related '-arch' flags from each - # config variables not explicitly overriden by an environment + # config variables not explicitly overridden by an environment # variable. If the error was for some other reason, we hope the # failure will show up again when trying to compile an extension # module. @@ -292,7 +292,7 @@ sdk = m.group(1) if not os.path.exists(sdk): for cv in _UNIVERSAL_CONFIG_VARS: - # Do not alter a config var explicitly overriden by env var + # Do not alter a config var explicitly overridden by env var if cv in _config_vars and cv not in os.environ: flags = _config_vars[cv] flags = re.sub(r'-isysroot\s+\S+(?:\s|$)', ' ', flags) diff --git a/lib-python/2.7/_pyio.py b/lib-python/2.7/_pyio.py --- a/lib-python/2.7/_pyio.py +++ b/lib-python/2.7/_pyio.py @@ -277,8 +277,9 @@ may raise a IOError when operations they do not support are called. The basic type used for binary data read from or written to a file is - bytes. bytearrays are accepted too, and in some cases (such as - readinto) needed. Text I/O classes work with str data. + the bytes type. Method arguments may also be bytearray or memoryview of + arrays of bytes. In some cases, such as readinto, a writable object such + as bytearray is required. Text I/O classes work with unicode data. Note that calling any method (even inquiries) on a closed stream is undefined. Implementations may raise IOError in this case. @@ -420,7 +421,7 @@ return self.__closed def _checkClosed(self, msg=None): - """Internal: raise an ValueError if file is closed + """Internal: raise a ValueError if file is closed """ if self.closed: raise ValueError("I/O operation on closed file." @@ -649,7 +650,6 @@ Raises BlockingIOError if the underlying raw stream has no data at the moment. """ - # XXX This ought to work with anything that supports the buffer API data = self.read(len(b)) n = len(data) try: @@ -664,8 +664,7 @@ def write(self, b): """Write the given buffer to the IO stream. - Return the number of bytes written, which is never less than - len(b). + Return the number of bytes written, which is always len(b). Raises BlockingIOError if the buffer is full and the underlying raw stream cannot accept more data at the moment. @@ -997,7 +996,7 @@ break avail += len(chunk) chunks.append(chunk) - # n is more then avail only when an EOF occurred or when + # n is more than avail only when an EOF occurred or when # read() would have blocked. n = min(n, avail) out = b"".join(chunks) @@ -2022,7 +2021,7 @@ def __repr__(self): # TextIOWrapper tells the encoding in its repr. In StringIO, - # that's a implementation detail. + # that's an implementation detail. return object.__repr__(self) @property diff --git a/lib-python/2.7/_strptime.py b/lib-python/2.7/_strptime.py --- a/lib-python/2.7/_strptime.py +++ b/lib-python/2.7/_strptime.py @@ -75,6 +75,8 @@ self.__calc_date_time() if _getlang() != self.lang: raise ValueError("locale changed during initialization") + if time.tzname != self.tzname or time.daylight != self.daylight: + raise ValueError("timezone changed during initialization") def __pad(self, seq, front): # Add '' to seq to either the front (is True), else the back. @@ -159,15 +161,17 @@ def __calc_timezone(self): # Set self.timezone by using time.tzname. - # Do not worry about possibility of time.tzname[0] == timetzname[1] - # and time.daylight; handle that in strptime . + # Do not worry about possibility of time.tzname[0] == time.tzname[1] + # and time.daylight; handle that in strptime. try: time.tzset() except AttributeError: pass - no_saving = frozenset(["utc", "gmt", time.tzname[0].lower()]) - if time.daylight: - has_saving = frozenset([time.tzname[1].lower()]) + self.tzname = time.tzname + self.daylight = time.daylight + no_saving = frozenset(["utc", "gmt", self.tzname[0].lower()]) + if self.daylight: + has_saving = frozenset([self.tzname[1].lower()]) else: has_saving = frozenset() self.timezone = (no_saving, has_saving) @@ -296,12 +300,15 @@ """Return a time struct based on the input string and the format string.""" global _TimeRE_cache, _regex_cache with _cache_lock: - if _getlang() != _TimeRE_cache.locale_time.lang: + locale_time = _TimeRE_cache.locale_time + if (_getlang() != locale_time.lang or + time.tzname != locale_time.tzname or + time.daylight != locale_time.daylight): _TimeRE_cache = TimeRE() _regex_cache.clear() + locale_time = _TimeRE_cache.locale_time if len(_regex_cache) > _CACHE_MAX_SIZE: _regex_cache.clear() - locale_time = _TimeRE_cache.locale_time format_regex = _regex_cache.get(format) if not format_regex: try: @@ -438,6 +445,10 @@ week_starts_Mon = True if week_of_year_start == 0 else False julian = _calc_julian_from_U_or_W(year, week_of_year, weekday, week_starts_Mon) + if julian <= 0: + year -= 1 + yday = 366 if calendar.isleap(year) else 365 + julian += yday # Cannot pre-calculate datetime_date() since can change in Julian # calculation and thus could have different value for the day of the week # calculation. diff --git a/lib-python/2.7/base64.py b/lib-python/2.7/base64.py --- a/lib-python/2.7/base64.py +++ b/lib-python/2.7/base64.py @@ -64,9 +64,10 @@ length 2 (additional characters are ignored) which specifies the alternative alphabet used instead of the '+' and '/' characters. - The decoded string is returned. A TypeError is raised if s were - incorrectly padded or if there are non-alphabet characters present in the - string. + The decoded string is returned. A TypeError is raised if s is + incorrectly padded. Characters that are neither in the normal base-64 + alphabet nor the alternative alphabet are discarded prior to the padding + check. """ if altchars is not None: s = s.translate(string.maketrans(altchars[:2], '+/')) @@ -87,9 +88,10 @@ def standard_b64decode(s): """Decode a string encoded with the standard Base64 alphabet. - s is the string to decode. The decoded string is returned. A TypeError - is raised if the string is incorrectly padded or if there are non-alphabet - characters present in the string. + Argument s is the string to decode. The decoded string is returned. A + TypeError is raised if the string is incorrectly padded. Characters that + are not in the standard alphabet are discarded prior to the padding + check. """ return b64decode(s) @@ -97,19 +99,20 @@ _urlsafe_decode_translation = string.maketrans(b'-_', b'+/') def urlsafe_b64encode(s): - """Encode a string using a url-safe Base64 alphabet. + """Encode a string using the URL- and filesystem-safe Base64 alphabet. - s is the string to encode. The encoded string is returned. The alphabet - uses '-' instead of '+' and '_' instead of '/'. + Argument s is the string to encode. The encoded string is returned. The + alphabet uses '-' instead of '+' and '_' instead of '/'. """ return b64encode(s).translate(_urlsafe_encode_translation) def urlsafe_b64decode(s): - """Decode a string encoded with the standard Base64 alphabet. + """Decode a string using the URL- and filesystem-safe Base64 alphabet. - s is the string to decode. The decoded string is returned. A TypeError - is raised if the string is incorrectly padded or if there are non-alphabet - characters present in the string. + Argument s is the string to decode. The decoded string is returned. A + TypeError is raised if the string is incorrectly padded. Characters that + are not in the URL-safe base-64 alphabet, and are not a plus '+' or slash + '/', are discarded prior to the padding check. The alphabet uses '-' instead of '+' and '_' instead of '/'. """ @@ -267,7 +270,7 @@ a lowercase alphabet is acceptable as input. For security purposes, the default is False. - The decoded string is returned. A TypeError is raised if s were + The decoded string is returned. A TypeError is raised if s is incorrectly padded or if there are non-alphabet characters present in the string. """ diff --git a/lib-python/2.7/bsddb/test/test_all.py b/lib-python/2.7/bsddb/test/test_all.py --- a/lib-python/2.7/bsddb/test/test_all.py +++ b/lib-python/2.7/bsddb/test/test_all.py @@ -523,7 +523,7 @@ return path -# This path can be overriden via "set_test_path_prefix()". +# This path can be overridden via "set_test_path_prefix()". import os, os.path get_new_path.prefix=os.path.join(os.environ.get("TMPDIR", os.path.join(os.sep,"tmp")), "z-Berkeley_DB") diff --git a/lib-python/2.7/bsddb/test/test_lock.py b/lib-python/2.7/bsddb/test/test_lock.py --- a/lib-python/2.7/bsddb/test/test_lock.py +++ b/lib-python/2.7/bsddb/test/test_lock.py @@ -41,7 +41,7 @@ print "locker ID: %s" % anID lock = self.env.lock_get(anID, "some locked thing", db.DB_LOCK_WRITE) if verbose: - print "Aquired lock: %s" % lock + print "Acquired lock: %s" % lock self.env.lock_put(lock) if verbose: print "Released lock: %s" % lock @@ -158,7 +158,7 @@ for i in xrange(1000) : lock = self.env.lock_get(anID, "some locked thing", lockType) if verbose: - print "%s: Aquired %s lock: %s" % (name, lt, lock) + print "%s: Acquired %s lock: %s" % (name, lt, lock) self.env.lock_put(lock) if verbose: diff --git a/lib-python/2.7/calendar.py b/lib-python/2.7/calendar.py --- a/lib-python/2.7/calendar.py +++ b/lib-python/2.7/calendar.py @@ -142,7 +142,7 @@ def iterweekdays(self): """ - Return a iterator for one week of weekday numbers starting with the + Return an iterator for one week of weekday numbers starting with the configured first one. """ for i in range(self.firstweekday, self.firstweekday + 7): diff --git a/lib-python/2.7/cgi.py b/lib-python/2.7/cgi.py --- a/lib-python/2.7/cgi.py +++ b/lib-python/2.7/cgi.py @@ -175,7 +175,7 @@ # parse query string function called from urlparse, -# this is done in order to maintain backward compatiblity. +# this is done in order to maintain backward compatibility. def parse_qs(qs, keep_blank_values=0, strict_parsing=0): """Parse a query given as a string argument.""" diff --git a/lib-python/2.7/collections.py b/lib-python/2.7/collections.py --- a/lib-python/2.7/collections.py +++ b/lib-python/2.7/collections.py @@ -1,3 +1,15 @@ +'''This module implements specialized container datatypes providing +alternatives to Python's general purpose built-in containers, dict, +list, set, and tuple. + +* namedtuple factory function for creating tuple subclasses with named fields +* deque list-like container with fast appends and pops on either end +* Counter dict subclass for counting hashable objects +* OrderedDict dict subclass that remembers the order entries were added +* defaultdict dict subclass that calls a factory function to supply missing values + +''' + __all__ = ['Counter', 'deque', 'defaultdict', 'namedtuple', 'OrderedDict'] # For bootstrapping reasons, the collection ABCs are defined in _abcoll.py. # They should however be considered an integral part of collections.py. @@ -302,7 +314,7 @@ >>> x, y = p # unpack like a regular tuple >>> x, y (11, 22) - >>> p.x + p.y # fields also accessable by name + >>> p.x + p.y # fields also accessible by name 33 >>> d = p._asdict() # convert to a dictionary >>> d['x'] diff --git a/lib-python/2.7/copy.py b/lib-python/2.7/copy.py --- a/lib-python/2.7/copy.py +++ b/lib-python/2.7/copy.py @@ -315,7 +315,7 @@ if n > 2: state = info[2] else: - state = {} + state = None if n > 3: listiter = info[3] else: @@ -329,7 +329,7 @@ y = callable(*args) memo[id(x)] = y - if state: + if state is not None: if deep: state = deepcopy(state, memo) if hasattr(y, '__setstate__'): diff --git a/lib-python/2.7/ctypes/__init__.py b/lib-python/2.7/ctypes/__init__.py --- a/lib-python/2.7/ctypes/__init__.py +++ b/lib-python/2.7/ctypes/__init__.py @@ -1,6 +1,3 @@ -###################################################################### -# This file should be kept compatible with Python 2.3, see PEP 291. # -###################################################################### """create and manipulate C data types in Python""" import os as _os, sys as _sys @@ -386,8 +383,8 @@ return func class PyDLL(CDLL): - """This class represents the Python library itself. It allows to - access Python API functions. The GIL is not released, and + """This class represents the Python library itself. It allows + accessing Python API functions. The GIL is not released, and Python exceptions are handled correctly. """ _func_flags_ = _FUNCFLAG_CDECL | _FUNCFLAG_PYTHONAPI diff --git a/lib-python/2.7/ctypes/_endian.py b/lib-python/2.7/ctypes/_endian.py --- a/lib-python/2.7/ctypes/_endian.py +++ b/lib-python/2.7/ctypes/_endian.py @@ -1,6 +1,3 @@ -###################################################################### -# This file should be kept compatible with Python 2.3, see PEP 291. # -###################################################################### import sys from ctypes import * diff --git a/lib-python/2.7/ctypes/macholib/README.ctypes b/lib-python/2.7/ctypes/macholib/README.ctypes --- a/lib-python/2.7/ctypes/macholib/README.ctypes +++ b/lib-python/2.7/ctypes/macholib/README.ctypes @@ -1,4 +1,4 @@ -Files in this directory from from Bob Ippolito's py2app. +Files in this directory come from Bob Ippolito's py2app. License: Any components of the py2app suite may be distributed under the MIT or PSF open source licenses. diff --git a/lib-python/2.7/ctypes/macholib/__init__.py b/lib-python/2.7/ctypes/macholib/__init__.py --- a/lib-python/2.7/ctypes/macholib/__init__.py +++ b/lib-python/2.7/ctypes/macholib/__init__.py @@ -1,6 +1,3 @@ -###################################################################### -# This file should be kept compatible with Python 2.3, see PEP 291. # -###################################################################### """ Enough Mach-O to make your head spin. diff --git a/lib-python/2.7/ctypes/macholib/dyld.py b/lib-python/2.7/ctypes/macholib/dyld.py --- a/lib-python/2.7/ctypes/macholib/dyld.py +++ b/lib-python/2.7/ctypes/macholib/dyld.py @@ -1,6 +1,3 @@ -###################################################################### -# This file should be kept compatible with Python 2.3, see PEP 291. # -###################################################################### """ dyld emulation """ diff --git a/lib-python/2.7/ctypes/macholib/dylib.py b/lib-python/2.7/ctypes/macholib/dylib.py --- a/lib-python/2.7/ctypes/macholib/dylib.py +++ b/lib-python/2.7/ctypes/macholib/dylib.py @@ -1,6 +1,3 @@ -###################################################################### -# This file should be kept compatible with Python 2.3, see PEP 291. # -###################################################################### """ Generic dylib path manipulation """ diff --git a/lib-python/2.7/ctypes/macholib/framework.py b/lib-python/2.7/ctypes/macholib/framework.py --- a/lib-python/2.7/ctypes/macholib/framework.py +++ b/lib-python/2.7/ctypes/macholib/framework.py @@ -1,6 +1,3 @@ -###################################################################### -# This file should be kept compatible with Python 2.3, see PEP 291. # -###################################################################### """ Generic framework path manipulation """ diff --git a/lib-python/2.7/ctypes/test/test_arrays.py b/lib-python/2.7/ctypes/test/test_arrays.py --- a/lib-python/2.7/ctypes/test/test_arrays.py +++ b/lib-python/2.7/ctypes/test/test_arrays.py @@ -24,20 +24,24 @@ self.assertEqual(len(ia), alen) # slot values ok? - values = [ia[i] for i in range(len(init))] + values = [ia[i] for i in range(alen)] self.assertEqual(values, init) + # out-of-bounds accesses should be caught + with self.assertRaises(IndexError): ia[alen] + with self.assertRaises(IndexError): ia[-alen-1] + # change the items from operator import setitem new_values = range(42, 42+alen) [setitem(ia, n, new_values[n]) for n in range(alen)] - values = [ia[i] for i in range(len(init))] + values = [ia[i] for i in range(alen)] self.assertEqual(values, new_values) # are the items initialized to 0? ia = int_array() - values = [ia[i] for i in range(len(init))] - self.assertEqual(values, [0] * len(init)) + values = [ia[i] for i in range(alen)] + self.assertEqual(values, [0] * alen) # Too many initializers should be caught self.assertRaises(IndexError, int_array, *range(alen*2)) diff --git a/lib-python/2.7/ctypes/test/test_pointers.py b/lib-python/2.7/ctypes/test/test_pointers.py --- a/lib-python/2.7/ctypes/test/test_pointers.py +++ b/lib-python/2.7/ctypes/test/test_pointers.py @@ -53,9 +53,13 @@ # C code: # int x = 12321; # res = &x - res.contents = c_int(12321) + x = c_int(12321) + res.contents = x self.assertEqual(i.value, 54345) + x.value = -99 + self.assertEqual(res.contents.value, -99) + def test_callbacks_with_pointers(self): # a function type receiving a pointer PROTOTYPE = CFUNCTYPE(c_int, POINTER(c_int)) @@ -128,9 +132,10 @@ def test_basic(self): p = pointer(c_int(42)) - # Although a pointer can be indexed, it ha no length + # Although a pointer can be indexed, it has no length self.assertRaises(TypeError, len, p) self.assertEqual(p[0], 42) + self.assertEqual(p[0:1], [42]) self.assertEqual(p.contents.value, 42) def test_charpp(self): diff --git a/lib-python/2.7/ctypes/test/test_structures.py b/lib-python/2.7/ctypes/test/test_structures.py --- a/lib-python/2.7/ctypes/test/test_structures.py +++ b/lib-python/2.7/ctypes/test/test_structures.py @@ -326,6 +326,7 @@ "(Phone) : " "expected string or Unicode object, int found") else: + # Compatibility no longer strictly required self.assertEqual(msg, "(Phone) exceptions.TypeError: " "expected string or Unicode object, int found") diff --git a/lib-python/2.7/ctypes/test/test_values.py b/lib-python/2.7/ctypes/test/test_values.py --- a/lib-python/2.7/ctypes/test/test_values.py +++ b/lib-python/2.7/ctypes/test/test_values.py @@ -22,8 +22,7 @@ ctdll = CDLL(_ctypes_test.__file__) self.assertRaises(ValueError, c_int.in_dll, ctdll, "Undefined_Symbol") - at unittest.skipUnless(sys.platform == 'win32', 'Windows-specific test') -class Win_ValuesTestCase(unittest.TestCase): +class PythonValuesTestCase(unittest.TestCase): """This test only works when python itself is a dll/shared library""" def test_optimizeflag(self): diff --git a/lib-python/2.7/ctypes/util.py b/lib-python/2.7/ctypes/util.py --- a/lib-python/2.7/ctypes/util.py +++ b/lib-python/2.7/ctypes/util.py @@ -1,6 +1,3 @@ -###################################################################### -# This file should be kept compatible with Python 2.3, see PEP 291. # -###################################################################### import sys, os # find_library(name) returns the pathname of a library, or None. @@ -137,16 +134,13 @@ cmd = 'if ! type objdump >/dev/null 2>&1; then exit 10; fi;' \ "objdump -p -j .dynamic 2>/dev/null " + f f = os.popen(cmd) - dump = f.read() - rv = f.close() + try: + dump = f.read() + finally: + rv = f.close() if rv == 10: raise OSError, 'objdump command not found' - f = os.popen(cmd) - try: - data = f.read() - finally: - f.close() - res = re.search(r'\sSONAME\s+([^\s]+)', data) + res = re.search(r'\sSONAME\s+([^\s]+)', dump) if not res: return None return res.group(1) @@ -191,6 +185,7 @@ else: cmd = 'env LC_ALL=C /usr/bin/crle 2>/dev/null' + paths = None for line in os.popen(cmd).readlines(): line = line.strip() if line.startswith('Default Library Path (ELF):'): @@ -228,7 +223,7 @@ # XXX assuming GLIBC's ldconfig (with option -p) expr = r'\s+(lib%s\.[^\s]+)\s+\(%s' % (re.escape(name), abi_type) - f = os.popen('/sbin/ldconfig -p 2>/dev/null') + f = os.popen('LC_ALL=C LANG=C /sbin/ldconfig -p 2>/dev/null') try: data = f.read() finally: diff --git a/lib-python/2.7/ctypes/wintypes.py b/lib-python/2.7/ctypes/wintypes.py --- a/lib-python/2.7/ctypes/wintypes.py +++ b/lib-python/2.7/ctypes/wintypes.py @@ -1,7 +1,3 @@ -###################################################################### -# This file should be kept compatible with Python 2.3, see PEP 291. # -###################################################################### - # The most useful windows datatypes from ctypes import * diff --git a/lib-python/2.7/decimal.py b/lib-python/2.7/decimal.py --- a/lib-python/2.7/decimal.py +++ b/lib-python/2.7/decimal.py @@ -224,7 +224,7 @@ class ConversionSyntax(InvalidOperation): """Trying to convert badly formed string. - This occurs and signals invalid-operation if an string is being + This occurs and signals invalid-operation if a string is being converted to a number and it does not conform to the numeric string syntax. The result is [0,qNaN]. """ @@ -1082,7 +1082,7 @@ def __pos__(self, context=None): """Returns a copy, unless it is a sNaN. - Rounds the number (if more then precision digits) + Rounds the number (if more than precision digits) """ if self._is_special: ans = self._check_nans(context=context) diff --git a/lib-python/2.7/difflib.py b/lib-python/2.7/difflib.py --- a/lib-python/2.7/difflib.py +++ b/lib-python/2.7/difflib.py @@ -1487,7 +1487,7 @@ yield _make_line(lines,'-',0), None, True continue elif s.startswith(('--?+', '--+', '- ')): - # in delete block and see a intraline change or unchanged line + # in delete block and see an intraline change or unchanged line # coming: yield the delete line and then blanks from_line,to_line = _make_line(lines,'-',0), None num_blanks_to_yield,num_blanks_pending = num_blanks_pending-1,0 diff --git a/lib-python/2.7/distutils/ccompiler.py b/lib-python/2.7/distutils/ccompiler.py --- a/lib-python/2.7/distutils/ccompiler.py +++ b/lib-python/2.7/distutils/ccompiler.py @@ -842,8 +842,9 @@ def library_filename(self, libname, lib_type='static', # or 'shared' strip_dir=0, output_dir=''): assert output_dir is not None - if lib_type not in ("static", "shared", "dylib"): - raise ValueError, "'lib_type' must be \"static\", \"shared\" or \"dylib\"" + if lib_type not in ("static", "shared", "dylib", "xcode_stub"): + raise ValueError, ("""'lib_type' must be "static", "shared", """ + """"dylib", or "xcode_stub".""") fmt = getattr(self, lib_type + "_lib_format") ext = getattr(self, lib_type + "_lib_extension") diff --git a/lib-python/2.7/distutils/cygwinccompiler.py b/lib-python/2.7/distutils/cygwinccompiler.py --- a/lib-python/2.7/distutils/cygwinccompiler.py +++ b/lib-python/2.7/distutils/cygwinccompiler.py @@ -10,9 +10,9 @@ # # * if you use a msvc compiled python version (1.5.2) # 1. you have to insert a __GNUC__ section in its config.h -# 2. you have to generate a import library for its dll +# 2. you have to generate an import library for its dll # - create a def-file for python??.dll -# - create a import library using +# - create an import library using # dlltool --dllname python15.dll --def python15.def \ # --output-lib libpython15.a # diff --git a/lib-python/2.7/distutils/msvc9compiler.py b/lib-python/2.7/distutils/msvc9compiler.py --- a/lib-python/2.7/distutils/msvc9compiler.py +++ b/lib-python/2.7/distutils/msvc9compiler.py @@ -54,7 +54,7 @@ # A map keyed by get_platform() return values to values accepted by # 'vcvarsall.bat'. Note a cross-compile may combine these (eg, 'x86_amd64' is -# the param to cross-compile on x86 targetting amd64.) +# the param to cross-compile on x86 targeting amd64.) PLAT_TO_VCVARS = { 'win32' : 'x86', 'win-amd64' : 'amd64', diff --git a/lib-python/2.7/distutils/tests/test_build_ext.py b/lib-python/2.7/distutils/tests/test_build_ext.py --- a/lib-python/2.7/distutils/tests/test_build_ext.py +++ b/lib-python/2.7/distutils/tests/test_build_ext.py @@ -251,7 +251,7 @@ def test_compiler_option(self): # cmd.compiler is an option and - # should not be overriden by a compiler instance + # should not be overridden by a compiler instance # when the command is run dist = Distribution() cmd = build_ext(dist) diff --git a/lib-python/2.7/distutils/tests/test_unixccompiler.py b/lib-python/2.7/distutils/tests/test_unixccompiler.py --- a/lib-python/2.7/distutils/tests/test_unixccompiler.py +++ b/lib-python/2.7/distutils/tests/test_unixccompiler.py @@ -135,7 +135,7 @@ self.assertEqual(self.cc.linker_so[0], 'my_cc') @unittest.skipUnless(sys.platform == 'darwin', 'test only relevant for OS X') - def test_osx_explict_ldshared(self): + def test_osx_explicit_ldshared(self): # Issue #18080: # ensure that setting CC env variable does not change # explicit LDSHARED setting for linker diff --git a/lib-python/2.7/distutils/unixccompiler.py b/lib-python/2.7/distutils/unixccompiler.py --- a/lib-python/2.7/distutils/unixccompiler.py +++ b/lib-python/2.7/distutils/unixccompiler.py @@ -79,7 +79,9 @@ static_lib_extension = ".a" shared_lib_extension = ".so" dylib_lib_extension = ".dylib" + xcode_stub_lib_extension = ".tbd" static_lib_format = shared_lib_format = dylib_lib_format = "lib%s%s" + xcode_stub_lib_format = dylib_lib_format if sys.platform == "cygwin": exe_extension = ".exe" @@ -245,12 +247,28 @@ def find_library_file(self, dirs, lib, debug=0): shared_f = self.library_filename(lib, lib_type='shared') dylib_f = self.library_filename(lib, lib_type='dylib') + xcode_stub_f = self.library_filename(lib, lib_type='xcode_stub') static_f = self.library_filename(lib, lib_type='static') if sys.platform == 'darwin': # On OSX users can specify an alternate SDK using # '-isysroot', calculate the SDK root if it is specified # (and use it further on) + # + # Note that, as of Xcode 7, Apple SDKs may contain textual stub + # libraries with .tbd extensions rather than the normal .dylib + # shared libraries installed in /. The Apple compiler tool + # chain handles this transparently but it can cause problems + # for programs that are being built with an SDK and searching + # for specific libraries. Callers of find_library_file need to + # keep in mind that the base filename of the returned SDK library + # file might have a different extension from that of the library + # file installed on the running system, for example: + # /Applications/Xcode.app/Contents/Developer/Platforms/ + # MacOSX.platform/Developer/SDKs/MacOSX10.11.sdk/ + # usr/lib/libedit.tbd + # vs + # /usr/lib/libedit.dylib cflags = sysconfig.get_config_var('CFLAGS') m = re.search(r'-isysroot\s+(\S+)', cflags) if m is None: @@ -264,6 +282,7 @@ shared = os.path.join(dir, shared_f) dylib = os.path.join(dir, dylib_f) static = os.path.join(dir, static_f) + xcode_stub = os.path.join(dir, xcode_stub_f) if sys.platform == 'darwin' and ( dir.startswith('/System/') or ( @@ -272,6 +291,7 @@ shared = os.path.join(sysroot, dir[1:], shared_f) dylib = os.path.join(sysroot, dir[1:], dylib_f) static = os.path.join(sysroot, dir[1:], static_f) + xcode_stub = os.path.join(sysroot, dir[1:], xcode_stub_f) # We're second-guessing the linker here, with not much hard # data to go on: GCC seems to prefer the shared library, so I'm @@ -279,6 +299,8 @@ # ignoring even GCC's "-static" option. So sue me. if os.path.exists(dylib): return dylib + elif os.path.exists(xcode_stub): + return xcode_stub elif os.path.exists(shared): return shared elif os.path.exists(static): diff --git a/lib-python/2.7/email/message.py b/lib-python/2.7/email/message.py --- a/lib-python/2.7/email/message.py +++ b/lib-python/2.7/email/message.py @@ -579,7 +579,7 @@ message, it will be set to "text/plain" and the new parameter and value will be appended as per RFC 2045. - An alternate header can specified in the header argument, and all + An alternate header can be specified in the header argument, and all parameters will be quoted as necessary unless requote is False. If charset is specified, the parameter will be encoded according to RFC diff --git a/lib-python/2.7/email/parser.py b/lib-python/2.7/email/parser.py --- a/lib-python/2.7/email/parser.py +++ b/lib-python/2.7/email/parser.py @@ -23,7 +23,7 @@ textual representation of the message. The string must be formatted as a block of RFC 2822 headers and header - continuation lines, optionally preceeded by a `Unix-from' header. The + continuation lines, optionally preceded by a `Unix-from' header. The header block is terminated either by the end of the string or by a blank line. diff --git a/lib-python/2.7/encodings/utf_32.py b/lib-python/2.7/encodings/utf_32.py --- a/lib-python/2.7/encodings/utf_32.py +++ b/lib-python/2.7/encodings/utf_32.py @@ -68,7 +68,7 @@ self.decoder = None def getstate(self): - # additonal state info from the base class must be None here, + # additional state info from the base class must be None here, # as it isn't passed along to the caller state = codecs.BufferedIncrementalDecoder.getstate(self)[0] # additional state info we pass to the caller: diff --git a/lib-python/2.7/ensurepip/__init__.py b/lib-python/2.7/ensurepip/__init__.py --- a/lib-python/2.7/ensurepip/__init__.py +++ b/lib-python/2.7/ensurepip/__init__.py @@ -12,9 +12,9 @@ __all__ = ["version", "bootstrap"] -_SETUPTOOLS_VERSION = "18.2" +_SETUPTOOLS_VERSION = "20.10.1" -_PIP_VERSION = "7.1.2" +_PIP_VERSION = "8.1.1" # pip currently requires ssl support, so we try to provide a nicer # error message when that is missing (http://bugs.python.org/issue19744) diff --git a/lib-python/2.7/ensurepip/_bundled/pip-7.1.2-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/pip-7.1.2-py2.py3-none-any.whl deleted file mode 100644 index 5e490155f0ca7f4ddb64c93c39fb2efb8795cd08..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 GIT binary patch [cut] diff --git a/lib-python/2.7/ensurepip/_bundled/pip-8.1.1-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/pip-8.1.1-py2.py3-none-any.whl new file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..8632eb7af04c6337f0442a878ecb99cd2b1a67e0 GIT binary patch [cut] diff --git a/lib-python/2.7/ensurepip/_bundled/setuptools-18.2-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/setuptools-18.2-py2.py3-none-any.whl deleted file mode 100644 index f4288d68e074466894d8a2342e113737df7b7649..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 GIT binary patch [cut] diff --git a/lib-python/2.7/ensurepip/_bundled/setuptools-20.10.1-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/setuptools-20.10.1-py2.py3-none-any.whl new file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..9d1319a24aba103fe956ef6298e3649efacc0b93 GIT binary patch [cut] diff --git a/lib-python/2.7/fileinput.py b/lib-python/2.7/fileinput.py --- a/lib-python/2.7/fileinput.py +++ b/lib-python/2.7/fileinput.py @@ -64,13 +64,6 @@ disabled when standard input is read. XXX The current implementation does not work for MS-DOS 8+3 filesystems. -Performance: this module is unfortunately one of the slower ways of -processing large numbers of input lines. Nevertheless, a significant -speed-up has been obtained by using readlines(bufsize) instead of -readline(). A new keyword argument, bufsize=N, is present on the -input() function and the FileInput() class to override the default -buffer size. - XXX Possible additions: - optional getopt argument processing @@ -86,6 +79,7 @@ _state = None +# No longer used DEFAULT_BUFSIZE = 8*1024 def input(files=None, inplace=0, backup="", bufsize=0, @@ -207,17 +201,14 @@ self._files = files self._inplace = inplace self._backup = backup - self._bufsize = bufsize or DEFAULT_BUFSIZE self._savestdout = None self._output = None self._filename = None - self._lineno = 0 + self._startlineno = 0 self._filelineno = 0 self._file = None self._isstdin = False self._backupfilename = None - self._buffer = [] - self._bufindex = 0 # restrict mode argument to reading modes if mode not in ('r', 'rU', 'U', 'rb'): raise ValueError("FileInput opening mode must be one of " @@ -242,22 +233,18 @@ return self def next(self): - try: - line = self._buffer[self._bufindex] - except IndexError: - pass - else: - self._bufindex += 1 - self._lineno += 1 - self._filelineno += 1 - return line - line = self.readline() - if not line: - raise StopIteration - return line + while 1: + line = self._readline() + if line: + self._filelineno += 1 + return line + if not self._file: + raise StopIteration + self.nextfile() + # repeat with next file def __getitem__(self, i): - if i != self._lineno: + if i != self.lineno(): raise RuntimeError, "accessing lines out of order" try: return self.next() @@ -277,7 +264,11 @@ output.close() finally: file = self._file - self._file = 0 + self._file = None + try: + del self._readline # restore FileInput._readline + except AttributeError: + pass try: if file and not self._isstdin: file.close() @@ -289,75 +280,72 @@ except OSError: pass self._isstdin = False - self._buffer = [] - self._bufindex = 0 def readline(self): - try: - line = self._buffer[self._bufindex] - except IndexError: - pass + while 1: + line = self._readline() + if line: + self._filelineno += 1 + return line + if not self._file: + return line + self.nextfile() + # repeat with next file + + def _readline(self): + if not self._files: + return "" + self._filename = self._files[0] + self._files = self._files[1:] + self._startlineno = self.lineno() + self._filelineno = 0 + self._file = None + self._isstdin = False + self._backupfilename = 0 + if self._filename == '-': + self._filename = '' + self._file = sys.stdin + self._isstdin = True else: - self._bufindex += 1 - self._lineno += 1 - self._filelineno += 1 - return line - if not self._file: - if not self._files: - return "" - self._filename = self._files[0] - self._files = self._files[1:] - self._filelineno = 0 - self._file = None - self._isstdin = False - self._backupfilename = 0 - if self._filename == '-': - self._filename = '' - self._file = sys.stdin - self._isstdin = True + if self._inplace: + self._backupfilename = ( + self._filename + (self._backup or os.extsep+"bak")) + try: os.unlink(self._backupfilename) + except os.error: pass + # The next few lines may raise IOError + os.rename(self._filename, self._backupfilename) + self._file = open(self._backupfilename, self._mode) + try: + perm = os.fstat(self._file.fileno()).st_mode + except OSError: + self._output = open(self._filename, "w") + else: + fd = os.open(self._filename, + os.O_CREAT | os.O_WRONLY | os.O_TRUNC, + perm) + self._output = os.fdopen(fd, "w") + try: + if hasattr(os, 'chmod'): + os.chmod(self._filename, perm) + except OSError: + pass + self._savestdout = sys.stdout + sys.stdout = self._output else: - if self._inplace: - self._backupfilename = ( - self._filename + (self._backup or os.extsep+"bak")) - try: os.unlink(self._backupfilename) - except os.error: pass - # The next few lines may raise IOError - os.rename(self._filename, self._backupfilename) - self._file = open(self._backupfilename, self._mode) - try: - perm = os.fstat(self._file.fileno()).st_mode - except OSError: - self._output = open(self._filename, "w") - else: - fd = os.open(self._filename, - os.O_CREAT | os.O_WRONLY | os.O_TRUNC, - perm) - self._output = os.fdopen(fd, "w") - try: - if hasattr(os, 'chmod'): - os.chmod(self._filename, perm) - except OSError: - pass - self._savestdout = sys.stdout - sys.stdout = self._output + # This may raise IOError + if self._openhook: + self._file = self._openhook(self._filename, self._mode) else: - # This may raise IOError - if self._openhook: - self._file = self._openhook(self._filename, self._mode) - else: - self._file = open(self._filename, self._mode) - self._buffer = self._file.readlines(self._bufsize) - self._bufindex = 0 - if not self._buffer: - self.nextfile() - # Recursive call - return self.readline() + self._file = open(self._filename, self._mode) + + self._readline = self._file.readline # hide FileInput._readline + return self._readline() def filename(self): return self._filename def lineno(self): - return self._lineno + return self._startlineno + self._filelineno def filelineno(self): return self._filelineno diff --git a/lib-python/2.7/getopt.py b/lib-python/2.7/getopt.py --- a/lib-python/2.7/getopt.py +++ b/lib-python/2.7/getopt.py @@ -28,7 +28,7 @@ # - RETURN_IN_ORDER option # - GNU extension with '-' as first character of option string # - optional arguments, specified by double colons -# - a option string with a W followed by semicolon should +# - an option string with a W followed by semicolon should # treat "-W foo" as "--foo" __all__ = ["GetoptError","error","getopt","gnu_getopt"] diff --git a/lib-python/2.7/gzip.py b/lib-python/2.7/gzip.py --- a/lib-python/2.7/gzip.py +++ b/lib-python/2.7/gzip.py @@ -55,7 +55,7 @@ a file object. When fileobj is not None, the filename argument is only used to be - included in the gzip file header, which may includes the original + included in the gzip file header, which may include the original filename of the uncompressed file. It defaults to the filename of fileobj, if discernible; otherwise, it defaults to the empty string, and in this case the original filename is not included in the header. diff --git a/lib-python/2.7/heapq.py b/lib-python/2.7/heapq.py --- a/lib-python/2.7/heapq.py +++ b/lib-python/2.7/heapq.py @@ -56,7 +56,7 @@ In the tree above, each cell `k' is topping `2*k+1' and `2*k+2'. In -an usual binary tournament we see in sports, each cell is the winner +a usual binary tournament we see in sports, each cell is the winner over the two cells it tops, and we can trace the winner down the tree to see all opponents s/he had. However, in many computer applications of such tournaments, we do not need to trace the history of a winner. diff --git a/lib-python/2.7/hotshot/__init__.py b/lib-python/2.7/hotshot/__init__.py --- a/lib-python/2.7/hotshot/__init__.py +++ b/lib-python/2.7/hotshot/__init__.py @@ -72,7 +72,7 @@ Additional positional and keyword arguments may be passed along; the result of the call is returned, and exceptions are - allowed to propogate cleanly, while ensuring that profiling is + allowed to propagate cleanly, while ensuring that profiling is disabled on the way out. """ return self._prof.runcall(func, args, kw) diff --git a/lib-python/2.7/idlelib/AutoComplete.py b/lib-python/2.7/idlelib/AutoComplete.py --- a/lib-python/2.7/idlelib/AutoComplete.py +++ b/lib-python/2.7/idlelib/AutoComplete.py @@ -1,6 +1,6 @@ """AutoComplete.py - An IDLE extension for automatically completing names. -This extension can complete either attribute names of file names. It can pop +This extension can complete either attribute names or file names. It can pop a window with all available names, for the user to select from. """ import os @@ -67,7 +67,7 @@ def try_open_completions_event(self, event): """Happens when it would be nice to open a completion list, but not - really necessary, for example after an dot, so function + really necessary, for example after a dot, so function calls won't be made. """ lastchar = self.text.get("insert-1c") diff --git a/lib-python/2.7/idlelib/CREDITS.txt b/lib-python/2.7/idlelib/CREDITS.txt --- a/lib-python/2.7/idlelib/CREDITS.txt +++ b/lib-python/2.7/idlelib/CREDITS.txt @@ -24,7 +24,7 @@ integration, debugger integration and persistent breakpoints). Scott David Daniels, Tal Einat, Hernan Foffani, Christos Georgiou, -Jim Jewett, Martin v. L�wis, Jason Orendorff, Guilherme Polo, Josh Robb, +Jim Jewett, Martin v. Löwis, Jason Orendorff, Guilherme Polo, Josh Robb, Nigel Rowe, Bruce Sherwood, Jeff Shute, and Weeble have submitted useful patches. Thanks, guys! diff --git a/lib-python/2.7/idlelib/ChangeLog b/lib-python/2.7/idlelib/ChangeLog --- a/lib-python/2.7/idlelib/ChangeLog +++ b/lib-python/2.7/idlelib/ChangeLog @@ -1574,7 +1574,7 @@ * Attic/PopupMenu.py: Pass a root to the help window. * SearchBinding.py: - Add parent argument to 'to to line number' dialog box. + Add parent argument to 'go to line number' dialog box. Sat Oct 10 19:15:32 1998 Guido van Rossum diff --git a/lib-python/2.7/idlelib/Debugger.py b/lib-python/2.7/idlelib/Debugger.py --- a/lib-python/2.7/idlelib/Debugger.py +++ b/lib-python/2.7/idlelib/Debugger.py @@ -373,7 +373,7 @@ def __init__(self, master, flist, gui): if macosxSupport.isAquaTk(): # At least on with the stock AquaTk version on OSX 10.4 you'll - # get an shaking GUI that eventually kills IDLE if the width + # get a shaking GUI that eventually kills IDLE if the width # argument is specified. ScrolledList.__init__(self, master) else: diff --git a/lib-python/2.7/idlelib/IOBinding.py b/lib-python/2.7/idlelib/IOBinding.py --- a/lib-python/2.7/idlelib/IOBinding.py +++ b/lib-python/2.7/idlelib/IOBinding.py @@ -17,6 +17,8 @@ import tkMessageBox from SimpleDialog import SimpleDialog +from idlelib.configHandler import idleConf + # Try setting the locale, so that we can find out # what encoding to use try: @@ -63,7 +65,7 @@ encoding = encoding.lower() -coding_re = re.compile(r'^[ \t\f]*#.*coding[:=][ \t]*([-\w.]+)') +coding_re = re.compile(r'^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)') blank_re = re.compile(r'^[ \t\f]*(?:[#\r\n]|$)') class EncodingMessage(SimpleDialog): @@ -137,7 +139,6 @@ raise LookupError, "Unknown encoding "+name return name - class IOBinding: def __init__(self, editwin): @@ -567,7 +568,6 @@ def _io_binding(parent): # htest # from Tkinter import Toplevel, Text - from idlelib.configHandler import idleConf root = Toplevel(parent) root.title("Test IOBinding") @@ -578,15 +578,24 @@ self.text = text self.flist = None self.text.bind("", self.open) + self.text.bind('', self.printer) self.text.bind("", self.save) + self.text.bind("", self.saveas) + self.text.bind('', self.savecopy) def get_saved(self): return 0 def set_saved(self, flag): pass def reset_undo(self): pass + def update_recent_files_list(self, filename): pass def open(self, event): self.text.event_generate("<>") + def printer(self, event): + self.text.event_generate("<>") def save(self, event): self.text.event_generate("<>") - def update_recent_files_list(s, f): pass + def saveas(self, event): + self.text.event_generate("<>") + def savecopy(self, event): + self.text.event_generate("<>") text = Text(root) text.pack() diff --git a/lib-python/2.7/idlelib/MultiCall.py b/lib-python/2.7/idlelib/MultiCall.py --- a/lib-python/2.7/idlelib/MultiCall.py +++ b/lib-python/2.7/idlelib/MultiCall.py @@ -101,7 +101,7 @@ self.widget.unbind(self.widgetinst, self.sequence, self.handlerid) # An int in range(1 << len(_modifiers)) represents a combination of modifiers -# (if the least significent bit is on, _modifiers[0] is on, and so on). +# (if the least significant bit is on, _modifiers[0] is on, and so on). # _state_subsets gives for each combination of modifiers, or *state*, # a list of the states which are a subset of it. This list is ordered by the # number of modifiers is the state - the most specific state comes first. diff --git a/lib-python/2.7/idlelib/NEWS.txt b/lib-python/2.7/idlelib/NEWS.txt --- a/lib-python/2.7/idlelib/NEWS.txt +++ b/lib-python/2.7/idlelib/NEWS.txt @@ -1,3 +1,35 @@ +What's New in IDLE 2.7.12? +========================== +*Release date: 2015-06-30?* + +- Issue #5124: Paste with text selected now replaces the selection on X11. + This matches how paste works on Windows, Mac, most modern Linux apps, + and ttk widgets. Original patch by Serhiy Storchaka. + +- Issue #24759: Make clear in idlelib.idle_test.__init__ that the directory + is a private implementation of test.test_idle and tool for maintainers. + +- Issue #26673: When tk reports font size as 0, change to size 10. + Such fonts on Linux prevented the configuration dialog from opening. + +- Issue #27044: Add ConfigDialog.remove_var_callbacks to stop memory leaks. + +- In the 'IDLE-console differences' section of the IDLE doc, clarify + how running with IDLE affects sys.modules and the standard streams. + +- Issue #25507: fix incorrect change in IOBinding that prevented printing. + Change also prevented saving shell window with non-ascii characters. + Augment IOBinding htest to include all major IOBinding functions. + +- Issue #25905: Revert unwanted conversion of ' to � RIGHT SINGLE QUOTATION + MARK in README.txt and open this and NEWS.txt with 'ascii'. + Re-encode CREDITS.txt to utf-8 and open it with 'utf-8'. + +- Issue #26417: Prevent spurious errors and incorrect defaults when + installing IDLE 2.7 on OS X: default configuration settings are + no longer installed from OS X specific copies. + + What's New in IDLE 2.7.11? ========================== *Release date: 2015-12-06* @@ -162,7 +194,7 @@ move version to end. - Issue #14105: Idle debugger breakpoints no longer disappear - when inseting or deleting lines. + when inserting or deleting lines. What's New in IDLE 2.7.8? diff --git a/lib-python/2.7/idlelib/PyShell.py b/lib-python/2.7/idlelib/PyShell.py --- a/lib-python/2.7/idlelib/PyShell.py +++ b/lib-python/2.7/idlelib/PyShell.py @@ -1408,6 +1408,17 @@ self.shell.close() +def fix_x11_paste(root): + "Make paste replace selection on x11. See issue #5124." + if root._windowingsystem == 'x11': + for cls in 'Text', 'Entry', 'Spinbox': + root.bind_class( + cls, + '<>', + 'catch {%W delete sel.first sel.last}\n' + + root.bind_class(cls, '<>')) + + usage_msg = """\ USAGE: idle [-deins] [-t title] [file]* @@ -1537,8 +1548,10 @@ 'editor-on-startup', type='bool') enable_edit = enable_edit or edit_start enable_shell = enable_shell or not enable_edit + # start editor and/or shell windows: root = Tk(className="Idle") + root.withdraw() # set application icon icondir = os.path.join(os.path.dirname(__file__), 'Icons') @@ -1553,7 +1566,7 @@ root.tk.call('wm', 'iconphoto', str(root), "-default", *icons) fixwordbreaks(root) - root.withdraw() + fix_x11_paste(root) flist = PyShellFileList(root) macosxSupport.setupApp(root, flist) diff --git a/lib-python/2.7/idlelib/README.txt b/lib-python/2.7/idlelib/README.txt --- a/lib-python/2.7/idlelib/README.txt +++ b/lib-python/2.7/idlelib/README.txt @@ -1,6 +1,6 @@ README.txt: an index to idlelib files and the IDLE menu. -IDLE is Python�s Integrated Development and Learning +IDLE is Python's Integrated Development and Learning Environment. The user documentation is part of the Library Reference and is available in IDLE by selecting Help => IDLE Help. This README documents idlelib for IDLE developers and curious users. diff --git a/lib-python/2.7/idlelib/WidgetRedirector.py b/lib-python/2.7/idlelib/WidgetRedirector.py --- a/lib-python/2.7/idlelib/WidgetRedirector.py +++ b/lib-python/2.7/idlelib/WidgetRedirector.py @@ -68,7 +68,7 @@ '''Return OriginalCommand(operation) after registering function. Registration adds an operation: function pair to ._operations. - It also adds an widget function attribute that masks the Tkinter + It also adds a widget function attribute that masks the Tkinter class instance method. Method masking operates independently from command dispatch. diff --git a/lib-python/2.7/idlelib/aboutDialog.py b/lib-python/2.7/idlelib/aboutDialog.py --- a/lib-python/2.7/idlelib/aboutDialog.py +++ b/lib-python/2.7/idlelib/aboutDialog.py @@ -110,6 +110,7 @@ command=self.ShowIDLECredits) idle_credits_b.pack(side=LEFT, padx=10, pady=10) + # License, et all, are of type _sitebuiltins._Printer def ShowLicense(self): self.display_printer_text('About - License', license) @@ -119,14 +120,16 @@ def ShowPythonCredits(self): self.display_printer_text('About - Python Credits', credits) + # Encode CREDITS.txt to utf-8 for proper version of Loewis. + # Specify others as ascii until need utf-8, so catch errors. def ShowIDLECredits(self): - self.display_file_text('About - Credits', 'CREDITS.txt', 'iso-8859-1') + self.display_file_text('About - Credits', 'CREDITS.txt', 'utf-8') def ShowIDLEAbout(self): - self.display_file_text('About - Readme', 'README.txt') + self.display_file_text('About - Readme', 'README.txt', 'ascii') def ShowIDLENEWS(self): - self.display_file_text('About - NEWS', 'NEWS.txt') + self.display_file_text('About - NEWS', 'NEWS.txt', 'utf-8') def display_printer_text(self, title, printer): printer._Printer__setup() @@ -141,5 +144,7 @@ self.destroy() if __name__ == '__main__': + import unittest + unittest.main('idlelib.idle_test.test_helpabout', verbosity=2, exit=False) from idlelib.idle_test.htest import run run(AboutDialog) diff --git a/lib-python/2.7/idlelib/configDialog.py b/lib-python/2.7/idlelib/configDialog.py --- a/lib-python/2.7/idlelib/configDialog.py +++ b/lib-python/2.7/idlelib/configDialog.py @@ -500,6 +500,16 @@ self.autoSave.trace_variable('w', self.VarChanged_autoSave) self.encoding.trace_variable('w', self.VarChanged_encoding) + def remove_var_callbacks(self): + for var in ( + self.fontSize, self.fontName, self.fontBold, + self.spaceNum, self.colour, self.builtinTheme, + self.customTheme, self.themeIsBuiltin, self.highlightTarget, + self.keyBinding, self.builtinKeys, self.customKeys, + self.keysAreBuiltin, self.winWidth, self.winHeight, + self.startupEdit, self.autoSave, self.encoding,): + var.trace_vdelete('w', var.trace_vinfo()[0][1]) + def VarChanged_font(self, *params): '''When one font attribute changes, save them all, as they are not independent from each other. In particular, when we are @@ -1213,7 +1223,7 @@ All values are treated as text, and it is up to the user to supply reasonable values. The only exception to this are the 'enable*' options, - which are boolean, and can be toggled with an True/False button. + which are boolean, and can be toggled with a True/False button. """ parent = self.parent frame = self.tabPages.pages['Extensions'].frame diff --git a/lib-python/2.7/idlelib/configHandler.py b/lib-python/2.7/idlelib/configHandler.py --- a/lib-python/2.7/idlelib/configHandler.py +++ b/lib-python/2.7/idlelib/configHandler.py @@ -721,7 +721,7 @@ actualFont = Font.actual(f) family = actualFont['family'] size = actualFont['size'] - if size < 0: + if size <= 0: size = 10 # if font in pixels, ignore actual size bold = actualFont['weight']=='bold' return (family, size, 'bold' if bold else 'normal') diff --git a/lib-python/2.7/idlelib/help.html b/lib-python/2.7/idlelib/help.html --- a/lib-python/2.7/idlelib/help.html +++ b/lib-python/2.7/idlelib/help.html @@ -6,15 +6,15 @@ - 24.6. IDLE — Python 2.7.10 documentation + 24.6. IDLE — Python 2.7.11 documentation - + - + @@ -40,8 +40,8 @@ - -
+ +
-
+

24.6. IDLE

IDLE is Python’s Integrated Development and Learning Environment.

IDLE has the following features:

    -
  • coded in 100% pure Python, using the tkinter GUI toolkit
    • +
  • coded in 100% pure Python, using the tkinter GUI toolkit
  • cross-platform: works mostly the same on Windows, Unix, and Mac OS X
  • Python shell window (interactive interpreter) with colorizing of code input, output, and error messages
    • @@ -163,7 +163,7 @@
    Find Selection
    Search for the currently selected string, if there is one.
    Find in Files...
    -
    Open a file search dialog. Put results in an new output window.
    +
    Open a file search dialog. Put results in a new output window.
    Replace...
    Open a search-and-replace dialog.
    Go to Line
    @@ -224,10 +224,10 @@
    Run Module
    Do Check Module (above). If no error, restart the shell to clean the environment, then execute the module. Output is displayed in the Shell -window. Note that output requires use of print or write. +window. Note that output requires use of print or write. When execution is complete, the Shell retains focus and displays a prompt. At this point, one may interactively explore the result of execution. -This is similar to executing a file with python -i file at a command +This is similar to executing a file with python -i file at a command line.
@@ -339,47 +339,47 @@

24.6.2. Editing and navigation

-

In this section, ‘C’ refers to the Control key on Windows and Unix and -the Command key on Mac OSX.

+

In this section, ‘C’ refers to the Control key on Windows and Unix and +the Command key on Mac OSX.

    -

  • Backspace deletes to the left; Del deletes to the right

    +

  • Backspace deletes to the left; Del deletes to the right

    • -

  • C-Backspace delete word left; C-Del delete word to the right

    +

  • C-Backspace delete word left; C-Del delete word to the right

    • -

  • Arrow keys and Page Up/Page Down to move around

    +

  • Arrow keys and Page Up/Page Down to move around

    • -

  • C-LeftArrow and C-RightArrow moves by words

    +

  • C-LeftArrow and C-RightArrow moves by words

    • -

  • Home/End go to begin/end of line

    +

  • Home/End go to begin/end of line

    • -

  • C-Home/C-End go to begin/end of file

    +

  • C-Home/C-End go to begin/end of file

  • Some useful Emacs bindings are inherited from Tcl/Tk:

      -
    • C-a beginning of line
      • -
    • C-e end of line
      • -
    • C-k kill line (but doesn’t put it in clipboard)
      • -
    • C-l center window around the insertion point
      • -
    • C-b go backwards one character without deleting (usually you can +
    • C-a beginning of line
      • +
    • C-e end of line
      • +
    • C-k kill line (but doesn’t put it in clipboard)
      • +
    • C-l center window around the insertion point
      • +
    • C-b go backwards one character without deleting (usually you can also use the cursor key for this)
      • -
    • C-f go forward one character without deleting (usually you can +
    • C-f go forward one character without deleting (usually you can also use the cursor key for this)
      • -
    • C-p go up one line (usually you can also use the cursor key for +
    • C-p go up one line (usually you can also use the cursor key for this)
      • -
    • C-d delete next character
      • +
    • C-d delete next character
-

Standard keybindings (like C-c to copy and C-v to paste) +

Standard keybindings (like C-c to copy and C-v to paste) may work. Keybindings are selected in the Configure IDLE dialog.

24.6.2.1. Automatic indentation

After a block-opening statement, the next line is indented by 4 spaces (in the Python Shell window by one tab). After certain keywords (break, return etc.) -the next line is dedented. In leading indentation, Backspace deletes up -to 4 spaces if they are there. Tab inserts spaces (in the Python +the next line is dedented. In leading indentation, Backspace deletes up +to 4 spaces if they are there. Tab inserts spaces (in the Python Shell window one tab), number depends on Indent width. Currently tabs are restricted to four spaces due to Tcl/Tk limitations.

See also the indent/dedent region commands in the edit menu.

@@ -394,25 +394,25 @@ of those characters (plus zero or more other characters) a tab is typed the ACW will open immediately if a possible continuation is found.

If there is only one possible completion for the characters entered, a -Tab will supply that completion without opening the ACW.

+Tab will supply that completion without opening the ACW.

‘Show Completions’ will force open a completions window, by default the -C-space will open a completions window. In an empty +C-space will open a completions window. In an empty string, this will contain the files in the current directory. On a blank line, it will contain the built-in and user-defined functions and classes in the current name spaces, plus any modules imported. If some characters have been entered, the ACW will attempt to be more specific.

If a string of characters is typed, the ACW selection will jump to the -entry most closely matching those characters. Entering a tab will +entry most closely matching those characters. Entering a tab will cause the longest non-ambiguous match to be entered in the Editor window or -Shell. Two tab in a row will supply the current ACW selection, as +Shell. Two tab in a row will supply the current ACW selection, as will return or a double click. Cursor keys, Page Up/Down, mouse selection, and the scroll wheel all operate on the ACW.

“Hidden” attributes can be accessed by typing the beginning of hidden name after a ‘.’, e.g. ‘_’. This allows access to modules with -__all__ set, or to class-private attributes.

+__all__ set, or to class-private attributes.

Completions and the ‘Expand Word’ facility can save a lot of typing!

Completions are currently limited to those in the namespaces. Names in -an Editor window which are not via __main__ and sys.modules will +an Editor window which are not via __main__ and sys.modules will not be found. Run the module once with your imports to correct this situation. Note that IDLE itself places quite a few modules in sys.modules, so much can be found by default, e.g. the re module.

@@ -421,10 +421,10 @@

24.6.2.3. Calltips

-

A calltip is shown when one types ( after the name of an acccessible +

A calltip is shown when one types ( after the name of an acccessible function. A name expression may include dots and subscripts. A calltip remains until it is clicked, the cursor is moved out of the argument area, -or ) is typed. When the cursor is in the argument part of a definition, +or ) is typed. When the cursor is in the argument part of a definition, the menu or shortcut display a calltip.

A calltip consists of the function signature and the first line of the docstring. For builtins without an accessible signature, the calltip @@ -433,11 +433,11 @@

The set of accessible functions depends on what modules have been imported into the user process, including those imported by Idle itself, and what definitions have been run, all since the last restart.

-

For example, restart the Shell and enter itertools.count(. A calltip +

For example, restart the Shell and enter itertools.count(. A calltip appears because Idle imports itertools into the user process for its own use. -(This could change.) Enter turtle.write( and nothing appears. Idle does +(This could change.) Enter turtle.write( and nothing appears. Idle does not import turtle. The menu or shortcut do nothing either. Enter -import turtle and then turtle.write( will work.

+import turtle and then turtle.write( will work.

In an editor, import statements have no effect until one runs the file. One might want to run a file after writing the import statements at the top, or immediately run an existing file before editing.

@@ -445,17 +445,17 @@

24.6.2.4. Python Shell window

    -

  • C-c interrupts executing command

    +

  • C-c interrupts executing command

    • -

  • C-d sends end-of-file; closes window if typed at a >>> prompt

    +

  • C-d sends end-of-file; closes window if typed at a >>> prompt

    • -

  • Alt-/ (Expand word) is also useful to reduce typing

    +

  • Alt-/ (Expand word) is also useful to reduce typing

    Command history

      -
    • Alt-p retrieves previous command matching what you have typed. On -OS X use C-p.
      • -
    • Alt-n retrieves next. On OS X use C-n.
      • -
    • Return while on any previous command retrieves that command
      • +
    • Alt-p retrieves previous command matching what you have typed. On +OS X use C-p.
      • +
    • Alt-n retrieves next. On OS X use C-n.
      • +
    • Return while on any previous command retrieves that command
@@ -465,8 +465,8 @@

Idle defaults to black on white text, but colors text with special meanings. For the shell, these are shell output, shell error, user output, and user error. For Python code, at the shell prompt or in an editor, these are -keywords, builtin class and function names, names following class and -def, strings, and comments. For any text window, these are the cursor (when +keywords, builtin class and function names, names following class and +def, strings, and comments. For any text window, these are the cursor (when present), found text (when possible), and selected text.

Text coloring is done in the background, so uncolorized text is occasionally visible. To change the color scheme, use the Configure IDLE dialog @@ -476,15 +476,15 @@

24.6.3. Startup and code execution

-

Upon startup with the -s option, IDLE will execute the file referenced by -the environment variables IDLESTARTUP or PYTHONSTARTUP. -IDLE first checks for IDLESTARTUP; if IDLESTARTUP is present the file -referenced is run. If IDLESTARTUP is not present, IDLE checks for -PYTHONSTARTUP. Files referenced by these environment variables are +

Upon startup with the -s option, IDLE will execute the file referenced by +the environment variables IDLESTARTUP or PYTHONSTARTUP. +IDLE first checks for IDLESTARTUP; if IDLESTARTUP is present the file +referenced is run. If IDLESTARTUP is not present, IDLE checks for +PYTHONSTARTUP. Files referenced by these environment variables are convenient places to store functions that are used frequently from the IDLE shell, or for executing import statements to import common modules.

-

In addition, Tk also loads a startup file if it is present. Note that the -Tk file is loaded unconditionally. This additional file is .Idle.py and is +

In addition, Tk also loads a startup file if it is present. Note that the +Tk file is loaded unconditionally. This additional file is .Idle.py and is looked for in the user’s home directory. Statements in this file will be executed in the Tk namespace, so this file is not useful for importing functions to be used from IDLE’s Python shell.

@@ -505,25 +505,27 @@

If there are arguments:

    -
  • If -, -c, or r is used, all arguments are placed in -sys.argv[1:...] and sys.argv[0] is set to '', '-c', -or '-r'. No editor window is opened, even if that is the default +
  • If -, -c, or r is used, all arguments are placed in +sys.argv[1:...] and sys.argv[0] is set to '', '-c', +or '-r'. No editor window is opened, even if that is the default set in the Options dialog.
  • Otherwise, arguments are files opened for editing and -sys.argv reflects the arguments passed to IDLE itself.
    • +sys.argv reflects the arguments passed to IDLE itself.

24.6.3.2. IDLE-console differences

As much as possible, the result of executing Python code with IDLE is the same as executing the same code in a console window. However, the different -interface and operation occasionally affects results.

-

For instance, IDLE normally executes user code in a separate process from -the IDLE GUI itself. The IDLE versions of sys.stdin, .stdout, and .stderr in the -execution process get input from and send output to the GUI process, -which keeps control of the keyboard and screen. This is normally transparent, -but code that access these object will see different attribute values. -Also, functions that directly access the keyboard and screen will not work.

+interface and operation occasionally affects visible results. For instance, +sys.modules starts with more entries.

+

IDLE also replaces sys.stdin, sys.stdout, and sys.stderr with +objects that get input from and send output to the Shell window. +When this window has the focus, it controls the keyboard and screen. +This is normally transparent, but functions that directly access the keyboard +and screen will not work. If sys is reset with reload(sys), +IDLE’s changes are lost and things like input, raw_input, and +print will not work correctly.

With IDLE’s Shell, one enters, edits, and recalls complete statements. Some consoles only work with a single physical line at a time.

@@ -595,7 +597,7 @@
-
+