[issue14776] Add SystemTap static markers
Dave Malcolm
report at bugs.python.org
Wed May 16 20:46:40 CEST 2012
Dave Malcolm <dmalcolm at redhat.com> added the comment:
On Wed, 2012-05-16 at 16:41 +0000, Antoine Pitrou wrote:
> Antoine Pitrou <pitrou at free.fr> added the comment:
>
> I tried the patch under Mageia 1 and got the following failure:
>
> test_systemtap skipped -- Test systemtap script did not run; stderr was: b"Pass 1: parsed user script and 72 library script(s) using 56252virt/20964res/1828shr kb, in 70usr/0sys/82real ms.\nPass 2: analyzed script: 1 probe(s), 1 function(s), 0 embed(s), 0 global(s) using 56648virt/21492res/1900shr kb, in 10usr/0sys/3real ms.\nPass 3: using cached /home/antoine/.systemtap/cache/15/stap_155c3565481f113c929ad94e10c2b48e_779.c\nPass 4: using cached /home/antoine/.systemtap/cache/15/stap_155c3565481f113c929ad94e10c2b48e_779.ko\nPass 5: starting run.\nPass 5: run completed in 0usr/0sys/3real ms.\nPass 5: run failed. Try again with another '--vp 00001' option.\n"
Thanks. I'm attaching an updated patch which reworks test_systemtap so
that
(a) it turns up the verbosity of stap invocations from "-v" to "-vv"
(b) it tests the "hello world" stap script on a trivial binary
("true") before attempting to invoke sys.executable under stap
What does the error message look like with the extra verbosity?
The chmod invocation sounds like an issue with how Mageia have packaged
SystemTap, assuming that you're using a packaged build of systemtap.
----------
Added file: http://bugs.python.org/file25615/cpython-systemtap-2012-05-16-001.patch
_______________________________________
Python tracker <report at bugs.python.org>
<http://bugs.python.org/issue14776>
_______________________________________
-------------- next part --------------
diff -r 0f6a6f59b002 .hgignore
--- a/.hgignore Wed May 16 19:08:36 2012 +0200
+++ b/.hgignore Wed May 16 13:34:52 2012 -0400
@@ -19,6 +19,7 @@
python.exe$
reflog.txt$
tags$
+Python/pysystemtap.h
Lib/plat-mac/errors.rsrc.df.rsrc
Doc/tools/sphinx/
Doc/tools/docutils/
diff -r 0f6a6f59b002 Lib/test/test_systemtap.py
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/Lib/test/test_systemtap.py Wed May 16 13:34:52 2012 -0400
@@ -0,0 +1,205 @@
+# Verify that systemtap static probes work
+#
+import subprocess
+import sys
+import sysconfig
+import os
+import unittest
+
+from test.support import run_unittest, TESTFN, unlink
+
+if '--with-systemtap' not in sysconfig.get_config_var('CONFIG_ARGS'):
+ raise unittest.SkipTest("Python was not configured --with-systemtap")
+
+try:
+ _, stap_version = subprocess.Popen(["stap", "-V"],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ ).communicate()
+except OSError:
+ # This is what "no stap" looks like. There may, however, be other
+ # errors that manifest this way too.
+ raise unittest.SkipTest("Couldn't find stap on the path")
+
+def invoke_systemtap_script(script, cmd):
+ # Start a child process, probing with the given systemtap script
+ # (passed as stdin to the "stap" tool)
+ # The script should be a bytes instance
+ # Return (stdout, stderr) pair
+
+ p = subprocess.Popen(["stap", "-", '-vv', '-c', cmd],
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ out, err = p.communicate(input=script)
+ return out, err
+
+# Verify that stap can run a simple "hello world"-style script
+# This can fail for various reasons:
+# - missing kernel headers
+# - permissions (a non-root user needs to be in the "stapdev" group)
+TRIVIAL_STAP_SCRIPT = b'probe begin { println("hello world") exit () }'
+
+out, err = invoke_systemtap_script(TRIVIAL_STAP_SCRIPT, 'true')
+if out != b'hello world\n':
+ raise unittest.SkipTest("Test systemtap script did not run; stderr was: %s" % err)
+
+# We don't expect stderr to be empty, since we're invoking stap with "-vv": stap
+# will (we hope) generate debugging output on stderr.
+
+def invoke_python_under_systemtap(script, pythoncode=None, pythonfile=None):
+ # Start a child python process, probing with the given systemtap script
+ # (passed as stdin to the "stap" tool)
+ # The script should be a bytes instance
+ # Return (stdout, stderr) pair
+
+ if pythonfile:
+ pythoncmd = '%s %s' % (sys.executable, pythonfile)
+ else:
+ pythoncmd = '%s -c %r' % (sys.executable, pythoncode)
+
+ # The process tree of a stap invocation of a command goes through
+ # something like this:
+ # stap ->fork/exec(staprun; exec stapio ->f/e(-c cmd); exec staprun -r)
+ # and this trip through setuid leads to LD_LIBRARY_PATH being dropped,
+ # which would lead to an --enable-shared build of python failing to be
+ # find its libpython, with an error like:
+ # error while loading shared libraries: libpython3.3dm.so.1.0: cannot
+ # open shared object file: No such file or directory
+ # Hence we need to jump through some hoops to expose LD_LIBRARY_PATH to
+ # the invoked python process:
+ LD_LIBRARY_PATH = os.environ.get('LD_LIBRARY_PATH', '')
+ if LD_LIBRARY_PATH:
+ pythoncmd = 'env LD_LIBRARY_PATH=%s ' % LD_LIBRARY_PATH + pythoncmd
+
+ return invoke_systemtap_script(script, pythoncmd)
+
+# When using the static markers, we need to supply the prefix of a systemtap
+# dotted probe point that containing the marker.
+# See http://sourceware.org/systemtap/langref/Probe_points.html
+#
+# We need to determine if this is a shared-library build
+#
+# Note that sysconfig can get this wrong; see:
+# http://bugs.python.org/issue14774
+#
+if '--enable-shared' in sysconfig.get_config_var('CONFIG_ARGS'):
+ # For a shared-library build, the markers are in library(INSTSONAME):
+ INSTSONAME = sysconfig.get_config_var('INSTSONAME')
+ probe_prefix = 'process("%s").library("%s")' % (sys.executable, INSTSONAME)
+else:
+ # For a non-shared-library build, we can simply use sys.executable:
+ probe_prefix = 'process("%s")' % sys.executable
+
+# The following script ought to generate lots of lines showing recursive
+# function entry and return, of the form:
+# 11408 python(8274): => __contains__ in Lib/_abcoll.py:362
+# 11414 python(8274): => __getitem__ in Lib/os.py:425
+# 11418 python(8274): => encode in Lib/os.py:490
+# 11424 python(8274): <= encode in Lib/os.py:493
+# 11428 python(8274): <= __getitem__ in Lib/os.py:426
+# 11433 python(8274): <= __contains__ in Lib/_abcoll.py:366
+# where the column are:
+# - time in microseconds since start of script
+# - name of executable
+# - PID of process
+# and the remainder indicates the call/return hierarchy
+
+hierarchy_script = ('''
+probe %s.mark("function__entry") {
+ filename = user_string($arg1);
+ funcname = user_string($arg2);
+ lineno = $arg3;
+
+ printf("%%s => %%s in %%s:%%d\\n", thread_indent(1), funcname, filename, lineno);
+}
+
+probe %s.mark("function__return") {
+ filename = user_string($arg1);
+ funcname = user_string($arg2);
+ lineno = $arg3;
+
+ printf("%%s <= %%s in %%s:%%d\\n", thread_indent(-1), funcname, filename, lineno);
+}
+''' % (probe_prefix, probe_prefix)).encode('utf-8')
+
+
+class ErrorDumper:
+ # A context manager that dumps extra information if an exception is raised,
+ # to help track down why the problem occurred
+ def __init__(self, out, err):
+ self.out = out
+ self.err = err
+
+ def __enter__(self):
+ pass
+
+ def __exit__(self, type_, value, traceback):
+ if type_:
+ # an exception is being raised:
+ print('stdout: %s' % out.decode())
+ print('stderr: %s' % err.decode())
+
+class SystemtapTests(unittest.TestCase):
+
+ def test_invoking_python(self):
+ # Ensure that we can invoke python under stap, with a trivial stap
+ # script:
+ out, err = invoke_python_under_systemtap(
+ b'probe begin { println("hello from stap") exit () }',
+ pythoncode="print('hello from python')")
+ with ErrorDumper(out, err):
+ self.assertIn(b'hello from stap', out)
+ self.assertIn(b'hello from python', out)
+
+ def test_function_entry(self):
+ # Ensure that the function_entry static marker works
+ out, err = invoke_python_under_systemtap(hierarchy_script)
+ # stdout ought to contain various lines showing recursive function
+ # entry and return (see above)
+
+ # Uncomment this for debugging purposes:
+ # print(out.decode('utf-8'))
+
+ # Executing the cmdline-supplied "pass":
+ # 0 python(8274): => <module> in <string>:1
+ # 5 python(8274): <= <module> in <string>:1
+ with ErrorDumper(out, err):
+ self.assertIn(b'=> <module> in <string>:1', out,
+ msg="stdout: %s\nstderr: %s\n" % (out, err))
+
+ def test_encoding(self):
+ # Ensure that scripts and function names containing non-Latin 1 code
+ # points are handled:
+ pythonfile = TESTFN + '_���.py'
+ try:
+ unlink(pythonfile)
+ f = open(pythonfile, "wb")
+ f.write("""
+# Sample script with non-ASCII filename, for use by test_systemtap.py
+# Implicitly UTF-8
+
+def ������������():
+ '''Function with non-ASCII identifier; I believe this reads "mojibake"'''
+ print("hello world!")
+
+������������()
+""".encode('utf-8'))
+ f.close()
+
+ out, err = invoke_python_under_systemtap(hierarchy_script,
+ pythonfile=pythonfile)
+ out_utf8 = out.decode('utf-8')
+ with ErrorDumper(out, err):
+ self.assertIn('=> <module> in %s:5' % pythonfile, out_utf8)
+ self.assertIn(' => ������������ in %s:5' % pythonfile, out_utf8)
+ self.assertIn(' <= ������������ in %s:7' % pythonfile, out_utf8)
+ self.assertIn('<= <module> in %s:9' % pythonfile, out_utf8)
+ finally:
+ unlink(pythonfile)
+
+def test_main():
+ run_unittest(SystemtapTests)
+
+if __name__ == "__main__":
+ test_main()
diff -r 0f6a6f59b002 Makefile.pre.in
--- a/Makefile.pre.in Wed May 16 19:08:36 2012 +0200
+++ b/Makefile.pre.in Wed May 16 13:34:52 2012 -0400
@@ -359,6 +359,7 @@
Python/formatter_unicode.o \
Python/fileutils.o \
Python/$(DYNLOADFILE) \
+ @SYSTEMTAPOBJS@ \
$(LIBOBJS) \
$(MACHDEP_OBJS) \
$(THREADOBJ)
@@ -708,7 +709,9 @@
$(OPCODETARGETS_H): $(OPCODETARGETGEN_FILES)
$(OPCODETARGETGEN) $(OPCODETARGETS_H)
-Python/ceval.o: $(OPCODETARGETS_H) $(srcdir)/Python/ceval_gil.h
+Python/ceval.o: $(OPCODETARGETS_H) $(srcdir)/Python/ceval_gil.h \
+ $(srcdir)/Python/ceval_systemtap.h \
+ $(srcdir)/Python/pysystemtap.h
Python/formatter_unicode.o: $(srcdir)/Python/formatter_unicode.c \
$(BYTESTR_DEPS)
@@ -719,6 +722,13 @@
$(srcdir)/Objects/typeslots.inc: $(srcdir)/Include/typeslots.h $(srcdir)/Objects/typeslots.py
$(PYTHON) $(srcdir)/Objects/typeslots.py < $(srcdir)/Include/typeslots.h > $(srcdir)/Objects/typeslots.inc
+# Only needed with --with-systemtap; not a public header:
+$(srcdir)/Python/pysystemtap.h: $(srcdir)/Python/pysystemtap.d
+ dtrace -o $@ $(DFLAGS) -C -h -s $(srcdir)/Python/pysystemtap.d
+
+Python/pysystemtap.o: $(srcdir)/Python/pysystemtap.d Python/ceval.o
+ dtrace -o $@ $(DFLAGS) -C -G -s $(srcdir)/Python/pysystemtap.d Python/ceval.o
+
############################################################################
# Header files
diff -r 0f6a6f59b002 Python/ceval.c
--- a/Python/ceval.c Wed May 16 19:08:36 2012 +0200
+++ b/Python/ceval.c Wed May 16 13:34:52 2012 -0400
@@ -18,6 +18,8 @@
#include <ctype.h>
+#include "ceval_systemtap.h"
+
#ifndef WITH_TSC
#define READ_TIMESTAMP(var)
@@ -1160,6 +1162,10 @@
}
}
+ if (PYTHON_FUNCTION_ENTRY_ENABLED()) {
+ systemtap_function_entry(f);
+ }
+
co = f->f_code;
names = co->co_names;
consts = co->co_consts;
@@ -3077,6 +3083,11 @@
/* pop frame */
exit_eval_frame:
+
+ if (PYTHON_FUNCTION_RETURN_ENABLED()) {
+ systemtap_function_return(f);
+ }
+
Py_LeaveRecursiveCall();
tstate->frame = f->f_back;
diff -r 0f6a6f59b002 Python/ceval_systemtap.h
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/Python/ceval_systemtap.h Wed May 16 13:34:52 2012 -0400
@@ -0,0 +1,86 @@
+/*
+ Support for SystemTap static markers
+*/
+
+#ifdef WITH_SYSTEMTAP
+
+#include "pysystemtap.h"
+
+/*
+ A struct to hold all of the information gathered when one of the traceable
+ markers is triggered
+*/
+struct frame_marker_info
+{
+ PyObject *filename_obj;
+ PyObject *funcname_obj;
+ const char *filename;
+ const char *funcname;
+ int lineno;
+};
+
+static void
+get_frame_marker_info(PyFrameObject *f, struct frame_marker_info *fmi)
+{
+ PyObject *ptype;
+ PyObject *pvalue;
+ PyObject *ptraceback;
+
+ PyErr_Fetch(&ptype, &pvalue, &ptraceback);
+
+ fmi->filename_obj = PyUnicode_EncodeFSDefault(f->f_code->co_filename);
+ if (fmi->filename_obj) {
+ fmi->filename = PyBytes_AsString(fmi->filename_obj);
+ } else {
+ fmi->filename = NULL;
+ }
+
+ fmi->funcname_obj = PyUnicode_AsUTF8String(f->f_code->co_name);
+ if (fmi->funcname_obj) {
+ fmi->funcname = PyBytes_AsString(fmi->funcname_obj);
+ } else {
+ fmi->funcname = NULL;
+ }
+
+ fmi->lineno = PyCode_Addr2Line(f->f_code, f->f_lasti);
+
+ PyErr_Restore(ptype, pvalue, ptraceback);
+
+}
+
+static void
+release_frame_marker_info(struct frame_marker_info *fmi)
+{
+ Py_XDECREF(fmi->filename_obj);
+ Py_XDECREF(fmi->funcname_obj);
+}
+
+static void
+systemtap_function_entry(PyFrameObject *f)
+{
+ struct frame_marker_info fmi;
+ get_frame_marker_info(f, &fmi);
+ PYTHON_FUNCTION_ENTRY(fmi.filename, fmi.funcname, fmi.lineno);
+ release_frame_marker_info(&fmi);
+}
+
+static void
+systemtap_function_return(PyFrameObject *f)
+{
+ struct frame_marker_info fmi;
+ get_frame_marker_info(f, &fmi);
+ PYTHON_FUNCTION_RETURN(fmi.filename, fmi.funcname, fmi.lineno);
+ release_frame_marker_info(&fmi);
+}
+
+#else /* #ifdef WITH_SYSTEMTAP */
+
+/*
+ When configured --without-systemtap, everything compiles away to nothing:
+*/
+#define PYTHON_FUNCTION_ENTRY_ENABLED() 0
+#define PYTHON_FUNCTION_RETURN_ENABLED() 0
+#define systemtap_function_entry(f)
+#define systemtap_function_return(f)
+
+#endif
diff -r 0f6a6f59b002 Python/pysystemtap.d
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/Python/pysystemtap.d Wed May 16 13:34:52 2012 -0400
@@ -0,0 +1,4 @@
+provider python {
+ probe function__entry(const char *, const char *, int);
+ probe function__return(const char *, const char *, int);
+};
diff -r 0f6a6f59b002 configure
--- a/configure Wed May 16 19:08:36 2012 +0200
+++ b/configure Wed May 16 13:34:52 2012 -0400
@@ -618,6 +618,7 @@
MACHDEP_OBJS
DYNLOADFILE
DLINCLDIR
+SYSTEMTAPOBJS
THREADOBJ
LDLAST
USE_THREAD_MODULE
@@ -773,6 +774,7 @@
with_tsc
with_pymalloc
with_valgrind
+with_systemtap
with_fpectl
with_libm
with_libc
@@ -1450,6 +1452,7 @@
--with(out)-tsc enable/disable timestamp counter profile
--with(out)-pymalloc disable/enable specialized mallocs
--with-valgrind Enable Valgrind support
+ --with(out)-systemtap disable/enable SystemTap support
--with-fpectl enable SIGFPE catching
--with-libm=STRING math library
--with-libc=STRING C library
@@ -9492,6 +9495,29 @@
OPT="-DDYNAMIC_ANNOTATIONS_ENABLED=1 $OPT"
fi
+# Check for systemtap support
+# On Linux, /usr/bin/dtrace is in fact a shim to SystemTap
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-systemtap" >&5
+$as_echo_n "checking for --with-systemtap... " >&6; }
+
+# Check whether --with-systemtap was given.
+if test "${with_systemtap+set}" = set; then :
+ withval=$with_systemtap;
+else
+ with_systemtap=no
+fi
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_systemtap" >&5
+$as_echo "$with_systemtap" >&6; }
+if test "$with_systemtap" != no; then
+
+$as_echo "#define WITH_SYSTEMTAP 1" >>confdefs.h
+
+ SYSTEMTAPOBJS="Python/pysystemtap.o"
+fi
+
+
+
# -I${DLINCLDIR} is added to the compile rule for importdl.o
DLINCLDIR=.
diff -r 0f6a6f59b002 configure.ac
--- a/configure.ac Wed May 16 19:08:36 2012 +0200
+++ b/configure.ac Wed May 16 13:34:52 2012 -0400
@@ -2537,6 +2537,21 @@
OPT="-DDYNAMIC_ANNOTATIONS_ENABLED=1 $OPT"
fi
+# Check for systemtap support
+# On Linux, /usr/bin/dtrace is in fact a shim to SystemTap
+AC_MSG_CHECKING([for --with-systemtap])
+AC_ARG_WITH([systemtap],
+ AC_HELP_STRING([--with(out)-systemtap], [disable/enable SystemTap support]),,
+ with_systemtap=no)
+AC_MSG_RESULT([$with_systemtap])
+if test "$with_systemtap" != no; then
+ AC_DEFINE(WITH_SYSTEMTAP, 1,
+ [Define if you want to compile in SystemTap support])
+ SYSTEMTAPOBJS="Python/pysystemtap.o"
+fi
+
+AC_SUBST(SYSTEMTAPOBJS)
+
# -I${DLINCLDIR} is added to the compile rule for importdl.o
AC_SUBST(DLINCLDIR)
DLINCLDIR=.
diff -r 0f6a6f59b002 pyconfig.h.in
--- a/pyconfig.h.in Wed May 16 19:08:36 2012 +0200
+++ b/pyconfig.h.in Wed May 16 13:34:52 2012 -0400
@@ -1306,6 +1306,9 @@
/* Define if you want to compile in Python-specific mallocs */
#undef WITH_PYMALLOC
+/* Define if you want to compile in SystemTap support */
+#undef WITH_SYSTEMTAP
+
/* Define if you want to compile in rudimentary thread support */
#undef WITH_THREAD
More information about the Python-bugs-list
mailing list