diff -r 0f6a6f59b002 .hgignore --- a/.hgignore Wed May 16 19:08:36 2012 +0200 +++ b/.hgignore Wed May 16 13:34:52 2012 -0400 @@ -19,6 +19,7 @@ python.exe$ reflog.txt$ tags$ +Python/pysystemtap.h Lib/plat-mac/errors.rsrc.df.rsrc Doc/tools/sphinx/ Doc/tools/docutils/ diff -r 0f6a6f59b002 Lib/test/test_systemtap.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/test/test_systemtap.py Wed May 16 13:34:52 2012 -0400 @@ -0,0 +1,205 @@ +# Verify that systemtap static probes work +# +import subprocess +import sys +import sysconfig +import os +import unittest + +from test.support import run_unittest, TESTFN, unlink + +if '--with-systemtap' not in sysconfig.get_config_var('CONFIG_ARGS'): + raise unittest.SkipTest("Python was not configured --with-systemtap") + +try: + _, stap_version = subprocess.Popen(["stap", "-V"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ).communicate() +except OSError: + # This is what "no stap" looks like. There may, however, be other + # errors that manifest this way too. + raise unittest.SkipTest("Couldn't find stap on the path") + +def invoke_systemtap_script(script, cmd): + # Start a child process, probing with the given systemtap script + # (passed as stdin to the "stap" tool) + # The script should be a bytes instance + # Return (stdout, stderr) pair + + p = subprocess.Popen(["stap", "-", '-vv', '-c', cmd], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + out, err = p.communicate(input=script) + return out, err + +# Verify that stap can run a simple "hello world"-style script +# This can fail for various reasons: +# - missing kernel headers +# - permissions (a non-root user needs to be in the "stapdev" group) +TRIVIAL_STAP_SCRIPT = b'probe begin { println("hello world") exit () }' + +out, err = invoke_systemtap_script(TRIVIAL_STAP_SCRIPT, 'true') +if out != b'hello world\n': + raise unittest.SkipTest("Test systemtap script did not run; stderr was: %s" % err) + +# We don't expect stderr to be empty, since we're invoking stap with "-vv": stap +# will (we hope) generate debugging output on stderr. + +def invoke_python_under_systemtap(script, pythoncode=None, pythonfile=None): + # Start a child python process, probing with the given systemtap script + # (passed as stdin to the "stap" tool) + # The script should be a bytes instance + # Return (stdout, stderr) pair + + if pythonfile: + pythoncmd = '%s %s' % (sys.executable, pythonfile) + else: + pythoncmd = '%s -c %r' % (sys.executable, pythoncode) + + # The process tree of a stap invocation of a command goes through + # something like this: + # stap ->fork/exec(staprun; exec stapio ->f/e(-c cmd); exec staprun -r) + # and this trip through setuid leads to LD_LIBRARY_PATH being dropped, + # which would lead to an --enable-shared build of python failing to be + # find its libpython, with an error like: + # error while loading shared libraries: libpython3.3dm.so.1.0: cannot + # open shared object file: No such file or directory + # Hence we need to jump through some hoops to expose LD_LIBRARY_PATH to + # the invoked python process: + LD_LIBRARY_PATH = os.environ.get('LD_LIBRARY_PATH', '') + if LD_LIBRARY_PATH: + pythoncmd = 'env LD_LIBRARY_PATH=%s ' % LD_LIBRARY_PATH + pythoncmd + + return invoke_systemtap_script(script, pythoncmd) + +# When using the static markers, we need to supply the prefix of a systemtap +# dotted probe point that containing the marker. +# See http://sourceware.org/systemtap/langref/Probe_points.html +# +# We need to determine if this is a shared-library build +# +# Note that sysconfig can get this wrong; see: +# http://bugs.python.org/issue14774 +# +if '--enable-shared' in sysconfig.get_config_var('CONFIG_ARGS'): + # For a shared-library build, the markers are in library(INSTSONAME): + INSTSONAME = sysconfig.get_config_var('INSTSONAME') + probe_prefix = 'process("%s").library("%s")' % (sys.executable, INSTSONAME) +else: + # For a non-shared-library build, we can simply use sys.executable: + probe_prefix = 'process("%s")' % sys.executable + +# The following script ought to generate lots of lines showing recursive +# function entry and return, of the form: +# 11408 python(8274): => __contains__ in Lib/_abcoll.py:362 +# 11414 python(8274): => __getitem__ in Lib/os.py:425 +# 11418 python(8274): => encode in Lib/os.py:490 +# 11424 python(8274): <= encode in Lib/os.py:493 +# 11428 python(8274): <= __getitem__ in Lib/os.py:426 +# 11433 python(8274): <= __contains__ in Lib/_abcoll.py:366 +# where the column are: +# - time in microseconds since start of script +# - name of executable +# - PID of process +# and the remainder indicates the call/return hierarchy + +hierarchy_script = (''' +probe %s.mark("function__entry") { + filename = user_string($arg1); + funcname = user_string($arg2); + lineno = $arg3; + + printf("%%s => %%s in %%s:%%d\\n", thread_indent(1), funcname, filename, lineno); +} + +probe %s.mark("function__return") { + filename = user_string($arg1); + funcname = user_string($arg2); + lineno = $arg3; + + printf("%%s <= %%s in %%s:%%d\\n", thread_indent(-1), funcname, filename, lineno); +} +''' % (probe_prefix, probe_prefix)).encode('utf-8') + + +class ErrorDumper: + # A context manager that dumps extra information if an exception is raised, + # to help track down why the problem occurred + def __init__(self, out, err): + self.out = out + self.err = err + + def __enter__(self): + pass + + def __exit__(self, type_, value, traceback): + if type_: + # an exception is being raised: + print('stdout: %s' % out.decode()) + print('stderr: %s' % err.decode()) + +class SystemtapTests(unittest.TestCase): + + def test_invoking_python(self): + # Ensure that we can invoke python under stap, with a trivial stap + # script: + out, err = invoke_python_under_systemtap( + b'probe begin { println("hello from stap") exit () }', + pythoncode="print('hello from python')") + with ErrorDumper(out, err): + self.assertIn(b'hello from stap', out) + self.assertIn(b'hello from python', out) + + def test_function_entry(self): + # Ensure that the function_entry static marker works + out, err = invoke_python_under_systemtap(hierarchy_script) + # stdout ought to contain various lines showing recursive function + # entry and return (see above) + + # Uncomment this for debugging purposes: + # print(out.decode('utf-8')) + + # Executing the cmdline-supplied "pass": + # 0 python(8274): => in :1 + # 5 python(8274): <= in :1 + with ErrorDumper(out, err): + self.assertIn(b'=> in :1', out, + msg="stdout: %s\nstderr: %s\n" % (out, err)) + + def test_encoding(self): + # Ensure that scripts and function names containing non-Latin 1 code + # points are handled: + pythonfile = TESTFN + '_☠.py' + try: + unlink(pythonfile) + f = open(pythonfile, "wb") + f.write(""" +# Sample script with non-ASCII filename, for use by test_systemtap.py +# Implicitly UTF-8 + +def 文字化け(): + '''Function with non-ASCII identifier; I believe this reads "mojibake"''' + print("hello world!") + +文字化け() +""".encode('utf-8')) + f.close() + + out, err = invoke_python_under_systemtap(hierarchy_script, + pythonfile=pythonfile) + out_utf8 = out.decode('utf-8') + with ErrorDumper(out, err): + self.assertIn('=> in %s:5' % pythonfile, out_utf8) + self.assertIn(' => 文字化け in %s:5' % pythonfile, out_utf8) + self.assertIn(' <= 文字化け in %s:7' % pythonfile, out_utf8) + self.assertIn('<= in %s:9' % pythonfile, out_utf8) + finally: + unlink(pythonfile) + +def test_main(): + run_unittest(SystemtapTests) + +if __name__ == "__main__": + test_main() diff -r 0f6a6f59b002 Makefile.pre.in --- a/Makefile.pre.in Wed May 16 19:08:36 2012 +0200 +++ b/Makefile.pre.in Wed May 16 13:34:52 2012 -0400 @@ -359,6 +359,7 @@ Python/formatter_unicode.o \ Python/fileutils.o \ Python/$(DYNLOADFILE) \ + @SYSTEMTAPOBJS@ \ $(LIBOBJS) \ $(MACHDEP_OBJS) \ $(THREADOBJ) @@ -708,7 +709,9 @@ $(OPCODETARGETS_H): $(OPCODETARGETGEN_FILES) $(OPCODETARGETGEN) $(OPCODETARGETS_H) -Python/ceval.o: $(OPCODETARGETS_H) $(srcdir)/Python/ceval_gil.h +Python/ceval.o: $(OPCODETARGETS_H) $(srcdir)/Python/ceval_gil.h \ + $(srcdir)/Python/ceval_systemtap.h \ + $(srcdir)/Python/pysystemtap.h Python/formatter_unicode.o: $(srcdir)/Python/formatter_unicode.c \ $(BYTESTR_DEPS) @@ -719,6 +722,13 @@ $(srcdir)/Objects/typeslots.inc: $(srcdir)/Include/typeslots.h $(srcdir)/Objects/typeslots.py $(PYTHON) $(srcdir)/Objects/typeslots.py < $(srcdir)/Include/typeslots.h > $(srcdir)/Objects/typeslots.inc +# Only needed with --with-systemtap; not a public header: +$(srcdir)/Python/pysystemtap.h: $(srcdir)/Python/pysystemtap.d + dtrace -o $@ $(DFLAGS) -C -h -s $(srcdir)/Python/pysystemtap.d + +Python/pysystemtap.o: $(srcdir)/Python/pysystemtap.d Python/ceval.o + dtrace -o $@ $(DFLAGS) -C -G -s $(srcdir)/Python/pysystemtap.d Python/ceval.o + ############################################################################ # Header files diff -r 0f6a6f59b002 Python/ceval.c --- a/Python/ceval.c Wed May 16 19:08:36 2012 +0200 +++ b/Python/ceval.c Wed May 16 13:34:52 2012 -0400 @@ -18,6 +18,8 @@ #include +#include "ceval_systemtap.h" + #ifndef WITH_TSC #define READ_TIMESTAMP(var) @@ -1160,6 +1162,10 @@ } } + if (PYTHON_FUNCTION_ENTRY_ENABLED()) { + systemtap_function_entry(f); + } + co = f->f_code; names = co->co_names; consts = co->co_consts; @@ -3077,6 +3083,11 @@ /* pop frame */ exit_eval_frame: + + if (PYTHON_FUNCTION_RETURN_ENABLED()) { + systemtap_function_return(f); + } + Py_LeaveRecursiveCall(); tstate->frame = f->f_back; diff -r 0f6a6f59b002 Python/ceval_systemtap.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Python/ceval_systemtap.h Wed May 16 13:34:52 2012 -0400 @@ -0,0 +1,86 @@ +/* + Support for SystemTap static markers +*/ + +#ifdef WITH_SYSTEMTAP + +#include "pysystemtap.h" + +/* + A struct to hold all of the information gathered when one of the traceable + markers is triggered +*/ +struct frame_marker_info +{ + PyObject *filename_obj; + PyObject *funcname_obj; + const char *filename; + const char *funcname; + int lineno; +}; + +static void +get_frame_marker_info(PyFrameObject *f, struct frame_marker_info *fmi) +{ + PyObject *ptype; + PyObject *pvalue; + PyObject *ptraceback; + + PyErr_Fetch(&ptype, &pvalue, &ptraceback); + + fmi->filename_obj = PyUnicode_EncodeFSDefault(f->f_code->co_filename); + if (fmi->filename_obj) { + fmi->filename = PyBytes_AsString(fmi->filename_obj); + } else { + fmi->filename = NULL; + } + + fmi->funcname_obj = PyUnicode_AsUTF8String(f->f_code->co_name); + if (fmi->funcname_obj) { + fmi->funcname = PyBytes_AsString(fmi->funcname_obj); + } else { + fmi->funcname = NULL; + } + + fmi->lineno = PyCode_Addr2Line(f->f_code, f->f_lasti); + + PyErr_Restore(ptype, pvalue, ptraceback); + +} + +static void +release_frame_marker_info(struct frame_marker_info *fmi) +{ + Py_XDECREF(fmi->filename_obj); + Py_XDECREF(fmi->funcname_obj); +} + +static void +systemtap_function_entry(PyFrameObject *f) +{ + struct frame_marker_info fmi; + get_frame_marker_info(f, &fmi); + PYTHON_FUNCTION_ENTRY(fmi.filename, fmi.funcname, fmi.lineno); + release_frame_marker_info(&fmi); +} + +static void +systemtap_function_return(PyFrameObject *f) +{ + struct frame_marker_info fmi; + get_frame_marker_info(f, &fmi); + PYTHON_FUNCTION_RETURN(fmi.filename, fmi.funcname, fmi.lineno); + release_frame_marker_info(&fmi); +} + +#else /* #ifdef WITH_SYSTEMTAP */ + +/* + When configured --without-systemtap, everything compiles away to nothing: +*/ +#define PYTHON_FUNCTION_ENTRY_ENABLED() 0 +#define PYTHON_FUNCTION_RETURN_ENABLED() 0 +#define systemtap_function_entry(f) +#define systemtap_function_return(f) + +#endif diff -r 0f6a6f59b002 Python/pysystemtap.d --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Python/pysystemtap.d Wed May 16 13:34:52 2012 -0400 @@ -0,0 +1,4 @@ +provider python { + probe function__entry(const char *, const char *, int); + probe function__return(const char *, const char *, int); +}; diff -r 0f6a6f59b002 configure --- a/configure Wed May 16 19:08:36 2012 +0200 +++ b/configure Wed May 16 13:34:52 2012 -0400 @@ -618,6 +618,7 @@ MACHDEP_OBJS DYNLOADFILE DLINCLDIR +SYSTEMTAPOBJS THREADOBJ LDLAST USE_THREAD_MODULE @@ -773,6 +774,7 @@ with_tsc with_pymalloc with_valgrind +with_systemtap with_fpectl with_libm with_libc @@ -1450,6 +1452,7 @@ --with(out)-tsc enable/disable timestamp counter profile --with(out)-pymalloc disable/enable specialized mallocs --with-valgrind Enable Valgrind support + --with(out)-systemtap disable/enable SystemTap support --with-fpectl enable SIGFPE catching --with-libm=STRING math library --with-libc=STRING C library @@ -9492,6 +9495,29 @@ OPT="-DDYNAMIC_ANNOTATIONS_ENABLED=1 $OPT" fi +# Check for systemtap support +# On Linux, /usr/bin/dtrace is in fact a shim to SystemTap +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-systemtap" >&5 +$as_echo_n "checking for --with-systemtap... " >&6; } + +# Check whether --with-systemtap was given. +if test "${with_systemtap+set}" = set; then : + withval=$with_systemtap; +else + with_systemtap=no +fi + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_systemtap" >&5 +$as_echo "$with_systemtap" >&6; } +if test "$with_systemtap" != no; then + +$as_echo "#define WITH_SYSTEMTAP 1" >>confdefs.h + + SYSTEMTAPOBJS="Python/pysystemtap.o" +fi + + + # -I${DLINCLDIR} is added to the compile rule for importdl.o DLINCLDIR=. diff -r 0f6a6f59b002 configure.ac --- a/configure.ac Wed May 16 19:08:36 2012 +0200 +++ b/configure.ac Wed May 16 13:34:52 2012 -0400 @@ -2537,6 +2537,21 @@ OPT="-DDYNAMIC_ANNOTATIONS_ENABLED=1 $OPT" fi +# Check for systemtap support +# On Linux, /usr/bin/dtrace is in fact a shim to SystemTap +AC_MSG_CHECKING([for --with-systemtap]) +AC_ARG_WITH([systemtap], + AC_HELP_STRING([--with(out)-systemtap], [disable/enable SystemTap support]),, + with_systemtap=no) +AC_MSG_RESULT([$with_systemtap]) +if test "$with_systemtap" != no; then + AC_DEFINE(WITH_SYSTEMTAP, 1, + [Define if you want to compile in SystemTap support]) + SYSTEMTAPOBJS="Python/pysystemtap.o" +fi + +AC_SUBST(SYSTEMTAPOBJS) + # -I${DLINCLDIR} is added to the compile rule for importdl.o AC_SUBST(DLINCLDIR) DLINCLDIR=. diff -r 0f6a6f59b002 pyconfig.h.in --- a/pyconfig.h.in Wed May 16 19:08:36 2012 +0200 +++ b/pyconfig.h.in Wed May 16 13:34:52 2012 -0400 @@ -1306,6 +1306,9 @@ /* Define if you want to compile in Python-specific mallocs */ #undef WITH_PYMALLOC +/* Define if you want to compile in SystemTap support */ +#undef WITH_SYSTEMTAP + /* Define if you want to compile in rudimentary thread support */ #undef WITH_THREAD