[pypy-commit] pypy reverse-debugger: in-progress
arigo
pypy.commits at gmail.com
Mon Jun 27 10:13:12 EDT 2016
Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r85399:6df99c485ecf
Date: 2016-06-27 16:14 +0200
http://bitbucket.org/pypy/pypy/changeset/6df99c485ecf/
Log: in-progress
diff --git a/rpython/translator/c/src/mem.c b/rpython/translator/c/src/mem.c
--- a/rpython/translator/c/src/mem.c
+++ b/rpython/translator/c/src/mem.c
@@ -107,6 +107,7 @@
#else
/* see revdb.c */
RPY_EXTERN void rpy_reverse_db_next_dead(void *);
+RPY_EXTERN int rpy_reverse_db_fq_register(void *);
#endif
static void mem_boehm_ignore(char *msg, GC_word arg)
@@ -121,7 +122,7 @@
GC_set_warn_proc(mem_boehm_ignore);
}
-void boehm_fq_callback(void *obj, void *rawfqueue)
+static void boehm_fq_callback(void *obj, void *rawfqueue)
{
struct boehm_fq_s **fqueue = rawfqueue;
struct boehm_fq_s *node = GC_malloc(sizeof(void *) * 2);
@@ -132,6 +133,16 @@
*fqueue = node;
}
+void boehm_fq_register(struct boehm_fq_s **fqueue, void *obj)
+{
+#ifdef RPY_REVERSE_DEBUGGER
+ /* this function returns 0 when recording, or 1 when replaying */
+ if (rpy_reverse_db_fq_register(obj))
+ return;
+#endif
+ GC_REGISTER_FINALIZER(obj, boehm_fq_callback, fqueue, NULL, NULL);
+}
+
void *boehm_fq_next_dead(struct boehm_fq_s **fqueue)
{
struct boehm_fq_s *node = *fqueue;
diff --git a/rpython/translator/c/src/mem.h b/rpython/translator/c/src/mem.h
--- a/rpython/translator/c/src/mem.h
+++ b/rpython/translator/c/src/mem.h
@@ -107,7 +107,7 @@
struct boehm_fq_s;
RPY_EXTERN struct boehm_fq_s *boehm_fq_queues[];
RPY_EXTERN void (*boehm_fq_trigger[])(void);
-RPY_EXTERN void boehm_fq_callback(void *, void *);
+RPY_EXTERN void boehm_fq_register(struct boehm_fq_s **, void *);
RPY_EXTERN void *boehm_fq_next_dead(struct boehm_fq_s **);
#define OP_GC__DISABLE_FINALIZERS(r) boehm_gc_finalizer_lock++
@@ -115,8 +115,7 @@
boehm_gc_finalizer_notifier())
#define OP_BOEHM_FQ_REGISTER(tagindex, obj, r) \
- GC_REGISTER_FINALIZER(obj, boehm_fq_callback, \
- boehm_fq_queues + tagindex, NULL, NULL)
+ boehm_fq_register(boehm_fq_queues + tagindex, obj)
#define OP_BOEHM_FQ_NEXT_DEAD(tagindex, r) \
r = boehm_fq_next_dead(boehm_fq_queues + tagindex)
diff --git a/rpython/translator/revdb/interact.py b/rpython/translator/revdb/interact.py
--- a/rpython/translator/revdb/interact.py
+++ b/rpython/translator/revdb/interact.py
@@ -1,4 +1,4 @@
-import sys, re
+import sys, os, re
import subprocess, socket
import traceback
from contextlib import contextmanager
@@ -15,12 +15,15 @@
def __init__(self, revdb_log_filename, executable=None):
with open(revdb_log_filename, 'rb') as f:
header = f.readline()
- fields = header.split('\t')
+ assert header.endswith('\n')
+ fields = header[:-1].split('\t')
if len(fields) < 2 or fields[0] != 'RevDB:':
raise ValueError("file %r is not a RevDB log" % (
revdb_log_filename,))
if executable is None:
executable = fields[1]
+ if not os.path.isfile(executable):
+ raise ValueError("executable %r not found" % (executable,))
self.pgroup = ReplayProcessGroup(executable, revdb_log_filename)
self.print_extra_pending_info = None
diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -11,6 +11,7 @@
#include <ctype.h>
#include <setjmp.h>
#include <signal.h>
+#include <search.h>
#include "structdef.h"
#include "forwarddecl.h"
@@ -73,6 +74,11 @@
void rpy_reverse_db_teardown(void)
{
uint64_t stop_points;
+ if (RPY_RDB_REPLAY) {
+ /* hack: prevents RPY_REVDB_EMIT() from calling
+ rpy_reverse_db_fetch(), which has nothing more to fetch now */
+ rpy_revdb.buf_limit += 1;
+ }
RPY_REVDB_EMIT(stop_points = rpy_revdb.stop_point_seen; ,
uint64_t _e, stop_points);
@@ -173,8 +179,9 @@
{
ssize_t content_size = current_packet_size();
if (content_size != 0) {
+ char *p = rpy_rev_buffer;
assert(0 < content_size && content_size <= 32767);
- *(int16_t *)rpy_rev_buffer = content_size;
+ *(int16_t *)p = content_size;
flush_buffer();
}
}
@@ -204,6 +211,8 @@
Then, call boehm_fq_trigger(), which calls finalizer_trigger().
*/
int i;
+ char *p = rpy_rev_buffer;
+
rpy_reverse_db_flush();
GC_invoke_finalizers();
@@ -219,7 +228,7 @@
"record_stop_point emitted unexpectedly to the rdb log\n");
exit(1);
}
- *(int16_t *)rpy_rev_buffer = ASYNC_FINALIZER_TRIGGER;
+ *(int16_t *)p = ASYNC_FINALIZER_TRIGGER;
memcpy(rpy_revdb.buf_p, &rpy_revdb.stop_point_seen, sizeof(uint64_t));
rpy_revdb.buf_p += sizeof(uint64_t);
flush_buffer();
@@ -294,7 +303,8 @@
exit(1);
}
if (pwrite(rpy_rev_fileno, &new, 1, offset) != 1) {
- fprintf(stderr, "can't patch log position %lld\n", offset);
+ fprintf(stderr, "can't patch log position %lld\n",
+ (long long)offset);
exit(1);
}
}
@@ -446,6 +456,7 @@
static uint64_t stopped_time;
static uint64_t stopped_uid;
static uint64_t total_stop_points;
+static uint64_t finalizer_trigger_saved_break;
static jmp_buf jmp_buf_cancel_execution;
static void (*pending_after_forward)(void);
static RPyString *empty_string;
@@ -453,6 +464,7 @@
static int last_recorded_breakpoint_num;
static char breakpoint_mode;
static uint64_t *future_ids, *future_next_id;
+static void *finalizer_tree;
static void attach_gdb(void)
{
@@ -606,6 +618,11 @@
/* ignore the SIGCHLD signals so that child processes don't become
zombies */
signal(SIGCHLD, SIG_IGN);
+
+ /* initiate the read, which is always at least one byte ahead of
+ RPY_REVDB_EMIT() in order to detect the ASYNC_* operations
+ early enough. */
+ rpy_reverse_db_fetch(__FILE__, __LINE__);
}
static void fetch_more(ssize_t keep, ssize_t expected_size)
@@ -622,15 +639,19 @@
}
RPY_EXTERN
-char *rpy_reverse_db_fetch(const char *file, int line)
+void rpy_reverse_db_fetch(const char *file, int line)
{
if (!flag_io_disabled) {
ssize_t keep;
ssize_t full_packet_size;
+ int16_t header;
+
if (rpy_revdb.buf_limit != rpy_revdb.buf_p) {
fprintf(stderr, "bad log format: incomplete packet\n");
exit(1);
}
+
+ fetch_more_if_needed:
keep = rpy_revdb.buf_readend - rpy_revdb.buf_p;
assert(keep >= 0);
@@ -639,13 +660,41 @@
fetch_more(keep, sizeof(int16_t));
keep = rpy_revdb.buf_readend - rpy_rev_buffer;
}
- full_packet_size = sizeof(int16_t) + *(int16_t *)rpy_revdb.buf_p;
- if (keep < full_packet_size) {
+ header = *(int16_t *)rpy_revdb.buf_p;
+ if (header < 0) {
+ int64_t bp;
+
+ switch (header) {
+
+ case ASYNC_FINALIZER_TRIGGER:
+ if (finalizer_trigger_saved_break != 0) {
+ fprintf(stderr, "unexpected multiple "
+ "ASYNC_FINALIZER_TRIGGER\n");
+ exit(1);
+ }
+ full_packet_size = sizeof(int16_t) + sizeof(int64_t);
+ if (keep < full_packet_size)
+ fetch_more(keep, full_packet_size);
+ memcpy(&bp, rpy_revdb.buf_p + sizeof(int16_t), sizeof(int64_t));
+ rpy_revdb.buf_p += full_packet_size;
+ if (bp <= rpy_revdb.stop_point_seen) {
+ fprintf(stderr, "invalid finalizer break point\n");
+ exit(1);
+ }
+ finalizer_trigger_saved_break = rpy_revdb.stop_point_break;
+ rpy_revdb.stop_point_break = bp;
+ goto fetch_more_if_needed;
+
+ default:
+ fprintf(stderr, "bad packet header %d", (int)header);
+ exit(1);
+ }
+ }
+ full_packet_size = sizeof(int16_t) + header;
+ if (keep < full_packet_size)
fetch_more(keep, full_packet_size);
- }
rpy_revdb.buf_limit = rpy_revdb.buf_p + full_packet_size;
rpy_revdb.buf_p += sizeof(int16_t);
- return rpy_revdb.buf_p;
}
else {
/* this is called when we are in execute_rpy_command(): we are
@@ -672,8 +721,8 @@
disabled_exc[1] = pypy_g_ExcData.ed_exc_value;
pypy_g_ExcData.ed_exc_type = NULL;
pypy_g_ExcData.ed_exc_value = NULL;
- rpy_revdb.buf_p = NULL;
- rpy_revdb.buf_limit = NULL;
+ rpy_revdb.buf_p = rpy_rev_buffer; /* anything readable */
+ rpy_revdb.buf_limit = rpy_rev_buffer; /* same as buf_p */
flag_io_disabled = 1;
}
@@ -724,7 +773,7 @@
(long long)stop_points);
exit(1);
}
- if (rpy_revdb.buf_p != rpy_revdb.buf_limit ||
+ if (rpy_revdb.buf_p != rpy_revdb.buf_limit - 1 ||
read(rpy_rev_fileno, dummy, 1) > 0) {
fprintf(stderr, "RevDB file error: corrupted file (too much data?)\n");
exit(1);
@@ -900,6 +949,10 @@
static void replay_stop_point(void)
{
+ if (finalizer_trigger_saved_break != 0) {
+ abort();
+ }
+
while (rpy_revdb.stop_point_break == rpy_revdb.stop_point_seen) {
save_state();
breakpoint_mode = 0;
@@ -911,8 +964,6 @@
}
else {
rpy_revdb_command_t cmd;
- if (((int64_t)stopped_uid) < 0)
- attach_gdb();
write_answer(ANSWER_READY, stopped_time, stopped_uid, 0);
read_sock(&cmd, sizeof(cmd));
@@ -1028,5 +1079,32 @@
return uid;
}
+static int _ftree_compare(const void *obj1, const void *obj2)
+{
+ const struct pypy_header0 *h1 = obj1;
+ const struct pypy_header0 *h2 = obj2;
+ if (h1->h_uid < h2->h_uid)
+ return -1;
+ if (h1->h_uid == h2->h_uid)
+ return 0;
+ else
+ return 1;
+}
+
+RPY_EXTERN
+int rpy_reverse_db_fq_register(void *obj)
+{
+ if (!RPY_RDB_REPLAY) {
+ return 0; /* recording */
+ }
+ else {
+ void *added = tsearch(obj, &finalizer_tree, _ftree_compare);
+ if (added != obj) {
+ fprintf(stderr, "tsearch: duplicate object\n");
+ exit(1);
+ }
+ return 1; /* replaying */
+ }
+}
/* ------------------------------------------------------------ */
diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h
--- a/rpython/translator/revdb/src-revdb/revdb_include.h
+++ b/rpython/translator/revdb/src-revdb/revdb_include.h
@@ -52,15 +52,13 @@
decl_e; \
char *_src = rpy_revdb.buf_p; \
char *_end1 = _src + sizeof(_e); \
- if (_end1 > rpy_revdb.buf_limit) { \
- _src = rpy_reverse_db_fetch(__FILE__, __LINE__); \
- _end1 = _src + sizeof(_e); \
- } \
+ memcpy(&_e, _src, sizeof(_e)); \
rpy_revdb.buf_p = _end1; \
- memcpy(&_e, _src, sizeof(_e)); \
_RPY_REVDB_PRINT((stderr, "%s:%d: read %0*llx\n", \
__FILE__, __LINE__, \
2 * sizeof(_e), (unsigned long long)_e)); \
+ if (_end1 >= rpy_revdb.buf_limit) \
+ rpy_reverse_db_fetch(__FILE__, __LINE__); \
variable = _e; \
}
@@ -116,7 +114,7 @@
rpy_reverse_db_call_destructor(obj)
RPY_EXTERN void rpy_reverse_db_flush(void);
-RPY_EXTERN char *rpy_reverse_db_fetch(const char *file, int line);
+RPY_EXTERN void rpy_reverse_db_fetch(const char *file, int line);
RPY_EXTERN void rpy_reverse_db_stop_point(void);
RPY_EXTERN void rpy_reverse_db_send_answer(int cmd, int64_t arg1, int64_t arg2,
int64_t arg3, RPyString *extra);
@@ -129,6 +127,7 @@
RPY_EXTERN void *rpy_reverse_db_weakref_create(void *target);
RPY_EXTERN void *rpy_reverse_db_weakref_deref(void *weakref);
//RPY_EXTERN void rpy_reverse_db_call_destructor(void *obj);
+RPY_EXTERN int rpy_reverse_db_fq_register(void *obj);
RPY_EXTERN void rpy_reverse_db_next_dead(void *result);
/* ------------------------------------------------------------ */
diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py
--- a/rpython/translator/revdb/test/test_basic.py
+++ b/rpython/translator/revdb/test/test_basic.py
@@ -84,10 +84,11 @@
return self.cur == len(self.buffer)
-def compile(self, entry_point, argtypes, backendopt=True,
+def compile(self, entry_point, backendopt=True,
withsmallfuncsets=None):
t = Translation(entry_point, None, gc="boehm")
self.t = t
+ t.set_backend_extra_options(c_debug_defines=True)
t.config.translation.reverse_debugger = True
t.config.translation.lldebug0 = True
if withsmallfuncsets is not None:
@@ -127,7 +128,7 @@
def main(argv):
print argv[1:]
return 9
- self.compile(main, [], backendopt=False)
+ self.compile(main, backendopt=False)
assert self.run('abc d') == '[abc, d]\n'
rdb = self.fetch_rdb([self.exename, 'abc', 'd'])
# write() call
@@ -143,7 +144,7 @@
objectmodel.compute_identity_hash(argv),
objectmodel.compute_identity_hash(argv)]
return 9
- self.compile(main, [], backendopt=False)
+ self.compile(main, backendopt=False)
out = self.run('Xx')
match = re.match(r'\[(-?\d+), \1, \1]\n', out)
assert match
@@ -167,7 +168,7 @@
def main(argv):
print lst[len(argv) & 1].x
return 9
- self.compile(main, [], backendopt=False)
+ self.compile(main, backendopt=False)
out = self.run('Xx')
assert out == '42\n'
rdb = self.fetch_rdb([self.exename, 'Xx'])
@@ -190,7 +191,7 @@
def main(argv):
print lst[len(argv) & 1].x
return 9
- self.compile(main, [], backendopt=False)
+ self.compile(main, backendopt=False)
out = self.run('Xx')
assert out == '41\n'
rdb = self.fetch_rdb([self.exename, 'Xx'])
@@ -220,7 +221,7 @@
x = f3 # now can be f1 or f2 or f3
print x()
return 9
- self.compile(main, [], backendopt=False, withsmallfuncsets=limit)
+ self.compile(main, backendopt=False, withsmallfuncsets=limit)
for input, expected_output in [
('2 3', '111\n'),
('2 3 4', '222\n'),
@@ -265,7 +266,7 @@
for x in lst:
print revdb.get_unique_id(x)
return 9
- compile(cls, main, [], backendopt=False)
+ compile(cls, main, backendopt=False)
assert run(cls, 'abc d ef') == ('abc\nd\nef\n'
'3\n0\n12\n15\n17\n')
rdb = fetch_rdb(cls, [cls.exename, 'abc', 'd', 'ef'])
@@ -324,8 +325,7 @@
debug_print('<<<', cmd.c_cmd, cmd.c_arg1,
cmd.c_arg2, cmd.c_arg3, extra, '>>>')
if extra == 'oops':
- for i in range(1000):
- print 42 # I/O not permitted
+ print 42 # I/O not permitted
if extra == 'raise-and-catch':
try:
g(extra)
@@ -373,7 +373,7 @@
revdb.stop_point()
print op
return 9
- compile(cls, main, [], backendopt=False)
+ compile(cls, main, backendopt=False)
assert run(cls, 'abc d ef') == 'abc\nd\nef\n'
def test_run_blip(self):
diff --git a/rpython/translator/revdb/test/test_process.py b/rpython/translator/revdb/test/test_process.py
--- a/rpython/translator/revdb/test/test_process.py
+++ b/rpython/translator/revdb/test/test_process.py
@@ -37,7 +37,7 @@
revdb.stop_point()
print op
return 9
- compile(cls, main, [], backendopt=False)
+ compile(cls, main, backendopt=False)
assert run(cls, 'abc d ef g h i j k l m') == (
'abc\nd\nef\ng\nh\ni\nj\nk\nl\nm\n')
diff --git a/rpython/translator/revdb/test/test_weak.py b/rpython/translator/revdb/test/test_weak.py
--- a/rpython/translator/revdb/test/test_weak.py
+++ b/rpython/translator/revdb/test/test_weak.py
@@ -26,6 +26,52 @@
ASYNC_FINALIZER_TRIGGER = 0xff46 - 2**16
+def get_finalizer_queue_main():
+ from rpython.rtyper.lltypesystem import lltype, rffi
+ from rpython.translator.tool.cbuild import ExternalCompilationInfo
+ eci = ExternalCompilationInfo(
+ pre_include_bits=["#define foobar(x) x\n"])
+ foobar = rffi.llexternal('foobar', [lltype.Signed], lltype.Signed,
+ compilation_info=eci)
+ class Glob:
+ pass
+ glob = Glob()
+ class X:
+ pass
+ class MyFinalizerQueue(rgc.FinalizerQueue):
+ Class = X
+ def finalizer_trigger(self):
+ glob.ping = True
+ fq = MyFinalizerQueue()
+ #
+ def main(argv):
+ glob.ping = False
+ lst1 = [X() for i in range(256)]
+ lst = [X() for i in range(3000)]
+ for i, x in enumerate(lst):
+ x.baz = i
+ fq.register_finalizer(x)
+ for i in range(3000):
+ lst[i] = None
+ if i % 300 == 150:
+ rgc.collect()
+ revdb.stop_point()
+ j = i + glob.ping * 1000000
+ assert foobar(j) == j
+ if glob.ping:
+ glob.ping = False
+ total = 0
+ while True:
+ x = fq.next_dead()
+ if x is None:
+ break
+ total = intmask(total * 3 + x.baz)
+ assert foobar(total) == total
+ keepalive_until_here(lst1)
+ return 9
+ return main
+
+
class TestRecording(BaseRecordingTests):
def test_weakref_create(self):
@@ -39,7 +85,7 @@
glob.r2 = weakref.ref(X())
glob.r3 = weakref.ref(X())
return 9
- self.compile(main, [], backendopt=False)
+ self.compile(main, backendopt=False)
out = self.run('Xx')
rdb = self.fetch_rdb([self.exename, 'Xx'])
# find the extra WEAKREF_DEAD
@@ -63,7 +109,7 @@
assert r1() is x1 # (*)
assert r2() is x2 # (*)
return 9
- self.compile(main, [], backendopt=False)
+ self.compile(main, backendopt=False)
out = self.run('Xx')
rdb = self.fetch_rdb([self.exename, 'Xx'])
# find the 2 + 16998 first WEAKREF_xxx (all "(*)" but the last two)
@@ -87,56 +133,15 @@
rgc.collect()
revdb.stop_point()
return 9
- self.compile(main, [], backendopt=False)
+ self.compile(main, backendopt=False)
out = self.run('Xx')
rdb = self.fetch_rdb([self.exename, 'Xx'])
x = rdb.next('q'); assert x == 3000 # number of stop points
assert rdb.done()
def test_finalizer_queue(self):
- from rpython.rtyper.lltypesystem import lltype, rffi
- from rpython.translator.tool.cbuild import ExternalCompilationInfo
- eci = ExternalCompilationInfo(
- pre_include_bits=["#define foobar(x) x\n"])
- foobar = rffi.llexternal('foobar', [lltype.Signed], lltype.Signed,
- compilation_info=eci)
- class Glob:
- pass
- glob = Glob()
- class X:
- pass
- class MyFinalizerQueue(rgc.FinalizerQueue):
- Class = X
- def finalizer_trigger(self):
- glob.ping = True
- fq = MyFinalizerQueue()
- #
- def main(argv):
- glob.ping = False
- lst1 = [X() for i in range(256)]
- lst = [X() for i in range(3000)]
- for i, x in enumerate(lst):
- x.baz = i
- fq.register_finalizer(x)
- for i in range(3000):
- lst[i] = None
- if i % 300 == 150:
- rgc.collect()
- revdb.stop_point()
- j = i + glob.ping * 1000000
- assert foobar(j) == j
- if glob.ping:
- glob.ping = False
- total = 0
- while True:
- x = fq.next_dead()
- if x is None:
- break
- total = intmask(total * 3 + x.baz)
- assert foobar(total) == total
- keepalive_until_here(lst1)
- return 9
- self.compile(main, [], backendopt=False)
+ main = get_finalizer_queue_main()
+ self.compile(main, backendopt=False)
out = self.run('Xx')
rdb = self.fetch_rdb([self.exename, 'Xx'])
uid_seen = set()
@@ -196,7 +201,7 @@
assert foobar(x) == x
print x
return 9
- self.compile(main, [], backendopt=False)
+ self.compile(main, backendopt=False)
out = self.run('Xx')
assert 1500 < int(out) <= 3000
rdb = self.fetch_rdb([self.exename, 'Xx'])
@@ -212,7 +217,7 @@
assert rdb.done()
-class TestReplaying(InteractiveTests):
+class TestReplayingWeakref(InteractiveTests):
expected_stop_points = 1
def setup_class(cls):
@@ -255,7 +260,7 @@
keepalive_until_here(keepalive)
revdb.stop_point()
return 9
- compile(cls, main, [], backendopt=False)
+ compile(cls, main, backendopt=False)
output = run(cls, '')
lines = output.splitlines()
assert lines[-1].startswith('prebuilt') and lines[-1].endswith(
@@ -268,3 +273,18 @@
# the asserts are replayed; if we get here it means they passed again
child.send(Message(CMD_FORWARD, 1))
child.expect(ANSWER_AT_END)
+
+
+class TestReplayingFinalizerQueue(InteractiveTests):
+ expected_stop_points = 3000
+
+ def setup_class(cls):
+ from rpython.translator.revdb.test.test_basic import compile, run
+ main = get_finalizer_queue_main()
+ compile(cls, main, backendopt=False)
+ run(cls, '')
+
+ def test_replaying_finalizer_queue(self):
+ child = self.replay()
+ child.send(Message(CMD_FORWARD, 3001))
+ child.expect(ANSWER_AT_END)
More information about the pypy-commit
mailing list