[pypy-commit] stmgc use-gcc: hg merge default
arigo
noreply at buildbot.pypy.org
Thu Jul 2 13:10:27 CEST 2015
Author: Armin Rigo <arigo at tunes.org>
Branch: use-gcc
Changeset: r1885:e7c5946182cc
Date: 2015-07-02 11:38 +0100
http://bitbucket.org/pypy/stmgc/changeset/e7c5946182cc/
Log: hg merge default
diff too long, truncating to 2000 out of 5492 lines
diff --git a/c7/demo/Makefile b/c7/demo/Makefile
--- a/c7/demo/Makefile
+++ b/c7/demo/Makefile
@@ -19,20 +19,18 @@
COMMON = -I.. -pthread -lrt -g -Wall -Werror -DSTM_LARGEMALLOC_TEST
-CC = gcc-seg-gs
-
# note that 'build' is partially optimized but still contains all asserts
debug-%: %.c ${H_FILES} ${C_FILES}
- $(CC) $(COMMON) -DSTM_DEBUGPRINT -DSTM_GC_NURSERY=128 -O0 \
+ clang $(COMMON) -DSTM_DEBUGPRINT -DSTM_GC_NURSERY=128 -O0 \
$< -o debug-$* ../stmgc.c
build-%: %.c ${H_FILES} ${C_FILES}
- $(CC) $(COMMON) -DSTM_GC_NURSERY=128 -O1 $< -o build-$* ../stmgc.c
+ clang $(COMMON) -DSTM_GC_NURSERY=128 -O1 $< -o build-$* ../stmgc.c
release-%: %.c ${H_FILES} ${C_FILES}
- $(CC) $(COMMON) -DNDEBUG -O2 $< -o release-$* ../stmgc.c
+ clang $(COMMON) -DNDEBUG -O2 $< -o release-$* ../stmgc.c
release-htm-%: %.c ../../htm-c7/stmgc.? ../../htm-c7/htm.h
- $(CC) $(COMMON) -O2 $< -o release-htm-$* ../../htm-c7/stmgc.c -DUSE_HTM
+ clang $(COMMON) -O2 $< -o release-htm-$* ../../htm-c7/stmgc.c -DUSE_HTM
diff --git a/c7/demo/demo2.c b/c7/demo/demo2.c
--- a/c7/demo/demo2.c
+++ b/c7/demo/demo2.c
@@ -216,7 +216,7 @@
void teardown_list(void)
{
- STM_POP_ROOT_DROP(stm_thread_local);
+ STM_POP_ROOT_RET(stm_thread_local);
}
@@ -256,7 +256,6 @@
stm_rewind_jmp_leaveframe(&stm_thread_local, &rjbuf);
unregister_thread_local();
status = sem_post(&done); assert(status == 0);
- (void)status;
return NULL;
}
@@ -294,7 +293,6 @@
rewind_jmp_buf rjbuf;
status = sem_init(&done, 0, 0); assert(status == 0);
- (void)status;
stm_setup();
stm_register_thread_local(&stm_thread_local);
diff --git a/c7/demo/demo_random.c b/c7/demo/demo_random.c
--- a/c7/demo/demo_random.c
+++ b/c7/demo/demo_random.c
@@ -412,7 +412,6 @@
stm_unregister_thread_local(&stm_thread_local);
status = sem_post(&done); assert(status == 0);
- (void)status;
return NULL;
}
diff --git a/c7/demo/demo_random2.c b/c7/demo/demo_random2.c
--- a/c7/demo/demo_random2.c
+++ b/c7/demo/demo_random2.c
@@ -435,7 +435,6 @@
stm_unregister_thread_local(&stm_thread_local);
status = sem_post(&done); assert(status == 0);
- (void)status;
return NULL;
}
diff --git a/c7/demo/test_shadowstack.c b/c7/demo/test_shadowstack.c
--- a/c7/demo/test_shadowstack.c
+++ b/c7/demo/test_shadowstack.c
@@ -54,7 +54,7 @@
then do a major collection. It should still be found by the
tracing logic. */
stm_start_transaction(&stm_thread_local);
- STM_POP_ROOT_DROP(stm_thread_local);
+ STM_POP_ROOT_RET(stm_thread_local);
STM_POP_ROOT(stm_thread_local, node);
assert(node->value == 129821);
STM_PUSH_ROOT(stm_thread_local, NULL);
diff --git a/c7/gdb/gdb_stm.py b/c7/gdb/gdb_stm.py
--- a/c7/gdb/gdb_stm.py
+++ b/c7/gdb/gdb_stm.py
@@ -34,6 +34,12 @@
raise
Func(func.__name__)
+def int_(x):
+ if isinstance(x, gdb.Value):
+ T = gdb.lookup_type('long')
+ x = x.cast(T)
+ return int(x)
+
# -------------------------------------------------------
_nb_segments = None
@@ -43,26 +49,26 @@
def get_nb_segments():
global _nb_segments
if _nb_segments is None:
- _nb_segments = int(gdb.parse_and_eval('_stm_nb_segments'))
+ _nb_segments = int_(gdb.parse_and_eval('_stm_nb_segments'))
assert 1 < _nb_segments <= 240
return _nb_segments
def get_segment_size():
global _segment_size
if _segment_size is None:
- nb_pages = int(gdb.parse_and_eval('_stm_segment_nb_pages'))
+ nb_pages = int_(gdb.parse_and_eval('_stm_segment_nb_pages'))
_segment_size = nb_pages * 4096
return _segment_size
def get_psegment_ofs():
global _psegment_ofs
if _psegment_ofs is None:
- _psegment_ofs = int(gdb.parse_and_eval('_stm_psegment_ofs'))
+ _psegment_ofs = int_(gdb.parse_and_eval('_stm_psegment_ofs'))
return _psegment_ofs
def get_segment_base(segment_id):
assert 0 <= segment_id <= get_nb_segments()
- base = int(gdb.parse_and_eval('stm_object_pages'))
+ base = int_(gdb.parse_and_eval('stm_object_pages'))
return base + get_segment_size() * segment_id
def get_psegment(segment_id, field=''):
@@ -72,13 +78,15 @@
% (get_segment_size() * segment_id + get_psegment_ofs(), field))
def thread_to_segment_id(thread_id):
- base = int(gdb.parse_and_eval('stm_object_pages'))
+ base = int_(gdb.parse_and_eval('stm_object_pages'))
for j in range(1, get_nb_segments() + 1):
- ts = get_psegment(j, '->transaction_state')
- if int(ts) != 0:
- ti = get_psegment(j, '->pub.running_thread->creating_pthread[0]')
- if int(ti) == thread_id:
- return j
+ #ti = get_psegment(j, '->pub.running_thread->creating_pthread[0]')
+ ti = get_psegment(j, '->running_pthread')
+ if int_(ti) == thread_id:
+ ts = get_psegment(j, '->transaction_state')
+ if int_(ts) == 0:
+ print >> sys.stderr, "note: transaction_state == 0"
+ return j
raise Exception("thread not found: %r" % (thread_id,))
def interactive_segment_base(thread=None):
@@ -92,10 +100,10 @@
thread_id = int(fields[2], 16)
segment_id = thread_to_segment_id(thread_id)
elif thread.type.code == gdb.TYPE_CODE_INT:
- if 0 <= int(thread) < 256:
- segment_id = int(thread)
+ if 0 <= int_(thread) < 256:
+ segment_id = int_(thread)
else:
- thread_id = int(thread)
+ thread_id = int_(thread)
segment_id = thread_to_segment_id(thread_id)
else:
raise TypeError("'thread' argument must be an int or not given")
@@ -105,12 +113,14 @@
def gc(p=None, thread=None):
sb = interactive_segment_base(thread)
if p is not None and p.type.code == gdb.TYPE_CODE_PTR:
- return gdb.Value(sb + int(p)).cast(p.type).dereference()
- elif p is None or int(p) == 0:
+ return gdb.Value(sb + int_(p)).cast(p.type).dereference()
+ else:
+ if p is None:
+ p = 0
+ else:
+ p = int_(p)
T = gdb.lookup_type('char').pointer()
- return gdb.Value(sb).cast(T)
- else:
- raise TypeError("gc() first argument must be a GC pointer or 0")
+ return gdb.Value(sb + p).cast(T)
@gdb_function
def psegment(thread=None):
diff --git a/c7/stm/core.c b/c7/stm/core.c
--- a/c7/stm/core.c
+++ b/c7/stm/core.c
@@ -45,6 +45,7 @@
#endif
}
+__attribute__((always_inline))
static void write_slowpath_overflow_obj(object_t *obj, bool mark_card)
{
/* An overflow object is an object from the same transaction, but
@@ -78,6 +79,7 @@
}
}
+__attribute__((always_inline))
static void write_slowpath_common(object_t *obj, bool mark_card)
{
assert(_seems_to_be_running_transaction());
@@ -221,7 +223,6 @@
check_flag_write_barrier(obj);
}
-__attribute__((flatten))
void _stm_write_slowpath(object_t *obj)
{
write_slowpath_common(obj, /*mark_card=*/false);
@@ -240,7 +241,6 @@
return (size >= _STM_MIN_CARD_OBJ_SIZE);
}
-__attribute__((flatten))
char _stm_write_slowpath_card_extra(object_t *obj)
{
/* the PyPy JIT calls this function directly if it finds that an
diff --git a/c7/stm/forksupport.c b/c7/stm/forksupport.c
--- a/c7/stm/forksupport.c
+++ b/c7/stm/forksupport.c
@@ -58,7 +58,7 @@
/* Make a new mmap at some other address, but of the same size as
the standard mmap at stm_object_pages
*/
- int big_copy_fd = -1;
+ int big_copy_fd;
char *big_copy = setup_mmap("stmgc's fork support", &big_copy_fd);
/* Copy all the data from the two ranges of objects (large, small)
diff --git a/c7/stm/fprintcolor.c b/c7/stm/fprintcolor.c
--- a/c7/stm/fprintcolor.c
+++ b/c7/stm/fprintcolor.c
@@ -1,5 +1,3 @@
-#include <stdarg.h>
-
/* ------------------------------------------------------------ */
#ifdef STM_DEBUGPRINT
/* ------------------------------------------------------------ */
diff --git a/c7/stmgc.h b/c7/stmgc.h
--- a/c7/stmgc.h
+++ b/c7/stmgc.h
@@ -20,15 +20,7 @@
#endif
-#ifdef __SEG_GS /* on a custom patched gcc */
-# define TLPREFIX __seg_gs
-# define _STM_RM_SUFFIX :8
-#elif defined(__clang__) /* on a clang, hopefully made bug-free */
-# define TLPREFIX __attribute__((address_space(256)))
-# define _STM_RM_SUFFIX /* nothing */
-#else
-# error "needs either a GCC with __seg_gs support, or a bug-freed clang"
-#endif
+#define TLPREFIX __attribute__((address_space(256)))
typedef TLPREFIX struct object_s object_t;
typedef TLPREFIX struct stm_segment_info_s stm_segment_info_t;
@@ -42,11 +34,11 @@
'STM_SEGMENT->transaction_read_version' if and only if the
object was read in the current transaction. The nurseries
also have corresponding read markers, but they are never used. */
- unsigned char rm _STM_RM_SUFFIX;
+ uint8_t rm;
};
struct stm_segment_info_s {
- unsigned int transaction_read_version;
+ uint8_t transaction_read_version;
int segment_num;
char *segment_base;
stm_char *nursery_current;
@@ -296,7 +288,6 @@
#define STM_PUSH_ROOT(tl, p) ((tl).shadowstack++->ss = (object_t *)(p))
#define STM_POP_ROOT(tl, p) ((p) = (typeof(p))((--(tl).shadowstack)->ss))
#define STM_POP_ROOT_RET(tl) ((--(tl).shadowstack)->ss)
-#define STM_POP_ROOT_DROP(tl) ((void)(--(tl).shadowstack))
/* Every thread needs to have a corresponding stm_thread_local_t
@@ -311,12 +302,7 @@
/* At some key places, like the entry point of the thread and in the
function with the interpreter's dispatch loop, you need to declare
- a local variable of type 'rewind_jmp_buf' and call these macros.
- IMPORTANT: a function in which you call stm_rewind_jmp_enterframe()
- must never change the value of its own arguments! If they are
- passed on the stack, gcc can change the value directly there, but
- we're missing the logic to save/restore this part!
-*/
+ a local variable of type 'rewind_jmp_buf' and call these macros. */
#define stm_rewind_jmp_enterprepframe(tl, rjbuf) \
rewind_jmp_enterprepframe(&(tl)->rjthread, rjbuf, (tl)->shadowstack)
#define stm_rewind_jmp_enterframe(tl, rjbuf) \
@@ -520,7 +506,7 @@
#define STM_POP_MARKER(tl) ({ \
object_t *_popped = STM_POP_ROOT_RET(tl); \
- STM_POP_ROOT_DROP(tl); \
+ STM_POP_ROOT_RET(tl); \
_popped; \
})
diff --git a/c7/test/common.py b/c7/test/common.py
--- a/c7/test/common.py
+++ b/c7/test/common.py
@@ -3,7 +3,7 @@
assert sys.maxint == 9223372036854775807, "requires a 64-bit environment"
# ----------
-os.environ['CC'] = 'gcc-seg-gs'
+os.environ['CC'] = 'clang'
parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
diff --git a/c7/test/support.py b/c7/test/support.py
--- a/c7/test/support.py
+++ b/c7/test/support.py
@@ -478,8 +478,7 @@
],
undef_macros=['NDEBUG'],
include_dirs=[parent_dir],
- extra_compile_args=['-g', '-O0', '-Werror', #, '-ferror-limit=1', for clang
- '-Wfatal-errors'], # for gcc
+ extra_compile_args=['-g', '-O0', '-Werror', '-ferror-limit=1'],
extra_link_args=['-g', '-lrt'],
force_generic_engine=True)
diff --git a/c7/test/test_list.py b/c7/test/test_list.py
--- a/c7/test/test_list.py
+++ b/c7/test/test_list.py
@@ -56,7 +56,7 @@
''', define_macros=[('STM_TESTS', '1')],
undef_macros=['NDEBUG'],
include_dirs=[parent_dir],
- extra_compile_args=['-g', '-O0', '-Werror'], #, '-ferror-limit=1'],
+ extra_compile_args=['-g', '-O0', '-Werror', '-ferror-limit=1'],
force_generic_engine=True)
# ____________________________________________________________
diff --git a/c7/test/test_rewind.c b/c7/test/test_rewind.c
--- a/c7/test/test_rewind.c
+++ b/c7/test/test_rewind.c
@@ -174,26 +174,12 @@
void foo(int *x) { ++*x; }
__attribute__((noinline))
-void f6(int c1, int c2, int c3, int c4, int c5, int c6, int c7,
- int c8, int c9, int c10, int c11, int c12, int c13)
+void f6(int a1, int a2, int a3, int a4, int a5, int a6, int a7,
+ int a8, int a9, int a10, int a11, int a12, int a13)
{
rewind_jmp_buf buf;
rewind_jmp_enterframe(>hread, &buf, NULL);
- int a1 = c1;
- int a2 = c2;
- int a3 = c3;
- int a4 = c4;
- int a5 = c5;
- int a6 = c6;
- int a7 = c7;
- int a8 = c8;
- int a9 = c9;
- int a10 = c10;
- int a11 = c11;
- int a12 = c12;
- int a13 = c13;
-
rewind_jmp_setjmp(>hread, NULL);
gevent(a1); gevent(a2); gevent(a3); gevent(a4);
gevent(a5); gevent(a6); gevent(a7); gevent(a8);
diff --git a/c7/test/test_rewind.py b/c7/test/test_rewind.py
--- a/c7/test/test_rewind.py
+++ b/c7/test/test_rewind.py
@@ -1,11 +1,11 @@
import os
def run_test(opt):
- err = os.system("gcc-seg-gs -g -O%s -Werror -DRJBUF_CUSTOM_MALLOC -I../stm"
+ err = os.system("clang -g -O%s -Werror -DRJBUF_CUSTOM_MALLOC -I../stm"
" -o test_rewind_O%s test_rewind.c ../stm/rewind_setjmp.c"
% (opt, opt))
if err != 0:
- raise OSError("gcc-seg-gs failed on test_rewind.c")
+ raise OSError("clang failed on test_rewind.c")
for testnum in [1, 2, 3, 4, 5, 6, 7, "TL1", "TL2"]:
print '=== O%s: RUNNING TEST %s ===' % (opt, testnum)
err = os.system("./test_rewind_O%s %s" % (opt, testnum))
diff --git a/c8/CALL_RELEASE_GIL b/c8/CALL_RELEASE_GIL
new file mode 100644
--- /dev/null
+++ b/c8/CALL_RELEASE_GIL
@@ -0,0 +1,120 @@
+
+c8-gil-like
+===========
+
+A branch to have "GIL-like" behavior for inevitable transactions: one
+not-too-short inevitable transaction that is passed around multiple
+threads.
+
+The goal is to have good fast-case behavior with the PyPy JIT around
+CALL_RELEASE_GIL. This is how it works in default (with shadowstack):
+
+
+- "rpy_fastgil" is a global variable. The value 0 means the GIL is
+ definitely unlocked; the value 1 means it is probably locked (it is
+ actually locked only if some mutex object is acquired too).
+
+- before CALL_RELEASE_GIL, we know that we have the GIL and we need to
+ release it. So we know that "rpy_fastgil" is 1, and we just write 0
+ there.
+
+- then we do the external call.
+
+- after CALL_RELEASE_GIL, two cases:
+
+ - if "rpy_fastgil" has been changed to 1 by some other thread *or*
+ if the (non-thread-local) shadowstack pointer changed, then we
+ call reacqgil_addr();
+
+ - otherwise, we swap rpy_fastgil back to 1 and we're done.
+
+- if the external call is long enough, a different thread will notice
+ that rpy_fastgil == 0 by regular polling, and grab the GIL for
+ itself by swapping it back to 1. (The changes from 0 to 1 are done
+ with atomic instructions.)
+
+- a different mechanism is used when we voluntarily release the GIL,
+ based on the mutex mentioned above. The mutex is also used by the
+ the reacqgil_addr() function if it actually needs to wait.
+
+
+Plan for porting this idea to stmgc:
+
+- we add a few macros to stmgc.h which can be used by C code, around
+ external calls; and we also inline these macros manually around
+ CALL_RELEASE_GIL in PyPy's JIT.
+
+- we add the "detached" mode to inevitable transactions: it means that
+ no thread is actively running this inevitable transaction for now,
+ but it was not committed yet. It is meant to be reattached, by the
+ same or a different thread.
+
+- we add a global variable, "stm_detached_inevitable_from_thread". It
+ is equal to the stm_thread_local pointer of the thread that detached
+ inevitable transaction (like rpy_fastgil == 0), or NULL if there is
+ no detached inevitable transaction (like rpy_fastgil == 1).
+
+- the macro stm_detach_inevitable_transaction() simply writes the
+ current thread's stm_thread_local pointer into the global variable
+ stm_detached_inevitable_from_thread. It can only be used if the
+ current transaction is inevitable (and in particular the inevitable
+ transaction was not detached already, because we're running it).
+ After the macro is called, the current thread is assumed not to be
+ running in a transaction any more (no more object or shadowstack
+ access).
+
+- the macro stm_reattach_transaction() does an atomic swap on
+ stm_detached_inevitable_from_thread to change it to NULL. If the
+ old value was equal to our own stm_thread_local pointer, we are done. If
+ not, we call a helper, _stm_reattach_transaction().
+
+- we also add the macro stm_detach_transation(). If the current
+ thread is inevitable it calls stm_detach_inevitable_transaction().
+ Otherwise it calls a helper, _stm_detach_noninevitable_transaction().
+
+- _stm_reattach_transaction(old): called with the old value from
+ stm_detached_inevitable_from_thread (which was swapped to be NULL just
+ now). If old != NULL, this swap had the effect that we took over
+ the inevitable transaction originally detached from a different
+ thread; we need to fix a few things like the stm_thread_local and %gs but
+ then we can continue running this reattached inevitable transaction.
+ If old == NULL, we need to fall back to the current
+ stm_start_transaction(). (A priori, there is no need to wait at
+ this point. The waiting point is later, in the optional
+ stm_become_inevitable()).
+
+- _stm_detach_noninevitable_transaction(): we try to make the
+ transaction inevitable. If it works we can then use
+ stm_detach_inevitable_transaction(). On the other hand, if we can't
+ make it inevitable without waiting, then instead we just commit it
+ and continue. In the latter case,
+ stm_detached_inevitable_from_thread is still NULL.
+
+- other place to fix: major collections. Maybe simply look inside
+ stm_detached_inevitable_from_thread, and if not NULL, grab the
+ inevitable transaction and commit it now. Or maybe not. The point
+ is that we need to prevent a thread from asynchronously grabbing it
+ by an atomic swap of stm_detached_inevitable_from_thread; instead,
+ the parallel threads that finish their external calls should all
+ find NULL in this variable and call _stm_reattach_transaction()
+ which will wait for the major GC to end.
+
+- stm_become_inevitable(): if it finds a detached inevitable
+ transaction, it should attach and commit it as a way to get rid of
+ it. This is why it might be better to call directly
+ stm_start_inevitable_transaction() when possible: that one is
+ allowed to attach to a detached inevitable transaction and simply
+ return, unlike stm_become_inevitable() which must continue running
+ the existing transaction.
+
+- commit logic of a non-inevitable transaction: we wait if there is
+ an inevitable transaction. Here too, if the inevitable transaction
+ is found to be detached, we could just commit it now. Or, a better
+ approach: if we find a detached inevitable transaction we grab it
+ temporarily, and commit only the *non-inevitable* transaction if it
+ doesn't conflict. The inevitable transaction is then detached
+ again. (Note that the conflict detection is: we don't commit any
+ write to any of the objects in the inevitable transaction's
+ read-set. This relies on inevitable threads maintaining their
+ read-set correctly, which should be the case in PyPy, but needs to
+ be checked.)
diff --git a/c8/LOCKS b/c8/LOCKS
new file mode 100644
--- /dev/null
+++ b/c8/LOCKS
@@ -0,0 +1,83 @@
+
+
+main lock-free operation
+========================
+
+The main lock-free operation is at commit time: the compare-and-swap
+that attaches a new log entry after 'last_commit_log_entry'.
+
+
+
+modification_lock
+=================
+
+one per segment.
+
+acquired on segment N when we want to read or write the segment N's
+copy of 'modified_old_objects', the backup copies, etc.
+
+an important user is _stm_validate(): it locks the current segment,
+and all the other segments out of which it is going to read data
+
+could be improved, because _stm_validate() writes into the current
+segment but only reads the other ones. So far it mostly serializes
+calls to _stm_validate(): if we have two of them starting at roughly
+the same time, they need both to acquire the modification_lock of at
+least the segment that did the most recent commit --- even though it
+could proceed in parallel if they could both realize that they only
+want to read from that same segment.
+
+same, handle_segfault_in_page() acquires two modification_locks: the
+current segment (which needs to be written to), and the
+'copy_from_segnum' (which only needs to be read from).
+
+the current segment modification_lock is also acquired briefly
+whenever we change our segment's 'modified_old_objects'.
+
+_validate_and_attach() needs to have its own segment's
+modification_lock *around* the compare-and-swap, so that
+_stm_validate() sees either the commit not done and the backup copies
+still in modified_old_objects, or the commit done and no backup copies
+any more.
+
+
+--- UPDATE: modification_lock is now done with pthread_rwlock_xxx().
+
+
+
+privatization_lock
+==================
+
+one per segment. Works like a single "reader-writer" lock: each
+segment acquires either only its copy ("reader") or all of them
+("writer").
+
+"Reader" status is needed to call get_page_status_in().
+"Writer" status is needed to call set_page_status_in/page_mark_(in)accessible.
+
+Essential "writers":
+- handle_segfault_in_page(), but it only writes the status for the current seg
+
+Essential "readers":
+- _stm_validate()
+- push_large_overflow_objects_to_other_segments()
+- nursery.c calling synchronize_object_enqueue()
+
+
+
+mutex and conditions
+====================
+
+There is also one global mutex and a few condition codes. It's
+unclear if these are still the best solution.
+
+The mutex is acquired in stm_start_transaction() and in
+stm_commit_transaction(). The main purpose is to wait for or signal
+the C_SEGMENT_FREE condition code.
+
+The C_AT_SAFE_POINT and C_REQUEST_REMOVED condition codes are used by
+synchronize_all_threads(). That's used only in rare cases, for
+example because we want to start a major collection.
+
+The mutex also needs to be acquired for rewind_longjmp's setjmp() and
+longjmp() equivalent.
diff --git a/c8/TODO b/c8/TODO
--- a/c8/TODO
+++ b/c8/TODO
@@ -1,17 +1,11 @@
+
+- fix markers (e.g. become_inevitable doesn't seem to show up)
- improve sync of small objs on commit (see FLAG_SYNC_LARGE in nursery.c)
-- non-zeroed nursery:
- read-the-docs benchmark shows 8% time spent in memset of throw_away_nursery
-
- reshare pages:
make seg0 MAP_SHARED in order to re-share private pages during major GC
-- avoid usleep(10) when waiting for an inevitable transaction:
- we do this sleep when we try to commit and another inev transaction is
- currently running. idea: signal the inev transaction to do the commit
- for us
-
- maybe re-implement the "please commit soon" signal
- the highest_overflow_number can overflow after 2**30 non-collect-time
@@ -28,3 +22,9 @@
- avoid __builtin_frame_address(0) in precisely the performance-critical
functions like the interpreter main loop
+
+
+---------------------------
+DONE:
+- non-zeroed nursery:
+ read-the-docs benchmark shows 8% time spent in memset of throw_away_nursery
diff --git a/c8/demo/demo_random.c b/c8/demo/demo_random.c
--- a/c8/demo/demo_random.c
+++ b/c8/demo/demo_random.c
@@ -8,6 +8,8 @@
#include <sys/wait.h>
#include "stmgc.h"
+#include "stm/fprintcolor.h"
+#include "stm/fprintcolor.c"
#define NUMTHREADS 2
#define STEPS_PER_THREAD 500
@@ -48,8 +50,10 @@
int num_roots;
int num_roots_at_transaction_start;
int steps_left;
+ long globally_unique;
};
__thread struct thread_data td;
+static long progress = 1;
struct thread_data *_get_td(void)
{
@@ -57,9 +61,16 @@
}
+long check_size(long size)
+{
+ assert(size >= sizeof(struct node_s));
+ assert(size <= sizeof(struct node_s) + 4096*70);
+ return size;
+}
+
ssize_t stmcb_size_rounded_up(struct object_s *ob)
{
- return ((struct node_s*)ob)->my_size;
+ return check_size(((struct node_s*)ob)->my_size);
}
void stmcb_trace(struct object_s *obj, void visit(object_t **))
@@ -69,7 +80,8 @@
/* and the same value at the end: */
/* note, ->next may be the same as last_next */
- nodeptr_t *last_next = (nodeptr_t*)((char*)n + n->my_size - sizeof(void*));
+ nodeptr_t *last_next = (nodeptr_t*)((char*)n + check_size(n->my_size)
+ - sizeof(void*));
assert(n->next == *last_next);
@@ -113,36 +125,36 @@
}
}
-void reload_roots()
-{
- int i;
- assert(td.num_roots == td.num_roots_at_transaction_start);
- for (i = td.num_roots_at_transaction_start - 1; i >= 0; i--) {
- if (td.roots[i])
- STM_POP_ROOT(stm_thread_local, td.roots[i]);
- }
-
- for (i = 0; i < td.num_roots_at_transaction_start; i++) {
- if (td.roots[i])
- STM_PUSH_ROOT(stm_thread_local, td.roots[i]);
- }
-}
-
void push_roots()
{
int i;
+ assert(td.num_roots_at_transaction_start <= td.num_roots);
for (i = td.num_roots_at_transaction_start; i < td.num_roots; i++) {
if (td.roots[i])
STM_PUSH_ROOT(stm_thread_local, td.roots[i]);
}
+ STM_SEGMENT->no_safe_point_here = 0;
}
void pop_roots()
{
int i;
- for (i = td.num_roots - 1; i >= td.num_roots_at_transaction_start; i--) {
- if (td.roots[i])
+ STM_SEGMENT->no_safe_point_here = 1;
+
+ assert(td.num_roots_at_transaction_start <= td.num_roots);
+ for (i = td.num_roots - 1; i >= 0; i--) {
+ if (td.roots[i]) {
STM_POP_ROOT(stm_thread_local, td.roots[i]);
+ assert(td.roots[i]);
+ }
+ }
+
+ dprintf(("stm_is_inevitable() = %d\n", (int)stm_is_inevitable()));
+ for (i = 0; i < td.num_roots_at_transaction_start; i++) {
+ if (td.roots[i]) {
+ dprintf(("root %d: %p\n", i, td.roots[i]));
+ STM_PUSH_ROOT(stm_thread_local, td.roots[i]);
+ }
}
}
@@ -150,6 +162,7 @@
{
int i;
assert(idx >= td.num_roots_at_transaction_start);
+ assert(idx < td.num_roots);
for (i = idx; i < td.num_roots - 1; i++)
td.roots[i] = td.roots[i + 1];
@@ -158,6 +171,7 @@
void add_root(objptr_t r)
{
+ assert(td.num_roots_at_transaction_start <= td.num_roots);
if (r && td.num_roots < MAXROOTS) {
td.roots[td.num_roots++] = r;
}
@@ -184,7 +198,8 @@
nodeptr_t n = (nodeptr_t)p;
/* and the same value at the end: */
- nodeptr_t TLPREFIX *last_next = (nodeptr_t TLPREFIX *)((stm_char*)n + n->my_size - sizeof(void*));
+ nodeptr_t TLPREFIX *last_next = (nodeptr_t TLPREFIX *)((stm_char*)n +
+ check_size(n->my_size) - sizeof(void*));
assert(n->next == *last_next);
n->next = (nodeptr_t)v;
*last_next = (nodeptr_t)v;
@@ -196,7 +211,8 @@
nodeptr_t n = (nodeptr_t)p;
/* and the same value at the end: */
- nodeptr_t TLPREFIX *last_next = (nodeptr_t TLPREFIX *)((stm_char*)n + n->my_size - sizeof(void*));
+ nodeptr_t TLPREFIX *last_next = (nodeptr_t TLPREFIX *)((stm_char*)n +
+ check_size(n->my_size) - sizeof(void*));
OPT_ASSERT(n->next == *last_next);
return n->next;
@@ -229,7 +245,7 @@
sizeof(struct node_s) + (get_rand(100000) & ~15),
sizeof(struct node_s) + 4096,
sizeof(struct node_s) + 4096*70};
- size_t size = sizes[get_rand(4)];
+ size_t size = check_size(sizes[get_rand(4)]);
p = stm_allocate(size);
nodeptr_t n = (nodeptr_t)p;
n->sig = SIGNATURE;
@@ -240,7 +256,6 @@
n->next = NULL;
*last_next = NULL;
pop_roots();
- /* reload_roots not necessary, all are old after start_transaction */
break;
case 4: // read and validate 'p'
read_barrier(p);
@@ -288,6 +303,15 @@
return p;
}
+static void end_gut(void)
+{
+ if (td.globally_unique != 0) {
+ fprintf(stderr, "[GUT END]");
+ assert(progress == td.globally_unique);
+ td.globally_unique = 0;
+ stm_resume_all_other_threads();
+ }
+}
objptr_t do_step(objptr_t p)
{
@@ -308,8 +332,14 @@
return NULL;
} else if (get_rand(240) == 1) {
push_roots();
- stm_become_globally_unique_transaction(&stm_thread_local, "really");
- fprintf(stderr, "[GUT/%d]", (int)STM_SEGMENT->segment_num);
+ if (td.globally_unique == 0) {
+ stm_stop_all_other_threads();
+ td.globally_unique = progress;
+ fprintf(stderr, "[GUT/%d]", (int)STM_SEGMENT->segment_num);
+ }
+ else {
+ end_gut();
+ }
pop_roots();
return NULL;
}
@@ -347,37 +377,53 @@
objptr_t p;
- stm_start_transaction(&stm_thread_local);
+ stm_enter_transactional_zone(&stm_thread_local);
assert(td.num_roots >= td.num_roots_at_transaction_start);
td.num_roots = td.num_roots_at_transaction_start;
p = NULL;
pop_roots(); /* does nothing.. */
- reload_roots();
while (td.steps_left-->0) {
if (td.steps_left % 8 == 0)
fprintf(stdout, "#");
- assert(p == NULL || ((nodeptr_t)p)->sig == SIGNATURE);
+ int local_seg = STM_SEGMENT->segment_num;
+ int p_sig = p == NULL ? 0 : ((nodeptr_t)p)->sig;
+
+ assert(p == NULL || p_sig == SIGNATURE);
+ (void)local_seg;
+ (void)p_sig;
+
+ if (!td.globally_unique)
+ ++progress; /* racy, but good enough */
p = do_step(p);
if (p == (objptr_t)-1) {
push_roots();
+ end_gut();
long call_fork = (arg != NULL && *(long *)arg);
if (call_fork == 0) { /* common case */
- stm_commit_transaction();
- td.num_roots_at_transaction_start = td.num_roots;
- if (get_rand(100) < 98) {
- stm_start_transaction(&stm_thread_local);
- } else {
- stm_start_inevitable_transaction(&stm_thread_local);
+ if (get_rand(100) < 50) {
+ stm_leave_transactional_zone(&stm_thread_local);
+ /* Nothing here; it's unlikely that a different thread
+ manages to steal the detached inev transaction.
+ Give them a little chance with a usleep(). */
+ dprintf(("sleep...\n"));
+ usleep(1);
+ dprintf(("sleep done\n"));
+ td.num_roots_at_transaction_start = td.num_roots;
+ stm_enter_transactional_zone(&stm_thread_local);
+ }
+ else {
+ _stm_commit_transaction();
+ td.num_roots_at_transaction_start = td.num_roots;
+ _stm_start_transaction(&stm_thread_local);
}
td.num_roots = td.num_roots_at_transaction_start;
p = NULL;
pop_roots();
- reload_roots();
}
else {
/* run a fork() inside the transaction */
@@ -401,16 +447,17 @@
}
}
push_roots();
- stm_commit_transaction();
+ end_gut();
+ stm_force_transaction_break(&stm_thread_local);
/* even out the shadow stack before leaveframe: */
- stm_start_inevitable_transaction(&stm_thread_local);
+ stm_become_inevitable(&stm_thread_local, "before leaveframe");
while (td.num_roots > 0) {
td.num_roots--;
objptr_t t;
STM_POP_ROOT(stm_thread_local, t);
}
- stm_commit_transaction();
+ stm_leave_transactional_zone(&stm_thread_local);
stm_rewind_jmp_leaveframe(&stm_thread_local, &rjbuf);
stm_unregister_thread_local(&stm_thread_local);
diff --git a/c8/demo/demo_random2.c b/c8/demo/demo_random2.c
--- a/c8/demo/demo_random2.c
+++ b/c8/demo/demo_random2.c
@@ -8,6 +8,8 @@
#include <sys/wait.h>
#include "stmgc.h"
+#include "stm/fprintcolor.h"
+#include "stm/fprintcolor.c"
#define NUMTHREADS 3
#define STEPS_PER_THREAD 50000
@@ -52,8 +54,10 @@
int active_roots_num;
long roots_on_ss;
long roots_on_ss_at_tr_start;
+ long globally_unique;
};
__thread struct thread_data td;
+static long progress = 1;
struct thread_data *_get_td(void)
{
@@ -61,9 +65,16 @@
}
+long check_size(long size)
+{
+ assert(size >= sizeof(struct node_s));
+ assert(size <= sizeof(struct node_s) + 4096*70);
+ return size;
+}
+
ssize_t stmcb_size_rounded_up(struct object_s *ob)
{
- return ((struct node_s*)ob)->my_size;
+ return check_size(((struct node_s*)ob)->my_size);
}
void stmcb_trace(struct object_s *obj, void visit(object_t **))
@@ -73,7 +84,8 @@
/* and the same value at the end: */
/* note, ->next may be the same as last_next */
- nodeptr_t *last_next = (nodeptr_t*)((char*)n + n->my_size - sizeof(void*));
+ nodeptr_t *last_next = (nodeptr_t*)((char*)n + check_size(n->my_size)
+ - sizeof(void*));
assert(n->next == *last_next);
@@ -193,7 +205,8 @@
nodeptr_t n = (nodeptr_t)p;
/* and the same value at the end: */
- nodeptr_t TLPREFIX *last_next = (nodeptr_t TLPREFIX *)((stm_char*)n + n->my_size - sizeof(void*));
+ nodeptr_t TLPREFIX *last_next = (nodeptr_t TLPREFIX *)((stm_char*)n +
+ check_size(n->my_size) - sizeof(void*));
assert(n->next == *last_next);
n->next = (nodeptr_t)v;
*last_next = (nodeptr_t)v;
@@ -205,7 +218,8 @@
nodeptr_t n = (nodeptr_t)p;
/* and the same value at the end: */
- nodeptr_t TLPREFIX *last_next = (nodeptr_t TLPREFIX *)((stm_char*)n + n->my_size - sizeof(void*));
+ nodeptr_t TLPREFIX *last_next = (nodeptr_t TLPREFIX *)((stm_char*)n +
+ check_size(n->my_size) - sizeof(void*));
OPT_ASSERT(n->next == *last_next);
return n->next;
@@ -239,6 +253,7 @@
sizeof(struct node_s)+32, sizeof(struct node_s)+48,
sizeof(struct node_s) + (get_rand(100000) & ~15)};
size_t size = sizes[get_rand(sizeof(sizes) / sizeof(size_t))];
+ size = check_size(size);
p = stm_allocate(size);
nodeptr_t n = (nodeptr_t)p;
n->sig = SIGNATURE;
@@ -296,6 +311,16 @@
return p;
}
+static void end_gut(void)
+{
+ if (td.globally_unique != 0) {
+ fprintf(stderr, "[GUT END]");
+ assert(progress == td.globally_unique);
+ td.globally_unique = 0;
+ stm_resume_all_other_threads();
+ }
+}
+
void frame_loop();
objptr_t do_step(objptr_t p)
{
@@ -309,13 +334,22 @@
p = simple_events(p, _r);
} else if (get_rand(20) == 1) {
long pushed = push_roots();
- stm_commit_transaction();
- td.roots_on_ss_at_tr_start = td.roots_on_ss;
-
- if (get_rand(100) < 98) {
- stm_start_transaction(&stm_thread_local);
- } else {
- stm_start_inevitable_transaction(&stm_thread_local);
+ end_gut();
+ if (get_rand(100) < 95) {
+ stm_leave_transactional_zone(&stm_thread_local);
+ /* Nothing here; it's unlikely that a different thread
+ manages to steal the detached inev transaction.
+ Give them a little chance with a usleep(). */
+ dprintf(("sleep...\n"));
+ usleep(1);
+ dprintf(("sleep done\n"));
+ td.roots_on_ss_at_tr_start = td.roots_on_ss;
+ stm_enter_transactional_zone(&stm_thread_local);
+ }
+ else {
+ _stm_commit_transaction();
+ td.roots_on_ss_at_tr_start = td.roots_on_ss;
+ _stm_start_transaction(&stm_thread_local);
}
td.roots_on_ss = td.roots_on_ss_at_tr_start;
td.active_roots_num = 0;
@@ -331,15 +365,21 @@
} else if (get_rand(20) == 1) {
long pushed = push_roots();
stm_become_inevitable(&stm_thread_local, "please");
- assert(stm_is_inevitable());
+ assert(stm_is_inevitable(&stm_thread_local));
pop_roots(pushed);
p= NULL;
} else if (get_rand(20) == 1) {
p = (objptr_t)-1; // possibly fork
- } else if (get_rand(20) == 1) {
+ } else if (get_rand(100) == 1) {
long pushed = push_roots();
- stm_become_globally_unique_transaction(&stm_thread_local, "really");
- fprintf(stderr, "[GUT/%d]", (int)STM_SEGMENT->segment_num);
+ if (td.globally_unique == 0) {
+ stm_stop_all_other_threads();
+ td.globally_unique = progress;
+ fprintf(stderr, "[GUT/%d]", (int)STM_SEGMENT->segment_num);
+ }
+ else {
+ end_gut();
+ }
pop_roots(pushed);
p = NULL;
}
@@ -364,6 +404,8 @@
p = do_step(p);
+ if (!td.globally_unique)
+ ++progress; /* racy, but good enough */
if (p == (objptr_t)-1) {
p = NULL;
@@ -371,6 +413,7 @@
long call_fork = (thread_may_fork != NULL && *(long *)thread_may_fork);
if (call_fork) { /* common case */
long pushed = push_roots();
+ end_gut();
/* run a fork() inside the transaction */
printf("========== FORK =========\n");
*(long*)thread_may_fork = 0;
@@ -426,7 +469,7 @@
setup_thread();
td.roots_on_ss_at_tr_start = 0;
- stm_start_transaction(&stm_thread_local);
+ stm_enter_transactional_zone(&stm_thread_local);
td.roots_on_ss = td.roots_on_ss_at_tr_start;
td.active_roots_num = 0;
@@ -435,7 +478,8 @@
frame_loop();
}
- stm_commit_transaction();
+ end_gut();
+ stm_leave_transactional_zone(&stm_thread_local);
stm_rewind_jmp_leaveframe(&stm_thread_local, &rjbuf);
stm_unregister_thread_local(&stm_thread_local);
diff --git a/c8/demo/demo_simple.c b/c8/demo/demo_simple.c
--- a/c8/demo/demo_simple.c
+++ b/c8/demo/demo_simple.c
@@ -70,18 +70,20 @@
object_t *tmp;
int i = 0;
+
+ stm_enter_transactional_zone(&stm_thread_local);
while (i < ITERS) {
- stm_start_transaction(&stm_thread_local);
tl_counter++;
if (i % 500 < 250)
STM_PUSH_ROOT(stm_thread_local, stm_allocate(16));//gl_counter++;
else
STM_POP_ROOT(stm_thread_local, tmp);
- stm_commit_transaction();
+ stm_force_transaction_break(&stm_thread_local);
i++;
}
OPT_ASSERT(org == (char *)stm_thread_local.shadowstack);
+ stm_leave_transactional_zone(&stm_thread_local);
stm_rewind_jmp_leaveframe(&stm_thread_local, &rjbuf);
stm_unregister_thread_local(&stm_thread_local);
diff --git a/c8/demo/test_shadowstack.c b/c8/demo/test_shadowstack.c
--- a/c8/demo/test_shadowstack.c
+++ b/c8/demo/test_shadowstack.c
@@ -43,17 +43,16 @@
stm_register_thread_local(&stm_thread_local);
stm_rewind_jmp_enterframe(&stm_thread_local, &rjbuf);
- stm_start_transaction(&stm_thread_local);
+ stm_enter_transactional_zone(&stm_thread_local);
node_t *node = (node_t *)stm_allocate(sizeof(struct node_s));
node->value = 129821;
STM_PUSH_ROOT(stm_thread_local, node);
STM_PUSH_ROOT(stm_thread_local, 333); /* odd value */
- stm_commit_transaction();
/* now in a new transaction, pop the node off the shadowstack, but
then do a major collection. It should still be found by the
tracing logic. */
- stm_start_transaction(&stm_thread_local);
+ stm_force_transaction_break(&stm_thread_local);
STM_POP_ROOT_RET(stm_thread_local);
STM_POP_ROOT(stm_thread_local, node);
assert(node->value == 129821);
diff --git a/c8/stm/atomic.h b/c8/stm/atomic.h
--- a/c8/stm/atomic.h
+++ b/c8/stm/atomic.h
@@ -24,24 +24,37 @@
#if defined(__i386__) || defined(__amd64__)
-# define HAVE_FULL_EXCHANGE_INSN
static inline void spin_loop(void) { asm("pause" : : : "memory"); }
static inline void write_fence(void) { asm("" : : : "memory"); }
+/*# define atomic_exchange(ptr, old, new) do { \
+ (old) = __sync_lock_test_and_set(ptr, new); \
+ } while (0)*/
#else
static inline void spin_loop(void) { asm("" : : : "memory"); }
static inline void write_fence(void) { __sync_synchronize(); }
+/*# define atomic_exchange(ptr, old, new) do { \
+ (old) = *(ptr); \
+ } while (UNLIKELY(!__sync_bool_compare_and_swap(ptr, old, new))); */
+
#endif
-#define spinlock_acquire(lock) \
- do { if (LIKELY(__sync_lock_test_and_set(&(lock), 1) == 0)) break; \
- spin_loop(); } while (1)
-#define spinlock_release(lock) \
- do { assert((lock) == 1); \
- __sync_lock_release(&(lock)); } while (0)
+static inline void _spinlock_acquire(uint8_t *plock) {
+ retry:
+ if (__builtin_expect(__sync_lock_test_and_set(plock, 1) != 0, 0)) {
+ spin_loop();
+ goto retry;
+ }
+}
+static inline void _spinlock_release(uint8_t *plock) {
+ assert(*plock == 1);
+ __sync_lock_release(plock);
+}
+#define spinlock_acquire(lock) _spinlock_acquire(&(lock))
+#define spinlock_release(lock) _spinlock_release(&(lock))
#endif /* _STM_ATOMIC_H */
diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -50,8 +50,8 @@
char *src_segment_base = (from_segnum >= 0 ? get_segment_base(from_segnum)
: NULL);
- assert(IMPLY(from_segnum >= 0, get_priv_segment(from_segnum)->modification_lock));
- assert(STM_PSEGMENT->modification_lock);
+ assert(IMPLY(from_segnum >= 0, modification_lock_check_rdlock(from_segnum)));
+ assert(modification_lock_check_wrlock(STM_SEGMENT->segment_num));
long my_segnum = STM_SEGMENT->segment_num;
DEBUG_EXPECT_SEGFAULT(false);
@@ -131,7 +131,7 @@
struct stm_commit_log_entry_s *from,
struct stm_commit_log_entry_s *to)
{
- assert(STM_PSEGMENT->modification_lock);
+ assert(modification_lock_check_wrlock(STM_SEGMENT->segment_num));
assert(from->rev_num >= to->rev_num);
/* walk BACKWARDS the commit log and update the page 'pagenum',
initially at revision 'from', until we reach the revision 'to'. */
@@ -199,8 +199,8 @@
/* before copying anything, acquire modification locks from our and
the other segment */
- uint64_t to_lock = (1UL << copy_from_segnum)| (1UL << my_segnum);
- acquire_modification_lock_set(to_lock);
+ uint64_t to_lock = (1UL << copy_from_segnum);
+ acquire_modification_lock_set(to_lock, my_segnum);
pagecopy(get_virtual_page(my_segnum, pagenum),
get_virtual_page(copy_from_segnum, pagenum));
@@ -223,7 +223,7 @@
if (src_version->rev_num > target_version->rev_num)
go_to_the_past(pagenum, src_version, target_version);
- release_modification_lock_set(to_lock);
+ release_modification_lock_set(to_lock, my_segnum);
release_all_privatization_locks();
}
@@ -324,10 +324,7 @@
/* Don't check this 'cl'. This entry is already checked */
if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) {
- //assert(first_cl->next == INEV_RUNNING);
- /* the above assert may fail when running a major collection
- while the commit of the inevitable transaction is in progress
- and the element is already attached */
+ assert(first_cl->next == INEV_RUNNING);
return true;
}
@@ -357,7 +354,7 @@
}
/* Find the set of segments we need to copy from and lock them: */
- uint64_t segments_to_lock = 1UL << my_segnum;
+ uint64_t segments_to_lock = 0;
cl = first_cl;
while ((next_cl = cl->next) != NULL) {
if (next_cl == INEV_RUNNING) {
@@ -375,8 +372,8 @@
/* HERE */
- acquire_privatization_lock(STM_SEGMENT->segment_num);
- acquire_modification_lock_set(segments_to_lock);
+ acquire_privatization_lock(my_segnum);
+ acquire_modification_lock_set(segments_to_lock, my_segnum);
/* import objects from first_cl to last_cl: */
@@ -466,8 +463,8 @@
}
/* done with modifications */
- release_modification_lock_set(segments_to_lock);
- release_privatization_lock(STM_SEGMENT->segment_num);
+ release_modification_lock_set(segments_to_lock, my_segnum);
+ release_privatization_lock(my_segnum);
}
return !needs_abort;
@@ -494,115 +491,121 @@
}
-static void wait_for_other_inevitable(struct stm_commit_log_entry_s *old)
-{
- timing_event(STM_SEGMENT->running_thread, STM_WAIT_OTHER_INEVITABLE);
-
- while (old->next == INEV_RUNNING && !safe_point_requested()) {
- spin_loop();
- usleep(10); /* XXXXXX */
- }
- timing_event(STM_SEGMENT->running_thread, STM_WAIT_DONE);
-}
-
static void reset_wb_executed_flags(void);
static void readd_wb_executed_flags(void);
static void check_all_write_barrier_flags(char *segbase, struct list_s *list);
+static void wait_for_inevitable(void)
+{
+ intptr_t detached = 0;
+
+ s_mutex_lock();
+ wait_some_more:
+ if (safe_point_requested()) {
+ /* XXXXXX if the safe point below aborts, in
+ _validate_and_attach(), 'new' leaks */
+ enter_safe_point_if_requested();
+ }
+ else if (STM_PSEGMENT->last_commit_log_entry->next == INEV_RUNNING) {
+ /* loop until C_SEGMENT_FREE_OR_SAFE_POINT_REQ is signalled, but
+ try to detach an inevitable transaction regularly */
+ detached = fetch_detached_transaction();
+ if (detached == 0) {
+ EMIT_WAIT(STM_WAIT_OTHER_INEVITABLE);
+ if (!cond_wait_timeout(C_SEGMENT_FREE_OR_SAFE_POINT_REQ, 0.00001))
+ goto wait_some_more;
+ }
+ }
+ EMIT_WAIT_DONE();
+ s_mutex_unlock();
+
+ if (detached != 0)
+ commit_fetched_detached_transaction(detached);
+}
+
+/* This is called to do stm_validate() and then attach 'new' at the
+ head of the 'commit_log_root' chained list. This function sleeps
+ and retries until it succeeds or aborts.
+*/
static void _validate_and_attach(struct stm_commit_log_entry_s *new)
{
struct stm_commit_log_entry_s *old;
OPT_ASSERT(new != NULL);
- /* we are attaching a real CL entry: */
- bool is_commit = new != INEV_RUNNING;
+ OPT_ASSERT(new != INEV_RUNNING);
- while (1) {
- if (!_stm_validate()) {
- if (new != INEV_RUNNING)
- free_cle((struct stm_commit_log_entry_s*)new);
- stm_abort_transaction();
- }
+ soon_finished_or_inevitable_thread_segment();
+
+ retry_from_start:
+ if (!_stm_validate()) {
+ free_cle(new);
+ stm_abort_transaction();
+ }
#if STM_TESTS
- if (STM_PSEGMENT->transaction_state != TS_INEVITABLE
- && STM_PSEGMENT->last_commit_log_entry->next == INEV_RUNNING) {
- /* abort for tests... */
- stm_abort_transaction();
- }
+ if (STM_PSEGMENT->transaction_state != TS_INEVITABLE
+ && STM_PSEGMENT->last_commit_log_entry->next == INEV_RUNNING) {
+ /* abort for tests... */
+ stm_abort_transaction();
+ }
#endif
- if (is_commit) {
- /* we must not remove the WB_EXECUTED flags before validation as
- it is part of a condition in import_objects() called by
- copy_bk_objs_in_page_from to not overwrite our modifications.
- So we do it here: */
- reset_wb_executed_flags();
- check_all_write_barrier_flags(STM_SEGMENT->segment_base,
- STM_PSEGMENT->modified_old_objects);
-
- /* need to remove the entries in modified_old_objects "at the same
- time" as the attach to commit log. Otherwise, another thread may
- see the new CL entry, import it, look for backup copies in this
- segment and find the old backup copies! */
- acquire_modification_lock(STM_SEGMENT->segment_num);
- }
-
- /* try to attach to commit log: */
- old = STM_PSEGMENT->last_commit_log_entry;
- if (old->next == NULL) {
- if (new != INEV_RUNNING) /* INEVITABLE */
- new->rev_num = old->rev_num + 1;
-
- if (__sync_bool_compare_and_swap(&old->next, NULL, new))
- break; /* success! */
- }
-
- if (is_commit) {
- release_modification_lock(STM_SEGMENT->segment_num);
- /* XXX: unfortunately, if we failed to attach our CL entry,
- we have to re-add the WB_EXECUTED flags before we try to
- validate again because of said condition (s.a) */
- readd_wb_executed_flags();
- }
-
- if (old->next == INEV_RUNNING && !safe_point_requested()) {
- /* we failed because there is an INEV transaction running */
- /* XXXXXX for now just sleep. We should really ask to inev
- transaction to do the commit for us, and then we can
- continue running. */
- dprintf(("_validate_and_attach(%p) failed, "
- "waiting for inevitable\n", new));
- wait_for_other_inevitable(old);
- }
-
- dprintf(("_validate_and_attach(%p) failed, enter safepoint\n", new));
-
- /* check for requested safe point. otherwise an INEV transaction
- may try to commit but cannot because of the busy-loop here. */
- /* minor gc is fine here because we did one immediately before, so
- there are no young objs anyway. major gc is fine because the
- modified_old_objects list is still populated with the same
- cl-entry objs */
- /* XXXXXXXX: memory leak if we happen to do a major gc, we get aborted
- in major_do_validation_and_minor_collections, and don't free 'new' */
- _stm_collectable_safe_point();
+ if (STM_PSEGMENT->last_commit_log_entry->next == INEV_RUNNING) {
+ wait_for_inevitable();
+ goto retry_from_start; /* redo _stm_validate() now */
}
- if (is_commit) {
+ /* we must not remove the WB_EXECUTED flags before validation as
+ it is part of a condition in import_objects() called by
+ copy_bk_objs_in_page_from to not overwrite our modifications.
+ So we do it here: */
+ reset_wb_executed_flags();
+ check_all_write_barrier_flags(STM_SEGMENT->segment_base,
+ STM_PSEGMENT->modified_old_objects);
+
+ /* need to remove the entries in modified_old_objects "at the same
+ time" as the attach to commit log. Otherwise, another thread may
+ see the new CL entry, import it, look for backup copies in this
+ segment and find the old backup copies! */
+ acquire_modification_lock_wr(STM_SEGMENT->segment_num);
+
+ /* try to attach to commit log: */
+ old = STM_PSEGMENT->last_commit_log_entry;
+ new->rev_num = old->rev_num + 1;
+ if (__sync_bool_compare_and_swap(&old->next, NULL, new)) {
+ /* success! */
/* compare with _validate_and_add_to_commit_log */
- STM_PSEGMENT->transaction_state = TS_NONE;
- STM_PSEGMENT->safe_point = SP_NO_TRANSACTION;
-
list_clear(STM_PSEGMENT->modified_old_objects);
STM_PSEGMENT->last_commit_log_entry = new;
- release_modification_lock(STM_SEGMENT->segment_num);
+ release_modification_lock_wr(STM_SEGMENT->segment_num);
+ }
+ else {
+ /* fail */
+ release_modification_lock_wr(STM_SEGMENT->segment_num);
+ /* XXX: unfortunately, if we failed to attach our CL entry,
+ we have to re-add the WB_EXECUTED flags before we try to
+ validate again because of said condition (s.a) */
+ readd_wb_executed_flags();
+
+ dprintf(("_validate_and_attach(%p) failed, retrying\n", new));
+ goto retry_from_start;
}
}
-static void _validate_and_turn_inevitable(void)
+/* This is called to do stm_validate() and then attach INEV_RUNNING to
+ the head of the 'commit_log_root' chained list. This function
+ may succeed or fail (or abort).
+*/
+static bool _validate_and_turn_inevitable(void)
{
- _validate_and_attach((struct stm_commit_log_entry_s *)INEV_RUNNING);
+ struct stm_commit_log_entry_s *old;
+
+ if (!_stm_validate())
+ stm_abort_transaction();
+
+ /* try to attach to commit log: */
+ old = STM_PSEGMENT->last_commit_log_entry;
+ return __sync_bool_compare_and_swap(&old->next, NULL, INEV_RUNNING);
}
static void _validate_and_add_to_commit_log(void)
@@ -611,6 +614,8 @@
new = _create_commit_log_entry();
if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) {
+ assert(_stm_detached_inevitable_from_thread == 0); /* running it */
+
old = STM_PSEGMENT->last_commit_log_entry;
new->rev_num = old->rev_num + 1;
OPT_ASSERT(old->next == INEV_RUNNING);
@@ -621,14 +626,15 @@
STM_PSEGMENT->modified_old_objects);
/* compare with _validate_and_attach: */
- STM_PSEGMENT->transaction_state = TS_NONE;
- STM_PSEGMENT->safe_point = SP_NO_TRANSACTION;
+ acquire_modification_lock_wr(STM_SEGMENT->segment_num);
list_clear(STM_PSEGMENT->modified_old_objects);
STM_PSEGMENT->last_commit_log_entry = new;
/* do it: */
bool yes = __sync_bool_compare_and_swap(&old->next, INEV_RUNNING, new);
OPT_ASSERT(yes);
+
+ release_modification_lock_wr(STM_SEGMENT->segment_num);
}
else {
_validate_and_attach(new);
@@ -692,7 +698,7 @@
increment_total_allocated(slice_sz);
memcpy(bk_slice, realobj + slice_off, slice_sz);
- acquire_modification_lock(STM_SEGMENT->segment_num);
+ acquire_modification_lock_wr(STM_SEGMENT->segment_num);
/* !! follows layout of "struct stm_undo_s" !! */
STM_PSEGMENT->modified_old_objects = list_append3(
STM_PSEGMENT->modified_old_objects,
@@ -700,7 +706,7 @@
(uintptr_t)bk_slice, /* bk_addr */
NEW_SLICE(slice_off, slice_sz));
dprintf(("> append slice %p, off=%lu, sz=%lu\n", bk_slice, slice_off, slice_sz));
- release_modification_lock(STM_SEGMENT->segment_num);
+ release_modification_lock_wr(STM_SEGMENT->segment_num);
slice_off += slice_sz;
}
@@ -896,6 +902,8 @@
static void touch_all_pages_of_obj(object_t *obj, size_t obj_size)
{
+ /* XXX should it be simpler, just really trying to read a dummy
+ byte in each page? */
int my_segnum = STM_SEGMENT->segment_num;
uintptr_t end_page, first_page = ((uintptr_t)obj) / 4096UL;
@@ -1121,11 +1129,12 @@
-static void _stm_start_transaction(stm_thread_local_t *tl)
+static void _do_start_transaction(stm_thread_local_t *tl)
{
assert(!_stm_in_transaction(tl));
+ tl->wait_event_emitted = 0;
- while (!acquire_thread_segment(tl)) {}
+ acquire_thread_segment(tl);
/* GS invalid before this point! */
assert(STM_PSEGMENT->safe_point == SP_NO_TRANSACTION);
@@ -1138,7 +1147,9 @@
#endif
STM_PSEGMENT->shadowstack_at_start_of_transaction = tl->shadowstack;
STM_PSEGMENT->threadlocal_at_start_of_transaction = tl->thread_local_obj;
-
+ STM_PSEGMENT->total_throw_away_nursery = 0;
+ assert(tl->self_or_0_if_atomic == (intptr_t)tl); /* not atomic */
+ assert(STM_PSEGMENT->atomic_nesting_levels == 0);
assert(list_is_empty(STM_PSEGMENT->modified_old_objects));
assert(list_is_empty(STM_PSEGMENT->large_overflow_objects));
@@ -1150,6 +1161,7 @@
assert(tree_is_cleared(STM_PSEGMENT->callbacks_on_commit_and_abort[1]));
assert(list_is_empty(STM_PSEGMENT->young_objects_with_light_finalizers));
assert(STM_PSEGMENT->finalizers == NULL);
+ assert(STM_PSEGMENT->active_queues == NULL);
#ifndef NDEBUG
/* this should not be used when objects_pointing_to_nursery == NULL */
STM_PSEGMENT->position_markers_len_old = 99999999999999999L;
@@ -1179,35 +1191,34 @@
stm_validate();
}
-long stm_start_transaction(stm_thread_local_t *tl)
+#ifdef STM_NO_AUTOMATIC_SETJMP
+static int did_abort = 0;
+#endif
+
+long _stm_start_transaction(stm_thread_local_t *tl)
{
s_mutex_lock();
#ifdef STM_NO_AUTOMATIC_SETJMP
- long repeat_count = 0; /* test/support.py */
+ long repeat_count = did_abort; /* test/support.py */
+ did_abort = 0;
#else
long repeat_count = stm_rewind_jmp_setjmp(tl);
#endif
- _stm_start_transaction(tl);
+ _do_start_transaction(tl);
+
+ if (repeat_count == 0) { /* else, 'nursery_mark' was already set
+ in abort_data_structures_from_segment_num() */
+ STM_SEGMENT->nursery_mark = ((stm_char *)_stm_nursery_start +
+ stm_fill_mark_nursery_bytes);
+ }
return repeat_count;
}
-void stm_start_inevitable_transaction(stm_thread_local_t *tl)
-{
- /* used to be more efficient, starting directly an inevitable transaction,
- but there is no real point any more, I believe */
- rewind_jmp_buf rjbuf;
- stm_rewind_jmp_enterframe(tl, &rjbuf);
-
- stm_start_transaction(tl);
- stm_become_inevitable(tl, "start_inevitable_transaction");
-
- stm_rewind_jmp_leaveframe(tl, &rjbuf);
-}
-
#ifdef STM_NO_AUTOMATIC_SETJMP
void _test_run_abort(stm_thread_local_t *tl) __attribute__((noreturn));
-int stm_is_inevitable(void)
+int stm_is_inevitable(stm_thread_local_t *tl)
{
+ assert(STM_SEGMENT->running_thread == tl);
switch (STM_PSEGMENT->transaction_state) {
case TS_REGULAR: return 0;
case TS_INEVITABLE: return 1;
@@ -1222,6 +1233,7 @@
{
stm_thread_local_t *tl = STM_SEGMENT->running_thread;
+ assert(_has_mutex());
STM_PSEGMENT->safe_point = SP_NO_TRANSACTION;
STM_PSEGMENT->transaction_state = TS_NONE;
@@ -1229,7 +1241,15 @@
list_clear(STM_PSEGMENT->objects_pointing_to_nursery);
list_clear(STM_PSEGMENT->old_objects_with_cards_set);
list_clear(STM_PSEGMENT->large_overflow_objects);
- timing_event(tl, event);
+ if (tl != NULL)
+ timing_event(tl, event);
+
+ /* If somebody is waiting for us to reach a safe point, we simply
+ signal it now and leave this transaction. This should be enough
+ for synchronize_all_threads() to retry and notice that we are
+ no longer SP_RUNNING. */
+ if (STM_SEGMENT->nursery_end != NURSERY_END)
+ cond_signal(C_AT_SAFE_POINT);
release_thread_segment(tl);
/* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */
@@ -1278,24 +1298,62 @@
}
-void stm_commit_transaction(void)
+void _stm_commit_transaction(void)
+{
+ assert(STM_PSEGMENT->running_pthread == pthread_self());
+ _core_commit_transaction(/*external=*/ false);
+}
+
+static void _core_commit_transaction(bool external)
{
exec_local_finalizers();
assert(!_has_mutex());
assert(STM_PSEGMENT->safe_point == SP_RUNNING);
- assert(STM_PSEGMENT->running_pthread == pthread_self());
+ assert(STM_PSEGMENT->transaction_state != TS_NONE);
+ if (globally_unique_transaction) {
+ stm_fatalerror("cannot commit between stm_stop_all_other_threads "
+ "and stm_resume_all_other_threads");
+ }
+ if (STM_PSEGMENT->atomic_nesting_levels > 0) {
+ stm_fatalerror("cannot commit between stm_enable_atomic "
+ "and stm_disable_atomic");
+ }
+ assert(STM_SEGMENT->running_thread->self_or_0_if_atomic ==
+ (intptr_t)(STM_SEGMENT->running_thread));
+ assert(STM_SEGMENT->running_thread->wait_event_emitted == 0);
- dprintf(("> stm_commit_transaction()\n"));
- minor_collection(1);
+ dprintf(("> stm_commit_transaction(external=%d)\n", (int)external));
+ minor_collection(/*commit=*/ true, external);
+ if (!external && is_major_collection_requested()) {
+ s_mutex_lock();
+ if (is_major_collection_requested()) { /* if still true */
+ major_collection_with_mutex();
+ }
+ s_mutex_unlock();
+ }
push_large_overflow_objects_to_other_segments();
/* push before validate. otherwise they are reachable too early */
+ if (external) {
+ /* from this point on, unlink the original 'stm_thread_local_t *'
+ from its segment. Better do it as soon as possible, because
+ other threads might be spin-looping, waiting for the -1 to
+ disappear. */
+ STM_SEGMENT->running_thread = NULL;
+ write_fence();
+ assert(_stm_detached_inevitable_from_thread == -1);
+ _stm_detached_inevitable_from_thread = 0;
+ }
+
bool was_inev = STM_PSEGMENT->transaction_state == TS_INEVITABLE;
_validate_and_add_to_commit_log();
- stm_rewind_jmp_forget(STM_SEGMENT->running_thread);
+ if (!was_inev) {
+ assert(!external);
+ stm_rewind_jmp_forget(STM_SEGMENT->running_thread);
+ }
/* XXX do we still need a s_mutex_lock() section here? */
s_mutex_lock();
@@ -1310,25 +1368,15 @@
STM_PSEGMENT->overflow_number_has_been_used = false;
}
+ if (STM_PSEGMENT->active_queues)
+ queues_deactivate_all(get_priv_segment(STM_SEGMENT->segment_num),
+ /*at_commit=*/true);
+
invoke_and_clear_user_callbacks(0); /* for commit */
- /* >>>>> there may be a FORK() happening in the safepoint below <<<<<*/
- enter_safe_point_if_requested();
- assert(STM_SEGMENT->nursery_end == NURSERY_END);
-
- /* if a major collection is required, do it here */
- if (is_major_collection_requested()) {
- major_collection_with_mutex();
- }
-
- _verify_cards_cleared_in_all_lists(get_priv_segment(STM_SEGMENT->segment_num));
-
- if (globally_unique_transaction && was_inev) {
- committed_globally_unique_transaction();
- }
-
/* done */
stm_thread_local_t *tl = STM_SEGMENT->running_thread;
+ assert(external == (tl == NULL));
_finish_transaction(STM_TRANSACTION_COMMIT);
/* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */
@@ -1336,7 +1384,8 @@
/* between transactions, call finalizers. this will execute
a transaction itself */
- invoke_general_finalizers(tl);
+ if (tl != NULL)
+ invoke_general_finalizers(tl);
}
static void reset_modified_from_backup_copies(int segment_num)
@@ -1345,7 +1394,7 @@
#pragma push_macro("STM_SEGMENT")
#undef STM_PSEGMENT
#undef STM_SEGMENT
- assert(get_priv_segment(segment_num)->modification_lock);
+ assert(modification_lock_check_wrlock(segment_num));
struct stm_priv_segment_info_s *pseg = get_priv_segment(segment_num);
struct list_s *list = pseg->modified_old_objects;
@@ -1397,7 +1446,7 @@
abort_finalizers(pseg);
- long bytes_in_nursery = throw_away_nursery(pseg);
+ throw_away_nursery(pseg);
/* clear CARD_MARKED on objs (don't care about CARD_MARKED_OLD) */
LIST_FOREACH_R(pseg->old_objects_with_cards_set, object_t * /*item*/,
@@ -1407,9 +1456,9 @@
_reset_object_cards(pseg, item, CARD_CLEAR, false, false);
});
- acquire_modification_lock(segment_num);
+ acquire_modification_lock_wr(segment_num);
reset_modified_from_backup_copies(segment_num);
- release_modification_lock(segment_num);
+ release_modification_lock_wr(segment_num);
_verify_cards_cleared_in_all_lists(pseg);
stm_thread_local_t *tl = pseg->pub.running_thread;
@@ -1431,7 +1480,29 @@
assert(tl->shadowstack == pseg->shadowstack_at_start_of_transaction);
#endif
tl->thread_local_obj = pseg->threadlocal_at_start_of_transaction;
- tl->last_abort__bytes_in_nursery = bytes_in_nursery;
+
+ if (pseg->active_queues)
+ queues_deactivate_all(pseg, /*at_commit=*/false);
+
+
+ /* Set the next nursery_mark: first compute the value that
+ nursery_mark must have had at the start of the aborted transaction */
+ stm_char *old_mark =pseg->pub.nursery_mark + pseg->total_throw_away_nursery;
+
+ /* This means that the limit, in term of bytes, was: */
+ uintptr_t old_limit = old_mark - (stm_char *)_stm_nursery_start;
+
+ /* If 'total_throw_away_nursery' is smaller than old_limit, use that */
+ if (pseg->total_throw_away_nursery < old_limit)
+ old_limit = pseg->total_throw_away_nursery;
+
+ /* Now set the new limit to 90% of the old limit */
+ pseg->pub.nursery_mark = ((stm_char *)_stm_nursery_start +
+ (uintptr_t)(old_limit * 0.9));
+
+#ifdef STM_NO_AUTOMATIC_SETJMP
+ did_abort = 1;
+#endif
list_clear(pseg->objects_pointing_to_nursery);
list_clear(pseg->old_objects_with_cards_set);
@@ -1452,6 +1523,8 @@
abort_data_structures_from_segment_num(STM_SEGMENT->segment_num);
stm_thread_local_t *tl = STM_SEGMENT->running_thread;
+ tl->self_or_0_if_atomic = (intptr_t)tl; /* clear the 'atomic' flag */
+ STM_PSEGMENT->atomic_nesting_levels = 0;
if (tl->mem_clear_on_abort)
memset(tl->mem_clear_on_abort, 0, tl->mem_bytes_to_clear_on_abort);
@@ -1500,36 +1573,84 @@
void _stm_become_inevitable(const char *msg)
{
- if (STM_PSEGMENT->transaction_state == TS_REGULAR) {
+ int num_waits = 0;
+
+ timing_become_inevitable();
+
+ retry_from_start:
+ assert(STM_PSEGMENT->transaction_state == TS_REGULAR);
+ _stm_collectable_safe_point();
+
+ if (msg != MSG_INEV_DONT_SLEEP) {
dprintf(("become_inevitable: %s\n", msg));
- _stm_collectable_safe_point();
- timing_become_inevitable();
- _validate_and_turn_inevitable();
- STM_PSEGMENT->transaction_state = TS_INEVITABLE;
+ if (any_soon_finished_or_inevitable_thread_segment() &&
+ num_waits <= NB_SEGMENTS) {
+#if STM_TESTS /* for tests: another transaction */
+ stm_abort_transaction(); /* is already inevitable, abort */
+#endif
- stm_rewind_jmp_forget(STM_SEGMENT->running_thread);
- invoke_and_clear_user_callbacks(0); /* for commit */
+ bool timed_out = false;
+
+ s_mutex_lock();
+ if (any_soon_finished_or_inevitable_thread_segment() &&
+ !safe_point_requested()) {
+
+ /* wait until C_SEGMENT_FREE_OR_SAFE_POINT_REQ is signalled */
+ EMIT_WAIT(STM_WAIT_OTHER_INEVITABLE);
+ if (!cond_wait_timeout(C_SEGMENT_FREE_OR_SAFE_POINT_REQ,
+ 0.000054321))
+ timed_out = true;
+ }
+ s_mutex_unlock();
+
+ if (timed_out) {
+ /* try to detach another inevitable transaction, but
+ only after waiting a bit. This is necessary to avoid
+ deadlocks in some situations, which are hopefully
+ not too common. We don't want two threads constantly
+ detaching each other. */
+ intptr_t detached = fetch_detached_transaction();
+ if (detached != 0) {
+ EMIT_WAIT_DONE();
+ commit_fetched_detached_transaction(detached);
+ }
+ }
+ else {
+ num_waits++;
+ }
+ goto retry_from_start;
+ }
+ EMIT_WAIT_DONE();
+ if (!_validate_and_turn_inevitable())
+ goto retry_from_start;
}
else {
- assert(STM_PSEGMENT->transaction_state == TS_INEVITABLE);
+ if (!_validate_and_turn_inevitable())
+ return;
}
+ soon_finished_or_inevitable_thread_segment();
+ STM_PSEGMENT->transaction_state = TS_INEVITABLE;
+
+ stm_rewind_jmp_forget(STM_SEGMENT->running_thread);
+ invoke_and_clear_user_callbacks(0); /* for commit */
}
+#if 0
void stm_become_globally_unique_transaction(stm_thread_local_t *tl,
const char *msg)
{
- stm_become_inevitable(tl, msg); /* may still abort */
+ stm_become_inevitable(tl, msg);
s_mutex_lock();
synchronize_all_threads(STOP_OTHERS_AND_BECOME_GLOBALLY_UNIQUE);
s_mutex_unlock();
}
-
+#endif
void stm_stop_all_other_threads(void)
{
- if (!stm_is_inevitable()) /* may still abort */
+ if (!stm_is_inevitable(STM_SEGMENT->running_thread)) /* may still abort */
_stm_become_inevitable("stop_all_other_threads");
s_mutex_lock();
diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -74,11 +74,6 @@
struct stm_priv_segment_info_s {
struct stm_segment_info_s pub;
- /* lock protecting from concurrent modification of
- 'modified_old_objects', page-revision-changes, ...
- Always acquired in global order of segments to avoid deadlocks. */
- uint8_t modification_lock;
-
/* All the old objects (older than the current transaction) that
the current transaction attempts to modify. This is used to
track the STM status: these are old objects that where written
@@ -122,13 +117,15 @@
bool minor_collect_will_commit_now;
struct tree_s *callbacks_on_commit_and_abort[2];
+ struct tree_s *active_queues;
+ uint8_t active_queues_lock;
/* This is the number stored in the overflowed objects (a multiple of
GCFLAG_OVERFLOW_NUMBER_bit0). It is incremented when the
transaction is done, but only if we actually overflowed any
object; otherwise, no object has got this number. */
+ bool overflow_number_has_been_used;
uint32_t overflow_number;
- bool overflow_number_has_been_used;
struct stm_commit_log_entry_s *last_commit_log_entry;
@@ -157,6 +154,12 @@
stm_char *sq_fragments[SYNC_QUEUE_SIZE];
int sq_fragsizes[SYNC_QUEUE_SIZE];
int sq_len;
+
+ /* For nursery_mark */
+ uintptr_t total_throw_away_nursery;
+
+ /* For stm_enable_atomic() */
+ uintptr_t atomic_nesting_levels;
};
enum /* safe_point */ {
@@ -164,6 +167,7 @@
SP_RUNNING,
SP_WAIT_FOR_C_REQUEST_REMOVED,
SP_WAIT_FOR_C_AT_SAFE_POINT,
+ SP_WAIT_FOR_INEV,
#ifdef STM_TESTS
SP_WAIT_FOR_OTHER_THREAD,
#endif
@@ -175,6 +179,12 @@
TS_INEVITABLE,
};
+#define MSG_INEV_DONT_SLEEP ((const char *)1)
+
More information about the pypy-commit
mailing list