[pypy-commit] stmgc default: Merge with c8-private-pages
Raemi
noreply at buildbot.pypy.org
Wed Jan 21 15:47:40 CET 2015
Author: Remi Meier <remi.meier at inf.ethz.ch>
Branch:
Changeset: r1565:a043f17bba3c
Date: 2015-01-21 15:48 +0100
http://bitbucket.org/pypy/stmgc/changeset/a043f17bba3c/
Log: Merge with c8-private-pages
diff too long, truncating to 2000 out of 9435 lines
diff --git a/c7/stm/gcpage.c b/c7/stm/gcpage.c
--- a/c7/stm/gcpage.c
+++ b/c7/stm/gcpage.c
@@ -436,8 +436,8 @@
get_priv_segment(i)->modified_old_objects,
object_t * /*item*/,
({
- mark_visited_test_and_set(item);
- mark_trace(item, stm_object_pages); /* shared version */
+ if (!mark_visited_test_and_set(item))
+ mark_trace(item, stm_object_pages); /* shared version */
mark_trace(item, base); /* private version */
}));
}
diff --git a/c8/demo/demo_random2.c b/c8/demo/demo_random2.c
--- a/c8/demo/demo_random2.c
+++ b/c8/demo/demo_random2.c
@@ -10,10 +10,10 @@
#include "stmgc.h"
#define NUMTHREADS 3
-#define STEPS_PER_THREAD 500
-#define THREAD_STARTS 1000 // how many restarts of threads
+#define STEPS_PER_THREAD 50000
+#define THREAD_STARTS 100 // how many restarts of threads
#define PREBUILT_ROOTS 3
-#define FORKS 3
+#define FORKS 0
#define ACTIVE_ROOTS_SET_SIZE 100 // max num of roots created/alive in one transaction
#define MAX_ROOTS_ON_SS 1000 // max on shadow stack
@@ -232,11 +232,13 @@
break;
case 3: // allocate fresh 'p'
pushed = push_roots();
- size_t sizes[4] = {sizeof(struct node_s),
- sizeof(struct node_s) + (get_rand(100000) & ~15),
- sizeof(struct node_s) + 4096,
- sizeof(struct node_s) + 4096*70};
- size_t size = sizes[get_rand(4)];
+ size_t sizes[] = {
+ sizeof(struct node_s), sizeof(struct node_s)+16,
+ sizeof(struct node_s), sizeof(struct node_s)+16,
+ sizeof(struct node_s)+32, sizeof(struct node_s)+48,
+ sizeof(struct node_s)+32, sizeof(struct node_s)+48,
+ sizeof(struct node_s) + (get_rand(100000) & ~15)};
+ size_t size = sizes[get_rand(sizeof(sizes) / sizeof(size_t))];
p = stm_allocate(size);
((nodeptr_t)p)->sig = SIGNATURE;
((nodeptr_t)p)->my_size = size;
@@ -352,7 +354,7 @@
/* "interpreter main loop": this is one "application-frame" */
while (td.steps_left-->0 && get_rand(10) != 0) {
if (td.steps_left % 8 == 0)
- fprintf(stdout, "#");
+ fprintf(stderr, "#");
assert(p == NULL || ((nodeptr_t)p)->sig == SIGNATURE);
@@ -461,7 +463,7 @@
.next = NULL
};
- stm_start_inevitable_transaction(&stm_thread_local);
+ //stm_start_inevitable_transaction(&stm_thread_local);
for (i = 0; i < PREBUILT_ROOTS; i++) {
void* new_templ = malloc(sizeof(struct node_s));
memcpy(new_templ, &prebuilt_template, sizeof(struct node_s));
@@ -474,7 +476,7 @@
((nodeptr_t)prebuilt_roots[i])->my_hash = hash;
}
}
- stm_commit_transaction();
+ //stm_commit_transaction();
}
int main(void)
@@ -493,10 +495,11 @@
stm_setup();
+ setup_globals();
+
stm_register_thread_local(&stm_thread_local);
stm_rewind_jmp_enterframe(&stm_thread_local, &rjbuf);
- setup_globals();
int thread_starts = NUMTHREADS * THREAD_STARTS;
for (i = 0; i < NUMTHREADS; i++) {
diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -2,235 +2,520 @@
# error "must be compiled via stmgc.c"
#endif
-#include <signal.h>
+/* General helper: copies objects into our own segment, from some
+ source described by a range of 'struct stm_undo_s'. Maybe later
+ we could specialize this function to avoid the checks in the
+ inner loop.
+*/
+static void import_objects(
+ int from_segnum, /* or -1: from undo->backup,
+ or -2: from undo->backup if not modified */
+ uintptr_t pagenum, /* or -1: "all accessible" */
+ struct stm_undo_s *undo,
+ struct stm_undo_s *end)
+{
+ char *src_segment_base = (from_segnum >= 0 ? get_segment_base(from_segnum)
+ : NULL);
+ assert(IMPLY(from_segnum >= 0, get_priv_segment(from_segnum)->modification_lock));
+ assert(STM_PSEGMENT->modification_lock);
+
+ DEBUG_EXPECT_SEGFAULT(false);
+ for (; undo < end; undo++) {
+ object_t *obj = undo->object;
+ stm_char *oslice = ((stm_char *)obj) + SLICE_OFFSET(undo->slice);
+ uintptr_t current_page_num = ((uintptr_t)oslice) / 4096;
+
+ if (pagenum == -1) {
+ if (get_page_status_in(STM_SEGMENT->segment_num,
+ current_page_num) == PAGE_NO_ACCESS)
+ continue;
+ }
+ else {
+ if (current_page_num != pagenum)
+ continue;
+ }
+
+ if (from_segnum == -2 && _stm_was_read(obj) && (obj->stm_flags & GCFLAG_WB_EXECUTED)) {
+ /* called from stm_validate():
+ > if not was_read(), we certainly didn't modify
+ > if not WB_EXECUTED, we may have read from the obj in a different page but
+ did not modify it (should not occur right now, but future proof!)
+ only the WB_EXECUTED alone is not enough, since we may have imported from a
+ segment's private page (which had the flag set) */
+ assert(IMPLY(_stm_was_read(obj), (obj->stm_flags & GCFLAG_WB_EXECUTED))); /* for now */
+ continue; /* only copy unmodified */
+ }
+
+ /* XXX: if the next assert is always true, we should never get a segfault
+ in this function at all. So the DEBUG_EXPECT_SEGFAULT is correct. */
+ assert((get_page_status_in(STM_SEGMENT->segment_num,
+ current_page_num) != PAGE_NO_ACCESS));
+
+ dprintf(("import slice seg=%d obj=%p off=%lu sz=%d pg=%lu\n",
+ from_segnum, obj, SLICE_OFFSET(undo->slice),
+ SLICE_SIZE(undo->slice), current_page_num));
+ char *src, *dst;
+ if (src_segment_base != NULL)
+ src = REAL_ADDRESS(src_segment_base, oslice);
+ else
+ src = undo->backup;
+ dst = REAL_ADDRESS(STM_SEGMENT->segment_base, oslice);
+ memcpy(dst, src, SLICE_SIZE(undo->slice));
+
+ if (src_segment_base == NULL && SLICE_OFFSET(undo->slice) == 0) {
+ /* check that restored obj doesn't have WB_EXECUTED */
+ assert(!(obj->stm_flags & GCFLAG_WB_EXECUTED));
+ }
+ }
+ DEBUG_EXPECT_SEGFAULT(true);
+}
+
+
+/* ############# signal handler ############# */
+
+static void copy_bk_objs_in_page_from(int from_segnum, uintptr_t pagenum,
+ bool only_if_not_modified)
+{
+ /* looks at all bk copies of objects overlapping page 'pagenum' and
+ copies the part in 'pagenum' back to the current segment */
+ dprintf(("copy_bk_objs_in_page_from(%d, %ld, %d)\n",
+ from_segnum, (long)pagenum, only_if_not_modified));
+
+ struct list_s *list = get_priv_segment(from_segnum)->modified_old_objects;
+ struct stm_undo_s *undo = (struct stm_undo_s *)list->items;
+ struct stm_undo_s *end = (struct stm_undo_s *)(list->items + list->count);
+
+ import_objects(only_if_not_modified ? -2 : -1,
+ pagenum, undo, end);
+}
+
+static void go_to_the_past(uintptr_t pagenum,
+ struct stm_commit_log_entry_s *from,
+ struct stm_commit_log_entry_s *to)
+{
+ assert(STM_PSEGMENT->modification_lock);
+ assert(from->rev_num >= to->rev_num);
+ /* walk BACKWARDS the commit log and update the page 'pagenum',
+ initially at revision 'from', until we reach the revision 'to'. */
+
+ /* XXXXXXX Recursive algo for now, fix this! */
+ if (from != to) {
+ struct stm_commit_log_entry_s *cl = to->next;
+ go_to_the_past(pagenum, from, cl);
+
+ struct stm_undo_s *undo = cl->written;
+ struct stm_undo_s *end = cl->written + cl->written_count;
+
+ import_objects(-1, pagenum, undo, end);
+ }
+}
+
+
+
+static void handle_segfault_in_page(uintptr_t pagenum)
+{
+ /* assumes page 'pagenum' is ACCESS_NONE, privatizes it,
+ and validates to newest revision */
+
+ dprintf(("handle_segfault_in_page(%lu), seg %d\n", pagenum, STM_SEGMENT->segment_num));
+
+ /* XXX: bad, but no deadlocks: */
+ acquire_all_privatization_locks();
+
+ long i;
+ int my_segnum = STM_SEGMENT->segment_num;
+
+ assert(get_page_status_in(my_segnum, pagenum) == PAGE_NO_ACCESS);
+
+ /* find who has the most recent revision of our page */
+ int copy_from_segnum = -1;
+ uint64_t most_recent_rev = 0;
+ for (i = 1; i < NB_SEGMENTS; i++) {
+ if (i == my_segnum)
+ continue;
+
+ struct stm_commit_log_entry_s *log_entry;
+ log_entry = get_priv_segment(i)->last_commit_log_entry;
+ if (get_page_status_in(i, pagenum) != PAGE_NO_ACCESS
+ && (copy_from_segnum == -1 || log_entry->rev_num > most_recent_rev)) {
+ copy_from_segnum = i;
+ most_recent_rev = log_entry->rev_num;
+ }
+ }
+ OPT_ASSERT(copy_from_segnum != my_segnum);
+
+ /* make our page write-ready */
+ page_mark_accessible(my_segnum, pagenum);
+
+ if (copy_from_segnum == -1) {
+ /* this page is only accessible in the sharing segment so far (new
+ allocation). We can thus simply mark it accessible here. */
+ pagecopy(get_virtual_page(my_segnum, pagenum),
+ get_virtual_page(0, pagenum));
+ release_all_privatization_locks();
+ return;
+ }
+
+ /* before copying anything, acquire modification locks from our and
+ the other segment */
+ uint64_t to_lock = (1UL << copy_from_segnum)| (1UL << my_segnum);
+ acquire_modification_lock_set(to_lock);
+ pagecopy(get_virtual_page(my_segnum, pagenum),
+ get_virtual_page(copy_from_segnum, pagenum));
+
+ /* if there were modifications in the page, revert them. */
+ copy_bk_objs_in_page_from(copy_from_segnum, pagenum, false);
+
+ /* we need to go from 'src_version' to 'target_version'. This
+ might need a walk into the past. */
+ struct stm_commit_log_entry_s *src_version, *target_version;
+ src_version = get_priv_segment(copy_from_segnum)->last_commit_log_entry;
+ target_version = STM_PSEGMENT->last_commit_log_entry;
+
+
+ dprintf(("handle_segfault_in_page: rev %lu to rev %lu\n",
+ src_version->rev_num, target_version->rev_num));
+ /* adapt revision of page to our revision:
+ if our rev is higher than the page we copy from, everything
+ is fine as we never read/modified the page anyway
+ */
+ if (src_version->rev_num > target_version->rev_num)
+ go_to_the_past(pagenum, src_version, target_version);
+
+ release_modification_lock_set(to_lock);
+ release_all_privatization_locks();
+}
+
+static void _signal_handler(int sig, siginfo_t *siginfo, void *context)
+{
+ assert(_stm_segfault_expected > 0);
+
+ int saved_errno = errno;
+ char *addr = siginfo->si_addr;
+ dprintf(("si_addr: %p\n", addr));
+ if (addr == NULL || addr < stm_object_pages ||
+ addr >= stm_object_pages+TOTAL_MEMORY) {
+ /* actual segfault, unrelated to stmgc */
+ fprintf(stderr, "Segmentation fault: accessing %p\n", addr);
+ abort();
+ }
+
+ int segnum = get_segment_of_linear_address(addr);
+ if (segnum != STM_SEGMENT->segment_num) {
+ fprintf(stderr, "Segmentation fault: accessing %p (seg %d) from"
+ " seg %d\n", addr, segnum, STM_SEGMENT->segment_num);
+ abort();
+ }
+ dprintf(("-> segment: %d\n", segnum));
+
+ char *seg_base = STM_SEGMENT->segment_base;
+ uintptr_t pagenum = ((char*)addr - seg_base) / 4096UL;
+ if (pagenum < END_NURSERY_PAGE) {
+ fprintf(stderr, "Segmentation fault: accessing %p (seg %d "
+ "page %lu)\n", addr, segnum, pagenum);
+ abort();
+ }
+
+ DEBUG_EXPECT_SEGFAULT(false);
+ handle_segfault_in_page(pagenum);
+ DEBUG_EXPECT_SEGFAULT(true);
+
+ errno = saved_errno;
+ /* now return and retry */
+}
/* ############# commit log ############# */
void _dbg_print_commit_log()
{
- volatile struct stm_commit_log_entry_s *cl;
- cl = (volatile struct stm_commit_log_entry_s *)&commit_log_root;
+ struct stm_commit_log_entry_s *cl = &commit_log_root;
- fprintf(stderr, "root (%p, %d)\n", cl->next, cl->segment_num);
+ fprintf(stderr, "commit log:\n");
while ((cl = cl->next)) {
- if ((uintptr_t)cl == -1) {
- fprintf(stderr, "INEVITABLE\n");
+ if (cl == INEV_RUNNING) {
+ fprintf(stderr, " INEVITABLE\n");
return;
}
- size_t i = 0;
- fprintf(stderr, " elem (%p, %d)\n", cl->next, cl->segment_num);
- object_t *obj;
- while ((obj = cl->written[i])) {
- fprintf(stderr, "-> %p\n", obj);
- i++;
- };
+ fprintf(stderr, " entry at %p: seg %d, rev %lu\n", cl, cl->segment_num, cl->rev_num);
+ struct stm_undo_s *undo = cl->written;
+ struct stm_undo_s *end = undo + cl->written_count;
+ for (; undo < end; undo++) {
+ fprintf(stderr, " obj %p, size %d, ofs %lu: ", undo->object,
+ SLICE_SIZE(undo->slice), SLICE_OFFSET(undo->slice));
+ /* long i; */
+ /* for (i=0; i<SLICE_SIZE(undo->slice); i += 8) */
+ /* fprintf(stderr, " 0x%016lx", *(long *)(undo->backup + i)); */
+ fprintf(stderr, "\n");
+ }
}
}
-static void _update_obj_from(int from_seg, object_t *obj)
+static void reset_modified_from_backup_copies(int segment_num); /* forward */
+
+static bool _stm_validate()
{
- /* during validation this looks up the obj in the
- from_seg (backup or normal) and copies the version
- over the current segment's one */
- size_t obj_size;
- char *realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj);
- uintptr_t pagenum = (uintptr_t)obj / 4096UL;
+ /* returns true if we reached a valid state, or false if
+ we need to abort now */
+ dprintf(("_stm_validate()\n"));
+ /* go from last known entry in commit log to the
+ most current one and apply all changes done
+ by other transactions. Abort if we have read one of
+ the committed objs. */
+ struct stm_commit_log_entry_s *first_cl = STM_PSEGMENT->last_commit_log_entry;
+ struct stm_commit_log_entry_s *next_cl, *last_cl, *cl;
+ int my_segnum = STM_SEGMENT->segment_num;
+ /* Don't check this 'cl'. This entry is already checked */
- OPT_ASSERT(!is_shared_log_page(pagenum));
- assert(is_private_log_page_in(STM_SEGMENT->segment_num, pagenum));
- assert(is_private_log_page_in(from_seg, pagenum));
+ if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) {
+ //assert(first_cl->next == INEV_RUNNING);
+ /* the above assert may fail when running a major collection
+ while the commit of the inevitable transaction is in progress */
+ return true;
+ }
- /* look the obj up in the other segment's modified_old_objects to
- get its backup copy: */
- acquire_modified_objs_lock(from_seg);
+ bool needs_abort = false;
- wlog_t *item;
- struct tree_s *tree = get_priv_segment(from_seg)->modified_old_objects;
- TREE_FIND(tree, (uintptr_t)obj, item, goto not_found);
+ while(1) {
+ /* retry IF: */
+ /* if at the time of "HERE" (s.b.) there happen to be
+ more commits (and bk copies) then it could be that
+ copy_bk_objs_in_page_from (s.b.) reads a bk copy that
+ is itself more recent than last_cl. This is fixed
+ by re-validating. */
+ first_cl = STM_PSEGMENT->last_commit_log_entry;
+ if (first_cl->next == NULL)
+ break;
- obj_size = stmcb_size_rounded_up((struct object_s*)item->val);
- memcpy(realobj, (char*)item->val, obj_size);
- assert(obj->stm_flags & GCFLAG_WRITE_BARRIER);
- release_modified_objs_lock(from_seg);
- return;
+ if (first_cl->next == INEV_RUNNING) {
+ /* need to reach safe point if an INEV transaction
+ is waiting for us, otherwise deadlock */
+ break;
+ }
- not_found:
- /* copy from page directly (obj is unmodified) */
- obj_size = stmcb_size_rounded_up(
- (struct object_s*)REAL_ADDRESS(get_segment_base(from_seg), obj));
- memcpy(realobj,
- REAL_ADDRESS(get_segment_base(from_seg), obj),
- obj_size);
- obj->stm_flags |= GCFLAG_WRITE_BARRIER; /* may already be gone */
- release_modified_objs_lock(from_seg);
+ /* Find the set of segments we need to copy from and lock them: */
+ uint64_t segments_to_lock = 1UL << my_segnum;
+ cl = first_cl;
+ while ((next_cl = cl->next) != NULL) {
+ if (next_cl == INEV_RUNNING) {
+ /* only validate entries up to INEV */
+ break;
+ }
+ assert(next_cl->rev_num > cl->rev_num);
+ cl = next_cl;
+
+ if (cl->written_count) {
+ segments_to_lock |= (1UL << cl->segment_num);
+ }
+ }
+ last_cl = cl;
+
+ /* HERE */
+
+ acquire_privatization_lock(STM_SEGMENT->segment_num);
+ acquire_modification_lock_set(segments_to_lock);
+
+
+ /* import objects from first_cl to last_cl: */
+ if (first_cl != last_cl) {
+ uint64_t segment_really_copied_from = 0UL;
+
+ cl = first_cl;
+ while ((cl = cl->next) != NULL) {
+ if (!needs_abort) {
+ struct stm_undo_s *undo = cl->written;
+ struct stm_undo_s *end = cl->written + cl->written_count;
+ for (; undo < end; undo++) {
+ if (_stm_was_read(undo->object)) {
+ /* first reset all modified objects from the backup
+ copies as soon as the first conflict is detected;
+ then we will proceed below to update our segment from
+ the old (but unmodified) version to the newer version.
+ */
+ reset_modified_from_backup_copies(my_segnum);
+ needs_abort = true;
+
+ dprintf(("_stm_validate() failed for obj %p\n", undo->object));
+ break;
+ }
+ }
+ }
+
+ if (cl->written_count) {
+ struct stm_undo_s *undo = cl->written;
+ struct stm_undo_s *end = cl->written + cl->written_count;
+
+ segment_really_copied_from |= (1UL << cl->segment_num);
+
+ import_objects(cl->segment_num, -1, undo, end);
+
+ /* here we can actually have our own modified version, so
+ make sure to only copy things that are not modified in our
+ segment... (if we do not abort) */
+ copy_bk_objs_in_page_from
+ (cl->segment_num, -1, /* any page */
+ !needs_abort); /* if we abort, we still want to copy everything */
+ }
+
+ /* last fully validated entry */
+ STM_PSEGMENT->last_commit_log_entry = cl;
+ if (cl == last_cl)
+ break;
+ }
+ assert(cl == last_cl);
+
+ /* XXX: this optimization fails in test_basic.py, bug3 */
+ /* OPT_ASSERT(segment_really_copied_from < (1 << NB_SEGMENTS)); */
+ /* int segnum; */
+ /* for (segnum = 1; segnum < NB_SEGMENTS; segnum++) { */
+ /* if (segment_really_copied_from & (1UL << segnum)) { */
+ /* /\* here we can actually have our own modified version, so */
+ /* make sure to only copy things that are not modified in our */
+ /* segment... (if we do not abort) *\/ */
+ /* copy_bk_objs_in_page_from( */
+ /* segnum, -1, /\* any page *\/ */
+ /* !needs_abort); /\* if we abort, we still want to copy everything *\/ */
+ /* } */
+ /* } */
+ }
+
+ /* done with modifications */
+ release_modification_lock_set(segments_to_lock);
+ release_privatization_lock(STM_SEGMENT->segment_num);
+ }
+
+ return !needs_abort;
}
-void stm_validate(void *free_if_abort)
-{
- /* go from last known entry in commit log to the
- most current one and apply all changes done
- by other transactions. Abort if we read one of
- the committed objs. */
- if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) {
- assert((uintptr_t)STM_PSEGMENT->last_commit_log_entry->next == -1);
- return;
- }
-
- volatile struct stm_commit_log_entry_s *cl, *prev_cl;
- cl = prev_cl = (volatile struct stm_commit_log_entry_s *)
- STM_PSEGMENT->last_commit_log_entry;
-
- bool needs_abort = false;
- /* Don't check 'cl'. This entry is already checked */
- while ((cl = cl->next)) {
- if ((uintptr_t)cl == -1) {
- /* there is an inevitable transaction running */
-#if STM_TESTS
- free(free_if_abort);
- stm_abort_transaction();
-#endif
- cl = prev_cl;
- _stm_collectable_safe_point();
- continue;
- }
- prev_cl = cl;
-
- OPT_ASSERT(cl->segment_num >= 0 && cl->segment_num < NB_SEGMENTS);
-
- object_t *obj;
- size_t i = 0;
- while ((obj = cl->written[i])) {
- _update_obj_from(cl->segment_num, obj);
-
- if (_stm_was_read(obj)) {
- needs_abort = true;
-
- /* if we wrote this obj, we need to free its backup and
- remove it from modified_old_objects because
- we would otherwise overwrite the updated obj on abort */
- acquire_modified_objs_lock(STM_SEGMENT->segment_num);
- wlog_t *item;
- struct tree_s *tree = STM_PSEGMENT->modified_old_objects;
- TREE_FIND(tree, (uintptr_t)obj, item, goto not_found);
-
- free((void*)item->val);
- TREE_FIND_DELETE(tree, item);
-
- not_found:
- /* nothing todo */
- release_modified_objs_lock(STM_SEGMENT->segment_num);
- }
-
- i++;
- };
-
- /* last fully validated entry */
- STM_PSEGMENT->last_commit_log_entry = (struct stm_commit_log_entry_s *)cl;
- }
-
- if (needs_abort) {
- free(free_if_abort);
- stm_abort_transaction();
- }
-}
-
-static struct stm_commit_log_entry_s *_create_commit_log_entry()
+static struct stm_commit_log_entry_s *_create_commit_log_entry(void)
{
/* puts all modified_old_objects in a new commit log entry */
// we don't need the privatization lock, as we are only
// reading from modified_old_objs and nobody but us can change it
- struct tree_s *tree = STM_PSEGMENT->modified_old_objects;
- size_t count = tree_count(tree);
- size_t byte_len = sizeof(struct stm_commit_log_entry_s) + (count + 1) * sizeof(object_t*);
+ struct list_s *list = STM_PSEGMENT->modified_old_objects;
+ OPT_ASSERT((list_count(list) % 3) == 0);
+ size_t count = list_count(list) / 3;
+ size_t byte_len = sizeof(struct stm_commit_log_entry_s) +
+ count * sizeof(struct stm_undo_s);
struct stm_commit_log_entry_s *result = malloc(byte_len);
result->next = NULL;
result->segment_num = STM_SEGMENT->segment_num;
-
- int i = 0;
- wlog_t *item;
- TREE_LOOP_FORWARD(tree, item); {
- result->written[i] = (object_t*)item->addr;
- i++;
- } TREE_LOOP_END;
-
- OPT_ASSERT(count == i);
- result->written[count] = NULL;
-
+ result->rev_num = -1; /* invalid */
+ result->written_count = count;
+ memcpy(result->written, list->items, count * sizeof(struct stm_undo_s));
return result;
}
-static void _validate_and_add_to_commit_log()
+
+static void reset_wb_executed_flags(void);
+static void readd_wb_executed_flags(void);
+static void check_all_write_barrier_flags(char *segbase, struct list_s *list);
+
+static void _validate_and_attach(struct stm_commit_log_entry_s *new)
{
- struct stm_commit_log_entry_s *new;
- volatile struct stm_commit_log_entry_s **to;
+ struct stm_commit_log_entry_s *old;
+
+ OPT_ASSERT(new != NULL);
+ /* we are attaching a real CL entry: */
+ bool is_commit = new != INEV_RUNNING;
+
+ while (1) {
+ if (!_stm_validate()) {
+ if (new != INEV_RUNNING)
+ free(new);
+ stm_abort_transaction();
+ }
+
+#if STM_TESTS
+ if (STM_PSEGMENT->transaction_state != TS_INEVITABLE
+ && STM_PSEGMENT->last_commit_log_entry->next == INEV_RUNNING) {
+ /* abort for tests... */
+ stm_abort_transaction();
+ }
+#endif
+
+ if (is_commit) {
+ /* we must not remove the WB_EXECUTED flags before validation as
+ it is part of a condition in import_objects() called by
+ copy_bk_objs_in_page_from to not overwrite our modifications.
+ So we do it here: */
+ reset_wb_executed_flags();
+ check_all_write_barrier_flags(STM_SEGMENT->segment_base,
+ STM_PSEGMENT->modified_old_objects);
+ }
+
+ /* try to attach to commit log: */
+ old = STM_PSEGMENT->last_commit_log_entry;
+ if (old->next == NULL) {
+ if (new != INEV_RUNNING) /* INEVITABLE */
+ new->rev_num = old->rev_num + 1;
+
+ if (__sync_bool_compare_and_swap(&old->next, NULL, new))
+ break; /* success! */
+ } else if (old->next == INEV_RUNNING) {
+ /* we failed because there is an INEV transaction running */
+ usleep(10);
+ }
+
+ if (is_commit) {
+ /* XXX: unfortunately, if we failed to attach our CL entry,
+ we have to re-add the WB_EXECUTED flags before we try to
+ validate again because of said condition (s.a) */
+ readd_wb_executed_flags();
+ }
+
+ dprintf(("_validate_and_attach(%p) failed, enter safepoint\n", new));
+
+ /* check for requested safe point. otherwise an INEV transaction
+ may try to commit but cannot because of the busy-loop here. */
+ _stm_collectable_safe_point();
+ }
+}
+
+static void _validate_and_turn_inevitable(void)
+{
+ _validate_and_attach((struct stm_commit_log_entry_s *)INEV_RUNNING);
+}
+
+static void _validate_and_add_to_commit_log(void)
+{
+ struct stm_commit_log_entry_s *old, *new;
new = _create_commit_log_entry();
if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) {
- OPT_ASSERT((uintptr_t)STM_PSEGMENT->last_commit_log_entry->next == -1);
+ old = STM_PSEGMENT->last_commit_log_entry;
+ new->rev_num = old->rev_num + 1;
+ OPT_ASSERT(old->next == INEV_RUNNING);
- to = &(STM_PSEGMENT->last_commit_log_entry->next);
- bool yes = __sync_bool_compare_and_swap(to, (void*)-1, new);
+ /* WB_EXECUTED must be removed before we attach */
+ reset_wb_executed_flags();
+ check_all_write_barrier_flags(STM_SEGMENT->segment_base,
+ STM_PSEGMENT->modified_old_objects);
+
+ bool yes = __sync_bool_compare_and_swap(&old->next, INEV_RUNNING, new);
OPT_ASSERT(yes);
- return;
+ }
+ else {
+ _validate_and_attach(new);
}
- /* regular transaction: */
- do {
- stm_validate(new);
-
- /* try attaching to commit log: */
- to = &(STM_PSEGMENT->last_commit_log_entry->next);
- } while (!__sync_bool_compare_and_swap(to, NULL, new));
-}
-
-static void _validate_and_turn_inevitable()
-{
- struct stm_commit_log_entry_s *new;
- volatile struct stm_commit_log_entry_s **to;
-
- new = (struct stm_commit_log_entry_s*)-1;
- do {
- stm_validate(NULL);
-
- /* try attaching to commit log: */
- to = &(STM_PSEGMENT->last_commit_log_entry->next);
- } while (!__sync_bool_compare_and_swap(to, NULL, new));
+ acquire_modification_lock(STM_SEGMENT->segment_num);
+ list_clear(STM_PSEGMENT->modified_old_objects);
+ STM_PSEGMENT->last_commit_log_entry = new;
+ release_modification_lock(STM_SEGMENT->segment_num);
}
/* ############# STM ############# */
+void stm_validate()
+{
+ if (!_stm_validate())
+ stm_abort_transaction();
+}
-void _privatize_shared_page(uintptr_t pagenum)
-{
- /* privatize pages of obj for our segment iff previously
- the pages were fully shared. */
-#ifndef NDEBUG
- long l;
- for (l = 0; l < NB_SEGMENTS; l++) {
- assert(get_priv_segment(l)->privatization_lock);
- }
-#endif
-
- uintptr_t i;
- int my_segnum = STM_SEGMENT->segment_num;
-
- assert(is_shared_log_page(pagenum));
- char *src = (char*)(get_virt_page_of(0, pagenum) * 4096UL);
-
- for (i = 1; i < NB_SEGMENTS; i++) {
- assert(!is_private_log_page_in(i, pagenum));
-
- page_privatize_in(i, pagenum, src);
- }
- set_page_private_in(0, pagenum);
-
- OPT_ASSERT(is_private_log_page_in(my_segnum, pagenum));
- assert(!is_shared_log_page(pagenum));
-}
void _stm_write_slowpath(object_t *obj)
{
@@ -246,56 +531,100 @@
realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj);
obj_size = stmcb_size_rounded_up((struct object_s *)realobj);
/* get the last page containing data from the object */
- end_page = (((uintptr_t)obj) + obj_size - 1) / 4096UL;
+ if (LIKELY(is_small_uniform(obj))) {
+ end_page = first_page;
+ } else {
+ end_page = (((uintptr_t)obj) + obj_size - 1) / 4096UL;
+ }
/* add to read set: */
stm_read(obj);
- /* create backup copy: */
- struct object_s *bk_obj = malloc(obj_size);
- memcpy(bk_obj, realobj, obj_size);
+ if (obj->stm_flags & GCFLAG_WB_EXECUTED) {
+ /* already executed WB once in this transaction. do GC
+ part again: */
+ dprintf(("write_slowpath-fast(%p)\n", obj));
+ obj->stm_flags &= ~GCFLAG_WRITE_BARRIER;
+ LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, obj);
+ return;
+ }
- /* if there are shared pages, privatize them */
+ assert(!(obj->stm_flags & GCFLAG_WB_EXECUTED));
+ dprintf(("write_slowpath(%p): sz=%lu\n", obj, obj_size));
+
+ retry:
+ /* privatize pages: */
+ /* XXX don't always acquire all locks... */
+ acquire_all_privatization_locks();
uintptr_t page;
for (page = first_page; page <= end_page; page++) {
- if (is_shared_log_page(page)) {
- long i;
- for (i = 0; i < NB_SEGMENTS; i++) {
- acquire_privatization_lock(i);
- }
- if (is_shared_log_page(page))
- _privatize_shared_page(page);
- for (i = NB_SEGMENTS-1; i >= 0; i--) {
- release_privatization_lock(i);
- }
+ if (get_page_status_in(my_segnum, page) == PAGE_NO_ACCESS) {
+ /* XXX: slow? */
+ release_all_privatization_locks();
+
+ volatile char *dummy = REAL_ADDRESS(STM_SEGMENT->segment_base, page * 4096UL);
+ *dummy; /* force segfault */
+
+ goto retry;
}
}
- /* pages not shared anymore. but we still may have
- only a read protected page ourselves: */
+ /* all pages are private to us and we hold the privatization_locks so
+ we are allowed to modify them */
- acquire_privatization_lock(my_segnum);
- OPT_ASSERT(is_private_log_page_in(my_segnum, first_page));
+ /* phew, now add the obj to the write-set and register the
+ backup copy. */
+ /* XXX: we should not be here at all fiddling with page status
+ if 'obj' is merely an overflow object. FIX ME, likely by copying
+ the overflow number logic from c7. */
- /* remove the WRITE_BARRIER flag */
+ DEBUG_EXPECT_SEGFAULT(false);
+
+ acquire_modification_lock(STM_SEGMENT->segment_num);
+ uintptr_t slice_sz;
+ uintptr_t in_page_offset = (uintptr_t)obj % 4096UL;
+ uintptr_t remaining_obj_sz = obj_size;
+ for (page = first_page; page <= end_page; page++) {
+ /* XXX Maybe also use mprotect() again to mark pages of the object as read-only, and
+ only stick it into modified_old_objects page-by-page? Maybe it's
+ possible to do card-marking that way, too. */
+ OPT_ASSERT(remaining_obj_sz);
+
+ slice_sz = remaining_obj_sz;
+ if (in_page_offset + slice_sz > 4096UL) {
+ /* not over page boundaries */
+ slice_sz = 4096UL - in_page_offset;
+ }
+
+ size_t slice_off = obj_size - remaining_obj_sz;
+
+ /* make backup slice: */
+ char *bk_slice = malloc(slice_sz);
+ memcpy(bk_slice, realobj + slice_off, slice_sz);
+
+ STM_PSEGMENT->modified_old_objects = list_append3(
+ STM_PSEGMENT->modified_old_objects,
+ (uintptr_t)obj, /* obj */
+ (uintptr_t)bk_slice, /* bk_addr */
+ NEW_SLICE(slice_off, slice_sz));
+
+ remaining_obj_sz -= slice_sz;
+ in_page_offset = (in_page_offset + slice_sz) % 4096UL; /* mostly 0 */
+ }
+ OPT_ASSERT(remaining_obj_sz == 0);
+
+ /* remove the WRITE_BARRIER flag and add WB_EXECUTED */
obj->stm_flags &= ~GCFLAG_WRITE_BARRIER;
+ obj->stm_flags |= GCFLAG_WB_EXECUTED;
+
+ DEBUG_EXPECT_SEGFAULT(true);
+
+ release_modification_lock(STM_SEGMENT->segment_num);
+ /* done fiddling with protection and privatization */
+ release_all_privatization_locks();
/* also add it to the GC list for minor collections */
LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, obj);
-
- /* done fiddling with protection and privatization */
- release_privatization_lock(my_segnum);
-
- /* phew, now add the obj to the write-set and register the
- backup copy. */
- /* XXX: possibly slow check; try overflow objs again? */
- if (!tree_contains(STM_PSEGMENT->modified_old_objects, (uintptr_t)obj)) {
- acquire_modified_objs_lock(my_segnum);
- tree_insert(STM_PSEGMENT->modified_old_objects,
- (uintptr_t)obj, (uintptr_t)bk_obj);
- release_modified_objs_lock(my_segnum);
- }
-
}
static void reset_transaction_read_version(void)
@@ -308,7 +637,7 @@
(long)(NB_READMARKER_PAGES * 4096UL)));
if (mmap(readmarkers, NB_READMARKER_PAGES * 4096UL,
PROT_READ | PROT_WRITE,
- MAP_FIXED | MAP_PAGES_FLAGS, -1, 0) != readmarkers) {
+ MAP_FIXED | MAP_SHARED | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0) != readmarkers) {
/* fall-back */
#if STM_TESTS
stm_fatalerror("reset_transaction_read_version: %m");
@@ -318,15 +647,40 @@
STM_SEGMENT->transaction_read_version = 1;
}
+static void reset_wb_executed_flags(void)
+{
+ dprintf(("reset_wb_executed_flags()\n"));
+ struct list_s *list = STM_PSEGMENT->modified_old_objects;
+ struct stm_undo_s *undo = (struct stm_undo_s *)list->items;
+ struct stm_undo_s *end = (struct stm_undo_s *)(list->items + list->count);
+
+ for (; undo < end; undo++) {
+ object_t *obj = undo->object;
+ obj->stm_flags &= ~GCFLAG_WB_EXECUTED;
+ }
+}
+
+static void readd_wb_executed_flags(void)
+{
+ dprintf(("readd_wb_executed_flags()\n"));
+ struct list_s *list = STM_PSEGMENT->modified_old_objects;
+ struct stm_undo_s *undo = (struct stm_undo_s *)list->items;
+ struct stm_undo_s *end = (struct stm_undo_s *)(list->items + list->count);
+
+ for (; undo < end; undo++) {
+ object_t *obj = undo->object;
+ obj->stm_flags |= GCFLAG_WB_EXECUTED;
+ }
+}
+
+
+
static void _stm_start_transaction(stm_thread_local_t *tl)
{
assert(!_stm_in_transaction(tl));
- retry:
-
- if (!acquire_thread_segment(tl))
- goto retry;
+ while (!acquire_thread_segment(tl)) {}
/* GS invalid before this point! */
assert(STM_PSEGMENT->safe_point == SP_NO_TRANSACTION);
@@ -337,11 +691,12 @@
STM_PSEGMENT->running_pthread = pthread_self();
#endif
STM_PSEGMENT->shadowstack_at_start_of_transaction = tl->shadowstack;
+ STM_PSEGMENT->threadlocal_at_start_of_transaction = tl->thread_local_obj;
enter_safe_point_if_requested();
- dprintf(("start_transaction\n"));
+ dprintf(("> start_transaction\n"));
- s_mutex_unlock();
+ s_mutex_unlock(); // XXX it's probably possible to not acquire this here
uint8_t old_rv = STM_SEGMENT->transaction_read_version;
STM_SEGMENT->transaction_read_version = old_rv + 1;
@@ -349,7 +704,8 @@
reset_transaction_read_version();
}
- assert(tree_is_cleared(STM_PSEGMENT->modified_old_objects));
+ assert(list_is_empty(STM_PSEGMENT->modified_old_objects));
+ assert(list_is_empty(STM_PSEGMENT->new_objects));
assert(list_is_empty(STM_PSEGMENT->objects_pointing_to_nursery));
assert(tree_is_cleared(STM_PSEGMENT->young_outside_nursery));
assert(tree_is_cleared(STM_PSEGMENT->nursery_objects_shadows));
@@ -358,7 +714,7 @@
check_nursery_at_transaction_start();
- stm_validate(NULL);
+ stm_validate();
}
long stm_start_transaction(stm_thread_local_t *tl)
@@ -401,40 +757,56 @@
STM_PSEGMENT->safe_point = SP_NO_TRANSACTION;
STM_PSEGMENT->transaction_state = TS_NONE;
list_clear(STM_PSEGMENT->objects_pointing_to_nursery);
+ list_clear(STM_PSEGMENT->new_objects);
release_thread_segment(tl);
/* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */
}
+static void check_all_write_barrier_flags(char *segbase, struct list_s *list)
+{
+#ifndef NDEBUG
+ struct stm_undo_s *undo = (struct stm_undo_s *)list->items;
+ struct stm_undo_s *end = (struct stm_undo_s *)(list->items + list->count);
+ for (; undo < end; undo++) {
+ object_t *obj = undo->object;
+ struct object_s *dst = (struct object_s*)REAL_ADDRESS(segbase, obj);
+ assert(dst->stm_flags & GCFLAG_WRITE_BARRIER);
+ assert(!(dst->stm_flags & GCFLAG_WB_EXECUTED));
+ }
+#endif
+}
+
+static void push_new_objects_to_other_segments(void)
+{
+ acquire_privatization_lock(STM_SEGMENT->segment_num);
+ LIST_FOREACH_R(STM_PSEGMENT->new_objects, object_t *,
+ ({
+ assert(item->stm_flags & GCFLAG_WB_EXECUTED);
+ item->stm_flags &= ~GCFLAG_WB_EXECUTED;
+ synchronize_object_enqueue(item);
+ }));
+ synchronize_objects_flush();
+ release_privatization_lock(STM_SEGMENT->segment_num);
+}
+
+
void stm_commit_transaction(void)
{
assert(!_has_mutex());
assert(STM_PSEGMENT->safe_point == SP_RUNNING);
assert(STM_PSEGMENT->running_pthread == pthread_self());
- dprintf(("stm_commit_transaction()\n"));
+ dprintf(("> stm_commit_transaction()\n"));
minor_collection(1);
+ push_new_objects_to_other_segments();
+
_validate_and_add_to_commit_log();
- /* clear WRITE_BARRIER flags, free all backup copies,
- and clear the tree: */
- acquire_modified_objs_lock(STM_SEGMENT->segment_num);
-
- struct tree_s *tree = STM_PSEGMENT->modified_old_objects;
- wlog_t *item;
- TREE_LOOP_FORWARD(tree, item); {
- object_t *obj = (object_t*)item->addr;
- struct object_s* bk_obj = (struct object_s *)item->val;
- free(bk_obj);
- obj->stm_flags |= GCFLAG_WRITE_BARRIER;
- } TREE_LOOP_END;
- tree_clear(tree);
-
- release_modified_objs_lock(STM_SEGMENT->segment_num);
-
invoke_and_clear_user_callbacks(0); /* for commit */
+ /* XXX do we still need a s_mutex_lock() section here? */
s_mutex_lock();
enter_safe_point_if_requested();
assert(STM_SEGMENT->nursery_end == NURSERY_END);
@@ -452,35 +824,37 @@
s_mutex_unlock();
}
-void reset_modified_from_backup_copies(int segment_num)
+static void reset_modified_from_backup_copies(int segment_num)
{
#pragma push_macro("STM_PSEGMENT")
#pragma push_macro("STM_SEGMENT")
#undef STM_PSEGMENT
#undef STM_SEGMENT
- acquire_modified_objs_lock(segment_num);
+ assert(get_priv_segment(segment_num)->modification_lock);
struct stm_priv_segment_info_s *pseg = get_priv_segment(segment_num);
- struct tree_s *tree = pseg->modified_old_objects;
- wlog_t *item;
- TREE_LOOP_FORWARD(tree, item); {
- object_t *obj = (object_t*)item->addr;
- struct object_s* bk_obj = (struct object_s *)item->val;
- size_t obj_size;
+ struct list_s *list = pseg->modified_old_objects;
+ struct stm_undo_s *undo = (struct stm_undo_s *)list->items;
+ struct stm_undo_s *end = (struct stm_undo_s *)(list->items + list->count);
- obj_size = stmcb_size_rounded_up(bk_obj);
+ for (; undo < end; undo++) {
+ object_t *obj = undo->object;
+ char *dst = REAL_ADDRESS(pseg->pub.segment_base, obj);
- memcpy(REAL_ADDRESS(pseg->pub.segment_base, obj),
- bk_obj, obj_size);
- assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); /* not written */
+ memcpy(dst + SLICE_OFFSET(undo->slice),
+ undo->backup,
+ SLICE_SIZE(undo->slice));
- free(bk_obj);
- } TREE_LOOP_END;
+ dprintf(("reset_modified_from_backup_copies(%d): obj=%p off=%lu bk=%p\n",
+ segment_num, obj, SLICE_OFFSET(undo->slice), undo->backup));
- tree_clear(tree);
+ free(undo->backup);
+ }
- release_modified_objs_lock(segment_num);
+ /* check that all objects have the GCFLAG_WRITE_BARRIER afterwards */
+ check_all_write_barrier_flags(pseg->pub.segment_base, list);
+ list_clear(list);
#pragma pop_macro("STM_SEGMENT")
#pragma pop_macro("STM_PSEGMENT")
}
@@ -505,7 +879,9 @@
long bytes_in_nursery = throw_away_nursery(pseg);
+ acquire_modification_lock(segment_num);
reset_modified_from_backup_copies(segment_num);
+ release_modification_lock(segment_num);
stm_thread_local_t *tl = pseg->pub.running_thread;
#ifdef STM_NO_AUTOMATIC_SETJMP
@@ -525,8 +901,11 @@
stm_rewind_jmp_restore_shadowstack(tl);
assert(tl->shadowstack == pseg->shadowstack_at_start_of_transaction);
#endif
-tl->last_abort__bytes_in_nursery = bytes_in_nursery;
+ tl->thread_local_obj = pseg->threadlocal_at_start_of_transaction;
+ tl->last_abort__bytes_in_nursery = bytes_in_nursery;
+ list_clear(pseg->objects_pointing_to_nursery);
+ list_clear(pseg->new_objects);
#pragma pop_macro("STM_SEGMENT")
#pragma pop_macro("STM_PSEGMENT")
}
@@ -613,3 +992,95 @@
synchronize_all_threads(STOP_OTHERS_AND_BECOME_GLOBALLY_UNIQUE);
s_mutex_unlock();
}
+
+
+
+static inline void _synchronize_fragment(stm_char *frag, ssize_t frag_size)
+{
+ /* double-check that the result fits in one page */
+ assert(frag_size > 0);
+ assert(frag_size + ((uintptr_t)frag & 4095) <= 4096);
+
+ /* XXX: is it possible to just add to the queue iff the pages
+ of the fragment need syncing to other segments? (keep privatization
+ lock until the "flush") */
+
+ /* Enqueue this object (or fragemnt of object) */
+ if (STM_PSEGMENT->sq_len == SYNC_QUEUE_SIZE)
+ synchronize_objects_flush();
+ STM_PSEGMENT->sq_fragments[STM_PSEGMENT->sq_len] = frag;
+ STM_PSEGMENT->sq_fragsizes[STM_PSEGMENT->sq_len] = frag_size;
+ ++STM_PSEGMENT->sq_len;
+}
+
+static void synchronize_object_enqueue(object_t *obj)
+{
+ assert(!_is_young(obj));
+ assert(STM_PSEGMENT->privatization_lock);
+ assert(obj->stm_flags & GCFLAG_WRITE_BARRIER);
+ assert(!(obj->stm_flags & GCFLAG_WB_EXECUTED));
+
+ ssize_t obj_size = stmcb_size_rounded_up(
+ (struct object_s *)REAL_ADDRESS(STM_SEGMENT->segment_base, obj));
+ OPT_ASSERT(obj_size >= 16);
+
+ if (LIKELY(is_small_uniform(obj))) {
+ OPT_ASSERT(obj_size <= GC_LAST_SMALL_SIZE);
+ _synchronize_fragment((stm_char *)obj, obj_size);
+ return;
+ }
+
+ /* else, a more complicated case for large objects, to copy
+ around data only within the needed pages
+ */
+ uintptr_t start = (uintptr_t)obj;
+ uintptr_t end = start + obj_size;
+
+ do {
+ uintptr_t copy_up_to = (start + 4096) & ~4095; /* end of page */
+ if (copy_up_to >= end) {
+ copy_up_to = end; /* this is the last fragment */
+ }
+ uintptr_t copy_size = copy_up_to - start;
+
+ _synchronize_fragment((stm_char *)start, copy_size);
+
+ start = copy_up_to;
+ } while (start != end);
+}
+
+static void synchronize_objects_flush(void)
+{
+ long j = STM_PSEGMENT->sq_len;
+ if (j == 0)
+ return;
+ STM_PSEGMENT->sq_len = 0;
+
+ dprintf(("synchronize_objects_flush(): %ld fragments\n", j));
+
+ assert(STM_PSEGMENT->privatization_lock);
+ DEBUG_EXPECT_SEGFAULT(false);
+
+ long i, myself = STM_SEGMENT->segment_num;
+ do {
+ --j;
+ stm_char *frag = STM_PSEGMENT->sq_fragments[j];
+ uintptr_t page = ((uintptr_t)frag) / 4096UL;
+ ssize_t frag_size = STM_PSEGMENT->sq_fragsizes[j];
+
+ char *src = REAL_ADDRESS(STM_SEGMENT->segment_base, frag);
+ for (i = 0; i < NB_SEGMENTS; i++) {
+ if (i == myself)
+ continue;
+
+ if (get_page_status_in(i, page) != PAGE_NO_ACCESS) {
+ /* shared or private, but never segfault */
+ char *dst = REAL_ADDRESS(get_segment_base(i), frag);
+ dprintf(("-> flush %p to seg %lu, sz=%lu\n", frag, i, frag_size));
+ memcpy(dst, src, frag_size);
+ }
+ }
+ } while (j > 0);
+
+ DEBUG_EXPECT_SEGFAULT(true);
+}
diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -6,6 +6,8 @@
#include <sys/mman.h>
#include <errno.h>
#include <pthread.h>
+#include <signal.h>
+
/************************************************************/
@@ -15,9 +17,8 @@
#define NB_PAGES (2500*256) // 2500MB
-#define NB_SEGMENTS STM_NB_SEGMENTS
+#define NB_SEGMENTS (STM_NB_SEGMENTS+1) /* +1 for sharing seg 0 */
#define NB_SEGMENTS_MAX 240 /* don't increase NB_SEGMENTS past this */
-#define MAP_PAGES_FLAGS (MAP_SHARED | MAP_ANONYMOUS | MAP_NORESERVE)
#define NB_NURSERY_PAGES (STM_GC_NURSERY/4)
#define TOTAL_MEMORY (NB_PAGES * 4096UL * NB_SEGMENTS)
@@ -25,6 +26,7 @@
#define FIRST_OBJECT_PAGE ((READMARKER_END + 4095) / 4096UL)
#define FIRST_NURSERY_PAGE FIRST_OBJECT_PAGE
#define END_NURSERY_PAGE (FIRST_NURSERY_PAGE + NB_NURSERY_PAGES)
+#define NB_SHARED_PAGES (NB_PAGES - END_NURSERY_PAGE)
#define READMARKER_START ((FIRST_OBJECT_PAGE * 4096UL) >> 4)
#define FIRST_READMARKER_PAGE (READMARKER_START / 4096UL)
@@ -35,9 +37,15 @@
enum /* stm_flags */ {
GCFLAG_WRITE_BARRIER = _STM_GCFLAG_WRITE_BARRIER,
GCFLAG_HAS_SHADOW = 0x02,
+ GCFLAG_WB_EXECUTED = 0x04,
+ GCFLAG_VISITED = 0x08,
};
+
+#define SYNC_QUEUE_SIZE 31
+
+
/************************************************************/
@@ -48,27 +56,58 @@
struct stm_priv_segment_info_s {
struct stm_segment_info_s pub;
- uint8_t modified_objs_lock;
- struct tree_s *modified_old_objects;
+ /* lock protecting from concurrent modification of
+ 'modified_old_objects', page-revision-changes, ...
+ Always acquired in global order of segments to avoid deadlocks. */
+ uint8_t modification_lock;
+
+ /* All the old objects (older than the current transaction) that
+ the current transaction attempts to modify. This is used to
+ track the STM status: these are old objects that where written
+ to and that will need to be recorded in the commit log. The
+ list contains three entries for every such object, in the same
+ format as 'struct stm_undo_s' below.
+ */
+ struct list_s *modified_old_objects;
+
struct list_s *objects_pointing_to_nursery;
struct tree_s *young_outside_nursery;
struct tree_s *nursery_objects_shadows;
- uint8_t privatization_lock;
+ /* list of objects created in the current transaction and
+ that survived at least one minor collection. They need
+ to be synchronized to other segments on commit, but they
+ do not need to be in the commit log entry. */
+ struct list_s *new_objects;
+
+ uint8_t privatization_lock; // XXX KILL
uint8_t safe_point;
uint8_t transaction_state;
+ /* Temp for minor collection */
+ bool minor_collect_will_commit_now;
+
struct tree_s *callbacks_on_commit_and_abort[2];
struct stm_commit_log_entry_s *last_commit_log_entry;
struct stm_shadowentry_s *shadowstack_at_start_of_transaction;
+ object_t *threadlocal_at_start_of_transaction;
/* For debugging */
#ifndef NDEBUG
pthread_t running_pthread;
#endif
+
+ /* This is for smallmalloc.c */
+ struct small_malloc_data_s small_malloc_data;
+
+ /* The sync queue used to synchronize newly allocated objs to
+ other segments */
+ stm_char *sq_fragments[SYNC_QUEUE_SIZE];
+ int sq_fragsizes[SYNC_QUEUE_SIZE];
+ int sq_len;
};
enum /* safe_point */ {
@@ -88,16 +127,41 @@
};
/* Commit Log things */
+struct stm_undo_s {
+ object_t *object; /* the object that is modified */
+ char *backup; /* some backup data (a slice of the original obj) */
+ uint64_t slice; /* location and size of this slice (cannot cross
+ pages). The size is in the lower 2 bytes, and
+ the offset in the remaining 6 bytes. */
+};
+#define SLICE_OFFSET(slice) ((slice) >> 16)
+#define SLICE_SIZE(slice) ((int)((slice) & 0xFFFF))
+#define NEW_SLICE(offset, size) (((uint64_t)(offset)) << 16 | (size))
+
+/* The model is: we have a global chained list, from 'commit_log_root',
+ of 'struct stm_commit_log_entry_s' entries. Every one is fully
+ read-only apart from the 'next' field. Every one stands for one
+ commit that occurred. It lists the old objects that were modified
+ in this commit, and their attached "undo logs" --- that is, the
+ data from 'written[n].backup' is the content of (slices of) the
+ object as they were *before* that commit occurred.
+*/
+#define INEV_RUNNING ((void*)-1)
struct stm_commit_log_entry_s {
- volatile struct stm_commit_log_entry_s *next;
+ struct stm_commit_log_entry_s *volatile next;
int segment_num;
- object_t *written[]; /* terminated with a NULL ptr */
+ uint64_t rev_num;
+ size_t written_count;
+ struct stm_undo_s written[];
};
-static struct stm_commit_log_entry_s commit_log_root = {NULL, -1};
+static struct stm_commit_log_entry_s commit_log_root = {NULL, -1, 0, 0};
+#ifndef STM_TESTS
static char *stm_object_pages;
-static int stm_object_pages_fd;
+#else
+char *stm_object_pages;
+#endif
static stm_thread_local_t *stm_all_thread_locals = NULL;
@@ -133,7 +197,11 @@
static stm_thread_local_t *abort_with_mutex_no_longjmp(void);
static void abort_data_structures_from_segment_num(int segment_num);
+static void synchronize_object_enqueue(object_t *obj);
+static void synchronize_objects_flush(void);
+static void _signal_handler(int sig, siginfo_t *siginfo, void *context);
+static bool _stm_validate();
static inline void _duck(void) {
/* put a call to _duck() between two instructions that set 0 into
@@ -154,12 +222,83 @@
spinlock_release(get_priv_segment(segnum)->privatization_lock);
}
-static inline void acquire_modified_objs_lock(int segnum)
+static inline bool all_privatization_locks_acquired()
{
- spinlock_acquire(get_priv_segment(segnum)->modified_objs_lock);
+#ifndef NDEBUG
+ long l;
+ for (l = 0; l < NB_SEGMENTS; l++) {
+ if (!get_priv_segment(l)->privatization_lock)
+ return false;
+ }
+ return true;
+#else
+ abort();
+#endif
}
-static inline void release_modified_objs_lock(int segnum)
+static inline void acquire_all_privatization_locks()
{
- spinlock_release(get_priv_segment(segnum)->modified_objs_lock);
+ /* XXX: don't do for the sharing seg0 */
+ long l;
+ for (l = 0; l < NB_SEGMENTS; l++) {
+ acquire_privatization_lock(l);
+ }
}
+
+static inline void release_all_privatization_locks()
+{
+ long l;
+ for (l = NB_SEGMENTS-1; l >= 0; l--) {
+ release_privatization_lock(l);
+ }
+}
+
+
+
+/* Modification locks are used to prevent copying from a segment
+ where either the revision of some pages is inconsistent with the
+ rest, or the modified_old_objects list is being modified (bk_copys).
+
+ Lock ordering: acquire privatization lock around acquiring a set
+ of modification locks!
+*/
+
+static inline void acquire_modification_lock(int segnum)
+{
+ spinlock_acquire(get_priv_segment(segnum)->modification_lock);
+}
+
+static inline void release_modification_lock(int segnum)
+{
+ spinlock_release(get_priv_segment(segnum)->modification_lock);
+}
+
+static inline void acquire_modification_lock_set(uint64_t seg_set)
+{
+ assert(NB_SEGMENTS <= 64);
+ OPT_ASSERT(seg_set < (1 << NB_SEGMENTS));
+
+ /* acquire locks in global order */
+ int i;
+ for (i = 0; i < NB_SEGMENTS; i++) {
+ if ((seg_set & (1 << i)) == 0)
+ continue;
+
+ spinlock_acquire(get_priv_segment(i)->modification_lock);
+ }
+}
+
+static inline void release_modification_lock_set(uint64_t seg_set)
+{
+ assert(NB_SEGMENTS <= 64);
+ OPT_ASSERT(seg_set < (1 << NB_SEGMENTS));
+
+ int i;
+ for (i = 0; i < NB_SEGMENTS; i++) {
+ if ((seg_set & (1 << i)) == 0)
+ continue;
+
+ assert(get_priv_segment(i)->modification_lock);
+ spinlock_release(get_priv_segment(i)->modification_lock);
+ }
+}
diff --git a/c8/stm/fprintcolor.c b/c8/stm/fprintcolor.c
--- a/c8/stm/fprintcolor.c
+++ b/c8/stm/fprintcolor.c
@@ -8,7 +8,8 @@
char buffer[2048];
va_list ap;
int result;
- int size = (int)sprintf(buffer, "\033[%dm[%d,%lx] ", dprintfcolor(),
+ int size = (int)sprintf(buffer, "\033[%dm[%d,%d,%lx] ",
+ dprintfcolor(), STM_SEGMENT->segment_num,
(int)getpid(), (long)pthread_self());
assert(size >= 0);
diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c
--- a/c8/stm/gcpage.c
+++ b/c8/stm/gcpage.c
@@ -2,42 +2,79 @@
# error "must be compiled via stmgc.c"
#endif
+static struct list_s *testing_prebuilt_objs = NULL;
+static struct tree_s *tree_prebuilt_objs = NULL; /* XXX refactor */
+
static void setup_gcpage(void)
{
+ char *base = stm_object_pages + END_NURSERY_PAGE * 4096UL;
+ uintptr_t length = NB_SHARED_PAGES * 4096UL;
+ _stm_largemalloc_init_arena(base, length);
+
uninitialized_page_start = stm_object_pages + END_NURSERY_PAGE * 4096UL;
- uninitialized_page_stop = stm_object_pages + NB_PAGES * 4096UL;
+ uninitialized_page_stop = uninitialized_page_start + NB_SHARED_PAGES * 4096UL;
}
static void teardown_gcpage(void)
{
-}
-
-static void setup_N_pages(char *pages_addr, uint64_t num)
-{
- long i;
- for (i = 0; i < NB_SEGMENTS; i++) {
- acquire_privatization_lock(i);
- }
- pages_initialize_shared((pages_addr - stm_object_pages) / 4096UL, num);
- for (i = NB_SEGMENTS-1; i >= 0; i--) {
- release_privatization_lock(i);
+ LIST_FREE(testing_prebuilt_objs);
+ if (tree_prebuilt_objs != NULL) {
+ tree_free(tree_prebuilt_objs);
+ tree_prebuilt_objs = NULL;
}
}
+
+static void setup_N_pages(char *pages_addr, long num)
+{
+ /* make pages accessible in sharing segment only (pages already
+ PROT_READ/WRITE (see setup.c), but not marked accessible as page
+ status). */
+
+ /* lock acquiring maybe not necessary because the affected pages don't
+ need privatization protection. (but there is an assert right
+ now to enforce that XXXXXX) */
+ acquire_all_privatization_locks();
+
+ uintptr_t p = (pages_addr - stm_object_pages) / 4096UL;
+ dprintf(("setup_N_pages(%p, %lu): pagenum %lu\n", pages_addr, num, p));
+ while (num-->0) {
+ /* XXX: page_range_mark_accessible() */
+ page_mark_accessible(0, p + num);
+ }
+
+ release_all_privatization_locks();
+}
+
+
static int lock_growth_large = 0;
-static char *allocate_outside_nursery_large(uint64_t size)
+static stm_char *allocate_outside_nursery_large(uint64_t size)
{
- /* XXX: real allocation */
+ /* Allocate the object with largemalloc.c from the lower addresses. */
+ char *addr = _stm_large_malloc(size);
+ if (addr == NULL)
+ stm_fatalerror("not enough memory!");
+
+ if (LIKELY(addr + size <= uninitialized_page_start)) {
+ dprintf(("allocate_outside_nursery_large(%lu): %p, page=%lu\n",
+ size, (char*)(addr - stm_object_pages),
+ (uintptr_t)(addr - stm_object_pages) / 4096UL));
+
+ return (stm_char*)(addr - stm_object_pages);
+ }
+
+
+ /* uncommon case: need to initialize some more pages */
spinlock_acquire(lock_growth_large);
- char *addr = uninitialized_page_start;
char *start = uninitialized_page_start;
- if (addr + size > start) { /* XXX: always for now */
+ if (addr + size > start) {
uintptr_t npages;
- npages = (addr + size - start) / 4096UL + 1;
+ npages = (addr + size - start) / 4096UL;
+ npages += GCPAGE_NUM_PAGES;
if (uninitialized_page_stop - start < npages * 4096UL) {
stm_fatalerror("out of memory!"); /* XXX */
}
@@ -48,26 +85,500 @@
stm_fatalerror("uninitialized_page_start changed?");
}
}
- dprintf(("allocate_outside_nursery_large(%lu): %p, seg=%d, page=%lu\n",
- size, addr, get_segment_of_linear_address(addr),
- (addr - get_segment_base(get_segment_of_linear_address(addr))) / 4096UL));
+
+ dprintf(("allocate_outside_nursery_large(%lu): %p, page=%lu\n",
+ size, (char*)(addr - stm_object_pages),
+ (uintptr_t)(addr - stm_object_pages) / 4096UL));
spinlock_release(lock_growth_large);
- return addr;
+ return (stm_char*)(addr - stm_object_pages);
}
object_t *_stm_allocate_old(ssize_t size_rounded_up)
{
/* only for tests xxx but stm_setup_prebuilt() uses this now too */
- char *p = allocate_outside_nursery_large(size_rounded_up);
- memset(p, 0, size_rounded_up);
+ stm_char *p = allocate_outside_nursery_large(size_rounded_up);
+ object_t *o = (object_t *)p;
- object_t *o = (object_t *)(p - stm_object_pages);
+ // sharing seg0 needs to be current:
+ assert(STM_SEGMENT->segment_num == 0);
+ memset(REAL_ADDRESS(STM_SEGMENT->segment_base, o), 0, size_rounded_up);
o->stm_flags = GCFLAG_WRITE_BARRIER;
+ if (testing_prebuilt_objs == NULL)
+ testing_prebuilt_objs = list_create();
+ LIST_APPEND(testing_prebuilt_objs, o);
+
dprintf(("allocate_old(%lu): %p, seg=%d, page=%lu\n",
size_rounded_up, p,
- get_segment_of_linear_address(p),
- (p - STM_SEGMENT->segment_base) / 4096UL));
+ get_segment_of_linear_address(stm_object_pages + (uintptr_t)p),
+ (uintptr_t)p / 4096UL));
return o;
}
+
+
+/************************************************************/
+
+
+static void major_collection_if_requested(void)
+{
+ assert(!_has_mutex());
+ if (!is_major_collection_requested())
+ return;
+
+ s_mutex_lock();
+
+ if (is_major_collection_requested()) { /* if still true */
+
+ synchronize_all_threads(STOP_OTHERS_UNTIL_MUTEX_UNLOCK);
+
+ if (is_major_collection_requested()) { /* if *still* true */
+ major_collection_now_at_safe_point();
+ }
+
+ }
+
+ s_mutex_unlock();
+}
+
+
+/************************************************************/
+
+/* objects to trace are traced in the sharing seg0 or in a
+ certain segment if there exist modifications there.
+ All other segments' versions should be identical to seg0's
+ version and thus don't need tracing. */
+static struct list_s *marked_objects_to_trace;
+
+/* we use the sharing seg0's pages for the GCFLAG_VISITED flag */
+
+static inline struct object_s *mark_loc(object_t *obj)
+{
+ /* uses the memory in seg0 for marking: */
+ struct object_s *result = (struct object_s*)REAL_ADDRESS(stm_object_pages, obj);
+ return result;
+}
+
+static inline bool mark_visited_test(object_t *obj)
+{
+ struct object_s *realobj = mark_loc(obj);
+ return !!(realobj->stm_flags & GCFLAG_VISITED);
+}
+
+static inline bool mark_visited_test_and_set(object_t *obj)
+{
+ struct object_s *realobj = mark_loc(obj);
+ if (realobj->stm_flags & GCFLAG_VISITED) {
+ return true;
+ }
+ else {
+ realobj->stm_flags |= GCFLAG_VISITED;
+ return false;
+ }
+}
+
+static inline bool mark_visited_test_and_clear(object_t *obj)
+{
+ struct object_s *realobj = mark_loc(obj);
+ if (realobj->stm_flags & GCFLAG_VISITED) {
+ realobj->stm_flags &= ~GCFLAG_VISITED;
+ return true;
+ }
+ else {
+ return false;
+ }
+}
+
+
+/************************************************************/
+
+
+static bool is_new_object(object_t *obj)
+{
+ struct object_s *realobj = (struct object_s*)REAL_ADDRESS(stm_object_pages, obj); /* seg0 */
+ return realobj->stm_flags & GCFLAG_WB_EXECUTED;
+}
+
+
+static inline void mark_record_trace(object_t **pobj)
+{
+ /* takes a normal pointer to a thread-local pointer to an object */
+ object_t *obj = *pobj;
+
+ /* Note: this obj might be visited already, but from a different
+ segment. We ignore this case and skip re-visiting the object
+ anyway. The idea is that such an object is old (not from the
+ current transaction), otherwise it would not be possible to see
+ it in two segments; and moreover it is not modified, otherwise
+ mark_trace() would have been called on two different segments
+ already. That means that this object is identical in all
+ segments and only needs visiting once. (It may actually be in a
+ shared page, or maybe not.)
+ */
+ if (obj == NULL || mark_visited_test_and_set(obj))
+ return; /* already visited this object */
+
+ LIST_APPEND(marked_objects_to_trace, obj);
+}
+
+
+static void mark_and_trace(object_t *obj, char *segment_base)
+{
+ /* mark the obj and trace all reachable objs from it */
+
+ assert(list_is_empty(marked_objects_to_trace));
+
+ /* trace into the object (the version from 'segment_base') */
+ struct object_s *realobj =
+ (struct object_s *)REAL_ADDRESS(segment_base, obj);
+ stmcb_trace(realobj, &mark_record_trace);
+
+ /* trace all references found in sharing seg0 (should always be
+ up-to-date and not cause segfaults, except for new objs) */
+ while (!list_is_empty(marked_objects_to_trace)) {
+ obj = (object_t *)list_pop_item(marked_objects_to_trace);
+
+ char *base = is_new_object(obj) ? segment_base : stm_object_pages;
+ realobj = (struct object_s *)REAL_ADDRESS(base, obj);
+ stmcb_trace(realobj, &mark_record_trace);
+ }
+}
+
+static inline void mark_visit_object(object_t *obj, char *segment_base)
+{
+ /* if already visited, don't trace */
+ if (obj == NULL || mark_visited_test_and_set(obj))
+ return;
+ mark_and_trace(obj, segment_base);
+}
+
+
+static void mark_visit_possibly_new_object(char *segment_base, object_t *obj)
+{
+ /* if newly allocated object, we trace in segment_base, otherwise in
+ the sharing seg0 */
+ if (obj == NULL)
+ return;
+
+ if (is_new_object(obj)) {
+ mark_visit_object(obj, segment_base);
+ } else {
+ mark_visit_object(obj, stm_object_pages);
+ }
+}
+
+static void *mark_visit_objects_from_ss(void *_, const void *slice, size_t size)
+{
+ const struct stm_shadowentry_s *p, *end;
+ p = (const struct stm_shadowentry_s *)slice;
+ end = (const struct stm_shadowentry_s *)(slice + size);
+ for (; p < end; p++)
+ if ((((uintptr_t)p->ss) & 3) == 0) {
+ assert(!is_new_object(p->ss));
+ mark_visit_object(p->ss, stm_object_pages); // seg0
+ }
+ return NULL;
+}
+
+static void assert_obj_accessible_in(long segnum, object_t *obj)
+{
+#ifndef NDEBUG
+ uintptr_t page = (uintptr_t)obj / 4096UL;
+ assert(get_page_status_in(segnum, page) == PAGE_ACCESSIBLE);
+
+ struct object_s *realobj =
+ (struct object_s *)REAL_ADDRESS(get_segment_base(segnum), obj);
+
+ size_t obj_size = stmcb_size_rounded_up(realobj);
+ uintptr_t count = obj_size / 4096UL + 1;
+ while (count--> 0) {
+ assert(get_page_status_in(segnum, page) == PAGE_ACCESSIBLE);
+ page++;
+ }
+#endif
+}
+
+
+
+static void mark_visit_from_modified_objects(void)
+{
+ /* look for modified objects in segments and mark all of them
+ for further tracing (XXX: don't if we are going to share
+ some of the pages) */
+
+ long i;
+ for (i = 1; i < NB_SEGMENTS; i++) {
+ char *base = get_segment_base(i);
+
+ struct list_s *lst = get_priv_segment(i)->modified_old_objects;
+ struct stm_undo_s *modified = (struct stm_undo_s *)lst->items;
+ struct stm_undo_s *end = (struct stm_undo_s *)(lst->items + lst->count);
+
+ for (; modified < end; modified++) {
+ object_t *obj = modified->object;
+ /* All modified objs have all pages accessible for now.
+ This is because we create a backup of the whole obj
+ and thus make all pages accessible. */
+ assert_obj_accessible_in(i, obj);
+
+ assert(!is_new_object(obj)); /* should never be in that list */
+
+ if (!mark_visited_test_and_set(obj)) {
+ /* trace shared, committed version */
+ mark_and_trace(obj, stm_object_pages);
+ }
+ mark_and_trace(obj, base); /* private, modified version */
+ }
+ }
+}
+
+static void mark_visit_from_roots(void)
+{
+ if (testing_prebuilt_objs != NULL) {
+ LIST_FOREACH_R(testing_prebuilt_objs, object_t * /*item*/,
+ mark_visit_object(item, stm_object_pages)); // seg0
+ }
+
+ stm_thread_local_t *tl = stm_all_thread_locals;
+ do {
+ /* look at all objs on the shadow stack (they are old but may
+ be uncommitted so far, so only exist in the associated_segment_num).
+
+ IF they are uncommitted new objs, trace in the actual segment,
+ otherwise, since we just executed a minor collection, they were
+ all synced to the sharing seg0. Thus we can trace them there.
+
+ If they were again modified since then, they were traced
+ by mark_visit_from_modified_object() already.
+ */
+
+ /* only for new, uncommitted objects:
+ If 'tl' is currently running, its 'associated_segment_num'
+ field is the segment number that contains the correct
+ version of its overflowed objects. */
+ char *segment_base = get_segment_base(tl->associated_segment_num);
+
+ struct stm_shadowentry_s *current = tl->shadowstack;
+ struct stm_shadowentry_s *base = tl->shadowstack_base;
+ while (current-- != base) {
+ if ((((uintptr_t)current->ss) & 3) == 0) {
+ mark_visit_possibly_new_object(segment_base, current->ss);
+ }
+ }
+
+ mark_visit_possibly_new_object(segment_base, tl->thread_local_obj);
+
+ tl = tl->next;
+ } while (tl != stm_all_thread_locals);
+
+ /* also visit all objs in the rewind-shadowstack */
+ long i;
+ for (i = 1; i < NB_SEGMENTS; i++) {
+ if (get_priv_segment(i)->transaction_state != TS_NONE) {
+ mark_visit_possibly_new_object(
+ get_segment_base(i),
+ get_priv_segment(i)->threadlocal_at_start_of_transaction);
+
+ stm_rewind_jmp_enum_shadowstack(
+ get_segment(i)->running_thread,
+ mark_visit_objects_from_ss);
+ }
+ }
+}
+
+static void ready_new_objects(void)
+{
+#pragma push_macro("STM_PSEGMENT")
+#pragma push_macro("STM_SEGMENT")
+#undef STM_PSEGMENT
+#undef STM_SEGMENT
+ /* objs in new_objects only have garbage in the sharing seg0,
+ since it is used to mark objs as visited, we must make
+ sure the flag is cleared at the start of a major collection.
+ (XXX: ^^^ may be optional if we have the part below)
+
+ Also, we need to be able to recognize these objects in order
+ to only trace them in the segment they are valid in. So we
+ also make sure to set WB_EXECUTED in the sharing seg0. No
+ other objs than new_objects have WB_EXECUTED in seg0 (since
+ there can only be committed versions there).
+ */
+
+ long i;
+ for (i = 1; i < NB_SEGMENTS; i++) {
+ struct stm_priv_segment_info_s *pseg = get_priv_segment(i);
+ struct list_s *lst = pseg->new_objects;
+
+ LIST_FOREACH_R(lst, object_t* /*item*/,
+ ({
+ struct object_s *realobj;
+ /* WB_EXECUTED always set in this segment */
+ assert(realobj = (struct object_s*)REAL_ADDRESS(pseg->pub.segment_base, item));
+ assert(realobj->stm_flags & GCFLAG_WB_EXECUTED);
+
+ /* clear VISITED and ensure WB_EXECUTED in seg0 */
+ mark_visited_test_and_clear(item);
+ realobj = (struct object_s*)REAL_ADDRESS(stm_object_pages, item);
+ realobj->stm_flags |= GCFLAG_WB_EXECUTED;
+ }));
+ }
+#pragma pop_macro("STM_SEGMENT")
+#pragma pop_macro("STM_PSEGMENT")
+}
+
+
+static void clean_up_segment_lists(void)
+{
+#pragma push_macro("STM_PSEGMENT")
+#pragma push_macro("STM_SEGMENT")
+#undef STM_PSEGMENT
+#undef STM_SEGMENT
+
+ long i;
+ for (i = 1; i < NB_SEGMENTS; i++) {
+ struct stm_priv_segment_info_s *pseg = get_priv_segment(i);
+ struct list_s *lst;
More information about the pypy-commit
mailing list