[pypy-commit] stmgc default: Simplify the forking logic by enforcing that stm_object_pages are

[arigo]

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1615:ac07eddb02d5
Date: 2015-02-09 15:57 +0100
http://bitbucket.org/pypy/stmgc/changeset/ac07eddb02d5/

Log:	Simplify the forking logic by enforcing that stm_object_pages are
	private pages all the way to END_NURSERY_PAGE, and shared pages
	afterwards. Removes the need for page_is_null(), which seems one
	the main sources of fork slowness.

diff --git a/c7/stm/core.c b/c7/stm/core.c
--- a/c7/stm/core.c
+++ b/c7/stm/core.c
@@ -309,18 +309,19 @@
     /* force-reset all read markers to 0 */
 
     char *readmarkers = REAL_ADDRESS(STM_SEGMENT->segment_base,
-                                     FIRST_READMARKER_PAGE * 4096UL);
+                                     FIRST_OLD_RM_PAGE * 4096UL);
+    uintptr_t num_bytes = 4096UL *
+        (NB_READMARKER_PAGES - (FIRST_OLD_RM_PAGE - FIRST_READMARKER_PAGE));
+
     dprintf(("reset_transaction_read_version: %p %ld\n", readmarkers,
-             (long)(NB_READMARKER_PAGES * 4096UL)));
+             (long)num_bytes));
 
-    if (mmap(readmarkers, NB_READMARKER_PAGES * 4096UL,
+    if (mmap(readmarkers, num_bytes,
              PROT_READ | PROT_WRITE,
-             MAP_FIXED | MAP_PAGES_FLAGS, -1, 0) != readmarkers) {
-        /* fall-back */
-#if STM_TESTS
+             MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE,
+             -1, 0) != readmarkers) {
+        /* failed */
         stm_fatalerror("reset_transaction_read_version: %m");
-#endif
-        memset(readmarkers, 0, NB_READMARKER_PAGES * 4096UL);
     }
     STM_SEGMENT->transaction_read_version = 1;
 }
diff --git a/c7/stm/forksupport.c b/c7/stm/forksupport.c
--- a/c7/stm/forksupport.c
+++ b/c7/stm/forksupport.c
@@ -12,16 +12,6 @@
 static stm_thread_local_t *fork_this_tl;
 static bool fork_was_in_transaction;
 
-static bool page_is_null(char *p)
-{
-    long *q = (long *)p;
-    long i;
-    for (i = 0; i < 4096 / sizeof(long); i++)
-        if (q[i] != 0)
-            return false;
-    return true;
-}
-
 
 static void forksupport_prepare(void)
 {
@@ -71,27 +61,6 @@
     int big_copy_fd;
     char *big_copy = setup_mmap("stmgc's fork support", &big_copy_fd);
 
-    /* Copy each of the segment infos into the new mmap, nurseries,
-       and associated read markers
-     */
-    long i;
-    for (i = 1; i <= NB_SEGMENTS; i++) {
-        char *src, *dst;
-        struct stm_priv_segment_info_s *psrc = get_priv_segment(i);
-        dst = big_copy + (((char *)psrc) - stm_object_pages);
-        *(struct stm_priv_segment_info_s *)dst = *psrc;
-
-        src = get_segment_base(i) + FIRST_READMARKER_PAGE * 4096UL;
-        dst = big_copy + (src - stm_object_pages);
-        long j;
-        for (j = 0; j < END_NURSERY_PAGE - FIRST_READMARKER_PAGE; j++) {
-            if (!page_is_null(src))
-                pagecopy(dst, src);
-            src += 4096;
-            dst += 4096;
-        }
-    }
-
     /* Copy all the data from the two ranges of objects (large, small)
        into the new mmap
     */
@@ -201,16 +170,24 @@
        just release these locks early */
     s_mutex_unlock();
 
-    /* Move the copy of the mmap over the old one, overwriting it
-       and thus freeing the old mapping in this process
+    /* Move the copy of the mmap over the old one, overwriting it,
+       with "holes" for each segment's read markers (which are already
+       MAP_PRIVATE and shouldn't be overwritten).  Then free the copy.
     */
     assert(fork_big_copy != NULL);
     assert(stm_object_pages != NULL);
-    void *res = mremap(fork_big_copy, TOTAL_MEMORY, TOTAL_MEMORY,
-                       MREMAP_MAYMOVE | MREMAP_FIXED,
-                       stm_object_pages);
-    if (res != stm_object_pages)
-        stm_fatalerror("after fork: mremap failed: %m");
+
+    long j;
+    for (j = 0; j <= NB_SEGMENTS; j++) {
+        char *dst = get_segment_base(j) + END_NURSERY_PAGE * 4096UL;
+        char *src = fork_big_copy + (dst - stm_object_pages);
+        uintptr_t num_bytes = (NB_PAGES - END_NURSERY_PAGE) * 4096UL;
+        void *res = mremap(src, num_bytes, num_bytes,
+                           MREMAP_MAYMOVE | MREMAP_FIXED, dst);
+        if (res != dst)
+            stm_fatalerror("after fork: mremap failed: %m");
+    }
+    munmap(fork_big_copy, TOTAL_MEMORY);
     fork_big_copy = NULL;
     close_fd_mmap(stm_object_pages_fd);
     stm_object_pages_fd = fork_big_copy_fd;
diff --git a/c7/stm/setup.c b/c7/stm/setup.c
--- a/c7/stm/setup.c
+++ b/c7/stm/setup.c
@@ -99,9 +99,21 @@
 
     stm_object_pages = setup_mmap("initial stm_object_pages mmap()",
                                   &stm_object_pages_fd);
+
+    /* remap MAP_PRIVATE pages on the initial part of each segment
+       in stm_object_pages */
+    long i;
+    for (i = 0; i <= NB_SEGMENTS; i++) {
+        char *res;
+        res = mmap(get_segment_base(i), END_NURSERY_PAGE * 4096UL,
+                   PROT_READ | PROT_WRITE,
+                   MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE,
+                   -1, 0);
+        if (res == MAP_FAILED)
+            stm_fatalerror("mmap(private) failed: %m");
+    }
     setup_protection_settings();
 
-    long i;
     for (i = 1; i <= NB_SEGMENTS; i++) {
         char *segment_base = get_segment_base(i);
 
@@ -133,16 +145,11 @@
         pr->old_objects_with_light_finalizers = list_create();
         pr->overflow_number = GCFLAG_OVERFLOW_NUMBER_bit0 * i;
         highest_overflow_number = pr->overflow_number;
-        pr->pub.transaction_read_version = 0xff;
+        pr->pub.transaction_read_version = 0;
     }
 
     /* The pages are shared lazily, as remap_file_pages() takes a relatively
        long time for each page.
-
-       The read markers are initially zero, but we set anyway
-       transaction_read_version to 0xff in order to force the first
-       transaction to "clear" the read markers by mapping a different,
-       private range of addresses.
     */
 
     setup_sync();