[pypy-commit] stmgc use-gcc: "Detect" stack overflows by checking if a segfault is caused by

[arigo]

Author: Armin Rigo <arigo at tunes.org>
Branch: use-gcc
Changeset: r1891:8b922e88252b
Date: 2015-07-02 19:33 +0100
http://bitbucket.org/pypy/stmgc/changeset/8b922e88252b/

Log:	"Detect" stack overflows by checking if a segfault is caused by
	access in the protected page at the end of the shadowstack.

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -238,6 +238,7 @@
         addr >= stm_object_pages+TOTAL_MEMORY) {
         /* actual segfault, unrelated to stmgc */
         fprintf(stderr, "Segmentation fault: accessing %p\n", addr);
+        detect_shadowstack_overflow(addr);
         abort();
     }
 
diff --git a/c8/stm/setup.c b/c8/stm/setup.c
--- a/c8/stm/setup.c
+++ b/c8/stm/setup.c
@@ -186,12 +186,12 @@
     teardown_modification_locks();
 }
 
-static void _shadowstack_trap_page(char *start, int prot)
+static char *_shadowstack_trap_page(struct stm_shadowentry_s *base)
 {
     size_t bsize = STM_SHADOW_STACK_DEPTH * sizeof(struct stm_shadowentry_s);
-    char *end = start + bsize + 4095;
+    char *end = ((char *)base) + bsize + 4095;
     end -= (((uintptr_t)end) & 4095);
-    mprotect(end, 4096, prot);
+    return end;
 }
 
 static void _init_shadow_stack(stm_thread_local_t *tl)
@@ -203,9 +203,9 @@
 
     /* set up a trap page: if the shadowstack overflows, it will
        crash in a clean segfault */
-    _shadowstack_trap_page(start, PROT_NONE);
+    struct stm_shadowentry_s *s = (struct stm_shadowentry_s *)start;
+    mprotect(_shadowstack_trap_page(s), 4096, PROT_NONE);
 
-    struct stm_shadowentry_s *s = (struct stm_shadowentry_s *)start;
     tl->shadowstack = s;
     tl->shadowstack_base = s;
     STM_PUSH_ROOT(*tl, -1);
@@ -216,8 +216,8 @@
     assert(tl->shadowstack > tl->shadowstack_base);
     assert(tl->shadowstack_base->ss == (object_t *)-1);
 
-    char *start = (char *)tl->shadowstack_base;
-    _shadowstack_trap_page(start, PROT_READ | PROT_WRITE);
+    char *trap = _shadowstack_trap_page(tl->shadowstack_base);
+    mprotect(trap, 4096, PROT_READ | PROT_WRITE);
 
     free(tl->shadowstack_base);
     tl->shadowstack = NULL;
@@ -298,3 +298,19 @@
 {
     return tl->next != NULL;
 }
+
+static void detect_shadowstack_overflow(char *addr)
+{
+    if (addr == NULL)
+        return;
+    stm_thread_local_t *tl = stm_all_thread_locals;
+    while (tl != NULL) {
+        char *trap = _shadowstack_trap_page(tl->shadowstack_base);
+        if (trap <= addr && addr <= trap + 4095) {
+            fprintf(stderr, "This is caused by a stack overflow.\n"
+                "Sorry, proper RuntimeError support is not implemented yet.\n");
+            return;
+        }
+        tl = tl->next;
+    }
+}
diff --git a/c8/stm/setup.h b/c8/stm/setup.h
--- a/c8/stm/setup.h
+++ b/c8/stm/setup.h
@@ -1,6 +1,7 @@
 static void setup_mmap(char *reason);
 static void setup_protection_settings(void);
 static pthread_t *_get_cpth(stm_thread_local_t *);
+static void detect_shadowstack_overflow(char *);
 
 #ifndef NDEBUG
 static __thread long _stm_segfault_expected = 1;