[pypy-commit] stmgc card-marking: some clarifications and adding the second fastpath to stm_write_card()

[Raemi]

Author: Remi Meier <remi.meier at inf.ethz.ch>
Branch: card-marking
Changeset: r1242:008b3c48e38b
Date: 2014-05-23 09:55 +0200
http://bitbucket.org/pypy/stmgc/changeset/008b3c48e38b/

Log:	some clarifications and adding the second fastpath to
	stm_write_card()

diff --git a/c7/stm/core.c b/c7/stm/core.c
--- a/c7/stm/core.c
+++ b/c7/stm/core.c
@@ -40,7 +40,7 @@
 #endif
 }
 
-static void _stm_mark_card(object_t *obj, uintptr_t card_index)
+void _stm_mark_card(object_t *obj, uintptr_t card_index)
 {
     assert(card_index > 0);
 
@@ -115,12 +115,27 @@
     assert(_seems_to_be_running_transaction());
     assert(!_is_young(obj));
     assert(obj->stm_flags & GCFLAG_WRITE_BARRIER);
+    assert(IMPLY(card_index, (card_index - 1) * CARD_SIZE < stmcb_size_rounded_up(
+                     (struct object_s*)REAL_ADDRESS(STM_SEGMENT->segment_base, obj))));
+
+    uintptr_t base_lock_idx = get_write_lock_idx((uintptr_t)obj);
+    uint8_t lock_num = STM_PSEGMENT->write_lock_num;
+    assert(base_lock_idx < sizeof(write_locks));
 
     if (!(obj->stm_flags & GCFLAG_HAS_CARDS))
         card_index = 0;         /* assume no cards */
 
-    assert(IMPLY(card_index, (card_index - 1) * CARD_SIZE < stmcb_size_rounded_up(
-                (struct object_s*)REAL_ADDRESS(STM_SEGMENT->segment_base, obj))));
+    /* if card_index and obj->stm_flags & CARDS_SET:
+           directly mark the card of obj at card_index
+           return (no STM part needed)
+       -> see stmgc.h */
+    /* if CARDS_SET, we entered here at least once, so we own the write_lock
+       OR this is an overflow object and the write_lock is not owned */
+    OPT_ASSERT(
+        IMPLY(obj->stm_flags & GCFLAG_CARDS_SET,
+              (!IS_OVERFLOW_OBJ(STM_PSEGMENT, obj) && write_locks[base_lock_idx] == lock_num)
+              || (IS_OVERFLOW_OBJ(STM_PSEGMENT, obj) && write_locks[base_lock_idx] == 0)
+              ));
 
     if (_stm_write_slowpath_overflow_objs(obj, card_index))
         return;
@@ -136,9 +151,7 @@
        'modified_old_objects' (but, because it had GCFLAG_WRITE_BARRIER,
        not in 'objects_pointing_to_nursery').  We'll detect this case
        by finding that we already own the write-lock. */
-    uintptr_t base_lock_idx = get_write_lock_idx((uintptr_t)obj);
-    uint8_t lock_num = STM_PSEGMENT->write_lock_num;
-    assert(base_lock_idx < sizeof(write_locks));
+
  retry:
     if (write_locks[base_lock_idx] == 0) {
         /* A lock to prevent reading garbage from
diff --git a/c7/stm/core.h b/c7/stm/core.h
--- a/c7/stm/core.h
+++ b/c7/stm/core.h
@@ -233,8 +233,8 @@
 
 #define REAL_ADDRESS(segment_base, src)   ((segment_base) + (uintptr_t)(src))
 
-#define IS_OVERFLOW_OBJ(pseg, obj) ((obj->stm_flags & -GCFLAG_OVERFLOW_NUMBER_bit0) \
-                                    == pseg->overflow_number)
+#define IS_OVERFLOW_OBJ(pseg, obj) (((obj)->stm_flags & -GCFLAG_OVERFLOW_NUMBER_bit0) \
+                                    == (pseg)->overflow_number)
 
 static inline uintptr_t get_card_index(uintptr_t byte_offset) {
     return (byte_offset / CARD_SIZE) + 1;
diff --git a/c7/stmgc.h b/c7/stmgc.h
--- a/c7/stmgc.h
+++ b/c7/stmgc.h
@@ -108,6 +108,7 @@
    but it's not exposed to C code so far */
 void _stm_write_slowpath(object_t *);
 void _stm_write_slowpath_card(object_t *, uintptr_t);
+void _stm_mark_card(object_t *obj, uintptr_t card_index);
 object_t *_stm_allocate_slowpath(ssize_t);
 object_t *_stm_allocate_external(ssize_t);
 void _stm_become_inevitable(const char*);
@@ -221,16 +222,23 @@
 }
 
 /* The following is a GC-optimized barrier that works on the granularity
-   of CARD_SIZE. It can only be used on objects one called stm_use_cards()
-   on. It has the same purpose as stm_write() for TM.
+   of CARD_SIZE. It can only be used on objects one any object, but only
+   helps with those that were internally marked with GCFLAG_HAS_CARDS
+   It has the same purpose as stm_write() for TM.
    'index' is the byte-offset into the object divided by _STM_CARD_SIZE
    plus 1: (offset // CARD_SIZE) + 1
 */
 __attribute__((always_inline))
 static inline void stm_write_card(object_t *obj, uintptr_t index)
 {
-    if (UNLIKELY((obj->stm_flags & _STM_GCFLAG_WRITE_BARRIER) != 0))
-        _stm_write_slowpath_card(obj, index);
+    if (UNLIKELY((obj->stm_flags & _STM_GCFLAG_WRITE_BARRIER) != 0)) {
+        if (LIKELY((obj->stm_flags & _STM_GCFLAG_CARDS_SET) != 0)) {
+            /* XXX: check how well clang optimizes this */
+            _stm_mark_card(obj, index);
+        } else {
+            _stm_write_slowpath_card(obj, index);
+        }
+    }
 }
 
 /* Must be provided by the user of this library.