[pypy-commit] pypy stm-thread-2: in-progress: trying to get test_stmgcintf working again
arigo
noreply at buildbot.pypy.org
Wed Sep 5 09:14:57 CEST 2012
Author: Armin Rigo <arigo at tunes.org>
Branch: stm-thread-2
Changeset: r57139:175127a5208d
Date: 2012-09-04 16:39 +0200
http://bitbucket.org/pypy/pypy/changeset/175127a5208d/
Log: in-progress: trying to get test_stmgcintf working again
diff --git a/pypy/rlib/rstm.py b/pypy/rlib/rstm.py
--- a/pypy/rlib/rstm.py
+++ b/pypy/rlib/rstm.py
@@ -97,15 +97,7 @@
ll_assert(arg is None or isinstance(arg, argcls),
"perform_transaction: wrong class")
before_external_call()
- # Passing around the GC object 'arg' is a bit delicate. At this point
- # it has been saved as a global, but 'arg' likely points to the object
- # with an offset 2, which is the flag used for "used to be a local".
- # We have to revert this flag here...
llarg = cast_instance_to_base_ptr(arg)
- llarg = rffi.cast(lltype.Signed, llarg)
- llarg &= ~2
- llarg = rffi.cast(rffi.VOIDP, llarg)
- #
adr_of_top = llop.gc_adr_of_root_stack_top(llmemory.Address)
callback = _get_stm_callback(func, argcls)
llcallback = llhelper(stmgcintf.StmOperations.CALLBACK_TX, callback)
diff --git a/pypy/rpython/memory/gc/stmgc.py b/pypy/rpython/memory/gc/stmgc.py
--- a/pypy/rpython/memory/gc/stmgc.py
+++ b/pypy/rpython/memory/gc/stmgc.py
@@ -15,11 +15,14 @@
first_gcflag = 1 << (LONG_BIT//2)
-# Terminology:
+# Documentation:
+# https://bitbucket.org/pypy/extradoc/raw/extradoc/talk/stm2012/stmimpl.rst
+#
+# Terminology used here:
#
# - Objects can be LOCAL or GLOBAL. This is what STM is based on.
# Local objects are not visible to any other transaction, whereas
-# global objects are, and need care.
+# global objects are, and are read-only.
#
# - Each object lives either in the shared area, or in a thread-local
# nursery. The shared area contains:
@@ -37,7 +40,7 @@
# are in the shared area. Getting the write barrier right for both
# this and the general STM mechanisms is tricky, so for now this GC
# is not actually generational (slow when running long transactions
-# or before running transactions at all).
+# or before running transactions at all). (XXX fix me)
#
# - So far, the GC is always running in "transactional" mode. Later,
# it would be possible to speed it up in case there is only one
@@ -48,16 +51,15 @@
# - GCFLAG_GLOBAL: identifies GLOBAL objects. All prebuilt objects
# start as GLOBAL; conversely, all freshly allocated objects start
# as LOCAL, and become GLOBAL if they survive an end-of-transaction.
-# All objects that are GLOBAL are immortal for now
+# All objects that are GLOBAL are immortal and leaking for now
# (global_collect() will be done later).
#
-# - GCFLAG_WAS_COPIED: means that the object is either a LOCAL COPY
-# or, if GLOBAL, then it has or had at least one LOCAL COPY. See
-# below.
+# - GCFLAG_POSSIBLY_OUTDATED: see stmimpl.rst. Used by C.
+# - GCFLAG_NOT_WRITTEN: see stmimpl.rst. Used by C.
+# - GCFLAG_LOCAL_COPY: see stmimpl.rst. Used by C.
#
-# - GCFLAG_VISITED: used during collections to flag objects found to be
-# surviving. Between collections, it must be set on the LOCAL COPY
-# objects, and only on them.
+# - GCFLAG_VISITED: used temporarily to mark local objects found to be
+# surviving during a collection.
#
# - GCFLAG_HAS_SHADOW: set on nursery objects whose id() or identityhash()
# was taken. Means that we already have a corresponding object allocated
@@ -65,14 +67,6 @@
#
# - GCFLAG_FIXED_HASH: only on some prebuilt objects. For identityhash().
#
-# When the mutator (= the program outside the GC) wants to write to an
-# object, stm_writebarrier() does something special on GLOBAL objects:
-#
-# - In transactional mode (always for now), the write barrier creates
-# a LOCAL COPY of the object and returns it (or, if already created by
-# the same transaction, finds it again). The mapping from GLOBAL to
-# LOCAL COPY objects is maintained by C code (see tldict_lookup()).
-#
# Invariant: between two transactions, all objects visible from the current
# thread are always GLOBAL. In particular:
#
@@ -91,39 +85,42 @@
# pointer to any LOCAL object at all.
#
# - A special case is the end-of-transaction collection, done by the same
-# local_collection() with a twist: all pointers to a LOCAL COPY object
-# are replaced with pointers to the corresponding GLOBAL original. When
-# it is done, we mark all surviving LOCAL objects as GLOBAL too, and we
-# are back to the situation where this thread sees only GLOBAL objects.
-# What we leave to the C code to do "as the finishing touch" is to copy
-# transactionally the content of the LOCAL COPY objects back over the
-# GLOBAL originals; before this is done, the transaction can be aborted
-# at any point with no visible side-effect on any object that other
-# threads can see.
+# local_collection() with a twist: all surviving objects (after being
+# copied out of the nursery) receive the flags GLOBAL and NOT_WRITTEN.
+# At the end, we are back to the situation where this thread sees only
+# GLOBAL objects. This is PerformLocalCollect() in stmimpl.rst.
+# It is done just before CommitTransaction(), implemented in C.
#
-# All objects have an address-sized 'version' field in their header. On
-# GLOBAL objects, it is used as a version by C code to handle STM (it must
-# be set to 0 when the object first turns GLOBAL). On the LOCAL objects,
-# though, it is abused here in the GC:
+# All objects have an address-sized 'revision' field in their header.
+# It is generally used by the C code (marked with (*) below), but it
+# is also (ab)used by the GC itself (marked with (!) below).
#
-# - if GCFLAG_WAS_COPIED, it points to the GLOBAL original.
+# - for local objects with GCFLAG_LOCAL_COPY, it points to the GLOBAL
+# original (*).
#
-# - if GCFLAG_HAS_SHADOW, to the shadow object outside the nursery.
-# (It is not used on other nursery objects before collection.)
+# - if GCFLAG_HAS_SHADOW, it points to the shadow object outside the
+# nursery (!). (It is not used on other nursery objects before
+# collection.)
#
# - it contains the 'next' object of the 'sharedarea_tls.chained_list'
-# list, which describes all LOCAL objects malloced outside the nursery.
+# list, which describes all LOCAL objects malloced outside the
+# nursery (!).
#
-# - for nursery objects, during collection, if they are copied outside
-# the nursery, they grow GCFLAG_VISITED and their 'version' points
-# to the fresh copy.
+# - during collection, the nursery objects that are copied outside
+# the nursery grow GCFLAG_VISITED and their 'revision' points
+# to the new copy (!).
+#
+# - on any GLOBAL object, 'revision' is managed by C code (*).
+# It must be initialized to 1 when the GC code turns a LOCAL object
+# into a GLOBAL one.
#
-GCFLAG_GLOBAL = first_gcflag << 0 # keep in sync with et.c
-GCFLAG_WAS_COPIED = first_gcflag << 1 # keep in sync with et.c
-GCFLAG_VISITED = first_gcflag << 2
-GCFLAG_HAS_SHADOW = first_gcflag << 3
-GCFLAG_FIXED_HASH = first_gcflag << 4
+GCFLAG_GLOBAL = first_gcflag << 0 # keep in sync with et.h
+GCFLAG_POSSIBLY_OUTDATED = first_gcflag << 1 # keep in sync with et.h
+GCFLAG_NOT_WRITTEN = first_gcflag << 2 # keep in sync with et.h
+GCFLAG_LOCAL_COPY = first_gcflag << 3 # keep in sync with et.h
+GCFLAG_HAS_SHADOW = first_gcflag << 4
+GCFLAG_FIXED_HASH = first_gcflag << 5
def always_inline(fn):
@@ -138,12 +135,11 @@
_alloc_flavor_ = "raw"
inline_simple_malloc = True
inline_simple_malloc_varsize = True
- #needs_write_barrier = "stm"
prebuilt_gc_objects_are_static_roots = False
malloc_zero_filled = True # xxx?
HDR = lltype.Struct('header', ('tid', lltype.Signed),
- ('version', llmemory.Address))
+ ('revision', lltype.Unsigned))
typeid_is_in_field = 'tid'
withhash_flag_is_in_field = 'tid', GCFLAG_FIXED_HASH
@@ -182,8 +178,6 @@
return self.get_size(obj)
self._stm_getsize = _stm_getsize
#
- ##for size, TYPE in PRIMITIVE_SIZES.items():
- ## self.declare_reader(size, TYPE)
self.declare_write_barrier()
def setup(self):
diff --git a/pypy/translator/stm/src_stm/et.c b/pypy/translator/stm/src_stm/et.c
--- a/pypy/translator/stm/src_stm/et.c
+++ b/pypy/translator/stm/src_stm/et.c
@@ -18,13 +18,14 @@
#define PYPY_DEBUG_FILE stderr
#include "et.h"
-#include "atomic_ops.h"
/************************************************************/
+typedef Unsigned revision_t;
#define INEVITABLE ((revision_t)-1)
#define LOCKED ((revision_t)-0x10000)
+#include "atomic_ops.h"
#include "lists.c"
/************************************************************/
@@ -47,6 +48,11 @@
struct FXCache recent_reads_cache;
};
+struct gcroot_s {
+ gcptr R, L;
+ revision_t v;
+};
+
static volatile revision_t global_cur_time = 2; /* always even */
static volatile revision_t next_locked_value = LOCKED + 3; /* always odd */
static __thread struct tx_descriptor *thread_descriptor = NULL;
@@ -495,7 +501,7 @@
return 1;
}
-revision_t CommitTransaction(void)
+void CommitTransaction(void)
{
revision_t cur_time;
struct tx_descriptor *d = thread_descriptor;
@@ -539,7 +545,6 @@
d->num_commits++;
d->active = 0;
- return cur_time;
}
/************************************************************/
@@ -593,7 +598,7 @@
/************************************************************/
-revision_t DescriptorInit(void)
+void DescriptorInit(void)
{
assert(thread_descriptor == NULL);
if (1)
@@ -628,7 +633,6 @@
(long)d->my_lock, (long)pthread_self());
PYPY_DEBUG_STOP("stm-init");
#endif
- return d->my_lock;
}
}
@@ -673,3 +677,5 @@
free(d);
}
+
+#include "rpyintf.c"
diff --git a/pypy/translator/stm/src_stm/et.h b/pypy/translator/stm/src_stm/et.h
--- a/pypy/translator/stm/src_stm/et.h
+++ b/pypy/translator/stm/src_stm/et.h
@@ -57,14 +57,7 @@
(__builtin_expect((((gcptr)(R))->h_tid & GCFLAG_NOT_WRITTEN) == 0, 1) ? \
(R) : (typeof(R))_WriteBarrierFromReady((gcptr)(R)))
-/* declared in structdef.h:
-struct gcroot_s {
- gcptr R, L;
- Signed v;
-};*/
-
void BeginTransaction(jmp_buf *);
-struct gcroot_s *FindRootsForLocalCollect(void);
int _FakeReach(gcptr);
void CommitTransaction(void);
void BecomeInevitable(void);
@@ -82,7 +75,11 @@
//gcptr _NonTransactionalReadBarrier(gcptr);
-extern size_t pypy_g__stm_getsize(gcptr);
-
+extern Signed pypy_g__stm_getsize(gcptr);
+void stm_set_tls(void *newtls);
+void *stm_get_tls(void);
+void stm_del_tls(void);
+gcptr stm_tldict_lookup(gcptr); /* for tests only */
+void stm_tldict_add(gcptr, gcptr); /* for tests only */
#endif /* _ET_H */
diff --git a/pypy/translator/stm/src_stm/lists.c b/pypy/translator/stm/src_stm/lists.c
--- a/pypy/translator/stm/src_stm/lists.c
+++ b/pypy/translator/stm/src_stm/lists.c
@@ -256,7 +256,7 @@
too simple for now. */
#define FX_ENTRIES 32
-#define FX_SIZE (FX_ENTRIES * sizeof(uintptr_t))
+#define FX_SIZE (FX_ENTRIES * sizeof(revision_t))
#define FX_THRESHOLD 5
#if FX_THRESHOLD >= FX_ENTRIES * 4 /* == lower bound on FX_SIZE */
@@ -264,7 +264,7 @@
#endif
struct FXCache {
- uintptr_t cache[FX_ENTRIES];
+ revision_t cache[FX_ENTRIES];
};
static void fxcache_clear(struct FXCache *fxcache)
@@ -278,11 +278,11 @@
If it is already, increment its value and returns 1.
If it we reach FX_THRESHOLD, returns 2.
*/
- uintptr_t uitem = (uintptr_t)item;
- uintptr_t *entry = (uintptr_t *)
- (((char *)fxcache->cache) + (uitem & (FX_SIZE-sizeof(uintptr_t))));
- uintptr_t stored_key = uitem & -FX_SIZE;
- uintptr_t value = stored_key ^ *entry;
+ revision_t uitem = (revision_t)item;
+ revision_t *entry = (revision_t *)
+ (((char *)fxcache->cache) + (uitem & (FX_SIZE-sizeof(revision_t))));
+ revision_t stored_key = uitem & -FX_SIZE;
+ revision_t value = stored_key ^ *entry;
if (value >= FX_SIZE)
{
/* not in the cache: evict the colliding item (no associativity) */
diff --git a/pypy/translator/stm/src_stm/rpyintf.c b/pypy/translator/stm/src_stm/rpyintf.c
new file mode 100644
--- /dev/null
+++ b/pypy/translator/stm/src_stm/rpyintf.c
@@ -0,0 +1,35 @@
+
+static __thread void *rpython_tls_object;
+
+void stm_set_tls(void *newtls)
+{
+ rpython_tls_object = newtls;
+}
+
+void *stm_get_tls(void)
+{
+ return rpython_tls_object;
+}
+
+void stm_del_tls(void)
+{
+ rpython_tls_object = NULL;
+}
+
+gcptr stm_tldict_lookup(gcptr key)
+{
+ struct tx_descriptor *d = thread_descriptor;
+ wlog_t* found;
+ G2L_FIND(d->global_to_local, key, found, goto not_found);
+ return found->val;
+
+ not_found:
+ return NULL;
+}
+
+void stm_tldict_add(gcptr key, gcptr value)
+{
+ struct tx_descriptor *d = thread_descriptor;
+ assert(d != NULL);
+ g2l_insert(&d->global_to_local, key, value);
+}
diff --git a/pypy/translator/stm/stmgcintf.py b/pypy/translator/stm/stmgcintf.py
--- a/pypy/translator/stm/stmgcintf.py
+++ b/pypy/translator/stm/stmgcintf.py
@@ -27,14 +27,6 @@
class StmOperations(object):
- PRIMITIVE_SIZES = {1: lltype.Char,
- 2: rffi.SHORT,
- 4: rffi.INT,
- 8: lltype.SignedLongLong,
- '8f': rffi.DOUBLE,
- '4f': rffi.FLOAT}
-
- INIT_DONE = lltype.Ptr(lltype.FuncType([], lltype.Void))
CALLBACK_TX = lltype.Ptr(lltype.FuncType([rffi.VOIDP, lltype.Signed],
lltype.Signed))
GETSIZE = lltype.Ptr(lltype.FuncType([llmemory.Address],
diff --git a/pypy/translator/stm/test/test_stmgcintf.c b/pypy/translator/stm/test/test_stmgcintf.c
--- a/pypy/translator/stm/test/test_stmgcintf.c
+++ b/pypy/translator/stm/test/test_stmgcintf.c
@@ -2,9 +2,12 @@
#define PYPY_LONG_BIT (sizeof(long) * 8)
+typedef long Signed;
+typedef unsigned long Unsigned;
+
struct pypy_header0 {
long h_tid;
- void *h_version;
+ Unsigned h_revision;
};
struct pypy_pypy_rlib_rstm_Transaction0 {
@@ -34,10 +37,10 @@
#include "src_stm/et.c"
-long (*cb_getsize)(void *);
+long (*cb_getsize)(gcptr);
void (*cb_enum_callback)(void *, void *, void *);
-long pypy_g__stm_getsize(void *a) {
+long pypy_g__stm_getsize(gcptr a) {
assert(cb_getsize != NULL);
return cb_getsize(a);
}
@@ -196,6 +199,7 @@
/************************************************************/
+#if 0
void test_read_main_thread(void)
{
S1 s1;
@@ -282,7 +286,7 @@
s2->last_16_bytes[i] = 'A' + i;
stm_tldict_add(&sg_global, s2);
}
-long size_getter_cb(void *x)
+long size_getter_cb(gcptr x)
{
return offsetof(S1, last_16_bytes) + 15;
}
@@ -318,6 +322,7 @@
void test_copy_transactional_to_raw(void) {
run_in_transaction(copy_transactional_to_raw, '.');
}
+#endif
/************************************************************/
@@ -349,8 +354,8 @@
S1 s1[15];
int i;
for (i=0; i<15; i++) {
- s1[i].header.h_tid = GCFLAG_GLOBAL;
- s1[i].header.h_version = NULL;
+ s1[i].header.h_tid = GCFLAG_PREBUILT;
+ s1[i].header.h_revision = REV_INITIAL;
s1[i].value1 = 48+i;
}
for (i=0; i<15; i++) {
More information about the pypy-commit
mailing list