[pypy-commit] pypy quad-color-gc: Update qcgc codebase
ntruessel
pypy.commits at gmail.com
Sun Aug 28 05:55:32 EDT 2016
Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r86636:041e0b606934
Date: 2016-08-27 16:36 +0200
http://bitbucket.org/pypy/pypy/changeset/041e0b606934/
Log: Update qcgc codebase
diff --git a/rpython/translator/c/src/qcgc/allocator.c b/rpython/translator/c/src/qcgc/allocator.c
--- a/rpython/translator/c/src/qcgc/allocator.c
+++ b/rpython/translator/c/src/qcgc/allocator.c
@@ -2,12 +2,14 @@
#include <assert.h>
#include <stdbool.h>
+#include <stdlib.h>
#include <string.h>
+#include "hugeblocktable.h"
+
QCGC_STATIC size_t bytes_to_cells(size_t bytes);
QCGC_STATIC void bump_allocator_assign(cell_t *ptr, size_t cells);
-QCGC_STATIC cell_t *bump_allocator_allocate(size_t cells);
QCGC_STATIC void bump_allocator_advance(size_t cells);
QCGC_STATIC bool is_small(size_t cells);
@@ -15,7 +17,6 @@
QCGC_STATIC size_t large_index(size_t cells);
QCGC_STATIC size_t small_index_to_cells(size_t index);
-QCGC_STATIC cell_t *fit_allocator_allocate(size_t cells);
QCGC_STATIC cell_t *fit_allocator_small_first_fit(size_t index, size_t cells);
QCGC_STATIC cell_t *fit_allocator_large_fit(size_t index, size_t cells);
QCGC_STATIC cell_t *fit_allocator_large_first_fit(size_t index, size_t cells);
@@ -61,28 +62,6 @@
free(qcgc_allocator_state.arenas);
}
-cell_t *qcgc_allocator_allocate(size_t bytes) {
- size_t size_in_cells = bytes_to_cells(bytes);
-#if CHECKED
- assert(size_in_cells > 0);
- assert(size_in_cells <= QCGC_ARENA_CELLS_COUNT - QCGC_ARENA_FIRST_CELL_INDEX);
-#endif
- cell_t *result;
-
- // TODO: Implement switch for bump/fit allocator
- if (true) {
- result = bump_allocator_allocate(size_in_cells);
- } else {
- result = fit_allocator_allocate(size_in_cells);
- }
-
- qcgc_arena_mark_allocated(result, size_in_cells);
-#if QCGC_INIT_ZERO
- memset(result, 0, bytes);
-#endif
- return result;
-}
-
void qcgc_fit_allocator_add(cell_t *ptr, size_t cells) {
if (cells > 0) {
if (is_small(cells)) {
@@ -111,36 +90,75 @@
qcgc_allocator_state.bump_state.remaining_cells -= cells;
}
-QCGC_STATIC cell_t *bump_allocator_allocate(size_t cells) {
+/*******************************************************************************
+ * Allocators *
+ ******************************************************************************/
+
+object_t *qcgc_bump_allocate(size_t bytes) {
+ size_t cells = bytes_to_cells(bytes);
if (cells > qcgc_allocator_state.bump_state.remaining_cells) {
// Grab a new arena
+ // FIXME: Add remaining memory to fit allocator
arena_t *arena = qcgc_arena_create();
bump_allocator_assign(&(arena->cells[QCGC_ARENA_FIRST_CELL_INDEX]),
QCGC_ARENA_CELLS_COUNT - QCGC_ARENA_FIRST_CELL_INDEX);
qcgc_allocator_state.arenas =
qcgc_arena_bag_add(qcgc_allocator_state.arenas, arena);
}
- cell_t *result = qcgc_allocator_state.bump_state.bump_ptr;
+ cell_t *mem = qcgc_allocator_state.bump_state.bump_ptr;
bump_allocator_advance(cells);
+
+ qcgc_arena_mark_allocated(mem, cells);
+ object_t *result = (object_t *) mem;
+
+#if QCGC_INIT_ZERO
+ memset(result, 0, cells * sizeof(cell_t));
+#endif
+
+ result->flags |= QCGC_GRAY_FLAG;
return result;
}
-QCGC_STATIC cell_t *fit_allocator_allocate(size_t cells) {
- cell_t *result;
+object_t *qcgc_fit_allocate(size_t bytes) {
+ size_t cells = bytes_to_cells(bytes);
+ cell_t *mem;
if (is_small(cells)) {
size_t index = small_index(cells);
- result = fit_allocator_small_first_fit(index, cells);
+ mem = fit_allocator_small_first_fit(index, cells);
} else {
size_t index = large_index(cells);
- result = fit_allocator_large_fit(index, cells);
+ mem = fit_allocator_large_fit(index, cells);
}
- if (result == NULL) {
- // No valid block found
- result = bump_allocator_allocate(cells);
+ if (mem == NULL) {
+ return NULL;
}
+ qcgc_arena_mark_allocated(mem, cells);
+ object_t *result = (object_t *) mem;
+
+#if QCGC_INIT_ZERO
+ memset(result, 0, cells * sizeof(cell_t));
+#endif
+
+ result->flags |= QCGC_GRAY_FLAG;
+ return result;
+}
+
+/**
+ * Constraints:
+ * - Zero initialized
+ * - Aligned to arena size
+ * - Multiple of arena size
+ * - No header, metadata stored in hash-map
+ */
+object_t *qcgc_large_allocate(size_t bytes) {
+ object_t *result = aligned_alloc(QCGC_ARENA_SIZE, bytes);
+#if QCGC_INIT_ZERO
+ memset(result, 0, bytes);
+#endif
+ qcgc_hbtable_insert(result);
return result;
}
diff --git a/rpython/translator/c/src/qcgc/allocator.h b/rpython/translator/c/src/qcgc/allocator.h
--- a/rpython/translator/c/src/qcgc/allocator.h
+++ b/rpython/translator/c/src/qcgc/allocator.h
@@ -57,13 +57,32 @@
void qcgc_allocator_destroy(void);
/**
- * Allocate new memory region
+ * Allocate new memory region using fit allocator
+ *
+ * @param bytes Desired size of the memory region in bytes
+ * @return Pointer to memory large enough to hold size bytes, NULL in case of
+ * errors or if there is no block sufficently large block, already zero
+ * initialized if QCGC_INIT_ZERO is set
+ */
+object_t *qcgc_fit_allocate(size_t bytes);
+
+/**
+ * Allocate new memory region using bump allocator
*
* @param bytes Desired size of the memory region in bytes
* @return Pointer to memory large enough to hold size bytes, NULL in case of
* errors, already zero initialized if QCGC_INIT_ZERO is set
*/
-cell_t *qcgc_allocator_allocate(size_t bytes);
+object_t *qcgc_bump_allocate(size_t bytes);
+
+/**
+ * Allocate new memory region using huge block allocator
+ *
+ * @param bytes Desired size of the memory region in bytes
+ * @return Pointer to memory large enough to hold size bytes, NULL in case of
+ * errors, already zero initialized if QCGC_INIT_ZERO is set
+ */
+object_t *qcgc_large_allocate(size_t bytes);
/**
diff --git a/rpython/translator/c/src/qcgc/bag.c b/rpython/translator/c/src/qcgc/bag.c
--- a/rpython/translator/c/src/qcgc/bag.c
+++ b/rpython/translator/c/src/qcgc/bag.c
@@ -5,3 +5,4 @@
DEFINE_BAG(arena_bag, arena_t *);
DEFINE_BAG(linear_free_list, cell_t *);
DEFINE_BAG(exp_free_list, struct exp_free_list_item_s);
+DEFINE_BAG(hbbucket, struct hbtable_entry_s);
diff --git a/rpython/translator/c/src/qcgc/bag.h b/rpython/translator/c/src/qcgc/bag.h
--- a/rpython/translator/c/src/qcgc/bag.h
+++ b/rpython/translator/c/src/qcgc/bag.h
@@ -14,13 +14,23 @@
type items[]; \
} name##_t; \
\
+__attribute__ ((warn_unused_result)) \
name##_t *qcgc_##name##_create(size_t size); \
+ \
+__attribute__ ((warn_unused_result)) \
name##_t *qcgc_##name##_add(name##_t *self, type item); \
+ \
+__attribute__ ((warn_unused_result)) \
name##_t *qcgc_##name##_remove_index(name##_t *self, size_t index);
#define DEFINE_BAG(name, type) \
+ \
QCGC_STATIC size_t name##_size(size_t size); \
+ \
+__attribute__ ((warn_unused_result)) \
QCGC_STATIC name##_t *name##_grow(name##_t *self); \
+ \
+__attribute__ ((warn_unused_result)) \
QCGC_STATIC name##_t *name##_shrink(name##_t *self); \
\
name##_t *qcgc_##name##_create(size_t size) { \
@@ -77,6 +87,12 @@
size_t size;
};
+struct hbtable_entry_s {
+ object_t *object;
+ bool mark_flag;
+};
+
DECLARE_BAG(arena_bag, arena_t *);
DECLARE_BAG(linear_free_list, cell_t *);
DECLARE_BAG(exp_free_list, struct exp_free_list_item_s);
+DECLARE_BAG(hbbucket, struct hbtable_entry_s);
diff --git a/rpython/translator/c/src/qcgc/gc_state.h b/rpython/translator/c/src/qcgc/gc_state.h
--- a/rpython/translator/c/src/qcgc/gc_state.h
+++ b/rpython/translator/c/src/qcgc/gc_state.h
@@ -25,6 +25,7 @@
struct qcgc_state {
shadow_stack_t *shadow_stack;
shadow_stack_t *prebuilt_objects;
+ gray_stack_t *gp_gray_stack;
size_t gray_stack_size;
gc_phase_t phase;
} qcgc_state;
diff --git a/rpython/translator/c/src/qcgc/gray_stack.h b/rpython/translator/c/src/qcgc/gray_stack.h
--- a/rpython/translator/c/src/qcgc/gray_stack.h
+++ b/rpython/translator/c/src/qcgc/gray_stack.h
@@ -12,8 +12,13 @@
object_t *items[];
} gray_stack_t;
+__attribute__ ((warn_unused_result))
gray_stack_t *qcgc_gray_stack_create(size_t size);
+__attribute__ ((warn_unused_result))
gray_stack_t *qcgc_gray_stack_push(gray_stack_t *stack, object_t *item);
+
object_t *qcgc_gray_stack_top(gray_stack_t *stack);
+
+__attribute__ ((warn_unused_result))
gray_stack_t *qcgc_gray_stack_pop(gray_stack_t *stack);
diff --git a/rpython/translator/c/src/qcgc/hugeblocktable.c b/rpython/translator/c/src/qcgc/hugeblocktable.c
new file mode 100644
--- /dev/null
+++ b/rpython/translator/c/src/qcgc/hugeblocktable.c
@@ -0,0 +1,80 @@
+#include "hugeblocktable.h"
+
+#include <assert.h>
+
+#include "gc_state.h"
+
+QCGC_STATIC size_t bucket(object_t *object);
+
+void qcgc_hbtable_initialize(void) {
+ qcgc_hbtable.mark_flag_ref = false;
+ for (size_t i = 0; i < QCGC_HBTABLE_BUCKETS; i++) {
+ qcgc_hbtable.bucket[i] = qcgc_hbbucket_create(4);
+ }
+}
+
+void qcgc_hbtable_destroy(void) {
+ for (size_t i = 0; i < QCGC_HBTABLE_BUCKETS; i++) {
+ free(qcgc_hbtable.bucket[i]);
+ }
+}
+
+void qcgc_hbtable_insert(object_t *object) {
+ size_t i = bucket(object);
+ qcgc_hbtable.bucket[i] = qcgc_hbbucket_add(qcgc_hbtable.bucket[i],
+ (struct hbtable_entry_s) {
+ .object = object,
+ .mark_flag = !qcgc_hbtable.mark_flag_ref});
+}
+
+bool qcgc_hbtable_mark(object_t *object) {
+ hbbucket_t *b = qcgc_hbtable.bucket[bucket(object)];
+ size_t count = b->count;
+ for (size_t i = 0; i < count; i++) {
+ if (b->items[i].object == object) {
+ if (b->items[i].mark_flag != qcgc_hbtable.mark_flag_ref) {
+ b->items[i].mark_flag = qcgc_hbtable.mark_flag_ref;
+ return true;
+ }
+ return false;
+ }
+ }
+#if CHECKED
+ assert(false);
+#endif
+ return false;
+}
+
+bool qcgc_hbtable_is_marked(object_t *object) {
+ hbbucket_t *b = qcgc_hbtable.bucket[bucket(object)];
+ size_t count = b->count;
+ for (size_t i = 0; i < count; i++) {
+ if (b->items[i].object == object) {
+ return b->items[i].mark_flag == qcgc_hbtable.mark_flag_ref;
+ }
+ }
+ return false;
+}
+
+void qcgc_hbtable_sweep(void) {
+ for (size_t i = 0; i < QCGC_HBTABLE_BUCKETS; i++) {
+ hbbucket_t *b = qcgc_hbtable.bucket[i];
+ size_t j = 0;
+ while(j < b->count) {
+ if (b->items[j].mark_flag != qcgc_hbtable.mark_flag_ref) {
+ // White object
+ free(b->items[j].object);
+ b = qcgc_hbbucket_remove_index(b, j);
+ } else {
+ // Black object
+ j++;
+ }
+ }
+ qcgc_hbtable.bucket[i] = b;
+ }
+ qcgc_hbtable.mark_flag_ref = !qcgc_hbtable.mark_flag_ref;
+}
+
+QCGC_STATIC size_t bucket(object_t *object) {
+ return ((uintptr_t) object >> (QCGC_ARENA_SIZE_EXP)) % QCGC_HBTABLE_BUCKETS;
+}
diff --git a/rpython/translator/c/src/qcgc/hugeblocktable.h b/rpython/translator/c/src/qcgc/hugeblocktable.h
new file mode 100644
--- /dev/null
+++ b/rpython/translator/c/src/qcgc/hugeblocktable.h
@@ -0,0 +1,24 @@
+#pragma once
+
+#include "config.h"
+
+#include <stdbool.h>
+
+#include "bag.h"
+#include "object.h"
+#include "gray_stack.h"
+
+// Choosing a prime number, hoping for good results
+#define QCGC_HBTABLE_BUCKETS 61
+
+struct hbtable_s {
+ bool mark_flag_ref;
+ hbbucket_t *bucket[QCGC_HBTABLE_BUCKETS];
+} qcgc_hbtable;
+
+void qcgc_hbtable_initialize(void);
+void qcgc_hbtable_destroy(void);
+void qcgc_hbtable_insert(object_t *object);
+bool qcgc_hbtable_mark(object_t *object);
+bool qcgc_hbtable_is_marked(object_t *object);
+void qcgc_hbtable_sweep(void);
diff --git a/rpython/translator/c/src/qcgc/mark_list.c b/rpython/translator/c/src/qcgc/mark_list.c
deleted file mode 100644
--- a/rpython/translator/c/src/qcgc/mark_list.c
+++ /dev/null
@@ -1,180 +0,0 @@
-#include "mark_list.h"
-
-#include <assert.h>
-
-#include <stdlib.h>
-#include <string.h>
-
-QCGC_STATIC mark_list_t *qcgc_mark_list_grow(mark_list_t *list);
-QCGC_STATIC void qcgc_mark_list_check_invariant(mark_list_t *list);
-
-mark_list_t *qcgc_mark_list_create(size_t initial_size) {
- size_t length = (initial_size + QCGC_MARK_LIST_SEGMENT_SIZE - 1) / QCGC_MARK_LIST_SEGMENT_SIZE;
- length += (size_t) length == 0;
- mark_list_t *result = (mark_list_t *)
- malloc(sizeof(mark_list_t) + length * sizeof(object_t **));
- result->head = 0;
- result->tail = 0;
- result->length = length;
- result->insert_index = 0;
- result->count = 0;
- result->segments[result->head] = (object_t **)
- calloc(QCGC_MARK_LIST_SEGMENT_SIZE, sizeof(object_t *));
-#if CHECKED
- qcgc_mark_list_check_invariant(result);
-#endif
- return result;
-}
-
-void qcgc_mark_list_destroy(mark_list_t *list) {
-#if CHECKED
- qcgc_mark_list_check_invariant(list);
-#endif
-
- size_t i = list->head;
- while (i != list->tail) {
- free(list->segments[i]);
- i = (i + 1) % list->length;
- }
- free(list->segments[list->tail]);
- free(list);
-}
-
-mark_list_t *qcgc_mark_list_push(mark_list_t *list, object_t *object) {
-#if CHECKED
- assert(list != NULL);
- assert(object != NULL);
-
- qcgc_mark_list_check_invariant(list);
- size_t old_count = list->count;
-#endif
- if (list->insert_index >= QCGC_MARK_LIST_SEGMENT_SIZE) {
- if ((list->tail + 1) % list->length == list->head) {
- list = qcgc_mark_list_grow(list);
- }
- list->insert_index = 0;
- list->tail = (list->tail + 1) % list->length;
- list->segments[list->tail] = (object_t **)
- calloc(QCGC_MARK_LIST_SEGMENT_SIZE, sizeof(object_t *));
- }
- list->segments[list->tail][list->insert_index] = object;
- list->insert_index++;
- list->count++;
-#if CHECKED
- assert(list->count == old_count + 1);
- assert(list->segments[list->tail][list->insert_index - 1] == object);
- qcgc_mark_list_check_invariant(list);
-#endif
- return list;
-}
-
-mark_list_t *qcgc_mark_list_push_all(mark_list_t *list,
- object_t **objects, size_t count) {
-#if CHECKED
- assert(list != NULL);
- assert(objects != NULL);
-
- qcgc_mark_list_check_invariant(list);
-
- size_t old_count = list->count;
- for (size_t i = 0; i < count; i++) {
- assert(objects[i] != NULL);
- }
-#endif
- // FIXME: Optimize or remove
- for (size_t i = 0; i < count; i++) {
- list = qcgc_mark_list_push(list, objects[i]);
- }
-#if CHECKED
- assert(list->count == old_count + count);
- qcgc_mark_list_check_invariant(list);
-#endif
- return list;
-}
-
-object_t **qcgc_mark_list_get_head_segment(mark_list_t *list) {
-#if CHECKED
- assert(list != NULL);
- assert(list->segments[list->head] != NULL);
- qcgc_mark_list_check_invariant(list);
-#endif
- return list->segments[list->head];
-}
-
-mark_list_t *qcgc_mark_list_drop_head_segment(mark_list_t *list) {
-#if CHECKED
- assert(list != NULL);
- size_t old_head = list->head;
- size_t old_tail = list->tail;
- qcgc_mark_list_check_invariant(list);
-#endif
- if (list->head != list->tail) {
- free(list->segments[list->head]);
- list->segments[list->head] = NULL;
- list->head = (list->head + 1) % list->length;
- list->count -= QCGC_MARK_LIST_SEGMENT_SIZE;
- } else {
- memset(list->segments[list->head], 0,
- sizeof(object_t *) * QCGC_MARK_LIST_SEGMENT_SIZE);
- list->insert_index = 0;
- list->count = 0;
- }
-#if CHECKED
- assert(old_tail == list->tail);
- if (old_head == old_tail) {
- assert(old_head == list->head);
- } else {
- assert((old_head + 1) % list->length == list->head);
- }
- qcgc_mark_list_check_invariant(list);
-#endif
- return list;
-}
-
-QCGC_STATIC mark_list_t *qcgc_mark_list_grow(mark_list_t *list) {
-#if CHECKED
- assert(list != NULL);
- size_t old_length = list->length;
- size_t old_tail = list->tail;
- qcgc_mark_list_check_invariant(list);
-#endif
- mark_list_t *new_list = (mark_list_t *) realloc(list,
- sizeof(mark_list_t) + 2 * list->length * sizeof(object_t **));
- if (new_list->tail < new_list->head) {
- memcpy(new_list->segments + new_list->length,
- new_list->segments, (new_list->tail + 1) * sizeof(object_t **));
- new_list->tail = new_list->length + new_list->tail;
- }
- new_list->length = 2 * new_list->length;
-#if CHECKED
- assert(new_list->length == 2 * old_length);
- if (old_tail < new_list->head) {
- assert(new_list->tail == old_tail + old_length);
- for (size_t i = 0; i < old_tail; i++) {
- assert(new_list->segments[i] == new_list->segments[i + old_length]);
- }
- } else {
- assert(new_list->tail == old_tail);
- }
- qcgc_mark_list_check_invariant(new_list);
-#endif
- return new_list;
-}
-
-QCGC_STATIC void qcgc_mark_list_check_invariant(mark_list_t *list) {
- assert(list->head < list->length);
- assert(list->tail < list->length);
- assert(list->count == (list->tail - list->head + list->length) % list->length * QCGC_MARK_LIST_SEGMENT_SIZE + list->insert_index);
- for (size_t i = 0; i < list->length; i++) {
- if ((list->head <= i && i <= list->tail) || (list->tail < list->head &&
- (i <= list->tail || i >= list->head))) {
- for (size_t j = 0; j < QCGC_MARK_LIST_SEGMENT_SIZE; j++) {
- if (i != list->tail || j < list->insert_index) {
- assert(list->segments[i][j] != NULL);
- } else {
- assert(list->segments[i][j] == NULL);
- }
- }
- }
- }
-}
diff --git a/rpython/translator/c/src/qcgc/mark_list.h b/rpython/translator/c/src/qcgc/mark_list.h
deleted file mode 100644
--- a/rpython/translator/c/src/qcgc/mark_list.h
+++ /dev/null
@@ -1,35 +0,0 @@
-/**
- * @file mark_list.h
- *
- * Object list for marking step
- */
-
-#pragma once
-
-#include "config.h"
-
-#include <stddef.h>
-
-#include "object.h"
-
-/**
- * Mark list - circular buffer.
- */
-typedef struct mark_list_s {
- size_t head;
- size_t tail;
- size_t length;
- size_t count;
- size_t insert_index;
- object_t **segments[];
-} mark_list_t;
-
-mark_list_t *qcgc_mark_list_create(size_t initial_size);
-void qcgc_mark_list_destroy(mark_list_t *list);
-
-mark_list_t *qcgc_mark_list_push(mark_list_t *list, object_t *object);
-mark_list_t *qcgc_mark_list_push_all(mark_list_t *list,
- object_t **objects, size_t count);
-
-object_t **qcgc_mark_list_get_head_segment(mark_list_t *list);
-mark_list_t *qcgc_mark_list_drop_head_segment(mark_list_t *list);
diff --git a/rpython/translator/c/src/qcgc/object.h b/rpython/translator/c/src/qcgc/object.h
--- a/rpython/translator/c/src/qcgc/object.h
+++ b/rpython/translator/c/src/qcgc/object.h
@@ -4,8 +4,9 @@
#include "config.h"
#include <stdint.h>
-#define QCGC_GRAY_FLAG 0x01
-#define QCGC_PREBUILT_OBJECT 0x02
+#define QCGC_GRAY_FLAG (1<<0)
+#define QCGC_PREBUILT_OBJECT (1<<1)
+#define QCGC_PREBUILT_REGISTERED (1<<2)
typedef struct object_s {
uint32_t flags;
diff --git a/rpython/translator/c/src/qcgc/qcgc.c b/rpython/translator/c/src/qcgc/qcgc.c
--- a/rpython/translator/c/src/qcgc/qcgc.c
+++ b/rpython/translator/c/src/qcgc/qcgc.c
@@ -7,15 +7,14 @@
#include <string.h>
#include "allocator.h"
+#include "hugeblocktable.h"
#include "event_logger.h"
// TODO: Eventually move to own header?
#define MAX(a,b) (((a)>(b))?(a):(b))
#define MIN(a,b) (((a)<(b))?(a):(b))
-void qcgc_mark(void);
-void qcgc_mark_all(void);
-void qcgc_mark_incremental(void);
+void qcgc_mark(bool incremental);
void qcgc_pop_object(object_t *object);
void qcgc_push_object(object_t *object);
void qcgc_sweep(void);
@@ -23,22 +22,34 @@
void qcgc_initialize(void) {
qcgc_state.shadow_stack = qcgc_shadow_stack_create(QCGC_SHADOWSTACK_SIZE);
qcgc_state.prebuilt_objects = qcgc_shadow_stack_create(16); //XXX
+ qcgc_state.gp_gray_stack = qcgc_gray_stack_create(16); // XXX
qcgc_state.gray_stack_size = 0;
qcgc_state.phase = GC_PAUSE;
qcgc_allocator_initialize();
+ qcgc_hbtable_initialize();
qcgc_event_logger_initialize();
}
void qcgc_destroy(void) {
qcgc_event_logger_destroy();
+ qcgc_hbtable_destroy();
qcgc_allocator_destroy();
free(qcgc_state.shadow_stack);
+ free(qcgc_state.prebuilt_objects);
+ free(qcgc_state.gp_gray_stack);
}
/**
* Shadow stack
*/
void qcgc_shadowstack_push(object_t *object) {
+#if CHECKED
+ assert((object->flags & QCGC_PREBUILT_OBJECT) == 0);
+#endif
+ if (qcgc_state.phase != GC_PAUSE) {
+ qcgc_state.phase = GC_MARK;
+ qcgc_push_object(object);
+ }
qcgc_state.shadow_stack =
qcgc_shadow_stack_push(qcgc_state.shadow_stack, object);
}
@@ -56,18 +67,45 @@
#if CHECKED
assert(object != NULL);
#endif
- if ((object->flags & QCGC_GRAY_FLAG) == 0) {
- object->flags |= QCGC_GRAY_FLAG;
- if ((object->flags & QCGC_PREBUILT_OBJECT) != 0) {
- // Save prebuilt object into list
- qcgc_shadow_stack_push(qcgc_state.prebuilt_objects, object);
- } else if (qcgc_state.phase != GC_PAUSE) {
- if (qcgc_arena_get_blocktype((cell_t *) object) == BLOCK_BLACK) {
- // This was black before, push it to gray stack again
- arena_t *arena = qcgc_arena_addr((cell_t *) object);
- arena->gray_stack = qcgc_gray_stack_push(
- arena->gray_stack, object);
- }
+ if ((object->flags & QCGC_GRAY_FLAG) != 0) {
+ // Already gray, skip
+ return;
+ }
+ object->flags |= QCGC_GRAY_FLAG;
+
+ // Register prebuilt object if necessary
+ if (((object->flags & QCGC_PREBUILT_OBJECT) != 0) &&
+ ((object->flags & QCGC_PREBUILT_REGISTERED) == 0)) {
+ object->flags |= QCGC_PREBUILT_REGISTERED;
+ qcgc_state.prebuilt_objects = qcgc_shadow_stack_push(
+ qcgc_state.prebuilt_objects, object);
+ }
+
+ if (qcgc_state.phase == GC_PAUSE) {
+ return; // We are done
+ }
+
+ // Triggered barrier, we must not collect now
+ qcgc_state.phase = GC_MARK;
+
+ // Test reachability of object and push if neccessary
+ if ((object->flags & QCGC_PREBUILT_OBJECT) != 0) {
+ // NOTE: No mark test here, as prebuilt objects are always reachable
+ // Push prebuilt object to general purpose gray stack
+ qcgc_state.gp_gray_stack = qcgc_gray_stack_push(
+ qcgc_state.gp_gray_stack, object);
+ } else if ((object_t *) qcgc_arena_addr((cell_t *) object) == object) {
+ if (qcgc_hbtable_is_marked(object)) {
+ // Push huge block to general purpose gray stack
+ qcgc_state.gp_gray_stack = qcgc_gray_stack_push(
+ qcgc_state.gp_gray_stack, object);
+ }
+ } else {
+ if (qcgc_arena_get_blocktype((cell_t *) object) == BLOCK_BLACK) {
+ // This was black before, push it to gray stack again
+ arena_t *arena = qcgc_arena_addr((cell_t *) object);
+ arena->gray_stack = qcgc_gray_stack_push(
+ arena->gray_stack, object);
}
}
}
@@ -81,14 +119,31 @@
qcgc_event_logger_log(EVENT_ALLOCATE_START, sizeof(size_t),
(uint8_t *) &size);
#endif
+ object_t *result;
+ if (size <= QCGC_LARGE_ALLOC_THRESHOLD) {
+ // Use bump / fit allocator
+ if (true) { // FIXME: Implement reasonable switch
+ result = qcgc_bump_allocate(size);
+ } else {
+ result = qcgc_fit_allocate(size);
- object_t *result = (object_t *) qcgc_allocator_allocate(size);
- result->flags |= QCGC_GRAY_FLAG;
+ // Fallback to bump allocator
+ if (result == NULL) {
+ result = qcgc_bump_allocate(size);
+ }
+ }
+ } else {
+ // Use huge block allocator
+ result = qcgc_large_allocate(size);
+ }
#if LOG_ALLOCATION
qcgc_event_logger_log(EVENT_ALLOCATE_DONE, sizeof(object_t *),
(uint8_t *) &result);
#endif
+#if CHECKED
+ assert(qcgc_state.phase != GC_COLLECT);
+#endif
return result;
}
@@ -121,78 +176,69 @@
}
}
-void qcgc_mark(void) {
- qcgc_mark_all();
-}
-
-void qcgc_mark_all(void) {
+void qcgc_mark(bool incremental) {
#if CHECKED
assert(qcgc_state.phase == GC_PAUSE || qcgc_state.phase == GC_MARK);
#endif
+ // FIXME: Log some more information
qcgc_event_logger_log(EVENT_MARK_START, 0, NULL);
- qcgc_state.phase = GC_MARK;
+ if (qcgc_state.phase == GC_PAUSE) {
+ qcgc_state.phase = GC_MARK;
- // Push all roots
- for (size_t i = 0; i < qcgc_state.shadow_stack->count; i++) {
- qcgc_push_object(qcgc_state.shadow_stack->items[i]);
+ // If we do this for the first time, push all roots.
+ // All further changes to the roots (new additions) will be added
+ // by qcgc_shadowstack_push
+ for (size_t i = 0; i < qcgc_state.shadow_stack->count; i++) {
+ qcgc_push_object(qcgc_state.shadow_stack->items[i]);
+ }
+
+ // If we do this for the first time, push all prebuilt objects.
+ // All further changes to prebuilt objects will go to the gp_gray_stack
+ // because of the write barrier
+ size_t count = qcgc_state.prebuilt_objects->count;
+ for (size_t i = 0; i < count; i++) {
+ qcgc_state.gp_gray_stack = qcgc_gray_stack_push(
+ qcgc_state.gp_gray_stack,
+ qcgc_state.prebuilt_objects->items[i]);
+ }
}
- // Trace all prebuilt objects
- for (size_t i = 0; i < qcgc_state.prebuilt_objects->count; i++) {
- qcgc_trace_cb(qcgc_state.prebuilt_objects->items[i], &qcgc_push_object);
- }
+ while (qcgc_state.gray_stack_size > 0) {
+ // General purpose gray stack (prebuilt objects and huge blocks)
+ size_t to_process = (incremental ?
+ MIN(qcgc_state.gp_gray_stack->index,
+ MAX(qcgc_state.gp_gray_stack->index / 2, QCGC_INC_MARK_MIN)) :
+ (qcgc_state.gp_gray_stack->index));
- while(qcgc_state.gray_stack_size > 0) {
+ while (to_process > 0) {
+ object_t *top = qcgc_gray_stack_top(qcgc_state.gp_gray_stack);
+ qcgc_state.gp_gray_stack =
+ qcgc_gray_stack_pop(qcgc_state.gp_gray_stack);
+ qcgc_pop_object(top);
+ to_process--;
+ }
+
+ // Arena gray stacks
for (size_t i = 0; i < qcgc_allocator_state.arenas->count; i++) {
arena_t *arena = qcgc_allocator_state.arenas->items[i];
- while (arena->gray_stack->index > 0) {
+ to_process = (incremental ?
+ MIN(arena->gray_stack->index,
+ MAX(arena->gray_stack->index / 2, QCGC_INC_MARK_MIN)) :
+ (arena->gray_stack->index));
+
+ while (to_process > 0) {
object_t *top =
qcgc_gray_stack_top(arena->gray_stack);
arena->gray_stack =
qcgc_gray_stack_pop(arena->gray_stack);
qcgc_pop_object(top);
+ to_process--;
}
}
- }
- qcgc_state.phase = GC_COLLECT;
-
- qcgc_event_logger_log(EVENT_MARK_DONE, 0, NULL);
-}
-
-void qcgc_mark_incremental(void) {
-#if CHECKED
- assert(qcgc_state.phase == GC_PAUSE || qcgc_state.phase == GC_MARK);
-#endif
- unsigned long gray_stack_size = qcgc_state.gray_stack_size;
- qcgc_event_logger_log(EVENT_INCMARK_START, sizeof(gray_stack_size),
- (uint8_t *) &gray_stack_size);
-
- qcgc_state.phase = GC_MARK;
-
- // Push all roots
- for (size_t i = 0; i < qcgc_state.shadow_stack->count; i++) {
- qcgc_push_object(qcgc_state.shadow_stack->items[i]);
- }
-
- // Trace all prebuilt objects
- for (size_t i = 0; i < qcgc_state.prebuilt_objects->count; i++) {
- qcgc_trace_cb(qcgc_state.prebuilt_objects->items[i], &qcgc_push_object);
- }
-
- for (size_t i = 0; i < qcgc_allocator_state.arenas->count; i++) {
- arena_t *arena = qcgc_allocator_state.arenas->items[i];
- size_t initial_stack_size = arena->gray_stack->index;
- size_t to_process = MIN(arena->gray_stack->index,
- MAX(initial_stack_size / 2, QCGC_INC_MARK_MIN));
- while (to_process > 0) {
- object_t *top =
- qcgc_gray_stack_top(arena->gray_stack);
- arena->gray_stack =
- qcgc_gray_stack_pop(arena->gray_stack);
- qcgc_pop_object(top);
- to_process--;
+ if (incremental) {
+ break; // Execute loop once for incremental collection
}
}
@@ -200,21 +246,29 @@
qcgc_state.phase = GC_COLLECT;
}
- gray_stack_size = qcgc_state.gray_stack_size;
- qcgc_event_logger_log(EVENT_INCMARK_START, sizeof(gray_stack_size),
- (uint8_t *) &gray_stack_size);
+ // FIXME: Log some more information
+ qcgc_event_logger_log(EVENT_MARK_DONE, 0, NULL);
+#if CHECKED
+ assert(incremental || (qcgc_state.phase = GC_COLLECT));
+#endif
}
void qcgc_pop_object(object_t *object) {
#if CHECKED
assert(object != NULL);
assert((object->flags & QCGC_GRAY_FLAG) == QCGC_GRAY_FLAG);
- assert(qcgc_arena_get_blocktype((cell_t *) object) == BLOCK_BLACK);
+ if (((object->flags & QCGC_PREBUILT_OBJECT) == 0) &&
+ ((object_t *) qcgc_arena_addr((cell_t *) object) != object)) {
+ assert(qcgc_arena_get_blocktype((cell_t *) object) == BLOCK_BLACK);
+ }
#endif
object->flags &= ~QCGC_GRAY_FLAG;
qcgc_trace_cb(object, &qcgc_push_object);
#if CHECKED
- assert(qcgc_get_mark_color(object) == MARK_COLOR_BLACK);
+ if (((object->flags & QCGC_PREBUILT_OBJECT) == 0) &&
+ ((object_t *) qcgc_arena_addr((cell_t *) object) != object)) {
+ assert(qcgc_get_mark_color(object) == MARK_COLOR_BLACK);
+ }
#endif
}
@@ -224,8 +278,17 @@
assert(qcgc_state.phase == GC_MARK);
#endif
if (object != NULL) {
+ if ((object_t *) qcgc_arena_addr((cell_t *) object) == object) {
+ if (qcgc_hbtable_mark(object)) {
+ // Did mark it / was white before
+ object->flags |= QCGC_GRAY_FLAG;
+ qcgc_state.gp_gray_stack = qcgc_gray_stack_push(
+ qcgc_state.gp_gray_stack, object);
+ }
+ return; // Skip tests
+ }
if ((object->flags & QCGC_PREBUILT_OBJECT) != 0) {
- return;
+ return; // Prebuilt objects are always black, no pushing here
}
if (qcgc_arena_get_blocktype((cell_t *) object) == BLOCK_WHITE) {
object->flags |= QCGC_GRAY_FLAG;
@@ -258,6 +321,7 @@
qcgc_event_logger_log(EVENT_SWEEP_START, sizeof(arena_count),
(uint8_t *) &arena_count);
+ qcgc_hbtable_sweep();
for (size_t i = 0; i < qcgc_allocator_state.arenas->count; i++) {
qcgc_arena_sweep(qcgc_allocator_state.arenas->items[i]);
}
@@ -267,6 +331,6 @@
}
void qcgc_collect(void) {
- qcgc_mark();
+ qcgc_mark(false);
qcgc_sweep();
}
diff --git a/rpython/translator/c/src/qcgc/shadow_stack.h b/rpython/translator/c/src/qcgc/shadow_stack.h
--- a/rpython/translator/c/src/qcgc/shadow_stack.h
+++ b/rpython/translator/c/src/qcgc/shadow_stack.h
@@ -12,8 +12,13 @@
object_t *items[];
} shadow_stack_t;
+__attribute__ ((warn_unused_result))
shadow_stack_t *qcgc_shadow_stack_create(size_t size);
+__attribute__ ((warn_unused_result))
shadow_stack_t *qcgc_shadow_stack_push(shadow_stack_t *stack, object_t *item);
+
object_t *qcgc_shadow_stack_top(shadow_stack_t *stack);
+
+__attribute__ ((warn_unused_result))
shadow_stack_t *qcgc_shadow_stack_pop(shadow_stack_t *stack);
More information about the pypy-commit
mailing list