void **free_list;
MSBlockInfo *next_free;
void **pin_queue_start;
+#ifdef SGEN_CONCURRENT_MARK
+ guint8 *cardtable_mod_union;
+#endif
mword mark_words [MS_NUM_MARK_WORDS];
};
header = (MSBlockHeader*) info->block;
header->info = info;
#endif
+#ifdef SGEN_CONCURRENT_MARK
+ info->cardtable_mod_union = NULL;
+#endif
update_heap_boundaries_for_block (info);
}
static void*
-alloc_obj_par (int size, gboolean pinned, gboolean has_references)
+alloc_obj_par (MonoVTable *vtable, int size, gboolean pinned, gboolean has_references)
{
int size_index = MS_BLOCK_OBJ_SIZE_INDEX (size);
MSBlockInfo **free_blocks_local = FREE_BLOCKS_LOCAL (pinned, has_references);
}
}
- /*
- * FIXME: This should not be necessary because it'll be
- * overwritten by the vtable immediately.
- */
- *(void**)obj = NULL;
+ *(MonoVTable**)obj = vtable;
+
+#ifdef SGEN_CONCURRENT_MARK
+ g_assert_not_reached ();
+#endif
return obj;
}
static void*
-major_par_alloc_object (int size, gboolean has_references)
+major_par_alloc_object (MonoVTable *vtable, int size, gboolean has_references)
{
- return alloc_obj_par (size, FALSE, has_references);
+ return alloc_obj_par (vtable, size, FALSE, has_references);
}
#endif
static void*
-alloc_obj (int size, gboolean pinned, gboolean has_references)
+alloc_obj (MonoVTable *vtable, int size, gboolean pinned, gboolean has_references)
{
int size_index = MS_BLOCK_OBJ_SIZE_INDEX (size);
MSBlockInfo **free_blocks = FREE_BLOCKS (pinned, has_references);
obj = unlink_slot_from_free_list_uncontested (free_blocks, size_index);
- /*
- * FIXME: This should not be necessary because it'll be
- * overwritten by the vtable immediately.
- */
- *(void**)obj = NULL;
+ *(MonoVTable**)obj = vtable;
+
+#ifdef SGEN_CONCURRENT_MARK
+ if (obj && sgen_remember_major_object_for_concurrent_mark (obj)) {
+ MSBlockInfo *block = MS_BLOCK_FOR_OBJ (obj);
+ int word, bit;
+ MS_CALC_MARK_BIT (word, bit, obj);
+ MS_SET_MARK_BIT (block, word, bit);
+ binary_protocol_mark (obj, NULL, size);
+ }
+#endif
return obj;
}
static void*
-major_alloc_object (int size, gboolean has_references)
+major_alloc_object (MonoVTable *vtable, int size, gboolean has_references)
{
- return alloc_obj (size, FALSE, has_references);
+ return alloc_obj (vtable, size, FALSE, has_references);
}
/*
/* size is a multiple of SGEN_ALLOC_ALIGN */
static void*
-major_alloc_small_pinned_obj (size_t size, gboolean has_references)
+major_alloc_small_pinned_obj (MonoVTable *vtable, size_t size, gboolean has_references)
{
void *res;
ms_wait_for_sweep_done ();
- res = alloc_obj (size, TRUE, has_references);
+ res = alloc_obj (vtable, size, TRUE, has_references);
/*If we failed to alloc memory, we better try releasing memory
*as pinned alloc is requested by the runtime.
*/
if (!res) {
- sgen_perform_collection (0, GENERATION_OLD, "pinned alloc failure");
- res = alloc_obj (size, TRUE, has_references);
+ sgen_perform_collection (0, GENERATION_OLD, "pinned alloc failure", TRUE);
+ res = alloc_obj (vtable, size, TRUE, has_references);
}
return res;
}
old_num_sections = num_major_sections;
- obj = alloc_obj (size, FALSE, SGEN_VTABLE_HAS_REFERENCES (vtable));
+ obj = alloc_obj (vtable, size, FALSE, SGEN_VTABLE_HAS_REFERENCES (vtable));
if (G_LIKELY (obj)) {
- *(MonoVTable**)obj = vtable;
HEAVY_STAT (++stat_objects_alloced_degraded);
HEAVY_STAT (stat_bytes_alloced_degraded += size);
g_assert (num_major_sections >= old_num_sections);
static void
pin_major_object (char *obj, SgenGrayQueue *queue)
{
+#ifdef SGEN_CONCURRENT_MARK
+ g_assert_not_reached ();
+#else
MSBlockInfo *block = MS_BLOCK_FOR_OBJ (obj);
block->has_pinned = TRUE;
MS_MARK_OBJECT_AND_ENQUEUE (obj, block, queue);
+#endif
}
#include "sgen-major-copy-object.h"
objsize = SGEN_ALIGN_UP (sgen_par_object_get_size (vt, (MonoObject*)obj));
has_references = SGEN_VTABLE_HAS_REFERENCES (vt);
- destination = sgen_minor_collector.par_alloc_for_promotion (obj, objsize, has_references);
+ destination = sgen_minor_collector.par_alloc_for_promotion (vt, obj, objsize, has_references);
if (G_UNLIKELY (!destination)) {
if (!sgen_ptr_in_nursery (obj)) {
int size_index;
return;
}
- /*
- * We do this before the CAS because we want to make
- * sure that if another thread sees the destination
- * pointer the VTable is already in place. Not doing
- * this can crash binary protocols.
- */
- *(MonoVTable**)destination = vt;
-
if (SGEN_CAS_PTR (obj, (void*)((mword)destination | SGEN_FORWARDED_BIT), vt) == vt) {
gboolean was_marked;
g_assert (!SGEN_OBJECT_IS_FORWARDED (obj));
if (sgen_ptr_in_nursery (obj)) {
- g_assert (SGEN_OBJECT_IS_PINNED (obj));
+ g_assert (!SGEN_OBJECT_IS_PINNED (obj));
} else {
#ifdef FIXED_HEAP
if (MS_PTR_IN_SMALL_MAJOR_HEAP (obj))
} else {
if (sgen_los_object_is_pinned (obj))
return;
- binary_protocol_pin (obj, (gpointer)SGEN_LOAD_VTABLE (obj), sgen_safe_object_get_size ((MonoObject*)obj));
if (G_UNLIKELY (MONO_GC_OBJ_PINNED_ENABLED ())) {
MonoVTable *vt = (MonoVTable*)SGEN_LOAD_VTABLE (obj);
MONO_GC_OBJ_PINNED ((mword)obj, sgen_safe_object_get_size (obj), vt->klass->name_space, vt->klass->name, GENERATION_OLD);
count = MS_BLOCK_FREE / block->obj_size;
+#ifdef SGEN_CONCURRENT_MARK
+ if (block->cardtable_mod_union) {
+ sgen_free_internal_dynamic (block->cardtable_mod_union, CARDS_PER_BLOCK, INTERNAL_MEM_CARDTABLE_MOD_UNION);
+ block->cardtable_mod_union = NULL;
+ }
+#endif
+
/* Count marked objects in the block */
for (i = 0; i < MS_NUM_MARK_WORDS; ++i) {
nused += bitcount (block->mark_words [i]);
if (lazy_sweep) {
MSBlockInfo **iter;
+ MONO_GC_SWEEP_BEGIN (GENERATION_OLD, TRUE);
+
iter = &all_blocks;
while (*iter) {
MSBlockInfo *block = *iter;
iter = &block->next;
}
+
+ MONO_GC_SWEEP_END (GENERATION_OLD, TRUE);
}
}
#define MS_OBJ_ALLOCED_FAST(o,b) (*(void**)(o) && (*(char**)(o) < (b) || *(char**)(o) >= (b) + MS_BLOCK_SIZE))
static void
-major_scan_card_table (SgenGrayQueue *queue)
+major_scan_card_table (gboolean mod_union, SgenGrayQueue *queue)
{
MSBlockInfo *block;
ScanObjectFunc scan_func = sgen_get_current_object_ops ()->scan_object;
#endif
char *obj, *end, *base;
+ if (mod_union) {
+#ifdef SGEN_CONCURRENT_MARK
+ cards = block->cardtable_mod_union;
+ g_assert (cards);
+#else
+ g_assert_not_reached ();
+#endif
+ } else {
/*We can avoid the extra copy since the remark cardtable was cleaned before */
#ifdef SGEN_HAVE_OVERLAPPING_CARDS
- cards = sgen_card_table_get_card_scan_address ((mword)block_start);
+ cards = sgen_card_table_get_card_scan_address ((mword)block_start);
#else
- cards = cards_data;
- if (!sgen_card_table_get_card_data (cards_data, (mword)block_start, CARDS_PER_BLOCK))
- continue;
+ cards = cards_data;
+ if (!sgen_card_table_get_card_data (cards_data, (mword)block_start, CARDS_PER_BLOCK))
+ continue;
#endif
+ }
if (!block->swept)
sweep_block (block);
base = sgen_card_table_align_pointer (obj);
while (obj < end) {
- if (MS_OBJ_ALLOCED_FAST (obj, block_start)) {
- int card_offset = (obj - base) >> CARD_BITS;
- sgen_cardtable_scan_object (obj, block_obj_size, cards + card_offset, queue);
+ int card_offset;
+
+ if (!MS_OBJ_ALLOCED_FAST (obj, block_start))
+ goto next_large;
+
+ if (mod_union) {
+ /* FIXME: do this more efficiently */
+ int w, b;
+ MS_CALC_MARK_BIT (w, b, obj);
+ if (!MS_MARK_BIT (block, w, b))
+ goto next_large;
}
+
+ card_offset = (obj - base) >> CARD_BITS;
+ sgen_cardtable_scan_object (obj, block_obj_size, cards + card_offset, queue);
+
+ next_large:
obj += block_obj_size;
}
} else {
* Cards aliasing happens in powers of two, so as long as major blocks are aligned to their
* sizes, they won't overflow the cardtable overlap modulus.
*/
- card_data = card_base = sgen_card_table_get_card_scan_address ((mword)block_start);
+ if (mod_union) {
+#ifdef SGEN_CONCURRENT_MARK
+ card_data = card_base = block->cardtable_mod_union;
+ /*
+ * This happens when the nursery
+ * collection that precedes finishing
+ * the concurrent collection allocates
+ * new major blocks.
+ */
+ if (!card_data)
+ continue;
+#else
+ g_assert_not_reached ();
+#endif
+ } else {
+ card_data = card_base = sgen_card_table_get_card_scan_address ((mword)block_start);
+ }
card_data_end = card_data + CARDS_PER_BLOCK;
for (card_data = initial_skip_card (card_data); card_data < card_data_end; ++card_data) { //card_data = skip_card (card_data + 1, card_data_end)) {
obj = (char*)MS_BLOCK_OBJ_FAST (block_start, block_obj_size, index);
while (obj < end) {
- if (MS_OBJ_ALLOCED_FAST (obj, block_start)) {
- HEAVY_STAT (++scanned_objects);
- scan_func (obj, queue);
+ if (!MS_OBJ_ALLOCED_FAST (obj, block_start))
+ goto next_small;
+
+ if (mod_union) {
+ /* FIXME: do this more efficiently */
+ int w, b;
+ MS_CALC_MARK_BIT (w, b, obj);
+ if (!MS_MARK_BIT (block, w, b))
+ goto next_small;
}
+
+ HEAVY_STAT (++scanned_objects);
+ scan_func (obj, queue);
+ next_small:
obj += block_obj_size;
}
HEAVY_STAT (if (*card_data) ++remarked_cards);
}
} END_FOREACH_BLOCK;
}
+
+#ifdef SGEN_CONCURRENT_MARK
+static void
+update_cardtable_mod_union (void)
+{
+ MSBlockInfo *block;
+
+ FOREACH_BLOCK (block) {
+ guint8 *cards;
+ gboolean init = FALSE;
+
+ if (!block->cardtable_mod_union) {
+ block->cardtable_mod_union = sgen_alloc_internal_dynamic (CARDS_PER_BLOCK,
+ INTERNAL_MEM_CARDTABLE_MOD_UNION, TRUE);
+ init = TRUE;
+ }
+
+ cards = sgen_card_table_get_card_scan_address ((mword)block->block);
+ if (init) {
+ memcpy (block->cardtable_mod_union, cards, CARDS_PER_BLOCK);
+ } else {
+ int i;
+ for (i = 0; i < CARDS_PER_BLOCK; ++i)
+ block->cardtable_mod_union [i] |= cards [i];
+ }
+ } END_FOREACH_BLOCK;
+}
+#endif
#endif
static gboolean
#undef pthread_create
static void
-post_param_init (void)
+post_param_init (SgenMajorCollector *collector)
{
if (concurrent_sweep) {
if (!mono_native_thread_create (&ms_sweep_thread, ms_sweep_thread_func, NULL)) {
exit (1);
}
}
+
+ collector->sweeps_lazily = lazy_sweep;
}
void
#ifdef SGEN_HAVE_CARDTABLE
collector->scan_card_table = major_scan_card_table;
collector->iterate_live_block_ranges = (void*)(void*) major_iterate_live_block_ranges;
+#ifdef SGEN_CONCURRENT_MARK
+ collector->update_cardtable_mod_union = update_cardtable_mod_union;
+#endif
#endif
collector->init_to_space = major_init_to_space;
collector->sweep = major_sweep;
collector->major_ops.copy_or_mark_object = major_copy_or_mark_object;
collector->major_ops.scan_object = major_scan_object;
+#ifdef SGEN_CONCURRENT_MARK
+ collector->major_ops.scan_vtype = major_scan_vtype;
+#endif
#ifdef SGEN_HAVE_CARDTABLE
/*cardtable requires major pages to be 8 cards aligned*/
projects / mono.git / blobdiff