When the concurrent mark ends, we have another concurrent phase where we scan the cardtable trying to clean as many cards as possible. The way we achieve this is by first copying all the mod union cards (for each block/los_obj) to a separate temporary card_table. When scanning we will only scan these cards. Therefore we will first preclean these cards in the actual mod_union card_tables, followed by a full memory barrier (which assures us that, when scanning, we will observe the changes that led to the corresponding card marks) and then by a normal mod_union scan, which can remark some of these cards, without any races with the preclean phase.
#define ARRAY_OBJ_INDEX(ptr,array,elem_size) (((char*)(ptr) - ((char*)(array) + G_STRUCT_OFFSET (MonoArray, vector))) / (elem_size))
gboolean
-sgen_client_cardtable_scan_object (GCObject *obj, mword block_obj_size, guint8 *cards, gboolean mod_union, ScanCopyContext ctx)
+sgen_client_cardtable_scan_object (GCObject *obj, mword block_obj_size, guint8 *cards, ScanCopyContext ctx)
{
MonoVTable *vt = SGEN_LOAD_VTABLE (obj);
MonoClass *klass = vt->klass;
*out_num_cards = num_cards;
}
+/* Preclean cards and saves the cards that need to be scanned afterwards in cards_preclean */
+void
+sgen_card_table_preclean_mod_union (guint8 *cards, guint8 *cards_preclean, size_t num_cards)
+{
+ size_t i;
+
+ memcpy (cards_preclean, cards, num_cards);
+ for (i = 0; i < num_cards; i++) {
+ if (cards_preclean [i]) {
+ cards [i] = 0;
+ }
+ }
+ /*
+ * When precleaning we need to make sure the card cleaning
+ * takes place before the object is scanned. If we don't
+ * do this we could finish scanning the object and, before
+ * the cleaning of the card takes place, another thread
+ * could dirty the object, mark the mod_union card only for
+ * us to clean it back, without scanning the object again.
+ */
+ mono_memory_barrier ();
+}
+
#ifdef SGEN_HAVE_OVERLAPPING_CARDS
static void
sgen_card_table_clear_cards ();
#endif
SGEN_TV_GETTIME (atv);
- sgen_get_major_collector ()->scan_card_table (FALSE, ctx);
+ sgen_get_major_collector ()->scan_card_table (CARDTABLE_SCAN_GLOBAL, ctx);
SGEN_TV_GETTIME (btv);
last_major_scan_time = SGEN_TV_ELAPSED (atv, btv);
major_card_scan_time += last_major_scan_time;
- sgen_los_scan_card_table (FALSE, ctx);
+ sgen_los_scan_card_table (CARDTABLE_SCAN_GLOBAL, ctx);
SGEN_TV_GETTIME (atv);
last_los_scan_time = SGEN_TV_ELAPSED (btv, atv);
los_card_scan_time += last_los_scan_time;
#endif
void
-sgen_cardtable_scan_object (GCObject *obj, mword block_obj_size, guint8 *cards, gboolean mod_union, ScanCopyContext ctx)
+sgen_cardtable_scan_object (GCObject *obj, mword block_obj_size, guint8 *cards, ScanCopyContext ctx)
{
HEAVY_STAT (++large_objects);
- if (sgen_client_cardtable_scan_object (obj, block_obj_size, cards, mod_union, ctx))
+ if (sgen_client_cardtable_scan_object (obj, block_obj_size, cards, ctx))
return;
HEAVY_STAT (++bloby_objects);
void* sgen_card_table_align_pointer (void *ptr);
void sgen_card_table_mark_range (mword address, mword size);
void sgen_cardtable_scan_object (GCObject *obj, mword obj_size, guint8 *cards,
- gboolean mod_union, ScanCopyContext ctx);
+ ScanCopyContext ctx);
gboolean sgen_card_table_get_card_data (guint8 *dest, mword address, mword cards);
void sgen_card_table_update_mod_union_from_cards (guint8 *dest, guint8 *start_card, size_t num_cards);
void sgen_card_table_update_mod_union (guint8 *dest, char *obj, mword obj_size, size_t *out_num_cards);
+void sgen_card_table_preclean_mod_union (guint8 *cards, guint8 *cards_preclean, size_t num_cards);
guint8* sgen_get_card_table_configuration (int *shift_bits, gpointer *mask);
* parts of the object based on which cards are marked, do so and return TRUE. Otherwise,
* return FALSE.
*/
-gboolean sgen_client_cardtable_scan_object (GCObject *obj, mword block_obj_size, guint8 *cards, gboolean mod_union, ScanCopyContext ctx);
+gboolean sgen_client_cardtable_scan_object (GCObject *obj, mword block_obj_size, guint8 *cards, ScanCopyContext ctx);
/*
* Called after nursery objects have been pinned. No action is necessary.
ScanCopyContext ctx = CONTEXT_FROM_OBJECT_OPERATIONS (job_data->ops, sgen_workers_get_job_gray_queue (worker_data));
g_assert (concurrent_collection_in_progress);
- major_collector.scan_card_table (TRUE, ctx);
+ major_collector.scan_card_table (CARDTABLE_SCAN_MOD_UNION, ctx);
}
static void
ScanCopyContext ctx = CONTEXT_FROM_OBJECT_OPERATIONS (job_data->ops, sgen_workers_get_job_gray_queue (worker_data));
g_assert (concurrent_collection_in_progress);
- sgen_los_scan_card_table (TRUE, ctx);
+ sgen_los_scan_card_table (CARDTABLE_SCAN_MOD_UNION, ctx);
+}
+
+static void
+job_mod_union_preclean (void *worker_data_untyped, SgenThreadPoolJob *job)
+{
+ WorkerData *worker_data = (WorkerData *)worker_data_untyped;
+ ScanJob *job_data = (ScanJob*)job;
+ ScanCopyContext ctx = CONTEXT_FROM_OBJECT_OPERATIONS (job_data->ops, sgen_workers_get_job_gray_queue (worker_data));
+
+ g_assert (concurrent_collection_in_progress);
+
+ major_collector.scan_card_table (CARDTABLE_SCAN_MOD_UNION_PRECLEAN, ctx);
+ sgen_los_scan_card_table (CARDTABLE_SCAN_MOD_UNION_PRECLEAN, ctx);
}
static void
* collector we start the workers after pinning.
*/
if (mode == COPY_OR_MARK_FROM_ROOTS_START_CONCURRENT) {
- sgen_workers_start_all_workers (object_ops);
+ ScanJob *sj;
+ /* Mod union preclean job */
+ sj = (ScanJob*)sgen_thread_pool_job_alloc ("preclean mod union cardtable", job_mod_union_preclean, sizeof (ScanJob));
+ sj->ops = object_ops;
+ sgen_workers_start_all_workers (object_ops, &sj->job);
gray_queue_enable_redirect (WORKERS_DISTRIBUTE_GRAY_QUEUE);
} else if (mode == COPY_OR_MARK_FROM_ROOTS_FINISH_CONCURRENT) {
if (sgen_workers_have_idle_work ()) {
- sgen_workers_start_all_workers (object_ops);
+ sgen_workers_start_all_workers (object_ops, NULL);
sgen_workers_join ();
}
}
size_t num_unique_scanned_objects;
} ScannedObjectCounts;
+typedef enum {
+ CARDTABLE_SCAN_GLOBAL = 0,
+ CARDTABLE_SCAN_MOD_UNION = 1,
+ CARDTABLE_SCAN_MOD_UNION_PRECLEAN = CARDTABLE_SCAN_MOD_UNION | 2,
+} CardTableScanType;
+
typedef struct _SgenMajorCollector SgenMajorCollector;
struct _SgenMajorCollector {
size_t section_size;
void (*free_non_pinned_object) (GCObject *obj, size_t size);
void (*pin_objects) (SgenGrayQueue *queue);
void (*pin_major_object) (GCObject *obj, SgenGrayQueue *queue);
- void (*scan_card_table) (gboolean mod_union, ScanCopyContext ctx);
+ void (*scan_card_table) (CardTableScanType scan_type, ScanCopyContext ctx);
void (*iterate_live_block_ranges) (sgen_cardtable_block_callback callback);
void (*update_cardtable_mod_union) (void);
void (*init_to_space) (void);
gboolean sgen_ptr_is_in_los (char *ptr, char **start);
void sgen_los_iterate_objects (IterateObjectCallbackFunc cb, void *user_data);
void sgen_los_iterate_live_block_ranges (sgen_cardtable_block_callback callback);
-void sgen_los_scan_card_table (gboolean mod_union, ScanCopyContext ctx);
+void sgen_los_scan_card_table (CardTableScanType scan_type, ScanCopyContext ctx);
void sgen_los_update_cardtable_mod_union (void);
void sgen_los_count_cards (long long *num_total_cards, long long *num_marked_cards);
gboolean sgen_los_is_valid_object (char *object);
}
void
-sgen_los_scan_card_table (gboolean mod_union, ScanCopyContext ctx)
+sgen_los_scan_card_table (CardTableScanType scan_type, ScanCopyContext ctx)
{
LOSObject *obj;
- binary_protocol_los_card_table_scan_start (sgen_timestamp (), mod_union);
+ binary_protocol_los_card_table_scan_start (sgen_timestamp (), scan_type & CARDTABLE_SCAN_MOD_UNION);
for (obj = los_object_list; obj; obj = obj->next) {
+ mword num_cards = 0;
guint8 *cards;
if (!SGEN_OBJECT_HAS_REFERENCES (obj->data))
continue;
- if (mod_union) {
+ if (scan_type & CARDTABLE_SCAN_MOD_UNION) {
if (!sgen_los_object_is_pinned (obj->data))
continue;
cards = get_cardtable_mod_union_for_object (obj);
g_assert (cards);
+ if (scan_type == CARDTABLE_SCAN_MOD_UNION_PRECLEAN) {
+ guint8 *cards_preclean;
+ mword obj_size = sgen_los_object_size (obj);
+ num_cards = sgen_card_table_number_of_cards_in_range ((mword) obj->data, obj_size);
+ cards_preclean = (guint8 *)sgen_alloc_internal_dynamic (num_cards, INTERNAL_MEM_CARDTABLE_MOD_UNION, TRUE);
+
+ sgen_card_table_preclean_mod_union (cards, cards_preclean, num_cards);
+
+ cards = cards_preclean;
+ }
} else {
cards = NULL;
}
- sgen_cardtable_scan_object (obj->data, sgen_los_object_size (obj), cards, mod_union, ctx);
+ sgen_cardtable_scan_object (obj->data, sgen_los_object_size (obj), cards, ctx);
+
+ if (scan_type == CARDTABLE_SCAN_MOD_UNION_PRECLEAN)
+ sgen_free_internal_dynamic (cards, num_cards, INTERNAL_MEM_CARDTABLE_MOD_UNION);
}
- binary_protocol_los_card_table_scan_end (sgen_timestamp (), mod_union);
+ binary_protocol_los_card_table_scan_end (sgen_timestamp (), scan_type & CARDTABLE_SCAN_MOD_UNION);
}
void
#include "mono/sgen/sgen-workers.h"
#include "mono/sgen/sgen-thread-pool.h"
#include "mono/sgen/sgen-client.h"
-#include "mono/utils/mono-membar.h"
+#include "mono/utils/mono-memory-model.h"
#if defined(ARCH_MIN_MS_BLOCK_SIZE) && defined(ARCH_MIN_MS_BLOCK_SIZE_SHIFT)
#define MS_BLOCK_SIZE ARCH_MIN_MS_BLOCK_SIZE
} else {
sgen_los_mark_mod_union_card (obj, ptr);
}
-
binary_protocol_mod_union_remset (obj, ptr, value_obj, SGEN_LOAD_VTABLE (value_obj));
}
#define COPY_OR_MARK_CONCURRENT_WITH_EVACUATION
#define COPY_OR_MARK_FUNCTION_NAME major_copy_or_mark_object_concurrent_with_evacuation
#define SCAN_OBJECT_FUNCTION_NAME major_scan_object_concurrent_with_evacuation
+#define SCAN_VTYPE_FUNCTION_NAME major_scan_vtype_concurrent_with_evacuation
+#define SCAN_PTR_FIELD_FUNCTION_NAME major_scan_ptr_field_concurrent_with_evacuation
#define DRAIN_GRAY_STACK_FUNCTION_NAME drain_gray_stack_concurrent_with_evacuation
#include "sgen-marksweep-drain-gray-stack.h"
#define MS_OBJ_ALLOCED_FAST(o,b) (*(void**)(o) && (*(char**)(o) < (b) || *(char**)(o) >= (b) + MS_BLOCK_SIZE))
static void
-scan_card_table_for_block (MSBlockInfo *block, gboolean mod_union, ScanCopyContext ctx)
+scan_card_table_for_block (MSBlockInfo *block, CardTableScanType scan_type, ScanCopyContext ctx)
{
SgenGrayQueue *queue = ctx.queue;
ScanObjectFunc scan_func = ctx.ops->scan_object;
#ifndef SGEN_HAVE_OVERLAPPING_CARDS
guint8 cards_copy [CARDS_PER_BLOCK];
#endif
+ guint8 cards_preclean [CARDS_PER_BLOCK];
gboolean small_objects;
int block_obj_size;
char *block_start;
guint8 *card_data_end;
char *scan_front = NULL;
+ /* The concurrent mark doesn't enter evacuating blocks */
+ if (scan_type == CARDTABLE_SCAN_MOD_UNION_PRECLEAN && major_block_is_evacuating (block))
+ return;
+
block_obj_size = block->obj_size;
small_objects = block_obj_size < CARD_SIZE_IN_BYTES;
* Cards aliasing happens in powers of two, so as long as major blocks are aligned to their
* sizes, they won't overflow the cardtable overlap modulus.
*/
- if (mod_union) {
+ if (scan_type & CARDTABLE_SCAN_MOD_UNION) {
card_data = card_base = block->cardtable_mod_union;
/*
* This happens when the nursery collection that precedes finishing
*/
if (!card_data)
return;
+
+ if (scan_type == CARDTABLE_SCAN_MOD_UNION_PRECLEAN) {
+ sgen_card_table_preclean_mod_union (card_data, cards_preclean, CARDS_PER_BLOCK);
+ card_data = card_base = cards_preclean;
+ }
} else {
#ifdef SGEN_HAVE_OVERLAPPING_CARDS
card_data = card_base = sgen_card_table_get_card_scan_address ((mword)block_start);
if (obj < scan_front || !MS_OBJ_ALLOCED_FAST (obj, block_start))
goto next_object;
- if (mod_union) {
+ if (scan_type & CARDTABLE_SCAN_MOD_UNION) {
/* FIXME: do this more efficiently */
int w, b;
MS_CALC_MARK_BIT (w, b, obj);
scan_func (object, sgen_obj_get_descriptor (object), queue);
} else {
size_t offset = sgen_card_table_get_card_offset (obj, block_start);
- sgen_cardtable_scan_object (object, block_obj_size, card_base + offset, mod_union, ctx);
+ sgen_cardtable_scan_object (object, block_obj_size, card_base + offset, ctx);
}
next_object:
obj += block_obj_size;
}
static void
-major_scan_card_table (gboolean mod_union, ScanCopyContext ctx)
+major_scan_card_table (CardTableScanType scan_type, ScanCopyContext ctx)
{
MSBlockInfo *block;
gboolean has_references;
if (!concurrent_mark)
- g_assert (!mod_union);
+ g_assert (scan_type == CARDTABLE_SCAN_GLOBAL);
major_finish_sweep_checking ();
- binary_protocol_major_card_table_scan_start (sgen_timestamp (), mod_union);
+ binary_protocol_major_card_table_scan_start (sgen_timestamp (), scan_type & CARDTABLE_SCAN_MOD_UNION);
FOREACH_BLOCK_HAS_REFERENCES_NO_LOCK (block, has_references) {
#ifdef PREFETCH_CARDS
int prefetch_index = __index + 6;
if (prefetch_index < allocated_blocks.next_slot) {
MSBlockInfo *prefetch_block = BLOCK_UNTAG (*sgen_array_list_get_slot (&allocated_blocks, prefetch_index));
PREFETCH_READ (prefetch_block);
- if (!mod_union) {
+ if (scan_type == CARDTABLE_SCAN_GLOBAL) {
guint8 *prefetch_cards = sgen_card_table_get_card_scan_address ((mword)MS_BLOCK_FOR_BLOCK_INFO (prefetch_block));
PREFETCH_WRITE (prefetch_cards);
PREFETCH_WRITE (prefetch_cards + 32);
if (!has_references)
continue;
- scan_card_table_for_block (block, mod_union, ctx);
+ scan_card_table_for_block (block, scan_type, ctx);
} END_FOREACH_BLOCK_NO_LOCK;
- binary_protocol_major_card_table_scan_end (sgen_timestamp (), mod_union);
+ binary_protocol_major_card_table_scan_end (sgen_timestamp (), scan_type & CARDTABLE_SCAN_MOD_UNION);
}
static void
if (is_concurrent) {
collector->major_ops_concurrent_start.copy_or_mark_object = major_copy_or_mark_object_concurrent_canonical;
collector->major_ops_concurrent_start.scan_object = major_scan_object_concurrent_with_evacuation;
+ collector->major_ops_concurrent_start.scan_vtype = major_scan_vtype_concurrent_with_evacuation;
+ collector->major_ops_concurrent_start.scan_ptr_field = major_scan_ptr_field_concurrent_with_evacuation;
collector->major_ops_concurrent_start.drain_gray_stack = drain_gray_stack_concurrent;
collector->major_ops_concurrent_finish.copy_or_mark_object = major_copy_or_mark_object_concurrent_finish_canonical;
static volatile State workers_state;
static SgenObjectOperations * volatile idle_func_object_ops;
+static SgenThreadPoolJob * volatile preclean_job;
static guint64 stat_workers_num_finished;
sgen_drain_gray_stack (ctx);
} else {
- worker_try_finish ();
+ SgenThreadPoolJob *job = preclean_job;
+ if (job) {
+ sgen_thread_pool_job_enqueue (job);
+ preclean_job = NULL;
+ } else {
+ worker_try_finish ();
+ }
}
}
void
sgen_workers_stop_all_workers (void)
{
+ preclean_job = NULL;
+ mono_memory_write_barrier ();
forced_stop = TRUE;
sgen_thread_pool_wait_for_all_jobs ();
}
void
-sgen_workers_start_all_workers (SgenObjectOperations *object_ops)
+sgen_workers_start_all_workers (SgenObjectOperations *object_ops, SgenThreadPoolJob *job)
{
forced_stop = FALSE;
idle_func_object_ops = object_ops;
+ preclean_job = job;
mono_memory_write_barrier ();
sgen_workers_ensure_awake ();
void sgen_workers_init (int num_workers);
void sgen_workers_stop_all_workers (void);
-void sgen_workers_start_all_workers (SgenObjectOperations *object_ops);
+void sgen_workers_start_all_workers (SgenObjectOperations *object_ops, SgenThreadPoolJob *finish_job);
void sgen_workers_ensure_awake (void);
void sgen_workers_init_distribute_gray_queue (void);
void sgen_workers_enqueue_job (SgenThreadPoolJob *job, gboolean enqueue);
projects / mono.git / commitdiff