SgenMajorCollector major_collector;
SgenMinorCollector sgen_minor_collector;
static GrayQueue gray_queue;
+static GrayQueue remember_major_objects_gray_queue;
static SgenRemeberedSet remset;
/* The gray queue to use from the main collection thread. */
-static SgenGrayQueue*
-sgen_workers_get_main_thread_queue (void)
-{
- if (sgen_collection_is_parallel () || sgen_collection_is_concurrent ())
- return sgen_workers_get_distribute_gray_queue ();
- return &gray_queue;
-}
-
-#define WORKERS_DISTRIBUTE_GRAY_QUEUE (sgen_workers_get_main_thread_queue ())
+#define WORKERS_DISTRIBUTE_GRAY_QUEUE (&gray_queue)
/*
* The gray queue a worker job must use. If we're not parallel or
return worker_data ? &worker_data->private_gray_queue : WORKERS_DISTRIBUTE_GRAY_QUEUE;
}
-static LOCK_DECLARE (workers_distribute_gray_queue_mutex);
-
gboolean
sgen_remember_major_object_for_concurrent_mark (char *obj)
{
- gboolean need_lock = current_collection_generation != GENERATION_NURSERY;
-
if (!major_collector.is_concurrent)
return FALSE;
if (!concurrent_collection_in_progress)
return FALSE;
- if (need_lock)
- mono_mutex_lock (&workers_distribute_gray_queue_mutex);
-
- sgen_gray_object_enqueue (sgen_workers_get_distribute_gray_queue (), obj);
-
- if (need_lock)
- mono_mutex_unlock (&workers_distribute_gray_queue_mutex);
+ GRAY_OBJECT_ENQUEUE (&remember_major_objects_gray_queue, obj);
return TRUE;
}
}
}
+static void
+gray_queue_redirect (SgenGrayQueue *queue)
+{
+ gboolean wake = FALSE;
+
+
+ for (;;) {
+ GrayQueueSection *section = sgen_gray_object_dequeue_section (queue);
+ if (!section)
+ break;
+ sgen_section_gray_queue_enqueue (queue->alloc_prepare_data, section);
+ wake = TRUE;
+ }
+
+ if (wake) {
+ g_assert (concurrent_collection_in_progress);
+ sgen_workers_wake_up_all ();
+ }
+}
+
static void
init_gray_queue (void)
{
if (sgen_collection_is_parallel () || sgen_collection_is_concurrent ()) {
- sgen_gray_object_queue_init_invalid (&gray_queue);
sgen_workers_init_distribute_gray_queue ();
+ sgen_gray_object_queue_init_with_alloc_prepare (&gray_queue, NULL,
+ gray_queue_redirect, sgen_workers_get_distribute_section_gray_queue ());
} else {
- sgen_gray_object_queue_init (&gray_queue, NULL, FALSE);
+ sgen_gray_object_queue_init (&gray_queue, NULL);
+ }
+
+ if (major_collector.is_concurrent) {
+ sgen_gray_object_queue_init_with_alloc_prepare (&remember_major_objects_gray_queue, NULL,
+ gray_queue_redirect, sgen_workers_get_distribute_section_gray_queue ());
+ } else {
+ sgen_gray_object_queue_init_invalid (&remember_major_objects_gray_queue);
}
}
time_minor_scan_thread_data += TV_ELAPSED (btv, atv);
btv = atv;
- if (sgen_collection_is_parallel () || sgen_collection_is_concurrent ()) {
- while (!sgen_gray_object_queue_is_empty (WORKERS_DISTRIBUTE_GRAY_QUEUE)) {
- sgen_workers_distribute_gray_queue_sections ();
- g_usleep (1000);
- }
- }
- sgen_workers_join ();
+ g_assert (!sgen_collection_is_parallel () && !sgen_collection_is_concurrent ());
if (sgen_collection_is_parallel () || sgen_collection_is_concurrent ())
g_assert (sgen_gray_object_queue_is_empty (&gray_queue));
static void
wait_for_workers_to_finish (void)
{
+ g_assert (sgen_gray_object_queue_is_empty (&remember_major_objects_gray_queue));
+
if (major_collector.is_parallel || major_collector.is_concurrent) {
- while (!sgen_gray_object_queue_is_empty (WORKERS_DISTRIBUTE_GRAY_QUEUE)) {
- sgen_workers_distribute_gray_queue_sections ();
- g_usleep (1000);
- }
+ gray_queue_redirect (&gray_queue);
+ sgen_workers_join ();
}
- sgen_workers_join ();
- if (major_collector.is_parallel || major_collector.is_concurrent)
- g_assert (sgen_gray_object_queue_is_empty (&gray_queue));
+ g_assert (sgen_gray_object_queue_is_empty (&gray_queue));
#ifdef SGEN_DEBUG_INTERNAL_ALLOC
main_gc_thread = NULL;
major_copy_or_mark_from_roots (NULL, TRUE, scan_mod_union);
wait_for_workers_to_finish ();
+ g_assert (sgen_gray_object_queue_is_empty (&gray_queue));
+
if (do_concurrent_checks)
check_nursery_is_clean ();
}
+ g_assert (sgen_section_gray_queue_is_empty (sgen_workers_get_distribute_section_gray_queue ()));
+
/* all the objects in the heap */
finish_gray_stack (heap_start, heap_end, GENERATION_OLD, &gray_queue);
TV_GETTIME (atv);
// FIXME: store reason and pass it when finishing
major_start_collection (NULL);
- sgen_workers_distribute_gray_queue_sections ();
- g_assert (sgen_gray_object_queue_is_empty (sgen_workers_get_distribute_gray_queue ()));
-
+ gray_queue_redirect (&gray_queue);
sgen_workers_wait_for_jobs ();
MONO_GC_CONCURRENT_START_END (GENERATION_OLD);
}
static gboolean
-major_update_or_finish_concurrent_collection (void)
+major_update_or_finish_concurrent_collection (gboolean force_finish)
{
MONO_GC_CONCURRENT_UPDATE_FINISH_BEGIN (GENERATION_OLD);
major_collector.update_cardtable_mod_union ();
sgen_los_update_cardtable_mod_union ();
- if (!sgen_workers_all_done ()) {
+ g_assert (sgen_gray_object_queue_is_empty (&gray_queue));
+ g_assert (sgen_gray_object_queue_is_empty (&remember_major_objects_gray_queue));
+
+ if (!force_finish && !sgen_workers_all_done ()) {
MONO_GC_CONCURRENT_UPDATE_END (GENERATION_OLD);
g_print ("workers not done\n");
}
collect_nursery ();
+ gray_queue_redirect (&remember_major_objects_gray_queue);
current_collection_generation = GENERATION_OLD;
major_finish_collection ("finishing", -1, TRUE);
sgen_stop_world (generation_to_collect);
if (concurrent_collection_in_progress) {
- g_assert (generation_to_collect == GENERATION_NURSERY);
g_print ("finishing concurrent collection\n");
- if (major_update_or_finish_concurrent_collection ())
+ if (major_update_or_finish_concurrent_collection (generation_to_collect == GENERATION_OLD))
goto done;
}
overflow_generation_to_collect = GENERATION_OLD;
overflow_reason = "Minor overflow";
}
- if (concurrent_collection_in_progress)
+ if (concurrent_collection_in_progress) {
+ gray_queue_redirect (&remember_major_objects_gray_queue);
sgen_workers_wake_up_all ();
+ }
} else {
- if (major_collector.is_concurrent)
+ if (major_collector.is_concurrent) {
+ g_assert (!concurrent_collection_in_progress);
collect_nursery ();
+ }
if (major_collector.is_concurrent && !wait_to_finish) {
major_start_concurrent_collection (reason);
}
done:
+ g_assert (sgen_gray_object_queue_is_empty (&gray_queue));
+ g_assert (sgen_gray_object_queue_is_empty (&remember_major_objects_gray_queue));
+
sgen_restart_world (generation_to_collect, infos);
mono_profiler_gc_event (MONO_GC_EVENT_END, generation_to_collect);
if (minor_collector_opt)
g_free (minor_collector_opt);
- if (major_collector.is_concurrent)
- LOCK_INIT (workers_distribute_gray_queue_mutex);
-
alloc_nursery ();
if ((env = getenv ("MONO_GC_DEBUG"))) {
#define GRAY_QUEUE_LENGTH_LIMIT 64
-static void
-lock_queue (SgenGrayQueue *queue)
-{
- if (!queue->locked)
- return;
-
- mono_mutex_lock (&queue->lock);
-}
-
-static void
-unlock_queue (SgenGrayQueue *queue)
-{
- if (!queue->locked)
- return;
-
- mono_mutex_unlock (&queue->lock);
-}
-
void
sgen_gray_object_alloc_queue_section (SgenGrayQueue *queue)
{
GrayQueueSection *section;
- lock_queue (queue);
-
if (queue->alloc_prepare_func)
queue->alloc_prepare_func (queue);
/* Link it with the others */
section->next = queue->first;
queue->first = section;
-
- unlock_queue (queue);
}
void
SGEN_ASSERT (9, obj, "enqueueing a null object");
//sgen_check_objref (obj);
- lock_queue (queue);
-
#ifdef SGEN_CHECK_GRAY_OBJECT_ENQUEUE
if (queue->enqueue_check_func)
- queue->enqueue_check_func (queue, obj);
+ queue->enqueue_check_func (obj);
#endif
if (G_UNLIKELY (!queue->first || queue->first->end == SGEN_GRAY_QUEUE_SECTION_SIZE))
sgen_gray_object_alloc_queue_section (queue);
SGEN_ASSERT (9, queue->first->end < SGEN_GRAY_QUEUE_SECTION_SIZE, "gray queue %p overflow, first %p, end %d", queue, queue->first, queue->first->end);
queue->first->objects [queue->first->end++] = obj;
-
- unlock_queue (queue);
}
char*
if (sgen_gray_object_queue_is_empty (queue))
return NULL;
- if (queue->locked) {
- lock_queue (queue);
- if (sgen_gray_object_queue_is_empty (queue)) {
- unlock_queue (queue);
- return NULL;
- }
- }
-
SGEN_ASSERT (9, queue->first->end, "gray queue %p underflow, first %p, end %d", queue, queue->first, queue->first->end);
obj = queue->first->objects [--queue->first->end];
queue->free_list = section;
}
- unlock_queue (queue);
-
return obj;
}
if (!queue->first)
return NULL;
- if (queue->locked) {
- lock_queue (queue);
- if (!queue->first) {
- unlock_queue (queue);
- return NULL;
- }
- }
-
section = queue->first;
queue->first = section->next;
section->next = NULL;
- unlock_queue (queue);
-
return section;
}
void
sgen_gray_object_enqueue_section (SgenGrayQueue *queue, GrayQueueSection *section)
{
- lock_queue (queue);
-
section->next = queue->first;
queue->first = section;
#ifdef SGEN_CHECK_GRAY_OBJECT_ENQUEUE
if (queue->enqueue_check_func) {
int i;
for (i = 0; i < section->end; ++i)
- queue->enqueue_check_func (queue, section->objects [i]);
+ queue->enqueue_check_func (section->objects [i]);
}
#endif
-
- unlock_queue (queue);
}
void
-sgen_gray_object_queue_init (SgenGrayQueue *queue, GrayQueueEnqueueCheckFunc enqueue_check_func, gboolean locked)
+sgen_gray_object_queue_init (SgenGrayQueue *queue, GrayQueueEnqueueCheckFunc enqueue_check_func)
{
GrayQueueSection *section, *next;
int i;
queue->enqueue_check_func = enqueue_check_func;
#endif
- queue->locked = locked;
- if (locked) {
- mono_mutexattr_t attr;
- mono_mutexattr_init (&attr);
- mono_mutexattr_settype (&attr, MONO_MUTEX_RECURSIVE);
- mono_mutex_init (&queue->lock, &attr);
- }
-
/* Free the extra sections allocated during the last collection */
i = 0;
for (section = queue->free_list; section && i < GRAY_QUEUE_LENGTH_LIMIT - 1; section = section->next)
void
sgen_gray_object_queue_init_invalid (SgenGrayQueue *queue)
{
- sgen_gray_object_queue_init (queue, NULL, FALSE);
+ sgen_gray_object_queue_init (queue, FALSE);
queue->alloc_prepare_func = invalid_prepare_func;
queue->alloc_prepare_data = NULL;
}
void
sgen_gray_object_queue_init_with_alloc_prepare (SgenGrayQueue *queue, GrayQueueEnqueueCheckFunc enqueue_check_func,
- gboolean locked,
GrayQueueAllocPrepareFunc alloc_prepare_func, void *data)
{
- sgen_gray_object_queue_init (queue, enqueue_check_func, locked);
+ sgen_gray_object_queue_init (queue, enqueue_check_func);
queue->alloc_prepare_func = alloc_prepare_func;
queue->alloc_prepare_data = data;
}
+static void
+lock_section_queue (SgenSectionGrayQueue *queue)
+{
+ if (!queue->locked)
+ return;
+
+ mono_mutex_lock (&queue->lock);
+}
+
+static void
+unlock_section_queue (SgenSectionGrayQueue *queue)
+{
+ if (!queue->locked)
+ return;
+
+ mono_mutex_unlock (&queue->lock);
+}
+
+void
+sgen_section_gray_queue_init (SgenSectionGrayQueue *queue, gboolean locked, GrayQueueEnqueueCheckFunc enqueue_check_func)
+{
+ g_assert (sgen_section_gray_queue_is_empty (queue));
+
+ queue->locked = locked;
+ if (locked) {
+ mono_mutexattr_t attr;
+ mono_mutexattr_init (&attr);
+ mono_mutexattr_settype (&attr, MONO_MUTEX_RECURSIVE);
+ mono_mutex_init (&queue->lock, &attr);
+ }
+
+#ifdef SGEN_CHECK_GRAY_OBJECT_ENQUEUE
+ queue->enqueue_check_func = enqueue_check_func;
+#endif
+}
+
+gboolean
+sgen_section_gray_queue_is_empty (SgenSectionGrayQueue *queue)
+{
+ return !queue->first;
+}
+
+GrayQueueSection*
+sgen_section_gray_queue_dequeue (SgenSectionGrayQueue *queue)
+{
+ GrayQueueSection *section;
+
+ lock_section_queue (queue);
+
+ if (queue->first) {
+ section = queue->first;
+ queue->first = section->next;
+
+ section->next = NULL;
+ } else {
+ section = NULL;
+ }
+
+ unlock_section_queue (queue);
+
+ return section;
+}
+
+void
+sgen_section_gray_queue_enqueue (SgenSectionGrayQueue *queue, GrayQueueSection *section)
+{
+ lock_section_queue (queue);
+
+ section->next = queue->first;
+ queue->first = section;
+#ifdef SGEN_CHECK_GRAY_OBJECT_ENQUEUE
+ if (queue->enqueue_check_func) {
+ int i;
+ for (i = 0; i < section->end; ++i)
+ queue->enqueue_check_func (section->objects [i]);
+ }
+#endif
+
+ unlock_section_queue (queue);
+}
+
#endif
typedef struct _SgenGrayQueue SgenGrayQueue;
typedef void (*GrayQueueAllocPrepareFunc) (SgenGrayQueue*);
-typedef void (*GrayQueueEnqueueCheckFunc) (SgenGrayQueue*, char*);
+typedef void (*GrayQueueEnqueueCheckFunc) (char*);
struct _SgenGrayQueue {
GrayQueueSection *first;
GrayQueueEnqueueCheckFunc enqueue_check_func;
#endif
void *alloc_prepare_data;
+};
+
+typedef struct _SgenSectionGrayQueue SgenSectionGrayQueue;
+
+struct _SgenSectionGrayQueue {
+ GrayQueueSection *first;
gboolean locked;
mono_mutex_t lock;
+#ifdef SGEN_CHECK_GRAY_OBJECT_ENQUEUE
+ GrayQueueEnqueueCheckFunc enqueue_check_func;
+#endif
};
void sgen_gray_object_enqueue (SgenGrayQueue *queue, char *obj) MONO_INTERNAL;
char* sgen_gray_object_dequeue (SgenGrayQueue *queue) MONO_INTERNAL;
GrayQueueSection* sgen_gray_object_dequeue_section (SgenGrayQueue *queue) MONO_INTERNAL;
void sgen_gray_object_enqueue_section (SgenGrayQueue *queue, GrayQueueSection *section) MONO_INTERNAL;
-void sgen_gray_object_queue_init (SgenGrayQueue *queue, GrayQueueEnqueueCheckFunc enqueue_check_func, gboolean locked) MONO_INTERNAL;
+void sgen_gray_object_queue_init (SgenGrayQueue *queue, GrayQueueEnqueueCheckFunc enqueue_check_func) MONO_INTERNAL;
void sgen_gray_object_queue_init_invalid (SgenGrayQueue *queue) MONO_INTERNAL;
void sgen_gray_object_queue_init_with_alloc_prepare (SgenGrayQueue *queue, GrayQueueEnqueueCheckFunc enqueue_check_func,
- gboolean locked,
GrayQueueAllocPrepareFunc func, void *data) MONO_INTERNAL;
void sgen_gray_object_alloc_queue_section (SgenGrayQueue *queue) MONO_INTERNAL;
void sgen_gray_object_free_queue_section (GrayQueueSection *section) MONO_INTERNAL;
+void sgen_section_gray_queue_init (SgenSectionGrayQueue *queue, gboolean locked,
+ GrayQueueEnqueueCheckFunc enqueue_check_func) MONO_INTERNAL;
+gboolean sgen_section_gray_queue_is_empty (SgenSectionGrayQueue *queue) MONO_INTERNAL;
+GrayQueueSection* sgen_section_gray_queue_dequeue (SgenSectionGrayQueue *queue) MONO_INTERNAL;
+void sgen_section_gray_queue_enqueue (SgenSectionGrayQueue *queue, GrayQueueSection *section) MONO_INTERNAL;
+
static inline gboolean
sgen_gray_object_queue_is_empty (SgenGrayQueue *queue)
{
static WorkerData *workers_data;
static WorkerData workers_gc_thread_data;
-static SgenGrayQueue workers_distribute_gray_queue;
+static SgenSectionGrayQueue workers_distribute_gray_queue;
static gboolean workers_distribute_gray_queue_inited;
static volatile gboolean workers_gc_in_progress = FALSE;
*/
major = sgen_get_major_collector ();
if (major->is_concurrent || major->is_parallel) {
- GrayQueueSection *section = sgen_gray_object_dequeue_section (&workers_distribute_gray_queue);
+ GrayQueueSection *section = sgen_section_gray_queue_dequeue (&workers_distribute_gray_queue);
if (section) {
sgen_gray_object_enqueue_section (&data->private_gray_queue, section);
return TRUE;
}
static void
-concurrent_enqueue_check (SgenGrayQueue *queue, char *obj)
+concurrent_enqueue_check (char *obj)
{
g_assert (!sgen_ptr_in_nursery (obj));
g_assert (SGEN_LOAD_VTABLE (obj));
{
sgen_gray_object_queue_init_with_alloc_prepare (&data->private_gray_queue,
sgen_get_major_collector ()->is_concurrent ? concurrent_enqueue_check : NULL,
- FALSE,
workers_gray_queue_share_redirect, data);
}
return NULL;
}
-void
-sgen_workers_distribute_gray_queue_sections (void)
-{
- if (!collection_needs_workers ())
- return;
-
- workers_gray_queue_share_redirect (&workers_distribute_gray_queue);
-}
-
static void
init_distribute_gray_queue (gboolean locked)
{
if (workers_distribute_gray_queue_inited) {
- g_assert (sgen_gray_object_queue_is_empty (&workers_distribute_gray_queue));
+ g_assert (sgen_section_gray_queue_is_empty (&workers_distribute_gray_queue));
g_assert (!workers_distribute_gray_queue.locked == !locked);
return;
}
- sgen_gray_object_queue_init_with_alloc_prepare (&workers_distribute_gray_queue,
- sgen_get_major_collector ()->is_concurrent ? concurrent_enqueue_check : NULL,
- locked,
- workers_gray_queue_share_redirect, &workers_gc_thread_data);
+ sgen_section_gray_queue_init (&workers_distribute_gray_queue, locked,
+ sgen_get_major_collector ()->is_concurrent ? concurrent_enqueue_check : NULL);
workers_distribute_gray_queue_inited = TRUE;
}
return;
g_assert (sgen_gray_object_queue_is_empty (&workers_gc_thread_data.private_gray_queue));
- g_assert (sgen_gray_object_queue_is_empty (&workers_distribute_gray_queue));
g_assert (workers_gc_in_progress);
workers_gc_in_progress = FALSE;
g_assert (workers_done_posted);
+ g_assert (sgen_section_gray_queue_is_empty (&workers_distribute_gray_queue));
g_assert (!workers_gc_thread_data.stealable_stack_fill);
g_assert (sgen_gray_object_queue_is_empty (&workers_gc_thread_data.private_gray_queue));
for (i = 0; i < workers_num; ++i) {
return FALSE;
}
-gboolean
-sgen_workers_is_distributed_queue (SgenGrayQueue *queue)
-{
- return queue == &workers_distribute_gray_queue;
-}
-
-SgenGrayQueue*
-sgen_workers_get_distribute_gray_queue (void)
+SgenSectionGrayQueue*
+sgen_workers_get_distribute_section_gray_queue (void)
{
return &workers_distribute_gray_queue;
}
void sgen_workers_reset_data (void) MONO_INTERNAL;
void sgen_workers_join (void) MONO_INTERNAL;
gboolean sgen_workers_all_done (void) MONO_INTERNAL;
-gboolean sgen_workers_is_distributed_queue (SgenGrayQueue *queue) MONO_INTERNAL;
-SgenGrayQueue* sgen_workers_get_distribute_gray_queue (void) MONO_INTERNAL;
+SgenSectionGrayQueue* sgen_workers_get_distribute_section_gray_queue (void) MONO_INTERNAL;
#endif
projects / mono.git / commitdiff