[sgen] Use non-synchronized gray stack when only one worker is running

diff --git a/mono/sgen/sgen-copy-object.h b/mono/sgen/sgen-copy-object.h
index 8faebc836df639657a28625e7435c0d723fcfbc5..4c3eae0f09e3db3c7736872a54640a0565f722f0 100644 (file)
--- a/mono/sgen/sgen-copy-object.h
+++ b/mono/sgen/sgen-copy-object.h
@@ -83,7 +83,7 @@ copy_object_no_checks (GCObject *obj, SgenGrayQueue *queue)
 
        if (has_references) {
                SGEN_LOG (9, "Enqueuing gray object %p (%s)", destination, sgen_client_vtable_get_name (vt));
-               GRAY_OBJECT_ENQUEUE (queue, (GCObject *)destination, sgen_vtable_get_descriptor (vt));
+               GRAY_OBJECT_ENQUEUE_SERIAL (queue, (GCObject *)destination, sgen_vtable_get_descriptor (vt));
        }
 
        return (GCObject *)destination;
@@ -120,7 +120,7 @@ copy_object_no_checks_par (GCObject *obj, SgenGrayQueue *queue)
                        /* In a racing case, only the worker that allocated the object enqueues it */
                        if (has_references) {
                                SGEN_LOG (9, "Enqueuing gray object %p (%s)", destination, sgen_client_vtable_get_name (vt));
-                               GRAY_OBJECT_ENQUEUE (queue, (GCObject *)destination, sgen_vtable_get_descriptor (vt));
+                               GRAY_OBJECT_ENQUEUE_PARALLEL (queue, (GCObject *)destination, sgen_vtable_get_descriptor (vt));
                        }
                } else {
                        destination = final_destination;

diff --git a/mono/sgen/sgen-gc.c b/mono/sgen/sgen-gc.c
index fa92d0c58b706069629a937469c595944d27b21b..799ca85bc55dbf21acfae942b0a9d5541ddcc00c 100644 (file)
--- a/mono/sgen/sgen-gc.c
+++ b/mono/sgen/sgen-gc.c
@@ -519,7 +519,7 @@ sgen_drain_gray_stack (ScanCopyContext ctx)
        for (;;) {
                GCObject *obj;
                SgenDescriptor desc;
-               GRAY_OBJECT_DEQUEUE (queue, &obj, &desc);
+               GRAY_OBJECT_DEQUEUE_PARALLEL (queue, &obj, &desc);
                if (!obj)
                        return TRUE;
                SGEN_LOG (9, "Precise gray object scan %p (%s)", obj, sgen_client_vtable_get_name (SGEN_LOAD_VTABLE (obj)));
@@ -674,7 +674,7 @@ pin_objects_from_nursery_pin_queue (gboolean do_scan_objects, ScanCopyContext ct
                                        safe_object_get_size (obj_to_pin));
 
                        pin_object (obj_to_pin);
-                       GRAY_OBJECT_ENQUEUE (queue, obj_to_pin, desc);
+                       GRAY_OBJECT_ENQUEUE_SERIAL (queue, obj_to_pin, desc);
                        sgen_pin_stats_register_object (obj_to_pin, GENERATION_NURSERY);
                        definitely_pinned [count] = obj_to_pin;
                        count++;
@@ -724,7 +724,7 @@ sgen_pin_object (GCObject *object, SgenGrayQueue *queue)
        ++objects_pinned;
        sgen_pin_stats_register_object (object, GENERATION_NURSERY);
 
-       GRAY_OBJECT_ENQUEUE (queue, object, sgen_obj_get_descriptor_safe (object));
+       GRAY_OBJECT_ENQUEUE_SERIAL (queue, object, sgen_obj_get_descriptor_safe (object));
 }
 
 /* Sort the addresses in array in increasing order.
@@ -1840,7 +1840,7 @@ major_copy_or_mark_from_roots (SgenGrayQueue *gc_thread_gray_queue, size_t *old_
                        }
                        sgen_los_pin_object (bigobj->data);
                        if (SGEN_OBJECT_HAS_REFERENCES (bigobj->data))
-                               GRAY_OBJECT_ENQUEUE (gc_thread_gray_queue, bigobj->data, sgen_obj_get_descriptor ((GCObject*)bigobj->data));
+                               GRAY_OBJECT_ENQUEUE_SERIAL (gc_thread_gray_queue, bigobj->data, sgen_obj_get_descriptor ((GCObject*)bigobj->data));
                        sgen_pin_stats_register_object (bigobj->data, GENERATION_OLD);
                        SGEN_LOG (6, "Marked large object %p (%s) size: %lu from roots", bigobj->data,
                                        sgen_client_vtable_get_name (SGEN_LOAD_VTABLE (bigobj->data)),

diff --git a/mono/sgen/sgen-gray.c b/mono/sgen/sgen-gray.c
index bb8f58c05904e77743be1c6bd7f45f598feb1d23..256b18ecbe6e86965c1aef14bbfb5796442d3399 100644 (file)
--- a/mono/sgen/sgen-gray.c
+++ b/mono/sgen/sgen-gray.c
@@ -45,7 +45,7 @@ guint64 stat_gray_queue_dequeue_slow_path;
 static GrayQueueSection *last_gray_queue_free_list;
 
 void
-sgen_gray_object_alloc_queue_section (SgenGrayQueue *queue)
+sgen_gray_object_alloc_queue_section (SgenGrayQueue *queue, gboolean is_parallel)
 {
        GrayQueueSection *section;
 
@@ -77,15 +77,19 @@ sgen_gray_object_alloc_queue_section (SgenGrayQueue *queue)
        queue->first = section;
        queue->cursor = section->entries - 1;
 
-       mono_memory_write_barrier ();
-       /*
-        * FIXME
-        * we could probably optimize the code to only rely on the write barrier
-        * for synchronization with the stealer thread. Additionally we could also
-        * do a write barrier once every other gray queue change, and request
-        * to have a minimum of sections before stealing, to keep consistency.
-        */
-       InterlockedIncrement (&queue->num_sections);
+       if (is_parallel) {
+               mono_memory_write_barrier ();
+               /*
+                * FIXME
+                * we could probably optimize the code to only rely on the write barrier
+                * for synchronization with the stealer thread. Additionally we could also
+                * do a write barrier once every other gray queue change, and request
+                * to have a minimum of sections before stealing, to keep consistency.
+                */
+               InterlockedIncrement (&queue->num_sections);
+       } else {
+               queue->num_sections++;
+       }
 }
 
 void
@@ -104,7 +108,7 @@ sgen_gray_object_free_queue_section (GrayQueueSection *section)
  */
 
 void
-sgen_gray_object_enqueue (SgenGrayQueue *queue, GCObject *obj, SgenDescriptor desc)
+sgen_gray_object_enqueue (SgenGrayQueue *queue, GCObject *obj, SgenDescriptor desc, gboolean is_parallel)
 {
        GrayQueueEntry entry = SGEN_GRAY_QUEUE_ENTRY (obj, desc);
 
@@ -128,7 +132,7 @@ sgen_gray_object_enqueue (SgenGrayQueue *queue, GCObject *obj, SgenDescriptor de
                        queue->first->size = SGEN_GRAY_QUEUE_SECTION_SIZE;
                }
 
-               sgen_gray_object_alloc_queue_section (queue);
+               sgen_gray_object_alloc_queue_section (queue, is_parallel);
        }
        STATE_ASSERT (queue->first, GRAY_QUEUE_SECTION_STATE_ENQUEUED);
        SGEN_ASSERT (9, queue->cursor <= GRAY_LAST_CURSOR_POSITION (queue->first), "gray queue %p overflow, first %p, cursor %p", queue, queue->first, queue->cursor);
@@ -172,7 +176,7 @@ sgen_gray_object_spread (SgenGrayQueue *queue, int num_sections)
 
        /* Allocate all needed sections */
        while (queue->num_sections < num_sections_final)
-               sgen_gray_object_alloc_queue_section (queue);
+               sgen_gray_object_alloc_queue_section (queue, TRUE);
 
        /* Spread out the elements in the sections. By design, sections at the end are fuller. */
        section_start = queue->first;
@@ -202,7 +206,7 @@ sgen_gray_object_spread (SgenGrayQueue *queue, int num_sections)
 }
 
 GrayQueueEntry
-sgen_gray_object_dequeue (SgenGrayQueue *queue)
+sgen_gray_object_dequeue (SgenGrayQueue *queue, gboolean is_parallel)
 {
        GrayQueueEntry entry;
 
@@ -224,11 +228,14 @@ sgen_gray_object_dequeue (SgenGrayQueue *queue)
 
        if (G_UNLIKELY (queue->cursor < GRAY_FIRST_CURSOR_POSITION (queue->first))) {
                GrayQueueSection *section;
-               gint32 old_num_sections;
+               gint32 old_num_sections = 0;
 
-               old_num_sections = InterlockedDecrement (&queue->num_sections);
+               if (is_parallel)
+                       old_num_sections = InterlockedDecrement (&queue->num_sections);
+               else
+                       queue->num_sections--;
 
-               if (old_num_sections <= 0) {
+               if (is_parallel && old_num_sections <= 0) {
                        mono_os_mutex_lock (&queue->steal_mutex);
                }
 
@@ -247,7 +254,7 @@ sgen_gray_object_dequeue (SgenGrayQueue *queue)
                queue->free_list = section;
                queue->cursor = queue->first ? queue->first->entries + queue->first->size - 1 : NULL;
 
-               if (old_num_sections <= 0) {
+               if (is_parallel && old_num_sections <= 0) {
                        mono_os_mutex_unlock (&queue->steal_mutex);
                }
        }
@@ -333,7 +340,7 @@ sgen_gray_object_steal_section (SgenGrayQueue *queue)
 }
 
 void
-sgen_gray_object_enqueue_section (SgenGrayQueue *queue, GrayQueueSection *section)
+sgen_gray_object_enqueue_section (SgenGrayQueue *queue, GrayQueueSection *section, gboolean is_parallel)
 {
        STATE_TRANSITION (section, GRAY_QUEUE_SECTION_STATE_FLOATING, GRAY_QUEUE_SECTION_STATE_ENQUEUED);
 
@@ -355,8 +362,12 @@ sgen_gray_object_enqueue_section (SgenGrayQueue *queue, GrayQueueSection *sectio
                        queue->enqueue_check_func (section->entries [i].obj);
        }
 #endif
-       mono_memory_write_barrier ();
-       InterlockedIncrement (&queue->num_sections);
+       if (is_parallel) {
+               mono_memory_write_barrier ();
+               InterlockedIncrement (&queue->num_sections);
+       } else {
+               queue->num_sections++;
+       }
 }
 
 void

diff --git a/mono/sgen/sgen-gray.h b/mono/sgen/sgen-gray.h
index a380a18bb0a916e9a71682f427d685f0064122a2..b2f69d0586d2dd296a42c0d573fbc4c78a2d677e 100644 (file)
--- a/mono/sgen/sgen-gray.h
+++ b/mono/sgen/sgen-gray.h
@@ -65,6 +65,11 @@ struct _GrayQueueEntry {
 
 #define SGEN_GRAY_QUEUE_ENTRY(obj,desc)        { (obj), (desc) }
 
+#define GRAY_OBJECT_ENQUEUE_SERIAL(queue, obj, desc) (GRAY_OBJECT_ENQUEUE (queue, obj, desc, FALSE))
+#define GRAY_OBJECT_ENQUEUE_PARALLEL(queue, obj, desc) (GRAY_OBJECT_ENQUEUE (queue, obj, desc, TRUE))
+#define GRAY_OBJECT_DEQUEUE_SERIAL(queue, obj, desc) (GRAY_OBJECT_DEQUEUE (queue, obj, desc, FALSE))
+#define GRAY_OBJECT_DEQUEUE_PARALLEL(queue, obj, desc) (GRAY_OBJECT_DEQUEUE (queue, obj, desc, TRUE))
+
 /*
  * This is a stack now instead of a queue, so the most recently added items are removed
  * first, improving cache locality, and keeping the stack size manageable.
@@ -125,17 +130,17 @@ extern guint64 stat_gray_queue_dequeue_slow_path;
 
 void sgen_init_gray_queues (void);
 
-void sgen_gray_object_enqueue (SgenGrayQueue *queue, GCObject *obj, SgenDescriptor desc);
-GrayQueueEntry sgen_gray_object_dequeue (SgenGrayQueue *queue);
+void sgen_gray_object_enqueue (SgenGrayQueue *queue, GCObject *obj, SgenDescriptor desc, gboolean is_parallel);
+GrayQueueEntry sgen_gray_object_dequeue (SgenGrayQueue *queue, gboolean is_parallel);
 GrayQueueSection* sgen_gray_object_dequeue_section (SgenGrayQueue *queue);
 GrayQueueSection* sgen_gray_object_steal_section (SgenGrayQueue *queue);
 void sgen_gray_object_spread (SgenGrayQueue *queue, int num_sections);
-void sgen_gray_object_enqueue_section (SgenGrayQueue *queue, GrayQueueSection *section);
+void sgen_gray_object_enqueue_section (SgenGrayQueue *queue, GrayQueueSection *section, gboolean is_parallel);
 void sgen_gray_object_queue_trim_free_list (SgenGrayQueue *queue);
 void sgen_gray_object_queue_init (SgenGrayQueue *queue, GrayQueueEnqueueCheckFunc enqueue_check_func, gboolean reuse_free_list);
 void sgen_gray_object_queue_dispose (SgenGrayQueue *queue);
 void sgen_gray_object_queue_deinit (SgenGrayQueue *queue);
-void sgen_gray_object_alloc_queue_section (SgenGrayQueue *queue);
+void sgen_gray_object_alloc_queue_section (SgenGrayQueue *queue, gboolean is_parallel);
 void sgen_gray_object_free_queue_section (GrayQueueSection *section);
 
 void sgen_section_gray_queue_init (SgenSectionGrayQueue *queue, gboolean locked,
@@ -153,13 +158,13 @@ sgen_gray_object_queue_is_empty (SgenGrayQueue *queue)
 }
 
 static inline MONO_ALWAYS_INLINE void
-GRAY_OBJECT_ENQUEUE (SgenGrayQueue *queue, GCObject *obj, SgenDescriptor desc)
+GRAY_OBJECT_ENQUEUE (SgenGrayQueue *queue, GCObject *obj, SgenDescriptor desc, gboolean is_parallel)
 {
 #if SGEN_MAX_DEBUG_LEVEL >= 9
-       sgen_gray_object_enqueue (queue, obj, desc);
+       sgen_gray_object_enqueue (queue, obj, desc, is_parallel);
 #else
        if (G_UNLIKELY (!queue->first || queue->cursor == GRAY_LAST_CURSOR_POSITION (queue->first))) {
-               sgen_gray_object_enqueue (queue, obj, desc);
+               sgen_gray_object_enqueue (queue, obj, desc, is_parallel);
        } else {
                GrayQueueEntry entry = SGEN_GRAY_QUEUE_ENTRY (obj, desc);
 
@@ -174,11 +179,11 @@ GRAY_OBJECT_ENQUEUE (SgenGrayQueue *queue, GCObject *obj, SgenDescriptor desc)
 }
 
 static inline MONO_ALWAYS_INLINE void
-GRAY_OBJECT_DEQUEUE (SgenGrayQueue *queue, GCObject** obj, SgenDescriptor *desc)
+GRAY_OBJECT_DEQUEUE (SgenGrayQueue *queue, GCObject** obj, SgenDescriptor *desc, gboolean is_parallel)
 {
        GrayQueueEntry entry;
 #if SGEN_MAX_DEBUG_LEVEL >= 9
-       entry = sgen_gray_object_dequeue (queue);
+       entry = sgen_gray_object_dequeue (queue, is_parallel);
        *obj = entry.obj;
        *desc = entry.desc;
 #else
@@ -187,7 +192,7 @@ GRAY_OBJECT_DEQUEUE (SgenGrayQueue *queue, GCObject** obj, SgenDescriptor *desc)
 
                *obj = NULL;
        } else if (G_UNLIKELY (queue->cursor == GRAY_FIRST_CURSOR_POSITION (queue->first))) {
-               entry = sgen_gray_object_dequeue (queue);
+               entry = sgen_gray_object_dequeue (queue, is_parallel);
                *obj = entry.obj;
                *desc = entry.desc;
        } else {

diff --git a/mono/sgen/sgen-marksweep-drain-gray-stack.h b/mono/sgen/sgen-marksweep-drain-gray-stack.h
index 370fbb0d42789490a01da15c88ffaa9cab3f3a15..53b213e72f7e5e8bedb9c01d4929ce581ccb0c2e 100644 (file)
--- a/mono/sgen/sgen-marksweep-drain-gray-stack.h
+++ b/mono/sgen/sgen-marksweep-drain-gray-stack.h
@@ -204,7 +204,11 @@ COPY_OR_MARK_FUNCTION_NAME (GCObject **ptr, GCObject *obj, SgenGrayQueue *queue)
                        if (first) {
                                binary_protocol_pin (obj, (gpointer)SGEN_LOAD_VTABLE (obj), sgen_safe_object_get_size (obj));
                                if (SGEN_OBJECT_HAS_REFERENCES (obj))
-                                       GRAY_OBJECT_ENQUEUE (queue, obj, desc);
+#ifdef COPY_OR_MARK_PARALLEL
+                                       GRAY_OBJECT_ENQUEUE_PARALLEL (queue, obj, desc);
+#else
+                                       GRAY_OBJECT_ENQUEUE_SERIAL (queue, obj, desc);
+#endif
                        }
                }
                return FALSE;
@@ -324,7 +328,11 @@ DRAIN_GRAY_STACK_FUNCTION_NAME (SgenGrayQueue *queue)
 
                HEAVY_STAT (++stat_drain_loops);
 
-               GRAY_OBJECT_DEQUEUE (queue, &obj, &desc);
+#if defined(COPY_OR_MARK_PARALLEL)
+               GRAY_OBJECT_DEQUEUE_PARALLEL (queue, &obj, &desc);
+#else
+               GRAY_OBJECT_DEQUEUE_SERIAL (queue, &obj, &desc);
+#endif
                if (!obj)
                        return TRUE;
 

diff --git a/mono/sgen/sgen-marksweep.c b/mono/sgen/sgen-marksweep.c
index fe7bb9f2a3d3e58fae7d7a826e813884af02d365..b280749a46a3dd8f9d045cd6de5e6ae7ee4a7e16 100644 (file)
--- a/mono/sgen/sgen-marksweep.c
+++ b/mono/sgen/sgen-marksweep.c
@@ -1145,7 +1145,7 @@ major_block_is_evacuating (MSBlockInfo *block)
                if (!MS_MARK_BIT ((block), __word, __bit)) {            \
                        MS_SET_MARK_BIT ((block), __word, __bit);       \
                        if (sgen_gc_descr_has_references (desc))                        \
-                               GRAY_OBJECT_ENQUEUE ((queue), (obj), (desc)); \
+                               GRAY_OBJECT_ENQUEUE_SERIAL ((queue), (obj), (desc)); \
                        binary_protocol_mark ((obj), (gpointer)SGEN_LOAD_VTABLE ((obj)), sgen_safe_object_get_size ((obj))); \
                        INC_NUM_MAJOR_OBJECTS_MARKED ();                \
                }                                                       \
@@ -1158,7 +1158,7 @@ major_block_is_evacuating (MSBlockInfo *block)
                MS_SET_MARK_BIT_PAR ((block), __word, __bit, first);    \
                if (first) {                                            \
                        if (sgen_gc_descr_has_references (desc))        \
-                               GRAY_OBJECT_ENQUEUE ((queue), (obj), (desc)); \
+                               GRAY_OBJECT_ENQUEUE_PARALLEL ((queue), (obj), (desc)); \
                        binary_protocol_mark ((obj), (gpointer)SGEN_LOAD_VTABLE ((obj)), sgen_safe_object_get_size ((obj))); \
                        INC_NUM_MAJOR_OBJECTS_MARKED ();                \
                }                                                       \

diff --git a/mono/sgen/sgen-nursery-allocator.c b/mono/sgen/sgen-nursery-allocator.c
index 785f46d5645c183043c8f09d047f4f2f3e7efeca..77a3aa866f743573c2c04da3c52352506d970e0a 100644 (file)
--- a/mono/sgen/sgen-nursery-allocator.c
+++ b/mono/sgen/sgen-nursery-allocator.c
@@ -742,7 +742,7 @@ sgen_build_nursery_fragments (GCMemSection *nursery_section, SgenGrayQueue *unpi
 
                if (addr0 < addr1) {
                        if (unpin_queue)
-                               GRAY_OBJECT_ENQUEUE (unpin_queue, (GCObject*)addr0, sgen_obj_get_descriptor_safe ((GCObject*)addr0));
+                               GRAY_OBJECT_ENQUEUE_SERIAL (unpin_queue, (GCObject*)addr0, sgen_obj_get_descriptor_safe ((GCObject*)addr0));
                        else
                                SGEN_UNPIN_OBJECT (addr0);
                        size = SGEN_ALIGN_UP (sgen_safe_object_get_size ((GCObject*)addr0));

diff --git a/mono/sgen/sgen-workers.c b/mono/sgen/sgen-workers.c
index 80e8ae36ffe46f927b9711f53556c676910bdcab..90960d89becb5bb9cea8171efbce900edf83c1e4 100644 (file)
--- a/mono/sgen/sgen-workers.c
+++ b/mono/sgen/sgen-workers.c
@@ -206,7 +206,7 @@ workers_get_work (WorkerData *data)
        if (major->is_concurrent) {
                GrayQueueSection *section = sgen_section_gray_queue_dequeue (&workers_distribute_gray_queue);
                if (section) {
-                       sgen_gray_object_enqueue_section (&data->private_gray_queue, section);
+                       sgen_gray_object_enqueue_section (&data->private_gray_queue, section, major->is_parallel);
                        return TRUE;
                }
        }
@@ -238,7 +238,7 @@ workers_steal_work (WorkerData *data)
        }
 
        if (section) {
-               sgen_gray_object_enqueue_section (&data->private_gray_queue, section);
+               sgen_gray_object_enqueue_section (&data->private_gray_queue, section, TRUE);
                return TRUE;
        }