[sgen] Separate concurrent M&S object operations into start, concurrent, finish.

diff --git a/mono/metadata/sgen-gc.c b/mono/metadata/sgen-gc.c
index deb36fb215d37a5dcc3c77ed4a16bd17c158ff5c..3f9ecaaf2b5d0cde9a19560f2d4a51f0624ab719 100644 (file)
--- a/mono/metadata/sgen-gc.c
+++ b/mono/metadata/sgen-gc.c
@@ -2700,9 +2700,9 @@ major_start_collection (gboolean concurrent, size_t *old_next_pin_slot)
                g_assert (major_collector.is_concurrent);
                concurrent_collection_in_progress = TRUE;
 
-               object_ops = &major_collector.major_concurrent_ops;
+               object_ops = &major_collector.major_ops_concurrent_start;
        } else {
-               object_ops = &major_collector.major_ops;
+               object_ops = &major_collector.major_ops_serial;
        }
 
        reset_pinned_from_failed_allocation ();
@@ -2737,7 +2737,7 @@ major_finish_collection (const char *reason, size_t old_next_pin_slot, gboolean
        TV_GETTIME (btv);
 
        if (concurrent_collection_in_progress) {
-               object_ops = &major_collector.major_concurrent_ops;
+               object_ops = &major_collector.major_ops_concurrent_finish;
 
                sgen_workers_signal_start_nursery_collection_and_wait ();
 
@@ -2760,7 +2760,7 @@ major_finish_collection (const char *reason, size_t old_next_pin_slot, gboolean
                        check_nursery_is_clean ();
        } else {
                SGEN_ASSERT (0, !scan_whole_nursery, "scan_whole_nursery only applies to concurrent collections");
-               object_ops = &major_collector.major_ops;
+               object_ops = &major_collector.major_ops_serial;
        }
 
        /*

diff --git a/mono/metadata/sgen-gc.h b/mono/metadata/sgen-gc.h
index b480135a7385c323616f80bd7d5265760da77231..6fb3ee2c29ee2174b5c8c587408018e1408b50ce 100644 (file)
--- a/mono/metadata/sgen-gc.h
+++ b/mono/metadata/sgen-gc.h
@@ -667,8 +667,10 @@ struct _SgenMajorCollector {
        void* (*alloc_small_pinned_obj) (MonoVTable *vtable, size_t size, gboolean has_references);
        void* (*alloc_degraded) (MonoVTable *vtable, size_t size);
 
-       SgenObjectOperations major_ops;
-       SgenObjectOperations major_concurrent_ops;
+       SgenObjectOperations major_ops_serial;
+       SgenObjectOperations major_ops_concurrent_start;
+       SgenObjectOperations major_ops_concurrent;
+       SgenObjectOperations major_ops_concurrent_finish;
 
        void* (*alloc_object) (MonoVTable *vtable, size_t size, gboolean has_references);
        void (*free_pinned_object) (char *obj, size_t size);

diff --git a/mono/metadata/sgen-marksweep-scan-object-concurrent.h b/mono/metadata/sgen-marksweep-scan-object-concurrent.h
index d21c363fbae550d3ab9ab4eb459a8d7b1638b5c8..dcfe8f905ab2c88da020dc12ca74693d4a0d6483 100644 (file)
--- a/mono/metadata/sgen-marksweep-scan-object-concurrent.h
+++ b/mono/metadata/sgen-marksweep-scan-object-concurrent.h
@@ -39,7 +39,7 @@ extern guint64 stat_scan_object_called_major;
                binary_protocol_scan_process_reference ((obj), (ptr), __old); \
                if (__old && !sgen_ptr_in_nursery (__old)) {            \
                        PREFETCH_READ (__old);                  \
-                       major_copy_or_mark_object_with_evacuation_concurrent ((ptr), __old, queue); \
+                       major_copy_or_mark_object_concurrent ((ptr), __old, queue); \
                } else {                                                \
                        if (G_UNLIKELY (sgen_ptr_in_nursery (__old) && !sgen_ptr_in_nursery ((ptr)))) \
                                sgen_add_to_global_remset ((ptr), __old); \
@@ -49,7 +49,7 @@ extern guint64 stat_scan_object_called_major;
 /* FIXME: Unify this with optimized code in sgen-marksweep.c. */
 
 static void
-major_scan_object_no_mark_concurrent (char *start, mword desc, SgenGrayQueue *queue)
+major_scan_object_no_mark_concurrent_anywhere (char *start, mword desc, SgenGrayQueue *queue)
 {
        SGEN_OBJECT_LAYOUT_STATISTICS_DECLARE_BITMAP;
 
@@ -67,6 +67,19 @@ major_scan_object_no_mark_concurrent (char *start, mword desc, SgenGrayQueue *qu
        HEAVY_STAT (++stat_scan_object_called_major);
 }
 
+static void
+major_scan_object_no_mark_concurrent_start_finish (char *start, mword desc, SgenGrayQueue *queue)
+{
+       major_scan_object_no_mark_concurrent_anywhere (start, desc, queue);
+}
+
+static void
+major_scan_object_no_mark_concurrent (char *start, mword desc, SgenGrayQueue *queue)
+{
+       SGEN_ASSERT (0, !sgen_ptr_in_nursery (start), "Why are we scanning nursery objects in the concurrent collector?");
+       major_scan_object_no_mark_concurrent_anywhere (start, desc, queue);
+}
+
 static void
 major_scan_vtype_concurrent (char *start, mword desc, SgenGrayQueue *queue BINARY_PROTOCOL_ARG (size_t size))
 {

diff --git a/mono/metadata/sgen-marksweep.c b/mono/metadata/sgen-marksweep.c
index 77cd360fd346ad037304a02c7489fcdf4d0620f9..0f3ef12c9b1ba56e72617fde28b7177fe24e3c68 100644 (file)
--- a/mono/metadata/sgen-marksweep.c
+++ b/mono/metadata/sgen-marksweep.c
@@ -1073,7 +1073,7 @@ pin_major_object (char *obj, SgenGrayQueue *queue)
 #include "sgen-major-copy-object.h"
 
 static void
-major_copy_or_mark_object_with_evacuation_concurrent (void **ptr, void *obj, SgenGrayQueue *queue)
+major_copy_or_mark_object_concurrent (void **ptr, void *obj, SgenGrayQueue *queue)
 {
        SGEN_ASSERT (9, sgen_concurrent_collection_in_progress (), "Why are we scanning concurrently when there's no concurrent collection on?");
        SGEN_ASSERT (9, !sgen_workers_are_working () || sgen_thread_pool_is_thread_pool_thread (mono_native_thread_id_get ()), "We must not scan from two threads at the same time!");
@@ -1195,7 +1195,7 @@ major_copy_or_mark_object_canonical (void **ptr, SgenGrayQueue *queue)
 static void
 major_copy_or_mark_object_concurrent_canonical (void **ptr, SgenGrayQueue *queue)
 {
-       major_copy_or_mark_object_with_evacuation_concurrent (ptr, *ptr, queue);
+       major_copy_or_mark_object_concurrent (ptr, *ptr, queue);
 }
 
 static void
@@ -2460,12 +2460,18 @@ sgen_marksweep_init_internal (SgenMajorCollector *collector, gboolean is_concurr
        collector->describe_pointer = major_describe_pointer;
        collector->count_cards = major_count_cards;
 
-       collector->major_ops.copy_or_mark_object = major_copy_or_mark_object_canonical;
-       collector->major_ops.scan_object = major_scan_object_with_evacuation;
+       collector->major_ops_serial.copy_or_mark_object = major_copy_or_mark_object_canonical;
+       collector->major_ops_serial.scan_object = major_scan_object_with_evacuation;
        if (is_concurrent) {
-               collector->major_concurrent_ops.copy_or_mark_object = major_copy_or_mark_object_concurrent_canonical;
-               collector->major_concurrent_ops.scan_object = major_scan_object_no_mark_concurrent;
-               collector->major_concurrent_ops.scan_vtype = major_scan_vtype_concurrent;
+               collector->major_ops_concurrent_start.copy_or_mark_object = major_copy_or_mark_object_concurrent_canonical;
+               collector->major_ops_concurrent_start.scan_object = major_scan_object_no_mark_concurrent_start_finish;
+
+               collector->major_ops_concurrent.copy_or_mark_object = major_copy_or_mark_object_concurrent_canonical;
+               collector->major_ops_concurrent.scan_object = major_scan_object_no_mark_concurrent;
+
+               collector->major_ops_concurrent_finish.copy_or_mark_object = major_copy_or_mark_object_concurrent_canonical;
+               collector->major_ops_concurrent_finish.scan_object = major_scan_object_no_mark_concurrent_start_finish;
+               collector->major_ops_concurrent_finish.scan_vtype = major_scan_vtype_concurrent;
        }
 
 #if !defined (FIXED_HEAP) && !defined (SGEN_PARALLEL_MARK)

diff --git a/mono/metadata/sgen-workers.c b/mono/metadata/sgen-workers.c
index b9a2f5808e8fbe9d4c10ffc20c8b188c44c8e189..b5f8062d10543699376b5d5c3e43a4c32665c9a0 100644 (file)
--- a/mono/metadata/sgen-workers.c
+++ b/mono/metadata/sgen-workers.c
@@ -257,6 +257,7 @@ marker_idle_func (void *data_untyped)
        if (!continue_idle_func ())
                return;
 
+       SGEN_ASSERT (0, sgen_concurrent_collection_in_progress (), "The worker should only mark in concurrent collections.");
        SGEN_ASSERT (0, sgen_get_current_collection_generation () != GENERATION_NURSERY, "Why are we doing work while there's a nursery collection happening?");
 
        if (workers_state == STATE_WORK_ENQUEUED) {
@@ -265,9 +266,7 @@ marker_idle_func (void *data_untyped)
        }
 
        if (!sgen_gray_object_queue_is_empty (&data->private_gray_queue) || workers_get_work (data)) {
-               SgenObjectOperations *ops = sgen_concurrent_collection_in_progress ()
-                       ? &major->major_concurrent_ops
-                       : &major->major_ops;
+               SgenObjectOperations *ops = &major->major_ops_concurrent;
                ScanCopyContext ctx = CONTEXT_FROM_OBJECT_OPERATIONS (ops, &data->private_gray_queue);
 
                SGEN_ASSERT (0, !sgen_gray_object_queue_is_empty (&data->private_gray_queue), "How is our gray queue empty if we just got work?");