[sgen] Evacuation for the concurrent collector

The concurrent mark no longer follows references into blocks that are evacuated. The cardtable is marked instead and all the objects will be moved during the finishing pause.

diff --git a/mono/sgen/sgen-marksweep-drain-gray-stack.h b/mono/sgen/sgen-marksweep-drain-gray-stack.h
index dce3be7344ad528c712be144fd775d38becfa295..d2f0893520d44b1c99dc6d3ce9d75261ba968da7 100644 (file)
--- a/mono/sgen/sgen-marksweep-drain-gray-stack.h
+++ b/mono/sgen/sgen-marksweep-drain-gray-stack.h
@@ -166,16 +166,21 @@ COPY_OR_MARK_FUNCTION_NAME (GCObject **ptr, GCObject *obj, SgenGrayQueue *queue)
 
                        block = MS_BLOCK_FOR_OBJ (obj);
 
+#ifdef COPY_OR_MARK_CONCURRENT
+                       if (G_UNLIKELY (major_block_is_evacuating (block))) {
+                               /*
+                                * We don't copy within the concurrent phase. These objects will
+                                * be handled below in the finishing pause, by scanning the mod-union
+                                * card table.
+                                */
+                               return FALSE;
+                       }
+#endif
+
 #ifdef COPY_OR_MARK_WITH_EVACUATION
-                       {
-                               int size_index = block->obj_size_index;
-
-                               if (evacuate_block_obj_sizes [size_index] && !block->has_pinned) {
-                                       HEAVY_STAT (++stat_optimized_copy_major_small_evacuate);
-                                       if (block->is_to_space)
-                                               return FALSE;
-                                       goto do_copy_object;
-                               }
+                       if (major_block_is_evacuating (block)) {
+                               HEAVY_STAT (++stat_optimized_copy_major_small_evacuate);
+                               goto do_copy_object;
                        }
 #endif
 
@@ -220,8 +225,15 @@ SCAN_OBJECT_FUNCTION_NAME (GCObject *full_object, SgenDescriptor desc, SgenGrayQ
                GCObject *__old = *(ptr);                               \
                binary_protocol_scan_process_reference ((obj), (ptr), __old); \
                if (__old && !sgen_ptr_in_nursery (__old)) {            \
-                       PREFETCH_READ (__old);                          \
-                       COPY_OR_MARK_FUNCTION_NAME ((ptr), __old, queue); \
+                       MSBlockInfo *block = MS_BLOCK_FOR_OBJ (__old);  \
+                       if (G_UNLIKELY (!sgen_ptr_in_nursery (ptr) &&   \
+                                       sgen_safe_object_is_small (__old, sgen_obj_get_descriptor (__old) & DESC_TYPE_MASK) && \
+                                       major_block_is_evacuating (block))) { \
+                               mark_mod_union_card ((full_object), (void**)(ptr)); \
+                       } else {                                        \
+                               PREFETCH_READ (__old);                  \
+                               COPY_OR_MARK_FUNCTION_NAME ((ptr), __old, queue); \
+                       }                                               \
                } else {                                                \
                        if (G_UNLIKELY (sgen_ptr_in_nursery (__old) && !sgen_ptr_in_nursery ((ptr)))) \
                                mark_mod_union_card ((full_object), (void**)(ptr)); \

diff --git a/mono/sgen/sgen-marksweep-scan-object-concurrent.h b/mono/sgen/sgen-marksweep-scan-object-concurrent.h
index 8ebd8eee2a14a2a05ae799eb74d6f12b6076bd73..fe080905de93ebf0dcfdc97855800efe10658f7d 100644 (file)
--- a/mono/sgen/sgen-marksweep-scan-object-concurrent.h
+++ b/mono/sgen/sgen-marksweep-scan-object-concurrent.h
@@ -35,7 +35,7 @@
                 void *__old = *(ptr);                                   \
                 binary_protocol_scan_process_reference ((obj), (ptr), __old); \
                 if (__old) {                                            \
-                        gboolean __still_in_nursery = major_copy_or_mark_object_no_evacuation ((ptr), __old, queue); \
+                        gboolean __still_in_nursery = major_copy_or_mark_object_with_evacuation ((ptr), __old, queue); \
                         if (G_UNLIKELY (__still_in_nursery && !sgen_ptr_in_nursery ((ptr)) && !SGEN_OBJECT_IS_CEMENTED (*(ptr)))) { \
                                 void *__copy = *(ptr);                  \
                                 sgen_add_to_global_remset ((ptr), __copy); \

diff --git a/mono/sgen/sgen-marksweep.c b/mono/sgen/sgen-marksweep.c
index 2fd0413e5d08fcaed700849f1e117c9a2e221d67..39e49d206ad6717b25e64e78d307004009e2e4c7 100644 (file)
--- a/mono/sgen/sgen-marksweep.c
+++ b/mono/sgen/sgen-marksweep.c
@@ -532,10 +532,11 @@ ms_alloc_block (int size_index, gboolean pinned, gboolean has_references)
         * Blocks that are to-space are not evacuated from.  During an major collection
         * blocks are allocated for two reasons: evacuating objects from the nursery and
         * evacuating them from major blocks marked for evacuation.  In both cases we don't
-        * want further evacuation.
+        * want further evacuation. We also don't want to evacuate objects allocated during
+        * the concurrent mark since it would add pointless stress on the finishing pause.
         */
-       info->is_to_space = (sgen_get_current_collection_generation () == GENERATION_OLD);
-       info->state = (info->is_to_space || sgen_concurrent_collection_in_progress ()) ? BLOCK_STATE_MARKING : BLOCK_STATE_SWEPT;
+       info->is_to_space = (sgen_get_current_collection_generation () == GENERATION_OLD) || sgen_concurrent_collection_in_progress ();
+       info->state = info->is_to_space ? BLOCK_STATE_MARKING : BLOCK_STATE_SWEPT;
        SGEN_ASSERT (6, !sweep_in_progress () || info->state == BLOCK_STATE_SWEPT, "How do we add a new block to be swept while sweeping?");
        info->cardtable_mod_union = NULL;
 
@@ -1078,6 +1079,16 @@ mark_mod_union_card (GCObject *obj, void **ptr)
        }
 }
 
+static inline gboolean
+major_block_is_evacuating (MSBlockInfo *block)
+{
+       if (evacuate_block_obj_sizes [block->obj_size_index] &&
+                       !block->has_pinned &&
+                       !block->is_to_space)
+               return TRUE;
+       return FALSE;
+}
+
 #define LOAD_VTABLE    SGEN_LOAD_VTABLE
 
 #define MS_MARK_OBJECT_AND_ENQUEUE_CHECKED(obj,desc,block,queue) do {  \
@@ -1220,7 +1231,7 @@ major_copy_or_mark_object_concurrent_canonical (GCObject **ptr, SgenGrayQueue *q
 static void
 major_copy_or_mark_object_concurrent_finish_canonical (GCObject **ptr, SgenGrayQueue *queue)
 {
-       major_copy_or_mark_object_no_evacuation (ptr, *ptr, queue);
+       major_copy_or_mark_object_with_evacuation (ptr, *ptr, queue);
 }
 
 static void
@@ -2479,9 +2490,9 @@ sgen_marksweep_init_internal (SgenMajorCollector *collector, gboolean is_concurr
                collector->major_ops_concurrent_start.drain_gray_stack = drain_gray_stack_concurrent;
 
                collector->major_ops_concurrent_finish.copy_or_mark_object = major_copy_or_mark_object_concurrent_finish_canonical;
-               collector->major_ops_concurrent_finish.scan_object = major_scan_object_no_evacuation;
+               collector->major_ops_concurrent_finish.scan_object = major_scan_object_with_evacuation;
                collector->major_ops_concurrent_finish.scan_vtype = major_scan_vtype_concurrent_finish;
-               collector->major_ops_concurrent_finish.drain_gray_stack = drain_gray_stack_no_evacuation;
+               collector->major_ops_concurrent_finish.drain_gray_stack = drain_gray_stack;
        }
 
 #ifdef HEAVY_STATISTICS