[sgen] Use separate object ops with the concurrent collector when there is no evacuation

When we concurrently drain the gray stack, we check to see if there are any blocks that need evacuation. If there aren't, we can drain the gray stack using functions that don't check evacuation related block information. As we do with the serial collector.

diff --git a/mono/sgen/sgen-marksweep-drain-gray-stack.h b/mono/sgen/sgen-marksweep-drain-gray-stack.h
index 56efcf2694d669d25ba25535ffa9867d15237ec1..2ddb9b050e33c59ff982ad878049bab7f720dda8 100644 (file)
--- a/mono/sgen/sgen-marksweep-drain-gray-stack.h
+++ b/mono/sgen/sgen-marksweep-drain-gray-stack.h
@@ -52,12 +52,12 @@ COPY_OR_MARK_FUNCTION_NAME (GCObject **ptr, GCObject *obj, SgenGrayQueue *queue)
 #endif
 
        SGEN_ASSERT (9, obj, "null object from pointer %p", ptr);
-#ifndef COPY_OR_MARK_CONCURRENT
+#if !defined(COPY_OR_MARK_CONCURRENT) && !defined(COPY_OR_MARK_CONCURRENT_WITH_EVACUATION)
        SGEN_ASSERT (9, current_collection_generation == GENERATION_OLD, "old gen parallel allocator called from a %d collection", current_collection_generation);
 #endif
 
        if (sgen_ptr_in_nursery (obj)) {
-#ifndef COPY_OR_MARK_CONCURRENT
+#if !defined(COPY_OR_MARK_CONCURRENT) && !defined(COPY_OR_MARK_CONCURRENT_WITH_EVACUATION)
                int word, bit;
                GCObject *forwarded, *old_obj;
                mword vtable_word = *(mword*)obj;
@@ -166,7 +166,7 @@ COPY_OR_MARK_FUNCTION_NAME (GCObject **ptr, GCObject *obj, SgenGrayQueue *queue)
 
                        block = MS_BLOCK_FOR_OBJ (obj);
 
-#ifdef COPY_OR_MARK_CONCURRENT
+#ifdef COPY_OR_MARK_CONCURRENT_WITH_EVACUATION
                        if (G_UNLIKELY (major_block_is_evacuating (block))) {
                                /*
                                 * We don't copy within the concurrent phase. These objects will
@@ -220,7 +220,7 @@ SCAN_OBJECT_FUNCTION_NAME (GCObject *full_object, SgenDescriptor desc, SgenGrayQ
        /* Now scan the object. */
 
 #undef HANDLE_PTR
-#ifdef COPY_OR_MARK_CONCURRENT
+#if defined(COPY_OR_MARK_CONCURRENT_WITH_EVACUATION)
 #define HANDLE_PTR(ptr,obj)    do {                                    \
                GCObject *__old = *(ptr);                               \
                binary_protocol_scan_process_reference ((full_object), (ptr), __old); \
@@ -239,6 +239,18 @@ SCAN_OBJECT_FUNCTION_NAME (GCObject *full_object, SgenDescriptor desc, SgenGrayQ
                                mark_mod_union_card ((full_object), (void**)(ptr), __old); \
                        }                                               \
                } while (0)
+#elif defined(COPY_OR_MARK_CONCURRENT)
+#define HANDLE_PTR(ptr,obj)    do {                                    \
+               GCObject *__old = *(ptr);                               \
+               binary_protocol_scan_process_reference ((full_object), (ptr), __old); \
+               if (__old && !sgen_ptr_in_nursery (__old)) {            \
+                       PREFETCH_READ (__old);                  \
+                       COPY_OR_MARK_FUNCTION_NAME ((ptr), __old, queue); \
+               } else {                                                \
+                       if (G_UNLIKELY (sgen_ptr_in_nursery (__old) && !sgen_ptr_in_nursery ((ptr)))) \
+                               mark_mod_union_card ((full_object), (void**)(ptr), __old); \
+                       }                                               \
+               } while (0)
 #else
 #define HANDLE_PTR(ptr,obj)    do {                                    \
                GCObject *__old = *(ptr);                                       \
@@ -260,7 +272,7 @@ SCAN_OBJECT_FUNCTION_NAME (GCObject *full_object, SgenDescriptor desc, SgenGrayQ
 static gboolean
 DRAIN_GRAY_STACK_FUNCTION_NAME (SgenGrayQueue *queue)
 {
-#ifdef COPY_OR_MARK_CONCURRENT
+#if defined(COPY_OR_MARK_CONCURRENT) || defined(COPY_OR_MARK_CONCURRENT_WITH_EVACUATION)
        int i;
        for (i = 0; i < 32; i++) {
 #else

diff --git a/mono/sgen/sgen-marksweep.c b/mono/sgen/sgen-marksweep.c
index ba92ab36e729c385e29eef1870151a17cd7e7612..27d0372d6ebbbdc499a7d32307ee4719ca6cd66d 100644 (file)
--- a/mono/sgen/sgen-marksweep.c
+++ b/mono/sgen/sgen-marksweep.c
@@ -1190,29 +1190,49 @@ static guint64 stat_drain_loops;
 
 #undef COPY_OR_MARK_WITH_EVACUATION
 #define COPY_OR_MARK_CONCURRENT
-#define COPY_OR_MARK_FUNCTION_NAME     major_copy_or_mark_object_concurrent
-#define SCAN_OBJECT_FUNCTION_NAME      major_scan_object_concurrent
-#define DRAIN_GRAY_STACK_FUNCTION_NAME drain_gray_stack_concurrent
+#define COPY_OR_MARK_FUNCTION_NAME     major_copy_or_mark_object_concurrent_no_evacuation
+#define SCAN_OBJECT_FUNCTION_NAME      major_scan_object_concurrent_no_evacuation
+#define DRAIN_GRAY_STACK_FUNCTION_NAME drain_gray_stack_concurrent_no_evacuation
 #include "sgen-marksweep-drain-gray-stack.h"
 
-static gboolean
-drain_gray_stack (SgenGrayQueue *queue)
+#undef COPY_OR_MARK_CONCURRENT
+#define COPY_OR_MARK_CONCURRENT_WITH_EVACUATION
+#define COPY_OR_MARK_FUNCTION_NAME     major_copy_or_mark_object_concurrent_with_evacuation
+#define SCAN_OBJECT_FUNCTION_NAME      major_scan_object_concurrent_with_evacuation
+#define DRAIN_GRAY_STACK_FUNCTION_NAME drain_gray_stack_concurrent_with_evacuation
+#include "sgen-marksweep-drain-gray-stack.h"
+
+static inline gboolean
+major_is_evacuating (void)
 {
-       gboolean evacuation = FALSE;
        int i;
        for (i = 0; i < num_block_obj_sizes; ++i) {
                if (evacuate_block_obj_sizes [i]) {
-                       evacuation = TRUE;
-                       break;
+                       return TRUE;
                }
        }
 
-       if (evacuation)
+       return FALSE;
+}
+
+static gboolean
+drain_gray_stack (SgenGrayQueue *queue)
+{
+       if (major_is_evacuating ())
                return drain_gray_stack_with_evacuation (queue);
        else
                return drain_gray_stack_no_evacuation (queue);
 }
 
+static gboolean
+drain_gray_stack_concurrent (SgenGrayQueue *queue)
+{
+       if (major_is_evacuating ())
+               return drain_gray_stack_concurrent_with_evacuation (queue);
+       else
+               return drain_gray_stack_concurrent_no_evacuation (queue);
+}
+
 #include "sgen-marksweep-scan-object-concurrent.h"
 
 static void
@@ -1224,7 +1244,7 @@ major_copy_or_mark_object_canonical (GCObject **ptr, SgenGrayQueue *queue)
 static void
 major_copy_or_mark_object_concurrent_canonical (GCObject **ptr, SgenGrayQueue *queue)
 {
-       major_copy_or_mark_object_concurrent (ptr, *ptr, queue);
+       major_copy_or_mark_object_concurrent_with_evacuation (ptr, *ptr, queue);
 }
 
 static void
@@ -2471,7 +2491,7 @@ sgen_marksweep_init_internal (SgenMajorCollector *collector, gboolean is_concurr
        collector->major_ops_serial.drain_gray_stack = drain_gray_stack;
        if (is_concurrent) {
                collector->major_ops_concurrent_start.copy_or_mark_object = major_copy_or_mark_object_concurrent_canonical;
-               collector->major_ops_concurrent_start.scan_object = major_scan_object_concurrent;
+               collector->major_ops_concurrent_start.scan_object = major_scan_object_concurrent_with_evacuation;
                collector->major_ops_concurrent_start.drain_gray_stack = drain_gray_stack_concurrent;
 
                collector->major_ops_concurrent_finish.copy_or_mark_object = major_copy_or_mark_object_concurrent_finish_canonical;