[sgen] Support evacuation in the optimized mark/scan loop.

We actually implement two separate mark/scan loops: One that doesn't
support evacuation and one that does.  We dynamically choose which one
to use.  The evacuation-less loop is faster, so we use it whenever we can.

diff --git a/mono/metadata/sgen-major-scan-object.h b/mono/metadata/sgen-major-scan-object.h
index d3ed61f42a0dd94e217c934b9b4ad9681e623e2c..df872a51484647adfebf5cd3f6ca2afee30c6fbf 100644 (file)
--- a/mono/metadata/sgen-major-scan-object.h
+++ b/mono/metadata/sgen-major-scan-object.h
@@ -55,7 +55,7 @@ extern long long stat_scan_object_called_major;
                if (__old && FOLLOW_OBJECT (__old)) {                   \
                        void *__copy;                                   \
                        PREFETCH_DYNAMIC_HEAP (__old);                  \
-                       CONCURRENT_NAME (major_copy_or_mark_object) ((ptr), __old, queue); \
+                       CONCURRENT_NAME (major_copy_or_mark_object_with_evacuation) ((ptr), __old, queue); \
                        __copy = *(ptr);                                \
                        SGEN_COND_LOG (9, __old != __copy, "Overwrote field at %p with %p (was: %p)", (ptr), *(ptr), __old); \
                        if (G_UNLIKELY (sgen_ptr_in_nursery (__copy) && !sgen_ptr_in_nursery ((ptr)) && !SGEN_OBJECT_IS_CEMENTED (__copy))) \

diff --git a/mono/metadata/sgen-marksweep-drain-gray-stack.h b/mono/metadata/sgen-marksweep-drain-gray-stack.h
index 2c56899f9b87a35c5a63934a03f6f7ed8d1f8295..d64d46bd56648f915143ac709e42b4d352328f43 100644 (file)
--- a/mono/metadata/sgen-marksweep-drain-gray-stack.h
+++ b/mono/metadata/sgen-marksweep-drain-gray-stack.h
@@ -76,7 +76,9 @@ COPY_OR_MARK_FUNCTION_NAME (void **ptr, void *obj, SgenGrayQueue *queue)
                if (sgen_nursery_is_to_space (obj))
                        return TRUE;
 
+#ifdef COPY_OR_MARK_WITH_EVACUATION
        do_copy_object:
+#endif
                old_obj = obj;
                obj = copy_object_no_checks (obj, queue);
                SGEN_ASSERT (0, old_obj != obj, "Cannot handle copy object failure.");
@@ -110,23 +112,24 @@ COPY_OR_MARK_FUNCTION_NAME (void **ptr, void *obj, SgenGrayQueue *queue)
                return FALSE;
        } else {
 #ifdef SGEN_MARK_ON_ENQUEUE
-               char *forwarded;
                mword vtable_word = *(mword*)obj;
                mword desc = sgen_vtable_get_descriptor ((MonoVTable*)vtable_word);
                int type = desc & 7;
 
                HEAVY_STAT (++stat_optimized_copy_major);
 
-               if ((forwarded = SGEN_VTABLE_IS_FORWARDED (vtable_word))) {
-                       HEAVY_STAT (++stat_optimized_copy_major_forwarded);
-                       *ptr = forwarded;
-                       return FALSE;
+#ifdef COPY_OR_MARK_WITH_EVACUATION
+               {
+                       char *forwarded;
+                       if ((forwarded = SGEN_VTABLE_IS_FORWARDED (vtable_word))) {
+                               HEAVY_STAT (++stat_optimized_copy_major_forwarded);
+                               *ptr = forwarded;
+                               return FALSE;
+                       }
                }
+#endif
 
                if (type <= DESC_TYPE_MAX_SMALL_OBJ || SGEN_ALIGN_UP (sgen_safe_object_get_size ((MonoObject*)obj)) <= SGEN_MAX_SMALL_OBJ_SIZE) {
-                       int size_index;
-                       gboolean evacuate;
-
 #ifdef HEAVY_STATISTICS
                        if (type <= DESC_TYPE_MAX_SMALL_OBJ)
                                ++stat_optimized_copy_major_small_fast;
@@ -135,15 +138,19 @@ COPY_OR_MARK_FUNCTION_NAME (void **ptr, void *obj, SgenGrayQueue *queue)
 #endif
 
                        block = MS_BLOCK_FOR_OBJ (obj);
-                       size_index = block->obj_size_index;
-                       evacuate = evacuate_block_obj_sizes [size_index];
-
-                       if (evacuate && !block->has_pinned) {
-                               HEAVY_STAT (++stat_optimized_copy_major_small_evacuate);
-                               if (block->is_to_space)
-                                       return FALSE;
-                               goto do_copy_object;
+
+#ifdef COPY_OR_MARK_WITH_EVACUATION
+                       {
+                               int size_index = block->obj_size_index;
+
+                               if (evacuate_block_obj_sizes [size_index] && !block->has_pinned) {
+                                       HEAVY_STAT (++stat_optimized_copy_major_small_evacuate);
+                                       if (block->is_to_space)
+                                               return FALSE;
+                                       goto do_copy_object;
+                               }
                        }
+#endif
 
                        MS_MARK_OBJECT_AND_ENQUEUE (obj, desc, block, queue);
                } else {
@@ -281,4 +288,5 @@ DRAIN_GRAY_STACK_FUNCTION_NAME (ScanCopyContext ctx)
 }
 
 #undef COPY_OR_MARK_FUNCTION_NAME
+#undef COPY_OR_MARK_WITH_EVACUATION
 #undef DRAIN_GRAY_STACK_FUNCTION_NAME

diff --git a/mono/metadata/sgen-marksweep.c b/mono/metadata/sgen-marksweep.c
index 06057613a6ae37c0ec65d1b17af3d5cb8eda2a8d..544e2c82849208640b5f6233fef214c6cb6e0943 100644 (file)
--- a/mono/metadata/sgen-marksweep.c
+++ b/mono/metadata/sgen-marksweep.c
@@ -936,7 +936,7 @@ pin_major_object (char *obj, SgenGrayQueue *queue)
 
 #ifdef SGEN_HAVE_CONCURRENT_MARK
 static void
-major_copy_or_mark_object_concurrent (void **ptr, void *obj, SgenGrayQueue *queue)
+major_copy_or_mark_object_with_evacuation_concurrent (void **ptr, void *obj, SgenGrayQueue *queue)
 {
        SGEN_ASSERT (9, sgen_concurrent_collection_in_progress (), "Why are we scanning concurrently when there's no concurrent collection on?");
        SGEN_ASSERT (9, !sgen_workers_are_working () || sgen_is_worker_thread (mono_native_thread_id_get ()), "We must not scan from two threads at the same time!");
@@ -1014,10 +1014,32 @@ static long long stat_drain_loops;
 
 static void major_scan_object (char *start, mword desc, SgenGrayQueue *queue);
 
-#define COPY_OR_MARK_FUNCTION_NAME     major_copy_or_mark_object
-#define DRAIN_GRAY_STACK_FUNCTION_NAME drain_gray_stack
+#define COPY_OR_MARK_FUNCTION_NAME     major_copy_or_mark_object_no_evacuation
+#define DRAIN_GRAY_STACK_FUNCTION_NAME drain_gray_stack_no_evacuation
 #include "sgen-marksweep-drain-gray-stack.h"
 
+#define COPY_OR_MARK_WITH_EVACUATION
+#define COPY_OR_MARK_FUNCTION_NAME     major_copy_or_mark_object_with_evacuation
+#define DRAIN_GRAY_STACK_FUNCTION_NAME drain_gray_stack_with_evacuation
+#include "sgen-marksweep-drain-gray-stack.h"
+
+static gboolean
+drain_gray_stack (ScanCopyContext ctx)
+{
+       gboolean evacuation = FALSE;
+       for (int i = 0; i < num_block_obj_sizes; ++i) {
+               if (evacuate_block_obj_sizes [i]) {
+                       evacuation = TRUE;
+                       break;
+               }
+       }
+
+       if (evacuation)
+               return drain_gray_stack_with_evacuation (ctx);
+       else
+               return drain_gray_stack_no_evacuation (ctx);
+}
+
 #include "sgen-major-scan-object.h"
 
 #ifdef SGEN_HAVE_CONCURRENT_MARK
@@ -1057,14 +1079,14 @@ sgen_gray_object_dequeue_fast (SgenGrayQueue *queue, char** obj, mword *desc) {
 static void
 major_copy_or_mark_object_canonical (void **ptr, SgenGrayQueue *queue)
 {
-       major_copy_or_mark_object (ptr, *ptr, queue);
+       major_copy_or_mark_object_with_evacuation (ptr, *ptr, queue);
 }
 
 #ifdef SGEN_HAVE_CONCURRENT_MARK
 static void
 major_copy_or_mark_object_concurrent_canonical (void **ptr, SgenGrayQueue *queue)
 {
-       major_copy_or_mark_object_concurrent (ptr, *ptr, queue);
+       major_copy_or_mark_object_with_evacuation_concurrent (ptr, *ptr, queue);
 }
 #endif