Merge pull request #2485 from ludovic-henry/monoerror-mono_marshal_alloc

[mono.git] / mono / sgen / sgen-marksweep-drain-gray-stack.h

diff --git a/mono/sgen/sgen-marksweep-drain-gray-stack.h b/mono/sgen/sgen-marksweep-drain-gray-stack.h
index 01d276ad06df9c15d84703b677bf3974505f655b..fc2161e7d3ec9583b1a72c5ae37515da335e4bcf 100644 (file)
--- a/mono/sgen/sgen-marksweep-drain-gray-stack.h
+++ b/mono/sgen/sgen-marksweep-drain-gray-stack.h
@@ -52,9 +52,12 @@ COPY_OR_MARK_FUNCTION_NAME (GCObject **ptr, GCObject *obj, SgenGrayQueue *queue)
 #endif
 
        SGEN_ASSERT (9, obj, "null object from pointer %p", ptr);
+#if !defined(COPY_OR_MARK_CONCURRENT) && !defined(COPY_OR_MARK_CONCURRENT_WITH_EVACUATION)
        SGEN_ASSERT (9, current_collection_generation == GENERATION_OLD, "old gen parallel allocator called from a %d collection", current_collection_generation);
+#endif
 
        if (sgen_ptr_in_nursery (obj)) {
+#if !defined(COPY_OR_MARK_CONCURRENT) && !defined(COPY_OR_MARK_CONCURRENT_WITH_EVACUATION)
                int word, bit;
                GCObject *forwarded, *old_obj;
                mword vtable_word = *(mword*)obj;
@@ -71,7 +74,7 @@ COPY_OR_MARK_FUNCTION_NAME (GCObject **ptr, GCObject *obj, SgenGrayQueue *queue)
                        HEAVY_STAT (++stat_optimized_copy_nursery_pinned);
                        return TRUE;
                }
-               if ((forwarded = SGEN_VTABLE_IS_FORWARDED (vtable_word))) {
+               if ((forwarded = (GCObject *)SGEN_VTABLE_IS_FORWARDED (vtable_word))) {
                        HEAVY_STAT (++stat_optimized_copy_nursery_forwarded);
                        SGEN_UPDATE_REFERENCE (ptr, forwarded);
                        return sgen_ptr_in_nursery (forwarded);
@@ -128,6 +131,7 @@ COPY_OR_MARK_FUNCTION_NAME (GCObject **ptr, GCObject *obj, SgenGrayQueue *queue)
                binary_protocol_mark (obj, (gpointer)LOAD_VTABLE (obj), sgen_safe_object_get_size (obj));
 
                return FALSE;
+#endif
        } else {
                mword vtable_word = *(mword*)obj;
                SgenDescriptor desc;
@@ -138,7 +142,7 @@ COPY_OR_MARK_FUNCTION_NAME (GCObject **ptr, GCObject *obj, SgenGrayQueue *queue)
 #ifdef COPY_OR_MARK_WITH_EVACUATION
                {
                        GCObject *forwarded;
-                       if ((forwarded = SGEN_VTABLE_IS_FORWARDED (vtable_word))) {
+                       if ((forwarded = (GCObject *)SGEN_VTABLE_IS_FORWARDED (vtable_word))) {
                                HEAVY_STAT (++stat_optimized_copy_major_forwarded);
                                SGEN_UPDATE_REFERENCE (ptr, forwarded);
                                SGEN_ASSERT (9, !sgen_ptr_in_nursery (forwarded), "Cannot be forwarded to nursery.");
@@ -162,16 +166,21 @@ COPY_OR_MARK_FUNCTION_NAME (GCObject **ptr, GCObject *obj, SgenGrayQueue *queue)
 
                        block = MS_BLOCK_FOR_OBJ (obj);
 
+#ifdef COPY_OR_MARK_CONCURRENT_WITH_EVACUATION
+                       if (G_UNLIKELY (major_block_is_evacuating (block))) {
+                               /*
+                                * We don't copy within the concurrent phase. These objects will
+                                * be handled below in the finishing pause, by scanning the mod-union
+                                * card table.
+                                */
+                               return FALSE;
+                       }
+#endif
+
 #ifdef COPY_OR_MARK_WITH_EVACUATION
-                       {
-                               int size_index = block->obj_size_index;
-
-                               if (evacuate_block_obj_sizes [size_index] && !block->has_pinned) {
-                                       HEAVY_STAT (++stat_optimized_copy_major_small_evacuate);
-                                       if (block->is_to_space)
-                                               return FALSE;
-                                       goto do_copy_object;
-                               }
+                       if (major_block_is_evacuating (block)) {
+                               HEAVY_STAT (++stat_optimized_copy_major_small_evacuate);
+                               goto do_copy_object;
                        }
 #endif
 
@@ -189,14 +198,14 @@ COPY_OR_MARK_FUNCTION_NAME (GCObject **ptr, GCObject *obj, SgenGrayQueue *queue)
                }
                return FALSE;
        }
-       SGEN_ASSERT (0, FALSE, "How is this happening?");
-       return FALSE;
+
+       return TRUE;
 }
 
 static void
-SCAN_OBJECT_FUNCTION_NAME (GCObject *obj, SgenDescriptor desc, SgenGrayQueue *queue)
+SCAN_OBJECT_FUNCTION_NAME (GCObject *full_object, SgenDescriptor desc, SgenGrayQueue *queue)
 {
-       char *start = (char*)obj;
+       char *start = (char*)full_object;
 
 #ifdef HEAVY_STATISTICS
        ++stat_optimized_major_scan;
@@ -211,30 +220,63 @@ SCAN_OBJECT_FUNCTION_NAME (GCObject *obj, SgenDescriptor desc, SgenGrayQueue *qu
        /* Now scan the object. */
 
 #undef HANDLE_PTR
+#if defined(COPY_OR_MARK_CONCURRENT_WITH_EVACUATION)
+#define HANDLE_PTR(ptr,obj)    do {                                    \
+               GCObject *__old = *(ptr);                               \
+               binary_protocol_scan_process_reference ((full_object), (ptr), __old); \
+               if (__old && !sgen_ptr_in_nursery (__old)) {            \
+                       if (G_UNLIKELY (!sgen_ptr_in_nursery (ptr) &&   \
+                                       sgen_safe_object_is_small (__old, sgen_obj_get_descriptor (__old) & DESC_TYPE_MASK) && \
+                                       major_block_is_evacuating (MS_BLOCK_FOR_OBJ (__old)))) { \
+                               mark_mod_union_card ((full_object), (void**)(ptr), __old); \
+                       } else {                                        \
+                               PREFETCH_READ (__old);                  \
+                               COPY_OR_MARK_FUNCTION_NAME ((ptr), __old, queue); \
+                       }                                               \
+               } else {                                                \
+                       if (G_UNLIKELY (sgen_ptr_in_nursery (__old) && !sgen_ptr_in_nursery ((ptr)))) \
+                               mark_mod_union_card ((full_object), (void**)(ptr), __old); \
+                       }                                               \
+               } while (0)
+#elif defined(COPY_OR_MARK_CONCURRENT)
 #define HANDLE_PTR(ptr,obj)    do {                                    \
-               void *__old = *(ptr);                                   \
-               binary_protocol_scan_process_reference ((obj), (ptr), __old); \
+               GCObject *__old = *(ptr);                               \
+               binary_protocol_scan_process_reference ((full_object), (ptr), __old); \
+               if (__old && !sgen_ptr_in_nursery (__old)) {            \
+                       PREFETCH_READ (__old);                  \
+                       COPY_OR_MARK_FUNCTION_NAME ((ptr), __old, queue); \
+               } else {                                                \
+                       if (G_UNLIKELY (sgen_ptr_in_nursery (__old) && !sgen_ptr_in_nursery ((ptr)))) \
+                               mark_mod_union_card ((full_object), (void**)(ptr), __old); \
+                       }                                               \
+               } while (0)
+#else
+#define HANDLE_PTR(ptr,obj)    do {                                    \
+               GCObject *__old = *(ptr);                                       \
+               binary_protocol_scan_process_reference ((full_object), (ptr), __old); \
                if (__old) {                                            \
                        gboolean __still_in_nursery = COPY_OR_MARK_FUNCTION_NAME ((ptr), __old, queue); \
                        if (G_UNLIKELY (__still_in_nursery && !sgen_ptr_in_nursery ((ptr)) && !SGEN_OBJECT_IS_CEMENTED (*(ptr)))) { \
-                               void *__copy = *(ptr);                  \
+                               GCObject *__copy = *(ptr);                      \
                                sgen_add_to_global_remset ((ptr), __copy); \
                        }                                               \
                }                                                       \
        } while (0)
+#endif
 
 #define SCAN_OBJECT_PROTOCOL
 #include "sgen-scan-object.h"
 }
 
 static gboolean
-DRAIN_GRAY_STACK_FUNCTION_NAME (ScanCopyContext ctx)
+DRAIN_GRAY_STACK_FUNCTION_NAME (SgenGrayQueue *queue)
 {
-       SgenGrayQueue *queue = ctx.queue;
-
-       SGEN_ASSERT (0, ctx.ops->scan_object == major_scan_object_with_evacuation, "Wrong scan function");
-
+#if defined(COPY_OR_MARK_CONCURRENT) || defined(COPY_OR_MARK_CONCURRENT_WITH_EVACUATION)
+       int i;
+       for (i = 0; i < 32; i++) {
+#else
        for (;;) {
+#endif
                GCObject *obj;
                SgenDescriptor desc;
 
@@ -244,8 +286,9 @@ DRAIN_GRAY_STACK_FUNCTION_NAME (ScanCopyContext ctx)
                if (!obj)
                        return TRUE;
 
-               SCAN_OBJECT_FUNCTION_NAME (obj, desc, ctx.queue);
+               SCAN_OBJECT_FUNCTION_NAME (obj, desc, queue);
        }
+       return FALSE;
 }
 
 #undef COPY_OR_MARK_FUNCTION_NAME