Sanitize assertions in the unmanaged object allocator.

[mono.git] / mono / metadata / sgen-gc.c

diff --git a/mono/metadata/sgen-gc.c b/mono/metadata/sgen-gc.c
index b916ec5f3cd66d0d9addb54b4060a159e13e92b1..f6c5b89b380e4ce1574e7a8c0c5eec8c0d967b92 100644 (file)
--- a/mono/metadata/sgen-gc.c
+++ b/mono/metadata/sgen-gc.c
@@ -3609,8 +3609,6 @@ mono_gc_alloc_obj_nolock (MonoVTable *vtable, size_t size)
                        g_assert (*p == NULL);
                        *p = vtable;
 
-                       g_assert (TLAB_NEXT == new_next);
-
                        return p;
                }
 
@@ -3625,8 +3623,8 @@ mono_gc_alloc_obj_nolock (MonoVTable *vtable, size_t size)
                 * This avoids taking again the GC lock when registering, but this is moot when
                 * doing thread-local allocation, so it may not be a good idea.
                 */
-               g_assert (TLAB_NEXT == new_next);
                if (TLAB_NEXT >= TLAB_REAL_END) {
+                       int available_in_tlab;
                        /* 
                         * Run out of space in the TLAB. When this happens, some amount of space
                         * remains in the TLAB, but not enough to satisfy the current allocation
@@ -3644,7 +3642,8 @@ mono_gc_alloc_obj_nolock (MonoVTable *vtable, size_t size)
                                return p;
                        }
 
-                       if (size > tlab_size) {
+                       available_in_tlab = TLAB_REAL_END - TLAB_NEXT;
+                       if (size > tlab_size || available_in_tlab > SGEN_MAX_NURSERY_WASTE) {
                                /* Allocate directly from the nursery */
                                do {
                                        p = mono_sgen_nursery_alloc (size);
@@ -3671,7 +3670,7 @@ mono_gc_alloc_obj_nolock (MonoVTable *vtable, size_t size)
                                int alloc_size = 0;
                                if (TLAB_START)
                                        DEBUG (3, fprintf (gc_debug_file, "Retire TLAB: %p-%p [%ld]\n", TLAB_START, TLAB_REAL_END, (long)(TLAB_REAL_END - TLAB_NEXT - size)));
-                               mono_sgen_nursery_retire_region (p, TLAB_REAL_END - (char*)p);
+                               mono_sgen_nursery_retire_region (p, available_in_tlab);
 
                                do {
                                        p = mono_sgen_nursery_alloc_range (tlab_size, size, &alloc_size);
@@ -3705,7 +3704,6 @@ mono_gc_alloc_obj_nolock (MonoVTable *vtable, size_t size)
                                /* Allocate from the TLAB */
                                p = (void*)TLAB_NEXT;
                                TLAB_NEXT += size;
-                               g_assert (TLAB_NEXT <= TLAB_REAL_END);
 
                                nursery_section->scan_starts [((char*)p - (char*)nursery_section->data)/SCAN_START_SIZE] = (char*)p;
                        }
@@ -3750,26 +3748,37 @@ mono_gc_try_alloc_obj_nolock (MonoVTable *vtable, size_t size)
 
                if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
                        memset (p, 0, size);
-               g_assert (*p == NULL);
        } else {
+               int available_in_tlab;
+               char *real_end;
                /* tlab_next and tlab_temp_end are TLS vars so accessing them might be expensive */
 
                p = (void**)TLAB_NEXT;
                /* FIXME: handle overflow */
                new_next = (char*)p + size;
-               TLAB_NEXT = new_next;
 
-               /*First case, we overflowed the tlab, get a new one*/
-               if (G_UNLIKELY (new_next >= TLAB_REAL_END)) {
+               real_end = TLAB_REAL_END;
+               available_in_tlab = real_end - (char*)p;
+
+               if (G_LIKELY (new_next < real_end)) {
+                       TLAB_NEXT = new_next;
+               } else if (available_in_tlab > SGEN_MAX_NURSERY_WASTE) {
+                       /* Allocate directly from the nursery */
+                       p = mono_sgen_nursery_alloc (size);
+                       if (!p)
+                               return NULL;
+
+                       if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
+                               memset (p, 0, size);                    
+               } else {
                        int alloc_size = 0;
 
-                       mono_sgen_nursery_retire_region (p, TLAB_REAL_END - (char*)p);
+                       mono_sgen_nursery_retire_region (p, available_in_tlab);
                        new_next = mono_sgen_nursery_alloc_range (tlab_size, size, &alloc_size);
                        p = (void**)new_next;
                        if (!p)
                                return NULL;
 
-                       g_assert (alloc_size >= size);
                        TLAB_START = (char*)new_next;
                        TLAB_NEXT = new_next + size;
                        TLAB_REAL_END = new_next + alloc_size;
@@ -3778,10 +3787,8 @@ mono_gc_try_alloc_obj_nolock (MonoVTable *vtable, size_t size)
                        if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
                                memset (new_next, 0, alloc_size);
                        new_next += size;
-                       g_assert (*p == NULL);
                }
 
-
                /* Second case, we overflowed temp end */
                if (G_UNLIKELY (new_next >= TLAB_TEMP_END)) {
                        nursery_section->scan_starts [((char*)p - (char*)nursery_section->data)/SCAN_START_SIZE] = (char*)p;
@@ -3789,8 +3796,6 @@ mono_gc_try_alloc_obj_nolock (MonoVTable *vtable, size_t size)
                        TLAB_TEMP_END = MIN (TLAB_REAL_END, TLAB_NEXT + SCAN_START_SIZE);
                        DEBUG (5, fprintf (gc_debug_file, "Expanding local alloc: %p-%p\n", TLAB_NEXT, TLAB_TEMP_END));         
                }
-
-               g_assert (TLAB_NEXT == new_next);
        }
 
        /* 
@@ -7017,11 +7022,6 @@ create_allocator (int atype)
        mono_mb_emit_byte (mb, CEE_ADD);
        mono_mb_emit_stloc (mb, new_next_var);
 
-       /* tlab_next = new_next */
-       mono_mb_emit_ldloc (mb, tlab_next_addr_var);
-       mono_mb_emit_ldloc (mb, new_next_var);
-       mono_mb_emit_byte (mb, CEE_STIND_I);
-
        /* if (G_LIKELY (new_next < tlab_temp_end)) */
        mono_mb_emit_ldloc (mb, new_next_var);
        EMIT_TLS_ACCESS (mb, tlab_temp_end, tlab_temp_end_offset);
@@ -7052,6 +7052,11 @@ create_allocator (int atype)
 
        /* FIXME: Memory barrier */
 
+       /* tlab_next = new_next */
+       mono_mb_emit_ldloc (mb, tlab_next_addr_var);
+       mono_mb_emit_ldloc (mb, new_next_var);
+       mono_mb_emit_byte (mb, CEE_STIND_I);
+
        /* *p = vtable; */
        mono_mb_emit_ldloc (mb, p_var);
        mono_mb_emit_ldarg (mb, 0);