mono/sgen/sgen-marksweep-drain-gray-stack.h

   1 /*
   2  * sgen-marksweep-drain-gray-stack.h: The copy/mark and gray stack
   3  *     draining functions of the M&S major collector.
   4  *
   5  * Copyright (C) 2014 Xamarin Inc
   6  *
   7  * This library is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU Library General Public
   9  * License 2.0 as published by the Free Software Foundation;
  10  *
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Library General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Library General Public
  17  * License 2.0 along with this library; if not, write to the Free
  18  * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  19  */
  20
  21 /*
  22  * COPY_OR_MARK_FUNCTION_NAME must be defined to be the function name of the copy/mark
  23  * function.
  24  *
  25  * SCAN_OBJECT_FUNCTION_NAME must be defined to be the function name of the object scanning
  26  * function.
  27  *
  28  * DRAIN_GRAY_STACK_FUNCTION_NAME must be defined to be the function name of the gray stack
  29  * draining function.
  30  *
  31  * Define COPY_OR_MARK_WITH_EVACUATION to support evacuation.
  32  */
  33
  34 /* Returns whether the object is still in the nursery. */
  35 static inline MONO_ALWAYS_INLINE gboolean
  36 COPY_OR_MARK_FUNCTION_NAME (GCObject **ptr, GCObject *obj, SgenGrayQueue *queue)
  37 {
  38         MSBlockInfo *block;
  39
  40 #ifdef HEAVY_STATISTICS
  41         ++stat_optimized_copy;
  42         {
  43                 char *forwarded;
  44                 SgenDescriptor desc;
  45                 if ((forwarded = SGEN_OBJECT_IS_FORWARDED (obj)))
  46                         desc = sgen_obj_get_descriptor_safe (forwarded);
  47                 else
  48                         desc = sgen_obj_get_descriptor_safe (obj);
  49
  50                 sgen_descriptor_count_copied_object (desc);
  51         }
  52 #endif
  53
  54         SGEN_ASSERT (9, obj, "null object from pointer %p", ptr);
  55 #ifndef COPY_OR_MARK_CONCURRENT
  56         SGEN_ASSERT (9, current_collection_generation == GENERATION_OLD, "old gen parallel allocator called from a %d collection", current_collection_generation);
  57 #endif
  58
  59         if (sgen_ptr_in_nursery (obj)) {
  60 #ifndef COPY_OR_MARK_CONCURRENT
  61                 int word, bit;
  62                 GCObject *forwarded, *old_obj;
  63                 mword vtable_word = *(mword*)obj;
  64
  65                 HEAVY_STAT (++stat_optimized_copy_nursery);
  66
  67 #if SGEN_MAX_DEBUG_LEVEL >= 9
  68                 if (sgen_nursery_is_to_space (obj))
  69                         SGEN_ASSERT (9, !SGEN_VTABLE_IS_PINNED (vtable_word) && !SGEN_VTABLE_IS_FORWARDED (vtable_word), "To-space object can't be pinned or forwarded.");
  70 #endif
  71
  72                 if (SGEN_VTABLE_IS_PINNED (vtable_word)) {
  73                         SGEN_ASSERT (9, !SGEN_VTABLE_IS_FORWARDED (vtable_word), "Cannot be both pinned and forwarded.");
  74                         HEAVY_STAT (++stat_optimized_copy_nursery_pinned);
  75                         return TRUE;
  76                 }
  77                 if ((forwarded = SGEN_VTABLE_IS_FORWARDED (vtable_word))) {
  78                         HEAVY_STAT (++stat_optimized_copy_nursery_forwarded);
  79                         SGEN_UPDATE_REFERENCE (ptr, forwarded);
  80                         return sgen_ptr_in_nursery (forwarded);
  81                 }
  82
  83                 /* An object in the nursery To Space has already been copied and grayed. Nothing to do. */
  84                 if (sgen_nursery_is_to_space (obj))
  85                         return TRUE;
  86
  87 #ifdef COPY_OR_MARK_WITH_EVACUATION
  88         do_copy_object:
  89 #endif
  90                 old_obj = obj;
  91                 obj = copy_object_no_checks (obj, queue);
  92                 if (G_UNLIKELY (old_obj == obj)) {
  93                         /*
  94                          * If we fail to evacuate an object we just stop doing it for a
  95                          * given block size as all other will surely fail too.
  96                          */
  97                         /* FIXME: test this case somehow. */
  98                         if (!sgen_ptr_in_nursery (obj)) {
  99                                 int size_index;
 100                                 block = MS_BLOCK_FOR_OBJ (obj);
 101                                 size_index = block->obj_size_index;
 102                                 evacuate_block_obj_sizes [size_index] = FALSE;
 103                                 MS_MARK_OBJECT_AND_ENQUEUE (obj, sgen_obj_get_descriptor (obj), block, queue);
 104                                 return FALSE;
 105                         }
 106                         return TRUE;
 107                 }
 108                 HEAVY_STAT (++stat_objects_copied_major);
 109                 SGEN_UPDATE_REFERENCE (ptr, obj);
 110
 111                 if (sgen_ptr_in_nursery (obj))
 112                         return TRUE;
 113
 114                 /*
 115                  * FIXME: See comment for copy_object_no_checks().  If
 116                  * we have that, we can let the allocation function
 117                  * give us the block info, too, and we won't have to
 118                  * re-fetch it.
 119                  *
 120                  * FIXME (2): We should rework this to avoid all those nursery checks.
 121                  */
 122                 /*
 123                  * For the split nursery allocator the object might
 124                  * still be in the nursery despite having being
 125                  * promoted, in which case we can't mark it.
 126                  */
 127                 block = MS_BLOCK_FOR_OBJ (obj);
 128                 MS_CALC_MARK_BIT (word, bit, obj);
 129                 SGEN_ASSERT (9, !MS_MARK_BIT (block, word, bit), "object %p already marked", obj);
 130                 MS_SET_MARK_BIT (block, word, bit);
 131                 binary_protocol_mark (obj, (gpointer)LOAD_VTABLE (obj), sgen_safe_object_get_size (obj));
 132
 133                 return FALSE;
 134 #endif
 135         } else {
 136                 mword vtable_word = *(mword*)obj;
 137                 SgenDescriptor desc;
 138                 int type;
 139
 140                 HEAVY_STAT (++stat_optimized_copy_major);
 141
 142 #ifdef COPY_OR_MARK_WITH_EVACUATION
 143                 {
 144                         GCObject *forwarded;
 145                         if ((forwarded = SGEN_VTABLE_IS_FORWARDED (vtable_word))) {
 146                                 HEAVY_STAT (++stat_optimized_copy_major_forwarded);
 147                                 SGEN_UPDATE_REFERENCE (ptr, forwarded);
 148                                 SGEN_ASSERT (9, !sgen_ptr_in_nursery (forwarded), "Cannot be forwarded to nursery.");
 149                                 return FALSE;
 150                         }
 151                 }
 152 #endif
 153
 154                 SGEN_ASSERT (9, !SGEN_VTABLE_IS_PINNED (vtable_word), "Pinned object in non-pinned block?");
 155
 156                 desc = sgen_vtable_get_descriptor ((GCVTable)vtable_word);
 157                 type = desc & DESC_TYPE_MASK;
 158
 159                 if (sgen_safe_object_is_small (obj, type)) {
 160 #ifdef HEAVY_STATISTICS
 161                         if (type <= DESC_TYPE_MAX_SMALL_OBJ)
 162                                 ++stat_optimized_copy_major_small_fast;
 163                         else
 164                                 ++stat_optimized_copy_major_small_slow;
 165 #endif
 166
 167                         block = MS_BLOCK_FOR_OBJ (obj);
 168
 169 #ifdef COPY_OR_MARK_WITH_EVACUATION
 170                         {
 171                                 int size_index = block->obj_size_index;
 172
 173                                 if (evacuate_block_obj_sizes [size_index] && !block->has_pinned) {
 174                                         HEAVY_STAT (++stat_optimized_copy_major_small_evacuate);
 175                                         if (block->is_to_space)
 176                                                 return FALSE;
 177                                         goto do_copy_object;
 178                                 }
 179                         }
 180 #endif
 181
 182                         MS_MARK_OBJECT_AND_ENQUEUE (obj, desc, block, queue);
 183                 } else {
 184                         HEAVY_STAT (++stat_optimized_copy_major_large);
 185
 186                         if (sgen_los_object_is_pinned (obj))
 187                                 return FALSE;
 188                         binary_protocol_pin (obj, (gpointer)SGEN_LOAD_VTABLE (obj), sgen_safe_object_get_size (obj));
 189
 190                         sgen_los_pin_object (obj);
 191                         if (SGEN_OBJECT_HAS_REFERENCES (obj))
 192                                 GRAY_OBJECT_ENQUEUE (queue, obj, sgen_obj_get_descriptor (obj));
 193                 }
 194                 return FALSE;
 195         }
 196
 197         return TRUE;
 198 }
 199
 200 static void
 201 SCAN_OBJECT_FUNCTION_NAME (GCObject *full_object, SgenDescriptor desc, SgenGrayQueue *queue)
 202 {
 203         char *start = (char*)full_object;
 204
 205 #ifdef HEAVY_STATISTICS
 206         ++stat_optimized_major_scan;
 207         if (!sgen_gc_descr_has_references (desc))
 208                 ++stat_optimized_major_scan_no_refs;
 209         sgen_descriptor_count_scanned_object (desc);
 210 #endif
 211 #ifdef SGEN_HEAVY_BINARY_PROTOCOL
 212         add_scanned_object (start);
 213 #endif
 214
 215         /* Now scan the object. */
 216
 217 #undef HANDLE_PTR
 218 #ifdef COPY_OR_MARK_CONCURRENT
 219 #define HANDLE_PTR(ptr,obj)     do {                                    \
 220                 GCObject *__old = *(ptr);                               \
 221                 binary_protocol_scan_process_reference ((obj), (ptr), __old); \
 222                 if (__old && !sgen_ptr_in_nursery (__old)) {            \
 223                         PREFETCH_READ (__old);                          \
 224                         COPY_OR_MARK_FUNCTION_NAME ((ptr), __old, queue); \
 225                 } else {                                                \
 226                         if (G_UNLIKELY (sgen_ptr_in_nursery (__old) && !sgen_ptr_in_nursery ((ptr)))) \
 227                                 mark_mod_union_card ((full_object), (void**)(ptr)); \
 228                         }                                               \
 229                 } while (0)
 230 #else
 231 #define HANDLE_PTR(ptr,obj)     do {                                    \
 232                 void *__old = *(ptr);                                   \
 233                 binary_protocol_scan_process_reference ((obj), (ptr), __old); \
 234                 if (__old) {                                            \
 235                         gboolean __still_in_nursery = COPY_OR_MARK_FUNCTION_NAME ((ptr), __old, queue); \
 236                         if (G_UNLIKELY (__still_in_nursery && !sgen_ptr_in_nursery ((ptr)) && !SGEN_OBJECT_IS_CEMENTED (*(ptr)))) { \
 237                                 void *__copy = *(ptr);                  \
 238                                 sgen_add_to_global_remset ((ptr), __copy); \
 239                         }                                               \
 240                 }                                                       \
 241         } while (0)
 242 #endif
 243
 244 #define SCAN_OBJECT_PROTOCOL
 245 #include "sgen-scan-object.h"
 246 }
 247
 248 static gboolean
 249 DRAIN_GRAY_STACK_FUNCTION_NAME (SgenGrayQueue *queue)
 250 {
 251 #ifdef COPY_OR_MARK_CONCURRENT
 252         int i;
 253         for (i = 0; i < 32; i++) {
 254 #else
 255         for (;;) {
 256 #endif
 257                 GCObject *obj;
 258                 SgenDescriptor desc;
 259
 260                 HEAVY_STAT (++stat_drain_loops);
 261
 262                 GRAY_OBJECT_DEQUEUE (queue, &obj, &desc);
 263                 if (!obj)
 264                         return TRUE;
 265
 266                 SCAN_OBJECT_FUNCTION_NAME (obj, desc, queue);
 267         }
 268         return FALSE;
 269 }
 270
 271 #undef COPY_OR_MARK_FUNCTION_NAME
 272 #undef COPY_OR_MARK_WITH_EVACUATION
 273 #undef SCAN_OBJECT_FUNCTION_NAME
 274 #undef DRAIN_GRAY_STACK_FUNCTION_NAME