mono/sgen/sgen-marksweep-drain-gray-stack.h

   1 /*
   2  * sgen-marksweep-drain-gray-stack.h: The copy/mark and gray stack
   3  *     draining functions of the M&S major collector.
   4  *
   5  * Copyright (C) 2014 Xamarin Inc
   6  *
   7  * This library is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU Library General Public
   9  * License 2.0 as published by the Free Software Foundation;
  10  *
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Library General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Library General Public
  17  * License 2.0 along with this library; if not, write to the Free
  18  * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  19  */
  20
  21 /*
  22  * COPY_OR_MARK_FUNCTION_NAME must be defined to be the function name of the copy/mark
  23  * function.
  24  *
  25  * SCAN_OBJECT_FUNCTION_NAME must be defined to be the function name of the object scanning
  26  * function.
  27  *
  28  * DRAIN_GRAY_STACK_FUNCTION_NAME must be defined to be the function name of the gray stack
  29  * draining function.
  30  *
  31  * Define COPY_OR_MARK_WITH_EVACUATION to support evacuation.
  32  */
  33
  34 /* Returns whether the object is still in the nursery. */
  35 static inline MONO_ALWAYS_INLINE gboolean
  36 COPY_OR_MARK_FUNCTION_NAME (GCObject **ptr, GCObject *obj, SgenGrayQueue *queue)
  37 {
  38         MSBlockInfo *block;
  39
  40 #ifdef HEAVY_STATISTICS
  41         ++stat_optimized_copy;
  42         {
  43                 char *forwarded;
  44                 SgenDescriptor desc;
  45                 if ((forwarded = SGEN_OBJECT_IS_FORWARDED (obj)))
  46                         desc = sgen_obj_get_descriptor_safe (forwarded);
  47                 else
  48                         desc = sgen_obj_get_descriptor_safe (obj);
  49
  50                 sgen_descriptor_count_copied_object (desc);
  51         }
  52 #endif
  53
  54         SGEN_ASSERT (9, obj, "null object from pointer %p", ptr);
  55 #ifndef COPY_OR_MARK_CONCURRENT
  56         SGEN_ASSERT (9, current_collection_generation == GENERATION_OLD, "old gen parallel allocator called from a %d collection", current_collection_generation);
  57 #endif
  58
  59         if (sgen_ptr_in_nursery (obj)) {
  60 #ifndef COPY_OR_MARK_CONCURRENT
  61                 int word, bit;
  62                 GCObject *forwarded, *old_obj;
  63                 mword vtable_word = *(mword*)obj;
  64
  65                 HEAVY_STAT (++stat_optimized_copy_nursery);
  66
  67 #if SGEN_MAX_DEBUG_LEVEL >= 9
  68                 if (sgen_nursery_is_to_space (obj))
  69                         SGEN_ASSERT (9, !SGEN_VTABLE_IS_PINNED (vtable_word) && !SGEN_VTABLE_IS_FORWARDED (vtable_word), "To-space object can't be pinned or forwarded.");
  70 #endif
  71
  72                 if (SGEN_VTABLE_IS_PINNED (vtable_word)) {
  73                         SGEN_ASSERT (9, !SGEN_VTABLE_IS_FORWARDED (vtable_word), "Cannot be both pinned and forwarded.");
  74                         HEAVY_STAT (++stat_optimized_copy_nursery_pinned);
  75                         return TRUE;
  76                 }
  77                 if ((forwarded = (GCObject *)SGEN_VTABLE_IS_FORWARDED (vtable_word))) {
  78                         HEAVY_STAT (++stat_optimized_copy_nursery_forwarded);
  79                         SGEN_UPDATE_REFERENCE (ptr, forwarded);
  80                         return sgen_ptr_in_nursery (forwarded);
  81                 }
  82
  83                 /* An object in the nursery To Space has already been copied and grayed. Nothing to do. */
  84                 if (sgen_nursery_is_to_space (obj))
  85                         return TRUE;
  86
  87 #ifdef COPY_OR_MARK_WITH_EVACUATION
  88         do_copy_object:
  89 #endif
  90                 old_obj = obj;
  91                 obj = copy_object_no_checks (obj, queue);
  92                 if (G_UNLIKELY (old_obj == obj)) {
  93                         /*
  94                          * If we fail to evacuate an object we just stop doing it for a
  95                          * given block size as all other will surely fail too.
  96                          */
  97                         /* FIXME: test this case somehow. */
  98                         if (!sgen_ptr_in_nursery (obj)) {
  99                                 int size_index;
 100                                 block = MS_BLOCK_FOR_OBJ (obj);
 101                                 size_index = block->obj_size_index;
 102                                 evacuate_block_obj_sizes [size_index] = FALSE;
 103                                 MS_MARK_OBJECT_AND_ENQUEUE (obj, sgen_obj_get_descriptor (obj), block, queue);
 104                                 return FALSE;
 105                         }
 106                         return TRUE;
 107                 }
 108                 HEAVY_STAT (++stat_objects_copied_major);
 109                 SGEN_UPDATE_REFERENCE (ptr, obj);
 110
 111                 if (sgen_ptr_in_nursery (obj))
 112                         return TRUE;
 113
 114                 /*
 115                  * FIXME: See comment for copy_object_no_checks().  If
 116                  * we have that, we can let the allocation function
 117                  * give us the block info, too, and we won't have to
 118                  * re-fetch it.
 119                  *
 120                  * FIXME (2): We should rework this to avoid all those nursery checks.
 121                  */
 122                 /*
 123                  * For the split nursery allocator the object might
 124                  * still be in the nursery despite having being
 125                  * promoted, in which case we can't mark it.
 126                  */
 127                 block = MS_BLOCK_FOR_OBJ (obj);
 128                 MS_CALC_MARK_BIT (word, bit, obj);
 129                 SGEN_ASSERT (9, !MS_MARK_BIT (block, word, bit), "object %p already marked", obj);
 130                 MS_SET_MARK_BIT (block, word, bit);
 131                 binary_protocol_mark (obj, (gpointer)LOAD_VTABLE (obj), sgen_safe_object_get_size (obj));
 132
 133                 return FALSE;
 134 #endif
 135         } else {
 136                 mword vtable_word = *(mword*)obj;
 137                 SgenDescriptor desc;
 138                 int type;
 139
 140                 HEAVY_STAT (++stat_optimized_copy_major);
 141
 142 #ifdef COPY_OR_MARK_WITH_EVACUATION
 143                 {
 144                         GCObject *forwarded;
 145                         if ((forwarded = (GCObject *)SGEN_VTABLE_IS_FORWARDED (vtable_word))) {
 146                                 HEAVY_STAT (++stat_optimized_copy_major_forwarded);
 147                                 SGEN_UPDATE_REFERENCE (ptr, forwarded);
 148                                 SGEN_ASSERT (9, !sgen_ptr_in_nursery (forwarded), "Cannot be forwarded to nursery.");
 149                                 return FALSE;
 150                         }
 151                 }
 152 #endif
 153
 154                 SGEN_ASSERT (9, !SGEN_VTABLE_IS_PINNED (vtable_word), "Pinned object in non-pinned block?");
 155
 156                 desc = sgen_vtable_get_descriptor ((GCVTable)vtable_word);
 157                 type = desc & DESC_TYPE_MASK;
 158
 159                 if (sgen_safe_object_is_small (obj, type)) {
 160 #ifdef HEAVY_STATISTICS
 161                         if (type <= DESC_TYPE_MAX_SMALL_OBJ)
 162                                 ++stat_optimized_copy_major_small_fast;
 163                         else
 164                                 ++stat_optimized_copy_major_small_slow;
 165 #endif
 166
 167                         block = MS_BLOCK_FOR_OBJ (obj);
 168
 169 #ifdef COPY_OR_MARK_CONCURRENT
 170                         if (G_UNLIKELY (major_block_is_evacuating (block))) {
 171                                 /*
 172                                  * We don't copy within the concurrent phase. These objects will
 173                                  * be handled below in the finishing pause, by scanning the mod-union
 174                                  * card table.
 175                                  */
 176                                 return FALSE;
 177                         }
 178 #endif
 179
 180 #ifdef COPY_OR_MARK_WITH_EVACUATION
 181                         if (major_block_is_evacuating (block)) {
 182                                 HEAVY_STAT (++stat_optimized_copy_major_small_evacuate);
 183                                 goto do_copy_object;
 184                         }
 185 #endif
 186
 187                         MS_MARK_OBJECT_AND_ENQUEUE (obj, desc, block, queue);
 188                 } else {
 189                         HEAVY_STAT (++stat_optimized_copy_major_large);
 190
 191                         if (sgen_los_object_is_pinned (obj))
 192                                 return FALSE;
 193                         binary_protocol_pin (obj, (gpointer)SGEN_LOAD_VTABLE (obj), sgen_safe_object_get_size (obj));
 194
 195                         sgen_los_pin_object (obj);
 196                         if (SGEN_OBJECT_HAS_REFERENCES (obj))
 197                                 GRAY_OBJECT_ENQUEUE (queue, obj, sgen_obj_get_descriptor (obj));
 198                 }
 199                 return FALSE;
 200         }
 201
 202         return TRUE;
 203 }
 204
 205 static void
 206 SCAN_OBJECT_FUNCTION_NAME (GCObject *full_object, SgenDescriptor desc, SgenGrayQueue *queue)
 207 {
 208         char *start = (char*)full_object;
 209
 210 #ifdef HEAVY_STATISTICS
 211         ++stat_optimized_major_scan;
 212         if (!sgen_gc_descr_has_references (desc))
 213                 ++stat_optimized_major_scan_no_refs;
 214         sgen_descriptor_count_scanned_object (desc);
 215 #endif
 216 #ifdef SGEN_HEAVY_BINARY_PROTOCOL
 217         add_scanned_object (start);
 218 #endif
 219
 220         /* Now scan the object. */
 221
 222 #undef HANDLE_PTR
 223 #ifdef COPY_OR_MARK_CONCURRENT
 224 #define HANDLE_PTR(ptr,obj)     do {                                    \
 225                 GCObject *__old = *(ptr);                               \
 226                 binary_protocol_scan_process_reference ((full_object), (ptr), __old); \
 227                 if (__old && !sgen_ptr_in_nursery (__old)) {            \
 228                         MSBlockInfo *block = MS_BLOCK_FOR_OBJ (__old);  \
 229                         if (G_UNLIKELY (!sgen_ptr_in_nursery (ptr) &&   \
 230                                         sgen_safe_object_is_small (__old, sgen_obj_get_descriptor (__old) & DESC_TYPE_MASK) && \
 231                                         major_block_is_evacuating (block))) { \
 232                                 mark_mod_union_card ((full_object), (void**)(ptr), __old); \
 233                         } else {                                        \
 234                                 PREFETCH_READ (__old);                  \
 235                                 COPY_OR_MARK_FUNCTION_NAME ((ptr), __old, queue); \
 236                         }                                               \
 237                 } else {                                                \
 238                         if (G_UNLIKELY (sgen_ptr_in_nursery (__old) && !sgen_ptr_in_nursery ((ptr)))) \
 239                                 mark_mod_union_card ((full_object), (void**)(ptr), __old); \
 240                         }                                               \
 241                 } while (0)
 242 #else
 243 #define HANDLE_PTR(ptr,obj)     do {                                    \
 244                 GCObject *__old = *(ptr);                                       \
 245                 binary_protocol_scan_process_reference ((full_object), (ptr), __old); \
 246                 if (__old) {                                            \
 247                         gboolean __still_in_nursery = COPY_OR_MARK_FUNCTION_NAME ((ptr), __old, queue); \
 248                         if (G_UNLIKELY (__still_in_nursery && !sgen_ptr_in_nursery ((ptr)) && !SGEN_OBJECT_IS_CEMENTED (*(ptr)))) { \
 249                                 GCObject *__copy = *(ptr);                      \
 250                                 sgen_add_to_global_remset ((ptr), __copy); \
 251                         }                                               \
 252                 }                                                       \
 253         } while (0)
 254 #endif
 255
 256 #define SCAN_OBJECT_PROTOCOL
 257 #include "sgen-scan-object.h"
 258 }
 259
 260 static gboolean
 261 DRAIN_GRAY_STACK_FUNCTION_NAME (SgenGrayQueue *queue)
 262 {
 263 #ifdef COPY_OR_MARK_CONCURRENT
 264         int i;
 265         for (i = 0; i < 32; i++) {
 266 #else
 267         for (;;) {
 268 #endif
 269                 GCObject *obj;
 270                 SgenDescriptor desc;
 271
 272                 HEAVY_STAT (++stat_drain_loops);
 273
 274                 GRAY_OBJECT_DEQUEUE (queue, &obj, &desc);
 275                 if (!obj)
 276                         return TRUE;
 277
 278                 SCAN_OBJECT_FUNCTION_NAME (obj, desc, queue);
 279         }
 280         return FALSE;
 281 }
 282
 283 #undef COPY_OR_MARK_FUNCTION_NAME
 284 #undef COPY_OR_MARK_WITH_EVACUATION
 285 #undef SCAN_OBJECT_FUNCTION_NAME
 286 #undef DRAIN_GRAY_STACK_FUNCTION_NAME