mono/sgen/sgen-marksweep-drain-gray-stack.h

   1 /*
   2  * sgen-marksweep-drain-gray-stack.h: The copy/mark and gray stack
   3  *     draining functions of the M&S major collector.
   4  *
   5  * Copyright (C) 2014 Xamarin Inc
   6  *
   7  * Licensed under the MIT license. See LICENSE file in the project root for full license information.
   8  */
   9
  10 /*
  11  * COPY_OR_MARK_FUNCTION_NAME must be defined to be the function name of the copy/mark
  12  * function.
  13  *
  14  * SCAN_OBJECT_FUNCTION_NAME must be defined to be the function name of the object scanning
  15  * function.
  16  *
  17  * DRAIN_GRAY_STACK_FUNCTION_NAME must be defined to be the function name of the gray stack
  18  * draining function.
  19  *
  20  * Define COPY_OR_MARK_WITH_EVACUATION to support evacuation.
  21  */
  22
  23 /* Returns whether the object is still in the nursery. */
  24 static inline MONO_ALWAYS_INLINE gboolean
  25 COPY_OR_MARK_FUNCTION_NAME (GCObject **ptr, GCObject *obj, SgenGrayQueue *queue)
  26 {
  27         MSBlockInfo *block;
  28
  29 #ifdef HEAVY_STATISTICS
  30         ++stat_optimized_copy;
  31         {
  32                 GCObject *forwarded;
  33                 SgenDescriptor desc;
  34                 if ((forwarded = SGEN_OBJECT_IS_FORWARDED (obj)))
  35                         desc = sgen_obj_get_descriptor_safe (forwarded);
  36                 else
  37                         desc = sgen_obj_get_descriptor_safe (obj);
  38
  39                 sgen_descriptor_count_copied_object (desc);
  40         }
  41 #endif
  42
  43         SGEN_ASSERT (9, obj, "null object from pointer %p", ptr);
  44 #if !defined(COPY_OR_MARK_CONCURRENT) && !defined(COPY_OR_MARK_CONCURRENT_WITH_EVACUATION)
  45         SGEN_ASSERT (9, current_collection_generation == GENERATION_OLD, "old gen parallel allocator called from a %d collection", current_collection_generation);
  46 #endif
  47
  48         if (sgen_ptr_in_nursery (obj)) {
  49 #if !defined(COPY_OR_MARK_CONCURRENT) && !defined(COPY_OR_MARK_CONCURRENT_WITH_EVACUATION)
  50                 int word, bit;
  51                 GCObject *forwarded, *old_obj;
  52                 mword vtable_word = *(mword*)obj;
  53
  54                 HEAVY_STAT (++stat_optimized_copy_nursery);
  55
  56 #if SGEN_MAX_DEBUG_LEVEL >= 9
  57                 if (sgen_nursery_is_to_space (obj))
  58                         SGEN_ASSERT (9, !SGEN_VTABLE_IS_PINNED (vtable_word) && !SGEN_VTABLE_IS_FORWARDED (vtable_word), "To-space object can't be pinned or forwarded.");
  59 #endif
  60
  61                 if (SGEN_VTABLE_IS_PINNED (vtable_word)) {
  62                         SGEN_ASSERT (9, !SGEN_VTABLE_IS_FORWARDED (vtable_word), "Cannot be both pinned and forwarded.");
  63                         HEAVY_STAT (++stat_optimized_copy_nursery_pinned);
  64                         return TRUE;
  65                 }
  66                 if ((forwarded = (GCObject *)SGEN_VTABLE_IS_FORWARDED (vtable_word))) {
  67                         HEAVY_STAT (++stat_optimized_copy_nursery_forwarded);
  68                         SGEN_UPDATE_REFERENCE (ptr, forwarded);
  69                         return sgen_ptr_in_nursery (forwarded);
  70                 }
  71
  72                 /* An object in the nursery To Space has already been copied and grayed. Nothing to do. */
  73                 if (sgen_nursery_is_to_space (obj))
  74                         return TRUE;
  75
  76 #ifdef COPY_OR_MARK_WITH_EVACUATION
  77         do_copy_object:
  78 #endif
  79                 old_obj = obj;
  80                 obj = copy_object_no_checks (obj, queue);
  81                 if (G_UNLIKELY (old_obj == obj)) {
  82                         /*
  83                          * If we fail to evacuate an object we just stop doing it for a
  84                          * given block size as all other will surely fail too.
  85                          */
  86                         /* FIXME: test this case somehow. */
  87                         if (!sgen_ptr_in_nursery (obj)) {
  88                                 int size_index;
  89                                 block = MS_BLOCK_FOR_OBJ (obj);
  90                                 size_index = block->obj_size_index;
  91                                 evacuate_block_obj_sizes [size_index] = FALSE;
  92                                 MS_MARK_OBJECT_AND_ENQUEUE (obj, sgen_obj_get_descriptor (obj), block, queue);
  93                                 return FALSE;
  94                         }
  95                         return TRUE;
  96                 }
  97                 HEAVY_STAT (++stat_objects_copied_major);
  98                 SGEN_UPDATE_REFERENCE (ptr, obj);
  99
 100                 if (sgen_ptr_in_nursery (obj))
 101                         return TRUE;
 102
 103                 /*
 104                  * FIXME: See comment for copy_object_no_checks().  If
 105                  * we have that, we can let the allocation function
 106                  * give us the block info, too, and we won't have to
 107                  * re-fetch it.
 108                  *
 109                  * FIXME (2): We should rework this to avoid all those nursery checks.
 110                  */
 111                 /*
 112                  * For the split nursery allocator the object might
 113                  * still be in the nursery despite having being
 114                  * promoted, in which case we can't mark it.
 115                  */
 116                 block = MS_BLOCK_FOR_OBJ (obj);
 117                 MS_CALC_MARK_BIT (word, bit, obj);
 118                 SGEN_ASSERT (9, !MS_MARK_BIT (block, word, bit), "object %p already marked", obj);
 119                 MS_SET_MARK_BIT (block, word, bit);
 120                 binary_protocol_mark (obj, (gpointer)LOAD_VTABLE (obj), sgen_safe_object_get_size (obj));
 121
 122                 return FALSE;
 123 #endif
 124         } else {
 125                 mword vtable_word = *(mword*)obj;
 126                 SgenDescriptor desc;
 127                 int type;
 128
 129                 HEAVY_STAT (++stat_optimized_copy_major);
 130
 131 #ifdef COPY_OR_MARK_WITH_EVACUATION
 132                 {
 133                         GCObject *forwarded;
 134                         if ((forwarded = (GCObject *)SGEN_VTABLE_IS_FORWARDED (vtable_word))) {
 135                                 HEAVY_STAT (++stat_optimized_copy_major_forwarded);
 136                                 SGEN_UPDATE_REFERENCE (ptr, forwarded);
 137                                 SGEN_ASSERT (9, !sgen_ptr_in_nursery (forwarded), "Cannot be forwarded to nursery.");
 138                                 return FALSE;
 139                         }
 140                 }
 141 #endif
 142
 143                 SGEN_ASSERT (9, !SGEN_VTABLE_IS_PINNED (vtable_word), "Pinned object in non-pinned block?");
 144
 145                 desc = sgen_vtable_get_descriptor ((GCVTable)vtable_word);
 146                 type = desc & DESC_TYPE_MASK;
 147
 148                 if (sgen_safe_object_is_small (obj, type)) {
 149 #ifdef HEAVY_STATISTICS
 150                         if (type <= DESC_TYPE_MAX_SMALL_OBJ)
 151                                 ++stat_optimized_copy_major_small_fast;
 152                         else
 153                                 ++stat_optimized_copy_major_small_slow;
 154 #endif
 155
 156                         block = MS_BLOCK_FOR_OBJ (obj);
 157
 158 #ifdef COPY_OR_MARK_CONCURRENT_WITH_EVACUATION
 159                         if (G_UNLIKELY (major_block_is_evacuating (block))) {
 160                                 /*
 161                                  * We don't copy within the concurrent phase. These objects will
 162                                  * be handled below in the finishing pause, by scanning the mod-union
 163                                  * card table.
 164                                  */
 165                                 return FALSE;
 166                         }
 167 #endif
 168
 169 #ifdef COPY_OR_MARK_WITH_EVACUATION
 170                         if (major_block_is_evacuating (block)) {
 171                                 HEAVY_STAT (++stat_optimized_copy_major_small_evacuate);
 172                                 goto do_copy_object;
 173                         }
 174 #endif
 175
 176                         MS_MARK_OBJECT_AND_ENQUEUE (obj, desc, block, queue);
 177                 } else {
 178                         HEAVY_STAT (++stat_optimized_copy_major_large);
 179
 180                         if (sgen_los_object_is_pinned (obj))
 181                                 return FALSE;
 182                         binary_protocol_pin (obj, (gpointer)SGEN_LOAD_VTABLE (obj), sgen_safe_object_get_size (obj));
 183
 184                         sgen_los_pin_object (obj);
 185                         if (SGEN_OBJECT_HAS_REFERENCES (obj))
 186                                 GRAY_OBJECT_ENQUEUE (queue, obj, sgen_obj_get_descriptor (obj));
 187                 }
 188                 return FALSE;
 189         }
 190
 191         return TRUE;
 192 }
 193
 194 static void
 195 SCAN_OBJECT_FUNCTION_NAME (GCObject *full_object, SgenDescriptor desc, SgenGrayQueue *queue)
 196 {
 197         char *start = (char*)full_object;
 198
 199 #ifdef HEAVY_STATISTICS
 200         ++stat_optimized_major_scan;
 201         if (!sgen_gc_descr_has_references (desc))
 202                 ++stat_optimized_major_scan_no_refs;
 203         sgen_descriptor_count_scanned_object (desc);
 204 #endif
 205 #ifdef SGEN_HEAVY_BINARY_PROTOCOL
 206         add_scanned_object (start);
 207 #endif
 208
 209         /* Now scan the object. */
 210
 211 #undef HANDLE_PTR
 212 #if defined(COPY_OR_MARK_CONCURRENT_WITH_EVACUATION)
 213 #define HANDLE_PTR(ptr,obj)     do {                                    \
 214                 GCObject *__old = *(ptr);                               \
 215                 binary_protocol_scan_process_reference ((full_object), (ptr), __old); \
 216                 if (__old && !sgen_ptr_in_nursery (__old)) {            \
 217                         if (G_UNLIKELY (!sgen_ptr_in_nursery (ptr) &&   \
 218                                         sgen_safe_object_is_small (__old, sgen_obj_get_descriptor (__old) & DESC_TYPE_MASK) && \
 219                                         major_block_is_evacuating (MS_BLOCK_FOR_OBJ (__old)))) { \
 220                                 mark_mod_union_card ((full_object), (void**)(ptr), __old); \
 221                         } else {                                        \
 222                                 PREFETCH_READ (__old);                  \
 223                                 COPY_OR_MARK_FUNCTION_NAME ((ptr), __old, queue); \
 224                         }                                               \
 225                 } else {                                                \
 226                         if (G_UNLIKELY (sgen_ptr_in_nursery (__old) && !sgen_ptr_in_nursery ((ptr)))) \
 227                                 mark_mod_union_card ((full_object), (void**)(ptr), __old); \
 228                         }                                               \
 229                 } while (0)
 230 #elif defined(COPY_OR_MARK_CONCURRENT)
 231 #define HANDLE_PTR(ptr,obj)     do {                                    \
 232                 GCObject *__old = *(ptr);                               \
 233                 binary_protocol_scan_process_reference ((full_object), (ptr), __old); \
 234                 if (__old && !sgen_ptr_in_nursery (__old)) {            \
 235                         PREFETCH_READ (__old);                  \
 236                         COPY_OR_MARK_FUNCTION_NAME ((ptr), __old, queue); \
 237                 } else {                                                \
 238                         if (G_UNLIKELY (sgen_ptr_in_nursery (__old) && !sgen_ptr_in_nursery ((ptr)))) \
 239                                 mark_mod_union_card ((full_object), (void**)(ptr), __old); \
 240                         }                                               \
 241                 } while (0)
 242 #else
 243 #define HANDLE_PTR(ptr,obj)     do {                                    \
 244                 GCObject *__old = *(ptr);                                       \
 245                 binary_protocol_scan_process_reference ((full_object), (ptr), __old); \
 246                 if (__old) {                                            \
 247                         gboolean __still_in_nursery = COPY_OR_MARK_FUNCTION_NAME ((ptr), __old, queue); \
 248                         if (G_UNLIKELY (__still_in_nursery && !sgen_ptr_in_nursery ((ptr)) && !SGEN_OBJECT_IS_CEMENTED (*(ptr)))) { \
 249                                 GCObject *__copy = *(ptr);                      \
 250                                 sgen_add_to_global_remset ((ptr), __copy); \
 251                         }                                               \
 252                 }                                                       \
 253         } while (0)
 254 #endif
 255
 256 #define SCAN_OBJECT_PROTOCOL
 257 #include "sgen-scan-object.h"
 258 }
 259
 260 #ifdef SCAN_VTYPE_FUNCTION_NAME
 261 static void
 262 SCAN_VTYPE_FUNCTION_NAME (GCObject *full_object, char *start, SgenDescriptor desc, SgenGrayQueue *queue BINARY_PROTOCOL_ARG (size_t size))
 263 {
 264         SGEN_OBJECT_LAYOUT_STATISTICS_DECLARE_BITMAP;
 265
 266 #ifdef HEAVY_STATISTICS
 267         /* FIXME: We're half scanning this object.  How do we account for that? */
 268         //add_scanned_object (start);
 269 #endif
 270
 271         /* The descriptors include info about the object header as well */
 272         start -= SGEN_CLIENT_OBJECT_HEADER_SIZE;
 273
 274         /* We use the same HANDLE_PTR from the obj scan function */
 275 #define SCAN_OBJECT_NOVTABLE
 276 #define SCAN_OBJECT_PROTOCOL
 277 #include "sgen-scan-object.h"
 278
 279         SGEN_OBJECT_LAYOUT_STATISTICS_COMMIT_BITMAP;
 280 }
 281 #endif
 282
 283 #ifdef SCAN_PTR_FIELD_FUNCTION_NAME
 284 static void
 285 SCAN_PTR_FIELD_FUNCTION_NAME (GCObject *full_object, GCObject **ptr, SgenGrayQueue *queue)
 286 {
 287         HANDLE_PTR (ptr, NULL);
 288 }
 289 #endif
 290
 291 static gboolean
 292 DRAIN_GRAY_STACK_FUNCTION_NAME (SgenGrayQueue *queue)
 293 {
 294 #if defined(COPY_OR_MARK_CONCURRENT) || defined(COPY_OR_MARK_CONCURRENT_WITH_EVACUATION)
 295         int i;
 296         for (i = 0; i < 32; i++) {
 297 #else
 298         for (;;) {
 299 #endif
 300                 GCObject *obj;
 301                 SgenDescriptor desc;
 302
 303                 HEAVY_STAT (++stat_drain_loops);
 304
 305                 GRAY_OBJECT_DEQUEUE (queue, &obj, &desc);
 306                 if (!obj)
 307                         return TRUE;
 308
 309                 SCAN_OBJECT_FUNCTION_NAME (obj, desc, queue);
 310         }
 311         return FALSE;
 312 }
 313
 314 #undef COPY_OR_MARK_FUNCTION_NAME
 315 #undef COPY_OR_MARK_WITH_EVACUATION
 316 #undef COPY_OR_MARK_CONCURRENT
 317 #undef COPY_OR_MARK_CONCURRENT_WITH_EVACUATION
 318 #undef SCAN_OBJECT_FUNCTION_NAME
 319 #undef SCAN_VTYPE_FUNCTION_NAME
 320 #undef SCAN_PTR_FIELD_FUNCTION_NAME
 321 #undef DRAIN_GRAY_STACK_FUNCTION_NAME