mono/sgen/sgen-los.c

   1 /*
   2  * sgen-los.c: Large objects space.
   3  *
   4  * Author:
   5  *      Paolo Molaro (lupus@ximian.com)
   6  *
   7  * Copyright 2005-2010 Novell, Inc (http://www.novell.com)
   8  *
   9  * Thread start/stop adapted from Boehm's GC:
  10  * Copyright (c) 1994 by Xerox Corporation.  All rights reserved.
  11  * Copyright (c) 1996 by Silicon Graphics.  All rights reserved.
  12  * Copyright (c) 1998 by Fergus Henderson.  All rights reserved.
  13  * Copyright (c) 2000-2004 by Hewlett-Packard Company.  All rights reserved.
  14  * Copyright 2001-2003 Ximian, Inc
  15  * Copyright 2003-2010 Novell, Inc.
  16  * Copyright (C) 2012 Xamarin Inc
  17  *
  18  * Licensed under the MIT license. See LICENSE file in the project root for full license information.
  19  */
  20
  21 #include "config.h"
  22
  23 #ifdef HAVE_SGEN_GC
  24
  25 #include <string.h>
  26
  27 #include "mono/sgen/sgen-gc.h"
  28 #include "mono/sgen/sgen-protocol.h"
  29 #include "mono/sgen/sgen-cardtable.h"
  30 #include "mono/sgen/sgen-memory-governor.h"
  31 #include "mono/sgen/sgen-client.h"
  32
  33 #define LOS_SECTION_SIZE        (1024 * 1024)
  34
  35 /*
  36  * This shouldn't be much smaller or larger than MAX_SMALL_OBJ_SIZE.
  37  * Must be at least sizeof (LOSSection).
  38  */
  39 #define LOS_CHUNK_SIZE          4096
  40 #define LOS_CHUNK_BITS          12
  41
  42 /* Largest object that can be allocated in a section. */
  43 #define LOS_SECTION_OBJECT_LIMIT        (LOS_SECTION_SIZE - LOS_CHUNK_SIZE - sizeof (LOSObject))
  44 //#define LOS_SECTION_OBJECT_LIMIT      0
  45 #define LOS_SECTION_NUM_CHUNKS          ((LOS_SECTION_SIZE >> LOS_CHUNK_BITS) - 1)
  46
  47 #define LOS_SECTION_FOR_OBJ(obj)        ((LOSSection*)((mword)(obj) & ~(mword)(LOS_SECTION_SIZE - 1)))
  48 #define LOS_CHUNK_INDEX(obj,section)    (((char*)(obj) - (char*)(section)) >> LOS_CHUNK_BITS)
  49
  50 #define LOS_NUM_FAST_SIZES              32
  51
  52 typedef struct _LOSFreeChunks LOSFreeChunks;
  53 struct _LOSFreeChunks {
  54         LOSFreeChunks *next_size;
  55         size_t size;
  56 };
  57
  58 typedef struct _LOSSection LOSSection;
  59 struct _LOSSection {
  60         LOSSection *next;
  61         size_t num_free_chunks;
  62         unsigned char *free_chunk_map;
  63 };
  64
  65 /* We allow read only access on the list while sweep is not running */
  66 LOSObject *los_object_list = NULL;
  67 /* Memory used by LOS objects */
  68 mword los_memory_usage = 0;
  69 /* Total memory used by the LOS allocator */
  70 mword los_memory_usage_total = 0;
  71
  72 static LOSSection *los_sections = NULL;
  73 static LOSFreeChunks *los_fast_free_lists [LOS_NUM_FAST_SIZES]; /* 0 is for larger sizes */
  74 static mword los_num_objects = 0;
  75 static int los_num_sections = 0;
  76
  77 //#define USE_MALLOC
  78 //#define LOS_CONSISTENCY_CHECK
  79 //#define LOS_DUMMY
  80
  81 #ifdef LOS_DUMMY
  82 #define LOS_SEGMENT_SIZE        (4096 * 1024)
  83
  84 static char *los_segment = NULL;
  85 static int los_segment_index = 0;
  86 #endif
  87
  88 mword
  89 sgen_los_object_size (LOSObject *obj)
  90 {
  91         return obj->size & ~1L;
  92 }
  93
  94 #ifdef LOS_CONSISTENCY_CHECK
  95 static void
  96 los_consistency_check (void)
  97 {
  98         LOSSection *section;
  99         LOSObject *obj;
 100         int i;
 101         mword memory_usage = 0;
 102
 103         for (obj = los_object_list; obj; obj = obj->next) {
 104                 mword obj_size = sgen_los_object_size (obj);
 105                 char *end = obj->data + obj_size;
 106                 int start_index, num_chunks;
 107
 108                 memory_usage += obj_size;
 109
 110                 if (obj_size > LOS_SECTION_OBJECT_LIMIT)
 111                         continue;
 112
 113                 section = LOS_SECTION_FOR_OBJ (obj);
 114
 115                 g_assert (end <= (char*)section + LOS_SECTION_SIZE);
 116
 117                 start_index = LOS_CHUNK_INDEX (obj, section);
 118                 num_chunks = (obj_size + sizeof (LOSObject) + LOS_CHUNK_SIZE - 1) >> LOS_CHUNK_BITS;
 119                 for (i = start_index; i < start_index + num_chunks; ++i)
 120                         g_assert (!section->free_chunk_map [i]);
 121         }
 122
 123         for (i = 0; i < LOS_NUM_FAST_SIZES; ++i) {
 124                 LOSFreeChunks *size_chunks;
 125                 for (size_chunks = los_fast_free_lists [i]; size_chunks; size_chunks = size_chunks->next_size) {
 126                         LOSSection *section = LOS_SECTION_FOR_OBJ (size_chunks);
 127                         int j, num_chunks, start_index;
 128
 129                         if (i == 0)
 130                                 g_assert (size_chunks->size >= LOS_NUM_FAST_SIZES * LOS_CHUNK_SIZE);
 131                         else
 132                                 g_assert (size_chunks->size == i * LOS_CHUNK_SIZE);
 133
 134                         num_chunks = size_chunks->size >> LOS_CHUNK_BITS;
 135                         start_index = LOS_CHUNK_INDEX (size_chunks, section);
 136                         for (j = start_index; j < start_index + num_chunks; ++j)
 137                                 g_assert (section->free_chunk_map [j]);
 138                 }
 139         }
 140
 141         g_assert (los_memory_usage == memory_usage);
 142 }
 143 #endif
 144
 145 static void
 146 add_free_chunk (LOSFreeChunks *free_chunks, size_t size)
 147 {
 148         size_t num_chunks = size >> LOS_CHUNK_BITS;
 149
 150         free_chunks->size = size;
 151
 152         if (num_chunks >= LOS_NUM_FAST_SIZES)
 153                 num_chunks = 0;
 154         free_chunks->next_size = los_fast_free_lists [num_chunks];
 155         los_fast_free_lists [num_chunks] = free_chunks;
 156 }
 157
 158 static LOSFreeChunks*
 159 get_from_size_list (LOSFreeChunks **list, size_t size)
 160 {
 161         LOSFreeChunks *free_chunks = NULL;
 162         LOSSection *section;
 163         size_t i, num_chunks, start_index;
 164
 165
 166         g_assert ((size & (LOS_CHUNK_SIZE - 1)) == 0);
 167
 168         while (*list) {
 169                 free_chunks = *list;
 170                 if (free_chunks->size >= size)
 171                         break;
 172                 list = &(*list)->next_size;
 173         }
 174
 175         if (!*list)
 176                 return NULL;
 177
 178         *list = free_chunks->next_size;
 179
 180         if (free_chunks->size > size)
 181                 add_free_chunk ((LOSFreeChunks*)((char*)free_chunks + size), free_chunks->size - size);
 182
 183         num_chunks = size >> LOS_CHUNK_BITS;
 184
 185         section = LOS_SECTION_FOR_OBJ (free_chunks);
 186
 187         start_index = LOS_CHUNK_INDEX (free_chunks, section);
 188         for (i = start_index; i < start_index + num_chunks; ++i) {
 189                 g_assert (section->free_chunk_map [i]);
 190                 section->free_chunk_map [i] = 0;
 191         }
 192
 193         section->num_free_chunks -= size >> LOS_CHUNK_BITS;
 194         g_assert (section->num_free_chunks >= 0);
 195
 196         return free_chunks;
 197 }
 198
 199 static LOSObject*
 200 randomize_los_object_start (gpointer addr, size_t obj_size, size_t alloced_size, size_t addr_alignment)
 201 {
 202         size_t offset = 0;
 203         if (alloced_size != obj_size) {
 204                 /*
 205                  * We want to get a random offset between 0 and (alloced_size - obj_size)
 206                  * We do a prime multiplication to avoid usage of functions which might not
 207                  * be thread/signal safe (like rand ()). We subtract 1 to avoid common
 208                  * power by 2 factors.
 209                  */
 210                 offset = SGEN_ALIGN_DOWN ((((size_t)addr - 1) * 2654435761u) % (alloced_size - obj_size));
 211         }
 212         SGEN_ASSERT (0, (alloced_size - obj_size) < addr_alignment, "Why are we wasting one entire chunk for a los object ?");
 213         /* Randomize the location within the reserved chunks to improve cache performance */
 214         return (LOSObject*)((guint8*)addr + offset);
 215
 216 }
 217
 218 static LOSObject*
 219 get_los_section_memory (size_t size)
 220 {
 221         LOSSection *section;
 222         LOSFreeChunks *free_chunks;
 223         size_t num_chunks;
 224         size_t obj_size = size;
 225
 226         size = SGEN_ALIGN_UP_TO (size, LOS_CHUNK_SIZE);
 227
 228         num_chunks = size >> LOS_CHUNK_BITS;
 229
 230         g_assert (size > 0 && size - sizeof (LOSObject) <= LOS_SECTION_OBJECT_LIMIT);
 231         g_assert (num_chunks > 0);
 232
 233  retry:
 234         if (num_chunks >= LOS_NUM_FAST_SIZES) {
 235                 free_chunks = get_from_size_list (&los_fast_free_lists [0], size);
 236         } else {
 237                 size_t i;
 238                 for (i = num_chunks; i < LOS_NUM_FAST_SIZES; ++i) {
 239                         free_chunks = get_from_size_list (&los_fast_free_lists [i], size);
 240                         if (free_chunks)
 241                                 break;
 242                 }
 243                 if (!free_chunks)
 244                         free_chunks = get_from_size_list (&los_fast_free_lists [0], size);
 245         }
 246
 247         if (free_chunks) {
 248                 return randomize_los_object_start (free_chunks, obj_size, size, LOS_CHUNK_SIZE);
 249         }
 250
 251         if (!sgen_memgov_try_alloc_space (LOS_SECTION_SIZE, SPACE_LOS))
 252                 return NULL;
 253
 254         section = (LOSSection *)sgen_alloc_os_memory_aligned (LOS_SECTION_SIZE, LOS_SECTION_SIZE, (SgenAllocFlags)(SGEN_ALLOC_HEAP | SGEN_ALLOC_ACTIVATE), NULL);
 255
 256         if (!section)
 257                 return NULL;
 258
 259         free_chunks = (LOSFreeChunks*)((char*)section + LOS_CHUNK_SIZE);
 260         free_chunks->size = LOS_SECTION_SIZE - LOS_CHUNK_SIZE;
 261         free_chunks->next_size = los_fast_free_lists [0];
 262         los_fast_free_lists [0] = free_chunks;
 263
 264         section->num_free_chunks = LOS_SECTION_NUM_CHUNKS;
 265
 266         section->free_chunk_map = (unsigned char*)section + sizeof (LOSSection);
 267         g_assert (sizeof (LOSSection) + LOS_SECTION_NUM_CHUNKS + 1 <= LOS_CHUNK_SIZE);
 268         section->free_chunk_map [0] = 0;
 269         memset (section->free_chunk_map + 1, 1, LOS_SECTION_NUM_CHUNKS);
 270
 271         section->next = los_sections;
 272         los_sections = section;
 273
 274         los_memory_usage_total += LOS_SECTION_SIZE;
 275         ++los_num_sections;
 276
 277         goto retry;
 278 }
 279
 280 static void
 281 free_los_section_memory (LOSObject *obj, size_t size)
 282 {
 283         LOSSection *section = LOS_SECTION_FOR_OBJ (obj);
 284         size_t num_chunks, i, start_index;
 285
 286         size = SGEN_ALIGN_UP_TO (size, LOS_CHUNK_SIZE);
 287
 288         num_chunks = size >> LOS_CHUNK_BITS;
 289
 290         g_assert (size > 0 && size - sizeof (LOSObject) <= LOS_SECTION_OBJECT_LIMIT);
 291         g_assert (num_chunks > 0);
 292
 293         section->num_free_chunks += num_chunks;
 294         g_assert (section->num_free_chunks <= LOS_SECTION_NUM_CHUNKS);
 295
 296         /*
 297          * We could free the LOS section here if it's empty, but we
 298          * can't unless we also remove its free chunks from the fast
 299          * free lists.  Instead, we do it in los_sweep().
 300          */
 301
 302         start_index = LOS_CHUNK_INDEX (obj, section);
 303         for (i = start_index; i < start_index + num_chunks; ++i) {
 304                 g_assert (!section->free_chunk_map [i]);
 305                 section->free_chunk_map [i] = 1;
 306         }
 307
 308         add_free_chunk ((LOSFreeChunks*)SGEN_ALIGN_DOWN_TO ((mword)obj, LOS_CHUNK_SIZE), size);
 309 }
 310
 311 void
 312 sgen_los_free_object (LOSObject *obj)
 313 {
 314         if (obj->cardtable_mod_union)
 315                 sgen_card_table_free_mod_union (obj->cardtable_mod_union, (char*)obj->data, sgen_los_object_size (obj));
 316
 317 #ifndef LOS_DUMMY
 318         mword size = sgen_los_object_size (obj);
 319         SGEN_LOG (4, "Freed large object %p, size %lu", obj->data, (unsigned long)size);
 320         binary_protocol_empty (obj->data, size);
 321
 322         los_memory_usage -= size;
 323         los_num_objects--;
 324
 325 #ifdef USE_MALLOC
 326         free (obj);
 327 #else
 328         if (size > LOS_SECTION_OBJECT_LIMIT) {
 329                 int pagesize = mono_pagesize ();
 330                 size += sizeof (LOSObject);
 331                 size = SGEN_ALIGN_UP_TO (size, pagesize);
 332                 sgen_free_os_memory ((gpointer)SGEN_ALIGN_DOWN_TO ((mword)obj, pagesize), size, SGEN_ALLOC_HEAP);
 333                 los_memory_usage_total -= size;
 334                 sgen_memgov_release_space (size, SPACE_LOS);
 335         } else {
 336                 free_los_section_memory (obj, size + sizeof (LOSObject));
 337 #ifdef LOS_CONSISTENCY_CHECKS
 338                 los_consistency_check ();
 339 #endif
 340         }
 341 #endif
 342 #endif
 343 }
 344
 345 /*
 346  * Objects with size >= MAX_SMALL_SIZE are allocated in the large object space.
 347  * They are currently kept track of with a linked list.
 348  * They don't move, so there is no need to pin them during collection
 349  * and we avoid the memcpy overhead.
 350  */
 351 void*
 352 sgen_los_alloc_large_inner (GCVTable vtable, size_t size)
 353 {
 354         LOSObject *obj = NULL;
 355         void **vtslot;
 356
 357         g_assert (size > SGEN_MAX_SMALL_OBJ_SIZE);
 358         g_assert ((size & 1) == 0);
 359
 360         /*
 361          * size + sizeof (LOSObject) <= SSIZE_MAX - (mono_pagesize () - 1)
 362          *
 363          * therefore:
 364          *
 365          * size <= SSIZE_MAX - (mono_pagesize () - 1) - sizeof (LOSObject)
 366          */
 367         if (size > SSIZE_MAX - (mono_pagesize () - 1) - sizeof (LOSObject))
 368                 return NULL;
 369
 370 #ifdef LOS_DUMMY
 371         if (!los_segment)
 372                 los_segment = sgen_alloc_os_memory (LOS_SEGMENT_SIZE, SGEN_ALLOC_HEAP | SGEN_ALLOC_ACTIVATE, NULL);
 373         los_segment_index = ALIGN_UP (los_segment_index);
 374
 375         obj = (LOSObject*)(los_segment + los_segment_index);
 376         los_segment_index += size + sizeof (LOSObject);
 377         g_assert (los_segment_index <= LOS_SEGMENT_SIZE);
 378 #else
 379         sgen_ensure_free_space (size, GENERATION_OLD);
 380
 381 #ifdef USE_MALLOC
 382         obj = malloc (size + sizeof (LOSObject));
 383         memset (obj, 0, size + sizeof (LOSObject));
 384 #else
 385         if (size > LOS_SECTION_OBJECT_LIMIT) {
 386                 size_t obj_size = size + sizeof (LOSObject);
 387                 int pagesize = mono_pagesize ();
 388                 size_t alloc_size = SGEN_ALIGN_UP_TO (obj_size, pagesize);
 389                 if (sgen_memgov_try_alloc_space (alloc_size, SPACE_LOS)) {
 390                         obj = (LOSObject *)sgen_alloc_os_memory (alloc_size, (SgenAllocFlags)(SGEN_ALLOC_HEAP | SGEN_ALLOC_ACTIVATE), NULL);
 391                         if (obj) {
 392                                 los_memory_usage_total += alloc_size;
 393                                 obj = randomize_los_object_start (obj, obj_size, alloc_size, pagesize);
 394                         }
 395                 }
 396         } else {
 397                 obj = get_los_section_memory (size + sizeof (LOSObject));
 398                 if (obj)
 399                         memset (obj, 0, size + sizeof (LOSObject));
 400         }
 401 #endif
 402 #endif
 403         if (!obj)
 404                 return NULL;
 405         g_assert (!((mword)obj->data & (SGEN_ALLOC_ALIGN - 1)));
 406         obj->size = size;
 407         vtslot = (void**)obj->data;
 408         *vtslot = vtable;
 409         sgen_update_heap_boundaries ((mword)obj->data, (mword)obj->data + size);
 410         obj->next = los_object_list;
 411         /*
 412          * We need a memory barrier so we don't expose as head of the los object list
 413          * a LOSObject that doesn't have its fields initialized.
 414          */
 415         mono_memory_write_barrier ();
 416         los_object_list = obj;
 417         los_memory_usage += size;
 418         los_num_objects++;
 419         SGEN_LOG (4, "Allocated large object %p, vtable: %p (%s), size: %zd", obj->data, vtable, sgen_client_vtable_get_name (vtable), size);
 420         binary_protocol_alloc (obj->data, vtable, size, sgen_client_get_provenance ());
 421
 422 #ifdef LOS_CONSISTENCY_CHECK
 423         los_consistency_check ();
 424 #endif
 425
 426         return obj->data;
 427 }
 428
 429 static void sgen_los_unpin_object (GCObject *data);
 430
 431 void
 432 sgen_los_sweep (void)
 433 {
 434         LOSObject *bigobj, *prevbo;
 435         LOSSection *section, *prev;
 436         int i;
 437         int num_sections = 0;
 438
 439         /* sweep the big objects list */
 440         prevbo = NULL;
 441         for (bigobj = los_object_list; bigobj;) {
 442                 SGEN_ASSERT (0, !SGEN_OBJECT_IS_PINNED (bigobj->data), "Who pinned a LOS object?");
 443
 444                 if (sgen_los_object_is_pinned (bigobj->data)) {
 445                         if (bigobj->cardtable_mod_union) {
 446                                 mword obj_size = sgen_los_object_size (bigobj);
 447                                 mword num_cards = sgen_card_table_number_of_cards_in_range ((mword) bigobj->data, obj_size);
 448                                 memset (bigobj->cardtable_mod_union, 0, num_cards);
 449                         }
 450
 451                         sgen_los_unpin_object (bigobj->data);
 452                         sgen_update_heap_boundaries ((mword)bigobj->data, (mword)bigobj->data + sgen_los_object_size (bigobj));
 453                 } else {
 454                         LOSObject *to_free;
 455                         /* not referenced anywhere, so we can free it */
 456                         if (prevbo)
 457                                 prevbo->next = bigobj->next;
 458                         else
 459                                 los_object_list = bigobj->next;
 460                         to_free = bigobj;
 461                         bigobj = bigobj->next;
 462                         sgen_los_free_object (to_free);
 463                         continue;
 464                 }
 465                 prevbo = bigobj;
 466                 bigobj = bigobj->next;
 467         }
 468
 469         /* Try to free memory */
 470         for (i = 0; i < LOS_NUM_FAST_SIZES; ++i)
 471                 los_fast_free_lists [i] = NULL;
 472
 473         prev = NULL;
 474         section = los_sections;
 475         while (section) {
 476                 if (section->num_free_chunks == LOS_SECTION_NUM_CHUNKS) {
 477                         LOSSection *next = section->next;
 478                         if (prev)
 479                                 prev->next = next;
 480                         else
 481                                 los_sections = next;
 482                         sgen_free_os_memory (section, LOS_SECTION_SIZE, SGEN_ALLOC_HEAP);
 483                         sgen_memgov_release_space (LOS_SECTION_SIZE, SPACE_LOS);
 484                         section = next;
 485                         --los_num_sections;
 486                         los_memory_usage_total -= LOS_SECTION_SIZE;
 487                         continue;
 488                 }
 489
 490                 for (i = 0; i <= LOS_SECTION_NUM_CHUNKS; ++i) {
 491                         if (section->free_chunk_map [i]) {
 492                                 int j;
 493                                 for (j = i + 1; j <= LOS_SECTION_NUM_CHUNKS && section->free_chunk_map [j]; ++j)
 494                                         ;
 495                                 add_free_chunk ((LOSFreeChunks*)((char*)section + (i << LOS_CHUNK_BITS)), (j - i) << LOS_CHUNK_BITS);
 496                                 i = j - 1;
 497                         }
 498                 }
 499
 500                 prev = section;
 501                 section = section->next;
 502
 503                 ++num_sections;
 504         }
 505
 506 #ifdef LOS_CONSISTENCY_CHECK
 507         los_consistency_check ();
 508 #endif
 509
 510         /*
 511         g_print ("LOS sections: %d  objects: %d  usage: %d\n", num_sections, los_num_objects, los_memory_usage);
 512         for (i = 0; i < LOS_NUM_FAST_SIZES; ++i) {
 513                 int num_chunks = 0;
 514                 LOSFreeChunks *free_chunks;
 515                 for (free_chunks = los_fast_free_lists [i]; free_chunks; free_chunks = free_chunks->next_size)
 516                         ++num_chunks;
 517                 g_print ("  %d: %d\n", i, num_chunks);
 518         }
 519         */
 520
 521         g_assert (los_num_sections == num_sections);
 522 }
 523
 524 gboolean
 525 sgen_ptr_is_in_los (char *ptr, char **start)
 526 {
 527         LOSObject *obj;
 528
 529         *start = NULL;
 530         for (obj = los_object_list; obj; obj = obj->next) {
 531                 char *end = (char*)obj->data + sgen_los_object_size (obj);
 532
 533                 if (ptr >= (char*)obj->data && ptr < end) {
 534                         *start = (char*)obj->data;
 535                         return TRUE;
 536                 }
 537         }
 538         return FALSE;
 539 }
 540
 541 void
 542 sgen_los_iterate_objects (IterateObjectCallbackFunc cb, void *user_data)
 543 {
 544         LOSObject *obj;
 545
 546         for (obj = los_object_list; obj; obj = obj->next)
 547                 cb (obj->data, sgen_los_object_size (obj), user_data);
 548 }
 549
 550 gboolean
 551 sgen_los_is_valid_object (char *object)
 552 {
 553         LOSObject *obj;
 554
 555         for (obj = los_object_list; obj; obj = obj->next) {
 556                 if ((char*)obj->data == object)
 557                         return TRUE;
 558         }
 559         return FALSE;
 560 }
 561
 562 gboolean
 563 mono_sgen_los_describe_pointer (char *ptr)
 564 {
 565         LOSObject *obj;
 566
 567         for (obj = los_object_list; obj; obj = obj->next) {
 568                 const char *los_kind;
 569                 mword size;
 570                 gboolean pinned;
 571
 572                 if ((char*)obj->data > ptr || (char*)obj->data + sgen_los_object_size (obj) <= ptr)
 573                         continue;
 574
 575                 size = sgen_los_object_size (obj);
 576                 pinned = sgen_los_object_is_pinned (obj->data);
 577
 578                 if (size > LOS_SECTION_OBJECT_LIMIT)
 579                         los_kind = "huge-los-ptr";
 580                 else
 581                         los_kind = "los-ptr";
 582
 583                 if ((char*)obj->data == ptr) {
 584                         SGEN_LOG (0, "%s (size %d pin %d)\n", los_kind, (int)size, pinned ? 1 : 0);
 585                 } else {
 586                         SGEN_LOG (0, "%s (interior-ptr offset %zd size %d pin %d)",
 587                                         los_kind, ptr - (char*)obj->data, (int)size, pinned ? 1 : 0);
 588                 }
 589
 590                 return TRUE;
 591         }
 592         return FALSE;
 593 }
 594
 595 void
 596 sgen_los_iterate_live_block_ranges (sgen_cardtable_block_callback callback)
 597 {
 598         LOSObject *obj;
 599         for (obj = los_object_list; obj; obj = obj->next) {
 600                 GCVTable vt = SGEN_LOAD_VTABLE (obj->data);
 601                 if (SGEN_VTABLE_HAS_REFERENCES (vt))
 602                         callback ((mword)obj->data, sgen_los_object_size (obj));
 603         }
 604 }
 605
 606 static guint8*
 607 get_cardtable_mod_union_for_object (LOSObject *obj)
 608 {
 609         mword size = sgen_los_object_size (obj);
 610         guint8 *mod_union = obj->cardtable_mod_union;
 611         guint8 *other;
 612         if (mod_union)
 613                 return mod_union;
 614         mod_union = sgen_card_table_alloc_mod_union ((char*)obj->data, size);
 615         other = (guint8 *)SGEN_CAS_PTR ((gpointer*)&obj->cardtable_mod_union, mod_union, NULL);
 616         if (!other) {
 617                 SGEN_ASSERT (0, obj->cardtable_mod_union == mod_union, "Why did CAS not replace?");
 618                 return mod_union;
 619         }
 620         sgen_card_table_free_mod_union (mod_union, (char*)obj->data, size);
 621         return other;
 622 }
 623
 624 void
 625 sgen_los_scan_card_table (CardTableScanType scan_type, ScanCopyContext ctx)
 626 {
 627         LOSObject *obj;
 628
 629         binary_protocol_los_card_table_scan_start (sgen_timestamp (), scan_type & CARDTABLE_SCAN_MOD_UNION);
 630         for (obj = los_object_list; obj; obj = obj->next) {
 631                 mword num_cards = 0;
 632                 guint8 *cards;
 633
 634                 if (!SGEN_OBJECT_HAS_REFERENCES (obj->data))
 635                         continue;
 636
 637                 if (scan_type & CARDTABLE_SCAN_MOD_UNION) {
 638                         if (!sgen_los_object_is_pinned (obj->data))
 639                                 continue;
 640
 641                         cards = get_cardtable_mod_union_for_object (obj);
 642                         g_assert (cards);
 643                         if (scan_type == CARDTABLE_SCAN_MOD_UNION_PRECLEAN) {
 644                                 guint8 *cards_preclean;
 645                                 mword obj_size = sgen_los_object_size (obj);
 646                                 num_cards = sgen_card_table_number_of_cards_in_range ((mword) obj->data, obj_size);
 647                                 cards_preclean = (guint8 *)sgen_alloc_internal_dynamic (num_cards, INTERNAL_MEM_CARDTABLE_MOD_UNION, TRUE);
 648
 649                                 sgen_card_table_preclean_mod_union (cards, cards_preclean, num_cards);
 650
 651                                 cards = cards_preclean;
 652                         }
 653                 } else {
 654                         cards = NULL;
 655                 }
 656
 657                 sgen_cardtable_scan_object (obj->data, sgen_los_object_size (obj), cards, ctx);
 658
 659                 if (scan_type == CARDTABLE_SCAN_MOD_UNION_PRECLEAN)
 660                         sgen_free_internal_dynamic (cards, num_cards, INTERNAL_MEM_CARDTABLE_MOD_UNION);
 661         }
 662         binary_protocol_los_card_table_scan_end (sgen_timestamp (), scan_type & CARDTABLE_SCAN_MOD_UNION);
 663 }
 664
 665 void
 666 sgen_los_count_cards (long long *num_total_cards, long long *num_marked_cards)
 667 {
 668         LOSObject *obj;
 669         long long total_cards = 0;
 670         long long marked_cards = 0;
 671
 672         for (obj = los_object_list; obj; obj = obj->next) {
 673                 int i;
 674                 guint8 *cards = sgen_card_table_get_card_scan_address ((mword) obj->data);
 675                 guint8 *cards_end = sgen_card_table_get_card_scan_address ((mword) obj->data + sgen_los_object_size (obj) - 1);
 676                 mword num_cards = (cards_end - cards) + 1;
 677
 678                 if (!SGEN_OBJECT_HAS_REFERENCES (obj->data))
 679                         continue;
 680
 681                 total_cards += num_cards;
 682                 for (i = 0; i < num_cards; ++i) {
 683                         if (cards [i])
 684                                 ++marked_cards;
 685                 }
 686         }
 687
 688         *num_total_cards = total_cards;
 689         *num_marked_cards = marked_cards;
 690 }
 691
 692 void
 693 sgen_los_update_cardtable_mod_union (void)
 694 {
 695         LOSObject *obj;
 696
 697         for (obj = los_object_list; obj; obj = obj->next) {
 698                 if (!SGEN_OBJECT_HAS_REFERENCES (obj->data))
 699                         continue;
 700                 sgen_card_table_update_mod_union (get_cardtable_mod_union_for_object (obj),
 701                                 (char*)obj->data, sgen_los_object_size (obj), NULL);
 702         }
 703 }
 704
 705 LOSObject*
 706 sgen_los_header_for_object (GCObject *data)
 707 {
 708 #if _MSC_VER
 709         return (LOSObject*)((char*)data - (int)(&(((LOSObject*)0)->data)));
 710 #else
 711         return (LOSObject*)((char*)data - sizeof (LOSObject));
 712 #endif
 713 }
 714
 715 void
 716 sgen_los_pin_object (GCObject *data)
 717 {
 718         LOSObject *obj = sgen_los_header_for_object (data);
 719         obj->size = obj->size | 1;
 720         binary_protocol_pin (data, (gpointer)SGEN_LOAD_VTABLE (data), sgen_safe_object_get_size (data));
 721 }
 722
 723 static void
 724 sgen_los_unpin_object (GCObject *data)
 725 {
 726         LOSObject *obj = sgen_los_header_for_object (data);
 727         obj->size = sgen_los_object_size (obj);
 728 }
 729
 730 gboolean
 731 sgen_los_object_is_pinned (GCObject *data)
 732 {
 733         LOSObject *obj = sgen_los_header_for_object (data);
 734         return obj->size & 1;
 735 }
 736
 737 void
 738 sgen_los_mark_mod_union_card (GCObject *mono_obj, void **ptr)
 739 {
 740         LOSObject *obj = sgen_los_header_for_object (mono_obj);
 741         guint8 *mod_union = get_cardtable_mod_union_for_object (obj);
 742         /* The LOSObject structure is not represented within the card space */
 743         size_t offset = sgen_card_table_get_card_offset ((char*)ptr, (char*)sgen_card_table_align_pointer((char*)mono_obj));
 744         SGEN_ASSERT (0, mod_union, "FIXME: optionally allocate the mod union if it's not here and CAS it in.");
 745         mod_union [offset] = 1;
 746 }
 747
 748 #endif /* HAVE_SGEN_GC */