mono/sgen/sgen-los.c

   1 /*
   2  * sgen-los.c: Large objects space.
   3  *
   4  * Author:
   5  *      Paolo Molaro (lupus@ximian.com)
   6  *
   7  * Copyright 2005-2010 Novell, Inc (http://www.novell.com)
   8  *
   9  * Thread start/stop adapted from Boehm's GC:
  10  * Copyright (c) 1994 by Xerox Corporation.  All rights reserved.
  11  * Copyright (c) 1996 by Silicon Graphics.  All rights reserved.
  12  * Copyright (c) 1998 by Fergus Henderson.  All rights reserved.
  13  * Copyright (c) 2000-2004 by Hewlett-Packard Company.  All rights reserved.
  14  * Copyright 2001-2003 Ximian, Inc
  15  * Copyright 2003-2010 Novell, Inc.
  16  * Copyright (C) 2012 Xamarin Inc
  17  *
  18  * This library is free software; you can redistribute it and/or
  19  * modify it under the terms of the GNU Library General Public
  20  * License 2.0 as published by the Free Software Foundation;
  21  *
  22  * This library is distributed in the hope that it will be useful,
  23  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  24  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  25  * Library General Public License for more details.
  26  *
  27  * You should have received a copy of the GNU Library General Public
  28  * License 2.0 along with this library; if not, write to the Free
  29  * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  30  */
  31
  32 #include "config.h"
  33
  34 #ifdef HAVE_SGEN_GC
  35
  36 #include <string.h>
  37
  38 #include "mono/sgen/sgen-gc.h"
  39 #include "mono/sgen/sgen-protocol.h"
  40 #include "mono/sgen/sgen-cardtable.h"
  41 #include "mono/sgen/sgen-memory-governor.h"
  42 #include "mono/sgen/sgen-client.h"
  43
  44 #define LOS_SECTION_SIZE        (1024 * 1024)
  45
  46 /*
  47  * This shouldn't be much smaller or larger than MAX_SMALL_OBJ_SIZE.
  48  * Must be at least sizeof (LOSSection).
  49  */
  50 #define LOS_CHUNK_SIZE          4096
  51 #define LOS_CHUNK_BITS          12
  52
  53 /* Largest object that can be allocated in a section. */
  54 #define LOS_SECTION_OBJECT_LIMIT        (LOS_SECTION_SIZE - LOS_CHUNK_SIZE - sizeof (LOSObject))
  55 //#define LOS_SECTION_OBJECT_LIMIT      0
  56 #define LOS_SECTION_NUM_CHUNKS          ((LOS_SECTION_SIZE >> LOS_CHUNK_BITS) - 1)
  57
  58 #define LOS_SECTION_FOR_OBJ(obj)        ((LOSSection*)((mword)(obj) & ~(mword)(LOS_SECTION_SIZE - 1)))
  59 #define LOS_CHUNK_INDEX(obj,section)    (((char*)(obj) - (char*)(section)) >> LOS_CHUNK_BITS)
  60
  61 #define LOS_NUM_FAST_SIZES              32
  62
  63 typedef struct _LOSFreeChunks LOSFreeChunks;
  64 struct _LOSFreeChunks {
  65         LOSFreeChunks *next_size;
  66         size_t size;
  67 };
  68
  69 typedef struct _LOSSection LOSSection;
  70 struct _LOSSection {
  71         LOSSection *next;
  72         size_t num_free_chunks;
  73         unsigned char *free_chunk_map;
  74 };
  75
  76 /* We allow read only access on the list while sweep is not running */
  77 LOSObject *los_object_list = NULL;
  78 mword los_memory_usage = 0;
  79
  80 static LOSSection *los_sections = NULL;
  81 static LOSFreeChunks *los_fast_free_lists [LOS_NUM_FAST_SIZES]; /* 0 is for larger sizes */
  82 static mword los_num_objects = 0;
  83 static int los_num_sections = 0;
  84
  85 //#define USE_MALLOC
  86 //#define LOS_CONSISTENCY_CHECK
  87 //#define LOS_DUMMY
  88
  89 #ifdef LOS_DUMMY
  90 #define LOS_SEGMENT_SIZE        (4096 * 1024)
  91
  92 static char *los_segment = NULL;
  93 static int los_segment_index = 0;
  94 #endif
  95
  96 mword
  97 sgen_los_object_size (LOSObject *obj)
  98 {
  99         return obj->size & ~1L;
 100 }
 101
 102 #ifdef LOS_CONSISTENCY_CHECK
 103 static void
 104 los_consistency_check (void)
 105 {
 106         LOSSection *section;
 107         LOSObject *obj;
 108         int i;
 109         mword memory_usage = 0;
 110
 111         for (obj = los_object_list; obj; obj = obj->next) {
 112                 mword obj_size = sgen_los_object_size (obj);
 113                 char *end = obj->data + obj_size;
 114                 int start_index, num_chunks;
 115
 116                 memory_usage += obj_size;
 117
 118                 if (obj_size > LOS_SECTION_OBJECT_LIMIT)
 119                         continue;
 120
 121                 section = LOS_SECTION_FOR_OBJ (obj);
 122
 123                 g_assert (end <= (char*)section + LOS_SECTION_SIZE);
 124
 125                 start_index = LOS_CHUNK_INDEX (obj, section);
 126                 num_chunks = (obj_size + sizeof (LOSObject) + LOS_CHUNK_SIZE - 1) >> LOS_CHUNK_BITS;
 127                 for (i = start_index; i < start_index + num_chunks; ++i)
 128                         g_assert (!section->free_chunk_map [i]);
 129         }
 130
 131         for (i = 0; i < LOS_NUM_FAST_SIZES; ++i) {
 132                 LOSFreeChunks *size_chunks;
 133                 for (size_chunks = los_fast_free_lists [i]; size_chunks; size_chunks = size_chunks->next_size) {
 134                         LOSSection *section = LOS_SECTION_FOR_OBJ (size_chunks);
 135                         int j, num_chunks, start_index;
 136
 137                         if (i == 0)
 138                                 g_assert (size_chunks->size >= LOS_NUM_FAST_SIZES * LOS_CHUNK_SIZE);
 139                         else
 140                                 g_assert (size_chunks->size == i * LOS_CHUNK_SIZE);
 141
 142                         num_chunks = size_chunks->size >> LOS_CHUNK_BITS;
 143                         start_index = LOS_CHUNK_INDEX (size_chunks, section);
 144                         for (j = start_index; j < start_index + num_chunks; ++j)
 145                                 g_assert (section->free_chunk_map [j]);
 146                 }
 147         }
 148
 149         g_assert (los_memory_usage == memory_usage);
 150 }
 151 #endif
 152
 153 static void
 154 add_free_chunk (LOSFreeChunks *free_chunks, size_t size)
 155 {
 156         size_t num_chunks = size >> LOS_CHUNK_BITS;
 157
 158         free_chunks->size = size;
 159
 160         if (num_chunks >= LOS_NUM_FAST_SIZES)
 161                 num_chunks = 0;
 162         free_chunks->next_size = los_fast_free_lists [num_chunks];
 163         los_fast_free_lists [num_chunks] = free_chunks;
 164 }
 165
 166 static LOSFreeChunks*
 167 get_from_size_list (LOSFreeChunks **list, size_t size)
 168 {
 169         LOSFreeChunks *free_chunks = NULL;
 170         LOSSection *section;
 171         size_t i, num_chunks, start_index;
 172
 173
 174         g_assert ((size & (LOS_CHUNK_SIZE - 1)) == 0);
 175
 176         while (*list) {
 177                 free_chunks = *list;
 178                 if (free_chunks->size >= size)
 179                         break;
 180                 list = &(*list)->next_size;
 181         }
 182
 183         if (!*list)
 184                 return NULL;
 185
 186         *list = free_chunks->next_size;
 187
 188         if (free_chunks->size > size)
 189                 add_free_chunk ((LOSFreeChunks*)((char*)free_chunks + size), free_chunks->size - size);
 190
 191         num_chunks = size >> LOS_CHUNK_BITS;
 192
 193         section = LOS_SECTION_FOR_OBJ (free_chunks);
 194
 195         start_index = LOS_CHUNK_INDEX (free_chunks, section);
 196         for (i = start_index; i < start_index + num_chunks; ++i) {
 197                 g_assert (section->free_chunk_map [i]);
 198                 section->free_chunk_map [i] = 0;
 199         }
 200
 201         section->num_free_chunks -= size >> LOS_CHUNK_BITS;
 202         g_assert (section->num_free_chunks >= 0);
 203
 204         return free_chunks;
 205 }
 206
 207 static LOSObject*
 208 randomize_los_object_start (gpointer addr, size_t obj_size, size_t alloced_size, size_t addr_alignment)
 209 {
 210         size_t offset = 0;
 211         if (alloced_size != obj_size) {
 212                 /*
 213                  * We want to get a random offset between 0 and (alloced_size - obj_size)
 214                  * We do a prime multiplication to avoid usage of functions which might not
 215                  * be thread/signal safe (like rand ()). We subtract 1 to avoid common
 216                  * power by 2 factors.
 217                  */
 218                 offset = SGEN_ALIGN_DOWN ((((size_t)addr - 1) * 2654435761u) % (alloced_size - obj_size));
 219         }
 220         SGEN_ASSERT (0, (alloced_size - obj_size) < addr_alignment, "Why are we wasting one entire chunk for a los object ?");
 221         /* Randomize the location within the reserved chunks to improve cache performance */
 222         return (LOSObject*)((guint8*)addr + offset);
 223
 224 }
 225
 226 static LOSObject*
 227 get_los_section_memory (size_t size)
 228 {
 229         LOSSection *section;
 230         LOSFreeChunks *free_chunks;
 231         size_t num_chunks;
 232         size_t obj_size = size;
 233
 234         size = SGEN_ALIGN_UP_TO (size, LOS_CHUNK_SIZE);
 235
 236         num_chunks = size >> LOS_CHUNK_BITS;
 237
 238         g_assert (size > 0 && size - sizeof (LOSObject) <= LOS_SECTION_OBJECT_LIMIT);
 239         g_assert (num_chunks > 0);
 240
 241  retry:
 242         if (num_chunks >= LOS_NUM_FAST_SIZES) {
 243                 free_chunks = get_from_size_list (&los_fast_free_lists [0], size);
 244         } else {
 245                 size_t i;
 246                 for (i = num_chunks; i < LOS_NUM_FAST_SIZES; ++i) {
 247                         free_chunks = get_from_size_list (&los_fast_free_lists [i], size);
 248                         if (free_chunks)
 249                                 break;
 250                 }
 251                 if (!free_chunks)
 252                         free_chunks = get_from_size_list (&los_fast_free_lists [0], size);
 253         }
 254
 255         if (free_chunks) {
 256                 return randomize_los_object_start (free_chunks, obj_size, size, LOS_CHUNK_SIZE);
 257         }
 258
 259         if (!sgen_memgov_try_alloc_space (LOS_SECTION_SIZE, SPACE_LOS))
 260                 return NULL;
 261
 262         section = (LOSSection *)sgen_alloc_os_memory_aligned (LOS_SECTION_SIZE, LOS_SECTION_SIZE, (SgenAllocFlags)(SGEN_ALLOC_HEAP | SGEN_ALLOC_ACTIVATE), NULL);
 263
 264         if (!section)
 265                 return NULL;
 266
 267         free_chunks = (LOSFreeChunks*)((char*)section + LOS_CHUNK_SIZE);
 268         free_chunks->size = LOS_SECTION_SIZE - LOS_CHUNK_SIZE;
 269         free_chunks->next_size = los_fast_free_lists [0];
 270         los_fast_free_lists [0] = free_chunks;
 271
 272         section->num_free_chunks = LOS_SECTION_NUM_CHUNKS;
 273
 274         section->free_chunk_map = (unsigned char*)section + sizeof (LOSSection);
 275         g_assert (sizeof (LOSSection) + LOS_SECTION_NUM_CHUNKS + 1 <= LOS_CHUNK_SIZE);
 276         section->free_chunk_map [0] = 0;
 277         memset (section->free_chunk_map + 1, 1, LOS_SECTION_NUM_CHUNKS);
 278
 279         section->next = los_sections;
 280         los_sections = section;
 281
 282         ++los_num_sections;
 283
 284         goto retry;
 285 }
 286
 287 static void
 288 free_los_section_memory (LOSObject *obj, size_t size)
 289 {
 290         LOSSection *section = LOS_SECTION_FOR_OBJ (obj);
 291         size_t num_chunks, i, start_index;
 292
 293         size = SGEN_ALIGN_UP_TO (size, LOS_CHUNK_SIZE);
 294
 295         num_chunks = size >> LOS_CHUNK_BITS;
 296
 297         g_assert (size > 0 && size - sizeof (LOSObject) <= LOS_SECTION_OBJECT_LIMIT);
 298         g_assert (num_chunks > 0);
 299
 300         section->num_free_chunks += num_chunks;
 301         g_assert (section->num_free_chunks <= LOS_SECTION_NUM_CHUNKS);
 302
 303         /*
 304          * We could free the LOS section here if it's empty, but we
 305          * can't unless we also remove its free chunks from the fast
 306          * free lists.  Instead, we do it in los_sweep().
 307          */
 308
 309         start_index = LOS_CHUNK_INDEX (obj, section);
 310         for (i = start_index; i < start_index + num_chunks; ++i) {
 311                 g_assert (!section->free_chunk_map [i]);
 312                 section->free_chunk_map [i] = 1;
 313         }
 314
 315         add_free_chunk ((LOSFreeChunks*)SGEN_ALIGN_DOWN_TO ((mword)obj, LOS_CHUNK_SIZE), size);
 316 }
 317
 318 void
 319 sgen_los_free_object (LOSObject *obj)
 320 {
 321         SGEN_ASSERT (0, !obj->cardtable_mod_union, "We should never free a LOS object with a mod-union table.");
 322
 323 #ifndef LOS_DUMMY
 324         mword size = sgen_los_object_size (obj);
 325         SGEN_LOG (4, "Freed large object %p, size %lu", obj->data, (unsigned long)size);
 326         binary_protocol_empty (obj->data, size);
 327
 328         los_memory_usage -= size;
 329         los_num_objects--;
 330
 331 #ifdef USE_MALLOC
 332         free (obj);
 333 #else
 334         if (size > LOS_SECTION_OBJECT_LIMIT) {
 335                 int pagesize = mono_pagesize ();
 336                 size += sizeof (LOSObject);
 337                 size = SGEN_ALIGN_UP_TO (size, pagesize);
 338                 sgen_free_os_memory ((gpointer)SGEN_ALIGN_DOWN_TO ((mword)obj, pagesize), size, SGEN_ALLOC_HEAP);
 339                 sgen_memgov_release_space (size, SPACE_LOS);
 340         } else {
 341                 free_los_section_memory (obj, size + sizeof (LOSObject));
 342 #ifdef LOS_CONSISTENCY_CHECKS
 343                 los_consistency_check ();
 344 #endif
 345         }
 346 #endif
 347 #endif
 348 }
 349
 350 /*
 351  * Objects with size >= MAX_SMALL_SIZE are allocated in the large object space.
 352  * They are currently kept track of with a linked list.
 353  * They don't move, so there is no need to pin them during collection
 354  * and we avoid the memcpy overhead.
 355  */
 356 void*
 357 sgen_los_alloc_large_inner (GCVTable vtable, size_t size)
 358 {
 359         LOSObject *obj = NULL;
 360         void **vtslot;
 361
 362         g_assert (size > SGEN_MAX_SMALL_OBJ_SIZE);
 363         g_assert ((size & 1) == 0);
 364
 365         /*
 366          * size + sizeof (LOSObject) <= SSIZE_MAX - (mono_pagesize () - 1)
 367          *
 368          * therefore:
 369          *
 370          * size <= SSIZE_MAX - (mono_pagesize () - 1) - sizeof (LOSObject)
 371          */
 372         if (size > SSIZE_MAX - (mono_pagesize () - 1) - sizeof (LOSObject))
 373                 return NULL;
 374
 375 #ifdef LOS_DUMMY
 376         if (!los_segment)
 377                 los_segment = sgen_alloc_os_memory (LOS_SEGMENT_SIZE, SGEN_ALLOC_HEAP | SGEN_ALLOC_ACTIVATE, NULL);
 378         los_segment_index = ALIGN_UP (los_segment_index);
 379
 380         obj = (LOSObject*)(los_segment + los_segment_index);
 381         los_segment_index += size + sizeof (LOSObject);
 382         g_assert (los_segment_index <= LOS_SEGMENT_SIZE);
 383 #else
 384         sgen_ensure_free_space (size, GENERATION_OLD);
 385
 386 #ifdef USE_MALLOC
 387         obj = malloc (size + sizeof (LOSObject));
 388         memset (obj, 0, size + sizeof (LOSObject));
 389 #else
 390         if (size > LOS_SECTION_OBJECT_LIMIT) {
 391                 size_t obj_size = size + sizeof (LOSObject);
 392                 int pagesize = mono_pagesize ();
 393                 size_t alloc_size = SGEN_ALIGN_UP_TO (obj_size, pagesize);
 394                 if (sgen_memgov_try_alloc_space (alloc_size, SPACE_LOS)) {
 395                         obj = (LOSObject *)sgen_alloc_os_memory (alloc_size, (SgenAllocFlags)(SGEN_ALLOC_HEAP | SGEN_ALLOC_ACTIVATE), NULL);
 396                         if (obj)
 397                                 obj = randomize_los_object_start (obj, obj_size, alloc_size, pagesize);
 398                 }
 399         } else {
 400                 obj = get_los_section_memory (size + sizeof (LOSObject));
 401                 if (obj)
 402                         memset (obj, 0, size + sizeof (LOSObject));
 403         }
 404 #endif
 405 #endif
 406         if (!obj)
 407                 return NULL;
 408         g_assert (!((mword)obj->data & (SGEN_ALLOC_ALIGN - 1)));
 409         obj->size = size;
 410         vtslot = (void**)obj->data;
 411         *vtslot = vtable;
 412         sgen_update_heap_boundaries ((mword)obj->data, (mword)obj->data + size);
 413         obj->next = los_object_list;
 414         /*
 415          * We need a memory barrier so we don't expose as head of the los object list
 416          * a LOSObject that doesn't have its fields initialized.
 417          */
 418         mono_memory_write_barrier ();
 419         los_object_list = obj;
 420         los_memory_usage += size;
 421         los_num_objects++;
 422         SGEN_LOG (4, "Allocated large object %p, vtable: %p (%s), size: %zd", obj->data, vtable, sgen_client_vtable_get_name (vtable), size);
 423         binary_protocol_alloc (obj->data, vtable, size, sgen_client_get_provenance ());
 424
 425 #ifdef LOS_CONSISTENCY_CHECK
 426         los_consistency_check ();
 427 #endif
 428
 429         return obj->data;
 430 }
 431
 432 static void sgen_los_unpin_object (GCObject *data);
 433
 434 void
 435 sgen_los_sweep (void)
 436 {
 437         LOSObject *bigobj, *prevbo;
 438         LOSSection *section, *prev;
 439         int i;
 440         int num_sections = 0;
 441
 442         /* sweep the big objects list */
 443         prevbo = NULL;
 444         for (bigobj = los_object_list; bigobj;) {
 445                 SGEN_ASSERT (0, !SGEN_OBJECT_IS_PINNED (bigobj->data), "Who pinned a LOS object?");
 446
 447                 if (bigobj->cardtable_mod_union) {
 448                         sgen_card_table_free_mod_union (bigobj->cardtable_mod_union, (char*)bigobj->data, sgen_los_object_size (bigobj));
 449                         bigobj->cardtable_mod_union = NULL;
 450                 }
 451
 452                 if (sgen_los_object_is_pinned (bigobj->data)) {
 453                         sgen_los_unpin_object (bigobj->data);
 454                         sgen_update_heap_boundaries ((mword)bigobj->data, (mword)bigobj->data + sgen_los_object_size (bigobj));
 455                 } else {
 456                         LOSObject *to_free;
 457                         /* not referenced anywhere, so we can free it */
 458                         if (prevbo)
 459                                 prevbo->next = bigobj->next;
 460                         else
 461                                 los_object_list = bigobj->next;
 462                         to_free = bigobj;
 463                         bigobj = bigobj->next;
 464                         sgen_los_free_object (to_free);
 465                         continue;
 466                 }
 467                 prevbo = bigobj;
 468                 bigobj = bigobj->next;
 469         }
 470
 471         /* Try to free memory */
 472         for (i = 0; i < LOS_NUM_FAST_SIZES; ++i)
 473                 los_fast_free_lists [i] = NULL;
 474
 475         prev = NULL;
 476         section = los_sections;
 477         while (section) {
 478                 if (section->num_free_chunks == LOS_SECTION_NUM_CHUNKS) {
 479                         LOSSection *next = section->next;
 480                         if (prev)
 481                                 prev->next = next;
 482                         else
 483                                 los_sections = next;
 484                         sgen_free_os_memory (section, LOS_SECTION_SIZE, SGEN_ALLOC_HEAP);
 485                         sgen_memgov_release_space (LOS_SECTION_SIZE, SPACE_LOS);
 486                         section = next;
 487                         --los_num_sections;
 488                         continue;
 489                 }
 490
 491                 for (i = 0; i <= LOS_SECTION_NUM_CHUNKS; ++i) {
 492                         if (section->free_chunk_map [i]) {
 493                                 int j;
 494                                 for (j = i + 1; j <= LOS_SECTION_NUM_CHUNKS && section->free_chunk_map [j]; ++j)
 495                                         ;
 496                                 add_free_chunk ((LOSFreeChunks*)((char*)section + (i << LOS_CHUNK_BITS)), (j - i) << LOS_CHUNK_BITS);
 497                                 i = j - 1;
 498                         }
 499                 }
 500
 501                 prev = section;
 502                 section = section->next;
 503
 504                 ++num_sections;
 505         }
 506
 507 #ifdef LOS_CONSISTENCY_CHECK
 508         los_consistency_check ();
 509 #endif
 510
 511         /*
 512         g_print ("LOS sections: %d  objects: %d  usage: %d\n", num_sections, los_num_objects, los_memory_usage);
 513         for (i = 0; i < LOS_NUM_FAST_SIZES; ++i) {
 514                 int num_chunks = 0;
 515                 LOSFreeChunks *free_chunks;
 516                 for (free_chunks = los_fast_free_lists [i]; free_chunks; free_chunks = free_chunks->next_size)
 517                         ++num_chunks;
 518                 g_print ("  %d: %d\n", i, num_chunks);
 519         }
 520         */
 521
 522         g_assert (los_num_sections == num_sections);
 523 }
 524
 525 gboolean
 526 sgen_ptr_is_in_los (char *ptr, char **start)
 527 {
 528         LOSObject *obj;
 529
 530         *start = NULL;
 531         for (obj = los_object_list; obj; obj = obj->next) {
 532                 char *end = (char*)obj->data + sgen_los_object_size (obj);
 533
 534                 if (ptr >= (char*)obj->data && ptr < end) {
 535                         *start = (char*)obj->data;
 536                         return TRUE;
 537                 }
 538         }
 539         return FALSE;
 540 }
 541
 542 void
 543 sgen_los_iterate_objects (IterateObjectCallbackFunc cb, void *user_data)
 544 {
 545         LOSObject *obj;
 546
 547         for (obj = los_object_list; obj; obj = obj->next)
 548                 cb (obj->data, sgen_los_object_size (obj), user_data);
 549 }
 550
 551 gboolean
 552 sgen_los_is_valid_object (char *object)
 553 {
 554         LOSObject *obj;
 555
 556         for (obj = los_object_list; obj; obj = obj->next) {
 557                 if ((char*)obj->data == object)
 558                         return TRUE;
 559         }
 560         return FALSE;
 561 }
 562
 563 gboolean
 564 mono_sgen_los_describe_pointer (char *ptr)
 565 {
 566         LOSObject *obj;
 567
 568         for (obj = los_object_list; obj; obj = obj->next) {
 569                 const char *los_kind;
 570                 mword size;
 571                 gboolean pinned;
 572
 573                 if ((char*)obj->data > ptr || (char*)obj->data + sgen_los_object_size (obj) <= ptr)
 574                         continue;
 575
 576                 size = sgen_los_object_size (obj);
 577                 pinned = sgen_los_object_is_pinned (obj->data);
 578
 579                 if (size > LOS_SECTION_OBJECT_LIMIT)
 580                         los_kind = "huge-los-ptr";
 581                 else
 582                         los_kind = "los-ptr";
 583
 584                 if ((char*)obj->data == ptr) {
 585                         SGEN_LOG (0, "%s (size %d pin %d)\n", los_kind, (int)size, pinned ? 1 : 0);
 586                 } else {
 587                         SGEN_LOG (0, "%s (interior-ptr offset %zd size %d pin %d)",
 588                                         los_kind, ptr - (char*)obj->data, (int)size, pinned ? 1 : 0);
 589                 }
 590
 591                 return TRUE;
 592         }
 593         return FALSE;
 594 }
 595
 596 void
 597 sgen_los_iterate_live_block_ranges (sgen_cardtable_block_callback callback)
 598 {
 599         LOSObject *obj;
 600         for (obj = los_object_list; obj; obj = obj->next) {
 601                 GCVTable vt = SGEN_LOAD_VTABLE (obj->data);
 602                 if (SGEN_VTABLE_HAS_REFERENCES (vt))
 603                         callback ((mword)obj->data, sgen_los_object_size (obj));
 604         }
 605 }
 606
 607 static guint8*
 608 get_cardtable_mod_union_for_object (LOSObject *obj)
 609 {
 610         mword size = sgen_los_object_size (obj);
 611         guint8 *mod_union = obj->cardtable_mod_union;
 612         guint8 *other;
 613         if (mod_union)
 614                 return mod_union;
 615         mod_union = sgen_card_table_alloc_mod_union ((char*)obj->data, size);
 616         other = (guint8 *)SGEN_CAS_PTR ((gpointer*)&obj->cardtable_mod_union, mod_union, NULL);
 617         if (!other) {
 618                 SGEN_ASSERT (0, obj->cardtable_mod_union == mod_union, "Why did CAS not replace?");
 619                 return mod_union;
 620         }
 621         sgen_card_table_free_mod_union (mod_union, (char*)obj->data, size);
 622         return other;
 623 }
 624
 625 void
 626 sgen_los_scan_card_table (CardTableScanType scan_type, ScanCopyContext ctx)
 627 {
 628         LOSObject *obj;
 629
 630         binary_protocol_los_card_table_scan_start (sgen_timestamp (), scan_type & CARDTABLE_SCAN_MOD_UNION);
 631         for (obj = los_object_list; obj; obj = obj->next) {
 632                 mword num_cards = 0;
 633                 guint8 *cards;
 634
 635                 if (!SGEN_OBJECT_HAS_REFERENCES (obj->data))
 636                         continue;
 637
 638                 if (scan_type & CARDTABLE_SCAN_MOD_UNION) {
 639                         if (!sgen_los_object_is_pinned (obj->data))
 640                                 continue;
 641
 642                         cards = get_cardtable_mod_union_for_object (obj);
 643                         g_assert (cards);
 644                         if (scan_type == CARDTABLE_SCAN_MOD_UNION_PRECLEAN) {
 645                                 guint8 *cards_preclean;
 646                                 mword obj_size = sgen_los_object_size (obj);
 647                                 num_cards = sgen_card_table_number_of_cards_in_range ((mword) obj->data, obj_size);
 648                                 cards_preclean = (guint8 *)sgen_alloc_internal_dynamic (num_cards, INTERNAL_MEM_CARDTABLE_MOD_UNION, TRUE);
 649
 650                                 sgen_card_table_preclean_mod_union (cards, cards_preclean, num_cards);
 651
 652                                 cards = cards_preclean;
 653                         }
 654                 } else {
 655                         cards = NULL;
 656                 }
 657
 658                 sgen_cardtable_scan_object (obj->data, sgen_los_object_size (obj), cards, ctx);
 659
 660                 if (scan_type == CARDTABLE_SCAN_MOD_UNION_PRECLEAN)
 661                         sgen_free_internal_dynamic (cards, num_cards, INTERNAL_MEM_CARDTABLE_MOD_UNION);
 662         }
 663         binary_protocol_los_card_table_scan_end (sgen_timestamp (), scan_type & CARDTABLE_SCAN_MOD_UNION);
 664 }
 665
 666 void
 667 sgen_los_count_cards (long long *num_total_cards, long long *num_marked_cards)
 668 {
 669         LOSObject *obj;
 670         long long total_cards = 0;
 671         long long marked_cards = 0;
 672
 673         for (obj = los_object_list; obj; obj = obj->next) {
 674                 int i;
 675                 guint8 *cards = sgen_card_table_get_card_scan_address ((mword) obj->data);
 676                 guint8 *cards_end = sgen_card_table_get_card_scan_address ((mword) obj->data + sgen_los_object_size (obj) - 1);
 677                 mword num_cards = (cards_end - cards) + 1;
 678
 679                 if (!SGEN_OBJECT_HAS_REFERENCES (obj->data))
 680                         continue;
 681
 682                 total_cards += num_cards;
 683                 for (i = 0; i < num_cards; ++i) {
 684                         if (cards [i])
 685                                 ++marked_cards;
 686                 }
 687         }
 688
 689         *num_total_cards = total_cards;
 690         *num_marked_cards = marked_cards;
 691 }
 692
 693 void
 694 sgen_los_update_cardtable_mod_union (void)
 695 {
 696         LOSObject *obj;
 697
 698         for (obj = los_object_list; obj; obj = obj->next) {
 699                 if (!SGEN_OBJECT_HAS_REFERENCES (obj->data))
 700                         continue;
 701                 sgen_card_table_update_mod_union (get_cardtable_mod_union_for_object (obj),
 702                                 (char*)obj->data, sgen_los_object_size (obj), NULL);
 703         }
 704 }
 705
 706 LOSObject*
 707 sgen_los_header_for_object (GCObject *data)
 708 {
 709 #if _MSC_VER
 710         return (LOSObject*)((char*)data - (int)(&(((LOSObject*)0)->data)));
 711 #else
 712         return (LOSObject*)((char*)data - sizeof (LOSObject));
 713 #endif
 714 }
 715
 716 void
 717 sgen_los_pin_object (GCObject *data)
 718 {
 719         LOSObject *obj = sgen_los_header_for_object (data);
 720         obj->size = obj->size | 1;
 721         binary_protocol_pin (data, (gpointer)SGEN_LOAD_VTABLE (data), sgen_safe_object_get_size (data));
 722 }
 723
 724 static void
 725 sgen_los_unpin_object (GCObject *data)
 726 {
 727         LOSObject *obj = sgen_los_header_for_object (data);
 728         obj->size = sgen_los_object_size (obj);
 729 }
 730
 731 gboolean
 732 sgen_los_object_is_pinned (GCObject *data)
 733 {
 734         LOSObject *obj = sgen_los_header_for_object (data);
 735         return obj->size & 1;
 736 }
 737
 738 void
 739 sgen_los_mark_mod_union_card (GCObject *mono_obj, void **ptr)
 740 {
 741         LOSObject *obj = sgen_los_header_for_object (mono_obj);
 742         guint8 *mod_union = get_cardtable_mod_union_for_object (obj);
 743         /* The LOSObject structure is not represented within the card space */
 744         size_t offset = sgen_card_table_get_card_offset ((char*)ptr, (char*)sgen_card_table_align_pointer((char*)mono_obj));
 745         SGEN_ASSERT (0, mod_union, "FIXME: optionally allocate the mod union if it's not here and CAS it in.");
 746         mod_union [offset] = 1;
 747 }
 748
 749 #endif /* HAVE_SGEN_GC */