mono/sgen/sgen-los.c

   1 /**
   2  * \file
   3  * Large objects space.
   4  *
   5  * Author:
   6  *      Paolo Molaro (lupus@ximian.com)
   7  *
   8  * Copyright 2005-2010 Novell, Inc (http://www.novell.com)
   9  *
  10  * Thread start/stop adapted from Boehm's GC:
  11  * Copyright (c) 1994 by Xerox Corporation.  All rights reserved.
  12  * Copyright (c) 1996 by Silicon Graphics.  All rights reserved.
  13  * Copyright (c) 1998 by Fergus Henderson.  All rights reserved.
  14  * Copyright (c) 2000-2004 by Hewlett-Packard Company.  All rights reserved.
  15  * Copyright 2001-2003 Ximian, Inc
  16  * Copyright 2003-2010 Novell, Inc.
  17  * Copyright (C) 2012 Xamarin Inc
  18  *
  19  * Licensed under the MIT license. See LICENSE file in the project root for full license information.
  20  */
  21
  22 #include "config.h"
  23
  24 #ifdef HAVE_SGEN_GC
  25
  26 #include <string.h>
  27
  28 #include "mono/sgen/sgen-gc.h"
  29 #include "mono/sgen/sgen-protocol.h"
  30 #include "mono/sgen/sgen-cardtable.h"
  31 #include "mono/sgen/sgen-memory-governor.h"
  32 #include "mono/sgen/sgen-client.h"
  33
  34 #define LOS_SECTION_SIZE        (1024 * 1024)
  35
  36 /*
  37  * This shouldn't be much smaller or larger than MAX_SMALL_OBJ_SIZE.
  38  * Must be at least sizeof (LOSSection).
  39  */
  40 #define LOS_CHUNK_SIZE          4096
  41 #define LOS_CHUNK_BITS          12
  42
  43 /* Largest object that can be allocated in a section. */
  44 #define LOS_SECTION_OBJECT_LIMIT        (LOS_SECTION_SIZE - LOS_CHUNK_SIZE - sizeof (LOSObject))
  45 //#define LOS_SECTION_OBJECT_LIMIT      0
  46 #define LOS_SECTION_NUM_CHUNKS          ((LOS_SECTION_SIZE >> LOS_CHUNK_BITS) - 1)
  47
  48 #define LOS_SECTION_FOR_OBJ(obj)        ((LOSSection*)((mword)(obj) & ~(mword)(LOS_SECTION_SIZE - 1)))
  49 #define LOS_CHUNK_INDEX(obj,section)    (((char*)(obj) - (char*)(section)) >> LOS_CHUNK_BITS)
  50
  51 #define LOS_NUM_FAST_SIZES              32
  52
  53 typedef struct _LOSFreeChunks LOSFreeChunks;
  54 struct _LOSFreeChunks {
  55         LOSFreeChunks *next_size;
  56         size_t size;
  57 };
  58
  59 typedef struct _LOSSection LOSSection;
  60 struct _LOSSection {
  61         LOSSection *next;
  62         size_t num_free_chunks;
  63         unsigned char *free_chunk_map;
  64 };
  65
  66 /* We allow read only access on the list while sweep is not running */
  67 LOSObject *los_object_list = NULL;
  68 /* Memory used by LOS objects */
  69 mword los_memory_usage = 0;
  70 /* Total memory used by the LOS allocator */
  71 mword los_memory_usage_total = 0;
  72
  73 static LOSSection *los_sections = NULL;
  74 static LOSFreeChunks *los_fast_free_lists [LOS_NUM_FAST_SIZES]; /* 0 is for larger sizes */
  75 static mword los_num_objects = 0;
  76 static int los_num_sections = 0;
  77
  78 //#define USE_MALLOC
  79 //#define LOS_CONSISTENCY_CHECK
  80 //#define LOS_DUMMY
  81
  82 #ifdef LOS_DUMMY
  83 #define LOS_SEGMENT_SIZE        (4096 * 1024)
  84
  85 static char *los_segment = NULL;
  86 static int los_segment_index = 0;
  87 #endif
  88
  89 mword
  90 sgen_los_object_size (LOSObject *obj)
  91 {
  92         return obj->size & ~1L;
  93 }
  94
  95 #ifdef LOS_CONSISTENCY_CHECK
  96 static void
  97 los_consistency_check (void)
  98 {
  99         LOSSection *section;
 100         LOSObject *obj;
 101         int i;
 102         mword memory_usage = 0;
 103
 104         for (obj = los_object_list; obj; obj = obj->next) {
 105                 mword obj_size = sgen_los_object_size (obj);
 106                 char *end = obj->data + obj_size;
 107                 int start_index, num_chunks;
 108
 109                 memory_usage += obj_size;
 110
 111                 if (obj_size > LOS_SECTION_OBJECT_LIMIT)
 112                         continue;
 113
 114                 section = LOS_SECTION_FOR_OBJ (obj);
 115
 116                 g_assert (end <= (char*)section + LOS_SECTION_SIZE);
 117
 118                 start_index = LOS_CHUNK_INDEX (obj, section);
 119                 num_chunks = (obj_size + sizeof (LOSObject) + LOS_CHUNK_SIZE - 1) >> LOS_CHUNK_BITS;
 120                 for (i = start_index; i < start_index + num_chunks; ++i)
 121                         g_assert (!section->free_chunk_map [i]);
 122         }
 123
 124         for (i = 0; i < LOS_NUM_FAST_SIZES; ++i) {
 125                 LOSFreeChunks *size_chunks;
 126                 for (size_chunks = los_fast_free_lists [i]; size_chunks; size_chunks = size_chunks->next_size) {
 127                         LOSSection *section = LOS_SECTION_FOR_OBJ (size_chunks);
 128                         int j, num_chunks, start_index;
 129
 130                         if (i == 0)
 131                                 g_assert (size_chunks->size >= LOS_NUM_FAST_SIZES * LOS_CHUNK_SIZE);
 132                         else
 133                                 g_assert (size_chunks->size == i * LOS_CHUNK_SIZE);
 134
 135                         num_chunks = size_chunks->size >> LOS_CHUNK_BITS;
 136                         start_index = LOS_CHUNK_INDEX (size_chunks, section);
 137                         for (j = start_index; j < start_index + num_chunks; ++j)
 138                                 g_assert (section->free_chunk_map [j]);
 139                 }
 140         }
 141
 142         g_assert (los_memory_usage == memory_usage);
 143 }
 144 #endif
 145
 146 static void
 147 add_free_chunk (LOSFreeChunks *free_chunks, size_t size)
 148 {
 149         size_t num_chunks = size >> LOS_CHUNK_BITS;
 150
 151         free_chunks->size = size;
 152
 153         if (num_chunks >= LOS_NUM_FAST_SIZES)
 154                 num_chunks = 0;
 155         free_chunks->next_size = los_fast_free_lists [num_chunks];
 156         los_fast_free_lists [num_chunks] = free_chunks;
 157 }
 158
 159 static LOSFreeChunks*
 160 get_from_size_list (LOSFreeChunks **list, size_t size)
 161 {
 162         LOSFreeChunks *free_chunks = NULL;
 163         LOSSection *section;
 164         size_t i, num_chunks, start_index;
 165
 166
 167         g_assert ((size & (LOS_CHUNK_SIZE - 1)) == 0);
 168
 169         while (*list) {
 170                 free_chunks = *list;
 171                 if (free_chunks->size >= size)
 172                         break;
 173                 list = &(*list)->next_size;
 174         }
 175
 176         if (!*list)
 177                 return NULL;
 178
 179         *list = free_chunks->next_size;
 180
 181         if (free_chunks->size > size)
 182                 add_free_chunk ((LOSFreeChunks*)((char*)free_chunks + size), free_chunks->size - size);
 183
 184         num_chunks = size >> LOS_CHUNK_BITS;
 185
 186         section = LOS_SECTION_FOR_OBJ (free_chunks);
 187
 188         start_index = LOS_CHUNK_INDEX (free_chunks, section);
 189         for (i = start_index; i < start_index + num_chunks; ++i) {
 190                 g_assert (section->free_chunk_map [i]);
 191                 section->free_chunk_map [i] = 0;
 192         }
 193
 194         section->num_free_chunks -= size >> LOS_CHUNK_BITS;
 195         g_assert (section->num_free_chunks >= 0);
 196
 197         return free_chunks;
 198 }
 199
 200 static LOSObject*
 201 randomize_los_object_start (gpointer addr, size_t obj_size, size_t alloced_size, size_t addr_alignment)
 202 {
 203         size_t offset = 0;
 204         if (alloced_size != obj_size) {
 205                 /*
 206                  * We want to get a random offset between 0 and (alloced_size - obj_size)
 207                  * We do a prime multiplication to avoid usage of functions which might not
 208                  * be thread/signal safe (like rand ()). We subtract 1 to avoid common
 209                  * power by 2 factors.
 210                  */
 211                 offset = SGEN_ALIGN_DOWN ((((size_t)addr - 1) * 2654435761u) % (alloced_size - obj_size));
 212         }
 213         SGEN_ASSERT (0, (alloced_size - obj_size) < addr_alignment, "Why are we wasting one entire chunk for a los object ?");
 214         /* Randomize the location within the reserved chunks to improve cache performance */
 215         return (LOSObject*)((guint8*)addr + offset);
 216
 217 }
 218
 219 static LOSObject*
 220 get_los_section_memory (size_t size)
 221 {
 222         LOSSection *section;
 223         LOSFreeChunks *free_chunks;
 224         size_t num_chunks;
 225         size_t obj_size = size;
 226
 227         size = SGEN_ALIGN_UP_TO (size, LOS_CHUNK_SIZE);
 228
 229         num_chunks = size >> LOS_CHUNK_BITS;
 230
 231         g_assert (size > 0 && size - sizeof (LOSObject) <= LOS_SECTION_OBJECT_LIMIT);
 232         g_assert (num_chunks > 0);
 233
 234  retry:
 235         if (num_chunks >= LOS_NUM_FAST_SIZES) {
 236                 free_chunks = get_from_size_list (&los_fast_free_lists [0], size);
 237         } else {
 238                 size_t i;
 239                 for (i = num_chunks; i < LOS_NUM_FAST_SIZES; ++i) {
 240                         free_chunks = get_from_size_list (&los_fast_free_lists [i], size);
 241                         if (free_chunks)
 242                                 break;
 243                 }
 244                 if (!free_chunks)
 245                         free_chunks = get_from_size_list (&los_fast_free_lists [0], size);
 246         }
 247
 248         if (free_chunks) {
 249                 return randomize_los_object_start (free_chunks, obj_size, size, LOS_CHUNK_SIZE);
 250         }
 251
 252         if (!sgen_memgov_try_alloc_space (LOS_SECTION_SIZE, SPACE_LOS))
 253                 return NULL;
 254
 255         section = (LOSSection *)sgen_alloc_os_memory_aligned (LOS_SECTION_SIZE, LOS_SECTION_SIZE, (SgenAllocFlags)(SGEN_ALLOC_HEAP | SGEN_ALLOC_ACTIVATE), NULL, MONO_MEM_ACCOUNT_SGEN_LOS);
 256
 257         if (!section)
 258                 return NULL;
 259
 260         free_chunks = (LOSFreeChunks*)((char*)section + LOS_CHUNK_SIZE);
 261         free_chunks->size = LOS_SECTION_SIZE - LOS_CHUNK_SIZE;
 262         free_chunks->next_size = los_fast_free_lists [0];
 263         los_fast_free_lists [0] = free_chunks;
 264
 265         section->num_free_chunks = LOS_SECTION_NUM_CHUNKS;
 266
 267         section->free_chunk_map = (unsigned char*)section + sizeof (LOSSection);
 268         g_assert (sizeof (LOSSection) + LOS_SECTION_NUM_CHUNKS + 1 <= LOS_CHUNK_SIZE);
 269         section->free_chunk_map [0] = 0;
 270         memset (section->free_chunk_map + 1, 1, LOS_SECTION_NUM_CHUNKS);
 271
 272         section->next = los_sections;
 273         los_sections = section;
 274
 275         los_memory_usage_total += LOS_SECTION_SIZE;
 276         ++los_num_sections;
 277
 278         goto retry;
 279 }
 280
 281 static void
 282 free_los_section_memory (LOSObject *obj, size_t size)
 283 {
 284         LOSSection *section = LOS_SECTION_FOR_OBJ (obj);
 285         size_t num_chunks, i, start_index;
 286
 287         size = SGEN_ALIGN_UP_TO (size, LOS_CHUNK_SIZE);
 288
 289         num_chunks = size >> LOS_CHUNK_BITS;
 290
 291         g_assert (size > 0 && size - sizeof (LOSObject) <= LOS_SECTION_OBJECT_LIMIT);
 292         g_assert (num_chunks > 0);
 293
 294         section->num_free_chunks += num_chunks;
 295         g_assert (section->num_free_chunks <= LOS_SECTION_NUM_CHUNKS);
 296
 297         /*
 298          * We could free the LOS section here if it's empty, but we
 299          * can't unless we also remove its free chunks from the fast
 300          * free lists.  Instead, we do it in los_sweep().
 301          */
 302
 303         start_index = LOS_CHUNK_INDEX (obj, section);
 304         for (i = start_index; i < start_index + num_chunks; ++i) {
 305                 g_assert (!section->free_chunk_map [i]);
 306                 section->free_chunk_map [i] = 1;
 307         }
 308
 309         add_free_chunk ((LOSFreeChunks*)SGEN_ALIGN_DOWN_TO ((mword)obj, LOS_CHUNK_SIZE), size);
 310 }
 311
 312 void
 313 sgen_los_free_object (LOSObject *obj)
 314 {
 315         if (obj->cardtable_mod_union)
 316                 sgen_card_table_free_mod_union (obj->cardtable_mod_union, (char*)obj->data, sgen_los_object_size (obj));
 317
 318 #ifndef LOS_DUMMY
 319         mword size = sgen_los_object_size (obj);
 320         SGEN_LOG (4, "Freed large object %p, size %lu", obj->data, (unsigned long)size);
 321         binary_protocol_empty (obj->data, size);
 322
 323         los_memory_usage -= size;
 324         los_num_objects--;
 325
 326 #ifdef USE_MALLOC
 327         g_free (obj);
 328 #else
 329         if (size > LOS_SECTION_OBJECT_LIMIT) {
 330                 int pagesize = mono_pagesize ();
 331                 size += sizeof (LOSObject);
 332                 size = SGEN_ALIGN_UP_TO (size, pagesize);
 333                 sgen_free_os_memory ((gpointer)SGEN_ALIGN_DOWN_TO ((mword)obj, pagesize), size, SGEN_ALLOC_HEAP, MONO_MEM_ACCOUNT_SGEN_LOS);
 334                 los_memory_usage_total -= size;
 335                 sgen_memgov_release_space (size, SPACE_LOS);
 336         } else {
 337                 free_los_section_memory (obj, size + sizeof (LOSObject));
 338 #ifdef LOS_CONSISTENCY_CHECKS
 339                 los_consistency_check ();
 340 #endif
 341         }
 342 #endif
 343 #endif
 344 }
 345
 346 /*
 347  * Objects with size >= MAX_SMALL_SIZE are allocated in the large object space.
 348  * They are currently kept track of with a linked list.
 349  * They don't move, so there is no need to pin them during collection
 350  * and we avoid the memcpy overhead.
 351  */
 352 void*
 353 sgen_los_alloc_large_inner (GCVTable vtable, size_t size)
 354 {
 355         LOSObject *obj = NULL;
 356         void **vtslot;
 357
 358         g_assert (size > SGEN_MAX_SMALL_OBJ_SIZE);
 359         g_assert ((size & 1) == 0);
 360
 361         /*
 362          * size + sizeof (LOSObject) <= SSIZE_MAX - (mono_pagesize () - 1)
 363          *
 364          * therefore:
 365          *
 366          * size <= SSIZE_MAX - (mono_pagesize () - 1) - sizeof (LOSObject)
 367          */
 368         if (size > SSIZE_MAX - (mono_pagesize () - 1) - sizeof (LOSObject))
 369                 return NULL;
 370
 371 #ifdef LOS_DUMMY
 372         if (!los_segment)
 373                 los_segment = sgen_alloc_os_memory (LOS_SEGMENT_SIZE, SGEN_ALLOC_HEAP | SGEN_ALLOC_ACTIVATE, NULL);
 374         los_segment_index = ALIGN_UP (los_segment_index);
 375
 376         obj = (LOSObject*)(los_segment + los_segment_index);
 377         los_segment_index += size + sizeof (LOSObject);
 378         g_assert (los_segment_index <= LOS_SEGMENT_SIZE);
 379 #else
 380         sgen_ensure_free_space (size, GENERATION_OLD);
 381
 382 #ifdef USE_MALLOC
 383         obj = g_malloc (size + sizeof (LOSObject));
 384         memset (obj, 0, size + sizeof (LOSObject));
 385 #else
 386         if (size > LOS_SECTION_OBJECT_LIMIT) {
 387                 size_t obj_size = size + sizeof (LOSObject);
 388                 int pagesize = mono_pagesize ();
 389                 size_t alloc_size = SGEN_ALIGN_UP_TO (obj_size, pagesize);
 390                 if (sgen_memgov_try_alloc_space (alloc_size, SPACE_LOS)) {
 391                         obj = (LOSObject *)sgen_alloc_os_memory (alloc_size, (SgenAllocFlags)(SGEN_ALLOC_HEAP | SGEN_ALLOC_ACTIVATE), NULL, MONO_MEM_ACCOUNT_SGEN_LOS);
 392                         if (obj) {
 393                                 los_memory_usage_total += alloc_size;
 394                                 obj = randomize_los_object_start (obj, obj_size, alloc_size, pagesize);
 395                         }
 396                 }
 397         } else {
 398                 obj = get_los_section_memory (size + sizeof (LOSObject));
 399                 if (obj)
 400                         memset (obj, 0, size + sizeof (LOSObject));
 401         }
 402 #endif
 403 #endif
 404         if (!obj)
 405                 return NULL;
 406         g_assert (!((mword)obj->data & (SGEN_ALLOC_ALIGN - 1)));
 407         obj->size = size;
 408         vtslot = (void**)obj->data;
 409         *vtslot = vtable;
 410         sgen_update_heap_boundaries ((mword)obj->data, (mword)obj->data + size);
 411         obj->next = los_object_list;
 412         /*
 413          * We need a memory barrier so we don't expose as head of the los object list
 414          * a LOSObject that doesn't have its fields initialized.
 415          */
 416         mono_memory_write_barrier ();
 417         los_object_list = obj;
 418         los_memory_usage += size;
 419         los_num_objects++;
 420         SGEN_LOG (4, "Allocated large object %p, vtable: %p (%s), size: %zd", obj->data, vtable, sgen_client_vtable_get_name (vtable), size);
 421         binary_protocol_alloc (obj->data, vtable, size, sgen_client_get_provenance ());
 422
 423 #ifdef LOS_CONSISTENCY_CHECK
 424         los_consistency_check ();
 425 #endif
 426
 427         return obj->data;
 428 }
 429
 430 static void sgen_los_unpin_object (GCObject *data);
 431
 432 void
 433 sgen_los_sweep (void)
 434 {
 435         LOSObject *bigobj, *prevbo;
 436         LOSSection *section, *prev;
 437         int i;
 438         int num_sections = 0;
 439
 440         /* sweep the big objects list */
 441         prevbo = NULL;
 442         for (bigobj = los_object_list; bigobj;) {
 443                 SGEN_ASSERT (0, !SGEN_OBJECT_IS_PINNED (bigobj->data), "Who pinned a LOS object?");
 444
 445                 if (sgen_los_object_is_pinned (bigobj->data)) {
 446                         if (bigobj->cardtable_mod_union) {
 447                                 mword obj_size = sgen_los_object_size (bigobj);
 448                                 mword num_cards = sgen_card_table_number_of_cards_in_range ((mword) bigobj->data, obj_size);
 449                                 memset (bigobj->cardtable_mod_union, 0, num_cards);
 450                         }
 451
 452                         sgen_los_unpin_object (bigobj->data);
 453                         sgen_update_heap_boundaries ((mword)bigobj->data, (mword)bigobj->data + sgen_los_object_size (bigobj));
 454                 } else {
 455                         LOSObject *to_free;
 456                         /* not referenced anywhere, so we can free it */
 457                         if (prevbo)
 458                                 prevbo->next = bigobj->next;
 459                         else
 460                                 los_object_list = bigobj->next;
 461                         to_free = bigobj;
 462                         bigobj = bigobj->next;
 463                         sgen_los_free_object (to_free);
 464                         continue;
 465                 }
 466                 prevbo = bigobj;
 467                 bigobj = bigobj->next;
 468         }
 469
 470         /* Try to free memory */
 471         for (i = 0; i < LOS_NUM_FAST_SIZES; ++i)
 472                 los_fast_free_lists [i] = NULL;
 473
 474         prev = NULL;
 475         section = los_sections;
 476         while (section) {
 477                 if (section->num_free_chunks == LOS_SECTION_NUM_CHUNKS) {
 478                         LOSSection *next = section->next;
 479                         if (prev)
 480                                 prev->next = next;
 481                         else
 482                                 los_sections = next;
 483                         sgen_free_os_memory (section, LOS_SECTION_SIZE, SGEN_ALLOC_HEAP, MONO_MEM_ACCOUNT_SGEN_LOS);
 484                         sgen_memgov_release_space (LOS_SECTION_SIZE, SPACE_LOS);
 485                         section = next;
 486                         --los_num_sections;
 487                         los_memory_usage_total -= LOS_SECTION_SIZE;
 488                         continue;
 489                 }
 490
 491                 for (i = 0; i <= LOS_SECTION_NUM_CHUNKS; ++i) {
 492                         if (section->free_chunk_map [i]) {
 493                                 int j;
 494                                 for (j = i + 1; j <= LOS_SECTION_NUM_CHUNKS && section->free_chunk_map [j]; ++j)
 495                                         ;
 496                                 add_free_chunk ((LOSFreeChunks*)((char*)section + (i << LOS_CHUNK_BITS)), (j - i) << LOS_CHUNK_BITS);
 497                                 i = j - 1;
 498                         }
 499                 }
 500
 501                 prev = section;
 502                 section = section->next;
 503
 504                 ++num_sections;
 505         }
 506
 507 #ifdef LOS_CONSISTENCY_CHECK
 508         los_consistency_check ();
 509 #endif
 510
 511         /*
 512         g_print ("LOS sections: %d  objects: %d  usage: %d\n", num_sections, los_num_objects, los_memory_usage);
 513         for (i = 0; i < LOS_NUM_FAST_SIZES; ++i) {
 514                 int num_chunks = 0;
 515                 LOSFreeChunks *free_chunks;
 516                 for (free_chunks = los_fast_free_lists [i]; free_chunks; free_chunks = free_chunks->next_size)
 517                         ++num_chunks;
 518                 g_print ("  %d: %d\n", i, num_chunks);
 519         }
 520         */
 521
 522         g_assert (los_num_sections == num_sections);
 523 }
 524
 525 gboolean
 526 sgen_ptr_is_in_los (char *ptr, char **start)
 527 {
 528         LOSObject *obj;
 529
 530         if (start)
 531                 *start = NULL;
 532         for (obj = los_object_list; obj; obj = obj->next) {
 533                 char *end = (char*)obj->data + sgen_los_object_size (obj);
 534
 535                 if (ptr >= (char*)obj->data && ptr < end) {
 536                         if (start)
 537                                 *start = (char*)obj->data;
 538                         return TRUE;
 539                 }
 540         }
 541         return FALSE;
 542 }
 543
 544 void
 545 sgen_los_iterate_objects (IterateObjectCallbackFunc cb, void *user_data)
 546 {
 547         LOSObject *obj;
 548
 549         for (obj = los_object_list; obj; obj = obj->next)
 550                 cb (obj->data, sgen_los_object_size (obj), user_data);
 551 }
 552
 553 gboolean
 554 sgen_los_is_valid_object (char *object)
 555 {
 556         LOSObject *obj;
 557
 558         for (obj = los_object_list; obj; obj = obj->next) {
 559                 if ((char*)obj->data == object)
 560                         return TRUE;
 561         }
 562         return FALSE;
 563 }
 564
 565 gboolean
 566 mono_sgen_los_describe_pointer (char *ptr)
 567 {
 568         LOSObject *obj;
 569
 570         for (obj = los_object_list; obj; obj = obj->next) {
 571                 const char *los_kind;
 572                 mword size;
 573                 gboolean pinned;
 574
 575                 if ((char*)obj->data > ptr || (char*)obj->data + sgen_los_object_size (obj) <= ptr)
 576                         continue;
 577
 578                 size = sgen_los_object_size (obj);
 579                 pinned = sgen_los_object_is_pinned (obj->data);
 580
 581                 if (size > LOS_SECTION_OBJECT_LIMIT)
 582                         los_kind = "huge-los-ptr";
 583                 else
 584                         los_kind = "los-ptr";
 585
 586                 if ((char*)obj->data == ptr) {
 587                         SGEN_LOG (0, "%s (size %d pin %d)\n", los_kind, (int)size, pinned ? 1 : 0);
 588                 } else {
 589                         SGEN_LOG (0, "%s (interior-ptr offset %zd size %d pin %d)",
 590                                         los_kind, ptr - (char*)obj->data, (int)size, pinned ? 1 : 0);
 591                 }
 592
 593                 return TRUE;
 594         }
 595         return FALSE;
 596 }
 597
 598 void
 599 sgen_los_iterate_live_block_ranges (sgen_cardtable_block_callback callback)
 600 {
 601         LOSObject *obj;
 602         for (obj = los_object_list; obj; obj = obj->next) {
 603                 GCVTable vt = SGEN_LOAD_VTABLE (obj->data);
 604                 if (SGEN_VTABLE_HAS_REFERENCES (vt))
 605                         callback ((mword)obj->data, sgen_los_object_size (obj));
 606         }
 607 }
 608
 609 static guint8*
 610 get_cardtable_mod_union_for_object (LOSObject *obj)
 611 {
 612         mword size = sgen_los_object_size (obj);
 613         guint8 *mod_union = obj->cardtable_mod_union;
 614         guint8 *other;
 615         if (mod_union)
 616                 return mod_union;
 617         mod_union = sgen_card_table_alloc_mod_union ((char*)obj->data, size);
 618         other = (guint8 *)SGEN_CAS_PTR ((gpointer*)&obj->cardtable_mod_union, mod_union, NULL);
 619         if (!other) {
 620                 SGEN_ASSERT (0, obj->cardtable_mod_union == mod_union, "Why did CAS not replace?");
 621                 return mod_union;
 622         }
 623         sgen_card_table_free_mod_union (mod_union, (char*)obj->data, size);
 624         return other;
 625 }
 626
 627 void
 628 sgen_los_scan_card_table (CardTableScanType scan_type, ScanCopyContext ctx, int job_index, int job_split_count)
 629 {
 630         LOSObject *obj;
 631         int i = 0;
 632
 633         binary_protocol_los_card_table_scan_start (sgen_timestamp (), scan_type & CARDTABLE_SCAN_MOD_UNION);
 634         for (obj = los_object_list; obj; obj = obj->next, i++) {
 635                 mword num_cards = 0;
 636                 guint8 *cards;
 637
 638                 if (i % job_split_count != job_index)
 639                         continue;
 640
 641                 if (!SGEN_OBJECT_HAS_REFERENCES (obj->data))
 642                         continue;
 643
 644                 if (scan_type & CARDTABLE_SCAN_MOD_UNION) {
 645                         if (!sgen_los_object_is_pinned (obj->data))
 646                                 continue;
 647
 648                         if (!obj->cardtable_mod_union)
 649                                 continue;
 650
 651                         cards = get_cardtable_mod_union_for_object (obj);
 652                         g_assert (cards);
 653                         if (scan_type == CARDTABLE_SCAN_MOD_UNION_PRECLEAN) {
 654                                 guint8 *cards_preclean;
 655                                 mword obj_size = sgen_los_object_size (obj);
 656                                 num_cards = sgen_card_table_number_of_cards_in_range ((mword) obj->data, obj_size);
 657                                 cards_preclean = (guint8 *)sgen_alloc_internal_dynamic (num_cards, INTERNAL_MEM_CARDTABLE_MOD_UNION, TRUE);
 658
 659                                 sgen_card_table_preclean_mod_union (cards, cards_preclean, num_cards);
 660
 661                                 cards = cards_preclean;
 662                         }
 663                 } else {
 664                         cards = NULL;
 665                 }
 666
 667                 sgen_cardtable_scan_object (obj->data, sgen_los_object_size (obj), cards, ctx);
 668
 669                 if (scan_type == CARDTABLE_SCAN_MOD_UNION_PRECLEAN)
 670                         sgen_free_internal_dynamic (cards, num_cards, INTERNAL_MEM_CARDTABLE_MOD_UNION);
 671         }
 672         binary_protocol_los_card_table_scan_end (sgen_timestamp (), scan_type & CARDTABLE_SCAN_MOD_UNION);
 673 }
 674
 675 void
 676 sgen_los_count_cards (long long *num_total_cards, long long *num_marked_cards)
 677 {
 678         LOSObject *obj;
 679         long long total_cards = 0;
 680         long long marked_cards = 0;
 681
 682         for (obj = los_object_list; obj; obj = obj->next) {
 683                 int i;
 684                 guint8 *cards = sgen_card_table_get_card_scan_address ((mword) obj->data);
 685                 guint8 *cards_end = sgen_card_table_get_card_scan_address ((mword) obj->data + sgen_los_object_size (obj) - 1);
 686                 mword num_cards = (cards_end - cards) + 1;
 687
 688                 if (!SGEN_OBJECT_HAS_REFERENCES (obj->data))
 689                         continue;
 690
 691                 total_cards += num_cards;
 692                 for (i = 0; i < num_cards; ++i) {
 693                         if (cards [i])
 694                                 ++marked_cards;
 695                 }
 696         }
 697
 698         *num_total_cards = total_cards;
 699         *num_marked_cards = marked_cards;
 700 }
 701
 702 void
 703 sgen_los_update_cardtable_mod_union (void)
 704 {
 705         LOSObject *obj;
 706
 707         for (obj = los_object_list; obj; obj = obj->next) {
 708                 if (!SGEN_OBJECT_HAS_REFERENCES (obj->data))
 709                         continue;
 710                 sgen_card_table_update_mod_union (get_cardtable_mod_union_for_object (obj),
 711                                 (char*)obj->data, sgen_los_object_size (obj), NULL);
 712         }
 713 }
 714
 715 LOSObject*
 716 sgen_los_header_for_object (GCObject *data)
 717 {
 718 #if _MSC_VER
 719         return (LOSObject*)((char*)data - (int)(&(((LOSObject*)0)->data)));
 720 #else
 721         return (LOSObject*)((char*)data - sizeof (LOSObject));
 722 #endif
 723 }
 724
 725 void
 726 sgen_los_pin_object (GCObject *data)
 727 {
 728         LOSObject *obj = sgen_los_header_for_object (data);
 729         obj->size = obj->size | 1;
 730         binary_protocol_pin (data, (gpointer)SGEN_LOAD_VTABLE (data), sgen_safe_object_get_size (data));
 731 }
 732
 733 gboolean
 734 sgen_los_pin_object_par (GCObject *data)
 735 {
 736         LOSObject *obj = sgen_los_header_for_object (data);
 737         mword old_size = obj->size;
 738         if (old_size & 1)
 739                 return FALSE;
 740 #if SIZEOF_VOID_P == 4
 741         old_size = InterlockedCompareExchange ((volatile gint32*)&obj->size, old_size | 1, old_size);
 742 #else
 743         old_size = InterlockedCompareExchange64 ((volatile gint64*)&obj->size, old_size | 1, old_size);
 744 #endif
 745         if (old_size & 1)
 746                 return FALSE;
 747         binary_protocol_pin (data, (gpointer)SGEN_LOAD_VTABLE (data), sgen_safe_object_get_size (data));
 748         return TRUE;
 749 }
 750
 751 static void
 752 sgen_los_unpin_object (GCObject *data)
 753 {
 754         LOSObject *obj = sgen_los_header_for_object (data);
 755         obj->size = sgen_los_object_size (obj);
 756 }
 757
 758 gboolean
 759 sgen_los_object_is_pinned (GCObject *data)
 760 {
 761         LOSObject *obj = sgen_los_header_for_object (data);
 762         return obj->size & 1;
 763 }
 764
 765 void
 766 sgen_los_mark_mod_union_card (GCObject *mono_obj, void **ptr)
 767 {
 768         LOSObject *obj = sgen_los_header_for_object (mono_obj);
 769         guint8 *mod_union = get_cardtable_mod_union_for_object (obj);
 770         /* The LOSObject structure is not represented within the card space */
 771         size_t offset = sgen_card_table_get_card_offset ((char*)ptr, (char*)sgen_card_table_align_pointer((char*)mono_obj));
 772         SGEN_ASSERT (0, mod_union, "FIXME: optionally allocate the mod union if it's not here and CAS it in.");
 773         mod_union [offset] = 1;
 774 }
 775
 776 #endif /* HAVE_SGEN_GC */