2 * sgen-los.c: Large objects space.
5 * Paolo Molaro (lupus@ximian.com)
7 * Copyright 2005-2010 Novell, Inc (http://www.novell.com)
9 * Thread start/stop adapted from Boehm's GC:
10 * Copyright (c) 1994 by Xerox Corporation. All rights reserved.
11 * Copyright (c) 1996 by Silicon Graphics. All rights reserved.
12 * Copyright (c) 1998 by Fergus Henderson. All rights reserved.
13 * Copyright (c) 2000-2004 by Hewlett-Packard Company. All rights reserved.
14 * Copyright 2001-2003 Ximian, Inc
15 * Copyright 2003-2010 Novell, Inc.
16 * Copyright (C) 2012 Xamarin Inc
18 * Licensed under the MIT license. See LICENSE file in the project root for full license information.
27 #include "mono/sgen/sgen-gc.h"
28 #include "mono/sgen/sgen-protocol.h"
29 #include "mono/sgen/sgen-cardtable.h"
30 #include "mono/sgen/sgen-memory-governor.h"
31 #include "mono/sgen/sgen-client.h"
33 #define LOS_SECTION_SIZE (1024 * 1024)
36 * This shouldn't be much smaller or larger than MAX_SMALL_OBJ_SIZE.
37 * Must be at least sizeof (LOSSection).
39 #define LOS_CHUNK_SIZE 4096
40 #define LOS_CHUNK_BITS 12
42 /* Largest object that can be allocated in a section. */
43 #define LOS_SECTION_OBJECT_LIMIT (LOS_SECTION_SIZE - LOS_CHUNK_SIZE - sizeof (LOSObject))
44 //#define LOS_SECTION_OBJECT_LIMIT 0
45 #define LOS_SECTION_NUM_CHUNKS ((LOS_SECTION_SIZE >> LOS_CHUNK_BITS) - 1)
47 #define LOS_SECTION_FOR_OBJ(obj) ((LOSSection*)((mword)(obj) & ~(mword)(LOS_SECTION_SIZE - 1)))
48 #define LOS_CHUNK_INDEX(obj,section) (((char*)(obj) - (char*)(section)) >> LOS_CHUNK_BITS)
50 #define LOS_NUM_FAST_SIZES 32
52 typedef struct _LOSFreeChunks LOSFreeChunks;
53 struct _LOSFreeChunks {
54 LOSFreeChunks *next_size;
58 typedef struct _LOSSection LOSSection;
61 size_t num_free_chunks;
62 unsigned char *free_chunk_map;
65 /* We allow read only access on the list while sweep is not running */
66 LOSObject *los_object_list = NULL;
67 /* Memory used by LOS objects */
68 mword los_memory_usage = 0;
69 /* Total memory used by the LOS allocator */
70 mword los_memory_usage_total = 0;
72 static LOSSection *los_sections = NULL;
73 static LOSFreeChunks *los_fast_free_lists [LOS_NUM_FAST_SIZES]; /* 0 is for larger sizes */
74 static mword los_num_objects = 0;
75 static int los_num_sections = 0;
78 //#define LOS_CONSISTENCY_CHECK
82 #define LOS_SEGMENT_SIZE (4096 * 1024)
84 static char *los_segment = NULL;
85 static int los_segment_index = 0;
89 sgen_los_object_size (LOSObject *obj)
91 return obj->size & ~1L;
94 #ifdef LOS_CONSISTENCY_CHECK
96 los_consistency_check (void)
101 mword memory_usage = 0;
103 for (obj = los_object_list; obj; obj = obj->next) {
104 mword obj_size = sgen_los_object_size (obj);
105 char *end = obj->data + obj_size;
106 int start_index, num_chunks;
108 memory_usage += obj_size;
110 if (obj_size > LOS_SECTION_OBJECT_LIMIT)
113 section = LOS_SECTION_FOR_OBJ (obj);
115 g_assert (end <= (char*)section + LOS_SECTION_SIZE);
117 start_index = LOS_CHUNK_INDEX (obj, section);
118 num_chunks = (obj_size + sizeof (LOSObject) + LOS_CHUNK_SIZE - 1) >> LOS_CHUNK_BITS;
119 for (i = start_index; i < start_index + num_chunks; ++i)
120 g_assert (!section->free_chunk_map [i]);
123 for (i = 0; i < LOS_NUM_FAST_SIZES; ++i) {
124 LOSFreeChunks *size_chunks;
125 for (size_chunks = los_fast_free_lists [i]; size_chunks; size_chunks = size_chunks->next_size) {
126 LOSSection *section = LOS_SECTION_FOR_OBJ (size_chunks);
127 int j, num_chunks, start_index;
130 g_assert (size_chunks->size >= LOS_NUM_FAST_SIZES * LOS_CHUNK_SIZE);
132 g_assert (size_chunks->size == i * LOS_CHUNK_SIZE);
134 num_chunks = size_chunks->size >> LOS_CHUNK_BITS;
135 start_index = LOS_CHUNK_INDEX (size_chunks, section);
136 for (j = start_index; j < start_index + num_chunks; ++j)
137 g_assert (section->free_chunk_map [j]);
141 g_assert (los_memory_usage == memory_usage);
146 add_free_chunk (LOSFreeChunks *free_chunks, size_t size)
148 size_t num_chunks = size >> LOS_CHUNK_BITS;
150 free_chunks->size = size;
152 if (num_chunks >= LOS_NUM_FAST_SIZES)
154 free_chunks->next_size = los_fast_free_lists [num_chunks];
155 los_fast_free_lists [num_chunks] = free_chunks;
158 static LOSFreeChunks*
159 get_from_size_list (LOSFreeChunks **list, size_t size)
161 LOSFreeChunks *free_chunks = NULL;
163 size_t i, num_chunks, start_index;
166 g_assert ((size & (LOS_CHUNK_SIZE - 1)) == 0);
170 if (free_chunks->size >= size)
172 list = &(*list)->next_size;
178 *list = free_chunks->next_size;
180 if (free_chunks->size > size)
181 add_free_chunk ((LOSFreeChunks*)((char*)free_chunks + size), free_chunks->size - size);
183 num_chunks = size >> LOS_CHUNK_BITS;
185 section = LOS_SECTION_FOR_OBJ (free_chunks);
187 start_index = LOS_CHUNK_INDEX (free_chunks, section);
188 for (i = start_index; i < start_index + num_chunks; ++i) {
189 g_assert (section->free_chunk_map [i]);
190 section->free_chunk_map [i] = 0;
193 section->num_free_chunks -= size >> LOS_CHUNK_BITS;
194 g_assert (section->num_free_chunks >= 0);
200 randomize_los_object_start (gpointer addr, size_t obj_size, size_t alloced_size, size_t addr_alignment)
203 if (alloced_size != obj_size) {
205 * We want to get a random offset between 0 and (alloced_size - obj_size)
206 * We do a prime multiplication to avoid usage of functions which might not
207 * be thread/signal safe (like rand ()). We subtract 1 to avoid common
208 * power by 2 factors.
210 offset = SGEN_ALIGN_DOWN ((((size_t)addr - 1) * 2654435761u) % (alloced_size - obj_size));
212 SGEN_ASSERT (0, (alloced_size - obj_size) < addr_alignment, "Why are we wasting one entire chunk for a los object ?");
213 /* Randomize the location within the reserved chunks to improve cache performance */
214 return (LOSObject*)((guint8*)addr + offset);
219 get_los_section_memory (size_t size)
222 LOSFreeChunks *free_chunks;
224 size_t obj_size = size;
226 size = SGEN_ALIGN_UP_TO (size, LOS_CHUNK_SIZE);
228 num_chunks = size >> LOS_CHUNK_BITS;
230 g_assert (size > 0 && size - sizeof (LOSObject) <= LOS_SECTION_OBJECT_LIMIT);
231 g_assert (num_chunks > 0);
234 if (num_chunks >= LOS_NUM_FAST_SIZES) {
235 free_chunks = get_from_size_list (&los_fast_free_lists [0], size);
238 for (i = num_chunks; i < LOS_NUM_FAST_SIZES; ++i) {
239 free_chunks = get_from_size_list (&los_fast_free_lists [i], size);
244 free_chunks = get_from_size_list (&los_fast_free_lists [0], size);
248 return randomize_los_object_start (free_chunks, obj_size, size, LOS_CHUNK_SIZE);
251 if (!sgen_memgov_try_alloc_space (LOS_SECTION_SIZE, SPACE_LOS))
254 section = (LOSSection *)sgen_alloc_os_memory_aligned (LOS_SECTION_SIZE, LOS_SECTION_SIZE, (SgenAllocFlags)(SGEN_ALLOC_HEAP | SGEN_ALLOC_ACTIVATE), NULL, MONO_MEM_ACCOUNT_SGEN_LOS);
259 free_chunks = (LOSFreeChunks*)((char*)section + LOS_CHUNK_SIZE);
260 free_chunks->size = LOS_SECTION_SIZE - LOS_CHUNK_SIZE;
261 free_chunks->next_size = los_fast_free_lists [0];
262 los_fast_free_lists [0] = free_chunks;
264 section->num_free_chunks = LOS_SECTION_NUM_CHUNKS;
266 section->free_chunk_map = (unsigned char*)section + sizeof (LOSSection);
267 g_assert (sizeof (LOSSection) + LOS_SECTION_NUM_CHUNKS + 1 <= LOS_CHUNK_SIZE);
268 section->free_chunk_map [0] = 0;
269 memset (section->free_chunk_map + 1, 1, LOS_SECTION_NUM_CHUNKS);
271 section->next = los_sections;
272 los_sections = section;
274 los_memory_usage_total += LOS_SECTION_SIZE;
281 free_los_section_memory (LOSObject *obj, size_t size)
283 LOSSection *section = LOS_SECTION_FOR_OBJ (obj);
284 size_t num_chunks, i, start_index;
286 size = SGEN_ALIGN_UP_TO (size, LOS_CHUNK_SIZE);
288 num_chunks = size >> LOS_CHUNK_BITS;
290 g_assert (size > 0 && size - sizeof (LOSObject) <= LOS_SECTION_OBJECT_LIMIT);
291 g_assert (num_chunks > 0);
293 section->num_free_chunks += num_chunks;
294 g_assert (section->num_free_chunks <= LOS_SECTION_NUM_CHUNKS);
297 * We could free the LOS section here if it's empty, but we
298 * can't unless we also remove its free chunks from the fast
299 * free lists. Instead, we do it in los_sweep().
302 start_index = LOS_CHUNK_INDEX (obj, section);
303 for (i = start_index; i < start_index + num_chunks; ++i) {
304 g_assert (!section->free_chunk_map [i]);
305 section->free_chunk_map [i] = 1;
308 add_free_chunk ((LOSFreeChunks*)SGEN_ALIGN_DOWN_TO ((mword)obj, LOS_CHUNK_SIZE), size);
312 sgen_los_free_object (LOSObject *obj)
314 if (obj->cardtable_mod_union)
315 sgen_card_table_free_mod_union (obj->cardtable_mod_union, (char*)obj->data, sgen_los_object_size (obj));
318 mword size = sgen_los_object_size (obj);
319 SGEN_LOG (4, "Freed large object %p, size %lu", obj->data, (unsigned long)size);
320 binary_protocol_empty (obj->data, size);
322 los_memory_usage -= size;
328 if (size > LOS_SECTION_OBJECT_LIMIT) {
329 int pagesize = mono_pagesize ();
330 size += sizeof (LOSObject);
331 size = SGEN_ALIGN_UP_TO (size, pagesize);
332 sgen_free_os_memory ((gpointer)SGEN_ALIGN_DOWN_TO ((mword)obj, pagesize), size, SGEN_ALLOC_HEAP, MONO_MEM_ACCOUNT_SGEN_LOS);
333 los_memory_usage_total -= size;
334 sgen_memgov_release_space (size, SPACE_LOS);
336 free_los_section_memory (obj, size + sizeof (LOSObject));
337 #ifdef LOS_CONSISTENCY_CHECKS
338 los_consistency_check ();
346 * Objects with size >= MAX_SMALL_SIZE are allocated in the large object space.
347 * They are currently kept track of with a linked list.
348 * They don't move, so there is no need to pin them during collection
349 * and we avoid the memcpy overhead.
352 sgen_los_alloc_large_inner (GCVTable vtable, size_t size)
354 LOSObject *obj = NULL;
357 g_assert (size > SGEN_MAX_SMALL_OBJ_SIZE);
358 g_assert ((size & 1) == 0);
361 * size + sizeof (LOSObject) <= SSIZE_MAX - (mono_pagesize () - 1)
365 * size <= SSIZE_MAX - (mono_pagesize () - 1) - sizeof (LOSObject)
367 if (size > SSIZE_MAX - (mono_pagesize () - 1) - sizeof (LOSObject))
372 los_segment = sgen_alloc_os_memory (LOS_SEGMENT_SIZE, SGEN_ALLOC_HEAP | SGEN_ALLOC_ACTIVATE, NULL);
373 los_segment_index = ALIGN_UP (los_segment_index);
375 obj = (LOSObject*)(los_segment + los_segment_index);
376 los_segment_index += size + sizeof (LOSObject);
377 g_assert (los_segment_index <= LOS_SEGMENT_SIZE);
379 sgen_ensure_free_space (size, GENERATION_OLD);
382 obj = malloc (size + sizeof (LOSObject));
383 memset (obj, 0, size + sizeof (LOSObject));
385 if (size > LOS_SECTION_OBJECT_LIMIT) {
386 size_t obj_size = size + sizeof (LOSObject);
387 int pagesize = mono_pagesize ();
388 size_t alloc_size = SGEN_ALIGN_UP_TO (obj_size, pagesize);
389 if (sgen_memgov_try_alloc_space (alloc_size, SPACE_LOS)) {
390 obj = (LOSObject *)sgen_alloc_os_memory (alloc_size, (SgenAllocFlags)(SGEN_ALLOC_HEAP | SGEN_ALLOC_ACTIVATE), NULL, MONO_MEM_ACCOUNT_SGEN_LOS);
392 los_memory_usage_total += alloc_size;
393 obj = randomize_los_object_start (obj, obj_size, alloc_size, pagesize);
397 obj = get_los_section_memory (size + sizeof (LOSObject));
399 memset (obj, 0, size + sizeof (LOSObject));
405 g_assert (!((mword)obj->data & (SGEN_ALLOC_ALIGN - 1)));
407 vtslot = (void**)obj->data;
409 sgen_update_heap_boundaries ((mword)obj->data, (mword)obj->data + size);
410 obj->next = los_object_list;
412 * We need a memory barrier so we don't expose as head of the los object list
413 * a LOSObject that doesn't have its fields initialized.
415 mono_memory_write_barrier ();
416 los_object_list = obj;
417 los_memory_usage += size;
419 SGEN_LOG (4, "Allocated large object %p, vtable: %p (%s), size: %zd", obj->data, vtable, sgen_client_vtable_get_name (vtable), size);
420 binary_protocol_alloc (obj->data, vtable, size, sgen_client_get_provenance ());
422 #ifdef LOS_CONSISTENCY_CHECK
423 los_consistency_check ();
429 static void sgen_los_unpin_object (GCObject *data);
432 sgen_los_sweep (void)
434 LOSObject *bigobj, *prevbo;
435 LOSSection *section, *prev;
437 int num_sections = 0;
439 /* sweep the big objects list */
441 for (bigobj = los_object_list; bigobj;) {
442 SGEN_ASSERT (0, !SGEN_OBJECT_IS_PINNED (bigobj->data), "Who pinned a LOS object?");
444 if (sgen_los_object_is_pinned (bigobj->data)) {
445 if (bigobj->cardtable_mod_union) {
446 mword obj_size = sgen_los_object_size (bigobj);
447 mword num_cards = sgen_card_table_number_of_cards_in_range ((mword) bigobj->data, obj_size);
448 memset (bigobj->cardtable_mod_union, 0, num_cards);
451 sgen_los_unpin_object (bigobj->data);
452 sgen_update_heap_boundaries ((mword)bigobj->data, (mword)bigobj->data + sgen_los_object_size (bigobj));
455 /* not referenced anywhere, so we can free it */
457 prevbo->next = bigobj->next;
459 los_object_list = bigobj->next;
461 bigobj = bigobj->next;
462 sgen_los_free_object (to_free);
466 bigobj = bigobj->next;
469 /* Try to free memory */
470 for (i = 0; i < LOS_NUM_FAST_SIZES; ++i)
471 los_fast_free_lists [i] = NULL;
474 section = los_sections;
476 if (section->num_free_chunks == LOS_SECTION_NUM_CHUNKS) {
477 LOSSection *next = section->next;
482 sgen_free_os_memory (section, LOS_SECTION_SIZE, SGEN_ALLOC_HEAP, MONO_MEM_ACCOUNT_SGEN_LOS);
483 sgen_memgov_release_space (LOS_SECTION_SIZE, SPACE_LOS);
486 los_memory_usage_total -= LOS_SECTION_SIZE;
490 for (i = 0; i <= LOS_SECTION_NUM_CHUNKS; ++i) {
491 if (section->free_chunk_map [i]) {
493 for (j = i + 1; j <= LOS_SECTION_NUM_CHUNKS && section->free_chunk_map [j]; ++j)
495 add_free_chunk ((LOSFreeChunks*)((char*)section + (i << LOS_CHUNK_BITS)), (j - i) << LOS_CHUNK_BITS);
501 section = section->next;
506 #ifdef LOS_CONSISTENCY_CHECK
507 los_consistency_check ();
511 g_print ("LOS sections: %d objects: %d usage: %d\n", num_sections, los_num_objects, los_memory_usage);
512 for (i = 0; i < LOS_NUM_FAST_SIZES; ++i) {
514 LOSFreeChunks *free_chunks;
515 for (free_chunks = los_fast_free_lists [i]; free_chunks; free_chunks = free_chunks->next_size)
517 g_print (" %d: %d\n", i, num_chunks);
521 g_assert (los_num_sections == num_sections);
525 sgen_ptr_is_in_los (char *ptr, char **start)
531 for (obj = los_object_list; obj; obj = obj->next) {
532 char *end = (char*)obj->data + sgen_los_object_size (obj);
534 if (ptr >= (char*)obj->data && ptr < end) {
536 *start = (char*)obj->data;
544 sgen_los_iterate_objects (IterateObjectCallbackFunc cb, void *user_data)
548 for (obj = los_object_list; obj; obj = obj->next)
549 cb (obj->data, sgen_los_object_size (obj), user_data);
553 sgen_los_is_valid_object (char *object)
557 for (obj = los_object_list; obj; obj = obj->next) {
558 if ((char*)obj->data == object)
565 mono_sgen_los_describe_pointer (char *ptr)
569 for (obj = los_object_list; obj; obj = obj->next) {
570 const char *los_kind;
574 if ((char*)obj->data > ptr || (char*)obj->data + sgen_los_object_size (obj) <= ptr)
577 size = sgen_los_object_size (obj);
578 pinned = sgen_los_object_is_pinned (obj->data);
580 if (size > LOS_SECTION_OBJECT_LIMIT)
581 los_kind = "huge-los-ptr";
583 los_kind = "los-ptr";
585 if ((char*)obj->data == ptr) {
586 SGEN_LOG (0, "%s (size %d pin %d)\n", los_kind, (int)size, pinned ? 1 : 0);
588 SGEN_LOG (0, "%s (interior-ptr offset %zd size %d pin %d)",
589 los_kind, ptr - (char*)obj->data, (int)size, pinned ? 1 : 0);
598 sgen_los_iterate_live_block_ranges (sgen_cardtable_block_callback callback)
601 for (obj = los_object_list; obj; obj = obj->next) {
602 GCVTable vt = SGEN_LOAD_VTABLE (obj->data);
603 if (SGEN_VTABLE_HAS_REFERENCES (vt))
604 callback ((mword)obj->data, sgen_los_object_size (obj));
609 get_cardtable_mod_union_for_object (LOSObject *obj)
611 mword size = sgen_los_object_size (obj);
612 guint8 *mod_union = obj->cardtable_mod_union;
616 mod_union = sgen_card_table_alloc_mod_union ((char*)obj->data, size);
617 other = (guint8 *)SGEN_CAS_PTR ((gpointer*)&obj->cardtable_mod_union, mod_union, NULL);
619 SGEN_ASSERT (0, obj->cardtable_mod_union == mod_union, "Why did CAS not replace?");
622 sgen_card_table_free_mod_union (mod_union, (char*)obj->data, size);
627 sgen_los_scan_card_table (CardTableScanType scan_type, ScanCopyContext ctx)
631 binary_protocol_los_card_table_scan_start (sgen_timestamp (), scan_type & CARDTABLE_SCAN_MOD_UNION);
632 for (obj = los_object_list; obj; obj = obj->next) {
636 if (!SGEN_OBJECT_HAS_REFERENCES (obj->data))
639 if (scan_type & CARDTABLE_SCAN_MOD_UNION) {
640 if (!sgen_los_object_is_pinned (obj->data))
643 cards = get_cardtable_mod_union_for_object (obj);
645 if (scan_type == CARDTABLE_SCAN_MOD_UNION_PRECLEAN) {
646 guint8 *cards_preclean;
647 mword obj_size = sgen_los_object_size (obj);
648 num_cards = sgen_card_table_number_of_cards_in_range ((mword) obj->data, obj_size);
649 cards_preclean = (guint8 *)sgen_alloc_internal_dynamic (num_cards, INTERNAL_MEM_CARDTABLE_MOD_UNION, TRUE);
651 sgen_card_table_preclean_mod_union (cards, cards_preclean, num_cards);
653 cards = cards_preclean;
659 sgen_cardtable_scan_object (obj->data, sgen_los_object_size (obj), cards, ctx);
661 if (scan_type == CARDTABLE_SCAN_MOD_UNION_PRECLEAN)
662 sgen_free_internal_dynamic (cards, num_cards, INTERNAL_MEM_CARDTABLE_MOD_UNION);
664 binary_protocol_los_card_table_scan_end (sgen_timestamp (), scan_type & CARDTABLE_SCAN_MOD_UNION);
668 sgen_los_count_cards (long long *num_total_cards, long long *num_marked_cards)
671 long long total_cards = 0;
672 long long marked_cards = 0;
674 for (obj = los_object_list; obj; obj = obj->next) {
676 guint8 *cards = sgen_card_table_get_card_scan_address ((mword) obj->data);
677 guint8 *cards_end = sgen_card_table_get_card_scan_address ((mword) obj->data + sgen_los_object_size (obj) - 1);
678 mword num_cards = (cards_end - cards) + 1;
680 if (!SGEN_OBJECT_HAS_REFERENCES (obj->data))
683 total_cards += num_cards;
684 for (i = 0; i < num_cards; ++i) {
690 *num_total_cards = total_cards;
691 *num_marked_cards = marked_cards;
695 sgen_los_update_cardtable_mod_union (void)
699 for (obj = los_object_list; obj; obj = obj->next) {
700 if (!SGEN_OBJECT_HAS_REFERENCES (obj->data))
702 sgen_card_table_update_mod_union (get_cardtable_mod_union_for_object (obj),
703 (char*)obj->data, sgen_los_object_size (obj), NULL);
708 sgen_los_header_for_object (GCObject *data)
711 return (LOSObject*)((char*)data - (int)(&(((LOSObject*)0)->data)));
713 return (LOSObject*)((char*)data - sizeof (LOSObject));
718 sgen_los_pin_object (GCObject *data)
720 LOSObject *obj = sgen_los_header_for_object (data);
721 obj->size = obj->size | 1;
722 binary_protocol_pin (data, (gpointer)SGEN_LOAD_VTABLE (data), sgen_safe_object_get_size (data));
726 sgen_los_unpin_object (GCObject *data)
728 LOSObject *obj = sgen_los_header_for_object (data);
729 obj->size = sgen_los_object_size (obj);
733 sgen_los_object_is_pinned (GCObject *data)
735 LOSObject *obj = sgen_los_header_for_object (data);
736 return obj->size & 1;
740 sgen_los_mark_mod_union_card (GCObject *mono_obj, void **ptr)
742 LOSObject *obj = sgen_los_header_for_object (mono_obj);
743 guint8 *mod_union = get_cardtable_mod_union_for_object (obj);
744 /* The LOSObject structure is not represented within the card space */
745 size_t offset = sgen_card_table_get_card_offset ((char*)ptr, (char*)sgen_card_table_align_pointer((char*)mono_obj));
746 SGEN_ASSERT (0, mod_union, "FIXME: optionally allocate the mod union if it's not here and CAS it in.");
747 mod_union [offset] = 1;
750 #endif /* HAVE_SGEN_GC */