2 * sgen-gc.c: Simple generational GC.
5 * Paolo Molaro (lupus@ximian.com)
7 * Copyright 2005-2009 Novell, Inc (http://www.novell.com)
9 * Thread start/stop adapted from Boehm's GC:
10 * Copyright (c) 1994 by Xerox Corporation. All rights reserved.
11 * Copyright (c) 1996 by Silicon Graphics. All rights reserved.
12 * Copyright (c) 1998 by Fergus Henderson. All rights reserved.
13 * Copyright (c) 2000-2004 by Hewlett-Packard Company. All rights reserved.
15 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
16 * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
18 * Permission is hereby granted to use or copy this program
19 * for any purpose, provided the above notices are retained on all copies.
20 * Permission to modify the code and to distribute modified code is granted,
21 * provided the above notices are retained, and a notice that the code was
22 * modified is included with the above copyright notice.
24 * All the rest of the code is LGPL.
26 * Important: allocation provides always zeroed memory, having to do
27 * a memset after allocation is deadly for performance.
28 * Memory usage at startup is currently as follows:
30 * 64 KB internal space
32 * We should provide a small memory config with half the sizes
34 * We currently try to make as few mono assumptions as possible:
35 * 1) 2-word header with no GC pointers in it (first vtable, second to store the
37 * 2) gc descriptor is the second word in the vtable (first word in the class)
38 * 3) 8 byte alignment is the minimum and enough (not true for special structures, FIXME)
39 * 4) there is a function to get an object's size and the number of
40 * elements in an array.
41 * 5) we know the special way bounds are allocated for complex arrays
43 * Always try to keep stack usage to a minimum: no recursive behaviour
44 * and no large stack allocs.
46 * General description.
47 * Objects are initially allocated in a nursery using a fast bump-pointer technique.
48 * When the nursery is full we start a nursery collection: this is performed with a
50 * When the old generation is full we start a copying GC of the old generation as well:
51 * this will be changed to mark/compact in the future.
52 * The things that complicate this description are:
53 * *) pinned objects: we can't move them so we need to keep track of them
54 * *) no precise info of the thread stacks and registers: we need to be able to
55 * quickly find the objects that may be referenced conservatively and pin them
56 * (this makes the first issues more important)
57 * *) large objects are too expensive to be dealt with using copying GC: we handle them
58 * with mark/sweep during major collections
59 * *) some objects need to not move even if they are small (interned strings, Type handles):
60 * we use mark/sweep for them, too: they are not allocated in the nursery, but inside
61 * PinnedChunks regions
66 *) change the jit to emit write barrier calls when needed (we
67 can have specialized write barriers): done with icalls, still need to
68 use some specialized barriers
69 *) we could have a function pointer in MonoClass to implement
70 customized write barriers for value types
71 *) the write barrier code could be isolated in a couple of functions: when a
72 thread is stopped if it's inside the barrier it is let go again
73 until we stop outside of them (not really needed, see below GC-safe points)
74 *) investigate the stuff needed to advance a thread to a GC-safe
75 point (single-stepping, read from unmapped memory etc) and implement it
76 Not needed yet: since we treat the objects reachable from the stack/regs as
77 roots, we store the ptr and exec the write barrier so there is no race.
78 We may need this to solve the issue with setting the length of arrays and strings.
79 We may need this also for handling precise info on stacks, even simple things
80 as having uninitialized data on the stack and having to wait for the prolog
81 to zero it. Not an issue for the last frame that we scan conservatively.
82 We could always not trust the value in the slots anyway.
83 *) make the jit info table lock free
84 *) modify the jit to save info about references in stack locations:
85 this can be done just for locals as a start, so that at least
86 part of the stack is handled precisely.
87 *) Make the debug printf stuff thread and signal safe.
88 *) test/fix 64 bit issues
89 *) test/fix endianess issues
91 *) add batch moving profile info
92 *) add more timing info
93 *) there is a possible race when an array or string is created: the vtable is set,
94 but the length is set only later so if the GC needs to scan the object in that window,
95 it won't get the correct size for the object. The object can't have references and it will
96 be pinned, but a free memory fragment may be created that overlaps with it.
97 We should change the array max_length field to be at the same offset as the string length:
98 this way we can have a single special alloc function for them that sets the length.
99 Multi-dim arrays have the same issue for rank == 1 for the bounds data.
100 *) implement a card table as the write barrier instead of remembered sets?
101 *) some sort of blacklist support?
102 *) fin_ready_list and critical_fin_list are part of the root set, too
103 *) consider lowering the large object min size to 16/32KB or so and benchmark
104 *) once mark-compact is implemented we could still keep the
105 copying collector for the old generation and use it if we think
106 it is better (small heaps and no pinning object in the old
108 *) avoid the memory store from copy_object when not needed.
109 *) optimize the write barriers fastpath to happen in managed code
110 *) add an option to mmap the whole heap in one chunk: it makes for many
111 simplifications in the checks (put the nursery at the top and just use a single
112 check for inclusion/exclusion): the issue this has is that on 32 bit systems it's
113 not flexible (too much of the address space may be used by default or we can't
114 increase the heap as needed) and we'd need a race-free mechanism to return memory
115 back to the system (mprotect(PROT_NONE) will still keep the memory allocated if it
116 was written to, munmap is needed, but the following mmap may not find the same segment
118 *) memzero the fragments after restarting the world and optionally a smaller chunk at a time
119 *) an additional strategy to realloc/expand the nursery when fully pinned is to start
120 allocating objects in the old generation. This means that we can't optimize away write
121 barrier calls in ctors (but that is not valid for other reasons, too).
122 *) add write barriers to the Clone methods
130 #include <semaphore.h>
134 #include "metadata/metadata-internals.h"
135 #include "metadata/class-internals.h"
136 #include "metadata/gc-internal.h"
137 #include "metadata/object-internals.h"
138 #include "metadata/threads.h"
139 #include "metadata/sgen-gc.h"
140 #include "metadata/sgen-archdep.h"
141 #include "metadata/mono-gc.h"
142 #include "metadata/method-builder.h"
143 #include "metadata/profiler-private.h"
144 #include "metadata/monitor.h"
145 #include "metadata/threadpool-internals.h"
146 #include "metadata/mempool-internals.h"
147 #include "metadata/marshal.h"
148 #include "utils/mono-mmap.h"
149 #include "utils/mono-time.h"
150 #include "utils/mono-semaphore.h"
151 #include "utils/mono-counters.h"
153 #include <mono/utils/memcheck.h>
155 #define OPDEF(a,b,c,d,e,f,g,h,i,j) \
159 #include "mono/cil/opcode.def"
166 * ######################################################################
167 * ######## Types and constants used by the GC.
168 * ######################################################################
170 #if SIZEOF_VOID_P == 4
171 typedef guint32 mword;
173 typedef guint64 mword;
176 static int gc_initialized = 0;
177 static int gc_debug_level = 0;
178 static FILE* gc_debug_file;
179 /* If set, do a minor collection before every allocation */
180 static gboolean collect_before_allocs = FALSE;
181 /* If set, do a heap consistency check before each minor collection */
182 static gboolean consistency_check_at_minor_collection = FALSE;
183 /* If set, check that there are no references to the domain left at domain unload */
184 static gboolean xdomain_checks = FALSE;
185 /* If not null, dump the heap after each collection into this file */
186 static FILE *heap_dump_file = NULL;
187 /* If set, mark stacks conservatively, even if precise marking is possible */
188 static gboolean conservative_stack_mark = FALSE;
191 * Turning on heavy statistics will turn off the managed allocator and
192 * the managed write barrier.
194 //#define HEAVY_STATISTICS
196 #ifdef HEAVY_STATISTICS
197 #define HEAVY_STAT(x) x
199 #define HEAVY_STAT(x)
202 #ifdef HEAVY_STATISTICS
203 static long stat_objects_alloced = 0;
204 static long stat_copy_object_called_nursery = 0;
205 static long stat_objects_copied_nursery = 0;
206 static long stat_copy_object_called_major = 0;
207 static long stat_objects_copied_major = 0;
209 static long stat_copy_object_failed_from_space = 0;
210 static long stat_copy_object_failed_forwarded = 0;
211 static long stat_copy_object_failed_pinned = 0;
212 static long stat_copy_object_failed_large_pinned = 0;
213 static long stat_copy_object_failed_to_space = 0;
215 static long stat_store_remsets = 0;
216 static long stat_store_remsets_unique = 0;
217 static long stat_saved_remsets_1 = 0;
218 static long stat_saved_remsets_2 = 0;
219 static long stat_global_remsets_added = 0;
220 static long stat_global_remsets_processed = 0;
222 static long num_copy_object_called = 0;
223 static long num_objects_copied = 0;
225 static int stat_wbarrier_set_field = 0;
226 static int stat_wbarrier_set_arrayref = 0;
227 static int stat_wbarrier_arrayref_copy = 0;
228 static int stat_wbarrier_generic_store = 0;
229 static int stat_wbarrier_generic_store_remset = 0;
230 static int stat_wbarrier_set_root = 0;
231 static int stat_wbarrier_value_copy = 0;
232 static int stat_wbarrier_object_copy = 0;
235 static long pinned_chunk_bytes_alloced = 0;
236 static long large_internal_bytes_alloced = 0;
239 INTERNAL_MEM_PIN_QUEUE,
240 INTERNAL_MEM_FRAGMENT,
241 INTERNAL_MEM_SECTION,
242 INTERNAL_MEM_SCAN_STARTS,
243 INTERNAL_MEM_FIN_TABLE,
244 INTERNAL_MEM_FINALIZE_ENTRY,
245 INTERNAL_MEM_DISLINK_TABLE,
246 INTERNAL_MEM_DISLINK,
247 INTERNAL_MEM_ROOTS_TABLE,
248 INTERNAL_MEM_ROOT_RECORD,
249 INTERNAL_MEM_STATISTICS,
251 INTERNAL_MEM_GRAY_QUEUE,
252 INTERNAL_MEM_STORE_REMSET,
256 static long small_internal_mem_bytes [INTERNAL_MEM_MAX];
260 mono_gc_flush_info (void)
262 fflush (gc_debug_file);
266 #define MAX_DEBUG_LEVEL 8
267 #define DEBUG(level,a) do {if (G_UNLIKELY ((level) <= MAX_DEBUG_LEVEL && (level) <= gc_debug_level)) a;} while (0)
269 #define TV_DECLARE(name) gint64 name
270 #define TV_GETTIME(tv) tv = mono_100ns_ticks ()
271 #define TV_ELAPSED(start,end) (int)((end-start) / 10)
273 #define ALIGN_TO(val,align) ((((guint64)val) + ((align) - 1)) & ~((align) - 1))
275 #define GC_BITS_PER_WORD (sizeof (mword) * 8)
283 typedef struct _Block Block;
289 /* each request from the OS ends up in a GCMemSection */
290 typedef struct _GCMemSection GCMemSection;
291 struct _GCMemSection {
295 /* pointer where more data could be allocated if it fits */
299 * scan starts is an array of pointers to objects equally spaced in the allocation area
300 * They let use quickly find pinned objects from pinning pointers.
303 /* in major collections indexes in the pin_queue for objects that pin this section */
306 unsigned short num_scan_start;
307 gboolean is_to_space;
310 #define SIZEOF_GC_MEM_SECTION ((sizeof (GCMemSection) + 7) & ~7)
312 /* large object space struct: 64+ KB */
313 /* we could make this limit much smaller to avoid memcpy copy
314 * and potentially have more room in the GC descriptor: need to measure
315 * This also means that such small OS objects will need to be
316 * allocated in a different way (using pinned chunks).
317 * We may want to put large but smaller than 64k objects in the fixed space
318 * when we move the object from one generation to another (to limit the
319 * pig in the snake effect).
320 * Note: it may be worth to have an optimized copy function, since we can
321 * assume that objects are aligned and have a multiple of 8 size.
322 * FIXME: This structure needs to be a multiple of 8 bytes in size: this is not
323 * true if MONO_ZERO_LEN_ARRAY is nonzero.
325 typedef struct _LOSObject LOSObject;
328 mword size; /* this is the object size */
329 int dummy; /* to have a sizeof (LOSObject) a multiple of ALLOC_ALIGN and data starting at same alignment */
332 char data [MONO_ZERO_LEN_ARRAY];
335 /* Pinned objects are allocated in the LOS space if bigger than half a page
336 * or from freelists otherwise. We assume that pinned objects are relatively few
337 * and they have a slow dying speed (like interned strings, thread objects).
338 * As such they will be collected only at major collections.
339 * free lists are not global: when we need memory we allocate a PinnedChunk.
340 * Each pinned chunk is made of several pages, the first of wich is used
341 * internally for bookeeping (here think of a page as 4KB). The bookeeping
342 * includes the freelists vectors and info about the object size of each page
343 * in the pinned chunk. So, when needed, a free page is found in a pinned chunk,
344 * a size is assigned to it, the page is divided in the proper chunks and each
345 * chunk is added to the freelist. To not waste space, the remaining space in the
346 * first page is used as objects of size 16 or 32 (need to measure which are more
348 * We use this same structure to allocate memory used internally by the GC, so
349 * we never use malloc/free if we need to alloc during collection: the world is stopped
350 * and malloc/free will deadlock.
351 * When we want to iterate over pinned objects, we just scan a page at a time
352 * linearly according to the size of objects in the page: the next pointer used to link
353 * the items in the freelist uses the same word as the vtable. Since we keep freelists
354 * for each pinned chunk, if the word points outside the pinned chunk it means
356 * We could avoid this expensive scanning in creative ways. We could have a policy
357 * of putting in the pinned space only objects we know about that have no struct fields
358 * with references and we can easily use a even expensive write barrier for them,
359 * since pointer writes on such objects should be rare.
360 * The best compromise is to just alloc interned strings and System.MonoType in them.
361 * It would be nice to allocate MonoThread in it, too: must check that we properly
362 * use write barriers so we don't have to do any expensive scanning of the whole pinned
363 * chunk list during minor collections. We can avoid it now because we alloc in it only
364 * reference-free objects.
366 #define PINNED_FIRST_SLOT_SIZE (sizeof (gpointer) * 4)
367 #define MAX_FREELIST_SIZE 2048
368 #define PINNED_PAGE_SIZE (4096)
369 #define PINNED_CHUNK_MIN_SIZE (4096*8)
370 typedef struct _PinnedChunk PinnedChunk;
371 struct _PinnedChunk {
374 int *page_sizes; /* a 0 means the page is still unused */
377 void *data [1]; /* page sizes and free lists are stored here */
380 /* The method used to clear the nursery */
381 /* Clearing at nursery collections is the safest, but has bad interactions with caches.
382 * Clearing at TLAB creation is much faster, but more complex and it might expose hard
387 CLEAR_AT_TLAB_CREATION
388 } NurseryClearPolicy;
390 static NurseryClearPolicy nursery_clear_policy = CLEAR_AT_TLAB_CREATION;
393 * If this is set, the nursery is aligned to an address aligned to its size, ie.
394 * a 1MB nursery will be aligned to an address divisible by 1MB. This allows us to
395 * speed up ptr_in_nursery () checks which are very frequent. This requires the
396 * nursery size to be a compile time constant.
398 #define ALIGN_NURSERY 1
401 * The young generation is divided into fragments. This is because
402 * we can hand one fragments to a thread for lock-less fast alloc and
403 * because the young generation ends up fragmented anyway by pinned objects.
404 * Once a collection is done, a list of fragments is created. When doing
405 * thread local alloc we use smallish nurseries so we allow new threads to
406 * allocate memory from gen0 without triggering a collection. Threads that
407 * are found to allocate lots of memory are given bigger fragments. This
408 * should make the finalizer thread use little nursery memory after a while.
409 * We should start assigning threads very small fragments: if there are many
410 * threads the nursery will be full of reserved space that the threads may not
411 * use at all, slowing down allocation speed.
412 * Thread local allocation is done from areas of memory Hotspot calls Thread Local
413 * Allocation Buffers (TLABs).
415 typedef struct _Fragment Fragment;
419 char *fragment_start;
420 char *fragment_limit; /* the current soft limit for allocation */
424 /* the runtime can register areas of memory as roots: we keep two lists of roots,
425 * a pinned root set for conservatively scanned roots and a normal one for
426 * precisely scanned roots (currently implemented as a single list).
428 typedef struct _RootRecord RootRecord;
436 /* for use with write barriers */
437 typedef struct _RememberedSet RememberedSet;
438 struct _RememberedSet {
442 mword data [MONO_ZERO_LEN_ARRAY];
446 * We're never actually using the first element. It's always set to
447 * NULL to simplify the elimination of consecutive duplicate
450 #define STORE_REMSET_BUFFER_SIZE 1024
452 typedef struct _GenericStoreRememberedSet GenericStoreRememberedSet;
453 struct _GenericStoreRememberedSet {
454 GenericStoreRememberedSet *next;
455 /* We need one entry less because the first entry of store
456 remset buffers is always a dummy and we don't copy it. */
457 gpointer data [STORE_REMSET_BUFFER_SIZE - 1];
460 /* we have 4 possible values in the low 2 bits */
462 REMSET_LOCATION, /* just a pointer to the exact location */
463 REMSET_RANGE, /* range of pointer fields */
464 REMSET_OBJECT, /* mark all the object for scanning */
465 REMSET_OTHER, /* all others */
466 REMSET_TYPE_MASK = 0x3
469 /* Subtypes of REMSET_OTHER */
471 REMSET_VTYPE, /* a valuetype array described by a gc descriptor and a count */
472 REMSET_ROOT_LOCATION, /* a location inside a root */
475 #ifdef HAVE_KW_THREAD
476 static __thread RememberedSet *remembered_set MONO_TLS_FAST;
478 static pthread_key_t remembered_set_key;
479 static RememberedSet *global_remset;
480 static RememberedSet *freed_thread_remsets;
481 //static int store_to_global_remset = 0;
482 static GenericStoreRememberedSet *generic_store_remsets = NULL;
484 /* FIXME: later choose a size that takes into account the RememberedSet struct
485 * and doesn't waste any alloc paddin space.
487 #define DEFAULT_REMSET_SIZE 1024
488 static RememberedSet* alloc_remset (int size, gpointer id);
490 /* Structure that corresponds to a MonoVTable: desc is a mword so requires
491 * no cast from a pointer to an integer
498 /* these bits are set in the object vtable: we could merge them since an object can be
499 * either pinned or forwarded but not both.
500 * We store them in the vtable slot because the bits are used in the sync block for
501 * other purposes: if we merge them and alloc the sync blocks aligned to 8 bytes, we can change
502 * this and use bit 3 in the syncblock (with the lower two bits both set for forwarded, that
503 * would be an invalid combination for the monitor and hash code).
504 * The values are already shifted.
505 * The forwarding address is stored in the sync block.
507 #define FORWARDED_BIT 1
509 #define VTABLE_BITS_MASK 0x3
511 /* returns NULL if not forwarded, or the forwarded address */
512 #define object_is_forwarded(obj) (((mword*)(obj))[0] & FORWARDED_BIT? (void*)(((mword*)(obj))[1]): NULL)
513 /* set the forwarded address fw_addr for object obj */
514 #define forward_object(obj,fw_addr) do { \
515 ((mword*)(obj))[0] |= FORWARDED_BIT; \
516 ((mword*)(obj))[1] = (mword)(fw_addr); \
519 #define object_is_pinned(obj) (((mword*)(obj))[0] & PINNED_BIT)
520 #define pin_object(obj) do { \
521 ((mword*)(obj))[0] |= PINNED_BIT; \
523 #define unpin_object(obj) do { \
524 ((mword*)(obj))[0] &= ~PINNED_BIT; \
528 #define ptr_in_nursery(ptr) (((mword)(ptr) & ~((1 << DEFAULT_NURSERY_BITS) - 1)) == (mword)nursery_start)
530 #define ptr_in_nursery(ptr) ((char*)(ptr) >= nursery_start && (char*)(ptr) < nursery_real_end)
534 * Since we set bits in the vtable, use the macro to load it from the pointer to
535 * an object that is potentially pinned.
537 #define LOAD_VTABLE(addr) ((*(mword*)(addr)) & ~VTABLE_BITS_MASK)
540 safe_name (void* obj)
542 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (obj);
543 return vt->klass->name;
547 safe_object_get_size (MonoObject* o)
549 MonoClass *klass = ((MonoVTable*)LOAD_VTABLE (o))->klass;
550 if (klass == mono_defaults.string_class) {
551 return sizeof (MonoString) + 2 * mono_string_length ((MonoString*) o) + 2;
552 } else if (klass->rank) {
553 MonoArray *array = (MonoArray*)o;
554 size_t size = sizeof (MonoArray) + mono_array_element_size (klass) * mono_array_length (array);
555 if (G_UNLIKELY (array->bounds)) {
556 size += sizeof (mono_array_size_t) - 1;
557 size &= ~(sizeof (mono_array_size_t) - 1);
558 size += sizeof (MonoArrayBounds) * klass->rank;
562 /* from a created object: the class must be inited already */
563 return klass->instance_size;
568 * ######################################################################
569 * ######## Global data.
570 * ######################################################################
572 static LOCK_DECLARE (gc_mutex);
573 static int gc_disabled = 0;
574 static int num_minor_gcs = 0;
575 static int num_major_gcs = 0;
577 /* good sizes are 512KB-1MB: larger ones increase a lot memzeroing time */
578 //#define DEFAULT_NURSERY_SIZE (1024*512*125+4096*118)
579 #define DEFAULT_NURSERY_SIZE (1024*512*2)
580 /* The number of trailing 0 bits in DEFAULT_NURSERY_SIZE */
581 #define DEFAULT_NURSERY_BITS 20
582 #define MAJOR_SECTION_SIZE (128*1024)
583 #define BLOCK_FOR_OBJECT(o) ((Block*)(((mword)(o)) & ~(MAJOR_SECTION_SIZE - 1)))
584 #define MAJOR_SECTION_FOR_OBJECT(o) ((GCMemSection*)BLOCK_FOR_OBJECT ((o)))
585 #define MIN_MINOR_COLLECTION_SECTION_ALLOWANCE (DEFAULT_NURSERY_SIZE * 3 / MAJOR_SECTION_SIZE)
586 #define DEFAULT_LOS_COLLECTION_TARGET (DEFAULT_NURSERY_SIZE * 2)
587 /* to quickly find the head of an object pinned by a conservative address
588 * we keep track of the objects allocated for each SCAN_START_SIZE memory
589 * chunk in the nursery or other memory sections. Larger values have less
590 * memory overhead and bigger runtime cost. 4-8 KB are reasonable values.
592 #define SCAN_START_SIZE (4096*2)
593 /* the minimum size of a fragment that we consider useful for allocation */
594 #define FRAGMENT_MIN_SIZE (512)
595 /* This is a fixed value used for pinned chunks, not the system pagesize */
596 #define FREELIST_PAGESIZE 4096
598 static mword pagesize = 4096;
599 static mword nursery_size = DEFAULT_NURSERY_SIZE;
600 static int degraded_mode = 0;
602 static int minor_collection_section_allowance = MIN_MINOR_COLLECTION_SECTION_ALLOWANCE;
603 static int minor_collection_sections_alloced = 0;
604 static int num_major_sections = 0;
606 static LOSObject *los_object_list = NULL;
607 static mword los_memory_usage = 0;
608 static mword los_num_objects = 0;
609 static mword next_los_collection = 2*1024*1024; /* 2 MB, need to tune */
610 static mword total_alloc = 0;
611 /* use this to tune when to do a major/minor collection */
612 static mword memory_pressure = 0;
614 static GCMemSection *section_list = NULL;
615 static GCMemSection *nursery_section = NULL;
616 static mword lowest_heap_address = ~(mword)0;
617 static mword highest_heap_address = 0;
619 typedef struct _FinalizeEntry FinalizeEntry;
620 struct _FinalizeEntry {
625 typedef struct _FinalizeEntryHashTable FinalizeEntryHashTable;
626 struct _FinalizeEntryHashTable {
627 FinalizeEntry **table;
632 typedef struct _DisappearingLink DisappearingLink;
633 struct _DisappearingLink {
634 DisappearingLink *next;
638 typedef struct _DisappearingLinkHashTable DisappearingLinkHashTable;
639 struct _DisappearingLinkHashTable {
640 DisappearingLink **table;
645 #define LARGE_INTERNAL_MEM_HEADER_MAGIC 0x7d289f3a
647 typedef struct _LargeInternalMemHeader LargeInternalMemHeader;
648 struct _LargeInternalMemHeader {
661 * The link pointer is hidden by negating each bit. We use the lowest
662 * bit of the link (before negation) to store whether it needs
663 * resurrection tracking.
665 #define HIDE_POINTER(p,t) ((gpointer)(~((gulong)(p)|((t)?1:0))))
666 #define REVEAL_POINTER(p) ((gpointer)((~(gulong)(p))&~3L))
668 #define DISLINK_OBJECT(d) (REVEAL_POINTER (*(d)->link))
669 #define DISLINK_TRACK(d) ((~(gulong)(*(d)->link)) & 1)
672 * The finalizable hash has the object as the key, the
673 * disappearing_link hash, has the link address as key.
675 static FinalizeEntryHashTable minor_finalizable_hash;
676 static FinalizeEntryHashTable major_finalizable_hash;
677 /* objects that are ready to be finalized */
678 static FinalizeEntry *fin_ready_list = NULL;
679 static FinalizeEntry *critical_fin_list = NULL;
681 static DisappearingLinkHashTable minor_disappearing_link_hash;
682 static DisappearingLinkHashTable major_disappearing_link_hash;
684 static int num_ready_finalizers = 0;
685 static int no_finalize = 0;
687 /* keep each size a multiple of ALLOC_ALIGN */
688 /* on 64 bit systems 8 is likely completely unused. */
689 static const int freelist_sizes [] = {
690 8, 16, 24, 32, 40, 48, 64, 80,
691 96, 128, 160, 192, 224, 256, 320, 384,
692 448, 512, 584, 680, 816, 1024, 1360, 2048};
693 #define FREELIST_NUM_SLOTS (sizeof (freelist_sizes) / sizeof (freelist_sizes [0]))
695 static char* max_pinned_chunk_addr = NULL;
696 static char* min_pinned_chunk_addr = (char*)-1;
697 /* pinned_chunk_list is used for allocations of objects that are never moved */
698 static PinnedChunk *pinned_chunk_list = NULL;
699 /* internal_chunk_list is used for allocating structures needed by the GC */
700 static PinnedChunk *internal_chunk_list = NULL;
703 obj_is_from_pinned_alloc (char *p)
705 return BLOCK_FOR_OBJECT (p)->role == MEMORY_ROLE_PINNED;
708 static int slot_for_size (size_t size);
711 free_pinned_object (PinnedChunk *chunk, char *obj, size_t size)
713 void **p = (void**)obj;
714 int slot = slot_for_size (size);
716 g_assert (obj >= (char*)chunk->start_data && obj < ((char*)chunk + chunk->num_pages * FREELIST_PAGESIZE));
717 *p = chunk->free_list [slot];
718 chunk->free_list [slot] = p;
722 ROOT_TYPE_NORMAL = 0, /* "normal" roots */
723 ROOT_TYPE_PINNED = 1, /* roots without a GC descriptor */
724 ROOT_TYPE_WBARRIER = 2, /* roots with a write barrier */
728 /* registered roots: the key to the hash is the root start address */
730 * Different kinds of roots are kept separate to speed up pin_from_roots () for example.
732 static RootRecord **roots_hash [ROOT_TYPE_NUM] = { NULL, NULL };
733 static int roots_hash_size [ROOT_TYPE_NUM] = { 0, 0, 0 };
734 static mword roots_size = 0; /* amount of memory in the root set */
735 static int num_roots_entries [ROOT_TYPE_NUM] = { 0, 0, 0 };
738 * The current allocation cursors
739 * We allocate objects in the nursery.
740 * The nursery is the area between nursery_start and nursery_real_end.
741 * Allocation is done from a Thread Local Allocation Buffer (TLAB). TLABs are allocated
742 * from nursery fragments.
743 * tlab_next is the pointer to the space inside the TLAB where the next object will
745 * tlab_temp_end is the pointer to the end of the temporary space reserved for
746 * the allocation: it allows us to set the scan starts at reasonable intervals.
747 * tlab_real_end points to the end of the TLAB.
748 * nursery_frag_real_end points to the end of the currently used nursery fragment.
749 * nursery_first_pinned_start points to the start of the first pinned object in the nursery
750 * nursery_last_pinned_end points to the end of the last pinned object in the nursery
751 * At the next allocation, the area of the nursery where objects can be present is
752 * between MIN(nursery_first_pinned_start, first_fragment_start) and
753 * MAX(nursery_last_pinned_end, nursery_frag_real_end)
755 static char *nursery_start = NULL;
757 /* eventually share with MonoThread? */
758 typedef struct _SgenThreadInfo SgenThreadInfo;
760 struct _SgenThreadInfo {
761 SgenThreadInfo *next;
763 unsigned int stop_count; /* to catch duplicate signals */
768 void *stack_start_limit;
769 char **tlab_next_addr;
770 char **tlab_start_addr;
771 char **tlab_temp_end_addr;
772 char **tlab_real_end_addr;
773 gpointer **store_remset_buffer_addr;
774 long *store_remset_buffer_index_addr;
775 RememberedSet *remset;
776 gpointer runtime_data;
777 gpointer stopped_ip; /* only valid if the thread is stopped */
778 MonoDomain *stopped_domain; /* ditto */
779 gpointer *stopped_regs; /* ditto */
780 #ifndef HAVE_KW_THREAD
785 gpointer *store_remset_buffer;
786 long store_remset_buffer_index;
790 #ifdef HAVE_KW_THREAD
791 #define TLAB_ACCESS_INIT
792 #define TLAB_START tlab_start
793 #define TLAB_NEXT tlab_next
794 #define TLAB_TEMP_END tlab_temp_end
795 #define TLAB_REAL_END tlab_real_end
796 #define REMEMBERED_SET remembered_set
797 #define STORE_REMSET_BUFFER store_remset_buffer
798 #define STORE_REMSET_BUFFER_INDEX store_remset_buffer_index
800 static pthread_key_t thread_info_key;
801 #define TLAB_ACCESS_INIT SgenThreadInfo *__thread_info__ = pthread_getspecific (thread_info_key)
802 #define TLAB_START (__thread_info__->tlab_start)
803 #define TLAB_NEXT (__thread_info__->tlab_next)
804 #define TLAB_TEMP_END (__thread_info__->tlab_temp_end)
805 #define TLAB_REAL_END (__thread_info__->tlab_real_end)
806 #define REMEMBERED_SET (__thread_info__->remset)
807 #define STORE_REMSET_BUFFER (__thread_info__->store_remset_buffer)
808 #define STORE_REMSET_BUFFER_INDEX (__thread_info__->store_remset_buffer_index)
812 * FIXME: What is faster, a TLS variable pointing to a structure, or separate TLS
813 * variables for next+temp_end ?
815 #ifdef HAVE_KW_THREAD
816 static __thread char *tlab_start;
817 static __thread char *tlab_next;
818 static __thread char *tlab_temp_end;
819 static __thread char *tlab_real_end;
820 static __thread gpointer *store_remset_buffer;
821 static __thread long store_remset_buffer_index;
822 /* Used by the managed allocator/wbarrier */
823 static __thread char **tlab_next_addr;
824 static __thread char *stack_end;
825 static __thread long *store_remset_buffer_index_addr;
827 static char *nursery_next = NULL;
828 static char *nursery_frag_real_end = NULL;
829 static char *nursery_real_end = NULL;
830 //static char *nursery_first_pinned_start = NULL;
831 static char *nursery_last_pinned_end = NULL;
833 /* The size of a TLAB */
834 /* The bigger the value, the less often we have to go to the slow path to allocate a new
835 * one, but the more space is wasted by threads not allocating much memory.
837 * FIXME: Make this self-tuning for each thread.
839 static guint32 tlab_size = (1024 * 4);
841 /* fragments that are free and ready to be used for allocation */
842 static Fragment *nursery_fragments = NULL;
843 /* freeelist of fragment structures */
844 static Fragment *fragment_freelist = NULL;
847 * used when moving the objects
849 static char *to_space_bumper = NULL;
850 static char *to_space_top = NULL;
851 static GCMemSection *to_space_section = NULL;
853 /* objects bigger then this go into the large object space */
854 #define MAX_SMALL_OBJ_SIZE MAX_FREELIST_SIZE
856 /* Functions supplied by the runtime to be called by the GC */
857 static MonoGCCallbacks gc_callbacks;
860 * ######################################################################
861 * ######## Macros and function declarations.
862 * ######################################################################
865 #define UPDATE_HEAP_BOUNDARIES(low,high) do { \
866 if ((mword)(low) < lowest_heap_address) \
867 lowest_heap_address = (mword)(low); \
868 if ((mword)(high) > highest_heap_address) \
869 highest_heap_address = (mword)(high); \
871 #define ADDR_IN_HEAP_BOUNDARIES(addr) ((p) >= lowest_heap_address && (p) < highest_heap_address)
874 align_pointer (void *ptr)
876 mword p = (mword)ptr;
877 p += sizeof (gpointer) - 1;
878 p &= ~ (sizeof (gpointer) - 1);
882 /* forward declarations */
883 static void* get_internal_mem (size_t size, int type);
884 static void free_internal_mem (void *addr, int type);
885 static void* get_os_memory (size_t size, int activate);
886 static void free_os_memory (void *addr, size_t size);
887 static G_GNUC_UNUSED void report_internal_mem_usage (void);
889 static int stop_world (void);
890 static int restart_world (void);
891 static void scan_thread_data (void *start_nursery, void *end_nursery, gboolean precise);
892 static void scan_from_remsets (void *start_nursery, void *end_nursery);
893 static void find_pinning_ref_from_thread (char *obj, size_t size);
894 static void update_current_thread_stack (void *start);
895 static GCMemSection* alloc_major_section (void);
896 static void finalize_in_range (char *start, char *end, int generation);
897 static void add_or_remove_disappearing_link (MonoObject *obj, void **link, gboolean track, int generation);
898 static void null_link_in_range (char *start, char *end, int generation);
899 static void null_links_for_domain (MonoDomain *domain, int generation);
900 static gboolean search_fragment_for_size (size_t size);
901 static void mark_pinned_from_addresses (PinnedChunk *chunk, void **start, void **end);
902 static void clear_remsets (void);
903 static void clear_tlabs (void);
904 typedef void (*ScanPinnedObjectCallbackFunc) (PinnedChunk*, char*, size_t, void*);
905 static void scan_pinned_objects (ScanPinnedObjectCallbackFunc callback, void *callback_data);
906 static void sweep_pinned_objects (void);
907 static void scan_from_pinned_objects (char *addr_start, char *addr_end);
908 static void free_large_object (LOSObject *obj);
909 static void free_major_section (GCMemSection *section);
910 static void to_space_expand (void);
912 static void mono_gc_register_disappearing_link (MonoObject *obj, void **link, gboolean track);
914 void describe_ptr (char *ptr);
915 void check_consistency (void);
916 char* check_object (char *start);
918 void mono_gc_scan_for_specific_ref (MonoObject *key);
921 * ######################################################################
922 * ######## GC descriptors
923 * ######################################################################
924 * Used to quickly get the info the GC needs about an object: size and
925 * where the references are held.
927 /* objects are aligned to 8 bytes boundaries
928 * A descriptor is a pointer in MonoVTable, so 32 or 64 bits of size.
929 * The low 3 bits define the type of the descriptor. The other bits
930 * depend on the type.
931 * As a general rule the 13 remaining low bits define the size, either
932 * of the whole object or of the elements in the arrays. While for objects
933 * the size is already in bytes, for arrays we need to shift, because
934 * array elements might be smaller than 8 bytes. In case of arrays, we
935 * use two bits to describe what the additional high bits represents,
936 * so the default behaviour can handle element sizes less than 2048 bytes.
937 * The high 16 bits, if 0 it means the object is pointer-free.
938 * This design should make it easy and fast to skip over ptr-free data.
939 * The first 4 types should cover >95% of the objects.
940 * Note that since the size of objects is limited to 64K, larger objects
941 * will be allocated in the large object heap.
942 * If we want 4-bytes alignment, we need to put vector and small bitmap
946 DESC_TYPE_RUN_LENGTH, /* 16 bits aligned byte size | 1-3 (offset, numptr) bytes tuples */
947 DESC_TYPE_SMALL_BITMAP, /* 16 bits aligned byte size | 16-48 bit bitmap */
948 DESC_TYPE_STRING, /* nothing */
949 DESC_TYPE_COMPLEX, /* index for bitmap into complex_descriptors */
950 DESC_TYPE_VECTOR, /* 10 bits element size | 1 bit array | 2 bits desc | element desc */
951 DESC_TYPE_ARRAY, /* 10 bits element size | 1 bit array | 2 bits desc | element desc */
952 DESC_TYPE_LARGE_BITMAP, /* | 29-61 bitmap bits */
953 DESC_TYPE_COMPLEX_ARR, /* index for bitmap into complex_descriptors */
954 /* subtypes for arrays and vectors */
955 DESC_TYPE_V_PTRFREE = 0,/* there are no refs: keep first so it has a zero value */
956 DESC_TYPE_V_REFS, /* all the array elements are refs */
957 DESC_TYPE_V_RUN_LEN, /* elements are run-length encoded as DESC_TYPE_RUN_LENGTH */
958 DESC_TYPE_V_BITMAP /* elements are as the bitmap in DESC_TYPE_SMALL_BITMAP */
961 #define OBJECT_HEADER_WORDS (sizeof(MonoObject)/sizeof(gpointer))
962 #define LOW_TYPE_BITS 3
963 #define SMALL_BITMAP_SHIFT 16
964 #define SMALL_BITMAP_SIZE (GC_BITS_PER_WORD - SMALL_BITMAP_SHIFT)
965 #define VECTOR_INFO_SHIFT 14
966 #define VECTOR_ELSIZE_SHIFT 3
967 #define LARGE_BITMAP_SIZE (GC_BITS_PER_WORD - LOW_TYPE_BITS)
968 #define MAX_SMALL_SIZE ((1 << SMALL_BITMAP_SHIFT) - 1)
969 #define SMALL_SIZE_MASK 0xfff8
970 #define MAX_ELEMENT_SIZE 0x3ff
971 #define ELEMENT_SIZE_MASK (0x3ff << LOW_TYPE_BITS)
972 #define VECTOR_SUBTYPE_PTRFREE (DESC_TYPE_V_PTRFREE << VECTOR_INFO_SHIFT)
973 #define VECTOR_SUBTYPE_REFS (DESC_TYPE_V_REFS << VECTOR_INFO_SHIFT)
974 #define VECTOR_SUBTYPE_RUN_LEN (DESC_TYPE_V_RUN_LEN << VECTOR_INFO_SHIFT)
975 #define VECTOR_SUBTYPE_BITMAP (DESC_TYPE_V_BITMAP << VECTOR_INFO_SHIFT)
977 #define ALLOC_ALIGN 8
980 /* Root bitmap descriptors are simpler: the lower three bits describe the type
981 * and we either have 30/62 bitmap bits or nibble-based run-length,
982 * or a complex descriptor, or a user defined marker function.
985 ROOT_DESC_CONSERVATIVE, /* 0, so matches NULL value */
990 ROOT_DESC_TYPE_MASK = 0x7,
991 ROOT_DESC_TYPE_SHIFT = 3,
994 #define MAKE_ROOT_DESC(type,val) ((type) | ((val) << ROOT_DESC_TYPE_SHIFT))
996 #define MAX_USER_DESCRIPTORS 16
998 static gsize* complex_descriptors = NULL;
999 static int complex_descriptors_size = 0;
1000 static int complex_descriptors_next = 0;
1001 static MonoGCMarkFunc user_descriptors [MAX_USER_DESCRIPTORS];
1002 static int user_descriptors_next = 0;
1005 alloc_complex_descriptor (gsize *bitmap, int numbits)
1007 int nwords = numbits/GC_BITS_PER_WORD + 2;
1012 res = complex_descriptors_next;
1013 /* linear search, so we don't have duplicates with domain load/unload
1014 * this should not be performance critical or we'd have bigger issues
1015 * (the number and size of complex descriptors should be small).
1017 for (i = 0; i < complex_descriptors_next; ) {
1018 if (complex_descriptors [i] == nwords) {
1019 int j, found = TRUE;
1020 for (j = 0; j < nwords - 1; ++j) {
1021 if (complex_descriptors [i + 1 + j] != bitmap [j]) {
1031 i += complex_descriptors [i];
1033 if (complex_descriptors_next + nwords > complex_descriptors_size) {
1034 int new_size = complex_descriptors_size * 2 + nwords;
1035 complex_descriptors = g_realloc (complex_descriptors, new_size * sizeof (gsize));
1036 complex_descriptors_size = new_size;
1038 DEBUG (6, fprintf (gc_debug_file, "Complex descriptor %d, size: %d (total desc memory: %d)\n", res, nwords, complex_descriptors_size));
1039 complex_descriptors_next += nwords;
1040 complex_descriptors [res] = nwords;
1041 for (i = 0; i < nwords - 1; ++i) {
1042 complex_descriptors [res + 1 + i] = bitmap [i];
1043 DEBUG (6, fprintf (gc_debug_file, "\tvalue: %p\n", (void*)complex_descriptors [res + 1 + i]));
1050 * Descriptor builders.
1053 mono_gc_make_descr_for_string (gsize *bitmap, int numbits)
1055 return (void*) DESC_TYPE_STRING;
1059 mono_gc_make_descr_for_object (gsize *bitmap, int numbits, size_t obj_size)
1061 int first_set = -1, num_set = 0, last_set = -1, i;
1063 size_t stored_size = obj_size;
1064 stored_size += ALLOC_ALIGN - 1;
1065 stored_size &= ~(ALLOC_ALIGN - 1);
1066 for (i = 0; i < numbits; ++i) {
1067 if (bitmap [i / GC_BITS_PER_WORD] & ((gsize)1 << (i % GC_BITS_PER_WORD))) {
1074 if (stored_size <= MAX_SMALL_OBJ_SIZE) {
1075 /* check run-length encoding first: one byte offset, one byte number of pointers
1076 * on 64 bit archs, we can have 3 runs, just one on 32.
1077 * It may be better to use nibbles.
1079 if (first_set < 0) {
1080 desc = DESC_TYPE_RUN_LENGTH | stored_size;
1081 DEBUG (6, fprintf (gc_debug_file, "Ptrfree descriptor %p, size: %zd\n", (void*)desc, stored_size));
1082 return (void*) desc;
1083 } else if (first_set < 256 && num_set < 256 && (first_set + num_set == last_set + 1)) {
1084 desc = DESC_TYPE_RUN_LENGTH | stored_size | (first_set << 16) | (num_set << 24);
1085 DEBUG (6, fprintf (gc_debug_file, "Runlen descriptor %p, size: %zd, first set: %d, num set: %d\n", (void*)desc, stored_size, first_set, num_set));
1086 return (void*) desc;
1088 /* we know the 2-word header is ptr-free */
1089 if (last_set < SMALL_BITMAP_SIZE + OBJECT_HEADER_WORDS) {
1090 desc = DESC_TYPE_SMALL_BITMAP | stored_size | ((*bitmap >> OBJECT_HEADER_WORDS) << SMALL_BITMAP_SHIFT);
1091 DEBUG (6, fprintf (gc_debug_file, "Smallbitmap descriptor %p, size: %zd, last set: %d\n", (void*)desc, stored_size, last_set));
1092 return (void*) desc;
1095 /* we know the 2-word header is ptr-free */
1096 if (last_set < LARGE_BITMAP_SIZE + OBJECT_HEADER_WORDS) {
1097 desc = DESC_TYPE_LARGE_BITMAP | ((*bitmap >> OBJECT_HEADER_WORDS) << LOW_TYPE_BITS);
1098 DEBUG (6, fprintf (gc_debug_file, "Largebitmap descriptor %p, size: %zd, last set: %d\n", (void*)desc, stored_size, last_set));
1099 return (void*) desc;
1101 /* it's a complex object ... */
1102 desc = DESC_TYPE_COMPLEX | (alloc_complex_descriptor (bitmap, last_set + 1) << LOW_TYPE_BITS);
1103 return (void*) desc;
1106 /* If the array holds references, numbits == 1 and the first bit is set in elem_bitmap */
1108 mono_gc_make_descr_for_array (int vector, gsize *elem_bitmap, int numbits, size_t elem_size)
1110 int first_set = -1, num_set = 0, last_set = -1, i;
1111 mword desc = vector? DESC_TYPE_VECTOR: DESC_TYPE_ARRAY;
1112 for (i = 0; i < numbits; ++i) {
1113 if (elem_bitmap [i / GC_BITS_PER_WORD] & ((gsize)1 << (i % GC_BITS_PER_WORD))) {
1120 if (elem_size <= MAX_ELEMENT_SIZE) {
1121 desc |= elem_size << VECTOR_ELSIZE_SHIFT;
1123 return (void*)(desc | VECTOR_SUBTYPE_PTRFREE);
1125 /* Note: we also handle structs with just ref fields */
1126 if (num_set * sizeof (gpointer) == elem_size) {
1127 return (void*)(desc | VECTOR_SUBTYPE_REFS | ((gssize)(-1) << 16));
1129 /* FIXME: try run-len first */
1130 /* Note: we can't skip the object header here, because it's not present */
1131 if (last_set <= SMALL_BITMAP_SIZE) {
1132 return (void*)(desc | VECTOR_SUBTYPE_BITMAP | (*elem_bitmap << 16));
1135 /* it's am array of complex structs ... */
1136 desc = DESC_TYPE_COMPLEX_ARR;
1137 desc |= alloc_complex_descriptor (elem_bitmap, last_set + 1) << LOW_TYPE_BITS;
1138 return (void*) desc;
1141 /* Return the bitmap encoded by a descriptor */
1143 mono_gc_get_bitmap_for_descr (void *descr, int *numbits)
1145 mword d = (mword)descr;
1149 case DESC_TYPE_RUN_LENGTH: {
1150 int first_set = (d >> 16) & 0xff;
1151 int num_set = (d >> 24) & 0xff;
1154 bitmap = g_new0 (gsize, (first_set + num_set + 7) / 8);
1156 for (i = first_set; i < first_set + num_set; ++i)
1157 bitmap [i / GC_BITS_PER_WORD] |= ((gsize)1 << (i % GC_BITS_PER_WORD));
1159 *numbits = first_set + num_set;
1163 case DESC_TYPE_SMALL_BITMAP:
1164 bitmap = g_new0 (gsize, 1);
1166 bitmap [0] = (d >> SMALL_BITMAP_SHIFT) << OBJECT_HEADER_WORDS;
1168 *numbits = GC_BITS_PER_WORD;
1172 g_assert_not_reached ();
1176 /* helper macros to scan and traverse objects, macros because we resue them in many functions */
1177 #define STRING_SIZE(size,str) do { \
1178 (size) = sizeof (MonoString) + 2 * (mono_string_length ((MonoString*)(str)) + 1); \
1179 (size) += (ALLOC_ALIGN - 1); \
1180 (size) &= ~(ALLOC_ALIGN - 1); \
1183 #define OBJ_RUN_LEN_SIZE(size,desc,obj) do { \
1184 (size) = (desc) & 0xfff8; \
1187 #define OBJ_BITMAP_SIZE(size,desc,obj) do { \
1188 (size) = (desc) & 0xfff8; \
1191 //#define PREFETCH(addr) __asm__ __volatile__ (" prefetchnta %0": : "m"(*(char *)(addr)))
1192 #define PREFETCH(addr)
1194 /* code using these macros must define a HANDLE_PTR(ptr) macro that does the work */
1195 #define OBJ_RUN_LEN_FOREACH_PTR(desc,obj) do { \
1196 if ((desc) & 0xffff0000) { \
1197 /* there are pointers */ \
1198 void **_objptr_end; \
1199 void **_objptr = (void**)(obj); \
1200 _objptr += ((desc) >> 16) & 0xff; \
1201 _objptr_end = _objptr + (((desc) >> 24) & 0xff); \
1202 while (_objptr < _objptr_end) { \
1203 HANDLE_PTR (_objptr, (obj)); \
1209 /* a bitmap desc means that there are pointer references or we'd have
1210 * choosen run-length, instead: add an assert to check.
1212 #define OBJ_BITMAP_FOREACH_PTR(desc,obj) do { \
1213 /* there are pointers */ \
1214 void **_objptr = (void**)(obj); \
1215 gsize _bmap = (desc) >> 16; \
1216 _objptr += OBJECT_HEADER_WORDS; \
1218 if ((_bmap & 1)) { \
1219 HANDLE_PTR (_objptr, (obj)); \
1226 #define OBJ_LARGE_BITMAP_FOREACH_PTR(vt,obj) do { \
1227 /* there are pointers */ \
1228 void **_objptr = (void**)(obj); \
1229 gsize _bmap = (vt)->desc >> LOW_TYPE_BITS; \
1230 _objptr += OBJECT_HEADER_WORDS; \
1232 if ((_bmap & 1)) { \
1233 HANDLE_PTR (_objptr, (obj)); \
1240 #define OBJ_COMPLEX_FOREACH_PTR(vt,obj) do { \
1241 /* there are pointers */ \
1242 void **_objptr = (void**)(obj); \
1243 gsize *bitmap_data = complex_descriptors + ((vt)->desc >> LOW_TYPE_BITS); \
1244 int bwords = (*bitmap_data) - 1; \
1245 void **start_run = _objptr; \
1248 MonoObject *myobj = (MonoObject*)obj; \
1249 g_print ("found %d at %p (0x%zx): %s.%s\n", bwords, (obj), (vt)->desc, myobj->vtable->klass->name_space, myobj->vtable->klass->name); \
1251 while (bwords-- > 0) { \
1252 gsize _bmap = *bitmap_data++; \
1253 _objptr = start_run; \
1254 /*g_print ("bitmap: 0x%x/%d at %p\n", _bmap, bwords, _objptr);*/ \
1256 if ((_bmap & 1)) { \
1257 HANDLE_PTR (_objptr, (obj)); \
1262 start_run += GC_BITS_PER_WORD; \
1266 /* this one is untested */
1267 #define OBJ_COMPLEX_ARR_FOREACH_PTR(vt,obj) do { \
1268 /* there are pointers */ \
1269 gsize *mbitmap_data = complex_descriptors + ((vt)->desc >> LOW_TYPE_BITS); \
1270 int mbwords = (*mbitmap_data++) - 1; \
1271 int el_size = mono_array_element_size (((MonoObject*)(obj))->vtable->klass); \
1272 char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
1273 char *e_end = e_start + el_size * mono_array_length ((MonoArray*)(obj)); \
1275 MonoObject *myobj = (MonoObject*)start; \
1276 g_print ("found %d at %p (0x%zx): %s.%s\n", mbwords, (obj), (vt)->desc, myobj->vtable->klass->name_space, myobj->vtable->klass->name); \
1278 while (e_start < e_end) { \
1279 void **_objptr = (void**)e_start; \
1280 gsize *bitmap_data = mbitmap_data; \
1281 unsigned int bwords = mbwords; \
1282 while (bwords-- > 0) { \
1283 gsize _bmap = *bitmap_data++; \
1284 void **start_run = _objptr; \
1285 /*g_print ("bitmap: 0x%x\n", _bmap);*/ \
1287 if ((_bmap & 1)) { \
1288 HANDLE_PTR (_objptr, (obj)); \
1293 _objptr = start_run + GC_BITS_PER_WORD; \
1295 e_start += el_size; \
1299 #define OBJ_VECTOR_FOREACH_PTR(vt,obj) do { \
1300 /* note: 0xffffc000 excludes DESC_TYPE_V_PTRFREE */ \
1301 if ((vt)->desc & 0xffffc000) { \
1302 int el_size = ((vt)->desc >> 3) & MAX_ELEMENT_SIZE; \
1303 /* there are pointers */ \
1304 int etype = (vt)->desc & 0xc000; \
1305 if (etype == (DESC_TYPE_V_REFS << 14)) { \
1306 void **p = (void**)((char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector)); \
1307 void **end_refs = (void**)((char*)p + el_size * mono_array_length ((MonoArray*)(obj))); \
1308 /* Note: this code can handle also arrays of struct with only references in them */ \
1309 while (p < end_refs) { \
1310 HANDLE_PTR (p, (obj)); \
1313 } else if (etype == DESC_TYPE_V_RUN_LEN << 14) { \
1314 int offset = ((vt)->desc >> 16) & 0xff; \
1315 int num_refs = ((vt)->desc >> 24) & 0xff; \
1316 char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
1317 char *e_end = e_start + el_size * mono_array_length ((MonoArray*)(obj)); \
1318 while (e_start < e_end) { \
1319 void **p = (void**)e_start; \
1322 for (i = 0; i < num_refs; ++i) { \
1323 HANDLE_PTR (p + i, (obj)); \
1325 e_start += el_size; \
1327 } else if (etype == DESC_TYPE_V_BITMAP << 14) { \
1328 char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
1329 char *e_end = e_start + el_size * mono_array_length ((MonoArray*)(obj)); \
1330 while (e_start < e_end) { \
1331 void **p = (void**)e_start; \
1332 gsize _bmap = (vt)->desc >> 16; \
1333 /* Note: there is no object header here to skip */ \
1335 if ((_bmap & 1)) { \
1336 HANDLE_PTR (p, (obj)); \
1341 e_start += el_size; \
1347 #define COUNT_OBJECT_TYPES do { \
1348 switch (desc & 0x7) { \
1349 case DESC_TYPE_STRING: type_str++; break; \
1350 case DESC_TYPE_RUN_LENGTH: type_rlen++; break; \
1351 case DESC_TYPE_ARRAY: case DESC_TYPE_VECTOR: type_vector++; break; \
1352 case DESC_TYPE_SMALL_BITMAP: type_bitmap++; break; \
1353 case DESC_TYPE_LARGE_BITMAP: type_lbit++; break; \
1354 case DESC_TYPE_COMPLEX: type_complex++; break; \
1355 case DESC_TYPE_COMPLEX_ARR: type_complex++; break; \
1356 default: g_assert_not_reached (); \
1362 * ######################################################################
1363 * ######## Detecting and removing garbage.
1364 * ######################################################################
1365 * This section of code deals with detecting the objects no longer in use
1366 * and reclaiming the memory.
1370 static mword new_obj_references = 0;
1371 static mword obj_references_checked = 0;
1374 #define HANDLE_PTR(ptr,obj) do { \
1375 if (*(ptr) && (char*)*(ptr) >= nursery_start && (char*)*(ptr) < nursery_next) { \
1376 new_obj_references++; \
1377 /*printf ("bogus ptr %p found at %p in object %p (%s.%s)\n", *(ptr), (ptr), o, o->vtable->klass->name_space, o->vtable->klass->name);*/ \
1379 obj_references_checked++; \
1383 static void __attribute__((noinline))
1384 scan_area (char *start, char *end)
1387 int type_str = 0, type_rlen = 0, type_bitmap = 0, type_vector = 0, type_lbit = 0, type_complex = 0;
1388 new_obj_references = 0;
1389 obj_references_checked = 0;
1390 while (start < end) {
1391 if (!*(void**)start) {
1392 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
1395 vt = (GCVTable*)LOAD_VTABLE (start);
1396 DEBUG (8, fprintf (gc_debug_file, "Scanning object %p, vtable: %p (%s)\n", start, vt, vt->klass->name));
1398 MonoObject *obj = (MonoObject*)start;
1399 g_print ("found at %p (0x%zx): %s.%s\n", start, vt->desc, obj->vtable->klass->name_space, obj->vtable->klass->name);
1402 #define SCAN_OBJECT_ACTION COUNT_OBJECT_TYPES
1403 #include "sgen-scan-object.h"
1405 /*printf ("references to new nursery %p-%p (size: %dk): %d, checked: %d\n", old_start, end, (end-old_start)/1024, new_obj_references, obj_references_checked);
1406 printf ("\tstrings: %d, runl: %d, vector: %d, bitmaps: %d, lbitmaps: %d, complex: %d\n",
1407 type_str, type_rlen, type_vector, type_bitmap, type_lbit, type_complex);*/
1412 is_xdomain_ref_allowed (gpointer *ptr, char *obj, MonoDomain *domain)
1414 MonoObject *o = (MonoObject*)(obj);
1415 MonoObject *ref = (MonoObject*)*(ptr);
1416 int offset = (char*)(ptr) - (char*)o;
1418 if (o->vtable->klass == mono_defaults.thread_class && offset == G_STRUCT_OFFSET (MonoThread, internal_thread))
1420 if (o->vtable->klass == mono_defaults.internal_thread_class && offset == G_STRUCT_OFFSET (MonoInternalThread, current_appcontext))
1422 if (mono_class_has_parent (o->vtable->klass, mono_defaults.real_proxy_class) &&
1423 offset == G_STRUCT_OFFSET (MonoRealProxy, unwrapped_server))
1425 /* Thread.cached_culture_info */
1426 if (!strcmp (ref->vtable->klass->name_space, "System.Globalization") &&
1427 !strcmp (ref->vtable->klass->name, "CultureInfo") &&
1428 !strcmp(o->vtable->klass->name_space, "System") &&
1429 !strcmp(o->vtable->klass->name, "Object[]"))
1432 * at System.IO.MemoryStream.InternalConstructor (byte[],int,int,bool,bool) [0x0004d] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.IO/MemoryStream.cs:121
1433 * at System.IO.MemoryStream..ctor (byte[]) [0x00017] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.IO/MemoryStream.cs:81
1434 * at (wrapper remoting-invoke-with-check) System.IO.MemoryStream..ctor (byte[]) <IL 0x00020, 0xffffffff>
1435 * at System.Runtime.Remoting.Messaging.CADMethodCallMessage.GetArguments () [0x0000d] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Messaging/CADMessages.cs:327
1436 * at System.Runtime.Remoting.Messaging.MethodCall..ctor (System.Runtime.Remoting.Messaging.CADMethodCallMessage) [0x00017] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Messaging/MethodCall.cs:87
1437 * at System.AppDomain.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage,byte[]&,System.Runtime.Remoting.Messaging.CADMethodReturnMessage&) [0x00018] in /home/schani/Work/novell/trunk/mcs/class/corlib/System/AppDomain.cs:1213
1438 * at (wrapper remoting-invoke-with-check) System.AppDomain.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage,byte[]&,System.Runtime.Remoting.Messaging.CADMethodReturnMessage&) <IL 0x0003d, 0xffffffff>
1439 * at System.Runtime.Remoting.Channels.CrossAppDomainSink.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage) [0x00008] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Channels/CrossAppDomainChannel.cs:198
1440 * at (wrapper runtime-invoke) object.runtime_invoke_CrossAppDomainSink/ProcessMessageRes_object_object (object,intptr,intptr,intptr) <IL 0x0004c, 0xffffffff>
1442 if (!strcmp (ref->vtable->klass->name_space, "System") &&
1443 !strcmp (ref->vtable->klass->name, "Byte[]") &&
1444 !strcmp (o->vtable->klass->name_space, "System.IO") &&
1445 !strcmp (o->vtable->klass->name, "MemoryStream"))
1447 /* append_job() in threadpool.c */
1448 if (!strcmp (ref->vtable->klass->name_space, "System.Runtime.Remoting.Messaging") &&
1449 !strcmp (ref->vtable->klass->name, "AsyncResult") &&
1450 !strcmp (o->vtable->klass->name_space, "System") &&
1451 !strcmp (o->vtable->klass->name, "Object[]") &&
1452 mono_thread_pool_is_queue_array ((MonoArray*) o))
1458 check_reference_for_xdomain (gpointer *ptr, char *obj, MonoDomain *domain)
1460 MonoObject *o = (MonoObject*)(obj);
1461 MonoObject *ref = (MonoObject*)*(ptr);
1462 int offset = (char*)(ptr) - (char*)o;
1464 MonoClassField *field;
1467 if (!ref || ref->vtable->domain == domain)
1469 if (is_xdomain_ref_allowed (ptr, obj, domain))
1473 for (class = o->vtable->klass; class; class = class->parent) {
1476 for (i = 0; i < class->field.count; ++i) {
1477 if (class->fields[i].offset == offset) {
1478 field = &class->fields[i];
1486 if (ref->vtable->klass == mono_defaults.string_class)
1487 str = mono_string_to_utf8 ((MonoString*)ref);
1490 g_print ("xdomain reference in %p (%s.%s) at offset %d (%s) to %p (%s.%s) (%s) - pointed to by:\n",
1491 o, o->vtable->klass->name_space, o->vtable->klass->name,
1492 offset, field ? field->name : "",
1493 ref, ref->vtable->klass->name_space, ref->vtable->klass->name, str ? str : "");
1494 mono_gc_scan_for_specific_ref (o);
1500 #define HANDLE_PTR(ptr,obj) check_reference_for_xdomain ((ptr), (obj), domain)
1503 scan_object_for_xdomain_refs (char *start)
1505 MonoDomain *domain = ((MonoObject*)start)->vtable->domain;
1507 #include "sgen-scan-object.h"
1513 scan_area_for_xdomain_refs (char *start, char *end)
1515 while (start < end) {
1516 if (!*(void**)start) {
1517 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
1521 start = scan_object_for_xdomain_refs (start);
1526 #define HANDLE_PTR(ptr,obj) do { \
1527 if ((MonoObject*)*(ptr) == key) { \
1528 g_print ("found ref to %p in object %p (%s) at offset %zd\n", \
1529 key, (obj), safe_name ((obj)), ((char*)(ptr) - (char*)(obj))); \
1534 scan_object_for_specific_ref (char *start, MonoObject *key)
1536 #include "sgen-scan-object.h"
1542 scan_area_for_specific_ref (char *start, char *end, MonoObject *key)
1544 while (start < end) {
1545 if (!*(void**)start) {
1546 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
1550 start = scan_object_for_specific_ref (start, key);
1555 scan_pinned_object_for_specific_ref_callback (PinnedChunk *chunk, char *obj, size_t size, MonoObject *key)
1557 scan_object_for_specific_ref (obj, key);
1561 check_root_obj_specific_ref (RootRecord *root, MonoObject *key, MonoObject *obj)
1565 g_print ("found ref to %p in root record %p\n", key, root);
1568 static MonoObject *check_key = NULL;
1569 static RootRecord *check_root = NULL;
1572 check_root_obj_specific_ref_from_marker (void *obj)
1574 check_root_obj_specific_ref (check_root, check_key, obj);
1579 scan_roots_for_specific_ref (MonoObject *key, int root_type)
1584 for (i = 0; i < roots_hash_size [root_type]; ++i) {
1585 for (root = roots_hash [root_type][i]; root; root = root->next) {
1586 void **start_root = (void**)root->start_root;
1587 mword desc = root->root_desc;
1591 switch (desc & ROOT_DESC_TYPE_MASK) {
1592 case ROOT_DESC_BITMAP:
1593 desc >>= ROOT_DESC_TYPE_SHIFT;
1596 check_root_obj_specific_ref (root, key, *start_root);
1601 case ROOT_DESC_COMPLEX: {
1602 gsize *bitmap_data = complex_descriptors + (desc >> ROOT_DESC_TYPE_SHIFT);
1603 int bwords = (*bitmap_data) - 1;
1604 void **start_run = start_root;
1606 while (bwords-- > 0) {
1607 gsize bmap = *bitmap_data++;
1608 void **objptr = start_run;
1611 check_root_obj_specific_ref (root, key, *objptr);
1615 start_run += GC_BITS_PER_WORD;
1619 case ROOT_DESC_USER: {
1620 MonoGCMarkFunc marker = user_descriptors [desc >> ROOT_DESC_TYPE_SHIFT];
1621 marker (start_root, check_root_obj_specific_ref_from_marker);
1624 case ROOT_DESC_RUN_LEN:
1625 g_assert_not_reached ();
1627 g_assert_not_reached ();
1636 mono_gc_scan_for_specific_ref (MonoObject *key)
1638 GCMemSection *section;
1643 for (section = section_list; section; section = section->block.next)
1644 scan_area_for_specific_ref (section->data, section->end_data, key);
1646 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1647 scan_object_for_specific_ref (bigobj->data, key);
1649 scan_pinned_objects ((ScanPinnedObjectCallbackFunc)scan_pinned_object_for_specific_ref_callback, key);
1651 scan_roots_for_specific_ref (key, ROOT_TYPE_NORMAL);
1652 scan_roots_for_specific_ref (key, ROOT_TYPE_WBARRIER);
1654 for (i = 0; i < roots_hash_size [ROOT_TYPE_PINNED]; ++i) {
1655 for (root = roots_hash [ROOT_TYPE_PINNED][i]; root; root = root->next) {
1656 void **ptr = (void**)root->start_root;
1658 while (ptr < (void**)root->end_root) {
1659 check_root_obj_specific_ref (root, *ptr, key);
1667 need_remove_object_for_domain (char *start, MonoDomain *domain)
1669 if (mono_object_domain (start) == domain) {
1670 DEBUG (1, fprintf (gc_debug_file, "Need to cleanup object %p\n", start));
1677 process_object_for_domain_clearing (char *start, MonoDomain *domain)
1679 GCVTable *vt = (GCVTable*)LOAD_VTABLE (start);
1680 if (vt->klass == mono_defaults.internal_thread_class)
1681 g_assert (mono_object_domain (start) == mono_get_root_domain ());
1682 /* The object could be a proxy for an object in the domain
1684 if (mono_class_has_parent (vt->klass, mono_defaults.real_proxy_class)) {
1685 MonoObject *server = ((MonoRealProxy*)start)->unwrapped_server;
1687 /* The server could already have been zeroed out, so
1688 we need to check for that, too. */
1689 if (server && (!LOAD_VTABLE (server) || mono_object_domain (server) == domain)) {
1690 DEBUG (1, fprintf (gc_debug_file, "Cleaning up remote pointer in %p to object %p\n",
1692 ((MonoRealProxy*)start)->unwrapped_server = NULL;
1697 static void __attribute__((noinline))
1698 scan_area_for_domain (MonoDomain *domain, char *start, char *end)
1703 while (start < end) {
1704 if (!*(void**)start) {
1705 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
1708 vt = (GCVTable*)LOAD_VTABLE (start);
1709 process_object_for_domain_clearing (start, domain);
1710 remove = need_remove_object_for_domain (start, domain);
1711 if (remove && ((MonoObject*)start)->synchronisation) {
1712 void **dislink = mono_monitor_get_object_monitor_weak_link ((MonoObject*)start);
1714 mono_gc_register_disappearing_link (NULL, dislink, FALSE);
1717 #define SCAN_OBJECT_NOSCAN
1718 #define SCAN_OBJECT_ACTION do { \
1719 if (remove) memset (start, 0, skip_size); \
1721 #include "sgen-scan-object.h"
1725 static MonoDomain *check_domain = NULL;
1728 check_obj_not_in_domain (void *o)
1730 g_assert (((MonoObject*)o)->vtable->domain != check_domain);
1735 scan_for_registered_roots_in_domain (MonoDomain *domain, int root_type)
1739 check_domain = domain;
1740 for (i = 0; i < roots_hash_size [root_type]; ++i) {
1741 for (root = roots_hash [root_type][i]; root; root = root->next) {
1742 void **start_root = (void**)root->start_root;
1743 mword desc = root->root_desc;
1745 /* The MonoDomain struct is allowed to hold
1746 references to objects in its own domain. */
1747 if (start_root == (void**)domain)
1750 switch (desc & ROOT_DESC_TYPE_MASK) {
1751 case ROOT_DESC_BITMAP:
1752 desc >>= ROOT_DESC_TYPE_SHIFT;
1754 if ((desc & 1) && *start_root)
1755 check_obj_not_in_domain (*start_root);
1760 case ROOT_DESC_COMPLEX: {
1761 gsize *bitmap_data = complex_descriptors + (desc >> ROOT_DESC_TYPE_SHIFT);
1762 int bwords = (*bitmap_data) - 1;
1763 void **start_run = start_root;
1765 while (bwords-- > 0) {
1766 gsize bmap = *bitmap_data++;
1767 void **objptr = start_run;
1769 if ((bmap & 1) && *objptr)
1770 check_obj_not_in_domain (*objptr);
1774 start_run += GC_BITS_PER_WORD;
1778 case ROOT_DESC_USER: {
1779 MonoGCMarkFunc marker = user_descriptors [desc >> ROOT_DESC_TYPE_SHIFT];
1780 marker (start_root, check_obj_not_in_domain);
1783 case ROOT_DESC_RUN_LEN:
1784 g_assert_not_reached ();
1786 g_assert_not_reached ();
1790 check_domain = NULL;
1794 clear_domain_process_pinned_object_callback (PinnedChunk *chunk, char *obj, size_t size, MonoDomain *domain)
1796 process_object_for_domain_clearing (obj, domain);
1800 clear_domain_free_pinned_object_callback (PinnedChunk *chunk, char *obj, size_t size, MonoDomain *domain)
1802 if (need_remove_object_for_domain (obj, domain))
1803 free_pinned_object (chunk, obj, size);
1807 scan_pinned_object_for_xdomain_refs_callback (PinnedChunk *chunk, char *obj, size_t size, gpointer dummy)
1809 scan_object_for_xdomain_refs (obj);
1813 check_for_xdomain_refs (void)
1815 GCMemSection *section;
1818 for (section = section_list; section; section = section->block.next)
1819 scan_area_for_xdomain_refs (section->data, section->end_data);
1821 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1822 scan_object_for_xdomain_refs (bigobj->data);
1824 scan_pinned_objects (scan_pinned_object_for_xdomain_refs_callback, NULL);
1828 * When appdomains are unloaded we can easily remove objects that have finalizers,
1829 * but all the others could still be present in random places on the heap.
1830 * We need a sweep to get rid of them even though it's going to be costly
1832 * The reason we need to remove them is because we access the vtable and class
1833 * structures to know the object size and the reference bitmap: once the domain is
1834 * unloaded the point to random memory.
1837 mono_gc_clear_domain (MonoDomain * domain)
1839 GCMemSection *section;
1840 LOSObject *bigobj, *prev;
1845 /* Clear all remaining nursery fragments */
1846 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION) {
1847 g_assert (nursery_next <= nursery_frag_real_end);
1848 memset (nursery_next, 0, nursery_frag_real_end - nursery_next);
1849 for (frag = nursery_fragments; frag; frag = frag->next) {
1850 memset (frag->fragment_start, 0, frag->fragment_end - frag->fragment_start);
1854 if (xdomain_checks && domain != mono_get_root_domain ()) {
1855 scan_for_registered_roots_in_domain (domain, ROOT_TYPE_NORMAL);
1856 scan_for_registered_roots_in_domain (domain, ROOT_TYPE_WBARRIER);
1857 check_for_xdomain_refs ();
1860 for (section = section_list; section; section = section->block.next) {
1861 scan_area_for_domain (domain, section->data, section->end_data);
1864 /* We need two passes over pinned and large objects because
1865 freeing such an object gives its memory back to the OS (in
1866 the case of large objects) or obliterates its vtable
1867 (pinned objects), but we might need to dereference a
1868 pointer from an object to another object if the first
1869 object is a proxy. */
1870 scan_pinned_objects ((ScanPinnedObjectCallbackFunc)clear_domain_process_pinned_object_callback, domain);
1871 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1872 process_object_for_domain_clearing (bigobj->data, domain);
1875 for (bigobj = los_object_list; bigobj;) {
1876 if (need_remove_object_for_domain (bigobj->data, domain)) {
1877 LOSObject *to_free = bigobj;
1879 prev->next = bigobj->next;
1881 los_object_list = bigobj->next;
1882 bigobj = bigobj->next;
1883 DEBUG (1, fprintf (gc_debug_file, "Freeing large object %p\n",
1885 free_large_object (to_free);
1889 bigobj = bigobj->next;
1891 scan_pinned_objects ((ScanPinnedObjectCallbackFunc)clear_domain_free_pinned_object_callback, domain);
1893 for (i = GENERATION_NURSERY; i < GENERATION_MAX; ++i)
1894 null_links_for_domain (domain, i);
1900 * add_to_global_remset:
1902 * The global remset contains locations which point into newspace after
1903 * a minor collection. This can happen if the objects they point to are pinned.
1906 add_to_global_remset (gpointer ptr, gboolean root)
1910 DEBUG (8, fprintf (gc_debug_file, "Adding global remset for %p\n", ptr));
1913 g_assert (!ptr_in_nursery (ptr) && ptr_in_nursery (*(gpointer*)ptr));
1915 HEAVY_STAT (++stat_global_remsets_added);
1918 * FIXME: If an object remains pinned, we need to add it at every minor collection.
1919 * To avoid uncontrolled growth of the global remset, only add each pointer once.
1921 if (global_remset->store_next + 3 < global_remset->end_set) {
1923 *(global_remset->store_next++) = (mword)ptr | REMSET_OTHER;
1924 *(global_remset->store_next++) = (mword)REMSET_ROOT_LOCATION;
1926 *(global_remset->store_next++) = (mword)ptr;
1930 rs = alloc_remset (global_remset->end_set - global_remset->data, NULL);
1931 rs->next = global_remset;
1934 *(global_remset->store_next++) = (mword)ptr | REMSET_OTHER;
1935 *(global_remset->store_next++) = (mword)REMSET_ROOT_LOCATION;
1937 *(global_remset->store_next++) = (mword)ptr;
1941 int global_rs_size = 0;
1943 for (rs = global_remset; rs; rs = rs->next) {
1944 global_rs_size += rs->store_next - rs->data;
1946 DEBUG (4, fprintf (gc_debug_file, "Global remset now has size %d\n", global_rs_size));
1950 #include "sgen-gray.c"
1953 * This is how the copying happens from the nursery to the old generation.
1954 * We assume that at this time all the pinned objects have been identified and
1956 * We run scan_object() for each pinned object so that each referenced
1957 * objects if possible are copied. The new gray objects created can have
1958 * scan_object() run on them right away, too.
1959 * Then we run copy_object() for the precisely tracked roots. At this point
1960 * all the roots are either gray or black. We run scan_object() on the gray
1961 * objects until no more gray objects are created.
1962 * At the end of the process we walk again the pinned list and we unmark
1963 * the pinned flag. As we go we also create the list of free space for use
1964 * in the next allocation runs.
1966 * We need to remember objects from the old generation that point to the new one
1967 * (or just addresses?).
1969 * copy_object could be made into a macro once debugged (use inline for now).
1972 static char* __attribute__((noinline))
1973 copy_object (char *obj, char *from_space_start, char *from_space_end)
1975 static void *copy_labels [] = { &&LAB_0, &&LAB_1, &&LAB_2, &&LAB_3, &&LAB_4, &&LAB_5, &&LAB_6, &&LAB_7, &&LAB_8 };
1981 HEAVY_STAT (++num_copy_object_called);
1983 if (!(obj >= from_space_start && obj < from_space_end)) {
1984 DEBUG (9, fprintf (gc_debug_file, "Not copying %p because it's not in from space (%p-%p)\n",
1985 obj, from_space_start, from_space_end));
1986 HEAVY_STAT (++stat_copy_object_failed_from_space);
1990 DEBUG (9, fprintf (gc_debug_file, "Precise copy of %p", obj));
1993 * obj must belong to one of:
1998 * 4. a non-to-space section of the major heap
1999 * 5. a to-space section of the major heap
2001 * In addition, objects in 1, 2 and 4 might also be pinned.
2002 * Objects in 1 and 4 might be forwarded.
2004 * Before we can copy the object we must make sure that we are
2005 * allowed to, i.e. that the object not pinned, not already
2006 * forwarded and doesn't belong to the LOS, a pinned chunk, or
2007 * a to-space section.
2009 * We are usually called for to-space objects (5) when we have
2010 * two remset entries for the same reference. The first entry
2011 * copies the object and updates the reference and the second
2012 * calls us with the updated reference that points into
2013 * to-space. There might also be other circumstances where we
2014 * get to-space objects.
2017 if ((forwarded = object_is_forwarded (obj))) {
2018 g_assert (((MonoVTable*)LOAD_VTABLE(obj))->gc_descr);
2019 DEBUG (9, fprintf (gc_debug_file, " (already forwarded to %p)\n", forwarded));
2020 HEAVY_STAT (++stat_copy_object_failed_forwarded);
2023 if (object_is_pinned (obj)) {
2024 g_assert (((MonoVTable*)LOAD_VTABLE(obj))->gc_descr);
2025 DEBUG (9, fprintf (gc_debug_file, " (pinned, no change)\n"));
2026 HEAVY_STAT (++stat_copy_object_failed_pinned);
2030 objsize = safe_object_get_size ((MonoObject*)obj);
2031 objsize += ALLOC_ALIGN - 1;
2032 objsize &= ~(ALLOC_ALIGN - 1);
2034 if (ptr_in_nursery (obj))
2038 * At this point we know obj is not pinned, not forwarded and
2039 * belongs to 2, 3, 4, or 5.
2041 * LOS object (2) are simple, at least until we always follow
2042 * the rule: if objsize > MAX_SMALL_OBJ_SIZE, pin the object
2043 * and return it. At the end of major collections, we walk
2044 * the los list and if the object is pinned, it is marked,
2045 * otherwise it can be freed.
2047 * Pinned chunks (3) and major heap sections (4, 5) both
2048 * reside in blocks, which are always aligned, so once we've
2049 * eliminated LOS objects, we can just access the block and
2050 * see whether it's a pinned chunk or a major heap section.
2052 if (G_UNLIKELY (objsize > MAX_SMALL_OBJ_SIZE || obj_is_from_pinned_alloc (obj))) {
2053 DEBUG (9, fprintf (gc_debug_file, " (marked LOS/Pinned %p (%s), size: %zd)\n", obj, safe_name (obj), objsize));
2055 HEAVY_STAT (++stat_copy_object_failed_large_pinned);
2060 * Now we know the object is in a major heap section. All we
2061 * need to do is check whether it's already in to-space (5) or
2064 if (MAJOR_SECTION_FOR_OBJECT (obj)->is_to_space) {
2065 g_assert (objsize <= MAX_SMALL_OBJ_SIZE);
2066 DEBUG (9, fprintf (gc_debug_file, " (already copied)\n"));
2067 HEAVY_STAT (++stat_copy_object_failed_to_space);
2072 DEBUG (9, fprintf (gc_debug_file, " (to %p, %s size: %zd)\n", to_space_bumper, ((MonoObject*)obj)->vtable->klass->name, objsize));
2074 HEAVY_STAT (++num_objects_copied);
2076 /* Make sure we have enough space available */
2077 if (to_space_bumper + objsize > to_space_top) {
2079 g_assert (to_space_bumper + objsize <= to_space_top);
2082 if (objsize <= sizeof (gpointer) * 8) {
2083 mword *dest = (mword*)to_space_bumper;
2084 goto *copy_labels [objsize / sizeof (gpointer)];
2086 (dest) [7] = ((mword*)obj) [7];
2088 (dest) [6] = ((mword*)obj) [6];
2090 (dest) [5] = ((mword*)obj) [5];
2092 (dest) [4] = ((mword*)obj) [4];
2094 (dest) [3] = ((mword*)obj) [3];
2096 (dest) [2] = ((mword*)obj) [2];
2098 (dest) [1] = ((mword*)obj) [1];
2100 (dest) [0] = ((mword*)obj) [0];
2108 char* edi = to_space_bumper;
2109 __asm__ __volatile__(
2111 : "=&c" (ecx), "=&D" (edi), "=&S" (esi)
2112 : "0" (objsize/4), "1" (edi),"2" (esi)
2117 memcpy (to_space_bumper, obj, objsize);
2120 /* adjust array->bounds */
2121 vt = ((MonoObject*)obj)->vtable;
2122 g_assert (vt->gc_descr);
2123 if (G_UNLIKELY (vt->rank && ((MonoArray*)obj)->bounds)) {
2124 MonoArray *array = (MonoArray*)to_space_bumper;
2125 array->bounds = (MonoArrayBounds*)((char*)to_space_bumper + ((char*)((MonoArray*)obj)->bounds - (char*)obj));
2126 DEBUG (9, fprintf (gc_debug_file, "Array instance %p: size: %zd, rank: %d, length: %d\n", array, objsize, vt->rank, mono_array_length (array)));
2128 /* set the forwarding pointer */
2129 forward_object (obj, to_space_bumper);
2130 obj = to_space_bumper;
2131 to_space_section->scan_starts [((char*)obj - (char*)to_space_section->data)/SCAN_START_SIZE] = obj;
2132 to_space_bumper += objsize;
2133 DEBUG (9, fprintf (gc_debug_file, "Enqueuing gray object %p (%s)\n", obj, safe_name (obj)));
2134 gray_object_enqueue (obj);
2135 DEBUG (8, g_assert (to_space_bumper <= to_space_top));
2140 #define HANDLE_PTR(ptr,obj) do { \
2141 void *__old = *(ptr); \
2144 *(ptr) = __copy = copy_object (__old, from_start, from_end); \
2145 DEBUG (9, if (__old != __copy) fprintf (gc_debug_file, "Overwrote field at %p with %p (was: %p)\n", (ptr), *(ptr), __old)); \
2146 if (G_UNLIKELY (ptr_in_nursery (__copy) && !ptr_in_nursery ((ptr)))) \
2147 add_to_global_remset ((ptr), FALSE); \
2152 * Scan the object pointed to by @start for references to
2153 * other objects between @from_start and @from_end and copy
2154 * them to the gray_objects area.
2155 * Returns a pointer to the end of the object.
2158 scan_object (char *start, char* from_start, char* from_end)
2160 #include "sgen-scan-object.h"
2168 * Scan objects in the gray stack until the stack is empty. This should be called
2169 * frequently after each object is copied, to achieve better locality and cache
2173 drain_gray_stack (char *start_addr, char *end_addr)
2177 while ((obj = gray_object_dequeue ())) {
2178 DEBUG (9, fprintf (gc_debug_file, "Precise gray object scan %p (%s)\n", obj, safe_name (obj)));
2179 scan_object (obj, start_addr, end_addr);
2186 * Scan the valuetype pointed to by START, described by DESC for references to
2187 * other objects between @from_start and @from_end and copy them to the gray_objects area.
2188 * Returns a pointer to the end of the object.
2191 scan_vtype (char *start, mword desc, char* from_start, char* from_end)
2195 /* The descriptors include info about the MonoObject header as well */
2196 start -= sizeof (MonoObject);
2198 switch (desc & 0x7) {
2199 case DESC_TYPE_RUN_LENGTH:
2200 OBJ_RUN_LEN_FOREACH_PTR (desc,start);
2201 OBJ_RUN_LEN_SIZE (skip_size, desc, start);
2202 g_assert (skip_size);
2203 return start + skip_size;
2204 case DESC_TYPE_SMALL_BITMAP:
2205 OBJ_BITMAP_FOREACH_PTR (desc,start);
2206 OBJ_BITMAP_SIZE (skip_size, desc, start);
2207 return start + skip_size;
2208 case DESC_TYPE_LARGE_BITMAP:
2209 case DESC_TYPE_COMPLEX:
2211 g_assert_not_reached ();
2214 // The other descriptors can't happen with vtypes
2215 g_assert_not_reached ();
2221 #include "sgen-pinning-stats.c"
2224 * Addresses from start to end are already sorted. This function finds
2225 * the object header for each address and pins the object. The
2226 * addresses must be inside the passed section. The (start of the)
2227 * address array is overwritten with the addresses of the actually
2228 * pinned objects. Return the number of pinned objects.
2231 pin_objects_from_addresses (GCMemSection *section, void **start, void **end, void *start_nursery, void *end_nursery)
2236 void *last_obj = NULL;
2237 size_t last_obj_size = 0;
2240 void **definitely_pinned = start;
2241 while (start < end) {
2243 /* the range check should be reduntant */
2244 if (addr != last && addr >= start_nursery && addr < end_nursery) {
2245 DEBUG (5, fprintf (gc_debug_file, "Considering pinning addr %p\n", addr));
2246 /* multiple pointers to the same object */
2247 if (addr >= last_obj && (char*)addr < (char*)last_obj + last_obj_size) {
2251 idx = ((char*)addr - (char*)section->data) / SCAN_START_SIZE;
2252 search_start = (void*)section->scan_starts [idx];
2253 if (!search_start || search_start > addr) {
2256 search_start = section->scan_starts [idx];
2257 if (search_start && search_start <= addr)
2260 if (!search_start || search_start > addr)
2261 search_start = start_nursery;
2263 if (search_start < last_obj)
2264 search_start = (char*)last_obj + last_obj_size;
2265 /* now addr should be in an object a short distance from search_start
2266 * Note that search_start must point to zeroed mem or point to an object.
2269 if (!*(void**)search_start) {
2270 mword p = (mword)search_start;
2271 p += sizeof (gpointer);
2272 p += ALLOC_ALIGN - 1;
2273 p &= ~(ALLOC_ALIGN - 1);
2274 search_start = (void*)p;
2277 last_obj = search_start;
2278 last_obj_size = safe_object_get_size ((MonoObject*)search_start);
2279 last_obj_size += ALLOC_ALIGN - 1;
2280 last_obj_size &= ~(ALLOC_ALIGN - 1);
2281 DEBUG (8, fprintf (gc_debug_file, "Pinned try match %p (%s), size %zd\n", last_obj, safe_name (last_obj), last_obj_size));
2282 if (addr >= search_start && (char*)addr < (char*)last_obj + last_obj_size) {
2283 DEBUG (4, fprintf (gc_debug_file, "Pinned object %p, vtable %p (%s), count %d\n", search_start, *(void**)search_start, safe_name (search_start), count));
2284 pin_object (search_start);
2286 pin_stats_register_object (search_start, last_obj_size);
2287 definitely_pinned [count] = search_start;
2291 /* skip to the next object */
2292 search_start = (void*)((char*)search_start + last_obj_size);
2293 } while (search_start <= addr);
2294 /* we either pinned the correct object or we ignored the addr because
2295 * it points to unused zeroed memory.
2301 //printf ("effective pinned: %d (at the end: %d)\n", count, (char*)end_nursery - (char*)last);
2305 static void** pin_queue;
2306 static int pin_queue_size = 0;
2307 static int next_pin_slot = 0;
2312 gap = (gap * 10) / 13;
2313 if (gap == 9 || gap == 10)
2322 compare_addr (const void *a, const void *b)
2324 return *(const void **)a - *(const void **)b;
2328 /* sort the addresses in array in increasing order */
2330 sort_addresses (void **array, int size)
2333 * qsort is slower as predicted.
2334 * qsort (array, size, sizeof (gpointer), compare_addr);
2341 gap = new_gap (gap);
2344 for (i = 0; i < end; i++) {
2346 if (array [i] > array [j]) {
2347 void* val = array [i];
2348 array [i] = array [j];
2353 if (gap == 1 && !swapped)
2358 static G_GNUC_UNUSED void
2359 print_nursery_gaps (void* start_nursery, void *end_nursery)
2362 gpointer first = start_nursery;
2364 for (i = 0; i < next_pin_slot; ++i) {
2365 next = pin_queue [i];
2366 fprintf (gc_debug_file, "Nursery range: %p-%p, size: %zd\n", first, next, (char*)next-(char*)first);
2370 fprintf (gc_debug_file, "Nursery range: %p-%p, size: %zd\n", first, next, (char*)next-(char*)first);
2373 /* reduce the info in the pin queue, removing duplicate pointers and sorting them */
2375 optimize_pin_queue (int start_slot)
2377 void **start, **cur, **end;
2378 /* sort and uniq pin_queue: we just sort and we let the rest discard multiple values */
2379 /* it may be better to keep ranges of pinned memory instead of individually pinning objects */
2380 DEBUG (5, fprintf (gc_debug_file, "Sorting pin queue, size: %d\n", next_pin_slot));
2381 if ((next_pin_slot - start_slot) > 1)
2382 sort_addresses (pin_queue + start_slot, next_pin_slot - start_slot);
2383 start = cur = pin_queue + start_slot;
2384 end = pin_queue + next_pin_slot;
2387 while (*start == *cur && cur < end)
2391 next_pin_slot = start - pin_queue;
2392 DEBUG (5, fprintf (gc_debug_file, "Pin queue reduced to size: %d\n", next_pin_slot));
2393 //DEBUG (6, print_nursery_gaps (start_nursery, end_nursery));
2398 optimized_pin_queue_search (void *addr)
2400 int first = 0, last = next_pin_slot;
2401 while (first < last) {
2402 int middle = first + ((last - first) >> 1);
2403 if (addr <= pin_queue [middle])
2408 g_assert (first == last);
2413 find_optimized_pin_queue_area (void *start, void *end, int *first, int *last)
2415 *first = optimized_pin_queue_search (start);
2416 *last = optimized_pin_queue_search (end);
2420 realloc_pin_queue (void)
2422 int new_size = pin_queue_size? pin_queue_size + pin_queue_size/2: 1024;
2423 void **new_pin = get_internal_mem (sizeof (void*) * new_size, INTERNAL_MEM_PIN_QUEUE);
2424 memcpy (new_pin, pin_queue, sizeof (void*) * next_pin_slot);
2425 free_internal_mem (pin_queue, INTERNAL_MEM_PIN_QUEUE);
2426 pin_queue = new_pin;
2427 pin_queue_size = new_size;
2428 DEBUG (4, fprintf (gc_debug_file, "Reallocated pin queue to size: %d\n", new_size));
2431 #include "sgen-pinning.c"
2434 * Scan the memory between start and end and queue values which could be pointers
2435 * to the area between start_nursery and end_nursery for later consideration.
2436 * Typically used for thread stacks.
2439 conservatively_pin_objects_from (void **start, void **end, void *start_nursery, void *end_nursery, int pin_type)
2442 while (start < end) {
2443 if (*start >= start_nursery && *start < end_nursery) {
2445 * *start can point to the middle of an object
2446 * note: should we handle pointing at the end of an object?
2447 * pinning in C# code disallows pointing at the end of an object
2448 * but there is some small chance that an optimizing C compiler
2449 * may keep the only reference to an object by pointing
2450 * at the end of it. We ignore this small chance for now.
2451 * Pointers to the end of an object are indistinguishable
2452 * from pointers to the start of the next object in memory
2453 * so if we allow that we'd need to pin two objects...
2454 * We queue the pointer in an array, the
2455 * array will then be sorted and uniqued. This way
2456 * we can coalesce several pinning pointers and it should
2457 * be faster since we'd do a memory scan with increasing
2458 * addresses. Note: we can align the address to the allocation
2459 * alignment, so the unique process is more effective.
2461 mword addr = (mword)*start;
2462 addr &= ~(ALLOC_ALIGN - 1);
2463 if (addr >= (mword)start_nursery && addr < (mword)end_nursery)
2464 pin_stage_ptr ((void*)addr);
2466 pin_stats_register_address ((char*)addr, pin_type);
2467 DEBUG (6, if (count) fprintf (gc_debug_file, "Pinning address %p\n", (void*)addr));
2472 DEBUG (7, if (count) fprintf (gc_debug_file, "found %d potential pinned heap pointers\n", count));
2476 * If generation is 0, just mark objects in the nursery, the others we don't care,
2477 * since they are not going to move anyway.
2478 * There are different areas that are scanned for pinned pointers:
2479 * *) the thread stacks (when jit support is ready only the unmanaged frames)
2480 * *) the pinned handle table
2481 * *) the pinned roots
2483 * Note: when we'll use a write barrier for old to new gen references, we need to
2484 * keep track of old gen objects that point to pinned new gen objects because in that
2485 * case the referenced object will be moved maybe at the next collection, but there
2486 * is no write in the old generation area where the pinned object is referenced
2487 * and we may not consider it as reachable.
2489 static G_GNUC_UNUSED void
2490 mark_pinned_objects (int generation)
2495 * Debugging function: find in the conservative roots where @obj is being pinned.
2497 static G_GNUC_UNUSED void
2498 find_pinning_reference (char *obj, size_t size)
2502 char *endobj = obj + size;
2503 for (i = 0; i < roots_hash_size [0]; ++i) {
2504 for (root = roots_hash [0][i]; root; root = root->next) {
2505 /* if desc is non-null it has precise info */
2506 if (!root->root_desc) {
2507 char ** start = (char**)root->start_root;
2508 while (start < (char**)root->end_root) {
2509 if (*start >= obj && *start < endobj) {
2510 DEBUG (0, fprintf (gc_debug_file, "Object %p referenced in pinned roots %p-%p (at %p in record %p)\n", obj, root->start_root, root->end_root, start, root));
2517 find_pinning_ref_from_thread (obj, size);
2521 * The first thing we do in a collection is to identify pinned objects.
2522 * This function considers all the areas of memory that need to be
2523 * conservatively scanned.
2526 pin_from_roots (void *start_nursery, void *end_nursery)
2530 DEBUG (2, fprintf (gc_debug_file, "Scanning pinned roots (%d bytes, %d/%d entries)\n", (int)roots_size, num_roots_entries [ROOT_TYPE_NORMAL], num_roots_entries [ROOT_TYPE_PINNED]));
2531 /* objects pinned from the API are inside these roots */
2532 for (i = 0; i < roots_hash_size [ROOT_TYPE_PINNED]; ++i) {
2533 for (root = roots_hash [ROOT_TYPE_PINNED][i]; root; root = root->next) {
2534 DEBUG (6, fprintf (gc_debug_file, "Pinned roots %p-%p\n", root->start_root, root->end_root));
2535 conservatively_pin_objects_from ((void**)root->start_root, (void**)root->end_root, start_nursery, end_nursery, PIN_TYPE_OTHER);
2538 /* now deal with the thread stacks
2539 * in the future we should be able to conservatively scan only:
2540 * *) the cpu registers
2541 * *) the unmanaged stack frames
2542 * *) the _last_ managed stack frame
2543 * *) pointers slots in managed frames
2545 scan_thread_data (start_nursery, end_nursery, FALSE);
2547 evacuate_pin_staging_area ();
2550 /* Copy function called from user defined mark functions */
2551 static char *user_copy_n_start;
2552 static char *user_copy_n_end;
2555 user_copy (void *addr)
2558 return copy_object (addr, user_copy_n_start, user_copy_n_end);
2564 * The memory area from start_root to end_root contains pointers to objects.
2565 * Their position is precisely described by @desc (this means that the pointer
2566 * can be either NULL or the pointer to the start of an object).
2567 * This functions copies them to to_space updates them.
2570 precisely_scan_objects_from (void** start_root, void** end_root, char* n_start, char *n_end, mword desc)
2572 switch (desc & ROOT_DESC_TYPE_MASK) {
2573 case ROOT_DESC_BITMAP:
2574 desc >>= ROOT_DESC_TYPE_SHIFT;
2576 if ((desc & 1) && *start_root) {
2577 *start_root = copy_object (*start_root, n_start, n_end);
2578 DEBUG (9, fprintf (gc_debug_file, "Overwrote root at %p with %p\n", start_root, *start_root));
2579 drain_gray_stack (n_start, n_end);
2585 case ROOT_DESC_COMPLEX: {
2586 gsize *bitmap_data = complex_descriptors + (desc >> ROOT_DESC_TYPE_SHIFT);
2587 int bwords = (*bitmap_data) - 1;
2588 void **start_run = start_root;
2590 while (bwords-- > 0) {
2591 gsize bmap = *bitmap_data++;
2592 void **objptr = start_run;
2594 if ((bmap & 1) && *objptr) {
2595 *objptr = copy_object (*objptr, n_start, n_end);
2596 DEBUG (9, fprintf (gc_debug_file, "Overwrote root at %p with %p\n", objptr, *objptr));
2597 drain_gray_stack (n_start, n_end);
2602 start_run += GC_BITS_PER_WORD;
2606 case ROOT_DESC_USER: {
2607 MonoGCMarkFunc marker = user_descriptors [desc >> ROOT_DESC_TYPE_SHIFT];
2609 user_copy_n_start = n_start;
2610 user_copy_n_end = n_end;
2611 marker (start_root, user_copy);
2614 case ROOT_DESC_RUN_LEN:
2615 g_assert_not_reached ();
2617 g_assert_not_reached ();
2622 alloc_fragment (void)
2624 Fragment *frag = fragment_freelist;
2626 fragment_freelist = frag->next;
2630 frag = get_internal_mem (sizeof (Fragment), INTERNAL_MEM_FRAGMENT);
2635 /* size must be a power of 2 */
2637 get_os_memory_aligned (mword size, gboolean activate)
2639 /* Allocate twice the memory to be able to put the block on an aligned address */
2640 char *mem = get_os_memory (size * 2, activate);
2645 aligned = (char*)((mword)(mem + (size - 1)) & ~(size - 1));
2646 g_assert (aligned >= mem && aligned + size <= mem + size * 2 && !((mword)aligned & (size - 1)));
2649 free_os_memory (mem, aligned - mem);
2650 if (aligned + size < mem + size * 2)
2651 free_os_memory (aligned + size, (mem + size * 2) - (aligned + size));
2657 * Allocate and setup the data structures needed to be able to allocate objects
2658 * in the nursery. The nursery is stored in nursery_section.
2661 alloc_nursery (void)
2663 GCMemSection *section;
2669 if (nursery_section)
2671 DEBUG (2, fprintf (gc_debug_file, "Allocating nursery size: %zd\n", nursery_size));
2672 /* later we will alloc a larger area for the nursery but only activate
2673 * what we need. The rest will be used as expansion if we have too many pinned
2674 * objects in the existing nursery.
2676 /* FIXME: handle OOM */
2677 section = get_internal_mem (SIZEOF_GC_MEM_SECTION, INTERNAL_MEM_SECTION);
2679 g_assert (nursery_size == DEFAULT_NURSERY_SIZE);
2680 alloc_size = nursery_size;
2681 #ifdef ALIGN_NURSERY
2682 data = get_os_memory_aligned (alloc_size, TRUE);
2684 data = get_os_memory (alloc_size, TRUE);
2686 nursery_start = data;
2687 nursery_real_end = nursery_start + nursery_size;
2688 UPDATE_HEAP_BOUNDARIES (nursery_start, nursery_real_end);
2689 nursery_next = nursery_start;
2690 total_alloc += alloc_size;
2691 DEBUG (4, fprintf (gc_debug_file, "Expanding nursery size (%p-%p): %zd, total: %zd\n", data, data + alloc_size, nursery_size, total_alloc));
2692 section->data = section->next_data = data;
2693 section->size = alloc_size;
2694 section->end_data = nursery_real_end;
2695 scan_starts = alloc_size / SCAN_START_SIZE;
2696 section->scan_starts = get_internal_mem (sizeof (char*) * scan_starts, INTERNAL_MEM_SCAN_STARTS);
2697 section->num_scan_start = scan_starts;
2698 section->block.role = MEMORY_ROLE_GEN0;
2700 /* add to the section list */
2701 section->block.next = section_list;
2702 section_list = section;
2704 nursery_section = section;
2706 /* Setup the single first large fragment */
2707 frag = alloc_fragment ();
2708 frag->fragment_start = nursery_start;
2709 frag->fragment_limit = nursery_start;
2710 frag->fragment_end = nursery_real_end;
2711 nursery_frag_real_end = nursery_real_end;
2712 /* FIXME: frag here is lost */
2716 scan_finalizer_entries (FinalizeEntry *list, char *start, char *end) {
2719 for (fin = list; fin; fin = fin->next) {
2722 DEBUG (5, fprintf (gc_debug_file, "Scan of fin ready object: %p (%s)\n", fin->object, safe_name (fin->object)));
2723 fin->object = copy_object (fin->object, start, end);
2728 * Update roots in the old generation. Since we currently don't have the
2729 * info from the write barriers, we just scan all the objects.
2731 static G_GNUC_UNUSED void
2732 scan_old_generation (char *start, char* end)
2734 GCMemSection *section;
2735 LOSObject *big_object;
2738 for (section = section_list; section; section = section->block.next) {
2739 if (section == nursery_section)
2741 DEBUG (2, fprintf (gc_debug_file, "Scan of old section: %p-%p, size: %d\n", section->data, section->next_data, (int)(section->next_data - section->data)));
2742 /* we have to deal with zeroed holes in old generation (truncated strings ...) */
2744 while (p < section->next_data) {
2749 DEBUG (8, fprintf (gc_debug_file, "Precise old object scan of %p (%s)\n", p, safe_name (p)));
2750 p = scan_object (p, start, end);
2753 /* scan the old object space, too */
2754 for (big_object = los_object_list; big_object; big_object = big_object->next) {
2755 DEBUG (5, fprintf (gc_debug_file, "Scan of big object: %p (%s), size: %zd\n", big_object->data, safe_name (big_object->data), big_object->size));
2756 scan_object (big_object->data, start, end);
2758 /* scan the list of objects ready for finalization */
2759 scan_finalizer_entries (fin_ready_list, start, end);
2760 scan_finalizer_entries (critical_fin_list, start, end);
2763 static mword fragment_total = 0;
2765 * We found a fragment of free memory in the nursery: memzero it and if
2766 * it is big enough, add it to the list of fragments that can be used for
2770 add_nursery_frag (size_t frag_size, char* frag_start, char* frag_end)
2773 DEBUG (4, fprintf (gc_debug_file, "Found empty fragment: %p-%p, size: %zd\n", frag_start, frag_end, frag_size));
2774 /* memsetting just the first chunk start is bound to provide better cache locality */
2775 if (nursery_clear_policy == CLEAR_AT_GC)
2776 memset (frag_start, 0, frag_size);
2777 /* Not worth dealing with smaller fragments: need to tune */
2778 if (frag_size >= FRAGMENT_MIN_SIZE) {
2779 fragment = alloc_fragment ();
2780 fragment->fragment_start = frag_start;
2781 fragment->fragment_limit = frag_start;
2782 fragment->fragment_end = frag_end;
2783 fragment->next = nursery_fragments;
2784 nursery_fragments = fragment;
2785 fragment_total += frag_size;
2787 /* Clear unused fragments, pinning depends on this */
2788 memset (frag_start, 0, frag_size);
2793 scan_needed_big_objects (char *start_addr, char *end_addr)
2795 LOSObject *big_object;
2797 for (big_object = los_object_list; big_object; big_object = big_object->next) {
2798 if (!big_object->scanned && object_is_pinned (big_object->data)) {
2799 DEBUG (5, fprintf (gc_debug_file, "Scan of big object: %p (%s), size: %zd\n", big_object->data, safe_name (big_object->data), big_object->size));
2800 scan_object (big_object->data, start_addr, end_addr);
2801 big_object->scanned = TRUE;
2809 generation_name (int generation)
2811 switch (generation) {
2812 case GENERATION_NURSERY: return "nursery";
2813 case GENERATION_OLD: return "old";
2814 default: g_assert_not_reached ();
2818 static DisappearingLinkHashTable*
2819 get_dislink_hash_table (int generation)
2821 switch (generation) {
2822 case GENERATION_NURSERY: return &minor_disappearing_link_hash;
2823 case GENERATION_OLD: return &major_disappearing_link_hash;
2824 default: g_assert_not_reached ();
2828 static FinalizeEntryHashTable*
2829 get_finalize_entry_hash_table (int generation)
2831 switch (generation) {
2832 case GENERATION_NURSERY: return &minor_finalizable_hash;
2833 case GENERATION_OLD: return &major_finalizable_hash;
2834 default: g_assert_not_reached ();
2839 new_to_space_section (void)
2841 /* FIXME: if the current to_space_section is empty, we don't
2842 have to allocate a new one */
2844 to_space_section = alloc_major_section ();
2845 to_space_bumper = to_space_section->next_data;
2846 to_space_top = to_space_section->end_data;
2850 to_space_set_next_data (void)
2852 g_assert (to_space_bumper >= to_space_section->next_data && to_space_bumper <= to_space_section->end_data);
2853 to_space_section->next_data = to_space_bumper;
2857 to_space_expand (void)
2859 if (to_space_section) {
2860 g_assert (to_space_top == to_space_section->end_data);
2861 to_space_set_next_data ();
2864 new_to_space_section ();
2868 unset_to_space (void)
2870 /* between collections the to_space_bumper is invalidated
2871 because degraded allocations might occur, so we set it to
2872 NULL, just to make it explicit */
2873 to_space_bumper = NULL;
2875 /* don't unset to_space_section if we implement the FIXME in
2876 new_to_space_section */
2877 to_space_section = NULL;
2881 object_is_in_to_space (char *obj)
2886 if (ptr_in_nursery (obj))
2889 objsize = safe_object_get_size ((MonoObject*)obj);
2890 objsize += ALLOC_ALIGN - 1;
2891 objsize &= ~(ALLOC_ALIGN - 1);
2894 if (objsize > MAX_SMALL_OBJ_SIZE)
2898 if (obj_is_from_pinned_alloc (obj))
2901 /* now we know it's in a major heap section */
2902 return MAJOR_SECTION_FOR_OBJECT (obj)->is_to_space;
2906 finish_gray_stack (char *start_addr, char *end_addr, int generation)
2910 int fin_ready, bigo_scanned_num;
2913 * We copied all the reachable objects. Now it's the time to copy
2914 * the objects that were not referenced by the roots, but by the copied objects.
2915 * we built a stack of objects pointed to by gray_start: they are
2916 * additional roots and we may add more items as we go.
2917 * We loop until gray_start == gray_objects which means no more objects have
2918 * been added. Note this is iterative: no recursion is involved.
2919 * We need to walk the LO list as well in search of marked big objects
2920 * (use a flag since this is needed only on major collections). We need to loop
2921 * here as well, so keep a counter of marked LO (increasing it in copy_object).
2922 * To achieve better cache locality and cache usage, we drain the gray stack
2923 * frequently, after each object is copied, and just finish the work here.
2925 drain_gray_stack (start_addr, end_addr);
2927 //scan_old_generation (start_addr, end_addr);
2928 DEBUG (2, fprintf (gc_debug_file, "%s generation done\n", generation_name (generation)));
2929 /* walk the finalization queue and move also the objects that need to be
2930 * finalized: use the finalized objects as new roots so the objects they depend
2931 * on are also not reclaimed. As with the roots above, only objects in the nursery
2932 * are marked/copied.
2933 * We need a loop here, since objects ready for finalizers may reference other objects
2934 * that are fin-ready. Speedup with a flag?
2937 fin_ready = num_ready_finalizers;
2938 finalize_in_range (start_addr, end_addr, generation);
2939 if (generation == GENERATION_OLD)
2940 finalize_in_range (nursery_start, nursery_real_end, GENERATION_NURSERY);
2941 bigo_scanned_num = scan_needed_big_objects (start_addr, end_addr);
2943 /* drain the new stack that might have been created */
2944 DEBUG (6, fprintf (gc_debug_file, "Precise scan of gray area post fin\n"));
2945 drain_gray_stack (start_addr, end_addr);
2946 } while (fin_ready != num_ready_finalizers || bigo_scanned_num);
2948 DEBUG (2, fprintf (gc_debug_file, "Finalize queue handling scan for %s generation: %d usecs\n", generation_name (generation), TV_ELAPSED (atv, btv)));
2951 * handle disappearing links
2952 * Note we do this after checking the finalization queue because if an object
2953 * survives (at least long enough to be finalized) we don't clear the link.
2954 * This also deals with a possible issue with the monitor reclamation: with the Boehm
2955 * GC a finalized object my lose the monitor because it is cleared before the finalizer is
2958 g_assert (gray_object_queue_is_empty ());
2960 null_link_in_range (start_addr, end_addr, generation);
2961 if (generation == GENERATION_OLD)
2962 null_link_in_range (start_addr, end_addr, GENERATION_NURSERY);
2963 if (gray_object_queue_is_empty ())
2965 drain_gray_stack (start_addr, end_addr);
2968 g_assert (gray_object_queue_is_empty ());
2969 /* DEBUG (2, fprintf (gc_debug_file, "Copied from %s to old space: %d bytes (%p-%p)\n", generation_name (generation), (int)(to_space_bumper - to_space), to_space, to_space_bumper)); */
2970 to_space_set_next_data ();
2973 static int last_num_pinned = 0;
2976 build_nursery_fragments (int start_pin, int end_pin)
2978 char *frag_start, *frag_end;
2982 while (nursery_fragments) {
2983 Fragment *next = nursery_fragments->next;
2984 nursery_fragments->next = fragment_freelist;
2985 fragment_freelist = nursery_fragments;
2986 nursery_fragments = next;
2988 frag_start = nursery_start;
2990 /* clear scan starts */
2991 memset (nursery_section->scan_starts, 0, nursery_section->num_scan_start * sizeof (gpointer));
2992 for (i = start_pin; i < end_pin; ++i) {
2993 frag_end = pin_queue [i];
2994 /* remove the pin bit from pinned objects */
2995 unpin_object (frag_end);
2996 nursery_section->scan_starts [((char*)frag_end - (char*)nursery_section->data)/SCAN_START_SIZE] = frag_end;
2997 frag_size = frag_end - frag_start;
2999 add_nursery_frag (frag_size, frag_start, frag_end);
3000 frag_size = safe_object_get_size ((MonoObject*)pin_queue [i]);
3001 frag_size += ALLOC_ALIGN - 1;
3002 frag_size &= ~(ALLOC_ALIGN - 1);
3003 frag_start = (char*)pin_queue [i] + frag_size;
3005 nursery_last_pinned_end = frag_start;
3006 frag_end = nursery_real_end;
3007 frag_size = frag_end - frag_start;
3009 add_nursery_frag (frag_size, frag_start, frag_end);
3010 if (!nursery_fragments) {
3011 DEBUG (1, fprintf (gc_debug_file, "Nursery fully pinned (%d)\n", end_pin - start_pin));
3012 for (i = start_pin; i < end_pin; ++i) {
3013 DEBUG (3, fprintf (gc_debug_file, "Bastard pinning obj %p (%s), size: %d\n", pin_queue [i], safe_name (pin_queue [i]), safe_object_get_size (pin_queue [i])));
3018 nursery_next = nursery_frag_real_end = NULL;
3020 /* Clear TLABs for all threads */
3024 /* FIXME: later reduce code duplication here with the above
3025 * We don't keep track of section fragments for non-nursery sections yet, so
3029 build_section_fragments (GCMemSection *section)
3032 char *frag_start, *frag_end;
3035 /* clear scan starts */
3036 memset (section->scan_starts, 0, section->num_scan_start * sizeof (gpointer));
3037 frag_start = section->data;
3038 section->next_data = section->data;
3039 for (i = section->pin_queue_start; i < section->pin_queue_end; ++i) {
3040 frag_end = pin_queue [i];
3041 /* remove the pin bit from pinned objects */
3042 unpin_object (frag_end);
3043 if (frag_end >= section->data + section->size) {
3044 frag_end = section->data + section->size;
3046 section->scan_starts [((char*)frag_end - (char*)section->data)/SCAN_START_SIZE] = frag_end;
3048 frag_size = frag_end - frag_start;
3050 memset (frag_start, 0, frag_size);
3051 frag_size = safe_object_get_size ((MonoObject*)pin_queue [i]);
3052 frag_size += ALLOC_ALIGN - 1;
3053 frag_size &= ~(ALLOC_ALIGN - 1);
3054 frag_start = (char*)pin_queue [i] + frag_size;
3055 section->next_data = MAX (section->next_data, frag_start);
3057 frag_end = section->end_data;
3058 frag_size = frag_end - frag_start;
3060 memset (frag_start, 0, frag_size);
3064 scan_from_registered_roots (char *addr_start, char *addr_end, int root_type)
3068 for (i = 0; i < roots_hash_size [root_type]; ++i) {
3069 for (root = roots_hash [root_type][i]; root; root = root->next) {
3070 DEBUG (6, fprintf (gc_debug_file, "Precise root scan %p-%p (desc: %p)\n", root->start_root, root->end_root, (void*)root->root_desc));
3071 precisely_scan_objects_from ((void**)root->start_root, (void**)root->end_root, addr_start, addr_end, root->root_desc);
3077 dump_occupied (char *start, char *end, char *section_start)
3079 fprintf (heap_dump_file, "<occupied offset=\"%zd\" size=\"%zd\"/>\n", start - section_start, end - start);
3083 dump_section (GCMemSection *section, const char *type)
3085 char *start = section->data;
3086 char *end = section->data + section->size;
3087 char *occ_start = NULL;
3089 char *old_start = NULL; /* just for debugging */
3091 fprintf (heap_dump_file, "<section type=\"%s\" size=\"%zu\">\n", type, section->size);
3093 while (start < end) {
3097 if (!*(void**)start) {
3099 dump_occupied (occ_start, start, section->data);
3102 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
3105 g_assert (start < section->next_data);
3110 vt = (GCVTable*)LOAD_VTABLE (start);
3113 size = safe_object_get_size ((MonoObject*) start);
3114 size += ALLOC_ALIGN - 1;
3115 size &= ~(ALLOC_ALIGN - 1);
3118 fprintf (heap_dump_file, "<object offset=\"%d\" class=\"%s.%s\" size=\"%d\"/>\n",
3119 start - section->data,
3120 vt->klass->name_space, vt->klass->name,
3128 dump_occupied (occ_start, start, section->data);
3130 fprintf (heap_dump_file, "</section>\n");
3134 dump_heap (const char *type, int num, const char *reason)
3136 static char const *internal_mem_names [] = { "pin-queue", "fragment", "section", "scan-starts",
3137 "fin-table", "finalize-entry", "dislink-table",
3138 "dislink", "roots-table", "root-record", "statistics",
3139 "remset", "gray-queue", "store-remset" };
3141 GCMemSection *section;
3145 fprintf (heap_dump_file, "<collection type=\"%s\" num=\"%d\"", type, num);
3147 fprintf (heap_dump_file, " reason=\"%s\"", reason);
3148 fprintf (heap_dump_file, ">\n");
3149 fprintf (heap_dump_file, "<other-mem-usage type=\"pinned-chunks\" size=\"%ld\"/>\n", pinned_chunk_bytes_alloced);
3150 fprintf (heap_dump_file, "<other-mem-usage type=\"large-internal\" size=\"%ld\"/>\n", large_internal_bytes_alloced);
3151 fprintf (heap_dump_file, "<other-mem-usage type=\"mempools\" size=\"%ld\"/>\n", mono_mempool_get_bytes_allocated ());
3152 for (i = 0; i < INTERNAL_MEM_MAX; ++i)
3153 fprintf (heap_dump_file, "<other-mem-usage type=\"%s\" size=\"%ld\"/>\n", internal_mem_names [i], small_internal_mem_bytes [i]);
3154 fprintf (heap_dump_file, "<pinned type=\"stack\" bytes=\"%zu\"/>\n", pinned_byte_counts [PIN_TYPE_STACK]);
3155 /* fprintf (heap_dump_file, "<pinned type=\"static-data\" bytes=\"%d\"/>\n", pinned_byte_counts [PIN_TYPE_STATIC_DATA]); */
3156 fprintf (heap_dump_file, "<pinned type=\"other\" bytes=\"%zu\"/>\n", pinned_byte_counts [PIN_TYPE_OTHER]);
3158 dump_section (nursery_section, "nursery");
3160 for (section = section_list; section; section = section->block.next) {
3161 if (section != nursery_section)
3162 dump_section (section, "old");
3165 fprintf (heap_dump_file, "<los>\n");
3166 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
3167 MonoObject *obj = (MonoObject*) bigobj->data;
3168 MonoClass *class = mono_object_class (obj);
3170 fprintf (heap_dump_file, "<object class=\"%s.%s\" size=\"%d\"/>\n",
3171 class->name_space, class->name,
3172 safe_object_get_size (obj));
3174 fprintf (heap_dump_file, "</los>\n");
3176 fprintf (heap_dump_file, "</collection>\n");
3182 static gboolean inited = FALSE;
3184 #ifdef HEAVY_STATISTICS
3185 num_copy_object_called = 0;
3186 num_objects_copied = 0;
3192 #ifdef HEAVY_STATISTICS
3193 mono_counters_register ("WBarrier set field", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_field);
3194 mono_counters_register ("WBarrier set arrayref", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_arrayref);
3195 mono_counters_register ("WBarrier arrayref copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_arrayref_copy);
3196 mono_counters_register ("WBarrier generic store called", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_generic_store);
3197 mono_counters_register ("WBarrier generic store stored", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_generic_store_remset);
3198 mono_counters_register ("WBarrier set root", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_root);
3199 mono_counters_register ("WBarrier value copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_value_copy);
3200 mono_counters_register ("WBarrier object copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_object_copy);
3202 mono_counters_register ("# objects allocated", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_alloced);
3203 mono_counters_register ("# copy_object() called (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_called_nursery);
3204 mono_counters_register ("# objects copied (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_copied_nursery);
3205 mono_counters_register ("# copy_object() called (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_called_major);
3206 mono_counters_register ("# objects copied (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_copied_major);
3208 mono_counters_register ("# copy_object() failed from space", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_failed_from_space);
3209 mono_counters_register ("# copy_object() failed forwarded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_failed_forwarded);
3210 mono_counters_register ("# copy_object() failed pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_failed_pinned);
3211 mono_counters_register ("# copy_object() failed large or pinned chunk", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_failed_large_pinned);
3212 mono_counters_register ("# copy_object() failed to space", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_failed_to_space);
3214 mono_counters_register ("Store remsets", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_store_remsets);
3215 mono_counters_register ("Unique store remsets", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_store_remsets_unique);
3216 mono_counters_register ("Saved remsets 1", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_saved_remsets_1);
3217 mono_counters_register ("Saved remsets 2", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_saved_remsets_2);
3218 mono_counters_register ("Global remsets added", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_added);
3219 mono_counters_register ("Global remsets processed", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_processed);
3226 commit_stats (int generation)
3228 #ifdef HEAVY_STATISTICS
3229 if (generation == GENERATION_NURSERY) {
3230 stat_copy_object_called_nursery += num_copy_object_called;
3231 stat_objects_copied_nursery += num_objects_copied;
3233 g_assert (generation == GENERATION_OLD);
3234 stat_copy_object_called_major += num_copy_object_called;
3235 stat_objects_copied_major += num_objects_copied;
3241 * Collect objects in the nursery. Returns whether to trigger a major
3245 collect_nursery (size_t requested_size)
3247 size_t max_garbage_amount;
3249 char *orig_nursery_next;
3251 GCMemSection *section;
3252 int old_num_major_sections = num_major_sections;
3253 int sections_alloced;
3254 TV_DECLARE (all_atv);
3255 TV_DECLARE (all_btv);
3262 orig_nursery_next = nursery_next;
3263 nursery_next = MAX (nursery_next, nursery_last_pinned_end);
3264 /* FIXME: optimize later to use the higher address where an object can be present */
3265 nursery_next = MAX (nursery_next, nursery_real_end);
3267 if (consistency_check_at_minor_collection)
3268 check_consistency ();
3270 DEBUG (1, fprintf (gc_debug_file, "Start nursery collection %d %p-%p, size: %d\n", num_minor_gcs, nursery_start, nursery_next, (int)(nursery_next - nursery_start)));
3271 max_garbage_amount = nursery_next - nursery_start;
3272 g_assert (nursery_section->size >= max_garbage_amount);
3274 /* Clear all remaining nursery fragments, pinning depends on this */
3275 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION) {
3276 g_assert (orig_nursery_next <= nursery_frag_real_end);
3277 memset (orig_nursery_next, 0, nursery_frag_real_end - orig_nursery_next);
3278 for (frag = nursery_fragments; frag; frag = frag->next) {
3279 memset (frag->fragment_start, 0, frag->fragment_end - frag->fragment_start);
3284 check_for_xdomain_refs ();
3286 nursery_section->next_data = nursery_next;
3288 if (!to_space_section) {
3289 new_to_space_section ();
3291 /* we might have done degraded allocation since the
3293 g_assert (to_space_bumper <= to_space_section->next_data);
3294 to_space_bumper = to_space_section->next_data;
3296 to_space_section->is_to_space = TRUE;
3298 gray_object_queue_init ();
3301 mono_stats.minor_gc_count ++;
3302 /* world must be stopped already */
3303 TV_GETTIME (all_atv);
3305 /* pin from pinned handles */
3307 pin_from_roots (nursery_start, nursery_next);
3308 /* identify pinned objects */
3309 optimize_pin_queue (0);
3310 next_pin_slot = pin_objects_from_addresses (nursery_section, pin_queue, pin_queue + next_pin_slot, nursery_start, nursery_next);
3312 DEBUG (2, fprintf (gc_debug_file, "Finding pinned pointers: %d in %d usecs\n", next_pin_slot, TV_ELAPSED (atv, btv)));
3313 DEBUG (4, fprintf (gc_debug_file, "Start scan with %d pinned objects\n", next_pin_slot));
3316 * walk all the roots and copy the young objects to the old generation,
3317 * starting from to_space
3320 scan_from_remsets (nursery_start, nursery_next);
3321 /* we don't have complete write barrier yet, so we scan all the old generation sections */
3323 DEBUG (2, fprintf (gc_debug_file, "Old generation scan: %d usecs\n", TV_ELAPSED (btv, atv)));
3325 /* the pinned objects are roots */
3326 for (i = 0; i < next_pin_slot; ++i) {
3327 DEBUG (6, fprintf (gc_debug_file, "Precise object scan %d of pinned %p (%s)\n", i, pin_queue [i], safe_name (pin_queue [i])));
3328 scan_object (pin_queue [i], nursery_start, nursery_next);
3330 /* registered roots, this includes static fields */
3331 scan_from_registered_roots (nursery_start, nursery_next, ROOT_TYPE_NORMAL);
3332 scan_from_registered_roots (nursery_start, nursery_next, ROOT_TYPE_WBARRIER);
3333 scan_thread_data (nursery_start, nursery_next, TRUE);
3334 /* alloc_pinned objects */
3335 scan_from_pinned_objects (nursery_start, nursery_next);
3337 DEBUG (2, fprintf (gc_debug_file, "Root scan: %d usecs\n", TV_ELAPSED (atv, btv)));
3339 finish_gray_stack (nursery_start, nursery_next, GENERATION_NURSERY);
3341 /* walk the pin_queue, build up the fragment list of free memory, unmark
3342 * pinned objects as we go, memzero() the empty fragments so they are ready for the
3345 build_nursery_fragments (0, next_pin_slot);
3347 DEBUG (2, fprintf (gc_debug_file, "Fragment creation: %d usecs, %zd bytes available\n", TV_ELAPSED (btv, atv), fragment_total));
3349 for (section = section_list; section; section = section->block.next) {
3350 if (section->is_to_space)
3351 section->is_to_space = FALSE;
3354 TV_GETTIME (all_btv);
3355 mono_stats.minor_gc_time_usecs += TV_ELAPSED (all_atv, all_btv);
3358 dump_heap ("minor", num_minor_gcs - 1, NULL);
3360 /* prepare the pin queue for the next collection */
3361 last_num_pinned = next_pin_slot;
3363 if (fin_ready_list || critical_fin_list) {
3364 DEBUG (4, fprintf (gc_debug_file, "Finalizer-thread wakeup: ready %d\n", num_ready_finalizers));
3365 mono_gc_finalize_notify ();
3369 g_assert (gray_object_queue_is_empty ());
3371 commit_stats (GENERATION_NURSERY);
3373 sections_alloced = num_major_sections - old_num_major_sections;
3374 minor_collection_sections_alloced += sections_alloced;
3376 return minor_collection_sections_alloced > minor_collection_section_allowance;
3380 scan_from_pinned_chunk_if_marked (PinnedChunk *chunk, char *obj, size_t size, void *dummy)
3382 if (object_is_pinned (obj))
3383 scan_object (obj, NULL, (char*)-1);
3387 major_collection (const char *reason)
3389 GCMemSection *section, *prev_section;
3390 LOSObject *bigobj, *prevbo;
3394 TV_DECLARE (all_atv);
3395 TV_DECLARE (all_btv);
3398 /* FIXME: only use these values for the precise scan
3399 * note that to_space pointers should be excluded anyway...
3401 char *heap_start = NULL;
3402 char *heap_end = (char*)-1;
3403 size_t copy_space_required = 0;
3404 int old_num_major_sections = num_major_sections;
3405 int num_major_sections_saved, save_target, allowance_target;
3410 DEBUG (1, fprintf (gc_debug_file, "Start major collection %d\n", num_major_gcs));
3412 mono_stats.major_gc_count ++;
3414 /* Clear all remaining nursery fragments, pinning depends on this */
3415 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION) {
3416 g_assert (nursery_next <= nursery_frag_real_end);
3417 memset (nursery_next, 0, nursery_frag_real_end - nursery_next);
3418 for (frag = nursery_fragments; frag; frag = frag->next) {
3419 memset (frag->fragment_start, 0, frag->fragment_end - frag->fragment_start);
3424 check_for_xdomain_refs ();
3427 * FIXME: implement Mark/Compact
3428 * Until that is done, we can just apply mostly the same alg as for the nursery:
3429 * this means we need a big section to potentially copy all the other sections, so
3430 * it is not ideal specially with large heaps.
3432 if (g_getenv ("MONO_GC_NO_MAJOR")) {
3433 collect_nursery (0);
3436 TV_GETTIME (all_atv);
3437 /* FIXME: make sure the nursery next_data ptr is updated */
3438 nursery_section->next_data = nursery_real_end;
3439 /* we should also coalesce scanning from sections close to each other
3440 * and deal with pointers outside of the sections later.
3442 /* The remsets are not useful for a major collection */
3444 /* world must be stopped already */
3447 DEBUG (6, fprintf (gc_debug_file, "Collecting pinned addresses\n"));
3448 pin_from_roots ((void*)lowest_heap_address, (void*)highest_heap_address);
3449 optimize_pin_queue (0);
3452 * pin_queue now contains all candidate pointers, sorted and
3453 * uniqued. We must do two passes now to figure out which
3454 * objects are pinned.
3456 * The first is to find within the pin_queue the area for each
3457 * section. This requires that the pin_queue be sorted. We
3458 * also process the LOS objects and pinned chunks here.
3460 * The second, destructive, pass is to reduce the section
3461 * areas to pointers to the actually pinned objects.
3463 DEBUG (6, fprintf (gc_debug_file, "Pinning from sections\n"));
3464 /* first pass for the sections */
3465 for (section = section_list; section; section = section->block.next) {
3467 DEBUG (6, fprintf (gc_debug_file, "Pinning from section %p (%p-%p)\n", section, section->data, section->end_data));
3468 find_optimized_pin_queue_area (section->data, section->end_data, &start, &end);
3469 DEBUG (6, fprintf (gc_debug_file, "Found %d pinning addresses in section %p (%d-%d)\n",
3470 end - start, section, start, end));
3471 section->pin_queue_start = start;
3472 section->pin_queue_end = end;
3474 /* identify possible pointers to the insize of large objects */
3475 DEBUG (6, fprintf (gc_debug_file, "Pinning from large objects\n"));
3476 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
3478 find_optimized_pin_queue_area (bigobj->data, (char*)bigobj->data + bigobj->size, &start, &end);
3480 pin_object (bigobj->data);
3482 pin_stats_register_object ((char*) bigobj->data, safe_object_get_size ((MonoObject*) bigobj->data));
3483 DEBUG (6, fprintf (gc_debug_file, "Marked large object %p (%s) size: %zd from roots\n", bigobj->data, safe_name (bigobj->data), bigobj->size));
3486 /* look for pinned addresses for pinned-alloc objects */
3487 DEBUG (6, fprintf (gc_debug_file, "Pinning from pinned-alloc objects\n"));
3488 for (chunk = pinned_chunk_list; chunk; chunk = chunk->block.next) {
3490 find_optimized_pin_queue_area (chunk->start_data, (char*)chunk + chunk->num_pages * FREELIST_PAGESIZE, &start, &end);
3492 mark_pinned_from_addresses (chunk, pin_queue + start, pin_queue + end);
3494 /* second pass for the sections */
3495 for (section = section_list; section; section = section->block.next) {
3496 int start = section->pin_queue_start;
3497 int end = section->pin_queue_end;
3500 reduced_to = pin_objects_from_addresses (section, pin_queue + start, pin_queue + end,
3501 section->data, section->next_data);
3502 section->pin_queue_start = start;
3503 section->pin_queue_end = start + reduced_to;
3505 copy_space_required += (char*)section->next_data - (char*)section->data;
3509 DEBUG (2, fprintf (gc_debug_file, "Finding pinned pointers: %d in %d usecs\n", next_pin_slot, TV_ELAPSED (atv, btv)));
3510 DEBUG (4, fprintf (gc_debug_file, "Start scan with %d pinned objects\n", next_pin_slot));
3512 new_to_space_section ();
3513 gray_object_queue_init ();
3515 /* the old generation doesn't need to be scanned (no remembered sets or card
3516 * table needed either): the only objects that must survive are those pinned and
3517 * those referenced by the precise roots.
3518 * mark any section without pinned objects, so we can free it since we will be able to
3519 * move all the objects.
3521 /* the pinned objects are roots (big objects are included in this list, too) */
3522 for (section = section_list; section; section = section->block.next) {
3523 for (i = section->pin_queue_start; i < section->pin_queue_end; ++i) {
3524 DEBUG (6, fprintf (gc_debug_file, "Precise object scan %d of pinned %p (%s)\n",
3525 i, pin_queue [i], safe_name (pin_queue [i])));
3526 scan_object (pin_queue [i], heap_start, heap_end);
3529 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
3530 if (object_is_pinned (bigobj->data)) {
3531 DEBUG (6, fprintf (gc_debug_file, "Precise object scan pinned LOS object %p (%s)\n",
3532 bigobj->data, safe_name (bigobj->data)));
3533 scan_object (bigobj->data, heap_start, heap_end);
3536 scan_pinned_objects (scan_from_pinned_chunk_if_marked, NULL);
3537 /* registered roots, this includes static fields */
3538 scan_from_registered_roots (heap_start, heap_end, ROOT_TYPE_NORMAL);
3539 scan_from_registered_roots (heap_start, heap_end, ROOT_TYPE_WBARRIER);
3541 scan_thread_data (heap_start, heap_end, TRUE);
3542 /* alloc_pinned objects */
3543 scan_from_pinned_objects (heap_start, heap_end);
3544 /* scan the list of objects ready for finalization */
3545 scan_finalizer_entries (fin_ready_list, heap_start, heap_end);
3546 scan_finalizer_entries (critical_fin_list, heap_start, heap_end);
3548 DEBUG (2, fprintf (gc_debug_file, "Root scan: %d usecs\n", TV_ELAPSED (btv, atv)));
3550 /* we need to go over the big object list to see if any was marked and scan it
3551 * And we need to make this in a loop, considering that objects referenced by finalizable
3552 * objects could reference big objects (this happens in finish_gray_stack ())
3554 scan_needed_big_objects (heap_start, heap_end);
3555 /* all the objects in the heap */
3556 finish_gray_stack (heap_start, heap_end, GENERATION_OLD);
3560 /* sweep the big objects list */
3562 for (bigobj = los_object_list; bigobj;) {
3563 if (object_is_pinned (bigobj->data)) {
3564 unpin_object (bigobj->data);
3565 bigobj->scanned = FALSE;
3568 /* not referenced anywhere, so we can free it */
3570 prevbo->next = bigobj->next;
3572 los_object_list = bigobj->next;
3574 bigobj = bigobj->next;
3575 free_large_object (to_free);
3579 bigobj = bigobj->next;
3581 /* unpin objects from the pinned chunks and free the unmarked ones */
3582 sweep_pinned_objects ();
3584 /* free the unused sections */
3585 prev_section = NULL;
3586 for (section = section_list; section;) {
3587 /* to_space doesn't need handling here and the nursery is special */
3588 if (section->is_to_space || section == nursery_section) {
3589 if (section->is_to_space)
3590 section->is_to_space = FALSE;
3591 prev_section = section;
3592 section = section->block.next;
3595 /* no pinning object, so the section is free */
3596 if (section->pin_queue_start == section->pin_queue_end) {
3597 GCMemSection *to_free;
3599 prev_section->block.next = section->block.next;
3601 section_list = section->block.next;
3603 section = section->block.next;
3604 free_major_section (to_free);
3607 DEBUG (6, fprintf (gc_debug_file, "Section %p has still pinned objects (%d)\n", section, section->pin_queue_end - section->pin_queue_start));
3608 build_section_fragments (section);
3610 prev_section = section;
3611 section = section->block.next;
3614 /* walk the pin_queue, build up the fragment list of free memory, unmark
3615 * pinned objects as we go, memzero() the empty fragments so they are ready for the
3618 build_nursery_fragments (nursery_section->pin_queue_start, nursery_section->pin_queue_end);
3620 TV_GETTIME (all_btv);
3621 mono_stats.major_gc_time_usecs += TV_ELAPSED (all_atv, all_btv);
3624 dump_heap ("major", num_major_gcs - 1, reason);
3626 /* prepare the pin queue for the next collection */
3628 if (fin_ready_list || critical_fin_list) {
3629 DEBUG (4, fprintf (gc_debug_file, "Finalizer-thread wakeup: ready %d\n", num_ready_finalizers));
3630 mono_gc_finalize_notify ();
3634 g_assert (gray_object_queue_is_empty ());
3636 commit_stats (GENERATION_OLD);
3638 num_major_sections_saved = MAX (old_num_major_sections - num_major_sections, 1);
3640 save_target = num_major_sections / 2;
3641 allowance_target = save_target * minor_collection_sections_alloced / num_major_sections_saved;
3643 minor_collection_section_allowance = MAX (MIN (allowance_target, num_major_sections), MIN_MINOR_COLLECTION_SECTION_ALLOWANCE);
3646 printf ("alloced %d saved %d target %d allowance %d\n",
3647 minor_collection_sections_alloced, num_major_sections_saved, allowance_target,
3648 minor_collection_section_allowance);
3651 minor_collection_sections_alloced = 0;
3655 * Allocate a new section of memory to be used as old generation.
3657 static GCMemSection*
3658 alloc_major_section (void)
3660 GCMemSection *section;
3663 section = get_os_memory_aligned (MAJOR_SECTION_SIZE, TRUE);
3664 section->next_data = section->data = (char*)section + SIZEOF_GC_MEM_SECTION;
3665 g_assert (!((mword)section->data & 7));
3666 section->size = MAJOR_SECTION_SIZE - SIZEOF_GC_MEM_SECTION;
3667 section->end_data = section->data + section->size;
3668 UPDATE_HEAP_BOUNDARIES (section->data, section->end_data);
3669 total_alloc += section->size;
3670 DEBUG (3, fprintf (gc_debug_file, "New major heap section: (%p-%p), total: %zd\n", section->data, section->end_data, total_alloc));
3671 scan_starts = section->size / SCAN_START_SIZE;
3672 section->scan_starts = get_internal_mem (sizeof (char*) * scan_starts, INTERNAL_MEM_SCAN_STARTS);
3673 section->num_scan_start = scan_starts;
3674 section->block.role = MEMORY_ROLE_GEN1;
3675 section->is_to_space = TRUE;
3677 /* add to the section list */
3678 section->block.next = section_list;
3679 section_list = section;
3681 ++num_major_sections;
3687 free_major_section (GCMemSection *section)
3689 DEBUG (3, fprintf (gc_debug_file, "Freed major section %p (%p-%p)\n", section, section->data, section->end_data));
3690 free_internal_mem (section->scan_starts, INTERNAL_MEM_SCAN_STARTS);
3691 free_os_memory (section, MAJOR_SECTION_SIZE);
3692 total_alloc -= MAJOR_SECTION_SIZE - SIZEOF_GC_MEM_SECTION;
3694 --num_major_sections;
3698 * When deciding if it's better to collect or to expand, keep track
3699 * of how much garbage was reclaimed with the last collection: if it's too
3701 * This is called when we could not allocate a small object.
3703 static void __attribute__((noinline))
3704 minor_collect_or_expand_inner (size_t size)
3706 int do_minor_collection = 1;
3708 if (!nursery_section) {
3712 if (do_minor_collection) {
3714 if (collect_nursery (size))
3715 major_collection ("minor overflow");
3716 DEBUG (2, fprintf (gc_debug_file, "Heap size: %zd, LOS size: %zd\n", total_alloc, los_memory_usage));
3718 /* this also sets the proper pointers for the next allocation */
3719 if (!search_fragment_for_size (size)) {
3721 /* TypeBuilder and MonoMethod are killing mcs with fragmentation */
3722 DEBUG (1, fprintf (gc_debug_file, "nursery collection didn't find enough room for %zd alloc (%d pinned)\n", size, last_num_pinned));
3723 for (i = 0; i < last_num_pinned; ++i) {
3724 DEBUG (3, fprintf (gc_debug_file, "Bastard pinning obj %p (%s), size: %d\n", pin_queue [i], safe_name (pin_queue [i]), safe_object_get_size (pin_queue [i])));
3729 //report_internal_mem_usage ();
3733 * ######################################################################
3734 * ######## Memory allocation from the OS
3735 * ######################################################################
3736 * This section of code deals with getting memory from the OS and
3737 * allocating memory for GC-internal data structures.
3738 * Internal memory can be handled with a freelist for small objects.
3742 * Allocate a big chunk of memory from the OS (usually 64KB to several megabytes).
3743 * This must not require any lock.
3746 get_os_memory (size_t size, int activate)
3749 unsigned long prot_flags = activate? MONO_MMAP_READ|MONO_MMAP_WRITE: MONO_MMAP_NONE;
3751 prot_flags |= MONO_MMAP_PRIVATE | MONO_MMAP_ANON;
3752 size += pagesize - 1;
3753 size &= ~(pagesize - 1);
3754 ptr = mono_valloc (0, size, prot_flags);
3759 * Free the memory returned by get_os_memory (), returning it to the OS.
3762 free_os_memory (void *addr, size_t size)
3764 mono_vfree (addr, size);
3771 report_pinned_chunk (PinnedChunk *chunk, int seq) {
3773 int i, free_pages, num_free, free_mem;
3775 for (i = 0; i < chunk->num_pages; ++i) {
3776 if (!chunk->page_sizes [i])
3779 printf ("Pinned chunk %d at %p, size: %d, pages: %d, free: %d\n", seq, chunk, chunk->num_pages * FREELIST_PAGESIZE, chunk->num_pages, free_pages);
3780 free_mem = FREELIST_PAGESIZE * free_pages;
3781 for (i = 0; i < FREELIST_NUM_SLOTS; ++i) {
3782 if (!chunk->free_list [i])
3785 p = chunk->free_list [i];
3790 printf ("\tfree list of size %d, %d items\n", freelist_sizes [i], num_free);
3791 free_mem += freelist_sizes [i] * num_free;
3793 printf ("\tfree memory in chunk: %d\n", free_mem);
3799 static G_GNUC_UNUSED void
3800 report_internal_mem_usage (void) {
3803 printf ("Internal memory usage:\n");
3805 for (chunk = internal_chunk_list; chunk; chunk = chunk->block.next) {
3806 report_pinned_chunk (chunk, i++);
3808 printf ("Pinned memory usage:\n");
3810 for (chunk = pinned_chunk_list; chunk; chunk = chunk->block.next) {
3811 report_pinned_chunk (chunk, i++);
3816 * the array of pointers from @start to @end contains conservative
3817 * pointers to objects inside @chunk: mark each referenced object
3821 mark_pinned_from_addresses (PinnedChunk *chunk, void **start, void **end)
3823 for (; start < end; start++) {
3824 char *addr = *start;
3825 int offset = (char*)addr - (char*)chunk;
3826 int page = offset / FREELIST_PAGESIZE;
3827 int obj_offset = page == 0? offset - ((char*)chunk->start_data - (char*)chunk): offset % FREELIST_PAGESIZE;
3828 int slot_size = chunk->page_sizes [page];
3830 /* the page is not allocated */
3833 /* would be faster if we restrict the sizes to power of two,
3834 * but that's a waste of memory: need to measure. it could reduce
3835 * fragmentation since there are less pages needed, if for example
3836 * someone interns strings of each size we end up with one page per
3837 * interned string (still this is just ~40 KB): with more fine-grained sizes
3838 * this increases the number of used pages.
3841 obj_offset /= slot_size;
3842 obj_offset *= slot_size;
3843 addr = (char*)chunk->start_data + obj_offset;
3845 obj_offset /= slot_size;
3846 obj_offset *= slot_size;
3847 addr = (char*)chunk + page * FREELIST_PAGESIZE + obj_offset;
3850 /* if the vtable is inside the chunk it's on the freelist, so skip */
3851 if (*ptr && (*ptr < (void*)chunk->start_data || *ptr > (void*)((char*)chunk + chunk->num_pages * FREELIST_PAGESIZE))) {
3854 pin_stats_register_object ((char*) addr, safe_object_get_size ((MonoObject*) addr));
3855 DEBUG (6, fprintf (gc_debug_file, "Marked pinned object %p (%s) from roots\n", addr, safe_name (addr)));
3861 scan_pinned_objects (ScanPinnedObjectCallbackFunc callback, void *callback_data)
3868 for (chunk = pinned_chunk_list; chunk; chunk = chunk->block.next) {
3869 end_chunk = (char*)chunk + chunk->num_pages * FREELIST_PAGESIZE;
3870 DEBUG (6, fprintf (gc_debug_file, "Scanning pinned chunk %p (range: %p-%p)\n", chunk, chunk->start_data, end_chunk));
3871 for (i = 0; i < chunk->num_pages; ++i) {
3872 obj_size = chunk->page_sizes [i];
3875 p = i? (char*)chunk + i * FREELIST_PAGESIZE: chunk->start_data;
3876 endp = i? p + FREELIST_PAGESIZE: (char*)chunk + FREELIST_PAGESIZE;
3877 DEBUG (6, fprintf (gc_debug_file, "Page %d (size: %d, range: %p-%p)\n", i, obj_size, p, endp));
3878 while (p + obj_size <= endp) {
3880 DEBUG (9, fprintf (gc_debug_file, "Considering %p (vtable: %p)\n", ptr, *ptr));
3881 /* if the first word (the vtable) is outside the chunk we have an object */
3882 if (*ptr && (*ptr < (void*)chunk || *ptr >= end_chunk))
3883 callback (chunk, (char*)ptr, obj_size, callback_data);
3891 sweep_pinned_objects_callback (PinnedChunk *chunk, char *ptr, size_t size, void *data)
3893 if (object_is_pinned (ptr)) {
3895 DEBUG (6, fprintf (gc_debug_file, "Unmarked pinned object %p (%s)\n", ptr, safe_name (ptr)));
3897 DEBUG (6, fprintf (gc_debug_file, "Freeing unmarked pinned object %p (%s)\n", ptr, safe_name (ptr)));
3898 free_pinned_object (chunk, ptr, size);
3903 sweep_pinned_objects (void)
3905 scan_pinned_objects (sweep_pinned_objects_callback, NULL);
3909 scan_object_callback (PinnedChunk *chunk, char *ptr, size_t size, char **data)
3911 DEBUG (6, fprintf (gc_debug_file, "Precise object scan of alloc_pinned %p (%s)\n", ptr, safe_name (ptr)));
3912 /* FIXME: Put objects without references into separate chunks
3913 which do not need to be scanned */
3914 scan_object (ptr, data [0], data [1]);
3918 scan_from_pinned_objects (char *addr_start, char *addr_end)
3920 char *data [2] = { addr_start, addr_end };
3921 scan_pinned_objects ((ScanPinnedObjectCallbackFunc)scan_object_callback, data);
3925 * Find the slot number in the freelist for memory chunks that
3926 * can contain @size objects.
3929 slot_for_size (size_t size)
3932 /* do a binary search or lookup table later. */
3933 for (slot = 0; slot < FREELIST_NUM_SLOTS; ++slot) {
3934 if (freelist_sizes [slot] >= size)
3937 g_assert_not_reached ();
3942 * Build a free list for @size memory chunks from the memory area between
3943 * start_page and end_page.
3946 build_freelist (PinnedChunk *chunk, int slot, int size, char *start_page, char *end_page)
3950 /*g_print ("building freelist for slot %d, size %d in %p\n", slot, size, chunk);*/
3951 p = (void**)start_page;
3952 end = (void**)(end_page - size);
3953 g_assert (!chunk->free_list [slot]);
3954 chunk->free_list [slot] = p;
3955 while ((char*)p + size <= (char*)end) {
3957 *p = (void*)((char*)p + size);
3961 /*g_print ("%d items created, max: %d\n", count, (end_page - start_page) / size);*/
3965 alloc_pinned_chunk (void)
3969 int size = MAJOR_SECTION_SIZE;
3971 chunk = get_os_memory_aligned (size, TRUE);
3972 chunk->block.role = MEMORY_ROLE_PINNED;
3974 UPDATE_HEAP_BOUNDARIES (chunk, ((char*)chunk + size));
3975 total_alloc += size;
3976 pinned_chunk_bytes_alloced += size;
3978 /* setup the bookeeping fields */
3979 chunk->num_pages = size / FREELIST_PAGESIZE;
3980 offset = G_STRUCT_OFFSET (PinnedChunk, data);
3981 chunk->page_sizes = (void*)((char*)chunk + offset);
3982 offset += sizeof (int) * chunk->num_pages;
3983 offset += ALLOC_ALIGN - 1;
3984 offset &= ~(ALLOC_ALIGN - 1);
3985 chunk->free_list = (void*)((char*)chunk + offset);
3986 offset += sizeof (void*) * FREELIST_NUM_SLOTS;
3987 offset += ALLOC_ALIGN - 1;
3988 offset &= ~(ALLOC_ALIGN - 1);
3989 chunk->start_data = (void*)((char*)chunk + offset);
3991 /* allocate the first page to the freelist */
3992 chunk->page_sizes [0] = PINNED_FIRST_SLOT_SIZE;
3993 build_freelist (chunk, slot_for_size (PINNED_FIRST_SLOT_SIZE), PINNED_FIRST_SLOT_SIZE, chunk->start_data, ((char*)chunk + FREELIST_PAGESIZE));
3994 DEBUG (4, fprintf (gc_debug_file, "Allocated pinned chunk %p, size: %d\n", chunk, size));
3995 min_pinned_chunk_addr = MIN (min_pinned_chunk_addr, (char*)chunk->start_data);
3996 max_pinned_chunk_addr = MAX (max_pinned_chunk_addr, ((char*)chunk + size));
4000 /* assumes freelist for slot is empty, so try to alloc a new page */
4002 get_chunk_freelist (PinnedChunk *chunk, int slot)
4006 p = chunk->free_list [slot];
4008 chunk->free_list [slot] = *p;
4011 for (i = 0; i < chunk->num_pages; ++i) {
4013 if (chunk->page_sizes [i])
4015 size = freelist_sizes [slot];
4016 chunk->page_sizes [i] = size;
4017 build_freelist (chunk, slot, size, (char*)chunk + FREELIST_PAGESIZE * i, (char*)chunk + FREELIST_PAGESIZE * (i + 1));
4021 p = chunk->free_list [slot];
4023 chunk->free_list [slot] = *p;
4030 alloc_from_freelist (size_t size)
4034 PinnedChunk *pchunk;
4035 slot = slot_for_size (size);
4036 /*g_print ("using slot %d for size %d (slot size: %d)\n", slot, size, freelist_sizes [slot]);*/
4037 g_assert (size <= freelist_sizes [slot]);
4038 for (pchunk = pinned_chunk_list; pchunk; pchunk = pchunk->block.next) {
4039 void **p = pchunk->free_list [slot];
4041 /*g_print ("found freelist for slot %d in chunk %p, returning %p, next %p\n", slot, pchunk, p, *p);*/
4042 pchunk->free_list [slot] = *p;
4046 for (pchunk = pinned_chunk_list; pchunk; pchunk = pchunk->block.next) {
4047 res = get_chunk_freelist (pchunk, slot);
4051 pchunk = alloc_pinned_chunk ();
4052 /* FIXME: handle OOM */
4053 pchunk->block.next = pinned_chunk_list;
4054 pinned_chunk_list = pchunk;
4055 res = get_chunk_freelist (pchunk, slot);
4059 /* used for the GC-internal data structures */
4060 /* FIXME: add support for bigger sizes by allocating more than one page
4064 get_internal_mem (size_t size, int type)
4068 PinnedChunk *pchunk;
4070 if (size > freelist_sizes [FREELIST_NUM_SLOTS - 1]) {
4071 LargeInternalMemHeader *mh;
4073 size += sizeof (LargeInternalMemHeader);
4074 mh = get_os_memory (size, TRUE);
4075 mh->magic = LARGE_INTERNAL_MEM_HEADER_MAGIC;
4078 large_internal_bytes_alloced += size;
4083 slot = slot_for_size (size);
4084 g_assert (size <= freelist_sizes [slot]);
4086 small_internal_mem_bytes [type] += freelist_sizes [slot];
4088 for (pchunk = internal_chunk_list; pchunk; pchunk = pchunk->block.next) {
4089 void **p = pchunk->free_list [slot];
4091 pchunk->free_list [slot] = *p;
4092 memset (p, 0, size);
4096 for (pchunk = internal_chunk_list; pchunk; pchunk = pchunk->block.next) {
4097 res = get_chunk_freelist (pchunk, slot);
4099 memset (res, 0, size);
4103 pchunk = alloc_pinned_chunk ();
4104 /* FIXME: handle OOM */
4105 pchunk->block.next = internal_chunk_list;
4106 internal_chunk_list = pchunk;
4107 res = get_chunk_freelist (pchunk, slot);
4108 memset (res, 0, size);
4113 free_internal_mem (void *addr, int type)
4115 PinnedChunk *pchunk;
4116 LargeInternalMemHeader *mh;
4119 for (pchunk = internal_chunk_list; pchunk; pchunk = pchunk->block.next) {
4120 /*printf ("trying to free %p in %p (pages: %d)\n", addr, pchunk, pchunk->num_pages);*/
4121 if (addr >= (void*)pchunk && (char*)addr < (char*)pchunk + pchunk->num_pages * FREELIST_PAGESIZE) {
4122 int offset = (char*)addr - (char*)pchunk;
4123 int page = offset / FREELIST_PAGESIZE;
4124 int slot = slot_for_size (pchunk->page_sizes [page]);
4126 *p = pchunk->free_list [slot];
4127 pchunk->free_list [slot] = p;
4129 small_internal_mem_bytes [type] -= freelist_sizes [slot];
4134 mh = (LargeInternalMemHeader*)((char*)addr - G_STRUCT_OFFSET (LargeInternalMemHeader, data));
4135 g_assert (mh->magic == LARGE_INTERNAL_MEM_HEADER_MAGIC);
4136 large_internal_bytes_alloced -= mh->size;
4137 free_os_memory (mh, mh->size);
4141 * ######################################################################
4142 * ######## Object allocation
4143 * ######################################################################
4144 * This section of code deals with allocating memory for objects.
4145 * There are several ways:
4146 * *) allocate large objects
4147 * *) allocate normal objects
4148 * *) fast lock-free allocation
4149 * *) allocation of pinned objects
4153 free_large_object (LOSObject *obj)
4155 size_t size = obj->size;
4156 DEBUG (4, fprintf (gc_debug_file, "Freed large object %p, size %zd\n", obj->data, obj->size));
4158 los_memory_usage -= size;
4159 size += sizeof (LOSObject);
4160 size += pagesize - 1;
4161 size &= ~(pagesize - 1);
4162 total_alloc -= size;
4164 free_os_memory (obj, size);
4168 * Objects with size >= 64KB are allocated in the large object space.
4169 * They are currently kept track of with a linked list.
4170 * They don't move, so there is no need to pin them during collection
4171 * and we avoid the memcpy overhead.
4173 static void* __attribute__((noinline))
4174 alloc_large_inner (MonoVTable *vtable, size_t size)
4179 int just_did_major_gc = FALSE;
4181 g_assert (size > MAX_SMALL_OBJ_SIZE);
4183 if (los_memory_usage > next_los_collection) {
4184 DEBUG (4, fprintf (gc_debug_file, "Should trigger major collection: req size %zd (los already: %zu, limit: %zu)\n", size, los_memory_usage, next_los_collection));
4185 just_did_major_gc = TRUE;
4187 major_collection ("LOS overflow");
4189 /* later increase based on a percent of the heap size */
4190 next_los_collection = los_memory_usage + 5*1024*1024;
4193 alloc_size += sizeof (LOSObject);
4194 alloc_size += pagesize - 1;
4195 alloc_size &= ~(pagesize - 1);
4196 /* FIXME: handle OOM */
4197 obj = get_os_memory (alloc_size, TRUE);
4199 vtslot = (void**)obj->data;
4201 total_alloc += alloc_size;
4202 UPDATE_HEAP_BOUNDARIES (obj->data, (char*)obj->data + size);
4203 obj->next = los_object_list;
4204 los_object_list = obj;
4205 los_memory_usage += size;
4207 DEBUG (4, fprintf (gc_debug_file, "Allocated large object %p, vtable: %p (%s), size: %zd\n", obj->data, vtable, vtable->klass->name, size));
4211 /* check if we have a suitable fragment in nursery_fragments to be able to allocate
4212 * an object of size @size
4213 * Return FALSE if not found (which means we need a collection)
4216 search_fragment_for_size (size_t size)
4218 Fragment *frag, *prev;
4219 DEBUG (4, fprintf (gc_debug_file, "Searching nursery fragment %p, size: %zd\n", nursery_frag_real_end, size));
4221 if (nursery_frag_real_end > nursery_next && nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
4222 /* Clear the remaining space, pinning depends on this */
4223 memset (nursery_next, 0, nursery_frag_real_end - nursery_next);
4226 for (frag = nursery_fragments; frag; frag = frag->next) {
4227 if (size <= (frag->fragment_end - frag->fragment_start)) {
4228 /* remove from the list */
4230 prev->next = frag->next;
4232 nursery_fragments = frag->next;
4233 nursery_next = frag->fragment_start;
4234 nursery_frag_real_end = frag->fragment_end;
4236 DEBUG (4, fprintf (gc_debug_file, "Using nursery fragment %p-%p, size: %zd (req: %zd)\n", nursery_next, nursery_frag_real_end, nursery_frag_real_end - nursery_next, size));
4237 frag->next = fragment_freelist;
4238 fragment_freelist = frag;
4247 * size is already rounded up and we hold the GC lock.
4250 alloc_degraded (MonoVTable *vtable, size_t size)
4252 GCMemSection *section;
4254 g_assert (size <= MAX_SMALL_OBJ_SIZE);
4255 for (section = section_list; section; section = section->block.next) {
4256 if (section != nursery_section && (section->end_data - section->next_data) >= size) {
4257 p = (void**)section->next_data;
4262 section = alloc_major_section ();
4263 section->is_to_space = FALSE;
4264 /* FIXME: handle OOM */
4265 p = (void**)section->next_data;
4267 section->next_data += size;
4268 degraded_mode += size;
4269 DEBUG (3, fprintf (gc_debug_file, "Allocated (degraded) object %p, vtable: %p (%s), size: %zd in section %p\n", p, vtable, vtable->klass->name, size, section));
4275 * Provide a variant that takes just the vtable for small fixed-size objects.
4276 * The aligned size is already computed and stored in vt->gc_descr.
4277 * Note: every SCAN_START_SIZE or so we are given the chance to do some special
4278 * processing. We can keep track of where objects start, for example,
4279 * so when we scan the thread stacks for pinned objects, we can start
4280 * a search for the pinned object in SCAN_START_SIZE chunks.
4283 mono_gc_alloc_obj_nolock (MonoVTable *vtable, size_t size)
4285 /* FIXME: handle OOM */
4291 HEAVY_STAT (++stat_objects_alloced);
4293 size += ALLOC_ALIGN - 1;
4294 size &= ~(ALLOC_ALIGN - 1);
4296 g_assert (vtable->gc_descr);
4298 if (G_UNLIKELY (collect_before_allocs)) {
4299 if (nursery_section) {
4301 collect_nursery (0);
4303 if (!degraded_mode && !search_fragment_for_size (size)) {
4305 g_assert_not_reached ();
4311 * We must already have the lock here instead of after the
4312 * fast path because we might be interrupted in the fast path
4313 * (after confirming that new_next < TLAB_TEMP_END) by the GC,
4314 * and we'll end up allocating an object in a fragment which
4315 * no longer belongs to us.
4317 * The managed allocator does not do this, but it's treated
4318 * specially by the world-stopping code.
4321 if (size > MAX_SMALL_OBJ_SIZE) {
4322 p = alloc_large_inner (vtable, size);
4324 /* tlab_next and tlab_temp_end are TLS vars so accessing them might be expensive */
4326 p = (void**)TLAB_NEXT;
4327 /* FIXME: handle overflow */
4328 new_next = (char*)p + size;
4329 TLAB_NEXT = new_next;
4331 if (G_LIKELY (new_next < TLAB_TEMP_END)) {
4335 * FIXME: We might need a memory barrier here so the change to tlab_next is
4336 * visible before the vtable store.
4339 DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
4340 g_assert (*p == NULL);
4343 g_assert (TLAB_NEXT == new_next);
4350 /* there are two cases: the object is too big or we run out of space in the TLAB */
4351 /* we also reach here when the thread does its first allocation after a minor
4352 * collection, since the tlab_ variables are initialized to NULL.
4353 * there can be another case (from ORP), if we cooperate with the runtime a bit:
4354 * objects that need finalizers can have the high bit set in their size
4355 * so the above check fails and we can readily add the object to the queue.
4356 * This avoids taking again the GC lock when registering, but this is moot when
4357 * doing thread-local allocation, so it may not be a good idea.
4359 g_assert (TLAB_NEXT == new_next);
4360 if (TLAB_NEXT >= TLAB_REAL_END) {
4362 * Run out of space in the TLAB. When this happens, some amount of space
4363 * remains in the TLAB, but not enough to satisfy the current allocation
4364 * request. Currently, we retire the TLAB in all cases, later we could
4365 * keep it if the remaining space is above a treshold, and satisfy the
4366 * allocation directly from the nursery.
4369 /* when running in degraded mode, we continue allocing that way
4370 * for a while, to decrease the number of useless nursery collections.
4372 if (degraded_mode && degraded_mode < DEFAULT_NURSERY_SIZE) {
4373 p = alloc_degraded (vtable, size);
4377 if (size > tlab_size) {
4378 /* Allocate directly from the nursery */
4379 if (nursery_next + size >= nursery_frag_real_end) {
4380 if (!search_fragment_for_size (size)) {
4381 minor_collect_or_expand_inner (size);
4382 if (degraded_mode) {
4383 p = alloc_degraded (vtable, size);
4389 p = (void*)nursery_next;
4390 nursery_next += size;
4391 if (nursery_next > nursery_frag_real_end) {
4396 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
4397 memset (p, 0, size);
4400 DEBUG (3, fprintf (gc_debug_file, "Retire TLAB: %p-%p [%ld]\n", TLAB_START, TLAB_REAL_END, (long)(TLAB_REAL_END - TLAB_NEXT - size)));
4402 if (nursery_next + tlab_size >= nursery_frag_real_end) {
4403 res = search_fragment_for_size (tlab_size);
4405 minor_collect_or_expand_inner (tlab_size);
4406 if (degraded_mode) {
4407 p = alloc_degraded (vtable, size);
4413 /* Allocate a new TLAB from the current nursery fragment */
4414 TLAB_START = nursery_next;
4415 nursery_next += tlab_size;
4416 TLAB_NEXT = TLAB_START;
4417 TLAB_REAL_END = TLAB_START + tlab_size;
4418 TLAB_TEMP_END = TLAB_START + MIN (SCAN_START_SIZE, tlab_size);
4420 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
4421 memset (TLAB_START, 0, tlab_size);
4423 /* Allocate from the TLAB */
4424 p = (void*)TLAB_NEXT;
4426 g_assert (TLAB_NEXT <= TLAB_REAL_END);
4428 nursery_section->scan_starts [((char*)p - (char*)nursery_section->data)/SCAN_START_SIZE] = (char*)p;
4431 /* Reached tlab_temp_end */
4433 /* record the scan start so we can find pinned objects more easily */
4434 nursery_section->scan_starts [((char*)p - (char*)nursery_section->data)/SCAN_START_SIZE] = (char*)p;
4435 /* we just bump tlab_temp_end as well */
4436 TLAB_TEMP_END = MIN (TLAB_REAL_END, TLAB_NEXT + SCAN_START_SIZE);
4437 DEBUG (5, fprintf (gc_debug_file, "Expanding local alloc: %p-%p\n", TLAB_NEXT, TLAB_TEMP_END));
4441 DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
4448 mono_gc_alloc_obj (MonoVTable *vtable, size_t size)
4452 res = mono_gc_alloc_obj_nolock (vtable, size);
4458 mono_gc_alloc_vector (MonoVTable *vtable, size_t size, mono_array_size_t max_length)
4464 arr = mono_gc_alloc_obj_nolock (vtable, size);
4465 arr->max_length = max_length;
4473 mono_gc_alloc_array (MonoVTable *vtable, size_t size, mono_array_size_t max_length, mono_array_size_t bounds_size)
4476 MonoArrayBounds *bounds;
4480 arr = mono_gc_alloc_obj_nolock (vtable, size);
4481 arr->max_length = max_length;
4483 bounds = (MonoArrayBounds*)((char*)arr + size - bounds_size);
4484 arr->bounds = bounds;
4492 mono_gc_alloc_string (MonoVTable *vtable, size_t size, gint32 len)
4498 str = mono_gc_alloc_obj_nolock (vtable, size);
4507 * To be used for interned strings and possibly MonoThread, reflection handles.
4508 * We may want to explicitly free these objects.
4511 mono_gc_alloc_pinned_obj (MonoVTable *vtable, size_t size)
4513 /* FIXME: handle OOM */
4515 size += ALLOC_ALIGN - 1;
4516 size &= ~(ALLOC_ALIGN - 1);
4518 if (size > MAX_FREELIST_SIZE) {
4519 /* large objects are always pinned anyway */
4520 p = alloc_large_inner (vtable, size);
4522 p = alloc_from_freelist (size);
4523 memset (p, 0, size);
4525 DEBUG (6, fprintf (gc_debug_file, "Allocated pinned object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
4532 * ######################################################################
4533 * ######## Finalization support
4534 * ######################################################################
4538 * this is valid for the nursery: if the object has been forwarded it means it's
4539 * still refrenced from a root. If it is pinned it's still alive as well.
4540 * Return TRUE if @obj is ready to be finalized.
4542 #define object_is_fin_ready(obj) (!object_is_pinned (obj) && !object_is_forwarded (obj))
4545 is_critical_finalizer (FinalizeEntry *entry)
4550 if (!mono_defaults.critical_finalizer_object)
4553 obj = entry->object;
4554 class = ((MonoVTable*)LOAD_VTABLE (obj))->klass;
4556 return mono_class_has_parent (class, mono_defaults.critical_finalizer_object);
4560 queue_finalization_entry (FinalizeEntry *entry) {
4561 if (is_critical_finalizer (entry)) {
4562 entry->next = critical_fin_list;
4563 critical_fin_list = entry;
4565 entry->next = fin_ready_list;
4566 fin_ready_list = entry;
4570 /* LOCKING: requires that the GC lock is held */
4572 rehash_fin_table (FinalizeEntryHashTable *hash_table)
4574 FinalizeEntry **finalizable_hash = hash_table->table;
4575 mword finalizable_hash_size = hash_table->size;
4578 FinalizeEntry **new_hash;
4579 FinalizeEntry *entry, *next;
4580 int new_size = g_spaced_primes_closest (hash_table->num_registered);
4582 new_hash = get_internal_mem (new_size * sizeof (FinalizeEntry*), INTERNAL_MEM_FIN_TABLE);
4583 for (i = 0; i < finalizable_hash_size; ++i) {
4584 for (entry = finalizable_hash [i]; entry; entry = next) {
4585 hash = mono_object_hash (entry->object) % new_size;
4587 entry->next = new_hash [hash];
4588 new_hash [hash] = entry;
4591 free_internal_mem (finalizable_hash, INTERNAL_MEM_FIN_TABLE);
4592 hash_table->table = new_hash;
4593 hash_table->size = new_size;
4596 /* LOCKING: requires that the GC lock is held */
4598 rehash_fin_table_if_necessary (FinalizeEntryHashTable *hash_table)
4600 if (hash_table->num_registered >= hash_table->size * 2)
4601 rehash_fin_table (hash_table);
4604 /* LOCKING: requires that the GC lock is held */
4606 finalize_in_range (char *start, char *end, int generation)
4608 FinalizeEntryHashTable *hash_table = get_finalize_entry_hash_table (generation);
4609 FinalizeEntry *entry, *prev;
4611 FinalizeEntry **finalizable_hash = hash_table->table;
4612 mword finalizable_hash_size = hash_table->size;
4616 for (i = 0; i < finalizable_hash_size; ++i) {
4618 for (entry = finalizable_hash [i]; entry;) {
4619 if ((char*)entry->object >= start && (char*)entry->object < end && !object_is_in_to_space (entry->object)) {
4620 gboolean is_fin_ready = object_is_fin_ready (entry->object);
4621 char *copy = copy_object (entry->object, start, end);
4624 FinalizeEntry *next;
4625 /* remove and put in fin_ready_list */
4627 prev->next = entry->next;
4629 finalizable_hash [i] = entry->next;
4631 num_ready_finalizers++;
4632 hash_table->num_registered--;
4633 queue_finalization_entry (entry);
4634 /* Make it survive */
4635 from = entry->object;
4636 entry->object = copy;
4637 DEBUG (5, fprintf (gc_debug_file, "Queueing object for finalization: %p (%s) (was at %p) (%d/%d)\n", entry->object, safe_name (entry->object), from, num_ready_finalizers, hash_table->num_registered));
4641 char *from = entry->object;
4642 if (hash_table == &minor_finalizable_hash && !ptr_in_nursery (copy)) {
4643 FinalizeEntry *next = entry->next;
4644 unsigned int major_hash;
4645 /* remove from the list */
4647 prev->next = entry->next;
4649 finalizable_hash [i] = entry->next;
4650 hash_table->num_registered--;
4652 entry->object = copy;
4654 /* insert it into the major hash */
4655 rehash_fin_table_if_necessary (&major_finalizable_hash);
4656 major_hash = mono_object_hash ((MonoObject*) copy) %
4657 major_finalizable_hash.size;
4658 entry->next = major_finalizable_hash.table [major_hash];
4659 major_finalizable_hash.table [major_hash] = entry;
4660 major_finalizable_hash.num_registered++;
4662 DEBUG (5, fprintf (gc_debug_file, "Promoting finalization of object %p (%s) (was at %p) to major table\n", copy, safe_name (copy), from));
4667 /* update pointer */
4668 DEBUG (5, fprintf (gc_debug_file, "Updating object for finalization: %p (%s) (was at %p)\n", entry->object, safe_name (entry->object), from));
4669 entry->object = copy;
4674 entry = entry->next;
4679 /* LOCKING: requires that the GC lock is held */
4681 null_link_in_range (char *start, char *end, int generation)
4683 DisappearingLinkHashTable *hash = get_dislink_hash_table (generation);
4684 DisappearingLink **disappearing_link_hash = hash->table;
4685 int disappearing_link_hash_size = hash->size;
4686 DisappearingLink *entry, *prev;
4688 if (!hash->num_links)
4690 for (i = 0; i < disappearing_link_hash_size; ++i) {
4692 for (entry = disappearing_link_hash [i]; entry;) {
4693 char *object = DISLINK_OBJECT (entry);
4694 if (object >= start && object < end && !object_is_in_to_space (object)) {
4695 gboolean track = DISLINK_TRACK (entry);
4696 if (!track && object_is_fin_ready (object)) {
4697 void **p = entry->link;
4698 DisappearingLink *old;
4700 /* remove from list */
4702 prev->next = entry->next;
4704 disappearing_link_hash [i] = entry->next;
4705 DEBUG (5, fprintf (gc_debug_file, "Dislink nullified at %p to GCed object %p\n", p, object));
4707 free_internal_mem (entry, INTERNAL_MEM_DISLINK);
4712 char *copy = copy_object (object, start, end);
4714 /* Update pointer if it's moved. If the object
4715 * has been moved out of the nursery, we need to
4716 * remove the link from the minor hash table to
4719 * FIXME: what if an object is moved earlier?
4722 if (hash == &minor_disappearing_link_hash && !ptr_in_nursery (copy)) {
4723 void **link = entry->link;
4724 DisappearingLink *old;
4725 /* remove from list */
4727 prev->next = entry->next;
4729 disappearing_link_hash [i] = entry->next;
4731 free_internal_mem (entry, INTERNAL_MEM_DISLINK);
4735 add_or_remove_disappearing_link ((MonoObject*)copy, link,
4736 track, GENERATION_OLD);
4738 DEBUG (5, fprintf (gc_debug_file, "Upgraded dislink at %p to major because object %p moved to %p\n", link, object, copy));
4742 /* We set the track resurrection bit to
4743 * FALSE if the object is to be finalized
4744 * so that the object can be collected in
4745 * the next cycle (i.e. after it was
4748 *entry->link = HIDE_POINTER (copy,
4749 object_is_fin_ready (object) ? FALSE : track);
4750 DEBUG (5, fprintf (gc_debug_file, "Updated dislink at %p to %p\n", entry->link, DISLINK_OBJECT (entry)));
4755 entry = entry->next;
4760 /* LOCKING: requires that the GC lock is held */
4762 null_links_for_domain (MonoDomain *domain, int generation)
4764 DisappearingLinkHashTable *hash = get_dislink_hash_table (generation);
4765 DisappearingLink **disappearing_link_hash = hash->table;
4766 int disappearing_link_hash_size = hash->size;
4767 DisappearingLink *entry, *prev;
4769 for (i = 0; i < disappearing_link_hash_size; ++i) {
4771 for (entry = disappearing_link_hash [i]; entry; ) {
4772 char *object = DISLINK_OBJECT (entry);
4773 /* FIXME: actually there should be no object
4774 left in the domain with a non-null vtable
4775 (provided we remove the Thread special
4777 if (object && (!((MonoObject*)object)->vtable || mono_object_domain (object) == domain)) {
4778 DisappearingLink *next = entry->next;
4783 disappearing_link_hash [i] = next;
4785 if (*(entry->link)) {
4786 *(entry->link) = NULL;
4787 g_warning ("Disappearing link %p not freed", entry->link);
4789 free_internal_mem (entry, INTERNAL_MEM_DISLINK);
4796 entry = entry->next;
4801 /* LOCKING: requires that the GC lock is held */
4803 finalizers_for_domain (MonoDomain *domain, MonoObject **out_array, int out_size,
4804 FinalizeEntryHashTable *hash_table)
4806 FinalizeEntry **finalizable_hash = hash_table->table;
4807 mword finalizable_hash_size = hash_table->size;
4808 FinalizeEntry *entry, *prev;
4811 if (no_finalize || !out_size || !out_array)
4814 for (i = 0; i < finalizable_hash_size; ++i) {
4816 for (entry = finalizable_hash [i]; entry;) {
4817 if (mono_object_domain (entry->object) == domain) {
4818 FinalizeEntry *next;
4819 /* remove and put in out_array */
4821 prev->next = entry->next;
4823 finalizable_hash [i] = entry->next;
4825 hash_table->num_registered--;
4826 out_array [count ++] = entry->object;
4827 DEBUG (5, fprintf (gc_debug_file, "Collecting object for finalization: %p (%s) (%d/%d)\n", entry->object, safe_name (entry->object), num_ready_finalizers, hash_table->num_registered));
4829 if (count == out_size)
4834 entry = entry->next;
4841 * mono_gc_finalizers_for_domain:
4842 * @domain: the unloading appdomain
4843 * @out_array: output array
4844 * @out_size: size of output array
4846 * Store inside @out_array up to @out_size objects that belong to the unloading
4847 * appdomain @domain. Returns the number of stored items. Can be called repeteadly
4848 * until it returns 0.
4849 * The items are removed from the finalizer data structure, so the caller is supposed
4851 * @out_array should be on the stack to allow the GC to know the objects are still alive.
4854 mono_gc_finalizers_for_domain (MonoDomain *domain, MonoObject **out_array, int out_size)
4859 result = finalizers_for_domain (domain, out_array, out_size, &minor_finalizable_hash);
4860 if (result < out_size) {
4861 result += finalizers_for_domain (domain, out_array + result, out_size - result,
4862 &major_finalizable_hash);
4870 register_for_finalization (MonoObject *obj, void *user_data, int generation)
4872 FinalizeEntryHashTable *hash_table = get_finalize_entry_hash_table (generation);
4873 FinalizeEntry **finalizable_hash;
4874 mword finalizable_hash_size;
4875 FinalizeEntry *entry, *prev;
4879 g_assert (user_data == NULL || user_data == mono_gc_run_finalize);
4880 hash = mono_object_hash (obj);
4882 rehash_fin_table_if_necessary (hash_table);
4883 finalizable_hash = hash_table->table;
4884 finalizable_hash_size = hash_table->size;
4885 hash %= finalizable_hash_size;
4887 for (entry = finalizable_hash [hash]; entry; entry = entry->next) {
4888 if (entry->object == obj) {
4890 /* remove from the list */
4892 prev->next = entry->next;
4894 finalizable_hash [hash] = entry->next;
4895 hash_table->num_registered--;
4896 DEBUG (5, fprintf (gc_debug_file, "Removed finalizer %p for object: %p (%s) (%d)\n", entry, obj, obj->vtable->klass->name, hash_table->num_registered));
4897 free_internal_mem (entry, INTERNAL_MEM_FINALIZE_ENTRY);
4905 /* request to deregister, but already out of the list */
4909 entry = get_internal_mem (sizeof (FinalizeEntry), INTERNAL_MEM_FINALIZE_ENTRY);
4910 entry->object = obj;
4911 entry->next = finalizable_hash [hash];
4912 finalizable_hash [hash] = entry;
4913 hash_table->num_registered++;
4914 DEBUG (5, fprintf (gc_debug_file, "Added finalizer %p for object: %p (%s) (%d) to %s table\n", entry, obj, obj->vtable->klass->name, hash_table->num_registered, generation_name (generation)));
4919 mono_gc_register_for_finalization (MonoObject *obj, void *user_data)
4921 if (ptr_in_nursery (obj))
4922 register_for_finalization (obj, user_data, GENERATION_NURSERY);
4924 register_for_finalization (obj, user_data, GENERATION_OLD);
4928 rehash_dislink (DisappearingLinkHashTable *hash_table)
4930 DisappearingLink **disappearing_link_hash = hash_table->table;
4931 int disappearing_link_hash_size = hash_table->size;
4934 DisappearingLink **new_hash;
4935 DisappearingLink *entry, *next;
4936 int new_size = g_spaced_primes_closest (hash_table->num_links);
4938 new_hash = get_internal_mem (new_size * sizeof (DisappearingLink*), INTERNAL_MEM_DISLINK_TABLE);
4939 for (i = 0; i < disappearing_link_hash_size; ++i) {
4940 for (entry = disappearing_link_hash [i]; entry; entry = next) {
4941 hash = mono_aligned_addr_hash (entry->link) % new_size;
4943 entry->next = new_hash [hash];
4944 new_hash [hash] = entry;
4947 free_internal_mem (disappearing_link_hash, INTERNAL_MEM_DISLINK_TABLE);
4948 hash_table->table = new_hash;
4949 hash_table->size = new_size;
4952 /* LOCKING: assumes the GC lock is held */
4954 add_or_remove_disappearing_link (MonoObject *obj, void **link, gboolean track, int generation)
4956 DisappearingLinkHashTable *hash_table = get_dislink_hash_table (generation);
4957 DisappearingLink *entry, *prev;
4959 DisappearingLink **disappearing_link_hash = hash_table->table;
4960 int disappearing_link_hash_size = hash_table->size;
4962 if (hash_table->num_links >= disappearing_link_hash_size * 2) {
4963 rehash_dislink (hash_table);
4964 disappearing_link_hash = hash_table->table;
4965 disappearing_link_hash_size = hash_table->size;
4967 /* FIXME: add check that link is not in the heap */
4968 hash = mono_aligned_addr_hash (link) % disappearing_link_hash_size;
4969 entry = disappearing_link_hash [hash];
4971 for (; entry; entry = entry->next) {
4972 /* link already added */
4973 if (link == entry->link) {
4974 /* NULL obj means remove */
4977 prev->next = entry->next;
4979 disappearing_link_hash [hash] = entry->next;
4980 hash_table->num_links--;
4981 DEBUG (5, fprintf (gc_debug_file, "Removed dislink %p (%d) from %s table\n", entry, hash_table->num_links, generation_name (generation)));
4982 free_internal_mem (entry, INTERNAL_MEM_DISLINK);
4985 *link = HIDE_POINTER (obj, track); /* we allow the change of object */
4993 entry = get_internal_mem (sizeof (DisappearingLink), INTERNAL_MEM_DISLINK);
4994 *link = HIDE_POINTER (obj, track);
4996 entry->next = disappearing_link_hash [hash];
4997 disappearing_link_hash [hash] = entry;
4998 hash_table->num_links++;
4999 DEBUG (5, fprintf (gc_debug_file, "Added dislink %p for object: %p (%s) at %p to %s table\n", entry, obj, obj->vtable->klass->name, link, generation_name (generation)));
5002 /* LOCKING: assumes the GC lock is held */
5004 mono_gc_register_disappearing_link (MonoObject *obj, void **link, gboolean track)
5006 add_or_remove_disappearing_link (NULL, link, FALSE, GENERATION_NURSERY);
5007 add_or_remove_disappearing_link (NULL, link, FALSE, GENERATION_OLD);
5009 if (ptr_in_nursery (obj))
5010 add_or_remove_disappearing_link (obj, link, track, GENERATION_NURSERY);
5012 add_or_remove_disappearing_link (obj, link, track, GENERATION_OLD);
5017 mono_gc_invoke_finalizers (void)
5019 FinalizeEntry *entry = NULL;
5020 gboolean entry_is_critical;
5023 /* FIXME: batch to reduce lock contention */
5024 while (fin_ready_list || critical_fin_list) {
5028 FinalizeEntry **list = entry_is_critical ? &critical_fin_list : &fin_ready_list;
5030 /* We have finalized entry in the last
5031 interation, now we need to remove it from
5034 *list = entry->next;
5036 FinalizeEntry *e = *list;
5037 while (e->next != entry)
5039 e->next = entry->next;
5041 free_internal_mem (entry, INTERNAL_MEM_FINALIZE_ENTRY);
5045 /* Now look for the first non-null entry. */
5046 for (entry = fin_ready_list; entry && !entry->object; entry = entry->next)
5049 entry_is_critical = FALSE;
5051 entry_is_critical = TRUE;
5052 for (entry = critical_fin_list; entry && !entry->object; entry = entry->next)
5057 g_assert (entry->object);
5058 num_ready_finalizers--;
5059 obj = entry->object;
5060 entry->object = NULL;
5061 DEBUG (7, fprintf (gc_debug_file, "Finalizing object %p (%s)\n", obj, safe_name (obj)));
5069 g_assert (entry->object == NULL);
5071 /* the object is on the stack so it is pinned */
5072 /*g_print ("Calling finalizer for object: %p (%s)\n", entry->object, safe_name (entry->object));*/
5073 mono_gc_run_finalize (obj, NULL);
5080 mono_gc_pending_finalizers (void)
5082 return fin_ready_list || critical_fin_list;
5085 /* Negative value to remove */
5087 mono_gc_add_memory_pressure (gint64 value)
5089 /* FIXME: Use interlocked functions */
5091 memory_pressure += value;
5096 * ######################################################################
5097 * ######## registered roots support
5098 * ######################################################################
5102 rehash_roots (gboolean pinned)
5106 RootRecord **new_hash;
5107 RootRecord *entry, *next;
5110 new_size = g_spaced_primes_closest (num_roots_entries [pinned]);
5111 new_hash = get_internal_mem (new_size * sizeof (RootRecord*), INTERNAL_MEM_ROOTS_TABLE);
5112 for (i = 0; i < roots_hash_size [pinned]; ++i) {
5113 for (entry = roots_hash [pinned][i]; entry; entry = next) {
5114 hash = mono_aligned_addr_hash (entry->start_root) % new_size;
5116 entry->next = new_hash [hash];
5117 new_hash [hash] = entry;
5120 free_internal_mem (roots_hash [pinned], INTERNAL_MEM_ROOTS_TABLE);
5121 roots_hash [pinned] = new_hash;
5122 roots_hash_size [pinned] = new_size;
5126 find_root (int root_type, char *start, guint32 addr_hash)
5128 RootRecord *new_root;
5130 guint32 hash = addr_hash % roots_hash_size [root_type];
5131 for (new_root = roots_hash [root_type][hash]; new_root; new_root = new_root->next) {
5132 /* we allow changing the size and the descriptor (for thread statics etc) */
5133 if (new_root->start_root == start) {
5142 * We do not coalesce roots.
5145 mono_gc_register_root_inner (char *start, size_t size, void *descr, int root_type)
5147 RootRecord *new_root;
5148 unsigned int hash, addr_hash = mono_aligned_addr_hash (start);
5151 for (i = 0; i < ROOT_TYPE_NUM; ++i) {
5152 if (num_roots_entries [i] >= roots_hash_size [i] * 2)
5155 for (i = 0; i < ROOT_TYPE_NUM; ++i) {
5156 new_root = find_root (i, start, addr_hash);
5157 /* we allow changing the size and the descriptor (for thread statics etc) */
5159 size_t old_size = new_root->end_root - new_root->start_root;
5160 new_root->end_root = new_root->start_root + size;
5161 g_assert (((new_root->root_desc != 0) && (descr != NULL)) ||
5162 ((new_root->root_desc == 0) && (descr == NULL)));
5163 new_root->root_desc = (mword)descr;
5165 roots_size -= old_size;
5170 new_root = get_internal_mem (sizeof (RootRecord), INTERNAL_MEM_ROOT_RECORD);
5172 new_root->start_root = start;
5173 new_root->end_root = new_root->start_root + size;
5174 new_root->root_desc = (mword)descr;
5176 hash = addr_hash % roots_hash_size [root_type];
5177 num_roots_entries [root_type]++;
5178 new_root->next = roots_hash [root_type] [hash];
5179 roots_hash [root_type][hash] = new_root;
5180 DEBUG (3, fprintf (gc_debug_file, "Added root %p for range: %p-%p, descr: %p (%d/%d bytes)\n", new_root, new_root->start_root, new_root->end_root, descr, (int)size, (int)roots_size));
5190 mono_gc_register_root (char *start, size_t size, void *descr)
5192 return mono_gc_register_root_inner (start, size, descr, descr ? ROOT_TYPE_NORMAL : ROOT_TYPE_PINNED);
5196 mono_gc_register_root_wbarrier (char *start, size_t size, void *descr)
5198 return mono_gc_register_root_inner (start, size, descr, ROOT_TYPE_WBARRIER);
5202 mono_gc_deregister_root (char* addr)
5204 RootRecord *tmp, *prev;
5205 unsigned int hash, addr_hash = mono_aligned_addr_hash (addr);
5209 for (root_type = 0; root_type < ROOT_TYPE_NUM; ++root_type) {
5210 hash = addr_hash % roots_hash_size [root_type];
5211 tmp = roots_hash [root_type][hash];
5214 if (tmp->start_root == (char*)addr) {
5216 prev->next = tmp->next;
5218 roots_hash [root_type][hash] = tmp->next;
5219 roots_size -= (tmp->end_root - tmp->start_root);
5220 num_roots_entries [root_type]--;
5221 DEBUG (3, fprintf (gc_debug_file, "Removed root %p for range: %p-%p\n", tmp, tmp->start_root, tmp->end_root));
5222 free_internal_mem (tmp, INTERNAL_MEM_ROOT_RECORD);
5233 * ######################################################################
5234 * ######## Thread handling (stop/start code)
5235 * ######################################################################
5238 /* FIXME: handle large/small config */
5239 #define THREAD_HASH_SIZE 11
5240 #define HASH_PTHREAD_T(id) (((unsigned int)(id) >> 4) * 2654435761u)
5242 static SgenThreadInfo* thread_table [THREAD_HASH_SIZE];
5244 #if USE_SIGNAL_BASED_START_STOP_WORLD
5246 static MonoSemType suspend_ack_semaphore;
5247 static MonoSemType *suspend_ack_semaphore_ptr;
5248 static unsigned int global_stop_count = 0;
5250 static int suspend_signal_num = SIGXFSZ;
5252 static int suspend_signal_num = SIGPWR;
5254 static int restart_signal_num = SIGXCPU;
5255 static sigset_t suspend_signal_mask;
5256 static mword cur_thread_regs [ARCH_NUM_REGS] = {0};
5258 /* LOCKING: assumes the GC lock is held */
5259 static SgenThreadInfo*
5260 thread_info_lookup (ARCH_THREAD_TYPE id)
5262 unsigned int hash = HASH_PTHREAD_T (id) % THREAD_HASH_SIZE;
5263 SgenThreadInfo *info;
5265 info = thread_table [hash];
5266 while (info && !ARCH_THREAD_EQUALS (info->id, id)) {
5273 update_current_thread_stack (void *start)
5275 void *ptr = cur_thread_regs;
5276 SgenThreadInfo *info = thread_info_lookup (ARCH_GET_THREAD ());
5277 info->stack_start = align_pointer (&ptr);
5278 g_assert (info->stack_start >= info->stack_start_limit && info->stack_start < info->stack_end);
5279 ARCH_STORE_REGS (ptr);
5280 info->stopped_regs = ptr;
5281 if (gc_callbacks.thread_suspend_func)
5282 gc_callbacks.thread_suspend_func (info->runtime_data, NULL);
5286 signal_desc (int signum)
5288 if (signum == suspend_signal_num)
5290 if (signum == restart_signal_num)
5296 * Define this and use the "xdomain-checks" MONO_GC_DEBUG option to
5297 * have cross-domain checks in the write barrier.
5299 //#define XDOMAIN_CHECKS_IN_WBARRIER
5301 #ifndef HEAVY_STATISTICS
5302 #define MANAGED_ALLOCATION
5303 #ifndef XDOMAIN_CHECKS_IN_WBARRIER
5304 #define MANAGED_WBARRIER
5309 is_ip_in_managed_allocator (MonoDomain *domain, gpointer ip);
5312 wait_for_suspend_ack (int count)
5316 for (i = 0; i < count; ++i) {
5317 while ((result = MONO_SEM_WAIT (suspend_ack_semaphore_ptr)) != 0) {
5318 if (errno != EINTR) {
5319 g_error ("sem_wait ()");
5325 /* LOCKING: assumes the GC lock is held */
5327 thread_handshake (int signum)
5329 int count, i, result;
5330 SgenThreadInfo *info;
5331 pthread_t me = pthread_self ();
5334 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5335 for (info = thread_table [i]; info; info = info->next) {
5336 DEBUG (4, fprintf (gc_debug_file, "considering thread %p for signal %d (%s)\n", info, signum, signal_desc (signum)));
5337 if (ARCH_THREAD_EQUALS (info->id, me)) {
5338 DEBUG (4, fprintf (gc_debug_file, "Skip (equal): %p, %p\n", (void*)me, (void*)info->id));
5341 /*if (signum == suspend_signal_num && info->stop_count == global_stop_count)
5343 result = pthread_kill (info->id, signum);
5345 DEBUG (4, fprintf (gc_debug_file, "thread %p signal sent\n", info));
5348 DEBUG (4, fprintf (gc_debug_file, "thread %p signal failed: %d (%s)\n", (void*)info->id, result, strerror (result)));
5354 wait_for_suspend_ack (count);
5360 restart_threads_until_none_in_managed_allocator (void)
5362 SgenThreadInfo *info;
5363 int i, result, num_threads_died = 0;
5364 int sleep_duration = -1;
5367 int restart_count = 0, restarted_count = 0;
5368 /* restart all threads that stopped in the
5370 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5371 for (info = thread_table [i]; info; info = info->next) {
5374 if (!info->stack_start ||
5375 is_ip_in_managed_allocator (info->stopped_domain, info->stopped_ip)) {
5376 result = pthread_kill (info->id, restart_signal_num);
5383 /* we set the stopped_ip to
5384 NULL for threads which
5385 we're not restarting so
5386 that we can easily identify
5388 info->stopped_ip = NULL;
5389 info->stopped_domain = NULL;
5393 /* if no threads were restarted, we're done */
5394 if (restart_count == 0)
5397 /* wait for the threads to signal their restart */
5398 wait_for_suspend_ack (restart_count);
5400 if (sleep_duration < 0) {
5404 g_usleep (sleep_duration);
5405 sleep_duration += 10;
5408 /* stop them again */
5409 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5410 for (info = thread_table [i]; info; info = info->next) {
5411 if (info->skip || info->stopped_ip == NULL)
5413 result = pthread_kill (info->id, suspend_signal_num);
5421 /* some threads might have died */
5422 num_threads_died += restart_count - restarted_count;
5423 /* wait for the threads to signal their suspension
5425 wait_for_suspend_ack (restart_count);
5428 return num_threads_died;
5431 /* LOCKING: assumes the GC lock is held (by the stopping thread) */
5433 suspend_handler (int sig, siginfo_t *siginfo, void *context)
5435 SgenThreadInfo *info;
5438 int old_errno = errno;
5439 gpointer regs [ARCH_NUM_REGS];
5440 gpointer stack_start;
5442 id = pthread_self ();
5443 info = thread_info_lookup (id);
5444 info->stopped_domain = mono_domain_get ();
5445 info->stopped_ip = (gpointer) ARCH_SIGCTX_IP (context);
5446 stop_count = global_stop_count;
5447 /* duplicate signal */
5448 if (0 && info->stop_count == stop_count) {
5452 #ifdef HAVE_KW_THREAD
5453 /* update the remset info in the thread data structure */
5454 info->remset = remembered_set;
5456 stack_start = (char*) ARCH_SIGCTX_SP (context) - REDZONE_SIZE;
5457 /* If stack_start is not within the limits, then don't set it
5458 in info and we will be restarted. */
5459 if (stack_start >= info->stack_start_limit && info->stack_start <= info->stack_end) {
5460 info->stack_start = stack_start;
5462 ARCH_COPY_SIGCTX_REGS (regs, context);
5463 info->stopped_regs = regs;
5465 g_assert (!info->stack_start);
5468 /* Notify the JIT */
5469 if (gc_callbacks.thread_suspend_func)
5470 gc_callbacks.thread_suspend_func (info->runtime_data, context);
5472 /* notify the waiting thread */
5473 MONO_SEM_POST (suspend_ack_semaphore_ptr);
5474 info->stop_count = stop_count;
5476 /* wait until we receive the restart signal */
5479 sigsuspend (&suspend_signal_mask);
5480 } while (info->signal != restart_signal_num);
5482 /* notify the waiting thread */
5483 MONO_SEM_POST (suspend_ack_semaphore_ptr);
5489 restart_handler (int sig)
5491 SgenThreadInfo *info;
5492 int old_errno = errno;
5494 info = thread_info_lookup (pthread_self ());
5495 info->signal = restart_signal_num;
5500 static TV_DECLARE (stop_world_time);
5501 static unsigned long max_pause_usec = 0;
5503 /* LOCKING: assumes the GC lock is held */
5509 update_current_thread_stack (&count);
5511 global_stop_count++;
5512 DEBUG (3, fprintf (gc_debug_file, "stopping world n %d from %p %p\n", global_stop_count, thread_info_lookup (ARCH_GET_THREAD ()), (gpointer)ARCH_GET_THREAD ()));
5513 TV_GETTIME (stop_world_time);
5514 count = thread_handshake (suspend_signal_num);
5515 count -= restart_threads_until_none_in_managed_allocator ();
5516 g_assert (count >= 0);
5517 DEBUG (3, fprintf (gc_debug_file, "world stopped %d thread(s)\n", count));
5521 /* LOCKING: assumes the GC lock is held */
5523 restart_world (void)
5526 SgenThreadInfo *info;
5527 TV_DECLARE (end_sw);
5530 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5531 for (info = thread_table [i]; info; info = info->next) {
5532 info->stack_start = NULL;
5533 info->stopped_regs = NULL;
5537 count = thread_handshake (restart_signal_num);
5538 TV_GETTIME (end_sw);
5539 usec = TV_ELAPSED (stop_world_time, end_sw);
5540 max_pause_usec = MAX (usec, max_pause_usec);
5541 DEBUG (2, fprintf (gc_debug_file, "restarted %d thread(s) (pause time: %d usec, max: %d)\n", count, (int)usec, (int)max_pause_usec));
5545 #endif /* USE_SIGNAL_BASED_START_STOP_WORLD */
5548 mono_gc_set_gc_callbacks (MonoGCCallbacks *callbacks)
5550 gc_callbacks = *callbacks;
5553 /* Variables holding start/end nursery so it won't have to be passed at every call */
5554 static void *scan_area_arg_start, *scan_area_arg_end;
5557 mono_gc_conservatively_scan_area (void *start, void *end)
5559 conservatively_pin_objects_from (start, end, scan_area_arg_start, scan_area_arg_end, PIN_TYPE_STACK);
5563 mono_gc_scan_object (void *obj)
5565 return copy_object (obj, scan_area_arg_start, scan_area_arg_end);
5569 * Mark from thread stacks and registers.
5572 scan_thread_data (void *start_nursery, void *end_nursery, gboolean precise)
5575 SgenThreadInfo *info;
5577 scan_area_arg_start = start_nursery;
5578 scan_area_arg_end = end_nursery;
5580 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5581 for (info = thread_table [i]; info; info = info->next) {
5583 DEBUG (2, fprintf (gc_debug_file, "Skipping dead thread %p, range: %p-%p, size: %zd\n", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start));
5586 DEBUG (3, fprintf (gc_debug_file, "Scanning thread %p, range: %p-%p, size: %zd, pinned=%d\n", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start, next_pin_slot));
5587 if (gc_callbacks.thread_mark_func && !conservative_stack_mark)
5588 gc_callbacks.thread_mark_func (info->runtime_data, info->stack_start, info->stack_end, precise);
5590 conservatively_pin_objects_from (info->stack_start, info->stack_end, start_nursery, end_nursery, PIN_TYPE_STACK);
5593 conservatively_pin_objects_from (info->stopped_regs, info->stopped_regs + ARCH_NUM_REGS,
5594 start_nursery, end_nursery, PIN_TYPE_STACK);
5600 find_pinning_ref_from_thread (char *obj, size_t size)
5603 SgenThreadInfo *info;
5604 char *endobj = obj + size;
5606 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5607 for (info = thread_table [i]; info; info = info->next) {
5608 char **start = (char**)info->stack_start;
5611 while (start < (char**)info->stack_end) {
5612 if (*start >= obj && *start < endobj) {
5613 DEBUG (0, fprintf (gc_debug_file, "Object %p referenced in thread %p (id %p) at %p, stack: %p-%p\n", obj, info, (gpointer)info->id, start, info->stack_start, info->stack_end));
5618 /* FIXME: check info->stopped_regs */
5624 ptr_on_stack (void *ptr)
5626 gpointer stack_start = &stack_start;
5627 SgenThreadInfo *info = thread_info_lookup (ARCH_GET_THREAD ());
5629 if (ptr >= stack_start && ptr < (gpointer)info->stack_end)
5635 handle_remset (mword *p, void *start_nursery, void *end_nursery, gboolean global)
5642 HEAVY_STAT (++stat_global_remsets_processed);
5644 /* FIXME: exclude stack locations */
5645 switch ((*p) & REMSET_TYPE_MASK) {
5646 case REMSET_LOCATION:
5648 //__builtin_prefetch (ptr);
5649 if (((void*)ptr < start_nursery || (void*)ptr >= end_nursery)) {
5650 *ptr = copy_object (*ptr, start_nursery, end_nursery);
5651 DEBUG (9, fprintf (gc_debug_file, "Overwrote remset at %p with %p\n", ptr, *ptr));
5652 if (!global && *ptr >= start_nursery && *ptr < end_nursery) {
5654 * If the object is pinned, each reference to it from nonpinned objects
5655 * becomes part of the global remset, which can grow very large.
5657 DEBUG (9, fprintf (gc_debug_file, "Add to global remset because of pinning %p (%p %s)\n", ptr, *ptr, safe_name (*ptr)));
5658 add_to_global_remset (ptr, FALSE);
5661 DEBUG (9, fprintf (gc_debug_file, "Skipping remset at %p holding %p\n", ptr, *ptr));
5665 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
5666 if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery))
5669 while (count-- > 0) {
5670 *ptr = copy_object (*ptr, start_nursery, end_nursery);
5671 DEBUG (9, fprintf (gc_debug_file, "Overwrote remset at %p with %p (count: %d)\n", ptr, *ptr, (int)count));
5672 if (!global && *ptr >= start_nursery && *ptr < end_nursery)
5673 add_to_global_remset (ptr, FALSE);
5678 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
5679 if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery))
5681 scan_object ((char*)ptr, start_nursery, end_nursery);
5683 case REMSET_OTHER: {
5684 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
5688 if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery))
5693 ptr = (void**) scan_vtype ((char*)ptr, desc, start_nursery, end_nursery);
5695 case REMSET_ROOT_LOCATION:
5696 /* Same as REMSET_LOCATION, but the address is not required to be in the heap */
5697 *ptr = copy_object (*ptr, start_nursery, end_nursery);
5698 DEBUG (9, fprintf (gc_debug_file, "Overwrote root location remset at %p with %p\n", ptr, *ptr));
5699 if (!global && *ptr >= start_nursery && *ptr < end_nursery) {
5701 * If the object is pinned, each reference to it from nonpinned objects
5702 * becomes part of the global remset, which can grow very large.
5704 DEBUG (9, fprintf (gc_debug_file, "Add to global remset because of pinning %p (%p %s)\n", ptr, *ptr, safe_name (*ptr)));
5705 add_to_global_remset (ptr, TRUE);
5709 g_assert_not_reached ();
5714 g_assert_not_reached ();
5719 #ifdef HEAVY_STATISTICS
5721 collect_store_remsets (RememberedSet *remset, mword *bumper)
5723 mword *p = remset->data;
5728 while (p < remset->store_next) {
5729 switch ((*p) & REMSET_TYPE_MASK) {
5730 case REMSET_LOCATION:
5733 ++stat_saved_remsets_1;
5735 if (*p == last1 || *p == last2) {
5736 ++stat_saved_remsets_2;
5754 case REMSET_ROOT_LOCATION:
5758 g_assert_not_reached ();
5762 g_assert_not_reached ();
5772 RememberedSet *remset;
5774 SgenThreadInfo *info;
5776 mword *addresses, *bumper, *p, *r;
5778 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5779 for (info = thread_table [i]; info; info = info->next) {
5780 for (remset = info->remset; remset; remset = remset->next)
5781 size += remset->store_next - remset->data;
5784 for (remset = freed_thread_remsets; remset; remset = remset->next)
5785 size += remset->store_next - remset->data;
5786 for (remset = global_remset; remset; remset = remset->next)
5787 size += remset->store_next - remset->data;
5789 bumper = addresses = get_internal_mem (sizeof (mword) * size, INTERNAL_MEM_STATISTICS);
5791 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5792 for (info = thread_table [i]; info; info = info->next) {
5793 for (remset = info->remset; remset; remset = remset->next)
5794 bumper = collect_store_remsets (remset, bumper);
5797 for (remset = global_remset; remset; remset = remset->next)
5798 bumper = collect_store_remsets (remset, bumper);
5799 for (remset = freed_thread_remsets; remset; remset = remset->next)
5800 bumper = collect_store_remsets (remset, bumper);
5802 g_assert (bumper <= addresses + size);
5804 stat_store_remsets += bumper - addresses;
5806 sort_addresses ((void**)addresses, bumper - addresses);
5809 while (r < bumper) {
5815 stat_store_remsets_unique += p - addresses;
5817 free_internal_mem (addresses, INTERNAL_MEM_STATISTICS);
5822 clear_thread_store_remset_buffer (SgenThreadInfo *info)
5824 *info->store_remset_buffer_index_addr = 0;
5825 memset (*info->store_remset_buffer_addr, 0, sizeof (gpointer) * STORE_REMSET_BUFFER_SIZE);
5829 scan_from_remsets (void *start_nursery, void *end_nursery)
5832 SgenThreadInfo *info;
5833 RememberedSet *remset;
5834 GenericStoreRememberedSet *store_remset;
5835 mword *p, *next_p, *store_pos;
5837 #ifdef HEAVY_STATISTICS
5841 /* the global one */
5842 for (remset = global_remset; remset; remset = remset->next) {
5843 DEBUG (4, fprintf (gc_debug_file, "Scanning global remset range: %p-%p, size: %zd\n", remset->data, remset->store_next, remset->store_next - remset->data));
5844 store_pos = remset->data;
5845 for (p = remset->data; p < remset->store_next; p = next_p) {
5848 next_p = handle_remset (p, start_nursery, end_nursery, TRUE);
5851 * Clear global remsets of locations which no longer point to the
5852 * nursery. Otherwise, they could grow indefinitely between major
5855 ptr = (p [0] & ~REMSET_TYPE_MASK);
5856 if ((p [0] & REMSET_TYPE_MASK) == REMSET_LOCATION) {
5857 if (ptr_in_nursery (*(void**)ptr))
5858 *store_pos ++ = p [0];
5860 g_assert ((p [0] & REMSET_TYPE_MASK) == REMSET_OTHER);
5861 g_assert (p [1] == REMSET_ROOT_LOCATION);
5862 if (ptr_in_nursery (*(void**)ptr)) {
5863 *store_pos ++ = p [0];
5864 *store_pos ++ = p [1];
5869 /* Truncate the remset */
5870 remset->store_next = store_pos;
5873 /* the generic store ones */
5874 store_remset = generic_store_remsets;
5875 while (store_remset) {
5876 GenericStoreRememberedSet *next = store_remset->next;
5878 for (i = 0; i < STORE_REMSET_BUFFER_SIZE - 1; ++i) {
5879 gpointer addr = store_remset->data [i];
5881 handle_remset ((mword*)&addr, start_nursery, end_nursery, FALSE);
5884 free_internal_mem (store_remset, INTERNAL_MEM_STORE_REMSET);
5886 store_remset = next;
5888 generic_store_remsets = NULL;
5890 /* the per-thread ones */
5891 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5892 for (info = thread_table [i]; info; info = info->next) {
5893 RememberedSet *next;
5895 for (remset = info->remset; remset; remset = next) {
5896 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for thread %p, range: %p-%p, size: %zd\n", info, remset->data, remset->store_next, remset->store_next - remset->data));
5897 for (p = remset->data; p < remset->store_next;) {
5898 p = handle_remset (p, start_nursery, end_nursery, FALSE);
5900 remset->store_next = remset->data;
5901 next = remset->next;
5902 remset->next = NULL;
5903 if (remset != info->remset) {
5904 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
5905 free_internal_mem (remset, INTERNAL_MEM_REMSET);
5908 for (j = 0; j < *info->store_remset_buffer_index_addr; ++j)
5909 handle_remset ((mword*)*info->store_remset_buffer_addr + j + 1, start_nursery, end_nursery, FALSE);
5910 clear_thread_store_remset_buffer (info);
5914 /* the freed thread ones */
5915 while (freed_thread_remsets) {
5916 RememberedSet *next;
5917 remset = freed_thread_remsets;
5918 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for freed thread, range: %p-%p, size: %zd\n", remset->data, remset->store_next, remset->store_next - remset->data));
5919 for (p = remset->data; p < remset->store_next;) {
5920 p = handle_remset (p, start_nursery, end_nursery, FALSE);
5922 next = remset->next;
5923 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
5924 free_internal_mem (remset, INTERNAL_MEM_REMSET);
5925 freed_thread_remsets = next;
5930 * Clear the info in the remembered sets: we're doing a major collection, so
5931 * the per-thread ones are not needed and the global ones will be reconstructed
5935 clear_remsets (void)
5938 SgenThreadInfo *info;
5939 RememberedSet *remset, *next;
5941 /* the global list */
5942 for (remset = global_remset; remset; remset = next) {
5943 remset->store_next = remset->data;
5944 next = remset->next;
5945 remset->next = NULL;
5946 if (remset != global_remset) {
5947 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
5948 free_internal_mem (remset, INTERNAL_MEM_REMSET);
5951 /* the generic store ones */
5952 while (generic_store_remsets) {
5953 GenericStoreRememberedSet *gs_next = generic_store_remsets->next;
5954 free_internal_mem (generic_store_remsets, INTERNAL_MEM_STORE_REMSET);
5955 generic_store_remsets = gs_next;
5957 /* the per-thread ones */
5958 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5959 for (info = thread_table [i]; info; info = info->next) {
5960 for (remset = info->remset; remset; remset = next) {
5961 remset->store_next = remset->data;
5962 next = remset->next;
5963 remset->next = NULL;
5964 if (remset != info->remset) {
5965 DEBUG (1, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
5966 free_internal_mem (remset, INTERNAL_MEM_REMSET);
5969 clear_thread_store_remset_buffer (info);
5973 /* the freed thread ones */
5974 while (freed_thread_remsets) {
5975 next = freed_thread_remsets->next;
5976 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", freed_thread_remsets->data));
5977 free_internal_mem (freed_thread_remsets, INTERNAL_MEM_REMSET);
5978 freed_thread_remsets = next;
5983 * Clear the thread local TLAB variables for all threads.
5988 SgenThreadInfo *info;
5991 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5992 for (info = thread_table [i]; info; info = info->next) {
5993 /* A new TLAB will be allocated when the thread does its first allocation */
5994 *info->tlab_start_addr = NULL;
5995 *info->tlab_next_addr = NULL;
5996 *info->tlab_temp_end_addr = NULL;
5997 *info->tlab_real_end_addr = NULL;
6002 /* LOCKING: assumes the GC lock is held */
6003 static SgenThreadInfo*
6004 gc_register_current_thread (void *addr)
6007 SgenThreadInfo* info = malloc (sizeof (SgenThreadInfo));
6008 #ifndef HAVE_KW_THREAD
6009 SgenThreadInfo *__thread_info__ = info;
6015 #ifndef HAVE_KW_THREAD
6016 info->tlab_start = info->tlab_next = info->tlab_temp_end = info->tlab_real_end = NULL;
6018 g_assert (!pthread_getspecific (thread_info_key));
6019 pthread_setspecific (thread_info_key, info);
6022 info->id = ARCH_GET_THREAD ();
6023 info->stop_count = -1;
6026 info->stack_start = NULL;
6027 info->tlab_start_addr = &TLAB_START;
6028 info->tlab_next_addr = &TLAB_NEXT;
6029 info->tlab_temp_end_addr = &TLAB_TEMP_END;
6030 info->tlab_real_end_addr = &TLAB_REAL_END;
6031 info->store_remset_buffer_addr = &STORE_REMSET_BUFFER;
6032 info->store_remset_buffer_index_addr = &STORE_REMSET_BUFFER_INDEX;
6033 info->stopped_ip = NULL;
6034 info->stopped_domain = NULL;
6035 info->stopped_regs = NULL;
6037 #ifdef HAVE_KW_THREAD
6038 tlab_next_addr = &tlab_next;
6039 store_remset_buffer_index_addr = &store_remset_buffer_index;
6042 /* try to get it with attributes first */
6043 #if defined(HAVE_PTHREAD_GETATTR_NP) && defined(HAVE_PTHREAD_ATTR_GETSTACK)
6047 pthread_attr_t attr;
6048 pthread_getattr_np (pthread_self (), &attr);
6049 pthread_attr_getstack (&attr, &sstart, &size);
6050 info->stack_start_limit = sstart;
6051 info->stack_end = (char*)sstart + size;
6052 pthread_attr_destroy (&attr);
6054 #elif defined(HAVE_PTHREAD_GET_STACKSIZE_NP) && defined(HAVE_PTHREAD_GET_STACKADDR_NP)
6055 info->stack_end = (char*)pthread_get_stackaddr_np (pthread_self ());
6058 /* FIXME: we assume the stack grows down */
6059 gsize stack_bottom = (gsize)addr;
6060 stack_bottom += 4095;
6061 stack_bottom &= ~4095;
6062 info->stack_end = (char*)stack_bottom;
6066 #ifdef HAVE_KW_THREAD
6067 stack_end = info->stack_end;
6070 /* hash into the table */
6071 hash = HASH_PTHREAD_T (info->id) % THREAD_HASH_SIZE;
6072 info->next = thread_table [hash];
6073 thread_table [hash] = info;
6075 info->remset = alloc_remset (DEFAULT_REMSET_SIZE, info);
6076 pthread_setspecific (remembered_set_key, info->remset);
6077 #ifdef HAVE_KW_THREAD
6078 remembered_set = info->remset;
6081 STORE_REMSET_BUFFER = get_internal_mem (sizeof (gpointer) * STORE_REMSET_BUFFER_SIZE, INTERNAL_MEM_STORE_REMSET);
6082 STORE_REMSET_BUFFER_INDEX = 0;
6084 DEBUG (3, fprintf (gc_debug_file, "registered thread %p (%p) (hash: %d)\n", info, (gpointer)info->id, hash));
6086 if (gc_callbacks.thread_attach_func)
6087 info->runtime_data = gc_callbacks.thread_attach_func ();
6093 add_generic_store_remset_from_buffer (gpointer *buffer)
6095 GenericStoreRememberedSet *remset = get_internal_mem (sizeof (GenericStoreRememberedSet), INTERNAL_MEM_STORE_REMSET);
6096 memcpy (remset->data, buffer + 1, sizeof (gpointer) * (STORE_REMSET_BUFFER_SIZE - 1));
6097 remset->next = generic_store_remsets;
6098 generic_store_remsets = remset;
6102 unregister_current_thread (void)
6105 SgenThreadInfo *prev = NULL;
6107 RememberedSet *rset;
6108 ARCH_THREAD_TYPE id = ARCH_GET_THREAD ();
6110 hash = HASH_PTHREAD_T (id) % THREAD_HASH_SIZE;
6111 p = thread_table [hash];
6113 DEBUG (3, fprintf (gc_debug_file, "unregister thread %p (%p)\n", p, (gpointer)p->id));
6114 while (!ARCH_THREAD_EQUALS (p->id, id)) {
6119 thread_table [hash] = p->next;
6121 prev->next = p->next;
6124 if (freed_thread_remsets) {
6125 for (rset = p->remset; rset->next; rset = rset->next)
6127 rset->next = freed_thread_remsets;
6128 freed_thread_remsets = p->remset;
6130 freed_thread_remsets = p->remset;
6133 if (*p->store_remset_buffer_index_addr)
6134 add_generic_store_remset_from_buffer (*p->store_remset_buffer_addr);
6135 free_internal_mem (*p->store_remset_buffer_addr, INTERNAL_MEM_STORE_REMSET);
6140 unregister_thread (void *k)
6142 g_assert (!mono_domain_get ());
6144 unregister_current_thread ();
6149 mono_gc_register_thread (void *baseptr)
6151 SgenThreadInfo *info;
6155 info = thread_info_lookup (ARCH_GET_THREAD ());
6157 info = gc_register_current_thread (baseptr);
6159 return info != NULL;
6162 #if USE_PTHREAD_INTERCEPT
6164 #undef pthread_create
6166 #undef pthread_detach
6169 void *(*start_routine) (void *);
6172 MonoSemType registered;
6173 } SgenThreadStartInfo;
6176 gc_start_thread (void *arg)
6178 SgenThreadStartInfo *start_info = arg;
6179 SgenThreadInfo* info;
6180 void *t_arg = start_info->arg;
6181 void *(*start_func) (void*) = start_info->start_routine;
6186 info = gc_register_current_thread (&result);
6188 post_result = MONO_SEM_POST (&(start_info->registered));
6189 g_assert (!post_result);
6190 result = start_func (t_arg);
6191 g_assert (!mono_domain_get ());
6193 * this is done by the pthread key dtor
6195 unregister_current_thread ();
6203 mono_gc_pthread_create (pthread_t *new_thread, const pthread_attr_t *attr, void *(*start_routine)(void *), void *arg)
6205 SgenThreadStartInfo *start_info;
6208 start_info = malloc (sizeof (SgenThreadStartInfo));
6211 result = MONO_SEM_INIT (&(start_info->registered), 0);
6213 start_info->arg = arg;
6214 start_info->start_routine = start_routine;
6216 result = pthread_create (new_thread, attr, gc_start_thread, start_info);
6218 while (MONO_SEM_WAIT (&(start_info->registered)) != 0) {
6219 /*if (EINTR != errno) ABORT("sem_wait failed"); */
6222 MONO_SEM_DESTROY (&(start_info->registered));
6228 mono_gc_pthread_join (pthread_t thread, void **retval)
6230 return pthread_join (thread, retval);
6234 mono_gc_pthread_detach (pthread_t thread)
6236 return pthread_detach (thread);
6239 #endif /* USE_PTHREAD_INTERCEPT */
6242 * ######################################################################
6243 * ######## Write barriers
6244 * ######################################################################
6247 static RememberedSet*
6248 alloc_remset (int size, gpointer id) {
6249 RememberedSet* res = get_internal_mem (sizeof (RememberedSet) + (size * sizeof (gpointer)), INTERNAL_MEM_REMSET);
6250 res->store_next = res->data;
6251 res->end_set = res->data + size;
6253 DEBUG (4, fprintf (gc_debug_file, "Allocated remset size %d at %p for %p\n", size, res->data, id));
6258 * Note: the write barriers first do the needed GC work and then do the actual store:
6259 * this way the value is visible to the conservative GC scan after the write barrier
6260 * itself. If a GC interrupts the barrier in the middle, value will be kept alive by
6261 * the conservative scan, otherwise by the remembered set scan. FIXME: figure out what
6262 * happens when we need to record which pointers contain references to the new generation.
6263 * The write barrier will be executed, but the pointer is still not stored.
6266 mono_gc_wbarrier_set_field (MonoObject *obj, gpointer field_ptr, MonoObject* value)
6270 HEAVY_STAT (++stat_wbarrier_set_field);
6271 if (ptr_in_nursery (field_ptr)) {
6272 *(void**)field_ptr = value;
6275 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", field_ptr));
6277 rs = REMEMBERED_SET;
6278 if (rs->store_next < rs->end_set) {
6279 *(rs->store_next++) = (mword)field_ptr;
6280 *(void**)field_ptr = value;
6284 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6285 rs->next = REMEMBERED_SET;
6286 REMEMBERED_SET = rs;
6287 #ifdef HAVE_KW_THREAD
6288 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6290 *(rs->store_next++) = (mword)field_ptr;
6291 *(void**)field_ptr = value;
6296 mono_gc_wbarrier_set_arrayref (MonoArray *arr, gpointer slot_ptr, MonoObject* value)
6300 HEAVY_STAT (++stat_wbarrier_set_arrayref);
6301 if (ptr_in_nursery (slot_ptr)) {
6302 *(void**)slot_ptr = value;
6305 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", slot_ptr));
6307 rs = REMEMBERED_SET;
6308 if (rs->store_next < rs->end_set) {
6309 *(rs->store_next++) = (mword)slot_ptr;
6310 *(void**)slot_ptr = value;
6314 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6315 rs->next = REMEMBERED_SET;
6316 REMEMBERED_SET = rs;
6317 #ifdef HAVE_KW_THREAD
6318 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6320 *(rs->store_next++) = (mword)slot_ptr;
6321 *(void**)slot_ptr = value;
6326 mono_gc_wbarrier_arrayref_copy (gpointer dest_ptr, gpointer src_ptr, int count)
6330 HEAVY_STAT (++stat_wbarrier_arrayref_copy);
6332 memmove (dest_ptr, src_ptr, count * sizeof (gpointer));
6333 if (ptr_in_nursery (dest_ptr)) {
6337 rs = REMEMBERED_SET;
6338 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p, %d\n", dest_ptr, count));
6339 if (rs->store_next + 1 < rs->end_set) {
6340 *(rs->store_next++) = (mword)dest_ptr | REMSET_RANGE;
6341 *(rs->store_next++) = count;
6345 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6346 rs->next = REMEMBERED_SET;
6347 REMEMBERED_SET = rs;
6348 #ifdef HAVE_KW_THREAD
6349 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6351 *(rs->store_next++) = (mword)dest_ptr | REMSET_RANGE;
6352 *(rs->store_next++) = count;
6357 find_object_for_ptr_in_area (char *ptr, char *start, char *end)
6359 while (start < end) {
6362 if (!*(void**)start) {
6363 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
6369 #define SCAN_OBJECT_NOSCAN
6370 #include "sgen-scan-object.h"
6372 if (ptr >= old_start && ptr < start)
6379 static char *found_obj;
6382 find_object_for_ptr_in_pinned_chunk_callback (PinnedChunk *chunk, char *obj, size_t size, char *ptr)
6384 if (ptr >= obj && ptr < obj + size) {
6385 g_assert (!found_obj);
6390 /* for use in the debugger */
6391 char* find_object_for_ptr (char *ptr);
6393 find_object_for_ptr (char *ptr)
6395 GCMemSection *section;
6398 for (section = section_list; section; section = section->block.next) {
6399 if (ptr >= section->data && ptr < section->end_data)
6400 return find_object_for_ptr_in_area (ptr, section->data, section->end_data);
6403 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
6404 if (ptr >= bigobj->data && ptr < bigobj->data + bigobj->size)
6405 return bigobj->data;
6409 scan_pinned_objects ((ScanPinnedObjectCallbackFunc)find_object_for_ptr_in_pinned_chunk_callback, ptr);
6414 evacuate_remset_buffer (void)
6419 buffer = STORE_REMSET_BUFFER;
6421 add_generic_store_remset_from_buffer (buffer);
6422 memset (buffer, 0, sizeof (gpointer) * STORE_REMSET_BUFFER_SIZE);
6424 STORE_REMSET_BUFFER_INDEX = 0;
6428 mono_gc_wbarrier_generic_nostore (gpointer ptr)
6434 HEAVY_STAT (++stat_wbarrier_generic_store);
6436 #ifdef XDOMAIN_CHECKS_IN_WBARRIER
6437 /* FIXME: ptr_in_heap must be called with the GC lock held */
6438 if (xdomain_checks && *(MonoObject**)ptr && ptr_in_heap (ptr)) {
6439 char *start = find_object_for_ptr (ptr);
6440 MonoObject *value = *(MonoObject**)ptr;
6444 MonoObject *obj = (MonoObject*)start;
6445 if (obj->vtable->domain != value->vtable->domain)
6446 g_assert (is_xdomain_ref_allowed (ptr, start, obj->vtable->domain));
6453 if (ptr_in_nursery (ptr) || ptr_on_stack (ptr) || !ptr_in_nursery (*(gpointer*)ptr)) {
6454 DEBUG (8, fprintf (gc_debug_file, "Skipping remset at %p\n", ptr));
6459 buffer = STORE_REMSET_BUFFER;
6460 index = STORE_REMSET_BUFFER_INDEX;
6461 /* This simple optimization eliminates a sizable portion of
6462 entries. Comparing it to the last but one entry as well
6463 doesn't eliminate significantly more entries. */
6464 if (buffer [index] == ptr) {
6469 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", ptr));
6470 HEAVY_STAT (++stat_wbarrier_generic_store_remset);
6473 if (index >= STORE_REMSET_BUFFER_SIZE) {
6474 evacuate_remset_buffer ();
6475 index = STORE_REMSET_BUFFER_INDEX;
6476 g_assert (index == 0);
6479 buffer [index] = ptr;
6480 STORE_REMSET_BUFFER_INDEX = index;
6486 mono_gc_wbarrier_generic_store (gpointer ptr, MonoObject* value)
6488 DEBUG (8, fprintf (gc_debug_file, "Wbarrier store at %p to %p (%s)\n", ptr, value, value ? safe_name (value) : "null"));
6489 *(void**)ptr = value;
6490 if (ptr_in_nursery (value))
6491 mono_gc_wbarrier_generic_nostore (ptr);
6495 mono_gc_wbarrier_set_root (gpointer ptr, MonoObject *value)
6499 HEAVY_STAT (++stat_wbarrier_set_root);
6500 if (ptr_in_nursery (ptr))
6502 DEBUG (8, fprintf (gc_debug_file, "Adding root remset at %p (%s)\n", ptr, value ? safe_name (value) : "null"));
6504 rs = REMEMBERED_SET;
6505 if (rs->store_next + 2 < rs->end_set) {
6506 *(rs->store_next++) = (mword)ptr | REMSET_OTHER;
6507 *(rs->store_next++) = (mword)REMSET_ROOT_LOCATION;
6508 *(void**)ptr = value;
6511 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6512 rs->next = REMEMBERED_SET;
6513 REMEMBERED_SET = rs;
6514 #ifdef HAVE_KW_THREAD
6515 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6517 *(rs->store_next++) = (mword)ptr | REMSET_OTHER;
6518 *(rs->store_next++) = (mword)REMSET_ROOT_LOCATION;
6520 *(void**)ptr = value;
6524 mono_gc_wbarrier_value_copy (gpointer dest, gpointer src, int count, MonoClass *klass)
6528 HEAVY_STAT (++stat_wbarrier_value_copy);
6529 g_assert (klass->valuetype);
6531 memmove (dest, src, count * mono_class_value_size (klass, NULL));
6532 rs = REMEMBERED_SET;
6533 if (ptr_in_nursery (dest) || ptr_on_stack (dest)) {
6537 g_assert (klass->gc_descr_inited);
6538 DEBUG (8, fprintf (gc_debug_file, "Adding value remset at %p, count %d, descr %p for class %s (%p)\n", dest, count, klass->gc_descr, klass->name, klass));
6540 if (rs->store_next + 3 < rs->end_set) {
6541 *(rs->store_next++) = (mword)dest | REMSET_OTHER;
6542 *(rs->store_next++) = (mword)REMSET_VTYPE;
6543 *(rs->store_next++) = (mword)klass->gc_descr;
6544 *(rs->store_next++) = (mword)count;
6548 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6549 rs->next = REMEMBERED_SET;
6550 REMEMBERED_SET = rs;
6551 #ifdef HAVE_KW_THREAD
6552 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6554 *(rs->store_next++) = (mword)dest | REMSET_OTHER;
6555 *(rs->store_next++) = (mword)REMSET_VTYPE;
6556 *(rs->store_next++) = (mword)klass->gc_descr;
6557 *(rs->store_next++) = (mword)count;
6562 * mono_gc_wbarrier_object_copy:
6564 * Write barrier to call when obj is the result of a clone or copy of an object.
6567 mono_gc_wbarrier_object_copy (MonoObject* obj, MonoObject *src)
6573 HEAVY_STAT (++stat_wbarrier_object_copy);
6574 rs = REMEMBERED_SET;
6575 DEBUG (1, fprintf (gc_debug_file, "Adding object remset for %p\n", obj));
6576 size = mono_object_class (obj)->instance_size;
6578 /* do not copy the sync state */
6579 memcpy ((char*)obj + sizeof (MonoObject), (char*)src + sizeof (MonoObject),
6580 size - sizeof (MonoObject));
6581 if (ptr_in_nursery (obj) || ptr_on_stack (obj)) {
6585 if (rs->store_next < rs->end_set) {
6586 *(rs->store_next++) = (mword)obj | REMSET_OBJECT;
6590 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6591 rs->next = REMEMBERED_SET;
6592 REMEMBERED_SET = rs;
6593 #ifdef HAVE_KW_THREAD
6594 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6596 *(rs->store_next++) = (mword)obj | REMSET_OBJECT;
6601 * ######################################################################
6602 * ######## Collector debugging
6603 * ######################################################################
6606 const char*descriptor_types [] = {
6618 describe_ptr (char *ptr)
6620 GCMemSection *section;
6625 if (ptr_in_nursery (ptr)) {
6626 printf ("Pointer inside nursery.\n");
6628 for (section = section_list; section;) {
6629 if (ptr >= section->data && ptr < section->data + section->size)
6631 section = section->block.next;
6635 printf ("Pointer inside oldspace.\n");
6636 } else if (obj_is_from_pinned_alloc (ptr)) {
6637 printf ("Pointer is inside a pinned chunk.\n");
6639 printf ("Pointer unknown.\n");
6644 if (object_is_pinned (ptr))
6645 printf ("Object is pinned.\n");
6647 if (object_is_forwarded (ptr))
6648 printf ("Object is forwared.\n");
6650 // FIXME: Handle pointers to the inside of objects
6651 vtable = (MonoVTable*)LOAD_VTABLE (ptr);
6653 printf ("VTable: %p\n", vtable);
6654 if (vtable == NULL) {
6655 printf ("VTable is invalid (empty).\n");
6658 if (ptr_in_nursery (vtable)) {
6659 printf ("VTable is invalid (points inside nursery).\n");
6662 printf ("Class: %s\n", vtable->klass->name);
6664 desc = ((GCVTable*)vtable)->desc;
6665 printf ("Descriptor: %lx\n", (long)desc);
6668 printf ("Descriptor type: %d (%s)\n", type, descriptor_types [type]);
6672 find_in_remset_loc (mword *p, char *addr, gboolean *found)
6678 switch ((*p) & REMSET_TYPE_MASK) {
6679 case REMSET_LOCATION:
6680 if (*p == (mword)addr)
6684 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
6686 if ((void**)addr >= ptr && (void**)addr < ptr + count)
6690 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
6691 count = safe_object_get_size ((MonoObject*)ptr);
6692 count += (ALLOC_ALIGN - 1);
6693 count &= (ALLOC_ALIGN - 1);
6694 count /= sizeof (mword);
6695 if ((void**)addr >= ptr && (void**)addr < ptr + count)
6698 case REMSET_OTHER: {
6701 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
6705 switch (desc & 0x7) {
6706 case DESC_TYPE_RUN_LENGTH:
6707 OBJ_RUN_LEN_SIZE (skip_size, desc, ptr);
6708 /* The descriptor includes the size of MonoObject */
6709 skip_size -= sizeof (MonoObject);
6711 if ((void**)addr >= ptr && (void**)addr < ptr + (skip_size / sizeof (gpointer)))
6716 g_assert_not_reached ();
6720 case REMSET_ROOT_LOCATION:
6723 g_assert_not_reached ();
6728 g_assert_not_reached ();
6734 * Return whenever ADDR occurs in the remembered sets
6737 find_in_remsets (char *addr)
6740 SgenThreadInfo *info;
6741 RememberedSet *remset;
6742 GenericStoreRememberedSet *store_remset;
6744 gboolean found = FALSE;
6746 /* the global one */
6747 for (remset = global_remset; remset; remset = remset->next) {
6748 DEBUG (4, fprintf (gc_debug_file, "Scanning global remset range: %p-%p, size: %zd\n", remset->data, remset->store_next, remset->store_next - remset->data));
6749 for (p = remset->data; p < remset->store_next;) {
6750 p = find_in_remset_loc (p, addr, &found);
6756 /* the generic store ones */
6757 for (store_remset = generic_store_remsets; store_remset; store_remset = store_remset->next) {
6758 for (i = 0; i < STORE_REMSET_BUFFER_SIZE - 1; ++i) {
6759 if (store_remset->data [i] == addr)
6764 /* the per-thread ones */
6765 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
6766 for (info = thread_table [i]; info; info = info->next) {
6768 for (remset = info->remset; remset; remset = remset->next) {
6769 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for thread %p, range: %p-%p, size: %zd\n", info, remset->data, remset->store_next, remset->store_next - remset->data));
6770 for (p = remset->data; p < remset->store_next;) {
6771 p = find_in_remset_loc (p, addr, &found);
6776 for (j = 0; j < *info->store_remset_buffer_index_addr; ++j) {
6777 if ((*info->store_remset_buffer_addr) [j + 1] == addr)
6783 /* the freed thread ones */
6784 for (remset = freed_thread_remsets; remset; remset = remset->next) {
6785 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for freed thread, range: %p-%p, size: %zd\n", remset->data, remset->store_next, remset->store_next - remset->data));
6786 for (p = remset->data; p < remset->store_next;) {
6787 p = find_in_remset_loc (p, addr, &found);
6797 #define HANDLE_PTR(ptr,obj) do { \
6798 if (*(ptr) && (char*)*(ptr) >= nursery_start && (char*)*(ptr) < nursery_next) { \
6799 if (!find_in_remsets ((char*)(ptr))) { \
6800 fprintf (gc_debug_file, "Oldspace->newspace reference %p at offset %zd in object %p (%s.%s) not found in remsets.\n", *(ptr), (char*)(ptr) - (char*)(obj), (obj), ((MonoObject*)(obj))->vtable->klass->name_space, ((MonoObject*)(obj))->vtable->klass->name); \
6801 g_assert_not_reached (); \
6807 * Check that each object reference inside the area which points into the nursery
6808 * can be found in the remembered sets.
6810 static void __attribute__((noinline))
6811 check_remsets_for_area (char *start, char *end)
6814 int type_str = 0, type_rlen = 0, type_bitmap = 0, type_vector = 0, type_lbit = 0, type_complex = 0;
6815 while (start < end) {
6816 if (!*(void**)start) {
6817 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
6820 vt = (GCVTable*)LOAD_VTABLE (start);
6821 DEBUG (8, fprintf (gc_debug_file, "Scanning object %p, vtable: %p (%s)\n", start, vt, vt->klass->name));
6823 MonoObject *obj = (MonoObject*)start;
6824 g_print ("found at %p (0x%lx): %s.%s\n", start, (long)vt->desc, obj->vtable->klass->name_space, obj->vtable->klass->name);
6827 #define SCAN_OBJECT_ACTION COUNT_OBJECT_TYPES
6828 #include "sgen-scan-object.h"
6833 * Perform consistency check of the heap.
6835 * Assumes the world is stopped.
6838 check_consistency (void)
6840 GCMemSection *section;
6842 // Need to add more checks
6843 // FIXME: Create a general heap enumeration function and use that
6845 DEBUG (1, fprintf (gc_debug_file, "Begin heap consistency check...\n"));
6847 // Check that oldspace->newspace pointers are registered with the collector
6848 for (section = section_list; section; section = section->block.next) {
6849 if (section->block.role == MEMORY_ROLE_GEN0)
6851 DEBUG (2, fprintf (gc_debug_file, "Scan of old section: %p-%p, size: %d\n", section->data, section->next_data, (int)(section->next_data - section->data)));
6852 check_remsets_for_area (section->data, section->next_data);
6855 DEBUG (1, fprintf (gc_debug_file, "Heap consistency check done.\n"));
6858 /* Check that the reference is valid */
6860 #define HANDLE_PTR(ptr,obj) do { \
6862 g_assert (safe_name (*(ptr)) != NULL); \
6869 * Perform consistency check on an object. Currently we only check that the
6870 * reference fields are valid.
6873 check_object (char *start)
6878 #include "sgen-scan-object.h"
6884 * ######################################################################
6885 * ######## Other mono public interface functions.
6886 * ######################################################################
6890 mono_gc_collect (int generation)
6894 if (generation == 0) {
6895 collect_nursery (0);
6897 major_collection ("user request");
6904 mono_gc_max_generation (void)
6910 mono_gc_collection_count (int generation)
6912 if (generation == 0)
6913 return num_minor_gcs;
6914 return num_major_gcs;
6918 mono_gc_get_used_size (void)
6921 GCMemSection *section;
6923 tot = los_memory_usage;
6924 for (section = section_list; section; section = section->block.next) {
6925 /* this is approximate... */
6926 tot += section->next_data - section->data;
6928 /* FIXME: account for pinned objects */
6934 mono_gc_get_heap_size (void)
6940 mono_gc_disable (void)
6948 mono_gc_enable (void)
6956 mono_gc_get_los_limit (void)
6958 return MAX_SMALL_OBJ_SIZE;
6962 mono_object_is_alive (MonoObject* o)
6968 mono_gc_get_generation (MonoObject *obj)
6970 if (ptr_in_nursery (obj))
6976 mono_gc_enable_events (void)
6981 mono_gc_weak_link_add (void **link_addr, MonoObject *obj, gboolean track)
6984 mono_gc_register_disappearing_link (obj, link_addr, track);
6989 mono_gc_weak_link_remove (void **link_addr)
6992 mono_gc_register_disappearing_link (NULL, link_addr, FALSE);
6997 mono_gc_weak_link_get (void **link_addr)
7001 return (MonoObject*) REVEAL_POINTER (*link_addr);
7005 mono_gc_make_descr_from_bitmap (gsize *bitmap, int numbits)
7007 if (numbits < ((sizeof (*bitmap) * 8) - ROOT_DESC_TYPE_SHIFT)) {
7008 return (void*)MAKE_ROOT_DESC (ROOT_DESC_BITMAP, bitmap [0]);
7010 mword complex = alloc_complex_descriptor (bitmap, numbits + 1);
7011 return (void*)MAKE_ROOT_DESC (ROOT_DESC_COMPLEX, complex);
7016 mono_gc_make_root_descr_user (MonoGCMarkFunc marker)
7020 g_assert (user_descriptors_next < MAX_USER_DESCRIPTORS);
7021 descr = (void*)MAKE_ROOT_DESC (ROOT_DESC_USER, (mword)user_descriptors_next);
7022 user_descriptors [user_descriptors_next ++] = marker;
7028 mono_gc_alloc_fixed (size_t size, void *descr)
7030 /* FIXME: do a single allocation */
7031 void *res = calloc (1, size);
7034 if (!mono_gc_register_root (res, size, descr)) {
7042 mono_gc_free_fixed (void* addr)
7044 mono_gc_deregister_root (addr);
7049 mono_gc_is_gc_thread (void)
7053 result = thread_info_lookup (ARCH_GET_THREAD ()) != NULL;
7059 mono_gc_base_init (void)
7063 struct sigaction sinfo;
7065 LOCK_INIT (gc_mutex);
7067 if (gc_initialized) {
7071 pagesize = mono_pagesize ();
7072 gc_debug_file = stderr;
7073 if ((env = getenv ("MONO_GC_DEBUG"))) {
7074 opts = g_strsplit (env, ",", -1);
7075 for (ptr = opts; ptr && *ptr; ptr ++) {
7077 if (opt [0] >= '0' && opt [0] <= '9') {
7078 gc_debug_level = atoi (opt);
7083 char *rf = g_strdup_printf ("%s.%d", opt, getpid ());
7084 gc_debug_file = fopen (rf, "wb");
7086 gc_debug_file = stderr;
7089 } else if (!strcmp (opt, "collect-before-allocs")) {
7090 collect_before_allocs = TRUE;
7091 } else if (!strcmp (opt, "check-at-minor-collections")) {
7092 consistency_check_at_minor_collection = TRUE;
7093 } else if (!strcmp (opt, "xdomain-checks")) {
7094 xdomain_checks = TRUE;
7095 } else if (!strcmp (opt, "clear-at-gc")) {
7096 nursery_clear_policy = CLEAR_AT_GC;
7097 } else if (!strcmp (opt, "conservative-stack-mark")) {
7098 conservative_stack_mark = TRUE;
7099 } else if (g_str_has_prefix (opt, "heap-dump=")) {
7100 char *filename = strchr (opt, '=') + 1;
7101 nursery_clear_policy = CLEAR_AT_GC;
7102 heap_dump_file = fopen (filename, "w");
7104 fprintf (heap_dump_file, "<sgen-dump>\n");
7106 fprintf (stderr, "Invalid format for the MONO_GC_DEBUG env variable: '%s'\n", env);
7107 fprintf (stderr, "The format is: MONO_GC_DEBUG=[l[:filename]|<option>]+ where l is a debug level 0-9.\n");
7108 fprintf (stderr, "Valid options are: collect-before-allocs, check-at-minor-collections, xdomain-checks, clear-at-gc.\n");
7115 suspend_ack_semaphore_ptr = &suspend_ack_semaphore;
7116 MONO_SEM_INIT (&suspend_ack_semaphore, 0);
7118 sigfillset (&sinfo.sa_mask);
7119 sinfo.sa_flags = SA_RESTART | SA_SIGINFO;
7120 sinfo.sa_sigaction = suspend_handler;
7121 if (sigaction (suspend_signal_num, &sinfo, NULL) != 0) {
7122 g_error ("failed sigaction");
7125 sinfo.sa_handler = restart_handler;
7126 if (sigaction (restart_signal_num, &sinfo, NULL) != 0) {
7127 g_error ("failed sigaction");
7130 sigfillset (&suspend_signal_mask);
7131 sigdelset (&suspend_signal_mask, restart_signal_num);
7133 global_remset = alloc_remset (1024, NULL);
7134 global_remset->next = NULL;
7136 pthread_key_create (&remembered_set_key, unregister_thread);
7138 #ifndef HAVE_KW_THREAD
7139 pthread_key_create (&thread_info_key, NULL);
7142 gc_initialized = TRUE;
7144 mono_gc_register_thread (&sinfo);
7148 mono_gc_get_suspend_signal (void)
7150 return suspend_signal_num;
7160 #ifdef HAVE_KW_THREAD
7161 #define EMIT_TLS_ACCESS(mb,dummy,offset) do { \
7162 mono_mb_emit_byte ((mb), MONO_CUSTOM_PREFIX); \
7163 mono_mb_emit_byte ((mb), CEE_MONO_TLS); \
7164 mono_mb_emit_i4 ((mb), (offset)); \
7167 #define EMIT_TLS_ACCESS(mb,member,dummy) do { \
7168 mono_mb_emit_byte ((mb), MONO_CUSTOM_PREFIX); \
7169 mono_mb_emit_byte ((mb), CEE_MONO_TLS); \
7170 mono_mb_emit_i4 ((mb), thread_info_key); \
7171 mono_mb_emit_icon ((mb), G_STRUCT_OFFSET (SgenThreadInfo, member)); \
7172 mono_mb_emit_byte ((mb), CEE_ADD); \
7173 mono_mb_emit_byte ((mb), CEE_LDIND_I); \
7177 #ifdef MANAGED_ALLOCATION
7178 /* FIXME: Do this in the JIT, where specialized allocation sequences can be created
7179 * for each class. This is currently not easy to do, as it is hard to generate basic
7180 * blocks + branches, but it is easy with the linear IL codebase.
7182 * For this to work we'd need to solve the TLAB race, first. Now we
7183 * require the allocator to be in a few known methods to make sure
7184 * that they are executed atomically via the restart mechanism.
7187 create_allocator (int atype)
7189 int p_var, size_var;
7190 guint32 slowpath_branch, max_size_branch;
7191 MonoMethodBuilder *mb;
7193 MonoMethodSignature *csig;
7194 static gboolean registered = FALSE;
7195 int tlab_next_addr_var, new_next_var;
7197 const char *name = NULL;
7198 AllocatorWrapperInfo *info;
7200 #ifdef HAVE_KW_THREAD
7201 int tlab_next_addr_offset = -1;
7202 int tlab_temp_end_offset = -1;
7204 MONO_THREAD_VAR_OFFSET (tlab_next_addr, tlab_next_addr_offset);
7205 MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
7207 g_assert (tlab_next_addr_offset != -1);
7208 g_assert (tlab_temp_end_offset != -1);
7212 mono_register_jit_icall (mono_gc_alloc_obj, "mono_gc_alloc_obj", mono_create_icall_signature ("object ptr int"), FALSE);
7213 mono_register_jit_icall (mono_gc_alloc_vector, "mono_gc_alloc_vector", mono_create_icall_signature ("object ptr int int"), FALSE);
7217 if (atype == ATYPE_SMALL) {
7219 name = "AllocSmall";
7220 } else if (atype == ATYPE_NORMAL) {
7223 } else if (atype == ATYPE_VECTOR) {
7225 name = "AllocVector";
7227 g_assert_not_reached ();
7230 csig = mono_metadata_signature_alloc (mono_defaults.corlib, num_params);
7231 csig->ret = &mono_defaults.object_class->byval_arg;
7232 for (i = 0; i < num_params; ++i)
7233 csig->params [i] = &mono_defaults.int_class->byval_arg;
7235 mb = mono_mb_new (mono_defaults.object_class, name, MONO_WRAPPER_ALLOC);
7236 size_var = mono_mb_add_local (mb, &mono_defaults.int32_class->byval_arg);
7237 if (atype == ATYPE_NORMAL || atype == ATYPE_SMALL) {
7238 /* size = vtable->klass->instance_size; */
7239 mono_mb_emit_ldarg (mb, 0);
7240 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoVTable, klass));
7241 mono_mb_emit_byte (mb, CEE_ADD);
7242 mono_mb_emit_byte (mb, CEE_LDIND_I);
7243 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoClass, instance_size));
7244 mono_mb_emit_byte (mb, CEE_ADD);
7245 /* FIXME: assert instance_size stays a 4 byte integer */
7246 mono_mb_emit_byte (mb, CEE_LDIND_U4);
7247 mono_mb_emit_stloc (mb, size_var);
7248 } else if (atype == ATYPE_VECTOR) {
7249 MonoExceptionClause *clause;
7251 MonoClass *oom_exc_class;
7254 /* n > MONO_ARRAY_MAX_INDEX -> OverflowException */
7255 mono_mb_emit_ldarg (mb, 1);
7256 mono_mb_emit_icon (mb, MONO_ARRAY_MAX_INDEX);
7257 pos = mono_mb_emit_short_branch (mb, CEE_BLE_UN_S);
7258 mono_mb_emit_exception (mb, "OverflowException", NULL);
7259 mono_mb_patch_short_branch (mb, pos);
7261 clause = mono_image_alloc0 (mono_defaults.corlib, sizeof (MonoExceptionClause));
7262 clause->try_offset = mono_mb_get_label (mb);
7264 /* vtable->klass->sizes.element_size */
7265 mono_mb_emit_ldarg (mb, 0);
7266 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoVTable, klass));
7267 mono_mb_emit_byte (mb, CEE_ADD);
7268 mono_mb_emit_byte (mb, CEE_LDIND_I);
7269 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoClass, sizes.element_size));
7270 mono_mb_emit_byte (mb, CEE_ADD);
7271 mono_mb_emit_byte (mb, CEE_LDIND_U4);
7274 mono_mb_emit_ldarg (mb, 1);
7275 mono_mb_emit_byte (mb, CEE_MUL_OVF_UN);
7276 /* + sizeof (MonoArray) */
7277 mono_mb_emit_icon (mb, sizeof (MonoArray));
7278 mono_mb_emit_byte (mb, CEE_ADD_OVF_UN);
7279 mono_mb_emit_stloc (mb, size_var);
7281 pos_leave = mono_mb_emit_branch (mb, CEE_LEAVE);
7284 clause->flags = MONO_EXCEPTION_CLAUSE_NONE;
7285 clause->try_len = mono_mb_get_pos (mb) - clause->try_offset;
7286 clause->data.catch_class = mono_class_from_name (mono_defaults.corlib,
7287 "System", "OverflowException");
7288 g_assert (clause->data.catch_class);
7289 clause->handler_offset = mono_mb_get_label (mb);
7291 oom_exc_class = mono_class_from_name (mono_defaults.corlib,
7292 "System", "OutOfMemoryException");
7293 g_assert (oom_exc_class);
7294 ctor = mono_class_get_method_from_name (oom_exc_class, ".ctor", 0);
7297 mono_mb_emit_byte (mb, CEE_POP);
7298 mono_mb_emit_op (mb, CEE_NEWOBJ, ctor);
7299 mono_mb_emit_byte (mb, CEE_THROW);
7301 clause->handler_len = mono_mb_get_pos (mb) - clause->handler_offset;
7302 mono_mb_set_clauses (mb, 1, clause);
7303 mono_mb_patch_branch (mb, pos_leave);
7306 g_assert_not_reached ();
7309 /* size += ALLOC_ALIGN - 1; */
7310 mono_mb_emit_ldloc (mb, size_var);
7311 mono_mb_emit_icon (mb, ALLOC_ALIGN - 1);
7312 mono_mb_emit_byte (mb, CEE_ADD);
7313 /* size &= ~(ALLOC_ALIGN - 1); */
7314 mono_mb_emit_icon (mb, ~(ALLOC_ALIGN - 1));
7315 mono_mb_emit_byte (mb, CEE_AND);
7316 mono_mb_emit_stloc (mb, size_var);
7318 /* if (size > MAX_SMALL_OBJ_SIZE) goto slowpath */
7319 if (atype != ATYPE_SMALL) {
7320 mono_mb_emit_ldloc (mb, size_var);
7321 mono_mb_emit_icon (mb, MAX_SMALL_OBJ_SIZE);
7322 max_size_branch = mono_mb_emit_short_branch (mb, MONO_CEE_BGT_S);
7326 * We need to modify tlab_next, but the JIT only supports reading, so we read
7327 * another tls var holding its address instead.
7330 /* tlab_next_addr (local) = tlab_next_addr (TLS var) */
7331 tlab_next_addr_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7332 EMIT_TLS_ACCESS (mb, tlab_next_addr, tlab_next_addr_offset);
7333 mono_mb_emit_stloc (mb, tlab_next_addr_var);
7335 /* p = (void**)tlab_next; */
7336 p_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7337 mono_mb_emit_ldloc (mb, tlab_next_addr_var);
7338 mono_mb_emit_byte (mb, CEE_LDIND_I);
7339 mono_mb_emit_stloc (mb, p_var);
7341 /* new_next = (char*)p + size; */
7342 new_next_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7343 mono_mb_emit_ldloc (mb, p_var);
7344 mono_mb_emit_ldloc (mb, size_var);
7345 mono_mb_emit_byte (mb, CEE_CONV_I);
7346 mono_mb_emit_byte (mb, CEE_ADD);
7347 mono_mb_emit_stloc (mb, new_next_var);
7349 /* tlab_next = new_next */
7350 mono_mb_emit_ldloc (mb, tlab_next_addr_var);
7351 mono_mb_emit_ldloc (mb, new_next_var);
7352 mono_mb_emit_byte (mb, CEE_STIND_I);
7354 /* if (G_LIKELY (new_next < tlab_temp_end)) */
7355 mono_mb_emit_ldloc (mb, new_next_var);
7356 EMIT_TLS_ACCESS (mb, tlab_temp_end, tlab_temp_end_offset);
7357 slowpath_branch = mono_mb_emit_short_branch (mb, MONO_CEE_BLT_UN_S);
7360 if (atype != ATYPE_SMALL)
7361 mono_mb_patch_short_branch (mb, max_size_branch);
7363 mono_mb_emit_byte (mb, MONO_CUSTOM_PREFIX);
7364 mono_mb_emit_byte (mb, CEE_MONO_NOT_TAKEN);
7366 /* FIXME: mono_gc_alloc_obj takes a 'size_t' as an argument, not an int32 */
7367 mono_mb_emit_ldarg (mb, 0);
7368 mono_mb_emit_ldloc (mb, size_var);
7369 if (atype == ATYPE_NORMAL || atype == ATYPE_SMALL) {
7370 mono_mb_emit_icall (mb, mono_gc_alloc_obj);
7371 } else if (atype == ATYPE_VECTOR) {
7372 mono_mb_emit_ldarg (mb, 1);
7373 mono_mb_emit_icall (mb, mono_gc_alloc_vector);
7375 g_assert_not_reached ();
7377 mono_mb_emit_byte (mb, CEE_RET);
7380 mono_mb_patch_short_branch (mb, slowpath_branch);
7382 /* FIXME: Memory barrier */
7385 mono_mb_emit_ldloc (mb, p_var);
7386 mono_mb_emit_ldarg (mb, 0);
7387 mono_mb_emit_byte (mb, CEE_STIND_I);
7389 if (atype == ATYPE_VECTOR) {
7390 /* arr->max_length = max_length; */
7391 mono_mb_emit_ldloc (mb, p_var);
7392 mono_mb_emit_ldflda (mb, G_STRUCT_OFFSET (MonoArray, max_length));
7393 mono_mb_emit_ldarg (mb, 1);
7394 mono_mb_emit_byte (mb, CEE_STIND_I);
7398 mono_mb_emit_ldloc (mb, p_var);
7399 mono_mb_emit_byte (mb, CEE_RET);
7401 res = mono_mb_create_method (mb, csig, 8);
7403 mono_method_get_header (res)->init_locals = FALSE;
7405 info = mono_image_alloc0 (mono_defaults.corlib, sizeof (AllocatorWrapperInfo));
7406 info->alloc_type = atype;
7407 mono_marshal_set_wrapper_info (res, info);
7413 static MonoMethod* alloc_method_cache [ATYPE_NUM];
7414 static MonoMethod *write_barrier_method;
7417 is_ip_in_managed_allocator (MonoDomain *domain, gpointer ip)
7425 ji = mono_jit_info_table_find (domain, ip);
7428 method = ji->method;
7430 if (method == write_barrier_method)
7432 for (i = 0; i < ATYPE_NUM; ++i)
7433 if (method == alloc_method_cache [i])
7439 * Generate an allocator method implementing the fast path of mono_gc_alloc_obj ().
7440 * The signature of the called method is:
7441 * object allocate (MonoVTable *vtable)
7444 mono_gc_get_managed_allocator (MonoVTable *vtable, gboolean for_box)
7446 #ifdef MANAGED_ALLOCATION
7447 MonoClass *klass = vtable->klass;
7449 #ifdef HAVE_KW_THREAD
7450 int tlab_next_offset = -1;
7451 int tlab_temp_end_offset = -1;
7452 MONO_THREAD_VAR_OFFSET (tlab_next, tlab_next_offset);
7453 MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
7455 if (tlab_next_offset == -1 || tlab_temp_end_offset == -1)
7459 if (!mono_runtime_has_tls_get ())
7461 if (klass->instance_size > tlab_size)
7463 if (klass->has_finalize || klass->marshalbyref || (mono_profiler_get_events () & MONO_PROFILE_ALLOCATIONS))
7467 if (klass->byval_arg.type == MONO_TYPE_STRING)
7469 if (collect_before_allocs)
7472 if (ALIGN_TO (klass->instance_size, ALLOC_ALIGN) < MAX_SMALL_OBJ_SIZE)
7473 return mono_gc_get_managed_allocator_by_type (ATYPE_SMALL);
7475 return mono_gc_get_managed_allocator_by_type (ATYPE_NORMAL);
7482 mono_gc_get_managed_array_allocator (MonoVTable *vtable, int rank)
7484 #ifdef MANAGED_ALLOCATION
7485 MonoClass *klass = vtable->klass;
7487 #ifdef HAVE_KW_THREAD
7488 int tlab_next_offset = -1;
7489 int tlab_temp_end_offset = -1;
7490 MONO_THREAD_VAR_OFFSET (tlab_next, tlab_next_offset);
7491 MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
7493 if (tlab_next_offset == -1 || tlab_temp_end_offset == -1)
7499 if (!mono_runtime_has_tls_get ())
7501 if (mono_profiler_get_events () & MONO_PROFILE_ALLOCATIONS)
7503 if (collect_before_allocs)
7505 g_assert (!klass->has_finalize && !klass->marshalbyref);
7507 return mono_gc_get_managed_allocator_by_type (ATYPE_VECTOR);
7514 mono_gc_get_managed_allocator_by_type (int atype)
7516 #ifdef MANAGED_ALLOCATION
7519 if (!mono_runtime_has_tls_get ())
7522 mono_loader_lock ();
7523 res = alloc_method_cache [atype];
7525 res = alloc_method_cache [atype] = create_allocator (atype);
7526 mono_loader_unlock ();
7534 mono_gc_get_managed_allocator_types (void)
7541 mono_gc_get_write_barrier (void)
7544 MonoMethodBuilder *mb;
7545 MonoMethodSignature *sig;
7546 #ifdef MANAGED_WBARRIER
7547 int label_no_wb_1, label_no_wb_2, label_no_wb_3, label_no_wb_4, label_need_wb, label_slow_path;
7548 int buffer_var, buffer_index_var, dummy_var;
7550 #ifdef HAVE_KW_THREAD
7551 int stack_end_offset = -1, store_remset_buffer_offset = -1;
7552 int store_remset_buffer_index_offset = -1, store_remset_buffer_index_addr_offset = -1;
7554 MONO_THREAD_VAR_OFFSET (stack_end, stack_end_offset);
7555 g_assert (stack_end_offset != -1);
7556 MONO_THREAD_VAR_OFFSET (store_remset_buffer, store_remset_buffer_offset);
7557 g_assert (store_remset_buffer_offset != -1);
7558 MONO_THREAD_VAR_OFFSET (store_remset_buffer_index, store_remset_buffer_index_offset);
7559 g_assert (store_remset_buffer_index_offset != -1);
7560 MONO_THREAD_VAR_OFFSET (store_remset_buffer_index_addr, store_remset_buffer_index_addr_offset);
7561 g_assert (store_remset_buffer_index_addr_offset != -1);
7565 // FIXME: Maybe create a separate version for ctors (the branch would be
7566 // correctly predicted more times)
7567 if (write_barrier_method)
7568 return write_barrier_method;
7570 /* Create the IL version of mono_gc_barrier_generic_store () */
7571 sig = mono_metadata_signature_alloc (mono_defaults.corlib, 1);
7572 sig->ret = &mono_defaults.void_class->byval_arg;
7573 sig->params [0] = &mono_defaults.int_class->byval_arg;
7575 mb = mono_mb_new (mono_defaults.object_class, "wbarrier", MONO_WRAPPER_WRITE_BARRIER);
7577 #ifdef MANAGED_WBARRIER
7578 if (mono_runtime_has_tls_get ()) {
7579 #ifdef ALIGN_NURSERY
7580 // if (ptr_in_nursery (ptr)) return;
7582 * Masking out the bits might be faster, but we would have to use 64 bit
7583 * immediates, which might be slower.
7585 mono_mb_emit_ldarg (mb, 0);
7586 mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
7587 mono_mb_emit_byte (mb, CEE_SHR_UN);
7588 mono_mb_emit_icon (mb, (mword)nursery_start >> DEFAULT_NURSERY_BITS);
7589 label_no_wb_1 = mono_mb_emit_branch (mb, CEE_BEQ);
7591 // if (!ptr_in_nursery (*ptr)) return;
7592 mono_mb_emit_ldarg (mb, 0);
7593 mono_mb_emit_byte (mb, CEE_LDIND_I);
7594 mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
7595 mono_mb_emit_byte (mb, CEE_SHR_UN);
7596 mono_mb_emit_icon (mb, (mword)nursery_start >> DEFAULT_NURSERY_BITS);
7597 label_no_wb_2 = mono_mb_emit_branch (mb, CEE_BNE_UN);
7600 g_assert_not_reached ();
7603 // if (ptr >= stack_end) goto need_wb;
7604 mono_mb_emit_ldarg (mb, 0);
7605 EMIT_TLS_ACCESS (mb, stack_end, stack_end_offset);
7606 label_need_wb = mono_mb_emit_branch (mb, CEE_BGE_UN);
7608 // if (ptr >= stack_start) return;
7609 dummy_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7610 mono_mb_emit_ldarg (mb, 0);
7611 mono_mb_emit_ldloc_addr (mb, dummy_var);
7612 label_no_wb_3 = mono_mb_emit_branch (mb, CEE_BGE_UN);
7615 mono_mb_patch_branch (mb, label_need_wb);
7617 // buffer = STORE_REMSET_BUFFER;
7618 buffer_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7619 EMIT_TLS_ACCESS (mb, store_remset_buffer, store_remset_buffer_offset);
7620 mono_mb_emit_stloc (mb, buffer_var);
7622 // buffer_index = STORE_REMSET_BUFFER_INDEX;
7623 buffer_index_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7624 EMIT_TLS_ACCESS (mb, store_remset_buffer_index, store_remset_buffer_index_offset);
7625 mono_mb_emit_stloc (mb, buffer_index_var);
7627 // if (buffer [buffer_index] == ptr) return;
7628 mono_mb_emit_ldloc (mb, buffer_var);
7629 mono_mb_emit_ldloc (mb, buffer_index_var);
7630 g_assert (sizeof (gpointer) == 4 || sizeof (gpointer) == 8);
7631 mono_mb_emit_icon (mb, sizeof (gpointer) == 4 ? 2 : 3);
7632 mono_mb_emit_byte (mb, CEE_SHL);
7633 mono_mb_emit_byte (mb, CEE_ADD);
7634 mono_mb_emit_byte (mb, CEE_LDIND_I);
7635 mono_mb_emit_ldarg (mb, 0);
7636 label_no_wb_4 = mono_mb_emit_branch (mb, CEE_BEQ);
7639 mono_mb_emit_ldloc (mb, buffer_index_var);
7640 mono_mb_emit_icon (mb, 1);
7641 mono_mb_emit_byte (mb, CEE_ADD);
7642 mono_mb_emit_stloc (mb, buffer_index_var);
7644 // if (buffer_index >= STORE_REMSET_BUFFER_SIZE) goto slow_path;
7645 mono_mb_emit_ldloc (mb, buffer_index_var);
7646 mono_mb_emit_icon (mb, STORE_REMSET_BUFFER_SIZE);
7647 label_slow_path = mono_mb_emit_branch (mb, CEE_BGE);
7649 // buffer [buffer_index] = ptr;
7650 mono_mb_emit_ldloc (mb, buffer_var);
7651 mono_mb_emit_ldloc (mb, buffer_index_var);
7652 g_assert (sizeof (gpointer) == 4 || sizeof (gpointer) == 8);
7653 mono_mb_emit_icon (mb, sizeof (gpointer) == 4 ? 2 : 3);
7654 mono_mb_emit_byte (mb, CEE_SHL);
7655 mono_mb_emit_byte (mb, CEE_ADD);
7656 mono_mb_emit_ldarg (mb, 0);
7657 mono_mb_emit_byte (mb, CEE_STIND_I);
7659 // STORE_REMSET_BUFFER_INDEX = buffer_index;
7660 EMIT_TLS_ACCESS (mb, store_remset_buffer_index_addr, store_remset_buffer_index_addr_offset);
7661 mono_mb_emit_ldloc (mb, buffer_index_var);
7662 mono_mb_emit_byte (mb, CEE_STIND_I);
7665 mono_mb_patch_branch (mb, label_no_wb_1);
7666 mono_mb_patch_branch (mb, label_no_wb_2);
7667 mono_mb_patch_branch (mb, label_no_wb_3);
7668 mono_mb_patch_branch (mb, label_no_wb_4);
7669 mono_mb_emit_byte (mb, CEE_RET);
7672 mono_mb_patch_branch (mb, label_slow_path);
7676 mono_mb_emit_ldarg (mb, 0);
7677 mono_mb_emit_icall (mb, mono_gc_wbarrier_generic_nostore);
7678 mono_mb_emit_byte (mb, CEE_RET);
7680 res = mono_mb_create_method (mb, sig, 16);
7683 mono_loader_lock ();
7684 if (write_barrier_method) {
7685 /* Already created */
7686 mono_free_method (res);
7688 /* double-checked locking */
7689 mono_memory_barrier ();
7690 write_barrier_method = res;
7692 mono_loader_unlock ();
7694 return write_barrier_method;
7697 #endif /* HAVE_SGEN_GC */