2 * sgen-gc.c: Simple generational GC.
5 * Paolo Molaro (lupus@ximian.com)
7 * Copyright 2005-2010 Novell, Inc (http://www.novell.com)
9 * Thread start/stop adapted from Boehm's GC:
10 * Copyright (c) 1994 by Xerox Corporation. All rights reserved.
11 * Copyright (c) 1996 by Silicon Graphics. All rights reserved.
12 * Copyright (c) 1998 by Fergus Henderson. All rights reserved.
13 * Copyright (c) 2000-2004 by Hewlett-Packard Company. All rights reserved.
15 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
16 * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
18 * Permission is hereby granted to use or copy this program
19 * for any purpose, provided the above notices are retained on all copies.
20 * Permission to modify the code and to distribute modified code is granted,
21 * provided the above notices are retained, and a notice that the code was
22 * modified is included with the above copyright notice.
25 * Copyright 2001-2003 Ximian, Inc
26 * Copyright 2003-2010 Novell, Inc.
28 * Permission is hereby granted, free of charge, to any person obtaining
29 * a copy of this software and associated documentation files (the
30 * "Software"), to deal in the Software without restriction, including
31 * without limitation the rights to use, copy, modify, merge, publish,
32 * distribute, sublicense, and/or sell copies of the Software, and to
33 * permit persons to whom the Software is furnished to do so, subject to
34 * the following conditions:
36 * The above copyright notice and this permission notice shall be
37 * included in all copies or substantial portions of the Software.
39 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
40 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
41 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
42 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
43 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
44 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
45 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
48 * Important: allocation provides always zeroed memory, having to do
49 * a memset after allocation is deadly for performance.
50 * Memory usage at startup is currently as follows:
52 * 64 KB internal space
54 * We should provide a small memory config with half the sizes
56 * We currently try to make as few mono assumptions as possible:
57 * 1) 2-word header with no GC pointers in it (first vtable, second to store the
59 * 2) gc descriptor is the second word in the vtable (first word in the class)
60 * 3) 8 byte alignment is the minimum and enough (not true for special structures, FIXME)
61 * 4) there is a function to get an object's size and the number of
62 * elements in an array.
63 * 5) we know the special way bounds are allocated for complex arrays
65 * Always try to keep stack usage to a minimum: no recursive behaviour
66 * and no large stack allocs.
68 * General description.
69 * Objects are initially allocated in a nursery using a fast bump-pointer technique.
70 * When the nursery is full we start a nursery collection: this is performed with a
72 * When the old generation is full we start a copying GC of the old generation as well:
73 * this will be changed to mark/compact in the future.
74 * The things that complicate this description are:
75 * *) pinned objects: we can't move them so we need to keep track of them
76 * *) no precise info of the thread stacks and registers: we need to be able to
77 * quickly find the objects that may be referenced conservatively and pin them
78 * (this makes the first issues more important)
79 * *) large objects are too expensive to be dealt with using copying GC: we handle them
80 * with mark/sweep during major collections
81 * *) some objects need to not move even if they are small (interned strings, Type handles):
82 * we use mark/sweep for them, too: they are not allocated in the nursery, but inside
83 * PinnedChunks regions
88 *) change the jit to emit write barrier calls when needed (we
89 can have specialized write barriers): done with icalls, still need to
90 use some specialized barriers
91 *) we could have a function pointer in MonoClass to implement
92 customized write barriers for value types
93 *) the write barrier code could be isolated in a couple of functions: when a
94 thread is stopped if it's inside the barrier it is let go again
95 until we stop outside of them (not really needed, see below GC-safe points)
96 *) investigate the stuff needed to advance a thread to a GC-safe
97 point (single-stepping, read from unmapped memory etc) and implement it
98 Not needed yet: since we treat the objects reachable from the stack/regs as
99 roots, we store the ptr and exec the write barrier so there is no race.
100 We may need this to solve the issue with setting the length of arrays and strings.
101 We may need this also for handling precise info on stacks, even simple things
102 as having uninitialized data on the stack and having to wait for the prolog
103 to zero it. Not an issue for the last frame that we scan conservatively.
104 We could always not trust the value in the slots anyway.
105 *) make the jit info table lock free
106 *) modify the jit to save info about references in stack locations:
107 this can be done just for locals as a start, so that at least
108 part of the stack is handled precisely.
109 *) Make the debug printf stuff thread and signal safe.
110 *) test/fix 64 bit issues
111 *) test/fix endianess issues
113 *) add batch moving profile info
114 *) add more timing info
115 *) there is a possible race when an array or string is created: the vtable is set,
116 but the length is set only later so if the GC needs to scan the object in that window,
117 it won't get the correct size for the object. The object can't have references and it will
118 be pinned, but a free memory fragment may be created that overlaps with it.
119 We should change the array max_length field to be at the same offset as the string length:
120 this way we can have a single special alloc function for them that sets the length.
121 Multi-dim arrays have the same issue for rank == 1 for the bounds data.
122 *) implement a card table as the write barrier instead of remembered sets?
123 *) some sort of blacklist support?
124 *) fin_ready_list and critical_fin_list are part of the root set, too
125 *) consider lowering the large object min size to 16/32KB or so and benchmark
126 *) once mark-compact is implemented we could still keep the
127 copying collector for the old generation and use it if we think
128 it is better (small heaps and no pinning object in the old
130 *) avoid the memory store from copy_object when not needed.
131 *) optimize the write barriers fastpath to happen in managed code
132 *) add an option to mmap the whole heap in one chunk: it makes for many
133 simplifications in the checks (put the nursery at the top and just use a single
134 check for inclusion/exclusion): the issue this has is that on 32 bit systems it's
135 not flexible (too much of the address space may be used by default or we can't
136 increase the heap as needed) and we'd need a race-free mechanism to return memory
137 back to the system (mprotect(PROT_NONE) will still keep the memory allocated if it
138 was written to, munmap is needed, but the following mmap may not find the same segment
140 *) memzero the fragments after restarting the world and optionally a smaller chunk at a time
141 *) an additional strategy to realloc/expand the nursery when fully pinned is to start
142 allocating objects in the old generation. This means that we can't optimize away write
143 barrier calls in ctors (but that is not valid for other reasons, too).
144 *) add write barriers to the Clone methods
152 #include <semaphore.h>
156 #include "metadata/metadata-internals.h"
157 #include "metadata/class-internals.h"
158 #include "metadata/gc-internal.h"
159 #include "metadata/object-internals.h"
160 #include "metadata/threads.h"
161 #include "metadata/sgen-gc.h"
162 #include "metadata/sgen-archdep.h"
163 #include "metadata/mono-gc.h"
164 #include "metadata/method-builder.h"
165 #include "metadata/profiler-private.h"
166 #include "metadata/monitor.h"
167 #include "metadata/threadpool-internals.h"
168 #include "metadata/mempool-internals.h"
169 #include "metadata/marshal.h"
170 #include "utils/mono-mmap.h"
171 #include "utils/mono-time.h"
172 #include "utils/mono-semaphore.h"
173 #include "utils/mono-counters.h"
175 #include <mono/utils/memcheck.h>
177 #define OPDEF(a,b,c,d,e,f,g,h,i,j) \
181 #include "mono/cil/opcode.def"
188 * ######################################################################
189 * ######## Types and constants used by the GC.
190 * ######################################################################
192 #if SIZEOF_VOID_P == 4
193 typedef guint32 mword;
195 typedef guint64 mword;
198 static int gc_initialized = 0;
199 static int gc_debug_level = 0;
200 static FILE* gc_debug_file;
201 /* If set, do a minor collection before every allocation */
202 static gboolean collect_before_allocs = FALSE;
203 /* If set, do a heap consistency check before each minor collection */
204 static gboolean consistency_check_at_minor_collection = FALSE;
205 /* If set, check that there are no references to the domain left at domain unload */
206 static gboolean xdomain_checks = FALSE;
207 /* If not null, dump the heap after each collection into this file */
208 static FILE *heap_dump_file = NULL;
209 /* If set, mark stacks conservatively, even if precise marking is possible */
210 static gboolean conservative_stack_mark = FALSE;
211 /* If set, do a plausibility check on the scan_starts before and after
213 static gboolean do_scan_starts_check = FALSE;
216 * Turning on heavy statistics will turn off the managed allocator and
217 * the managed write barrier.
219 //#define HEAVY_STATISTICS
221 #ifdef HEAVY_STATISTICS
222 #define HEAVY_STAT(x) x
224 #define HEAVY_STAT(x)
227 #ifdef HEAVY_STATISTICS
228 static long stat_objects_alloced = 0;
229 static long stat_objects_alloced_degraded = 0;
230 static long stat_copy_object_called_nursery = 0;
231 static long stat_objects_copied_nursery = 0;
232 static long stat_copy_object_called_major = 0;
233 static long stat_objects_copied_major = 0;
235 static long stat_copy_object_failed_from_space = 0;
236 static long stat_copy_object_failed_forwarded = 0;
237 static long stat_copy_object_failed_pinned = 0;
238 static long stat_copy_object_failed_large_pinned = 0;
239 static long stat_copy_object_failed_to_space = 0;
241 static long stat_store_remsets = 0;
242 static long stat_store_remsets_unique = 0;
243 static long stat_saved_remsets_1 = 0;
244 static long stat_saved_remsets_2 = 0;
245 static long stat_global_remsets_added = 0;
246 static long stat_global_remsets_processed = 0;
248 static long num_copy_object_called = 0;
249 static long num_objects_copied = 0;
251 static int stat_wbarrier_set_field = 0;
252 static int stat_wbarrier_set_arrayref = 0;
253 static int stat_wbarrier_arrayref_copy = 0;
254 static int stat_wbarrier_generic_store = 0;
255 static int stat_wbarrier_generic_store_remset = 0;
256 static int stat_wbarrier_set_root = 0;
257 static int stat_wbarrier_value_copy = 0;
258 static int stat_wbarrier_object_copy = 0;
261 static long pinned_chunk_bytes_alloced = 0;
262 static long large_internal_bytes_alloced = 0;
265 INTERNAL_MEM_PIN_QUEUE,
266 INTERNAL_MEM_FRAGMENT,
267 INTERNAL_MEM_SECTION,
268 INTERNAL_MEM_SCAN_STARTS,
269 INTERNAL_MEM_FIN_TABLE,
270 INTERNAL_MEM_FINALIZE_ENTRY,
271 INTERNAL_MEM_DISLINK_TABLE,
272 INTERNAL_MEM_DISLINK,
273 INTERNAL_MEM_ROOTS_TABLE,
274 INTERNAL_MEM_ROOT_RECORD,
275 INTERNAL_MEM_STATISTICS,
277 INTERNAL_MEM_GRAY_QUEUE,
278 INTERNAL_MEM_STORE_REMSET,
282 static long small_internal_mem_bytes [INTERNAL_MEM_MAX];
286 mono_gc_flush_info (void)
288 fflush (gc_debug_file);
292 #define MAX_DEBUG_LEVEL 8
293 #define DEBUG(level,a) do {if (G_UNLIKELY ((level) <= MAX_DEBUG_LEVEL && (level) <= gc_debug_level)) a;} while (0)
295 #define TV_DECLARE(name) gint64 name
296 #define TV_GETTIME(tv) tv = mono_100ns_ticks ()
297 #define TV_ELAPSED(start,end) (int)((end-start) / 10)
299 #define ALIGN_TO(val,align) ((((guint64)val) + ((align) - 1)) & ~((align) - 1))
301 #define GC_BITS_PER_WORD (sizeof (mword) * 8)
309 typedef struct _Block Block;
315 /* each request from the OS ends up in a GCMemSection */
316 typedef struct _GCMemSection GCMemSection;
317 struct _GCMemSection {
321 /* pointer where more data could be allocated if it fits */
325 * scan starts is an array of pointers to objects equally spaced in the allocation area
326 * They let use quickly find pinned objects from pinning pointers.
329 /* in major collections indexes in the pin_queue for objects that pin this section */
332 unsigned short num_scan_start;
333 gboolean is_to_space;
336 #define SIZEOF_GC_MEM_SECTION ((sizeof (GCMemSection) + 7) & ~7)
338 /* large object space struct: 64+ KB */
339 /* we could make this limit much smaller to avoid memcpy copy
340 * and potentially have more room in the GC descriptor: need to measure
341 * This also means that such small OS objects will need to be
342 * allocated in a different way (using pinned chunks).
343 * We may want to put large but smaller than 64k objects in the fixed space
344 * when we move the object from one generation to another (to limit the
345 * pig in the snake effect).
346 * Note: it may be worth to have an optimized copy function, since we can
347 * assume that objects are aligned and have a multiple of 8 size.
348 * FIXME: This structure needs to be a multiple of 8 bytes in size: this is not
349 * true if MONO_ZERO_LEN_ARRAY is nonzero.
351 typedef struct _LOSObject LOSObject;
354 mword size; /* this is the object size */
355 int dummy; /* to have a sizeof (LOSObject) a multiple of ALLOC_ALIGN and data starting at same alignment */
358 char data [MONO_ZERO_LEN_ARRAY];
361 /* Pinned objects are allocated in the LOS space if bigger than half a page
362 * or from freelists otherwise. We assume that pinned objects are relatively few
363 * and they have a slow dying speed (like interned strings, thread objects).
364 * As such they will be collected only at major collections.
365 * free lists are not global: when we need memory we allocate a PinnedChunk.
366 * Each pinned chunk is made of several pages, the first of wich is used
367 * internally for bookeeping (here think of a page as 4KB). The bookeeping
368 * includes the freelists vectors and info about the object size of each page
369 * in the pinned chunk. So, when needed, a free page is found in a pinned chunk,
370 * a size is assigned to it, the page is divided in the proper chunks and each
371 * chunk is added to the freelist. To not waste space, the remaining space in the
372 * first page is used as objects of size 16 or 32 (need to measure which are more
374 * We use this same structure to allocate memory used internally by the GC, so
375 * we never use malloc/free if we need to alloc during collection: the world is stopped
376 * and malloc/free will deadlock.
377 * When we want to iterate over pinned objects, we just scan a page at a time
378 * linearly according to the size of objects in the page: the next pointer used to link
379 * the items in the freelist uses the same word as the vtable. Since we keep freelists
380 * for each pinned chunk, if the word points outside the pinned chunk it means
382 * We could avoid this expensive scanning in creative ways. We could have a policy
383 * of putting in the pinned space only objects we know about that have no struct fields
384 * with references and we can easily use a even expensive write barrier for them,
385 * since pointer writes on such objects should be rare.
386 * The best compromise is to just alloc interned strings and System.MonoType in them.
387 * It would be nice to allocate MonoThread in it, too: must check that we properly
388 * use write barriers so we don't have to do any expensive scanning of the whole pinned
389 * chunk list during minor collections. We can avoid it now because we alloc in it only
390 * reference-free objects.
392 #define PINNED_FIRST_SLOT_SIZE (sizeof (gpointer) * 4)
393 #define MAX_FREELIST_SIZE 2048
394 #define PINNED_PAGE_SIZE (4096)
395 #define PINNED_CHUNK_MIN_SIZE (4096*8)
396 typedef struct _PinnedChunk PinnedChunk;
397 struct _PinnedChunk {
400 int *page_sizes; /* a 0 means the page is still unused */
403 void *data [1]; /* page sizes and free lists are stored here */
406 /* The method used to clear the nursery */
407 /* Clearing at nursery collections is the safest, but has bad interactions with caches.
408 * Clearing at TLAB creation is much faster, but more complex and it might expose hard
413 CLEAR_AT_TLAB_CREATION
414 } NurseryClearPolicy;
416 static NurseryClearPolicy nursery_clear_policy = CLEAR_AT_TLAB_CREATION;
419 * If this is set, the nursery is aligned to an address aligned to its size, ie.
420 * a 1MB nursery will be aligned to an address divisible by 1MB. This allows us to
421 * speed up ptr_in_nursery () checks which are very frequent. This requires the
422 * nursery size to be a compile time constant.
424 #define ALIGN_NURSERY 1
427 * The young generation is divided into fragments. This is because
428 * we can hand one fragments to a thread for lock-less fast alloc and
429 * because the young generation ends up fragmented anyway by pinned objects.
430 * Once a collection is done, a list of fragments is created. When doing
431 * thread local alloc we use smallish nurseries so we allow new threads to
432 * allocate memory from gen0 without triggering a collection. Threads that
433 * are found to allocate lots of memory are given bigger fragments. This
434 * should make the finalizer thread use little nursery memory after a while.
435 * We should start assigning threads very small fragments: if there are many
436 * threads the nursery will be full of reserved space that the threads may not
437 * use at all, slowing down allocation speed.
438 * Thread local allocation is done from areas of memory Hotspot calls Thread Local
439 * Allocation Buffers (TLABs).
441 typedef struct _Fragment Fragment;
445 char *fragment_start;
446 char *fragment_limit; /* the current soft limit for allocation */
450 /* the runtime can register areas of memory as roots: we keep two lists of roots,
451 * a pinned root set for conservatively scanned roots and a normal one for
452 * precisely scanned roots (currently implemented as a single list).
454 typedef struct _RootRecord RootRecord;
462 /* for use with write barriers */
463 typedef struct _RememberedSet RememberedSet;
464 struct _RememberedSet {
468 mword data [MONO_ZERO_LEN_ARRAY];
472 * We're never actually using the first element. It's always set to
473 * NULL to simplify the elimination of consecutive duplicate
476 #define STORE_REMSET_BUFFER_SIZE 1024
478 typedef struct _GenericStoreRememberedSet GenericStoreRememberedSet;
479 struct _GenericStoreRememberedSet {
480 GenericStoreRememberedSet *next;
481 /* We need one entry less because the first entry of store
482 remset buffers is always a dummy and we don't copy it. */
483 gpointer data [STORE_REMSET_BUFFER_SIZE - 1];
486 /* we have 4 possible values in the low 2 bits */
488 REMSET_LOCATION, /* just a pointer to the exact location */
489 REMSET_RANGE, /* range of pointer fields */
490 REMSET_OBJECT, /* mark all the object for scanning */
491 REMSET_OTHER, /* all others */
492 REMSET_TYPE_MASK = 0x3
495 /* Subtypes of REMSET_OTHER */
497 REMSET_VTYPE, /* a valuetype array described by a gc descriptor and a count */
498 REMSET_ROOT_LOCATION, /* a location inside a root */
501 #ifdef HAVE_KW_THREAD
502 static __thread RememberedSet *remembered_set MONO_TLS_FAST;
504 static pthread_key_t remembered_set_key;
505 static RememberedSet *global_remset;
506 static RememberedSet *freed_thread_remsets;
507 //static int store_to_global_remset = 0;
508 static GenericStoreRememberedSet *generic_store_remsets = NULL;
510 /* FIXME: later choose a size that takes into account the RememberedSet struct
511 * and doesn't waste any alloc paddin space.
513 #define DEFAULT_REMSET_SIZE 1024
514 static RememberedSet* alloc_remset (int size, gpointer id);
516 /* Structure that corresponds to a MonoVTable: desc is a mword so requires
517 * no cast from a pointer to an integer
524 /* these bits are set in the object vtable: we could merge them since an object can be
525 * either pinned or forwarded but not both.
526 * We store them in the vtable slot because the bits are used in the sync block for
527 * other purposes: if we merge them and alloc the sync blocks aligned to 8 bytes, we can change
528 * this and use bit 3 in the syncblock (with the lower two bits both set for forwarded, that
529 * would be an invalid combination for the monitor and hash code).
530 * The values are already shifted.
531 * The forwarding address is stored in the sync block.
533 #define FORWARDED_BIT 1
535 #define VTABLE_BITS_MASK 0x3
537 /* returns NULL if not forwarded, or the forwarded address */
538 #define object_is_forwarded(obj) (((mword*)(obj))[0] & FORWARDED_BIT? (void*)(((mword*)(obj))[1]): NULL)
539 /* set the forwarded address fw_addr for object obj */
540 #define forward_object(obj,fw_addr) do { \
541 ((mword*)(obj))[0] |= FORWARDED_BIT; \
542 ((mword*)(obj))[1] = (mword)(fw_addr); \
545 #define object_is_pinned(obj) (((mword*)(obj))[0] & PINNED_BIT)
546 #define pin_object(obj) do { \
547 ((mword*)(obj))[0] |= PINNED_BIT; \
549 #define unpin_object(obj) do { \
550 ((mword*)(obj))[0] &= ~PINNED_BIT; \
554 #define ptr_in_nursery(ptr) (((mword)(ptr) & ~((1 << DEFAULT_NURSERY_BITS) - 1)) == (mword)nursery_start)
556 #define ptr_in_nursery(ptr) ((char*)(ptr) >= nursery_start && (char*)(ptr) < nursery_real_end)
560 * Since we set bits in the vtable, use the macro to load it from the pointer to
561 * an object that is potentially pinned.
563 #define LOAD_VTABLE(addr) ((*(mword*)(addr)) & ~VTABLE_BITS_MASK)
566 safe_name (void* obj)
568 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (obj);
569 return vt->klass->name;
573 safe_object_get_size (MonoObject* o)
575 MonoClass *klass = ((MonoVTable*)LOAD_VTABLE (o))->klass;
576 if (klass == mono_defaults.string_class) {
577 return sizeof (MonoString) + 2 * mono_string_length ((MonoString*) o) + 2;
578 } else if (klass->rank) {
579 MonoArray *array = (MonoArray*)o;
580 size_t size = sizeof (MonoArray) + klass->sizes.element_size * mono_array_length (array);
581 if (G_UNLIKELY (array->bounds)) {
582 size += sizeof (mono_array_size_t) - 1;
583 size &= ~(sizeof (mono_array_size_t) - 1);
584 size += sizeof (MonoArrayBounds) * klass->rank;
588 /* from a created object: the class must be inited already */
589 return klass->instance_size;
594 * ######################################################################
595 * ######## Global data.
596 * ######################################################################
598 static LOCK_DECLARE (gc_mutex);
599 static int gc_disabled = 0;
600 static int num_minor_gcs = 0;
601 static int num_major_gcs = 0;
603 /* good sizes are 512KB-1MB: larger ones increase a lot memzeroing time */
604 //#define DEFAULT_NURSERY_SIZE (1024*512*125+4096*118)
605 #define DEFAULT_NURSERY_SIZE (1024*512*2)
606 /* The number of trailing 0 bits in DEFAULT_NURSERY_SIZE */
607 #define DEFAULT_NURSERY_BITS 20
608 #define MAJOR_SECTION_SIZE (128*1024)
609 #define BLOCK_FOR_OBJECT(o) ((Block*)(((mword)(o)) & ~(MAJOR_SECTION_SIZE - 1)))
610 #define MAJOR_SECTION_FOR_OBJECT(o) ((GCMemSection*)BLOCK_FOR_OBJECT ((o)))
611 #define MIN_MINOR_COLLECTION_SECTION_ALLOWANCE (DEFAULT_NURSERY_SIZE * 3 / MAJOR_SECTION_SIZE)
612 #define DEFAULT_LOS_COLLECTION_TARGET (DEFAULT_NURSERY_SIZE * 2)
613 /* to quickly find the head of an object pinned by a conservative address
614 * we keep track of the objects allocated for each SCAN_START_SIZE memory
615 * chunk in the nursery or other memory sections. Larger values have less
616 * memory overhead and bigger runtime cost. 4-8 KB are reasonable values.
618 #define SCAN_START_SIZE (4096*2)
619 /* the minimum size of a fragment that we consider useful for allocation */
620 #define FRAGMENT_MIN_SIZE (512)
621 /* This is a fixed value used for pinned chunks, not the system pagesize */
622 #define FREELIST_PAGESIZE 4096
624 static mword pagesize = 4096;
625 static mword nursery_size = DEFAULT_NURSERY_SIZE;
626 static int degraded_mode = 0;
628 static int minor_collection_section_allowance = MIN_MINOR_COLLECTION_SECTION_ALLOWANCE;
629 static int minor_collection_sections_alloced = 0;
630 static int num_major_sections = 0;
632 static LOSObject *los_object_list = NULL;
633 static mword los_memory_usage = 0;
634 static mword los_num_objects = 0;
635 static mword next_los_collection = 2*1024*1024; /* 2 MB, need to tune */
636 static mword total_alloc = 0;
637 /* use this to tune when to do a major/minor collection */
638 static mword memory_pressure = 0;
640 static GCMemSection *section_list = NULL;
641 static GCMemSection *nursery_section = NULL;
642 static mword lowest_heap_address = ~(mword)0;
643 static mword highest_heap_address = 0;
645 typedef struct _FinalizeEntry FinalizeEntry;
646 struct _FinalizeEntry {
651 typedef struct _FinalizeEntryHashTable FinalizeEntryHashTable;
652 struct _FinalizeEntryHashTable {
653 FinalizeEntry **table;
658 typedef struct _DisappearingLink DisappearingLink;
659 struct _DisappearingLink {
660 DisappearingLink *next;
664 typedef struct _DisappearingLinkHashTable DisappearingLinkHashTable;
665 struct _DisappearingLinkHashTable {
666 DisappearingLink **table;
671 #define LARGE_INTERNAL_MEM_HEADER_MAGIC 0x7d289f3a
673 typedef struct _LargeInternalMemHeader LargeInternalMemHeader;
674 struct _LargeInternalMemHeader {
687 * The link pointer is hidden by negating each bit. We use the lowest
688 * bit of the link (before negation) to store whether it needs
689 * resurrection tracking.
691 #define HIDE_POINTER(p,t) ((gpointer)(~((gulong)(p)|((t)?1:0))))
692 #define REVEAL_POINTER(p) ((gpointer)((~(gulong)(p))&~3L))
694 #define DISLINK_OBJECT(d) (REVEAL_POINTER (*(d)->link))
695 #define DISLINK_TRACK(d) ((~(gulong)(*(d)->link)) & 1)
698 * The finalizable hash has the object as the key, the
699 * disappearing_link hash, has the link address as key.
701 static FinalizeEntryHashTable minor_finalizable_hash;
702 static FinalizeEntryHashTable major_finalizable_hash;
703 /* objects that are ready to be finalized */
704 static FinalizeEntry *fin_ready_list = NULL;
705 static FinalizeEntry *critical_fin_list = NULL;
707 static DisappearingLinkHashTable minor_disappearing_link_hash;
708 static DisappearingLinkHashTable major_disappearing_link_hash;
710 static int num_ready_finalizers = 0;
711 static int no_finalize = 0;
713 /* keep each size a multiple of ALLOC_ALIGN */
714 /* on 64 bit systems 8 is likely completely unused. */
715 static const int freelist_sizes [] = {
716 8, 16, 24, 32, 40, 48, 64, 80,
717 96, 128, 160, 192, 224, 256, 320, 384,
718 448, 512, 584, 680, 816, 1024, 1360, 2048};
719 #define FREELIST_NUM_SLOTS (sizeof (freelist_sizes) / sizeof (freelist_sizes [0]))
721 static char* max_pinned_chunk_addr = NULL;
722 static char* min_pinned_chunk_addr = (char*)-1;
723 /* pinned_chunk_list is used for allocations of objects that are never moved */
724 static PinnedChunk *pinned_chunk_list = NULL;
725 /* internal_chunk_list is used for allocating structures needed by the GC */
726 static PinnedChunk *internal_chunk_list = NULL;
729 obj_is_from_pinned_alloc (char *p)
731 return BLOCK_FOR_OBJECT (p)->role == MEMORY_ROLE_PINNED;
734 static int slot_for_size (size_t size);
737 free_pinned_object (PinnedChunk *chunk, char *obj, size_t size)
739 void **p = (void**)obj;
740 int slot = slot_for_size (size);
742 g_assert (obj >= (char*)chunk->start_data && obj < ((char*)chunk + chunk->num_pages * FREELIST_PAGESIZE));
743 *p = chunk->free_list [slot];
744 chunk->free_list [slot] = p;
748 ROOT_TYPE_NORMAL = 0, /* "normal" roots */
749 ROOT_TYPE_PINNED = 1, /* roots without a GC descriptor */
750 ROOT_TYPE_WBARRIER = 2, /* roots with a write barrier */
754 /* registered roots: the key to the hash is the root start address */
756 * Different kinds of roots are kept separate to speed up pin_from_roots () for example.
758 static RootRecord **roots_hash [ROOT_TYPE_NUM] = { NULL, NULL };
759 static int roots_hash_size [ROOT_TYPE_NUM] = { 0, 0, 0 };
760 static mword roots_size = 0; /* amount of memory in the root set */
761 static int num_roots_entries [ROOT_TYPE_NUM] = { 0, 0, 0 };
764 * The current allocation cursors
765 * We allocate objects in the nursery.
766 * The nursery is the area between nursery_start and nursery_real_end.
767 * Allocation is done from a Thread Local Allocation Buffer (TLAB). TLABs are allocated
768 * from nursery fragments.
769 * tlab_next is the pointer to the space inside the TLAB where the next object will
771 * tlab_temp_end is the pointer to the end of the temporary space reserved for
772 * the allocation: it allows us to set the scan starts at reasonable intervals.
773 * tlab_real_end points to the end of the TLAB.
774 * nursery_frag_real_end points to the end of the currently used nursery fragment.
775 * nursery_first_pinned_start points to the start of the first pinned object in the nursery
776 * nursery_last_pinned_end points to the end of the last pinned object in the nursery
777 * At the next allocation, the area of the nursery where objects can be present is
778 * between MIN(nursery_first_pinned_start, first_fragment_start) and
779 * MAX(nursery_last_pinned_end, nursery_frag_real_end)
781 static char *nursery_start = NULL;
783 /* eventually share with MonoThread? */
784 typedef struct _SgenThreadInfo SgenThreadInfo;
786 struct _SgenThreadInfo {
787 SgenThreadInfo *next;
789 unsigned int stop_count; /* to catch duplicate signals */
792 volatile int in_critical_region;
795 void *stack_start_limit;
796 char **tlab_next_addr;
797 char **tlab_start_addr;
798 char **tlab_temp_end_addr;
799 char **tlab_real_end_addr;
800 gpointer **store_remset_buffer_addr;
801 long *store_remset_buffer_index_addr;
802 RememberedSet *remset;
803 gpointer runtime_data;
804 gpointer stopped_ip; /* only valid if the thread is stopped */
805 MonoDomain *stopped_domain; /* ditto */
806 gpointer *stopped_regs; /* ditto */
807 #ifndef HAVE_KW_THREAD
812 gpointer *store_remset_buffer;
813 long store_remset_buffer_index;
817 #ifdef HAVE_KW_THREAD
818 #define TLAB_ACCESS_INIT
819 #define TLAB_START tlab_start
820 #define TLAB_NEXT tlab_next
821 #define TLAB_TEMP_END tlab_temp_end
822 #define TLAB_REAL_END tlab_real_end
823 #define REMEMBERED_SET remembered_set
824 #define STORE_REMSET_BUFFER store_remset_buffer
825 #define STORE_REMSET_BUFFER_INDEX store_remset_buffer_index
826 #define IN_CRITICAL_REGION thread_info->in_critical_region
828 static pthread_key_t thread_info_key;
829 #define TLAB_ACCESS_INIT SgenThreadInfo *__thread_info__ = pthread_getspecific (thread_info_key)
830 #define TLAB_START (__thread_info__->tlab_start)
831 #define TLAB_NEXT (__thread_info__->tlab_next)
832 #define TLAB_TEMP_END (__thread_info__->tlab_temp_end)
833 #define TLAB_REAL_END (__thread_info__->tlab_real_end)
834 #define REMEMBERED_SET (__thread_info__->remset)
835 #define STORE_REMSET_BUFFER (__thread_info__->store_remset_buffer)
836 #define STORE_REMSET_BUFFER_INDEX (__thread_info__->store_remset_buffer_index)
837 #define IN_CRITICAL_REGION (__thread_info__->in_critical_region)
840 /* we use the memory barrier only to prevent compiler reordering (a memory constraint may be enough) */
841 #define ENTER_CRITICAL_REGION do {IN_CRITICAL_REGION = 1;mono_memory_barrier ();} while (0)
842 #define EXIT_CRITICAL_REGION do {IN_CRITICAL_REGION = 0;mono_memory_barrier ();} while (0)
845 * FIXME: What is faster, a TLS variable pointing to a structure, or separate TLS
846 * variables for next+temp_end ?
848 #ifdef HAVE_KW_THREAD
849 static __thread SgenThreadInfo *thread_info;
850 static __thread char *tlab_start;
851 static __thread char *tlab_next;
852 static __thread char *tlab_temp_end;
853 static __thread char *tlab_real_end;
854 static __thread gpointer *store_remset_buffer;
855 static __thread long store_remset_buffer_index;
856 /* Used by the managed allocator/wbarrier */
857 static __thread char **tlab_next_addr;
858 static __thread char *stack_end;
859 static __thread long *store_remset_buffer_index_addr;
861 static char *nursery_next = NULL;
862 static char *nursery_frag_real_end = NULL;
863 static char *nursery_real_end = NULL;
864 //static char *nursery_first_pinned_start = NULL;
865 static char *nursery_last_pinned_end = NULL;
867 /* The size of a TLAB */
868 /* The bigger the value, the less often we have to go to the slow path to allocate a new
869 * one, but the more space is wasted by threads not allocating much memory.
871 * FIXME: Make this self-tuning for each thread.
873 static guint32 tlab_size = (1024 * 4);
875 /* fragments that are free and ready to be used for allocation */
876 static Fragment *nursery_fragments = NULL;
877 /* freeelist of fragment structures */
878 static Fragment *fragment_freelist = NULL;
881 * used when moving the objects
883 static char *to_space_bumper = NULL;
884 static char *to_space_top = NULL;
885 static GCMemSection *to_space_section = NULL;
887 /* objects bigger then this go into the large object space */
888 #define MAX_SMALL_OBJ_SIZE MAX_FREELIST_SIZE
890 /* Functions supplied by the runtime to be called by the GC */
891 static MonoGCCallbacks gc_callbacks;
894 * ######################################################################
895 * ######## Macros and function declarations.
896 * ######################################################################
899 #define UPDATE_HEAP_BOUNDARIES(low,high) do { \
900 if ((mword)(low) < lowest_heap_address) \
901 lowest_heap_address = (mword)(low); \
902 if ((mword)(high) > highest_heap_address) \
903 highest_heap_address = (mword)(high); \
905 #define ADDR_IN_HEAP_BOUNDARIES(addr) ((p) >= lowest_heap_address && (p) < highest_heap_address)
908 align_pointer (void *ptr)
910 mword p = (mword)ptr;
911 p += sizeof (gpointer) - 1;
912 p &= ~ (sizeof (gpointer) - 1);
916 /* forward declarations */
917 static void* get_internal_mem (size_t size, int type);
918 static void free_internal_mem (void *addr, int type);
919 static void* get_os_memory (size_t size, int activate);
920 static void free_os_memory (void *addr, size_t size);
921 static G_GNUC_UNUSED void report_internal_mem_usage (void);
923 static int stop_world (void);
924 static int restart_world (void);
925 static void scan_thread_data (void *start_nursery, void *end_nursery, gboolean precise);
926 static void scan_from_remsets (void *start_nursery, void *end_nursery);
927 static void find_pinning_ref_from_thread (char *obj, size_t size);
928 static void update_current_thread_stack (void *start);
929 static GCMemSection* alloc_major_section (void);
930 static void finalize_in_range (char *start, char *end, int generation);
931 static void add_or_remove_disappearing_link (MonoObject *obj, void **link, gboolean track, int generation);
932 static void null_link_in_range (char *start, char *end, int generation);
933 static void null_links_for_domain (MonoDomain *domain, int generation);
934 static gboolean search_fragment_for_size (size_t size);
935 static void mark_pinned_from_addresses (PinnedChunk *chunk, void **start, void **end);
936 static void clear_remsets (void);
937 static void clear_tlabs (void);
938 typedef void (*ScanPinnedObjectCallbackFunc) (PinnedChunk*, char*, size_t, void*);
939 static void scan_pinned_objects (ScanPinnedObjectCallbackFunc callback, void *callback_data);
940 static void sweep_pinned_objects (void);
941 static void scan_from_pinned_objects (char *addr_start, char *addr_end);
942 static void free_large_object (LOSObject *obj);
943 static void free_major_section (GCMemSection *section);
944 static void to_space_expand (void);
946 static void mono_gc_register_disappearing_link (MonoObject *obj, void **link, gboolean track);
948 void describe_ptr (char *ptr);
949 void check_consistency (void);
950 char* check_object (char *start);
952 void mono_gc_scan_for_specific_ref (MonoObject *key);
955 * ######################################################################
956 * ######## GC descriptors
957 * ######################################################################
958 * Used to quickly get the info the GC needs about an object: size and
959 * where the references are held.
961 /* objects are aligned to 8 bytes boundaries
962 * A descriptor is a pointer in MonoVTable, so 32 or 64 bits of size.
963 * The low 3 bits define the type of the descriptor. The other bits
964 * depend on the type.
965 * As a general rule the 13 remaining low bits define the size, either
966 * of the whole object or of the elements in the arrays. While for objects
967 * the size is already in bytes, for arrays we need to shift, because
968 * array elements might be smaller than 8 bytes. In case of arrays, we
969 * use two bits to describe what the additional high bits represents,
970 * so the default behaviour can handle element sizes less than 2048 bytes.
971 * The high 16 bits, if 0 it means the object is pointer-free.
972 * This design should make it easy and fast to skip over ptr-free data.
973 * The first 4 types should cover >95% of the objects.
974 * Note that since the size of objects is limited to 64K, larger objects
975 * will be allocated in the large object heap.
976 * If we want 4-bytes alignment, we need to put vector and small bitmap
980 DESC_TYPE_RUN_LENGTH, /* 16 bits aligned byte size | 1-3 (offset, numptr) bytes tuples */
981 DESC_TYPE_SMALL_BITMAP, /* 16 bits aligned byte size | 16-48 bit bitmap */
982 DESC_TYPE_STRING, /* nothing */
983 DESC_TYPE_COMPLEX, /* index for bitmap into complex_descriptors */
984 DESC_TYPE_VECTOR, /* 10 bits element size | 1 bit array | 2 bits desc | element desc */
985 DESC_TYPE_ARRAY, /* 10 bits element size | 1 bit array | 2 bits desc | element desc */
986 DESC_TYPE_LARGE_BITMAP, /* | 29-61 bitmap bits */
987 DESC_TYPE_COMPLEX_ARR, /* index for bitmap into complex_descriptors */
988 /* subtypes for arrays and vectors */
989 DESC_TYPE_V_PTRFREE = 0,/* there are no refs: keep first so it has a zero value */
990 DESC_TYPE_V_REFS, /* all the array elements are refs */
991 DESC_TYPE_V_RUN_LEN, /* elements are run-length encoded as DESC_TYPE_RUN_LENGTH */
992 DESC_TYPE_V_BITMAP /* elements are as the bitmap in DESC_TYPE_SMALL_BITMAP */
995 #define OBJECT_HEADER_WORDS (sizeof(MonoObject)/sizeof(gpointer))
996 #define LOW_TYPE_BITS 3
997 #define SMALL_BITMAP_SHIFT 16
998 #define SMALL_BITMAP_SIZE (GC_BITS_PER_WORD - SMALL_BITMAP_SHIFT)
999 #define VECTOR_INFO_SHIFT 14
1000 #define VECTOR_ELSIZE_SHIFT 3
1001 #define LARGE_BITMAP_SIZE (GC_BITS_PER_WORD - LOW_TYPE_BITS)
1002 #define MAX_SMALL_SIZE ((1 << SMALL_BITMAP_SHIFT) - 1)
1003 #define SMALL_SIZE_MASK 0xfff8
1004 #define MAX_ELEMENT_SIZE 0x3ff
1005 #define ELEMENT_SIZE_MASK (0x3ff << LOW_TYPE_BITS)
1006 #define VECTOR_SUBTYPE_PTRFREE (DESC_TYPE_V_PTRFREE << VECTOR_INFO_SHIFT)
1007 #define VECTOR_SUBTYPE_REFS (DESC_TYPE_V_REFS << VECTOR_INFO_SHIFT)
1008 #define VECTOR_SUBTYPE_RUN_LEN (DESC_TYPE_V_RUN_LEN << VECTOR_INFO_SHIFT)
1009 #define VECTOR_SUBTYPE_BITMAP (DESC_TYPE_V_BITMAP << VECTOR_INFO_SHIFT)
1011 #define ALLOC_ALIGN 8
1014 /* Root bitmap descriptors are simpler: the lower three bits describe the type
1015 * and we either have 30/62 bitmap bits or nibble-based run-length,
1016 * or a complex descriptor, or a user defined marker function.
1019 ROOT_DESC_CONSERVATIVE, /* 0, so matches NULL value */
1024 ROOT_DESC_TYPE_MASK = 0x7,
1025 ROOT_DESC_TYPE_SHIFT = 3,
1028 #define MAKE_ROOT_DESC(type,val) ((type) | ((val) << ROOT_DESC_TYPE_SHIFT))
1030 #define MAX_USER_DESCRIPTORS 16
1032 static gsize* complex_descriptors = NULL;
1033 static int complex_descriptors_size = 0;
1034 static int complex_descriptors_next = 0;
1035 static MonoGCMarkFunc user_descriptors [MAX_USER_DESCRIPTORS];
1036 static int user_descriptors_next = 0;
1039 alloc_complex_descriptor (gsize *bitmap, int numbits)
1043 numbits = ALIGN_TO (numbits, GC_BITS_PER_WORD);
1044 nwords = numbits / GC_BITS_PER_WORD + 1;
1047 res = complex_descriptors_next;
1048 /* linear search, so we don't have duplicates with domain load/unload
1049 * this should not be performance critical or we'd have bigger issues
1050 * (the number and size of complex descriptors should be small).
1052 for (i = 0; i < complex_descriptors_next; ) {
1053 if (complex_descriptors [i] == nwords) {
1054 int j, found = TRUE;
1055 for (j = 0; j < nwords - 1; ++j) {
1056 if (complex_descriptors [i + 1 + j] != bitmap [j]) {
1066 i += complex_descriptors [i];
1068 if (complex_descriptors_next + nwords > complex_descriptors_size) {
1069 int new_size = complex_descriptors_size * 2 + nwords;
1070 complex_descriptors = g_realloc (complex_descriptors, new_size * sizeof (gsize));
1071 complex_descriptors_size = new_size;
1073 DEBUG (6, fprintf (gc_debug_file, "Complex descriptor %d, size: %d (total desc memory: %d)\n", res, nwords, complex_descriptors_size));
1074 complex_descriptors_next += nwords;
1075 complex_descriptors [res] = nwords;
1076 for (i = 0; i < nwords - 1; ++i) {
1077 complex_descriptors [res + 1 + i] = bitmap [i];
1078 DEBUG (6, fprintf (gc_debug_file, "\tvalue: %p\n", (void*)complex_descriptors [res + 1 + i]));
1085 * Descriptor builders.
1088 mono_gc_make_descr_for_string (gsize *bitmap, int numbits)
1090 return (void*) DESC_TYPE_STRING;
1094 mono_gc_make_descr_for_object (gsize *bitmap, int numbits, size_t obj_size)
1096 int first_set = -1, num_set = 0, last_set = -1, i;
1098 size_t stored_size = obj_size;
1099 stored_size += ALLOC_ALIGN - 1;
1100 stored_size &= ~(ALLOC_ALIGN - 1);
1101 for (i = 0; i < numbits; ++i) {
1102 if (bitmap [i / GC_BITS_PER_WORD] & ((gsize)1 << (i % GC_BITS_PER_WORD))) {
1109 if (stored_size <= MAX_SMALL_OBJ_SIZE) {
1110 /* check run-length encoding first: one byte offset, one byte number of pointers
1111 * on 64 bit archs, we can have 3 runs, just one on 32.
1112 * It may be better to use nibbles.
1114 if (first_set < 0) {
1115 desc = DESC_TYPE_RUN_LENGTH | stored_size;
1116 DEBUG (6, fprintf (gc_debug_file, "Ptrfree descriptor %p, size: %zd\n", (void*)desc, stored_size));
1117 return (void*) desc;
1118 } else if (first_set < 256 && num_set < 256 && (first_set + num_set == last_set + 1)) {
1119 desc = DESC_TYPE_RUN_LENGTH | stored_size | (first_set << 16) | (num_set << 24);
1120 DEBUG (6, fprintf (gc_debug_file, "Runlen descriptor %p, size: %zd, first set: %d, num set: %d\n", (void*)desc, stored_size, first_set, num_set));
1121 return (void*) desc;
1123 /* we know the 2-word header is ptr-free */
1124 if (last_set < SMALL_BITMAP_SIZE + OBJECT_HEADER_WORDS) {
1125 desc = DESC_TYPE_SMALL_BITMAP | stored_size | ((*bitmap >> OBJECT_HEADER_WORDS) << SMALL_BITMAP_SHIFT);
1126 DEBUG (6, fprintf (gc_debug_file, "Smallbitmap descriptor %p, size: %zd, last set: %d\n", (void*)desc, stored_size, last_set));
1127 return (void*) desc;
1130 /* we know the 2-word header is ptr-free */
1131 if (last_set < LARGE_BITMAP_SIZE + OBJECT_HEADER_WORDS) {
1132 desc = DESC_TYPE_LARGE_BITMAP | ((*bitmap >> OBJECT_HEADER_WORDS) << LOW_TYPE_BITS);
1133 DEBUG (6, fprintf (gc_debug_file, "Largebitmap descriptor %p, size: %zd, last set: %d\n", (void*)desc, stored_size, last_set));
1134 return (void*) desc;
1136 /* it's a complex object ... */
1137 desc = DESC_TYPE_COMPLEX | (alloc_complex_descriptor (bitmap, last_set + 1) << LOW_TYPE_BITS);
1138 return (void*) desc;
1141 /* If the array holds references, numbits == 1 and the first bit is set in elem_bitmap */
1143 mono_gc_make_descr_for_array (int vector, gsize *elem_bitmap, int numbits, size_t elem_size)
1145 int first_set = -1, num_set = 0, last_set = -1, i;
1146 mword desc = vector? DESC_TYPE_VECTOR: DESC_TYPE_ARRAY;
1147 for (i = 0; i < numbits; ++i) {
1148 if (elem_bitmap [i / GC_BITS_PER_WORD] & ((gsize)1 << (i % GC_BITS_PER_WORD))) {
1155 if (elem_size <= MAX_ELEMENT_SIZE) {
1156 desc |= elem_size << VECTOR_ELSIZE_SHIFT;
1158 return (void*)(desc | VECTOR_SUBTYPE_PTRFREE);
1160 /* Note: we also handle structs with just ref fields */
1161 if (num_set * sizeof (gpointer) == elem_size) {
1162 return (void*)(desc | VECTOR_SUBTYPE_REFS | ((gssize)(-1) << 16));
1164 /* FIXME: try run-len first */
1165 /* Note: we can't skip the object header here, because it's not present */
1166 if (last_set <= SMALL_BITMAP_SIZE) {
1167 return (void*)(desc | VECTOR_SUBTYPE_BITMAP | (*elem_bitmap << 16));
1170 /* it's am array of complex structs ... */
1171 desc = DESC_TYPE_COMPLEX_ARR;
1172 desc |= alloc_complex_descriptor (elem_bitmap, last_set + 1) << LOW_TYPE_BITS;
1173 return (void*) desc;
1176 /* Return the bitmap encoded by a descriptor */
1178 mono_gc_get_bitmap_for_descr (void *descr, int *numbits)
1180 mword d = (mword)descr;
1184 case DESC_TYPE_RUN_LENGTH: {
1185 int first_set = (d >> 16) & 0xff;
1186 int num_set = (d >> 24) & 0xff;
1189 bitmap = g_new0 (gsize, (first_set + num_set + 7) / 8);
1191 for (i = first_set; i < first_set + num_set; ++i)
1192 bitmap [i / GC_BITS_PER_WORD] |= ((gsize)1 << (i % GC_BITS_PER_WORD));
1194 *numbits = first_set + num_set;
1198 case DESC_TYPE_SMALL_BITMAP:
1199 bitmap = g_new0 (gsize, 1);
1201 bitmap [0] = (d >> SMALL_BITMAP_SHIFT) << OBJECT_HEADER_WORDS;
1203 *numbits = GC_BITS_PER_WORD;
1207 g_assert_not_reached ();
1211 /* helper macros to scan and traverse objects, macros because we resue them in many functions */
1212 #define STRING_SIZE(size,str) do { \
1213 (size) = sizeof (MonoString) + 2 * mono_string_length ((MonoString*)(str)) + 2; \
1214 (size) += (ALLOC_ALIGN - 1); \
1215 (size) &= ~(ALLOC_ALIGN - 1); \
1218 #define OBJ_RUN_LEN_SIZE(size,desc,obj) do { \
1219 (size) = (desc) & 0xfff8; \
1222 #define OBJ_BITMAP_SIZE(size,desc,obj) do { \
1223 (size) = (desc) & 0xfff8; \
1226 //#define PREFETCH(addr) __asm__ __volatile__ (" prefetchnta %0": : "m"(*(char *)(addr)))
1227 #define PREFETCH(addr)
1229 /* code using these macros must define a HANDLE_PTR(ptr) macro that does the work */
1230 #define OBJ_RUN_LEN_FOREACH_PTR(desc,obj) do { \
1231 if ((desc) & 0xffff0000) { \
1232 /* there are pointers */ \
1233 void **_objptr_end; \
1234 void **_objptr = (void**)(obj); \
1235 _objptr += ((desc) >> 16) & 0xff; \
1236 _objptr_end = _objptr + (((desc) >> 24) & 0xff); \
1237 while (_objptr < _objptr_end) { \
1238 HANDLE_PTR (_objptr, (obj)); \
1244 /* a bitmap desc means that there are pointer references or we'd have
1245 * choosen run-length, instead: add an assert to check.
1247 #define OBJ_BITMAP_FOREACH_PTR(desc,obj) do { \
1248 /* there are pointers */ \
1249 void **_objptr = (void**)(obj); \
1250 gsize _bmap = (desc) >> 16; \
1251 _objptr += OBJECT_HEADER_WORDS; \
1253 if ((_bmap & 1)) { \
1254 HANDLE_PTR (_objptr, (obj)); \
1261 #define OBJ_LARGE_BITMAP_FOREACH_PTR(vt,obj) do { \
1262 /* there are pointers */ \
1263 void **_objptr = (void**)(obj); \
1264 gsize _bmap = (vt)->desc >> LOW_TYPE_BITS; \
1265 _objptr += OBJECT_HEADER_WORDS; \
1267 if ((_bmap & 1)) { \
1268 HANDLE_PTR (_objptr, (obj)); \
1275 #define OBJ_COMPLEX_FOREACH_PTR(vt,obj) do { \
1276 /* there are pointers */ \
1277 void **_objptr = (void**)(obj); \
1278 gsize *bitmap_data = complex_descriptors + ((vt)->desc >> LOW_TYPE_BITS); \
1279 int bwords = (*bitmap_data) - 1; \
1280 void **start_run = _objptr; \
1283 MonoObject *myobj = (MonoObject*)obj; \
1284 g_print ("found %d at %p (0x%zx): %s.%s\n", bwords, (obj), (vt)->desc, myobj->vtable->klass->name_space, myobj->vtable->klass->name); \
1286 while (bwords-- > 0) { \
1287 gsize _bmap = *bitmap_data++; \
1288 _objptr = start_run; \
1289 /*g_print ("bitmap: 0x%x/%d at %p\n", _bmap, bwords, _objptr);*/ \
1291 if ((_bmap & 1)) { \
1292 HANDLE_PTR (_objptr, (obj)); \
1297 start_run += GC_BITS_PER_WORD; \
1301 /* this one is untested */
1302 #define OBJ_COMPLEX_ARR_FOREACH_PTR(vt,obj) do { \
1303 /* there are pointers */ \
1304 gsize *mbitmap_data = complex_descriptors + ((vt)->desc >> LOW_TYPE_BITS); \
1305 int mbwords = (*mbitmap_data++) - 1; \
1306 int el_size = mono_array_element_size (((MonoObject*)(obj))->vtable->klass); \
1307 char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
1308 char *e_end = e_start + el_size * mono_array_length ((MonoArray*)(obj)); \
1310 MonoObject *myobj = (MonoObject*)start; \
1311 g_print ("found %d at %p (0x%zx): %s.%s\n", mbwords, (obj), (vt)->desc, myobj->vtable->klass->name_space, myobj->vtable->klass->name); \
1313 while (e_start < e_end) { \
1314 void **_objptr = (void**)e_start; \
1315 gsize *bitmap_data = mbitmap_data; \
1316 unsigned int bwords = mbwords; \
1317 while (bwords-- > 0) { \
1318 gsize _bmap = *bitmap_data++; \
1319 void **start_run = _objptr; \
1320 /*g_print ("bitmap: 0x%x\n", _bmap);*/ \
1322 if ((_bmap & 1)) { \
1323 HANDLE_PTR (_objptr, (obj)); \
1328 _objptr = start_run + GC_BITS_PER_WORD; \
1330 e_start += el_size; \
1334 #define OBJ_VECTOR_FOREACH_PTR(vt,obj) do { \
1335 /* note: 0xffffc000 excludes DESC_TYPE_V_PTRFREE */ \
1336 if ((vt)->desc & 0xffffc000) { \
1337 int el_size = ((vt)->desc >> 3) & MAX_ELEMENT_SIZE; \
1338 /* there are pointers */ \
1339 int etype = (vt)->desc & 0xc000; \
1340 if (etype == (DESC_TYPE_V_REFS << 14)) { \
1341 void **p = (void**)((char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector)); \
1342 void **end_refs = (void**)((char*)p + el_size * mono_array_length ((MonoArray*)(obj))); \
1343 /* Note: this code can handle also arrays of struct with only references in them */ \
1344 while (p < end_refs) { \
1345 HANDLE_PTR (p, (obj)); \
1348 } else if (etype == DESC_TYPE_V_RUN_LEN << 14) { \
1349 int offset = ((vt)->desc >> 16) & 0xff; \
1350 int num_refs = ((vt)->desc >> 24) & 0xff; \
1351 char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
1352 char *e_end = e_start + el_size * mono_array_length ((MonoArray*)(obj)); \
1353 while (e_start < e_end) { \
1354 void **p = (void**)e_start; \
1357 for (i = 0; i < num_refs; ++i) { \
1358 HANDLE_PTR (p + i, (obj)); \
1360 e_start += el_size; \
1362 } else if (etype == DESC_TYPE_V_BITMAP << 14) { \
1363 char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
1364 char *e_end = e_start + el_size * mono_array_length ((MonoArray*)(obj)); \
1365 while (e_start < e_end) { \
1366 void **p = (void**)e_start; \
1367 gsize _bmap = (vt)->desc >> 16; \
1368 /* Note: there is no object header here to skip */ \
1370 if ((_bmap & 1)) { \
1371 HANDLE_PTR (p, (obj)); \
1376 e_start += el_size; \
1382 #define COUNT_OBJECT_TYPES do { \
1383 switch (desc & 0x7) { \
1384 case DESC_TYPE_STRING: type_str++; break; \
1385 case DESC_TYPE_RUN_LENGTH: type_rlen++; break; \
1386 case DESC_TYPE_ARRAY: case DESC_TYPE_VECTOR: type_vector++; break; \
1387 case DESC_TYPE_SMALL_BITMAP: type_bitmap++; break; \
1388 case DESC_TYPE_LARGE_BITMAP: type_lbit++; break; \
1389 case DESC_TYPE_COMPLEX: type_complex++; break; \
1390 case DESC_TYPE_COMPLEX_ARR: type_complex++; break; \
1391 default: g_assert_not_reached (); \
1397 * ######################################################################
1398 * ######## Detecting and removing garbage.
1399 * ######################################################################
1400 * This section of code deals with detecting the objects no longer in use
1401 * and reclaiming the memory.
1405 static mword new_obj_references = 0;
1406 static mword obj_references_checked = 0;
1409 #define HANDLE_PTR(ptr,obj) do { \
1410 if (*(ptr) && (char*)*(ptr) >= nursery_start && (char*)*(ptr) < nursery_next) { \
1411 new_obj_references++; \
1412 /*printf ("bogus ptr %p found at %p in object %p (%s.%s)\n", *(ptr), (ptr), o, o->vtable->klass->name_space, o->vtable->klass->name);*/ \
1414 obj_references_checked++; \
1418 static void __attribute__((noinline))
1419 scan_area (char *start, char *end)
1422 int type_str = 0, type_rlen = 0, type_bitmap = 0, type_vector = 0, type_lbit = 0, type_complex = 0;
1423 new_obj_references = 0;
1424 obj_references_checked = 0;
1425 while (start < end) {
1426 if (!*(void**)start) {
1427 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
1430 vt = (GCVTable*)LOAD_VTABLE (start);
1431 DEBUG (8, fprintf (gc_debug_file, "Scanning object %p, vtable: %p (%s)\n", start, vt, vt->klass->name));
1433 MonoObject *obj = (MonoObject*)start;
1434 g_print ("found at %p (0x%zx): %s.%s\n", start, vt->desc, obj->vtable->klass->name_space, obj->vtable->klass->name);
1437 #define SCAN_OBJECT_ACTION COUNT_OBJECT_TYPES
1438 #include "sgen-scan-object.h"
1440 /*printf ("references to new nursery %p-%p (size: %dk): %d, checked: %d\n", old_start, end, (end-old_start)/1024, new_obj_references, obj_references_checked);
1441 printf ("\tstrings: %d, runl: %d, vector: %d, bitmaps: %d, lbitmaps: %d, complex: %d\n",
1442 type_str, type_rlen, type_vector, type_bitmap, type_lbit, type_complex);*/
1447 is_xdomain_ref_allowed (gpointer *ptr, char *obj, MonoDomain *domain)
1449 MonoObject *o = (MonoObject*)(obj);
1450 MonoObject *ref = (MonoObject*)*(ptr);
1451 int offset = (char*)(ptr) - (char*)o;
1453 if (o->vtable->klass == mono_defaults.thread_class && offset == G_STRUCT_OFFSET (MonoThread, internal_thread))
1455 if (o->vtable->klass == mono_defaults.internal_thread_class && offset == G_STRUCT_OFFSET (MonoInternalThread, current_appcontext))
1457 if (mono_class_has_parent (o->vtable->klass, mono_defaults.real_proxy_class) &&
1458 offset == G_STRUCT_OFFSET (MonoRealProxy, unwrapped_server))
1460 /* Thread.cached_culture_info */
1461 if (!strcmp (ref->vtable->klass->name_space, "System.Globalization") &&
1462 !strcmp (ref->vtable->klass->name, "CultureInfo") &&
1463 !strcmp(o->vtable->klass->name_space, "System") &&
1464 !strcmp(o->vtable->klass->name, "Object[]"))
1467 * at System.IO.MemoryStream.InternalConstructor (byte[],int,int,bool,bool) [0x0004d] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.IO/MemoryStream.cs:121
1468 * at System.IO.MemoryStream..ctor (byte[]) [0x00017] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.IO/MemoryStream.cs:81
1469 * at (wrapper remoting-invoke-with-check) System.IO.MemoryStream..ctor (byte[]) <IL 0x00020, 0xffffffff>
1470 * at System.Runtime.Remoting.Messaging.CADMethodCallMessage.GetArguments () [0x0000d] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Messaging/CADMessages.cs:327
1471 * at System.Runtime.Remoting.Messaging.MethodCall..ctor (System.Runtime.Remoting.Messaging.CADMethodCallMessage) [0x00017] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Messaging/MethodCall.cs:87
1472 * at System.AppDomain.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage,byte[]&,System.Runtime.Remoting.Messaging.CADMethodReturnMessage&) [0x00018] in /home/schani/Work/novell/trunk/mcs/class/corlib/System/AppDomain.cs:1213
1473 * at (wrapper remoting-invoke-with-check) System.AppDomain.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage,byte[]&,System.Runtime.Remoting.Messaging.CADMethodReturnMessage&) <IL 0x0003d, 0xffffffff>
1474 * at System.Runtime.Remoting.Channels.CrossAppDomainSink.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage) [0x00008] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Channels/CrossAppDomainChannel.cs:198
1475 * at (wrapper runtime-invoke) object.runtime_invoke_CrossAppDomainSink/ProcessMessageRes_object_object (object,intptr,intptr,intptr) <IL 0x0004c, 0xffffffff>
1477 if (!strcmp (ref->vtable->klass->name_space, "System") &&
1478 !strcmp (ref->vtable->klass->name, "Byte[]") &&
1479 !strcmp (o->vtable->klass->name_space, "System.IO") &&
1480 !strcmp (o->vtable->klass->name, "MemoryStream"))
1482 /* append_job() in threadpool.c */
1483 if (!strcmp (ref->vtable->klass->name_space, "System.Runtime.Remoting.Messaging") &&
1484 !strcmp (ref->vtable->klass->name, "AsyncResult") &&
1485 !strcmp (o->vtable->klass->name_space, "System") &&
1486 !strcmp (o->vtable->klass->name, "Object[]") &&
1487 mono_thread_pool_is_queue_array ((MonoArray*) o))
1493 check_reference_for_xdomain (gpointer *ptr, char *obj, MonoDomain *domain)
1495 MonoObject *o = (MonoObject*)(obj);
1496 MonoObject *ref = (MonoObject*)*(ptr);
1497 int offset = (char*)(ptr) - (char*)o;
1499 MonoClassField *field;
1502 if (!ref || ref->vtable->domain == domain)
1504 if (is_xdomain_ref_allowed (ptr, obj, domain))
1508 for (class = o->vtable->klass; class; class = class->parent) {
1511 for (i = 0; i < class->field.count; ++i) {
1512 if (class->fields[i].offset == offset) {
1513 field = &class->fields[i];
1521 if (ref->vtable->klass == mono_defaults.string_class)
1522 str = mono_string_to_utf8 ((MonoString*)ref);
1525 g_print ("xdomain reference in %p (%s.%s) at offset %d (%s) to %p (%s.%s) (%s) - pointed to by:\n",
1526 o, o->vtable->klass->name_space, o->vtable->klass->name,
1527 offset, field ? field->name : "",
1528 ref, ref->vtable->klass->name_space, ref->vtable->klass->name, str ? str : "");
1529 mono_gc_scan_for_specific_ref (o);
1535 #define HANDLE_PTR(ptr,obj) check_reference_for_xdomain ((ptr), (obj), domain)
1538 scan_object_for_xdomain_refs (char *start)
1540 MonoDomain *domain = ((MonoObject*)start)->vtable->domain;
1542 #include "sgen-scan-object.h"
1548 scan_area_for_xdomain_refs (char *start, char *end)
1550 while (start < end) {
1551 if (!*(void**)start) {
1552 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
1556 start = scan_object_for_xdomain_refs (start);
1561 #define HANDLE_PTR(ptr,obj) do { \
1562 if ((MonoObject*)*(ptr) == key) { \
1563 g_print ("found ref to %p in object %p (%s) at offset %zd\n", \
1564 key, (obj), safe_name ((obj)), ((char*)(ptr) - (char*)(obj))); \
1569 scan_object_for_specific_ref (char *start, MonoObject *key)
1571 #include "sgen-scan-object.h"
1577 scan_area_for_specific_ref (char *start, char *end, MonoObject *key)
1579 while (start < end) {
1580 if (!*(void**)start) {
1581 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
1585 start = scan_object_for_specific_ref (start, key);
1590 scan_pinned_object_for_specific_ref_callback (PinnedChunk *chunk, char *obj, size_t size, MonoObject *key)
1592 scan_object_for_specific_ref (obj, key);
1596 check_root_obj_specific_ref (RootRecord *root, MonoObject *key, MonoObject *obj)
1600 g_print ("found ref to %p in root record %p\n", key, root);
1603 static MonoObject *check_key = NULL;
1604 static RootRecord *check_root = NULL;
1607 check_root_obj_specific_ref_from_marker (void *obj)
1609 check_root_obj_specific_ref (check_root, check_key, obj);
1614 scan_roots_for_specific_ref (MonoObject *key, int root_type)
1619 for (i = 0; i < roots_hash_size [root_type]; ++i) {
1620 for (root = roots_hash [root_type][i]; root; root = root->next) {
1621 void **start_root = (void**)root->start_root;
1622 mword desc = root->root_desc;
1626 switch (desc & ROOT_DESC_TYPE_MASK) {
1627 case ROOT_DESC_BITMAP:
1628 desc >>= ROOT_DESC_TYPE_SHIFT;
1631 check_root_obj_specific_ref (root, key, *start_root);
1636 case ROOT_DESC_COMPLEX: {
1637 gsize *bitmap_data = complex_descriptors + (desc >> ROOT_DESC_TYPE_SHIFT);
1638 int bwords = (*bitmap_data) - 1;
1639 void **start_run = start_root;
1641 while (bwords-- > 0) {
1642 gsize bmap = *bitmap_data++;
1643 void **objptr = start_run;
1646 check_root_obj_specific_ref (root, key, *objptr);
1650 start_run += GC_BITS_PER_WORD;
1654 case ROOT_DESC_USER: {
1655 MonoGCMarkFunc marker = user_descriptors [desc >> ROOT_DESC_TYPE_SHIFT];
1656 marker (start_root, check_root_obj_specific_ref_from_marker);
1659 case ROOT_DESC_RUN_LEN:
1660 g_assert_not_reached ();
1662 g_assert_not_reached ();
1671 mono_gc_scan_for_specific_ref (MonoObject *key)
1673 GCMemSection *section;
1678 for (section = section_list; section; section = section->block.next)
1679 scan_area_for_specific_ref (section->data, section->end_data, key);
1681 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1682 scan_object_for_specific_ref (bigobj->data, key);
1684 scan_pinned_objects ((ScanPinnedObjectCallbackFunc)scan_pinned_object_for_specific_ref_callback, key);
1686 scan_roots_for_specific_ref (key, ROOT_TYPE_NORMAL);
1687 scan_roots_for_specific_ref (key, ROOT_TYPE_WBARRIER);
1689 for (i = 0; i < roots_hash_size [ROOT_TYPE_PINNED]; ++i) {
1690 for (root = roots_hash [ROOT_TYPE_PINNED][i]; root; root = root->next) {
1691 void **ptr = (void**)root->start_root;
1693 while (ptr < (void**)root->end_root) {
1694 check_root_obj_specific_ref (root, *ptr, key);
1701 //#define BINARY_PROTOCOL
1702 #include "sgen-protocol.c"
1705 need_remove_object_for_domain (char *start, MonoDomain *domain)
1707 if (mono_object_domain (start) == domain) {
1708 DEBUG (1, fprintf (gc_debug_file, "Need to cleanup object %p\n", start));
1709 binary_protocol_cleanup (start, (gpointer)LOAD_VTABLE (start), safe_object_get_size ((MonoObject*)start));
1716 process_object_for_domain_clearing (char *start, MonoDomain *domain)
1718 GCVTable *vt = (GCVTable*)LOAD_VTABLE (start);
1719 if (vt->klass == mono_defaults.internal_thread_class)
1720 g_assert (mono_object_domain (start) == mono_get_root_domain ());
1721 /* The object could be a proxy for an object in the domain
1723 if (mono_class_has_parent (vt->klass, mono_defaults.real_proxy_class)) {
1724 MonoObject *server = ((MonoRealProxy*)start)->unwrapped_server;
1726 /* The server could already have been zeroed out, so
1727 we need to check for that, too. */
1728 if (server && (!LOAD_VTABLE (server) || mono_object_domain (server) == domain)) {
1729 DEBUG (1, fprintf (gc_debug_file, "Cleaning up remote pointer in %p to object %p\n",
1731 ((MonoRealProxy*)start)->unwrapped_server = NULL;
1736 static void __attribute__((noinline))
1737 scan_area_for_domain (MonoDomain *domain, char *start, char *end)
1742 while (start < end) {
1743 if (!*(void**)start) {
1744 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
1747 vt = (GCVTable*)LOAD_VTABLE (start);
1748 process_object_for_domain_clearing (start, domain);
1749 remove = need_remove_object_for_domain (start, domain);
1750 if (remove && ((MonoObject*)start)->synchronisation) {
1751 void **dislink = mono_monitor_get_object_monitor_weak_link ((MonoObject*)start);
1753 mono_gc_register_disappearing_link (NULL, dislink, FALSE);
1756 #define SCAN_OBJECT_NOSCAN
1757 #define SCAN_OBJECT_ACTION do { \
1758 if (remove) memset (start, 0, skip_size); \
1760 #include "sgen-scan-object.h"
1764 static MonoDomain *check_domain = NULL;
1767 check_obj_not_in_domain (void *o)
1769 g_assert (((MonoObject*)o)->vtable->domain != check_domain);
1774 scan_for_registered_roots_in_domain (MonoDomain *domain, int root_type)
1778 check_domain = domain;
1779 for (i = 0; i < roots_hash_size [root_type]; ++i) {
1780 for (root = roots_hash [root_type][i]; root; root = root->next) {
1781 void **start_root = (void**)root->start_root;
1782 mword desc = root->root_desc;
1784 /* The MonoDomain struct is allowed to hold
1785 references to objects in its own domain. */
1786 if (start_root == (void**)domain)
1789 switch (desc & ROOT_DESC_TYPE_MASK) {
1790 case ROOT_DESC_BITMAP:
1791 desc >>= ROOT_DESC_TYPE_SHIFT;
1793 if ((desc & 1) && *start_root)
1794 check_obj_not_in_domain (*start_root);
1799 case ROOT_DESC_COMPLEX: {
1800 gsize *bitmap_data = complex_descriptors + (desc >> ROOT_DESC_TYPE_SHIFT);
1801 int bwords = (*bitmap_data) - 1;
1802 void **start_run = start_root;
1804 while (bwords-- > 0) {
1805 gsize bmap = *bitmap_data++;
1806 void **objptr = start_run;
1808 if ((bmap & 1) && *objptr)
1809 check_obj_not_in_domain (*objptr);
1813 start_run += GC_BITS_PER_WORD;
1817 case ROOT_DESC_USER: {
1818 MonoGCMarkFunc marker = user_descriptors [desc >> ROOT_DESC_TYPE_SHIFT];
1819 marker (start_root, check_obj_not_in_domain);
1822 case ROOT_DESC_RUN_LEN:
1823 g_assert_not_reached ();
1825 g_assert_not_reached ();
1829 check_domain = NULL;
1833 clear_domain_process_pinned_object_callback (PinnedChunk *chunk, char *obj, size_t size, MonoDomain *domain)
1835 process_object_for_domain_clearing (obj, domain);
1839 clear_domain_free_pinned_object_callback (PinnedChunk *chunk, char *obj, size_t size, MonoDomain *domain)
1841 if (need_remove_object_for_domain (obj, domain))
1842 free_pinned_object (chunk, obj, size);
1846 scan_pinned_object_for_xdomain_refs_callback (PinnedChunk *chunk, char *obj, size_t size, gpointer dummy)
1848 scan_object_for_xdomain_refs (obj);
1852 check_for_xdomain_refs (void)
1854 GCMemSection *section;
1857 for (section = section_list; section; section = section->block.next)
1858 scan_area_for_xdomain_refs (section->data, section->end_data);
1860 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1861 scan_object_for_xdomain_refs (bigobj->data);
1863 scan_pinned_objects (scan_pinned_object_for_xdomain_refs_callback, NULL);
1867 * When appdomains are unloaded we can easily remove objects that have finalizers,
1868 * but all the others could still be present in random places on the heap.
1869 * We need a sweep to get rid of them even though it's going to be costly
1871 * The reason we need to remove them is because we access the vtable and class
1872 * structures to know the object size and the reference bitmap: once the domain is
1873 * unloaded the point to random memory.
1876 mono_gc_clear_domain (MonoDomain * domain)
1878 GCMemSection *section;
1879 LOSObject *bigobj, *prev;
1884 /* Clear all remaining nursery fragments */
1885 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION) {
1886 g_assert (nursery_next <= nursery_frag_real_end);
1887 memset (nursery_next, 0, nursery_frag_real_end - nursery_next);
1888 for (frag = nursery_fragments; frag; frag = frag->next) {
1889 memset (frag->fragment_start, 0, frag->fragment_end - frag->fragment_start);
1893 if (xdomain_checks && domain != mono_get_root_domain ()) {
1894 scan_for_registered_roots_in_domain (domain, ROOT_TYPE_NORMAL);
1895 scan_for_registered_roots_in_domain (domain, ROOT_TYPE_WBARRIER);
1896 check_for_xdomain_refs ();
1899 for (section = section_list; section; section = section->block.next) {
1900 scan_area_for_domain (domain, section->data, section->end_data);
1903 /* We need two passes over pinned and large objects because
1904 freeing such an object gives its memory back to the OS (in
1905 the case of large objects) or obliterates its vtable
1906 (pinned objects), but we might need to dereference a
1907 pointer from an object to another object if the first
1908 object is a proxy. */
1909 scan_pinned_objects ((ScanPinnedObjectCallbackFunc)clear_domain_process_pinned_object_callback, domain);
1910 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1911 process_object_for_domain_clearing (bigobj->data, domain);
1914 for (bigobj = los_object_list; bigobj;) {
1915 if (need_remove_object_for_domain (bigobj->data, domain)) {
1916 LOSObject *to_free = bigobj;
1918 prev->next = bigobj->next;
1920 los_object_list = bigobj->next;
1921 bigobj = bigobj->next;
1922 DEBUG (1, fprintf (gc_debug_file, "Freeing large object %p\n",
1924 free_large_object (to_free);
1928 bigobj = bigobj->next;
1930 scan_pinned_objects ((ScanPinnedObjectCallbackFunc)clear_domain_free_pinned_object_callback, domain);
1932 for (i = GENERATION_NURSERY; i < GENERATION_MAX; ++i)
1933 null_links_for_domain (domain, i);
1939 * add_to_global_remset:
1941 * The global remset contains locations which point into newspace after
1942 * a minor collection. This can happen if the objects they point to are pinned.
1945 add_to_global_remset (gpointer ptr, gboolean root)
1949 DEBUG (8, fprintf (gc_debug_file, "Adding global remset for %p\n", ptr));
1950 binary_protocol_global_remset (ptr, *(gpointer*)ptr, (gpointer)LOAD_VTABLE (*(gpointer*)ptr));
1953 g_assert (!ptr_in_nursery (ptr) && ptr_in_nursery (*(gpointer*)ptr));
1955 HEAVY_STAT (++stat_global_remsets_added);
1958 * FIXME: If an object remains pinned, we need to add it at every minor collection.
1959 * To avoid uncontrolled growth of the global remset, only add each pointer once.
1961 if (global_remset->store_next + 3 < global_remset->end_set) {
1963 *(global_remset->store_next++) = (mword)ptr | REMSET_OTHER;
1964 *(global_remset->store_next++) = (mword)REMSET_ROOT_LOCATION;
1966 *(global_remset->store_next++) = (mword)ptr;
1970 rs = alloc_remset (global_remset->end_set - global_remset->data, NULL);
1971 rs->next = global_remset;
1974 *(global_remset->store_next++) = (mword)ptr | REMSET_OTHER;
1975 *(global_remset->store_next++) = (mword)REMSET_ROOT_LOCATION;
1977 *(global_remset->store_next++) = (mword)ptr;
1981 int global_rs_size = 0;
1983 for (rs = global_remset; rs; rs = rs->next) {
1984 global_rs_size += rs->store_next - rs->data;
1986 DEBUG (4, fprintf (gc_debug_file, "Global remset now has size %d\n", global_rs_size));
1990 #define MOVED_OBJECTS_NUM 64
1991 static void *moved_objects [MOVED_OBJECTS_NUM];
1992 static int moved_objects_idx = 0;
1994 #include "sgen-gray.c"
1997 * This is how the copying happens from the nursery to the old generation.
1998 * We assume that at this time all the pinned objects have been identified and
2000 * We run scan_object() for each pinned object so that each referenced
2001 * objects if possible are copied. The new gray objects created can have
2002 * scan_object() run on them right away, too.
2003 * Then we run copy_object() for the precisely tracked roots. At this point
2004 * all the roots are either gray or black. We run scan_object() on the gray
2005 * objects until no more gray objects are created.
2006 * At the end of the process we walk again the pinned list and we unmark
2007 * the pinned flag. As we go we also create the list of free space for use
2008 * in the next allocation runs.
2010 * We need to remember objects from the old generation that point to the new one
2011 * (or just addresses?).
2013 * copy_object could be made into a macro once debugged (use inline for now).
2016 static char* __attribute__((noinline))
2017 copy_object (char *obj, char *from_space_start, char *from_space_end)
2019 static void *copy_labels [] = { &&LAB_0, &&LAB_1, &&LAB_2, &&LAB_3, &&LAB_4, &&LAB_5, &&LAB_6, &&LAB_7, &&LAB_8 };
2025 HEAVY_STAT (++num_copy_object_called);
2027 if (!(obj >= from_space_start && obj < from_space_end)) {
2028 DEBUG (9, fprintf (gc_debug_file, "Not copying %p because it's not in from space (%p-%p)\n",
2029 obj, from_space_start, from_space_end));
2030 HEAVY_STAT (++stat_copy_object_failed_from_space);
2034 DEBUG (9, fprintf (gc_debug_file, "Precise copy of %p", obj));
2037 * obj must belong to one of:
2042 * 4. a non-to-space section of the major heap
2043 * 5. a to-space section of the major heap
2045 * In addition, objects in 1, 2 and 4 might also be pinned.
2046 * Objects in 1 and 4 might be forwarded.
2048 * Before we can copy the object we must make sure that we are
2049 * allowed to, i.e. that the object not pinned, not already
2050 * forwarded and doesn't belong to the LOS, a pinned chunk, or
2051 * a to-space section.
2053 * We are usually called for to-space objects (5) when we have
2054 * two remset entries for the same reference. The first entry
2055 * copies the object and updates the reference and the second
2056 * calls us with the updated reference that points into
2057 * to-space. There might also be other circumstances where we
2058 * get to-space objects.
2061 if ((forwarded = object_is_forwarded (obj))) {
2062 g_assert (((MonoVTable*)LOAD_VTABLE(obj))->gc_descr);
2063 DEBUG (9, fprintf (gc_debug_file, " (already forwarded to %p)\n", forwarded));
2064 HEAVY_STAT (++stat_copy_object_failed_forwarded);
2067 if (object_is_pinned (obj)) {
2068 g_assert (((MonoVTable*)LOAD_VTABLE(obj))->gc_descr);
2069 DEBUG (9, fprintf (gc_debug_file, " (pinned, no change)\n"));
2070 HEAVY_STAT (++stat_copy_object_failed_pinned);
2074 objsize = safe_object_get_size ((MonoObject*)obj);
2075 objsize += ALLOC_ALIGN - 1;
2076 objsize &= ~(ALLOC_ALIGN - 1);
2078 if (ptr_in_nursery (obj))
2082 * At this point we know obj is not pinned, not forwarded and
2083 * belongs to 2, 3, 4, or 5.
2085 * LOS object (2) are simple, at least until we always follow
2086 * the rule: if objsize > MAX_SMALL_OBJ_SIZE, pin the object
2087 * and return it. At the end of major collections, we walk
2088 * the los list and if the object is pinned, it is marked,
2089 * otherwise it can be freed.
2091 * Pinned chunks (3) and major heap sections (4, 5) both
2092 * reside in blocks, which are always aligned, so once we've
2093 * eliminated LOS objects, we can just access the block and
2094 * see whether it's a pinned chunk or a major heap section.
2096 if (G_UNLIKELY (objsize > MAX_SMALL_OBJ_SIZE || obj_is_from_pinned_alloc (obj))) {
2097 DEBUG (9, fprintf (gc_debug_file, " (marked LOS/Pinned %p (%s), size: %zd)\n", obj, safe_name (obj), objsize));
2098 binary_protocol_pin (obj, (gpointer)LOAD_VTABLE (obj), safe_object_get_size ((MonoObject*)obj));
2100 HEAVY_STAT (++stat_copy_object_failed_large_pinned);
2105 * Now we know the object is in a major heap section. All we
2106 * need to do is check whether it's already in to-space (5) or
2109 if (MAJOR_SECTION_FOR_OBJECT (obj)->is_to_space) {
2110 g_assert (objsize <= MAX_SMALL_OBJ_SIZE);
2111 DEBUG (9, fprintf (gc_debug_file, " (already copied)\n"));
2112 HEAVY_STAT (++stat_copy_object_failed_to_space);
2117 DEBUG (9, fprintf (gc_debug_file, " (to %p, %s size: %zd)\n", to_space_bumper, ((MonoObject*)obj)->vtable->klass->name, objsize));
2118 binary_protocol_copy (obj, to_space_bumper, ((MonoObject*)obj)->vtable, objsize);
2120 HEAVY_STAT (++num_objects_copied);
2122 /* Make sure we have enough space available */
2123 if (to_space_bumper + objsize > to_space_top) {
2125 g_assert (to_space_bumper + objsize <= to_space_top);
2128 if (objsize <= sizeof (gpointer) * 8) {
2129 mword *dest = (mword*)to_space_bumper;
2130 goto *copy_labels [objsize / sizeof (gpointer)];
2132 (dest) [7] = ((mword*)obj) [7];
2134 (dest) [6] = ((mword*)obj) [6];
2136 (dest) [5] = ((mword*)obj) [5];
2138 (dest) [4] = ((mword*)obj) [4];
2140 (dest) [3] = ((mword*)obj) [3];
2142 (dest) [2] = ((mword*)obj) [2];
2144 (dest) [1] = ((mword*)obj) [1];
2146 (dest) [0] = ((mword*)obj) [0];
2154 char* edi = to_space_bumper;
2155 __asm__ __volatile__(
2157 : "=&c" (ecx), "=&D" (edi), "=&S" (esi)
2158 : "0" (objsize/4), "1" (edi),"2" (esi)
2163 memcpy (to_space_bumper, obj, objsize);
2166 /* adjust array->bounds */
2167 vt = ((MonoObject*)obj)->vtable;
2168 g_assert (vt->gc_descr);
2169 if (G_UNLIKELY (vt->rank && ((MonoArray*)obj)->bounds)) {
2170 MonoArray *array = (MonoArray*)to_space_bumper;
2171 array->bounds = (MonoArrayBounds*)((char*)to_space_bumper + ((char*)((MonoArray*)obj)->bounds - (char*)obj));
2172 DEBUG (9, fprintf (gc_debug_file, "Array instance %p: size: %zd, rank: %d, length: %d\n", array, objsize, vt->rank, mono_array_length (array)));
2174 /* set the forwarding pointer */
2175 forward_object (obj, to_space_bumper);
2176 if (G_UNLIKELY (mono_profiler_events & MONO_PROFILE_GC_MOVES)) {
2177 if (moved_objects_idx == MOVED_OBJECTS_NUM) {
2178 mono_profiler_gc_moves (moved_objects, moved_objects_idx);
2179 moved_objects_idx = 0;
2181 moved_objects [moved_objects_idx++] = obj;
2182 moved_objects [moved_objects_idx++] = to_space_bumper;
2184 obj = to_space_bumper;
2185 to_space_section->scan_starts [((char*)obj - (char*)to_space_section->data)/SCAN_START_SIZE] = obj;
2186 to_space_bumper += objsize;
2187 DEBUG (9, fprintf (gc_debug_file, "Enqueuing gray object %p (%s)\n", obj, safe_name (obj)));
2188 gray_object_enqueue (obj);
2189 DEBUG (8, g_assert (to_space_bumper <= to_space_top));
2194 #define HANDLE_PTR(ptr,obj) do { \
2195 void *__old = *(ptr); \
2198 *(ptr) = __copy = copy_object (__old, from_start, from_end); \
2199 DEBUG (9, if (__old != __copy) fprintf (gc_debug_file, "Overwrote field at %p with %p (was: %p)\n", (ptr), *(ptr), __old)); \
2200 if (G_UNLIKELY (ptr_in_nursery (__copy) && !ptr_in_nursery ((ptr)))) \
2201 add_to_global_remset ((ptr), FALSE); \
2206 * Scan the object pointed to by @start for references to
2207 * other objects between @from_start and @from_end and copy
2208 * them to the gray_objects area.
2209 * Returns a pointer to the end of the object.
2212 scan_object (char *start, char* from_start, char* from_end)
2214 #include "sgen-scan-object.h"
2222 * Scan objects in the gray stack until the stack is empty. This should be called
2223 * frequently after each object is copied, to achieve better locality and cache
2227 drain_gray_stack (char *start_addr, char *end_addr)
2231 while ((obj = gray_object_dequeue ())) {
2232 DEBUG (9, fprintf (gc_debug_file, "Precise gray object scan %p (%s)\n", obj, safe_name (obj)));
2233 scan_object (obj, start_addr, end_addr);
2240 * Scan the valuetype pointed to by START, described by DESC for references to
2241 * other objects between @from_start and @from_end and copy them to the gray_objects area.
2242 * Returns a pointer to the end of the object.
2245 scan_vtype (char *start, mword desc, char* from_start, char* from_end)
2249 /* The descriptors include info about the MonoObject header as well */
2250 start -= sizeof (MonoObject);
2252 switch (desc & 0x7) {
2253 case DESC_TYPE_RUN_LENGTH:
2254 OBJ_RUN_LEN_FOREACH_PTR (desc,start);
2255 OBJ_RUN_LEN_SIZE (skip_size, desc, start);
2256 g_assert (skip_size);
2257 return start + skip_size;
2258 case DESC_TYPE_SMALL_BITMAP:
2259 OBJ_BITMAP_FOREACH_PTR (desc,start);
2260 OBJ_BITMAP_SIZE (skip_size, desc, start);
2261 return start + skip_size;
2262 case DESC_TYPE_LARGE_BITMAP:
2263 case DESC_TYPE_COMPLEX:
2265 g_assert_not_reached ();
2268 // The other descriptors can't happen with vtypes
2269 g_assert_not_reached ();
2275 #include "sgen-pinning-stats.c"
2278 * Addresses from start to end are already sorted. This function finds
2279 * the object header for each address and pins the object. The
2280 * addresses must be inside the passed section. The (start of the)
2281 * address array is overwritten with the addresses of the actually
2282 * pinned objects. Return the number of pinned objects.
2285 pin_objects_from_addresses (GCMemSection *section, void **start, void **end, void *start_nursery, void *end_nursery)
2290 void *last_obj = NULL;
2291 size_t last_obj_size = 0;
2294 void **definitely_pinned = start;
2295 while (start < end) {
2297 /* the range check should be reduntant */
2298 if (addr != last && addr >= start_nursery && addr < end_nursery) {
2299 DEBUG (5, fprintf (gc_debug_file, "Considering pinning addr %p\n", addr));
2300 /* multiple pointers to the same object */
2301 if (addr >= last_obj && (char*)addr < (char*)last_obj + last_obj_size) {
2305 idx = ((char*)addr - (char*)section->data) / SCAN_START_SIZE;
2306 g_assert (idx < section->num_scan_start);
2307 search_start = (void*)section->scan_starts [idx];
2308 if (!search_start || search_start > addr) {
2311 search_start = section->scan_starts [idx];
2312 if (search_start && search_start <= addr)
2315 if (!search_start || search_start > addr)
2316 search_start = start_nursery;
2318 if (search_start < last_obj)
2319 search_start = (char*)last_obj + last_obj_size;
2320 /* now addr should be in an object a short distance from search_start
2321 * Note that search_start must point to zeroed mem or point to an object.
2324 if (!*(void**)search_start) {
2325 mword p = (mword)search_start;
2326 p += sizeof (gpointer);
2327 p += ALLOC_ALIGN - 1;
2328 p &= ~(ALLOC_ALIGN - 1);
2329 search_start = (void*)p;
2332 last_obj = search_start;
2333 last_obj_size = safe_object_get_size ((MonoObject*)search_start);
2334 last_obj_size += ALLOC_ALIGN - 1;
2335 last_obj_size &= ~(ALLOC_ALIGN - 1);
2336 DEBUG (8, fprintf (gc_debug_file, "Pinned try match %p (%s), size %zd\n", last_obj, safe_name (last_obj), last_obj_size));
2337 if (addr >= search_start && (char*)addr < (char*)last_obj + last_obj_size) {
2338 DEBUG (4, fprintf (gc_debug_file, "Pinned object %p, vtable %p (%s), count %d\n", search_start, *(void**)search_start, safe_name (search_start), count));
2339 binary_protocol_pin (search_start, (gpointer)LOAD_VTABLE (search_start), safe_object_get_size (search_start));
2340 pin_object (search_start);
2342 pin_stats_register_object (search_start, last_obj_size);
2343 definitely_pinned [count] = search_start;
2347 /* skip to the next object */
2348 search_start = (void*)((char*)search_start + last_obj_size);
2349 } while (search_start <= addr);
2350 /* we either pinned the correct object or we ignored the addr because
2351 * it points to unused zeroed memory.
2357 //printf ("effective pinned: %d (at the end: %d)\n", count, (char*)end_nursery - (char*)last);
2361 static void** pin_queue;
2362 static int pin_queue_size = 0;
2363 static int next_pin_slot = 0;
2368 gap = (gap * 10) / 13;
2369 if (gap == 9 || gap == 10)
2378 compare_addr (const void *a, const void *b)
2380 return *(const void **)a - *(const void **)b;
2384 /* sort the addresses in array in increasing order */
2386 sort_addresses (void **array, int size)
2389 * qsort is slower as predicted.
2390 * qsort (array, size, sizeof (gpointer), compare_addr);
2397 gap = new_gap (gap);
2400 for (i = 0; i < end; i++) {
2402 if (array [i] > array [j]) {
2403 void* val = array [i];
2404 array [i] = array [j];
2409 if (gap == 1 && !swapped)
2414 static G_GNUC_UNUSED void
2415 print_nursery_gaps (void* start_nursery, void *end_nursery)
2418 gpointer first = start_nursery;
2420 for (i = 0; i < next_pin_slot; ++i) {
2421 next = pin_queue [i];
2422 fprintf (gc_debug_file, "Nursery range: %p-%p, size: %zd\n", first, next, (char*)next-(char*)first);
2426 fprintf (gc_debug_file, "Nursery range: %p-%p, size: %zd\n", first, next, (char*)next-(char*)first);
2429 /* reduce the info in the pin queue, removing duplicate pointers and sorting them */
2431 optimize_pin_queue (int start_slot)
2433 void **start, **cur, **end;
2434 /* sort and uniq pin_queue: we just sort and we let the rest discard multiple values */
2435 /* it may be better to keep ranges of pinned memory instead of individually pinning objects */
2436 DEBUG (5, fprintf (gc_debug_file, "Sorting pin queue, size: %d\n", next_pin_slot));
2437 if ((next_pin_slot - start_slot) > 1)
2438 sort_addresses (pin_queue + start_slot, next_pin_slot - start_slot);
2439 start = cur = pin_queue + start_slot;
2440 end = pin_queue + next_pin_slot;
2443 while (*start == *cur && cur < end)
2447 next_pin_slot = start - pin_queue;
2448 DEBUG (5, fprintf (gc_debug_file, "Pin queue reduced to size: %d\n", next_pin_slot));
2449 //DEBUG (6, print_nursery_gaps (start_nursery, end_nursery));
2454 optimized_pin_queue_search (void *addr)
2456 int first = 0, last = next_pin_slot;
2457 while (first < last) {
2458 int middle = first + ((last - first) >> 1);
2459 if (addr <= pin_queue [middle])
2464 g_assert (first == last);
2469 find_optimized_pin_queue_area (void *start, void *end, int *first, int *last)
2471 *first = optimized_pin_queue_search (start);
2472 *last = optimized_pin_queue_search (end);
2476 realloc_pin_queue (void)
2478 int new_size = pin_queue_size? pin_queue_size + pin_queue_size/2: 1024;
2479 void **new_pin = get_internal_mem (sizeof (void*) * new_size, INTERNAL_MEM_PIN_QUEUE);
2480 memcpy (new_pin, pin_queue, sizeof (void*) * next_pin_slot);
2481 free_internal_mem (pin_queue, INTERNAL_MEM_PIN_QUEUE);
2482 pin_queue = new_pin;
2483 pin_queue_size = new_size;
2484 DEBUG (4, fprintf (gc_debug_file, "Reallocated pin queue to size: %d\n", new_size));
2487 #include "sgen-pinning.c"
2490 * Scan the memory between start and end and queue values which could be pointers
2491 * to the area between start_nursery and end_nursery for later consideration.
2492 * Typically used for thread stacks.
2495 conservatively_pin_objects_from (void **start, void **end, void *start_nursery, void *end_nursery, int pin_type)
2498 while (start < end) {
2499 if (*start >= start_nursery && *start < end_nursery) {
2501 * *start can point to the middle of an object
2502 * note: should we handle pointing at the end of an object?
2503 * pinning in C# code disallows pointing at the end of an object
2504 * but there is some small chance that an optimizing C compiler
2505 * may keep the only reference to an object by pointing
2506 * at the end of it. We ignore this small chance for now.
2507 * Pointers to the end of an object are indistinguishable
2508 * from pointers to the start of the next object in memory
2509 * so if we allow that we'd need to pin two objects...
2510 * We queue the pointer in an array, the
2511 * array will then be sorted and uniqued. This way
2512 * we can coalesce several pinning pointers and it should
2513 * be faster since we'd do a memory scan with increasing
2514 * addresses. Note: we can align the address to the allocation
2515 * alignment, so the unique process is more effective.
2517 mword addr = (mword)*start;
2518 addr &= ~(ALLOC_ALIGN - 1);
2519 if (addr >= (mword)start_nursery && addr < (mword)end_nursery)
2520 pin_stage_ptr ((void*)addr);
2522 pin_stats_register_address ((char*)addr, pin_type);
2523 DEBUG (6, if (count) fprintf (gc_debug_file, "Pinning address %p\n", (void*)addr));
2528 DEBUG (7, if (count) fprintf (gc_debug_file, "found %d potential pinned heap pointers\n", count));
2532 * If generation is 0, just mark objects in the nursery, the others we don't care,
2533 * since they are not going to move anyway.
2534 * There are different areas that are scanned for pinned pointers:
2535 * *) the thread stacks (when jit support is ready only the unmanaged frames)
2536 * *) the pinned handle table
2537 * *) the pinned roots
2539 * Note: when we'll use a write barrier for old to new gen references, we need to
2540 * keep track of old gen objects that point to pinned new gen objects because in that
2541 * case the referenced object will be moved maybe at the next collection, but there
2542 * is no write in the old generation area where the pinned object is referenced
2543 * and we may not consider it as reachable.
2545 static G_GNUC_UNUSED void
2546 mark_pinned_objects (int generation)
2551 * Debugging function: find in the conservative roots where @obj is being pinned.
2553 static G_GNUC_UNUSED void
2554 find_pinning_reference (char *obj, size_t size)
2558 char *endobj = obj + size;
2559 for (i = 0; i < roots_hash_size [0]; ++i) {
2560 for (root = roots_hash [0][i]; root; root = root->next) {
2561 /* if desc is non-null it has precise info */
2562 if (!root->root_desc) {
2563 char ** start = (char**)root->start_root;
2564 while (start < (char**)root->end_root) {
2565 if (*start >= obj && *start < endobj) {
2566 DEBUG (0, fprintf (gc_debug_file, "Object %p referenced in pinned roots %p-%p (at %p in record %p)\n", obj, root->start_root, root->end_root, start, root));
2573 find_pinning_ref_from_thread (obj, size);
2577 * The first thing we do in a collection is to identify pinned objects.
2578 * This function considers all the areas of memory that need to be
2579 * conservatively scanned.
2582 pin_from_roots (void *start_nursery, void *end_nursery)
2586 DEBUG (2, fprintf (gc_debug_file, "Scanning pinned roots (%d bytes, %d/%d entries)\n", (int)roots_size, num_roots_entries [ROOT_TYPE_NORMAL], num_roots_entries [ROOT_TYPE_PINNED]));
2587 /* objects pinned from the API are inside these roots */
2588 for (i = 0; i < roots_hash_size [ROOT_TYPE_PINNED]; ++i) {
2589 for (root = roots_hash [ROOT_TYPE_PINNED][i]; root; root = root->next) {
2590 DEBUG (6, fprintf (gc_debug_file, "Pinned roots %p-%p\n", root->start_root, root->end_root));
2591 conservatively_pin_objects_from ((void**)root->start_root, (void**)root->end_root, start_nursery, end_nursery, PIN_TYPE_OTHER);
2594 /* now deal with the thread stacks
2595 * in the future we should be able to conservatively scan only:
2596 * *) the cpu registers
2597 * *) the unmanaged stack frames
2598 * *) the _last_ managed stack frame
2599 * *) pointers slots in managed frames
2601 scan_thread_data (start_nursery, end_nursery, FALSE);
2603 evacuate_pin_staging_area ();
2606 /* Copy function called from user defined mark functions */
2607 static char *user_copy_n_start;
2608 static char *user_copy_n_end;
2611 user_copy (void *addr)
2614 return copy_object (addr, user_copy_n_start, user_copy_n_end);
2620 * The memory area from start_root to end_root contains pointers to objects.
2621 * Their position is precisely described by @desc (this means that the pointer
2622 * can be either NULL or the pointer to the start of an object).
2623 * This functions copies them to to_space updates them.
2626 precisely_scan_objects_from (void** start_root, void** end_root, char* n_start, char *n_end, mword desc)
2628 switch (desc & ROOT_DESC_TYPE_MASK) {
2629 case ROOT_DESC_BITMAP:
2630 desc >>= ROOT_DESC_TYPE_SHIFT;
2632 if ((desc & 1) && *start_root) {
2633 *start_root = copy_object (*start_root, n_start, n_end);
2634 DEBUG (9, fprintf (gc_debug_file, "Overwrote root at %p with %p\n", start_root, *start_root));
2635 drain_gray_stack (n_start, n_end);
2641 case ROOT_DESC_COMPLEX: {
2642 gsize *bitmap_data = complex_descriptors + (desc >> ROOT_DESC_TYPE_SHIFT);
2643 int bwords = (*bitmap_data) - 1;
2644 void **start_run = start_root;
2646 while (bwords-- > 0) {
2647 gsize bmap = *bitmap_data++;
2648 void **objptr = start_run;
2650 if ((bmap & 1) && *objptr) {
2651 *objptr = copy_object (*objptr, n_start, n_end);
2652 DEBUG (9, fprintf (gc_debug_file, "Overwrote root at %p with %p\n", objptr, *objptr));
2653 drain_gray_stack (n_start, n_end);
2658 start_run += GC_BITS_PER_WORD;
2662 case ROOT_DESC_USER: {
2663 MonoGCMarkFunc marker = user_descriptors [desc >> ROOT_DESC_TYPE_SHIFT];
2665 user_copy_n_start = n_start;
2666 user_copy_n_end = n_end;
2667 marker (start_root, user_copy);
2670 case ROOT_DESC_RUN_LEN:
2671 g_assert_not_reached ();
2673 g_assert_not_reached ();
2678 alloc_fragment (void)
2680 Fragment *frag = fragment_freelist;
2682 fragment_freelist = frag->next;
2686 frag = get_internal_mem (sizeof (Fragment), INTERNAL_MEM_FRAGMENT);
2691 /* size must be a power of 2 */
2693 get_os_memory_aligned (mword size, gboolean activate)
2695 /* Allocate twice the memory to be able to put the block on an aligned address */
2696 char *mem = get_os_memory (size * 2, activate);
2701 aligned = (char*)((mword)(mem + (size - 1)) & ~(size - 1));
2702 g_assert (aligned >= mem && aligned + size <= mem + size * 2 && !((mword)aligned & (size - 1)));
2705 free_os_memory (mem, aligned - mem);
2706 if (aligned + size < mem + size * 2)
2707 free_os_memory (aligned + size, (mem + size * 2) - (aligned + size));
2713 * Allocate and setup the data structures needed to be able to allocate objects
2714 * in the nursery. The nursery is stored in nursery_section.
2717 alloc_nursery (void)
2719 GCMemSection *section;
2725 if (nursery_section)
2727 DEBUG (2, fprintf (gc_debug_file, "Allocating nursery size: %zd\n", nursery_size));
2728 /* later we will alloc a larger area for the nursery but only activate
2729 * what we need. The rest will be used as expansion if we have too many pinned
2730 * objects in the existing nursery.
2732 /* FIXME: handle OOM */
2733 section = get_internal_mem (SIZEOF_GC_MEM_SECTION, INTERNAL_MEM_SECTION);
2735 g_assert (nursery_size == DEFAULT_NURSERY_SIZE);
2736 alloc_size = nursery_size;
2737 #ifdef ALIGN_NURSERY
2738 data = get_os_memory_aligned (alloc_size, TRUE);
2740 data = get_os_memory (alloc_size, TRUE);
2742 nursery_start = data;
2743 nursery_real_end = nursery_start + nursery_size;
2744 UPDATE_HEAP_BOUNDARIES (nursery_start, nursery_real_end);
2745 nursery_next = nursery_start;
2746 total_alloc += alloc_size;
2747 DEBUG (4, fprintf (gc_debug_file, "Expanding nursery size (%p-%p): %zd, total: %zd\n", data, data + alloc_size, nursery_size, total_alloc));
2748 section->data = section->next_data = data;
2749 section->size = alloc_size;
2750 section->end_data = nursery_real_end;
2751 scan_starts = (alloc_size + SCAN_START_SIZE - 1) / SCAN_START_SIZE;
2752 section->scan_starts = get_internal_mem (sizeof (char*) * scan_starts, INTERNAL_MEM_SCAN_STARTS);
2753 section->num_scan_start = scan_starts;
2754 section->block.role = MEMORY_ROLE_GEN0;
2756 /* add to the section list */
2757 section->block.next = section_list;
2758 section_list = section;
2760 nursery_section = section;
2762 /* Setup the single first large fragment */
2763 frag = alloc_fragment ();
2764 frag->fragment_start = nursery_start;
2765 frag->fragment_limit = nursery_start;
2766 frag->fragment_end = nursery_real_end;
2767 nursery_frag_real_end = nursery_real_end;
2768 /* FIXME: frag here is lost */
2772 scan_finalizer_entries (FinalizeEntry *list, char *start, char *end) {
2775 for (fin = list; fin; fin = fin->next) {
2778 DEBUG (5, fprintf (gc_debug_file, "Scan of fin ready object: %p (%s)\n", fin->object, safe_name (fin->object)));
2779 fin->object = copy_object (fin->object, start, end);
2784 * Update roots in the old generation. Since we currently don't have the
2785 * info from the write barriers, we just scan all the objects.
2787 static G_GNUC_UNUSED void
2788 scan_old_generation (char *start, char* end)
2790 GCMemSection *section;
2791 LOSObject *big_object;
2794 for (section = section_list; section; section = section->block.next) {
2795 if (section == nursery_section)
2797 DEBUG (2, fprintf (gc_debug_file, "Scan of old section: %p-%p, size: %d\n", section->data, section->next_data, (int)(section->next_data - section->data)));
2798 /* we have to deal with zeroed holes in old generation (truncated strings ...) */
2800 while (p < section->next_data) {
2805 DEBUG (8, fprintf (gc_debug_file, "Precise old object scan of %p (%s)\n", p, safe_name (p)));
2806 p = scan_object (p, start, end);
2809 /* scan the old object space, too */
2810 for (big_object = los_object_list; big_object; big_object = big_object->next) {
2811 DEBUG (5, fprintf (gc_debug_file, "Scan of big object: %p (%s), size: %zd\n", big_object->data, safe_name (big_object->data), big_object->size));
2812 scan_object (big_object->data, start, end);
2814 /* scan the list of objects ready for finalization */
2815 scan_finalizer_entries (fin_ready_list, start, end);
2816 scan_finalizer_entries (critical_fin_list, start, end);
2819 static mword fragment_total = 0;
2821 * We found a fragment of free memory in the nursery: memzero it and if
2822 * it is big enough, add it to the list of fragments that can be used for
2826 add_nursery_frag (size_t frag_size, char* frag_start, char* frag_end)
2829 DEBUG (4, fprintf (gc_debug_file, "Found empty fragment: %p-%p, size: %zd\n", frag_start, frag_end, frag_size));
2830 binary_protocol_empty (frag_start, frag_size);
2831 /* memsetting just the first chunk start is bound to provide better cache locality */
2832 if (nursery_clear_policy == CLEAR_AT_GC)
2833 memset (frag_start, 0, frag_size);
2834 /* Not worth dealing with smaller fragments: need to tune */
2835 if (frag_size >= FRAGMENT_MIN_SIZE) {
2836 fragment = alloc_fragment ();
2837 fragment->fragment_start = frag_start;
2838 fragment->fragment_limit = frag_start;
2839 fragment->fragment_end = frag_end;
2840 fragment->next = nursery_fragments;
2841 nursery_fragments = fragment;
2842 fragment_total += frag_size;
2844 /* Clear unused fragments, pinning depends on this */
2845 memset (frag_start, 0, frag_size);
2850 scan_needed_big_objects (char *start_addr, char *end_addr)
2852 LOSObject *big_object;
2854 for (big_object = los_object_list; big_object; big_object = big_object->next) {
2855 if (!big_object->scanned && object_is_pinned (big_object->data)) {
2856 DEBUG (5, fprintf (gc_debug_file, "Scan of big object: %p (%s), size: %zd\n", big_object->data, safe_name (big_object->data), big_object->size));
2857 scan_object (big_object->data, start_addr, end_addr);
2858 drain_gray_stack (start_addr, end_addr);
2859 big_object->scanned = TRUE;
2867 generation_name (int generation)
2869 switch (generation) {
2870 case GENERATION_NURSERY: return "nursery";
2871 case GENERATION_OLD: return "old";
2872 default: g_assert_not_reached ();
2876 static DisappearingLinkHashTable*
2877 get_dislink_hash_table (int generation)
2879 switch (generation) {
2880 case GENERATION_NURSERY: return &minor_disappearing_link_hash;
2881 case GENERATION_OLD: return &major_disappearing_link_hash;
2882 default: g_assert_not_reached ();
2886 static FinalizeEntryHashTable*
2887 get_finalize_entry_hash_table (int generation)
2889 switch (generation) {
2890 case GENERATION_NURSERY: return &minor_finalizable_hash;
2891 case GENERATION_OLD: return &major_finalizable_hash;
2892 default: g_assert_not_reached ();
2897 new_to_space_section (void)
2899 /* FIXME: if the current to_space_section is empty, we don't
2900 have to allocate a new one */
2902 to_space_section = alloc_major_section ();
2903 to_space_bumper = to_space_section->next_data;
2904 to_space_top = to_space_section->end_data;
2908 to_space_set_next_data (void)
2910 g_assert (to_space_bumper >= to_space_section->next_data && to_space_bumper <= to_space_section->end_data);
2911 to_space_section->next_data = to_space_bumper;
2915 to_space_expand (void)
2917 if (to_space_section) {
2918 g_assert (to_space_top == to_space_section->end_data);
2919 to_space_set_next_data ();
2922 new_to_space_section ();
2926 unset_to_space (void)
2928 /* between collections the to_space_bumper is invalidated
2929 because degraded allocations might occur, so we set it to
2930 NULL, just to make it explicit */
2931 to_space_bumper = NULL;
2933 /* don't unset to_space_section if we implement the FIXME in
2934 new_to_space_section */
2935 to_space_section = NULL;
2939 object_is_in_to_space (char *obj)
2944 if (ptr_in_nursery (obj))
2947 objsize = safe_object_get_size ((MonoObject*)obj);
2948 objsize += ALLOC_ALIGN - 1;
2949 objsize &= ~(ALLOC_ALIGN - 1);
2952 if (objsize > MAX_SMALL_OBJ_SIZE)
2956 if (obj_is_from_pinned_alloc (obj))
2959 /* now we know it's in a major heap section */
2960 return MAJOR_SECTION_FOR_OBJECT (obj)->is_to_space;
2964 finish_gray_stack (char *start_addr, char *end_addr, int generation)
2968 int fin_ready, bigo_scanned_num;
2971 * We copied all the reachable objects. Now it's the time to copy
2972 * the objects that were not referenced by the roots, but by the copied objects.
2973 * we built a stack of objects pointed to by gray_start: they are
2974 * additional roots and we may add more items as we go.
2975 * We loop until gray_start == gray_objects which means no more objects have
2976 * been added. Note this is iterative: no recursion is involved.
2977 * We need to walk the LO list as well in search of marked big objects
2978 * (use a flag since this is needed only on major collections). We need to loop
2979 * here as well, so keep a counter of marked LO (increasing it in copy_object).
2980 * To achieve better cache locality and cache usage, we drain the gray stack
2981 * frequently, after each object is copied, and just finish the work here.
2983 drain_gray_stack (start_addr, end_addr);
2985 //scan_old_generation (start_addr, end_addr);
2986 DEBUG (2, fprintf (gc_debug_file, "%s generation done\n", generation_name (generation)));
2987 /* walk the finalization queue and move also the objects that need to be
2988 * finalized: use the finalized objects as new roots so the objects they depend
2989 * on are also not reclaimed. As with the roots above, only objects in the nursery
2990 * are marked/copied.
2991 * We need a loop here, since objects ready for finalizers may reference other objects
2992 * that are fin-ready. Speedup with a flag?
2995 fin_ready = num_ready_finalizers;
2996 finalize_in_range (start_addr, end_addr, generation);
2997 if (generation == GENERATION_OLD)
2998 finalize_in_range (nursery_start, nursery_real_end, GENERATION_NURSERY);
2999 bigo_scanned_num = scan_needed_big_objects (start_addr, end_addr);
3001 /* drain the new stack that might have been created */
3002 DEBUG (6, fprintf (gc_debug_file, "Precise scan of gray area post fin\n"));
3003 drain_gray_stack (start_addr, end_addr);
3004 } while (fin_ready != num_ready_finalizers || bigo_scanned_num);
3006 DEBUG (2, fprintf (gc_debug_file, "Finalize queue handling scan for %s generation: %d usecs\n", generation_name (generation), TV_ELAPSED (atv, btv)));
3009 * handle disappearing links
3010 * Note we do this after checking the finalization queue because if an object
3011 * survives (at least long enough to be finalized) we don't clear the link.
3012 * This also deals with a possible issue with the monitor reclamation: with the Boehm
3013 * GC a finalized object my lose the monitor because it is cleared before the finalizer is
3016 g_assert (gray_object_queue_is_empty ());
3018 null_link_in_range (start_addr, end_addr, generation);
3019 if (generation == GENERATION_OLD)
3020 null_link_in_range (start_addr, end_addr, GENERATION_NURSERY);
3021 if (gray_object_queue_is_empty ())
3023 drain_gray_stack (start_addr, end_addr);
3026 g_assert (gray_object_queue_is_empty ());
3027 /* DEBUG (2, fprintf (gc_debug_file, "Copied from %s to old space: %d bytes (%p-%p)\n", generation_name (generation), (int)(to_space_bumper - to_space), to_space, to_space_bumper)); */
3028 to_space_set_next_data ();
3032 check_scan_starts (void)
3034 GCMemSection *section;
3036 if (!do_scan_starts_check)
3038 for (section = section_list; section; section = section->block.next) {
3039 for (i = 0; i < section->num_scan_start; ++i) {
3040 if (section->scan_starts [i]) {
3041 guint size = safe_object_get_size ((MonoObject*) section->scan_starts [i]);
3042 g_assert (size >= sizeof (MonoObject) && size <= MAX_SMALL_OBJ_SIZE);
3048 static int last_num_pinned = 0;
3051 build_nursery_fragments (int start_pin, int end_pin)
3053 char *frag_start, *frag_end;
3057 while (nursery_fragments) {
3058 Fragment *next = nursery_fragments->next;
3059 nursery_fragments->next = fragment_freelist;
3060 fragment_freelist = nursery_fragments;
3061 nursery_fragments = next;
3063 frag_start = nursery_start;
3065 /* clear scan starts */
3066 memset (nursery_section->scan_starts, 0, nursery_section->num_scan_start * sizeof (gpointer));
3067 for (i = start_pin; i < end_pin; ++i) {
3068 frag_end = pin_queue [i];
3069 /* remove the pin bit from pinned objects */
3070 unpin_object (frag_end);
3071 nursery_section->scan_starts [((char*)frag_end - (char*)nursery_section->data)/SCAN_START_SIZE] = frag_end;
3072 frag_size = frag_end - frag_start;
3074 add_nursery_frag (frag_size, frag_start, frag_end);
3075 frag_size = safe_object_get_size ((MonoObject*)pin_queue [i]);
3076 frag_size += ALLOC_ALIGN - 1;
3077 frag_size &= ~(ALLOC_ALIGN - 1);
3078 frag_start = (char*)pin_queue [i] + frag_size;
3080 nursery_last_pinned_end = frag_start;
3081 frag_end = nursery_real_end;
3082 frag_size = frag_end - frag_start;
3084 add_nursery_frag (frag_size, frag_start, frag_end);
3085 if (!nursery_fragments) {
3086 DEBUG (1, fprintf (gc_debug_file, "Nursery fully pinned (%d)\n", end_pin - start_pin));
3087 for (i = start_pin; i < end_pin; ++i) {
3088 DEBUG (3, fprintf (gc_debug_file, "Bastard pinning obj %p (%s), size: %d\n", pin_queue [i], safe_name (pin_queue [i]), safe_object_get_size (pin_queue [i])));
3093 nursery_next = nursery_frag_real_end = NULL;
3095 /* Clear TLABs for all threads */
3099 /* FIXME: later reduce code duplication here with the above
3100 * We don't keep track of section fragments for non-nursery sections yet, so
3104 build_section_fragments (GCMemSection *section)
3107 char *frag_start, *frag_end;
3110 /* clear scan starts */
3111 memset (section->scan_starts, 0, section->num_scan_start * sizeof (gpointer));
3112 frag_start = section->data;
3113 section->next_data = section->data;
3114 for (i = section->pin_queue_start; i < section->pin_queue_end; ++i) {
3115 frag_end = pin_queue [i];
3116 /* remove the pin bit from pinned objects */
3117 unpin_object (frag_end);
3118 if (frag_end >= section->data + section->size) {
3119 frag_end = section->data + section->size;
3121 section->scan_starts [((char*)frag_end - (char*)section->data)/SCAN_START_SIZE] = frag_end;
3123 frag_size = frag_end - frag_start;
3125 binary_protocol_empty (frag_start, frag_size);
3126 memset (frag_start, 0, frag_size);
3128 frag_size = safe_object_get_size ((MonoObject*)pin_queue [i]);
3129 frag_size += ALLOC_ALIGN - 1;
3130 frag_size &= ~(ALLOC_ALIGN - 1);
3131 frag_start = (char*)pin_queue [i] + frag_size;
3132 section->next_data = MAX (section->next_data, frag_start);
3134 frag_end = section->end_data;
3135 frag_size = frag_end - frag_start;
3137 binary_protocol_empty (frag_start, frag_size);
3138 memset (frag_start, 0, frag_size);
3143 scan_from_registered_roots (char *addr_start, char *addr_end, int root_type)
3147 for (i = 0; i < roots_hash_size [root_type]; ++i) {
3148 for (root = roots_hash [root_type][i]; root; root = root->next) {
3149 DEBUG (6, fprintf (gc_debug_file, "Precise root scan %p-%p (desc: %p)\n", root->start_root, root->end_root, (void*)root->root_desc));
3150 precisely_scan_objects_from ((void**)root->start_root, (void**)root->end_root, addr_start, addr_end, root->root_desc);
3156 dump_occupied (char *start, char *end, char *section_start)
3158 fprintf (heap_dump_file, "<occupied offset=\"%zd\" size=\"%zd\"/>\n", start - section_start, end - start);
3162 dump_section (GCMemSection *section, const char *type)
3164 char *start = section->data;
3165 char *end = section->data + section->size;
3166 char *occ_start = NULL;
3168 char *old_start = NULL; /* just for debugging */
3170 fprintf (heap_dump_file, "<section type=\"%s\" size=\"%zu\">\n", type, section->size);
3172 while (start < end) {
3176 if (!*(void**)start) {
3178 dump_occupied (occ_start, start, section->data);
3181 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
3184 g_assert (start < section->next_data);
3189 vt = (GCVTable*)LOAD_VTABLE (start);
3192 size = safe_object_get_size ((MonoObject*) start);
3193 size += ALLOC_ALIGN - 1;
3194 size &= ~(ALLOC_ALIGN - 1);
3197 fprintf (heap_dump_file, "<object offset=\"%d\" class=\"%s.%s\" size=\"%d\"/>\n",
3198 start - section->data,
3199 vt->klass->name_space, vt->klass->name,
3207 dump_occupied (occ_start, start, section->data);
3209 fprintf (heap_dump_file, "</section>\n");
3213 dump_heap (const char *type, int num, const char *reason)
3215 static char const *internal_mem_names [] = { "pin-queue", "fragment", "section", "scan-starts",
3216 "fin-table", "finalize-entry", "dislink-table",
3217 "dislink", "roots-table", "root-record", "statistics",
3218 "remset", "gray-queue", "store-remset" };
3220 GCMemSection *section;
3224 fprintf (heap_dump_file, "<collection type=\"%s\" num=\"%d\"", type, num);
3226 fprintf (heap_dump_file, " reason=\"%s\"", reason);
3227 fprintf (heap_dump_file, ">\n");
3228 fprintf (heap_dump_file, "<other-mem-usage type=\"pinned-chunks\" size=\"%ld\"/>\n", pinned_chunk_bytes_alloced);
3229 fprintf (heap_dump_file, "<other-mem-usage type=\"large-internal\" size=\"%ld\"/>\n", large_internal_bytes_alloced);
3230 fprintf (heap_dump_file, "<other-mem-usage type=\"mempools\" size=\"%ld\"/>\n", mono_mempool_get_bytes_allocated ());
3231 for (i = 0; i < INTERNAL_MEM_MAX; ++i)
3232 fprintf (heap_dump_file, "<other-mem-usage type=\"%s\" size=\"%ld\"/>\n", internal_mem_names [i], small_internal_mem_bytes [i]);
3233 fprintf (heap_dump_file, "<pinned type=\"stack\" bytes=\"%zu\"/>\n", pinned_byte_counts [PIN_TYPE_STACK]);
3234 /* fprintf (heap_dump_file, "<pinned type=\"static-data\" bytes=\"%d\"/>\n", pinned_byte_counts [PIN_TYPE_STATIC_DATA]); */
3235 fprintf (heap_dump_file, "<pinned type=\"other\" bytes=\"%zu\"/>\n", pinned_byte_counts [PIN_TYPE_OTHER]);
3237 dump_section (nursery_section, "nursery");
3239 for (section = section_list; section; section = section->block.next) {
3240 if (section != nursery_section)
3241 dump_section (section, "old");
3244 fprintf (heap_dump_file, "<los>\n");
3245 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
3246 MonoObject *obj = (MonoObject*) bigobj->data;
3247 MonoClass *class = mono_object_class (obj);
3249 fprintf (heap_dump_file, "<object class=\"%s.%s\" size=\"%d\"/>\n",
3250 class->name_space, class->name,
3251 safe_object_get_size (obj));
3253 fprintf (heap_dump_file, "</los>\n");
3255 fprintf (heap_dump_file, "</collection>\n");
3261 static gboolean inited = FALSE;
3263 #ifdef HEAVY_STATISTICS
3264 num_copy_object_called = 0;
3265 num_objects_copied = 0;
3271 #ifdef HEAVY_STATISTICS
3272 mono_counters_register ("WBarrier set field", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_field);
3273 mono_counters_register ("WBarrier set arrayref", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_arrayref);
3274 mono_counters_register ("WBarrier arrayref copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_arrayref_copy);
3275 mono_counters_register ("WBarrier generic store called", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_generic_store);
3276 mono_counters_register ("WBarrier generic store stored", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_generic_store_remset);
3277 mono_counters_register ("WBarrier set root", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_root);
3278 mono_counters_register ("WBarrier value copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_value_copy);
3279 mono_counters_register ("WBarrier object copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_object_copy);
3281 mono_counters_register ("# objects allocated", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_alloced);
3282 mono_counters_register ("# objects allocated degraded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_alloced_degraded);
3283 mono_counters_register ("# copy_object() called (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_called_nursery);
3284 mono_counters_register ("# objects copied (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_copied_nursery);
3285 mono_counters_register ("# copy_object() called (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_called_major);
3286 mono_counters_register ("# objects copied (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_copied_major);
3288 mono_counters_register ("# copy_object() failed from space", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_failed_from_space);
3289 mono_counters_register ("# copy_object() failed forwarded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_failed_forwarded);
3290 mono_counters_register ("# copy_object() failed pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_failed_pinned);
3291 mono_counters_register ("# copy_object() failed large or pinned chunk", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_failed_large_pinned);
3292 mono_counters_register ("# copy_object() failed to space", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_failed_to_space);
3294 mono_counters_register ("Store remsets", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_store_remsets);
3295 mono_counters_register ("Unique store remsets", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_store_remsets_unique);
3296 mono_counters_register ("Saved remsets 1", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_saved_remsets_1);
3297 mono_counters_register ("Saved remsets 2", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_saved_remsets_2);
3298 mono_counters_register ("Global remsets added", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_added);
3299 mono_counters_register ("Global remsets processed", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_processed);
3306 commit_stats (int generation)
3308 #ifdef HEAVY_STATISTICS
3309 if (generation == GENERATION_NURSERY) {
3310 stat_copy_object_called_nursery += num_copy_object_called;
3311 stat_objects_copied_nursery += num_objects_copied;
3313 g_assert (generation == GENERATION_OLD);
3314 stat_copy_object_called_major += num_copy_object_called;
3315 stat_objects_copied_major += num_objects_copied;
3321 * Collect objects in the nursery. Returns whether to trigger a major
3325 collect_nursery (size_t requested_size)
3327 size_t max_garbage_amount;
3329 char *orig_nursery_next;
3331 GCMemSection *section;
3332 int old_num_major_sections = num_major_sections;
3333 int sections_alloced;
3334 TV_DECLARE (all_atv);
3335 TV_DECLARE (all_btv);
3340 binary_protocol_collection (GENERATION_NURSERY);
3341 check_scan_starts ();
3344 orig_nursery_next = nursery_next;
3345 nursery_next = MAX (nursery_next, nursery_last_pinned_end);
3346 /* FIXME: optimize later to use the higher address where an object can be present */
3347 nursery_next = MAX (nursery_next, nursery_real_end);
3349 DEBUG (1, fprintf (gc_debug_file, "Start nursery collection %d %p-%p, size: %d\n", num_minor_gcs, nursery_start, nursery_next, (int)(nursery_next - nursery_start)));
3350 max_garbage_amount = nursery_next - nursery_start;
3351 g_assert (nursery_section->size >= max_garbage_amount);
3353 /* Clear all remaining nursery fragments, pinning depends on this */
3354 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION) {
3355 g_assert (orig_nursery_next <= nursery_frag_real_end);
3356 memset (orig_nursery_next, 0, nursery_frag_real_end - orig_nursery_next);
3357 for (frag = nursery_fragments; frag; frag = frag->next) {
3358 memset (frag->fragment_start, 0, frag->fragment_end - frag->fragment_start);
3363 check_for_xdomain_refs ();
3365 nursery_section->next_data = nursery_next;
3367 if (!to_space_section) {
3368 new_to_space_section ();
3370 /* we might have done degraded allocation since the
3372 g_assert (to_space_bumper <= to_space_section->next_data);
3373 to_space_bumper = to_space_section->next_data;
3375 to_space_section->is_to_space = TRUE;
3377 gray_object_queue_init ();
3380 mono_stats.minor_gc_count ++;
3381 /* world must be stopped already */
3382 TV_GETTIME (all_atv);
3384 /* pin from pinned handles */
3386 pin_from_roots (nursery_start, nursery_next);
3387 /* identify pinned objects */
3388 optimize_pin_queue (0);
3389 next_pin_slot = pin_objects_from_addresses (nursery_section, pin_queue, pin_queue + next_pin_slot, nursery_start, nursery_next);
3391 DEBUG (2, fprintf (gc_debug_file, "Finding pinned pointers: %d in %d usecs\n", next_pin_slot, TV_ELAPSED (atv, btv)));
3392 DEBUG (4, fprintf (gc_debug_file, "Start scan with %d pinned objects\n", next_pin_slot));
3394 if (consistency_check_at_minor_collection)
3395 check_consistency ();
3398 * walk all the roots and copy the young objects to the old generation,
3399 * starting from to_space
3402 scan_from_remsets (nursery_start, nursery_next);
3403 /* we don't have complete write barrier yet, so we scan all the old generation sections */
3405 DEBUG (2, fprintf (gc_debug_file, "Old generation scan: %d usecs\n", TV_ELAPSED (btv, atv)));
3407 /* the pinned objects are roots */
3408 for (i = 0; i < next_pin_slot; ++i) {
3409 DEBUG (6, fprintf (gc_debug_file, "Precise object scan %d of pinned %p (%s)\n", i, pin_queue [i], safe_name (pin_queue [i])));
3410 scan_object (pin_queue [i], nursery_start, nursery_next);
3412 /* registered roots, this includes static fields */
3413 scan_from_registered_roots (nursery_start, nursery_next, ROOT_TYPE_NORMAL);
3414 scan_from_registered_roots (nursery_start, nursery_next, ROOT_TYPE_WBARRIER);
3415 scan_thread_data (nursery_start, nursery_next, TRUE);
3416 /* alloc_pinned objects */
3417 scan_from_pinned_objects (nursery_start, nursery_next);
3419 DEBUG (2, fprintf (gc_debug_file, "Root scan: %d usecs\n", TV_ELAPSED (atv, btv)));
3421 finish_gray_stack (nursery_start, nursery_next, GENERATION_NURSERY);
3423 /* walk the pin_queue, build up the fragment list of free memory, unmark
3424 * pinned objects as we go, memzero() the empty fragments so they are ready for the
3427 build_nursery_fragments (0, next_pin_slot);
3429 DEBUG (2, fprintf (gc_debug_file, "Fragment creation: %d usecs, %zd bytes available\n", TV_ELAPSED (btv, atv), fragment_total));
3431 for (section = section_list; section; section = section->block.next) {
3432 if (section->is_to_space)
3433 section->is_to_space = FALSE;
3436 TV_GETTIME (all_btv);
3437 mono_stats.minor_gc_time_usecs += TV_ELAPSED (all_atv, all_btv);
3440 dump_heap ("minor", num_minor_gcs - 1, NULL);
3442 /* prepare the pin queue for the next collection */
3443 last_num_pinned = next_pin_slot;
3445 if (fin_ready_list || critical_fin_list) {
3446 DEBUG (4, fprintf (gc_debug_file, "Finalizer-thread wakeup: ready %d\n", num_ready_finalizers));
3447 mono_gc_finalize_notify ();
3451 g_assert (gray_object_queue_is_empty ());
3453 commit_stats (GENERATION_NURSERY);
3455 check_scan_starts ();
3457 sections_alloced = num_major_sections - old_num_major_sections;
3458 minor_collection_sections_alloced += sections_alloced;
3460 return minor_collection_sections_alloced > minor_collection_section_allowance;
3464 scan_from_pinned_chunk_if_marked (PinnedChunk *chunk, char *obj, size_t size, void *dummy)
3466 if (object_is_pinned (obj))
3467 scan_object (obj, NULL, (char*)-1);
3471 major_collection (const char *reason)
3473 GCMemSection *section, *prev_section;
3474 LOSObject *bigobj, *prevbo;
3478 TV_DECLARE (all_atv);
3479 TV_DECLARE (all_btv);
3482 /* FIXME: only use these values for the precise scan
3483 * note that to_space pointers should be excluded anyway...
3485 char *heap_start = NULL;
3486 char *heap_end = (char*)-1;
3487 size_t copy_space_required = 0;
3488 int old_num_major_sections = num_major_sections;
3489 int num_major_sections_saved, save_target, allowance_target;
3492 binary_protocol_collection (GENERATION_OLD);
3493 check_scan_starts ();
3496 DEBUG (1, fprintf (gc_debug_file, "Start major collection %d\n", num_major_gcs));
3498 mono_stats.major_gc_count ++;
3500 /* Clear all remaining nursery fragments, pinning depends on this */
3501 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION) {
3502 g_assert (nursery_next <= nursery_frag_real_end);
3503 memset (nursery_next, 0, nursery_frag_real_end - nursery_next);
3504 for (frag = nursery_fragments; frag; frag = frag->next) {
3505 memset (frag->fragment_start, 0, frag->fragment_end - frag->fragment_start);
3510 check_for_xdomain_refs ();
3513 * FIXME: implement Mark/Compact
3514 * Until that is done, we can just apply mostly the same alg as for the nursery:
3515 * this means we need a big section to potentially copy all the other sections, so
3516 * it is not ideal specially with large heaps.
3518 if (g_getenv ("MONO_GC_NO_MAJOR")) {
3519 collect_nursery (0);
3522 TV_GETTIME (all_atv);
3523 /* FIXME: make sure the nursery next_data ptr is updated */
3524 nursery_section->next_data = nursery_real_end;
3525 /* we should also coalesce scanning from sections close to each other
3526 * and deal with pointers outside of the sections later.
3528 /* The remsets are not useful for a major collection */
3530 /* world must be stopped already */
3533 DEBUG (6, fprintf (gc_debug_file, "Collecting pinned addresses\n"));
3534 pin_from_roots ((void*)lowest_heap_address, (void*)highest_heap_address);
3535 optimize_pin_queue (0);
3538 * pin_queue now contains all candidate pointers, sorted and
3539 * uniqued. We must do two passes now to figure out which
3540 * objects are pinned.
3542 * The first is to find within the pin_queue the area for each
3543 * section. This requires that the pin_queue be sorted. We
3544 * also process the LOS objects and pinned chunks here.
3546 * The second, destructive, pass is to reduce the section
3547 * areas to pointers to the actually pinned objects.
3549 DEBUG (6, fprintf (gc_debug_file, "Pinning from sections\n"));
3550 /* first pass for the sections */
3551 for (section = section_list; section; section = section->block.next) {
3553 DEBUG (6, fprintf (gc_debug_file, "Pinning from section %p (%p-%p)\n", section, section->data, section->end_data));
3554 find_optimized_pin_queue_area (section->data, section->end_data, &start, &end);
3555 DEBUG (6, fprintf (gc_debug_file, "Found %d pinning addresses in section %p (%d-%d)\n",
3556 end - start, section, start, end));
3557 section->pin_queue_start = start;
3558 section->pin_queue_end = end;
3560 /* identify possible pointers to the insize of large objects */
3561 DEBUG (6, fprintf (gc_debug_file, "Pinning from large objects\n"));
3562 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
3564 find_optimized_pin_queue_area (bigobj->data, (char*)bigobj->data + bigobj->size, &start, &end);
3566 pin_object (bigobj->data);
3568 pin_stats_register_object ((char*) bigobj->data, safe_object_get_size ((MonoObject*) bigobj->data));
3569 DEBUG (6, fprintf (gc_debug_file, "Marked large object %p (%s) size: %zd from roots\n", bigobj->data, safe_name (bigobj->data), bigobj->size));
3572 /* look for pinned addresses for pinned-alloc objects */
3573 DEBUG (6, fprintf (gc_debug_file, "Pinning from pinned-alloc objects\n"));
3574 for (chunk = pinned_chunk_list; chunk; chunk = chunk->block.next) {
3576 find_optimized_pin_queue_area (chunk->start_data, (char*)chunk + chunk->num_pages * FREELIST_PAGESIZE, &start, &end);
3578 mark_pinned_from_addresses (chunk, pin_queue + start, pin_queue + end);
3580 /* second pass for the sections */
3581 for (section = section_list; section; section = section->block.next) {
3582 int start = section->pin_queue_start;
3583 int end = section->pin_queue_end;
3586 reduced_to = pin_objects_from_addresses (section, pin_queue + start, pin_queue + end,
3587 section->data, section->next_data);
3588 section->pin_queue_start = start;
3589 section->pin_queue_end = start + reduced_to;
3591 copy_space_required += (char*)section->next_data - (char*)section->data;
3595 DEBUG (2, fprintf (gc_debug_file, "Finding pinned pointers: %d in %d usecs\n", next_pin_slot, TV_ELAPSED (atv, btv)));
3596 DEBUG (4, fprintf (gc_debug_file, "Start scan with %d pinned objects\n", next_pin_slot));
3598 new_to_space_section ();
3599 gray_object_queue_init ();
3601 /* the old generation doesn't need to be scanned (no remembered sets or card
3602 * table needed either): the only objects that must survive are those pinned and
3603 * those referenced by the precise roots.
3604 * mark any section without pinned objects, so we can free it since we will be able to
3605 * move all the objects.
3607 /* the pinned objects are roots (big objects are included in this list, too) */
3608 for (section = section_list; section; section = section->block.next) {
3609 for (i = section->pin_queue_start; i < section->pin_queue_end; ++i) {
3610 DEBUG (6, fprintf (gc_debug_file, "Precise object scan %d of pinned %p (%s)\n",
3611 i, pin_queue [i], safe_name (pin_queue [i])));
3612 scan_object (pin_queue [i], heap_start, heap_end);
3615 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
3616 if (object_is_pinned (bigobj->data)) {
3617 DEBUG (6, fprintf (gc_debug_file, "Precise object scan pinned LOS object %p (%s)\n",
3618 bigobj->data, safe_name (bigobj->data)));
3619 scan_object (bigobj->data, heap_start, heap_end);
3622 scan_pinned_objects (scan_from_pinned_chunk_if_marked, NULL);
3623 /* registered roots, this includes static fields */
3624 scan_from_registered_roots (heap_start, heap_end, ROOT_TYPE_NORMAL);
3625 scan_from_registered_roots (heap_start, heap_end, ROOT_TYPE_WBARRIER);
3627 scan_thread_data (heap_start, heap_end, TRUE);
3628 /* alloc_pinned objects */
3629 scan_from_pinned_objects (heap_start, heap_end);
3630 /* scan the list of objects ready for finalization */
3631 scan_finalizer_entries (fin_ready_list, heap_start, heap_end);
3632 scan_finalizer_entries (critical_fin_list, heap_start, heap_end);
3634 DEBUG (2, fprintf (gc_debug_file, "Root scan: %d usecs\n", TV_ELAPSED (btv, atv)));
3636 /* we need to go over the big object list to see if any was marked and scan it
3637 * And we need to make this in a loop, considering that objects referenced by finalizable
3638 * objects could reference big objects (this happens in finish_gray_stack ())
3640 scan_needed_big_objects (heap_start, heap_end);
3641 /* all the objects in the heap */
3642 finish_gray_stack (heap_start, heap_end, GENERATION_OLD);
3646 /* sweep the big objects list */
3648 for (bigobj = los_object_list; bigobj;) {
3649 if (object_is_pinned (bigobj->data)) {
3650 unpin_object (bigobj->data);
3651 bigobj->scanned = FALSE;
3654 /* not referenced anywhere, so we can free it */
3656 prevbo->next = bigobj->next;
3658 los_object_list = bigobj->next;
3660 bigobj = bigobj->next;
3661 free_large_object (to_free);
3665 bigobj = bigobj->next;
3667 /* unpin objects from the pinned chunks and free the unmarked ones */
3668 sweep_pinned_objects ();
3670 /* free the unused sections */
3671 prev_section = NULL;
3672 for (section = section_list; section;) {
3673 /* to_space doesn't need handling here and the nursery is special */
3674 if (section->is_to_space || section == nursery_section) {
3675 if (section->is_to_space)
3676 section->is_to_space = FALSE;
3677 prev_section = section;
3678 section = section->block.next;
3681 /* no pinning object, so the section is free */
3682 if (section->pin_queue_start == section->pin_queue_end) {
3683 GCMemSection *to_free;
3685 prev_section->block.next = section->block.next;
3687 section_list = section->block.next;
3689 section = section->block.next;
3690 free_major_section (to_free);
3693 DEBUG (6, fprintf (gc_debug_file, "Section %p has still pinned objects (%d)\n", section, section->pin_queue_end - section->pin_queue_start));
3694 build_section_fragments (section);
3696 prev_section = section;
3697 section = section->block.next;
3700 /* walk the pin_queue, build up the fragment list of free memory, unmark
3701 * pinned objects as we go, memzero() the empty fragments so they are ready for the
3704 build_nursery_fragments (nursery_section->pin_queue_start, nursery_section->pin_queue_end);
3706 TV_GETTIME (all_btv);
3707 mono_stats.major_gc_time_usecs += TV_ELAPSED (all_atv, all_btv);
3710 dump_heap ("major", num_major_gcs - 1, reason);
3712 /* prepare the pin queue for the next collection */
3714 if (fin_ready_list || critical_fin_list) {
3715 DEBUG (4, fprintf (gc_debug_file, "Finalizer-thread wakeup: ready %d\n", num_ready_finalizers));
3716 mono_gc_finalize_notify ();
3720 g_assert (gray_object_queue_is_empty ());
3722 commit_stats (GENERATION_OLD);
3724 num_major_sections_saved = MAX (old_num_major_sections - num_major_sections, 1);
3726 save_target = num_major_sections / 2;
3727 allowance_target = save_target * minor_collection_sections_alloced / num_major_sections_saved;
3729 minor_collection_section_allowance = MAX (MIN (allowance_target, num_major_sections), MIN_MINOR_COLLECTION_SECTION_ALLOWANCE);
3732 printf ("alloced %d saved %d target %d allowance %d\n",
3733 minor_collection_sections_alloced, num_major_sections_saved, allowance_target,
3734 minor_collection_section_allowance);
3737 minor_collection_sections_alloced = 0;
3739 check_scan_starts ();
3743 * Allocate a new section of memory to be used as old generation.
3745 static GCMemSection*
3746 alloc_major_section (void)
3748 GCMemSection *section;
3751 section = get_os_memory_aligned (MAJOR_SECTION_SIZE, TRUE);
3752 section->next_data = section->data = (char*)section + SIZEOF_GC_MEM_SECTION;
3753 g_assert (!((mword)section->data & 7));
3754 section->size = MAJOR_SECTION_SIZE - SIZEOF_GC_MEM_SECTION;
3755 section->end_data = section->data + section->size;
3756 UPDATE_HEAP_BOUNDARIES (section->data, section->end_data);
3757 total_alloc += section->size;
3758 DEBUG (3, fprintf (gc_debug_file, "New major heap section: (%p-%p), total: %zd\n", section->data, section->end_data, total_alloc));
3759 scan_starts = (section->size + SCAN_START_SIZE - 1) / SCAN_START_SIZE;
3760 section->scan_starts = get_internal_mem (sizeof (char*) * scan_starts, INTERNAL_MEM_SCAN_STARTS);
3761 section->num_scan_start = scan_starts;
3762 section->block.role = MEMORY_ROLE_GEN1;
3763 section->is_to_space = TRUE;
3765 /* add to the section list */
3766 section->block.next = section_list;
3767 section_list = section;
3769 ++num_major_sections;
3775 free_major_section (GCMemSection *section)
3777 DEBUG (3, fprintf (gc_debug_file, "Freed major section %p (%p-%p)\n", section, section->data, section->end_data));
3778 free_internal_mem (section->scan_starts, INTERNAL_MEM_SCAN_STARTS);
3779 free_os_memory (section, MAJOR_SECTION_SIZE);
3780 total_alloc -= MAJOR_SECTION_SIZE - SIZEOF_GC_MEM_SECTION;
3782 --num_major_sections;
3786 * When deciding if it's better to collect or to expand, keep track
3787 * of how much garbage was reclaimed with the last collection: if it's too
3789 * This is called when we could not allocate a small object.
3791 static void __attribute__((noinline))
3792 minor_collect_or_expand_inner (size_t size)
3794 int do_minor_collection = 1;
3796 if (!nursery_section) {
3800 if (do_minor_collection) {
3802 if (collect_nursery (size))
3803 major_collection ("minor overflow");
3804 DEBUG (2, fprintf (gc_debug_file, "Heap size: %zd, LOS size: %zd\n", total_alloc, los_memory_usage));
3806 /* this also sets the proper pointers for the next allocation */
3807 if (!search_fragment_for_size (size)) {
3809 /* TypeBuilder and MonoMethod are killing mcs with fragmentation */
3810 DEBUG (1, fprintf (gc_debug_file, "nursery collection didn't find enough room for %zd alloc (%d pinned)\n", size, last_num_pinned));
3811 for (i = 0; i < last_num_pinned; ++i) {
3812 DEBUG (3, fprintf (gc_debug_file, "Bastard pinning obj %p (%s), size: %d\n", pin_queue [i], safe_name (pin_queue [i]), safe_object_get_size (pin_queue [i])));
3817 //report_internal_mem_usage ();
3821 * ######################################################################
3822 * ######## Memory allocation from the OS
3823 * ######################################################################
3824 * This section of code deals with getting memory from the OS and
3825 * allocating memory for GC-internal data structures.
3826 * Internal memory can be handled with a freelist for small objects.
3830 * Allocate a big chunk of memory from the OS (usually 64KB to several megabytes).
3831 * This must not require any lock.
3834 get_os_memory (size_t size, int activate)
3837 unsigned long prot_flags = activate? MONO_MMAP_READ|MONO_MMAP_WRITE: MONO_MMAP_NONE;
3839 prot_flags |= MONO_MMAP_PRIVATE | MONO_MMAP_ANON;
3840 size += pagesize - 1;
3841 size &= ~(pagesize - 1);
3842 ptr = mono_valloc (0, size, prot_flags);
3847 * Free the memory returned by get_os_memory (), returning it to the OS.
3850 free_os_memory (void *addr, size_t size)
3852 mono_vfree (addr, size);
3859 report_pinned_chunk (PinnedChunk *chunk, int seq) {
3861 int i, free_pages, num_free, free_mem;
3863 for (i = 0; i < chunk->num_pages; ++i) {
3864 if (!chunk->page_sizes [i])
3867 printf ("Pinned chunk %d at %p, size: %d, pages: %d, free: %d\n", seq, chunk, chunk->num_pages * FREELIST_PAGESIZE, chunk->num_pages, free_pages);
3868 free_mem = FREELIST_PAGESIZE * free_pages;
3869 for (i = 0; i < FREELIST_NUM_SLOTS; ++i) {
3870 if (!chunk->free_list [i])
3873 p = chunk->free_list [i];
3878 printf ("\tfree list of size %d, %d items\n", freelist_sizes [i], num_free);
3879 free_mem += freelist_sizes [i] * num_free;
3881 printf ("\tfree memory in chunk: %d\n", free_mem);
3887 static G_GNUC_UNUSED void
3888 report_internal_mem_usage (void) {
3891 printf ("Internal memory usage:\n");
3893 for (chunk = internal_chunk_list; chunk; chunk = chunk->block.next) {
3894 report_pinned_chunk (chunk, i++);
3896 printf ("Pinned memory usage:\n");
3898 for (chunk = pinned_chunk_list; chunk; chunk = chunk->block.next) {
3899 report_pinned_chunk (chunk, i++);
3904 * the array of pointers from @start to @end contains conservative
3905 * pointers to objects inside @chunk: mark each referenced object
3909 mark_pinned_from_addresses (PinnedChunk *chunk, void **start, void **end)
3911 for (; start < end; start++) {
3912 char *addr = *start;
3913 int offset = (char*)addr - (char*)chunk;
3914 int page = offset / FREELIST_PAGESIZE;
3915 int obj_offset = page == 0? offset - ((char*)chunk->start_data - (char*)chunk): offset % FREELIST_PAGESIZE;
3916 int slot_size = chunk->page_sizes [page];
3918 /* the page is not allocated */
3921 /* would be faster if we restrict the sizes to power of two,
3922 * but that's a waste of memory: need to measure. it could reduce
3923 * fragmentation since there are less pages needed, if for example
3924 * someone interns strings of each size we end up with one page per
3925 * interned string (still this is just ~40 KB): with more fine-grained sizes
3926 * this increases the number of used pages.
3929 obj_offset /= slot_size;
3930 obj_offset *= slot_size;
3931 addr = (char*)chunk->start_data + obj_offset;
3933 obj_offset /= slot_size;
3934 obj_offset *= slot_size;
3935 addr = (char*)chunk + page * FREELIST_PAGESIZE + obj_offset;
3938 /* if the vtable is inside the chunk it's on the freelist, so skip */
3939 if (*ptr && (*ptr < (void*)chunk->start_data || *ptr > (void*)((char*)chunk + chunk->num_pages * FREELIST_PAGESIZE))) {
3940 binary_protocol_pin (addr, (gpointer)LOAD_VTABLE (addr), safe_object_get_size ((MonoObject*)addr));
3943 pin_stats_register_object ((char*) addr, safe_object_get_size ((MonoObject*) addr));
3944 DEBUG (6, fprintf (gc_debug_file, "Marked pinned object %p (%s) from roots\n", addr, safe_name (addr)));
3950 scan_pinned_objects (ScanPinnedObjectCallbackFunc callback, void *callback_data)
3957 for (chunk = pinned_chunk_list; chunk; chunk = chunk->block.next) {
3958 end_chunk = (char*)chunk + chunk->num_pages * FREELIST_PAGESIZE;
3959 DEBUG (6, fprintf (gc_debug_file, "Scanning pinned chunk %p (range: %p-%p)\n", chunk, chunk->start_data, end_chunk));
3960 for (i = 0; i < chunk->num_pages; ++i) {
3961 obj_size = chunk->page_sizes [i];
3964 p = i? (char*)chunk + i * FREELIST_PAGESIZE: chunk->start_data;
3965 endp = i? p + FREELIST_PAGESIZE: (char*)chunk + FREELIST_PAGESIZE;
3966 DEBUG (6, fprintf (gc_debug_file, "Page %d (size: %d, range: %p-%p)\n", i, obj_size, p, endp));
3967 while (p + obj_size <= endp) {
3969 DEBUG (9, fprintf (gc_debug_file, "Considering %p (vtable: %p)\n", ptr, *ptr));
3970 /* if the first word (the vtable) is outside the chunk we have an object */
3971 if (*ptr && (*ptr < (void*)chunk || *ptr >= end_chunk))
3972 callback (chunk, (char*)ptr, obj_size, callback_data);
3980 sweep_pinned_objects_callback (PinnedChunk *chunk, char *ptr, size_t size, void *data)
3982 if (object_is_pinned (ptr)) {
3984 DEBUG (6, fprintf (gc_debug_file, "Unmarked pinned object %p (%s)\n", ptr, safe_name (ptr)));
3986 DEBUG (6, fprintf (gc_debug_file, "Freeing unmarked pinned object %p (%s)\n", ptr, safe_name (ptr)));
3987 free_pinned_object (chunk, ptr, size);
3992 sweep_pinned_objects (void)
3994 scan_pinned_objects (sweep_pinned_objects_callback, NULL);
3998 scan_object_callback (PinnedChunk *chunk, char *ptr, size_t size, char **data)
4000 DEBUG (6, fprintf (gc_debug_file, "Precise object scan of alloc_pinned %p (%s)\n", ptr, safe_name (ptr)));
4001 /* FIXME: Put objects without references into separate chunks
4002 which do not need to be scanned */
4003 scan_object (ptr, data [0], data [1]);
4007 scan_from_pinned_objects (char *addr_start, char *addr_end)
4009 char *data [2] = { addr_start, addr_end };
4010 scan_pinned_objects ((ScanPinnedObjectCallbackFunc)scan_object_callback, data);
4014 * Find the slot number in the freelist for memory chunks that
4015 * can contain @size objects.
4018 slot_for_size (size_t size)
4021 /* do a binary search or lookup table later. */
4022 for (slot = 0; slot < FREELIST_NUM_SLOTS; ++slot) {
4023 if (freelist_sizes [slot] >= size)
4026 g_assert_not_reached ();
4031 * Build a free list for @size memory chunks from the memory area between
4032 * start_page and end_page.
4035 build_freelist (PinnedChunk *chunk, int slot, int size, char *start_page, char *end_page)
4039 /*g_print ("building freelist for slot %d, size %d in %p\n", slot, size, chunk);*/
4040 p = (void**)start_page;
4041 end = (void**)(end_page - size);
4042 g_assert (!chunk->free_list [slot]);
4043 chunk->free_list [slot] = p;
4044 while ((char*)p + size <= (char*)end) {
4046 *p = (void*)((char*)p + size);
4050 /*g_print ("%d items created, max: %d\n", count, (end_page - start_page) / size);*/
4054 alloc_pinned_chunk (void)
4058 int size = MAJOR_SECTION_SIZE;
4060 chunk = get_os_memory_aligned (size, TRUE);
4061 chunk->block.role = MEMORY_ROLE_PINNED;
4063 UPDATE_HEAP_BOUNDARIES (chunk, ((char*)chunk + size));
4064 total_alloc += size;
4065 pinned_chunk_bytes_alloced += size;
4067 /* setup the bookeeping fields */
4068 chunk->num_pages = size / FREELIST_PAGESIZE;
4069 offset = G_STRUCT_OFFSET (PinnedChunk, data);
4070 chunk->page_sizes = (void*)((char*)chunk + offset);
4071 offset += sizeof (int) * chunk->num_pages;
4072 offset += ALLOC_ALIGN - 1;
4073 offset &= ~(ALLOC_ALIGN - 1);
4074 chunk->free_list = (void*)((char*)chunk + offset);
4075 offset += sizeof (void*) * FREELIST_NUM_SLOTS;
4076 offset += ALLOC_ALIGN - 1;
4077 offset &= ~(ALLOC_ALIGN - 1);
4078 chunk->start_data = (void*)((char*)chunk + offset);
4080 /* allocate the first page to the freelist */
4081 chunk->page_sizes [0] = PINNED_FIRST_SLOT_SIZE;
4082 build_freelist (chunk, slot_for_size (PINNED_FIRST_SLOT_SIZE), PINNED_FIRST_SLOT_SIZE, chunk->start_data, ((char*)chunk + FREELIST_PAGESIZE));
4083 DEBUG (4, fprintf (gc_debug_file, "Allocated pinned chunk %p, size: %d\n", chunk, size));
4084 min_pinned_chunk_addr = MIN (min_pinned_chunk_addr, (char*)chunk->start_data);
4085 max_pinned_chunk_addr = MAX (max_pinned_chunk_addr, ((char*)chunk + size));
4089 /* assumes freelist for slot is empty, so try to alloc a new page */
4091 get_chunk_freelist (PinnedChunk *chunk, int slot)
4095 p = chunk->free_list [slot];
4097 chunk->free_list [slot] = *p;
4100 for (i = 0; i < chunk->num_pages; ++i) {
4102 if (chunk->page_sizes [i])
4104 size = freelist_sizes [slot];
4105 chunk->page_sizes [i] = size;
4106 build_freelist (chunk, slot, size, (char*)chunk + FREELIST_PAGESIZE * i, (char*)chunk + FREELIST_PAGESIZE * (i + 1));
4110 p = chunk->free_list [slot];
4112 chunk->free_list [slot] = *p;
4119 alloc_from_freelist (size_t size)
4123 PinnedChunk *pchunk;
4124 slot = slot_for_size (size);
4125 /*g_print ("using slot %d for size %d (slot size: %d)\n", slot, size, freelist_sizes [slot]);*/
4126 g_assert (size <= freelist_sizes [slot]);
4127 for (pchunk = pinned_chunk_list; pchunk; pchunk = pchunk->block.next) {
4128 void **p = pchunk->free_list [slot];
4130 /*g_print ("found freelist for slot %d in chunk %p, returning %p, next %p\n", slot, pchunk, p, *p);*/
4131 pchunk->free_list [slot] = *p;
4135 for (pchunk = pinned_chunk_list; pchunk; pchunk = pchunk->block.next) {
4136 res = get_chunk_freelist (pchunk, slot);
4140 pchunk = alloc_pinned_chunk ();
4141 /* FIXME: handle OOM */
4142 pchunk->block.next = pinned_chunk_list;
4143 pinned_chunk_list = pchunk;
4144 res = get_chunk_freelist (pchunk, slot);
4148 /* used for the GC-internal data structures */
4149 /* FIXME: add support for bigger sizes by allocating more than one page
4153 get_internal_mem (size_t size, int type)
4157 PinnedChunk *pchunk;
4159 if (size > freelist_sizes [FREELIST_NUM_SLOTS - 1]) {
4160 LargeInternalMemHeader *mh;
4162 size += sizeof (LargeInternalMemHeader);
4163 mh = get_os_memory (size, TRUE);
4164 mh->magic = LARGE_INTERNAL_MEM_HEADER_MAGIC;
4167 large_internal_bytes_alloced += size;
4172 slot = slot_for_size (size);
4173 g_assert (size <= freelist_sizes [slot]);
4175 small_internal_mem_bytes [type] += freelist_sizes [slot];
4177 for (pchunk = internal_chunk_list; pchunk; pchunk = pchunk->block.next) {
4178 void **p = pchunk->free_list [slot];
4180 pchunk->free_list [slot] = *p;
4181 memset (p, 0, size);
4185 for (pchunk = internal_chunk_list; pchunk; pchunk = pchunk->block.next) {
4186 res = get_chunk_freelist (pchunk, slot);
4188 memset (res, 0, size);
4192 pchunk = alloc_pinned_chunk ();
4193 /* FIXME: handle OOM */
4194 pchunk->block.next = internal_chunk_list;
4195 internal_chunk_list = pchunk;
4196 res = get_chunk_freelist (pchunk, slot);
4197 memset (res, 0, size);
4202 free_internal_mem (void *addr, int type)
4204 PinnedChunk *pchunk;
4205 LargeInternalMemHeader *mh;
4208 for (pchunk = internal_chunk_list; pchunk; pchunk = pchunk->block.next) {
4209 /*printf ("trying to free %p in %p (pages: %d)\n", addr, pchunk, pchunk->num_pages);*/
4210 if (addr >= (void*)pchunk && (char*)addr < (char*)pchunk + pchunk->num_pages * FREELIST_PAGESIZE) {
4211 int offset = (char*)addr - (char*)pchunk;
4212 int page = offset / FREELIST_PAGESIZE;
4213 int slot = slot_for_size (pchunk->page_sizes [page]);
4215 *p = pchunk->free_list [slot];
4216 pchunk->free_list [slot] = p;
4218 small_internal_mem_bytes [type] -= freelist_sizes [slot];
4223 mh = (LargeInternalMemHeader*)((char*)addr - G_STRUCT_OFFSET (LargeInternalMemHeader, data));
4224 g_assert (mh->magic == LARGE_INTERNAL_MEM_HEADER_MAGIC);
4225 large_internal_bytes_alloced -= mh->size;
4226 free_os_memory (mh, mh->size);
4230 * ######################################################################
4231 * ######## Object allocation
4232 * ######################################################################
4233 * This section of code deals with allocating memory for objects.
4234 * There are several ways:
4235 * *) allocate large objects
4236 * *) allocate normal objects
4237 * *) fast lock-free allocation
4238 * *) allocation of pinned objects
4242 free_large_object (LOSObject *obj)
4244 size_t size = obj->size;
4245 DEBUG (4, fprintf (gc_debug_file, "Freed large object %p, size %zd\n", obj->data, obj->size));
4246 binary_protocol_empty (obj->data, obj->size);
4248 los_memory_usage -= size;
4249 size += sizeof (LOSObject);
4250 size += pagesize - 1;
4251 size &= ~(pagesize - 1);
4252 total_alloc -= size;
4254 free_os_memory (obj, size);
4258 * Objects with size >= 64KB are allocated in the large object space.
4259 * They are currently kept track of with a linked list.
4260 * They don't move, so there is no need to pin them during collection
4261 * and we avoid the memcpy overhead.
4263 static void* __attribute__((noinline))
4264 alloc_large_inner (MonoVTable *vtable, size_t size)
4269 int just_did_major_gc = FALSE;
4271 g_assert (size > MAX_SMALL_OBJ_SIZE);
4273 if (los_memory_usage > next_los_collection) {
4274 DEBUG (4, fprintf (gc_debug_file, "Should trigger major collection: req size %zd (los already: %zu, limit: %zu)\n", size, los_memory_usage, next_los_collection));
4275 just_did_major_gc = TRUE;
4277 major_collection ("LOS overflow");
4279 /* later increase based on a percent of the heap size */
4280 next_los_collection = los_memory_usage + 5*1024*1024;
4283 alloc_size += sizeof (LOSObject);
4284 alloc_size += pagesize - 1;
4285 alloc_size &= ~(pagesize - 1);
4286 /* FIXME: handle OOM */
4287 obj = get_os_memory (alloc_size, TRUE);
4289 vtslot = (void**)obj->data;
4291 total_alloc += alloc_size;
4292 UPDATE_HEAP_BOUNDARIES (obj->data, (char*)obj->data + size);
4293 obj->next = los_object_list;
4294 los_object_list = obj;
4295 los_memory_usage += size;
4297 DEBUG (4, fprintf (gc_debug_file, "Allocated large object %p, vtable: %p (%s), size: %zd\n", obj->data, vtable, vtable->klass->name, size));
4298 binary_protocol_alloc (obj->data, vtable, size);
4302 /* check if we have a suitable fragment in nursery_fragments to be able to allocate
4303 * an object of size @size
4304 * Return FALSE if not found (which means we need a collection)
4307 search_fragment_for_size (size_t size)
4309 Fragment *frag, *prev;
4310 DEBUG (4, fprintf (gc_debug_file, "Searching nursery fragment %p, size: %zd\n", nursery_frag_real_end, size));
4312 if (nursery_frag_real_end > nursery_next && nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
4313 /* Clear the remaining space, pinning depends on this */
4314 memset (nursery_next, 0, nursery_frag_real_end - nursery_next);
4317 for (frag = nursery_fragments; frag; frag = frag->next) {
4318 if (size <= (frag->fragment_end - frag->fragment_start)) {
4319 /* remove from the list */
4321 prev->next = frag->next;
4323 nursery_fragments = frag->next;
4324 nursery_next = frag->fragment_start;
4325 nursery_frag_real_end = frag->fragment_end;
4327 DEBUG (4, fprintf (gc_debug_file, "Using nursery fragment %p-%p, size: %zd (req: %zd)\n", nursery_next, nursery_frag_real_end, nursery_frag_real_end - nursery_next, size));
4328 frag->next = fragment_freelist;
4329 fragment_freelist = frag;
4338 * size is already rounded up and we hold the GC lock.
4341 alloc_degraded (MonoVTable *vtable, size_t size)
4343 GCMemSection *section;
4345 g_assert (size <= MAX_SMALL_OBJ_SIZE);
4346 HEAVY_STAT (++stat_objects_alloced_degraded);
4347 for (section = section_list; section; section = section->block.next) {
4348 if (section != nursery_section && (section->end_data - section->next_data) >= size) {
4349 p = (void**)section->next_data;
4354 section = alloc_major_section ();
4355 section->is_to_space = FALSE;
4356 /* FIXME: handle OOM */
4357 p = (void**)section->next_data;
4359 section->next_data += size;
4360 degraded_mode += size;
4361 DEBUG (3, fprintf (gc_debug_file, "Allocated (degraded) object %p, vtable: %p (%s), size: %zd in section %p\n", p, vtable, vtable->klass->name, size, section));
4367 * Provide a variant that takes just the vtable for small fixed-size objects.
4368 * The aligned size is already computed and stored in vt->gc_descr.
4369 * Note: every SCAN_START_SIZE or so we are given the chance to do some special
4370 * processing. We can keep track of where objects start, for example,
4371 * so when we scan the thread stacks for pinned objects, we can start
4372 * a search for the pinned object in SCAN_START_SIZE chunks.
4375 mono_gc_alloc_obj_nolock (MonoVTable *vtable, size_t size)
4377 /* FIXME: handle OOM */
4383 HEAVY_STAT (++stat_objects_alloced);
4385 size += ALLOC_ALIGN - 1;
4386 size &= ~(ALLOC_ALIGN - 1);
4388 g_assert (vtable->gc_descr);
4390 if (G_UNLIKELY (collect_before_allocs)) {
4391 if (nursery_section) {
4393 collect_nursery (0);
4395 if (!degraded_mode && !search_fragment_for_size (size)) {
4397 g_assert_not_reached ();
4403 * We must already have the lock here instead of after the
4404 * fast path because we might be interrupted in the fast path
4405 * (after confirming that new_next < TLAB_TEMP_END) by the GC,
4406 * and we'll end up allocating an object in a fragment which
4407 * no longer belongs to us.
4409 * The managed allocator does not do this, but it's treated
4410 * specially by the world-stopping code.
4413 if (size > MAX_SMALL_OBJ_SIZE) {
4414 p = alloc_large_inner (vtable, size);
4416 /* tlab_next and tlab_temp_end are TLS vars so accessing them might be expensive */
4418 p = (void**)TLAB_NEXT;
4419 /* FIXME: handle overflow */
4420 new_next = (char*)p + size;
4421 TLAB_NEXT = new_next;
4423 if (G_LIKELY (new_next < TLAB_TEMP_END)) {
4427 * FIXME: We might need a memory barrier here so the change to tlab_next is
4428 * visible before the vtable store.
4431 DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
4432 binary_protocol_alloc (p , vtable, size);
4433 g_assert (*p == NULL);
4436 g_assert (TLAB_NEXT == new_next);
4443 /* there are two cases: the object is too big or we run out of space in the TLAB */
4444 /* we also reach here when the thread does its first allocation after a minor
4445 * collection, since the tlab_ variables are initialized to NULL.
4446 * there can be another case (from ORP), if we cooperate with the runtime a bit:
4447 * objects that need finalizers can have the high bit set in their size
4448 * so the above check fails and we can readily add the object to the queue.
4449 * This avoids taking again the GC lock when registering, but this is moot when
4450 * doing thread-local allocation, so it may not be a good idea.
4452 g_assert (TLAB_NEXT == new_next);
4453 if (TLAB_NEXT >= TLAB_REAL_END) {
4455 * Run out of space in the TLAB. When this happens, some amount of space
4456 * remains in the TLAB, but not enough to satisfy the current allocation
4457 * request. Currently, we retire the TLAB in all cases, later we could
4458 * keep it if the remaining space is above a treshold, and satisfy the
4459 * allocation directly from the nursery.
4462 /* when running in degraded mode, we continue allocing that way
4463 * for a while, to decrease the number of useless nursery collections.
4465 if (degraded_mode && degraded_mode < DEFAULT_NURSERY_SIZE) {
4466 p = alloc_degraded (vtable, size);
4470 if (size > tlab_size) {
4471 /* Allocate directly from the nursery */
4472 if (nursery_next + size >= nursery_frag_real_end) {
4473 if (!search_fragment_for_size (size)) {
4474 minor_collect_or_expand_inner (size);
4475 if (degraded_mode) {
4476 p = alloc_degraded (vtable, size);
4482 p = (void*)nursery_next;
4483 nursery_next += size;
4484 if (nursery_next > nursery_frag_real_end) {
4489 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
4490 memset (p, 0, size);
4493 DEBUG (3, fprintf (gc_debug_file, "Retire TLAB: %p-%p [%ld]\n", TLAB_START, TLAB_REAL_END, (long)(TLAB_REAL_END - TLAB_NEXT - size)));
4495 if (nursery_next + tlab_size >= nursery_frag_real_end) {
4496 res = search_fragment_for_size (tlab_size);
4498 minor_collect_or_expand_inner (tlab_size);
4499 if (degraded_mode) {
4500 p = alloc_degraded (vtable, size);
4506 /* Allocate a new TLAB from the current nursery fragment */
4507 TLAB_START = nursery_next;
4508 nursery_next += tlab_size;
4509 TLAB_NEXT = TLAB_START;
4510 TLAB_REAL_END = TLAB_START + tlab_size;
4511 TLAB_TEMP_END = TLAB_START + MIN (SCAN_START_SIZE, tlab_size);
4513 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
4514 memset (TLAB_START, 0, tlab_size);
4516 /* Allocate from the TLAB */
4517 p = (void*)TLAB_NEXT;
4519 g_assert (TLAB_NEXT <= TLAB_REAL_END);
4521 nursery_section->scan_starts [((char*)p - (char*)nursery_section->data)/SCAN_START_SIZE] = (char*)p;
4524 /* Reached tlab_temp_end */
4526 /* record the scan start so we can find pinned objects more easily */
4527 nursery_section->scan_starts [((char*)p - (char*)nursery_section->data)/SCAN_START_SIZE] = (char*)p;
4528 /* we just bump tlab_temp_end as well */
4529 TLAB_TEMP_END = MIN (TLAB_REAL_END, TLAB_NEXT + SCAN_START_SIZE);
4530 DEBUG (5, fprintf (gc_debug_file, "Expanding local alloc: %p-%p\n", TLAB_NEXT, TLAB_TEMP_END));
4534 DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
4535 binary_protocol_alloc (p, vtable, size);
4542 mono_gc_try_alloc_obj_nolock (MonoVTable *vtable, size_t size)
4548 size += ALLOC_ALIGN - 1;
4549 size &= ~(ALLOC_ALIGN - 1);
4551 g_assert (vtable->gc_descr);
4552 if (size <= MAX_SMALL_OBJ_SIZE) {
4553 /* tlab_next and tlab_temp_end are TLS vars so accessing them might be expensive */
4555 p = (void**)TLAB_NEXT;
4556 /* FIXME: handle overflow */
4557 new_next = (char*)p + size;
4558 TLAB_NEXT = new_next;
4560 if (G_LIKELY (new_next < TLAB_TEMP_END)) {
4564 * FIXME: We might need a memory barrier here so the change to tlab_next is
4565 * visible before the vtable store.
4568 HEAVY_STAT (++stat_objects_alloced);
4570 DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
4571 binary_protocol_alloc (p, vtable, size);
4572 g_assert (*p == NULL);
4575 g_assert (TLAB_NEXT == new_next);
4584 mono_gc_alloc_obj (MonoVTable *vtable, size_t size)
4587 #ifndef DISABLE_CRITICAL_REGION
4589 ENTER_CRITICAL_REGION;
4590 res = mono_gc_try_alloc_obj_nolock (vtable, size);
4592 EXIT_CRITICAL_REGION;
4595 EXIT_CRITICAL_REGION;
4598 res = mono_gc_alloc_obj_nolock (vtable, size);
4604 mono_gc_alloc_vector (MonoVTable *vtable, size_t size, uintptr_t max_length)
4607 #ifndef DISABLE_CRITICAL_REGION
4609 ENTER_CRITICAL_REGION;
4610 arr = mono_gc_try_alloc_obj_nolock (vtable, size);
4612 arr->max_length = max_length;
4613 EXIT_CRITICAL_REGION;
4616 EXIT_CRITICAL_REGION;
4621 arr = mono_gc_alloc_obj_nolock (vtable, size);
4622 arr->max_length = max_length;
4630 mono_gc_alloc_array (MonoVTable *vtable, size_t size, uintptr_t max_length, uintptr_t bounds_size)
4633 MonoArrayBounds *bounds;
4637 arr = mono_gc_alloc_obj_nolock (vtable, size);
4638 arr->max_length = max_length;
4640 bounds = (MonoArrayBounds*)((char*)arr + size - bounds_size);
4641 arr->bounds = bounds;
4649 mono_gc_alloc_string (MonoVTable *vtable, size_t size, gint32 len)
4652 #ifndef DISABLE_CRITICAL_REGION
4654 ENTER_CRITICAL_REGION;
4655 str = mono_gc_try_alloc_obj_nolock (vtable, size);
4658 EXIT_CRITICAL_REGION;
4661 EXIT_CRITICAL_REGION;
4666 str = mono_gc_alloc_obj_nolock (vtable, size);
4675 * To be used for interned strings and possibly MonoThread, reflection handles.
4676 * We may want to explicitly free these objects.
4679 mono_gc_alloc_pinned_obj (MonoVTable *vtable, size_t size)
4681 /* FIXME: handle OOM */
4683 size += ALLOC_ALIGN - 1;
4684 size &= ~(ALLOC_ALIGN - 1);
4686 if (size > MAX_FREELIST_SIZE) {
4687 /* large objects are always pinned anyway */
4688 p = alloc_large_inner (vtable, size);
4690 p = alloc_from_freelist (size);
4691 memset (p, 0, size);
4693 DEBUG (6, fprintf (gc_debug_file, "Allocated pinned object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
4694 binary_protocol_alloc (p, vtable, size);
4701 * ######################################################################
4702 * ######## Finalization support
4703 * ######################################################################
4707 * this is valid for the nursery: if the object has been forwarded it means it's
4708 * still refrenced from a root. If it is pinned it's still alive as well.
4709 * Return TRUE if @obj is ready to be finalized.
4711 #define object_is_fin_ready(obj) (!object_is_pinned (obj) && !object_is_forwarded (obj))
4714 is_critical_finalizer (FinalizeEntry *entry)
4719 if (!mono_defaults.critical_finalizer_object)
4722 obj = entry->object;
4723 class = ((MonoVTable*)LOAD_VTABLE (obj))->klass;
4725 return mono_class_has_parent (class, mono_defaults.critical_finalizer_object);
4729 queue_finalization_entry (FinalizeEntry *entry) {
4730 if (is_critical_finalizer (entry)) {
4731 entry->next = critical_fin_list;
4732 critical_fin_list = entry;
4734 entry->next = fin_ready_list;
4735 fin_ready_list = entry;
4739 /* LOCKING: requires that the GC lock is held */
4741 rehash_fin_table (FinalizeEntryHashTable *hash_table)
4743 FinalizeEntry **finalizable_hash = hash_table->table;
4744 mword finalizable_hash_size = hash_table->size;
4747 FinalizeEntry **new_hash;
4748 FinalizeEntry *entry, *next;
4749 int new_size = g_spaced_primes_closest (hash_table->num_registered);
4751 new_hash = get_internal_mem (new_size * sizeof (FinalizeEntry*), INTERNAL_MEM_FIN_TABLE);
4752 for (i = 0; i < finalizable_hash_size; ++i) {
4753 for (entry = finalizable_hash [i]; entry; entry = next) {
4754 hash = mono_object_hash (entry->object) % new_size;
4756 entry->next = new_hash [hash];
4757 new_hash [hash] = entry;
4760 free_internal_mem (finalizable_hash, INTERNAL_MEM_FIN_TABLE);
4761 hash_table->table = new_hash;
4762 hash_table->size = new_size;
4765 /* LOCKING: requires that the GC lock is held */
4767 rehash_fin_table_if_necessary (FinalizeEntryHashTable *hash_table)
4769 if (hash_table->num_registered >= hash_table->size * 2)
4770 rehash_fin_table (hash_table);
4773 /* LOCKING: requires that the GC lock is held */
4775 finalize_in_range (char *start, char *end, int generation)
4777 FinalizeEntryHashTable *hash_table = get_finalize_entry_hash_table (generation);
4778 FinalizeEntry *entry, *prev;
4780 FinalizeEntry **finalizable_hash = hash_table->table;
4781 mword finalizable_hash_size = hash_table->size;
4785 for (i = 0; i < finalizable_hash_size; ++i) {
4787 for (entry = finalizable_hash [i]; entry;) {
4788 if ((char*)entry->object >= start && (char*)entry->object < end && !object_is_in_to_space (entry->object)) {
4789 gboolean is_fin_ready = object_is_fin_ready (entry->object);
4790 char *copy = copy_object (entry->object, start, end);
4793 FinalizeEntry *next;
4794 /* remove and put in fin_ready_list */
4796 prev->next = entry->next;
4798 finalizable_hash [i] = entry->next;
4800 num_ready_finalizers++;
4801 hash_table->num_registered--;
4802 queue_finalization_entry (entry);
4803 /* Make it survive */
4804 from = entry->object;
4805 entry->object = copy;
4806 DEBUG (5, fprintf (gc_debug_file, "Queueing object for finalization: %p (%s) (was at %p) (%d/%d)\n", entry->object, safe_name (entry->object), from, num_ready_finalizers, hash_table->num_registered));
4810 char *from = entry->object;
4811 if (hash_table == &minor_finalizable_hash && !ptr_in_nursery (copy)) {
4812 FinalizeEntry *next = entry->next;
4813 unsigned int major_hash;
4814 /* remove from the list */
4816 prev->next = entry->next;
4818 finalizable_hash [i] = entry->next;
4819 hash_table->num_registered--;
4821 entry->object = copy;
4823 /* insert it into the major hash */
4824 rehash_fin_table_if_necessary (&major_finalizable_hash);
4825 major_hash = mono_object_hash ((MonoObject*) copy) %
4826 major_finalizable_hash.size;
4827 entry->next = major_finalizable_hash.table [major_hash];
4828 major_finalizable_hash.table [major_hash] = entry;
4829 major_finalizable_hash.num_registered++;
4831 DEBUG (5, fprintf (gc_debug_file, "Promoting finalization of object %p (%s) (was at %p) to major table\n", copy, safe_name (copy), from));
4836 /* update pointer */
4837 DEBUG (5, fprintf (gc_debug_file, "Updating object for finalization: %p (%s) (was at %p)\n", entry->object, safe_name (entry->object), from));
4838 entry->object = copy;
4843 entry = entry->next;
4848 /* LOCKING: requires that the GC lock is held */
4850 null_link_in_range (char *start, char *end, int generation)
4852 DisappearingLinkHashTable *hash = get_dislink_hash_table (generation);
4853 DisappearingLink **disappearing_link_hash = hash->table;
4854 int disappearing_link_hash_size = hash->size;
4855 DisappearingLink *entry, *prev;
4857 if (!hash->num_links)
4859 for (i = 0; i < disappearing_link_hash_size; ++i) {
4861 for (entry = disappearing_link_hash [i]; entry;) {
4862 char *object = DISLINK_OBJECT (entry);
4863 if (object >= start && object < end && !object_is_in_to_space (object)) {
4864 gboolean track = DISLINK_TRACK (entry);
4865 if (!track && object_is_fin_ready (object)) {
4866 void **p = entry->link;
4867 DisappearingLink *old;
4869 /* remove from list */
4871 prev->next = entry->next;
4873 disappearing_link_hash [i] = entry->next;
4874 DEBUG (5, fprintf (gc_debug_file, "Dislink nullified at %p to GCed object %p\n", p, object));
4876 free_internal_mem (entry, INTERNAL_MEM_DISLINK);
4881 char *copy = copy_object (object, start, end);
4883 /* Update pointer if it's moved. If the object
4884 * has been moved out of the nursery, we need to
4885 * remove the link from the minor hash table to
4888 * FIXME: what if an object is moved earlier?
4891 if (hash == &minor_disappearing_link_hash && !ptr_in_nursery (copy)) {
4892 void **link = entry->link;
4893 DisappearingLink *old;
4894 /* remove from list */
4896 prev->next = entry->next;
4898 disappearing_link_hash [i] = entry->next;
4900 free_internal_mem (entry, INTERNAL_MEM_DISLINK);
4904 add_or_remove_disappearing_link ((MonoObject*)copy, link,
4905 track, GENERATION_OLD);
4907 DEBUG (5, fprintf (gc_debug_file, "Upgraded dislink at %p to major because object %p moved to %p\n", link, object, copy));
4911 /* We set the track resurrection bit to
4912 * FALSE if the object is to be finalized
4913 * so that the object can be collected in
4914 * the next cycle (i.e. after it was
4917 *entry->link = HIDE_POINTER (copy,
4918 object_is_fin_ready (object) ? FALSE : track);
4919 DEBUG (5, fprintf (gc_debug_file, "Updated dislink at %p to %p\n", entry->link, DISLINK_OBJECT (entry)));
4924 entry = entry->next;
4929 /* LOCKING: requires that the GC lock is held */
4931 null_links_for_domain (MonoDomain *domain, int generation)
4933 DisappearingLinkHashTable *hash = get_dislink_hash_table (generation);
4934 DisappearingLink **disappearing_link_hash = hash->table;
4935 int disappearing_link_hash_size = hash->size;
4936 DisappearingLink *entry, *prev;
4938 for (i = 0; i < disappearing_link_hash_size; ++i) {
4940 for (entry = disappearing_link_hash [i]; entry; ) {
4941 char *object = DISLINK_OBJECT (entry);
4942 /* FIXME: actually there should be no object
4943 left in the domain with a non-null vtable
4944 (provided we remove the Thread special
4946 if (object && (!((MonoObject*)object)->vtable || mono_object_domain (object) == domain)) {
4947 DisappearingLink *next = entry->next;
4952 disappearing_link_hash [i] = next;
4954 if (*(entry->link)) {
4955 *(entry->link) = NULL;
4956 g_warning ("Disappearing link %p not freed", entry->link);
4958 free_internal_mem (entry, INTERNAL_MEM_DISLINK);
4965 entry = entry->next;
4970 /* LOCKING: requires that the GC lock is held */
4972 finalizers_for_domain (MonoDomain *domain, MonoObject **out_array, int out_size,
4973 FinalizeEntryHashTable *hash_table)
4975 FinalizeEntry **finalizable_hash = hash_table->table;
4976 mword finalizable_hash_size = hash_table->size;
4977 FinalizeEntry *entry, *prev;
4980 if (no_finalize || !out_size || !out_array)
4983 for (i = 0; i < finalizable_hash_size; ++i) {
4985 for (entry = finalizable_hash [i]; entry;) {
4986 if (mono_object_domain (entry->object) == domain) {
4987 FinalizeEntry *next;
4988 /* remove and put in out_array */
4990 prev->next = entry->next;
4992 finalizable_hash [i] = entry->next;
4994 hash_table->num_registered--;
4995 out_array [count ++] = entry->object;
4996 DEBUG (5, fprintf (gc_debug_file, "Collecting object for finalization: %p (%s) (%d/%d)\n", entry->object, safe_name (entry->object), num_ready_finalizers, hash_table->num_registered));
4998 if (count == out_size)
5003 entry = entry->next;
5010 * mono_gc_finalizers_for_domain:
5011 * @domain: the unloading appdomain
5012 * @out_array: output array
5013 * @out_size: size of output array
5015 * Store inside @out_array up to @out_size objects that belong to the unloading
5016 * appdomain @domain. Returns the number of stored items. Can be called repeteadly
5017 * until it returns 0.
5018 * The items are removed from the finalizer data structure, so the caller is supposed
5020 * @out_array should be on the stack to allow the GC to know the objects are still alive.
5023 mono_gc_finalizers_for_domain (MonoDomain *domain, MonoObject **out_array, int out_size)
5028 result = finalizers_for_domain (domain, out_array, out_size, &minor_finalizable_hash);
5029 if (result < out_size) {
5030 result += finalizers_for_domain (domain, out_array + result, out_size - result,
5031 &major_finalizable_hash);
5039 register_for_finalization (MonoObject *obj, void *user_data, int generation)
5041 FinalizeEntryHashTable *hash_table = get_finalize_entry_hash_table (generation);
5042 FinalizeEntry **finalizable_hash;
5043 mword finalizable_hash_size;
5044 FinalizeEntry *entry, *prev;
5048 g_assert (user_data == NULL || user_data == mono_gc_run_finalize);
5049 hash = mono_object_hash (obj);
5051 rehash_fin_table_if_necessary (hash_table);
5052 finalizable_hash = hash_table->table;
5053 finalizable_hash_size = hash_table->size;
5054 hash %= finalizable_hash_size;
5056 for (entry = finalizable_hash [hash]; entry; entry = entry->next) {
5057 if (entry->object == obj) {
5059 /* remove from the list */
5061 prev->next = entry->next;
5063 finalizable_hash [hash] = entry->next;
5064 hash_table->num_registered--;
5065 DEBUG (5, fprintf (gc_debug_file, "Removed finalizer %p for object: %p (%s) (%d)\n", entry, obj, obj->vtable->klass->name, hash_table->num_registered));
5066 free_internal_mem (entry, INTERNAL_MEM_FINALIZE_ENTRY);
5074 /* request to deregister, but already out of the list */
5078 entry = get_internal_mem (sizeof (FinalizeEntry), INTERNAL_MEM_FINALIZE_ENTRY);
5079 entry->object = obj;
5080 entry->next = finalizable_hash [hash];
5081 finalizable_hash [hash] = entry;
5082 hash_table->num_registered++;
5083 DEBUG (5, fprintf (gc_debug_file, "Added finalizer %p for object: %p (%s) (%d) to %s table\n", entry, obj, obj->vtable->klass->name, hash_table->num_registered, generation_name (generation)));
5088 mono_gc_register_for_finalization (MonoObject *obj, void *user_data)
5090 if (ptr_in_nursery (obj))
5091 register_for_finalization (obj, user_data, GENERATION_NURSERY);
5093 register_for_finalization (obj, user_data, GENERATION_OLD);
5097 rehash_dislink (DisappearingLinkHashTable *hash_table)
5099 DisappearingLink **disappearing_link_hash = hash_table->table;
5100 int disappearing_link_hash_size = hash_table->size;
5103 DisappearingLink **new_hash;
5104 DisappearingLink *entry, *next;
5105 int new_size = g_spaced_primes_closest (hash_table->num_links);
5107 new_hash = get_internal_mem (new_size * sizeof (DisappearingLink*), INTERNAL_MEM_DISLINK_TABLE);
5108 for (i = 0; i < disappearing_link_hash_size; ++i) {
5109 for (entry = disappearing_link_hash [i]; entry; entry = next) {
5110 hash = mono_aligned_addr_hash (entry->link) % new_size;
5112 entry->next = new_hash [hash];
5113 new_hash [hash] = entry;
5116 free_internal_mem (disappearing_link_hash, INTERNAL_MEM_DISLINK_TABLE);
5117 hash_table->table = new_hash;
5118 hash_table->size = new_size;
5121 /* LOCKING: assumes the GC lock is held */
5123 add_or_remove_disappearing_link (MonoObject *obj, void **link, gboolean track, int generation)
5125 DisappearingLinkHashTable *hash_table = get_dislink_hash_table (generation);
5126 DisappearingLink *entry, *prev;
5128 DisappearingLink **disappearing_link_hash = hash_table->table;
5129 int disappearing_link_hash_size = hash_table->size;
5131 if (hash_table->num_links >= disappearing_link_hash_size * 2) {
5132 rehash_dislink (hash_table);
5133 disappearing_link_hash = hash_table->table;
5134 disappearing_link_hash_size = hash_table->size;
5136 /* FIXME: add check that link is not in the heap */
5137 hash = mono_aligned_addr_hash (link) % disappearing_link_hash_size;
5138 entry = disappearing_link_hash [hash];
5140 for (; entry; entry = entry->next) {
5141 /* link already added */
5142 if (link == entry->link) {
5143 /* NULL obj means remove */
5146 prev->next = entry->next;
5148 disappearing_link_hash [hash] = entry->next;
5149 hash_table->num_links--;
5150 DEBUG (5, fprintf (gc_debug_file, "Removed dislink %p (%d) from %s table\n", entry, hash_table->num_links, generation_name (generation)));
5151 free_internal_mem (entry, INTERNAL_MEM_DISLINK);
5154 *link = HIDE_POINTER (obj, track); /* we allow the change of object */
5162 entry = get_internal_mem (sizeof (DisappearingLink), INTERNAL_MEM_DISLINK);
5163 *link = HIDE_POINTER (obj, track);
5165 entry->next = disappearing_link_hash [hash];
5166 disappearing_link_hash [hash] = entry;
5167 hash_table->num_links++;
5168 DEBUG (5, fprintf (gc_debug_file, "Added dislink %p for object: %p (%s) at %p to %s table\n", entry, obj, obj->vtable->klass->name, link, generation_name (generation)));
5171 /* LOCKING: assumes the GC lock is held */
5173 mono_gc_register_disappearing_link (MonoObject *obj, void **link, gboolean track)
5175 add_or_remove_disappearing_link (NULL, link, FALSE, GENERATION_NURSERY);
5176 add_or_remove_disappearing_link (NULL, link, FALSE, GENERATION_OLD);
5178 if (ptr_in_nursery (obj))
5179 add_or_remove_disappearing_link (obj, link, track, GENERATION_NURSERY);
5181 add_or_remove_disappearing_link (obj, link, track, GENERATION_OLD);
5186 mono_gc_invoke_finalizers (void)
5188 FinalizeEntry *entry = NULL;
5189 gboolean entry_is_critical = FALSE;
5192 /* FIXME: batch to reduce lock contention */
5193 while (fin_ready_list || critical_fin_list) {
5197 FinalizeEntry **list = entry_is_critical ? &critical_fin_list : &fin_ready_list;
5199 /* We have finalized entry in the last
5200 interation, now we need to remove it from
5203 *list = entry->next;
5205 FinalizeEntry *e = *list;
5206 while (e->next != entry)
5208 e->next = entry->next;
5210 free_internal_mem (entry, INTERNAL_MEM_FINALIZE_ENTRY);
5214 /* Now look for the first non-null entry. */
5215 for (entry = fin_ready_list; entry && !entry->object; entry = entry->next)
5218 entry_is_critical = FALSE;
5220 entry_is_critical = TRUE;
5221 for (entry = critical_fin_list; entry && !entry->object; entry = entry->next)
5226 g_assert (entry->object);
5227 num_ready_finalizers--;
5228 obj = entry->object;
5229 entry->object = NULL;
5230 DEBUG (7, fprintf (gc_debug_file, "Finalizing object %p (%s)\n", obj, safe_name (obj)));
5238 g_assert (entry->object == NULL);
5240 /* the object is on the stack so it is pinned */
5241 /*g_print ("Calling finalizer for object: %p (%s)\n", entry->object, safe_name (entry->object));*/
5242 mono_gc_run_finalize (obj, NULL);
5249 mono_gc_pending_finalizers (void)
5251 return fin_ready_list || critical_fin_list;
5254 /* Negative value to remove */
5256 mono_gc_add_memory_pressure (gint64 value)
5258 /* FIXME: Use interlocked functions */
5260 memory_pressure += value;
5265 * ######################################################################
5266 * ######## registered roots support
5267 * ######################################################################
5271 rehash_roots (gboolean pinned)
5275 RootRecord **new_hash;
5276 RootRecord *entry, *next;
5279 new_size = g_spaced_primes_closest (num_roots_entries [pinned]);
5280 new_hash = get_internal_mem (new_size * sizeof (RootRecord*), INTERNAL_MEM_ROOTS_TABLE);
5281 for (i = 0; i < roots_hash_size [pinned]; ++i) {
5282 for (entry = roots_hash [pinned][i]; entry; entry = next) {
5283 hash = mono_aligned_addr_hash (entry->start_root) % new_size;
5285 entry->next = new_hash [hash];
5286 new_hash [hash] = entry;
5289 free_internal_mem (roots_hash [pinned], INTERNAL_MEM_ROOTS_TABLE);
5290 roots_hash [pinned] = new_hash;
5291 roots_hash_size [pinned] = new_size;
5295 find_root (int root_type, char *start, guint32 addr_hash)
5297 RootRecord *new_root;
5299 guint32 hash = addr_hash % roots_hash_size [root_type];
5300 for (new_root = roots_hash [root_type][hash]; new_root; new_root = new_root->next) {
5301 /* we allow changing the size and the descriptor (for thread statics etc) */
5302 if (new_root->start_root == start) {
5311 * We do not coalesce roots.
5314 mono_gc_register_root_inner (char *start, size_t size, void *descr, int root_type)
5316 RootRecord *new_root;
5317 unsigned int hash, addr_hash = mono_aligned_addr_hash (start);
5320 for (i = 0; i < ROOT_TYPE_NUM; ++i) {
5321 if (num_roots_entries [i] >= roots_hash_size [i] * 2)
5324 for (i = 0; i < ROOT_TYPE_NUM; ++i) {
5325 new_root = find_root (i, start, addr_hash);
5326 /* we allow changing the size and the descriptor (for thread statics etc) */
5328 size_t old_size = new_root->end_root - new_root->start_root;
5329 new_root->end_root = new_root->start_root + size;
5330 g_assert (((new_root->root_desc != 0) && (descr != NULL)) ||
5331 ((new_root->root_desc == 0) && (descr == NULL)));
5332 new_root->root_desc = (mword)descr;
5334 roots_size -= old_size;
5339 new_root = get_internal_mem (sizeof (RootRecord), INTERNAL_MEM_ROOT_RECORD);
5341 new_root->start_root = start;
5342 new_root->end_root = new_root->start_root + size;
5343 new_root->root_desc = (mword)descr;
5345 hash = addr_hash % roots_hash_size [root_type];
5346 num_roots_entries [root_type]++;
5347 new_root->next = roots_hash [root_type] [hash];
5348 roots_hash [root_type][hash] = new_root;
5349 DEBUG (3, fprintf (gc_debug_file, "Added root %p for range: %p-%p, descr: %p (%d/%d bytes)\n", new_root, new_root->start_root, new_root->end_root, descr, (int)size, (int)roots_size));
5359 mono_gc_register_root (char *start, size_t size, void *descr)
5361 return mono_gc_register_root_inner (start, size, descr, descr ? ROOT_TYPE_NORMAL : ROOT_TYPE_PINNED);
5365 mono_gc_register_root_wbarrier (char *start, size_t size, void *descr)
5367 return mono_gc_register_root_inner (start, size, descr, ROOT_TYPE_WBARRIER);
5371 mono_gc_deregister_root (char* addr)
5373 RootRecord *tmp, *prev;
5374 unsigned int hash, addr_hash = mono_aligned_addr_hash (addr);
5378 for (root_type = 0; root_type < ROOT_TYPE_NUM; ++root_type) {
5379 hash = addr_hash % roots_hash_size [root_type];
5380 tmp = roots_hash [root_type][hash];
5383 if (tmp->start_root == (char*)addr) {
5385 prev->next = tmp->next;
5387 roots_hash [root_type][hash] = tmp->next;
5388 roots_size -= (tmp->end_root - tmp->start_root);
5389 num_roots_entries [root_type]--;
5390 DEBUG (3, fprintf (gc_debug_file, "Removed root %p for range: %p-%p\n", tmp, tmp->start_root, tmp->end_root));
5391 free_internal_mem (tmp, INTERNAL_MEM_ROOT_RECORD);
5402 * ######################################################################
5403 * ######## Thread handling (stop/start code)
5404 * ######################################################################
5407 /* FIXME: handle large/small config */
5408 #define THREAD_HASH_SIZE 11
5409 #define HASH_PTHREAD_T(id) (((unsigned int)(id) >> 4) * 2654435761u)
5411 static SgenThreadInfo* thread_table [THREAD_HASH_SIZE];
5413 #if USE_SIGNAL_BASED_START_STOP_WORLD
5415 static MonoSemType suspend_ack_semaphore;
5416 static MonoSemType *suspend_ack_semaphore_ptr;
5417 static unsigned int global_stop_count = 0;
5419 static int suspend_signal_num = SIGXFSZ;
5421 static int suspend_signal_num = SIGPWR;
5423 static int restart_signal_num = SIGXCPU;
5424 static sigset_t suspend_signal_mask;
5425 static mword cur_thread_regs [ARCH_NUM_REGS] = {0};
5427 /* LOCKING: assumes the GC lock is held */
5428 static SgenThreadInfo*
5429 thread_info_lookup (ARCH_THREAD_TYPE id)
5431 unsigned int hash = HASH_PTHREAD_T (id) % THREAD_HASH_SIZE;
5432 SgenThreadInfo *info;
5434 info = thread_table [hash];
5435 while (info && !ARCH_THREAD_EQUALS (info->id, id)) {
5442 update_current_thread_stack (void *start)
5444 void *ptr = cur_thread_regs;
5445 SgenThreadInfo *info = thread_info_lookup (ARCH_GET_THREAD ());
5447 info->stack_start = align_pointer (&ptr);
5448 g_assert (info->stack_start >= info->stack_start_limit && info->stack_start < info->stack_end);
5449 ARCH_STORE_REGS (ptr);
5450 info->stopped_regs = ptr;
5451 if (gc_callbacks.thread_suspend_func)
5452 gc_callbacks.thread_suspend_func (info->runtime_data, NULL);
5456 signal_desc (int signum)
5458 if (signum == suspend_signal_num)
5460 if (signum == restart_signal_num)
5466 * Define this and use the "xdomain-checks" MONO_GC_DEBUG option to
5467 * have cross-domain checks in the write barrier.
5469 //#define XDOMAIN_CHECKS_IN_WBARRIER
5471 #ifndef HEAVY_STATISTICS
5472 #define MANAGED_ALLOCATION
5473 #ifndef XDOMAIN_CHECKS_IN_WBARRIER
5474 #define MANAGED_WBARRIER
5479 is_ip_in_managed_allocator (MonoDomain *domain, gpointer ip);
5482 wait_for_suspend_ack (int count)
5486 for (i = 0; i < count; ++i) {
5487 while ((result = MONO_SEM_WAIT (suspend_ack_semaphore_ptr)) != 0) {
5488 if (errno != EINTR) {
5489 g_error ("sem_wait ()");
5495 /* LOCKING: assumes the GC lock is held */
5497 thread_handshake (int signum)
5499 int count, i, result;
5500 SgenThreadInfo *info;
5501 pthread_t me = pthread_self ();
5504 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5505 for (info = thread_table [i]; info; info = info->next) {
5506 DEBUG (4, fprintf (gc_debug_file, "considering thread %p for signal %d (%s)\n", info, signum, signal_desc (signum)));
5507 if (ARCH_THREAD_EQUALS (info->id, me)) {
5508 DEBUG (4, fprintf (gc_debug_file, "Skip (equal): %p, %p\n", (void*)me, (void*)info->id));
5511 /*if (signum == suspend_signal_num && info->stop_count == global_stop_count)
5513 result = pthread_kill (info->id, signum);
5515 DEBUG (4, fprintf (gc_debug_file, "thread %p signal sent\n", info));
5518 DEBUG (4, fprintf (gc_debug_file, "thread %p signal failed: %d (%s)\n", (void*)info->id, result, strerror (result)));
5524 wait_for_suspend_ack (count);
5530 restart_threads_until_none_in_managed_allocator (void)
5532 SgenThreadInfo *info;
5533 int i, result, num_threads_died = 0;
5534 int sleep_duration = -1;
5537 int restart_count = 0, restarted_count = 0;
5538 /* restart all threads that stopped in the
5540 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5541 for (info = thread_table [i]; info; info = info->next) {
5544 if (!info->stack_start || info->in_critical_region ||
5545 is_ip_in_managed_allocator (info->stopped_domain, info->stopped_ip)) {
5546 binary_protocol_thread_restart ((gpointer)info->id);
5547 result = pthread_kill (info->id, restart_signal_num);
5554 /* we set the stopped_ip to
5555 NULL for threads which
5556 we're not restarting so
5557 that we can easily identify
5559 info->stopped_ip = NULL;
5560 info->stopped_domain = NULL;
5564 /* if no threads were restarted, we're done */
5565 if (restart_count == 0)
5568 /* wait for the threads to signal their restart */
5569 wait_for_suspend_ack (restart_count);
5571 if (sleep_duration < 0) {
5575 g_usleep (sleep_duration);
5576 sleep_duration += 10;
5579 /* stop them again */
5580 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5581 for (info = thread_table [i]; info; info = info->next) {
5582 if (info->skip || info->stopped_ip == NULL)
5584 result = pthread_kill (info->id, suspend_signal_num);
5592 /* some threads might have died */
5593 num_threads_died += restart_count - restarted_count;
5594 /* wait for the threads to signal their suspension
5596 wait_for_suspend_ack (restart_count);
5599 return num_threads_died;
5602 /* LOCKING: assumes the GC lock is held (by the stopping thread) */
5604 suspend_handler (int sig, siginfo_t *siginfo, void *context)
5606 SgenThreadInfo *info;
5609 int old_errno = errno;
5610 gpointer regs [ARCH_NUM_REGS];
5611 gpointer stack_start;
5613 id = pthread_self ();
5614 info = thread_info_lookup (id);
5615 info->stopped_domain = mono_domain_get ();
5616 info->stopped_ip = (gpointer) ARCH_SIGCTX_IP (context);
5617 stop_count = global_stop_count;
5618 /* duplicate signal */
5619 if (0 && info->stop_count == stop_count) {
5623 #ifdef HAVE_KW_THREAD
5624 /* update the remset info in the thread data structure */
5625 info->remset = remembered_set;
5627 stack_start = (char*) ARCH_SIGCTX_SP (context) - REDZONE_SIZE;
5628 /* If stack_start is not within the limits, then don't set it
5629 in info and we will be restarted. */
5630 if (stack_start >= info->stack_start_limit && info->stack_start <= info->stack_end) {
5631 info->stack_start = stack_start;
5633 ARCH_COPY_SIGCTX_REGS (regs, context);
5634 info->stopped_regs = regs;
5636 g_assert (!info->stack_start);
5639 /* Notify the JIT */
5640 if (gc_callbacks.thread_suspend_func)
5641 gc_callbacks.thread_suspend_func (info->runtime_data, context);
5643 DEBUG (4, fprintf (gc_debug_file, "Posting suspend_ack_semaphore for suspend from %p %p\n", info, (gpointer)ARCH_GET_THREAD ()));
5644 /* notify the waiting thread */
5645 MONO_SEM_POST (suspend_ack_semaphore_ptr);
5646 info->stop_count = stop_count;
5648 /* wait until we receive the restart signal */
5651 sigsuspend (&suspend_signal_mask);
5652 } while (info->signal != restart_signal_num);
5654 DEBUG (4, fprintf (gc_debug_file, "Posting suspend_ack_semaphore for resume from %p %p\n", info, (gpointer)ARCH_GET_THREAD ()));
5655 /* notify the waiting thread */
5656 MONO_SEM_POST (suspend_ack_semaphore_ptr);
5662 restart_handler (int sig)
5664 SgenThreadInfo *info;
5665 int old_errno = errno;
5667 info = thread_info_lookup (pthread_self ());
5668 info->signal = restart_signal_num;
5669 DEBUG (4, fprintf (gc_debug_file, "Restart handler in %p %p\n", info, (gpointer)ARCH_GET_THREAD ()));
5674 static TV_DECLARE (stop_world_time);
5675 static unsigned long max_pause_usec = 0;
5677 /* LOCKING: assumes the GC lock is held */
5683 update_current_thread_stack (&count);
5685 global_stop_count++;
5686 DEBUG (3, fprintf (gc_debug_file, "stopping world n %d from %p %p\n", global_stop_count, thread_info_lookup (ARCH_GET_THREAD ()), (gpointer)ARCH_GET_THREAD ()));
5687 TV_GETTIME (stop_world_time);
5688 count = thread_handshake (suspend_signal_num);
5689 count -= restart_threads_until_none_in_managed_allocator ();
5690 g_assert (count >= 0);
5691 DEBUG (3, fprintf (gc_debug_file, "world stopped %d thread(s)\n", count));
5695 /* LOCKING: assumes the GC lock is held */
5697 restart_world (void)
5700 SgenThreadInfo *info;
5701 TV_DECLARE (end_sw);
5704 /* notify the profiler of the leftovers */
5705 if (G_UNLIKELY (mono_profiler_events & MONO_PROFILE_GC_MOVES)) {
5706 if (moved_objects_idx) {
5707 mono_profiler_gc_moves (moved_objects, moved_objects_idx);
5708 moved_objects_idx = 0;
5711 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5712 for (info = thread_table [i]; info; info = info->next) {
5713 info->stack_start = NULL;
5714 info->stopped_regs = NULL;
5718 count = thread_handshake (restart_signal_num);
5719 TV_GETTIME (end_sw);
5720 usec = TV_ELAPSED (stop_world_time, end_sw);
5721 max_pause_usec = MAX (usec, max_pause_usec);
5722 DEBUG (2, fprintf (gc_debug_file, "restarted %d thread(s) (pause time: %d usec, max: %d)\n", count, (int)usec, (int)max_pause_usec));
5726 #endif /* USE_SIGNAL_BASED_START_STOP_WORLD */
5729 mono_gc_set_gc_callbacks (MonoGCCallbacks *callbacks)
5731 gc_callbacks = *callbacks;
5734 /* Variables holding start/end nursery so it won't have to be passed at every call */
5735 static void *scan_area_arg_start, *scan_area_arg_end;
5738 mono_gc_conservatively_scan_area (void *start, void *end)
5740 conservatively_pin_objects_from (start, end, scan_area_arg_start, scan_area_arg_end, PIN_TYPE_STACK);
5744 mono_gc_scan_object (void *obj)
5746 return copy_object (obj, scan_area_arg_start, scan_area_arg_end);
5750 * Mark from thread stacks and registers.
5753 scan_thread_data (void *start_nursery, void *end_nursery, gboolean precise)
5756 SgenThreadInfo *info;
5758 scan_area_arg_start = start_nursery;
5759 scan_area_arg_end = end_nursery;
5761 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5762 for (info = thread_table [i]; info; info = info->next) {
5764 DEBUG (2, fprintf (gc_debug_file, "Skipping dead thread %p, range: %p-%p, size: %zd\n", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start));
5767 DEBUG (3, fprintf (gc_debug_file, "Scanning thread %p, range: %p-%p, size: %zd, pinned=%d\n", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start, next_pin_slot));
5768 if (gc_callbacks.thread_mark_func && !conservative_stack_mark)
5769 gc_callbacks.thread_mark_func (info->runtime_data, info->stack_start, info->stack_end, precise);
5771 conservatively_pin_objects_from (info->stack_start, info->stack_end, start_nursery, end_nursery, PIN_TYPE_STACK);
5774 conservatively_pin_objects_from (info->stopped_regs, info->stopped_regs + ARCH_NUM_REGS,
5775 start_nursery, end_nursery, PIN_TYPE_STACK);
5781 find_pinning_ref_from_thread (char *obj, size_t size)
5784 SgenThreadInfo *info;
5785 char *endobj = obj + size;
5787 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5788 for (info = thread_table [i]; info; info = info->next) {
5789 char **start = (char**)info->stack_start;
5792 while (start < (char**)info->stack_end) {
5793 if (*start >= obj && *start < endobj) {
5794 DEBUG (0, fprintf (gc_debug_file, "Object %p referenced in thread %p (id %p) at %p, stack: %p-%p\n", obj, info, (gpointer)info->id, start, info->stack_start, info->stack_end));
5799 /* FIXME: check info->stopped_regs */
5805 ptr_on_stack (void *ptr)
5807 gpointer stack_start = &stack_start;
5808 SgenThreadInfo *info = thread_info_lookup (ARCH_GET_THREAD ());
5810 if (ptr >= stack_start && ptr < (gpointer)info->stack_end)
5816 handle_remset (mword *p, void *start_nursery, void *end_nursery, gboolean global)
5823 HEAVY_STAT (++stat_global_remsets_processed);
5825 /* FIXME: exclude stack locations */
5826 switch ((*p) & REMSET_TYPE_MASK) {
5827 case REMSET_LOCATION:
5829 //__builtin_prefetch (ptr);
5830 if (((void*)ptr < start_nursery || (void*)ptr >= end_nursery)) {
5831 gpointer old = *ptr;
5832 *ptr = copy_object (*ptr, start_nursery, end_nursery);
5833 DEBUG (9, fprintf (gc_debug_file, "Overwrote remset at %p with %p\n", ptr, *ptr));
5835 binary_protocol_ptr_update (ptr, old, *ptr, (gpointer)LOAD_VTABLE (*ptr), safe_object_get_size (*ptr));
5836 if (!global && *ptr >= start_nursery && *ptr < end_nursery) {
5838 * If the object is pinned, each reference to it from nonpinned objects
5839 * becomes part of the global remset, which can grow very large.
5841 DEBUG (9, fprintf (gc_debug_file, "Add to global remset because of pinning %p (%p %s)\n", ptr, *ptr, safe_name (*ptr)));
5842 add_to_global_remset (ptr, FALSE);
5845 DEBUG (9, fprintf (gc_debug_file, "Skipping remset at %p holding %p\n", ptr, *ptr));
5849 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
5850 if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery))
5853 while (count-- > 0) {
5854 *ptr = copy_object (*ptr, start_nursery, end_nursery);
5855 DEBUG (9, fprintf (gc_debug_file, "Overwrote remset at %p with %p (count: %d)\n", ptr, *ptr, (int)count));
5856 if (!global && *ptr >= start_nursery && *ptr < end_nursery)
5857 add_to_global_remset (ptr, FALSE);
5862 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
5863 if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery))
5865 scan_object ((char*)ptr, start_nursery, end_nursery);
5867 case REMSET_OTHER: {
5868 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
5872 if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery))
5877 ptr = (void**) scan_vtype ((char*)ptr, desc, start_nursery, end_nursery);
5879 case REMSET_ROOT_LOCATION:
5880 /* Same as REMSET_LOCATION, but the address is not required to be in the heap */
5881 *ptr = copy_object (*ptr, start_nursery, end_nursery);
5882 DEBUG (9, fprintf (gc_debug_file, "Overwrote root location remset at %p with %p\n", ptr, *ptr));
5883 if (!global && *ptr >= start_nursery && *ptr < end_nursery) {
5885 * If the object is pinned, each reference to it from nonpinned objects
5886 * becomes part of the global remset, which can grow very large.
5888 DEBUG (9, fprintf (gc_debug_file, "Add to global remset because of pinning %p (%p %s)\n", ptr, *ptr, safe_name (*ptr)));
5889 add_to_global_remset (ptr, TRUE);
5893 g_assert_not_reached ();
5898 g_assert_not_reached ();
5903 #ifdef HEAVY_STATISTICS
5905 collect_store_remsets (RememberedSet *remset, mword *bumper)
5907 mword *p = remset->data;
5912 while (p < remset->store_next) {
5913 switch ((*p) & REMSET_TYPE_MASK) {
5914 case REMSET_LOCATION:
5917 ++stat_saved_remsets_1;
5919 if (*p == last1 || *p == last2) {
5920 ++stat_saved_remsets_2;
5938 case REMSET_ROOT_LOCATION:
5942 g_assert_not_reached ();
5946 g_assert_not_reached ();
5956 RememberedSet *remset;
5958 SgenThreadInfo *info;
5960 mword *addresses, *bumper, *p, *r;
5962 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5963 for (info = thread_table [i]; info; info = info->next) {
5964 for (remset = info->remset; remset; remset = remset->next)
5965 size += remset->store_next - remset->data;
5968 for (remset = freed_thread_remsets; remset; remset = remset->next)
5969 size += remset->store_next - remset->data;
5970 for (remset = global_remset; remset; remset = remset->next)
5971 size += remset->store_next - remset->data;
5973 bumper = addresses = get_internal_mem (sizeof (mword) * size, INTERNAL_MEM_STATISTICS);
5975 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5976 for (info = thread_table [i]; info; info = info->next) {
5977 for (remset = info->remset; remset; remset = remset->next)
5978 bumper = collect_store_remsets (remset, bumper);
5981 for (remset = global_remset; remset; remset = remset->next)
5982 bumper = collect_store_remsets (remset, bumper);
5983 for (remset = freed_thread_remsets; remset; remset = remset->next)
5984 bumper = collect_store_remsets (remset, bumper);
5986 g_assert (bumper <= addresses + size);
5988 stat_store_remsets += bumper - addresses;
5990 sort_addresses ((void**)addresses, bumper - addresses);
5993 while (r < bumper) {
5999 stat_store_remsets_unique += p - addresses;
6001 free_internal_mem (addresses, INTERNAL_MEM_STATISTICS);
6006 clear_thread_store_remset_buffer (SgenThreadInfo *info)
6008 *info->store_remset_buffer_index_addr = 0;
6009 memset (*info->store_remset_buffer_addr, 0, sizeof (gpointer) * STORE_REMSET_BUFFER_SIZE);
6013 scan_from_remsets (void *start_nursery, void *end_nursery)
6016 SgenThreadInfo *info;
6017 RememberedSet *remset;
6018 GenericStoreRememberedSet *store_remset;
6019 mword *p, *next_p, *store_pos;
6021 #ifdef HEAVY_STATISTICS
6025 /* the global one */
6026 for (remset = global_remset; remset; remset = remset->next) {
6027 DEBUG (4, fprintf (gc_debug_file, "Scanning global remset range: %p-%p, size: %zd\n", remset->data, remset->store_next, remset->store_next - remset->data));
6028 store_pos = remset->data;
6029 for (p = remset->data; p < remset->store_next; p = next_p) {
6032 next_p = handle_remset (p, start_nursery, end_nursery, TRUE);
6035 * Clear global remsets of locations which no longer point to the
6036 * nursery. Otherwise, they could grow indefinitely between major
6039 ptr = (p [0] & ~REMSET_TYPE_MASK);
6040 if ((p [0] & REMSET_TYPE_MASK) == REMSET_LOCATION) {
6041 if (ptr_in_nursery (*(void**)ptr))
6042 *store_pos ++ = p [0];
6044 g_assert ((p [0] & REMSET_TYPE_MASK) == REMSET_OTHER);
6045 g_assert (p [1] == REMSET_ROOT_LOCATION);
6046 if (ptr_in_nursery (*(void**)ptr)) {
6047 *store_pos ++ = p [0];
6048 *store_pos ++ = p [1];
6053 /* Truncate the remset */
6054 remset->store_next = store_pos;
6057 /* the generic store ones */
6058 store_remset = generic_store_remsets;
6059 while (store_remset) {
6060 GenericStoreRememberedSet *next = store_remset->next;
6062 for (i = 0; i < STORE_REMSET_BUFFER_SIZE - 1; ++i) {
6063 gpointer addr = store_remset->data [i];
6065 handle_remset ((mword*)&addr, start_nursery, end_nursery, FALSE);
6068 free_internal_mem (store_remset, INTERNAL_MEM_STORE_REMSET);
6070 store_remset = next;
6072 generic_store_remsets = NULL;
6074 /* the per-thread ones */
6075 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
6076 for (info = thread_table [i]; info; info = info->next) {
6077 RememberedSet *next;
6079 for (remset = info->remset; remset; remset = next) {
6080 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for thread %p, range: %p-%p, size: %zd\n", info, remset->data, remset->store_next, remset->store_next - remset->data));
6081 for (p = remset->data; p < remset->store_next;) {
6082 p = handle_remset (p, start_nursery, end_nursery, FALSE);
6084 remset->store_next = remset->data;
6085 next = remset->next;
6086 remset->next = NULL;
6087 if (remset != info->remset) {
6088 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
6089 free_internal_mem (remset, INTERNAL_MEM_REMSET);
6092 for (j = 0; j < *info->store_remset_buffer_index_addr; ++j)
6093 handle_remset ((mword*)*info->store_remset_buffer_addr + j + 1, start_nursery, end_nursery, FALSE);
6094 clear_thread_store_remset_buffer (info);
6098 /* the freed thread ones */
6099 while (freed_thread_remsets) {
6100 RememberedSet *next;
6101 remset = freed_thread_remsets;
6102 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for freed thread, range: %p-%p, size: %zd\n", remset->data, remset->store_next, remset->store_next - remset->data));
6103 for (p = remset->data; p < remset->store_next;) {
6104 p = handle_remset (p, start_nursery, end_nursery, FALSE);
6106 next = remset->next;
6107 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
6108 free_internal_mem (remset, INTERNAL_MEM_REMSET);
6109 freed_thread_remsets = next;
6114 * Clear the info in the remembered sets: we're doing a major collection, so
6115 * the per-thread ones are not needed and the global ones will be reconstructed
6119 clear_remsets (void)
6122 SgenThreadInfo *info;
6123 RememberedSet *remset, *next;
6125 /* the global list */
6126 for (remset = global_remset; remset; remset = next) {
6127 remset->store_next = remset->data;
6128 next = remset->next;
6129 remset->next = NULL;
6130 if (remset != global_remset) {
6131 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
6132 free_internal_mem (remset, INTERNAL_MEM_REMSET);
6135 /* the generic store ones */
6136 while (generic_store_remsets) {
6137 GenericStoreRememberedSet *gs_next = generic_store_remsets->next;
6138 free_internal_mem (generic_store_remsets, INTERNAL_MEM_STORE_REMSET);
6139 generic_store_remsets = gs_next;
6141 /* the per-thread ones */
6142 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
6143 for (info = thread_table [i]; info; info = info->next) {
6144 for (remset = info->remset; remset; remset = next) {
6145 remset->store_next = remset->data;
6146 next = remset->next;
6147 remset->next = NULL;
6148 if (remset != info->remset) {
6149 DEBUG (1, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
6150 free_internal_mem (remset, INTERNAL_MEM_REMSET);
6153 clear_thread_store_remset_buffer (info);
6157 /* the freed thread ones */
6158 while (freed_thread_remsets) {
6159 next = freed_thread_remsets->next;
6160 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", freed_thread_remsets->data));
6161 free_internal_mem (freed_thread_remsets, INTERNAL_MEM_REMSET);
6162 freed_thread_remsets = next;
6167 * Clear the thread local TLAB variables for all threads.
6172 SgenThreadInfo *info;
6175 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
6176 for (info = thread_table [i]; info; info = info->next) {
6177 /* A new TLAB will be allocated when the thread does its first allocation */
6178 *info->tlab_start_addr = NULL;
6179 *info->tlab_next_addr = NULL;
6180 *info->tlab_temp_end_addr = NULL;
6181 *info->tlab_real_end_addr = NULL;
6186 /* LOCKING: assumes the GC lock is held */
6187 static SgenThreadInfo*
6188 gc_register_current_thread (void *addr)
6191 SgenThreadInfo* info = malloc (sizeof (SgenThreadInfo));
6192 #ifndef HAVE_KW_THREAD
6193 SgenThreadInfo *__thread_info__ = info;
6199 memset (info, 0, sizeof (SgenThreadInfo));
6200 #ifndef HAVE_KW_THREAD
6201 info->tlab_start = info->tlab_next = info->tlab_temp_end = info->tlab_real_end = NULL;
6203 g_assert (!pthread_getspecific (thread_info_key));
6204 pthread_setspecific (thread_info_key, info);
6209 info->id = ARCH_GET_THREAD ();
6210 info->stop_count = -1;
6213 info->stack_start = NULL;
6214 info->tlab_start_addr = &TLAB_START;
6215 info->tlab_next_addr = &TLAB_NEXT;
6216 info->tlab_temp_end_addr = &TLAB_TEMP_END;
6217 info->tlab_real_end_addr = &TLAB_REAL_END;
6218 info->store_remset_buffer_addr = &STORE_REMSET_BUFFER;
6219 info->store_remset_buffer_index_addr = &STORE_REMSET_BUFFER_INDEX;
6220 info->stopped_ip = NULL;
6221 info->stopped_domain = NULL;
6222 info->stopped_regs = NULL;
6224 binary_protocol_thread_register ((gpointer)info->id);
6226 #ifdef HAVE_KW_THREAD
6227 tlab_next_addr = &tlab_next;
6228 store_remset_buffer_index_addr = &store_remset_buffer_index;
6231 /* try to get it with attributes first */
6232 #if defined(HAVE_PTHREAD_GETATTR_NP) && defined(HAVE_PTHREAD_ATTR_GETSTACK)
6236 pthread_attr_t attr;
6237 pthread_getattr_np (pthread_self (), &attr);
6238 pthread_attr_getstack (&attr, &sstart, &size);
6239 info->stack_start_limit = sstart;
6240 info->stack_end = (char*)sstart + size;
6241 pthread_attr_destroy (&attr);
6243 #elif defined(HAVE_PTHREAD_GET_STACKSIZE_NP) && defined(HAVE_PTHREAD_GET_STACKADDR_NP)
6244 info->stack_end = (char*)pthread_get_stackaddr_np (pthread_self ());
6247 /* FIXME: we assume the stack grows down */
6248 gsize stack_bottom = (gsize)addr;
6249 stack_bottom += 4095;
6250 stack_bottom &= ~4095;
6251 info->stack_end = (char*)stack_bottom;
6255 #ifdef HAVE_KW_THREAD
6256 stack_end = info->stack_end;
6259 /* hash into the table */
6260 hash = HASH_PTHREAD_T (info->id) % THREAD_HASH_SIZE;
6261 info->next = thread_table [hash];
6262 thread_table [hash] = info;
6264 info->remset = alloc_remset (DEFAULT_REMSET_SIZE, info);
6265 pthread_setspecific (remembered_set_key, info->remset);
6266 #ifdef HAVE_KW_THREAD
6267 remembered_set = info->remset;
6270 STORE_REMSET_BUFFER = get_internal_mem (sizeof (gpointer) * STORE_REMSET_BUFFER_SIZE, INTERNAL_MEM_STORE_REMSET);
6271 STORE_REMSET_BUFFER_INDEX = 0;
6273 DEBUG (3, fprintf (gc_debug_file, "registered thread %p (%p) (hash: %d)\n", info, (gpointer)info->id, hash));
6275 if (gc_callbacks.thread_attach_func)
6276 info->runtime_data = gc_callbacks.thread_attach_func ();
6282 add_generic_store_remset_from_buffer (gpointer *buffer)
6284 GenericStoreRememberedSet *remset = get_internal_mem (sizeof (GenericStoreRememberedSet), INTERNAL_MEM_STORE_REMSET);
6285 memcpy (remset->data, buffer + 1, sizeof (gpointer) * (STORE_REMSET_BUFFER_SIZE - 1));
6286 remset->next = generic_store_remsets;
6287 generic_store_remsets = remset;
6291 unregister_current_thread (void)
6294 SgenThreadInfo *prev = NULL;
6296 RememberedSet *rset;
6297 ARCH_THREAD_TYPE id = ARCH_GET_THREAD ();
6299 binary_protocol_thread_unregister ((gpointer)id);
6301 hash = HASH_PTHREAD_T (id) % THREAD_HASH_SIZE;
6302 p = thread_table [hash];
6304 DEBUG (3, fprintf (gc_debug_file, "unregister thread %p (%p)\n", p, (gpointer)p->id));
6305 while (!ARCH_THREAD_EQUALS (p->id, id)) {
6310 thread_table [hash] = p->next;
6312 prev->next = p->next;
6315 if (freed_thread_remsets) {
6316 for (rset = p->remset; rset->next; rset = rset->next)
6318 rset->next = freed_thread_remsets;
6319 freed_thread_remsets = p->remset;
6321 freed_thread_remsets = p->remset;
6324 if (*p->store_remset_buffer_index_addr)
6325 add_generic_store_remset_from_buffer (*p->store_remset_buffer_addr);
6326 free_internal_mem (*p->store_remset_buffer_addr, INTERNAL_MEM_STORE_REMSET);
6331 unregister_thread (void *k)
6333 g_assert (!mono_domain_get ());
6335 unregister_current_thread ();
6340 mono_gc_register_thread (void *baseptr)
6342 SgenThreadInfo *info;
6346 info = thread_info_lookup (ARCH_GET_THREAD ());
6348 info = gc_register_current_thread (baseptr);
6350 return info != NULL;
6353 #if USE_PTHREAD_INTERCEPT
6355 #undef pthread_create
6357 #undef pthread_detach
6360 void *(*start_routine) (void *);
6363 MonoSemType registered;
6364 } SgenThreadStartInfo;
6367 gc_start_thread (void *arg)
6369 SgenThreadStartInfo *start_info = arg;
6370 SgenThreadInfo* info;
6371 void *t_arg = start_info->arg;
6372 void *(*start_func) (void*) = start_info->start_routine;
6377 info = gc_register_current_thread (&result);
6379 post_result = MONO_SEM_POST (&(start_info->registered));
6380 g_assert (!post_result);
6381 result = start_func (t_arg);
6382 g_assert (!mono_domain_get ());
6384 * this is done by the pthread key dtor
6386 unregister_current_thread ();
6394 mono_gc_pthread_create (pthread_t *new_thread, const pthread_attr_t *attr, void *(*start_routine)(void *), void *arg)
6396 SgenThreadStartInfo *start_info;
6399 start_info = malloc (sizeof (SgenThreadStartInfo));
6402 result = MONO_SEM_INIT (&(start_info->registered), 0);
6404 start_info->arg = arg;
6405 start_info->start_routine = start_routine;
6407 result = pthread_create (new_thread, attr, gc_start_thread, start_info);
6409 while (MONO_SEM_WAIT (&(start_info->registered)) != 0) {
6410 /*if (EINTR != errno) ABORT("sem_wait failed"); */
6413 MONO_SEM_DESTROY (&(start_info->registered));
6419 mono_gc_pthread_join (pthread_t thread, void **retval)
6421 return pthread_join (thread, retval);
6425 mono_gc_pthread_detach (pthread_t thread)
6427 return pthread_detach (thread);
6430 #endif /* USE_PTHREAD_INTERCEPT */
6433 * ######################################################################
6434 * ######## Write barriers
6435 * ######################################################################
6438 static RememberedSet*
6439 alloc_remset (int size, gpointer id) {
6440 RememberedSet* res = get_internal_mem (sizeof (RememberedSet) + (size * sizeof (gpointer)), INTERNAL_MEM_REMSET);
6441 res->store_next = res->data;
6442 res->end_set = res->data + size;
6444 DEBUG (4, fprintf (gc_debug_file, "Allocated remset size %d at %p for %p\n", size, res->data, id));
6449 * Note: the write barriers first do the needed GC work and then do the actual store:
6450 * this way the value is visible to the conservative GC scan after the write barrier
6451 * itself. If a GC interrupts the barrier in the middle, value will be kept alive by
6452 * the conservative scan, otherwise by the remembered set scan. FIXME: figure out what
6453 * happens when we need to record which pointers contain references to the new generation.
6454 * The write barrier will be executed, but the pointer is still not stored.
6457 mono_gc_wbarrier_set_field (MonoObject *obj, gpointer field_ptr, MonoObject* value)
6461 HEAVY_STAT (++stat_wbarrier_set_field);
6462 if (ptr_in_nursery (field_ptr)) {
6463 *(void**)field_ptr = value;
6466 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", field_ptr));
6468 rs = REMEMBERED_SET;
6469 if (rs->store_next < rs->end_set) {
6470 *(rs->store_next++) = (mword)field_ptr;
6471 *(void**)field_ptr = value;
6475 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6476 rs->next = REMEMBERED_SET;
6477 REMEMBERED_SET = rs;
6478 #ifdef HAVE_KW_THREAD
6479 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6481 *(rs->store_next++) = (mword)field_ptr;
6482 *(void**)field_ptr = value;
6487 mono_gc_wbarrier_set_arrayref (MonoArray *arr, gpointer slot_ptr, MonoObject* value)
6491 HEAVY_STAT (++stat_wbarrier_set_arrayref);
6492 if (ptr_in_nursery (slot_ptr)) {
6493 *(void**)slot_ptr = value;
6496 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", slot_ptr));
6498 rs = REMEMBERED_SET;
6499 if (rs->store_next < rs->end_set) {
6500 *(rs->store_next++) = (mword)slot_ptr;
6501 *(void**)slot_ptr = value;
6505 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6506 rs->next = REMEMBERED_SET;
6507 REMEMBERED_SET = rs;
6508 #ifdef HAVE_KW_THREAD
6509 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6511 *(rs->store_next++) = (mword)slot_ptr;
6512 *(void**)slot_ptr = value;
6517 mono_gc_wbarrier_arrayref_copy (gpointer dest_ptr, gpointer src_ptr, int count)
6521 HEAVY_STAT (++stat_wbarrier_arrayref_copy);
6523 memmove (dest_ptr, src_ptr, count * sizeof (gpointer));
6524 if (ptr_in_nursery (dest_ptr)) {
6528 rs = REMEMBERED_SET;
6529 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p, %d\n", dest_ptr, count));
6530 if (rs->store_next + 1 < rs->end_set) {
6531 *(rs->store_next++) = (mword)dest_ptr | REMSET_RANGE;
6532 *(rs->store_next++) = count;
6536 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6537 rs->next = REMEMBERED_SET;
6538 REMEMBERED_SET = rs;
6539 #ifdef HAVE_KW_THREAD
6540 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6542 *(rs->store_next++) = (mword)dest_ptr | REMSET_RANGE;
6543 *(rs->store_next++) = count;
6548 find_object_for_ptr_in_area (char *ptr, char *start, char *end)
6550 while (start < end) {
6553 if (!*(void**)start) {
6554 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
6560 #define SCAN_OBJECT_NOSCAN
6561 #include "sgen-scan-object.h"
6563 if (ptr >= old_start && ptr < start)
6570 static char *found_obj;
6573 find_object_for_ptr_in_pinned_chunk_callback (PinnedChunk *chunk, char *obj, size_t size, char *ptr)
6575 if (ptr >= obj && ptr < obj + size) {
6576 g_assert (!found_obj);
6581 /* for use in the debugger */
6582 char* find_object_for_ptr (char *ptr);
6584 find_object_for_ptr (char *ptr)
6586 GCMemSection *section;
6589 for (section = section_list; section; section = section->block.next) {
6590 if (ptr >= section->data && ptr < section->end_data)
6591 return find_object_for_ptr_in_area (ptr, section->data, section->end_data);
6594 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
6595 if (ptr >= bigobj->data && ptr < bigobj->data + bigobj->size)
6596 return bigobj->data;
6600 scan_pinned_objects ((ScanPinnedObjectCallbackFunc)find_object_for_ptr_in_pinned_chunk_callback, ptr);
6605 evacuate_remset_buffer (void)
6610 buffer = STORE_REMSET_BUFFER;
6612 add_generic_store_remset_from_buffer (buffer);
6613 memset (buffer, 0, sizeof (gpointer) * STORE_REMSET_BUFFER_SIZE);
6615 STORE_REMSET_BUFFER_INDEX = 0;
6619 mono_gc_wbarrier_generic_nostore (gpointer ptr)
6625 HEAVY_STAT (++stat_wbarrier_generic_store);
6627 #ifdef XDOMAIN_CHECKS_IN_WBARRIER
6628 /* FIXME: ptr_in_heap must be called with the GC lock held */
6629 if (xdomain_checks && *(MonoObject**)ptr && ptr_in_heap (ptr)) {
6630 char *start = find_object_for_ptr (ptr);
6631 MonoObject *value = *(MonoObject**)ptr;
6635 MonoObject *obj = (MonoObject*)start;
6636 if (obj->vtable->domain != value->vtable->domain)
6637 g_assert (is_xdomain_ref_allowed (ptr, start, obj->vtable->domain));
6645 if (*(gpointer*)ptr)
6646 binary_protocol_wbarrier (ptr, *(gpointer*)ptr, (gpointer)LOAD_VTABLE (*(gpointer*)ptr));
6648 if (ptr_in_nursery (ptr) || ptr_on_stack (ptr) || !ptr_in_nursery (*(gpointer*)ptr)) {
6649 DEBUG (8, fprintf (gc_debug_file, "Skipping remset at %p\n", ptr));
6654 buffer = STORE_REMSET_BUFFER;
6655 index = STORE_REMSET_BUFFER_INDEX;
6656 /* This simple optimization eliminates a sizable portion of
6657 entries. Comparing it to the last but one entry as well
6658 doesn't eliminate significantly more entries. */
6659 if (buffer [index] == ptr) {
6664 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", ptr));
6665 HEAVY_STAT (++stat_wbarrier_generic_store_remset);
6668 if (index >= STORE_REMSET_BUFFER_SIZE) {
6669 evacuate_remset_buffer ();
6670 index = STORE_REMSET_BUFFER_INDEX;
6671 g_assert (index == 0);
6674 buffer [index] = ptr;
6675 STORE_REMSET_BUFFER_INDEX = index;
6681 mono_gc_wbarrier_generic_store (gpointer ptr, MonoObject* value)
6683 DEBUG (8, fprintf (gc_debug_file, "Wbarrier store at %p to %p (%s)\n", ptr, value, value ? safe_name (value) : "null"));
6684 *(void**)ptr = value;
6685 if (ptr_in_nursery (value))
6686 mono_gc_wbarrier_generic_nostore (ptr);
6690 mono_gc_wbarrier_set_root (gpointer ptr, MonoObject *value)
6694 HEAVY_STAT (++stat_wbarrier_set_root);
6695 if (ptr_in_nursery (ptr))
6697 DEBUG (8, fprintf (gc_debug_file, "Adding root remset at %p (%s)\n", ptr, value ? safe_name (value) : "null"));
6699 rs = REMEMBERED_SET;
6700 if (rs->store_next + 2 < rs->end_set) {
6701 *(rs->store_next++) = (mword)ptr | REMSET_OTHER;
6702 *(rs->store_next++) = (mword)REMSET_ROOT_LOCATION;
6703 *(void**)ptr = value;
6706 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6707 rs->next = REMEMBERED_SET;
6708 REMEMBERED_SET = rs;
6709 #ifdef HAVE_KW_THREAD
6710 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6712 *(rs->store_next++) = (mword)ptr | REMSET_OTHER;
6713 *(rs->store_next++) = (mword)REMSET_ROOT_LOCATION;
6715 *(void**)ptr = value;
6719 mono_gc_wbarrier_value_copy (gpointer dest, gpointer src, int count, MonoClass *klass)
6723 HEAVY_STAT (++stat_wbarrier_value_copy);
6724 g_assert (klass->valuetype);
6726 memmove (dest, src, count * mono_class_value_size (klass, NULL));
6727 rs = REMEMBERED_SET;
6728 if (ptr_in_nursery (dest) || ptr_on_stack (dest)) {
6732 g_assert (klass->gc_descr_inited);
6733 DEBUG (8, fprintf (gc_debug_file, "Adding value remset at %p, count %d, descr %p for class %s (%p)\n", dest, count, klass->gc_descr, klass->name, klass));
6735 if (rs->store_next + 3 < rs->end_set) {
6736 *(rs->store_next++) = (mword)dest | REMSET_OTHER;
6737 *(rs->store_next++) = (mword)REMSET_VTYPE;
6738 *(rs->store_next++) = (mword)klass->gc_descr;
6739 *(rs->store_next++) = (mword)count;
6743 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6744 rs->next = REMEMBERED_SET;
6745 REMEMBERED_SET = rs;
6746 #ifdef HAVE_KW_THREAD
6747 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6749 *(rs->store_next++) = (mword)dest | REMSET_OTHER;
6750 *(rs->store_next++) = (mword)REMSET_VTYPE;
6751 *(rs->store_next++) = (mword)klass->gc_descr;
6752 *(rs->store_next++) = (mword)count;
6757 * mono_gc_wbarrier_object_copy:
6759 * Write barrier to call when obj is the result of a clone or copy of an object.
6762 mono_gc_wbarrier_object_copy (MonoObject* obj, MonoObject *src)
6768 HEAVY_STAT (++stat_wbarrier_object_copy);
6769 rs = REMEMBERED_SET;
6770 DEBUG (1, fprintf (gc_debug_file, "Adding object remset for %p\n", obj));
6771 size = mono_object_class (obj)->instance_size;
6773 /* do not copy the sync state */
6774 memcpy ((char*)obj + sizeof (MonoObject), (char*)src + sizeof (MonoObject),
6775 size - sizeof (MonoObject));
6776 if (ptr_in_nursery (obj) || ptr_on_stack (obj)) {
6780 if (rs->store_next < rs->end_set) {
6781 *(rs->store_next++) = (mword)obj | REMSET_OBJECT;
6785 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6786 rs->next = REMEMBERED_SET;
6787 REMEMBERED_SET = rs;
6788 #ifdef HAVE_KW_THREAD
6789 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6791 *(rs->store_next++) = (mword)obj | REMSET_OBJECT;
6796 * ######################################################################
6797 * ######## Collector debugging
6798 * ######################################################################
6801 const char*descriptor_types [] = {
6813 describe_ptr (char *ptr)
6815 GCMemSection *section;
6820 if (ptr_in_nursery (ptr)) {
6821 printf ("Pointer inside nursery.\n");
6823 for (section = section_list; section;) {
6824 if (ptr >= section->data && ptr < section->data + section->size)
6826 section = section->block.next;
6830 printf ("Pointer inside oldspace.\n");
6831 } else if (obj_is_from_pinned_alloc (ptr)) {
6832 printf ("Pointer is inside a pinned chunk.\n");
6834 printf ("Pointer unknown.\n");
6839 if (object_is_pinned (ptr))
6840 printf ("Object is pinned.\n");
6842 if (object_is_forwarded (ptr))
6843 printf ("Object is forwared.\n");
6845 // FIXME: Handle pointers to the inside of objects
6846 vtable = (MonoVTable*)LOAD_VTABLE (ptr);
6848 printf ("VTable: %p\n", vtable);
6849 if (vtable == NULL) {
6850 printf ("VTable is invalid (empty).\n");
6853 if (ptr_in_nursery (vtable)) {
6854 printf ("VTable is invalid (points inside nursery).\n");
6857 printf ("Class: %s\n", vtable->klass->name);
6859 desc = ((GCVTable*)vtable)->desc;
6860 printf ("Descriptor: %lx\n", (long)desc);
6863 printf ("Descriptor type: %d (%s)\n", type, descriptor_types [type]);
6867 find_in_remset_loc (mword *p, char *addr, gboolean *found)
6873 switch ((*p) & REMSET_TYPE_MASK) {
6874 case REMSET_LOCATION:
6875 if (*p == (mword)addr)
6879 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
6881 if ((void**)addr >= ptr && (void**)addr < ptr + count)
6885 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
6886 count = safe_object_get_size ((MonoObject*)ptr);
6887 count += (ALLOC_ALIGN - 1);
6888 count &= (ALLOC_ALIGN - 1);
6889 count /= sizeof (mword);
6890 if ((void**)addr >= ptr && (void**)addr < ptr + count)
6893 case REMSET_OTHER: {
6896 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
6900 switch (desc & 0x7) {
6901 case DESC_TYPE_RUN_LENGTH:
6902 OBJ_RUN_LEN_SIZE (skip_size, desc, ptr);
6904 case DESC_TYPE_SMALL_BITMAP:
6905 OBJ_BITMAP_SIZE (skip_size, desc, start);
6909 g_assert_not_reached ();
6912 /* The descriptor includes the size of MonoObject */
6913 skip_size -= sizeof (MonoObject);
6915 if ((void**)addr >= ptr && (void**)addr < ptr + (skip_size / sizeof (gpointer)))
6919 case REMSET_ROOT_LOCATION:
6922 g_assert_not_reached ();
6927 g_assert_not_reached ();
6933 * Return whenever ADDR occurs in the remembered sets
6936 find_in_remsets (char *addr)
6939 SgenThreadInfo *info;
6940 RememberedSet *remset;
6941 GenericStoreRememberedSet *store_remset;
6943 gboolean found = FALSE;
6945 /* the global one */
6946 for (remset = global_remset; remset; remset = remset->next) {
6947 DEBUG (4, fprintf (gc_debug_file, "Scanning global remset range: %p-%p, size: %zd\n", remset->data, remset->store_next, remset->store_next - remset->data));
6948 for (p = remset->data; p < remset->store_next;) {
6949 p = find_in_remset_loc (p, addr, &found);
6955 /* the generic store ones */
6956 for (store_remset = generic_store_remsets; store_remset; store_remset = store_remset->next) {
6957 for (i = 0; i < STORE_REMSET_BUFFER_SIZE - 1; ++i) {
6958 if (store_remset->data [i] == addr)
6963 /* the per-thread ones */
6964 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
6965 for (info = thread_table [i]; info; info = info->next) {
6967 for (remset = info->remset; remset; remset = remset->next) {
6968 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for thread %p, range: %p-%p, size: %zd\n", info, remset->data, remset->store_next, remset->store_next - remset->data));
6969 for (p = remset->data; p < remset->store_next;) {
6970 p = find_in_remset_loc (p, addr, &found);
6975 for (j = 0; j < *info->store_remset_buffer_index_addr; ++j) {
6976 if ((*info->store_remset_buffer_addr) [j + 1] == addr)
6982 /* the freed thread ones */
6983 for (remset = freed_thread_remsets; remset; remset = remset->next) {
6984 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for freed thread, range: %p-%p, size: %zd\n", remset->data, remset->store_next, remset->store_next - remset->data));
6985 for (p = remset->data; p < remset->store_next;) {
6986 p = find_in_remset_loc (p, addr, &found);
6995 static gboolean missing_remsets;
6998 * We let a missing remset slide if the target object is pinned,
6999 * because the store might have happened but the remset not yet added,
7000 * but in that case the target must be pinned. We might theoretically
7001 * miss some missing remsets this way, but it's very unlikely.
7004 #define HANDLE_PTR(ptr,obj) do { \
7005 if (*(ptr) && (char*)*(ptr) >= nursery_start && (char*)*(ptr) < nursery_next) { \
7006 if (!find_in_remsets ((char*)(ptr))) { \
7007 fprintf (gc_debug_file, "Oldspace->newspace reference %p at offset %zd in object %p (%s.%s) not found in remsets.\n", *(ptr), (char*)(ptr) - (char*)(obj), (obj), ((MonoObject*)(obj))->vtable->klass->name_space, ((MonoObject*)(obj))->vtable->klass->name); \
7008 binary_protocol_missing_remset ((obj), (gpointer)LOAD_VTABLE ((obj)), (char*)(ptr) - (char*)(obj), *(ptr), (gpointer)LOAD_VTABLE(*(ptr)), object_is_pinned (*(ptr))); \
7009 if (!object_is_pinned (*(ptr))) \
7010 missing_remsets = TRUE; \
7016 * Check that each object reference inside the area which points into the nursery
7017 * can be found in the remembered sets.
7019 static void __attribute__((noinline))
7020 check_remsets_for_area (char *start, char *end)
7023 int type_str = 0, type_rlen = 0, type_bitmap = 0, type_vector = 0, type_lbit = 0, type_complex = 0;
7024 while (start < end) {
7025 if (!*(void**)start) {
7026 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
7029 vt = (GCVTable*)LOAD_VTABLE (start);
7030 DEBUG (8, fprintf (gc_debug_file, "Scanning object %p, vtable: %p (%s)\n", start, vt, vt->klass->name));
7032 MonoObject *obj = (MonoObject*)start;
7033 g_print ("found at %p (0x%lx): %s.%s\n", start, (long)vt->desc, obj->vtable->klass->name_space, obj->vtable->klass->name);
7036 #define SCAN_OBJECT_ACTION COUNT_OBJECT_TYPES
7037 #include "sgen-scan-object.h"
7042 * Perform consistency check of the heap.
7044 * Assumes the world is stopped.
7047 check_consistency (void)
7049 GCMemSection *section;
7051 // Need to add more checks
7052 // FIXME: Create a general heap enumeration function and use that
7054 missing_remsets = FALSE;
7056 DEBUG (1, fprintf (gc_debug_file, "Begin heap consistency check...\n"));
7058 // Check that oldspace->newspace pointers are registered with the collector
7059 for (section = section_list; section; section = section->block.next) {
7060 if (section->block.role == MEMORY_ROLE_GEN0)
7062 DEBUG (2, fprintf (gc_debug_file, "Scan of old section: %p-%p, size: %d\n", section->data, section->next_data, (int)(section->next_data - section->data)));
7063 check_remsets_for_area (section->data, section->next_data);
7066 DEBUG (1, fprintf (gc_debug_file, "Heap consistency check done.\n"));
7068 #ifdef BINARY_PROTOCOL
7069 if (!binary_protocol_file)
7071 g_assert (!missing_remsets);
7074 /* Check that the reference is valid */
7076 #define HANDLE_PTR(ptr,obj) do { \
7078 g_assert (safe_name (*(ptr)) != NULL); \
7085 * Perform consistency check on an object. Currently we only check that the
7086 * reference fields are valid.
7089 check_object (char *start)
7094 #include "sgen-scan-object.h"
7100 * ######################################################################
7101 * ######## Other mono public interface functions.
7102 * ######################################################################
7106 mono_gc_collect (int generation)
7110 if (generation == 0) {
7111 collect_nursery (0);
7113 major_collection ("user request");
7120 mono_gc_max_generation (void)
7126 mono_gc_collection_count (int generation)
7128 if (generation == 0)
7129 return num_minor_gcs;
7130 return num_major_gcs;
7134 mono_gc_get_used_size (void)
7137 GCMemSection *section;
7139 tot = los_memory_usage;
7140 for (section = section_list; section; section = section->block.next) {
7141 /* this is approximate... */
7142 tot += section->next_data - section->data;
7144 /* FIXME: account for pinned objects */
7150 mono_gc_get_heap_size (void)
7156 mono_gc_disable (void)
7164 mono_gc_enable (void)
7172 mono_gc_get_los_limit (void)
7174 return MAX_SMALL_OBJ_SIZE;
7178 mono_object_is_alive (MonoObject* o)
7184 mono_gc_get_generation (MonoObject *obj)
7186 if (ptr_in_nursery (obj))
7192 mono_gc_enable_events (void)
7197 mono_gc_weak_link_add (void **link_addr, MonoObject *obj, gboolean track)
7200 mono_gc_register_disappearing_link (obj, link_addr, track);
7205 mono_gc_weak_link_remove (void **link_addr)
7208 mono_gc_register_disappearing_link (NULL, link_addr, FALSE);
7213 mono_gc_weak_link_get (void **link_addr)
7217 return (MonoObject*) REVEAL_POINTER (*link_addr);
7221 mono_gc_make_descr_from_bitmap (gsize *bitmap, int numbits)
7223 if (numbits < ((sizeof (*bitmap) * 8) - ROOT_DESC_TYPE_SHIFT)) {
7224 return (void*)MAKE_ROOT_DESC (ROOT_DESC_BITMAP, bitmap [0]);
7226 mword complex = alloc_complex_descriptor (bitmap, numbits);
7227 return (void*)MAKE_ROOT_DESC (ROOT_DESC_COMPLEX, complex);
7232 mono_gc_make_root_descr_user (MonoGCMarkFunc marker)
7236 g_assert (user_descriptors_next < MAX_USER_DESCRIPTORS);
7237 descr = (void*)MAKE_ROOT_DESC (ROOT_DESC_USER, (mword)user_descriptors_next);
7238 user_descriptors [user_descriptors_next ++] = marker;
7244 mono_gc_alloc_fixed (size_t size, void *descr)
7246 /* FIXME: do a single allocation */
7247 void *res = calloc (1, size);
7250 if (!mono_gc_register_root (res, size, descr)) {
7258 mono_gc_free_fixed (void* addr)
7260 mono_gc_deregister_root (addr);
7265 mono_gc_is_gc_thread (void)
7269 result = thread_info_lookup (ARCH_GET_THREAD ()) != NULL;
7275 mono_gc_base_init (void)
7279 struct sigaction sinfo;
7281 LOCK_INIT (gc_mutex);
7283 if (gc_initialized) {
7287 pagesize = mono_pagesize ();
7288 gc_debug_file = stderr;
7289 if ((env = getenv ("MONO_GC_DEBUG"))) {
7290 opts = g_strsplit (env, ",", -1);
7291 for (ptr = opts; ptr && *ptr; ptr ++) {
7293 if (opt [0] >= '0' && opt [0] <= '9') {
7294 gc_debug_level = atoi (opt);
7299 char *rf = g_strdup_printf ("%s.%d", opt, getpid ());
7300 gc_debug_file = fopen (rf, "wb");
7302 gc_debug_file = stderr;
7305 } else if (!strcmp (opt, "collect-before-allocs")) {
7306 collect_before_allocs = TRUE;
7307 } else if (!strcmp (opt, "check-at-minor-collections")) {
7308 consistency_check_at_minor_collection = TRUE;
7309 } else if (!strcmp (opt, "xdomain-checks")) {
7310 xdomain_checks = TRUE;
7311 } else if (!strcmp (opt, "clear-at-gc")) {
7312 nursery_clear_policy = CLEAR_AT_GC;
7313 } else if (!strcmp (opt, "conservative-stack-mark")) {
7314 conservative_stack_mark = TRUE;
7315 } else if (!strcmp (opt, "check-scan-starts")) {
7316 do_scan_starts_check = TRUE;
7317 } else if (g_str_has_prefix (opt, "heap-dump=")) {
7318 char *filename = strchr (opt, '=') + 1;
7319 nursery_clear_policy = CLEAR_AT_GC;
7320 heap_dump_file = fopen (filename, "w");
7322 fprintf (heap_dump_file, "<sgen-dump>\n");
7323 #ifdef BINARY_PROTOCOL
7324 } else if (g_str_has_prefix (opt, "binary-protocol=")) {
7325 char *filename = strchr (opt, '=') + 1;
7326 binary_protocol_file = fopen (filename, "w");
7329 fprintf (stderr, "Invalid format for the MONO_GC_DEBUG env variable: '%s'\n", env);
7330 fprintf (stderr, "The format is: MONO_GC_DEBUG=[l[:filename]|<option>]+ where l is a debug level 0-9.\n");
7331 fprintf (stderr, "Valid options are: collect-before-allocs, check-at-minor-collections, xdomain-checks, clear-at-gc.\n");
7338 suspend_ack_semaphore_ptr = &suspend_ack_semaphore;
7339 MONO_SEM_INIT (&suspend_ack_semaphore, 0);
7341 sigfillset (&sinfo.sa_mask);
7342 sinfo.sa_flags = SA_RESTART | SA_SIGINFO;
7343 sinfo.sa_sigaction = suspend_handler;
7344 if (sigaction (suspend_signal_num, &sinfo, NULL) != 0) {
7345 g_error ("failed sigaction");
7348 sinfo.sa_handler = restart_handler;
7349 if (sigaction (restart_signal_num, &sinfo, NULL) != 0) {
7350 g_error ("failed sigaction");
7353 sigfillset (&suspend_signal_mask);
7354 sigdelset (&suspend_signal_mask, restart_signal_num);
7356 global_remset = alloc_remset (1024, NULL);
7357 global_remset->next = NULL;
7359 pthread_key_create (&remembered_set_key, unregister_thread);
7361 #ifndef HAVE_KW_THREAD
7362 pthread_key_create (&thread_info_key, NULL);
7365 gc_initialized = TRUE;
7367 mono_gc_register_thread (&sinfo);
7371 mono_gc_get_suspend_signal (void)
7373 return suspend_signal_num;
7383 #ifdef HAVE_KW_THREAD
7384 #define EMIT_TLS_ACCESS(mb,dummy,offset) do { \
7385 mono_mb_emit_byte ((mb), MONO_CUSTOM_PREFIX); \
7386 mono_mb_emit_byte ((mb), CEE_MONO_TLS); \
7387 mono_mb_emit_i4 ((mb), (offset)); \
7390 #define EMIT_TLS_ACCESS(mb,member,dummy) do { \
7391 mono_mb_emit_byte ((mb), MONO_CUSTOM_PREFIX); \
7392 mono_mb_emit_byte ((mb), CEE_MONO_TLS); \
7393 mono_mb_emit_i4 ((mb), thread_info_key); \
7394 mono_mb_emit_icon ((mb), G_STRUCT_OFFSET (SgenThreadInfo, member)); \
7395 mono_mb_emit_byte ((mb), CEE_ADD); \
7396 mono_mb_emit_byte ((mb), CEE_LDIND_I); \
7400 #ifdef MANAGED_ALLOCATION
7401 /* FIXME: Do this in the JIT, where specialized allocation sequences can be created
7402 * for each class. This is currently not easy to do, as it is hard to generate basic
7403 * blocks + branches, but it is easy with the linear IL codebase.
7405 * For this to work we'd need to solve the TLAB race, first. Now we
7406 * require the allocator to be in a few known methods to make sure
7407 * that they are executed atomically via the restart mechanism.
7410 create_allocator (int atype)
7412 int p_var, size_var;
7413 guint32 slowpath_branch, max_size_branch;
7414 MonoMethodBuilder *mb;
7416 MonoMethodSignature *csig;
7417 static gboolean registered = FALSE;
7418 int tlab_next_addr_var, new_next_var;
7420 const char *name = NULL;
7421 AllocatorWrapperInfo *info;
7423 #ifdef HAVE_KW_THREAD
7424 int tlab_next_addr_offset = -1;
7425 int tlab_temp_end_offset = -1;
7427 MONO_THREAD_VAR_OFFSET (tlab_next_addr, tlab_next_addr_offset);
7428 MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
7430 g_assert (tlab_next_addr_offset != -1);
7431 g_assert (tlab_temp_end_offset != -1);
7435 mono_register_jit_icall (mono_gc_alloc_obj, "mono_gc_alloc_obj", mono_create_icall_signature ("object ptr int"), FALSE);
7436 mono_register_jit_icall (mono_gc_alloc_vector, "mono_gc_alloc_vector", mono_create_icall_signature ("object ptr int int"), FALSE);
7440 if (atype == ATYPE_SMALL) {
7442 name = "AllocSmall";
7443 } else if (atype == ATYPE_NORMAL) {
7446 } else if (atype == ATYPE_VECTOR) {
7448 name = "AllocVector";
7450 g_assert_not_reached ();
7453 csig = mono_metadata_signature_alloc (mono_defaults.corlib, num_params);
7454 csig->ret = &mono_defaults.object_class->byval_arg;
7455 for (i = 0; i < num_params; ++i)
7456 csig->params [i] = &mono_defaults.int_class->byval_arg;
7458 mb = mono_mb_new (mono_defaults.object_class, name, MONO_WRAPPER_ALLOC);
7459 size_var = mono_mb_add_local (mb, &mono_defaults.int32_class->byval_arg);
7460 if (atype == ATYPE_NORMAL || atype == ATYPE_SMALL) {
7461 /* size = vtable->klass->instance_size; */
7462 mono_mb_emit_ldarg (mb, 0);
7463 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoVTable, klass));
7464 mono_mb_emit_byte (mb, CEE_ADD);
7465 mono_mb_emit_byte (mb, CEE_LDIND_I);
7466 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoClass, instance_size));
7467 mono_mb_emit_byte (mb, CEE_ADD);
7468 /* FIXME: assert instance_size stays a 4 byte integer */
7469 mono_mb_emit_byte (mb, CEE_LDIND_U4);
7470 mono_mb_emit_stloc (mb, size_var);
7471 } else if (atype == ATYPE_VECTOR) {
7472 MonoExceptionClause *clause;
7474 MonoClass *oom_exc_class;
7477 /* n > MONO_ARRAY_MAX_INDEX -> OverflowException */
7478 mono_mb_emit_ldarg (mb, 1);
7479 mono_mb_emit_icon (mb, MONO_ARRAY_MAX_INDEX);
7480 pos = mono_mb_emit_short_branch (mb, CEE_BLE_UN_S);
7481 mono_mb_emit_exception (mb, "OverflowException", NULL);
7482 mono_mb_patch_short_branch (mb, pos);
7484 clause = mono_image_alloc0 (mono_defaults.corlib, sizeof (MonoExceptionClause));
7485 clause->try_offset = mono_mb_get_label (mb);
7487 /* vtable->klass->sizes.element_size */
7488 mono_mb_emit_ldarg (mb, 0);
7489 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoVTable, klass));
7490 mono_mb_emit_byte (mb, CEE_ADD);
7491 mono_mb_emit_byte (mb, CEE_LDIND_I);
7492 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoClass, sizes.element_size));
7493 mono_mb_emit_byte (mb, CEE_ADD);
7494 mono_mb_emit_byte (mb, CEE_LDIND_U4);
7497 mono_mb_emit_ldarg (mb, 1);
7498 mono_mb_emit_byte (mb, CEE_MUL_OVF_UN);
7499 /* + sizeof (MonoArray) */
7500 mono_mb_emit_icon (mb, sizeof (MonoArray));
7501 mono_mb_emit_byte (mb, CEE_ADD_OVF_UN);
7502 mono_mb_emit_stloc (mb, size_var);
7504 pos_leave = mono_mb_emit_branch (mb, CEE_LEAVE);
7507 clause->flags = MONO_EXCEPTION_CLAUSE_NONE;
7508 clause->try_len = mono_mb_get_pos (mb) - clause->try_offset;
7509 clause->data.catch_class = mono_class_from_name (mono_defaults.corlib,
7510 "System", "OverflowException");
7511 g_assert (clause->data.catch_class);
7512 clause->handler_offset = mono_mb_get_label (mb);
7514 oom_exc_class = mono_class_from_name (mono_defaults.corlib,
7515 "System", "OutOfMemoryException");
7516 g_assert (oom_exc_class);
7517 ctor = mono_class_get_method_from_name (oom_exc_class, ".ctor", 0);
7520 mono_mb_emit_byte (mb, CEE_POP);
7521 mono_mb_emit_op (mb, CEE_NEWOBJ, ctor);
7522 mono_mb_emit_byte (mb, CEE_THROW);
7524 clause->handler_len = mono_mb_get_pos (mb) - clause->handler_offset;
7525 mono_mb_set_clauses (mb, 1, clause);
7526 mono_mb_patch_branch (mb, pos_leave);
7529 g_assert_not_reached ();
7532 /* size += ALLOC_ALIGN - 1; */
7533 mono_mb_emit_ldloc (mb, size_var);
7534 mono_mb_emit_icon (mb, ALLOC_ALIGN - 1);
7535 mono_mb_emit_byte (mb, CEE_ADD);
7536 /* size &= ~(ALLOC_ALIGN - 1); */
7537 mono_mb_emit_icon (mb, ~(ALLOC_ALIGN - 1));
7538 mono_mb_emit_byte (mb, CEE_AND);
7539 mono_mb_emit_stloc (mb, size_var);
7541 /* if (size > MAX_SMALL_OBJ_SIZE) goto slowpath */
7542 if (atype != ATYPE_SMALL) {
7543 mono_mb_emit_ldloc (mb, size_var);
7544 mono_mb_emit_icon (mb, MAX_SMALL_OBJ_SIZE);
7545 max_size_branch = mono_mb_emit_short_branch (mb, MONO_CEE_BGT_S);
7549 * We need to modify tlab_next, but the JIT only supports reading, so we read
7550 * another tls var holding its address instead.
7553 /* tlab_next_addr (local) = tlab_next_addr (TLS var) */
7554 tlab_next_addr_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7555 EMIT_TLS_ACCESS (mb, tlab_next_addr, tlab_next_addr_offset);
7556 mono_mb_emit_stloc (mb, tlab_next_addr_var);
7558 /* p = (void**)tlab_next; */
7559 p_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7560 mono_mb_emit_ldloc (mb, tlab_next_addr_var);
7561 mono_mb_emit_byte (mb, CEE_LDIND_I);
7562 mono_mb_emit_stloc (mb, p_var);
7564 /* new_next = (char*)p + size; */
7565 new_next_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7566 mono_mb_emit_ldloc (mb, p_var);
7567 mono_mb_emit_ldloc (mb, size_var);
7568 mono_mb_emit_byte (mb, CEE_CONV_I);
7569 mono_mb_emit_byte (mb, CEE_ADD);
7570 mono_mb_emit_stloc (mb, new_next_var);
7572 /* tlab_next = new_next */
7573 mono_mb_emit_ldloc (mb, tlab_next_addr_var);
7574 mono_mb_emit_ldloc (mb, new_next_var);
7575 mono_mb_emit_byte (mb, CEE_STIND_I);
7577 /* if (G_LIKELY (new_next < tlab_temp_end)) */
7578 mono_mb_emit_ldloc (mb, new_next_var);
7579 EMIT_TLS_ACCESS (mb, tlab_temp_end, tlab_temp_end_offset);
7580 slowpath_branch = mono_mb_emit_short_branch (mb, MONO_CEE_BLT_UN_S);
7583 if (atype != ATYPE_SMALL)
7584 mono_mb_patch_short_branch (mb, max_size_branch);
7586 mono_mb_emit_byte (mb, MONO_CUSTOM_PREFIX);
7587 mono_mb_emit_byte (mb, CEE_MONO_NOT_TAKEN);
7589 /* FIXME: mono_gc_alloc_obj takes a 'size_t' as an argument, not an int32 */
7590 mono_mb_emit_ldarg (mb, 0);
7591 mono_mb_emit_ldloc (mb, size_var);
7592 if (atype == ATYPE_NORMAL || atype == ATYPE_SMALL) {
7593 mono_mb_emit_icall (mb, mono_gc_alloc_obj);
7594 } else if (atype == ATYPE_VECTOR) {
7595 mono_mb_emit_ldarg (mb, 1);
7596 mono_mb_emit_icall (mb, mono_gc_alloc_vector);
7598 g_assert_not_reached ();
7600 mono_mb_emit_byte (mb, CEE_RET);
7603 mono_mb_patch_short_branch (mb, slowpath_branch);
7605 /* FIXME: Memory barrier */
7608 mono_mb_emit_ldloc (mb, p_var);
7609 mono_mb_emit_ldarg (mb, 0);
7610 mono_mb_emit_byte (mb, CEE_STIND_I);
7612 if (atype == ATYPE_VECTOR) {
7613 /* arr->max_length = max_length; */
7614 mono_mb_emit_ldloc (mb, p_var);
7615 mono_mb_emit_ldflda (mb, G_STRUCT_OFFSET (MonoArray, max_length));
7616 mono_mb_emit_ldarg (mb, 1);
7617 mono_mb_emit_byte (mb, CEE_STIND_I);
7621 mono_mb_emit_ldloc (mb, p_var);
7622 mono_mb_emit_byte (mb, CEE_RET);
7624 res = mono_mb_create_method (mb, csig, 8);
7626 mono_method_get_header (res)->init_locals = FALSE;
7628 info = mono_image_alloc0 (mono_defaults.corlib, sizeof (AllocatorWrapperInfo));
7629 info->alloc_type = atype;
7630 mono_marshal_set_wrapper_info (res, info);
7636 static MonoMethod* alloc_method_cache [ATYPE_NUM];
7637 static MonoMethod *write_barrier_method;
7640 is_ip_in_managed_allocator (MonoDomain *domain, gpointer ip)
7648 ji = mono_jit_info_table_find (domain, ip);
7651 method = ji->method;
7653 if (method == write_barrier_method)
7655 for (i = 0; i < ATYPE_NUM; ++i)
7656 if (method == alloc_method_cache [i])
7662 * Generate an allocator method implementing the fast path of mono_gc_alloc_obj ().
7663 * The signature of the called method is:
7664 * object allocate (MonoVTable *vtable)
7667 mono_gc_get_managed_allocator (MonoVTable *vtable, gboolean for_box)
7669 #ifdef MANAGED_ALLOCATION
7670 MonoClass *klass = vtable->klass;
7672 #ifdef HAVE_KW_THREAD
7673 int tlab_next_offset = -1;
7674 int tlab_temp_end_offset = -1;
7675 MONO_THREAD_VAR_OFFSET (tlab_next, tlab_next_offset);
7676 MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
7678 if (tlab_next_offset == -1 || tlab_temp_end_offset == -1)
7682 if (!mono_runtime_has_tls_get ())
7684 if (klass->instance_size > tlab_size)
7686 if (klass->has_finalize || klass->marshalbyref || (mono_profiler_get_events () & MONO_PROFILE_ALLOCATIONS))
7690 if (klass->byval_arg.type == MONO_TYPE_STRING)
7692 if (collect_before_allocs)
7695 if (ALIGN_TO (klass->instance_size, ALLOC_ALIGN) < MAX_SMALL_OBJ_SIZE)
7696 return mono_gc_get_managed_allocator_by_type (ATYPE_SMALL);
7698 return mono_gc_get_managed_allocator_by_type (ATYPE_NORMAL);
7705 mono_gc_get_managed_array_allocator (MonoVTable *vtable, int rank)
7707 #ifdef MANAGED_ALLOCATION
7708 MonoClass *klass = vtable->klass;
7710 #ifdef HAVE_KW_THREAD
7711 int tlab_next_offset = -1;
7712 int tlab_temp_end_offset = -1;
7713 MONO_THREAD_VAR_OFFSET (tlab_next, tlab_next_offset);
7714 MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
7716 if (tlab_next_offset == -1 || tlab_temp_end_offset == -1)
7722 if (!mono_runtime_has_tls_get ())
7724 if (mono_profiler_get_events () & MONO_PROFILE_ALLOCATIONS)
7726 if (collect_before_allocs)
7728 g_assert (!klass->has_finalize && !klass->marshalbyref);
7730 return mono_gc_get_managed_allocator_by_type (ATYPE_VECTOR);
7737 mono_gc_get_managed_allocator_by_type (int atype)
7739 #ifdef MANAGED_ALLOCATION
7742 if (!mono_runtime_has_tls_get ())
7745 mono_loader_lock ();
7746 res = alloc_method_cache [atype];
7748 res = alloc_method_cache [atype] = create_allocator (atype);
7749 mono_loader_unlock ();
7757 mono_gc_get_managed_allocator_types (void)
7764 mono_gc_get_write_barrier (void)
7767 MonoMethodBuilder *mb;
7768 MonoMethodSignature *sig;
7769 #ifdef MANAGED_WBARRIER
7770 int label_no_wb_1, label_no_wb_2, label_no_wb_3, label_no_wb_4, label_need_wb, label_slow_path;
7771 #ifndef ALIGN_NURSERY
7772 int label_continue_1, label_continue_2, label_no_wb_5;
7773 int dereferenced_var;
7775 int buffer_var, buffer_index_var, dummy_var;
7777 #ifdef HAVE_KW_THREAD
7778 int stack_end_offset = -1, store_remset_buffer_offset = -1;
7779 int store_remset_buffer_index_offset = -1, store_remset_buffer_index_addr_offset = -1;
7781 MONO_THREAD_VAR_OFFSET (stack_end, stack_end_offset);
7782 g_assert (stack_end_offset != -1);
7783 MONO_THREAD_VAR_OFFSET (store_remset_buffer, store_remset_buffer_offset);
7784 g_assert (store_remset_buffer_offset != -1);
7785 MONO_THREAD_VAR_OFFSET (store_remset_buffer_index, store_remset_buffer_index_offset);
7786 g_assert (store_remset_buffer_index_offset != -1);
7787 MONO_THREAD_VAR_OFFSET (store_remset_buffer_index_addr, store_remset_buffer_index_addr_offset);
7788 g_assert (store_remset_buffer_index_addr_offset != -1);
7792 // FIXME: Maybe create a separate version for ctors (the branch would be
7793 // correctly predicted more times)
7794 if (write_barrier_method)
7795 return write_barrier_method;
7797 /* Create the IL version of mono_gc_barrier_generic_store () */
7798 sig = mono_metadata_signature_alloc (mono_defaults.corlib, 1);
7799 sig->ret = &mono_defaults.void_class->byval_arg;
7800 sig->params [0] = &mono_defaults.int_class->byval_arg;
7802 mb = mono_mb_new (mono_defaults.object_class, "wbarrier", MONO_WRAPPER_WRITE_BARRIER);
7804 #ifdef MANAGED_WBARRIER
7805 if (mono_runtime_has_tls_get ()) {
7806 #ifdef ALIGN_NURSERY
7807 // if (ptr_in_nursery (ptr)) return;
7809 * Masking out the bits might be faster, but we would have to use 64 bit
7810 * immediates, which might be slower.
7812 mono_mb_emit_ldarg (mb, 0);
7813 mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
7814 mono_mb_emit_byte (mb, CEE_SHR_UN);
7815 mono_mb_emit_icon (mb, (mword)nursery_start >> DEFAULT_NURSERY_BITS);
7816 label_no_wb_1 = mono_mb_emit_branch (mb, CEE_BEQ);
7818 // if (!ptr_in_nursery (*ptr)) return;
7819 mono_mb_emit_ldarg (mb, 0);
7820 mono_mb_emit_byte (mb, CEE_LDIND_I);
7821 mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
7822 mono_mb_emit_byte (mb, CEE_SHR_UN);
7823 mono_mb_emit_icon (mb, (mword)nursery_start >> DEFAULT_NURSERY_BITS);
7824 label_no_wb_2 = mono_mb_emit_branch (mb, CEE_BNE_UN);
7827 // if (ptr < (nursery_start)) goto continue;
7828 mono_mb_emit_ldarg (mb, 0);
7829 mono_mb_emit_ptr (mb, (gpointer) nursery_start);
7830 label_continue_1 = mono_mb_emit_branch (mb, CEE_BLT);
7832 // if (ptr >= nursery_real_end)) goto continue;
7833 mono_mb_emit_ldarg (mb, 0);
7834 mono_mb_emit_ptr (mb, (gpointer) nursery_real_end);
7835 label_continue_2 = mono_mb_emit_branch (mb, CEE_BGE);
7838 label_no_wb_1 = mono_mb_emit_branch (mb, CEE_BR);
7841 mono_mb_patch_branch (mb, label_continue_1);
7842 mono_mb_patch_branch (mb, label_continue_2);
7844 // Dereference and store in local var
7845 dereferenced_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7846 mono_mb_emit_ldarg (mb, 0);
7847 mono_mb_emit_byte (mb, CEE_LDIND_I);
7848 mono_mb_emit_stloc (mb, dereferenced_var);
7850 // if (*ptr < nursery_start) return;
7851 mono_mb_emit_ldloc (mb, dereferenced_var);
7852 mono_mb_emit_ptr (mb, (gpointer) nursery_start);
7853 label_no_wb_2 = mono_mb_emit_branch (mb, CEE_BLT);
7855 // if (*ptr >= nursery_end) return;
7856 mono_mb_emit_ldloc (mb, dereferenced_var);
7857 mono_mb_emit_ptr (mb, (gpointer) nursery_real_end);
7858 label_no_wb_5 = mono_mb_emit_branch (mb, CEE_BGE);
7861 // if (ptr >= stack_end) goto need_wb;
7862 mono_mb_emit_ldarg (mb, 0);
7863 EMIT_TLS_ACCESS (mb, stack_end, stack_end_offset);
7864 label_need_wb = mono_mb_emit_branch (mb, CEE_BGE_UN);
7866 // if (ptr >= stack_start) return;
7867 dummy_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7868 mono_mb_emit_ldarg (mb, 0);
7869 mono_mb_emit_ldloc_addr (mb, dummy_var);
7870 label_no_wb_3 = mono_mb_emit_branch (mb, CEE_BGE_UN);
7873 mono_mb_patch_branch (mb, label_need_wb);
7875 // buffer = STORE_REMSET_BUFFER;
7876 buffer_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7877 EMIT_TLS_ACCESS (mb, store_remset_buffer, store_remset_buffer_offset);
7878 mono_mb_emit_stloc (mb, buffer_var);
7880 // buffer_index = STORE_REMSET_BUFFER_INDEX;
7881 buffer_index_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7882 EMIT_TLS_ACCESS (mb, store_remset_buffer_index, store_remset_buffer_index_offset);
7883 mono_mb_emit_stloc (mb, buffer_index_var);
7885 // if (buffer [buffer_index] == ptr) return;
7886 mono_mb_emit_ldloc (mb, buffer_var);
7887 mono_mb_emit_ldloc (mb, buffer_index_var);
7888 g_assert (sizeof (gpointer) == 4 || sizeof (gpointer) == 8);
7889 mono_mb_emit_icon (mb, sizeof (gpointer) == 4 ? 2 : 3);
7890 mono_mb_emit_byte (mb, CEE_SHL);
7891 mono_mb_emit_byte (mb, CEE_ADD);
7892 mono_mb_emit_byte (mb, CEE_LDIND_I);
7893 mono_mb_emit_ldarg (mb, 0);
7894 label_no_wb_4 = mono_mb_emit_branch (mb, CEE_BEQ);
7897 mono_mb_emit_ldloc (mb, buffer_index_var);
7898 mono_mb_emit_icon (mb, 1);
7899 mono_mb_emit_byte (mb, CEE_ADD);
7900 mono_mb_emit_stloc (mb, buffer_index_var);
7902 // if (buffer_index >= STORE_REMSET_BUFFER_SIZE) goto slow_path;
7903 mono_mb_emit_ldloc (mb, buffer_index_var);
7904 mono_mb_emit_icon (mb, STORE_REMSET_BUFFER_SIZE);
7905 label_slow_path = mono_mb_emit_branch (mb, CEE_BGE);
7907 // buffer [buffer_index] = ptr;
7908 mono_mb_emit_ldloc (mb, buffer_var);
7909 mono_mb_emit_ldloc (mb, buffer_index_var);
7910 g_assert (sizeof (gpointer) == 4 || sizeof (gpointer) == 8);
7911 mono_mb_emit_icon (mb, sizeof (gpointer) == 4 ? 2 : 3);
7912 mono_mb_emit_byte (mb, CEE_SHL);
7913 mono_mb_emit_byte (mb, CEE_ADD);
7914 mono_mb_emit_ldarg (mb, 0);
7915 mono_mb_emit_byte (mb, CEE_STIND_I);
7917 // STORE_REMSET_BUFFER_INDEX = buffer_index;
7918 EMIT_TLS_ACCESS (mb, store_remset_buffer_index_addr, store_remset_buffer_index_addr_offset);
7919 mono_mb_emit_ldloc (mb, buffer_index_var);
7920 mono_mb_emit_byte (mb, CEE_STIND_I);
7923 mono_mb_patch_branch (mb, label_no_wb_1);
7924 mono_mb_patch_branch (mb, label_no_wb_2);
7925 mono_mb_patch_branch (mb, label_no_wb_3);
7926 mono_mb_patch_branch (mb, label_no_wb_4);
7927 #ifndef ALIGN_NURSERY
7928 mono_mb_patch_branch (mb, label_no_wb_5);
7930 mono_mb_emit_byte (mb, CEE_RET);
7933 mono_mb_patch_branch (mb, label_slow_path);
7937 mono_mb_emit_ldarg (mb, 0);
7938 mono_mb_emit_icall (mb, mono_gc_wbarrier_generic_nostore);
7939 mono_mb_emit_byte (mb, CEE_RET);
7941 res = mono_mb_create_method (mb, sig, 16);
7944 mono_loader_lock ();
7945 if (write_barrier_method) {
7946 /* Already created */
7947 mono_free_method (res);
7949 /* double-checked locking */
7950 mono_memory_barrier ();
7951 write_barrier_method = res;
7953 mono_loader_unlock ();
7955 return write_barrier_method;
7958 #endif /* HAVE_SGEN_GC */