2 * sgen-gc.c: Simple generational GC.
5 * Paolo Molaro (lupus@ximian.com)
7 * Copyright 2005-2010 Novell, Inc (http://www.novell.com)
9 * Thread start/stop adapted from Boehm's GC:
10 * Copyright (c) 1994 by Xerox Corporation. All rights reserved.
11 * Copyright (c) 1996 by Silicon Graphics. All rights reserved.
12 * Copyright (c) 1998 by Fergus Henderson. All rights reserved.
13 * Copyright (c) 2000-2004 by Hewlett-Packard Company. All rights reserved.
15 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
16 * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
18 * Permission is hereby granted to use or copy this program
19 * for any purpose, provided the above notices are retained on all copies.
20 * Permission to modify the code and to distribute modified code is granted,
21 * provided the above notices are retained, and a notice that the code was
22 * modified is included with the above copyright notice.
25 * Copyright 2001-2003 Ximian, Inc
26 * Copyright 2003-2010 Novell, Inc.
28 * Permission is hereby granted, free of charge, to any person obtaining
29 * a copy of this software and associated documentation files (the
30 * "Software"), to deal in the Software without restriction, including
31 * without limitation the rights to use, copy, modify, merge, publish,
32 * distribute, sublicense, and/or sell copies of the Software, and to
33 * permit persons to whom the Software is furnished to do so, subject to
34 * the following conditions:
36 * The above copyright notice and this permission notice shall be
37 * included in all copies or substantial portions of the Software.
39 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
40 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
41 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
42 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
43 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
44 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
45 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
48 * Important: allocation provides always zeroed memory, having to do
49 * a memset after allocation is deadly for performance.
50 * Memory usage at startup is currently as follows:
52 * 64 KB internal space
54 * We should provide a small memory config with half the sizes
56 * We currently try to make as few mono assumptions as possible:
57 * 1) 2-word header with no GC pointers in it (first vtable, second to store the
59 * 2) gc descriptor is the second word in the vtable (first word in the class)
60 * 3) 8 byte alignment is the minimum and enough (not true for special structures (SIMD), FIXME)
61 * 4) there is a function to get an object's size and the number of
62 * elements in an array.
63 * 5) we know the special way bounds are allocated for complex arrays
64 * 6) we know about proxies and how to treat them when domains are unloaded
66 * Always try to keep stack usage to a minimum: no recursive behaviour
67 * and no large stack allocs.
69 * General description.
70 * Objects are initially allocated in a nursery using a fast bump-pointer technique.
71 * When the nursery is full we start a nursery collection: this is performed with a
73 * When the old generation is full we start a copying GC of the old generation as well:
74 * this will be changed to mark&sweep with copying when fragmentation becomes to severe
75 * in the future. Maybe we'll even do both during the same collection like IMMIX.
77 * The things that complicate this description are:
78 * *) pinned objects: we can't move them so we need to keep track of them
79 * *) no precise info of the thread stacks and registers: we need to be able to
80 * quickly find the objects that may be referenced conservatively and pin them
81 * (this makes the first issues more important)
82 * *) large objects are too expensive to be dealt with using copying GC: we handle them
83 * with mark/sweep during major collections
84 * *) some objects need to not move even if they are small (interned strings, Type handles):
85 * we use mark/sweep for them, too: they are not allocated in the nursery, but inside
86 * PinnedChunks regions
92 *) we could have a function pointer in MonoClass to implement
93 customized write barriers for value types
95 *) investigate the stuff needed to advance a thread to a GC-safe
96 point (single-stepping, read from unmapped memory etc) and implement it.
97 This would enable us to inline allocations and write barriers, for example,
98 or at least parts of them, like the write barrier checks.
99 We may need this also for handling precise info on stacks, even simple things
100 as having uninitialized data on the stack and having to wait for the prolog
101 to zero it. Not an issue for the last frame that we scan conservatively.
102 We could always not trust the value in the slots anyway.
104 *) modify the jit to save info about references in stack locations:
105 this can be done just for locals as a start, so that at least
106 part of the stack is handled precisely.
108 *) test/fix endianess issues
110 *) Implement a card table as the write barrier instead of remembered
111 sets? Card tables are not easy to implement with our current
112 memory layout. We have several different kinds of major heap
113 objects: Small objects in regular blocks, small objects in pinned
114 chunks and LOS objects. If we just have a pointer we have no way
115 to tell which kind of object it points into, therefore we cannot
116 know where its card table is. The least we have to do to make
117 this happen is to get rid of write barriers for indirect stores.
120 *) Get rid of write barriers for indirect stores. We can do this by
121 telling the GC to wbarrier-register an object once we do an ldloca
122 or ldelema on it, and to unregister it once it's not used anymore
123 (it can only travel downwards on the stack). The problem with
124 unregistering is that it needs to happen eventually no matter
125 what, even if exceptions are thrown, the thread aborts, etc.
126 Rodrigo suggested that we could do only the registering part and
127 let the collector find out (pessimistically) when it's safe to
128 unregister, namely when the stack pointer of the thread that
129 registered the object is higher than it was when the registering
130 happened. This might make for a good first implementation to get
131 some data on performance.
133 *) Some sort of blacklist support? Blacklists is a concept from the
134 Boehm GC: if during a conservative scan we find pointers to an
135 area which we might use as heap, we mark that area as unusable, so
136 pointer retention by random pinning pointers is reduced.
138 *) experiment with max small object size (very small right now - 2kb,
139 because it's tied to the max freelist size)
141 *) add an option to mmap the whole heap in one chunk: it makes for many
142 simplifications in the checks (put the nursery at the top and just use a single
143 check for inclusion/exclusion): the issue this has is that on 32 bit systems it's
144 not flexible (too much of the address space may be used by default or we can't
145 increase the heap as needed) and we'd need a race-free mechanism to return memory
146 back to the system (mprotect(PROT_NONE) will still keep the memory allocated if it
147 was written to, munmap is needed, but the following mmap may not find the same segment
150 *) memzero the major fragments after restarting the world and optionally a smaller
153 *) investigate having fragment zeroing threads
155 *) separate locks for finalization and other minor stuff to reduce
158 *) try a different copying order to improve memory locality
160 *) a thread abort after a store but before the write barrier will
161 prevent the write barrier from executing
163 *) specialized dynamically generated markers/copiers
165 *) Dynamically adjust TLAB size to the number of threads. If we have
166 too many threads that do allocation, we might need smaller TLABs,
167 and we might get better performance with larger TLABs if we only
168 have a handful of threads. We could sum up the space left in all
169 assigned TLABs and if that's more than some percentage of the
170 nursery size, reduce the TLAB size.
172 *) Explore placing unreachable objects on unused nursery memory.
173 Instead of memset'ng a region to zero, place an int[] covering it.
174 A good place to start is add_nursery_frag. The tricky thing here is
175 placing those objects atomically outside of a collection.
185 #include <semaphore.h>
190 #include "metadata/metadata-internals.h"
191 #include "metadata/class-internals.h"
192 #include "metadata/gc-internal.h"
193 #include "metadata/object-internals.h"
194 #include "metadata/threads.h"
195 #include "metadata/sgen-gc.h"
196 #include "metadata/sgen-archdep.h"
197 #include "metadata/mono-gc.h"
198 #include "metadata/method-builder.h"
199 #include "metadata/profiler-private.h"
200 #include "metadata/monitor.h"
201 #include "metadata/threadpool-internals.h"
202 #include "metadata/mempool-internals.h"
203 #include "metadata/marshal.h"
204 #include "utils/mono-mmap.h"
205 #include "utils/mono-time.h"
206 #include "utils/mono-semaphore.h"
207 #include "utils/mono-counters.h"
209 #include <mono/utils/memcheck.h>
211 #define OPDEF(a,b,c,d,e,f,g,h,i,j) \
215 #include "mono/cil/opcode.def"
222 * ######################################################################
223 * ######## Types and constants used by the GC.
224 * ######################################################################
226 #if SIZEOF_VOID_P == 4
227 typedef guint32 mword;
229 typedef guint64 mword;
232 static int gc_initialized = 0;
233 static int gc_debug_level = 0;
234 static FILE* gc_debug_file;
235 /* If set, do a minor collection before every allocation */
236 static gboolean collect_before_allocs = FALSE;
237 /* If set, do a heap consistency check before each minor collection */
238 static gboolean consistency_check_at_minor_collection = FALSE;
239 /* If set, check that there are no references to the domain left at domain unload */
240 static gboolean xdomain_checks = FALSE;
241 /* If not null, dump the heap after each collection into this file */
242 static FILE *heap_dump_file = NULL;
243 /* If set, mark stacks conservatively, even if precise marking is possible */
244 static gboolean conservative_stack_mark = TRUE;
245 /* If set, do a plausibility check on the scan_starts before and after
247 static gboolean do_scan_starts_check = FALSE;
250 * Turning on heavy statistics will turn off the managed allocator and
251 * the managed write barrier.
253 //#define HEAVY_STATISTICS
255 #ifdef HEAVY_STATISTICS
256 #define HEAVY_STAT(x) x
258 #define HEAVY_STAT(x)
261 #ifdef HEAVY_STATISTICS
262 static long long stat_objects_alloced = 0;
263 static long long stat_bytes_alloced = 0;
264 static long long stat_objects_alloced_degraded = 0;
265 static long long stat_bytes_alloced_degraded = 0;
266 static long long stat_bytes_alloced_los = 0;
268 static long long stat_copy_object_called_nursery = 0;
269 static long long stat_objects_copied_nursery = 0;
270 static long long stat_copy_object_called_major = 0;
271 static long long stat_objects_copied_major = 0;
273 static long long stat_scan_object_called_nursery = 0;
274 static long long stat_scan_object_called_major = 0;
276 static long long stat_nursery_copy_object_failed_from_space = 0;
277 static long long stat_nursery_copy_object_failed_forwarded = 0;
278 static long long stat_nursery_copy_object_failed_pinned = 0;
280 static long long stat_store_remsets = 0;
281 static long long stat_store_remsets_unique = 0;
282 static long long stat_saved_remsets_1 = 0;
283 static long long stat_saved_remsets_2 = 0;
284 static long long stat_global_remsets_added = 0;
285 static long long stat_global_remsets_readded = 0;
286 static long long stat_global_remsets_processed = 0;
287 static long long stat_global_remsets_discarded = 0;
289 static long long stat_wasted_fragments_used = 0;
290 static long long stat_wasted_fragments_bytes = 0;
292 static int stat_wbarrier_set_field = 0;
293 static int stat_wbarrier_set_arrayref = 0;
294 static int stat_wbarrier_arrayref_copy = 0;
295 static int stat_wbarrier_generic_store = 0;
296 static int stat_wbarrier_generic_store_remset = 0;
297 static int stat_wbarrier_set_root = 0;
298 static int stat_wbarrier_value_copy = 0;
299 static int stat_wbarrier_object_copy = 0;
302 static long long time_minor_pre_collection_fragment_clear = 0;
303 static long long time_minor_pinning = 0;
304 static long long time_minor_scan_remsets = 0;
305 static long long time_minor_scan_pinned = 0;
306 static long long time_minor_scan_registered_roots = 0;
307 static long long time_minor_scan_thread_data = 0;
308 static long long time_minor_finish_gray_stack = 0;
309 static long long time_minor_fragment_creation = 0;
311 static long long time_major_pre_collection_fragment_clear = 0;
312 static long long time_major_pinning = 0;
313 static long long time_major_scan_pinned = 0;
314 static long long time_major_scan_registered_roots = 0;
315 static long long time_major_scan_thread_data = 0;
316 static long long time_major_scan_alloc_pinned = 0;
317 static long long time_major_scan_finalized = 0;
318 static long long time_major_scan_big_objects = 0;
319 static long long time_major_finish_gray_stack = 0;
320 static long long time_major_sweep = 0;
321 static long long time_major_fragment_creation = 0;
323 static long long pinned_chunk_bytes_alloced = 0;
324 static long long large_internal_bytes_alloced = 0;
326 /* Keep in sync with internal_mem_names in dump_heap()! */
328 INTERNAL_MEM_PIN_QUEUE,
329 INTERNAL_MEM_FRAGMENT,
330 INTERNAL_MEM_SECTION,
331 INTERNAL_MEM_SCAN_STARTS,
332 INTERNAL_MEM_FIN_TABLE,
333 INTERNAL_MEM_FINALIZE_ENTRY,
334 INTERNAL_MEM_DISLINK_TABLE,
335 INTERNAL_MEM_DISLINK,
336 INTERNAL_MEM_ROOTS_TABLE,
337 INTERNAL_MEM_ROOT_RECORD,
338 INTERNAL_MEM_STATISTICS,
340 INTERNAL_MEM_GRAY_QUEUE,
341 INTERNAL_MEM_STORE_REMSET,
342 INTERNAL_MEM_MS_TABLES,
343 INTERNAL_MEM_MS_BLOCK_INFO,
344 INTERNAL_MEM_EPHEMERON_LINK,
348 static long small_internal_mem_bytes [INTERNAL_MEM_MAX];
352 mono_gc_flush_info (void)
354 fflush (gc_debug_file);
358 #define MAX_DEBUG_LEVEL 2
359 #define DEBUG(level,a) do {if (G_UNLIKELY ((level) <= MAX_DEBUG_LEVEL && (level) <= gc_debug_level)) a;} while (0)
361 /* Define this to allow the user to change some of the constants by specifying
362 * their values in the MONO_GC_PARAMS environmental variable. See
363 * mono_gc_base_init for details. */
364 #define USER_CONFIG 1
366 #define TV_DECLARE(name) gint64 name
367 #define TV_GETTIME(tv) tv = mono_100ns_ticks ()
368 #define TV_ELAPSED(start,end) (int)((end-start) / 10)
369 #define TV_ELAPSED_MS(start,end) ((TV_ELAPSED((start),(end)) + 500) / 1000)
371 #define ALIGN_TO(val,align) ((((guint64)val) + ((align) - 1)) & ~((align) - 1))
373 #define GC_BITS_PER_WORD (sizeof (mword) * 8)
381 typedef struct _Block Block;
387 /* each request from the OS ends up in a GCMemSection */
388 typedef struct _GCMemSection GCMemSection;
389 struct _GCMemSection {
393 /* pointer where more data could be allocated if it fits */
397 * scan starts is an array of pointers to objects equally spaced in the allocation area
398 * They let use quickly find pinned objects from pinning pointers.
401 /* in major collections indexes in the pin_queue for objects that pin this section */
404 unsigned short num_scan_start;
405 gboolean is_to_space;
408 #define SIZEOF_GC_MEM_SECTION ((sizeof (GCMemSection) + 7) & ~7)
410 /* large object space struct: 64+ KB */
411 /* we could make this limit much smaller to avoid memcpy copy
412 * and potentially have more room in the GC descriptor: need to measure
413 * This also means that such small OS objects will need to be
414 * allocated in a different way (using pinned chunks).
415 * We may want to put large but smaller than 64k objects in the fixed space
416 * when we move the object from one generation to another (to limit the
417 * pig in the snake effect).
418 * Note: it may be worth to have an optimized copy function, since we can
419 * assume that objects are aligned and have a multiple of 8 size.
420 * FIXME: This structure needs to be a multiple of 8 bytes in size: this is not
421 * true if MONO_ZERO_LEN_ARRAY is nonzero.
423 typedef struct _LOSObject LOSObject;
426 mword size; /* this is the object size */
428 int dummy; /* to have a sizeof (LOSObject) a multiple of ALLOC_ALIGN and data starting at same alignment */
429 char data [MONO_ZERO_LEN_ARRAY];
432 /* Pinned objects are allocated in the LOS space if bigger than half a page
433 * or from freelists otherwise. We assume that pinned objects are relatively few
434 * and they have a slow dying speed (like interned strings, thread objects).
435 * As such they will be collected only at major collections.
436 * free lists are not global: when we need memory we allocate a PinnedChunk.
437 * Each pinned chunk is made of several pages, the first of wich is used
438 * internally for bookeeping (here think of a page as 4KB). The bookeeping
439 * includes the freelists vectors and info about the object size of each page
440 * in the pinned chunk. So, when needed, a free page is found in a pinned chunk,
441 * a size is assigned to it, the page is divided in the proper chunks and each
442 * chunk is added to the freelist. To not waste space, the remaining space in the
443 * first page is used as objects of size 16 or 32 (need to measure which are more
445 * We use this same structure to allocate memory used internally by the GC, so
446 * we never use malloc/free if we need to alloc during collection: the world is stopped
447 * and malloc/free will deadlock.
448 * When we want to iterate over pinned objects, we just scan a page at a time
449 * linearly according to the size of objects in the page: the next pointer used to link
450 * the items in the freelist uses the same word as the vtable. Since we keep freelists
451 * for each pinned chunk, if the word points outside the pinned chunk it means
453 * We could avoid this expensive scanning in creative ways. We could have a policy
454 * of putting in the pinned space only objects we know about that have no struct fields
455 * with references and we can easily use a even expensive write barrier for them,
456 * since pointer writes on such objects should be rare.
457 * The best compromise is to just alloc interned strings and System.MonoType in them.
458 * It would be nice to allocate MonoThread in it, too: must check that we properly
459 * use write barriers so we don't have to do any expensive scanning of the whole pinned
460 * chunk list during minor collections. We can avoid it now because we alloc in it only
461 * reference-free objects.
463 #define PINNED_FIRST_SLOT_SIZE (sizeof (gpointer) * 4)
464 #define MAX_FREELIST_SIZE 2048
465 #define PINNED_PAGE_SIZE (4096)
466 #define PINNED_CHUNK_MIN_SIZE (4096*8)
467 typedef struct _PinnedChunk PinnedChunk;
468 struct _PinnedChunk {
471 int *page_sizes; /* a 0 means the page is still unused */
474 void *data [1]; /* page sizes and free lists are stored here */
477 /* The method used to clear the nursery */
478 /* Clearing at nursery collections is the safest, but has bad interactions with caches.
479 * Clearing at TLAB creation is much faster, but more complex and it might expose hard
484 CLEAR_AT_TLAB_CREATION
485 } NurseryClearPolicy;
487 static NurseryClearPolicy nursery_clear_policy = CLEAR_AT_TLAB_CREATION;
490 * If this is set, the nursery is aligned to an address aligned to its size, ie.
491 * a 1MB nursery will be aligned to an address divisible by 1MB. This allows us to
492 * speed up ptr_in_nursery () checks which are very frequent. This requires the
493 * nursery size to be a compile time constant.
495 #define ALIGN_NURSERY 1
498 * The young generation is divided into fragments. This is because
499 * we can hand one fragments to a thread for lock-less fast alloc and
500 * because the young generation ends up fragmented anyway by pinned objects.
501 * Once a collection is done, a list of fragments is created. When doing
502 * thread local alloc we use smallish nurseries so we allow new threads to
503 * allocate memory from gen0 without triggering a collection. Threads that
504 * are found to allocate lots of memory are given bigger fragments. This
505 * should make the finalizer thread use little nursery memory after a while.
506 * We should start assigning threads very small fragments: if there are many
507 * threads the nursery will be full of reserved space that the threads may not
508 * use at all, slowing down allocation speed.
509 * Thread local allocation is done from areas of memory Hotspot calls Thread Local
510 * Allocation Buffers (TLABs).
512 typedef struct _Fragment Fragment;
516 char *fragment_start;
517 char *fragment_limit; /* the current soft limit for allocation */
521 /* the runtime can register areas of memory as roots: we keep two lists of roots,
522 * a pinned root set for conservatively scanned roots and a normal one for
523 * precisely scanned roots (currently implemented as a single list).
525 typedef struct _RootRecord RootRecord;
533 /* for use with write barriers */
534 typedef struct _RememberedSet RememberedSet;
535 struct _RememberedSet {
539 mword data [MONO_ZERO_LEN_ARRAY];
543 * We're never actually using the first element. It's always set to
544 * NULL to simplify the elimination of consecutive duplicate
547 #define STORE_REMSET_BUFFER_SIZE 1024
549 typedef struct _GenericStoreRememberedSet GenericStoreRememberedSet;
550 struct _GenericStoreRememberedSet {
551 GenericStoreRememberedSet *next;
552 /* We need one entry less because the first entry of store
553 remset buffers is always a dummy and we don't copy it. */
554 gpointer data [STORE_REMSET_BUFFER_SIZE - 1];
557 /* we have 4 possible values in the low 2 bits */
559 REMSET_LOCATION, /* just a pointer to the exact location */
560 REMSET_RANGE, /* range of pointer fields */
561 REMSET_OBJECT, /* mark all the object for scanning */
562 REMSET_VTYPE, /* a valuetype array described by a gc descriptor and a count */
563 REMSET_TYPE_MASK = 0x3
566 #ifdef HAVE_KW_THREAD
567 static __thread RememberedSet *remembered_set MONO_TLS_FAST;
569 static pthread_key_t remembered_set_key;
570 static RememberedSet *global_remset;
571 static RememberedSet *freed_thread_remsets;
572 static GenericStoreRememberedSet *generic_store_remsets = NULL;
574 /*A two slots cache for recently inserted remsets */
575 static gpointer global_remset_cache [2];
577 /* FIXME: later choose a size that takes into account the RememberedSet struct
578 * and doesn't waste any alloc paddin space.
580 #define DEFAULT_REMSET_SIZE 1024
581 static RememberedSet* alloc_remset (int size, gpointer id);
583 /* Structure that corresponds to a MonoVTable: desc is a mword so requires
584 * no cast from a pointer to an integer
591 /* these bits are set in the object vtable: we could merge them since an object can be
592 * either pinned or forwarded but not both.
593 * We store them in the vtable slot because the bits are used in the sync block for
594 * other purposes: if we merge them and alloc the sync blocks aligned to 8 bytes, we can change
595 * this and use bit 3 in the syncblock (with the lower two bits both set for forwarded, that
596 * would be an invalid combination for the monitor and hash code).
597 * The values are already shifted.
598 * The forwarding address is stored in the sync block.
600 #define FORWARDED_BIT 1
602 #define VTABLE_BITS_MASK 0x3
604 /* returns NULL if not forwarded, or the forwarded address */
605 #define object_is_forwarded(obj) (((mword*)(obj))[0] & FORWARDED_BIT? (void*)(((mword*)(obj))[1]): NULL)
606 /* set the forwarded address fw_addr for object obj */
607 #define forward_object(obj,fw_addr) do { \
608 ((mword*)(obj))[0] |= FORWARDED_BIT; \
609 ((mword*)(obj))[1] = (mword)(fw_addr); \
612 #define object_is_pinned(obj) (((mword*)(obj))[0] & PINNED_BIT)
613 #define pin_object(obj) do { \
614 ((mword*)(obj))[0] |= PINNED_BIT; \
616 #define unpin_object(obj) do { \
617 ((mword*)(obj))[0] &= ~PINNED_BIT; \
621 #define ptr_in_nursery(ptr) (((mword)(ptr) & ~((1 << DEFAULT_NURSERY_BITS) - 1)) == (mword)nursery_start)
623 #define ptr_in_nursery(ptr) ((char*)(ptr) >= nursery_start && (char*)(ptr) < nursery_real_end)
627 * Since we set bits in the vtable, use the macro to load it from the pointer to
628 * an object that is potentially pinned.
630 #define LOAD_VTABLE(addr) ((*(mword*)(addr)) & ~VTABLE_BITS_MASK)
633 safe_name (void* obj)
635 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (obj);
636 return vt->klass->name;
640 safe_object_get_size (MonoObject* o)
642 MonoClass *klass = ((MonoVTable*)LOAD_VTABLE (o))->klass;
643 if (klass == mono_defaults.string_class) {
644 return sizeof (MonoString) + 2 * mono_string_length_fast ((MonoString*) o) + 2;
645 } else if (klass->rank) {
646 MonoArray *array = (MonoArray*)o;
647 size_t size = sizeof (MonoArray) + klass->sizes.element_size * mono_array_length_fast (array);
648 if (G_UNLIKELY (array->bounds)) {
649 size += sizeof (mono_array_size_t) - 1;
650 size &= ~(sizeof (mono_array_size_t) - 1);
651 size += sizeof (MonoArrayBounds) * klass->rank;
655 /* from a created object: the class must be inited already */
656 return klass->instance_size;
661 * ######################################################################
662 * ######## Global data.
663 * ######################################################################
665 static LOCK_DECLARE (gc_mutex);
666 static int gc_disabled = 0;
667 static int num_minor_gcs = 0;
668 static int num_major_gcs = 0;
672 /* good sizes are 512KB-1MB: larger ones increase a lot memzeroing time */
673 #define DEFAULT_NURSERY_SIZE (default_nursery_size)
674 static int default_nursery_size = (1 << 22);
676 /* The number of trailing 0 bits in DEFAULT_NURSERY_SIZE */
677 #define DEFAULT_NURSERY_BITS (default_nursery_bits)
678 static int default_nursery_bits = 22;
683 #define DEFAULT_NURSERY_SIZE (4*1024*1024)
685 #define DEFAULT_NURSERY_BITS 22
690 #define MIN_MINOR_COLLECTION_ALLOWANCE (DEFAULT_NURSERY_SIZE * 4)
691 /* to quickly find the head of an object pinned by a conservative address
692 * we keep track of the objects allocated for each SCAN_START_SIZE memory
693 * chunk in the nursery or other memory sections. Larger values have less
694 * memory overhead and bigger runtime cost. 4-8 KB are reasonable values.
696 #define SCAN_START_SIZE (4096*2)
697 /* the minimum size of a fragment that we consider useful for allocation */
698 #define FRAGMENT_MIN_SIZE (512)
699 /* This is a fixed value used for pinned chunks, not the system pagesize */
700 #define FREELIST_PAGESIZE 4096
702 static mword pagesize = 4096;
703 static mword nursery_size;
704 static int degraded_mode = 0;
706 static LOSObject *los_object_list = NULL;
707 static mword los_memory_usage = 0;
708 static mword last_los_memory_usage = 0;
709 static mword los_num_objects = 0;
710 static mword total_alloc = 0;
711 /* use this to tune when to do a major/minor collection */
712 static mword memory_pressure = 0;
713 static int minor_collection_allowance;
714 static int minor_collection_sections_alloced = 0;
716 static GCMemSection *nursery_section = NULL;
717 static mword lowest_heap_address = ~(mword)0;
718 static mword highest_heap_address = 0;
720 static LOCK_DECLARE (interruption_mutex);
722 typedef struct _FinalizeEntry FinalizeEntry;
723 struct _FinalizeEntry {
728 typedef struct _FinalizeEntryHashTable FinalizeEntryHashTable;
729 struct _FinalizeEntryHashTable {
730 FinalizeEntry **table;
735 typedef struct _DisappearingLink DisappearingLink;
736 struct _DisappearingLink {
737 DisappearingLink *next;
741 typedef struct _DisappearingLinkHashTable DisappearingLinkHashTable;
742 struct _DisappearingLinkHashTable {
743 DisappearingLink **table;
748 typedef struct _EphemeronLinkNode EphemeronLinkNode;
750 struct _EphemeronLinkNode {
751 EphemeronLinkNode *next;
760 #define LARGE_INTERNAL_MEM_HEADER_MAGIC 0x7d289f3a
762 typedef struct _LargeInternalMemHeader LargeInternalMemHeader;
763 struct _LargeInternalMemHeader {
775 int current_collection_generation = -1;
778 * The link pointer is hidden by negating each bit. We use the lowest
779 * bit of the link (before negation) to store whether it needs
780 * resurrection tracking.
782 #define HIDE_POINTER(p,t) ((gpointer)(~((gulong)(p)|((t)?1:0))))
783 #define REVEAL_POINTER(p) ((gpointer)((~(gulong)(p))&~3L))
785 #define DISLINK_OBJECT(d) (REVEAL_POINTER (*(d)->link))
786 #define DISLINK_TRACK(d) ((~(gulong)(*(d)->link)) & 1)
789 * The finalizable hash has the object as the key, the
790 * disappearing_link hash, has the link address as key.
792 static FinalizeEntryHashTable minor_finalizable_hash;
793 static FinalizeEntryHashTable major_finalizable_hash;
794 /* objects that are ready to be finalized */
795 static FinalizeEntry *fin_ready_list = NULL;
796 static FinalizeEntry *critical_fin_list = NULL;
798 static DisappearingLinkHashTable minor_disappearing_link_hash;
799 static DisappearingLinkHashTable major_disappearing_link_hash;
801 static EphemeronLinkNode *ephemeron_list;
803 static int num_ready_finalizers = 0;
804 static int no_finalize = 0;
806 /* keep each size a multiple of ALLOC_ALIGN */
807 /* on 64 bit systems 8 is likely completely unused. */
808 static const int freelist_sizes [] = {
809 8, 16, 24, 32, 40, 48, 64, 80,
810 96, 128, 160, 192, 224, 256, 320, 384,
811 448, 512, 584, 680, 816, 1024, 1360, 2048};
812 #define FREELIST_NUM_SLOTS (sizeof (freelist_sizes) / sizeof (freelist_sizes [0]))
814 /* This is also the MAJOR_SECTION_SIZE for the copying major
816 #define PINNED_CHUNK_SIZE (128 * 1024)
818 /* internal_chunk_list is used for allocating structures needed by the GC */
819 static PinnedChunk *internal_chunk_list = NULL;
821 static int slot_for_size (size_t size);
824 ROOT_TYPE_NORMAL = 0, /* "normal" roots */
825 ROOT_TYPE_PINNED = 1, /* roots without a GC descriptor */
826 ROOT_TYPE_WBARRIER = 2, /* roots with a write barrier */
830 /* registered roots: the key to the hash is the root start address */
832 * Different kinds of roots are kept separate to speed up pin_from_roots () for example.
834 static RootRecord **roots_hash [ROOT_TYPE_NUM] = { NULL, NULL };
835 static int roots_hash_size [ROOT_TYPE_NUM] = { 0, 0, 0 };
836 static mword roots_size = 0; /* amount of memory in the root set */
837 static int num_roots_entries [ROOT_TYPE_NUM] = { 0, 0, 0 };
840 * The current allocation cursors
841 * We allocate objects in the nursery.
842 * The nursery is the area between nursery_start and nursery_real_end.
843 * Allocation is done from a Thread Local Allocation Buffer (TLAB). TLABs are allocated
844 * from nursery fragments.
845 * tlab_next is the pointer to the space inside the TLAB where the next object will
847 * tlab_temp_end is the pointer to the end of the temporary space reserved for
848 * the allocation: it allows us to set the scan starts at reasonable intervals.
849 * tlab_real_end points to the end of the TLAB.
850 * nursery_frag_real_end points to the end of the currently used nursery fragment.
851 * nursery_first_pinned_start points to the start of the first pinned object in the nursery
852 * nursery_last_pinned_end points to the end of the last pinned object in the nursery
853 * At the next allocation, the area of the nursery where objects can be present is
854 * between MIN(nursery_first_pinned_start, first_fragment_start) and
855 * MAX(nursery_last_pinned_end, nursery_frag_real_end)
857 static char *nursery_start = NULL;
859 /* eventually share with MonoThread? */
860 typedef struct _SgenThreadInfo SgenThreadInfo;
862 struct _SgenThreadInfo {
863 SgenThreadInfo *next;
865 unsigned int stop_count; /* to catch duplicate signals */
868 volatile int in_critical_region;
871 void *stack_start_limit;
872 char **tlab_next_addr;
873 char **tlab_start_addr;
874 char **tlab_temp_end_addr;
875 char **tlab_real_end_addr;
876 gpointer **store_remset_buffer_addr;
877 long *store_remset_buffer_index_addr;
878 RememberedSet *remset;
879 gpointer runtime_data;
880 gpointer stopped_ip; /* only valid if the thread is stopped */
881 MonoDomain *stopped_domain; /* ditto */
882 gpointer *stopped_regs; /* ditto */
883 #ifndef HAVE_KW_THREAD
888 gpointer *store_remset_buffer;
889 long store_remset_buffer_index;
893 #ifdef HAVE_KW_THREAD
894 #define TLAB_ACCESS_INIT
895 #define TLAB_START tlab_start
896 #define TLAB_NEXT tlab_next
897 #define TLAB_TEMP_END tlab_temp_end
898 #define TLAB_REAL_END tlab_real_end
899 #define REMEMBERED_SET remembered_set
900 #define STORE_REMSET_BUFFER store_remset_buffer
901 #define STORE_REMSET_BUFFER_INDEX store_remset_buffer_index
902 #define IN_CRITICAL_REGION thread_info->in_critical_region
904 static pthread_key_t thread_info_key;
905 #define TLAB_ACCESS_INIT SgenThreadInfo *__thread_info__ = pthread_getspecific (thread_info_key)
906 #define TLAB_START (__thread_info__->tlab_start)
907 #define TLAB_NEXT (__thread_info__->tlab_next)
908 #define TLAB_TEMP_END (__thread_info__->tlab_temp_end)
909 #define TLAB_REAL_END (__thread_info__->tlab_real_end)
910 #define REMEMBERED_SET (__thread_info__->remset)
911 #define STORE_REMSET_BUFFER (__thread_info__->store_remset_buffer)
912 #define STORE_REMSET_BUFFER_INDEX (__thread_info__->store_remset_buffer_index)
913 #define IN_CRITICAL_REGION (__thread_info__->in_critical_region)
916 /* we use the memory barrier only to prevent compiler reordering (a memory constraint may be enough) */
917 #define ENTER_CRITICAL_REGION do {IN_CRITICAL_REGION = 1;mono_memory_barrier ();} while (0)
918 #define EXIT_CRITICAL_REGION do {IN_CRITICAL_REGION = 0;mono_memory_barrier ();} while (0)
921 * FIXME: What is faster, a TLS variable pointing to a structure, or separate TLS
922 * variables for next+temp_end ?
924 #ifdef HAVE_KW_THREAD
925 static __thread SgenThreadInfo *thread_info;
926 static __thread char *tlab_start;
927 static __thread char *tlab_next;
928 static __thread char *tlab_temp_end;
929 static __thread char *tlab_real_end;
930 static __thread gpointer *store_remset_buffer;
931 static __thread long store_remset_buffer_index;
932 /* Used by the managed allocator/wbarrier */
933 static __thread char **tlab_next_addr;
934 static __thread char *stack_end;
935 static __thread long *store_remset_buffer_index_addr;
937 static char *nursery_next = NULL;
938 static char *nursery_frag_real_end = NULL;
939 static char *nursery_real_end = NULL;
940 static char *nursery_last_pinned_end = NULL;
942 /* The size of a TLAB */
943 /* The bigger the value, the less often we have to go to the slow path to allocate a new
944 * one, but the more space is wasted by threads not allocating much memory.
946 * FIXME: Make this self-tuning for each thread.
948 static guint32 tlab_size = (1024 * 4);
950 /*How much space is tolerable to be wasted from the current fragment when allocating a new TLAB*/
951 #define MAX_NURSERY_TLAB_WASTE 512
953 /* fragments that are free and ready to be used for allocation */
954 static Fragment *nursery_fragments = NULL;
955 /* freeelist of fragment structures */
956 static Fragment *fragment_freelist = NULL;
959 * Objects bigger then this go into the large object space. This size
960 * has a few constraints. It must fit into the major heap, which in
961 * the case of the copying collector means that it must fit into a
962 * pinned chunk. It must also play well with the GC descriptors, some
963 * of which (DESC_TYPE_RUN_LENGTH, DESC_TYPE_SMALL_BITMAP) encode the
966 #define MAX_SMALL_OBJ_SIZE 2040
968 /* Functions supplied by the runtime to be called by the GC */
969 static MonoGCCallbacks gc_callbacks;
971 #define ALLOC_ALIGN 8
972 #define ALLOC_ALIGN_BITS 3
974 #define MOVED_OBJECTS_NUM 64
975 static void *moved_objects [MOVED_OBJECTS_NUM];
976 static int moved_objects_idx = 0;
979 * ######################################################################
980 * ######## Macros and function declarations.
981 * ######################################################################
984 #define UPDATE_HEAP_BOUNDARIES(low,high) do { \
985 if ((mword)(low) < lowest_heap_address) \
986 lowest_heap_address = (mword)(low); \
987 if ((mword)(high) > highest_heap_address) \
988 highest_heap_address = (mword)(high); \
990 #define ADDR_IN_HEAP_BOUNDARIES(addr) ((p) >= lowest_heap_address && (p) < highest_heap_address)
993 align_pointer (void *ptr)
995 mword p = (mword)ptr;
996 p += sizeof (gpointer) - 1;
997 p &= ~ (sizeof (gpointer) - 1);
1001 typedef void (*CopyOrMarkObjectFunc) (void**);
1002 typedef char* (*ScanObjectFunc) (char*);
1004 /* forward declarations */
1005 static void* get_internal_mem (size_t size, int type);
1006 static void free_internal_mem (void *addr, int type);
1007 static void* get_os_memory (size_t size, int activate);
1008 static void* get_os_memory_aligned (mword size, mword alignment, gboolean activate);
1009 static void free_os_memory (void *addr, size_t size);
1010 static G_GNUC_UNUSED void report_internal_mem_usage (void);
1012 static int stop_world (void);
1013 static int restart_world (void);
1014 static void add_to_global_remset (gpointer ptr);
1015 static void scan_thread_data (void *start_nursery, void *end_nursery, gboolean precise);
1016 static void scan_from_remsets (void *start_nursery, void *end_nursery);
1017 static void scan_from_registered_roots (CopyOrMarkObjectFunc copy_func, char *addr_start, char *addr_end, int root_type);
1018 static void scan_finalizer_entries (CopyOrMarkObjectFunc copy_func, FinalizeEntry *list);
1019 static void find_pinning_ref_from_thread (char *obj, size_t size);
1020 static void update_current_thread_stack (void *start);
1021 static void finalize_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, int generation);
1022 static void add_or_remove_disappearing_link (MonoObject *obj, void **link, gboolean track, int generation);
1023 static void null_link_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, int generation);
1024 static void null_links_for_domain (MonoDomain *domain, int generation);
1025 static gboolean search_fragment_for_size (size_t size);
1026 static int search_fragment_for_size_range (size_t desired_size, size_t minimum_size);
1027 static void build_nursery_fragments (int start_pin, int end_pin);
1028 static void clear_nursery_fragments (char *next);
1029 static void pin_from_roots (void *start_nursery, void *end_nursery);
1030 static int pin_objects_from_addresses (GCMemSection *section, void **start, void **end, void *start_nursery, void *end_nursery);
1031 static void pin_objects_in_section (GCMemSection *section);
1032 static void optimize_pin_queue (int start_slot);
1033 static void clear_remsets (void);
1034 static void clear_tlabs (void);
1035 typedef void (*IterateObjectCallbackFunc) (char*, size_t, void*);
1036 static void scan_area_with_callback (char *start, char *end, IterateObjectCallbackFunc callback, void *data);
1037 static void scan_object (char *start);
1038 static void major_scan_object (char *start);
1039 static void* copy_object_no_checks (void *obj);
1040 static void copy_object (void **obj_slot);
1041 static void* get_chunk_freelist (PinnedChunk *chunk, int slot);
1042 static PinnedChunk* alloc_pinned_chunk (void);
1043 static void free_large_object (LOSObject *obj);
1044 static void sort_addresses (void **array, int size);
1045 static void drain_gray_stack (void);
1046 static void finish_gray_stack (char *start_addr, char *end_addr, int generation);
1047 static gboolean need_major_collection (void);
1049 static void mono_gc_register_disappearing_link (MonoObject *obj, void **link, gboolean track);
1051 void describe_ptr (char *ptr);
1052 static void check_consistency (void);
1053 static void check_major_refs (void);
1054 static void check_section_scan_starts (GCMemSection *section);
1055 static void check_scan_starts (void);
1056 static void check_for_xdomain_refs (void);
1057 static void dump_occupied (char *start, char *end, char *section_start);
1058 static void dump_section (GCMemSection *section, const char *type);
1059 static void dump_heap (const char *type, int num, const char *reason);
1060 static void commit_stats (int generation);
1061 static void report_pinned_chunk (PinnedChunk *chunk, int seq);
1063 void mono_gc_scan_for_specific_ref (MonoObject *key);
1065 static void init_stats (void);
1067 static int mark_ephemerons_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end);
1068 static void clear_unreachable_ephemerons (CopyOrMarkObjectFunc copy_func, char *start, char *end);
1069 static void null_ephemerons_for_domain (MonoDomain *domain);
1071 //#define BINARY_PROTOCOL
1072 #include "sgen-protocol.c"
1073 #include "sgen-pinning.c"
1074 #include "sgen-pinning-stats.c"
1075 #include "sgen-gray.c"
1078 * ######################################################################
1079 * ######## GC descriptors
1080 * ######################################################################
1081 * Used to quickly get the info the GC needs about an object: size and
1082 * where the references are held.
1084 /* objects are aligned to 8 bytes boundaries
1085 * A descriptor is a pointer in MonoVTable, so 32 or 64 bits of size.
1086 * The low 3 bits define the type of the descriptor. The other bits
1087 * depend on the type.
1088 * As a general rule the 13 remaining low bits define the size, either
1089 * of the whole object or of the elements in the arrays. While for objects
1090 * the size is already in bytes, for arrays we need to shift, because
1091 * array elements might be smaller than 8 bytes. In case of arrays, we
1092 * use two bits to describe what the additional high bits represents,
1093 * so the default behaviour can handle element sizes less than 2048 bytes.
1094 * The high 16 bits, if 0 it means the object is pointer-free.
1095 * This design should make it easy and fast to skip over ptr-free data.
1096 * The first 4 types should cover >95% of the objects.
1097 * Note that since the size of objects is limited to 64K, larger objects
1098 * will be allocated in the large object heap.
1099 * If we want 4-bytes alignment, we need to put vector and small bitmap
1104 * We don't use 0 so that 0 isn't a valid GC descriptor. No
1105 * deep reason for this other than to be able to identify a
1106 * non-inited descriptor for debugging.
1108 * If an object contains no references, its GC descriptor is
1109 * always DESC_TYPE_RUN_LENGTH, without a size, no exceptions.
1110 * This is so that we can quickly check for that in
1111 * copy_object_no_checks(), without having to fetch the
1114 DESC_TYPE_RUN_LENGTH = 1, /* 15 bits aligned byte size | 1-3 (offset, numptr) bytes tuples */
1115 DESC_TYPE_SMALL_BITMAP, /* 15 bits aligned byte size | 16-48 bit bitmap */
1116 DESC_TYPE_COMPLEX, /* index for bitmap into complex_descriptors */
1117 DESC_TYPE_VECTOR, /* 10 bits element size | 1 bit array | 2 bits desc | element desc */
1118 DESC_TYPE_ARRAY, /* 10 bits element size | 1 bit array | 2 bits desc | element desc */
1119 DESC_TYPE_LARGE_BITMAP, /* | 29-61 bitmap bits */
1120 DESC_TYPE_COMPLEX_ARR, /* index for bitmap into complex_descriptors */
1121 /* subtypes for arrays and vectors */
1122 DESC_TYPE_V_PTRFREE = 0,/* there are no refs: keep first so it has a zero value */
1123 DESC_TYPE_V_REFS, /* all the array elements are refs */
1124 DESC_TYPE_V_RUN_LEN, /* elements are run-length encoded as DESC_TYPE_RUN_LENGTH */
1125 DESC_TYPE_V_BITMAP /* elements are as the bitmap in DESC_TYPE_SMALL_BITMAP */
1128 #define OBJECT_HEADER_WORDS (sizeof(MonoObject)/sizeof(gpointer))
1129 #define LOW_TYPE_BITS 3
1130 #define SMALL_BITMAP_SHIFT 16
1131 #define SMALL_BITMAP_SIZE (GC_BITS_PER_WORD - SMALL_BITMAP_SHIFT)
1132 #define VECTOR_INFO_SHIFT 14
1133 #define VECTOR_ELSIZE_SHIFT 3
1134 #define LARGE_BITMAP_SIZE (GC_BITS_PER_WORD - LOW_TYPE_BITS)
1135 #define MAX_ELEMENT_SIZE 0x3ff
1136 #define VECTOR_SUBTYPE_PTRFREE (DESC_TYPE_V_PTRFREE << VECTOR_INFO_SHIFT)
1137 #define VECTOR_SUBTYPE_REFS (DESC_TYPE_V_REFS << VECTOR_INFO_SHIFT)
1138 #define VECTOR_SUBTYPE_RUN_LEN (DESC_TYPE_V_RUN_LEN << VECTOR_INFO_SHIFT)
1139 #define VECTOR_SUBTYPE_BITMAP (DESC_TYPE_V_BITMAP << VECTOR_INFO_SHIFT)
1142 /* Root bitmap descriptors are simpler: the lower three bits describe the type
1143 * and we either have 30/62 bitmap bits or nibble-based run-length,
1144 * or a complex descriptor, or a user defined marker function.
1147 ROOT_DESC_CONSERVATIVE, /* 0, so matches NULL value */
1152 ROOT_DESC_TYPE_MASK = 0x7,
1153 ROOT_DESC_TYPE_SHIFT = 3,
1156 #define MAKE_ROOT_DESC(type,val) ((type) | ((val) << ROOT_DESC_TYPE_SHIFT))
1158 #define MAX_USER_DESCRIPTORS 16
1160 static gsize* complex_descriptors = NULL;
1161 static int complex_descriptors_size = 0;
1162 static int complex_descriptors_next = 0;
1163 static MonoGCRootMarkFunc user_descriptors [MAX_USER_DESCRIPTORS];
1164 static int user_descriptors_next = 0;
1167 alloc_complex_descriptor (gsize *bitmap, int numbits)
1171 numbits = ALIGN_TO (numbits, GC_BITS_PER_WORD);
1172 nwords = numbits / GC_BITS_PER_WORD + 1;
1175 res = complex_descriptors_next;
1176 /* linear search, so we don't have duplicates with domain load/unload
1177 * this should not be performance critical or we'd have bigger issues
1178 * (the number and size of complex descriptors should be small).
1180 for (i = 0; i < complex_descriptors_next; ) {
1181 if (complex_descriptors [i] == nwords) {
1182 int j, found = TRUE;
1183 for (j = 0; j < nwords - 1; ++j) {
1184 if (complex_descriptors [i + 1 + j] != bitmap [j]) {
1194 i += complex_descriptors [i];
1196 if (complex_descriptors_next + nwords > complex_descriptors_size) {
1197 int new_size = complex_descriptors_size * 2 + nwords;
1198 complex_descriptors = g_realloc (complex_descriptors, new_size * sizeof (gsize));
1199 complex_descriptors_size = new_size;
1201 DEBUG (6, fprintf (gc_debug_file, "Complex descriptor %d, size: %d (total desc memory: %d)\n", res, nwords, complex_descriptors_size));
1202 complex_descriptors_next += nwords;
1203 complex_descriptors [res] = nwords;
1204 for (i = 0; i < nwords - 1; ++i) {
1205 complex_descriptors [res + 1 + i] = bitmap [i];
1206 DEBUG (6, fprintf (gc_debug_file, "\tvalue: %p\n", (void*)complex_descriptors [res + 1 + i]));
1213 * Descriptor builders.
1216 mono_gc_make_descr_for_string (gsize *bitmap, int numbits)
1218 return (void*) DESC_TYPE_RUN_LENGTH;
1222 mono_gc_make_descr_for_object (gsize *bitmap, int numbits, size_t obj_size)
1224 int first_set = -1, num_set = 0, last_set = -1, i;
1226 size_t stored_size = obj_size;
1227 for (i = 0; i < numbits; ++i) {
1228 if (bitmap [i / GC_BITS_PER_WORD] & ((gsize)1 << (i % GC_BITS_PER_WORD))) {
1236 * We don't encode the size of types that don't contain
1237 * references because they might not be aligned, i.e. the
1238 * bottom two bits might be set, which would clash with the
1239 * bits we need to encode the descriptor type. Since we don't
1240 * use the encoded size to skip objects, other than for
1241 * processing remsets, in which case only the positions of
1242 * references are relevant, this is not a problem.
1245 return DESC_TYPE_RUN_LENGTH;
1246 g_assert (!(stored_size & 0x3));
1247 if (stored_size <= MAX_SMALL_OBJ_SIZE) {
1248 /* check run-length encoding first: one byte offset, one byte number of pointers
1249 * on 64 bit archs, we can have 3 runs, just one on 32.
1250 * It may be better to use nibbles.
1252 if (first_set < 0) {
1253 desc = DESC_TYPE_RUN_LENGTH | (stored_size << 1);
1254 DEBUG (6, fprintf (gc_debug_file, "Ptrfree descriptor %p, size: %zd\n", (void*)desc, stored_size));
1255 return (void*) desc;
1256 } else if (first_set < 256 && num_set < 256 && (first_set + num_set == last_set + 1)) {
1257 desc = DESC_TYPE_RUN_LENGTH | (stored_size << 1) | (first_set << 16) | (num_set << 24);
1258 DEBUG (6, fprintf (gc_debug_file, "Runlen descriptor %p, size: %zd, first set: %d, num set: %d\n", (void*)desc, stored_size, first_set, num_set));
1259 return (void*) desc;
1261 /* we know the 2-word header is ptr-free */
1262 if (last_set < SMALL_BITMAP_SIZE + OBJECT_HEADER_WORDS) {
1263 desc = DESC_TYPE_SMALL_BITMAP | (stored_size << 1) | ((*bitmap >> OBJECT_HEADER_WORDS) << SMALL_BITMAP_SHIFT);
1264 DEBUG (6, fprintf (gc_debug_file, "Smallbitmap descriptor %p, size: %zd, last set: %d\n", (void*)desc, stored_size, last_set));
1265 return (void*) desc;
1268 /* we know the 2-word header is ptr-free */
1269 if (last_set < LARGE_BITMAP_SIZE + OBJECT_HEADER_WORDS) {
1270 desc = DESC_TYPE_LARGE_BITMAP | ((*bitmap >> OBJECT_HEADER_WORDS) << LOW_TYPE_BITS);
1271 DEBUG (6, fprintf (gc_debug_file, "Largebitmap descriptor %p, size: %zd, last set: %d\n", (void*)desc, stored_size, last_set));
1272 return (void*) desc;
1274 /* it's a complex object ... */
1275 desc = DESC_TYPE_COMPLEX | (alloc_complex_descriptor (bitmap, last_set + 1) << LOW_TYPE_BITS);
1276 return (void*) desc;
1279 /* If the array holds references, numbits == 1 and the first bit is set in elem_bitmap */
1281 mono_gc_make_descr_for_array (int vector, gsize *elem_bitmap, int numbits, size_t elem_size)
1283 int first_set = -1, num_set = 0, last_set = -1, i;
1284 mword desc = vector? DESC_TYPE_VECTOR: DESC_TYPE_ARRAY;
1285 for (i = 0; i < numbits; ++i) {
1286 if (elem_bitmap [i / GC_BITS_PER_WORD] & ((gsize)1 << (i % GC_BITS_PER_WORD))) {
1293 /* See comment at the definition of DESC_TYPE_RUN_LENGTH. */
1295 return DESC_TYPE_RUN_LENGTH;
1296 if (elem_size <= MAX_ELEMENT_SIZE) {
1297 desc |= elem_size << VECTOR_ELSIZE_SHIFT;
1299 return (void*)(desc | VECTOR_SUBTYPE_PTRFREE);
1301 /* Note: we also handle structs with just ref fields */
1302 if (num_set * sizeof (gpointer) == elem_size) {
1303 return (void*)(desc | VECTOR_SUBTYPE_REFS | ((gssize)(-1) << 16));
1305 /* FIXME: try run-len first */
1306 /* Note: we can't skip the object header here, because it's not present */
1307 if (last_set <= SMALL_BITMAP_SIZE) {
1308 return (void*)(desc | VECTOR_SUBTYPE_BITMAP | (*elem_bitmap << 16));
1311 /* it's am array of complex structs ... */
1312 desc = DESC_TYPE_COMPLEX_ARR;
1313 desc |= alloc_complex_descriptor (elem_bitmap, last_set + 1) << LOW_TYPE_BITS;
1314 return (void*) desc;
1317 /* Return the bitmap encoded by a descriptor */
1319 mono_gc_get_bitmap_for_descr (void *descr, int *numbits)
1321 mword d = (mword)descr;
1325 case DESC_TYPE_RUN_LENGTH: {
1326 int first_set = (d >> 16) & 0xff;
1327 int num_set = (d >> 24) & 0xff;
1330 bitmap = g_new0 (gsize, (first_set + num_set + 7) / 8);
1332 for (i = first_set; i < first_set + num_set; ++i)
1333 bitmap [i / GC_BITS_PER_WORD] |= ((gsize)1 << (i % GC_BITS_PER_WORD));
1335 *numbits = first_set + num_set;
1339 case DESC_TYPE_SMALL_BITMAP:
1340 bitmap = g_new0 (gsize, 1);
1342 bitmap [0] = (d >> SMALL_BITMAP_SHIFT) << OBJECT_HEADER_WORDS;
1344 *numbits = GC_BITS_PER_WORD;
1348 g_assert_not_reached ();
1352 /* helper macros to scan and traverse objects, macros because we resue them in many functions */
1353 #define STRING_SIZE(size,str) do { \
1354 (size) = sizeof (MonoString) + 2 * mono_string_length_fast ((MonoString*)(str)) + 2; \
1355 (size) += (ALLOC_ALIGN - 1); \
1356 (size) &= ~(ALLOC_ALIGN - 1); \
1359 #define OBJ_RUN_LEN_SIZE(size,desc,obj) do { \
1360 (size) = ((desc) & 0xfff8) >> 1; \
1363 #define OBJ_BITMAP_SIZE(size,desc,obj) do { \
1364 (size) = ((desc) & 0xfff8) >> 1; \
1367 //#define PREFETCH(addr) __asm__ __volatile__ (" prefetchnta %0": : "m"(*(char *)(addr)))
1368 #define PREFETCH(addr)
1370 /* code using these macros must define a HANDLE_PTR(ptr) macro that does the work */
1371 #define OBJ_RUN_LEN_FOREACH_PTR(desc,obj) do { \
1372 if ((desc) & 0xffff0000) { \
1373 /* there are pointers */ \
1374 void **_objptr_end; \
1375 void **_objptr = (void**)(obj); \
1376 _objptr += ((desc) >> 16) & 0xff; \
1377 _objptr_end = _objptr + (((desc) >> 24) & 0xff); \
1378 while (_objptr < _objptr_end) { \
1379 HANDLE_PTR (_objptr, (obj)); \
1385 /* a bitmap desc means that there are pointer references or we'd have
1386 * choosen run-length, instead: add an assert to check.
1388 #define OBJ_BITMAP_FOREACH_PTR(desc,obj) do { \
1389 /* there are pointers */ \
1390 void **_objptr = (void**)(obj); \
1391 gsize _bmap = (desc) >> 16; \
1392 _objptr += OBJECT_HEADER_WORDS; \
1394 if ((_bmap & 1)) { \
1395 HANDLE_PTR (_objptr, (obj)); \
1402 #define OBJ_LARGE_BITMAP_FOREACH_PTR(vt,obj) do { \
1403 /* there are pointers */ \
1404 void **_objptr = (void**)(obj); \
1405 gsize _bmap = (vt)->desc >> LOW_TYPE_BITS; \
1406 _objptr += OBJECT_HEADER_WORDS; \
1408 if ((_bmap & 1)) { \
1409 HANDLE_PTR (_objptr, (obj)); \
1416 #define OBJ_COMPLEX_FOREACH_PTR(vt,obj) do { \
1417 /* there are pointers */ \
1418 void **_objptr = (void**)(obj); \
1419 gsize *bitmap_data = complex_descriptors + ((vt)->desc >> LOW_TYPE_BITS); \
1420 int bwords = (*bitmap_data) - 1; \
1421 void **start_run = _objptr; \
1424 MonoObject *myobj = (MonoObject*)obj; \
1425 g_print ("found %d at %p (0x%zx): %s.%s\n", bwords, (obj), (vt)->desc, myobj->vtable->klass->name_space, myobj->vtable->klass->name); \
1427 while (bwords-- > 0) { \
1428 gsize _bmap = *bitmap_data++; \
1429 _objptr = start_run; \
1430 /*g_print ("bitmap: 0x%x/%d at %p\n", _bmap, bwords, _objptr);*/ \
1432 if ((_bmap & 1)) { \
1433 HANDLE_PTR (_objptr, (obj)); \
1438 start_run += GC_BITS_PER_WORD; \
1442 /* this one is untested */
1443 #define OBJ_COMPLEX_ARR_FOREACH_PTR(vt,obj) do { \
1444 /* there are pointers */ \
1445 gsize *mbitmap_data = complex_descriptors + ((vt)->desc >> LOW_TYPE_BITS); \
1446 int mbwords = (*mbitmap_data++) - 1; \
1447 int el_size = mono_array_element_size (((MonoObject*)(obj))->vtable->klass); \
1448 char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
1449 char *e_end = e_start + el_size * mono_array_length_fast ((MonoArray*)(obj)); \
1451 MonoObject *myobj = (MonoObject*)start; \
1452 g_print ("found %d at %p (0x%zx): %s.%s\n", mbwords, (obj), (vt)->desc, myobj->vtable->klass->name_space, myobj->vtable->klass->name); \
1454 while (e_start < e_end) { \
1455 void **_objptr = (void**)e_start; \
1456 gsize *bitmap_data = mbitmap_data; \
1457 unsigned int bwords = mbwords; \
1458 while (bwords-- > 0) { \
1459 gsize _bmap = *bitmap_data++; \
1460 void **start_run = _objptr; \
1461 /*g_print ("bitmap: 0x%x\n", _bmap);*/ \
1463 if ((_bmap & 1)) { \
1464 HANDLE_PTR (_objptr, (obj)); \
1469 _objptr = start_run + GC_BITS_PER_WORD; \
1471 e_start += el_size; \
1475 #define OBJ_VECTOR_FOREACH_PTR(vt,obj) do { \
1476 /* note: 0xffffc000 excludes DESC_TYPE_V_PTRFREE */ \
1477 if ((vt)->desc & 0xffffc000) { \
1478 int el_size = ((vt)->desc >> 3) & MAX_ELEMENT_SIZE; \
1479 /* there are pointers */ \
1480 int etype = (vt)->desc & 0xc000; \
1481 if (etype == (DESC_TYPE_V_REFS << 14)) { \
1482 void **p = (void**)((char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector)); \
1483 void **end_refs = (void**)((char*)p + el_size * mono_array_length_fast ((MonoArray*)(obj))); \
1484 /* Note: this code can handle also arrays of struct with only references in them */ \
1485 while (p < end_refs) { \
1486 HANDLE_PTR (p, (obj)); \
1489 } else if (etype == DESC_TYPE_V_RUN_LEN << 14) { \
1490 int offset = ((vt)->desc >> 16) & 0xff; \
1491 int num_refs = ((vt)->desc >> 24) & 0xff; \
1492 char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
1493 char *e_end = e_start + el_size * mono_array_length_fast ((MonoArray*)(obj)); \
1494 while (e_start < e_end) { \
1495 void **p = (void**)e_start; \
1498 for (i = 0; i < num_refs; ++i) { \
1499 HANDLE_PTR (p + i, (obj)); \
1501 e_start += el_size; \
1503 } else if (etype == DESC_TYPE_V_BITMAP << 14) { \
1504 char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
1505 char *e_end = e_start + el_size * mono_array_length_fast ((MonoArray*)(obj)); \
1506 while (e_start < e_end) { \
1507 void **p = (void**)e_start; \
1508 gsize _bmap = (vt)->desc >> 16; \
1509 /* Note: there is no object header here to skip */ \
1511 if ((_bmap & 1)) { \
1512 HANDLE_PTR (p, (obj)); \
1517 e_start += el_size; \
1523 #include "sgen-major-copying.c"
1524 //#include "sgen-marksweep.c"
1527 is_xdomain_ref_allowed (gpointer *ptr, char *obj, MonoDomain *domain)
1529 MonoObject *o = (MonoObject*)(obj);
1530 MonoObject *ref = (MonoObject*)*(ptr);
1531 int offset = (char*)(ptr) - (char*)o;
1533 if (o->vtable->klass == mono_defaults.thread_class && offset == G_STRUCT_OFFSET (MonoThread, internal_thread))
1535 if (o->vtable->klass == mono_defaults.internal_thread_class && offset == G_STRUCT_OFFSET (MonoInternalThread, current_appcontext))
1537 if (mono_class_has_parent (o->vtable->klass, mono_defaults.real_proxy_class) &&
1538 offset == G_STRUCT_OFFSET (MonoRealProxy, unwrapped_server))
1540 /* Thread.cached_culture_info */
1541 if (!strcmp (ref->vtable->klass->name_space, "System.Globalization") &&
1542 !strcmp (ref->vtable->klass->name, "CultureInfo") &&
1543 !strcmp(o->vtable->klass->name_space, "System") &&
1544 !strcmp(o->vtable->klass->name, "Object[]"))
1547 * at System.IO.MemoryStream.InternalConstructor (byte[],int,int,bool,bool) [0x0004d] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.IO/MemoryStream.cs:121
1548 * at System.IO.MemoryStream..ctor (byte[]) [0x00017] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.IO/MemoryStream.cs:81
1549 * at (wrapper remoting-invoke-with-check) System.IO.MemoryStream..ctor (byte[]) <IL 0x00020, 0xffffffff>
1550 * at System.Runtime.Remoting.Messaging.CADMethodCallMessage.GetArguments () [0x0000d] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Messaging/CADMessages.cs:327
1551 * at System.Runtime.Remoting.Messaging.MethodCall..ctor (System.Runtime.Remoting.Messaging.CADMethodCallMessage) [0x00017] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Messaging/MethodCall.cs:87
1552 * at System.AppDomain.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage,byte[]&,System.Runtime.Remoting.Messaging.CADMethodReturnMessage&) [0x00018] in /home/schani/Work/novell/trunk/mcs/class/corlib/System/AppDomain.cs:1213
1553 * at (wrapper remoting-invoke-with-check) System.AppDomain.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage,byte[]&,System.Runtime.Remoting.Messaging.CADMethodReturnMessage&) <IL 0x0003d, 0xffffffff>
1554 * at System.Runtime.Remoting.Channels.CrossAppDomainSink.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage) [0x00008] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Channels/CrossAppDomainChannel.cs:198
1555 * at (wrapper runtime-invoke) object.runtime_invoke_CrossAppDomainSink/ProcessMessageRes_object_object (object,intptr,intptr,intptr) <IL 0x0004c, 0xffffffff>
1557 if (!strcmp (ref->vtable->klass->name_space, "System") &&
1558 !strcmp (ref->vtable->klass->name, "Byte[]") &&
1559 !strcmp (o->vtable->klass->name_space, "System.IO") &&
1560 !strcmp (o->vtable->klass->name, "MemoryStream"))
1562 /* append_job() in threadpool.c */
1563 if (!strcmp (ref->vtable->klass->name_space, "System.Runtime.Remoting.Messaging") &&
1564 !strcmp (ref->vtable->klass->name, "AsyncResult") &&
1565 !strcmp (o->vtable->klass->name_space, "System") &&
1566 !strcmp (o->vtable->klass->name, "Object[]") &&
1567 mono_thread_pool_is_queue_array ((MonoArray*) o))
1573 check_reference_for_xdomain (gpointer *ptr, char *obj, MonoDomain *domain)
1575 MonoObject *o = (MonoObject*)(obj);
1576 MonoObject *ref = (MonoObject*)*(ptr);
1577 int offset = (char*)(ptr) - (char*)o;
1579 MonoClassField *field;
1582 if (!ref || ref->vtable->domain == domain)
1584 if (is_xdomain_ref_allowed (ptr, obj, domain))
1588 for (class = o->vtable->klass; class; class = class->parent) {
1591 for (i = 0; i < class->field.count; ++i) {
1592 if (class->fields[i].offset == offset) {
1593 field = &class->fields[i];
1601 if (ref->vtable->klass == mono_defaults.string_class)
1602 str = mono_string_to_utf8 ((MonoString*)ref);
1605 g_print ("xdomain reference in %p (%s.%s) at offset %d (%s) to %p (%s.%s) (%s) - pointed to by:\n",
1606 o, o->vtable->klass->name_space, o->vtable->klass->name,
1607 offset, field ? field->name : "",
1608 ref, ref->vtable->klass->name_space, ref->vtable->klass->name, str ? str : "");
1609 mono_gc_scan_for_specific_ref (o);
1615 #define HANDLE_PTR(ptr,obj) check_reference_for_xdomain ((ptr), (obj), domain)
1618 scan_object_for_xdomain_refs (char *start, mword size, void *data)
1620 MonoDomain *domain = ((MonoObject*)start)->vtable->domain;
1622 #include "sgen-scan-object.h"
1626 #define HANDLE_PTR(ptr,obj) do { \
1627 if ((MonoObject*)*(ptr) == key) { \
1628 g_print ("found ref to %p in object %p (%s) at offset %zd\n", \
1629 key, (obj), safe_name ((obj)), ((char*)(ptr) - (char*)(obj))); \
1634 scan_object_for_specific_ref (char *start, MonoObject *key)
1636 #include "sgen-scan-object.h"
1640 scan_area_with_callback (char *start, char *end, IterateObjectCallbackFunc callback, void *data)
1642 while (start < end) {
1644 if (!*(void**)start) {
1645 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
1649 size = safe_object_get_size ((MonoObject*) start);
1650 size += ALLOC_ALIGN - 1;
1651 size &= ~(ALLOC_ALIGN - 1);
1653 callback (start, size, data);
1660 scan_object_for_specific_ref_callback (char *obj, size_t size, MonoObject *key)
1662 scan_object_for_specific_ref (obj, key);
1666 check_root_obj_specific_ref (RootRecord *root, MonoObject *key, MonoObject *obj)
1670 g_print ("found ref to %p in root record %p\n", key, root);
1673 static MonoObject *check_key = NULL;
1674 static RootRecord *check_root = NULL;
1677 check_root_obj_specific_ref_from_marker (void **obj)
1679 check_root_obj_specific_ref (check_root, check_key, *obj);
1683 scan_roots_for_specific_ref (MonoObject *key, int root_type)
1688 for (i = 0; i < roots_hash_size [root_type]; ++i) {
1689 for (root = roots_hash [root_type][i]; root; root = root->next) {
1690 void **start_root = (void**)root->start_root;
1691 mword desc = root->root_desc;
1695 switch (desc & ROOT_DESC_TYPE_MASK) {
1696 case ROOT_DESC_BITMAP:
1697 desc >>= ROOT_DESC_TYPE_SHIFT;
1700 check_root_obj_specific_ref (root, key, *start_root);
1705 case ROOT_DESC_COMPLEX: {
1706 gsize *bitmap_data = complex_descriptors + (desc >> ROOT_DESC_TYPE_SHIFT);
1707 int bwords = (*bitmap_data) - 1;
1708 void **start_run = start_root;
1710 while (bwords-- > 0) {
1711 gsize bmap = *bitmap_data++;
1712 void **objptr = start_run;
1715 check_root_obj_specific_ref (root, key, *objptr);
1719 start_run += GC_BITS_PER_WORD;
1723 case ROOT_DESC_USER: {
1724 MonoGCRootMarkFunc marker = user_descriptors [desc >> ROOT_DESC_TYPE_SHIFT];
1725 marker (start_root, check_root_obj_specific_ref_from_marker);
1728 case ROOT_DESC_RUN_LEN:
1729 g_assert_not_reached ();
1731 g_assert_not_reached ();
1740 mono_gc_scan_for_specific_ref (MonoObject *key)
1746 scan_area_with_callback (nursery_section->data, nursery_section->end_data,
1747 (IterateObjectCallbackFunc)scan_object_for_specific_ref_callback, key);
1749 major_iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)scan_object_for_specific_ref_callback, key);
1751 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1752 scan_object_for_specific_ref (bigobj->data, key);
1754 scan_roots_for_specific_ref (key, ROOT_TYPE_NORMAL);
1755 scan_roots_for_specific_ref (key, ROOT_TYPE_WBARRIER);
1757 for (i = 0; i < roots_hash_size [ROOT_TYPE_PINNED]; ++i) {
1758 for (root = roots_hash [ROOT_TYPE_PINNED][i]; root; root = root->next) {
1759 void **ptr = (void**)root->start_root;
1761 while (ptr < (void**)root->end_root) {
1762 check_root_obj_specific_ref (root, *ptr, key);
1769 /* Clear all remaining nursery fragments */
1771 clear_nursery_fragments (char *next)
1774 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION) {
1775 g_assert (next <= nursery_frag_real_end);
1776 memset (next, 0, nursery_frag_real_end - next);
1777 for (frag = nursery_fragments; frag; frag = frag->next) {
1778 memset (frag->fragment_start, 0, frag->fragment_end - frag->fragment_start);
1784 need_remove_object_for_domain (char *start, MonoDomain *domain)
1786 if (mono_object_domain (start) == domain) {
1787 DEBUG (4, fprintf (gc_debug_file, "Need to cleanup object %p\n", start));
1788 binary_protocol_cleanup (start, (gpointer)LOAD_VTABLE (start), safe_object_get_size ((MonoObject*)start));
1795 process_object_for_domain_clearing (char *start, MonoDomain *domain)
1797 GCVTable *vt = (GCVTable*)LOAD_VTABLE (start);
1798 if (vt->klass == mono_defaults.internal_thread_class)
1799 g_assert (mono_object_domain (start) == mono_get_root_domain ());
1800 /* The object could be a proxy for an object in the domain
1802 if (mono_class_has_parent (vt->klass, mono_defaults.real_proxy_class)) {
1803 MonoObject *server = ((MonoRealProxy*)start)->unwrapped_server;
1805 /* The server could already have been zeroed out, so
1806 we need to check for that, too. */
1807 if (server && (!LOAD_VTABLE (server) || mono_object_domain (server) == domain)) {
1808 DEBUG (4, fprintf (gc_debug_file, "Cleaning up remote pointer in %p to object %p\n",
1810 ((MonoRealProxy*)start)->unwrapped_server = NULL;
1815 static MonoDomain *check_domain = NULL;
1818 check_obj_not_in_domain (void **o)
1820 g_assert (((MonoObject*)(*o))->vtable->domain != check_domain);
1824 scan_for_registered_roots_in_domain (MonoDomain *domain, int root_type)
1828 check_domain = domain;
1829 for (i = 0; i < roots_hash_size [root_type]; ++i) {
1830 for (root = roots_hash [root_type][i]; root; root = root->next) {
1831 void **start_root = (void**)root->start_root;
1832 mword desc = root->root_desc;
1834 /* The MonoDomain struct is allowed to hold
1835 references to objects in its own domain. */
1836 if (start_root == (void**)domain)
1839 switch (desc & ROOT_DESC_TYPE_MASK) {
1840 case ROOT_DESC_BITMAP:
1841 desc >>= ROOT_DESC_TYPE_SHIFT;
1843 if ((desc & 1) && *start_root)
1844 check_obj_not_in_domain (*start_root);
1849 case ROOT_DESC_COMPLEX: {
1850 gsize *bitmap_data = complex_descriptors + (desc >> ROOT_DESC_TYPE_SHIFT);
1851 int bwords = (*bitmap_data) - 1;
1852 void **start_run = start_root;
1854 while (bwords-- > 0) {
1855 gsize bmap = *bitmap_data++;
1856 void **objptr = start_run;
1858 if ((bmap & 1) && *objptr)
1859 check_obj_not_in_domain (*objptr);
1863 start_run += GC_BITS_PER_WORD;
1867 case ROOT_DESC_USER: {
1868 MonoGCRootMarkFunc marker = user_descriptors [desc >> ROOT_DESC_TYPE_SHIFT];
1869 marker (start_root, check_obj_not_in_domain);
1872 case ROOT_DESC_RUN_LEN:
1873 g_assert_not_reached ();
1875 g_assert_not_reached ();
1879 check_domain = NULL;
1883 check_for_xdomain_refs (void)
1887 scan_area_with_callback (nursery_section->data, nursery_section->end_data, scan_object_for_xdomain_refs, NULL);
1889 major_iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)scan_object_for_xdomain_refs, NULL);
1891 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1892 scan_object_for_xdomain_refs (bigobj->data, bigobj->size, NULL);
1896 clear_domain_process_object (char *obj, MonoDomain *domain)
1900 process_object_for_domain_clearing (obj, domain);
1901 remove = need_remove_object_for_domain (obj, domain);
1903 if (remove && ((MonoObject*)obj)->synchronisation) {
1904 void **dislink = mono_monitor_get_object_monitor_weak_link ((MonoObject*)obj);
1906 mono_gc_register_disappearing_link (NULL, dislink, FALSE);
1913 clear_domain_process_minor_object_callback (char *obj, size_t size, MonoDomain *domain)
1915 if (clear_domain_process_object (obj, domain))
1916 memset (obj, 0, size);
1920 clear_domain_process_major_object_callback (char *obj, size_t size, MonoDomain *domain)
1922 clear_domain_process_object (obj, domain);
1926 clear_domain_free_major_non_pinned_object_callback (char *obj, size_t size, MonoDomain *domain)
1928 if (need_remove_object_for_domain (obj, domain))
1929 major_free_non_pinned_object (obj, size);
1933 clear_domain_free_major_pinned_object_callback (char *obj, size_t size, MonoDomain *domain)
1935 if (need_remove_object_for_domain (obj, domain))
1936 free_pinned_object (obj, size);
1940 * When appdomains are unloaded we can easily remove objects that have finalizers,
1941 * but all the others could still be present in random places on the heap.
1942 * We need a sweep to get rid of them even though it's going to be costly
1944 * The reason we need to remove them is because we access the vtable and class
1945 * structures to know the object size and the reference bitmap: once the domain is
1946 * unloaded the point to random memory.
1949 mono_gc_clear_domain (MonoDomain * domain)
1951 LOSObject *bigobj, *prev;
1956 clear_nursery_fragments (nursery_next);
1958 if (xdomain_checks && domain != mono_get_root_domain ()) {
1959 scan_for_registered_roots_in_domain (domain, ROOT_TYPE_NORMAL);
1960 scan_for_registered_roots_in_domain (domain, ROOT_TYPE_WBARRIER);
1961 check_for_xdomain_refs ();
1964 scan_area_with_callback (nursery_section->data, nursery_section->end_data,
1965 (IterateObjectCallbackFunc)clear_domain_process_minor_object_callback, domain);
1967 /*Ephemerons and dislinks must be processed before LOS since they might end up pointing
1968 to memory returned to the OS.*/
1969 null_ephemerons_for_domain (domain);
1971 for (i = GENERATION_NURSERY; i < GENERATION_MAX; ++i)
1972 null_links_for_domain (domain, i);
1974 /* We need two passes over major and large objects because
1975 freeing such objects might give their memory back to the OS
1976 (in the case of large objects) or obliterate its vtable
1977 (pinned objects with major-copying or pinned and non-pinned
1978 objects with major-mark&sweep), but we might need to
1979 dereference a pointer from an object to another object if
1980 the first object is a proxy. */
1981 major_iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)clear_domain_process_major_object_callback, domain);
1982 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1983 clear_domain_process_object (bigobj->data, domain);
1986 for (bigobj = los_object_list; bigobj;) {
1987 if (need_remove_object_for_domain (bigobj->data, domain)) {
1988 LOSObject *to_free = bigobj;
1990 prev->next = bigobj->next;
1992 los_object_list = bigobj->next;
1993 bigobj = bigobj->next;
1994 DEBUG (4, fprintf (gc_debug_file, "Freeing large object %p\n",
1996 free_large_object (to_free);
2000 bigobj = bigobj->next;
2002 major_iterate_objects (TRUE, FALSE, (IterateObjectCallbackFunc)clear_domain_free_major_non_pinned_object_callback, domain);
2003 major_iterate_objects (FALSE, TRUE, (IterateObjectCallbackFunc)clear_domain_free_major_pinned_object_callback, domain);
2009 global_remset_cache_clear (void)
2011 memset (global_remset_cache, 0, sizeof (global_remset_cache));
2015 * Tries to check if a given remset location was already added to the global remset.
2018 * A 2 entry, LRU cache of recently saw location remsets.
2020 * It's hand-coded instead of done using loops to reduce the number of memory references on cache hit.
2022 * Returns TRUE is the element was added..
2025 global_remset_location_was_not_added (gpointer ptr)
2028 gpointer first = global_remset_cache [0], second;
2030 HEAVY_STAT (++stat_global_remsets_discarded);
2034 second = global_remset_cache [1];
2036 if (second == ptr) {
2037 /*Move the second to the front*/
2038 global_remset_cache [0] = second;
2039 global_remset_cache [1] = first;
2041 HEAVY_STAT (++stat_global_remsets_discarded);
2045 global_remset_cache [0] = second;
2046 global_remset_cache [1] = ptr;
2051 * add_to_global_remset:
2053 * The global remset contains locations which point into newspace after
2054 * a minor collection. This can happen if the objects they point to are pinned.
2057 add_to_global_remset (gpointer ptr)
2061 g_assert (!ptr_in_nursery (ptr) && ptr_in_nursery (*(gpointer*)ptr));
2063 if (!global_remset_location_was_not_added (ptr))
2066 DEBUG (8, fprintf (gc_debug_file, "Adding global remset for %p\n", ptr));
2067 binary_protocol_global_remset (ptr, *(gpointer*)ptr, (gpointer)LOAD_VTABLE (*(gpointer*)ptr));
2069 HEAVY_STAT (++stat_global_remsets_added);
2072 * FIXME: If an object remains pinned, we need to add it at every minor collection.
2073 * To avoid uncontrolled growth of the global remset, only add each pointer once.
2075 if (global_remset->store_next + 3 < global_remset->end_set) {
2076 *(global_remset->store_next++) = (mword)ptr;
2079 rs = alloc_remset (global_remset->end_set - global_remset->data, NULL);
2080 rs->next = global_remset;
2082 *(global_remset->store_next++) = (mword)ptr;
2085 int global_rs_size = 0;
2087 for (rs = global_remset; rs; rs = rs->next) {
2088 global_rs_size += rs->store_next - rs->data;
2090 DEBUG (4, fprintf (gc_debug_file, "Global remset now has size %d\n", global_rs_size));
2095 * FIXME: allocate before calling this function and pass the
2096 * destination address.
2099 copy_object_no_checks (void *obj)
2101 static const void *copy_labels [] = { &&LAB_0, &&LAB_1, &&LAB_2, &&LAB_3, &&LAB_4, &&LAB_5, &&LAB_6, &&LAB_7, &&LAB_8 };
2105 MonoVTable *vt = ((MonoObject*)obj)->vtable;
2106 gboolean has_references = vt->gc_descr != DESC_TYPE_RUN_LENGTH;
2108 objsize = safe_object_get_size ((MonoObject*)obj);
2109 objsize += ALLOC_ALIGN - 1;
2110 objsize &= ~(ALLOC_ALIGN - 1);
2112 DEBUG (9, g_assert (vt->klass->inited));
2113 MAJOR_GET_COPY_OBJECT_SPACE (destination, objsize, has_references);
2115 DEBUG (9, fprintf (gc_debug_file, " (to %p, %s size: %zd)\n", destination, ((MonoObject*)obj)->vtable->klass->name, objsize));
2116 binary_protocol_copy (obj, destination, ((MonoObject*)obj)->vtable, objsize);
2118 if (objsize <= sizeof (gpointer) * 8) {
2119 mword *dest = (mword*)destination;
2120 goto *copy_labels [objsize / sizeof (gpointer)];
2122 (dest) [7] = ((mword*)obj) [7];
2124 (dest) [6] = ((mword*)obj) [6];
2126 (dest) [5] = ((mword*)obj) [5];
2128 (dest) [4] = ((mword*)obj) [4];
2130 (dest) [3] = ((mword*)obj) [3];
2132 (dest) [2] = ((mword*)obj) [2];
2134 (dest) [1] = ((mword*)obj) [1];
2136 (dest) [0] = ((mword*)obj) [0];
2144 char* edi = destination;
2145 __asm__ __volatile__(
2147 : "=&c" (ecx), "=&D" (edi), "=&S" (esi)
2148 : "0" (objsize/4), "1" (edi),"2" (esi)
2153 memcpy (destination, obj, objsize);
2156 /* adjust array->bounds */
2157 DEBUG (9, g_assert (vt->gc_descr));
2158 if (G_UNLIKELY (vt->rank && ((MonoArray*)obj)->bounds)) {
2159 MonoArray *array = (MonoArray*)destination;
2160 array->bounds = (MonoArrayBounds*)((char*)destination + ((char*)((MonoArray*)obj)->bounds - (char*)obj));
2161 DEBUG (9, fprintf (gc_debug_file, "Array instance %p: size: %zd, rank: %d, length: %d\n", array, objsize, vt->rank, mono_array_length (array)));
2163 /* set the forwarding pointer */
2164 forward_object (obj, destination);
2165 if (G_UNLIKELY (mono_profiler_events & MONO_PROFILE_GC_MOVES)) {
2166 if (moved_objects_idx == MOVED_OBJECTS_NUM) {
2167 mono_profiler_gc_moves (moved_objects, moved_objects_idx);
2168 moved_objects_idx = 0;
2170 moved_objects [moved_objects_idx++] = obj;
2171 moved_objects [moved_objects_idx++] = destination;
2174 if (has_references) {
2175 DEBUG (9, fprintf (gc_debug_file, "Enqueuing gray object %p (%s)\n", obj, safe_name (obj)));
2176 GRAY_OBJECT_ENQUEUE (obj);
2182 * This is how the copying happens from the nursery to the old generation.
2183 * We assume that at this time all the pinned objects have been identified and
2185 * We run scan_object() for each pinned object so that each referenced
2186 * objects if possible are copied. The new gray objects created can have
2187 * scan_object() run on them right away, too.
2188 * Then we run copy_object() for the precisely tracked roots. At this point
2189 * all the roots are either gray or black. We run scan_object() on the gray
2190 * objects until no more gray objects are created.
2191 * At the end of the process we walk again the pinned list and we unmark
2192 * the pinned flag. As we go we also create the list of free space for use
2193 * in the next allocation runs.
2195 * We need to remember objects from the old generation that point to the new one
2196 * (or just addresses?).
2198 * copy_object could be made into a macro once debugged (use inline for now).
2201 static void __attribute__((noinline))
2202 copy_object (void **obj_slot)
2205 char *obj = *obj_slot;
2207 DEBUG (9, g_assert (current_collection_generation == GENERATION_NURSERY));
2209 HEAVY_STAT (++stat_copy_object_called_nursery);
2211 if (!ptr_in_nursery (obj)) {
2212 HEAVY_STAT (++stat_nursery_copy_object_failed_from_space);
2216 DEBUG (9, fprintf (gc_debug_file, "Precise copy of %p from %p", obj, obj_slot));
2219 * Before we can copy the object we must make sure that we are
2220 * allowed to, i.e. that the object not pinned or not already
2224 if ((forwarded = object_is_forwarded (obj))) {
2225 DEBUG (9, g_assert (((MonoVTable*)LOAD_VTABLE(obj))->gc_descr));
2226 DEBUG (9, fprintf (gc_debug_file, " (already forwarded to %p)\n", forwarded));
2227 HEAVY_STAT (++stat_nursery_copy_object_failed_forwarded);
2228 *obj_slot = forwarded;
2231 if (object_is_pinned (obj)) {
2232 DEBUG (9, g_assert (((MonoVTable*)LOAD_VTABLE(obj))->gc_descr));
2233 DEBUG (9, fprintf (gc_debug_file, " (pinned, no change)\n"));
2234 HEAVY_STAT (++stat_nursery_copy_object_failed_pinned);
2238 HEAVY_STAT (++stat_objects_copied_nursery);
2240 *obj_slot = copy_object_no_checks (obj);
2244 #define HANDLE_PTR(ptr,obj) do { \
2245 void *__old = *(ptr); \
2248 copy_object ((ptr)); \
2250 DEBUG (9, if (__old != __copy) fprintf (gc_debug_file, "Overwrote field at %p with %p (was: %p)\n", (ptr), *(ptr), __old)); \
2251 if (G_UNLIKELY (ptr_in_nursery (__copy) && !ptr_in_nursery ((ptr)))) \
2252 add_to_global_remset ((ptr)); \
2257 * Scan the object pointed to by @start for references to
2258 * other objects between @from_start and @from_end and copy
2259 * them to the gray_objects area.
2262 scan_object (char *start)
2264 #include "sgen-scan-object.h"
2266 HEAVY_STAT (++stat_scan_object_called_nursery);
2272 * Scan the valuetype pointed to by START, described by DESC for references to
2273 * other objects between @from_start and @from_end and copy them to the gray_objects area.
2274 * Returns a pointer to the end of the object.
2277 scan_vtype (char *start, mword desc, char* from_start, char* from_end)
2281 /* The descriptors include info about the MonoObject header as well */
2282 start -= sizeof (MonoObject);
2284 switch (desc & 0x7) {
2285 case DESC_TYPE_RUN_LENGTH:
2286 OBJ_RUN_LEN_FOREACH_PTR (desc,start);
2287 OBJ_RUN_LEN_SIZE (skip_size, desc, start);
2288 g_assert (skip_size);
2289 return start + skip_size;
2290 case DESC_TYPE_SMALL_BITMAP:
2291 OBJ_BITMAP_FOREACH_PTR (desc,start);
2292 OBJ_BITMAP_SIZE (skip_size, desc, start);
2293 return start + skip_size;
2294 case DESC_TYPE_LARGE_BITMAP:
2295 case DESC_TYPE_COMPLEX:
2297 g_assert_not_reached ();
2300 // The other descriptors can't happen with vtypes
2301 g_assert_not_reached ();
2308 #define HANDLE_PTR(ptr,obj) do { \
2309 void *__old = *(ptr); \
2312 major_copy_or_mark_object ((ptr)); \
2314 DEBUG (9, if (__old != __copy) fprintf (gc_debug_file, "Overwrote field at %p with %p (was: %p)\n", (ptr), *(ptr), __old)); \
2315 if (G_UNLIKELY (ptr_in_nursery (__copy) && !ptr_in_nursery ((ptr)))) \
2316 add_to_global_remset ((ptr)); \
2321 major_scan_object (char *start)
2323 #include "sgen-scan-object.h"
2325 HEAVY_STAT (++stat_scan_object_called_major);
2331 * Scan objects in the gray stack until the stack is empty. This should be called
2332 * frequently after each object is copied, to achieve better locality and cache
2336 drain_gray_stack (void)
2340 if (current_collection_generation == GENERATION_NURSERY) {
2342 GRAY_OBJECT_DEQUEUE (obj);
2345 DEBUG (9, fprintf (gc_debug_file, "Precise gray object scan %p (%s)\n", obj, safe_name (obj)));
2350 GRAY_OBJECT_DEQUEUE (obj);
2353 DEBUG (9, fprintf (gc_debug_file, "Precise gray object scan %p (%s)\n", obj, safe_name (obj)));
2354 major_scan_object (obj);
2360 * Addresses from start to end are already sorted. This function finds
2361 * the object header for each address and pins the object. The
2362 * addresses must be inside the passed section. The (start of the)
2363 * address array is overwritten with the addresses of the actually
2364 * pinned objects. Return the number of pinned objects.
2367 pin_objects_from_addresses (GCMemSection *section, void **start, void **end, void *start_nursery, void *end_nursery)
2372 void *last_obj = NULL;
2373 size_t last_obj_size = 0;
2376 void **definitely_pinned = start;
2377 while (start < end) {
2379 /* the range check should be reduntant */
2380 if (addr != last && addr >= start_nursery && addr < end_nursery) {
2381 DEBUG (5, fprintf (gc_debug_file, "Considering pinning addr %p\n", addr));
2382 /* multiple pointers to the same object */
2383 if (addr >= last_obj && (char*)addr < (char*)last_obj + last_obj_size) {
2387 idx = ((char*)addr - (char*)section->data) / SCAN_START_SIZE;
2388 g_assert (idx < section->num_scan_start);
2389 search_start = (void*)section->scan_starts [idx];
2390 if (!search_start || search_start > addr) {
2393 search_start = section->scan_starts [idx];
2394 if (search_start && search_start <= addr)
2397 if (!search_start || search_start > addr)
2398 search_start = start_nursery;
2400 if (search_start < last_obj)
2401 search_start = (char*)last_obj + last_obj_size;
2402 /* now addr should be in an object a short distance from search_start
2403 * Note that search_start must point to zeroed mem or point to an object.
2406 if (!*(void**)search_start) {
2407 mword p = (mword)search_start;
2408 p += sizeof (gpointer);
2409 p += ALLOC_ALIGN - 1;
2410 p &= ~(ALLOC_ALIGN - 1);
2411 search_start = (void*)p;
2414 last_obj = search_start;
2415 last_obj_size = safe_object_get_size ((MonoObject*)search_start);
2416 last_obj_size += ALLOC_ALIGN - 1;
2417 last_obj_size &= ~(ALLOC_ALIGN - 1);
2418 DEBUG (8, fprintf (gc_debug_file, "Pinned try match %p (%s), size %zd\n", last_obj, safe_name (last_obj), last_obj_size));
2419 if (addr >= search_start && (char*)addr < (char*)last_obj + last_obj_size) {
2420 DEBUG (4, fprintf (gc_debug_file, "Pinned object %p, vtable %p (%s), count %d\n", search_start, *(void**)search_start, safe_name (search_start), count));
2421 binary_protocol_pin (search_start, (gpointer)LOAD_VTABLE (search_start), safe_object_get_size (search_start));
2422 pin_object (search_start);
2423 GRAY_OBJECT_ENQUEUE (search_start);
2425 pin_stats_register_object (search_start, last_obj_size);
2426 definitely_pinned [count] = search_start;
2430 /* skip to the next object */
2431 search_start = (void*)((char*)search_start + last_obj_size);
2432 } while (search_start <= addr);
2433 /* we either pinned the correct object or we ignored the addr because
2434 * it points to unused zeroed memory.
2440 //printf ("effective pinned: %d (at the end: %d)\n", count, (char*)end_nursery - (char*)last);
2445 pin_objects_in_section (GCMemSection *section)
2447 int start = section->pin_queue_start;
2448 int end = section->pin_queue_end;
2451 reduced_to = pin_objects_from_addresses (section, pin_queue + start, pin_queue + end,
2452 section->data, section->next_data);
2453 section->pin_queue_start = start;
2454 section->pin_queue_end = start + reduced_to;
2461 gap = (gap * 10) / 13;
2462 if (gap == 9 || gap == 10)
2469 /* Sort the addresses in array in increasing order.
2470 * Done using a by-the book heap sort. Which has decent and stable performance, is pretty cache efficient.
2473 sort_addresses (void **array, int size)
2478 for (i = 1; i < size; ++i) {
2481 int parent = (child - 1) / 2;
2483 if (array [parent] >= array [child])
2486 tmp = array [parent];
2487 array [parent] = array [child];
2488 array [child] = tmp;
2494 for (i = size - 1; i > 0; --i) {
2497 array [i] = array [0];
2503 while (root * 2 + 1 <= end) {
2504 int child = root * 2 + 1;
2506 if (child < end && array [child] < array [child + 1])
2508 if (array [root] >= array [child])
2512 array [root] = array [child];
2513 array [child] = tmp;
2520 static G_GNUC_UNUSED void
2521 print_nursery_gaps (void* start_nursery, void *end_nursery)
2524 gpointer first = start_nursery;
2526 for (i = 0; i < next_pin_slot; ++i) {
2527 next = pin_queue [i];
2528 fprintf (gc_debug_file, "Nursery range: %p-%p, size: %zd\n", first, next, (char*)next-(char*)first);
2532 fprintf (gc_debug_file, "Nursery range: %p-%p, size: %zd\n", first, next, (char*)next-(char*)first);
2535 /* reduce the info in the pin queue, removing duplicate pointers and sorting them */
2537 optimize_pin_queue (int start_slot)
2539 void **start, **cur, **end;
2540 /* sort and uniq pin_queue: we just sort and we let the rest discard multiple values */
2541 /* it may be better to keep ranges of pinned memory instead of individually pinning objects */
2542 DEBUG (5, fprintf (gc_debug_file, "Sorting pin queue, size: %d\n", next_pin_slot));
2543 if ((next_pin_slot - start_slot) > 1)
2544 sort_addresses (pin_queue + start_slot, next_pin_slot - start_slot);
2545 start = cur = pin_queue + start_slot;
2546 end = pin_queue + next_pin_slot;
2549 while (*start == *cur && cur < end)
2553 next_pin_slot = start - pin_queue;
2554 DEBUG (5, fprintf (gc_debug_file, "Pin queue reduced to size: %d\n", next_pin_slot));
2555 //DEBUG (6, print_nursery_gaps (start_nursery, end_nursery));
2560 * Scan the memory between start and end and queue values which could be pointers
2561 * to the area between start_nursery and end_nursery for later consideration.
2562 * Typically used for thread stacks.
2565 conservatively_pin_objects_from (void **start, void **end, void *start_nursery, void *end_nursery, int pin_type)
2568 while (start < end) {
2569 if (*start >= start_nursery && *start < end_nursery) {
2571 * *start can point to the middle of an object
2572 * note: should we handle pointing at the end of an object?
2573 * pinning in C# code disallows pointing at the end of an object
2574 * but there is some small chance that an optimizing C compiler
2575 * may keep the only reference to an object by pointing
2576 * at the end of it. We ignore this small chance for now.
2577 * Pointers to the end of an object are indistinguishable
2578 * from pointers to the start of the next object in memory
2579 * so if we allow that we'd need to pin two objects...
2580 * We queue the pointer in an array, the
2581 * array will then be sorted and uniqued. This way
2582 * we can coalesce several pinning pointers and it should
2583 * be faster since we'd do a memory scan with increasing
2584 * addresses. Note: we can align the address to the allocation
2585 * alignment, so the unique process is more effective.
2587 mword addr = (mword)*start;
2588 addr &= ~(ALLOC_ALIGN - 1);
2589 if (addr >= (mword)start_nursery && addr < (mword)end_nursery)
2590 pin_stage_ptr ((void*)addr);
2592 pin_stats_register_address ((char*)addr, pin_type);
2593 DEBUG (6, if (count) fprintf (gc_debug_file, "Pinning address %p\n", (void*)addr));
2598 DEBUG (7, if (count) fprintf (gc_debug_file, "found %d potential pinned heap pointers\n", count));
2602 * Debugging function: find in the conservative roots where @obj is being pinned.
2604 static G_GNUC_UNUSED void
2605 find_pinning_reference (char *obj, size_t size)
2609 char *endobj = obj + size;
2610 for (i = 0; i < roots_hash_size [0]; ++i) {
2611 for (root = roots_hash [0][i]; root; root = root->next) {
2612 /* if desc is non-null it has precise info */
2613 if (!root->root_desc) {
2614 char ** start = (char**)root->start_root;
2615 while (start < (char**)root->end_root) {
2616 if (*start >= obj && *start < endobj) {
2617 DEBUG (0, fprintf (gc_debug_file, "Object %p referenced in pinned roots %p-%p (at %p in record %p)\n", obj, root->start_root, root->end_root, start, root));
2624 find_pinning_ref_from_thread (obj, size);
2628 * The first thing we do in a collection is to identify pinned objects.
2629 * This function considers all the areas of memory that need to be
2630 * conservatively scanned.
2633 pin_from_roots (void *start_nursery, void *end_nursery)
2637 DEBUG (2, fprintf (gc_debug_file, "Scanning pinned roots (%d bytes, %d/%d entries)\n", (int)roots_size, num_roots_entries [ROOT_TYPE_NORMAL], num_roots_entries [ROOT_TYPE_PINNED]));
2638 /* objects pinned from the API are inside these roots */
2639 for (i = 0; i < roots_hash_size [ROOT_TYPE_PINNED]; ++i) {
2640 for (root = roots_hash [ROOT_TYPE_PINNED][i]; root; root = root->next) {
2641 DEBUG (6, fprintf (gc_debug_file, "Pinned roots %p-%p\n", root->start_root, root->end_root));
2642 conservatively_pin_objects_from ((void**)root->start_root, (void**)root->end_root, start_nursery, end_nursery, PIN_TYPE_OTHER);
2645 /* now deal with the thread stacks
2646 * in the future we should be able to conservatively scan only:
2647 * *) the cpu registers
2648 * *) the unmanaged stack frames
2649 * *) the _last_ managed stack frame
2650 * *) pointers slots in managed frames
2652 scan_thread_data (start_nursery, end_nursery, FALSE);
2654 evacuate_pin_staging_area ();
2658 * The memory area from start_root to end_root contains pointers to objects.
2659 * Their position is precisely described by @desc (this means that the pointer
2660 * can be either NULL or the pointer to the start of an object).
2661 * This functions copies them to to_space updates them.
2664 precisely_scan_objects_from (CopyOrMarkObjectFunc copy_func, void** start_root, void** end_root, char* n_start, char *n_end, mword desc)
2666 switch (desc & ROOT_DESC_TYPE_MASK) {
2667 case ROOT_DESC_BITMAP:
2668 desc >>= ROOT_DESC_TYPE_SHIFT;
2670 if ((desc & 1) && *start_root) {
2671 copy_func (start_root);
2672 DEBUG (9, fprintf (gc_debug_file, "Overwrote root at %p with %p\n", start_root, *start_root));
2673 drain_gray_stack ();
2679 case ROOT_DESC_COMPLEX: {
2680 gsize *bitmap_data = complex_descriptors + (desc >> ROOT_DESC_TYPE_SHIFT);
2681 int bwords = (*bitmap_data) - 1;
2682 void **start_run = start_root;
2684 while (bwords-- > 0) {
2685 gsize bmap = *bitmap_data++;
2686 void **objptr = start_run;
2688 if ((bmap & 1) && *objptr) {
2690 DEBUG (9, fprintf (gc_debug_file, "Overwrote root at %p with %p\n", objptr, *objptr));
2691 drain_gray_stack ();
2696 start_run += GC_BITS_PER_WORD;
2700 case ROOT_DESC_USER: {
2701 MonoGCRootMarkFunc marker = user_descriptors [desc >> ROOT_DESC_TYPE_SHIFT];
2702 marker (start_root, copy_func);
2705 case ROOT_DESC_RUN_LEN:
2706 g_assert_not_reached ();
2708 g_assert_not_reached ();
2713 alloc_fragment (void)
2715 Fragment *frag = fragment_freelist;
2717 fragment_freelist = frag->next;
2721 frag = get_internal_mem (sizeof (Fragment), INTERNAL_MEM_FRAGMENT);
2726 /* size must be a power of 2 */
2728 get_os_memory_aligned (mword size, mword alignment, gboolean activate)
2730 /* Allocate twice the memory to be able to put the block on an aligned address */
2731 char *mem = get_os_memory (size + alignment, activate);
2736 aligned = (char*)((mword)(mem + (alignment - 1)) & ~(alignment - 1));
2737 g_assert (aligned >= mem && aligned + size <= mem + size + alignment && !((mword)aligned & (alignment - 1)));
2740 free_os_memory (mem, aligned - mem);
2741 if (aligned + size < mem + size + alignment)
2742 free_os_memory (aligned + size, (mem + size + alignment) - (aligned + size));
2748 * Allocate and setup the data structures needed to be able to allocate objects
2749 * in the nursery. The nursery is stored in nursery_section.
2752 alloc_nursery (void)
2754 GCMemSection *section;
2760 if (nursery_section)
2762 DEBUG (2, fprintf (gc_debug_file, "Allocating nursery size: %zd\n", nursery_size));
2763 /* later we will alloc a larger area for the nursery but only activate
2764 * what we need. The rest will be used as expansion if we have too many pinned
2765 * objects in the existing nursery.
2767 /* FIXME: handle OOM */
2768 section = get_internal_mem (SIZEOF_GC_MEM_SECTION, INTERNAL_MEM_SECTION);
2770 g_assert (nursery_size == DEFAULT_NURSERY_SIZE);
2771 alloc_size = nursery_size;
2772 #ifdef ALIGN_NURSERY
2773 data = get_os_memory_aligned (alloc_size, alloc_size, TRUE);
2775 data = get_os_memory (alloc_size, TRUE);
2777 nursery_start = data;
2778 nursery_real_end = nursery_start + nursery_size;
2779 UPDATE_HEAP_BOUNDARIES (nursery_start, nursery_real_end);
2780 nursery_next = nursery_start;
2781 total_alloc += alloc_size;
2782 DEBUG (4, fprintf (gc_debug_file, "Expanding nursery size (%p-%p): %zd, total: %zd\n", data, data + alloc_size, nursery_size, total_alloc));
2783 section->data = section->next_data = data;
2784 section->size = alloc_size;
2785 section->end_data = nursery_real_end;
2786 scan_starts = (alloc_size + SCAN_START_SIZE - 1) / SCAN_START_SIZE;
2787 section->scan_starts = get_internal_mem (sizeof (char*) * scan_starts, INTERNAL_MEM_SCAN_STARTS);
2788 section->num_scan_start = scan_starts;
2789 section->block.role = MEMORY_ROLE_GEN0;
2790 section->block.next = NULL;
2792 nursery_section = section;
2794 /* Setup the single first large fragment */
2795 frag = alloc_fragment ();
2796 frag->fragment_start = nursery_start;
2797 frag->fragment_limit = nursery_start;
2798 frag->fragment_end = nursery_real_end;
2799 nursery_frag_real_end = nursery_real_end;
2800 /* FIXME: frag here is lost */
2804 scan_finalizer_entries (CopyOrMarkObjectFunc copy_func, FinalizeEntry *list) {
2807 for (fin = list; fin; fin = fin->next) {
2810 DEBUG (5, fprintf (gc_debug_file, "Scan of fin ready object: %p (%s)\n", fin->object, safe_name (fin->object)));
2811 copy_func (&fin->object);
2815 static mword fragment_total = 0;
2817 * We found a fragment of free memory in the nursery: memzero it and if
2818 * it is big enough, add it to the list of fragments that can be used for
2822 add_nursery_frag (size_t frag_size, char* frag_start, char* frag_end)
2825 DEBUG (4, fprintf (gc_debug_file, "Found empty fragment: %p-%p, size: %zd\n", frag_start, frag_end, frag_size));
2826 binary_protocol_empty (frag_start, frag_size);
2827 /* memsetting just the first chunk start is bound to provide better cache locality */
2828 if (nursery_clear_policy == CLEAR_AT_GC)
2829 memset (frag_start, 0, frag_size);
2830 /* Not worth dealing with smaller fragments: need to tune */
2831 if (frag_size >= FRAGMENT_MIN_SIZE) {
2832 fragment = alloc_fragment ();
2833 fragment->fragment_start = frag_start;
2834 fragment->fragment_limit = frag_start;
2835 fragment->fragment_end = frag_end;
2836 fragment->next = nursery_fragments;
2837 nursery_fragments = fragment;
2838 fragment_total += frag_size;
2840 /* Clear unused fragments, pinning depends on this */
2841 /*TODO place an int[] here instead of the memset if size justify it*/
2842 memset (frag_start, 0, frag_size);
2847 generation_name (int generation)
2849 switch (generation) {
2850 case GENERATION_NURSERY: return "nursery";
2851 case GENERATION_OLD: return "old";
2852 default: g_assert_not_reached ();
2856 static DisappearingLinkHashTable*
2857 get_dislink_hash_table (int generation)
2859 switch (generation) {
2860 case GENERATION_NURSERY: return &minor_disappearing_link_hash;
2861 case GENERATION_OLD: return &major_disappearing_link_hash;
2862 default: g_assert_not_reached ();
2866 static FinalizeEntryHashTable*
2867 get_finalize_entry_hash_table (int generation)
2869 switch (generation) {
2870 case GENERATION_NURSERY: return &minor_finalizable_hash;
2871 case GENERATION_OLD: return &major_finalizable_hash;
2872 default: g_assert_not_reached ();
2877 finish_gray_stack (char *start_addr, char *end_addr, int generation)
2882 int ephemeron_rounds = 0;
2883 CopyOrMarkObjectFunc copy_func = current_collection_generation == GENERATION_NURSERY ? copy_object : major_copy_or_mark_object;
2886 * We copied all the reachable objects. Now it's the time to copy
2887 * the objects that were not referenced by the roots, but by the copied objects.
2888 * we built a stack of objects pointed to by gray_start: they are
2889 * additional roots and we may add more items as we go.
2890 * We loop until gray_start == gray_objects which means no more objects have
2891 * been added. Note this is iterative: no recursion is involved.
2892 * We need to walk the LO list as well in search of marked big objects
2893 * (use a flag since this is needed only on major collections). We need to loop
2894 * here as well, so keep a counter of marked LO (increasing it in copy_object).
2895 * To achieve better cache locality and cache usage, we drain the gray stack
2896 * frequently, after each object is copied, and just finish the work here.
2898 drain_gray_stack ();
2900 DEBUG (2, fprintf (gc_debug_file, "%s generation done\n", generation_name (generation)));
2901 /* walk the finalization queue and move also the objects that need to be
2902 * finalized: use the finalized objects as new roots so the objects they depend
2903 * on are also not reclaimed. As with the roots above, only objects in the nursery
2904 * are marked/copied.
2905 * We need a loop here, since objects ready for finalizers may reference other objects
2906 * that are fin-ready. Speedup with a flag?
2910 * Walk the ephemeron tables marking all values with reachable keys. This must be completely done
2911 * before processing finalizable objects to avoid finalizing reachable values.
2913 * It must be done inside the finalizaters loop since objects must not be removed from CWT tables
2914 * while they are been finalized.
2916 int done_with_ephemerons = 0;
2918 done_with_ephemerons = mark_ephemerons_in_range (copy_func, start_addr, end_addr);
2919 drain_gray_stack ();
2921 } while (!done_with_ephemerons);
2923 fin_ready = num_ready_finalizers;
2924 finalize_in_range (copy_func, start_addr, end_addr, generation);
2925 if (generation == GENERATION_OLD)
2926 finalize_in_range (copy_func, nursery_start, nursery_real_end, GENERATION_NURSERY);
2928 /* drain the new stack that might have been created */
2929 DEBUG (6, fprintf (gc_debug_file, "Precise scan of gray area post fin\n"));
2930 drain_gray_stack ();
2931 } while (fin_ready != num_ready_finalizers);
2934 * Clear ephemeron pairs with unreachable keys.
2935 * We pass the copy func so we can figure out if an array was promoted or not.
2937 clear_unreachable_ephemerons (copy_func, start_addr, end_addr);
2940 DEBUG (2, fprintf (gc_debug_file, "Finalize queue handling scan for %s generation: %d usecs %d ephemeron roundss\n", generation_name (generation), TV_ELAPSED (atv, btv), ephemeron_rounds));
2943 * handle disappearing links
2944 * Note we do this after checking the finalization queue because if an object
2945 * survives (at least long enough to be finalized) we don't clear the link.
2946 * This also deals with a possible issue with the monitor reclamation: with the Boehm
2947 * GC a finalized object my lose the monitor because it is cleared before the finalizer is
2950 g_assert (gray_object_queue_is_empty ());
2952 null_link_in_range (copy_func, start_addr, end_addr, generation);
2953 if (generation == GENERATION_OLD)
2954 null_link_in_range (copy_func, start_addr, end_addr, GENERATION_NURSERY);
2955 if (gray_object_queue_is_empty ())
2957 drain_gray_stack ();
2960 g_assert (gray_object_queue_is_empty ());
2964 check_section_scan_starts (GCMemSection *section)
2967 for (i = 0; i < section->num_scan_start; ++i) {
2968 if (section->scan_starts [i]) {
2969 guint size = safe_object_get_size ((MonoObject*) section->scan_starts [i]);
2970 g_assert (size >= sizeof (MonoObject) && size <= MAX_SMALL_OBJ_SIZE);
2976 check_scan_starts (void)
2978 if (!do_scan_starts_check)
2980 check_section_scan_starts (nursery_section);
2981 major_check_scan_starts ();
2984 static int last_num_pinned = 0;
2987 build_nursery_fragments (int start_pin, int end_pin)
2989 char *frag_start, *frag_end;
2993 while (nursery_fragments) {
2994 Fragment *next = nursery_fragments->next;
2995 nursery_fragments->next = fragment_freelist;
2996 fragment_freelist = nursery_fragments;
2997 nursery_fragments = next;
2999 frag_start = nursery_start;
3001 /* clear scan starts */
3002 memset (nursery_section->scan_starts, 0, nursery_section->num_scan_start * sizeof (gpointer));
3003 for (i = start_pin; i < end_pin; ++i) {
3004 frag_end = pin_queue [i];
3005 /* remove the pin bit from pinned objects */
3006 unpin_object (frag_end);
3007 nursery_section->scan_starts [((char*)frag_end - (char*)nursery_section->data)/SCAN_START_SIZE] = frag_end;
3008 frag_size = frag_end - frag_start;
3010 add_nursery_frag (frag_size, frag_start, frag_end);
3011 frag_size = safe_object_get_size ((MonoObject*)pin_queue [i]);
3012 frag_size += ALLOC_ALIGN - 1;
3013 frag_size &= ~(ALLOC_ALIGN - 1);
3014 frag_start = (char*)pin_queue [i] + frag_size;
3016 nursery_last_pinned_end = frag_start;
3017 frag_end = nursery_real_end;
3018 frag_size = frag_end - frag_start;
3020 add_nursery_frag (frag_size, frag_start, frag_end);
3021 if (!nursery_fragments) {
3022 DEBUG (1, fprintf (gc_debug_file, "Nursery fully pinned (%d)\n", end_pin - start_pin));
3023 for (i = start_pin; i < end_pin; ++i) {
3024 DEBUG (3, fprintf (gc_debug_file, "Bastard pinning obj %p (%s), size: %d\n", pin_queue [i], safe_name (pin_queue [i]), safe_object_get_size (pin_queue [i])));
3029 nursery_next = nursery_frag_real_end = NULL;
3031 /* Clear TLABs for all threads */
3036 scan_from_registered_roots (CopyOrMarkObjectFunc copy_func, char *addr_start, char *addr_end, int root_type)
3040 for (i = 0; i < roots_hash_size [root_type]; ++i) {
3041 for (root = roots_hash [root_type][i]; root; root = root->next) {
3042 DEBUG (6, fprintf (gc_debug_file, "Precise root scan %p-%p (desc: %p)\n", root->start_root, root->end_root, (void*)root->root_desc));
3043 precisely_scan_objects_from (copy_func, (void**)root->start_root, (void**)root->end_root, addr_start, addr_end, root->root_desc);
3049 dump_occupied (char *start, char *end, char *section_start)
3051 fprintf (heap_dump_file, "<occupied offset=\"%zd\" size=\"%zd\"/>\n", start - section_start, end - start);
3055 dump_section (GCMemSection *section, const char *type)
3057 char *start = section->data;
3058 char *end = section->data + section->size;
3059 char *occ_start = NULL;
3061 char *old_start = NULL; /* just for debugging */
3063 fprintf (heap_dump_file, "<section type=\"%s\" size=\"%zu\">\n", type, section->size);
3065 while (start < end) {
3069 if (!*(void**)start) {
3071 dump_occupied (occ_start, start, section->data);
3074 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
3077 g_assert (start < section->next_data);
3082 vt = (GCVTable*)LOAD_VTABLE (start);
3085 size = safe_object_get_size ((MonoObject*) start);
3086 size += ALLOC_ALIGN - 1;
3087 size &= ~(ALLOC_ALIGN - 1);
3090 fprintf (heap_dump_file, "<object offset=\"%d\" class=\"%s.%s\" size=\"%d\"/>\n",
3091 start - section->data,
3092 vt->klass->name_space, vt->klass->name,
3100 dump_occupied (occ_start, start, section->data);
3102 fprintf (heap_dump_file, "</section>\n");
3106 dump_object (MonoObject *obj, gboolean dump_location)
3108 static char class_name [1024];
3110 MonoClass *class = mono_object_class (obj);
3114 * Python's XML parser is too stupid to parse angle brackets
3115 * in strings, so we just ignore them;
3118 while (class->name [i] && j < sizeof (class_name) - 1) {
3119 if (!strchr ("<>\"", class->name [i]))
3120 class_name [j++] = class->name [i];
3123 g_assert (j < sizeof (class_name));
3126 fprintf (heap_dump_file, "<object class=\"%s.%s\" size=\"%d\"",
3127 class->name_space, class_name,
3128 safe_object_get_size (obj));
3129 if (dump_location) {
3130 const char *location;
3131 if (ptr_in_nursery (obj))
3132 location = "nursery";
3133 else if (safe_object_get_size (obj) <= MAX_SMALL_OBJ_SIZE)
3137 fprintf (heap_dump_file, " location=\"%s\"", location);
3139 fprintf (heap_dump_file, "/>\n");
3143 dump_heap (const char *type, int num, const char *reason)
3145 static char const *internal_mem_names [] = { "pin-queue", "fragment", "section", "scan-starts",
3146 "fin-table", "finalize-entry", "dislink-table",
3147 "dislink", "roots-table", "root-record", "statistics",
3148 "remset", "gray-queue", "store-remset", "marksweep-tables",
3149 "marksweep-block-info", "ephemeron-link" };
3155 fprintf (heap_dump_file, "<collection type=\"%s\" num=\"%d\"", type, num);
3157 fprintf (heap_dump_file, " reason=\"%s\"", reason);
3158 fprintf (heap_dump_file, ">\n");
3159 fprintf (heap_dump_file, "<other-mem-usage type=\"pinned-chunks\" size=\"%lld\"/>\n", pinned_chunk_bytes_alloced);
3160 fprintf (heap_dump_file, "<other-mem-usage type=\"large-internal\" size=\"%lld\"/>\n", large_internal_bytes_alloced);
3161 fprintf (heap_dump_file, "<other-mem-usage type=\"mempools\" size=\"%ld\"/>\n", mono_mempool_get_bytes_allocated ());
3162 for (i = 0; i < INTERNAL_MEM_MAX; ++i)
3163 fprintf (heap_dump_file, "<other-mem-usage type=\"%s\" size=\"%ld\"/>\n", internal_mem_names [i], small_internal_mem_bytes [i]);
3164 fprintf (heap_dump_file, "<pinned type=\"stack\" bytes=\"%zu\"/>\n", pinned_byte_counts [PIN_TYPE_STACK]);
3165 /* fprintf (heap_dump_file, "<pinned type=\"static-data\" bytes=\"%d\"/>\n", pinned_byte_counts [PIN_TYPE_STATIC_DATA]); */
3166 fprintf (heap_dump_file, "<pinned type=\"other\" bytes=\"%zu\"/>\n", pinned_byte_counts [PIN_TYPE_OTHER]);
3168 fprintf (heap_dump_file, "<pinned-objects>\n");
3169 for (list = pinned_objects; list; list = list->next)
3170 dump_object (list->obj, TRUE);
3171 fprintf (heap_dump_file, "</pinned-objects>\n");
3173 dump_section (nursery_section, "nursery");
3177 fprintf (heap_dump_file, "<los>\n");
3178 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
3179 dump_object ((MonoObject*)bigobj->data, FALSE);
3180 fprintf (heap_dump_file, "</los>\n");
3182 fprintf (heap_dump_file, "</collection>\n");
3188 static gboolean inited = FALSE;
3193 mono_counters_register ("Minor fragment clear", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_pre_collection_fragment_clear);
3194 mono_counters_register ("Minor pinning", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_pinning);
3195 mono_counters_register ("Minor scan remsets", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_scan_remsets);
3196 mono_counters_register ("Minor scan pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_scan_pinned);
3197 mono_counters_register ("Minor scan registered roots", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_scan_registered_roots);
3198 mono_counters_register ("Minor scan thread data", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_scan_thread_data);
3199 mono_counters_register ("Minor finish gray stack", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_finish_gray_stack);
3200 mono_counters_register ("Minor fragment creation", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_fragment_creation);
3202 mono_counters_register ("Major fragment clear", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_pre_collection_fragment_clear);
3203 mono_counters_register ("Major pinning", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_pinning);
3204 mono_counters_register ("Major scan pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_pinned);
3205 mono_counters_register ("Major scan registered roots", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_registered_roots);
3206 mono_counters_register ("Major scan thread data", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_thread_data);
3207 mono_counters_register ("Major scan alloc_pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_alloc_pinned);
3208 mono_counters_register ("Major scan finalized", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_finalized);
3209 mono_counters_register ("Major scan big objects", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_big_objects);
3210 mono_counters_register ("Major finish gray stack", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_finish_gray_stack);
3211 mono_counters_register ("Major sweep", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_sweep);
3212 mono_counters_register ("Major fragment creation", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_fragment_creation);
3214 #ifdef HEAVY_STATISTICS
3215 mono_counters_register ("WBarrier set field", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_field);
3216 mono_counters_register ("WBarrier set arrayref", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_arrayref);
3217 mono_counters_register ("WBarrier arrayref copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_arrayref_copy);
3218 mono_counters_register ("WBarrier generic store called", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_generic_store);
3219 mono_counters_register ("WBarrier generic store stored", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_generic_store_remset);
3220 mono_counters_register ("WBarrier set root", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_root);
3221 mono_counters_register ("WBarrier value copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_value_copy);
3222 mono_counters_register ("WBarrier object copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_object_copy);
3224 mono_counters_register ("# objects allocated", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_alloced);
3225 mono_counters_register ("bytes allocated", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_bytes_alloced);
3226 mono_counters_register ("# objects allocated degraded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_alloced_degraded);
3227 mono_counters_register ("bytes allocated degraded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_bytes_alloced_degraded);
3228 mono_counters_register ("bytes allocated in LOS", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_bytes_alloced_los);
3230 mono_counters_register ("# copy_object() called (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_called_nursery);
3231 mono_counters_register ("# objects copied (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_copied_nursery);
3232 mono_counters_register ("# copy_object() called (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_called_major);
3233 mono_counters_register ("# objects copied (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_copied_major);
3235 mono_counters_register ("# scan_object() called (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_scan_object_called_nursery);
3236 mono_counters_register ("# scan_object() called (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_scan_object_called_major);
3238 mono_counters_register ("# nursery copy_object() failed from space", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_from_space);
3239 mono_counters_register ("# nursery copy_object() failed forwarded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_forwarded);
3240 mono_counters_register ("# nursery copy_object() failed pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_pinned);
3242 mono_counters_register ("# wasted fragments used", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_wasted_fragments_used);
3243 mono_counters_register ("bytes in wasted fragments", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_wasted_fragments_bytes);
3245 mono_counters_register ("Store remsets", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_store_remsets);
3246 mono_counters_register ("Unique store remsets", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_store_remsets_unique);
3247 mono_counters_register ("Saved remsets 1", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_saved_remsets_1);
3248 mono_counters_register ("Saved remsets 2", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_saved_remsets_2);
3249 mono_counters_register ("Global remsets added", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_added);
3250 mono_counters_register ("Global remsets re-added", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_readded);
3251 mono_counters_register ("Global remsets processed", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_processed);
3252 mono_counters_register ("Global remsets discarded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_discarded);
3260 need_major_collection (void)
3262 mword los_alloced = los_memory_usage - MIN (last_los_memory_usage, los_memory_usage);
3263 return minor_collection_sections_alloced * MAJOR_SECTION_SIZE + los_alloced > minor_collection_allowance;
3267 * Collect objects in the nursery. Returns whether to trigger a major
3271 collect_nursery (size_t requested_size)
3273 size_t max_garbage_amount;
3274 char *orig_nursery_next;
3275 TV_DECLARE (all_atv);
3276 TV_DECLARE (all_btv);
3280 current_collection_generation = GENERATION_NURSERY;
3283 binary_protocol_collection (GENERATION_NURSERY);
3284 check_scan_starts ();
3287 orig_nursery_next = nursery_next;
3288 nursery_next = MAX (nursery_next, nursery_last_pinned_end);
3289 /* FIXME: optimize later to use the higher address where an object can be present */
3290 nursery_next = MAX (nursery_next, nursery_real_end);
3292 DEBUG (1, fprintf (gc_debug_file, "Start nursery collection %d %p-%p, size: %d\n", num_minor_gcs, nursery_start, nursery_next, (int)(nursery_next - nursery_start)));
3293 max_garbage_amount = nursery_next - nursery_start;
3294 g_assert (nursery_section->size >= max_garbage_amount);
3296 /* world must be stopped already */
3297 TV_GETTIME (all_atv);
3300 /* Pinning depends on this */
3301 clear_nursery_fragments (orig_nursery_next);
3304 time_minor_pre_collection_fragment_clear += TV_ELAPSED_MS (atv, btv);
3307 check_for_xdomain_refs ();
3309 nursery_section->next_data = nursery_next;
3311 major_start_nursery_collection ();
3313 gray_object_queue_init ();
3316 mono_stats.minor_gc_count ++;
3318 global_remset_cache_clear ();
3320 /* pin from pinned handles */
3322 pin_from_roots (nursery_start, nursery_next);
3323 /* identify pinned objects */
3324 optimize_pin_queue (0);
3325 next_pin_slot = pin_objects_from_addresses (nursery_section, pin_queue, pin_queue + next_pin_slot, nursery_start, nursery_next);
3326 nursery_section->pin_queue_start = 0;
3327 nursery_section->pin_queue_end = next_pin_slot;
3329 time_minor_pinning += TV_ELAPSED_MS (btv, atv);
3330 DEBUG (2, fprintf (gc_debug_file, "Finding pinned pointers: %d in %d usecs\n", next_pin_slot, TV_ELAPSED (btv, atv)));
3331 DEBUG (4, fprintf (gc_debug_file, "Start scan with %d pinned objects\n", next_pin_slot));
3333 if (consistency_check_at_minor_collection)
3334 check_consistency ();
3337 * walk all the roots and copy the young objects to the old generation,
3338 * starting from to_space
3341 scan_from_remsets (nursery_start, nursery_next);
3342 /* we don't have complete write barrier yet, so we scan all the old generation sections */
3344 time_minor_scan_remsets += TV_ELAPSED_MS (atv, btv);
3345 DEBUG (2, fprintf (gc_debug_file, "Old generation scan: %d usecs\n", TV_ELAPSED (atv, btv)));
3347 drain_gray_stack ();
3350 time_minor_scan_pinned += TV_ELAPSED_MS (btv, atv);
3351 /* registered roots, this includes static fields */
3352 scan_from_registered_roots (copy_object, nursery_start, nursery_next, ROOT_TYPE_NORMAL);
3353 scan_from_registered_roots (copy_object, nursery_start, nursery_next, ROOT_TYPE_WBARRIER);
3355 time_minor_scan_registered_roots += TV_ELAPSED_MS (atv, btv);
3357 scan_thread_data (nursery_start, nursery_next, TRUE);
3359 time_minor_scan_thread_data += TV_ELAPSED_MS (btv, atv);
3362 finish_gray_stack (nursery_start, nursery_next, GENERATION_NURSERY);
3364 time_minor_finish_gray_stack += TV_ELAPSED_MS (btv, atv);
3366 /* walk the pin_queue, build up the fragment list of free memory, unmark
3367 * pinned objects as we go, memzero() the empty fragments so they are ready for the
3370 build_nursery_fragments (0, next_pin_slot);
3372 time_minor_fragment_creation += TV_ELAPSED_MS (atv, btv);
3373 DEBUG (2, fprintf (gc_debug_file, "Fragment creation: %d usecs, %zd bytes available\n", TV_ELAPSED (atv, btv), fragment_total));
3375 if (consistency_check_at_minor_collection)
3376 check_major_refs ();
3378 major_finish_nursery_collection ();
3380 TV_GETTIME (all_btv);
3381 mono_stats.minor_gc_time_usecs += TV_ELAPSED (all_atv, all_btv);
3384 dump_heap ("minor", num_minor_gcs - 1, NULL);
3386 /* prepare the pin queue for the next collection */
3387 last_num_pinned = next_pin_slot;
3389 if (fin_ready_list || critical_fin_list) {
3390 DEBUG (4, fprintf (gc_debug_file, "Finalizer-thread wakeup: ready %d\n", num_ready_finalizers));
3391 mono_gc_finalize_notify ();
3395 g_assert (gray_object_queue_is_empty ());
3397 check_scan_starts ();
3399 current_collection_generation = -1;
3401 return need_major_collection ();
3405 major_do_collection (const char *reason)
3407 LOSObject *bigobj, *prevbo;
3408 TV_DECLARE (all_atv);
3409 TV_DECLARE (all_btv);
3412 /* FIXME: only use these values for the precise scan
3413 * note that to_space pointers should be excluded anyway...
3415 char *heap_start = NULL;
3416 char *heap_end = (char*)-1;
3417 int old_num_major_sections = num_major_sections;
3418 int num_major_sections_saved, save_target, allowance_target;
3419 mword los_memory_saved, los_memory_alloced, old_los_memory_usage;
3422 * A domain could have been freed, resulting in
3423 * los_memory_usage being less than last_los_memory_usage.
3425 los_memory_alloced = los_memory_usage - MIN (last_los_memory_usage, los_memory_usage);
3426 old_los_memory_usage = los_memory_usage;
3428 //count_ref_nonref_objs ();
3429 //consistency_check ();
3432 binary_protocol_collection (GENERATION_OLD);
3433 check_scan_starts ();
3434 gray_object_queue_init ();
3437 DEBUG (1, fprintf (gc_debug_file, "Start major collection %d\n", num_major_gcs));
3439 mono_stats.major_gc_count ++;
3441 /* world must be stopped already */
3442 TV_GETTIME (all_atv);
3445 /* Pinning depends on this */
3446 clear_nursery_fragments (nursery_next);
3449 time_major_pre_collection_fragment_clear += TV_ELAPSED_MS (atv, btv);
3452 check_for_xdomain_refs ();
3454 nursery_section->next_data = nursery_real_end;
3455 /* we should also coalesce scanning from sections close to each other
3456 * and deal with pointers outside of the sections later.
3458 /* The remsets are not useful for a major collection */
3460 global_remset_cache_clear ();
3464 DEBUG (6, fprintf (gc_debug_file, "Collecting pinned addresses\n"));
3465 pin_from_roots ((void*)lowest_heap_address, (void*)highest_heap_address);
3466 optimize_pin_queue (0);
3469 * pin_queue now contains all candidate pointers, sorted and
3470 * uniqued. We must do two passes now to figure out which
3471 * objects are pinned.
3473 * The first is to find within the pin_queue the area for each
3474 * section. This requires that the pin_queue be sorted. We
3475 * also process the LOS objects and pinned chunks here.
3477 * The second, destructive, pass is to reduce the section
3478 * areas to pointers to the actually pinned objects.
3480 DEBUG (6, fprintf (gc_debug_file, "Pinning from sections\n"));
3481 /* first pass for the sections */
3482 find_section_pin_queue_start_end (nursery_section);
3483 major_find_pin_queue_start_ends ();
3484 /* identify possible pointers to the insize of large objects */
3485 DEBUG (6, fprintf (gc_debug_file, "Pinning from large objects\n"));
3486 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
3488 find_optimized_pin_queue_area (bigobj->data, (char*)bigobj->data + bigobj->size, &start, &end);
3490 pin_object (bigobj->data);
3491 /* FIXME: only enqueue if object has references */
3492 GRAY_OBJECT_ENQUEUE (bigobj->data);
3494 pin_stats_register_object ((char*) bigobj->data, safe_object_get_size ((MonoObject*) bigobj->data));
3495 DEBUG (6, fprintf (gc_debug_file, "Marked large object %p (%s) size: %zd from roots\n", bigobj->data, safe_name (bigobj->data), bigobj->size));
3498 /* second pass for the sections */
3499 pin_objects_in_section (nursery_section);
3500 major_pin_objects ();
3503 time_major_pinning += TV_ELAPSED_MS (atv, btv);
3504 DEBUG (2, fprintf (gc_debug_file, "Finding pinned pointers: %d in %d usecs\n", next_pin_slot, TV_ELAPSED (atv, btv)));
3505 DEBUG (4, fprintf (gc_debug_file, "Start scan with %d pinned objects\n", next_pin_slot));
3507 major_init_to_space ();
3509 drain_gray_stack ();
3512 time_major_scan_pinned += TV_ELAPSED_MS (btv, atv);
3514 /* registered roots, this includes static fields */
3515 scan_from_registered_roots (major_copy_or_mark_object, heap_start, heap_end, ROOT_TYPE_NORMAL);
3516 scan_from_registered_roots (major_copy_or_mark_object, heap_start, heap_end, ROOT_TYPE_WBARRIER);
3518 time_major_scan_registered_roots += TV_ELAPSED_MS (atv, btv);
3521 /* FIXME: This is the wrong place for this, because it does
3523 scan_thread_data (heap_start, heap_end, TRUE);
3525 time_major_scan_thread_data += TV_ELAPSED_MS (btv, atv);
3528 time_major_scan_alloc_pinned += TV_ELAPSED_MS (atv, btv);
3530 /* scan the list of objects ready for finalization */
3531 scan_finalizer_entries (major_copy_or_mark_object, fin_ready_list);
3532 scan_finalizer_entries (major_copy_or_mark_object, critical_fin_list);
3534 time_major_scan_finalized += TV_ELAPSED_MS (btv, atv);
3535 DEBUG (2, fprintf (gc_debug_file, "Root scan: %d usecs\n", TV_ELAPSED (btv, atv)));
3538 time_major_scan_big_objects += TV_ELAPSED_MS (atv, btv);
3540 /* all the objects in the heap */
3541 finish_gray_stack (heap_start, heap_end, GENERATION_OLD);
3543 time_major_finish_gray_stack += TV_ELAPSED_MS (btv, atv);
3545 /* sweep the big objects list */
3547 for (bigobj = los_object_list; bigobj;) {
3548 if (object_is_pinned (bigobj->data)) {
3549 unpin_object (bigobj->data);
3552 /* not referenced anywhere, so we can free it */
3554 prevbo->next = bigobj->next;
3556 los_object_list = bigobj->next;
3558 bigobj = bigobj->next;
3559 free_large_object (to_free);
3563 bigobj = bigobj->next;
3569 time_major_sweep += TV_ELAPSED_MS (atv, btv);
3571 /* walk the pin_queue, build up the fragment list of free memory, unmark
3572 * pinned objects as we go, memzero() the empty fragments so they are ready for the
3575 build_nursery_fragments (nursery_section->pin_queue_start, nursery_section->pin_queue_end);
3578 time_major_fragment_creation += TV_ELAPSED_MS (btv, atv);
3580 TV_GETTIME (all_btv);
3581 mono_stats.major_gc_time_usecs += TV_ELAPSED (all_atv, all_btv);
3584 dump_heap ("major", num_major_gcs - 1, reason);
3586 /* prepare the pin queue for the next collection */
3588 if (fin_ready_list || critical_fin_list) {
3589 DEBUG (4, fprintf (gc_debug_file, "Finalizer-thread wakeup: ready %d\n", num_ready_finalizers));
3590 mono_gc_finalize_notify ();
3594 g_assert (gray_object_queue_is_empty ());
3596 num_major_sections_saved = MAX (old_num_major_sections - num_major_sections, 0);
3597 los_memory_saved = MAX (old_los_memory_usage - los_memory_usage, 1);
3599 save_target = ((num_major_sections * MAJOR_SECTION_SIZE) + los_memory_saved) / 2;
3601 * We aim to allow the allocation of as many sections as is
3602 * necessary to reclaim save_target sections in the next
3603 * collection. We assume the collection pattern won't change.
3604 * In the last cycle, we had num_major_sections_saved for
3605 * minor_collection_sections_alloced. Assuming things won't
3606 * change, this must be the same ratio as save_target for
3607 * allowance_target, i.e.
3609 * num_major_sections_saved save_target
3610 * --------------------------------- == ----------------
3611 * minor_collection_sections_alloced allowance_target
3615 allowance_target = (mword)((double)save_target * (double)(minor_collection_sections_alloced * MAJOR_SECTION_SIZE + los_memory_alloced) / (double)(num_major_sections_saved * MAJOR_SECTION_SIZE + los_memory_saved));
3617 minor_collection_allowance = MAX (MIN (allowance_target, num_major_sections * MAJOR_SECTION_SIZE + los_memory_usage), MIN_MINOR_COLLECTION_ALLOWANCE);
3619 minor_collection_sections_alloced = 0;
3620 last_los_memory_usage = los_memory_usage;
3622 check_scan_starts ();
3624 //consistency_check ();
3628 major_collection (const char *reason)
3630 if (g_getenv ("MONO_GC_NO_MAJOR")) {
3631 collect_nursery (0);
3635 current_collection_generation = GENERATION_OLD;
3636 major_do_collection (reason);
3637 current_collection_generation = -1;
3641 * When deciding if it's better to collect or to expand, keep track
3642 * of how much garbage was reclaimed with the last collection: if it's too
3644 * This is called when we could not allocate a small object.
3646 static void __attribute__((noinline))
3647 minor_collect_or_expand_inner (size_t size)
3649 int do_minor_collection = 1;
3651 if (!nursery_section) {
3655 if (do_minor_collection) {
3657 if (collect_nursery (size))
3658 major_collection ("minor overflow");
3659 DEBUG (2, fprintf (gc_debug_file, "Heap size: %zd, LOS size: %zd\n", total_alloc, los_memory_usage));
3661 /* this also sets the proper pointers for the next allocation */
3662 if (!search_fragment_for_size (size)) {
3664 /* TypeBuilder and MonoMethod are killing mcs with fragmentation */
3665 DEBUG (1, fprintf (gc_debug_file, "nursery collection didn't find enough room for %zd alloc (%d pinned)\n", size, last_num_pinned));
3666 for (i = 0; i < last_num_pinned; ++i) {
3667 DEBUG (3, fprintf (gc_debug_file, "Bastard pinning obj %p (%s), size: %d\n", pin_queue [i], safe_name (pin_queue [i]), safe_object_get_size (pin_queue [i])));
3672 //report_internal_mem_usage ();
3676 * ######################################################################
3677 * ######## Memory allocation from the OS
3678 * ######################################################################
3679 * This section of code deals with getting memory from the OS and
3680 * allocating memory for GC-internal data structures.
3681 * Internal memory can be handled with a freelist for small objects.
3685 * Allocate a big chunk of memory from the OS (usually 64KB to several megabytes).
3686 * This must not require any lock.
3689 get_os_memory (size_t size, int activate)
3692 unsigned long prot_flags = activate? MONO_MMAP_READ|MONO_MMAP_WRITE: MONO_MMAP_NONE;
3694 prot_flags |= MONO_MMAP_PRIVATE | MONO_MMAP_ANON;
3695 size += pagesize - 1;
3696 size &= ~(pagesize - 1);
3697 ptr = mono_valloc (0, size, prot_flags);
3702 * Free the memory returned by get_os_memory (), returning it to the OS.
3705 free_os_memory (void *addr, size_t size)
3707 mono_vfree (addr, size);
3714 report_pinned_chunk (PinnedChunk *chunk, int seq) {
3716 int i, free_pages, num_free, free_mem;
3718 for (i = 0; i < chunk->num_pages; ++i) {
3719 if (!chunk->page_sizes [i])
3722 printf ("Pinned chunk %d at %p, size: %d, pages: %d, free: %d\n", seq, chunk, chunk->num_pages * FREELIST_PAGESIZE, chunk->num_pages, free_pages);
3723 free_mem = FREELIST_PAGESIZE * free_pages;
3724 for (i = 0; i < FREELIST_NUM_SLOTS; ++i) {
3725 if (!chunk->free_list [i])
3728 p = chunk->free_list [i];
3733 printf ("\tfree list of size %d, %d items\n", freelist_sizes [i], num_free);
3734 free_mem += freelist_sizes [i] * num_free;
3736 printf ("\tfree memory in chunk: %d\n", free_mem);
3742 static G_GNUC_UNUSED void
3743 report_internal_mem_usage (void) {
3746 printf ("Internal memory usage:\n");
3748 for (chunk = internal_chunk_list; chunk; chunk = chunk->block.next) {
3749 report_pinned_chunk (chunk, i++);
3751 printf ("Pinned memory usage:\n");
3752 major_report_pinned_memory_usage ();
3756 * Find the slot number in the freelist for memory chunks that
3757 * can contain @size objects.
3760 slot_for_size (size_t size)
3763 /* do a binary search or lookup table later. */
3764 for (slot = 0; slot < FREELIST_NUM_SLOTS; ++slot) {
3765 if (freelist_sizes [slot] >= size)
3768 g_assert_not_reached ();
3773 * Build a free list for @size memory chunks from the memory area between
3774 * start_page and end_page.
3777 build_freelist (PinnedChunk *chunk, int slot, int size, char *start_page, char *end_page)
3781 /*g_print ("building freelist for slot %d, size %d in %p\n", slot, size, chunk);*/
3782 p = (void**)start_page;
3783 end = (void**)(end_page - size);
3784 g_assert (!chunk->free_list [slot]);
3785 chunk->free_list [slot] = p;
3786 while ((char*)p + size <= (char*)end) {
3788 *p = (void*)((char*)p + size);
3792 /*g_print ("%d items created, max: %d\n", count, (end_page - start_page) / size);*/
3796 alloc_pinned_chunk (void)
3800 int size = PINNED_CHUNK_SIZE;
3802 chunk = get_os_memory_aligned (size, size, TRUE);
3803 chunk->block.role = MEMORY_ROLE_PINNED;
3805 UPDATE_HEAP_BOUNDARIES (chunk, ((char*)chunk + size));
3806 total_alloc += size;
3807 pinned_chunk_bytes_alloced += size;
3809 /* setup the bookeeping fields */
3810 chunk->num_pages = size / FREELIST_PAGESIZE;
3811 offset = G_STRUCT_OFFSET (PinnedChunk, data);
3812 chunk->page_sizes = (void*)((char*)chunk + offset);
3813 offset += sizeof (int) * chunk->num_pages;
3814 offset += ALLOC_ALIGN - 1;
3815 offset &= ~(ALLOC_ALIGN - 1);
3816 chunk->free_list = (void*)((char*)chunk + offset);
3817 offset += sizeof (void*) * FREELIST_NUM_SLOTS;
3818 offset += ALLOC_ALIGN - 1;
3819 offset &= ~(ALLOC_ALIGN - 1);
3820 chunk->start_data = (void*)((char*)chunk + offset);
3822 /* allocate the first page to the freelist */
3823 chunk->page_sizes [0] = PINNED_FIRST_SLOT_SIZE;
3824 build_freelist (chunk, slot_for_size (PINNED_FIRST_SLOT_SIZE), PINNED_FIRST_SLOT_SIZE, chunk->start_data, ((char*)chunk + FREELIST_PAGESIZE));
3825 DEBUG (4, fprintf (gc_debug_file, "Allocated pinned chunk %p, size: %d\n", chunk, size));
3829 /* assumes freelist for slot is empty, so try to alloc a new page */
3831 get_chunk_freelist (PinnedChunk *chunk, int slot)
3835 p = chunk->free_list [slot];
3837 chunk->free_list [slot] = *p;
3840 for (i = 0; i < chunk->num_pages; ++i) {
3842 if (chunk->page_sizes [i])
3844 size = freelist_sizes [slot];
3845 chunk->page_sizes [i] = size;
3846 build_freelist (chunk, slot, size, (char*)chunk + FREELIST_PAGESIZE * i, (char*)chunk + FREELIST_PAGESIZE * (i + 1));
3850 p = chunk->free_list [slot];
3852 chunk->free_list [slot] = *p;
3858 /* used for the GC-internal data structures */
3860 get_internal_mem (size_t size, int type)
3864 PinnedChunk *pchunk;
3866 if (size > freelist_sizes [FREELIST_NUM_SLOTS - 1]) {
3867 LargeInternalMemHeader *mh;
3869 size += sizeof (LargeInternalMemHeader);
3870 mh = get_os_memory (size, TRUE);
3871 mh->magic = LARGE_INTERNAL_MEM_HEADER_MAGIC;
3874 large_internal_bytes_alloced += size;
3879 slot = slot_for_size (size);
3880 g_assert (size <= freelist_sizes [slot]);
3882 small_internal_mem_bytes [type] += freelist_sizes [slot];
3884 for (pchunk = internal_chunk_list; pchunk; pchunk = pchunk->block.next) {
3885 void **p = pchunk->free_list [slot];
3887 pchunk->free_list [slot] = *p;
3888 memset (p, 0, size);
3892 for (pchunk = internal_chunk_list; pchunk; pchunk = pchunk->block.next) {
3893 res = get_chunk_freelist (pchunk, slot);
3895 memset (res, 0, size);
3899 pchunk = alloc_pinned_chunk ();
3900 /* FIXME: handle OOM */
3901 pchunk->block.next = internal_chunk_list;
3902 internal_chunk_list = pchunk;
3903 res = get_chunk_freelist (pchunk, slot);
3904 memset (res, 0, size);
3909 free_internal_mem (void *addr, int type)
3911 PinnedChunk *pchunk;
3912 LargeInternalMemHeader *mh;
3915 for (pchunk = internal_chunk_list; pchunk; pchunk = pchunk->block.next) {
3916 /*printf ("trying to free %p in %p (pages: %d)\n", addr, pchunk, pchunk->num_pages);*/
3917 if (addr >= (void*)pchunk && (char*)addr < (char*)pchunk + pchunk->num_pages * FREELIST_PAGESIZE) {
3918 int offset = (char*)addr - (char*)pchunk;
3919 int page = offset / FREELIST_PAGESIZE;
3920 int slot = slot_for_size (pchunk->page_sizes [page]);
3922 *p = pchunk->free_list [slot];
3923 pchunk->free_list [slot] = p;
3925 small_internal_mem_bytes [type] -= freelist_sizes [slot];
3930 mh = (LargeInternalMemHeader*)((char*)addr - G_STRUCT_OFFSET (LargeInternalMemHeader, data));
3931 g_assert (mh->magic == LARGE_INTERNAL_MEM_HEADER_MAGIC);
3932 large_internal_bytes_alloced -= mh->size;
3933 free_os_memory (mh, mh->size);
3937 * ######################################################################
3938 * ######## Object allocation
3939 * ######################################################################
3940 * This section of code deals with allocating memory for objects.
3941 * There are several ways:
3942 * *) allocate large objects
3943 * *) allocate normal objects
3944 * *) fast lock-free allocation
3945 * *) allocation of pinned objects
3949 free_large_object (LOSObject *obj)
3951 size_t size = obj->size;
3952 DEBUG (4, fprintf (gc_debug_file, "Freed large object %p, size %zd\n", obj->data, obj->size));
3953 binary_protocol_empty (obj->data, obj->size);
3955 los_memory_usage -= size;
3956 size += sizeof (LOSObject);
3957 size += pagesize - 1;
3958 size &= ~(pagesize - 1);
3959 total_alloc -= size;
3961 free_os_memory (obj, size);
3965 * Objects with size >= 64KB are allocated in the large object space.
3966 * They are currently kept track of with a linked list.
3967 * They don't move, so there is no need to pin them during collection
3968 * and we avoid the memcpy overhead.
3970 static void* __attribute__((noinline))
3971 alloc_large_inner (MonoVTable *vtable, size_t size)
3977 g_assert (size > MAX_SMALL_OBJ_SIZE);
3979 if (need_major_collection ()) {
3980 DEBUG (4, fprintf (gc_debug_file, "Should trigger major collection: req size %zd (los already: %zu)\n", size, los_memory_usage));
3982 major_collection ("LOS overflow");
3986 alloc_size += sizeof (LOSObject);
3987 alloc_size += pagesize - 1;
3988 alloc_size &= ~(pagesize - 1);
3989 /* FIXME: handle OOM */
3990 obj = get_os_memory (alloc_size, TRUE);
3991 g_assert (!((mword)obj->data & (ALLOC_ALIGN - 1)));
3993 vtslot = (void**)obj->data;
3995 total_alloc += alloc_size;
3996 UPDATE_HEAP_BOUNDARIES (obj->data, (char*)obj->data + size);
3997 obj->next = los_object_list;
3998 los_object_list = obj;
3999 los_memory_usage += size;
4001 DEBUG (4, fprintf (gc_debug_file, "Allocated large object %p, vtable: %p (%s), size: %zd\n", obj->data, vtable, vtable->klass->name, size));
4002 binary_protocol_alloc (obj->data, vtable, size);
4007 setup_fragment (Fragment *frag, Fragment *prev, size_t size)
4009 /* remove from the list */
4011 prev->next = frag->next;
4013 nursery_fragments = frag->next;
4014 nursery_next = frag->fragment_start;
4015 nursery_frag_real_end = frag->fragment_end;
4017 DEBUG (4, fprintf (gc_debug_file, "Using nursery fragment %p-%p, size: %zd (req: %zd)\n", nursery_next, nursery_frag_real_end, nursery_frag_real_end - nursery_next, size));
4018 frag->next = fragment_freelist;
4019 fragment_freelist = frag;
4022 /* check if we have a suitable fragment in nursery_fragments to be able to allocate
4023 * an object of size @size
4024 * Return FALSE if not found (which means we need a collection)
4027 search_fragment_for_size (size_t size)
4029 Fragment *frag, *prev;
4030 DEBUG (4, fprintf (gc_debug_file, "Searching nursery fragment %p, size: %zd\n", nursery_frag_real_end, size));
4032 if (nursery_frag_real_end > nursery_next && nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
4033 /* Clear the remaining space, pinning depends on this */
4034 memset (nursery_next, 0, nursery_frag_real_end - nursery_next);
4037 for (frag = nursery_fragments; frag; frag = frag->next) {
4038 if (size <= (frag->fragment_end - frag->fragment_start)) {
4039 setup_fragment (frag, prev, size);
4048 * Same as search_fragment_for_size but if search for @desired_size fails, try to satisfy @minimum_size.
4049 * This improves nursery usage.
4052 search_fragment_for_size_range (size_t desired_size, size_t minimum_size)
4054 Fragment *frag, *prev, *min_prev;
4055 DEBUG (4, fprintf (gc_debug_file, "Searching nursery fragment %p, desired size: %zd minimum size %zd\n", nursery_frag_real_end, desired_size, minimum_size));
4057 if (nursery_frag_real_end > nursery_next && nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
4058 /* Clear the remaining space, pinning depends on this */
4059 memset (nursery_next, 0, nursery_frag_real_end - nursery_next);
4061 min_prev = GINT_TO_POINTER (-1);
4064 for (frag = nursery_fragments; frag; frag = frag->next) {
4065 int frag_size = frag->fragment_end - frag->fragment_start;
4066 if (desired_size <= frag_size) {
4067 setup_fragment (frag, prev, desired_size);
4068 return desired_size;
4070 if (minimum_size <= frag_size)
4076 if (min_prev != GINT_TO_POINTER (-1)) {
4079 frag = min_prev->next;
4081 frag = nursery_fragments;
4083 frag_size = frag->fragment_end - frag->fragment_start;
4084 HEAVY_STAT (++stat_wasted_fragments_used);
4085 HEAVY_STAT (stat_wasted_fragments_bytes += frag_size);
4087 setup_fragment (frag, min_prev, minimum_size);
4095 * Provide a variant that takes just the vtable for small fixed-size objects.
4096 * The aligned size is already computed and stored in vt->gc_descr.
4097 * Note: every SCAN_START_SIZE or so we are given the chance to do some special
4098 * processing. We can keep track of where objects start, for example,
4099 * so when we scan the thread stacks for pinned objects, we can start
4100 * a search for the pinned object in SCAN_START_SIZE chunks.
4103 mono_gc_alloc_obj_nolock (MonoVTable *vtable, size_t size)
4105 /* FIXME: handle OOM */
4110 HEAVY_STAT (++stat_objects_alloced);
4111 if (size <= MAX_SMALL_OBJ_SIZE)
4112 HEAVY_STAT (stat_bytes_alloced += size);
4114 HEAVY_STAT (stat_bytes_alloced_los += size);
4116 size += ALLOC_ALIGN - 1;
4117 size &= ~(ALLOC_ALIGN - 1);
4119 g_assert (vtable->gc_descr);
4121 if (G_UNLIKELY (collect_before_allocs)) {
4122 if (nursery_section) {
4124 collect_nursery (0);
4126 if (!degraded_mode && !search_fragment_for_size (size)) {
4128 g_assert_not_reached ();
4134 * We must already have the lock here instead of after the
4135 * fast path because we might be interrupted in the fast path
4136 * (after confirming that new_next < TLAB_TEMP_END) by the GC,
4137 * and we'll end up allocating an object in a fragment which
4138 * no longer belongs to us.
4140 * The managed allocator does not do this, but it's treated
4141 * specially by the world-stopping code.
4144 if (size > MAX_SMALL_OBJ_SIZE) {
4145 p = alloc_large_inner (vtable, size);
4147 /* tlab_next and tlab_temp_end are TLS vars so accessing them might be expensive */
4149 p = (void**)TLAB_NEXT;
4150 /* FIXME: handle overflow */
4151 new_next = (char*)p + size;
4152 TLAB_NEXT = new_next;
4154 if (G_LIKELY (new_next < TLAB_TEMP_END)) {
4158 * FIXME: We might need a memory barrier here so the change to tlab_next is
4159 * visible before the vtable store.
4162 DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
4163 binary_protocol_alloc (p , vtable, size);
4164 g_assert (*p == NULL);
4167 g_assert (TLAB_NEXT == new_next);
4174 /* there are two cases: the object is too big or we run out of space in the TLAB */
4175 /* we also reach here when the thread does its first allocation after a minor
4176 * collection, since the tlab_ variables are initialized to NULL.
4177 * there can be another case (from ORP), if we cooperate with the runtime a bit:
4178 * objects that need finalizers can have the high bit set in their size
4179 * so the above check fails and we can readily add the object to the queue.
4180 * This avoids taking again the GC lock when registering, but this is moot when
4181 * doing thread-local allocation, so it may not be a good idea.
4183 g_assert (TLAB_NEXT == new_next);
4184 if (TLAB_NEXT >= TLAB_REAL_END) {
4186 * Run out of space in the TLAB. When this happens, some amount of space
4187 * remains in the TLAB, but not enough to satisfy the current allocation
4188 * request. Currently, we retire the TLAB in all cases, later we could
4189 * keep it if the remaining space is above a treshold, and satisfy the
4190 * allocation directly from the nursery.
4193 /* when running in degraded mode, we continue allocing that way
4194 * for a while, to decrease the number of useless nursery collections.
4196 if (degraded_mode && degraded_mode < DEFAULT_NURSERY_SIZE) {
4197 p = alloc_degraded (vtable, size);
4201 /*FIXME This codepath is current deadcode since tlab_size > MAX_SMALL_OBJ_SIZE*/
4202 if (size > tlab_size) {
4203 /* Allocate directly from the nursery */
4204 if (nursery_next + size >= nursery_frag_real_end) {
4205 if (!search_fragment_for_size (size)) {
4206 minor_collect_or_expand_inner (size);
4207 if (degraded_mode) {
4208 p = alloc_degraded (vtable, size);
4214 p = (void*)nursery_next;
4215 nursery_next += size;
4216 if (nursery_next > nursery_frag_real_end) {
4221 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
4222 memset (p, 0, size);
4224 int alloc_size = tlab_size;
4225 int available_in_nursery = nursery_frag_real_end - nursery_next;
4227 DEBUG (3, fprintf (gc_debug_file, "Retire TLAB: %p-%p [%ld]\n", TLAB_START, TLAB_REAL_END, (long)(TLAB_REAL_END - TLAB_NEXT - size)));
4229 if (alloc_size >= available_in_nursery) {
4230 if (available_in_nursery > MAX_NURSERY_TLAB_WASTE && available_in_nursery > size) {
4231 alloc_size = available_in_nursery;
4233 alloc_size = search_fragment_for_size_range (tlab_size, size);
4235 alloc_size = tlab_size;
4236 minor_collect_or_expand_inner (tlab_size);
4237 if (degraded_mode) {
4238 p = alloc_degraded (vtable, size);
4245 /* Allocate a new TLAB from the current nursery fragment */
4246 TLAB_START = nursery_next;
4247 nursery_next += alloc_size;
4248 TLAB_NEXT = TLAB_START;
4249 TLAB_REAL_END = TLAB_START + alloc_size;
4250 TLAB_TEMP_END = TLAB_START + MIN (SCAN_START_SIZE, alloc_size);
4252 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
4253 memset (TLAB_START, 0, alloc_size);
4255 /* Allocate from the TLAB */
4256 p = (void*)TLAB_NEXT;
4258 g_assert (TLAB_NEXT <= TLAB_REAL_END);
4260 nursery_section->scan_starts [((char*)p - (char*)nursery_section->data)/SCAN_START_SIZE] = (char*)p;
4263 /* Reached tlab_temp_end */
4265 /* record the scan start so we can find pinned objects more easily */
4266 nursery_section->scan_starts [((char*)p - (char*)nursery_section->data)/SCAN_START_SIZE] = (char*)p;
4267 /* we just bump tlab_temp_end as well */
4268 TLAB_TEMP_END = MIN (TLAB_REAL_END, TLAB_NEXT + SCAN_START_SIZE);
4269 DEBUG (5, fprintf (gc_debug_file, "Expanding local alloc: %p-%p\n", TLAB_NEXT, TLAB_TEMP_END));
4273 DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
4274 binary_protocol_alloc (p, vtable, size);
4281 mono_gc_try_alloc_obj_nolock (MonoVTable *vtable, size_t size)
4287 size += ALLOC_ALIGN - 1;
4288 size &= ~(ALLOC_ALIGN - 1);
4290 g_assert (vtable->gc_descr);
4291 if (size <= MAX_SMALL_OBJ_SIZE) {
4292 /* tlab_next and tlab_temp_end are TLS vars so accessing them might be expensive */
4294 p = (void**)TLAB_NEXT;
4295 /* FIXME: handle overflow */
4296 new_next = (char*)p + size;
4297 TLAB_NEXT = new_next;
4299 if (G_LIKELY (new_next < TLAB_TEMP_END)) {
4303 * FIXME: We might need a memory barrier here so the change to tlab_next is
4304 * visible before the vtable store.
4307 HEAVY_STAT (++stat_objects_alloced);
4308 HEAVY_STAT (stat_bytes_alloced += size);
4310 DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
4311 binary_protocol_alloc (p, vtable, size);
4312 g_assert (*p == NULL);
4315 g_assert (TLAB_NEXT == new_next);
4324 mono_gc_alloc_obj (MonoVTable *vtable, size_t size)
4327 #ifndef DISABLE_CRITICAL_REGION
4329 ENTER_CRITICAL_REGION;
4330 res = mono_gc_try_alloc_obj_nolock (vtable, size);
4332 EXIT_CRITICAL_REGION;
4335 EXIT_CRITICAL_REGION;
4338 res = mono_gc_alloc_obj_nolock (vtable, size);
4344 mono_gc_alloc_vector (MonoVTable *vtable, size_t size, uintptr_t max_length)
4347 #ifndef DISABLE_CRITICAL_REGION
4349 ENTER_CRITICAL_REGION;
4350 arr = mono_gc_try_alloc_obj_nolock (vtable, size);
4352 arr->max_length = max_length;
4353 EXIT_CRITICAL_REGION;
4356 EXIT_CRITICAL_REGION;
4361 arr = mono_gc_alloc_obj_nolock (vtable, size);
4362 arr->max_length = max_length;
4370 mono_gc_alloc_array (MonoVTable *vtable, size_t size, uintptr_t max_length, uintptr_t bounds_size)
4373 MonoArrayBounds *bounds;
4377 arr = mono_gc_alloc_obj_nolock (vtable, size);
4378 arr->max_length = max_length;
4380 bounds = (MonoArrayBounds*)((char*)arr + size - bounds_size);
4381 arr->bounds = bounds;
4389 mono_gc_alloc_string (MonoVTable *vtable, size_t size, gint32 len)
4392 #ifndef DISABLE_CRITICAL_REGION
4394 ENTER_CRITICAL_REGION;
4395 str = mono_gc_try_alloc_obj_nolock (vtable, size);
4398 EXIT_CRITICAL_REGION;
4401 EXIT_CRITICAL_REGION;
4406 str = mono_gc_alloc_obj_nolock (vtable, size);
4415 * To be used for interned strings and possibly MonoThread, reflection handles.
4416 * We may want to explicitly free these objects.
4419 mono_gc_alloc_pinned_obj (MonoVTable *vtable, size_t size)
4421 /* FIXME: handle OOM */
4423 size += ALLOC_ALIGN - 1;
4424 size &= ~(ALLOC_ALIGN - 1);
4426 if (size > MAX_SMALL_OBJ_SIZE) {
4427 /* large objects are always pinned anyway */
4428 p = alloc_large_inner (vtable, size);
4430 DEBUG (9, g_assert (vtable->klass->inited));
4431 p = major_alloc_small_pinned_obj (size, vtable->klass->has_references);
4433 DEBUG (6, fprintf (gc_debug_file, "Allocated pinned object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
4434 binary_protocol_alloc (p, vtable, size);
4441 * ######################################################################
4442 * ######## Finalization support
4443 * ######################################################################
4447 * this is valid for the nursery: if the object has been forwarded it means it's
4448 * still refrenced from a root. If it is pinned it's still alive as well.
4449 * Return TRUE if @obj is ready to be finalized.
4451 #define object_is_fin_ready(obj) (!object_is_pinned (obj) && !object_is_forwarded (obj))
4454 is_critical_finalizer (FinalizeEntry *entry)
4459 if (!mono_defaults.critical_finalizer_object)
4462 obj = entry->object;
4463 class = ((MonoVTable*)LOAD_VTABLE (obj))->klass;
4465 return mono_class_has_parent (class, mono_defaults.critical_finalizer_object);
4469 queue_finalization_entry (FinalizeEntry *entry) {
4470 if (is_critical_finalizer (entry)) {
4471 entry->next = critical_fin_list;
4472 critical_fin_list = entry;
4474 entry->next = fin_ready_list;
4475 fin_ready_list = entry;
4479 /* LOCKING: requires that the GC lock is held */
4481 rehash_fin_table (FinalizeEntryHashTable *hash_table)
4483 FinalizeEntry **finalizable_hash = hash_table->table;
4484 mword finalizable_hash_size = hash_table->size;
4487 FinalizeEntry **new_hash;
4488 FinalizeEntry *entry, *next;
4489 int new_size = g_spaced_primes_closest (hash_table->num_registered);
4491 new_hash = get_internal_mem (new_size * sizeof (FinalizeEntry*), INTERNAL_MEM_FIN_TABLE);
4492 for (i = 0; i < finalizable_hash_size; ++i) {
4493 for (entry = finalizable_hash [i]; entry; entry = next) {
4494 hash = mono_object_hash (entry->object) % new_size;
4496 entry->next = new_hash [hash];
4497 new_hash [hash] = entry;
4500 free_internal_mem (finalizable_hash, INTERNAL_MEM_FIN_TABLE);
4501 hash_table->table = new_hash;
4502 hash_table->size = new_size;
4505 /* LOCKING: requires that the GC lock is held */
4507 rehash_fin_table_if_necessary (FinalizeEntryHashTable *hash_table)
4509 if (hash_table->num_registered >= hash_table->size * 2)
4510 rehash_fin_table (hash_table);
4513 /* LOCKING: requires that the GC lock is held */
4515 finalize_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, int generation)
4517 FinalizeEntryHashTable *hash_table = get_finalize_entry_hash_table (generation);
4518 FinalizeEntry *entry, *prev;
4520 FinalizeEntry **finalizable_hash = hash_table->table;
4521 mword finalizable_hash_size = hash_table->size;
4525 for (i = 0; i < finalizable_hash_size; ++i) {
4527 for (entry = finalizable_hash [i]; entry;) {
4528 if ((char*)entry->object >= start && (char*)entry->object < end && !major_is_object_live (entry->object)) {
4529 gboolean is_fin_ready = object_is_fin_ready (entry->object);
4530 char *copy = entry->object;
4531 copy_func ((void**)©);
4534 FinalizeEntry *next;
4535 /* remove and put in fin_ready_list */
4537 prev->next = entry->next;
4539 finalizable_hash [i] = entry->next;
4541 num_ready_finalizers++;
4542 hash_table->num_registered--;
4543 queue_finalization_entry (entry);
4544 /* Make it survive */
4545 from = entry->object;
4546 entry->object = copy;
4547 DEBUG (5, fprintf (gc_debug_file, "Queueing object for finalization: %p (%s) (was at %p) (%d/%d)\n", entry->object, safe_name (entry->object), from, num_ready_finalizers, hash_table->num_registered));
4551 char *from = entry->object;
4552 if (hash_table == &minor_finalizable_hash && !ptr_in_nursery (copy)) {
4553 FinalizeEntry *next = entry->next;
4554 unsigned int major_hash;
4555 /* remove from the list */
4557 prev->next = entry->next;
4559 finalizable_hash [i] = entry->next;
4560 hash_table->num_registered--;
4562 entry->object = copy;
4564 /* insert it into the major hash */
4565 rehash_fin_table_if_necessary (&major_finalizable_hash);
4566 major_hash = mono_object_hash ((MonoObject*) copy) %
4567 major_finalizable_hash.size;
4568 entry->next = major_finalizable_hash.table [major_hash];
4569 major_finalizable_hash.table [major_hash] = entry;
4570 major_finalizable_hash.num_registered++;
4572 DEBUG (5, fprintf (gc_debug_file, "Promoting finalization of object %p (%s) (was at %p) to major table\n", copy, safe_name (copy), from));
4577 /* update pointer */
4578 DEBUG (5, fprintf (gc_debug_file, "Updating object for finalization: %p (%s) (was at %p)\n", entry->object, safe_name (entry->object), from));
4579 entry->object = copy;
4584 entry = entry->next;
4590 object_is_reachable (char *object, char *start, char *end)
4592 /*This happens for non nursery objects during minor collections. We just treat all objects as alive.*/
4593 if (object < start || object >= end)
4595 return !object_is_fin_ready (object) || major_is_object_live (object);
4598 /* LOCKING: requires that the GC lock is held */
4600 null_ephemerons_for_domain (MonoDomain *domain)
4602 EphemeronLinkNode *current = ephemeron_list, *prev = NULL;
4605 MonoObject *object = (MonoObject*)current->array;
4607 if (object && !object->vtable) {
4608 EphemeronLinkNode *tmp = current;
4611 prev->next = current->next;
4613 ephemeron_list = current->next;
4615 current = current->next;
4616 free_internal_mem (tmp, INTERNAL_MEM_EPHEMERON_LINK);
4619 current = current->next;
4624 /* LOCKING: requires that the GC lock is held */
4626 clear_unreachable_ephemerons (CopyOrMarkObjectFunc copy_func, char *start, char *end)
4628 int was_in_nursery, was_promoted;
4629 EphemeronLinkNode *current = ephemeron_list, *prev = NULL;
4631 Ephemeron *cur, *array_end;
4635 char *object = current->array;
4637 if (!object_is_reachable (object, start, end)) {
4638 EphemeronLinkNode *tmp = current;
4640 DEBUG (5, fprintf (gc_debug_file, "Dead Ephemeron array at %p\n", object));
4643 prev->next = current->next;
4645 ephemeron_list = current->next;
4647 current = current->next;
4648 free_internal_mem (tmp, INTERNAL_MEM_EPHEMERON_LINK);
4653 was_in_nursery = ptr_in_nursery (object);
4654 copy_func ((void**)&object);
4655 current->array = object;
4657 /*The array was promoted, add global remsets for key/values left behind in nursery.*/
4658 was_promoted = was_in_nursery && !ptr_in_nursery (object);
4660 DEBUG (5, fprintf (gc_debug_file, "Clearing unreachable entries for ephemeron array at %p\n", object));
4662 array = (MonoArray*)object;
4663 cur = mono_array_addr (array, Ephemeron, 0);
4664 array_end = cur + mono_array_length_fast (array);
4665 tombstone = (char*)((MonoVTable*)LOAD_VTABLE (object))->domain->ephemeron_tombstone;
4667 for (; cur < array_end; ++cur) {
4668 char *key = (char*)cur->key;
4670 if (!key || key == tombstone)
4673 DEBUG (5, fprintf (gc_debug_file, "[%d] key %p (%s) value %p (%s)\n", cur - mono_array_addr (array, Ephemeron, 0),
4674 key, object_is_reachable (key, start, end) ? "reachable" : "unreachable",
4675 cur->value, cur->value && object_is_reachable (cur->value, start, end) ? "reachable" : "unreachable"));
4677 if (!object_is_reachable (key, start, end)) {
4678 cur->key = tombstone;
4684 if (ptr_in_nursery (key)) {/*key was not promoted*/
4685 DEBUG (5, fprintf (gc_debug_file, "\tAdded remset to key %p\n", key));
4686 add_to_global_remset (&cur->key);
4688 if (ptr_in_nursery (cur->value)) {/*value was not promoted*/
4689 DEBUG (5, fprintf (gc_debug_file, "\tAdded remset to value %p\n", cur->value));
4690 add_to_global_remset (&cur->value);
4695 current = current->next;
4699 /* LOCKING: requires that the GC lock is held */
4701 mark_ephemerons_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end)
4703 int nothing_marked = 1;
4704 EphemeronLinkNode *current = ephemeron_list;
4706 Ephemeron *cur, *array_end;
4709 for (current = ephemeron_list; current; current = current->next) {
4710 char *object = current->array;
4711 DEBUG (5, fprintf (gc_debug_file, "Ephemeron array at %p\n", object));
4713 /*We ignore arrays in old gen during minor collections since all objects are promoted by the remset machinery.*/
4714 if (object < start || object >= end)
4717 /*It has to be alive*/
4718 if (!object_is_reachable (object, start, end)) {
4719 DEBUG (5, fprintf (gc_debug_file, "\tnot reachable\n"));
4723 copy_func ((void**)&object);
4725 array = (MonoArray*)object;
4726 cur = mono_array_addr (array, Ephemeron, 0);
4727 array_end = cur + mono_array_length_fast (array);
4728 tombstone = (char*)((MonoVTable*)LOAD_VTABLE (object))->domain->ephemeron_tombstone;
4730 for (; cur < array_end; ++cur) {
4731 char *key = cur->key;
4733 if (!key || key == tombstone)
4736 DEBUG (5, fprintf (gc_debug_file, "[%d] key %p (%s) value %p (%s)\n", cur - mono_array_addr (array, Ephemeron, 0),
4737 key, object_is_reachable (key, start, end) ? "reachable" : "unreachable",
4738 cur->value, cur->value && object_is_reachable (cur->value, start, end) ? "reachable" : "unreachable"));
4740 if (object_is_reachable (key, start, end)) {
4741 char *value = cur->value;
4743 copy_func ((void**)&cur->key);
4745 if (!object_is_reachable (value, start, end))
4747 copy_func ((void**)&cur->value);
4753 DEBUG (5, fprintf (gc_debug_file, "Ephemeron run finished. Is it done %d\n", nothing_marked));
4754 return nothing_marked;
4757 /* LOCKING: requires that the GC lock is held */
4759 null_link_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, int generation)
4761 DisappearingLinkHashTable *hash = get_dislink_hash_table (generation);
4762 DisappearingLink **disappearing_link_hash = hash->table;
4763 int disappearing_link_hash_size = hash->size;
4764 DisappearingLink *entry, *prev;
4766 if (!hash->num_links)
4768 for (i = 0; i < disappearing_link_hash_size; ++i) {
4770 for (entry = disappearing_link_hash [i]; entry;) {
4771 char *object = DISLINK_OBJECT (entry);
4772 if (object >= start && object < end && !major_is_object_live (object)) {
4773 gboolean track = DISLINK_TRACK (entry);
4774 if (!track && object_is_fin_ready (object)) {
4775 void **p = entry->link;
4776 DisappearingLink *old;
4778 /* remove from list */
4780 prev->next = entry->next;
4782 disappearing_link_hash [i] = entry->next;
4783 DEBUG (5, fprintf (gc_debug_file, "Dislink nullified at %p to GCed object %p\n", p, object));
4785 free_internal_mem (entry, INTERNAL_MEM_DISLINK);
4790 char *copy = object;
4791 copy_func ((void**)©);
4793 /* Update pointer if it's moved. If the object
4794 * has been moved out of the nursery, we need to
4795 * remove the link from the minor hash table to
4798 * FIXME: what if an object is moved earlier?
4801 if (hash == &minor_disappearing_link_hash && !ptr_in_nursery (copy)) {
4802 void **link = entry->link;
4803 DisappearingLink *old;
4804 /* remove from list */
4806 prev->next = entry->next;
4808 disappearing_link_hash [i] = entry->next;
4810 free_internal_mem (entry, INTERNAL_MEM_DISLINK);
4814 add_or_remove_disappearing_link ((MonoObject*)copy, link,
4815 track, GENERATION_OLD);
4817 DEBUG (5, fprintf (gc_debug_file, "Upgraded dislink at %p to major because object %p moved to %p\n", link, object, copy));
4821 /* We set the track resurrection bit to
4822 * FALSE if the object is to be finalized
4823 * so that the object can be collected in
4824 * the next cycle (i.e. after it was
4827 *entry->link = HIDE_POINTER (copy,
4828 object_is_fin_ready (object) ? FALSE : track);
4829 DEBUG (5, fprintf (gc_debug_file, "Updated dislink at %p to %p\n", entry->link, DISLINK_OBJECT (entry)));
4834 entry = entry->next;
4839 /* LOCKING: requires that the GC lock is held */
4841 null_links_for_domain (MonoDomain *domain, int generation)
4843 DisappearingLinkHashTable *hash = get_dislink_hash_table (generation);
4844 DisappearingLink **disappearing_link_hash = hash->table;
4845 int disappearing_link_hash_size = hash->size;
4846 DisappearingLink *entry, *prev;
4848 for (i = 0; i < disappearing_link_hash_size; ++i) {
4850 for (entry = disappearing_link_hash [i]; entry; ) {
4851 char *object = DISLINK_OBJECT (entry);
4852 if (object && !((MonoObject*)object)->vtable) {
4853 DisappearingLink *next = entry->next;
4858 disappearing_link_hash [i] = next;
4860 if (*(entry->link)) {
4861 *(entry->link) = NULL;
4862 g_warning ("Disappearing link %p not freed", entry->link);
4864 free_internal_mem (entry, INTERNAL_MEM_DISLINK);
4871 entry = entry->next;
4876 /* LOCKING: requires that the GC lock is held */
4878 finalizers_for_domain (MonoDomain *domain, MonoObject **out_array, int out_size,
4879 FinalizeEntryHashTable *hash_table)
4881 FinalizeEntry **finalizable_hash = hash_table->table;
4882 mword finalizable_hash_size = hash_table->size;
4883 FinalizeEntry *entry, *prev;
4886 if (no_finalize || !out_size || !out_array)
4889 for (i = 0; i < finalizable_hash_size; ++i) {
4891 for (entry = finalizable_hash [i]; entry;) {
4892 if (mono_object_domain (entry->object) == domain) {
4893 FinalizeEntry *next;
4894 /* remove and put in out_array */
4896 prev->next = entry->next;
4898 finalizable_hash [i] = entry->next;
4900 hash_table->num_registered--;
4901 out_array [count ++] = entry->object;
4902 DEBUG (5, fprintf (gc_debug_file, "Collecting object for finalization: %p (%s) (%d/%d)\n", entry->object, safe_name (entry->object), num_ready_finalizers, hash_table->num_registered));
4904 if (count == out_size)
4909 entry = entry->next;
4916 * mono_gc_finalizers_for_domain:
4917 * @domain: the unloading appdomain
4918 * @out_array: output array
4919 * @out_size: size of output array
4921 * Store inside @out_array up to @out_size objects that belong to the unloading
4922 * appdomain @domain. Returns the number of stored items. Can be called repeteadly
4923 * until it returns 0.
4924 * The items are removed from the finalizer data structure, so the caller is supposed
4926 * @out_array should be on the stack to allow the GC to know the objects are still alive.
4929 mono_gc_finalizers_for_domain (MonoDomain *domain, MonoObject **out_array, int out_size)
4934 result = finalizers_for_domain (domain, out_array, out_size, &minor_finalizable_hash);
4935 if (result < out_size) {
4936 result += finalizers_for_domain (domain, out_array + result, out_size - result,
4937 &major_finalizable_hash);
4945 register_for_finalization (MonoObject *obj, void *user_data, int generation)
4947 FinalizeEntryHashTable *hash_table = get_finalize_entry_hash_table (generation);
4948 FinalizeEntry **finalizable_hash;
4949 mword finalizable_hash_size;
4950 FinalizeEntry *entry, *prev;
4954 g_assert (user_data == NULL || user_data == mono_gc_run_finalize);
4955 hash = mono_object_hash (obj);
4957 rehash_fin_table_if_necessary (hash_table);
4958 finalizable_hash = hash_table->table;
4959 finalizable_hash_size = hash_table->size;
4960 hash %= finalizable_hash_size;
4962 for (entry = finalizable_hash [hash]; entry; entry = entry->next) {
4963 if (entry->object == obj) {
4965 /* remove from the list */
4967 prev->next = entry->next;
4969 finalizable_hash [hash] = entry->next;
4970 hash_table->num_registered--;
4971 DEBUG (5, fprintf (gc_debug_file, "Removed finalizer %p for object: %p (%s) (%d)\n", entry, obj, obj->vtable->klass->name, hash_table->num_registered));
4972 free_internal_mem (entry, INTERNAL_MEM_FINALIZE_ENTRY);
4980 /* request to deregister, but already out of the list */
4984 entry = get_internal_mem (sizeof (FinalizeEntry), INTERNAL_MEM_FINALIZE_ENTRY);
4985 entry->object = obj;
4986 entry->next = finalizable_hash [hash];
4987 finalizable_hash [hash] = entry;
4988 hash_table->num_registered++;
4989 DEBUG (5, fprintf (gc_debug_file, "Added finalizer %p for object: %p (%s) (%d) to %s table\n", entry, obj, obj->vtable->klass->name, hash_table->num_registered, generation_name (generation)));
4994 mono_gc_register_for_finalization (MonoObject *obj, void *user_data)
4996 if (ptr_in_nursery (obj))
4997 register_for_finalization (obj, user_data, GENERATION_NURSERY);
4999 register_for_finalization (obj, user_data, GENERATION_OLD);
5003 rehash_dislink (DisappearingLinkHashTable *hash_table)
5005 DisappearingLink **disappearing_link_hash = hash_table->table;
5006 int disappearing_link_hash_size = hash_table->size;
5009 DisappearingLink **new_hash;
5010 DisappearingLink *entry, *next;
5011 int new_size = g_spaced_primes_closest (hash_table->num_links);
5013 new_hash = get_internal_mem (new_size * sizeof (DisappearingLink*), INTERNAL_MEM_DISLINK_TABLE);
5014 for (i = 0; i < disappearing_link_hash_size; ++i) {
5015 for (entry = disappearing_link_hash [i]; entry; entry = next) {
5016 hash = mono_aligned_addr_hash (entry->link) % new_size;
5018 entry->next = new_hash [hash];
5019 new_hash [hash] = entry;
5022 free_internal_mem (disappearing_link_hash, INTERNAL_MEM_DISLINK_TABLE);
5023 hash_table->table = new_hash;
5024 hash_table->size = new_size;
5027 /* LOCKING: assumes the GC lock is held */
5029 add_or_remove_disappearing_link (MonoObject *obj, void **link, gboolean track, int generation)
5031 DisappearingLinkHashTable *hash_table = get_dislink_hash_table (generation);
5032 DisappearingLink *entry, *prev;
5034 DisappearingLink **disappearing_link_hash = hash_table->table;
5035 int disappearing_link_hash_size = hash_table->size;
5037 if (hash_table->num_links >= disappearing_link_hash_size * 2) {
5038 rehash_dislink (hash_table);
5039 disappearing_link_hash = hash_table->table;
5040 disappearing_link_hash_size = hash_table->size;
5042 /* FIXME: add check that link is not in the heap */
5043 hash = mono_aligned_addr_hash (link) % disappearing_link_hash_size;
5044 entry = disappearing_link_hash [hash];
5046 for (; entry; entry = entry->next) {
5047 /* link already added */
5048 if (link == entry->link) {
5049 /* NULL obj means remove */
5052 prev->next = entry->next;
5054 disappearing_link_hash [hash] = entry->next;
5055 hash_table->num_links--;
5056 DEBUG (5, fprintf (gc_debug_file, "Removed dislink %p (%d) from %s table\n", entry, hash_table->num_links, generation_name (generation)));
5057 free_internal_mem (entry, INTERNAL_MEM_DISLINK);
5060 *link = HIDE_POINTER (obj, track); /* we allow the change of object */
5068 entry = get_internal_mem (sizeof (DisappearingLink), INTERNAL_MEM_DISLINK);
5069 *link = HIDE_POINTER (obj, track);
5071 entry->next = disappearing_link_hash [hash];
5072 disappearing_link_hash [hash] = entry;
5073 hash_table->num_links++;
5074 DEBUG (5, fprintf (gc_debug_file, "Added dislink %p for object: %p (%s) at %p to %s table\n", entry, obj, obj->vtable->klass->name, link, generation_name (generation)));
5077 /* LOCKING: assumes the GC lock is held */
5079 mono_gc_register_disappearing_link (MonoObject *obj, void **link, gboolean track)
5081 add_or_remove_disappearing_link (NULL, link, FALSE, GENERATION_NURSERY);
5082 add_or_remove_disappearing_link (NULL, link, FALSE, GENERATION_OLD);
5084 if (ptr_in_nursery (obj))
5085 add_or_remove_disappearing_link (obj, link, track, GENERATION_NURSERY);
5087 add_or_remove_disappearing_link (obj, link, track, GENERATION_OLD);
5092 mono_gc_invoke_finalizers (void)
5094 FinalizeEntry *entry = NULL;
5095 gboolean entry_is_critical = FALSE;
5098 /* FIXME: batch to reduce lock contention */
5099 while (fin_ready_list || critical_fin_list) {
5103 FinalizeEntry **list = entry_is_critical ? &critical_fin_list : &fin_ready_list;
5105 /* We have finalized entry in the last
5106 interation, now we need to remove it from
5109 *list = entry->next;
5111 FinalizeEntry *e = *list;
5112 while (e->next != entry)
5114 e->next = entry->next;
5116 free_internal_mem (entry, INTERNAL_MEM_FINALIZE_ENTRY);
5120 /* Now look for the first non-null entry. */
5121 for (entry = fin_ready_list; entry && !entry->object; entry = entry->next)
5124 entry_is_critical = FALSE;
5126 entry_is_critical = TRUE;
5127 for (entry = critical_fin_list; entry && !entry->object; entry = entry->next)
5132 g_assert (entry->object);
5133 num_ready_finalizers--;
5134 obj = entry->object;
5135 entry->object = NULL;
5136 DEBUG (7, fprintf (gc_debug_file, "Finalizing object %p (%s)\n", obj, safe_name (obj)));
5144 g_assert (entry->object == NULL);
5146 /* the object is on the stack so it is pinned */
5147 /*g_print ("Calling finalizer for object: %p (%s)\n", entry->object, safe_name (entry->object));*/
5148 mono_gc_run_finalize (obj, NULL);
5155 mono_gc_pending_finalizers (void)
5157 return fin_ready_list || critical_fin_list;
5160 /* Negative value to remove */
5162 mono_gc_add_memory_pressure (gint64 value)
5164 /* FIXME: Use interlocked functions */
5166 memory_pressure += value;
5171 * ######################################################################
5172 * ######## registered roots support
5173 * ######################################################################
5177 rehash_roots (gboolean pinned)
5181 RootRecord **new_hash;
5182 RootRecord *entry, *next;
5185 new_size = g_spaced_primes_closest (num_roots_entries [pinned]);
5186 new_hash = get_internal_mem (new_size * sizeof (RootRecord*), INTERNAL_MEM_ROOTS_TABLE);
5187 for (i = 0; i < roots_hash_size [pinned]; ++i) {
5188 for (entry = roots_hash [pinned][i]; entry; entry = next) {
5189 hash = mono_aligned_addr_hash (entry->start_root) % new_size;
5191 entry->next = new_hash [hash];
5192 new_hash [hash] = entry;
5195 free_internal_mem (roots_hash [pinned], INTERNAL_MEM_ROOTS_TABLE);
5196 roots_hash [pinned] = new_hash;
5197 roots_hash_size [pinned] = new_size;
5201 find_root (int root_type, char *start, guint32 addr_hash)
5203 RootRecord *new_root;
5205 guint32 hash = addr_hash % roots_hash_size [root_type];
5206 for (new_root = roots_hash [root_type][hash]; new_root; new_root = new_root->next) {
5207 /* we allow changing the size and the descriptor (for thread statics etc) */
5208 if (new_root->start_root == start) {
5217 * We do not coalesce roots.
5220 mono_gc_register_root_inner (char *start, size_t size, void *descr, int root_type)
5222 RootRecord *new_root;
5223 unsigned int hash, addr_hash = mono_aligned_addr_hash (start);
5226 for (i = 0; i < ROOT_TYPE_NUM; ++i) {
5227 if (num_roots_entries [i] >= roots_hash_size [i] * 2)
5230 for (i = 0; i < ROOT_TYPE_NUM; ++i) {
5231 new_root = find_root (i, start, addr_hash);
5232 /* we allow changing the size and the descriptor (for thread statics etc) */
5234 size_t old_size = new_root->end_root - new_root->start_root;
5235 new_root->end_root = new_root->start_root + size;
5236 g_assert (((new_root->root_desc != 0) && (descr != NULL)) ||
5237 ((new_root->root_desc == 0) && (descr == NULL)));
5238 new_root->root_desc = (mword)descr;
5240 roots_size -= old_size;
5245 new_root = get_internal_mem (sizeof (RootRecord), INTERNAL_MEM_ROOT_RECORD);
5247 new_root->start_root = start;
5248 new_root->end_root = new_root->start_root + size;
5249 new_root->root_desc = (mword)descr;
5251 hash = addr_hash % roots_hash_size [root_type];
5252 num_roots_entries [root_type]++;
5253 new_root->next = roots_hash [root_type] [hash];
5254 roots_hash [root_type][hash] = new_root;
5255 DEBUG (3, fprintf (gc_debug_file, "Added root %p for range: %p-%p, descr: %p (%d/%d bytes)\n", new_root, new_root->start_root, new_root->end_root, descr, (int)size, (int)roots_size));
5265 mono_gc_register_root (char *start, size_t size, void *descr)
5267 return mono_gc_register_root_inner (start, size, descr, descr ? ROOT_TYPE_NORMAL : ROOT_TYPE_PINNED);
5271 mono_gc_register_root_wbarrier (char *start, size_t size, void *descr)
5273 return mono_gc_register_root_inner (start, size, descr, ROOT_TYPE_WBARRIER);
5277 mono_gc_deregister_root (char* addr)
5279 RootRecord *tmp, *prev;
5280 unsigned int hash, addr_hash = mono_aligned_addr_hash (addr);
5284 for (root_type = 0; root_type < ROOT_TYPE_NUM; ++root_type) {
5285 hash = addr_hash % roots_hash_size [root_type];
5286 tmp = roots_hash [root_type][hash];
5289 if (tmp->start_root == (char*)addr) {
5291 prev->next = tmp->next;
5293 roots_hash [root_type][hash] = tmp->next;
5294 roots_size -= (tmp->end_root - tmp->start_root);
5295 num_roots_entries [root_type]--;
5296 DEBUG (3, fprintf (gc_debug_file, "Removed root %p for range: %p-%p\n", tmp, tmp->start_root, tmp->end_root));
5297 free_internal_mem (tmp, INTERNAL_MEM_ROOT_RECORD);
5308 * ######################################################################
5309 * ######## Thread handling (stop/start code)
5310 * ######################################################################
5313 /* FIXME: handle large/small config */
5314 #define THREAD_HASH_SIZE 11
5315 #define HASH_PTHREAD_T(id) (((unsigned int)(id) >> 4) * 2654435761u)
5317 static SgenThreadInfo* thread_table [THREAD_HASH_SIZE];
5319 #if USE_SIGNAL_BASED_START_STOP_WORLD
5321 static MonoSemType suspend_ack_semaphore;
5322 static MonoSemType *suspend_ack_semaphore_ptr;
5323 static unsigned int global_stop_count = 0;
5325 static int suspend_signal_num = SIGXFSZ;
5327 static int suspend_signal_num = SIGPWR;
5329 static int restart_signal_num = SIGXCPU;
5330 static sigset_t suspend_signal_mask;
5331 static mword cur_thread_regs [ARCH_NUM_REGS] = {0};
5333 /* LOCKING: assumes the GC lock is held */
5334 static SgenThreadInfo*
5335 thread_info_lookup (ARCH_THREAD_TYPE id)
5337 unsigned int hash = HASH_PTHREAD_T (id) % THREAD_HASH_SIZE;
5338 SgenThreadInfo *info;
5340 info = thread_table [hash];
5341 while (info && !ARCH_THREAD_EQUALS (info->id, id)) {
5348 update_current_thread_stack (void *start)
5350 void *ptr = cur_thread_regs;
5351 SgenThreadInfo *info = thread_info_lookup (ARCH_GET_THREAD ());
5353 info->stack_start = align_pointer (&ptr);
5354 g_assert (info->stack_start >= info->stack_start_limit && info->stack_start < info->stack_end);
5355 ARCH_STORE_REGS (ptr);
5356 info->stopped_regs = ptr;
5357 if (gc_callbacks.thread_suspend_func)
5358 gc_callbacks.thread_suspend_func (info->runtime_data, NULL);
5362 signal_desc (int signum)
5364 if (signum == suspend_signal_num)
5366 if (signum == restart_signal_num)
5372 * Define this and use the "xdomain-checks" MONO_GC_DEBUG option to
5373 * have cross-domain checks in the write barrier.
5375 //#define XDOMAIN_CHECKS_IN_WBARRIER
5377 #ifndef BINARY_PROTOCOL
5378 #ifndef HEAVY_STATISTICS
5379 #define MANAGED_ALLOCATION
5380 #ifndef XDOMAIN_CHECKS_IN_WBARRIER
5381 #define MANAGED_WBARRIER
5387 is_ip_in_managed_allocator (MonoDomain *domain, gpointer ip);
5390 wait_for_suspend_ack (int count)
5394 for (i = 0; i < count; ++i) {
5395 while ((result = MONO_SEM_WAIT (suspend_ack_semaphore_ptr)) != 0) {
5396 if (errno != EINTR) {
5397 g_error ("sem_wait ()");
5403 /* LOCKING: assumes the GC lock is held */
5405 thread_handshake (int signum)
5407 int count, i, result;
5408 SgenThreadInfo *info;
5409 pthread_t me = pthread_self ();
5412 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5413 for (info = thread_table [i]; info; info = info->next) {
5414 DEBUG (4, fprintf (gc_debug_file, "considering thread %p for signal %d (%s)\n", info, signum, signal_desc (signum)));
5415 if (ARCH_THREAD_EQUALS (info->id, me)) {
5416 DEBUG (4, fprintf (gc_debug_file, "Skip (equal): %p, %p\n", (void*)me, (void*)info->id));
5419 /*if (signum == suspend_signal_num && info->stop_count == global_stop_count)
5421 result = pthread_kill (info->id, signum);
5423 DEBUG (4, fprintf (gc_debug_file, "thread %p signal sent\n", info));
5426 DEBUG (4, fprintf (gc_debug_file, "thread %p signal failed: %d (%s)\n", (void*)info->id, result, strerror (result)));
5432 wait_for_suspend_ack (count);
5438 restart_threads_until_none_in_managed_allocator (void)
5440 SgenThreadInfo *info;
5441 int i, result, num_threads_died = 0;
5442 int sleep_duration = -1;
5445 int restart_count = 0, restarted_count = 0;
5446 /* restart all threads that stopped in the
5448 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5449 for (info = thread_table [i]; info; info = info->next) {
5452 if (!info->stack_start || info->in_critical_region ||
5453 is_ip_in_managed_allocator (info->stopped_domain, info->stopped_ip)) {
5454 binary_protocol_thread_restart ((gpointer)info->id);
5455 result = pthread_kill (info->id, restart_signal_num);
5462 /* we set the stopped_ip to
5463 NULL for threads which
5464 we're not restarting so
5465 that we can easily identify
5467 info->stopped_ip = NULL;
5468 info->stopped_domain = NULL;
5472 /* if no threads were restarted, we're done */
5473 if (restart_count == 0)
5476 /* wait for the threads to signal their restart */
5477 wait_for_suspend_ack (restart_count);
5479 if (sleep_duration < 0) {
5483 g_usleep (sleep_duration);
5484 sleep_duration += 10;
5487 /* stop them again */
5488 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5489 for (info = thread_table [i]; info; info = info->next) {
5490 if (info->skip || info->stopped_ip == NULL)
5492 result = pthread_kill (info->id, suspend_signal_num);
5500 /* some threads might have died */
5501 num_threads_died += restart_count - restarted_count;
5502 /* wait for the threads to signal their suspension
5504 wait_for_suspend_ack (restart_count);
5507 return num_threads_died;
5510 /* LOCKING: assumes the GC lock is held (by the stopping thread) */
5512 suspend_handler (int sig, siginfo_t *siginfo, void *context)
5514 SgenThreadInfo *info;
5517 int old_errno = errno;
5518 gpointer regs [ARCH_NUM_REGS];
5519 gpointer stack_start;
5521 id = pthread_self ();
5522 info = thread_info_lookup (id);
5523 info->stopped_domain = mono_domain_get ();
5524 info->stopped_ip = (gpointer) ARCH_SIGCTX_IP (context);
5525 stop_count = global_stop_count;
5526 /* duplicate signal */
5527 if (0 && info->stop_count == stop_count) {
5531 #ifdef HAVE_KW_THREAD
5532 /* update the remset info in the thread data structure */
5533 info->remset = remembered_set;
5535 stack_start = (char*) ARCH_SIGCTX_SP (context) - REDZONE_SIZE;
5536 /* If stack_start is not within the limits, then don't set it
5537 in info and we will be restarted. */
5538 if (stack_start >= info->stack_start_limit && info->stack_start <= info->stack_end) {
5539 info->stack_start = stack_start;
5541 ARCH_COPY_SIGCTX_REGS (regs, context);
5542 info->stopped_regs = regs;
5544 g_assert (!info->stack_start);
5547 /* Notify the JIT */
5548 if (gc_callbacks.thread_suspend_func)
5549 gc_callbacks.thread_suspend_func (info->runtime_data, context);
5551 DEBUG (4, fprintf (gc_debug_file, "Posting suspend_ack_semaphore for suspend from %p %p\n", info, (gpointer)ARCH_GET_THREAD ()));
5552 /* notify the waiting thread */
5553 MONO_SEM_POST (suspend_ack_semaphore_ptr);
5554 info->stop_count = stop_count;
5556 /* wait until we receive the restart signal */
5559 sigsuspend (&suspend_signal_mask);
5560 } while (info->signal != restart_signal_num);
5562 DEBUG (4, fprintf (gc_debug_file, "Posting suspend_ack_semaphore for resume from %p %p\n", info, (gpointer)ARCH_GET_THREAD ()));
5563 /* notify the waiting thread */
5564 MONO_SEM_POST (suspend_ack_semaphore_ptr);
5570 restart_handler (int sig)
5572 SgenThreadInfo *info;
5573 int old_errno = errno;
5575 info = thread_info_lookup (pthread_self ());
5576 info->signal = restart_signal_num;
5577 DEBUG (4, fprintf (gc_debug_file, "Restart handler in %p %p\n", info, (gpointer)ARCH_GET_THREAD ()));
5583 acquire_gc_locks (void)
5589 release_gc_locks (void)
5591 UNLOCK_INTERRUPTION;
5594 static TV_DECLARE (stop_world_time);
5595 static unsigned long max_pause_usec = 0;
5597 /* LOCKING: assumes the GC lock is held */
5603 acquire_gc_locks ();
5605 update_current_thread_stack (&count);
5607 global_stop_count++;
5608 DEBUG (3, fprintf (gc_debug_file, "stopping world n %d from %p %p\n", global_stop_count, thread_info_lookup (ARCH_GET_THREAD ()), (gpointer)ARCH_GET_THREAD ()));
5609 TV_GETTIME (stop_world_time);
5610 count = thread_handshake (suspend_signal_num);
5611 count -= restart_threads_until_none_in_managed_allocator ();
5612 g_assert (count >= 0);
5613 DEBUG (3, fprintf (gc_debug_file, "world stopped %d thread(s)\n", count));
5617 /* LOCKING: assumes the GC lock is held */
5619 restart_world (void)
5622 SgenThreadInfo *info;
5623 TV_DECLARE (end_sw);
5626 /* notify the profiler of the leftovers */
5627 if (G_UNLIKELY (mono_profiler_events & MONO_PROFILE_GC_MOVES)) {
5628 if (moved_objects_idx) {
5629 mono_profiler_gc_moves (moved_objects, moved_objects_idx);
5630 moved_objects_idx = 0;
5633 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5634 for (info = thread_table [i]; info; info = info->next) {
5635 info->stack_start = NULL;
5636 info->stopped_regs = NULL;
5640 release_gc_locks ();
5642 count = thread_handshake (restart_signal_num);
5643 TV_GETTIME (end_sw);
5644 usec = TV_ELAPSED (stop_world_time, end_sw);
5645 max_pause_usec = MAX (usec, max_pause_usec);
5646 DEBUG (2, fprintf (gc_debug_file, "restarted %d thread(s) (pause time: %d usec, max: %d)\n", count, (int)usec, (int)max_pause_usec));
5650 #endif /* USE_SIGNAL_BASED_START_STOP_WORLD */
5653 mono_gc_set_gc_callbacks (MonoGCCallbacks *callbacks)
5655 gc_callbacks = *callbacks;
5658 /* Variables holding start/end nursery so it won't have to be passed at every call */
5659 static void *scan_area_arg_start, *scan_area_arg_end;
5662 mono_gc_conservatively_scan_area (void *start, void *end)
5664 conservatively_pin_objects_from (start, end, scan_area_arg_start, scan_area_arg_end, PIN_TYPE_STACK);
5668 mono_gc_scan_object (void *obj)
5670 if (current_collection_generation == GENERATION_NURSERY)
5673 major_copy_or_mark_object (&obj);
5678 * Mark from thread stacks and registers.
5681 scan_thread_data (void *start_nursery, void *end_nursery, gboolean precise)
5684 SgenThreadInfo *info;
5686 scan_area_arg_start = start_nursery;
5687 scan_area_arg_end = end_nursery;
5689 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5690 for (info = thread_table [i]; info; info = info->next) {
5692 DEBUG (3, fprintf (gc_debug_file, "Skipping dead thread %p, range: %p-%p, size: %zd\n", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start));
5695 DEBUG (3, fprintf (gc_debug_file, "Scanning thread %p, range: %p-%p, size: %zd, pinned=%d\n", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start, next_pin_slot));
5696 if (gc_callbacks.thread_mark_func && !conservative_stack_mark)
5697 gc_callbacks.thread_mark_func (info->runtime_data, info->stack_start, info->stack_end, precise);
5699 conservatively_pin_objects_from (info->stack_start, info->stack_end, start_nursery, end_nursery, PIN_TYPE_STACK);
5702 conservatively_pin_objects_from (info->stopped_regs, info->stopped_regs + ARCH_NUM_REGS,
5703 start_nursery, end_nursery, PIN_TYPE_STACK);
5709 find_pinning_ref_from_thread (char *obj, size_t size)
5712 SgenThreadInfo *info;
5713 char *endobj = obj + size;
5715 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5716 for (info = thread_table [i]; info; info = info->next) {
5717 char **start = (char**)info->stack_start;
5720 while (start < (char**)info->stack_end) {
5721 if (*start >= obj && *start < endobj) {
5722 DEBUG (0, fprintf (gc_debug_file, "Object %p referenced in thread %p (id %p) at %p, stack: %p-%p\n", obj, info, (gpointer)info->id, start, info->stack_start, info->stack_end));
5727 /* FIXME: check info->stopped_regs */
5733 ptr_on_stack (void *ptr)
5735 gpointer stack_start = &stack_start;
5736 SgenThreadInfo *info = thread_info_lookup (ARCH_GET_THREAD ());
5738 if (ptr >= stack_start && ptr < (gpointer)info->stack_end)
5744 handle_remset (mword *p, void *start_nursery, void *end_nursery, gboolean global)
5751 HEAVY_STAT (++stat_global_remsets_processed);
5753 /* FIXME: exclude stack locations */
5754 switch ((*p) & REMSET_TYPE_MASK) {
5755 case REMSET_LOCATION:
5757 //__builtin_prefetch (ptr);
5758 if (((void*)ptr < start_nursery || (void*)ptr >= end_nursery)) {
5759 gpointer old = *ptr;
5761 DEBUG (9, fprintf (gc_debug_file, "Overwrote remset at %p with %p\n", ptr, *ptr));
5763 binary_protocol_ptr_update (ptr, old, *ptr, (gpointer)LOAD_VTABLE (*ptr), safe_object_get_size (*ptr));
5764 if (!global && *ptr >= start_nursery && *ptr < end_nursery) {
5766 * If the object is pinned, each reference to it from nonpinned objects
5767 * becomes part of the global remset, which can grow very large.
5769 DEBUG (9, fprintf (gc_debug_file, "Add to global remset because of pinning %p (%p %s)\n", ptr, *ptr, safe_name (*ptr)));
5770 add_to_global_remset (ptr);
5773 DEBUG (9, fprintf (gc_debug_file, "Skipping remset at %p holding %p\n", ptr, *ptr));
5777 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
5778 if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery))
5781 while (count-- > 0) {
5783 DEBUG (9, fprintf (gc_debug_file, "Overwrote remset at %p with %p (count: %d)\n", ptr, *ptr, (int)count));
5784 if (!global && *ptr >= start_nursery && *ptr < end_nursery)
5785 add_to_global_remset (ptr);
5790 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
5791 if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery))
5793 scan_object ((char*)ptr);
5795 case REMSET_VTYPE: {
5796 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
5797 if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery))
5802 ptr = (void**) scan_vtype ((char*)ptr, desc, start_nursery, end_nursery);
5806 g_assert_not_reached ();
5811 #ifdef HEAVY_STATISTICS
5813 collect_store_remsets (RememberedSet *remset, mword *bumper)
5815 mword *p = remset->data;
5820 while (p < remset->store_next) {
5821 switch ((*p) & REMSET_TYPE_MASK) {
5822 case REMSET_LOCATION:
5825 ++stat_saved_remsets_1;
5827 if (*p == last1 || *p == last2) {
5828 ++stat_saved_remsets_2;
5845 g_assert_not_reached ();
5855 RememberedSet *remset;
5857 SgenThreadInfo *info;
5859 mword *addresses, *bumper, *p, *r;
5861 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5862 for (info = thread_table [i]; info; info = info->next) {
5863 for (remset = info->remset; remset; remset = remset->next)
5864 size += remset->store_next - remset->data;
5867 for (remset = freed_thread_remsets; remset; remset = remset->next)
5868 size += remset->store_next - remset->data;
5869 for (remset = global_remset; remset; remset = remset->next)
5870 size += remset->store_next - remset->data;
5872 bumper = addresses = get_internal_mem (sizeof (mword) * size, INTERNAL_MEM_STATISTICS);
5874 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5875 for (info = thread_table [i]; info; info = info->next) {
5876 for (remset = info->remset; remset; remset = remset->next)
5877 bumper = collect_store_remsets (remset, bumper);
5880 for (remset = global_remset; remset; remset = remset->next)
5881 bumper = collect_store_remsets (remset, bumper);
5882 for (remset = freed_thread_remsets; remset; remset = remset->next)
5883 bumper = collect_store_remsets (remset, bumper);
5885 g_assert (bumper <= addresses + size);
5887 stat_store_remsets += bumper - addresses;
5889 sort_addresses ((void**)addresses, bumper - addresses);
5892 while (r < bumper) {
5898 stat_store_remsets_unique += p - addresses;
5900 free_internal_mem (addresses, INTERNAL_MEM_STATISTICS);
5905 clear_thread_store_remset_buffer (SgenThreadInfo *info)
5907 *info->store_remset_buffer_index_addr = 0;
5908 memset (*info->store_remset_buffer_addr, 0, sizeof (gpointer) * STORE_REMSET_BUFFER_SIZE);
5912 scan_from_remsets (void *start_nursery, void *end_nursery)
5915 SgenThreadInfo *info;
5916 RememberedSet *remset;
5917 GenericStoreRememberedSet *store_remset;
5918 mword *p, *next_p, *store_pos;
5920 #ifdef HEAVY_STATISTICS
5924 /* the global one */
5925 for (remset = global_remset; remset; remset = remset->next) {
5926 DEBUG (4, fprintf (gc_debug_file, "Scanning global remset range: %p-%p, size: %zd\n", remset->data, remset->store_next, remset->store_next - remset->data));
5927 store_pos = remset->data;
5928 for (p = remset->data; p < remset->store_next; p = next_p) {
5931 /*Ignore previously processed remset.*/
5932 if (!global_remset_location_was_not_added (ptr)) {
5937 next_p = handle_remset (p, start_nursery, end_nursery, TRUE);
5940 * Clear global remsets of locations which no longer point to the
5941 * nursery. Otherwise, they could grow indefinitely between major
5944 * Since all global remsets are location remsets, we don't need to unmask the pointer.
5946 if (ptr_in_nursery (*ptr)) {
5947 *store_pos ++ = p [0];
5948 HEAVY_STAT (++stat_global_remsets_readded);
5952 /* Truncate the remset */
5953 remset->store_next = store_pos;
5956 /* the generic store ones */
5957 store_remset = generic_store_remsets;
5958 while (store_remset) {
5959 GenericStoreRememberedSet *next = store_remset->next;
5961 for (i = 0; i < STORE_REMSET_BUFFER_SIZE - 1; ++i) {
5962 gpointer addr = store_remset->data [i];
5964 handle_remset ((mword*)&addr, start_nursery, end_nursery, FALSE);
5967 free_internal_mem (store_remset, INTERNAL_MEM_STORE_REMSET);
5969 store_remset = next;
5971 generic_store_remsets = NULL;
5973 /* the per-thread ones */
5974 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5975 for (info = thread_table [i]; info; info = info->next) {
5976 RememberedSet *next;
5978 for (remset = info->remset; remset; remset = next) {
5979 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for thread %p, range: %p-%p, size: %zd\n", info, remset->data, remset->store_next, remset->store_next - remset->data));
5980 for (p = remset->data; p < remset->store_next;) {
5981 p = handle_remset (p, start_nursery, end_nursery, FALSE);
5983 remset->store_next = remset->data;
5984 next = remset->next;
5985 remset->next = NULL;
5986 if (remset != info->remset) {
5987 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
5988 free_internal_mem (remset, INTERNAL_MEM_REMSET);
5991 for (j = 0; j < *info->store_remset_buffer_index_addr; ++j)
5992 handle_remset ((mword*)*info->store_remset_buffer_addr + j + 1, start_nursery, end_nursery, FALSE);
5993 clear_thread_store_remset_buffer (info);
5997 /* the freed thread ones */
5998 while (freed_thread_remsets) {
5999 RememberedSet *next;
6000 remset = freed_thread_remsets;
6001 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for freed thread, range: %p-%p, size: %zd\n", remset->data, remset->store_next, remset->store_next - remset->data));
6002 for (p = remset->data; p < remset->store_next;) {
6003 p = handle_remset (p, start_nursery, end_nursery, FALSE);
6005 next = remset->next;
6006 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
6007 free_internal_mem (remset, INTERNAL_MEM_REMSET);
6008 freed_thread_remsets = next;
6013 * Clear the info in the remembered sets: we're doing a major collection, so
6014 * the per-thread ones are not needed and the global ones will be reconstructed
6018 clear_remsets (void)
6021 SgenThreadInfo *info;
6022 RememberedSet *remset, *next;
6024 /* the global list */
6025 for (remset = global_remset; remset; remset = next) {
6026 remset->store_next = remset->data;
6027 next = remset->next;
6028 remset->next = NULL;
6029 if (remset != global_remset) {
6030 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
6031 free_internal_mem (remset, INTERNAL_MEM_REMSET);
6034 /* the generic store ones */
6035 while (generic_store_remsets) {
6036 GenericStoreRememberedSet *gs_next = generic_store_remsets->next;
6037 free_internal_mem (generic_store_remsets, INTERNAL_MEM_STORE_REMSET);
6038 generic_store_remsets = gs_next;
6040 /* the per-thread ones */
6041 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
6042 for (info = thread_table [i]; info; info = info->next) {
6043 for (remset = info->remset; remset; remset = next) {
6044 remset->store_next = remset->data;
6045 next = remset->next;
6046 remset->next = NULL;
6047 if (remset != info->remset) {
6048 DEBUG (3, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
6049 free_internal_mem (remset, INTERNAL_MEM_REMSET);
6052 clear_thread_store_remset_buffer (info);
6056 /* the freed thread ones */
6057 while (freed_thread_remsets) {
6058 next = freed_thread_remsets->next;
6059 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", freed_thread_remsets->data));
6060 free_internal_mem (freed_thread_remsets, INTERNAL_MEM_REMSET);
6061 freed_thread_remsets = next;
6066 * Clear the thread local TLAB variables for all threads.
6071 SgenThreadInfo *info;
6074 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
6075 for (info = thread_table [i]; info; info = info->next) {
6076 /* A new TLAB will be allocated when the thread does its first allocation */
6077 *info->tlab_start_addr = NULL;
6078 *info->tlab_next_addr = NULL;
6079 *info->tlab_temp_end_addr = NULL;
6080 *info->tlab_real_end_addr = NULL;
6085 /* LOCKING: assumes the GC lock is held */
6086 static SgenThreadInfo*
6087 gc_register_current_thread (void *addr)
6090 SgenThreadInfo* info = malloc (sizeof (SgenThreadInfo));
6091 #ifndef HAVE_KW_THREAD
6092 SgenThreadInfo *__thread_info__ = info;
6098 memset (info, 0, sizeof (SgenThreadInfo));
6099 #ifndef HAVE_KW_THREAD
6100 info->tlab_start = info->tlab_next = info->tlab_temp_end = info->tlab_real_end = NULL;
6102 g_assert (!pthread_getspecific (thread_info_key));
6103 pthread_setspecific (thread_info_key, info);
6108 info->id = ARCH_GET_THREAD ();
6109 info->stop_count = -1;
6112 info->stack_start = NULL;
6113 info->tlab_start_addr = &TLAB_START;
6114 info->tlab_next_addr = &TLAB_NEXT;
6115 info->tlab_temp_end_addr = &TLAB_TEMP_END;
6116 info->tlab_real_end_addr = &TLAB_REAL_END;
6117 info->store_remset_buffer_addr = &STORE_REMSET_BUFFER;
6118 info->store_remset_buffer_index_addr = &STORE_REMSET_BUFFER_INDEX;
6119 info->stopped_ip = NULL;
6120 info->stopped_domain = NULL;
6121 info->stopped_regs = NULL;
6123 binary_protocol_thread_register ((gpointer)info->id);
6125 #ifdef HAVE_KW_THREAD
6126 tlab_next_addr = &tlab_next;
6127 store_remset_buffer_index_addr = &store_remset_buffer_index;
6130 /* try to get it with attributes first */
6131 #if defined(HAVE_PTHREAD_GETATTR_NP) && defined(HAVE_PTHREAD_ATTR_GETSTACK)
6135 pthread_attr_t attr;
6136 pthread_getattr_np (pthread_self (), &attr);
6137 pthread_attr_getstack (&attr, &sstart, &size);
6138 info->stack_start_limit = sstart;
6139 info->stack_end = (char*)sstart + size;
6140 pthread_attr_destroy (&attr);
6142 #elif defined(HAVE_PTHREAD_GET_STACKSIZE_NP) && defined(HAVE_PTHREAD_GET_STACKADDR_NP)
6143 info->stack_end = (char*)pthread_get_stackaddr_np (pthread_self ());
6144 info->stack_start_limit = (char*)info->stack_end - pthread_get_stacksize_np (pthread_self ());
6147 /* FIXME: we assume the stack grows down */
6148 gsize stack_bottom = (gsize)addr;
6149 stack_bottom += 4095;
6150 stack_bottom &= ~4095;
6151 info->stack_end = (char*)stack_bottom;
6155 #ifdef HAVE_KW_THREAD
6156 stack_end = info->stack_end;
6159 /* hash into the table */
6160 hash = HASH_PTHREAD_T (info->id) % THREAD_HASH_SIZE;
6161 info->next = thread_table [hash];
6162 thread_table [hash] = info;
6164 info->remset = alloc_remset (DEFAULT_REMSET_SIZE, info);
6165 pthread_setspecific (remembered_set_key, info->remset);
6166 #ifdef HAVE_KW_THREAD
6167 remembered_set = info->remset;
6170 STORE_REMSET_BUFFER = get_internal_mem (sizeof (gpointer) * STORE_REMSET_BUFFER_SIZE, INTERNAL_MEM_STORE_REMSET);
6171 STORE_REMSET_BUFFER_INDEX = 0;
6173 DEBUG (3, fprintf (gc_debug_file, "registered thread %p (%p) (hash: %d)\n", info, (gpointer)info->id, hash));
6175 if (gc_callbacks.thread_attach_func)
6176 info->runtime_data = gc_callbacks.thread_attach_func ();
6182 add_generic_store_remset_from_buffer (gpointer *buffer)
6184 GenericStoreRememberedSet *remset = get_internal_mem (sizeof (GenericStoreRememberedSet), INTERNAL_MEM_STORE_REMSET);
6185 memcpy (remset->data, buffer + 1, sizeof (gpointer) * (STORE_REMSET_BUFFER_SIZE - 1));
6186 remset->next = generic_store_remsets;
6187 generic_store_remsets = remset;
6191 unregister_current_thread (void)
6194 SgenThreadInfo *prev = NULL;
6196 RememberedSet *rset;
6197 ARCH_THREAD_TYPE id = ARCH_GET_THREAD ();
6199 binary_protocol_thread_unregister ((gpointer)id);
6201 hash = HASH_PTHREAD_T (id) % THREAD_HASH_SIZE;
6202 p = thread_table [hash];
6204 DEBUG (3, fprintf (gc_debug_file, "unregister thread %p (%p)\n", p, (gpointer)p->id));
6205 while (!ARCH_THREAD_EQUALS (p->id, id)) {
6210 thread_table [hash] = p->next;
6212 prev->next = p->next;
6215 if (freed_thread_remsets) {
6216 for (rset = p->remset; rset->next; rset = rset->next)
6218 rset->next = freed_thread_remsets;
6219 freed_thread_remsets = p->remset;
6221 freed_thread_remsets = p->remset;
6224 if (*p->store_remset_buffer_index_addr)
6225 add_generic_store_remset_from_buffer (*p->store_remset_buffer_addr);
6226 free_internal_mem (*p->store_remset_buffer_addr, INTERNAL_MEM_STORE_REMSET);
6231 unregister_thread (void *k)
6233 g_assert (!mono_domain_get ());
6235 unregister_current_thread ();
6240 mono_gc_register_thread (void *baseptr)
6242 SgenThreadInfo *info;
6246 info = thread_info_lookup (ARCH_GET_THREAD ());
6248 info = gc_register_current_thread (baseptr);
6250 return info != NULL;
6253 #if USE_PTHREAD_INTERCEPT
6255 #undef pthread_create
6257 #undef pthread_detach
6260 void *(*start_routine) (void *);
6263 MonoSemType registered;
6264 } SgenThreadStartInfo;
6267 gc_start_thread (void *arg)
6269 SgenThreadStartInfo *start_info = arg;
6270 SgenThreadInfo* info;
6271 void *t_arg = start_info->arg;
6272 void *(*start_func) (void*) = start_info->start_routine;
6277 info = gc_register_current_thread (&result);
6279 post_result = MONO_SEM_POST (&(start_info->registered));
6280 g_assert (!post_result);
6281 result = start_func (t_arg);
6282 g_assert (!mono_domain_get ());
6284 * this is done by the pthread key dtor
6286 unregister_current_thread ();
6294 mono_gc_pthread_create (pthread_t *new_thread, const pthread_attr_t *attr, void *(*start_routine)(void *), void *arg)
6296 SgenThreadStartInfo *start_info;
6299 start_info = malloc (sizeof (SgenThreadStartInfo));
6302 result = MONO_SEM_INIT (&(start_info->registered), 0);
6304 start_info->arg = arg;
6305 start_info->start_routine = start_routine;
6307 result = pthread_create (new_thread, attr, gc_start_thread, start_info);
6309 while (MONO_SEM_WAIT (&(start_info->registered)) != 0) {
6310 /*if (EINTR != errno) ABORT("sem_wait failed"); */
6313 MONO_SEM_DESTROY (&(start_info->registered));
6319 mono_gc_pthread_join (pthread_t thread, void **retval)
6321 return pthread_join (thread, retval);
6325 mono_gc_pthread_detach (pthread_t thread)
6327 return pthread_detach (thread);
6330 #endif /* USE_PTHREAD_INTERCEPT */
6333 * ######################################################################
6334 * ######## Write barriers
6335 * ######################################################################
6338 static RememberedSet*
6339 alloc_remset (int size, gpointer id) {
6340 RememberedSet* res = get_internal_mem (sizeof (RememberedSet) + (size * sizeof (gpointer)), INTERNAL_MEM_REMSET);
6341 res->store_next = res->data;
6342 res->end_set = res->data + size;
6344 DEBUG (4, fprintf (gc_debug_file, "Allocated remset size %d at %p for %p\n", size, res->data, id));
6349 * Note: the write barriers first do the needed GC work and then do the actual store:
6350 * this way the value is visible to the conservative GC scan after the write barrier
6351 * itself. If a GC interrupts the barrier in the middle, value will be kept alive by
6352 * the conservative scan, otherwise by the remembered set scan.
6355 mono_gc_wbarrier_set_field (MonoObject *obj, gpointer field_ptr, MonoObject* value)
6359 HEAVY_STAT (++stat_wbarrier_set_field);
6360 if (ptr_in_nursery (field_ptr)) {
6361 *(void**)field_ptr = value;
6364 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", field_ptr));
6366 rs = REMEMBERED_SET;
6367 if (rs->store_next < rs->end_set) {
6368 *(rs->store_next++) = (mword)field_ptr;
6369 *(void**)field_ptr = value;
6373 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6374 rs->next = REMEMBERED_SET;
6375 REMEMBERED_SET = rs;
6376 #ifdef HAVE_KW_THREAD
6377 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6379 *(rs->store_next++) = (mword)field_ptr;
6380 *(void**)field_ptr = value;
6385 mono_gc_wbarrier_set_arrayref (MonoArray *arr, gpointer slot_ptr, MonoObject* value)
6389 HEAVY_STAT (++stat_wbarrier_set_arrayref);
6390 if (ptr_in_nursery (slot_ptr)) {
6391 *(void**)slot_ptr = value;
6394 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", slot_ptr));
6396 rs = REMEMBERED_SET;
6397 if (rs->store_next < rs->end_set) {
6398 *(rs->store_next++) = (mword)slot_ptr;
6399 *(void**)slot_ptr = value;
6403 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6404 rs->next = REMEMBERED_SET;
6405 REMEMBERED_SET = rs;
6406 #ifdef HAVE_KW_THREAD
6407 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6409 *(rs->store_next++) = (mword)slot_ptr;
6410 *(void**)slot_ptr = value;
6415 mono_gc_wbarrier_arrayref_copy (gpointer dest_ptr, gpointer src_ptr, int count)
6419 HEAVY_STAT (++stat_wbarrier_arrayref_copy);
6421 memmove (dest_ptr, src_ptr, count * sizeof (gpointer));
6422 if (ptr_in_nursery (dest_ptr)) {
6426 rs = REMEMBERED_SET;
6427 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p, %d\n", dest_ptr, count));
6428 if (rs->store_next + 1 < rs->end_set) {
6429 *(rs->store_next++) = (mword)dest_ptr | REMSET_RANGE;
6430 *(rs->store_next++) = count;
6434 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6435 rs->next = REMEMBERED_SET;
6436 REMEMBERED_SET = rs;
6437 #ifdef HAVE_KW_THREAD
6438 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6440 *(rs->store_next++) = (mword)dest_ptr | REMSET_RANGE;
6441 *(rs->store_next++) = count;
6445 static char *found_obj;
6448 find_object_for_ptr_callback (char *obj, size_t size, char *ptr)
6450 if (ptr >= obj && ptr < obj + size) {
6451 g_assert (!found_obj);
6456 /* for use in the debugger */
6457 char* find_object_for_ptr (char *ptr);
6459 find_object_for_ptr (char *ptr)
6463 if (ptr >= nursery_section->data && ptr < nursery_section->end_data) {
6465 scan_area_with_callback (nursery_section->data, nursery_section->end_data,
6466 (IterateObjectCallbackFunc)find_object_for_ptr_callback, ptr);
6471 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
6472 if (ptr >= bigobj->data && ptr < bigobj->data + bigobj->size)
6473 return bigobj->data;
6477 * Very inefficient, but this is debugging code, supposed to
6478 * be called from gdb, so we don't care.
6481 major_iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)find_object_for_ptr_callback, ptr);
6486 evacuate_remset_buffer (void)
6491 buffer = STORE_REMSET_BUFFER;
6493 add_generic_store_remset_from_buffer (buffer);
6494 memset (buffer, 0, sizeof (gpointer) * STORE_REMSET_BUFFER_SIZE);
6496 STORE_REMSET_BUFFER_INDEX = 0;
6500 mono_gc_wbarrier_generic_nostore (gpointer ptr)
6506 HEAVY_STAT (++stat_wbarrier_generic_store);
6508 #ifdef XDOMAIN_CHECKS_IN_WBARRIER
6509 /* FIXME: ptr_in_heap must be called with the GC lock held */
6510 if (xdomain_checks && *(MonoObject**)ptr && ptr_in_heap (ptr)) {
6511 char *start = find_object_for_ptr (ptr);
6512 MonoObject *value = *(MonoObject**)ptr;
6516 MonoObject *obj = (MonoObject*)start;
6517 if (obj->vtable->domain != value->vtable->domain)
6518 g_assert (is_xdomain_ref_allowed (ptr, start, obj->vtable->domain));
6526 if (*(gpointer*)ptr)
6527 binary_protocol_wbarrier (ptr, *(gpointer*)ptr, (gpointer)LOAD_VTABLE (*(gpointer*)ptr));
6529 if (ptr_in_nursery (ptr) || ptr_on_stack (ptr) || !ptr_in_nursery (*(gpointer*)ptr)) {
6530 DEBUG (8, fprintf (gc_debug_file, "Skipping remset at %p\n", ptr));
6535 buffer = STORE_REMSET_BUFFER;
6536 index = STORE_REMSET_BUFFER_INDEX;
6537 /* This simple optimization eliminates a sizable portion of
6538 entries. Comparing it to the last but one entry as well
6539 doesn't eliminate significantly more entries. */
6540 if (buffer [index] == ptr) {
6545 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", ptr));
6546 HEAVY_STAT (++stat_wbarrier_generic_store_remset);
6549 if (index >= STORE_REMSET_BUFFER_SIZE) {
6550 evacuate_remset_buffer ();
6551 index = STORE_REMSET_BUFFER_INDEX;
6552 g_assert (index == 0);
6555 buffer [index] = ptr;
6556 STORE_REMSET_BUFFER_INDEX = index;
6562 mono_gc_wbarrier_generic_store (gpointer ptr, MonoObject* value)
6564 DEBUG (8, fprintf (gc_debug_file, "Wbarrier store at %p to %p (%s)\n", ptr, value, value ? safe_name (value) : "null"));
6565 *(void**)ptr = value;
6566 if (ptr_in_nursery (value))
6567 mono_gc_wbarrier_generic_nostore (ptr);
6571 mono_gc_wbarrier_value_copy (gpointer dest, gpointer src, int count, MonoClass *klass)
6575 HEAVY_STAT (++stat_wbarrier_value_copy);
6576 g_assert (klass->valuetype);
6578 memmove (dest, src, count * mono_class_value_size (klass, NULL));
6579 rs = REMEMBERED_SET;
6580 if (ptr_in_nursery (dest) || ptr_on_stack (dest) || !klass->has_references) {
6584 g_assert (klass->gc_descr_inited);
6585 DEBUG (8, fprintf (gc_debug_file, "Adding value remset at %p, count %d, descr %p for class %s (%p)\n", dest, count, klass->gc_descr, klass->name, klass));
6587 if (rs->store_next + 3 < rs->end_set) {
6588 *(rs->store_next++) = (mword)dest | REMSET_VTYPE;
6589 *(rs->store_next++) = (mword)klass->gc_descr;
6590 *(rs->store_next++) = (mword)count;
6594 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6595 rs->next = REMEMBERED_SET;
6596 REMEMBERED_SET = rs;
6597 #ifdef HAVE_KW_THREAD
6598 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6600 *(rs->store_next++) = (mword)dest | REMSET_VTYPE;
6601 *(rs->store_next++) = (mword)klass->gc_descr;
6602 *(rs->store_next++) = (mword)count;
6607 * mono_gc_wbarrier_object_copy:
6609 * Write barrier to call when obj is the result of a clone or copy of an object.
6612 mono_gc_wbarrier_object_copy (MonoObject* obj, MonoObject *src)
6618 HEAVY_STAT (++stat_wbarrier_object_copy);
6619 rs = REMEMBERED_SET;
6620 DEBUG (6, fprintf (gc_debug_file, "Adding object remset for %p\n", obj));
6621 size = mono_object_class (obj)->instance_size;
6623 /* do not copy the sync state */
6624 memcpy ((char*)obj + sizeof (MonoObject), (char*)src + sizeof (MonoObject),
6625 size - sizeof (MonoObject));
6626 if (ptr_in_nursery (obj) || ptr_on_stack (obj)) {
6630 if (rs->store_next < rs->end_set) {
6631 *(rs->store_next++) = (mword)obj | REMSET_OBJECT;
6635 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6636 rs->next = REMEMBERED_SET;
6637 REMEMBERED_SET = rs;
6638 #ifdef HAVE_KW_THREAD
6639 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6641 *(rs->store_next++) = (mword)obj | REMSET_OBJECT;
6646 * ######################################################################
6647 * ######## Collector debugging
6648 * ######################################################################
6651 const char*descriptor_types [] = {
6663 describe_ptr (char *ptr)
6669 if (ptr_in_nursery (ptr)) {
6670 printf ("Pointer inside nursery.\n");
6672 if (major_ptr_is_in_non_pinned_space (ptr)) {
6673 printf ("Pointer inside oldspace.\n");
6674 } else if (obj_is_from_pinned_alloc (ptr)) {
6675 printf ("Pointer is inside a pinned chunk.\n");
6677 printf ("Pointer unknown.\n");
6682 if (object_is_pinned (ptr))
6683 printf ("Object is pinned.\n");
6685 if (object_is_forwarded (ptr))
6686 printf ("Object is forwared.\n");
6688 // FIXME: Handle pointers to the inside of objects
6689 vtable = (MonoVTable*)LOAD_VTABLE (ptr);
6691 printf ("VTable: %p\n", vtable);
6692 if (vtable == NULL) {
6693 printf ("VTable is invalid (empty).\n");
6696 if (ptr_in_nursery (vtable)) {
6697 printf ("VTable is invalid (points inside nursery).\n");
6700 printf ("Class: %s\n", vtable->klass->name);
6702 desc = ((GCVTable*)vtable)->desc;
6703 printf ("Descriptor: %lx\n", (long)desc);
6706 printf ("Descriptor type: %d (%s)\n", type, descriptor_types [type]);
6710 find_in_remset_loc (mword *p, char *addr, gboolean *found)
6716 switch ((*p) & REMSET_TYPE_MASK) {
6717 case REMSET_LOCATION:
6718 if (*p == (mword)addr)
6722 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
6724 if ((void**)addr >= ptr && (void**)addr < ptr + count)
6728 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
6729 count = safe_object_get_size ((MonoObject*)ptr);
6730 count += (ALLOC_ALIGN - 1);
6731 count &= (ALLOC_ALIGN - 1);
6732 count /= sizeof (mword);
6733 if ((void**)addr >= ptr && (void**)addr < ptr + count)
6737 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
6741 switch (desc & 0x7) {
6742 case DESC_TYPE_RUN_LENGTH:
6743 OBJ_RUN_LEN_SIZE (skip_size, desc, ptr);
6745 case DESC_TYPE_SMALL_BITMAP:
6746 OBJ_BITMAP_SIZE (skip_size, desc, start);
6750 g_assert_not_reached ();
6753 /* The descriptor includes the size of MonoObject */
6754 skip_size -= sizeof (MonoObject);
6756 if ((void**)addr >= ptr && (void**)addr < ptr + (skip_size / sizeof (gpointer)))
6761 g_assert_not_reached ();
6767 * Return whenever ADDR occurs in the remembered sets
6770 find_in_remsets (char *addr)
6773 SgenThreadInfo *info;
6774 RememberedSet *remset;
6775 GenericStoreRememberedSet *store_remset;
6777 gboolean found = FALSE;
6779 /* the global one */
6780 for (remset = global_remset; remset; remset = remset->next) {
6781 DEBUG (4, fprintf (gc_debug_file, "Scanning global remset range: %p-%p, size: %zd\n", remset->data, remset->store_next, remset->store_next - remset->data));
6782 for (p = remset->data; p < remset->store_next;) {
6783 p = find_in_remset_loc (p, addr, &found);
6789 /* the generic store ones */
6790 for (store_remset = generic_store_remsets; store_remset; store_remset = store_remset->next) {
6791 for (i = 0; i < STORE_REMSET_BUFFER_SIZE - 1; ++i) {
6792 if (store_remset->data [i] == addr)
6797 /* the per-thread ones */
6798 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
6799 for (info = thread_table [i]; info; info = info->next) {
6801 for (remset = info->remset; remset; remset = remset->next) {
6802 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for thread %p, range: %p-%p, size: %zd\n", info, remset->data, remset->store_next, remset->store_next - remset->data));
6803 for (p = remset->data; p < remset->store_next;) {
6804 p = find_in_remset_loc (p, addr, &found);
6809 for (j = 0; j < *info->store_remset_buffer_index_addr; ++j) {
6810 if ((*info->store_remset_buffer_addr) [j + 1] == addr)
6816 /* the freed thread ones */
6817 for (remset = freed_thread_remsets; remset; remset = remset->next) {
6818 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for freed thread, range: %p-%p, size: %zd\n", remset->data, remset->store_next, remset->store_next - remset->data));
6819 for (p = remset->data; p < remset->store_next;) {
6820 p = find_in_remset_loc (p, addr, &found);
6829 static gboolean missing_remsets;
6832 * We let a missing remset slide if the target object is pinned,
6833 * because the store might have happened but the remset not yet added,
6834 * but in that case the target must be pinned. We might theoretically
6835 * miss some missing remsets this way, but it's very unlikely.
6838 #define HANDLE_PTR(ptr,obj) do { \
6839 if (*(ptr) && (char*)*(ptr) >= nursery_start && (char*)*(ptr) < nursery_next) { \
6840 if (!find_in_remsets ((char*)(ptr))) { \
6841 fprintf (gc_debug_file, "Oldspace->newspace reference %p at offset %zd in object %p (%s.%s) not found in remsets.\n", *(ptr), (char*)(ptr) - (char*)(obj), (obj), ((MonoObject*)(obj))->vtable->klass->name_space, ((MonoObject*)(obj))->vtable->klass->name); \
6842 binary_protocol_missing_remset ((obj), (gpointer)LOAD_VTABLE ((obj)), (char*)(ptr) - (char*)(obj), *(ptr), (gpointer)LOAD_VTABLE(*(ptr)), object_is_pinned (*(ptr))); \
6843 if (!object_is_pinned (*(ptr))) \
6844 missing_remsets = TRUE; \
6850 * Check that each object reference which points into the nursery can
6851 * be found in the remembered sets.
6854 check_consistency_callback (char *start, size_t size, void *dummy)
6856 GCVTable *vt = (GCVTable*)LOAD_VTABLE (start);
6857 DEBUG (8, fprintf (gc_debug_file, "Scanning object %p, vtable: %p (%s)\n", start, vt, vt->klass->name));
6859 #define SCAN_OBJECT_ACTION
6860 #include "sgen-scan-object.h"
6864 * Perform consistency check of the heap.
6866 * Assumes the world is stopped.
6869 check_consistency (void)
6873 // Need to add more checks
6875 missing_remsets = FALSE;
6877 DEBUG (1, fprintf (gc_debug_file, "Begin heap consistency check...\n"));
6879 // Check that oldspace->newspace pointers are registered with the collector
6880 major_iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)check_consistency_callback, NULL);
6882 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
6883 check_consistency_callback (bigobj->data, bigobj->size, NULL);
6885 DEBUG (1, fprintf (gc_debug_file, "Heap consistency check done.\n"));
6887 #ifdef BINARY_PROTOCOL
6888 if (!binary_protocol_file)
6890 g_assert (!missing_remsets);
6895 #define HANDLE_PTR(ptr,obj) do { \
6897 g_assert (LOAD_VTABLE (*(ptr))); \
6901 check_major_refs_callback (char *start, size_t size, void *dummy)
6903 #define SCAN_OBJECT_ACTION
6904 #include "sgen-scan-object.h"
6908 check_major_refs (void)
6912 major_iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)check_major_refs_callback, NULL);
6914 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
6915 check_major_refs_callback (bigobj->data, bigobj->size, NULL);
6918 /* Check that the reference is valid */
6920 #define HANDLE_PTR(ptr,obj) do { \
6922 g_assert (safe_name (*(ptr)) != NULL); \
6929 * Perform consistency check on an object. Currently we only check that the
6930 * reference fields are valid.
6933 check_object (char *start)
6938 #include "sgen-scan-object.h"
6942 * ######################################################################
6943 * ######## Other mono public interface functions.
6944 * ######################################################################
6948 mono_gc_collect (int generation)
6952 if (generation == 0) {
6953 collect_nursery (0);
6955 major_collection ("user request");
6962 mono_gc_max_generation (void)
6968 mono_gc_collection_count (int generation)
6970 if (generation == 0)
6971 return num_minor_gcs;
6972 return num_major_gcs;
6976 mono_gc_get_used_size (void)
6980 tot = los_memory_usage;
6981 tot += nursery_section->next_data - nursery_section->data;
6982 tot += major_get_used_size ();
6983 /* FIXME: account for pinned objects */
6989 mono_gc_get_heap_size (void)
6995 mono_gc_disable (void)
7003 mono_gc_enable (void)
7011 mono_gc_get_los_limit (void)
7013 return MAX_SMALL_OBJ_SIZE;
7017 mono_object_is_alive (MonoObject* o)
7023 mono_gc_get_generation (MonoObject *obj)
7025 if (ptr_in_nursery (obj))
7031 mono_gc_enable_events (void)
7036 mono_gc_weak_link_add (void **link_addr, MonoObject *obj, gboolean track)
7039 mono_gc_register_disappearing_link (obj, link_addr, track);
7044 mono_gc_weak_link_remove (void **link_addr)
7047 mono_gc_register_disappearing_link (NULL, link_addr, FALSE);
7052 mono_gc_weak_link_get (void **link_addr)
7056 return (MonoObject*) REVEAL_POINTER (*link_addr);
7060 mono_gc_ephemeron_array_add (MonoObject *obj)
7062 EphemeronLinkNode *node;
7066 node = get_internal_mem (sizeof (EphemeronLinkNode), INTERNAL_MEM_EPHEMERON_LINK);
7071 node->array = (char*)obj;
7072 node->next = ephemeron_list;
7073 ephemeron_list = node;
7075 DEBUG (5, fprintf (gc_debug_file, "Registered ephemeron array %p\n", obj));
7082 mono_gc_make_descr_from_bitmap (gsize *bitmap, int numbits)
7084 if (numbits < ((sizeof (*bitmap) * 8) - ROOT_DESC_TYPE_SHIFT)) {
7085 return (void*)MAKE_ROOT_DESC (ROOT_DESC_BITMAP, bitmap [0]);
7087 mword complex = alloc_complex_descriptor (bitmap, numbits);
7088 return (void*)MAKE_ROOT_DESC (ROOT_DESC_COMPLEX, complex);
7093 mono_gc_make_root_descr_user (MonoGCRootMarkFunc marker)
7097 g_assert (user_descriptors_next < MAX_USER_DESCRIPTORS);
7098 descr = (void*)MAKE_ROOT_DESC (ROOT_DESC_USER, (mword)user_descriptors_next);
7099 user_descriptors [user_descriptors_next ++] = marker;
7105 mono_gc_alloc_fixed (size_t size, void *descr)
7107 /* FIXME: do a single allocation */
7108 void *res = calloc (1, size);
7111 if (!mono_gc_register_root (res, size, descr)) {
7119 mono_gc_free_fixed (void* addr)
7121 mono_gc_deregister_root (addr);
7126 mono_gc_invoke_with_gc_lock (MonoGCLockedCallbackFunc func, void *data)
7130 result = func (data);
7131 UNLOCK_INTERRUPTION;
7136 mono_gc_is_gc_thread (void)
7140 result = thread_info_lookup (ARCH_GET_THREAD ()) != NULL;
7147 /* Tries to extract a number from the passed string, taking in to account m, k
7150 parse_environment_string_extract_number (gchar *str, glong *out)
7153 int len = strlen (str), shift = 0;
7155 gboolean is_suffix = FALSE;
7158 switch (str [len - 1]) {
7169 suffix = str [len - 1];
7174 val = strtol (str, &endptr, 10);
7176 if ((errno == ERANGE && (val == LONG_MAX || val == LONG_MIN))
7177 || (errno != 0 && val == 0) || (endptr == str))
7181 if (*(endptr + 1)) /* Invalid string. */
7193 mono_gc_base_init (void)
7197 struct sigaction sinfo;
7199 LOCK_INIT (gc_mutex);
7201 if (gc_initialized) {
7205 pagesize = mono_pagesize ();
7206 gc_debug_file = stderr;
7210 if ((env = getenv ("MONO_GC_PARAMS"))) {
7211 if (g_str_has_prefix (env, "nursery-size")) {
7214 while (env [index] && env [index++] != '=')
7216 if (env [index] && parse_environment_string_extract_number (env
7218 default_nursery_size = val;
7219 #ifdef ALIGN_NURSERY
7220 if ((val & (val - 1))) {
7221 fprintf (stderr, "The nursery size must be a power of two.\n");
7225 default_nursery_bits = 0;
7226 while (1 << (++ default_nursery_bits) != default_nursery_size)
7230 fprintf (stderr, "nursery-size must be an integer.\n");
7234 fprintf (stderr, "MONO_GC_PARAMS must be of the form 'nursery-size=N' (where N is an integer, possibly with a k, m or a g suffix).\n");
7241 nursery_size = DEFAULT_NURSERY_SIZE;
7242 minor_collection_allowance = MIN_MINOR_COLLECTION_ALLOWANCE;
7246 if ((env = getenv ("MONO_GC_DEBUG"))) {
7247 opts = g_strsplit (env, ",", -1);
7248 for (ptr = opts; ptr && *ptr; ptr ++) {
7250 if (opt [0] >= '0' && opt [0] <= '9') {
7251 gc_debug_level = atoi (opt);
7256 char *rf = g_strdup_printf ("%s.%d", opt, getpid ());
7257 gc_debug_file = fopen (rf, "wb");
7259 gc_debug_file = stderr;
7262 } else if (!strcmp (opt, "collect-before-allocs")) {
7263 collect_before_allocs = TRUE;
7264 } else if (!strcmp (opt, "check-at-minor-collections")) {
7265 consistency_check_at_minor_collection = TRUE;
7266 nursery_clear_policy = CLEAR_AT_GC;
7267 } else if (!strcmp (opt, "xdomain-checks")) {
7268 xdomain_checks = TRUE;
7269 } else if (!strcmp (opt, "clear-at-gc")) {
7270 nursery_clear_policy = CLEAR_AT_GC;
7271 } else if (!strcmp (opt, "conservative-stack-mark")) {
7272 conservative_stack_mark = TRUE;
7273 } else if (!strcmp (opt, "check-scan-starts")) {
7274 do_scan_starts_check = TRUE;
7275 } else if (g_str_has_prefix (opt, "heap-dump=")) {
7276 char *filename = strchr (opt, '=') + 1;
7277 nursery_clear_policy = CLEAR_AT_GC;
7278 heap_dump_file = fopen (filename, "w");
7280 fprintf (heap_dump_file, "<sgen-dump>\n");
7281 #ifdef BINARY_PROTOCOL
7282 } else if (g_str_has_prefix (opt, "binary-protocol=")) {
7283 char *filename = strchr (opt, '=') + 1;
7284 binary_protocol_file = fopen (filename, "w");
7287 fprintf (stderr, "Invalid format for the MONO_GC_DEBUG env variable: '%s'\n", env);
7288 fprintf (stderr, "The format is: MONO_GC_DEBUG=[l[:filename]|<option>]+ where l is a debug level 0-9.\n");
7289 fprintf (stderr, "Valid options are: collect-before-allocs, check-at-minor-collections, xdomain-checks, clear-at-gc.\n");
7296 suspend_ack_semaphore_ptr = &suspend_ack_semaphore;
7297 MONO_SEM_INIT (&suspend_ack_semaphore, 0);
7299 sigfillset (&sinfo.sa_mask);
7300 sinfo.sa_flags = SA_RESTART | SA_SIGINFO;
7301 sinfo.sa_sigaction = suspend_handler;
7302 if (sigaction (suspend_signal_num, &sinfo, NULL) != 0) {
7303 g_error ("failed sigaction");
7306 sinfo.sa_handler = restart_handler;
7307 if (sigaction (restart_signal_num, &sinfo, NULL) != 0) {
7308 g_error ("failed sigaction");
7311 sigfillset (&suspend_signal_mask);
7312 sigdelset (&suspend_signal_mask, restart_signal_num);
7314 global_remset = alloc_remset (1024, NULL);
7315 global_remset->next = NULL;
7317 pthread_key_create (&remembered_set_key, unregister_thread);
7319 #ifndef HAVE_KW_THREAD
7320 pthread_key_create (&thread_info_key, NULL);
7323 gc_initialized = TRUE;
7325 mono_gc_register_thread (&sinfo);
7329 mono_gc_get_suspend_signal (void)
7331 return suspend_signal_num;
7341 #ifdef HAVE_KW_THREAD
7342 #define EMIT_TLS_ACCESS(mb,dummy,offset) do { \
7343 mono_mb_emit_byte ((mb), MONO_CUSTOM_PREFIX); \
7344 mono_mb_emit_byte ((mb), CEE_MONO_TLS); \
7345 mono_mb_emit_i4 ((mb), (offset)); \
7348 #define EMIT_TLS_ACCESS(mb,member,dummy) do { \
7349 mono_mb_emit_byte ((mb), MONO_CUSTOM_PREFIX); \
7350 mono_mb_emit_byte ((mb), CEE_MONO_TLS); \
7351 mono_mb_emit_i4 ((mb), thread_info_key); \
7352 mono_mb_emit_icon ((mb), G_STRUCT_OFFSET (SgenThreadInfo, member)); \
7353 mono_mb_emit_byte ((mb), CEE_ADD); \
7354 mono_mb_emit_byte ((mb), CEE_LDIND_I); \
7358 #ifdef MANAGED_ALLOCATION
7359 /* FIXME: Do this in the JIT, where specialized allocation sequences can be created
7360 * for each class. This is currently not easy to do, as it is hard to generate basic
7361 * blocks + branches, but it is easy with the linear IL codebase.
7363 * For this to work we'd need to solve the TLAB race, first. Now we
7364 * require the allocator to be in a few known methods to make sure
7365 * that they are executed atomically via the restart mechanism.
7368 create_allocator (int atype)
7370 int p_var, size_var;
7371 guint32 slowpath_branch, max_size_branch;
7372 MonoMethodBuilder *mb;
7374 MonoMethodSignature *csig;
7375 static gboolean registered = FALSE;
7376 int tlab_next_addr_var, new_next_var;
7378 const char *name = NULL;
7379 AllocatorWrapperInfo *info;
7381 #ifdef HAVE_KW_THREAD
7382 int tlab_next_addr_offset = -1;
7383 int tlab_temp_end_offset = -1;
7385 MONO_THREAD_VAR_OFFSET (tlab_next_addr, tlab_next_addr_offset);
7386 MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
7388 g_assert (tlab_next_addr_offset != -1);
7389 g_assert (tlab_temp_end_offset != -1);
7393 mono_register_jit_icall (mono_gc_alloc_obj, "mono_gc_alloc_obj", mono_create_icall_signature ("object ptr int"), FALSE);
7394 mono_register_jit_icall (mono_gc_alloc_vector, "mono_gc_alloc_vector", mono_create_icall_signature ("object ptr int int"), FALSE);
7398 if (atype == ATYPE_SMALL) {
7400 name = "AllocSmall";
7401 } else if (atype == ATYPE_NORMAL) {
7404 } else if (atype == ATYPE_VECTOR) {
7406 name = "AllocVector";
7408 g_assert_not_reached ();
7411 csig = mono_metadata_signature_alloc (mono_defaults.corlib, num_params);
7412 csig->ret = &mono_defaults.object_class->byval_arg;
7413 for (i = 0; i < num_params; ++i)
7414 csig->params [i] = &mono_defaults.int_class->byval_arg;
7416 mb = mono_mb_new (mono_defaults.object_class, name, MONO_WRAPPER_ALLOC);
7417 size_var = mono_mb_add_local (mb, &mono_defaults.int32_class->byval_arg);
7418 if (atype == ATYPE_NORMAL || atype == ATYPE_SMALL) {
7419 /* size = vtable->klass->instance_size; */
7420 mono_mb_emit_ldarg (mb, 0);
7421 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoVTable, klass));
7422 mono_mb_emit_byte (mb, CEE_ADD);
7423 mono_mb_emit_byte (mb, CEE_LDIND_I);
7424 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoClass, instance_size));
7425 mono_mb_emit_byte (mb, CEE_ADD);
7426 /* FIXME: assert instance_size stays a 4 byte integer */
7427 mono_mb_emit_byte (mb, CEE_LDIND_U4);
7428 mono_mb_emit_stloc (mb, size_var);
7429 } else if (atype == ATYPE_VECTOR) {
7430 MonoExceptionClause *clause;
7432 MonoClass *oom_exc_class;
7435 /* n > MONO_ARRAY_MAX_INDEX -> OverflowException */
7436 mono_mb_emit_ldarg (mb, 1);
7437 mono_mb_emit_icon (mb, MONO_ARRAY_MAX_INDEX);
7438 pos = mono_mb_emit_short_branch (mb, CEE_BLE_UN_S);
7439 mono_mb_emit_exception (mb, "OverflowException", NULL);
7440 mono_mb_patch_short_branch (mb, pos);
7442 clause = mono_image_alloc0 (mono_defaults.corlib, sizeof (MonoExceptionClause));
7443 clause->try_offset = mono_mb_get_label (mb);
7445 /* vtable->klass->sizes.element_size */
7446 mono_mb_emit_ldarg (mb, 0);
7447 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoVTable, klass));
7448 mono_mb_emit_byte (mb, CEE_ADD);
7449 mono_mb_emit_byte (mb, CEE_LDIND_I);
7450 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoClass, sizes.element_size));
7451 mono_mb_emit_byte (mb, CEE_ADD);
7452 mono_mb_emit_byte (mb, CEE_LDIND_U4);
7455 mono_mb_emit_ldarg (mb, 1);
7456 mono_mb_emit_byte (mb, CEE_MUL_OVF_UN);
7457 /* + sizeof (MonoArray) */
7458 mono_mb_emit_icon (mb, sizeof (MonoArray));
7459 mono_mb_emit_byte (mb, CEE_ADD_OVF_UN);
7460 mono_mb_emit_stloc (mb, size_var);
7462 pos_leave = mono_mb_emit_branch (mb, CEE_LEAVE);
7465 clause->flags = MONO_EXCEPTION_CLAUSE_NONE;
7466 clause->try_len = mono_mb_get_pos (mb) - clause->try_offset;
7467 clause->data.catch_class = mono_class_from_name (mono_defaults.corlib,
7468 "System", "OverflowException");
7469 g_assert (clause->data.catch_class);
7470 clause->handler_offset = mono_mb_get_label (mb);
7472 oom_exc_class = mono_class_from_name (mono_defaults.corlib,
7473 "System", "OutOfMemoryException");
7474 g_assert (oom_exc_class);
7475 ctor = mono_class_get_method_from_name (oom_exc_class, ".ctor", 0);
7478 mono_mb_emit_byte (mb, CEE_POP);
7479 mono_mb_emit_op (mb, CEE_NEWOBJ, ctor);
7480 mono_mb_emit_byte (mb, CEE_THROW);
7482 clause->handler_len = mono_mb_get_pos (mb) - clause->handler_offset;
7483 mono_mb_set_clauses (mb, 1, clause);
7484 mono_mb_patch_branch (mb, pos_leave);
7487 g_assert_not_reached ();
7490 /* size += ALLOC_ALIGN - 1; */
7491 mono_mb_emit_ldloc (mb, size_var);
7492 mono_mb_emit_icon (mb, ALLOC_ALIGN - 1);
7493 mono_mb_emit_byte (mb, CEE_ADD);
7494 /* size &= ~(ALLOC_ALIGN - 1); */
7495 mono_mb_emit_icon (mb, ~(ALLOC_ALIGN - 1));
7496 mono_mb_emit_byte (mb, CEE_AND);
7497 mono_mb_emit_stloc (mb, size_var);
7499 /* if (size > MAX_SMALL_OBJ_SIZE) goto slowpath */
7500 if (atype != ATYPE_SMALL) {
7501 mono_mb_emit_ldloc (mb, size_var);
7502 mono_mb_emit_icon (mb, MAX_SMALL_OBJ_SIZE);
7503 max_size_branch = mono_mb_emit_short_branch (mb, MONO_CEE_BGT_S);
7507 * We need to modify tlab_next, but the JIT only supports reading, so we read
7508 * another tls var holding its address instead.
7511 /* tlab_next_addr (local) = tlab_next_addr (TLS var) */
7512 tlab_next_addr_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7513 EMIT_TLS_ACCESS (mb, tlab_next_addr, tlab_next_addr_offset);
7514 mono_mb_emit_stloc (mb, tlab_next_addr_var);
7516 /* p = (void**)tlab_next; */
7517 p_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7518 mono_mb_emit_ldloc (mb, tlab_next_addr_var);
7519 mono_mb_emit_byte (mb, CEE_LDIND_I);
7520 mono_mb_emit_stloc (mb, p_var);
7522 /* new_next = (char*)p + size; */
7523 new_next_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7524 mono_mb_emit_ldloc (mb, p_var);
7525 mono_mb_emit_ldloc (mb, size_var);
7526 mono_mb_emit_byte (mb, CEE_CONV_I);
7527 mono_mb_emit_byte (mb, CEE_ADD);
7528 mono_mb_emit_stloc (mb, new_next_var);
7530 /* tlab_next = new_next */
7531 mono_mb_emit_ldloc (mb, tlab_next_addr_var);
7532 mono_mb_emit_ldloc (mb, new_next_var);
7533 mono_mb_emit_byte (mb, CEE_STIND_I);
7535 /* if (G_LIKELY (new_next < tlab_temp_end)) */
7536 mono_mb_emit_ldloc (mb, new_next_var);
7537 EMIT_TLS_ACCESS (mb, tlab_temp_end, tlab_temp_end_offset);
7538 slowpath_branch = mono_mb_emit_short_branch (mb, MONO_CEE_BLT_UN_S);
7541 if (atype != ATYPE_SMALL)
7542 mono_mb_patch_short_branch (mb, max_size_branch);
7544 mono_mb_emit_byte (mb, MONO_CUSTOM_PREFIX);
7545 mono_mb_emit_byte (mb, CEE_MONO_NOT_TAKEN);
7547 /* FIXME: mono_gc_alloc_obj takes a 'size_t' as an argument, not an int32 */
7548 mono_mb_emit_ldarg (mb, 0);
7549 mono_mb_emit_ldloc (mb, size_var);
7550 if (atype == ATYPE_NORMAL || atype == ATYPE_SMALL) {
7551 mono_mb_emit_icall (mb, mono_gc_alloc_obj);
7552 } else if (atype == ATYPE_VECTOR) {
7553 mono_mb_emit_ldarg (mb, 1);
7554 mono_mb_emit_icall (mb, mono_gc_alloc_vector);
7556 g_assert_not_reached ();
7558 mono_mb_emit_byte (mb, CEE_RET);
7561 mono_mb_patch_short_branch (mb, slowpath_branch);
7563 /* FIXME: Memory barrier */
7566 mono_mb_emit_ldloc (mb, p_var);
7567 mono_mb_emit_ldarg (mb, 0);
7568 mono_mb_emit_byte (mb, CEE_STIND_I);
7570 if (atype == ATYPE_VECTOR) {
7571 /* arr->max_length = max_length; */
7572 mono_mb_emit_ldloc (mb, p_var);
7573 mono_mb_emit_ldflda (mb, G_STRUCT_OFFSET (MonoArray, max_length));
7574 mono_mb_emit_ldarg (mb, 1);
7575 mono_mb_emit_byte (mb, CEE_STIND_I);
7579 mono_mb_emit_ldloc (mb, p_var);
7580 mono_mb_emit_byte (mb, CEE_RET);
7582 res = mono_mb_create_method (mb, csig, 8);
7584 mono_method_get_header (res)->init_locals = FALSE;
7586 info = mono_image_alloc0 (mono_defaults.corlib, sizeof (AllocatorWrapperInfo));
7587 info->alloc_type = atype;
7588 mono_marshal_set_wrapper_info (res, info);
7594 static MonoMethod* alloc_method_cache [ATYPE_NUM];
7595 static MonoMethod *write_barrier_method;
7598 is_ip_in_managed_allocator (MonoDomain *domain, gpointer ip)
7606 ji = mono_jit_info_table_find (domain, ip);
7609 method = ji->method;
7611 if (method == write_barrier_method)
7613 for (i = 0; i < ATYPE_NUM; ++i)
7614 if (method == alloc_method_cache [i])
7620 * Generate an allocator method implementing the fast path of mono_gc_alloc_obj ().
7621 * The signature of the called method is:
7622 * object allocate (MonoVTable *vtable)
7625 mono_gc_get_managed_allocator (MonoVTable *vtable, gboolean for_box)
7627 #ifdef MANAGED_ALLOCATION
7628 MonoClass *klass = vtable->klass;
7630 #ifdef HAVE_KW_THREAD
7631 int tlab_next_offset = -1;
7632 int tlab_temp_end_offset = -1;
7633 MONO_THREAD_VAR_OFFSET (tlab_next, tlab_next_offset);
7634 MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
7636 if (tlab_next_offset == -1 || tlab_temp_end_offset == -1)
7640 if (!mono_runtime_has_tls_get ())
7642 if (klass->instance_size > tlab_size)
7644 if (klass->has_finalize || klass->marshalbyref || (mono_profiler_get_events () & MONO_PROFILE_ALLOCATIONS))
7648 if (klass->byval_arg.type == MONO_TYPE_STRING)
7650 if (collect_before_allocs)
7653 if (ALIGN_TO (klass->instance_size, ALLOC_ALIGN) < MAX_SMALL_OBJ_SIZE)
7654 return mono_gc_get_managed_allocator_by_type (ATYPE_SMALL);
7656 return mono_gc_get_managed_allocator_by_type (ATYPE_NORMAL);
7663 mono_gc_get_managed_array_allocator (MonoVTable *vtable, int rank)
7665 #ifdef MANAGED_ALLOCATION
7666 MonoClass *klass = vtable->klass;
7668 #ifdef HAVE_KW_THREAD
7669 int tlab_next_offset = -1;
7670 int tlab_temp_end_offset = -1;
7671 MONO_THREAD_VAR_OFFSET (tlab_next, tlab_next_offset);
7672 MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
7674 if (tlab_next_offset == -1 || tlab_temp_end_offset == -1)
7680 if (!mono_runtime_has_tls_get ())
7682 if (mono_profiler_get_events () & MONO_PROFILE_ALLOCATIONS)
7684 if (collect_before_allocs)
7686 g_assert (!klass->has_finalize && !klass->marshalbyref);
7688 return mono_gc_get_managed_allocator_by_type (ATYPE_VECTOR);
7695 mono_gc_get_managed_allocator_by_type (int atype)
7697 #ifdef MANAGED_ALLOCATION
7700 if (!mono_runtime_has_tls_get ())
7703 mono_loader_lock ();
7704 res = alloc_method_cache [atype];
7706 res = alloc_method_cache [atype] = create_allocator (atype);
7707 mono_loader_unlock ();
7715 mono_gc_get_managed_allocator_types (void)
7722 mono_gc_get_write_barrier (void)
7725 MonoMethodBuilder *mb;
7726 MonoMethodSignature *sig;
7727 #ifdef MANAGED_WBARRIER
7728 int label_no_wb_1, label_no_wb_2, label_no_wb_3, label_no_wb_4, label_need_wb, label_slow_path;
7729 #ifndef ALIGN_NURSERY
7730 int label_continue_1, label_continue_2, label_no_wb_5;
7731 int dereferenced_var;
7733 int buffer_var, buffer_index_var, dummy_var;
7735 #ifdef HAVE_KW_THREAD
7736 int stack_end_offset = -1, store_remset_buffer_offset = -1;
7737 int store_remset_buffer_index_offset = -1, store_remset_buffer_index_addr_offset = -1;
7739 MONO_THREAD_VAR_OFFSET (stack_end, stack_end_offset);
7740 g_assert (stack_end_offset != -1);
7741 MONO_THREAD_VAR_OFFSET (store_remset_buffer, store_remset_buffer_offset);
7742 g_assert (store_remset_buffer_offset != -1);
7743 MONO_THREAD_VAR_OFFSET (store_remset_buffer_index, store_remset_buffer_index_offset);
7744 g_assert (store_remset_buffer_index_offset != -1);
7745 MONO_THREAD_VAR_OFFSET (store_remset_buffer_index_addr, store_remset_buffer_index_addr_offset);
7746 g_assert (store_remset_buffer_index_addr_offset != -1);
7750 // FIXME: Maybe create a separate version for ctors (the branch would be
7751 // correctly predicted more times)
7752 if (write_barrier_method)
7753 return write_barrier_method;
7755 /* Create the IL version of mono_gc_barrier_generic_store () */
7756 sig = mono_metadata_signature_alloc (mono_defaults.corlib, 1);
7757 sig->ret = &mono_defaults.void_class->byval_arg;
7758 sig->params [0] = &mono_defaults.int_class->byval_arg;
7760 mb = mono_mb_new (mono_defaults.object_class, "wbarrier", MONO_WRAPPER_WRITE_BARRIER);
7762 #ifdef MANAGED_WBARRIER
7763 if (mono_runtime_has_tls_get ()) {
7764 #ifdef ALIGN_NURSERY
7765 // if (ptr_in_nursery (ptr)) return;
7767 * Masking out the bits might be faster, but we would have to use 64 bit
7768 * immediates, which might be slower.
7770 mono_mb_emit_ldarg (mb, 0);
7771 mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
7772 mono_mb_emit_byte (mb, CEE_SHR_UN);
7773 mono_mb_emit_icon (mb, (mword)nursery_start >> DEFAULT_NURSERY_BITS);
7774 label_no_wb_1 = mono_mb_emit_branch (mb, CEE_BEQ);
7776 // if (!ptr_in_nursery (*ptr)) return;
7777 mono_mb_emit_ldarg (mb, 0);
7778 mono_mb_emit_byte (mb, CEE_LDIND_I);
7779 mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
7780 mono_mb_emit_byte (mb, CEE_SHR_UN);
7781 mono_mb_emit_icon (mb, (mword)nursery_start >> DEFAULT_NURSERY_BITS);
7782 label_no_wb_2 = mono_mb_emit_branch (mb, CEE_BNE_UN);
7785 // if (ptr < (nursery_start)) goto continue;
7786 mono_mb_emit_ldarg (mb, 0);
7787 mono_mb_emit_ptr (mb, (gpointer) nursery_start);
7788 label_continue_1 = mono_mb_emit_branch (mb, CEE_BLT);
7790 // if (ptr >= nursery_real_end)) goto continue;
7791 mono_mb_emit_ldarg (mb, 0);
7792 mono_mb_emit_ptr (mb, (gpointer) nursery_real_end);
7793 label_continue_2 = mono_mb_emit_branch (mb, CEE_BGE);
7796 label_no_wb_1 = mono_mb_emit_branch (mb, CEE_BR);
7799 mono_mb_patch_branch (mb, label_continue_1);
7800 mono_mb_patch_branch (mb, label_continue_2);
7802 // Dereference and store in local var
7803 dereferenced_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7804 mono_mb_emit_ldarg (mb, 0);
7805 mono_mb_emit_byte (mb, CEE_LDIND_I);
7806 mono_mb_emit_stloc (mb, dereferenced_var);
7808 // if (*ptr < nursery_start) return;
7809 mono_mb_emit_ldloc (mb, dereferenced_var);
7810 mono_mb_emit_ptr (mb, (gpointer) nursery_start);
7811 label_no_wb_2 = mono_mb_emit_branch (mb, CEE_BLT);
7813 // if (*ptr >= nursery_end) return;
7814 mono_mb_emit_ldloc (mb, dereferenced_var);
7815 mono_mb_emit_ptr (mb, (gpointer) nursery_real_end);
7816 label_no_wb_5 = mono_mb_emit_branch (mb, CEE_BGE);
7819 // if (ptr >= stack_end) goto need_wb;
7820 mono_mb_emit_ldarg (mb, 0);
7821 EMIT_TLS_ACCESS (mb, stack_end, stack_end_offset);
7822 label_need_wb = mono_mb_emit_branch (mb, CEE_BGE_UN);
7824 // if (ptr >= stack_start) return;
7825 dummy_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7826 mono_mb_emit_ldarg (mb, 0);
7827 mono_mb_emit_ldloc_addr (mb, dummy_var);
7828 label_no_wb_3 = mono_mb_emit_branch (mb, CEE_BGE_UN);
7831 mono_mb_patch_branch (mb, label_need_wb);
7833 // buffer = STORE_REMSET_BUFFER;
7834 buffer_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7835 EMIT_TLS_ACCESS (mb, store_remset_buffer, store_remset_buffer_offset);
7836 mono_mb_emit_stloc (mb, buffer_var);
7838 // buffer_index = STORE_REMSET_BUFFER_INDEX;
7839 buffer_index_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7840 EMIT_TLS_ACCESS (mb, store_remset_buffer_index, store_remset_buffer_index_offset);
7841 mono_mb_emit_stloc (mb, buffer_index_var);
7843 // if (buffer [buffer_index] == ptr) return;
7844 mono_mb_emit_ldloc (mb, buffer_var);
7845 mono_mb_emit_ldloc (mb, buffer_index_var);
7846 g_assert (sizeof (gpointer) == 4 || sizeof (gpointer) == 8);
7847 mono_mb_emit_icon (mb, sizeof (gpointer) == 4 ? 2 : 3);
7848 mono_mb_emit_byte (mb, CEE_SHL);
7849 mono_mb_emit_byte (mb, CEE_ADD);
7850 mono_mb_emit_byte (mb, CEE_LDIND_I);
7851 mono_mb_emit_ldarg (mb, 0);
7852 label_no_wb_4 = mono_mb_emit_branch (mb, CEE_BEQ);
7855 mono_mb_emit_ldloc (mb, buffer_index_var);
7856 mono_mb_emit_icon (mb, 1);
7857 mono_mb_emit_byte (mb, CEE_ADD);
7858 mono_mb_emit_stloc (mb, buffer_index_var);
7860 // if (buffer_index >= STORE_REMSET_BUFFER_SIZE) goto slow_path;
7861 mono_mb_emit_ldloc (mb, buffer_index_var);
7862 mono_mb_emit_icon (mb, STORE_REMSET_BUFFER_SIZE);
7863 label_slow_path = mono_mb_emit_branch (mb, CEE_BGE);
7865 // buffer [buffer_index] = ptr;
7866 mono_mb_emit_ldloc (mb, buffer_var);
7867 mono_mb_emit_ldloc (mb, buffer_index_var);
7868 g_assert (sizeof (gpointer) == 4 || sizeof (gpointer) == 8);
7869 mono_mb_emit_icon (mb, sizeof (gpointer) == 4 ? 2 : 3);
7870 mono_mb_emit_byte (mb, CEE_SHL);
7871 mono_mb_emit_byte (mb, CEE_ADD);
7872 mono_mb_emit_ldarg (mb, 0);
7873 mono_mb_emit_byte (mb, CEE_STIND_I);
7875 // STORE_REMSET_BUFFER_INDEX = buffer_index;
7876 EMIT_TLS_ACCESS (mb, store_remset_buffer_index_addr, store_remset_buffer_index_addr_offset);
7877 mono_mb_emit_ldloc (mb, buffer_index_var);
7878 mono_mb_emit_byte (mb, CEE_STIND_I);
7881 mono_mb_patch_branch (mb, label_no_wb_1);
7882 mono_mb_patch_branch (mb, label_no_wb_2);
7883 mono_mb_patch_branch (mb, label_no_wb_3);
7884 mono_mb_patch_branch (mb, label_no_wb_4);
7885 #ifndef ALIGN_NURSERY
7886 mono_mb_patch_branch (mb, label_no_wb_5);
7888 mono_mb_emit_byte (mb, CEE_RET);
7891 mono_mb_patch_branch (mb, label_slow_path);
7895 mono_mb_emit_ldarg (mb, 0);
7896 mono_mb_emit_icall (mb, mono_gc_wbarrier_generic_nostore);
7897 mono_mb_emit_byte (mb, CEE_RET);
7899 res = mono_mb_create_method (mb, sig, 16);
7902 mono_loader_lock ();
7903 if (write_barrier_method) {
7904 /* Already created */
7905 mono_free_method (res);
7907 /* double-checked locking */
7908 mono_memory_barrier ();
7909 write_barrier_method = res;
7911 mono_loader_unlock ();
7913 return write_barrier_method;
7916 #endif /* HAVE_SGEN_GC */