2 * sgen-gc.c: Simple generational GC.
5 * Paolo Molaro (lupus@ximian.com)
7 * Copyright 2005-2010 Novell, Inc (http://www.novell.com)
9 * Thread start/stop adapted from Boehm's GC:
10 * Copyright (c) 1994 by Xerox Corporation. All rights reserved.
11 * Copyright (c) 1996 by Silicon Graphics. All rights reserved.
12 * Copyright (c) 1998 by Fergus Henderson. All rights reserved.
13 * Copyright (c) 2000-2004 by Hewlett-Packard Company. All rights reserved.
15 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
16 * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
18 * Permission is hereby granted to use or copy this program
19 * for any purpose, provided the above notices are retained on all copies.
20 * Permission to modify the code and to distribute modified code is granted,
21 * provided the above notices are retained, and a notice that the code was
22 * modified is included with the above copyright notice.
25 * Copyright 2001-2003 Ximian, Inc
26 * Copyright 2003-2010 Novell, Inc.
28 * Permission is hereby granted, free of charge, to any person obtaining
29 * a copy of this software and associated documentation files (the
30 * "Software"), to deal in the Software without restriction, including
31 * without limitation the rights to use, copy, modify, merge, publish,
32 * distribute, sublicense, and/or sell copies of the Software, and to
33 * permit persons to whom the Software is furnished to do so, subject to
34 * the following conditions:
36 * The above copyright notice and this permission notice shall be
37 * included in all copies or substantial portions of the Software.
39 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
40 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
41 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
42 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
43 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
44 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
45 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
48 * Important: allocation provides always zeroed memory, having to do
49 * a memset after allocation is deadly for performance.
50 * Memory usage at startup is currently as follows:
52 * 64 KB internal space
54 * We should provide a small memory config with half the sizes
56 * We currently try to make as few mono assumptions as possible:
57 * 1) 2-word header with no GC pointers in it (first vtable, second to store the
59 * 2) gc descriptor is the second word in the vtable (first word in the class)
60 * 3) 8 byte alignment is the minimum and enough (not true for special structures (SIMD), FIXME)
61 * 4) there is a function to get an object's size and the number of
62 * elements in an array.
63 * 5) we know the special way bounds are allocated for complex arrays
64 * 6) we know about proxies and how to treat them when domains are unloaded
66 * Always try to keep stack usage to a minimum: no recursive behaviour
67 * and no large stack allocs.
69 * General description.
70 * Objects are initially allocated in a nursery using a fast bump-pointer technique.
71 * When the nursery is full we start a nursery collection: this is performed with a
73 * When the old generation is full we start a copying GC of the old generation as well:
74 * this will be changed to mark&sweep with copying when fragmentation becomes to severe
75 * in the future. Maybe we'll even do both during the same collection like IMMIX.
77 * The things that complicate this description are:
78 * *) pinned objects: we can't move them so we need to keep track of them
79 * *) no precise info of the thread stacks and registers: we need to be able to
80 * quickly find the objects that may be referenced conservatively and pin them
81 * (this makes the first issues more important)
82 * *) large objects are too expensive to be dealt with using copying GC: we handle them
83 * with mark/sweep during major collections
84 * *) some objects need to not move even if they are small (interned strings, Type handles):
85 * we use mark/sweep for them, too: they are not allocated in the nursery, but inside
86 * PinnedChunks regions
92 *) we could have a function pointer in MonoClass to implement
93 customized write barriers for value types
95 *) investigate the stuff needed to advance a thread to a GC-safe
96 point (single-stepping, read from unmapped memory etc) and implement it.
97 This would enable us to inline allocations and write barriers, for example,
98 or at least parts of them, like the write barrier checks.
99 We may need this also for handling precise info on stacks, even simple things
100 as having uninitialized data on the stack and having to wait for the prolog
101 to zero it. Not an issue for the last frame that we scan conservatively.
102 We could always not trust the value in the slots anyway.
104 *) modify the jit to save info about references in stack locations:
105 this can be done just for locals as a start, so that at least
106 part of the stack is handled precisely.
108 *) test/fix endianess issues
110 *) Implement a card table as the write barrier instead of remembered
111 sets? Card tables are not easy to implement with our current
112 memory layout. We have several different kinds of major heap
113 objects: Small objects in regular blocks, small objects in pinned
114 chunks and LOS objects. If we just have a pointer we have no way
115 to tell which kind of object it points into, therefore we cannot
116 know where its card table is. The least we have to do to make
117 this happen is to get rid of write barriers for indirect stores.
120 *) Get rid of write barriers for indirect stores. We can do this by
121 telling the GC to wbarrier-register an object once we do an ldloca
122 or ldelema on it, and to unregister it once it's not used anymore
123 (it can only travel downwards on the stack). The problem with
124 unregistering is that it needs to happen eventually no matter
125 what, even if exceptions are thrown, the thread aborts, etc.
126 Rodrigo suggested that we could do only the registering part and
127 let the collector find out (pessimistically) when it's safe to
128 unregister, namely when the stack pointer of the thread that
129 registered the object is higher than it was when the registering
130 happened. This might make for a good first implementation to get
131 some data on performance.
133 *) Some sort of blacklist support? Blacklists is a concept from the
134 Boehm GC: if during a conservative scan we find pointers to an
135 area which we might use as heap, we mark that area as unusable, so
136 pointer retention by random pinning pointers is reduced.
138 *) experiment with max small object size (very small right now - 2kb,
139 because it's tied to the max freelist size)
141 *) add an option to mmap the whole heap in one chunk: it makes for many
142 simplifications in the checks (put the nursery at the top and just use a single
143 check for inclusion/exclusion): the issue this has is that on 32 bit systems it's
144 not flexible (too much of the address space may be used by default or we can't
145 increase the heap as needed) and we'd need a race-free mechanism to return memory
146 back to the system (mprotect(PROT_NONE) will still keep the memory allocated if it
147 was written to, munmap is needed, but the following mmap may not find the same segment
150 *) memzero the major fragments after restarting the world and optionally a smaller
153 *) investigate having fragment zeroing threads
155 *) separate locks for finalization and other minor stuff to reduce
158 *) try a different copying order to improve memory locality
160 *) a thread abort after a store but before the write barrier will
161 prevent the write barrier from executing
163 *) specialized dynamically generated markers/copiers
165 *) Dynamically adjust TLAB size to the number of threads. If we have
166 too many threads that do allocation, we might need smaller TLABs,
167 and we might get better performance with larger TLABs if we only
168 have a handful of threads. We could sum up the space left in all
169 assigned TLABs and if that's more than some percentage of the
170 nursery size, reduce the TLAB size.
172 *) Explore placing unreachable objects on unused nursery memory.
173 Instead of memset'ng a region to zero, place an int[] covering it.
174 A good place to start is add_nursery_frag. The tricky thing here is
175 placing those objects atomically outside of a collection.
185 #include <semaphore.h>
194 #define _XOPEN_SOURCE
196 #include "metadata/metadata-internals.h"
197 #include "metadata/class-internals.h"
198 #include "metadata/gc-internal.h"
199 #include "metadata/object-internals.h"
200 #include "metadata/threads.h"
201 #include "metadata/sgen-gc.h"
202 #include "metadata/sgen-cardtable.h"
203 #include "metadata/sgen-archdep.h"
204 #include "metadata/mono-gc.h"
205 #include "metadata/method-builder.h"
206 #include "metadata/profiler-private.h"
207 #include "metadata/monitor.h"
208 #include "metadata/threadpool-internals.h"
209 #include "metadata/mempool-internals.h"
210 #include "metadata/marshal.h"
211 #include "utils/mono-mmap.h"
212 #include "utils/mono-time.h"
213 #include "utils/mono-semaphore.h"
214 #include "utils/mono-counters.h"
215 #include "utils/mono-proclib.h"
217 #include <mono/utils/memcheck.h>
219 #if defined(__MACH__)
220 #include "utils/mach-support.h"
223 #define OPDEF(a,b,c,d,e,f,g,h,i,j) \
227 #include "mono/cil/opcode.def"
233 #undef pthread_create
235 #undef pthread_detach
238 * ######################################################################
239 * ######## Types and constants used by the GC.
240 * ######################################################################
243 static int gc_initialized = 0;
244 /* If set, do a minor collection before every allocation */
245 static gboolean collect_before_allocs = FALSE;
246 /* If set, do a heap consistency check before each minor collection */
247 static gboolean consistency_check_at_minor_collection = FALSE;
248 /* If set, check that there are no references to the domain left at domain unload */
249 static gboolean xdomain_checks = FALSE;
250 /* If not null, dump the heap after each collection into this file */
251 static FILE *heap_dump_file = NULL;
252 /* If set, mark stacks conservatively, even if precise marking is possible */
253 static gboolean conservative_stack_mark = TRUE;
254 /* If set, do a plausibility check on the scan_starts before and after
256 static gboolean do_scan_starts_check = FALSE;
258 #ifdef HEAVY_STATISTICS
259 static long long stat_objects_alloced = 0;
260 static long long stat_bytes_alloced = 0;
261 long long stat_objects_alloced_degraded = 0;
262 long long stat_bytes_alloced_degraded = 0;
263 static long long stat_bytes_alloced_los = 0;
265 long long stat_copy_object_called_nursery = 0;
266 long long stat_objects_copied_nursery = 0;
267 long long stat_copy_object_called_major = 0;
268 long long stat_objects_copied_major = 0;
270 long long stat_scan_object_called_nursery = 0;
271 long long stat_scan_object_called_major = 0;
273 long long stat_nursery_copy_object_failed_from_space = 0;
274 long long stat_nursery_copy_object_failed_forwarded = 0;
275 long long stat_nursery_copy_object_failed_pinned = 0;
277 static long long stat_store_remsets = 0;
278 static long long stat_store_remsets_unique = 0;
279 static long long stat_saved_remsets_1 = 0;
280 static long long stat_saved_remsets_2 = 0;
281 static long long stat_local_remsets_processed = 0;
282 static long long stat_global_remsets_added = 0;
283 static long long stat_global_remsets_readded = 0;
284 static long long stat_global_remsets_processed = 0;
285 static long long stat_global_remsets_discarded = 0;
287 static long long stat_wasted_fragments_used = 0;
288 static long long stat_wasted_fragments_bytes = 0;
290 static int stat_wbarrier_set_field = 0;
291 static int stat_wbarrier_set_arrayref = 0;
292 static int stat_wbarrier_arrayref_copy = 0;
293 static int stat_wbarrier_generic_store = 0;
294 static int stat_wbarrier_generic_store_remset = 0;
295 static int stat_wbarrier_set_root = 0;
296 static int stat_wbarrier_value_copy = 0;
297 static int stat_wbarrier_object_copy = 0;
300 static long long time_minor_pre_collection_fragment_clear = 0;
301 static long long time_minor_pinning = 0;
302 static long long time_minor_scan_remsets = 0;
303 static long long time_minor_scan_card_table = 0;
304 static long long time_minor_scan_pinned = 0;
305 static long long time_minor_scan_registered_roots = 0;
306 static long long time_minor_scan_thread_data = 0;
307 static long long time_minor_finish_gray_stack = 0;
308 static long long time_minor_fragment_creation = 0;
310 static long long time_major_pre_collection_fragment_clear = 0;
311 static long long time_major_pinning = 0;
312 static long long time_major_scan_pinned = 0;
313 static long long time_major_scan_registered_roots = 0;
314 static long long time_major_scan_thread_data = 0;
315 static long long time_major_scan_alloc_pinned = 0;
316 static long long time_major_scan_finalized = 0;
317 static long long time_major_scan_big_objects = 0;
318 static long long time_major_finish_gray_stack = 0;
319 static long long time_major_free_bigobjs = 0;
320 static long long time_major_los_sweep = 0;
321 static long long time_major_sweep = 0;
322 static long long time_major_fragment_creation = 0;
324 #define DEBUG(level,a) do {if (G_UNLIKELY ((level) <= SGEN_MAX_DEBUG_LEVEL && (level) <= gc_debug_level)) a;} while (0)
326 static int gc_debug_level = 0;
327 static FILE* gc_debug_file;
331 mono_gc_flush_info (void)
333 fflush (gc_debug_file);
338 * Define this to allow the user to change the nursery size by
339 * specifying its value in the MONO_GC_PARAMS environmental
340 * variable. See mono_gc_base_init for details.
342 #define USER_CONFIG 1
344 #define TV_DECLARE(name) gint64 name
345 #define TV_GETTIME(tv) tv = mono_100ns_ticks ()
346 #define TV_ELAPSED(start,end) (int)((end-start) / 10)
347 #define TV_ELAPSED_MS(start,end) ((TV_ELAPSED((start),(end)) + 500) / 1000)
349 #define ALIGN_TO(val,align) ((((guint64)val) + ((align) - 1)) & ~((align) - 1))
351 /* The method used to clear the nursery */
352 /* Clearing at nursery collections is the safest, but has bad interactions with caches.
353 * Clearing at TLAB creation is much faster, but more complex and it might expose hard
358 CLEAR_AT_TLAB_CREATION
359 } NurseryClearPolicy;
361 static NurseryClearPolicy nursery_clear_policy = CLEAR_AT_TLAB_CREATION;
364 * The young generation is divided into fragments. This is because
365 * we can hand one fragments to a thread for lock-less fast alloc and
366 * because the young generation ends up fragmented anyway by pinned objects.
367 * Once a collection is done, a list of fragments is created. When doing
368 * thread local alloc we use smallish nurseries so we allow new threads to
369 * allocate memory from gen0 without triggering a collection. Threads that
370 * are found to allocate lots of memory are given bigger fragments. This
371 * should make the finalizer thread use little nursery memory after a while.
372 * We should start assigning threads very small fragments: if there are many
373 * threads the nursery will be full of reserved space that the threads may not
374 * use at all, slowing down allocation speed.
375 * Thread local allocation is done from areas of memory Hotspot calls Thread Local
376 * Allocation Buffers (TLABs).
378 typedef struct _Fragment Fragment;
382 char *fragment_start;
383 char *fragment_limit; /* the current soft limit for allocation */
387 /* the runtime can register areas of memory as roots: we keep two lists of roots,
388 * a pinned root set for conservatively scanned roots and a normal one for
389 * precisely scanned roots (currently implemented as a single list).
391 typedef struct _RootRecord RootRecord;
400 * We're never actually using the first element. It's always set to
401 * NULL to simplify the elimination of consecutive duplicate
404 #define STORE_REMSET_BUFFER_SIZE 1024
406 typedef struct _GenericStoreRememberedSet GenericStoreRememberedSet;
407 struct _GenericStoreRememberedSet {
408 GenericStoreRememberedSet *next;
409 /* We need one entry less because the first entry of store
410 remset buffers is always a dummy and we don't copy it. */
411 gpointer data [STORE_REMSET_BUFFER_SIZE - 1];
414 /* we have 4 possible values in the low 2 bits */
416 REMSET_LOCATION, /* just a pointer to the exact location */
417 REMSET_RANGE, /* range of pointer fields */
418 REMSET_OBJECT, /* mark all the object for scanning */
419 REMSET_VTYPE, /* a valuetype array described by a gc descriptor and a count */
420 REMSET_TYPE_MASK = 0x3
423 #ifdef HAVE_KW_THREAD
424 static __thread RememberedSet *remembered_set MONO_TLS_FAST;
426 static pthread_key_t remembered_set_key;
427 static RememberedSet *global_remset;
428 static RememberedSet *freed_thread_remsets;
429 static GenericStoreRememberedSet *generic_store_remsets = NULL;
431 /*A two slots cache for recently inserted remsets */
432 static gpointer global_remset_cache [2];
434 /* FIXME: later choose a size that takes into account the RememberedSet struct
435 * and doesn't waste any alloc paddin space.
437 #define DEFAULT_REMSET_SIZE 1024
438 static RememberedSet* alloc_remset (int size, gpointer id);
440 #define object_is_forwarded SGEN_OBJECT_IS_FORWARDED
441 #define object_is_pinned SGEN_OBJECT_IS_PINNED
442 #define pin_object SGEN_PIN_OBJECT
443 #define unpin_object SGEN_UNPIN_OBJECT
445 #define ptr_in_nursery(p) (SGEN_PTR_IN_NURSERY ((p), DEFAULT_NURSERY_BITS, nursery_start, nursery_real_end))
447 #define LOAD_VTABLE SGEN_LOAD_VTABLE
450 safe_name (void* obj)
452 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (obj);
453 return vt->klass->name;
456 #define safe_object_get_size mono_sgen_safe_object_get_size
459 * ######################################################################
460 * ######## Global data.
461 * ######################################################################
463 static LOCK_DECLARE (gc_mutex);
464 static int gc_disabled = 0;
465 static int num_minor_gcs = 0;
466 static int num_major_gcs = 0;
468 static gboolean use_cardtable;
472 /* good sizes are 512KB-1MB: larger ones increase a lot memzeroing time */
473 #define DEFAULT_NURSERY_SIZE (default_nursery_size)
474 static int default_nursery_size = (1 << 22);
475 #ifdef SGEN_ALIGN_NURSERY
476 /* The number of trailing 0 bits in DEFAULT_NURSERY_SIZE */
477 #define DEFAULT_NURSERY_BITS (default_nursery_bits)
478 static int default_nursery_bits = 22;
483 #define DEFAULT_NURSERY_SIZE (4*1024*1024)
484 #ifdef SGEN_ALIGN_NURSERY
485 #define DEFAULT_NURSERY_BITS 22
490 #ifndef SGEN_ALIGN_NURSERY
491 #define DEFAULT_NURSERY_BITS -1
494 #define MIN_MINOR_COLLECTION_ALLOWANCE (DEFAULT_NURSERY_SIZE * 4)
496 #define SCAN_START_SIZE SGEN_SCAN_START_SIZE
498 /* the minimum size of a fragment that we consider useful for allocation */
499 #define FRAGMENT_MIN_SIZE (512)
501 static mword pagesize = 4096;
502 static mword nursery_size;
503 static int degraded_mode = 0;
505 static mword total_alloc = 0;
506 /* use this to tune when to do a major/minor collection */
507 static mword memory_pressure = 0;
508 static mword minor_collection_allowance;
509 static int minor_collection_sections_alloced = 0;
511 static GCMemSection *nursery_section = NULL;
512 static mword lowest_heap_address = ~(mword)0;
513 static mword highest_heap_address = 0;
515 static LOCK_DECLARE (interruption_mutex);
516 static LOCK_DECLARE (global_remset_mutex);
517 static LOCK_DECLARE (pin_queue_mutex);
519 #define LOCK_GLOBAL_REMSET pthread_mutex_lock (&global_remset_mutex)
520 #define UNLOCK_GLOBAL_REMSET pthread_mutex_unlock (&global_remset_mutex)
522 #define LOCK_PIN_QUEUE pthread_mutex_lock (&pin_queue_mutex)
523 #define UNLOCK_PIN_QUEUE pthread_mutex_unlock (&pin_queue_mutex)
525 typedef struct _FinalizeEntry FinalizeEntry;
526 struct _FinalizeEntry {
531 typedef struct _FinalizeEntryHashTable FinalizeEntryHashTable;
532 struct _FinalizeEntryHashTable {
533 FinalizeEntry **table;
538 typedef struct _DisappearingLink DisappearingLink;
539 struct _DisappearingLink {
540 DisappearingLink *next;
544 typedef struct _DisappearingLinkHashTable DisappearingLinkHashTable;
545 struct _DisappearingLinkHashTable {
546 DisappearingLink **table;
551 typedef struct _EphemeronLinkNode EphemeronLinkNode;
553 struct _EphemeronLinkNode {
554 EphemeronLinkNode *next;
563 int current_collection_generation = -1;
566 * The link pointer is hidden by negating each bit. We use the lowest
567 * bit of the link (before negation) to store whether it needs
568 * resurrection tracking.
570 #define HIDE_POINTER(p,t) ((gpointer)(~((gulong)(p)|((t)?1:0))))
571 #define REVEAL_POINTER(p) ((gpointer)((~(gulong)(p))&~3L))
573 #define DISLINK_OBJECT(d) (REVEAL_POINTER (*(d)->link))
574 #define DISLINK_TRACK(d) ((~(gulong)(*(d)->link)) & 1)
577 * The finalizable hash has the object as the key, the
578 * disappearing_link hash, has the link address as key.
580 static FinalizeEntryHashTable minor_finalizable_hash;
581 static FinalizeEntryHashTable major_finalizable_hash;
582 /* objects that are ready to be finalized */
583 static FinalizeEntry *fin_ready_list = NULL;
584 static FinalizeEntry *critical_fin_list = NULL;
586 static DisappearingLinkHashTable minor_disappearing_link_hash;
587 static DisappearingLinkHashTable major_disappearing_link_hash;
589 static EphemeronLinkNode *ephemeron_list;
591 static int num_ready_finalizers = 0;
592 static int no_finalize = 0;
595 ROOT_TYPE_NORMAL = 0, /* "normal" roots */
596 ROOT_TYPE_PINNED = 1, /* roots without a GC descriptor */
597 ROOT_TYPE_WBARRIER = 2, /* roots with a write barrier */
601 /* registered roots: the key to the hash is the root start address */
603 * Different kinds of roots are kept separate to speed up pin_from_roots () for example.
605 static RootRecord **roots_hash [ROOT_TYPE_NUM] = { NULL, NULL };
606 static int roots_hash_size [ROOT_TYPE_NUM] = { 0, 0, 0 };
607 static mword roots_size = 0; /* amount of memory in the root set */
608 static int num_roots_entries [ROOT_TYPE_NUM] = { 0, 0, 0 };
611 * The current allocation cursors
612 * We allocate objects in the nursery.
613 * The nursery is the area between nursery_start and nursery_real_end.
614 * Allocation is done from a Thread Local Allocation Buffer (TLAB). TLABs are allocated
615 * from nursery fragments.
616 * tlab_next is the pointer to the space inside the TLAB where the next object will
618 * tlab_temp_end is the pointer to the end of the temporary space reserved for
619 * the allocation: it allows us to set the scan starts at reasonable intervals.
620 * tlab_real_end points to the end of the TLAB.
621 * nursery_frag_real_end points to the end of the currently used nursery fragment.
622 * nursery_first_pinned_start points to the start of the first pinned object in the nursery
623 * nursery_last_pinned_end points to the end of the last pinned object in the nursery
624 * At the next allocation, the area of the nursery where objects can be present is
625 * between MIN(nursery_first_pinned_start, first_fragment_start) and
626 * MAX(nursery_last_pinned_end, nursery_frag_real_end)
628 static char *nursery_start = NULL;
630 #ifdef HAVE_KW_THREAD
631 #define TLAB_ACCESS_INIT
632 #define TLAB_START tlab_start
633 #define TLAB_NEXT tlab_next
634 #define TLAB_TEMP_END tlab_temp_end
635 #define TLAB_REAL_END tlab_real_end
636 #define REMEMBERED_SET remembered_set
637 #define STORE_REMSET_BUFFER store_remset_buffer
638 #define STORE_REMSET_BUFFER_INDEX store_remset_buffer_index
639 #define IN_CRITICAL_REGION thread_info->in_critical_region
641 static pthread_key_t thread_info_key;
642 #define TLAB_ACCESS_INIT SgenThreadInfo *__thread_info__ = pthread_getspecific (thread_info_key)
643 #define TLAB_START (__thread_info__->tlab_start)
644 #define TLAB_NEXT (__thread_info__->tlab_next)
645 #define TLAB_TEMP_END (__thread_info__->tlab_temp_end)
646 #define TLAB_REAL_END (__thread_info__->tlab_real_end)
647 #define REMEMBERED_SET (__thread_info__->remset)
648 #define STORE_REMSET_BUFFER (__thread_info__->store_remset_buffer)
649 #define STORE_REMSET_BUFFER_INDEX (__thread_info__->store_remset_buffer_index)
650 #define IN_CRITICAL_REGION (__thread_info__->in_critical_region)
653 /* we use the memory barrier only to prevent compiler reordering (a memory constraint may be enough) */
654 #define ENTER_CRITICAL_REGION do {IN_CRITICAL_REGION = 1;mono_memory_barrier ();} while (0)
655 #define EXIT_CRITICAL_REGION do {IN_CRITICAL_REGION = 0;mono_memory_barrier ();} while (0)
658 * FIXME: What is faster, a TLS variable pointing to a structure, or separate TLS
659 * variables for next+temp_end ?
661 #ifdef HAVE_KW_THREAD
662 static __thread SgenThreadInfo *thread_info;
663 static __thread char *tlab_start;
664 static __thread char *tlab_next;
665 static __thread char *tlab_temp_end;
666 static __thread char *tlab_real_end;
667 static __thread gpointer *store_remset_buffer;
668 static __thread long store_remset_buffer_index;
669 /* Used by the managed allocator/wbarrier */
670 static __thread char **tlab_next_addr;
671 static __thread char *stack_end;
672 static __thread long *store_remset_buffer_index_addr;
674 static char *nursery_next = NULL;
675 static char *nursery_frag_real_end = NULL;
676 static char *nursery_real_end = NULL;
677 static char *nursery_last_pinned_end = NULL;
679 /* The size of a TLAB */
680 /* The bigger the value, the less often we have to go to the slow path to allocate a new
681 * one, but the more space is wasted by threads not allocating much memory.
683 * FIXME: Make this self-tuning for each thread.
685 static guint32 tlab_size = (1024 * 4);
687 /*How much space is tolerable to be wasted from the current fragment when allocating a new TLAB*/
688 #define MAX_NURSERY_TLAB_WASTE 512
690 /* fragments that are free and ready to be used for allocation */
691 static Fragment *nursery_fragments = NULL;
692 /* freeelist of fragment structures */
693 static Fragment *fragment_freelist = NULL;
695 #define MAX_SMALL_OBJ_SIZE SGEN_MAX_SMALL_OBJ_SIZE
697 /* Functions supplied by the runtime to be called by the GC */
698 static MonoGCCallbacks gc_callbacks;
700 #define ALLOC_ALIGN SGEN_ALLOC_ALIGN
701 #define ALLOC_ALIGN_BITS SGEN_ALLOC_ALIGN_BITS
703 #define ALIGN_UP SGEN_ALIGN_UP
705 #define MOVED_OBJECTS_NUM 64
706 static void *moved_objects [MOVED_OBJECTS_NUM];
707 static int moved_objects_idx = 0;
709 /* Vtable of the objects used to fill out nursery fragments before a collection */
710 static MonoVTable *array_fill_vtable;
713 * ######################################################################
714 * ######## Heap size accounting
715 * ######################################################################
718 static mword max_heap_size = ((mword)0)- ((mword)1);
719 static mword allocated_heap;
721 /*Object was pinned during the current collection*/
722 static mword objects_pinned;
725 mono_sgen_release_space (mword size, int space)
727 allocated_heap -= size;
731 available_free_space (void)
733 return max_heap_size - MIN (allocated_heap, max_heap_size);
737 mono_sgen_try_alloc_space (mword size, int space)
739 if (available_free_space () < size)
742 allocated_heap += size;
747 init_heap_size_limits (glong max_heap)
752 if (max_heap < nursery_size * 4) {
753 fprintf (stderr, "max-heap-size must be at least 4 times larger than nursery size.\n");
756 max_heap_size = max_heap - nursery_size;
760 * ######################################################################
761 * ######## Macros and function declarations.
762 * ######################################################################
765 #define ADDR_IN_HEAP_BOUNDARIES(addr) ((p) >= lowest_heap_address && (p) < highest_heap_address)
768 align_pointer (void *ptr)
770 mword p = (mword)ptr;
771 p += sizeof (gpointer) - 1;
772 p &= ~ (sizeof (gpointer) - 1);
776 typedef SgenGrayQueue GrayQueue;
778 typedef void (*CopyOrMarkObjectFunc) (void**, GrayQueue*);
779 typedef char* (*ScanObjectFunc) (char*, GrayQueue*);
781 /* forward declarations */
782 static int stop_world (int generation);
783 static int restart_world (int generation);
784 static void scan_thread_data (void *start_nursery, void *end_nursery, gboolean precise);
785 static void scan_from_remsets (void *start_nursery, void *end_nursery, GrayQueue *queue);
786 static void scan_from_registered_roots (CopyOrMarkObjectFunc copy_func, char *addr_start, char *addr_end, int root_type, GrayQueue *queue);
787 static void scan_finalizer_entries (CopyOrMarkObjectFunc copy_func, FinalizeEntry *list, GrayQueue *queue);
788 static void find_pinning_ref_from_thread (char *obj, size_t size);
789 static void update_current_thread_stack (void *start);
790 static void finalize_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, int generation, GrayQueue *queue);
791 static void add_or_remove_disappearing_link (MonoObject *obj, void **link, gboolean track, int generation);
792 static void null_link_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, int generation, GrayQueue *queue);
793 static void null_links_for_domain (MonoDomain *domain, int generation);
794 static gboolean search_fragment_for_size (size_t size);
795 static int search_fragment_for_size_range (size_t desired_size, size_t minimum_size);
796 static void clear_nursery_fragments (char *next);
797 static void pin_from_roots (void *start_nursery, void *end_nursery);
798 static int pin_objects_from_addresses (GCMemSection *section, void **start, void **end, void *start_nursery, void *end_nursery, GrayQueue *queue);
799 static void optimize_pin_queue (int start_slot);
800 static void clear_remsets (void);
801 static void clear_tlabs (void);
802 static void sort_addresses (void **array, int size);
803 static void drain_gray_stack (GrayQueue *queue);
804 static void finish_gray_stack (char *start_addr, char *end_addr, int generation, GrayQueue *queue);
805 static gboolean need_major_collection (mword space_needed);
806 static void major_collection (const char *reason);
808 static void mono_gc_register_disappearing_link (MonoObject *obj, void **link, gboolean track);
810 void describe_ptr (char *ptr);
811 void check_object (char *start);
813 static void check_consistency (void);
814 static void check_major_refs (void);
815 static void check_scan_starts (void);
816 static void check_for_xdomain_refs (void);
817 static void dump_heap (const char *type, int num, const char *reason);
819 void mono_gc_scan_for_specific_ref (MonoObject *key);
821 static void init_stats (void);
823 static int mark_ephemerons_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, GrayQueue *queue);
824 static void clear_unreachable_ephemerons (CopyOrMarkObjectFunc copy_func, char *start, char *end, GrayQueue *queue);
825 static void null_ephemerons_for_domain (MonoDomain *domain);
827 SgenMajorCollector major_collector;
829 #include "sgen-protocol.c"
830 #include "sgen-pinning.c"
831 #include "sgen-pinning-stats.c"
832 #include "sgen-gray.c"
833 #include "sgen-workers.c"
834 #include "sgen-los.c"
835 #include "sgen-cardtable.c"
837 /* Root bitmap descriptors are simpler: the lower three bits describe the type
838 * and we either have 30/62 bitmap bits or nibble-based run-length,
839 * or a complex descriptor, or a user defined marker function.
842 ROOT_DESC_CONSERVATIVE, /* 0, so matches NULL value */
847 ROOT_DESC_TYPE_MASK = 0x7,
848 ROOT_DESC_TYPE_SHIFT = 3,
851 #define MAKE_ROOT_DESC(type,val) ((type) | ((val) << ROOT_DESC_TYPE_SHIFT))
853 #define MAX_USER_DESCRIPTORS 16
855 static gsize* complex_descriptors = NULL;
856 static int complex_descriptors_size = 0;
857 static int complex_descriptors_next = 0;
858 static MonoGCRootMarkFunc user_descriptors [MAX_USER_DESCRIPTORS];
859 static int user_descriptors_next = 0;
862 alloc_complex_descriptor (gsize *bitmap, int numbits)
866 numbits = ALIGN_TO (numbits, GC_BITS_PER_WORD);
867 nwords = numbits / GC_BITS_PER_WORD + 1;
870 res = complex_descriptors_next;
871 /* linear search, so we don't have duplicates with domain load/unload
872 * this should not be performance critical or we'd have bigger issues
873 * (the number and size of complex descriptors should be small).
875 for (i = 0; i < complex_descriptors_next; ) {
876 if (complex_descriptors [i] == nwords) {
878 for (j = 0; j < nwords - 1; ++j) {
879 if (complex_descriptors [i + 1 + j] != bitmap [j]) {
889 i += complex_descriptors [i];
891 if (complex_descriptors_next + nwords > complex_descriptors_size) {
892 int new_size = complex_descriptors_size * 2 + nwords;
893 complex_descriptors = g_realloc (complex_descriptors, new_size * sizeof (gsize));
894 complex_descriptors_size = new_size;
896 DEBUG (6, fprintf (gc_debug_file, "Complex descriptor %d, size: %d (total desc memory: %d)\n", res, nwords, complex_descriptors_size));
897 complex_descriptors_next += nwords;
898 complex_descriptors [res] = nwords;
899 for (i = 0; i < nwords - 1; ++i) {
900 complex_descriptors [res + 1 + i] = bitmap [i];
901 DEBUG (6, fprintf (gc_debug_file, "\tvalue: %p\n", (void*)complex_descriptors [res + 1 + i]));
908 mono_sgen_get_complex_descriptor (GCVTable *vt)
910 return complex_descriptors + (vt->desc >> LOW_TYPE_BITS);
914 * Descriptor builders.
917 mono_gc_make_descr_for_string (gsize *bitmap, int numbits)
919 return (void*) DESC_TYPE_RUN_LENGTH;
923 mono_gc_make_descr_for_object (gsize *bitmap, int numbits, size_t obj_size)
925 int first_set = -1, num_set = 0, last_set = -1, i;
927 size_t stored_size = obj_size;
928 for (i = 0; i < numbits; ++i) {
929 if (bitmap [i / GC_BITS_PER_WORD] & ((gsize)1 << (i % GC_BITS_PER_WORD))) {
937 * We don't encode the size of types that don't contain
938 * references because they might not be aligned, i.e. the
939 * bottom two bits might be set, which would clash with the
940 * bits we need to encode the descriptor type. Since we don't
941 * use the encoded size to skip objects, other than for
942 * processing remsets, in which case only the positions of
943 * references are relevant, this is not a problem.
946 return (void*)DESC_TYPE_RUN_LENGTH;
947 g_assert (!(stored_size & 0x3));
948 if (stored_size <= MAX_SMALL_OBJ_SIZE) {
949 /* check run-length encoding first: one byte offset, one byte number of pointers
950 * on 64 bit archs, we can have 3 runs, just one on 32.
951 * It may be better to use nibbles.
954 desc = DESC_TYPE_RUN_LENGTH | (stored_size << 1);
955 DEBUG (6, fprintf (gc_debug_file, "Ptrfree descriptor %p, size: %zd\n", (void*)desc, stored_size));
957 } else if (first_set < 256 && num_set < 256 && (first_set + num_set == last_set + 1)) {
958 desc = DESC_TYPE_RUN_LENGTH | (stored_size << 1) | (first_set << 16) | (num_set << 24);
959 DEBUG (6, fprintf (gc_debug_file, "Runlen descriptor %p, size: %zd, first set: %d, num set: %d\n", (void*)desc, stored_size, first_set, num_set));
962 /* we know the 2-word header is ptr-free */
963 if (last_set < SMALL_BITMAP_SIZE + OBJECT_HEADER_WORDS) {
964 desc = DESC_TYPE_SMALL_BITMAP | (stored_size << 1) | ((*bitmap >> OBJECT_HEADER_WORDS) << SMALL_BITMAP_SHIFT);
965 DEBUG (6, fprintf (gc_debug_file, "Smallbitmap descriptor %p, size: %zd, last set: %d\n", (void*)desc, stored_size, last_set));
969 /* we know the 2-word header is ptr-free */
970 if (last_set < LARGE_BITMAP_SIZE + OBJECT_HEADER_WORDS) {
971 desc = DESC_TYPE_LARGE_BITMAP | ((*bitmap >> OBJECT_HEADER_WORDS) << LOW_TYPE_BITS);
972 DEBUG (6, fprintf (gc_debug_file, "Largebitmap descriptor %p, size: %zd, last set: %d\n", (void*)desc, stored_size, last_set));
975 /* it's a complex object ... */
976 desc = DESC_TYPE_COMPLEX | (alloc_complex_descriptor (bitmap, last_set + 1) << LOW_TYPE_BITS);
980 /* If the array holds references, numbits == 1 and the first bit is set in elem_bitmap */
982 mono_gc_make_descr_for_array (int vector, gsize *elem_bitmap, int numbits, size_t elem_size)
984 int first_set = -1, num_set = 0, last_set = -1, i;
985 mword desc = vector? DESC_TYPE_VECTOR: DESC_TYPE_ARRAY;
986 for (i = 0; i < numbits; ++i) {
987 if (elem_bitmap [i / GC_BITS_PER_WORD] & ((gsize)1 << (i % GC_BITS_PER_WORD))) {
994 /* See comment at the definition of DESC_TYPE_RUN_LENGTH. */
996 return (void*)DESC_TYPE_RUN_LENGTH;
997 if (elem_size <= MAX_ELEMENT_SIZE) {
998 desc |= elem_size << VECTOR_ELSIZE_SHIFT;
1000 return (void*)(desc | VECTOR_SUBTYPE_PTRFREE);
1002 /* Note: we also handle structs with just ref fields */
1003 if (num_set * sizeof (gpointer) == elem_size) {
1004 return (void*)(desc | VECTOR_SUBTYPE_REFS | ((gssize)(-1) << 16));
1006 /* FIXME: try run-len first */
1007 /* Note: we can't skip the object header here, because it's not present */
1008 if (last_set <= SMALL_BITMAP_SIZE) {
1009 return (void*)(desc | VECTOR_SUBTYPE_BITMAP | (*elem_bitmap << 16));
1012 /* it's am array of complex structs ... */
1013 desc = DESC_TYPE_COMPLEX_ARR;
1014 desc |= alloc_complex_descriptor (elem_bitmap, last_set + 1) << LOW_TYPE_BITS;
1015 return (void*) desc;
1018 /* Return the bitmap encoded by a descriptor */
1020 mono_gc_get_bitmap_for_descr (void *descr, int *numbits)
1022 mword d = (mword)descr;
1026 case DESC_TYPE_RUN_LENGTH: {
1027 int first_set = (d >> 16) & 0xff;
1028 int num_set = (d >> 24) & 0xff;
1031 bitmap = g_new0 (gsize, (first_set + num_set + 7) / 8);
1033 for (i = first_set; i < first_set + num_set; ++i)
1034 bitmap [i / GC_BITS_PER_WORD] |= ((gsize)1 << (i % GC_BITS_PER_WORD));
1036 *numbits = first_set + num_set;
1040 case DESC_TYPE_SMALL_BITMAP:
1041 bitmap = g_new0 (gsize, 1);
1043 bitmap [0] = (d >> SMALL_BITMAP_SHIFT) << OBJECT_HEADER_WORDS;
1045 *numbits = GC_BITS_PER_WORD;
1049 g_assert_not_reached ();
1054 is_xdomain_ref_allowed (gpointer *ptr, char *obj, MonoDomain *domain)
1056 MonoObject *o = (MonoObject*)(obj);
1057 MonoObject *ref = (MonoObject*)*(ptr);
1058 int offset = (char*)(ptr) - (char*)o;
1060 if (o->vtable->klass == mono_defaults.thread_class && offset == G_STRUCT_OFFSET (MonoThread, internal_thread))
1062 if (o->vtable->klass == mono_defaults.internal_thread_class && offset == G_STRUCT_OFFSET (MonoInternalThread, current_appcontext))
1064 if (mono_class_has_parent (o->vtable->klass, mono_defaults.real_proxy_class) &&
1065 offset == G_STRUCT_OFFSET (MonoRealProxy, unwrapped_server))
1067 /* Thread.cached_culture_info */
1068 if (!strcmp (ref->vtable->klass->name_space, "System.Globalization") &&
1069 !strcmp (ref->vtable->klass->name, "CultureInfo") &&
1070 !strcmp(o->vtable->klass->name_space, "System") &&
1071 !strcmp(o->vtable->klass->name, "Object[]"))
1074 * at System.IO.MemoryStream.InternalConstructor (byte[],int,int,bool,bool) [0x0004d] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.IO/MemoryStream.cs:121
1075 * at System.IO.MemoryStream..ctor (byte[]) [0x00017] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.IO/MemoryStream.cs:81
1076 * at (wrapper remoting-invoke-with-check) System.IO.MemoryStream..ctor (byte[]) <IL 0x00020, 0xffffffff>
1077 * at System.Runtime.Remoting.Messaging.CADMethodCallMessage.GetArguments () [0x0000d] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Messaging/CADMessages.cs:327
1078 * at System.Runtime.Remoting.Messaging.MethodCall..ctor (System.Runtime.Remoting.Messaging.CADMethodCallMessage) [0x00017] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Messaging/MethodCall.cs:87
1079 * at System.AppDomain.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage,byte[]&,System.Runtime.Remoting.Messaging.CADMethodReturnMessage&) [0x00018] in /home/schani/Work/novell/trunk/mcs/class/corlib/System/AppDomain.cs:1213
1080 * at (wrapper remoting-invoke-with-check) System.AppDomain.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage,byte[]&,System.Runtime.Remoting.Messaging.CADMethodReturnMessage&) <IL 0x0003d, 0xffffffff>
1081 * at System.Runtime.Remoting.Channels.CrossAppDomainSink.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage) [0x00008] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Channels/CrossAppDomainChannel.cs:198
1082 * at (wrapper runtime-invoke) object.runtime_invoke_CrossAppDomainSink/ProcessMessageRes_object_object (object,intptr,intptr,intptr) <IL 0x0004c, 0xffffffff>
1084 if (!strcmp (ref->vtable->klass->name_space, "System") &&
1085 !strcmp (ref->vtable->klass->name, "Byte[]") &&
1086 !strcmp (o->vtable->klass->name_space, "System.IO") &&
1087 !strcmp (o->vtable->klass->name, "MemoryStream"))
1089 /* append_job() in threadpool.c */
1090 if (!strcmp (ref->vtable->klass->name_space, "System.Runtime.Remoting.Messaging") &&
1091 !strcmp (ref->vtable->klass->name, "AsyncResult") &&
1092 !strcmp (o->vtable->klass->name_space, "System") &&
1093 !strcmp (o->vtable->klass->name, "Object[]") &&
1094 mono_thread_pool_is_queue_array ((MonoArray*) o))
1100 check_reference_for_xdomain (gpointer *ptr, char *obj, MonoDomain *domain)
1102 MonoObject *o = (MonoObject*)(obj);
1103 MonoObject *ref = (MonoObject*)*(ptr);
1104 int offset = (char*)(ptr) - (char*)o;
1106 MonoClassField *field;
1109 if (!ref || ref->vtable->domain == domain)
1111 if (is_xdomain_ref_allowed (ptr, obj, domain))
1115 for (class = o->vtable->klass; class; class = class->parent) {
1118 for (i = 0; i < class->field.count; ++i) {
1119 if (class->fields[i].offset == offset) {
1120 field = &class->fields[i];
1128 if (ref->vtable->klass == mono_defaults.string_class)
1129 str = mono_string_to_utf8 ((MonoString*)ref);
1132 g_print ("xdomain reference in %p (%s.%s) at offset %d (%s) to %p (%s.%s) (%s) - pointed to by:\n",
1133 o, o->vtable->klass->name_space, o->vtable->klass->name,
1134 offset, field ? field->name : "",
1135 ref, ref->vtable->klass->name_space, ref->vtable->klass->name, str ? str : "");
1136 mono_gc_scan_for_specific_ref (o);
1142 #define HANDLE_PTR(ptr,obj) check_reference_for_xdomain ((ptr), (obj), domain)
1145 scan_object_for_xdomain_refs (char *start, mword size, void *data)
1147 MonoDomain *domain = ((MonoObject*)start)->vtable->domain;
1149 #include "sgen-scan-object.h"
1153 #define HANDLE_PTR(ptr,obj) do { \
1154 if ((MonoObject*)*(ptr) == key) { \
1155 g_print ("found ref to %p in object %p (%s) at offset %td\n", \
1156 key, (obj), safe_name ((obj)), ((char*)(ptr) - (char*)(obj))); \
1161 scan_object_for_specific_ref (char *start, MonoObject *key)
1163 #include "sgen-scan-object.h"
1167 mono_sgen_scan_area_with_callback (char *start, char *end, IterateObjectCallbackFunc callback, void *data)
1169 while (start < end) {
1171 if (!*(void**)start) {
1172 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
1176 size = ALIGN_UP (safe_object_get_size ((MonoObject*) start));
1178 callback (start, size, data);
1185 scan_object_for_specific_ref_callback (char *obj, size_t size, MonoObject *key)
1187 scan_object_for_specific_ref (obj, key);
1191 check_root_obj_specific_ref (RootRecord *root, MonoObject *key, MonoObject *obj)
1195 g_print ("found ref to %p in root record %p\n", key, root);
1198 static MonoObject *check_key = NULL;
1199 static RootRecord *check_root = NULL;
1202 check_root_obj_specific_ref_from_marker (void **obj)
1204 check_root_obj_specific_ref (check_root, check_key, *obj);
1208 scan_roots_for_specific_ref (MonoObject *key, int root_type)
1213 for (i = 0; i < roots_hash_size [root_type]; ++i) {
1214 for (root = roots_hash [root_type][i]; root; root = root->next) {
1215 void **start_root = (void**)root->start_root;
1216 mword desc = root->root_desc;
1220 switch (desc & ROOT_DESC_TYPE_MASK) {
1221 case ROOT_DESC_BITMAP:
1222 desc >>= ROOT_DESC_TYPE_SHIFT;
1225 check_root_obj_specific_ref (root, key, *start_root);
1230 case ROOT_DESC_COMPLEX: {
1231 gsize *bitmap_data = complex_descriptors + (desc >> ROOT_DESC_TYPE_SHIFT);
1232 int bwords = (*bitmap_data) - 1;
1233 void **start_run = start_root;
1235 while (bwords-- > 0) {
1236 gsize bmap = *bitmap_data++;
1237 void **objptr = start_run;
1240 check_root_obj_specific_ref (root, key, *objptr);
1244 start_run += GC_BITS_PER_WORD;
1248 case ROOT_DESC_USER: {
1249 MonoGCRootMarkFunc marker = user_descriptors [desc >> ROOT_DESC_TYPE_SHIFT];
1250 marker (start_root, check_root_obj_specific_ref_from_marker);
1253 case ROOT_DESC_RUN_LEN:
1254 g_assert_not_reached ();
1256 g_assert_not_reached ();
1265 mono_gc_scan_for_specific_ref (MonoObject *key)
1271 mono_sgen_scan_area_with_callback (nursery_section->data, nursery_section->end_data,
1272 (IterateObjectCallbackFunc)scan_object_for_specific_ref_callback, key);
1274 major_collector.iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)scan_object_for_specific_ref_callback, key);
1276 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1277 scan_object_for_specific_ref (bigobj->data, key);
1279 scan_roots_for_specific_ref (key, ROOT_TYPE_NORMAL);
1280 scan_roots_for_specific_ref (key, ROOT_TYPE_WBARRIER);
1282 for (i = 0; i < roots_hash_size [ROOT_TYPE_PINNED]; ++i) {
1283 for (root = roots_hash [ROOT_TYPE_PINNED][i]; root; root = root->next) {
1284 void **ptr = (void**)root->start_root;
1286 while (ptr < (void**)root->end_root) {
1287 check_root_obj_specific_ref (root, *ptr, key);
1295 clear_current_nursery_fragment (char *next)
1297 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION) {
1298 g_assert (next <= nursery_frag_real_end);
1299 DEBUG (4, fprintf (gc_debug_file, "Clear nursery frag %p-%p\n", next, nursery_frag_real_end));
1300 memset (next, 0, nursery_frag_real_end - next);
1304 /* Clear all remaining nursery fragments */
1306 clear_nursery_fragments (char *next)
1309 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION) {
1310 clear_current_nursery_fragment (next);
1311 for (frag = nursery_fragments; frag; frag = frag->next) {
1312 DEBUG (4, fprintf (gc_debug_file, "Clear nursery frag %p-%p\n", frag->fragment_start, frag->fragment_end));
1313 memset (frag->fragment_start, 0, frag->fragment_end - frag->fragment_start);
1319 need_remove_object_for_domain (char *start, MonoDomain *domain)
1321 if (mono_object_domain (start) == domain) {
1322 DEBUG (4, fprintf (gc_debug_file, "Need to cleanup object %p\n", start));
1323 binary_protocol_cleanup (start, (gpointer)LOAD_VTABLE (start), safe_object_get_size ((MonoObject*)start));
1330 process_object_for_domain_clearing (char *start, MonoDomain *domain)
1332 GCVTable *vt = (GCVTable*)LOAD_VTABLE (start);
1333 if (vt->klass == mono_defaults.internal_thread_class)
1334 g_assert (mono_object_domain (start) == mono_get_root_domain ());
1335 /* The object could be a proxy for an object in the domain
1337 if (mono_class_has_parent (vt->klass, mono_defaults.real_proxy_class)) {
1338 MonoObject *server = ((MonoRealProxy*)start)->unwrapped_server;
1340 /* The server could already have been zeroed out, so
1341 we need to check for that, too. */
1342 if (server && (!LOAD_VTABLE (server) || mono_object_domain (server) == domain)) {
1343 DEBUG (4, fprintf (gc_debug_file, "Cleaning up remote pointer in %p to object %p\n",
1345 ((MonoRealProxy*)start)->unwrapped_server = NULL;
1350 static MonoDomain *check_domain = NULL;
1353 check_obj_not_in_domain (void **o)
1355 g_assert (((MonoObject*)(*o))->vtable->domain != check_domain);
1359 scan_for_registered_roots_in_domain (MonoDomain *domain, int root_type)
1363 check_domain = domain;
1364 for (i = 0; i < roots_hash_size [root_type]; ++i) {
1365 for (root = roots_hash [root_type][i]; root; root = root->next) {
1366 void **start_root = (void**)root->start_root;
1367 mword desc = root->root_desc;
1369 /* The MonoDomain struct is allowed to hold
1370 references to objects in its own domain. */
1371 if (start_root == (void**)domain)
1374 switch (desc & ROOT_DESC_TYPE_MASK) {
1375 case ROOT_DESC_BITMAP:
1376 desc >>= ROOT_DESC_TYPE_SHIFT;
1378 if ((desc & 1) && *start_root)
1379 check_obj_not_in_domain (*start_root);
1384 case ROOT_DESC_COMPLEX: {
1385 gsize *bitmap_data = complex_descriptors + (desc >> ROOT_DESC_TYPE_SHIFT);
1386 int bwords = (*bitmap_data) - 1;
1387 void **start_run = start_root;
1389 while (bwords-- > 0) {
1390 gsize bmap = *bitmap_data++;
1391 void **objptr = start_run;
1393 if ((bmap & 1) && *objptr)
1394 check_obj_not_in_domain (*objptr);
1398 start_run += GC_BITS_PER_WORD;
1402 case ROOT_DESC_USER: {
1403 MonoGCRootMarkFunc marker = user_descriptors [desc >> ROOT_DESC_TYPE_SHIFT];
1404 marker (start_root, check_obj_not_in_domain);
1407 case ROOT_DESC_RUN_LEN:
1408 g_assert_not_reached ();
1410 g_assert_not_reached ();
1414 check_domain = NULL;
1418 check_for_xdomain_refs (void)
1422 mono_sgen_scan_area_with_callback (nursery_section->data, nursery_section->end_data,
1423 (IterateObjectCallbackFunc)scan_object_for_xdomain_refs, NULL);
1425 major_collector.iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)scan_object_for_xdomain_refs, NULL);
1427 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1428 scan_object_for_xdomain_refs (bigobj->data, bigobj->size, NULL);
1432 clear_domain_process_object (char *obj, MonoDomain *domain)
1436 process_object_for_domain_clearing (obj, domain);
1437 remove = need_remove_object_for_domain (obj, domain);
1439 if (remove && ((MonoObject*)obj)->synchronisation) {
1440 void **dislink = mono_monitor_get_object_monitor_weak_link ((MonoObject*)obj);
1442 mono_gc_register_disappearing_link (NULL, dislink, FALSE);
1449 clear_domain_process_minor_object_callback (char *obj, size_t size, MonoDomain *domain)
1451 if (clear_domain_process_object (obj, domain))
1452 memset (obj, 0, size);
1456 clear_domain_process_major_object_callback (char *obj, size_t size, MonoDomain *domain)
1458 clear_domain_process_object (obj, domain);
1462 clear_domain_free_major_non_pinned_object_callback (char *obj, size_t size, MonoDomain *domain)
1464 if (need_remove_object_for_domain (obj, domain))
1465 major_collector.free_non_pinned_object (obj, size);
1469 clear_domain_free_major_pinned_object_callback (char *obj, size_t size, MonoDomain *domain)
1471 if (need_remove_object_for_domain (obj, domain))
1472 major_collector.free_pinned_object (obj, size);
1476 * When appdomains are unloaded we can easily remove objects that have finalizers,
1477 * but all the others could still be present in random places on the heap.
1478 * We need a sweep to get rid of them even though it's going to be costly
1480 * The reason we need to remove them is because we access the vtable and class
1481 * structures to know the object size and the reference bitmap: once the domain is
1482 * unloaded the point to random memory.
1485 mono_gc_clear_domain (MonoDomain * domain)
1487 LOSObject *bigobj, *prev;
1492 clear_nursery_fragments (nursery_next);
1494 if (xdomain_checks && domain != mono_get_root_domain ()) {
1495 scan_for_registered_roots_in_domain (domain, ROOT_TYPE_NORMAL);
1496 scan_for_registered_roots_in_domain (domain, ROOT_TYPE_WBARRIER);
1497 check_for_xdomain_refs ();
1500 mono_sgen_scan_area_with_callback (nursery_section->data, nursery_section->end_data,
1501 (IterateObjectCallbackFunc)clear_domain_process_minor_object_callback, domain);
1503 /*Ephemerons and dislinks must be processed before LOS since they might end up pointing
1504 to memory returned to the OS.*/
1505 null_ephemerons_for_domain (domain);
1507 for (i = GENERATION_NURSERY; i < GENERATION_MAX; ++i)
1508 null_links_for_domain (domain, i);
1510 /* We need two passes over major and large objects because
1511 freeing such objects might give their memory back to the OS
1512 (in the case of large objects) or obliterate its vtable
1513 (pinned objects with major-copying or pinned and non-pinned
1514 objects with major-mark&sweep), but we might need to
1515 dereference a pointer from an object to another object if
1516 the first object is a proxy. */
1517 major_collector.iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)clear_domain_process_major_object_callback, domain);
1518 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1519 clear_domain_process_object (bigobj->data, domain);
1522 for (bigobj = los_object_list; bigobj;) {
1523 if (need_remove_object_for_domain (bigobj->data, domain)) {
1524 LOSObject *to_free = bigobj;
1526 prev->next = bigobj->next;
1528 los_object_list = bigobj->next;
1529 bigobj = bigobj->next;
1530 DEBUG (4, fprintf (gc_debug_file, "Freeing large object %p\n",
1532 free_large_object (to_free);
1536 bigobj = bigobj->next;
1538 major_collector.iterate_objects (TRUE, FALSE, (IterateObjectCallbackFunc)clear_domain_free_major_non_pinned_object_callback, domain);
1539 major_collector.iterate_objects (FALSE, TRUE, (IterateObjectCallbackFunc)clear_domain_free_major_pinned_object_callback, domain);
1545 global_remset_cache_clear (void)
1547 memset (global_remset_cache, 0, sizeof (global_remset_cache));
1551 * Tries to check if a given remset location was already added to the global remset.
1554 * A 2 entry, LRU cache of recently saw location remsets.
1556 * It's hand-coded instead of done using loops to reduce the number of memory references on cache hit.
1558 * Returns TRUE is the element was added..
1561 global_remset_location_was_not_added (gpointer ptr)
1564 gpointer first = global_remset_cache [0], second;
1566 HEAVY_STAT (++stat_global_remsets_discarded);
1570 second = global_remset_cache [1];
1572 if (second == ptr) {
1573 /*Move the second to the front*/
1574 global_remset_cache [0] = second;
1575 global_remset_cache [1] = first;
1577 HEAVY_STAT (++stat_global_remsets_discarded);
1581 global_remset_cache [0] = second;
1582 global_remset_cache [1] = ptr;
1587 * mono_sgen_add_to_global_remset:
1589 * The global remset contains locations which point into newspace after
1590 * a minor collection. This can happen if the objects they point to are pinned.
1592 * LOCKING: If called from a parallel collector, the global remset
1593 * lock must be held. For serial collectors that is not necessary.
1596 mono_sgen_add_to_global_remset (gpointer ptr)
1601 if (use_cardtable) {
1602 sgen_card_table_mark_address ((mword)ptr);
1606 g_assert (!ptr_in_nursery (ptr) && ptr_in_nursery (*(gpointer*)ptr));
1608 lock = (current_collection_generation == GENERATION_OLD && major_collector.is_parallel);
1612 if (!global_remset_location_was_not_added (ptr))
1615 DEBUG (8, fprintf (gc_debug_file, "Adding global remset for %p\n", ptr));
1616 binary_protocol_global_remset (ptr, *(gpointer*)ptr, (gpointer)LOAD_VTABLE (*(gpointer*)ptr));
1618 HEAVY_STAT (++stat_global_remsets_added);
1621 * FIXME: If an object remains pinned, we need to add it at every minor collection.
1622 * To avoid uncontrolled growth of the global remset, only add each pointer once.
1624 if (global_remset->store_next + 3 < global_remset->end_set) {
1625 *(global_remset->store_next++) = (mword)ptr;
1628 rs = alloc_remset (global_remset->end_set - global_remset->data, NULL);
1629 rs->next = global_remset;
1631 *(global_remset->store_next++) = (mword)ptr;
1634 int global_rs_size = 0;
1636 for (rs = global_remset; rs; rs = rs->next) {
1637 global_rs_size += rs->store_next - rs->data;
1639 DEBUG (4, fprintf (gc_debug_file, "Global remset now has size %d\n", global_rs_size));
1644 UNLOCK_GLOBAL_REMSET;
1650 * Scan objects in the gray stack until the stack is empty. This should be called
1651 * frequently after each object is copied, to achieve better locality and cache
1655 drain_gray_stack (GrayQueue *queue)
1659 if (current_collection_generation == GENERATION_NURSERY) {
1661 GRAY_OBJECT_DEQUEUE (queue, obj);
1664 DEBUG (9, fprintf (gc_debug_file, "Precise gray object scan %p (%s)\n", obj, safe_name (obj)));
1665 major_collector.minor_scan_object (obj, queue);
1668 if (major_collector.is_parallel && queue == &workers_distribute_gray_queue)
1672 GRAY_OBJECT_DEQUEUE (queue, obj);
1675 DEBUG (9, fprintf (gc_debug_file, "Precise gray object scan %p (%s)\n", obj, safe_name (obj)));
1676 major_collector.major_scan_object (obj, queue);
1682 * Addresses from start to end are already sorted. This function finds
1683 * the object header for each address and pins the object. The
1684 * addresses must be inside the passed section. The (start of the)
1685 * address array is overwritten with the addresses of the actually
1686 * pinned objects. Return the number of pinned objects.
1689 pin_objects_from_addresses (GCMemSection *section, void **start, void **end, void *start_nursery, void *end_nursery, GrayQueue *queue)
1694 void *last_obj = NULL;
1695 size_t last_obj_size = 0;
1698 void **definitely_pinned = start;
1702 * The code below starts the search from an entry in scan_starts, which might point into a nursery
1703 * fragment containing random data. Clearing the nursery fragments takes a lot of time, and searching
1704 * though them too, so lay arrays at each location inside a fragment where a search can start:
1705 * - scan_locations[i]
1707 * - the start of each fragment (the last_obj + last_obj case)
1708 * The third encompasses the first two, since scan_locations [i] can't point inside a nursery fragment.
1710 for (frag = nursery_fragments; frag; frag = frag->next) {
1713 g_assert (frag->fragment_end - frag->fragment_start >= sizeof (MonoArray));
1714 o = (MonoArray*)frag->fragment_start;
1715 memset (o, 0, sizeof (MonoArray));
1716 g_assert (array_fill_vtable);
1717 o->obj.vtable = array_fill_vtable;
1718 /* Mark this as not a real object */
1719 o->obj.synchronisation = GINT_TO_POINTER (-1);
1720 o->max_length = (frag->fragment_end - frag->fragment_start) - sizeof (MonoArray);
1721 g_assert (frag->fragment_start + safe_object_get_size ((MonoObject*)o) == frag->fragment_end);
1724 while (start < end) {
1726 /* the range check should be reduntant */
1727 if (addr != last && addr >= start_nursery && addr < end_nursery) {
1728 DEBUG (5, fprintf (gc_debug_file, "Considering pinning addr %p\n", addr));
1729 /* multiple pointers to the same object */
1730 if (addr >= last_obj && (char*)addr < (char*)last_obj + last_obj_size) {
1734 idx = ((char*)addr - (char*)section->data) / SCAN_START_SIZE;
1735 g_assert (idx < section->num_scan_start);
1736 search_start = (void*)section->scan_starts [idx];
1737 if (!search_start || search_start > addr) {
1740 search_start = section->scan_starts [idx];
1741 if (search_start && search_start <= addr)
1744 if (!search_start || search_start > addr)
1745 search_start = start_nursery;
1747 if (search_start < last_obj)
1748 search_start = (char*)last_obj + last_obj_size;
1749 /* now addr should be in an object a short distance from search_start
1750 * Note that search_start must point to zeroed mem or point to an object.
1754 if (!*(void**)search_start) {
1755 /* Consistency check */
1757 for (frag = nursery_fragments; frag; frag = frag->next) {
1758 if (search_start >= frag->fragment_start && search_start < frag->fragment_end)
1759 g_assert_not_reached ();
1763 search_start = (void*)ALIGN_UP ((mword)search_start + sizeof (gpointer));
1766 last_obj = search_start;
1767 last_obj_size = ALIGN_UP (safe_object_get_size ((MonoObject*)search_start));
1769 if (((MonoObject*)last_obj)->synchronisation == GINT_TO_POINTER (-1)) {
1770 /* Marks the beginning of a nursery fragment, skip */
1772 DEBUG (8, fprintf (gc_debug_file, "Pinned try match %p (%s), size %zd\n", last_obj, safe_name (last_obj), last_obj_size));
1773 if (addr >= search_start && (char*)addr < (char*)last_obj + last_obj_size) {
1774 DEBUG (4, fprintf (gc_debug_file, "Pinned object %p, vtable %p (%s), count %d\n", search_start, *(void**)search_start, safe_name (search_start), count));
1775 binary_protocol_pin (search_start, (gpointer)LOAD_VTABLE (search_start), safe_object_get_size (search_start));
1776 pin_object (search_start);
1777 GRAY_OBJECT_ENQUEUE (queue, search_start);
1779 mono_sgen_pin_stats_register_object (search_start, last_obj_size);
1780 definitely_pinned [count] = search_start;
1785 /* skip to the next object */
1786 search_start = (void*)((char*)search_start + last_obj_size);
1787 } while (search_start <= addr);
1788 /* we either pinned the correct object or we ignored the addr because
1789 * it points to unused zeroed memory.
1795 //printf ("effective pinned: %d (at the end: %d)\n", count, (char*)end_nursery - (char*)last);
1800 mono_sgen_pin_objects_in_section (GCMemSection *section, GrayQueue *queue)
1802 int num_entries = section->pin_queue_num_entries;
1804 void **start = section->pin_queue_start;
1806 reduced_to = pin_objects_from_addresses (section, start, start + num_entries,
1807 section->data, section->next_data, queue);
1808 section->pin_queue_num_entries = reduced_to;
1810 section->pin_queue_start = NULL;
1816 mono_sgen_pin_object (void *object, GrayQueue *queue)
1818 if (major_collector.is_parallel) {
1820 /*object arrives pinned*/
1821 pin_stage_ptr (object);
1825 SGEN_PIN_OBJECT (object);
1826 pin_stage_ptr (object);
1829 GRAY_OBJECT_ENQUEUE (queue, object);
1832 /* Sort the addresses in array in increasing order.
1833 * Done using a by-the book heap sort. Which has decent and stable performance, is pretty cache efficient.
1836 sort_addresses (void **array, int size)
1841 for (i = 1; i < size; ++i) {
1844 int parent = (child - 1) / 2;
1846 if (array [parent] >= array [child])
1849 tmp = array [parent];
1850 array [parent] = array [child];
1851 array [child] = tmp;
1857 for (i = size - 1; i > 0; --i) {
1860 array [i] = array [0];
1866 while (root * 2 + 1 <= end) {
1867 int child = root * 2 + 1;
1869 if (child < end && array [child] < array [child + 1])
1871 if (array [root] >= array [child])
1875 array [root] = array [child];
1876 array [child] = tmp;
1883 static G_GNUC_UNUSED void
1884 print_nursery_gaps (void* start_nursery, void *end_nursery)
1887 gpointer first = start_nursery;
1889 for (i = 0; i < next_pin_slot; ++i) {
1890 next = pin_queue [i];
1891 fprintf (gc_debug_file, "Nursery range: %p-%p, size: %td\n", first, next, (char*)next-(char*)first);
1895 fprintf (gc_debug_file, "Nursery range: %p-%p, size: %td\n", first, next, (char*)next-(char*)first);
1898 /* reduce the info in the pin queue, removing duplicate pointers and sorting them */
1900 optimize_pin_queue (int start_slot)
1902 void **start, **cur, **end;
1903 /* sort and uniq pin_queue: we just sort and we let the rest discard multiple values */
1904 /* it may be better to keep ranges of pinned memory instead of individually pinning objects */
1905 DEBUG (5, fprintf (gc_debug_file, "Sorting pin queue, size: %d\n", next_pin_slot));
1906 if ((next_pin_slot - start_slot) > 1)
1907 sort_addresses (pin_queue + start_slot, next_pin_slot - start_slot);
1908 start = cur = pin_queue + start_slot;
1909 end = pin_queue + next_pin_slot;
1912 while (*start == *cur && cur < end)
1916 next_pin_slot = start - pin_queue;
1917 DEBUG (5, fprintf (gc_debug_file, "Pin queue reduced to size: %d\n", next_pin_slot));
1918 //DEBUG (6, print_nursery_gaps (start_nursery, end_nursery));
1923 * Scan the memory between start and end and queue values which could be pointers
1924 * to the area between start_nursery and end_nursery for later consideration.
1925 * Typically used for thread stacks.
1928 conservatively_pin_objects_from (void **start, void **end, void *start_nursery, void *end_nursery, int pin_type)
1931 while (start < end) {
1932 if (*start >= start_nursery && *start < end_nursery) {
1934 * *start can point to the middle of an object
1935 * note: should we handle pointing at the end of an object?
1936 * pinning in C# code disallows pointing at the end of an object
1937 * but there is some small chance that an optimizing C compiler
1938 * may keep the only reference to an object by pointing
1939 * at the end of it. We ignore this small chance for now.
1940 * Pointers to the end of an object are indistinguishable
1941 * from pointers to the start of the next object in memory
1942 * so if we allow that we'd need to pin two objects...
1943 * We queue the pointer in an array, the
1944 * array will then be sorted and uniqued. This way
1945 * we can coalesce several pinning pointers and it should
1946 * be faster since we'd do a memory scan with increasing
1947 * addresses. Note: we can align the address to the allocation
1948 * alignment, so the unique process is more effective.
1950 mword addr = (mword)*start;
1951 addr &= ~(ALLOC_ALIGN - 1);
1952 if (addr >= (mword)start_nursery && addr < (mword)end_nursery)
1953 pin_stage_ptr ((void*)addr);
1955 pin_stats_register_address ((char*)addr, pin_type);
1956 DEBUG (6, if (count) fprintf (gc_debug_file, "Pinning address %p\n", (void*)addr));
1961 DEBUG (7, if (count) fprintf (gc_debug_file, "found %d potential pinned heap pointers\n", count));
1965 * Debugging function: find in the conservative roots where @obj is being pinned.
1967 static G_GNUC_UNUSED void
1968 find_pinning_reference (char *obj, size_t size)
1972 char *endobj = obj + size;
1973 for (i = 0; i < roots_hash_size [0]; ++i) {
1974 for (root = roots_hash [0][i]; root; root = root->next) {
1975 /* if desc is non-null it has precise info */
1976 if (!root->root_desc) {
1977 char ** start = (char**)root->start_root;
1978 while (start < (char**)root->end_root) {
1979 if (*start >= obj && *start < endobj) {
1980 DEBUG (0, fprintf (gc_debug_file, "Object %p referenced in pinned roots %p-%p (at %p in record %p)\n", obj, root->start_root, root->end_root, start, root));
1987 find_pinning_ref_from_thread (obj, size);
1991 * The first thing we do in a collection is to identify pinned objects.
1992 * This function considers all the areas of memory that need to be
1993 * conservatively scanned.
1996 pin_from_roots (void *start_nursery, void *end_nursery)
2000 DEBUG (2, fprintf (gc_debug_file, "Scanning pinned roots (%d bytes, %d/%d entries)\n", (int)roots_size, num_roots_entries [ROOT_TYPE_NORMAL], num_roots_entries [ROOT_TYPE_PINNED]));
2001 /* objects pinned from the API are inside these roots */
2002 for (i = 0; i < roots_hash_size [ROOT_TYPE_PINNED]; ++i) {
2003 for (root = roots_hash [ROOT_TYPE_PINNED][i]; root; root = root->next) {
2004 DEBUG (6, fprintf (gc_debug_file, "Pinned roots %p-%p\n", root->start_root, root->end_root));
2005 conservatively_pin_objects_from ((void**)root->start_root, (void**)root->end_root, start_nursery, end_nursery, PIN_TYPE_OTHER);
2008 /* now deal with the thread stacks
2009 * in the future we should be able to conservatively scan only:
2010 * *) the cpu registers
2011 * *) the unmanaged stack frames
2012 * *) the _last_ managed stack frame
2013 * *) pointers slots in managed frames
2015 scan_thread_data (start_nursery, end_nursery, FALSE);
2017 evacuate_pin_staging_area ();
2020 static CopyOrMarkObjectFunc user_copy_or_mark_func;
2021 static GrayQueue *user_copy_or_mark_queue;
2024 single_arg_user_copy_or_mark (void **obj)
2026 user_copy_or_mark_func (obj, user_copy_or_mark_queue);
2030 * The memory area from start_root to end_root contains pointers to objects.
2031 * Their position is precisely described by @desc (this means that the pointer
2032 * can be either NULL or the pointer to the start of an object).
2033 * This functions copies them to to_space updates them.
2035 * This function is not thread-safe!
2038 precisely_scan_objects_from (CopyOrMarkObjectFunc copy_func, void** start_root, void** end_root, char* n_start, char *n_end, mword desc, GrayQueue *queue)
2040 switch (desc & ROOT_DESC_TYPE_MASK) {
2041 case ROOT_DESC_BITMAP:
2042 desc >>= ROOT_DESC_TYPE_SHIFT;
2044 if ((desc & 1) && *start_root) {
2045 copy_func (start_root, queue);
2046 DEBUG (9, fprintf (gc_debug_file, "Overwrote root at %p with %p\n", start_root, *start_root));
2047 drain_gray_stack (queue);
2053 case ROOT_DESC_COMPLEX: {
2054 gsize *bitmap_data = complex_descriptors + (desc >> ROOT_DESC_TYPE_SHIFT);
2055 int bwords = (*bitmap_data) - 1;
2056 void **start_run = start_root;
2058 while (bwords-- > 0) {
2059 gsize bmap = *bitmap_data++;
2060 void **objptr = start_run;
2062 if ((bmap & 1) && *objptr) {
2063 copy_func (objptr, queue);
2064 DEBUG (9, fprintf (gc_debug_file, "Overwrote root at %p with %p\n", objptr, *objptr));
2065 drain_gray_stack (queue);
2070 start_run += GC_BITS_PER_WORD;
2074 case ROOT_DESC_USER: {
2075 MonoGCRootMarkFunc marker = user_descriptors [desc >> ROOT_DESC_TYPE_SHIFT];
2076 user_copy_or_mark_func = copy_func;
2077 user_copy_or_mark_queue = queue;
2078 marker (start_root, single_arg_user_copy_or_mark);
2079 user_copy_or_mark_func = NULL;
2080 user_copy_or_mark_queue = NULL;
2083 case ROOT_DESC_RUN_LEN:
2084 g_assert_not_reached ();
2086 g_assert_not_reached ();
2091 mono_sgen_update_heap_boundaries (mword low, mword high)
2096 old = lowest_heap_address;
2099 } while (SGEN_CAS_PTR ((gpointer*)&lowest_heap_address, (gpointer)low, (gpointer)old) != (gpointer)old);
2102 old = highest_heap_address;
2105 } while (SGEN_CAS_PTR ((gpointer*)&highest_heap_address, (gpointer)high, (gpointer)old) != (gpointer)old);
2109 alloc_fragment (void)
2111 Fragment *frag = fragment_freelist;
2113 fragment_freelist = frag->next;
2117 frag = mono_sgen_alloc_internal (INTERNAL_MEM_FRAGMENT);
2122 /* size must be a power of 2 */
2124 mono_sgen_alloc_os_memory_aligned (mword size, mword alignment, gboolean activate)
2126 /* Allocate twice the memory to be able to put the block on an aligned address */
2127 char *mem = mono_sgen_alloc_os_memory (size + alignment, activate);
2132 aligned = (char*)((mword)(mem + (alignment - 1)) & ~(alignment - 1));
2133 g_assert (aligned >= mem && aligned + size <= mem + size + alignment && !((mword)aligned & (alignment - 1)));
2136 mono_sgen_free_os_memory (mem, aligned - mem);
2137 if (aligned + size < mem + size + alignment)
2138 mono_sgen_free_os_memory (aligned + size, (mem + size + alignment) - (aligned + size));
2144 * Allocate and setup the data structures needed to be able to allocate objects
2145 * in the nursery. The nursery is stored in nursery_section.
2148 alloc_nursery (void)
2150 GCMemSection *section;
2156 if (nursery_section)
2158 DEBUG (2, fprintf (gc_debug_file, "Allocating nursery size: %lu\n", (unsigned long)nursery_size));
2159 /* later we will alloc a larger area for the nursery but only activate
2160 * what we need. The rest will be used as expansion if we have too many pinned
2161 * objects in the existing nursery.
2163 /* FIXME: handle OOM */
2164 section = mono_sgen_alloc_internal (INTERNAL_MEM_SECTION);
2166 g_assert (nursery_size == DEFAULT_NURSERY_SIZE);
2167 alloc_size = nursery_size;
2168 #ifdef SGEN_ALIGN_NURSERY
2169 data = major_collector.alloc_heap (alloc_size, alloc_size, DEFAULT_NURSERY_BITS);
2171 data = major_collector.alloc_heap (alloc_size, 0, DEFAULT_NURSERY_BITS);
2173 nursery_start = data;
2174 nursery_real_end = nursery_start + nursery_size;
2175 mono_sgen_update_heap_boundaries ((mword)nursery_start, (mword)nursery_real_end);
2176 nursery_next = nursery_start;
2177 DEBUG (4, fprintf (gc_debug_file, "Expanding nursery size (%p-%p): %lu, total: %lu\n", data, data + alloc_size, (unsigned long)nursery_size, (unsigned long)total_alloc));
2178 section->data = section->next_data = data;
2179 section->size = alloc_size;
2180 section->end_data = nursery_real_end;
2181 scan_starts = (alloc_size + SCAN_START_SIZE - 1) / SCAN_START_SIZE;
2182 section->scan_starts = mono_sgen_alloc_internal_dynamic (sizeof (char*) * scan_starts, INTERNAL_MEM_SCAN_STARTS);
2183 section->num_scan_start = scan_starts;
2184 section->block.role = MEMORY_ROLE_GEN0;
2185 section->block.next = NULL;
2187 nursery_section = section;
2189 /* Setup the single first large fragment */
2190 frag = alloc_fragment ();
2191 frag->fragment_start = nursery_start;
2192 frag->fragment_limit = nursery_start;
2193 frag->fragment_end = nursery_real_end;
2194 nursery_frag_real_end = nursery_real_end;
2195 /* FIXME: frag here is lost */
2199 mono_gc_get_nursery (int *shift_bits, size_t *size)
2201 *size = nursery_size;
2202 #ifdef SGEN_ALIGN_NURSERY
2203 *shift_bits = DEFAULT_NURSERY_BITS;
2207 return nursery_start;
2211 scan_finalizer_entries (CopyOrMarkObjectFunc copy_func, FinalizeEntry *list, GrayQueue *queue)
2215 for (fin = list; fin; fin = fin->next) {
2218 DEBUG (5, fprintf (gc_debug_file, "Scan of fin ready object: %p (%s)\n", fin->object, safe_name (fin->object)));
2219 copy_func (&fin->object, queue);
2223 static mword fragment_total = 0;
2225 * We found a fragment of free memory in the nursery: memzero it and if
2226 * it is big enough, add it to the list of fragments that can be used for
2230 add_nursery_frag (size_t frag_size, char* frag_start, char* frag_end)
2233 DEBUG (4, fprintf (gc_debug_file, "Found empty fragment: %p-%p, size: %zd\n", frag_start, frag_end, frag_size));
2234 binary_protocol_empty (frag_start, frag_size);
2235 /* Not worth dealing with smaller fragments: need to tune */
2236 if (frag_size >= FRAGMENT_MIN_SIZE) {
2237 /* memsetting just the first chunk start is bound to provide better cache locality */
2238 if (nursery_clear_policy == CLEAR_AT_GC)
2239 memset (frag_start, 0, frag_size);
2241 fragment = alloc_fragment ();
2242 fragment->fragment_start = frag_start;
2243 fragment->fragment_limit = frag_start;
2244 fragment->fragment_end = frag_end;
2245 fragment->next = nursery_fragments;
2246 nursery_fragments = fragment;
2247 fragment_total += frag_size;
2249 /* Clear unused fragments, pinning depends on this */
2250 /*TODO place an int[] here instead of the memset if size justify it*/
2251 memset (frag_start, 0, frag_size);
2256 generation_name (int generation)
2258 switch (generation) {
2259 case GENERATION_NURSERY: return "nursery";
2260 case GENERATION_OLD: return "old";
2261 default: g_assert_not_reached ();
2265 static DisappearingLinkHashTable*
2266 get_dislink_hash_table (int generation)
2268 switch (generation) {
2269 case GENERATION_NURSERY: return &minor_disappearing_link_hash;
2270 case GENERATION_OLD: return &major_disappearing_link_hash;
2271 default: g_assert_not_reached ();
2275 static FinalizeEntryHashTable*
2276 get_finalize_entry_hash_table (int generation)
2278 switch (generation) {
2279 case GENERATION_NURSERY: return &minor_finalizable_hash;
2280 case GENERATION_OLD: return &major_finalizable_hash;
2281 default: g_assert_not_reached ();
2286 finish_gray_stack (char *start_addr, char *end_addr, int generation, GrayQueue *queue)
2291 int ephemeron_rounds = 0;
2292 CopyOrMarkObjectFunc copy_func = current_collection_generation == GENERATION_NURSERY ? major_collector.copy_object : major_collector.copy_or_mark_object;
2295 * We copied all the reachable objects. Now it's the time to copy
2296 * the objects that were not referenced by the roots, but by the copied objects.
2297 * we built a stack of objects pointed to by gray_start: they are
2298 * additional roots and we may add more items as we go.
2299 * We loop until gray_start == gray_objects which means no more objects have
2300 * been added. Note this is iterative: no recursion is involved.
2301 * We need to walk the LO list as well in search of marked big objects
2302 * (use a flag since this is needed only on major collections). We need to loop
2303 * here as well, so keep a counter of marked LO (increasing it in copy_object).
2304 * To achieve better cache locality and cache usage, we drain the gray stack
2305 * frequently, after each object is copied, and just finish the work here.
2307 drain_gray_stack (queue);
2309 DEBUG (2, fprintf (gc_debug_file, "%s generation done\n", generation_name (generation)));
2310 /* walk the finalization queue and move also the objects that need to be
2311 * finalized: use the finalized objects as new roots so the objects they depend
2312 * on are also not reclaimed. As with the roots above, only objects in the nursery
2313 * are marked/copied.
2314 * We need a loop here, since objects ready for finalizers may reference other objects
2315 * that are fin-ready. Speedup with a flag?
2319 * Walk the ephemeron tables marking all values with reachable keys. This must be completely done
2320 * before processing finalizable objects to avoid finalizing reachable values.
2322 * It must be done inside the finalizaters loop since objects must not be removed from CWT tables
2323 * while they are been finalized.
2325 int done_with_ephemerons = 0;
2327 done_with_ephemerons = mark_ephemerons_in_range (copy_func, start_addr, end_addr, queue);
2328 drain_gray_stack (queue);
2330 } while (!done_with_ephemerons);
2332 fin_ready = num_ready_finalizers;
2333 finalize_in_range (copy_func, start_addr, end_addr, generation, queue);
2334 if (generation == GENERATION_OLD)
2335 finalize_in_range (copy_func, nursery_start, nursery_real_end, GENERATION_NURSERY, queue);
2337 /* drain the new stack that might have been created */
2338 DEBUG (6, fprintf (gc_debug_file, "Precise scan of gray area post fin\n"));
2339 drain_gray_stack (queue);
2340 } while (fin_ready != num_ready_finalizers);
2343 * Clear ephemeron pairs with unreachable keys.
2344 * We pass the copy func so we can figure out if an array was promoted or not.
2346 clear_unreachable_ephemerons (copy_func, start_addr, end_addr, queue);
2349 DEBUG (2, fprintf (gc_debug_file, "Finalize queue handling scan for %s generation: %d usecs %d ephemeron roundss\n", generation_name (generation), TV_ELAPSED (atv, btv), ephemeron_rounds));
2352 * handle disappearing links
2353 * Note we do this after checking the finalization queue because if an object
2354 * survives (at least long enough to be finalized) we don't clear the link.
2355 * This also deals with a possible issue with the monitor reclamation: with the Boehm
2356 * GC a finalized object my lose the monitor because it is cleared before the finalizer is
2359 g_assert (gray_object_queue_is_empty (queue));
2361 null_link_in_range (copy_func, start_addr, end_addr, generation, queue);
2362 if (generation == GENERATION_OLD)
2363 null_link_in_range (copy_func, start_addr, end_addr, GENERATION_NURSERY, queue);
2364 if (gray_object_queue_is_empty (queue))
2366 drain_gray_stack (queue);
2369 g_assert (gray_object_queue_is_empty (queue));
2373 mono_sgen_check_section_scan_starts (GCMemSection *section)
2376 for (i = 0; i < section->num_scan_start; ++i) {
2377 if (section->scan_starts [i]) {
2378 guint size = safe_object_get_size ((MonoObject*) section->scan_starts [i]);
2379 g_assert (size >= sizeof (MonoObject) && size <= MAX_SMALL_OBJ_SIZE);
2385 check_scan_starts (void)
2387 if (!do_scan_starts_check)
2389 mono_sgen_check_section_scan_starts (nursery_section);
2390 major_collector.check_scan_starts ();
2393 static int last_num_pinned = 0;
2396 build_nursery_fragments (void **start, int num_entries)
2398 char *frag_start, *frag_end;
2402 while (nursery_fragments) {
2403 Fragment *next = nursery_fragments->next;
2404 nursery_fragments->next = fragment_freelist;
2405 fragment_freelist = nursery_fragments;
2406 nursery_fragments = next;
2408 frag_start = nursery_start;
2410 /* clear scan starts */
2411 memset (nursery_section->scan_starts, 0, nursery_section->num_scan_start * sizeof (gpointer));
2412 for (i = 0; i < num_entries; ++i) {
2413 frag_end = start [i];
2414 /* remove the pin bit from pinned objects */
2415 unpin_object (frag_end);
2416 nursery_section->scan_starts [((char*)frag_end - (char*)nursery_section->data)/SCAN_START_SIZE] = frag_end;
2417 frag_size = frag_end - frag_start;
2419 add_nursery_frag (frag_size, frag_start, frag_end);
2420 frag_size = ALIGN_UP (safe_object_get_size ((MonoObject*)start [i]));
2421 frag_start = (char*)start [i] + frag_size;
2423 nursery_last_pinned_end = frag_start;
2424 frag_end = nursery_real_end;
2425 frag_size = frag_end - frag_start;
2427 add_nursery_frag (frag_size, frag_start, frag_end);
2428 if (!nursery_fragments) {
2429 DEBUG (1, fprintf (gc_debug_file, "Nursery fully pinned (%d)\n", num_entries));
2430 for (i = 0; i < num_entries; ++i) {
2431 DEBUG (3, fprintf (gc_debug_file, "Bastard pinning obj %p (%s), size: %d\n", start [i], safe_name (start [i]), safe_object_get_size (start [i])));
2436 nursery_next = nursery_frag_real_end = NULL;
2438 /* Clear TLABs for all threads */
2443 scan_from_registered_roots (CopyOrMarkObjectFunc copy_func, char *addr_start, char *addr_end, int root_type, GrayQueue *queue)
2447 for (i = 0; i < roots_hash_size [root_type]; ++i) {
2448 for (root = roots_hash [root_type][i]; root; root = root->next) {
2449 DEBUG (6, fprintf (gc_debug_file, "Precise root scan %p-%p (desc: %p)\n", root->start_root, root->end_root, (void*)root->root_desc));
2450 precisely_scan_objects_from (copy_func, (void**)root->start_root, (void**)root->end_root, addr_start, addr_end, root->root_desc, queue);
2456 mono_sgen_dump_occupied (char *start, char *end, char *section_start)
2458 fprintf (heap_dump_file, "<occupied offset=\"%td\" size=\"%td\"/>\n", start - section_start, end - start);
2462 mono_sgen_dump_section (GCMemSection *section, const char *type)
2464 char *start = section->data;
2465 char *end = section->data + section->size;
2466 char *occ_start = NULL;
2468 char *old_start = NULL; /* just for debugging */
2470 fprintf (heap_dump_file, "<section type=\"%s\" size=\"%lu\">\n", type, (unsigned long)section->size);
2472 while (start < end) {
2476 if (!*(void**)start) {
2478 mono_sgen_dump_occupied (occ_start, start, section->data);
2481 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
2484 g_assert (start < section->next_data);
2489 vt = (GCVTable*)LOAD_VTABLE (start);
2492 size = ALIGN_UP (safe_object_get_size ((MonoObject*) start));
2495 fprintf (heap_dump_file, "<object offset=\"%d\" class=\"%s.%s\" size=\"%d\"/>\n",
2496 start - section->data,
2497 vt->klass->name_space, vt->klass->name,
2505 mono_sgen_dump_occupied (occ_start, start, section->data);
2507 fprintf (heap_dump_file, "</section>\n");
2511 dump_object (MonoObject *obj, gboolean dump_location)
2513 static char class_name [1024];
2515 MonoClass *class = mono_object_class (obj);
2519 * Python's XML parser is too stupid to parse angle brackets
2520 * in strings, so we just ignore them;
2523 while (class->name [i] && j < sizeof (class_name) - 1) {
2524 if (!strchr ("<>\"", class->name [i]))
2525 class_name [j++] = class->name [i];
2528 g_assert (j < sizeof (class_name));
2531 fprintf (heap_dump_file, "<object class=\"%s.%s\" size=\"%d\"",
2532 class->name_space, class_name,
2533 safe_object_get_size (obj));
2534 if (dump_location) {
2535 const char *location;
2536 if (ptr_in_nursery (obj))
2537 location = "nursery";
2538 else if (safe_object_get_size (obj) <= MAX_SMALL_OBJ_SIZE)
2542 fprintf (heap_dump_file, " location=\"%s\"", location);
2544 fprintf (heap_dump_file, "/>\n");
2548 dump_heap (const char *type, int num, const char *reason)
2553 fprintf (heap_dump_file, "<collection type=\"%s\" num=\"%d\"", type, num);
2555 fprintf (heap_dump_file, " reason=\"%s\"", reason);
2556 fprintf (heap_dump_file, ">\n");
2557 fprintf (heap_dump_file, "<other-mem-usage type=\"mempools\" size=\"%ld\"/>\n", mono_mempool_get_bytes_allocated ());
2558 mono_sgen_dump_internal_mem_usage (heap_dump_file);
2559 fprintf (heap_dump_file, "<pinned type=\"stack\" bytes=\"%zu\"/>\n", pinned_byte_counts [PIN_TYPE_STACK]);
2560 /* fprintf (heap_dump_file, "<pinned type=\"static-data\" bytes=\"%d\"/>\n", pinned_byte_counts [PIN_TYPE_STATIC_DATA]); */
2561 fprintf (heap_dump_file, "<pinned type=\"other\" bytes=\"%zu\"/>\n", pinned_byte_counts [PIN_TYPE_OTHER]);
2563 fprintf (heap_dump_file, "<pinned-objects>\n");
2564 for (list = pinned_objects; list; list = list->next)
2565 dump_object (list->obj, TRUE);
2566 fprintf (heap_dump_file, "</pinned-objects>\n");
2568 mono_sgen_dump_section (nursery_section, "nursery");
2570 major_collector.dump_heap (heap_dump_file);
2572 fprintf (heap_dump_file, "<los>\n");
2573 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
2574 dump_object ((MonoObject*)bigobj->data, FALSE);
2575 fprintf (heap_dump_file, "</los>\n");
2577 fprintf (heap_dump_file, "</collection>\n");
2581 mono_sgen_register_moved_object (void *obj, void *destination)
2583 g_assert (mono_profiler_events & MONO_PROFILE_GC_MOVES);
2585 /* FIXME: handle this for parallel collector */
2586 g_assert (!major_collector.is_parallel);
2588 if (moved_objects_idx == MOVED_OBJECTS_NUM) {
2589 mono_profiler_gc_moves (moved_objects, moved_objects_idx);
2590 moved_objects_idx = 0;
2592 moved_objects [moved_objects_idx++] = obj;
2593 moved_objects [moved_objects_idx++] = destination;
2599 static gboolean inited = FALSE;
2604 mono_counters_register ("Minor fragment clear", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_pre_collection_fragment_clear);
2605 mono_counters_register ("Minor pinning", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_pinning);
2606 mono_counters_register ("Minor scan remsets", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_scan_remsets);
2607 mono_counters_register ("Minor scan cardtables", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_scan_card_table);
2608 mono_counters_register ("Minor scan pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_scan_pinned);
2609 mono_counters_register ("Minor scan registered roots", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_scan_registered_roots);
2610 mono_counters_register ("Minor scan thread data", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_scan_thread_data);
2611 mono_counters_register ("Minor finish gray stack", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_finish_gray_stack);
2612 mono_counters_register ("Minor fragment creation", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_fragment_creation);
2614 mono_counters_register ("Major fragment clear", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_pre_collection_fragment_clear);
2615 mono_counters_register ("Major pinning", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_pinning);
2616 mono_counters_register ("Major scan pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_pinned);
2617 mono_counters_register ("Major scan registered roots", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_registered_roots);
2618 mono_counters_register ("Major scan thread data", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_thread_data);
2619 mono_counters_register ("Major scan alloc_pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_alloc_pinned);
2620 mono_counters_register ("Major scan finalized", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_finalized);
2621 mono_counters_register ("Major scan big objects", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_big_objects);
2622 mono_counters_register ("Major finish gray stack", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_finish_gray_stack);
2623 mono_counters_register ("Major free big objects", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_free_bigobjs);
2624 mono_counters_register ("Major LOS sweep", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_los_sweep);
2625 mono_counters_register ("Major sweep", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_sweep);
2626 mono_counters_register ("Major fragment creation", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_fragment_creation);
2629 #ifdef HEAVY_STATISTICS
2630 mono_counters_register ("WBarrier set field", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_field);
2631 mono_counters_register ("WBarrier set arrayref", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_arrayref);
2632 mono_counters_register ("WBarrier arrayref copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_arrayref_copy);
2633 mono_counters_register ("WBarrier generic store called", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_generic_store);
2634 mono_counters_register ("WBarrier generic store stored", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_generic_store_remset);
2635 mono_counters_register ("WBarrier set root", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_root);
2636 mono_counters_register ("WBarrier value copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_value_copy);
2637 mono_counters_register ("WBarrier object copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_object_copy);
2639 mono_counters_register ("# objects allocated", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_alloced);
2640 mono_counters_register ("bytes allocated", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_bytes_alloced);
2641 mono_counters_register ("# objects allocated degraded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_alloced_degraded);
2642 mono_counters_register ("bytes allocated degraded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_bytes_alloced_degraded);
2643 mono_counters_register ("bytes allocated in LOS", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_bytes_alloced_los);
2645 mono_counters_register ("# copy_object() called (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_called_nursery);
2646 mono_counters_register ("# objects copied (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_copied_nursery);
2647 mono_counters_register ("# copy_object() called (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_called_major);
2648 mono_counters_register ("# objects copied (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_copied_major);
2650 mono_counters_register ("# scan_object() called (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_scan_object_called_nursery);
2651 mono_counters_register ("# scan_object() called (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_scan_object_called_major);
2653 mono_counters_register ("# nursery copy_object() failed from space", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_from_space);
2654 mono_counters_register ("# nursery copy_object() failed forwarded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_forwarded);
2655 mono_counters_register ("# nursery copy_object() failed pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_pinned);
2657 mono_counters_register ("# wasted fragments used", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_wasted_fragments_used);
2658 mono_counters_register ("bytes in wasted fragments", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_wasted_fragments_bytes);
2660 mono_counters_register ("Store remsets", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_store_remsets);
2661 mono_counters_register ("Unique store remsets", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_store_remsets_unique);
2662 mono_counters_register ("Saved remsets 1", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_saved_remsets_1);
2663 mono_counters_register ("Saved remsets 2", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_saved_remsets_2);
2664 mono_counters_register ("Non-global remsets processed", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_local_remsets_processed);
2665 mono_counters_register ("Global remsets added", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_added);
2666 mono_counters_register ("Global remsets re-added", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_readded);
2667 mono_counters_register ("Global remsets processed", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_processed);
2668 mono_counters_register ("Global remsets discarded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_discarded);
2675 need_major_collection (mword space_needed)
2677 mword los_alloced = los_memory_usage - MIN (last_los_memory_usage, los_memory_usage);
2678 return (space_needed > available_free_space ()) ||
2679 minor_collection_sections_alloced * major_collector.section_size + los_alloced > minor_collection_allowance;
2683 * Collect objects in the nursery. Returns whether to trigger a major
2687 collect_nursery (size_t requested_size)
2689 gboolean needs_major;
2690 size_t max_garbage_amount;
2691 char *orig_nursery_next;
2692 TV_DECLARE (all_atv);
2693 TV_DECLARE (all_btv);
2697 mono_perfcounters->gc_collections0++;
2699 current_collection_generation = GENERATION_NURSERY;
2701 binary_protocol_collection (GENERATION_NURSERY);
2702 check_scan_starts ();
2706 orig_nursery_next = nursery_next;
2707 nursery_next = MAX (nursery_next, nursery_last_pinned_end);
2708 /* FIXME: optimize later to use the higher address where an object can be present */
2709 nursery_next = MAX (nursery_next, nursery_real_end);
2711 DEBUG (1, fprintf (gc_debug_file, "Start nursery collection %d %p-%p, size: %d\n", num_minor_gcs, nursery_start, nursery_next, (int)(nursery_next - nursery_start)));
2712 max_garbage_amount = nursery_next - nursery_start;
2713 g_assert (nursery_section->size >= max_garbage_amount);
2715 /* world must be stopped already */
2716 TV_GETTIME (all_atv);
2719 /* Pinning no longer depends on clearing all nursery fragments */
2720 clear_current_nursery_fragment (orig_nursery_next);
2723 time_minor_pre_collection_fragment_clear += TV_ELAPSED_MS (atv, btv);
2726 check_for_xdomain_refs ();
2728 nursery_section->next_data = nursery_next;
2730 major_collector.start_nursery_collection ();
2732 gray_object_queue_init (&gray_queue, mono_sgen_get_unmanaged_allocator ());
2735 mono_stats.minor_gc_count ++;
2737 global_remset_cache_clear ();
2739 /* pin from pinned handles */
2741 mono_profiler_gc_event (MONO_GC_EVENT_MARK_START, 0);
2742 pin_from_roots (nursery_start, nursery_next);
2743 /* identify pinned objects */
2744 optimize_pin_queue (0);
2745 next_pin_slot = pin_objects_from_addresses (nursery_section, pin_queue, pin_queue + next_pin_slot, nursery_start, nursery_next, &gray_queue);
2746 nursery_section->pin_queue_start = pin_queue;
2747 nursery_section->pin_queue_num_entries = next_pin_slot;
2749 time_minor_pinning += TV_ELAPSED_MS (btv, atv);
2750 DEBUG (2, fprintf (gc_debug_file, "Finding pinned pointers: %d in %d usecs\n", next_pin_slot, TV_ELAPSED (btv, atv)));
2751 DEBUG (4, fprintf (gc_debug_file, "Start scan with %d pinned objects\n", next_pin_slot));
2753 if (consistency_check_at_minor_collection)
2754 check_consistency ();
2757 * walk all the roots and copy the young objects to the old generation,
2758 * starting from to_space
2761 scan_from_remsets (nursery_start, nursery_next, &gray_queue);
2762 /* we don't have complete write barrier yet, so we scan all the old generation sections */
2764 time_minor_scan_remsets += TV_ELAPSED_MS (atv, btv);
2765 DEBUG (2, fprintf (gc_debug_file, "Old generation scan: %d usecs\n", TV_ELAPSED (atv, btv)));
2767 if (use_cardtable) {
2769 card_tables_collect_stats (TRUE);
2770 scan_from_card_tables (nursery_start, nursery_next, &gray_queue);
2772 time_minor_scan_card_table += TV_ELAPSED_MS (atv, btv);
2775 drain_gray_stack (&gray_queue);
2778 time_minor_scan_pinned += TV_ELAPSED_MS (btv, atv);
2779 /* registered roots, this includes static fields */
2780 scan_from_registered_roots (major_collector.copy_object, nursery_start, nursery_next, ROOT_TYPE_NORMAL, &gray_queue);
2781 scan_from_registered_roots (major_collector.copy_object, nursery_start, nursery_next, ROOT_TYPE_WBARRIER, &gray_queue);
2783 time_minor_scan_registered_roots += TV_ELAPSED_MS (atv, btv);
2785 scan_thread_data (nursery_start, nursery_next, TRUE);
2787 time_minor_scan_thread_data += TV_ELAPSED_MS (btv, atv);
2790 finish_gray_stack (nursery_start, nursery_next, GENERATION_NURSERY, &gray_queue);
2792 time_minor_finish_gray_stack += TV_ELAPSED_MS (btv, atv);
2793 mono_profiler_gc_event (MONO_GC_EVENT_MARK_END, 0);
2795 if (objects_pinned) {
2796 evacuate_pin_staging_area ();
2797 optimize_pin_queue (0);
2798 nursery_section->pin_queue_start = pin_queue;
2799 nursery_section->pin_queue_num_entries = next_pin_slot;
2802 /* walk the pin_queue, build up the fragment list of free memory, unmark
2803 * pinned objects as we go, memzero() the empty fragments so they are ready for the
2806 mono_profiler_gc_event (MONO_GC_EVENT_RECLAIM_START, 0);
2807 build_nursery_fragments (pin_queue, next_pin_slot);
2808 mono_profiler_gc_event (MONO_GC_EVENT_RECLAIM_END, 0);
2810 time_minor_fragment_creation += TV_ELAPSED_MS (atv, btv);
2811 DEBUG (2, fprintf (gc_debug_file, "Fragment creation: %d usecs, %lu bytes available\n", TV_ELAPSED (atv, btv), (unsigned long)fragment_total));
2813 if (consistency_check_at_minor_collection)
2814 check_major_refs ();
2816 major_collector.finish_nursery_collection ();
2818 TV_GETTIME (all_btv);
2819 mono_stats.minor_gc_time_usecs += TV_ELAPSED (all_atv, all_btv);
2822 dump_heap ("minor", num_minor_gcs - 1, NULL);
2824 /* prepare the pin queue for the next collection */
2825 last_num_pinned = next_pin_slot;
2827 if (fin_ready_list || critical_fin_list) {
2828 DEBUG (4, fprintf (gc_debug_file, "Finalizer-thread wakeup: ready %d\n", num_ready_finalizers));
2829 mono_gc_finalize_notify ();
2833 g_assert (gray_object_queue_is_empty (&gray_queue));
2836 card_tables_collect_stats (FALSE);
2838 check_scan_starts ();
2840 binary_protocol_flush_buffers (FALSE);
2842 /*objects are late pinned because of lack of memory, so a major is a good call*/
2843 needs_major = need_major_collection (0) || objects_pinned;
2844 current_collection_generation = -1;
2851 major_do_collection (const char *reason)
2853 LOSObject *bigobj, *prevbo;
2854 TV_DECLARE (all_atv);
2855 TV_DECLARE (all_btv);
2858 /* FIXME: only use these values for the precise scan
2859 * note that to_space pointers should be excluded anyway...
2861 char *heap_start = NULL;
2862 char *heap_end = (char*)-1;
2863 int old_num_major_sections = major_collector.get_num_major_sections ();
2864 int num_major_sections, num_major_sections_saved, save_target, allowance_target;
2865 int old_next_pin_slot;
2866 mword los_memory_saved, los_memory_alloced, old_los_memory_usage;
2868 mono_perfcounters->gc_collections1++;
2871 * A domain could have been freed, resulting in
2872 * los_memory_usage being less than last_los_memory_usage.
2874 los_memory_alloced = los_memory_usage - MIN (last_los_memory_usage, los_memory_usage);
2875 old_los_memory_usage = los_memory_usage;
2878 //count_ref_nonref_objs ();
2879 //consistency_check ();
2881 binary_protocol_collection (GENERATION_OLD);
2882 check_scan_starts ();
2883 gray_object_queue_init (&gray_queue, mono_sgen_get_unmanaged_allocator ());
2884 if (major_collector.is_parallel)
2885 gray_object_queue_init (&workers_distribute_gray_queue, mono_sgen_get_unmanaged_allocator ());
2888 DEBUG (1, fprintf (gc_debug_file, "Start major collection %d\n", num_major_gcs));
2890 mono_stats.major_gc_count ++;
2892 /* world must be stopped already */
2893 TV_GETTIME (all_atv);
2896 /* Pinning depends on this */
2897 clear_nursery_fragments (nursery_next);
2900 time_major_pre_collection_fragment_clear += TV_ELAPSED_MS (atv, btv);
2903 check_for_xdomain_refs ();
2905 nursery_section->next_data = nursery_real_end;
2906 /* we should also coalesce scanning from sections close to each other
2907 * and deal with pointers outside of the sections later.
2910 if (major_collector.start_major_collection)
2911 major_collector.start_major_collection ();
2913 /* The remsets are not useful for a major collection */
2915 global_remset_cache_clear ();
2917 card_table_clear ();
2921 DEBUG (6, fprintf (gc_debug_file, "Collecting pinned addresses\n"));
2922 pin_from_roots ((void*)lowest_heap_address, (void*)highest_heap_address);
2923 optimize_pin_queue (0);
2926 * pin_queue now contains all candidate pointers, sorted and
2927 * uniqued. We must do two passes now to figure out which
2928 * objects are pinned.
2930 * The first is to find within the pin_queue the area for each
2931 * section. This requires that the pin_queue be sorted. We
2932 * also process the LOS objects and pinned chunks here.
2934 * The second, destructive, pass is to reduce the section
2935 * areas to pointers to the actually pinned objects.
2937 DEBUG (6, fprintf (gc_debug_file, "Pinning from sections\n"));
2938 /* first pass for the sections */
2939 mono_sgen_find_section_pin_queue_start_end (nursery_section);
2940 major_collector.find_pin_queue_start_ends (WORKERS_DISTRIBUTE_GRAY_QUEUE);
2941 /* identify possible pointers to the insize of large objects */
2942 DEBUG (6, fprintf (gc_debug_file, "Pinning from large objects\n"));
2943 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
2945 if (mono_sgen_find_optimized_pin_queue_area (bigobj->data, (char*)bigobj->data + bigobj->size, &dummy)) {
2946 pin_object (bigobj->data);
2947 /* FIXME: only enqueue if object has references */
2948 GRAY_OBJECT_ENQUEUE (WORKERS_DISTRIBUTE_GRAY_QUEUE, bigobj->data);
2950 mono_sgen_pin_stats_register_object ((char*) bigobj->data, safe_object_get_size ((MonoObject*) bigobj->data));
2951 DEBUG (6, fprintf (gc_debug_file, "Marked large object %p (%s) size: %lu from roots\n", bigobj->data, safe_name (bigobj->data), (unsigned long)bigobj->size));
2954 /* second pass for the sections */
2955 mono_sgen_pin_objects_in_section (nursery_section, WORKERS_DISTRIBUTE_GRAY_QUEUE);
2956 major_collector.pin_objects (WORKERS_DISTRIBUTE_GRAY_QUEUE);
2957 old_next_pin_slot = next_pin_slot;
2960 time_major_pinning += TV_ELAPSED_MS (atv, btv);
2961 DEBUG (2, fprintf (gc_debug_file, "Finding pinned pointers: %d in %d usecs\n", next_pin_slot, TV_ELAPSED (atv, btv)));
2962 DEBUG (4, fprintf (gc_debug_file, "Start scan with %d pinned objects\n", next_pin_slot));
2964 major_collector.init_to_space ();
2966 workers_start_all_workers (1);
2969 time_major_scan_pinned += TV_ELAPSED_MS (btv, atv);
2971 /* registered roots, this includes static fields */
2972 scan_from_registered_roots (major_collector.copy_or_mark_object, heap_start, heap_end, ROOT_TYPE_NORMAL, WORKERS_DISTRIBUTE_GRAY_QUEUE);
2973 scan_from_registered_roots (major_collector.copy_or_mark_object, heap_start, heap_end, ROOT_TYPE_WBARRIER, WORKERS_DISTRIBUTE_GRAY_QUEUE);
2975 time_major_scan_registered_roots += TV_ELAPSED_MS (atv, btv);
2978 /* FIXME: This is the wrong place for this, because it does
2980 scan_thread_data (heap_start, heap_end, TRUE);
2982 time_major_scan_thread_data += TV_ELAPSED_MS (btv, atv);
2985 time_major_scan_alloc_pinned += TV_ELAPSED_MS (atv, btv);
2987 /* scan the list of objects ready for finalization */
2988 scan_finalizer_entries (major_collector.copy_or_mark_object, fin_ready_list, WORKERS_DISTRIBUTE_GRAY_QUEUE);
2989 scan_finalizer_entries (major_collector.copy_or_mark_object, critical_fin_list, WORKERS_DISTRIBUTE_GRAY_QUEUE);
2991 time_major_scan_finalized += TV_ELAPSED_MS (btv, atv);
2992 DEBUG (2, fprintf (gc_debug_file, "Root scan: %d usecs\n", TV_ELAPSED (btv, atv)));
2995 time_major_scan_big_objects += TV_ELAPSED_MS (atv, btv);
2997 if (major_collector.is_parallel) {
2998 while (!gray_object_queue_is_empty (WORKERS_DISTRIBUTE_GRAY_QUEUE)) {
2999 workers_distribute_gray_queue_sections ();
3003 workers_change_num_working (-1);
3006 if (major_collector.is_parallel)
3007 g_assert (gray_object_queue_is_empty (&gray_queue));
3009 /* all the objects in the heap */
3010 finish_gray_stack (heap_start, heap_end, GENERATION_OLD, &gray_queue);
3012 time_major_finish_gray_stack += TV_ELAPSED_MS (btv, atv);
3014 if (objects_pinned) {
3015 /*This is slow, but we just OOM'd*/
3016 mono_sgen_pin_queue_clear_discarded_entries (nursery_section, old_next_pin_slot);
3017 evacuate_pin_staging_area ();
3018 optimize_pin_queue (0);
3019 mono_sgen_find_section_pin_queue_start_end (nursery_section);
3023 /* sweep the big objects list */
3025 for (bigobj = los_object_list; bigobj;) {
3026 if (object_is_pinned (bigobj->data)) {
3027 unpin_object (bigobj->data);
3030 /* not referenced anywhere, so we can free it */
3032 prevbo->next = bigobj->next;
3034 los_object_list = bigobj->next;
3036 bigobj = bigobj->next;
3037 free_large_object (to_free);
3041 bigobj = bigobj->next;
3045 time_major_free_bigobjs += TV_ELAPSED_MS (atv, btv);
3050 time_major_los_sweep += TV_ELAPSED_MS (btv, atv);
3052 major_collector.sweep ();
3055 time_major_sweep += TV_ELAPSED_MS (atv, btv);
3057 /* walk the pin_queue, build up the fragment list of free memory, unmark
3058 * pinned objects as we go, memzero() the empty fragments so they are ready for the
3061 build_nursery_fragments (nursery_section->pin_queue_start, nursery_section->pin_queue_num_entries);
3064 time_major_fragment_creation += TV_ELAPSED_MS (btv, atv);
3066 TV_GETTIME (all_btv);
3067 mono_stats.major_gc_time_usecs += TV_ELAPSED (all_atv, all_btv);
3070 dump_heap ("major", num_major_gcs - 1, reason);
3072 /* prepare the pin queue for the next collection */
3074 if (fin_ready_list || critical_fin_list) {
3075 DEBUG (4, fprintf (gc_debug_file, "Finalizer-thread wakeup: ready %d\n", num_ready_finalizers));
3076 mono_gc_finalize_notify ();
3080 g_assert (gray_object_queue_is_empty (&gray_queue));
3082 num_major_sections = major_collector.get_num_major_sections ();
3084 num_major_sections_saved = MAX (old_num_major_sections - num_major_sections, 0);
3085 los_memory_saved = MAX (old_los_memory_usage - los_memory_usage, 1);
3087 save_target = ((num_major_sections * major_collector.section_size) + los_memory_saved) / 2;
3089 * We aim to allow the allocation of as many sections as is
3090 * necessary to reclaim save_target sections in the next
3091 * collection. We assume the collection pattern won't change.
3092 * In the last cycle, we had num_major_sections_saved for
3093 * minor_collection_sections_alloced. Assuming things won't
3094 * change, this must be the same ratio as save_target for
3095 * allowance_target, i.e.
3097 * num_major_sections_saved save_target
3098 * --------------------------------- == ----------------
3099 * minor_collection_sections_alloced allowance_target
3103 allowance_target = (mword)((double)save_target * (double)(minor_collection_sections_alloced * major_collector.section_size + los_memory_alloced) / (double)(num_major_sections_saved * major_collector.section_size + los_memory_saved));
3105 minor_collection_allowance = MAX (MIN (allowance_target, num_major_sections * major_collector.section_size + los_memory_usage), MIN_MINOR_COLLECTION_ALLOWANCE);
3107 minor_collection_sections_alloced = 0;
3108 last_los_memory_usage = los_memory_usage;
3110 major_collector.finish_major_collection ();
3112 check_scan_starts ();
3114 binary_protocol_flush_buffers (FALSE);
3116 //consistency_check ();
3120 major_collection (const char *reason)
3122 if (g_getenv ("MONO_GC_NO_MAJOR")) {
3123 collect_nursery (0);
3127 current_collection_generation = GENERATION_OLD;
3128 major_do_collection (reason);
3129 current_collection_generation = -1;
3133 sgen_collect_major_no_lock (const char *reason)
3135 mono_profiler_gc_event (MONO_GC_EVENT_START, 1);
3137 major_collection (reason);
3139 mono_profiler_gc_event (MONO_GC_EVENT_END, 1);
3143 * When deciding if it's better to collect or to expand, keep track
3144 * of how much garbage was reclaimed with the last collection: if it's too
3146 * This is called when we could not allocate a small object.
3148 static void __attribute__((noinline))
3149 minor_collect_or_expand_inner (size_t size)
3151 int do_minor_collection = 1;
3153 g_assert (nursery_section);
3154 if (do_minor_collection) {
3155 mono_profiler_gc_event (MONO_GC_EVENT_START, 0);
3157 if (collect_nursery (size)) {
3158 mono_profiler_gc_event (MONO_GC_EVENT_START, 1);
3159 major_collection ("minor overflow");
3160 /* keep events symmetric */
3161 mono_profiler_gc_event (MONO_GC_EVENT_END, 1);
3163 DEBUG (2, fprintf (gc_debug_file, "Heap size: %lu, LOS size: %lu\n", (unsigned long)total_alloc, (unsigned long)los_memory_usage));
3165 /* this also sets the proper pointers for the next allocation */
3166 if (!search_fragment_for_size (size)) {
3168 /* TypeBuilder and MonoMethod are killing mcs with fragmentation */
3169 DEBUG (1, fprintf (gc_debug_file, "nursery collection didn't find enough room for %zd alloc (%d pinned)\n", size, last_num_pinned));
3170 for (i = 0; i < last_num_pinned; ++i) {
3171 DEBUG (3, fprintf (gc_debug_file, "Bastard pinning obj %p (%s), size: %d\n", pin_queue [i], safe_name (pin_queue [i]), safe_object_get_size (pin_queue [i])));
3175 mono_profiler_gc_event (MONO_GC_EVENT_END, 0);
3177 //report_internal_mem_usage ();
3181 * ######################################################################
3182 * ######## Memory allocation from the OS
3183 * ######################################################################
3184 * This section of code deals with getting memory from the OS and
3185 * allocating memory for GC-internal data structures.
3186 * Internal memory can be handled with a freelist for small objects.
3192 G_GNUC_UNUSED static void
3193 report_internal_mem_usage (void)
3195 printf ("Internal memory usage:\n");
3196 mono_sgen_report_internal_mem_usage ();
3197 printf ("Pinned memory usage:\n");
3198 major_collector.report_pinned_memory_usage ();
3202 * Allocate a big chunk of memory from the OS (usually 64KB to several megabytes).
3203 * This must not require any lock.
3206 mono_sgen_alloc_os_memory (size_t size, int activate)
3209 unsigned long prot_flags = activate? MONO_MMAP_READ|MONO_MMAP_WRITE: MONO_MMAP_NONE;
3211 prot_flags |= MONO_MMAP_PRIVATE | MONO_MMAP_ANON;
3212 size += pagesize - 1;
3213 size &= ~(pagesize - 1);
3214 ptr = mono_valloc (0, size, prot_flags);
3216 total_alloc += size;
3221 * Free the memory returned by mono_sgen_alloc_os_memory (), returning it to the OS.
3224 mono_sgen_free_os_memory (void *addr, size_t size)
3226 mono_vfree (addr, size);
3228 size += pagesize - 1;
3229 size &= ~(pagesize - 1);
3231 total_alloc -= size;
3235 * ######################################################################
3236 * ######## Object allocation
3237 * ######################################################################
3238 * This section of code deals with allocating memory for objects.
3239 * There are several ways:
3240 * *) allocate large objects
3241 * *) allocate normal objects
3242 * *) fast lock-free allocation
3243 * *) allocation of pinned objects
3247 setup_fragment (Fragment *frag, Fragment *prev, size_t size)
3249 /* remove from the list */
3251 prev->next = frag->next;
3253 nursery_fragments = frag->next;
3254 nursery_next = frag->fragment_start;
3255 nursery_frag_real_end = frag->fragment_end;
3257 DEBUG (4, fprintf (gc_debug_file, "Using nursery fragment %p-%p, size: %td (req: %zd)\n", nursery_next, nursery_frag_real_end, nursery_frag_real_end - nursery_next, size));
3258 frag->next = fragment_freelist;
3259 fragment_freelist = frag;
3262 /* check if we have a suitable fragment in nursery_fragments to be able to allocate
3263 * an object of size @size
3264 * Return FALSE if not found (which means we need a collection)
3267 search_fragment_for_size (size_t size)
3269 Fragment *frag, *prev;
3270 DEBUG (4, fprintf (gc_debug_file, "Searching nursery fragment %p, size: %zd\n", nursery_frag_real_end, size));
3272 if (nursery_frag_real_end > nursery_next && nursery_clear_policy == CLEAR_AT_TLAB_CREATION) {
3273 /* Clear the remaining space, pinning depends on this */
3274 memset (nursery_next, 0, nursery_frag_real_end - nursery_next);
3278 for (frag = nursery_fragments; frag; frag = frag->next) {
3279 if (size <= (frag->fragment_end - frag->fragment_start)) {
3280 setup_fragment (frag, prev, size);
3289 * Same as search_fragment_for_size but if search for @desired_size fails, try to satisfy @minimum_size.
3290 * This improves nursery usage.
3293 search_fragment_for_size_range (size_t desired_size, size_t minimum_size)
3295 Fragment *frag, *prev, *min_prev;
3296 DEBUG (4, fprintf (gc_debug_file, "Searching nursery fragment %p, desired size: %zd minimum size %zd\n", nursery_frag_real_end, desired_size, minimum_size));
3298 if (nursery_frag_real_end > nursery_next && nursery_clear_policy == CLEAR_AT_TLAB_CREATION) {
3299 /* Clear the remaining space, pinning depends on this */
3300 memset (nursery_next, 0, nursery_frag_real_end - nursery_next);
3303 min_prev = GINT_TO_POINTER (-1);
3306 for (frag = nursery_fragments; frag; frag = frag->next) {
3307 int frag_size = frag->fragment_end - frag->fragment_start;
3308 if (desired_size <= frag_size) {
3309 setup_fragment (frag, prev, desired_size);
3310 return desired_size;
3312 if (minimum_size <= frag_size)
3318 if (min_prev != GINT_TO_POINTER (-1)) {
3321 frag = min_prev->next;
3323 frag = nursery_fragments;
3325 frag_size = frag->fragment_end - frag->fragment_start;
3326 HEAVY_STAT (++stat_wasted_fragments_used);
3327 HEAVY_STAT (stat_wasted_fragments_bytes += frag_size);
3329 setup_fragment (frag, min_prev, minimum_size);
3337 alloc_degraded (MonoVTable *vtable, size_t size)
3339 if (need_major_collection (0)) {
3340 mono_profiler_gc_event (MONO_GC_EVENT_START, 1);
3342 major_collection ("degraded overflow");
3344 mono_profiler_gc_event (MONO_GC_EVENT_END, 1);
3347 degraded_mode += size;
3348 return major_collector.alloc_degraded (vtable, size);
3352 * Provide a variant that takes just the vtable for small fixed-size objects.
3353 * The aligned size is already computed and stored in vt->gc_descr.
3354 * Note: every SCAN_START_SIZE or so we are given the chance to do some special
3355 * processing. We can keep track of where objects start, for example,
3356 * so when we scan the thread stacks for pinned objects, we can start
3357 * a search for the pinned object in SCAN_START_SIZE chunks.
3360 mono_gc_alloc_obj_nolock (MonoVTable *vtable, size_t size)
3362 /* FIXME: handle OOM */
3367 HEAVY_STAT (++stat_objects_alloced);
3368 if (size <= MAX_SMALL_OBJ_SIZE)
3369 HEAVY_STAT (stat_bytes_alloced += size);
3371 HEAVY_STAT (stat_bytes_alloced_los += size);
3373 size = ALIGN_UP (size);
3375 g_assert (vtable->gc_descr);
3377 if (G_UNLIKELY (collect_before_allocs)) {
3378 if (nursery_section) {
3379 mono_profiler_gc_event (MONO_GC_EVENT_START, 0);
3381 collect_nursery (0);
3383 mono_profiler_gc_event (MONO_GC_EVENT_END, 0);
3384 if (!degraded_mode && !search_fragment_for_size (size)) {
3386 g_assert_not_reached ();
3392 * We must already have the lock here instead of after the
3393 * fast path because we might be interrupted in the fast path
3394 * (after confirming that new_next < TLAB_TEMP_END) by the GC,
3395 * and we'll end up allocating an object in a fragment which
3396 * no longer belongs to us.
3398 * The managed allocator does not do this, but it's treated
3399 * specially by the world-stopping code.
3402 if (size > MAX_SMALL_OBJ_SIZE) {
3403 p = alloc_large_inner (vtable, size);
3405 /* tlab_next and tlab_temp_end are TLS vars so accessing them might be expensive */
3407 p = (void**)TLAB_NEXT;
3408 /* FIXME: handle overflow */
3409 new_next = (char*)p + size;
3410 TLAB_NEXT = new_next;
3412 if (G_LIKELY (new_next < TLAB_TEMP_END)) {
3416 * FIXME: We might need a memory barrier here so the change to tlab_next is
3417 * visible before the vtable store.
3420 DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
3421 binary_protocol_alloc (p , vtable, size);
3422 g_assert (*p == NULL);
3425 g_assert (TLAB_NEXT == new_next);
3432 /* there are two cases: the object is too big or we run out of space in the TLAB */
3433 /* we also reach here when the thread does its first allocation after a minor
3434 * collection, since the tlab_ variables are initialized to NULL.
3435 * there can be another case (from ORP), if we cooperate with the runtime a bit:
3436 * objects that need finalizers can have the high bit set in their size
3437 * so the above check fails and we can readily add the object to the queue.
3438 * This avoids taking again the GC lock when registering, but this is moot when
3439 * doing thread-local allocation, so it may not be a good idea.
3441 g_assert (TLAB_NEXT == new_next);
3442 if (TLAB_NEXT >= TLAB_REAL_END) {
3444 * Run out of space in the TLAB. When this happens, some amount of space
3445 * remains in the TLAB, but not enough to satisfy the current allocation
3446 * request. Currently, we retire the TLAB in all cases, later we could
3447 * keep it if the remaining space is above a treshold, and satisfy the
3448 * allocation directly from the nursery.
3451 /* when running in degraded mode, we continue allocing that way
3452 * for a while, to decrease the number of useless nursery collections.
3454 if (degraded_mode && degraded_mode < DEFAULT_NURSERY_SIZE) {
3455 p = alloc_degraded (vtable, size);
3456 binary_protocol_alloc_degraded (p, vtable, size);
3460 /*FIXME This codepath is current deadcode since tlab_size > MAX_SMALL_OBJ_SIZE*/
3461 if (size > tlab_size) {
3462 /* Allocate directly from the nursery */
3463 if (nursery_next + size >= nursery_frag_real_end) {
3464 if (!search_fragment_for_size (size)) {
3465 minor_collect_or_expand_inner (size);
3466 if (degraded_mode) {
3467 p = alloc_degraded (vtable, size);
3468 binary_protocol_alloc_degraded (p, vtable, size);
3474 p = (void*)nursery_next;
3475 nursery_next += size;
3476 if (nursery_next > nursery_frag_real_end) {
3481 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION) {
3482 memset (p, 0, size);
3485 int alloc_size = tlab_size;
3486 int available_in_nursery = nursery_frag_real_end - nursery_next;
3488 DEBUG (3, fprintf (gc_debug_file, "Retire TLAB: %p-%p [%ld]\n", TLAB_START, TLAB_REAL_END, (long)(TLAB_REAL_END - TLAB_NEXT - size)));
3490 if (alloc_size >= available_in_nursery) {
3491 if (available_in_nursery > MAX_NURSERY_TLAB_WASTE && available_in_nursery > size) {
3492 alloc_size = available_in_nursery;
3494 alloc_size = search_fragment_for_size_range (tlab_size, size);
3496 alloc_size = tlab_size;
3497 minor_collect_or_expand_inner (tlab_size);
3498 if (degraded_mode) {
3499 p = alloc_degraded (vtable, size);
3500 binary_protocol_alloc_degraded (p, vtable, size);
3507 /* Allocate a new TLAB from the current nursery fragment */
3508 TLAB_START = nursery_next;
3509 nursery_next += alloc_size;
3510 TLAB_NEXT = TLAB_START;
3511 TLAB_REAL_END = TLAB_START + alloc_size;
3512 TLAB_TEMP_END = TLAB_START + MIN (SCAN_START_SIZE, alloc_size);
3514 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION) {
3515 memset (TLAB_START, 0, alloc_size);
3518 /* Allocate from the TLAB */
3519 p = (void*)TLAB_NEXT;
3521 g_assert (TLAB_NEXT <= TLAB_REAL_END);
3523 nursery_section->scan_starts [((char*)p - (char*)nursery_section->data)/SCAN_START_SIZE] = (char*)p;
3526 /* Reached tlab_temp_end */
3528 /* record the scan start so we can find pinned objects more easily */
3529 nursery_section->scan_starts [((char*)p - (char*)nursery_section->data)/SCAN_START_SIZE] = (char*)p;
3530 /* we just bump tlab_temp_end as well */
3531 TLAB_TEMP_END = MIN (TLAB_REAL_END, TLAB_NEXT + SCAN_START_SIZE);
3532 DEBUG (5, fprintf (gc_debug_file, "Expanding local alloc: %p-%p\n", TLAB_NEXT, TLAB_TEMP_END));
3537 DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
3538 binary_protocol_alloc (p, vtable, size);
3546 mono_gc_try_alloc_obj_nolock (MonoVTable *vtable, size_t size)
3552 size = ALIGN_UP (size);
3554 g_assert (vtable->gc_descr);
3555 if (size <= MAX_SMALL_OBJ_SIZE) {
3556 /* tlab_next and tlab_temp_end are TLS vars so accessing them might be expensive */
3558 p = (void**)TLAB_NEXT;
3559 /* FIXME: handle overflow */
3560 new_next = (char*)p + size;
3561 TLAB_NEXT = new_next;
3563 if (G_LIKELY (new_next < TLAB_TEMP_END)) {
3567 * FIXME: We might need a memory barrier here so the change to tlab_next is
3568 * visible before the vtable store.
3571 HEAVY_STAT (++stat_objects_alloced);
3572 HEAVY_STAT (stat_bytes_alloced += size);
3574 DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
3575 binary_protocol_alloc (p, vtable, size);
3576 g_assert (*p == NULL);
3579 g_assert (TLAB_NEXT == new_next);
3588 mono_gc_alloc_obj (MonoVTable *vtable, size_t size)
3591 #ifndef DISABLE_CRITICAL_REGION
3593 ENTER_CRITICAL_REGION;
3594 res = mono_gc_try_alloc_obj_nolock (vtable, size);
3596 EXIT_CRITICAL_REGION;
3599 EXIT_CRITICAL_REGION;
3602 res = mono_gc_alloc_obj_nolock (vtable, size);
3604 if (G_UNLIKELY (!res))
3605 return mono_gc_out_of_memory (size);
3610 mono_gc_alloc_vector (MonoVTable *vtable, size_t size, uintptr_t max_length)
3613 #ifndef DISABLE_CRITICAL_REGION
3615 ENTER_CRITICAL_REGION;
3616 arr = mono_gc_try_alloc_obj_nolock (vtable, size);
3618 arr->max_length = max_length;
3619 EXIT_CRITICAL_REGION;
3622 EXIT_CRITICAL_REGION;
3627 arr = mono_gc_alloc_obj_nolock (vtable, size);
3628 if (G_UNLIKELY (!arr)) {
3630 return mono_gc_out_of_memory (size);
3633 arr->max_length = max_length;
3641 mono_gc_alloc_array (MonoVTable *vtable, size_t size, uintptr_t max_length, uintptr_t bounds_size)
3644 MonoArrayBounds *bounds;
3648 arr = mono_gc_alloc_obj_nolock (vtable, size);
3649 if (G_UNLIKELY (!arr)) {
3651 return mono_gc_out_of_memory (size);
3654 arr->max_length = max_length;
3656 bounds = (MonoArrayBounds*)((char*)arr + size - bounds_size);
3657 arr->bounds = bounds;
3665 mono_gc_alloc_string (MonoVTable *vtable, size_t size, gint32 len)
3668 #ifndef DISABLE_CRITICAL_REGION
3670 ENTER_CRITICAL_REGION;
3671 str = mono_gc_try_alloc_obj_nolock (vtable, size);
3674 EXIT_CRITICAL_REGION;
3677 EXIT_CRITICAL_REGION;
3682 str = mono_gc_alloc_obj_nolock (vtable, size);
3683 if (G_UNLIKELY (!str)) {
3685 return mono_gc_out_of_memory (size);
3696 * To be used for interned strings and possibly MonoThread, reflection handles.
3697 * We may want to explicitly free these objects.
3700 mono_gc_alloc_pinned_obj (MonoVTable *vtable, size_t size)
3703 size = ALIGN_UP (size);
3706 if (size > MAX_SMALL_OBJ_SIZE) {
3707 /* large objects are always pinned anyway */
3708 p = alloc_large_inner (vtable, size);
3710 DEBUG (9, g_assert (vtable->klass->inited));
3711 p = major_collector.alloc_small_pinned_obj (size, vtable->klass->has_references);
3714 DEBUG (6, fprintf (gc_debug_file, "Allocated pinned object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
3715 binary_protocol_alloc_pinned (p, vtable, size);
3723 * ######################################################################
3724 * ######## Finalization support
3725 * ######################################################################
3729 * this is valid for the nursery: if the object has been forwarded it means it's
3730 * still refrenced from a root. If it is pinned it's still alive as well.
3731 * Return TRUE if @obj is ready to be finalized.
3733 #define object_is_fin_ready(obj) (!object_is_pinned (obj) && !object_is_forwarded (obj))
3736 is_critical_finalizer (FinalizeEntry *entry)
3741 if (!mono_defaults.critical_finalizer_object)
3744 obj = entry->object;
3745 class = ((MonoVTable*)LOAD_VTABLE (obj))->klass;
3747 return mono_class_has_parent (class, mono_defaults.critical_finalizer_object);
3751 queue_finalization_entry (FinalizeEntry *entry) {
3752 if (is_critical_finalizer (entry)) {
3753 entry->next = critical_fin_list;
3754 critical_fin_list = entry;
3756 entry->next = fin_ready_list;
3757 fin_ready_list = entry;
3761 /* LOCKING: requires that the GC lock is held */
3763 rehash_fin_table (FinalizeEntryHashTable *hash_table)
3765 FinalizeEntry **finalizable_hash = hash_table->table;
3766 mword finalizable_hash_size = hash_table->size;
3769 FinalizeEntry **new_hash;
3770 FinalizeEntry *entry, *next;
3771 int new_size = g_spaced_primes_closest (hash_table->num_registered);
3773 new_hash = mono_sgen_alloc_internal_dynamic (new_size * sizeof (FinalizeEntry*), INTERNAL_MEM_FIN_TABLE);
3774 for (i = 0; i < finalizable_hash_size; ++i) {
3775 for (entry = finalizable_hash [i]; entry; entry = next) {
3776 hash = mono_object_hash (entry->object) % new_size;
3778 entry->next = new_hash [hash];
3779 new_hash [hash] = entry;
3782 mono_sgen_free_internal_dynamic (finalizable_hash, finalizable_hash_size * sizeof (FinalizeEntry*), INTERNAL_MEM_FIN_TABLE);
3783 hash_table->table = new_hash;
3784 hash_table->size = new_size;
3787 /* LOCKING: requires that the GC lock is held */
3789 rehash_fin_table_if_necessary (FinalizeEntryHashTable *hash_table)
3791 if (hash_table->num_registered >= hash_table->size * 2)
3792 rehash_fin_table (hash_table);
3795 /* LOCKING: requires that the GC lock is held */
3797 finalize_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, int generation, GrayQueue *queue)
3799 FinalizeEntryHashTable *hash_table = get_finalize_entry_hash_table (generation);
3800 FinalizeEntry *entry, *prev;
3802 FinalizeEntry **finalizable_hash = hash_table->table;
3803 mword finalizable_hash_size = hash_table->size;
3807 for (i = 0; i < finalizable_hash_size; ++i) {
3809 for (entry = finalizable_hash [i]; entry;) {
3810 if ((char*)entry->object >= start && (char*)entry->object < end && !major_collector.is_object_live (entry->object)) {
3811 gboolean is_fin_ready = object_is_fin_ready (entry->object);
3812 char *copy = entry->object;
3813 copy_func ((void**)©, queue);
3816 FinalizeEntry *next;
3817 /* remove and put in fin_ready_list */
3819 prev->next = entry->next;
3821 finalizable_hash [i] = entry->next;
3823 num_ready_finalizers++;
3824 hash_table->num_registered--;
3825 queue_finalization_entry (entry);
3826 /* Make it survive */
3827 from = entry->object;
3828 entry->object = copy;
3829 DEBUG (5, fprintf (gc_debug_file, "Queueing object for finalization: %p (%s) (was at %p) (%d/%d)\n", entry->object, safe_name (entry->object), from, num_ready_finalizers, hash_table->num_registered));
3833 char *from = entry->object;
3834 if (hash_table == &minor_finalizable_hash && !ptr_in_nursery (copy)) {
3835 FinalizeEntry *next = entry->next;
3836 unsigned int major_hash;
3837 /* remove from the list */
3839 prev->next = entry->next;
3841 finalizable_hash [i] = entry->next;
3842 hash_table->num_registered--;
3844 entry->object = copy;
3846 /* insert it into the major hash */
3847 rehash_fin_table_if_necessary (&major_finalizable_hash);
3848 major_hash = mono_object_hash ((MonoObject*) copy) %
3849 major_finalizable_hash.size;
3850 entry->next = major_finalizable_hash.table [major_hash];
3851 major_finalizable_hash.table [major_hash] = entry;
3852 major_finalizable_hash.num_registered++;
3854 DEBUG (5, fprintf (gc_debug_file, "Promoting finalization of object %p (%s) (was at %p) to major table\n", copy, safe_name (copy), from));
3859 /* update pointer */
3860 DEBUG (5, fprintf (gc_debug_file, "Updating object for finalization: %p (%s) (was at %p)\n", entry->object, safe_name (entry->object), from));
3861 entry->object = copy;
3866 entry = entry->next;
3872 object_is_reachable (char *object, char *start, char *end)
3874 /*This happens for non nursery objects during minor collections. We just treat all objects as alive.*/
3875 if (object < start || object >= end)
3877 return !object_is_fin_ready (object) || major_collector.is_object_live (object);
3880 /* LOCKING: requires that the GC lock is held */
3882 null_ephemerons_for_domain (MonoDomain *domain)
3884 EphemeronLinkNode *current = ephemeron_list, *prev = NULL;
3887 MonoObject *object = (MonoObject*)current->array;
3889 if (object && !object->vtable) {
3890 EphemeronLinkNode *tmp = current;
3893 prev->next = current->next;
3895 ephemeron_list = current->next;
3897 current = current->next;
3898 mono_sgen_free_internal (tmp, INTERNAL_MEM_EPHEMERON_LINK);
3901 current = current->next;
3906 /* LOCKING: requires that the GC lock is held */
3908 clear_unreachable_ephemerons (CopyOrMarkObjectFunc copy_func, char *start, char *end, GrayQueue *queue)
3910 int was_in_nursery, was_promoted;
3911 EphemeronLinkNode *current = ephemeron_list, *prev = NULL;
3913 Ephemeron *cur, *array_end;
3917 char *object = current->array;
3919 if (!object_is_reachable (object, start, end)) {
3920 EphemeronLinkNode *tmp = current;
3922 DEBUG (5, fprintf (gc_debug_file, "Dead Ephemeron array at %p\n", object));
3925 prev->next = current->next;
3927 ephemeron_list = current->next;
3929 current = current->next;
3930 mono_sgen_free_internal (tmp, INTERNAL_MEM_EPHEMERON_LINK);
3935 was_in_nursery = ptr_in_nursery (object);
3936 copy_func ((void**)&object, queue);
3937 current->array = object;
3939 /*The array was promoted, add global remsets for key/values left behind in nursery.*/
3940 was_promoted = was_in_nursery && !ptr_in_nursery (object);
3942 DEBUG (5, fprintf (gc_debug_file, "Clearing unreachable entries for ephemeron array at %p\n", object));
3944 array = (MonoArray*)object;
3945 cur = mono_array_addr (array, Ephemeron, 0);
3946 array_end = cur + mono_array_length_fast (array);
3947 tombstone = (char*)((MonoVTable*)LOAD_VTABLE (object))->domain->ephemeron_tombstone;
3949 for (; cur < array_end; ++cur) {
3950 char *key = (char*)cur->key;
3952 if (!key || key == tombstone)
3955 DEBUG (5, fprintf (gc_debug_file, "[%td] key %p (%s) value %p (%s)\n", cur - mono_array_addr (array, Ephemeron, 0),
3956 key, object_is_reachable (key, start, end) ? "reachable" : "unreachable",
3957 cur->value, cur->value && object_is_reachable (cur->value, start, end) ? "reachable" : "unreachable"));
3959 if (!object_is_reachable (key, start, end)) {
3960 cur->key = tombstone;
3966 if (ptr_in_nursery (key)) {/*key was not promoted*/
3967 DEBUG (5, fprintf (gc_debug_file, "\tAdded remset to key %p\n", key));
3968 mono_sgen_add_to_global_remset (&cur->key);
3970 if (ptr_in_nursery (cur->value)) {/*value was not promoted*/
3971 DEBUG (5, fprintf (gc_debug_file, "\tAdded remset to value %p\n", cur->value));
3972 mono_sgen_add_to_global_remset (&cur->value);
3977 current = current->next;
3981 /* LOCKING: requires that the GC lock is held */
3983 mark_ephemerons_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, GrayQueue *queue)
3985 int nothing_marked = 1;
3986 EphemeronLinkNode *current = ephemeron_list;
3988 Ephemeron *cur, *array_end;
3991 for (current = ephemeron_list; current; current = current->next) {
3992 char *object = current->array;
3993 DEBUG (5, fprintf (gc_debug_file, "Ephemeron array at %p\n", object));
3995 /*We ignore arrays in old gen during minor collections since all objects are promoted by the remset machinery.*/
3996 if (object < start || object >= end)
3999 /*It has to be alive*/
4000 if (!object_is_reachable (object, start, end)) {
4001 DEBUG (5, fprintf (gc_debug_file, "\tnot reachable\n"));
4005 copy_func ((void**)&object, queue);
4007 array = (MonoArray*)object;
4008 cur = mono_array_addr (array, Ephemeron, 0);
4009 array_end = cur + mono_array_length_fast (array);
4010 tombstone = (char*)((MonoVTable*)LOAD_VTABLE (object))->domain->ephemeron_tombstone;
4012 for (; cur < array_end; ++cur) {
4013 char *key = cur->key;
4015 if (!key || key == tombstone)
4018 DEBUG (5, fprintf (gc_debug_file, "[%td] key %p (%s) value %p (%s)\n", cur - mono_array_addr (array, Ephemeron, 0),
4019 key, object_is_reachable (key, start, end) ? "reachable" : "unreachable",
4020 cur->value, cur->value && object_is_reachable (cur->value, start, end) ? "reachable" : "unreachable"));
4022 if (object_is_reachable (key, start, end)) {
4023 char *value = cur->value;
4025 copy_func ((void**)&cur->key, queue);
4027 if (!object_is_reachable (value, start, end))
4029 copy_func ((void**)&cur->value, queue);
4035 DEBUG (5, fprintf (gc_debug_file, "Ephemeron run finished. Is it done %d\n", nothing_marked));
4036 return nothing_marked;
4039 /* LOCKING: requires that the GC lock is held */
4041 null_link_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, int generation, GrayQueue *queue)
4043 DisappearingLinkHashTable *hash = get_dislink_hash_table (generation);
4044 DisappearingLink **disappearing_link_hash = hash->table;
4045 int disappearing_link_hash_size = hash->size;
4046 DisappearingLink *entry, *prev;
4048 if (!hash->num_links)
4050 for (i = 0; i < disappearing_link_hash_size; ++i) {
4052 for (entry = disappearing_link_hash [i]; entry;) {
4053 char *object = DISLINK_OBJECT (entry);
4054 if (object >= start && object < end && !major_collector.is_object_live (object)) {
4055 gboolean track = DISLINK_TRACK (entry);
4056 if (!track && object_is_fin_ready (object)) {
4057 void **p = entry->link;
4058 DisappearingLink *old;
4060 /* remove from list */
4062 prev->next = entry->next;
4064 disappearing_link_hash [i] = entry->next;
4065 DEBUG (5, fprintf (gc_debug_file, "Dislink nullified at %p to GCed object %p\n", p, object));
4067 mono_sgen_free_internal (entry, INTERNAL_MEM_DISLINK);
4072 char *copy = object;
4073 copy_func ((void**)©, queue);
4075 /* Update pointer if it's moved. If the object
4076 * has been moved out of the nursery, we need to
4077 * remove the link from the minor hash table to
4080 * FIXME: what if an object is moved earlier?
4083 if (hash == &minor_disappearing_link_hash && !ptr_in_nursery (copy)) {
4084 void **link = entry->link;
4085 DisappearingLink *old;
4086 /* remove from list */
4088 prev->next = entry->next;
4090 disappearing_link_hash [i] = entry->next;
4092 mono_sgen_free_internal (entry, INTERNAL_MEM_DISLINK);
4096 add_or_remove_disappearing_link ((MonoObject*)copy, link,
4097 track, GENERATION_OLD);
4099 DEBUG (5, fprintf (gc_debug_file, "Upgraded dislink at %p to major because object %p moved to %p\n", link, object, copy));
4103 /* We set the track resurrection bit to
4104 * FALSE if the object is to be finalized
4105 * so that the object can be collected in
4106 * the next cycle (i.e. after it was
4109 *entry->link = HIDE_POINTER (copy,
4110 object_is_fin_ready (object) ? FALSE : track);
4111 DEBUG (5, fprintf (gc_debug_file, "Updated dislink at %p to %p\n", entry->link, DISLINK_OBJECT (entry)));
4116 entry = entry->next;
4121 /* LOCKING: requires that the GC lock is held */
4123 null_links_for_domain (MonoDomain *domain, int generation)
4125 DisappearingLinkHashTable *hash = get_dislink_hash_table (generation);
4126 DisappearingLink **disappearing_link_hash = hash->table;
4127 int disappearing_link_hash_size = hash->size;
4128 DisappearingLink *entry, *prev;
4130 for (i = 0; i < disappearing_link_hash_size; ++i) {
4132 for (entry = disappearing_link_hash [i]; entry; ) {
4133 char *object = DISLINK_OBJECT (entry);
4134 if (object && !((MonoObject*)object)->vtable) {
4135 DisappearingLink *next = entry->next;
4140 disappearing_link_hash [i] = next;
4142 if (*(entry->link)) {
4143 *(entry->link) = NULL;
4144 g_warning ("Disappearing link %p not freed", entry->link);
4146 mono_sgen_free_internal (entry, INTERNAL_MEM_DISLINK);
4153 entry = entry->next;
4158 /* LOCKING: requires that the GC lock is held */
4160 finalizers_for_domain (MonoDomain *domain, MonoObject **out_array, int out_size,
4161 FinalizeEntryHashTable *hash_table)
4163 FinalizeEntry **finalizable_hash = hash_table->table;
4164 mword finalizable_hash_size = hash_table->size;
4165 FinalizeEntry *entry, *prev;
4168 if (no_finalize || !out_size || !out_array)
4171 for (i = 0; i < finalizable_hash_size; ++i) {
4173 for (entry = finalizable_hash [i]; entry;) {
4174 if (mono_object_domain (entry->object) == domain) {
4175 FinalizeEntry *next;
4176 /* remove and put in out_array */
4178 prev->next = entry->next;
4180 finalizable_hash [i] = entry->next;
4182 hash_table->num_registered--;
4183 out_array [count ++] = entry->object;
4184 DEBUG (5, fprintf (gc_debug_file, "Collecting object for finalization: %p (%s) (%d/%d)\n", entry->object, safe_name (entry->object), num_ready_finalizers, hash_table->num_registered));
4186 if (count == out_size)
4191 entry = entry->next;
4198 * mono_gc_finalizers_for_domain:
4199 * @domain: the unloading appdomain
4200 * @out_array: output array
4201 * @out_size: size of output array
4203 * Store inside @out_array up to @out_size objects that belong to the unloading
4204 * appdomain @domain. Returns the number of stored items. Can be called repeteadly
4205 * until it returns 0.
4206 * The items are removed from the finalizer data structure, so the caller is supposed
4208 * @out_array should be on the stack to allow the GC to know the objects are still alive.
4211 mono_gc_finalizers_for_domain (MonoDomain *domain, MonoObject **out_array, int out_size)
4216 result = finalizers_for_domain (domain, out_array, out_size, &minor_finalizable_hash);
4217 if (result < out_size) {
4218 result += finalizers_for_domain (domain, out_array + result, out_size - result,
4219 &major_finalizable_hash);
4227 register_for_finalization (MonoObject *obj, void *user_data, int generation)
4229 FinalizeEntryHashTable *hash_table = get_finalize_entry_hash_table (generation);
4230 FinalizeEntry **finalizable_hash;
4231 mword finalizable_hash_size;
4232 FinalizeEntry *entry, *prev;
4236 g_assert (user_data == NULL || user_data == mono_gc_run_finalize);
4237 hash = mono_object_hash (obj);
4239 rehash_fin_table_if_necessary (hash_table);
4240 finalizable_hash = hash_table->table;
4241 finalizable_hash_size = hash_table->size;
4242 hash %= finalizable_hash_size;
4244 for (entry = finalizable_hash [hash]; entry; entry = entry->next) {
4245 if (entry->object == obj) {
4247 /* remove from the list */
4249 prev->next = entry->next;
4251 finalizable_hash [hash] = entry->next;
4252 hash_table->num_registered--;
4253 DEBUG (5, fprintf (gc_debug_file, "Removed finalizer %p for object: %p (%s) (%d)\n", entry, obj, obj->vtable->klass->name, hash_table->num_registered));
4254 mono_sgen_free_internal (entry, INTERNAL_MEM_FINALIZE_ENTRY);
4262 /* request to deregister, but already out of the list */
4266 entry = mono_sgen_alloc_internal (INTERNAL_MEM_FINALIZE_ENTRY);
4267 entry->object = obj;
4268 entry->next = finalizable_hash [hash];
4269 finalizable_hash [hash] = entry;
4270 hash_table->num_registered++;
4271 DEBUG (5, fprintf (gc_debug_file, "Added finalizer %p for object: %p (%s) (%d) to %s table\n", entry, obj, obj->vtable->klass->name, hash_table->num_registered, generation_name (generation)));
4276 mono_gc_register_for_finalization (MonoObject *obj, void *user_data)
4278 if (ptr_in_nursery (obj))
4279 register_for_finalization (obj, user_data, GENERATION_NURSERY);
4281 register_for_finalization (obj, user_data, GENERATION_OLD);
4285 rehash_dislink (DisappearingLinkHashTable *hash_table)
4287 DisappearingLink **disappearing_link_hash = hash_table->table;
4288 int disappearing_link_hash_size = hash_table->size;
4291 DisappearingLink **new_hash;
4292 DisappearingLink *entry, *next;
4293 int new_size = g_spaced_primes_closest (hash_table->num_links);
4295 new_hash = mono_sgen_alloc_internal_dynamic (new_size * sizeof (DisappearingLink*), INTERNAL_MEM_DISLINK_TABLE);
4296 for (i = 0; i < disappearing_link_hash_size; ++i) {
4297 for (entry = disappearing_link_hash [i]; entry; entry = next) {
4298 hash = mono_aligned_addr_hash (entry->link) % new_size;
4300 entry->next = new_hash [hash];
4301 new_hash [hash] = entry;
4304 mono_sgen_free_internal_dynamic (disappearing_link_hash,
4305 disappearing_link_hash_size * sizeof (DisappearingLink*), INTERNAL_MEM_DISLINK_TABLE);
4306 hash_table->table = new_hash;
4307 hash_table->size = new_size;
4310 /* LOCKING: assumes the GC lock is held */
4312 add_or_remove_disappearing_link (MonoObject *obj, void **link, gboolean track, int generation)
4314 DisappearingLinkHashTable *hash_table = get_dislink_hash_table (generation);
4315 DisappearingLink *entry, *prev;
4317 DisappearingLink **disappearing_link_hash = hash_table->table;
4318 int disappearing_link_hash_size = hash_table->size;
4320 if (hash_table->num_links >= disappearing_link_hash_size * 2) {
4321 rehash_dislink (hash_table);
4322 disappearing_link_hash = hash_table->table;
4323 disappearing_link_hash_size = hash_table->size;
4325 /* FIXME: add check that link is not in the heap */
4326 hash = mono_aligned_addr_hash (link) % disappearing_link_hash_size;
4327 entry = disappearing_link_hash [hash];
4329 for (; entry; entry = entry->next) {
4330 /* link already added */
4331 if (link == entry->link) {
4332 /* NULL obj means remove */
4335 prev->next = entry->next;
4337 disappearing_link_hash [hash] = entry->next;
4338 hash_table->num_links--;
4339 DEBUG (5, fprintf (gc_debug_file, "Removed dislink %p (%d) from %s table\n", entry, hash_table->num_links, generation_name (generation)));
4340 mono_sgen_free_internal (entry, INTERNAL_MEM_DISLINK);
4343 *link = HIDE_POINTER (obj, track); /* we allow the change of object */
4351 entry = mono_sgen_alloc_internal (INTERNAL_MEM_DISLINK);
4352 *link = HIDE_POINTER (obj, track);
4354 entry->next = disappearing_link_hash [hash];
4355 disappearing_link_hash [hash] = entry;
4356 hash_table->num_links++;
4357 DEBUG (5, fprintf (gc_debug_file, "Added dislink %p for object: %p (%s) at %p to %s table\n", entry, obj, obj->vtable->klass->name, link, generation_name (generation)));
4360 /* LOCKING: assumes the GC lock is held */
4362 mono_gc_register_disappearing_link (MonoObject *obj, void **link, gboolean track)
4364 add_or_remove_disappearing_link (NULL, link, FALSE, GENERATION_NURSERY);
4365 add_or_remove_disappearing_link (NULL, link, FALSE, GENERATION_OLD);
4367 if (ptr_in_nursery (obj))
4368 add_or_remove_disappearing_link (obj, link, track, GENERATION_NURSERY);
4370 add_or_remove_disappearing_link (obj, link, track, GENERATION_OLD);
4375 mono_gc_invoke_finalizers (void)
4377 FinalizeEntry *entry = NULL;
4378 gboolean entry_is_critical = FALSE;
4381 /* FIXME: batch to reduce lock contention */
4382 while (fin_ready_list || critical_fin_list) {
4386 FinalizeEntry **list = entry_is_critical ? &critical_fin_list : &fin_ready_list;
4388 /* We have finalized entry in the last
4389 interation, now we need to remove it from
4392 *list = entry->next;
4394 FinalizeEntry *e = *list;
4395 while (e->next != entry)
4397 e->next = entry->next;
4399 mono_sgen_free_internal (entry, INTERNAL_MEM_FINALIZE_ENTRY);
4403 /* Now look for the first non-null entry. */
4404 for (entry = fin_ready_list; entry && !entry->object; entry = entry->next)
4407 entry_is_critical = FALSE;
4409 entry_is_critical = TRUE;
4410 for (entry = critical_fin_list; entry && !entry->object; entry = entry->next)
4415 g_assert (entry->object);
4416 num_ready_finalizers--;
4417 obj = entry->object;
4418 entry->object = NULL;
4419 DEBUG (7, fprintf (gc_debug_file, "Finalizing object %p (%s)\n", obj, safe_name (obj)));
4427 g_assert (entry->object == NULL);
4429 /* the object is on the stack so it is pinned */
4430 /*g_print ("Calling finalizer for object: %p (%s)\n", entry->object, safe_name (entry->object));*/
4431 mono_gc_run_finalize (obj, NULL);
4438 mono_gc_pending_finalizers (void)
4440 return fin_ready_list || critical_fin_list;
4443 /* Negative value to remove */
4445 mono_gc_add_memory_pressure (gint64 value)
4447 /* FIXME: Use interlocked functions */
4449 memory_pressure += value;
4454 mono_sgen_register_major_sections_alloced (int num_sections)
4456 minor_collection_sections_alloced += num_sections;
4460 mono_sgen_get_minor_collection_allowance (void)
4462 return minor_collection_allowance;
4466 * ######################################################################
4467 * ######## registered roots support
4468 * ######################################################################
4472 rehash_roots (gboolean pinned)
4476 RootRecord **new_hash;
4477 RootRecord *entry, *next;
4480 new_size = g_spaced_primes_closest (num_roots_entries [pinned]);
4481 new_hash = mono_sgen_alloc_internal_dynamic (new_size * sizeof (RootRecord*), INTERNAL_MEM_ROOTS_TABLE);
4482 for (i = 0; i < roots_hash_size [pinned]; ++i) {
4483 for (entry = roots_hash [pinned][i]; entry; entry = next) {
4484 hash = mono_aligned_addr_hash (entry->start_root) % new_size;
4486 entry->next = new_hash [hash];
4487 new_hash [hash] = entry;
4490 mono_sgen_free_internal_dynamic (roots_hash [pinned], roots_hash_size [pinned] * sizeof (RootRecord*), INTERNAL_MEM_ROOTS_TABLE);
4491 roots_hash [pinned] = new_hash;
4492 roots_hash_size [pinned] = new_size;
4496 find_root (int root_type, char *start, guint32 addr_hash)
4498 RootRecord *new_root;
4500 guint32 hash = addr_hash % roots_hash_size [root_type];
4501 for (new_root = roots_hash [root_type][hash]; new_root; new_root = new_root->next) {
4502 /* we allow changing the size and the descriptor (for thread statics etc) */
4503 if (new_root->start_root == start) {
4512 * We do not coalesce roots.
4515 mono_gc_register_root_inner (char *start, size_t size, void *descr, int root_type)
4517 RootRecord *new_root;
4518 unsigned int hash, addr_hash = mono_aligned_addr_hash (start);
4521 for (i = 0; i < ROOT_TYPE_NUM; ++i) {
4522 if (num_roots_entries [i] >= roots_hash_size [i] * 2)
4525 for (i = 0; i < ROOT_TYPE_NUM; ++i) {
4526 new_root = find_root (i, start, addr_hash);
4527 /* we allow changing the size and the descriptor (for thread statics etc) */
4529 size_t old_size = new_root->end_root - new_root->start_root;
4530 new_root->end_root = new_root->start_root + size;
4531 g_assert (((new_root->root_desc != 0) && (descr != NULL)) ||
4532 ((new_root->root_desc == 0) && (descr == NULL)));
4533 new_root->root_desc = (mword)descr;
4535 roots_size -= old_size;
4540 new_root = mono_sgen_alloc_internal (INTERNAL_MEM_ROOT_RECORD);
4542 new_root->start_root = start;
4543 new_root->end_root = new_root->start_root + size;
4544 new_root->root_desc = (mword)descr;
4546 hash = addr_hash % roots_hash_size [root_type];
4547 num_roots_entries [root_type]++;
4548 new_root->next = roots_hash [root_type] [hash];
4549 roots_hash [root_type][hash] = new_root;
4550 DEBUG (3, fprintf (gc_debug_file, "Added root %p for range: %p-%p, descr: %p (%d/%d bytes)\n", new_root, new_root->start_root, new_root->end_root, descr, (int)size, (int)roots_size));
4560 mono_gc_register_root (char *start, size_t size, void *descr)
4562 return mono_gc_register_root_inner (start, size, descr, descr ? ROOT_TYPE_NORMAL : ROOT_TYPE_PINNED);
4566 mono_gc_register_root_wbarrier (char *start, size_t size, void *descr)
4568 return mono_gc_register_root_inner (start, size, descr, ROOT_TYPE_WBARRIER);
4572 mono_gc_deregister_root (char* addr)
4574 RootRecord *tmp, *prev;
4575 unsigned int hash, addr_hash = mono_aligned_addr_hash (addr);
4579 for (root_type = 0; root_type < ROOT_TYPE_NUM; ++root_type) {
4580 hash = addr_hash % roots_hash_size [root_type];
4581 tmp = roots_hash [root_type][hash];
4584 if (tmp->start_root == (char*)addr) {
4586 prev->next = tmp->next;
4588 roots_hash [root_type][hash] = tmp->next;
4589 roots_size -= (tmp->end_root - tmp->start_root);
4590 num_roots_entries [root_type]--;
4591 DEBUG (3, fprintf (gc_debug_file, "Removed root %p for range: %p-%p\n", tmp, tmp->start_root, tmp->end_root));
4592 mono_sgen_free_internal (tmp, INTERNAL_MEM_ROOT_RECORD);
4603 * ######################################################################
4604 * ######## Thread handling (stop/start code)
4605 * ######################################################################
4608 /* FIXME: handle large/small config */
4609 #define HASH_PTHREAD_T(id) (((unsigned int)(id) >> 4) * 2654435761u)
4611 static SgenThreadInfo* thread_table [THREAD_HASH_SIZE];
4613 #if USE_SIGNAL_BASED_START_STOP_WORLD
4615 static MonoSemType suspend_ack_semaphore;
4616 static MonoSemType *suspend_ack_semaphore_ptr;
4617 static unsigned int global_stop_count = 0;
4619 static sigset_t suspend_signal_mask;
4620 static mword cur_thread_regs [ARCH_NUM_REGS] = {0};
4622 /* LOCKING: assumes the GC lock is held */
4624 mono_sgen_get_thread_table (void)
4626 return thread_table;
4630 mono_sgen_thread_info_lookup (ARCH_THREAD_TYPE id)
4632 unsigned int hash = HASH_PTHREAD_T (id) % THREAD_HASH_SIZE;
4633 SgenThreadInfo *info;
4635 info = thread_table [hash];
4636 while (info && !ARCH_THREAD_EQUALS (info->id, id)) {
4643 update_current_thread_stack (void *start)
4645 void *ptr = cur_thread_regs;
4646 SgenThreadInfo *info = mono_sgen_thread_info_lookup (ARCH_GET_THREAD ());
4648 info->stack_start = align_pointer (&ptr);
4649 g_assert (info->stack_start >= info->stack_start_limit && info->stack_start < info->stack_end);
4650 ARCH_STORE_REGS (ptr);
4651 info->stopped_regs = ptr;
4652 if (gc_callbacks.thread_suspend_func)
4653 gc_callbacks.thread_suspend_func (info->runtime_data, NULL);
4657 * Define this and use the "xdomain-checks" MONO_GC_DEBUG option to
4658 * have cross-domain checks in the write barrier.
4660 //#define XDOMAIN_CHECKS_IN_WBARRIER
4662 #ifndef SGEN_BINARY_PROTOCOL
4663 #ifndef HEAVY_STATISTICS
4664 #define MANAGED_ALLOCATION
4665 #ifndef XDOMAIN_CHECKS_IN_WBARRIER
4666 #define MANAGED_WBARRIER
4672 is_ip_in_managed_allocator (MonoDomain *domain, gpointer ip);
4675 mono_sgen_wait_for_suspend_ack (int count)
4679 for (i = 0; i < count; ++i) {
4680 while ((result = MONO_SEM_WAIT (suspend_ack_semaphore_ptr)) != 0) {
4681 if (errno != EINTR) {
4682 g_error ("sem_wait ()");
4689 restart_threads_until_none_in_managed_allocator (void)
4691 SgenThreadInfo *info;
4692 int i, result, num_threads_died = 0;
4693 int sleep_duration = -1;
4696 int restart_count = 0, restarted_count = 0;
4697 /* restart all threads that stopped in the
4699 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
4700 for (info = thread_table [i]; info; info = info->next) {
4703 if (!info->stack_start || info->in_critical_region ||
4704 is_ip_in_managed_allocator (info->stopped_domain, info->stopped_ip)) {
4705 binary_protocol_thread_restart ((gpointer)info->id);
4706 #if defined(__MACH__) && MONO_MACH_ARCH_SUPPORTED
4707 result = thread_resume (pthread_mach_thread_np (info->id));
4709 result = pthread_kill (info->id, restart_signal_num);
4717 /* we set the stopped_ip to
4718 NULL for threads which
4719 we're not restarting so
4720 that we can easily identify
4722 info->stopped_ip = NULL;
4723 info->stopped_domain = NULL;
4727 /* if no threads were restarted, we're done */
4728 if (restart_count == 0)
4731 #if defined(__MACH__) && MONO_MACH_ARCH_SUPPORTED
4732 /* mach thread_resume is synchronous so we dont need to wait for them */
4734 /* wait for the threads to signal their restart */
4735 mono_sgen_wait_for_suspend_ack (restart_count);
4738 if (sleep_duration < 0) {
4742 g_usleep (sleep_duration);
4743 sleep_duration += 10;
4746 /* stop them again */
4747 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
4748 for (info = thread_table [i]; info; info = info->next) {
4749 if (info->skip || info->stopped_ip == NULL)
4751 #if defined(__MACH__) && MONO_MACH_ARCH_SUPPORTED
4752 result = thread_suspend (pthread_mach_thread_np (info->id));
4754 result = pthread_kill (info->id, suspend_signal_num);
4763 /* some threads might have died */
4764 num_threads_died += restart_count - restarted_count;
4765 #if defined(__MACH__) && MONO_MACH_ARCH_SUPPORTED
4766 /* mach thread_resume is synchronous so we dont need to wait for them */
4768 /* wait for the threads to signal their suspension
4770 mono_sgen_wait_for_suspend_ack (restart_count);
4774 return num_threads_died;
4777 /* LOCKING: assumes the GC lock is held (by the stopping thread) */
4779 suspend_handler (int sig, siginfo_t *siginfo, void *context)
4781 SgenThreadInfo *info;
4784 int old_errno = errno;
4785 gpointer regs [ARCH_NUM_REGS];
4786 gpointer stack_start;
4788 id = pthread_self ();
4789 info = mono_sgen_thread_info_lookup (id);
4790 info->stopped_domain = mono_domain_get ();
4791 info->stopped_ip = (gpointer) ARCH_SIGCTX_IP (context);
4792 stop_count = global_stop_count;
4793 /* duplicate signal */
4794 if (0 && info->stop_count == stop_count) {
4798 #ifdef HAVE_KW_THREAD
4799 /* update the remset info in the thread data structure */
4800 info->remset = remembered_set;
4802 stack_start = (char*) ARCH_SIGCTX_SP (context) - REDZONE_SIZE;
4803 /* If stack_start is not within the limits, then don't set it
4804 in info and we will be restarted. */
4805 if (stack_start >= info->stack_start_limit && info->stack_start <= info->stack_end) {
4806 info->stack_start = stack_start;
4808 ARCH_COPY_SIGCTX_REGS (regs, context);
4809 info->stopped_regs = regs;
4811 g_assert (!info->stack_start);
4814 /* Notify the JIT */
4815 if (gc_callbacks.thread_suspend_func)
4816 gc_callbacks.thread_suspend_func (info->runtime_data, context);
4818 DEBUG (4, fprintf (gc_debug_file, "Posting suspend_ack_semaphore for suspend from %p %p\n", info, (gpointer)ARCH_GET_THREAD ()));
4819 /* notify the waiting thread */
4820 MONO_SEM_POST (suspend_ack_semaphore_ptr);
4821 info->stop_count = stop_count;
4823 /* wait until we receive the restart signal */
4826 sigsuspend (&suspend_signal_mask);
4827 } while (info->signal != restart_signal_num);
4829 DEBUG (4, fprintf (gc_debug_file, "Posting suspend_ack_semaphore for resume from %p %p\n", info, (gpointer)ARCH_GET_THREAD ()));
4830 /* notify the waiting thread */
4831 MONO_SEM_POST (suspend_ack_semaphore_ptr);
4837 restart_handler (int sig)
4839 SgenThreadInfo *info;
4840 int old_errno = errno;
4842 info = mono_sgen_thread_info_lookup (pthread_self ());
4843 info->signal = restart_signal_num;
4844 DEBUG (4, fprintf (gc_debug_file, "Restart handler in %p %p\n", info, (gpointer)ARCH_GET_THREAD ()));
4850 acquire_gc_locks (void)
4856 release_gc_locks (void)
4858 UNLOCK_INTERRUPTION;
4861 static TV_DECLARE (stop_world_time);
4862 static unsigned long max_pause_usec = 0;
4864 /* LOCKING: assumes the GC lock is held */
4866 stop_world (int generation)
4870 mono_profiler_gc_event (MONO_GC_EVENT_PRE_STOP_WORLD, generation);
4871 acquire_gc_locks ();
4873 update_current_thread_stack (&count);
4875 global_stop_count++;
4876 DEBUG (3, fprintf (gc_debug_file, "stopping world n %d from %p %p\n", global_stop_count, mono_sgen_thread_info_lookup (ARCH_GET_THREAD ()), (gpointer)ARCH_GET_THREAD ()));
4877 TV_GETTIME (stop_world_time);
4878 count = mono_sgen_thread_handshake (suspend_signal_num);
4879 count -= restart_threads_until_none_in_managed_allocator ();
4880 g_assert (count >= 0);
4881 DEBUG (3, fprintf (gc_debug_file, "world stopped %d thread(s)\n", count));
4882 mono_profiler_gc_event (MONO_GC_EVENT_POST_STOP_WORLD, generation);
4886 /* LOCKING: assumes the GC lock is held */
4888 restart_world (int generation)
4891 SgenThreadInfo *info;
4892 TV_DECLARE (end_sw);
4895 /* notify the profiler of the leftovers */
4896 if (G_UNLIKELY (mono_profiler_events & MONO_PROFILE_GC_MOVES)) {
4897 if (moved_objects_idx) {
4898 mono_profiler_gc_moves (moved_objects, moved_objects_idx);
4899 moved_objects_idx = 0;
4902 mono_profiler_gc_event (MONO_GC_EVENT_PRE_START_WORLD, generation);
4903 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
4904 for (info = thread_table [i]; info; info = info->next) {
4905 info->stack_start = NULL;
4906 info->stopped_regs = NULL;
4910 release_gc_locks ();
4912 count = mono_sgen_thread_handshake (restart_signal_num);
4913 TV_GETTIME (end_sw);
4914 usec = TV_ELAPSED (stop_world_time, end_sw);
4915 max_pause_usec = MAX (usec, max_pause_usec);
4916 DEBUG (2, fprintf (gc_debug_file, "restarted %d thread(s) (pause time: %d usec, max: %d)\n", count, (int)usec, (int)max_pause_usec));
4917 mono_profiler_gc_event (MONO_GC_EVENT_POST_START_WORLD, generation);
4921 #endif /* USE_SIGNAL_BASED_START_STOP_WORLD */
4924 mono_sgen_get_current_collection_generation (void)
4926 return current_collection_generation;
4930 mono_gc_set_gc_callbacks (MonoGCCallbacks *callbacks)
4932 gc_callbacks = *callbacks;
4936 mono_gc_get_gc_callbacks ()
4938 return &gc_callbacks;
4941 /* Variables holding start/end nursery so it won't have to be passed at every call */
4942 static void *scan_area_arg_start, *scan_area_arg_end;
4945 mono_gc_conservatively_scan_area (void *start, void *end)
4947 conservatively_pin_objects_from (start, end, scan_area_arg_start, scan_area_arg_end, PIN_TYPE_STACK);
4951 mono_gc_scan_object (void *obj)
4953 g_assert_not_reached ();
4954 if (current_collection_generation == GENERATION_NURSERY)
4955 major_collector.copy_object (&obj, &gray_queue);
4957 major_collector.copy_or_mark_object (&obj, &gray_queue);
4962 * Mark from thread stacks and registers.
4965 scan_thread_data (void *start_nursery, void *end_nursery, gboolean precise)
4968 SgenThreadInfo *info;
4970 scan_area_arg_start = start_nursery;
4971 scan_area_arg_end = end_nursery;
4973 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
4974 for (info = thread_table [i]; info; info = info->next) {
4976 DEBUG (3, fprintf (gc_debug_file, "Skipping dead thread %p, range: %p-%p, size: %td\n", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start));
4979 DEBUG (3, fprintf (gc_debug_file, "Scanning thread %p, range: %p-%p, size: %td, pinned=%d\n", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start, next_pin_slot));
4980 if (gc_callbacks.thread_mark_func && !conservative_stack_mark)
4981 gc_callbacks.thread_mark_func (info->runtime_data, info->stack_start, info->stack_end, precise);
4983 conservatively_pin_objects_from (info->stack_start, info->stack_end, start_nursery, end_nursery, PIN_TYPE_STACK);
4986 conservatively_pin_objects_from (info->stopped_regs, info->stopped_regs + ARCH_NUM_REGS,
4987 start_nursery, end_nursery, PIN_TYPE_STACK);
4993 find_pinning_ref_from_thread (char *obj, size_t size)
4996 SgenThreadInfo *info;
4997 char *endobj = obj + size;
4999 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5000 for (info = thread_table [i]; info; info = info->next) {
5001 char **start = (char**)info->stack_start;
5004 while (start < (char**)info->stack_end) {
5005 if (*start >= obj && *start < endobj) {
5006 DEBUG (0, fprintf (gc_debug_file, "Object %p referenced in thread %p (id %p) at %p, stack: %p-%p\n", obj, info, (gpointer)info->id, start, info->stack_start, info->stack_end));
5011 /* FIXME: check info->stopped_regs */
5017 ptr_on_stack (void *ptr)
5019 gpointer stack_start = &stack_start;
5020 SgenThreadInfo *info = mono_sgen_thread_info_lookup (ARCH_GET_THREAD ());
5022 if (ptr >= stack_start && ptr < (gpointer)info->stack_end)
5028 handle_remset (mword *p, void *start_nursery, void *end_nursery, gboolean global, GrayQueue *queue)
5035 HEAVY_STAT (++stat_global_remsets_processed);
5037 HEAVY_STAT (++stat_local_remsets_processed);
5039 /* FIXME: exclude stack locations */
5040 switch ((*p) & REMSET_TYPE_MASK) {
5041 case REMSET_LOCATION:
5043 //__builtin_prefetch (ptr);
5044 if (((void*)ptr < start_nursery || (void*)ptr >= end_nursery)) {
5045 gpointer old = *ptr;
5046 major_collector.copy_object (ptr, queue);
5047 DEBUG (9, fprintf (gc_debug_file, "Overwrote remset at %p with %p\n", ptr, *ptr));
5049 binary_protocol_ptr_update (ptr, old, *ptr, (gpointer)LOAD_VTABLE (*ptr), safe_object_get_size (*ptr));
5050 if (!global && *ptr >= start_nursery && *ptr < end_nursery) {
5052 * If the object is pinned, each reference to it from nonpinned objects
5053 * becomes part of the global remset, which can grow very large.
5055 DEBUG (9, fprintf (gc_debug_file, "Add to global remset because of pinning %p (%p %s)\n", ptr, *ptr, safe_name (*ptr)));
5056 mono_sgen_add_to_global_remset (ptr);
5059 DEBUG (9, fprintf (gc_debug_file, "Skipping remset at %p holding %p\n", ptr, *ptr));
5063 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
5064 if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery))
5067 while (count-- > 0) {
5068 major_collector.copy_object (ptr, queue);
5069 DEBUG (9, fprintf (gc_debug_file, "Overwrote remset at %p with %p (count: %d)\n", ptr, *ptr, (int)count));
5070 if (!global && *ptr >= start_nursery && *ptr < end_nursery)
5071 mono_sgen_add_to_global_remset (ptr);
5076 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
5077 if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery))
5079 major_collector.minor_scan_object ((char*)ptr, queue);
5081 case REMSET_VTYPE: {
5082 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
5083 if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery))
5088 ptr = (void**) major_collector.minor_scan_vtype ((char*)ptr, desc, start_nursery, end_nursery, queue);
5092 g_assert_not_reached ();
5097 #ifdef HEAVY_STATISTICS
5099 collect_store_remsets (RememberedSet *remset, mword *bumper)
5101 mword *p = remset->data;
5106 while (p < remset->store_next) {
5107 switch ((*p) & REMSET_TYPE_MASK) {
5108 case REMSET_LOCATION:
5111 ++stat_saved_remsets_1;
5113 if (*p == last1 || *p == last2) {
5114 ++stat_saved_remsets_2;
5131 g_assert_not_reached ();
5141 RememberedSet *remset;
5143 SgenThreadInfo *info;
5145 mword *addresses, *bumper, *p, *r;
5147 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5148 for (info = thread_table [i]; info; info = info->next) {
5149 for (remset = info->remset; remset; remset = remset->next)
5150 size += remset->store_next - remset->data;
5153 for (remset = freed_thread_remsets; remset; remset = remset->next)
5154 size += remset->store_next - remset->data;
5155 for (remset = global_remset; remset; remset = remset->next)
5156 size += remset->store_next - remset->data;
5158 bumper = addresses = mono_sgen_alloc_internal_dynamic (sizeof (mword) * size, INTERNAL_MEM_STATISTICS);
5160 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5161 for (info = thread_table [i]; info; info = info->next) {
5162 for (remset = info->remset; remset; remset = remset->next)
5163 bumper = collect_store_remsets (remset, bumper);
5166 for (remset = global_remset; remset; remset = remset->next)
5167 bumper = collect_store_remsets (remset, bumper);
5168 for (remset = freed_thread_remsets; remset; remset = remset->next)
5169 bumper = collect_store_remsets (remset, bumper);
5171 g_assert (bumper <= addresses + size);
5173 stat_store_remsets += bumper - addresses;
5175 sort_addresses ((void**)addresses, bumper - addresses);
5178 while (r < bumper) {
5184 stat_store_remsets_unique += p - addresses;
5186 mono_sgen_free_internal_dynamic (addresses, sizeof (mword) * size, INTERNAL_MEM_STATISTICS);
5191 clear_thread_store_remset_buffer (SgenThreadInfo *info)
5193 *info->store_remset_buffer_index_addr = 0;
5194 memset (*info->store_remset_buffer_addr, 0, sizeof (gpointer) * STORE_REMSET_BUFFER_SIZE);
5198 remset_byte_size (RememberedSet *remset)
5200 return sizeof (RememberedSet) + (remset->end_set - remset->data) * sizeof (gpointer);
5204 scan_from_remsets (void *start_nursery, void *end_nursery, GrayQueue *queue)
5207 SgenThreadInfo *info;
5208 RememberedSet *remset;
5209 GenericStoreRememberedSet *store_remset;
5210 mword *p, *next_p, *store_pos;
5212 #ifdef HEAVY_STATISTICS
5216 /* the global one */
5217 for (remset = global_remset; remset; remset = remset->next) {
5218 DEBUG (4, fprintf (gc_debug_file, "Scanning global remset range: %p-%p, size: %td\n", remset->data, remset->store_next, remset->store_next - remset->data));
5219 store_pos = remset->data;
5220 for (p = remset->data; p < remset->store_next; p = next_p) {
5221 void **ptr = (void**)p [0];
5223 /*Ignore previously processed remset.*/
5224 if (!global_remset_location_was_not_added (ptr)) {
5229 next_p = handle_remset (p, start_nursery, end_nursery, TRUE, queue);
5232 * Clear global remsets of locations which no longer point to the
5233 * nursery. Otherwise, they could grow indefinitely between major
5236 * Since all global remsets are location remsets, we don't need to unmask the pointer.
5238 if (ptr_in_nursery (*ptr)) {
5239 *store_pos ++ = p [0];
5240 HEAVY_STAT (++stat_global_remsets_readded);
5244 /* Truncate the remset */
5245 remset->store_next = store_pos;
5248 /* the generic store ones */
5249 store_remset = generic_store_remsets;
5250 while (store_remset) {
5251 GenericStoreRememberedSet *next = store_remset->next;
5253 for (i = 0; i < STORE_REMSET_BUFFER_SIZE - 1; ++i) {
5254 gpointer addr = store_remset->data [i];
5256 handle_remset ((mword*)&addr, start_nursery, end_nursery, FALSE, queue);
5259 mono_sgen_free_internal (store_remset, INTERNAL_MEM_STORE_REMSET);
5261 store_remset = next;
5263 generic_store_remsets = NULL;
5265 /* the per-thread ones */
5266 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5267 for (info = thread_table [i]; info; info = info->next) {
5268 RememberedSet *next;
5270 for (remset = info->remset; remset; remset = next) {
5271 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for thread %p, range: %p-%p, size: %td\n", info, remset->data, remset->store_next, remset->store_next - remset->data));
5272 for (p = remset->data; p < remset->store_next;)
5273 p = handle_remset (p, start_nursery, end_nursery, FALSE, queue);
5274 remset->store_next = remset->data;
5275 next = remset->next;
5276 remset->next = NULL;
5277 if (remset != info->remset) {
5278 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
5279 mono_sgen_free_internal_dynamic (remset, remset_byte_size (remset), INTERNAL_MEM_REMSET);
5282 for (j = 0; j < *info->store_remset_buffer_index_addr; ++j)
5283 handle_remset ((mword*)*info->store_remset_buffer_addr + j + 1, start_nursery, end_nursery, FALSE, queue);
5284 clear_thread_store_remset_buffer (info);
5288 /* the freed thread ones */
5289 while (freed_thread_remsets) {
5290 RememberedSet *next;
5291 remset = freed_thread_remsets;
5292 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for freed thread, range: %p-%p, size: %td\n", remset->data, remset->store_next, remset->store_next - remset->data));
5293 for (p = remset->data; p < remset->store_next;)
5294 p = handle_remset (p, start_nursery, end_nursery, FALSE, queue);
5295 next = remset->next;
5296 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
5297 mono_sgen_free_internal_dynamic (remset, remset_byte_size (remset), INTERNAL_MEM_REMSET);
5298 freed_thread_remsets = next;
5303 * Clear the info in the remembered sets: we're doing a major collection, so
5304 * the per-thread ones are not needed and the global ones will be reconstructed
5308 clear_remsets (void)
5311 SgenThreadInfo *info;
5312 RememberedSet *remset, *next;
5314 /* the global list */
5315 for (remset = global_remset; remset; remset = next) {
5316 remset->store_next = remset->data;
5317 next = remset->next;
5318 remset->next = NULL;
5319 if (remset != global_remset) {
5320 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
5321 mono_sgen_free_internal_dynamic (remset, remset_byte_size (remset), INTERNAL_MEM_REMSET);
5324 /* the generic store ones */
5325 while (generic_store_remsets) {
5326 GenericStoreRememberedSet *gs_next = generic_store_remsets->next;
5327 mono_sgen_free_internal (generic_store_remsets, INTERNAL_MEM_STORE_REMSET);
5328 generic_store_remsets = gs_next;
5330 /* the per-thread ones */
5331 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5332 for (info = thread_table [i]; info; info = info->next) {
5333 for (remset = info->remset; remset; remset = next) {
5334 remset->store_next = remset->data;
5335 next = remset->next;
5336 remset->next = NULL;
5337 if (remset != info->remset) {
5338 DEBUG (3, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
5339 mono_sgen_free_internal_dynamic (remset, remset_byte_size (remset), INTERNAL_MEM_REMSET);
5342 clear_thread_store_remset_buffer (info);
5346 /* the freed thread ones */
5347 while (freed_thread_remsets) {
5348 next = freed_thread_remsets->next;
5349 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", freed_thread_remsets->data));
5350 mono_sgen_free_internal_dynamic (freed_thread_remsets, remset_byte_size (freed_thread_remsets), INTERNAL_MEM_REMSET);
5351 freed_thread_remsets = next;
5356 * Clear the thread local TLAB variables for all threads.
5361 SgenThreadInfo *info;
5364 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5365 for (info = thread_table [i]; info; info = info->next) {
5366 /* A new TLAB will be allocated when the thread does its first allocation */
5367 *info->tlab_start_addr = NULL;
5368 *info->tlab_next_addr = NULL;
5369 *info->tlab_temp_end_addr = NULL;
5370 *info->tlab_real_end_addr = NULL;
5375 /* LOCKING: assumes the GC lock is held */
5376 static SgenThreadInfo*
5377 gc_register_current_thread (void *addr)
5380 SgenThreadInfo* info = malloc (sizeof (SgenThreadInfo));
5381 #ifndef HAVE_KW_THREAD
5382 SgenThreadInfo *__thread_info__ = info;
5388 memset (info, 0, sizeof (SgenThreadInfo));
5389 #ifndef HAVE_KW_THREAD
5390 info->tlab_start = info->tlab_next = info->tlab_temp_end = info->tlab_real_end = NULL;
5392 g_assert (!pthread_getspecific (thread_info_key));
5393 pthread_setspecific (thread_info_key, info);
5398 info->id = ARCH_GET_THREAD ();
5399 info->stop_count = -1;
5402 info->stack_start = NULL;
5403 info->tlab_start_addr = &TLAB_START;
5404 info->tlab_next_addr = &TLAB_NEXT;
5405 info->tlab_temp_end_addr = &TLAB_TEMP_END;
5406 info->tlab_real_end_addr = &TLAB_REAL_END;
5407 info->store_remset_buffer_addr = &STORE_REMSET_BUFFER;
5408 info->store_remset_buffer_index_addr = &STORE_REMSET_BUFFER_INDEX;
5409 info->stopped_ip = NULL;
5410 info->stopped_domain = NULL;
5411 info->stopped_regs = NULL;
5413 binary_protocol_thread_register ((gpointer)info->id);
5415 #ifdef HAVE_KW_THREAD
5416 tlab_next_addr = &tlab_next;
5417 store_remset_buffer_index_addr = &store_remset_buffer_index;
5420 /* try to get it with attributes first */
5421 #if defined(HAVE_PTHREAD_GETATTR_NP) && defined(HAVE_PTHREAD_ATTR_GETSTACK)
5425 pthread_attr_t attr;
5426 pthread_getattr_np (pthread_self (), &attr);
5427 pthread_attr_getstack (&attr, &sstart, &size);
5428 info->stack_start_limit = sstart;
5429 info->stack_end = (char*)sstart + size;
5430 pthread_attr_destroy (&attr);
5432 #elif defined(HAVE_PTHREAD_GET_STACKSIZE_NP) && defined(HAVE_PTHREAD_GET_STACKADDR_NP)
5433 info->stack_end = (char*)pthread_get_stackaddr_np (pthread_self ());
5434 info->stack_start_limit = (char*)info->stack_end - pthread_get_stacksize_np (pthread_self ());
5437 /* FIXME: we assume the stack grows down */
5438 gsize stack_bottom = (gsize)addr;
5439 stack_bottom += 4095;
5440 stack_bottom &= ~4095;
5441 info->stack_end = (char*)stack_bottom;
5445 #ifdef HAVE_KW_THREAD
5446 stack_end = info->stack_end;
5449 /* hash into the table */
5450 hash = HASH_PTHREAD_T (info->id) % THREAD_HASH_SIZE;
5451 info->next = thread_table [hash];
5452 thread_table [hash] = info;
5454 info->remset = alloc_remset (DEFAULT_REMSET_SIZE, info);
5455 pthread_setspecific (remembered_set_key, info->remset);
5456 #ifdef HAVE_KW_THREAD
5457 remembered_set = info->remset;
5460 STORE_REMSET_BUFFER = mono_sgen_alloc_internal (INTERNAL_MEM_STORE_REMSET);
5461 STORE_REMSET_BUFFER_INDEX = 0;
5463 DEBUG (3, fprintf (gc_debug_file, "registered thread %p (%p) (hash: %d)\n", info, (gpointer)info->id, hash));
5465 if (gc_callbacks.thread_attach_func)
5466 info->runtime_data = gc_callbacks.thread_attach_func ();
5472 add_generic_store_remset_from_buffer (gpointer *buffer)
5474 GenericStoreRememberedSet *remset = mono_sgen_alloc_internal (INTERNAL_MEM_STORE_REMSET);
5475 memcpy (remset->data, buffer + 1, sizeof (gpointer) * (STORE_REMSET_BUFFER_SIZE - 1));
5476 remset->next = generic_store_remsets;
5477 generic_store_remsets = remset;
5481 unregister_current_thread (void)
5484 SgenThreadInfo *prev = NULL;
5486 RememberedSet *rset;
5487 ARCH_THREAD_TYPE id = ARCH_GET_THREAD ();
5489 binary_protocol_thread_unregister ((gpointer)id);
5491 hash = HASH_PTHREAD_T (id) % THREAD_HASH_SIZE;
5492 p = thread_table [hash];
5494 DEBUG (3, fprintf (gc_debug_file, "unregister thread %p (%p)\n", p, (gpointer)p->id));
5495 while (!ARCH_THREAD_EQUALS (p->id, id)) {
5500 thread_table [hash] = p->next;
5502 prev->next = p->next;
5505 if (freed_thread_remsets) {
5506 for (rset = p->remset; rset->next; rset = rset->next)
5508 rset->next = freed_thread_remsets;
5509 freed_thread_remsets = p->remset;
5511 freed_thread_remsets = p->remset;
5514 if (*p->store_remset_buffer_index_addr)
5515 add_generic_store_remset_from_buffer (*p->store_remset_buffer_addr);
5516 mono_sgen_free_internal (*p->store_remset_buffer_addr, INTERNAL_MEM_STORE_REMSET);
5521 unregister_thread (void *k)
5523 g_assert (!mono_domain_get ());
5525 unregister_current_thread ();
5530 mono_gc_register_thread (void *baseptr)
5532 SgenThreadInfo *info;
5536 info = mono_sgen_thread_info_lookup (ARCH_GET_THREAD ());
5538 info = gc_register_current_thread (baseptr);
5541 /* Need a better place to initialize this */
5542 if (!array_fill_vtable && mono_get_root_domain ()) {
5543 array_fill_vtable = mono_class_vtable (mono_get_root_domain (), mono_array_class_get (mono_defaults.byte_class, 1));
5546 return info != NULL;
5549 #if USE_PTHREAD_INTERCEPT
5552 void *(*start_routine) (void *);
5555 MonoSemType registered;
5556 } SgenThreadStartInfo;
5559 gc_start_thread (void *arg)
5561 SgenThreadStartInfo *start_info = arg;
5562 SgenThreadInfo* info;
5563 void *t_arg = start_info->arg;
5564 void *(*start_func) (void*) = start_info->start_routine;
5569 info = gc_register_current_thread (&result);
5571 post_result = MONO_SEM_POST (&(start_info->registered));
5572 g_assert (!post_result);
5573 result = start_func (t_arg);
5574 g_assert (!mono_domain_get ());
5576 * this is done by the pthread key dtor
5578 unregister_current_thread ();
5586 mono_gc_pthread_create (pthread_t *new_thread, const pthread_attr_t *attr, void *(*start_routine)(void *), void *arg)
5588 SgenThreadStartInfo *start_info;
5591 start_info = malloc (sizeof (SgenThreadStartInfo));
5594 MONO_SEM_INIT (&(start_info->registered), 0);
5595 start_info->arg = arg;
5596 start_info->start_routine = start_routine;
5598 result = pthread_create (new_thread, attr, gc_start_thread, start_info);
5600 while (MONO_SEM_WAIT (&(start_info->registered)) != 0) {
5601 /*if (EINTR != errno) ABORT("sem_wait failed"); */
5604 MONO_SEM_DESTROY (&(start_info->registered));
5610 mono_gc_pthread_join (pthread_t thread, void **retval)
5612 return pthread_join (thread, retval);
5616 mono_gc_pthread_detach (pthread_t thread)
5618 return pthread_detach (thread);
5621 #endif /* USE_PTHREAD_INTERCEPT */
5624 * ######################################################################
5625 * ######## Write barriers
5626 * ######################################################################
5630 * This causes the compile to extend the liveness of 'v' till the call to dummy_use
5633 dummy_use (gpointer v) {
5634 __asm__ volatile ("" : "=r"(v) : "r"(v));
5638 static RememberedSet*
5639 alloc_remset (int size, gpointer id) {
5640 RememberedSet* res = mono_sgen_alloc_internal_dynamic (sizeof (RememberedSet) + (size * sizeof (gpointer)), INTERNAL_MEM_REMSET);
5641 res->store_next = res->data;
5642 res->end_set = res->data + size;
5644 DEBUG (4, fprintf (gc_debug_file, "Allocated remset size %d at %p for %p\n", size, res->data, id));
5649 * Note: the write barriers first do the needed GC work and then do the actual store:
5650 * this way the value is visible to the conservative GC scan after the write barrier
5651 * itself. If a GC interrupts the barrier in the middle, value will be kept alive by
5652 * the conservative scan, otherwise by the remembered set scan.
5655 mono_gc_wbarrier_set_field (MonoObject *obj, gpointer field_ptr, MonoObject* value)
5657 HEAVY_STAT (++stat_wbarrier_set_field);
5658 if (ptr_in_nursery (field_ptr)) {
5659 *(void**)field_ptr = value;
5662 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", field_ptr));
5663 if (use_cardtable) {
5664 *(void**)field_ptr = value;
5665 if (ptr_in_nursery (value))
5666 sgen_card_table_mark_address ((mword)field_ptr);
5673 rs = REMEMBERED_SET;
5674 if (rs->store_next < rs->end_set) {
5675 *(rs->store_next++) = (mword)field_ptr;
5676 *(void**)field_ptr = value;
5680 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
5681 rs->next = REMEMBERED_SET;
5682 REMEMBERED_SET = rs;
5683 #ifdef HAVE_KW_THREAD
5684 mono_sgen_thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
5686 *(rs->store_next++) = (mword)field_ptr;
5687 *(void**)field_ptr = value;
5693 mono_gc_wbarrier_set_arrayref (MonoArray *arr, gpointer slot_ptr, MonoObject* value)
5695 HEAVY_STAT (++stat_wbarrier_set_arrayref);
5696 if (ptr_in_nursery (slot_ptr)) {
5697 *(void**)slot_ptr = value;
5700 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", slot_ptr));
5701 if (use_cardtable) {
5702 *(void**)slot_ptr = value;
5703 if (ptr_in_nursery (value))
5704 sgen_card_table_mark_address ((mword)slot_ptr);
5711 rs = REMEMBERED_SET;
5712 if (rs->store_next < rs->end_set) {
5713 *(rs->store_next++) = (mword)slot_ptr;
5714 *(void**)slot_ptr = value;
5718 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
5719 rs->next = REMEMBERED_SET;
5720 REMEMBERED_SET = rs;
5721 #ifdef HAVE_KW_THREAD
5722 mono_sgen_thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
5724 *(rs->store_next++) = (mword)slot_ptr;
5725 *(void**)slot_ptr = value;
5731 mono_gc_wbarrier_arrayref_copy (gpointer dest_ptr, gpointer src_ptr, int count)
5733 HEAVY_STAT (++stat_wbarrier_arrayref_copy);
5734 /*This check can be done without taking a lock since dest_ptr array is pinned*/
5735 if (ptr_in_nursery (dest_ptr) || count <= 0) {
5736 memmove (dest_ptr, src_ptr, count * sizeof (gpointer));
5740 if (use_cardtable) {
5741 gpointer *dest = dest_ptr;
5742 gpointer *src = src_ptr;
5744 /*overlapping that required backward copying*/
5745 if (src < dest && (src + count) > dest) {
5746 gpointer *start = dest;
5750 for (; dest >= start; --src, --dest) {
5751 gpointer value = *src;
5753 if (ptr_in_nursery (value))
5754 sgen_card_table_mark_address ((mword)dest);
5758 gpointer *end = dest + count;
5759 for (; dest < end; ++src, ++dest) {
5760 gpointer value = *src;
5762 if (ptr_in_nursery (value))
5763 sgen_card_table_mark_address ((mword)dest);
5771 memmove (dest_ptr, src_ptr, count * sizeof (gpointer));
5773 rs = REMEMBERED_SET;
5774 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p, %d\n", dest_ptr, count));
5775 if (rs->store_next + 1 < rs->end_set) {
5776 *(rs->store_next++) = (mword)dest_ptr | REMSET_RANGE;
5777 *(rs->store_next++) = count;
5781 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
5782 rs->next = REMEMBERED_SET;
5783 REMEMBERED_SET = rs;
5784 #ifdef HAVE_KW_THREAD
5785 mono_sgen_thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
5787 *(rs->store_next++) = (mword)dest_ptr | REMSET_RANGE;
5788 *(rs->store_next++) = count;
5794 static char *found_obj;
5797 find_object_for_ptr_callback (char *obj, size_t size, char *ptr)
5799 if (ptr >= obj && ptr < obj + size) {
5800 g_assert (!found_obj);
5805 /* for use in the debugger */
5806 char* find_object_for_ptr (char *ptr);
5808 find_object_for_ptr (char *ptr)
5812 if (ptr >= nursery_section->data && ptr < nursery_section->end_data) {
5814 mono_sgen_scan_area_with_callback (nursery_section->data, nursery_section->end_data,
5815 (IterateObjectCallbackFunc)find_object_for_ptr_callback, ptr);
5820 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
5821 if (ptr >= bigobj->data && ptr < bigobj->data + bigobj->size)
5822 return bigobj->data;
5826 * Very inefficient, but this is debugging code, supposed to
5827 * be called from gdb, so we don't care.
5830 major_collector.iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)find_object_for_ptr_callback, ptr);
5835 evacuate_remset_buffer (void)
5840 buffer = STORE_REMSET_BUFFER;
5842 add_generic_store_remset_from_buffer (buffer);
5843 memset (buffer, 0, sizeof (gpointer) * STORE_REMSET_BUFFER_SIZE);
5845 STORE_REMSET_BUFFER_INDEX = 0;
5849 mono_gc_wbarrier_generic_nostore (gpointer ptr)
5855 HEAVY_STAT (++stat_wbarrier_generic_store);
5857 #ifdef XDOMAIN_CHECKS_IN_WBARRIER
5858 /* FIXME: ptr_in_heap must be called with the GC lock held */
5859 if (xdomain_checks && *(MonoObject**)ptr && ptr_in_heap (ptr)) {
5860 char *start = find_object_for_ptr (ptr);
5861 MonoObject *value = *(MonoObject**)ptr;
5865 MonoObject *obj = (MonoObject*)start;
5866 if (obj->vtable->domain != value->vtable->domain)
5867 g_assert (is_xdomain_ref_allowed (ptr, start, obj->vtable->domain));
5873 if (*(gpointer*)ptr)
5874 binary_protocol_wbarrier (ptr, *(gpointer*)ptr, (gpointer)LOAD_VTABLE (*(gpointer*)ptr));
5876 if (ptr_in_nursery (ptr) || ptr_on_stack (ptr) || !ptr_in_nursery (*(gpointer*)ptr)) {
5877 DEBUG (8, fprintf (gc_debug_file, "Skipping remset at %p\n", ptr));
5881 if (use_cardtable) {
5882 if (ptr_in_nursery(*(gpointer*)ptr))
5883 sgen_card_table_mark_address ((mword)ptr);
5889 buffer = STORE_REMSET_BUFFER;
5890 index = STORE_REMSET_BUFFER_INDEX;
5891 /* This simple optimization eliminates a sizable portion of
5892 entries. Comparing it to the last but one entry as well
5893 doesn't eliminate significantly more entries. */
5894 if (buffer [index] == ptr) {
5899 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", ptr));
5900 HEAVY_STAT (++stat_wbarrier_generic_store_remset);
5903 if (index >= STORE_REMSET_BUFFER_SIZE) {
5904 evacuate_remset_buffer ();
5905 index = STORE_REMSET_BUFFER_INDEX;
5906 g_assert (index == 0);
5909 buffer [index] = ptr;
5910 STORE_REMSET_BUFFER_INDEX = index;
5916 mono_gc_wbarrier_generic_store (gpointer ptr, MonoObject* value)
5918 DEBUG (8, fprintf (gc_debug_file, "Wbarrier store at %p to %p (%s)\n", ptr, value, value ? safe_name (value) : "null"));
5919 *(void**)ptr = value;
5920 if (ptr_in_nursery (value))
5921 mono_gc_wbarrier_generic_nostore (ptr);
5925 void mono_gc_wbarrier_value_copy_bitmap (gpointer _dest, gpointer _src, int size, unsigned bitmap)
5927 mword *dest = _dest;
5932 mono_gc_wbarrier_generic_store (dest, (MonoObject*)*src);
5937 size -= SIZEOF_VOID_P;
5944 mono_gc_wbarrier_value_copy (gpointer dest, gpointer src, int count, MonoClass *klass)
5947 size_t size = count * mono_class_value_size (klass, NULL);
5949 HEAVY_STAT (++stat_wbarrier_value_copy);
5950 g_assert (klass->valuetype);
5952 memmove (dest, src, size);
5953 if (use_cardtable) {
5954 sgen_card_table_mark_range ((mword)dest, size);
5956 rs = REMEMBERED_SET;
5957 if (ptr_in_nursery (dest) || ptr_on_stack (dest) || !klass->has_references) {
5961 g_assert (klass->gc_descr_inited);
5962 DEBUG (8, fprintf (gc_debug_file, "Adding value remset at %p, count %d, descr %p for class %s (%p)\n", dest, count, klass->gc_descr, klass->name, klass));
5964 if (rs->store_next + 3 < rs->end_set) {
5965 *(rs->store_next++) = (mword)dest | REMSET_VTYPE;
5966 *(rs->store_next++) = (mword)klass->gc_descr;
5967 *(rs->store_next++) = (mword)count;
5971 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
5972 rs->next = REMEMBERED_SET;
5973 REMEMBERED_SET = rs;
5974 #ifdef HAVE_KW_THREAD
5975 mono_sgen_thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
5977 *(rs->store_next++) = (mword)dest | REMSET_VTYPE;
5978 *(rs->store_next++) = (mword)klass->gc_descr;
5979 *(rs->store_next++) = (mword)count;
5985 * mono_gc_wbarrier_object_copy:
5987 * Write barrier to call when obj is the result of a clone or copy of an object.
5990 mono_gc_wbarrier_object_copy (MonoObject* obj, MonoObject *src)
5996 HEAVY_STAT (++stat_wbarrier_object_copy);
5997 rs = REMEMBERED_SET;
5998 DEBUG (6, fprintf (gc_debug_file, "Adding object remset for %p\n", obj));
5999 size = mono_object_class (obj)->instance_size;
6001 /* do not copy the sync state */
6002 memcpy ((char*)obj + sizeof (MonoObject), (char*)src + sizeof (MonoObject),
6003 size - sizeof (MonoObject));
6004 if (ptr_in_nursery (obj) || ptr_on_stack (obj)) {
6008 if (rs->store_next < rs->end_set) {
6009 *(rs->store_next++) = (mword)obj | REMSET_OBJECT;
6013 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6014 rs->next = REMEMBERED_SET;
6015 REMEMBERED_SET = rs;
6016 #ifdef HAVE_KW_THREAD
6017 mono_sgen_thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6019 *(rs->store_next++) = (mword)obj | REMSET_OBJECT;
6024 * ######################################################################
6025 * ######## Collector debugging
6026 * ######################################################################
6029 const char*descriptor_types [] = {
6041 describe_ptr (char *ptr)
6047 if (ptr_in_nursery (ptr)) {
6048 printf ("Pointer inside nursery.\n");
6050 if (major_collector.ptr_is_in_non_pinned_space (ptr)) {
6051 printf ("Pointer inside oldspace.\n");
6052 } else if (major_collector.obj_is_from_pinned_alloc (ptr)) {
6053 printf ("Pointer is inside a pinned chunk.\n");
6055 printf ("Pointer unknown.\n");
6060 if (object_is_pinned (ptr))
6061 printf ("Object is pinned.\n");
6063 if (object_is_forwarded (ptr))
6064 printf ("Object is forwared.\n");
6066 // FIXME: Handle pointers to the inside of objects
6067 vtable = (MonoVTable*)LOAD_VTABLE (ptr);
6069 printf ("VTable: %p\n", vtable);
6070 if (vtable == NULL) {
6071 printf ("VTable is invalid (empty).\n");
6074 if (ptr_in_nursery (vtable)) {
6075 printf ("VTable is invalid (points inside nursery).\n");
6078 printf ("Class: %s\n", vtable->klass->name);
6080 desc = ((GCVTable*)vtable)->desc;
6081 printf ("Descriptor: %lx\n", (long)desc);
6084 printf ("Descriptor type: %d (%s)\n", type, descriptor_types [type]);
6088 find_in_remset_loc (mword *p, char *addr, gboolean *found)
6094 switch ((*p) & REMSET_TYPE_MASK) {
6095 case REMSET_LOCATION:
6096 if (*p == (mword)addr)
6100 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
6102 if ((void**)addr >= ptr && (void**)addr < ptr + count)
6106 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
6107 count = safe_object_get_size ((MonoObject*)ptr);
6108 count = ALIGN_UP (count);
6109 count /= sizeof (mword);
6110 if ((void**)addr >= ptr && (void**)addr < ptr + count)
6114 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
6118 switch (desc & 0x7) {
6119 case DESC_TYPE_RUN_LENGTH:
6120 OBJ_RUN_LEN_SIZE (skip_size, desc, ptr);
6122 case DESC_TYPE_SMALL_BITMAP:
6123 OBJ_BITMAP_SIZE (skip_size, desc, start);
6127 g_assert_not_reached ();
6130 /* The descriptor includes the size of MonoObject */
6131 skip_size -= sizeof (MonoObject);
6133 if ((void**)addr >= ptr && (void**)addr < ptr + (skip_size / sizeof (gpointer)))
6138 g_assert_not_reached ();
6144 * Return whenever ADDR occurs in the remembered sets
6147 find_in_remsets (char *addr)
6150 SgenThreadInfo *info;
6151 RememberedSet *remset;
6152 GenericStoreRememberedSet *store_remset;
6154 gboolean found = FALSE;
6156 /* the global one */
6157 for (remset = global_remset; remset; remset = remset->next) {
6158 DEBUG (4, fprintf (gc_debug_file, "Scanning global remset range: %p-%p, size: %td\n", remset->data, remset->store_next, remset->store_next - remset->data));
6159 for (p = remset->data; p < remset->store_next;) {
6160 p = find_in_remset_loc (p, addr, &found);
6166 /* the generic store ones */
6167 for (store_remset = generic_store_remsets; store_remset; store_remset = store_remset->next) {
6168 for (i = 0; i < STORE_REMSET_BUFFER_SIZE - 1; ++i) {
6169 if (store_remset->data [i] == addr)
6174 /* the per-thread ones */
6175 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
6176 for (info = thread_table [i]; info; info = info->next) {
6178 for (remset = info->remset; remset; remset = remset->next) {
6179 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for thread %p, range: %p-%p, size: %td\n", info, remset->data, remset->store_next, remset->store_next - remset->data));
6180 for (p = remset->data; p < remset->store_next;) {
6181 p = find_in_remset_loc (p, addr, &found);
6186 for (j = 0; j < *info->store_remset_buffer_index_addr; ++j) {
6187 if ((*info->store_remset_buffer_addr) [j + 1] == addr)
6193 /* the freed thread ones */
6194 for (remset = freed_thread_remsets; remset; remset = remset->next) {
6195 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for freed thread, range: %p-%p, size: %td\n", remset->data, remset->store_next, remset->store_next - remset->data));
6196 for (p = remset->data; p < remset->store_next;) {
6197 p = find_in_remset_loc (p, addr, &found);
6206 static gboolean missing_remsets;
6209 * We let a missing remset slide if the target object is pinned,
6210 * because the store might have happened but the remset not yet added,
6211 * but in that case the target must be pinned. We might theoretically
6212 * miss some missing remsets this way, but it's very unlikely.
6215 #define HANDLE_PTR(ptr,obj) do { \
6216 if (*(ptr) && (char*)*(ptr) >= nursery_start && (char*)*(ptr) < nursery_next) { \
6217 if (!find_in_remsets ((char*)(ptr)) && (!use_cardtable || !sgen_card_table_address_is_marked ((mword)ptr))) { \
6218 fprintf (gc_debug_file, "Oldspace->newspace reference %p at offset %td in object %p (%s.%s) not found in remsets.\n", *(ptr), (char*)(ptr) - (char*)(obj), (obj), ((MonoObject*)(obj))->vtable->klass->name_space, ((MonoObject*)(obj))->vtable->klass->name); \
6219 binary_protocol_missing_remset ((obj), (gpointer)LOAD_VTABLE ((obj)), (char*)(ptr) - (char*)(obj), *(ptr), (gpointer)LOAD_VTABLE(*(ptr)), object_is_pinned (*(ptr))); \
6220 if (!object_is_pinned (*(ptr))) \
6221 missing_remsets = TRUE; \
6227 * Check that each object reference which points into the nursery can
6228 * be found in the remembered sets.
6231 check_consistency_callback (char *start, size_t size, void *dummy)
6233 GCVTable *vt = (GCVTable*)LOAD_VTABLE (start);
6234 DEBUG (8, fprintf (gc_debug_file, "Scanning object %p, vtable: %p (%s)\n", start, vt, vt->klass->name));
6236 #define SCAN_OBJECT_ACTION
6237 #include "sgen-scan-object.h"
6241 * Perform consistency check of the heap.
6243 * Assumes the world is stopped.
6246 check_consistency (void)
6250 // Need to add more checks
6252 missing_remsets = FALSE;
6254 DEBUG (1, fprintf (gc_debug_file, "Begin heap consistency check...\n"));
6256 // Check that oldspace->newspace pointers are registered with the collector
6257 major_collector.iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)check_consistency_callback, NULL);
6259 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
6260 check_consistency_callback (bigobj->data, bigobj->size, NULL);
6262 DEBUG (1, fprintf (gc_debug_file, "Heap consistency check done.\n"));
6264 #ifdef SGEN_BINARY_PROTOCOL
6265 if (!binary_protocol_file)
6267 g_assert (!missing_remsets);
6272 #define HANDLE_PTR(ptr,obj) do { \
6273 if (*(ptr) && !LOAD_VTABLE (*(ptr))) \
6274 g_error ("Could not load vtable for obj %p slot %d (size %d)", obj, (char*)ptr - (char*)obj, safe_object_get_size ((MonoObject*)obj)); \
6278 check_major_refs_callback (char *start, size_t size, void *dummy)
6280 #define SCAN_OBJECT_ACTION
6281 #include "sgen-scan-object.h"
6285 check_major_refs (void)
6289 major_collector.iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)check_major_refs_callback, NULL);
6291 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
6292 check_major_refs_callback (bigobj->data, bigobj->size, NULL);
6295 /* Check that the reference is valid */
6297 #define HANDLE_PTR(ptr,obj) do { \
6299 g_assert (safe_name (*(ptr)) != NULL); \
6306 * Perform consistency check on an object. Currently we only check that the
6307 * reference fields are valid.
6310 check_object (char *start)
6315 #include "sgen-scan-object.h"
6319 * ######################################################################
6320 * ######## Other mono public interface functions.
6321 * ######################################################################
6324 #define REFS_SIZE 128
6327 MonoGCReferences callback;
6331 MonoObject *refs [REFS_SIZE];
6335 #define HANDLE_PTR(ptr,obj) do { \
6337 if (hwi->count == REFS_SIZE) { \
6338 hwi->callback ((MonoObject*)start, mono_object_class (start), hwi->called? 0: size, hwi->count, hwi->refs, hwi->data); \
6342 hwi->refs [hwi->count++] = *(ptr); \
6347 collect_references (HeapWalkInfo *hwi, char *start, size_t size)
6349 #include "sgen-scan-object.h"
6353 walk_references (char *start, size_t size, void *data)
6355 HeapWalkInfo *hwi = data;
6358 collect_references (hwi, start, size);
6359 if (hwi->count || !hwi->called)
6360 hwi->callback ((MonoObject*)start, mono_object_class (start), hwi->called? 0: size, hwi->count, hwi->refs, hwi->data);
6364 * mono_gc_walk_heap:
6365 * @flags: flags for future use
6366 * @callback: a function pointer called for each object in the heap
6367 * @data: a user data pointer that is passed to callback
6369 * This function can be used to iterate over all the live objects in the heap:
6370 * for each object, @callback is invoked, providing info about the object's
6371 * location in memory, its class, its size and the objects it references.
6372 * The object references may be buffered, so the callback may be invoked
6373 * multiple times for the same object: in all but the first call, the size
6374 * argument will be zero.
6375 * Note that this function can be only called in the #MONO_GC_EVENT_PRE_START_WORLD
6376 * profiler event handler.
6378 * Returns: a non-zero value if the GC doesn't support heap walking
6381 mono_gc_walk_heap (int flags, MonoGCReferences callback, void *data)
6387 hwi.callback = callback;
6390 clear_nursery_fragments (nursery_next);
6391 mono_sgen_scan_area_with_callback (nursery_section->data, nursery_section->end_data, walk_references, &hwi);
6393 major_collector.iterate_objects (TRUE, TRUE, walk_references, &hwi);
6395 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
6396 walk_references (bigobj->data, bigobj->size, &hwi);
6401 mono_gc_collect (int generation)
6406 mono_profiler_gc_event (MONO_GC_EVENT_START, generation);
6407 stop_world (generation);
6408 if (generation == 0) {
6409 collect_nursery (0);
6411 major_collection ("user request");
6413 restart_world (generation);
6414 mono_profiler_gc_event (MONO_GC_EVENT_END, generation);
6419 mono_gc_max_generation (void)
6425 mono_gc_collection_count (int generation)
6427 if (generation == 0)
6428 return num_minor_gcs;
6429 return num_major_gcs;
6433 mono_gc_get_used_size (void)
6437 tot = los_memory_usage;
6438 tot += nursery_section->next_data - nursery_section->data;
6439 tot += major_collector.get_used_size ();
6440 /* FIXME: account for pinned objects */
6446 mono_gc_get_heap_size (void)
6452 mono_gc_disable (void)
6460 mono_gc_enable (void)
6468 mono_gc_get_los_limit (void)
6470 return MAX_SMALL_OBJ_SIZE;
6474 mono_object_is_alive (MonoObject* o)
6480 mono_gc_get_generation (MonoObject *obj)
6482 if (ptr_in_nursery (obj))
6488 mono_gc_enable_events (void)
6493 mono_gc_weak_link_add (void **link_addr, MonoObject *obj, gboolean track)
6496 mono_gc_register_disappearing_link (obj, link_addr, track);
6501 mono_gc_weak_link_remove (void **link_addr)
6504 mono_gc_register_disappearing_link (NULL, link_addr, FALSE);
6509 mono_gc_weak_link_get (void **link_addr)
6513 return (MonoObject*) REVEAL_POINTER (*link_addr);
6517 mono_gc_ephemeron_array_add (MonoObject *obj)
6519 EphemeronLinkNode *node;
6523 node = mono_sgen_alloc_internal (INTERNAL_MEM_EPHEMERON_LINK);
6528 node->array = (char*)obj;
6529 node->next = ephemeron_list;
6530 ephemeron_list = node;
6532 DEBUG (5, fprintf (gc_debug_file, "Registered ephemeron array %p\n", obj));
6539 mono_gc_make_descr_from_bitmap (gsize *bitmap, int numbits)
6542 return (void*)MAKE_ROOT_DESC (ROOT_DESC_BITMAP, 0);
6543 } else if (numbits < ((sizeof (*bitmap) * 8) - ROOT_DESC_TYPE_SHIFT)) {
6544 return (void*)MAKE_ROOT_DESC (ROOT_DESC_BITMAP, bitmap [0]);
6546 mword complex = alloc_complex_descriptor (bitmap, numbits);
6547 return (void*)MAKE_ROOT_DESC (ROOT_DESC_COMPLEX, complex);
6551 static void *all_ref_root_descrs [32];
6554 mono_gc_make_root_descr_all_refs (int numbits)
6559 if (numbits < 32 && all_ref_root_descrs [numbits])
6560 return all_ref_root_descrs [numbits];
6562 gc_bitmap = g_malloc0 (ALIGN_TO (numbits, 8) + 1);
6563 memset (gc_bitmap, 0xff, numbits / 8);
6565 gc_bitmap [numbits / 8] = (1 << (numbits % 8)) - 1;
6566 descr = mono_gc_make_descr_from_bitmap (gc_bitmap, numbits);
6570 all_ref_root_descrs [numbits] = descr;
6576 mono_gc_make_root_descr_user (MonoGCRootMarkFunc marker)
6580 g_assert (user_descriptors_next < MAX_USER_DESCRIPTORS);
6581 descr = (void*)MAKE_ROOT_DESC (ROOT_DESC_USER, (mword)user_descriptors_next);
6582 user_descriptors [user_descriptors_next ++] = marker;
6588 mono_gc_alloc_fixed (size_t size, void *descr)
6590 /* FIXME: do a single allocation */
6591 void *res = calloc (1, size);
6594 if (!mono_gc_register_root (res, size, descr)) {
6602 mono_gc_free_fixed (void* addr)
6604 mono_gc_deregister_root (addr);
6609 mono_gc_invoke_with_gc_lock (MonoGCLockedCallbackFunc func, void *data)
6613 result = func (data);
6614 UNLOCK_INTERRUPTION;
6619 mono_gc_is_gc_thread (void)
6623 result = mono_sgen_thread_info_lookup (ARCH_GET_THREAD ()) != NULL;
6629 mono_gc_base_init (void)
6633 char *major_collector_opt = NULL;
6634 struct sigaction sinfo;
6637 #ifdef PLATFORM_ANDROID
6638 g_assert_not_reached ();
6641 /* the gc_initialized guard seems to imply this method is
6642 idempotent, but LOCK_INIT(gc_mutex) might not be. It's
6643 defined in sgen-gc.h as nothing, so there's no danger at
6645 LOCK_INIT (gc_mutex);
6647 if (gc_initialized) {
6651 pagesize = mono_pagesize ();
6652 gc_debug_file = stderr;
6654 LOCK_INIT (interruption_mutex);
6655 LOCK_INIT (global_remset_mutex);
6656 LOCK_INIT (pin_queue_mutex);
6658 if ((env = getenv ("MONO_GC_PARAMS"))) {
6659 opts = g_strsplit (env, ",", -1);
6660 for (ptr = opts; *ptr; ++ptr) {
6662 if (g_str_has_prefix (opt, "major=")) {
6663 opt = strchr (opt, '=') + 1;
6664 major_collector_opt = g_strdup (opt);
6672 mono_sgen_init_internal_allocator ();
6674 mono_sgen_register_fixed_internal_mem_type (INTERNAL_MEM_FRAGMENT, sizeof (Fragment));
6675 mono_sgen_register_fixed_internal_mem_type (INTERNAL_MEM_SECTION, SGEN_SIZEOF_GC_MEM_SECTION);
6676 mono_sgen_register_fixed_internal_mem_type (INTERNAL_MEM_FINALIZE_ENTRY, sizeof (FinalizeEntry));
6677 mono_sgen_register_fixed_internal_mem_type (INTERNAL_MEM_DISLINK, sizeof (DisappearingLink));
6678 mono_sgen_register_fixed_internal_mem_type (INTERNAL_MEM_ROOT_RECORD, sizeof (RootRecord));
6679 mono_sgen_register_fixed_internal_mem_type (INTERNAL_MEM_GRAY_QUEUE, sizeof (GrayQueueSection));
6680 g_assert (sizeof (GenericStoreRememberedSet) == sizeof (gpointer) * STORE_REMSET_BUFFER_SIZE);
6681 mono_sgen_register_fixed_internal_mem_type (INTERNAL_MEM_STORE_REMSET, sizeof (GenericStoreRememberedSet));
6682 mono_sgen_register_fixed_internal_mem_type (INTERNAL_MEM_EPHEMERON_LINK, sizeof (EphemeronLinkNode));
6684 if (!major_collector_opt || !strcmp (major_collector_opt, "marksweep")) {
6685 mono_sgen_marksweep_init (&major_collector);
6686 } else if (!major_collector_opt || !strcmp (major_collector_opt, "marksweep-fixed")) {
6687 mono_sgen_marksweep_fixed_init (&major_collector);
6688 } else if (!major_collector_opt || !strcmp (major_collector_opt, "marksweep-par")) {
6689 mono_sgen_marksweep_par_init (&major_collector);
6690 workers_init (mono_cpu_count ());
6691 } else if (!major_collector_opt || !strcmp (major_collector_opt, "marksweep-fixed-par")) {
6692 mono_sgen_marksweep_fixed_par_init (&major_collector);
6693 workers_init (mono_cpu_count ());
6694 } else if (!strcmp (major_collector_opt, "copying")) {
6695 mono_sgen_copying_init (&major_collector);
6697 fprintf (stderr, "Unknown major collector `%s'.\n", major_collector_opt);
6701 #ifdef SGEN_HAVE_CARDTABLE
6702 use_cardtable = major_collector.supports_cardtable;
6704 use_cardtable = FALSE;
6708 for (ptr = opts; *ptr; ++ptr) {
6710 if (g_str_has_prefix (opt, "major="))
6712 if (g_str_has_prefix (opt, "wbarrier=")) {
6713 opt = strchr (opt, '=') + 1;
6714 if (strcmp (opt, "remset") == 0) {
6715 use_cardtable = FALSE;
6716 } else if (strcmp (opt, "cardtable") == 0) {
6717 if (!use_cardtable) {
6718 if (major_collector.supports_cardtable)
6719 fprintf (stderr, "The cardtable write barrier is not supported on this platform.\n");
6721 fprintf (stderr, "The major collector does not support the cardtable write barrier.\n");
6727 if (g_str_has_prefix (opt, "max-heap-size=")) {
6728 opt = strchr (opt, '=') + 1;
6729 if (*opt && mono_gc_parse_environment_string_extract_number (opt, &max_heap)) {
6730 if ((max_heap & (mono_pagesize () - 1))) {
6731 fprintf (stderr, "max-heap-size size must be a multiple of %d.\n", mono_pagesize ());
6735 fprintf (stderr, "max-heap-size must be an integer.\n");
6741 if (g_str_has_prefix (opt, "nursery-size=")) {
6743 opt = strchr (opt, '=') + 1;
6744 if (*opt && mono_gc_parse_environment_string_extract_number (opt, &val)) {
6745 default_nursery_size = val;
6746 #ifdef SGEN_ALIGN_NURSERY
6747 if ((val & (val - 1))) {
6748 fprintf (stderr, "The nursery size must be a power of two.\n");
6752 default_nursery_bits = 0;
6753 while (1 << (++ default_nursery_bits) != default_nursery_size)
6757 fprintf (stderr, "nursery-size must be an integer.\n");
6763 if (!(major_collector.handle_gc_param && major_collector.handle_gc_param (opt))) {
6764 fprintf (stderr, "MONO_GC_PARAMS must be a comma-delimited list of one or more of the following:\n");
6765 fprintf (stderr, " max-heap-size=N (where N is an integer, possibly with a k, m or a g suffix)\n");
6766 fprintf (stderr, " nursery-size=N (where N is an integer, possibly with a k, m or a g suffix)\n");
6767 fprintf (stderr, " major=COLLECTOR (where COLLECTOR is `marksweep', `marksweep-par' or `copying')\n");
6768 fprintf (stderr, " wbarrier=WBARRIER (where WBARRIER is `remset' or `cardtable')\n");
6769 if (major_collector.print_gc_param_usage)
6770 major_collector.print_gc_param_usage ();
6777 if (major_collector_opt)
6778 g_free (major_collector_opt);
6780 nursery_size = DEFAULT_NURSERY_SIZE;
6781 minor_collection_allowance = MIN_MINOR_COLLECTION_ALLOWANCE;
6782 init_heap_size_limits (max_heap);
6786 if ((env = getenv ("MONO_GC_DEBUG"))) {
6787 opts = g_strsplit (env, ",", -1);
6788 for (ptr = opts; ptr && *ptr; ptr ++) {
6790 if (opt [0] >= '0' && opt [0] <= '9') {
6791 gc_debug_level = atoi (opt);
6796 char *rf = g_strdup_printf ("%s.%d", opt, getpid ());
6797 gc_debug_file = fopen (rf, "wb");
6799 gc_debug_file = stderr;
6802 } else if (!strcmp (opt, "collect-before-allocs")) {
6803 collect_before_allocs = TRUE;
6804 } else if (!strcmp (opt, "check-at-minor-collections")) {
6805 consistency_check_at_minor_collection = TRUE;
6806 nursery_clear_policy = CLEAR_AT_GC;
6807 } else if (!strcmp (opt, "xdomain-checks")) {
6808 xdomain_checks = TRUE;
6809 } else if (!strcmp (opt, "clear-at-gc")) {
6810 nursery_clear_policy = CLEAR_AT_GC;
6811 } else if (!strcmp (opt, "conservative-stack-mark")) {
6812 conservative_stack_mark = TRUE;
6813 } else if (!strcmp (opt, "check-scan-starts")) {
6814 do_scan_starts_check = TRUE;
6815 } else if (g_str_has_prefix (opt, "heap-dump=")) {
6816 char *filename = strchr (opt, '=') + 1;
6817 nursery_clear_policy = CLEAR_AT_GC;
6818 heap_dump_file = fopen (filename, "w");
6820 fprintf (heap_dump_file, "<sgen-dump>\n");
6821 #ifdef SGEN_BINARY_PROTOCOL
6822 } else if (g_str_has_prefix (opt, "binary-protocol=")) {
6823 char *filename = strchr (opt, '=') + 1;
6824 binary_protocol_file = fopen (filename, "w");
6827 fprintf (stderr, "Invalid format for the MONO_GC_DEBUG env variable: '%s'\n", env);
6828 fprintf (stderr, "The format is: MONO_GC_DEBUG=[l[:filename]|<option>]+ where l is a debug level 0-9.\n");
6829 fprintf (stderr, "Valid options are: collect-before-allocs, check-at-minor-collections, xdomain-checks, clear-at-gc.\n");
6836 suspend_ack_semaphore_ptr = &suspend_ack_semaphore;
6837 MONO_SEM_INIT (&suspend_ack_semaphore, 0);
6839 sigfillset (&sinfo.sa_mask);
6840 sinfo.sa_flags = SA_RESTART | SA_SIGINFO;
6841 sinfo.sa_sigaction = suspend_handler;
6842 if (sigaction (suspend_signal_num, &sinfo, NULL) != 0) {
6843 g_error ("failed sigaction");
6846 sinfo.sa_handler = restart_handler;
6847 if (sigaction (restart_signal_num, &sinfo, NULL) != 0) {
6848 g_error ("failed sigaction");
6851 sigfillset (&suspend_signal_mask);
6852 sigdelset (&suspend_signal_mask, restart_signal_num);
6854 global_remset = alloc_remset (1024, NULL);
6855 global_remset->next = NULL;
6857 pthread_key_create (&remembered_set_key, unregister_thread);
6859 #ifndef HAVE_KW_THREAD
6860 pthread_key_create (&thread_info_key, NULL);
6866 gc_initialized = TRUE;
6868 mono_gc_register_thread (&sinfo);
6872 mono_gc_get_suspend_signal (void)
6874 return suspend_signal_num;
6884 #ifdef HAVE_KW_THREAD
6885 #define EMIT_TLS_ACCESS(mb,dummy,offset) do { \
6886 mono_mb_emit_byte ((mb), MONO_CUSTOM_PREFIX); \
6887 mono_mb_emit_byte ((mb), CEE_MONO_TLS); \
6888 mono_mb_emit_i4 ((mb), (offset)); \
6891 #define EMIT_TLS_ACCESS(mb,member,dummy) do { \
6892 mono_mb_emit_byte ((mb), MONO_CUSTOM_PREFIX); \
6893 mono_mb_emit_byte ((mb), CEE_MONO_TLS); \
6894 mono_mb_emit_i4 ((mb), thread_info_key); \
6895 mono_mb_emit_icon ((mb), G_STRUCT_OFFSET (SgenThreadInfo, member)); \
6896 mono_mb_emit_byte ((mb), CEE_ADD); \
6897 mono_mb_emit_byte ((mb), CEE_LDIND_I); \
6901 #ifdef MANAGED_ALLOCATION
6902 /* FIXME: Do this in the JIT, where specialized allocation sequences can be created
6903 * for each class. This is currently not easy to do, as it is hard to generate basic
6904 * blocks + branches, but it is easy with the linear IL codebase.
6906 * For this to work we'd need to solve the TLAB race, first. Now we
6907 * require the allocator to be in a few known methods to make sure
6908 * that they are executed atomically via the restart mechanism.
6911 create_allocator (int atype)
6913 int p_var, size_var;
6914 guint32 slowpath_branch, max_size_branch;
6915 MonoMethodBuilder *mb;
6917 MonoMethodSignature *csig;
6918 static gboolean registered = FALSE;
6919 int tlab_next_addr_var, new_next_var;
6921 const char *name = NULL;
6922 AllocatorWrapperInfo *info;
6924 #ifdef HAVE_KW_THREAD
6925 int tlab_next_addr_offset = -1;
6926 int tlab_temp_end_offset = -1;
6928 MONO_THREAD_VAR_OFFSET (tlab_next_addr, tlab_next_addr_offset);
6929 MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
6931 g_assert (tlab_next_addr_offset != -1);
6932 g_assert (tlab_temp_end_offset != -1);
6936 mono_register_jit_icall (mono_gc_alloc_obj, "mono_gc_alloc_obj", mono_create_icall_signature ("object ptr int"), FALSE);
6937 mono_register_jit_icall (mono_gc_alloc_vector, "mono_gc_alloc_vector", mono_create_icall_signature ("object ptr int int"), FALSE);
6941 if (atype == ATYPE_SMALL) {
6943 name = "AllocSmall";
6944 } else if (atype == ATYPE_NORMAL) {
6947 } else if (atype == ATYPE_VECTOR) {
6949 name = "AllocVector";
6951 g_assert_not_reached ();
6954 csig = mono_metadata_signature_alloc (mono_defaults.corlib, num_params);
6955 csig->ret = &mono_defaults.object_class->byval_arg;
6956 for (i = 0; i < num_params; ++i)
6957 csig->params [i] = &mono_defaults.int_class->byval_arg;
6959 mb = mono_mb_new (mono_defaults.object_class, name, MONO_WRAPPER_ALLOC);
6960 size_var = mono_mb_add_local (mb, &mono_defaults.int32_class->byval_arg);
6961 if (atype == ATYPE_NORMAL || atype == ATYPE_SMALL) {
6962 /* size = vtable->klass->instance_size; */
6963 mono_mb_emit_ldarg (mb, 0);
6964 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoVTable, klass));
6965 mono_mb_emit_byte (mb, CEE_ADD);
6966 mono_mb_emit_byte (mb, CEE_LDIND_I);
6967 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoClass, instance_size));
6968 mono_mb_emit_byte (mb, CEE_ADD);
6969 /* FIXME: assert instance_size stays a 4 byte integer */
6970 mono_mb_emit_byte (mb, CEE_LDIND_U4);
6971 mono_mb_emit_stloc (mb, size_var);
6972 } else if (atype == ATYPE_VECTOR) {
6973 MonoExceptionClause *clause;
6975 MonoClass *oom_exc_class;
6978 /* n > MONO_ARRAY_MAX_INDEX -> OverflowException */
6979 mono_mb_emit_ldarg (mb, 1);
6980 mono_mb_emit_icon (mb, MONO_ARRAY_MAX_INDEX);
6981 pos = mono_mb_emit_short_branch (mb, CEE_BLE_UN_S);
6982 mono_mb_emit_exception (mb, "OverflowException", NULL);
6983 mono_mb_patch_short_branch (mb, pos);
6985 clause = mono_image_alloc0 (mono_defaults.corlib, sizeof (MonoExceptionClause));
6986 clause->try_offset = mono_mb_get_label (mb);
6988 /* vtable->klass->sizes.element_size */
6989 mono_mb_emit_ldarg (mb, 0);
6990 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoVTable, klass));
6991 mono_mb_emit_byte (mb, CEE_ADD);
6992 mono_mb_emit_byte (mb, CEE_LDIND_I);
6993 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoClass, sizes.element_size));
6994 mono_mb_emit_byte (mb, CEE_ADD);
6995 mono_mb_emit_byte (mb, CEE_LDIND_U4);
6998 mono_mb_emit_ldarg (mb, 1);
6999 mono_mb_emit_byte (mb, CEE_MUL_OVF_UN);
7000 /* + sizeof (MonoArray) */
7001 mono_mb_emit_icon (mb, sizeof (MonoArray));
7002 mono_mb_emit_byte (mb, CEE_ADD_OVF_UN);
7003 mono_mb_emit_stloc (mb, size_var);
7005 pos_leave = mono_mb_emit_branch (mb, CEE_LEAVE);
7008 clause->flags = MONO_EXCEPTION_CLAUSE_NONE;
7009 clause->try_len = mono_mb_get_pos (mb) - clause->try_offset;
7010 clause->data.catch_class = mono_class_from_name (mono_defaults.corlib,
7011 "System", "OverflowException");
7012 g_assert (clause->data.catch_class);
7013 clause->handler_offset = mono_mb_get_label (mb);
7015 oom_exc_class = mono_class_from_name (mono_defaults.corlib,
7016 "System", "OutOfMemoryException");
7017 g_assert (oom_exc_class);
7018 ctor = mono_class_get_method_from_name (oom_exc_class, ".ctor", 0);
7021 mono_mb_emit_byte (mb, CEE_POP);
7022 mono_mb_emit_op (mb, CEE_NEWOBJ, ctor);
7023 mono_mb_emit_byte (mb, CEE_THROW);
7025 clause->handler_len = mono_mb_get_pos (mb) - clause->handler_offset;
7026 mono_mb_set_clauses (mb, 1, clause);
7027 mono_mb_patch_branch (mb, pos_leave);
7030 g_assert_not_reached ();
7033 /* size += ALLOC_ALIGN - 1; */
7034 mono_mb_emit_ldloc (mb, size_var);
7035 mono_mb_emit_icon (mb, ALLOC_ALIGN - 1);
7036 mono_mb_emit_byte (mb, CEE_ADD);
7037 /* size &= ~(ALLOC_ALIGN - 1); */
7038 mono_mb_emit_icon (mb, ~(ALLOC_ALIGN - 1));
7039 mono_mb_emit_byte (mb, CEE_AND);
7040 mono_mb_emit_stloc (mb, size_var);
7042 /* if (size > MAX_SMALL_OBJ_SIZE) goto slowpath */
7043 if (atype != ATYPE_SMALL) {
7044 mono_mb_emit_ldloc (mb, size_var);
7045 mono_mb_emit_icon (mb, MAX_SMALL_OBJ_SIZE);
7046 max_size_branch = mono_mb_emit_short_branch (mb, MONO_CEE_BGT_S);
7050 * We need to modify tlab_next, but the JIT only supports reading, so we read
7051 * another tls var holding its address instead.
7054 /* tlab_next_addr (local) = tlab_next_addr (TLS var) */
7055 tlab_next_addr_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7056 EMIT_TLS_ACCESS (mb, tlab_next_addr, tlab_next_addr_offset);
7057 mono_mb_emit_stloc (mb, tlab_next_addr_var);
7059 /* p = (void**)tlab_next; */
7060 p_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7061 mono_mb_emit_ldloc (mb, tlab_next_addr_var);
7062 mono_mb_emit_byte (mb, CEE_LDIND_I);
7063 mono_mb_emit_stloc (mb, p_var);
7065 /* new_next = (char*)p + size; */
7066 new_next_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7067 mono_mb_emit_ldloc (mb, p_var);
7068 mono_mb_emit_ldloc (mb, size_var);
7069 mono_mb_emit_byte (mb, CEE_CONV_I);
7070 mono_mb_emit_byte (mb, CEE_ADD);
7071 mono_mb_emit_stloc (mb, new_next_var);
7073 /* tlab_next = new_next */
7074 mono_mb_emit_ldloc (mb, tlab_next_addr_var);
7075 mono_mb_emit_ldloc (mb, new_next_var);
7076 mono_mb_emit_byte (mb, CEE_STIND_I);
7078 /* if (G_LIKELY (new_next < tlab_temp_end)) */
7079 mono_mb_emit_ldloc (mb, new_next_var);
7080 EMIT_TLS_ACCESS (mb, tlab_temp_end, tlab_temp_end_offset);
7081 slowpath_branch = mono_mb_emit_short_branch (mb, MONO_CEE_BLT_UN_S);
7084 if (atype != ATYPE_SMALL)
7085 mono_mb_patch_short_branch (mb, max_size_branch);
7087 mono_mb_emit_byte (mb, MONO_CUSTOM_PREFIX);
7088 mono_mb_emit_byte (mb, CEE_MONO_NOT_TAKEN);
7090 /* FIXME: mono_gc_alloc_obj takes a 'size_t' as an argument, not an int32 */
7091 mono_mb_emit_ldarg (mb, 0);
7092 mono_mb_emit_ldloc (mb, size_var);
7093 if (atype == ATYPE_NORMAL || atype == ATYPE_SMALL) {
7094 mono_mb_emit_icall (mb, mono_gc_alloc_obj);
7095 } else if (atype == ATYPE_VECTOR) {
7096 mono_mb_emit_ldarg (mb, 1);
7097 mono_mb_emit_icall (mb, mono_gc_alloc_vector);
7099 g_assert_not_reached ();
7101 mono_mb_emit_byte (mb, CEE_RET);
7104 mono_mb_patch_short_branch (mb, slowpath_branch);
7106 /* FIXME: Memory barrier */
7109 mono_mb_emit_ldloc (mb, p_var);
7110 mono_mb_emit_ldarg (mb, 0);
7111 mono_mb_emit_byte (mb, CEE_STIND_I);
7113 if (atype == ATYPE_VECTOR) {
7114 /* arr->max_length = max_length; */
7115 mono_mb_emit_ldloc (mb, p_var);
7116 mono_mb_emit_ldflda (mb, G_STRUCT_OFFSET (MonoArray, max_length));
7117 mono_mb_emit_ldarg (mb, 1);
7118 mono_mb_emit_byte (mb, CEE_STIND_I);
7122 mono_mb_emit_ldloc (mb, p_var);
7123 mono_mb_emit_byte (mb, CEE_RET);
7125 res = mono_mb_create_method (mb, csig, 8);
7127 mono_method_get_header (res)->init_locals = FALSE;
7129 info = mono_image_alloc0 (mono_defaults.corlib, sizeof (AllocatorWrapperInfo));
7130 info->gc_name = "sgen";
7131 info->alloc_type = atype;
7132 mono_marshal_set_wrapper_info (res, info);
7139 mono_gc_get_gc_name (void)
7144 static MonoMethod* alloc_method_cache [ATYPE_NUM];
7145 static MonoMethod *write_barrier_method;
7148 is_ip_in_managed_allocator (MonoDomain *domain, gpointer ip)
7154 if (!mono_thread_internal_current ())
7155 /* Happens during thread attach */
7160 ji = mono_jit_info_table_find (domain, ip);
7163 method = ji->method;
7165 if (method == write_barrier_method)
7167 for (i = 0; i < ATYPE_NUM; ++i)
7168 if (method == alloc_method_cache [i])
7174 * Generate an allocator method implementing the fast path of mono_gc_alloc_obj ().
7175 * The signature of the called method is:
7176 * object allocate (MonoVTable *vtable)
7179 mono_gc_get_managed_allocator (MonoVTable *vtable, gboolean for_box)
7181 #ifdef MANAGED_ALLOCATION
7182 MonoClass *klass = vtable->klass;
7184 #ifdef HAVE_KW_THREAD
7185 int tlab_next_offset = -1;
7186 int tlab_temp_end_offset = -1;
7187 MONO_THREAD_VAR_OFFSET (tlab_next, tlab_next_offset);
7188 MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
7190 if (tlab_next_offset == -1 || tlab_temp_end_offset == -1)
7194 if (!mono_runtime_has_tls_get ())
7196 if (klass->instance_size > tlab_size)
7198 if (klass->has_finalize || klass->marshalbyref || (mono_profiler_get_events () & MONO_PROFILE_ALLOCATIONS))
7202 if (klass->byval_arg.type == MONO_TYPE_STRING)
7204 if (collect_before_allocs)
7207 if (ALIGN_TO (klass->instance_size, ALLOC_ALIGN) < MAX_SMALL_OBJ_SIZE)
7208 return mono_gc_get_managed_allocator_by_type (ATYPE_SMALL);
7210 return mono_gc_get_managed_allocator_by_type (ATYPE_NORMAL);
7217 mono_gc_get_managed_array_allocator (MonoVTable *vtable, int rank)
7219 #ifdef MANAGED_ALLOCATION
7220 MonoClass *klass = vtable->klass;
7222 #ifdef HAVE_KW_THREAD
7223 int tlab_next_offset = -1;
7224 int tlab_temp_end_offset = -1;
7225 MONO_THREAD_VAR_OFFSET (tlab_next, tlab_next_offset);
7226 MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
7228 if (tlab_next_offset == -1 || tlab_temp_end_offset == -1)
7234 if (!mono_runtime_has_tls_get ())
7236 if (mono_profiler_get_events () & MONO_PROFILE_ALLOCATIONS)
7238 if (collect_before_allocs)
7240 g_assert (!klass->has_finalize && !klass->marshalbyref);
7242 return mono_gc_get_managed_allocator_by_type (ATYPE_VECTOR);
7249 mono_gc_get_managed_allocator_by_type (int atype)
7251 #ifdef MANAGED_ALLOCATION
7254 if (!mono_runtime_has_tls_get ())
7257 mono_loader_lock ();
7258 res = alloc_method_cache [atype];
7260 res = alloc_method_cache [atype] = create_allocator (atype);
7261 mono_loader_unlock ();
7269 mono_gc_get_managed_allocator_types (void)
7276 mono_gc_get_write_barrier (void)
7279 MonoMethodBuilder *mb;
7280 MonoMethodSignature *sig;
7281 #ifdef MANAGED_WBARRIER
7282 int label_no_wb_1, label_no_wb_2, label_no_wb_3, label_no_wb_4, label_need_wb, label_slow_path;
7283 #ifndef SGEN_ALIGN_NURSERY
7284 int label_continue_1, label_continue_2, label_no_wb_5;
7285 int dereferenced_var;
7287 int buffer_var, buffer_index_var, dummy_var;
7289 #ifdef HAVE_KW_THREAD
7290 int stack_end_offset = -1, store_remset_buffer_offset = -1;
7291 int store_remset_buffer_index_offset = -1, store_remset_buffer_index_addr_offset = -1;
7293 MONO_THREAD_VAR_OFFSET (stack_end, stack_end_offset);
7294 g_assert (stack_end_offset != -1);
7295 MONO_THREAD_VAR_OFFSET (store_remset_buffer, store_remset_buffer_offset);
7296 g_assert (store_remset_buffer_offset != -1);
7297 MONO_THREAD_VAR_OFFSET (store_remset_buffer_index, store_remset_buffer_index_offset);
7298 g_assert (store_remset_buffer_index_offset != -1);
7299 MONO_THREAD_VAR_OFFSET (store_remset_buffer_index_addr, store_remset_buffer_index_addr_offset);
7300 g_assert (store_remset_buffer_index_addr_offset != -1);
7304 g_assert (!use_cardtable);
7306 // FIXME: Maybe create a separate version for ctors (the branch would be
7307 // correctly predicted more times)
7308 if (write_barrier_method)
7309 return write_barrier_method;
7311 /* Create the IL version of mono_gc_barrier_generic_store () */
7312 sig = mono_metadata_signature_alloc (mono_defaults.corlib, 1);
7313 sig->ret = &mono_defaults.void_class->byval_arg;
7314 sig->params [0] = &mono_defaults.int_class->byval_arg;
7316 mb = mono_mb_new (mono_defaults.object_class, "wbarrier", MONO_WRAPPER_WRITE_BARRIER);
7318 #ifdef MANAGED_WBARRIER
7319 if (mono_runtime_has_tls_get ()) {
7320 #ifdef SGEN_ALIGN_NURSERY
7321 // if (ptr_in_nursery (ptr)) return;
7323 * Masking out the bits might be faster, but we would have to use 64 bit
7324 * immediates, which might be slower.
7326 mono_mb_emit_ldarg (mb, 0);
7327 mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
7328 mono_mb_emit_byte (mb, CEE_SHR_UN);
7329 mono_mb_emit_icon (mb, (mword)nursery_start >> DEFAULT_NURSERY_BITS);
7330 label_no_wb_1 = mono_mb_emit_branch (mb, CEE_BEQ);
7332 // if (!ptr_in_nursery (*ptr)) return;
7333 mono_mb_emit_ldarg (mb, 0);
7334 mono_mb_emit_byte (mb, CEE_LDIND_I);
7335 mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
7336 mono_mb_emit_byte (mb, CEE_SHR_UN);
7337 mono_mb_emit_icon (mb, (mword)nursery_start >> DEFAULT_NURSERY_BITS);
7338 label_no_wb_2 = mono_mb_emit_branch (mb, CEE_BNE_UN);
7341 // if (ptr < (nursery_start)) goto continue;
7342 mono_mb_emit_ldarg (mb, 0);
7343 mono_mb_emit_ptr (mb, (gpointer) nursery_start);
7344 label_continue_1 = mono_mb_emit_branch (mb, CEE_BLT);
7346 // if (ptr >= nursery_real_end)) goto continue;
7347 mono_mb_emit_ldarg (mb, 0);
7348 mono_mb_emit_ptr (mb, (gpointer) nursery_real_end);
7349 label_continue_2 = mono_mb_emit_branch (mb, CEE_BGE);
7352 label_no_wb_1 = mono_mb_emit_branch (mb, CEE_BR);
7355 mono_mb_patch_branch (mb, label_continue_1);
7356 mono_mb_patch_branch (mb, label_continue_2);
7358 // Dereference and store in local var
7359 dereferenced_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7360 mono_mb_emit_ldarg (mb, 0);
7361 mono_mb_emit_byte (mb, CEE_LDIND_I);
7362 mono_mb_emit_stloc (mb, dereferenced_var);
7364 // if (*ptr < nursery_start) return;
7365 mono_mb_emit_ldloc (mb, dereferenced_var);
7366 mono_mb_emit_ptr (mb, (gpointer) nursery_start);
7367 label_no_wb_2 = mono_mb_emit_branch (mb, CEE_BLT);
7369 // if (*ptr >= nursery_end) return;
7370 mono_mb_emit_ldloc (mb, dereferenced_var);
7371 mono_mb_emit_ptr (mb, (gpointer) nursery_real_end);
7372 label_no_wb_5 = mono_mb_emit_branch (mb, CEE_BGE);
7375 // if (ptr >= stack_end) goto need_wb;
7376 mono_mb_emit_ldarg (mb, 0);
7377 EMIT_TLS_ACCESS (mb, stack_end, stack_end_offset);
7378 label_need_wb = mono_mb_emit_branch (mb, CEE_BGE_UN);
7380 // if (ptr >= stack_start) return;
7381 dummy_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7382 mono_mb_emit_ldarg (mb, 0);
7383 mono_mb_emit_ldloc_addr (mb, dummy_var);
7384 label_no_wb_3 = mono_mb_emit_branch (mb, CEE_BGE_UN);
7387 mono_mb_patch_branch (mb, label_need_wb);
7389 // buffer = STORE_REMSET_BUFFER;
7390 buffer_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7391 EMIT_TLS_ACCESS (mb, store_remset_buffer, store_remset_buffer_offset);
7392 mono_mb_emit_stloc (mb, buffer_var);
7394 // buffer_index = STORE_REMSET_BUFFER_INDEX;
7395 buffer_index_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7396 EMIT_TLS_ACCESS (mb, store_remset_buffer_index, store_remset_buffer_index_offset);
7397 mono_mb_emit_stloc (mb, buffer_index_var);
7399 // if (buffer [buffer_index] == ptr) return;
7400 mono_mb_emit_ldloc (mb, buffer_var);
7401 mono_mb_emit_ldloc (mb, buffer_index_var);
7402 g_assert (sizeof (gpointer) == 4 || sizeof (gpointer) == 8);
7403 mono_mb_emit_icon (mb, sizeof (gpointer) == 4 ? 2 : 3);
7404 mono_mb_emit_byte (mb, CEE_SHL);
7405 mono_mb_emit_byte (mb, CEE_ADD);
7406 mono_mb_emit_byte (mb, CEE_LDIND_I);
7407 mono_mb_emit_ldarg (mb, 0);
7408 label_no_wb_4 = mono_mb_emit_branch (mb, CEE_BEQ);
7411 mono_mb_emit_ldloc (mb, buffer_index_var);
7412 mono_mb_emit_icon (mb, 1);
7413 mono_mb_emit_byte (mb, CEE_ADD);
7414 mono_mb_emit_stloc (mb, buffer_index_var);
7416 // if (buffer_index >= STORE_REMSET_BUFFER_SIZE) goto slow_path;
7417 mono_mb_emit_ldloc (mb, buffer_index_var);
7418 mono_mb_emit_icon (mb, STORE_REMSET_BUFFER_SIZE);
7419 label_slow_path = mono_mb_emit_branch (mb, CEE_BGE);
7421 // buffer [buffer_index] = ptr;
7422 mono_mb_emit_ldloc (mb, buffer_var);
7423 mono_mb_emit_ldloc (mb, buffer_index_var);
7424 g_assert (sizeof (gpointer) == 4 || sizeof (gpointer) == 8);
7425 mono_mb_emit_icon (mb, sizeof (gpointer) == 4 ? 2 : 3);
7426 mono_mb_emit_byte (mb, CEE_SHL);
7427 mono_mb_emit_byte (mb, CEE_ADD);
7428 mono_mb_emit_ldarg (mb, 0);
7429 mono_mb_emit_byte (mb, CEE_STIND_I);
7431 // STORE_REMSET_BUFFER_INDEX = buffer_index;
7432 EMIT_TLS_ACCESS (mb, store_remset_buffer_index_addr, store_remset_buffer_index_addr_offset);
7433 mono_mb_emit_ldloc (mb, buffer_index_var);
7434 mono_mb_emit_byte (mb, CEE_STIND_I);
7437 mono_mb_patch_branch (mb, label_no_wb_1);
7438 mono_mb_patch_branch (mb, label_no_wb_2);
7439 mono_mb_patch_branch (mb, label_no_wb_3);
7440 mono_mb_patch_branch (mb, label_no_wb_4);
7441 #ifndef SGEN_ALIGN_NURSERY
7442 mono_mb_patch_branch (mb, label_no_wb_5);
7444 mono_mb_emit_byte (mb, CEE_RET);
7447 mono_mb_patch_branch (mb, label_slow_path);
7451 mono_mb_emit_ldarg (mb, 0);
7452 mono_mb_emit_icall (mb, mono_gc_wbarrier_generic_nostore);
7453 mono_mb_emit_byte (mb, CEE_RET);
7455 res = mono_mb_create_method (mb, sig, 16);
7458 mono_loader_lock ();
7459 if (write_barrier_method) {
7460 /* Already created */
7461 mono_free_method (res);
7463 /* double-checked locking */
7464 mono_memory_barrier ();
7465 write_barrier_method = res;
7467 mono_loader_unlock ();
7469 return write_barrier_method;
7473 mono_gc_get_description (void)
7475 return g_strdup ("sgen");
7479 mono_gc_set_desktop_mode (void)
7484 mono_gc_is_moving (void)
7490 mono_gc_is_disabled (void)
7496 mono_sgen_debug_printf (int level, const char *format, ...)
7500 if (level > gc_debug_level)
7503 va_start (ap, format);
7504 vfprintf (gc_debug_file, format, ap);
7508 #endif /* HAVE_SGEN_GC */