2 * sgen-gc.c: Simple generational GC.
5 * Paolo Molaro (lupus@ximian.com)
7 * Copyright 2005-2010 Novell, Inc (http://www.novell.com)
9 * Thread start/stop adapted from Boehm's GC:
10 * Copyright (c) 1994 by Xerox Corporation. All rights reserved.
11 * Copyright (c) 1996 by Silicon Graphics. All rights reserved.
12 * Copyright (c) 1998 by Fergus Henderson. All rights reserved.
13 * Copyright (c) 2000-2004 by Hewlett-Packard Company. All rights reserved.
15 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
16 * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
18 * Permission is hereby granted to use or copy this program
19 * for any purpose, provided the above notices are retained on all copies.
20 * Permission to modify the code and to distribute modified code is granted,
21 * provided the above notices are retained, and a notice that the code was
22 * modified is included with the above copyright notice.
25 * Copyright 2001-2003 Ximian, Inc
26 * Copyright 2003-2010 Novell, Inc.
28 * Permission is hereby granted, free of charge, to any person obtaining
29 * a copy of this software and associated documentation files (the
30 * "Software"), to deal in the Software without restriction, including
31 * without limitation the rights to use, copy, modify, merge, publish,
32 * distribute, sublicense, and/or sell copies of the Software, and to
33 * permit persons to whom the Software is furnished to do so, subject to
34 * the following conditions:
36 * The above copyright notice and this permission notice shall be
37 * included in all copies or substantial portions of the Software.
39 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
40 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
41 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
42 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
43 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
44 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
45 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
48 * Important: allocation provides always zeroed memory, having to do
49 * a memset after allocation is deadly for performance.
50 * Memory usage at startup is currently as follows:
52 * 64 KB internal space
54 * We should provide a small memory config with half the sizes
56 * We currently try to make as few mono assumptions as possible:
57 * 1) 2-word header with no GC pointers in it (first vtable, second to store the
59 * 2) gc descriptor is the second word in the vtable (first word in the class)
60 * 3) 8 byte alignment is the minimum and enough (not true for special structures (SIMD), FIXME)
61 * 4) there is a function to get an object's size and the number of
62 * elements in an array.
63 * 5) we know the special way bounds are allocated for complex arrays
64 * 6) we know about proxies and how to treat them when domains are unloaded
66 * Always try to keep stack usage to a minimum: no recursive behaviour
67 * and no large stack allocs.
69 * General description.
70 * Objects are initially allocated in a nursery using a fast bump-pointer technique.
71 * When the nursery is full we start a nursery collection: this is performed with a
73 * When the old generation is full we start a copying GC of the old generation as well:
74 * this will be changed to mark&sweep with copying when fragmentation becomes to severe
75 * in the future. Maybe we'll even do both during the same collection like IMMIX.
77 * The things that complicate this description are:
78 * *) pinned objects: we can't move them so we need to keep track of them
79 * *) no precise info of the thread stacks and registers: we need to be able to
80 * quickly find the objects that may be referenced conservatively and pin them
81 * (this makes the first issues more important)
82 * *) large objects are too expensive to be dealt with using copying GC: we handle them
83 * with mark/sweep during major collections
84 * *) some objects need to not move even if they are small (interned strings, Type handles):
85 * we use mark/sweep for them, too: they are not allocated in the nursery, but inside
86 * PinnedChunks regions
92 *) we could have a function pointer in MonoClass to implement
93 customized write barriers for value types
95 *) investigate the stuff needed to advance a thread to a GC-safe
96 point (single-stepping, read from unmapped memory etc) and implement it.
97 This would enable us to inline allocations and write barriers, for example,
98 or at least parts of them, like the write barrier checks.
99 We may need this also for handling precise info on stacks, even simple things
100 as having uninitialized data on the stack and having to wait for the prolog
101 to zero it. Not an issue for the last frame that we scan conservatively.
102 We could always not trust the value in the slots anyway.
104 *) modify the jit to save info about references in stack locations:
105 this can be done just for locals as a start, so that at least
106 part of the stack is handled precisely.
108 *) test/fix endianess issues
110 *) Implement a card table as the write barrier instead of remembered
111 sets? Card tables are not easy to implement with our current
112 memory layout. We have several different kinds of major heap
113 objects: Small objects in regular blocks, small objects in pinned
114 chunks and LOS objects. If we just have a pointer we have no way
115 to tell which kind of object it points into, therefore we cannot
116 know where its card table is. The least we have to do to make
117 this happen is to get rid of write barriers for indirect stores.
120 *) Get rid of write barriers for indirect stores. We can do this by
121 telling the GC to wbarrier-register an object once we do an ldloca
122 or ldelema on it, and to unregister it once it's not used anymore
123 (it can only travel downwards on the stack). The problem with
124 unregistering is that it needs to happen eventually no matter
125 what, even if exceptions are thrown, the thread aborts, etc.
126 Rodrigo suggested that we could do only the registering part and
127 let the collector find out (pessimistically) when it's safe to
128 unregister, namely when the stack pointer of the thread that
129 registered the object is higher than it was when the registering
130 happened. This might make for a good first implementation to get
131 some data on performance.
133 *) Some sort of blacklist support? Blacklists is a concept from the
134 Boehm GC: if during a conservative scan we find pointers to an
135 area which we might use as heap, we mark that area as unusable, so
136 pointer retention by random pinning pointers is reduced.
138 *) experiment with max small object size (very small right now - 2kb,
139 because it's tied to the max freelist size)
141 *) add an option to mmap the whole heap in one chunk: it makes for many
142 simplifications in the checks (put the nursery at the top and just use a single
143 check for inclusion/exclusion): the issue this has is that on 32 bit systems it's
144 not flexible (too much of the address space may be used by default or we can't
145 increase the heap as needed) and we'd need a race-free mechanism to return memory
146 back to the system (mprotect(PROT_NONE) will still keep the memory allocated if it
147 was written to, munmap is needed, but the following mmap may not find the same segment
150 *) memzero the major fragments after restarting the world and optionally a smaller
153 *) investigate having fragment zeroing threads
155 *) separate locks for finalization and other minor stuff to reduce
158 *) try a different copying order to improve memory locality
160 *) a thread abort after a store but before the write barrier will
161 prevent the write barrier from executing
163 *) specialized dynamically generated markers/copiers
165 *) Dynamically adjust TLAB size to the number of threads. If we have
166 too many threads that do allocation, we might need smaller TLABs,
167 and we might get better performance with larger TLABs if we only
168 have a handful of threads. We could sum up the space left in all
169 assigned TLABs and if that's more than some percentage of the
170 nursery size, reduce the TLAB size.
172 *) Explore placing unreachable objects on unused nursery memory.
173 Instead of memset'ng a region to zero, place an int[] covering it.
174 A good place to start is add_nursery_frag. The tricky thing here is
175 placing those objects atomically outside of a collection.
185 #include <semaphore.h>
194 #define _XOPEN_SOURCE
196 #include "metadata/metadata-internals.h"
197 #include "metadata/class-internals.h"
198 #include "metadata/gc-internal.h"
199 #include "metadata/object-internals.h"
200 #include "metadata/threads.h"
201 #include "metadata/sgen-gc.h"
202 #include "metadata/sgen-cardtable.h"
203 #include "metadata/sgen-archdep.h"
204 #include "metadata/mono-gc.h"
205 #include "metadata/method-builder.h"
206 #include "metadata/profiler-private.h"
207 #include "metadata/monitor.h"
208 #include "metadata/threadpool-internals.h"
209 #include "metadata/mempool-internals.h"
210 #include "metadata/marshal.h"
211 #include "utils/mono-mmap.h"
212 #include "utils/mono-time.h"
213 #include "utils/mono-semaphore.h"
214 #include "utils/mono-counters.h"
215 #include "utils/mono-proclib.h"
217 #include <mono/utils/memcheck.h>
219 #if defined(__MACH__)
220 #include "utils/mach-support.h"
223 #define OPDEF(a,b,c,d,e,f,g,h,i,j) \
227 #include "mono/cil/opcode.def"
233 #undef pthread_create
235 #undef pthread_detach
238 * ######################################################################
239 * ######## Types and constants used by the GC.
240 * ######################################################################
243 static int gc_initialized = 0;
244 /* If set, do a minor collection before every allocation */
245 static gboolean collect_before_allocs = FALSE;
246 /* If set, do a heap consistency check before each minor collection */
247 static gboolean consistency_check_at_minor_collection = FALSE;
248 /* If set, check that there are no references to the domain left at domain unload */
249 static gboolean xdomain_checks = FALSE;
250 /* If not null, dump the heap after each collection into this file */
251 static FILE *heap_dump_file = NULL;
252 /* If set, mark stacks conservatively, even if precise marking is possible */
253 static gboolean conservative_stack_mark = TRUE;
254 /* If set, do a plausibility check on the scan_starts before and after
256 static gboolean do_scan_starts_check = FALSE;
258 #ifdef HEAVY_STATISTICS
259 static long long stat_objects_alloced = 0;
260 static long long stat_bytes_alloced = 0;
261 long long stat_objects_alloced_degraded = 0;
262 long long stat_bytes_alloced_degraded = 0;
263 static long long stat_bytes_alloced_los = 0;
265 long long stat_copy_object_called_nursery = 0;
266 long long stat_objects_copied_nursery = 0;
267 long long stat_copy_object_called_major = 0;
268 long long stat_objects_copied_major = 0;
270 long long stat_scan_object_called_nursery = 0;
271 long long stat_scan_object_called_major = 0;
273 long long stat_nursery_copy_object_failed_from_space = 0;
274 long long stat_nursery_copy_object_failed_forwarded = 0;
275 long long stat_nursery_copy_object_failed_pinned = 0;
277 static long long stat_store_remsets = 0;
278 static long long stat_store_remsets_unique = 0;
279 static long long stat_saved_remsets_1 = 0;
280 static long long stat_saved_remsets_2 = 0;
281 static long long stat_local_remsets_processed = 0;
282 static long long stat_global_remsets_added = 0;
283 static long long stat_global_remsets_readded = 0;
284 static long long stat_global_remsets_processed = 0;
285 static long long stat_global_remsets_discarded = 0;
287 static long long stat_wasted_fragments_used = 0;
288 static long long stat_wasted_fragments_bytes = 0;
290 static int stat_wbarrier_set_field = 0;
291 static int stat_wbarrier_set_arrayref = 0;
292 static int stat_wbarrier_arrayref_copy = 0;
293 static int stat_wbarrier_generic_store = 0;
294 static int stat_wbarrier_generic_store_remset = 0;
295 static int stat_wbarrier_set_root = 0;
296 static int stat_wbarrier_value_copy = 0;
297 static int stat_wbarrier_object_copy = 0;
300 static long long time_minor_pre_collection_fragment_clear = 0;
301 static long long time_minor_pinning = 0;
302 static long long time_minor_scan_remsets = 0;
303 static long long time_minor_scan_card_table = 0;
304 static long long time_minor_scan_pinned = 0;
305 static long long time_minor_scan_registered_roots = 0;
306 static long long time_minor_scan_thread_data = 0;
307 static long long time_minor_finish_gray_stack = 0;
308 static long long time_minor_fragment_creation = 0;
310 static long long time_major_pre_collection_fragment_clear = 0;
311 static long long time_major_pinning = 0;
312 static long long time_major_scan_pinned = 0;
313 static long long time_major_scan_registered_roots = 0;
314 static long long time_major_scan_thread_data = 0;
315 static long long time_major_scan_alloc_pinned = 0;
316 static long long time_major_scan_finalized = 0;
317 static long long time_major_scan_big_objects = 0;
318 static long long time_major_finish_gray_stack = 0;
319 static long long time_major_free_bigobjs = 0;
320 static long long time_major_los_sweep = 0;
321 static long long time_major_sweep = 0;
322 static long long time_major_fragment_creation = 0;
324 #define DEBUG(level,a) do {if (G_UNLIKELY ((level) <= SGEN_MAX_DEBUG_LEVEL && (level) <= gc_debug_level)) a;} while (0)
326 static int gc_debug_level = 0;
327 static FILE* gc_debug_file;
331 mono_gc_flush_info (void)
333 fflush (gc_debug_file);
338 * Define this to allow the user to change the nursery size by
339 * specifying its value in the MONO_GC_PARAMS environmental
340 * variable. See mono_gc_base_init for details.
342 #define USER_CONFIG 1
344 #define TV_DECLARE(name) gint64 name
345 #define TV_GETTIME(tv) tv = mono_100ns_ticks ()
346 #define TV_ELAPSED(start,end) (int)((end-start) / 10)
347 #define TV_ELAPSED_MS(start,end) ((TV_ELAPSED((start),(end)) + 500) / 1000)
349 #define ALIGN_TO(val,align) ((((guint64)val) + ((align) - 1)) & ~((align) - 1))
351 /* The method used to clear the nursery */
352 /* Clearing at nursery collections is the safest, but has bad interactions with caches.
353 * Clearing at TLAB creation is much faster, but more complex and it might expose hard
358 CLEAR_AT_TLAB_CREATION
359 } NurseryClearPolicy;
361 static NurseryClearPolicy nursery_clear_policy = CLEAR_AT_TLAB_CREATION;
364 * The young generation is divided into fragments. This is because
365 * we can hand one fragments to a thread for lock-less fast alloc and
366 * because the young generation ends up fragmented anyway by pinned objects.
367 * Once a collection is done, a list of fragments is created. When doing
368 * thread local alloc we use smallish nurseries so we allow new threads to
369 * allocate memory from gen0 without triggering a collection. Threads that
370 * are found to allocate lots of memory are given bigger fragments. This
371 * should make the finalizer thread use little nursery memory after a while.
372 * We should start assigning threads very small fragments: if there are many
373 * threads the nursery will be full of reserved space that the threads may not
374 * use at all, slowing down allocation speed.
375 * Thread local allocation is done from areas of memory Hotspot calls Thread Local
376 * Allocation Buffers (TLABs).
378 typedef struct _Fragment Fragment;
382 char *fragment_start;
383 char *fragment_limit; /* the current soft limit for allocation */
387 /* the runtime can register areas of memory as roots: we keep two lists of roots,
388 * a pinned root set for conservatively scanned roots and a normal one for
389 * precisely scanned roots (currently implemented as a single list).
391 typedef struct _RootRecord RootRecord;
400 * We're never actually using the first element. It's always set to
401 * NULL to simplify the elimination of consecutive duplicate
404 #define STORE_REMSET_BUFFER_SIZE 1024
406 typedef struct _GenericStoreRememberedSet GenericStoreRememberedSet;
407 struct _GenericStoreRememberedSet {
408 GenericStoreRememberedSet *next;
409 /* We need one entry less because the first entry of store
410 remset buffers is always a dummy and we don't copy it. */
411 gpointer data [STORE_REMSET_BUFFER_SIZE - 1];
414 /* we have 4 possible values in the low 2 bits */
416 REMSET_LOCATION, /* just a pointer to the exact location */
417 REMSET_RANGE, /* range of pointer fields */
418 REMSET_OBJECT, /* mark all the object for scanning */
419 REMSET_VTYPE, /* a valuetype array described by a gc descriptor and a count */
420 REMSET_TYPE_MASK = 0x3
423 #ifdef HAVE_KW_THREAD
424 static __thread RememberedSet *remembered_set MONO_TLS_FAST;
426 static pthread_key_t remembered_set_key;
427 static RememberedSet *global_remset;
428 static RememberedSet *freed_thread_remsets;
429 static GenericStoreRememberedSet *generic_store_remsets = NULL;
431 /*A two slots cache for recently inserted remsets */
432 static gpointer global_remset_cache [2];
434 /* FIXME: later choose a size that takes into account the RememberedSet struct
435 * and doesn't waste any alloc paddin space.
437 #define DEFAULT_REMSET_SIZE 1024
438 static RememberedSet* alloc_remset (int size, gpointer id);
440 #define object_is_forwarded SGEN_OBJECT_IS_FORWARDED
441 #define object_is_pinned SGEN_OBJECT_IS_PINNED
442 #define pin_object SGEN_PIN_OBJECT
443 #define unpin_object SGEN_UNPIN_OBJECT
445 #define ptr_in_nursery(p) (SGEN_PTR_IN_NURSERY ((p), DEFAULT_NURSERY_BITS, nursery_start, nursery_real_end))
447 #define LOAD_VTABLE SGEN_LOAD_VTABLE
450 safe_name (void* obj)
452 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (obj);
453 return vt->klass->name;
456 #define safe_object_get_size mono_sgen_safe_object_get_size
459 * ######################################################################
460 * ######## Global data.
461 * ######################################################################
463 static LOCK_DECLARE (gc_mutex);
464 static int gc_disabled = 0;
465 static int num_minor_gcs = 0;
466 static int num_major_gcs = 0;
468 static gboolean use_cardtable;
472 /* good sizes are 512KB-1MB: larger ones increase a lot memzeroing time */
473 #define DEFAULT_NURSERY_SIZE (default_nursery_size)
474 static int default_nursery_size = (1 << 22);
475 #ifdef SGEN_ALIGN_NURSERY
476 /* The number of trailing 0 bits in DEFAULT_NURSERY_SIZE */
477 #define DEFAULT_NURSERY_BITS (default_nursery_bits)
478 static int default_nursery_bits = 22;
483 #define DEFAULT_NURSERY_SIZE (4*1024*1024)
484 #ifdef SGEN_ALIGN_NURSERY
485 #define DEFAULT_NURSERY_BITS 22
490 #ifndef SGEN_ALIGN_NURSERY
491 #define DEFAULT_NURSERY_BITS -1
494 #define MIN_MINOR_COLLECTION_ALLOWANCE (DEFAULT_NURSERY_SIZE * 4)
496 #define SCAN_START_SIZE SGEN_SCAN_START_SIZE
498 /* the minimum size of a fragment that we consider useful for allocation */
499 #define FRAGMENT_MIN_SIZE (512)
501 static mword pagesize = 4096;
502 static mword nursery_size;
503 static int degraded_mode = 0;
505 static mword total_alloc = 0;
506 /* use this to tune when to do a major/minor collection */
507 static mword memory_pressure = 0;
508 static mword minor_collection_allowance;
509 static int minor_collection_sections_alloced = 0;
511 static GCMemSection *nursery_section = NULL;
512 static mword lowest_heap_address = ~(mword)0;
513 static mword highest_heap_address = 0;
515 static LOCK_DECLARE (interruption_mutex);
516 static LOCK_DECLARE (global_remset_mutex);
518 #define LOCK_GLOBAL_REMSET pthread_mutex_lock (&global_remset_mutex)
519 #define UNLOCK_GLOBAL_REMSET pthread_mutex_unlock (&global_remset_mutex)
521 typedef struct _FinalizeEntry FinalizeEntry;
522 struct _FinalizeEntry {
527 typedef struct _FinalizeEntryHashTable FinalizeEntryHashTable;
528 struct _FinalizeEntryHashTable {
529 FinalizeEntry **table;
534 typedef struct _DisappearingLink DisappearingLink;
535 struct _DisappearingLink {
536 DisappearingLink *next;
540 typedef struct _DisappearingLinkHashTable DisappearingLinkHashTable;
541 struct _DisappearingLinkHashTable {
542 DisappearingLink **table;
547 typedef struct _EphemeronLinkNode EphemeronLinkNode;
549 struct _EphemeronLinkNode {
550 EphemeronLinkNode *next;
565 int current_collection_generation = -1;
568 * The link pointer is hidden by negating each bit. We use the lowest
569 * bit of the link (before negation) to store whether it needs
570 * resurrection tracking.
572 #define HIDE_POINTER(p,t) ((gpointer)(~((gulong)(p)|((t)?1:0))))
573 #define REVEAL_POINTER(p) ((gpointer)((~(gulong)(p))&~3L))
575 #define DISLINK_OBJECT(d) (REVEAL_POINTER (*(d)->link))
576 #define DISLINK_TRACK(d) ((~(gulong)(*(d)->link)) & 1)
579 * The finalizable hash has the object as the key, the
580 * disappearing_link hash, has the link address as key.
582 static FinalizeEntryHashTable minor_finalizable_hash;
583 static FinalizeEntryHashTable major_finalizable_hash;
584 /* objects that are ready to be finalized */
585 static FinalizeEntry *fin_ready_list = NULL;
586 static FinalizeEntry *critical_fin_list = NULL;
588 static DisappearingLinkHashTable minor_disappearing_link_hash;
589 static DisappearingLinkHashTable major_disappearing_link_hash;
591 static EphemeronLinkNode *ephemeron_list;
593 static int num_ready_finalizers = 0;
594 static int no_finalize = 0;
597 ROOT_TYPE_NORMAL = 0, /* "normal" roots */
598 ROOT_TYPE_PINNED = 1, /* roots without a GC descriptor */
599 ROOT_TYPE_WBARRIER = 2, /* roots with a write barrier */
603 /* registered roots: the key to the hash is the root start address */
605 * Different kinds of roots are kept separate to speed up pin_from_roots () for example.
607 static RootRecord **roots_hash [ROOT_TYPE_NUM] = { NULL, NULL };
608 static int roots_hash_size [ROOT_TYPE_NUM] = { 0, 0, 0 };
609 static mword roots_size = 0; /* amount of memory in the root set */
610 static int num_roots_entries [ROOT_TYPE_NUM] = { 0, 0, 0 };
613 * The current allocation cursors
614 * We allocate objects in the nursery.
615 * The nursery is the area between nursery_start and nursery_real_end.
616 * Allocation is done from a Thread Local Allocation Buffer (TLAB). TLABs are allocated
617 * from nursery fragments.
618 * tlab_next is the pointer to the space inside the TLAB where the next object will
620 * tlab_temp_end is the pointer to the end of the temporary space reserved for
621 * the allocation: it allows us to set the scan starts at reasonable intervals.
622 * tlab_real_end points to the end of the TLAB.
623 * nursery_frag_real_end points to the end of the currently used nursery fragment.
624 * nursery_first_pinned_start points to the start of the first pinned object in the nursery
625 * nursery_last_pinned_end points to the end of the last pinned object in the nursery
626 * At the next allocation, the area of the nursery where objects can be present is
627 * between MIN(nursery_first_pinned_start, first_fragment_start) and
628 * MAX(nursery_last_pinned_end, nursery_frag_real_end)
630 static char *nursery_start = NULL;
632 #ifdef HAVE_KW_THREAD
633 #define TLAB_ACCESS_INIT
634 #define TLAB_START tlab_start
635 #define TLAB_NEXT tlab_next
636 #define TLAB_TEMP_END tlab_temp_end
637 #define TLAB_REAL_END tlab_real_end
638 #define REMEMBERED_SET remembered_set
639 #define STORE_REMSET_BUFFER store_remset_buffer
640 #define STORE_REMSET_BUFFER_INDEX store_remset_buffer_index
641 #define IN_CRITICAL_REGION thread_info->in_critical_region
643 static pthread_key_t thread_info_key;
644 #define TLAB_ACCESS_INIT SgenThreadInfo *__thread_info__ = pthread_getspecific (thread_info_key)
645 #define TLAB_START (__thread_info__->tlab_start)
646 #define TLAB_NEXT (__thread_info__->tlab_next)
647 #define TLAB_TEMP_END (__thread_info__->tlab_temp_end)
648 #define TLAB_REAL_END (__thread_info__->tlab_real_end)
649 #define REMEMBERED_SET (__thread_info__->remset)
650 #define STORE_REMSET_BUFFER (__thread_info__->store_remset_buffer)
651 #define STORE_REMSET_BUFFER_INDEX (__thread_info__->store_remset_buffer_index)
652 #define IN_CRITICAL_REGION (__thread_info__->in_critical_region)
655 /* we use the memory barrier only to prevent compiler reordering (a memory constraint may be enough) */
656 #define ENTER_CRITICAL_REGION do {IN_CRITICAL_REGION = 1;mono_memory_barrier ();} while (0)
657 #define EXIT_CRITICAL_REGION do {IN_CRITICAL_REGION = 0;mono_memory_barrier ();} while (0)
660 * FIXME: What is faster, a TLS variable pointing to a structure, or separate TLS
661 * variables for next+temp_end ?
663 #ifdef HAVE_KW_THREAD
664 static __thread SgenThreadInfo *thread_info;
665 static __thread char *tlab_start;
666 static __thread char *tlab_next;
667 static __thread char *tlab_temp_end;
668 static __thread char *tlab_real_end;
669 static __thread gpointer *store_remset_buffer;
670 static __thread long store_remset_buffer_index;
671 /* Used by the managed allocator/wbarrier */
672 static __thread char **tlab_next_addr;
673 static __thread char *stack_end;
674 static __thread long *store_remset_buffer_index_addr;
676 static char *nursery_next = NULL;
677 static char *nursery_frag_real_end = NULL;
678 static char *nursery_real_end = NULL;
679 static char *nursery_last_pinned_end = NULL;
681 /* The size of a TLAB */
682 /* The bigger the value, the less often we have to go to the slow path to allocate a new
683 * one, but the more space is wasted by threads not allocating much memory.
685 * FIXME: Make this self-tuning for each thread.
687 static guint32 tlab_size = (1024 * 4);
689 /*How much space is tolerable to be wasted from the current fragment when allocating a new TLAB*/
690 #define MAX_NURSERY_TLAB_WASTE 512
692 /* fragments that are free and ready to be used for allocation */
693 static Fragment *nursery_fragments = NULL;
694 /* freeelist of fragment structures */
695 static Fragment *fragment_freelist = NULL;
697 #define MAX_SMALL_OBJ_SIZE SGEN_MAX_SMALL_OBJ_SIZE
699 /* Functions supplied by the runtime to be called by the GC */
700 static MonoGCCallbacks gc_callbacks;
702 #define ALLOC_ALIGN SGEN_ALLOC_ALIGN
703 #define ALLOC_ALIGN_BITS SGEN_ALLOC_ALIGN_BITS
705 #define ALIGN_UP SGEN_ALIGN_UP
707 #define MOVED_OBJECTS_NUM 64
708 static void *moved_objects [MOVED_OBJECTS_NUM];
709 static int moved_objects_idx = 0;
711 /* Vtable of the objects used to fill out nursery fragments before a collection */
712 static MonoVTable *array_fill_vtable;
715 * ######################################################################
716 * ######## Macros and function declarations.
717 * ######################################################################
720 #define ADDR_IN_HEAP_BOUNDARIES(addr) ((p) >= lowest_heap_address && (p) < highest_heap_address)
723 align_pointer (void *ptr)
725 mword p = (mword)ptr;
726 p += sizeof (gpointer) - 1;
727 p &= ~ (sizeof (gpointer) - 1);
731 typedef SgenGrayQueue GrayQueue;
733 typedef void (*CopyOrMarkObjectFunc) (void**, GrayQueue*);
734 typedef char* (*ScanObjectFunc) (char*, GrayQueue*);
736 /* forward declarations */
737 static int stop_world (int generation);
738 static int restart_world (int generation);
739 static void scan_thread_data (void *start_nursery, void *end_nursery, gboolean precise);
740 static void scan_from_remsets (void *start_nursery, void *end_nursery, GrayQueue *queue);
741 static void scan_from_registered_roots (CopyOrMarkObjectFunc copy_func, char *addr_start, char *addr_end, int root_type, GrayQueue *queue);
742 static void scan_finalizer_entries (CopyOrMarkObjectFunc copy_func, FinalizeEntry *list, GrayQueue *queue);
743 static void find_pinning_ref_from_thread (char *obj, size_t size);
744 static void update_current_thread_stack (void *start);
745 static void finalize_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, int generation, GrayQueue *queue);
746 static void add_or_remove_disappearing_link (MonoObject *obj, void **link, gboolean track, int generation);
747 static void null_link_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, int generation, GrayQueue *queue);
748 static void null_links_for_domain (MonoDomain *domain, int generation);
749 static gboolean search_fragment_for_size (size_t size);
750 static int search_fragment_for_size_range (size_t desired_size, size_t minimum_size);
751 static void clear_nursery_fragments (char *next);
752 static void pin_from_roots (void *start_nursery, void *end_nursery);
753 static int pin_objects_from_addresses (GCMemSection *section, void **start, void **end, void *start_nursery, void *end_nursery, GrayQueue *queue);
754 static void optimize_pin_queue (int start_slot);
755 static void clear_remsets (void);
756 static void clear_tlabs (void);
757 static void sort_addresses (void **array, int size);
758 static void drain_gray_stack (GrayQueue *queue);
759 static void finish_gray_stack (char *start_addr, char *end_addr, int generation, GrayQueue *queue);
760 static gboolean need_major_collection (void);
761 static void major_collection (const char *reason);
763 static void mono_gc_register_disappearing_link (MonoObject *obj, void **link, gboolean track);
765 void describe_ptr (char *ptr);
766 void check_object (char *start);
768 static void check_consistency (void);
769 static void check_major_refs (void);
770 static void check_scan_starts (void);
771 static void check_for_xdomain_refs (void);
772 static void dump_heap (const char *type, int num, const char *reason);
774 void mono_gc_scan_for_specific_ref (MonoObject *key);
776 static void init_stats (void);
778 static int mark_ephemerons_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, GrayQueue *queue);
779 static void clear_unreachable_ephemerons (CopyOrMarkObjectFunc copy_func, char *start, char *end, GrayQueue *queue);
780 static void null_ephemerons_for_domain (MonoDomain *domain);
782 SgenMajorCollector major_collector;
784 #include "sgen-protocol.c"
785 #include "sgen-pinning.c"
786 #include "sgen-pinning-stats.c"
787 #include "sgen-gray.c"
788 #include "sgen-workers.c"
789 #include "sgen-los.c"
790 #include "sgen-cardtable.c"
792 /* Root bitmap descriptors are simpler: the lower three bits describe the type
793 * and we either have 30/62 bitmap bits or nibble-based run-length,
794 * or a complex descriptor, or a user defined marker function.
797 ROOT_DESC_CONSERVATIVE, /* 0, so matches NULL value */
802 ROOT_DESC_TYPE_MASK = 0x7,
803 ROOT_DESC_TYPE_SHIFT = 3,
806 #define MAKE_ROOT_DESC(type,val) ((type) | ((val) << ROOT_DESC_TYPE_SHIFT))
808 #define MAX_USER_DESCRIPTORS 16
810 static gsize* complex_descriptors = NULL;
811 static int complex_descriptors_size = 0;
812 static int complex_descriptors_next = 0;
813 static MonoGCRootMarkFunc user_descriptors [MAX_USER_DESCRIPTORS];
814 static int user_descriptors_next = 0;
817 alloc_complex_descriptor (gsize *bitmap, int numbits)
821 numbits = ALIGN_TO (numbits, GC_BITS_PER_WORD);
822 nwords = numbits / GC_BITS_PER_WORD + 1;
825 res = complex_descriptors_next;
826 /* linear search, so we don't have duplicates with domain load/unload
827 * this should not be performance critical or we'd have bigger issues
828 * (the number and size of complex descriptors should be small).
830 for (i = 0; i < complex_descriptors_next; ) {
831 if (complex_descriptors [i] == nwords) {
833 for (j = 0; j < nwords - 1; ++j) {
834 if (complex_descriptors [i + 1 + j] != bitmap [j]) {
844 i += complex_descriptors [i];
846 if (complex_descriptors_next + nwords > complex_descriptors_size) {
847 int new_size = complex_descriptors_size * 2 + nwords;
848 complex_descriptors = g_realloc (complex_descriptors, new_size * sizeof (gsize));
849 complex_descriptors_size = new_size;
851 DEBUG (6, fprintf (gc_debug_file, "Complex descriptor %d, size: %d (total desc memory: %d)\n", res, nwords, complex_descriptors_size));
852 complex_descriptors_next += nwords;
853 complex_descriptors [res] = nwords;
854 for (i = 0; i < nwords - 1; ++i) {
855 complex_descriptors [res + 1 + i] = bitmap [i];
856 DEBUG (6, fprintf (gc_debug_file, "\tvalue: %p\n", (void*)complex_descriptors [res + 1 + i]));
863 mono_sgen_get_complex_descriptor (GCVTable *vt)
865 return complex_descriptors + (vt->desc >> LOW_TYPE_BITS);
869 * Descriptor builders.
872 mono_gc_make_descr_for_string (gsize *bitmap, int numbits)
874 return (void*) DESC_TYPE_RUN_LENGTH;
878 mono_gc_make_descr_for_object (gsize *bitmap, int numbits, size_t obj_size)
880 int first_set = -1, num_set = 0, last_set = -1, i;
882 size_t stored_size = obj_size;
883 for (i = 0; i < numbits; ++i) {
884 if (bitmap [i / GC_BITS_PER_WORD] & ((gsize)1 << (i % GC_BITS_PER_WORD))) {
892 * We don't encode the size of types that don't contain
893 * references because they might not be aligned, i.e. the
894 * bottom two bits might be set, which would clash with the
895 * bits we need to encode the descriptor type. Since we don't
896 * use the encoded size to skip objects, other than for
897 * processing remsets, in which case only the positions of
898 * references are relevant, this is not a problem.
901 return (void*)DESC_TYPE_RUN_LENGTH;
902 g_assert (!(stored_size & 0x3));
903 if (stored_size <= MAX_SMALL_OBJ_SIZE) {
904 /* check run-length encoding first: one byte offset, one byte number of pointers
905 * on 64 bit archs, we can have 3 runs, just one on 32.
906 * It may be better to use nibbles.
909 desc = DESC_TYPE_RUN_LENGTH | (stored_size << 1);
910 DEBUG (6, fprintf (gc_debug_file, "Ptrfree descriptor %p, size: %zd\n", (void*)desc, stored_size));
912 } else if (first_set < 256 && num_set < 256 && (first_set + num_set == last_set + 1)) {
913 desc = DESC_TYPE_RUN_LENGTH | (stored_size << 1) | (first_set << 16) | (num_set << 24);
914 DEBUG (6, fprintf (gc_debug_file, "Runlen descriptor %p, size: %zd, first set: %d, num set: %d\n", (void*)desc, stored_size, first_set, num_set));
917 /* we know the 2-word header is ptr-free */
918 if (last_set < SMALL_BITMAP_SIZE + OBJECT_HEADER_WORDS) {
919 desc = DESC_TYPE_SMALL_BITMAP | (stored_size << 1) | ((*bitmap >> OBJECT_HEADER_WORDS) << SMALL_BITMAP_SHIFT);
920 DEBUG (6, fprintf (gc_debug_file, "Smallbitmap descriptor %p, size: %zd, last set: %d\n", (void*)desc, stored_size, last_set));
924 /* we know the 2-word header is ptr-free */
925 if (last_set < LARGE_BITMAP_SIZE + OBJECT_HEADER_WORDS) {
926 desc = DESC_TYPE_LARGE_BITMAP | ((*bitmap >> OBJECT_HEADER_WORDS) << LOW_TYPE_BITS);
927 DEBUG (6, fprintf (gc_debug_file, "Largebitmap descriptor %p, size: %zd, last set: %d\n", (void*)desc, stored_size, last_set));
930 /* it's a complex object ... */
931 desc = DESC_TYPE_COMPLEX | (alloc_complex_descriptor (bitmap, last_set + 1) << LOW_TYPE_BITS);
935 /* If the array holds references, numbits == 1 and the first bit is set in elem_bitmap */
937 mono_gc_make_descr_for_array (int vector, gsize *elem_bitmap, int numbits, size_t elem_size)
939 int first_set = -1, num_set = 0, last_set = -1, i;
940 mword desc = vector? DESC_TYPE_VECTOR: DESC_TYPE_ARRAY;
941 for (i = 0; i < numbits; ++i) {
942 if (elem_bitmap [i / GC_BITS_PER_WORD] & ((gsize)1 << (i % GC_BITS_PER_WORD))) {
949 /* See comment at the definition of DESC_TYPE_RUN_LENGTH. */
951 return (void*)DESC_TYPE_RUN_LENGTH;
952 if (elem_size <= MAX_ELEMENT_SIZE) {
953 desc |= elem_size << VECTOR_ELSIZE_SHIFT;
955 return (void*)(desc | VECTOR_SUBTYPE_PTRFREE);
957 /* Note: we also handle structs with just ref fields */
958 if (num_set * sizeof (gpointer) == elem_size) {
959 return (void*)(desc | VECTOR_SUBTYPE_REFS | ((gssize)(-1) << 16));
961 /* FIXME: try run-len first */
962 /* Note: we can't skip the object header here, because it's not present */
963 if (last_set <= SMALL_BITMAP_SIZE) {
964 return (void*)(desc | VECTOR_SUBTYPE_BITMAP | (*elem_bitmap << 16));
967 /* it's am array of complex structs ... */
968 desc = DESC_TYPE_COMPLEX_ARR;
969 desc |= alloc_complex_descriptor (elem_bitmap, last_set + 1) << LOW_TYPE_BITS;
973 /* Return the bitmap encoded by a descriptor */
975 mono_gc_get_bitmap_for_descr (void *descr, int *numbits)
977 mword d = (mword)descr;
981 case DESC_TYPE_RUN_LENGTH: {
982 int first_set = (d >> 16) & 0xff;
983 int num_set = (d >> 24) & 0xff;
986 bitmap = g_new0 (gsize, (first_set + num_set + 7) / 8);
988 for (i = first_set; i < first_set + num_set; ++i)
989 bitmap [i / GC_BITS_PER_WORD] |= ((gsize)1 << (i % GC_BITS_PER_WORD));
991 *numbits = first_set + num_set;
995 case DESC_TYPE_SMALL_BITMAP:
996 bitmap = g_new0 (gsize, 1);
998 bitmap [0] = (d >> SMALL_BITMAP_SHIFT) << OBJECT_HEADER_WORDS;
1000 *numbits = GC_BITS_PER_WORD;
1004 g_assert_not_reached ();
1009 is_xdomain_ref_allowed (gpointer *ptr, char *obj, MonoDomain *domain)
1011 MonoObject *o = (MonoObject*)(obj);
1012 MonoObject *ref = (MonoObject*)*(ptr);
1013 int offset = (char*)(ptr) - (char*)o;
1015 if (o->vtable->klass == mono_defaults.thread_class && offset == G_STRUCT_OFFSET (MonoThread, internal_thread))
1017 if (o->vtable->klass == mono_defaults.internal_thread_class && offset == G_STRUCT_OFFSET (MonoInternalThread, current_appcontext))
1019 if (mono_class_has_parent (o->vtable->klass, mono_defaults.real_proxy_class) &&
1020 offset == G_STRUCT_OFFSET (MonoRealProxy, unwrapped_server))
1022 /* Thread.cached_culture_info */
1023 if (!strcmp (ref->vtable->klass->name_space, "System.Globalization") &&
1024 !strcmp (ref->vtable->klass->name, "CultureInfo") &&
1025 !strcmp(o->vtable->klass->name_space, "System") &&
1026 !strcmp(o->vtable->klass->name, "Object[]"))
1029 * at System.IO.MemoryStream.InternalConstructor (byte[],int,int,bool,bool) [0x0004d] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.IO/MemoryStream.cs:121
1030 * at System.IO.MemoryStream..ctor (byte[]) [0x00017] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.IO/MemoryStream.cs:81
1031 * at (wrapper remoting-invoke-with-check) System.IO.MemoryStream..ctor (byte[]) <IL 0x00020, 0xffffffff>
1032 * at System.Runtime.Remoting.Messaging.CADMethodCallMessage.GetArguments () [0x0000d] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Messaging/CADMessages.cs:327
1033 * at System.Runtime.Remoting.Messaging.MethodCall..ctor (System.Runtime.Remoting.Messaging.CADMethodCallMessage) [0x00017] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Messaging/MethodCall.cs:87
1034 * at System.AppDomain.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage,byte[]&,System.Runtime.Remoting.Messaging.CADMethodReturnMessage&) [0x00018] in /home/schani/Work/novell/trunk/mcs/class/corlib/System/AppDomain.cs:1213
1035 * at (wrapper remoting-invoke-with-check) System.AppDomain.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage,byte[]&,System.Runtime.Remoting.Messaging.CADMethodReturnMessage&) <IL 0x0003d, 0xffffffff>
1036 * at System.Runtime.Remoting.Channels.CrossAppDomainSink.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage) [0x00008] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Channels/CrossAppDomainChannel.cs:198
1037 * at (wrapper runtime-invoke) object.runtime_invoke_CrossAppDomainSink/ProcessMessageRes_object_object (object,intptr,intptr,intptr) <IL 0x0004c, 0xffffffff>
1039 if (!strcmp (ref->vtable->klass->name_space, "System") &&
1040 !strcmp (ref->vtable->klass->name, "Byte[]") &&
1041 !strcmp (o->vtable->klass->name_space, "System.IO") &&
1042 !strcmp (o->vtable->klass->name, "MemoryStream"))
1044 /* append_job() in threadpool.c */
1045 if (!strcmp (ref->vtable->klass->name_space, "System.Runtime.Remoting.Messaging") &&
1046 !strcmp (ref->vtable->klass->name, "AsyncResult") &&
1047 !strcmp (o->vtable->klass->name_space, "System") &&
1048 !strcmp (o->vtable->klass->name, "Object[]") &&
1049 mono_thread_pool_is_queue_array ((MonoArray*) o))
1055 check_reference_for_xdomain (gpointer *ptr, char *obj, MonoDomain *domain)
1057 MonoObject *o = (MonoObject*)(obj);
1058 MonoObject *ref = (MonoObject*)*(ptr);
1059 int offset = (char*)(ptr) - (char*)o;
1061 MonoClassField *field;
1064 if (!ref || ref->vtable->domain == domain)
1066 if (is_xdomain_ref_allowed (ptr, obj, domain))
1070 for (class = o->vtable->klass; class; class = class->parent) {
1073 for (i = 0; i < class->field.count; ++i) {
1074 if (class->fields[i].offset == offset) {
1075 field = &class->fields[i];
1083 if (ref->vtable->klass == mono_defaults.string_class)
1084 str = mono_string_to_utf8 ((MonoString*)ref);
1087 g_print ("xdomain reference in %p (%s.%s) at offset %d (%s) to %p (%s.%s) (%s) - pointed to by:\n",
1088 o, o->vtable->klass->name_space, o->vtable->klass->name,
1089 offset, field ? field->name : "",
1090 ref, ref->vtable->klass->name_space, ref->vtable->klass->name, str ? str : "");
1091 mono_gc_scan_for_specific_ref (o);
1097 #define HANDLE_PTR(ptr,obj) check_reference_for_xdomain ((ptr), (obj), domain)
1100 scan_object_for_xdomain_refs (char *start, mword size, void *data)
1102 MonoDomain *domain = ((MonoObject*)start)->vtable->domain;
1104 #include "sgen-scan-object.h"
1108 #define HANDLE_PTR(ptr,obj) do { \
1109 if ((MonoObject*)*(ptr) == key) { \
1110 g_print ("found ref to %p in object %p (%s) at offset %td\n", \
1111 key, (obj), safe_name ((obj)), ((char*)(ptr) - (char*)(obj))); \
1116 scan_object_for_specific_ref (char *start, MonoObject *key)
1118 #include "sgen-scan-object.h"
1122 mono_sgen_scan_area_with_callback (char *start, char *end, IterateObjectCallbackFunc callback, void *data)
1124 while (start < end) {
1126 if (!*(void**)start) {
1127 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
1131 size = ALIGN_UP (safe_object_get_size ((MonoObject*) start));
1133 callback (start, size, data);
1140 scan_object_for_specific_ref_callback (char *obj, size_t size, MonoObject *key)
1142 scan_object_for_specific_ref (obj, key);
1146 check_root_obj_specific_ref (RootRecord *root, MonoObject *key, MonoObject *obj)
1150 g_print ("found ref to %p in root record %p\n", key, root);
1153 static MonoObject *check_key = NULL;
1154 static RootRecord *check_root = NULL;
1157 check_root_obj_specific_ref_from_marker (void **obj)
1159 check_root_obj_specific_ref (check_root, check_key, *obj);
1163 scan_roots_for_specific_ref (MonoObject *key, int root_type)
1168 for (i = 0; i < roots_hash_size [root_type]; ++i) {
1169 for (root = roots_hash [root_type][i]; root; root = root->next) {
1170 void **start_root = (void**)root->start_root;
1171 mword desc = root->root_desc;
1175 switch (desc & ROOT_DESC_TYPE_MASK) {
1176 case ROOT_DESC_BITMAP:
1177 desc >>= ROOT_DESC_TYPE_SHIFT;
1180 check_root_obj_specific_ref (root, key, *start_root);
1185 case ROOT_DESC_COMPLEX: {
1186 gsize *bitmap_data = complex_descriptors + (desc >> ROOT_DESC_TYPE_SHIFT);
1187 int bwords = (*bitmap_data) - 1;
1188 void **start_run = start_root;
1190 while (bwords-- > 0) {
1191 gsize bmap = *bitmap_data++;
1192 void **objptr = start_run;
1195 check_root_obj_specific_ref (root, key, *objptr);
1199 start_run += GC_BITS_PER_WORD;
1203 case ROOT_DESC_USER: {
1204 MonoGCRootMarkFunc marker = user_descriptors [desc >> ROOT_DESC_TYPE_SHIFT];
1205 marker (start_root, check_root_obj_specific_ref_from_marker);
1208 case ROOT_DESC_RUN_LEN:
1209 g_assert_not_reached ();
1211 g_assert_not_reached ();
1220 mono_gc_scan_for_specific_ref (MonoObject *key)
1226 mono_sgen_scan_area_with_callback (nursery_section->data, nursery_section->end_data,
1227 (IterateObjectCallbackFunc)scan_object_for_specific_ref_callback, key);
1229 major_collector.iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)scan_object_for_specific_ref_callback, key);
1231 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1232 scan_object_for_specific_ref (bigobj->data, key);
1234 scan_roots_for_specific_ref (key, ROOT_TYPE_NORMAL);
1235 scan_roots_for_specific_ref (key, ROOT_TYPE_WBARRIER);
1237 for (i = 0; i < roots_hash_size [ROOT_TYPE_PINNED]; ++i) {
1238 for (root = roots_hash [ROOT_TYPE_PINNED][i]; root; root = root->next) {
1239 void **ptr = (void**)root->start_root;
1241 while (ptr < (void**)root->end_root) {
1242 check_root_obj_specific_ref (root, *ptr, key);
1250 clear_current_nursery_fragment (char *next)
1252 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION) {
1253 g_assert (next <= nursery_frag_real_end);
1254 DEBUG (4, fprintf (gc_debug_file, "Clear nursery frag %p-%p\n", next, nursery_frag_real_end));
1255 memset (next, 0, nursery_frag_real_end - next);
1259 /* Clear all remaining nursery fragments */
1261 clear_nursery_fragments (char *next)
1264 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION) {
1265 clear_current_nursery_fragment (next);
1266 for (frag = nursery_fragments; frag; frag = frag->next) {
1267 DEBUG (4, fprintf (gc_debug_file, "Clear nursery frag %p-%p\n", frag->fragment_start, frag->fragment_end));
1268 memset (frag->fragment_start, 0, frag->fragment_end - frag->fragment_start);
1274 need_remove_object_for_domain (char *start, MonoDomain *domain)
1276 if (mono_object_domain (start) == domain) {
1277 DEBUG (4, fprintf (gc_debug_file, "Need to cleanup object %p\n", start));
1278 binary_protocol_cleanup (start, (gpointer)LOAD_VTABLE (start), safe_object_get_size ((MonoObject*)start));
1285 process_object_for_domain_clearing (char *start, MonoDomain *domain)
1287 GCVTable *vt = (GCVTable*)LOAD_VTABLE (start);
1288 if (vt->klass == mono_defaults.internal_thread_class)
1289 g_assert (mono_object_domain (start) == mono_get_root_domain ());
1290 /* The object could be a proxy for an object in the domain
1292 if (mono_class_has_parent (vt->klass, mono_defaults.real_proxy_class)) {
1293 MonoObject *server = ((MonoRealProxy*)start)->unwrapped_server;
1295 /* The server could already have been zeroed out, so
1296 we need to check for that, too. */
1297 if (server && (!LOAD_VTABLE (server) || mono_object_domain (server) == domain)) {
1298 DEBUG (4, fprintf (gc_debug_file, "Cleaning up remote pointer in %p to object %p\n",
1300 ((MonoRealProxy*)start)->unwrapped_server = NULL;
1305 static MonoDomain *check_domain = NULL;
1308 check_obj_not_in_domain (void **o)
1310 g_assert (((MonoObject*)(*o))->vtable->domain != check_domain);
1314 scan_for_registered_roots_in_domain (MonoDomain *domain, int root_type)
1318 check_domain = domain;
1319 for (i = 0; i < roots_hash_size [root_type]; ++i) {
1320 for (root = roots_hash [root_type][i]; root; root = root->next) {
1321 void **start_root = (void**)root->start_root;
1322 mword desc = root->root_desc;
1324 /* The MonoDomain struct is allowed to hold
1325 references to objects in its own domain. */
1326 if (start_root == (void**)domain)
1329 switch (desc & ROOT_DESC_TYPE_MASK) {
1330 case ROOT_DESC_BITMAP:
1331 desc >>= ROOT_DESC_TYPE_SHIFT;
1333 if ((desc & 1) && *start_root)
1334 check_obj_not_in_domain (*start_root);
1339 case ROOT_DESC_COMPLEX: {
1340 gsize *bitmap_data = complex_descriptors + (desc >> ROOT_DESC_TYPE_SHIFT);
1341 int bwords = (*bitmap_data) - 1;
1342 void **start_run = start_root;
1344 while (bwords-- > 0) {
1345 gsize bmap = *bitmap_data++;
1346 void **objptr = start_run;
1348 if ((bmap & 1) && *objptr)
1349 check_obj_not_in_domain (*objptr);
1353 start_run += GC_BITS_PER_WORD;
1357 case ROOT_DESC_USER: {
1358 MonoGCRootMarkFunc marker = user_descriptors [desc >> ROOT_DESC_TYPE_SHIFT];
1359 marker (start_root, check_obj_not_in_domain);
1362 case ROOT_DESC_RUN_LEN:
1363 g_assert_not_reached ();
1365 g_assert_not_reached ();
1369 check_domain = NULL;
1373 check_for_xdomain_refs (void)
1377 mono_sgen_scan_area_with_callback (nursery_section->data, nursery_section->end_data,
1378 (IterateObjectCallbackFunc)scan_object_for_xdomain_refs, NULL);
1380 major_collector.iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)scan_object_for_xdomain_refs, NULL);
1382 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1383 scan_object_for_xdomain_refs (bigobj->data, bigobj->size, NULL);
1387 clear_domain_process_object (char *obj, MonoDomain *domain)
1391 process_object_for_domain_clearing (obj, domain);
1392 remove = need_remove_object_for_domain (obj, domain);
1394 if (remove && ((MonoObject*)obj)->synchronisation) {
1395 void **dislink = mono_monitor_get_object_monitor_weak_link ((MonoObject*)obj);
1397 mono_gc_register_disappearing_link (NULL, dislink, FALSE);
1404 clear_domain_process_minor_object_callback (char *obj, size_t size, MonoDomain *domain)
1406 if (clear_domain_process_object (obj, domain))
1407 memset (obj, 0, size);
1411 clear_domain_process_major_object_callback (char *obj, size_t size, MonoDomain *domain)
1413 clear_domain_process_object (obj, domain);
1417 clear_domain_free_major_non_pinned_object_callback (char *obj, size_t size, MonoDomain *domain)
1419 if (need_remove_object_for_domain (obj, domain))
1420 major_collector.free_non_pinned_object (obj, size);
1424 clear_domain_free_major_pinned_object_callback (char *obj, size_t size, MonoDomain *domain)
1426 if (need_remove_object_for_domain (obj, domain))
1427 major_collector.free_pinned_object (obj, size);
1431 * When appdomains are unloaded we can easily remove objects that have finalizers,
1432 * but all the others could still be present in random places on the heap.
1433 * We need a sweep to get rid of them even though it's going to be costly
1435 * The reason we need to remove them is because we access the vtable and class
1436 * structures to know the object size and the reference bitmap: once the domain is
1437 * unloaded the point to random memory.
1440 mono_gc_clear_domain (MonoDomain * domain)
1442 LOSObject *bigobj, *prev;
1447 clear_nursery_fragments (nursery_next);
1449 if (xdomain_checks && domain != mono_get_root_domain ()) {
1450 scan_for_registered_roots_in_domain (domain, ROOT_TYPE_NORMAL);
1451 scan_for_registered_roots_in_domain (domain, ROOT_TYPE_WBARRIER);
1452 check_for_xdomain_refs ();
1455 mono_sgen_scan_area_with_callback (nursery_section->data, nursery_section->end_data,
1456 (IterateObjectCallbackFunc)clear_domain_process_minor_object_callback, domain);
1458 /*Ephemerons and dislinks must be processed before LOS since they might end up pointing
1459 to memory returned to the OS.*/
1460 null_ephemerons_for_domain (domain);
1462 for (i = GENERATION_NURSERY; i < GENERATION_MAX; ++i)
1463 null_links_for_domain (domain, i);
1465 /* We need two passes over major and large objects because
1466 freeing such objects might give their memory back to the OS
1467 (in the case of large objects) or obliterate its vtable
1468 (pinned objects with major-copying or pinned and non-pinned
1469 objects with major-mark&sweep), but we might need to
1470 dereference a pointer from an object to another object if
1471 the first object is a proxy. */
1472 major_collector.iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)clear_domain_process_major_object_callback, domain);
1473 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1474 clear_domain_process_object (bigobj->data, domain);
1477 for (bigobj = los_object_list; bigobj;) {
1478 if (need_remove_object_for_domain (bigobj->data, domain)) {
1479 LOSObject *to_free = bigobj;
1481 prev->next = bigobj->next;
1483 los_object_list = bigobj->next;
1484 bigobj = bigobj->next;
1485 DEBUG (4, fprintf (gc_debug_file, "Freeing large object %p\n",
1487 free_large_object (to_free);
1491 bigobj = bigobj->next;
1493 major_collector.iterate_objects (TRUE, FALSE, (IterateObjectCallbackFunc)clear_domain_free_major_non_pinned_object_callback, domain);
1494 major_collector.iterate_objects (FALSE, TRUE, (IterateObjectCallbackFunc)clear_domain_free_major_pinned_object_callback, domain);
1500 global_remset_cache_clear (void)
1502 memset (global_remset_cache, 0, sizeof (global_remset_cache));
1506 * Tries to check if a given remset location was already added to the global remset.
1509 * A 2 entry, LRU cache of recently saw location remsets.
1511 * It's hand-coded instead of done using loops to reduce the number of memory references on cache hit.
1513 * Returns TRUE is the element was added..
1516 global_remset_location_was_not_added (gpointer ptr)
1519 gpointer first = global_remset_cache [0], second;
1521 HEAVY_STAT (++stat_global_remsets_discarded);
1525 second = global_remset_cache [1];
1527 if (second == ptr) {
1528 /*Move the second to the front*/
1529 global_remset_cache [0] = second;
1530 global_remset_cache [1] = first;
1532 HEAVY_STAT (++stat_global_remsets_discarded);
1536 global_remset_cache [0] = second;
1537 global_remset_cache [1] = ptr;
1542 * mono_sgen_add_to_global_remset:
1544 * The global remset contains locations which point into newspace after
1545 * a minor collection. This can happen if the objects they point to are pinned.
1547 * LOCKING: If called from a parallel collector, the global remset
1548 * lock must be held. For serial collectors that is not necessary.
1551 mono_sgen_add_to_global_remset (gpointer ptr)
1556 if (use_cardtable) {
1557 sgen_card_table_mark_address ((mword)ptr);
1561 g_assert (!ptr_in_nursery (ptr) && ptr_in_nursery (*(gpointer*)ptr));
1563 lock = (current_collection_generation == GENERATION_OLD && major_collector.is_parallel);
1567 if (!global_remset_location_was_not_added (ptr))
1570 DEBUG (8, fprintf (gc_debug_file, "Adding global remset for %p\n", ptr));
1571 binary_protocol_global_remset (ptr, *(gpointer*)ptr, (gpointer)LOAD_VTABLE (*(gpointer*)ptr));
1573 HEAVY_STAT (++stat_global_remsets_added);
1576 * FIXME: If an object remains pinned, we need to add it at every minor collection.
1577 * To avoid uncontrolled growth of the global remset, only add each pointer once.
1579 if (global_remset->store_next + 3 < global_remset->end_set) {
1580 *(global_remset->store_next++) = (mword)ptr;
1583 rs = alloc_remset (global_remset->end_set - global_remset->data, NULL);
1584 rs->next = global_remset;
1586 *(global_remset->store_next++) = (mword)ptr;
1589 int global_rs_size = 0;
1591 for (rs = global_remset; rs; rs = rs->next) {
1592 global_rs_size += rs->store_next - rs->data;
1594 DEBUG (4, fprintf (gc_debug_file, "Global remset now has size %d\n", global_rs_size));
1599 UNLOCK_GLOBAL_REMSET;
1605 * Scan objects in the gray stack until the stack is empty. This should be called
1606 * frequently after each object is copied, to achieve better locality and cache
1610 drain_gray_stack (GrayQueue *queue)
1614 if (current_collection_generation == GENERATION_NURSERY) {
1616 GRAY_OBJECT_DEQUEUE (queue, obj);
1619 DEBUG (9, fprintf (gc_debug_file, "Precise gray object scan %p (%s)\n", obj, safe_name (obj)));
1620 major_collector.minor_scan_object (obj, queue);
1623 if (major_collector.is_parallel && queue == &workers_distribute_gray_queue)
1627 GRAY_OBJECT_DEQUEUE (queue, obj);
1630 DEBUG (9, fprintf (gc_debug_file, "Precise gray object scan %p (%s)\n", obj, safe_name (obj)));
1631 major_collector.major_scan_object (obj, queue);
1637 * Addresses from start to end are already sorted. This function finds
1638 * the object header for each address and pins the object. The
1639 * addresses must be inside the passed section. The (start of the)
1640 * address array is overwritten with the addresses of the actually
1641 * pinned objects. Return the number of pinned objects.
1644 pin_objects_from_addresses (GCMemSection *section, void **start, void **end, void *start_nursery, void *end_nursery, GrayQueue *queue)
1649 void *last_obj = NULL;
1650 size_t last_obj_size = 0;
1653 void **definitely_pinned = start;
1657 * The code below starts the search from an entry in scan_starts, which might point into a nursery
1658 * fragment containing random data. Clearing the nursery fragments takes a lot of time, and searching
1659 * though them too, so lay arrays at each location inside a fragment where a search can start:
1660 * - scan_locations[i]
1662 * - the start of each fragment (the last_obj + last_obj case)
1663 * The third encompasses the first two, since scan_locations [i] can't point inside a nursery fragment.
1665 for (frag = nursery_fragments; frag; frag = frag->next) {
1668 g_assert (frag->fragment_end - frag->fragment_start >= sizeof (MonoArray));
1669 o = (MonoArray*)frag->fragment_start;
1670 memset (o, 0, sizeof (MonoArray));
1671 g_assert (array_fill_vtable);
1672 o->obj.vtable = array_fill_vtable;
1673 /* Mark this as not a real object */
1674 o->obj.synchronisation = GINT_TO_POINTER (-1);
1675 o->max_length = (frag->fragment_end - frag->fragment_start) - sizeof (MonoArray);
1676 g_assert (frag->fragment_start + safe_object_get_size ((MonoObject*)o) == frag->fragment_end);
1679 while (start < end) {
1681 /* the range check should be reduntant */
1682 if (addr != last && addr >= start_nursery && addr < end_nursery) {
1683 DEBUG (5, fprintf (gc_debug_file, "Considering pinning addr %p\n", addr));
1684 /* multiple pointers to the same object */
1685 if (addr >= last_obj && (char*)addr < (char*)last_obj + last_obj_size) {
1689 idx = ((char*)addr - (char*)section->data) / SCAN_START_SIZE;
1690 g_assert (idx < section->num_scan_start);
1691 search_start = (void*)section->scan_starts [idx];
1692 if (!search_start || search_start > addr) {
1695 search_start = section->scan_starts [idx];
1696 if (search_start && search_start <= addr)
1699 if (!search_start || search_start > addr)
1700 search_start = start_nursery;
1702 if (search_start < last_obj)
1703 search_start = (char*)last_obj + last_obj_size;
1704 /* now addr should be in an object a short distance from search_start
1705 * Note that search_start must point to zeroed mem or point to an object.
1709 if (!*(void**)search_start) {
1710 /* Consistency check */
1712 for (frag = nursery_fragments; frag; frag = frag->next) {
1713 if (search_start >= frag->fragment_start && search_start < frag->fragment_end)
1714 g_assert_not_reached ();
1718 search_start = (void*)ALIGN_UP ((mword)search_start + sizeof (gpointer));
1721 last_obj = search_start;
1722 last_obj_size = ALIGN_UP (safe_object_get_size ((MonoObject*)search_start));
1724 if (((MonoObject*)last_obj)->synchronisation == GINT_TO_POINTER (-1)) {
1725 /* Marks the beginning of a nursery fragment, skip */
1727 DEBUG (8, fprintf (gc_debug_file, "Pinned try match %p (%s), size %zd\n", last_obj, safe_name (last_obj), last_obj_size));
1728 if (addr >= search_start && (char*)addr < (char*)last_obj + last_obj_size) {
1729 DEBUG (4, fprintf (gc_debug_file, "Pinned object %p, vtable %p (%s), count %d\n", search_start, *(void**)search_start, safe_name (search_start), count));
1730 binary_protocol_pin (search_start, (gpointer)LOAD_VTABLE (search_start), safe_object_get_size (search_start));
1731 pin_object (search_start);
1732 GRAY_OBJECT_ENQUEUE (queue, search_start);
1734 mono_sgen_pin_stats_register_object (search_start, last_obj_size);
1735 definitely_pinned [count] = search_start;
1740 /* skip to the next object */
1741 search_start = (void*)((char*)search_start + last_obj_size);
1742 } while (search_start <= addr);
1743 /* we either pinned the correct object or we ignored the addr because
1744 * it points to unused zeroed memory.
1750 //printf ("effective pinned: %d (at the end: %d)\n", count, (char*)end_nursery - (char*)last);
1755 mono_sgen_pin_objects_in_section (GCMemSection *section, GrayQueue *queue)
1757 int num_entries = section->pin_queue_num_entries;
1759 void **start = section->pin_queue_start;
1761 reduced_to = pin_objects_from_addresses (section, start, start + num_entries,
1762 section->data, section->next_data, queue);
1763 section->pin_queue_num_entries = reduced_to;
1765 section->pin_queue_start = NULL;
1769 /* Sort the addresses in array in increasing order.
1770 * Done using a by-the book heap sort. Which has decent and stable performance, is pretty cache efficient.
1773 sort_addresses (void **array, int size)
1778 for (i = 1; i < size; ++i) {
1781 int parent = (child - 1) / 2;
1783 if (array [parent] >= array [child])
1786 tmp = array [parent];
1787 array [parent] = array [child];
1788 array [child] = tmp;
1794 for (i = size - 1; i > 0; --i) {
1797 array [i] = array [0];
1803 while (root * 2 + 1 <= end) {
1804 int child = root * 2 + 1;
1806 if (child < end && array [child] < array [child + 1])
1808 if (array [root] >= array [child])
1812 array [root] = array [child];
1813 array [child] = tmp;
1820 static G_GNUC_UNUSED void
1821 print_nursery_gaps (void* start_nursery, void *end_nursery)
1824 gpointer first = start_nursery;
1826 for (i = 0; i < next_pin_slot; ++i) {
1827 next = pin_queue [i];
1828 fprintf (gc_debug_file, "Nursery range: %p-%p, size: %td\n", first, next, (char*)next-(char*)first);
1832 fprintf (gc_debug_file, "Nursery range: %p-%p, size: %td\n", first, next, (char*)next-(char*)first);
1835 /* reduce the info in the pin queue, removing duplicate pointers and sorting them */
1837 optimize_pin_queue (int start_slot)
1839 void **start, **cur, **end;
1840 /* sort and uniq pin_queue: we just sort and we let the rest discard multiple values */
1841 /* it may be better to keep ranges of pinned memory instead of individually pinning objects */
1842 DEBUG (5, fprintf (gc_debug_file, "Sorting pin queue, size: %d\n", next_pin_slot));
1843 if ((next_pin_slot - start_slot) > 1)
1844 sort_addresses (pin_queue + start_slot, next_pin_slot - start_slot);
1845 start = cur = pin_queue + start_slot;
1846 end = pin_queue + next_pin_slot;
1849 while (*start == *cur && cur < end)
1853 next_pin_slot = start - pin_queue;
1854 DEBUG (5, fprintf (gc_debug_file, "Pin queue reduced to size: %d\n", next_pin_slot));
1855 //DEBUG (6, print_nursery_gaps (start_nursery, end_nursery));
1860 * Scan the memory between start and end and queue values which could be pointers
1861 * to the area between start_nursery and end_nursery for later consideration.
1862 * Typically used for thread stacks.
1865 conservatively_pin_objects_from (void **start, void **end, void *start_nursery, void *end_nursery, int pin_type)
1868 while (start < end) {
1869 if (*start >= start_nursery && *start < end_nursery) {
1871 * *start can point to the middle of an object
1872 * note: should we handle pointing at the end of an object?
1873 * pinning in C# code disallows pointing at the end of an object
1874 * but there is some small chance that an optimizing C compiler
1875 * may keep the only reference to an object by pointing
1876 * at the end of it. We ignore this small chance for now.
1877 * Pointers to the end of an object are indistinguishable
1878 * from pointers to the start of the next object in memory
1879 * so if we allow that we'd need to pin two objects...
1880 * We queue the pointer in an array, the
1881 * array will then be sorted and uniqued. This way
1882 * we can coalesce several pinning pointers and it should
1883 * be faster since we'd do a memory scan with increasing
1884 * addresses. Note: we can align the address to the allocation
1885 * alignment, so the unique process is more effective.
1887 mword addr = (mword)*start;
1888 addr &= ~(ALLOC_ALIGN - 1);
1889 if (addr >= (mword)start_nursery && addr < (mword)end_nursery)
1890 pin_stage_ptr ((void*)addr);
1892 pin_stats_register_address ((char*)addr, pin_type);
1893 DEBUG (6, if (count) fprintf (gc_debug_file, "Pinning address %p\n", (void*)addr));
1898 DEBUG (7, if (count) fprintf (gc_debug_file, "found %d potential pinned heap pointers\n", count));
1902 * Debugging function: find in the conservative roots where @obj is being pinned.
1904 static G_GNUC_UNUSED void
1905 find_pinning_reference (char *obj, size_t size)
1909 char *endobj = obj + size;
1910 for (i = 0; i < roots_hash_size [0]; ++i) {
1911 for (root = roots_hash [0][i]; root; root = root->next) {
1912 /* if desc is non-null it has precise info */
1913 if (!root->root_desc) {
1914 char ** start = (char**)root->start_root;
1915 while (start < (char**)root->end_root) {
1916 if (*start >= obj && *start < endobj) {
1917 DEBUG (0, fprintf (gc_debug_file, "Object %p referenced in pinned roots %p-%p (at %p in record %p)\n", obj, root->start_root, root->end_root, start, root));
1924 find_pinning_ref_from_thread (obj, size);
1928 * The first thing we do in a collection is to identify pinned objects.
1929 * This function considers all the areas of memory that need to be
1930 * conservatively scanned.
1933 pin_from_roots (void *start_nursery, void *end_nursery)
1937 DEBUG (2, fprintf (gc_debug_file, "Scanning pinned roots (%d bytes, %d/%d entries)\n", (int)roots_size, num_roots_entries [ROOT_TYPE_NORMAL], num_roots_entries [ROOT_TYPE_PINNED]));
1938 /* objects pinned from the API are inside these roots */
1939 for (i = 0; i < roots_hash_size [ROOT_TYPE_PINNED]; ++i) {
1940 for (root = roots_hash [ROOT_TYPE_PINNED][i]; root; root = root->next) {
1941 DEBUG (6, fprintf (gc_debug_file, "Pinned roots %p-%p\n", root->start_root, root->end_root));
1942 conservatively_pin_objects_from ((void**)root->start_root, (void**)root->end_root, start_nursery, end_nursery, PIN_TYPE_OTHER);
1945 /* now deal with the thread stacks
1946 * in the future we should be able to conservatively scan only:
1947 * *) the cpu registers
1948 * *) the unmanaged stack frames
1949 * *) the _last_ managed stack frame
1950 * *) pointers slots in managed frames
1952 scan_thread_data (start_nursery, end_nursery, FALSE);
1954 evacuate_pin_staging_area ();
1957 static CopyOrMarkObjectFunc user_copy_or_mark_func;
1958 static GrayQueue *user_copy_or_mark_queue;
1961 single_arg_user_copy_or_mark (void **obj)
1963 user_copy_or_mark_func (obj, user_copy_or_mark_queue);
1967 * The memory area from start_root to end_root contains pointers to objects.
1968 * Their position is precisely described by @desc (this means that the pointer
1969 * can be either NULL or the pointer to the start of an object).
1970 * This functions copies them to to_space updates them.
1972 * This function is not thread-safe!
1975 precisely_scan_objects_from (CopyOrMarkObjectFunc copy_func, void** start_root, void** end_root, char* n_start, char *n_end, mword desc, GrayQueue *queue)
1977 switch (desc & ROOT_DESC_TYPE_MASK) {
1978 case ROOT_DESC_BITMAP:
1979 desc >>= ROOT_DESC_TYPE_SHIFT;
1981 if ((desc & 1) && *start_root) {
1982 copy_func (start_root, queue);
1983 DEBUG (9, fprintf (gc_debug_file, "Overwrote root at %p with %p\n", start_root, *start_root));
1984 drain_gray_stack (queue);
1990 case ROOT_DESC_COMPLEX: {
1991 gsize *bitmap_data = complex_descriptors + (desc >> ROOT_DESC_TYPE_SHIFT);
1992 int bwords = (*bitmap_data) - 1;
1993 void **start_run = start_root;
1995 while (bwords-- > 0) {
1996 gsize bmap = *bitmap_data++;
1997 void **objptr = start_run;
1999 if ((bmap & 1) && *objptr) {
2000 copy_func (objptr, queue);
2001 DEBUG (9, fprintf (gc_debug_file, "Overwrote root at %p with %p\n", objptr, *objptr));
2002 drain_gray_stack (queue);
2007 start_run += GC_BITS_PER_WORD;
2011 case ROOT_DESC_USER: {
2012 MonoGCRootMarkFunc marker = user_descriptors [desc >> ROOT_DESC_TYPE_SHIFT];
2013 user_copy_or_mark_func = copy_func;
2014 user_copy_or_mark_queue = queue;
2015 marker (start_root, single_arg_user_copy_or_mark);
2016 user_copy_or_mark_func = NULL;
2017 user_copy_or_mark_queue = NULL;
2020 case ROOT_DESC_RUN_LEN:
2021 g_assert_not_reached ();
2023 g_assert_not_reached ();
2028 mono_sgen_update_heap_boundaries (mword low, mword high)
2033 old = lowest_heap_address;
2036 } while (SGEN_CAS_PTR ((gpointer*)&lowest_heap_address, (gpointer)low, (gpointer)old) != (gpointer)old);
2039 old = highest_heap_address;
2042 } while (SGEN_CAS_PTR ((gpointer*)&highest_heap_address, (gpointer)high, (gpointer)old) != (gpointer)old);
2046 alloc_fragment (void)
2048 Fragment *frag = fragment_freelist;
2050 fragment_freelist = frag->next;
2054 frag = mono_sgen_alloc_internal (INTERNAL_MEM_FRAGMENT);
2059 /* size must be a power of 2 */
2061 mono_sgen_alloc_os_memory_aligned (mword size, mword alignment, gboolean activate)
2063 /* Allocate twice the memory to be able to put the block on an aligned address */
2064 char *mem = mono_sgen_alloc_os_memory (size + alignment, activate);
2069 aligned = (char*)((mword)(mem + (alignment - 1)) & ~(alignment - 1));
2070 g_assert (aligned >= mem && aligned + size <= mem + size + alignment && !((mword)aligned & (alignment - 1)));
2073 mono_sgen_free_os_memory (mem, aligned - mem);
2074 if (aligned + size < mem + size + alignment)
2075 mono_sgen_free_os_memory (aligned + size, (mem + size + alignment) - (aligned + size));
2081 * Allocate and setup the data structures needed to be able to allocate objects
2082 * in the nursery. The nursery is stored in nursery_section.
2085 alloc_nursery (void)
2087 GCMemSection *section;
2093 if (nursery_section)
2095 DEBUG (2, fprintf (gc_debug_file, "Allocating nursery size: %lu\n", (unsigned long)nursery_size));
2096 /* later we will alloc a larger area for the nursery but only activate
2097 * what we need. The rest will be used as expansion if we have too many pinned
2098 * objects in the existing nursery.
2100 /* FIXME: handle OOM */
2101 section = mono_sgen_alloc_internal (INTERNAL_MEM_SECTION);
2103 g_assert (nursery_size == DEFAULT_NURSERY_SIZE);
2104 alloc_size = nursery_size;
2105 #ifdef SGEN_ALIGN_NURSERY
2106 data = major_collector.alloc_heap (alloc_size, alloc_size, DEFAULT_NURSERY_BITS);
2108 data = major_collector.alloc_heap (alloc_size, 0, DEFAULT_NURSERY_BITS);
2110 nursery_start = data;
2111 nursery_real_end = nursery_start + nursery_size;
2112 mono_sgen_update_heap_boundaries ((mword)nursery_start, (mword)nursery_real_end);
2113 nursery_next = nursery_start;
2114 DEBUG (4, fprintf (gc_debug_file, "Expanding nursery size (%p-%p): %lu, total: %lu\n", data, data + alloc_size, (unsigned long)nursery_size, (unsigned long)total_alloc));
2115 section->data = section->next_data = data;
2116 section->size = alloc_size;
2117 section->end_data = nursery_real_end;
2118 scan_starts = (alloc_size + SCAN_START_SIZE - 1) / SCAN_START_SIZE;
2119 section->scan_starts = mono_sgen_alloc_internal_dynamic (sizeof (char*) * scan_starts, INTERNAL_MEM_SCAN_STARTS);
2120 section->num_scan_start = scan_starts;
2121 section->block.role = MEMORY_ROLE_GEN0;
2122 section->block.next = NULL;
2124 nursery_section = section;
2126 /* Setup the single first large fragment */
2127 frag = alloc_fragment ();
2128 frag->fragment_start = nursery_start;
2129 frag->fragment_limit = nursery_start;
2130 frag->fragment_end = nursery_real_end;
2131 nursery_frag_real_end = nursery_real_end;
2132 /* FIXME: frag here is lost */
2136 mono_gc_get_nursery (int *shift_bits, size_t *size)
2138 *size = nursery_size;
2139 #ifdef SGEN_ALIGN_NURSERY
2140 *shift_bits = DEFAULT_NURSERY_BITS;
2144 return nursery_start;
2148 scan_finalizer_entries (CopyOrMarkObjectFunc copy_func, FinalizeEntry *list, GrayQueue *queue)
2152 for (fin = list; fin; fin = fin->next) {
2155 DEBUG (5, fprintf (gc_debug_file, "Scan of fin ready object: %p (%s)\n", fin->object, safe_name (fin->object)));
2156 copy_func (&fin->object, queue);
2160 static mword fragment_total = 0;
2162 * We found a fragment of free memory in the nursery: memzero it and if
2163 * it is big enough, add it to the list of fragments that can be used for
2167 add_nursery_frag (size_t frag_size, char* frag_start, char* frag_end)
2170 DEBUG (4, fprintf (gc_debug_file, "Found empty fragment: %p-%p, size: %zd\n", frag_start, frag_end, frag_size));
2171 binary_protocol_empty (frag_start, frag_size);
2172 /* Not worth dealing with smaller fragments: need to tune */
2173 if (frag_size >= FRAGMENT_MIN_SIZE) {
2174 /* memsetting just the first chunk start is bound to provide better cache locality */
2175 if (nursery_clear_policy == CLEAR_AT_GC)
2176 memset (frag_start, 0, frag_size);
2178 fragment = alloc_fragment ();
2179 fragment->fragment_start = frag_start;
2180 fragment->fragment_limit = frag_start;
2181 fragment->fragment_end = frag_end;
2182 fragment->next = nursery_fragments;
2183 nursery_fragments = fragment;
2184 fragment_total += frag_size;
2186 /* Clear unused fragments, pinning depends on this */
2187 /*TODO place an int[] here instead of the memset if size justify it*/
2188 memset (frag_start, 0, frag_size);
2193 generation_name (int generation)
2195 switch (generation) {
2196 case GENERATION_NURSERY: return "nursery";
2197 case GENERATION_OLD: return "old";
2198 default: g_assert_not_reached ();
2202 static DisappearingLinkHashTable*
2203 get_dislink_hash_table (int generation)
2205 switch (generation) {
2206 case GENERATION_NURSERY: return &minor_disappearing_link_hash;
2207 case GENERATION_OLD: return &major_disappearing_link_hash;
2208 default: g_assert_not_reached ();
2212 static FinalizeEntryHashTable*
2213 get_finalize_entry_hash_table (int generation)
2215 switch (generation) {
2216 case GENERATION_NURSERY: return &minor_finalizable_hash;
2217 case GENERATION_OLD: return &major_finalizable_hash;
2218 default: g_assert_not_reached ();
2223 finish_gray_stack (char *start_addr, char *end_addr, int generation, GrayQueue *queue)
2228 int ephemeron_rounds = 0;
2229 CopyOrMarkObjectFunc copy_func = current_collection_generation == GENERATION_NURSERY ? major_collector.copy_object : major_collector.copy_or_mark_object;
2232 * We copied all the reachable objects. Now it's the time to copy
2233 * the objects that were not referenced by the roots, but by the copied objects.
2234 * we built a stack of objects pointed to by gray_start: they are
2235 * additional roots and we may add more items as we go.
2236 * We loop until gray_start == gray_objects which means no more objects have
2237 * been added. Note this is iterative: no recursion is involved.
2238 * We need to walk the LO list as well in search of marked big objects
2239 * (use a flag since this is needed only on major collections). We need to loop
2240 * here as well, so keep a counter of marked LO (increasing it in copy_object).
2241 * To achieve better cache locality and cache usage, we drain the gray stack
2242 * frequently, after each object is copied, and just finish the work here.
2244 drain_gray_stack (queue);
2246 DEBUG (2, fprintf (gc_debug_file, "%s generation done\n", generation_name (generation)));
2247 /* walk the finalization queue and move also the objects that need to be
2248 * finalized: use the finalized objects as new roots so the objects they depend
2249 * on are also not reclaimed. As with the roots above, only objects in the nursery
2250 * are marked/copied.
2251 * We need a loop here, since objects ready for finalizers may reference other objects
2252 * that are fin-ready. Speedup with a flag?
2256 * Walk the ephemeron tables marking all values with reachable keys. This must be completely done
2257 * before processing finalizable objects to avoid finalizing reachable values.
2259 * It must be done inside the finalizaters loop since objects must not be removed from CWT tables
2260 * while they are been finalized.
2262 int done_with_ephemerons = 0;
2264 done_with_ephemerons = mark_ephemerons_in_range (copy_func, start_addr, end_addr, queue);
2265 drain_gray_stack (queue);
2267 } while (!done_with_ephemerons);
2269 fin_ready = num_ready_finalizers;
2270 finalize_in_range (copy_func, start_addr, end_addr, generation, queue);
2271 if (generation == GENERATION_OLD)
2272 finalize_in_range (copy_func, nursery_start, nursery_real_end, GENERATION_NURSERY, queue);
2274 /* drain the new stack that might have been created */
2275 DEBUG (6, fprintf (gc_debug_file, "Precise scan of gray area post fin\n"));
2276 drain_gray_stack (queue);
2277 } while (fin_ready != num_ready_finalizers);
2280 * Clear ephemeron pairs with unreachable keys.
2281 * We pass the copy func so we can figure out if an array was promoted or not.
2283 clear_unreachable_ephemerons (copy_func, start_addr, end_addr, queue);
2286 DEBUG (2, fprintf (gc_debug_file, "Finalize queue handling scan for %s generation: %d usecs %d ephemeron roundss\n", generation_name (generation), TV_ELAPSED (atv, btv), ephemeron_rounds));
2289 * handle disappearing links
2290 * Note we do this after checking the finalization queue because if an object
2291 * survives (at least long enough to be finalized) we don't clear the link.
2292 * This also deals with a possible issue with the monitor reclamation: with the Boehm
2293 * GC a finalized object my lose the monitor because it is cleared before the finalizer is
2296 g_assert (gray_object_queue_is_empty (queue));
2298 null_link_in_range (copy_func, start_addr, end_addr, generation, queue);
2299 if (generation == GENERATION_OLD)
2300 null_link_in_range (copy_func, start_addr, end_addr, GENERATION_NURSERY, queue);
2301 if (gray_object_queue_is_empty (queue))
2303 drain_gray_stack (queue);
2306 g_assert (gray_object_queue_is_empty (queue));
2310 mono_sgen_check_section_scan_starts (GCMemSection *section)
2313 for (i = 0; i < section->num_scan_start; ++i) {
2314 if (section->scan_starts [i]) {
2315 guint size = safe_object_get_size ((MonoObject*) section->scan_starts [i]);
2316 g_assert (size >= sizeof (MonoObject) && size <= MAX_SMALL_OBJ_SIZE);
2322 check_scan_starts (void)
2324 if (!do_scan_starts_check)
2326 mono_sgen_check_section_scan_starts (nursery_section);
2327 major_collector.check_scan_starts ();
2330 static int last_num_pinned = 0;
2333 build_nursery_fragments (void **start, int num_entries)
2335 char *frag_start, *frag_end;
2339 while (nursery_fragments) {
2340 Fragment *next = nursery_fragments->next;
2341 nursery_fragments->next = fragment_freelist;
2342 fragment_freelist = nursery_fragments;
2343 nursery_fragments = next;
2345 frag_start = nursery_start;
2347 /* clear scan starts */
2348 memset (nursery_section->scan_starts, 0, nursery_section->num_scan_start * sizeof (gpointer));
2349 for (i = 0; i < num_entries; ++i) {
2350 frag_end = start [i];
2351 /* remove the pin bit from pinned objects */
2352 unpin_object (frag_end);
2353 nursery_section->scan_starts [((char*)frag_end - (char*)nursery_section->data)/SCAN_START_SIZE] = frag_end;
2354 frag_size = frag_end - frag_start;
2356 add_nursery_frag (frag_size, frag_start, frag_end);
2357 frag_size = ALIGN_UP (safe_object_get_size ((MonoObject*)start [i]));
2358 frag_start = (char*)start [i] + frag_size;
2360 nursery_last_pinned_end = frag_start;
2361 frag_end = nursery_real_end;
2362 frag_size = frag_end - frag_start;
2364 add_nursery_frag (frag_size, frag_start, frag_end);
2365 if (!nursery_fragments) {
2366 DEBUG (1, fprintf (gc_debug_file, "Nursery fully pinned (%d)\n", num_entries));
2367 for (i = 0; i < num_entries; ++i) {
2368 DEBUG (3, fprintf (gc_debug_file, "Bastard pinning obj %p (%s), size: %d\n", start [i], safe_name (start [i]), safe_object_get_size (start [i])));
2373 nursery_next = nursery_frag_real_end = NULL;
2375 /* Clear TLABs for all threads */
2380 scan_from_registered_roots (CopyOrMarkObjectFunc copy_func, char *addr_start, char *addr_end, int root_type, GrayQueue *queue)
2384 for (i = 0; i < roots_hash_size [root_type]; ++i) {
2385 for (root = roots_hash [root_type][i]; root; root = root->next) {
2386 DEBUG (6, fprintf (gc_debug_file, "Precise root scan %p-%p (desc: %p)\n", root->start_root, root->end_root, (void*)root->root_desc));
2387 precisely_scan_objects_from (copy_func, (void**)root->start_root, (void**)root->end_root, addr_start, addr_end, root->root_desc, queue);
2393 mono_sgen_dump_occupied (char *start, char *end, char *section_start)
2395 fprintf (heap_dump_file, "<occupied offset=\"%td\" size=\"%td\"/>\n", start - section_start, end - start);
2399 mono_sgen_dump_section (GCMemSection *section, const char *type)
2401 char *start = section->data;
2402 char *end = section->data + section->size;
2403 char *occ_start = NULL;
2405 char *old_start = NULL; /* just for debugging */
2407 fprintf (heap_dump_file, "<section type=\"%s\" size=\"%lu\">\n", type, (unsigned long)section->size);
2409 while (start < end) {
2413 if (!*(void**)start) {
2415 mono_sgen_dump_occupied (occ_start, start, section->data);
2418 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
2421 g_assert (start < section->next_data);
2426 vt = (GCVTable*)LOAD_VTABLE (start);
2429 size = ALIGN_UP (safe_object_get_size ((MonoObject*) start));
2432 fprintf (heap_dump_file, "<object offset=\"%d\" class=\"%s.%s\" size=\"%d\"/>\n",
2433 start - section->data,
2434 vt->klass->name_space, vt->klass->name,
2442 mono_sgen_dump_occupied (occ_start, start, section->data);
2444 fprintf (heap_dump_file, "</section>\n");
2448 dump_object (MonoObject *obj, gboolean dump_location)
2450 static char class_name [1024];
2452 MonoClass *class = mono_object_class (obj);
2456 * Python's XML parser is too stupid to parse angle brackets
2457 * in strings, so we just ignore them;
2460 while (class->name [i] && j < sizeof (class_name) - 1) {
2461 if (!strchr ("<>\"", class->name [i]))
2462 class_name [j++] = class->name [i];
2465 g_assert (j < sizeof (class_name));
2468 fprintf (heap_dump_file, "<object class=\"%s.%s\" size=\"%d\"",
2469 class->name_space, class_name,
2470 safe_object_get_size (obj));
2471 if (dump_location) {
2472 const char *location;
2473 if (ptr_in_nursery (obj))
2474 location = "nursery";
2475 else if (safe_object_get_size (obj) <= MAX_SMALL_OBJ_SIZE)
2479 fprintf (heap_dump_file, " location=\"%s\"", location);
2481 fprintf (heap_dump_file, "/>\n");
2485 dump_heap (const char *type, int num, const char *reason)
2490 fprintf (heap_dump_file, "<collection type=\"%s\" num=\"%d\"", type, num);
2492 fprintf (heap_dump_file, " reason=\"%s\"", reason);
2493 fprintf (heap_dump_file, ">\n");
2494 fprintf (heap_dump_file, "<other-mem-usage type=\"mempools\" size=\"%ld\"/>\n", mono_mempool_get_bytes_allocated ());
2495 mono_sgen_dump_internal_mem_usage (heap_dump_file);
2496 fprintf (heap_dump_file, "<pinned type=\"stack\" bytes=\"%zu\"/>\n", pinned_byte_counts [PIN_TYPE_STACK]);
2497 /* fprintf (heap_dump_file, "<pinned type=\"static-data\" bytes=\"%d\"/>\n", pinned_byte_counts [PIN_TYPE_STATIC_DATA]); */
2498 fprintf (heap_dump_file, "<pinned type=\"other\" bytes=\"%zu\"/>\n", pinned_byte_counts [PIN_TYPE_OTHER]);
2500 fprintf (heap_dump_file, "<pinned-objects>\n");
2501 for (list = pinned_objects; list; list = list->next)
2502 dump_object (list->obj, TRUE);
2503 fprintf (heap_dump_file, "</pinned-objects>\n");
2505 mono_sgen_dump_section (nursery_section, "nursery");
2507 major_collector.dump_heap (heap_dump_file);
2509 fprintf (heap_dump_file, "<los>\n");
2510 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
2511 dump_object ((MonoObject*)bigobj->data, FALSE);
2512 fprintf (heap_dump_file, "</los>\n");
2514 fprintf (heap_dump_file, "</collection>\n");
2518 mono_sgen_register_moved_object (void *obj, void *destination)
2520 g_assert (mono_profiler_events & MONO_PROFILE_GC_MOVES);
2522 /* FIXME: handle this for parallel collector */
2523 g_assert (!major_collector.is_parallel);
2525 if (moved_objects_idx == MOVED_OBJECTS_NUM) {
2526 mono_profiler_gc_moves (moved_objects, moved_objects_idx);
2527 moved_objects_idx = 0;
2529 moved_objects [moved_objects_idx++] = obj;
2530 moved_objects [moved_objects_idx++] = destination;
2536 static gboolean inited = FALSE;
2541 mono_counters_register ("Minor fragment clear", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_pre_collection_fragment_clear);
2542 mono_counters_register ("Minor pinning", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_pinning);
2543 mono_counters_register ("Minor scan remsets", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_scan_remsets);
2544 mono_counters_register ("Minor scan cardtables", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_scan_card_table);
2545 mono_counters_register ("Minor scan pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_scan_pinned);
2546 mono_counters_register ("Minor scan registered roots", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_scan_registered_roots);
2547 mono_counters_register ("Minor scan thread data", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_scan_thread_data);
2548 mono_counters_register ("Minor finish gray stack", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_finish_gray_stack);
2549 mono_counters_register ("Minor fragment creation", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_fragment_creation);
2551 mono_counters_register ("Major fragment clear", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_pre_collection_fragment_clear);
2552 mono_counters_register ("Major pinning", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_pinning);
2553 mono_counters_register ("Major scan pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_pinned);
2554 mono_counters_register ("Major scan registered roots", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_registered_roots);
2555 mono_counters_register ("Major scan thread data", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_thread_data);
2556 mono_counters_register ("Major scan alloc_pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_alloc_pinned);
2557 mono_counters_register ("Major scan finalized", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_finalized);
2558 mono_counters_register ("Major scan big objects", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_big_objects);
2559 mono_counters_register ("Major finish gray stack", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_finish_gray_stack);
2560 mono_counters_register ("Major free big objects", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_free_bigobjs);
2561 mono_counters_register ("Major LOS sweep", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_los_sweep);
2562 mono_counters_register ("Major sweep", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_sweep);
2563 mono_counters_register ("Major fragment creation", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_fragment_creation);
2566 #ifdef HEAVY_STATISTICS
2567 mono_counters_register ("WBarrier set field", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_field);
2568 mono_counters_register ("WBarrier set arrayref", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_arrayref);
2569 mono_counters_register ("WBarrier arrayref copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_arrayref_copy);
2570 mono_counters_register ("WBarrier generic store called", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_generic_store);
2571 mono_counters_register ("WBarrier generic store stored", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_generic_store_remset);
2572 mono_counters_register ("WBarrier set root", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_root);
2573 mono_counters_register ("WBarrier value copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_value_copy);
2574 mono_counters_register ("WBarrier object copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_object_copy);
2576 mono_counters_register ("# objects allocated", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_alloced);
2577 mono_counters_register ("bytes allocated", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_bytes_alloced);
2578 mono_counters_register ("# objects allocated degraded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_alloced_degraded);
2579 mono_counters_register ("bytes allocated degraded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_bytes_alloced_degraded);
2580 mono_counters_register ("bytes allocated in LOS", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_bytes_alloced_los);
2582 mono_counters_register ("# copy_object() called (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_called_nursery);
2583 mono_counters_register ("# objects copied (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_copied_nursery);
2584 mono_counters_register ("# copy_object() called (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_called_major);
2585 mono_counters_register ("# objects copied (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_copied_major);
2587 mono_counters_register ("# scan_object() called (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_scan_object_called_nursery);
2588 mono_counters_register ("# scan_object() called (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_scan_object_called_major);
2590 mono_counters_register ("# nursery copy_object() failed from space", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_from_space);
2591 mono_counters_register ("# nursery copy_object() failed forwarded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_forwarded);
2592 mono_counters_register ("# nursery copy_object() failed pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_pinned);
2594 mono_counters_register ("# wasted fragments used", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_wasted_fragments_used);
2595 mono_counters_register ("bytes in wasted fragments", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_wasted_fragments_bytes);
2597 mono_counters_register ("Store remsets", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_store_remsets);
2598 mono_counters_register ("Unique store remsets", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_store_remsets_unique);
2599 mono_counters_register ("Saved remsets 1", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_saved_remsets_1);
2600 mono_counters_register ("Saved remsets 2", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_saved_remsets_2);
2601 mono_counters_register ("Non-global remsets processed", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_local_remsets_processed);
2602 mono_counters_register ("Global remsets added", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_added);
2603 mono_counters_register ("Global remsets re-added", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_readded);
2604 mono_counters_register ("Global remsets processed", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_processed);
2605 mono_counters_register ("Global remsets discarded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_discarded);
2612 need_major_collection (void)
2614 mword los_alloced = los_memory_usage - MIN (last_los_memory_usage, los_memory_usage);
2615 return minor_collection_sections_alloced * major_collector.section_size + los_alloced > minor_collection_allowance;
2619 * Collect objects in the nursery. Returns whether to trigger a major
2623 collect_nursery (size_t requested_size)
2625 size_t max_garbage_amount;
2626 char *orig_nursery_next;
2627 TV_DECLARE (all_atv);
2628 TV_DECLARE (all_btv);
2632 mono_perfcounters->gc_collections0++;
2634 current_collection_generation = GENERATION_NURSERY;
2636 binary_protocol_collection (GENERATION_NURSERY);
2637 check_scan_starts ();
2640 orig_nursery_next = nursery_next;
2641 nursery_next = MAX (nursery_next, nursery_last_pinned_end);
2642 /* FIXME: optimize later to use the higher address where an object can be present */
2643 nursery_next = MAX (nursery_next, nursery_real_end);
2645 DEBUG (1, fprintf (gc_debug_file, "Start nursery collection %d %p-%p, size: %d\n", num_minor_gcs, nursery_start, nursery_next, (int)(nursery_next - nursery_start)));
2646 max_garbage_amount = nursery_next - nursery_start;
2647 g_assert (nursery_section->size >= max_garbage_amount);
2649 /* world must be stopped already */
2650 TV_GETTIME (all_atv);
2653 /* Pinning no longer depends on clearing all nursery fragments */
2654 clear_current_nursery_fragment (orig_nursery_next);
2657 time_minor_pre_collection_fragment_clear += TV_ELAPSED_MS (atv, btv);
2660 check_for_xdomain_refs ();
2662 nursery_section->next_data = nursery_next;
2664 major_collector.start_nursery_collection ();
2666 gray_object_queue_init (&gray_queue, mono_sgen_get_unmanaged_allocator ());
2669 mono_stats.minor_gc_count ++;
2671 global_remset_cache_clear ();
2673 /* pin from pinned handles */
2675 mono_profiler_gc_event (MONO_GC_EVENT_MARK_START, 0);
2676 pin_from_roots (nursery_start, nursery_next);
2677 /* identify pinned objects */
2678 optimize_pin_queue (0);
2679 next_pin_slot = pin_objects_from_addresses (nursery_section, pin_queue, pin_queue + next_pin_slot, nursery_start, nursery_next, &gray_queue);
2680 nursery_section->pin_queue_start = pin_queue;
2681 nursery_section->pin_queue_num_entries = next_pin_slot;
2683 time_minor_pinning += TV_ELAPSED_MS (btv, atv);
2684 DEBUG (2, fprintf (gc_debug_file, "Finding pinned pointers: %d in %d usecs\n", next_pin_slot, TV_ELAPSED (btv, atv)));
2685 DEBUG (4, fprintf (gc_debug_file, "Start scan with %d pinned objects\n", next_pin_slot));
2687 if (consistency_check_at_minor_collection)
2688 check_consistency ();
2691 * walk all the roots and copy the young objects to the old generation,
2692 * starting from to_space
2695 scan_from_remsets (nursery_start, nursery_next, &gray_queue);
2696 /* we don't have complete write barrier yet, so we scan all the old generation sections */
2698 time_minor_scan_remsets += TV_ELAPSED_MS (atv, btv);
2699 DEBUG (2, fprintf (gc_debug_file, "Old generation scan: %d usecs\n", TV_ELAPSED (atv, btv)));
2701 if (use_cardtable) {
2703 card_tables_collect_stats (TRUE);
2704 scan_from_card_tables (nursery_start, nursery_next, &gray_queue);
2706 time_minor_scan_card_table += TV_ELAPSED_MS (atv, btv);
2709 drain_gray_stack (&gray_queue);
2712 time_minor_scan_pinned += TV_ELAPSED_MS (btv, atv);
2713 /* registered roots, this includes static fields */
2714 scan_from_registered_roots (major_collector.copy_object, nursery_start, nursery_next, ROOT_TYPE_NORMAL, &gray_queue);
2715 scan_from_registered_roots (major_collector.copy_object, nursery_start, nursery_next, ROOT_TYPE_WBARRIER, &gray_queue);
2717 time_minor_scan_registered_roots += TV_ELAPSED_MS (atv, btv);
2719 scan_thread_data (nursery_start, nursery_next, TRUE);
2721 time_minor_scan_thread_data += TV_ELAPSED_MS (btv, atv);
2724 finish_gray_stack (nursery_start, nursery_next, GENERATION_NURSERY, &gray_queue);
2726 time_minor_finish_gray_stack += TV_ELAPSED_MS (btv, atv);
2727 mono_profiler_gc_event (MONO_GC_EVENT_MARK_END, 0);
2729 /* walk the pin_queue, build up the fragment list of free memory, unmark
2730 * pinned objects as we go, memzero() the empty fragments so they are ready for the
2733 mono_profiler_gc_event (MONO_GC_EVENT_RECLAIM_START, 0);
2734 build_nursery_fragments (pin_queue, next_pin_slot);
2735 mono_profiler_gc_event (MONO_GC_EVENT_RECLAIM_END, 0);
2737 time_minor_fragment_creation += TV_ELAPSED_MS (atv, btv);
2738 DEBUG (2, fprintf (gc_debug_file, "Fragment creation: %d usecs, %lu bytes available\n", TV_ELAPSED (atv, btv), (unsigned long)fragment_total));
2740 if (consistency_check_at_minor_collection)
2741 check_major_refs ();
2743 major_collector.finish_nursery_collection ();
2745 TV_GETTIME (all_btv);
2746 mono_stats.minor_gc_time_usecs += TV_ELAPSED (all_atv, all_btv);
2749 dump_heap ("minor", num_minor_gcs - 1, NULL);
2751 /* prepare the pin queue for the next collection */
2752 last_num_pinned = next_pin_slot;
2754 if (fin_ready_list || critical_fin_list) {
2755 DEBUG (4, fprintf (gc_debug_file, "Finalizer-thread wakeup: ready %d\n", num_ready_finalizers));
2756 mono_gc_finalize_notify ();
2760 g_assert (gray_object_queue_is_empty (&gray_queue));
2763 card_tables_collect_stats (FALSE);
2765 check_scan_starts ();
2767 binary_protocol_flush_buffers (FALSE);
2769 current_collection_generation = -1;
2771 return need_major_collection ();
2775 major_do_collection (const char *reason)
2777 LOSObject *bigobj, *prevbo;
2778 TV_DECLARE (all_atv);
2779 TV_DECLARE (all_btv);
2782 /* FIXME: only use these values for the precise scan
2783 * note that to_space pointers should be excluded anyway...
2785 char *heap_start = NULL;
2786 char *heap_end = (char*)-1;
2787 int old_num_major_sections = major_collector.get_num_major_sections ();
2788 int num_major_sections, num_major_sections_saved, save_target, allowance_target;
2789 mword los_memory_saved, los_memory_alloced, old_los_memory_usage;
2791 mono_perfcounters->gc_collections1++;
2794 * A domain could have been freed, resulting in
2795 * los_memory_usage being less than last_los_memory_usage.
2797 los_memory_alloced = los_memory_usage - MIN (last_los_memory_usage, los_memory_usage);
2798 old_los_memory_usage = los_memory_usage;
2800 //count_ref_nonref_objs ();
2801 //consistency_check ();
2803 binary_protocol_collection (GENERATION_OLD);
2804 check_scan_starts ();
2805 gray_object_queue_init (&gray_queue, mono_sgen_get_unmanaged_allocator ());
2806 if (major_collector.is_parallel)
2807 gray_object_queue_init (&workers_distribute_gray_queue, mono_sgen_get_unmanaged_allocator ());
2810 DEBUG (1, fprintf (gc_debug_file, "Start major collection %d\n", num_major_gcs));
2812 mono_stats.major_gc_count ++;
2814 /* world must be stopped already */
2815 TV_GETTIME (all_atv);
2818 /* Pinning depends on this */
2819 clear_nursery_fragments (nursery_next);
2822 time_major_pre_collection_fragment_clear += TV_ELAPSED_MS (atv, btv);
2825 check_for_xdomain_refs ();
2827 nursery_section->next_data = nursery_real_end;
2828 /* we should also coalesce scanning from sections close to each other
2829 * and deal with pointers outside of the sections later.
2831 /* The remsets are not useful for a major collection */
2833 global_remset_cache_clear ();
2835 card_table_clear ();
2839 DEBUG (6, fprintf (gc_debug_file, "Collecting pinned addresses\n"));
2840 pin_from_roots ((void*)lowest_heap_address, (void*)highest_heap_address);
2841 optimize_pin_queue (0);
2844 * pin_queue now contains all candidate pointers, sorted and
2845 * uniqued. We must do two passes now to figure out which
2846 * objects are pinned.
2848 * The first is to find within the pin_queue the area for each
2849 * section. This requires that the pin_queue be sorted. We
2850 * also process the LOS objects and pinned chunks here.
2852 * The second, destructive, pass is to reduce the section
2853 * areas to pointers to the actually pinned objects.
2855 DEBUG (6, fprintf (gc_debug_file, "Pinning from sections\n"));
2856 /* first pass for the sections */
2857 mono_sgen_find_section_pin_queue_start_end (nursery_section);
2858 major_collector.find_pin_queue_start_ends (WORKERS_DISTRIBUTE_GRAY_QUEUE);
2859 /* identify possible pointers to the insize of large objects */
2860 DEBUG (6, fprintf (gc_debug_file, "Pinning from large objects\n"));
2861 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
2863 if (mono_sgen_find_optimized_pin_queue_area (bigobj->data, (char*)bigobj->data + bigobj->size, &dummy)) {
2864 pin_object (bigobj->data);
2865 /* FIXME: only enqueue if object has references */
2866 GRAY_OBJECT_ENQUEUE (WORKERS_DISTRIBUTE_GRAY_QUEUE, bigobj->data);
2868 mono_sgen_pin_stats_register_object ((char*) bigobj->data, safe_object_get_size ((MonoObject*) bigobj->data));
2869 DEBUG (6, fprintf (gc_debug_file, "Marked large object %p (%s) size: %lu from roots\n", bigobj->data, safe_name (bigobj->data), (unsigned long)bigobj->size));
2872 /* second pass for the sections */
2873 mono_sgen_pin_objects_in_section (nursery_section, WORKERS_DISTRIBUTE_GRAY_QUEUE);
2874 major_collector.pin_objects (WORKERS_DISTRIBUTE_GRAY_QUEUE);
2877 time_major_pinning += TV_ELAPSED_MS (atv, btv);
2878 DEBUG (2, fprintf (gc_debug_file, "Finding pinned pointers: %d in %d usecs\n", next_pin_slot, TV_ELAPSED (atv, btv)));
2879 DEBUG (4, fprintf (gc_debug_file, "Start scan with %d pinned objects\n", next_pin_slot));
2881 major_collector.init_to_space ();
2883 workers_start_all_workers (1);
2886 time_major_scan_pinned += TV_ELAPSED_MS (btv, atv);
2888 /* registered roots, this includes static fields */
2889 scan_from_registered_roots (major_collector.copy_or_mark_object, heap_start, heap_end, ROOT_TYPE_NORMAL, WORKERS_DISTRIBUTE_GRAY_QUEUE);
2890 scan_from_registered_roots (major_collector.copy_or_mark_object, heap_start, heap_end, ROOT_TYPE_WBARRIER, WORKERS_DISTRIBUTE_GRAY_QUEUE);
2892 time_major_scan_registered_roots += TV_ELAPSED_MS (atv, btv);
2895 /* FIXME: This is the wrong place for this, because it does
2897 scan_thread_data (heap_start, heap_end, TRUE);
2899 time_major_scan_thread_data += TV_ELAPSED_MS (btv, atv);
2902 time_major_scan_alloc_pinned += TV_ELAPSED_MS (atv, btv);
2904 /* scan the list of objects ready for finalization */
2905 scan_finalizer_entries (major_collector.copy_or_mark_object, fin_ready_list, WORKERS_DISTRIBUTE_GRAY_QUEUE);
2906 scan_finalizer_entries (major_collector.copy_or_mark_object, critical_fin_list, WORKERS_DISTRIBUTE_GRAY_QUEUE);
2908 time_major_scan_finalized += TV_ELAPSED_MS (btv, atv);
2909 DEBUG (2, fprintf (gc_debug_file, "Root scan: %d usecs\n", TV_ELAPSED (btv, atv)));
2912 time_major_scan_big_objects += TV_ELAPSED_MS (atv, btv);
2914 if (major_collector.is_parallel) {
2915 while (!gray_object_queue_is_empty (WORKERS_DISTRIBUTE_GRAY_QUEUE)) {
2916 workers_distribute_gray_queue_sections ();
2920 workers_change_num_working (-1);
2923 if (major_collector.is_parallel)
2924 g_assert (gray_object_queue_is_empty (&gray_queue));
2926 /* all the objects in the heap */
2927 finish_gray_stack (heap_start, heap_end, GENERATION_OLD, &gray_queue);
2929 time_major_finish_gray_stack += TV_ELAPSED_MS (btv, atv);
2931 /* sweep the big objects list */
2933 for (bigobj = los_object_list; bigobj;) {
2934 if (object_is_pinned (bigobj->data)) {
2935 unpin_object (bigobj->data);
2938 /* not referenced anywhere, so we can free it */
2940 prevbo->next = bigobj->next;
2942 los_object_list = bigobj->next;
2944 bigobj = bigobj->next;
2945 free_large_object (to_free);
2949 bigobj = bigobj->next;
2953 time_major_free_bigobjs += TV_ELAPSED_MS (atv, btv);
2958 time_major_los_sweep += TV_ELAPSED_MS (btv, atv);
2960 major_collector.sweep ();
2963 time_major_sweep += TV_ELAPSED_MS (atv, btv);
2965 /* walk the pin_queue, build up the fragment list of free memory, unmark
2966 * pinned objects as we go, memzero() the empty fragments so they are ready for the
2969 build_nursery_fragments (nursery_section->pin_queue_start, nursery_section->pin_queue_num_entries);
2972 time_major_fragment_creation += TV_ELAPSED_MS (btv, atv);
2974 TV_GETTIME (all_btv);
2975 mono_stats.major_gc_time_usecs += TV_ELAPSED (all_atv, all_btv);
2978 dump_heap ("major", num_major_gcs - 1, reason);
2980 /* prepare the pin queue for the next collection */
2982 if (fin_ready_list || critical_fin_list) {
2983 DEBUG (4, fprintf (gc_debug_file, "Finalizer-thread wakeup: ready %d\n", num_ready_finalizers));
2984 mono_gc_finalize_notify ();
2988 g_assert (gray_object_queue_is_empty (&gray_queue));
2990 num_major_sections = major_collector.get_num_major_sections ();
2992 num_major_sections_saved = MAX (old_num_major_sections - num_major_sections, 0);
2993 los_memory_saved = MAX (old_los_memory_usage - los_memory_usage, 1);
2995 save_target = ((num_major_sections * major_collector.section_size) + los_memory_saved) / 2;
2997 * We aim to allow the allocation of as many sections as is
2998 * necessary to reclaim save_target sections in the next
2999 * collection. We assume the collection pattern won't change.
3000 * In the last cycle, we had num_major_sections_saved for
3001 * minor_collection_sections_alloced. Assuming things won't
3002 * change, this must be the same ratio as save_target for
3003 * allowance_target, i.e.
3005 * num_major_sections_saved save_target
3006 * --------------------------------- == ----------------
3007 * minor_collection_sections_alloced allowance_target
3011 allowance_target = (mword)((double)save_target * (double)(minor_collection_sections_alloced * major_collector.section_size + los_memory_alloced) / (double)(num_major_sections_saved * major_collector.section_size + los_memory_saved));
3013 minor_collection_allowance = MAX (MIN (allowance_target, num_major_sections * major_collector.section_size + los_memory_usage), MIN_MINOR_COLLECTION_ALLOWANCE);
3015 minor_collection_sections_alloced = 0;
3016 last_los_memory_usage = los_memory_usage;
3018 major_collector.finish_major_collection ();
3020 check_scan_starts ();
3022 binary_protocol_flush_buffers (FALSE);
3024 //consistency_check ();
3028 major_collection (const char *reason)
3030 if (g_getenv ("MONO_GC_NO_MAJOR")) {
3031 collect_nursery (0);
3035 current_collection_generation = GENERATION_OLD;
3036 major_do_collection (reason);
3037 current_collection_generation = -1;
3041 * When deciding if it's better to collect or to expand, keep track
3042 * of how much garbage was reclaimed with the last collection: if it's too
3044 * This is called when we could not allocate a small object.
3046 static void __attribute__((noinline))
3047 minor_collect_or_expand_inner (size_t size)
3049 int do_minor_collection = 1;
3051 g_assert (nursery_section);
3052 if (do_minor_collection) {
3053 mono_profiler_gc_event (MONO_GC_EVENT_START, 0);
3055 if (collect_nursery (size)) {
3056 mono_profiler_gc_event (MONO_GC_EVENT_START, 1);
3057 major_collection ("minor overflow");
3058 /* keep events symmetric */
3059 mono_profiler_gc_event (MONO_GC_EVENT_END, 1);
3061 DEBUG (2, fprintf (gc_debug_file, "Heap size: %lu, LOS size: %lu\n", (unsigned long)total_alloc, (unsigned long)los_memory_usage));
3063 /* this also sets the proper pointers for the next allocation */
3064 if (!search_fragment_for_size (size)) {
3066 /* TypeBuilder and MonoMethod are killing mcs with fragmentation */
3067 DEBUG (1, fprintf (gc_debug_file, "nursery collection didn't find enough room for %zd alloc (%d pinned)\n", size, last_num_pinned));
3068 for (i = 0; i < last_num_pinned; ++i) {
3069 DEBUG (3, fprintf (gc_debug_file, "Bastard pinning obj %p (%s), size: %d\n", pin_queue [i], safe_name (pin_queue [i]), safe_object_get_size (pin_queue [i])));
3073 mono_profiler_gc_event (MONO_GC_EVENT_END, 0);
3075 //report_internal_mem_usage ();
3079 * ######################################################################
3080 * ######## Memory allocation from the OS
3081 * ######################################################################
3082 * This section of code deals with getting memory from the OS and
3083 * allocating memory for GC-internal data structures.
3084 * Internal memory can be handled with a freelist for small objects.
3090 G_GNUC_UNUSED static void
3091 report_internal_mem_usage (void)
3093 printf ("Internal memory usage:\n");
3094 mono_sgen_report_internal_mem_usage ();
3095 printf ("Pinned memory usage:\n");
3096 major_collector.report_pinned_memory_usage ();
3100 * Allocate a big chunk of memory from the OS (usually 64KB to several megabytes).
3101 * This must not require any lock.
3104 mono_sgen_alloc_os_memory (size_t size, int activate)
3107 unsigned long prot_flags = activate? MONO_MMAP_READ|MONO_MMAP_WRITE: MONO_MMAP_NONE;
3109 prot_flags |= MONO_MMAP_PRIVATE | MONO_MMAP_ANON;
3110 size += pagesize - 1;
3111 size &= ~(pagesize - 1);
3112 ptr = mono_valloc (0, size, prot_flags);
3114 total_alloc += size;
3119 * Free the memory returned by mono_sgen_alloc_os_memory (), returning it to the OS.
3122 mono_sgen_free_os_memory (void *addr, size_t size)
3124 mono_vfree (addr, size);
3126 size += pagesize - 1;
3127 size &= ~(pagesize - 1);
3129 total_alloc -= size;
3133 * ######################################################################
3134 * ######## Object allocation
3135 * ######################################################################
3136 * This section of code deals with allocating memory for objects.
3137 * There are several ways:
3138 * *) allocate large objects
3139 * *) allocate normal objects
3140 * *) fast lock-free allocation
3141 * *) allocation of pinned objects
3145 setup_fragment (Fragment *frag, Fragment *prev, size_t size)
3147 /* remove from the list */
3149 prev->next = frag->next;
3151 nursery_fragments = frag->next;
3152 nursery_next = frag->fragment_start;
3153 nursery_frag_real_end = frag->fragment_end;
3155 DEBUG (4, fprintf (gc_debug_file, "Using nursery fragment %p-%p, size: %td (req: %zd)\n", nursery_next, nursery_frag_real_end, nursery_frag_real_end - nursery_next, size));
3156 frag->next = fragment_freelist;
3157 fragment_freelist = frag;
3160 /* check if we have a suitable fragment in nursery_fragments to be able to allocate
3161 * an object of size @size
3162 * Return FALSE if not found (which means we need a collection)
3165 search_fragment_for_size (size_t size)
3167 Fragment *frag, *prev;
3168 DEBUG (4, fprintf (gc_debug_file, "Searching nursery fragment %p, size: %zd\n", nursery_frag_real_end, size));
3170 if (nursery_frag_real_end > nursery_next && nursery_clear_policy == CLEAR_AT_TLAB_CREATION) {
3171 /* Clear the remaining space, pinning depends on this */
3172 memset (nursery_next, 0, nursery_frag_real_end - nursery_next);
3176 for (frag = nursery_fragments; frag; frag = frag->next) {
3177 if (size <= (frag->fragment_end - frag->fragment_start)) {
3178 setup_fragment (frag, prev, size);
3187 * Same as search_fragment_for_size but if search for @desired_size fails, try to satisfy @minimum_size.
3188 * This improves nursery usage.
3191 search_fragment_for_size_range (size_t desired_size, size_t minimum_size)
3193 Fragment *frag, *prev, *min_prev;
3194 DEBUG (4, fprintf (gc_debug_file, "Searching nursery fragment %p, desired size: %zd minimum size %zd\n", nursery_frag_real_end, desired_size, minimum_size));
3196 if (nursery_frag_real_end > nursery_next && nursery_clear_policy == CLEAR_AT_TLAB_CREATION) {
3197 /* Clear the remaining space, pinning depends on this */
3198 memset (nursery_next, 0, nursery_frag_real_end - nursery_next);
3201 min_prev = GINT_TO_POINTER (-1);
3204 for (frag = nursery_fragments; frag; frag = frag->next) {
3205 int frag_size = frag->fragment_end - frag->fragment_start;
3206 if (desired_size <= frag_size) {
3207 setup_fragment (frag, prev, desired_size);
3208 return desired_size;
3210 if (minimum_size <= frag_size)
3216 if (min_prev != GINT_TO_POINTER (-1)) {
3219 frag = min_prev->next;
3221 frag = nursery_fragments;
3223 frag_size = frag->fragment_end - frag->fragment_start;
3224 HEAVY_STAT (++stat_wasted_fragments_used);
3225 HEAVY_STAT (stat_wasted_fragments_bytes += frag_size);
3227 setup_fragment (frag, min_prev, minimum_size);
3235 alloc_degraded (MonoVTable *vtable, size_t size)
3237 if (need_major_collection ()) {
3238 mono_profiler_gc_event (MONO_GC_EVENT_START, 1);
3240 major_collection ("degraded overflow");
3242 mono_profiler_gc_event (MONO_GC_EVENT_END, 1);
3245 degraded_mode += size;
3246 return major_collector.alloc_degraded (vtable, size);
3250 * Provide a variant that takes just the vtable for small fixed-size objects.
3251 * The aligned size is already computed and stored in vt->gc_descr.
3252 * Note: every SCAN_START_SIZE or so we are given the chance to do some special
3253 * processing. We can keep track of where objects start, for example,
3254 * so when we scan the thread stacks for pinned objects, we can start
3255 * a search for the pinned object in SCAN_START_SIZE chunks.
3258 mono_gc_alloc_obj_nolock (MonoVTable *vtable, size_t size)
3260 /* FIXME: handle OOM */
3265 HEAVY_STAT (++stat_objects_alloced);
3266 if (size <= MAX_SMALL_OBJ_SIZE)
3267 HEAVY_STAT (stat_bytes_alloced += size);
3269 HEAVY_STAT (stat_bytes_alloced_los += size);
3271 size = ALIGN_UP (size);
3273 g_assert (vtable->gc_descr);
3275 if (G_UNLIKELY (collect_before_allocs)) {
3276 if (nursery_section) {
3277 mono_profiler_gc_event (MONO_GC_EVENT_START, 0);
3279 collect_nursery (0);
3281 mono_profiler_gc_event (MONO_GC_EVENT_END, 0);
3282 if (!degraded_mode && !search_fragment_for_size (size)) {
3284 g_assert_not_reached ();
3290 * We must already have the lock here instead of after the
3291 * fast path because we might be interrupted in the fast path
3292 * (after confirming that new_next < TLAB_TEMP_END) by the GC,
3293 * and we'll end up allocating an object in a fragment which
3294 * no longer belongs to us.
3296 * The managed allocator does not do this, but it's treated
3297 * specially by the world-stopping code.
3300 if (size > MAX_SMALL_OBJ_SIZE) {
3301 p = alloc_large_inner (vtable, size);
3303 /* tlab_next and tlab_temp_end are TLS vars so accessing them might be expensive */
3305 p = (void**)TLAB_NEXT;
3306 /* FIXME: handle overflow */
3307 new_next = (char*)p + size;
3308 TLAB_NEXT = new_next;
3310 if (G_LIKELY (new_next < TLAB_TEMP_END)) {
3314 * FIXME: We might need a memory barrier here so the change to tlab_next is
3315 * visible before the vtable store.
3318 DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
3319 binary_protocol_alloc (p , vtable, size);
3320 g_assert (*p == NULL);
3323 g_assert (TLAB_NEXT == new_next);
3330 /* there are two cases: the object is too big or we run out of space in the TLAB */
3331 /* we also reach here when the thread does its first allocation after a minor
3332 * collection, since the tlab_ variables are initialized to NULL.
3333 * there can be another case (from ORP), if we cooperate with the runtime a bit:
3334 * objects that need finalizers can have the high bit set in their size
3335 * so the above check fails and we can readily add the object to the queue.
3336 * This avoids taking again the GC lock when registering, but this is moot when
3337 * doing thread-local allocation, so it may not be a good idea.
3339 g_assert (TLAB_NEXT == new_next);
3340 if (TLAB_NEXT >= TLAB_REAL_END) {
3342 * Run out of space in the TLAB. When this happens, some amount of space
3343 * remains in the TLAB, but not enough to satisfy the current allocation
3344 * request. Currently, we retire the TLAB in all cases, later we could
3345 * keep it if the remaining space is above a treshold, and satisfy the
3346 * allocation directly from the nursery.
3349 /* when running in degraded mode, we continue allocing that way
3350 * for a while, to decrease the number of useless nursery collections.
3352 if (degraded_mode && degraded_mode < DEFAULT_NURSERY_SIZE) {
3353 p = alloc_degraded (vtable, size);
3354 binary_protocol_alloc_degraded (p, vtable, size);
3358 /*FIXME This codepath is current deadcode since tlab_size > MAX_SMALL_OBJ_SIZE*/
3359 if (size > tlab_size) {
3360 /* Allocate directly from the nursery */
3361 if (nursery_next + size >= nursery_frag_real_end) {
3362 if (!search_fragment_for_size (size)) {
3363 minor_collect_or_expand_inner (size);
3364 if (degraded_mode) {
3365 p = alloc_degraded (vtable, size);
3366 binary_protocol_alloc_degraded (p, vtable, size);
3372 p = (void*)nursery_next;
3373 nursery_next += size;
3374 if (nursery_next > nursery_frag_real_end) {
3379 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION) {
3380 memset (p, 0, size);
3383 int alloc_size = tlab_size;
3384 int available_in_nursery = nursery_frag_real_end - nursery_next;
3386 DEBUG (3, fprintf (gc_debug_file, "Retire TLAB: %p-%p [%ld]\n", TLAB_START, TLAB_REAL_END, (long)(TLAB_REAL_END - TLAB_NEXT - size)));
3388 if (alloc_size >= available_in_nursery) {
3389 if (available_in_nursery > MAX_NURSERY_TLAB_WASTE && available_in_nursery > size) {
3390 alloc_size = available_in_nursery;
3392 alloc_size = search_fragment_for_size_range (tlab_size, size);
3394 alloc_size = tlab_size;
3395 minor_collect_or_expand_inner (tlab_size);
3396 if (degraded_mode) {
3397 p = alloc_degraded (vtable, size);
3398 binary_protocol_alloc_degraded (p, vtable, size);
3405 /* Allocate a new TLAB from the current nursery fragment */
3406 TLAB_START = nursery_next;
3407 nursery_next += alloc_size;
3408 TLAB_NEXT = TLAB_START;
3409 TLAB_REAL_END = TLAB_START + alloc_size;
3410 TLAB_TEMP_END = TLAB_START + MIN (SCAN_START_SIZE, alloc_size);
3412 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION) {
3413 memset (TLAB_START, 0, alloc_size);
3416 /* Allocate from the TLAB */
3417 p = (void*)TLAB_NEXT;
3419 g_assert (TLAB_NEXT <= TLAB_REAL_END);
3421 nursery_section->scan_starts [((char*)p - (char*)nursery_section->data)/SCAN_START_SIZE] = (char*)p;
3424 /* Reached tlab_temp_end */
3426 /* record the scan start so we can find pinned objects more easily */
3427 nursery_section->scan_starts [((char*)p - (char*)nursery_section->data)/SCAN_START_SIZE] = (char*)p;
3428 /* we just bump tlab_temp_end as well */
3429 TLAB_TEMP_END = MIN (TLAB_REAL_END, TLAB_NEXT + SCAN_START_SIZE);
3430 DEBUG (5, fprintf (gc_debug_file, "Expanding local alloc: %p-%p\n", TLAB_NEXT, TLAB_TEMP_END));
3434 DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
3435 binary_protocol_alloc (p, vtable, size);
3442 mono_gc_try_alloc_obj_nolock (MonoVTable *vtable, size_t size)
3448 size = ALIGN_UP (size);
3450 g_assert (vtable->gc_descr);
3451 if (size <= MAX_SMALL_OBJ_SIZE) {
3452 /* tlab_next and tlab_temp_end are TLS vars so accessing them might be expensive */
3454 p = (void**)TLAB_NEXT;
3455 /* FIXME: handle overflow */
3456 new_next = (char*)p + size;
3457 TLAB_NEXT = new_next;
3459 if (G_LIKELY (new_next < TLAB_TEMP_END)) {
3463 * FIXME: We might need a memory barrier here so the change to tlab_next is
3464 * visible before the vtable store.
3467 HEAVY_STAT (++stat_objects_alloced);
3468 HEAVY_STAT (stat_bytes_alloced += size);
3470 DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
3471 binary_protocol_alloc (p, vtable, size);
3472 g_assert (*p == NULL);
3475 g_assert (TLAB_NEXT == new_next);
3484 mono_gc_alloc_obj (MonoVTable *vtable, size_t size)
3487 #ifndef DISABLE_CRITICAL_REGION
3489 ENTER_CRITICAL_REGION;
3490 res = mono_gc_try_alloc_obj_nolock (vtable, size);
3492 EXIT_CRITICAL_REGION;
3495 EXIT_CRITICAL_REGION;
3498 res = mono_gc_alloc_obj_nolock (vtable, size);
3504 mono_gc_alloc_vector (MonoVTable *vtable, size_t size, uintptr_t max_length)
3507 #ifndef DISABLE_CRITICAL_REGION
3509 ENTER_CRITICAL_REGION;
3510 arr = mono_gc_try_alloc_obj_nolock (vtable, size);
3512 arr->max_length = max_length;
3513 EXIT_CRITICAL_REGION;
3516 EXIT_CRITICAL_REGION;
3521 arr = mono_gc_alloc_obj_nolock (vtable, size);
3522 arr->max_length = max_length;
3530 mono_gc_alloc_array (MonoVTable *vtable, size_t size, uintptr_t max_length, uintptr_t bounds_size)
3533 MonoArrayBounds *bounds;
3537 arr = mono_gc_alloc_obj_nolock (vtable, size);
3538 arr->max_length = max_length;
3540 bounds = (MonoArrayBounds*)((char*)arr + size - bounds_size);
3541 arr->bounds = bounds;
3549 mono_gc_alloc_string (MonoVTable *vtable, size_t size, gint32 len)
3552 #ifndef DISABLE_CRITICAL_REGION
3554 ENTER_CRITICAL_REGION;
3555 str = mono_gc_try_alloc_obj_nolock (vtable, size);
3558 EXIT_CRITICAL_REGION;
3561 EXIT_CRITICAL_REGION;
3566 str = mono_gc_alloc_obj_nolock (vtable, size);
3575 * To be used for interned strings and possibly MonoThread, reflection handles.
3576 * We may want to explicitly free these objects.
3579 mono_gc_alloc_pinned_obj (MonoVTable *vtable, size_t size)
3581 /* FIXME: handle OOM */
3583 size = ALIGN_UP (size);
3585 if (size > MAX_SMALL_OBJ_SIZE) {
3586 /* large objects are always pinned anyway */
3587 p = alloc_large_inner (vtable, size);
3589 DEBUG (9, g_assert (vtable->klass->inited));
3590 p = major_collector.alloc_small_pinned_obj (size, vtable->klass->has_references);
3592 DEBUG (6, fprintf (gc_debug_file, "Allocated pinned object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
3593 binary_protocol_alloc_pinned (p, vtable, size);
3600 * ######################################################################
3601 * ######## Finalization support
3602 * ######################################################################
3606 * this is valid for the nursery: if the object has been forwarded it means it's
3607 * still refrenced from a root. If it is pinned it's still alive as well.
3608 * Return TRUE if @obj is ready to be finalized.
3610 #define object_is_fin_ready(obj) (!object_is_pinned (obj) && !object_is_forwarded (obj))
3613 is_critical_finalizer (FinalizeEntry *entry)
3618 if (!mono_defaults.critical_finalizer_object)
3621 obj = entry->object;
3622 class = ((MonoVTable*)LOAD_VTABLE (obj))->klass;
3624 return mono_class_has_parent (class, mono_defaults.critical_finalizer_object);
3628 queue_finalization_entry (FinalizeEntry *entry) {
3629 if (is_critical_finalizer (entry)) {
3630 entry->next = critical_fin_list;
3631 critical_fin_list = entry;
3633 entry->next = fin_ready_list;
3634 fin_ready_list = entry;
3638 /* LOCKING: requires that the GC lock is held */
3640 rehash_fin_table (FinalizeEntryHashTable *hash_table)
3642 FinalizeEntry **finalizable_hash = hash_table->table;
3643 mword finalizable_hash_size = hash_table->size;
3646 FinalizeEntry **new_hash;
3647 FinalizeEntry *entry, *next;
3648 int new_size = g_spaced_primes_closest (hash_table->num_registered);
3650 new_hash = mono_sgen_alloc_internal_dynamic (new_size * sizeof (FinalizeEntry*), INTERNAL_MEM_FIN_TABLE);
3651 for (i = 0; i < finalizable_hash_size; ++i) {
3652 for (entry = finalizable_hash [i]; entry; entry = next) {
3653 hash = mono_object_hash (entry->object) % new_size;
3655 entry->next = new_hash [hash];
3656 new_hash [hash] = entry;
3659 mono_sgen_free_internal_dynamic (finalizable_hash, finalizable_hash_size * sizeof (FinalizeEntry*), INTERNAL_MEM_FIN_TABLE);
3660 hash_table->table = new_hash;
3661 hash_table->size = new_size;
3664 /* LOCKING: requires that the GC lock is held */
3666 rehash_fin_table_if_necessary (FinalizeEntryHashTable *hash_table)
3668 if (hash_table->num_registered >= hash_table->size * 2)
3669 rehash_fin_table (hash_table);
3672 /* LOCKING: requires that the GC lock is held */
3674 finalize_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, int generation, GrayQueue *queue)
3676 FinalizeEntryHashTable *hash_table = get_finalize_entry_hash_table (generation);
3677 FinalizeEntry *entry, *prev;
3679 FinalizeEntry **finalizable_hash = hash_table->table;
3680 mword finalizable_hash_size = hash_table->size;
3684 for (i = 0; i < finalizable_hash_size; ++i) {
3686 for (entry = finalizable_hash [i]; entry;) {
3687 if ((char*)entry->object >= start && (char*)entry->object < end && !major_collector.is_object_live (entry->object)) {
3688 gboolean is_fin_ready = object_is_fin_ready (entry->object);
3689 char *copy = entry->object;
3690 copy_func ((void**)©, queue);
3693 FinalizeEntry *next;
3694 /* remove and put in fin_ready_list */
3696 prev->next = entry->next;
3698 finalizable_hash [i] = entry->next;
3700 num_ready_finalizers++;
3701 hash_table->num_registered--;
3702 queue_finalization_entry (entry);
3703 /* Make it survive */
3704 from = entry->object;
3705 entry->object = copy;
3706 DEBUG (5, fprintf (gc_debug_file, "Queueing object for finalization: %p (%s) (was at %p) (%d/%d)\n", entry->object, safe_name (entry->object), from, num_ready_finalizers, hash_table->num_registered));
3710 char *from = entry->object;
3711 if (hash_table == &minor_finalizable_hash && !ptr_in_nursery (copy)) {
3712 FinalizeEntry *next = entry->next;
3713 unsigned int major_hash;
3714 /* remove from the list */
3716 prev->next = entry->next;
3718 finalizable_hash [i] = entry->next;
3719 hash_table->num_registered--;
3721 entry->object = copy;
3723 /* insert it into the major hash */
3724 rehash_fin_table_if_necessary (&major_finalizable_hash);
3725 major_hash = mono_object_hash ((MonoObject*) copy) %
3726 major_finalizable_hash.size;
3727 entry->next = major_finalizable_hash.table [major_hash];
3728 major_finalizable_hash.table [major_hash] = entry;
3729 major_finalizable_hash.num_registered++;
3731 DEBUG (5, fprintf (gc_debug_file, "Promoting finalization of object %p (%s) (was at %p) to major table\n", copy, safe_name (copy), from));
3736 /* update pointer */
3737 DEBUG (5, fprintf (gc_debug_file, "Updating object for finalization: %p (%s) (was at %p)\n", entry->object, safe_name (entry->object), from));
3738 entry->object = copy;
3743 entry = entry->next;
3749 object_is_reachable (char *object, char *start, char *end)
3751 /*This happens for non nursery objects during minor collections. We just treat all objects as alive.*/
3752 if (object < start || object >= end)
3754 return !object_is_fin_ready (object) || major_collector.is_object_live (object);
3757 /* LOCKING: requires that the GC lock is held */
3759 null_ephemerons_for_domain (MonoDomain *domain)
3761 EphemeronLinkNode *current = ephemeron_list, *prev = NULL;
3764 MonoObject *object = (MonoObject*)current->array;
3766 if (object && !object->vtable) {
3767 EphemeronLinkNode *tmp = current;
3770 prev->next = current->next;
3772 ephemeron_list = current->next;
3774 current = current->next;
3775 mono_sgen_free_internal (tmp, INTERNAL_MEM_EPHEMERON_LINK);
3778 current = current->next;
3783 /* LOCKING: requires that the GC lock is held */
3785 clear_unreachable_ephemerons (CopyOrMarkObjectFunc copy_func, char *start, char *end, GrayQueue *queue)
3787 int was_in_nursery, was_promoted;
3788 EphemeronLinkNode *current = ephemeron_list, *prev = NULL;
3790 Ephemeron *cur, *array_end;
3794 char *object = current->array;
3796 if (!object_is_reachable (object, start, end)) {
3797 EphemeronLinkNode *tmp = current;
3799 DEBUG (5, fprintf (gc_debug_file, "Dead Ephemeron array at %p\n", object));
3802 prev->next = current->next;
3804 ephemeron_list = current->next;
3806 current = current->next;
3807 mono_sgen_free_internal (tmp, INTERNAL_MEM_EPHEMERON_LINK);
3812 was_in_nursery = ptr_in_nursery (object);
3813 copy_func ((void**)&object, queue);
3814 current->array = object;
3816 /*The array was promoted, add global remsets for key/values left behind in nursery.*/
3817 was_promoted = was_in_nursery && !ptr_in_nursery (object);
3819 DEBUG (5, fprintf (gc_debug_file, "Clearing unreachable entries for ephemeron array at %p\n", object));
3821 array = (MonoArray*)object;
3822 cur = mono_array_addr (array, Ephemeron, 0);
3823 array_end = cur + mono_array_length_fast (array);
3824 tombstone = (char*)((MonoVTable*)LOAD_VTABLE (object))->domain->ephemeron_tombstone;
3826 for (; cur < array_end; ++cur) {
3827 char *key = (char*)cur->key;
3829 if (!key || key == tombstone)
3832 DEBUG (5, fprintf (gc_debug_file, "[%td] key %p (%s) value %p (%s)\n", cur - mono_array_addr (array, Ephemeron, 0),
3833 key, object_is_reachable (key, start, end) ? "reachable" : "unreachable",
3834 cur->value, cur->value && object_is_reachable (cur->value, start, end) ? "reachable" : "unreachable"));
3836 if (!object_is_reachable (key, start, end)) {
3837 cur->key = tombstone;
3843 if (ptr_in_nursery (key)) {/*key was not promoted*/
3844 DEBUG (5, fprintf (gc_debug_file, "\tAdded remset to key %p\n", key));
3845 mono_sgen_add_to_global_remset (&cur->key);
3847 if (ptr_in_nursery (cur->value)) {/*value was not promoted*/
3848 DEBUG (5, fprintf (gc_debug_file, "\tAdded remset to value %p\n", cur->value));
3849 mono_sgen_add_to_global_remset (&cur->value);
3854 current = current->next;
3858 /* LOCKING: requires that the GC lock is held */
3860 mark_ephemerons_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, GrayQueue *queue)
3862 int nothing_marked = 1;
3863 EphemeronLinkNode *current = ephemeron_list;
3865 Ephemeron *cur, *array_end;
3868 for (current = ephemeron_list; current; current = current->next) {
3869 char *object = current->array;
3870 DEBUG (5, fprintf (gc_debug_file, "Ephemeron array at %p\n", object));
3872 /*We ignore arrays in old gen during minor collections since all objects are promoted by the remset machinery.*/
3873 if (object < start || object >= end)
3876 /*It has to be alive*/
3877 if (!object_is_reachable (object, start, end)) {
3878 DEBUG (5, fprintf (gc_debug_file, "\tnot reachable\n"));
3882 copy_func ((void**)&object, queue);
3884 array = (MonoArray*)object;
3885 cur = mono_array_addr (array, Ephemeron, 0);
3886 array_end = cur + mono_array_length_fast (array);
3887 tombstone = (char*)((MonoVTable*)LOAD_VTABLE (object))->domain->ephemeron_tombstone;
3889 for (; cur < array_end; ++cur) {
3890 char *key = cur->key;
3892 if (!key || key == tombstone)
3895 DEBUG (5, fprintf (gc_debug_file, "[%td] key %p (%s) value %p (%s)\n", cur - mono_array_addr (array, Ephemeron, 0),
3896 key, object_is_reachable (key, start, end) ? "reachable" : "unreachable",
3897 cur->value, cur->value && object_is_reachable (cur->value, start, end) ? "reachable" : "unreachable"));
3899 if (object_is_reachable (key, start, end)) {
3900 char *value = cur->value;
3902 copy_func ((void**)&cur->key, queue);
3904 if (!object_is_reachable (value, start, end))
3906 copy_func ((void**)&cur->value, queue);
3912 DEBUG (5, fprintf (gc_debug_file, "Ephemeron run finished. Is it done %d\n", nothing_marked));
3913 return nothing_marked;
3916 /* LOCKING: requires that the GC lock is held */
3918 null_link_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, int generation, GrayQueue *queue)
3920 DisappearingLinkHashTable *hash = get_dislink_hash_table (generation);
3921 DisappearingLink **disappearing_link_hash = hash->table;
3922 int disappearing_link_hash_size = hash->size;
3923 DisappearingLink *entry, *prev;
3925 if (!hash->num_links)
3927 for (i = 0; i < disappearing_link_hash_size; ++i) {
3929 for (entry = disappearing_link_hash [i]; entry;) {
3930 char *object = DISLINK_OBJECT (entry);
3931 if (object >= start && object < end && !major_collector.is_object_live (object)) {
3932 gboolean track = DISLINK_TRACK (entry);
3933 if (!track && object_is_fin_ready (object)) {
3934 void **p = entry->link;
3935 DisappearingLink *old;
3937 /* remove from list */
3939 prev->next = entry->next;
3941 disappearing_link_hash [i] = entry->next;
3942 DEBUG (5, fprintf (gc_debug_file, "Dislink nullified at %p to GCed object %p\n", p, object));
3944 mono_sgen_free_internal (entry, INTERNAL_MEM_DISLINK);
3949 char *copy = object;
3950 copy_func ((void**)©, queue);
3952 /* Update pointer if it's moved. If the object
3953 * has been moved out of the nursery, we need to
3954 * remove the link from the minor hash table to
3957 * FIXME: what if an object is moved earlier?
3960 if (hash == &minor_disappearing_link_hash && !ptr_in_nursery (copy)) {
3961 void **link = entry->link;
3962 DisappearingLink *old;
3963 /* remove from list */
3965 prev->next = entry->next;
3967 disappearing_link_hash [i] = entry->next;
3969 mono_sgen_free_internal (entry, INTERNAL_MEM_DISLINK);
3973 add_or_remove_disappearing_link ((MonoObject*)copy, link,
3974 track, GENERATION_OLD);
3976 DEBUG (5, fprintf (gc_debug_file, "Upgraded dislink at %p to major because object %p moved to %p\n", link, object, copy));
3980 /* We set the track resurrection bit to
3981 * FALSE if the object is to be finalized
3982 * so that the object can be collected in
3983 * the next cycle (i.e. after it was
3986 *entry->link = HIDE_POINTER (copy,
3987 object_is_fin_ready (object) ? FALSE : track);
3988 DEBUG (5, fprintf (gc_debug_file, "Updated dislink at %p to %p\n", entry->link, DISLINK_OBJECT (entry)));
3993 entry = entry->next;
3998 /* LOCKING: requires that the GC lock is held */
4000 null_links_for_domain (MonoDomain *domain, int generation)
4002 DisappearingLinkHashTable *hash = get_dislink_hash_table (generation);
4003 DisappearingLink **disappearing_link_hash = hash->table;
4004 int disappearing_link_hash_size = hash->size;
4005 DisappearingLink *entry, *prev;
4007 for (i = 0; i < disappearing_link_hash_size; ++i) {
4009 for (entry = disappearing_link_hash [i]; entry; ) {
4010 char *object = DISLINK_OBJECT (entry);
4011 if (object && !((MonoObject*)object)->vtable) {
4012 DisappearingLink *next = entry->next;
4017 disappearing_link_hash [i] = next;
4019 if (*(entry->link)) {
4020 *(entry->link) = NULL;
4021 g_warning ("Disappearing link %p not freed", entry->link);
4023 mono_sgen_free_internal (entry, INTERNAL_MEM_DISLINK);
4030 entry = entry->next;
4035 /* LOCKING: requires that the GC lock is held */
4037 finalizers_for_domain (MonoDomain *domain, MonoObject **out_array, int out_size,
4038 FinalizeEntryHashTable *hash_table)
4040 FinalizeEntry **finalizable_hash = hash_table->table;
4041 mword finalizable_hash_size = hash_table->size;
4042 FinalizeEntry *entry, *prev;
4045 if (no_finalize || !out_size || !out_array)
4048 for (i = 0; i < finalizable_hash_size; ++i) {
4050 for (entry = finalizable_hash [i]; entry;) {
4051 if (mono_object_domain (entry->object) == domain) {
4052 FinalizeEntry *next;
4053 /* remove and put in out_array */
4055 prev->next = entry->next;
4057 finalizable_hash [i] = entry->next;
4059 hash_table->num_registered--;
4060 out_array [count ++] = entry->object;
4061 DEBUG (5, fprintf (gc_debug_file, "Collecting object for finalization: %p (%s) (%d/%d)\n", entry->object, safe_name (entry->object), num_ready_finalizers, hash_table->num_registered));
4063 if (count == out_size)
4068 entry = entry->next;
4075 * mono_gc_finalizers_for_domain:
4076 * @domain: the unloading appdomain
4077 * @out_array: output array
4078 * @out_size: size of output array
4080 * Store inside @out_array up to @out_size objects that belong to the unloading
4081 * appdomain @domain. Returns the number of stored items. Can be called repeteadly
4082 * until it returns 0.
4083 * The items are removed from the finalizer data structure, so the caller is supposed
4085 * @out_array should be on the stack to allow the GC to know the objects are still alive.
4088 mono_gc_finalizers_for_domain (MonoDomain *domain, MonoObject **out_array, int out_size)
4093 result = finalizers_for_domain (domain, out_array, out_size, &minor_finalizable_hash);
4094 if (result < out_size) {
4095 result += finalizers_for_domain (domain, out_array + result, out_size - result,
4096 &major_finalizable_hash);
4104 register_for_finalization (MonoObject *obj, void *user_data, int generation)
4106 FinalizeEntryHashTable *hash_table = get_finalize_entry_hash_table (generation);
4107 FinalizeEntry **finalizable_hash;
4108 mword finalizable_hash_size;
4109 FinalizeEntry *entry, *prev;
4113 g_assert (user_data == NULL || user_data == mono_gc_run_finalize);
4114 hash = mono_object_hash (obj);
4116 rehash_fin_table_if_necessary (hash_table);
4117 finalizable_hash = hash_table->table;
4118 finalizable_hash_size = hash_table->size;
4119 hash %= finalizable_hash_size;
4121 for (entry = finalizable_hash [hash]; entry; entry = entry->next) {
4122 if (entry->object == obj) {
4124 /* remove from the list */
4126 prev->next = entry->next;
4128 finalizable_hash [hash] = entry->next;
4129 hash_table->num_registered--;
4130 DEBUG (5, fprintf (gc_debug_file, "Removed finalizer %p for object: %p (%s) (%d)\n", entry, obj, obj->vtable->klass->name, hash_table->num_registered));
4131 mono_sgen_free_internal (entry, INTERNAL_MEM_FINALIZE_ENTRY);
4139 /* request to deregister, but already out of the list */
4143 entry = mono_sgen_alloc_internal (INTERNAL_MEM_FINALIZE_ENTRY);
4144 entry->object = obj;
4145 entry->next = finalizable_hash [hash];
4146 finalizable_hash [hash] = entry;
4147 hash_table->num_registered++;
4148 DEBUG (5, fprintf (gc_debug_file, "Added finalizer %p for object: %p (%s) (%d) to %s table\n", entry, obj, obj->vtable->klass->name, hash_table->num_registered, generation_name (generation)));
4153 mono_gc_register_for_finalization (MonoObject *obj, void *user_data)
4155 if (ptr_in_nursery (obj))
4156 register_for_finalization (obj, user_data, GENERATION_NURSERY);
4158 register_for_finalization (obj, user_data, GENERATION_OLD);
4162 rehash_dislink (DisappearingLinkHashTable *hash_table)
4164 DisappearingLink **disappearing_link_hash = hash_table->table;
4165 int disappearing_link_hash_size = hash_table->size;
4168 DisappearingLink **new_hash;
4169 DisappearingLink *entry, *next;
4170 int new_size = g_spaced_primes_closest (hash_table->num_links);
4172 new_hash = mono_sgen_alloc_internal_dynamic (new_size * sizeof (DisappearingLink*), INTERNAL_MEM_DISLINK_TABLE);
4173 for (i = 0; i < disappearing_link_hash_size; ++i) {
4174 for (entry = disappearing_link_hash [i]; entry; entry = next) {
4175 hash = mono_aligned_addr_hash (entry->link) % new_size;
4177 entry->next = new_hash [hash];
4178 new_hash [hash] = entry;
4181 mono_sgen_free_internal_dynamic (disappearing_link_hash,
4182 disappearing_link_hash_size * sizeof (DisappearingLink*), INTERNAL_MEM_DISLINK_TABLE);
4183 hash_table->table = new_hash;
4184 hash_table->size = new_size;
4187 /* LOCKING: assumes the GC lock is held */
4189 add_or_remove_disappearing_link (MonoObject *obj, void **link, gboolean track, int generation)
4191 DisappearingLinkHashTable *hash_table = get_dislink_hash_table (generation);
4192 DisappearingLink *entry, *prev;
4194 DisappearingLink **disappearing_link_hash = hash_table->table;
4195 int disappearing_link_hash_size = hash_table->size;
4197 if (hash_table->num_links >= disappearing_link_hash_size * 2) {
4198 rehash_dislink (hash_table);
4199 disappearing_link_hash = hash_table->table;
4200 disappearing_link_hash_size = hash_table->size;
4202 /* FIXME: add check that link is not in the heap */
4203 hash = mono_aligned_addr_hash (link) % disappearing_link_hash_size;
4204 entry = disappearing_link_hash [hash];
4206 for (; entry; entry = entry->next) {
4207 /* link already added */
4208 if (link == entry->link) {
4209 /* NULL obj means remove */
4212 prev->next = entry->next;
4214 disappearing_link_hash [hash] = entry->next;
4215 hash_table->num_links--;
4216 DEBUG (5, fprintf (gc_debug_file, "Removed dislink %p (%d) from %s table\n", entry, hash_table->num_links, generation_name (generation)));
4217 mono_sgen_free_internal (entry, INTERNAL_MEM_DISLINK);
4220 *link = HIDE_POINTER (obj, track); /* we allow the change of object */
4228 entry = mono_sgen_alloc_internal (INTERNAL_MEM_DISLINK);
4229 *link = HIDE_POINTER (obj, track);
4231 entry->next = disappearing_link_hash [hash];
4232 disappearing_link_hash [hash] = entry;
4233 hash_table->num_links++;
4234 DEBUG (5, fprintf (gc_debug_file, "Added dislink %p for object: %p (%s) at %p to %s table\n", entry, obj, obj->vtable->klass->name, link, generation_name (generation)));
4237 /* LOCKING: assumes the GC lock is held */
4239 mono_gc_register_disappearing_link (MonoObject *obj, void **link, gboolean track)
4241 add_or_remove_disappearing_link (NULL, link, FALSE, GENERATION_NURSERY);
4242 add_or_remove_disappearing_link (NULL, link, FALSE, GENERATION_OLD);
4244 if (ptr_in_nursery (obj))
4245 add_or_remove_disappearing_link (obj, link, track, GENERATION_NURSERY);
4247 add_or_remove_disappearing_link (obj, link, track, GENERATION_OLD);
4252 mono_gc_invoke_finalizers (void)
4254 FinalizeEntry *entry = NULL;
4255 gboolean entry_is_critical = FALSE;
4258 /* FIXME: batch to reduce lock contention */
4259 while (fin_ready_list || critical_fin_list) {
4263 FinalizeEntry **list = entry_is_critical ? &critical_fin_list : &fin_ready_list;
4265 /* We have finalized entry in the last
4266 interation, now we need to remove it from
4269 *list = entry->next;
4271 FinalizeEntry *e = *list;
4272 while (e->next != entry)
4274 e->next = entry->next;
4276 mono_sgen_free_internal (entry, INTERNAL_MEM_FINALIZE_ENTRY);
4280 /* Now look for the first non-null entry. */
4281 for (entry = fin_ready_list; entry && !entry->object; entry = entry->next)
4284 entry_is_critical = FALSE;
4286 entry_is_critical = TRUE;
4287 for (entry = critical_fin_list; entry && !entry->object; entry = entry->next)
4292 g_assert (entry->object);
4293 num_ready_finalizers--;
4294 obj = entry->object;
4295 entry->object = NULL;
4296 DEBUG (7, fprintf (gc_debug_file, "Finalizing object %p (%s)\n", obj, safe_name (obj)));
4304 g_assert (entry->object == NULL);
4306 /* the object is on the stack so it is pinned */
4307 /*g_print ("Calling finalizer for object: %p (%s)\n", entry->object, safe_name (entry->object));*/
4308 mono_gc_run_finalize (obj, NULL);
4315 mono_gc_pending_finalizers (void)
4317 return fin_ready_list || critical_fin_list;
4320 /* Negative value to remove */
4322 mono_gc_add_memory_pressure (gint64 value)
4324 /* FIXME: Use interlocked functions */
4326 memory_pressure += value;
4331 mono_sgen_register_major_sections_alloced (int num_sections)
4333 minor_collection_sections_alloced += num_sections;
4337 mono_sgen_get_minor_collection_allowance (void)
4339 return minor_collection_allowance;
4343 * ######################################################################
4344 * ######## registered roots support
4345 * ######################################################################
4349 rehash_roots (gboolean pinned)
4353 RootRecord **new_hash;
4354 RootRecord *entry, *next;
4357 new_size = g_spaced_primes_closest (num_roots_entries [pinned]);
4358 new_hash = mono_sgen_alloc_internal_dynamic (new_size * sizeof (RootRecord*), INTERNAL_MEM_ROOTS_TABLE);
4359 for (i = 0; i < roots_hash_size [pinned]; ++i) {
4360 for (entry = roots_hash [pinned][i]; entry; entry = next) {
4361 hash = mono_aligned_addr_hash (entry->start_root) % new_size;
4363 entry->next = new_hash [hash];
4364 new_hash [hash] = entry;
4367 mono_sgen_free_internal_dynamic (roots_hash [pinned], roots_hash_size [pinned] * sizeof (RootRecord*), INTERNAL_MEM_ROOTS_TABLE);
4368 roots_hash [pinned] = new_hash;
4369 roots_hash_size [pinned] = new_size;
4373 find_root (int root_type, char *start, guint32 addr_hash)
4375 RootRecord *new_root;
4377 guint32 hash = addr_hash % roots_hash_size [root_type];
4378 for (new_root = roots_hash [root_type][hash]; new_root; new_root = new_root->next) {
4379 /* we allow changing the size and the descriptor (for thread statics etc) */
4380 if (new_root->start_root == start) {
4389 * We do not coalesce roots.
4392 mono_gc_register_root_inner (char *start, size_t size, void *descr, int root_type)
4394 RootRecord *new_root;
4395 unsigned int hash, addr_hash = mono_aligned_addr_hash (start);
4398 for (i = 0; i < ROOT_TYPE_NUM; ++i) {
4399 if (num_roots_entries [i] >= roots_hash_size [i] * 2)
4402 for (i = 0; i < ROOT_TYPE_NUM; ++i) {
4403 new_root = find_root (i, start, addr_hash);
4404 /* we allow changing the size and the descriptor (for thread statics etc) */
4406 size_t old_size = new_root->end_root - new_root->start_root;
4407 new_root->end_root = new_root->start_root + size;
4408 g_assert (((new_root->root_desc != 0) && (descr != NULL)) ||
4409 ((new_root->root_desc == 0) && (descr == NULL)));
4410 new_root->root_desc = (mword)descr;
4412 roots_size -= old_size;
4417 new_root = mono_sgen_alloc_internal (INTERNAL_MEM_ROOT_RECORD);
4419 new_root->start_root = start;
4420 new_root->end_root = new_root->start_root + size;
4421 new_root->root_desc = (mword)descr;
4423 hash = addr_hash % roots_hash_size [root_type];
4424 num_roots_entries [root_type]++;
4425 new_root->next = roots_hash [root_type] [hash];
4426 roots_hash [root_type][hash] = new_root;
4427 DEBUG (3, fprintf (gc_debug_file, "Added root %p for range: %p-%p, descr: %p (%d/%d bytes)\n", new_root, new_root->start_root, new_root->end_root, descr, (int)size, (int)roots_size));
4437 mono_gc_register_root (char *start, size_t size, void *descr)
4439 return mono_gc_register_root_inner (start, size, descr, descr ? ROOT_TYPE_NORMAL : ROOT_TYPE_PINNED);
4443 mono_gc_register_root_wbarrier (char *start, size_t size, void *descr)
4445 return mono_gc_register_root_inner (start, size, descr, ROOT_TYPE_WBARRIER);
4449 mono_gc_deregister_root (char* addr)
4451 RootRecord *tmp, *prev;
4452 unsigned int hash, addr_hash = mono_aligned_addr_hash (addr);
4456 for (root_type = 0; root_type < ROOT_TYPE_NUM; ++root_type) {
4457 hash = addr_hash % roots_hash_size [root_type];
4458 tmp = roots_hash [root_type][hash];
4461 if (tmp->start_root == (char*)addr) {
4463 prev->next = tmp->next;
4465 roots_hash [root_type][hash] = tmp->next;
4466 roots_size -= (tmp->end_root - tmp->start_root);
4467 num_roots_entries [root_type]--;
4468 DEBUG (3, fprintf (gc_debug_file, "Removed root %p for range: %p-%p\n", tmp, tmp->start_root, tmp->end_root));
4469 mono_sgen_free_internal (tmp, INTERNAL_MEM_ROOT_RECORD);
4480 * ######################################################################
4481 * ######## Thread handling (stop/start code)
4482 * ######################################################################
4485 /* FIXME: handle large/small config */
4486 #define HASH_PTHREAD_T(id) (((unsigned int)(id) >> 4) * 2654435761u)
4488 static SgenThreadInfo* thread_table [THREAD_HASH_SIZE];
4490 #if USE_SIGNAL_BASED_START_STOP_WORLD
4492 static MonoSemType suspend_ack_semaphore;
4493 static MonoSemType *suspend_ack_semaphore_ptr;
4494 static unsigned int global_stop_count = 0;
4496 static sigset_t suspend_signal_mask;
4497 static mword cur_thread_regs [ARCH_NUM_REGS] = {0};
4499 /* LOCKING: assumes the GC lock is held */
4501 mono_sgen_get_thread_table (void)
4503 return thread_table;
4507 mono_sgen_thread_info_lookup (ARCH_THREAD_TYPE id)
4509 unsigned int hash = HASH_PTHREAD_T (id) % THREAD_HASH_SIZE;
4510 SgenThreadInfo *info;
4512 info = thread_table [hash];
4513 while (info && !ARCH_THREAD_EQUALS (info->id, id)) {
4520 update_current_thread_stack (void *start)
4522 void *ptr = cur_thread_regs;
4523 SgenThreadInfo *info = mono_sgen_thread_info_lookup (ARCH_GET_THREAD ());
4525 info->stack_start = align_pointer (&ptr);
4526 g_assert (info->stack_start >= info->stack_start_limit && info->stack_start < info->stack_end);
4527 ARCH_STORE_REGS (ptr);
4528 info->stopped_regs = ptr;
4529 if (gc_callbacks.thread_suspend_func)
4530 gc_callbacks.thread_suspend_func (info->runtime_data, NULL);
4534 * Define this and use the "xdomain-checks" MONO_GC_DEBUG option to
4535 * have cross-domain checks in the write barrier.
4537 //#define XDOMAIN_CHECKS_IN_WBARRIER
4539 #ifndef SGEN_BINARY_PROTOCOL
4540 #ifndef HEAVY_STATISTICS
4541 #define MANAGED_ALLOCATION
4542 #ifndef XDOMAIN_CHECKS_IN_WBARRIER
4543 #define MANAGED_WBARRIER
4549 is_ip_in_managed_allocator (MonoDomain *domain, gpointer ip);
4552 mono_sgen_wait_for_suspend_ack (int count)
4556 for (i = 0; i < count; ++i) {
4557 while ((result = MONO_SEM_WAIT (suspend_ack_semaphore_ptr)) != 0) {
4558 if (errno != EINTR) {
4559 g_error ("sem_wait ()");
4566 restart_threads_until_none_in_managed_allocator (void)
4568 SgenThreadInfo *info;
4569 int i, result, num_threads_died = 0;
4570 int sleep_duration = -1;
4573 int restart_count = 0, restarted_count = 0;
4574 /* restart all threads that stopped in the
4576 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
4577 for (info = thread_table [i]; info; info = info->next) {
4580 if (!info->stack_start || info->in_critical_region ||
4581 is_ip_in_managed_allocator (info->stopped_domain, info->stopped_ip)) {
4582 binary_protocol_thread_restart ((gpointer)info->id);
4583 #if defined(__MACH__) && MONO_MACH_ARCH_SUPPORTED
4584 result = thread_resume (pthread_mach_thread_np (info->id));
4586 result = pthread_kill (info->id, restart_signal_num);
4594 /* we set the stopped_ip to
4595 NULL for threads which
4596 we're not restarting so
4597 that we can easily identify
4599 info->stopped_ip = NULL;
4600 info->stopped_domain = NULL;
4604 /* if no threads were restarted, we're done */
4605 if (restart_count == 0)
4608 #if defined(__MACH__) && MONO_MACH_ARCH_SUPPORTED
4609 /* mach thread_resume is synchronous so we dont need to wait for them */
4611 /* wait for the threads to signal their restart */
4612 mono_sgen_wait_for_suspend_ack (restart_count);
4615 if (sleep_duration < 0) {
4619 g_usleep (sleep_duration);
4620 sleep_duration += 10;
4623 /* stop them again */
4624 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
4625 for (info = thread_table [i]; info; info = info->next) {
4626 if (info->skip || info->stopped_ip == NULL)
4628 #if defined(__MACH__) && MONO_MACH_ARCH_SUPPORTED
4629 result = thread_suspend (pthread_mach_thread_np (info->id));
4631 result = pthread_kill (info->id, suspend_signal_num);
4640 /* some threads might have died */
4641 num_threads_died += restart_count - restarted_count;
4642 #if defined(__MACH__) && MONO_MACH_ARCH_SUPPORTED
4643 /* mach thread_resume is synchronous so we dont need to wait for them */
4645 /* wait for the threads to signal their suspension
4647 mono_sgen_wait_for_suspend_ack (restart_count);
4651 return num_threads_died;
4654 /* LOCKING: assumes the GC lock is held (by the stopping thread) */
4656 suspend_handler (int sig, siginfo_t *siginfo, void *context)
4658 SgenThreadInfo *info;
4661 int old_errno = errno;
4662 gpointer regs [ARCH_NUM_REGS];
4663 gpointer stack_start;
4665 id = pthread_self ();
4666 info = mono_sgen_thread_info_lookup (id);
4667 info->stopped_domain = mono_domain_get ();
4668 info->stopped_ip = (gpointer) ARCH_SIGCTX_IP (context);
4669 stop_count = global_stop_count;
4670 /* duplicate signal */
4671 if (0 && info->stop_count == stop_count) {
4675 #ifdef HAVE_KW_THREAD
4676 /* update the remset info in the thread data structure */
4677 info->remset = remembered_set;
4679 stack_start = (char*) ARCH_SIGCTX_SP (context) - REDZONE_SIZE;
4680 /* If stack_start is not within the limits, then don't set it
4681 in info and we will be restarted. */
4682 if (stack_start >= info->stack_start_limit && info->stack_start <= info->stack_end) {
4683 info->stack_start = stack_start;
4685 ARCH_COPY_SIGCTX_REGS (regs, context);
4686 info->stopped_regs = regs;
4688 g_assert (!info->stack_start);
4691 /* Notify the JIT */
4692 if (gc_callbacks.thread_suspend_func)
4693 gc_callbacks.thread_suspend_func (info->runtime_data, context);
4695 DEBUG (4, fprintf (gc_debug_file, "Posting suspend_ack_semaphore for suspend from %p %p\n", info, (gpointer)ARCH_GET_THREAD ()));
4696 /* notify the waiting thread */
4697 MONO_SEM_POST (suspend_ack_semaphore_ptr);
4698 info->stop_count = stop_count;
4700 /* wait until we receive the restart signal */
4703 sigsuspend (&suspend_signal_mask);
4704 } while (info->signal != restart_signal_num);
4706 DEBUG (4, fprintf (gc_debug_file, "Posting suspend_ack_semaphore for resume from %p %p\n", info, (gpointer)ARCH_GET_THREAD ()));
4707 /* notify the waiting thread */
4708 MONO_SEM_POST (suspend_ack_semaphore_ptr);
4714 restart_handler (int sig)
4716 SgenThreadInfo *info;
4717 int old_errno = errno;
4719 info = mono_sgen_thread_info_lookup (pthread_self ());
4720 info->signal = restart_signal_num;
4721 DEBUG (4, fprintf (gc_debug_file, "Restart handler in %p %p\n", info, (gpointer)ARCH_GET_THREAD ()));
4727 acquire_gc_locks (void)
4733 release_gc_locks (void)
4735 UNLOCK_INTERRUPTION;
4738 static TV_DECLARE (stop_world_time);
4739 static unsigned long max_pause_usec = 0;
4741 /* LOCKING: assumes the GC lock is held */
4743 stop_world (int generation)
4747 mono_profiler_gc_event (MONO_GC_EVENT_PRE_STOP_WORLD, generation);
4748 acquire_gc_locks ();
4750 update_current_thread_stack (&count);
4752 global_stop_count++;
4753 DEBUG (3, fprintf (gc_debug_file, "stopping world n %d from %p %p\n", global_stop_count, mono_sgen_thread_info_lookup (ARCH_GET_THREAD ()), (gpointer)ARCH_GET_THREAD ()));
4754 TV_GETTIME (stop_world_time);
4755 count = mono_sgen_thread_handshake (suspend_signal_num);
4756 count -= restart_threads_until_none_in_managed_allocator ();
4757 g_assert (count >= 0);
4758 DEBUG (3, fprintf (gc_debug_file, "world stopped %d thread(s)\n", count));
4759 mono_profiler_gc_event (MONO_GC_EVENT_POST_STOP_WORLD, generation);
4763 /* LOCKING: assumes the GC lock is held */
4765 restart_world (int generation)
4768 SgenThreadInfo *info;
4769 TV_DECLARE (end_sw);
4772 /* notify the profiler of the leftovers */
4773 if (G_UNLIKELY (mono_profiler_events & MONO_PROFILE_GC_MOVES)) {
4774 if (moved_objects_idx) {
4775 mono_profiler_gc_moves (moved_objects, moved_objects_idx);
4776 moved_objects_idx = 0;
4779 mono_profiler_gc_event (MONO_GC_EVENT_PRE_START_WORLD, generation);
4780 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
4781 for (info = thread_table [i]; info; info = info->next) {
4782 info->stack_start = NULL;
4783 info->stopped_regs = NULL;
4787 release_gc_locks ();
4789 count = mono_sgen_thread_handshake (restart_signal_num);
4790 TV_GETTIME (end_sw);
4791 usec = TV_ELAPSED (stop_world_time, end_sw);
4792 max_pause_usec = MAX (usec, max_pause_usec);
4793 DEBUG (2, fprintf (gc_debug_file, "restarted %d thread(s) (pause time: %d usec, max: %d)\n", count, (int)usec, (int)max_pause_usec));
4794 mono_profiler_gc_event (MONO_GC_EVENT_POST_START_WORLD, generation);
4798 #endif /* USE_SIGNAL_BASED_START_STOP_WORLD */
4801 mono_gc_set_gc_callbacks (MonoGCCallbacks *callbacks)
4803 gc_callbacks = *callbacks;
4807 mono_gc_get_gc_callbacks ()
4809 return &gc_callbacks;
4812 /* Variables holding start/end nursery so it won't have to be passed at every call */
4813 static void *scan_area_arg_start, *scan_area_arg_end;
4816 mono_gc_conservatively_scan_area (void *start, void *end)
4818 conservatively_pin_objects_from (start, end, scan_area_arg_start, scan_area_arg_end, PIN_TYPE_STACK);
4822 mono_gc_scan_object (void *obj)
4824 g_assert_not_reached ();
4825 if (current_collection_generation == GENERATION_NURSERY)
4826 major_collector.copy_object (&obj, &gray_queue);
4828 major_collector.copy_or_mark_object (&obj, &gray_queue);
4833 * Mark from thread stacks and registers.
4836 scan_thread_data (void *start_nursery, void *end_nursery, gboolean precise)
4839 SgenThreadInfo *info;
4841 scan_area_arg_start = start_nursery;
4842 scan_area_arg_end = end_nursery;
4844 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
4845 for (info = thread_table [i]; info; info = info->next) {
4847 DEBUG (3, fprintf (gc_debug_file, "Skipping dead thread %p, range: %p-%p, size: %td\n", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start));
4850 DEBUG (3, fprintf (gc_debug_file, "Scanning thread %p, range: %p-%p, size: %td, pinned=%d\n", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start, next_pin_slot));
4851 if (gc_callbacks.thread_mark_func && !conservative_stack_mark)
4852 gc_callbacks.thread_mark_func (info->runtime_data, info->stack_start, info->stack_end, precise);
4854 conservatively_pin_objects_from (info->stack_start, info->stack_end, start_nursery, end_nursery, PIN_TYPE_STACK);
4857 conservatively_pin_objects_from (info->stopped_regs, info->stopped_regs + ARCH_NUM_REGS,
4858 start_nursery, end_nursery, PIN_TYPE_STACK);
4864 find_pinning_ref_from_thread (char *obj, size_t size)
4867 SgenThreadInfo *info;
4868 char *endobj = obj + size;
4870 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
4871 for (info = thread_table [i]; info; info = info->next) {
4872 char **start = (char**)info->stack_start;
4875 while (start < (char**)info->stack_end) {
4876 if (*start >= obj && *start < endobj) {
4877 DEBUG (0, fprintf (gc_debug_file, "Object %p referenced in thread %p (id %p) at %p, stack: %p-%p\n", obj, info, (gpointer)info->id, start, info->stack_start, info->stack_end));
4882 /* FIXME: check info->stopped_regs */
4888 ptr_on_stack (void *ptr)
4890 gpointer stack_start = &stack_start;
4891 SgenThreadInfo *info = mono_sgen_thread_info_lookup (ARCH_GET_THREAD ());
4893 if (ptr >= stack_start && ptr < (gpointer)info->stack_end)
4899 handle_remset (mword *p, void *start_nursery, void *end_nursery, gboolean global, GrayQueue *queue)
4906 HEAVY_STAT (++stat_global_remsets_processed);
4908 HEAVY_STAT (++stat_local_remsets_processed);
4910 /* FIXME: exclude stack locations */
4911 switch ((*p) & REMSET_TYPE_MASK) {
4912 case REMSET_LOCATION:
4914 //__builtin_prefetch (ptr);
4915 if (((void*)ptr < start_nursery || (void*)ptr >= end_nursery)) {
4916 gpointer old = *ptr;
4917 major_collector.copy_object (ptr, queue);
4918 DEBUG (9, fprintf (gc_debug_file, "Overwrote remset at %p with %p\n", ptr, *ptr));
4920 binary_protocol_ptr_update (ptr, old, *ptr, (gpointer)LOAD_VTABLE (*ptr), safe_object_get_size (*ptr));
4921 if (!global && *ptr >= start_nursery && *ptr < end_nursery) {
4923 * If the object is pinned, each reference to it from nonpinned objects
4924 * becomes part of the global remset, which can grow very large.
4926 DEBUG (9, fprintf (gc_debug_file, "Add to global remset because of pinning %p (%p %s)\n", ptr, *ptr, safe_name (*ptr)));
4927 mono_sgen_add_to_global_remset (ptr);
4930 DEBUG (9, fprintf (gc_debug_file, "Skipping remset at %p holding %p\n", ptr, *ptr));
4934 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
4935 if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery))
4938 while (count-- > 0) {
4939 major_collector.copy_object (ptr, queue);
4940 DEBUG (9, fprintf (gc_debug_file, "Overwrote remset at %p with %p (count: %d)\n", ptr, *ptr, (int)count));
4941 if (!global && *ptr >= start_nursery && *ptr < end_nursery)
4942 mono_sgen_add_to_global_remset (ptr);
4947 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
4948 if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery))
4950 major_collector.minor_scan_object ((char*)ptr, queue);
4952 case REMSET_VTYPE: {
4953 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
4954 if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery))
4959 ptr = (void**) major_collector.minor_scan_vtype ((char*)ptr, desc, start_nursery, end_nursery, queue);
4963 g_assert_not_reached ();
4968 #ifdef HEAVY_STATISTICS
4970 collect_store_remsets (RememberedSet *remset, mword *bumper)
4972 mword *p = remset->data;
4977 while (p < remset->store_next) {
4978 switch ((*p) & REMSET_TYPE_MASK) {
4979 case REMSET_LOCATION:
4982 ++stat_saved_remsets_1;
4984 if (*p == last1 || *p == last2) {
4985 ++stat_saved_remsets_2;
5002 g_assert_not_reached ();
5012 RememberedSet *remset;
5014 SgenThreadInfo *info;
5016 mword *addresses, *bumper, *p, *r;
5018 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5019 for (info = thread_table [i]; info; info = info->next) {
5020 for (remset = info->remset; remset; remset = remset->next)
5021 size += remset->store_next - remset->data;
5024 for (remset = freed_thread_remsets; remset; remset = remset->next)
5025 size += remset->store_next - remset->data;
5026 for (remset = global_remset; remset; remset = remset->next)
5027 size += remset->store_next - remset->data;
5029 bumper = addresses = mono_sgen_alloc_internal_dynamic (sizeof (mword) * size, INTERNAL_MEM_STATISTICS);
5031 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5032 for (info = thread_table [i]; info; info = info->next) {
5033 for (remset = info->remset; remset; remset = remset->next)
5034 bumper = collect_store_remsets (remset, bumper);
5037 for (remset = global_remset; remset; remset = remset->next)
5038 bumper = collect_store_remsets (remset, bumper);
5039 for (remset = freed_thread_remsets; remset; remset = remset->next)
5040 bumper = collect_store_remsets (remset, bumper);
5042 g_assert (bumper <= addresses + size);
5044 stat_store_remsets += bumper - addresses;
5046 sort_addresses ((void**)addresses, bumper - addresses);
5049 while (r < bumper) {
5055 stat_store_remsets_unique += p - addresses;
5057 mono_sgen_free_internal_dynamic (addresses, sizeof (mword) * size, INTERNAL_MEM_STATISTICS);
5062 clear_thread_store_remset_buffer (SgenThreadInfo *info)
5064 *info->store_remset_buffer_index_addr = 0;
5065 memset (*info->store_remset_buffer_addr, 0, sizeof (gpointer) * STORE_REMSET_BUFFER_SIZE);
5069 remset_byte_size (RememberedSet *remset)
5071 return sizeof (RememberedSet) + (remset->end_set - remset->data) * sizeof (gpointer);
5075 scan_from_remsets (void *start_nursery, void *end_nursery, GrayQueue *queue)
5078 SgenThreadInfo *info;
5079 RememberedSet *remset;
5080 GenericStoreRememberedSet *store_remset;
5081 mword *p, *next_p, *store_pos;
5083 #ifdef HEAVY_STATISTICS
5087 /* the global one */
5088 for (remset = global_remset; remset; remset = remset->next) {
5089 DEBUG (4, fprintf (gc_debug_file, "Scanning global remset range: %p-%p, size: %td\n", remset->data, remset->store_next, remset->store_next - remset->data));
5090 store_pos = remset->data;
5091 for (p = remset->data; p < remset->store_next; p = next_p) {
5092 void **ptr = (void**)p [0];
5094 /*Ignore previously processed remset.*/
5095 if (!global_remset_location_was_not_added (ptr)) {
5100 next_p = handle_remset (p, start_nursery, end_nursery, TRUE, queue);
5103 * Clear global remsets of locations which no longer point to the
5104 * nursery. Otherwise, they could grow indefinitely between major
5107 * Since all global remsets are location remsets, we don't need to unmask the pointer.
5109 if (ptr_in_nursery (*ptr)) {
5110 *store_pos ++ = p [0];
5111 HEAVY_STAT (++stat_global_remsets_readded);
5115 /* Truncate the remset */
5116 remset->store_next = store_pos;
5119 /* the generic store ones */
5120 store_remset = generic_store_remsets;
5121 while (store_remset) {
5122 GenericStoreRememberedSet *next = store_remset->next;
5124 for (i = 0; i < STORE_REMSET_BUFFER_SIZE - 1; ++i) {
5125 gpointer addr = store_remset->data [i];
5127 handle_remset ((mword*)&addr, start_nursery, end_nursery, FALSE, queue);
5130 mono_sgen_free_internal (store_remset, INTERNAL_MEM_STORE_REMSET);
5132 store_remset = next;
5134 generic_store_remsets = NULL;
5136 /* the per-thread ones */
5137 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5138 for (info = thread_table [i]; info; info = info->next) {
5139 RememberedSet *next;
5141 for (remset = info->remset; remset; remset = next) {
5142 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for thread %p, range: %p-%p, size: %td\n", info, remset->data, remset->store_next, remset->store_next - remset->data));
5143 for (p = remset->data; p < remset->store_next;)
5144 p = handle_remset (p, start_nursery, end_nursery, FALSE, queue);
5145 remset->store_next = remset->data;
5146 next = remset->next;
5147 remset->next = NULL;
5148 if (remset != info->remset) {
5149 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
5150 mono_sgen_free_internal_dynamic (remset, remset_byte_size (remset), INTERNAL_MEM_REMSET);
5153 for (j = 0; j < *info->store_remset_buffer_index_addr; ++j)
5154 handle_remset ((mword*)*info->store_remset_buffer_addr + j + 1, start_nursery, end_nursery, FALSE, queue);
5155 clear_thread_store_remset_buffer (info);
5159 /* the freed thread ones */
5160 while (freed_thread_remsets) {
5161 RememberedSet *next;
5162 remset = freed_thread_remsets;
5163 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for freed thread, range: %p-%p, size: %td\n", remset->data, remset->store_next, remset->store_next - remset->data));
5164 for (p = remset->data; p < remset->store_next;)
5165 p = handle_remset (p, start_nursery, end_nursery, FALSE, queue);
5166 next = remset->next;
5167 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
5168 mono_sgen_free_internal_dynamic (remset, remset_byte_size (remset), INTERNAL_MEM_REMSET);
5169 freed_thread_remsets = next;
5174 * Clear the info in the remembered sets: we're doing a major collection, so
5175 * the per-thread ones are not needed and the global ones will be reconstructed
5179 clear_remsets (void)
5182 SgenThreadInfo *info;
5183 RememberedSet *remset, *next;
5185 /* the global list */
5186 for (remset = global_remset; remset; remset = next) {
5187 remset->store_next = remset->data;
5188 next = remset->next;
5189 remset->next = NULL;
5190 if (remset != global_remset) {
5191 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
5192 mono_sgen_free_internal_dynamic (remset, remset_byte_size (remset), INTERNAL_MEM_REMSET);
5195 /* the generic store ones */
5196 while (generic_store_remsets) {
5197 GenericStoreRememberedSet *gs_next = generic_store_remsets->next;
5198 mono_sgen_free_internal (generic_store_remsets, INTERNAL_MEM_STORE_REMSET);
5199 generic_store_remsets = gs_next;
5201 /* the per-thread ones */
5202 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5203 for (info = thread_table [i]; info; info = info->next) {
5204 for (remset = info->remset; remset; remset = next) {
5205 remset->store_next = remset->data;
5206 next = remset->next;
5207 remset->next = NULL;
5208 if (remset != info->remset) {
5209 DEBUG (3, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
5210 mono_sgen_free_internal_dynamic (remset, remset_byte_size (remset), INTERNAL_MEM_REMSET);
5213 clear_thread_store_remset_buffer (info);
5217 /* the freed thread ones */
5218 while (freed_thread_remsets) {
5219 next = freed_thread_remsets->next;
5220 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", freed_thread_remsets->data));
5221 mono_sgen_free_internal_dynamic (freed_thread_remsets, remset_byte_size (freed_thread_remsets), INTERNAL_MEM_REMSET);
5222 freed_thread_remsets = next;
5227 * Clear the thread local TLAB variables for all threads.
5232 SgenThreadInfo *info;
5235 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5236 for (info = thread_table [i]; info; info = info->next) {
5237 /* A new TLAB will be allocated when the thread does its first allocation */
5238 *info->tlab_start_addr = NULL;
5239 *info->tlab_next_addr = NULL;
5240 *info->tlab_temp_end_addr = NULL;
5241 *info->tlab_real_end_addr = NULL;
5246 /* LOCKING: assumes the GC lock is held */
5247 static SgenThreadInfo*
5248 gc_register_current_thread (void *addr)
5251 SgenThreadInfo* info = malloc (sizeof (SgenThreadInfo));
5252 #ifndef HAVE_KW_THREAD
5253 SgenThreadInfo *__thread_info__ = info;
5259 memset (info, 0, sizeof (SgenThreadInfo));
5260 #ifndef HAVE_KW_THREAD
5261 info->tlab_start = info->tlab_next = info->tlab_temp_end = info->tlab_real_end = NULL;
5263 g_assert (!pthread_getspecific (thread_info_key));
5264 pthread_setspecific (thread_info_key, info);
5269 info->id = ARCH_GET_THREAD ();
5270 info->stop_count = -1;
5273 info->stack_start = NULL;
5274 info->tlab_start_addr = &TLAB_START;
5275 info->tlab_next_addr = &TLAB_NEXT;
5276 info->tlab_temp_end_addr = &TLAB_TEMP_END;
5277 info->tlab_real_end_addr = &TLAB_REAL_END;
5278 info->store_remset_buffer_addr = &STORE_REMSET_BUFFER;
5279 info->store_remset_buffer_index_addr = &STORE_REMSET_BUFFER_INDEX;
5280 info->stopped_ip = NULL;
5281 info->stopped_domain = NULL;
5282 info->stopped_regs = NULL;
5284 binary_protocol_thread_register ((gpointer)info->id);
5286 #ifdef HAVE_KW_THREAD
5287 tlab_next_addr = &tlab_next;
5288 store_remset_buffer_index_addr = &store_remset_buffer_index;
5291 /* try to get it with attributes first */
5292 #if defined(HAVE_PTHREAD_GETATTR_NP) && defined(HAVE_PTHREAD_ATTR_GETSTACK)
5296 pthread_attr_t attr;
5297 pthread_getattr_np (pthread_self (), &attr);
5298 pthread_attr_getstack (&attr, &sstart, &size);
5299 info->stack_start_limit = sstart;
5300 info->stack_end = (char*)sstart + size;
5301 pthread_attr_destroy (&attr);
5303 #elif defined(HAVE_PTHREAD_GET_STACKSIZE_NP) && defined(HAVE_PTHREAD_GET_STACKADDR_NP)
5304 info->stack_end = (char*)pthread_get_stackaddr_np (pthread_self ());
5305 info->stack_start_limit = (char*)info->stack_end - pthread_get_stacksize_np (pthread_self ());
5308 /* FIXME: we assume the stack grows down */
5309 gsize stack_bottom = (gsize)addr;
5310 stack_bottom += 4095;
5311 stack_bottom &= ~4095;
5312 info->stack_end = (char*)stack_bottom;
5316 #ifdef HAVE_KW_THREAD
5317 stack_end = info->stack_end;
5320 /* hash into the table */
5321 hash = HASH_PTHREAD_T (info->id) % THREAD_HASH_SIZE;
5322 info->next = thread_table [hash];
5323 thread_table [hash] = info;
5325 info->remset = alloc_remset (DEFAULT_REMSET_SIZE, info);
5326 pthread_setspecific (remembered_set_key, info->remset);
5327 #ifdef HAVE_KW_THREAD
5328 remembered_set = info->remset;
5331 STORE_REMSET_BUFFER = mono_sgen_alloc_internal (INTERNAL_MEM_STORE_REMSET);
5332 STORE_REMSET_BUFFER_INDEX = 0;
5334 DEBUG (3, fprintf (gc_debug_file, "registered thread %p (%p) (hash: %d)\n", info, (gpointer)info->id, hash));
5336 if (gc_callbacks.thread_attach_func)
5337 info->runtime_data = gc_callbacks.thread_attach_func ();
5343 add_generic_store_remset_from_buffer (gpointer *buffer)
5345 GenericStoreRememberedSet *remset = mono_sgen_alloc_internal (INTERNAL_MEM_STORE_REMSET);
5346 memcpy (remset->data, buffer + 1, sizeof (gpointer) * (STORE_REMSET_BUFFER_SIZE - 1));
5347 remset->next = generic_store_remsets;
5348 generic_store_remsets = remset;
5352 unregister_current_thread (void)
5355 SgenThreadInfo *prev = NULL;
5357 RememberedSet *rset;
5358 ARCH_THREAD_TYPE id = ARCH_GET_THREAD ();
5360 binary_protocol_thread_unregister ((gpointer)id);
5362 hash = HASH_PTHREAD_T (id) % THREAD_HASH_SIZE;
5363 p = thread_table [hash];
5365 DEBUG (3, fprintf (gc_debug_file, "unregister thread %p (%p)\n", p, (gpointer)p->id));
5366 while (!ARCH_THREAD_EQUALS (p->id, id)) {
5371 thread_table [hash] = p->next;
5373 prev->next = p->next;
5376 if (freed_thread_remsets) {
5377 for (rset = p->remset; rset->next; rset = rset->next)
5379 rset->next = freed_thread_remsets;
5380 freed_thread_remsets = p->remset;
5382 freed_thread_remsets = p->remset;
5385 if (*p->store_remset_buffer_index_addr)
5386 add_generic_store_remset_from_buffer (*p->store_remset_buffer_addr);
5387 mono_sgen_free_internal (*p->store_remset_buffer_addr, INTERNAL_MEM_STORE_REMSET);
5392 unregister_thread (void *k)
5394 g_assert (!mono_domain_get ());
5396 unregister_current_thread ();
5401 mono_gc_register_thread (void *baseptr)
5403 SgenThreadInfo *info;
5407 info = mono_sgen_thread_info_lookup (ARCH_GET_THREAD ());
5409 info = gc_register_current_thread (baseptr);
5412 /* Need a better place to initialize this */
5413 if (!array_fill_vtable && mono_get_root_domain ()) {
5414 array_fill_vtable = mono_class_vtable (mono_get_root_domain (), mono_array_class_get (mono_defaults.byte_class, 1));
5417 return info != NULL;
5420 #if USE_PTHREAD_INTERCEPT
5423 void *(*start_routine) (void *);
5426 MonoSemType registered;
5427 } SgenThreadStartInfo;
5430 gc_start_thread (void *arg)
5432 SgenThreadStartInfo *start_info = arg;
5433 SgenThreadInfo* info;
5434 void *t_arg = start_info->arg;
5435 void *(*start_func) (void*) = start_info->start_routine;
5440 info = gc_register_current_thread (&result);
5442 post_result = MONO_SEM_POST (&(start_info->registered));
5443 g_assert (!post_result);
5444 result = start_func (t_arg);
5445 g_assert (!mono_domain_get ());
5447 * this is done by the pthread key dtor
5449 unregister_current_thread ();
5457 mono_gc_pthread_create (pthread_t *new_thread, const pthread_attr_t *attr, void *(*start_routine)(void *), void *arg)
5459 SgenThreadStartInfo *start_info;
5462 start_info = malloc (sizeof (SgenThreadStartInfo));
5465 MONO_SEM_INIT (&(start_info->registered), 0);
5466 start_info->arg = arg;
5467 start_info->start_routine = start_routine;
5469 result = pthread_create (new_thread, attr, gc_start_thread, start_info);
5471 while (MONO_SEM_WAIT (&(start_info->registered)) != 0) {
5472 /*if (EINTR != errno) ABORT("sem_wait failed"); */
5475 MONO_SEM_DESTROY (&(start_info->registered));
5481 mono_gc_pthread_join (pthread_t thread, void **retval)
5483 return pthread_join (thread, retval);
5487 mono_gc_pthread_detach (pthread_t thread)
5489 return pthread_detach (thread);
5492 #endif /* USE_PTHREAD_INTERCEPT */
5495 * ######################################################################
5496 * ######## Write barriers
5497 * ######################################################################
5501 * This causes the compile to extend the liveness of 'v' till the call to dummy_use
5504 dummy_use (gpointer v) {
5505 __asm__ volatile ("" : "=r"(v) : "r"(v));
5509 static RememberedSet*
5510 alloc_remset (int size, gpointer id) {
5511 RememberedSet* res = mono_sgen_alloc_internal_dynamic (sizeof (RememberedSet) + (size * sizeof (gpointer)), INTERNAL_MEM_REMSET);
5512 res->store_next = res->data;
5513 res->end_set = res->data + size;
5515 DEBUG (4, fprintf (gc_debug_file, "Allocated remset size %d at %p for %p\n", size, res->data, id));
5520 * Note: the write barriers first do the needed GC work and then do the actual store:
5521 * this way the value is visible to the conservative GC scan after the write barrier
5522 * itself. If a GC interrupts the barrier in the middle, value will be kept alive by
5523 * the conservative scan, otherwise by the remembered set scan.
5526 mono_gc_wbarrier_set_field (MonoObject *obj, gpointer field_ptr, MonoObject* value)
5528 HEAVY_STAT (++stat_wbarrier_set_field);
5529 if (ptr_in_nursery (field_ptr)) {
5530 *(void**)field_ptr = value;
5533 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", field_ptr));
5534 if (use_cardtable) {
5535 *(void**)field_ptr = value;
5536 if (ptr_in_nursery (value))
5537 sgen_card_table_mark_address ((mword)field_ptr);
5544 rs = REMEMBERED_SET;
5545 if (rs->store_next < rs->end_set) {
5546 *(rs->store_next++) = (mword)field_ptr;
5547 *(void**)field_ptr = value;
5551 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
5552 rs->next = REMEMBERED_SET;
5553 REMEMBERED_SET = rs;
5554 #ifdef HAVE_KW_THREAD
5555 mono_sgen_thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
5557 *(rs->store_next++) = (mword)field_ptr;
5558 *(void**)field_ptr = value;
5564 mono_gc_wbarrier_set_arrayref (MonoArray *arr, gpointer slot_ptr, MonoObject* value)
5566 HEAVY_STAT (++stat_wbarrier_set_arrayref);
5567 if (ptr_in_nursery (slot_ptr)) {
5568 *(void**)slot_ptr = value;
5571 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", slot_ptr));
5572 if (use_cardtable) {
5573 *(void**)slot_ptr = value;
5574 if (ptr_in_nursery (value))
5575 sgen_card_table_mark_address ((mword)slot_ptr);
5582 rs = REMEMBERED_SET;
5583 if (rs->store_next < rs->end_set) {
5584 *(rs->store_next++) = (mword)slot_ptr;
5585 *(void**)slot_ptr = value;
5589 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
5590 rs->next = REMEMBERED_SET;
5591 REMEMBERED_SET = rs;
5592 #ifdef HAVE_KW_THREAD
5593 mono_sgen_thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
5595 *(rs->store_next++) = (mword)slot_ptr;
5596 *(void**)slot_ptr = value;
5602 mono_gc_wbarrier_arrayref_copy (gpointer dest_ptr, gpointer src_ptr, int count)
5604 HEAVY_STAT (++stat_wbarrier_arrayref_copy);
5605 /*This check can be done without taking a lock since dest_ptr array is pinned*/
5606 if (ptr_in_nursery (dest_ptr) || count <= 0) {
5607 memmove (dest_ptr, src_ptr, count * sizeof (gpointer));
5611 if (use_cardtable) {
5612 gpointer *dest = dest_ptr;
5613 gpointer *src = src_ptr;
5615 /*overlapping that required backward copying*/
5616 if (src < dest && (src + count) > dest) {
5617 gpointer *start = dest;
5621 for (; dest >= start; --src, --dest) {
5622 gpointer value = *src;
5624 if (ptr_in_nursery (value))
5625 sgen_card_table_mark_address ((mword)dest);
5629 gpointer *end = dest + count;
5630 for (; dest < end; ++src, ++dest) {
5631 gpointer value = *src;
5633 if (ptr_in_nursery (value))
5634 sgen_card_table_mark_address ((mword)dest);
5642 memmove (dest_ptr, src_ptr, count * sizeof (gpointer));
5644 rs = REMEMBERED_SET;
5645 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p, %d\n", dest_ptr, count));
5646 if (rs->store_next + 1 < rs->end_set) {
5647 *(rs->store_next++) = (mword)dest_ptr | REMSET_RANGE;
5648 *(rs->store_next++) = count;
5652 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
5653 rs->next = REMEMBERED_SET;
5654 REMEMBERED_SET = rs;
5655 #ifdef HAVE_KW_THREAD
5656 mono_sgen_thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
5658 *(rs->store_next++) = (mword)dest_ptr | REMSET_RANGE;
5659 *(rs->store_next++) = count;
5665 static char *found_obj;
5668 find_object_for_ptr_callback (char *obj, size_t size, char *ptr)
5670 if (ptr >= obj && ptr < obj + size) {
5671 g_assert (!found_obj);
5676 /* for use in the debugger */
5677 char* find_object_for_ptr (char *ptr);
5679 find_object_for_ptr (char *ptr)
5683 if (ptr >= nursery_section->data && ptr < nursery_section->end_data) {
5685 mono_sgen_scan_area_with_callback (nursery_section->data, nursery_section->end_data,
5686 (IterateObjectCallbackFunc)find_object_for_ptr_callback, ptr);
5691 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
5692 if (ptr >= bigobj->data && ptr < bigobj->data + bigobj->size)
5693 return bigobj->data;
5697 * Very inefficient, but this is debugging code, supposed to
5698 * be called from gdb, so we don't care.
5701 major_collector.iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)find_object_for_ptr_callback, ptr);
5706 evacuate_remset_buffer (void)
5711 buffer = STORE_REMSET_BUFFER;
5713 add_generic_store_remset_from_buffer (buffer);
5714 memset (buffer, 0, sizeof (gpointer) * STORE_REMSET_BUFFER_SIZE);
5716 STORE_REMSET_BUFFER_INDEX = 0;
5720 mono_gc_wbarrier_generic_nostore (gpointer ptr)
5726 HEAVY_STAT (++stat_wbarrier_generic_store);
5728 #ifdef XDOMAIN_CHECKS_IN_WBARRIER
5729 /* FIXME: ptr_in_heap must be called with the GC lock held */
5730 if (xdomain_checks && *(MonoObject**)ptr && ptr_in_heap (ptr)) {
5731 char *start = find_object_for_ptr (ptr);
5732 MonoObject *value = *(MonoObject**)ptr;
5736 MonoObject *obj = (MonoObject*)start;
5737 if (obj->vtable->domain != value->vtable->domain)
5738 g_assert (is_xdomain_ref_allowed (ptr, start, obj->vtable->domain));
5744 if (*(gpointer*)ptr)
5745 binary_protocol_wbarrier (ptr, *(gpointer*)ptr, (gpointer)LOAD_VTABLE (*(gpointer*)ptr));
5747 if (ptr_in_nursery (ptr) || ptr_on_stack (ptr) || !ptr_in_nursery (*(gpointer*)ptr)) {
5748 DEBUG (8, fprintf (gc_debug_file, "Skipping remset at %p\n", ptr));
5752 if (use_cardtable) {
5753 if (ptr_in_nursery(*(gpointer*)ptr))
5754 sgen_card_table_mark_address ((mword)ptr);
5760 buffer = STORE_REMSET_BUFFER;
5761 index = STORE_REMSET_BUFFER_INDEX;
5762 /* This simple optimization eliminates a sizable portion of
5763 entries. Comparing it to the last but one entry as well
5764 doesn't eliminate significantly more entries. */
5765 if (buffer [index] == ptr) {
5770 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", ptr));
5771 HEAVY_STAT (++stat_wbarrier_generic_store_remset);
5774 if (index >= STORE_REMSET_BUFFER_SIZE) {
5775 evacuate_remset_buffer ();
5776 index = STORE_REMSET_BUFFER_INDEX;
5777 g_assert (index == 0);
5780 buffer [index] = ptr;
5781 STORE_REMSET_BUFFER_INDEX = index;
5787 mono_gc_wbarrier_generic_store (gpointer ptr, MonoObject* value)
5789 DEBUG (8, fprintf (gc_debug_file, "Wbarrier store at %p to %p (%s)\n", ptr, value, value ? safe_name (value) : "null"));
5790 *(void**)ptr = value;
5791 if (ptr_in_nursery (value))
5792 mono_gc_wbarrier_generic_nostore (ptr);
5796 void mono_gc_wbarrier_value_copy_bitmap (gpointer _dest, gpointer _src, int size, unsigned bitmap)
5798 mword *dest = _dest;
5803 mono_gc_wbarrier_generic_store (dest, (MonoObject*)*src);
5808 size -= SIZEOF_VOID_P;
5815 mono_gc_wbarrier_value_copy (gpointer dest, gpointer src, int count, MonoClass *klass)
5818 size_t size = count * mono_class_value_size (klass, NULL);
5820 HEAVY_STAT (++stat_wbarrier_value_copy);
5821 g_assert (klass->valuetype);
5823 memmove (dest, src, size);
5824 if (use_cardtable) {
5825 sgen_card_table_mark_range ((mword)dest, size);
5827 rs = REMEMBERED_SET;
5828 if (ptr_in_nursery (dest) || ptr_on_stack (dest) || !klass->has_references) {
5832 g_assert (klass->gc_descr_inited);
5833 DEBUG (8, fprintf (gc_debug_file, "Adding value remset at %p, count %d, descr %p for class %s (%p)\n", dest, count, klass->gc_descr, klass->name, klass));
5835 if (rs->store_next + 3 < rs->end_set) {
5836 *(rs->store_next++) = (mword)dest | REMSET_VTYPE;
5837 *(rs->store_next++) = (mword)klass->gc_descr;
5838 *(rs->store_next++) = (mword)count;
5842 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
5843 rs->next = REMEMBERED_SET;
5844 REMEMBERED_SET = rs;
5845 #ifdef HAVE_KW_THREAD
5846 mono_sgen_thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
5848 *(rs->store_next++) = (mword)dest | REMSET_VTYPE;
5849 *(rs->store_next++) = (mword)klass->gc_descr;
5850 *(rs->store_next++) = (mword)count;
5856 * mono_gc_wbarrier_object_copy:
5858 * Write barrier to call when obj is the result of a clone or copy of an object.
5861 mono_gc_wbarrier_object_copy (MonoObject* obj, MonoObject *src)
5867 HEAVY_STAT (++stat_wbarrier_object_copy);
5868 rs = REMEMBERED_SET;
5869 DEBUG (6, fprintf (gc_debug_file, "Adding object remset for %p\n", obj));
5870 size = mono_object_class (obj)->instance_size;
5872 /* do not copy the sync state */
5873 memcpy ((char*)obj + sizeof (MonoObject), (char*)src + sizeof (MonoObject),
5874 size - sizeof (MonoObject));
5875 if (ptr_in_nursery (obj) || ptr_on_stack (obj)) {
5879 if (rs->store_next < rs->end_set) {
5880 *(rs->store_next++) = (mword)obj | REMSET_OBJECT;
5884 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
5885 rs->next = REMEMBERED_SET;
5886 REMEMBERED_SET = rs;
5887 #ifdef HAVE_KW_THREAD
5888 mono_sgen_thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
5890 *(rs->store_next++) = (mword)obj | REMSET_OBJECT;
5895 * ######################################################################
5896 * ######## Collector debugging
5897 * ######################################################################
5900 const char*descriptor_types [] = {
5912 describe_ptr (char *ptr)
5918 if (ptr_in_nursery (ptr)) {
5919 printf ("Pointer inside nursery.\n");
5921 if (major_collector.ptr_is_in_non_pinned_space (ptr)) {
5922 printf ("Pointer inside oldspace.\n");
5923 } else if (major_collector.obj_is_from_pinned_alloc (ptr)) {
5924 printf ("Pointer is inside a pinned chunk.\n");
5926 printf ("Pointer unknown.\n");
5931 if (object_is_pinned (ptr))
5932 printf ("Object is pinned.\n");
5934 if (object_is_forwarded (ptr))
5935 printf ("Object is forwared.\n");
5937 // FIXME: Handle pointers to the inside of objects
5938 vtable = (MonoVTable*)LOAD_VTABLE (ptr);
5940 printf ("VTable: %p\n", vtable);
5941 if (vtable == NULL) {
5942 printf ("VTable is invalid (empty).\n");
5945 if (ptr_in_nursery (vtable)) {
5946 printf ("VTable is invalid (points inside nursery).\n");
5949 printf ("Class: %s\n", vtable->klass->name);
5951 desc = ((GCVTable*)vtable)->desc;
5952 printf ("Descriptor: %lx\n", (long)desc);
5955 printf ("Descriptor type: %d (%s)\n", type, descriptor_types [type]);
5959 find_in_remset_loc (mword *p, char *addr, gboolean *found)
5965 switch ((*p) & REMSET_TYPE_MASK) {
5966 case REMSET_LOCATION:
5967 if (*p == (mword)addr)
5971 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
5973 if ((void**)addr >= ptr && (void**)addr < ptr + count)
5977 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
5978 count = safe_object_get_size ((MonoObject*)ptr);
5979 count = ALIGN_UP (count);
5980 count /= sizeof (mword);
5981 if ((void**)addr >= ptr && (void**)addr < ptr + count)
5985 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
5989 switch (desc & 0x7) {
5990 case DESC_TYPE_RUN_LENGTH:
5991 OBJ_RUN_LEN_SIZE (skip_size, desc, ptr);
5993 case DESC_TYPE_SMALL_BITMAP:
5994 OBJ_BITMAP_SIZE (skip_size, desc, start);
5998 g_assert_not_reached ();
6001 /* The descriptor includes the size of MonoObject */
6002 skip_size -= sizeof (MonoObject);
6004 if ((void**)addr >= ptr && (void**)addr < ptr + (skip_size / sizeof (gpointer)))
6009 g_assert_not_reached ();
6015 * Return whenever ADDR occurs in the remembered sets
6018 find_in_remsets (char *addr)
6021 SgenThreadInfo *info;
6022 RememberedSet *remset;
6023 GenericStoreRememberedSet *store_remset;
6025 gboolean found = FALSE;
6027 /* the global one */
6028 for (remset = global_remset; remset; remset = remset->next) {
6029 DEBUG (4, fprintf (gc_debug_file, "Scanning global remset range: %p-%p, size: %td\n", remset->data, remset->store_next, remset->store_next - remset->data));
6030 for (p = remset->data; p < remset->store_next;) {
6031 p = find_in_remset_loc (p, addr, &found);
6037 /* the generic store ones */
6038 for (store_remset = generic_store_remsets; store_remset; store_remset = store_remset->next) {
6039 for (i = 0; i < STORE_REMSET_BUFFER_SIZE - 1; ++i) {
6040 if (store_remset->data [i] == addr)
6045 /* the per-thread ones */
6046 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
6047 for (info = thread_table [i]; info; info = info->next) {
6049 for (remset = info->remset; remset; remset = remset->next) {
6050 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for thread %p, range: %p-%p, size: %td\n", info, remset->data, remset->store_next, remset->store_next - remset->data));
6051 for (p = remset->data; p < remset->store_next;) {
6052 p = find_in_remset_loc (p, addr, &found);
6057 for (j = 0; j < *info->store_remset_buffer_index_addr; ++j) {
6058 if ((*info->store_remset_buffer_addr) [j + 1] == addr)
6064 /* the freed thread ones */
6065 for (remset = freed_thread_remsets; remset; remset = remset->next) {
6066 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for freed thread, range: %p-%p, size: %td\n", remset->data, remset->store_next, remset->store_next - remset->data));
6067 for (p = remset->data; p < remset->store_next;) {
6068 p = find_in_remset_loc (p, addr, &found);
6077 static gboolean missing_remsets;
6080 * We let a missing remset slide if the target object is pinned,
6081 * because the store might have happened but the remset not yet added,
6082 * but in that case the target must be pinned. We might theoretically
6083 * miss some missing remsets this way, but it's very unlikely.
6086 #define HANDLE_PTR(ptr,obj) do { \
6087 if (*(ptr) && (char*)*(ptr) >= nursery_start && (char*)*(ptr) < nursery_next) { \
6088 if (!find_in_remsets ((char*)(ptr)) && (!use_cardtable || !sgen_card_table_address_is_marked ((mword)ptr))) { \
6089 fprintf (gc_debug_file, "Oldspace->newspace reference %p at offset %td in object %p (%s.%s) not found in remsets.\n", *(ptr), (char*)(ptr) - (char*)(obj), (obj), ((MonoObject*)(obj))->vtable->klass->name_space, ((MonoObject*)(obj))->vtable->klass->name); \
6090 binary_protocol_missing_remset ((obj), (gpointer)LOAD_VTABLE ((obj)), (char*)(ptr) - (char*)(obj), *(ptr), (gpointer)LOAD_VTABLE(*(ptr)), object_is_pinned (*(ptr))); \
6091 if (!object_is_pinned (*(ptr))) \
6092 missing_remsets = TRUE; \
6098 * Check that each object reference which points into the nursery can
6099 * be found in the remembered sets.
6102 check_consistency_callback (char *start, size_t size, void *dummy)
6104 GCVTable *vt = (GCVTable*)LOAD_VTABLE (start);
6105 DEBUG (8, fprintf (gc_debug_file, "Scanning object %p, vtable: %p (%s)\n", start, vt, vt->klass->name));
6107 #define SCAN_OBJECT_ACTION
6108 #include "sgen-scan-object.h"
6112 * Perform consistency check of the heap.
6114 * Assumes the world is stopped.
6117 check_consistency (void)
6121 // Need to add more checks
6123 missing_remsets = FALSE;
6125 DEBUG (1, fprintf (gc_debug_file, "Begin heap consistency check...\n"));
6127 // Check that oldspace->newspace pointers are registered with the collector
6128 major_collector.iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)check_consistency_callback, NULL);
6130 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
6131 check_consistency_callback (bigobj->data, bigobj->size, NULL);
6133 DEBUG (1, fprintf (gc_debug_file, "Heap consistency check done.\n"));
6135 #ifdef SGEN_BINARY_PROTOCOL
6136 if (!binary_protocol_file)
6138 g_assert (!missing_remsets);
6143 #define HANDLE_PTR(ptr,obj) do { \
6144 if (*(ptr) && !LOAD_VTABLE (*(ptr))) \
6145 g_error ("Could not load vtable for obj %p slot %d (size %d)", obj, (char*)ptr - (char*)obj, safe_object_get_size ((MonoObject*)obj)); \
6149 check_major_refs_callback (char *start, size_t size, void *dummy)
6151 #define SCAN_OBJECT_ACTION
6152 #include "sgen-scan-object.h"
6156 check_major_refs (void)
6160 major_collector.iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)check_major_refs_callback, NULL);
6162 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
6163 check_major_refs_callback (bigobj->data, bigobj->size, NULL);
6166 /* Check that the reference is valid */
6168 #define HANDLE_PTR(ptr,obj) do { \
6170 g_assert (safe_name (*(ptr)) != NULL); \
6177 * Perform consistency check on an object. Currently we only check that the
6178 * reference fields are valid.
6181 check_object (char *start)
6186 #include "sgen-scan-object.h"
6190 * ######################################################################
6191 * ######## Other mono public interface functions.
6192 * ######################################################################
6195 #define REFS_SIZE 128
6198 MonoGCReferences callback;
6202 MonoObject *refs [REFS_SIZE];
6206 #define HANDLE_PTR(ptr,obj) do { \
6208 if (hwi->count == REFS_SIZE) { \
6209 hwi->callback ((MonoObject*)start, mono_object_class (start), hwi->called? 0: size, hwi->count, hwi->refs, hwi->data); \
6213 hwi->refs [hwi->count++] = *(ptr); \
6218 collect_references (HeapWalkInfo *hwi, char *start, size_t size)
6220 #include "sgen-scan-object.h"
6224 walk_references (char *start, size_t size, void *data)
6226 HeapWalkInfo *hwi = data;
6229 collect_references (hwi, start, size);
6230 if (hwi->count || !hwi->called)
6231 hwi->callback ((MonoObject*)start, mono_object_class (start), hwi->called? 0: size, hwi->count, hwi->refs, hwi->data);
6235 * mono_gc_walk_heap:
6236 * @flags: flags for future use
6237 * @callback: a function pointer called for each object in the heap
6238 * @data: a user data pointer that is passed to callback
6240 * This function can be used to iterate over all the live objects in the heap:
6241 * for each object, @callback is invoked, providing info about the object's
6242 * location in memory, its class, its size and the objects it references.
6243 * The object references may be buffered, so the callback may be invoked
6244 * multiple times for the same object: in all but the first call, the size
6245 * argument will be zero.
6246 * Note that this function can be only called in the #MONO_GC_EVENT_PRE_START_WORLD
6247 * profiler event handler.
6249 * Returns: a non-zero value if the GC doesn't support heap walking
6252 mono_gc_walk_heap (int flags, MonoGCReferences callback, void *data)
6258 hwi.callback = callback;
6261 clear_nursery_fragments (nursery_next);
6262 mono_sgen_scan_area_with_callback (nursery_section->data, nursery_section->end_data, walk_references, &hwi);
6264 major_collector.iterate_objects (TRUE, TRUE, walk_references, &hwi);
6266 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
6267 walk_references (bigobj->data, bigobj->size, &hwi);
6272 mono_gc_collect (int generation)
6277 mono_profiler_gc_event (MONO_GC_EVENT_START, generation);
6278 stop_world (generation);
6279 if (generation == 0) {
6280 collect_nursery (0);
6282 major_collection ("user request");
6284 restart_world (generation);
6285 mono_profiler_gc_event (MONO_GC_EVENT_END, generation);
6290 mono_gc_max_generation (void)
6296 mono_gc_collection_count (int generation)
6298 if (generation == 0)
6299 return num_minor_gcs;
6300 return num_major_gcs;
6304 mono_gc_get_used_size (void)
6308 tot = los_memory_usage;
6309 tot += nursery_section->next_data - nursery_section->data;
6310 tot += major_collector.get_used_size ();
6311 /* FIXME: account for pinned objects */
6317 mono_gc_get_heap_size (void)
6323 mono_gc_disable (void)
6331 mono_gc_enable (void)
6339 mono_gc_get_los_limit (void)
6341 return MAX_SMALL_OBJ_SIZE;
6345 mono_object_is_alive (MonoObject* o)
6351 mono_gc_get_generation (MonoObject *obj)
6353 if (ptr_in_nursery (obj))
6359 mono_gc_enable_events (void)
6364 mono_gc_weak_link_add (void **link_addr, MonoObject *obj, gboolean track)
6367 mono_gc_register_disappearing_link (obj, link_addr, track);
6372 mono_gc_weak_link_remove (void **link_addr)
6375 mono_gc_register_disappearing_link (NULL, link_addr, FALSE);
6380 mono_gc_weak_link_get (void **link_addr)
6384 return (MonoObject*) REVEAL_POINTER (*link_addr);
6388 mono_gc_ephemeron_array_add (MonoObject *obj)
6390 EphemeronLinkNode *node;
6394 node = mono_sgen_alloc_internal (INTERNAL_MEM_EPHEMERON_LINK);
6399 node->array = (char*)obj;
6400 node->next = ephemeron_list;
6401 ephemeron_list = node;
6403 DEBUG (5, fprintf (gc_debug_file, "Registered ephemeron array %p\n", obj));
6410 mono_gc_make_descr_from_bitmap (gsize *bitmap, int numbits)
6413 return (void*)MAKE_ROOT_DESC (ROOT_DESC_BITMAP, 0);
6414 } else if (numbits < ((sizeof (*bitmap) * 8) - ROOT_DESC_TYPE_SHIFT)) {
6415 return (void*)MAKE_ROOT_DESC (ROOT_DESC_BITMAP, bitmap [0]);
6417 mword complex = alloc_complex_descriptor (bitmap, numbits);
6418 return (void*)MAKE_ROOT_DESC (ROOT_DESC_COMPLEX, complex);
6422 static void *all_ref_root_descrs [32];
6425 mono_gc_make_root_descr_all_refs (int numbits)
6430 if (numbits < 32 && all_ref_root_descrs [numbits])
6431 return all_ref_root_descrs [numbits];
6433 gc_bitmap = g_malloc0 (ALIGN_TO (numbits, 8) + 1);
6434 memset (gc_bitmap, 0xff, numbits / 8);
6436 gc_bitmap [numbits / 8] = (1 << (numbits % 8)) - 1;
6437 descr = mono_gc_make_descr_from_bitmap (gc_bitmap, numbits);
6441 all_ref_root_descrs [numbits] = descr;
6447 mono_gc_make_root_descr_user (MonoGCRootMarkFunc marker)
6451 g_assert (user_descriptors_next < MAX_USER_DESCRIPTORS);
6452 descr = (void*)MAKE_ROOT_DESC (ROOT_DESC_USER, (mword)user_descriptors_next);
6453 user_descriptors [user_descriptors_next ++] = marker;
6459 mono_gc_alloc_fixed (size_t size, void *descr)
6461 /* FIXME: do a single allocation */
6462 void *res = calloc (1, size);
6465 if (!mono_gc_register_root (res, size, descr)) {
6473 mono_gc_free_fixed (void* addr)
6475 mono_gc_deregister_root (addr);
6480 mono_gc_invoke_with_gc_lock (MonoGCLockedCallbackFunc func, void *data)
6484 result = func (data);
6485 UNLOCK_INTERRUPTION;
6490 mono_gc_is_gc_thread (void)
6494 result = mono_sgen_thread_info_lookup (ARCH_GET_THREAD ()) != NULL;
6499 /* Tries to extract a number from the passed string, taking in to account m, k
6502 mono_sgen_parse_environment_string_extract_number (const char *str, glong *out)
6505 int len = strlen (str), shift = 0;
6507 gboolean is_suffix = FALSE;
6510 switch (str [len - 1]) {
6521 suffix = str [len - 1];
6526 val = strtol (str, &endptr, 10);
6528 if ((errno == ERANGE && (val == LONG_MAX || val == LONG_MIN))
6529 || (errno != 0 && val == 0) || (endptr == str))
6533 if (*(endptr + 1)) /* Invalid string. */
6543 mono_gc_base_init (void)
6547 char *major_collector_opt = NULL;
6548 struct sigaction sinfo;
6550 #ifdef PLATFORM_ANDROID
6551 g_assert_not_reached ();
6554 /* the gc_initialized guard seems to imply this method is
6555 idempotent, but LOCK_INIT(gc_mutex) might not be. It's
6556 defined in sgen-gc.h as nothing, so there's no danger at
6558 LOCK_INIT (gc_mutex);
6560 if (gc_initialized) {
6564 pagesize = mono_pagesize ();
6565 gc_debug_file = stderr;
6567 LOCK_INIT (interruption_mutex);
6568 LOCK_INIT (global_remset_mutex);
6570 if ((env = getenv ("MONO_GC_PARAMS"))) {
6571 opts = g_strsplit (env, ",", -1);
6572 for (ptr = opts; *ptr; ++ptr) {
6574 if (g_str_has_prefix (opt, "major=")) {
6575 opt = strchr (opt, '=') + 1;
6576 major_collector_opt = g_strdup (opt);
6584 mono_sgen_init_internal_allocator ();
6586 mono_sgen_register_fixed_internal_mem_type (INTERNAL_MEM_FRAGMENT, sizeof (Fragment));
6587 mono_sgen_register_fixed_internal_mem_type (INTERNAL_MEM_SECTION, SGEN_SIZEOF_GC_MEM_SECTION);
6588 mono_sgen_register_fixed_internal_mem_type (INTERNAL_MEM_FINALIZE_ENTRY, sizeof (FinalizeEntry));
6589 mono_sgen_register_fixed_internal_mem_type (INTERNAL_MEM_DISLINK, sizeof (DisappearingLink));
6590 mono_sgen_register_fixed_internal_mem_type (INTERNAL_MEM_ROOT_RECORD, sizeof (RootRecord));
6591 mono_sgen_register_fixed_internal_mem_type (INTERNAL_MEM_GRAY_QUEUE, sizeof (GrayQueueSection));
6592 g_assert (sizeof (GenericStoreRememberedSet) == sizeof (gpointer) * STORE_REMSET_BUFFER_SIZE);
6593 mono_sgen_register_fixed_internal_mem_type (INTERNAL_MEM_STORE_REMSET, sizeof (GenericStoreRememberedSet));
6594 mono_sgen_register_fixed_internal_mem_type (INTERNAL_MEM_EPHEMERON_LINK, sizeof (EphemeronLinkNode));
6596 if (!major_collector_opt || !strcmp (major_collector_opt, "marksweep")) {
6597 mono_sgen_marksweep_init (&major_collector);
6598 } else if (!major_collector_opt || !strcmp (major_collector_opt, "marksweep-fixed")) {
6599 mono_sgen_marksweep_fixed_init (&major_collector);
6600 } else if (!major_collector_opt || !strcmp (major_collector_opt, "marksweep-par")) {
6601 mono_sgen_marksweep_par_init (&major_collector);
6602 workers_init (mono_cpu_count ());
6603 } else if (!major_collector_opt || !strcmp (major_collector_opt, "marksweep-fixed-par")) {
6604 mono_sgen_marksweep_fixed_par_init (&major_collector);
6605 workers_init (mono_cpu_count ());
6606 } else if (!strcmp (major_collector_opt, "copying")) {
6607 mono_sgen_copying_init (&major_collector);
6609 fprintf (stderr, "Unknown major collector `%s'.\n", major_collector_opt);
6613 #ifdef SGEN_HAVE_CARDTABLE
6614 use_cardtable = major_collector.supports_cardtable;
6616 use_cardtable = FALSE;
6620 for (ptr = opts; *ptr; ++ptr) {
6622 if (g_str_has_prefix (opt, "major="))
6624 if (g_str_has_prefix (opt, "wbarrier=")) {
6625 opt = strchr (opt, '=') + 1;
6626 if (strcmp (opt, "remset") == 0) {
6627 use_cardtable = FALSE;
6628 } else if (strcmp (opt, "cardtable") == 0) {
6629 if (!use_cardtable) {
6630 if (major_collector.supports_cardtable)
6631 fprintf (stderr, "The cardtable write barrier is not supported on this platform.\n");
6633 fprintf (stderr, "The major collector does not support the cardtable write barrier.\n");
6640 if (g_str_has_prefix (opt, "nursery-size=")) {
6642 opt = strchr (opt, '=') + 1;
6643 if (*opt && mono_sgen_parse_environment_string_extract_number (opt, &val)) {
6644 default_nursery_size = val;
6645 #ifdef SGEN_ALIGN_NURSERY
6646 if ((val & (val - 1))) {
6647 fprintf (stderr, "The nursery size must be a power of two.\n");
6651 default_nursery_bits = 0;
6652 while (1 << (++ default_nursery_bits) != default_nursery_size)
6656 fprintf (stderr, "nursery-size must be an integer.\n");
6662 if (!(major_collector.handle_gc_param && major_collector.handle_gc_param (opt))) {
6663 fprintf (stderr, "MONO_GC_PARAMS must be a comma-delimited list of one or more of the following:\n");
6664 fprintf (stderr, " nursery-size=N (where N is an integer, possibly with a k, m or a g suffix)\n");
6665 fprintf (stderr, " major=COLLECTOR (where COLLECTOR is `marksweep', `marksweep-par' or `copying')\n");
6666 fprintf (stderr, " wbarrier=WBARRIER (where WBARRIER is `remset' or `cardtable')\n");
6667 if (major_collector.print_gc_param_usage)
6668 major_collector.print_gc_param_usage ();
6675 if (major_collector_opt)
6676 g_free (major_collector_opt);
6678 nursery_size = DEFAULT_NURSERY_SIZE;
6679 minor_collection_allowance = MIN_MINOR_COLLECTION_ALLOWANCE;
6683 if ((env = getenv ("MONO_GC_DEBUG"))) {
6684 opts = g_strsplit (env, ",", -1);
6685 for (ptr = opts; ptr && *ptr; ptr ++) {
6687 if (opt [0] >= '0' && opt [0] <= '9') {
6688 gc_debug_level = atoi (opt);
6693 char *rf = g_strdup_printf ("%s.%d", opt, getpid ());
6694 gc_debug_file = fopen (rf, "wb");
6696 gc_debug_file = stderr;
6699 } else if (!strcmp (opt, "collect-before-allocs")) {
6700 collect_before_allocs = TRUE;
6701 } else if (!strcmp (opt, "check-at-minor-collections")) {
6702 consistency_check_at_minor_collection = TRUE;
6703 nursery_clear_policy = CLEAR_AT_GC;
6704 } else if (!strcmp (opt, "xdomain-checks")) {
6705 xdomain_checks = TRUE;
6706 } else if (!strcmp (opt, "clear-at-gc")) {
6707 nursery_clear_policy = CLEAR_AT_GC;
6708 } else if (!strcmp (opt, "conservative-stack-mark")) {
6709 conservative_stack_mark = TRUE;
6710 } else if (!strcmp (opt, "check-scan-starts")) {
6711 do_scan_starts_check = TRUE;
6712 } else if (g_str_has_prefix (opt, "heap-dump=")) {
6713 char *filename = strchr (opt, '=') + 1;
6714 nursery_clear_policy = CLEAR_AT_GC;
6715 heap_dump_file = fopen (filename, "w");
6717 fprintf (heap_dump_file, "<sgen-dump>\n");
6718 #ifdef SGEN_BINARY_PROTOCOL
6719 } else if (g_str_has_prefix (opt, "binary-protocol=")) {
6720 char *filename = strchr (opt, '=') + 1;
6721 binary_protocol_file = fopen (filename, "w");
6724 fprintf (stderr, "Invalid format for the MONO_GC_DEBUG env variable: '%s'\n", env);
6725 fprintf (stderr, "The format is: MONO_GC_DEBUG=[l[:filename]|<option>]+ where l is a debug level 0-9.\n");
6726 fprintf (stderr, "Valid options are: collect-before-allocs, check-at-minor-collections, xdomain-checks, clear-at-gc.\n");
6733 suspend_ack_semaphore_ptr = &suspend_ack_semaphore;
6734 MONO_SEM_INIT (&suspend_ack_semaphore, 0);
6736 sigfillset (&sinfo.sa_mask);
6737 sinfo.sa_flags = SA_RESTART | SA_SIGINFO;
6738 sinfo.sa_sigaction = suspend_handler;
6739 if (sigaction (suspend_signal_num, &sinfo, NULL) != 0) {
6740 g_error ("failed sigaction");
6743 sinfo.sa_handler = restart_handler;
6744 if (sigaction (restart_signal_num, &sinfo, NULL) != 0) {
6745 g_error ("failed sigaction");
6748 sigfillset (&suspend_signal_mask);
6749 sigdelset (&suspend_signal_mask, restart_signal_num);
6751 global_remset = alloc_remset (1024, NULL);
6752 global_remset->next = NULL;
6754 pthread_key_create (&remembered_set_key, unregister_thread);
6756 #ifndef HAVE_KW_THREAD
6757 pthread_key_create (&thread_info_key, NULL);
6763 gc_initialized = TRUE;
6765 mono_gc_register_thread (&sinfo);
6769 mono_gc_get_suspend_signal (void)
6771 return suspend_signal_num;
6781 #ifdef HAVE_KW_THREAD
6782 #define EMIT_TLS_ACCESS(mb,dummy,offset) do { \
6783 mono_mb_emit_byte ((mb), MONO_CUSTOM_PREFIX); \
6784 mono_mb_emit_byte ((mb), CEE_MONO_TLS); \
6785 mono_mb_emit_i4 ((mb), (offset)); \
6788 #define EMIT_TLS_ACCESS(mb,member,dummy) do { \
6789 mono_mb_emit_byte ((mb), MONO_CUSTOM_PREFIX); \
6790 mono_mb_emit_byte ((mb), CEE_MONO_TLS); \
6791 mono_mb_emit_i4 ((mb), thread_info_key); \
6792 mono_mb_emit_icon ((mb), G_STRUCT_OFFSET (SgenThreadInfo, member)); \
6793 mono_mb_emit_byte ((mb), CEE_ADD); \
6794 mono_mb_emit_byte ((mb), CEE_LDIND_I); \
6798 #ifdef MANAGED_ALLOCATION
6799 /* FIXME: Do this in the JIT, where specialized allocation sequences can be created
6800 * for each class. This is currently not easy to do, as it is hard to generate basic
6801 * blocks + branches, but it is easy with the linear IL codebase.
6803 * For this to work we'd need to solve the TLAB race, first. Now we
6804 * require the allocator to be in a few known methods to make sure
6805 * that they are executed atomically via the restart mechanism.
6808 create_allocator (int atype)
6810 int p_var, size_var;
6811 guint32 slowpath_branch, max_size_branch;
6812 MonoMethodBuilder *mb;
6814 MonoMethodSignature *csig;
6815 static gboolean registered = FALSE;
6816 int tlab_next_addr_var, new_next_var;
6818 const char *name = NULL;
6819 AllocatorWrapperInfo *info;
6821 #ifdef HAVE_KW_THREAD
6822 int tlab_next_addr_offset = -1;
6823 int tlab_temp_end_offset = -1;
6825 MONO_THREAD_VAR_OFFSET (tlab_next_addr, tlab_next_addr_offset);
6826 MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
6828 g_assert (tlab_next_addr_offset != -1);
6829 g_assert (tlab_temp_end_offset != -1);
6833 mono_register_jit_icall (mono_gc_alloc_obj, "mono_gc_alloc_obj", mono_create_icall_signature ("object ptr int"), FALSE);
6834 mono_register_jit_icall (mono_gc_alloc_vector, "mono_gc_alloc_vector", mono_create_icall_signature ("object ptr int int"), FALSE);
6838 if (atype == ATYPE_SMALL) {
6840 name = "AllocSmall";
6841 } else if (atype == ATYPE_NORMAL) {
6844 } else if (atype == ATYPE_VECTOR) {
6846 name = "AllocVector";
6848 g_assert_not_reached ();
6851 csig = mono_metadata_signature_alloc (mono_defaults.corlib, num_params);
6852 csig->ret = &mono_defaults.object_class->byval_arg;
6853 for (i = 0; i < num_params; ++i)
6854 csig->params [i] = &mono_defaults.int_class->byval_arg;
6856 mb = mono_mb_new (mono_defaults.object_class, name, MONO_WRAPPER_ALLOC);
6857 size_var = mono_mb_add_local (mb, &mono_defaults.int32_class->byval_arg);
6858 if (atype == ATYPE_NORMAL || atype == ATYPE_SMALL) {
6859 /* size = vtable->klass->instance_size; */
6860 mono_mb_emit_ldarg (mb, 0);
6861 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoVTable, klass));
6862 mono_mb_emit_byte (mb, CEE_ADD);
6863 mono_mb_emit_byte (mb, CEE_LDIND_I);
6864 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoClass, instance_size));
6865 mono_mb_emit_byte (mb, CEE_ADD);
6866 /* FIXME: assert instance_size stays a 4 byte integer */
6867 mono_mb_emit_byte (mb, CEE_LDIND_U4);
6868 mono_mb_emit_stloc (mb, size_var);
6869 } else if (atype == ATYPE_VECTOR) {
6870 MonoExceptionClause *clause;
6872 MonoClass *oom_exc_class;
6875 /* n > MONO_ARRAY_MAX_INDEX -> OverflowException */
6876 mono_mb_emit_ldarg (mb, 1);
6877 mono_mb_emit_icon (mb, MONO_ARRAY_MAX_INDEX);
6878 pos = mono_mb_emit_short_branch (mb, CEE_BLE_UN_S);
6879 mono_mb_emit_exception (mb, "OverflowException", NULL);
6880 mono_mb_patch_short_branch (mb, pos);
6882 clause = mono_image_alloc0 (mono_defaults.corlib, sizeof (MonoExceptionClause));
6883 clause->try_offset = mono_mb_get_label (mb);
6885 /* vtable->klass->sizes.element_size */
6886 mono_mb_emit_ldarg (mb, 0);
6887 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoVTable, klass));
6888 mono_mb_emit_byte (mb, CEE_ADD);
6889 mono_mb_emit_byte (mb, CEE_LDIND_I);
6890 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoClass, sizes.element_size));
6891 mono_mb_emit_byte (mb, CEE_ADD);
6892 mono_mb_emit_byte (mb, CEE_LDIND_U4);
6895 mono_mb_emit_ldarg (mb, 1);
6896 mono_mb_emit_byte (mb, CEE_MUL_OVF_UN);
6897 /* + sizeof (MonoArray) */
6898 mono_mb_emit_icon (mb, sizeof (MonoArray));
6899 mono_mb_emit_byte (mb, CEE_ADD_OVF_UN);
6900 mono_mb_emit_stloc (mb, size_var);
6902 pos_leave = mono_mb_emit_branch (mb, CEE_LEAVE);
6905 clause->flags = MONO_EXCEPTION_CLAUSE_NONE;
6906 clause->try_len = mono_mb_get_pos (mb) - clause->try_offset;
6907 clause->data.catch_class = mono_class_from_name (mono_defaults.corlib,
6908 "System", "OverflowException");
6909 g_assert (clause->data.catch_class);
6910 clause->handler_offset = mono_mb_get_label (mb);
6912 oom_exc_class = mono_class_from_name (mono_defaults.corlib,
6913 "System", "OutOfMemoryException");
6914 g_assert (oom_exc_class);
6915 ctor = mono_class_get_method_from_name (oom_exc_class, ".ctor", 0);
6918 mono_mb_emit_byte (mb, CEE_POP);
6919 mono_mb_emit_op (mb, CEE_NEWOBJ, ctor);
6920 mono_mb_emit_byte (mb, CEE_THROW);
6922 clause->handler_len = mono_mb_get_pos (mb) - clause->handler_offset;
6923 mono_mb_set_clauses (mb, 1, clause);
6924 mono_mb_patch_branch (mb, pos_leave);
6927 g_assert_not_reached ();
6930 /* size += ALLOC_ALIGN - 1; */
6931 mono_mb_emit_ldloc (mb, size_var);
6932 mono_mb_emit_icon (mb, ALLOC_ALIGN - 1);
6933 mono_mb_emit_byte (mb, CEE_ADD);
6934 /* size &= ~(ALLOC_ALIGN - 1); */
6935 mono_mb_emit_icon (mb, ~(ALLOC_ALIGN - 1));
6936 mono_mb_emit_byte (mb, CEE_AND);
6937 mono_mb_emit_stloc (mb, size_var);
6939 /* if (size > MAX_SMALL_OBJ_SIZE) goto slowpath */
6940 if (atype != ATYPE_SMALL) {
6941 mono_mb_emit_ldloc (mb, size_var);
6942 mono_mb_emit_icon (mb, MAX_SMALL_OBJ_SIZE);
6943 max_size_branch = mono_mb_emit_short_branch (mb, MONO_CEE_BGT_S);
6947 * We need to modify tlab_next, but the JIT only supports reading, so we read
6948 * another tls var holding its address instead.
6951 /* tlab_next_addr (local) = tlab_next_addr (TLS var) */
6952 tlab_next_addr_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
6953 EMIT_TLS_ACCESS (mb, tlab_next_addr, tlab_next_addr_offset);
6954 mono_mb_emit_stloc (mb, tlab_next_addr_var);
6956 /* p = (void**)tlab_next; */
6957 p_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
6958 mono_mb_emit_ldloc (mb, tlab_next_addr_var);
6959 mono_mb_emit_byte (mb, CEE_LDIND_I);
6960 mono_mb_emit_stloc (mb, p_var);
6962 /* new_next = (char*)p + size; */
6963 new_next_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
6964 mono_mb_emit_ldloc (mb, p_var);
6965 mono_mb_emit_ldloc (mb, size_var);
6966 mono_mb_emit_byte (mb, CEE_CONV_I);
6967 mono_mb_emit_byte (mb, CEE_ADD);
6968 mono_mb_emit_stloc (mb, new_next_var);
6970 /* tlab_next = new_next */
6971 mono_mb_emit_ldloc (mb, tlab_next_addr_var);
6972 mono_mb_emit_ldloc (mb, new_next_var);
6973 mono_mb_emit_byte (mb, CEE_STIND_I);
6975 /* if (G_LIKELY (new_next < tlab_temp_end)) */
6976 mono_mb_emit_ldloc (mb, new_next_var);
6977 EMIT_TLS_ACCESS (mb, tlab_temp_end, tlab_temp_end_offset);
6978 slowpath_branch = mono_mb_emit_short_branch (mb, MONO_CEE_BLT_UN_S);
6981 if (atype != ATYPE_SMALL)
6982 mono_mb_patch_short_branch (mb, max_size_branch);
6984 mono_mb_emit_byte (mb, MONO_CUSTOM_PREFIX);
6985 mono_mb_emit_byte (mb, CEE_MONO_NOT_TAKEN);
6987 /* FIXME: mono_gc_alloc_obj takes a 'size_t' as an argument, not an int32 */
6988 mono_mb_emit_ldarg (mb, 0);
6989 mono_mb_emit_ldloc (mb, size_var);
6990 if (atype == ATYPE_NORMAL || atype == ATYPE_SMALL) {
6991 mono_mb_emit_icall (mb, mono_gc_alloc_obj);
6992 } else if (atype == ATYPE_VECTOR) {
6993 mono_mb_emit_ldarg (mb, 1);
6994 mono_mb_emit_icall (mb, mono_gc_alloc_vector);
6996 g_assert_not_reached ();
6998 mono_mb_emit_byte (mb, CEE_RET);
7001 mono_mb_patch_short_branch (mb, slowpath_branch);
7003 /* FIXME: Memory barrier */
7006 mono_mb_emit_ldloc (mb, p_var);
7007 mono_mb_emit_ldarg (mb, 0);
7008 mono_mb_emit_byte (mb, CEE_STIND_I);
7010 if (atype == ATYPE_VECTOR) {
7011 /* arr->max_length = max_length; */
7012 mono_mb_emit_ldloc (mb, p_var);
7013 mono_mb_emit_ldflda (mb, G_STRUCT_OFFSET (MonoArray, max_length));
7014 mono_mb_emit_ldarg (mb, 1);
7015 mono_mb_emit_byte (mb, CEE_STIND_I);
7019 mono_mb_emit_ldloc (mb, p_var);
7020 mono_mb_emit_byte (mb, CEE_RET);
7022 res = mono_mb_create_method (mb, csig, 8);
7024 mono_method_get_header (res)->init_locals = FALSE;
7026 info = mono_image_alloc0 (mono_defaults.corlib, sizeof (AllocatorWrapperInfo));
7027 info->gc_name = "sgen";
7028 info->alloc_type = atype;
7029 mono_marshal_set_wrapper_info (res, info);
7036 mono_gc_get_gc_name (void)
7041 static MonoMethod* alloc_method_cache [ATYPE_NUM];
7042 static MonoMethod *write_barrier_method;
7045 is_ip_in_managed_allocator (MonoDomain *domain, gpointer ip)
7051 if (!mono_thread_internal_current ())
7052 /* Happens during thread attach */
7057 ji = mono_jit_info_table_find (domain, ip);
7060 method = ji->method;
7062 if (method == write_barrier_method)
7064 for (i = 0; i < ATYPE_NUM; ++i)
7065 if (method == alloc_method_cache [i])
7071 * Generate an allocator method implementing the fast path of mono_gc_alloc_obj ().
7072 * The signature of the called method is:
7073 * object allocate (MonoVTable *vtable)
7076 mono_gc_get_managed_allocator (MonoVTable *vtable, gboolean for_box)
7078 #ifdef MANAGED_ALLOCATION
7079 MonoClass *klass = vtable->klass;
7081 #ifdef HAVE_KW_THREAD
7082 int tlab_next_offset = -1;
7083 int tlab_temp_end_offset = -1;
7084 MONO_THREAD_VAR_OFFSET (tlab_next, tlab_next_offset);
7085 MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
7087 if (tlab_next_offset == -1 || tlab_temp_end_offset == -1)
7091 if (!mono_runtime_has_tls_get ())
7093 if (klass->instance_size > tlab_size)
7095 if (klass->has_finalize || klass->marshalbyref || (mono_profiler_get_events () & MONO_PROFILE_ALLOCATIONS))
7099 if (klass->byval_arg.type == MONO_TYPE_STRING)
7101 if (collect_before_allocs)
7104 if (ALIGN_TO (klass->instance_size, ALLOC_ALIGN) < MAX_SMALL_OBJ_SIZE)
7105 return mono_gc_get_managed_allocator_by_type (ATYPE_SMALL);
7107 return mono_gc_get_managed_allocator_by_type (ATYPE_NORMAL);
7114 mono_gc_get_managed_array_allocator (MonoVTable *vtable, int rank)
7116 #ifdef MANAGED_ALLOCATION
7117 MonoClass *klass = vtable->klass;
7119 #ifdef HAVE_KW_THREAD
7120 int tlab_next_offset = -1;
7121 int tlab_temp_end_offset = -1;
7122 MONO_THREAD_VAR_OFFSET (tlab_next, tlab_next_offset);
7123 MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
7125 if (tlab_next_offset == -1 || tlab_temp_end_offset == -1)
7131 if (!mono_runtime_has_tls_get ())
7133 if (mono_profiler_get_events () & MONO_PROFILE_ALLOCATIONS)
7135 if (collect_before_allocs)
7137 g_assert (!klass->has_finalize && !klass->marshalbyref);
7139 return mono_gc_get_managed_allocator_by_type (ATYPE_VECTOR);
7146 mono_gc_get_managed_allocator_by_type (int atype)
7148 #ifdef MANAGED_ALLOCATION
7151 if (!mono_runtime_has_tls_get ())
7154 mono_loader_lock ();
7155 res = alloc_method_cache [atype];
7157 res = alloc_method_cache [atype] = create_allocator (atype);
7158 mono_loader_unlock ();
7166 mono_gc_get_managed_allocator_types (void)
7173 mono_gc_get_write_barrier (void)
7176 MonoMethodBuilder *mb;
7177 MonoMethodSignature *sig;
7178 #ifdef MANAGED_WBARRIER
7179 int label_no_wb_1, label_no_wb_2, label_no_wb_3, label_no_wb_4, label_need_wb, label_slow_path;
7180 #ifndef SGEN_ALIGN_NURSERY
7181 int label_continue_1, label_continue_2, label_no_wb_5;
7182 int dereferenced_var;
7184 int buffer_var, buffer_index_var, dummy_var;
7186 #ifdef HAVE_KW_THREAD
7187 int stack_end_offset = -1, store_remset_buffer_offset = -1;
7188 int store_remset_buffer_index_offset = -1, store_remset_buffer_index_addr_offset = -1;
7190 MONO_THREAD_VAR_OFFSET (stack_end, stack_end_offset);
7191 g_assert (stack_end_offset != -1);
7192 MONO_THREAD_VAR_OFFSET (store_remset_buffer, store_remset_buffer_offset);
7193 g_assert (store_remset_buffer_offset != -1);
7194 MONO_THREAD_VAR_OFFSET (store_remset_buffer_index, store_remset_buffer_index_offset);
7195 g_assert (store_remset_buffer_index_offset != -1);
7196 MONO_THREAD_VAR_OFFSET (store_remset_buffer_index_addr, store_remset_buffer_index_addr_offset);
7197 g_assert (store_remset_buffer_index_addr_offset != -1);
7201 g_assert (!use_cardtable);
7203 // FIXME: Maybe create a separate version for ctors (the branch would be
7204 // correctly predicted more times)
7205 if (write_barrier_method)
7206 return write_barrier_method;
7208 /* Create the IL version of mono_gc_barrier_generic_store () */
7209 sig = mono_metadata_signature_alloc (mono_defaults.corlib, 1);
7210 sig->ret = &mono_defaults.void_class->byval_arg;
7211 sig->params [0] = &mono_defaults.int_class->byval_arg;
7213 mb = mono_mb_new (mono_defaults.object_class, "wbarrier", MONO_WRAPPER_WRITE_BARRIER);
7215 #ifdef MANAGED_WBARRIER
7216 if (mono_runtime_has_tls_get ()) {
7217 #ifdef SGEN_ALIGN_NURSERY
7218 // if (ptr_in_nursery (ptr)) return;
7220 * Masking out the bits might be faster, but we would have to use 64 bit
7221 * immediates, which might be slower.
7223 mono_mb_emit_ldarg (mb, 0);
7224 mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
7225 mono_mb_emit_byte (mb, CEE_SHR_UN);
7226 mono_mb_emit_icon (mb, (mword)nursery_start >> DEFAULT_NURSERY_BITS);
7227 label_no_wb_1 = mono_mb_emit_branch (mb, CEE_BEQ);
7229 // if (!ptr_in_nursery (*ptr)) return;
7230 mono_mb_emit_ldarg (mb, 0);
7231 mono_mb_emit_byte (mb, CEE_LDIND_I);
7232 mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
7233 mono_mb_emit_byte (mb, CEE_SHR_UN);
7234 mono_mb_emit_icon (mb, (mword)nursery_start >> DEFAULT_NURSERY_BITS);
7235 label_no_wb_2 = mono_mb_emit_branch (mb, CEE_BNE_UN);
7238 // if (ptr < (nursery_start)) goto continue;
7239 mono_mb_emit_ldarg (mb, 0);
7240 mono_mb_emit_ptr (mb, (gpointer) nursery_start);
7241 label_continue_1 = mono_mb_emit_branch (mb, CEE_BLT);
7243 // if (ptr >= nursery_real_end)) goto continue;
7244 mono_mb_emit_ldarg (mb, 0);
7245 mono_mb_emit_ptr (mb, (gpointer) nursery_real_end);
7246 label_continue_2 = mono_mb_emit_branch (mb, CEE_BGE);
7249 label_no_wb_1 = mono_mb_emit_branch (mb, CEE_BR);
7252 mono_mb_patch_branch (mb, label_continue_1);
7253 mono_mb_patch_branch (mb, label_continue_2);
7255 // Dereference and store in local var
7256 dereferenced_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7257 mono_mb_emit_ldarg (mb, 0);
7258 mono_mb_emit_byte (mb, CEE_LDIND_I);
7259 mono_mb_emit_stloc (mb, dereferenced_var);
7261 // if (*ptr < nursery_start) return;
7262 mono_mb_emit_ldloc (mb, dereferenced_var);
7263 mono_mb_emit_ptr (mb, (gpointer) nursery_start);
7264 label_no_wb_2 = mono_mb_emit_branch (mb, CEE_BLT);
7266 // if (*ptr >= nursery_end) return;
7267 mono_mb_emit_ldloc (mb, dereferenced_var);
7268 mono_mb_emit_ptr (mb, (gpointer) nursery_real_end);
7269 label_no_wb_5 = mono_mb_emit_branch (mb, CEE_BGE);
7272 // if (ptr >= stack_end) goto need_wb;
7273 mono_mb_emit_ldarg (mb, 0);
7274 EMIT_TLS_ACCESS (mb, stack_end, stack_end_offset);
7275 label_need_wb = mono_mb_emit_branch (mb, CEE_BGE_UN);
7277 // if (ptr >= stack_start) return;
7278 dummy_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7279 mono_mb_emit_ldarg (mb, 0);
7280 mono_mb_emit_ldloc_addr (mb, dummy_var);
7281 label_no_wb_3 = mono_mb_emit_branch (mb, CEE_BGE_UN);
7284 mono_mb_patch_branch (mb, label_need_wb);
7286 // buffer = STORE_REMSET_BUFFER;
7287 buffer_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7288 EMIT_TLS_ACCESS (mb, store_remset_buffer, store_remset_buffer_offset);
7289 mono_mb_emit_stloc (mb, buffer_var);
7291 // buffer_index = STORE_REMSET_BUFFER_INDEX;
7292 buffer_index_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7293 EMIT_TLS_ACCESS (mb, store_remset_buffer_index, store_remset_buffer_index_offset);
7294 mono_mb_emit_stloc (mb, buffer_index_var);
7296 // if (buffer [buffer_index] == ptr) return;
7297 mono_mb_emit_ldloc (mb, buffer_var);
7298 mono_mb_emit_ldloc (mb, buffer_index_var);
7299 g_assert (sizeof (gpointer) == 4 || sizeof (gpointer) == 8);
7300 mono_mb_emit_icon (mb, sizeof (gpointer) == 4 ? 2 : 3);
7301 mono_mb_emit_byte (mb, CEE_SHL);
7302 mono_mb_emit_byte (mb, CEE_ADD);
7303 mono_mb_emit_byte (mb, CEE_LDIND_I);
7304 mono_mb_emit_ldarg (mb, 0);
7305 label_no_wb_4 = mono_mb_emit_branch (mb, CEE_BEQ);
7308 mono_mb_emit_ldloc (mb, buffer_index_var);
7309 mono_mb_emit_icon (mb, 1);
7310 mono_mb_emit_byte (mb, CEE_ADD);
7311 mono_mb_emit_stloc (mb, buffer_index_var);
7313 // if (buffer_index >= STORE_REMSET_BUFFER_SIZE) goto slow_path;
7314 mono_mb_emit_ldloc (mb, buffer_index_var);
7315 mono_mb_emit_icon (mb, STORE_REMSET_BUFFER_SIZE);
7316 label_slow_path = mono_mb_emit_branch (mb, CEE_BGE);
7318 // buffer [buffer_index] = ptr;
7319 mono_mb_emit_ldloc (mb, buffer_var);
7320 mono_mb_emit_ldloc (mb, buffer_index_var);
7321 g_assert (sizeof (gpointer) == 4 || sizeof (gpointer) == 8);
7322 mono_mb_emit_icon (mb, sizeof (gpointer) == 4 ? 2 : 3);
7323 mono_mb_emit_byte (mb, CEE_SHL);
7324 mono_mb_emit_byte (mb, CEE_ADD);
7325 mono_mb_emit_ldarg (mb, 0);
7326 mono_mb_emit_byte (mb, CEE_STIND_I);
7328 // STORE_REMSET_BUFFER_INDEX = buffer_index;
7329 EMIT_TLS_ACCESS (mb, store_remset_buffer_index_addr, store_remset_buffer_index_addr_offset);
7330 mono_mb_emit_ldloc (mb, buffer_index_var);
7331 mono_mb_emit_byte (mb, CEE_STIND_I);
7334 mono_mb_patch_branch (mb, label_no_wb_1);
7335 mono_mb_patch_branch (mb, label_no_wb_2);
7336 mono_mb_patch_branch (mb, label_no_wb_3);
7337 mono_mb_patch_branch (mb, label_no_wb_4);
7338 #ifndef SGEN_ALIGN_NURSERY
7339 mono_mb_patch_branch (mb, label_no_wb_5);
7341 mono_mb_emit_byte (mb, CEE_RET);
7344 mono_mb_patch_branch (mb, label_slow_path);
7348 mono_mb_emit_ldarg (mb, 0);
7349 mono_mb_emit_icall (mb, mono_gc_wbarrier_generic_nostore);
7350 mono_mb_emit_byte (mb, CEE_RET);
7352 res = mono_mb_create_method (mb, sig, 16);
7355 mono_loader_lock ();
7356 if (write_barrier_method) {
7357 /* Already created */
7358 mono_free_method (res);
7360 /* double-checked locking */
7361 mono_memory_barrier ();
7362 write_barrier_method = res;
7364 mono_loader_unlock ();
7366 return write_barrier_method;
7370 mono_gc_get_description (void)
7372 return g_strdup ("sgen");
7376 mono_gc_set_desktop_mode (void)
7381 mono_gc_is_moving (void)
7387 mono_gc_is_disabled (void)
7393 mono_sgen_debug_printf (int level, const char *format, ...)
7397 if (level > gc_debug_level)
7400 va_start (ap, format);
7401 vfprintf (gc_debug_file, format, ap);
7405 #endif /* HAVE_SGEN_GC */