2 * sgen-gc.c: Simple generational GC.
5 * Paolo Molaro (lupus@ximian.com)
7 * Copyright 2005-2010 Novell, Inc (http://www.novell.com)
9 * Thread start/stop adapted from Boehm's GC:
10 * Copyright (c) 1994 by Xerox Corporation. All rights reserved.
11 * Copyright (c) 1996 by Silicon Graphics. All rights reserved.
12 * Copyright (c) 1998 by Fergus Henderson. All rights reserved.
13 * Copyright (c) 2000-2004 by Hewlett-Packard Company. All rights reserved.
15 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
16 * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
18 * Permission is hereby granted to use or copy this program
19 * for any purpose, provided the above notices are retained on all copies.
20 * Permission to modify the code and to distribute modified code is granted,
21 * provided the above notices are retained, and a notice that the code was
22 * modified is included with the above copyright notice.
25 * Copyright 2001-2003 Ximian, Inc
26 * Copyright 2003-2010 Novell, Inc.
28 * Permission is hereby granted, free of charge, to any person obtaining
29 * a copy of this software and associated documentation files (the
30 * "Software"), to deal in the Software without restriction, including
31 * without limitation the rights to use, copy, modify, merge, publish,
32 * distribute, sublicense, and/or sell copies of the Software, and to
33 * permit persons to whom the Software is furnished to do so, subject to
34 * the following conditions:
36 * The above copyright notice and this permission notice shall be
37 * included in all copies or substantial portions of the Software.
39 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
40 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
41 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
42 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
43 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
44 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
45 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
48 * Important: allocation provides always zeroed memory, having to do
49 * a memset after allocation is deadly for performance.
50 * Memory usage at startup is currently as follows:
52 * 64 KB internal space
54 * We should provide a small memory config with half the sizes
56 * We currently try to make as few mono assumptions as possible:
57 * 1) 2-word header with no GC pointers in it (first vtable, second to store the
59 * 2) gc descriptor is the second word in the vtable (first word in the class)
60 * 3) 8 byte alignment is the minimum and enough (not true for special structures (SIMD), FIXME)
61 * 4) there is a function to get an object's size and the number of
62 * elements in an array.
63 * 5) we know the special way bounds are allocated for complex arrays
64 * 6) we know about proxies and how to treat them when domains are unloaded
66 * Always try to keep stack usage to a minimum: no recursive behaviour
67 * and no large stack allocs.
69 * General description.
70 * Objects are initially allocated in a nursery using a fast bump-pointer technique.
71 * When the nursery is full we start a nursery collection: this is performed with a
73 * When the old generation is full we start a copying GC of the old generation as well:
74 * this will be changed to mark&sweep with copying when fragmentation becomes to severe
75 * in the future. Maybe we'll even do both during the same collection like IMMIX.
77 * The things that complicate this description are:
78 * *) pinned objects: we can't move them so we need to keep track of them
79 * *) no precise info of the thread stacks and registers: we need to be able to
80 * quickly find the objects that may be referenced conservatively and pin them
81 * (this makes the first issues more important)
82 * *) large objects are too expensive to be dealt with using copying GC: we handle them
83 * with mark/sweep during major collections
84 * *) some objects need to not move even if they are small (interned strings, Type handles):
85 * we use mark/sweep for them, too: they are not allocated in the nursery, but inside
86 * PinnedChunks regions
92 *) we could have a function pointer in MonoClass to implement
93 customized write barriers for value types
95 *) investigate the stuff needed to advance a thread to a GC-safe
96 point (single-stepping, read from unmapped memory etc) and implement it.
97 This would enable us to inline allocations and write barriers, for example,
98 or at least parts of them, like the write barrier checks.
99 We may need this also for handling precise info on stacks, even simple things
100 as having uninitialized data on the stack and having to wait for the prolog
101 to zero it. Not an issue for the last frame that we scan conservatively.
102 We could always not trust the value in the slots anyway.
104 *) modify the jit to save info about references in stack locations:
105 this can be done just for locals as a start, so that at least
106 part of the stack is handled precisely.
108 *) test/fix endianess issues
110 *) Implement a card table as the write barrier instead of remembered
111 sets? Card tables are not easy to implement with our current
112 memory layout. We have several different kinds of major heap
113 objects: Small objects in regular blocks, small objects in pinned
114 chunks and LOS objects. If we just have a pointer we have no way
115 to tell which kind of object it points into, therefore we cannot
116 know where its card table is. The least we have to do to make
117 this happen is to get rid of write barriers for indirect stores.
120 *) Get rid of write barriers for indirect stores. We can do this by
121 telling the GC to wbarrier-register an object once we do an ldloca
122 or ldelema on it, and to unregister it once it's not used anymore
123 (it can only travel downwards on the stack). The problem with
124 unregistering is that it needs to happen eventually no matter
125 what, even if exceptions are thrown, the thread aborts, etc.
126 Rodrigo suggested that we could do only the registering part and
127 let the collector find out (pessimistically) when it's safe to
128 unregister, namely when the stack pointer of the thread that
129 registered the object is higher than it was when the registering
130 happened. This might make for a good first implementation to get
131 some data on performance.
133 *) Some sort of blacklist support? Blacklists is a concept from the
134 Boehm GC: if during a conservative scan we find pointers to an
135 area which we might use as heap, we mark that area as unusable, so
136 pointer retention by random pinning pointers is reduced.
138 *) experiment with max small object size (very small right now - 2kb,
139 because it's tied to the max freelist size)
141 *) add an option to mmap the whole heap in one chunk: it makes for many
142 simplifications in the checks (put the nursery at the top and just use a single
143 check for inclusion/exclusion): the issue this has is that on 32 bit systems it's
144 not flexible (too much of the address space may be used by default or we can't
145 increase the heap as needed) and we'd need a race-free mechanism to return memory
146 back to the system (mprotect(PROT_NONE) will still keep the memory allocated if it
147 was written to, munmap is needed, but the following mmap may not find the same segment
150 *) memzero the major fragments after restarting the world and optionally a smaller
153 *) investigate having fragment zeroing threads
155 *) separate locks for finalization and other minor stuff to reduce
158 *) try a different copying order to improve memory locality
160 *) a thread abort after a store but before the write barrier will
161 prevent the write barrier from executing
163 *) specialized dynamically generated markers/copiers
165 *) Dynamically adjust TLAB size to the number of threads. If we have
166 too many threads that do allocation, we might need smaller TLABs,
167 and we might get better performance with larger TLABs if we only
168 have a handful of threads. We could sum up the space left in all
169 assigned TLABs and if that's more than some percentage of the
170 nursery size, reduce the TLAB size.
172 *) Explore placing unreachable objects on unused nursery memory.
173 Instead of memset'ng a region to zero, place an int[] covering it.
174 A good place to start is add_nursery_frag. The tricky thing here is
175 placing those objects atomically outside of a collection.
185 #include <semaphore.h>
194 #define _XOPEN_SOURCE
196 #include "metadata/metadata-internals.h"
197 #include "metadata/class-internals.h"
198 #include "metadata/gc-internal.h"
199 #include "metadata/object-internals.h"
200 #include "metadata/threads.h"
201 #include "metadata/sgen-gc.h"
202 #include "metadata/sgen-archdep.h"
203 #include "metadata/mono-gc.h"
204 #include "metadata/method-builder.h"
205 #include "metadata/profiler-private.h"
206 #include "metadata/monitor.h"
207 #include "metadata/threadpool-internals.h"
208 #include "metadata/mempool-internals.h"
209 #include "metadata/marshal.h"
210 #include "utils/mono-mmap.h"
211 #include "utils/mono-time.h"
212 #include "utils/mono-semaphore.h"
213 #include "utils/mono-counters.h"
215 #include <mono/utils/memcheck.h>
217 #if defined(__MACH__)
218 #include "utils/mach-support.h"
221 #define OPDEF(a,b,c,d,e,f,g,h,i,j) \
225 #include "mono/cil/opcode.def"
231 #undef pthread_create
233 #undef pthread_detach
236 * ######################################################################
237 * ######## Types and constants used by the GC.
238 * ######################################################################
241 static int gc_initialized = 0;
242 /* If set, do a minor collection before every allocation */
243 static gboolean collect_before_allocs = FALSE;
244 /* If set, do a heap consistency check before each minor collection */
245 static gboolean consistency_check_at_minor_collection = FALSE;
246 /* If set, check that there are no references to the domain left at domain unload */
247 static gboolean xdomain_checks = FALSE;
248 /* If not null, dump the heap after each collection into this file */
249 static FILE *heap_dump_file = NULL;
250 /* If set, mark stacks conservatively, even if precise marking is possible */
251 static gboolean conservative_stack_mark = TRUE;
252 /* If set, do a plausibility check on the scan_starts before and after
254 static gboolean do_scan_starts_check = FALSE;
256 #ifdef HEAVY_STATISTICS
257 static long long stat_objects_alloced = 0;
258 static long long stat_bytes_alloced = 0;
259 static long long stat_objects_alloced_degraded = 0;
260 static long long stat_bytes_alloced_degraded = 0;
261 static long long stat_bytes_alloced_los = 0;
263 static long long stat_copy_object_called_nursery = 0;
264 static long long stat_objects_copied_nursery = 0;
265 static long long stat_copy_object_called_major = 0;
266 static long long stat_objects_copied_major = 0;
268 static long long stat_scan_object_called_nursery = 0;
269 static long long stat_scan_object_called_major = 0;
271 static long long stat_nursery_copy_object_failed_from_space = 0;
272 static long long stat_nursery_copy_object_failed_forwarded = 0;
273 static long long stat_nursery_copy_object_failed_pinned = 0;
275 static long long stat_store_remsets = 0;
276 static long long stat_store_remsets_unique = 0;
277 static long long stat_saved_remsets_1 = 0;
278 static long long stat_saved_remsets_2 = 0;
279 static long long stat_global_remsets_added = 0;
280 static long long stat_global_remsets_readded = 0;
281 static long long stat_global_remsets_processed = 0;
282 static long long stat_global_remsets_discarded = 0;
284 static long long stat_wasted_fragments_used = 0;
285 static long long stat_wasted_fragments_bytes = 0;
287 static int stat_wbarrier_set_field = 0;
288 static int stat_wbarrier_set_arrayref = 0;
289 static int stat_wbarrier_arrayref_copy = 0;
290 static int stat_wbarrier_generic_store = 0;
291 static int stat_wbarrier_generic_store_remset = 0;
292 static int stat_wbarrier_set_root = 0;
293 static int stat_wbarrier_value_copy = 0;
294 static int stat_wbarrier_object_copy = 0;
297 static long long time_minor_pre_collection_fragment_clear = 0;
298 static long long time_minor_pinning = 0;
299 static long long time_minor_scan_remsets = 0;
300 static long long time_minor_scan_pinned = 0;
301 static long long time_minor_scan_registered_roots = 0;
302 static long long time_minor_scan_thread_data = 0;
303 static long long time_minor_finish_gray_stack = 0;
304 static long long time_minor_fragment_creation = 0;
306 static long long time_major_pre_collection_fragment_clear = 0;
307 static long long time_major_pinning = 0;
308 static long long time_major_scan_pinned = 0;
309 static long long time_major_scan_registered_roots = 0;
310 static long long time_major_scan_thread_data = 0;
311 static long long time_major_scan_alloc_pinned = 0;
312 static long long time_major_scan_finalized = 0;
313 static long long time_major_scan_big_objects = 0;
314 static long long time_major_finish_gray_stack = 0;
315 static long long time_major_free_bigobjs = 0;
316 static long long time_major_los_sweep = 0;
317 static long long time_major_sweep = 0;
318 static long long time_major_fragment_creation = 0;
320 #define MAX_DEBUG_LEVEL 2
321 #define DEBUG(level,a) do {if (G_UNLIKELY ((level) <= MAX_DEBUG_LEVEL && (level) <= gc_debug_level)) a;} while (0)
323 static int gc_debug_level = 0;
324 static FILE* gc_debug_file;
328 mono_gc_flush_info (void)
330 fflush (gc_debug_file);
334 /* Define this to allow the user to change some of the constants by specifying
335 * their values in the MONO_GC_PARAMS environmental variable. See
336 * mono_gc_base_init for details. */
337 #define USER_CONFIG 1
339 #define TV_DECLARE(name) gint64 name
340 #define TV_GETTIME(tv) tv = mono_100ns_ticks ()
341 #define TV_ELAPSED(start,end) (int)((end-start) / 10)
342 #define TV_ELAPSED_MS(start,end) ((TV_ELAPSED((start),(end)) + 500) / 1000)
344 #define ALIGN_TO(val,align) ((((guint64)val) + ((align) - 1)) & ~((align) - 1))
346 #define GC_BITS_PER_WORD (sizeof (mword) * 8)
348 /* each request from the OS ends up in a GCMemSection */
349 typedef struct _GCMemSection GCMemSection;
350 struct _GCMemSection {
354 /* pointer where more data could be allocated if it fits */
358 * scan starts is an array of pointers to objects equally spaced in the allocation area
359 * They let use quickly find pinned objects from pinning pointers.
362 /* in major collections indexes in the pin_queue for objects that pin this section */
365 unsigned short num_scan_start;
366 gboolean is_to_space;
369 #define SIZEOF_GC_MEM_SECTION ((sizeof (GCMemSection) + 7) & ~7)
371 /* The method used to clear the nursery */
372 /* Clearing at nursery collections is the safest, but has bad interactions with caches.
373 * Clearing at TLAB creation is much faster, but more complex and it might expose hard
378 CLEAR_AT_TLAB_CREATION
379 } NurseryClearPolicy;
381 static NurseryClearPolicy nursery_clear_policy = CLEAR_AT_TLAB_CREATION;
384 * If this is set, the nursery is aligned to an address aligned to its size, ie.
385 * a 1MB nursery will be aligned to an address divisible by 1MB. This allows us to
386 * speed up ptr_in_nursery () checks which are very frequent. This requires the
387 * nursery size to be a compile time constant.
389 #define ALIGN_NURSERY 1
392 * The young generation is divided into fragments. This is because
393 * we can hand one fragments to a thread for lock-less fast alloc and
394 * because the young generation ends up fragmented anyway by pinned objects.
395 * Once a collection is done, a list of fragments is created. When doing
396 * thread local alloc we use smallish nurseries so we allow new threads to
397 * allocate memory from gen0 without triggering a collection. Threads that
398 * are found to allocate lots of memory are given bigger fragments. This
399 * should make the finalizer thread use little nursery memory after a while.
400 * We should start assigning threads very small fragments: if there are many
401 * threads the nursery will be full of reserved space that the threads may not
402 * use at all, slowing down allocation speed.
403 * Thread local allocation is done from areas of memory Hotspot calls Thread Local
404 * Allocation Buffers (TLABs).
406 typedef struct _Fragment Fragment;
410 char *fragment_start;
411 char *fragment_limit; /* the current soft limit for allocation */
415 /* the runtime can register areas of memory as roots: we keep two lists of roots,
416 * a pinned root set for conservatively scanned roots and a normal one for
417 * precisely scanned roots (currently implemented as a single list).
419 typedef struct _RootRecord RootRecord;
428 * We're never actually using the first element. It's always set to
429 * NULL to simplify the elimination of consecutive duplicate
432 #define STORE_REMSET_BUFFER_SIZE 1024
434 typedef struct _GenericStoreRememberedSet GenericStoreRememberedSet;
435 struct _GenericStoreRememberedSet {
436 GenericStoreRememberedSet *next;
437 /* We need one entry less because the first entry of store
438 remset buffers is always a dummy and we don't copy it. */
439 gpointer data [STORE_REMSET_BUFFER_SIZE - 1];
442 /* we have 4 possible values in the low 2 bits */
444 REMSET_LOCATION, /* just a pointer to the exact location */
445 REMSET_RANGE, /* range of pointer fields */
446 REMSET_OBJECT, /* mark all the object for scanning */
447 REMSET_VTYPE, /* a valuetype array described by a gc descriptor and a count */
448 REMSET_TYPE_MASK = 0x3
451 #ifdef HAVE_KW_THREAD
452 static __thread RememberedSet *remembered_set MONO_TLS_FAST;
454 static pthread_key_t remembered_set_key;
455 static RememberedSet *global_remset;
456 static RememberedSet *freed_thread_remsets;
457 static GenericStoreRememberedSet *generic_store_remsets = NULL;
459 /*A two slots cache for recently inserted remsets */
460 static gpointer global_remset_cache [2];
462 /* FIXME: later choose a size that takes into account the RememberedSet struct
463 * and doesn't waste any alloc paddin space.
465 #define DEFAULT_REMSET_SIZE 1024
466 static RememberedSet* alloc_remset (int size, gpointer id);
468 /* Structure that corresponds to a MonoVTable: desc is a mword so requires
469 * no cast from a pointer to an integer
476 /* these bits are set in the object vtable: we could merge them since an object can be
477 * either pinned or forwarded but not both.
478 * We store them in the vtable slot because the bits are used in the sync block for
479 * other purposes: if we merge them and alloc the sync blocks aligned to 8 bytes, we can change
480 * this and use bit 3 in the syncblock (with the lower two bits both set for forwarded, that
481 * would be an invalid combination for the monitor and hash code).
482 * The values are already shifted.
483 * The forwarding address is stored in the sync block.
485 #define FORWARDED_BIT 1
487 #define VTABLE_BITS_MASK 0x3
489 /* returns NULL if not forwarded, or the forwarded address */
490 #define object_is_forwarded(obj) (((mword*)(obj))[0] & FORWARDED_BIT ? (void*)(((mword*)(obj))[0] & ~VTABLE_BITS_MASK) : NULL)
491 /* set the forwarded address fw_addr for object obj */
492 #define forward_object(obj,fw_addr) do { \
493 ((mword*)(obj))[0] = (mword)(fw_addr) | FORWARDED_BIT; \
496 #define object_is_pinned(obj) (((mword*)(obj))[0] & PINNED_BIT)
497 #define pin_object(obj) do { \
498 ((mword*)(obj))[0] |= PINNED_BIT; \
500 #define unpin_object(obj) do { \
501 ((mword*)(obj))[0] &= ~PINNED_BIT; \
505 #define ptr_in_nursery(ptr) (((mword)(ptr) & ~((1 << DEFAULT_NURSERY_BITS) - 1)) == (mword)nursery_start)
507 #define ptr_in_nursery(ptr) ((char*)(ptr) >= nursery_start && (char*)(ptr) < nursery_real_end)
511 * Since we set bits in the vtable, use the macro to load it from the pointer to
512 * an object that is potentially pinned.
514 #define LOAD_VTABLE(addr) ((*(mword*)(addr)) & ~VTABLE_BITS_MASK)
516 #define VTABLE_HAS_REFERENCES(vt) (((MonoVTable*)(vt))->gc_descr != (void*)DESC_TYPE_RUN_LENGTH)
519 safe_name (void* obj)
521 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (obj);
522 return vt->klass->name;
526 * This function can be called on an object whose first word, the
527 * vtable field, is not intact. This is necessary for the parallel
531 par_object_get_size (MonoVTable *vtable, MonoObject* o)
533 MonoClass *klass = vtable->klass;
535 * We depend on mono_string_length_fast and
536 * mono_array_length_fast not using the object's vtable.
538 if (klass == mono_defaults.string_class) {
539 return sizeof (MonoString) + 2 * mono_string_length_fast ((MonoString*) o) + 2;
540 } else if (klass->rank) {
541 MonoArray *array = (MonoArray*)o;
542 size_t size = sizeof (MonoArray) + klass->sizes.element_size * mono_array_length_fast (array);
543 if (G_UNLIKELY (array->bounds)) {
544 size += sizeof (mono_array_size_t) - 1;
545 size &= ~(sizeof (mono_array_size_t) - 1);
546 size += sizeof (MonoArrayBounds) * klass->rank;
550 /* from a created object: the class must be inited already */
551 return klass->instance_size;
555 #define safe_object_get_size(o) par_object_get_size ((MonoVTable*)LOAD_VTABLE ((o)), (o))
558 * ######################################################################
559 * ######## Global data.
560 * ######################################################################
562 static LOCK_DECLARE (gc_mutex);
563 static int gc_disabled = 0;
564 static int num_minor_gcs = 0;
565 static int num_major_gcs = 0;
569 /* good sizes are 512KB-1MB: larger ones increase a lot memzeroing time */
570 #define DEFAULT_NURSERY_SIZE (default_nursery_size)
571 static int default_nursery_size = (1 << 22);
573 /* The number of trailing 0 bits in DEFAULT_NURSERY_SIZE */
574 #define DEFAULT_NURSERY_BITS (default_nursery_bits)
575 static int default_nursery_bits = 22;
580 #define DEFAULT_NURSERY_SIZE (4*1024*1024)
582 #define DEFAULT_NURSERY_BITS 22
587 #define MIN_MINOR_COLLECTION_ALLOWANCE (DEFAULT_NURSERY_SIZE * 4)
588 /* to quickly find the head of an object pinned by a conservative address
589 * we keep track of the objects allocated for each SCAN_START_SIZE memory
590 * chunk in the nursery or other memory sections. Larger values have less
591 * memory overhead and bigger runtime cost. 4-8 KB are reasonable values.
593 #define SCAN_START_SIZE (4096*2)
594 /* the minimum size of a fragment that we consider useful for allocation */
595 #define FRAGMENT_MIN_SIZE (512)
597 static mword pagesize = 4096;
598 static mword nursery_size;
599 static int degraded_mode = 0;
601 static mword total_alloc = 0;
602 /* use this to tune when to do a major/minor collection */
603 static mword memory_pressure = 0;
604 static int minor_collection_allowance;
605 static int minor_collection_sections_alloced = 0;
607 static GCMemSection *nursery_section = NULL;
608 static mword lowest_heap_address = ~(mword)0;
609 static mword highest_heap_address = 0;
611 static LOCK_DECLARE (interruption_mutex);
613 typedef struct _FinalizeEntry FinalizeEntry;
614 struct _FinalizeEntry {
619 typedef struct _FinalizeEntryHashTable FinalizeEntryHashTable;
620 struct _FinalizeEntryHashTable {
621 FinalizeEntry **table;
626 typedef struct _DisappearingLink DisappearingLink;
627 struct _DisappearingLink {
628 DisappearingLink *next;
632 typedef struct _DisappearingLinkHashTable DisappearingLinkHashTable;
633 struct _DisappearingLinkHashTable {
634 DisappearingLink **table;
639 typedef struct _EphemeronLinkNode EphemeronLinkNode;
641 struct _EphemeronLinkNode {
642 EphemeronLinkNode *next;
657 int current_collection_generation = -1;
660 * The link pointer is hidden by negating each bit. We use the lowest
661 * bit of the link (before negation) to store whether it needs
662 * resurrection tracking.
664 #define HIDE_POINTER(p,t) ((gpointer)(~((gulong)(p)|((t)?1:0))))
665 #define REVEAL_POINTER(p) ((gpointer)((~(gulong)(p))&~3L))
667 #define DISLINK_OBJECT(d) (REVEAL_POINTER (*(d)->link))
668 #define DISLINK_TRACK(d) ((~(gulong)(*(d)->link)) & 1)
671 * The finalizable hash has the object as the key, the
672 * disappearing_link hash, has the link address as key.
674 static FinalizeEntryHashTable minor_finalizable_hash;
675 static FinalizeEntryHashTable major_finalizable_hash;
676 /* objects that are ready to be finalized */
677 static FinalizeEntry *fin_ready_list = NULL;
678 static FinalizeEntry *critical_fin_list = NULL;
680 static DisappearingLinkHashTable minor_disappearing_link_hash;
681 static DisappearingLinkHashTable major_disappearing_link_hash;
683 static EphemeronLinkNode *ephemeron_list;
685 static int num_ready_finalizers = 0;
686 static int no_finalize = 0;
689 ROOT_TYPE_NORMAL = 0, /* "normal" roots */
690 ROOT_TYPE_PINNED = 1, /* roots without a GC descriptor */
691 ROOT_TYPE_WBARRIER = 2, /* roots with a write barrier */
695 /* registered roots: the key to the hash is the root start address */
697 * Different kinds of roots are kept separate to speed up pin_from_roots () for example.
699 static RootRecord **roots_hash [ROOT_TYPE_NUM] = { NULL, NULL };
700 static int roots_hash_size [ROOT_TYPE_NUM] = { 0, 0, 0 };
701 static mword roots_size = 0; /* amount of memory in the root set */
702 static int num_roots_entries [ROOT_TYPE_NUM] = { 0, 0, 0 };
705 * The current allocation cursors
706 * We allocate objects in the nursery.
707 * The nursery is the area between nursery_start and nursery_real_end.
708 * Allocation is done from a Thread Local Allocation Buffer (TLAB). TLABs are allocated
709 * from nursery fragments.
710 * tlab_next is the pointer to the space inside the TLAB where the next object will
712 * tlab_temp_end is the pointer to the end of the temporary space reserved for
713 * the allocation: it allows us to set the scan starts at reasonable intervals.
714 * tlab_real_end points to the end of the TLAB.
715 * nursery_frag_real_end points to the end of the currently used nursery fragment.
716 * nursery_first_pinned_start points to the start of the first pinned object in the nursery
717 * nursery_last_pinned_end points to the end of the last pinned object in the nursery
718 * At the next allocation, the area of the nursery where objects can be present is
719 * between MIN(nursery_first_pinned_start, first_fragment_start) and
720 * MAX(nursery_last_pinned_end, nursery_frag_real_end)
722 static char *nursery_start = NULL;
724 #ifdef HAVE_KW_THREAD
725 #define TLAB_ACCESS_INIT
726 #define TLAB_START tlab_start
727 #define TLAB_NEXT tlab_next
728 #define TLAB_TEMP_END tlab_temp_end
729 #define TLAB_REAL_END tlab_real_end
730 #define REMEMBERED_SET remembered_set
731 #define STORE_REMSET_BUFFER store_remset_buffer
732 #define STORE_REMSET_BUFFER_INDEX store_remset_buffer_index
733 #define IN_CRITICAL_REGION thread_info->in_critical_region
735 static pthread_key_t thread_info_key;
736 #define TLAB_ACCESS_INIT SgenThreadInfo *__thread_info__ = pthread_getspecific (thread_info_key)
737 #define TLAB_START (__thread_info__->tlab_start)
738 #define TLAB_NEXT (__thread_info__->tlab_next)
739 #define TLAB_TEMP_END (__thread_info__->tlab_temp_end)
740 #define TLAB_REAL_END (__thread_info__->tlab_real_end)
741 #define REMEMBERED_SET (__thread_info__->remset)
742 #define STORE_REMSET_BUFFER (__thread_info__->store_remset_buffer)
743 #define STORE_REMSET_BUFFER_INDEX (__thread_info__->store_remset_buffer_index)
744 #define IN_CRITICAL_REGION (__thread_info__->in_critical_region)
747 /* we use the memory barrier only to prevent compiler reordering (a memory constraint may be enough) */
748 #define ENTER_CRITICAL_REGION do {IN_CRITICAL_REGION = 1;mono_memory_barrier ();} while (0)
749 #define EXIT_CRITICAL_REGION do {IN_CRITICAL_REGION = 0;mono_memory_barrier ();} while (0)
752 * FIXME: What is faster, a TLS variable pointing to a structure, or separate TLS
753 * variables for next+temp_end ?
755 #ifdef HAVE_KW_THREAD
756 static __thread SgenThreadInfo *thread_info;
757 static __thread char *tlab_start;
758 static __thread char *tlab_next;
759 static __thread char *tlab_temp_end;
760 static __thread char *tlab_real_end;
761 static __thread gpointer *store_remset_buffer;
762 static __thread long store_remset_buffer_index;
763 /* Used by the managed allocator/wbarrier */
764 static __thread char **tlab_next_addr;
765 static __thread char *stack_end;
766 static __thread long *store_remset_buffer_index_addr;
768 static char *nursery_next = NULL;
769 static char *nursery_frag_real_end = NULL;
770 static char *nursery_real_end = NULL;
771 static char *nursery_last_pinned_end = NULL;
773 /* The size of a TLAB */
774 /* The bigger the value, the less often we have to go to the slow path to allocate a new
775 * one, but the more space is wasted by threads not allocating much memory.
777 * FIXME: Make this self-tuning for each thread.
779 static guint32 tlab_size = (1024 * 4);
781 /*How much space is tolerable to be wasted from the current fragment when allocating a new TLAB*/
782 #define MAX_NURSERY_TLAB_WASTE 512
784 /* fragments that are free and ready to be used for allocation */
785 static Fragment *nursery_fragments = NULL;
786 /* freeelist of fragment structures */
787 static Fragment *fragment_freelist = NULL;
790 * Objects bigger then this go into the large object space. This size
791 * has a few constraints. It must fit into the major heap, which in
792 * the case of the copying collector means that it must fit into a
793 * pinned chunk. It must also play well with the GC descriptors, some
794 * of which (DESC_TYPE_RUN_LENGTH, DESC_TYPE_SMALL_BITMAP) encode the
797 #define MAX_SMALL_OBJ_SIZE 8000
799 /* Functions supplied by the runtime to be called by the GC */
800 static MonoGCCallbacks gc_callbacks;
802 #define ALLOC_ALIGN SGEN_ALLOC_ALIGN
803 #define ALLOC_ALIGN_BITS SGEN_ALLOC_ALIGN_BITS
805 #define ALIGN_UP SGEN_ALIGN_UP
807 #define MOVED_OBJECTS_NUM 64
808 static void *moved_objects [MOVED_OBJECTS_NUM];
809 static int moved_objects_idx = 0;
812 * ######################################################################
813 * ######## Macros and function declarations.
814 * ######################################################################
817 #define ADDR_IN_HEAP_BOUNDARIES(addr) ((p) >= lowest_heap_address && (p) < highest_heap_address)
820 align_pointer (void *ptr)
822 mword p = (mword)ptr;
823 p += sizeof (gpointer) - 1;
824 p &= ~ (sizeof (gpointer) - 1);
828 typedef struct _GrayQueue GrayQueue;
830 typedef void (*CopyOrMarkObjectFunc) (void**, GrayQueue*);
831 typedef char* (*ScanObjectFunc) (char*, GrayQueue*);
833 /* forward declarations */
834 static int stop_world (void);
835 static int restart_world (void);
836 static void add_to_global_remset (gpointer ptr);
837 static void scan_thread_data (void *start_nursery, void *end_nursery, gboolean precise);
838 static void scan_from_remsets (void *start_nursery, void *end_nursery, GrayQueue *queue);
839 static void scan_from_registered_roots (CopyOrMarkObjectFunc copy_func, char *addr_start, char *addr_end, int root_type, GrayQueue *queue);
840 static void scan_finalizer_entries (CopyOrMarkObjectFunc copy_func, FinalizeEntry *list, GrayQueue *queue);
841 static void find_pinning_ref_from_thread (char *obj, size_t size);
842 static void update_current_thread_stack (void *start);
843 static void finalize_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, int generation, GrayQueue *queue);
844 static void add_or_remove_disappearing_link (MonoObject *obj, void **link, gboolean track, int generation);
845 static void null_link_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, int generation, GrayQueue *queue);
846 static void null_links_for_domain (MonoDomain *domain, int generation);
847 static gboolean search_fragment_for_size (size_t size);
848 static int search_fragment_for_size_range (size_t desired_size, size_t minimum_size);
849 static void build_nursery_fragments (int start_pin, int end_pin);
850 static void clear_nursery_fragments (char *next);
851 static void pin_from_roots (void *start_nursery, void *end_nursery);
852 static int pin_objects_from_addresses (GCMemSection *section, void **start, void **end, void *start_nursery, void *end_nursery, GrayQueue *queue);
853 static void pin_objects_in_section (GCMemSection *section, GrayQueue *queue);
854 static void optimize_pin_queue (int start_slot);
855 static void clear_remsets (void);
856 static void clear_tlabs (void);
857 static void scan_area_with_callback (char *start, char *end, IterateObjectCallbackFunc callback, void *data);
858 static void scan_object (char *start, GrayQueue *queue);
859 static void major_scan_object (char *start, GrayQueue *queue);
860 static void par_copy_object_no_checks (char *destination, MonoVTable *vt, void *obj, mword objsize, GrayQueue *queue);
861 static void* copy_object_no_checks (void *obj, GrayQueue *queue);
862 static void copy_object (void **obj_slot, GrayQueue *queue);
863 static void sort_addresses (void **array, int size);
864 static void drain_gray_stack (GrayQueue *queue);
865 static void finish_gray_stack (char *start_addr, char *end_addr, int generation, GrayQueue *queue);
866 static gboolean need_major_collection (void);
867 static void major_collection (const char *reason);
869 static void mono_gc_register_disappearing_link (MonoObject *obj, void **link, gboolean track);
871 void describe_ptr (char *ptr);
872 void check_object (char *start);
874 static void check_consistency (void);
875 static void check_major_refs (void);
876 static void check_section_scan_starts (GCMemSection *section);
877 static void check_scan_starts (void);
878 static void check_for_xdomain_refs (void);
879 static void dump_occupied (char *start, char *end, char *section_start);
880 static void dump_section (GCMemSection *section, const char *type);
881 static void dump_heap (const char *type, int num, const char *reason);
883 void mono_gc_scan_for_specific_ref (MonoObject *key);
885 static void init_stats (void);
887 static int mark_ephemerons_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, GrayQueue *queue);
888 static void clear_unreachable_ephemerons (CopyOrMarkObjectFunc copy_func, char *start, char *end, GrayQueue *queue);
889 static void null_ephemerons_for_domain (MonoDomain *domain);
891 //#define BINARY_PROTOCOL
892 #include "sgen-protocol.c"
893 #include "sgen-pinning.c"
894 #include "sgen-pinning-stats.c"
895 #include "sgen-gray.c"
896 #include "sgen-los.c"
899 * ######################################################################
900 * ######## GC descriptors
901 * ######################################################################
902 * Used to quickly get the info the GC needs about an object: size and
903 * where the references are held.
905 /* objects are aligned to 8 bytes boundaries
906 * A descriptor is a pointer in MonoVTable, so 32 or 64 bits of size.
907 * The low 3 bits define the type of the descriptor. The other bits
908 * depend on the type.
909 * As a general rule the 13 remaining low bits define the size, either
910 * of the whole object or of the elements in the arrays. While for objects
911 * the size is already in bytes, for arrays we need to shift, because
912 * array elements might be smaller than 8 bytes. In case of arrays, we
913 * use two bits to describe what the additional high bits represents,
914 * so the default behaviour can handle element sizes less than 2048 bytes.
915 * The high 16 bits, if 0 it means the object is pointer-free.
916 * This design should make it easy and fast to skip over ptr-free data.
917 * The first 4 types should cover >95% of the objects.
918 * Note that since the size of objects is limited to 64K, larger objects
919 * will be allocated in the large object heap.
920 * If we want 4-bytes alignment, we need to put vector and small bitmap
925 * We don't use 0 so that 0 isn't a valid GC descriptor. No
926 * deep reason for this other than to be able to identify a
927 * non-inited descriptor for debugging.
929 * If an object contains no references, its GC descriptor is
930 * always DESC_TYPE_RUN_LENGTH, without a size, no exceptions.
931 * This is so that we can quickly check for that in
932 * copy_object_no_checks(), without having to fetch the
935 DESC_TYPE_RUN_LENGTH = 1, /* 15 bits aligned byte size | 1-3 (offset, numptr) bytes tuples */
936 DESC_TYPE_SMALL_BITMAP, /* 15 bits aligned byte size | 16-48 bit bitmap */
937 DESC_TYPE_COMPLEX, /* index for bitmap into complex_descriptors */
938 DESC_TYPE_VECTOR, /* 10 bits element size | 1 bit array | 2 bits desc | element desc */
939 DESC_TYPE_ARRAY, /* 10 bits element size | 1 bit array | 2 bits desc | element desc */
940 DESC_TYPE_LARGE_BITMAP, /* | 29-61 bitmap bits */
941 DESC_TYPE_COMPLEX_ARR, /* index for bitmap into complex_descriptors */
942 /* subtypes for arrays and vectors */
943 DESC_TYPE_V_PTRFREE = 0,/* there are no refs: keep first so it has a zero value */
944 DESC_TYPE_V_REFS, /* all the array elements are refs */
945 DESC_TYPE_V_RUN_LEN, /* elements are run-length encoded as DESC_TYPE_RUN_LENGTH */
946 DESC_TYPE_V_BITMAP /* elements are as the bitmap in DESC_TYPE_SMALL_BITMAP */
949 #define OBJECT_HEADER_WORDS (sizeof(MonoObject)/sizeof(gpointer))
950 #define LOW_TYPE_BITS 3
951 #define SMALL_BITMAP_SHIFT 16
952 #define SMALL_BITMAP_SIZE (GC_BITS_PER_WORD - SMALL_BITMAP_SHIFT)
953 #define VECTOR_INFO_SHIFT 14
954 #define VECTOR_ELSIZE_SHIFT 3
955 #define LARGE_BITMAP_SIZE (GC_BITS_PER_WORD - LOW_TYPE_BITS)
956 #define MAX_ELEMENT_SIZE 0x3ff
957 #define VECTOR_SUBTYPE_PTRFREE (DESC_TYPE_V_PTRFREE << VECTOR_INFO_SHIFT)
958 #define VECTOR_SUBTYPE_REFS (DESC_TYPE_V_REFS << VECTOR_INFO_SHIFT)
959 #define VECTOR_SUBTYPE_RUN_LEN (DESC_TYPE_V_RUN_LEN << VECTOR_INFO_SHIFT)
960 #define VECTOR_SUBTYPE_BITMAP (DESC_TYPE_V_BITMAP << VECTOR_INFO_SHIFT)
963 /* Root bitmap descriptors are simpler: the lower three bits describe the type
964 * and we either have 30/62 bitmap bits or nibble-based run-length,
965 * or a complex descriptor, or a user defined marker function.
968 ROOT_DESC_CONSERVATIVE, /* 0, so matches NULL value */
973 ROOT_DESC_TYPE_MASK = 0x7,
974 ROOT_DESC_TYPE_SHIFT = 3,
977 #define MAKE_ROOT_DESC(type,val) ((type) | ((val) << ROOT_DESC_TYPE_SHIFT))
979 #define MAX_USER_DESCRIPTORS 16
981 static gsize* complex_descriptors = NULL;
982 static int complex_descriptors_size = 0;
983 static int complex_descriptors_next = 0;
984 static MonoGCRootMarkFunc user_descriptors [MAX_USER_DESCRIPTORS];
985 static int user_descriptors_next = 0;
988 alloc_complex_descriptor (gsize *bitmap, int numbits)
992 numbits = ALIGN_TO (numbits, GC_BITS_PER_WORD);
993 nwords = numbits / GC_BITS_PER_WORD + 1;
996 res = complex_descriptors_next;
997 /* linear search, so we don't have duplicates with domain load/unload
998 * this should not be performance critical or we'd have bigger issues
999 * (the number and size of complex descriptors should be small).
1001 for (i = 0; i < complex_descriptors_next; ) {
1002 if (complex_descriptors [i] == nwords) {
1003 int j, found = TRUE;
1004 for (j = 0; j < nwords - 1; ++j) {
1005 if (complex_descriptors [i + 1 + j] != bitmap [j]) {
1015 i += complex_descriptors [i];
1017 if (complex_descriptors_next + nwords > complex_descriptors_size) {
1018 int new_size = complex_descriptors_size * 2 + nwords;
1019 complex_descriptors = g_realloc (complex_descriptors, new_size * sizeof (gsize));
1020 complex_descriptors_size = new_size;
1022 DEBUG (6, fprintf (gc_debug_file, "Complex descriptor %d, size: %d (total desc memory: %d)\n", res, nwords, complex_descriptors_size));
1023 complex_descriptors_next += nwords;
1024 complex_descriptors [res] = nwords;
1025 for (i = 0; i < nwords - 1; ++i) {
1026 complex_descriptors [res + 1 + i] = bitmap [i];
1027 DEBUG (6, fprintf (gc_debug_file, "\tvalue: %p\n", (void*)complex_descriptors [res + 1 + i]));
1034 * Descriptor builders.
1037 mono_gc_make_descr_for_string (gsize *bitmap, int numbits)
1039 return (void*) DESC_TYPE_RUN_LENGTH;
1043 mono_gc_make_descr_for_object (gsize *bitmap, int numbits, size_t obj_size)
1045 int first_set = -1, num_set = 0, last_set = -1, i;
1047 size_t stored_size = obj_size;
1048 for (i = 0; i < numbits; ++i) {
1049 if (bitmap [i / GC_BITS_PER_WORD] & ((gsize)1 << (i % GC_BITS_PER_WORD))) {
1057 * We don't encode the size of types that don't contain
1058 * references because they might not be aligned, i.e. the
1059 * bottom two bits might be set, which would clash with the
1060 * bits we need to encode the descriptor type. Since we don't
1061 * use the encoded size to skip objects, other than for
1062 * processing remsets, in which case only the positions of
1063 * references are relevant, this is not a problem.
1066 return (void*)DESC_TYPE_RUN_LENGTH;
1067 g_assert (!(stored_size & 0x3));
1068 if (stored_size <= MAX_SMALL_OBJ_SIZE) {
1069 /* check run-length encoding first: one byte offset, one byte number of pointers
1070 * on 64 bit archs, we can have 3 runs, just one on 32.
1071 * It may be better to use nibbles.
1073 if (first_set < 0) {
1074 desc = DESC_TYPE_RUN_LENGTH | (stored_size << 1);
1075 DEBUG (6, fprintf (gc_debug_file, "Ptrfree descriptor %p, size: %zd\n", (void*)desc, stored_size));
1076 return (void*) desc;
1077 } else if (first_set < 256 && num_set < 256 && (first_set + num_set == last_set + 1)) {
1078 desc = DESC_TYPE_RUN_LENGTH | (stored_size << 1) | (first_set << 16) | (num_set << 24);
1079 DEBUG (6, fprintf (gc_debug_file, "Runlen descriptor %p, size: %zd, first set: %d, num set: %d\n", (void*)desc, stored_size, first_set, num_set));
1080 return (void*) desc;
1082 /* we know the 2-word header is ptr-free */
1083 if (last_set < SMALL_BITMAP_SIZE + OBJECT_HEADER_WORDS) {
1084 desc = DESC_TYPE_SMALL_BITMAP | (stored_size << 1) | ((*bitmap >> OBJECT_HEADER_WORDS) << SMALL_BITMAP_SHIFT);
1085 DEBUG (6, fprintf (gc_debug_file, "Smallbitmap descriptor %p, size: %zd, last set: %d\n", (void*)desc, stored_size, last_set));
1086 return (void*) desc;
1089 /* we know the 2-word header is ptr-free */
1090 if (last_set < LARGE_BITMAP_SIZE + OBJECT_HEADER_WORDS) {
1091 desc = DESC_TYPE_LARGE_BITMAP | ((*bitmap >> OBJECT_HEADER_WORDS) << LOW_TYPE_BITS);
1092 DEBUG (6, fprintf (gc_debug_file, "Largebitmap descriptor %p, size: %zd, last set: %d\n", (void*)desc, stored_size, last_set));
1093 return (void*) desc;
1095 /* it's a complex object ... */
1096 desc = DESC_TYPE_COMPLEX | (alloc_complex_descriptor (bitmap, last_set + 1) << LOW_TYPE_BITS);
1097 return (void*) desc;
1100 /* If the array holds references, numbits == 1 and the first bit is set in elem_bitmap */
1102 mono_gc_make_descr_for_array (int vector, gsize *elem_bitmap, int numbits, size_t elem_size)
1104 int first_set = -1, num_set = 0, last_set = -1, i;
1105 mword desc = vector? DESC_TYPE_VECTOR: DESC_TYPE_ARRAY;
1106 for (i = 0; i < numbits; ++i) {
1107 if (elem_bitmap [i / GC_BITS_PER_WORD] & ((gsize)1 << (i % GC_BITS_PER_WORD))) {
1114 /* See comment at the definition of DESC_TYPE_RUN_LENGTH. */
1116 return (void*)DESC_TYPE_RUN_LENGTH;
1117 if (elem_size <= MAX_ELEMENT_SIZE) {
1118 desc |= elem_size << VECTOR_ELSIZE_SHIFT;
1120 return (void*)(desc | VECTOR_SUBTYPE_PTRFREE);
1122 /* Note: we also handle structs with just ref fields */
1123 if (num_set * sizeof (gpointer) == elem_size) {
1124 return (void*)(desc | VECTOR_SUBTYPE_REFS | ((gssize)(-1) << 16));
1126 /* FIXME: try run-len first */
1127 /* Note: we can't skip the object header here, because it's not present */
1128 if (last_set <= SMALL_BITMAP_SIZE) {
1129 return (void*)(desc | VECTOR_SUBTYPE_BITMAP | (*elem_bitmap << 16));
1132 /* it's am array of complex structs ... */
1133 desc = DESC_TYPE_COMPLEX_ARR;
1134 desc |= alloc_complex_descriptor (elem_bitmap, last_set + 1) << LOW_TYPE_BITS;
1135 return (void*) desc;
1138 /* Return the bitmap encoded by a descriptor */
1140 mono_gc_get_bitmap_for_descr (void *descr, int *numbits)
1142 mword d = (mword)descr;
1146 case DESC_TYPE_RUN_LENGTH: {
1147 int first_set = (d >> 16) & 0xff;
1148 int num_set = (d >> 24) & 0xff;
1151 bitmap = g_new0 (gsize, (first_set + num_set + 7) / 8);
1153 for (i = first_set; i < first_set + num_set; ++i)
1154 bitmap [i / GC_BITS_PER_WORD] |= ((gsize)1 << (i % GC_BITS_PER_WORD));
1156 *numbits = first_set + num_set;
1160 case DESC_TYPE_SMALL_BITMAP:
1161 bitmap = g_new0 (gsize, 1);
1163 bitmap [0] = (d >> SMALL_BITMAP_SHIFT) << OBJECT_HEADER_WORDS;
1165 *numbits = GC_BITS_PER_WORD;
1169 g_assert_not_reached ();
1173 /* helper macros to scan and traverse objects, macros because we resue them in many functions */
1174 #define OBJ_RUN_LEN_SIZE(size,desc,obj) do { \
1175 (size) = ((desc) & 0xfff8) >> 1; \
1178 #define OBJ_BITMAP_SIZE(size,desc,obj) do { \
1179 (size) = ((desc) & 0xfff8) >> 1; \
1182 //#define PREFETCH(addr) __asm__ __volatile__ (" prefetchnta %0": : "m"(*(char *)(addr)))
1183 #define PREFETCH(addr)
1185 /* code using these macros must define a HANDLE_PTR(ptr) macro that does the work */
1186 #define OBJ_RUN_LEN_FOREACH_PTR(desc,obj) do { \
1187 if ((desc) & 0xffff0000) { \
1188 /* there are pointers */ \
1189 void **_objptr_end; \
1190 void **_objptr = (void**)(obj); \
1191 _objptr += ((desc) >> 16) & 0xff; \
1192 _objptr_end = _objptr + (((desc) >> 24) & 0xff); \
1193 while (_objptr < _objptr_end) { \
1194 HANDLE_PTR (_objptr, (obj)); \
1200 /* a bitmap desc means that there are pointer references or we'd have
1201 * choosen run-length, instead: add an assert to check.
1203 #define OBJ_BITMAP_FOREACH_PTR(desc,obj) do { \
1204 /* there are pointers */ \
1205 void **_objptr = (void**)(obj); \
1206 gsize _bmap = (desc) >> 16; \
1207 _objptr += OBJECT_HEADER_WORDS; \
1209 if ((_bmap & 1)) { \
1210 HANDLE_PTR (_objptr, (obj)); \
1217 #define OBJ_LARGE_BITMAP_FOREACH_PTR(vt,obj) do { \
1218 /* there are pointers */ \
1219 void **_objptr = (void**)(obj); \
1220 gsize _bmap = (vt)->desc >> LOW_TYPE_BITS; \
1221 _objptr += OBJECT_HEADER_WORDS; \
1223 if ((_bmap & 1)) { \
1224 HANDLE_PTR (_objptr, (obj)); \
1231 #define OBJ_COMPLEX_FOREACH_PTR(vt,obj) do { \
1232 /* there are pointers */ \
1233 void **_objptr = (void**)(obj); \
1234 gsize *bitmap_data = complex_descriptors + ((vt)->desc >> LOW_TYPE_BITS); \
1235 int bwords = (*bitmap_data) - 1; \
1236 void **start_run = _objptr; \
1239 MonoObject *myobj = (MonoObject*)obj; \
1240 g_print ("found %d at %p (0x%zx): %s.%s\n", bwords, (obj), (vt)->desc, myobj->vtable->klass->name_space, myobj->vtable->klass->name); \
1242 while (bwords-- > 0) { \
1243 gsize _bmap = *bitmap_data++; \
1244 _objptr = start_run; \
1245 /*g_print ("bitmap: 0x%x/%d at %p\n", _bmap, bwords, _objptr);*/ \
1247 if ((_bmap & 1)) { \
1248 HANDLE_PTR (_objptr, (obj)); \
1253 start_run += GC_BITS_PER_WORD; \
1257 /* this one is untested */
1258 #define OBJ_COMPLEX_ARR_FOREACH_PTR(vt,obj) do { \
1259 /* there are pointers */ \
1260 gsize *mbitmap_data = complex_descriptors + ((vt)->desc >> LOW_TYPE_BITS); \
1261 int mbwords = (*mbitmap_data++) - 1; \
1262 int el_size = mono_array_element_size (vt->klass); \
1263 char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
1264 char *e_end = e_start + el_size * mono_array_length_fast ((MonoArray*)(obj)); \
1266 g_print ("found %d at %p (0x%zx): %s.%s\n", mbwords, (obj), (vt)->desc, vt->klass->name_space, vt->klass->name); \
1267 while (e_start < e_end) { \
1268 void **_objptr = (void**)e_start; \
1269 gsize *bitmap_data = mbitmap_data; \
1270 unsigned int bwords = mbwords; \
1271 while (bwords-- > 0) { \
1272 gsize _bmap = *bitmap_data++; \
1273 void **start_run = _objptr; \
1274 /*g_print ("bitmap: 0x%x\n", _bmap);*/ \
1276 if ((_bmap & 1)) { \
1277 HANDLE_PTR (_objptr, (obj)); \
1282 _objptr = start_run + GC_BITS_PER_WORD; \
1284 e_start += el_size; \
1288 #define OBJ_VECTOR_FOREACH_PTR(vt,obj) do { \
1289 /* note: 0xffffc000 excludes DESC_TYPE_V_PTRFREE */ \
1290 if ((vt)->desc & 0xffffc000) { \
1291 int el_size = ((vt)->desc >> 3) & MAX_ELEMENT_SIZE; \
1292 /* there are pointers */ \
1293 int etype = (vt)->desc & 0xc000; \
1294 if (etype == (DESC_TYPE_V_REFS << 14)) { \
1295 void **p = (void**)((char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector)); \
1296 void **end_refs = (void**)((char*)p + el_size * mono_array_length_fast ((MonoArray*)(obj))); \
1297 /* Note: this code can handle also arrays of struct with only references in them */ \
1298 while (p < end_refs) { \
1299 HANDLE_PTR (p, (obj)); \
1302 } else if (etype == DESC_TYPE_V_RUN_LEN << 14) { \
1303 int offset = ((vt)->desc >> 16) & 0xff; \
1304 int num_refs = ((vt)->desc >> 24) & 0xff; \
1305 char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
1306 char *e_end = e_start + el_size * mono_array_length_fast ((MonoArray*)(obj)); \
1307 while (e_start < e_end) { \
1308 void **p = (void**)e_start; \
1311 for (i = 0; i < num_refs; ++i) { \
1312 HANDLE_PTR (p + i, (obj)); \
1314 e_start += el_size; \
1316 } else if (etype == DESC_TYPE_V_BITMAP << 14) { \
1317 char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
1318 char *e_end = e_start + el_size * mono_array_length_fast ((MonoArray*)(obj)); \
1319 while (e_start < e_end) { \
1320 void **p = (void**)e_start; \
1321 gsize _bmap = (vt)->desc >> 16; \
1322 /* Note: there is no object header here to skip */ \
1324 if ((_bmap & 1)) { \
1325 HANDLE_PTR (p, (obj)); \
1330 e_start += el_size; \
1336 //#include "sgen-major-copying.c"
1337 #include "sgen-marksweep.c"
1340 is_xdomain_ref_allowed (gpointer *ptr, char *obj, MonoDomain *domain)
1342 MonoObject *o = (MonoObject*)(obj);
1343 MonoObject *ref = (MonoObject*)*(ptr);
1344 int offset = (char*)(ptr) - (char*)o;
1346 if (o->vtable->klass == mono_defaults.thread_class && offset == G_STRUCT_OFFSET (MonoThread, internal_thread))
1348 if (o->vtable->klass == mono_defaults.internal_thread_class && offset == G_STRUCT_OFFSET (MonoInternalThread, current_appcontext))
1350 if (mono_class_has_parent (o->vtable->klass, mono_defaults.real_proxy_class) &&
1351 offset == G_STRUCT_OFFSET (MonoRealProxy, unwrapped_server))
1353 /* Thread.cached_culture_info */
1354 if (!strcmp (ref->vtable->klass->name_space, "System.Globalization") &&
1355 !strcmp (ref->vtable->klass->name, "CultureInfo") &&
1356 !strcmp(o->vtable->klass->name_space, "System") &&
1357 !strcmp(o->vtable->klass->name, "Object[]"))
1360 * at System.IO.MemoryStream.InternalConstructor (byte[],int,int,bool,bool) [0x0004d] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.IO/MemoryStream.cs:121
1361 * at System.IO.MemoryStream..ctor (byte[]) [0x00017] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.IO/MemoryStream.cs:81
1362 * at (wrapper remoting-invoke-with-check) System.IO.MemoryStream..ctor (byte[]) <IL 0x00020, 0xffffffff>
1363 * at System.Runtime.Remoting.Messaging.CADMethodCallMessage.GetArguments () [0x0000d] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Messaging/CADMessages.cs:327
1364 * at System.Runtime.Remoting.Messaging.MethodCall..ctor (System.Runtime.Remoting.Messaging.CADMethodCallMessage) [0x00017] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Messaging/MethodCall.cs:87
1365 * at System.AppDomain.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage,byte[]&,System.Runtime.Remoting.Messaging.CADMethodReturnMessage&) [0x00018] in /home/schani/Work/novell/trunk/mcs/class/corlib/System/AppDomain.cs:1213
1366 * at (wrapper remoting-invoke-with-check) System.AppDomain.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage,byte[]&,System.Runtime.Remoting.Messaging.CADMethodReturnMessage&) <IL 0x0003d, 0xffffffff>
1367 * at System.Runtime.Remoting.Channels.CrossAppDomainSink.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage) [0x00008] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Channels/CrossAppDomainChannel.cs:198
1368 * at (wrapper runtime-invoke) object.runtime_invoke_CrossAppDomainSink/ProcessMessageRes_object_object (object,intptr,intptr,intptr) <IL 0x0004c, 0xffffffff>
1370 if (!strcmp (ref->vtable->klass->name_space, "System") &&
1371 !strcmp (ref->vtable->klass->name, "Byte[]") &&
1372 !strcmp (o->vtable->klass->name_space, "System.IO") &&
1373 !strcmp (o->vtable->klass->name, "MemoryStream"))
1375 /* append_job() in threadpool.c */
1376 if (!strcmp (ref->vtable->klass->name_space, "System.Runtime.Remoting.Messaging") &&
1377 !strcmp (ref->vtable->klass->name, "AsyncResult") &&
1378 !strcmp (o->vtable->klass->name_space, "System") &&
1379 !strcmp (o->vtable->klass->name, "Object[]") &&
1380 mono_thread_pool_is_queue_array ((MonoArray*) o))
1386 check_reference_for_xdomain (gpointer *ptr, char *obj, MonoDomain *domain)
1388 MonoObject *o = (MonoObject*)(obj);
1389 MonoObject *ref = (MonoObject*)*(ptr);
1390 int offset = (char*)(ptr) - (char*)o;
1392 MonoClassField *field;
1395 if (!ref || ref->vtable->domain == domain)
1397 if (is_xdomain_ref_allowed (ptr, obj, domain))
1401 for (class = o->vtable->klass; class; class = class->parent) {
1404 for (i = 0; i < class->field.count; ++i) {
1405 if (class->fields[i].offset == offset) {
1406 field = &class->fields[i];
1414 if (ref->vtable->klass == mono_defaults.string_class)
1415 str = mono_string_to_utf8 ((MonoString*)ref);
1418 g_print ("xdomain reference in %p (%s.%s) at offset %d (%s) to %p (%s.%s) (%s) - pointed to by:\n",
1419 o, o->vtable->klass->name_space, o->vtable->klass->name,
1420 offset, field ? field->name : "",
1421 ref, ref->vtable->klass->name_space, ref->vtable->klass->name, str ? str : "");
1422 mono_gc_scan_for_specific_ref (o);
1428 #define HANDLE_PTR(ptr,obj) check_reference_for_xdomain ((ptr), (obj), domain)
1431 scan_object_for_xdomain_refs (char *start, mword size, void *data)
1433 MonoDomain *domain = ((MonoObject*)start)->vtable->domain;
1435 #include "sgen-scan-object.h"
1439 #define HANDLE_PTR(ptr,obj) do { \
1440 if ((MonoObject*)*(ptr) == key) { \
1441 g_print ("found ref to %p in object %p (%s) at offset %td\n", \
1442 key, (obj), safe_name ((obj)), ((char*)(ptr) - (char*)(obj))); \
1447 scan_object_for_specific_ref (char *start, MonoObject *key)
1449 #include "sgen-scan-object.h"
1453 scan_area_with_callback (char *start, char *end, IterateObjectCallbackFunc callback, void *data)
1455 while (start < end) {
1457 if (!*(void**)start) {
1458 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
1462 size = ALIGN_UP (safe_object_get_size ((MonoObject*) start));
1464 callback (start, size, data);
1471 scan_object_for_specific_ref_callback (char *obj, size_t size, MonoObject *key)
1473 scan_object_for_specific_ref (obj, key);
1477 check_root_obj_specific_ref (RootRecord *root, MonoObject *key, MonoObject *obj)
1481 g_print ("found ref to %p in root record %p\n", key, root);
1484 static MonoObject *check_key = NULL;
1485 static RootRecord *check_root = NULL;
1488 check_root_obj_specific_ref_from_marker (void **obj)
1490 check_root_obj_specific_ref (check_root, check_key, *obj);
1494 scan_roots_for_specific_ref (MonoObject *key, int root_type)
1499 for (i = 0; i < roots_hash_size [root_type]; ++i) {
1500 for (root = roots_hash [root_type][i]; root; root = root->next) {
1501 void **start_root = (void**)root->start_root;
1502 mword desc = root->root_desc;
1506 switch (desc & ROOT_DESC_TYPE_MASK) {
1507 case ROOT_DESC_BITMAP:
1508 desc >>= ROOT_DESC_TYPE_SHIFT;
1511 check_root_obj_specific_ref (root, key, *start_root);
1516 case ROOT_DESC_COMPLEX: {
1517 gsize *bitmap_data = complex_descriptors + (desc >> ROOT_DESC_TYPE_SHIFT);
1518 int bwords = (*bitmap_data) - 1;
1519 void **start_run = start_root;
1521 while (bwords-- > 0) {
1522 gsize bmap = *bitmap_data++;
1523 void **objptr = start_run;
1526 check_root_obj_specific_ref (root, key, *objptr);
1530 start_run += GC_BITS_PER_WORD;
1534 case ROOT_DESC_USER: {
1535 MonoGCRootMarkFunc marker = user_descriptors [desc >> ROOT_DESC_TYPE_SHIFT];
1536 marker (start_root, check_root_obj_specific_ref_from_marker);
1539 case ROOT_DESC_RUN_LEN:
1540 g_assert_not_reached ();
1542 g_assert_not_reached ();
1551 mono_gc_scan_for_specific_ref (MonoObject *key)
1557 scan_area_with_callback (nursery_section->data, nursery_section->end_data,
1558 (IterateObjectCallbackFunc)scan_object_for_specific_ref_callback, key);
1560 major_iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)scan_object_for_specific_ref_callback, key);
1562 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1563 scan_object_for_specific_ref (bigobj->data, key);
1565 scan_roots_for_specific_ref (key, ROOT_TYPE_NORMAL);
1566 scan_roots_for_specific_ref (key, ROOT_TYPE_WBARRIER);
1568 for (i = 0; i < roots_hash_size [ROOT_TYPE_PINNED]; ++i) {
1569 for (root = roots_hash [ROOT_TYPE_PINNED][i]; root; root = root->next) {
1570 void **ptr = (void**)root->start_root;
1572 while (ptr < (void**)root->end_root) {
1573 check_root_obj_specific_ref (root, *ptr, key);
1580 /* Clear all remaining nursery fragments */
1582 clear_nursery_fragments (char *next)
1585 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION) {
1586 g_assert (next <= nursery_frag_real_end);
1587 memset (next, 0, nursery_frag_real_end - next);
1588 for (frag = nursery_fragments; frag; frag = frag->next) {
1589 memset (frag->fragment_start, 0, frag->fragment_end - frag->fragment_start);
1595 need_remove_object_for_domain (char *start, MonoDomain *domain)
1597 if (mono_object_domain (start) == domain) {
1598 DEBUG (4, fprintf (gc_debug_file, "Need to cleanup object %p\n", start));
1599 binary_protocol_cleanup (start, (gpointer)LOAD_VTABLE (start), safe_object_get_size ((MonoObject*)start));
1606 process_object_for_domain_clearing (char *start, MonoDomain *domain)
1608 GCVTable *vt = (GCVTable*)LOAD_VTABLE (start);
1609 if (vt->klass == mono_defaults.internal_thread_class)
1610 g_assert (mono_object_domain (start) == mono_get_root_domain ());
1611 /* The object could be a proxy for an object in the domain
1613 if (mono_class_has_parent (vt->klass, mono_defaults.real_proxy_class)) {
1614 MonoObject *server = ((MonoRealProxy*)start)->unwrapped_server;
1616 /* The server could already have been zeroed out, so
1617 we need to check for that, too. */
1618 if (server && (!LOAD_VTABLE (server) || mono_object_domain (server) == domain)) {
1619 DEBUG (4, fprintf (gc_debug_file, "Cleaning up remote pointer in %p to object %p\n",
1621 ((MonoRealProxy*)start)->unwrapped_server = NULL;
1626 static MonoDomain *check_domain = NULL;
1629 check_obj_not_in_domain (void **o)
1631 g_assert (((MonoObject*)(*o))->vtable->domain != check_domain);
1635 scan_for_registered_roots_in_domain (MonoDomain *domain, int root_type)
1639 check_domain = domain;
1640 for (i = 0; i < roots_hash_size [root_type]; ++i) {
1641 for (root = roots_hash [root_type][i]; root; root = root->next) {
1642 void **start_root = (void**)root->start_root;
1643 mword desc = root->root_desc;
1645 /* The MonoDomain struct is allowed to hold
1646 references to objects in its own domain. */
1647 if (start_root == (void**)domain)
1650 switch (desc & ROOT_DESC_TYPE_MASK) {
1651 case ROOT_DESC_BITMAP:
1652 desc >>= ROOT_DESC_TYPE_SHIFT;
1654 if ((desc & 1) && *start_root)
1655 check_obj_not_in_domain (*start_root);
1660 case ROOT_DESC_COMPLEX: {
1661 gsize *bitmap_data = complex_descriptors + (desc >> ROOT_DESC_TYPE_SHIFT);
1662 int bwords = (*bitmap_data) - 1;
1663 void **start_run = start_root;
1665 while (bwords-- > 0) {
1666 gsize bmap = *bitmap_data++;
1667 void **objptr = start_run;
1669 if ((bmap & 1) && *objptr)
1670 check_obj_not_in_domain (*objptr);
1674 start_run += GC_BITS_PER_WORD;
1678 case ROOT_DESC_USER: {
1679 MonoGCRootMarkFunc marker = user_descriptors [desc >> ROOT_DESC_TYPE_SHIFT];
1680 marker (start_root, check_obj_not_in_domain);
1683 case ROOT_DESC_RUN_LEN:
1684 g_assert_not_reached ();
1686 g_assert_not_reached ();
1690 check_domain = NULL;
1694 check_for_xdomain_refs (void)
1698 scan_area_with_callback (nursery_section->data, nursery_section->end_data, (IterateObjectCallbackFunc)scan_object_for_xdomain_refs, NULL);
1700 major_iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)scan_object_for_xdomain_refs, NULL);
1702 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1703 scan_object_for_xdomain_refs (bigobj->data, bigobj->size, NULL);
1707 clear_domain_process_object (char *obj, MonoDomain *domain)
1711 process_object_for_domain_clearing (obj, domain);
1712 remove = need_remove_object_for_domain (obj, domain);
1714 if (remove && ((MonoObject*)obj)->synchronisation) {
1715 void **dislink = mono_monitor_get_object_monitor_weak_link ((MonoObject*)obj);
1717 mono_gc_register_disappearing_link (NULL, dislink, FALSE);
1724 clear_domain_process_minor_object_callback (char *obj, size_t size, MonoDomain *domain)
1726 if (clear_domain_process_object (obj, domain))
1727 memset (obj, 0, size);
1731 clear_domain_process_major_object_callback (char *obj, size_t size, MonoDomain *domain)
1733 clear_domain_process_object (obj, domain);
1737 clear_domain_free_major_non_pinned_object_callback (char *obj, size_t size, MonoDomain *domain)
1739 if (need_remove_object_for_domain (obj, domain))
1740 major_free_non_pinned_object (obj, size);
1744 clear_domain_free_major_pinned_object_callback (char *obj, size_t size, MonoDomain *domain)
1746 if (need_remove_object_for_domain (obj, domain))
1747 free_pinned_object (obj, size);
1751 * When appdomains are unloaded we can easily remove objects that have finalizers,
1752 * but all the others could still be present in random places on the heap.
1753 * We need a sweep to get rid of them even though it's going to be costly
1755 * The reason we need to remove them is because we access the vtable and class
1756 * structures to know the object size and the reference bitmap: once the domain is
1757 * unloaded the point to random memory.
1760 mono_gc_clear_domain (MonoDomain * domain)
1762 LOSObject *bigobj, *prev;
1767 clear_nursery_fragments (nursery_next);
1769 if (xdomain_checks && domain != mono_get_root_domain ()) {
1770 scan_for_registered_roots_in_domain (domain, ROOT_TYPE_NORMAL);
1771 scan_for_registered_roots_in_domain (domain, ROOT_TYPE_WBARRIER);
1772 check_for_xdomain_refs ();
1775 scan_area_with_callback (nursery_section->data, nursery_section->end_data,
1776 (IterateObjectCallbackFunc)clear_domain_process_minor_object_callback, domain);
1778 /*Ephemerons and dislinks must be processed before LOS since they might end up pointing
1779 to memory returned to the OS.*/
1780 null_ephemerons_for_domain (domain);
1782 for (i = GENERATION_NURSERY; i < GENERATION_MAX; ++i)
1783 null_links_for_domain (domain, i);
1785 /* We need two passes over major and large objects because
1786 freeing such objects might give their memory back to the OS
1787 (in the case of large objects) or obliterate its vtable
1788 (pinned objects with major-copying or pinned and non-pinned
1789 objects with major-mark&sweep), but we might need to
1790 dereference a pointer from an object to another object if
1791 the first object is a proxy. */
1792 major_iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)clear_domain_process_major_object_callback, domain);
1793 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1794 clear_domain_process_object (bigobj->data, domain);
1797 for (bigobj = los_object_list; bigobj;) {
1798 if (need_remove_object_for_domain (bigobj->data, domain)) {
1799 LOSObject *to_free = bigobj;
1801 prev->next = bigobj->next;
1803 los_object_list = bigobj->next;
1804 bigobj = bigobj->next;
1805 DEBUG (4, fprintf (gc_debug_file, "Freeing large object %p\n",
1807 free_large_object (to_free);
1811 bigobj = bigobj->next;
1813 major_iterate_objects (TRUE, FALSE, (IterateObjectCallbackFunc)clear_domain_free_major_non_pinned_object_callback, domain);
1814 major_iterate_objects (FALSE, TRUE, (IterateObjectCallbackFunc)clear_domain_free_major_pinned_object_callback, domain);
1820 global_remset_cache_clear (void)
1822 memset (global_remset_cache, 0, sizeof (global_remset_cache));
1826 * Tries to check if a given remset location was already added to the global remset.
1829 * A 2 entry, LRU cache of recently saw location remsets.
1831 * It's hand-coded instead of done using loops to reduce the number of memory references on cache hit.
1833 * Returns TRUE is the element was added..
1836 global_remset_location_was_not_added (gpointer ptr)
1839 gpointer first = global_remset_cache [0], second;
1841 HEAVY_STAT (++stat_global_remsets_discarded);
1845 second = global_remset_cache [1];
1847 if (second == ptr) {
1848 /*Move the second to the front*/
1849 global_remset_cache [0] = second;
1850 global_remset_cache [1] = first;
1852 HEAVY_STAT (++stat_global_remsets_discarded);
1856 global_remset_cache [0] = second;
1857 global_remset_cache [1] = ptr;
1862 * add_to_global_remset:
1864 * The global remset contains locations which point into newspace after
1865 * a minor collection. This can happen if the objects they point to are pinned.
1868 add_to_global_remset (gpointer ptr)
1872 g_assert (!ptr_in_nursery (ptr) && ptr_in_nursery (*(gpointer*)ptr));
1874 if (!global_remset_location_was_not_added (ptr))
1877 DEBUG (8, fprintf (gc_debug_file, "Adding global remset for %p\n", ptr));
1878 binary_protocol_global_remset (ptr, *(gpointer*)ptr, (gpointer)LOAD_VTABLE (*(gpointer*)ptr));
1880 HEAVY_STAT (++stat_global_remsets_added);
1883 * FIXME: If an object remains pinned, we need to add it at every minor collection.
1884 * To avoid uncontrolled growth of the global remset, only add each pointer once.
1886 if (global_remset->store_next + 3 < global_remset->end_set) {
1887 *(global_remset->store_next++) = (mword)ptr;
1890 rs = alloc_remset (global_remset->end_set - global_remset->data, NULL);
1891 rs->next = global_remset;
1893 *(global_remset->store_next++) = (mword)ptr;
1896 int global_rs_size = 0;
1898 for (rs = global_remset; rs; rs = rs->next) {
1899 global_rs_size += rs->store_next - rs->data;
1901 DEBUG (4, fprintf (gc_debug_file, "Global remset now has size %d\n", global_rs_size));
1906 * This function can be used even if the vtable of obj is not valid
1907 * anymore, which is the case in the parallel collector.
1910 par_copy_object_no_checks (char *destination, MonoVTable *vt, void *obj, mword objsize, GrayQueue *queue)
1912 static const void *copy_labels [] = { &&LAB_0, &&LAB_1, &&LAB_2, &&LAB_3, &&LAB_4, &&LAB_5, &&LAB_6, &&LAB_7, &&LAB_8 };
1914 DEBUG (9, g_assert (vt->klass->inited));
1915 DEBUG (9, fprintf (gc_debug_file, " (to %p, %s size: %lu)\n", destination, ((MonoObject*)obj)->vtable->klass->name, (unsigned long)objsize));
1916 binary_protocol_copy (obj, destination, vt, objsize);
1918 *(MonoVTable**)destination = vt;
1919 if (objsize <= sizeof (gpointer) * 8) {
1920 mword *dest = (mword*)destination;
1921 goto *copy_labels [objsize / sizeof (gpointer)];
1923 (dest) [7] = ((mword*)obj) [7];
1925 (dest) [6] = ((mword*)obj) [6];
1927 (dest) [5] = ((mword*)obj) [5];
1929 (dest) [4] = ((mword*)obj) [4];
1931 (dest) [3] = ((mword*)obj) [3];
1933 (dest) [2] = ((mword*)obj) [2];
1935 (dest) [1] = ((mword*)obj) [1];
1941 memcpy (destination + sizeof (mword), (char*)obj + sizeof (mword), objsize - sizeof (mword));
1943 /* adjust array->bounds */
1944 DEBUG (9, g_assert (vt->gc_descr));
1945 if (G_UNLIKELY (vt->rank && ((MonoArray*)obj)->bounds)) {
1946 MonoArray *array = (MonoArray*)destination;
1947 array->bounds = (MonoArrayBounds*)((char*)destination + ((char*)((MonoArray*)obj)->bounds - (char*)obj));
1948 DEBUG (9, fprintf (gc_debug_file, "Array instance %p: size: %lu, rank: %d, length: %lu\n", array, (unsigned long)objsize, vt->rank, (unsigned long)mono_array_length (array)));
1950 if (G_UNLIKELY (mono_profiler_events & MONO_PROFILE_GC_MOVES)) {
1951 /* FIXME: handle this for parallel collector */
1952 #ifdef SGEN_PARALLEL_MARK
1953 g_assert_not_reached ();
1955 if (moved_objects_idx == MOVED_OBJECTS_NUM) {
1956 mono_profiler_gc_moves (moved_objects, moved_objects_idx);
1957 moved_objects_idx = 0;
1959 moved_objects [moved_objects_idx++] = obj;
1960 moved_objects [moved_objects_idx++] = destination;
1964 DEBUG (9, fprintf (gc_debug_file, "Enqueuing gray object %p (%s)\n", obj, safe_name (obj)));
1965 GRAY_OBJECT_ENQUEUE (queue, obj);
1970 copy_object_no_checks (void *obj, GrayQueue *queue)
1972 MonoVTable *vt = ((MonoObject*)obj)->vtable;
1973 gboolean has_references = VTABLE_HAS_REFERENCES (vt);
1974 mword objsize = ALIGN_UP (par_object_get_size (vt, (MonoObject*)obj));
1977 MAJOR_GET_COPY_OBJECT_SPACE (destination, objsize, has_references);
1979 par_copy_object_no_checks (destination, vt, obj, objsize, has_references ? queue : NULL);
1981 /* set the forwarding pointer */
1982 forward_object (obj, destination);
1988 * This is how the copying happens from the nursery to the old generation.
1989 * We assume that at this time all the pinned objects have been identified and
1991 * We run scan_object() for each pinned object so that each referenced
1992 * objects if possible are copied. The new gray objects created can have
1993 * scan_object() run on them right away, too.
1994 * Then we run copy_object() for the precisely tracked roots. At this point
1995 * all the roots are either gray or black. We run scan_object() on the gray
1996 * objects until no more gray objects are created.
1997 * At the end of the process we walk again the pinned list and we unmark
1998 * the pinned flag. As we go we also create the list of free space for use
1999 * in the next allocation runs.
2001 * We need to remember objects from the old generation that point to the new one
2002 * (or just addresses?).
2004 * copy_object could be made into a macro once debugged (use inline for now).
2007 static void __attribute__((noinline))
2008 copy_object (void **obj_slot, GrayQueue *queue)
2011 char *obj = *obj_slot;
2013 DEBUG (9, g_assert (current_collection_generation == GENERATION_NURSERY));
2015 HEAVY_STAT (++stat_copy_object_called_nursery);
2017 if (!ptr_in_nursery (obj)) {
2018 HEAVY_STAT (++stat_nursery_copy_object_failed_from_space);
2022 DEBUG (9, fprintf (gc_debug_file, "Precise copy of %p from %p", obj, obj_slot));
2025 * Before we can copy the object we must make sure that we are
2026 * allowed to, i.e. that the object not pinned or not already
2030 if ((forwarded = object_is_forwarded (obj))) {
2031 DEBUG (9, g_assert (((MonoVTable*)LOAD_VTABLE(obj))->gc_descr));
2032 DEBUG (9, fprintf (gc_debug_file, " (already forwarded to %p)\n", forwarded));
2033 HEAVY_STAT (++stat_nursery_copy_object_failed_forwarded);
2034 *obj_slot = forwarded;
2037 if (object_is_pinned (obj)) {
2038 DEBUG (9, g_assert (((MonoVTable*)LOAD_VTABLE(obj))->gc_descr));
2039 DEBUG (9, fprintf (gc_debug_file, " (pinned, no change)\n"));
2040 HEAVY_STAT (++stat_nursery_copy_object_failed_pinned);
2044 HEAVY_STAT (++stat_objects_copied_nursery);
2046 *obj_slot = copy_object_no_checks (obj, queue);
2050 #define HANDLE_PTR(ptr,obj) do { \
2051 void *__old = *(ptr); \
2054 copy_object ((ptr), queue); \
2056 DEBUG (9, if (__old != __copy) fprintf (gc_debug_file, "Overwrote field at %p with %p (was: %p)\n", (ptr), *(ptr), __old)); \
2057 if (G_UNLIKELY (ptr_in_nursery (__copy) && !ptr_in_nursery ((ptr)))) \
2058 add_to_global_remset ((ptr)); \
2063 * Scan the object pointed to by @start for references to
2064 * other objects between @from_start and @from_end and copy
2065 * them to the gray_objects area.
2068 scan_object (char *start, GrayQueue *queue)
2070 #include "sgen-scan-object.h"
2072 HEAVY_STAT (++stat_scan_object_called_nursery);
2078 * Scan the valuetype pointed to by START, described by DESC for references to
2079 * other objects between @from_start and @from_end and copy them to the gray_objects area.
2080 * Returns a pointer to the end of the object.
2083 scan_vtype (char *start, mword desc, char* from_start, char* from_end, GrayQueue *queue)
2087 /* The descriptors include info about the MonoObject header as well */
2088 start -= sizeof (MonoObject);
2090 switch (desc & 0x7) {
2091 case DESC_TYPE_RUN_LENGTH:
2092 OBJ_RUN_LEN_FOREACH_PTR (desc,start);
2093 OBJ_RUN_LEN_SIZE (skip_size, desc, start);
2094 g_assert (skip_size);
2095 return start + skip_size;
2096 case DESC_TYPE_SMALL_BITMAP:
2097 OBJ_BITMAP_FOREACH_PTR (desc,start);
2098 OBJ_BITMAP_SIZE (skip_size, desc, start);
2099 return start + skip_size;
2100 case DESC_TYPE_LARGE_BITMAP:
2101 case DESC_TYPE_COMPLEX:
2103 g_assert_not_reached ();
2106 // The other descriptors can't happen with vtypes
2107 g_assert_not_reached ();
2114 #define HANDLE_PTR(ptr,obj) do { \
2115 void *__old = *(ptr); \
2118 major_copy_or_mark_object ((ptr), queue); \
2120 DEBUG (9, if (__old != __copy) fprintf (gc_debug_file, "Overwrote field at %p with %p (was: %p)\n", (ptr), *(ptr), __old)); \
2121 if (G_UNLIKELY (ptr_in_nursery (__copy) && !ptr_in_nursery ((ptr)))) \
2122 add_to_global_remset ((ptr)); \
2127 major_scan_object (char *start, GrayQueue *queue)
2129 #include "sgen-scan-object.h"
2131 HEAVY_STAT (++stat_scan_object_called_major);
2137 * Scan objects in the gray stack until the stack is empty. This should be called
2138 * frequently after each object is copied, to achieve better locality and cache
2142 drain_gray_stack (GrayQueue *queue)
2146 if (current_collection_generation == GENERATION_NURSERY) {
2148 GRAY_OBJECT_DEQUEUE (queue, obj);
2151 DEBUG (9, fprintf (gc_debug_file, "Precise gray object scan %p (%s)\n", obj, safe_name (obj)));
2152 scan_object (obj, queue);
2156 GRAY_OBJECT_DEQUEUE (queue, obj);
2159 DEBUG (9, fprintf (gc_debug_file, "Precise gray object scan %p (%s)\n", obj, safe_name (obj)));
2160 major_scan_object (obj, queue);
2166 * Addresses from start to end are already sorted. This function finds
2167 * the object header for each address and pins the object. The
2168 * addresses must be inside the passed section. The (start of the)
2169 * address array is overwritten with the addresses of the actually
2170 * pinned objects. Return the number of pinned objects.
2173 pin_objects_from_addresses (GCMemSection *section, void **start, void **end, void *start_nursery, void *end_nursery, GrayQueue *queue)
2178 void *last_obj = NULL;
2179 size_t last_obj_size = 0;
2182 void **definitely_pinned = start;
2183 while (start < end) {
2185 /* the range check should be reduntant */
2186 if (addr != last && addr >= start_nursery && addr < end_nursery) {
2187 DEBUG (5, fprintf (gc_debug_file, "Considering pinning addr %p\n", addr));
2188 /* multiple pointers to the same object */
2189 if (addr >= last_obj && (char*)addr < (char*)last_obj + last_obj_size) {
2193 idx = ((char*)addr - (char*)section->data) / SCAN_START_SIZE;
2194 g_assert (idx < section->num_scan_start);
2195 search_start = (void*)section->scan_starts [idx];
2196 if (!search_start || search_start > addr) {
2199 search_start = section->scan_starts [idx];
2200 if (search_start && search_start <= addr)
2203 if (!search_start || search_start > addr)
2204 search_start = start_nursery;
2206 if (search_start < last_obj)
2207 search_start = (char*)last_obj + last_obj_size;
2208 /* now addr should be in an object a short distance from search_start
2209 * Note that search_start must point to zeroed mem or point to an object.
2212 if (!*(void**)search_start) {
2213 search_start = (void*)ALIGN_UP ((mword)search_start + sizeof (gpointer));
2216 last_obj = search_start;
2217 last_obj_size = ALIGN_UP (safe_object_get_size ((MonoObject*)search_start));
2218 DEBUG (8, fprintf (gc_debug_file, "Pinned try match %p (%s), size %zd\n", last_obj, safe_name (last_obj), last_obj_size));
2219 if (addr >= search_start && (char*)addr < (char*)last_obj + last_obj_size) {
2220 DEBUG (4, fprintf (gc_debug_file, "Pinned object %p, vtable %p (%s), count %d\n", search_start, *(void**)search_start, safe_name (search_start), count));
2221 binary_protocol_pin (search_start, (gpointer)LOAD_VTABLE (search_start), safe_object_get_size (search_start));
2222 pin_object (search_start);
2223 GRAY_OBJECT_ENQUEUE (queue, search_start);
2225 pin_stats_register_object (search_start, last_obj_size);
2226 definitely_pinned [count] = search_start;
2230 /* skip to the next object */
2231 search_start = (void*)((char*)search_start + last_obj_size);
2232 } while (search_start <= addr);
2233 /* we either pinned the correct object or we ignored the addr because
2234 * it points to unused zeroed memory.
2240 //printf ("effective pinned: %d (at the end: %d)\n", count, (char*)end_nursery - (char*)last);
2245 pin_objects_in_section (GCMemSection *section, GrayQueue *queue)
2247 int start = section->pin_queue_start;
2248 int end = section->pin_queue_end;
2251 reduced_to = pin_objects_from_addresses (section, pin_queue + start, pin_queue + end,
2252 section->data, section->next_data, queue);
2253 section->pin_queue_start = start;
2254 section->pin_queue_end = start + reduced_to;
2258 /* Sort the addresses in array in increasing order.
2259 * Done using a by-the book heap sort. Which has decent and stable performance, is pretty cache efficient.
2262 sort_addresses (void **array, int size)
2267 for (i = 1; i < size; ++i) {
2270 int parent = (child - 1) / 2;
2272 if (array [parent] >= array [child])
2275 tmp = array [parent];
2276 array [parent] = array [child];
2277 array [child] = tmp;
2283 for (i = size - 1; i > 0; --i) {
2286 array [i] = array [0];
2292 while (root * 2 + 1 <= end) {
2293 int child = root * 2 + 1;
2295 if (child < end && array [child] < array [child + 1])
2297 if (array [root] >= array [child])
2301 array [root] = array [child];
2302 array [child] = tmp;
2309 static G_GNUC_UNUSED void
2310 print_nursery_gaps (void* start_nursery, void *end_nursery)
2313 gpointer first = start_nursery;
2315 for (i = 0; i < next_pin_slot; ++i) {
2316 next = pin_queue [i];
2317 fprintf (gc_debug_file, "Nursery range: %p-%p, size: %td\n", first, next, (char*)next-(char*)first);
2321 fprintf (gc_debug_file, "Nursery range: %p-%p, size: %td\n", first, next, (char*)next-(char*)first);
2324 /* reduce the info in the pin queue, removing duplicate pointers and sorting them */
2326 optimize_pin_queue (int start_slot)
2328 void **start, **cur, **end;
2329 /* sort and uniq pin_queue: we just sort and we let the rest discard multiple values */
2330 /* it may be better to keep ranges of pinned memory instead of individually pinning objects */
2331 DEBUG (5, fprintf (gc_debug_file, "Sorting pin queue, size: %d\n", next_pin_slot));
2332 if ((next_pin_slot - start_slot) > 1)
2333 sort_addresses (pin_queue + start_slot, next_pin_slot - start_slot);
2334 start = cur = pin_queue + start_slot;
2335 end = pin_queue + next_pin_slot;
2338 while (*start == *cur && cur < end)
2342 next_pin_slot = start - pin_queue;
2343 DEBUG (5, fprintf (gc_debug_file, "Pin queue reduced to size: %d\n", next_pin_slot));
2344 //DEBUG (6, print_nursery_gaps (start_nursery, end_nursery));
2349 * Scan the memory between start and end and queue values which could be pointers
2350 * to the area between start_nursery and end_nursery for later consideration.
2351 * Typically used for thread stacks.
2354 conservatively_pin_objects_from (void **start, void **end, void *start_nursery, void *end_nursery, int pin_type)
2357 while (start < end) {
2358 if (*start >= start_nursery && *start < end_nursery) {
2360 * *start can point to the middle of an object
2361 * note: should we handle pointing at the end of an object?
2362 * pinning in C# code disallows pointing at the end of an object
2363 * but there is some small chance that an optimizing C compiler
2364 * may keep the only reference to an object by pointing
2365 * at the end of it. We ignore this small chance for now.
2366 * Pointers to the end of an object are indistinguishable
2367 * from pointers to the start of the next object in memory
2368 * so if we allow that we'd need to pin two objects...
2369 * We queue the pointer in an array, the
2370 * array will then be sorted and uniqued. This way
2371 * we can coalesce several pinning pointers and it should
2372 * be faster since we'd do a memory scan with increasing
2373 * addresses. Note: we can align the address to the allocation
2374 * alignment, so the unique process is more effective.
2376 mword addr = (mword)*start;
2377 addr &= ~(ALLOC_ALIGN - 1);
2378 if (addr >= (mword)start_nursery && addr < (mword)end_nursery)
2379 pin_stage_ptr ((void*)addr);
2381 pin_stats_register_address ((char*)addr, pin_type);
2382 DEBUG (6, if (count) fprintf (gc_debug_file, "Pinning address %p\n", (void*)addr));
2387 DEBUG (7, if (count) fprintf (gc_debug_file, "found %d potential pinned heap pointers\n", count));
2391 * Debugging function: find in the conservative roots where @obj is being pinned.
2393 static G_GNUC_UNUSED void
2394 find_pinning_reference (char *obj, size_t size)
2398 char *endobj = obj + size;
2399 for (i = 0; i < roots_hash_size [0]; ++i) {
2400 for (root = roots_hash [0][i]; root; root = root->next) {
2401 /* if desc is non-null it has precise info */
2402 if (!root->root_desc) {
2403 char ** start = (char**)root->start_root;
2404 while (start < (char**)root->end_root) {
2405 if (*start >= obj && *start < endobj) {
2406 DEBUG (0, fprintf (gc_debug_file, "Object %p referenced in pinned roots %p-%p (at %p in record %p)\n", obj, root->start_root, root->end_root, start, root));
2413 find_pinning_ref_from_thread (obj, size);
2417 * The first thing we do in a collection is to identify pinned objects.
2418 * This function considers all the areas of memory that need to be
2419 * conservatively scanned.
2422 pin_from_roots (void *start_nursery, void *end_nursery)
2426 DEBUG (2, fprintf (gc_debug_file, "Scanning pinned roots (%d bytes, %d/%d entries)\n", (int)roots_size, num_roots_entries [ROOT_TYPE_NORMAL], num_roots_entries [ROOT_TYPE_PINNED]));
2427 /* objects pinned from the API are inside these roots */
2428 for (i = 0; i < roots_hash_size [ROOT_TYPE_PINNED]; ++i) {
2429 for (root = roots_hash [ROOT_TYPE_PINNED][i]; root; root = root->next) {
2430 DEBUG (6, fprintf (gc_debug_file, "Pinned roots %p-%p\n", root->start_root, root->end_root));
2431 conservatively_pin_objects_from ((void**)root->start_root, (void**)root->end_root, start_nursery, end_nursery, PIN_TYPE_OTHER);
2434 /* now deal with the thread stacks
2435 * in the future we should be able to conservatively scan only:
2436 * *) the cpu registers
2437 * *) the unmanaged stack frames
2438 * *) the _last_ managed stack frame
2439 * *) pointers slots in managed frames
2441 scan_thread_data (start_nursery, end_nursery, FALSE);
2443 evacuate_pin_staging_area ();
2446 static CopyOrMarkObjectFunc user_copy_or_mark_func;
2447 static GrayQueue *user_copy_or_mark_queue;
2450 single_arg_user_copy_or_mark (void **obj)
2452 user_copy_or_mark_func (obj, user_copy_or_mark_queue);
2456 * The memory area from start_root to end_root contains pointers to objects.
2457 * Their position is precisely described by @desc (this means that the pointer
2458 * can be either NULL or the pointer to the start of an object).
2459 * This functions copies them to to_space updates them.
2461 * This function is not thread-safe!
2464 precisely_scan_objects_from (CopyOrMarkObjectFunc copy_func, void** start_root, void** end_root, char* n_start, char *n_end, mword desc, GrayQueue *queue)
2466 switch (desc & ROOT_DESC_TYPE_MASK) {
2467 case ROOT_DESC_BITMAP:
2468 desc >>= ROOT_DESC_TYPE_SHIFT;
2470 if ((desc & 1) && *start_root) {
2471 copy_func (start_root, queue);
2472 DEBUG (9, fprintf (gc_debug_file, "Overwrote root at %p with %p\n", start_root, *start_root));
2473 drain_gray_stack (queue);
2479 case ROOT_DESC_COMPLEX: {
2480 gsize *bitmap_data = complex_descriptors + (desc >> ROOT_DESC_TYPE_SHIFT);
2481 int bwords = (*bitmap_data) - 1;
2482 void **start_run = start_root;
2484 while (bwords-- > 0) {
2485 gsize bmap = *bitmap_data++;
2486 void **objptr = start_run;
2488 if ((bmap & 1) && *objptr) {
2489 copy_func (objptr, queue);
2490 DEBUG (9, fprintf (gc_debug_file, "Overwrote root at %p with %p\n", objptr, *objptr));
2491 drain_gray_stack (queue);
2496 start_run += GC_BITS_PER_WORD;
2500 case ROOT_DESC_USER: {
2501 MonoGCRootMarkFunc marker = user_descriptors [desc >> ROOT_DESC_TYPE_SHIFT];
2502 user_copy_or_mark_func = copy_func;
2503 user_copy_or_mark_queue = queue;
2504 marker (start_root, single_arg_user_copy_or_mark);
2505 user_copy_or_mark_func = NULL;
2506 user_copy_or_mark_queue = NULL;
2509 case ROOT_DESC_RUN_LEN:
2510 g_assert_not_reached ();
2512 g_assert_not_reached ();
2517 mono_sgen_update_heap_boundaries (mword low, mword high)
2522 old = lowest_heap_address;
2525 } while (SGEN_CAS_PTR ((gpointer*)&lowest_heap_address, (gpointer)low, (gpointer)old) != (gpointer)old);
2528 old = highest_heap_address;
2531 } while (SGEN_CAS_PTR ((gpointer*)&highest_heap_address, (gpointer)high, (gpointer)old) != (gpointer)old);
2535 alloc_fragment (void)
2537 Fragment *frag = fragment_freelist;
2539 fragment_freelist = frag->next;
2543 frag = mono_sgen_alloc_internal (INTERNAL_MEM_FRAGMENT);
2548 /* size must be a power of 2 */
2550 mono_sgen_alloc_os_memory_aligned (mword size, mword alignment, gboolean activate)
2552 /* Allocate twice the memory to be able to put the block on an aligned address */
2553 char *mem = mono_sgen_alloc_os_memory (size + alignment, activate);
2558 aligned = (char*)((mword)(mem + (alignment - 1)) & ~(alignment - 1));
2559 g_assert (aligned >= mem && aligned + size <= mem + size + alignment && !((mword)aligned & (alignment - 1)));
2562 mono_sgen_free_os_memory (mem, aligned - mem);
2563 if (aligned + size < mem + size + alignment)
2564 mono_sgen_free_os_memory (aligned + size, (mem + size + alignment) - (aligned + size));
2570 * Allocate and setup the data structures needed to be able to allocate objects
2571 * in the nursery. The nursery is stored in nursery_section.
2574 alloc_nursery (void)
2576 GCMemSection *section;
2582 if (nursery_section)
2584 DEBUG (2, fprintf (gc_debug_file, "Allocating nursery size: %lu\n", (unsigned long)nursery_size));
2585 /* later we will alloc a larger area for the nursery but only activate
2586 * what we need. The rest will be used as expansion if we have too many pinned
2587 * objects in the existing nursery.
2589 /* FIXME: handle OOM */
2590 section = mono_sgen_alloc_internal (INTERNAL_MEM_SECTION);
2592 g_assert (nursery_size == DEFAULT_NURSERY_SIZE);
2593 alloc_size = nursery_size;
2594 #ifdef ALIGN_NURSERY
2595 data = mono_sgen_alloc_os_memory_aligned (alloc_size, alloc_size, TRUE);
2597 data = mono_sgen_alloc_os_memory (alloc_size, TRUE);
2599 nursery_start = data;
2600 nursery_real_end = nursery_start + nursery_size;
2601 mono_sgen_update_heap_boundaries ((mword)nursery_start, (mword)nursery_real_end);
2602 nursery_next = nursery_start;
2603 DEBUG (4, fprintf (gc_debug_file, "Expanding nursery size (%p-%p): %lu, total: %lu\n", data, data + alloc_size, (unsigned long)nursery_size, (unsigned long)total_alloc));
2604 section->data = section->next_data = data;
2605 section->size = alloc_size;
2606 section->end_data = nursery_real_end;
2607 scan_starts = (alloc_size + SCAN_START_SIZE - 1) / SCAN_START_SIZE;
2608 section->scan_starts = mono_sgen_alloc_internal_dynamic (sizeof (char*) * scan_starts, INTERNAL_MEM_SCAN_STARTS);
2609 section->num_scan_start = scan_starts;
2610 section->block.role = MEMORY_ROLE_GEN0;
2611 section->block.next = NULL;
2613 nursery_section = section;
2615 /* Setup the single first large fragment */
2616 frag = alloc_fragment ();
2617 frag->fragment_start = nursery_start;
2618 frag->fragment_limit = nursery_start;
2619 frag->fragment_end = nursery_real_end;
2620 nursery_frag_real_end = nursery_real_end;
2621 /* FIXME: frag here is lost */
2625 scan_finalizer_entries (CopyOrMarkObjectFunc copy_func, FinalizeEntry *list, GrayQueue *queue)
2629 for (fin = list; fin; fin = fin->next) {
2632 DEBUG (5, fprintf (gc_debug_file, "Scan of fin ready object: %p (%s)\n", fin->object, safe_name (fin->object)));
2633 copy_func (&fin->object, queue);
2637 static mword fragment_total = 0;
2639 * We found a fragment of free memory in the nursery: memzero it and if
2640 * it is big enough, add it to the list of fragments that can be used for
2644 add_nursery_frag (size_t frag_size, char* frag_start, char* frag_end)
2647 DEBUG (4, fprintf (gc_debug_file, "Found empty fragment: %p-%p, size: %zd\n", frag_start, frag_end, frag_size));
2648 binary_protocol_empty (frag_start, frag_size);
2649 /* memsetting just the first chunk start is bound to provide better cache locality */
2650 if (nursery_clear_policy == CLEAR_AT_GC)
2651 memset (frag_start, 0, frag_size);
2652 /* Not worth dealing with smaller fragments: need to tune */
2653 if (frag_size >= FRAGMENT_MIN_SIZE) {
2654 fragment = alloc_fragment ();
2655 fragment->fragment_start = frag_start;
2656 fragment->fragment_limit = frag_start;
2657 fragment->fragment_end = frag_end;
2658 fragment->next = nursery_fragments;
2659 nursery_fragments = fragment;
2660 fragment_total += frag_size;
2662 /* Clear unused fragments, pinning depends on this */
2663 /*TODO place an int[] here instead of the memset if size justify it*/
2664 memset (frag_start, 0, frag_size);
2669 generation_name (int generation)
2671 switch (generation) {
2672 case GENERATION_NURSERY: return "nursery";
2673 case GENERATION_OLD: return "old";
2674 default: g_assert_not_reached ();
2678 static DisappearingLinkHashTable*
2679 get_dislink_hash_table (int generation)
2681 switch (generation) {
2682 case GENERATION_NURSERY: return &minor_disappearing_link_hash;
2683 case GENERATION_OLD: return &major_disappearing_link_hash;
2684 default: g_assert_not_reached ();
2688 static FinalizeEntryHashTable*
2689 get_finalize_entry_hash_table (int generation)
2691 switch (generation) {
2692 case GENERATION_NURSERY: return &minor_finalizable_hash;
2693 case GENERATION_OLD: return &major_finalizable_hash;
2694 default: g_assert_not_reached ();
2699 finish_gray_stack (char *start_addr, char *end_addr, int generation, GrayQueue *queue)
2704 int ephemeron_rounds = 0;
2705 CopyOrMarkObjectFunc copy_func = current_collection_generation == GENERATION_NURSERY ? copy_object : major_copy_or_mark_object;
2708 * We copied all the reachable objects. Now it's the time to copy
2709 * the objects that were not referenced by the roots, but by the copied objects.
2710 * we built a stack of objects pointed to by gray_start: they are
2711 * additional roots and we may add more items as we go.
2712 * We loop until gray_start == gray_objects which means no more objects have
2713 * been added. Note this is iterative: no recursion is involved.
2714 * We need to walk the LO list as well in search of marked big objects
2715 * (use a flag since this is needed only on major collections). We need to loop
2716 * here as well, so keep a counter of marked LO (increasing it in copy_object).
2717 * To achieve better cache locality and cache usage, we drain the gray stack
2718 * frequently, after each object is copied, and just finish the work here.
2720 drain_gray_stack (queue);
2722 DEBUG (2, fprintf (gc_debug_file, "%s generation done\n", generation_name (generation)));
2723 /* walk the finalization queue and move also the objects that need to be
2724 * finalized: use the finalized objects as new roots so the objects they depend
2725 * on are also not reclaimed. As with the roots above, only objects in the nursery
2726 * are marked/copied.
2727 * We need a loop here, since objects ready for finalizers may reference other objects
2728 * that are fin-ready. Speedup with a flag?
2732 * Walk the ephemeron tables marking all values with reachable keys. This must be completely done
2733 * before processing finalizable objects to avoid finalizing reachable values.
2735 * It must be done inside the finalizaters loop since objects must not be removed from CWT tables
2736 * while they are been finalized.
2738 int done_with_ephemerons = 0;
2740 done_with_ephemerons = mark_ephemerons_in_range (copy_func, start_addr, end_addr, queue);
2741 drain_gray_stack (queue);
2743 } while (!done_with_ephemerons);
2745 fin_ready = num_ready_finalizers;
2746 finalize_in_range (copy_func, start_addr, end_addr, generation, queue);
2747 if (generation == GENERATION_OLD)
2748 finalize_in_range (copy_func, nursery_start, nursery_real_end, GENERATION_NURSERY, queue);
2750 /* drain the new stack that might have been created */
2751 DEBUG (6, fprintf (gc_debug_file, "Precise scan of gray area post fin\n"));
2752 drain_gray_stack (queue);
2753 } while (fin_ready != num_ready_finalizers);
2756 * Clear ephemeron pairs with unreachable keys.
2757 * We pass the copy func so we can figure out if an array was promoted or not.
2759 clear_unreachable_ephemerons (copy_func, start_addr, end_addr, queue);
2762 DEBUG (2, fprintf (gc_debug_file, "Finalize queue handling scan for %s generation: %d usecs %d ephemeron roundss\n", generation_name (generation), TV_ELAPSED (atv, btv), ephemeron_rounds));
2765 * handle disappearing links
2766 * Note we do this after checking the finalization queue because if an object
2767 * survives (at least long enough to be finalized) we don't clear the link.
2768 * This also deals with a possible issue with the monitor reclamation: with the Boehm
2769 * GC a finalized object my lose the monitor because it is cleared before the finalizer is
2772 g_assert (gray_object_queue_is_empty (queue));
2774 null_link_in_range (copy_func, start_addr, end_addr, generation, queue);
2775 if (generation == GENERATION_OLD)
2776 null_link_in_range (copy_func, start_addr, end_addr, GENERATION_NURSERY, queue);
2777 if (gray_object_queue_is_empty (queue))
2779 drain_gray_stack (queue);
2782 g_assert (gray_object_queue_is_empty (queue));
2786 check_section_scan_starts (GCMemSection *section)
2789 for (i = 0; i < section->num_scan_start; ++i) {
2790 if (section->scan_starts [i]) {
2791 guint size = safe_object_get_size ((MonoObject*) section->scan_starts [i]);
2792 g_assert (size >= sizeof (MonoObject) && size <= MAX_SMALL_OBJ_SIZE);
2798 check_scan_starts (void)
2800 if (!do_scan_starts_check)
2802 check_section_scan_starts (nursery_section);
2803 major_check_scan_starts ();
2806 static int last_num_pinned = 0;
2809 build_nursery_fragments (int start_pin, int end_pin)
2811 char *frag_start, *frag_end;
2815 while (nursery_fragments) {
2816 Fragment *next = nursery_fragments->next;
2817 nursery_fragments->next = fragment_freelist;
2818 fragment_freelist = nursery_fragments;
2819 nursery_fragments = next;
2821 frag_start = nursery_start;
2823 /* clear scan starts */
2824 memset (nursery_section->scan_starts, 0, nursery_section->num_scan_start * sizeof (gpointer));
2825 for (i = start_pin; i < end_pin; ++i) {
2826 frag_end = pin_queue [i];
2827 /* remove the pin bit from pinned objects */
2828 unpin_object (frag_end);
2829 nursery_section->scan_starts [((char*)frag_end - (char*)nursery_section->data)/SCAN_START_SIZE] = frag_end;
2830 frag_size = frag_end - frag_start;
2832 add_nursery_frag (frag_size, frag_start, frag_end);
2833 frag_size = ALIGN_UP (safe_object_get_size ((MonoObject*)pin_queue [i]));
2834 frag_start = (char*)pin_queue [i] + frag_size;
2836 nursery_last_pinned_end = frag_start;
2837 frag_end = nursery_real_end;
2838 frag_size = frag_end - frag_start;
2840 add_nursery_frag (frag_size, frag_start, frag_end);
2841 if (!nursery_fragments) {
2842 DEBUG (1, fprintf (gc_debug_file, "Nursery fully pinned (%d)\n", end_pin - start_pin));
2843 for (i = start_pin; i < end_pin; ++i) {
2844 DEBUG (3, fprintf (gc_debug_file, "Bastard pinning obj %p (%s), size: %d\n", pin_queue [i], safe_name (pin_queue [i]), safe_object_get_size (pin_queue [i])));
2849 nursery_next = nursery_frag_real_end = NULL;
2851 /* Clear TLABs for all threads */
2856 scan_from_registered_roots (CopyOrMarkObjectFunc copy_func, char *addr_start, char *addr_end, int root_type, GrayQueue *queue)
2860 for (i = 0; i < roots_hash_size [root_type]; ++i) {
2861 for (root = roots_hash [root_type][i]; root; root = root->next) {
2862 DEBUG (6, fprintf (gc_debug_file, "Precise root scan %p-%p (desc: %p)\n", root->start_root, root->end_root, (void*)root->root_desc));
2863 precisely_scan_objects_from (copy_func, (void**)root->start_root, (void**)root->end_root, addr_start, addr_end, root->root_desc, queue);
2869 dump_occupied (char *start, char *end, char *section_start)
2871 fprintf (heap_dump_file, "<occupied offset=\"%td\" size=\"%td\"/>\n", start - section_start, end - start);
2875 dump_section (GCMemSection *section, const char *type)
2877 char *start = section->data;
2878 char *end = section->data + section->size;
2879 char *occ_start = NULL;
2881 char *old_start = NULL; /* just for debugging */
2883 fprintf (heap_dump_file, "<section type=\"%s\" size=\"%lu\">\n", type, (unsigned long)section->size);
2885 while (start < end) {
2889 if (!*(void**)start) {
2891 dump_occupied (occ_start, start, section->data);
2894 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
2897 g_assert (start < section->next_data);
2902 vt = (GCVTable*)LOAD_VTABLE (start);
2905 size = ALIGN_UP (safe_object_get_size ((MonoObject*) start));
2908 fprintf (heap_dump_file, "<object offset=\"%d\" class=\"%s.%s\" size=\"%d\"/>\n",
2909 start - section->data,
2910 vt->klass->name_space, vt->klass->name,
2918 dump_occupied (occ_start, start, section->data);
2920 fprintf (heap_dump_file, "</section>\n");
2924 dump_object (MonoObject *obj, gboolean dump_location)
2926 static char class_name [1024];
2928 MonoClass *class = mono_object_class (obj);
2932 * Python's XML parser is too stupid to parse angle brackets
2933 * in strings, so we just ignore them;
2936 while (class->name [i] && j < sizeof (class_name) - 1) {
2937 if (!strchr ("<>\"", class->name [i]))
2938 class_name [j++] = class->name [i];
2941 g_assert (j < sizeof (class_name));
2944 fprintf (heap_dump_file, "<object class=\"%s.%s\" size=\"%d\"",
2945 class->name_space, class_name,
2946 safe_object_get_size (obj));
2947 if (dump_location) {
2948 const char *location;
2949 if (ptr_in_nursery (obj))
2950 location = "nursery";
2951 else if (safe_object_get_size (obj) <= MAX_SMALL_OBJ_SIZE)
2955 fprintf (heap_dump_file, " location=\"%s\"", location);
2957 fprintf (heap_dump_file, "/>\n");
2961 dump_heap (const char *type, int num, const char *reason)
2966 fprintf (heap_dump_file, "<collection type=\"%s\" num=\"%d\"", type, num);
2968 fprintf (heap_dump_file, " reason=\"%s\"", reason);
2969 fprintf (heap_dump_file, ">\n");
2970 fprintf (heap_dump_file, "<other-mem-usage type=\"mempools\" size=\"%ld\"/>\n", mono_mempool_get_bytes_allocated ());
2971 mono_sgen_dump_internal_mem_usage (heap_dump_file);
2972 fprintf (heap_dump_file, "<pinned type=\"stack\" bytes=\"%zu\"/>\n", pinned_byte_counts [PIN_TYPE_STACK]);
2973 /* fprintf (heap_dump_file, "<pinned type=\"static-data\" bytes=\"%d\"/>\n", pinned_byte_counts [PIN_TYPE_STATIC_DATA]); */
2974 fprintf (heap_dump_file, "<pinned type=\"other\" bytes=\"%zu\"/>\n", pinned_byte_counts [PIN_TYPE_OTHER]);
2976 fprintf (heap_dump_file, "<pinned-objects>\n");
2977 for (list = pinned_objects; list; list = list->next)
2978 dump_object (list->obj, TRUE);
2979 fprintf (heap_dump_file, "</pinned-objects>\n");
2981 dump_section (nursery_section, "nursery");
2985 fprintf (heap_dump_file, "<los>\n");
2986 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
2987 dump_object ((MonoObject*)bigobj->data, FALSE);
2988 fprintf (heap_dump_file, "</los>\n");
2990 fprintf (heap_dump_file, "</collection>\n");
2996 static gboolean inited = FALSE;
3001 mono_counters_register ("Minor fragment clear", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_pre_collection_fragment_clear);
3002 mono_counters_register ("Minor pinning", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_pinning);
3003 mono_counters_register ("Minor scan remsets", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_scan_remsets);
3004 mono_counters_register ("Minor scan pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_scan_pinned);
3005 mono_counters_register ("Minor scan registered roots", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_scan_registered_roots);
3006 mono_counters_register ("Minor scan thread data", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_scan_thread_data);
3007 mono_counters_register ("Minor finish gray stack", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_finish_gray_stack);
3008 mono_counters_register ("Minor fragment creation", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_fragment_creation);
3010 mono_counters_register ("Major fragment clear", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_pre_collection_fragment_clear);
3011 mono_counters_register ("Major pinning", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_pinning);
3012 mono_counters_register ("Major scan pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_pinned);
3013 mono_counters_register ("Major scan registered roots", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_registered_roots);
3014 mono_counters_register ("Major scan thread data", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_thread_data);
3015 mono_counters_register ("Major scan alloc_pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_alloc_pinned);
3016 mono_counters_register ("Major scan finalized", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_finalized);
3017 mono_counters_register ("Major scan big objects", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_big_objects);
3018 mono_counters_register ("Major finish gray stack", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_finish_gray_stack);
3019 mono_counters_register ("Major free big objects", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_free_bigobjs);
3020 mono_counters_register ("Major LOS sweep", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_los_sweep);
3021 mono_counters_register ("Major sweep", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_sweep);
3022 mono_counters_register ("Major fragment creation", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_fragment_creation);
3024 #ifdef HEAVY_STATISTICS
3025 mono_counters_register ("WBarrier set field", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_field);
3026 mono_counters_register ("WBarrier set arrayref", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_arrayref);
3027 mono_counters_register ("WBarrier arrayref copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_arrayref_copy);
3028 mono_counters_register ("WBarrier generic store called", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_generic_store);
3029 mono_counters_register ("WBarrier generic store stored", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_generic_store_remset);
3030 mono_counters_register ("WBarrier set root", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_root);
3031 mono_counters_register ("WBarrier value copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_value_copy);
3032 mono_counters_register ("WBarrier object copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_object_copy);
3034 mono_counters_register ("# objects allocated", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_alloced);
3035 mono_counters_register ("bytes allocated", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_bytes_alloced);
3036 mono_counters_register ("# objects allocated degraded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_alloced_degraded);
3037 mono_counters_register ("bytes allocated degraded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_bytes_alloced_degraded);
3038 mono_counters_register ("bytes allocated in LOS", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_bytes_alloced_los);
3040 mono_counters_register ("# copy_object() called (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_called_nursery);
3041 mono_counters_register ("# objects copied (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_copied_nursery);
3042 mono_counters_register ("# copy_object() called (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_called_major);
3043 mono_counters_register ("# objects copied (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_copied_major);
3045 mono_counters_register ("# scan_object() called (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_scan_object_called_nursery);
3046 mono_counters_register ("# scan_object() called (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_scan_object_called_major);
3048 mono_counters_register ("# nursery copy_object() failed from space", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_from_space);
3049 mono_counters_register ("# nursery copy_object() failed forwarded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_forwarded);
3050 mono_counters_register ("# nursery copy_object() failed pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_pinned);
3052 mono_counters_register ("# wasted fragments used", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_wasted_fragments_used);
3053 mono_counters_register ("bytes in wasted fragments", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_wasted_fragments_bytes);
3055 mono_counters_register ("Store remsets", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_store_remsets);
3056 mono_counters_register ("Unique store remsets", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_store_remsets_unique);
3057 mono_counters_register ("Saved remsets 1", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_saved_remsets_1);
3058 mono_counters_register ("Saved remsets 2", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_saved_remsets_2);
3059 mono_counters_register ("Global remsets added", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_added);
3060 mono_counters_register ("Global remsets re-added", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_readded);
3061 mono_counters_register ("Global remsets processed", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_processed);
3062 mono_counters_register ("Global remsets discarded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_discarded);
3069 need_major_collection (void)
3071 mword los_alloced = los_memory_usage - MIN (last_los_memory_usage, los_memory_usage);
3072 return minor_collection_sections_alloced * MAJOR_SECTION_SIZE + los_alloced > minor_collection_allowance;
3076 * Collect objects in the nursery. Returns whether to trigger a major
3080 collect_nursery (size_t requested_size)
3082 size_t max_garbage_amount;
3083 char *orig_nursery_next;
3084 TV_DECLARE (all_atv);
3085 TV_DECLARE (all_btv);
3089 current_collection_generation = GENERATION_NURSERY;
3091 binary_protocol_collection (GENERATION_NURSERY);
3092 check_scan_starts ();
3095 orig_nursery_next = nursery_next;
3096 nursery_next = MAX (nursery_next, nursery_last_pinned_end);
3097 /* FIXME: optimize later to use the higher address where an object can be present */
3098 nursery_next = MAX (nursery_next, nursery_real_end);
3100 DEBUG (1, fprintf (gc_debug_file, "Start nursery collection %d %p-%p, size: %d\n", num_minor_gcs, nursery_start, nursery_next, (int)(nursery_next - nursery_start)));
3101 max_garbage_amount = nursery_next - nursery_start;
3102 g_assert (nursery_section->size >= max_garbage_amount);
3104 /* world must be stopped already */
3105 TV_GETTIME (all_atv);
3108 /* Pinning depends on this */
3109 clear_nursery_fragments (orig_nursery_next);
3112 time_minor_pre_collection_fragment_clear += TV_ELAPSED_MS (atv, btv);
3115 check_for_xdomain_refs ();
3117 nursery_section->next_data = nursery_next;
3119 major_start_nursery_collection ();
3121 gray_object_queue_init (&gray_queue);
3124 mono_stats.minor_gc_count ++;
3126 global_remset_cache_clear ();
3128 /* pin from pinned handles */
3130 pin_from_roots (nursery_start, nursery_next);
3131 /* identify pinned objects */
3132 optimize_pin_queue (0);
3133 next_pin_slot = pin_objects_from_addresses (nursery_section, pin_queue, pin_queue + next_pin_slot, nursery_start, nursery_next, &gray_queue);
3134 nursery_section->pin_queue_start = 0;
3135 nursery_section->pin_queue_end = next_pin_slot;
3137 time_minor_pinning += TV_ELAPSED_MS (btv, atv);
3138 DEBUG (2, fprintf (gc_debug_file, "Finding pinned pointers: %d in %d usecs\n", next_pin_slot, TV_ELAPSED (btv, atv)));
3139 DEBUG (4, fprintf (gc_debug_file, "Start scan with %d pinned objects\n", next_pin_slot));
3141 if (consistency_check_at_minor_collection)
3142 check_consistency ();
3145 * walk all the roots and copy the young objects to the old generation,
3146 * starting from to_space
3149 scan_from_remsets (nursery_start, nursery_next, &gray_queue);
3150 /* we don't have complete write barrier yet, so we scan all the old generation sections */
3152 time_minor_scan_remsets += TV_ELAPSED_MS (atv, btv);
3153 DEBUG (2, fprintf (gc_debug_file, "Old generation scan: %d usecs\n", TV_ELAPSED (atv, btv)));
3155 drain_gray_stack (&gray_queue);
3158 time_minor_scan_pinned += TV_ELAPSED_MS (btv, atv);
3159 /* registered roots, this includes static fields */
3160 scan_from_registered_roots (copy_object, nursery_start, nursery_next, ROOT_TYPE_NORMAL, &gray_queue);
3161 scan_from_registered_roots (copy_object, nursery_start, nursery_next, ROOT_TYPE_WBARRIER, &gray_queue);
3163 time_minor_scan_registered_roots += TV_ELAPSED_MS (atv, btv);
3165 scan_thread_data (nursery_start, nursery_next, TRUE);
3167 time_minor_scan_thread_data += TV_ELAPSED_MS (btv, atv);
3170 finish_gray_stack (nursery_start, nursery_next, GENERATION_NURSERY, &gray_queue);
3172 time_minor_finish_gray_stack += TV_ELAPSED_MS (btv, atv);
3174 /* walk the pin_queue, build up the fragment list of free memory, unmark
3175 * pinned objects as we go, memzero() the empty fragments so they are ready for the
3178 build_nursery_fragments (0, next_pin_slot);
3180 time_minor_fragment_creation += TV_ELAPSED_MS (atv, btv);
3181 DEBUG (2, fprintf (gc_debug_file, "Fragment creation: %d usecs, %lu bytes available\n", TV_ELAPSED (atv, btv), (unsigned long)fragment_total));
3183 if (consistency_check_at_minor_collection)
3184 check_major_refs ();
3186 major_finish_nursery_collection ();
3188 TV_GETTIME (all_btv);
3189 mono_stats.minor_gc_time_usecs += TV_ELAPSED (all_atv, all_btv);
3192 dump_heap ("minor", num_minor_gcs - 1, NULL);
3194 /* prepare the pin queue for the next collection */
3195 last_num_pinned = next_pin_slot;
3197 if (fin_ready_list || critical_fin_list) {
3198 DEBUG (4, fprintf (gc_debug_file, "Finalizer-thread wakeup: ready %d\n", num_ready_finalizers));
3199 mono_gc_finalize_notify ();
3203 g_assert (gray_object_queue_is_empty (&gray_queue));
3205 check_scan_starts ();
3207 binary_protocol_flush_buffers ();
3209 current_collection_generation = -1;
3211 return need_major_collection ();
3215 major_do_collection (const char *reason)
3217 LOSObject *bigobj, *prevbo;
3218 TV_DECLARE (all_atv);
3219 TV_DECLARE (all_btv);
3222 /* FIXME: only use these values for the precise scan
3223 * note that to_space pointers should be excluded anyway...
3225 char *heap_start = NULL;
3226 char *heap_end = (char*)-1;
3227 int old_num_major_sections = num_major_sections;
3228 int num_major_sections_saved, save_target, allowance_target;
3229 mword los_memory_saved, los_memory_alloced, old_los_memory_usage;
3232 * A domain could have been freed, resulting in
3233 * los_memory_usage being less than last_los_memory_usage.
3235 los_memory_alloced = los_memory_usage - MIN (last_los_memory_usage, los_memory_usage);
3236 old_los_memory_usage = los_memory_usage;
3238 //count_ref_nonref_objs ();
3239 //consistency_check ();
3241 binary_protocol_collection (GENERATION_OLD);
3242 check_scan_starts ();
3243 gray_object_queue_init (&gray_queue);
3246 DEBUG (1, fprintf (gc_debug_file, "Start major collection %d\n", num_major_gcs));
3248 mono_stats.major_gc_count ++;
3250 /* world must be stopped already */
3251 TV_GETTIME (all_atv);
3254 /* Pinning depends on this */
3255 clear_nursery_fragments (nursery_next);
3258 time_major_pre_collection_fragment_clear += TV_ELAPSED_MS (atv, btv);
3261 check_for_xdomain_refs ();
3263 nursery_section->next_data = nursery_real_end;
3264 /* we should also coalesce scanning from sections close to each other
3265 * and deal with pointers outside of the sections later.
3267 /* The remsets are not useful for a major collection */
3269 global_remset_cache_clear ();
3273 DEBUG (6, fprintf (gc_debug_file, "Collecting pinned addresses\n"));
3274 pin_from_roots ((void*)lowest_heap_address, (void*)highest_heap_address);
3275 optimize_pin_queue (0);
3278 * pin_queue now contains all candidate pointers, sorted and
3279 * uniqued. We must do two passes now to figure out which
3280 * objects are pinned.
3282 * The first is to find within the pin_queue the area for each
3283 * section. This requires that the pin_queue be sorted. We
3284 * also process the LOS objects and pinned chunks here.
3286 * The second, destructive, pass is to reduce the section
3287 * areas to pointers to the actually pinned objects.
3289 DEBUG (6, fprintf (gc_debug_file, "Pinning from sections\n"));
3290 /* first pass for the sections */
3291 find_section_pin_queue_start_end (nursery_section);
3292 major_find_pin_queue_start_ends (&gray_queue);
3293 /* identify possible pointers to the insize of large objects */
3294 DEBUG (6, fprintf (gc_debug_file, "Pinning from large objects\n"));
3295 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
3297 find_optimized_pin_queue_area (bigobj->data, (char*)bigobj->data + bigobj->size, &start, &end);
3299 pin_object (bigobj->data);
3300 /* FIXME: only enqueue if object has references */
3301 GRAY_OBJECT_ENQUEUE (&gray_queue, bigobj->data);
3303 pin_stats_register_object ((char*) bigobj->data, safe_object_get_size ((MonoObject*) bigobj->data));
3304 DEBUG (6, fprintf (gc_debug_file, "Marked large object %p (%s) size: %lu from roots\n", bigobj->data, safe_name (bigobj->data), (unsigned long)bigobj->size));
3307 /* second pass for the sections */
3308 pin_objects_in_section (nursery_section, &gray_queue);
3309 major_pin_objects (&gray_queue);
3312 time_major_pinning += TV_ELAPSED_MS (atv, btv);
3313 DEBUG (2, fprintf (gc_debug_file, "Finding pinned pointers: %d in %d usecs\n", next_pin_slot, TV_ELAPSED (atv, btv)));
3314 DEBUG (4, fprintf (gc_debug_file, "Start scan with %d pinned objects\n", next_pin_slot));
3316 major_init_to_space ();
3318 drain_gray_stack (&gray_queue);
3321 time_major_scan_pinned += TV_ELAPSED_MS (btv, atv);
3323 /* registered roots, this includes static fields */
3324 scan_from_registered_roots (major_copy_or_mark_object, heap_start, heap_end, ROOT_TYPE_NORMAL, &gray_queue);
3325 scan_from_registered_roots (major_copy_or_mark_object, heap_start, heap_end, ROOT_TYPE_WBARRIER, &gray_queue);
3327 time_major_scan_registered_roots += TV_ELAPSED_MS (atv, btv);
3330 /* FIXME: This is the wrong place for this, because it does
3332 scan_thread_data (heap_start, heap_end, TRUE);
3334 time_major_scan_thread_data += TV_ELAPSED_MS (btv, atv);
3337 time_major_scan_alloc_pinned += TV_ELAPSED_MS (atv, btv);
3339 /* scan the list of objects ready for finalization */
3340 scan_finalizer_entries (major_copy_or_mark_object, fin_ready_list, &gray_queue);
3341 scan_finalizer_entries (major_copy_or_mark_object, critical_fin_list, &gray_queue);
3343 time_major_scan_finalized += TV_ELAPSED_MS (btv, atv);
3344 DEBUG (2, fprintf (gc_debug_file, "Root scan: %d usecs\n", TV_ELAPSED (btv, atv)));
3347 time_major_scan_big_objects += TV_ELAPSED_MS (atv, btv);
3349 /* all the objects in the heap */
3350 finish_gray_stack (heap_start, heap_end, GENERATION_OLD, &gray_queue);
3352 time_major_finish_gray_stack += TV_ELAPSED_MS (btv, atv);
3354 /* sweep the big objects list */
3356 for (bigobj = los_object_list; bigobj;) {
3357 if (object_is_pinned (bigobj->data)) {
3358 unpin_object (bigobj->data);
3361 /* not referenced anywhere, so we can free it */
3363 prevbo->next = bigobj->next;
3365 los_object_list = bigobj->next;
3367 bigobj = bigobj->next;
3368 free_large_object (to_free);
3372 bigobj = bigobj->next;
3376 time_major_free_bigobjs += TV_ELAPSED_MS (atv, btv);
3381 time_major_los_sweep += TV_ELAPSED_MS (btv, atv);
3386 time_major_sweep += TV_ELAPSED_MS (atv, btv);
3388 /* walk the pin_queue, build up the fragment list of free memory, unmark
3389 * pinned objects as we go, memzero() the empty fragments so they are ready for the
3392 build_nursery_fragments (nursery_section->pin_queue_start, nursery_section->pin_queue_end);
3395 time_major_fragment_creation += TV_ELAPSED_MS (btv, atv);
3397 TV_GETTIME (all_btv);
3398 mono_stats.major_gc_time_usecs += TV_ELAPSED (all_atv, all_btv);
3401 dump_heap ("major", num_major_gcs - 1, reason);
3403 /* prepare the pin queue for the next collection */
3405 if (fin_ready_list || critical_fin_list) {
3406 DEBUG (4, fprintf (gc_debug_file, "Finalizer-thread wakeup: ready %d\n", num_ready_finalizers));
3407 mono_gc_finalize_notify ();
3411 g_assert (gray_object_queue_is_empty (&gray_queue));
3413 num_major_sections_saved = MAX (old_num_major_sections - num_major_sections, 0);
3414 los_memory_saved = MAX (old_los_memory_usage - los_memory_usage, 1);
3416 save_target = ((num_major_sections * MAJOR_SECTION_SIZE) + los_memory_saved) / 2;
3418 * We aim to allow the allocation of as many sections as is
3419 * necessary to reclaim save_target sections in the next
3420 * collection. We assume the collection pattern won't change.
3421 * In the last cycle, we had num_major_sections_saved for
3422 * minor_collection_sections_alloced. Assuming things won't
3423 * change, this must be the same ratio as save_target for
3424 * allowance_target, i.e.
3426 * num_major_sections_saved save_target
3427 * --------------------------------- == ----------------
3428 * minor_collection_sections_alloced allowance_target
3432 allowance_target = (mword)((double)save_target * (double)(minor_collection_sections_alloced * MAJOR_SECTION_SIZE + los_memory_alloced) / (double)(num_major_sections_saved * MAJOR_SECTION_SIZE + los_memory_saved));
3434 minor_collection_allowance = MAX (MIN (allowance_target, num_major_sections * MAJOR_SECTION_SIZE + los_memory_usage), MIN_MINOR_COLLECTION_ALLOWANCE);
3436 minor_collection_sections_alloced = 0;
3437 last_los_memory_usage = los_memory_usage;
3439 major_finish_major_collection ();
3441 check_scan_starts ();
3443 binary_protocol_flush_buffers ();
3445 //consistency_check ();
3449 major_collection (const char *reason)
3451 if (g_getenv ("MONO_GC_NO_MAJOR")) {
3452 collect_nursery (0);
3456 current_collection_generation = GENERATION_OLD;
3457 major_do_collection (reason);
3458 current_collection_generation = -1;
3462 * When deciding if it's better to collect or to expand, keep track
3463 * of how much garbage was reclaimed with the last collection: if it's too
3465 * This is called when we could not allocate a small object.
3467 static void __attribute__((noinline))
3468 minor_collect_or_expand_inner (size_t size)
3470 int do_minor_collection = 1;
3472 if (!nursery_section) {
3476 if (do_minor_collection) {
3478 if (collect_nursery (size))
3479 major_collection ("minor overflow");
3480 DEBUG (2, fprintf (gc_debug_file, "Heap size: %lu, LOS size: %lu\n", (unsigned long)total_alloc, (unsigned long)los_memory_usage));
3482 /* this also sets the proper pointers for the next allocation */
3483 if (!search_fragment_for_size (size)) {
3485 /* TypeBuilder and MonoMethod are killing mcs with fragmentation */
3486 DEBUG (1, fprintf (gc_debug_file, "nursery collection didn't find enough room for %zd alloc (%d pinned)\n", size, last_num_pinned));
3487 for (i = 0; i < last_num_pinned; ++i) {
3488 DEBUG (3, fprintf (gc_debug_file, "Bastard pinning obj %p (%s), size: %d\n", pin_queue [i], safe_name (pin_queue [i]), safe_object_get_size (pin_queue [i])));
3493 //report_internal_mem_usage ();
3497 * ######################################################################
3498 * ######## Memory allocation from the OS
3499 * ######################################################################
3500 * This section of code deals with getting memory from the OS and
3501 * allocating memory for GC-internal data structures.
3502 * Internal memory can be handled with a freelist for small objects.
3508 G_GNUC_UNUSED static void
3509 report_internal_mem_usage (void)
3511 printf ("Internal memory usage:\n");
3512 mono_sgen_report_internal_mem_usage ();
3513 printf ("Pinned memory usage:\n");
3514 major_report_pinned_memory_usage ();
3518 * Allocate a big chunk of memory from the OS (usually 64KB to several megabytes).
3519 * This must not require any lock.
3522 mono_sgen_alloc_os_memory (size_t size, int activate)
3525 unsigned long prot_flags = activate? MONO_MMAP_READ|MONO_MMAP_WRITE: MONO_MMAP_NONE;
3527 prot_flags |= MONO_MMAP_PRIVATE | MONO_MMAP_ANON;
3528 size += pagesize - 1;
3529 size &= ~(pagesize - 1);
3530 ptr = mono_valloc (0, size, prot_flags);
3532 total_alloc += size;
3537 * Free the memory returned by mono_sgen_alloc_os_memory (), returning it to the OS.
3540 mono_sgen_free_os_memory (void *addr, size_t size)
3542 mono_vfree (addr, size);
3544 size += pagesize - 1;
3545 size &= ~(pagesize - 1);
3547 total_alloc -= size;
3551 * ######################################################################
3552 * ######## Object allocation
3553 * ######################################################################
3554 * This section of code deals with allocating memory for objects.
3555 * There are several ways:
3556 * *) allocate large objects
3557 * *) allocate normal objects
3558 * *) fast lock-free allocation
3559 * *) allocation of pinned objects
3563 setup_fragment (Fragment *frag, Fragment *prev, size_t size)
3565 /* remove from the list */
3567 prev->next = frag->next;
3569 nursery_fragments = frag->next;
3570 nursery_next = frag->fragment_start;
3571 nursery_frag_real_end = frag->fragment_end;
3573 DEBUG (4, fprintf (gc_debug_file, "Using nursery fragment %p-%p, size: %td (req: %zd)\n", nursery_next, nursery_frag_real_end, nursery_frag_real_end - nursery_next, size));
3574 frag->next = fragment_freelist;
3575 fragment_freelist = frag;
3578 /* check if we have a suitable fragment in nursery_fragments to be able to allocate
3579 * an object of size @size
3580 * Return FALSE if not found (which means we need a collection)
3583 search_fragment_for_size (size_t size)
3585 Fragment *frag, *prev;
3586 DEBUG (4, fprintf (gc_debug_file, "Searching nursery fragment %p, size: %zd\n", nursery_frag_real_end, size));
3588 if (nursery_frag_real_end > nursery_next && nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
3589 /* Clear the remaining space, pinning depends on this */
3590 memset (nursery_next, 0, nursery_frag_real_end - nursery_next);
3593 for (frag = nursery_fragments; frag; frag = frag->next) {
3594 if (size <= (frag->fragment_end - frag->fragment_start)) {
3595 setup_fragment (frag, prev, size);
3604 * Same as search_fragment_for_size but if search for @desired_size fails, try to satisfy @minimum_size.
3605 * This improves nursery usage.
3608 search_fragment_for_size_range (size_t desired_size, size_t minimum_size)
3610 Fragment *frag, *prev, *min_prev;
3611 DEBUG (4, fprintf (gc_debug_file, "Searching nursery fragment %p, desired size: %zd minimum size %zd\n", nursery_frag_real_end, desired_size, minimum_size));
3613 if (nursery_frag_real_end > nursery_next && nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
3614 /* Clear the remaining space, pinning depends on this */
3615 memset (nursery_next, 0, nursery_frag_real_end - nursery_next);
3617 min_prev = GINT_TO_POINTER (-1);
3620 for (frag = nursery_fragments; frag; frag = frag->next) {
3621 int frag_size = frag->fragment_end - frag->fragment_start;
3622 if (desired_size <= frag_size) {
3623 setup_fragment (frag, prev, desired_size);
3624 return desired_size;
3626 if (minimum_size <= frag_size)
3632 if (min_prev != GINT_TO_POINTER (-1)) {
3635 frag = min_prev->next;
3637 frag = nursery_fragments;
3639 frag_size = frag->fragment_end - frag->fragment_start;
3640 HEAVY_STAT (++stat_wasted_fragments_used);
3641 HEAVY_STAT (stat_wasted_fragments_bytes += frag_size);
3643 setup_fragment (frag, min_prev, minimum_size);
3651 alloc_degraded (MonoVTable *vtable, size_t size)
3653 if (need_major_collection ()) {
3655 major_collection ("degraded overflow");
3659 return major_alloc_degraded (vtable, size);
3663 * Provide a variant that takes just the vtable for small fixed-size objects.
3664 * The aligned size is already computed and stored in vt->gc_descr.
3665 * Note: every SCAN_START_SIZE or so we are given the chance to do some special
3666 * processing. We can keep track of where objects start, for example,
3667 * so when we scan the thread stacks for pinned objects, we can start
3668 * a search for the pinned object in SCAN_START_SIZE chunks.
3671 mono_gc_alloc_obj_nolock (MonoVTable *vtable, size_t size)
3673 /* FIXME: handle OOM */
3678 HEAVY_STAT (++stat_objects_alloced);
3679 if (size <= MAX_SMALL_OBJ_SIZE)
3680 HEAVY_STAT (stat_bytes_alloced += size);
3682 HEAVY_STAT (stat_bytes_alloced_los += size);
3684 size = ALIGN_UP (size);
3686 g_assert (vtable->gc_descr);
3688 if (G_UNLIKELY (collect_before_allocs)) {
3689 if (nursery_section) {
3691 collect_nursery (0);
3693 if (!degraded_mode && !search_fragment_for_size (size)) {
3695 g_assert_not_reached ();
3701 * We must already have the lock here instead of after the
3702 * fast path because we might be interrupted in the fast path
3703 * (after confirming that new_next < TLAB_TEMP_END) by the GC,
3704 * and we'll end up allocating an object in a fragment which
3705 * no longer belongs to us.
3707 * The managed allocator does not do this, but it's treated
3708 * specially by the world-stopping code.
3711 if (size > MAX_SMALL_OBJ_SIZE) {
3712 p = alloc_large_inner (vtable, size);
3714 /* tlab_next and tlab_temp_end are TLS vars so accessing them might be expensive */
3716 p = (void**)TLAB_NEXT;
3717 /* FIXME: handle overflow */
3718 new_next = (char*)p + size;
3719 TLAB_NEXT = new_next;
3721 if (G_LIKELY (new_next < TLAB_TEMP_END)) {
3725 * FIXME: We might need a memory barrier here so the change to tlab_next is
3726 * visible before the vtable store.
3729 DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
3730 binary_protocol_alloc (p , vtable, size);
3731 g_assert (*p == NULL);
3734 g_assert (TLAB_NEXT == new_next);
3741 /* there are two cases: the object is too big or we run out of space in the TLAB */
3742 /* we also reach here when the thread does its first allocation after a minor
3743 * collection, since the tlab_ variables are initialized to NULL.
3744 * there can be another case (from ORP), if we cooperate with the runtime a bit:
3745 * objects that need finalizers can have the high bit set in their size
3746 * so the above check fails and we can readily add the object to the queue.
3747 * This avoids taking again the GC lock when registering, but this is moot when
3748 * doing thread-local allocation, so it may not be a good idea.
3750 g_assert (TLAB_NEXT == new_next);
3751 if (TLAB_NEXT >= TLAB_REAL_END) {
3753 * Run out of space in the TLAB. When this happens, some amount of space
3754 * remains in the TLAB, but not enough to satisfy the current allocation
3755 * request. Currently, we retire the TLAB in all cases, later we could
3756 * keep it if the remaining space is above a treshold, and satisfy the
3757 * allocation directly from the nursery.
3760 /* when running in degraded mode, we continue allocing that way
3761 * for a while, to decrease the number of useless nursery collections.
3763 if (degraded_mode && degraded_mode < DEFAULT_NURSERY_SIZE) {
3764 p = alloc_degraded (vtable, size);
3765 binary_protocol_alloc_degraded (p, vtable, size);
3769 /*FIXME This codepath is current deadcode since tlab_size > MAX_SMALL_OBJ_SIZE*/
3770 if (size > tlab_size) {
3771 /* Allocate directly from the nursery */
3772 if (nursery_next + size >= nursery_frag_real_end) {
3773 if (!search_fragment_for_size (size)) {
3774 minor_collect_or_expand_inner (size);
3775 if (degraded_mode) {
3776 p = alloc_degraded (vtable, size);
3777 binary_protocol_alloc_degraded (p, vtable, size);
3783 p = (void*)nursery_next;
3784 nursery_next += size;
3785 if (nursery_next > nursery_frag_real_end) {
3790 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
3791 memset (p, 0, size);
3793 int alloc_size = tlab_size;
3794 int available_in_nursery = nursery_frag_real_end - nursery_next;
3796 DEBUG (3, fprintf (gc_debug_file, "Retire TLAB: %p-%p [%ld]\n", TLAB_START, TLAB_REAL_END, (long)(TLAB_REAL_END - TLAB_NEXT - size)));
3798 if (alloc_size >= available_in_nursery) {
3799 if (available_in_nursery > MAX_NURSERY_TLAB_WASTE && available_in_nursery > size) {
3800 alloc_size = available_in_nursery;
3802 alloc_size = search_fragment_for_size_range (tlab_size, size);
3804 alloc_size = tlab_size;
3805 minor_collect_or_expand_inner (tlab_size);
3806 if (degraded_mode) {
3807 p = alloc_degraded (vtable, size);
3808 binary_protocol_alloc_degraded (p, vtable, size);
3815 /* Allocate a new TLAB from the current nursery fragment */
3816 TLAB_START = nursery_next;
3817 nursery_next += alloc_size;
3818 TLAB_NEXT = TLAB_START;
3819 TLAB_REAL_END = TLAB_START + alloc_size;
3820 TLAB_TEMP_END = TLAB_START + MIN (SCAN_START_SIZE, alloc_size);
3822 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
3823 memset (TLAB_START, 0, alloc_size);
3825 /* Allocate from the TLAB */
3826 p = (void*)TLAB_NEXT;
3828 g_assert (TLAB_NEXT <= TLAB_REAL_END);
3830 nursery_section->scan_starts [((char*)p - (char*)nursery_section->data)/SCAN_START_SIZE] = (char*)p;
3833 /* Reached tlab_temp_end */
3835 /* record the scan start so we can find pinned objects more easily */
3836 nursery_section->scan_starts [((char*)p - (char*)nursery_section->data)/SCAN_START_SIZE] = (char*)p;
3837 /* we just bump tlab_temp_end as well */
3838 TLAB_TEMP_END = MIN (TLAB_REAL_END, TLAB_NEXT + SCAN_START_SIZE);
3839 DEBUG (5, fprintf (gc_debug_file, "Expanding local alloc: %p-%p\n", TLAB_NEXT, TLAB_TEMP_END));
3843 DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
3844 binary_protocol_alloc (p, vtable, size);
3851 mono_gc_try_alloc_obj_nolock (MonoVTable *vtable, size_t size)
3857 size = ALIGN_UP (size);
3859 g_assert (vtable->gc_descr);
3860 if (size <= MAX_SMALL_OBJ_SIZE) {
3861 /* tlab_next and tlab_temp_end are TLS vars so accessing them might be expensive */
3863 p = (void**)TLAB_NEXT;
3864 /* FIXME: handle overflow */
3865 new_next = (char*)p + size;
3866 TLAB_NEXT = new_next;
3868 if (G_LIKELY (new_next < TLAB_TEMP_END)) {
3872 * FIXME: We might need a memory barrier here so the change to tlab_next is
3873 * visible before the vtable store.
3876 HEAVY_STAT (++stat_objects_alloced);
3877 HEAVY_STAT (stat_bytes_alloced += size);
3879 DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
3880 binary_protocol_alloc (p, vtable, size);
3881 g_assert (*p == NULL);
3884 g_assert (TLAB_NEXT == new_next);
3893 mono_gc_alloc_obj (MonoVTable *vtable, size_t size)
3896 #ifndef DISABLE_CRITICAL_REGION
3898 ENTER_CRITICAL_REGION;
3899 res = mono_gc_try_alloc_obj_nolock (vtable, size);
3901 EXIT_CRITICAL_REGION;
3904 EXIT_CRITICAL_REGION;
3907 res = mono_gc_alloc_obj_nolock (vtable, size);
3913 mono_gc_alloc_vector (MonoVTable *vtable, size_t size, uintptr_t max_length)
3916 #ifndef DISABLE_CRITICAL_REGION
3918 ENTER_CRITICAL_REGION;
3919 arr = mono_gc_try_alloc_obj_nolock (vtable, size);
3921 arr->max_length = max_length;
3922 EXIT_CRITICAL_REGION;
3925 EXIT_CRITICAL_REGION;
3930 arr = mono_gc_alloc_obj_nolock (vtable, size);
3931 arr->max_length = max_length;
3939 mono_gc_alloc_array (MonoVTable *vtable, size_t size, uintptr_t max_length, uintptr_t bounds_size)
3942 MonoArrayBounds *bounds;
3946 arr = mono_gc_alloc_obj_nolock (vtable, size);
3947 arr->max_length = max_length;
3949 bounds = (MonoArrayBounds*)((char*)arr + size - bounds_size);
3950 arr->bounds = bounds;
3958 mono_gc_alloc_string (MonoVTable *vtable, size_t size, gint32 len)
3961 #ifndef DISABLE_CRITICAL_REGION
3963 ENTER_CRITICAL_REGION;
3964 str = mono_gc_try_alloc_obj_nolock (vtable, size);
3967 EXIT_CRITICAL_REGION;
3970 EXIT_CRITICAL_REGION;
3975 str = mono_gc_alloc_obj_nolock (vtable, size);
3984 * To be used for interned strings and possibly MonoThread, reflection handles.
3985 * We may want to explicitly free these objects.
3988 mono_gc_alloc_pinned_obj (MonoVTable *vtable, size_t size)
3990 /* FIXME: handle OOM */
3992 size = ALIGN_UP (size);
3994 if (size > MAX_SMALL_OBJ_SIZE) {
3995 /* large objects are always pinned anyway */
3996 p = alloc_large_inner (vtable, size);
3998 DEBUG (9, g_assert (vtable->klass->inited));
3999 p = major_alloc_small_pinned_obj (size, vtable->klass->has_references);
4001 DEBUG (6, fprintf (gc_debug_file, "Allocated pinned object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
4002 binary_protocol_alloc_pinned (p, vtable, size);
4009 * ######################################################################
4010 * ######## Finalization support
4011 * ######################################################################
4015 * this is valid for the nursery: if the object has been forwarded it means it's
4016 * still refrenced from a root. If it is pinned it's still alive as well.
4017 * Return TRUE if @obj is ready to be finalized.
4019 #define object_is_fin_ready(obj) (!object_is_pinned (obj) && !object_is_forwarded (obj))
4022 is_critical_finalizer (FinalizeEntry *entry)
4027 if (!mono_defaults.critical_finalizer_object)
4030 obj = entry->object;
4031 class = ((MonoVTable*)LOAD_VTABLE (obj))->klass;
4033 return mono_class_has_parent (class, mono_defaults.critical_finalizer_object);
4037 queue_finalization_entry (FinalizeEntry *entry) {
4038 if (is_critical_finalizer (entry)) {
4039 entry->next = critical_fin_list;
4040 critical_fin_list = entry;
4042 entry->next = fin_ready_list;
4043 fin_ready_list = entry;
4047 /* LOCKING: requires that the GC lock is held */
4049 rehash_fin_table (FinalizeEntryHashTable *hash_table)
4051 FinalizeEntry **finalizable_hash = hash_table->table;
4052 mword finalizable_hash_size = hash_table->size;
4055 FinalizeEntry **new_hash;
4056 FinalizeEntry *entry, *next;
4057 int new_size = g_spaced_primes_closest (hash_table->num_registered);
4059 new_hash = mono_sgen_alloc_internal_dynamic (new_size * sizeof (FinalizeEntry*), INTERNAL_MEM_FIN_TABLE);
4060 for (i = 0; i < finalizable_hash_size; ++i) {
4061 for (entry = finalizable_hash [i]; entry; entry = next) {
4062 hash = mono_object_hash (entry->object) % new_size;
4064 entry->next = new_hash [hash];
4065 new_hash [hash] = entry;
4068 mono_sgen_free_internal_dynamic (finalizable_hash, finalizable_hash_size * sizeof (FinalizeEntry*), INTERNAL_MEM_FIN_TABLE);
4069 hash_table->table = new_hash;
4070 hash_table->size = new_size;
4073 /* LOCKING: requires that the GC lock is held */
4075 rehash_fin_table_if_necessary (FinalizeEntryHashTable *hash_table)
4077 if (hash_table->num_registered >= hash_table->size * 2)
4078 rehash_fin_table (hash_table);
4081 /* LOCKING: requires that the GC lock is held */
4083 finalize_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, int generation, GrayQueue *queue)
4085 FinalizeEntryHashTable *hash_table = get_finalize_entry_hash_table (generation);
4086 FinalizeEntry *entry, *prev;
4088 FinalizeEntry **finalizable_hash = hash_table->table;
4089 mword finalizable_hash_size = hash_table->size;
4093 for (i = 0; i < finalizable_hash_size; ++i) {
4095 for (entry = finalizable_hash [i]; entry;) {
4096 if ((char*)entry->object >= start && (char*)entry->object < end && !major_is_object_live (entry->object)) {
4097 gboolean is_fin_ready = object_is_fin_ready (entry->object);
4098 char *copy = entry->object;
4099 copy_func ((void**)©, queue);
4102 FinalizeEntry *next;
4103 /* remove and put in fin_ready_list */
4105 prev->next = entry->next;
4107 finalizable_hash [i] = entry->next;
4109 num_ready_finalizers++;
4110 hash_table->num_registered--;
4111 queue_finalization_entry (entry);
4112 /* Make it survive */
4113 from = entry->object;
4114 entry->object = copy;
4115 DEBUG (5, fprintf (gc_debug_file, "Queueing object for finalization: %p (%s) (was at %p) (%d/%d)\n", entry->object, safe_name (entry->object), from, num_ready_finalizers, hash_table->num_registered));
4119 char *from = entry->object;
4120 if (hash_table == &minor_finalizable_hash && !ptr_in_nursery (copy)) {
4121 FinalizeEntry *next = entry->next;
4122 unsigned int major_hash;
4123 /* remove from the list */
4125 prev->next = entry->next;
4127 finalizable_hash [i] = entry->next;
4128 hash_table->num_registered--;
4130 entry->object = copy;
4132 /* insert it into the major hash */
4133 rehash_fin_table_if_necessary (&major_finalizable_hash);
4134 major_hash = mono_object_hash ((MonoObject*) copy) %
4135 major_finalizable_hash.size;
4136 entry->next = major_finalizable_hash.table [major_hash];
4137 major_finalizable_hash.table [major_hash] = entry;
4138 major_finalizable_hash.num_registered++;
4140 DEBUG (5, fprintf (gc_debug_file, "Promoting finalization of object %p (%s) (was at %p) to major table\n", copy, safe_name (copy), from));
4145 /* update pointer */
4146 DEBUG (5, fprintf (gc_debug_file, "Updating object for finalization: %p (%s) (was at %p)\n", entry->object, safe_name (entry->object), from));
4147 entry->object = copy;
4152 entry = entry->next;
4158 object_is_reachable (char *object, char *start, char *end)
4160 /*This happens for non nursery objects during minor collections. We just treat all objects as alive.*/
4161 if (object < start || object >= end)
4163 return !object_is_fin_ready (object) || major_is_object_live (object);
4166 /* LOCKING: requires that the GC lock is held */
4168 null_ephemerons_for_domain (MonoDomain *domain)
4170 EphemeronLinkNode *current = ephemeron_list, *prev = NULL;
4173 MonoObject *object = (MonoObject*)current->array;
4175 if (object && !object->vtable) {
4176 EphemeronLinkNode *tmp = current;
4179 prev->next = current->next;
4181 ephemeron_list = current->next;
4183 current = current->next;
4184 mono_sgen_free_internal (tmp, INTERNAL_MEM_EPHEMERON_LINK);
4187 current = current->next;
4192 /* LOCKING: requires that the GC lock is held */
4194 clear_unreachable_ephemerons (CopyOrMarkObjectFunc copy_func, char *start, char *end, GrayQueue *queue)
4196 int was_in_nursery, was_promoted;
4197 EphemeronLinkNode *current = ephemeron_list, *prev = NULL;
4199 Ephemeron *cur, *array_end;
4203 char *object = current->array;
4205 if (!object_is_reachable (object, start, end)) {
4206 EphemeronLinkNode *tmp = current;
4208 DEBUG (5, fprintf (gc_debug_file, "Dead Ephemeron array at %p\n", object));
4211 prev->next = current->next;
4213 ephemeron_list = current->next;
4215 current = current->next;
4216 mono_sgen_free_internal (tmp, INTERNAL_MEM_EPHEMERON_LINK);
4221 was_in_nursery = ptr_in_nursery (object);
4222 copy_func ((void**)&object, queue);
4223 current->array = object;
4225 /*The array was promoted, add global remsets for key/values left behind in nursery.*/
4226 was_promoted = was_in_nursery && !ptr_in_nursery (object);
4228 DEBUG (5, fprintf (gc_debug_file, "Clearing unreachable entries for ephemeron array at %p\n", object));
4230 array = (MonoArray*)object;
4231 cur = mono_array_addr (array, Ephemeron, 0);
4232 array_end = cur + mono_array_length_fast (array);
4233 tombstone = (char*)((MonoVTable*)LOAD_VTABLE (object))->domain->ephemeron_tombstone;
4235 for (; cur < array_end; ++cur) {
4236 char *key = (char*)cur->key;
4238 if (!key || key == tombstone)
4241 DEBUG (5, fprintf (gc_debug_file, "[%td] key %p (%s) value %p (%s)\n", cur - mono_array_addr (array, Ephemeron, 0),
4242 key, object_is_reachable (key, start, end) ? "reachable" : "unreachable",
4243 cur->value, cur->value && object_is_reachable (cur->value, start, end) ? "reachable" : "unreachable"));
4245 if (!object_is_reachable (key, start, end)) {
4246 cur->key = tombstone;
4252 if (ptr_in_nursery (key)) {/*key was not promoted*/
4253 DEBUG (5, fprintf (gc_debug_file, "\tAdded remset to key %p\n", key));
4254 add_to_global_remset (&cur->key);
4256 if (ptr_in_nursery (cur->value)) {/*value was not promoted*/
4257 DEBUG (5, fprintf (gc_debug_file, "\tAdded remset to value %p\n", cur->value));
4258 add_to_global_remset (&cur->value);
4263 current = current->next;
4267 /* LOCKING: requires that the GC lock is held */
4269 mark_ephemerons_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, GrayQueue *queue)
4271 int nothing_marked = 1;
4272 EphemeronLinkNode *current = ephemeron_list;
4274 Ephemeron *cur, *array_end;
4277 for (current = ephemeron_list; current; current = current->next) {
4278 char *object = current->array;
4279 DEBUG (5, fprintf (gc_debug_file, "Ephemeron array at %p\n", object));
4281 /*We ignore arrays in old gen during minor collections since all objects are promoted by the remset machinery.*/
4282 if (object < start || object >= end)
4285 /*It has to be alive*/
4286 if (!object_is_reachable (object, start, end)) {
4287 DEBUG (5, fprintf (gc_debug_file, "\tnot reachable\n"));
4291 copy_func ((void**)&object, queue);
4293 array = (MonoArray*)object;
4294 cur = mono_array_addr (array, Ephemeron, 0);
4295 array_end = cur + mono_array_length_fast (array);
4296 tombstone = (char*)((MonoVTable*)LOAD_VTABLE (object))->domain->ephemeron_tombstone;
4298 for (; cur < array_end; ++cur) {
4299 char *key = cur->key;
4301 if (!key || key == tombstone)
4304 DEBUG (5, fprintf (gc_debug_file, "[%td] key %p (%s) value %p (%s)\n", cur - mono_array_addr (array, Ephemeron, 0),
4305 key, object_is_reachable (key, start, end) ? "reachable" : "unreachable",
4306 cur->value, cur->value && object_is_reachable (cur->value, start, end) ? "reachable" : "unreachable"));
4308 if (object_is_reachable (key, start, end)) {
4309 char *value = cur->value;
4311 copy_func ((void**)&cur->key, queue);
4313 if (!object_is_reachable (value, start, end))
4315 copy_func ((void**)&cur->value, queue);
4321 DEBUG (5, fprintf (gc_debug_file, "Ephemeron run finished. Is it done %d\n", nothing_marked));
4322 return nothing_marked;
4325 /* LOCKING: requires that the GC lock is held */
4327 null_link_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, int generation, GrayQueue *queue)
4329 DisappearingLinkHashTable *hash = get_dislink_hash_table (generation);
4330 DisappearingLink **disappearing_link_hash = hash->table;
4331 int disappearing_link_hash_size = hash->size;
4332 DisappearingLink *entry, *prev;
4334 if (!hash->num_links)
4336 for (i = 0; i < disappearing_link_hash_size; ++i) {
4338 for (entry = disappearing_link_hash [i]; entry;) {
4339 char *object = DISLINK_OBJECT (entry);
4340 if (object >= start && object < end && !major_is_object_live (object)) {
4341 gboolean track = DISLINK_TRACK (entry);
4342 if (!track && object_is_fin_ready (object)) {
4343 void **p = entry->link;
4344 DisappearingLink *old;
4346 /* remove from list */
4348 prev->next = entry->next;
4350 disappearing_link_hash [i] = entry->next;
4351 DEBUG (5, fprintf (gc_debug_file, "Dislink nullified at %p to GCed object %p\n", p, object));
4353 mono_sgen_free_internal (entry, INTERNAL_MEM_DISLINK);
4358 char *copy = object;
4359 copy_func ((void**)©, queue);
4361 /* Update pointer if it's moved. If the object
4362 * has been moved out of the nursery, we need to
4363 * remove the link from the minor hash table to
4366 * FIXME: what if an object is moved earlier?
4369 if (hash == &minor_disappearing_link_hash && !ptr_in_nursery (copy)) {
4370 void **link = entry->link;
4371 DisappearingLink *old;
4372 /* remove from list */
4374 prev->next = entry->next;
4376 disappearing_link_hash [i] = entry->next;
4378 mono_sgen_free_internal (entry, INTERNAL_MEM_DISLINK);
4382 add_or_remove_disappearing_link ((MonoObject*)copy, link,
4383 track, GENERATION_OLD);
4385 DEBUG (5, fprintf (gc_debug_file, "Upgraded dislink at %p to major because object %p moved to %p\n", link, object, copy));
4389 /* We set the track resurrection bit to
4390 * FALSE if the object is to be finalized
4391 * so that the object can be collected in
4392 * the next cycle (i.e. after it was
4395 *entry->link = HIDE_POINTER (copy,
4396 object_is_fin_ready (object) ? FALSE : track);
4397 DEBUG (5, fprintf (gc_debug_file, "Updated dislink at %p to %p\n", entry->link, DISLINK_OBJECT (entry)));
4402 entry = entry->next;
4407 /* LOCKING: requires that the GC lock is held */
4409 null_links_for_domain (MonoDomain *domain, int generation)
4411 DisappearingLinkHashTable *hash = get_dislink_hash_table (generation);
4412 DisappearingLink **disappearing_link_hash = hash->table;
4413 int disappearing_link_hash_size = hash->size;
4414 DisappearingLink *entry, *prev;
4416 for (i = 0; i < disappearing_link_hash_size; ++i) {
4418 for (entry = disappearing_link_hash [i]; entry; ) {
4419 char *object = DISLINK_OBJECT (entry);
4420 if (object && !((MonoObject*)object)->vtable) {
4421 DisappearingLink *next = entry->next;
4426 disappearing_link_hash [i] = next;
4428 if (*(entry->link)) {
4429 *(entry->link) = NULL;
4430 g_warning ("Disappearing link %p not freed", entry->link);
4432 mono_sgen_free_internal (entry, INTERNAL_MEM_DISLINK);
4439 entry = entry->next;
4444 /* LOCKING: requires that the GC lock is held */
4446 finalizers_for_domain (MonoDomain *domain, MonoObject **out_array, int out_size,
4447 FinalizeEntryHashTable *hash_table)
4449 FinalizeEntry **finalizable_hash = hash_table->table;
4450 mword finalizable_hash_size = hash_table->size;
4451 FinalizeEntry *entry, *prev;
4454 if (no_finalize || !out_size || !out_array)
4457 for (i = 0; i < finalizable_hash_size; ++i) {
4459 for (entry = finalizable_hash [i]; entry;) {
4460 if (mono_object_domain (entry->object) == domain) {
4461 FinalizeEntry *next;
4462 /* remove and put in out_array */
4464 prev->next = entry->next;
4466 finalizable_hash [i] = entry->next;
4468 hash_table->num_registered--;
4469 out_array [count ++] = entry->object;
4470 DEBUG (5, fprintf (gc_debug_file, "Collecting object for finalization: %p (%s) (%d/%d)\n", entry->object, safe_name (entry->object), num_ready_finalizers, hash_table->num_registered));
4472 if (count == out_size)
4477 entry = entry->next;
4484 * mono_gc_finalizers_for_domain:
4485 * @domain: the unloading appdomain
4486 * @out_array: output array
4487 * @out_size: size of output array
4489 * Store inside @out_array up to @out_size objects that belong to the unloading
4490 * appdomain @domain. Returns the number of stored items. Can be called repeteadly
4491 * until it returns 0.
4492 * The items are removed from the finalizer data structure, so the caller is supposed
4494 * @out_array should be on the stack to allow the GC to know the objects are still alive.
4497 mono_gc_finalizers_for_domain (MonoDomain *domain, MonoObject **out_array, int out_size)
4502 result = finalizers_for_domain (domain, out_array, out_size, &minor_finalizable_hash);
4503 if (result < out_size) {
4504 result += finalizers_for_domain (domain, out_array + result, out_size - result,
4505 &major_finalizable_hash);
4513 register_for_finalization (MonoObject *obj, void *user_data, int generation)
4515 FinalizeEntryHashTable *hash_table = get_finalize_entry_hash_table (generation);
4516 FinalizeEntry **finalizable_hash;
4517 mword finalizable_hash_size;
4518 FinalizeEntry *entry, *prev;
4522 g_assert (user_data == NULL || user_data == mono_gc_run_finalize);
4523 hash = mono_object_hash (obj);
4525 rehash_fin_table_if_necessary (hash_table);
4526 finalizable_hash = hash_table->table;
4527 finalizable_hash_size = hash_table->size;
4528 hash %= finalizable_hash_size;
4530 for (entry = finalizable_hash [hash]; entry; entry = entry->next) {
4531 if (entry->object == obj) {
4533 /* remove from the list */
4535 prev->next = entry->next;
4537 finalizable_hash [hash] = entry->next;
4538 hash_table->num_registered--;
4539 DEBUG (5, fprintf (gc_debug_file, "Removed finalizer %p for object: %p (%s) (%d)\n", entry, obj, obj->vtable->klass->name, hash_table->num_registered));
4540 mono_sgen_free_internal (entry, INTERNAL_MEM_FINALIZE_ENTRY);
4548 /* request to deregister, but already out of the list */
4552 entry = mono_sgen_alloc_internal (INTERNAL_MEM_FINALIZE_ENTRY);
4553 entry->object = obj;
4554 entry->next = finalizable_hash [hash];
4555 finalizable_hash [hash] = entry;
4556 hash_table->num_registered++;
4557 DEBUG (5, fprintf (gc_debug_file, "Added finalizer %p for object: %p (%s) (%d) to %s table\n", entry, obj, obj->vtable->klass->name, hash_table->num_registered, generation_name (generation)));
4562 mono_gc_register_for_finalization (MonoObject *obj, void *user_data)
4564 if (ptr_in_nursery (obj))
4565 register_for_finalization (obj, user_data, GENERATION_NURSERY);
4567 register_for_finalization (obj, user_data, GENERATION_OLD);
4571 rehash_dislink (DisappearingLinkHashTable *hash_table)
4573 DisappearingLink **disappearing_link_hash = hash_table->table;
4574 int disappearing_link_hash_size = hash_table->size;
4577 DisappearingLink **new_hash;
4578 DisappearingLink *entry, *next;
4579 int new_size = g_spaced_primes_closest (hash_table->num_links);
4581 new_hash = mono_sgen_alloc_internal_dynamic (new_size * sizeof (DisappearingLink*), INTERNAL_MEM_DISLINK_TABLE);
4582 for (i = 0; i < disappearing_link_hash_size; ++i) {
4583 for (entry = disappearing_link_hash [i]; entry; entry = next) {
4584 hash = mono_aligned_addr_hash (entry->link) % new_size;
4586 entry->next = new_hash [hash];
4587 new_hash [hash] = entry;
4590 mono_sgen_free_internal_dynamic (disappearing_link_hash,
4591 disappearing_link_hash_size * sizeof (DisappearingLink*), INTERNAL_MEM_DISLINK_TABLE);
4592 hash_table->table = new_hash;
4593 hash_table->size = new_size;
4596 /* LOCKING: assumes the GC lock is held */
4598 add_or_remove_disappearing_link (MonoObject *obj, void **link, gboolean track, int generation)
4600 DisappearingLinkHashTable *hash_table = get_dislink_hash_table (generation);
4601 DisappearingLink *entry, *prev;
4603 DisappearingLink **disappearing_link_hash = hash_table->table;
4604 int disappearing_link_hash_size = hash_table->size;
4606 if (hash_table->num_links >= disappearing_link_hash_size * 2) {
4607 rehash_dislink (hash_table);
4608 disappearing_link_hash = hash_table->table;
4609 disappearing_link_hash_size = hash_table->size;
4611 /* FIXME: add check that link is not in the heap */
4612 hash = mono_aligned_addr_hash (link) % disappearing_link_hash_size;
4613 entry = disappearing_link_hash [hash];
4615 for (; entry; entry = entry->next) {
4616 /* link already added */
4617 if (link == entry->link) {
4618 /* NULL obj means remove */
4621 prev->next = entry->next;
4623 disappearing_link_hash [hash] = entry->next;
4624 hash_table->num_links--;
4625 DEBUG (5, fprintf (gc_debug_file, "Removed dislink %p (%d) from %s table\n", entry, hash_table->num_links, generation_name (generation)));
4626 mono_sgen_free_internal (entry, INTERNAL_MEM_DISLINK);
4629 *link = HIDE_POINTER (obj, track); /* we allow the change of object */
4637 entry = mono_sgen_alloc_internal (INTERNAL_MEM_DISLINK);
4638 *link = HIDE_POINTER (obj, track);
4640 entry->next = disappearing_link_hash [hash];
4641 disappearing_link_hash [hash] = entry;
4642 hash_table->num_links++;
4643 DEBUG (5, fprintf (gc_debug_file, "Added dislink %p for object: %p (%s) at %p to %s table\n", entry, obj, obj->vtable->klass->name, link, generation_name (generation)));
4646 /* LOCKING: assumes the GC lock is held */
4648 mono_gc_register_disappearing_link (MonoObject *obj, void **link, gboolean track)
4650 add_or_remove_disappearing_link (NULL, link, FALSE, GENERATION_NURSERY);
4651 add_or_remove_disappearing_link (NULL, link, FALSE, GENERATION_OLD);
4653 if (ptr_in_nursery (obj))
4654 add_or_remove_disappearing_link (obj, link, track, GENERATION_NURSERY);
4656 add_or_remove_disappearing_link (obj, link, track, GENERATION_OLD);
4661 mono_gc_invoke_finalizers (void)
4663 FinalizeEntry *entry = NULL;
4664 gboolean entry_is_critical = FALSE;
4667 /* FIXME: batch to reduce lock contention */
4668 while (fin_ready_list || critical_fin_list) {
4672 FinalizeEntry **list = entry_is_critical ? &critical_fin_list : &fin_ready_list;
4674 /* We have finalized entry in the last
4675 interation, now we need to remove it from
4678 *list = entry->next;
4680 FinalizeEntry *e = *list;
4681 while (e->next != entry)
4683 e->next = entry->next;
4685 mono_sgen_free_internal (entry, INTERNAL_MEM_FINALIZE_ENTRY);
4689 /* Now look for the first non-null entry. */
4690 for (entry = fin_ready_list; entry && !entry->object; entry = entry->next)
4693 entry_is_critical = FALSE;
4695 entry_is_critical = TRUE;
4696 for (entry = critical_fin_list; entry && !entry->object; entry = entry->next)
4701 g_assert (entry->object);
4702 num_ready_finalizers--;
4703 obj = entry->object;
4704 entry->object = NULL;
4705 DEBUG (7, fprintf (gc_debug_file, "Finalizing object %p (%s)\n", obj, safe_name (obj)));
4713 g_assert (entry->object == NULL);
4715 /* the object is on the stack so it is pinned */
4716 /*g_print ("Calling finalizer for object: %p (%s)\n", entry->object, safe_name (entry->object));*/
4717 mono_gc_run_finalize (obj, NULL);
4724 mono_gc_pending_finalizers (void)
4726 return fin_ready_list || critical_fin_list;
4729 /* Negative value to remove */
4731 mono_gc_add_memory_pressure (gint64 value)
4733 /* FIXME: Use interlocked functions */
4735 memory_pressure += value;
4740 * ######################################################################
4741 * ######## registered roots support
4742 * ######################################################################
4746 rehash_roots (gboolean pinned)
4750 RootRecord **new_hash;
4751 RootRecord *entry, *next;
4754 new_size = g_spaced_primes_closest (num_roots_entries [pinned]);
4755 new_hash = mono_sgen_alloc_internal_dynamic (new_size * sizeof (RootRecord*), INTERNAL_MEM_ROOTS_TABLE);
4756 for (i = 0; i < roots_hash_size [pinned]; ++i) {
4757 for (entry = roots_hash [pinned][i]; entry; entry = next) {
4758 hash = mono_aligned_addr_hash (entry->start_root) % new_size;
4760 entry->next = new_hash [hash];
4761 new_hash [hash] = entry;
4764 mono_sgen_free_internal_dynamic (roots_hash [pinned], roots_hash_size [pinned] * sizeof (RootRecord*), INTERNAL_MEM_ROOTS_TABLE);
4765 roots_hash [pinned] = new_hash;
4766 roots_hash_size [pinned] = new_size;
4770 find_root (int root_type, char *start, guint32 addr_hash)
4772 RootRecord *new_root;
4774 guint32 hash = addr_hash % roots_hash_size [root_type];
4775 for (new_root = roots_hash [root_type][hash]; new_root; new_root = new_root->next) {
4776 /* we allow changing the size and the descriptor (for thread statics etc) */
4777 if (new_root->start_root == start) {
4786 * We do not coalesce roots.
4789 mono_gc_register_root_inner (char *start, size_t size, void *descr, int root_type)
4791 RootRecord *new_root;
4792 unsigned int hash, addr_hash = mono_aligned_addr_hash (start);
4795 for (i = 0; i < ROOT_TYPE_NUM; ++i) {
4796 if (num_roots_entries [i] >= roots_hash_size [i] * 2)
4799 for (i = 0; i < ROOT_TYPE_NUM; ++i) {
4800 new_root = find_root (i, start, addr_hash);
4801 /* we allow changing the size and the descriptor (for thread statics etc) */
4803 size_t old_size = new_root->end_root - new_root->start_root;
4804 new_root->end_root = new_root->start_root + size;
4805 g_assert (((new_root->root_desc != 0) && (descr != NULL)) ||
4806 ((new_root->root_desc == 0) && (descr == NULL)));
4807 new_root->root_desc = (mword)descr;
4809 roots_size -= old_size;
4814 new_root = mono_sgen_alloc_internal (INTERNAL_MEM_ROOT_RECORD);
4816 new_root->start_root = start;
4817 new_root->end_root = new_root->start_root + size;
4818 new_root->root_desc = (mword)descr;
4820 hash = addr_hash % roots_hash_size [root_type];
4821 num_roots_entries [root_type]++;
4822 new_root->next = roots_hash [root_type] [hash];
4823 roots_hash [root_type][hash] = new_root;
4824 DEBUG (3, fprintf (gc_debug_file, "Added root %p for range: %p-%p, descr: %p (%d/%d bytes)\n", new_root, new_root->start_root, new_root->end_root, descr, (int)size, (int)roots_size));
4834 mono_gc_register_root (char *start, size_t size, void *descr)
4836 return mono_gc_register_root_inner (start, size, descr, descr ? ROOT_TYPE_NORMAL : ROOT_TYPE_PINNED);
4840 mono_gc_register_root_wbarrier (char *start, size_t size, void *descr)
4842 return mono_gc_register_root_inner (start, size, descr, ROOT_TYPE_WBARRIER);
4846 mono_gc_deregister_root (char* addr)
4848 RootRecord *tmp, *prev;
4849 unsigned int hash, addr_hash = mono_aligned_addr_hash (addr);
4853 for (root_type = 0; root_type < ROOT_TYPE_NUM; ++root_type) {
4854 hash = addr_hash % roots_hash_size [root_type];
4855 tmp = roots_hash [root_type][hash];
4858 if (tmp->start_root == (char*)addr) {
4860 prev->next = tmp->next;
4862 roots_hash [root_type][hash] = tmp->next;
4863 roots_size -= (tmp->end_root - tmp->start_root);
4864 num_roots_entries [root_type]--;
4865 DEBUG (3, fprintf (gc_debug_file, "Removed root %p for range: %p-%p\n", tmp, tmp->start_root, tmp->end_root));
4866 mono_sgen_free_internal (tmp, INTERNAL_MEM_ROOT_RECORD);
4877 * ######################################################################
4878 * ######## Thread handling (stop/start code)
4879 * ######################################################################
4882 /* FIXME: handle large/small config */
4883 #define HASH_PTHREAD_T(id) (((unsigned int)(id) >> 4) * 2654435761u)
4885 static SgenThreadInfo* thread_table [THREAD_HASH_SIZE];
4887 #if USE_SIGNAL_BASED_START_STOP_WORLD
4889 static MonoSemType suspend_ack_semaphore;
4890 static MonoSemType *suspend_ack_semaphore_ptr;
4891 static unsigned int global_stop_count = 0;
4893 static sigset_t suspend_signal_mask;
4894 static mword cur_thread_regs [ARCH_NUM_REGS] = {0};
4896 /* LOCKING: assumes the GC lock is held */
4898 mono_sgen_get_thread_table (void)
4900 return thread_table;
4904 mono_sgen_thread_info_lookup (ARCH_THREAD_TYPE id)
4906 unsigned int hash = HASH_PTHREAD_T (id) % THREAD_HASH_SIZE;
4907 SgenThreadInfo *info;
4909 info = thread_table [hash];
4910 while (info && !ARCH_THREAD_EQUALS (info->id, id)) {
4917 update_current_thread_stack (void *start)
4919 void *ptr = cur_thread_regs;
4920 SgenThreadInfo *info = mono_sgen_thread_info_lookup (ARCH_GET_THREAD ());
4922 info->stack_start = align_pointer (&ptr);
4923 g_assert (info->stack_start >= info->stack_start_limit && info->stack_start < info->stack_end);
4924 ARCH_STORE_REGS (ptr);
4925 info->stopped_regs = ptr;
4926 if (gc_callbacks.thread_suspend_func)
4927 gc_callbacks.thread_suspend_func (info->runtime_data, NULL);
4931 * Define this and use the "xdomain-checks" MONO_GC_DEBUG option to
4932 * have cross-domain checks in the write barrier.
4934 //#define XDOMAIN_CHECKS_IN_WBARRIER
4936 #ifndef BINARY_PROTOCOL
4937 #ifndef HEAVY_STATISTICS
4938 #define MANAGED_ALLOCATION
4939 #ifndef XDOMAIN_CHECKS_IN_WBARRIER
4940 #define MANAGED_WBARRIER
4946 is_ip_in_managed_allocator (MonoDomain *domain, gpointer ip);
4949 mono_sgen_wait_for_suspend_ack (int count)
4953 for (i = 0; i < count; ++i) {
4954 while ((result = MONO_SEM_WAIT (suspend_ack_semaphore_ptr)) != 0) {
4955 if (errno != EINTR) {
4956 g_error ("sem_wait ()");
4963 restart_threads_until_none_in_managed_allocator (void)
4965 SgenThreadInfo *info;
4966 int i, result, num_threads_died = 0;
4967 int sleep_duration = -1;
4970 int restart_count = 0, restarted_count = 0;
4971 /* restart all threads that stopped in the
4973 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
4974 for (info = thread_table [i]; info; info = info->next) {
4977 if (!info->stack_start || info->in_critical_region ||
4978 is_ip_in_managed_allocator (info->stopped_domain, info->stopped_ip)) {
4979 binary_protocol_thread_restart ((gpointer)info->id);
4980 #if defined(__MACH__) && MONO_MACH_ARCH_SUPPORTED
4981 result = thread_resume (pthread_mach_thread_np (info->id));
4983 result = pthread_kill (info->id, restart_signal_num);
4991 /* we set the stopped_ip to
4992 NULL for threads which
4993 we're not restarting so
4994 that we can easily identify
4996 info->stopped_ip = NULL;
4997 info->stopped_domain = NULL;
5001 /* if no threads were restarted, we're done */
5002 if (restart_count == 0)
5005 #if defined(__MACH__) && MONO_MACH_ARCH_SUPPORTED
5006 /* mach thread_resume is synchronous so we dont need to wait for them */
5008 /* wait for the threads to signal their restart */
5009 mono_sgen_wait_for_suspend_ack (restart_count);
5012 if (sleep_duration < 0) {
5016 g_usleep (sleep_duration);
5017 sleep_duration += 10;
5020 /* stop them again */
5021 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5022 for (info = thread_table [i]; info; info = info->next) {
5023 if (info->skip || info->stopped_ip == NULL)
5025 #if defined(__MACH__) && MONO_MACH_ARCH_SUPPORTED
5026 result = thread_suspend (pthread_mach_thread_np (info->id));
5028 result = pthread_kill (info->id, suspend_signal_num);
5037 /* some threads might have died */
5038 num_threads_died += restart_count - restarted_count;
5039 #if defined(__MACH__) && MONO_MACH_ARCH_SUPPORTED
5040 /* mach thread_resume is synchronous so we dont need to wait for them */
5042 /* wait for the threads to signal their suspension
5044 mono_sgen_wait_for_suspend_ack (restart_count);
5048 return num_threads_died;
5051 /* LOCKING: assumes the GC lock is held (by the stopping thread) */
5053 suspend_handler (int sig, siginfo_t *siginfo, void *context)
5055 SgenThreadInfo *info;
5058 int old_errno = errno;
5059 gpointer regs [ARCH_NUM_REGS];
5060 gpointer stack_start;
5062 id = pthread_self ();
5063 info = mono_sgen_thread_info_lookup (id);
5064 info->stopped_domain = mono_domain_get ();
5065 info->stopped_ip = (gpointer) ARCH_SIGCTX_IP (context);
5066 stop_count = global_stop_count;
5067 /* duplicate signal */
5068 if (0 && info->stop_count == stop_count) {
5072 #ifdef HAVE_KW_THREAD
5073 /* update the remset info in the thread data structure */
5074 info->remset = remembered_set;
5076 stack_start = (char*) ARCH_SIGCTX_SP (context) - REDZONE_SIZE;
5077 /* If stack_start is not within the limits, then don't set it
5078 in info and we will be restarted. */
5079 if (stack_start >= info->stack_start_limit && info->stack_start <= info->stack_end) {
5080 info->stack_start = stack_start;
5082 ARCH_COPY_SIGCTX_REGS (regs, context);
5083 info->stopped_regs = regs;
5085 g_assert (!info->stack_start);
5088 /* Notify the JIT */
5089 if (gc_callbacks.thread_suspend_func)
5090 gc_callbacks.thread_suspend_func (info->runtime_data, context);
5092 DEBUG (4, fprintf (gc_debug_file, "Posting suspend_ack_semaphore for suspend from %p %p\n", info, (gpointer)ARCH_GET_THREAD ()));
5093 /* notify the waiting thread */
5094 MONO_SEM_POST (suspend_ack_semaphore_ptr);
5095 info->stop_count = stop_count;
5097 /* wait until we receive the restart signal */
5100 sigsuspend (&suspend_signal_mask);
5101 } while (info->signal != restart_signal_num);
5103 DEBUG (4, fprintf (gc_debug_file, "Posting suspend_ack_semaphore for resume from %p %p\n", info, (gpointer)ARCH_GET_THREAD ()));
5104 /* notify the waiting thread */
5105 MONO_SEM_POST (suspend_ack_semaphore_ptr);
5111 restart_handler (int sig)
5113 SgenThreadInfo *info;
5114 int old_errno = errno;
5116 info = mono_sgen_thread_info_lookup (pthread_self ());
5117 info->signal = restart_signal_num;
5118 DEBUG (4, fprintf (gc_debug_file, "Restart handler in %p %p\n", info, (gpointer)ARCH_GET_THREAD ()));
5124 acquire_gc_locks (void)
5130 release_gc_locks (void)
5132 UNLOCK_INTERRUPTION;
5135 static TV_DECLARE (stop_world_time);
5136 static unsigned long max_pause_usec = 0;
5138 /* LOCKING: assumes the GC lock is held */
5144 acquire_gc_locks ();
5146 update_current_thread_stack (&count);
5148 global_stop_count++;
5149 DEBUG (3, fprintf (gc_debug_file, "stopping world n %d from %p %p\n", global_stop_count, mono_sgen_thread_info_lookup (ARCH_GET_THREAD ()), (gpointer)ARCH_GET_THREAD ()));
5150 TV_GETTIME (stop_world_time);
5151 count = mono_sgen_thread_handshake (suspend_signal_num);
5152 count -= restart_threads_until_none_in_managed_allocator ();
5153 g_assert (count >= 0);
5154 DEBUG (3, fprintf (gc_debug_file, "world stopped %d thread(s)\n", count));
5158 /* LOCKING: assumes the GC lock is held */
5160 restart_world (void)
5163 SgenThreadInfo *info;
5164 TV_DECLARE (end_sw);
5167 /* notify the profiler of the leftovers */
5168 if (G_UNLIKELY (mono_profiler_events & MONO_PROFILE_GC_MOVES)) {
5169 if (moved_objects_idx) {
5170 mono_profiler_gc_moves (moved_objects, moved_objects_idx);
5171 moved_objects_idx = 0;
5174 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5175 for (info = thread_table [i]; info; info = info->next) {
5176 info->stack_start = NULL;
5177 info->stopped_regs = NULL;
5181 release_gc_locks ();
5183 count = mono_sgen_thread_handshake (restart_signal_num);
5184 TV_GETTIME (end_sw);
5185 usec = TV_ELAPSED (stop_world_time, end_sw);
5186 max_pause_usec = MAX (usec, max_pause_usec);
5187 DEBUG (2, fprintf (gc_debug_file, "restarted %d thread(s) (pause time: %d usec, max: %d)\n", count, (int)usec, (int)max_pause_usec));
5191 #endif /* USE_SIGNAL_BASED_START_STOP_WORLD */
5194 mono_gc_set_gc_callbacks (MonoGCCallbacks *callbacks)
5196 gc_callbacks = *callbacks;
5200 mono_gc_get_gc_callbacks ()
5202 return &gc_callbacks;
5205 /* Variables holding start/end nursery so it won't have to be passed at every call */
5206 static void *scan_area_arg_start, *scan_area_arg_end;
5209 mono_gc_conservatively_scan_area (void *start, void *end)
5211 conservatively_pin_objects_from (start, end, scan_area_arg_start, scan_area_arg_end, PIN_TYPE_STACK);
5215 mono_gc_scan_object (void *obj)
5217 if (current_collection_generation == GENERATION_NURSERY)
5218 copy_object (&obj, &gray_queue);
5220 major_copy_or_mark_object (&obj, &gray_queue);
5225 * Mark from thread stacks and registers.
5228 scan_thread_data (void *start_nursery, void *end_nursery, gboolean precise)
5231 SgenThreadInfo *info;
5233 scan_area_arg_start = start_nursery;
5234 scan_area_arg_end = end_nursery;
5236 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5237 for (info = thread_table [i]; info; info = info->next) {
5239 DEBUG (3, fprintf (gc_debug_file, "Skipping dead thread %p, range: %p-%p, size: %td\n", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start));
5242 DEBUG (3, fprintf (gc_debug_file, "Scanning thread %p, range: %p-%p, size: %td, pinned=%d\n", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start, next_pin_slot));
5243 if (gc_callbacks.thread_mark_func && !conservative_stack_mark)
5244 gc_callbacks.thread_mark_func (info->runtime_data, info->stack_start, info->stack_end, precise);
5246 conservatively_pin_objects_from (info->stack_start, info->stack_end, start_nursery, end_nursery, PIN_TYPE_STACK);
5249 conservatively_pin_objects_from (info->stopped_regs, info->stopped_regs + ARCH_NUM_REGS,
5250 start_nursery, end_nursery, PIN_TYPE_STACK);
5256 find_pinning_ref_from_thread (char *obj, size_t size)
5259 SgenThreadInfo *info;
5260 char *endobj = obj + size;
5262 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5263 for (info = thread_table [i]; info; info = info->next) {
5264 char **start = (char**)info->stack_start;
5267 while (start < (char**)info->stack_end) {
5268 if (*start >= obj && *start < endobj) {
5269 DEBUG (0, fprintf (gc_debug_file, "Object %p referenced in thread %p (id %p) at %p, stack: %p-%p\n", obj, info, (gpointer)info->id, start, info->stack_start, info->stack_end));
5274 /* FIXME: check info->stopped_regs */
5280 ptr_on_stack (void *ptr)
5282 gpointer stack_start = &stack_start;
5283 SgenThreadInfo *info = mono_sgen_thread_info_lookup (ARCH_GET_THREAD ());
5285 if (ptr >= stack_start && ptr < (gpointer)info->stack_end)
5291 handle_remset (mword *p, void *start_nursery, void *end_nursery, gboolean global, GrayQueue *queue)
5298 HEAVY_STAT (++stat_global_remsets_processed);
5300 /* FIXME: exclude stack locations */
5301 switch ((*p) & REMSET_TYPE_MASK) {
5302 case REMSET_LOCATION:
5304 //__builtin_prefetch (ptr);
5305 if (((void*)ptr < start_nursery || (void*)ptr >= end_nursery)) {
5306 gpointer old = *ptr;
5307 copy_object (ptr, queue);
5308 DEBUG (9, fprintf (gc_debug_file, "Overwrote remset at %p with %p\n", ptr, *ptr));
5310 binary_protocol_ptr_update (ptr, old, *ptr, (gpointer)LOAD_VTABLE (*ptr), safe_object_get_size (*ptr));
5311 if (!global && *ptr >= start_nursery && *ptr < end_nursery) {
5313 * If the object is pinned, each reference to it from nonpinned objects
5314 * becomes part of the global remset, which can grow very large.
5316 DEBUG (9, fprintf (gc_debug_file, "Add to global remset because of pinning %p (%p %s)\n", ptr, *ptr, safe_name (*ptr)));
5317 add_to_global_remset (ptr);
5320 DEBUG (9, fprintf (gc_debug_file, "Skipping remset at %p holding %p\n", ptr, *ptr));
5324 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
5325 if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery))
5328 while (count-- > 0) {
5329 copy_object (ptr, queue);
5330 DEBUG (9, fprintf (gc_debug_file, "Overwrote remset at %p with %p (count: %d)\n", ptr, *ptr, (int)count));
5331 if (!global && *ptr >= start_nursery && *ptr < end_nursery)
5332 add_to_global_remset (ptr);
5337 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
5338 if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery))
5340 scan_object ((char*)ptr, queue);
5342 case REMSET_VTYPE: {
5343 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
5344 if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery))
5349 ptr = (void**) scan_vtype ((char*)ptr, desc, start_nursery, end_nursery, queue);
5353 g_assert_not_reached ();
5358 #ifdef HEAVY_STATISTICS
5360 collect_store_remsets (RememberedSet *remset, mword *bumper)
5362 mword *p = remset->data;
5367 while (p < remset->store_next) {
5368 switch ((*p) & REMSET_TYPE_MASK) {
5369 case REMSET_LOCATION:
5372 ++stat_saved_remsets_1;
5374 if (*p == last1 || *p == last2) {
5375 ++stat_saved_remsets_2;
5392 g_assert_not_reached ();
5402 RememberedSet *remset;
5404 SgenThreadInfo *info;
5406 mword *addresses, *bumper, *p, *r;
5408 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5409 for (info = thread_table [i]; info; info = info->next) {
5410 for (remset = info->remset; remset; remset = remset->next)
5411 size += remset->store_next - remset->data;
5414 for (remset = freed_thread_remsets; remset; remset = remset->next)
5415 size += remset->store_next - remset->data;
5416 for (remset = global_remset; remset; remset = remset->next)
5417 size += remset->store_next - remset->data;
5419 bumper = addresses = mono_sgen_alloc_internal_dynamic (sizeof (mword) * size, INTERNAL_MEM_STATISTICS);
5421 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5422 for (info = thread_table [i]; info; info = info->next) {
5423 for (remset = info->remset; remset; remset = remset->next)
5424 bumper = collect_store_remsets (remset, bumper);
5427 for (remset = global_remset; remset; remset = remset->next)
5428 bumper = collect_store_remsets (remset, bumper);
5429 for (remset = freed_thread_remsets; remset; remset = remset->next)
5430 bumper = collect_store_remsets (remset, bumper);
5432 g_assert (bumper <= addresses + size);
5434 stat_store_remsets += bumper - addresses;
5436 sort_addresses ((void**)addresses, bumper - addresses);
5439 while (r < bumper) {
5445 stat_store_remsets_unique += p - addresses;
5447 mono_sgen_free_internal_dynamic (addresses, sizeof (mword) * size, INTERNAL_MEM_STATISTICS);
5452 clear_thread_store_remset_buffer (SgenThreadInfo *info)
5454 *info->store_remset_buffer_index_addr = 0;
5455 memset (*info->store_remset_buffer_addr, 0, sizeof (gpointer) * STORE_REMSET_BUFFER_SIZE);
5459 remset_byte_size (RememberedSet *remset)
5461 return sizeof (RememberedSet) + (remset->end_set - remset->data) * sizeof (gpointer);
5465 scan_from_remsets (void *start_nursery, void *end_nursery, GrayQueue *queue)
5468 SgenThreadInfo *info;
5469 RememberedSet *remset;
5470 GenericStoreRememberedSet *store_remset;
5471 mword *p, *next_p, *store_pos;
5473 #ifdef HEAVY_STATISTICS
5477 /* the global one */
5478 for (remset = global_remset; remset; remset = remset->next) {
5479 DEBUG (4, fprintf (gc_debug_file, "Scanning global remset range: %p-%p, size: %td\n", remset->data, remset->store_next, remset->store_next - remset->data));
5480 store_pos = remset->data;
5481 for (p = remset->data; p < remset->store_next; p = next_p) {
5482 void **ptr = (void**)p [0];
5484 /*Ignore previously processed remset.*/
5485 if (!global_remset_location_was_not_added (ptr)) {
5490 next_p = handle_remset (p, start_nursery, end_nursery, TRUE, queue);
5493 * Clear global remsets of locations which no longer point to the
5494 * nursery. Otherwise, they could grow indefinitely between major
5497 * Since all global remsets are location remsets, we don't need to unmask the pointer.
5499 if (ptr_in_nursery (*ptr)) {
5500 *store_pos ++ = p [0];
5501 HEAVY_STAT (++stat_global_remsets_readded);
5505 /* Truncate the remset */
5506 remset->store_next = store_pos;
5509 /* the generic store ones */
5510 store_remset = generic_store_remsets;
5511 while (store_remset) {
5512 GenericStoreRememberedSet *next = store_remset->next;
5514 for (i = 0; i < STORE_REMSET_BUFFER_SIZE - 1; ++i) {
5515 gpointer addr = store_remset->data [i];
5517 handle_remset ((mword*)&addr, start_nursery, end_nursery, FALSE, queue);
5520 mono_sgen_free_internal (store_remset, INTERNAL_MEM_STORE_REMSET);
5522 store_remset = next;
5524 generic_store_remsets = NULL;
5526 /* the per-thread ones */
5527 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5528 for (info = thread_table [i]; info; info = info->next) {
5529 RememberedSet *next;
5531 for (remset = info->remset; remset; remset = next) {
5532 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for thread %p, range: %p-%p, size: %td\n", info, remset->data, remset->store_next, remset->store_next - remset->data));
5533 for (p = remset->data; p < remset->store_next;) {
5534 p = handle_remset (p, start_nursery, end_nursery, FALSE, queue);
5536 remset->store_next = remset->data;
5537 next = remset->next;
5538 remset->next = NULL;
5539 if (remset != info->remset) {
5540 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
5541 mono_sgen_free_internal_dynamic (remset, remset_byte_size (remset), INTERNAL_MEM_REMSET);
5544 for (j = 0; j < *info->store_remset_buffer_index_addr; ++j)
5545 handle_remset ((mword*)*info->store_remset_buffer_addr + j + 1, start_nursery, end_nursery, FALSE, queue);
5546 clear_thread_store_remset_buffer (info);
5550 /* the freed thread ones */
5551 while (freed_thread_remsets) {
5552 RememberedSet *next;
5553 remset = freed_thread_remsets;
5554 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for freed thread, range: %p-%p, size: %td\n", remset->data, remset->store_next, remset->store_next - remset->data));
5555 for (p = remset->data; p < remset->store_next;) {
5556 p = handle_remset (p, start_nursery, end_nursery, FALSE, queue);
5558 next = remset->next;
5559 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
5560 mono_sgen_free_internal_dynamic (remset, remset_byte_size (remset), INTERNAL_MEM_REMSET);
5561 freed_thread_remsets = next;
5566 * Clear the info in the remembered sets: we're doing a major collection, so
5567 * the per-thread ones are not needed and the global ones will be reconstructed
5571 clear_remsets (void)
5574 SgenThreadInfo *info;
5575 RememberedSet *remset, *next;
5577 /* the global list */
5578 for (remset = global_remset; remset; remset = next) {
5579 remset->store_next = remset->data;
5580 next = remset->next;
5581 remset->next = NULL;
5582 if (remset != global_remset) {
5583 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
5584 mono_sgen_free_internal_dynamic (remset, remset_byte_size (remset), INTERNAL_MEM_REMSET);
5587 /* the generic store ones */
5588 while (generic_store_remsets) {
5589 GenericStoreRememberedSet *gs_next = generic_store_remsets->next;
5590 mono_sgen_free_internal (generic_store_remsets, INTERNAL_MEM_STORE_REMSET);
5591 generic_store_remsets = gs_next;
5593 /* the per-thread ones */
5594 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5595 for (info = thread_table [i]; info; info = info->next) {
5596 for (remset = info->remset; remset; remset = next) {
5597 remset->store_next = remset->data;
5598 next = remset->next;
5599 remset->next = NULL;
5600 if (remset != info->remset) {
5601 DEBUG (3, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
5602 mono_sgen_free_internal_dynamic (remset, remset_byte_size (remset), INTERNAL_MEM_REMSET);
5605 clear_thread_store_remset_buffer (info);
5609 /* the freed thread ones */
5610 while (freed_thread_remsets) {
5611 next = freed_thread_remsets->next;
5612 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", freed_thread_remsets->data));
5613 mono_sgen_free_internal_dynamic (freed_thread_remsets, remset_byte_size (freed_thread_remsets), INTERNAL_MEM_REMSET);
5614 freed_thread_remsets = next;
5619 * Clear the thread local TLAB variables for all threads.
5624 SgenThreadInfo *info;
5627 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5628 for (info = thread_table [i]; info; info = info->next) {
5629 /* A new TLAB will be allocated when the thread does its first allocation */
5630 *info->tlab_start_addr = NULL;
5631 *info->tlab_next_addr = NULL;
5632 *info->tlab_temp_end_addr = NULL;
5633 *info->tlab_real_end_addr = NULL;
5638 /* LOCKING: assumes the GC lock is held */
5639 static SgenThreadInfo*
5640 gc_register_current_thread (void *addr)
5643 SgenThreadInfo* info = malloc (sizeof (SgenThreadInfo));
5644 #ifndef HAVE_KW_THREAD
5645 SgenThreadInfo *__thread_info__ = info;
5651 memset (info, 0, sizeof (SgenThreadInfo));
5652 #ifndef HAVE_KW_THREAD
5653 info->tlab_start = info->tlab_next = info->tlab_temp_end = info->tlab_real_end = NULL;
5655 g_assert (!pthread_getspecific (thread_info_key));
5656 pthread_setspecific (thread_info_key, info);
5661 info->id = ARCH_GET_THREAD ();
5662 info->stop_count = -1;
5665 info->stack_start = NULL;
5666 info->tlab_start_addr = &TLAB_START;
5667 info->tlab_next_addr = &TLAB_NEXT;
5668 info->tlab_temp_end_addr = &TLAB_TEMP_END;
5669 info->tlab_real_end_addr = &TLAB_REAL_END;
5670 info->store_remset_buffer_addr = &STORE_REMSET_BUFFER;
5671 info->store_remset_buffer_index_addr = &STORE_REMSET_BUFFER_INDEX;
5672 info->stopped_ip = NULL;
5673 info->stopped_domain = NULL;
5674 info->stopped_regs = NULL;
5676 binary_protocol_thread_register ((gpointer)info->id);
5678 #ifdef HAVE_KW_THREAD
5679 tlab_next_addr = &tlab_next;
5680 store_remset_buffer_index_addr = &store_remset_buffer_index;
5683 /* try to get it with attributes first */
5684 #if defined(HAVE_PTHREAD_GETATTR_NP) && defined(HAVE_PTHREAD_ATTR_GETSTACK)
5688 pthread_attr_t attr;
5689 pthread_getattr_np (pthread_self (), &attr);
5690 pthread_attr_getstack (&attr, &sstart, &size);
5691 info->stack_start_limit = sstart;
5692 info->stack_end = (char*)sstart + size;
5693 pthread_attr_destroy (&attr);
5695 #elif defined(HAVE_PTHREAD_GET_STACKSIZE_NP) && defined(HAVE_PTHREAD_GET_STACKADDR_NP)
5696 info->stack_end = (char*)pthread_get_stackaddr_np (pthread_self ());
5697 info->stack_start_limit = (char*)info->stack_end - pthread_get_stacksize_np (pthread_self ());
5700 /* FIXME: we assume the stack grows down */
5701 gsize stack_bottom = (gsize)addr;
5702 stack_bottom += 4095;
5703 stack_bottom &= ~4095;
5704 info->stack_end = (char*)stack_bottom;
5708 #ifdef HAVE_KW_THREAD
5709 stack_end = info->stack_end;
5712 /* hash into the table */
5713 hash = HASH_PTHREAD_T (info->id) % THREAD_HASH_SIZE;
5714 info->next = thread_table [hash];
5715 thread_table [hash] = info;
5717 info->remset = alloc_remset (DEFAULT_REMSET_SIZE, info);
5718 pthread_setspecific (remembered_set_key, info->remset);
5719 #ifdef HAVE_KW_THREAD
5720 remembered_set = info->remset;
5723 STORE_REMSET_BUFFER = mono_sgen_alloc_internal (INTERNAL_MEM_STORE_REMSET);
5724 STORE_REMSET_BUFFER_INDEX = 0;
5726 DEBUG (3, fprintf (gc_debug_file, "registered thread %p (%p) (hash: %d)\n", info, (gpointer)info->id, hash));
5728 if (gc_callbacks.thread_attach_func)
5729 info->runtime_data = gc_callbacks.thread_attach_func ();
5735 add_generic_store_remset_from_buffer (gpointer *buffer)
5737 GenericStoreRememberedSet *remset = mono_sgen_alloc_internal (INTERNAL_MEM_STORE_REMSET);
5738 memcpy (remset->data, buffer + 1, sizeof (gpointer) * (STORE_REMSET_BUFFER_SIZE - 1));
5739 remset->next = generic_store_remsets;
5740 generic_store_remsets = remset;
5744 unregister_current_thread (void)
5747 SgenThreadInfo *prev = NULL;
5749 RememberedSet *rset;
5750 ARCH_THREAD_TYPE id = ARCH_GET_THREAD ();
5752 binary_protocol_thread_unregister ((gpointer)id);
5754 hash = HASH_PTHREAD_T (id) % THREAD_HASH_SIZE;
5755 p = thread_table [hash];
5757 DEBUG (3, fprintf (gc_debug_file, "unregister thread %p (%p)\n", p, (gpointer)p->id));
5758 while (!ARCH_THREAD_EQUALS (p->id, id)) {
5763 thread_table [hash] = p->next;
5765 prev->next = p->next;
5768 if (freed_thread_remsets) {
5769 for (rset = p->remset; rset->next; rset = rset->next)
5771 rset->next = freed_thread_remsets;
5772 freed_thread_remsets = p->remset;
5774 freed_thread_remsets = p->remset;
5777 if (*p->store_remset_buffer_index_addr)
5778 add_generic_store_remset_from_buffer (*p->store_remset_buffer_addr);
5779 mono_sgen_free_internal (*p->store_remset_buffer_addr, INTERNAL_MEM_STORE_REMSET);
5784 unregister_thread (void *k)
5786 g_assert (!mono_domain_get ());
5788 unregister_current_thread ();
5793 mono_gc_register_thread (void *baseptr)
5795 SgenThreadInfo *info;
5799 info = mono_sgen_thread_info_lookup (ARCH_GET_THREAD ());
5801 info = gc_register_current_thread (baseptr);
5803 return info != NULL;
5806 #if USE_PTHREAD_INTERCEPT
5809 void *(*start_routine) (void *);
5812 MonoSemType registered;
5813 } SgenThreadStartInfo;
5816 gc_start_thread (void *arg)
5818 SgenThreadStartInfo *start_info = arg;
5819 SgenThreadInfo* info;
5820 void *t_arg = start_info->arg;
5821 void *(*start_func) (void*) = start_info->start_routine;
5826 info = gc_register_current_thread (&result);
5828 post_result = MONO_SEM_POST (&(start_info->registered));
5829 g_assert (!post_result);
5830 result = start_func (t_arg);
5831 g_assert (!mono_domain_get ());
5833 * this is done by the pthread key dtor
5835 unregister_current_thread ();
5843 mono_gc_pthread_create (pthread_t *new_thread, const pthread_attr_t *attr, void *(*start_routine)(void *), void *arg)
5845 SgenThreadStartInfo *start_info;
5848 start_info = malloc (sizeof (SgenThreadStartInfo));
5851 MONO_SEM_INIT (&(start_info->registered), 0);
5852 start_info->arg = arg;
5853 start_info->start_routine = start_routine;
5855 result = pthread_create (new_thread, attr, gc_start_thread, start_info);
5857 while (MONO_SEM_WAIT (&(start_info->registered)) != 0) {
5858 /*if (EINTR != errno) ABORT("sem_wait failed"); */
5861 MONO_SEM_DESTROY (&(start_info->registered));
5867 mono_gc_pthread_join (pthread_t thread, void **retval)
5869 return pthread_join (thread, retval);
5873 mono_gc_pthread_detach (pthread_t thread)
5875 return pthread_detach (thread);
5878 #endif /* USE_PTHREAD_INTERCEPT */
5881 * ######################################################################
5882 * ######## Write barriers
5883 * ######################################################################
5886 static RememberedSet*
5887 alloc_remset (int size, gpointer id) {
5888 RememberedSet* res = mono_sgen_alloc_internal_dynamic (sizeof (RememberedSet) + (size * sizeof (gpointer)), INTERNAL_MEM_REMSET);
5889 res->store_next = res->data;
5890 res->end_set = res->data + size;
5892 DEBUG (4, fprintf (gc_debug_file, "Allocated remset size %d at %p for %p\n", size, res->data, id));
5897 * Note: the write barriers first do the needed GC work and then do the actual store:
5898 * this way the value is visible to the conservative GC scan after the write barrier
5899 * itself. If a GC interrupts the barrier in the middle, value will be kept alive by
5900 * the conservative scan, otherwise by the remembered set scan.
5903 mono_gc_wbarrier_set_field (MonoObject *obj, gpointer field_ptr, MonoObject* value)
5907 HEAVY_STAT (++stat_wbarrier_set_field);
5908 if (ptr_in_nursery (field_ptr)) {
5909 *(void**)field_ptr = value;
5912 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", field_ptr));
5914 rs = REMEMBERED_SET;
5915 if (rs->store_next < rs->end_set) {
5916 *(rs->store_next++) = (mword)field_ptr;
5917 *(void**)field_ptr = value;
5921 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
5922 rs->next = REMEMBERED_SET;
5923 REMEMBERED_SET = rs;
5924 #ifdef HAVE_KW_THREAD
5925 mono_sgen_thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
5927 *(rs->store_next++) = (mword)field_ptr;
5928 *(void**)field_ptr = value;
5933 mono_gc_wbarrier_set_arrayref (MonoArray *arr, gpointer slot_ptr, MonoObject* value)
5937 HEAVY_STAT (++stat_wbarrier_set_arrayref);
5938 if (ptr_in_nursery (slot_ptr)) {
5939 *(void**)slot_ptr = value;
5942 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", slot_ptr));
5944 rs = REMEMBERED_SET;
5945 if (rs->store_next < rs->end_set) {
5946 *(rs->store_next++) = (mword)slot_ptr;
5947 *(void**)slot_ptr = value;
5951 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
5952 rs->next = REMEMBERED_SET;
5953 REMEMBERED_SET = rs;
5954 #ifdef HAVE_KW_THREAD
5955 mono_sgen_thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
5957 *(rs->store_next++) = (mword)slot_ptr;
5958 *(void**)slot_ptr = value;
5963 mono_gc_wbarrier_arrayref_copy (gpointer dest_ptr, gpointer src_ptr, int count)
5967 HEAVY_STAT (++stat_wbarrier_arrayref_copy);
5969 memmove (dest_ptr, src_ptr, count * sizeof (gpointer));
5970 if (ptr_in_nursery (dest_ptr)) {
5974 rs = REMEMBERED_SET;
5975 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p, %d\n", dest_ptr, count));
5976 if (rs->store_next + 1 < rs->end_set) {
5977 *(rs->store_next++) = (mword)dest_ptr | REMSET_RANGE;
5978 *(rs->store_next++) = count;
5982 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
5983 rs->next = REMEMBERED_SET;
5984 REMEMBERED_SET = rs;
5985 #ifdef HAVE_KW_THREAD
5986 mono_sgen_thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
5988 *(rs->store_next++) = (mword)dest_ptr | REMSET_RANGE;
5989 *(rs->store_next++) = count;
5993 static char *found_obj;
5996 find_object_for_ptr_callback (char *obj, size_t size, char *ptr)
5998 if (ptr >= obj && ptr < obj + size) {
5999 g_assert (!found_obj);
6004 /* for use in the debugger */
6005 char* find_object_for_ptr (char *ptr);
6007 find_object_for_ptr (char *ptr)
6011 if (ptr >= nursery_section->data && ptr < nursery_section->end_data) {
6013 scan_area_with_callback (nursery_section->data, nursery_section->end_data,
6014 (IterateObjectCallbackFunc)find_object_for_ptr_callback, ptr);
6019 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
6020 if (ptr >= bigobj->data && ptr < bigobj->data + bigobj->size)
6021 return bigobj->data;
6025 * Very inefficient, but this is debugging code, supposed to
6026 * be called from gdb, so we don't care.
6029 major_iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)find_object_for_ptr_callback, ptr);
6034 evacuate_remset_buffer (void)
6039 buffer = STORE_REMSET_BUFFER;
6041 add_generic_store_remset_from_buffer (buffer);
6042 memset (buffer, 0, sizeof (gpointer) * STORE_REMSET_BUFFER_SIZE);
6044 STORE_REMSET_BUFFER_INDEX = 0;
6048 mono_gc_wbarrier_generic_nostore (gpointer ptr)
6054 HEAVY_STAT (++stat_wbarrier_generic_store);
6056 #ifdef XDOMAIN_CHECKS_IN_WBARRIER
6057 /* FIXME: ptr_in_heap must be called with the GC lock held */
6058 if (xdomain_checks && *(MonoObject**)ptr && ptr_in_heap (ptr)) {
6059 char *start = find_object_for_ptr (ptr);
6060 MonoObject *value = *(MonoObject**)ptr;
6064 MonoObject *obj = (MonoObject*)start;
6065 if (obj->vtable->domain != value->vtable->domain)
6066 g_assert (is_xdomain_ref_allowed (ptr, start, obj->vtable->domain));
6074 if (*(gpointer*)ptr)
6075 binary_protocol_wbarrier (ptr, *(gpointer*)ptr, (gpointer)LOAD_VTABLE (*(gpointer*)ptr));
6077 if (ptr_in_nursery (ptr) || ptr_on_stack (ptr) || !ptr_in_nursery (*(gpointer*)ptr)) {
6078 DEBUG (8, fprintf (gc_debug_file, "Skipping remset at %p\n", ptr));
6083 buffer = STORE_REMSET_BUFFER;
6084 index = STORE_REMSET_BUFFER_INDEX;
6085 /* This simple optimization eliminates a sizable portion of
6086 entries. Comparing it to the last but one entry as well
6087 doesn't eliminate significantly more entries. */
6088 if (buffer [index] == ptr) {
6093 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", ptr));
6094 HEAVY_STAT (++stat_wbarrier_generic_store_remset);
6097 if (index >= STORE_REMSET_BUFFER_SIZE) {
6098 evacuate_remset_buffer ();
6099 index = STORE_REMSET_BUFFER_INDEX;
6100 g_assert (index == 0);
6103 buffer [index] = ptr;
6104 STORE_REMSET_BUFFER_INDEX = index;
6110 mono_gc_wbarrier_generic_store (gpointer ptr, MonoObject* value)
6112 DEBUG (8, fprintf (gc_debug_file, "Wbarrier store at %p to %p (%s)\n", ptr, value, value ? safe_name (value) : "null"));
6113 *(void**)ptr = value;
6114 if (ptr_in_nursery (value))
6115 mono_gc_wbarrier_generic_nostore (ptr);
6118 void mono_gc_wbarrier_value_copy_bitmap (gpointer _dest, gpointer _src, int size, unsigned bitmap)
6120 mword *dest = _dest;
6125 mono_gc_wbarrier_generic_store (dest, (MonoObject*)*src);
6130 size -= SIZEOF_VOID_P;
6137 mono_gc_wbarrier_value_copy (gpointer dest, gpointer src, int count, MonoClass *klass)
6141 HEAVY_STAT (++stat_wbarrier_value_copy);
6142 g_assert (klass->valuetype);
6144 memmove (dest, src, count * mono_class_value_size (klass, NULL));
6145 rs = REMEMBERED_SET;
6146 if (ptr_in_nursery (dest) || ptr_on_stack (dest) || !klass->has_references) {
6150 g_assert (klass->gc_descr_inited);
6151 DEBUG (8, fprintf (gc_debug_file, "Adding value remset at %p, count %d, descr %p for class %s (%p)\n", dest, count, klass->gc_descr, klass->name, klass));
6153 if (rs->store_next + 3 < rs->end_set) {
6154 *(rs->store_next++) = (mword)dest | REMSET_VTYPE;
6155 *(rs->store_next++) = (mword)klass->gc_descr;
6156 *(rs->store_next++) = (mword)count;
6160 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6161 rs->next = REMEMBERED_SET;
6162 REMEMBERED_SET = rs;
6163 #ifdef HAVE_KW_THREAD
6164 mono_sgen_thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6166 *(rs->store_next++) = (mword)dest | REMSET_VTYPE;
6167 *(rs->store_next++) = (mword)klass->gc_descr;
6168 *(rs->store_next++) = (mword)count;
6173 * mono_gc_wbarrier_object_copy:
6175 * Write barrier to call when obj is the result of a clone or copy of an object.
6178 mono_gc_wbarrier_object_copy (MonoObject* obj, MonoObject *src)
6184 HEAVY_STAT (++stat_wbarrier_object_copy);
6185 rs = REMEMBERED_SET;
6186 DEBUG (6, fprintf (gc_debug_file, "Adding object remset for %p\n", obj));
6187 size = mono_object_class (obj)->instance_size;
6189 /* do not copy the sync state */
6190 memcpy ((char*)obj + sizeof (MonoObject), (char*)src + sizeof (MonoObject),
6191 size - sizeof (MonoObject));
6192 if (ptr_in_nursery (obj) || ptr_on_stack (obj)) {
6196 if (rs->store_next < rs->end_set) {
6197 *(rs->store_next++) = (mword)obj | REMSET_OBJECT;
6201 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6202 rs->next = REMEMBERED_SET;
6203 REMEMBERED_SET = rs;
6204 #ifdef HAVE_KW_THREAD
6205 mono_sgen_thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6207 *(rs->store_next++) = (mword)obj | REMSET_OBJECT;
6212 * ######################################################################
6213 * ######## Collector debugging
6214 * ######################################################################
6217 const char*descriptor_types [] = {
6229 describe_ptr (char *ptr)
6235 if (ptr_in_nursery (ptr)) {
6236 printf ("Pointer inside nursery.\n");
6238 if (major_ptr_is_in_non_pinned_space (ptr)) {
6239 printf ("Pointer inside oldspace.\n");
6240 } else if (obj_is_from_pinned_alloc (ptr)) {
6241 printf ("Pointer is inside a pinned chunk.\n");
6243 printf ("Pointer unknown.\n");
6248 if (object_is_pinned (ptr))
6249 printf ("Object is pinned.\n");
6251 if (object_is_forwarded (ptr))
6252 printf ("Object is forwared.\n");
6254 // FIXME: Handle pointers to the inside of objects
6255 vtable = (MonoVTable*)LOAD_VTABLE (ptr);
6257 printf ("VTable: %p\n", vtable);
6258 if (vtable == NULL) {
6259 printf ("VTable is invalid (empty).\n");
6262 if (ptr_in_nursery (vtable)) {
6263 printf ("VTable is invalid (points inside nursery).\n");
6266 printf ("Class: %s\n", vtable->klass->name);
6268 desc = ((GCVTable*)vtable)->desc;
6269 printf ("Descriptor: %lx\n", (long)desc);
6272 printf ("Descriptor type: %d (%s)\n", type, descriptor_types [type]);
6276 find_in_remset_loc (mword *p, char *addr, gboolean *found)
6282 switch ((*p) & REMSET_TYPE_MASK) {
6283 case REMSET_LOCATION:
6284 if (*p == (mword)addr)
6288 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
6290 if ((void**)addr >= ptr && (void**)addr < ptr + count)
6294 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
6295 count = safe_object_get_size ((MonoObject*)ptr);
6296 count = ALIGN_UP (count);
6297 count /= sizeof (mword);
6298 if ((void**)addr >= ptr && (void**)addr < ptr + count)
6302 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
6306 switch (desc & 0x7) {
6307 case DESC_TYPE_RUN_LENGTH:
6308 OBJ_RUN_LEN_SIZE (skip_size, desc, ptr);
6310 case DESC_TYPE_SMALL_BITMAP:
6311 OBJ_BITMAP_SIZE (skip_size, desc, start);
6315 g_assert_not_reached ();
6318 /* The descriptor includes the size of MonoObject */
6319 skip_size -= sizeof (MonoObject);
6321 if ((void**)addr >= ptr && (void**)addr < ptr + (skip_size / sizeof (gpointer)))
6326 g_assert_not_reached ();
6332 * Return whenever ADDR occurs in the remembered sets
6335 find_in_remsets (char *addr)
6338 SgenThreadInfo *info;
6339 RememberedSet *remset;
6340 GenericStoreRememberedSet *store_remset;
6342 gboolean found = FALSE;
6344 /* the global one */
6345 for (remset = global_remset; remset; remset = remset->next) {
6346 DEBUG (4, fprintf (gc_debug_file, "Scanning global remset range: %p-%p, size: %td\n", remset->data, remset->store_next, remset->store_next - remset->data));
6347 for (p = remset->data; p < remset->store_next;) {
6348 p = find_in_remset_loc (p, addr, &found);
6354 /* the generic store ones */
6355 for (store_remset = generic_store_remsets; store_remset; store_remset = store_remset->next) {
6356 for (i = 0; i < STORE_REMSET_BUFFER_SIZE - 1; ++i) {
6357 if (store_remset->data [i] == addr)
6362 /* the per-thread ones */
6363 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
6364 for (info = thread_table [i]; info; info = info->next) {
6366 for (remset = info->remset; remset; remset = remset->next) {
6367 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for thread %p, range: %p-%p, size: %td\n", info, remset->data, remset->store_next, remset->store_next - remset->data));
6368 for (p = remset->data; p < remset->store_next;) {
6369 p = find_in_remset_loc (p, addr, &found);
6374 for (j = 0; j < *info->store_remset_buffer_index_addr; ++j) {
6375 if ((*info->store_remset_buffer_addr) [j + 1] == addr)
6381 /* the freed thread ones */
6382 for (remset = freed_thread_remsets; remset; remset = remset->next) {
6383 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for freed thread, range: %p-%p, size: %td\n", remset->data, remset->store_next, remset->store_next - remset->data));
6384 for (p = remset->data; p < remset->store_next;) {
6385 p = find_in_remset_loc (p, addr, &found);
6394 static gboolean missing_remsets;
6397 * We let a missing remset slide if the target object is pinned,
6398 * because the store might have happened but the remset not yet added,
6399 * but in that case the target must be pinned. We might theoretically
6400 * miss some missing remsets this way, but it's very unlikely.
6403 #define HANDLE_PTR(ptr,obj) do { \
6404 if (*(ptr) && (char*)*(ptr) >= nursery_start && (char*)*(ptr) < nursery_next) { \
6405 if (!find_in_remsets ((char*)(ptr))) { \
6406 fprintf (gc_debug_file, "Oldspace->newspace reference %p at offset %td in object %p (%s.%s) not found in remsets.\n", *(ptr), (char*)(ptr) - (char*)(obj), (obj), ((MonoObject*)(obj))->vtable->klass->name_space, ((MonoObject*)(obj))->vtable->klass->name); \
6407 binary_protocol_missing_remset ((obj), (gpointer)LOAD_VTABLE ((obj)), (char*)(ptr) - (char*)(obj), *(ptr), (gpointer)LOAD_VTABLE(*(ptr)), object_is_pinned (*(ptr))); \
6408 if (!object_is_pinned (*(ptr))) \
6409 missing_remsets = TRUE; \
6415 * Check that each object reference which points into the nursery can
6416 * be found in the remembered sets.
6419 check_consistency_callback (char *start, size_t size, void *dummy)
6421 GCVTable *vt = (GCVTable*)LOAD_VTABLE (start);
6422 DEBUG (8, fprintf (gc_debug_file, "Scanning object %p, vtable: %p (%s)\n", start, vt, vt->klass->name));
6424 #define SCAN_OBJECT_ACTION
6425 #include "sgen-scan-object.h"
6429 * Perform consistency check of the heap.
6431 * Assumes the world is stopped.
6434 check_consistency (void)
6438 // Need to add more checks
6440 missing_remsets = FALSE;
6442 DEBUG (1, fprintf (gc_debug_file, "Begin heap consistency check...\n"));
6444 // Check that oldspace->newspace pointers are registered with the collector
6445 major_iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)check_consistency_callback, NULL);
6447 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
6448 check_consistency_callback (bigobj->data, bigobj->size, NULL);
6450 DEBUG (1, fprintf (gc_debug_file, "Heap consistency check done.\n"));
6452 #ifdef BINARY_PROTOCOL
6453 if (!binary_protocol_file)
6455 g_assert (!missing_remsets);
6460 #define HANDLE_PTR(ptr,obj) do { \
6462 g_assert (LOAD_VTABLE (*(ptr))); \
6466 check_major_refs_callback (char *start, size_t size, void *dummy)
6468 #define SCAN_OBJECT_ACTION
6469 #include "sgen-scan-object.h"
6473 check_major_refs (void)
6477 major_iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)check_major_refs_callback, NULL);
6479 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
6480 check_major_refs_callback (bigobj->data, bigobj->size, NULL);
6483 /* Check that the reference is valid */
6485 #define HANDLE_PTR(ptr,obj) do { \
6487 g_assert (safe_name (*(ptr)) != NULL); \
6494 * Perform consistency check on an object. Currently we only check that the
6495 * reference fields are valid.
6498 check_object (char *start)
6503 #include "sgen-scan-object.h"
6507 * ######################################################################
6508 * ######## Other mono public interface functions.
6509 * ######################################################################
6513 mono_gc_collect (int generation)
6517 if (generation == 0) {
6518 collect_nursery (0);
6520 major_collection ("user request");
6527 mono_gc_max_generation (void)
6533 mono_gc_collection_count (int generation)
6535 if (generation == 0)
6536 return num_minor_gcs;
6537 return num_major_gcs;
6541 mono_gc_get_used_size (void)
6545 tot = los_memory_usage;
6546 tot += nursery_section->next_data - nursery_section->data;
6547 tot += major_get_used_size ();
6548 /* FIXME: account for pinned objects */
6554 mono_gc_get_heap_size (void)
6560 mono_gc_disable (void)
6568 mono_gc_enable (void)
6576 mono_gc_get_los_limit (void)
6578 return MAX_SMALL_OBJ_SIZE;
6582 mono_object_is_alive (MonoObject* o)
6588 mono_gc_get_generation (MonoObject *obj)
6590 if (ptr_in_nursery (obj))
6596 mono_gc_enable_events (void)
6601 mono_gc_weak_link_add (void **link_addr, MonoObject *obj, gboolean track)
6604 mono_gc_register_disappearing_link (obj, link_addr, track);
6609 mono_gc_weak_link_remove (void **link_addr)
6612 mono_gc_register_disappearing_link (NULL, link_addr, FALSE);
6617 mono_gc_weak_link_get (void **link_addr)
6621 return (MonoObject*) REVEAL_POINTER (*link_addr);
6625 mono_gc_ephemeron_array_add (MonoObject *obj)
6627 EphemeronLinkNode *node;
6631 node = mono_sgen_alloc_internal (INTERNAL_MEM_EPHEMERON_LINK);
6636 node->array = (char*)obj;
6637 node->next = ephemeron_list;
6638 ephemeron_list = node;
6640 DEBUG (5, fprintf (gc_debug_file, "Registered ephemeron array %p\n", obj));
6647 mono_gc_make_descr_from_bitmap (gsize *bitmap, int numbits)
6649 if (numbits < ((sizeof (*bitmap) * 8) - ROOT_DESC_TYPE_SHIFT)) {
6650 return (void*)MAKE_ROOT_DESC (ROOT_DESC_BITMAP, bitmap [0]);
6652 mword complex = alloc_complex_descriptor (bitmap, numbits);
6653 return (void*)MAKE_ROOT_DESC (ROOT_DESC_COMPLEX, complex);
6658 mono_gc_make_root_descr_user (MonoGCRootMarkFunc marker)
6662 g_assert (user_descriptors_next < MAX_USER_DESCRIPTORS);
6663 descr = (void*)MAKE_ROOT_DESC (ROOT_DESC_USER, (mword)user_descriptors_next);
6664 user_descriptors [user_descriptors_next ++] = marker;
6670 mono_gc_alloc_fixed (size_t size, void *descr)
6672 /* FIXME: do a single allocation */
6673 void *res = calloc (1, size);
6676 if (!mono_gc_register_root (res, size, descr)) {
6684 mono_gc_free_fixed (void* addr)
6686 mono_gc_deregister_root (addr);
6691 mono_gc_invoke_with_gc_lock (MonoGCLockedCallbackFunc func, void *data)
6695 result = func (data);
6696 UNLOCK_INTERRUPTION;
6701 mono_gc_is_gc_thread (void)
6705 result = mono_sgen_thread_info_lookup (ARCH_GET_THREAD ()) != NULL;
6712 /* Tries to extract a number from the passed string, taking in to account m, k
6715 parse_environment_string_extract_number (gchar *str, glong *out)
6718 int len = strlen (str), shift = 0;
6720 gboolean is_suffix = FALSE;
6723 switch (str [len - 1]) {
6734 suffix = str [len - 1];
6739 val = strtol (str, &endptr, 10);
6741 if ((errno == ERANGE && (val == LONG_MAX || val == LONG_MIN))
6742 || (errno != 0 && val == 0) || (endptr == str))
6746 if (*(endptr + 1)) /* Invalid string. */
6758 mono_gc_base_init (void)
6762 struct sigaction sinfo;
6764 LOCK_INIT (gc_mutex);
6766 if (gc_initialized) {
6770 pagesize = mono_pagesize ();
6771 gc_debug_file = stderr;
6775 if ((env = getenv ("MONO_GC_PARAMS"))) {
6776 if (g_str_has_prefix (env, "nursery-size")) {
6779 while (env [index] && env [index++] != '=')
6781 if (env [index] && parse_environment_string_extract_number (env
6783 default_nursery_size = val;
6784 #ifdef ALIGN_NURSERY
6785 if ((val & (val - 1))) {
6786 fprintf (stderr, "The nursery size must be a power of two.\n");
6790 default_nursery_bits = 0;
6791 while (1 << (++ default_nursery_bits) != default_nursery_size)
6795 fprintf (stderr, "nursery-size must be an integer.\n");
6799 fprintf (stderr, "MONO_GC_PARAMS must be of the form 'nursery-size=N' (where N is an integer, possibly with a k, m or a g suffix).\n");
6806 nursery_size = DEFAULT_NURSERY_SIZE;
6807 minor_collection_allowance = MIN_MINOR_COLLECTION_ALLOWANCE;
6810 mono_sgen_init_internal_allocator ();
6813 mono_sgen_register_fixed_internal_mem_type (INTERNAL_MEM_FRAGMENT, sizeof (Fragment));
6814 mono_sgen_register_fixed_internal_mem_type (INTERNAL_MEM_SECTION, SIZEOF_GC_MEM_SECTION);
6815 mono_sgen_register_fixed_internal_mem_type (INTERNAL_MEM_FINALIZE_ENTRY, sizeof (FinalizeEntry));
6816 mono_sgen_register_fixed_internal_mem_type (INTERNAL_MEM_DISLINK, sizeof (DisappearingLink));
6817 mono_sgen_register_fixed_internal_mem_type (INTERNAL_MEM_ROOT_RECORD, sizeof (RootRecord));
6818 mono_sgen_register_fixed_internal_mem_type (INTERNAL_MEM_GRAY_QUEUE, sizeof (GrayQueueSection));
6819 g_assert (sizeof (GenericStoreRememberedSet) == sizeof (gpointer) * STORE_REMSET_BUFFER_SIZE);
6820 mono_sgen_register_fixed_internal_mem_type (INTERNAL_MEM_STORE_REMSET, sizeof (GenericStoreRememberedSet));
6821 mono_sgen_register_fixed_internal_mem_type (INTERNAL_MEM_EPHEMERON_LINK, sizeof (EphemeronLinkNode));
6823 if ((env = getenv ("MONO_GC_DEBUG"))) {
6824 opts = g_strsplit (env, ",", -1);
6825 for (ptr = opts; ptr && *ptr; ptr ++) {
6827 if (opt [0] >= '0' && opt [0] <= '9') {
6828 gc_debug_level = atoi (opt);
6833 char *rf = g_strdup_printf ("%s.%d", opt, getpid ());
6834 gc_debug_file = fopen (rf, "wb");
6836 gc_debug_file = stderr;
6839 } else if (!strcmp (opt, "collect-before-allocs")) {
6840 collect_before_allocs = TRUE;
6841 } else if (!strcmp (opt, "check-at-minor-collections")) {
6842 consistency_check_at_minor_collection = TRUE;
6843 nursery_clear_policy = CLEAR_AT_GC;
6844 } else if (!strcmp (opt, "xdomain-checks")) {
6845 xdomain_checks = TRUE;
6846 } else if (!strcmp (opt, "clear-at-gc")) {
6847 nursery_clear_policy = CLEAR_AT_GC;
6848 } else if (!strcmp (opt, "conservative-stack-mark")) {
6849 conservative_stack_mark = TRUE;
6850 } else if (!strcmp (opt, "check-scan-starts")) {
6851 do_scan_starts_check = TRUE;
6852 } else if (g_str_has_prefix (opt, "heap-dump=")) {
6853 char *filename = strchr (opt, '=') + 1;
6854 nursery_clear_policy = CLEAR_AT_GC;
6855 heap_dump_file = fopen (filename, "w");
6857 fprintf (heap_dump_file, "<sgen-dump>\n");
6858 #ifdef BINARY_PROTOCOL
6859 } else if (g_str_has_prefix (opt, "binary-protocol=")) {
6860 char *filename = strchr (opt, '=') + 1;
6861 binary_protocol_file = fopen (filename, "w");
6864 fprintf (stderr, "Invalid format for the MONO_GC_DEBUG env variable: '%s'\n", env);
6865 fprintf (stderr, "The format is: MONO_GC_DEBUG=[l[:filename]|<option>]+ where l is a debug level 0-9.\n");
6866 fprintf (stderr, "Valid options are: collect-before-allocs, check-at-minor-collections, xdomain-checks, clear-at-gc.\n");
6873 suspend_ack_semaphore_ptr = &suspend_ack_semaphore;
6874 MONO_SEM_INIT (&suspend_ack_semaphore, 0);
6876 sigfillset (&sinfo.sa_mask);
6877 sinfo.sa_flags = SA_RESTART | SA_SIGINFO;
6878 sinfo.sa_sigaction = suspend_handler;
6879 if (sigaction (suspend_signal_num, &sinfo, NULL) != 0) {
6880 g_error ("failed sigaction");
6883 sinfo.sa_handler = restart_handler;
6884 if (sigaction (restart_signal_num, &sinfo, NULL) != 0) {
6885 g_error ("failed sigaction");
6888 sigfillset (&suspend_signal_mask);
6889 sigdelset (&suspend_signal_mask, restart_signal_num);
6891 global_remset = alloc_remset (1024, NULL);
6892 global_remset->next = NULL;
6894 pthread_key_create (&remembered_set_key, unregister_thread);
6896 #ifndef HAVE_KW_THREAD
6897 pthread_key_create (&thread_info_key, NULL);
6900 gc_initialized = TRUE;
6902 mono_gc_register_thread (&sinfo);
6906 mono_gc_get_suspend_signal (void)
6908 return suspend_signal_num;
6918 #ifdef HAVE_KW_THREAD
6919 #define EMIT_TLS_ACCESS(mb,dummy,offset) do { \
6920 mono_mb_emit_byte ((mb), MONO_CUSTOM_PREFIX); \
6921 mono_mb_emit_byte ((mb), CEE_MONO_TLS); \
6922 mono_mb_emit_i4 ((mb), (offset)); \
6925 #define EMIT_TLS_ACCESS(mb,member,dummy) do { \
6926 mono_mb_emit_byte ((mb), MONO_CUSTOM_PREFIX); \
6927 mono_mb_emit_byte ((mb), CEE_MONO_TLS); \
6928 mono_mb_emit_i4 ((mb), thread_info_key); \
6929 mono_mb_emit_icon ((mb), G_STRUCT_OFFSET (SgenThreadInfo, member)); \
6930 mono_mb_emit_byte ((mb), CEE_ADD); \
6931 mono_mb_emit_byte ((mb), CEE_LDIND_I); \
6935 #ifdef MANAGED_ALLOCATION
6936 /* FIXME: Do this in the JIT, where specialized allocation sequences can be created
6937 * for each class. This is currently not easy to do, as it is hard to generate basic
6938 * blocks + branches, but it is easy with the linear IL codebase.
6940 * For this to work we'd need to solve the TLAB race, first. Now we
6941 * require the allocator to be in a few known methods to make sure
6942 * that they are executed atomically via the restart mechanism.
6945 create_allocator (int atype)
6947 int p_var, size_var;
6948 guint32 slowpath_branch, max_size_branch;
6949 MonoMethodBuilder *mb;
6951 MonoMethodSignature *csig;
6952 static gboolean registered = FALSE;
6953 int tlab_next_addr_var, new_next_var;
6955 const char *name = NULL;
6956 AllocatorWrapperInfo *info;
6958 #ifdef HAVE_KW_THREAD
6959 int tlab_next_addr_offset = -1;
6960 int tlab_temp_end_offset = -1;
6962 MONO_THREAD_VAR_OFFSET (tlab_next_addr, tlab_next_addr_offset);
6963 MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
6965 g_assert (tlab_next_addr_offset != -1);
6966 g_assert (tlab_temp_end_offset != -1);
6970 mono_register_jit_icall (mono_gc_alloc_obj, "mono_gc_alloc_obj", mono_create_icall_signature ("object ptr int"), FALSE);
6971 mono_register_jit_icall (mono_gc_alloc_vector, "mono_gc_alloc_vector", mono_create_icall_signature ("object ptr int int"), FALSE);
6975 if (atype == ATYPE_SMALL) {
6977 name = "AllocSmall";
6978 } else if (atype == ATYPE_NORMAL) {
6981 } else if (atype == ATYPE_VECTOR) {
6983 name = "AllocVector";
6985 g_assert_not_reached ();
6988 csig = mono_metadata_signature_alloc (mono_defaults.corlib, num_params);
6989 csig->ret = &mono_defaults.object_class->byval_arg;
6990 for (i = 0; i < num_params; ++i)
6991 csig->params [i] = &mono_defaults.int_class->byval_arg;
6993 mb = mono_mb_new (mono_defaults.object_class, name, MONO_WRAPPER_ALLOC);
6994 size_var = mono_mb_add_local (mb, &mono_defaults.int32_class->byval_arg);
6995 if (atype == ATYPE_NORMAL || atype == ATYPE_SMALL) {
6996 /* size = vtable->klass->instance_size; */
6997 mono_mb_emit_ldarg (mb, 0);
6998 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoVTable, klass));
6999 mono_mb_emit_byte (mb, CEE_ADD);
7000 mono_mb_emit_byte (mb, CEE_LDIND_I);
7001 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoClass, instance_size));
7002 mono_mb_emit_byte (mb, CEE_ADD);
7003 /* FIXME: assert instance_size stays a 4 byte integer */
7004 mono_mb_emit_byte (mb, CEE_LDIND_U4);
7005 mono_mb_emit_stloc (mb, size_var);
7006 } else if (atype == ATYPE_VECTOR) {
7007 MonoExceptionClause *clause;
7009 MonoClass *oom_exc_class;
7012 /* n > MONO_ARRAY_MAX_INDEX -> OverflowException */
7013 mono_mb_emit_ldarg (mb, 1);
7014 mono_mb_emit_icon (mb, MONO_ARRAY_MAX_INDEX);
7015 pos = mono_mb_emit_short_branch (mb, CEE_BLE_UN_S);
7016 mono_mb_emit_exception (mb, "OverflowException", NULL);
7017 mono_mb_patch_short_branch (mb, pos);
7019 clause = mono_image_alloc0 (mono_defaults.corlib, sizeof (MonoExceptionClause));
7020 clause->try_offset = mono_mb_get_label (mb);
7022 /* vtable->klass->sizes.element_size */
7023 mono_mb_emit_ldarg (mb, 0);
7024 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoVTable, klass));
7025 mono_mb_emit_byte (mb, CEE_ADD);
7026 mono_mb_emit_byte (mb, CEE_LDIND_I);
7027 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoClass, sizes.element_size));
7028 mono_mb_emit_byte (mb, CEE_ADD);
7029 mono_mb_emit_byte (mb, CEE_LDIND_U4);
7032 mono_mb_emit_ldarg (mb, 1);
7033 mono_mb_emit_byte (mb, CEE_MUL_OVF_UN);
7034 /* + sizeof (MonoArray) */
7035 mono_mb_emit_icon (mb, sizeof (MonoArray));
7036 mono_mb_emit_byte (mb, CEE_ADD_OVF_UN);
7037 mono_mb_emit_stloc (mb, size_var);
7039 pos_leave = mono_mb_emit_branch (mb, CEE_LEAVE);
7042 clause->flags = MONO_EXCEPTION_CLAUSE_NONE;
7043 clause->try_len = mono_mb_get_pos (mb) - clause->try_offset;
7044 clause->data.catch_class = mono_class_from_name (mono_defaults.corlib,
7045 "System", "OverflowException");
7046 g_assert (clause->data.catch_class);
7047 clause->handler_offset = mono_mb_get_label (mb);
7049 oom_exc_class = mono_class_from_name (mono_defaults.corlib,
7050 "System", "OutOfMemoryException");
7051 g_assert (oom_exc_class);
7052 ctor = mono_class_get_method_from_name (oom_exc_class, ".ctor", 0);
7055 mono_mb_emit_byte (mb, CEE_POP);
7056 mono_mb_emit_op (mb, CEE_NEWOBJ, ctor);
7057 mono_mb_emit_byte (mb, CEE_THROW);
7059 clause->handler_len = mono_mb_get_pos (mb) - clause->handler_offset;
7060 mono_mb_set_clauses (mb, 1, clause);
7061 mono_mb_patch_branch (mb, pos_leave);
7064 g_assert_not_reached ();
7067 /* size += ALLOC_ALIGN - 1; */
7068 mono_mb_emit_ldloc (mb, size_var);
7069 mono_mb_emit_icon (mb, ALLOC_ALIGN - 1);
7070 mono_mb_emit_byte (mb, CEE_ADD);
7071 /* size &= ~(ALLOC_ALIGN - 1); */
7072 mono_mb_emit_icon (mb, ~(ALLOC_ALIGN - 1));
7073 mono_mb_emit_byte (mb, CEE_AND);
7074 mono_mb_emit_stloc (mb, size_var);
7076 /* if (size > MAX_SMALL_OBJ_SIZE) goto slowpath */
7077 if (atype != ATYPE_SMALL) {
7078 mono_mb_emit_ldloc (mb, size_var);
7079 mono_mb_emit_icon (mb, MAX_SMALL_OBJ_SIZE);
7080 max_size_branch = mono_mb_emit_short_branch (mb, MONO_CEE_BGT_S);
7084 * We need to modify tlab_next, but the JIT only supports reading, so we read
7085 * another tls var holding its address instead.
7088 /* tlab_next_addr (local) = tlab_next_addr (TLS var) */
7089 tlab_next_addr_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7090 EMIT_TLS_ACCESS (mb, tlab_next_addr, tlab_next_addr_offset);
7091 mono_mb_emit_stloc (mb, tlab_next_addr_var);
7093 /* p = (void**)tlab_next; */
7094 p_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7095 mono_mb_emit_ldloc (mb, tlab_next_addr_var);
7096 mono_mb_emit_byte (mb, CEE_LDIND_I);
7097 mono_mb_emit_stloc (mb, p_var);
7099 /* new_next = (char*)p + size; */
7100 new_next_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7101 mono_mb_emit_ldloc (mb, p_var);
7102 mono_mb_emit_ldloc (mb, size_var);
7103 mono_mb_emit_byte (mb, CEE_CONV_I);
7104 mono_mb_emit_byte (mb, CEE_ADD);
7105 mono_mb_emit_stloc (mb, new_next_var);
7107 /* tlab_next = new_next */
7108 mono_mb_emit_ldloc (mb, tlab_next_addr_var);
7109 mono_mb_emit_ldloc (mb, new_next_var);
7110 mono_mb_emit_byte (mb, CEE_STIND_I);
7112 /* if (G_LIKELY (new_next < tlab_temp_end)) */
7113 mono_mb_emit_ldloc (mb, new_next_var);
7114 EMIT_TLS_ACCESS (mb, tlab_temp_end, tlab_temp_end_offset);
7115 slowpath_branch = mono_mb_emit_short_branch (mb, MONO_CEE_BLT_UN_S);
7118 if (atype != ATYPE_SMALL)
7119 mono_mb_patch_short_branch (mb, max_size_branch);
7121 mono_mb_emit_byte (mb, MONO_CUSTOM_PREFIX);
7122 mono_mb_emit_byte (mb, CEE_MONO_NOT_TAKEN);
7124 /* FIXME: mono_gc_alloc_obj takes a 'size_t' as an argument, not an int32 */
7125 mono_mb_emit_ldarg (mb, 0);
7126 mono_mb_emit_ldloc (mb, size_var);
7127 if (atype == ATYPE_NORMAL || atype == ATYPE_SMALL) {
7128 mono_mb_emit_icall (mb, mono_gc_alloc_obj);
7129 } else if (atype == ATYPE_VECTOR) {
7130 mono_mb_emit_ldarg (mb, 1);
7131 mono_mb_emit_icall (mb, mono_gc_alloc_vector);
7133 g_assert_not_reached ();
7135 mono_mb_emit_byte (mb, CEE_RET);
7138 mono_mb_patch_short_branch (mb, slowpath_branch);
7140 /* FIXME: Memory barrier */
7143 mono_mb_emit_ldloc (mb, p_var);
7144 mono_mb_emit_ldarg (mb, 0);
7145 mono_mb_emit_byte (mb, CEE_STIND_I);
7147 if (atype == ATYPE_VECTOR) {
7148 /* arr->max_length = max_length; */
7149 mono_mb_emit_ldloc (mb, p_var);
7150 mono_mb_emit_ldflda (mb, G_STRUCT_OFFSET (MonoArray, max_length));
7151 mono_mb_emit_ldarg (mb, 1);
7152 mono_mb_emit_byte (mb, CEE_STIND_I);
7156 mono_mb_emit_ldloc (mb, p_var);
7157 mono_mb_emit_byte (mb, CEE_RET);
7159 res = mono_mb_create_method (mb, csig, 8);
7161 mono_method_get_header (res)->init_locals = FALSE;
7163 info = mono_image_alloc0 (mono_defaults.corlib, sizeof (AllocatorWrapperInfo));
7164 info->alloc_type = atype;
7165 mono_marshal_set_wrapper_info (res, info);
7171 static MonoMethod* alloc_method_cache [ATYPE_NUM];
7172 static MonoMethod *write_barrier_method;
7175 is_ip_in_managed_allocator (MonoDomain *domain, gpointer ip)
7183 ji = mono_jit_info_table_find (domain, ip);
7186 method = ji->method;
7188 if (method == write_barrier_method)
7190 for (i = 0; i < ATYPE_NUM; ++i)
7191 if (method == alloc_method_cache [i])
7197 * Generate an allocator method implementing the fast path of mono_gc_alloc_obj ().
7198 * The signature of the called method is:
7199 * object allocate (MonoVTable *vtable)
7202 mono_gc_get_managed_allocator (MonoVTable *vtable, gboolean for_box)
7204 #ifdef MANAGED_ALLOCATION
7205 MonoClass *klass = vtable->klass;
7207 #ifdef HAVE_KW_THREAD
7208 int tlab_next_offset = -1;
7209 int tlab_temp_end_offset = -1;
7210 MONO_THREAD_VAR_OFFSET (tlab_next, tlab_next_offset);
7211 MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
7213 if (tlab_next_offset == -1 || tlab_temp_end_offset == -1)
7217 if (!mono_runtime_has_tls_get ())
7219 if (klass->instance_size > tlab_size)
7221 if (klass->has_finalize || klass->marshalbyref || (mono_profiler_get_events () & MONO_PROFILE_ALLOCATIONS))
7225 if (klass->byval_arg.type == MONO_TYPE_STRING)
7227 if (collect_before_allocs)
7230 if (ALIGN_TO (klass->instance_size, ALLOC_ALIGN) < MAX_SMALL_OBJ_SIZE)
7231 return mono_gc_get_managed_allocator_by_type (ATYPE_SMALL);
7233 return mono_gc_get_managed_allocator_by_type (ATYPE_NORMAL);
7240 mono_gc_get_managed_array_allocator (MonoVTable *vtable, int rank)
7242 #ifdef MANAGED_ALLOCATION
7243 MonoClass *klass = vtable->klass;
7245 #ifdef HAVE_KW_THREAD
7246 int tlab_next_offset = -1;
7247 int tlab_temp_end_offset = -1;
7248 MONO_THREAD_VAR_OFFSET (tlab_next, tlab_next_offset);
7249 MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
7251 if (tlab_next_offset == -1 || tlab_temp_end_offset == -1)
7257 if (!mono_runtime_has_tls_get ())
7259 if (mono_profiler_get_events () & MONO_PROFILE_ALLOCATIONS)
7261 if (collect_before_allocs)
7263 g_assert (!klass->has_finalize && !klass->marshalbyref);
7265 return mono_gc_get_managed_allocator_by_type (ATYPE_VECTOR);
7272 mono_gc_get_managed_allocator_by_type (int atype)
7274 #ifdef MANAGED_ALLOCATION
7277 if (!mono_runtime_has_tls_get ())
7280 mono_loader_lock ();
7281 res = alloc_method_cache [atype];
7283 res = alloc_method_cache [atype] = create_allocator (atype);
7284 mono_loader_unlock ();
7292 mono_gc_get_managed_allocator_types (void)
7299 mono_gc_get_write_barrier (void)
7302 MonoMethodBuilder *mb;
7303 MonoMethodSignature *sig;
7304 #ifdef MANAGED_WBARRIER
7305 int label_no_wb_1, label_no_wb_2, label_no_wb_3, label_no_wb_4, label_need_wb, label_slow_path;
7306 #ifndef ALIGN_NURSERY
7307 int label_continue_1, label_continue_2, label_no_wb_5;
7308 int dereferenced_var;
7310 int buffer_var, buffer_index_var, dummy_var;
7312 #ifdef HAVE_KW_THREAD
7313 int stack_end_offset = -1, store_remset_buffer_offset = -1;
7314 int store_remset_buffer_index_offset = -1, store_remset_buffer_index_addr_offset = -1;
7316 MONO_THREAD_VAR_OFFSET (stack_end, stack_end_offset);
7317 g_assert (stack_end_offset != -1);
7318 MONO_THREAD_VAR_OFFSET (store_remset_buffer, store_remset_buffer_offset);
7319 g_assert (store_remset_buffer_offset != -1);
7320 MONO_THREAD_VAR_OFFSET (store_remset_buffer_index, store_remset_buffer_index_offset);
7321 g_assert (store_remset_buffer_index_offset != -1);
7322 MONO_THREAD_VAR_OFFSET (store_remset_buffer_index_addr, store_remset_buffer_index_addr_offset);
7323 g_assert (store_remset_buffer_index_addr_offset != -1);
7327 // FIXME: Maybe create a separate version for ctors (the branch would be
7328 // correctly predicted more times)
7329 if (write_barrier_method)
7330 return write_barrier_method;
7332 /* Create the IL version of mono_gc_barrier_generic_store () */
7333 sig = mono_metadata_signature_alloc (mono_defaults.corlib, 1);
7334 sig->ret = &mono_defaults.void_class->byval_arg;
7335 sig->params [0] = &mono_defaults.int_class->byval_arg;
7337 mb = mono_mb_new (mono_defaults.object_class, "wbarrier", MONO_WRAPPER_WRITE_BARRIER);
7339 #ifdef MANAGED_WBARRIER
7340 if (mono_runtime_has_tls_get ()) {
7341 #ifdef ALIGN_NURSERY
7342 // if (ptr_in_nursery (ptr)) return;
7344 * Masking out the bits might be faster, but we would have to use 64 bit
7345 * immediates, which might be slower.
7347 mono_mb_emit_ldarg (mb, 0);
7348 mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
7349 mono_mb_emit_byte (mb, CEE_SHR_UN);
7350 mono_mb_emit_icon (mb, (mword)nursery_start >> DEFAULT_NURSERY_BITS);
7351 label_no_wb_1 = mono_mb_emit_branch (mb, CEE_BEQ);
7353 // if (!ptr_in_nursery (*ptr)) return;
7354 mono_mb_emit_ldarg (mb, 0);
7355 mono_mb_emit_byte (mb, CEE_LDIND_I);
7356 mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
7357 mono_mb_emit_byte (mb, CEE_SHR_UN);
7358 mono_mb_emit_icon (mb, (mword)nursery_start >> DEFAULT_NURSERY_BITS);
7359 label_no_wb_2 = mono_mb_emit_branch (mb, CEE_BNE_UN);
7362 // if (ptr < (nursery_start)) goto continue;
7363 mono_mb_emit_ldarg (mb, 0);
7364 mono_mb_emit_ptr (mb, (gpointer) nursery_start);
7365 label_continue_1 = mono_mb_emit_branch (mb, CEE_BLT);
7367 // if (ptr >= nursery_real_end)) goto continue;
7368 mono_mb_emit_ldarg (mb, 0);
7369 mono_mb_emit_ptr (mb, (gpointer) nursery_real_end);
7370 label_continue_2 = mono_mb_emit_branch (mb, CEE_BGE);
7373 label_no_wb_1 = mono_mb_emit_branch (mb, CEE_BR);
7376 mono_mb_patch_branch (mb, label_continue_1);
7377 mono_mb_patch_branch (mb, label_continue_2);
7379 // Dereference and store in local var
7380 dereferenced_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7381 mono_mb_emit_ldarg (mb, 0);
7382 mono_mb_emit_byte (mb, CEE_LDIND_I);
7383 mono_mb_emit_stloc (mb, dereferenced_var);
7385 // if (*ptr < nursery_start) return;
7386 mono_mb_emit_ldloc (mb, dereferenced_var);
7387 mono_mb_emit_ptr (mb, (gpointer) nursery_start);
7388 label_no_wb_2 = mono_mb_emit_branch (mb, CEE_BLT);
7390 // if (*ptr >= nursery_end) return;
7391 mono_mb_emit_ldloc (mb, dereferenced_var);
7392 mono_mb_emit_ptr (mb, (gpointer) nursery_real_end);
7393 label_no_wb_5 = mono_mb_emit_branch (mb, CEE_BGE);
7396 // if (ptr >= stack_end) goto need_wb;
7397 mono_mb_emit_ldarg (mb, 0);
7398 EMIT_TLS_ACCESS (mb, stack_end, stack_end_offset);
7399 label_need_wb = mono_mb_emit_branch (mb, CEE_BGE_UN);
7401 // if (ptr >= stack_start) return;
7402 dummy_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7403 mono_mb_emit_ldarg (mb, 0);
7404 mono_mb_emit_ldloc_addr (mb, dummy_var);
7405 label_no_wb_3 = mono_mb_emit_branch (mb, CEE_BGE_UN);
7408 mono_mb_patch_branch (mb, label_need_wb);
7410 // buffer = STORE_REMSET_BUFFER;
7411 buffer_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7412 EMIT_TLS_ACCESS (mb, store_remset_buffer, store_remset_buffer_offset);
7413 mono_mb_emit_stloc (mb, buffer_var);
7415 // buffer_index = STORE_REMSET_BUFFER_INDEX;
7416 buffer_index_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7417 EMIT_TLS_ACCESS (mb, store_remset_buffer_index, store_remset_buffer_index_offset);
7418 mono_mb_emit_stloc (mb, buffer_index_var);
7420 // if (buffer [buffer_index] == ptr) return;
7421 mono_mb_emit_ldloc (mb, buffer_var);
7422 mono_mb_emit_ldloc (mb, buffer_index_var);
7423 g_assert (sizeof (gpointer) == 4 || sizeof (gpointer) == 8);
7424 mono_mb_emit_icon (mb, sizeof (gpointer) == 4 ? 2 : 3);
7425 mono_mb_emit_byte (mb, CEE_SHL);
7426 mono_mb_emit_byte (mb, CEE_ADD);
7427 mono_mb_emit_byte (mb, CEE_LDIND_I);
7428 mono_mb_emit_ldarg (mb, 0);
7429 label_no_wb_4 = mono_mb_emit_branch (mb, CEE_BEQ);
7432 mono_mb_emit_ldloc (mb, buffer_index_var);
7433 mono_mb_emit_icon (mb, 1);
7434 mono_mb_emit_byte (mb, CEE_ADD);
7435 mono_mb_emit_stloc (mb, buffer_index_var);
7437 // if (buffer_index >= STORE_REMSET_BUFFER_SIZE) goto slow_path;
7438 mono_mb_emit_ldloc (mb, buffer_index_var);
7439 mono_mb_emit_icon (mb, STORE_REMSET_BUFFER_SIZE);
7440 label_slow_path = mono_mb_emit_branch (mb, CEE_BGE);
7442 // buffer [buffer_index] = ptr;
7443 mono_mb_emit_ldloc (mb, buffer_var);
7444 mono_mb_emit_ldloc (mb, buffer_index_var);
7445 g_assert (sizeof (gpointer) == 4 || sizeof (gpointer) == 8);
7446 mono_mb_emit_icon (mb, sizeof (gpointer) == 4 ? 2 : 3);
7447 mono_mb_emit_byte (mb, CEE_SHL);
7448 mono_mb_emit_byte (mb, CEE_ADD);
7449 mono_mb_emit_ldarg (mb, 0);
7450 mono_mb_emit_byte (mb, CEE_STIND_I);
7452 // STORE_REMSET_BUFFER_INDEX = buffer_index;
7453 EMIT_TLS_ACCESS (mb, store_remset_buffer_index_addr, store_remset_buffer_index_addr_offset);
7454 mono_mb_emit_ldloc (mb, buffer_index_var);
7455 mono_mb_emit_byte (mb, CEE_STIND_I);
7458 mono_mb_patch_branch (mb, label_no_wb_1);
7459 mono_mb_patch_branch (mb, label_no_wb_2);
7460 mono_mb_patch_branch (mb, label_no_wb_3);
7461 mono_mb_patch_branch (mb, label_no_wb_4);
7462 #ifndef ALIGN_NURSERY
7463 mono_mb_patch_branch (mb, label_no_wb_5);
7465 mono_mb_emit_byte (mb, CEE_RET);
7468 mono_mb_patch_branch (mb, label_slow_path);
7472 mono_mb_emit_ldarg (mb, 0);
7473 mono_mb_emit_icall (mb, mono_gc_wbarrier_generic_nostore);
7474 mono_mb_emit_byte (mb, CEE_RET);
7476 res = mono_mb_create_method (mb, sig, 16);
7479 mono_loader_lock ();
7480 if (write_barrier_method) {
7481 /* Already created */
7482 mono_free_method (res);
7484 /* double-checked locking */
7485 mono_memory_barrier ();
7486 write_barrier_method = res;
7488 mono_loader_unlock ();
7490 return write_barrier_method;
7494 mono_gc_get_description (void)
7496 return g_strdup ("sgen");
7500 mono_gc_set_desktop_mode (void)
7505 mono_gc_is_moving (void)
7511 mono_gc_is_disabled (void)
7517 mono_sgen_is_worker_thread (pthread_t thread)
7523 mono_sgen_debug_printf (int level, const char *format, ...)
7527 if (level > gc_debug_level)
7530 va_start (ap, format);
7531 vfprintf (gc_debug_file, format, ap);
7535 #endif /* HAVE_SGEN_GC */