2 * sgen-gc.c: Simple generational GC.
5 * Paolo Molaro (lupus@ximian.com)
7 * Copyright 2005-2010 Novell, Inc (http://www.novell.com)
9 * Thread start/stop adapted from Boehm's GC:
10 * Copyright (c) 1994 by Xerox Corporation. All rights reserved.
11 * Copyright (c) 1996 by Silicon Graphics. All rights reserved.
12 * Copyright (c) 1998 by Fergus Henderson. All rights reserved.
13 * Copyright (c) 2000-2004 by Hewlett-Packard Company. All rights reserved.
15 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
16 * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
18 * Permission is hereby granted to use or copy this program
19 * for any purpose, provided the above notices are retained on all copies.
20 * Permission to modify the code and to distribute modified code is granted,
21 * provided the above notices are retained, and a notice that the code was
22 * modified is included with the above copyright notice.
25 * Copyright 2001-2003 Ximian, Inc
26 * Copyright 2003-2010 Novell, Inc.
28 * Permission is hereby granted, free of charge, to any person obtaining
29 * a copy of this software and associated documentation files (the
30 * "Software"), to deal in the Software without restriction, including
31 * without limitation the rights to use, copy, modify, merge, publish,
32 * distribute, sublicense, and/or sell copies of the Software, and to
33 * permit persons to whom the Software is furnished to do so, subject to
34 * the following conditions:
36 * The above copyright notice and this permission notice shall be
37 * included in all copies or substantial portions of the Software.
39 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
40 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
41 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
42 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
43 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
44 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
45 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
48 * Important: allocation provides always zeroed memory, having to do
49 * a memset after allocation is deadly for performance.
50 * Memory usage at startup is currently as follows:
52 * 64 KB internal space
54 * We should provide a small memory config with half the sizes
56 * We currently try to make as few mono assumptions as possible:
57 * 1) 2-word header with no GC pointers in it (first vtable, second to store the
59 * 2) gc descriptor is the second word in the vtable (first word in the class)
60 * 3) 8 byte alignment is the minimum and enough (not true for special structures (SIMD), FIXME)
61 * 4) there is a function to get an object's size and the number of
62 * elements in an array.
63 * 5) we know the special way bounds are allocated for complex arrays
64 * 6) we know about proxies and how to treat them when domains are unloaded
66 * Always try to keep stack usage to a minimum: no recursive behaviour
67 * and no large stack allocs.
69 * General description.
70 * Objects are initially allocated in a nursery using a fast bump-pointer technique.
71 * When the nursery is full we start a nursery collection: this is performed with a
73 * When the old generation is full we start a copying GC of the old generation as well:
74 * this will be changed to mark&sweep with copying when fragmentation becomes to severe
75 * in the future. Maybe we'll even do both during the same collection like IMMIX.
77 * The things that complicate this description are:
78 * *) pinned objects: we can't move them so we need to keep track of them
79 * *) no precise info of the thread stacks and registers: we need to be able to
80 * quickly find the objects that may be referenced conservatively and pin them
81 * (this makes the first issues more important)
82 * *) large objects are too expensive to be dealt with using copying GC: we handle them
83 * with mark/sweep during major collections
84 * *) some objects need to not move even if they are small (interned strings, Type handles):
85 * we use mark/sweep for them, too: they are not allocated in the nursery, but inside
86 * PinnedChunks regions
92 *) we could have a function pointer in MonoClass to implement
93 customized write barriers for value types
95 *) investigate the stuff needed to advance a thread to a GC-safe
96 point (single-stepping, read from unmapped memory etc) and implement it.
97 This would enable us to inline allocations and write barriers, for example,
98 or at least parts of them, like the write barrier checks.
99 We may need this also for handling precise info on stacks, even simple things
100 as having uninitialized data on the stack and having to wait for the prolog
101 to zero it. Not an issue for the last frame that we scan conservatively.
102 We could always not trust the value in the slots anyway.
104 *) modify the jit to save info about references in stack locations:
105 this can be done just for locals as a start, so that at least
106 part of the stack is handled precisely.
108 *) test/fix endianess issues
110 *) Implement a card table as the write barrier instead of remembered
111 sets? Card tables are not easy to implement with our current
112 memory layout. We have several different kinds of major heap
113 objects: Small objects in regular blocks, small objects in pinned
114 chunks and LOS objects. If we just have a pointer we have no way
115 to tell which kind of object it points into, therefore we cannot
116 know where its card table is. The least we have to do to make
117 this happen is to get rid of write barriers for indirect stores.
120 *) Get rid of write barriers for indirect stores. We can do this by
121 telling the GC to wbarrier-register an object once we do an ldloca
122 or ldelema on it, and to unregister it once it's not used anymore
123 (it can only travel downwards on the stack). The problem with
124 unregistering is that it needs to happen eventually no matter
125 what, even if exceptions are thrown, the thread aborts, etc.
126 Rodrigo suggested that we could do only the registering part and
127 let the collector find out (pessimistically) when it's safe to
128 unregister, namely when the stack pointer of the thread that
129 registered the object is higher than it was when the registering
130 happened. This might make for a good first implementation to get
131 some data on performance.
133 *) Some sort of blacklist support? Blacklists is a concept from the
134 Boehm GC: if during a conservative scan we find pointers to an
135 area which we might use as heap, we mark that area as unusable, so
136 pointer retention by random pinning pointers is reduced.
138 *) experiment with max small object size (very small right now - 2kb,
139 because it's tied to the max freelist size)
141 *) add an option to mmap the whole heap in one chunk: it makes for many
142 simplifications in the checks (put the nursery at the top and just use a single
143 check for inclusion/exclusion): the issue this has is that on 32 bit systems it's
144 not flexible (too much of the address space may be used by default or we can't
145 increase the heap as needed) and we'd need a race-free mechanism to return memory
146 back to the system (mprotect(PROT_NONE) will still keep the memory allocated if it
147 was written to, munmap is needed, but the following mmap may not find the same segment
150 *) memzero the major fragments after restarting the world and optionally a smaller
153 *) investigate having fragment zeroing threads
155 *) separate locks for finalization and other minor stuff to reduce
158 *) try a different copying order to improve memory locality
160 *) a thread abort after a store but before the write barrier will
161 prevent the write barrier from executing
163 *) specialized dynamically generated markers/copiers
165 *) Dynamically adjust TLAB size to the number of threads. If we have
166 too many threads that do allocation, we might need smaller TLABs,
167 and we might get better performance with larger TLABs if we only
168 have a handful of threads. We could sum up the space left in all
169 assigned TLABs and if that's more than some percentage of the
170 nursery size, reduce the TLAB size.
172 *) Explore placing unreachable objects on unused nursery memory.
173 Instead of memset'ng a region to zero, place an int[] covering it.
174 A good place to start is add_nursery_frag. The tricky thing here is
175 placing those objects atomically outside of a collection.
185 #include <semaphore.h>
194 #define _XOPEN_SOURCE
196 #include "metadata/metadata-internals.h"
197 #include "metadata/class-internals.h"
198 #include "metadata/gc-internal.h"
199 #include "metadata/object-internals.h"
200 #include "metadata/threads.h"
201 #include "metadata/sgen-gc.h"
202 #include "metadata/sgen-archdep.h"
203 #include "metadata/mono-gc.h"
204 #include "metadata/method-builder.h"
205 #include "metadata/profiler-private.h"
206 #include "metadata/monitor.h"
207 #include "metadata/threadpool-internals.h"
208 #include "metadata/mempool-internals.h"
209 #include "metadata/marshal.h"
210 #include "utils/mono-mmap.h"
211 #include "utils/mono-time.h"
212 #include "utils/mono-semaphore.h"
213 #include "utils/mono-counters.h"
215 #include <mono/utils/memcheck.h>
217 #if defined(__MACH__)
218 #include "utils/mach-support.h"
221 #define OPDEF(a,b,c,d,e,f,g,h,i,j) \
225 #include "mono/cil/opcode.def"
232 * ######################################################################
233 * ######## Types and constants used by the GC.
234 * ######################################################################
237 static int gc_initialized = 0;
238 static int gc_debug_level = 0;
239 static FILE* gc_debug_file;
240 /* If set, do a minor collection before every allocation */
241 static gboolean collect_before_allocs = FALSE;
242 /* If set, do a heap consistency check before each minor collection */
243 static gboolean consistency_check_at_minor_collection = FALSE;
244 /* If set, check that there are no references to the domain left at domain unload */
245 static gboolean xdomain_checks = FALSE;
246 /* If not null, dump the heap after each collection into this file */
247 static FILE *heap_dump_file = NULL;
248 /* If set, mark stacks conservatively, even if precise marking is possible */
249 static gboolean conservative_stack_mark = TRUE;
250 /* If set, do a plausibility check on the scan_starts before and after
252 static gboolean do_scan_starts_check = FALSE;
255 * Turning on heavy statistics will turn off the managed allocator and
256 * the managed write barrier.
258 //#define HEAVY_STATISTICS
260 #ifdef HEAVY_STATISTICS
261 #define HEAVY_STAT(x) x
263 #define HEAVY_STAT(x)
266 #ifdef HEAVY_STATISTICS
267 static long long stat_objects_alloced = 0;
268 static long long stat_bytes_alloced = 0;
269 static long long stat_objects_alloced_degraded = 0;
270 static long long stat_bytes_alloced_degraded = 0;
271 static long long stat_bytes_alloced_los = 0;
273 static long long stat_copy_object_called_nursery = 0;
274 static long long stat_objects_copied_nursery = 0;
275 static long long stat_copy_object_called_major = 0;
276 static long long stat_objects_copied_major = 0;
278 static long long stat_scan_object_called_nursery = 0;
279 static long long stat_scan_object_called_major = 0;
281 static long long stat_nursery_copy_object_failed_from_space = 0;
282 static long long stat_nursery_copy_object_failed_forwarded = 0;
283 static long long stat_nursery_copy_object_failed_pinned = 0;
285 static long long stat_store_remsets = 0;
286 static long long stat_store_remsets_unique = 0;
287 static long long stat_saved_remsets_1 = 0;
288 static long long stat_saved_remsets_2 = 0;
289 static long long stat_global_remsets_added = 0;
290 static long long stat_global_remsets_readded = 0;
291 static long long stat_global_remsets_processed = 0;
292 static long long stat_global_remsets_discarded = 0;
294 static long long stat_wasted_fragments_used = 0;
295 static long long stat_wasted_fragments_bytes = 0;
297 static int stat_wbarrier_set_field = 0;
298 static int stat_wbarrier_set_arrayref = 0;
299 static int stat_wbarrier_arrayref_copy = 0;
300 static int stat_wbarrier_generic_store = 0;
301 static int stat_wbarrier_generic_store_remset = 0;
302 static int stat_wbarrier_set_root = 0;
303 static int stat_wbarrier_value_copy = 0;
304 static int stat_wbarrier_object_copy = 0;
307 static long long time_minor_pre_collection_fragment_clear = 0;
308 static long long time_minor_pinning = 0;
309 static long long time_minor_scan_remsets = 0;
310 static long long time_minor_scan_pinned = 0;
311 static long long time_minor_scan_registered_roots = 0;
312 static long long time_minor_scan_thread_data = 0;
313 static long long time_minor_finish_gray_stack = 0;
314 static long long time_minor_fragment_creation = 0;
316 static long long time_major_pre_collection_fragment_clear = 0;
317 static long long time_major_pinning = 0;
318 static long long time_major_scan_pinned = 0;
319 static long long time_major_scan_registered_roots = 0;
320 static long long time_major_scan_thread_data = 0;
321 static long long time_major_scan_alloc_pinned = 0;
322 static long long time_major_scan_finalized = 0;
323 static long long time_major_scan_big_objects = 0;
324 static long long time_major_finish_gray_stack = 0;
325 static long long time_major_free_bigobjs = 0;
326 static long long time_major_los_sweep = 0;
327 static long long time_major_sweep = 0;
328 static long long time_major_fragment_creation = 0;
330 static long long pinned_chunk_bytes_alloced = 0;
331 static long long large_internal_bytes_alloced = 0;
333 /* Keep in sync with internal_mem_names in dump_heap()! */
335 INTERNAL_MEM_PIN_QUEUE,
336 INTERNAL_MEM_FRAGMENT,
337 INTERNAL_MEM_SECTION,
338 INTERNAL_MEM_SCAN_STARTS,
339 INTERNAL_MEM_FIN_TABLE,
340 INTERNAL_MEM_FINALIZE_ENTRY,
341 INTERNAL_MEM_DISLINK_TABLE,
342 INTERNAL_MEM_DISLINK,
343 INTERNAL_MEM_ROOTS_TABLE,
344 INTERNAL_MEM_ROOT_RECORD,
345 INTERNAL_MEM_STATISTICS,
347 INTERNAL_MEM_GRAY_QUEUE,
348 INTERNAL_MEM_STORE_REMSET,
349 INTERNAL_MEM_MS_TABLES,
350 INTERNAL_MEM_MS_BLOCK_INFO,
351 INTERNAL_MEM_EPHEMERON_LINK,
355 static long small_internal_mem_bytes [INTERNAL_MEM_MAX];
359 mono_gc_flush_info (void)
361 fflush (gc_debug_file);
365 /* Define this to allow the user to change some of the constants by specifying
366 * their values in the MONO_GC_PARAMS environmental variable. See
367 * mono_gc_base_init for details. */
368 #define USER_CONFIG 1
370 #define TV_DECLARE(name) gint64 name
371 #define TV_GETTIME(tv) tv = mono_100ns_ticks ()
372 #define TV_ELAPSED(start,end) (int)((end-start) / 10)
373 #define TV_ELAPSED_MS(start,end) ((TV_ELAPSED((start),(end)) + 500) / 1000)
375 #define ALIGN_TO(val,align) ((((guint64)val) + ((align) - 1)) & ~((align) - 1))
377 #define GC_BITS_PER_WORD (sizeof (mword) * 8)
385 typedef struct _Block Block;
391 /* each request from the OS ends up in a GCMemSection */
392 typedef struct _GCMemSection GCMemSection;
393 struct _GCMemSection {
397 /* pointer where more data could be allocated if it fits */
401 * scan starts is an array of pointers to objects equally spaced in the allocation area
402 * They let use quickly find pinned objects from pinning pointers.
405 /* in major collections indexes in the pin_queue for objects that pin this section */
408 unsigned short num_scan_start;
409 gboolean is_to_space;
412 #define SIZEOF_GC_MEM_SECTION ((sizeof (GCMemSection) + 7) & ~7)
414 /* Pinned objects are allocated in the LOS space if bigger than half a page
415 * or from freelists otherwise. We assume that pinned objects are relatively few
416 * and they have a slow dying speed (like interned strings, thread objects).
417 * As such they will be collected only at major collections.
418 * free lists are not global: when we need memory we allocate a PinnedChunk.
419 * Each pinned chunk is made of several pages, the first of wich is used
420 * internally for bookeeping (here think of a page as 4KB). The bookeeping
421 * includes the freelists vectors and info about the object size of each page
422 * in the pinned chunk. So, when needed, a free page is found in a pinned chunk,
423 * a size is assigned to it, the page is divided in the proper chunks and each
424 * chunk is added to the freelist. To not waste space, the remaining space in the
425 * first page is used as objects of size 16 or 32 (need to measure which are more
427 * We use this same structure to allocate memory used internally by the GC, so
428 * we never use malloc/free if we need to alloc during collection: the world is stopped
429 * and malloc/free will deadlock.
430 * When we want to iterate over pinned objects, we just scan a page at a time
431 * linearly according to the size of objects in the page: the next pointer used to link
432 * the items in the freelist uses the same word as the vtable. Since we keep freelists
433 * for each pinned chunk, if the word points outside the pinned chunk it means
435 * We could avoid this expensive scanning in creative ways. We could have a policy
436 * of putting in the pinned space only objects we know about that have no struct fields
437 * with references and we can easily use a even expensive write barrier for them,
438 * since pointer writes on such objects should be rare.
439 * The best compromise is to just alloc interned strings and System.MonoType in them.
440 * It would be nice to allocate MonoThread in it, too: must check that we properly
441 * use write barriers so we don't have to do any expensive scanning of the whole pinned
442 * chunk list during minor collections. We can avoid it now because we alloc in it only
443 * reference-free objects.
445 #define PINNED_FIRST_SLOT_SIZE (sizeof (gpointer) * 4)
446 #define MAX_FREELIST_SIZE 8192
447 typedef struct _PinnedChunk PinnedChunk;
448 struct _PinnedChunk {
451 int *page_sizes; /* a 0 means the page is still unused */
454 void *data [1]; /* page sizes and free lists are stored here */
457 /* The method used to clear the nursery */
458 /* Clearing at nursery collections is the safest, but has bad interactions with caches.
459 * Clearing at TLAB creation is much faster, but more complex and it might expose hard
464 CLEAR_AT_TLAB_CREATION
465 } NurseryClearPolicy;
467 static NurseryClearPolicy nursery_clear_policy = CLEAR_AT_TLAB_CREATION;
470 * If this is set, the nursery is aligned to an address aligned to its size, ie.
471 * a 1MB nursery will be aligned to an address divisible by 1MB. This allows us to
472 * speed up ptr_in_nursery () checks which are very frequent. This requires the
473 * nursery size to be a compile time constant.
475 #define ALIGN_NURSERY 1
478 * The young generation is divided into fragments. This is because
479 * we can hand one fragments to a thread for lock-less fast alloc and
480 * because the young generation ends up fragmented anyway by pinned objects.
481 * Once a collection is done, a list of fragments is created. When doing
482 * thread local alloc we use smallish nurseries so we allow new threads to
483 * allocate memory from gen0 without triggering a collection. Threads that
484 * are found to allocate lots of memory are given bigger fragments. This
485 * should make the finalizer thread use little nursery memory after a while.
486 * We should start assigning threads very small fragments: if there are many
487 * threads the nursery will be full of reserved space that the threads may not
488 * use at all, slowing down allocation speed.
489 * Thread local allocation is done from areas of memory Hotspot calls Thread Local
490 * Allocation Buffers (TLABs).
492 typedef struct _Fragment Fragment;
496 char *fragment_start;
497 char *fragment_limit; /* the current soft limit for allocation */
501 /* the runtime can register areas of memory as roots: we keep two lists of roots,
502 * a pinned root set for conservatively scanned roots and a normal one for
503 * precisely scanned roots (currently implemented as a single list).
505 typedef struct _RootRecord RootRecord;
514 * We're never actually using the first element. It's always set to
515 * NULL to simplify the elimination of consecutive duplicate
518 #define STORE_REMSET_BUFFER_SIZE 1024
520 typedef struct _GenericStoreRememberedSet GenericStoreRememberedSet;
521 struct _GenericStoreRememberedSet {
522 GenericStoreRememberedSet *next;
523 /* We need one entry less because the first entry of store
524 remset buffers is always a dummy and we don't copy it. */
525 gpointer data [STORE_REMSET_BUFFER_SIZE - 1];
528 /* we have 4 possible values in the low 2 bits */
530 REMSET_LOCATION, /* just a pointer to the exact location */
531 REMSET_RANGE, /* range of pointer fields */
532 REMSET_OBJECT, /* mark all the object for scanning */
533 REMSET_VTYPE, /* a valuetype array described by a gc descriptor and a count */
534 REMSET_TYPE_MASK = 0x3
537 #ifdef HAVE_KW_THREAD
538 static __thread RememberedSet *remembered_set MONO_TLS_FAST;
540 static pthread_key_t remembered_set_key;
541 static RememberedSet *global_remset;
542 static RememberedSet *freed_thread_remsets;
543 static GenericStoreRememberedSet *generic_store_remsets = NULL;
545 /*A two slots cache for recently inserted remsets */
546 static gpointer global_remset_cache [2];
548 /* FIXME: later choose a size that takes into account the RememberedSet struct
549 * and doesn't waste any alloc paddin space.
551 #define DEFAULT_REMSET_SIZE 1024
552 static RememberedSet* alloc_remset (int size, gpointer id);
554 /* Structure that corresponds to a MonoVTable: desc is a mword so requires
555 * no cast from a pointer to an integer
562 /* these bits are set in the object vtable: we could merge them since an object can be
563 * either pinned or forwarded but not both.
564 * We store them in the vtable slot because the bits are used in the sync block for
565 * other purposes: if we merge them and alloc the sync blocks aligned to 8 bytes, we can change
566 * this and use bit 3 in the syncblock (with the lower two bits both set for forwarded, that
567 * would be an invalid combination for the monitor and hash code).
568 * The values are already shifted.
569 * The forwarding address is stored in the sync block.
571 #define FORWARDED_BIT 1
573 #define VTABLE_BITS_MASK 0x3
575 /* returns NULL if not forwarded, or the forwarded address */
576 #define object_is_forwarded(obj) (((mword*)(obj))[0] & FORWARDED_BIT? (void*)(((mword*)(obj))[1]): NULL)
577 /* set the forwarded address fw_addr for object obj */
578 #define forward_object(obj,fw_addr) do { \
579 ((mword*)(obj))[0] |= FORWARDED_BIT; \
580 ((mword*)(obj))[1] = (mword)(fw_addr); \
583 #define object_is_pinned(obj) (((mword*)(obj))[0] & PINNED_BIT)
584 #define pin_object(obj) do { \
585 ((mword*)(obj))[0] |= PINNED_BIT; \
587 #define unpin_object(obj) do { \
588 ((mword*)(obj))[0] &= ~PINNED_BIT; \
592 #define ptr_in_nursery(ptr) (((mword)(ptr) & ~((1 << DEFAULT_NURSERY_BITS) - 1)) == (mword)nursery_start)
594 #define ptr_in_nursery(ptr) ((char*)(ptr) >= nursery_start && (char*)(ptr) < nursery_real_end)
598 * Since we set bits in the vtable, use the macro to load it from the pointer to
599 * an object that is potentially pinned.
601 #define LOAD_VTABLE(addr) ((*(mword*)(addr)) & ~VTABLE_BITS_MASK)
604 safe_name (void* obj)
606 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (obj);
607 return vt->klass->name;
611 safe_object_get_size (MonoObject* o)
613 MonoClass *klass = ((MonoVTable*)LOAD_VTABLE (o))->klass;
614 if (klass == mono_defaults.string_class) {
615 return sizeof (MonoString) + 2 * mono_string_length_fast ((MonoString*) o) + 2;
616 } else if (klass->rank) {
617 MonoArray *array = (MonoArray*)o;
618 size_t size = sizeof (MonoArray) + klass->sizes.element_size * mono_array_length_fast (array);
619 if (G_UNLIKELY (array->bounds)) {
620 size += sizeof (mono_array_size_t) - 1;
621 size &= ~(sizeof (mono_array_size_t) - 1);
622 size += sizeof (MonoArrayBounds) * klass->rank;
626 /* from a created object: the class must be inited already */
627 return klass->instance_size;
632 * ######################################################################
633 * ######## Global data.
634 * ######################################################################
636 static LOCK_DECLARE (gc_mutex);
637 static int gc_disabled = 0;
638 static int num_minor_gcs = 0;
639 static int num_major_gcs = 0;
643 /* good sizes are 512KB-1MB: larger ones increase a lot memzeroing time */
644 #define DEFAULT_NURSERY_SIZE (default_nursery_size)
645 static int default_nursery_size = (1 << 22);
647 /* The number of trailing 0 bits in DEFAULT_NURSERY_SIZE */
648 #define DEFAULT_NURSERY_BITS (default_nursery_bits)
649 static int default_nursery_bits = 22;
654 #define DEFAULT_NURSERY_SIZE (4*1024*1024)
656 #define DEFAULT_NURSERY_BITS 22
661 #define MIN_MINOR_COLLECTION_ALLOWANCE (DEFAULT_NURSERY_SIZE * 4)
662 /* to quickly find the head of an object pinned by a conservative address
663 * we keep track of the objects allocated for each SCAN_START_SIZE memory
664 * chunk in the nursery or other memory sections. Larger values have less
665 * memory overhead and bigger runtime cost. 4-8 KB are reasonable values.
667 #define SCAN_START_SIZE (4096*2)
668 /* the minimum size of a fragment that we consider useful for allocation */
669 #define FRAGMENT_MIN_SIZE (512)
670 /* This is a fixed value used for pinned chunks, not the system pagesize */
671 #define FREELIST_PAGESIZE (16*1024)
673 static mword pagesize = 4096;
674 static mword nursery_size;
675 static int degraded_mode = 0;
677 static mword total_alloc = 0;
678 /* use this to tune when to do a major/minor collection */
679 static mword memory_pressure = 0;
680 static int minor_collection_allowance;
681 static int minor_collection_sections_alloced = 0;
683 static GCMemSection *nursery_section = NULL;
684 static mword lowest_heap_address = ~(mword)0;
685 static mword highest_heap_address = 0;
687 static LOCK_DECLARE (interruption_mutex);
689 typedef struct _FinalizeEntry FinalizeEntry;
690 struct _FinalizeEntry {
695 typedef struct _FinalizeEntryHashTable FinalizeEntryHashTable;
696 struct _FinalizeEntryHashTable {
697 FinalizeEntry **table;
702 typedef struct _DisappearingLink DisappearingLink;
703 struct _DisappearingLink {
704 DisappearingLink *next;
708 typedef struct _DisappearingLinkHashTable DisappearingLinkHashTable;
709 struct _DisappearingLinkHashTable {
710 DisappearingLink **table;
715 typedef struct _EphemeronLinkNode EphemeronLinkNode;
717 struct _EphemeronLinkNode {
718 EphemeronLinkNode *next;
727 #define LARGE_INTERNAL_MEM_HEADER_MAGIC 0x7d289f3a
729 typedef struct _LargeInternalMemHeader LargeInternalMemHeader;
730 struct _LargeInternalMemHeader {
742 int current_collection_generation = -1;
745 * The link pointer is hidden by negating each bit. We use the lowest
746 * bit of the link (before negation) to store whether it needs
747 * resurrection tracking.
749 #define HIDE_POINTER(p,t) ((gpointer)(~((gulong)(p)|((t)?1:0))))
750 #define REVEAL_POINTER(p) ((gpointer)((~(gulong)(p))&~3L))
752 #define DISLINK_OBJECT(d) (REVEAL_POINTER (*(d)->link))
753 #define DISLINK_TRACK(d) ((~(gulong)(*(d)->link)) & 1)
756 * The finalizable hash has the object as the key, the
757 * disappearing_link hash, has the link address as key.
759 static FinalizeEntryHashTable minor_finalizable_hash;
760 static FinalizeEntryHashTable major_finalizable_hash;
761 /* objects that are ready to be finalized */
762 static FinalizeEntry *fin_ready_list = NULL;
763 static FinalizeEntry *critical_fin_list = NULL;
765 static DisappearingLinkHashTable minor_disappearing_link_hash;
766 static DisappearingLinkHashTable major_disappearing_link_hash;
768 static EphemeronLinkNode *ephemeron_list;
770 static int num_ready_finalizers = 0;
771 static int no_finalize = 0;
773 /* keep each size a multiple of ALLOC_ALIGN */
774 /* on 64 bit systems 8 is likely completely unused. */
775 static const int freelist_sizes [] = {
776 8, 16, 24, 32, 40, 48, 64, 80,
777 96, 128, 160, 192, 224, 256, 320, 384,
778 448, 512, 584, 680, 816, 1024, 1360, 2048,
779 2336, 2728, 3272, 4096, 5456, 8192 };
780 #define FREELIST_NUM_SLOTS (sizeof (freelist_sizes) / sizeof (freelist_sizes [0]))
782 /* This is also the MAJOR_SECTION_SIZE for the copying major
784 #define PINNED_CHUNK_SIZE (128 * 1024)
786 /* internal_chunk_list is used for allocating structures needed by the GC */
787 static PinnedChunk *internal_chunk_list = NULL;
789 static int slot_for_size (size_t size);
792 ROOT_TYPE_NORMAL = 0, /* "normal" roots */
793 ROOT_TYPE_PINNED = 1, /* roots without a GC descriptor */
794 ROOT_TYPE_WBARRIER = 2, /* roots with a write barrier */
798 /* registered roots: the key to the hash is the root start address */
800 * Different kinds of roots are kept separate to speed up pin_from_roots () for example.
802 static RootRecord **roots_hash [ROOT_TYPE_NUM] = { NULL, NULL };
803 static int roots_hash_size [ROOT_TYPE_NUM] = { 0, 0, 0 };
804 static mword roots_size = 0; /* amount of memory in the root set */
805 static int num_roots_entries [ROOT_TYPE_NUM] = { 0, 0, 0 };
808 * The current allocation cursors
809 * We allocate objects in the nursery.
810 * The nursery is the area between nursery_start and nursery_real_end.
811 * Allocation is done from a Thread Local Allocation Buffer (TLAB). TLABs are allocated
812 * from nursery fragments.
813 * tlab_next is the pointer to the space inside the TLAB where the next object will
815 * tlab_temp_end is the pointer to the end of the temporary space reserved for
816 * the allocation: it allows us to set the scan starts at reasonable intervals.
817 * tlab_real_end points to the end of the TLAB.
818 * nursery_frag_real_end points to the end of the currently used nursery fragment.
819 * nursery_first_pinned_start points to the start of the first pinned object in the nursery
820 * nursery_last_pinned_end points to the end of the last pinned object in the nursery
821 * At the next allocation, the area of the nursery where objects can be present is
822 * between MIN(nursery_first_pinned_start, first_fragment_start) and
823 * MAX(nursery_last_pinned_end, nursery_frag_real_end)
825 static char *nursery_start = NULL;
827 #ifdef HAVE_KW_THREAD
828 #define TLAB_ACCESS_INIT
829 #define TLAB_START tlab_start
830 #define TLAB_NEXT tlab_next
831 #define TLAB_TEMP_END tlab_temp_end
832 #define TLAB_REAL_END tlab_real_end
833 #define REMEMBERED_SET remembered_set
834 #define STORE_REMSET_BUFFER store_remset_buffer
835 #define STORE_REMSET_BUFFER_INDEX store_remset_buffer_index
836 #define IN_CRITICAL_REGION thread_info->in_critical_region
838 static pthread_key_t thread_info_key;
839 #define TLAB_ACCESS_INIT SgenThreadInfo *__thread_info__ = pthread_getspecific (thread_info_key)
840 #define TLAB_START (__thread_info__->tlab_start)
841 #define TLAB_NEXT (__thread_info__->tlab_next)
842 #define TLAB_TEMP_END (__thread_info__->tlab_temp_end)
843 #define TLAB_REAL_END (__thread_info__->tlab_real_end)
844 #define REMEMBERED_SET (__thread_info__->remset)
845 #define STORE_REMSET_BUFFER (__thread_info__->store_remset_buffer)
846 #define STORE_REMSET_BUFFER_INDEX (__thread_info__->store_remset_buffer_index)
847 #define IN_CRITICAL_REGION (__thread_info__->in_critical_region)
850 /* we use the memory barrier only to prevent compiler reordering (a memory constraint may be enough) */
851 #define ENTER_CRITICAL_REGION do {IN_CRITICAL_REGION = 1;mono_memory_barrier ();} while (0)
852 #define EXIT_CRITICAL_REGION do {IN_CRITICAL_REGION = 0;mono_memory_barrier ();} while (0)
855 * FIXME: What is faster, a TLS variable pointing to a structure, or separate TLS
856 * variables for next+temp_end ?
858 #ifdef HAVE_KW_THREAD
859 static __thread SgenThreadInfo *thread_info;
860 static __thread char *tlab_start;
861 static __thread char *tlab_next;
862 static __thread char *tlab_temp_end;
863 static __thread char *tlab_real_end;
864 static __thread gpointer *store_remset_buffer;
865 static __thread long store_remset_buffer_index;
866 /* Used by the managed allocator/wbarrier */
867 static __thread char **tlab_next_addr;
868 static __thread char *stack_end;
869 static __thread long *store_remset_buffer_index_addr;
871 static char *nursery_next = NULL;
872 static char *nursery_frag_real_end = NULL;
873 static char *nursery_real_end = NULL;
874 static char *nursery_last_pinned_end = NULL;
876 /* The size of a TLAB */
877 /* The bigger the value, the less often we have to go to the slow path to allocate a new
878 * one, but the more space is wasted by threads not allocating much memory.
880 * FIXME: Make this self-tuning for each thread.
882 static guint32 tlab_size = (1024 * 4);
884 /*How much space is tolerable to be wasted from the current fragment when allocating a new TLAB*/
885 #define MAX_NURSERY_TLAB_WASTE 512
887 /* fragments that are free and ready to be used for allocation */
888 static Fragment *nursery_fragments = NULL;
889 /* freeelist of fragment structures */
890 static Fragment *fragment_freelist = NULL;
893 * Objects bigger then this go into the large object space. This size
894 * has a few constraints. It must fit into the major heap, which in
895 * the case of the copying collector means that it must fit into a
896 * pinned chunk. It must also play well with the GC descriptors, some
897 * of which (DESC_TYPE_RUN_LENGTH, DESC_TYPE_SMALL_BITMAP) encode the
900 #define MAX_SMALL_OBJ_SIZE 8000
902 /* Functions supplied by the runtime to be called by the GC */
903 static MonoGCCallbacks gc_callbacks;
905 #define ALLOC_ALIGN 8
906 #define ALLOC_ALIGN_BITS 3
908 #define MOVED_OBJECTS_NUM 64
909 static void *moved_objects [MOVED_OBJECTS_NUM];
910 static int moved_objects_idx = 0;
913 * ######################################################################
914 * ######## Macros and function declarations.
915 * ######################################################################
918 #define UPDATE_HEAP_BOUNDARIES(low,high) do { \
919 if ((mword)(low) < lowest_heap_address) \
920 lowest_heap_address = (mword)(low); \
921 if ((mword)(high) > highest_heap_address) \
922 highest_heap_address = (mword)(high); \
924 #define ADDR_IN_HEAP_BOUNDARIES(addr) ((p) >= lowest_heap_address && (p) < highest_heap_address)
927 align_pointer (void *ptr)
929 mword p = (mword)ptr;
930 p += sizeof (gpointer) - 1;
931 p &= ~ (sizeof (gpointer) - 1);
935 typedef void (*CopyOrMarkObjectFunc) (void**);
936 typedef char* (*ScanObjectFunc) (char*);
938 /* forward declarations */
939 static void* get_internal_mem (size_t size, int type);
940 static void free_internal_mem (void *addr, int type);
941 static void* get_os_memory (size_t size, int activate);
942 static void* get_os_memory_aligned (mword size, mword alignment, gboolean activate);
943 static void free_os_memory (void *addr, size_t size);
944 static G_GNUC_UNUSED void report_internal_mem_usage (void);
946 static int stop_world (void);
947 static int restart_world (void);
948 static void add_to_global_remset (gpointer ptr);
949 static void scan_thread_data (void *start_nursery, void *end_nursery, gboolean precise);
950 static void scan_from_remsets (void *start_nursery, void *end_nursery);
951 static void scan_from_registered_roots (CopyOrMarkObjectFunc copy_func, char *addr_start, char *addr_end, int root_type);
952 static void scan_finalizer_entries (CopyOrMarkObjectFunc copy_func, FinalizeEntry *list);
953 static void find_pinning_ref_from_thread (char *obj, size_t size);
954 static void update_current_thread_stack (void *start);
955 static void finalize_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, int generation);
956 static void add_or_remove_disappearing_link (MonoObject *obj, void **link, gboolean track, int generation);
957 static void null_link_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, int generation);
958 static void null_links_for_domain (MonoDomain *domain, int generation);
959 static gboolean search_fragment_for_size (size_t size);
960 static int search_fragment_for_size_range (size_t desired_size, size_t minimum_size);
961 static void build_nursery_fragments (int start_pin, int end_pin);
962 static void clear_nursery_fragments (char *next);
963 static void pin_from_roots (void *start_nursery, void *end_nursery);
964 static int pin_objects_from_addresses (GCMemSection *section, void **start, void **end, void *start_nursery, void *end_nursery);
965 static void pin_objects_in_section (GCMemSection *section);
966 static void optimize_pin_queue (int start_slot);
967 static void clear_remsets (void);
968 static void clear_tlabs (void);
969 typedef void (*IterateObjectCallbackFunc) (char*, size_t, void*);
970 static void scan_area_with_callback (char *start, char *end, IterateObjectCallbackFunc callback, void *data);
971 static void scan_object (char *start);
972 static void major_scan_object (char *start);
973 static void* copy_object_no_checks (void *obj);
974 static void copy_object (void **obj_slot);
975 static void* get_chunk_freelist (PinnedChunk *chunk, int slot);
976 static PinnedChunk* alloc_pinned_chunk (void);
977 static void sort_addresses (void **array, int size);
978 static void drain_gray_stack (void);
979 static void finish_gray_stack (char *start_addr, char *end_addr, int generation);
980 static gboolean need_major_collection (void);
981 static void major_collection (const char *reason);
983 static void mono_gc_register_disappearing_link (MonoObject *obj, void **link, gboolean track);
985 void describe_ptr (char *ptr);
986 void check_object (char *start);
988 static void check_consistency (void);
989 static void check_major_refs (void);
990 static void check_section_scan_starts (GCMemSection *section);
991 static void check_scan_starts (void);
992 static void check_for_xdomain_refs (void);
993 static void dump_occupied (char *start, char *end, char *section_start);
994 static void dump_section (GCMemSection *section, const char *type);
995 static void dump_heap (const char *type, int num, const char *reason);
996 static void report_pinned_chunk (PinnedChunk *chunk, int seq);
998 void mono_gc_scan_for_specific_ref (MonoObject *key);
1000 static void init_stats (void);
1002 static int mark_ephemerons_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end);
1003 static void clear_unreachable_ephemerons (CopyOrMarkObjectFunc copy_func, char *start, char *end);
1004 static void null_ephemerons_for_domain (MonoDomain *domain);
1006 //#define BINARY_PROTOCOL
1007 #include "sgen-protocol.c"
1008 #include "sgen-pinning.c"
1009 #include "sgen-pinning-stats.c"
1010 #include "sgen-gray.c"
1011 #include "sgen-los.c"
1014 * ######################################################################
1015 * ######## GC descriptors
1016 * ######################################################################
1017 * Used to quickly get the info the GC needs about an object: size and
1018 * where the references are held.
1020 /* objects are aligned to 8 bytes boundaries
1021 * A descriptor is a pointer in MonoVTable, so 32 or 64 bits of size.
1022 * The low 3 bits define the type of the descriptor. The other bits
1023 * depend on the type.
1024 * As a general rule the 13 remaining low bits define the size, either
1025 * of the whole object or of the elements in the arrays. While for objects
1026 * the size is already in bytes, for arrays we need to shift, because
1027 * array elements might be smaller than 8 bytes. In case of arrays, we
1028 * use two bits to describe what the additional high bits represents,
1029 * so the default behaviour can handle element sizes less than 2048 bytes.
1030 * The high 16 bits, if 0 it means the object is pointer-free.
1031 * This design should make it easy and fast to skip over ptr-free data.
1032 * The first 4 types should cover >95% of the objects.
1033 * Note that since the size of objects is limited to 64K, larger objects
1034 * will be allocated in the large object heap.
1035 * If we want 4-bytes alignment, we need to put vector and small bitmap
1040 * We don't use 0 so that 0 isn't a valid GC descriptor. No
1041 * deep reason for this other than to be able to identify a
1042 * non-inited descriptor for debugging.
1044 * If an object contains no references, its GC descriptor is
1045 * always DESC_TYPE_RUN_LENGTH, without a size, no exceptions.
1046 * This is so that we can quickly check for that in
1047 * copy_object_no_checks(), without having to fetch the
1050 DESC_TYPE_RUN_LENGTH = 1, /* 15 bits aligned byte size | 1-3 (offset, numptr) bytes tuples */
1051 DESC_TYPE_SMALL_BITMAP, /* 15 bits aligned byte size | 16-48 bit bitmap */
1052 DESC_TYPE_COMPLEX, /* index for bitmap into complex_descriptors */
1053 DESC_TYPE_VECTOR, /* 10 bits element size | 1 bit array | 2 bits desc | element desc */
1054 DESC_TYPE_ARRAY, /* 10 bits element size | 1 bit array | 2 bits desc | element desc */
1055 DESC_TYPE_LARGE_BITMAP, /* | 29-61 bitmap bits */
1056 DESC_TYPE_COMPLEX_ARR, /* index for bitmap into complex_descriptors */
1057 /* subtypes for arrays and vectors */
1058 DESC_TYPE_V_PTRFREE = 0,/* there are no refs: keep first so it has a zero value */
1059 DESC_TYPE_V_REFS, /* all the array elements are refs */
1060 DESC_TYPE_V_RUN_LEN, /* elements are run-length encoded as DESC_TYPE_RUN_LENGTH */
1061 DESC_TYPE_V_BITMAP /* elements are as the bitmap in DESC_TYPE_SMALL_BITMAP */
1064 #define OBJECT_HEADER_WORDS (sizeof(MonoObject)/sizeof(gpointer))
1065 #define LOW_TYPE_BITS 3
1066 #define SMALL_BITMAP_SHIFT 16
1067 #define SMALL_BITMAP_SIZE (GC_BITS_PER_WORD - SMALL_BITMAP_SHIFT)
1068 #define VECTOR_INFO_SHIFT 14
1069 #define VECTOR_ELSIZE_SHIFT 3
1070 #define LARGE_BITMAP_SIZE (GC_BITS_PER_WORD - LOW_TYPE_BITS)
1071 #define MAX_ELEMENT_SIZE 0x3ff
1072 #define VECTOR_SUBTYPE_PTRFREE (DESC_TYPE_V_PTRFREE << VECTOR_INFO_SHIFT)
1073 #define VECTOR_SUBTYPE_REFS (DESC_TYPE_V_REFS << VECTOR_INFO_SHIFT)
1074 #define VECTOR_SUBTYPE_RUN_LEN (DESC_TYPE_V_RUN_LEN << VECTOR_INFO_SHIFT)
1075 #define VECTOR_SUBTYPE_BITMAP (DESC_TYPE_V_BITMAP << VECTOR_INFO_SHIFT)
1078 /* Root bitmap descriptors are simpler: the lower three bits describe the type
1079 * and we either have 30/62 bitmap bits or nibble-based run-length,
1080 * or a complex descriptor, or a user defined marker function.
1083 ROOT_DESC_CONSERVATIVE, /* 0, so matches NULL value */
1088 ROOT_DESC_TYPE_MASK = 0x7,
1089 ROOT_DESC_TYPE_SHIFT = 3,
1092 #define MAKE_ROOT_DESC(type,val) ((type) | ((val) << ROOT_DESC_TYPE_SHIFT))
1094 #define MAX_USER_DESCRIPTORS 16
1096 static gsize* complex_descriptors = NULL;
1097 static int complex_descriptors_size = 0;
1098 static int complex_descriptors_next = 0;
1099 static MonoGCRootMarkFunc user_descriptors [MAX_USER_DESCRIPTORS];
1100 static int user_descriptors_next = 0;
1103 alloc_complex_descriptor (gsize *bitmap, int numbits)
1107 numbits = ALIGN_TO (numbits, GC_BITS_PER_WORD);
1108 nwords = numbits / GC_BITS_PER_WORD + 1;
1111 res = complex_descriptors_next;
1112 /* linear search, so we don't have duplicates with domain load/unload
1113 * this should not be performance critical or we'd have bigger issues
1114 * (the number and size of complex descriptors should be small).
1116 for (i = 0; i < complex_descriptors_next; ) {
1117 if (complex_descriptors [i] == nwords) {
1118 int j, found = TRUE;
1119 for (j = 0; j < nwords - 1; ++j) {
1120 if (complex_descriptors [i + 1 + j] != bitmap [j]) {
1130 i += complex_descriptors [i];
1132 if (complex_descriptors_next + nwords > complex_descriptors_size) {
1133 int new_size = complex_descriptors_size * 2 + nwords;
1134 complex_descriptors = g_realloc (complex_descriptors, new_size * sizeof (gsize));
1135 complex_descriptors_size = new_size;
1137 DEBUG (6, fprintf (gc_debug_file, "Complex descriptor %d, size: %d (total desc memory: %d)\n", res, nwords, complex_descriptors_size));
1138 complex_descriptors_next += nwords;
1139 complex_descriptors [res] = nwords;
1140 for (i = 0; i < nwords - 1; ++i) {
1141 complex_descriptors [res + 1 + i] = bitmap [i];
1142 DEBUG (6, fprintf (gc_debug_file, "\tvalue: %p\n", (void*)complex_descriptors [res + 1 + i]));
1149 * Descriptor builders.
1152 mono_gc_make_descr_for_string (gsize *bitmap, int numbits)
1154 return (void*) DESC_TYPE_RUN_LENGTH;
1158 mono_gc_make_descr_for_object (gsize *bitmap, int numbits, size_t obj_size)
1160 int first_set = -1, num_set = 0, last_set = -1, i;
1162 size_t stored_size = obj_size;
1163 for (i = 0; i < numbits; ++i) {
1164 if (bitmap [i / GC_BITS_PER_WORD] & ((gsize)1 << (i % GC_BITS_PER_WORD))) {
1172 * We don't encode the size of types that don't contain
1173 * references because they might not be aligned, i.e. the
1174 * bottom two bits might be set, which would clash with the
1175 * bits we need to encode the descriptor type. Since we don't
1176 * use the encoded size to skip objects, other than for
1177 * processing remsets, in which case only the positions of
1178 * references are relevant, this is not a problem.
1181 return (void*)DESC_TYPE_RUN_LENGTH;
1182 g_assert (!(stored_size & 0x3));
1183 if (stored_size <= MAX_SMALL_OBJ_SIZE) {
1184 /* check run-length encoding first: one byte offset, one byte number of pointers
1185 * on 64 bit archs, we can have 3 runs, just one on 32.
1186 * It may be better to use nibbles.
1188 if (first_set < 0) {
1189 desc = DESC_TYPE_RUN_LENGTH | (stored_size << 1);
1190 DEBUG (6, fprintf (gc_debug_file, "Ptrfree descriptor %p, size: %zd\n", (void*)desc, stored_size));
1191 return (void*) desc;
1192 } else if (first_set < 256 && num_set < 256 && (first_set + num_set == last_set + 1)) {
1193 desc = DESC_TYPE_RUN_LENGTH | (stored_size << 1) | (first_set << 16) | (num_set << 24);
1194 DEBUG (6, fprintf (gc_debug_file, "Runlen descriptor %p, size: %zd, first set: %d, num set: %d\n", (void*)desc, stored_size, first_set, num_set));
1195 return (void*) desc;
1197 /* we know the 2-word header is ptr-free */
1198 if (last_set < SMALL_BITMAP_SIZE + OBJECT_HEADER_WORDS) {
1199 desc = DESC_TYPE_SMALL_BITMAP | (stored_size << 1) | ((*bitmap >> OBJECT_HEADER_WORDS) << SMALL_BITMAP_SHIFT);
1200 DEBUG (6, fprintf (gc_debug_file, "Smallbitmap descriptor %p, size: %zd, last set: %d\n", (void*)desc, stored_size, last_set));
1201 return (void*) desc;
1204 /* we know the 2-word header is ptr-free */
1205 if (last_set < LARGE_BITMAP_SIZE + OBJECT_HEADER_WORDS) {
1206 desc = DESC_TYPE_LARGE_BITMAP | ((*bitmap >> OBJECT_HEADER_WORDS) << LOW_TYPE_BITS);
1207 DEBUG (6, fprintf (gc_debug_file, "Largebitmap descriptor %p, size: %zd, last set: %d\n", (void*)desc, stored_size, last_set));
1208 return (void*) desc;
1210 /* it's a complex object ... */
1211 desc = DESC_TYPE_COMPLEX | (alloc_complex_descriptor (bitmap, last_set + 1) << LOW_TYPE_BITS);
1212 return (void*) desc;
1215 /* If the array holds references, numbits == 1 and the first bit is set in elem_bitmap */
1217 mono_gc_make_descr_for_array (int vector, gsize *elem_bitmap, int numbits, size_t elem_size)
1219 int first_set = -1, num_set = 0, last_set = -1, i;
1220 mword desc = vector? DESC_TYPE_VECTOR: DESC_TYPE_ARRAY;
1221 for (i = 0; i < numbits; ++i) {
1222 if (elem_bitmap [i / GC_BITS_PER_WORD] & ((gsize)1 << (i % GC_BITS_PER_WORD))) {
1229 /* See comment at the definition of DESC_TYPE_RUN_LENGTH. */
1231 return (void*)DESC_TYPE_RUN_LENGTH;
1232 if (elem_size <= MAX_ELEMENT_SIZE) {
1233 desc |= elem_size << VECTOR_ELSIZE_SHIFT;
1235 return (void*)(desc | VECTOR_SUBTYPE_PTRFREE);
1237 /* Note: we also handle structs with just ref fields */
1238 if (num_set * sizeof (gpointer) == elem_size) {
1239 return (void*)(desc | VECTOR_SUBTYPE_REFS | ((gssize)(-1) << 16));
1241 /* FIXME: try run-len first */
1242 /* Note: we can't skip the object header here, because it's not present */
1243 if (last_set <= SMALL_BITMAP_SIZE) {
1244 return (void*)(desc | VECTOR_SUBTYPE_BITMAP | (*elem_bitmap << 16));
1247 /* it's am array of complex structs ... */
1248 desc = DESC_TYPE_COMPLEX_ARR;
1249 desc |= alloc_complex_descriptor (elem_bitmap, last_set + 1) << LOW_TYPE_BITS;
1250 return (void*) desc;
1253 /* Return the bitmap encoded by a descriptor */
1255 mono_gc_get_bitmap_for_descr (void *descr, int *numbits)
1257 mword d = (mword)descr;
1261 case DESC_TYPE_RUN_LENGTH: {
1262 int first_set = (d >> 16) & 0xff;
1263 int num_set = (d >> 24) & 0xff;
1266 bitmap = g_new0 (gsize, (first_set + num_set + 7) / 8);
1268 for (i = first_set; i < first_set + num_set; ++i)
1269 bitmap [i / GC_BITS_PER_WORD] |= ((gsize)1 << (i % GC_BITS_PER_WORD));
1271 *numbits = first_set + num_set;
1275 case DESC_TYPE_SMALL_BITMAP:
1276 bitmap = g_new0 (gsize, 1);
1278 bitmap [0] = (d >> SMALL_BITMAP_SHIFT) << OBJECT_HEADER_WORDS;
1280 *numbits = GC_BITS_PER_WORD;
1284 g_assert_not_reached ();
1288 /* helper macros to scan and traverse objects, macros because we resue them in many functions */
1289 #define STRING_SIZE(size,str) do { \
1290 (size) = sizeof (MonoString) + 2 * mono_string_length_fast ((MonoString*)(str)) + 2; \
1291 (size) += (ALLOC_ALIGN - 1); \
1292 (size) &= ~(ALLOC_ALIGN - 1); \
1295 #define OBJ_RUN_LEN_SIZE(size,desc,obj) do { \
1296 (size) = ((desc) & 0xfff8) >> 1; \
1299 #define OBJ_BITMAP_SIZE(size,desc,obj) do { \
1300 (size) = ((desc) & 0xfff8) >> 1; \
1303 //#define PREFETCH(addr) __asm__ __volatile__ (" prefetchnta %0": : "m"(*(char *)(addr)))
1304 #define PREFETCH(addr)
1306 /* code using these macros must define a HANDLE_PTR(ptr) macro that does the work */
1307 #define OBJ_RUN_LEN_FOREACH_PTR(desc,obj) do { \
1308 if ((desc) & 0xffff0000) { \
1309 /* there are pointers */ \
1310 void **_objptr_end; \
1311 void **_objptr = (void**)(obj); \
1312 _objptr += ((desc) >> 16) & 0xff; \
1313 _objptr_end = _objptr + (((desc) >> 24) & 0xff); \
1314 while (_objptr < _objptr_end) { \
1315 HANDLE_PTR (_objptr, (obj)); \
1321 /* a bitmap desc means that there are pointer references or we'd have
1322 * choosen run-length, instead: add an assert to check.
1324 #define OBJ_BITMAP_FOREACH_PTR(desc,obj) do { \
1325 /* there are pointers */ \
1326 void **_objptr = (void**)(obj); \
1327 gsize _bmap = (desc) >> 16; \
1328 _objptr += OBJECT_HEADER_WORDS; \
1330 if ((_bmap & 1)) { \
1331 HANDLE_PTR (_objptr, (obj)); \
1338 #define OBJ_LARGE_BITMAP_FOREACH_PTR(vt,obj) do { \
1339 /* there are pointers */ \
1340 void **_objptr = (void**)(obj); \
1341 gsize _bmap = (vt)->desc >> LOW_TYPE_BITS; \
1342 _objptr += OBJECT_HEADER_WORDS; \
1344 if ((_bmap & 1)) { \
1345 HANDLE_PTR (_objptr, (obj)); \
1352 #define OBJ_COMPLEX_FOREACH_PTR(vt,obj) do { \
1353 /* there are pointers */ \
1354 void **_objptr = (void**)(obj); \
1355 gsize *bitmap_data = complex_descriptors + ((vt)->desc >> LOW_TYPE_BITS); \
1356 int bwords = (*bitmap_data) - 1; \
1357 void **start_run = _objptr; \
1360 MonoObject *myobj = (MonoObject*)obj; \
1361 g_print ("found %d at %p (0x%zx): %s.%s\n", bwords, (obj), (vt)->desc, myobj->vtable->klass->name_space, myobj->vtable->klass->name); \
1363 while (bwords-- > 0) { \
1364 gsize _bmap = *bitmap_data++; \
1365 _objptr = start_run; \
1366 /*g_print ("bitmap: 0x%x/%d at %p\n", _bmap, bwords, _objptr);*/ \
1368 if ((_bmap & 1)) { \
1369 HANDLE_PTR (_objptr, (obj)); \
1374 start_run += GC_BITS_PER_WORD; \
1378 /* this one is untested */
1379 #define OBJ_COMPLEX_ARR_FOREACH_PTR(vt,obj) do { \
1380 /* there are pointers */ \
1381 gsize *mbitmap_data = complex_descriptors + ((vt)->desc >> LOW_TYPE_BITS); \
1382 int mbwords = (*mbitmap_data++) - 1; \
1383 int el_size = mono_array_element_size (vt->klass); \
1384 char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
1385 char *e_end = e_start + el_size * mono_array_length_fast ((MonoArray*)(obj)); \
1387 g_print ("found %d at %p (0x%zx): %s.%s\n", mbwords, (obj), (vt)->desc, vt->klass->name_space, vt->klass->name); \
1388 while (e_start < e_end) { \
1389 void **_objptr = (void**)e_start; \
1390 gsize *bitmap_data = mbitmap_data; \
1391 unsigned int bwords = mbwords; \
1392 while (bwords-- > 0) { \
1393 gsize _bmap = *bitmap_data++; \
1394 void **start_run = _objptr; \
1395 /*g_print ("bitmap: 0x%x\n", _bmap);*/ \
1397 if ((_bmap & 1)) { \
1398 HANDLE_PTR (_objptr, (obj)); \
1403 _objptr = start_run + GC_BITS_PER_WORD; \
1405 e_start += el_size; \
1409 #define OBJ_VECTOR_FOREACH_PTR(vt,obj) do { \
1410 /* note: 0xffffc000 excludes DESC_TYPE_V_PTRFREE */ \
1411 if ((vt)->desc & 0xffffc000) { \
1412 int el_size = ((vt)->desc >> 3) & MAX_ELEMENT_SIZE; \
1413 /* there are pointers */ \
1414 int etype = (vt)->desc & 0xc000; \
1415 if (etype == (DESC_TYPE_V_REFS << 14)) { \
1416 void **p = (void**)((char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector)); \
1417 void **end_refs = (void**)((char*)p + el_size * mono_array_length_fast ((MonoArray*)(obj))); \
1418 /* Note: this code can handle also arrays of struct with only references in them */ \
1419 while (p < end_refs) { \
1420 HANDLE_PTR (p, (obj)); \
1423 } else if (etype == DESC_TYPE_V_RUN_LEN << 14) { \
1424 int offset = ((vt)->desc >> 16) & 0xff; \
1425 int num_refs = ((vt)->desc >> 24) & 0xff; \
1426 char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
1427 char *e_end = e_start + el_size * mono_array_length_fast ((MonoArray*)(obj)); \
1428 while (e_start < e_end) { \
1429 void **p = (void**)e_start; \
1432 for (i = 0; i < num_refs; ++i) { \
1433 HANDLE_PTR (p + i, (obj)); \
1435 e_start += el_size; \
1437 } else if (etype == DESC_TYPE_V_BITMAP << 14) { \
1438 char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
1439 char *e_end = e_start + el_size * mono_array_length_fast ((MonoArray*)(obj)); \
1440 while (e_start < e_end) { \
1441 void **p = (void**)e_start; \
1442 gsize _bmap = (vt)->desc >> 16; \
1443 /* Note: there is no object header here to skip */ \
1445 if ((_bmap & 1)) { \
1446 HANDLE_PTR (p, (obj)); \
1451 e_start += el_size; \
1457 //#include "sgen-major-copying.c"
1458 #include "sgen-marksweep.c"
1461 is_xdomain_ref_allowed (gpointer *ptr, char *obj, MonoDomain *domain)
1463 MonoObject *o = (MonoObject*)(obj);
1464 MonoObject *ref = (MonoObject*)*(ptr);
1465 int offset = (char*)(ptr) - (char*)o;
1467 if (o->vtable->klass == mono_defaults.thread_class && offset == G_STRUCT_OFFSET (MonoThread, internal_thread))
1469 if (o->vtable->klass == mono_defaults.internal_thread_class && offset == G_STRUCT_OFFSET (MonoInternalThread, current_appcontext))
1471 if (mono_class_has_parent (o->vtable->klass, mono_defaults.real_proxy_class) &&
1472 offset == G_STRUCT_OFFSET (MonoRealProxy, unwrapped_server))
1474 /* Thread.cached_culture_info */
1475 if (!strcmp (ref->vtable->klass->name_space, "System.Globalization") &&
1476 !strcmp (ref->vtable->klass->name, "CultureInfo") &&
1477 !strcmp(o->vtable->klass->name_space, "System") &&
1478 !strcmp(o->vtable->klass->name, "Object[]"))
1481 * at System.IO.MemoryStream.InternalConstructor (byte[],int,int,bool,bool) [0x0004d] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.IO/MemoryStream.cs:121
1482 * at System.IO.MemoryStream..ctor (byte[]) [0x00017] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.IO/MemoryStream.cs:81
1483 * at (wrapper remoting-invoke-with-check) System.IO.MemoryStream..ctor (byte[]) <IL 0x00020, 0xffffffff>
1484 * at System.Runtime.Remoting.Messaging.CADMethodCallMessage.GetArguments () [0x0000d] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Messaging/CADMessages.cs:327
1485 * at System.Runtime.Remoting.Messaging.MethodCall..ctor (System.Runtime.Remoting.Messaging.CADMethodCallMessage) [0x00017] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Messaging/MethodCall.cs:87
1486 * at System.AppDomain.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage,byte[]&,System.Runtime.Remoting.Messaging.CADMethodReturnMessage&) [0x00018] in /home/schani/Work/novell/trunk/mcs/class/corlib/System/AppDomain.cs:1213
1487 * at (wrapper remoting-invoke-with-check) System.AppDomain.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage,byte[]&,System.Runtime.Remoting.Messaging.CADMethodReturnMessage&) <IL 0x0003d, 0xffffffff>
1488 * at System.Runtime.Remoting.Channels.CrossAppDomainSink.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage) [0x00008] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Channels/CrossAppDomainChannel.cs:198
1489 * at (wrapper runtime-invoke) object.runtime_invoke_CrossAppDomainSink/ProcessMessageRes_object_object (object,intptr,intptr,intptr) <IL 0x0004c, 0xffffffff>
1491 if (!strcmp (ref->vtable->klass->name_space, "System") &&
1492 !strcmp (ref->vtable->klass->name, "Byte[]") &&
1493 !strcmp (o->vtable->klass->name_space, "System.IO") &&
1494 !strcmp (o->vtable->klass->name, "MemoryStream"))
1496 /* append_job() in threadpool.c */
1497 if (!strcmp (ref->vtable->klass->name_space, "System.Runtime.Remoting.Messaging") &&
1498 !strcmp (ref->vtable->klass->name, "AsyncResult") &&
1499 !strcmp (o->vtable->klass->name_space, "System") &&
1500 !strcmp (o->vtable->klass->name, "Object[]") &&
1501 mono_thread_pool_is_queue_array ((MonoArray*) o))
1507 check_reference_for_xdomain (gpointer *ptr, char *obj, MonoDomain *domain)
1509 MonoObject *o = (MonoObject*)(obj);
1510 MonoObject *ref = (MonoObject*)*(ptr);
1511 int offset = (char*)(ptr) - (char*)o;
1513 MonoClassField *field;
1516 if (!ref || ref->vtable->domain == domain)
1518 if (is_xdomain_ref_allowed (ptr, obj, domain))
1522 for (class = o->vtable->klass; class; class = class->parent) {
1525 for (i = 0; i < class->field.count; ++i) {
1526 if (class->fields[i].offset == offset) {
1527 field = &class->fields[i];
1535 if (ref->vtable->klass == mono_defaults.string_class)
1536 str = mono_string_to_utf8 ((MonoString*)ref);
1539 g_print ("xdomain reference in %p (%s.%s) at offset %d (%s) to %p (%s.%s) (%s) - pointed to by:\n",
1540 o, o->vtable->klass->name_space, o->vtable->klass->name,
1541 offset, field ? field->name : "",
1542 ref, ref->vtable->klass->name_space, ref->vtable->klass->name, str ? str : "");
1543 mono_gc_scan_for_specific_ref (o);
1549 #define HANDLE_PTR(ptr,obj) check_reference_for_xdomain ((ptr), (obj), domain)
1552 scan_object_for_xdomain_refs (char *start, mword size, void *data)
1554 MonoDomain *domain = ((MonoObject*)start)->vtable->domain;
1556 #include "sgen-scan-object.h"
1560 #define HANDLE_PTR(ptr,obj) do { \
1561 if ((MonoObject*)*(ptr) == key) { \
1562 g_print ("found ref to %p in object %p (%s) at offset %td\n", \
1563 key, (obj), safe_name ((obj)), ((char*)(ptr) - (char*)(obj))); \
1568 scan_object_for_specific_ref (char *start, MonoObject *key)
1570 #include "sgen-scan-object.h"
1574 scan_area_with_callback (char *start, char *end, IterateObjectCallbackFunc callback, void *data)
1576 while (start < end) {
1578 if (!*(void**)start) {
1579 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
1583 size = safe_object_get_size ((MonoObject*) start);
1584 size += ALLOC_ALIGN - 1;
1585 size &= ~(ALLOC_ALIGN - 1);
1587 callback (start, size, data);
1594 scan_object_for_specific_ref_callback (char *obj, size_t size, MonoObject *key)
1596 scan_object_for_specific_ref (obj, key);
1600 check_root_obj_specific_ref (RootRecord *root, MonoObject *key, MonoObject *obj)
1604 g_print ("found ref to %p in root record %p\n", key, root);
1607 static MonoObject *check_key = NULL;
1608 static RootRecord *check_root = NULL;
1611 check_root_obj_specific_ref_from_marker (void **obj)
1613 check_root_obj_specific_ref (check_root, check_key, *obj);
1617 scan_roots_for_specific_ref (MonoObject *key, int root_type)
1622 for (i = 0; i < roots_hash_size [root_type]; ++i) {
1623 for (root = roots_hash [root_type][i]; root; root = root->next) {
1624 void **start_root = (void**)root->start_root;
1625 mword desc = root->root_desc;
1629 switch (desc & ROOT_DESC_TYPE_MASK) {
1630 case ROOT_DESC_BITMAP:
1631 desc >>= ROOT_DESC_TYPE_SHIFT;
1634 check_root_obj_specific_ref (root, key, *start_root);
1639 case ROOT_DESC_COMPLEX: {
1640 gsize *bitmap_data = complex_descriptors + (desc >> ROOT_DESC_TYPE_SHIFT);
1641 int bwords = (*bitmap_data) - 1;
1642 void **start_run = start_root;
1644 while (bwords-- > 0) {
1645 gsize bmap = *bitmap_data++;
1646 void **objptr = start_run;
1649 check_root_obj_specific_ref (root, key, *objptr);
1653 start_run += GC_BITS_PER_WORD;
1657 case ROOT_DESC_USER: {
1658 MonoGCRootMarkFunc marker = user_descriptors [desc >> ROOT_DESC_TYPE_SHIFT];
1659 marker (start_root, check_root_obj_specific_ref_from_marker);
1662 case ROOT_DESC_RUN_LEN:
1663 g_assert_not_reached ();
1665 g_assert_not_reached ();
1674 mono_gc_scan_for_specific_ref (MonoObject *key)
1680 scan_area_with_callback (nursery_section->data, nursery_section->end_data,
1681 (IterateObjectCallbackFunc)scan_object_for_specific_ref_callback, key);
1683 major_iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)scan_object_for_specific_ref_callback, key);
1685 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1686 scan_object_for_specific_ref (bigobj->data, key);
1688 scan_roots_for_specific_ref (key, ROOT_TYPE_NORMAL);
1689 scan_roots_for_specific_ref (key, ROOT_TYPE_WBARRIER);
1691 for (i = 0; i < roots_hash_size [ROOT_TYPE_PINNED]; ++i) {
1692 for (root = roots_hash [ROOT_TYPE_PINNED][i]; root; root = root->next) {
1693 void **ptr = (void**)root->start_root;
1695 while (ptr < (void**)root->end_root) {
1696 check_root_obj_specific_ref (root, *ptr, key);
1703 /* Clear all remaining nursery fragments */
1705 clear_nursery_fragments (char *next)
1708 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION) {
1709 g_assert (next <= nursery_frag_real_end);
1710 memset (next, 0, nursery_frag_real_end - next);
1711 for (frag = nursery_fragments; frag; frag = frag->next) {
1712 memset (frag->fragment_start, 0, frag->fragment_end - frag->fragment_start);
1718 need_remove_object_for_domain (char *start, MonoDomain *domain)
1720 if (mono_object_domain (start) == domain) {
1721 DEBUG (4, fprintf (gc_debug_file, "Need to cleanup object %p\n", start));
1722 binary_protocol_cleanup (start, (gpointer)LOAD_VTABLE (start), safe_object_get_size ((MonoObject*)start));
1729 process_object_for_domain_clearing (char *start, MonoDomain *domain)
1731 GCVTable *vt = (GCVTable*)LOAD_VTABLE (start);
1732 if (vt->klass == mono_defaults.internal_thread_class)
1733 g_assert (mono_object_domain (start) == mono_get_root_domain ());
1734 /* The object could be a proxy for an object in the domain
1736 if (mono_class_has_parent (vt->klass, mono_defaults.real_proxy_class)) {
1737 MonoObject *server = ((MonoRealProxy*)start)->unwrapped_server;
1739 /* The server could already have been zeroed out, so
1740 we need to check for that, too. */
1741 if (server && (!LOAD_VTABLE (server) || mono_object_domain (server) == domain)) {
1742 DEBUG (4, fprintf (gc_debug_file, "Cleaning up remote pointer in %p to object %p\n",
1744 ((MonoRealProxy*)start)->unwrapped_server = NULL;
1749 static MonoDomain *check_domain = NULL;
1752 check_obj_not_in_domain (void **o)
1754 g_assert (((MonoObject*)(*o))->vtable->domain != check_domain);
1758 scan_for_registered_roots_in_domain (MonoDomain *domain, int root_type)
1762 check_domain = domain;
1763 for (i = 0; i < roots_hash_size [root_type]; ++i) {
1764 for (root = roots_hash [root_type][i]; root; root = root->next) {
1765 void **start_root = (void**)root->start_root;
1766 mword desc = root->root_desc;
1768 /* The MonoDomain struct is allowed to hold
1769 references to objects in its own domain. */
1770 if (start_root == (void**)domain)
1773 switch (desc & ROOT_DESC_TYPE_MASK) {
1774 case ROOT_DESC_BITMAP:
1775 desc >>= ROOT_DESC_TYPE_SHIFT;
1777 if ((desc & 1) && *start_root)
1778 check_obj_not_in_domain (*start_root);
1783 case ROOT_DESC_COMPLEX: {
1784 gsize *bitmap_data = complex_descriptors + (desc >> ROOT_DESC_TYPE_SHIFT);
1785 int bwords = (*bitmap_data) - 1;
1786 void **start_run = start_root;
1788 while (bwords-- > 0) {
1789 gsize bmap = *bitmap_data++;
1790 void **objptr = start_run;
1792 if ((bmap & 1) && *objptr)
1793 check_obj_not_in_domain (*objptr);
1797 start_run += GC_BITS_PER_WORD;
1801 case ROOT_DESC_USER: {
1802 MonoGCRootMarkFunc marker = user_descriptors [desc >> ROOT_DESC_TYPE_SHIFT];
1803 marker (start_root, check_obj_not_in_domain);
1806 case ROOT_DESC_RUN_LEN:
1807 g_assert_not_reached ();
1809 g_assert_not_reached ();
1813 check_domain = NULL;
1817 check_for_xdomain_refs (void)
1821 scan_area_with_callback (nursery_section->data, nursery_section->end_data, (IterateObjectCallbackFunc)scan_object_for_xdomain_refs, NULL);
1823 major_iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)scan_object_for_xdomain_refs, NULL);
1825 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1826 scan_object_for_xdomain_refs (bigobj->data, bigobj->size, NULL);
1830 clear_domain_process_object (char *obj, MonoDomain *domain)
1834 process_object_for_domain_clearing (obj, domain);
1835 remove = need_remove_object_for_domain (obj, domain);
1837 if (remove && ((MonoObject*)obj)->synchronisation) {
1838 void **dislink = mono_monitor_get_object_monitor_weak_link ((MonoObject*)obj);
1840 mono_gc_register_disappearing_link (NULL, dislink, FALSE);
1847 clear_domain_process_minor_object_callback (char *obj, size_t size, MonoDomain *domain)
1849 if (clear_domain_process_object (obj, domain))
1850 memset (obj, 0, size);
1854 clear_domain_process_major_object_callback (char *obj, size_t size, MonoDomain *domain)
1856 clear_domain_process_object (obj, domain);
1860 clear_domain_free_major_non_pinned_object_callback (char *obj, size_t size, MonoDomain *domain)
1862 if (need_remove_object_for_domain (obj, domain))
1863 major_free_non_pinned_object (obj, size);
1867 clear_domain_free_major_pinned_object_callback (char *obj, size_t size, MonoDomain *domain)
1869 if (need_remove_object_for_domain (obj, domain))
1870 free_pinned_object (obj, size);
1874 * When appdomains are unloaded we can easily remove objects that have finalizers,
1875 * but all the others could still be present in random places on the heap.
1876 * We need a sweep to get rid of them even though it's going to be costly
1878 * The reason we need to remove them is because we access the vtable and class
1879 * structures to know the object size and the reference bitmap: once the domain is
1880 * unloaded the point to random memory.
1883 mono_gc_clear_domain (MonoDomain * domain)
1885 LOSObject *bigobj, *prev;
1890 clear_nursery_fragments (nursery_next);
1892 if (xdomain_checks && domain != mono_get_root_domain ()) {
1893 scan_for_registered_roots_in_domain (domain, ROOT_TYPE_NORMAL);
1894 scan_for_registered_roots_in_domain (domain, ROOT_TYPE_WBARRIER);
1895 check_for_xdomain_refs ();
1898 scan_area_with_callback (nursery_section->data, nursery_section->end_data,
1899 (IterateObjectCallbackFunc)clear_domain_process_minor_object_callback, domain);
1901 /*Ephemerons and dislinks must be processed before LOS since they might end up pointing
1902 to memory returned to the OS.*/
1903 null_ephemerons_for_domain (domain);
1905 for (i = GENERATION_NURSERY; i < GENERATION_MAX; ++i)
1906 null_links_for_domain (domain, i);
1908 /* We need two passes over major and large objects because
1909 freeing such objects might give their memory back to the OS
1910 (in the case of large objects) or obliterate its vtable
1911 (pinned objects with major-copying or pinned and non-pinned
1912 objects with major-mark&sweep), but we might need to
1913 dereference a pointer from an object to another object if
1914 the first object is a proxy. */
1915 major_iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)clear_domain_process_major_object_callback, domain);
1916 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1917 clear_domain_process_object (bigobj->data, domain);
1920 for (bigobj = los_object_list; bigobj;) {
1921 if (need_remove_object_for_domain (bigobj->data, domain)) {
1922 LOSObject *to_free = bigobj;
1924 prev->next = bigobj->next;
1926 los_object_list = bigobj->next;
1927 bigobj = bigobj->next;
1928 DEBUG (4, fprintf (gc_debug_file, "Freeing large object %p\n",
1930 free_large_object (to_free);
1934 bigobj = bigobj->next;
1936 major_iterate_objects (TRUE, FALSE, (IterateObjectCallbackFunc)clear_domain_free_major_non_pinned_object_callback, domain);
1937 major_iterate_objects (FALSE, TRUE, (IterateObjectCallbackFunc)clear_domain_free_major_pinned_object_callback, domain);
1943 global_remset_cache_clear (void)
1945 memset (global_remset_cache, 0, sizeof (global_remset_cache));
1949 * Tries to check if a given remset location was already added to the global remset.
1952 * A 2 entry, LRU cache of recently saw location remsets.
1954 * It's hand-coded instead of done using loops to reduce the number of memory references on cache hit.
1956 * Returns TRUE is the element was added..
1959 global_remset_location_was_not_added (gpointer ptr)
1962 gpointer first = global_remset_cache [0], second;
1964 HEAVY_STAT (++stat_global_remsets_discarded);
1968 second = global_remset_cache [1];
1970 if (second == ptr) {
1971 /*Move the second to the front*/
1972 global_remset_cache [0] = second;
1973 global_remset_cache [1] = first;
1975 HEAVY_STAT (++stat_global_remsets_discarded);
1979 global_remset_cache [0] = second;
1980 global_remset_cache [1] = ptr;
1985 * add_to_global_remset:
1987 * The global remset contains locations which point into newspace after
1988 * a minor collection. This can happen if the objects they point to are pinned.
1991 add_to_global_remset (gpointer ptr)
1995 g_assert (!ptr_in_nursery (ptr) && ptr_in_nursery (*(gpointer*)ptr));
1997 if (!global_remset_location_was_not_added (ptr))
2000 DEBUG (8, fprintf (gc_debug_file, "Adding global remset for %p\n", ptr));
2001 binary_protocol_global_remset (ptr, *(gpointer*)ptr, (gpointer)LOAD_VTABLE (*(gpointer*)ptr));
2003 HEAVY_STAT (++stat_global_remsets_added);
2006 * FIXME: If an object remains pinned, we need to add it at every minor collection.
2007 * To avoid uncontrolled growth of the global remset, only add each pointer once.
2009 if (global_remset->store_next + 3 < global_remset->end_set) {
2010 *(global_remset->store_next++) = (mword)ptr;
2013 rs = alloc_remset (global_remset->end_set - global_remset->data, NULL);
2014 rs->next = global_remset;
2016 *(global_remset->store_next++) = (mword)ptr;
2019 int global_rs_size = 0;
2021 for (rs = global_remset; rs; rs = rs->next) {
2022 global_rs_size += rs->store_next - rs->data;
2024 DEBUG (4, fprintf (gc_debug_file, "Global remset now has size %d\n", global_rs_size));
2029 * FIXME: allocate before calling this function and pass the
2030 * destination address.
2033 copy_object_no_checks (void *obj)
2035 static const void *copy_labels [] = { &&LAB_0, &&LAB_1, &&LAB_2, &&LAB_3, &&LAB_4, &&LAB_5, &&LAB_6, &&LAB_7, &&LAB_8 };
2039 MonoVTable *vt = ((MonoObject*)obj)->vtable;
2040 gboolean has_references = vt->gc_descr != (void*)DESC_TYPE_RUN_LENGTH;
2042 objsize = safe_object_get_size ((MonoObject*)obj);
2043 objsize += ALLOC_ALIGN - 1;
2044 objsize &= ~(ALLOC_ALIGN - 1);
2046 DEBUG (9, g_assert (vt->klass->inited));
2047 MAJOR_GET_COPY_OBJECT_SPACE (destination, objsize, has_references);
2049 DEBUG (9, fprintf (gc_debug_file, " (to %p, %s size: %lu)\n", destination, ((MonoObject*)obj)->vtable->klass->name, (unsigned long)objsize));
2050 binary_protocol_copy (obj, destination, ((MonoObject*)obj)->vtable, objsize);
2052 if (objsize <= sizeof (gpointer) * 8) {
2053 mword *dest = (mword*)destination;
2054 goto *copy_labels [objsize / sizeof (gpointer)];
2056 (dest) [7] = ((mword*)obj) [7];
2058 (dest) [6] = ((mword*)obj) [6];
2060 (dest) [5] = ((mword*)obj) [5];
2062 (dest) [4] = ((mword*)obj) [4];
2064 (dest) [3] = ((mword*)obj) [3];
2066 (dest) [2] = ((mword*)obj) [2];
2068 (dest) [1] = ((mword*)obj) [1];
2070 (dest) [0] = ((mword*)obj) [0];
2078 char* edi = destination;
2079 __asm__ __volatile__(
2081 : "=&c" (ecx), "=&D" (edi), "=&S" (esi)
2082 : "0" (objsize/4), "1" (edi),"2" (esi)
2087 memcpy (destination, obj, objsize);
2090 /* adjust array->bounds */
2091 DEBUG (9, g_assert (vt->gc_descr));
2092 if (G_UNLIKELY (vt->rank && ((MonoArray*)obj)->bounds)) {
2093 MonoArray *array = (MonoArray*)destination;
2094 array->bounds = (MonoArrayBounds*)((char*)destination + ((char*)((MonoArray*)obj)->bounds - (char*)obj));
2095 DEBUG (9, fprintf (gc_debug_file, "Array instance %p: size: %lu, rank: %d, length: %lu\n", array, (unsigned long)objsize, vt->rank, (unsigned long)mono_array_length (array)));
2097 /* set the forwarding pointer */
2098 forward_object (obj, destination);
2099 if (G_UNLIKELY (mono_profiler_events & MONO_PROFILE_GC_MOVES)) {
2100 if (moved_objects_idx == MOVED_OBJECTS_NUM) {
2101 mono_profiler_gc_moves (moved_objects, moved_objects_idx);
2102 moved_objects_idx = 0;
2104 moved_objects [moved_objects_idx++] = obj;
2105 moved_objects [moved_objects_idx++] = destination;
2108 if (has_references) {
2109 DEBUG (9, fprintf (gc_debug_file, "Enqueuing gray object %p (%s)\n", obj, safe_name (obj)));
2110 GRAY_OBJECT_ENQUEUE (obj);
2116 * This is how the copying happens from the nursery to the old generation.
2117 * We assume that at this time all the pinned objects have been identified and
2119 * We run scan_object() for each pinned object so that each referenced
2120 * objects if possible are copied. The new gray objects created can have
2121 * scan_object() run on them right away, too.
2122 * Then we run copy_object() for the precisely tracked roots. At this point
2123 * all the roots are either gray or black. We run scan_object() on the gray
2124 * objects until no more gray objects are created.
2125 * At the end of the process we walk again the pinned list and we unmark
2126 * the pinned flag. As we go we also create the list of free space for use
2127 * in the next allocation runs.
2129 * We need to remember objects from the old generation that point to the new one
2130 * (or just addresses?).
2132 * copy_object could be made into a macro once debugged (use inline for now).
2135 static void __attribute__((noinline))
2136 copy_object (void **obj_slot)
2139 char *obj = *obj_slot;
2141 DEBUG (9, g_assert (current_collection_generation == GENERATION_NURSERY));
2143 HEAVY_STAT (++stat_copy_object_called_nursery);
2145 if (!ptr_in_nursery (obj)) {
2146 HEAVY_STAT (++stat_nursery_copy_object_failed_from_space);
2150 DEBUG (9, fprintf (gc_debug_file, "Precise copy of %p from %p", obj, obj_slot));
2153 * Before we can copy the object we must make sure that we are
2154 * allowed to, i.e. that the object not pinned or not already
2158 if ((forwarded = object_is_forwarded (obj))) {
2159 DEBUG (9, g_assert (((MonoVTable*)LOAD_VTABLE(obj))->gc_descr));
2160 DEBUG (9, fprintf (gc_debug_file, " (already forwarded to %p)\n", forwarded));
2161 HEAVY_STAT (++stat_nursery_copy_object_failed_forwarded);
2162 *obj_slot = forwarded;
2165 if (object_is_pinned (obj)) {
2166 DEBUG (9, g_assert (((MonoVTable*)LOAD_VTABLE(obj))->gc_descr));
2167 DEBUG (9, fprintf (gc_debug_file, " (pinned, no change)\n"));
2168 HEAVY_STAT (++stat_nursery_copy_object_failed_pinned);
2172 HEAVY_STAT (++stat_objects_copied_nursery);
2174 *obj_slot = copy_object_no_checks (obj);
2178 #define HANDLE_PTR(ptr,obj) do { \
2179 void *__old = *(ptr); \
2182 copy_object ((ptr)); \
2184 DEBUG (9, if (__old != __copy) fprintf (gc_debug_file, "Overwrote field at %p with %p (was: %p)\n", (ptr), *(ptr), __old)); \
2185 if (G_UNLIKELY (ptr_in_nursery (__copy) && !ptr_in_nursery ((ptr)))) \
2186 add_to_global_remset ((ptr)); \
2191 * Scan the object pointed to by @start for references to
2192 * other objects between @from_start and @from_end and copy
2193 * them to the gray_objects area.
2196 scan_object (char *start)
2198 #include "sgen-scan-object.h"
2200 HEAVY_STAT (++stat_scan_object_called_nursery);
2206 * Scan the valuetype pointed to by START, described by DESC for references to
2207 * other objects between @from_start and @from_end and copy them to the gray_objects area.
2208 * Returns a pointer to the end of the object.
2211 scan_vtype (char *start, mword desc, char* from_start, char* from_end)
2215 /* The descriptors include info about the MonoObject header as well */
2216 start -= sizeof (MonoObject);
2218 switch (desc & 0x7) {
2219 case DESC_TYPE_RUN_LENGTH:
2220 OBJ_RUN_LEN_FOREACH_PTR (desc,start);
2221 OBJ_RUN_LEN_SIZE (skip_size, desc, start);
2222 g_assert (skip_size);
2223 return start + skip_size;
2224 case DESC_TYPE_SMALL_BITMAP:
2225 OBJ_BITMAP_FOREACH_PTR (desc,start);
2226 OBJ_BITMAP_SIZE (skip_size, desc, start);
2227 return start + skip_size;
2228 case DESC_TYPE_LARGE_BITMAP:
2229 case DESC_TYPE_COMPLEX:
2231 g_assert_not_reached ();
2234 // The other descriptors can't happen with vtypes
2235 g_assert_not_reached ();
2242 #define HANDLE_PTR(ptr,obj) do { \
2243 void *__old = *(ptr); \
2246 major_copy_or_mark_object ((ptr)); \
2248 DEBUG (9, if (__old != __copy) fprintf (gc_debug_file, "Overwrote field at %p with %p (was: %p)\n", (ptr), *(ptr), __old)); \
2249 if (G_UNLIKELY (ptr_in_nursery (__copy) && !ptr_in_nursery ((ptr)))) \
2250 add_to_global_remset ((ptr)); \
2255 major_scan_object (char *start)
2257 #include "sgen-scan-object.h"
2259 HEAVY_STAT (++stat_scan_object_called_major);
2265 * Scan objects in the gray stack until the stack is empty. This should be called
2266 * frequently after each object is copied, to achieve better locality and cache
2270 drain_gray_stack (void)
2274 if (current_collection_generation == GENERATION_NURSERY) {
2276 GRAY_OBJECT_DEQUEUE (obj);
2279 DEBUG (9, fprintf (gc_debug_file, "Precise gray object scan %p (%s)\n", obj, safe_name (obj)));
2284 GRAY_OBJECT_DEQUEUE (obj);
2287 DEBUG (9, fprintf (gc_debug_file, "Precise gray object scan %p (%s)\n", obj, safe_name (obj)));
2288 major_scan_object (obj);
2294 * Addresses from start to end are already sorted. This function finds
2295 * the object header for each address and pins the object. The
2296 * addresses must be inside the passed section. The (start of the)
2297 * address array is overwritten with the addresses of the actually
2298 * pinned objects. Return the number of pinned objects.
2301 pin_objects_from_addresses (GCMemSection *section, void **start, void **end, void *start_nursery, void *end_nursery)
2306 void *last_obj = NULL;
2307 size_t last_obj_size = 0;
2310 void **definitely_pinned = start;
2311 while (start < end) {
2313 /* the range check should be reduntant */
2314 if (addr != last && addr >= start_nursery && addr < end_nursery) {
2315 DEBUG (5, fprintf (gc_debug_file, "Considering pinning addr %p\n", addr));
2316 /* multiple pointers to the same object */
2317 if (addr >= last_obj && (char*)addr < (char*)last_obj + last_obj_size) {
2321 idx = ((char*)addr - (char*)section->data) / SCAN_START_SIZE;
2322 g_assert (idx < section->num_scan_start);
2323 search_start = (void*)section->scan_starts [idx];
2324 if (!search_start || search_start > addr) {
2327 search_start = section->scan_starts [idx];
2328 if (search_start && search_start <= addr)
2331 if (!search_start || search_start > addr)
2332 search_start = start_nursery;
2334 if (search_start < last_obj)
2335 search_start = (char*)last_obj + last_obj_size;
2336 /* now addr should be in an object a short distance from search_start
2337 * Note that search_start must point to zeroed mem or point to an object.
2340 if (!*(void**)search_start) {
2341 mword p = (mword)search_start;
2342 p += sizeof (gpointer);
2343 p += ALLOC_ALIGN - 1;
2344 p &= ~(ALLOC_ALIGN - 1);
2345 search_start = (void*)p;
2348 last_obj = search_start;
2349 last_obj_size = safe_object_get_size ((MonoObject*)search_start);
2350 last_obj_size += ALLOC_ALIGN - 1;
2351 last_obj_size &= ~(ALLOC_ALIGN - 1);
2352 DEBUG (8, fprintf (gc_debug_file, "Pinned try match %p (%s), size %zd\n", last_obj, safe_name (last_obj), last_obj_size));
2353 if (addr >= search_start && (char*)addr < (char*)last_obj + last_obj_size) {
2354 DEBUG (4, fprintf (gc_debug_file, "Pinned object %p, vtable %p (%s), count %d\n", search_start, *(void**)search_start, safe_name (search_start), count));
2355 binary_protocol_pin (search_start, (gpointer)LOAD_VTABLE (search_start), safe_object_get_size (search_start));
2356 pin_object (search_start);
2357 GRAY_OBJECT_ENQUEUE (search_start);
2359 pin_stats_register_object (search_start, last_obj_size);
2360 definitely_pinned [count] = search_start;
2364 /* skip to the next object */
2365 search_start = (void*)((char*)search_start + last_obj_size);
2366 } while (search_start <= addr);
2367 /* we either pinned the correct object or we ignored the addr because
2368 * it points to unused zeroed memory.
2374 //printf ("effective pinned: %d (at the end: %d)\n", count, (char*)end_nursery - (char*)last);
2379 pin_objects_in_section (GCMemSection *section)
2381 int start = section->pin_queue_start;
2382 int end = section->pin_queue_end;
2385 reduced_to = pin_objects_from_addresses (section, pin_queue + start, pin_queue + end,
2386 section->data, section->next_data);
2387 section->pin_queue_start = start;
2388 section->pin_queue_end = start + reduced_to;
2392 /* Sort the addresses in array in increasing order.
2393 * Done using a by-the book heap sort. Which has decent and stable performance, is pretty cache efficient.
2396 sort_addresses (void **array, int size)
2401 for (i = 1; i < size; ++i) {
2404 int parent = (child - 1) / 2;
2406 if (array [parent] >= array [child])
2409 tmp = array [parent];
2410 array [parent] = array [child];
2411 array [child] = tmp;
2417 for (i = size - 1; i > 0; --i) {
2420 array [i] = array [0];
2426 while (root * 2 + 1 <= end) {
2427 int child = root * 2 + 1;
2429 if (child < end && array [child] < array [child + 1])
2431 if (array [root] >= array [child])
2435 array [root] = array [child];
2436 array [child] = tmp;
2443 static G_GNUC_UNUSED void
2444 print_nursery_gaps (void* start_nursery, void *end_nursery)
2447 gpointer first = start_nursery;
2449 for (i = 0; i < next_pin_slot; ++i) {
2450 next = pin_queue [i];
2451 fprintf (gc_debug_file, "Nursery range: %p-%p, size: %td\n", first, next, (char*)next-(char*)first);
2455 fprintf (gc_debug_file, "Nursery range: %p-%p, size: %td\n", first, next, (char*)next-(char*)first);
2458 /* reduce the info in the pin queue, removing duplicate pointers and sorting them */
2460 optimize_pin_queue (int start_slot)
2462 void **start, **cur, **end;
2463 /* sort and uniq pin_queue: we just sort and we let the rest discard multiple values */
2464 /* it may be better to keep ranges of pinned memory instead of individually pinning objects */
2465 DEBUG (5, fprintf (gc_debug_file, "Sorting pin queue, size: %d\n", next_pin_slot));
2466 if ((next_pin_slot - start_slot) > 1)
2467 sort_addresses (pin_queue + start_slot, next_pin_slot - start_slot);
2468 start = cur = pin_queue + start_slot;
2469 end = pin_queue + next_pin_slot;
2472 while (*start == *cur && cur < end)
2476 next_pin_slot = start - pin_queue;
2477 DEBUG (5, fprintf (gc_debug_file, "Pin queue reduced to size: %d\n", next_pin_slot));
2478 //DEBUG (6, print_nursery_gaps (start_nursery, end_nursery));
2483 * Scan the memory between start and end and queue values which could be pointers
2484 * to the area between start_nursery and end_nursery for later consideration.
2485 * Typically used for thread stacks.
2488 conservatively_pin_objects_from (void **start, void **end, void *start_nursery, void *end_nursery, int pin_type)
2491 while (start < end) {
2492 if (*start >= start_nursery && *start < end_nursery) {
2494 * *start can point to the middle of an object
2495 * note: should we handle pointing at the end of an object?
2496 * pinning in C# code disallows pointing at the end of an object
2497 * but there is some small chance that an optimizing C compiler
2498 * may keep the only reference to an object by pointing
2499 * at the end of it. We ignore this small chance for now.
2500 * Pointers to the end of an object are indistinguishable
2501 * from pointers to the start of the next object in memory
2502 * so if we allow that we'd need to pin two objects...
2503 * We queue the pointer in an array, the
2504 * array will then be sorted and uniqued. This way
2505 * we can coalesce several pinning pointers and it should
2506 * be faster since we'd do a memory scan with increasing
2507 * addresses. Note: we can align the address to the allocation
2508 * alignment, so the unique process is more effective.
2510 mword addr = (mword)*start;
2511 addr &= ~(ALLOC_ALIGN - 1);
2512 if (addr >= (mword)start_nursery && addr < (mword)end_nursery)
2513 pin_stage_ptr ((void*)addr);
2515 pin_stats_register_address ((char*)addr, pin_type);
2516 DEBUG (6, if (count) fprintf (gc_debug_file, "Pinning address %p\n", (void*)addr));
2521 DEBUG (7, if (count) fprintf (gc_debug_file, "found %d potential pinned heap pointers\n", count));
2525 * Debugging function: find in the conservative roots where @obj is being pinned.
2527 static G_GNUC_UNUSED void
2528 find_pinning_reference (char *obj, size_t size)
2532 char *endobj = obj + size;
2533 for (i = 0; i < roots_hash_size [0]; ++i) {
2534 for (root = roots_hash [0][i]; root; root = root->next) {
2535 /* if desc is non-null it has precise info */
2536 if (!root->root_desc) {
2537 char ** start = (char**)root->start_root;
2538 while (start < (char**)root->end_root) {
2539 if (*start >= obj && *start < endobj) {
2540 DEBUG (0, fprintf (gc_debug_file, "Object %p referenced in pinned roots %p-%p (at %p in record %p)\n", obj, root->start_root, root->end_root, start, root));
2547 find_pinning_ref_from_thread (obj, size);
2551 * The first thing we do in a collection is to identify pinned objects.
2552 * This function considers all the areas of memory that need to be
2553 * conservatively scanned.
2556 pin_from_roots (void *start_nursery, void *end_nursery)
2560 DEBUG (2, fprintf (gc_debug_file, "Scanning pinned roots (%d bytes, %d/%d entries)\n", (int)roots_size, num_roots_entries [ROOT_TYPE_NORMAL], num_roots_entries [ROOT_TYPE_PINNED]));
2561 /* objects pinned from the API are inside these roots */
2562 for (i = 0; i < roots_hash_size [ROOT_TYPE_PINNED]; ++i) {
2563 for (root = roots_hash [ROOT_TYPE_PINNED][i]; root; root = root->next) {
2564 DEBUG (6, fprintf (gc_debug_file, "Pinned roots %p-%p\n", root->start_root, root->end_root));
2565 conservatively_pin_objects_from ((void**)root->start_root, (void**)root->end_root, start_nursery, end_nursery, PIN_TYPE_OTHER);
2568 /* now deal with the thread stacks
2569 * in the future we should be able to conservatively scan only:
2570 * *) the cpu registers
2571 * *) the unmanaged stack frames
2572 * *) the _last_ managed stack frame
2573 * *) pointers slots in managed frames
2575 scan_thread_data (start_nursery, end_nursery, FALSE);
2577 evacuate_pin_staging_area ();
2581 * The memory area from start_root to end_root contains pointers to objects.
2582 * Their position is precisely described by @desc (this means that the pointer
2583 * can be either NULL or the pointer to the start of an object).
2584 * This functions copies them to to_space updates them.
2587 precisely_scan_objects_from (CopyOrMarkObjectFunc copy_func, void** start_root, void** end_root, char* n_start, char *n_end, mword desc)
2589 switch (desc & ROOT_DESC_TYPE_MASK) {
2590 case ROOT_DESC_BITMAP:
2591 desc >>= ROOT_DESC_TYPE_SHIFT;
2593 if ((desc & 1) && *start_root) {
2594 copy_func (start_root);
2595 DEBUG (9, fprintf (gc_debug_file, "Overwrote root at %p with %p\n", start_root, *start_root));
2596 drain_gray_stack ();
2602 case ROOT_DESC_COMPLEX: {
2603 gsize *bitmap_data = complex_descriptors + (desc >> ROOT_DESC_TYPE_SHIFT);
2604 int bwords = (*bitmap_data) - 1;
2605 void **start_run = start_root;
2607 while (bwords-- > 0) {
2608 gsize bmap = *bitmap_data++;
2609 void **objptr = start_run;
2611 if ((bmap & 1) && *objptr) {
2613 DEBUG (9, fprintf (gc_debug_file, "Overwrote root at %p with %p\n", objptr, *objptr));
2614 drain_gray_stack ();
2619 start_run += GC_BITS_PER_WORD;
2623 case ROOT_DESC_USER: {
2624 MonoGCRootMarkFunc marker = user_descriptors [desc >> ROOT_DESC_TYPE_SHIFT];
2625 marker (start_root, copy_func);
2628 case ROOT_DESC_RUN_LEN:
2629 g_assert_not_reached ();
2631 g_assert_not_reached ();
2636 alloc_fragment (void)
2638 Fragment *frag = fragment_freelist;
2640 fragment_freelist = frag->next;
2644 frag = get_internal_mem (sizeof (Fragment), INTERNAL_MEM_FRAGMENT);
2649 /* size must be a power of 2 */
2651 get_os_memory_aligned (mword size, mword alignment, gboolean activate)
2653 /* Allocate twice the memory to be able to put the block on an aligned address */
2654 char *mem = get_os_memory (size + alignment, activate);
2659 aligned = (char*)((mword)(mem + (alignment - 1)) & ~(alignment - 1));
2660 g_assert (aligned >= mem && aligned + size <= mem + size + alignment && !((mword)aligned & (alignment - 1)));
2663 free_os_memory (mem, aligned - mem);
2664 if (aligned + size < mem + size + alignment)
2665 free_os_memory (aligned + size, (mem + size + alignment) - (aligned + size));
2671 * Allocate and setup the data structures needed to be able to allocate objects
2672 * in the nursery. The nursery is stored in nursery_section.
2675 alloc_nursery (void)
2677 GCMemSection *section;
2683 if (nursery_section)
2685 DEBUG (2, fprintf (gc_debug_file, "Allocating nursery size: %lu\n", (unsigned long)nursery_size));
2686 /* later we will alloc a larger area for the nursery but only activate
2687 * what we need. The rest will be used as expansion if we have too many pinned
2688 * objects in the existing nursery.
2690 /* FIXME: handle OOM */
2691 section = get_internal_mem (SIZEOF_GC_MEM_SECTION, INTERNAL_MEM_SECTION);
2693 g_assert (nursery_size == DEFAULT_NURSERY_SIZE);
2694 alloc_size = nursery_size;
2695 #ifdef ALIGN_NURSERY
2696 data = get_os_memory_aligned (alloc_size, alloc_size, TRUE);
2698 data = get_os_memory (alloc_size, TRUE);
2700 nursery_start = data;
2701 nursery_real_end = nursery_start + nursery_size;
2702 UPDATE_HEAP_BOUNDARIES (nursery_start, nursery_real_end);
2703 nursery_next = nursery_start;
2704 total_alloc += alloc_size;
2705 DEBUG (4, fprintf (gc_debug_file, "Expanding nursery size (%p-%p): %lu, total: %lu\n", data, data + alloc_size, (unsigned long)nursery_size, (unsigned long)total_alloc));
2706 section->data = section->next_data = data;
2707 section->size = alloc_size;
2708 section->end_data = nursery_real_end;
2709 scan_starts = (alloc_size + SCAN_START_SIZE - 1) / SCAN_START_SIZE;
2710 section->scan_starts = get_internal_mem (sizeof (char*) * scan_starts, INTERNAL_MEM_SCAN_STARTS);
2711 section->num_scan_start = scan_starts;
2712 section->block.role = MEMORY_ROLE_GEN0;
2713 section->block.next = NULL;
2715 nursery_section = section;
2717 /* Setup the single first large fragment */
2718 frag = alloc_fragment ();
2719 frag->fragment_start = nursery_start;
2720 frag->fragment_limit = nursery_start;
2721 frag->fragment_end = nursery_real_end;
2722 nursery_frag_real_end = nursery_real_end;
2723 /* FIXME: frag here is lost */
2727 scan_finalizer_entries (CopyOrMarkObjectFunc copy_func, FinalizeEntry *list) {
2730 for (fin = list; fin; fin = fin->next) {
2733 DEBUG (5, fprintf (gc_debug_file, "Scan of fin ready object: %p (%s)\n", fin->object, safe_name (fin->object)));
2734 copy_func (&fin->object);
2738 static mword fragment_total = 0;
2740 * We found a fragment of free memory in the nursery: memzero it and if
2741 * it is big enough, add it to the list of fragments that can be used for
2745 add_nursery_frag (size_t frag_size, char* frag_start, char* frag_end)
2748 DEBUG (4, fprintf (gc_debug_file, "Found empty fragment: %p-%p, size: %zd\n", frag_start, frag_end, frag_size));
2749 binary_protocol_empty (frag_start, frag_size);
2750 /* memsetting just the first chunk start is bound to provide better cache locality */
2751 if (nursery_clear_policy == CLEAR_AT_GC)
2752 memset (frag_start, 0, frag_size);
2753 /* Not worth dealing with smaller fragments: need to tune */
2754 if (frag_size >= FRAGMENT_MIN_SIZE) {
2755 fragment = alloc_fragment ();
2756 fragment->fragment_start = frag_start;
2757 fragment->fragment_limit = frag_start;
2758 fragment->fragment_end = frag_end;
2759 fragment->next = nursery_fragments;
2760 nursery_fragments = fragment;
2761 fragment_total += frag_size;
2763 /* Clear unused fragments, pinning depends on this */
2764 /*TODO place an int[] here instead of the memset if size justify it*/
2765 memset (frag_start, 0, frag_size);
2770 generation_name (int generation)
2772 switch (generation) {
2773 case GENERATION_NURSERY: return "nursery";
2774 case GENERATION_OLD: return "old";
2775 default: g_assert_not_reached ();
2779 static DisappearingLinkHashTable*
2780 get_dislink_hash_table (int generation)
2782 switch (generation) {
2783 case GENERATION_NURSERY: return &minor_disappearing_link_hash;
2784 case GENERATION_OLD: return &major_disappearing_link_hash;
2785 default: g_assert_not_reached ();
2789 static FinalizeEntryHashTable*
2790 get_finalize_entry_hash_table (int generation)
2792 switch (generation) {
2793 case GENERATION_NURSERY: return &minor_finalizable_hash;
2794 case GENERATION_OLD: return &major_finalizable_hash;
2795 default: g_assert_not_reached ();
2800 finish_gray_stack (char *start_addr, char *end_addr, int generation)
2805 int ephemeron_rounds = 0;
2806 CopyOrMarkObjectFunc copy_func = current_collection_generation == GENERATION_NURSERY ? copy_object : major_copy_or_mark_object;
2809 * We copied all the reachable objects. Now it's the time to copy
2810 * the objects that were not referenced by the roots, but by the copied objects.
2811 * we built a stack of objects pointed to by gray_start: they are
2812 * additional roots and we may add more items as we go.
2813 * We loop until gray_start == gray_objects which means no more objects have
2814 * been added. Note this is iterative: no recursion is involved.
2815 * We need to walk the LO list as well in search of marked big objects
2816 * (use a flag since this is needed only on major collections). We need to loop
2817 * here as well, so keep a counter of marked LO (increasing it in copy_object).
2818 * To achieve better cache locality and cache usage, we drain the gray stack
2819 * frequently, after each object is copied, and just finish the work here.
2821 drain_gray_stack ();
2823 DEBUG (2, fprintf (gc_debug_file, "%s generation done\n", generation_name (generation)));
2824 /* walk the finalization queue and move also the objects that need to be
2825 * finalized: use the finalized objects as new roots so the objects they depend
2826 * on are also not reclaimed. As with the roots above, only objects in the nursery
2827 * are marked/copied.
2828 * We need a loop here, since objects ready for finalizers may reference other objects
2829 * that are fin-ready. Speedup with a flag?
2833 * Walk the ephemeron tables marking all values with reachable keys. This must be completely done
2834 * before processing finalizable objects to avoid finalizing reachable values.
2836 * It must be done inside the finalizaters loop since objects must not be removed from CWT tables
2837 * while they are been finalized.
2839 int done_with_ephemerons = 0;
2841 done_with_ephemerons = mark_ephemerons_in_range (copy_func, start_addr, end_addr);
2842 drain_gray_stack ();
2844 } while (!done_with_ephemerons);
2846 fin_ready = num_ready_finalizers;
2847 finalize_in_range (copy_func, start_addr, end_addr, generation);
2848 if (generation == GENERATION_OLD)
2849 finalize_in_range (copy_func, nursery_start, nursery_real_end, GENERATION_NURSERY);
2851 /* drain the new stack that might have been created */
2852 DEBUG (6, fprintf (gc_debug_file, "Precise scan of gray area post fin\n"));
2853 drain_gray_stack ();
2854 } while (fin_ready != num_ready_finalizers);
2857 * Clear ephemeron pairs with unreachable keys.
2858 * We pass the copy func so we can figure out if an array was promoted or not.
2860 clear_unreachable_ephemerons (copy_func, start_addr, end_addr);
2863 DEBUG (2, fprintf (gc_debug_file, "Finalize queue handling scan for %s generation: %d usecs %d ephemeron roundss\n", generation_name (generation), TV_ELAPSED (atv, btv), ephemeron_rounds));
2866 * handle disappearing links
2867 * Note we do this after checking the finalization queue because if an object
2868 * survives (at least long enough to be finalized) we don't clear the link.
2869 * This also deals with a possible issue with the monitor reclamation: with the Boehm
2870 * GC a finalized object my lose the monitor because it is cleared before the finalizer is
2873 g_assert (gray_object_queue_is_empty ());
2875 null_link_in_range (copy_func, start_addr, end_addr, generation);
2876 if (generation == GENERATION_OLD)
2877 null_link_in_range (copy_func, start_addr, end_addr, GENERATION_NURSERY);
2878 if (gray_object_queue_is_empty ())
2880 drain_gray_stack ();
2883 g_assert (gray_object_queue_is_empty ());
2887 check_section_scan_starts (GCMemSection *section)
2890 for (i = 0; i < section->num_scan_start; ++i) {
2891 if (section->scan_starts [i]) {
2892 guint size = safe_object_get_size ((MonoObject*) section->scan_starts [i]);
2893 g_assert (size >= sizeof (MonoObject) && size <= MAX_SMALL_OBJ_SIZE);
2899 check_scan_starts (void)
2901 if (!do_scan_starts_check)
2903 check_section_scan_starts (nursery_section);
2904 major_check_scan_starts ();
2907 static int last_num_pinned = 0;
2910 build_nursery_fragments (int start_pin, int end_pin)
2912 char *frag_start, *frag_end;
2916 while (nursery_fragments) {
2917 Fragment *next = nursery_fragments->next;
2918 nursery_fragments->next = fragment_freelist;
2919 fragment_freelist = nursery_fragments;
2920 nursery_fragments = next;
2922 frag_start = nursery_start;
2924 /* clear scan starts */
2925 memset (nursery_section->scan_starts, 0, nursery_section->num_scan_start * sizeof (gpointer));
2926 for (i = start_pin; i < end_pin; ++i) {
2927 frag_end = pin_queue [i];
2928 /* remove the pin bit from pinned objects */
2929 unpin_object (frag_end);
2930 nursery_section->scan_starts [((char*)frag_end - (char*)nursery_section->data)/SCAN_START_SIZE] = frag_end;
2931 frag_size = frag_end - frag_start;
2933 add_nursery_frag (frag_size, frag_start, frag_end);
2934 frag_size = safe_object_get_size ((MonoObject*)pin_queue [i]);
2935 frag_size += ALLOC_ALIGN - 1;
2936 frag_size &= ~(ALLOC_ALIGN - 1);
2937 frag_start = (char*)pin_queue [i] + frag_size;
2939 nursery_last_pinned_end = frag_start;
2940 frag_end = nursery_real_end;
2941 frag_size = frag_end - frag_start;
2943 add_nursery_frag (frag_size, frag_start, frag_end);
2944 if (!nursery_fragments) {
2945 DEBUG (1, fprintf (gc_debug_file, "Nursery fully pinned (%d)\n", end_pin - start_pin));
2946 for (i = start_pin; i < end_pin; ++i) {
2947 DEBUG (3, fprintf (gc_debug_file, "Bastard pinning obj %p (%s), size: %d\n", pin_queue [i], safe_name (pin_queue [i]), safe_object_get_size (pin_queue [i])));
2952 nursery_next = nursery_frag_real_end = NULL;
2954 /* Clear TLABs for all threads */
2959 scan_from_registered_roots (CopyOrMarkObjectFunc copy_func, char *addr_start, char *addr_end, int root_type)
2963 for (i = 0; i < roots_hash_size [root_type]; ++i) {
2964 for (root = roots_hash [root_type][i]; root; root = root->next) {
2965 DEBUG (6, fprintf (gc_debug_file, "Precise root scan %p-%p (desc: %p)\n", root->start_root, root->end_root, (void*)root->root_desc));
2966 precisely_scan_objects_from (copy_func, (void**)root->start_root, (void**)root->end_root, addr_start, addr_end, root->root_desc);
2972 dump_occupied (char *start, char *end, char *section_start)
2974 fprintf (heap_dump_file, "<occupied offset=\"%td\" size=\"%td\"/>\n", start - section_start, end - start);
2978 dump_section (GCMemSection *section, const char *type)
2980 char *start = section->data;
2981 char *end = section->data + section->size;
2982 char *occ_start = NULL;
2984 char *old_start = NULL; /* just for debugging */
2986 fprintf (heap_dump_file, "<section type=\"%s\" size=\"%lu\">\n", type, (unsigned long)section->size);
2988 while (start < end) {
2992 if (!*(void**)start) {
2994 dump_occupied (occ_start, start, section->data);
2997 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
3000 g_assert (start < section->next_data);
3005 vt = (GCVTable*)LOAD_VTABLE (start);
3008 size = safe_object_get_size ((MonoObject*) start);
3009 size += ALLOC_ALIGN - 1;
3010 size &= ~(ALLOC_ALIGN - 1);
3013 fprintf (heap_dump_file, "<object offset=\"%d\" class=\"%s.%s\" size=\"%d\"/>\n",
3014 start - section->data,
3015 vt->klass->name_space, vt->klass->name,
3023 dump_occupied (occ_start, start, section->data);
3025 fprintf (heap_dump_file, "</section>\n");
3029 dump_object (MonoObject *obj, gboolean dump_location)
3031 static char class_name [1024];
3033 MonoClass *class = mono_object_class (obj);
3037 * Python's XML parser is too stupid to parse angle brackets
3038 * in strings, so we just ignore them;
3041 while (class->name [i] && j < sizeof (class_name) - 1) {
3042 if (!strchr ("<>\"", class->name [i]))
3043 class_name [j++] = class->name [i];
3046 g_assert (j < sizeof (class_name));
3049 fprintf (heap_dump_file, "<object class=\"%s.%s\" size=\"%d\"",
3050 class->name_space, class_name,
3051 safe_object_get_size (obj));
3052 if (dump_location) {
3053 const char *location;
3054 if (ptr_in_nursery (obj))
3055 location = "nursery";
3056 else if (safe_object_get_size (obj) <= MAX_SMALL_OBJ_SIZE)
3060 fprintf (heap_dump_file, " location=\"%s\"", location);
3062 fprintf (heap_dump_file, "/>\n");
3066 dump_heap (const char *type, int num, const char *reason)
3068 static char const *internal_mem_names [] = { "pin-queue", "fragment", "section", "scan-starts",
3069 "fin-table", "finalize-entry", "dislink-table",
3070 "dislink", "roots-table", "root-record", "statistics",
3071 "remset", "gray-queue", "store-remset", "marksweep-tables",
3072 "marksweep-block-info", "ephemeron-link" };
3078 fprintf (heap_dump_file, "<collection type=\"%s\" num=\"%d\"", type, num);
3080 fprintf (heap_dump_file, " reason=\"%s\"", reason);
3081 fprintf (heap_dump_file, ">\n");
3082 fprintf (heap_dump_file, "<other-mem-usage type=\"pinned-chunks\" size=\"%lld\"/>\n", pinned_chunk_bytes_alloced);
3083 fprintf (heap_dump_file, "<other-mem-usage type=\"large-internal\" size=\"%lld\"/>\n", large_internal_bytes_alloced);
3084 fprintf (heap_dump_file, "<other-mem-usage type=\"mempools\" size=\"%ld\"/>\n", mono_mempool_get_bytes_allocated ());
3085 for (i = 0; i < INTERNAL_MEM_MAX; ++i)
3086 fprintf (heap_dump_file, "<other-mem-usage type=\"%s\" size=\"%ld\"/>\n", internal_mem_names [i], small_internal_mem_bytes [i]);
3087 fprintf (heap_dump_file, "<pinned type=\"stack\" bytes=\"%zu\"/>\n", pinned_byte_counts [PIN_TYPE_STACK]);
3088 /* fprintf (heap_dump_file, "<pinned type=\"static-data\" bytes=\"%d\"/>\n", pinned_byte_counts [PIN_TYPE_STATIC_DATA]); */
3089 fprintf (heap_dump_file, "<pinned type=\"other\" bytes=\"%zu\"/>\n", pinned_byte_counts [PIN_TYPE_OTHER]);
3091 fprintf (heap_dump_file, "<pinned-objects>\n");
3092 for (list = pinned_objects; list; list = list->next)
3093 dump_object (list->obj, TRUE);
3094 fprintf (heap_dump_file, "</pinned-objects>\n");
3096 dump_section (nursery_section, "nursery");
3100 fprintf (heap_dump_file, "<los>\n");
3101 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
3102 dump_object ((MonoObject*)bigobj->data, FALSE);
3103 fprintf (heap_dump_file, "</los>\n");
3105 fprintf (heap_dump_file, "</collection>\n");
3111 static gboolean inited = FALSE;
3116 mono_counters_register ("Minor fragment clear", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_pre_collection_fragment_clear);
3117 mono_counters_register ("Minor pinning", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_pinning);
3118 mono_counters_register ("Minor scan remsets", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_scan_remsets);
3119 mono_counters_register ("Minor scan pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_scan_pinned);
3120 mono_counters_register ("Minor scan registered roots", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_scan_registered_roots);
3121 mono_counters_register ("Minor scan thread data", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_scan_thread_data);
3122 mono_counters_register ("Minor finish gray stack", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_finish_gray_stack);
3123 mono_counters_register ("Minor fragment creation", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_fragment_creation);
3125 mono_counters_register ("Major fragment clear", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_pre_collection_fragment_clear);
3126 mono_counters_register ("Major pinning", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_pinning);
3127 mono_counters_register ("Major scan pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_pinned);
3128 mono_counters_register ("Major scan registered roots", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_registered_roots);
3129 mono_counters_register ("Major scan thread data", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_thread_data);
3130 mono_counters_register ("Major scan alloc_pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_alloc_pinned);
3131 mono_counters_register ("Major scan finalized", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_finalized);
3132 mono_counters_register ("Major scan big objects", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_big_objects);
3133 mono_counters_register ("Major finish gray stack", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_finish_gray_stack);
3134 mono_counters_register ("Major free big objects", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_free_bigobjs);
3135 mono_counters_register ("Major LOS sweep", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_los_sweep);
3136 mono_counters_register ("Major sweep", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_sweep);
3137 mono_counters_register ("Major fragment creation", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_fragment_creation);
3139 #ifdef HEAVY_STATISTICS
3140 mono_counters_register ("WBarrier set field", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_field);
3141 mono_counters_register ("WBarrier set arrayref", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_arrayref);
3142 mono_counters_register ("WBarrier arrayref copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_arrayref_copy);
3143 mono_counters_register ("WBarrier generic store called", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_generic_store);
3144 mono_counters_register ("WBarrier generic store stored", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_generic_store_remset);
3145 mono_counters_register ("WBarrier set root", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_root);
3146 mono_counters_register ("WBarrier value copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_value_copy);
3147 mono_counters_register ("WBarrier object copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_object_copy);
3149 mono_counters_register ("# objects allocated", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_alloced);
3150 mono_counters_register ("bytes allocated", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_bytes_alloced);
3151 mono_counters_register ("# objects allocated degraded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_alloced_degraded);
3152 mono_counters_register ("bytes allocated degraded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_bytes_alloced_degraded);
3153 mono_counters_register ("bytes allocated in LOS", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_bytes_alloced_los);
3155 mono_counters_register ("# copy_object() called (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_called_nursery);
3156 mono_counters_register ("# objects copied (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_copied_nursery);
3157 mono_counters_register ("# copy_object() called (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_called_major);
3158 mono_counters_register ("# objects copied (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_copied_major);
3160 mono_counters_register ("# scan_object() called (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_scan_object_called_nursery);
3161 mono_counters_register ("# scan_object() called (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_scan_object_called_major);
3163 mono_counters_register ("# nursery copy_object() failed from space", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_from_space);
3164 mono_counters_register ("# nursery copy_object() failed forwarded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_forwarded);
3165 mono_counters_register ("# nursery copy_object() failed pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_pinned);
3167 mono_counters_register ("# wasted fragments used", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_wasted_fragments_used);
3168 mono_counters_register ("bytes in wasted fragments", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_wasted_fragments_bytes);
3170 mono_counters_register ("Store remsets", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_store_remsets);
3171 mono_counters_register ("Unique store remsets", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_store_remsets_unique);
3172 mono_counters_register ("Saved remsets 1", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_saved_remsets_1);
3173 mono_counters_register ("Saved remsets 2", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_saved_remsets_2);
3174 mono_counters_register ("Global remsets added", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_added);
3175 mono_counters_register ("Global remsets re-added", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_readded);
3176 mono_counters_register ("Global remsets processed", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_processed);
3177 mono_counters_register ("Global remsets discarded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_discarded);
3185 need_major_collection (void)
3187 mword los_alloced = los_memory_usage - MIN (last_los_memory_usage, los_memory_usage);
3188 return minor_collection_sections_alloced * MAJOR_SECTION_SIZE + los_alloced > minor_collection_allowance;
3192 * Collect objects in the nursery. Returns whether to trigger a major
3196 collect_nursery (size_t requested_size)
3198 size_t max_garbage_amount;
3199 char *orig_nursery_next;
3200 TV_DECLARE (all_atv);
3201 TV_DECLARE (all_btv);
3205 current_collection_generation = GENERATION_NURSERY;
3208 binary_protocol_collection (GENERATION_NURSERY);
3209 check_scan_starts ();
3212 orig_nursery_next = nursery_next;
3213 nursery_next = MAX (nursery_next, nursery_last_pinned_end);
3214 /* FIXME: optimize later to use the higher address where an object can be present */
3215 nursery_next = MAX (nursery_next, nursery_real_end);
3217 DEBUG (1, fprintf (gc_debug_file, "Start nursery collection %d %p-%p, size: %d\n", num_minor_gcs, nursery_start, nursery_next, (int)(nursery_next - nursery_start)));
3218 max_garbage_amount = nursery_next - nursery_start;
3219 g_assert (nursery_section->size >= max_garbage_amount);
3221 /* world must be stopped already */
3222 TV_GETTIME (all_atv);
3225 /* Pinning depends on this */
3226 clear_nursery_fragments (orig_nursery_next);
3229 time_minor_pre_collection_fragment_clear += TV_ELAPSED_MS (atv, btv);
3232 check_for_xdomain_refs ();
3234 nursery_section->next_data = nursery_next;
3236 major_start_nursery_collection ();
3238 gray_object_queue_init ();
3241 mono_stats.minor_gc_count ++;
3243 global_remset_cache_clear ();
3245 /* pin from pinned handles */
3247 pin_from_roots (nursery_start, nursery_next);
3248 /* identify pinned objects */
3249 optimize_pin_queue (0);
3250 next_pin_slot = pin_objects_from_addresses (nursery_section, pin_queue, pin_queue + next_pin_slot, nursery_start, nursery_next);
3251 nursery_section->pin_queue_start = 0;
3252 nursery_section->pin_queue_end = next_pin_slot;
3254 time_minor_pinning += TV_ELAPSED_MS (btv, atv);
3255 DEBUG (2, fprintf (gc_debug_file, "Finding pinned pointers: %d in %d usecs\n", next_pin_slot, TV_ELAPSED (btv, atv)));
3256 DEBUG (4, fprintf (gc_debug_file, "Start scan with %d pinned objects\n", next_pin_slot));
3258 if (consistency_check_at_minor_collection)
3259 check_consistency ();
3262 * walk all the roots and copy the young objects to the old generation,
3263 * starting from to_space
3266 scan_from_remsets (nursery_start, nursery_next);
3267 /* we don't have complete write barrier yet, so we scan all the old generation sections */
3269 time_minor_scan_remsets += TV_ELAPSED_MS (atv, btv);
3270 DEBUG (2, fprintf (gc_debug_file, "Old generation scan: %d usecs\n", TV_ELAPSED (atv, btv)));
3272 drain_gray_stack ();
3275 time_minor_scan_pinned += TV_ELAPSED_MS (btv, atv);
3276 /* registered roots, this includes static fields */
3277 scan_from_registered_roots (copy_object, nursery_start, nursery_next, ROOT_TYPE_NORMAL);
3278 scan_from_registered_roots (copy_object, nursery_start, nursery_next, ROOT_TYPE_WBARRIER);
3280 time_minor_scan_registered_roots += TV_ELAPSED_MS (atv, btv);
3282 scan_thread_data (nursery_start, nursery_next, TRUE);
3284 time_minor_scan_thread_data += TV_ELAPSED_MS (btv, atv);
3287 finish_gray_stack (nursery_start, nursery_next, GENERATION_NURSERY);
3289 time_minor_finish_gray_stack += TV_ELAPSED_MS (btv, atv);
3291 /* walk the pin_queue, build up the fragment list of free memory, unmark
3292 * pinned objects as we go, memzero() the empty fragments so they are ready for the
3295 build_nursery_fragments (0, next_pin_slot);
3297 time_minor_fragment_creation += TV_ELAPSED_MS (atv, btv);
3298 DEBUG (2, fprintf (gc_debug_file, "Fragment creation: %d usecs, %lu bytes available\n", TV_ELAPSED (atv, btv), (unsigned long)fragment_total));
3300 if (consistency_check_at_minor_collection)
3301 check_major_refs ();
3303 major_finish_nursery_collection ();
3305 TV_GETTIME (all_btv);
3306 mono_stats.minor_gc_time_usecs += TV_ELAPSED (all_atv, all_btv);
3309 dump_heap ("minor", num_minor_gcs - 1, NULL);
3311 /* prepare the pin queue for the next collection */
3312 last_num_pinned = next_pin_slot;
3314 if (fin_ready_list || critical_fin_list) {
3315 DEBUG (4, fprintf (gc_debug_file, "Finalizer-thread wakeup: ready %d\n", num_ready_finalizers));
3316 mono_gc_finalize_notify ();
3320 g_assert (gray_object_queue_is_empty ());
3322 check_scan_starts ();
3324 current_collection_generation = -1;
3326 return need_major_collection ();
3330 major_do_collection (const char *reason)
3332 LOSObject *bigobj, *prevbo;
3333 TV_DECLARE (all_atv);
3334 TV_DECLARE (all_btv);
3337 /* FIXME: only use these values for the precise scan
3338 * note that to_space pointers should be excluded anyway...
3340 char *heap_start = NULL;
3341 char *heap_end = (char*)-1;
3342 int old_num_major_sections = num_major_sections;
3343 int num_major_sections_saved, save_target, allowance_target;
3344 mword los_memory_saved, los_memory_alloced, old_los_memory_usage;
3347 * A domain could have been freed, resulting in
3348 * los_memory_usage being less than last_los_memory_usage.
3350 los_memory_alloced = los_memory_usage - MIN (last_los_memory_usage, los_memory_usage);
3351 old_los_memory_usage = los_memory_usage;
3353 //count_ref_nonref_objs ();
3354 //consistency_check ();
3357 binary_protocol_collection (GENERATION_OLD);
3358 check_scan_starts ();
3359 gray_object_queue_init ();
3362 DEBUG (1, fprintf (gc_debug_file, "Start major collection %d\n", num_major_gcs));
3364 mono_stats.major_gc_count ++;
3366 /* world must be stopped already */
3367 TV_GETTIME (all_atv);
3370 /* Pinning depends on this */
3371 clear_nursery_fragments (nursery_next);
3374 time_major_pre_collection_fragment_clear += TV_ELAPSED_MS (atv, btv);
3377 check_for_xdomain_refs ();
3379 nursery_section->next_data = nursery_real_end;
3380 /* we should also coalesce scanning from sections close to each other
3381 * and deal with pointers outside of the sections later.
3383 /* The remsets are not useful for a major collection */
3385 global_remset_cache_clear ();
3389 DEBUG (6, fprintf (gc_debug_file, "Collecting pinned addresses\n"));
3390 pin_from_roots ((void*)lowest_heap_address, (void*)highest_heap_address);
3391 optimize_pin_queue (0);
3394 * pin_queue now contains all candidate pointers, sorted and
3395 * uniqued. We must do two passes now to figure out which
3396 * objects are pinned.
3398 * The first is to find within the pin_queue the area for each
3399 * section. This requires that the pin_queue be sorted. We
3400 * also process the LOS objects and pinned chunks here.
3402 * The second, destructive, pass is to reduce the section
3403 * areas to pointers to the actually pinned objects.
3405 DEBUG (6, fprintf (gc_debug_file, "Pinning from sections\n"));
3406 /* first pass for the sections */
3407 find_section_pin_queue_start_end (nursery_section);
3408 major_find_pin_queue_start_ends ();
3409 /* identify possible pointers to the insize of large objects */
3410 DEBUG (6, fprintf (gc_debug_file, "Pinning from large objects\n"));
3411 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
3413 find_optimized_pin_queue_area (bigobj->data, (char*)bigobj->data + bigobj->size, &start, &end);
3415 pin_object (bigobj->data);
3416 /* FIXME: only enqueue if object has references */
3417 GRAY_OBJECT_ENQUEUE (bigobj->data);
3419 pin_stats_register_object ((char*) bigobj->data, safe_object_get_size ((MonoObject*) bigobj->data));
3420 DEBUG (6, fprintf (gc_debug_file, "Marked large object %p (%s) size: %lu from roots\n", bigobj->data, safe_name (bigobj->data), (unsigned long)bigobj->size));
3423 /* second pass for the sections */
3424 pin_objects_in_section (nursery_section);
3425 major_pin_objects ();
3428 time_major_pinning += TV_ELAPSED_MS (atv, btv);
3429 DEBUG (2, fprintf (gc_debug_file, "Finding pinned pointers: %d in %d usecs\n", next_pin_slot, TV_ELAPSED (atv, btv)));
3430 DEBUG (4, fprintf (gc_debug_file, "Start scan with %d pinned objects\n", next_pin_slot));
3432 major_init_to_space ();
3434 drain_gray_stack ();
3437 time_major_scan_pinned += TV_ELAPSED_MS (btv, atv);
3439 /* registered roots, this includes static fields */
3440 scan_from_registered_roots (major_copy_or_mark_object, heap_start, heap_end, ROOT_TYPE_NORMAL);
3441 scan_from_registered_roots (major_copy_or_mark_object, heap_start, heap_end, ROOT_TYPE_WBARRIER);
3443 time_major_scan_registered_roots += TV_ELAPSED_MS (atv, btv);
3446 /* FIXME: This is the wrong place for this, because it does
3448 scan_thread_data (heap_start, heap_end, TRUE);
3450 time_major_scan_thread_data += TV_ELAPSED_MS (btv, atv);
3453 time_major_scan_alloc_pinned += TV_ELAPSED_MS (atv, btv);
3455 /* scan the list of objects ready for finalization */
3456 scan_finalizer_entries (major_copy_or_mark_object, fin_ready_list);
3457 scan_finalizer_entries (major_copy_or_mark_object, critical_fin_list);
3459 time_major_scan_finalized += TV_ELAPSED_MS (btv, atv);
3460 DEBUG (2, fprintf (gc_debug_file, "Root scan: %d usecs\n", TV_ELAPSED (btv, atv)));
3463 time_major_scan_big_objects += TV_ELAPSED_MS (atv, btv);
3465 /* all the objects in the heap */
3466 finish_gray_stack (heap_start, heap_end, GENERATION_OLD);
3468 time_major_finish_gray_stack += TV_ELAPSED_MS (btv, atv);
3470 /* sweep the big objects list */
3472 for (bigobj = los_object_list; bigobj;) {
3473 if (object_is_pinned (bigobj->data)) {
3474 unpin_object (bigobj->data);
3477 /* not referenced anywhere, so we can free it */
3479 prevbo->next = bigobj->next;
3481 los_object_list = bigobj->next;
3483 bigobj = bigobj->next;
3484 free_large_object (to_free);
3488 bigobj = bigobj->next;
3492 time_major_free_bigobjs += TV_ELAPSED_MS (atv, btv);
3497 time_major_los_sweep += TV_ELAPSED_MS (btv, atv);
3502 time_major_sweep += TV_ELAPSED_MS (atv, btv);
3504 /* walk the pin_queue, build up the fragment list of free memory, unmark
3505 * pinned objects as we go, memzero() the empty fragments so they are ready for the
3508 build_nursery_fragments (nursery_section->pin_queue_start, nursery_section->pin_queue_end);
3511 time_major_fragment_creation += TV_ELAPSED_MS (btv, atv);
3513 TV_GETTIME (all_btv);
3514 mono_stats.major_gc_time_usecs += TV_ELAPSED (all_atv, all_btv);
3517 dump_heap ("major", num_major_gcs - 1, reason);
3519 /* prepare the pin queue for the next collection */
3521 if (fin_ready_list || critical_fin_list) {
3522 DEBUG (4, fprintf (gc_debug_file, "Finalizer-thread wakeup: ready %d\n", num_ready_finalizers));
3523 mono_gc_finalize_notify ();
3527 g_assert (gray_object_queue_is_empty ());
3529 num_major_sections_saved = MAX (old_num_major_sections - num_major_sections, 0);
3530 los_memory_saved = MAX (old_los_memory_usage - los_memory_usage, 1);
3532 save_target = ((num_major_sections * MAJOR_SECTION_SIZE) + los_memory_saved) / 2;
3534 * We aim to allow the allocation of as many sections as is
3535 * necessary to reclaim save_target sections in the next
3536 * collection. We assume the collection pattern won't change.
3537 * In the last cycle, we had num_major_sections_saved for
3538 * minor_collection_sections_alloced. Assuming things won't
3539 * change, this must be the same ratio as save_target for
3540 * allowance_target, i.e.
3542 * num_major_sections_saved save_target
3543 * --------------------------------- == ----------------
3544 * minor_collection_sections_alloced allowance_target
3548 allowance_target = (mword)((double)save_target * (double)(minor_collection_sections_alloced * MAJOR_SECTION_SIZE + los_memory_alloced) / (double)(num_major_sections_saved * MAJOR_SECTION_SIZE + los_memory_saved));
3550 minor_collection_allowance = MAX (MIN (allowance_target, num_major_sections * MAJOR_SECTION_SIZE + los_memory_usage), MIN_MINOR_COLLECTION_ALLOWANCE);
3552 minor_collection_sections_alloced = 0;
3553 last_los_memory_usage = los_memory_usage;
3555 check_scan_starts ();
3557 //consistency_check ();
3561 major_collection (const char *reason)
3563 if (g_getenv ("MONO_GC_NO_MAJOR")) {
3564 collect_nursery (0);
3568 current_collection_generation = GENERATION_OLD;
3569 major_do_collection (reason);
3570 current_collection_generation = -1;
3574 * When deciding if it's better to collect or to expand, keep track
3575 * of how much garbage was reclaimed with the last collection: if it's too
3577 * This is called when we could not allocate a small object.
3579 static void __attribute__((noinline))
3580 minor_collect_or_expand_inner (size_t size)
3582 int do_minor_collection = 1;
3584 if (!nursery_section) {
3588 if (do_minor_collection) {
3590 if (collect_nursery (size))
3591 major_collection ("minor overflow");
3592 DEBUG (2, fprintf (gc_debug_file, "Heap size: %lu, LOS size: %lu\n", (unsigned long)total_alloc, (unsigned long)los_memory_usage));
3594 /* this also sets the proper pointers for the next allocation */
3595 if (!search_fragment_for_size (size)) {
3597 /* TypeBuilder and MonoMethod are killing mcs with fragmentation */
3598 DEBUG (1, fprintf (gc_debug_file, "nursery collection didn't find enough room for %zd alloc (%d pinned)\n", size, last_num_pinned));
3599 for (i = 0; i < last_num_pinned; ++i) {
3600 DEBUG (3, fprintf (gc_debug_file, "Bastard pinning obj %p (%s), size: %d\n", pin_queue [i], safe_name (pin_queue [i]), safe_object_get_size (pin_queue [i])));
3605 //report_internal_mem_usage ();
3609 * ######################################################################
3610 * ######## Memory allocation from the OS
3611 * ######################################################################
3612 * This section of code deals with getting memory from the OS and
3613 * allocating memory for GC-internal data structures.
3614 * Internal memory can be handled with a freelist for small objects.
3618 * Allocate a big chunk of memory from the OS (usually 64KB to several megabytes).
3619 * This must not require any lock.
3622 get_os_memory (size_t size, int activate)
3625 unsigned long prot_flags = activate? MONO_MMAP_READ|MONO_MMAP_WRITE: MONO_MMAP_NONE;
3627 prot_flags |= MONO_MMAP_PRIVATE | MONO_MMAP_ANON;
3628 size += pagesize - 1;
3629 size &= ~(pagesize - 1);
3630 ptr = mono_valloc (0, size, prot_flags);
3635 * Free the memory returned by get_os_memory (), returning it to the OS.
3638 free_os_memory (void *addr, size_t size)
3640 mono_vfree (addr, size);
3647 report_pinned_chunk (PinnedChunk *chunk, int seq) {
3649 int i, free_pages, num_free, free_mem;
3651 for (i = 0; i < chunk->num_pages; ++i) {
3652 if (!chunk->page_sizes [i])
3655 printf ("Pinned chunk %d at %p, size: %d, pages: %d, free: %d\n", seq, chunk, chunk->num_pages * FREELIST_PAGESIZE, chunk->num_pages, free_pages);
3656 free_mem = FREELIST_PAGESIZE * free_pages;
3657 for (i = 0; i < FREELIST_NUM_SLOTS; ++i) {
3658 if (!chunk->free_list [i])
3661 p = chunk->free_list [i];
3666 printf ("\tfree list of size %d, %d items\n", freelist_sizes [i], num_free);
3667 free_mem += freelist_sizes [i] * num_free;
3669 printf ("\tfree memory in chunk: %d\n", free_mem);
3675 static G_GNUC_UNUSED void
3676 report_internal_mem_usage (void) {
3679 printf ("Internal memory usage:\n");
3681 for (chunk = internal_chunk_list; chunk; chunk = chunk->block.next) {
3682 report_pinned_chunk (chunk, i++);
3684 printf ("Pinned memory usage:\n");
3685 major_report_pinned_memory_usage ();
3689 * Find the slot number in the freelist for memory chunks that
3690 * can contain @size objects.
3693 slot_for_size (size_t size)
3696 /* do a binary search or lookup table later. */
3697 for (slot = 0; slot < FREELIST_NUM_SLOTS; ++slot) {
3698 if (freelist_sizes [slot] >= size)
3701 g_assert_not_reached ();
3706 * Build a free list for @size memory chunks from the memory area between
3707 * start_page and end_page.
3710 build_freelist (PinnedChunk *chunk, int slot, int size, char *start_page, char *end_page)
3714 /*g_print ("building freelist for slot %d, size %d in %p\n", slot, size, chunk);*/
3715 p = (void**)start_page;
3716 end = (void**)(end_page - size);
3717 g_assert (!chunk->free_list [slot]);
3718 chunk->free_list [slot] = p;
3719 while ((char*)p + size <= (char*)end) {
3721 *p = (void*)((char*)p + size);
3725 /*g_print ("%d items created, max: %d\n", count, (end_page - start_page) / size);*/
3729 alloc_pinned_chunk (void)
3733 int size = PINNED_CHUNK_SIZE;
3735 chunk = get_os_memory_aligned (size, size, TRUE);
3736 chunk->block.role = MEMORY_ROLE_PINNED;
3738 UPDATE_HEAP_BOUNDARIES (chunk, ((char*)chunk + size));
3739 total_alloc += size;
3740 pinned_chunk_bytes_alloced += size;
3742 /* setup the bookeeping fields */
3743 chunk->num_pages = size / FREELIST_PAGESIZE;
3744 offset = G_STRUCT_OFFSET (PinnedChunk, data);
3745 chunk->page_sizes = (void*)((char*)chunk + offset);
3746 offset += sizeof (int) * chunk->num_pages;
3747 offset += ALLOC_ALIGN - 1;
3748 offset &= ~(ALLOC_ALIGN - 1);
3749 chunk->free_list = (void*)((char*)chunk + offset);
3750 offset += sizeof (void*) * FREELIST_NUM_SLOTS;
3751 offset += ALLOC_ALIGN - 1;
3752 offset &= ~(ALLOC_ALIGN - 1);
3753 chunk->start_data = (void*)((char*)chunk + offset);
3755 /* allocate the first page to the freelist */
3756 chunk->page_sizes [0] = PINNED_FIRST_SLOT_SIZE;
3757 build_freelist (chunk, slot_for_size (PINNED_FIRST_SLOT_SIZE), PINNED_FIRST_SLOT_SIZE, chunk->start_data, ((char*)chunk + FREELIST_PAGESIZE));
3758 DEBUG (4, fprintf (gc_debug_file, "Allocated pinned chunk %p, size: %d\n", chunk, size));
3762 /* assumes freelist for slot is empty, so try to alloc a new page */
3764 get_chunk_freelist (PinnedChunk *chunk, int slot)
3768 p = chunk->free_list [slot];
3770 chunk->free_list [slot] = *p;
3773 for (i = 0; i < chunk->num_pages; ++i) {
3775 if (chunk->page_sizes [i])
3777 size = freelist_sizes [slot];
3778 chunk->page_sizes [i] = size;
3779 build_freelist (chunk, slot, size, (char*)chunk + FREELIST_PAGESIZE * i, (char*)chunk + FREELIST_PAGESIZE * (i + 1));
3783 p = chunk->free_list [slot];
3785 chunk->free_list [slot] = *p;
3791 /* used for the GC-internal data structures */
3793 get_internal_mem (size_t size, int type)
3797 PinnedChunk *pchunk;
3799 if (size > freelist_sizes [FREELIST_NUM_SLOTS - 1]) {
3800 LargeInternalMemHeader *mh;
3802 size += sizeof (LargeInternalMemHeader);
3803 mh = get_os_memory (size, TRUE);
3804 mh->magic = LARGE_INTERNAL_MEM_HEADER_MAGIC;
3807 large_internal_bytes_alloced += size;
3812 slot = slot_for_size (size);
3813 g_assert (size <= freelist_sizes [slot]);
3815 small_internal_mem_bytes [type] += freelist_sizes [slot];
3817 for (pchunk = internal_chunk_list; pchunk; pchunk = pchunk->block.next) {
3818 void **p = pchunk->free_list [slot];
3820 pchunk->free_list [slot] = *p;
3821 memset (p, 0, size);
3825 for (pchunk = internal_chunk_list; pchunk; pchunk = pchunk->block.next) {
3826 res = get_chunk_freelist (pchunk, slot);
3828 memset (res, 0, size);
3832 pchunk = alloc_pinned_chunk ();
3833 /* FIXME: handle OOM */
3834 pchunk->block.next = internal_chunk_list;
3835 internal_chunk_list = pchunk;
3836 res = get_chunk_freelist (pchunk, slot);
3837 memset (res, 0, size);
3842 free_internal_mem (void *addr, int type)
3844 PinnedChunk *pchunk;
3845 LargeInternalMemHeader *mh;
3848 for (pchunk = internal_chunk_list; pchunk; pchunk = pchunk->block.next) {
3849 /*printf ("trying to free %p in %p (pages: %d)\n", addr, pchunk, pchunk->num_pages);*/
3850 if (addr >= (void*)pchunk && (char*)addr < (char*)pchunk + pchunk->num_pages * FREELIST_PAGESIZE) {
3851 int offset = (char*)addr - (char*)pchunk;
3852 int page = offset / FREELIST_PAGESIZE;
3853 int slot = slot_for_size (pchunk->page_sizes [page]);
3855 *p = pchunk->free_list [slot];
3856 pchunk->free_list [slot] = p;
3858 small_internal_mem_bytes [type] -= freelist_sizes [slot];
3863 mh = (LargeInternalMemHeader*)((char*)addr - G_STRUCT_OFFSET (LargeInternalMemHeader, data));
3864 g_assert (mh->magic == LARGE_INTERNAL_MEM_HEADER_MAGIC);
3865 large_internal_bytes_alloced -= mh->size;
3866 free_os_memory (mh, mh->size);
3870 * ######################################################################
3871 * ######## Object allocation
3872 * ######################################################################
3873 * This section of code deals with allocating memory for objects.
3874 * There are several ways:
3875 * *) allocate large objects
3876 * *) allocate normal objects
3877 * *) fast lock-free allocation
3878 * *) allocation of pinned objects
3882 setup_fragment (Fragment *frag, Fragment *prev, size_t size)
3884 /* remove from the list */
3886 prev->next = frag->next;
3888 nursery_fragments = frag->next;
3889 nursery_next = frag->fragment_start;
3890 nursery_frag_real_end = frag->fragment_end;
3892 DEBUG (4, fprintf (gc_debug_file, "Using nursery fragment %p-%p, size: %td (req: %zd)\n", nursery_next, nursery_frag_real_end, nursery_frag_real_end - nursery_next, size));
3893 frag->next = fragment_freelist;
3894 fragment_freelist = frag;
3897 /* check if we have a suitable fragment in nursery_fragments to be able to allocate
3898 * an object of size @size
3899 * Return FALSE if not found (which means we need a collection)
3902 search_fragment_for_size (size_t size)
3904 Fragment *frag, *prev;
3905 DEBUG (4, fprintf (gc_debug_file, "Searching nursery fragment %p, size: %zd\n", nursery_frag_real_end, size));
3907 if (nursery_frag_real_end > nursery_next && nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
3908 /* Clear the remaining space, pinning depends on this */
3909 memset (nursery_next, 0, nursery_frag_real_end - nursery_next);
3912 for (frag = nursery_fragments; frag; frag = frag->next) {
3913 if (size <= (frag->fragment_end - frag->fragment_start)) {
3914 setup_fragment (frag, prev, size);
3923 * Same as search_fragment_for_size but if search for @desired_size fails, try to satisfy @minimum_size.
3924 * This improves nursery usage.
3927 search_fragment_for_size_range (size_t desired_size, size_t minimum_size)
3929 Fragment *frag, *prev, *min_prev;
3930 DEBUG (4, fprintf (gc_debug_file, "Searching nursery fragment %p, desired size: %zd minimum size %zd\n", nursery_frag_real_end, desired_size, minimum_size));
3932 if (nursery_frag_real_end > nursery_next && nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
3933 /* Clear the remaining space, pinning depends on this */
3934 memset (nursery_next, 0, nursery_frag_real_end - nursery_next);
3936 min_prev = GINT_TO_POINTER (-1);
3939 for (frag = nursery_fragments; frag; frag = frag->next) {
3940 int frag_size = frag->fragment_end - frag->fragment_start;
3941 if (desired_size <= frag_size) {
3942 setup_fragment (frag, prev, desired_size);
3943 return desired_size;
3945 if (minimum_size <= frag_size)
3951 if (min_prev != GINT_TO_POINTER (-1)) {
3954 frag = min_prev->next;
3956 frag = nursery_fragments;
3958 frag_size = frag->fragment_end - frag->fragment_start;
3959 HEAVY_STAT (++stat_wasted_fragments_used);
3960 HEAVY_STAT (stat_wasted_fragments_bytes += frag_size);
3962 setup_fragment (frag, min_prev, minimum_size);
3970 alloc_degraded (MonoVTable *vtable, size_t size)
3972 if (need_major_collection ()) {
3974 major_collection ("degraded overflow");
3978 return major_alloc_degraded (vtable, size);
3982 * Provide a variant that takes just the vtable for small fixed-size objects.
3983 * The aligned size is already computed and stored in vt->gc_descr.
3984 * Note: every SCAN_START_SIZE or so we are given the chance to do some special
3985 * processing. We can keep track of where objects start, for example,
3986 * so when we scan the thread stacks for pinned objects, we can start
3987 * a search for the pinned object in SCAN_START_SIZE chunks.
3990 mono_gc_alloc_obj_nolock (MonoVTable *vtable, size_t size)
3992 /* FIXME: handle OOM */
3997 HEAVY_STAT (++stat_objects_alloced);
3998 if (size <= MAX_SMALL_OBJ_SIZE)
3999 HEAVY_STAT (stat_bytes_alloced += size);
4001 HEAVY_STAT (stat_bytes_alloced_los += size);
4003 size += ALLOC_ALIGN - 1;
4004 size &= ~(ALLOC_ALIGN - 1);
4006 g_assert (vtable->gc_descr);
4008 if (G_UNLIKELY (collect_before_allocs)) {
4009 if (nursery_section) {
4011 collect_nursery (0);
4013 if (!degraded_mode && !search_fragment_for_size (size)) {
4015 g_assert_not_reached ();
4021 * We must already have the lock here instead of after the
4022 * fast path because we might be interrupted in the fast path
4023 * (after confirming that new_next < TLAB_TEMP_END) by the GC,
4024 * and we'll end up allocating an object in a fragment which
4025 * no longer belongs to us.
4027 * The managed allocator does not do this, but it's treated
4028 * specially by the world-stopping code.
4031 if (size > MAX_SMALL_OBJ_SIZE) {
4032 p = alloc_large_inner (vtable, size);
4034 /* tlab_next and tlab_temp_end are TLS vars so accessing them might be expensive */
4036 p = (void**)TLAB_NEXT;
4037 /* FIXME: handle overflow */
4038 new_next = (char*)p + size;
4039 TLAB_NEXT = new_next;
4041 if (G_LIKELY (new_next < TLAB_TEMP_END)) {
4045 * FIXME: We might need a memory barrier here so the change to tlab_next is
4046 * visible before the vtable store.
4049 DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
4050 binary_protocol_alloc (p , vtable, size);
4051 g_assert (*p == NULL);
4054 g_assert (TLAB_NEXT == new_next);
4061 /* there are two cases: the object is too big or we run out of space in the TLAB */
4062 /* we also reach here when the thread does its first allocation after a minor
4063 * collection, since the tlab_ variables are initialized to NULL.
4064 * there can be another case (from ORP), if we cooperate with the runtime a bit:
4065 * objects that need finalizers can have the high bit set in their size
4066 * so the above check fails and we can readily add the object to the queue.
4067 * This avoids taking again the GC lock when registering, but this is moot when
4068 * doing thread-local allocation, so it may not be a good idea.
4070 g_assert (TLAB_NEXT == new_next);
4071 if (TLAB_NEXT >= TLAB_REAL_END) {
4073 * Run out of space in the TLAB. When this happens, some amount of space
4074 * remains in the TLAB, but not enough to satisfy the current allocation
4075 * request. Currently, we retire the TLAB in all cases, later we could
4076 * keep it if the remaining space is above a treshold, and satisfy the
4077 * allocation directly from the nursery.
4080 /* when running in degraded mode, we continue allocing that way
4081 * for a while, to decrease the number of useless nursery collections.
4083 if (degraded_mode && degraded_mode < DEFAULT_NURSERY_SIZE) {
4084 p = alloc_degraded (vtable, size);
4085 binary_protocol_alloc_degraded (p, vtable, size);
4089 /*FIXME This codepath is current deadcode since tlab_size > MAX_SMALL_OBJ_SIZE*/
4090 if (size > tlab_size) {
4091 /* Allocate directly from the nursery */
4092 if (nursery_next + size >= nursery_frag_real_end) {
4093 if (!search_fragment_for_size (size)) {
4094 minor_collect_or_expand_inner (size);
4095 if (degraded_mode) {
4096 p = alloc_degraded (vtable, size);
4097 binary_protocol_alloc_degraded (p, vtable, size);
4103 p = (void*)nursery_next;
4104 nursery_next += size;
4105 if (nursery_next > nursery_frag_real_end) {
4110 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
4111 memset (p, 0, size);
4113 int alloc_size = tlab_size;
4114 int available_in_nursery = nursery_frag_real_end - nursery_next;
4116 DEBUG (3, fprintf (gc_debug_file, "Retire TLAB: %p-%p [%ld]\n", TLAB_START, TLAB_REAL_END, (long)(TLAB_REAL_END - TLAB_NEXT - size)));
4118 if (alloc_size >= available_in_nursery) {
4119 if (available_in_nursery > MAX_NURSERY_TLAB_WASTE && available_in_nursery > size) {
4120 alloc_size = available_in_nursery;
4122 alloc_size = search_fragment_for_size_range (tlab_size, size);
4124 alloc_size = tlab_size;
4125 minor_collect_or_expand_inner (tlab_size);
4126 if (degraded_mode) {
4127 p = alloc_degraded (vtable, size);
4128 binary_protocol_alloc_degraded (p, vtable, size);
4135 /* Allocate a new TLAB from the current nursery fragment */
4136 TLAB_START = nursery_next;
4137 nursery_next += alloc_size;
4138 TLAB_NEXT = TLAB_START;
4139 TLAB_REAL_END = TLAB_START + alloc_size;
4140 TLAB_TEMP_END = TLAB_START + MIN (SCAN_START_SIZE, alloc_size);
4142 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
4143 memset (TLAB_START, 0, alloc_size);
4145 /* Allocate from the TLAB */
4146 p = (void*)TLAB_NEXT;
4148 g_assert (TLAB_NEXT <= TLAB_REAL_END);
4150 nursery_section->scan_starts [((char*)p - (char*)nursery_section->data)/SCAN_START_SIZE] = (char*)p;
4153 /* Reached tlab_temp_end */
4155 /* record the scan start so we can find pinned objects more easily */
4156 nursery_section->scan_starts [((char*)p - (char*)nursery_section->data)/SCAN_START_SIZE] = (char*)p;
4157 /* we just bump tlab_temp_end as well */
4158 TLAB_TEMP_END = MIN (TLAB_REAL_END, TLAB_NEXT + SCAN_START_SIZE);
4159 DEBUG (5, fprintf (gc_debug_file, "Expanding local alloc: %p-%p\n", TLAB_NEXT, TLAB_TEMP_END));
4163 DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
4164 binary_protocol_alloc (p, vtable, size);
4171 mono_gc_try_alloc_obj_nolock (MonoVTable *vtable, size_t size)
4177 size += ALLOC_ALIGN - 1;
4178 size &= ~(ALLOC_ALIGN - 1);
4180 g_assert (vtable->gc_descr);
4181 if (size <= MAX_SMALL_OBJ_SIZE) {
4182 /* tlab_next and tlab_temp_end are TLS vars so accessing them might be expensive */
4184 p = (void**)TLAB_NEXT;
4185 /* FIXME: handle overflow */
4186 new_next = (char*)p + size;
4187 TLAB_NEXT = new_next;
4189 if (G_LIKELY (new_next < TLAB_TEMP_END)) {
4193 * FIXME: We might need a memory barrier here so the change to tlab_next is
4194 * visible before the vtable store.
4197 HEAVY_STAT (++stat_objects_alloced);
4198 HEAVY_STAT (stat_bytes_alloced += size);
4200 DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
4201 binary_protocol_alloc (p, vtable, size);
4202 g_assert (*p == NULL);
4205 g_assert (TLAB_NEXT == new_next);
4214 mono_gc_alloc_obj (MonoVTable *vtable, size_t size)
4217 #ifndef DISABLE_CRITICAL_REGION
4219 ENTER_CRITICAL_REGION;
4220 res = mono_gc_try_alloc_obj_nolock (vtable, size);
4222 EXIT_CRITICAL_REGION;
4225 EXIT_CRITICAL_REGION;
4228 res = mono_gc_alloc_obj_nolock (vtable, size);
4234 mono_gc_alloc_vector (MonoVTable *vtable, size_t size, uintptr_t max_length)
4237 #ifndef DISABLE_CRITICAL_REGION
4239 ENTER_CRITICAL_REGION;
4240 arr = mono_gc_try_alloc_obj_nolock (vtable, size);
4242 arr->max_length = max_length;
4243 EXIT_CRITICAL_REGION;
4246 EXIT_CRITICAL_REGION;
4251 arr = mono_gc_alloc_obj_nolock (vtable, size);
4252 arr->max_length = max_length;
4260 mono_gc_alloc_array (MonoVTable *vtable, size_t size, uintptr_t max_length, uintptr_t bounds_size)
4263 MonoArrayBounds *bounds;
4267 arr = mono_gc_alloc_obj_nolock (vtable, size);
4268 arr->max_length = max_length;
4270 bounds = (MonoArrayBounds*)((char*)arr + size - bounds_size);
4271 arr->bounds = bounds;
4279 mono_gc_alloc_string (MonoVTable *vtable, size_t size, gint32 len)
4282 #ifndef DISABLE_CRITICAL_REGION
4284 ENTER_CRITICAL_REGION;
4285 str = mono_gc_try_alloc_obj_nolock (vtable, size);
4288 EXIT_CRITICAL_REGION;
4291 EXIT_CRITICAL_REGION;
4296 str = mono_gc_alloc_obj_nolock (vtable, size);
4305 * To be used for interned strings and possibly MonoThread, reflection handles.
4306 * We may want to explicitly free these objects.
4309 mono_gc_alloc_pinned_obj (MonoVTable *vtable, size_t size)
4311 /* FIXME: handle OOM */
4313 size += ALLOC_ALIGN - 1;
4314 size &= ~(ALLOC_ALIGN - 1);
4316 if (size > MAX_SMALL_OBJ_SIZE) {
4317 /* large objects are always pinned anyway */
4318 p = alloc_large_inner (vtable, size);
4320 DEBUG (9, g_assert (vtable->klass->inited));
4321 p = major_alloc_small_pinned_obj (size, vtable->klass->has_references);
4323 DEBUG (6, fprintf (gc_debug_file, "Allocated pinned object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
4324 binary_protocol_alloc_pinned (p, vtable, size);
4331 * ######################################################################
4332 * ######## Finalization support
4333 * ######################################################################
4337 * this is valid for the nursery: if the object has been forwarded it means it's
4338 * still refrenced from a root. If it is pinned it's still alive as well.
4339 * Return TRUE if @obj is ready to be finalized.
4341 #define object_is_fin_ready(obj) (!object_is_pinned (obj) && !object_is_forwarded (obj))
4344 is_critical_finalizer (FinalizeEntry *entry)
4349 if (!mono_defaults.critical_finalizer_object)
4352 obj = entry->object;
4353 class = ((MonoVTable*)LOAD_VTABLE (obj))->klass;
4355 return mono_class_has_parent (class, mono_defaults.critical_finalizer_object);
4359 queue_finalization_entry (FinalizeEntry *entry) {
4360 if (is_critical_finalizer (entry)) {
4361 entry->next = critical_fin_list;
4362 critical_fin_list = entry;
4364 entry->next = fin_ready_list;
4365 fin_ready_list = entry;
4369 /* LOCKING: requires that the GC lock is held */
4371 rehash_fin_table (FinalizeEntryHashTable *hash_table)
4373 FinalizeEntry **finalizable_hash = hash_table->table;
4374 mword finalizable_hash_size = hash_table->size;
4377 FinalizeEntry **new_hash;
4378 FinalizeEntry *entry, *next;
4379 int new_size = g_spaced_primes_closest (hash_table->num_registered);
4381 new_hash = get_internal_mem (new_size * sizeof (FinalizeEntry*), INTERNAL_MEM_FIN_TABLE);
4382 for (i = 0; i < finalizable_hash_size; ++i) {
4383 for (entry = finalizable_hash [i]; entry; entry = next) {
4384 hash = mono_object_hash (entry->object) % new_size;
4386 entry->next = new_hash [hash];
4387 new_hash [hash] = entry;
4390 free_internal_mem (finalizable_hash, INTERNAL_MEM_FIN_TABLE);
4391 hash_table->table = new_hash;
4392 hash_table->size = new_size;
4395 /* LOCKING: requires that the GC lock is held */
4397 rehash_fin_table_if_necessary (FinalizeEntryHashTable *hash_table)
4399 if (hash_table->num_registered >= hash_table->size * 2)
4400 rehash_fin_table (hash_table);
4403 /* LOCKING: requires that the GC lock is held */
4405 finalize_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, int generation)
4407 FinalizeEntryHashTable *hash_table = get_finalize_entry_hash_table (generation);
4408 FinalizeEntry *entry, *prev;
4410 FinalizeEntry **finalizable_hash = hash_table->table;
4411 mword finalizable_hash_size = hash_table->size;
4415 for (i = 0; i < finalizable_hash_size; ++i) {
4417 for (entry = finalizable_hash [i]; entry;) {
4418 if ((char*)entry->object >= start && (char*)entry->object < end && !major_is_object_live (entry->object)) {
4419 gboolean is_fin_ready = object_is_fin_ready (entry->object);
4420 char *copy = entry->object;
4421 copy_func ((void**)©);
4424 FinalizeEntry *next;
4425 /* remove and put in fin_ready_list */
4427 prev->next = entry->next;
4429 finalizable_hash [i] = entry->next;
4431 num_ready_finalizers++;
4432 hash_table->num_registered--;
4433 queue_finalization_entry (entry);
4434 /* Make it survive */
4435 from = entry->object;
4436 entry->object = copy;
4437 DEBUG (5, fprintf (gc_debug_file, "Queueing object for finalization: %p (%s) (was at %p) (%d/%d)\n", entry->object, safe_name (entry->object), from, num_ready_finalizers, hash_table->num_registered));
4441 char *from = entry->object;
4442 if (hash_table == &minor_finalizable_hash && !ptr_in_nursery (copy)) {
4443 FinalizeEntry *next = entry->next;
4444 unsigned int major_hash;
4445 /* remove from the list */
4447 prev->next = entry->next;
4449 finalizable_hash [i] = entry->next;
4450 hash_table->num_registered--;
4452 entry->object = copy;
4454 /* insert it into the major hash */
4455 rehash_fin_table_if_necessary (&major_finalizable_hash);
4456 major_hash = mono_object_hash ((MonoObject*) copy) %
4457 major_finalizable_hash.size;
4458 entry->next = major_finalizable_hash.table [major_hash];
4459 major_finalizable_hash.table [major_hash] = entry;
4460 major_finalizable_hash.num_registered++;
4462 DEBUG (5, fprintf (gc_debug_file, "Promoting finalization of object %p (%s) (was at %p) to major table\n", copy, safe_name (copy), from));
4467 /* update pointer */
4468 DEBUG (5, fprintf (gc_debug_file, "Updating object for finalization: %p (%s) (was at %p)\n", entry->object, safe_name (entry->object), from));
4469 entry->object = copy;
4474 entry = entry->next;
4480 object_is_reachable (char *object, char *start, char *end)
4482 /*This happens for non nursery objects during minor collections. We just treat all objects as alive.*/
4483 if (object < start || object >= end)
4485 return !object_is_fin_ready (object) || major_is_object_live (object);
4488 /* LOCKING: requires that the GC lock is held */
4490 null_ephemerons_for_domain (MonoDomain *domain)
4492 EphemeronLinkNode *current = ephemeron_list, *prev = NULL;
4495 MonoObject *object = (MonoObject*)current->array;
4497 if (object && !object->vtable) {
4498 EphemeronLinkNode *tmp = current;
4501 prev->next = current->next;
4503 ephemeron_list = current->next;
4505 current = current->next;
4506 free_internal_mem (tmp, INTERNAL_MEM_EPHEMERON_LINK);
4509 current = current->next;
4514 /* LOCKING: requires that the GC lock is held */
4516 clear_unreachable_ephemerons (CopyOrMarkObjectFunc copy_func, char *start, char *end)
4518 int was_in_nursery, was_promoted;
4519 EphemeronLinkNode *current = ephemeron_list, *prev = NULL;
4521 Ephemeron *cur, *array_end;
4525 char *object = current->array;
4527 if (!object_is_reachable (object, start, end)) {
4528 EphemeronLinkNode *tmp = current;
4530 DEBUG (5, fprintf (gc_debug_file, "Dead Ephemeron array at %p\n", object));
4533 prev->next = current->next;
4535 ephemeron_list = current->next;
4537 current = current->next;
4538 free_internal_mem (tmp, INTERNAL_MEM_EPHEMERON_LINK);
4543 was_in_nursery = ptr_in_nursery (object);
4544 copy_func ((void**)&object);
4545 current->array = object;
4547 /*The array was promoted, add global remsets for key/values left behind in nursery.*/
4548 was_promoted = was_in_nursery && !ptr_in_nursery (object);
4550 DEBUG (5, fprintf (gc_debug_file, "Clearing unreachable entries for ephemeron array at %p\n", object));
4552 array = (MonoArray*)object;
4553 cur = mono_array_addr (array, Ephemeron, 0);
4554 array_end = cur + mono_array_length_fast (array);
4555 tombstone = (char*)((MonoVTable*)LOAD_VTABLE (object))->domain->ephemeron_tombstone;
4557 for (; cur < array_end; ++cur) {
4558 char *key = (char*)cur->key;
4560 if (!key || key == tombstone)
4563 DEBUG (5, fprintf (gc_debug_file, "[%td] key %p (%s) value %p (%s)\n", cur - mono_array_addr (array, Ephemeron, 0),
4564 key, object_is_reachable (key, start, end) ? "reachable" : "unreachable",
4565 cur->value, cur->value && object_is_reachable (cur->value, start, end) ? "reachable" : "unreachable"));
4567 if (!object_is_reachable (key, start, end)) {
4568 cur->key = tombstone;
4574 if (ptr_in_nursery (key)) {/*key was not promoted*/
4575 DEBUG (5, fprintf (gc_debug_file, "\tAdded remset to key %p\n", key));
4576 add_to_global_remset (&cur->key);
4578 if (ptr_in_nursery (cur->value)) {/*value was not promoted*/
4579 DEBUG (5, fprintf (gc_debug_file, "\tAdded remset to value %p\n", cur->value));
4580 add_to_global_remset (&cur->value);
4585 current = current->next;
4589 /* LOCKING: requires that the GC lock is held */
4591 mark_ephemerons_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end)
4593 int nothing_marked = 1;
4594 EphemeronLinkNode *current = ephemeron_list;
4596 Ephemeron *cur, *array_end;
4599 for (current = ephemeron_list; current; current = current->next) {
4600 char *object = current->array;
4601 DEBUG (5, fprintf (gc_debug_file, "Ephemeron array at %p\n", object));
4603 /*We ignore arrays in old gen during minor collections since all objects are promoted by the remset machinery.*/
4604 if (object < start || object >= end)
4607 /*It has to be alive*/
4608 if (!object_is_reachable (object, start, end)) {
4609 DEBUG (5, fprintf (gc_debug_file, "\tnot reachable\n"));
4613 copy_func ((void**)&object);
4615 array = (MonoArray*)object;
4616 cur = mono_array_addr (array, Ephemeron, 0);
4617 array_end = cur + mono_array_length_fast (array);
4618 tombstone = (char*)((MonoVTable*)LOAD_VTABLE (object))->domain->ephemeron_tombstone;
4620 for (; cur < array_end; ++cur) {
4621 char *key = cur->key;
4623 if (!key || key == tombstone)
4626 DEBUG (5, fprintf (gc_debug_file, "[%td] key %p (%s) value %p (%s)\n", cur - mono_array_addr (array, Ephemeron, 0),
4627 key, object_is_reachable (key, start, end) ? "reachable" : "unreachable",
4628 cur->value, cur->value && object_is_reachable (cur->value, start, end) ? "reachable" : "unreachable"));
4630 if (object_is_reachable (key, start, end)) {
4631 char *value = cur->value;
4633 copy_func ((void**)&cur->key);
4635 if (!object_is_reachable (value, start, end))
4637 copy_func ((void**)&cur->value);
4643 DEBUG (5, fprintf (gc_debug_file, "Ephemeron run finished. Is it done %d\n", nothing_marked));
4644 return nothing_marked;
4647 /* LOCKING: requires that the GC lock is held */
4649 null_link_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, int generation)
4651 DisappearingLinkHashTable *hash = get_dislink_hash_table (generation);
4652 DisappearingLink **disappearing_link_hash = hash->table;
4653 int disappearing_link_hash_size = hash->size;
4654 DisappearingLink *entry, *prev;
4656 if (!hash->num_links)
4658 for (i = 0; i < disappearing_link_hash_size; ++i) {
4660 for (entry = disappearing_link_hash [i]; entry;) {
4661 char *object = DISLINK_OBJECT (entry);
4662 if (object >= start && object < end && !major_is_object_live (object)) {
4663 gboolean track = DISLINK_TRACK (entry);
4664 if (!track && object_is_fin_ready (object)) {
4665 void **p = entry->link;
4666 DisappearingLink *old;
4668 /* remove from list */
4670 prev->next = entry->next;
4672 disappearing_link_hash [i] = entry->next;
4673 DEBUG (5, fprintf (gc_debug_file, "Dislink nullified at %p to GCed object %p\n", p, object));
4675 free_internal_mem (entry, INTERNAL_MEM_DISLINK);
4680 char *copy = object;
4681 copy_func ((void**)©);
4683 /* Update pointer if it's moved. If the object
4684 * has been moved out of the nursery, we need to
4685 * remove the link from the minor hash table to
4688 * FIXME: what if an object is moved earlier?
4691 if (hash == &minor_disappearing_link_hash && !ptr_in_nursery (copy)) {
4692 void **link = entry->link;
4693 DisappearingLink *old;
4694 /* remove from list */
4696 prev->next = entry->next;
4698 disappearing_link_hash [i] = entry->next;
4700 free_internal_mem (entry, INTERNAL_MEM_DISLINK);
4704 add_or_remove_disappearing_link ((MonoObject*)copy, link,
4705 track, GENERATION_OLD);
4707 DEBUG (5, fprintf (gc_debug_file, "Upgraded dislink at %p to major because object %p moved to %p\n", link, object, copy));
4711 /* We set the track resurrection bit to
4712 * FALSE if the object is to be finalized
4713 * so that the object can be collected in
4714 * the next cycle (i.e. after it was
4717 *entry->link = HIDE_POINTER (copy,
4718 object_is_fin_ready (object) ? FALSE : track);
4719 DEBUG (5, fprintf (gc_debug_file, "Updated dislink at %p to %p\n", entry->link, DISLINK_OBJECT (entry)));
4724 entry = entry->next;
4729 /* LOCKING: requires that the GC lock is held */
4731 null_links_for_domain (MonoDomain *domain, int generation)
4733 DisappearingLinkHashTable *hash = get_dislink_hash_table (generation);
4734 DisappearingLink **disappearing_link_hash = hash->table;
4735 int disappearing_link_hash_size = hash->size;
4736 DisappearingLink *entry, *prev;
4738 for (i = 0; i < disappearing_link_hash_size; ++i) {
4740 for (entry = disappearing_link_hash [i]; entry; ) {
4741 char *object = DISLINK_OBJECT (entry);
4742 if (object && !((MonoObject*)object)->vtable) {
4743 DisappearingLink *next = entry->next;
4748 disappearing_link_hash [i] = next;
4750 if (*(entry->link)) {
4751 *(entry->link) = NULL;
4752 g_warning ("Disappearing link %p not freed", entry->link);
4754 free_internal_mem (entry, INTERNAL_MEM_DISLINK);
4761 entry = entry->next;
4766 /* LOCKING: requires that the GC lock is held */
4768 finalizers_for_domain (MonoDomain *domain, MonoObject **out_array, int out_size,
4769 FinalizeEntryHashTable *hash_table)
4771 FinalizeEntry **finalizable_hash = hash_table->table;
4772 mword finalizable_hash_size = hash_table->size;
4773 FinalizeEntry *entry, *prev;
4776 if (no_finalize || !out_size || !out_array)
4779 for (i = 0; i < finalizable_hash_size; ++i) {
4781 for (entry = finalizable_hash [i]; entry;) {
4782 if (mono_object_domain (entry->object) == domain) {
4783 FinalizeEntry *next;
4784 /* remove and put in out_array */
4786 prev->next = entry->next;
4788 finalizable_hash [i] = entry->next;
4790 hash_table->num_registered--;
4791 out_array [count ++] = entry->object;
4792 DEBUG (5, fprintf (gc_debug_file, "Collecting object for finalization: %p (%s) (%d/%d)\n", entry->object, safe_name (entry->object), num_ready_finalizers, hash_table->num_registered));
4794 if (count == out_size)
4799 entry = entry->next;
4806 * mono_gc_finalizers_for_domain:
4807 * @domain: the unloading appdomain
4808 * @out_array: output array
4809 * @out_size: size of output array
4811 * Store inside @out_array up to @out_size objects that belong to the unloading
4812 * appdomain @domain. Returns the number of stored items. Can be called repeteadly
4813 * until it returns 0.
4814 * The items are removed from the finalizer data structure, so the caller is supposed
4816 * @out_array should be on the stack to allow the GC to know the objects are still alive.
4819 mono_gc_finalizers_for_domain (MonoDomain *domain, MonoObject **out_array, int out_size)
4824 result = finalizers_for_domain (domain, out_array, out_size, &minor_finalizable_hash);
4825 if (result < out_size) {
4826 result += finalizers_for_domain (domain, out_array + result, out_size - result,
4827 &major_finalizable_hash);
4835 register_for_finalization (MonoObject *obj, void *user_data, int generation)
4837 FinalizeEntryHashTable *hash_table = get_finalize_entry_hash_table (generation);
4838 FinalizeEntry **finalizable_hash;
4839 mword finalizable_hash_size;
4840 FinalizeEntry *entry, *prev;
4844 g_assert (user_data == NULL || user_data == mono_gc_run_finalize);
4845 hash = mono_object_hash (obj);
4847 rehash_fin_table_if_necessary (hash_table);
4848 finalizable_hash = hash_table->table;
4849 finalizable_hash_size = hash_table->size;
4850 hash %= finalizable_hash_size;
4852 for (entry = finalizable_hash [hash]; entry; entry = entry->next) {
4853 if (entry->object == obj) {
4855 /* remove from the list */
4857 prev->next = entry->next;
4859 finalizable_hash [hash] = entry->next;
4860 hash_table->num_registered--;
4861 DEBUG (5, fprintf (gc_debug_file, "Removed finalizer %p for object: %p (%s) (%d)\n", entry, obj, obj->vtable->klass->name, hash_table->num_registered));
4862 free_internal_mem (entry, INTERNAL_MEM_FINALIZE_ENTRY);
4870 /* request to deregister, but already out of the list */
4874 entry = get_internal_mem (sizeof (FinalizeEntry), INTERNAL_MEM_FINALIZE_ENTRY);
4875 entry->object = obj;
4876 entry->next = finalizable_hash [hash];
4877 finalizable_hash [hash] = entry;
4878 hash_table->num_registered++;
4879 DEBUG (5, fprintf (gc_debug_file, "Added finalizer %p for object: %p (%s) (%d) to %s table\n", entry, obj, obj->vtable->klass->name, hash_table->num_registered, generation_name (generation)));
4884 mono_gc_register_for_finalization (MonoObject *obj, void *user_data)
4886 if (ptr_in_nursery (obj))
4887 register_for_finalization (obj, user_data, GENERATION_NURSERY);
4889 register_for_finalization (obj, user_data, GENERATION_OLD);
4893 rehash_dislink (DisappearingLinkHashTable *hash_table)
4895 DisappearingLink **disappearing_link_hash = hash_table->table;
4896 int disappearing_link_hash_size = hash_table->size;
4899 DisappearingLink **new_hash;
4900 DisappearingLink *entry, *next;
4901 int new_size = g_spaced_primes_closest (hash_table->num_links);
4903 new_hash = get_internal_mem (new_size * sizeof (DisappearingLink*), INTERNAL_MEM_DISLINK_TABLE);
4904 for (i = 0; i < disappearing_link_hash_size; ++i) {
4905 for (entry = disappearing_link_hash [i]; entry; entry = next) {
4906 hash = mono_aligned_addr_hash (entry->link) % new_size;
4908 entry->next = new_hash [hash];
4909 new_hash [hash] = entry;
4912 free_internal_mem (disappearing_link_hash, INTERNAL_MEM_DISLINK_TABLE);
4913 hash_table->table = new_hash;
4914 hash_table->size = new_size;
4917 /* LOCKING: assumes the GC lock is held */
4919 add_or_remove_disappearing_link (MonoObject *obj, void **link, gboolean track, int generation)
4921 DisappearingLinkHashTable *hash_table = get_dislink_hash_table (generation);
4922 DisappearingLink *entry, *prev;
4924 DisappearingLink **disappearing_link_hash = hash_table->table;
4925 int disappearing_link_hash_size = hash_table->size;
4927 if (hash_table->num_links >= disappearing_link_hash_size * 2) {
4928 rehash_dislink (hash_table);
4929 disappearing_link_hash = hash_table->table;
4930 disappearing_link_hash_size = hash_table->size;
4932 /* FIXME: add check that link is not in the heap */
4933 hash = mono_aligned_addr_hash (link) % disappearing_link_hash_size;
4934 entry = disappearing_link_hash [hash];
4936 for (; entry; entry = entry->next) {
4937 /* link already added */
4938 if (link == entry->link) {
4939 /* NULL obj means remove */
4942 prev->next = entry->next;
4944 disappearing_link_hash [hash] = entry->next;
4945 hash_table->num_links--;
4946 DEBUG (5, fprintf (gc_debug_file, "Removed dislink %p (%d) from %s table\n", entry, hash_table->num_links, generation_name (generation)));
4947 free_internal_mem (entry, INTERNAL_MEM_DISLINK);
4950 *link = HIDE_POINTER (obj, track); /* we allow the change of object */
4958 entry = get_internal_mem (sizeof (DisappearingLink), INTERNAL_MEM_DISLINK);
4959 *link = HIDE_POINTER (obj, track);
4961 entry->next = disappearing_link_hash [hash];
4962 disappearing_link_hash [hash] = entry;
4963 hash_table->num_links++;
4964 DEBUG (5, fprintf (gc_debug_file, "Added dislink %p for object: %p (%s) at %p to %s table\n", entry, obj, obj->vtable->klass->name, link, generation_name (generation)));
4967 /* LOCKING: assumes the GC lock is held */
4969 mono_gc_register_disappearing_link (MonoObject *obj, void **link, gboolean track)
4971 add_or_remove_disappearing_link (NULL, link, FALSE, GENERATION_NURSERY);
4972 add_or_remove_disappearing_link (NULL, link, FALSE, GENERATION_OLD);
4974 if (ptr_in_nursery (obj))
4975 add_or_remove_disappearing_link (obj, link, track, GENERATION_NURSERY);
4977 add_or_remove_disappearing_link (obj, link, track, GENERATION_OLD);
4982 mono_gc_invoke_finalizers (void)
4984 FinalizeEntry *entry = NULL;
4985 gboolean entry_is_critical = FALSE;
4988 /* FIXME: batch to reduce lock contention */
4989 while (fin_ready_list || critical_fin_list) {
4993 FinalizeEntry **list = entry_is_critical ? &critical_fin_list : &fin_ready_list;
4995 /* We have finalized entry in the last
4996 interation, now we need to remove it from
4999 *list = entry->next;
5001 FinalizeEntry *e = *list;
5002 while (e->next != entry)
5004 e->next = entry->next;
5006 free_internal_mem (entry, INTERNAL_MEM_FINALIZE_ENTRY);
5010 /* Now look for the first non-null entry. */
5011 for (entry = fin_ready_list; entry && !entry->object; entry = entry->next)
5014 entry_is_critical = FALSE;
5016 entry_is_critical = TRUE;
5017 for (entry = critical_fin_list; entry && !entry->object; entry = entry->next)
5022 g_assert (entry->object);
5023 num_ready_finalizers--;
5024 obj = entry->object;
5025 entry->object = NULL;
5026 DEBUG (7, fprintf (gc_debug_file, "Finalizing object %p (%s)\n", obj, safe_name (obj)));
5034 g_assert (entry->object == NULL);
5036 /* the object is on the stack so it is pinned */
5037 /*g_print ("Calling finalizer for object: %p (%s)\n", entry->object, safe_name (entry->object));*/
5038 mono_gc_run_finalize (obj, NULL);
5045 mono_gc_pending_finalizers (void)
5047 return fin_ready_list || critical_fin_list;
5050 /* Negative value to remove */
5052 mono_gc_add_memory_pressure (gint64 value)
5054 /* FIXME: Use interlocked functions */
5056 memory_pressure += value;
5061 * ######################################################################
5062 * ######## registered roots support
5063 * ######################################################################
5067 rehash_roots (gboolean pinned)
5071 RootRecord **new_hash;
5072 RootRecord *entry, *next;
5075 new_size = g_spaced_primes_closest (num_roots_entries [pinned]);
5076 new_hash = get_internal_mem (new_size * sizeof (RootRecord*), INTERNAL_MEM_ROOTS_TABLE);
5077 for (i = 0; i < roots_hash_size [pinned]; ++i) {
5078 for (entry = roots_hash [pinned][i]; entry; entry = next) {
5079 hash = mono_aligned_addr_hash (entry->start_root) % new_size;
5081 entry->next = new_hash [hash];
5082 new_hash [hash] = entry;
5085 free_internal_mem (roots_hash [pinned], INTERNAL_MEM_ROOTS_TABLE);
5086 roots_hash [pinned] = new_hash;
5087 roots_hash_size [pinned] = new_size;
5091 find_root (int root_type, char *start, guint32 addr_hash)
5093 RootRecord *new_root;
5095 guint32 hash = addr_hash % roots_hash_size [root_type];
5096 for (new_root = roots_hash [root_type][hash]; new_root; new_root = new_root->next) {
5097 /* we allow changing the size and the descriptor (for thread statics etc) */
5098 if (new_root->start_root == start) {
5107 * We do not coalesce roots.
5110 mono_gc_register_root_inner (char *start, size_t size, void *descr, int root_type)
5112 RootRecord *new_root;
5113 unsigned int hash, addr_hash = mono_aligned_addr_hash (start);
5116 for (i = 0; i < ROOT_TYPE_NUM; ++i) {
5117 if (num_roots_entries [i] >= roots_hash_size [i] * 2)
5120 for (i = 0; i < ROOT_TYPE_NUM; ++i) {
5121 new_root = find_root (i, start, addr_hash);
5122 /* we allow changing the size and the descriptor (for thread statics etc) */
5124 size_t old_size = new_root->end_root - new_root->start_root;
5125 new_root->end_root = new_root->start_root + size;
5126 g_assert (((new_root->root_desc != 0) && (descr != NULL)) ||
5127 ((new_root->root_desc == 0) && (descr == NULL)));
5128 new_root->root_desc = (mword)descr;
5130 roots_size -= old_size;
5135 new_root = get_internal_mem (sizeof (RootRecord), INTERNAL_MEM_ROOT_RECORD);
5137 new_root->start_root = start;
5138 new_root->end_root = new_root->start_root + size;
5139 new_root->root_desc = (mword)descr;
5141 hash = addr_hash % roots_hash_size [root_type];
5142 num_roots_entries [root_type]++;
5143 new_root->next = roots_hash [root_type] [hash];
5144 roots_hash [root_type][hash] = new_root;
5145 DEBUG (3, fprintf (gc_debug_file, "Added root %p for range: %p-%p, descr: %p (%d/%d bytes)\n", new_root, new_root->start_root, new_root->end_root, descr, (int)size, (int)roots_size));
5155 mono_gc_register_root (char *start, size_t size, void *descr)
5157 return mono_gc_register_root_inner (start, size, descr, descr ? ROOT_TYPE_NORMAL : ROOT_TYPE_PINNED);
5161 mono_gc_register_root_wbarrier (char *start, size_t size, void *descr)
5163 return mono_gc_register_root_inner (start, size, descr, ROOT_TYPE_WBARRIER);
5167 mono_gc_deregister_root (char* addr)
5169 RootRecord *tmp, *prev;
5170 unsigned int hash, addr_hash = mono_aligned_addr_hash (addr);
5174 for (root_type = 0; root_type < ROOT_TYPE_NUM; ++root_type) {
5175 hash = addr_hash % roots_hash_size [root_type];
5176 tmp = roots_hash [root_type][hash];
5179 if (tmp->start_root == (char*)addr) {
5181 prev->next = tmp->next;
5183 roots_hash [root_type][hash] = tmp->next;
5184 roots_size -= (tmp->end_root - tmp->start_root);
5185 num_roots_entries [root_type]--;
5186 DEBUG (3, fprintf (gc_debug_file, "Removed root %p for range: %p-%p\n", tmp, tmp->start_root, tmp->end_root));
5187 free_internal_mem (tmp, INTERNAL_MEM_ROOT_RECORD);
5198 * ######################################################################
5199 * ######## Thread handling (stop/start code)
5200 * ######################################################################
5203 /* FIXME: handle large/small config */
5204 #define HASH_PTHREAD_T(id) (((unsigned int)(id) >> 4) * 2654435761u)
5206 static SgenThreadInfo* thread_table [THREAD_HASH_SIZE];
5208 #if USE_SIGNAL_BASED_START_STOP_WORLD
5210 static MonoSemType suspend_ack_semaphore;
5211 static MonoSemType *suspend_ack_semaphore_ptr;
5212 static unsigned int global_stop_count = 0;
5214 static sigset_t suspend_signal_mask;
5215 static mword cur_thread_regs [ARCH_NUM_REGS] = {0};
5217 /* LOCKING: assumes the GC lock is held */
5219 mono_sgen_get_thread_table (void)
5221 return thread_table;
5225 mono_sgen_thread_info_lookup (ARCH_THREAD_TYPE id)
5227 unsigned int hash = HASH_PTHREAD_T (id) % THREAD_HASH_SIZE;
5228 SgenThreadInfo *info;
5230 info = thread_table [hash];
5231 while (info && !ARCH_THREAD_EQUALS (info->id, id)) {
5238 update_current_thread_stack (void *start)
5240 void *ptr = cur_thread_regs;
5241 SgenThreadInfo *info = mono_sgen_thread_info_lookup (ARCH_GET_THREAD ());
5243 info->stack_start = align_pointer (&ptr);
5244 g_assert (info->stack_start >= info->stack_start_limit && info->stack_start < info->stack_end);
5245 ARCH_STORE_REGS (ptr);
5246 info->stopped_regs = ptr;
5247 if (gc_callbacks.thread_suspend_func)
5248 gc_callbacks.thread_suspend_func (info->runtime_data, NULL);
5252 * Define this and use the "xdomain-checks" MONO_GC_DEBUG option to
5253 * have cross-domain checks in the write barrier.
5255 //#define XDOMAIN_CHECKS_IN_WBARRIER
5257 #ifndef BINARY_PROTOCOL
5258 #ifndef HEAVY_STATISTICS
5259 #define MANAGED_ALLOCATION
5260 #ifndef XDOMAIN_CHECKS_IN_WBARRIER
5261 #define MANAGED_WBARRIER
5267 is_ip_in_managed_allocator (MonoDomain *domain, gpointer ip);
5270 mono_sgen_wait_for_suspend_ack (int count)
5274 for (i = 0; i < count; ++i) {
5275 while ((result = MONO_SEM_WAIT (suspend_ack_semaphore_ptr)) != 0) {
5276 if (errno != EINTR) {
5277 g_error ("sem_wait ()");
5284 restart_threads_until_none_in_managed_allocator (void)
5286 SgenThreadInfo *info;
5287 int i, result, num_threads_died = 0;
5288 int sleep_duration = -1;
5291 int restart_count = 0, restarted_count = 0;
5292 /* restart all threads that stopped in the
5294 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5295 for (info = thread_table [i]; info; info = info->next) {
5298 if (!info->stack_start || info->in_critical_region ||
5299 is_ip_in_managed_allocator (info->stopped_domain, info->stopped_ip)) {
5300 binary_protocol_thread_restart ((gpointer)info->id);
5301 #if defined(__MACH__) && MONO_MACH_ARCH_SUPPORTED
5302 result = thread_resume (pthread_mach_thread_np (info->id));
5304 result = pthread_kill (info->id, restart_signal_num);
5312 /* we set the stopped_ip to
5313 NULL for threads which
5314 we're not restarting so
5315 that we can easily identify
5317 info->stopped_ip = NULL;
5318 info->stopped_domain = NULL;
5322 /* if no threads were restarted, we're done */
5323 if (restart_count == 0)
5326 #if defined(__MACH__) && MONO_MACH_ARCH_SUPPORTED
5327 /* mach thread_resume is synchronous so we dont need to wait for them */
5329 /* wait for the threads to signal their restart */
5330 mono_sgen_wait_for_suspend_ack (restart_count);
5333 if (sleep_duration < 0) {
5337 g_usleep (sleep_duration);
5338 sleep_duration += 10;
5341 /* stop them again */
5342 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5343 for (info = thread_table [i]; info; info = info->next) {
5344 if (info->skip || info->stopped_ip == NULL)
5346 #if defined(__MACH__) && MONO_MACH_ARCH_SUPPORTED
5347 result = thread_suspend (pthread_mach_thread_np (info->id));
5349 result = pthread_kill (info->id, suspend_signal_num);
5358 /* some threads might have died */
5359 num_threads_died += restart_count - restarted_count;
5360 #if defined(__MACH__) && MONO_MACH_ARCH_SUPPORTED
5361 /* mach thread_resume is synchronous so we dont need to wait for them */
5363 /* wait for the threads to signal their suspension
5365 mono_sgen_wait_for_suspend_ack (restart_count);
5369 return num_threads_died;
5372 /* LOCKING: assumes the GC lock is held (by the stopping thread) */
5374 suspend_handler (int sig, siginfo_t *siginfo, void *context)
5376 SgenThreadInfo *info;
5379 int old_errno = errno;
5380 gpointer regs [ARCH_NUM_REGS];
5381 gpointer stack_start;
5383 id = pthread_self ();
5384 info = mono_sgen_thread_info_lookup (id);
5385 info->stopped_domain = mono_domain_get ();
5386 info->stopped_ip = (gpointer) ARCH_SIGCTX_IP (context);
5387 stop_count = global_stop_count;
5388 /* duplicate signal */
5389 if (0 && info->stop_count == stop_count) {
5393 #ifdef HAVE_KW_THREAD
5394 /* update the remset info in the thread data structure */
5395 info->remset = remembered_set;
5397 stack_start = (char*) ARCH_SIGCTX_SP (context) - REDZONE_SIZE;
5398 /* If stack_start is not within the limits, then don't set it
5399 in info and we will be restarted. */
5400 if (stack_start >= info->stack_start_limit && info->stack_start <= info->stack_end) {
5401 info->stack_start = stack_start;
5403 ARCH_COPY_SIGCTX_REGS (regs, context);
5404 info->stopped_regs = regs;
5406 g_assert (!info->stack_start);
5409 /* Notify the JIT */
5410 if (gc_callbacks.thread_suspend_func)
5411 gc_callbacks.thread_suspend_func (info->runtime_data, context);
5413 DEBUG (4, fprintf (gc_debug_file, "Posting suspend_ack_semaphore for suspend from %p %p\n", info, (gpointer)ARCH_GET_THREAD ()));
5414 /* notify the waiting thread */
5415 MONO_SEM_POST (suspend_ack_semaphore_ptr);
5416 info->stop_count = stop_count;
5418 /* wait until we receive the restart signal */
5421 sigsuspend (&suspend_signal_mask);
5422 } while (info->signal != restart_signal_num);
5424 DEBUG (4, fprintf (gc_debug_file, "Posting suspend_ack_semaphore for resume from %p %p\n", info, (gpointer)ARCH_GET_THREAD ()));
5425 /* notify the waiting thread */
5426 MONO_SEM_POST (suspend_ack_semaphore_ptr);
5432 restart_handler (int sig)
5434 SgenThreadInfo *info;
5435 int old_errno = errno;
5437 info = mono_sgen_thread_info_lookup (pthread_self ());
5438 info->signal = restart_signal_num;
5439 DEBUG (4, fprintf (gc_debug_file, "Restart handler in %p %p\n", info, (gpointer)ARCH_GET_THREAD ()));
5445 acquire_gc_locks (void)
5451 release_gc_locks (void)
5453 UNLOCK_INTERRUPTION;
5456 static TV_DECLARE (stop_world_time);
5457 static unsigned long max_pause_usec = 0;
5459 /* LOCKING: assumes the GC lock is held */
5465 acquire_gc_locks ();
5467 update_current_thread_stack (&count);
5469 global_stop_count++;
5470 DEBUG (3, fprintf (gc_debug_file, "stopping world n %d from %p %p\n", global_stop_count, mono_sgen_thread_info_lookup (ARCH_GET_THREAD ()), (gpointer)ARCH_GET_THREAD ()));
5471 TV_GETTIME (stop_world_time);
5472 count = mono_sgen_thread_handshake (suspend_signal_num);
5473 count -= restart_threads_until_none_in_managed_allocator ();
5474 g_assert (count >= 0);
5475 DEBUG (3, fprintf (gc_debug_file, "world stopped %d thread(s)\n", count));
5479 /* LOCKING: assumes the GC lock is held */
5481 restart_world (void)
5484 SgenThreadInfo *info;
5485 TV_DECLARE (end_sw);
5488 /* notify the profiler of the leftovers */
5489 if (G_UNLIKELY (mono_profiler_events & MONO_PROFILE_GC_MOVES)) {
5490 if (moved_objects_idx) {
5491 mono_profiler_gc_moves (moved_objects, moved_objects_idx);
5492 moved_objects_idx = 0;
5495 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5496 for (info = thread_table [i]; info; info = info->next) {
5497 info->stack_start = NULL;
5498 info->stopped_regs = NULL;
5502 release_gc_locks ();
5504 count = mono_sgen_thread_handshake (restart_signal_num);
5505 TV_GETTIME (end_sw);
5506 usec = TV_ELAPSED (stop_world_time, end_sw);
5507 max_pause_usec = MAX (usec, max_pause_usec);
5508 DEBUG (2, fprintf (gc_debug_file, "restarted %d thread(s) (pause time: %d usec, max: %d)\n", count, (int)usec, (int)max_pause_usec));
5512 #endif /* USE_SIGNAL_BASED_START_STOP_WORLD */
5515 mono_gc_set_gc_callbacks (MonoGCCallbacks *callbacks)
5517 gc_callbacks = *callbacks;
5521 mono_gc_get_gc_callbacks ()
5523 return &gc_callbacks;
5526 /* Variables holding start/end nursery so it won't have to be passed at every call */
5527 static void *scan_area_arg_start, *scan_area_arg_end;
5530 mono_gc_conservatively_scan_area (void *start, void *end)
5532 conservatively_pin_objects_from (start, end, scan_area_arg_start, scan_area_arg_end, PIN_TYPE_STACK);
5536 mono_gc_scan_object (void *obj)
5538 if (current_collection_generation == GENERATION_NURSERY)
5541 major_copy_or_mark_object (&obj);
5546 * Mark from thread stacks and registers.
5549 scan_thread_data (void *start_nursery, void *end_nursery, gboolean precise)
5552 SgenThreadInfo *info;
5554 scan_area_arg_start = start_nursery;
5555 scan_area_arg_end = end_nursery;
5557 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5558 for (info = thread_table [i]; info; info = info->next) {
5560 DEBUG (3, fprintf (gc_debug_file, "Skipping dead thread %p, range: %p-%p, size: %td\n", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start));
5563 DEBUG (3, fprintf (gc_debug_file, "Scanning thread %p, range: %p-%p, size: %td, pinned=%d\n", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start, next_pin_slot));
5564 if (gc_callbacks.thread_mark_func && !conservative_stack_mark)
5565 gc_callbacks.thread_mark_func (info->runtime_data, info->stack_start, info->stack_end, precise);
5567 conservatively_pin_objects_from (info->stack_start, info->stack_end, start_nursery, end_nursery, PIN_TYPE_STACK);
5570 conservatively_pin_objects_from (info->stopped_regs, info->stopped_regs + ARCH_NUM_REGS,
5571 start_nursery, end_nursery, PIN_TYPE_STACK);
5577 find_pinning_ref_from_thread (char *obj, size_t size)
5580 SgenThreadInfo *info;
5581 char *endobj = obj + size;
5583 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5584 for (info = thread_table [i]; info; info = info->next) {
5585 char **start = (char**)info->stack_start;
5588 while (start < (char**)info->stack_end) {
5589 if (*start >= obj && *start < endobj) {
5590 DEBUG (0, fprintf (gc_debug_file, "Object %p referenced in thread %p (id %p) at %p, stack: %p-%p\n", obj, info, (gpointer)info->id, start, info->stack_start, info->stack_end));
5595 /* FIXME: check info->stopped_regs */
5601 ptr_on_stack (void *ptr)
5603 gpointer stack_start = &stack_start;
5604 SgenThreadInfo *info = mono_sgen_thread_info_lookup (ARCH_GET_THREAD ());
5606 if (ptr >= stack_start && ptr < (gpointer)info->stack_end)
5612 handle_remset (mword *p, void *start_nursery, void *end_nursery, gboolean global)
5619 HEAVY_STAT (++stat_global_remsets_processed);
5621 /* FIXME: exclude stack locations */
5622 switch ((*p) & REMSET_TYPE_MASK) {
5623 case REMSET_LOCATION:
5625 //__builtin_prefetch (ptr);
5626 if (((void*)ptr < start_nursery || (void*)ptr >= end_nursery)) {
5627 gpointer old = *ptr;
5629 DEBUG (9, fprintf (gc_debug_file, "Overwrote remset at %p with %p\n", ptr, *ptr));
5631 binary_protocol_ptr_update (ptr, old, *ptr, (gpointer)LOAD_VTABLE (*ptr), safe_object_get_size (*ptr));
5632 if (!global && *ptr >= start_nursery && *ptr < end_nursery) {
5634 * If the object is pinned, each reference to it from nonpinned objects
5635 * becomes part of the global remset, which can grow very large.
5637 DEBUG (9, fprintf (gc_debug_file, "Add to global remset because of pinning %p (%p %s)\n", ptr, *ptr, safe_name (*ptr)));
5638 add_to_global_remset (ptr);
5641 DEBUG (9, fprintf (gc_debug_file, "Skipping remset at %p holding %p\n", ptr, *ptr));
5645 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
5646 if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery))
5649 while (count-- > 0) {
5651 DEBUG (9, fprintf (gc_debug_file, "Overwrote remset at %p with %p (count: %d)\n", ptr, *ptr, (int)count));
5652 if (!global && *ptr >= start_nursery && *ptr < end_nursery)
5653 add_to_global_remset (ptr);
5658 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
5659 if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery))
5661 scan_object ((char*)ptr);
5663 case REMSET_VTYPE: {
5664 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
5665 if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery))
5670 ptr = (void**) scan_vtype ((char*)ptr, desc, start_nursery, end_nursery);
5674 g_assert_not_reached ();
5679 #ifdef HEAVY_STATISTICS
5681 collect_store_remsets (RememberedSet *remset, mword *bumper)
5683 mword *p = remset->data;
5688 while (p < remset->store_next) {
5689 switch ((*p) & REMSET_TYPE_MASK) {
5690 case REMSET_LOCATION:
5693 ++stat_saved_remsets_1;
5695 if (*p == last1 || *p == last2) {
5696 ++stat_saved_remsets_2;
5713 g_assert_not_reached ();
5723 RememberedSet *remset;
5725 SgenThreadInfo *info;
5727 mword *addresses, *bumper, *p, *r;
5729 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5730 for (info = thread_table [i]; info; info = info->next) {
5731 for (remset = info->remset; remset; remset = remset->next)
5732 size += remset->store_next - remset->data;
5735 for (remset = freed_thread_remsets; remset; remset = remset->next)
5736 size += remset->store_next - remset->data;
5737 for (remset = global_remset; remset; remset = remset->next)
5738 size += remset->store_next - remset->data;
5740 bumper = addresses = get_internal_mem (sizeof (mword) * size, INTERNAL_MEM_STATISTICS);
5742 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5743 for (info = thread_table [i]; info; info = info->next) {
5744 for (remset = info->remset; remset; remset = remset->next)
5745 bumper = collect_store_remsets (remset, bumper);
5748 for (remset = global_remset; remset; remset = remset->next)
5749 bumper = collect_store_remsets (remset, bumper);
5750 for (remset = freed_thread_remsets; remset; remset = remset->next)
5751 bumper = collect_store_remsets (remset, bumper);
5753 g_assert (bumper <= addresses + size);
5755 stat_store_remsets += bumper - addresses;
5757 sort_addresses ((void**)addresses, bumper - addresses);
5760 while (r < bumper) {
5766 stat_store_remsets_unique += p - addresses;
5768 free_internal_mem (addresses, INTERNAL_MEM_STATISTICS);
5773 clear_thread_store_remset_buffer (SgenThreadInfo *info)
5775 *info->store_remset_buffer_index_addr = 0;
5776 memset (*info->store_remset_buffer_addr, 0, sizeof (gpointer) * STORE_REMSET_BUFFER_SIZE);
5780 scan_from_remsets (void *start_nursery, void *end_nursery)
5783 SgenThreadInfo *info;
5784 RememberedSet *remset;
5785 GenericStoreRememberedSet *store_remset;
5786 mword *p, *next_p, *store_pos;
5788 #ifdef HEAVY_STATISTICS
5792 /* the global one */
5793 for (remset = global_remset; remset; remset = remset->next) {
5794 DEBUG (4, fprintf (gc_debug_file, "Scanning global remset range: %p-%p, size: %td\n", remset->data, remset->store_next, remset->store_next - remset->data));
5795 store_pos = remset->data;
5796 for (p = remset->data; p < remset->store_next; p = next_p) {
5797 void **ptr = (void**)p [0];
5799 /*Ignore previously processed remset.*/
5800 if (!global_remset_location_was_not_added (ptr)) {
5805 next_p = handle_remset (p, start_nursery, end_nursery, TRUE);
5808 * Clear global remsets of locations which no longer point to the
5809 * nursery. Otherwise, they could grow indefinitely between major
5812 * Since all global remsets are location remsets, we don't need to unmask the pointer.
5814 if (ptr_in_nursery (*ptr)) {
5815 *store_pos ++ = p [0];
5816 HEAVY_STAT (++stat_global_remsets_readded);
5820 /* Truncate the remset */
5821 remset->store_next = store_pos;
5824 /* the generic store ones */
5825 store_remset = generic_store_remsets;
5826 while (store_remset) {
5827 GenericStoreRememberedSet *next = store_remset->next;
5829 for (i = 0; i < STORE_REMSET_BUFFER_SIZE - 1; ++i) {
5830 gpointer addr = store_remset->data [i];
5832 handle_remset ((mword*)&addr, start_nursery, end_nursery, FALSE);
5835 free_internal_mem (store_remset, INTERNAL_MEM_STORE_REMSET);
5837 store_remset = next;
5839 generic_store_remsets = NULL;
5841 /* the per-thread ones */
5842 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5843 for (info = thread_table [i]; info; info = info->next) {
5844 RememberedSet *next;
5846 for (remset = info->remset; remset; remset = next) {
5847 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for thread %p, range: %p-%p, size: %td\n", info, remset->data, remset->store_next, remset->store_next - remset->data));
5848 for (p = remset->data; p < remset->store_next;) {
5849 p = handle_remset (p, start_nursery, end_nursery, FALSE);
5851 remset->store_next = remset->data;
5852 next = remset->next;
5853 remset->next = NULL;
5854 if (remset != info->remset) {
5855 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
5856 free_internal_mem (remset, INTERNAL_MEM_REMSET);
5859 for (j = 0; j < *info->store_remset_buffer_index_addr; ++j)
5860 handle_remset ((mword*)*info->store_remset_buffer_addr + j + 1, start_nursery, end_nursery, FALSE);
5861 clear_thread_store_remset_buffer (info);
5865 /* the freed thread ones */
5866 while (freed_thread_remsets) {
5867 RememberedSet *next;
5868 remset = freed_thread_remsets;
5869 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for freed thread, range: %p-%p, size: %td\n", remset->data, remset->store_next, remset->store_next - remset->data));
5870 for (p = remset->data; p < remset->store_next;) {
5871 p = handle_remset (p, start_nursery, end_nursery, FALSE);
5873 next = remset->next;
5874 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
5875 free_internal_mem (remset, INTERNAL_MEM_REMSET);
5876 freed_thread_remsets = next;
5881 * Clear the info in the remembered sets: we're doing a major collection, so
5882 * the per-thread ones are not needed and the global ones will be reconstructed
5886 clear_remsets (void)
5889 SgenThreadInfo *info;
5890 RememberedSet *remset, *next;
5892 /* the global list */
5893 for (remset = global_remset; remset; remset = next) {
5894 remset->store_next = remset->data;
5895 next = remset->next;
5896 remset->next = NULL;
5897 if (remset != global_remset) {
5898 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
5899 free_internal_mem (remset, INTERNAL_MEM_REMSET);
5902 /* the generic store ones */
5903 while (generic_store_remsets) {
5904 GenericStoreRememberedSet *gs_next = generic_store_remsets->next;
5905 free_internal_mem (generic_store_remsets, INTERNAL_MEM_STORE_REMSET);
5906 generic_store_remsets = gs_next;
5908 /* the per-thread ones */
5909 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5910 for (info = thread_table [i]; info; info = info->next) {
5911 for (remset = info->remset; remset; remset = next) {
5912 remset->store_next = remset->data;
5913 next = remset->next;
5914 remset->next = NULL;
5915 if (remset != info->remset) {
5916 DEBUG (3, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
5917 free_internal_mem (remset, INTERNAL_MEM_REMSET);
5920 clear_thread_store_remset_buffer (info);
5924 /* the freed thread ones */
5925 while (freed_thread_remsets) {
5926 next = freed_thread_remsets->next;
5927 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", freed_thread_remsets->data));
5928 free_internal_mem (freed_thread_remsets, INTERNAL_MEM_REMSET);
5929 freed_thread_remsets = next;
5934 * Clear the thread local TLAB variables for all threads.
5939 SgenThreadInfo *info;
5942 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5943 for (info = thread_table [i]; info; info = info->next) {
5944 /* A new TLAB will be allocated when the thread does its first allocation */
5945 *info->tlab_start_addr = NULL;
5946 *info->tlab_next_addr = NULL;
5947 *info->tlab_temp_end_addr = NULL;
5948 *info->tlab_real_end_addr = NULL;
5953 /* LOCKING: assumes the GC lock is held */
5954 static SgenThreadInfo*
5955 gc_register_current_thread (void *addr)
5958 SgenThreadInfo* info = malloc (sizeof (SgenThreadInfo));
5959 #ifndef HAVE_KW_THREAD
5960 SgenThreadInfo *__thread_info__ = info;
5966 memset (info, 0, sizeof (SgenThreadInfo));
5967 #ifndef HAVE_KW_THREAD
5968 info->tlab_start = info->tlab_next = info->tlab_temp_end = info->tlab_real_end = NULL;
5970 g_assert (!pthread_getspecific (thread_info_key));
5971 pthread_setspecific (thread_info_key, info);
5976 info->id = ARCH_GET_THREAD ();
5977 info->stop_count = -1;
5980 info->stack_start = NULL;
5981 info->tlab_start_addr = &TLAB_START;
5982 info->tlab_next_addr = &TLAB_NEXT;
5983 info->tlab_temp_end_addr = &TLAB_TEMP_END;
5984 info->tlab_real_end_addr = &TLAB_REAL_END;
5985 info->store_remset_buffer_addr = &STORE_REMSET_BUFFER;
5986 info->store_remset_buffer_index_addr = &STORE_REMSET_BUFFER_INDEX;
5987 info->stopped_ip = NULL;
5988 info->stopped_domain = NULL;
5989 info->stopped_regs = NULL;
5991 binary_protocol_thread_register ((gpointer)info->id);
5993 #ifdef HAVE_KW_THREAD
5994 tlab_next_addr = &tlab_next;
5995 store_remset_buffer_index_addr = &store_remset_buffer_index;
5998 /* try to get it with attributes first */
5999 #if defined(HAVE_PTHREAD_GETATTR_NP) && defined(HAVE_PTHREAD_ATTR_GETSTACK)
6003 pthread_attr_t attr;
6004 pthread_getattr_np (pthread_self (), &attr);
6005 pthread_attr_getstack (&attr, &sstart, &size);
6006 info->stack_start_limit = sstart;
6007 info->stack_end = (char*)sstart + size;
6008 pthread_attr_destroy (&attr);
6010 #elif defined(HAVE_PTHREAD_GET_STACKSIZE_NP) && defined(HAVE_PTHREAD_GET_STACKADDR_NP)
6011 info->stack_end = (char*)pthread_get_stackaddr_np (pthread_self ());
6012 info->stack_start_limit = (char*)info->stack_end - pthread_get_stacksize_np (pthread_self ());
6015 /* FIXME: we assume the stack grows down */
6016 gsize stack_bottom = (gsize)addr;
6017 stack_bottom += 4095;
6018 stack_bottom &= ~4095;
6019 info->stack_end = (char*)stack_bottom;
6023 #ifdef HAVE_KW_THREAD
6024 stack_end = info->stack_end;
6027 /* hash into the table */
6028 hash = HASH_PTHREAD_T (info->id) % THREAD_HASH_SIZE;
6029 info->next = thread_table [hash];
6030 thread_table [hash] = info;
6032 info->remset = alloc_remset (DEFAULT_REMSET_SIZE, info);
6033 pthread_setspecific (remembered_set_key, info->remset);
6034 #ifdef HAVE_KW_THREAD
6035 remembered_set = info->remset;
6038 STORE_REMSET_BUFFER = get_internal_mem (sizeof (gpointer) * STORE_REMSET_BUFFER_SIZE, INTERNAL_MEM_STORE_REMSET);
6039 STORE_REMSET_BUFFER_INDEX = 0;
6041 DEBUG (3, fprintf (gc_debug_file, "registered thread %p (%p) (hash: %d)\n", info, (gpointer)info->id, hash));
6043 if (gc_callbacks.thread_attach_func)
6044 info->runtime_data = gc_callbacks.thread_attach_func ();
6050 add_generic_store_remset_from_buffer (gpointer *buffer)
6052 GenericStoreRememberedSet *remset = get_internal_mem (sizeof (GenericStoreRememberedSet), INTERNAL_MEM_STORE_REMSET);
6053 memcpy (remset->data, buffer + 1, sizeof (gpointer) * (STORE_REMSET_BUFFER_SIZE - 1));
6054 remset->next = generic_store_remsets;
6055 generic_store_remsets = remset;
6059 unregister_current_thread (void)
6062 SgenThreadInfo *prev = NULL;
6064 RememberedSet *rset;
6065 ARCH_THREAD_TYPE id = ARCH_GET_THREAD ();
6067 binary_protocol_thread_unregister ((gpointer)id);
6069 hash = HASH_PTHREAD_T (id) % THREAD_HASH_SIZE;
6070 p = thread_table [hash];
6072 DEBUG (3, fprintf (gc_debug_file, "unregister thread %p (%p)\n", p, (gpointer)p->id));
6073 while (!ARCH_THREAD_EQUALS (p->id, id)) {
6078 thread_table [hash] = p->next;
6080 prev->next = p->next;
6083 if (freed_thread_remsets) {
6084 for (rset = p->remset; rset->next; rset = rset->next)
6086 rset->next = freed_thread_remsets;
6087 freed_thread_remsets = p->remset;
6089 freed_thread_remsets = p->remset;
6092 if (*p->store_remset_buffer_index_addr)
6093 add_generic_store_remset_from_buffer (*p->store_remset_buffer_addr);
6094 free_internal_mem (*p->store_remset_buffer_addr, INTERNAL_MEM_STORE_REMSET);
6099 unregister_thread (void *k)
6101 g_assert (!mono_domain_get ());
6103 unregister_current_thread ();
6108 mono_gc_register_thread (void *baseptr)
6110 SgenThreadInfo *info;
6114 info = mono_sgen_thread_info_lookup (ARCH_GET_THREAD ());
6116 info = gc_register_current_thread (baseptr);
6118 return info != NULL;
6121 #if USE_PTHREAD_INTERCEPT
6123 #undef pthread_create
6125 #undef pthread_detach
6128 void *(*start_routine) (void *);
6131 MonoSemType registered;
6132 } SgenThreadStartInfo;
6135 gc_start_thread (void *arg)
6137 SgenThreadStartInfo *start_info = arg;
6138 SgenThreadInfo* info;
6139 void *t_arg = start_info->arg;
6140 void *(*start_func) (void*) = start_info->start_routine;
6145 info = gc_register_current_thread (&result);
6147 post_result = MONO_SEM_POST (&(start_info->registered));
6148 g_assert (!post_result);
6149 result = start_func (t_arg);
6150 g_assert (!mono_domain_get ());
6152 * this is done by the pthread key dtor
6154 unregister_current_thread ();
6162 mono_gc_pthread_create (pthread_t *new_thread, const pthread_attr_t *attr, void *(*start_routine)(void *), void *arg)
6164 SgenThreadStartInfo *start_info;
6167 start_info = malloc (sizeof (SgenThreadStartInfo));
6170 result = MONO_SEM_INIT (&(start_info->registered), 0);
6172 start_info->arg = arg;
6173 start_info->start_routine = start_routine;
6175 result = pthread_create (new_thread, attr, gc_start_thread, start_info);
6177 while (MONO_SEM_WAIT (&(start_info->registered)) != 0) {
6178 /*if (EINTR != errno) ABORT("sem_wait failed"); */
6181 MONO_SEM_DESTROY (&(start_info->registered));
6187 mono_gc_pthread_join (pthread_t thread, void **retval)
6189 return pthread_join (thread, retval);
6193 mono_gc_pthread_detach (pthread_t thread)
6195 return pthread_detach (thread);
6198 #endif /* USE_PTHREAD_INTERCEPT */
6201 * ######################################################################
6202 * ######## Write barriers
6203 * ######################################################################
6206 static RememberedSet*
6207 alloc_remset (int size, gpointer id) {
6208 RememberedSet* res = get_internal_mem (sizeof (RememberedSet) + (size * sizeof (gpointer)), INTERNAL_MEM_REMSET);
6209 res->store_next = res->data;
6210 res->end_set = res->data + size;
6212 DEBUG (4, fprintf (gc_debug_file, "Allocated remset size %d at %p for %p\n", size, res->data, id));
6217 * Note: the write barriers first do the needed GC work and then do the actual store:
6218 * this way the value is visible to the conservative GC scan after the write barrier
6219 * itself. If a GC interrupts the barrier in the middle, value will be kept alive by
6220 * the conservative scan, otherwise by the remembered set scan.
6223 mono_gc_wbarrier_set_field (MonoObject *obj, gpointer field_ptr, MonoObject* value)
6227 HEAVY_STAT (++stat_wbarrier_set_field);
6228 if (ptr_in_nursery (field_ptr)) {
6229 *(void**)field_ptr = value;
6232 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", field_ptr));
6234 rs = REMEMBERED_SET;
6235 if (rs->store_next < rs->end_set) {
6236 *(rs->store_next++) = (mword)field_ptr;
6237 *(void**)field_ptr = value;
6241 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6242 rs->next = REMEMBERED_SET;
6243 REMEMBERED_SET = rs;
6244 #ifdef HAVE_KW_THREAD
6245 mono_sgen_thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6247 *(rs->store_next++) = (mword)field_ptr;
6248 *(void**)field_ptr = value;
6253 mono_gc_wbarrier_set_arrayref (MonoArray *arr, gpointer slot_ptr, MonoObject* value)
6257 HEAVY_STAT (++stat_wbarrier_set_arrayref);
6258 if (ptr_in_nursery (slot_ptr)) {
6259 *(void**)slot_ptr = value;
6262 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", slot_ptr));
6264 rs = REMEMBERED_SET;
6265 if (rs->store_next < rs->end_set) {
6266 *(rs->store_next++) = (mword)slot_ptr;
6267 *(void**)slot_ptr = value;
6271 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6272 rs->next = REMEMBERED_SET;
6273 REMEMBERED_SET = rs;
6274 #ifdef HAVE_KW_THREAD
6275 mono_sgen_thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6277 *(rs->store_next++) = (mword)slot_ptr;
6278 *(void**)slot_ptr = value;
6283 mono_gc_wbarrier_arrayref_copy (gpointer dest_ptr, gpointer src_ptr, int count)
6287 HEAVY_STAT (++stat_wbarrier_arrayref_copy);
6289 memmove (dest_ptr, src_ptr, count * sizeof (gpointer));
6290 if (ptr_in_nursery (dest_ptr)) {
6294 rs = REMEMBERED_SET;
6295 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p, %d\n", dest_ptr, count));
6296 if (rs->store_next + 1 < rs->end_set) {
6297 *(rs->store_next++) = (mword)dest_ptr | REMSET_RANGE;
6298 *(rs->store_next++) = count;
6302 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6303 rs->next = REMEMBERED_SET;
6304 REMEMBERED_SET = rs;
6305 #ifdef HAVE_KW_THREAD
6306 mono_sgen_thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6308 *(rs->store_next++) = (mword)dest_ptr | REMSET_RANGE;
6309 *(rs->store_next++) = count;
6313 static char *found_obj;
6316 find_object_for_ptr_callback (char *obj, size_t size, char *ptr)
6318 if (ptr >= obj && ptr < obj + size) {
6319 g_assert (!found_obj);
6324 /* for use in the debugger */
6325 char* find_object_for_ptr (char *ptr);
6327 find_object_for_ptr (char *ptr)
6331 if (ptr >= nursery_section->data && ptr < nursery_section->end_data) {
6333 scan_area_with_callback (nursery_section->data, nursery_section->end_data,
6334 (IterateObjectCallbackFunc)find_object_for_ptr_callback, ptr);
6339 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
6340 if (ptr >= bigobj->data && ptr < bigobj->data + bigobj->size)
6341 return bigobj->data;
6345 * Very inefficient, but this is debugging code, supposed to
6346 * be called from gdb, so we don't care.
6349 major_iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)find_object_for_ptr_callback, ptr);
6354 evacuate_remset_buffer (void)
6359 buffer = STORE_REMSET_BUFFER;
6361 add_generic_store_remset_from_buffer (buffer);
6362 memset (buffer, 0, sizeof (gpointer) * STORE_REMSET_BUFFER_SIZE);
6364 STORE_REMSET_BUFFER_INDEX = 0;
6368 mono_gc_wbarrier_generic_nostore (gpointer ptr)
6374 HEAVY_STAT (++stat_wbarrier_generic_store);
6376 #ifdef XDOMAIN_CHECKS_IN_WBARRIER
6377 /* FIXME: ptr_in_heap must be called with the GC lock held */
6378 if (xdomain_checks && *(MonoObject**)ptr && ptr_in_heap (ptr)) {
6379 char *start = find_object_for_ptr (ptr);
6380 MonoObject *value = *(MonoObject**)ptr;
6384 MonoObject *obj = (MonoObject*)start;
6385 if (obj->vtable->domain != value->vtable->domain)
6386 g_assert (is_xdomain_ref_allowed (ptr, start, obj->vtable->domain));
6394 if (*(gpointer*)ptr)
6395 binary_protocol_wbarrier (ptr, *(gpointer*)ptr, (gpointer)LOAD_VTABLE (*(gpointer*)ptr));
6397 if (ptr_in_nursery (ptr) || ptr_on_stack (ptr) || !ptr_in_nursery (*(gpointer*)ptr)) {
6398 DEBUG (8, fprintf (gc_debug_file, "Skipping remset at %p\n", ptr));
6403 buffer = STORE_REMSET_BUFFER;
6404 index = STORE_REMSET_BUFFER_INDEX;
6405 /* This simple optimization eliminates a sizable portion of
6406 entries. Comparing it to the last but one entry as well
6407 doesn't eliminate significantly more entries. */
6408 if (buffer [index] == ptr) {
6413 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", ptr));
6414 HEAVY_STAT (++stat_wbarrier_generic_store_remset);
6417 if (index >= STORE_REMSET_BUFFER_SIZE) {
6418 evacuate_remset_buffer ();
6419 index = STORE_REMSET_BUFFER_INDEX;
6420 g_assert (index == 0);
6423 buffer [index] = ptr;
6424 STORE_REMSET_BUFFER_INDEX = index;
6430 mono_gc_wbarrier_generic_store (gpointer ptr, MonoObject* value)
6432 DEBUG (8, fprintf (gc_debug_file, "Wbarrier store at %p to %p (%s)\n", ptr, value, value ? safe_name (value) : "null"));
6433 *(void**)ptr = value;
6434 if (ptr_in_nursery (value))
6435 mono_gc_wbarrier_generic_nostore (ptr);
6438 void mono_gc_wbarrier_value_copy_bitmap (gpointer _dest, gpointer _src, int size, unsigned bitmap)
6440 mword *dest = _dest;
6445 mono_gc_wbarrier_generic_store (dest, (MonoObject*)*src);
6450 size -= SIZEOF_VOID_P;
6457 mono_gc_wbarrier_value_copy (gpointer dest, gpointer src, int count, MonoClass *klass)
6461 HEAVY_STAT (++stat_wbarrier_value_copy);
6462 g_assert (klass->valuetype);
6464 memmove (dest, src, count * mono_class_value_size (klass, NULL));
6465 rs = REMEMBERED_SET;
6466 if (ptr_in_nursery (dest) || ptr_on_stack (dest) || !klass->has_references) {
6470 g_assert (klass->gc_descr_inited);
6471 DEBUG (8, fprintf (gc_debug_file, "Adding value remset at %p, count %d, descr %p for class %s (%p)\n", dest, count, klass->gc_descr, klass->name, klass));
6473 if (rs->store_next + 3 < rs->end_set) {
6474 *(rs->store_next++) = (mword)dest | REMSET_VTYPE;
6475 *(rs->store_next++) = (mword)klass->gc_descr;
6476 *(rs->store_next++) = (mword)count;
6480 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6481 rs->next = REMEMBERED_SET;
6482 REMEMBERED_SET = rs;
6483 #ifdef HAVE_KW_THREAD
6484 mono_sgen_thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6486 *(rs->store_next++) = (mword)dest | REMSET_VTYPE;
6487 *(rs->store_next++) = (mword)klass->gc_descr;
6488 *(rs->store_next++) = (mword)count;
6493 * mono_gc_wbarrier_object_copy:
6495 * Write barrier to call when obj is the result of a clone or copy of an object.
6498 mono_gc_wbarrier_object_copy (MonoObject* obj, MonoObject *src)
6504 HEAVY_STAT (++stat_wbarrier_object_copy);
6505 rs = REMEMBERED_SET;
6506 DEBUG (6, fprintf (gc_debug_file, "Adding object remset for %p\n", obj));
6507 size = mono_object_class (obj)->instance_size;
6509 /* do not copy the sync state */
6510 memcpy ((char*)obj + sizeof (MonoObject), (char*)src + sizeof (MonoObject),
6511 size - sizeof (MonoObject));
6512 if (ptr_in_nursery (obj) || ptr_on_stack (obj)) {
6516 if (rs->store_next < rs->end_set) {
6517 *(rs->store_next++) = (mword)obj | REMSET_OBJECT;
6521 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6522 rs->next = REMEMBERED_SET;
6523 REMEMBERED_SET = rs;
6524 #ifdef HAVE_KW_THREAD
6525 mono_sgen_thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6527 *(rs->store_next++) = (mword)obj | REMSET_OBJECT;
6532 * ######################################################################
6533 * ######## Collector debugging
6534 * ######################################################################
6537 const char*descriptor_types [] = {
6549 describe_ptr (char *ptr)
6555 if (ptr_in_nursery (ptr)) {
6556 printf ("Pointer inside nursery.\n");
6558 if (major_ptr_is_in_non_pinned_space (ptr)) {
6559 printf ("Pointer inside oldspace.\n");
6560 } else if (obj_is_from_pinned_alloc (ptr)) {
6561 printf ("Pointer is inside a pinned chunk.\n");
6563 printf ("Pointer unknown.\n");
6568 if (object_is_pinned (ptr))
6569 printf ("Object is pinned.\n");
6571 if (object_is_forwarded (ptr))
6572 printf ("Object is forwared.\n");
6574 // FIXME: Handle pointers to the inside of objects
6575 vtable = (MonoVTable*)LOAD_VTABLE (ptr);
6577 printf ("VTable: %p\n", vtable);
6578 if (vtable == NULL) {
6579 printf ("VTable is invalid (empty).\n");
6582 if (ptr_in_nursery (vtable)) {
6583 printf ("VTable is invalid (points inside nursery).\n");
6586 printf ("Class: %s\n", vtable->klass->name);
6588 desc = ((GCVTable*)vtable)->desc;
6589 printf ("Descriptor: %lx\n", (long)desc);
6592 printf ("Descriptor type: %d (%s)\n", type, descriptor_types [type]);
6596 find_in_remset_loc (mword *p, char *addr, gboolean *found)
6602 switch ((*p) & REMSET_TYPE_MASK) {
6603 case REMSET_LOCATION:
6604 if (*p == (mword)addr)
6608 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
6610 if ((void**)addr >= ptr && (void**)addr < ptr + count)
6614 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
6615 count = safe_object_get_size ((MonoObject*)ptr);
6616 count += (ALLOC_ALIGN - 1);
6617 count &= (ALLOC_ALIGN - 1);
6618 count /= sizeof (mword);
6619 if ((void**)addr >= ptr && (void**)addr < ptr + count)
6623 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
6627 switch (desc & 0x7) {
6628 case DESC_TYPE_RUN_LENGTH:
6629 OBJ_RUN_LEN_SIZE (skip_size, desc, ptr);
6631 case DESC_TYPE_SMALL_BITMAP:
6632 OBJ_BITMAP_SIZE (skip_size, desc, start);
6636 g_assert_not_reached ();
6639 /* The descriptor includes the size of MonoObject */
6640 skip_size -= sizeof (MonoObject);
6642 if ((void**)addr >= ptr && (void**)addr < ptr + (skip_size / sizeof (gpointer)))
6647 g_assert_not_reached ();
6653 * Return whenever ADDR occurs in the remembered sets
6656 find_in_remsets (char *addr)
6659 SgenThreadInfo *info;
6660 RememberedSet *remset;
6661 GenericStoreRememberedSet *store_remset;
6663 gboolean found = FALSE;
6665 /* the global one */
6666 for (remset = global_remset; remset; remset = remset->next) {
6667 DEBUG (4, fprintf (gc_debug_file, "Scanning global remset range: %p-%p, size: %td\n", remset->data, remset->store_next, remset->store_next - remset->data));
6668 for (p = remset->data; p < remset->store_next;) {
6669 p = find_in_remset_loc (p, addr, &found);
6675 /* the generic store ones */
6676 for (store_remset = generic_store_remsets; store_remset; store_remset = store_remset->next) {
6677 for (i = 0; i < STORE_REMSET_BUFFER_SIZE - 1; ++i) {
6678 if (store_remset->data [i] == addr)
6683 /* the per-thread ones */
6684 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
6685 for (info = thread_table [i]; info; info = info->next) {
6687 for (remset = info->remset; remset; remset = remset->next) {
6688 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for thread %p, range: %p-%p, size: %td\n", info, remset->data, remset->store_next, remset->store_next - remset->data));
6689 for (p = remset->data; p < remset->store_next;) {
6690 p = find_in_remset_loc (p, addr, &found);
6695 for (j = 0; j < *info->store_remset_buffer_index_addr; ++j) {
6696 if ((*info->store_remset_buffer_addr) [j + 1] == addr)
6702 /* the freed thread ones */
6703 for (remset = freed_thread_remsets; remset; remset = remset->next) {
6704 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for freed thread, range: %p-%p, size: %td\n", remset->data, remset->store_next, remset->store_next - remset->data));
6705 for (p = remset->data; p < remset->store_next;) {
6706 p = find_in_remset_loc (p, addr, &found);
6715 static gboolean missing_remsets;
6718 * We let a missing remset slide if the target object is pinned,
6719 * because the store might have happened but the remset not yet added,
6720 * but in that case the target must be pinned. We might theoretically
6721 * miss some missing remsets this way, but it's very unlikely.
6724 #define HANDLE_PTR(ptr,obj) do { \
6725 if (*(ptr) && (char*)*(ptr) >= nursery_start && (char*)*(ptr) < nursery_next) { \
6726 if (!find_in_remsets ((char*)(ptr))) { \
6727 fprintf (gc_debug_file, "Oldspace->newspace reference %p at offset %td in object %p (%s.%s) not found in remsets.\n", *(ptr), (char*)(ptr) - (char*)(obj), (obj), ((MonoObject*)(obj))->vtable->klass->name_space, ((MonoObject*)(obj))->vtable->klass->name); \
6728 binary_protocol_missing_remset ((obj), (gpointer)LOAD_VTABLE ((obj)), (char*)(ptr) - (char*)(obj), *(ptr), (gpointer)LOAD_VTABLE(*(ptr)), object_is_pinned (*(ptr))); \
6729 if (!object_is_pinned (*(ptr))) \
6730 missing_remsets = TRUE; \
6736 * Check that each object reference which points into the nursery can
6737 * be found in the remembered sets.
6740 check_consistency_callback (char *start, size_t size, void *dummy)
6742 GCVTable *vt = (GCVTable*)LOAD_VTABLE (start);
6743 DEBUG (8, fprintf (gc_debug_file, "Scanning object %p, vtable: %p (%s)\n", start, vt, vt->klass->name));
6745 #define SCAN_OBJECT_ACTION
6746 #include "sgen-scan-object.h"
6750 * Perform consistency check of the heap.
6752 * Assumes the world is stopped.
6755 check_consistency (void)
6759 // Need to add more checks
6761 missing_remsets = FALSE;
6763 DEBUG (1, fprintf (gc_debug_file, "Begin heap consistency check...\n"));
6765 // Check that oldspace->newspace pointers are registered with the collector
6766 major_iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)check_consistency_callback, NULL);
6768 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
6769 check_consistency_callback (bigobj->data, bigobj->size, NULL);
6771 DEBUG (1, fprintf (gc_debug_file, "Heap consistency check done.\n"));
6773 #ifdef BINARY_PROTOCOL
6774 if (!binary_protocol_file)
6776 g_assert (!missing_remsets);
6781 #define HANDLE_PTR(ptr,obj) do { \
6783 g_assert (LOAD_VTABLE (*(ptr))); \
6787 check_major_refs_callback (char *start, size_t size, void *dummy)
6789 #define SCAN_OBJECT_ACTION
6790 #include "sgen-scan-object.h"
6794 check_major_refs (void)
6798 major_iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)check_major_refs_callback, NULL);
6800 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
6801 check_major_refs_callback (bigobj->data, bigobj->size, NULL);
6804 /* Check that the reference is valid */
6806 #define HANDLE_PTR(ptr,obj) do { \
6808 g_assert (safe_name (*(ptr)) != NULL); \
6815 * Perform consistency check on an object. Currently we only check that the
6816 * reference fields are valid.
6819 check_object (char *start)
6824 #include "sgen-scan-object.h"
6828 * ######################################################################
6829 * ######## Other mono public interface functions.
6830 * ######################################################################
6834 mono_gc_collect (int generation)
6838 if (generation == 0) {
6839 collect_nursery (0);
6841 major_collection ("user request");
6848 mono_gc_max_generation (void)
6854 mono_gc_collection_count (int generation)
6856 if (generation == 0)
6857 return num_minor_gcs;
6858 return num_major_gcs;
6862 mono_gc_get_used_size (void)
6866 tot = los_memory_usage;
6867 tot += nursery_section->next_data - nursery_section->data;
6868 tot += major_get_used_size ();
6869 /* FIXME: account for pinned objects */
6875 mono_gc_get_heap_size (void)
6881 mono_gc_disable (void)
6889 mono_gc_enable (void)
6897 mono_gc_get_los_limit (void)
6899 return MAX_SMALL_OBJ_SIZE;
6903 mono_object_is_alive (MonoObject* o)
6909 mono_gc_get_generation (MonoObject *obj)
6911 if (ptr_in_nursery (obj))
6917 mono_gc_enable_events (void)
6922 mono_gc_weak_link_add (void **link_addr, MonoObject *obj, gboolean track)
6925 mono_gc_register_disappearing_link (obj, link_addr, track);
6930 mono_gc_weak_link_remove (void **link_addr)
6933 mono_gc_register_disappearing_link (NULL, link_addr, FALSE);
6938 mono_gc_weak_link_get (void **link_addr)
6942 return (MonoObject*) REVEAL_POINTER (*link_addr);
6946 mono_gc_ephemeron_array_add (MonoObject *obj)
6948 EphemeronLinkNode *node;
6952 node = get_internal_mem (sizeof (EphemeronLinkNode), INTERNAL_MEM_EPHEMERON_LINK);
6957 node->array = (char*)obj;
6958 node->next = ephemeron_list;
6959 ephemeron_list = node;
6961 DEBUG (5, fprintf (gc_debug_file, "Registered ephemeron array %p\n", obj));
6968 mono_gc_make_descr_from_bitmap (gsize *bitmap, int numbits)
6970 if (numbits < ((sizeof (*bitmap) * 8) - ROOT_DESC_TYPE_SHIFT)) {
6971 return (void*)MAKE_ROOT_DESC (ROOT_DESC_BITMAP, bitmap [0]);
6973 mword complex = alloc_complex_descriptor (bitmap, numbits);
6974 return (void*)MAKE_ROOT_DESC (ROOT_DESC_COMPLEX, complex);
6979 mono_gc_make_root_descr_user (MonoGCRootMarkFunc marker)
6983 g_assert (user_descriptors_next < MAX_USER_DESCRIPTORS);
6984 descr = (void*)MAKE_ROOT_DESC (ROOT_DESC_USER, (mword)user_descriptors_next);
6985 user_descriptors [user_descriptors_next ++] = marker;
6991 mono_gc_alloc_fixed (size_t size, void *descr)
6993 /* FIXME: do a single allocation */
6994 void *res = calloc (1, size);
6997 if (!mono_gc_register_root (res, size, descr)) {
7005 mono_gc_free_fixed (void* addr)
7007 mono_gc_deregister_root (addr);
7012 mono_gc_invoke_with_gc_lock (MonoGCLockedCallbackFunc func, void *data)
7016 result = func (data);
7017 UNLOCK_INTERRUPTION;
7022 mono_gc_is_gc_thread (void)
7026 result = mono_sgen_thread_info_lookup (ARCH_GET_THREAD ()) != NULL;
7033 /* Tries to extract a number from the passed string, taking in to account m, k
7036 parse_environment_string_extract_number (gchar *str, glong *out)
7039 int len = strlen (str), shift = 0;
7041 gboolean is_suffix = FALSE;
7044 switch (str [len - 1]) {
7055 suffix = str [len - 1];
7060 val = strtol (str, &endptr, 10);
7062 if ((errno == ERANGE && (val == LONG_MAX || val == LONG_MIN))
7063 || (errno != 0 && val == 0) || (endptr == str))
7067 if (*(endptr + 1)) /* Invalid string. */
7079 mono_gc_base_init (void)
7083 struct sigaction sinfo;
7085 LOCK_INIT (gc_mutex);
7087 if (gc_initialized) {
7091 pagesize = mono_pagesize ();
7092 gc_debug_file = stderr;
7096 if ((env = getenv ("MONO_GC_PARAMS"))) {
7097 if (g_str_has_prefix (env, "nursery-size")) {
7100 while (env [index] && env [index++] != '=')
7102 if (env [index] && parse_environment_string_extract_number (env
7104 default_nursery_size = val;
7105 #ifdef ALIGN_NURSERY
7106 if ((val & (val - 1))) {
7107 fprintf (stderr, "The nursery size must be a power of two.\n");
7111 default_nursery_bits = 0;
7112 while (1 << (++ default_nursery_bits) != default_nursery_size)
7116 fprintf (stderr, "nursery-size must be an integer.\n");
7120 fprintf (stderr, "MONO_GC_PARAMS must be of the form 'nursery-size=N' (where N is an integer, possibly with a k, m or a g suffix).\n");
7127 nursery_size = DEFAULT_NURSERY_SIZE;
7128 minor_collection_allowance = MIN_MINOR_COLLECTION_ALLOWANCE;
7132 if ((env = getenv ("MONO_GC_DEBUG"))) {
7133 opts = g_strsplit (env, ",", -1);
7134 for (ptr = opts; ptr && *ptr; ptr ++) {
7136 if (opt [0] >= '0' && opt [0] <= '9') {
7137 gc_debug_level = atoi (opt);
7142 char *rf = g_strdup_printf ("%s.%d", opt, getpid ());
7143 gc_debug_file = fopen (rf, "wb");
7145 gc_debug_file = stderr;
7148 } else if (!strcmp (opt, "collect-before-allocs")) {
7149 collect_before_allocs = TRUE;
7150 } else if (!strcmp (opt, "check-at-minor-collections")) {
7151 consistency_check_at_minor_collection = TRUE;
7152 nursery_clear_policy = CLEAR_AT_GC;
7153 } else if (!strcmp (opt, "xdomain-checks")) {
7154 xdomain_checks = TRUE;
7155 } else if (!strcmp (opt, "clear-at-gc")) {
7156 nursery_clear_policy = CLEAR_AT_GC;
7157 } else if (!strcmp (opt, "conservative-stack-mark")) {
7158 conservative_stack_mark = TRUE;
7159 } else if (!strcmp (opt, "check-scan-starts")) {
7160 do_scan_starts_check = TRUE;
7161 } else if (g_str_has_prefix (opt, "heap-dump=")) {
7162 char *filename = strchr (opt, '=') + 1;
7163 nursery_clear_policy = CLEAR_AT_GC;
7164 heap_dump_file = fopen (filename, "w");
7166 fprintf (heap_dump_file, "<sgen-dump>\n");
7167 #ifdef BINARY_PROTOCOL
7168 } else if (g_str_has_prefix (opt, "binary-protocol=")) {
7169 char *filename = strchr (opt, '=') + 1;
7170 binary_protocol_file = fopen (filename, "w");
7173 fprintf (stderr, "Invalid format for the MONO_GC_DEBUG env variable: '%s'\n", env);
7174 fprintf (stderr, "The format is: MONO_GC_DEBUG=[l[:filename]|<option>]+ where l is a debug level 0-9.\n");
7175 fprintf (stderr, "Valid options are: collect-before-allocs, check-at-minor-collections, xdomain-checks, clear-at-gc.\n");
7182 suspend_ack_semaphore_ptr = &suspend_ack_semaphore;
7183 MONO_SEM_INIT (&suspend_ack_semaphore, 0);
7185 sigfillset (&sinfo.sa_mask);
7186 sinfo.sa_flags = SA_RESTART | SA_SIGINFO;
7187 sinfo.sa_sigaction = suspend_handler;
7188 if (sigaction (suspend_signal_num, &sinfo, NULL) != 0) {
7189 g_error ("failed sigaction");
7192 sinfo.sa_handler = restart_handler;
7193 if (sigaction (restart_signal_num, &sinfo, NULL) != 0) {
7194 g_error ("failed sigaction");
7197 sigfillset (&suspend_signal_mask);
7198 sigdelset (&suspend_signal_mask, restart_signal_num);
7200 global_remset = alloc_remset (1024, NULL);
7201 global_remset->next = NULL;
7203 pthread_key_create (&remembered_set_key, unregister_thread);
7205 #ifndef HAVE_KW_THREAD
7206 pthread_key_create (&thread_info_key, NULL);
7209 gc_initialized = TRUE;
7211 mono_gc_register_thread (&sinfo);
7215 mono_gc_get_suspend_signal (void)
7217 return suspend_signal_num;
7227 #ifdef HAVE_KW_THREAD
7228 #define EMIT_TLS_ACCESS(mb,dummy,offset) do { \
7229 mono_mb_emit_byte ((mb), MONO_CUSTOM_PREFIX); \
7230 mono_mb_emit_byte ((mb), CEE_MONO_TLS); \
7231 mono_mb_emit_i4 ((mb), (offset)); \
7234 #define EMIT_TLS_ACCESS(mb,member,dummy) do { \
7235 mono_mb_emit_byte ((mb), MONO_CUSTOM_PREFIX); \
7236 mono_mb_emit_byte ((mb), CEE_MONO_TLS); \
7237 mono_mb_emit_i4 ((mb), thread_info_key); \
7238 mono_mb_emit_icon ((mb), G_STRUCT_OFFSET (SgenThreadInfo, member)); \
7239 mono_mb_emit_byte ((mb), CEE_ADD); \
7240 mono_mb_emit_byte ((mb), CEE_LDIND_I); \
7244 #ifdef MANAGED_ALLOCATION
7245 /* FIXME: Do this in the JIT, where specialized allocation sequences can be created
7246 * for each class. This is currently not easy to do, as it is hard to generate basic
7247 * blocks + branches, but it is easy with the linear IL codebase.
7249 * For this to work we'd need to solve the TLAB race, first. Now we
7250 * require the allocator to be in a few known methods to make sure
7251 * that they are executed atomically via the restart mechanism.
7254 create_allocator (int atype)
7256 int p_var, size_var;
7257 guint32 slowpath_branch, max_size_branch;
7258 MonoMethodBuilder *mb;
7260 MonoMethodSignature *csig;
7261 static gboolean registered = FALSE;
7262 int tlab_next_addr_var, new_next_var;
7264 const char *name = NULL;
7265 AllocatorWrapperInfo *info;
7267 #ifdef HAVE_KW_THREAD
7268 int tlab_next_addr_offset = -1;
7269 int tlab_temp_end_offset = -1;
7271 MONO_THREAD_VAR_OFFSET (tlab_next_addr, tlab_next_addr_offset);
7272 MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
7274 g_assert (tlab_next_addr_offset != -1);
7275 g_assert (tlab_temp_end_offset != -1);
7279 mono_register_jit_icall (mono_gc_alloc_obj, "mono_gc_alloc_obj", mono_create_icall_signature ("object ptr int"), FALSE);
7280 mono_register_jit_icall (mono_gc_alloc_vector, "mono_gc_alloc_vector", mono_create_icall_signature ("object ptr int int"), FALSE);
7284 if (atype == ATYPE_SMALL) {
7286 name = "AllocSmall";
7287 } else if (atype == ATYPE_NORMAL) {
7290 } else if (atype == ATYPE_VECTOR) {
7292 name = "AllocVector";
7294 g_assert_not_reached ();
7297 csig = mono_metadata_signature_alloc (mono_defaults.corlib, num_params);
7298 csig->ret = &mono_defaults.object_class->byval_arg;
7299 for (i = 0; i < num_params; ++i)
7300 csig->params [i] = &mono_defaults.int_class->byval_arg;
7302 mb = mono_mb_new (mono_defaults.object_class, name, MONO_WRAPPER_ALLOC);
7303 size_var = mono_mb_add_local (mb, &mono_defaults.int32_class->byval_arg);
7304 if (atype == ATYPE_NORMAL || atype == ATYPE_SMALL) {
7305 /* size = vtable->klass->instance_size; */
7306 mono_mb_emit_ldarg (mb, 0);
7307 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoVTable, klass));
7308 mono_mb_emit_byte (mb, CEE_ADD);
7309 mono_mb_emit_byte (mb, CEE_LDIND_I);
7310 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoClass, instance_size));
7311 mono_mb_emit_byte (mb, CEE_ADD);
7312 /* FIXME: assert instance_size stays a 4 byte integer */
7313 mono_mb_emit_byte (mb, CEE_LDIND_U4);
7314 mono_mb_emit_stloc (mb, size_var);
7315 } else if (atype == ATYPE_VECTOR) {
7316 MonoExceptionClause *clause;
7318 MonoClass *oom_exc_class;
7321 /* n > MONO_ARRAY_MAX_INDEX -> OverflowException */
7322 mono_mb_emit_ldarg (mb, 1);
7323 mono_mb_emit_icon (mb, MONO_ARRAY_MAX_INDEX);
7324 pos = mono_mb_emit_short_branch (mb, CEE_BLE_UN_S);
7325 mono_mb_emit_exception (mb, "OverflowException", NULL);
7326 mono_mb_patch_short_branch (mb, pos);
7328 clause = mono_image_alloc0 (mono_defaults.corlib, sizeof (MonoExceptionClause));
7329 clause->try_offset = mono_mb_get_label (mb);
7331 /* vtable->klass->sizes.element_size */
7332 mono_mb_emit_ldarg (mb, 0);
7333 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoVTable, klass));
7334 mono_mb_emit_byte (mb, CEE_ADD);
7335 mono_mb_emit_byte (mb, CEE_LDIND_I);
7336 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoClass, sizes.element_size));
7337 mono_mb_emit_byte (mb, CEE_ADD);
7338 mono_mb_emit_byte (mb, CEE_LDIND_U4);
7341 mono_mb_emit_ldarg (mb, 1);
7342 mono_mb_emit_byte (mb, CEE_MUL_OVF_UN);
7343 /* + sizeof (MonoArray) */
7344 mono_mb_emit_icon (mb, sizeof (MonoArray));
7345 mono_mb_emit_byte (mb, CEE_ADD_OVF_UN);
7346 mono_mb_emit_stloc (mb, size_var);
7348 pos_leave = mono_mb_emit_branch (mb, CEE_LEAVE);
7351 clause->flags = MONO_EXCEPTION_CLAUSE_NONE;
7352 clause->try_len = mono_mb_get_pos (mb) - clause->try_offset;
7353 clause->data.catch_class = mono_class_from_name (mono_defaults.corlib,
7354 "System", "OverflowException");
7355 g_assert (clause->data.catch_class);
7356 clause->handler_offset = mono_mb_get_label (mb);
7358 oom_exc_class = mono_class_from_name (mono_defaults.corlib,
7359 "System", "OutOfMemoryException");
7360 g_assert (oom_exc_class);
7361 ctor = mono_class_get_method_from_name (oom_exc_class, ".ctor", 0);
7364 mono_mb_emit_byte (mb, CEE_POP);
7365 mono_mb_emit_op (mb, CEE_NEWOBJ, ctor);
7366 mono_mb_emit_byte (mb, CEE_THROW);
7368 clause->handler_len = mono_mb_get_pos (mb) - clause->handler_offset;
7369 mono_mb_set_clauses (mb, 1, clause);
7370 mono_mb_patch_branch (mb, pos_leave);
7373 g_assert_not_reached ();
7376 /* size += ALLOC_ALIGN - 1; */
7377 mono_mb_emit_ldloc (mb, size_var);
7378 mono_mb_emit_icon (mb, ALLOC_ALIGN - 1);
7379 mono_mb_emit_byte (mb, CEE_ADD);
7380 /* size &= ~(ALLOC_ALIGN - 1); */
7381 mono_mb_emit_icon (mb, ~(ALLOC_ALIGN - 1));
7382 mono_mb_emit_byte (mb, CEE_AND);
7383 mono_mb_emit_stloc (mb, size_var);
7385 /* if (size > MAX_SMALL_OBJ_SIZE) goto slowpath */
7386 if (atype != ATYPE_SMALL) {
7387 mono_mb_emit_ldloc (mb, size_var);
7388 mono_mb_emit_icon (mb, MAX_SMALL_OBJ_SIZE);
7389 max_size_branch = mono_mb_emit_short_branch (mb, MONO_CEE_BGT_S);
7393 * We need to modify tlab_next, but the JIT only supports reading, so we read
7394 * another tls var holding its address instead.
7397 /* tlab_next_addr (local) = tlab_next_addr (TLS var) */
7398 tlab_next_addr_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7399 EMIT_TLS_ACCESS (mb, tlab_next_addr, tlab_next_addr_offset);
7400 mono_mb_emit_stloc (mb, tlab_next_addr_var);
7402 /* p = (void**)tlab_next; */
7403 p_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7404 mono_mb_emit_ldloc (mb, tlab_next_addr_var);
7405 mono_mb_emit_byte (mb, CEE_LDIND_I);
7406 mono_mb_emit_stloc (mb, p_var);
7408 /* new_next = (char*)p + size; */
7409 new_next_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7410 mono_mb_emit_ldloc (mb, p_var);
7411 mono_mb_emit_ldloc (mb, size_var);
7412 mono_mb_emit_byte (mb, CEE_CONV_I);
7413 mono_mb_emit_byte (mb, CEE_ADD);
7414 mono_mb_emit_stloc (mb, new_next_var);
7416 /* tlab_next = new_next */
7417 mono_mb_emit_ldloc (mb, tlab_next_addr_var);
7418 mono_mb_emit_ldloc (mb, new_next_var);
7419 mono_mb_emit_byte (mb, CEE_STIND_I);
7421 /* if (G_LIKELY (new_next < tlab_temp_end)) */
7422 mono_mb_emit_ldloc (mb, new_next_var);
7423 EMIT_TLS_ACCESS (mb, tlab_temp_end, tlab_temp_end_offset);
7424 slowpath_branch = mono_mb_emit_short_branch (mb, MONO_CEE_BLT_UN_S);
7427 if (atype != ATYPE_SMALL)
7428 mono_mb_patch_short_branch (mb, max_size_branch);
7430 mono_mb_emit_byte (mb, MONO_CUSTOM_PREFIX);
7431 mono_mb_emit_byte (mb, CEE_MONO_NOT_TAKEN);
7433 /* FIXME: mono_gc_alloc_obj takes a 'size_t' as an argument, not an int32 */
7434 mono_mb_emit_ldarg (mb, 0);
7435 mono_mb_emit_ldloc (mb, size_var);
7436 if (atype == ATYPE_NORMAL || atype == ATYPE_SMALL) {
7437 mono_mb_emit_icall (mb, mono_gc_alloc_obj);
7438 } else if (atype == ATYPE_VECTOR) {
7439 mono_mb_emit_ldarg (mb, 1);
7440 mono_mb_emit_icall (mb, mono_gc_alloc_vector);
7442 g_assert_not_reached ();
7444 mono_mb_emit_byte (mb, CEE_RET);
7447 mono_mb_patch_short_branch (mb, slowpath_branch);
7449 /* FIXME: Memory barrier */
7452 mono_mb_emit_ldloc (mb, p_var);
7453 mono_mb_emit_ldarg (mb, 0);
7454 mono_mb_emit_byte (mb, CEE_STIND_I);
7456 if (atype == ATYPE_VECTOR) {
7457 /* arr->max_length = max_length; */
7458 mono_mb_emit_ldloc (mb, p_var);
7459 mono_mb_emit_ldflda (mb, G_STRUCT_OFFSET (MonoArray, max_length));
7460 mono_mb_emit_ldarg (mb, 1);
7461 mono_mb_emit_byte (mb, CEE_STIND_I);
7465 mono_mb_emit_ldloc (mb, p_var);
7466 mono_mb_emit_byte (mb, CEE_RET);
7468 res = mono_mb_create_method (mb, csig, 8);
7470 mono_method_get_header (res)->init_locals = FALSE;
7472 info = mono_image_alloc0 (mono_defaults.corlib, sizeof (AllocatorWrapperInfo));
7473 info->alloc_type = atype;
7474 mono_marshal_set_wrapper_info (res, info);
7480 static MonoMethod* alloc_method_cache [ATYPE_NUM];
7481 static MonoMethod *write_barrier_method;
7484 is_ip_in_managed_allocator (MonoDomain *domain, gpointer ip)
7492 ji = mono_jit_info_table_find (domain, ip);
7495 method = ji->method;
7497 if (method == write_barrier_method)
7499 for (i = 0; i < ATYPE_NUM; ++i)
7500 if (method == alloc_method_cache [i])
7506 * Generate an allocator method implementing the fast path of mono_gc_alloc_obj ().
7507 * The signature of the called method is:
7508 * object allocate (MonoVTable *vtable)
7511 mono_gc_get_managed_allocator (MonoVTable *vtable, gboolean for_box)
7513 #ifdef MANAGED_ALLOCATION
7514 MonoClass *klass = vtable->klass;
7516 #ifdef HAVE_KW_THREAD
7517 int tlab_next_offset = -1;
7518 int tlab_temp_end_offset = -1;
7519 MONO_THREAD_VAR_OFFSET (tlab_next, tlab_next_offset);
7520 MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
7522 if (tlab_next_offset == -1 || tlab_temp_end_offset == -1)
7526 if (!mono_runtime_has_tls_get ())
7528 if (klass->instance_size > tlab_size)
7530 if (klass->has_finalize || klass->marshalbyref || (mono_profiler_get_events () & MONO_PROFILE_ALLOCATIONS))
7534 if (klass->byval_arg.type == MONO_TYPE_STRING)
7536 if (collect_before_allocs)
7539 if (ALIGN_TO (klass->instance_size, ALLOC_ALIGN) < MAX_SMALL_OBJ_SIZE)
7540 return mono_gc_get_managed_allocator_by_type (ATYPE_SMALL);
7542 return mono_gc_get_managed_allocator_by_type (ATYPE_NORMAL);
7549 mono_gc_get_managed_array_allocator (MonoVTable *vtable, int rank)
7551 #ifdef MANAGED_ALLOCATION
7552 MonoClass *klass = vtable->klass;
7554 #ifdef HAVE_KW_THREAD
7555 int tlab_next_offset = -1;
7556 int tlab_temp_end_offset = -1;
7557 MONO_THREAD_VAR_OFFSET (tlab_next, tlab_next_offset);
7558 MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
7560 if (tlab_next_offset == -1 || tlab_temp_end_offset == -1)
7566 if (!mono_runtime_has_tls_get ())
7568 if (mono_profiler_get_events () & MONO_PROFILE_ALLOCATIONS)
7570 if (collect_before_allocs)
7572 g_assert (!klass->has_finalize && !klass->marshalbyref);
7574 return mono_gc_get_managed_allocator_by_type (ATYPE_VECTOR);
7581 mono_gc_get_managed_allocator_by_type (int atype)
7583 #ifdef MANAGED_ALLOCATION
7586 if (!mono_runtime_has_tls_get ())
7589 mono_loader_lock ();
7590 res = alloc_method_cache [atype];
7592 res = alloc_method_cache [atype] = create_allocator (atype);
7593 mono_loader_unlock ();
7601 mono_gc_get_managed_allocator_types (void)
7608 mono_gc_get_write_barrier (void)
7611 MonoMethodBuilder *mb;
7612 MonoMethodSignature *sig;
7613 #ifdef MANAGED_WBARRIER
7614 int label_no_wb_1, label_no_wb_2, label_no_wb_3, label_no_wb_4, label_need_wb, label_slow_path;
7615 #ifndef ALIGN_NURSERY
7616 int label_continue_1, label_continue_2, label_no_wb_5;
7617 int dereferenced_var;
7619 int buffer_var, buffer_index_var, dummy_var;
7621 #ifdef HAVE_KW_THREAD
7622 int stack_end_offset = -1, store_remset_buffer_offset = -1;
7623 int store_remset_buffer_index_offset = -1, store_remset_buffer_index_addr_offset = -1;
7625 MONO_THREAD_VAR_OFFSET (stack_end, stack_end_offset);
7626 g_assert (stack_end_offset != -1);
7627 MONO_THREAD_VAR_OFFSET (store_remset_buffer, store_remset_buffer_offset);
7628 g_assert (store_remset_buffer_offset != -1);
7629 MONO_THREAD_VAR_OFFSET (store_remset_buffer_index, store_remset_buffer_index_offset);
7630 g_assert (store_remset_buffer_index_offset != -1);
7631 MONO_THREAD_VAR_OFFSET (store_remset_buffer_index_addr, store_remset_buffer_index_addr_offset);
7632 g_assert (store_remset_buffer_index_addr_offset != -1);
7636 // FIXME: Maybe create a separate version for ctors (the branch would be
7637 // correctly predicted more times)
7638 if (write_barrier_method)
7639 return write_barrier_method;
7641 /* Create the IL version of mono_gc_barrier_generic_store () */
7642 sig = mono_metadata_signature_alloc (mono_defaults.corlib, 1);
7643 sig->ret = &mono_defaults.void_class->byval_arg;
7644 sig->params [0] = &mono_defaults.int_class->byval_arg;
7646 mb = mono_mb_new (mono_defaults.object_class, "wbarrier", MONO_WRAPPER_WRITE_BARRIER);
7648 #ifdef MANAGED_WBARRIER
7649 if (mono_runtime_has_tls_get ()) {
7650 #ifdef ALIGN_NURSERY
7651 // if (ptr_in_nursery (ptr)) return;
7653 * Masking out the bits might be faster, but we would have to use 64 bit
7654 * immediates, which might be slower.
7656 mono_mb_emit_ldarg (mb, 0);
7657 mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
7658 mono_mb_emit_byte (mb, CEE_SHR_UN);
7659 mono_mb_emit_icon (mb, (mword)nursery_start >> DEFAULT_NURSERY_BITS);
7660 label_no_wb_1 = mono_mb_emit_branch (mb, CEE_BEQ);
7662 // if (!ptr_in_nursery (*ptr)) return;
7663 mono_mb_emit_ldarg (mb, 0);
7664 mono_mb_emit_byte (mb, CEE_LDIND_I);
7665 mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
7666 mono_mb_emit_byte (mb, CEE_SHR_UN);
7667 mono_mb_emit_icon (mb, (mword)nursery_start >> DEFAULT_NURSERY_BITS);
7668 label_no_wb_2 = mono_mb_emit_branch (mb, CEE_BNE_UN);
7671 // if (ptr < (nursery_start)) goto continue;
7672 mono_mb_emit_ldarg (mb, 0);
7673 mono_mb_emit_ptr (mb, (gpointer) nursery_start);
7674 label_continue_1 = mono_mb_emit_branch (mb, CEE_BLT);
7676 // if (ptr >= nursery_real_end)) goto continue;
7677 mono_mb_emit_ldarg (mb, 0);
7678 mono_mb_emit_ptr (mb, (gpointer) nursery_real_end);
7679 label_continue_2 = mono_mb_emit_branch (mb, CEE_BGE);
7682 label_no_wb_1 = mono_mb_emit_branch (mb, CEE_BR);
7685 mono_mb_patch_branch (mb, label_continue_1);
7686 mono_mb_patch_branch (mb, label_continue_2);
7688 // Dereference and store in local var
7689 dereferenced_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7690 mono_mb_emit_ldarg (mb, 0);
7691 mono_mb_emit_byte (mb, CEE_LDIND_I);
7692 mono_mb_emit_stloc (mb, dereferenced_var);
7694 // if (*ptr < nursery_start) return;
7695 mono_mb_emit_ldloc (mb, dereferenced_var);
7696 mono_mb_emit_ptr (mb, (gpointer) nursery_start);
7697 label_no_wb_2 = mono_mb_emit_branch (mb, CEE_BLT);
7699 // if (*ptr >= nursery_end) return;
7700 mono_mb_emit_ldloc (mb, dereferenced_var);
7701 mono_mb_emit_ptr (mb, (gpointer) nursery_real_end);
7702 label_no_wb_5 = mono_mb_emit_branch (mb, CEE_BGE);
7705 // if (ptr >= stack_end) goto need_wb;
7706 mono_mb_emit_ldarg (mb, 0);
7707 EMIT_TLS_ACCESS (mb, stack_end, stack_end_offset);
7708 label_need_wb = mono_mb_emit_branch (mb, CEE_BGE_UN);
7710 // if (ptr >= stack_start) return;
7711 dummy_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7712 mono_mb_emit_ldarg (mb, 0);
7713 mono_mb_emit_ldloc_addr (mb, dummy_var);
7714 label_no_wb_3 = mono_mb_emit_branch (mb, CEE_BGE_UN);
7717 mono_mb_patch_branch (mb, label_need_wb);
7719 // buffer = STORE_REMSET_BUFFER;
7720 buffer_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7721 EMIT_TLS_ACCESS (mb, store_remset_buffer, store_remset_buffer_offset);
7722 mono_mb_emit_stloc (mb, buffer_var);
7724 // buffer_index = STORE_REMSET_BUFFER_INDEX;
7725 buffer_index_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7726 EMIT_TLS_ACCESS (mb, store_remset_buffer_index, store_remset_buffer_index_offset);
7727 mono_mb_emit_stloc (mb, buffer_index_var);
7729 // if (buffer [buffer_index] == ptr) return;
7730 mono_mb_emit_ldloc (mb, buffer_var);
7731 mono_mb_emit_ldloc (mb, buffer_index_var);
7732 g_assert (sizeof (gpointer) == 4 || sizeof (gpointer) == 8);
7733 mono_mb_emit_icon (mb, sizeof (gpointer) == 4 ? 2 : 3);
7734 mono_mb_emit_byte (mb, CEE_SHL);
7735 mono_mb_emit_byte (mb, CEE_ADD);
7736 mono_mb_emit_byte (mb, CEE_LDIND_I);
7737 mono_mb_emit_ldarg (mb, 0);
7738 label_no_wb_4 = mono_mb_emit_branch (mb, CEE_BEQ);
7741 mono_mb_emit_ldloc (mb, buffer_index_var);
7742 mono_mb_emit_icon (mb, 1);
7743 mono_mb_emit_byte (mb, CEE_ADD);
7744 mono_mb_emit_stloc (mb, buffer_index_var);
7746 // if (buffer_index >= STORE_REMSET_BUFFER_SIZE) goto slow_path;
7747 mono_mb_emit_ldloc (mb, buffer_index_var);
7748 mono_mb_emit_icon (mb, STORE_REMSET_BUFFER_SIZE);
7749 label_slow_path = mono_mb_emit_branch (mb, CEE_BGE);
7751 // buffer [buffer_index] = ptr;
7752 mono_mb_emit_ldloc (mb, buffer_var);
7753 mono_mb_emit_ldloc (mb, buffer_index_var);
7754 g_assert (sizeof (gpointer) == 4 || sizeof (gpointer) == 8);
7755 mono_mb_emit_icon (mb, sizeof (gpointer) == 4 ? 2 : 3);
7756 mono_mb_emit_byte (mb, CEE_SHL);
7757 mono_mb_emit_byte (mb, CEE_ADD);
7758 mono_mb_emit_ldarg (mb, 0);
7759 mono_mb_emit_byte (mb, CEE_STIND_I);
7761 // STORE_REMSET_BUFFER_INDEX = buffer_index;
7762 EMIT_TLS_ACCESS (mb, store_remset_buffer_index_addr, store_remset_buffer_index_addr_offset);
7763 mono_mb_emit_ldloc (mb, buffer_index_var);
7764 mono_mb_emit_byte (mb, CEE_STIND_I);
7767 mono_mb_patch_branch (mb, label_no_wb_1);
7768 mono_mb_patch_branch (mb, label_no_wb_2);
7769 mono_mb_patch_branch (mb, label_no_wb_3);
7770 mono_mb_patch_branch (mb, label_no_wb_4);
7771 #ifndef ALIGN_NURSERY
7772 mono_mb_patch_branch (mb, label_no_wb_5);
7774 mono_mb_emit_byte (mb, CEE_RET);
7777 mono_mb_patch_branch (mb, label_slow_path);
7781 mono_mb_emit_ldarg (mb, 0);
7782 mono_mb_emit_icall (mb, mono_gc_wbarrier_generic_nostore);
7783 mono_mb_emit_byte (mb, CEE_RET);
7785 res = mono_mb_create_method (mb, sig, 16);
7788 mono_loader_lock ();
7789 if (write_barrier_method) {
7790 /* Already created */
7791 mono_free_method (res);
7793 /* double-checked locking */
7794 mono_memory_barrier ();
7795 write_barrier_method = res;
7797 mono_loader_unlock ();
7799 return write_barrier_method;
7803 mono_gc_get_description (void)
7805 return g_strdup ("sgen");
7809 mono_gc_set_desktop_mode (void)
7814 mono_gc_is_moving (void)
7820 mono_gc_is_disabled (void)
7825 #endif /* HAVE_SGEN_GC */