2 * sgen-gc.c: Simple generational GC.
5 * Paolo Molaro (lupus@ximian.com)
7 * Copyright 2005-2010 Novell, Inc (http://www.novell.com)
9 * Thread start/stop adapted from Boehm's GC:
10 * Copyright (c) 1994 by Xerox Corporation. All rights reserved.
11 * Copyright (c) 1996 by Silicon Graphics. All rights reserved.
12 * Copyright (c) 1998 by Fergus Henderson. All rights reserved.
13 * Copyright (c) 2000-2004 by Hewlett-Packard Company. All rights reserved.
15 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
16 * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
18 * Permission is hereby granted to use or copy this program
19 * for any purpose, provided the above notices are retained on all copies.
20 * Permission to modify the code and to distribute modified code is granted,
21 * provided the above notices are retained, and a notice that the code was
22 * modified is included with the above copyright notice.
25 * Copyright 2001-2003 Ximian, Inc
26 * Copyright 2003-2010 Novell, Inc.
28 * Permission is hereby granted, free of charge, to any person obtaining
29 * a copy of this software and associated documentation files (the
30 * "Software"), to deal in the Software without restriction, including
31 * without limitation the rights to use, copy, modify, merge, publish,
32 * distribute, sublicense, and/or sell copies of the Software, and to
33 * permit persons to whom the Software is furnished to do so, subject to
34 * the following conditions:
36 * The above copyright notice and this permission notice shall be
37 * included in all copies or substantial portions of the Software.
39 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
40 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
41 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
42 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
43 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
44 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
45 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
48 * Important: allocation provides always zeroed memory, having to do
49 * a memset after allocation is deadly for performance.
50 * Memory usage at startup is currently as follows:
52 * 64 KB internal space
54 * We should provide a small memory config with half the sizes
56 * We currently try to make as few mono assumptions as possible:
57 * 1) 2-word header with no GC pointers in it (first vtable, second to store the
59 * 2) gc descriptor is the second word in the vtable (first word in the class)
60 * 3) 8 byte alignment is the minimum and enough (not true for special structures (SIMD), FIXME)
61 * 4) there is a function to get an object's size and the number of
62 * elements in an array.
63 * 5) we know the special way bounds are allocated for complex arrays
64 * 6) we know about proxies and how to treat them when domains are unloaded
66 * Always try to keep stack usage to a minimum: no recursive behaviour
67 * and no large stack allocs.
69 * General description.
70 * Objects are initially allocated in a nursery using a fast bump-pointer technique.
71 * When the nursery is full we start a nursery collection: this is performed with a
73 * When the old generation is full we start a copying GC of the old generation as well:
74 * this will be changed to mark&sweep with copying when fragmentation becomes to severe
75 * in the future. Maybe we'll even do both during the same collection like IMMIX.
77 * The things that complicate this description are:
78 * *) pinned objects: we can't move them so we need to keep track of them
79 * *) no precise info of the thread stacks and registers: we need to be able to
80 * quickly find the objects that may be referenced conservatively and pin them
81 * (this makes the first issues more important)
82 * *) large objects are too expensive to be dealt with using copying GC: we handle them
83 * with mark/sweep during major collections
84 * *) some objects need to not move even if they are small (interned strings, Type handles):
85 * we use mark/sweep for them, too: they are not allocated in the nursery, but inside
86 * PinnedChunks regions
92 *) we could have a function pointer in MonoClass to implement
93 customized write barriers for value types
95 *) investigate the stuff needed to advance a thread to a GC-safe
96 point (single-stepping, read from unmapped memory etc) and implement it.
97 This would enable us to inline allocations and write barriers, for example,
98 or at least parts of them, like the write barrier checks.
99 We may need this also for handling precise info on stacks, even simple things
100 as having uninitialized data on the stack and having to wait for the prolog
101 to zero it. Not an issue for the last frame that we scan conservatively.
102 We could always not trust the value in the slots anyway.
104 *) modify the jit to save info about references in stack locations:
105 this can be done just for locals as a start, so that at least
106 part of the stack is handled precisely.
108 *) test/fix endianess issues
110 *) Implement a card table as the write barrier instead of remembered
111 sets? Card tables are not easy to implement with our current
112 memory layout. We have several different kinds of major heap
113 objects: Small objects in regular blocks, small objects in pinned
114 chunks and LOS objects. If we just have a pointer we have no way
115 to tell which kind of object it points into, therefore we cannot
116 know where its card table is. The least we have to do to make
117 this happen is to get rid of write barriers for indirect stores.
120 *) Get rid of write barriers for indirect stores. We can do this by
121 telling the GC to wbarrier-register an object once we do an ldloca
122 or ldelema on it, and to unregister it once it's not used anymore
123 (it can only travel downwards on the stack). The problem with
124 unregistering is that it needs to happen eventually no matter
125 what, even if exceptions are thrown, the thread aborts, etc.
126 Rodrigo suggested that we could do only the registering part and
127 let the collector find out (pessimistically) when it's safe to
128 unregister, namely when the stack pointer of the thread that
129 registered the object is higher than it was when the registering
130 happened. This might make for a good first implementation to get
131 some data on performance.
133 *) Some sort of blacklist support? Blacklists is a concept from the
134 Boehm GC: if during a conservative scan we find pointers to an
135 area which we might use as heap, we mark that area as unusable, so
136 pointer retention by random pinning pointers is reduced.
138 *) experiment with max small object size (very small right now - 2kb,
139 because it's tied to the max freelist size)
141 *) add an option to mmap the whole heap in one chunk: it makes for many
142 simplifications in the checks (put the nursery at the top and just use a single
143 check for inclusion/exclusion): the issue this has is that on 32 bit systems it's
144 not flexible (too much of the address space may be used by default or we can't
145 increase the heap as needed) and we'd need a race-free mechanism to return memory
146 back to the system (mprotect(PROT_NONE) will still keep the memory allocated if it
147 was written to, munmap is needed, but the following mmap may not find the same segment
150 *) memzero the major fragments after restarting the world and optionally a smaller
153 *) investigate having fragment zeroing threads
155 *) separate locks for finalization and other minor stuff to reduce
158 *) try a different copying order to improve memory locality
160 *) a thread abort after a store but before the write barrier will
161 prevent the write barrier from executing
163 *) specialized dynamically generated markers/copiers
165 *) Dynamically adjust TLAB size to the number of threads. If we have
166 too many threads that do allocation, we might need smaller TLABs,
167 and we might get better performance with larger TLABs if we only
168 have a handful of threads. We could sum up the space left in all
169 assigned TLABs and if that's more than some percentage of the
170 nursery size, reduce the TLAB size.
172 *) Explore placing unreachable objects on unused nursery memory.
173 Instead of memset'ng a region to zero, place an int[] covering it.
174 A good place to start is add_nursery_frag. The tricky thing here is
175 placing those objects atomically outside of a collection.
185 #include <semaphore.h>
194 #define _XOPEN_SOURCE
196 #include "metadata/metadata-internals.h"
197 #include "metadata/class-internals.h"
198 #include "metadata/gc-internal.h"
199 #include "metadata/object-internals.h"
200 #include "metadata/threads.h"
201 #include "metadata/sgen-gc.h"
202 #include "metadata/sgen-archdep.h"
203 #include "metadata/mono-gc.h"
204 #include "metadata/method-builder.h"
205 #include "metadata/profiler-private.h"
206 #include "metadata/monitor.h"
207 #include "metadata/threadpool-internals.h"
208 #include "metadata/mempool-internals.h"
209 #include "metadata/marshal.h"
210 #include "utils/mono-mmap.h"
211 #include "utils/mono-time.h"
212 #include "utils/mono-semaphore.h"
213 #include "utils/mono-counters.h"
215 #include <mono/utils/memcheck.h>
217 #if defined(__MACH__)
218 #include "utils/mach-support.h"
221 #define OPDEF(a,b,c,d,e,f,g,h,i,j) \
225 #include "mono/cil/opcode.def"
231 #undef pthread_create
233 #undef pthread_detach
236 * ######################################################################
237 * ######## Types and constants used by the GC.
238 * ######################################################################
241 static int gc_initialized = 0;
242 static int gc_debug_level = 0;
243 static FILE* gc_debug_file;
244 /* If set, do a minor collection before every allocation */
245 static gboolean collect_before_allocs = FALSE;
246 /* If set, do a heap consistency check before each minor collection */
247 static gboolean consistency_check_at_minor_collection = FALSE;
248 /* If set, check that there are no references to the domain left at domain unload */
249 static gboolean xdomain_checks = FALSE;
250 /* If not null, dump the heap after each collection into this file */
251 static FILE *heap_dump_file = NULL;
252 /* If set, mark stacks conservatively, even if precise marking is possible */
253 static gboolean conservative_stack_mark = TRUE;
254 /* If set, do a plausibility check on the scan_starts before and after
256 static gboolean do_scan_starts_check = FALSE;
259 * Turning on heavy statistics will turn off the managed allocator and
260 * the managed write barrier.
262 //#define HEAVY_STATISTICS
264 #ifdef HEAVY_STATISTICS
265 #define HEAVY_STAT(x) x
267 #define HEAVY_STAT(x)
270 #ifdef HEAVY_STATISTICS
271 static long long stat_objects_alloced = 0;
272 static long long stat_bytes_alloced = 0;
273 static long long stat_objects_alloced_degraded = 0;
274 static long long stat_bytes_alloced_degraded = 0;
275 static long long stat_bytes_alloced_los = 0;
277 static long long stat_copy_object_called_nursery = 0;
278 static long long stat_objects_copied_nursery = 0;
279 static long long stat_copy_object_called_major = 0;
280 static long long stat_objects_copied_major = 0;
282 static long long stat_scan_object_called_nursery = 0;
283 static long long stat_scan_object_called_major = 0;
285 static long long stat_nursery_copy_object_failed_from_space = 0;
286 static long long stat_nursery_copy_object_failed_forwarded = 0;
287 static long long stat_nursery_copy_object_failed_pinned = 0;
289 static long long stat_store_remsets = 0;
290 static long long stat_store_remsets_unique = 0;
291 static long long stat_saved_remsets_1 = 0;
292 static long long stat_saved_remsets_2 = 0;
293 static long long stat_global_remsets_added = 0;
294 static long long stat_global_remsets_readded = 0;
295 static long long stat_global_remsets_processed = 0;
296 static long long stat_global_remsets_discarded = 0;
298 static long long stat_wasted_fragments_used = 0;
299 static long long stat_wasted_fragments_bytes = 0;
301 static int stat_wbarrier_set_field = 0;
302 static int stat_wbarrier_set_arrayref = 0;
303 static int stat_wbarrier_arrayref_copy = 0;
304 static int stat_wbarrier_generic_store = 0;
305 static int stat_wbarrier_generic_store_remset = 0;
306 static int stat_wbarrier_set_root = 0;
307 static int stat_wbarrier_value_copy = 0;
308 static int stat_wbarrier_object_copy = 0;
310 static long long stat_internal_alloc = 0;
311 static long long stat_internal_alloc_loop1 = 0;
312 static long long stat_internal_alloc_loop2 = 0;
315 static long long time_minor_pre_collection_fragment_clear = 0;
316 static long long time_minor_pinning = 0;
317 static long long time_minor_scan_remsets = 0;
318 static long long time_minor_scan_pinned = 0;
319 static long long time_minor_scan_registered_roots = 0;
320 static long long time_minor_scan_thread_data = 0;
321 static long long time_minor_finish_gray_stack = 0;
322 static long long time_minor_fragment_creation = 0;
324 static long long time_major_pre_collection_fragment_clear = 0;
325 static long long time_major_pinning = 0;
326 static long long time_major_scan_pinned = 0;
327 static long long time_major_scan_registered_roots = 0;
328 static long long time_major_scan_thread_data = 0;
329 static long long time_major_scan_alloc_pinned = 0;
330 static long long time_major_scan_finalized = 0;
331 static long long time_major_scan_big_objects = 0;
332 static long long time_major_finish_gray_stack = 0;
333 static long long time_major_free_bigobjs = 0;
334 static long long time_major_los_sweep = 0;
335 static long long time_major_sweep = 0;
336 static long long time_major_fragment_creation = 0;
338 static long long pinned_chunk_bytes_alloced = 0;
339 static long long large_internal_bytes_alloced = 0;
341 /* Keep in sync with internal_mem_names in dump_heap()! */
343 INTERNAL_MEM_PIN_QUEUE,
344 INTERNAL_MEM_FRAGMENT,
345 INTERNAL_MEM_SECTION,
346 INTERNAL_MEM_SCAN_STARTS,
347 INTERNAL_MEM_FIN_TABLE,
348 INTERNAL_MEM_FINALIZE_ENTRY,
349 INTERNAL_MEM_DISLINK_TABLE,
350 INTERNAL_MEM_DISLINK,
351 INTERNAL_MEM_ROOTS_TABLE,
352 INTERNAL_MEM_ROOT_RECORD,
353 INTERNAL_MEM_STATISTICS,
355 INTERNAL_MEM_GRAY_QUEUE,
356 INTERNAL_MEM_STORE_REMSET,
357 INTERNAL_MEM_MS_TABLES,
358 INTERNAL_MEM_MS_BLOCK_INFO,
359 INTERNAL_MEM_EPHEMERON_LINK,
363 static long small_internal_mem_bytes [INTERNAL_MEM_MAX];
367 mono_gc_flush_info (void)
369 fflush (gc_debug_file);
373 /* Define this to allow the user to change some of the constants by specifying
374 * their values in the MONO_GC_PARAMS environmental variable. See
375 * mono_gc_base_init for details. */
376 #define USER_CONFIG 1
378 #define TV_DECLARE(name) gint64 name
379 #define TV_GETTIME(tv) tv = mono_100ns_ticks ()
380 #define TV_ELAPSED(start,end) (int)((end-start) / 10)
381 #define TV_ELAPSED_MS(start,end) ((TV_ELAPSED((start),(end)) + 500) / 1000)
383 #define ALIGN_TO(val,align) ((((guint64)val) + ((align) - 1)) & ~((align) - 1))
385 #define GC_BITS_PER_WORD (sizeof (mword) * 8)
393 typedef struct _Block Block;
399 /* each request from the OS ends up in a GCMemSection */
400 typedef struct _GCMemSection GCMemSection;
401 struct _GCMemSection {
405 /* pointer where more data could be allocated if it fits */
409 * scan starts is an array of pointers to objects equally spaced in the allocation area
410 * They let use quickly find pinned objects from pinning pointers.
413 /* in major collections indexes in the pin_queue for objects that pin this section */
416 unsigned short num_scan_start;
417 gboolean is_to_space;
420 #define SIZEOF_GC_MEM_SECTION ((sizeof (GCMemSection) + 7) & ~7)
422 /* Pinned objects are allocated in the LOS space if bigger than half a page
423 * or from freelists otherwise. We assume that pinned objects are relatively few
424 * and they have a slow dying speed (like interned strings, thread objects).
425 * As such they will be collected only at major collections.
426 * free lists are not global: when we need memory we allocate a PinnedChunk.
427 * Each pinned chunk is made of several pages, the first of wich is used
428 * internally for bookeeping (here think of a page as 4KB). The bookeeping
429 * includes the freelists vectors and info about the object size of each page
430 * in the pinned chunk. So, when needed, a free page is found in a pinned chunk,
431 * a size is assigned to it, the page is divided in the proper chunks and each
432 * chunk is added to the freelist. To not waste space, the remaining space in the
433 * first page is used as objects of size 16 or 32 (need to measure which are more
435 * We use this same structure to allocate memory used internally by the GC, so
436 * we never use malloc/free if we need to alloc during collection: the world is stopped
437 * and malloc/free will deadlock.
438 * When we want to iterate over pinned objects, we just scan a page at a time
439 * linearly according to the size of objects in the page: the next pointer used to link
440 * the items in the freelist uses the same word as the vtable. Since we keep freelists
441 * for each pinned chunk, if the word points outside the pinned chunk it means
443 * We could avoid this expensive scanning in creative ways. We could have a policy
444 * of putting in the pinned space only objects we know about that have no struct fields
445 * with references and we can easily use a even expensive write barrier for them,
446 * since pointer writes on such objects should be rare.
447 * The best compromise is to just alloc interned strings and System.MonoType in them.
448 * It would be nice to allocate MonoThread in it, too: must check that we properly
449 * use write barriers so we don't have to do any expensive scanning of the whole pinned
450 * chunk list during minor collections. We can avoid it now because we alloc in it only
451 * reference-free objects.
453 #define PINNED_FIRST_SLOT_SIZE (sizeof (gpointer) * 4)
454 #define MAX_FREELIST_SIZE 8192
455 typedef struct _PinnedChunk PinnedChunk;
456 struct _PinnedChunk {
459 int *page_sizes; /* a 0 means the page is still unused */
462 void *data [1]; /* page sizes and free lists are stored here */
465 /* The method used to clear the nursery */
466 /* Clearing at nursery collections is the safest, but has bad interactions with caches.
467 * Clearing at TLAB creation is much faster, but more complex and it might expose hard
472 CLEAR_AT_TLAB_CREATION
473 } NurseryClearPolicy;
475 static NurseryClearPolicy nursery_clear_policy = CLEAR_AT_TLAB_CREATION;
478 * If this is set, the nursery is aligned to an address aligned to its size, ie.
479 * a 1MB nursery will be aligned to an address divisible by 1MB. This allows us to
480 * speed up ptr_in_nursery () checks which are very frequent. This requires the
481 * nursery size to be a compile time constant.
483 #define ALIGN_NURSERY 1
486 * The young generation is divided into fragments. This is because
487 * we can hand one fragments to a thread for lock-less fast alloc and
488 * because the young generation ends up fragmented anyway by pinned objects.
489 * Once a collection is done, a list of fragments is created. When doing
490 * thread local alloc we use smallish nurseries so we allow new threads to
491 * allocate memory from gen0 without triggering a collection. Threads that
492 * are found to allocate lots of memory are given bigger fragments. This
493 * should make the finalizer thread use little nursery memory after a while.
494 * We should start assigning threads very small fragments: if there are many
495 * threads the nursery will be full of reserved space that the threads may not
496 * use at all, slowing down allocation speed.
497 * Thread local allocation is done from areas of memory Hotspot calls Thread Local
498 * Allocation Buffers (TLABs).
500 typedef struct _Fragment Fragment;
504 char *fragment_start;
505 char *fragment_limit; /* the current soft limit for allocation */
509 /* the runtime can register areas of memory as roots: we keep two lists of roots,
510 * a pinned root set for conservatively scanned roots and a normal one for
511 * precisely scanned roots (currently implemented as a single list).
513 typedef struct _RootRecord RootRecord;
522 * We're never actually using the first element. It's always set to
523 * NULL to simplify the elimination of consecutive duplicate
526 #define STORE_REMSET_BUFFER_SIZE 1024
528 typedef struct _GenericStoreRememberedSet GenericStoreRememberedSet;
529 struct _GenericStoreRememberedSet {
530 GenericStoreRememberedSet *next;
531 /* We need one entry less because the first entry of store
532 remset buffers is always a dummy and we don't copy it. */
533 gpointer data [STORE_REMSET_BUFFER_SIZE - 1];
536 /* we have 4 possible values in the low 2 bits */
538 REMSET_LOCATION, /* just a pointer to the exact location */
539 REMSET_RANGE, /* range of pointer fields */
540 REMSET_OBJECT, /* mark all the object for scanning */
541 REMSET_VTYPE, /* a valuetype array described by a gc descriptor and a count */
542 REMSET_TYPE_MASK = 0x3
545 #ifdef HAVE_KW_THREAD
546 static __thread RememberedSet *remembered_set MONO_TLS_FAST;
548 static pthread_key_t remembered_set_key;
549 static RememberedSet *global_remset;
550 static RememberedSet *freed_thread_remsets;
551 static GenericStoreRememberedSet *generic_store_remsets = NULL;
553 /*A two slots cache for recently inserted remsets */
554 static gpointer global_remset_cache [2];
556 /* FIXME: later choose a size that takes into account the RememberedSet struct
557 * and doesn't waste any alloc paddin space.
559 #define DEFAULT_REMSET_SIZE 1024
560 static RememberedSet* alloc_remset (int size, gpointer id);
562 /* Structure that corresponds to a MonoVTable: desc is a mword so requires
563 * no cast from a pointer to an integer
570 /* these bits are set in the object vtable: we could merge them since an object can be
571 * either pinned or forwarded but not both.
572 * We store them in the vtable slot because the bits are used in the sync block for
573 * other purposes: if we merge them and alloc the sync blocks aligned to 8 bytes, we can change
574 * this and use bit 3 in the syncblock (with the lower two bits both set for forwarded, that
575 * would be an invalid combination for the monitor and hash code).
576 * The values are already shifted.
577 * The forwarding address is stored in the sync block.
579 #define FORWARDED_BIT 1
581 #define VTABLE_BITS_MASK 0x3
583 /* returns NULL if not forwarded, or the forwarded address */
584 #define object_is_forwarded(obj) (((mword*)(obj))[0] & FORWARDED_BIT ? (void*)(((mword*)(obj))[0] & ~VTABLE_BITS_MASK) : NULL)
585 /* set the forwarded address fw_addr for object obj */
586 #define forward_object(obj,fw_addr) do { \
587 ((mword*)(obj))[0] = (mword)(fw_addr) | FORWARDED_BIT; \
590 #define object_is_pinned(obj) (((mword*)(obj))[0] & PINNED_BIT)
591 #define pin_object(obj) do { \
592 ((mword*)(obj))[0] |= PINNED_BIT; \
594 #define unpin_object(obj) do { \
595 ((mword*)(obj))[0] &= ~PINNED_BIT; \
599 #define ptr_in_nursery(ptr) (((mword)(ptr) & ~((1 << DEFAULT_NURSERY_BITS) - 1)) == (mword)nursery_start)
601 #define ptr_in_nursery(ptr) ((char*)(ptr) >= nursery_start && (char*)(ptr) < nursery_real_end)
605 * Since we set bits in the vtable, use the macro to load it from the pointer to
606 * an object that is potentially pinned.
608 #define LOAD_VTABLE(addr) ((*(mword*)(addr)) & ~VTABLE_BITS_MASK)
610 #define VTABLE_HAS_REFERENCES(vt) (((MonoVTable*)(vt))->gc_descr != (void*)DESC_TYPE_RUN_LENGTH)
613 safe_name (void* obj)
615 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (obj);
616 return vt->klass->name;
620 * This function can be called on an object whose first word, the
621 * vtable field, is not intact. This is necessary for the parallel
625 par_object_get_size (MonoVTable *vtable, MonoObject* o)
627 MonoClass *klass = vtable->klass;
629 * We depend on mono_string_length_fast and
630 * mono_array_length_fast not using the object's vtable.
632 if (klass == mono_defaults.string_class) {
633 return sizeof (MonoString) + 2 * mono_string_length_fast ((MonoString*) o) + 2;
634 } else if (klass->rank) {
635 MonoArray *array = (MonoArray*)o;
636 size_t size = sizeof (MonoArray) + klass->sizes.element_size * mono_array_length_fast (array);
637 if (G_UNLIKELY (array->bounds)) {
638 size += sizeof (mono_array_size_t) - 1;
639 size &= ~(sizeof (mono_array_size_t) - 1);
640 size += sizeof (MonoArrayBounds) * klass->rank;
644 /* from a created object: the class must be inited already */
645 return klass->instance_size;
649 #define safe_object_get_size(o) par_object_get_size ((MonoVTable*)LOAD_VTABLE ((o)), (o))
652 * ######################################################################
653 * ######## Global data.
654 * ######################################################################
656 static LOCK_DECLARE (gc_mutex);
657 static int gc_disabled = 0;
658 static int num_minor_gcs = 0;
659 static int num_major_gcs = 0;
663 /* good sizes are 512KB-1MB: larger ones increase a lot memzeroing time */
664 #define DEFAULT_NURSERY_SIZE (default_nursery_size)
665 static int default_nursery_size = (1 << 22);
667 /* The number of trailing 0 bits in DEFAULT_NURSERY_SIZE */
668 #define DEFAULT_NURSERY_BITS (default_nursery_bits)
669 static int default_nursery_bits = 22;
674 #define DEFAULT_NURSERY_SIZE (4*1024*1024)
676 #define DEFAULT_NURSERY_BITS 22
681 #define MIN_MINOR_COLLECTION_ALLOWANCE (DEFAULT_NURSERY_SIZE * 4)
682 /* to quickly find the head of an object pinned by a conservative address
683 * we keep track of the objects allocated for each SCAN_START_SIZE memory
684 * chunk in the nursery or other memory sections. Larger values have less
685 * memory overhead and bigger runtime cost. 4-8 KB are reasonable values.
687 #define SCAN_START_SIZE (4096*2)
688 /* the minimum size of a fragment that we consider useful for allocation */
689 #define FRAGMENT_MIN_SIZE (512)
690 /* This is a fixed value used for pinned chunks, not the system pagesize */
691 #define FREELIST_PAGESIZE (16*1024)
693 static mword pagesize = 4096;
694 static mword nursery_size;
695 static int degraded_mode = 0;
697 static mword total_alloc = 0;
698 /* use this to tune when to do a major/minor collection */
699 static mword memory_pressure = 0;
700 static int minor_collection_allowance;
701 static int minor_collection_sections_alloced = 0;
703 static GCMemSection *nursery_section = NULL;
704 static mword lowest_heap_address = ~(mword)0;
705 static mword highest_heap_address = 0;
707 static LOCK_DECLARE (interruption_mutex);
709 #ifdef SGEN_PARALLEL_MARK
710 static LOCK_DECLARE (internal_allocator_mutex);
713 typedef struct _FinalizeEntry FinalizeEntry;
714 struct _FinalizeEntry {
719 typedef struct _FinalizeEntryHashTable FinalizeEntryHashTable;
720 struct _FinalizeEntryHashTable {
721 FinalizeEntry **table;
726 typedef struct _DisappearingLink DisappearingLink;
727 struct _DisappearingLink {
728 DisappearingLink *next;
732 typedef struct _DisappearingLinkHashTable DisappearingLinkHashTable;
733 struct _DisappearingLinkHashTable {
734 DisappearingLink **table;
739 typedef struct _EphemeronLinkNode EphemeronLinkNode;
741 struct _EphemeronLinkNode {
742 EphemeronLinkNode *next;
751 #define LARGE_INTERNAL_MEM_HEADER_MAGIC 0x7d289f3a
753 typedef struct _LargeInternalMemHeader LargeInternalMemHeader;
754 struct _LargeInternalMemHeader {
766 int current_collection_generation = -1;
769 * The link pointer is hidden by negating each bit. We use the lowest
770 * bit of the link (before negation) to store whether it needs
771 * resurrection tracking.
773 #define HIDE_POINTER(p,t) ((gpointer)(~((gulong)(p)|((t)?1:0))))
774 #define REVEAL_POINTER(p) ((gpointer)((~(gulong)(p))&~3L))
776 #define DISLINK_OBJECT(d) (REVEAL_POINTER (*(d)->link))
777 #define DISLINK_TRACK(d) ((~(gulong)(*(d)->link)) & 1)
780 * The finalizable hash has the object as the key, the
781 * disappearing_link hash, has the link address as key.
783 static FinalizeEntryHashTable minor_finalizable_hash;
784 static FinalizeEntryHashTable major_finalizable_hash;
785 /* objects that are ready to be finalized */
786 static FinalizeEntry *fin_ready_list = NULL;
787 static FinalizeEntry *critical_fin_list = NULL;
789 static DisappearingLinkHashTable minor_disappearing_link_hash;
790 static DisappearingLinkHashTable major_disappearing_link_hash;
792 static EphemeronLinkNode *ephemeron_list;
794 static int num_ready_finalizers = 0;
795 static int no_finalize = 0;
797 /* keep each size a multiple of ALLOC_ALIGN */
798 /* on 64 bit systems 8 is likely completely unused. */
799 static const int freelist_sizes [] = {
800 8, 16, 24, 32, 40, 48, 64, 80,
801 96, 128, 160, 192, 224, 256, 320, 384,
802 448, 512, 584, 680, 816, 1024, 1360, 2048,
803 2336, 2728, 3272, 4096, 5456, 8192 };
804 #define FREELIST_NUM_SLOTS (sizeof (freelist_sizes) / sizeof (freelist_sizes [0]))
806 /* This is also the MAJOR_SECTION_SIZE for the copying major
808 #define PINNED_CHUNK_SIZE (128 * 1024)
810 /* internal_chunk_list is used for allocating structures needed by the GC */
811 static PinnedChunk *internal_chunk_list = NULL;
813 static int slot_for_size (size_t size);
816 ROOT_TYPE_NORMAL = 0, /* "normal" roots */
817 ROOT_TYPE_PINNED = 1, /* roots without a GC descriptor */
818 ROOT_TYPE_WBARRIER = 2, /* roots with a write barrier */
822 /* registered roots: the key to the hash is the root start address */
824 * Different kinds of roots are kept separate to speed up pin_from_roots () for example.
826 static RootRecord **roots_hash [ROOT_TYPE_NUM] = { NULL, NULL };
827 static int roots_hash_size [ROOT_TYPE_NUM] = { 0, 0, 0 };
828 static mword roots_size = 0; /* amount of memory in the root set */
829 static int num_roots_entries [ROOT_TYPE_NUM] = { 0, 0, 0 };
832 * The current allocation cursors
833 * We allocate objects in the nursery.
834 * The nursery is the area between nursery_start and nursery_real_end.
835 * Allocation is done from a Thread Local Allocation Buffer (TLAB). TLABs are allocated
836 * from nursery fragments.
837 * tlab_next is the pointer to the space inside the TLAB where the next object will
839 * tlab_temp_end is the pointer to the end of the temporary space reserved for
840 * the allocation: it allows us to set the scan starts at reasonable intervals.
841 * tlab_real_end points to the end of the TLAB.
842 * nursery_frag_real_end points to the end of the currently used nursery fragment.
843 * nursery_first_pinned_start points to the start of the first pinned object in the nursery
844 * nursery_last_pinned_end points to the end of the last pinned object in the nursery
845 * At the next allocation, the area of the nursery where objects can be present is
846 * between MIN(nursery_first_pinned_start, first_fragment_start) and
847 * MAX(nursery_last_pinned_end, nursery_frag_real_end)
849 static char *nursery_start = NULL;
851 #ifdef HAVE_KW_THREAD
852 #define TLAB_ACCESS_INIT
853 #define TLAB_START tlab_start
854 #define TLAB_NEXT tlab_next
855 #define TLAB_TEMP_END tlab_temp_end
856 #define TLAB_REAL_END tlab_real_end
857 #define REMEMBERED_SET remembered_set
858 #define STORE_REMSET_BUFFER store_remset_buffer
859 #define STORE_REMSET_BUFFER_INDEX store_remset_buffer_index
860 #define IN_CRITICAL_REGION thread_info->in_critical_region
862 static pthread_key_t thread_info_key;
863 #define TLAB_ACCESS_INIT SgenThreadInfo *__thread_info__ = pthread_getspecific (thread_info_key)
864 #define TLAB_START (__thread_info__->tlab_start)
865 #define TLAB_NEXT (__thread_info__->tlab_next)
866 #define TLAB_TEMP_END (__thread_info__->tlab_temp_end)
867 #define TLAB_REAL_END (__thread_info__->tlab_real_end)
868 #define REMEMBERED_SET (__thread_info__->remset)
869 #define STORE_REMSET_BUFFER (__thread_info__->store_remset_buffer)
870 #define STORE_REMSET_BUFFER_INDEX (__thread_info__->store_remset_buffer_index)
871 #define IN_CRITICAL_REGION (__thread_info__->in_critical_region)
874 /* we use the memory barrier only to prevent compiler reordering (a memory constraint may be enough) */
875 #define ENTER_CRITICAL_REGION do {IN_CRITICAL_REGION = 1;mono_memory_barrier ();} while (0)
876 #define EXIT_CRITICAL_REGION do {IN_CRITICAL_REGION = 0;mono_memory_barrier ();} while (0)
879 * FIXME: What is faster, a TLS variable pointing to a structure, or separate TLS
880 * variables for next+temp_end ?
882 #ifdef HAVE_KW_THREAD
883 static __thread SgenThreadInfo *thread_info;
884 static __thread char *tlab_start;
885 static __thread char *tlab_next;
886 static __thread char *tlab_temp_end;
887 static __thread char *tlab_real_end;
888 static __thread gpointer *store_remset_buffer;
889 static __thread long store_remset_buffer_index;
890 /* Used by the managed allocator/wbarrier */
891 static __thread char **tlab_next_addr;
892 static __thread char *stack_end;
893 static __thread long *store_remset_buffer_index_addr;
895 static char *nursery_next = NULL;
896 static char *nursery_frag_real_end = NULL;
897 static char *nursery_real_end = NULL;
898 static char *nursery_last_pinned_end = NULL;
900 /* The size of a TLAB */
901 /* The bigger the value, the less often we have to go to the slow path to allocate a new
902 * one, but the more space is wasted by threads not allocating much memory.
904 * FIXME: Make this self-tuning for each thread.
906 static guint32 tlab_size = (1024 * 4);
908 /*How much space is tolerable to be wasted from the current fragment when allocating a new TLAB*/
909 #define MAX_NURSERY_TLAB_WASTE 512
911 /* fragments that are free and ready to be used for allocation */
912 static Fragment *nursery_fragments = NULL;
913 /* freeelist of fragment structures */
914 static Fragment *fragment_freelist = NULL;
917 * Objects bigger then this go into the large object space. This size
918 * has a few constraints. It must fit into the major heap, which in
919 * the case of the copying collector means that it must fit into a
920 * pinned chunk. It must also play well with the GC descriptors, some
921 * of which (DESC_TYPE_RUN_LENGTH, DESC_TYPE_SMALL_BITMAP) encode the
924 #define MAX_SMALL_OBJ_SIZE 8000
926 /* Functions supplied by the runtime to be called by the GC */
927 static MonoGCCallbacks gc_callbacks;
929 #define ALLOC_ALIGN 8
930 #define ALLOC_ALIGN_BITS 3
932 #define ALIGN_UP(s) (((s)+(ALLOC_ALIGN-1)) & ~(ALLOC_ALIGN-1))
934 #define MOVED_OBJECTS_NUM 64
935 static void *moved_objects [MOVED_OBJECTS_NUM];
936 static int moved_objects_idx = 0;
939 * ######################################################################
940 * ######## Macros and function declarations.
941 * ######################################################################
944 #define ADDR_IN_HEAP_BOUNDARIES(addr) ((p) >= lowest_heap_address && (p) < highest_heap_address)
947 align_pointer (void *ptr)
949 mword p = (mword)ptr;
950 p += sizeof (gpointer) - 1;
951 p &= ~ (sizeof (gpointer) - 1);
955 typedef struct _GrayQueue GrayQueue;
957 typedef void (*CopyOrMarkObjectFunc) (void**, GrayQueue*);
958 typedef char* (*ScanObjectFunc) (char*, GrayQueue*);
960 /* forward declarations */
961 static void* get_internal_mem (size_t size, int type);
962 static void free_internal_mem (void *addr, int type);
963 static void* get_os_memory (size_t size, int activate);
964 static void* get_os_memory_aligned (mword size, mword alignment, gboolean activate);
965 static void free_os_memory (void *addr, size_t size);
966 static G_GNUC_UNUSED void report_internal_mem_usage (void);
968 static int stop_world (void);
969 static int restart_world (void);
970 static void add_to_global_remset (gpointer ptr);
971 static void scan_thread_data (void *start_nursery, void *end_nursery, gboolean precise);
972 static void scan_from_remsets (void *start_nursery, void *end_nursery, GrayQueue *queue);
973 static void scan_from_registered_roots (CopyOrMarkObjectFunc copy_func, char *addr_start, char *addr_end, int root_type, GrayQueue *queue);
974 static void scan_finalizer_entries (CopyOrMarkObjectFunc copy_func, FinalizeEntry *list, GrayQueue *queue);
975 static void find_pinning_ref_from_thread (char *obj, size_t size);
976 static void update_current_thread_stack (void *start);
977 static void finalize_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, int generation, GrayQueue *queue);
978 static void add_or_remove_disappearing_link (MonoObject *obj, void **link, gboolean track, int generation);
979 static void null_link_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, int generation, GrayQueue *queue);
980 static void null_links_for_domain (MonoDomain *domain, int generation);
981 static gboolean search_fragment_for_size (size_t size);
982 static int search_fragment_for_size_range (size_t desired_size, size_t minimum_size);
983 static void build_nursery_fragments (int start_pin, int end_pin);
984 static void clear_nursery_fragments (char *next);
985 static void pin_from_roots (void *start_nursery, void *end_nursery);
986 static int pin_objects_from_addresses (GCMemSection *section, void **start, void **end, void *start_nursery, void *end_nursery, GrayQueue *queue);
987 static void pin_objects_in_section (GCMemSection *section, GrayQueue *queue);
988 static void optimize_pin_queue (int start_slot);
989 static void clear_remsets (void);
990 static void clear_tlabs (void);
991 typedef void (*IterateObjectCallbackFunc) (char*, size_t, void*);
992 static void scan_area_with_callback (char *start, char *end, IterateObjectCallbackFunc callback, void *data);
993 static void scan_object (char *start, GrayQueue *queue);
994 static void major_scan_object (char *start, GrayQueue *queue);
995 static void par_copy_object_no_checks (char *destination, MonoVTable *vt, void *obj, mword objsize, GrayQueue *queue);
996 static void* copy_object_no_checks (void *obj, GrayQueue *queue);
997 static void copy_object (void **obj_slot, GrayQueue *queue);
998 static void* get_chunk_freelist (PinnedChunk *chunk, int slot);
999 static PinnedChunk* alloc_pinned_chunk (void);
1000 static void sort_addresses (void **array, int size);
1001 static void drain_gray_stack (GrayQueue *queue);
1002 static void finish_gray_stack (char *start_addr, char *end_addr, int generation, GrayQueue *queue);
1003 static gboolean need_major_collection (void);
1004 static void major_collection (const char *reason);
1005 static void update_heap_boundaries (mword low, mword high);
1007 static void mono_gc_register_disappearing_link (MonoObject *obj, void **link, gboolean track);
1009 void describe_ptr (char *ptr);
1010 void check_object (char *start);
1012 static void check_consistency (void);
1013 static void check_major_refs (void);
1014 static void check_section_scan_starts (GCMemSection *section);
1015 static void check_scan_starts (void);
1016 static void check_for_xdomain_refs (void);
1017 static void dump_occupied (char *start, char *end, char *section_start);
1018 static void dump_section (GCMemSection *section, const char *type);
1019 static void dump_heap (const char *type, int num, const char *reason);
1020 static void report_pinned_chunk (PinnedChunk *chunk, int seq);
1022 void mono_gc_scan_for_specific_ref (MonoObject *key);
1024 static void init_stats (void);
1026 static int mark_ephemerons_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, GrayQueue *queue);
1027 static void clear_unreachable_ephemerons (CopyOrMarkObjectFunc copy_func, char *start, char *end, GrayQueue *queue);
1028 static void null_ephemerons_for_domain (MonoDomain *domain);
1030 //#define BINARY_PROTOCOL
1031 #include "sgen-protocol.c"
1032 #include "sgen-pinning.c"
1033 #include "sgen-pinning-stats.c"
1034 #include "sgen-gray.c"
1035 #include "sgen-los.c"
1038 * ######################################################################
1039 * ######## GC descriptors
1040 * ######################################################################
1041 * Used to quickly get the info the GC needs about an object: size and
1042 * where the references are held.
1044 /* objects are aligned to 8 bytes boundaries
1045 * A descriptor is a pointer in MonoVTable, so 32 or 64 bits of size.
1046 * The low 3 bits define the type of the descriptor. The other bits
1047 * depend on the type.
1048 * As a general rule the 13 remaining low bits define the size, either
1049 * of the whole object or of the elements in the arrays. While for objects
1050 * the size is already in bytes, for arrays we need to shift, because
1051 * array elements might be smaller than 8 bytes. In case of arrays, we
1052 * use two bits to describe what the additional high bits represents,
1053 * so the default behaviour can handle element sizes less than 2048 bytes.
1054 * The high 16 bits, if 0 it means the object is pointer-free.
1055 * This design should make it easy and fast to skip over ptr-free data.
1056 * The first 4 types should cover >95% of the objects.
1057 * Note that since the size of objects is limited to 64K, larger objects
1058 * will be allocated in the large object heap.
1059 * If we want 4-bytes alignment, we need to put vector and small bitmap
1064 * We don't use 0 so that 0 isn't a valid GC descriptor. No
1065 * deep reason for this other than to be able to identify a
1066 * non-inited descriptor for debugging.
1068 * If an object contains no references, its GC descriptor is
1069 * always DESC_TYPE_RUN_LENGTH, without a size, no exceptions.
1070 * This is so that we can quickly check for that in
1071 * copy_object_no_checks(), without having to fetch the
1074 DESC_TYPE_RUN_LENGTH = 1, /* 15 bits aligned byte size | 1-3 (offset, numptr) bytes tuples */
1075 DESC_TYPE_SMALL_BITMAP, /* 15 bits aligned byte size | 16-48 bit bitmap */
1076 DESC_TYPE_COMPLEX, /* index for bitmap into complex_descriptors */
1077 DESC_TYPE_VECTOR, /* 10 bits element size | 1 bit array | 2 bits desc | element desc */
1078 DESC_TYPE_ARRAY, /* 10 bits element size | 1 bit array | 2 bits desc | element desc */
1079 DESC_TYPE_LARGE_BITMAP, /* | 29-61 bitmap bits */
1080 DESC_TYPE_COMPLEX_ARR, /* index for bitmap into complex_descriptors */
1081 /* subtypes for arrays and vectors */
1082 DESC_TYPE_V_PTRFREE = 0,/* there are no refs: keep first so it has a zero value */
1083 DESC_TYPE_V_REFS, /* all the array elements are refs */
1084 DESC_TYPE_V_RUN_LEN, /* elements are run-length encoded as DESC_TYPE_RUN_LENGTH */
1085 DESC_TYPE_V_BITMAP /* elements are as the bitmap in DESC_TYPE_SMALL_BITMAP */
1088 #define OBJECT_HEADER_WORDS (sizeof(MonoObject)/sizeof(gpointer))
1089 #define LOW_TYPE_BITS 3
1090 #define SMALL_BITMAP_SHIFT 16
1091 #define SMALL_BITMAP_SIZE (GC_BITS_PER_WORD - SMALL_BITMAP_SHIFT)
1092 #define VECTOR_INFO_SHIFT 14
1093 #define VECTOR_ELSIZE_SHIFT 3
1094 #define LARGE_BITMAP_SIZE (GC_BITS_PER_WORD - LOW_TYPE_BITS)
1095 #define MAX_ELEMENT_SIZE 0x3ff
1096 #define VECTOR_SUBTYPE_PTRFREE (DESC_TYPE_V_PTRFREE << VECTOR_INFO_SHIFT)
1097 #define VECTOR_SUBTYPE_REFS (DESC_TYPE_V_REFS << VECTOR_INFO_SHIFT)
1098 #define VECTOR_SUBTYPE_RUN_LEN (DESC_TYPE_V_RUN_LEN << VECTOR_INFO_SHIFT)
1099 #define VECTOR_SUBTYPE_BITMAP (DESC_TYPE_V_BITMAP << VECTOR_INFO_SHIFT)
1102 /* Root bitmap descriptors are simpler: the lower three bits describe the type
1103 * and we either have 30/62 bitmap bits or nibble-based run-length,
1104 * or a complex descriptor, or a user defined marker function.
1107 ROOT_DESC_CONSERVATIVE, /* 0, so matches NULL value */
1112 ROOT_DESC_TYPE_MASK = 0x7,
1113 ROOT_DESC_TYPE_SHIFT = 3,
1116 #define MAKE_ROOT_DESC(type,val) ((type) | ((val) << ROOT_DESC_TYPE_SHIFT))
1118 #define MAX_USER_DESCRIPTORS 16
1120 static gsize* complex_descriptors = NULL;
1121 static int complex_descriptors_size = 0;
1122 static int complex_descriptors_next = 0;
1123 static MonoGCRootMarkFunc user_descriptors [MAX_USER_DESCRIPTORS];
1124 static int user_descriptors_next = 0;
1127 alloc_complex_descriptor (gsize *bitmap, int numbits)
1131 numbits = ALIGN_TO (numbits, GC_BITS_PER_WORD);
1132 nwords = numbits / GC_BITS_PER_WORD + 1;
1135 res = complex_descriptors_next;
1136 /* linear search, so we don't have duplicates with domain load/unload
1137 * this should not be performance critical or we'd have bigger issues
1138 * (the number and size of complex descriptors should be small).
1140 for (i = 0; i < complex_descriptors_next; ) {
1141 if (complex_descriptors [i] == nwords) {
1142 int j, found = TRUE;
1143 for (j = 0; j < nwords - 1; ++j) {
1144 if (complex_descriptors [i + 1 + j] != bitmap [j]) {
1154 i += complex_descriptors [i];
1156 if (complex_descriptors_next + nwords > complex_descriptors_size) {
1157 int new_size = complex_descriptors_size * 2 + nwords;
1158 complex_descriptors = g_realloc (complex_descriptors, new_size * sizeof (gsize));
1159 complex_descriptors_size = new_size;
1161 DEBUG (6, fprintf (gc_debug_file, "Complex descriptor %d, size: %d (total desc memory: %d)\n", res, nwords, complex_descriptors_size));
1162 complex_descriptors_next += nwords;
1163 complex_descriptors [res] = nwords;
1164 for (i = 0; i < nwords - 1; ++i) {
1165 complex_descriptors [res + 1 + i] = bitmap [i];
1166 DEBUG (6, fprintf (gc_debug_file, "\tvalue: %p\n", (void*)complex_descriptors [res + 1 + i]));
1173 * Descriptor builders.
1176 mono_gc_make_descr_for_string (gsize *bitmap, int numbits)
1178 return (void*) DESC_TYPE_RUN_LENGTH;
1182 mono_gc_make_descr_for_object (gsize *bitmap, int numbits, size_t obj_size)
1184 int first_set = -1, num_set = 0, last_set = -1, i;
1186 size_t stored_size = obj_size;
1187 for (i = 0; i < numbits; ++i) {
1188 if (bitmap [i / GC_BITS_PER_WORD] & ((gsize)1 << (i % GC_BITS_PER_WORD))) {
1196 * We don't encode the size of types that don't contain
1197 * references because they might not be aligned, i.e. the
1198 * bottom two bits might be set, which would clash with the
1199 * bits we need to encode the descriptor type. Since we don't
1200 * use the encoded size to skip objects, other than for
1201 * processing remsets, in which case only the positions of
1202 * references are relevant, this is not a problem.
1205 return (void*)DESC_TYPE_RUN_LENGTH;
1206 g_assert (!(stored_size & 0x3));
1207 if (stored_size <= MAX_SMALL_OBJ_SIZE) {
1208 /* check run-length encoding first: one byte offset, one byte number of pointers
1209 * on 64 bit archs, we can have 3 runs, just one on 32.
1210 * It may be better to use nibbles.
1212 if (first_set < 0) {
1213 desc = DESC_TYPE_RUN_LENGTH | (stored_size << 1);
1214 DEBUG (6, fprintf (gc_debug_file, "Ptrfree descriptor %p, size: %zd\n", (void*)desc, stored_size));
1215 return (void*) desc;
1216 } else if (first_set < 256 && num_set < 256 && (first_set + num_set == last_set + 1)) {
1217 desc = DESC_TYPE_RUN_LENGTH | (stored_size << 1) | (first_set << 16) | (num_set << 24);
1218 DEBUG (6, fprintf (gc_debug_file, "Runlen descriptor %p, size: %zd, first set: %d, num set: %d\n", (void*)desc, stored_size, first_set, num_set));
1219 return (void*) desc;
1221 /* we know the 2-word header is ptr-free */
1222 if (last_set < SMALL_BITMAP_SIZE + OBJECT_HEADER_WORDS) {
1223 desc = DESC_TYPE_SMALL_BITMAP | (stored_size << 1) | ((*bitmap >> OBJECT_HEADER_WORDS) << SMALL_BITMAP_SHIFT);
1224 DEBUG (6, fprintf (gc_debug_file, "Smallbitmap descriptor %p, size: %zd, last set: %d\n", (void*)desc, stored_size, last_set));
1225 return (void*) desc;
1228 /* we know the 2-word header is ptr-free */
1229 if (last_set < LARGE_BITMAP_SIZE + OBJECT_HEADER_WORDS) {
1230 desc = DESC_TYPE_LARGE_BITMAP | ((*bitmap >> OBJECT_HEADER_WORDS) << LOW_TYPE_BITS);
1231 DEBUG (6, fprintf (gc_debug_file, "Largebitmap descriptor %p, size: %zd, last set: %d\n", (void*)desc, stored_size, last_set));
1232 return (void*) desc;
1234 /* it's a complex object ... */
1235 desc = DESC_TYPE_COMPLEX | (alloc_complex_descriptor (bitmap, last_set + 1) << LOW_TYPE_BITS);
1236 return (void*) desc;
1239 /* If the array holds references, numbits == 1 and the first bit is set in elem_bitmap */
1241 mono_gc_make_descr_for_array (int vector, gsize *elem_bitmap, int numbits, size_t elem_size)
1243 int first_set = -1, num_set = 0, last_set = -1, i;
1244 mword desc = vector? DESC_TYPE_VECTOR: DESC_TYPE_ARRAY;
1245 for (i = 0; i < numbits; ++i) {
1246 if (elem_bitmap [i / GC_BITS_PER_WORD] & ((gsize)1 << (i % GC_BITS_PER_WORD))) {
1253 /* See comment at the definition of DESC_TYPE_RUN_LENGTH. */
1255 return (void*)DESC_TYPE_RUN_LENGTH;
1256 if (elem_size <= MAX_ELEMENT_SIZE) {
1257 desc |= elem_size << VECTOR_ELSIZE_SHIFT;
1259 return (void*)(desc | VECTOR_SUBTYPE_PTRFREE);
1261 /* Note: we also handle structs with just ref fields */
1262 if (num_set * sizeof (gpointer) == elem_size) {
1263 return (void*)(desc | VECTOR_SUBTYPE_REFS | ((gssize)(-1) << 16));
1265 /* FIXME: try run-len first */
1266 /* Note: we can't skip the object header here, because it's not present */
1267 if (last_set <= SMALL_BITMAP_SIZE) {
1268 return (void*)(desc | VECTOR_SUBTYPE_BITMAP | (*elem_bitmap << 16));
1271 /* it's am array of complex structs ... */
1272 desc = DESC_TYPE_COMPLEX_ARR;
1273 desc |= alloc_complex_descriptor (elem_bitmap, last_set + 1) << LOW_TYPE_BITS;
1274 return (void*) desc;
1277 /* Return the bitmap encoded by a descriptor */
1279 mono_gc_get_bitmap_for_descr (void *descr, int *numbits)
1281 mword d = (mword)descr;
1285 case DESC_TYPE_RUN_LENGTH: {
1286 int first_set = (d >> 16) & 0xff;
1287 int num_set = (d >> 24) & 0xff;
1290 bitmap = g_new0 (gsize, (first_set + num_set + 7) / 8);
1292 for (i = first_set; i < first_set + num_set; ++i)
1293 bitmap [i / GC_BITS_PER_WORD] |= ((gsize)1 << (i % GC_BITS_PER_WORD));
1295 *numbits = first_set + num_set;
1299 case DESC_TYPE_SMALL_BITMAP:
1300 bitmap = g_new0 (gsize, 1);
1302 bitmap [0] = (d >> SMALL_BITMAP_SHIFT) << OBJECT_HEADER_WORDS;
1304 *numbits = GC_BITS_PER_WORD;
1308 g_assert_not_reached ();
1312 /* helper macros to scan and traverse objects, macros because we resue them in many functions */
1313 #define OBJ_RUN_LEN_SIZE(size,desc,obj) do { \
1314 (size) = ((desc) & 0xfff8) >> 1; \
1317 #define OBJ_BITMAP_SIZE(size,desc,obj) do { \
1318 (size) = ((desc) & 0xfff8) >> 1; \
1321 //#define PREFETCH(addr) __asm__ __volatile__ (" prefetchnta %0": : "m"(*(char *)(addr)))
1322 #define PREFETCH(addr)
1324 /* code using these macros must define a HANDLE_PTR(ptr) macro that does the work */
1325 #define OBJ_RUN_LEN_FOREACH_PTR(desc,obj) do { \
1326 if ((desc) & 0xffff0000) { \
1327 /* there are pointers */ \
1328 void **_objptr_end; \
1329 void **_objptr = (void**)(obj); \
1330 _objptr += ((desc) >> 16) & 0xff; \
1331 _objptr_end = _objptr + (((desc) >> 24) & 0xff); \
1332 while (_objptr < _objptr_end) { \
1333 HANDLE_PTR (_objptr, (obj)); \
1339 /* a bitmap desc means that there are pointer references or we'd have
1340 * choosen run-length, instead: add an assert to check.
1342 #define OBJ_BITMAP_FOREACH_PTR(desc,obj) do { \
1343 /* there are pointers */ \
1344 void **_objptr = (void**)(obj); \
1345 gsize _bmap = (desc) >> 16; \
1346 _objptr += OBJECT_HEADER_WORDS; \
1348 if ((_bmap & 1)) { \
1349 HANDLE_PTR (_objptr, (obj)); \
1356 #define OBJ_LARGE_BITMAP_FOREACH_PTR(vt,obj) do { \
1357 /* there are pointers */ \
1358 void **_objptr = (void**)(obj); \
1359 gsize _bmap = (vt)->desc >> LOW_TYPE_BITS; \
1360 _objptr += OBJECT_HEADER_WORDS; \
1362 if ((_bmap & 1)) { \
1363 HANDLE_PTR (_objptr, (obj)); \
1370 #define OBJ_COMPLEX_FOREACH_PTR(vt,obj) do { \
1371 /* there are pointers */ \
1372 void **_objptr = (void**)(obj); \
1373 gsize *bitmap_data = complex_descriptors + ((vt)->desc >> LOW_TYPE_BITS); \
1374 int bwords = (*bitmap_data) - 1; \
1375 void **start_run = _objptr; \
1378 MonoObject *myobj = (MonoObject*)obj; \
1379 g_print ("found %d at %p (0x%zx): %s.%s\n", bwords, (obj), (vt)->desc, myobj->vtable->klass->name_space, myobj->vtable->klass->name); \
1381 while (bwords-- > 0) { \
1382 gsize _bmap = *bitmap_data++; \
1383 _objptr = start_run; \
1384 /*g_print ("bitmap: 0x%x/%d at %p\n", _bmap, bwords, _objptr);*/ \
1386 if ((_bmap & 1)) { \
1387 HANDLE_PTR (_objptr, (obj)); \
1392 start_run += GC_BITS_PER_WORD; \
1396 /* this one is untested */
1397 #define OBJ_COMPLEX_ARR_FOREACH_PTR(vt,obj) do { \
1398 /* there are pointers */ \
1399 gsize *mbitmap_data = complex_descriptors + ((vt)->desc >> LOW_TYPE_BITS); \
1400 int mbwords = (*mbitmap_data++) - 1; \
1401 int el_size = mono_array_element_size (vt->klass); \
1402 char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
1403 char *e_end = e_start + el_size * mono_array_length_fast ((MonoArray*)(obj)); \
1405 g_print ("found %d at %p (0x%zx): %s.%s\n", mbwords, (obj), (vt)->desc, vt->klass->name_space, vt->klass->name); \
1406 while (e_start < e_end) { \
1407 void **_objptr = (void**)e_start; \
1408 gsize *bitmap_data = mbitmap_data; \
1409 unsigned int bwords = mbwords; \
1410 while (bwords-- > 0) { \
1411 gsize _bmap = *bitmap_data++; \
1412 void **start_run = _objptr; \
1413 /*g_print ("bitmap: 0x%x\n", _bmap);*/ \
1415 if ((_bmap & 1)) { \
1416 HANDLE_PTR (_objptr, (obj)); \
1421 _objptr = start_run + GC_BITS_PER_WORD; \
1423 e_start += el_size; \
1427 #define OBJ_VECTOR_FOREACH_PTR(vt,obj) do { \
1428 /* note: 0xffffc000 excludes DESC_TYPE_V_PTRFREE */ \
1429 if ((vt)->desc & 0xffffc000) { \
1430 int el_size = ((vt)->desc >> 3) & MAX_ELEMENT_SIZE; \
1431 /* there are pointers */ \
1432 int etype = (vt)->desc & 0xc000; \
1433 if (etype == (DESC_TYPE_V_REFS << 14)) { \
1434 void **p = (void**)((char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector)); \
1435 void **end_refs = (void**)((char*)p + el_size * mono_array_length_fast ((MonoArray*)(obj))); \
1436 /* Note: this code can handle also arrays of struct with only references in them */ \
1437 while (p < end_refs) { \
1438 HANDLE_PTR (p, (obj)); \
1441 } else if (etype == DESC_TYPE_V_RUN_LEN << 14) { \
1442 int offset = ((vt)->desc >> 16) & 0xff; \
1443 int num_refs = ((vt)->desc >> 24) & 0xff; \
1444 char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
1445 char *e_end = e_start + el_size * mono_array_length_fast ((MonoArray*)(obj)); \
1446 while (e_start < e_end) { \
1447 void **p = (void**)e_start; \
1450 for (i = 0; i < num_refs; ++i) { \
1451 HANDLE_PTR (p + i, (obj)); \
1453 e_start += el_size; \
1455 } else if (etype == DESC_TYPE_V_BITMAP << 14) { \
1456 char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
1457 char *e_end = e_start + el_size * mono_array_length_fast ((MonoArray*)(obj)); \
1458 while (e_start < e_end) { \
1459 void **p = (void**)e_start; \
1460 gsize _bmap = (vt)->desc >> 16; \
1461 /* Note: there is no object header here to skip */ \
1463 if ((_bmap & 1)) { \
1464 HANDLE_PTR (p, (obj)); \
1469 e_start += el_size; \
1475 //#include "sgen-major-copying.c"
1476 #include "sgen-marksweep.c"
1479 is_xdomain_ref_allowed (gpointer *ptr, char *obj, MonoDomain *domain)
1481 MonoObject *o = (MonoObject*)(obj);
1482 MonoObject *ref = (MonoObject*)*(ptr);
1483 int offset = (char*)(ptr) - (char*)o;
1485 if (o->vtable->klass == mono_defaults.thread_class && offset == G_STRUCT_OFFSET (MonoThread, internal_thread))
1487 if (o->vtable->klass == mono_defaults.internal_thread_class && offset == G_STRUCT_OFFSET (MonoInternalThread, current_appcontext))
1489 if (mono_class_has_parent (o->vtable->klass, mono_defaults.real_proxy_class) &&
1490 offset == G_STRUCT_OFFSET (MonoRealProxy, unwrapped_server))
1492 /* Thread.cached_culture_info */
1493 if (!strcmp (ref->vtable->klass->name_space, "System.Globalization") &&
1494 !strcmp (ref->vtable->klass->name, "CultureInfo") &&
1495 !strcmp(o->vtable->klass->name_space, "System") &&
1496 !strcmp(o->vtable->klass->name, "Object[]"))
1499 * at System.IO.MemoryStream.InternalConstructor (byte[],int,int,bool,bool) [0x0004d] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.IO/MemoryStream.cs:121
1500 * at System.IO.MemoryStream..ctor (byte[]) [0x00017] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.IO/MemoryStream.cs:81
1501 * at (wrapper remoting-invoke-with-check) System.IO.MemoryStream..ctor (byte[]) <IL 0x00020, 0xffffffff>
1502 * at System.Runtime.Remoting.Messaging.CADMethodCallMessage.GetArguments () [0x0000d] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Messaging/CADMessages.cs:327
1503 * at System.Runtime.Remoting.Messaging.MethodCall..ctor (System.Runtime.Remoting.Messaging.CADMethodCallMessage) [0x00017] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Messaging/MethodCall.cs:87
1504 * at System.AppDomain.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage,byte[]&,System.Runtime.Remoting.Messaging.CADMethodReturnMessage&) [0x00018] in /home/schani/Work/novell/trunk/mcs/class/corlib/System/AppDomain.cs:1213
1505 * at (wrapper remoting-invoke-with-check) System.AppDomain.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage,byte[]&,System.Runtime.Remoting.Messaging.CADMethodReturnMessage&) <IL 0x0003d, 0xffffffff>
1506 * at System.Runtime.Remoting.Channels.CrossAppDomainSink.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage) [0x00008] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Channels/CrossAppDomainChannel.cs:198
1507 * at (wrapper runtime-invoke) object.runtime_invoke_CrossAppDomainSink/ProcessMessageRes_object_object (object,intptr,intptr,intptr) <IL 0x0004c, 0xffffffff>
1509 if (!strcmp (ref->vtable->klass->name_space, "System") &&
1510 !strcmp (ref->vtable->klass->name, "Byte[]") &&
1511 !strcmp (o->vtable->klass->name_space, "System.IO") &&
1512 !strcmp (o->vtable->klass->name, "MemoryStream"))
1514 /* append_job() in threadpool.c */
1515 if (!strcmp (ref->vtable->klass->name_space, "System.Runtime.Remoting.Messaging") &&
1516 !strcmp (ref->vtable->klass->name, "AsyncResult") &&
1517 !strcmp (o->vtable->klass->name_space, "System") &&
1518 !strcmp (o->vtable->klass->name, "Object[]") &&
1519 mono_thread_pool_is_queue_array ((MonoArray*) o))
1525 check_reference_for_xdomain (gpointer *ptr, char *obj, MonoDomain *domain)
1527 MonoObject *o = (MonoObject*)(obj);
1528 MonoObject *ref = (MonoObject*)*(ptr);
1529 int offset = (char*)(ptr) - (char*)o;
1531 MonoClassField *field;
1534 if (!ref || ref->vtable->domain == domain)
1536 if (is_xdomain_ref_allowed (ptr, obj, domain))
1540 for (class = o->vtable->klass; class; class = class->parent) {
1543 for (i = 0; i < class->field.count; ++i) {
1544 if (class->fields[i].offset == offset) {
1545 field = &class->fields[i];
1553 if (ref->vtable->klass == mono_defaults.string_class)
1554 str = mono_string_to_utf8 ((MonoString*)ref);
1557 g_print ("xdomain reference in %p (%s.%s) at offset %d (%s) to %p (%s.%s) (%s) - pointed to by:\n",
1558 o, o->vtable->klass->name_space, o->vtable->klass->name,
1559 offset, field ? field->name : "",
1560 ref, ref->vtable->klass->name_space, ref->vtable->klass->name, str ? str : "");
1561 mono_gc_scan_for_specific_ref (o);
1567 #define HANDLE_PTR(ptr,obj) check_reference_for_xdomain ((ptr), (obj), domain)
1570 scan_object_for_xdomain_refs (char *start, mword size, void *data)
1572 MonoDomain *domain = ((MonoObject*)start)->vtable->domain;
1574 #include "sgen-scan-object.h"
1578 #define HANDLE_PTR(ptr,obj) do { \
1579 if ((MonoObject*)*(ptr) == key) { \
1580 g_print ("found ref to %p in object %p (%s) at offset %td\n", \
1581 key, (obj), safe_name ((obj)), ((char*)(ptr) - (char*)(obj))); \
1586 scan_object_for_specific_ref (char *start, MonoObject *key)
1588 #include "sgen-scan-object.h"
1592 scan_area_with_callback (char *start, char *end, IterateObjectCallbackFunc callback, void *data)
1594 while (start < end) {
1596 if (!*(void**)start) {
1597 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
1601 size = ALIGN_UP (safe_object_get_size ((MonoObject*) start));
1603 callback (start, size, data);
1610 scan_object_for_specific_ref_callback (char *obj, size_t size, MonoObject *key)
1612 scan_object_for_specific_ref (obj, key);
1616 check_root_obj_specific_ref (RootRecord *root, MonoObject *key, MonoObject *obj)
1620 g_print ("found ref to %p in root record %p\n", key, root);
1623 static MonoObject *check_key = NULL;
1624 static RootRecord *check_root = NULL;
1627 check_root_obj_specific_ref_from_marker (void **obj)
1629 check_root_obj_specific_ref (check_root, check_key, *obj);
1633 scan_roots_for_specific_ref (MonoObject *key, int root_type)
1638 for (i = 0; i < roots_hash_size [root_type]; ++i) {
1639 for (root = roots_hash [root_type][i]; root; root = root->next) {
1640 void **start_root = (void**)root->start_root;
1641 mword desc = root->root_desc;
1645 switch (desc & ROOT_DESC_TYPE_MASK) {
1646 case ROOT_DESC_BITMAP:
1647 desc >>= ROOT_DESC_TYPE_SHIFT;
1650 check_root_obj_specific_ref (root, key, *start_root);
1655 case ROOT_DESC_COMPLEX: {
1656 gsize *bitmap_data = complex_descriptors + (desc >> ROOT_DESC_TYPE_SHIFT);
1657 int bwords = (*bitmap_data) - 1;
1658 void **start_run = start_root;
1660 while (bwords-- > 0) {
1661 gsize bmap = *bitmap_data++;
1662 void **objptr = start_run;
1665 check_root_obj_specific_ref (root, key, *objptr);
1669 start_run += GC_BITS_PER_WORD;
1673 case ROOT_DESC_USER: {
1674 MonoGCRootMarkFunc marker = user_descriptors [desc >> ROOT_DESC_TYPE_SHIFT];
1675 marker (start_root, check_root_obj_specific_ref_from_marker);
1678 case ROOT_DESC_RUN_LEN:
1679 g_assert_not_reached ();
1681 g_assert_not_reached ();
1690 mono_gc_scan_for_specific_ref (MonoObject *key)
1696 scan_area_with_callback (nursery_section->data, nursery_section->end_data,
1697 (IterateObjectCallbackFunc)scan_object_for_specific_ref_callback, key);
1699 major_iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)scan_object_for_specific_ref_callback, key);
1701 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1702 scan_object_for_specific_ref (bigobj->data, key);
1704 scan_roots_for_specific_ref (key, ROOT_TYPE_NORMAL);
1705 scan_roots_for_specific_ref (key, ROOT_TYPE_WBARRIER);
1707 for (i = 0; i < roots_hash_size [ROOT_TYPE_PINNED]; ++i) {
1708 for (root = roots_hash [ROOT_TYPE_PINNED][i]; root; root = root->next) {
1709 void **ptr = (void**)root->start_root;
1711 while (ptr < (void**)root->end_root) {
1712 check_root_obj_specific_ref (root, *ptr, key);
1719 /* Clear all remaining nursery fragments */
1721 clear_nursery_fragments (char *next)
1724 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION) {
1725 g_assert (next <= nursery_frag_real_end);
1726 memset (next, 0, nursery_frag_real_end - next);
1727 for (frag = nursery_fragments; frag; frag = frag->next) {
1728 memset (frag->fragment_start, 0, frag->fragment_end - frag->fragment_start);
1734 need_remove_object_for_domain (char *start, MonoDomain *domain)
1736 if (mono_object_domain (start) == domain) {
1737 DEBUG (4, fprintf (gc_debug_file, "Need to cleanup object %p\n", start));
1738 binary_protocol_cleanup (start, (gpointer)LOAD_VTABLE (start), safe_object_get_size ((MonoObject*)start));
1745 process_object_for_domain_clearing (char *start, MonoDomain *domain)
1747 GCVTable *vt = (GCVTable*)LOAD_VTABLE (start);
1748 if (vt->klass == mono_defaults.internal_thread_class)
1749 g_assert (mono_object_domain (start) == mono_get_root_domain ());
1750 /* The object could be a proxy for an object in the domain
1752 if (mono_class_has_parent (vt->klass, mono_defaults.real_proxy_class)) {
1753 MonoObject *server = ((MonoRealProxy*)start)->unwrapped_server;
1755 /* The server could already have been zeroed out, so
1756 we need to check for that, too. */
1757 if (server && (!LOAD_VTABLE (server) || mono_object_domain (server) == domain)) {
1758 DEBUG (4, fprintf (gc_debug_file, "Cleaning up remote pointer in %p to object %p\n",
1760 ((MonoRealProxy*)start)->unwrapped_server = NULL;
1765 static MonoDomain *check_domain = NULL;
1768 check_obj_not_in_domain (void **o)
1770 g_assert (((MonoObject*)(*o))->vtable->domain != check_domain);
1774 scan_for_registered_roots_in_domain (MonoDomain *domain, int root_type)
1778 check_domain = domain;
1779 for (i = 0; i < roots_hash_size [root_type]; ++i) {
1780 for (root = roots_hash [root_type][i]; root; root = root->next) {
1781 void **start_root = (void**)root->start_root;
1782 mword desc = root->root_desc;
1784 /* The MonoDomain struct is allowed to hold
1785 references to objects in its own domain. */
1786 if (start_root == (void**)domain)
1789 switch (desc & ROOT_DESC_TYPE_MASK) {
1790 case ROOT_DESC_BITMAP:
1791 desc >>= ROOT_DESC_TYPE_SHIFT;
1793 if ((desc & 1) && *start_root)
1794 check_obj_not_in_domain (*start_root);
1799 case ROOT_DESC_COMPLEX: {
1800 gsize *bitmap_data = complex_descriptors + (desc >> ROOT_DESC_TYPE_SHIFT);
1801 int bwords = (*bitmap_data) - 1;
1802 void **start_run = start_root;
1804 while (bwords-- > 0) {
1805 gsize bmap = *bitmap_data++;
1806 void **objptr = start_run;
1808 if ((bmap & 1) && *objptr)
1809 check_obj_not_in_domain (*objptr);
1813 start_run += GC_BITS_PER_WORD;
1817 case ROOT_DESC_USER: {
1818 MonoGCRootMarkFunc marker = user_descriptors [desc >> ROOT_DESC_TYPE_SHIFT];
1819 marker (start_root, check_obj_not_in_domain);
1822 case ROOT_DESC_RUN_LEN:
1823 g_assert_not_reached ();
1825 g_assert_not_reached ();
1829 check_domain = NULL;
1833 check_for_xdomain_refs (void)
1837 scan_area_with_callback (nursery_section->data, nursery_section->end_data, (IterateObjectCallbackFunc)scan_object_for_xdomain_refs, NULL);
1839 major_iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)scan_object_for_xdomain_refs, NULL);
1841 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1842 scan_object_for_xdomain_refs (bigobj->data, bigobj->size, NULL);
1846 clear_domain_process_object (char *obj, MonoDomain *domain)
1850 process_object_for_domain_clearing (obj, domain);
1851 remove = need_remove_object_for_domain (obj, domain);
1853 if (remove && ((MonoObject*)obj)->synchronisation) {
1854 void **dislink = mono_monitor_get_object_monitor_weak_link ((MonoObject*)obj);
1856 mono_gc_register_disappearing_link (NULL, dislink, FALSE);
1863 clear_domain_process_minor_object_callback (char *obj, size_t size, MonoDomain *domain)
1865 if (clear_domain_process_object (obj, domain))
1866 memset (obj, 0, size);
1870 clear_domain_process_major_object_callback (char *obj, size_t size, MonoDomain *domain)
1872 clear_domain_process_object (obj, domain);
1876 clear_domain_free_major_non_pinned_object_callback (char *obj, size_t size, MonoDomain *domain)
1878 if (need_remove_object_for_domain (obj, domain))
1879 major_free_non_pinned_object (obj, size);
1883 clear_domain_free_major_pinned_object_callback (char *obj, size_t size, MonoDomain *domain)
1885 if (need_remove_object_for_domain (obj, domain))
1886 free_pinned_object (obj, size);
1890 * When appdomains are unloaded we can easily remove objects that have finalizers,
1891 * but all the others could still be present in random places on the heap.
1892 * We need a sweep to get rid of them even though it's going to be costly
1894 * The reason we need to remove them is because we access the vtable and class
1895 * structures to know the object size and the reference bitmap: once the domain is
1896 * unloaded the point to random memory.
1899 mono_gc_clear_domain (MonoDomain * domain)
1901 LOSObject *bigobj, *prev;
1906 clear_nursery_fragments (nursery_next);
1908 if (xdomain_checks && domain != mono_get_root_domain ()) {
1909 scan_for_registered_roots_in_domain (domain, ROOT_TYPE_NORMAL);
1910 scan_for_registered_roots_in_domain (domain, ROOT_TYPE_WBARRIER);
1911 check_for_xdomain_refs ();
1914 scan_area_with_callback (nursery_section->data, nursery_section->end_data,
1915 (IterateObjectCallbackFunc)clear_domain_process_minor_object_callback, domain);
1917 /*Ephemerons and dislinks must be processed before LOS since they might end up pointing
1918 to memory returned to the OS.*/
1919 null_ephemerons_for_domain (domain);
1921 for (i = GENERATION_NURSERY; i < GENERATION_MAX; ++i)
1922 null_links_for_domain (domain, i);
1924 /* We need two passes over major and large objects because
1925 freeing such objects might give their memory back to the OS
1926 (in the case of large objects) or obliterate its vtable
1927 (pinned objects with major-copying or pinned and non-pinned
1928 objects with major-mark&sweep), but we might need to
1929 dereference a pointer from an object to another object if
1930 the first object is a proxy. */
1931 major_iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)clear_domain_process_major_object_callback, domain);
1932 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1933 clear_domain_process_object (bigobj->data, domain);
1936 for (bigobj = los_object_list; bigobj;) {
1937 if (need_remove_object_for_domain (bigobj->data, domain)) {
1938 LOSObject *to_free = bigobj;
1940 prev->next = bigobj->next;
1942 los_object_list = bigobj->next;
1943 bigobj = bigobj->next;
1944 DEBUG (4, fprintf (gc_debug_file, "Freeing large object %p\n",
1946 free_large_object (to_free);
1950 bigobj = bigobj->next;
1952 major_iterate_objects (TRUE, FALSE, (IterateObjectCallbackFunc)clear_domain_free_major_non_pinned_object_callback, domain);
1953 major_iterate_objects (FALSE, TRUE, (IterateObjectCallbackFunc)clear_domain_free_major_pinned_object_callback, domain);
1959 global_remset_cache_clear (void)
1961 memset (global_remset_cache, 0, sizeof (global_remset_cache));
1965 * Tries to check if a given remset location was already added to the global remset.
1968 * A 2 entry, LRU cache of recently saw location remsets.
1970 * It's hand-coded instead of done using loops to reduce the number of memory references on cache hit.
1972 * Returns TRUE is the element was added..
1975 global_remset_location_was_not_added (gpointer ptr)
1978 gpointer first = global_remset_cache [0], second;
1980 HEAVY_STAT (++stat_global_remsets_discarded);
1984 second = global_remset_cache [1];
1986 if (second == ptr) {
1987 /*Move the second to the front*/
1988 global_remset_cache [0] = second;
1989 global_remset_cache [1] = first;
1991 HEAVY_STAT (++stat_global_remsets_discarded);
1995 global_remset_cache [0] = second;
1996 global_remset_cache [1] = ptr;
2001 * add_to_global_remset:
2003 * The global remset contains locations which point into newspace after
2004 * a minor collection. This can happen if the objects they point to are pinned.
2007 add_to_global_remset (gpointer ptr)
2011 g_assert (!ptr_in_nursery (ptr) && ptr_in_nursery (*(gpointer*)ptr));
2013 if (!global_remset_location_was_not_added (ptr))
2016 DEBUG (8, fprintf (gc_debug_file, "Adding global remset for %p\n", ptr));
2017 binary_protocol_global_remset (ptr, *(gpointer*)ptr, (gpointer)LOAD_VTABLE (*(gpointer*)ptr));
2019 HEAVY_STAT (++stat_global_remsets_added);
2022 * FIXME: If an object remains pinned, we need to add it at every minor collection.
2023 * To avoid uncontrolled growth of the global remset, only add each pointer once.
2025 if (global_remset->store_next + 3 < global_remset->end_set) {
2026 *(global_remset->store_next++) = (mword)ptr;
2029 rs = alloc_remset (global_remset->end_set - global_remset->data, NULL);
2030 rs->next = global_remset;
2032 *(global_remset->store_next++) = (mword)ptr;
2035 int global_rs_size = 0;
2037 for (rs = global_remset; rs; rs = rs->next) {
2038 global_rs_size += rs->store_next - rs->data;
2040 DEBUG (4, fprintf (gc_debug_file, "Global remset now has size %d\n", global_rs_size));
2045 * This function can be used even if the vtable of obj is not valid
2046 * anymore, which is the case in the parallel collector.
2049 par_copy_object_no_checks (char *destination, MonoVTable *vt, void *obj, mword objsize, GrayQueue *queue)
2051 static const void *copy_labels [] = { &&LAB_0, &&LAB_1, &&LAB_2, &&LAB_3, &&LAB_4, &&LAB_5, &&LAB_6, &&LAB_7, &&LAB_8 };
2053 DEBUG (9, g_assert (vt->klass->inited));
2054 DEBUG (9, fprintf (gc_debug_file, " (to %p, %s size: %lu)\n", destination, ((MonoObject*)obj)->vtable->klass->name, (unsigned long)objsize));
2055 binary_protocol_copy (obj, destination, vt, objsize);
2057 *(MonoVTable**)destination = vt;
2058 if (objsize <= sizeof (gpointer) * 8) {
2059 mword *dest = (mword*)destination;
2060 goto *copy_labels [objsize / sizeof (gpointer)];
2062 (dest) [7] = ((mword*)obj) [7];
2064 (dest) [6] = ((mword*)obj) [6];
2066 (dest) [5] = ((mword*)obj) [5];
2068 (dest) [4] = ((mword*)obj) [4];
2070 (dest) [3] = ((mword*)obj) [3];
2072 (dest) [2] = ((mword*)obj) [2];
2074 (dest) [1] = ((mword*)obj) [1];
2080 memcpy (destination + sizeof (mword), (char*)obj + sizeof (mword), objsize - sizeof (mword));
2082 /* adjust array->bounds */
2083 DEBUG (9, g_assert (vt->gc_descr));
2084 if (G_UNLIKELY (vt->rank && ((MonoArray*)obj)->bounds)) {
2085 MonoArray *array = (MonoArray*)destination;
2086 array->bounds = (MonoArrayBounds*)((char*)destination + ((char*)((MonoArray*)obj)->bounds - (char*)obj));
2087 DEBUG (9, fprintf (gc_debug_file, "Array instance %p: size: %lu, rank: %d, length: %lu\n", array, (unsigned long)objsize, vt->rank, (unsigned long)mono_array_length (array)));
2089 if (G_UNLIKELY (mono_profiler_events & MONO_PROFILE_GC_MOVES)) {
2090 /* FIXME: handle this for parallel collector */
2091 #ifdef SGEN_PARALLEL_MARK
2092 g_assert_not_reached ();
2094 if (moved_objects_idx == MOVED_OBJECTS_NUM) {
2095 mono_profiler_gc_moves (moved_objects, moved_objects_idx);
2096 moved_objects_idx = 0;
2098 moved_objects [moved_objects_idx++] = obj;
2099 moved_objects [moved_objects_idx++] = destination;
2103 DEBUG (9, fprintf (gc_debug_file, "Enqueuing gray object %p (%s)\n", obj, safe_name (obj)));
2104 GRAY_OBJECT_ENQUEUE (queue, obj);
2109 copy_object_no_checks (void *obj, GrayQueue *queue)
2111 MonoVTable *vt = ((MonoObject*)obj)->vtable;
2112 gboolean has_references = VTABLE_HAS_REFERENCES (vt);
2113 mword objsize = ALIGN_UP (par_object_get_size (vt, (MonoObject*)obj));
2116 MAJOR_GET_COPY_OBJECT_SPACE (destination, objsize, has_references);
2118 par_copy_object_no_checks (destination, vt, obj, objsize, has_references ? queue : NULL);
2120 /* set the forwarding pointer */
2121 forward_object (obj, destination);
2127 * This is how the copying happens from the nursery to the old generation.
2128 * We assume that at this time all the pinned objects have been identified and
2130 * We run scan_object() for each pinned object so that each referenced
2131 * objects if possible are copied. The new gray objects created can have
2132 * scan_object() run on them right away, too.
2133 * Then we run copy_object() for the precisely tracked roots. At this point
2134 * all the roots are either gray or black. We run scan_object() on the gray
2135 * objects until no more gray objects are created.
2136 * At the end of the process we walk again the pinned list and we unmark
2137 * the pinned flag. As we go we also create the list of free space for use
2138 * in the next allocation runs.
2140 * We need to remember objects from the old generation that point to the new one
2141 * (or just addresses?).
2143 * copy_object could be made into a macro once debugged (use inline for now).
2146 static void __attribute__((noinline))
2147 copy_object (void **obj_slot, GrayQueue *queue)
2150 char *obj = *obj_slot;
2152 DEBUG (9, g_assert (current_collection_generation == GENERATION_NURSERY));
2154 HEAVY_STAT (++stat_copy_object_called_nursery);
2156 if (!ptr_in_nursery (obj)) {
2157 HEAVY_STAT (++stat_nursery_copy_object_failed_from_space);
2161 DEBUG (9, fprintf (gc_debug_file, "Precise copy of %p from %p", obj, obj_slot));
2164 * Before we can copy the object we must make sure that we are
2165 * allowed to, i.e. that the object not pinned or not already
2169 if ((forwarded = object_is_forwarded (obj))) {
2170 DEBUG (9, g_assert (((MonoVTable*)LOAD_VTABLE(obj))->gc_descr));
2171 DEBUG (9, fprintf (gc_debug_file, " (already forwarded to %p)\n", forwarded));
2172 HEAVY_STAT (++stat_nursery_copy_object_failed_forwarded);
2173 *obj_slot = forwarded;
2176 if (object_is_pinned (obj)) {
2177 DEBUG (9, g_assert (((MonoVTable*)LOAD_VTABLE(obj))->gc_descr));
2178 DEBUG (9, fprintf (gc_debug_file, " (pinned, no change)\n"));
2179 HEAVY_STAT (++stat_nursery_copy_object_failed_pinned);
2183 HEAVY_STAT (++stat_objects_copied_nursery);
2185 *obj_slot = copy_object_no_checks (obj, queue);
2189 #define HANDLE_PTR(ptr,obj) do { \
2190 void *__old = *(ptr); \
2193 copy_object ((ptr), queue); \
2195 DEBUG (9, if (__old != __copy) fprintf (gc_debug_file, "Overwrote field at %p with %p (was: %p)\n", (ptr), *(ptr), __old)); \
2196 if (G_UNLIKELY (ptr_in_nursery (__copy) && !ptr_in_nursery ((ptr)))) \
2197 add_to_global_remset ((ptr)); \
2202 * Scan the object pointed to by @start for references to
2203 * other objects between @from_start and @from_end and copy
2204 * them to the gray_objects area.
2207 scan_object (char *start, GrayQueue *queue)
2209 #include "sgen-scan-object.h"
2211 HEAVY_STAT (++stat_scan_object_called_nursery);
2217 * Scan the valuetype pointed to by START, described by DESC for references to
2218 * other objects between @from_start and @from_end and copy them to the gray_objects area.
2219 * Returns a pointer to the end of the object.
2222 scan_vtype (char *start, mword desc, char* from_start, char* from_end, GrayQueue *queue)
2226 /* The descriptors include info about the MonoObject header as well */
2227 start -= sizeof (MonoObject);
2229 switch (desc & 0x7) {
2230 case DESC_TYPE_RUN_LENGTH:
2231 OBJ_RUN_LEN_FOREACH_PTR (desc,start);
2232 OBJ_RUN_LEN_SIZE (skip_size, desc, start);
2233 g_assert (skip_size);
2234 return start + skip_size;
2235 case DESC_TYPE_SMALL_BITMAP:
2236 OBJ_BITMAP_FOREACH_PTR (desc,start);
2237 OBJ_BITMAP_SIZE (skip_size, desc, start);
2238 return start + skip_size;
2239 case DESC_TYPE_LARGE_BITMAP:
2240 case DESC_TYPE_COMPLEX:
2242 g_assert_not_reached ();
2245 // The other descriptors can't happen with vtypes
2246 g_assert_not_reached ();
2253 #define HANDLE_PTR(ptr,obj) do { \
2254 void *__old = *(ptr); \
2257 major_copy_or_mark_object ((ptr), queue); \
2259 DEBUG (9, if (__old != __copy) fprintf (gc_debug_file, "Overwrote field at %p with %p (was: %p)\n", (ptr), *(ptr), __old)); \
2260 if (G_UNLIKELY (ptr_in_nursery (__copy) && !ptr_in_nursery ((ptr)))) \
2261 add_to_global_remset ((ptr)); \
2266 major_scan_object (char *start, GrayQueue *queue)
2268 #include "sgen-scan-object.h"
2270 HEAVY_STAT (++stat_scan_object_called_major);
2276 * Scan objects in the gray stack until the stack is empty. This should be called
2277 * frequently after each object is copied, to achieve better locality and cache
2281 drain_gray_stack (GrayQueue *queue)
2285 if (current_collection_generation == GENERATION_NURSERY) {
2287 GRAY_OBJECT_DEQUEUE (queue, obj);
2290 DEBUG (9, fprintf (gc_debug_file, "Precise gray object scan %p (%s)\n", obj, safe_name (obj)));
2291 scan_object (obj, queue);
2295 GRAY_OBJECT_DEQUEUE (queue, obj);
2298 DEBUG (9, fprintf (gc_debug_file, "Precise gray object scan %p (%s)\n", obj, safe_name (obj)));
2299 major_scan_object (obj, queue);
2305 * Addresses from start to end are already sorted. This function finds
2306 * the object header for each address and pins the object. The
2307 * addresses must be inside the passed section. The (start of the)
2308 * address array is overwritten with the addresses of the actually
2309 * pinned objects. Return the number of pinned objects.
2312 pin_objects_from_addresses (GCMemSection *section, void **start, void **end, void *start_nursery, void *end_nursery, GrayQueue *queue)
2317 void *last_obj = NULL;
2318 size_t last_obj_size = 0;
2321 void **definitely_pinned = start;
2322 while (start < end) {
2324 /* the range check should be reduntant */
2325 if (addr != last && addr >= start_nursery && addr < end_nursery) {
2326 DEBUG (5, fprintf (gc_debug_file, "Considering pinning addr %p\n", addr));
2327 /* multiple pointers to the same object */
2328 if (addr >= last_obj && (char*)addr < (char*)last_obj + last_obj_size) {
2332 idx = ((char*)addr - (char*)section->data) / SCAN_START_SIZE;
2333 g_assert (idx < section->num_scan_start);
2334 search_start = (void*)section->scan_starts [idx];
2335 if (!search_start || search_start > addr) {
2338 search_start = section->scan_starts [idx];
2339 if (search_start && search_start <= addr)
2342 if (!search_start || search_start > addr)
2343 search_start = start_nursery;
2345 if (search_start < last_obj)
2346 search_start = (char*)last_obj + last_obj_size;
2347 /* now addr should be in an object a short distance from search_start
2348 * Note that search_start must point to zeroed mem or point to an object.
2351 if (!*(void**)search_start) {
2352 search_start = (void*)ALIGN_UP ((mword)search_start + sizeof (gpointer));
2355 last_obj = search_start;
2356 last_obj_size = ALIGN_UP (safe_object_get_size ((MonoObject*)search_start));
2357 DEBUG (8, fprintf (gc_debug_file, "Pinned try match %p (%s), size %zd\n", last_obj, safe_name (last_obj), last_obj_size));
2358 if (addr >= search_start && (char*)addr < (char*)last_obj + last_obj_size) {
2359 DEBUG (4, fprintf (gc_debug_file, "Pinned object %p, vtable %p (%s), count %d\n", search_start, *(void**)search_start, safe_name (search_start), count));
2360 binary_protocol_pin (search_start, (gpointer)LOAD_VTABLE (search_start), safe_object_get_size (search_start));
2361 pin_object (search_start);
2362 GRAY_OBJECT_ENQUEUE (queue, search_start);
2364 pin_stats_register_object (search_start, last_obj_size);
2365 definitely_pinned [count] = search_start;
2369 /* skip to the next object */
2370 search_start = (void*)((char*)search_start + last_obj_size);
2371 } while (search_start <= addr);
2372 /* we either pinned the correct object or we ignored the addr because
2373 * it points to unused zeroed memory.
2379 //printf ("effective pinned: %d (at the end: %d)\n", count, (char*)end_nursery - (char*)last);
2384 pin_objects_in_section (GCMemSection *section, GrayQueue *queue)
2386 int start = section->pin_queue_start;
2387 int end = section->pin_queue_end;
2390 reduced_to = pin_objects_from_addresses (section, pin_queue + start, pin_queue + end,
2391 section->data, section->next_data, queue);
2392 section->pin_queue_start = start;
2393 section->pin_queue_end = start + reduced_to;
2397 /* Sort the addresses in array in increasing order.
2398 * Done using a by-the book heap sort. Which has decent and stable performance, is pretty cache efficient.
2401 sort_addresses (void **array, int size)
2406 for (i = 1; i < size; ++i) {
2409 int parent = (child - 1) / 2;
2411 if (array [parent] >= array [child])
2414 tmp = array [parent];
2415 array [parent] = array [child];
2416 array [child] = tmp;
2422 for (i = size - 1; i > 0; --i) {
2425 array [i] = array [0];
2431 while (root * 2 + 1 <= end) {
2432 int child = root * 2 + 1;
2434 if (child < end && array [child] < array [child + 1])
2436 if (array [root] >= array [child])
2440 array [root] = array [child];
2441 array [child] = tmp;
2448 static G_GNUC_UNUSED void
2449 print_nursery_gaps (void* start_nursery, void *end_nursery)
2452 gpointer first = start_nursery;
2454 for (i = 0; i < next_pin_slot; ++i) {
2455 next = pin_queue [i];
2456 fprintf (gc_debug_file, "Nursery range: %p-%p, size: %td\n", first, next, (char*)next-(char*)first);
2460 fprintf (gc_debug_file, "Nursery range: %p-%p, size: %td\n", first, next, (char*)next-(char*)first);
2463 /* reduce the info in the pin queue, removing duplicate pointers and sorting them */
2465 optimize_pin_queue (int start_slot)
2467 void **start, **cur, **end;
2468 /* sort and uniq pin_queue: we just sort and we let the rest discard multiple values */
2469 /* it may be better to keep ranges of pinned memory instead of individually pinning objects */
2470 DEBUG (5, fprintf (gc_debug_file, "Sorting pin queue, size: %d\n", next_pin_slot));
2471 if ((next_pin_slot - start_slot) > 1)
2472 sort_addresses (pin_queue + start_slot, next_pin_slot - start_slot);
2473 start = cur = pin_queue + start_slot;
2474 end = pin_queue + next_pin_slot;
2477 while (*start == *cur && cur < end)
2481 next_pin_slot = start - pin_queue;
2482 DEBUG (5, fprintf (gc_debug_file, "Pin queue reduced to size: %d\n", next_pin_slot));
2483 //DEBUG (6, print_nursery_gaps (start_nursery, end_nursery));
2488 * Scan the memory between start and end and queue values which could be pointers
2489 * to the area between start_nursery and end_nursery for later consideration.
2490 * Typically used for thread stacks.
2493 conservatively_pin_objects_from (void **start, void **end, void *start_nursery, void *end_nursery, int pin_type)
2496 while (start < end) {
2497 if (*start >= start_nursery && *start < end_nursery) {
2499 * *start can point to the middle of an object
2500 * note: should we handle pointing at the end of an object?
2501 * pinning in C# code disallows pointing at the end of an object
2502 * but there is some small chance that an optimizing C compiler
2503 * may keep the only reference to an object by pointing
2504 * at the end of it. We ignore this small chance for now.
2505 * Pointers to the end of an object are indistinguishable
2506 * from pointers to the start of the next object in memory
2507 * so if we allow that we'd need to pin two objects...
2508 * We queue the pointer in an array, the
2509 * array will then be sorted and uniqued. This way
2510 * we can coalesce several pinning pointers and it should
2511 * be faster since we'd do a memory scan with increasing
2512 * addresses. Note: we can align the address to the allocation
2513 * alignment, so the unique process is more effective.
2515 mword addr = (mword)*start;
2516 addr &= ~(ALLOC_ALIGN - 1);
2517 if (addr >= (mword)start_nursery && addr < (mword)end_nursery)
2518 pin_stage_ptr ((void*)addr);
2520 pin_stats_register_address ((char*)addr, pin_type);
2521 DEBUG (6, if (count) fprintf (gc_debug_file, "Pinning address %p\n", (void*)addr));
2526 DEBUG (7, if (count) fprintf (gc_debug_file, "found %d potential pinned heap pointers\n", count));
2530 * Debugging function: find in the conservative roots where @obj is being pinned.
2532 static G_GNUC_UNUSED void
2533 find_pinning_reference (char *obj, size_t size)
2537 char *endobj = obj + size;
2538 for (i = 0; i < roots_hash_size [0]; ++i) {
2539 for (root = roots_hash [0][i]; root; root = root->next) {
2540 /* if desc is non-null it has precise info */
2541 if (!root->root_desc) {
2542 char ** start = (char**)root->start_root;
2543 while (start < (char**)root->end_root) {
2544 if (*start >= obj && *start < endobj) {
2545 DEBUG (0, fprintf (gc_debug_file, "Object %p referenced in pinned roots %p-%p (at %p in record %p)\n", obj, root->start_root, root->end_root, start, root));
2552 find_pinning_ref_from_thread (obj, size);
2556 * The first thing we do in a collection is to identify pinned objects.
2557 * This function considers all the areas of memory that need to be
2558 * conservatively scanned.
2561 pin_from_roots (void *start_nursery, void *end_nursery)
2565 DEBUG (2, fprintf (gc_debug_file, "Scanning pinned roots (%d bytes, %d/%d entries)\n", (int)roots_size, num_roots_entries [ROOT_TYPE_NORMAL], num_roots_entries [ROOT_TYPE_PINNED]));
2566 /* objects pinned from the API are inside these roots */
2567 for (i = 0; i < roots_hash_size [ROOT_TYPE_PINNED]; ++i) {
2568 for (root = roots_hash [ROOT_TYPE_PINNED][i]; root; root = root->next) {
2569 DEBUG (6, fprintf (gc_debug_file, "Pinned roots %p-%p\n", root->start_root, root->end_root));
2570 conservatively_pin_objects_from ((void**)root->start_root, (void**)root->end_root, start_nursery, end_nursery, PIN_TYPE_OTHER);
2573 /* now deal with the thread stacks
2574 * in the future we should be able to conservatively scan only:
2575 * *) the cpu registers
2576 * *) the unmanaged stack frames
2577 * *) the _last_ managed stack frame
2578 * *) pointers slots in managed frames
2580 scan_thread_data (start_nursery, end_nursery, FALSE);
2582 evacuate_pin_staging_area ();
2585 static CopyOrMarkObjectFunc user_copy_or_mark_func;
2586 static GrayQueue *user_copy_or_mark_queue;
2589 single_arg_user_copy_or_mark (void **obj)
2591 user_copy_or_mark_func (obj, user_copy_or_mark_queue);
2595 * The memory area from start_root to end_root contains pointers to objects.
2596 * Their position is precisely described by @desc (this means that the pointer
2597 * can be either NULL or the pointer to the start of an object).
2598 * This functions copies them to to_space updates them.
2600 * This function is not thread-safe!
2603 precisely_scan_objects_from (CopyOrMarkObjectFunc copy_func, void** start_root, void** end_root, char* n_start, char *n_end, mword desc, GrayQueue *queue)
2605 switch (desc & ROOT_DESC_TYPE_MASK) {
2606 case ROOT_DESC_BITMAP:
2607 desc >>= ROOT_DESC_TYPE_SHIFT;
2609 if ((desc & 1) && *start_root) {
2610 copy_func (start_root, queue);
2611 DEBUG (9, fprintf (gc_debug_file, "Overwrote root at %p with %p\n", start_root, *start_root));
2612 drain_gray_stack (queue);
2618 case ROOT_DESC_COMPLEX: {
2619 gsize *bitmap_data = complex_descriptors + (desc >> ROOT_DESC_TYPE_SHIFT);
2620 int bwords = (*bitmap_data) - 1;
2621 void **start_run = start_root;
2623 while (bwords-- > 0) {
2624 gsize bmap = *bitmap_data++;
2625 void **objptr = start_run;
2627 if ((bmap & 1) && *objptr) {
2628 copy_func (objptr, queue);
2629 DEBUG (9, fprintf (gc_debug_file, "Overwrote root at %p with %p\n", objptr, *objptr));
2630 drain_gray_stack (queue);
2635 start_run += GC_BITS_PER_WORD;
2639 case ROOT_DESC_USER: {
2640 MonoGCRootMarkFunc marker = user_descriptors [desc >> ROOT_DESC_TYPE_SHIFT];
2641 user_copy_or_mark_func = copy_func;
2642 user_copy_or_mark_queue = queue;
2643 marker (start_root, single_arg_user_copy_or_mark);
2644 user_copy_or_mark_func = NULL;
2645 user_copy_or_mark_queue = NULL;
2648 case ROOT_DESC_RUN_LEN:
2649 g_assert_not_reached ();
2651 g_assert_not_reached ();
2656 update_heap_boundaries (mword low, mword high)
2661 old = lowest_heap_address;
2664 } while (SGEN_CAS_PTR ((gpointer*)&lowest_heap_address, (gpointer)low, (gpointer)old) != (gpointer)old);
2667 old = highest_heap_address;
2670 } while (SGEN_CAS_PTR ((gpointer*)&highest_heap_address, (gpointer)high, (gpointer)old) != (gpointer)old);
2674 alloc_fragment (void)
2676 Fragment *frag = fragment_freelist;
2678 fragment_freelist = frag->next;
2682 frag = get_internal_mem (sizeof (Fragment), INTERNAL_MEM_FRAGMENT);
2687 /* size must be a power of 2 */
2689 get_os_memory_aligned (mword size, mword alignment, gboolean activate)
2691 /* Allocate twice the memory to be able to put the block on an aligned address */
2692 char *mem = get_os_memory (size + alignment, activate);
2697 aligned = (char*)((mword)(mem + (alignment - 1)) & ~(alignment - 1));
2698 g_assert (aligned >= mem && aligned + size <= mem + size + alignment && !((mword)aligned & (alignment - 1)));
2701 free_os_memory (mem, aligned - mem);
2702 if (aligned + size < mem + size + alignment)
2703 free_os_memory (aligned + size, (mem + size + alignment) - (aligned + size));
2709 * Allocate and setup the data structures needed to be able to allocate objects
2710 * in the nursery. The nursery is stored in nursery_section.
2713 alloc_nursery (void)
2715 GCMemSection *section;
2721 if (nursery_section)
2723 DEBUG (2, fprintf (gc_debug_file, "Allocating nursery size: %lu\n", (unsigned long)nursery_size));
2724 /* later we will alloc a larger area for the nursery but only activate
2725 * what we need. The rest will be used as expansion if we have too many pinned
2726 * objects in the existing nursery.
2728 /* FIXME: handle OOM */
2729 section = get_internal_mem (SIZEOF_GC_MEM_SECTION, INTERNAL_MEM_SECTION);
2731 g_assert (nursery_size == DEFAULT_NURSERY_SIZE);
2732 alloc_size = nursery_size;
2733 #ifdef ALIGN_NURSERY
2734 data = get_os_memory_aligned (alloc_size, alloc_size, TRUE);
2736 data = get_os_memory (alloc_size, TRUE);
2738 nursery_start = data;
2739 nursery_real_end = nursery_start + nursery_size;
2740 update_heap_boundaries ((mword)nursery_start, (mword)nursery_real_end);
2741 nursery_next = nursery_start;
2742 total_alloc += alloc_size;
2743 DEBUG (4, fprintf (gc_debug_file, "Expanding nursery size (%p-%p): %lu, total: %lu\n", data, data + alloc_size, (unsigned long)nursery_size, (unsigned long)total_alloc));
2744 section->data = section->next_data = data;
2745 section->size = alloc_size;
2746 section->end_data = nursery_real_end;
2747 scan_starts = (alloc_size + SCAN_START_SIZE - 1) / SCAN_START_SIZE;
2748 section->scan_starts = get_internal_mem (sizeof (char*) * scan_starts, INTERNAL_MEM_SCAN_STARTS);
2749 section->num_scan_start = scan_starts;
2750 section->block.role = MEMORY_ROLE_GEN0;
2751 section->block.next = NULL;
2753 nursery_section = section;
2755 /* Setup the single first large fragment */
2756 frag = alloc_fragment ();
2757 frag->fragment_start = nursery_start;
2758 frag->fragment_limit = nursery_start;
2759 frag->fragment_end = nursery_real_end;
2760 nursery_frag_real_end = nursery_real_end;
2761 /* FIXME: frag here is lost */
2765 scan_finalizer_entries (CopyOrMarkObjectFunc copy_func, FinalizeEntry *list, GrayQueue *queue)
2769 for (fin = list; fin; fin = fin->next) {
2772 DEBUG (5, fprintf (gc_debug_file, "Scan of fin ready object: %p (%s)\n", fin->object, safe_name (fin->object)));
2773 copy_func (&fin->object, queue);
2777 static mword fragment_total = 0;
2779 * We found a fragment of free memory in the nursery: memzero it and if
2780 * it is big enough, add it to the list of fragments that can be used for
2784 add_nursery_frag (size_t frag_size, char* frag_start, char* frag_end)
2787 DEBUG (4, fprintf (gc_debug_file, "Found empty fragment: %p-%p, size: %zd\n", frag_start, frag_end, frag_size));
2788 binary_protocol_empty (frag_start, frag_size);
2789 /* memsetting just the first chunk start is bound to provide better cache locality */
2790 if (nursery_clear_policy == CLEAR_AT_GC)
2791 memset (frag_start, 0, frag_size);
2792 /* Not worth dealing with smaller fragments: need to tune */
2793 if (frag_size >= FRAGMENT_MIN_SIZE) {
2794 fragment = alloc_fragment ();
2795 fragment->fragment_start = frag_start;
2796 fragment->fragment_limit = frag_start;
2797 fragment->fragment_end = frag_end;
2798 fragment->next = nursery_fragments;
2799 nursery_fragments = fragment;
2800 fragment_total += frag_size;
2802 /* Clear unused fragments, pinning depends on this */
2803 /*TODO place an int[] here instead of the memset if size justify it*/
2804 memset (frag_start, 0, frag_size);
2809 generation_name (int generation)
2811 switch (generation) {
2812 case GENERATION_NURSERY: return "nursery";
2813 case GENERATION_OLD: return "old";
2814 default: g_assert_not_reached ();
2818 static DisappearingLinkHashTable*
2819 get_dislink_hash_table (int generation)
2821 switch (generation) {
2822 case GENERATION_NURSERY: return &minor_disappearing_link_hash;
2823 case GENERATION_OLD: return &major_disappearing_link_hash;
2824 default: g_assert_not_reached ();
2828 static FinalizeEntryHashTable*
2829 get_finalize_entry_hash_table (int generation)
2831 switch (generation) {
2832 case GENERATION_NURSERY: return &minor_finalizable_hash;
2833 case GENERATION_OLD: return &major_finalizable_hash;
2834 default: g_assert_not_reached ();
2839 finish_gray_stack (char *start_addr, char *end_addr, int generation, GrayQueue *queue)
2844 int ephemeron_rounds = 0;
2845 CopyOrMarkObjectFunc copy_func = current_collection_generation == GENERATION_NURSERY ? copy_object : major_copy_or_mark_object;
2848 * We copied all the reachable objects. Now it's the time to copy
2849 * the objects that were not referenced by the roots, but by the copied objects.
2850 * we built a stack of objects pointed to by gray_start: they are
2851 * additional roots and we may add more items as we go.
2852 * We loop until gray_start == gray_objects which means no more objects have
2853 * been added. Note this is iterative: no recursion is involved.
2854 * We need to walk the LO list as well in search of marked big objects
2855 * (use a flag since this is needed only on major collections). We need to loop
2856 * here as well, so keep a counter of marked LO (increasing it in copy_object).
2857 * To achieve better cache locality and cache usage, we drain the gray stack
2858 * frequently, after each object is copied, and just finish the work here.
2860 drain_gray_stack (queue);
2862 DEBUG (2, fprintf (gc_debug_file, "%s generation done\n", generation_name (generation)));
2863 /* walk the finalization queue and move also the objects that need to be
2864 * finalized: use the finalized objects as new roots so the objects they depend
2865 * on are also not reclaimed. As with the roots above, only objects in the nursery
2866 * are marked/copied.
2867 * We need a loop here, since objects ready for finalizers may reference other objects
2868 * that are fin-ready. Speedup with a flag?
2872 * Walk the ephemeron tables marking all values with reachable keys. This must be completely done
2873 * before processing finalizable objects to avoid finalizing reachable values.
2875 * It must be done inside the finalizaters loop since objects must not be removed from CWT tables
2876 * while they are been finalized.
2878 int done_with_ephemerons = 0;
2880 done_with_ephemerons = mark_ephemerons_in_range (copy_func, start_addr, end_addr, queue);
2881 drain_gray_stack (queue);
2883 } while (!done_with_ephemerons);
2885 fin_ready = num_ready_finalizers;
2886 finalize_in_range (copy_func, start_addr, end_addr, generation, queue);
2887 if (generation == GENERATION_OLD)
2888 finalize_in_range (copy_func, nursery_start, nursery_real_end, GENERATION_NURSERY, queue);
2890 /* drain the new stack that might have been created */
2891 DEBUG (6, fprintf (gc_debug_file, "Precise scan of gray area post fin\n"));
2892 drain_gray_stack (queue);
2893 } while (fin_ready != num_ready_finalizers);
2896 * Clear ephemeron pairs with unreachable keys.
2897 * We pass the copy func so we can figure out if an array was promoted or not.
2899 clear_unreachable_ephemerons (copy_func, start_addr, end_addr, queue);
2902 DEBUG (2, fprintf (gc_debug_file, "Finalize queue handling scan for %s generation: %d usecs %d ephemeron roundss\n", generation_name (generation), TV_ELAPSED (atv, btv), ephemeron_rounds));
2905 * handle disappearing links
2906 * Note we do this after checking the finalization queue because if an object
2907 * survives (at least long enough to be finalized) we don't clear the link.
2908 * This also deals with a possible issue with the monitor reclamation: with the Boehm
2909 * GC a finalized object my lose the monitor because it is cleared before the finalizer is
2912 g_assert (gray_object_queue_is_empty (queue));
2914 null_link_in_range (copy_func, start_addr, end_addr, generation, queue);
2915 if (generation == GENERATION_OLD)
2916 null_link_in_range (copy_func, start_addr, end_addr, GENERATION_NURSERY, queue);
2917 if (gray_object_queue_is_empty (queue))
2919 drain_gray_stack (queue);
2922 g_assert (gray_object_queue_is_empty (queue));
2926 check_section_scan_starts (GCMemSection *section)
2929 for (i = 0; i < section->num_scan_start; ++i) {
2930 if (section->scan_starts [i]) {
2931 guint size = safe_object_get_size ((MonoObject*) section->scan_starts [i]);
2932 g_assert (size >= sizeof (MonoObject) && size <= MAX_SMALL_OBJ_SIZE);
2938 check_scan_starts (void)
2940 if (!do_scan_starts_check)
2942 check_section_scan_starts (nursery_section);
2943 major_check_scan_starts ();
2946 static int last_num_pinned = 0;
2949 build_nursery_fragments (int start_pin, int end_pin)
2951 char *frag_start, *frag_end;
2955 while (nursery_fragments) {
2956 Fragment *next = nursery_fragments->next;
2957 nursery_fragments->next = fragment_freelist;
2958 fragment_freelist = nursery_fragments;
2959 nursery_fragments = next;
2961 frag_start = nursery_start;
2963 /* clear scan starts */
2964 memset (nursery_section->scan_starts, 0, nursery_section->num_scan_start * sizeof (gpointer));
2965 for (i = start_pin; i < end_pin; ++i) {
2966 frag_end = pin_queue [i];
2967 /* remove the pin bit from pinned objects */
2968 unpin_object (frag_end);
2969 nursery_section->scan_starts [((char*)frag_end - (char*)nursery_section->data)/SCAN_START_SIZE] = frag_end;
2970 frag_size = frag_end - frag_start;
2972 add_nursery_frag (frag_size, frag_start, frag_end);
2973 frag_size = ALIGN_UP (safe_object_get_size ((MonoObject*)pin_queue [i]));
2974 frag_start = (char*)pin_queue [i] + frag_size;
2976 nursery_last_pinned_end = frag_start;
2977 frag_end = nursery_real_end;
2978 frag_size = frag_end - frag_start;
2980 add_nursery_frag (frag_size, frag_start, frag_end);
2981 if (!nursery_fragments) {
2982 DEBUG (1, fprintf (gc_debug_file, "Nursery fully pinned (%d)\n", end_pin - start_pin));
2983 for (i = start_pin; i < end_pin; ++i) {
2984 DEBUG (3, fprintf (gc_debug_file, "Bastard pinning obj %p (%s), size: %d\n", pin_queue [i], safe_name (pin_queue [i]), safe_object_get_size (pin_queue [i])));
2989 nursery_next = nursery_frag_real_end = NULL;
2991 /* Clear TLABs for all threads */
2996 scan_from_registered_roots (CopyOrMarkObjectFunc copy_func, char *addr_start, char *addr_end, int root_type, GrayQueue *queue)
3000 for (i = 0; i < roots_hash_size [root_type]; ++i) {
3001 for (root = roots_hash [root_type][i]; root; root = root->next) {
3002 DEBUG (6, fprintf (gc_debug_file, "Precise root scan %p-%p (desc: %p)\n", root->start_root, root->end_root, (void*)root->root_desc));
3003 precisely_scan_objects_from (copy_func, (void**)root->start_root, (void**)root->end_root, addr_start, addr_end, root->root_desc, queue);
3009 dump_occupied (char *start, char *end, char *section_start)
3011 fprintf (heap_dump_file, "<occupied offset=\"%td\" size=\"%td\"/>\n", start - section_start, end - start);
3015 dump_section (GCMemSection *section, const char *type)
3017 char *start = section->data;
3018 char *end = section->data + section->size;
3019 char *occ_start = NULL;
3021 char *old_start = NULL; /* just for debugging */
3023 fprintf (heap_dump_file, "<section type=\"%s\" size=\"%lu\">\n", type, (unsigned long)section->size);
3025 while (start < end) {
3029 if (!*(void**)start) {
3031 dump_occupied (occ_start, start, section->data);
3034 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
3037 g_assert (start < section->next_data);
3042 vt = (GCVTable*)LOAD_VTABLE (start);
3045 size = ALIGN_UP (safe_object_get_size ((MonoObject*) start));
3048 fprintf (heap_dump_file, "<object offset=\"%d\" class=\"%s.%s\" size=\"%d\"/>\n",
3049 start - section->data,
3050 vt->klass->name_space, vt->klass->name,
3058 dump_occupied (occ_start, start, section->data);
3060 fprintf (heap_dump_file, "</section>\n");
3064 dump_object (MonoObject *obj, gboolean dump_location)
3066 static char class_name [1024];
3068 MonoClass *class = mono_object_class (obj);
3072 * Python's XML parser is too stupid to parse angle brackets
3073 * in strings, so we just ignore them;
3076 while (class->name [i] && j < sizeof (class_name) - 1) {
3077 if (!strchr ("<>\"", class->name [i]))
3078 class_name [j++] = class->name [i];
3081 g_assert (j < sizeof (class_name));
3084 fprintf (heap_dump_file, "<object class=\"%s.%s\" size=\"%d\"",
3085 class->name_space, class_name,
3086 safe_object_get_size (obj));
3087 if (dump_location) {
3088 const char *location;
3089 if (ptr_in_nursery (obj))
3090 location = "nursery";
3091 else if (safe_object_get_size (obj) <= MAX_SMALL_OBJ_SIZE)
3095 fprintf (heap_dump_file, " location=\"%s\"", location);
3097 fprintf (heap_dump_file, "/>\n");
3101 dump_heap (const char *type, int num, const char *reason)
3103 static char const *internal_mem_names [] = { "pin-queue", "fragment", "section", "scan-starts",
3104 "fin-table", "finalize-entry", "dislink-table",
3105 "dislink", "roots-table", "root-record", "statistics",
3106 "remset", "gray-queue", "store-remset", "marksweep-tables",
3107 "marksweep-block-info", "ephemeron-link" };
3113 fprintf (heap_dump_file, "<collection type=\"%s\" num=\"%d\"", type, num);
3115 fprintf (heap_dump_file, " reason=\"%s\"", reason);
3116 fprintf (heap_dump_file, ">\n");
3117 fprintf (heap_dump_file, "<other-mem-usage type=\"pinned-chunks\" size=\"%lld\"/>\n", pinned_chunk_bytes_alloced);
3118 fprintf (heap_dump_file, "<other-mem-usage type=\"large-internal\" size=\"%lld\"/>\n", large_internal_bytes_alloced);
3119 fprintf (heap_dump_file, "<other-mem-usage type=\"mempools\" size=\"%ld\"/>\n", mono_mempool_get_bytes_allocated ());
3120 for (i = 0; i < INTERNAL_MEM_MAX; ++i)
3121 fprintf (heap_dump_file, "<other-mem-usage type=\"%s\" size=\"%ld\"/>\n", internal_mem_names [i], small_internal_mem_bytes [i]);
3122 fprintf (heap_dump_file, "<pinned type=\"stack\" bytes=\"%zu\"/>\n", pinned_byte_counts [PIN_TYPE_STACK]);
3123 /* fprintf (heap_dump_file, "<pinned type=\"static-data\" bytes=\"%d\"/>\n", pinned_byte_counts [PIN_TYPE_STATIC_DATA]); */
3124 fprintf (heap_dump_file, "<pinned type=\"other\" bytes=\"%zu\"/>\n", pinned_byte_counts [PIN_TYPE_OTHER]);
3126 fprintf (heap_dump_file, "<pinned-objects>\n");
3127 for (list = pinned_objects; list; list = list->next)
3128 dump_object (list->obj, TRUE);
3129 fprintf (heap_dump_file, "</pinned-objects>\n");
3131 dump_section (nursery_section, "nursery");
3135 fprintf (heap_dump_file, "<los>\n");
3136 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
3137 dump_object ((MonoObject*)bigobj->data, FALSE);
3138 fprintf (heap_dump_file, "</los>\n");
3140 fprintf (heap_dump_file, "</collection>\n");
3146 static gboolean inited = FALSE;
3151 mono_counters_register ("Minor fragment clear", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_pre_collection_fragment_clear);
3152 mono_counters_register ("Minor pinning", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_pinning);
3153 mono_counters_register ("Minor scan remsets", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_scan_remsets);
3154 mono_counters_register ("Minor scan pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_scan_pinned);
3155 mono_counters_register ("Minor scan registered roots", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_scan_registered_roots);
3156 mono_counters_register ("Minor scan thread data", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_scan_thread_data);
3157 mono_counters_register ("Minor finish gray stack", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_finish_gray_stack);
3158 mono_counters_register ("Minor fragment creation", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_fragment_creation);
3160 mono_counters_register ("Major fragment clear", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_pre_collection_fragment_clear);
3161 mono_counters_register ("Major pinning", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_pinning);
3162 mono_counters_register ("Major scan pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_pinned);
3163 mono_counters_register ("Major scan registered roots", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_registered_roots);
3164 mono_counters_register ("Major scan thread data", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_thread_data);
3165 mono_counters_register ("Major scan alloc_pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_alloc_pinned);
3166 mono_counters_register ("Major scan finalized", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_finalized);
3167 mono_counters_register ("Major scan big objects", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_big_objects);
3168 mono_counters_register ("Major finish gray stack", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_finish_gray_stack);
3169 mono_counters_register ("Major free big objects", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_free_bigobjs);
3170 mono_counters_register ("Major LOS sweep", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_los_sweep);
3171 mono_counters_register ("Major sweep", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_sweep);
3172 mono_counters_register ("Major fragment creation", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_fragment_creation);
3174 #ifdef HEAVY_STATISTICS
3175 mono_counters_register ("WBarrier set field", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_field);
3176 mono_counters_register ("WBarrier set arrayref", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_arrayref);
3177 mono_counters_register ("WBarrier arrayref copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_arrayref_copy);
3178 mono_counters_register ("WBarrier generic store called", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_generic_store);
3179 mono_counters_register ("WBarrier generic store stored", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_generic_store_remset);
3180 mono_counters_register ("WBarrier set root", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_root);
3181 mono_counters_register ("WBarrier value copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_value_copy);
3182 mono_counters_register ("WBarrier object copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_object_copy);
3184 mono_counters_register ("# objects allocated", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_alloced);
3185 mono_counters_register ("bytes allocated", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_bytes_alloced);
3186 mono_counters_register ("# objects allocated degraded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_alloced_degraded);
3187 mono_counters_register ("bytes allocated degraded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_bytes_alloced_degraded);
3188 mono_counters_register ("bytes allocated in LOS", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_bytes_alloced_los);
3190 mono_counters_register ("# copy_object() called (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_called_nursery);
3191 mono_counters_register ("# objects copied (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_copied_nursery);
3192 mono_counters_register ("# copy_object() called (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_called_major);
3193 mono_counters_register ("# objects copied (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_copied_major);
3195 mono_counters_register ("# scan_object() called (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_scan_object_called_nursery);
3196 mono_counters_register ("# scan_object() called (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_scan_object_called_major);
3198 mono_counters_register ("# nursery copy_object() failed from space", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_from_space);
3199 mono_counters_register ("# nursery copy_object() failed forwarded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_forwarded);
3200 mono_counters_register ("# nursery copy_object() failed pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_pinned);
3202 mono_counters_register ("# wasted fragments used", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_wasted_fragments_used);
3203 mono_counters_register ("bytes in wasted fragments", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_wasted_fragments_bytes);
3205 mono_counters_register ("Store remsets", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_store_remsets);
3206 mono_counters_register ("Unique store remsets", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_store_remsets_unique);
3207 mono_counters_register ("Saved remsets 1", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_saved_remsets_1);
3208 mono_counters_register ("Saved remsets 2", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_saved_remsets_2);
3209 mono_counters_register ("Global remsets added", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_added);
3210 mono_counters_register ("Global remsets re-added", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_readded);
3211 mono_counters_register ("Global remsets processed", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_processed);
3212 mono_counters_register ("Global remsets discarded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_discarded);
3214 mono_counters_register ("Internal allocs", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_internal_alloc);
3215 mono_counters_register ("Internal alloc loop1", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_internal_alloc_loop1);
3216 mono_counters_register ("Internal alloc loop2", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_internal_alloc_loop2);
3223 need_major_collection (void)
3225 mword los_alloced = los_memory_usage - MIN (last_los_memory_usage, los_memory_usage);
3226 return minor_collection_sections_alloced * MAJOR_SECTION_SIZE + los_alloced > minor_collection_allowance;
3230 * Collect objects in the nursery. Returns whether to trigger a major
3234 collect_nursery (size_t requested_size)
3236 size_t max_garbage_amount;
3237 char *orig_nursery_next;
3238 TV_DECLARE (all_atv);
3239 TV_DECLARE (all_btv);
3243 current_collection_generation = GENERATION_NURSERY;
3246 binary_protocol_collection (GENERATION_NURSERY);
3247 check_scan_starts ();
3250 orig_nursery_next = nursery_next;
3251 nursery_next = MAX (nursery_next, nursery_last_pinned_end);
3252 /* FIXME: optimize later to use the higher address where an object can be present */
3253 nursery_next = MAX (nursery_next, nursery_real_end);
3255 DEBUG (1, fprintf (gc_debug_file, "Start nursery collection %d %p-%p, size: %d\n", num_minor_gcs, nursery_start, nursery_next, (int)(nursery_next - nursery_start)));
3256 max_garbage_amount = nursery_next - nursery_start;
3257 g_assert (nursery_section->size >= max_garbage_amount);
3259 /* world must be stopped already */
3260 TV_GETTIME (all_atv);
3263 /* Pinning depends on this */
3264 clear_nursery_fragments (orig_nursery_next);
3267 time_minor_pre_collection_fragment_clear += TV_ELAPSED_MS (atv, btv);
3270 check_for_xdomain_refs ();
3272 nursery_section->next_data = nursery_next;
3274 major_start_nursery_collection ();
3276 gray_object_queue_init (&gray_queue);
3279 mono_stats.minor_gc_count ++;
3281 global_remset_cache_clear ();
3283 /* pin from pinned handles */
3285 pin_from_roots (nursery_start, nursery_next);
3286 /* identify pinned objects */
3287 optimize_pin_queue (0);
3288 next_pin_slot = pin_objects_from_addresses (nursery_section, pin_queue, pin_queue + next_pin_slot, nursery_start, nursery_next, &gray_queue);
3289 nursery_section->pin_queue_start = 0;
3290 nursery_section->pin_queue_end = next_pin_slot;
3292 time_minor_pinning += TV_ELAPSED_MS (btv, atv);
3293 DEBUG (2, fprintf (gc_debug_file, "Finding pinned pointers: %d in %d usecs\n", next_pin_slot, TV_ELAPSED (btv, atv)));
3294 DEBUG (4, fprintf (gc_debug_file, "Start scan with %d pinned objects\n", next_pin_slot));
3296 if (consistency_check_at_minor_collection)
3297 check_consistency ();
3300 * walk all the roots and copy the young objects to the old generation,
3301 * starting from to_space
3304 scan_from_remsets (nursery_start, nursery_next, &gray_queue);
3305 /* we don't have complete write barrier yet, so we scan all the old generation sections */
3307 time_minor_scan_remsets += TV_ELAPSED_MS (atv, btv);
3308 DEBUG (2, fprintf (gc_debug_file, "Old generation scan: %d usecs\n", TV_ELAPSED (atv, btv)));
3310 drain_gray_stack (&gray_queue);
3313 time_minor_scan_pinned += TV_ELAPSED_MS (btv, atv);
3314 /* registered roots, this includes static fields */
3315 scan_from_registered_roots (copy_object, nursery_start, nursery_next, ROOT_TYPE_NORMAL, &gray_queue);
3316 scan_from_registered_roots (copy_object, nursery_start, nursery_next, ROOT_TYPE_WBARRIER, &gray_queue);
3318 time_minor_scan_registered_roots += TV_ELAPSED_MS (atv, btv);
3320 scan_thread_data (nursery_start, nursery_next, TRUE);
3322 time_minor_scan_thread_data += TV_ELAPSED_MS (btv, atv);
3325 finish_gray_stack (nursery_start, nursery_next, GENERATION_NURSERY, &gray_queue);
3327 time_minor_finish_gray_stack += TV_ELAPSED_MS (btv, atv);
3329 /* walk the pin_queue, build up the fragment list of free memory, unmark
3330 * pinned objects as we go, memzero() the empty fragments so they are ready for the
3333 build_nursery_fragments (0, next_pin_slot);
3335 time_minor_fragment_creation += TV_ELAPSED_MS (atv, btv);
3336 DEBUG (2, fprintf (gc_debug_file, "Fragment creation: %d usecs, %lu bytes available\n", TV_ELAPSED (atv, btv), (unsigned long)fragment_total));
3338 if (consistency_check_at_minor_collection)
3339 check_major_refs ();
3341 major_finish_nursery_collection ();
3343 TV_GETTIME (all_btv);
3344 mono_stats.minor_gc_time_usecs += TV_ELAPSED (all_atv, all_btv);
3347 dump_heap ("minor", num_minor_gcs - 1, NULL);
3349 /* prepare the pin queue for the next collection */
3350 last_num_pinned = next_pin_slot;
3352 if (fin_ready_list || critical_fin_list) {
3353 DEBUG (4, fprintf (gc_debug_file, "Finalizer-thread wakeup: ready %d\n", num_ready_finalizers));
3354 mono_gc_finalize_notify ();
3358 g_assert (gray_object_queue_is_empty (&gray_queue));
3360 check_scan_starts ();
3362 binary_protocol_flush_buffers ();
3364 current_collection_generation = -1;
3366 return need_major_collection ();
3370 major_do_collection (const char *reason)
3372 LOSObject *bigobj, *prevbo;
3373 TV_DECLARE (all_atv);
3374 TV_DECLARE (all_btv);
3377 /* FIXME: only use these values for the precise scan
3378 * note that to_space pointers should be excluded anyway...
3380 char *heap_start = NULL;
3381 char *heap_end = (char*)-1;
3382 int old_num_major_sections = num_major_sections;
3383 int num_major_sections_saved, save_target, allowance_target;
3384 mword los_memory_saved, los_memory_alloced, old_los_memory_usage;
3387 * A domain could have been freed, resulting in
3388 * los_memory_usage being less than last_los_memory_usage.
3390 los_memory_alloced = los_memory_usage - MIN (last_los_memory_usage, los_memory_usage);
3391 old_los_memory_usage = los_memory_usage;
3393 //count_ref_nonref_objs ();
3394 //consistency_check ();
3397 binary_protocol_collection (GENERATION_OLD);
3398 check_scan_starts ();
3399 gray_object_queue_init (&gray_queue);
3402 DEBUG (1, fprintf (gc_debug_file, "Start major collection %d\n", num_major_gcs));
3404 mono_stats.major_gc_count ++;
3406 /* world must be stopped already */
3407 TV_GETTIME (all_atv);
3410 /* Pinning depends on this */
3411 clear_nursery_fragments (nursery_next);
3414 time_major_pre_collection_fragment_clear += TV_ELAPSED_MS (atv, btv);
3417 check_for_xdomain_refs ();
3419 nursery_section->next_data = nursery_real_end;
3420 /* we should also coalesce scanning from sections close to each other
3421 * and deal with pointers outside of the sections later.
3423 /* The remsets are not useful for a major collection */
3425 global_remset_cache_clear ();
3429 DEBUG (6, fprintf (gc_debug_file, "Collecting pinned addresses\n"));
3430 pin_from_roots ((void*)lowest_heap_address, (void*)highest_heap_address);
3431 optimize_pin_queue (0);
3434 * pin_queue now contains all candidate pointers, sorted and
3435 * uniqued. We must do two passes now to figure out which
3436 * objects are pinned.
3438 * The first is to find within the pin_queue the area for each
3439 * section. This requires that the pin_queue be sorted. We
3440 * also process the LOS objects and pinned chunks here.
3442 * The second, destructive, pass is to reduce the section
3443 * areas to pointers to the actually pinned objects.
3445 DEBUG (6, fprintf (gc_debug_file, "Pinning from sections\n"));
3446 /* first pass for the sections */
3447 find_section_pin_queue_start_end (nursery_section);
3448 major_find_pin_queue_start_ends (&gray_queue);
3449 /* identify possible pointers to the insize of large objects */
3450 DEBUG (6, fprintf (gc_debug_file, "Pinning from large objects\n"));
3451 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
3453 find_optimized_pin_queue_area (bigobj->data, (char*)bigobj->data + bigobj->size, &start, &end);
3455 pin_object (bigobj->data);
3456 /* FIXME: only enqueue if object has references */
3457 GRAY_OBJECT_ENQUEUE (&gray_queue, bigobj->data);
3459 pin_stats_register_object ((char*) bigobj->data, safe_object_get_size ((MonoObject*) bigobj->data));
3460 DEBUG (6, fprintf (gc_debug_file, "Marked large object %p (%s) size: %lu from roots\n", bigobj->data, safe_name (bigobj->data), (unsigned long)bigobj->size));
3463 /* second pass for the sections */
3464 pin_objects_in_section (nursery_section, &gray_queue);
3465 major_pin_objects (&gray_queue);
3468 time_major_pinning += TV_ELAPSED_MS (atv, btv);
3469 DEBUG (2, fprintf (gc_debug_file, "Finding pinned pointers: %d in %d usecs\n", next_pin_slot, TV_ELAPSED (atv, btv)));
3470 DEBUG (4, fprintf (gc_debug_file, "Start scan with %d pinned objects\n", next_pin_slot));
3472 major_init_to_space ();
3474 drain_gray_stack (&gray_queue);
3477 time_major_scan_pinned += TV_ELAPSED_MS (btv, atv);
3479 /* registered roots, this includes static fields */
3480 scan_from_registered_roots (major_copy_or_mark_object, heap_start, heap_end, ROOT_TYPE_NORMAL, &gray_queue);
3481 scan_from_registered_roots (major_copy_or_mark_object, heap_start, heap_end, ROOT_TYPE_WBARRIER, &gray_queue);
3483 time_major_scan_registered_roots += TV_ELAPSED_MS (atv, btv);
3486 /* FIXME: This is the wrong place for this, because it does
3488 scan_thread_data (heap_start, heap_end, TRUE);
3490 time_major_scan_thread_data += TV_ELAPSED_MS (btv, atv);
3493 time_major_scan_alloc_pinned += TV_ELAPSED_MS (atv, btv);
3495 /* scan the list of objects ready for finalization */
3496 scan_finalizer_entries (major_copy_or_mark_object, fin_ready_list, &gray_queue);
3497 scan_finalizer_entries (major_copy_or_mark_object, critical_fin_list, &gray_queue);
3499 time_major_scan_finalized += TV_ELAPSED_MS (btv, atv);
3500 DEBUG (2, fprintf (gc_debug_file, "Root scan: %d usecs\n", TV_ELAPSED (btv, atv)));
3503 time_major_scan_big_objects += TV_ELAPSED_MS (atv, btv);
3505 /* all the objects in the heap */
3506 finish_gray_stack (heap_start, heap_end, GENERATION_OLD, &gray_queue);
3508 time_major_finish_gray_stack += TV_ELAPSED_MS (btv, atv);
3510 /* sweep the big objects list */
3512 for (bigobj = los_object_list; bigobj;) {
3513 if (object_is_pinned (bigobj->data)) {
3514 unpin_object (bigobj->data);
3517 /* not referenced anywhere, so we can free it */
3519 prevbo->next = bigobj->next;
3521 los_object_list = bigobj->next;
3523 bigobj = bigobj->next;
3524 free_large_object (to_free);
3528 bigobj = bigobj->next;
3532 time_major_free_bigobjs += TV_ELAPSED_MS (atv, btv);
3537 time_major_los_sweep += TV_ELAPSED_MS (btv, atv);
3542 time_major_sweep += TV_ELAPSED_MS (atv, btv);
3544 /* walk the pin_queue, build up the fragment list of free memory, unmark
3545 * pinned objects as we go, memzero() the empty fragments so they are ready for the
3548 build_nursery_fragments (nursery_section->pin_queue_start, nursery_section->pin_queue_end);
3551 time_major_fragment_creation += TV_ELAPSED_MS (btv, atv);
3553 TV_GETTIME (all_btv);
3554 mono_stats.major_gc_time_usecs += TV_ELAPSED (all_atv, all_btv);
3557 dump_heap ("major", num_major_gcs - 1, reason);
3559 /* prepare the pin queue for the next collection */
3561 if (fin_ready_list || critical_fin_list) {
3562 DEBUG (4, fprintf (gc_debug_file, "Finalizer-thread wakeup: ready %d\n", num_ready_finalizers));
3563 mono_gc_finalize_notify ();
3567 g_assert (gray_object_queue_is_empty (&gray_queue));
3569 num_major_sections_saved = MAX (old_num_major_sections - num_major_sections, 0);
3570 los_memory_saved = MAX (old_los_memory_usage - los_memory_usage, 1);
3572 save_target = ((num_major_sections * MAJOR_SECTION_SIZE) + los_memory_saved) / 2;
3574 * We aim to allow the allocation of as many sections as is
3575 * necessary to reclaim save_target sections in the next
3576 * collection. We assume the collection pattern won't change.
3577 * In the last cycle, we had num_major_sections_saved for
3578 * minor_collection_sections_alloced. Assuming things won't
3579 * change, this must be the same ratio as save_target for
3580 * allowance_target, i.e.
3582 * num_major_sections_saved save_target
3583 * --------------------------------- == ----------------
3584 * minor_collection_sections_alloced allowance_target
3588 allowance_target = (mword)((double)save_target * (double)(minor_collection_sections_alloced * MAJOR_SECTION_SIZE + los_memory_alloced) / (double)(num_major_sections_saved * MAJOR_SECTION_SIZE + los_memory_saved));
3590 minor_collection_allowance = MAX (MIN (allowance_target, num_major_sections * MAJOR_SECTION_SIZE + los_memory_usage), MIN_MINOR_COLLECTION_ALLOWANCE);
3592 minor_collection_sections_alloced = 0;
3593 last_los_memory_usage = los_memory_usage;
3595 major_finish_major_collection ();
3597 check_scan_starts ();
3599 binary_protocol_flush_buffers ();
3601 //consistency_check ();
3605 major_collection (const char *reason)
3607 if (g_getenv ("MONO_GC_NO_MAJOR")) {
3608 collect_nursery (0);
3612 current_collection_generation = GENERATION_OLD;
3613 major_do_collection (reason);
3614 current_collection_generation = -1;
3618 * When deciding if it's better to collect or to expand, keep track
3619 * of how much garbage was reclaimed with the last collection: if it's too
3621 * This is called when we could not allocate a small object.
3623 static void __attribute__((noinline))
3624 minor_collect_or_expand_inner (size_t size)
3626 int do_minor_collection = 1;
3628 if (!nursery_section) {
3632 if (do_minor_collection) {
3634 if (collect_nursery (size))
3635 major_collection ("minor overflow");
3636 DEBUG (2, fprintf (gc_debug_file, "Heap size: %lu, LOS size: %lu\n", (unsigned long)total_alloc, (unsigned long)los_memory_usage));
3638 /* this also sets the proper pointers for the next allocation */
3639 if (!search_fragment_for_size (size)) {
3641 /* TypeBuilder and MonoMethod are killing mcs with fragmentation */
3642 DEBUG (1, fprintf (gc_debug_file, "nursery collection didn't find enough room for %zd alloc (%d pinned)\n", size, last_num_pinned));
3643 for (i = 0; i < last_num_pinned; ++i) {
3644 DEBUG (3, fprintf (gc_debug_file, "Bastard pinning obj %p (%s), size: %d\n", pin_queue [i], safe_name (pin_queue [i]), safe_object_get_size (pin_queue [i])));
3649 //report_internal_mem_usage ();
3653 * ######################################################################
3654 * ######## Memory allocation from the OS
3655 * ######################################################################
3656 * This section of code deals with getting memory from the OS and
3657 * allocating memory for GC-internal data structures.
3658 * Internal memory can be handled with a freelist for small objects.
3662 * Allocate a big chunk of memory from the OS (usually 64KB to several megabytes).
3663 * This must not require any lock.
3666 get_os_memory (size_t size, int activate)
3669 unsigned long prot_flags = activate? MONO_MMAP_READ|MONO_MMAP_WRITE: MONO_MMAP_NONE;
3671 prot_flags |= MONO_MMAP_PRIVATE | MONO_MMAP_ANON;
3672 size += pagesize - 1;
3673 size &= ~(pagesize - 1);
3674 ptr = mono_valloc (0, size, prot_flags);
3679 * Free the memory returned by get_os_memory (), returning it to the OS.
3682 free_os_memory (void *addr, size_t size)
3684 mono_vfree (addr, size);
3691 report_pinned_chunk (PinnedChunk *chunk, int seq) {
3693 int i, free_pages, num_free, free_mem;
3695 for (i = 0; i < chunk->num_pages; ++i) {
3696 if (!chunk->page_sizes [i])
3699 printf ("Pinned chunk %d at %p, size: %d, pages: %d, free: %d\n", seq, chunk, chunk->num_pages * FREELIST_PAGESIZE, chunk->num_pages, free_pages);
3700 free_mem = FREELIST_PAGESIZE * free_pages;
3701 for (i = 0; i < FREELIST_NUM_SLOTS; ++i) {
3702 if (!chunk->free_list [i])
3705 p = chunk->free_list [i];
3710 printf ("\tfree list of size %d, %d items\n", freelist_sizes [i], num_free);
3711 free_mem += freelist_sizes [i] * num_free;
3713 printf ("\tfree memory in chunk: %d\n", free_mem);
3719 static G_GNUC_UNUSED void
3720 report_internal_mem_usage (void) {
3723 printf ("Internal memory usage:\n");
3725 for (chunk = internal_chunk_list; chunk; chunk = chunk->block.next) {
3726 report_pinned_chunk (chunk, i++);
3728 printf ("Pinned memory usage:\n");
3729 major_report_pinned_memory_usage ();
3733 * Find the slot number in the freelist for memory chunks that
3734 * can contain @size objects.
3737 slot_for_size (size_t size)
3740 /* do a binary search or lookup table later. */
3741 for (slot = 0; slot < FREELIST_NUM_SLOTS; ++slot) {
3742 if (freelist_sizes [slot] >= size)
3745 g_assert_not_reached ();
3750 * Build a free list for @size memory chunks from the memory area between
3751 * start_page and end_page.
3754 build_freelist (PinnedChunk *chunk, int slot, int size, char *start_page, char *end_page)
3758 /*g_print ("building freelist for slot %d, size %d in %p\n", slot, size, chunk);*/
3759 p = (void**)start_page;
3760 end = (void**)(end_page - size);
3761 g_assert (!chunk->free_list [slot]);
3762 chunk->free_list [slot] = p;
3763 while ((char*)p + size <= (char*)end) {
3765 *p = (void*)((char*)p + size);
3769 /*g_print ("%d items created, max: %d\n", count, (end_page - start_page) / size);*/
3772 /* LOCKING: requires the internal allocator lock to be held */
3774 alloc_pinned_chunk (void)
3778 int size = PINNED_CHUNK_SIZE;
3780 chunk = get_os_memory_aligned (size, size, TRUE);
3781 chunk->block.role = MEMORY_ROLE_PINNED;
3783 update_heap_boundaries ((mword)chunk, ((mword)chunk + size));
3784 total_alloc += size;
3785 pinned_chunk_bytes_alloced += size;
3787 /* setup the bookeeping fields */
3788 chunk->num_pages = size / FREELIST_PAGESIZE;
3789 offset = G_STRUCT_OFFSET (PinnedChunk, data);
3790 chunk->page_sizes = (void*)((char*)chunk + offset);
3791 offset += sizeof (int) * chunk->num_pages;
3792 offset = ALIGN_UP (offset);
3793 chunk->free_list = (void*)((char*)chunk + offset);
3794 offset += sizeof (void*) * FREELIST_NUM_SLOTS;
3795 offset = ALIGN_UP (offset);
3796 chunk->start_data = (void*)((char*)chunk + offset);
3798 /* allocate the first page to the freelist */
3799 chunk->page_sizes [0] = PINNED_FIRST_SLOT_SIZE;
3800 build_freelist (chunk, slot_for_size (PINNED_FIRST_SLOT_SIZE), PINNED_FIRST_SLOT_SIZE, chunk->start_data, ((char*)chunk + FREELIST_PAGESIZE));
3801 DEBUG (4, fprintf (gc_debug_file, "Allocated pinned chunk %p, size: %d\n", chunk, size));
3805 /* assumes freelist for slot is empty, so try to alloc a new page */
3807 get_chunk_freelist (PinnedChunk *chunk, int slot)
3811 p = chunk->free_list [slot];
3813 chunk->free_list [slot] = *p;
3816 for (i = 0; i < chunk->num_pages; ++i) {
3818 if (chunk->page_sizes [i])
3820 size = freelist_sizes [slot];
3821 chunk->page_sizes [i] = size;
3822 build_freelist (chunk, slot, size, (char*)chunk + FREELIST_PAGESIZE * i, (char*)chunk + FREELIST_PAGESIZE * (i + 1));
3826 p = chunk->free_list [slot];
3828 chunk->free_list [slot] = *p;
3834 /* used for the GC-internal data structures */
3836 get_internal_mem (size_t size, int type)
3840 PinnedChunk *pchunk;
3842 LOCK_INTERNAL_ALLOCATOR;
3844 HEAVY_STAT (++stat_internal_alloc);
3846 if (size > freelist_sizes [FREELIST_NUM_SLOTS - 1]) {
3847 LargeInternalMemHeader *mh;
3849 UNLOCK_INTERNAL_ALLOCATOR;
3851 size += sizeof (LargeInternalMemHeader);
3852 mh = get_os_memory (size, TRUE);
3853 mh->magic = LARGE_INTERNAL_MEM_HEADER_MAGIC;
3855 /* FIXME: do a CAS here */
3856 large_internal_bytes_alloced += size;
3860 slot = slot_for_size (size);
3861 g_assert (size <= freelist_sizes [slot]);
3863 small_internal_mem_bytes [type] += freelist_sizes [slot];
3865 for (pchunk = internal_chunk_list; pchunk; pchunk = pchunk->block.next) {
3866 void **p = pchunk->free_list [slot];
3867 HEAVY_STAT (++stat_internal_alloc_loop1);
3869 pchunk->free_list [slot] = *p;
3871 UNLOCK_INTERNAL_ALLOCATOR;
3873 memset (p, 0, size);
3877 for (pchunk = internal_chunk_list; pchunk; pchunk = pchunk->block.next) {
3878 HEAVY_STAT (++stat_internal_alloc_loop2);
3879 res = get_chunk_freelist (pchunk, slot);
3881 UNLOCK_INTERNAL_ALLOCATOR;
3883 memset (res, 0, size);
3887 pchunk = alloc_pinned_chunk ();
3888 /* FIXME: handle OOM */
3889 pchunk->block.next = internal_chunk_list;
3890 internal_chunk_list = pchunk;
3891 res = get_chunk_freelist (pchunk, slot);
3893 UNLOCK_INTERNAL_ALLOCATOR;
3895 memset (res, 0, size);
3900 free_internal_mem (void *addr, int type)
3902 PinnedChunk *pchunk;
3903 LargeInternalMemHeader *mh;
3907 LOCK_INTERNAL_ALLOCATOR;
3909 for (pchunk = internal_chunk_list; pchunk; pchunk = pchunk->block.next) {
3910 /*printf ("trying to free %p in %p (pages: %d)\n", addr, pchunk, pchunk->num_pages);*/
3911 if (addr >= (void*)pchunk && (char*)addr < (char*)pchunk + pchunk->num_pages * FREELIST_PAGESIZE) {
3912 int offset = (char*)addr - (char*)pchunk;
3913 int page = offset / FREELIST_PAGESIZE;
3914 int slot = slot_for_size (pchunk->page_sizes [page]);
3916 *p = pchunk->free_list [slot];
3917 pchunk->free_list [slot] = p;
3919 small_internal_mem_bytes [type] -= freelist_sizes [slot];
3921 UNLOCK_INTERNAL_ALLOCATOR;
3927 UNLOCK_INTERNAL_ALLOCATOR;
3929 mh = (LargeInternalMemHeader*)((char*)addr - G_STRUCT_OFFSET (LargeInternalMemHeader, data));
3930 g_assert (mh->magic == LARGE_INTERNAL_MEM_HEADER_MAGIC);
3931 /* FIXME: do a CAS */
3932 large_internal_bytes_alloced -= mh->size;
3933 free_os_memory (mh, mh->size);
3937 * ######################################################################
3938 * ######## Object allocation
3939 * ######################################################################
3940 * This section of code deals with allocating memory for objects.
3941 * There are several ways:
3942 * *) allocate large objects
3943 * *) allocate normal objects
3944 * *) fast lock-free allocation
3945 * *) allocation of pinned objects
3949 setup_fragment (Fragment *frag, Fragment *prev, size_t size)
3951 /* remove from the list */
3953 prev->next = frag->next;
3955 nursery_fragments = frag->next;
3956 nursery_next = frag->fragment_start;
3957 nursery_frag_real_end = frag->fragment_end;
3959 DEBUG (4, fprintf (gc_debug_file, "Using nursery fragment %p-%p, size: %td (req: %zd)\n", nursery_next, nursery_frag_real_end, nursery_frag_real_end - nursery_next, size));
3960 frag->next = fragment_freelist;
3961 fragment_freelist = frag;
3964 /* check if we have a suitable fragment in nursery_fragments to be able to allocate
3965 * an object of size @size
3966 * Return FALSE if not found (which means we need a collection)
3969 search_fragment_for_size (size_t size)
3971 Fragment *frag, *prev;
3972 DEBUG (4, fprintf (gc_debug_file, "Searching nursery fragment %p, size: %zd\n", nursery_frag_real_end, size));
3974 if (nursery_frag_real_end > nursery_next && nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
3975 /* Clear the remaining space, pinning depends on this */
3976 memset (nursery_next, 0, nursery_frag_real_end - nursery_next);
3979 for (frag = nursery_fragments; frag; frag = frag->next) {
3980 if (size <= (frag->fragment_end - frag->fragment_start)) {
3981 setup_fragment (frag, prev, size);
3990 * Same as search_fragment_for_size but if search for @desired_size fails, try to satisfy @minimum_size.
3991 * This improves nursery usage.
3994 search_fragment_for_size_range (size_t desired_size, size_t minimum_size)
3996 Fragment *frag, *prev, *min_prev;
3997 DEBUG (4, fprintf (gc_debug_file, "Searching nursery fragment %p, desired size: %zd minimum size %zd\n", nursery_frag_real_end, desired_size, minimum_size));
3999 if (nursery_frag_real_end > nursery_next && nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
4000 /* Clear the remaining space, pinning depends on this */
4001 memset (nursery_next, 0, nursery_frag_real_end - nursery_next);
4003 min_prev = GINT_TO_POINTER (-1);
4006 for (frag = nursery_fragments; frag; frag = frag->next) {
4007 int frag_size = frag->fragment_end - frag->fragment_start;
4008 if (desired_size <= frag_size) {
4009 setup_fragment (frag, prev, desired_size);
4010 return desired_size;
4012 if (minimum_size <= frag_size)
4018 if (min_prev != GINT_TO_POINTER (-1)) {
4021 frag = min_prev->next;
4023 frag = nursery_fragments;
4025 frag_size = frag->fragment_end - frag->fragment_start;
4026 HEAVY_STAT (++stat_wasted_fragments_used);
4027 HEAVY_STAT (stat_wasted_fragments_bytes += frag_size);
4029 setup_fragment (frag, min_prev, minimum_size);
4037 alloc_degraded (MonoVTable *vtable, size_t size)
4039 if (need_major_collection ()) {
4041 major_collection ("degraded overflow");
4045 return major_alloc_degraded (vtable, size);
4049 * Provide a variant that takes just the vtable for small fixed-size objects.
4050 * The aligned size is already computed and stored in vt->gc_descr.
4051 * Note: every SCAN_START_SIZE or so we are given the chance to do some special
4052 * processing. We can keep track of where objects start, for example,
4053 * so when we scan the thread stacks for pinned objects, we can start
4054 * a search for the pinned object in SCAN_START_SIZE chunks.
4057 mono_gc_alloc_obj_nolock (MonoVTable *vtable, size_t size)
4059 /* FIXME: handle OOM */
4064 HEAVY_STAT (++stat_objects_alloced);
4065 if (size <= MAX_SMALL_OBJ_SIZE)
4066 HEAVY_STAT (stat_bytes_alloced += size);
4068 HEAVY_STAT (stat_bytes_alloced_los += size);
4070 size = ALIGN_UP (size);
4072 g_assert (vtable->gc_descr);
4074 if (G_UNLIKELY (collect_before_allocs)) {
4075 if (nursery_section) {
4077 collect_nursery (0);
4079 if (!degraded_mode && !search_fragment_for_size (size)) {
4081 g_assert_not_reached ();
4087 * We must already have the lock here instead of after the
4088 * fast path because we might be interrupted in the fast path
4089 * (after confirming that new_next < TLAB_TEMP_END) by the GC,
4090 * and we'll end up allocating an object in a fragment which
4091 * no longer belongs to us.
4093 * The managed allocator does not do this, but it's treated
4094 * specially by the world-stopping code.
4097 if (size > MAX_SMALL_OBJ_SIZE) {
4098 p = alloc_large_inner (vtable, size);
4100 /* tlab_next and tlab_temp_end are TLS vars so accessing them might be expensive */
4102 p = (void**)TLAB_NEXT;
4103 /* FIXME: handle overflow */
4104 new_next = (char*)p + size;
4105 TLAB_NEXT = new_next;
4107 if (G_LIKELY (new_next < TLAB_TEMP_END)) {
4111 * FIXME: We might need a memory barrier here so the change to tlab_next is
4112 * visible before the vtable store.
4115 DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
4116 binary_protocol_alloc (p , vtable, size);
4117 g_assert (*p == NULL);
4120 g_assert (TLAB_NEXT == new_next);
4127 /* there are two cases: the object is too big or we run out of space in the TLAB */
4128 /* we also reach here when the thread does its first allocation after a minor
4129 * collection, since the tlab_ variables are initialized to NULL.
4130 * there can be another case (from ORP), if we cooperate with the runtime a bit:
4131 * objects that need finalizers can have the high bit set in their size
4132 * so the above check fails and we can readily add the object to the queue.
4133 * This avoids taking again the GC lock when registering, but this is moot when
4134 * doing thread-local allocation, so it may not be a good idea.
4136 g_assert (TLAB_NEXT == new_next);
4137 if (TLAB_NEXT >= TLAB_REAL_END) {
4139 * Run out of space in the TLAB. When this happens, some amount of space
4140 * remains in the TLAB, but not enough to satisfy the current allocation
4141 * request. Currently, we retire the TLAB in all cases, later we could
4142 * keep it if the remaining space is above a treshold, and satisfy the
4143 * allocation directly from the nursery.
4146 /* when running in degraded mode, we continue allocing that way
4147 * for a while, to decrease the number of useless nursery collections.
4149 if (degraded_mode && degraded_mode < DEFAULT_NURSERY_SIZE) {
4150 p = alloc_degraded (vtable, size);
4151 binary_protocol_alloc_degraded (p, vtable, size);
4155 /*FIXME This codepath is current deadcode since tlab_size > MAX_SMALL_OBJ_SIZE*/
4156 if (size > tlab_size) {
4157 /* Allocate directly from the nursery */
4158 if (nursery_next + size >= nursery_frag_real_end) {
4159 if (!search_fragment_for_size (size)) {
4160 minor_collect_or_expand_inner (size);
4161 if (degraded_mode) {
4162 p = alloc_degraded (vtable, size);
4163 binary_protocol_alloc_degraded (p, vtable, size);
4169 p = (void*)nursery_next;
4170 nursery_next += size;
4171 if (nursery_next > nursery_frag_real_end) {
4176 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
4177 memset (p, 0, size);
4179 int alloc_size = tlab_size;
4180 int available_in_nursery = nursery_frag_real_end - nursery_next;
4182 DEBUG (3, fprintf (gc_debug_file, "Retire TLAB: %p-%p [%ld]\n", TLAB_START, TLAB_REAL_END, (long)(TLAB_REAL_END - TLAB_NEXT - size)));
4184 if (alloc_size >= available_in_nursery) {
4185 if (available_in_nursery > MAX_NURSERY_TLAB_WASTE && available_in_nursery > size) {
4186 alloc_size = available_in_nursery;
4188 alloc_size = search_fragment_for_size_range (tlab_size, size);
4190 alloc_size = tlab_size;
4191 minor_collect_or_expand_inner (tlab_size);
4192 if (degraded_mode) {
4193 p = alloc_degraded (vtable, size);
4194 binary_protocol_alloc_degraded (p, vtable, size);
4201 /* Allocate a new TLAB from the current nursery fragment */
4202 TLAB_START = nursery_next;
4203 nursery_next += alloc_size;
4204 TLAB_NEXT = TLAB_START;
4205 TLAB_REAL_END = TLAB_START + alloc_size;
4206 TLAB_TEMP_END = TLAB_START + MIN (SCAN_START_SIZE, alloc_size);
4208 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
4209 memset (TLAB_START, 0, alloc_size);
4211 /* Allocate from the TLAB */
4212 p = (void*)TLAB_NEXT;
4214 g_assert (TLAB_NEXT <= TLAB_REAL_END);
4216 nursery_section->scan_starts [((char*)p - (char*)nursery_section->data)/SCAN_START_SIZE] = (char*)p;
4219 /* Reached tlab_temp_end */
4221 /* record the scan start so we can find pinned objects more easily */
4222 nursery_section->scan_starts [((char*)p - (char*)nursery_section->data)/SCAN_START_SIZE] = (char*)p;
4223 /* we just bump tlab_temp_end as well */
4224 TLAB_TEMP_END = MIN (TLAB_REAL_END, TLAB_NEXT + SCAN_START_SIZE);
4225 DEBUG (5, fprintf (gc_debug_file, "Expanding local alloc: %p-%p\n", TLAB_NEXT, TLAB_TEMP_END));
4229 DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
4230 binary_protocol_alloc (p, vtable, size);
4237 mono_gc_try_alloc_obj_nolock (MonoVTable *vtable, size_t size)
4243 size = ALIGN_UP (size);
4245 g_assert (vtable->gc_descr);
4246 if (size <= MAX_SMALL_OBJ_SIZE) {
4247 /* tlab_next and tlab_temp_end are TLS vars so accessing them might be expensive */
4249 p = (void**)TLAB_NEXT;
4250 /* FIXME: handle overflow */
4251 new_next = (char*)p + size;
4252 TLAB_NEXT = new_next;
4254 if (G_LIKELY (new_next < TLAB_TEMP_END)) {
4258 * FIXME: We might need a memory barrier here so the change to tlab_next is
4259 * visible before the vtable store.
4262 HEAVY_STAT (++stat_objects_alloced);
4263 HEAVY_STAT (stat_bytes_alloced += size);
4265 DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
4266 binary_protocol_alloc (p, vtable, size);
4267 g_assert (*p == NULL);
4270 g_assert (TLAB_NEXT == new_next);
4279 mono_gc_alloc_obj (MonoVTable *vtable, size_t size)
4282 #ifndef DISABLE_CRITICAL_REGION
4284 ENTER_CRITICAL_REGION;
4285 res = mono_gc_try_alloc_obj_nolock (vtable, size);
4287 EXIT_CRITICAL_REGION;
4290 EXIT_CRITICAL_REGION;
4293 res = mono_gc_alloc_obj_nolock (vtable, size);
4299 mono_gc_alloc_vector (MonoVTable *vtable, size_t size, uintptr_t max_length)
4302 #ifndef DISABLE_CRITICAL_REGION
4304 ENTER_CRITICAL_REGION;
4305 arr = mono_gc_try_alloc_obj_nolock (vtable, size);
4307 arr->max_length = max_length;
4308 EXIT_CRITICAL_REGION;
4311 EXIT_CRITICAL_REGION;
4316 arr = mono_gc_alloc_obj_nolock (vtable, size);
4317 arr->max_length = max_length;
4325 mono_gc_alloc_array (MonoVTable *vtable, size_t size, uintptr_t max_length, uintptr_t bounds_size)
4328 MonoArrayBounds *bounds;
4332 arr = mono_gc_alloc_obj_nolock (vtable, size);
4333 arr->max_length = max_length;
4335 bounds = (MonoArrayBounds*)((char*)arr + size - bounds_size);
4336 arr->bounds = bounds;
4344 mono_gc_alloc_string (MonoVTable *vtable, size_t size, gint32 len)
4347 #ifndef DISABLE_CRITICAL_REGION
4349 ENTER_CRITICAL_REGION;
4350 str = mono_gc_try_alloc_obj_nolock (vtable, size);
4353 EXIT_CRITICAL_REGION;
4356 EXIT_CRITICAL_REGION;
4361 str = mono_gc_alloc_obj_nolock (vtable, size);
4370 * To be used for interned strings and possibly MonoThread, reflection handles.
4371 * We may want to explicitly free these objects.
4374 mono_gc_alloc_pinned_obj (MonoVTable *vtable, size_t size)
4376 /* FIXME: handle OOM */
4378 size = ALIGN_UP (size);
4380 if (size > MAX_SMALL_OBJ_SIZE) {
4381 /* large objects are always pinned anyway */
4382 p = alloc_large_inner (vtable, size);
4384 DEBUG (9, g_assert (vtable->klass->inited));
4385 p = major_alloc_small_pinned_obj (size, vtable->klass->has_references);
4387 DEBUG (6, fprintf (gc_debug_file, "Allocated pinned object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
4388 binary_protocol_alloc_pinned (p, vtable, size);
4395 * ######################################################################
4396 * ######## Finalization support
4397 * ######################################################################
4401 * this is valid for the nursery: if the object has been forwarded it means it's
4402 * still refrenced from a root. If it is pinned it's still alive as well.
4403 * Return TRUE if @obj is ready to be finalized.
4405 #define object_is_fin_ready(obj) (!object_is_pinned (obj) && !object_is_forwarded (obj))
4408 is_critical_finalizer (FinalizeEntry *entry)
4413 if (!mono_defaults.critical_finalizer_object)
4416 obj = entry->object;
4417 class = ((MonoVTable*)LOAD_VTABLE (obj))->klass;
4419 return mono_class_has_parent (class, mono_defaults.critical_finalizer_object);
4423 queue_finalization_entry (FinalizeEntry *entry) {
4424 if (is_critical_finalizer (entry)) {
4425 entry->next = critical_fin_list;
4426 critical_fin_list = entry;
4428 entry->next = fin_ready_list;
4429 fin_ready_list = entry;
4433 /* LOCKING: requires that the GC lock is held */
4435 rehash_fin_table (FinalizeEntryHashTable *hash_table)
4437 FinalizeEntry **finalizable_hash = hash_table->table;
4438 mword finalizable_hash_size = hash_table->size;
4441 FinalizeEntry **new_hash;
4442 FinalizeEntry *entry, *next;
4443 int new_size = g_spaced_primes_closest (hash_table->num_registered);
4445 new_hash = get_internal_mem (new_size * sizeof (FinalizeEntry*), INTERNAL_MEM_FIN_TABLE);
4446 for (i = 0; i < finalizable_hash_size; ++i) {
4447 for (entry = finalizable_hash [i]; entry; entry = next) {
4448 hash = mono_object_hash (entry->object) % new_size;
4450 entry->next = new_hash [hash];
4451 new_hash [hash] = entry;
4454 free_internal_mem (finalizable_hash, INTERNAL_MEM_FIN_TABLE);
4455 hash_table->table = new_hash;
4456 hash_table->size = new_size;
4459 /* LOCKING: requires that the GC lock is held */
4461 rehash_fin_table_if_necessary (FinalizeEntryHashTable *hash_table)
4463 if (hash_table->num_registered >= hash_table->size * 2)
4464 rehash_fin_table (hash_table);
4467 /* LOCKING: requires that the GC lock is held */
4469 finalize_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, int generation, GrayQueue *queue)
4471 FinalizeEntryHashTable *hash_table = get_finalize_entry_hash_table (generation);
4472 FinalizeEntry *entry, *prev;
4474 FinalizeEntry **finalizable_hash = hash_table->table;
4475 mword finalizable_hash_size = hash_table->size;
4479 for (i = 0; i < finalizable_hash_size; ++i) {
4481 for (entry = finalizable_hash [i]; entry;) {
4482 if ((char*)entry->object >= start && (char*)entry->object < end && !major_is_object_live (entry->object)) {
4483 gboolean is_fin_ready = object_is_fin_ready (entry->object);
4484 char *copy = entry->object;
4485 copy_func ((void**)©, queue);
4488 FinalizeEntry *next;
4489 /* remove and put in fin_ready_list */
4491 prev->next = entry->next;
4493 finalizable_hash [i] = entry->next;
4495 num_ready_finalizers++;
4496 hash_table->num_registered--;
4497 queue_finalization_entry (entry);
4498 /* Make it survive */
4499 from = entry->object;
4500 entry->object = copy;
4501 DEBUG (5, fprintf (gc_debug_file, "Queueing object for finalization: %p (%s) (was at %p) (%d/%d)\n", entry->object, safe_name (entry->object), from, num_ready_finalizers, hash_table->num_registered));
4505 char *from = entry->object;
4506 if (hash_table == &minor_finalizable_hash && !ptr_in_nursery (copy)) {
4507 FinalizeEntry *next = entry->next;
4508 unsigned int major_hash;
4509 /* remove from the list */
4511 prev->next = entry->next;
4513 finalizable_hash [i] = entry->next;
4514 hash_table->num_registered--;
4516 entry->object = copy;
4518 /* insert it into the major hash */
4519 rehash_fin_table_if_necessary (&major_finalizable_hash);
4520 major_hash = mono_object_hash ((MonoObject*) copy) %
4521 major_finalizable_hash.size;
4522 entry->next = major_finalizable_hash.table [major_hash];
4523 major_finalizable_hash.table [major_hash] = entry;
4524 major_finalizable_hash.num_registered++;
4526 DEBUG (5, fprintf (gc_debug_file, "Promoting finalization of object %p (%s) (was at %p) to major table\n", copy, safe_name (copy), from));
4531 /* update pointer */
4532 DEBUG (5, fprintf (gc_debug_file, "Updating object for finalization: %p (%s) (was at %p)\n", entry->object, safe_name (entry->object), from));
4533 entry->object = copy;
4538 entry = entry->next;
4544 object_is_reachable (char *object, char *start, char *end)
4546 /*This happens for non nursery objects during minor collections. We just treat all objects as alive.*/
4547 if (object < start || object >= end)
4549 return !object_is_fin_ready (object) || major_is_object_live (object);
4552 /* LOCKING: requires that the GC lock is held */
4554 null_ephemerons_for_domain (MonoDomain *domain)
4556 EphemeronLinkNode *current = ephemeron_list, *prev = NULL;
4559 MonoObject *object = (MonoObject*)current->array;
4561 if (object && !object->vtable) {
4562 EphemeronLinkNode *tmp = current;
4565 prev->next = current->next;
4567 ephemeron_list = current->next;
4569 current = current->next;
4570 free_internal_mem (tmp, INTERNAL_MEM_EPHEMERON_LINK);
4573 current = current->next;
4578 /* LOCKING: requires that the GC lock is held */
4580 clear_unreachable_ephemerons (CopyOrMarkObjectFunc copy_func, char *start, char *end, GrayQueue *queue)
4582 int was_in_nursery, was_promoted;
4583 EphemeronLinkNode *current = ephemeron_list, *prev = NULL;
4585 Ephemeron *cur, *array_end;
4589 char *object = current->array;
4591 if (!object_is_reachable (object, start, end)) {
4592 EphemeronLinkNode *tmp = current;
4594 DEBUG (5, fprintf (gc_debug_file, "Dead Ephemeron array at %p\n", object));
4597 prev->next = current->next;
4599 ephemeron_list = current->next;
4601 current = current->next;
4602 free_internal_mem (tmp, INTERNAL_MEM_EPHEMERON_LINK);
4607 was_in_nursery = ptr_in_nursery (object);
4608 copy_func ((void**)&object, queue);
4609 current->array = object;
4611 /*The array was promoted, add global remsets for key/values left behind in nursery.*/
4612 was_promoted = was_in_nursery && !ptr_in_nursery (object);
4614 DEBUG (5, fprintf (gc_debug_file, "Clearing unreachable entries for ephemeron array at %p\n", object));
4616 array = (MonoArray*)object;
4617 cur = mono_array_addr (array, Ephemeron, 0);
4618 array_end = cur + mono_array_length_fast (array);
4619 tombstone = (char*)((MonoVTable*)LOAD_VTABLE (object))->domain->ephemeron_tombstone;
4621 for (; cur < array_end; ++cur) {
4622 char *key = (char*)cur->key;
4624 if (!key || key == tombstone)
4627 DEBUG (5, fprintf (gc_debug_file, "[%td] key %p (%s) value %p (%s)\n", cur - mono_array_addr (array, Ephemeron, 0),
4628 key, object_is_reachable (key, start, end) ? "reachable" : "unreachable",
4629 cur->value, cur->value && object_is_reachable (cur->value, start, end) ? "reachable" : "unreachable"));
4631 if (!object_is_reachable (key, start, end)) {
4632 cur->key = tombstone;
4638 if (ptr_in_nursery (key)) {/*key was not promoted*/
4639 DEBUG (5, fprintf (gc_debug_file, "\tAdded remset to key %p\n", key));
4640 add_to_global_remset (&cur->key);
4642 if (ptr_in_nursery (cur->value)) {/*value was not promoted*/
4643 DEBUG (5, fprintf (gc_debug_file, "\tAdded remset to value %p\n", cur->value));
4644 add_to_global_remset (&cur->value);
4649 current = current->next;
4653 /* LOCKING: requires that the GC lock is held */
4655 mark_ephemerons_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, GrayQueue *queue)
4657 int nothing_marked = 1;
4658 EphemeronLinkNode *current = ephemeron_list;
4660 Ephemeron *cur, *array_end;
4663 for (current = ephemeron_list; current; current = current->next) {
4664 char *object = current->array;
4665 DEBUG (5, fprintf (gc_debug_file, "Ephemeron array at %p\n", object));
4667 /*We ignore arrays in old gen during minor collections since all objects are promoted by the remset machinery.*/
4668 if (object < start || object >= end)
4671 /*It has to be alive*/
4672 if (!object_is_reachable (object, start, end)) {
4673 DEBUG (5, fprintf (gc_debug_file, "\tnot reachable\n"));
4677 copy_func ((void**)&object, queue);
4679 array = (MonoArray*)object;
4680 cur = mono_array_addr (array, Ephemeron, 0);
4681 array_end = cur + mono_array_length_fast (array);
4682 tombstone = (char*)((MonoVTable*)LOAD_VTABLE (object))->domain->ephemeron_tombstone;
4684 for (; cur < array_end; ++cur) {
4685 char *key = cur->key;
4687 if (!key || key == tombstone)
4690 DEBUG (5, fprintf (gc_debug_file, "[%td] key %p (%s) value %p (%s)\n", cur - mono_array_addr (array, Ephemeron, 0),
4691 key, object_is_reachable (key, start, end) ? "reachable" : "unreachable",
4692 cur->value, cur->value && object_is_reachable (cur->value, start, end) ? "reachable" : "unreachable"));
4694 if (object_is_reachable (key, start, end)) {
4695 char *value = cur->value;
4697 copy_func ((void**)&cur->key, queue);
4699 if (!object_is_reachable (value, start, end))
4701 copy_func ((void**)&cur->value, queue);
4707 DEBUG (5, fprintf (gc_debug_file, "Ephemeron run finished. Is it done %d\n", nothing_marked));
4708 return nothing_marked;
4711 /* LOCKING: requires that the GC lock is held */
4713 null_link_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, int generation, GrayQueue *queue)
4715 DisappearingLinkHashTable *hash = get_dislink_hash_table (generation);
4716 DisappearingLink **disappearing_link_hash = hash->table;
4717 int disappearing_link_hash_size = hash->size;
4718 DisappearingLink *entry, *prev;
4720 if (!hash->num_links)
4722 for (i = 0; i < disappearing_link_hash_size; ++i) {
4724 for (entry = disappearing_link_hash [i]; entry;) {
4725 char *object = DISLINK_OBJECT (entry);
4726 if (object >= start && object < end && !major_is_object_live (object)) {
4727 gboolean track = DISLINK_TRACK (entry);
4728 if (!track && object_is_fin_ready (object)) {
4729 void **p = entry->link;
4730 DisappearingLink *old;
4732 /* remove from list */
4734 prev->next = entry->next;
4736 disappearing_link_hash [i] = entry->next;
4737 DEBUG (5, fprintf (gc_debug_file, "Dislink nullified at %p to GCed object %p\n", p, object));
4739 free_internal_mem (entry, INTERNAL_MEM_DISLINK);
4744 char *copy = object;
4745 copy_func ((void**)©, queue);
4747 /* Update pointer if it's moved. If the object
4748 * has been moved out of the nursery, we need to
4749 * remove the link from the minor hash table to
4752 * FIXME: what if an object is moved earlier?
4755 if (hash == &minor_disappearing_link_hash && !ptr_in_nursery (copy)) {
4756 void **link = entry->link;
4757 DisappearingLink *old;
4758 /* remove from list */
4760 prev->next = entry->next;
4762 disappearing_link_hash [i] = entry->next;
4764 free_internal_mem (entry, INTERNAL_MEM_DISLINK);
4768 add_or_remove_disappearing_link ((MonoObject*)copy, link,
4769 track, GENERATION_OLD);
4771 DEBUG (5, fprintf (gc_debug_file, "Upgraded dislink at %p to major because object %p moved to %p\n", link, object, copy));
4775 /* We set the track resurrection bit to
4776 * FALSE if the object is to be finalized
4777 * so that the object can be collected in
4778 * the next cycle (i.e. after it was
4781 *entry->link = HIDE_POINTER (copy,
4782 object_is_fin_ready (object) ? FALSE : track);
4783 DEBUG (5, fprintf (gc_debug_file, "Updated dislink at %p to %p\n", entry->link, DISLINK_OBJECT (entry)));
4788 entry = entry->next;
4793 /* LOCKING: requires that the GC lock is held */
4795 null_links_for_domain (MonoDomain *domain, int generation)
4797 DisappearingLinkHashTable *hash = get_dislink_hash_table (generation);
4798 DisappearingLink **disappearing_link_hash = hash->table;
4799 int disappearing_link_hash_size = hash->size;
4800 DisappearingLink *entry, *prev;
4802 for (i = 0; i < disappearing_link_hash_size; ++i) {
4804 for (entry = disappearing_link_hash [i]; entry; ) {
4805 char *object = DISLINK_OBJECT (entry);
4806 if (object && !((MonoObject*)object)->vtable) {
4807 DisappearingLink *next = entry->next;
4812 disappearing_link_hash [i] = next;
4814 if (*(entry->link)) {
4815 *(entry->link) = NULL;
4816 g_warning ("Disappearing link %p not freed", entry->link);
4818 free_internal_mem (entry, INTERNAL_MEM_DISLINK);
4825 entry = entry->next;
4830 /* LOCKING: requires that the GC lock is held */
4832 finalizers_for_domain (MonoDomain *domain, MonoObject **out_array, int out_size,
4833 FinalizeEntryHashTable *hash_table)
4835 FinalizeEntry **finalizable_hash = hash_table->table;
4836 mword finalizable_hash_size = hash_table->size;
4837 FinalizeEntry *entry, *prev;
4840 if (no_finalize || !out_size || !out_array)
4843 for (i = 0; i < finalizable_hash_size; ++i) {
4845 for (entry = finalizable_hash [i]; entry;) {
4846 if (mono_object_domain (entry->object) == domain) {
4847 FinalizeEntry *next;
4848 /* remove and put in out_array */
4850 prev->next = entry->next;
4852 finalizable_hash [i] = entry->next;
4854 hash_table->num_registered--;
4855 out_array [count ++] = entry->object;
4856 DEBUG (5, fprintf (gc_debug_file, "Collecting object for finalization: %p (%s) (%d/%d)\n", entry->object, safe_name (entry->object), num_ready_finalizers, hash_table->num_registered));
4858 if (count == out_size)
4863 entry = entry->next;
4870 * mono_gc_finalizers_for_domain:
4871 * @domain: the unloading appdomain
4872 * @out_array: output array
4873 * @out_size: size of output array
4875 * Store inside @out_array up to @out_size objects that belong to the unloading
4876 * appdomain @domain. Returns the number of stored items. Can be called repeteadly
4877 * until it returns 0.
4878 * The items are removed from the finalizer data structure, so the caller is supposed
4880 * @out_array should be on the stack to allow the GC to know the objects are still alive.
4883 mono_gc_finalizers_for_domain (MonoDomain *domain, MonoObject **out_array, int out_size)
4888 result = finalizers_for_domain (domain, out_array, out_size, &minor_finalizable_hash);
4889 if (result < out_size) {
4890 result += finalizers_for_domain (domain, out_array + result, out_size - result,
4891 &major_finalizable_hash);
4899 register_for_finalization (MonoObject *obj, void *user_data, int generation)
4901 FinalizeEntryHashTable *hash_table = get_finalize_entry_hash_table (generation);
4902 FinalizeEntry **finalizable_hash;
4903 mword finalizable_hash_size;
4904 FinalizeEntry *entry, *prev;
4908 g_assert (user_data == NULL || user_data == mono_gc_run_finalize);
4909 hash = mono_object_hash (obj);
4911 rehash_fin_table_if_necessary (hash_table);
4912 finalizable_hash = hash_table->table;
4913 finalizable_hash_size = hash_table->size;
4914 hash %= finalizable_hash_size;
4916 for (entry = finalizable_hash [hash]; entry; entry = entry->next) {
4917 if (entry->object == obj) {
4919 /* remove from the list */
4921 prev->next = entry->next;
4923 finalizable_hash [hash] = entry->next;
4924 hash_table->num_registered--;
4925 DEBUG (5, fprintf (gc_debug_file, "Removed finalizer %p for object: %p (%s) (%d)\n", entry, obj, obj->vtable->klass->name, hash_table->num_registered));
4926 free_internal_mem (entry, INTERNAL_MEM_FINALIZE_ENTRY);
4934 /* request to deregister, but already out of the list */
4938 entry = get_internal_mem (sizeof (FinalizeEntry), INTERNAL_MEM_FINALIZE_ENTRY);
4939 entry->object = obj;
4940 entry->next = finalizable_hash [hash];
4941 finalizable_hash [hash] = entry;
4942 hash_table->num_registered++;
4943 DEBUG (5, fprintf (gc_debug_file, "Added finalizer %p for object: %p (%s) (%d) to %s table\n", entry, obj, obj->vtable->klass->name, hash_table->num_registered, generation_name (generation)));
4948 mono_gc_register_for_finalization (MonoObject *obj, void *user_data)
4950 if (ptr_in_nursery (obj))
4951 register_for_finalization (obj, user_data, GENERATION_NURSERY);
4953 register_for_finalization (obj, user_data, GENERATION_OLD);
4957 rehash_dislink (DisappearingLinkHashTable *hash_table)
4959 DisappearingLink **disappearing_link_hash = hash_table->table;
4960 int disappearing_link_hash_size = hash_table->size;
4963 DisappearingLink **new_hash;
4964 DisappearingLink *entry, *next;
4965 int new_size = g_spaced_primes_closest (hash_table->num_links);
4967 new_hash = get_internal_mem (new_size * sizeof (DisappearingLink*), INTERNAL_MEM_DISLINK_TABLE);
4968 for (i = 0; i < disappearing_link_hash_size; ++i) {
4969 for (entry = disappearing_link_hash [i]; entry; entry = next) {
4970 hash = mono_aligned_addr_hash (entry->link) % new_size;
4972 entry->next = new_hash [hash];
4973 new_hash [hash] = entry;
4976 free_internal_mem (disappearing_link_hash, INTERNAL_MEM_DISLINK_TABLE);
4977 hash_table->table = new_hash;
4978 hash_table->size = new_size;
4981 /* LOCKING: assumes the GC lock is held */
4983 add_or_remove_disappearing_link (MonoObject *obj, void **link, gboolean track, int generation)
4985 DisappearingLinkHashTable *hash_table = get_dislink_hash_table (generation);
4986 DisappearingLink *entry, *prev;
4988 DisappearingLink **disappearing_link_hash = hash_table->table;
4989 int disappearing_link_hash_size = hash_table->size;
4991 if (hash_table->num_links >= disappearing_link_hash_size * 2) {
4992 rehash_dislink (hash_table);
4993 disappearing_link_hash = hash_table->table;
4994 disappearing_link_hash_size = hash_table->size;
4996 /* FIXME: add check that link is not in the heap */
4997 hash = mono_aligned_addr_hash (link) % disappearing_link_hash_size;
4998 entry = disappearing_link_hash [hash];
5000 for (; entry; entry = entry->next) {
5001 /* link already added */
5002 if (link == entry->link) {
5003 /* NULL obj means remove */
5006 prev->next = entry->next;
5008 disappearing_link_hash [hash] = entry->next;
5009 hash_table->num_links--;
5010 DEBUG (5, fprintf (gc_debug_file, "Removed dislink %p (%d) from %s table\n", entry, hash_table->num_links, generation_name (generation)));
5011 free_internal_mem (entry, INTERNAL_MEM_DISLINK);
5014 *link = HIDE_POINTER (obj, track); /* we allow the change of object */
5022 entry = get_internal_mem (sizeof (DisappearingLink), INTERNAL_MEM_DISLINK);
5023 *link = HIDE_POINTER (obj, track);
5025 entry->next = disappearing_link_hash [hash];
5026 disappearing_link_hash [hash] = entry;
5027 hash_table->num_links++;
5028 DEBUG (5, fprintf (gc_debug_file, "Added dislink %p for object: %p (%s) at %p to %s table\n", entry, obj, obj->vtable->klass->name, link, generation_name (generation)));
5031 /* LOCKING: assumes the GC lock is held */
5033 mono_gc_register_disappearing_link (MonoObject *obj, void **link, gboolean track)
5035 add_or_remove_disappearing_link (NULL, link, FALSE, GENERATION_NURSERY);
5036 add_or_remove_disappearing_link (NULL, link, FALSE, GENERATION_OLD);
5038 if (ptr_in_nursery (obj))
5039 add_or_remove_disappearing_link (obj, link, track, GENERATION_NURSERY);
5041 add_or_remove_disappearing_link (obj, link, track, GENERATION_OLD);
5046 mono_gc_invoke_finalizers (void)
5048 FinalizeEntry *entry = NULL;
5049 gboolean entry_is_critical = FALSE;
5052 /* FIXME: batch to reduce lock contention */
5053 while (fin_ready_list || critical_fin_list) {
5057 FinalizeEntry **list = entry_is_critical ? &critical_fin_list : &fin_ready_list;
5059 /* We have finalized entry in the last
5060 interation, now we need to remove it from
5063 *list = entry->next;
5065 FinalizeEntry *e = *list;
5066 while (e->next != entry)
5068 e->next = entry->next;
5070 free_internal_mem (entry, INTERNAL_MEM_FINALIZE_ENTRY);
5074 /* Now look for the first non-null entry. */
5075 for (entry = fin_ready_list; entry && !entry->object; entry = entry->next)
5078 entry_is_critical = FALSE;
5080 entry_is_critical = TRUE;
5081 for (entry = critical_fin_list; entry && !entry->object; entry = entry->next)
5086 g_assert (entry->object);
5087 num_ready_finalizers--;
5088 obj = entry->object;
5089 entry->object = NULL;
5090 DEBUG (7, fprintf (gc_debug_file, "Finalizing object %p (%s)\n", obj, safe_name (obj)));
5098 g_assert (entry->object == NULL);
5100 /* the object is on the stack so it is pinned */
5101 /*g_print ("Calling finalizer for object: %p (%s)\n", entry->object, safe_name (entry->object));*/
5102 mono_gc_run_finalize (obj, NULL);
5109 mono_gc_pending_finalizers (void)
5111 return fin_ready_list || critical_fin_list;
5114 /* Negative value to remove */
5116 mono_gc_add_memory_pressure (gint64 value)
5118 /* FIXME: Use interlocked functions */
5120 memory_pressure += value;
5125 * ######################################################################
5126 * ######## registered roots support
5127 * ######################################################################
5131 rehash_roots (gboolean pinned)
5135 RootRecord **new_hash;
5136 RootRecord *entry, *next;
5139 new_size = g_spaced_primes_closest (num_roots_entries [pinned]);
5140 new_hash = get_internal_mem (new_size * sizeof (RootRecord*), INTERNAL_MEM_ROOTS_TABLE);
5141 for (i = 0; i < roots_hash_size [pinned]; ++i) {
5142 for (entry = roots_hash [pinned][i]; entry; entry = next) {
5143 hash = mono_aligned_addr_hash (entry->start_root) % new_size;
5145 entry->next = new_hash [hash];
5146 new_hash [hash] = entry;
5149 free_internal_mem (roots_hash [pinned], INTERNAL_MEM_ROOTS_TABLE);
5150 roots_hash [pinned] = new_hash;
5151 roots_hash_size [pinned] = new_size;
5155 find_root (int root_type, char *start, guint32 addr_hash)
5157 RootRecord *new_root;
5159 guint32 hash = addr_hash % roots_hash_size [root_type];
5160 for (new_root = roots_hash [root_type][hash]; new_root; new_root = new_root->next) {
5161 /* we allow changing the size and the descriptor (for thread statics etc) */
5162 if (new_root->start_root == start) {
5171 * We do not coalesce roots.
5174 mono_gc_register_root_inner (char *start, size_t size, void *descr, int root_type)
5176 RootRecord *new_root;
5177 unsigned int hash, addr_hash = mono_aligned_addr_hash (start);
5180 for (i = 0; i < ROOT_TYPE_NUM; ++i) {
5181 if (num_roots_entries [i] >= roots_hash_size [i] * 2)
5184 for (i = 0; i < ROOT_TYPE_NUM; ++i) {
5185 new_root = find_root (i, start, addr_hash);
5186 /* we allow changing the size and the descriptor (for thread statics etc) */
5188 size_t old_size = new_root->end_root - new_root->start_root;
5189 new_root->end_root = new_root->start_root + size;
5190 g_assert (((new_root->root_desc != 0) && (descr != NULL)) ||
5191 ((new_root->root_desc == 0) && (descr == NULL)));
5192 new_root->root_desc = (mword)descr;
5194 roots_size -= old_size;
5199 new_root = get_internal_mem (sizeof (RootRecord), INTERNAL_MEM_ROOT_RECORD);
5201 new_root->start_root = start;
5202 new_root->end_root = new_root->start_root + size;
5203 new_root->root_desc = (mword)descr;
5205 hash = addr_hash % roots_hash_size [root_type];
5206 num_roots_entries [root_type]++;
5207 new_root->next = roots_hash [root_type] [hash];
5208 roots_hash [root_type][hash] = new_root;
5209 DEBUG (3, fprintf (gc_debug_file, "Added root %p for range: %p-%p, descr: %p (%d/%d bytes)\n", new_root, new_root->start_root, new_root->end_root, descr, (int)size, (int)roots_size));
5219 mono_gc_register_root (char *start, size_t size, void *descr)
5221 return mono_gc_register_root_inner (start, size, descr, descr ? ROOT_TYPE_NORMAL : ROOT_TYPE_PINNED);
5225 mono_gc_register_root_wbarrier (char *start, size_t size, void *descr)
5227 return mono_gc_register_root_inner (start, size, descr, ROOT_TYPE_WBARRIER);
5231 mono_gc_deregister_root (char* addr)
5233 RootRecord *tmp, *prev;
5234 unsigned int hash, addr_hash = mono_aligned_addr_hash (addr);
5238 for (root_type = 0; root_type < ROOT_TYPE_NUM; ++root_type) {
5239 hash = addr_hash % roots_hash_size [root_type];
5240 tmp = roots_hash [root_type][hash];
5243 if (tmp->start_root == (char*)addr) {
5245 prev->next = tmp->next;
5247 roots_hash [root_type][hash] = tmp->next;
5248 roots_size -= (tmp->end_root - tmp->start_root);
5249 num_roots_entries [root_type]--;
5250 DEBUG (3, fprintf (gc_debug_file, "Removed root %p for range: %p-%p\n", tmp, tmp->start_root, tmp->end_root));
5251 free_internal_mem (tmp, INTERNAL_MEM_ROOT_RECORD);
5262 * ######################################################################
5263 * ######## Thread handling (stop/start code)
5264 * ######################################################################
5267 /* FIXME: handle large/small config */
5268 #define HASH_PTHREAD_T(id) (((unsigned int)(id) >> 4) * 2654435761u)
5270 static SgenThreadInfo* thread_table [THREAD_HASH_SIZE];
5272 #if USE_SIGNAL_BASED_START_STOP_WORLD
5274 static MonoSemType suspend_ack_semaphore;
5275 static MonoSemType *suspend_ack_semaphore_ptr;
5276 static unsigned int global_stop_count = 0;
5278 static sigset_t suspend_signal_mask;
5279 static mword cur_thread_regs [ARCH_NUM_REGS] = {0};
5281 /* LOCKING: assumes the GC lock is held */
5283 mono_sgen_get_thread_table (void)
5285 return thread_table;
5289 mono_sgen_thread_info_lookup (ARCH_THREAD_TYPE id)
5291 unsigned int hash = HASH_PTHREAD_T (id) % THREAD_HASH_SIZE;
5292 SgenThreadInfo *info;
5294 info = thread_table [hash];
5295 while (info && !ARCH_THREAD_EQUALS (info->id, id)) {
5302 update_current_thread_stack (void *start)
5304 void *ptr = cur_thread_regs;
5305 SgenThreadInfo *info = mono_sgen_thread_info_lookup (ARCH_GET_THREAD ());
5307 info->stack_start = align_pointer (&ptr);
5308 g_assert (info->stack_start >= info->stack_start_limit && info->stack_start < info->stack_end);
5309 ARCH_STORE_REGS (ptr);
5310 info->stopped_regs = ptr;
5311 if (gc_callbacks.thread_suspend_func)
5312 gc_callbacks.thread_suspend_func (info->runtime_data, NULL);
5316 * Define this and use the "xdomain-checks" MONO_GC_DEBUG option to
5317 * have cross-domain checks in the write barrier.
5319 //#define XDOMAIN_CHECKS_IN_WBARRIER
5321 #ifndef BINARY_PROTOCOL
5322 #ifndef HEAVY_STATISTICS
5323 #define MANAGED_ALLOCATION
5324 #ifndef XDOMAIN_CHECKS_IN_WBARRIER
5325 #define MANAGED_WBARRIER
5331 is_ip_in_managed_allocator (MonoDomain *domain, gpointer ip);
5334 mono_sgen_wait_for_suspend_ack (int count)
5338 for (i = 0; i < count; ++i) {
5339 while ((result = MONO_SEM_WAIT (suspend_ack_semaphore_ptr)) != 0) {
5340 if (errno != EINTR) {
5341 g_error ("sem_wait ()");
5348 restart_threads_until_none_in_managed_allocator (void)
5350 SgenThreadInfo *info;
5351 int i, result, num_threads_died = 0;
5352 int sleep_duration = -1;
5355 int restart_count = 0, restarted_count = 0;
5356 /* restart all threads that stopped in the
5358 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5359 for (info = thread_table [i]; info; info = info->next) {
5362 if (!info->stack_start || info->in_critical_region ||
5363 is_ip_in_managed_allocator (info->stopped_domain, info->stopped_ip)) {
5364 binary_protocol_thread_restart ((gpointer)info->id);
5365 #if defined(__MACH__) && MONO_MACH_ARCH_SUPPORTED
5366 result = thread_resume (pthread_mach_thread_np (info->id));
5368 result = pthread_kill (info->id, restart_signal_num);
5376 /* we set the stopped_ip to
5377 NULL for threads which
5378 we're not restarting so
5379 that we can easily identify
5381 info->stopped_ip = NULL;
5382 info->stopped_domain = NULL;
5386 /* if no threads were restarted, we're done */
5387 if (restart_count == 0)
5390 #if defined(__MACH__) && MONO_MACH_ARCH_SUPPORTED
5391 /* mach thread_resume is synchronous so we dont need to wait for them */
5393 /* wait for the threads to signal their restart */
5394 mono_sgen_wait_for_suspend_ack (restart_count);
5397 if (sleep_duration < 0) {
5401 g_usleep (sleep_duration);
5402 sleep_duration += 10;
5405 /* stop them again */
5406 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5407 for (info = thread_table [i]; info; info = info->next) {
5408 if (info->skip || info->stopped_ip == NULL)
5410 #if defined(__MACH__) && MONO_MACH_ARCH_SUPPORTED
5411 result = thread_suspend (pthread_mach_thread_np (info->id));
5413 result = pthread_kill (info->id, suspend_signal_num);
5422 /* some threads might have died */
5423 num_threads_died += restart_count - restarted_count;
5424 #if defined(__MACH__) && MONO_MACH_ARCH_SUPPORTED
5425 /* mach thread_resume is synchronous so we dont need to wait for them */
5427 /* wait for the threads to signal their suspension
5429 mono_sgen_wait_for_suspend_ack (restart_count);
5433 return num_threads_died;
5436 /* LOCKING: assumes the GC lock is held (by the stopping thread) */
5438 suspend_handler (int sig, siginfo_t *siginfo, void *context)
5440 SgenThreadInfo *info;
5443 int old_errno = errno;
5444 gpointer regs [ARCH_NUM_REGS];
5445 gpointer stack_start;
5447 id = pthread_self ();
5448 info = mono_sgen_thread_info_lookup (id);
5449 info->stopped_domain = mono_domain_get ();
5450 info->stopped_ip = (gpointer) ARCH_SIGCTX_IP (context);
5451 stop_count = global_stop_count;
5452 /* duplicate signal */
5453 if (0 && info->stop_count == stop_count) {
5457 #ifdef HAVE_KW_THREAD
5458 /* update the remset info in the thread data structure */
5459 info->remset = remembered_set;
5461 stack_start = (char*) ARCH_SIGCTX_SP (context) - REDZONE_SIZE;
5462 /* If stack_start is not within the limits, then don't set it
5463 in info and we will be restarted. */
5464 if (stack_start >= info->stack_start_limit && info->stack_start <= info->stack_end) {
5465 info->stack_start = stack_start;
5467 ARCH_COPY_SIGCTX_REGS (regs, context);
5468 info->stopped_regs = regs;
5470 g_assert (!info->stack_start);
5473 /* Notify the JIT */
5474 if (gc_callbacks.thread_suspend_func)
5475 gc_callbacks.thread_suspend_func (info->runtime_data, context);
5477 DEBUG (4, fprintf (gc_debug_file, "Posting suspend_ack_semaphore for suspend from %p %p\n", info, (gpointer)ARCH_GET_THREAD ()));
5478 /* notify the waiting thread */
5479 MONO_SEM_POST (suspend_ack_semaphore_ptr);
5480 info->stop_count = stop_count;
5482 /* wait until we receive the restart signal */
5485 sigsuspend (&suspend_signal_mask);
5486 } while (info->signal != restart_signal_num);
5488 DEBUG (4, fprintf (gc_debug_file, "Posting suspend_ack_semaphore for resume from %p %p\n", info, (gpointer)ARCH_GET_THREAD ()));
5489 /* notify the waiting thread */
5490 MONO_SEM_POST (suspend_ack_semaphore_ptr);
5496 restart_handler (int sig)
5498 SgenThreadInfo *info;
5499 int old_errno = errno;
5501 info = mono_sgen_thread_info_lookup (pthread_self ());
5502 info->signal = restart_signal_num;
5503 DEBUG (4, fprintf (gc_debug_file, "Restart handler in %p %p\n", info, (gpointer)ARCH_GET_THREAD ()));
5509 acquire_gc_locks (void)
5515 release_gc_locks (void)
5517 UNLOCK_INTERRUPTION;
5520 static TV_DECLARE (stop_world_time);
5521 static unsigned long max_pause_usec = 0;
5523 /* LOCKING: assumes the GC lock is held */
5529 acquire_gc_locks ();
5531 update_current_thread_stack (&count);
5533 global_stop_count++;
5534 DEBUG (3, fprintf (gc_debug_file, "stopping world n %d from %p %p\n", global_stop_count, mono_sgen_thread_info_lookup (ARCH_GET_THREAD ()), (gpointer)ARCH_GET_THREAD ()));
5535 TV_GETTIME (stop_world_time);
5536 count = mono_sgen_thread_handshake (suspend_signal_num);
5537 count -= restart_threads_until_none_in_managed_allocator ();
5538 g_assert (count >= 0);
5539 DEBUG (3, fprintf (gc_debug_file, "world stopped %d thread(s)\n", count));
5543 /* LOCKING: assumes the GC lock is held */
5545 restart_world (void)
5548 SgenThreadInfo *info;
5549 TV_DECLARE (end_sw);
5552 /* notify the profiler of the leftovers */
5553 if (G_UNLIKELY (mono_profiler_events & MONO_PROFILE_GC_MOVES)) {
5554 if (moved_objects_idx) {
5555 mono_profiler_gc_moves (moved_objects, moved_objects_idx);
5556 moved_objects_idx = 0;
5559 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5560 for (info = thread_table [i]; info; info = info->next) {
5561 info->stack_start = NULL;
5562 info->stopped_regs = NULL;
5566 release_gc_locks ();
5568 count = mono_sgen_thread_handshake (restart_signal_num);
5569 TV_GETTIME (end_sw);
5570 usec = TV_ELAPSED (stop_world_time, end_sw);
5571 max_pause_usec = MAX (usec, max_pause_usec);
5572 DEBUG (2, fprintf (gc_debug_file, "restarted %d thread(s) (pause time: %d usec, max: %d)\n", count, (int)usec, (int)max_pause_usec));
5576 #endif /* USE_SIGNAL_BASED_START_STOP_WORLD */
5579 mono_gc_set_gc_callbacks (MonoGCCallbacks *callbacks)
5581 gc_callbacks = *callbacks;
5585 mono_gc_get_gc_callbacks ()
5587 return &gc_callbacks;
5590 /* Variables holding start/end nursery so it won't have to be passed at every call */
5591 static void *scan_area_arg_start, *scan_area_arg_end;
5594 mono_gc_conservatively_scan_area (void *start, void *end)
5596 conservatively_pin_objects_from (start, end, scan_area_arg_start, scan_area_arg_end, PIN_TYPE_STACK);
5600 mono_gc_scan_object (void *obj)
5602 if (current_collection_generation == GENERATION_NURSERY)
5603 copy_object (&obj, &gray_queue);
5605 major_copy_or_mark_object (&obj, &gray_queue);
5610 * Mark from thread stacks and registers.
5613 scan_thread_data (void *start_nursery, void *end_nursery, gboolean precise)
5616 SgenThreadInfo *info;
5618 scan_area_arg_start = start_nursery;
5619 scan_area_arg_end = end_nursery;
5621 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5622 for (info = thread_table [i]; info; info = info->next) {
5624 DEBUG (3, fprintf (gc_debug_file, "Skipping dead thread %p, range: %p-%p, size: %td\n", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start));
5627 DEBUG (3, fprintf (gc_debug_file, "Scanning thread %p, range: %p-%p, size: %td, pinned=%d\n", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start, next_pin_slot));
5628 if (gc_callbacks.thread_mark_func && !conservative_stack_mark)
5629 gc_callbacks.thread_mark_func (info->runtime_data, info->stack_start, info->stack_end, precise);
5631 conservatively_pin_objects_from (info->stack_start, info->stack_end, start_nursery, end_nursery, PIN_TYPE_STACK);
5634 conservatively_pin_objects_from (info->stopped_regs, info->stopped_regs + ARCH_NUM_REGS,
5635 start_nursery, end_nursery, PIN_TYPE_STACK);
5641 find_pinning_ref_from_thread (char *obj, size_t size)
5644 SgenThreadInfo *info;
5645 char *endobj = obj + size;
5647 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5648 for (info = thread_table [i]; info; info = info->next) {
5649 char **start = (char**)info->stack_start;
5652 while (start < (char**)info->stack_end) {
5653 if (*start >= obj && *start < endobj) {
5654 DEBUG (0, fprintf (gc_debug_file, "Object %p referenced in thread %p (id %p) at %p, stack: %p-%p\n", obj, info, (gpointer)info->id, start, info->stack_start, info->stack_end));
5659 /* FIXME: check info->stopped_regs */
5665 ptr_on_stack (void *ptr)
5667 gpointer stack_start = &stack_start;
5668 SgenThreadInfo *info = mono_sgen_thread_info_lookup (ARCH_GET_THREAD ());
5670 if (ptr >= stack_start && ptr < (gpointer)info->stack_end)
5676 handle_remset (mword *p, void *start_nursery, void *end_nursery, gboolean global, GrayQueue *queue)
5683 HEAVY_STAT (++stat_global_remsets_processed);
5685 /* FIXME: exclude stack locations */
5686 switch ((*p) & REMSET_TYPE_MASK) {
5687 case REMSET_LOCATION:
5689 //__builtin_prefetch (ptr);
5690 if (((void*)ptr < start_nursery || (void*)ptr >= end_nursery)) {
5691 gpointer old = *ptr;
5692 copy_object (ptr, queue);
5693 DEBUG (9, fprintf (gc_debug_file, "Overwrote remset at %p with %p\n", ptr, *ptr));
5695 binary_protocol_ptr_update (ptr, old, *ptr, (gpointer)LOAD_VTABLE (*ptr), safe_object_get_size (*ptr));
5696 if (!global && *ptr >= start_nursery && *ptr < end_nursery) {
5698 * If the object is pinned, each reference to it from nonpinned objects
5699 * becomes part of the global remset, which can grow very large.
5701 DEBUG (9, fprintf (gc_debug_file, "Add to global remset because of pinning %p (%p %s)\n", ptr, *ptr, safe_name (*ptr)));
5702 add_to_global_remset (ptr);
5705 DEBUG (9, fprintf (gc_debug_file, "Skipping remset at %p holding %p\n", ptr, *ptr));
5709 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
5710 if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery))
5713 while (count-- > 0) {
5714 copy_object (ptr, queue);
5715 DEBUG (9, fprintf (gc_debug_file, "Overwrote remset at %p with %p (count: %d)\n", ptr, *ptr, (int)count));
5716 if (!global && *ptr >= start_nursery && *ptr < end_nursery)
5717 add_to_global_remset (ptr);
5722 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
5723 if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery))
5725 scan_object ((char*)ptr, queue);
5727 case REMSET_VTYPE: {
5728 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
5729 if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery))
5734 ptr = (void**) scan_vtype ((char*)ptr, desc, start_nursery, end_nursery, queue);
5738 g_assert_not_reached ();
5743 #ifdef HEAVY_STATISTICS
5745 collect_store_remsets (RememberedSet *remset, mword *bumper)
5747 mword *p = remset->data;
5752 while (p < remset->store_next) {
5753 switch ((*p) & REMSET_TYPE_MASK) {
5754 case REMSET_LOCATION:
5757 ++stat_saved_remsets_1;
5759 if (*p == last1 || *p == last2) {
5760 ++stat_saved_remsets_2;
5777 g_assert_not_reached ();
5787 RememberedSet *remset;
5789 SgenThreadInfo *info;
5791 mword *addresses, *bumper, *p, *r;
5793 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5794 for (info = thread_table [i]; info; info = info->next) {
5795 for (remset = info->remset; remset; remset = remset->next)
5796 size += remset->store_next - remset->data;
5799 for (remset = freed_thread_remsets; remset; remset = remset->next)
5800 size += remset->store_next - remset->data;
5801 for (remset = global_remset; remset; remset = remset->next)
5802 size += remset->store_next - remset->data;
5804 bumper = addresses = get_internal_mem (sizeof (mword) * size, INTERNAL_MEM_STATISTICS);
5806 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5807 for (info = thread_table [i]; info; info = info->next) {
5808 for (remset = info->remset; remset; remset = remset->next)
5809 bumper = collect_store_remsets (remset, bumper);
5812 for (remset = global_remset; remset; remset = remset->next)
5813 bumper = collect_store_remsets (remset, bumper);
5814 for (remset = freed_thread_remsets; remset; remset = remset->next)
5815 bumper = collect_store_remsets (remset, bumper);
5817 g_assert (bumper <= addresses + size);
5819 stat_store_remsets += bumper - addresses;
5821 sort_addresses ((void**)addresses, bumper - addresses);
5824 while (r < bumper) {
5830 stat_store_remsets_unique += p - addresses;
5832 free_internal_mem (addresses, INTERNAL_MEM_STATISTICS);
5837 clear_thread_store_remset_buffer (SgenThreadInfo *info)
5839 *info->store_remset_buffer_index_addr = 0;
5840 memset (*info->store_remset_buffer_addr, 0, sizeof (gpointer) * STORE_REMSET_BUFFER_SIZE);
5844 scan_from_remsets (void *start_nursery, void *end_nursery, GrayQueue *queue)
5847 SgenThreadInfo *info;
5848 RememberedSet *remset;
5849 GenericStoreRememberedSet *store_remset;
5850 mword *p, *next_p, *store_pos;
5852 #ifdef HEAVY_STATISTICS
5856 /* the global one */
5857 for (remset = global_remset; remset; remset = remset->next) {
5858 DEBUG (4, fprintf (gc_debug_file, "Scanning global remset range: %p-%p, size: %td\n", remset->data, remset->store_next, remset->store_next - remset->data));
5859 store_pos = remset->data;
5860 for (p = remset->data; p < remset->store_next; p = next_p) {
5861 void **ptr = (void**)p [0];
5863 /*Ignore previously processed remset.*/
5864 if (!global_remset_location_was_not_added (ptr)) {
5869 next_p = handle_remset (p, start_nursery, end_nursery, TRUE, queue);
5872 * Clear global remsets of locations which no longer point to the
5873 * nursery. Otherwise, they could grow indefinitely between major
5876 * Since all global remsets are location remsets, we don't need to unmask the pointer.
5878 if (ptr_in_nursery (*ptr)) {
5879 *store_pos ++ = p [0];
5880 HEAVY_STAT (++stat_global_remsets_readded);
5884 /* Truncate the remset */
5885 remset->store_next = store_pos;
5888 /* the generic store ones */
5889 store_remset = generic_store_remsets;
5890 while (store_remset) {
5891 GenericStoreRememberedSet *next = store_remset->next;
5893 for (i = 0; i < STORE_REMSET_BUFFER_SIZE - 1; ++i) {
5894 gpointer addr = store_remset->data [i];
5896 handle_remset ((mword*)&addr, start_nursery, end_nursery, FALSE, queue);
5899 free_internal_mem (store_remset, INTERNAL_MEM_STORE_REMSET);
5901 store_remset = next;
5903 generic_store_remsets = NULL;
5905 /* the per-thread ones */
5906 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5907 for (info = thread_table [i]; info; info = info->next) {
5908 RememberedSet *next;
5910 for (remset = info->remset; remset; remset = next) {
5911 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for thread %p, range: %p-%p, size: %td\n", info, remset->data, remset->store_next, remset->store_next - remset->data));
5912 for (p = remset->data; p < remset->store_next;) {
5913 p = handle_remset (p, start_nursery, end_nursery, FALSE, queue);
5915 remset->store_next = remset->data;
5916 next = remset->next;
5917 remset->next = NULL;
5918 if (remset != info->remset) {
5919 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
5920 free_internal_mem (remset, INTERNAL_MEM_REMSET);
5923 for (j = 0; j < *info->store_remset_buffer_index_addr; ++j)
5924 handle_remset ((mword*)*info->store_remset_buffer_addr + j + 1, start_nursery, end_nursery, FALSE, queue);
5925 clear_thread_store_remset_buffer (info);
5929 /* the freed thread ones */
5930 while (freed_thread_remsets) {
5931 RememberedSet *next;
5932 remset = freed_thread_remsets;
5933 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for freed thread, range: %p-%p, size: %td\n", remset->data, remset->store_next, remset->store_next - remset->data));
5934 for (p = remset->data; p < remset->store_next;) {
5935 p = handle_remset (p, start_nursery, end_nursery, FALSE, queue);
5937 next = remset->next;
5938 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
5939 free_internal_mem (remset, INTERNAL_MEM_REMSET);
5940 freed_thread_remsets = next;
5945 * Clear the info in the remembered sets: we're doing a major collection, so
5946 * the per-thread ones are not needed and the global ones will be reconstructed
5950 clear_remsets (void)
5953 SgenThreadInfo *info;
5954 RememberedSet *remset, *next;
5956 /* the global list */
5957 for (remset = global_remset; remset; remset = next) {
5958 remset->store_next = remset->data;
5959 next = remset->next;
5960 remset->next = NULL;
5961 if (remset != global_remset) {
5962 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
5963 free_internal_mem (remset, INTERNAL_MEM_REMSET);
5966 /* the generic store ones */
5967 while (generic_store_remsets) {
5968 GenericStoreRememberedSet *gs_next = generic_store_remsets->next;
5969 free_internal_mem (generic_store_remsets, INTERNAL_MEM_STORE_REMSET);
5970 generic_store_remsets = gs_next;
5972 /* the per-thread ones */
5973 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5974 for (info = thread_table [i]; info; info = info->next) {
5975 for (remset = info->remset; remset; remset = next) {
5976 remset->store_next = remset->data;
5977 next = remset->next;
5978 remset->next = NULL;
5979 if (remset != info->remset) {
5980 DEBUG (3, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
5981 free_internal_mem (remset, INTERNAL_MEM_REMSET);
5984 clear_thread_store_remset_buffer (info);
5988 /* the freed thread ones */
5989 while (freed_thread_remsets) {
5990 next = freed_thread_remsets->next;
5991 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", freed_thread_remsets->data));
5992 free_internal_mem (freed_thread_remsets, INTERNAL_MEM_REMSET);
5993 freed_thread_remsets = next;
5998 * Clear the thread local TLAB variables for all threads.
6003 SgenThreadInfo *info;
6006 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
6007 for (info = thread_table [i]; info; info = info->next) {
6008 /* A new TLAB will be allocated when the thread does its first allocation */
6009 *info->tlab_start_addr = NULL;
6010 *info->tlab_next_addr = NULL;
6011 *info->tlab_temp_end_addr = NULL;
6012 *info->tlab_real_end_addr = NULL;
6017 /* LOCKING: assumes the GC lock is held */
6018 static SgenThreadInfo*
6019 gc_register_current_thread (void *addr)
6022 SgenThreadInfo* info = malloc (sizeof (SgenThreadInfo));
6023 #ifndef HAVE_KW_THREAD
6024 SgenThreadInfo *__thread_info__ = info;
6030 memset (info, 0, sizeof (SgenThreadInfo));
6031 #ifndef HAVE_KW_THREAD
6032 info->tlab_start = info->tlab_next = info->tlab_temp_end = info->tlab_real_end = NULL;
6034 g_assert (!pthread_getspecific (thread_info_key));
6035 pthread_setspecific (thread_info_key, info);
6040 info->id = ARCH_GET_THREAD ();
6041 info->stop_count = -1;
6044 info->stack_start = NULL;
6045 info->tlab_start_addr = &TLAB_START;
6046 info->tlab_next_addr = &TLAB_NEXT;
6047 info->tlab_temp_end_addr = &TLAB_TEMP_END;
6048 info->tlab_real_end_addr = &TLAB_REAL_END;
6049 info->store_remset_buffer_addr = &STORE_REMSET_BUFFER;
6050 info->store_remset_buffer_index_addr = &STORE_REMSET_BUFFER_INDEX;
6051 info->stopped_ip = NULL;
6052 info->stopped_domain = NULL;
6053 info->stopped_regs = NULL;
6055 binary_protocol_thread_register ((gpointer)info->id);
6057 #ifdef HAVE_KW_THREAD
6058 tlab_next_addr = &tlab_next;
6059 store_remset_buffer_index_addr = &store_remset_buffer_index;
6062 /* try to get it with attributes first */
6063 #if defined(HAVE_PTHREAD_GETATTR_NP) && defined(HAVE_PTHREAD_ATTR_GETSTACK)
6067 pthread_attr_t attr;
6068 pthread_getattr_np (pthread_self (), &attr);
6069 pthread_attr_getstack (&attr, &sstart, &size);
6070 info->stack_start_limit = sstart;
6071 info->stack_end = (char*)sstart + size;
6072 pthread_attr_destroy (&attr);
6074 #elif defined(HAVE_PTHREAD_GET_STACKSIZE_NP) && defined(HAVE_PTHREAD_GET_STACKADDR_NP)
6075 info->stack_end = (char*)pthread_get_stackaddr_np (pthread_self ());
6076 info->stack_start_limit = (char*)info->stack_end - pthread_get_stacksize_np (pthread_self ());
6079 /* FIXME: we assume the stack grows down */
6080 gsize stack_bottom = (gsize)addr;
6081 stack_bottom += 4095;
6082 stack_bottom &= ~4095;
6083 info->stack_end = (char*)stack_bottom;
6087 #ifdef HAVE_KW_THREAD
6088 stack_end = info->stack_end;
6091 /* hash into the table */
6092 hash = HASH_PTHREAD_T (info->id) % THREAD_HASH_SIZE;
6093 info->next = thread_table [hash];
6094 thread_table [hash] = info;
6096 info->remset = alloc_remset (DEFAULT_REMSET_SIZE, info);
6097 pthread_setspecific (remembered_set_key, info->remset);
6098 #ifdef HAVE_KW_THREAD
6099 remembered_set = info->remset;
6102 STORE_REMSET_BUFFER = get_internal_mem (sizeof (gpointer) * STORE_REMSET_BUFFER_SIZE, INTERNAL_MEM_STORE_REMSET);
6103 STORE_REMSET_BUFFER_INDEX = 0;
6105 DEBUG (3, fprintf (gc_debug_file, "registered thread %p (%p) (hash: %d)\n", info, (gpointer)info->id, hash));
6107 if (gc_callbacks.thread_attach_func)
6108 info->runtime_data = gc_callbacks.thread_attach_func ();
6114 add_generic_store_remset_from_buffer (gpointer *buffer)
6116 GenericStoreRememberedSet *remset = get_internal_mem (sizeof (GenericStoreRememberedSet), INTERNAL_MEM_STORE_REMSET);
6117 memcpy (remset->data, buffer + 1, sizeof (gpointer) * (STORE_REMSET_BUFFER_SIZE - 1));
6118 remset->next = generic_store_remsets;
6119 generic_store_remsets = remset;
6123 unregister_current_thread (void)
6126 SgenThreadInfo *prev = NULL;
6128 RememberedSet *rset;
6129 ARCH_THREAD_TYPE id = ARCH_GET_THREAD ();
6131 binary_protocol_thread_unregister ((gpointer)id);
6133 hash = HASH_PTHREAD_T (id) % THREAD_HASH_SIZE;
6134 p = thread_table [hash];
6136 DEBUG (3, fprintf (gc_debug_file, "unregister thread %p (%p)\n", p, (gpointer)p->id));
6137 while (!ARCH_THREAD_EQUALS (p->id, id)) {
6142 thread_table [hash] = p->next;
6144 prev->next = p->next;
6147 if (freed_thread_remsets) {
6148 for (rset = p->remset; rset->next; rset = rset->next)
6150 rset->next = freed_thread_remsets;
6151 freed_thread_remsets = p->remset;
6153 freed_thread_remsets = p->remset;
6156 if (*p->store_remset_buffer_index_addr)
6157 add_generic_store_remset_from_buffer (*p->store_remset_buffer_addr);
6158 free_internal_mem (*p->store_remset_buffer_addr, INTERNAL_MEM_STORE_REMSET);
6163 unregister_thread (void *k)
6165 g_assert (!mono_domain_get ());
6167 unregister_current_thread ();
6172 mono_gc_register_thread (void *baseptr)
6174 SgenThreadInfo *info;
6178 info = mono_sgen_thread_info_lookup (ARCH_GET_THREAD ());
6180 info = gc_register_current_thread (baseptr);
6182 return info != NULL;
6185 #if USE_PTHREAD_INTERCEPT
6188 void *(*start_routine) (void *);
6191 MonoSemType registered;
6192 } SgenThreadStartInfo;
6195 gc_start_thread (void *arg)
6197 SgenThreadStartInfo *start_info = arg;
6198 SgenThreadInfo* info;
6199 void *t_arg = start_info->arg;
6200 void *(*start_func) (void*) = start_info->start_routine;
6205 info = gc_register_current_thread (&result);
6207 post_result = MONO_SEM_POST (&(start_info->registered));
6208 g_assert (!post_result);
6209 result = start_func (t_arg);
6210 g_assert (!mono_domain_get ());
6212 * this is done by the pthread key dtor
6214 unregister_current_thread ();
6222 mono_gc_pthread_create (pthread_t *new_thread, const pthread_attr_t *attr, void *(*start_routine)(void *), void *arg)
6224 SgenThreadStartInfo *start_info;
6227 start_info = malloc (sizeof (SgenThreadStartInfo));
6230 MONO_SEM_INIT (&(start_info->registered), 0);
6231 start_info->arg = arg;
6232 start_info->start_routine = start_routine;
6234 result = pthread_create (new_thread, attr, gc_start_thread, start_info);
6236 while (MONO_SEM_WAIT (&(start_info->registered)) != 0) {
6237 /*if (EINTR != errno) ABORT("sem_wait failed"); */
6240 MONO_SEM_DESTROY (&(start_info->registered));
6246 mono_gc_pthread_join (pthread_t thread, void **retval)
6248 return pthread_join (thread, retval);
6252 mono_gc_pthread_detach (pthread_t thread)
6254 return pthread_detach (thread);
6257 #endif /* USE_PTHREAD_INTERCEPT */
6260 * ######################################################################
6261 * ######## Write barriers
6262 * ######################################################################
6265 static RememberedSet*
6266 alloc_remset (int size, gpointer id) {
6267 RememberedSet* res = get_internal_mem (sizeof (RememberedSet) + (size * sizeof (gpointer)), INTERNAL_MEM_REMSET);
6268 res->store_next = res->data;
6269 res->end_set = res->data + size;
6271 DEBUG (4, fprintf (gc_debug_file, "Allocated remset size %d at %p for %p\n", size, res->data, id));
6276 * Note: the write barriers first do the needed GC work and then do the actual store:
6277 * this way the value is visible to the conservative GC scan after the write barrier
6278 * itself. If a GC interrupts the barrier in the middle, value will be kept alive by
6279 * the conservative scan, otherwise by the remembered set scan.
6282 mono_gc_wbarrier_set_field (MonoObject *obj, gpointer field_ptr, MonoObject* value)
6286 HEAVY_STAT (++stat_wbarrier_set_field);
6287 if (ptr_in_nursery (field_ptr)) {
6288 *(void**)field_ptr = value;
6291 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", field_ptr));
6293 rs = REMEMBERED_SET;
6294 if (rs->store_next < rs->end_set) {
6295 *(rs->store_next++) = (mword)field_ptr;
6296 *(void**)field_ptr = value;
6300 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6301 rs->next = REMEMBERED_SET;
6302 REMEMBERED_SET = rs;
6303 #ifdef HAVE_KW_THREAD
6304 mono_sgen_thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6306 *(rs->store_next++) = (mword)field_ptr;
6307 *(void**)field_ptr = value;
6312 mono_gc_wbarrier_set_arrayref (MonoArray *arr, gpointer slot_ptr, MonoObject* value)
6316 HEAVY_STAT (++stat_wbarrier_set_arrayref);
6317 if (ptr_in_nursery (slot_ptr)) {
6318 *(void**)slot_ptr = value;
6321 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", slot_ptr));
6323 rs = REMEMBERED_SET;
6324 if (rs->store_next < rs->end_set) {
6325 *(rs->store_next++) = (mword)slot_ptr;
6326 *(void**)slot_ptr = value;
6330 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6331 rs->next = REMEMBERED_SET;
6332 REMEMBERED_SET = rs;
6333 #ifdef HAVE_KW_THREAD
6334 mono_sgen_thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6336 *(rs->store_next++) = (mword)slot_ptr;
6337 *(void**)slot_ptr = value;
6342 mono_gc_wbarrier_arrayref_copy (gpointer dest_ptr, gpointer src_ptr, int count)
6346 HEAVY_STAT (++stat_wbarrier_arrayref_copy);
6348 memmove (dest_ptr, src_ptr, count * sizeof (gpointer));
6349 if (ptr_in_nursery (dest_ptr)) {
6353 rs = REMEMBERED_SET;
6354 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p, %d\n", dest_ptr, count));
6355 if (rs->store_next + 1 < rs->end_set) {
6356 *(rs->store_next++) = (mword)dest_ptr | REMSET_RANGE;
6357 *(rs->store_next++) = count;
6361 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6362 rs->next = REMEMBERED_SET;
6363 REMEMBERED_SET = rs;
6364 #ifdef HAVE_KW_THREAD
6365 mono_sgen_thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6367 *(rs->store_next++) = (mword)dest_ptr | REMSET_RANGE;
6368 *(rs->store_next++) = count;
6372 static char *found_obj;
6375 find_object_for_ptr_callback (char *obj, size_t size, char *ptr)
6377 if (ptr >= obj && ptr < obj + size) {
6378 g_assert (!found_obj);
6383 /* for use in the debugger */
6384 char* find_object_for_ptr (char *ptr);
6386 find_object_for_ptr (char *ptr)
6390 if (ptr >= nursery_section->data && ptr < nursery_section->end_data) {
6392 scan_area_with_callback (nursery_section->data, nursery_section->end_data,
6393 (IterateObjectCallbackFunc)find_object_for_ptr_callback, ptr);
6398 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
6399 if (ptr >= bigobj->data && ptr < bigobj->data + bigobj->size)
6400 return bigobj->data;
6404 * Very inefficient, but this is debugging code, supposed to
6405 * be called from gdb, so we don't care.
6408 major_iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)find_object_for_ptr_callback, ptr);
6413 evacuate_remset_buffer (void)
6418 buffer = STORE_REMSET_BUFFER;
6420 add_generic_store_remset_from_buffer (buffer);
6421 memset (buffer, 0, sizeof (gpointer) * STORE_REMSET_BUFFER_SIZE);
6423 STORE_REMSET_BUFFER_INDEX = 0;
6427 mono_gc_wbarrier_generic_nostore (gpointer ptr)
6433 HEAVY_STAT (++stat_wbarrier_generic_store);
6435 #ifdef XDOMAIN_CHECKS_IN_WBARRIER
6436 /* FIXME: ptr_in_heap must be called with the GC lock held */
6437 if (xdomain_checks && *(MonoObject**)ptr && ptr_in_heap (ptr)) {
6438 char *start = find_object_for_ptr (ptr);
6439 MonoObject *value = *(MonoObject**)ptr;
6443 MonoObject *obj = (MonoObject*)start;
6444 if (obj->vtable->domain != value->vtable->domain)
6445 g_assert (is_xdomain_ref_allowed (ptr, start, obj->vtable->domain));
6453 if (*(gpointer*)ptr)
6454 binary_protocol_wbarrier (ptr, *(gpointer*)ptr, (gpointer)LOAD_VTABLE (*(gpointer*)ptr));
6456 if (ptr_in_nursery (ptr) || ptr_on_stack (ptr) || !ptr_in_nursery (*(gpointer*)ptr)) {
6457 DEBUG (8, fprintf (gc_debug_file, "Skipping remset at %p\n", ptr));
6462 buffer = STORE_REMSET_BUFFER;
6463 index = STORE_REMSET_BUFFER_INDEX;
6464 /* This simple optimization eliminates a sizable portion of
6465 entries. Comparing it to the last but one entry as well
6466 doesn't eliminate significantly more entries. */
6467 if (buffer [index] == ptr) {
6472 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", ptr));
6473 HEAVY_STAT (++stat_wbarrier_generic_store_remset);
6476 if (index >= STORE_REMSET_BUFFER_SIZE) {
6477 evacuate_remset_buffer ();
6478 index = STORE_REMSET_BUFFER_INDEX;
6479 g_assert (index == 0);
6482 buffer [index] = ptr;
6483 STORE_REMSET_BUFFER_INDEX = index;
6489 mono_gc_wbarrier_generic_store (gpointer ptr, MonoObject* value)
6491 DEBUG (8, fprintf (gc_debug_file, "Wbarrier store at %p to %p (%s)\n", ptr, value, value ? safe_name (value) : "null"));
6492 *(void**)ptr = value;
6493 if (ptr_in_nursery (value))
6494 mono_gc_wbarrier_generic_nostore (ptr);
6497 void mono_gc_wbarrier_value_copy_bitmap (gpointer _dest, gpointer _src, int size, unsigned bitmap)
6499 mword *dest = _dest;
6504 mono_gc_wbarrier_generic_store (dest, (MonoObject*)*src);
6509 size -= SIZEOF_VOID_P;
6516 mono_gc_wbarrier_value_copy (gpointer dest, gpointer src, int count, MonoClass *klass)
6520 HEAVY_STAT (++stat_wbarrier_value_copy);
6521 g_assert (klass->valuetype);
6523 memmove (dest, src, count * mono_class_value_size (klass, NULL));
6524 rs = REMEMBERED_SET;
6525 if (ptr_in_nursery (dest) || ptr_on_stack (dest) || !klass->has_references) {
6529 g_assert (klass->gc_descr_inited);
6530 DEBUG (8, fprintf (gc_debug_file, "Adding value remset at %p, count %d, descr %p for class %s (%p)\n", dest, count, klass->gc_descr, klass->name, klass));
6532 if (rs->store_next + 3 < rs->end_set) {
6533 *(rs->store_next++) = (mword)dest | REMSET_VTYPE;
6534 *(rs->store_next++) = (mword)klass->gc_descr;
6535 *(rs->store_next++) = (mword)count;
6539 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6540 rs->next = REMEMBERED_SET;
6541 REMEMBERED_SET = rs;
6542 #ifdef HAVE_KW_THREAD
6543 mono_sgen_thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6545 *(rs->store_next++) = (mword)dest | REMSET_VTYPE;
6546 *(rs->store_next++) = (mword)klass->gc_descr;
6547 *(rs->store_next++) = (mword)count;
6552 * mono_gc_wbarrier_object_copy:
6554 * Write barrier to call when obj is the result of a clone or copy of an object.
6557 mono_gc_wbarrier_object_copy (MonoObject* obj, MonoObject *src)
6563 HEAVY_STAT (++stat_wbarrier_object_copy);
6564 rs = REMEMBERED_SET;
6565 DEBUG (6, fprintf (gc_debug_file, "Adding object remset for %p\n", obj));
6566 size = mono_object_class (obj)->instance_size;
6568 /* do not copy the sync state */
6569 memcpy ((char*)obj + sizeof (MonoObject), (char*)src + sizeof (MonoObject),
6570 size - sizeof (MonoObject));
6571 if (ptr_in_nursery (obj) || ptr_on_stack (obj)) {
6575 if (rs->store_next < rs->end_set) {
6576 *(rs->store_next++) = (mword)obj | REMSET_OBJECT;
6580 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6581 rs->next = REMEMBERED_SET;
6582 REMEMBERED_SET = rs;
6583 #ifdef HAVE_KW_THREAD
6584 mono_sgen_thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6586 *(rs->store_next++) = (mword)obj | REMSET_OBJECT;
6591 * ######################################################################
6592 * ######## Collector debugging
6593 * ######################################################################
6596 const char*descriptor_types [] = {
6608 describe_ptr (char *ptr)
6614 if (ptr_in_nursery (ptr)) {
6615 printf ("Pointer inside nursery.\n");
6617 if (major_ptr_is_in_non_pinned_space (ptr)) {
6618 printf ("Pointer inside oldspace.\n");
6619 } else if (obj_is_from_pinned_alloc (ptr)) {
6620 printf ("Pointer is inside a pinned chunk.\n");
6622 printf ("Pointer unknown.\n");
6627 if (object_is_pinned (ptr))
6628 printf ("Object is pinned.\n");
6630 if (object_is_forwarded (ptr))
6631 printf ("Object is forwared.\n");
6633 // FIXME: Handle pointers to the inside of objects
6634 vtable = (MonoVTable*)LOAD_VTABLE (ptr);
6636 printf ("VTable: %p\n", vtable);
6637 if (vtable == NULL) {
6638 printf ("VTable is invalid (empty).\n");
6641 if (ptr_in_nursery (vtable)) {
6642 printf ("VTable is invalid (points inside nursery).\n");
6645 printf ("Class: %s\n", vtable->klass->name);
6647 desc = ((GCVTable*)vtable)->desc;
6648 printf ("Descriptor: %lx\n", (long)desc);
6651 printf ("Descriptor type: %d (%s)\n", type, descriptor_types [type]);
6655 find_in_remset_loc (mword *p, char *addr, gboolean *found)
6661 switch ((*p) & REMSET_TYPE_MASK) {
6662 case REMSET_LOCATION:
6663 if (*p == (mword)addr)
6667 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
6669 if ((void**)addr >= ptr && (void**)addr < ptr + count)
6673 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
6674 count = safe_object_get_size ((MonoObject*)ptr);
6675 count = ALIGN_UP (count);
6676 count /= sizeof (mword);
6677 if ((void**)addr >= ptr && (void**)addr < ptr + count)
6681 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
6685 switch (desc & 0x7) {
6686 case DESC_TYPE_RUN_LENGTH:
6687 OBJ_RUN_LEN_SIZE (skip_size, desc, ptr);
6689 case DESC_TYPE_SMALL_BITMAP:
6690 OBJ_BITMAP_SIZE (skip_size, desc, start);
6694 g_assert_not_reached ();
6697 /* The descriptor includes the size of MonoObject */
6698 skip_size -= sizeof (MonoObject);
6700 if ((void**)addr >= ptr && (void**)addr < ptr + (skip_size / sizeof (gpointer)))
6705 g_assert_not_reached ();
6711 * Return whenever ADDR occurs in the remembered sets
6714 find_in_remsets (char *addr)
6717 SgenThreadInfo *info;
6718 RememberedSet *remset;
6719 GenericStoreRememberedSet *store_remset;
6721 gboolean found = FALSE;
6723 /* the global one */
6724 for (remset = global_remset; remset; remset = remset->next) {
6725 DEBUG (4, fprintf (gc_debug_file, "Scanning global remset range: %p-%p, size: %td\n", remset->data, remset->store_next, remset->store_next - remset->data));
6726 for (p = remset->data; p < remset->store_next;) {
6727 p = find_in_remset_loc (p, addr, &found);
6733 /* the generic store ones */
6734 for (store_remset = generic_store_remsets; store_remset; store_remset = store_remset->next) {
6735 for (i = 0; i < STORE_REMSET_BUFFER_SIZE - 1; ++i) {
6736 if (store_remset->data [i] == addr)
6741 /* the per-thread ones */
6742 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
6743 for (info = thread_table [i]; info; info = info->next) {
6745 for (remset = info->remset; remset; remset = remset->next) {
6746 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for thread %p, range: %p-%p, size: %td\n", info, remset->data, remset->store_next, remset->store_next - remset->data));
6747 for (p = remset->data; p < remset->store_next;) {
6748 p = find_in_remset_loc (p, addr, &found);
6753 for (j = 0; j < *info->store_remset_buffer_index_addr; ++j) {
6754 if ((*info->store_remset_buffer_addr) [j + 1] == addr)
6760 /* the freed thread ones */
6761 for (remset = freed_thread_remsets; remset; remset = remset->next) {
6762 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for freed thread, range: %p-%p, size: %td\n", remset->data, remset->store_next, remset->store_next - remset->data));
6763 for (p = remset->data; p < remset->store_next;) {
6764 p = find_in_remset_loc (p, addr, &found);
6773 static gboolean missing_remsets;
6776 * We let a missing remset slide if the target object is pinned,
6777 * because the store might have happened but the remset not yet added,
6778 * but in that case the target must be pinned. We might theoretically
6779 * miss some missing remsets this way, but it's very unlikely.
6782 #define HANDLE_PTR(ptr,obj) do { \
6783 if (*(ptr) && (char*)*(ptr) >= nursery_start && (char*)*(ptr) < nursery_next) { \
6784 if (!find_in_remsets ((char*)(ptr))) { \
6785 fprintf (gc_debug_file, "Oldspace->newspace reference %p at offset %td in object %p (%s.%s) not found in remsets.\n", *(ptr), (char*)(ptr) - (char*)(obj), (obj), ((MonoObject*)(obj))->vtable->klass->name_space, ((MonoObject*)(obj))->vtable->klass->name); \
6786 binary_protocol_missing_remset ((obj), (gpointer)LOAD_VTABLE ((obj)), (char*)(ptr) - (char*)(obj), *(ptr), (gpointer)LOAD_VTABLE(*(ptr)), object_is_pinned (*(ptr))); \
6787 if (!object_is_pinned (*(ptr))) \
6788 missing_remsets = TRUE; \
6794 * Check that each object reference which points into the nursery can
6795 * be found in the remembered sets.
6798 check_consistency_callback (char *start, size_t size, void *dummy)
6800 GCVTable *vt = (GCVTable*)LOAD_VTABLE (start);
6801 DEBUG (8, fprintf (gc_debug_file, "Scanning object %p, vtable: %p (%s)\n", start, vt, vt->klass->name));
6803 #define SCAN_OBJECT_ACTION
6804 #include "sgen-scan-object.h"
6808 * Perform consistency check of the heap.
6810 * Assumes the world is stopped.
6813 check_consistency (void)
6817 // Need to add more checks
6819 missing_remsets = FALSE;
6821 DEBUG (1, fprintf (gc_debug_file, "Begin heap consistency check...\n"));
6823 // Check that oldspace->newspace pointers are registered with the collector
6824 major_iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)check_consistency_callback, NULL);
6826 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
6827 check_consistency_callback (bigobj->data, bigobj->size, NULL);
6829 DEBUG (1, fprintf (gc_debug_file, "Heap consistency check done.\n"));
6831 #ifdef BINARY_PROTOCOL
6832 if (!binary_protocol_file)
6834 g_assert (!missing_remsets);
6839 #define HANDLE_PTR(ptr,obj) do { \
6841 g_assert (LOAD_VTABLE (*(ptr))); \
6845 check_major_refs_callback (char *start, size_t size, void *dummy)
6847 #define SCAN_OBJECT_ACTION
6848 #include "sgen-scan-object.h"
6852 check_major_refs (void)
6856 major_iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)check_major_refs_callback, NULL);
6858 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
6859 check_major_refs_callback (bigobj->data, bigobj->size, NULL);
6862 /* Check that the reference is valid */
6864 #define HANDLE_PTR(ptr,obj) do { \
6866 g_assert (safe_name (*(ptr)) != NULL); \
6873 * Perform consistency check on an object. Currently we only check that the
6874 * reference fields are valid.
6877 check_object (char *start)
6882 #include "sgen-scan-object.h"
6886 * ######################################################################
6887 * ######## Other mono public interface functions.
6888 * ######################################################################
6892 mono_gc_collect (int generation)
6896 if (generation == 0) {
6897 collect_nursery (0);
6899 major_collection ("user request");
6906 mono_gc_max_generation (void)
6912 mono_gc_collection_count (int generation)
6914 if (generation == 0)
6915 return num_minor_gcs;
6916 return num_major_gcs;
6920 mono_gc_get_used_size (void)
6924 tot = los_memory_usage;
6925 tot += nursery_section->next_data - nursery_section->data;
6926 tot += major_get_used_size ();
6927 /* FIXME: account for pinned objects */
6933 mono_gc_get_heap_size (void)
6939 mono_gc_disable (void)
6947 mono_gc_enable (void)
6955 mono_gc_get_los_limit (void)
6957 return MAX_SMALL_OBJ_SIZE;
6961 mono_object_is_alive (MonoObject* o)
6967 mono_gc_get_generation (MonoObject *obj)
6969 if (ptr_in_nursery (obj))
6975 mono_gc_enable_events (void)
6980 mono_gc_weak_link_add (void **link_addr, MonoObject *obj, gboolean track)
6983 mono_gc_register_disappearing_link (obj, link_addr, track);
6988 mono_gc_weak_link_remove (void **link_addr)
6991 mono_gc_register_disappearing_link (NULL, link_addr, FALSE);
6996 mono_gc_weak_link_get (void **link_addr)
7000 return (MonoObject*) REVEAL_POINTER (*link_addr);
7004 mono_gc_ephemeron_array_add (MonoObject *obj)
7006 EphemeronLinkNode *node;
7010 node = get_internal_mem (sizeof (EphemeronLinkNode), INTERNAL_MEM_EPHEMERON_LINK);
7015 node->array = (char*)obj;
7016 node->next = ephemeron_list;
7017 ephemeron_list = node;
7019 DEBUG (5, fprintf (gc_debug_file, "Registered ephemeron array %p\n", obj));
7026 mono_gc_make_descr_from_bitmap (gsize *bitmap, int numbits)
7028 if (numbits < ((sizeof (*bitmap) * 8) - ROOT_DESC_TYPE_SHIFT)) {
7029 return (void*)MAKE_ROOT_DESC (ROOT_DESC_BITMAP, bitmap [0]);
7031 mword complex = alloc_complex_descriptor (bitmap, numbits);
7032 return (void*)MAKE_ROOT_DESC (ROOT_DESC_COMPLEX, complex);
7037 mono_gc_make_root_descr_user (MonoGCRootMarkFunc marker)
7041 g_assert (user_descriptors_next < MAX_USER_DESCRIPTORS);
7042 descr = (void*)MAKE_ROOT_DESC (ROOT_DESC_USER, (mword)user_descriptors_next);
7043 user_descriptors [user_descriptors_next ++] = marker;
7049 mono_gc_alloc_fixed (size_t size, void *descr)
7051 /* FIXME: do a single allocation */
7052 void *res = calloc (1, size);
7055 if (!mono_gc_register_root (res, size, descr)) {
7063 mono_gc_free_fixed (void* addr)
7065 mono_gc_deregister_root (addr);
7070 mono_gc_invoke_with_gc_lock (MonoGCLockedCallbackFunc func, void *data)
7074 result = func (data);
7075 UNLOCK_INTERRUPTION;
7080 mono_gc_is_gc_thread (void)
7084 result = mono_sgen_thread_info_lookup (ARCH_GET_THREAD ()) != NULL;
7091 /* Tries to extract a number from the passed string, taking in to account m, k
7094 parse_environment_string_extract_number (gchar *str, glong *out)
7097 int len = strlen (str), shift = 0;
7099 gboolean is_suffix = FALSE;
7102 switch (str [len - 1]) {
7113 suffix = str [len - 1];
7118 val = strtol (str, &endptr, 10);
7120 if ((errno == ERANGE && (val == LONG_MAX || val == LONG_MIN))
7121 || (errno != 0 && val == 0) || (endptr == str))
7125 if (*(endptr + 1)) /* Invalid string. */
7137 mono_gc_base_init (void)
7141 struct sigaction sinfo;
7143 LOCK_INIT (gc_mutex);
7145 if (gc_initialized) {
7149 pagesize = mono_pagesize ();
7150 gc_debug_file = stderr;
7154 if ((env = getenv ("MONO_GC_PARAMS"))) {
7155 if (g_str_has_prefix (env, "nursery-size")) {
7158 while (env [index] && env [index++] != '=')
7160 if (env [index] && parse_environment_string_extract_number (env
7162 default_nursery_size = val;
7163 #ifdef ALIGN_NURSERY
7164 if ((val & (val - 1))) {
7165 fprintf (stderr, "The nursery size must be a power of two.\n");
7169 default_nursery_bits = 0;
7170 while (1 << (++ default_nursery_bits) != default_nursery_size)
7174 fprintf (stderr, "nursery-size must be an integer.\n");
7178 fprintf (stderr, "MONO_GC_PARAMS must be of the form 'nursery-size=N' (where N is an integer, possibly with a k, m or a g suffix).\n");
7185 nursery_size = DEFAULT_NURSERY_SIZE;
7186 minor_collection_allowance = MIN_MINOR_COLLECTION_ALLOWANCE;
7190 if ((env = getenv ("MONO_GC_DEBUG"))) {
7191 opts = g_strsplit (env, ",", -1);
7192 for (ptr = opts; ptr && *ptr; ptr ++) {
7194 if (opt [0] >= '0' && opt [0] <= '9') {
7195 gc_debug_level = atoi (opt);
7200 char *rf = g_strdup_printf ("%s.%d", opt, getpid ());
7201 gc_debug_file = fopen (rf, "wb");
7203 gc_debug_file = stderr;
7206 } else if (!strcmp (opt, "collect-before-allocs")) {
7207 collect_before_allocs = TRUE;
7208 } else if (!strcmp (opt, "check-at-minor-collections")) {
7209 consistency_check_at_minor_collection = TRUE;
7210 nursery_clear_policy = CLEAR_AT_GC;
7211 } else if (!strcmp (opt, "xdomain-checks")) {
7212 xdomain_checks = TRUE;
7213 } else if (!strcmp (opt, "clear-at-gc")) {
7214 nursery_clear_policy = CLEAR_AT_GC;
7215 } else if (!strcmp (opt, "conservative-stack-mark")) {
7216 conservative_stack_mark = TRUE;
7217 } else if (!strcmp (opt, "check-scan-starts")) {
7218 do_scan_starts_check = TRUE;
7219 } else if (g_str_has_prefix (opt, "heap-dump=")) {
7220 char *filename = strchr (opt, '=') + 1;
7221 nursery_clear_policy = CLEAR_AT_GC;
7222 heap_dump_file = fopen (filename, "w");
7224 fprintf (heap_dump_file, "<sgen-dump>\n");
7225 #ifdef BINARY_PROTOCOL
7226 } else if (g_str_has_prefix (opt, "binary-protocol=")) {
7227 char *filename = strchr (opt, '=') + 1;
7228 binary_protocol_file = fopen (filename, "w");
7231 fprintf (stderr, "Invalid format for the MONO_GC_DEBUG env variable: '%s'\n", env);
7232 fprintf (stderr, "The format is: MONO_GC_DEBUG=[l[:filename]|<option>]+ where l is a debug level 0-9.\n");
7233 fprintf (stderr, "Valid options are: collect-before-allocs, check-at-minor-collections, xdomain-checks, clear-at-gc.\n");
7240 suspend_ack_semaphore_ptr = &suspend_ack_semaphore;
7241 MONO_SEM_INIT (&suspend_ack_semaphore, 0);
7243 sigfillset (&sinfo.sa_mask);
7244 sinfo.sa_flags = SA_RESTART | SA_SIGINFO;
7245 sinfo.sa_sigaction = suspend_handler;
7246 if (sigaction (suspend_signal_num, &sinfo, NULL) != 0) {
7247 g_error ("failed sigaction");
7250 sinfo.sa_handler = restart_handler;
7251 if (sigaction (restart_signal_num, &sinfo, NULL) != 0) {
7252 g_error ("failed sigaction");
7255 sigfillset (&suspend_signal_mask);
7256 sigdelset (&suspend_signal_mask, restart_signal_num);
7258 global_remset = alloc_remset (1024, NULL);
7259 global_remset->next = NULL;
7261 pthread_key_create (&remembered_set_key, unregister_thread);
7263 #ifndef HAVE_KW_THREAD
7264 pthread_key_create (&thread_info_key, NULL);
7267 gc_initialized = TRUE;
7269 mono_gc_register_thread (&sinfo);
7273 mono_gc_get_suspend_signal (void)
7275 return suspend_signal_num;
7285 #ifdef HAVE_KW_THREAD
7286 #define EMIT_TLS_ACCESS(mb,dummy,offset) do { \
7287 mono_mb_emit_byte ((mb), MONO_CUSTOM_PREFIX); \
7288 mono_mb_emit_byte ((mb), CEE_MONO_TLS); \
7289 mono_mb_emit_i4 ((mb), (offset)); \
7292 #define EMIT_TLS_ACCESS(mb,member,dummy) do { \
7293 mono_mb_emit_byte ((mb), MONO_CUSTOM_PREFIX); \
7294 mono_mb_emit_byte ((mb), CEE_MONO_TLS); \
7295 mono_mb_emit_i4 ((mb), thread_info_key); \
7296 mono_mb_emit_icon ((mb), G_STRUCT_OFFSET (SgenThreadInfo, member)); \
7297 mono_mb_emit_byte ((mb), CEE_ADD); \
7298 mono_mb_emit_byte ((mb), CEE_LDIND_I); \
7302 #ifdef MANAGED_ALLOCATION
7303 /* FIXME: Do this in the JIT, where specialized allocation sequences can be created
7304 * for each class. This is currently not easy to do, as it is hard to generate basic
7305 * blocks + branches, but it is easy with the linear IL codebase.
7307 * For this to work we'd need to solve the TLAB race, first. Now we
7308 * require the allocator to be in a few known methods to make sure
7309 * that they are executed atomically via the restart mechanism.
7312 create_allocator (int atype)
7314 int p_var, size_var;
7315 guint32 slowpath_branch, max_size_branch;
7316 MonoMethodBuilder *mb;
7318 MonoMethodSignature *csig;
7319 static gboolean registered = FALSE;
7320 int tlab_next_addr_var, new_next_var;
7322 const char *name = NULL;
7323 AllocatorWrapperInfo *info;
7325 #ifdef HAVE_KW_THREAD
7326 int tlab_next_addr_offset = -1;
7327 int tlab_temp_end_offset = -1;
7329 MONO_THREAD_VAR_OFFSET (tlab_next_addr, tlab_next_addr_offset);
7330 MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
7332 g_assert (tlab_next_addr_offset != -1);
7333 g_assert (tlab_temp_end_offset != -1);
7337 mono_register_jit_icall (mono_gc_alloc_obj, "mono_gc_alloc_obj", mono_create_icall_signature ("object ptr int"), FALSE);
7338 mono_register_jit_icall (mono_gc_alloc_vector, "mono_gc_alloc_vector", mono_create_icall_signature ("object ptr int int"), FALSE);
7342 if (atype == ATYPE_SMALL) {
7344 name = "AllocSmall";
7345 } else if (atype == ATYPE_NORMAL) {
7348 } else if (atype == ATYPE_VECTOR) {
7350 name = "AllocVector";
7352 g_assert_not_reached ();
7355 csig = mono_metadata_signature_alloc (mono_defaults.corlib, num_params);
7356 csig->ret = &mono_defaults.object_class->byval_arg;
7357 for (i = 0; i < num_params; ++i)
7358 csig->params [i] = &mono_defaults.int_class->byval_arg;
7360 mb = mono_mb_new (mono_defaults.object_class, name, MONO_WRAPPER_ALLOC);
7361 size_var = mono_mb_add_local (mb, &mono_defaults.int32_class->byval_arg);
7362 if (atype == ATYPE_NORMAL || atype == ATYPE_SMALL) {
7363 /* size = vtable->klass->instance_size; */
7364 mono_mb_emit_ldarg (mb, 0);
7365 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoVTable, klass));
7366 mono_mb_emit_byte (mb, CEE_ADD);
7367 mono_mb_emit_byte (mb, CEE_LDIND_I);
7368 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoClass, instance_size));
7369 mono_mb_emit_byte (mb, CEE_ADD);
7370 /* FIXME: assert instance_size stays a 4 byte integer */
7371 mono_mb_emit_byte (mb, CEE_LDIND_U4);
7372 mono_mb_emit_stloc (mb, size_var);
7373 } else if (atype == ATYPE_VECTOR) {
7374 MonoExceptionClause *clause;
7376 MonoClass *oom_exc_class;
7379 /* n > MONO_ARRAY_MAX_INDEX -> OverflowException */
7380 mono_mb_emit_ldarg (mb, 1);
7381 mono_mb_emit_icon (mb, MONO_ARRAY_MAX_INDEX);
7382 pos = mono_mb_emit_short_branch (mb, CEE_BLE_UN_S);
7383 mono_mb_emit_exception (mb, "OverflowException", NULL);
7384 mono_mb_patch_short_branch (mb, pos);
7386 clause = mono_image_alloc0 (mono_defaults.corlib, sizeof (MonoExceptionClause));
7387 clause->try_offset = mono_mb_get_label (mb);
7389 /* vtable->klass->sizes.element_size */
7390 mono_mb_emit_ldarg (mb, 0);
7391 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoVTable, klass));
7392 mono_mb_emit_byte (mb, CEE_ADD);
7393 mono_mb_emit_byte (mb, CEE_LDIND_I);
7394 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoClass, sizes.element_size));
7395 mono_mb_emit_byte (mb, CEE_ADD);
7396 mono_mb_emit_byte (mb, CEE_LDIND_U4);
7399 mono_mb_emit_ldarg (mb, 1);
7400 mono_mb_emit_byte (mb, CEE_MUL_OVF_UN);
7401 /* + sizeof (MonoArray) */
7402 mono_mb_emit_icon (mb, sizeof (MonoArray));
7403 mono_mb_emit_byte (mb, CEE_ADD_OVF_UN);
7404 mono_mb_emit_stloc (mb, size_var);
7406 pos_leave = mono_mb_emit_branch (mb, CEE_LEAVE);
7409 clause->flags = MONO_EXCEPTION_CLAUSE_NONE;
7410 clause->try_len = mono_mb_get_pos (mb) - clause->try_offset;
7411 clause->data.catch_class = mono_class_from_name (mono_defaults.corlib,
7412 "System", "OverflowException");
7413 g_assert (clause->data.catch_class);
7414 clause->handler_offset = mono_mb_get_label (mb);
7416 oom_exc_class = mono_class_from_name (mono_defaults.corlib,
7417 "System", "OutOfMemoryException");
7418 g_assert (oom_exc_class);
7419 ctor = mono_class_get_method_from_name (oom_exc_class, ".ctor", 0);
7422 mono_mb_emit_byte (mb, CEE_POP);
7423 mono_mb_emit_op (mb, CEE_NEWOBJ, ctor);
7424 mono_mb_emit_byte (mb, CEE_THROW);
7426 clause->handler_len = mono_mb_get_pos (mb) - clause->handler_offset;
7427 mono_mb_set_clauses (mb, 1, clause);
7428 mono_mb_patch_branch (mb, pos_leave);
7431 g_assert_not_reached ();
7434 /* size += ALLOC_ALIGN - 1; */
7435 mono_mb_emit_ldloc (mb, size_var);
7436 mono_mb_emit_icon (mb, ALLOC_ALIGN - 1);
7437 mono_mb_emit_byte (mb, CEE_ADD);
7438 /* size &= ~(ALLOC_ALIGN - 1); */
7439 mono_mb_emit_icon (mb, ~(ALLOC_ALIGN - 1));
7440 mono_mb_emit_byte (mb, CEE_AND);
7441 mono_mb_emit_stloc (mb, size_var);
7443 /* if (size > MAX_SMALL_OBJ_SIZE) goto slowpath */
7444 if (atype != ATYPE_SMALL) {
7445 mono_mb_emit_ldloc (mb, size_var);
7446 mono_mb_emit_icon (mb, MAX_SMALL_OBJ_SIZE);
7447 max_size_branch = mono_mb_emit_short_branch (mb, MONO_CEE_BGT_S);
7451 * We need to modify tlab_next, but the JIT only supports reading, so we read
7452 * another tls var holding its address instead.
7455 /* tlab_next_addr (local) = tlab_next_addr (TLS var) */
7456 tlab_next_addr_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7457 EMIT_TLS_ACCESS (mb, tlab_next_addr, tlab_next_addr_offset);
7458 mono_mb_emit_stloc (mb, tlab_next_addr_var);
7460 /* p = (void**)tlab_next; */
7461 p_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7462 mono_mb_emit_ldloc (mb, tlab_next_addr_var);
7463 mono_mb_emit_byte (mb, CEE_LDIND_I);
7464 mono_mb_emit_stloc (mb, p_var);
7466 /* new_next = (char*)p + size; */
7467 new_next_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7468 mono_mb_emit_ldloc (mb, p_var);
7469 mono_mb_emit_ldloc (mb, size_var);
7470 mono_mb_emit_byte (mb, CEE_CONV_I);
7471 mono_mb_emit_byte (mb, CEE_ADD);
7472 mono_mb_emit_stloc (mb, new_next_var);
7474 /* tlab_next = new_next */
7475 mono_mb_emit_ldloc (mb, tlab_next_addr_var);
7476 mono_mb_emit_ldloc (mb, new_next_var);
7477 mono_mb_emit_byte (mb, CEE_STIND_I);
7479 /* if (G_LIKELY (new_next < tlab_temp_end)) */
7480 mono_mb_emit_ldloc (mb, new_next_var);
7481 EMIT_TLS_ACCESS (mb, tlab_temp_end, tlab_temp_end_offset);
7482 slowpath_branch = mono_mb_emit_short_branch (mb, MONO_CEE_BLT_UN_S);
7485 if (atype != ATYPE_SMALL)
7486 mono_mb_patch_short_branch (mb, max_size_branch);
7488 mono_mb_emit_byte (mb, MONO_CUSTOM_PREFIX);
7489 mono_mb_emit_byte (mb, CEE_MONO_NOT_TAKEN);
7491 /* FIXME: mono_gc_alloc_obj takes a 'size_t' as an argument, not an int32 */
7492 mono_mb_emit_ldarg (mb, 0);
7493 mono_mb_emit_ldloc (mb, size_var);
7494 if (atype == ATYPE_NORMAL || atype == ATYPE_SMALL) {
7495 mono_mb_emit_icall (mb, mono_gc_alloc_obj);
7496 } else if (atype == ATYPE_VECTOR) {
7497 mono_mb_emit_ldarg (mb, 1);
7498 mono_mb_emit_icall (mb, mono_gc_alloc_vector);
7500 g_assert_not_reached ();
7502 mono_mb_emit_byte (mb, CEE_RET);
7505 mono_mb_patch_short_branch (mb, slowpath_branch);
7507 /* FIXME: Memory barrier */
7510 mono_mb_emit_ldloc (mb, p_var);
7511 mono_mb_emit_ldarg (mb, 0);
7512 mono_mb_emit_byte (mb, CEE_STIND_I);
7514 if (atype == ATYPE_VECTOR) {
7515 /* arr->max_length = max_length; */
7516 mono_mb_emit_ldloc (mb, p_var);
7517 mono_mb_emit_ldflda (mb, G_STRUCT_OFFSET (MonoArray, max_length));
7518 mono_mb_emit_ldarg (mb, 1);
7519 mono_mb_emit_byte (mb, CEE_STIND_I);
7523 mono_mb_emit_ldloc (mb, p_var);
7524 mono_mb_emit_byte (mb, CEE_RET);
7526 res = mono_mb_create_method (mb, csig, 8);
7528 mono_method_get_header (res)->init_locals = FALSE;
7530 info = mono_image_alloc0 (mono_defaults.corlib, sizeof (AllocatorWrapperInfo));
7531 info->alloc_type = atype;
7532 mono_marshal_set_wrapper_info (res, info);
7538 static MonoMethod* alloc_method_cache [ATYPE_NUM];
7539 static MonoMethod *write_barrier_method;
7542 is_ip_in_managed_allocator (MonoDomain *domain, gpointer ip)
7550 ji = mono_jit_info_table_find (domain, ip);
7553 method = ji->method;
7555 if (method == write_barrier_method)
7557 for (i = 0; i < ATYPE_NUM; ++i)
7558 if (method == alloc_method_cache [i])
7564 * Generate an allocator method implementing the fast path of mono_gc_alloc_obj ().
7565 * The signature of the called method is:
7566 * object allocate (MonoVTable *vtable)
7569 mono_gc_get_managed_allocator (MonoVTable *vtable, gboolean for_box)
7571 #ifdef MANAGED_ALLOCATION
7572 MonoClass *klass = vtable->klass;
7574 #ifdef HAVE_KW_THREAD
7575 int tlab_next_offset = -1;
7576 int tlab_temp_end_offset = -1;
7577 MONO_THREAD_VAR_OFFSET (tlab_next, tlab_next_offset);
7578 MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
7580 if (tlab_next_offset == -1 || tlab_temp_end_offset == -1)
7584 if (!mono_runtime_has_tls_get ())
7586 if (klass->instance_size > tlab_size)
7588 if (klass->has_finalize || klass->marshalbyref || (mono_profiler_get_events () & MONO_PROFILE_ALLOCATIONS))
7592 if (klass->byval_arg.type == MONO_TYPE_STRING)
7594 if (collect_before_allocs)
7597 if (ALIGN_TO (klass->instance_size, ALLOC_ALIGN) < MAX_SMALL_OBJ_SIZE)
7598 return mono_gc_get_managed_allocator_by_type (ATYPE_SMALL);
7600 return mono_gc_get_managed_allocator_by_type (ATYPE_NORMAL);
7607 mono_gc_get_managed_array_allocator (MonoVTable *vtable, int rank)
7609 #ifdef MANAGED_ALLOCATION
7610 MonoClass *klass = vtable->klass;
7612 #ifdef HAVE_KW_THREAD
7613 int tlab_next_offset = -1;
7614 int tlab_temp_end_offset = -1;
7615 MONO_THREAD_VAR_OFFSET (tlab_next, tlab_next_offset);
7616 MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
7618 if (tlab_next_offset == -1 || tlab_temp_end_offset == -1)
7624 if (!mono_runtime_has_tls_get ())
7626 if (mono_profiler_get_events () & MONO_PROFILE_ALLOCATIONS)
7628 if (collect_before_allocs)
7630 g_assert (!klass->has_finalize && !klass->marshalbyref);
7632 return mono_gc_get_managed_allocator_by_type (ATYPE_VECTOR);
7639 mono_gc_get_managed_allocator_by_type (int atype)
7641 #ifdef MANAGED_ALLOCATION
7644 if (!mono_runtime_has_tls_get ())
7647 mono_loader_lock ();
7648 res = alloc_method_cache [atype];
7650 res = alloc_method_cache [atype] = create_allocator (atype);
7651 mono_loader_unlock ();
7659 mono_gc_get_managed_allocator_types (void)
7666 mono_gc_get_write_barrier (void)
7669 MonoMethodBuilder *mb;
7670 MonoMethodSignature *sig;
7671 #ifdef MANAGED_WBARRIER
7672 int label_no_wb_1, label_no_wb_2, label_no_wb_3, label_no_wb_4, label_need_wb, label_slow_path;
7673 #ifndef ALIGN_NURSERY
7674 int label_continue_1, label_continue_2, label_no_wb_5;
7675 int dereferenced_var;
7677 int buffer_var, buffer_index_var, dummy_var;
7679 #ifdef HAVE_KW_THREAD
7680 int stack_end_offset = -1, store_remset_buffer_offset = -1;
7681 int store_remset_buffer_index_offset = -1, store_remset_buffer_index_addr_offset = -1;
7683 MONO_THREAD_VAR_OFFSET (stack_end, stack_end_offset);
7684 g_assert (stack_end_offset != -1);
7685 MONO_THREAD_VAR_OFFSET (store_remset_buffer, store_remset_buffer_offset);
7686 g_assert (store_remset_buffer_offset != -1);
7687 MONO_THREAD_VAR_OFFSET (store_remset_buffer_index, store_remset_buffer_index_offset);
7688 g_assert (store_remset_buffer_index_offset != -1);
7689 MONO_THREAD_VAR_OFFSET (store_remset_buffer_index_addr, store_remset_buffer_index_addr_offset);
7690 g_assert (store_remset_buffer_index_addr_offset != -1);
7694 // FIXME: Maybe create a separate version for ctors (the branch would be
7695 // correctly predicted more times)
7696 if (write_barrier_method)
7697 return write_barrier_method;
7699 /* Create the IL version of mono_gc_barrier_generic_store () */
7700 sig = mono_metadata_signature_alloc (mono_defaults.corlib, 1);
7701 sig->ret = &mono_defaults.void_class->byval_arg;
7702 sig->params [0] = &mono_defaults.int_class->byval_arg;
7704 mb = mono_mb_new (mono_defaults.object_class, "wbarrier", MONO_WRAPPER_WRITE_BARRIER);
7706 #ifdef MANAGED_WBARRIER
7707 if (mono_runtime_has_tls_get ()) {
7708 #ifdef ALIGN_NURSERY
7709 // if (ptr_in_nursery (ptr)) return;
7711 * Masking out the bits might be faster, but we would have to use 64 bit
7712 * immediates, which might be slower.
7714 mono_mb_emit_ldarg (mb, 0);
7715 mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
7716 mono_mb_emit_byte (mb, CEE_SHR_UN);
7717 mono_mb_emit_icon (mb, (mword)nursery_start >> DEFAULT_NURSERY_BITS);
7718 label_no_wb_1 = mono_mb_emit_branch (mb, CEE_BEQ);
7720 // if (!ptr_in_nursery (*ptr)) return;
7721 mono_mb_emit_ldarg (mb, 0);
7722 mono_mb_emit_byte (mb, CEE_LDIND_I);
7723 mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
7724 mono_mb_emit_byte (mb, CEE_SHR_UN);
7725 mono_mb_emit_icon (mb, (mword)nursery_start >> DEFAULT_NURSERY_BITS);
7726 label_no_wb_2 = mono_mb_emit_branch (mb, CEE_BNE_UN);
7729 // if (ptr < (nursery_start)) goto continue;
7730 mono_mb_emit_ldarg (mb, 0);
7731 mono_mb_emit_ptr (mb, (gpointer) nursery_start);
7732 label_continue_1 = mono_mb_emit_branch (mb, CEE_BLT);
7734 // if (ptr >= nursery_real_end)) goto continue;
7735 mono_mb_emit_ldarg (mb, 0);
7736 mono_mb_emit_ptr (mb, (gpointer) nursery_real_end);
7737 label_continue_2 = mono_mb_emit_branch (mb, CEE_BGE);
7740 label_no_wb_1 = mono_mb_emit_branch (mb, CEE_BR);
7743 mono_mb_patch_branch (mb, label_continue_1);
7744 mono_mb_patch_branch (mb, label_continue_2);
7746 // Dereference and store in local var
7747 dereferenced_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7748 mono_mb_emit_ldarg (mb, 0);
7749 mono_mb_emit_byte (mb, CEE_LDIND_I);
7750 mono_mb_emit_stloc (mb, dereferenced_var);
7752 // if (*ptr < nursery_start) return;
7753 mono_mb_emit_ldloc (mb, dereferenced_var);
7754 mono_mb_emit_ptr (mb, (gpointer) nursery_start);
7755 label_no_wb_2 = mono_mb_emit_branch (mb, CEE_BLT);
7757 // if (*ptr >= nursery_end) return;
7758 mono_mb_emit_ldloc (mb, dereferenced_var);
7759 mono_mb_emit_ptr (mb, (gpointer) nursery_real_end);
7760 label_no_wb_5 = mono_mb_emit_branch (mb, CEE_BGE);
7763 // if (ptr >= stack_end) goto need_wb;
7764 mono_mb_emit_ldarg (mb, 0);
7765 EMIT_TLS_ACCESS (mb, stack_end, stack_end_offset);
7766 label_need_wb = mono_mb_emit_branch (mb, CEE_BGE_UN);
7768 // if (ptr >= stack_start) return;
7769 dummy_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7770 mono_mb_emit_ldarg (mb, 0);
7771 mono_mb_emit_ldloc_addr (mb, dummy_var);
7772 label_no_wb_3 = mono_mb_emit_branch (mb, CEE_BGE_UN);
7775 mono_mb_patch_branch (mb, label_need_wb);
7777 // buffer = STORE_REMSET_BUFFER;
7778 buffer_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7779 EMIT_TLS_ACCESS (mb, store_remset_buffer, store_remset_buffer_offset);
7780 mono_mb_emit_stloc (mb, buffer_var);
7782 // buffer_index = STORE_REMSET_BUFFER_INDEX;
7783 buffer_index_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7784 EMIT_TLS_ACCESS (mb, store_remset_buffer_index, store_remset_buffer_index_offset);
7785 mono_mb_emit_stloc (mb, buffer_index_var);
7787 // if (buffer [buffer_index] == ptr) return;
7788 mono_mb_emit_ldloc (mb, buffer_var);
7789 mono_mb_emit_ldloc (mb, buffer_index_var);
7790 g_assert (sizeof (gpointer) == 4 || sizeof (gpointer) == 8);
7791 mono_mb_emit_icon (mb, sizeof (gpointer) == 4 ? 2 : 3);
7792 mono_mb_emit_byte (mb, CEE_SHL);
7793 mono_mb_emit_byte (mb, CEE_ADD);
7794 mono_mb_emit_byte (mb, CEE_LDIND_I);
7795 mono_mb_emit_ldarg (mb, 0);
7796 label_no_wb_4 = mono_mb_emit_branch (mb, CEE_BEQ);
7799 mono_mb_emit_ldloc (mb, buffer_index_var);
7800 mono_mb_emit_icon (mb, 1);
7801 mono_mb_emit_byte (mb, CEE_ADD);
7802 mono_mb_emit_stloc (mb, buffer_index_var);
7804 // if (buffer_index >= STORE_REMSET_BUFFER_SIZE) goto slow_path;
7805 mono_mb_emit_ldloc (mb, buffer_index_var);
7806 mono_mb_emit_icon (mb, STORE_REMSET_BUFFER_SIZE);
7807 label_slow_path = mono_mb_emit_branch (mb, CEE_BGE);
7809 // buffer [buffer_index] = ptr;
7810 mono_mb_emit_ldloc (mb, buffer_var);
7811 mono_mb_emit_ldloc (mb, buffer_index_var);
7812 g_assert (sizeof (gpointer) == 4 || sizeof (gpointer) == 8);
7813 mono_mb_emit_icon (mb, sizeof (gpointer) == 4 ? 2 : 3);
7814 mono_mb_emit_byte (mb, CEE_SHL);
7815 mono_mb_emit_byte (mb, CEE_ADD);
7816 mono_mb_emit_ldarg (mb, 0);
7817 mono_mb_emit_byte (mb, CEE_STIND_I);
7819 // STORE_REMSET_BUFFER_INDEX = buffer_index;
7820 EMIT_TLS_ACCESS (mb, store_remset_buffer_index_addr, store_remset_buffer_index_addr_offset);
7821 mono_mb_emit_ldloc (mb, buffer_index_var);
7822 mono_mb_emit_byte (mb, CEE_STIND_I);
7825 mono_mb_patch_branch (mb, label_no_wb_1);
7826 mono_mb_patch_branch (mb, label_no_wb_2);
7827 mono_mb_patch_branch (mb, label_no_wb_3);
7828 mono_mb_patch_branch (mb, label_no_wb_4);
7829 #ifndef ALIGN_NURSERY
7830 mono_mb_patch_branch (mb, label_no_wb_5);
7832 mono_mb_emit_byte (mb, CEE_RET);
7835 mono_mb_patch_branch (mb, label_slow_path);
7839 mono_mb_emit_ldarg (mb, 0);
7840 mono_mb_emit_icall (mb, mono_gc_wbarrier_generic_nostore);
7841 mono_mb_emit_byte (mb, CEE_RET);
7843 res = mono_mb_create_method (mb, sig, 16);
7846 mono_loader_lock ();
7847 if (write_barrier_method) {
7848 /* Already created */
7849 mono_free_method (res);
7851 /* double-checked locking */
7852 mono_memory_barrier ();
7853 write_barrier_method = res;
7855 mono_loader_unlock ();
7857 return write_barrier_method;
7861 mono_gc_get_description (void)
7863 return g_strdup ("sgen");
7867 mono_gc_set_desktop_mode (void)
7872 mono_gc_is_moving (void)
7878 mono_gc_is_disabled (void)
7884 mono_sgen_is_worker_thread (pthread_t thread)
7889 #endif /* HAVE_SGEN_GC */