2 * sgen-gc.c: Simple generational GC.
5 * Paolo Molaro (lupus@ximian.com)
7 * Copyright 2005-2010 Novell, Inc (http://www.novell.com)
9 * Thread start/stop adapted from Boehm's GC:
10 * Copyright (c) 1994 by Xerox Corporation. All rights reserved.
11 * Copyright (c) 1996 by Silicon Graphics. All rights reserved.
12 * Copyright (c) 1998 by Fergus Henderson. All rights reserved.
13 * Copyright (c) 2000-2004 by Hewlett-Packard Company. All rights reserved.
15 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
16 * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
18 * Permission is hereby granted to use or copy this program
19 * for any purpose, provided the above notices are retained on all copies.
20 * Permission to modify the code and to distribute modified code is granted,
21 * provided the above notices are retained, and a notice that the code was
22 * modified is included with the above copyright notice.
25 * Copyright 2001-2003 Ximian, Inc
26 * Copyright 2003-2010 Novell, Inc.
28 * Permission is hereby granted, free of charge, to any person obtaining
29 * a copy of this software and associated documentation files (the
30 * "Software"), to deal in the Software without restriction, including
31 * without limitation the rights to use, copy, modify, merge, publish,
32 * distribute, sublicense, and/or sell copies of the Software, and to
33 * permit persons to whom the Software is furnished to do so, subject to
34 * the following conditions:
36 * The above copyright notice and this permission notice shall be
37 * included in all copies or substantial portions of the Software.
39 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
40 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
41 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
42 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
43 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
44 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
45 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
48 * Important: allocation provides always zeroed memory, having to do
49 * a memset after allocation is deadly for performance.
50 * Memory usage at startup is currently as follows:
52 * 64 KB internal space
54 * We should provide a small memory config with half the sizes
56 * We currently try to make as few mono assumptions as possible:
57 * 1) 2-word header with no GC pointers in it (first vtable, second to store the
59 * 2) gc descriptor is the second word in the vtable (first word in the class)
60 * 3) 8 byte alignment is the minimum and enough (not true for special structures (SIMD), FIXME)
61 * 4) there is a function to get an object's size and the number of
62 * elements in an array.
63 * 5) we know the special way bounds are allocated for complex arrays
64 * 6) we know about proxies and how to treat them when domains are unloaded
66 * Always try to keep stack usage to a minimum: no recursive behaviour
67 * and no large stack allocs.
69 * General description.
70 * Objects are initially allocated in a nursery using a fast bump-pointer technique.
71 * When the nursery is full we start a nursery collection: this is performed with a
73 * When the old generation is full we start a copying GC of the old generation as well:
74 * this will be changed to mark&sweep with copying when fragmentation becomes to severe
75 * in the future. Maybe we'll even do both during the same collection like IMMIX.
77 * The things that complicate this description are:
78 * *) pinned objects: we can't move them so we need to keep track of them
79 * *) no precise info of the thread stacks and registers: we need to be able to
80 * quickly find the objects that may be referenced conservatively and pin them
81 * (this makes the first issues more important)
82 * *) large objects are too expensive to be dealt with using copying GC: we handle them
83 * with mark/sweep during major collections
84 * *) some objects need to not move even if they are small (interned strings, Type handles):
85 * we use mark/sweep for them, too: they are not allocated in the nursery, but inside
86 * PinnedChunks regions
92 *) we could have a function pointer in MonoClass to implement
93 customized write barriers for value types
95 *) investigate the stuff needed to advance a thread to a GC-safe
96 point (single-stepping, read from unmapped memory etc) and implement it.
97 This would enable us to inline allocations and write barriers, for example,
98 or at least parts of them, like the write barrier checks.
99 We may need this also for handling precise info on stacks, even simple things
100 as having uninitialized data on the stack and having to wait for the prolog
101 to zero it. Not an issue for the last frame that we scan conservatively.
102 We could always not trust the value in the slots anyway.
104 *) modify the jit to save info about references in stack locations:
105 this can be done just for locals as a start, so that at least
106 part of the stack is handled precisely.
108 *) test/fix endianess issues
110 *) Implement a card table as the write barrier instead of remembered
111 sets? Card tables are not easy to implement with our current
112 memory layout. We have several different kinds of major heap
113 objects: Small objects in regular blocks, small objects in pinned
114 chunks and LOS objects. If we just have a pointer we have no way
115 to tell which kind of object it points into, therefore we cannot
116 know where its card table is. The least we have to do to make
117 this happen is to get rid of write barriers for indirect stores.
120 *) Get rid of write barriers for indirect stores. We can do this by
121 telling the GC to wbarrier-register an object once we do an ldloca
122 or ldelema on it, and to unregister it once it's not used anymore
123 (it can only travel downwards on the stack). The problem with
124 unregistering is that it needs to happen eventually no matter
125 what, even if exceptions are thrown, the thread aborts, etc.
126 Rodrigo suggested that we could do only the registering part and
127 let the collector find out (pessimistically) when it's safe to
128 unregister, namely when the stack pointer of the thread that
129 registered the object is higher than it was when the registering
130 happened. This might make for a good first implementation to get
131 some data on performance.
133 *) Some sort of blacklist support? Blacklists is a concept from the
134 Boehm GC: if during a conservative scan we find pointers to an
135 area which we might use as heap, we mark that area as unusable, so
136 pointer retention by random pinning pointers is reduced.
138 *) experiment with max small object size (very small right now - 2kb,
139 because it's tied to the max freelist size)
141 *) add an option to mmap the whole heap in one chunk: it makes for many
142 simplifications in the checks (put the nursery at the top and just use a single
143 check for inclusion/exclusion): the issue this has is that on 32 bit systems it's
144 not flexible (too much of the address space may be used by default or we can't
145 increase the heap as needed) and we'd need a race-free mechanism to return memory
146 back to the system (mprotect(PROT_NONE) will still keep the memory allocated if it
147 was written to, munmap is needed, but the following mmap may not find the same segment
150 *) memzero the major fragments after restarting the world and optionally a smaller
153 *) investigate having fragment zeroing threads
155 *) separate locks for finalization and other minor stuff to reduce
158 *) try a different copying order to improve memory locality
160 *) a thread abort after a store but before the write barrier will
161 prevent the write barrier from executing
163 *) specialized dynamically generated markers/copiers
165 *) Dynamically adjust TLAB size to the number of threads. If we have
166 too many threads that do allocation, we might need smaller TLABs,
167 and we might get better performance with larger TLABs if we only
168 have a handful of threads. We could sum up the space left in all
169 assigned TLABs and if that's more than some percentage of the
170 nursery size, reduce the TLAB size.
172 *) Explore placing unreachable objects on unused nursery memory.
173 Instead of memset'ng a region to zero, place an int[] covering it.
174 A good place to start is add_nursery_frag. The tricky thing here is
175 placing those objects atomically outside of a collection.
185 #include <semaphore.h>
194 #define _XOPEN_SOURCE
196 #include "metadata/metadata-internals.h"
197 #include "metadata/class-internals.h"
198 #include "metadata/gc-internal.h"
199 #include "metadata/object-internals.h"
200 #include "metadata/threads.h"
201 #include "metadata/sgen-gc.h"
202 #include "metadata/sgen-archdep.h"
203 #include "metadata/mono-gc.h"
204 #include "metadata/method-builder.h"
205 #include "metadata/profiler-private.h"
206 #include "metadata/monitor.h"
207 #include "metadata/threadpool-internals.h"
208 #include "metadata/mempool-internals.h"
209 #include "metadata/marshal.h"
210 #include "utils/mono-mmap.h"
211 #include "utils/mono-time.h"
212 #include "utils/mono-semaphore.h"
213 #include "utils/mono-counters.h"
215 #include <mono/utils/memcheck.h>
217 #if defined(__MACH__)
218 #include "utils/mach-support.h"
221 #define OPDEF(a,b,c,d,e,f,g,h,i,j) \
225 #include "mono/cil/opcode.def"
231 #undef pthread_create
233 #undef pthread_detach
236 * ######################################################################
237 * ######## Types and constants used by the GC.
238 * ######################################################################
241 static int gc_initialized = 0;
242 static int gc_debug_level = 0;
243 static FILE* gc_debug_file;
244 /* If set, do a minor collection before every allocation */
245 static gboolean collect_before_allocs = FALSE;
246 /* If set, do a heap consistency check before each minor collection */
247 static gboolean consistency_check_at_minor_collection = FALSE;
248 /* If set, check that there are no references to the domain left at domain unload */
249 static gboolean xdomain_checks = FALSE;
250 /* If not null, dump the heap after each collection into this file */
251 static FILE *heap_dump_file = NULL;
252 /* If set, mark stacks conservatively, even if precise marking is possible */
253 static gboolean conservative_stack_mark = TRUE;
254 /* If set, do a plausibility check on the scan_starts before and after
256 static gboolean do_scan_starts_check = FALSE;
259 * Turning on heavy statistics will turn off the managed allocator and
260 * the managed write barrier.
262 //#define HEAVY_STATISTICS
264 #ifdef HEAVY_STATISTICS
265 #define HEAVY_STAT(x) x
267 #define HEAVY_STAT(x)
270 #ifdef HEAVY_STATISTICS
271 static long long stat_objects_alloced = 0;
272 static long long stat_bytes_alloced = 0;
273 static long long stat_objects_alloced_degraded = 0;
274 static long long stat_bytes_alloced_degraded = 0;
275 static long long stat_bytes_alloced_los = 0;
277 static long long stat_copy_object_called_nursery = 0;
278 static long long stat_objects_copied_nursery = 0;
279 static long long stat_copy_object_called_major = 0;
280 static long long stat_objects_copied_major = 0;
282 static long long stat_scan_object_called_nursery = 0;
283 static long long stat_scan_object_called_major = 0;
285 static long long stat_nursery_copy_object_failed_from_space = 0;
286 static long long stat_nursery_copy_object_failed_forwarded = 0;
287 static long long stat_nursery_copy_object_failed_pinned = 0;
289 static long long stat_store_remsets = 0;
290 static long long stat_store_remsets_unique = 0;
291 static long long stat_saved_remsets_1 = 0;
292 static long long stat_saved_remsets_2 = 0;
293 static long long stat_global_remsets_added = 0;
294 static long long stat_global_remsets_readded = 0;
295 static long long stat_global_remsets_processed = 0;
296 static long long stat_global_remsets_discarded = 0;
298 static long long stat_wasted_fragments_used = 0;
299 static long long stat_wasted_fragments_bytes = 0;
301 static int stat_wbarrier_set_field = 0;
302 static int stat_wbarrier_set_arrayref = 0;
303 static int stat_wbarrier_arrayref_copy = 0;
304 static int stat_wbarrier_generic_store = 0;
305 static int stat_wbarrier_generic_store_remset = 0;
306 static int stat_wbarrier_set_root = 0;
307 static int stat_wbarrier_value_copy = 0;
308 static int stat_wbarrier_object_copy = 0;
311 static long long time_minor_pre_collection_fragment_clear = 0;
312 static long long time_minor_pinning = 0;
313 static long long time_minor_scan_remsets = 0;
314 static long long time_minor_scan_pinned = 0;
315 static long long time_minor_scan_registered_roots = 0;
316 static long long time_minor_scan_thread_data = 0;
317 static long long time_minor_finish_gray_stack = 0;
318 static long long time_minor_fragment_creation = 0;
320 static long long time_major_pre_collection_fragment_clear = 0;
321 static long long time_major_pinning = 0;
322 static long long time_major_scan_pinned = 0;
323 static long long time_major_scan_registered_roots = 0;
324 static long long time_major_scan_thread_data = 0;
325 static long long time_major_scan_alloc_pinned = 0;
326 static long long time_major_scan_finalized = 0;
327 static long long time_major_scan_big_objects = 0;
328 static long long time_major_finish_gray_stack = 0;
329 static long long time_major_free_bigobjs = 0;
330 static long long time_major_los_sweep = 0;
331 static long long time_major_sweep = 0;
332 static long long time_major_fragment_creation = 0;
334 static long long pinned_chunk_bytes_alloced = 0;
335 static long long large_internal_bytes_alloced = 0;
337 /* Keep in sync with internal_mem_names in dump_heap()! */
339 INTERNAL_MEM_PIN_QUEUE,
340 INTERNAL_MEM_FRAGMENT,
341 INTERNAL_MEM_SECTION,
342 INTERNAL_MEM_SCAN_STARTS,
343 INTERNAL_MEM_FIN_TABLE,
344 INTERNAL_MEM_FINALIZE_ENTRY,
345 INTERNAL_MEM_DISLINK_TABLE,
346 INTERNAL_MEM_DISLINK,
347 INTERNAL_MEM_ROOTS_TABLE,
348 INTERNAL_MEM_ROOT_RECORD,
349 INTERNAL_MEM_STATISTICS,
351 INTERNAL_MEM_GRAY_QUEUE,
352 INTERNAL_MEM_STORE_REMSET,
353 INTERNAL_MEM_MS_TABLES,
354 INTERNAL_MEM_MS_BLOCK_INFO,
355 INTERNAL_MEM_EPHEMERON_LINK,
359 static long small_internal_mem_bytes [INTERNAL_MEM_MAX];
363 mono_gc_flush_info (void)
365 fflush (gc_debug_file);
369 /* Define this to allow the user to change some of the constants by specifying
370 * their values in the MONO_GC_PARAMS environmental variable. See
371 * mono_gc_base_init for details. */
372 #define USER_CONFIG 1
374 #define TV_DECLARE(name) gint64 name
375 #define TV_GETTIME(tv) tv = mono_100ns_ticks ()
376 #define TV_ELAPSED(start,end) (int)((end-start) / 10)
377 #define TV_ELAPSED_MS(start,end) ((TV_ELAPSED((start),(end)) + 500) / 1000)
379 #define ALIGN_TO(val,align) ((((guint64)val) + ((align) - 1)) & ~((align) - 1))
381 #define GC_BITS_PER_WORD (sizeof (mword) * 8)
389 typedef struct _Block Block;
395 /* each request from the OS ends up in a GCMemSection */
396 typedef struct _GCMemSection GCMemSection;
397 struct _GCMemSection {
401 /* pointer where more data could be allocated if it fits */
405 * scan starts is an array of pointers to objects equally spaced in the allocation area
406 * They let use quickly find pinned objects from pinning pointers.
409 /* in major collections indexes in the pin_queue for objects that pin this section */
412 unsigned short num_scan_start;
413 gboolean is_to_space;
416 #define SIZEOF_GC_MEM_SECTION ((sizeof (GCMemSection) + 7) & ~7)
418 /* Pinned objects are allocated in the LOS space if bigger than half a page
419 * or from freelists otherwise. We assume that pinned objects are relatively few
420 * and they have a slow dying speed (like interned strings, thread objects).
421 * As such they will be collected only at major collections.
422 * free lists are not global: when we need memory we allocate a PinnedChunk.
423 * Each pinned chunk is made of several pages, the first of wich is used
424 * internally for bookeeping (here think of a page as 4KB). The bookeeping
425 * includes the freelists vectors and info about the object size of each page
426 * in the pinned chunk. So, when needed, a free page is found in a pinned chunk,
427 * a size is assigned to it, the page is divided in the proper chunks and each
428 * chunk is added to the freelist. To not waste space, the remaining space in the
429 * first page is used as objects of size 16 or 32 (need to measure which are more
431 * We use this same structure to allocate memory used internally by the GC, so
432 * we never use malloc/free if we need to alloc during collection: the world is stopped
433 * and malloc/free will deadlock.
434 * When we want to iterate over pinned objects, we just scan a page at a time
435 * linearly according to the size of objects in the page: the next pointer used to link
436 * the items in the freelist uses the same word as the vtable. Since we keep freelists
437 * for each pinned chunk, if the word points outside the pinned chunk it means
439 * We could avoid this expensive scanning in creative ways. We could have a policy
440 * of putting in the pinned space only objects we know about that have no struct fields
441 * with references and we can easily use a even expensive write barrier for them,
442 * since pointer writes on such objects should be rare.
443 * The best compromise is to just alloc interned strings and System.MonoType in them.
444 * It would be nice to allocate MonoThread in it, too: must check that we properly
445 * use write barriers so we don't have to do any expensive scanning of the whole pinned
446 * chunk list during minor collections. We can avoid it now because we alloc in it only
447 * reference-free objects.
449 #define PINNED_FIRST_SLOT_SIZE (sizeof (gpointer) * 4)
450 #define MAX_FREELIST_SIZE 8192
451 typedef struct _PinnedChunk PinnedChunk;
452 struct _PinnedChunk {
455 int *page_sizes; /* a 0 means the page is still unused */
458 void *data [1]; /* page sizes and free lists are stored here */
461 /* The method used to clear the nursery */
462 /* Clearing at nursery collections is the safest, but has bad interactions with caches.
463 * Clearing at TLAB creation is much faster, but more complex and it might expose hard
468 CLEAR_AT_TLAB_CREATION
469 } NurseryClearPolicy;
471 static NurseryClearPolicy nursery_clear_policy = CLEAR_AT_TLAB_CREATION;
474 * If this is set, the nursery is aligned to an address aligned to its size, ie.
475 * a 1MB nursery will be aligned to an address divisible by 1MB. This allows us to
476 * speed up ptr_in_nursery () checks which are very frequent. This requires the
477 * nursery size to be a compile time constant.
479 #define ALIGN_NURSERY 1
482 * The young generation is divided into fragments. This is because
483 * we can hand one fragments to a thread for lock-less fast alloc and
484 * because the young generation ends up fragmented anyway by pinned objects.
485 * Once a collection is done, a list of fragments is created. When doing
486 * thread local alloc we use smallish nurseries so we allow new threads to
487 * allocate memory from gen0 without triggering a collection. Threads that
488 * are found to allocate lots of memory are given bigger fragments. This
489 * should make the finalizer thread use little nursery memory after a while.
490 * We should start assigning threads very small fragments: if there are many
491 * threads the nursery will be full of reserved space that the threads may not
492 * use at all, slowing down allocation speed.
493 * Thread local allocation is done from areas of memory Hotspot calls Thread Local
494 * Allocation Buffers (TLABs).
496 typedef struct _Fragment Fragment;
500 char *fragment_start;
501 char *fragment_limit; /* the current soft limit for allocation */
505 /* the runtime can register areas of memory as roots: we keep two lists of roots,
506 * a pinned root set for conservatively scanned roots and a normal one for
507 * precisely scanned roots (currently implemented as a single list).
509 typedef struct _RootRecord RootRecord;
518 * We're never actually using the first element. It's always set to
519 * NULL to simplify the elimination of consecutive duplicate
522 #define STORE_REMSET_BUFFER_SIZE 1024
524 typedef struct _GenericStoreRememberedSet GenericStoreRememberedSet;
525 struct _GenericStoreRememberedSet {
526 GenericStoreRememberedSet *next;
527 /* We need one entry less because the first entry of store
528 remset buffers is always a dummy and we don't copy it. */
529 gpointer data [STORE_REMSET_BUFFER_SIZE - 1];
532 /* we have 4 possible values in the low 2 bits */
534 REMSET_LOCATION, /* just a pointer to the exact location */
535 REMSET_RANGE, /* range of pointer fields */
536 REMSET_OBJECT, /* mark all the object for scanning */
537 REMSET_VTYPE, /* a valuetype array described by a gc descriptor and a count */
538 REMSET_TYPE_MASK = 0x3
541 #ifdef HAVE_KW_THREAD
542 static __thread RememberedSet *remembered_set MONO_TLS_FAST;
544 static pthread_key_t remembered_set_key;
545 static RememberedSet *global_remset;
546 static RememberedSet *freed_thread_remsets;
547 static GenericStoreRememberedSet *generic_store_remsets = NULL;
549 /*A two slots cache for recently inserted remsets */
550 static gpointer global_remset_cache [2];
552 /* FIXME: later choose a size that takes into account the RememberedSet struct
553 * and doesn't waste any alloc paddin space.
555 #define DEFAULT_REMSET_SIZE 1024
556 static RememberedSet* alloc_remset (int size, gpointer id);
558 /* Structure that corresponds to a MonoVTable: desc is a mword so requires
559 * no cast from a pointer to an integer
566 /* these bits are set in the object vtable: we could merge them since an object can be
567 * either pinned or forwarded but not both.
568 * We store them in the vtable slot because the bits are used in the sync block for
569 * other purposes: if we merge them and alloc the sync blocks aligned to 8 bytes, we can change
570 * this and use bit 3 in the syncblock (with the lower two bits both set for forwarded, that
571 * would be an invalid combination for the monitor and hash code).
572 * The values are already shifted.
573 * The forwarding address is stored in the sync block.
575 #define FORWARDED_BIT 1
577 #define VTABLE_BITS_MASK 0x3
579 /* returns NULL if not forwarded, or the forwarded address */
580 #define object_is_forwarded(obj) (((mword*)(obj))[0] & FORWARDED_BIT ? (void*)(((mword*)(obj))[0] & ~VTABLE_BITS_MASK) : NULL)
581 /* set the forwarded address fw_addr for object obj */
582 #define forward_object(obj,fw_addr) do { \
583 ((mword*)(obj))[0] = (mword)(fw_addr) | FORWARDED_BIT; \
586 #define object_is_pinned(obj) (((mword*)(obj))[0] & PINNED_BIT)
587 #define pin_object(obj) do { \
588 ((mword*)(obj))[0] |= PINNED_BIT; \
590 #define unpin_object(obj) do { \
591 ((mword*)(obj))[0] &= ~PINNED_BIT; \
595 #define ptr_in_nursery(ptr) (((mword)(ptr) & ~((1 << DEFAULT_NURSERY_BITS) - 1)) == (mword)nursery_start)
597 #define ptr_in_nursery(ptr) ((char*)(ptr) >= nursery_start && (char*)(ptr) < nursery_real_end)
601 * Since we set bits in the vtable, use the macro to load it from the pointer to
602 * an object that is potentially pinned.
604 #define LOAD_VTABLE(addr) ((*(mword*)(addr)) & ~VTABLE_BITS_MASK)
607 safe_name (void* obj)
609 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (obj);
610 return vt->klass->name;
614 safe_object_get_size (MonoObject* o)
616 MonoClass *klass = ((MonoVTable*)LOAD_VTABLE (o))->klass;
617 if (klass == mono_defaults.string_class) {
618 return sizeof (MonoString) + 2 * mono_string_length_fast ((MonoString*) o) + 2;
619 } else if (klass->rank) {
620 MonoArray *array = (MonoArray*)o;
621 size_t size = sizeof (MonoArray) + klass->sizes.element_size * mono_array_length_fast (array);
622 if (G_UNLIKELY (array->bounds)) {
623 size += sizeof (mono_array_size_t) - 1;
624 size &= ~(sizeof (mono_array_size_t) - 1);
625 size += sizeof (MonoArrayBounds) * klass->rank;
629 /* from a created object: the class must be inited already */
630 return klass->instance_size;
635 * ######################################################################
636 * ######## Global data.
637 * ######################################################################
639 static LOCK_DECLARE (gc_mutex);
640 static int gc_disabled = 0;
641 static int num_minor_gcs = 0;
642 static int num_major_gcs = 0;
646 /* good sizes are 512KB-1MB: larger ones increase a lot memzeroing time */
647 #define DEFAULT_NURSERY_SIZE (default_nursery_size)
648 static int default_nursery_size = (1 << 22);
650 /* The number of trailing 0 bits in DEFAULT_NURSERY_SIZE */
651 #define DEFAULT_NURSERY_BITS (default_nursery_bits)
652 static int default_nursery_bits = 22;
657 #define DEFAULT_NURSERY_SIZE (4*1024*1024)
659 #define DEFAULT_NURSERY_BITS 22
664 #define MIN_MINOR_COLLECTION_ALLOWANCE (DEFAULT_NURSERY_SIZE * 4)
665 /* to quickly find the head of an object pinned by a conservative address
666 * we keep track of the objects allocated for each SCAN_START_SIZE memory
667 * chunk in the nursery or other memory sections. Larger values have less
668 * memory overhead and bigger runtime cost. 4-8 KB are reasonable values.
670 #define SCAN_START_SIZE (4096*2)
671 /* the minimum size of a fragment that we consider useful for allocation */
672 #define FRAGMENT_MIN_SIZE (512)
673 /* This is a fixed value used for pinned chunks, not the system pagesize */
674 #define FREELIST_PAGESIZE (16*1024)
676 static mword pagesize = 4096;
677 static mword nursery_size;
678 static int degraded_mode = 0;
680 static mword total_alloc = 0;
681 /* use this to tune when to do a major/minor collection */
682 static mword memory_pressure = 0;
683 static int minor_collection_allowance;
684 static int minor_collection_sections_alloced = 0;
686 static GCMemSection *nursery_section = NULL;
687 static mword lowest_heap_address = ~(mword)0;
688 static mword highest_heap_address = 0;
690 static LOCK_DECLARE (interruption_mutex);
692 typedef struct _FinalizeEntry FinalizeEntry;
693 struct _FinalizeEntry {
698 typedef struct _FinalizeEntryHashTable FinalizeEntryHashTable;
699 struct _FinalizeEntryHashTable {
700 FinalizeEntry **table;
705 typedef struct _DisappearingLink DisappearingLink;
706 struct _DisappearingLink {
707 DisappearingLink *next;
711 typedef struct _DisappearingLinkHashTable DisappearingLinkHashTable;
712 struct _DisappearingLinkHashTable {
713 DisappearingLink **table;
718 typedef struct _EphemeronLinkNode EphemeronLinkNode;
720 struct _EphemeronLinkNode {
721 EphemeronLinkNode *next;
730 #define LARGE_INTERNAL_MEM_HEADER_MAGIC 0x7d289f3a
732 typedef struct _LargeInternalMemHeader LargeInternalMemHeader;
733 struct _LargeInternalMemHeader {
745 int current_collection_generation = -1;
748 * The link pointer is hidden by negating each bit. We use the lowest
749 * bit of the link (before negation) to store whether it needs
750 * resurrection tracking.
752 #define HIDE_POINTER(p,t) ((gpointer)(~((gulong)(p)|((t)?1:0))))
753 #define REVEAL_POINTER(p) ((gpointer)((~(gulong)(p))&~3L))
755 #define DISLINK_OBJECT(d) (REVEAL_POINTER (*(d)->link))
756 #define DISLINK_TRACK(d) ((~(gulong)(*(d)->link)) & 1)
759 * The finalizable hash has the object as the key, the
760 * disappearing_link hash, has the link address as key.
762 static FinalizeEntryHashTable minor_finalizable_hash;
763 static FinalizeEntryHashTable major_finalizable_hash;
764 /* objects that are ready to be finalized */
765 static FinalizeEntry *fin_ready_list = NULL;
766 static FinalizeEntry *critical_fin_list = NULL;
768 static DisappearingLinkHashTable minor_disappearing_link_hash;
769 static DisappearingLinkHashTable major_disappearing_link_hash;
771 static EphemeronLinkNode *ephemeron_list;
773 static int num_ready_finalizers = 0;
774 static int no_finalize = 0;
776 /* keep each size a multiple of ALLOC_ALIGN */
777 /* on 64 bit systems 8 is likely completely unused. */
778 static const int freelist_sizes [] = {
779 8, 16, 24, 32, 40, 48, 64, 80,
780 96, 128, 160, 192, 224, 256, 320, 384,
781 448, 512, 584, 680, 816, 1024, 1360, 2048,
782 2336, 2728, 3272, 4096, 5456, 8192 };
783 #define FREELIST_NUM_SLOTS (sizeof (freelist_sizes) / sizeof (freelist_sizes [0]))
785 /* This is also the MAJOR_SECTION_SIZE for the copying major
787 #define PINNED_CHUNK_SIZE (128 * 1024)
789 /* internal_chunk_list is used for allocating structures needed by the GC */
790 static PinnedChunk *internal_chunk_list = NULL;
792 static int slot_for_size (size_t size);
795 ROOT_TYPE_NORMAL = 0, /* "normal" roots */
796 ROOT_TYPE_PINNED = 1, /* roots without a GC descriptor */
797 ROOT_TYPE_WBARRIER = 2, /* roots with a write barrier */
801 /* registered roots: the key to the hash is the root start address */
803 * Different kinds of roots are kept separate to speed up pin_from_roots () for example.
805 static RootRecord **roots_hash [ROOT_TYPE_NUM] = { NULL, NULL };
806 static int roots_hash_size [ROOT_TYPE_NUM] = { 0, 0, 0 };
807 static mword roots_size = 0; /* amount of memory in the root set */
808 static int num_roots_entries [ROOT_TYPE_NUM] = { 0, 0, 0 };
811 * The current allocation cursors
812 * We allocate objects in the nursery.
813 * The nursery is the area between nursery_start and nursery_real_end.
814 * Allocation is done from a Thread Local Allocation Buffer (TLAB). TLABs are allocated
815 * from nursery fragments.
816 * tlab_next is the pointer to the space inside the TLAB where the next object will
818 * tlab_temp_end is the pointer to the end of the temporary space reserved for
819 * the allocation: it allows us to set the scan starts at reasonable intervals.
820 * tlab_real_end points to the end of the TLAB.
821 * nursery_frag_real_end points to the end of the currently used nursery fragment.
822 * nursery_first_pinned_start points to the start of the first pinned object in the nursery
823 * nursery_last_pinned_end points to the end of the last pinned object in the nursery
824 * At the next allocation, the area of the nursery where objects can be present is
825 * between MIN(nursery_first_pinned_start, first_fragment_start) and
826 * MAX(nursery_last_pinned_end, nursery_frag_real_end)
828 static char *nursery_start = NULL;
830 #ifdef HAVE_KW_THREAD
831 #define TLAB_ACCESS_INIT
832 #define TLAB_START tlab_start
833 #define TLAB_NEXT tlab_next
834 #define TLAB_TEMP_END tlab_temp_end
835 #define TLAB_REAL_END tlab_real_end
836 #define REMEMBERED_SET remembered_set
837 #define STORE_REMSET_BUFFER store_remset_buffer
838 #define STORE_REMSET_BUFFER_INDEX store_remset_buffer_index
839 #define IN_CRITICAL_REGION thread_info->in_critical_region
841 static pthread_key_t thread_info_key;
842 #define TLAB_ACCESS_INIT SgenThreadInfo *__thread_info__ = pthread_getspecific (thread_info_key)
843 #define TLAB_START (__thread_info__->tlab_start)
844 #define TLAB_NEXT (__thread_info__->tlab_next)
845 #define TLAB_TEMP_END (__thread_info__->tlab_temp_end)
846 #define TLAB_REAL_END (__thread_info__->tlab_real_end)
847 #define REMEMBERED_SET (__thread_info__->remset)
848 #define STORE_REMSET_BUFFER (__thread_info__->store_remset_buffer)
849 #define STORE_REMSET_BUFFER_INDEX (__thread_info__->store_remset_buffer_index)
850 #define IN_CRITICAL_REGION (__thread_info__->in_critical_region)
853 /* we use the memory barrier only to prevent compiler reordering (a memory constraint may be enough) */
854 #define ENTER_CRITICAL_REGION do {IN_CRITICAL_REGION = 1;mono_memory_barrier ();} while (0)
855 #define EXIT_CRITICAL_REGION do {IN_CRITICAL_REGION = 0;mono_memory_barrier ();} while (0)
858 * FIXME: What is faster, a TLS variable pointing to a structure, or separate TLS
859 * variables for next+temp_end ?
861 #ifdef HAVE_KW_THREAD
862 static __thread SgenThreadInfo *thread_info;
863 static __thread char *tlab_start;
864 static __thread char *tlab_next;
865 static __thread char *tlab_temp_end;
866 static __thread char *tlab_real_end;
867 static __thread gpointer *store_remset_buffer;
868 static __thread long store_remset_buffer_index;
869 /* Used by the managed allocator/wbarrier */
870 static __thread char **tlab_next_addr;
871 static __thread char *stack_end;
872 static __thread long *store_remset_buffer_index_addr;
874 static char *nursery_next = NULL;
875 static char *nursery_frag_real_end = NULL;
876 static char *nursery_real_end = NULL;
877 static char *nursery_last_pinned_end = NULL;
879 /* The size of a TLAB */
880 /* The bigger the value, the less often we have to go to the slow path to allocate a new
881 * one, but the more space is wasted by threads not allocating much memory.
883 * FIXME: Make this self-tuning for each thread.
885 static guint32 tlab_size = (1024 * 4);
887 /*How much space is tolerable to be wasted from the current fragment when allocating a new TLAB*/
888 #define MAX_NURSERY_TLAB_WASTE 512
890 /* fragments that are free and ready to be used for allocation */
891 static Fragment *nursery_fragments = NULL;
892 /* freeelist of fragment structures */
893 static Fragment *fragment_freelist = NULL;
896 * Objects bigger then this go into the large object space. This size
897 * has a few constraints. It must fit into the major heap, which in
898 * the case of the copying collector means that it must fit into a
899 * pinned chunk. It must also play well with the GC descriptors, some
900 * of which (DESC_TYPE_RUN_LENGTH, DESC_TYPE_SMALL_BITMAP) encode the
903 #define MAX_SMALL_OBJ_SIZE 8000
905 /* Functions supplied by the runtime to be called by the GC */
906 static MonoGCCallbacks gc_callbacks;
908 #define ALLOC_ALIGN 8
909 #define ALLOC_ALIGN_BITS 3
911 #define ALIGN_UP(s) (((s)+(ALLOC_ALIGN-1)) & ~(ALLOC_ALIGN-1))
913 #define MOVED_OBJECTS_NUM 64
914 static void *moved_objects [MOVED_OBJECTS_NUM];
915 static int moved_objects_idx = 0;
918 * ######################################################################
919 * ######## Macros and function declarations.
920 * ######################################################################
923 #define UPDATE_HEAP_BOUNDARIES(low,high) do { \
924 if ((mword)(low) < lowest_heap_address) \
925 lowest_heap_address = (mword)(low); \
926 if ((mword)(high) > highest_heap_address) \
927 highest_heap_address = (mword)(high); \
929 #define ADDR_IN_HEAP_BOUNDARIES(addr) ((p) >= lowest_heap_address && (p) < highest_heap_address)
932 align_pointer (void *ptr)
934 mword p = (mword)ptr;
935 p += sizeof (gpointer) - 1;
936 p &= ~ (sizeof (gpointer) - 1);
940 typedef struct _GrayQueue GrayQueue;
942 typedef void (*CopyOrMarkObjectFunc) (void**, GrayQueue*);
943 typedef char* (*ScanObjectFunc) (char*, GrayQueue*);
945 /* forward declarations */
946 static void* get_internal_mem (size_t size, int type);
947 static void free_internal_mem (void *addr, int type);
948 static void* get_os_memory (size_t size, int activate);
949 static void* get_os_memory_aligned (mword size, mword alignment, gboolean activate);
950 static void free_os_memory (void *addr, size_t size);
951 static G_GNUC_UNUSED void report_internal_mem_usage (void);
953 static int stop_world (void);
954 static int restart_world (void);
955 static void add_to_global_remset (gpointer ptr);
956 static void scan_thread_data (void *start_nursery, void *end_nursery, gboolean precise);
957 static void scan_from_remsets (void *start_nursery, void *end_nursery, GrayQueue *queue);
958 static void scan_from_registered_roots (CopyOrMarkObjectFunc copy_func, char *addr_start, char *addr_end, int root_type, GrayQueue *queue);
959 static void scan_finalizer_entries (CopyOrMarkObjectFunc copy_func, FinalizeEntry *list, GrayQueue *queue);
960 static void find_pinning_ref_from_thread (char *obj, size_t size);
961 static void update_current_thread_stack (void *start);
962 static void finalize_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, int generation, GrayQueue *queue);
963 static void add_or_remove_disappearing_link (MonoObject *obj, void **link, gboolean track, int generation);
964 static void null_link_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, int generation, GrayQueue *queue);
965 static void null_links_for_domain (MonoDomain *domain, int generation);
966 static gboolean search_fragment_for_size (size_t size);
967 static int search_fragment_for_size_range (size_t desired_size, size_t minimum_size);
968 static void build_nursery_fragments (int start_pin, int end_pin);
969 static void clear_nursery_fragments (char *next);
970 static void pin_from_roots (void *start_nursery, void *end_nursery);
971 static int pin_objects_from_addresses (GCMemSection *section, void **start, void **end, void *start_nursery, void *end_nursery, GrayQueue *queue);
972 static void pin_objects_in_section (GCMemSection *section, GrayQueue *queue);
973 static void optimize_pin_queue (int start_slot);
974 static void clear_remsets (void);
975 static void clear_tlabs (void);
976 typedef void (*IterateObjectCallbackFunc) (char*, size_t, void*);
977 static void scan_area_with_callback (char *start, char *end, IterateObjectCallbackFunc callback, void *data);
978 static void scan_object (char *start, GrayQueue *queue);
979 static void major_scan_object (char *start, GrayQueue *queue);
980 static void* copy_object_no_checks (void *obj, GrayQueue *queue);
981 static void copy_object (void **obj_slot, GrayQueue *queue);
982 static void* get_chunk_freelist (PinnedChunk *chunk, int slot);
983 static PinnedChunk* alloc_pinned_chunk (void);
984 static void sort_addresses (void **array, int size);
985 static void drain_gray_stack (GrayQueue *queue);
986 static void finish_gray_stack (char *start_addr, char *end_addr, int generation, GrayQueue *queue);
987 static gboolean need_major_collection (void);
988 static void major_collection (const char *reason);
990 static void mono_gc_register_disappearing_link (MonoObject *obj, void **link, gboolean track);
992 void describe_ptr (char *ptr);
993 void check_object (char *start);
995 static void check_consistency (void);
996 static void check_major_refs (void);
997 static void check_section_scan_starts (GCMemSection *section);
998 static void check_scan_starts (void);
999 static void check_for_xdomain_refs (void);
1000 static void dump_occupied (char *start, char *end, char *section_start);
1001 static void dump_section (GCMemSection *section, const char *type);
1002 static void dump_heap (const char *type, int num, const char *reason);
1003 static void report_pinned_chunk (PinnedChunk *chunk, int seq);
1005 void mono_gc_scan_for_specific_ref (MonoObject *key);
1007 static void init_stats (void);
1009 static int mark_ephemerons_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, GrayQueue *queue);
1010 static void clear_unreachable_ephemerons (CopyOrMarkObjectFunc copy_func, char *start, char *end, GrayQueue *queue);
1011 static void null_ephemerons_for_domain (MonoDomain *domain);
1013 //#define BINARY_PROTOCOL
1014 #include "sgen-protocol.c"
1015 #include "sgen-pinning.c"
1016 #include "sgen-pinning-stats.c"
1017 #include "sgen-gray.c"
1018 #include "sgen-los.c"
1021 * ######################################################################
1022 * ######## GC descriptors
1023 * ######################################################################
1024 * Used to quickly get the info the GC needs about an object: size and
1025 * where the references are held.
1027 /* objects are aligned to 8 bytes boundaries
1028 * A descriptor is a pointer in MonoVTable, so 32 or 64 bits of size.
1029 * The low 3 bits define the type of the descriptor. The other bits
1030 * depend on the type.
1031 * As a general rule the 13 remaining low bits define the size, either
1032 * of the whole object or of the elements in the arrays. While for objects
1033 * the size is already in bytes, for arrays we need to shift, because
1034 * array elements might be smaller than 8 bytes. In case of arrays, we
1035 * use two bits to describe what the additional high bits represents,
1036 * so the default behaviour can handle element sizes less than 2048 bytes.
1037 * The high 16 bits, if 0 it means the object is pointer-free.
1038 * This design should make it easy and fast to skip over ptr-free data.
1039 * The first 4 types should cover >95% of the objects.
1040 * Note that since the size of objects is limited to 64K, larger objects
1041 * will be allocated in the large object heap.
1042 * If we want 4-bytes alignment, we need to put vector and small bitmap
1047 * We don't use 0 so that 0 isn't a valid GC descriptor. No
1048 * deep reason for this other than to be able to identify a
1049 * non-inited descriptor for debugging.
1051 * If an object contains no references, its GC descriptor is
1052 * always DESC_TYPE_RUN_LENGTH, without a size, no exceptions.
1053 * This is so that we can quickly check for that in
1054 * copy_object_no_checks(), without having to fetch the
1057 DESC_TYPE_RUN_LENGTH = 1, /* 15 bits aligned byte size | 1-3 (offset, numptr) bytes tuples */
1058 DESC_TYPE_SMALL_BITMAP, /* 15 bits aligned byte size | 16-48 bit bitmap */
1059 DESC_TYPE_COMPLEX, /* index for bitmap into complex_descriptors */
1060 DESC_TYPE_VECTOR, /* 10 bits element size | 1 bit array | 2 bits desc | element desc */
1061 DESC_TYPE_ARRAY, /* 10 bits element size | 1 bit array | 2 bits desc | element desc */
1062 DESC_TYPE_LARGE_BITMAP, /* | 29-61 bitmap bits */
1063 DESC_TYPE_COMPLEX_ARR, /* index for bitmap into complex_descriptors */
1064 /* subtypes for arrays and vectors */
1065 DESC_TYPE_V_PTRFREE = 0,/* there are no refs: keep first so it has a zero value */
1066 DESC_TYPE_V_REFS, /* all the array elements are refs */
1067 DESC_TYPE_V_RUN_LEN, /* elements are run-length encoded as DESC_TYPE_RUN_LENGTH */
1068 DESC_TYPE_V_BITMAP /* elements are as the bitmap in DESC_TYPE_SMALL_BITMAP */
1071 #define OBJECT_HEADER_WORDS (sizeof(MonoObject)/sizeof(gpointer))
1072 #define LOW_TYPE_BITS 3
1073 #define SMALL_BITMAP_SHIFT 16
1074 #define SMALL_BITMAP_SIZE (GC_BITS_PER_WORD - SMALL_BITMAP_SHIFT)
1075 #define VECTOR_INFO_SHIFT 14
1076 #define VECTOR_ELSIZE_SHIFT 3
1077 #define LARGE_BITMAP_SIZE (GC_BITS_PER_WORD - LOW_TYPE_BITS)
1078 #define MAX_ELEMENT_SIZE 0x3ff
1079 #define VECTOR_SUBTYPE_PTRFREE (DESC_TYPE_V_PTRFREE << VECTOR_INFO_SHIFT)
1080 #define VECTOR_SUBTYPE_REFS (DESC_TYPE_V_REFS << VECTOR_INFO_SHIFT)
1081 #define VECTOR_SUBTYPE_RUN_LEN (DESC_TYPE_V_RUN_LEN << VECTOR_INFO_SHIFT)
1082 #define VECTOR_SUBTYPE_BITMAP (DESC_TYPE_V_BITMAP << VECTOR_INFO_SHIFT)
1085 /* Root bitmap descriptors are simpler: the lower three bits describe the type
1086 * and we either have 30/62 bitmap bits or nibble-based run-length,
1087 * or a complex descriptor, or a user defined marker function.
1090 ROOT_DESC_CONSERVATIVE, /* 0, so matches NULL value */
1095 ROOT_DESC_TYPE_MASK = 0x7,
1096 ROOT_DESC_TYPE_SHIFT = 3,
1099 #define MAKE_ROOT_DESC(type,val) ((type) | ((val) << ROOT_DESC_TYPE_SHIFT))
1101 #define MAX_USER_DESCRIPTORS 16
1103 static gsize* complex_descriptors = NULL;
1104 static int complex_descriptors_size = 0;
1105 static int complex_descriptors_next = 0;
1106 static MonoGCRootMarkFunc user_descriptors [MAX_USER_DESCRIPTORS];
1107 static int user_descriptors_next = 0;
1110 alloc_complex_descriptor (gsize *bitmap, int numbits)
1114 numbits = ALIGN_TO (numbits, GC_BITS_PER_WORD);
1115 nwords = numbits / GC_BITS_PER_WORD + 1;
1118 res = complex_descriptors_next;
1119 /* linear search, so we don't have duplicates with domain load/unload
1120 * this should not be performance critical or we'd have bigger issues
1121 * (the number and size of complex descriptors should be small).
1123 for (i = 0; i < complex_descriptors_next; ) {
1124 if (complex_descriptors [i] == nwords) {
1125 int j, found = TRUE;
1126 for (j = 0; j < nwords - 1; ++j) {
1127 if (complex_descriptors [i + 1 + j] != bitmap [j]) {
1137 i += complex_descriptors [i];
1139 if (complex_descriptors_next + nwords > complex_descriptors_size) {
1140 int new_size = complex_descriptors_size * 2 + nwords;
1141 complex_descriptors = g_realloc (complex_descriptors, new_size * sizeof (gsize));
1142 complex_descriptors_size = new_size;
1144 DEBUG (6, fprintf (gc_debug_file, "Complex descriptor %d, size: %d (total desc memory: %d)\n", res, nwords, complex_descriptors_size));
1145 complex_descriptors_next += nwords;
1146 complex_descriptors [res] = nwords;
1147 for (i = 0; i < nwords - 1; ++i) {
1148 complex_descriptors [res + 1 + i] = bitmap [i];
1149 DEBUG (6, fprintf (gc_debug_file, "\tvalue: %p\n", (void*)complex_descriptors [res + 1 + i]));
1156 * Descriptor builders.
1159 mono_gc_make_descr_for_string (gsize *bitmap, int numbits)
1161 return (void*) DESC_TYPE_RUN_LENGTH;
1165 mono_gc_make_descr_for_object (gsize *bitmap, int numbits, size_t obj_size)
1167 int first_set = -1, num_set = 0, last_set = -1, i;
1169 size_t stored_size = obj_size;
1170 for (i = 0; i < numbits; ++i) {
1171 if (bitmap [i / GC_BITS_PER_WORD] & ((gsize)1 << (i % GC_BITS_PER_WORD))) {
1179 * We don't encode the size of types that don't contain
1180 * references because they might not be aligned, i.e. the
1181 * bottom two bits might be set, which would clash with the
1182 * bits we need to encode the descriptor type. Since we don't
1183 * use the encoded size to skip objects, other than for
1184 * processing remsets, in which case only the positions of
1185 * references are relevant, this is not a problem.
1188 return (void*)DESC_TYPE_RUN_LENGTH;
1189 g_assert (!(stored_size & 0x3));
1190 if (stored_size <= MAX_SMALL_OBJ_SIZE) {
1191 /* check run-length encoding first: one byte offset, one byte number of pointers
1192 * on 64 bit archs, we can have 3 runs, just one on 32.
1193 * It may be better to use nibbles.
1195 if (first_set < 0) {
1196 desc = DESC_TYPE_RUN_LENGTH | (stored_size << 1);
1197 DEBUG (6, fprintf (gc_debug_file, "Ptrfree descriptor %p, size: %zd\n", (void*)desc, stored_size));
1198 return (void*) desc;
1199 } else if (first_set < 256 && num_set < 256 && (first_set + num_set == last_set + 1)) {
1200 desc = DESC_TYPE_RUN_LENGTH | (stored_size << 1) | (first_set << 16) | (num_set << 24);
1201 DEBUG (6, fprintf (gc_debug_file, "Runlen descriptor %p, size: %zd, first set: %d, num set: %d\n", (void*)desc, stored_size, first_set, num_set));
1202 return (void*) desc;
1204 /* we know the 2-word header is ptr-free */
1205 if (last_set < SMALL_BITMAP_SIZE + OBJECT_HEADER_WORDS) {
1206 desc = DESC_TYPE_SMALL_BITMAP | (stored_size << 1) | ((*bitmap >> OBJECT_HEADER_WORDS) << SMALL_BITMAP_SHIFT);
1207 DEBUG (6, fprintf (gc_debug_file, "Smallbitmap descriptor %p, size: %zd, last set: %d\n", (void*)desc, stored_size, last_set));
1208 return (void*) desc;
1211 /* we know the 2-word header is ptr-free */
1212 if (last_set < LARGE_BITMAP_SIZE + OBJECT_HEADER_WORDS) {
1213 desc = DESC_TYPE_LARGE_BITMAP | ((*bitmap >> OBJECT_HEADER_WORDS) << LOW_TYPE_BITS);
1214 DEBUG (6, fprintf (gc_debug_file, "Largebitmap descriptor %p, size: %zd, last set: %d\n", (void*)desc, stored_size, last_set));
1215 return (void*) desc;
1217 /* it's a complex object ... */
1218 desc = DESC_TYPE_COMPLEX | (alloc_complex_descriptor (bitmap, last_set + 1) << LOW_TYPE_BITS);
1219 return (void*) desc;
1222 /* If the array holds references, numbits == 1 and the first bit is set in elem_bitmap */
1224 mono_gc_make_descr_for_array (int vector, gsize *elem_bitmap, int numbits, size_t elem_size)
1226 int first_set = -1, num_set = 0, last_set = -1, i;
1227 mword desc = vector? DESC_TYPE_VECTOR: DESC_TYPE_ARRAY;
1228 for (i = 0; i < numbits; ++i) {
1229 if (elem_bitmap [i / GC_BITS_PER_WORD] & ((gsize)1 << (i % GC_BITS_PER_WORD))) {
1236 /* See comment at the definition of DESC_TYPE_RUN_LENGTH. */
1238 return (void*)DESC_TYPE_RUN_LENGTH;
1239 if (elem_size <= MAX_ELEMENT_SIZE) {
1240 desc |= elem_size << VECTOR_ELSIZE_SHIFT;
1242 return (void*)(desc | VECTOR_SUBTYPE_PTRFREE);
1244 /* Note: we also handle structs with just ref fields */
1245 if (num_set * sizeof (gpointer) == elem_size) {
1246 return (void*)(desc | VECTOR_SUBTYPE_REFS | ((gssize)(-1) << 16));
1248 /* FIXME: try run-len first */
1249 /* Note: we can't skip the object header here, because it's not present */
1250 if (last_set <= SMALL_BITMAP_SIZE) {
1251 return (void*)(desc | VECTOR_SUBTYPE_BITMAP | (*elem_bitmap << 16));
1254 /* it's am array of complex structs ... */
1255 desc = DESC_TYPE_COMPLEX_ARR;
1256 desc |= alloc_complex_descriptor (elem_bitmap, last_set + 1) << LOW_TYPE_BITS;
1257 return (void*) desc;
1260 /* Return the bitmap encoded by a descriptor */
1262 mono_gc_get_bitmap_for_descr (void *descr, int *numbits)
1264 mword d = (mword)descr;
1268 case DESC_TYPE_RUN_LENGTH: {
1269 int first_set = (d >> 16) & 0xff;
1270 int num_set = (d >> 24) & 0xff;
1273 bitmap = g_new0 (gsize, (first_set + num_set + 7) / 8);
1275 for (i = first_set; i < first_set + num_set; ++i)
1276 bitmap [i / GC_BITS_PER_WORD] |= ((gsize)1 << (i % GC_BITS_PER_WORD));
1278 *numbits = first_set + num_set;
1282 case DESC_TYPE_SMALL_BITMAP:
1283 bitmap = g_new0 (gsize, 1);
1285 bitmap [0] = (d >> SMALL_BITMAP_SHIFT) << OBJECT_HEADER_WORDS;
1287 *numbits = GC_BITS_PER_WORD;
1291 g_assert_not_reached ();
1295 /* helper macros to scan and traverse objects, macros because we resue them in many functions */
1296 #define OBJ_RUN_LEN_SIZE(size,desc,obj) do { \
1297 (size) = ((desc) & 0xfff8) >> 1; \
1300 #define OBJ_BITMAP_SIZE(size,desc,obj) do { \
1301 (size) = ((desc) & 0xfff8) >> 1; \
1304 //#define PREFETCH(addr) __asm__ __volatile__ (" prefetchnta %0": : "m"(*(char *)(addr)))
1305 #define PREFETCH(addr)
1307 /* code using these macros must define a HANDLE_PTR(ptr) macro that does the work */
1308 #define OBJ_RUN_LEN_FOREACH_PTR(desc,obj) do { \
1309 if ((desc) & 0xffff0000) { \
1310 /* there are pointers */ \
1311 void **_objptr_end; \
1312 void **_objptr = (void**)(obj); \
1313 _objptr += ((desc) >> 16) & 0xff; \
1314 _objptr_end = _objptr + (((desc) >> 24) & 0xff); \
1315 while (_objptr < _objptr_end) { \
1316 HANDLE_PTR (_objptr, (obj)); \
1322 /* a bitmap desc means that there are pointer references or we'd have
1323 * choosen run-length, instead: add an assert to check.
1325 #define OBJ_BITMAP_FOREACH_PTR(desc,obj) do { \
1326 /* there are pointers */ \
1327 void **_objptr = (void**)(obj); \
1328 gsize _bmap = (desc) >> 16; \
1329 _objptr += OBJECT_HEADER_WORDS; \
1331 if ((_bmap & 1)) { \
1332 HANDLE_PTR (_objptr, (obj)); \
1339 #define OBJ_LARGE_BITMAP_FOREACH_PTR(vt,obj) do { \
1340 /* there are pointers */ \
1341 void **_objptr = (void**)(obj); \
1342 gsize _bmap = (vt)->desc >> LOW_TYPE_BITS; \
1343 _objptr += OBJECT_HEADER_WORDS; \
1345 if ((_bmap & 1)) { \
1346 HANDLE_PTR (_objptr, (obj)); \
1353 #define OBJ_COMPLEX_FOREACH_PTR(vt,obj) do { \
1354 /* there are pointers */ \
1355 void **_objptr = (void**)(obj); \
1356 gsize *bitmap_data = complex_descriptors + ((vt)->desc >> LOW_TYPE_BITS); \
1357 int bwords = (*bitmap_data) - 1; \
1358 void **start_run = _objptr; \
1361 MonoObject *myobj = (MonoObject*)obj; \
1362 g_print ("found %d at %p (0x%zx): %s.%s\n", bwords, (obj), (vt)->desc, myobj->vtable->klass->name_space, myobj->vtable->klass->name); \
1364 while (bwords-- > 0) { \
1365 gsize _bmap = *bitmap_data++; \
1366 _objptr = start_run; \
1367 /*g_print ("bitmap: 0x%x/%d at %p\n", _bmap, bwords, _objptr);*/ \
1369 if ((_bmap & 1)) { \
1370 HANDLE_PTR (_objptr, (obj)); \
1375 start_run += GC_BITS_PER_WORD; \
1379 /* this one is untested */
1380 #define OBJ_COMPLEX_ARR_FOREACH_PTR(vt,obj) do { \
1381 /* there are pointers */ \
1382 gsize *mbitmap_data = complex_descriptors + ((vt)->desc >> LOW_TYPE_BITS); \
1383 int mbwords = (*mbitmap_data++) - 1; \
1384 int el_size = mono_array_element_size (vt->klass); \
1385 char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
1386 char *e_end = e_start + el_size * mono_array_length_fast ((MonoArray*)(obj)); \
1388 g_print ("found %d at %p (0x%zx): %s.%s\n", mbwords, (obj), (vt)->desc, vt->klass->name_space, vt->klass->name); \
1389 while (e_start < e_end) { \
1390 void **_objptr = (void**)e_start; \
1391 gsize *bitmap_data = mbitmap_data; \
1392 unsigned int bwords = mbwords; \
1393 while (bwords-- > 0) { \
1394 gsize _bmap = *bitmap_data++; \
1395 void **start_run = _objptr; \
1396 /*g_print ("bitmap: 0x%x\n", _bmap);*/ \
1398 if ((_bmap & 1)) { \
1399 HANDLE_PTR (_objptr, (obj)); \
1404 _objptr = start_run + GC_BITS_PER_WORD; \
1406 e_start += el_size; \
1410 #define OBJ_VECTOR_FOREACH_PTR(vt,obj) do { \
1411 /* note: 0xffffc000 excludes DESC_TYPE_V_PTRFREE */ \
1412 if ((vt)->desc & 0xffffc000) { \
1413 int el_size = ((vt)->desc >> 3) & MAX_ELEMENT_SIZE; \
1414 /* there are pointers */ \
1415 int etype = (vt)->desc & 0xc000; \
1416 if (etype == (DESC_TYPE_V_REFS << 14)) { \
1417 void **p = (void**)((char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector)); \
1418 void **end_refs = (void**)((char*)p + el_size * mono_array_length_fast ((MonoArray*)(obj))); \
1419 /* Note: this code can handle also arrays of struct with only references in them */ \
1420 while (p < end_refs) { \
1421 HANDLE_PTR (p, (obj)); \
1424 } else if (etype == DESC_TYPE_V_RUN_LEN << 14) { \
1425 int offset = ((vt)->desc >> 16) & 0xff; \
1426 int num_refs = ((vt)->desc >> 24) & 0xff; \
1427 char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
1428 char *e_end = e_start + el_size * mono_array_length_fast ((MonoArray*)(obj)); \
1429 while (e_start < e_end) { \
1430 void **p = (void**)e_start; \
1433 for (i = 0; i < num_refs; ++i) { \
1434 HANDLE_PTR (p + i, (obj)); \
1436 e_start += el_size; \
1438 } else if (etype == DESC_TYPE_V_BITMAP << 14) { \
1439 char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
1440 char *e_end = e_start + el_size * mono_array_length_fast ((MonoArray*)(obj)); \
1441 while (e_start < e_end) { \
1442 void **p = (void**)e_start; \
1443 gsize _bmap = (vt)->desc >> 16; \
1444 /* Note: there is no object header here to skip */ \
1446 if ((_bmap & 1)) { \
1447 HANDLE_PTR (p, (obj)); \
1452 e_start += el_size; \
1458 //#include "sgen-major-copying.c"
1459 #include "sgen-marksweep.c"
1462 is_xdomain_ref_allowed (gpointer *ptr, char *obj, MonoDomain *domain)
1464 MonoObject *o = (MonoObject*)(obj);
1465 MonoObject *ref = (MonoObject*)*(ptr);
1466 int offset = (char*)(ptr) - (char*)o;
1468 if (o->vtable->klass == mono_defaults.thread_class && offset == G_STRUCT_OFFSET (MonoThread, internal_thread))
1470 if (o->vtable->klass == mono_defaults.internal_thread_class && offset == G_STRUCT_OFFSET (MonoInternalThread, current_appcontext))
1472 if (mono_class_has_parent (o->vtable->klass, mono_defaults.real_proxy_class) &&
1473 offset == G_STRUCT_OFFSET (MonoRealProxy, unwrapped_server))
1475 /* Thread.cached_culture_info */
1476 if (!strcmp (ref->vtable->klass->name_space, "System.Globalization") &&
1477 !strcmp (ref->vtable->klass->name, "CultureInfo") &&
1478 !strcmp(o->vtable->klass->name_space, "System") &&
1479 !strcmp(o->vtable->klass->name, "Object[]"))
1482 * at System.IO.MemoryStream.InternalConstructor (byte[],int,int,bool,bool) [0x0004d] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.IO/MemoryStream.cs:121
1483 * at System.IO.MemoryStream..ctor (byte[]) [0x00017] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.IO/MemoryStream.cs:81
1484 * at (wrapper remoting-invoke-with-check) System.IO.MemoryStream..ctor (byte[]) <IL 0x00020, 0xffffffff>
1485 * at System.Runtime.Remoting.Messaging.CADMethodCallMessage.GetArguments () [0x0000d] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Messaging/CADMessages.cs:327
1486 * at System.Runtime.Remoting.Messaging.MethodCall..ctor (System.Runtime.Remoting.Messaging.CADMethodCallMessage) [0x00017] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Messaging/MethodCall.cs:87
1487 * at System.AppDomain.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage,byte[]&,System.Runtime.Remoting.Messaging.CADMethodReturnMessage&) [0x00018] in /home/schani/Work/novell/trunk/mcs/class/corlib/System/AppDomain.cs:1213
1488 * at (wrapper remoting-invoke-with-check) System.AppDomain.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage,byte[]&,System.Runtime.Remoting.Messaging.CADMethodReturnMessage&) <IL 0x0003d, 0xffffffff>
1489 * at System.Runtime.Remoting.Channels.CrossAppDomainSink.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage) [0x00008] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Channels/CrossAppDomainChannel.cs:198
1490 * at (wrapper runtime-invoke) object.runtime_invoke_CrossAppDomainSink/ProcessMessageRes_object_object (object,intptr,intptr,intptr) <IL 0x0004c, 0xffffffff>
1492 if (!strcmp (ref->vtable->klass->name_space, "System") &&
1493 !strcmp (ref->vtable->klass->name, "Byte[]") &&
1494 !strcmp (o->vtable->klass->name_space, "System.IO") &&
1495 !strcmp (o->vtable->klass->name, "MemoryStream"))
1497 /* append_job() in threadpool.c */
1498 if (!strcmp (ref->vtable->klass->name_space, "System.Runtime.Remoting.Messaging") &&
1499 !strcmp (ref->vtable->klass->name, "AsyncResult") &&
1500 !strcmp (o->vtable->klass->name_space, "System") &&
1501 !strcmp (o->vtable->klass->name, "Object[]") &&
1502 mono_thread_pool_is_queue_array ((MonoArray*) o))
1508 check_reference_for_xdomain (gpointer *ptr, char *obj, MonoDomain *domain)
1510 MonoObject *o = (MonoObject*)(obj);
1511 MonoObject *ref = (MonoObject*)*(ptr);
1512 int offset = (char*)(ptr) - (char*)o;
1514 MonoClassField *field;
1517 if (!ref || ref->vtable->domain == domain)
1519 if (is_xdomain_ref_allowed (ptr, obj, domain))
1523 for (class = o->vtable->klass; class; class = class->parent) {
1526 for (i = 0; i < class->field.count; ++i) {
1527 if (class->fields[i].offset == offset) {
1528 field = &class->fields[i];
1536 if (ref->vtable->klass == mono_defaults.string_class)
1537 str = mono_string_to_utf8 ((MonoString*)ref);
1540 g_print ("xdomain reference in %p (%s.%s) at offset %d (%s) to %p (%s.%s) (%s) - pointed to by:\n",
1541 o, o->vtable->klass->name_space, o->vtable->klass->name,
1542 offset, field ? field->name : "",
1543 ref, ref->vtable->klass->name_space, ref->vtable->klass->name, str ? str : "");
1544 mono_gc_scan_for_specific_ref (o);
1550 #define HANDLE_PTR(ptr,obj) check_reference_for_xdomain ((ptr), (obj), domain)
1553 scan_object_for_xdomain_refs (char *start, mword size, void *data)
1555 MonoDomain *domain = ((MonoObject*)start)->vtable->domain;
1557 #include "sgen-scan-object.h"
1561 #define HANDLE_PTR(ptr,obj) do { \
1562 if ((MonoObject*)*(ptr) == key) { \
1563 g_print ("found ref to %p in object %p (%s) at offset %td\n", \
1564 key, (obj), safe_name ((obj)), ((char*)(ptr) - (char*)(obj))); \
1569 scan_object_for_specific_ref (char *start, MonoObject *key)
1571 #include "sgen-scan-object.h"
1575 scan_area_with_callback (char *start, char *end, IterateObjectCallbackFunc callback, void *data)
1577 while (start < end) {
1579 if (!*(void**)start) {
1580 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
1584 size = ALIGN_UP (safe_object_get_size ((MonoObject*) start));
1586 callback (start, size, data);
1593 scan_object_for_specific_ref_callback (char *obj, size_t size, MonoObject *key)
1595 scan_object_for_specific_ref (obj, key);
1599 check_root_obj_specific_ref (RootRecord *root, MonoObject *key, MonoObject *obj)
1603 g_print ("found ref to %p in root record %p\n", key, root);
1606 static MonoObject *check_key = NULL;
1607 static RootRecord *check_root = NULL;
1610 check_root_obj_specific_ref_from_marker (void **obj)
1612 check_root_obj_specific_ref (check_root, check_key, *obj);
1616 scan_roots_for_specific_ref (MonoObject *key, int root_type)
1621 for (i = 0; i < roots_hash_size [root_type]; ++i) {
1622 for (root = roots_hash [root_type][i]; root; root = root->next) {
1623 void **start_root = (void**)root->start_root;
1624 mword desc = root->root_desc;
1628 switch (desc & ROOT_DESC_TYPE_MASK) {
1629 case ROOT_DESC_BITMAP:
1630 desc >>= ROOT_DESC_TYPE_SHIFT;
1633 check_root_obj_specific_ref (root, key, *start_root);
1638 case ROOT_DESC_COMPLEX: {
1639 gsize *bitmap_data = complex_descriptors + (desc >> ROOT_DESC_TYPE_SHIFT);
1640 int bwords = (*bitmap_data) - 1;
1641 void **start_run = start_root;
1643 while (bwords-- > 0) {
1644 gsize bmap = *bitmap_data++;
1645 void **objptr = start_run;
1648 check_root_obj_specific_ref (root, key, *objptr);
1652 start_run += GC_BITS_PER_WORD;
1656 case ROOT_DESC_USER: {
1657 MonoGCRootMarkFunc marker = user_descriptors [desc >> ROOT_DESC_TYPE_SHIFT];
1658 marker (start_root, check_root_obj_specific_ref_from_marker);
1661 case ROOT_DESC_RUN_LEN:
1662 g_assert_not_reached ();
1664 g_assert_not_reached ();
1673 mono_gc_scan_for_specific_ref (MonoObject *key)
1679 scan_area_with_callback (nursery_section->data, nursery_section->end_data,
1680 (IterateObjectCallbackFunc)scan_object_for_specific_ref_callback, key);
1682 major_iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)scan_object_for_specific_ref_callback, key);
1684 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1685 scan_object_for_specific_ref (bigobj->data, key);
1687 scan_roots_for_specific_ref (key, ROOT_TYPE_NORMAL);
1688 scan_roots_for_specific_ref (key, ROOT_TYPE_WBARRIER);
1690 for (i = 0; i < roots_hash_size [ROOT_TYPE_PINNED]; ++i) {
1691 for (root = roots_hash [ROOT_TYPE_PINNED][i]; root; root = root->next) {
1692 void **ptr = (void**)root->start_root;
1694 while (ptr < (void**)root->end_root) {
1695 check_root_obj_specific_ref (root, *ptr, key);
1702 /* Clear all remaining nursery fragments */
1704 clear_nursery_fragments (char *next)
1707 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION) {
1708 g_assert (next <= nursery_frag_real_end);
1709 memset (next, 0, nursery_frag_real_end - next);
1710 for (frag = nursery_fragments; frag; frag = frag->next) {
1711 memset (frag->fragment_start, 0, frag->fragment_end - frag->fragment_start);
1717 need_remove_object_for_domain (char *start, MonoDomain *domain)
1719 if (mono_object_domain (start) == domain) {
1720 DEBUG (4, fprintf (gc_debug_file, "Need to cleanup object %p\n", start));
1721 binary_protocol_cleanup (start, (gpointer)LOAD_VTABLE (start), safe_object_get_size ((MonoObject*)start));
1728 process_object_for_domain_clearing (char *start, MonoDomain *domain)
1730 GCVTable *vt = (GCVTable*)LOAD_VTABLE (start);
1731 if (vt->klass == mono_defaults.internal_thread_class)
1732 g_assert (mono_object_domain (start) == mono_get_root_domain ());
1733 /* The object could be a proxy for an object in the domain
1735 if (mono_class_has_parent (vt->klass, mono_defaults.real_proxy_class)) {
1736 MonoObject *server = ((MonoRealProxy*)start)->unwrapped_server;
1738 /* The server could already have been zeroed out, so
1739 we need to check for that, too. */
1740 if (server && (!LOAD_VTABLE (server) || mono_object_domain (server) == domain)) {
1741 DEBUG (4, fprintf (gc_debug_file, "Cleaning up remote pointer in %p to object %p\n",
1743 ((MonoRealProxy*)start)->unwrapped_server = NULL;
1748 static MonoDomain *check_domain = NULL;
1751 check_obj_not_in_domain (void **o)
1753 g_assert (((MonoObject*)(*o))->vtable->domain != check_domain);
1757 scan_for_registered_roots_in_domain (MonoDomain *domain, int root_type)
1761 check_domain = domain;
1762 for (i = 0; i < roots_hash_size [root_type]; ++i) {
1763 for (root = roots_hash [root_type][i]; root; root = root->next) {
1764 void **start_root = (void**)root->start_root;
1765 mword desc = root->root_desc;
1767 /* The MonoDomain struct is allowed to hold
1768 references to objects in its own domain. */
1769 if (start_root == (void**)domain)
1772 switch (desc & ROOT_DESC_TYPE_MASK) {
1773 case ROOT_DESC_BITMAP:
1774 desc >>= ROOT_DESC_TYPE_SHIFT;
1776 if ((desc & 1) && *start_root)
1777 check_obj_not_in_domain (*start_root);
1782 case ROOT_DESC_COMPLEX: {
1783 gsize *bitmap_data = complex_descriptors + (desc >> ROOT_DESC_TYPE_SHIFT);
1784 int bwords = (*bitmap_data) - 1;
1785 void **start_run = start_root;
1787 while (bwords-- > 0) {
1788 gsize bmap = *bitmap_data++;
1789 void **objptr = start_run;
1791 if ((bmap & 1) && *objptr)
1792 check_obj_not_in_domain (*objptr);
1796 start_run += GC_BITS_PER_WORD;
1800 case ROOT_DESC_USER: {
1801 MonoGCRootMarkFunc marker = user_descriptors [desc >> ROOT_DESC_TYPE_SHIFT];
1802 marker (start_root, check_obj_not_in_domain);
1805 case ROOT_DESC_RUN_LEN:
1806 g_assert_not_reached ();
1808 g_assert_not_reached ();
1812 check_domain = NULL;
1816 check_for_xdomain_refs (void)
1820 scan_area_with_callback (nursery_section->data, nursery_section->end_data, (IterateObjectCallbackFunc)scan_object_for_xdomain_refs, NULL);
1822 major_iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)scan_object_for_xdomain_refs, NULL);
1824 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1825 scan_object_for_xdomain_refs (bigobj->data, bigobj->size, NULL);
1829 clear_domain_process_object (char *obj, MonoDomain *domain)
1833 process_object_for_domain_clearing (obj, domain);
1834 remove = need_remove_object_for_domain (obj, domain);
1836 if (remove && ((MonoObject*)obj)->synchronisation) {
1837 void **dislink = mono_monitor_get_object_monitor_weak_link ((MonoObject*)obj);
1839 mono_gc_register_disappearing_link (NULL, dislink, FALSE);
1846 clear_domain_process_minor_object_callback (char *obj, size_t size, MonoDomain *domain)
1848 if (clear_domain_process_object (obj, domain))
1849 memset (obj, 0, size);
1853 clear_domain_process_major_object_callback (char *obj, size_t size, MonoDomain *domain)
1855 clear_domain_process_object (obj, domain);
1859 clear_domain_free_major_non_pinned_object_callback (char *obj, size_t size, MonoDomain *domain)
1861 if (need_remove_object_for_domain (obj, domain))
1862 major_free_non_pinned_object (obj, size);
1866 clear_domain_free_major_pinned_object_callback (char *obj, size_t size, MonoDomain *domain)
1868 if (need_remove_object_for_domain (obj, domain))
1869 free_pinned_object (obj, size);
1873 * When appdomains are unloaded we can easily remove objects that have finalizers,
1874 * but all the others could still be present in random places on the heap.
1875 * We need a sweep to get rid of them even though it's going to be costly
1877 * The reason we need to remove them is because we access the vtable and class
1878 * structures to know the object size and the reference bitmap: once the domain is
1879 * unloaded the point to random memory.
1882 mono_gc_clear_domain (MonoDomain * domain)
1884 LOSObject *bigobj, *prev;
1889 clear_nursery_fragments (nursery_next);
1891 if (xdomain_checks && domain != mono_get_root_domain ()) {
1892 scan_for_registered_roots_in_domain (domain, ROOT_TYPE_NORMAL);
1893 scan_for_registered_roots_in_domain (domain, ROOT_TYPE_WBARRIER);
1894 check_for_xdomain_refs ();
1897 scan_area_with_callback (nursery_section->data, nursery_section->end_data,
1898 (IterateObjectCallbackFunc)clear_domain_process_minor_object_callback, domain);
1900 /*Ephemerons and dislinks must be processed before LOS since they might end up pointing
1901 to memory returned to the OS.*/
1902 null_ephemerons_for_domain (domain);
1904 for (i = GENERATION_NURSERY; i < GENERATION_MAX; ++i)
1905 null_links_for_domain (domain, i);
1907 /* We need two passes over major and large objects because
1908 freeing such objects might give their memory back to the OS
1909 (in the case of large objects) or obliterate its vtable
1910 (pinned objects with major-copying or pinned and non-pinned
1911 objects with major-mark&sweep), but we might need to
1912 dereference a pointer from an object to another object if
1913 the first object is a proxy. */
1914 major_iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)clear_domain_process_major_object_callback, domain);
1915 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1916 clear_domain_process_object (bigobj->data, domain);
1919 for (bigobj = los_object_list; bigobj;) {
1920 if (need_remove_object_for_domain (bigobj->data, domain)) {
1921 LOSObject *to_free = bigobj;
1923 prev->next = bigobj->next;
1925 los_object_list = bigobj->next;
1926 bigobj = bigobj->next;
1927 DEBUG (4, fprintf (gc_debug_file, "Freeing large object %p\n",
1929 free_large_object (to_free);
1933 bigobj = bigobj->next;
1935 major_iterate_objects (TRUE, FALSE, (IterateObjectCallbackFunc)clear_domain_free_major_non_pinned_object_callback, domain);
1936 major_iterate_objects (FALSE, TRUE, (IterateObjectCallbackFunc)clear_domain_free_major_pinned_object_callback, domain);
1942 global_remset_cache_clear (void)
1944 memset (global_remset_cache, 0, sizeof (global_remset_cache));
1948 * Tries to check if a given remset location was already added to the global remset.
1951 * A 2 entry, LRU cache of recently saw location remsets.
1953 * It's hand-coded instead of done using loops to reduce the number of memory references on cache hit.
1955 * Returns TRUE is the element was added..
1958 global_remset_location_was_not_added (gpointer ptr)
1961 gpointer first = global_remset_cache [0], second;
1963 HEAVY_STAT (++stat_global_remsets_discarded);
1967 second = global_remset_cache [1];
1969 if (second == ptr) {
1970 /*Move the second to the front*/
1971 global_remset_cache [0] = second;
1972 global_remset_cache [1] = first;
1974 HEAVY_STAT (++stat_global_remsets_discarded);
1978 global_remset_cache [0] = second;
1979 global_remset_cache [1] = ptr;
1984 * add_to_global_remset:
1986 * The global remset contains locations which point into newspace after
1987 * a minor collection. This can happen if the objects they point to are pinned.
1990 add_to_global_remset (gpointer ptr)
1994 g_assert (!ptr_in_nursery (ptr) && ptr_in_nursery (*(gpointer*)ptr));
1996 if (!global_remset_location_was_not_added (ptr))
1999 DEBUG (8, fprintf (gc_debug_file, "Adding global remset for %p\n", ptr));
2000 binary_protocol_global_remset (ptr, *(gpointer*)ptr, (gpointer)LOAD_VTABLE (*(gpointer*)ptr));
2002 HEAVY_STAT (++stat_global_remsets_added);
2005 * FIXME: If an object remains pinned, we need to add it at every minor collection.
2006 * To avoid uncontrolled growth of the global remset, only add each pointer once.
2008 if (global_remset->store_next + 3 < global_remset->end_set) {
2009 *(global_remset->store_next++) = (mword)ptr;
2012 rs = alloc_remset (global_remset->end_set - global_remset->data, NULL);
2013 rs->next = global_remset;
2015 *(global_remset->store_next++) = (mword)ptr;
2018 int global_rs_size = 0;
2020 for (rs = global_remset; rs; rs = rs->next) {
2021 global_rs_size += rs->store_next - rs->data;
2023 DEBUG (4, fprintf (gc_debug_file, "Global remset now has size %d\n", global_rs_size));
2028 * FIXME: allocate before calling this function and pass the
2029 * destination address.
2032 copy_object_no_checks (void *obj, GrayQueue *queue)
2034 static const void *copy_labels [] = { &&LAB_0, &&LAB_1, &&LAB_2, &&LAB_3, &&LAB_4, &&LAB_5, &&LAB_6, &&LAB_7, &&LAB_8 };
2038 MonoVTable *vt = ((MonoObject*)obj)->vtable;
2039 gboolean has_references = vt->gc_descr != (void*)DESC_TYPE_RUN_LENGTH;
2041 objsize = safe_object_get_size ((MonoObject*)obj);
2042 objsize += ALLOC_ALIGN - 1;
2043 objsize &= ~(ALLOC_ALIGN - 1);
2045 DEBUG (9, g_assert (vt->klass->inited));
2046 MAJOR_GET_COPY_OBJECT_SPACE (destination, objsize, has_references);
2048 DEBUG (9, fprintf (gc_debug_file, " (to %p, %s size: %lu)\n", destination, ((MonoObject*)obj)->vtable->klass->name, (unsigned long)objsize));
2049 binary_protocol_copy (obj, destination, ((MonoObject*)obj)->vtable, objsize);
2051 if (objsize <= sizeof (gpointer) * 8) {
2052 mword *dest = (mword*)destination;
2053 goto *copy_labels [objsize / sizeof (gpointer)];
2055 (dest) [7] = ((mword*)obj) [7];
2057 (dest) [6] = ((mword*)obj) [6];
2059 (dest) [5] = ((mword*)obj) [5];
2061 (dest) [4] = ((mword*)obj) [4];
2063 (dest) [3] = ((mword*)obj) [3];
2065 (dest) [2] = ((mword*)obj) [2];
2067 (dest) [1] = ((mword*)obj) [1];
2069 (dest) [0] = ((mword*)obj) [0];
2077 char* edi = destination;
2078 __asm__ __volatile__(
2080 : "=&c" (ecx), "=&D" (edi), "=&S" (esi)
2081 : "0" (objsize/4), "1" (edi),"2" (esi)
2086 memcpy (destination, obj, objsize);
2089 /* adjust array->bounds */
2090 DEBUG (9, g_assert (vt->gc_descr));
2091 if (G_UNLIKELY (vt->rank && ((MonoArray*)obj)->bounds)) {
2092 MonoArray *array = (MonoArray*)destination;
2093 array->bounds = (MonoArrayBounds*)((char*)destination + ((char*)((MonoArray*)obj)->bounds - (char*)obj));
2094 DEBUG (9, fprintf (gc_debug_file, "Array instance %p: size: %lu, rank: %d, length: %lu\n", array, (unsigned long)objsize, vt->rank, (unsigned long)mono_array_length (array)));
2096 /* set the forwarding pointer */
2097 forward_object (obj, destination);
2098 if (G_UNLIKELY (mono_profiler_events & MONO_PROFILE_GC_MOVES)) {
2099 if (moved_objects_idx == MOVED_OBJECTS_NUM) {
2100 mono_profiler_gc_moves (moved_objects, moved_objects_idx);
2101 moved_objects_idx = 0;
2103 moved_objects [moved_objects_idx++] = obj;
2104 moved_objects [moved_objects_idx++] = destination;
2107 if (has_references) {
2108 DEBUG (9, fprintf (gc_debug_file, "Enqueuing gray object %p (%s)\n", obj, safe_name (obj)));
2109 GRAY_OBJECT_ENQUEUE (queue, obj);
2115 * This is how the copying happens from the nursery to the old generation.
2116 * We assume that at this time all the pinned objects have been identified and
2118 * We run scan_object() for each pinned object so that each referenced
2119 * objects if possible are copied. The new gray objects created can have
2120 * scan_object() run on them right away, too.
2121 * Then we run copy_object() for the precisely tracked roots. At this point
2122 * all the roots are either gray or black. We run scan_object() on the gray
2123 * objects until no more gray objects are created.
2124 * At the end of the process we walk again the pinned list and we unmark
2125 * the pinned flag. As we go we also create the list of free space for use
2126 * in the next allocation runs.
2128 * We need to remember objects from the old generation that point to the new one
2129 * (or just addresses?).
2131 * copy_object could be made into a macro once debugged (use inline for now).
2134 static void __attribute__((noinline))
2135 copy_object (void **obj_slot, GrayQueue *queue)
2138 char *obj = *obj_slot;
2140 DEBUG (9, g_assert (current_collection_generation == GENERATION_NURSERY));
2142 HEAVY_STAT (++stat_copy_object_called_nursery);
2144 if (!ptr_in_nursery (obj)) {
2145 HEAVY_STAT (++stat_nursery_copy_object_failed_from_space);
2149 DEBUG (9, fprintf (gc_debug_file, "Precise copy of %p from %p", obj, obj_slot));
2152 * Before we can copy the object we must make sure that we are
2153 * allowed to, i.e. that the object not pinned or not already
2157 if ((forwarded = object_is_forwarded (obj))) {
2158 DEBUG (9, g_assert (((MonoVTable*)LOAD_VTABLE(obj))->gc_descr));
2159 DEBUG (9, fprintf (gc_debug_file, " (already forwarded to %p)\n", forwarded));
2160 HEAVY_STAT (++stat_nursery_copy_object_failed_forwarded);
2161 *obj_slot = forwarded;
2164 if (object_is_pinned (obj)) {
2165 DEBUG (9, g_assert (((MonoVTable*)LOAD_VTABLE(obj))->gc_descr));
2166 DEBUG (9, fprintf (gc_debug_file, " (pinned, no change)\n"));
2167 HEAVY_STAT (++stat_nursery_copy_object_failed_pinned);
2171 HEAVY_STAT (++stat_objects_copied_nursery);
2173 *obj_slot = copy_object_no_checks (obj, queue);
2177 #define HANDLE_PTR(ptr,obj) do { \
2178 void *__old = *(ptr); \
2181 copy_object ((ptr), queue); \
2183 DEBUG (9, if (__old != __copy) fprintf (gc_debug_file, "Overwrote field at %p with %p (was: %p)\n", (ptr), *(ptr), __old)); \
2184 if (G_UNLIKELY (ptr_in_nursery (__copy) && !ptr_in_nursery ((ptr)))) \
2185 add_to_global_remset ((ptr)); \
2190 * Scan the object pointed to by @start for references to
2191 * other objects between @from_start and @from_end and copy
2192 * them to the gray_objects area.
2195 scan_object (char *start, GrayQueue *queue)
2197 #include "sgen-scan-object.h"
2199 HEAVY_STAT (++stat_scan_object_called_nursery);
2205 * Scan the valuetype pointed to by START, described by DESC for references to
2206 * other objects between @from_start and @from_end and copy them to the gray_objects area.
2207 * Returns a pointer to the end of the object.
2210 scan_vtype (char *start, mword desc, char* from_start, char* from_end, GrayQueue *queue)
2214 /* The descriptors include info about the MonoObject header as well */
2215 start -= sizeof (MonoObject);
2217 switch (desc & 0x7) {
2218 case DESC_TYPE_RUN_LENGTH:
2219 OBJ_RUN_LEN_FOREACH_PTR (desc,start);
2220 OBJ_RUN_LEN_SIZE (skip_size, desc, start);
2221 g_assert (skip_size);
2222 return start + skip_size;
2223 case DESC_TYPE_SMALL_BITMAP:
2224 OBJ_BITMAP_FOREACH_PTR (desc,start);
2225 OBJ_BITMAP_SIZE (skip_size, desc, start);
2226 return start + skip_size;
2227 case DESC_TYPE_LARGE_BITMAP:
2228 case DESC_TYPE_COMPLEX:
2230 g_assert_not_reached ();
2233 // The other descriptors can't happen with vtypes
2234 g_assert_not_reached ();
2241 #define HANDLE_PTR(ptr,obj) do { \
2242 void *__old = *(ptr); \
2245 major_copy_or_mark_object ((ptr), queue); \
2247 DEBUG (9, if (__old != __copy) fprintf (gc_debug_file, "Overwrote field at %p with %p (was: %p)\n", (ptr), *(ptr), __old)); \
2248 if (G_UNLIKELY (ptr_in_nursery (__copy) && !ptr_in_nursery ((ptr)))) \
2249 add_to_global_remset ((ptr)); \
2254 major_scan_object (char *start, GrayQueue *queue)
2256 #include "sgen-scan-object.h"
2258 HEAVY_STAT (++stat_scan_object_called_major);
2264 * Scan objects in the gray stack until the stack is empty. This should be called
2265 * frequently after each object is copied, to achieve better locality and cache
2269 drain_gray_stack (GrayQueue *queue)
2273 if (current_collection_generation == GENERATION_NURSERY) {
2275 GRAY_OBJECT_DEQUEUE (queue, obj);
2278 DEBUG (9, fprintf (gc_debug_file, "Precise gray object scan %p (%s)\n", obj, safe_name (obj)));
2279 scan_object (obj, queue);
2283 GRAY_OBJECT_DEQUEUE (queue, obj);
2286 DEBUG (9, fprintf (gc_debug_file, "Precise gray object scan %p (%s)\n", obj, safe_name (obj)));
2287 major_scan_object (obj, queue);
2293 * Addresses from start to end are already sorted. This function finds
2294 * the object header for each address and pins the object. The
2295 * addresses must be inside the passed section. The (start of the)
2296 * address array is overwritten with the addresses of the actually
2297 * pinned objects. Return the number of pinned objects.
2300 pin_objects_from_addresses (GCMemSection *section, void **start, void **end, void *start_nursery, void *end_nursery, GrayQueue *queue)
2305 void *last_obj = NULL;
2306 size_t last_obj_size = 0;
2309 void **definitely_pinned = start;
2310 while (start < end) {
2312 /* the range check should be reduntant */
2313 if (addr != last && addr >= start_nursery && addr < end_nursery) {
2314 DEBUG (5, fprintf (gc_debug_file, "Considering pinning addr %p\n", addr));
2315 /* multiple pointers to the same object */
2316 if (addr >= last_obj && (char*)addr < (char*)last_obj + last_obj_size) {
2320 idx = ((char*)addr - (char*)section->data) / SCAN_START_SIZE;
2321 g_assert (idx < section->num_scan_start);
2322 search_start = (void*)section->scan_starts [idx];
2323 if (!search_start || search_start > addr) {
2326 search_start = section->scan_starts [idx];
2327 if (search_start && search_start <= addr)
2330 if (!search_start || search_start > addr)
2331 search_start = start_nursery;
2333 if (search_start < last_obj)
2334 search_start = (char*)last_obj + last_obj_size;
2335 /* now addr should be in an object a short distance from search_start
2336 * Note that search_start must point to zeroed mem or point to an object.
2339 if (!*(void**)search_start) {
2340 search_start = (void*)ALIGN_UP ((mword)search_start + sizeof (gpointer));
2343 last_obj = search_start;
2344 last_obj_size = ALIGN_UP (safe_object_get_size ((MonoObject*)search_start));
2345 DEBUG (8, fprintf (gc_debug_file, "Pinned try match %p (%s), size %zd\n", last_obj, safe_name (last_obj), last_obj_size));
2346 if (addr >= search_start && (char*)addr < (char*)last_obj + last_obj_size) {
2347 DEBUG (4, fprintf (gc_debug_file, "Pinned object %p, vtable %p (%s), count %d\n", search_start, *(void**)search_start, safe_name (search_start), count));
2348 binary_protocol_pin (search_start, (gpointer)LOAD_VTABLE (search_start), safe_object_get_size (search_start));
2349 pin_object (search_start);
2350 GRAY_OBJECT_ENQUEUE (queue, search_start);
2352 pin_stats_register_object (search_start, last_obj_size);
2353 definitely_pinned [count] = search_start;
2357 /* skip to the next object */
2358 search_start = (void*)((char*)search_start + last_obj_size);
2359 } while (search_start <= addr);
2360 /* we either pinned the correct object or we ignored the addr because
2361 * it points to unused zeroed memory.
2367 //printf ("effective pinned: %d (at the end: %d)\n", count, (char*)end_nursery - (char*)last);
2372 pin_objects_in_section (GCMemSection *section, GrayQueue *queue)
2374 int start = section->pin_queue_start;
2375 int end = section->pin_queue_end;
2378 reduced_to = pin_objects_from_addresses (section, pin_queue + start, pin_queue + end,
2379 section->data, section->next_data, queue);
2380 section->pin_queue_start = start;
2381 section->pin_queue_end = start + reduced_to;
2385 /* Sort the addresses in array in increasing order.
2386 * Done using a by-the book heap sort. Which has decent and stable performance, is pretty cache efficient.
2389 sort_addresses (void **array, int size)
2394 for (i = 1; i < size; ++i) {
2397 int parent = (child - 1) / 2;
2399 if (array [parent] >= array [child])
2402 tmp = array [parent];
2403 array [parent] = array [child];
2404 array [child] = tmp;
2410 for (i = size - 1; i > 0; --i) {
2413 array [i] = array [0];
2419 while (root * 2 + 1 <= end) {
2420 int child = root * 2 + 1;
2422 if (child < end && array [child] < array [child + 1])
2424 if (array [root] >= array [child])
2428 array [root] = array [child];
2429 array [child] = tmp;
2436 static G_GNUC_UNUSED void
2437 print_nursery_gaps (void* start_nursery, void *end_nursery)
2440 gpointer first = start_nursery;
2442 for (i = 0; i < next_pin_slot; ++i) {
2443 next = pin_queue [i];
2444 fprintf (gc_debug_file, "Nursery range: %p-%p, size: %td\n", first, next, (char*)next-(char*)first);
2448 fprintf (gc_debug_file, "Nursery range: %p-%p, size: %td\n", first, next, (char*)next-(char*)first);
2451 /* reduce the info in the pin queue, removing duplicate pointers and sorting them */
2453 optimize_pin_queue (int start_slot)
2455 void **start, **cur, **end;
2456 /* sort and uniq pin_queue: we just sort and we let the rest discard multiple values */
2457 /* it may be better to keep ranges of pinned memory instead of individually pinning objects */
2458 DEBUG (5, fprintf (gc_debug_file, "Sorting pin queue, size: %d\n", next_pin_slot));
2459 if ((next_pin_slot - start_slot) > 1)
2460 sort_addresses (pin_queue + start_slot, next_pin_slot - start_slot);
2461 start = cur = pin_queue + start_slot;
2462 end = pin_queue + next_pin_slot;
2465 while (*start == *cur && cur < end)
2469 next_pin_slot = start - pin_queue;
2470 DEBUG (5, fprintf (gc_debug_file, "Pin queue reduced to size: %d\n", next_pin_slot));
2471 //DEBUG (6, print_nursery_gaps (start_nursery, end_nursery));
2476 * Scan the memory between start and end and queue values which could be pointers
2477 * to the area between start_nursery and end_nursery for later consideration.
2478 * Typically used for thread stacks.
2481 conservatively_pin_objects_from (void **start, void **end, void *start_nursery, void *end_nursery, int pin_type)
2484 while (start < end) {
2485 if (*start >= start_nursery && *start < end_nursery) {
2487 * *start can point to the middle of an object
2488 * note: should we handle pointing at the end of an object?
2489 * pinning in C# code disallows pointing at the end of an object
2490 * but there is some small chance that an optimizing C compiler
2491 * may keep the only reference to an object by pointing
2492 * at the end of it. We ignore this small chance for now.
2493 * Pointers to the end of an object are indistinguishable
2494 * from pointers to the start of the next object in memory
2495 * so if we allow that we'd need to pin two objects...
2496 * We queue the pointer in an array, the
2497 * array will then be sorted and uniqued. This way
2498 * we can coalesce several pinning pointers and it should
2499 * be faster since we'd do a memory scan with increasing
2500 * addresses. Note: we can align the address to the allocation
2501 * alignment, so the unique process is more effective.
2503 mword addr = (mword)*start;
2504 addr &= ~(ALLOC_ALIGN - 1);
2505 if (addr >= (mword)start_nursery && addr < (mword)end_nursery)
2506 pin_stage_ptr ((void*)addr);
2508 pin_stats_register_address ((char*)addr, pin_type);
2509 DEBUG (6, if (count) fprintf (gc_debug_file, "Pinning address %p\n", (void*)addr));
2514 DEBUG (7, if (count) fprintf (gc_debug_file, "found %d potential pinned heap pointers\n", count));
2518 * Debugging function: find in the conservative roots where @obj is being pinned.
2520 static G_GNUC_UNUSED void
2521 find_pinning_reference (char *obj, size_t size)
2525 char *endobj = obj + size;
2526 for (i = 0; i < roots_hash_size [0]; ++i) {
2527 for (root = roots_hash [0][i]; root; root = root->next) {
2528 /* if desc is non-null it has precise info */
2529 if (!root->root_desc) {
2530 char ** start = (char**)root->start_root;
2531 while (start < (char**)root->end_root) {
2532 if (*start >= obj && *start < endobj) {
2533 DEBUG (0, fprintf (gc_debug_file, "Object %p referenced in pinned roots %p-%p (at %p in record %p)\n", obj, root->start_root, root->end_root, start, root));
2540 find_pinning_ref_from_thread (obj, size);
2544 * The first thing we do in a collection is to identify pinned objects.
2545 * This function considers all the areas of memory that need to be
2546 * conservatively scanned.
2549 pin_from_roots (void *start_nursery, void *end_nursery)
2553 DEBUG (2, fprintf (gc_debug_file, "Scanning pinned roots (%d bytes, %d/%d entries)\n", (int)roots_size, num_roots_entries [ROOT_TYPE_NORMAL], num_roots_entries [ROOT_TYPE_PINNED]));
2554 /* objects pinned from the API are inside these roots */
2555 for (i = 0; i < roots_hash_size [ROOT_TYPE_PINNED]; ++i) {
2556 for (root = roots_hash [ROOT_TYPE_PINNED][i]; root; root = root->next) {
2557 DEBUG (6, fprintf (gc_debug_file, "Pinned roots %p-%p\n", root->start_root, root->end_root));
2558 conservatively_pin_objects_from ((void**)root->start_root, (void**)root->end_root, start_nursery, end_nursery, PIN_TYPE_OTHER);
2561 /* now deal with the thread stacks
2562 * in the future we should be able to conservatively scan only:
2563 * *) the cpu registers
2564 * *) the unmanaged stack frames
2565 * *) the _last_ managed stack frame
2566 * *) pointers slots in managed frames
2568 scan_thread_data (start_nursery, end_nursery, FALSE);
2570 evacuate_pin_staging_area ();
2573 static CopyOrMarkObjectFunc user_copy_or_mark_func;
2574 static GrayQueue *user_copy_or_mark_queue;
2577 single_arg_user_copy_or_mark (void **obj)
2579 user_copy_or_mark_func (obj, user_copy_or_mark_queue);
2583 * The memory area from start_root to end_root contains pointers to objects.
2584 * Their position is precisely described by @desc (this means that the pointer
2585 * can be either NULL or the pointer to the start of an object).
2586 * This functions copies them to to_space updates them.
2588 * This function is not thread-safe!
2591 precisely_scan_objects_from (CopyOrMarkObjectFunc copy_func, void** start_root, void** end_root, char* n_start, char *n_end, mword desc, GrayQueue *queue)
2593 switch (desc & ROOT_DESC_TYPE_MASK) {
2594 case ROOT_DESC_BITMAP:
2595 desc >>= ROOT_DESC_TYPE_SHIFT;
2597 if ((desc & 1) && *start_root) {
2598 copy_func (start_root, queue);
2599 DEBUG (9, fprintf (gc_debug_file, "Overwrote root at %p with %p\n", start_root, *start_root));
2600 drain_gray_stack (queue);
2606 case ROOT_DESC_COMPLEX: {
2607 gsize *bitmap_data = complex_descriptors + (desc >> ROOT_DESC_TYPE_SHIFT);
2608 int bwords = (*bitmap_data) - 1;
2609 void **start_run = start_root;
2611 while (bwords-- > 0) {
2612 gsize bmap = *bitmap_data++;
2613 void **objptr = start_run;
2615 if ((bmap & 1) && *objptr) {
2616 copy_func (objptr, queue);
2617 DEBUG (9, fprintf (gc_debug_file, "Overwrote root at %p with %p\n", objptr, *objptr));
2618 drain_gray_stack (queue);
2623 start_run += GC_BITS_PER_WORD;
2627 case ROOT_DESC_USER: {
2628 MonoGCRootMarkFunc marker = user_descriptors [desc >> ROOT_DESC_TYPE_SHIFT];
2629 user_copy_or_mark_func = copy_func;
2630 user_copy_or_mark_queue = queue;
2631 marker (start_root, single_arg_user_copy_or_mark);
2632 user_copy_or_mark_func = NULL;
2633 user_copy_or_mark_queue = NULL;
2636 case ROOT_DESC_RUN_LEN:
2637 g_assert_not_reached ();
2639 g_assert_not_reached ();
2644 alloc_fragment (void)
2646 Fragment *frag = fragment_freelist;
2648 fragment_freelist = frag->next;
2652 frag = get_internal_mem (sizeof (Fragment), INTERNAL_MEM_FRAGMENT);
2657 /* size must be a power of 2 */
2659 get_os_memory_aligned (mword size, mword alignment, gboolean activate)
2661 /* Allocate twice the memory to be able to put the block on an aligned address */
2662 char *mem = get_os_memory (size + alignment, activate);
2667 aligned = (char*)((mword)(mem + (alignment - 1)) & ~(alignment - 1));
2668 g_assert (aligned >= mem && aligned + size <= mem + size + alignment && !((mword)aligned & (alignment - 1)));
2671 free_os_memory (mem, aligned - mem);
2672 if (aligned + size < mem + size + alignment)
2673 free_os_memory (aligned + size, (mem + size + alignment) - (aligned + size));
2679 * Allocate and setup the data structures needed to be able to allocate objects
2680 * in the nursery. The nursery is stored in nursery_section.
2683 alloc_nursery (void)
2685 GCMemSection *section;
2691 if (nursery_section)
2693 DEBUG (2, fprintf (gc_debug_file, "Allocating nursery size: %lu\n", (unsigned long)nursery_size));
2694 /* later we will alloc a larger area for the nursery but only activate
2695 * what we need. The rest will be used as expansion if we have too many pinned
2696 * objects in the existing nursery.
2698 /* FIXME: handle OOM */
2699 section = get_internal_mem (SIZEOF_GC_MEM_SECTION, INTERNAL_MEM_SECTION);
2701 g_assert (nursery_size == DEFAULT_NURSERY_SIZE);
2702 alloc_size = nursery_size;
2703 #ifdef ALIGN_NURSERY
2704 data = get_os_memory_aligned (alloc_size, alloc_size, TRUE);
2706 data = get_os_memory (alloc_size, TRUE);
2708 nursery_start = data;
2709 nursery_real_end = nursery_start + nursery_size;
2710 UPDATE_HEAP_BOUNDARIES (nursery_start, nursery_real_end);
2711 nursery_next = nursery_start;
2712 total_alloc += alloc_size;
2713 DEBUG (4, fprintf (gc_debug_file, "Expanding nursery size (%p-%p): %lu, total: %lu\n", data, data + alloc_size, (unsigned long)nursery_size, (unsigned long)total_alloc));
2714 section->data = section->next_data = data;
2715 section->size = alloc_size;
2716 section->end_data = nursery_real_end;
2717 scan_starts = (alloc_size + SCAN_START_SIZE - 1) / SCAN_START_SIZE;
2718 section->scan_starts = get_internal_mem (sizeof (char*) * scan_starts, INTERNAL_MEM_SCAN_STARTS);
2719 section->num_scan_start = scan_starts;
2720 section->block.role = MEMORY_ROLE_GEN0;
2721 section->block.next = NULL;
2723 nursery_section = section;
2725 /* Setup the single first large fragment */
2726 frag = alloc_fragment ();
2727 frag->fragment_start = nursery_start;
2728 frag->fragment_limit = nursery_start;
2729 frag->fragment_end = nursery_real_end;
2730 nursery_frag_real_end = nursery_real_end;
2731 /* FIXME: frag here is lost */
2735 scan_finalizer_entries (CopyOrMarkObjectFunc copy_func, FinalizeEntry *list, GrayQueue *queue)
2739 for (fin = list; fin; fin = fin->next) {
2742 DEBUG (5, fprintf (gc_debug_file, "Scan of fin ready object: %p (%s)\n", fin->object, safe_name (fin->object)));
2743 copy_func (&fin->object, queue);
2747 static mword fragment_total = 0;
2749 * We found a fragment of free memory in the nursery: memzero it and if
2750 * it is big enough, add it to the list of fragments that can be used for
2754 add_nursery_frag (size_t frag_size, char* frag_start, char* frag_end)
2757 DEBUG (4, fprintf (gc_debug_file, "Found empty fragment: %p-%p, size: %zd\n", frag_start, frag_end, frag_size));
2758 binary_protocol_empty (frag_start, frag_size);
2759 /* memsetting just the first chunk start is bound to provide better cache locality */
2760 if (nursery_clear_policy == CLEAR_AT_GC)
2761 memset (frag_start, 0, frag_size);
2762 /* Not worth dealing with smaller fragments: need to tune */
2763 if (frag_size >= FRAGMENT_MIN_SIZE) {
2764 fragment = alloc_fragment ();
2765 fragment->fragment_start = frag_start;
2766 fragment->fragment_limit = frag_start;
2767 fragment->fragment_end = frag_end;
2768 fragment->next = nursery_fragments;
2769 nursery_fragments = fragment;
2770 fragment_total += frag_size;
2772 /* Clear unused fragments, pinning depends on this */
2773 /*TODO place an int[] here instead of the memset if size justify it*/
2774 memset (frag_start, 0, frag_size);
2779 generation_name (int generation)
2781 switch (generation) {
2782 case GENERATION_NURSERY: return "nursery";
2783 case GENERATION_OLD: return "old";
2784 default: g_assert_not_reached ();
2788 static DisappearingLinkHashTable*
2789 get_dislink_hash_table (int generation)
2791 switch (generation) {
2792 case GENERATION_NURSERY: return &minor_disappearing_link_hash;
2793 case GENERATION_OLD: return &major_disappearing_link_hash;
2794 default: g_assert_not_reached ();
2798 static FinalizeEntryHashTable*
2799 get_finalize_entry_hash_table (int generation)
2801 switch (generation) {
2802 case GENERATION_NURSERY: return &minor_finalizable_hash;
2803 case GENERATION_OLD: return &major_finalizable_hash;
2804 default: g_assert_not_reached ();
2809 finish_gray_stack (char *start_addr, char *end_addr, int generation, GrayQueue *queue)
2814 int ephemeron_rounds = 0;
2815 CopyOrMarkObjectFunc copy_func = current_collection_generation == GENERATION_NURSERY ? copy_object : major_copy_or_mark_object;
2818 * We copied all the reachable objects. Now it's the time to copy
2819 * the objects that were not referenced by the roots, but by the copied objects.
2820 * we built a stack of objects pointed to by gray_start: they are
2821 * additional roots and we may add more items as we go.
2822 * We loop until gray_start == gray_objects which means no more objects have
2823 * been added. Note this is iterative: no recursion is involved.
2824 * We need to walk the LO list as well in search of marked big objects
2825 * (use a flag since this is needed only on major collections). We need to loop
2826 * here as well, so keep a counter of marked LO (increasing it in copy_object).
2827 * To achieve better cache locality and cache usage, we drain the gray stack
2828 * frequently, after each object is copied, and just finish the work here.
2830 drain_gray_stack (queue);
2832 DEBUG (2, fprintf (gc_debug_file, "%s generation done\n", generation_name (generation)));
2833 /* walk the finalization queue and move also the objects that need to be
2834 * finalized: use the finalized objects as new roots so the objects they depend
2835 * on are also not reclaimed. As with the roots above, only objects in the nursery
2836 * are marked/copied.
2837 * We need a loop here, since objects ready for finalizers may reference other objects
2838 * that are fin-ready. Speedup with a flag?
2842 * Walk the ephemeron tables marking all values with reachable keys. This must be completely done
2843 * before processing finalizable objects to avoid finalizing reachable values.
2845 * It must be done inside the finalizaters loop since objects must not be removed from CWT tables
2846 * while they are been finalized.
2848 int done_with_ephemerons = 0;
2850 done_with_ephemerons = mark_ephemerons_in_range (copy_func, start_addr, end_addr, queue);
2851 drain_gray_stack (queue);
2853 } while (!done_with_ephemerons);
2855 fin_ready = num_ready_finalizers;
2856 finalize_in_range (copy_func, start_addr, end_addr, generation, queue);
2857 if (generation == GENERATION_OLD)
2858 finalize_in_range (copy_func, nursery_start, nursery_real_end, GENERATION_NURSERY, queue);
2860 /* drain the new stack that might have been created */
2861 DEBUG (6, fprintf (gc_debug_file, "Precise scan of gray area post fin\n"));
2862 drain_gray_stack (queue);
2863 } while (fin_ready != num_ready_finalizers);
2866 * Clear ephemeron pairs with unreachable keys.
2867 * We pass the copy func so we can figure out if an array was promoted or not.
2869 clear_unreachable_ephemerons (copy_func, start_addr, end_addr, queue);
2872 DEBUG (2, fprintf (gc_debug_file, "Finalize queue handling scan for %s generation: %d usecs %d ephemeron roundss\n", generation_name (generation), TV_ELAPSED (atv, btv), ephemeron_rounds));
2875 * handle disappearing links
2876 * Note we do this after checking the finalization queue because if an object
2877 * survives (at least long enough to be finalized) we don't clear the link.
2878 * This also deals with a possible issue with the monitor reclamation: with the Boehm
2879 * GC a finalized object my lose the monitor because it is cleared before the finalizer is
2882 g_assert (gray_object_queue_is_empty (queue));
2884 null_link_in_range (copy_func, start_addr, end_addr, generation, queue);
2885 if (generation == GENERATION_OLD)
2886 null_link_in_range (copy_func, start_addr, end_addr, GENERATION_NURSERY, queue);
2887 if (gray_object_queue_is_empty (queue))
2889 drain_gray_stack (queue);
2892 g_assert (gray_object_queue_is_empty (queue));
2896 check_section_scan_starts (GCMemSection *section)
2899 for (i = 0; i < section->num_scan_start; ++i) {
2900 if (section->scan_starts [i]) {
2901 guint size = safe_object_get_size ((MonoObject*) section->scan_starts [i]);
2902 g_assert (size >= sizeof (MonoObject) && size <= MAX_SMALL_OBJ_SIZE);
2908 check_scan_starts (void)
2910 if (!do_scan_starts_check)
2912 check_section_scan_starts (nursery_section);
2913 major_check_scan_starts ();
2916 static int last_num_pinned = 0;
2919 build_nursery_fragments (int start_pin, int end_pin)
2921 char *frag_start, *frag_end;
2925 while (nursery_fragments) {
2926 Fragment *next = nursery_fragments->next;
2927 nursery_fragments->next = fragment_freelist;
2928 fragment_freelist = nursery_fragments;
2929 nursery_fragments = next;
2931 frag_start = nursery_start;
2933 /* clear scan starts */
2934 memset (nursery_section->scan_starts, 0, nursery_section->num_scan_start * sizeof (gpointer));
2935 for (i = start_pin; i < end_pin; ++i) {
2936 frag_end = pin_queue [i];
2937 /* remove the pin bit from pinned objects */
2938 unpin_object (frag_end);
2939 nursery_section->scan_starts [((char*)frag_end - (char*)nursery_section->data)/SCAN_START_SIZE] = frag_end;
2940 frag_size = frag_end - frag_start;
2942 add_nursery_frag (frag_size, frag_start, frag_end);
2943 frag_size = ALIGN_UP (safe_object_get_size ((MonoObject*)pin_queue [i]));
2944 frag_start = (char*)pin_queue [i] + frag_size;
2946 nursery_last_pinned_end = frag_start;
2947 frag_end = nursery_real_end;
2948 frag_size = frag_end - frag_start;
2950 add_nursery_frag (frag_size, frag_start, frag_end);
2951 if (!nursery_fragments) {
2952 DEBUG (1, fprintf (gc_debug_file, "Nursery fully pinned (%d)\n", end_pin - start_pin));
2953 for (i = start_pin; i < end_pin; ++i) {
2954 DEBUG (3, fprintf (gc_debug_file, "Bastard pinning obj %p (%s), size: %d\n", pin_queue [i], safe_name (pin_queue [i]), safe_object_get_size (pin_queue [i])));
2959 nursery_next = nursery_frag_real_end = NULL;
2961 /* Clear TLABs for all threads */
2966 scan_from_registered_roots (CopyOrMarkObjectFunc copy_func, char *addr_start, char *addr_end, int root_type, GrayQueue *queue)
2970 for (i = 0; i < roots_hash_size [root_type]; ++i) {
2971 for (root = roots_hash [root_type][i]; root; root = root->next) {
2972 DEBUG (6, fprintf (gc_debug_file, "Precise root scan %p-%p (desc: %p)\n", root->start_root, root->end_root, (void*)root->root_desc));
2973 precisely_scan_objects_from (copy_func, (void**)root->start_root, (void**)root->end_root, addr_start, addr_end, root->root_desc, queue);
2979 dump_occupied (char *start, char *end, char *section_start)
2981 fprintf (heap_dump_file, "<occupied offset=\"%td\" size=\"%td\"/>\n", start - section_start, end - start);
2985 dump_section (GCMemSection *section, const char *type)
2987 char *start = section->data;
2988 char *end = section->data + section->size;
2989 char *occ_start = NULL;
2991 char *old_start = NULL; /* just for debugging */
2993 fprintf (heap_dump_file, "<section type=\"%s\" size=\"%lu\">\n", type, (unsigned long)section->size);
2995 while (start < end) {
2999 if (!*(void**)start) {
3001 dump_occupied (occ_start, start, section->data);
3004 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
3007 g_assert (start < section->next_data);
3012 vt = (GCVTable*)LOAD_VTABLE (start);
3015 size = ALIGN_UP (safe_object_get_size ((MonoObject*) start));
3018 fprintf (heap_dump_file, "<object offset=\"%d\" class=\"%s.%s\" size=\"%d\"/>\n",
3019 start - section->data,
3020 vt->klass->name_space, vt->klass->name,
3028 dump_occupied (occ_start, start, section->data);
3030 fprintf (heap_dump_file, "</section>\n");
3034 dump_object (MonoObject *obj, gboolean dump_location)
3036 static char class_name [1024];
3038 MonoClass *class = mono_object_class (obj);
3042 * Python's XML parser is too stupid to parse angle brackets
3043 * in strings, so we just ignore them;
3046 while (class->name [i] && j < sizeof (class_name) - 1) {
3047 if (!strchr ("<>\"", class->name [i]))
3048 class_name [j++] = class->name [i];
3051 g_assert (j < sizeof (class_name));
3054 fprintf (heap_dump_file, "<object class=\"%s.%s\" size=\"%d\"",
3055 class->name_space, class_name,
3056 safe_object_get_size (obj));
3057 if (dump_location) {
3058 const char *location;
3059 if (ptr_in_nursery (obj))
3060 location = "nursery";
3061 else if (safe_object_get_size (obj) <= MAX_SMALL_OBJ_SIZE)
3065 fprintf (heap_dump_file, " location=\"%s\"", location);
3067 fprintf (heap_dump_file, "/>\n");
3071 dump_heap (const char *type, int num, const char *reason)
3073 static char const *internal_mem_names [] = { "pin-queue", "fragment", "section", "scan-starts",
3074 "fin-table", "finalize-entry", "dislink-table",
3075 "dislink", "roots-table", "root-record", "statistics",
3076 "remset", "gray-queue", "store-remset", "marksweep-tables",
3077 "marksweep-block-info", "ephemeron-link" };
3083 fprintf (heap_dump_file, "<collection type=\"%s\" num=\"%d\"", type, num);
3085 fprintf (heap_dump_file, " reason=\"%s\"", reason);
3086 fprintf (heap_dump_file, ">\n");
3087 fprintf (heap_dump_file, "<other-mem-usage type=\"pinned-chunks\" size=\"%lld\"/>\n", pinned_chunk_bytes_alloced);
3088 fprintf (heap_dump_file, "<other-mem-usage type=\"large-internal\" size=\"%lld\"/>\n", large_internal_bytes_alloced);
3089 fprintf (heap_dump_file, "<other-mem-usage type=\"mempools\" size=\"%ld\"/>\n", mono_mempool_get_bytes_allocated ());
3090 for (i = 0; i < INTERNAL_MEM_MAX; ++i)
3091 fprintf (heap_dump_file, "<other-mem-usage type=\"%s\" size=\"%ld\"/>\n", internal_mem_names [i], small_internal_mem_bytes [i]);
3092 fprintf (heap_dump_file, "<pinned type=\"stack\" bytes=\"%zu\"/>\n", pinned_byte_counts [PIN_TYPE_STACK]);
3093 /* fprintf (heap_dump_file, "<pinned type=\"static-data\" bytes=\"%d\"/>\n", pinned_byte_counts [PIN_TYPE_STATIC_DATA]); */
3094 fprintf (heap_dump_file, "<pinned type=\"other\" bytes=\"%zu\"/>\n", pinned_byte_counts [PIN_TYPE_OTHER]);
3096 fprintf (heap_dump_file, "<pinned-objects>\n");
3097 for (list = pinned_objects; list; list = list->next)
3098 dump_object (list->obj, TRUE);
3099 fprintf (heap_dump_file, "</pinned-objects>\n");
3101 dump_section (nursery_section, "nursery");
3105 fprintf (heap_dump_file, "<los>\n");
3106 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
3107 dump_object ((MonoObject*)bigobj->data, FALSE);
3108 fprintf (heap_dump_file, "</los>\n");
3110 fprintf (heap_dump_file, "</collection>\n");
3116 static gboolean inited = FALSE;
3121 mono_counters_register ("Minor fragment clear", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_pre_collection_fragment_clear);
3122 mono_counters_register ("Minor pinning", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_pinning);
3123 mono_counters_register ("Minor scan remsets", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_scan_remsets);
3124 mono_counters_register ("Minor scan pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_scan_pinned);
3125 mono_counters_register ("Minor scan registered roots", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_scan_registered_roots);
3126 mono_counters_register ("Minor scan thread data", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_scan_thread_data);
3127 mono_counters_register ("Minor finish gray stack", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_finish_gray_stack);
3128 mono_counters_register ("Minor fragment creation", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_fragment_creation);
3130 mono_counters_register ("Major fragment clear", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_pre_collection_fragment_clear);
3131 mono_counters_register ("Major pinning", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_pinning);
3132 mono_counters_register ("Major scan pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_pinned);
3133 mono_counters_register ("Major scan registered roots", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_registered_roots);
3134 mono_counters_register ("Major scan thread data", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_thread_data);
3135 mono_counters_register ("Major scan alloc_pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_alloc_pinned);
3136 mono_counters_register ("Major scan finalized", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_finalized);
3137 mono_counters_register ("Major scan big objects", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_big_objects);
3138 mono_counters_register ("Major finish gray stack", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_finish_gray_stack);
3139 mono_counters_register ("Major free big objects", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_free_bigobjs);
3140 mono_counters_register ("Major LOS sweep", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_los_sweep);
3141 mono_counters_register ("Major sweep", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_sweep);
3142 mono_counters_register ("Major fragment creation", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_fragment_creation);
3144 #ifdef HEAVY_STATISTICS
3145 mono_counters_register ("WBarrier set field", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_field);
3146 mono_counters_register ("WBarrier set arrayref", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_arrayref);
3147 mono_counters_register ("WBarrier arrayref copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_arrayref_copy);
3148 mono_counters_register ("WBarrier generic store called", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_generic_store);
3149 mono_counters_register ("WBarrier generic store stored", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_generic_store_remset);
3150 mono_counters_register ("WBarrier set root", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_root);
3151 mono_counters_register ("WBarrier value copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_value_copy);
3152 mono_counters_register ("WBarrier object copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_object_copy);
3154 mono_counters_register ("# objects allocated", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_alloced);
3155 mono_counters_register ("bytes allocated", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_bytes_alloced);
3156 mono_counters_register ("# objects allocated degraded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_alloced_degraded);
3157 mono_counters_register ("bytes allocated degraded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_bytes_alloced_degraded);
3158 mono_counters_register ("bytes allocated in LOS", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_bytes_alloced_los);
3160 mono_counters_register ("# copy_object() called (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_called_nursery);
3161 mono_counters_register ("# objects copied (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_copied_nursery);
3162 mono_counters_register ("# copy_object() called (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_called_major);
3163 mono_counters_register ("# objects copied (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_copied_major);
3165 mono_counters_register ("# scan_object() called (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_scan_object_called_nursery);
3166 mono_counters_register ("# scan_object() called (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_scan_object_called_major);
3168 mono_counters_register ("# nursery copy_object() failed from space", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_from_space);
3169 mono_counters_register ("# nursery copy_object() failed forwarded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_forwarded);
3170 mono_counters_register ("# nursery copy_object() failed pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_pinned);
3172 mono_counters_register ("# wasted fragments used", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_wasted_fragments_used);
3173 mono_counters_register ("bytes in wasted fragments", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_wasted_fragments_bytes);
3175 mono_counters_register ("Store remsets", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_store_remsets);
3176 mono_counters_register ("Unique store remsets", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_store_remsets_unique);
3177 mono_counters_register ("Saved remsets 1", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_saved_remsets_1);
3178 mono_counters_register ("Saved remsets 2", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_saved_remsets_2);
3179 mono_counters_register ("Global remsets added", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_added);
3180 mono_counters_register ("Global remsets re-added", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_readded);
3181 mono_counters_register ("Global remsets processed", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_processed);
3182 mono_counters_register ("Global remsets discarded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_discarded);
3190 need_major_collection (void)
3192 mword los_alloced = los_memory_usage - MIN (last_los_memory_usage, los_memory_usage);
3193 return minor_collection_sections_alloced * MAJOR_SECTION_SIZE + los_alloced > minor_collection_allowance;
3197 * Collect objects in the nursery. Returns whether to trigger a major
3201 collect_nursery (size_t requested_size)
3203 size_t max_garbage_amount;
3204 char *orig_nursery_next;
3205 TV_DECLARE (all_atv);
3206 TV_DECLARE (all_btv);
3210 current_collection_generation = GENERATION_NURSERY;
3213 binary_protocol_collection (GENERATION_NURSERY);
3214 check_scan_starts ();
3217 orig_nursery_next = nursery_next;
3218 nursery_next = MAX (nursery_next, nursery_last_pinned_end);
3219 /* FIXME: optimize later to use the higher address where an object can be present */
3220 nursery_next = MAX (nursery_next, nursery_real_end);
3222 DEBUG (1, fprintf (gc_debug_file, "Start nursery collection %d %p-%p, size: %d\n", num_minor_gcs, nursery_start, nursery_next, (int)(nursery_next - nursery_start)));
3223 max_garbage_amount = nursery_next - nursery_start;
3224 g_assert (nursery_section->size >= max_garbage_amount);
3226 /* world must be stopped already */
3227 TV_GETTIME (all_atv);
3230 /* Pinning depends on this */
3231 clear_nursery_fragments (orig_nursery_next);
3234 time_minor_pre_collection_fragment_clear += TV_ELAPSED_MS (atv, btv);
3237 check_for_xdomain_refs ();
3239 nursery_section->next_data = nursery_next;
3241 major_start_nursery_collection ();
3243 gray_object_queue_init (&gray_queue);
3246 mono_stats.minor_gc_count ++;
3248 global_remset_cache_clear ();
3250 /* pin from pinned handles */
3252 pin_from_roots (nursery_start, nursery_next);
3253 /* identify pinned objects */
3254 optimize_pin_queue (0);
3255 next_pin_slot = pin_objects_from_addresses (nursery_section, pin_queue, pin_queue + next_pin_slot, nursery_start, nursery_next, &gray_queue);
3256 nursery_section->pin_queue_start = 0;
3257 nursery_section->pin_queue_end = next_pin_slot;
3259 time_minor_pinning += TV_ELAPSED_MS (btv, atv);
3260 DEBUG (2, fprintf (gc_debug_file, "Finding pinned pointers: %d in %d usecs\n", next_pin_slot, TV_ELAPSED (btv, atv)));
3261 DEBUG (4, fprintf (gc_debug_file, "Start scan with %d pinned objects\n", next_pin_slot));
3263 if (consistency_check_at_minor_collection)
3264 check_consistency ();
3267 * walk all the roots and copy the young objects to the old generation,
3268 * starting from to_space
3271 scan_from_remsets (nursery_start, nursery_next, &gray_queue);
3272 /* we don't have complete write barrier yet, so we scan all the old generation sections */
3274 time_minor_scan_remsets += TV_ELAPSED_MS (atv, btv);
3275 DEBUG (2, fprintf (gc_debug_file, "Old generation scan: %d usecs\n", TV_ELAPSED (atv, btv)));
3277 drain_gray_stack (&gray_queue);
3280 time_minor_scan_pinned += TV_ELAPSED_MS (btv, atv);
3281 /* registered roots, this includes static fields */
3282 scan_from_registered_roots (copy_object, nursery_start, nursery_next, ROOT_TYPE_NORMAL, &gray_queue);
3283 scan_from_registered_roots (copy_object, nursery_start, nursery_next, ROOT_TYPE_WBARRIER, &gray_queue);
3285 time_minor_scan_registered_roots += TV_ELAPSED_MS (atv, btv);
3287 scan_thread_data (nursery_start, nursery_next, TRUE);
3289 time_minor_scan_thread_data += TV_ELAPSED_MS (btv, atv);
3292 finish_gray_stack (nursery_start, nursery_next, GENERATION_NURSERY, &gray_queue);
3294 time_minor_finish_gray_stack += TV_ELAPSED_MS (btv, atv);
3296 /* walk the pin_queue, build up the fragment list of free memory, unmark
3297 * pinned objects as we go, memzero() the empty fragments so they are ready for the
3300 build_nursery_fragments (0, next_pin_slot);
3302 time_minor_fragment_creation += TV_ELAPSED_MS (atv, btv);
3303 DEBUG (2, fprintf (gc_debug_file, "Fragment creation: %d usecs, %lu bytes available\n", TV_ELAPSED (atv, btv), (unsigned long)fragment_total));
3305 if (consistency_check_at_minor_collection)
3306 check_major_refs ();
3308 major_finish_nursery_collection ();
3310 TV_GETTIME (all_btv);
3311 mono_stats.minor_gc_time_usecs += TV_ELAPSED (all_atv, all_btv);
3314 dump_heap ("minor", num_minor_gcs - 1, NULL);
3316 /* prepare the pin queue for the next collection */
3317 last_num_pinned = next_pin_slot;
3319 if (fin_ready_list || critical_fin_list) {
3320 DEBUG (4, fprintf (gc_debug_file, "Finalizer-thread wakeup: ready %d\n", num_ready_finalizers));
3321 mono_gc_finalize_notify ();
3325 g_assert (gray_object_queue_is_empty (&gray_queue));
3327 check_scan_starts ();
3329 binary_protocol_flush_buffers ();
3331 current_collection_generation = -1;
3333 return need_major_collection ();
3337 major_do_collection (const char *reason)
3339 LOSObject *bigobj, *prevbo;
3340 TV_DECLARE (all_atv);
3341 TV_DECLARE (all_btv);
3344 /* FIXME: only use these values for the precise scan
3345 * note that to_space pointers should be excluded anyway...
3347 char *heap_start = NULL;
3348 char *heap_end = (char*)-1;
3349 int old_num_major_sections = num_major_sections;
3350 int num_major_sections_saved, save_target, allowance_target;
3351 mword los_memory_saved, los_memory_alloced, old_los_memory_usage;
3354 * A domain could have been freed, resulting in
3355 * los_memory_usage being less than last_los_memory_usage.
3357 los_memory_alloced = los_memory_usage - MIN (last_los_memory_usage, los_memory_usage);
3358 old_los_memory_usage = los_memory_usage;
3360 //count_ref_nonref_objs ();
3361 //consistency_check ();
3364 binary_protocol_collection (GENERATION_OLD);
3365 check_scan_starts ();
3366 gray_object_queue_init (&gray_queue);
3369 DEBUG (1, fprintf (gc_debug_file, "Start major collection %d\n", num_major_gcs));
3371 mono_stats.major_gc_count ++;
3373 /* world must be stopped already */
3374 TV_GETTIME (all_atv);
3377 /* Pinning depends on this */
3378 clear_nursery_fragments (nursery_next);
3381 time_major_pre_collection_fragment_clear += TV_ELAPSED_MS (atv, btv);
3384 check_for_xdomain_refs ();
3386 nursery_section->next_data = nursery_real_end;
3387 /* we should also coalesce scanning from sections close to each other
3388 * and deal with pointers outside of the sections later.
3390 /* The remsets are not useful for a major collection */
3392 global_remset_cache_clear ();
3396 DEBUG (6, fprintf (gc_debug_file, "Collecting pinned addresses\n"));
3397 pin_from_roots ((void*)lowest_heap_address, (void*)highest_heap_address);
3398 optimize_pin_queue (0);
3401 * pin_queue now contains all candidate pointers, sorted and
3402 * uniqued. We must do two passes now to figure out which
3403 * objects are pinned.
3405 * The first is to find within the pin_queue the area for each
3406 * section. This requires that the pin_queue be sorted. We
3407 * also process the LOS objects and pinned chunks here.
3409 * The second, destructive, pass is to reduce the section
3410 * areas to pointers to the actually pinned objects.
3412 DEBUG (6, fprintf (gc_debug_file, "Pinning from sections\n"));
3413 /* first pass for the sections */
3414 find_section_pin_queue_start_end (nursery_section);
3415 major_find_pin_queue_start_ends (&gray_queue);
3416 /* identify possible pointers to the insize of large objects */
3417 DEBUG (6, fprintf (gc_debug_file, "Pinning from large objects\n"));
3418 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
3420 find_optimized_pin_queue_area (bigobj->data, (char*)bigobj->data + bigobj->size, &start, &end);
3422 pin_object (bigobj->data);
3423 /* FIXME: only enqueue if object has references */
3424 GRAY_OBJECT_ENQUEUE (&gray_queue, bigobj->data);
3426 pin_stats_register_object ((char*) bigobj->data, safe_object_get_size ((MonoObject*) bigobj->data));
3427 DEBUG (6, fprintf (gc_debug_file, "Marked large object %p (%s) size: %lu from roots\n", bigobj->data, safe_name (bigobj->data), (unsigned long)bigobj->size));
3430 /* second pass for the sections */
3431 pin_objects_in_section (nursery_section, &gray_queue);
3432 major_pin_objects (&gray_queue);
3435 time_major_pinning += TV_ELAPSED_MS (atv, btv);
3436 DEBUG (2, fprintf (gc_debug_file, "Finding pinned pointers: %d in %d usecs\n", next_pin_slot, TV_ELAPSED (atv, btv)));
3437 DEBUG (4, fprintf (gc_debug_file, "Start scan with %d pinned objects\n", next_pin_slot));
3439 major_init_to_space ();
3441 drain_gray_stack (&gray_queue);
3444 time_major_scan_pinned += TV_ELAPSED_MS (btv, atv);
3446 /* registered roots, this includes static fields */
3447 scan_from_registered_roots (major_copy_or_mark_object, heap_start, heap_end, ROOT_TYPE_NORMAL, &gray_queue);
3448 scan_from_registered_roots (major_copy_or_mark_object, heap_start, heap_end, ROOT_TYPE_WBARRIER, &gray_queue);
3450 time_major_scan_registered_roots += TV_ELAPSED_MS (atv, btv);
3453 /* FIXME: This is the wrong place for this, because it does
3455 scan_thread_data (heap_start, heap_end, TRUE);
3457 time_major_scan_thread_data += TV_ELAPSED_MS (btv, atv);
3460 time_major_scan_alloc_pinned += TV_ELAPSED_MS (atv, btv);
3462 /* scan the list of objects ready for finalization */
3463 scan_finalizer_entries (major_copy_or_mark_object, fin_ready_list, &gray_queue);
3464 scan_finalizer_entries (major_copy_or_mark_object, critical_fin_list, &gray_queue);
3466 time_major_scan_finalized += TV_ELAPSED_MS (btv, atv);
3467 DEBUG (2, fprintf (gc_debug_file, "Root scan: %d usecs\n", TV_ELAPSED (btv, atv)));
3470 time_major_scan_big_objects += TV_ELAPSED_MS (atv, btv);
3472 /* all the objects in the heap */
3473 finish_gray_stack (heap_start, heap_end, GENERATION_OLD, &gray_queue);
3475 time_major_finish_gray_stack += TV_ELAPSED_MS (btv, atv);
3477 /* sweep the big objects list */
3479 for (bigobj = los_object_list; bigobj;) {
3480 if (object_is_pinned (bigobj->data)) {
3481 unpin_object (bigobj->data);
3484 /* not referenced anywhere, so we can free it */
3486 prevbo->next = bigobj->next;
3488 los_object_list = bigobj->next;
3490 bigobj = bigobj->next;
3491 free_large_object (to_free);
3495 bigobj = bigobj->next;
3499 time_major_free_bigobjs += TV_ELAPSED_MS (atv, btv);
3504 time_major_los_sweep += TV_ELAPSED_MS (btv, atv);
3509 time_major_sweep += TV_ELAPSED_MS (atv, btv);
3511 /* walk the pin_queue, build up the fragment list of free memory, unmark
3512 * pinned objects as we go, memzero() the empty fragments so they are ready for the
3515 build_nursery_fragments (nursery_section->pin_queue_start, nursery_section->pin_queue_end);
3518 time_major_fragment_creation += TV_ELAPSED_MS (btv, atv);
3520 TV_GETTIME (all_btv);
3521 mono_stats.major_gc_time_usecs += TV_ELAPSED (all_atv, all_btv);
3524 dump_heap ("major", num_major_gcs - 1, reason);
3526 /* prepare the pin queue for the next collection */
3528 if (fin_ready_list || critical_fin_list) {
3529 DEBUG (4, fprintf (gc_debug_file, "Finalizer-thread wakeup: ready %d\n", num_ready_finalizers));
3530 mono_gc_finalize_notify ();
3534 g_assert (gray_object_queue_is_empty (&gray_queue));
3536 num_major_sections_saved = MAX (old_num_major_sections - num_major_sections, 0);
3537 los_memory_saved = MAX (old_los_memory_usage - los_memory_usage, 1);
3539 save_target = ((num_major_sections * MAJOR_SECTION_SIZE) + los_memory_saved) / 2;
3541 * We aim to allow the allocation of as many sections as is
3542 * necessary to reclaim save_target sections in the next
3543 * collection. We assume the collection pattern won't change.
3544 * In the last cycle, we had num_major_sections_saved for
3545 * minor_collection_sections_alloced. Assuming things won't
3546 * change, this must be the same ratio as save_target for
3547 * allowance_target, i.e.
3549 * num_major_sections_saved save_target
3550 * --------------------------------- == ----------------
3551 * minor_collection_sections_alloced allowance_target
3555 allowance_target = (mword)((double)save_target * (double)(minor_collection_sections_alloced * MAJOR_SECTION_SIZE + los_memory_alloced) / (double)(num_major_sections_saved * MAJOR_SECTION_SIZE + los_memory_saved));
3557 minor_collection_allowance = MAX (MIN (allowance_target, num_major_sections * MAJOR_SECTION_SIZE + los_memory_usage), MIN_MINOR_COLLECTION_ALLOWANCE);
3559 minor_collection_sections_alloced = 0;
3560 last_los_memory_usage = los_memory_usage;
3562 major_finish_major_collection ();
3564 check_scan_starts ();
3566 binary_protocol_flush_buffers ();
3568 //consistency_check ();
3572 major_collection (const char *reason)
3574 if (g_getenv ("MONO_GC_NO_MAJOR")) {
3575 collect_nursery (0);
3579 current_collection_generation = GENERATION_OLD;
3580 major_do_collection (reason);
3581 current_collection_generation = -1;
3585 * When deciding if it's better to collect or to expand, keep track
3586 * of how much garbage was reclaimed with the last collection: if it's too
3588 * This is called when we could not allocate a small object.
3590 static void __attribute__((noinline))
3591 minor_collect_or_expand_inner (size_t size)
3593 int do_minor_collection = 1;
3595 if (!nursery_section) {
3599 if (do_minor_collection) {
3601 if (collect_nursery (size))
3602 major_collection ("minor overflow");
3603 DEBUG (2, fprintf (gc_debug_file, "Heap size: %lu, LOS size: %lu\n", (unsigned long)total_alloc, (unsigned long)los_memory_usage));
3605 /* this also sets the proper pointers for the next allocation */
3606 if (!search_fragment_for_size (size)) {
3608 /* TypeBuilder and MonoMethod are killing mcs with fragmentation */
3609 DEBUG (1, fprintf (gc_debug_file, "nursery collection didn't find enough room for %zd alloc (%d pinned)\n", size, last_num_pinned));
3610 for (i = 0; i < last_num_pinned; ++i) {
3611 DEBUG (3, fprintf (gc_debug_file, "Bastard pinning obj %p (%s), size: %d\n", pin_queue [i], safe_name (pin_queue [i]), safe_object_get_size (pin_queue [i])));
3616 //report_internal_mem_usage ();
3620 * ######################################################################
3621 * ######## Memory allocation from the OS
3622 * ######################################################################
3623 * This section of code deals with getting memory from the OS and
3624 * allocating memory for GC-internal data structures.
3625 * Internal memory can be handled with a freelist for small objects.
3629 * Allocate a big chunk of memory from the OS (usually 64KB to several megabytes).
3630 * This must not require any lock.
3633 get_os_memory (size_t size, int activate)
3636 unsigned long prot_flags = activate? MONO_MMAP_READ|MONO_MMAP_WRITE: MONO_MMAP_NONE;
3638 prot_flags |= MONO_MMAP_PRIVATE | MONO_MMAP_ANON;
3639 size += pagesize - 1;
3640 size &= ~(pagesize - 1);
3641 ptr = mono_valloc (0, size, prot_flags);
3646 * Free the memory returned by get_os_memory (), returning it to the OS.
3649 free_os_memory (void *addr, size_t size)
3651 mono_vfree (addr, size);
3658 report_pinned_chunk (PinnedChunk *chunk, int seq) {
3660 int i, free_pages, num_free, free_mem;
3662 for (i = 0; i < chunk->num_pages; ++i) {
3663 if (!chunk->page_sizes [i])
3666 printf ("Pinned chunk %d at %p, size: %d, pages: %d, free: %d\n", seq, chunk, chunk->num_pages * FREELIST_PAGESIZE, chunk->num_pages, free_pages);
3667 free_mem = FREELIST_PAGESIZE * free_pages;
3668 for (i = 0; i < FREELIST_NUM_SLOTS; ++i) {
3669 if (!chunk->free_list [i])
3672 p = chunk->free_list [i];
3677 printf ("\tfree list of size %d, %d items\n", freelist_sizes [i], num_free);
3678 free_mem += freelist_sizes [i] * num_free;
3680 printf ("\tfree memory in chunk: %d\n", free_mem);
3686 static G_GNUC_UNUSED void
3687 report_internal_mem_usage (void) {
3690 printf ("Internal memory usage:\n");
3692 for (chunk = internal_chunk_list; chunk; chunk = chunk->block.next) {
3693 report_pinned_chunk (chunk, i++);
3695 printf ("Pinned memory usage:\n");
3696 major_report_pinned_memory_usage ();
3700 * Find the slot number in the freelist for memory chunks that
3701 * can contain @size objects.
3704 slot_for_size (size_t size)
3707 /* do a binary search or lookup table later. */
3708 for (slot = 0; slot < FREELIST_NUM_SLOTS; ++slot) {
3709 if (freelist_sizes [slot] >= size)
3712 g_assert_not_reached ();
3717 * Build a free list for @size memory chunks from the memory area between
3718 * start_page and end_page.
3721 build_freelist (PinnedChunk *chunk, int slot, int size, char *start_page, char *end_page)
3725 /*g_print ("building freelist for slot %d, size %d in %p\n", slot, size, chunk);*/
3726 p = (void**)start_page;
3727 end = (void**)(end_page - size);
3728 g_assert (!chunk->free_list [slot]);
3729 chunk->free_list [slot] = p;
3730 while ((char*)p + size <= (char*)end) {
3732 *p = (void*)((char*)p + size);
3736 /*g_print ("%d items created, max: %d\n", count, (end_page - start_page) / size);*/
3740 alloc_pinned_chunk (void)
3744 int size = PINNED_CHUNK_SIZE;
3746 chunk = get_os_memory_aligned (size, size, TRUE);
3747 chunk->block.role = MEMORY_ROLE_PINNED;
3749 UPDATE_HEAP_BOUNDARIES (chunk, ((char*)chunk + size));
3750 total_alloc += size;
3751 pinned_chunk_bytes_alloced += size;
3753 /* setup the bookeeping fields */
3754 chunk->num_pages = size / FREELIST_PAGESIZE;
3755 offset = G_STRUCT_OFFSET (PinnedChunk, data);
3756 chunk->page_sizes = (void*)((char*)chunk + offset);
3757 offset += sizeof (int) * chunk->num_pages;
3758 offset = ALIGN_UP (offset);
3759 chunk->free_list = (void*)((char*)chunk + offset);
3760 offset += sizeof (void*) * FREELIST_NUM_SLOTS;
3761 offset = ALIGN_UP (offset);
3762 chunk->start_data = (void*)((char*)chunk + offset);
3764 /* allocate the first page to the freelist */
3765 chunk->page_sizes [0] = PINNED_FIRST_SLOT_SIZE;
3766 build_freelist (chunk, slot_for_size (PINNED_FIRST_SLOT_SIZE), PINNED_FIRST_SLOT_SIZE, chunk->start_data, ((char*)chunk + FREELIST_PAGESIZE));
3767 DEBUG (4, fprintf (gc_debug_file, "Allocated pinned chunk %p, size: %d\n", chunk, size));
3771 /* assumes freelist for slot is empty, so try to alloc a new page */
3773 get_chunk_freelist (PinnedChunk *chunk, int slot)
3777 p = chunk->free_list [slot];
3779 chunk->free_list [slot] = *p;
3782 for (i = 0; i < chunk->num_pages; ++i) {
3784 if (chunk->page_sizes [i])
3786 size = freelist_sizes [slot];
3787 chunk->page_sizes [i] = size;
3788 build_freelist (chunk, slot, size, (char*)chunk + FREELIST_PAGESIZE * i, (char*)chunk + FREELIST_PAGESIZE * (i + 1));
3792 p = chunk->free_list [slot];
3794 chunk->free_list [slot] = *p;
3800 /* used for the GC-internal data structures */
3802 get_internal_mem (size_t size, int type)
3806 PinnedChunk *pchunk;
3808 if (size > freelist_sizes [FREELIST_NUM_SLOTS - 1]) {
3809 LargeInternalMemHeader *mh;
3811 size += sizeof (LargeInternalMemHeader);
3812 mh = get_os_memory (size, TRUE);
3813 mh->magic = LARGE_INTERNAL_MEM_HEADER_MAGIC;
3816 large_internal_bytes_alloced += size;
3821 slot = slot_for_size (size);
3822 g_assert (size <= freelist_sizes [slot]);
3824 small_internal_mem_bytes [type] += freelist_sizes [slot];
3826 for (pchunk = internal_chunk_list; pchunk; pchunk = pchunk->block.next) {
3827 void **p = pchunk->free_list [slot];
3829 pchunk->free_list [slot] = *p;
3830 memset (p, 0, size);
3834 for (pchunk = internal_chunk_list; pchunk; pchunk = pchunk->block.next) {
3835 res = get_chunk_freelist (pchunk, slot);
3837 memset (res, 0, size);
3841 pchunk = alloc_pinned_chunk ();
3842 /* FIXME: handle OOM */
3843 pchunk->block.next = internal_chunk_list;
3844 internal_chunk_list = pchunk;
3845 res = get_chunk_freelist (pchunk, slot);
3846 memset (res, 0, size);
3851 free_internal_mem (void *addr, int type)
3853 PinnedChunk *pchunk;
3854 LargeInternalMemHeader *mh;
3857 for (pchunk = internal_chunk_list; pchunk; pchunk = pchunk->block.next) {
3858 /*printf ("trying to free %p in %p (pages: %d)\n", addr, pchunk, pchunk->num_pages);*/
3859 if (addr >= (void*)pchunk && (char*)addr < (char*)pchunk + pchunk->num_pages * FREELIST_PAGESIZE) {
3860 int offset = (char*)addr - (char*)pchunk;
3861 int page = offset / FREELIST_PAGESIZE;
3862 int slot = slot_for_size (pchunk->page_sizes [page]);
3864 *p = pchunk->free_list [slot];
3865 pchunk->free_list [slot] = p;
3867 small_internal_mem_bytes [type] -= freelist_sizes [slot];
3872 mh = (LargeInternalMemHeader*)((char*)addr - G_STRUCT_OFFSET (LargeInternalMemHeader, data));
3873 g_assert (mh->magic == LARGE_INTERNAL_MEM_HEADER_MAGIC);
3874 large_internal_bytes_alloced -= mh->size;
3875 free_os_memory (mh, mh->size);
3879 * ######################################################################
3880 * ######## Object allocation
3881 * ######################################################################
3882 * This section of code deals with allocating memory for objects.
3883 * There are several ways:
3884 * *) allocate large objects
3885 * *) allocate normal objects
3886 * *) fast lock-free allocation
3887 * *) allocation of pinned objects
3891 setup_fragment (Fragment *frag, Fragment *prev, size_t size)
3893 /* remove from the list */
3895 prev->next = frag->next;
3897 nursery_fragments = frag->next;
3898 nursery_next = frag->fragment_start;
3899 nursery_frag_real_end = frag->fragment_end;
3901 DEBUG (4, fprintf (gc_debug_file, "Using nursery fragment %p-%p, size: %td (req: %zd)\n", nursery_next, nursery_frag_real_end, nursery_frag_real_end - nursery_next, size));
3902 frag->next = fragment_freelist;
3903 fragment_freelist = frag;
3906 /* check if we have a suitable fragment in nursery_fragments to be able to allocate
3907 * an object of size @size
3908 * Return FALSE if not found (which means we need a collection)
3911 search_fragment_for_size (size_t size)
3913 Fragment *frag, *prev;
3914 DEBUG (4, fprintf (gc_debug_file, "Searching nursery fragment %p, size: %zd\n", nursery_frag_real_end, size));
3916 if (nursery_frag_real_end > nursery_next && nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
3917 /* Clear the remaining space, pinning depends on this */
3918 memset (nursery_next, 0, nursery_frag_real_end - nursery_next);
3921 for (frag = nursery_fragments; frag; frag = frag->next) {
3922 if (size <= (frag->fragment_end - frag->fragment_start)) {
3923 setup_fragment (frag, prev, size);
3932 * Same as search_fragment_for_size but if search for @desired_size fails, try to satisfy @minimum_size.
3933 * This improves nursery usage.
3936 search_fragment_for_size_range (size_t desired_size, size_t minimum_size)
3938 Fragment *frag, *prev, *min_prev;
3939 DEBUG (4, fprintf (gc_debug_file, "Searching nursery fragment %p, desired size: %zd minimum size %zd\n", nursery_frag_real_end, desired_size, minimum_size));
3941 if (nursery_frag_real_end > nursery_next && nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
3942 /* Clear the remaining space, pinning depends on this */
3943 memset (nursery_next, 0, nursery_frag_real_end - nursery_next);
3945 min_prev = GINT_TO_POINTER (-1);
3948 for (frag = nursery_fragments; frag; frag = frag->next) {
3949 int frag_size = frag->fragment_end - frag->fragment_start;
3950 if (desired_size <= frag_size) {
3951 setup_fragment (frag, prev, desired_size);
3952 return desired_size;
3954 if (minimum_size <= frag_size)
3960 if (min_prev != GINT_TO_POINTER (-1)) {
3963 frag = min_prev->next;
3965 frag = nursery_fragments;
3967 frag_size = frag->fragment_end - frag->fragment_start;
3968 HEAVY_STAT (++stat_wasted_fragments_used);
3969 HEAVY_STAT (stat_wasted_fragments_bytes += frag_size);
3971 setup_fragment (frag, min_prev, minimum_size);
3979 alloc_degraded (MonoVTable *vtable, size_t size)
3981 if (need_major_collection ()) {
3983 major_collection ("degraded overflow");
3987 return major_alloc_degraded (vtable, size);
3991 * Provide a variant that takes just the vtable for small fixed-size objects.
3992 * The aligned size is already computed and stored in vt->gc_descr.
3993 * Note: every SCAN_START_SIZE or so we are given the chance to do some special
3994 * processing. We can keep track of where objects start, for example,
3995 * so when we scan the thread stacks for pinned objects, we can start
3996 * a search for the pinned object in SCAN_START_SIZE chunks.
3999 mono_gc_alloc_obj_nolock (MonoVTable *vtable, size_t size)
4001 /* FIXME: handle OOM */
4006 HEAVY_STAT (++stat_objects_alloced);
4007 if (size <= MAX_SMALL_OBJ_SIZE)
4008 HEAVY_STAT (stat_bytes_alloced += size);
4010 HEAVY_STAT (stat_bytes_alloced_los += size);
4012 size = ALIGN_UP (size);
4014 g_assert (vtable->gc_descr);
4016 if (G_UNLIKELY (collect_before_allocs)) {
4017 if (nursery_section) {
4019 collect_nursery (0);
4021 if (!degraded_mode && !search_fragment_for_size (size)) {
4023 g_assert_not_reached ();
4029 * We must already have the lock here instead of after the
4030 * fast path because we might be interrupted in the fast path
4031 * (after confirming that new_next < TLAB_TEMP_END) by the GC,
4032 * and we'll end up allocating an object in a fragment which
4033 * no longer belongs to us.
4035 * The managed allocator does not do this, but it's treated
4036 * specially by the world-stopping code.
4039 if (size > MAX_SMALL_OBJ_SIZE) {
4040 p = alloc_large_inner (vtable, size);
4042 /* tlab_next and tlab_temp_end are TLS vars so accessing them might be expensive */
4044 p = (void**)TLAB_NEXT;
4045 /* FIXME: handle overflow */
4046 new_next = (char*)p + size;
4047 TLAB_NEXT = new_next;
4049 if (G_LIKELY (new_next < TLAB_TEMP_END)) {
4053 * FIXME: We might need a memory barrier here so the change to tlab_next is
4054 * visible before the vtable store.
4057 DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
4058 binary_protocol_alloc (p , vtable, size);
4059 g_assert (*p == NULL);
4062 g_assert (TLAB_NEXT == new_next);
4069 /* there are two cases: the object is too big or we run out of space in the TLAB */
4070 /* we also reach here when the thread does its first allocation after a minor
4071 * collection, since the tlab_ variables are initialized to NULL.
4072 * there can be another case (from ORP), if we cooperate with the runtime a bit:
4073 * objects that need finalizers can have the high bit set in their size
4074 * so the above check fails and we can readily add the object to the queue.
4075 * This avoids taking again the GC lock when registering, but this is moot when
4076 * doing thread-local allocation, so it may not be a good idea.
4078 g_assert (TLAB_NEXT == new_next);
4079 if (TLAB_NEXT >= TLAB_REAL_END) {
4081 * Run out of space in the TLAB. When this happens, some amount of space
4082 * remains in the TLAB, but not enough to satisfy the current allocation
4083 * request. Currently, we retire the TLAB in all cases, later we could
4084 * keep it if the remaining space is above a treshold, and satisfy the
4085 * allocation directly from the nursery.
4088 /* when running in degraded mode, we continue allocing that way
4089 * for a while, to decrease the number of useless nursery collections.
4091 if (degraded_mode && degraded_mode < DEFAULT_NURSERY_SIZE) {
4092 p = alloc_degraded (vtable, size);
4093 binary_protocol_alloc_degraded (p, vtable, size);
4097 /*FIXME This codepath is current deadcode since tlab_size > MAX_SMALL_OBJ_SIZE*/
4098 if (size > tlab_size) {
4099 /* Allocate directly from the nursery */
4100 if (nursery_next + size >= nursery_frag_real_end) {
4101 if (!search_fragment_for_size (size)) {
4102 minor_collect_or_expand_inner (size);
4103 if (degraded_mode) {
4104 p = alloc_degraded (vtable, size);
4105 binary_protocol_alloc_degraded (p, vtable, size);
4111 p = (void*)nursery_next;
4112 nursery_next += size;
4113 if (nursery_next > nursery_frag_real_end) {
4118 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
4119 memset (p, 0, size);
4121 int alloc_size = tlab_size;
4122 int available_in_nursery = nursery_frag_real_end - nursery_next;
4124 DEBUG (3, fprintf (gc_debug_file, "Retire TLAB: %p-%p [%ld]\n", TLAB_START, TLAB_REAL_END, (long)(TLAB_REAL_END - TLAB_NEXT - size)));
4126 if (alloc_size >= available_in_nursery) {
4127 if (available_in_nursery > MAX_NURSERY_TLAB_WASTE && available_in_nursery > size) {
4128 alloc_size = available_in_nursery;
4130 alloc_size = search_fragment_for_size_range (tlab_size, size);
4132 alloc_size = tlab_size;
4133 minor_collect_or_expand_inner (tlab_size);
4134 if (degraded_mode) {
4135 p = alloc_degraded (vtable, size);
4136 binary_protocol_alloc_degraded (p, vtable, size);
4143 /* Allocate a new TLAB from the current nursery fragment */
4144 TLAB_START = nursery_next;
4145 nursery_next += alloc_size;
4146 TLAB_NEXT = TLAB_START;
4147 TLAB_REAL_END = TLAB_START + alloc_size;
4148 TLAB_TEMP_END = TLAB_START + MIN (SCAN_START_SIZE, alloc_size);
4150 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
4151 memset (TLAB_START, 0, alloc_size);
4153 /* Allocate from the TLAB */
4154 p = (void*)TLAB_NEXT;
4156 g_assert (TLAB_NEXT <= TLAB_REAL_END);
4158 nursery_section->scan_starts [((char*)p - (char*)nursery_section->data)/SCAN_START_SIZE] = (char*)p;
4161 /* Reached tlab_temp_end */
4163 /* record the scan start so we can find pinned objects more easily */
4164 nursery_section->scan_starts [((char*)p - (char*)nursery_section->data)/SCAN_START_SIZE] = (char*)p;
4165 /* we just bump tlab_temp_end as well */
4166 TLAB_TEMP_END = MIN (TLAB_REAL_END, TLAB_NEXT + SCAN_START_SIZE);
4167 DEBUG (5, fprintf (gc_debug_file, "Expanding local alloc: %p-%p\n", TLAB_NEXT, TLAB_TEMP_END));
4171 DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
4172 binary_protocol_alloc (p, vtable, size);
4179 mono_gc_try_alloc_obj_nolock (MonoVTable *vtable, size_t size)
4185 size = ALIGN_UP (size);
4187 g_assert (vtable->gc_descr);
4188 if (size <= MAX_SMALL_OBJ_SIZE) {
4189 /* tlab_next and tlab_temp_end are TLS vars so accessing them might be expensive */
4191 p = (void**)TLAB_NEXT;
4192 /* FIXME: handle overflow */
4193 new_next = (char*)p + size;
4194 TLAB_NEXT = new_next;
4196 if (G_LIKELY (new_next < TLAB_TEMP_END)) {
4200 * FIXME: We might need a memory barrier here so the change to tlab_next is
4201 * visible before the vtable store.
4204 HEAVY_STAT (++stat_objects_alloced);
4205 HEAVY_STAT (stat_bytes_alloced += size);
4207 DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
4208 binary_protocol_alloc (p, vtable, size);
4209 g_assert (*p == NULL);
4212 g_assert (TLAB_NEXT == new_next);
4221 mono_gc_alloc_obj (MonoVTable *vtable, size_t size)
4224 #ifndef DISABLE_CRITICAL_REGION
4226 ENTER_CRITICAL_REGION;
4227 res = mono_gc_try_alloc_obj_nolock (vtable, size);
4229 EXIT_CRITICAL_REGION;
4232 EXIT_CRITICAL_REGION;
4235 res = mono_gc_alloc_obj_nolock (vtable, size);
4241 mono_gc_alloc_vector (MonoVTable *vtable, size_t size, uintptr_t max_length)
4244 #ifndef DISABLE_CRITICAL_REGION
4246 ENTER_CRITICAL_REGION;
4247 arr = mono_gc_try_alloc_obj_nolock (vtable, size);
4249 arr->max_length = max_length;
4250 EXIT_CRITICAL_REGION;
4253 EXIT_CRITICAL_REGION;
4258 arr = mono_gc_alloc_obj_nolock (vtable, size);
4259 arr->max_length = max_length;
4267 mono_gc_alloc_array (MonoVTable *vtable, size_t size, uintptr_t max_length, uintptr_t bounds_size)
4270 MonoArrayBounds *bounds;
4274 arr = mono_gc_alloc_obj_nolock (vtable, size);
4275 arr->max_length = max_length;
4277 bounds = (MonoArrayBounds*)((char*)arr + size - bounds_size);
4278 arr->bounds = bounds;
4286 mono_gc_alloc_string (MonoVTable *vtable, size_t size, gint32 len)
4289 #ifndef DISABLE_CRITICAL_REGION
4291 ENTER_CRITICAL_REGION;
4292 str = mono_gc_try_alloc_obj_nolock (vtable, size);
4295 EXIT_CRITICAL_REGION;
4298 EXIT_CRITICAL_REGION;
4303 str = mono_gc_alloc_obj_nolock (vtable, size);
4312 * To be used for interned strings and possibly MonoThread, reflection handles.
4313 * We may want to explicitly free these objects.
4316 mono_gc_alloc_pinned_obj (MonoVTable *vtable, size_t size)
4318 /* FIXME: handle OOM */
4320 size = ALIGN_UP (size);
4322 if (size > MAX_SMALL_OBJ_SIZE) {
4323 /* large objects are always pinned anyway */
4324 p = alloc_large_inner (vtable, size);
4326 DEBUG (9, g_assert (vtable->klass->inited));
4327 p = major_alloc_small_pinned_obj (size, vtable->klass->has_references);
4329 DEBUG (6, fprintf (gc_debug_file, "Allocated pinned object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
4330 binary_protocol_alloc_pinned (p, vtable, size);
4337 * ######################################################################
4338 * ######## Finalization support
4339 * ######################################################################
4343 * this is valid for the nursery: if the object has been forwarded it means it's
4344 * still refrenced from a root. If it is pinned it's still alive as well.
4345 * Return TRUE if @obj is ready to be finalized.
4347 #define object_is_fin_ready(obj) (!object_is_pinned (obj) && !object_is_forwarded (obj))
4350 is_critical_finalizer (FinalizeEntry *entry)
4355 if (!mono_defaults.critical_finalizer_object)
4358 obj = entry->object;
4359 class = ((MonoVTable*)LOAD_VTABLE (obj))->klass;
4361 return mono_class_has_parent (class, mono_defaults.critical_finalizer_object);
4365 queue_finalization_entry (FinalizeEntry *entry) {
4366 if (is_critical_finalizer (entry)) {
4367 entry->next = critical_fin_list;
4368 critical_fin_list = entry;
4370 entry->next = fin_ready_list;
4371 fin_ready_list = entry;
4375 /* LOCKING: requires that the GC lock is held */
4377 rehash_fin_table (FinalizeEntryHashTable *hash_table)
4379 FinalizeEntry **finalizable_hash = hash_table->table;
4380 mword finalizable_hash_size = hash_table->size;
4383 FinalizeEntry **new_hash;
4384 FinalizeEntry *entry, *next;
4385 int new_size = g_spaced_primes_closest (hash_table->num_registered);
4387 new_hash = get_internal_mem (new_size * sizeof (FinalizeEntry*), INTERNAL_MEM_FIN_TABLE);
4388 for (i = 0; i < finalizable_hash_size; ++i) {
4389 for (entry = finalizable_hash [i]; entry; entry = next) {
4390 hash = mono_object_hash (entry->object) % new_size;
4392 entry->next = new_hash [hash];
4393 new_hash [hash] = entry;
4396 free_internal_mem (finalizable_hash, INTERNAL_MEM_FIN_TABLE);
4397 hash_table->table = new_hash;
4398 hash_table->size = new_size;
4401 /* LOCKING: requires that the GC lock is held */
4403 rehash_fin_table_if_necessary (FinalizeEntryHashTable *hash_table)
4405 if (hash_table->num_registered >= hash_table->size * 2)
4406 rehash_fin_table (hash_table);
4409 /* LOCKING: requires that the GC lock is held */
4411 finalize_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, int generation, GrayQueue *queue)
4413 FinalizeEntryHashTable *hash_table = get_finalize_entry_hash_table (generation);
4414 FinalizeEntry *entry, *prev;
4416 FinalizeEntry **finalizable_hash = hash_table->table;
4417 mword finalizable_hash_size = hash_table->size;
4421 for (i = 0; i < finalizable_hash_size; ++i) {
4423 for (entry = finalizable_hash [i]; entry;) {
4424 if ((char*)entry->object >= start && (char*)entry->object < end && !major_is_object_live (entry->object)) {
4425 gboolean is_fin_ready = object_is_fin_ready (entry->object);
4426 char *copy = entry->object;
4427 copy_func ((void**)©, queue);
4430 FinalizeEntry *next;
4431 /* remove and put in fin_ready_list */
4433 prev->next = entry->next;
4435 finalizable_hash [i] = entry->next;
4437 num_ready_finalizers++;
4438 hash_table->num_registered--;
4439 queue_finalization_entry (entry);
4440 /* Make it survive */
4441 from = entry->object;
4442 entry->object = copy;
4443 DEBUG (5, fprintf (gc_debug_file, "Queueing object for finalization: %p (%s) (was at %p) (%d/%d)\n", entry->object, safe_name (entry->object), from, num_ready_finalizers, hash_table->num_registered));
4447 char *from = entry->object;
4448 if (hash_table == &minor_finalizable_hash && !ptr_in_nursery (copy)) {
4449 FinalizeEntry *next = entry->next;
4450 unsigned int major_hash;
4451 /* remove from the list */
4453 prev->next = entry->next;
4455 finalizable_hash [i] = entry->next;
4456 hash_table->num_registered--;
4458 entry->object = copy;
4460 /* insert it into the major hash */
4461 rehash_fin_table_if_necessary (&major_finalizable_hash);
4462 major_hash = mono_object_hash ((MonoObject*) copy) %
4463 major_finalizable_hash.size;
4464 entry->next = major_finalizable_hash.table [major_hash];
4465 major_finalizable_hash.table [major_hash] = entry;
4466 major_finalizable_hash.num_registered++;
4468 DEBUG (5, fprintf (gc_debug_file, "Promoting finalization of object %p (%s) (was at %p) to major table\n", copy, safe_name (copy), from));
4473 /* update pointer */
4474 DEBUG (5, fprintf (gc_debug_file, "Updating object for finalization: %p (%s) (was at %p)\n", entry->object, safe_name (entry->object), from));
4475 entry->object = copy;
4480 entry = entry->next;
4486 object_is_reachable (char *object, char *start, char *end)
4488 /*This happens for non nursery objects during minor collections. We just treat all objects as alive.*/
4489 if (object < start || object >= end)
4491 return !object_is_fin_ready (object) || major_is_object_live (object);
4494 /* LOCKING: requires that the GC lock is held */
4496 null_ephemerons_for_domain (MonoDomain *domain)
4498 EphemeronLinkNode *current = ephemeron_list, *prev = NULL;
4501 MonoObject *object = (MonoObject*)current->array;
4503 if (object && !object->vtable) {
4504 EphemeronLinkNode *tmp = current;
4507 prev->next = current->next;
4509 ephemeron_list = current->next;
4511 current = current->next;
4512 free_internal_mem (tmp, INTERNAL_MEM_EPHEMERON_LINK);
4515 current = current->next;
4520 /* LOCKING: requires that the GC lock is held */
4522 clear_unreachable_ephemerons (CopyOrMarkObjectFunc copy_func, char *start, char *end, GrayQueue *queue)
4524 int was_in_nursery, was_promoted;
4525 EphemeronLinkNode *current = ephemeron_list, *prev = NULL;
4527 Ephemeron *cur, *array_end;
4531 char *object = current->array;
4533 if (!object_is_reachable (object, start, end)) {
4534 EphemeronLinkNode *tmp = current;
4536 DEBUG (5, fprintf (gc_debug_file, "Dead Ephemeron array at %p\n", object));
4539 prev->next = current->next;
4541 ephemeron_list = current->next;
4543 current = current->next;
4544 free_internal_mem (tmp, INTERNAL_MEM_EPHEMERON_LINK);
4549 was_in_nursery = ptr_in_nursery (object);
4550 copy_func ((void**)&object, queue);
4551 current->array = object;
4553 /*The array was promoted, add global remsets for key/values left behind in nursery.*/
4554 was_promoted = was_in_nursery && !ptr_in_nursery (object);
4556 DEBUG (5, fprintf (gc_debug_file, "Clearing unreachable entries for ephemeron array at %p\n", object));
4558 array = (MonoArray*)object;
4559 cur = mono_array_addr (array, Ephemeron, 0);
4560 array_end = cur + mono_array_length_fast (array);
4561 tombstone = (char*)((MonoVTable*)LOAD_VTABLE (object))->domain->ephemeron_tombstone;
4563 for (; cur < array_end; ++cur) {
4564 char *key = (char*)cur->key;
4566 if (!key || key == tombstone)
4569 DEBUG (5, fprintf (gc_debug_file, "[%td] key %p (%s) value %p (%s)\n", cur - mono_array_addr (array, Ephemeron, 0),
4570 key, object_is_reachable (key, start, end) ? "reachable" : "unreachable",
4571 cur->value, cur->value && object_is_reachable (cur->value, start, end) ? "reachable" : "unreachable"));
4573 if (!object_is_reachable (key, start, end)) {
4574 cur->key = tombstone;
4580 if (ptr_in_nursery (key)) {/*key was not promoted*/
4581 DEBUG (5, fprintf (gc_debug_file, "\tAdded remset to key %p\n", key));
4582 add_to_global_remset (&cur->key);
4584 if (ptr_in_nursery (cur->value)) {/*value was not promoted*/
4585 DEBUG (5, fprintf (gc_debug_file, "\tAdded remset to value %p\n", cur->value));
4586 add_to_global_remset (&cur->value);
4591 current = current->next;
4595 /* LOCKING: requires that the GC lock is held */
4597 mark_ephemerons_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, GrayQueue *queue)
4599 int nothing_marked = 1;
4600 EphemeronLinkNode *current = ephemeron_list;
4602 Ephemeron *cur, *array_end;
4605 for (current = ephemeron_list; current; current = current->next) {
4606 char *object = current->array;
4607 DEBUG (5, fprintf (gc_debug_file, "Ephemeron array at %p\n", object));
4609 /*We ignore arrays in old gen during minor collections since all objects are promoted by the remset machinery.*/
4610 if (object < start || object >= end)
4613 /*It has to be alive*/
4614 if (!object_is_reachable (object, start, end)) {
4615 DEBUG (5, fprintf (gc_debug_file, "\tnot reachable\n"));
4619 copy_func ((void**)&object, queue);
4621 array = (MonoArray*)object;
4622 cur = mono_array_addr (array, Ephemeron, 0);
4623 array_end = cur + mono_array_length_fast (array);
4624 tombstone = (char*)((MonoVTable*)LOAD_VTABLE (object))->domain->ephemeron_tombstone;
4626 for (; cur < array_end; ++cur) {
4627 char *key = cur->key;
4629 if (!key || key == tombstone)
4632 DEBUG (5, fprintf (gc_debug_file, "[%td] key %p (%s) value %p (%s)\n", cur - mono_array_addr (array, Ephemeron, 0),
4633 key, object_is_reachable (key, start, end) ? "reachable" : "unreachable",
4634 cur->value, cur->value && object_is_reachable (cur->value, start, end) ? "reachable" : "unreachable"));
4636 if (object_is_reachable (key, start, end)) {
4637 char *value = cur->value;
4639 copy_func ((void**)&cur->key, queue);
4641 if (!object_is_reachable (value, start, end))
4643 copy_func ((void**)&cur->value, queue);
4649 DEBUG (5, fprintf (gc_debug_file, "Ephemeron run finished. Is it done %d\n", nothing_marked));
4650 return nothing_marked;
4653 /* LOCKING: requires that the GC lock is held */
4655 null_link_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, int generation, GrayQueue *queue)
4657 DisappearingLinkHashTable *hash = get_dislink_hash_table (generation);
4658 DisappearingLink **disappearing_link_hash = hash->table;
4659 int disappearing_link_hash_size = hash->size;
4660 DisappearingLink *entry, *prev;
4662 if (!hash->num_links)
4664 for (i = 0; i < disappearing_link_hash_size; ++i) {
4666 for (entry = disappearing_link_hash [i]; entry;) {
4667 char *object = DISLINK_OBJECT (entry);
4668 if (object >= start && object < end && !major_is_object_live (object)) {
4669 gboolean track = DISLINK_TRACK (entry);
4670 if (!track && object_is_fin_ready (object)) {
4671 void **p = entry->link;
4672 DisappearingLink *old;
4674 /* remove from list */
4676 prev->next = entry->next;
4678 disappearing_link_hash [i] = entry->next;
4679 DEBUG (5, fprintf (gc_debug_file, "Dislink nullified at %p to GCed object %p\n", p, object));
4681 free_internal_mem (entry, INTERNAL_MEM_DISLINK);
4686 char *copy = object;
4687 copy_func ((void**)©, queue);
4689 /* Update pointer if it's moved. If the object
4690 * has been moved out of the nursery, we need to
4691 * remove the link from the minor hash table to
4694 * FIXME: what if an object is moved earlier?
4697 if (hash == &minor_disappearing_link_hash && !ptr_in_nursery (copy)) {
4698 void **link = entry->link;
4699 DisappearingLink *old;
4700 /* remove from list */
4702 prev->next = entry->next;
4704 disappearing_link_hash [i] = entry->next;
4706 free_internal_mem (entry, INTERNAL_MEM_DISLINK);
4710 add_or_remove_disappearing_link ((MonoObject*)copy, link,
4711 track, GENERATION_OLD);
4713 DEBUG (5, fprintf (gc_debug_file, "Upgraded dislink at %p to major because object %p moved to %p\n", link, object, copy));
4717 /* We set the track resurrection bit to
4718 * FALSE if the object is to be finalized
4719 * so that the object can be collected in
4720 * the next cycle (i.e. after it was
4723 *entry->link = HIDE_POINTER (copy,
4724 object_is_fin_ready (object) ? FALSE : track);
4725 DEBUG (5, fprintf (gc_debug_file, "Updated dislink at %p to %p\n", entry->link, DISLINK_OBJECT (entry)));
4730 entry = entry->next;
4735 /* LOCKING: requires that the GC lock is held */
4737 null_links_for_domain (MonoDomain *domain, int generation)
4739 DisappearingLinkHashTable *hash = get_dislink_hash_table (generation);
4740 DisappearingLink **disappearing_link_hash = hash->table;
4741 int disappearing_link_hash_size = hash->size;
4742 DisappearingLink *entry, *prev;
4744 for (i = 0; i < disappearing_link_hash_size; ++i) {
4746 for (entry = disappearing_link_hash [i]; entry; ) {
4747 char *object = DISLINK_OBJECT (entry);
4748 if (object && !((MonoObject*)object)->vtable) {
4749 DisappearingLink *next = entry->next;
4754 disappearing_link_hash [i] = next;
4756 if (*(entry->link)) {
4757 *(entry->link) = NULL;
4758 g_warning ("Disappearing link %p not freed", entry->link);
4760 free_internal_mem (entry, INTERNAL_MEM_DISLINK);
4767 entry = entry->next;
4772 /* LOCKING: requires that the GC lock is held */
4774 finalizers_for_domain (MonoDomain *domain, MonoObject **out_array, int out_size,
4775 FinalizeEntryHashTable *hash_table)
4777 FinalizeEntry **finalizable_hash = hash_table->table;
4778 mword finalizable_hash_size = hash_table->size;
4779 FinalizeEntry *entry, *prev;
4782 if (no_finalize || !out_size || !out_array)
4785 for (i = 0; i < finalizable_hash_size; ++i) {
4787 for (entry = finalizable_hash [i]; entry;) {
4788 if (mono_object_domain (entry->object) == domain) {
4789 FinalizeEntry *next;
4790 /* remove and put in out_array */
4792 prev->next = entry->next;
4794 finalizable_hash [i] = entry->next;
4796 hash_table->num_registered--;
4797 out_array [count ++] = entry->object;
4798 DEBUG (5, fprintf (gc_debug_file, "Collecting object for finalization: %p (%s) (%d/%d)\n", entry->object, safe_name (entry->object), num_ready_finalizers, hash_table->num_registered));
4800 if (count == out_size)
4805 entry = entry->next;
4812 * mono_gc_finalizers_for_domain:
4813 * @domain: the unloading appdomain
4814 * @out_array: output array
4815 * @out_size: size of output array
4817 * Store inside @out_array up to @out_size objects that belong to the unloading
4818 * appdomain @domain. Returns the number of stored items. Can be called repeteadly
4819 * until it returns 0.
4820 * The items are removed from the finalizer data structure, so the caller is supposed
4822 * @out_array should be on the stack to allow the GC to know the objects are still alive.
4825 mono_gc_finalizers_for_domain (MonoDomain *domain, MonoObject **out_array, int out_size)
4830 result = finalizers_for_domain (domain, out_array, out_size, &minor_finalizable_hash);
4831 if (result < out_size) {
4832 result += finalizers_for_domain (domain, out_array + result, out_size - result,
4833 &major_finalizable_hash);
4841 register_for_finalization (MonoObject *obj, void *user_data, int generation)
4843 FinalizeEntryHashTable *hash_table = get_finalize_entry_hash_table (generation);
4844 FinalizeEntry **finalizable_hash;
4845 mword finalizable_hash_size;
4846 FinalizeEntry *entry, *prev;
4850 g_assert (user_data == NULL || user_data == mono_gc_run_finalize);
4851 hash = mono_object_hash (obj);
4853 rehash_fin_table_if_necessary (hash_table);
4854 finalizable_hash = hash_table->table;
4855 finalizable_hash_size = hash_table->size;
4856 hash %= finalizable_hash_size;
4858 for (entry = finalizable_hash [hash]; entry; entry = entry->next) {
4859 if (entry->object == obj) {
4861 /* remove from the list */
4863 prev->next = entry->next;
4865 finalizable_hash [hash] = entry->next;
4866 hash_table->num_registered--;
4867 DEBUG (5, fprintf (gc_debug_file, "Removed finalizer %p for object: %p (%s) (%d)\n", entry, obj, obj->vtable->klass->name, hash_table->num_registered));
4868 free_internal_mem (entry, INTERNAL_MEM_FINALIZE_ENTRY);
4876 /* request to deregister, but already out of the list */
4880 entry = get_internal_mem (sizeof (FinalizeEntry), INTERNAL_MEM_FINALIZE_ENTRY);
4881 entry->object = obj;
4882 entry->next = finalizable_hash [hash];
4883 finalizable_hash [hash] = entry;
4884 hash_table->num_registered++;
4885 DEBUG (5, fprintf (gc_debug_file, "Added finalizer %p for object: %p (%s) (%d) to %s table\n", entry, obj, obj->vtable->klass->name, hash_table->num_registered, generation_name (generation)));
4890 mono_gc_register_for_finalization (MonoObject *obj, void *user_data)
4892 if (ptr_in_nursery (obj))
4893 register_for_finalization (obj, user_data, GENERATION_NURSERY);
4895 register_for_finalization (obj, user_data, GENERATION_OLD);
4899 rehash_dislink (DisappearingLinkHashTable *hash_table)
4901 DisappearingLink **disappearing_link_hash = hash_table->table;
4902 int disappearing_link_hash_size = hash_table->size;
4905 DisappearingLink **new_hash;
4906 DisappearingLink *entry, *next;
4907 int new_size = g_spaced_primes_closest (hash_table->num_links);
4909 new_hash = get_internal_mem (new_size * sizeof (DisappearingLink*), INTERNAL_MEM_DISLINK_TABLE);
4910 for (i = 0; i < disappearing_link_hash_size; ++i) {
4911 for (entry = disappearing_link_hash [i]; entry; entry = next) {
4912 hash = mono_aligned_addr_hash (entry->link) % new_size;
4914 entry->next = new_hash [hash];
4915 new_hash [hash] = entry;
4918 free_internal_mem (disappearing_link_hash, INTERNAL_MEM_DISLINK_TABLE);
4919 hash_table->table = new_hash;
4920 hash_table->size = new_size;
4923 /* LOCKING: assumes the GC lock is held */
4925 add_or_remove_disappearing_link (MonoObject *obj, void **link, gboolean track, int generation)
4927 DisappearingLinkHashTable *hash_table = get_dislink_hash_table (generation);
4928 DisappearingLink *entry, *prev;
4930 DisappearingLink **disappearing_link_hash = hash_table->table;
4931 int disappearing_link_hash_size = hash_table->size;
4933 if (hash_table->num_links >= disappearing_link_hash_size * 2) {
4934 rehash_dislink (hash_table);
4935 disappearing_link_hash = hash_table->table;
4936 disappearing_link_hash_size = hash_table->size;
4938 /* FIXME: add check that link is not in the heap */
4939 hash = mono_aligned_addr_hash (link) % disappearing_link_hash_size;
4940 entry = disappearing_link_hash [hash];
4942 for (; entry; entry = entry->next) {
4943 /* link already added */
4944 if (link == entry->link) {
4945 /* NULL obj means remove */
4948 prev->next = entry->next;
4950 disappearing_link_hash [hash] = entry->next;
4951 hash_table->num_links--;
4952 DEBUG (5, fprintf (gc_debug_file, "Removed dislink %p (%d) from %s table\n", entry, hash_table->num_links, generation_name (generation)));
4953 free_internal_mem (entry, INTERNAL_MEM_DISLINK);
4956 *link = HIDE_POINTER (obj, track); /* we allow the change of object */
4964 entry = get_internal_mem (sizeof (DisappearingLink), INTERNAL_MEM_DISLINK);
4965 *link = HIDE_POINTER (obj, track);
4967 entry->next = disappearing_link_hash [hash];
4968 disappearing_link_hash [hash] = entry;
4969 hash_table->num_links++;
4970 DEBUG (5, fprintf (gc_debug_file, "Added dislink %p for object: %p (%s) at %p to %s table\n", entry, obj, obj->vtable->klass->name, link, generation_name (generation)));
4973 /* LOCKING: assumes the GC lock is held */
4975 mono_gc_register_disappearing_link (MonoObject *obj, void **link, gboolean track)
4977 add_or_remove_disappearing_link (NULL, link, FALSE, GENERATION_NURSERY);
4978 add_or_remove_disappearing_link (NULL, link, FALSE, GENERATION_OLD);
4980 if (ptr_in_nursery (obj))
4981 add_or_remove_disappearing_link (obj, link, track, GENERATION_NURSERY);
4983 add_or_remove_disappearing_link (obj, link, track, GENERATION_OLD);
4988 mono_gc_invoke_finalizers (void)
4990 FinalizeEntry *entry = NULL;
4991 gboolean entry_is_critical = FALSE;
4994 /* FIXME: batch to reduce lock contention */
4995 while (fin_ready_list || critical_fin_list) {
4999 FinalizeEntry **list = entry_is_critical ? &critical_fin_list : &fin_ready_list;
5001 /* We have finalized entry in the last
5002 interation, now we need to remove it from
5005 *list = entry->next;
5007 FinalizeEntry *e = *list;
5008 while (e->next != entry)
5010 e->next = entry->next;
5012 free_internal_mem (entry, INTERNAL_MEM_FINALIZE_ENTRY);
5016 /* Now look for the first non-null entry. */
5017 for (entry = fin_ready_list; entry && !entry->object; entry = entry->next)
5020 entry_is_critical = FALSE;
5022 entry_is_critical = TRUE;
5023 for (entry = critical_fin_list; entry && !entry->object; entry = entry->next)
5028 g_assert (entry->object);
5029 num_ready_finalizers--;
5030 obj = entry->object;
5031 entry->object = NULL;
5032 DEBUG (7, fprintf (gc_debug_file, "Finalizing object %p (%s)\n", obj, safe_name (obj)));
5040 g_assert (entry->object == NULL);
5042 /* the object is on the stack so it is pinned */
5043 /*g_print ("Calling finalizer for object: %p (%s)\n", entry->object, safe_name (entry->object));*/
5044 mono_gc_run_finalize (obj, NULL);
5051 mono_gc_pending_finalizers (void)
5053 return fin_ready_list || critical_fin_list;
5056 /* Negative value to remove */
5058 mono_gc_add_memory_pressure (gint64 value)
5060 /* FIXME: Use interlocked functions */
5062 memory_pressure += value;
5067 * ######################################################################
5068 * ######## registered roots support
5069 * ######################################################################
5073 rehash_roots (gboolean pinned)
5077 RootRecord **new_hash;
5078 RootRecord *entry, *next;
5081 new_size = g_spaced_primes_closest (num_roots_entries [pinned]);
5082 new_hash = get_internal_mem (new_size * sizeof (RootRecord*), INTERNAL_MEM_ROOTS_TABLE);
5083 for (i = 0; i < roots_hash_size [pinned]; ++i) {
5084 for (entry = roots_hash [pinned][i]; entry; entry = next) {
5085 hash = mono_aligned_addr_hash (entry->start_root) % new_size;
5087 entry->next = new_hash [hash];
5088 new_hash [hash] = entry;
5091 free_internal_mem (roots_hash [pinned], INTERNAL_MEM_ROOTS_TABLE);
5092 roots_hash [pinned] = new_hash;
5093 roots_hash_size [pinned] = new_size;
5097 find_root (int root_type, char *start, guint32 addr_hash)
5099 RootRecord *new_root;
5101 guint32 hash = addr_hash % roots_hash_size [root_type];
5102 for (new_root = roots_hash [root_type][hash]; new_root; new_root = new_root->next) {
5103 /* we allow changing the size and the descriptor (for thread statics etc) */
5104 if (new_root->start_root == start) {
5113 * We do not coalesce roots.
5116 mono_gc_register_root_inner (char *start, size_t size, void *descr, int root_type)
5118 RootRecord *new_root;
5119 unsigned int hash, addr_hash = mono_aligned_addr_hash (start);
5122 for (i = 0; i < ROOT_TYPE_NUM; ++i) {
5123 if (num_roots_entries [i] >= roots_hash_size [i] * 2)
5126 for (i = 0; i < ROOT_TYPE_NUM; ++i) {
5127 new_root = find_root (i, start, addr_hash);
5128 /* we allow changing the size and the descriptor (for thread statics etc) */
5130 size_t old_size = new_root->end_root - new_root->start_root;
5131 new_root->end_root = new_root->start_root + size;
5132 g_assert (((new_root->root_desc != 0) && (descr != NULL)) ||
5133 ((new_root->root_desc == 0) && (descr == NULL)));
5134 new_root->root_desc = (mword)descr;
5136 roots_size -= old_size;
5141 new_root = get_internal_mem (sizeof (RootRecord), INTERNAL_MEM_ROOT_RECORD);
5143 new_root->start_root = start;
5144 new_root->end_root = new_root->start_root + size;
5145 new_root->root_desc = (mword)descr;
5147 hash = addr_hash % roots_hash_size [root_type];
5148 num_roots_entries [root_type]++;
5149 new_root->next = roots_hash [root_type] [hash];
5150 roots_hash [root_type][hash] = new_root;
5151 DEBUG (3, fprintf (gc_debug_file, "Added root %p for range: %p-%p, descr: %p (%d/%d bytes)\n", new_root, new_root->start_root, new_root->end_root, descr, (int)size, (int)roots_size));
5161 mono_gc_register_root (char *start, size_t size, void *descr)
5163 return mono_gc_register_root_inner (start, size, descr, descr ? ROOT_TYPE_NORMAL : ROOT_TYPE_PINNED);
5167 mono_gc_register_root_wbarrier (char *start, size_t size, void *descr)
5169 return mono_gc_register_root_inner (start, size, descr, ROOT_TYPE_WBARRIER);
5173 mono_gc_deregister_root (char* addr)
5175 RootRecord *tmp, *prev;
5176 unsigned int hash, addr_hash = mono_aligned_addr_hash (addr);
5180 for (root_type = 0; root_type < ROOT_TYPE_NUM; ++root_type) {
5181 hash = addr_hash % roots_hash_size [root_type];
5182 tmp = roots_hash [root_type][hash];
5185 if (tmp->start_root == (char*)addr) {
5187 prev->next = tmp->next;
5189 roots_hash [root_type][hash] = tmp->next;
5190 roots_size -= (tmp->end_root - tmp->start_root);
5191 num_roots_entries [root_type]--;
5192 DEBUG (3, fprintf (gc_debug_file, "Removed root %p for range: %p-%p\n", tmp, tmp->start_root, tmp->end_root));
5193 free_internal_mem (tmp, INTERNAL_MEM_ROOT_RECORD);
5204 * ######################################################################
5205 * ######## Thread handling (stop/start code)
5206 * ######################################################################
5209 /* FIXME: handle large/small config */
5210 #define HASH_PTHREAD_T(id) (((unsigned int)(id) >> 4) * 2654435761u)
5212 static SgenThreadInfo* thread_table [THREAD_HASH_SIZE];
5214 #if USE_SIGNAL_BASED_START_STOP_WORLD
5216 static MonoSemType suspend_ack_semaphore;
5217 static MonoSemType *suspend_ack_semaphore_ptr;
5218 static unsigned int global_stop_count = 0;
5220 static sigset_t suspend_signal_mask;
5221 static mword cur_thread_regs [ARCH_NUM_REGS] = {0};
5223 /* LOCKING: assumes the GC lock is held */
5225 mono_sgen_get_thread_table (void)
5227 return thread_table;
5231 mono_sgen_thread_info_lookup (ARCH_THREAD_TYPE id)
5233 unsigned int hash = HASH_PTHREAD_T (id) % THREAD_HASH_SIZE;
5234 SgenThreadInfo *info;
5236 info = thread_table [hash];
5237 while (info && !ARCH_THREAD_EQUALS (info->id, id)) {
5244 update_current_thread_stack (void *start)
5246 void *ptr = cur_thread_regs;
5247 SgenThreadInfo *info = mono_sgen_thread_info_lookup (ARCH_GET_THREAD ());
5249 info->stack_start = align_pointer (&ptr);
5250 g_assert (info->stack_start >= info->stack_start_limit && info->stack_start < info->stack_end);
5251 ARCH_STORE_REGS (ptr);
5252 info->stopped_regs = ptr;
5253 if (gc_callbacks.thread_suspend_func)
5254 gc_callbacks.thread_suspend_func (info->runtime_data, NULL);
5258 * Define this and use the "xdomain-checks" MONO_GC_DEBUG option to
5259 * have cross-domain checks in the write barrier.
5261 //#define XDOMAIN_CHECKS_IN_WBARRIER
5263 #ifndef BINARY_PROTOCOL
5264 #ifndef HEAVY_STATISTICS
5265 #define MANAGED_ALLOCATION
5266 #ifndef XDOMAIN_CHECKS_IN_WBARRIER
5267 #define MANAGED_WBARRIER
5273 is_ip_in_managed_allocator (MonoDomain *domain, gpointer ip);
5276 mono_sgen_wait_for_suspend_ack (int count)
5280 for (i = 0; i < count; ++i) {
5281 while ((result = MONO_SEM_WAIT (suspend_ack_semaphore_ptr)) != 0) {
5282 if (errno != EINTR) {
5283 g_error ("sem_wait ()");
5290 restart_threads_until_none_in_managed_allocator (void)
5292 SgenThreadInfo *info;
5293 int i, result, num_threads_died = 0;
5294 int sleep_duration = -1;
5297 int restart_count = 0, restarted_count = 0;
5298 /* restart all threads that stopped in the
5300 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5301 for (info = thread_table [i]; info; info = info->next) {
5304 if (!info->stack_start || info->in_critical_region ||
5305 is_ip_in_managed_allocator (info->stopped_domain, info->stopped_ip)) {
5306 binary_protocol_thread_restart ((gpointer)info->id);
5307 #if defined(__MACH__) && MONO_MACH_ARCH_SUPPORTED
5308 result = thread_resume (pthread_mach_thread_np (info->id));
5310 result = pthread_kill (info->id, restart_signal_num);
5318 /* we set the stopped_ip to
5319 NULL for threads which
5320 we're not restarting so
5321 that we can easily identify
5323 info->stopped_ip = NULL;
5324 info->stopped_domain = NULL;
5328 /* if no threads were restarted, we're done */
5329 if (restart_count == 0)
5332 #if defined(__MACH__) && MONO_MACH_ARCH_SUPPORTED
5333 /* mach thread_resume is synchronous so we dont need to wait for them */
5335 /* wait for the threads to signal their restart */
5336 mono_sgen_wait_for_suspend_ack (restart_count);
5339 if (sleep_duration < 0) {
5343 g_usleep (sleep_duration);
5344 sleep_duration += 10;
5347 /* stop them again */
5348 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5349 for (info = thread_table [i]; info; info = info->next) {
5350 if (info->skip || info->stopped_ip == NULL)
5352 #if defined(__MACH__) && MONO_MACH_ARCH_SUPPORTED
5353 result = thread_suspend (pthread_mach_thread_np (info->id));
5355 result = pthread_kill (info->id, suspend_signal_num);
5364 /* some threads might have died */
5365 num_threads_died += restart_count - restarted_count;
5366 #if defined(__MACH__) && MONO_MACH_ARCH_SUPPORTED
5367 /* mach thread_resume is synchronous so we dont need to wait for them */
5369 /* wait for the threads to signal their suspension
5371 mono_sgen_wait_for_suspend_ack (restart_count);
5375 return num_threads_died;
5378 /* LOCKING: assumes the GC lock is held (by the stopping thread) */
5380 suspend_handler (int sig, siginfo_t *siginfo, void *context)
5382 SgenThreadInfo *info;
5385 int old_errno = errno;
5386 gpointer regs [ARCH_NUM_REGS];
5387 gpointer stack_start;
5389 id = pthread_self ();
5390 info = mono_sgen_thread_info_lookup (id);
5391 info->stopped_domain = mono_domain_get ();
5392 info->stopped_ip = (gpointer) ARCH_SIGCTX_IP (context);
5393 stop_count = global_stop_count;
5394 /* duplicate signal */
5395 if (0 && info->stop_count == stop_count) {
5399 #ifdef HAVE_KW_THREAD
5400 /* update the remset info in the thread data structure */
5401 info->remset = remembered_set;
5403 stack_start = (char*) ARCH_SIGCTX_SP (context) - REDZONE_SIZE;
5404 /* If stack_start is not within the limits, then don't set it
5405 in info and we will be restarted. */
5406 if (stack_start >= info->stack_start_limit && info->stack_start <= info->stack_end) {
5407 info->stack_start = stack_start;
5409 ARCH_COPY_SIGCTX_REGS (regs, context);
5410 info->stopped_regs = regs;
5412 g_assert (!info->stack_start);
5415 /* Notify the JIT */
5416 if (gc_callbacks.thread_suspend_func)
5417 gc_callbacks.thread_suspend_func (info->runtime_data, context);
5419 DEBUG (4, fprintf (gc_debug_file, "Posting suspend_ack_semaphore for suspend from %p %p\n", info, (gpointer)ARCH_GET_THREAD ()));
5420 /* notify the waiting thread */
5421 MONO_SEM_POST (suspend_ack_semaphore_ptr);
5422 info->stop_count = stop_count;
5424 /* wait until we receive the restart signal */
5427 sigsuspend (&suspend_signal_mask);
5428 } while (info->signal != restart_signal_num);
5430 DEBUG (4, fprintf (gc_debug_file, "Posting suspend_ack_semaphore for resume from %p %p\n", info, (gpointer)ARCH_GET_THREAD ()));
5431 /* notify the waiting thread */
5432 MONO_SEM_POST (suspend_ack_semaphore_ptr);
5438 restart_handler (int sig)
5440 SgenThreadInfo *info;
5441 int old_errno = errno;
5443 info = mono_sgen_thread_info_lookup (pthread_self ());
5444 info->signal = restart_signal_num;
5445 DEBUG (4, fprintf (gc_debug_file, "Restart handler in %p %p\n", info, (gpointer)ARCH_GET_THREAD ()));
5451 acquire_gc_locks (void)
5457 release_gc_locks (void)
5459 UNLOCK_INTERRUPTION;
5462 static TV_DECLARE (stop_world_time);
5463 static unsigned long max_pause_usec = 0;
5465 /* LOCKING: assumes the GC lock is held */
5471 acquire_gc_locks ();
5473 update_current_thread_stack (&count);
5475 global_stop_count++;
5476 DEBUG (3, fprintf (gc_debug_file, "stopping world n %d from %p %p\n", global_stop_count, mono_sgen_thread_info_lookup (ARCH_GET_THREAD ()), (gpointer)ARCH_GET_THREAD ()));
5477 TV_GETTIME (stop_world_time);
5478 count = mono_sgen_thread_handshake (suspend_signal_num);
5479 count -= restart_threads_until_none_in_managed_allocator ();
5480 g_assert (count >= 0);
5481 DEBUG (3, fprintf (gc_debug_file, "world stopped %d thread(s)\n", count));
5485 /* LOCKING: assumes the GC lock is held */
5487 restart_world (void)
5490 SgenThreadInfo *info;
5491 TV_DECLARE (end_sw);
5494 /* notify the profiler of the leftovers */
5495 if (G_UNLIKELY (mono_profiler_events & MONO_PROFILE_GC_MOVES)) {
5496 if (moved_objects_idx) {
5497 mono_profiler_gc_moves (moved_objects, moved_objects_idx);
5498 moved_objects_idx = 0;
5501 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5502 for (info = thread_table [i]; info; info = info->next) {
5503 info->stack_start = NULL;
5504 info->stopped_regs = NULL;
5508 release_gc_locks ();
5510 count = mono_sgen_thread_handshake (restart_signal_num);
5511 TV_GETTIME (end_sw);
5512 usec = TV_ELAPSED (stop_world_time, end_sw);
5513 max_pause_usec = MAX (usec, max_pause_usec);
5514 DEBUG (2, fprintf (gc_debug_file, "restarted %d thread(s) (pause time: %d usec, max: %d)\n", count, (int)usec, (int)max_pause_usec));
5518 #endif /* USE_SIGNAL_BASED_START_STOP_WORLD */
5521 mono_gc_set_gc_callbacks (MonoGCCallbacks *callbacks)
5523 gc_callbacks = *callbacks;
5527 mono_gc_get_gc_callbacks ()
5529 return &gc_callbacks;
5532 /* Variables holding start/end nursery so it won't have to be passed at every call */
5533 static void *scan_area_arg_start, *scan_area_arg_end;
5536 mono_gc_conservatively_scan_area (void *start, void *end)
5538 conservatively_pin_objects_from (start, end, scan_area_arg_start, scan_area_arg_end, PIN_TYPE_STACK);
5542 mono_gc_scan_object (void *obj)
5544 if (current_collection_generation == GENERATION_NURSERY)
5545 copy_object (&obj, &gray_queue);
5547 major_copy_or_mark_object (&obj, &gray_queue);
5552 * Mark from thread stacks and registers.
5555 scan_thread_data (void *start_nursery, void *end_nursery, gboolean precise)
5558 SgenThreadInfo *info;
5560 scan_area_arg_start = start_nursery;
5561 scan_area_arg_end = end_nursery;
5563 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5564 for (info = thread_table [i]; info; info = info->next) {
5566 DEBUG (3, fprintf (gc_debug_file, "Skipping dead thread %p, range: %p-%p, size: %td\n", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start));
5569 DEBUG (3, fprintf (gc_debug_file, "Scanning thread %p, range: %p-%p, size: %td, pinned=%d\n", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start, next_pin_slot));
5570 if (gc_callbacks.thread_mark_func && !conservative_stack_mark)
5571 gc_callbacks.thread_mark_func (info->runtime_data, info->stack_start, info->stack_end, precise);
5573 conservatively_pin_objects_from (info->stack_start, info->stack_end, start_nursery, end_nursery, PIN_TYPE_STACK);
5576 conservatively_pin_objects_from (info->stopped_regs, info->stopped_regs + ARCH_NUM_REGS,
5577 start_nursery, end_nursery, PIN_TYPE_STACK);
5583 find_pinning_ref_from_thread (char *obj, size_t size)
5586 SgenThreadInfo *info;
5587 char *endobj = obj + size;
5589 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5590 for (info = thread_table [i]; info; info = info->next) {
5591 char **start = (char**)info->stack_start;
5594 while (start < (char**)info->stack_end) {
5595 if (*start >= obj && *start < endobj) {
5596 DEBUG (0, fprintf (gc_debug_file, "Object %p referenced in thread %p (id %p) at %p, stack: %p-%p\n", obj, info, (gpointer)info->id, start, info->stack_start, info->stack_end));
5601 /* FIXME: check info->stopped_regs */
5607 ptr_on_stack (void *ptr)
5609 gpointer stack_start = &stack_start;
5610 SgenThreadInfo *info = mono_sgen_thread_info_lookup (ARCH_GET_THREAD ());
5612 if (ptr >= stack_start && ptr < (gpointer)info->stack_end)
5618 handle_remset (mword *p, void *start_nursery, void *end_nursery, gboolean global, GrayQueue *queue)
5625 HEAVY_STAT (++stat_global_remsets_processed);
5627 /* FIXME: exclude stack locations */
5628 switch ((*p) & REMSET_TYPE_MASK) {
5629 case REMSET_LOCATION:
5631 //__builtin_prefetch (ptr);
5632 if (((void*)ptr < start_nursery || (void*)ptr >= end_nursery)) {
5633 gpointer old = *ptr;
5634 copy_object (ptr, queue);
5635 DEBUG (9, fprintf (gc_debug_file, "Overwrote remset at %p with %p\n", ptr, *ptr));
5637 binary_protocol_ptr_update (ptr, old, *ptr, (gpointer)LOAD_VTABLE (*ptr), safe_object_get_size (*ptr));
5638 if (!global && *ptr >= start_nursery && *ptr < end_nursery) {
5640 * If the object is pinned, each reference to it from nonpinned objects
5641 * becomes part of the global remset, which can grow very large.
5643 DEBUG (9, fprintf (gc_debug_file, "Add to global remset because of pinning %p (%p %s)\n", ptr, *ptr, safe_name (*ptr)));
5644 add_to_global_remset (ptr);
5647 DEBUG (9, fprintf (gc_debug_file, "Skipping remset at %p holding %p\n", ptr, *ptr));
5651 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
5652 if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery))
5655 while (count-- > 0) {
5656 copy_object (ptr, queue);
5657 DEBUG (9, fprintf (gc_debug_file, "Overwrote remset at %p with %p (count: %d)\n", ptr, *ptr, (int)count));
5658 if (!global && *ptr >= start_nursery && *ptr < end_nursery)
5659 add_to_global_remset (ptr);
5664 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
5665 if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery))
5667 scan_object ((char*)ptr, queue);
5669 case REMSET_VTYPE: {
5670 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
5671 if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery))
5676 ptr = (void**) scan_vtype ((char*)ptr, desc, start_nursery, end_nursery, queue);
5680 g_assert_not_reached ();
5685 #ifdef HEAVY_STATISTICS
5687 collect_store_remsets (RememberedSet *remset, mword *bumper)
5689 mword *p = remset->data;
5694 while (p < remset->store_next) {
5695 switch ((*p) & REMSET_TYPE_MASK) {
5696 case REMSET_LOCATION:
5699 ++stat_saved_remsets_1;
5701 if (*p == last1 || *p == last2) {
5702 ++stat_saved_remsets_2;
5719 g_assert_not_reached ();
5729 RememberedSet *remset;
5731 SgenThreadInfo *info;
5733 mword *addresses, *bumper, *p, *r;
5735 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5736 for (info = thread_table [i]; info; info = info->next) {
5737 for (remset = info->remset; remset; remset = remset->next)
5738 size += remset->store_next - remset->data;
5741 for (remset = freed_thread_remsets; remset; remset = remset->next)
5742 size += remset->store_next - remset->data;
5743 for (remset = global_remset; remset; remset = remset->next)
5744 size += remset->store_next - remset->data;
5746 bumper = addresses = get_internal_mem (sizeof (mword) * size, INTERNAL_MEM_STATISTICS);
5748 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5749 for (info = thread_table [i]; info; info = info->next) {
5750 for (remset = info->remset; remset; remset = remset->next)
5751 bumper = collect_store_remsets (remset, bumper);
5754 for (remset = global_remset; remset; remset = remset->next)
5755 bumper = collect_store_remsets (remset, bumper);
5756 for (remset = freed_thread_remsets; remset; remset = remset->next)
5757 bumper = collect_store_remsets (remset, bumper);
5759 g_assert (bumper <= addresses + size);
5761 stat_store_remsets += bumper - addresses;
5763 sort_addresses ((void**)addresses, bumper - addresses);
5766 while (r < bumper) {
5772 stat_store_remsets_unique += p - addresses;
5774 free_internal_mem (addresses, INTERNAL_MEM_STATISTICS);
5779 clear_thread_store_remset_buffer (SgenThreadInfo *info)
5781 *info->store_remset_buffer_index_addr = 0;
5782 memset (*info->store_remset_buffer_addr, 0, sizeof (gpointer) * STORE_REMSET_BUFFER_SIZE);
5786 scan_from_remsets (void *start_nursery, void *end_nursery, GrayQueue *queue)
5789 SgenThreadInfo *info;
5790 RememberedSet *remset;
5791 GenericStoreRememberedSet *store_remset;
5792 mword *p, *next_p, *store_pos;
5794 #ifdef HEAVY_STATISTICS
5798 /* the global one */
5799 for (remset = global_remset; remset; remset = remset->next) {
5800 DEBUG (4, fprintf (gc_debug_file, "Scanning global remset range: %p-%p, size: %td\n", remset->data, remset->store_next, remset->store_next - remset->data));
5801 store_pos = remset->data;
5802 for (p = remset->data; p < remset->store_next; p = next_p) {
5803 void **ptr = (void**)p [0];
5805 /*Ignore previously processed remset.*/
5806 if (!global_remset_location_was_not_added (ptr)) {
5811 next_p = handle_remset (p, start_nursery, end_nursery, TRUE, queue);
5814 * Clear global remsets of locations which no longer point to the
5815 * nursery. Otherwise, they could grow indefinitely between major
5818 * Since all global remsets are location remsets, we don't need to unmask the pointer.
5820 if (ptr_in_nursery (*ptr)) {
5821 *store_pos ++ = p [0];
5822 HEAVY_STAT (++stat_global_remsets_readded);
5826 /* Truncate the remset */
5827 remset->store_next = store_pos;
5830 /* the generic store ones */
5831 store_remset = generic_store_remsets;
5832 while (store_remset) {
5833 GenericStoreRememberedSet *next = store_remset->next;
5835 for (i = 0; i < STORE_REMSET_BUFFER_SIZE - 1; ++i) {
5836 gpointer addr = store_remset->data [i];
5838 handle_remset ((mword*)&addr, start_nursery, end_nursery, FALSE, queue);
5841 free_internal_mem (store_remset, INTERNAL_MEM_STORE_REMSET);
5843 store_remset = next;
5845 generic_store_remsets = NULL;
5847 /* the per-thread ones */
5848 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5849 for (info = thread_table [i]; info; info = info->next) {
5850 RememberedSet *next;
5852 for (remset = info->remset; remset; remset = next) {
5853 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for thread %p, range: %p-%p, size: %td\n", info, remset->data, remset->store_next, remset->store_next - remset->data));
5854 for (p = remset->data; p < remset->store_next;) {
5855 p = handle_remset (p, start_nursery, end_nursery, FALSE, queue);
5857 remset->store_next = remset->data;
5858 next = remset->next;
5859 remset->next = NULL;
5860 if (remset != info->remset) {
5861 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
5862 free_internal_mem (remset, INTERNAL_MEM_REMSET);
5865 for (j = 0; j < *info->store_remset_buffer_index_addr; ++j)
5866 handle_remset ((mword*)*info->store_remset_buffer_addr + j + 1, start_nursery, end_nursery, FALSE, queue);
5867 clear_thread_store_remset_buffer (info);
5871 /* the freed thread ones */
5872 while (freed_thread_remsets) {
5873 RememberedSet *next;
5874 remset = freed_thread_remsets;
5875 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for freed thread, range: %p-%p, size: %td\n", remset->data, remset->store_next, remset->store_next - remset->data));
5876 for (p = remset->data; p < remset->store_next;) {
5877 p = handle_remset (p, start_nursery, end_nursery, FALSE, queue);
5879 next = remset->next;
5880 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
5881 free_internal_mem (remset, INTERNAL_MEM_REMSET);
5882 freed_thread_remsets = next;
5887 * Clear the info in the remembered sets: we're doing a major collection, so
5888 * the per-thread ones are not needed and the global ones will be reconstructed
5892 clear_remsets (void)
5895 SgenThreadInfo *info;
5896 RememberedSet *remset, *next;
5898 /* the global list */
5899 for (remset = global_remset; remset; remset = next) {
5900 remset->store_next = remset->data;
5901 next = remset->next;
5902 remset->next = NULL;
5903 if (remset != global_remset) {
5904 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
5905 free_internal_mem (remset, INTERNAL_MEM_REMSET);
5908 /* the generic store ones */
5909 while (generic_store_remsets) {
5910 GenericStoreRememberedSet *gs_next = generic_store_remsets->next;
5911 free_internal_mem (generic_store_remsets, INTERNAL_MEM_STORE_REMSET);
5912 generic_store_remsets = gs_next;
5914 /* the per-thread ones */
5915 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5916 for (info = thread_table [i]; info; info = info->next) {
5917 for (remset = info->remset; remset; remset = next) {
5918 remset->store_next = remset->data;
5919 next = remset->next;
5920 remset->next = NULL;
5921 if (remset != info->remset) {
5922 DEBUG (3, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
5923 free_internal_mem (remset, INTERNAL_MEM_REMSET);
5926 clear_thread_store_remset_buffer (info);
5930 /* the freed thread ones */
5931 while (freed_thread_remsets) {
5932 next = freed_thread_remsets->next;
5933 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", freed_thread_remsets->data));
5934 free_internal_mem (freed_thread_remsets, INTERNAL_MEM_REMSET);
5935 freed_thread_remsets = next;
5940 * Clear the thread local TLAB variables for all threads.
5945 SgenThreadInfo *info;
5948 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5949 for (info = thread_table [i]; info; info = info->next) {
5950 /* A new TLAB will be allocated when the thread does its first allocation */
5951 *info->tlab_start_addr = NULL;
5952 *info->tlab_next_addr = NULL;
5953 *info->tlab_temp_end_addr = NULL;
5954 *info->tlab_real_end_addr = NULL;
5959 /* LOCKING: assumes the GC lock is held */
5960 static SgenThreadInfo*
5961 gc_register_current_thread (void *addr)
5964 SgenThreadInfo* info = malloc (sizeof (SgenThreadInfo));
5965 #ifndef HAVE_KW_THREAD
5966 SgenThreadInfo *__thread_info__ = info;
5972 memset (info, 0, sizeof (SgenThreadInfo));
5973 #ifndef HAVE_KW_THREAD
5974 info->tlab_start = info->tlab_next = info->tlab_temp_end = info->tlab_real_end = NULL;
5976 g_assert (!pthread_getspecific (thread_info_key));
5977 pthread_setspecific (thread_info_key, info);
5982 info->id = ARCH_GET_THREAD ();
5983 info->stop_count = -1;
5986 info->stack_start = NULL;
5987 info->tlab_start_addr = &TLAB_START;
5988 info->tlab_next_addr = &TLAB_NEXT;
5989 info->tlab_temp_end_addr = &TLAB_TEMP_END;
5990 info->tlab_real_end_addr = &TLAB_REAL_END;
5991 info->store_remset_buffer_addr = &STORE_REMSET_BUFFER;
5992 info->store_remset_buffer_index_addr = &STORE_REMSET_BUFFER_INDEX;
5993 info->stopped_ip = NULL;
5994 info->stopped_domain = NULL;
5995 info->stopped_regs = NULL;
5997 binary_protocol_thread_register ((gpointer)info->id);
5999 #ifdef HAVE_KW_THREAD
6000 tlab_next_addr = &tlab_next;
6001 store_remset_buffer_index_addr = &store_remset_buffer_index;
6004 /* try to get it with attributes first */
6005 #if defined(HAVE_PTHREAD_GETATTR_NP) && defined(HAVE_PTHREAD_ATTR_GETSTACK)
6009 pthread_attr_t attr;
6010 pthread_getattr_np (pthread_self (), &attr);
6011 pthread_attr_getstack (&attr, &sstart, &size);
6012 info->stack_start_limit = sstart;
6013 info->stack_end = (char*)sstart + size;
6014 pthread_attr_destroy (&attr);
6016 #elif defined(HAVE_PTHREAD_GET_STACKSIZE_NP) && defined(HAVE_PTHREAD_GET_STACKADDR_NP)
6017 info->stack_end = (char*)pthread_get_stackaddr_np (pthread_self ());
6018 info->stack_start_limit = (char*)info->stack_end - pthread_get_stacksize_np (pthread_self ());
6021 /* FIXME: we assume the stack grows down */
6022 gsize stack_bottom = (gsize)addr;
6023 stack_bottom += 4095;
6024 stack_bottom &= ~4095;
6025 info->stack_end = (char*)stack_bottom;
6029 #ifdef HAVE_KW_THREAD
6030 stack_end = info->stack_end;
6033 /* hash into the table */
6034 hash = HASH_PTHREAD_T (info->id) % THREAD_HASH_SIZE;
6035 info->next = thread_table [hash];
6036 thread_table [hash] = info;
6038 info->remset = alloc_remset (DEFAULT_REMSET_SIZE, info);
6039 pthread_setspecific (remembered_set_key, info->remset);
6040 #ifdef HAVE_KW_THREAD
6041 remembered_set = info->remset;
6044 STORE_REMSET_BUFFER = get_internal_mem (sizeof (gpointer) * STORE_REMSET_BUFFER_SIZE, INTERNAL_MEM_STORE_REMSET);
6045 STORE_REMSET_BUFFER_INDEX = 0;
6047 DEBUG (3, fprintf (gc_debug_file, "registered thread %p (%p) (hash: %d)\n", info, (gpointer)info->id, hash));
6049 if (gc_callbacks.thread_attach_func)
6050 info->runtime_data = gc_callbacks.thread_attach_func ();
6056 add_generic_store_remset_from_buffer (gpointer *buffer)
6058 GenericStoreRememberedSet *remset = get_internal_mem (sizeof (GenericStoreRememberedSet), INTERNAL_MEM_STORE_REMSET);
6059 memcpy (remset->data, buffer + 1, sizeof (gpointer) * (STORE_REMSET_BUFFER_SIZE - 1));
6060 remset->next = generic_store_remsets;
6061 generic_store_remsets = remset;
6065 unregister_current_thread (void)
6068 SgenThreadInfo *prev = NULL;
6070 RememberedSet *rset;
6071 ARCH_THREAD_TYPE id = ARCH_GET_THREAD ();
6073 binary_protocol_thread_unregister ((gpointer)id);
6075 hash = HASH_PTHREAD_T (id) % THREAD_HASH_SIZE;
6076 p = thread_table [hash];
6078 DEBUG (3, fprintf (gc_debug_file, "unregister thread %p (%p)\n", p, (gpointer)p->id));
6079 while (!ARCH_THREAD_EQUALS (p->id, id)) {
6084 thread_table [hash] = p->next;
6086 prev->next = p->next;
6089 if (freed_thread_remsets) {
6090 for (rset = p->remset; rset->next; rset = rset->next)
6092 rset->next = freed_thread_remsets;
6093 freed_thread_remsets = p->remset;
6095 freed_thread_remsets = p->remset;
6098 if (*p->store_remset_buffer_index_addr)
6099 add_generic_store_remset_from_buffer (*p->store_remset_buffer_addr);
6100 free_internal_mem (*p->store_remset_buffer_addr, INTERNAL_MEM_STORE_REMSET);
6105 unregister_thread (void *k)
6107 g_assert (!mono_domain_get ());
6109 unregister_current_thread ();
6114 mono_gc_register_thread (void *baseptr)
6116 SgenThreadInfo *info;
6120 info = mono_sgen_thread_info_lookup (ARCH_GET_THREAD ());
6122 info = gc_register_current_thread (baseptr);
6124 return info != NULL;
6127 #if USE_PTHREAD_INTERCEPT
6130 void *(*start_routine) (void *);
6133 MonoSemType registered;
6134 } SgenThreadStartInfo;
6137 gc_start_thread (void *arg)
6139 SgenThreadStartInfo *start_info = arg;
6140 SgenThreadInfo* info;
6141 void *t_arg = start_info->arg;
6142 void *(*start_func) (void*) = start_info->start_routine;
6147 info = gc_register_current_thread (&result);
6149 post_result = MONO_SEM_POST (&(start_info->registered));
6150 g_assert (!post_result);
6151 result = start_func (t_arg);
6152 g_assert (!mono_domain_get ());
6154 * this is done by the pthread key dtor
6156 unregister_current_thread ();
6164 mono_gc_pthread_create (pthread_t *new_thread, const pthread_attr_t *attr, void *(*start_routine)(void *), void *arg)
6166 SgenThreadStartInfo *start_info;
6169 start_info = malloc (sizeof (SgenThreadStartInfo));
6172 result = MONO_SEM_INIT (&(start_info->registered), 0);
6174 start_info->arg = arg;
6175 start_info->start_routine = start_routine;
6177 result = pthread_create (new_thread, attr, gc_start_thread, start_info);
6179 while (MONO_SEM_WAIT (&(start_info->registered)) != 0) {
6180 /*if (EINTR != errno) ABORT("sem_wait failed"); */
6183 MONO_SEM_DESTROY (&(start_info->registered));
6189 mono_gc_pthread_join (pthread_t thread, void **retval)
6191 return pthread_join (thread, retval);
6195 mono_gc_pthread_detach (pthread_t thread)
6197 return pthread_detach (thread);
6200 #endif /* USE_PTHREAD_INTERCEPT */
6203 * ######################################################################
6204 * ######## Write barriers
6205 * ######################################################################
6208 static RememberedSet*
6209 alloc_remset (int size, gpointer id) {
6210 RememberedSet* res = get_internal_mem (sizeof (RememberedSet) + (size * sizeof (gpointer)), INTERNAL_MEM_REMSET);
6211 res->store_next = res->data;
6212 res->end_set = res->data + size;
6214 DEBUG (4, fprintf (gc_debug_file, "Allocated remset size %d at %p for %p\n", size, res->data, id));
6219 * Note: the write barriers first do the needed GC work and then do the actual store:
6220 * this way the value is visible to the conservative GC scan after the write barrier
6221 * itself. If a GC interrupts the barrier in the middle, value will be kept alive by
6222 * the conservative scan, otherwise by the remembered set scan.
6225 mono_gc_wbarrier_set_field (MonoObject *obj, gpointer field_ptr, MonoObject* value)
6229 HEAVY_STAT (++stat_wbarrier_set_field);
6230 if (ptr_in_nursery (field_ptr)) {
6231 *(void**)field_ptr = value;
6234 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", field_ptr));
6236 rs = REMEMBERED_SET;
6237 if (rs->store_next < rs->end_set) {
6238 *(rs->store_next++) = (mword)field_ptr;
6239 *(void**)field_ptr = value;
6243 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6244 rs->next = REMEMBERED_SET;
6245 REMEMBERED_SET = rs;
6246 #ifdef HAVE_KW_THREAD
6247 mono_sgen_thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6249 *(rs->store_next++) = (mword)field_ptr;
6250 *(void**)field_ptr = value;
6255 mono_gc_wbarrier_set_arrayref (MonoArray *arr, gpointer slot_ptr, MonoObject* value)
6259 HEAVY_STAT (++stat_wbarrier_set_arrayref);
6260 if (ptr_in_nursery (slot_ptr)) {
6261 *(void**)slot_ptr = value;
6264 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", slot_ptr));
6266 rs = REMEMBERED_SET;
6267 if (rs->store_next < rs->end_set) {
6268 *(rs->store_next++) = (mword)slot_ptr;
6269 *(void**)slot_ptr = value;
6273 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6274 rs->next = REMEMBERED_SET;
6275 REMEMBERED_SET = rs;
6276 #ifdef HAVE_KW_THREAD
6277 mono_sgen_thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6279 *(rs->store_next++) = (mword)slot_ptr;
6280 *(void**)slot_ptr = value;
6285 mono_gc_wbarrier_arrayref_copy (gpointer dest_ptr, gpointer src_ptr, int count)
6289 HEAVY_STAT (++stat_wbarrier_arrayref_copy);
6291 memmove (dest_ptr, src_ptr, count * sizeof (gpointer));
6292 if (ptr_in_nursery (dest_ptr)) {
6296 rs = REMEMBERED_SET;
6297 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p, %d\n", dest_ptr, count));
6298 if (rs->store_next + 1 < rs->end_set) {
6299 *(rs->store_next++) = (mword)dest_ptr | REMSET_RANGE;
6300 *(rs->store_next++) = count;
6304 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6305 rs->next = REMEMBERED_SET;
6306 REMEMBERED_SET = rs;
6307 #ifdef HAVE_KW_THREAD
6308 mono_sgen_thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6310 *(rs->store_next++) = (mword)dest_ptr | REMSET_RANGE;
6311 *(rs->store_next++) = count;
6315 static char *found_obj;
6318 find_object_for_ptr_callback (char *obj, size_t size, char *ptr)
6320 if (ptr >= obj && ptr < obj + size) {
6321 g_assert (!found_obj);
6326 /* for use in the debugger */
6327 char* find_object_for_ptr (char *ptr);
6329 find_object_for_ptr (char *ptr)
6333 if (ptr >= nursery_section->data && ptr < nursery_section->end_data) {
6335 scan_area_with_callback (nursery_section->data, nursery_section->end_data,
6336 (IterateObjectCallbackFunc)find_object_for_ptr_callback, ptr);
6341 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
6342 if (ptr >= bigobj->data && ptr < bigobj->data + bigobj->size)
6343 return bigobj->data;
6347 * Very inefficient, but this is debugging code, supposed to
6348 * be called from gdb, so we don't care.
6351 major_iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)find_object_for_ptr_callback, ptr);
6356 evacuate_remset_buffer (void)
6361 buffer = STORE_REMSET_BUFFER;
6363 add_generic_store_remset_from_buffer (buffer);
6364 memset (buffer, 0, sizeof (gpointer) * STORE_REMSET_BUFFER_SIZE);
6366 STORE_REMSET_BUFFER_INDEX = 0;
6370 mono_gc_wbarrier_generic_nostore (gpointer ptr)
6376 HEAVY_STAT (++stat_wbarrier_generic_store);
6378 #ifdef XDOMAIN_CHECKS_IN_WBARRIER
6379 /* FIXME: ptr_in_heap must be called with the GC lock held */
6380 if (xdomain_checks && *(MonoObject**)ptr && ptr_in_heap (ptr)) {
6381 char *start = find_object_for_ptr (ptr);
6382 MonoObject *value = *(MonoObject**)ptr;
6386 MonoObject *obj = (MonoObject*)start;
6387 if (obj->vtable->domain != value->vtable->domain)
6388 g_assert (is_xdomain_ref_allowed (ptr, start, obj->vtable->domain));
6396 if (*(gpointer*)ptr)
6397 binary_protocol_wbarrier (ptr, *(gpointer*)ptr, (gpointer)LOAD_VTABLE (*(gpointer*)ptr));
6399 if (ptr_in_nursery (ptr) || ptr_on_stack (ptr) || !ptr_in_nursery (*(gpointer*)ptr)) {
6400 DEBUG (8, fprintf (gc_debug_file, "Skipping remset at %p\n", ptr));
6405 buffer = STORE_REMSET_BUFFER;
6406 index = STORE_REMSET_BUFFER_INDEX;
6407 /* This simple optimization eliminates a sizable portion of
6408 entries. Comparing it to the last but one entry as well
6409 doesn't eliminate significantly more entries. */
6410 if (buffer [index] == ptr) {
6415 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", ptr));
6416 HEAVY_STAT (++stat_wbarrier_generic_store_remset);
6419 if (index >= STORE_REMSET_BUFFER_SIZE) {
6420 evacuate_remset_buffer ();
6421 index = STORE_REMSET_BUFFER_INDEX;
6422 g_assert (index == 0);
6425 buffer [index] = ptr;
6426 STORE_REMSET_BUFFER_INDEX = index;
6432 mono_gc_wbarrier_generic_store (gpointer ptr, MonoObject* value)
6434 DEBUG (8, fprintf (gc_debug_file, "Wbarrier store at %p to %p (%s)\n", ptr, value, value ? safe_name (value) : "null"));
6435 *(void**)ptr = value;
6436 if (ptr_in_nursery (value))
6437 mono_gc_wbarrier_generic_nostore (ptr);
6440 void mono_gc_wbarrier_value_copy_bitmap (gpointer _dest, gpointer _src, int size, unsigned bitmap)
6442 mword *dest = _dest;
6447 mono_gc_wbarrier_generic_store (dest, (MonoObject*)*src);
6452 size -= SIZEOF_VOID_P;
6459 mono_gc_wbarrier_value_copy (gpointer dest, gpointer src, int count, MonoClass *klass)
6463 HEAVY_STAT (++stat_wbarrier_value_copy);
6464 g_assert (klass->valuetype);
6466 memmove (dest, src, count * mono_class_value_size (klass, NULL));
6467 rs = REMEMBERED_SET;
6468 if (ptr_in_nursery (dest) || ptr_on_stack (dest) || !klass->has_references) {
6472 g_assert (klass->gc_descr_inited);
6473 DEBUG (8, fprintf (gc_debug_file, "Adding value remset at %p, count %d, descr %p for class %s (%p)\n", dest, count, klass->gc_descr, klass->name, klass));
6475 if (rs->store_next + 3 < rs->end_set) {
6476 *(rs->store_next++) = (mword)dest | REMSET_VTYPE;
6477 *(rs->store_next++) = (mword)klass->gc_descr;
6478 *(rs->store_next++) = (mword)count;
6482 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6483 rs->next = REMEMBERED_SET;
6484 REMEMBERED_SET = rs;
6485 #ifdef HAVE_KW_THREAD
6486 mono_sgen_thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6488 *(rs->store_next++) = (mword)dest | REMSET_VTYPE;
6489 *(rs->store_next++) = (mword)klass->gc_descr;
6490 *(rs->store_next++) = (mword)count;
6495 * mono_gc_wbarrier_object_copy:
6497 * Write barrier to call when obj is the result of a clone or copy of an object.
6500 mono_gc_wbarrier_object_copy (MonoObject* obj, MonoObject *src)
6506 HEAVY_STAT (++stat_wbarrier_object_copy);
6507 rs = REMEMBERED_SET;
6508 DEBUG (6, fprintf (gc_debug_file, "Adding object remset for %p\n", obj));
6509 size = mono_object_class (obj)->instance_size;
6511 /* do not copy the sync state */
6512 memcpy ((char*)obj + sizeof (MonoObject), (char*)src + sizeof (MonoObject),
6513 size - sizeof (MonoObject));
6514 if (ptr_in_nursery (obj) || ptr_on_stack (obj)) {
6518 if (rs->store_next < rs->end_set) {
6519 *(rs->store_next++) = (mword)obj | REMSET_OBJECT;
6523 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6524 rs->next = REMEMBERED_SET;
6525 REMEMBERED_SET = rs;
6526 #ifdef HAVE_KW_THREAD
6527 mono_sgen_thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6529 *(rs->store_next++) = (mword)obj | REMSET_OBJECT;
6534 * ######################################################################
6535 * ######## Collector debugging
6536 * ######################################################################
6539 const char*descriptor_types [] = {
6551 describe_ptr (char *ptr)
6557 if (ptr_in_nursery (ptr)) {
6558 printf ("Pointer inside nursery.\n");
6560 if (major_ptr_is_in_non_pinned_space (ptr)) {
6561 printf ("Pointer inside oldspace.\n");
6562 } else if (obj_is_from_pinned_alloc (ptr)) {
6563 printf ("Pointer is inside a pinned chunk.\n");
6565 printf ("Pointer unknown.\n");
6570 if (object_is_pinned (ptr))
6571 printf ("Object is pinned.\n");
6573 if (object_is_forwarded (ptr))
6574 printf ("Object is forwared.\n");
6576 // FIXME: Handle pointers to the inside of objects
6577 vtable = (MonoVTable*)LOAD_VTABLE (ptr);
6579 printf ("VTable: %p\n", vtable);
6580 if (vtable == NULL) {
6581 printf ("VTable is invalid (empty).\n");
6584 if (ptr_in_nursery (vtable)) {
6585 printf ("VTable is invalid (points inside nursery).\n");
6588 printf ("Class: %s\n", vtable->klass->name);
6590 desc = ((GCVTable*)vtable)->desc;
6591 printf ("Descriptor: %lx\n", (long)desc);
6594 printf ("Descriptor type: %d (%s)\n", type, descriptor_types [type]);
6598 find_in_remset_loc (mword *p, char *addr, gboolean *found)
6604 switch ((*p) & REMSET_TYPE_MASK) {
6605 case REMSET_LOCATION:
6606 if (*p == (mword)addr)
6610 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
6612 if ((void**)addr >= ptr && (void**)addr < ptr + count)
6616 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
6617 count = safe_object_get_size ((MonoObject*)ptr);
6618 count = ALIGN_UP (count);
6619 count /= sizeof (mword);
6620 if ((void**)addr >= ptr && (void**)addr < ptr + count)
6624 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
6628 switch (desc & 0x7) {
6629 case DESC_TYPE_RUN_LENGTH:
6630 OBJ_RUN_LEN_SIZE (skip_size, desc, ptr);
6632 case DESC_TYPE_SMALL_BITMAP:
6633 OBJ_BITMAP_SIZE (skip_size, desc, start);
6637 g_assert_not_reached ();
6640 /* The descriptor includes the size of MonoObject */
6641 skip_size -= sizeof (MonoObject);
6643 if ((void**)addr >= ptr && (void**)addr < ptr + (skip_size / sizeof (gpointer)))
6648 g_assert_not_reached ();
6654 * Return whenever ADDR occurs in the remembered sets
6657 find_in_remsets (char *addr)
6660 SgenThreadInfo *info;
6661 RememberedSet *remset;
6662 GenericStoreRememberedSet *store_remset;
6664 gboolean found = FALSE;
6666 /* the global one */
6667 for (remset = global_remset; remset; remset = remset->next) {
6668 DEBUG (4, fprintf (gc_debug_file, "Scanning global remset range: %p-%p, size: %td\n", remset->data, remset->store_next, remset->store_next - remset->data));
6669 for (p = remset->data; p < remset->store_next;) {
6670 p = find_in_remset_loc (p, addr, &found);
6676 /* the generic store ones */
6677 for (store_remset = generic_store_remsets; store_remset; store_remset = store_remset->next) {
6678 for (i = 0; i < STORE_REMSET_BUFFER_SIZE - 1; ++i) {
6679 if (store_remset->data [i] == addr)
6684 /* the per-thread ones */
6685 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
6686 for (info = thread_table [i]; info; info = info->next) {
6688 for (remset = info->remset; remset; remset = remset->next) {
6689 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for thread %p, range: %p-%p, size: %td\n", info, remset->data, remset->store_next, remset->store_next - remset->data));
6690 for (p = remset->data; p < remset->store_next;) {
6691 p = find_in_remset_loc (p, addr, &found);
6696 for (j = 0; j < *info->store_remset_buffer_index_addr; ++j) {
6697 if ((*info->store_remset_buffer_addr) [j + 1] == addr)
6703 /* the freed thread ones */
6704 for (remset = freed_thread_remsets; remset; remset = remset->next) {
6705 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for freed thread, range: %p-%p, size: %td\n", remset->data, remset->store_next, remset->store_next - remset->data));
6706 for (p = remset->data; p < remset->store_next;) {
6707 p = find_in_remset_loc (p, addr, &found);
6716 static gboolean missing_remsets;
6719 * We let a missing remset slide if the target object is pinned,
6720 * because the store might have happened but the remset not yet added,
6721 * but in that case the target must be pinned. We might theoretically
6722 * miss some missing remsets this way, but it's very unlikely.
6725 #define HANDLE_PTR(ptr,obj) do { \
6726 if (*(ptr) && (char*)*(ptr) >= nursery_start && (char*)*(ptr) < nursery_next) { \
6727 if (!find_in_remsets ((char*)(ptr))) { \
6728 fprintf (gc_debug_file, "Oldspace->newspace reference %p at offset %td in object %p (%s.%s) not found in remsets.\n", *(ptr), (char*)(ptr) - (char*)(obj), (obj), ((MonoObject*)(obj))->vtable->klass->name_space, ((MonoObject*)(obj))->vtable->klass->name); \
6729 binary_protocol_missing_remset ((obj), (gpointer)LOAD_VTABLE ((obj)), (char*)(ptr) - (char*)(obj), *(ptr), (gpointer)LOAD_VTABLE(*(ptr)), object_is_pinned (*(ptr))); \
6730 if (!object_is_pinned (*(ptr))) \
6731 missing_remsets = TRUE; \
6737 * Check that each object reference which points into the nursery can
6738 * be found in the remembered sets.
6741 check_consistency_callback (char *start, size_t size, void *dummy)
6743 GCVTable *vt = (GCVTable*)LOAD_VTABLE (start);
6744 DEBUG (8, fprintf (gc_debug_file, "Scanning object %p, vtable: %p (%s)\n", start, vt, vt->klass->name));
6746 #define SCAN_OBJECT_ACTION
6747 #include "sgen-scan-object.h"
6751 * Perform consistency check of the heap.
6753 * Assumes the world is stopped.
6756 check_consistency (void)
6760 // Need to add more checks
6762 missing_remsets = FALSE;
6764 DEBUG (1, fprintf (gc_debug_file, "Begin heap consistency check...\n"));
6766 // Check that oldspace->newspace pointers are registered with the collector
6767 major_iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)check_consistency_callback, NULL);
6769 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
6770 check_consistency_callback (bigobj->data, bigobj->size, NULL);
6772 DEBUG (1, fprintf (gc_debug_file, "Heap consistency check done.\n"));
6774 #ifdef BINARY_PROTOCOL
6775 if (!binary_protocol_file)
6777 g_assert (!missing_remsets);
6782 #define HANDLE_PTR(ptr,obj) do { \
6784 g_assert (LOAD_VTABLE (*(ptr))); \
6788 check_major_refs_callback (char *start, size_t size, void *dummy)
6790 #define SCAN_OBJECT_ACTION
6791 #include "sgen-scan-object.h"
6795 check_major_refs (void)
6799 major_iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)check_major_refs_callback, NULL);
6801 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
6802 check_major_refs_callback (bigobj->data, bigobj->size, NULL);
6805 /* Check that the reference is valid */
6807 #define HANDLE_PTR(ptr,obj) do { \
6809 g_assert (safe_name (*(ptr)) != NULL); \
6816 * Perform consistency check on an object. Currently we only check that the
6817 * reference fields are valid.
6820 check_object (char *start)
6825 #include "sgen-scan-object.h"
6829 * ######################################################################
6830 * ######## Other mono public interface functions.
6831 * ######################################################################
6835 mono_gc_collect (int generation)
6839 if (generation == 0) {
6840 collect_nursery (0);
6842 major_collection ("user request");
6849 mono_gc_max_generation (void)
6855 mono_gc_collection_count (int generation)
6857 if (generation == 0)
6858 return num_minor_gcs;
6859 return num_major_gcs;
6863 mono_gc_get_used_size (void)
6867 tot = los_memory_usage;
6868 tot += nursery_section->next_data - nursery_section->data;
6869 tot += major_get_used_size ();
6870 /* FIXME: account for pinned objects */
6876 mono_gc_get_heap_size (void)
6882 mono_gc_disable (void)
6890 mono_gc_enable (void)
6898 mono_gc_get_los_limit (void)
6900 return MAX_SMALL_OBJ_SIZE;
6904 mono_object_is_alive (MonoObject* o)
6910 mono_gc_get_generation (MonoObject *obj)
6912 if (ptr_in_nursery (obj))
6918 mono_gc_enable_events (void)
6923 mono_gc_weak_link_add (void **link_addr, MonoObject *obj, gboolean track)
6926 mono_gc_register_disappearing_link (obj, link_addr, track);
6931 mono_gc_weak_link_remove (void **link_addr)
6934 mono_gc_register_disappearing_link (NULL, link_addr, FALSE);
6939 mono_gc_weak_link_get (void **link_addr)
6943 return (MonoObject*) REVEAL_POINTER (*link_addr);
6947 mono_gc_ephemeron_array_add (MonoObject *obj)
6949 EphemeronLinkNode *node;
6953 node = get_internal_mem (sizeof (EphemeronLinkNode), INTERNAL_MEM_EPHEMERON_LINK);
6958 node->array = (char*)obj;
6959 node->next = ephemeron_list;
6960 ephemeron_list = node;
6962 DEBUG (5, fprintf (gc_debug_file, "Registered ephemeron array %p\n", obj));
6969 mono_gc_make_descr_from_bitmap (gsize *bitmap, int numbits)
6971 if (numbits < ((sizeof (*bitmap) * 8) - ROOT_DESC_TYPE_SHIFT)) {
6972 return (void*)MAKE_ROOT_DESC (ROOT_DESC_BITMAP, bitmap [0]);
6974 mword complex = alloc_complex_descriptor (bitmap, numbits);
6975 return (void*)MAKE_ROOT_DESC (ROOT_DESC_COMPLEX, complex);
6980 mono_gc_make_root_descr_user (MonoGCRootMarkFunc marker)
6984 g_assert (user_descriptors_next < MAX_USER_DESCRIPTORS);
6985 descr = (void*)MAKE_ROOT_DESC (ROOT_DESC_USER, (mword)user_descriptors_next);
6986 user_descriptors [user_descriptors_next ++] = marker;
6992 mono_gc_alloc_fixed (size_t size, void *descr)
6994 /* FIXME: do a single allocation */
6995 void *res = calloc (1, size);
6998 if (!mono_gc_register_root (res, size, descr)) {
7006 mono_gc_free_fixed (void* addr)
7008 mono_gc_deregister_root (addr);
7013 mono_gc_invoke_with_gc_lock (MonoGCLockedCallbackFunc func, void *data)
7017 result = func (data);
7018 UNLOCK_INTERRUPTION;
7023 mono_gc_is_gc_thread (void)
7027 result = mono_sgen_thread_info_lookup (ARCH_GET_THREAD ()) != NULL;
7034 /* Tries to extract a number from the passed string, taking in to account m, k
7037 parse_environment_string_extract_number (gchar *str, glong *out)
7040 int len = strlen (str), shift = 0;
7042 gboolean is_suffix = FALSE;
7045 switch (str [len - 1]) {
7056 suffix = str [len - 1];
7061 val = strtol (str, &endptr, 10);
7063 if ((errno == ERANGE && (val == LONG_MAX || val == LONG_MIN))
7064 || (errno != 0 && val == 0) || (endptr == str))
7068 if (*(endptr + 1)) /* Invalid string. */
7080 mono_gc_base_init (void)
7084 struct sigaction sinfo;
7086 LOCK_INIT (gc_mutex);
7088 if (gc_initialized) {
7092 pagesize = mono_pagesize ();
7093 gc_debug_file = stderr;
7097 if ((env = getenv ("MONO_GC_PARAMS"))) {
7098 if (g_str_has_prefix (env, "nursery-size")) {
7101 while (env [index] && env [index++] != '=')
7103 if (env [index] && parse_environment_string_extract_number (env
7105 default_nursery_size = val;
7106 #ifdef ALIGN_NURSERY
7107 if ((val & (val - 1))) {
7108 fprintf (stderr, "The nursery size must be a power of two.\n");
7112 default_nursery_bits = 0;
7113 while (1 << (++ default_nursery_bits) != default_nursery_size)
7117 fprintf (stderr, "nursery-size must be an integer.\n");
7121 fprintf (stderr, "MONO_GC_PARAMS must be of the form 'nursery-size=N' (where N is an integer, possibly with a k, m or a g suffix).\n");
7128 nursery_size = DEFAULT_NURSERY_SIZE;
7129 minor_collection_allowance = MIN_MINOR_COLLECTION_ALLOWANCE;
7133 if ((env = getenv ("MONO_GC_DEBUG"))) {
7134 opts = g_strsplit (env, ",", -1);
7135 for (ptr = opts; ptr && *ptr; ptr ++) {
7137 if (opt [0] >= '0' && opt [0] <= '9') {
7138 gc_debug_level = atoi (opt);
7143 char *rf = g_strdup_printf ("%s.%d", opt, getpid ());
7144 gc_debug_file = fopen (rf, "wb");
7146 gc_debug_file = stderr;
7149 } else if (!strcmp (opt, "collect-before-allocs")) {
7150 collect_before_allocs = TRUE;
7151 } else if (!strcmp (opt, "check-at-minor-collections")) {
7152 consistency_check_at_minor_collection = TRUE;
7153 nursery_clear_policy = CLEAR_AT_GC;
7154 } else if (!strcmp (opt, "xdomain-checks")) {
7155 xdomain_checks = TRUE;
7156 } else if (!strcmp (opt, "clear-at-gc")) {
7157 nursery_clear_policy = CLEAR_AT_GC;
7158 } else if (!strcmp (opt, "conservative-stack-mark")) {
7159 conservative_stack_mark = TRUE;
7160 } else if (!strcmp (opt, "check-scan-starts")) {
7161 do_scan_starts_check = TRUE;
7162 } else if (g_str_has_prefix (opt, "heap-dump=")) {
7163 char *filename = strchr (opt, '=') + 1;
7164 nursery_clear_policy = CLEAR_AT_GC;
7165 heap_dump_file = fopen (filename, "w");
7167 fprintf (heap_dump_file, "<sgen-dump>\n");
7168 #ifdef BINARY_PROTOCOL
7169 } else if (g_str_has_prefix (opt, "binary-protocol=")) {
7170 char *filename = strchr (opt, '=') + 1;
7171 binary_protocol_file = fopen (filename, "w");
7174 fprintf (stderr, "Invalid format for the MONO_GC_DEBUG env variable: '%s'\n", env);
7175 fprintf (stderr, "The format is: MONO_GC_DEBUG=[l[:filename]|<option>]+ where l is a debug level 0-9.\n");
7176 fprintf (stderr, "Valid options are: collect-before-allocs, check-at-minor-collections, xdomain-checks, clear-at-gc.\n");
7183 suspend_ack_semaphore_ptr = &suspend_ack_semaphore;
7184 MONO_SEM_INIT (&suspend_ack_semaphore, 0);
7186 sigfillset (&sinfo.sa_mask);
7187 sinfo.sa_flags = SA_RESTART | SA_SIGINFO;
7188 sinfo.sa_sigaction = suspend_handler;
7189 if (sigaction (suspend_signal_num, &sinfo, NULL) != 0) {
7190 g_error ("failed sigaction");
7193 sinfo.sa_handler = restart_handler;
7194 if (sigaction (restart_signal_num, &sinfo, NULL) != 0) {
7195 g_error ("failed sigaction");
7198 sigfillset (&suspend_signal_mask);
7199 sigdelset (&suspend_signal_mask, restart_signal_num);
7201 global_remset = alloc_remset (1024, NULL);
7202 global_remset->next = NULL;
7204 pthread_key_create (&remembered_set_key, unregister_thread);
7206 #ifndef HAVE_KW_THREAD
7207 pthread_key_create (&thread_info_key, NULL);
7210 gc_initialized = TRUE;
7212 mono_gc_register_thread (&sinfo);
7216 mono_gc_get_suspend_signal (void)
7218 return suspend_signal_num;
7228 #ifdef HAVE_KW_THREAD
7229 #define EMIT_TLS_ACCESS(mb,dummy,offset) do { \
7230 mono_mb_emit_byte ((mb), MONO_CUSTOM_PREFIX); \
7231 mono_mb_emit_byte ((mb), CEE_MONO_TLS); \
7232 mono_mb_emit_i4 ((mb), (offset)); \
7235 #define EMIT_TLS_ACCESS(mb,member,dummy) do { \
7236 mono_mb_emit_byte ((mb), MONO_CUSTOM_PREFIX); \
7237 mono_mb_emit_byte ((mb), CEE_MONO_TLS); \
7238 mono_mb_emit_i4 ((mb), thread_info_key); \
7239 mono_mb_emit_icon ((mb), G_STRUCT_OFFSET (SgenThreadInfo, member)); \
7240 mono_mb_emit_byte ((mb), CEE_ADD); \
7241 mono_mb_emit_byte ((mb), CEE_LDIND_I); \
7245 #ifdef MANAGED_ALLOCATION
7246 /* FIXME: Do this in the JIT, where specialized allocation sequences can be created
7247 * for each class. This is currently not easy to do, as it is hard to generate basic
7248 * blocks + branches, but it is easy with the linear IL codebase.
7250 * For this to work we'd need to solve the TLAB race, first. Now we
7251 * require the allocator to be in a few known methods to make sure
7252 * that they are executed atomically via the restart mechanism.
7255 create_allocator (int atype)
7257 int p_var, size_var;
7258 guint32 slowpath_branch, max_size_branch;
7259 MonoMethodBuilder *mb;
7261 MonoMethodSignature *csig;
7262 static gboolean registered = FALSE;
7263 int tlab_next_addr_var, new_next_var;
7265 const char *name = NULL;
7266 AllocatorWrapperInfo *info;
7268 #ifdef HAVE_KW_THREAD
7269 int tlab_next_addr_offset = -1;
7270 int tlab_temp_end_offset = -1;
7272 MONO_THREAD_VAR_OFFSET (tlab_next_addr, tlab_next_addr_offset);
7273 MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
7275 g_assert (tlab_next_addr_offset != -1);
7276 g_assert (tlab_temp_end_offset != -1);
7280 mono_register_jit_icall (mono_gc_alloc_obj, "mono_gc_alloc_obj", mono_create_icall_signature ("object ptr int"), FALSE);
7281 mono_register_jit_icall (mono_gc_alloc_vector, "mono_gc_alloc_vector", mono_create_icall_signature ("object ptr int int"), FALSE);
7285 if (atype == ATYPE_SMALL) {
7287 name = "AllocSmall";
7288 } else if (atype == ATYPE_NORMAL) {
7291 } else if (atype == ATYPE_VECTOR) {
7293 name = "AllocVector";
7295 g_assert_not_reached ();
7298 csig = mono_metadata_signature_alloc (mono_defaults.corlib, num_params);
7299 csig->ret = &mono_defaults.object_class->byval_arg;
7300 for (i = 0; i < num_params; ++i)
7301 csig->params [i] = &mono_defaults.int_class->byval_arg;
7303 mb = mono_mb_new (mono_defaults.object_class, name, MONO_WRAPPER_ALLOC);
7304 size_var = mono_mb_add_local (mb, &mono_defaults.int32_class->byval_arg);
7305 if (atype == ATYPE_NORMAL || atype == ATYPE_SMALL) {
7306 /* size = vtable->klass->instance_size; */
7307 mono_mb_emit_ldarg (mb, 0);
7308 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoVTable, klass));
7309 mono_mb_emit_byte (mb, CEE_ADD);
7310 mono_mb_emit_byte (mb, CEE_LDIND_I);
7311 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoClass, instance_size));
7312 mono_mb_emit_byte (mb, CEE_ADD);
7313 /* FIXME: assert instance_size stays a 4 byte integer */
7314 mono_mb_emit_byte (mb, CEE_LDIND_U4);
7315 mono_mb_emit_stloc (mb, size_var);
7316 } else if (atype == ATYPE_VECTOR) {
7317 MonoExceptionClause *clause;
7319 MonoClass *oom_exc_class;
7322 /* n > MONO_ARRAY_MAX_INDEX -> OverflowException */
7323 mono_mb_emit_ldarg (mb, 1);
7324 mono_mb_emit_icon (mb, MONO_ARRAY_MAX_INDEX);
7325 pos = mono_mb_emit_short_branch (mb, CEE_BLE_UN_S);
7326 mono_mb_emit_exception (mb, "OverflowException", NULL);
7327 mono_mb_patch_short_branch (mb, pos);
7329 clause = mono_image_alloc0 (mono_defaults.corlib, sizeof (MonoExceptionClause));
7330 clause->try_offset = mono_mb_get_label (mb);
7332 /* vtable->klass->sizes.element_size */
7333 mono_mb_emit_ldarg (mb, 0);
7334 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoVTable, klass));
7335 mono_mb_emit_byte (mb, CEE_ADD);
7336 mono_mb_emit_byte (mb, CEE_LDIND_I);
7337 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoClass, sizes.element_size));
7338 mono_mb_emit_byte (mb, CEE_ADD);
7339 mono_mb_emit_byte (mb, CEE_LDIND_U4);
7342 mono_mb_emit_ldarg (mb, 1);
7343 mono_mb_emit_byte (mb, CEE_MUL_OVF_UN);
7344 /* + sizeof (MonoArray) */
7345 mono_mb_emit_icon (mb, sizeof (MonoArray));
7346 mono_mb_emit_byte (mb, CEE_ADD_OVF_UN);
7347 mono_mb_emit_stloc (mb, size_var);
7349 pos_leave = mono_mb_emit_branch (mb, CEE_LEAVE);
7352 clause->flags = MONO_EXCEPTION_CLAUSE_NONE;
7353 clause->try_len = mono_mb_get_pos (mb) - clause->try_offset;
7354 clause->data.catch_class = mono_class_from_name (mono_defaults.corlib,
7355 "System", "OverflowException");
7356 g_assert (clause->data.catch_class);
7357 clause->handler_offset = mono_mb_get_label (mb);
7359 oom_exc_class = mono_class_from_name (mono_defaults.corlib,
7360 "System", "OutOfMemoryException");
7361 g_assert (oom_exc_class);
7362 ctor = mono_class_get_method_from_name (oom_exc_class, ".ctor", 0);
7365 mono_mb_emit_byte (mb, CEE_POP);
7366 mono_mb_emit_op (mb, CEE_NEWOBJ, ctor);
7367 mono_mb_emit_byte (mb, CEE_THROW);
7369 clause->handler_len = mono_mb_get_pos (mb) - clause->handler_offset;
7370 mono_mb_set_clauses (mb, 1, clause);
7371 mono_mb_patch_branch (mb, pos_leave);
7374 g_assert_not_reached ();
7377 /* size += ALLOC_ALIGN - 1; */
7378 mono_mb_emit_ldloc (mb, size_var);
7379 mono_mb_emit_icon (mb, ALLOC_ALIGN - 1);
7380 mono_mb_emit_byte (mb, CEE_ADD);
7381 /* size &= ~(ALLOC_ALIGN - 1); */
7382 mono_mb_emit_icon (mb, ~(ALLOC_ALIGN - 1));
7383 mono_mb_emit_byte (mb, CEE_AND);
7384 mono_mb_emit_stloc (mb, size_var);
7386 /* if (size > MAX_SMALL_OBJ_SIZE) goto slowpath */
7387 if (atype != ATYPE_SMALL) {
7388 mono_mb_emit_ldloc (mb, size_var);
7389 mono_mb_emit_icon (mb, MAX_SMALL_OBJ_SIZE);
7390 max_size_branch = mono_mb_emit_short_branch (mb, MONO_CEE_BGT_S);
7394 * We need to modify tlab_next, but the JIT only supports reading, so we read
7395 * another tls var holding its address instead.
7398 /* tlab_next_addr (local) = tlab_next_addr (TLS var) */
7399 tlab_next_addr_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7400 EMIT_TLS_ACCESS (mb, tlab_next_addr, tlab_next_addr_offset);
7401 mono_mb_emit_stloc (mb, tlab_next_addr_var);
7403 /* p = (void**)tlab_next; */
7404 p_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7405 mono_mb_emit_ldloc (mb, tlab_next_addr_var);
7406 mono_mb_emit_byte (mb, CEE_LDIND_I);
7407 mono_mb_emit_stloc (mb, p_var);
7409 /* new_next = (char*)p + size; */
7410 new_next_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7411 mono_mb_emit_ldloc (mb, p_var);
7412 mono_mb_emit_ldloc (mb, size_var);
7413 mono_mb_emit_byte (mb, CEE_CONV_I);
7414 mono_mb_emit_byte (mb, CEE_ADD);
7415 mono_mb_emit_stloc (mb, new_next_var);
7417 /* tlab_next = new_next */
7418 mono_mb_emit_ldloc (mb, tlab_next_addr_var);
7419 mono_mb_emit_ldloc (mb, new_next_var);
7420 mono_mb_emit_byte (mb, CEE_STIND_I);
7422 /* if (G_LIKELY (new_next < tlab_temp_end)) */
7423 mono_mb_emit_ldloc (mb, new_next_var);
7424 EMIT_TLS_ACCESS (mb, tlab_temp_end, tlab_temp_end_offset);
7425 slowpath_branch = mono_mb_emit_short_branch (mb, MONO_CEE_BLT_UN_S);
7428 if (atype != ATYPE_SMALL)
7429 mono_mb_patch_short_branch (mb, max_size_branch);
7431 mono_mb_emit_byte (mb, MONO_CUSTOM_PREFIX);
7432 mono_mb_emit_byte (mb, CEE_MONO_NOT_TAKEN);
7434 /* FIXME: mono_gc_alloc_obj takes a 'size_t' as an argument, not an int32 */
7435 mono_mb_emit_ldarg (mb, 0);
7436 mono_mb_emit_ldloc (mb, size_var);
7437 if (atype == ATYPE_NORMAL || atype == ATYPE_SMALL) {
7438 mono_mb_emit_icall (mb, mono_gc_alloc_obj);
7439 } else if (atype == ATYPE_VECTOR) {
7440 mono_mb_emit_ldarg (mb, 1);
7441 mono_mb_emit_icall (mb, mono_gc_alloc_vector);
7443 g_assert_not_reached ();
7445 mono_mb_emit_byte (mb, CEE_RET);
7448 mono_mb_patch_short_branch (mb, slowpath_branch);
7450 /* FIXME: Memory barrier */
7453 mono_mb_emit_ldloc (mb, p_var);
7454 mono_mb_emit_ldarg (mb, 0);
7455 mono_mb_emit_byte (mb, CEE_STIND_I);
7457 if (atype == ATYPE_VECTOR) {
7458 /* arr->max_length = max_length; */
7459 mono_mb_emit_ldloc (mb, p_var);
7460 mono_mb_emit_ldflda (mb, G_STRUCT_OFFSET (MonoArray, max_length));
7461 mono_mb_emit_ldarg (mb, 1);
7462 mono_mb_emit_byte (mb, CEE_STIND_I);
7466 mono_mb_emit_ldloc (mb, p_var);
7467 mono_mb_emit_byte (mb, CEE_RET);
7469 res = mono_mb_create_method (mb, csig, 8);
7471 mono_method_get_header (res)->init_locals = FALSE;
7473 info = mono_image_alloc0 (mono_defaults.corlib, sizeof (AllocatorWrapperInfo));
7474 info->alloc_type = atype;
7475 mono_marshal_set_wrapper_info (res, info);
7481 static MonoMethod* alloc_method_cache [ATYPE_NUM];
7482 static MonoMethod *write_barrier_method;
7485 is_ip_in_managed_allocator (MonoDomain *domain, gpointer ip)
7493 ji = mono_jit_info_table_find (domain, ip);
7496 method = ji->method;
7498 if (method == write_barrier_method)
7500 for (i = 0; i < ATYPE_NUM; ++i)
7501 if (method == alloc_method_cache [i])
7507 * Generate an allocator method implementing the fast path of mono_gc_alloc_obj ().
7508 * The signature of the called method is:
7509 * object allocate (MonoVTable *vtable)
7512 mono_gc_get_managed_allocator (MonoVTable *vtable, gboolean for_box)
7514 #ifdef MANAGED_ALLOCATION
7515 MonoClass *klass = vtable->klass;
7517 #ifdef HAVE_KW_THREAD
7518 int tlab_next_offset = -1;
7519 int tlab_temp_end_offset = -1;
7520 MONO_THREAD_VAR_OFFSET (tlab_next, tlab_next_offset);
7521 MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
7523 if (tlab_next_offset == -1 || tlab_temp_end_offset == -1)
7527 if (!mono_runtime_has_tls_get ())
7529 if (klass->instance_size > tlab_size)
7531 if (klass->has_finalize || klass->marshalbyref || (mono_profiler_get_events () & MONO_PROFILE_ALLOCATIONS))
7535 if (klass->byval_arg.type == MONO_TYPE_STRING)
7537 if (collect_before_allocs)
7540 if (ALIGN_TO (klass->instance_size, ALLOC_ALIGN) < MAX_SMALL_OBJ_SIZE)
7541 return mono_gc_get_managed_allocator_by_type (ATYPE_SMALL);
7543 return mono_gc_get_managed_allocator_by_type (ATYPE_NORMAL);
7550 mono_gc_get_managed_array_allocator (MonoVTable *vtable, int rank)
7552 #ifdef MANAGED_ALLOCATION
7553 MonoClass *klass = vtable->klass;
7555 #ifdef HAVE_KW_THREAD
7556 int tlab_next_offset = -1;
7557 int tlab_temp_end_offset = -1;
7558 MONO_THREAD_VAR_OFFSET (tlab_next, tlab_next_offset);
7559 MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
7561 if (tlab_next_offset == -1 || tlab_temp_end_offset == -1)
7567 if (!mono_runtime_has_tls_get ())
7569 if (mono_profiler_get_events () & MONO_PROFILE_ALLOCATIONS)
7571 if (collect_before_allocs)
7573 g_assert (!klass->has_finalize && !klass->marshalbyref);
7575 return mono_gc_get_managed_allocator_by_type (ATYPE_VECTOR);
7582 mono_gc_get_managed_allocator_by_type (int atype)
7584 #ifdef MANAGED_ALLOCATION
7587 if (!mono_runtime_has_tls_get ())
7590 mono_loader_lock ();
7591 res = alloc_method_cache [atype];
7593 res = alloc_method_cache [atype] = create_allocator (atype);
7594 mono_loader_unlock ();
7602 mono_gc_get_managed_allocator_types (void)
7609 mono_gc_get_write_barrier (void)
7612 MonoMethodBuilder *mb;
7613 MonoMethodSignature *sig;
7614 #ifdef MANAGED_WBARRIER
7615 int label_no_wb_1, label_no_wb_2, label_no_wb_3, label_no_wb_4, label_need_wb, label_slow_path;
7616 #ifndef ALIGN_NURSERY
7617 int label_continue_1, label_continue_2, label_no_wb_5;
7618 int dereferenced_var;
7620 int buffer_var, buffer_index_var, dummy_var;
7622 #ifdef HAVE_KW_THREAD
7623 int stack_end_offset = -1, store_remset_buffer_offset = -1;
7624 int store_remset_buffer_index_offset = -1, store_remset_buffer_index_addr_offset = -1;
7626 MONO_THREAD_VAR_OFFSET (stack_end, stack_end_offset);
7627 g_assert (stack_end_offset != -1);
7628 MONO_THREAD_VAR_OFFSET (store_remset_buffer, store_remset_buffer_offset);
7629 g_assert (store_remset_buffer_offset != -1);
7630 MONO_THREAD_VAR_OFFSET (store_remset_buffer_index, store_remset_buffer_index_offset);
7631 g_assert (store_remset_buffer_index_offset != -1);
7632 MONO_THREAD_VAR_OFFSET (store_remset_buffer_index_addr, store_remset_buffer_index_addr_offset);
7633 g_assert (store_remset_buffer_index_addr_offset != -1);
7637 // FIXME: Maybe create a separate version for ctors (the branch would be
7638 // correctly predicted more times)
7639 if (write_barrier_method)
7640 return write_barrier_method;
7642 /* Create the IL version of mono_gc_barrier_generic_store () */
7643 sig = mono_metadata_signature_alloc (mono_defaults.corlib, 1);
7644 sig->ret = &mono_defaults.void_class->byval_arg;
7645 sig->params [0] = &mono_defaults.int_class->byval_arg;
7647 mb = mono_mb_new (mono_defaults.object_class, "wbarrier", MONO_WRAPPER_WRITE_BARRIER);
7649 #ifdef MANAGED_WBARRIER
7650 if (mono_runtime_has_tls_get ()) {
7651 #ifdef ALIGN_NURSERY
7652 // if (ptr_in_nursery (ptr)) return;
7654 * Masking out the bits might be faster, but we would have to use 64 bit
7655 * immediates, which might be slower.
7657 mono_mb_emit_ldarg (mb, 0);
7658 mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
7659 mono_mb_emit_byte (mb, CEE_SHR_UN);
7660 mono_mb_emit_icon (mb, (mword)nursery_start >> DEFAULT_NURSERY_BITS);
7661 label_no_wb_1 = mono_mb_emit_branch (mb, CEE_BEQ);
7663 // if (!ptr_in_nursery (*ptr)) return;
7664 mono_mb_emit_ldarg (mb, 0);
7665 mono_mb_emit_byte (mb, CEE_LDIND_I);
7666 mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
7667 mono_mb_emit_byte (mb, CEE_SHR_UN);
7668 mono_mb_emit_icon (mb, (mword)nursery_start >> DEFAULT_NURSERY_BITS);
7669 label_no_wb_2 = mono_mb_emit_branch (mb, CEE_BNE_UN);
7672 // if (ptr < (nursery_start)) goto continue;
7673 mono_mb_emit_ldarg (mb, 0);
7674 mono_mb_emit_ptr (mb, (gpointer) nursery_start);
7675 label_continue_1 = mono_mb_emit_branch (mb, CEE_BLT);
7677 // if (ptr >= nursery_real_end)) goto continue;
7678 mono_mb_emit_ldarg (mb, 0);
7679 mono_mb_emit_ptr (mb, (gpointer) nursery_real_end);
7680 label_continue_2 = mono_mb_emit_branch (mb, CEE_BGE);
7683 label_no_wb_1 = mono_mb_emit_branch (mb, CEE_BR);
7686 mono_mb_patch_branch (mb, label_continue_1);
7687 mono_mb_patch_branch (mb, label_continue_2);
7689 // Dereference and store in local var
7690 dereferenced_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7691 mono_mb_emit_ldarg (mb, 0);
7692 mono_mb_emit_byte (mb, CEE_LDIND_I);
7693 mono_mb_emit_stloc (mb, dereferenced_var);
7695 // if (*ptr < nursery_start) return;
7696 mono_mb_emit_ldloc (mb, dereferenced_var);
7697 mono_mb_emit_ptr (mb, (gpointer) nursery_start);
7698 label_no_wb_2 = mono_mb_emit_branch (mb, CEE_BLT);
7700 // if (*ptr >= nursery_end) return;
7701 mono_mb_emit_ldloc (mb, dereferenced_var);
7702 mono_mb_emit_ptr (mb, (gpointer) nursery_real_end);
7703 label_no_wb_5 = mono_mb_emit_branch (mb, CEE_BGE);
7706 // if (ptr >= stack_end) goto need_wb;
7707 mono_mb_emit_ldarg (mb, 0);
7708 EMIT_TLS_ACCESS (mb, stack_end, stack_end_offset);
7709 label_need_wb = mono_mb_emit_branch (mb, CEE_BGE_UN);
7711 // if (ptr >= stack_start) return;
7712 dummy_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7713 mono_mb_emit_ldarg (mb, 0);
7714 mono_mb_emit_ldloc_addr (mb, dummy_var);
7715 label_no_wb_3 = mono_mb_emit_branch (mb, CEE_BGE_UN);
7718 mono_mb_patch_branch (mb, label_need_wb);
7720 // buffer = STORE_REMSET_BUFFER;
7721 buffer_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7722 EMIT_TLS_ACCESS (mb, store_remset_buffer, store_remset_buffer_offset);
7723 mono_mb_emit_stloc (mb, buffer_var);
7725 // buffer_index = STORE_REMSET_BUFFER_INDEX;
7726 buffer_index_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7727 EMIT_TLS_ACCESS (mb, store_remset_buffer_index, store_remset_buffer_index_offset);
7728 mono_mb_emit_stloc (mb, buffer_index_var);
7730 // if (buffer [buffer_index] == ptr) return;
7731 mono_mb_emit_ldloc (mb, buffer_var);
7732 mono_mb_emit_ldloc (mb, buffer_index_var);
7733 g_assert (sizeof (gpointer) == 4 || sizeof (gpointer) == 8);
7734 mono_mb_emit_icon (mb, sizeof (gpointer) == 4 ? 2 : 3);
7735 mono_mb_emit_byte (mb, CEE_SHL);
7736 mono_mb_emit_byte (mb, CEE_ADD);
7737 mono_mb_emit_byte (mb, CEE_LDIND_I);
7738 mono_mb_emit_ldarg (mb, 0);
7739 label_no_wb_4 = mono_mb_emit_branch (mb, CEE_BEQ);
7742 mono_mb_emit_ldloc (mb, buffer_index_var);
7743 mono_mb_emit_icon (mb, 1);
7744 mono_mb_emit_byte (mb, CEE_ADD);
7745 mono_mb_emit_stloc (mb, buffer_index_var);
7747 // if (buffer_index >= STORE_REMSET_BUFFER_SIZE) goto slow_path;
7748 mono_mb_emit_ldloc (mb, buffer_index_var);
7749 mono_mb_emit_icon (mb, STORE_REMSET_BUFFER_SIZE);
7750 label_slow_path = mono_mb_emit_branch (mb, CEE_BGE);
7752 // buffer [buffer_index] = ptr;
7753 mono_mb_emit_ldloc (mb, buffer_var);
7754 mono_mb_emit_ldloc (mb, buffer_index_var);
7755 g_assert (sizeof (gpointer) == 4 || sizeof (gpointer) == 8);
7756 mono_mb_emit_icon (mb, sizeof (gpointer) == 4 ? 2 : 3);
7757 mono_mb_emit_byte (mb, CEE_SHL);
7758 mono_mb_emit_byte (mb, CEE_ADD);
7759 mono_mb_emit_ldarg (mb, 0);
7760 mono_mb_emit_byte (mb, CEE_STIND_I);
7762 // STORE_REMSET_BUFFER_INDEX = buffer_index;
7763 EMIT_TLS_ACCESS (mb, store_remset_buffer_index_addr, store_remset_buffer_index_addr_offset);
7764 mono_mb_emit_ldloc (mb, buffer_index_var);
7765 mono_mb_emit_byte (mb, CEE_STIND_I);
7768 mono_mb_patch_branch (mb, label_no_wb_1);
7769 mono_mb_patch_branch (mb, label_no_wb_2);
7770 mono_mb_patch_branch (mb, label_no_wb_3);
7771 mono_mb_patch_branch (mb, label_no_wb_4);
7772 #ifndef ALIGN_NURSERY
7773 mono_mb_patch_branch (mb, label_no_wb_5);
7775 mono_mb_emit_byte (mb, CEE_RET);
7778 mono_mb_patch_branch (mb, label_slow_path);
7782 mono_mb_emit_ldarg (mb, 0);
7783 mono_mb_emit_icall (mb, mono_gc_wbarrier_generic_nostore);
7784 mono_mb_emit_byte (mb, CEE_RET);
7786 res = mono_mb_create_method (mb, sig, 16);
7789 mono_loader_lock ();
7790 if (write_barrier_method) {
7791 /* Already created */
7792 mono_free_method (res);
7794 /* double-checked locking */
7795 mono_memory_barrier ();
7796 write_barrier_method = res;
7798 mono_loader_unlock ();
7800 return write_barrier_method;
7804 mono_gc_get_description (void)
7806 return g_strdup ("sgen");
7810 mono_gc_set_desktop_mode (void)
7815 mono_gc_is_moving (void)
7821 mono_gc_is_disabled (void)
7827 mono_sgen_is_worker_thread (pthread_t thread)
7832 #endif /* HAVE_SGEN_GC */