2 * sgen-gc.c: Simple generational GC.
5 * Paolo Molaro (lupus@ximian.com)
7 * Copyright 2005-2010 Novell, Inc (http://www.novell.com)
9 * Thread start/stop adapted from Boehm's GC:
10 * Copyright (c) 1994 by Xerox Corporation. All rights reserved.
11 * Copyright (c) 1996 by Silicon Graphics. All rights reserved.
12 * Copyright (c) 1998 by Fergus Henderson. All rights reserved.
13 * Copyright (c) 2000-2004 by Hewlett-Packard Company. All rights reserved.
15 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
16 * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
18 * Permission is hereby granted to use or copy this program
19 * for any purpose, provided the above notices are retained on all copies.
20 * Permission to modify the code and to distribute modified code is granted,
21 * provided the above notices are retained, and a notice that the code was
22 * modified is included with the above copyright notice.
25 * Copyright 2001-2003 Ximian, Inc
26 * Copyright 2003-2010 Novell, Inc.
28 * Permission is hereby granted, free of charge, to any person obtaining
29 * a copy of this software and associated documentation files (the
30 * "Software"), to deal in the Software without restriction, including
31 * without limitation the rights to use, copy, modify, merge, publish,
32 * distribute, sublicense, and/or sell copies of the Software, and to
33 * permit persons to whom the Software is furnished to do so, subject to
34 * the following conditions:
36 * The above copyright notice and this permission notice shall be
37 * included in all copies or substantial portions of the Software.
39 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
40 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
41 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
42 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
43 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
44 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
45 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
48 * Important: allocation provides always zeroed memory, having to do
49 * a memset after allocation is deadly for performance.
50 * Memory usage at startup is currently as follows:
52 * 64 KB internal space
54 * We should provide a small memory config with half the sizes
56 * We currently try to make as few mono assumptions as possible:
57 * 1) 2-word header with no GC pointers in it (first vtable, second to store the
59 * 2) gc descriptor is the second word in the vtable (first word in the class)
60 * 3) 8 byte alignment is the minimum and enough (not true for special structures (SIMD), FIXME)
61 * 4) there is a function to get an object's size and the number of
62 * elements in an array.
63 * 5) we know the special way bounds are allocated for complex arrays
64 * 6) we know about proxies and how to treat them when domains are unloaded
66 * Always try to keep stack usage to a minimum: no recursive behaviour
67 * and no large stack allocs.
69 * General description.
70 * Objects are initially allocated in a nursery using a fast bump-pointer technique.
71 * When the nursery is full we start a nursery collection: this is performed with a
73 * When the old generation is full we start a copying GC of the old generation as well:
74 * this will be changed to mark&sweep with copying when fragmentation becomes to severe
75 * in the future. Maybe we'll even do both during the same collection like IMMIX.
77 * The things that complicate this description are:
78 * *) pinned objects: we can't move them so we need to keep track of them
79 * *) no precise info of the thread stacks and registers: we need to be able to
80 * quickly find the objects that may be referenced conservatively and pin them
81 * (this makes the first issues more important)
82 * *) large objects are too expensive to be dealt with using copying GC: we handle them
83 * with mark/sweep during major collections
84 * *) some objects need to not move even if they are small (interned strings, Type handles):
85 * we use mark/sweep for them, too: they are not allocated in the nursery, but inside
86 * PinnedChunks regions
92 *) we could have a function pointer in MonoClass to implement
93 customized write barriers for value types
95 *) investigate the stuff needed to advance a thread to a GC-safe
96 point (single-stepping, read from unmapped memory etc) and implement it.
97 This would enable us to inline allocations and write barriers, for example,
98 or at least parts of them, like the write barrier checks.
99 We may need this also for handling precise info on stacks, even simple things
100 as having uninitialized data on the stack and having to wait for the prolog
101 to zero it. Not an issue for the last frame that we scan conservatively.
102 We could always not trust the value in the slots anyway.
104 *) modify the jit to save info about references in stack locations:
105 this can be done just for locals as a start, so that at least
106 part of the stack is handled precisely.
108 *) test/fix endianess issues
110 *) Implement a card table as the write barrier instead of remembered
111 sets? Card tables are not easy to implement with our current
112 memory layout. We have several different kinds of major heap
113 objects: Small objects in regular blocks, small objects in pinned
114 chunks and LOS objects. If we just have a pointer we have no way
115 to tell which kind of object it points into, therefore we cannot
116 know where its card table is. The least we have to do to make
117 this happen is to get rid of write barriers for indirect stores.
120 *) Get rid of write barriers for indirect stores. We can do this by
121 telling the GC to wbarrier-register an object once we do an ldloca
122 or ldelema on it, and to unregister it once it's not used anymore
123 (it can only travel downwards on the stack). The problem with
124 unregistering is that it needs to happen eventually no matter
125 what, even if exceptions are thrown, the thread aborts, etc.
126 Rodrigo suggested that we could do only the registering part and
127 let the collector find out (pessimistically) when it's safe to
128 unregister, namely when the stack pointer of the thread that
129 registered the object is higher than it was when the registering
130 happened. This might make for a good first implementation to get
131 some data on performance.
133 *) Some sort of blacklist support? Blacklists is a concept from the
134 Boehm GC: if during a conservative scan we find pointers to an
135 area which we might use as heap, we mark that area as unusable, so
136 pointer retention by random pinning pointers is reduced.
138 *) experiment with max small object size (very small right now - 2kb,
139 because it's tied to the max freelist size)
141 *) add an option to mmap the whole heap in one chunk: it makes for many
142 simplifications in the checks (put the nursery at the top and just use a single
143 check for inclusion/exclusion): the issue this has is that on 32 bit systems it's
144 not flexible (too much of the address space may be used by default or we can't
145 increase the heap as needed) and we'd need a race-free mechanism to return memory
146 back to the system (mprotect(PROT_NONE) will still keep the memory allocated if it
147 was written to, munmap is needed, but the following mmap may not find the same segment
150 *) memzero the major fragments after restarting the world and optionally a smaller
153 *) investigate having fragment zeroing threads
155 *) separate locks for finalization and other minor stuff to reduce
158 *) try a different copying order to improve memory locality
160 *) a thread abort after a store but before the write barrier will
161 prevent the write barrier from executing
163 *) specialized dynamically generated markers/copiers
165 *) Dynamically adjust TLAB size to the number of threads. If we have
166 too many threads that do allocation, we might need smaller TLABs,
167 and we might get better performance with larger TLABs if we only
168 have a handful of threads. We could sum up the space left in all
169 assigned TLABs and if that's more than some percentage of the
170 nursery size, reduce the TLAB size.
179 #include <semaphore.h>
184 #include "metadata/metadata-internals.h"
185 #include "metadata/class-internals.h"
186 #include "metadata/gc-internal.h"
187 #include "metadata/object-internals.h"
188 #include "metadata/threads.h"
189 #include "metadata/sgen-gc.h"
190 #include "metadata/sgen-archdep.h"
191 #include "metadata/mono-gc.h"
192 #include "metadata/method-builder.h"
193 #include "metadata/profiler-private.h"
194 #include "metadata/monitor.h"
195 #include "metadata/threadpool-internals.h"
196 #include "metadata/mempool-internals.h"
197 #include "metadata/marshal.h"
198 #include "utils/mono-mmap.h"
199 #include "utils/mono-time.h"
200 #include "utils/mono-semaphore.h"
201 #include "utils/mono-counters.h"
203 #include <mono/utils/memcheck.h>
205 #define OPDEF(a,b,c,d,e,f,g,h,i,j) \
209 #include "mono/cil/opcode.def"
216 * ######################################################################
217 * ######## Types and constants used by the GC.
218 * ######################################################################
220 #if SIZEOF_VOID_P == 4
221 typedef guint32 mword;
223 typedef guint64 mword;
226 static int gc_initialized = 0;
227 static int gc_debug_level = 0;
228 static FILE* gc_debug_file;
229 /* If set, do a minor collection before every allocation */
230 static gboolean collect_before_allocs = FALSE;
231 /* If set, do a heap consistency check before each minor collection */
232 static gboolean consistency_check_at_minor_collection = FALSE;
233 /* If set, check that there are no references to the domain left at domain unload */
234 static gboolean xdomain_checks = FALSE;
235 /* If not null, dump the heap after each collection into this file */
236 static FILE *heap_dump_file = NULL;
237 /* If set, mark stacks conservatively, even if precise marking is possible */
238 static gboolean conservative_stack_mark = TRUE;
239 /* If set, do a plausibility check on the scan_starts before and after
241 static gboolean do_scan_starts_check = FALSE;
244 * Turning on heavy statistics will turn off the managed allocator and
245 * the managed write barrier.
247 //#define HEAVY_STATISTICS
249 #ifdef HEAVY_STATISTICS
250 #define HEAVY_STAT(x) x
252 #define HEAVY_STAT(x)
255 #ifdef HEAVY_STATISTICS
256 static long stat_objects_alloced = 0;
257 static long stat_bytes_alloced = 0;
258 static long stat_objects_alloced_degraded = 0;
259 static long stat_bytes_alloced_degraded = 0;
260 static long stat_bytes_alloced_los = 0;
261 static long stat_copy_object_called_nursery = 0;
262 static long stat_objects_copied_nursery = 0;
263 static long stat_copy_object_called_major = 0;
264 static long stat_objects_copied_major = 0;
266 static long stat_copy_object_failed_from_space = 0;
267 static long stat_copy_object_failed_forwarded = 0;
268 static long stat_copy_object_failed_pinned = 0;
269 static long stat_copy_object_failed_large_pinned = 0;
270 static long stat_copy_object_failed_to_space = 0;
272 static long stat_store_remsets = 0;
273 static long stat_store_remsets_unique = 0;
274 static long stat_saved_remsets_1 = 0;
275 static long stat_saved_remsets_2 = 0;
276 static long stat_global_remsets_added = 0;
277 static long stat_global_remsets_processed = 0;
279 static long num_copy_object_called = 0;
280 static long num_objects_copied = 0;
282 static int stat_wbarrier_set_field = 0;
283 static int stat_wbarrier_set_arrayref = 0;
284 static int stat_wbarrier_arrayref_copy = 0;
285 static int stat_wbarrier_generic_store = 0;
286 static int stat_wbarrier_generic_store_remset = 0;
287 static int stat_wbarrier_set_root = 0;
288 static int stat_wbarrier_value_copy = 0;
289 static int stat_wbarrier_object_copy = 0;
292 static long time_minor_pre_collection_fragment_clear = 0;
293 static long time_minor_pinning = 0;
294 static long time_minor_scan_remsets = 0;
295 static long time_minor_scan_pinned = 0;
296 static long time_minor_scan_registered_roots = 0;
297 static long time_minor_scan_thread_data = 0;
298 static long time_minor_finish_gray_stack = 0;
299 static long time_minor_fragment_creation = 0;
301 static long time_major_pre_collection_fragment_clear = 0;
302 static long time_major_pinning = 0;
303 static long time_major_scan_pinned = 0;
304 static long time_major_scan_registered_roots = 0;
305 static long time_major_scan_thread_data = 0;
306 static long time_major_scan_alloc_pinned = 0;
307 static long time_major_scan_finalized = 0;
308 static long time_major_scan_big_objects = 0;
309 static long time_major_finish_gray_stack = 0;
310 static long time_major_sweep = 0;
311 static long time_major_fragment_creation = 0;
313 static long pinned_chunk_bytes_alloced = 0;
314 static long large_internal_bytes_alloced = 0;
317 INTERNAL_MEM_PIN_QUEUE,
318 INTERNAL_MEM_FRAGMENT,
319 INTERNAL_MEM_SECTION,
320 INTERNAL_MEM_SCAN_STARTS,
321 INTERNAL_MEM_FIN_TABLE,
322 INTERNAL_MEM_FINALIZE_ENTRY,
323 INTERNAL_MEM_DISLINK_TABLE,
324 INTERNAL_MEM_DISLINK,
325 INTERNAL_MEM_ROOTS_TABLE,
326 INTERNAL_MEM_ROOT_RECORD,
327 INTERNAL_MEM_STATISTICS,
329 INTERNAL_MEM_GRAY_QUEUE,
330 INTERNAL_MEM_STORE_REMSET,
331 INTERNAL_MEM_MS_TABLES,
332 INTERNAL_MEM_MS_BLOCK_INFO,
336 static long small_internal_mem_bytes [INTERNAL_MEM_MAX];
340 mono_gc_flush_info (void)
342 fflush (gc_debug_file);
346 #define MAX_DEBUG_LEVEL 2
347 #define DEBUG(level,a) do {if (G_UNLIKELY ((level) <= MAX_DEBUG_LEVEL && (level) <= gc_debug_level)) a;} while (0)
349 /* Define this to allow the user to change some of the constants by specifying
350 * their values in the MONO_GC_PARAMS environmental variable. See
351 * mono_gc_base_init for details. */
352 #define USER_CONFIG 1
354 #define TV_DECLARE(name) gint64 name
355 #define TV_GETTIME(tv) tv = mono_100ns_ticks ()
356 #define TV_ELAPSED(start,end) (int)((end-start) / 10)
357 #define TV_ELAPSED_MS(start,end) ((TV_ELAPSED((start),(end)) + 500) / 1000)
359 #define ALIGN_TO(val,align) ((((guint64)val) + ((align) - 1)) & ~((align) - 1))
361 #define GC_BITS_PER_WORD (sizeof (mword) * 8)
369 typedef struct _Block Block;
375 /* each request from the OS ends up in a GCMemSection */
376 typedef struct _GCMemSection GCMemSection;
377 struct _GCMemSection {
381 /* pointer where more data could be allocated if it fits */
385 * scan starts is an array of pointers to objects equally spaced in the allocation area
386 * They let use quickly find pinned objects from pinning pointers.
389 /* in major collections indexes in the pin_queue for objects that pin this section */
392 unsigned short num_scan_start;
393 gboolean is_to_space;
396 #define SIZEOF_GC_MEM_SECTION ((sizeof (GCMemSection) + 7) & ~7)
398 /* large object space struct: 64+ KB */
399 /* we could make this limit much smaller to avoid memcpy copy
400 * and potentially have more room in the GC descriptor: need to measure
401 * This also means that such small OS objects will need to be
402 * allocated in a different way (using pinned chunks).
403 * We may want to put large but smaller than 64k objects in the fixed space
404 * when we move the object from one generation to another (to limit the
405 * pig in the snake effect).
406 * Note: it may be worth to have an optimized copy function, since we can
407 * assume that objects are aligned and have a multiple of 8 size.
408 * FIXME: This structure needs to be a multiple of 8 bytes in size: this is not
409 * true if MONO_ZERO_LEN_ARRAY is nonzero.
411 typedef struct _LOSObject LOSObject;
414 mword size; /* this is the object size */
416 int dummy; /* to have a sizeof (LOSObject) a multiple of ALLOC_ALIGN and data starting at same alignment */
417 char data [MONO_ZERO_LEN_ARRAY];
420 /* Pinned objects are allocated in the LOS space if bigger than half a page
421 * or from freelists otherwise. We assume that pinned objects are relatively few
422 * and they have a slow dying speed (like interned strings, thread objects).
423 * As such they will be collected only at major collections.
424 * free lists are not global: when we need memory we allocate a PinnedChunk.
425 * Each pinned chunk is made of several pages, the first of wich is used
426 * internally for bookeeping (here think of a page as 4KB). The bookeeping
427 * includes the freelists vectors and info about the object size of each page
428 * in the pinned chunk. So, when needed, a free page is found in a pinned chunk,
429 * a size is assigned to it, the page is divided in the proper chunks and each
430 * chunk is added to the freelist. To not waste space, the remaining space in the
431 * first page is used as objects of size 16 or 32 (need to measure which are more
433 * We use this same structure to allocate memory used internally by the GC, so
434 * we never use malloc/free if we need to alloc during collection: the world is stopped
435 * and malloc/free will deadlock.
436 * When we want to iterate over pinned objects, we just scan a page at a time
437 * linearly according to the size of objects in the page: the next pointer used to link
438 * the items in the freelist uses the same word as the vtable. Since we keep freelists
439 * for each pinned chunk, if the word points outside the pinned chunk it means
441 * We could avoid this expensive scanning in creative ways. We could have a policy
442 * of putting in the pinned space only objects we know about that have no struct fields
443 * with references and we can easily use a even expensive write barrier for them,
444 * since pointer writes on such objects should be rare.
445 * The best compromise is to just alloc interned strings and System.MonoType in them.
446 * It would be nice to allocate MonoThread in it, too: must check that we properly
447 * use write barriers so we don't have to do any expensive scanning of the whole pinned
448 * chunk list during minor collections. We can avoid it now because we alloc in it only
449 * reference-free objects.
451 #define PINNED_FIRST_SLOT_SIZE (sizeof (gpointer) * 4)
452 #define MAX_FREELIST_SIZE 2048
453 #define PINNED_PAGE_SIZE (4096)
454 #define PINNED_CHUNK_MIN_SIZE (4096*8)
455 typedef struct _PinnedChunk PinnedChunk;
456 struct _PinnedChunk {
459 int *page_sizes; /* a 0 means the page is still unused */
462 void *data [1]; /* page sizes and free lists are stored here */
465 /* The method used to clear the nursery */
466 /* Clearing at nursery collections is the safest, but has bad interactions with caches.
467 * Clearing at TLAB creation is much faster, but more complex and it might expose hard
472 CLEAR_AT_TLAB_CREATION
473 } NurseryClearPolicy;
475 static NurseryClearPolicy nursery_clear_policy = CLEAR_AT_TLAB_CREATION;
478 * If this is set, the nursery is aligned to an address aligned to its size, ie.
479 * a 1MB nursery will be aligned to an address divisible by 1MB. This allows us to
480 * speed up ptr_in_nursery () checks which are very frequent. This requires the
481 * nursery size to be a compile time constant.
483 #define ALIGN_NURSERY 1
486 * The young generation is divided into fragments. This is because
487 * we can hand one fragments to a thread for lock-less fast alloc and
488 * because the young generation ends up fragmented anyway by pinned objects.
489 * Once a collection is done, a list of fragments is created. When doing
490 * thread local alloc we use smallish nurseries so we allow new threads to
491 * allocate memory from gen0 without triggering a collection. Threads that
492 * are found to allocate lots of memory are given bigger fragments. This
493 * should make the finalizer thread use little nursery memory after a while.
494 * We should start assigning threads very small fragments: if there are many
495 * threads the nursery will be full of reserved space that the threads may not
496 * use at all, slowing down allocation speed.
497 * Thread local allocation is done from areas of memory Hotspot calls Thread Local
498 * Allocation Buffers (TLABs).
500 typedef struct _Fragment Fragment;
504 char *fragment_start;
505 char *fragment_limit; /* the current soft limit for allocation */
509 /* the runtime can register areas of memory as roots: we keep two lists of roots,
510 * a pinned root set for conservatively scanned roots and a normal one for
511 * precisely scanned roots (currently implemented as a single list).
513 typedef struct _RootRecord RootRecord;
521 /* for use with write barriers */
522 typedef struct _RememberedSet RememberedSet;
523 struct _RememberedSet {
527 mword data [MONO_ZERO_LEN_ARRAY];
531 * We're never actually using the first element. It's always set to
532 * NULL to simplify the elimination of consecutive duplicate
535 #define STORE_REMSET_BUFFER_SIZE 1024
537 typedef struct _GenericStoreRememberedSet GenericStoreRememberedSet;
538 struct _GenericStoreRememberedSet {
539 GenericStoreRememberedSet *next;
540 /* We need one entry less because the first entry of store
541 remset buffers is always a dummy and we don't copy it. */
542 gpointer data [STORE_REMSET_BUFFER_SIZE - 1];
545 /* we have 4 possible values in the low 2 bits */
547 REMSET_LOCATION, /* just a pointer to the exact location */
548 REMSET_RANGE, /* range of pointer fields */
549 REMSET_OBJECT, /* mark all the object for scanning */
550 REMSET_OTHER, /* all others */
551 REMSET_TYPE_MASK = 0x3
554 /* Subtypes of REMSET_OTHER */
556 REMSET_VTYPE, /* a valuetype array described by a gc descriptor and a count */
557 REMSET_ROOT_LOCATION, /* a location inside a root */
560 #ifdef HAVE_KW_THREAD
561 static __thread RememberedSet *remembered_set MONO_TLS_FAST;
563 static pthread_key_t remembered_set_key;
564 static RememberedSet *global_remset;
565 static RememberedSet *freed_thread_remsets;
566 static GenericStoreRememberedSet *generic_store_remsets = NULL;
568 /* FIXME: later choose a size that takes into account the RememberedSet struct
569 * and doesn't waste any alloc paddin space.
571 #define DEFAULT_REMSET_SIZE 1024
572 static RememberedSet* alloc_remset (int size, gpointer id);
574 /* Structure that corresponds to a MonoVTable: desc is a mword so requires
575 * no cast from a pointer to an integer
582 /* these bits are set in the object vtable: we could merge them since an object can be
583 * either pinned or forwarded but not both.
584 * We store them in the vtable slot because the bits are used in the sync block for
585 * other purposes: if we merge them and alloc the sync blocks aligned to 8 bytes, we can change
586 * this and use bit 3 in the syncblock (with the lower two bits both set for forwarded, that
587 * would be an invalid combination for the monitor and hash code).
588 * The values are already shifted.
589 * The forwarding address is stored in the sync block.
591 #define FORWARDED_BIT 1
593 #define VTABLE_BITS_MASK 0x3
595 /* returns NULL if not forwarded, or the forwarded address */
596 #define object_is_forwarded(obj) (((mword*)(obj))[0] & FORWARDED_BIT? (void*)(((mword*)(obj))[1]): NULL)
597 /* set the forwarded address fw_addr for object obj */
598 #define forward_object(obj,fw_addr) do { \
599 ((mword*)(obj))[0] |= FORWARDED_BIT; \
600 ((mword*)(obj))[1] = (mword)(fw_addr); \
603 #define object_is_pinned(obj) (((mword*)(obj))[0] & PINNED_BIT)
604 #define pin_object(obj) do { \
605 ((mword*)(obj))[0] |= PINNED_BIT; \
607 #define unpin_object(obj) do { \
608 ((mword*)(obj))[0] &= ~PINNED_BIT; \
612 #define ptr_in_nursery(ptr) (((mword)(ptr) & ~((1 << DEFAULT_NURSERY_BITS) - 1)) == (mword)nursery_start)
614 #define ptr_in_nursery(ptr) ((char*)(ptr) >= nursery_start && (char*)(ptr) < nursery_real_end)
618 * Since we set bits in the vtable, use the macro to load it from the pointer to
619 * an object that is potentially pinned.
621 #define LOAD_VTABLE(addr) ((*(mword*)(addr)) & ~VTABLE_BITS_MASK)
624 safe_name (void* obj)
626 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (obj);
627 return vt->klass->name;
631 safe_object_get_size (MonoObject* o)
633 MonoClass *klass = ((MonoVTable*)LOAD_VTABLE (o))->klass;
634 if (klass == mono_defaults.string_class) {
635 return sizeof (MonoString) + 2 * mono_string_length ((MonoString*) o) + 2;
636 } else if (klass->rank) {
637 MonoArray *array = (MonoArray*)o;
638 size_t size = sizeof (MonoArray) + klass->sizes.element_size * mono_array_length (array);
639 if (G_UNLIKELY (array->bounds)) {
640 size += sizeof (mono_array_size_t) - 1;
641 size &= ~(sizeof (mono_array_size_t) - 1);
642 size += sizeof (MonoArrayBounds) * klass->rank;
646 /* from a created object: the class must be inited already */
647 return klass->instance_size;
652 * ######################################################################
653 * ######## Global data.
654 * ######################################################################
656 static LOCK_DECLARE (gc_mutex);
657 static int gc_disabled = 0;
658 static int num_minor_gcs = 0;
659 static int num_major_gcs = 0;
663 /* good sizes are 512KB-1MB: larger ones increase a lot memzeroing time */
664 #define DEFAULT_NURSERY_SIZE (default_nursery_size)
665 static int default_nursery_size = (1 << 20);
667 /* The number of trailing 0 bits in DEFAULT_NURSERY_SIZE */
668 #define DEFAULT_NURSERY_BITS (default_nursery_bits)
669 static int default_nursery_bits = 20;
674 #define DEFAULT_NURSERY_SIZE (1024*512*2)
676 #define DEFAULT_NURSERY_BITS 20
681 #define MIN_LOS_ALLOWANCE (DEFAULT_NURSERY_SIZE * 2)
682 /* to quickly find the head of an object pinned by a conservative address
683 * we keep track of the objects allocated for each SCAN_START_SIZE memory
684 * chunk in the nursery or other memory sections. Larger values have less
685 * memory overhead and bigger runtime cost. 4-8 KB are reasonable values.
687 #define SCAN_START_SIZE (4096*2)
688 /* the minimum size of a fragment that we consider useful for allocation */
689 #define FRAGMENT_MIN_SIZE (512)
690 /* This is a fixed value used for pinned chunks, not the system pagesize */
691 #define FREELIST_PAGESIZE 4096
693 static mword pagesize = 4096;
694 static mword nursery_size;
695 static int degraded_mode = 0;
697 static LOSObject *los_object_list = NULL;
698 static mword los_memory_usage = 0;
699 static mword los_num_objects = 0;
700 static mword next_los_collection = 2*1024*1024; /* 2 MB, need to tune */
701 static mword total_alloc = 0;
702 /* use this to tune when to do a major/minor collection */
703 static mword memory_pressure = 0;
705 static GCMemSection *nursery_section = NULL;
706 static mword lowest_heap_address = ~(mword)0;
707 static mword highest_heap_address = 0;
709 static LOCK_DECLARE (interruption_mutex);
711 typedef struct _FinalizeEntry FinalizeEntry;
712 struct _FinalizeEntry {
717 typedef struct _FinalizeEntryHashTable FinalizeEntryHashTable;
718 struct _FinalizeEntryHashTable {
719 FinalizeEntry **table;
724 typedef struct _DisappearingLink DisappearingLink;
725 struct _DisappearingLink {
726 DisappearingLink *next;
730 typedef struct _DisappearingLinkHashTable DisappearingLinkHashTable;
731 struct _DisappearingLinkHashTable {
732 DisappearingLink **table;
737 #define LARGE_INTERNAL_MEM_HEADER_MAGIC 0x7d289f3a
739 typedef struct _LargeInternalMemHeader LargeInternalMemHeader;
740 struct _LargeInternalMemHeader {
752 int current_collection_generation = -1;
755 * The link pointer is hidden by negating each bit. We use the lowest
756 * bit of the link (before negation) to store whether it needs
757 * resurrection tracking.
759 #define HIDE_POINTER(p,t) ((gpointer)(~((gulong)(p)|((t)?1:0))))
760 #define REVEAL_POINTER(p) ((gpointer)((~(gulong)(p))&~3L))
762 #define DISLINK_OBJECT(d) (REVEAL_POINTER (*(d)->link))
763 #define DISLINK_TRACK(d) ((~(gulong)(*(d)->link)) & 1)
766 * The finalizable hash has the object as the key, the
767 * disappearing_link hash, has the link address as key.
769 static FinalizeEntryHashTable minor_finalizable_hash;
770 static FinalizeEntryHashTable major_finalizable_hash;
771 /* objects that are ready to be finalized */
772 static FinalizeEntry *fin_ready_list = NULL;
773 static FinalizeEntry *critical_fin_list = NULL;
775 static DisappearingLinkHashTable minor_disappearing_link_hash;
776 static DisappearingLinkHashTable major_disappearing_link_hash;
778 static int num_ready_finalizers = 0;
779 static int no_finalize = 0;
781 /* keep each size a multiple of ALLOC_ALIGN */
782 /* on 64 bit systems 8 is likely completely unused. */
783 static const int freelist_sizes [] = {
784 8, 16, 24, 32, 40, 48, 64, 80,
785 96, 128, 160, 192, 224, 256, 320, 384,
786 448, 512, 584, 680, 816, 1024, 1360, 2048};
787 #define FREELIST_NUM_SLOTS (sizeof (freelist_sizes) / sizeof (freelist_sizes [0]))
789 /* This is also the MAJOR_SECTION_SIZE for the copying major
791 #define PINNED_CHUNK_SIZE (128 * 1024)
793 /* internal_chunk_list is used for allocating structures needed by the GC */
794 static PinnedChunk *internal_chunk_list = NULL;
796 static int slot_for_size (size_t size);
799 ROOT_TYPE_NORMAL = 0, /* "normal" roots */
800 ROOT_TYPE_PINNED = 1, /* roots without a GC descriptor */
801 ROOT_TYPE_WBARRIER = 2, /* roots with a write barrier */
805 /* registered roots: the key to the hash is the root start address */
807 * Different kinds of roots are kept separate to speed up pin_from_roots () for example.
809 static RootRecord **roots_hash [ROOT_TYPE_NUM] = { NULL, NULL };
810 static int roots_hash_size [ROOT_TYPE_NUM] = { 0, 0, 0 };
811 static mword roots_size = 0; /* amount of memory in the root set */
812 static int num_roots_entries [ROOT_TYPE_NUM] = { 0, 0, 0 };
815 * The current allocation cursors
816 * We allocate objects in the nursery.
817 * The nursery is the area between nursery_start and nursery_real_end.
818 * Allocation is done from a Thread Local Allocation Buffer (TLAB). TLABs are allocated
819 * from nursery fragments.
820 * tlab_next is the pointer to the space inside the TLAB where the next object will
822 * tlab_temp_end is the pointer to the end of the temporary space reserved for
823 * the allocation: it allows us to set the scan starts at reasonable intervals.
824 * tlab_real_end points to the end of the TLAB.
825 * nursery_frag_real_end points to the end of the currently used nursery fragment.
826 * nursery_first_pinned_start points to the start of the first pinned object in the nursery
827 * nursery_last_pinned_end points to the end of the last pinned object in the nursery
828 * At the next allocation, the area of the nursery where objects can be present is
829 * between MIN(nursery_first_pinned_start, first_fragment_start) and
830 * MAX(nursery_last_pinned_end, nursery_frag_real_end)
832 static char *nursery_start = NULL;
834 /* eventually share with MonoThread? */
835 typedef struct _SgenThreadInfo SgenThreadInfo;
837 struct _SgenThreadInfo {
838 SgenThreadInfo *next;
840 unsigned int stop_count; /* to catch duplicate signals */
843 volatile int in_critical_region;
846 void *stack_start_limit;
847 char **tlab_next_addr;
848 char **tlab_start_addr;
849 char **tlab_temp_end_addr;
850 char **tlab_real_end_addr;
851 gpointer **store_remset_buffer_addr;
852 long *store_remset_buffer_index_addr;
853 RememberedSet *remset;
854 gpointer runtime_data;
855 gpointer stopped_ip; /* only valid if the thread is stopped */
856 MonoDomain *stopped_domain; /* ditto */
857 gpointer *stopped_regs; /* ditto */
858 #ifndef HAVE_KW_THREAD
863 gpointer *store_remset_buffer;
864 long store_remset_buffer_index;
868 #ifdef HAVE_KW_THREAD
869 #define TLAB_ACCESS_INIT
870 #define TLAB_START tlab_start
871 #define TLAB_NEXT tlab_next
872 #define TLAB_TEMP_END tlab_temp_end
873 #define TLAB_REAL_END tlab_real_end
874 #define REMEMBERED_SET remembered_set
875 #define STORE_REMSET_BUFFER store_remset_buffer
876 #define STORE_REMSET_BUFFER_INDEX store_remset_buffer_index
877 #define IN_CRITICAL_REGION thread_info->in_critical_region
879 static pthread_key_t thread_info_key;
880 #define TLAB_ACCESS_INIT SgenThreadInfo *__thread_info__ = pthread_getspecific (thread_info_key)
881 #define TLAB_START (__thread_info__->tlab_start)
882 #define TLAB_NEXT (__thread_info__->tlab_next)
883 #define TLAB_TEMP_END (__thread_info__->tlab_temp_end)
884 #define TLAB_REAL_END (__thread_info__->tlab_real_end)
885 #define REMEMBERED_SET (__thread_info__->remset)
886 #define STORE_REMSET_BUFFER (__thread_info__->store_remset_buffer)
887 #define STORE_REMSET_BUFFER_INDEX (__thread_info__->store_remset_buffer_index)
888 #define IN_CRITICAL_REGION (__thread_info__->in_critical_region)
891 /* we use the memory barrier only to prevent compiler reordering (a memory constraint may be enough) */
892 #define ENTER_CRITICAL_REGION do {IN_CRITICAL_REGION = 1;mono_memory_barrier ();} while (0)
893 #define EXIT_CRITICAL_REGION do {IN_CRITICAL_REGION = 0;mono_memory_barrier ();} while (0)
896 * FIXME: What is faster, a TLS variable pointing to a structure, or separate TLS
897 * variables for next+temp_end ?
899 #ifdef HAVE_KW_THREAD
900 static __thread SgenThreadInfo *thread_info;
901 static __thread char *tlab_start;
902 static __thread char *tlab_next;
903 static __thread char *tlab_temp_end;
904 static __thread char *tlab_real_end;
905 static __thread gpointer *store_remset_buffer;
906 static __thread long store_remset_buffer_index;
907 /* Used by the managed allocator/wbarrier */
908 static __thread char **tlab_next_addr;
909 static __thread char *stack_end;
910 static __thread long *store_remset_buffer_index_addr;
912 static char *nursery_next = NULL;
913 static char *nursery_frag_real_end = NULL;
914 static char *nursery_real_end = NULL;
915 static char *nursery_last_pinned_end = NULL;
917 /* The size of a TLAB */
918 /* The bigger the value, the less often we have to go to the slow path to allocate a new
919 * one, but the more space is wasted by threads not allocating much memory.
921 * FIXME: Make this self-tuning for each thread.
923 static guint32 tlab_size = (1024 * 4);
925 /* fragments that are free and ready to be used for allocation */
926 static Fragment *nursery_fragments = NULL;
927 /* freeelist of fragment structures */
928 static Fragment *fragment_freelist = NULL;
930 /* objects bigger then this go into the large object space */
931 #define MAX_SMALL_OBJ_SIZE 2040
933 /* Functions supplied by the runtime to be called by the GC */
934 static MonoGCCallbacks gc_callbacks;
936 #define ALLOC_ALIGN 8
938 #define MOVED_OBJECTS_NUM 64
939 static void *moved_objects [MOVED_OBJECTS_NUM];
940 static int moved_objects_idx = 0;
943 * ######################################################################
944 * ######## Macros and function declarations.
945 * ######################################################################
948 #define UPDATE_HEAP_BOUNDARIES(low,high) do { \
949 if ((mword)(low) < lowest_heap_address) \
950 lowest_heap_address = (mword)(low); \
951 if ((mword)(high) > highest_heap_address) \
952 highest_heap_address = (mword)(high); \
954 #define ADDR_IN_HEAP_BOUNDARIES(addr) ((p) >= lowest_heap_address && (p) < highest_heap_address)
957 align_pointer (void *ptr)
959 mword p = (mword)ptr;
960 p += sizeof (gpointer) - 1;
961 p &= ~ (sizeof (gpointer) - 1);
965 typedef void (*CopyOrMarkObjectFunc) (void**);
966 typedef char* (*ScanObjectFunc) (char*);
968 /* forward declarations */
969 static void* get_internal_mem (size_t size, int type);
970 static void free_internal_mem (void *addr, int type);
971 static void* get_os_memory (size_t size, int activate);
972 static void* get_os_memory_aligned (mword size, mword alignment, gboolean activate);
973 static void free_os_memory (void *addr, size_t size);
974 static G_GNUC_UNUSED void report_internal_mem_usage (void);
976 static int stop_world (void);
977 static int restart_world (void);
978 static void add_to_global_remset (gpointer ptr, gboolean root);
979 static void scan_thread_data (void *start_nursery, void *end_nursery, gboolean precise);
980 static void scan_from_remsets (void *start_nursery, void *end_nursery);
981 static void scan_from_registered_roots (CopyOrMarkObjectFunc copy_func, char *addr_start, char *addr_end, int root_type);
982 static void scan_finalizer_entries (CopyOrMarkObjectFunc copy_func, FinalizeEntry *list);
983 static void find_pinning_ref_from_thread (char *obj, size_t size);
984 static void update_current_thread_stack (void *start);
985 static void finalize_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, int generation);
986 static void add_or_remove_disappearing_link (MonoObject *obj, void **link, gboolean track, int generation);
987 static void null_link_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, int generation);
988 static void null_links_for_domain (MonoDomain *domain, int generation);
989 static gboolean search_fragment_for_size (size_t size);
990 static void build_nursery_fragments (int start_pin, int end_pin);
991 static void clear_nursery_fragments (char *next);
992 static void pin_from_roots (void *start_nursery, void *end_nursery);
993 static int pin_objects_from_addresses (GCMemSection *section, void **start, void **end, void *start_nursery, void *end_nursery);
994 static void pin_objects_in_section (GCMemSection *section);
995 static void optimize_pin_queue (int start_slot);
996 static void clear_remsets (void);
997 static void clear_tlabs (void);
998 typedef void (*IterateObjectCallbackFunc) (char*, size_t, void*);
999 static void scan_area_with_callback (char *start, char *end, IterateObjectCallbackFunc callback, void *data);
1000 static char* scan_object (char *start);
1001 static char* major_scan_object (char *start);
1002 static void* copy_object_no_checks (void *obj);
1003 static void copy_object (void **obj_slot);
1004 static void* get_chunk_freelist (PinnedChunk *chunk, int slot);
1005 static PinnedChunk* alloc_pinned_chunk (void);
1006 static void free_large_object (LOSObject *obj);
1007 static void sort_addresses (void **array, int size);
1008 static void drain_gray_stack (void);
1009 static void finish_gray_stack (char *start_addr, char *end_addr, int generation);
1011 static void mono_gc_register_disappearing_link (MonoObject *obj, void **link, gboolean track);
1013 void describe_ptr (char *ptr);
1014 static void check_consistency (void);
1015 static void check_section_scan_starts (GCMemSection *section);
1016 static void check_scan_starts (void);
1017 static void check_for_xdomain_refs (void);
1018 static void dump_section (GCMemSection *section, const char *type);
1019 static void dump_heap (const char *type, int num, const char *reason);
1020 static void commit_stats (int generation);
1021 static void report_pinned_chunk (PinnedChunk *chunk, int seq);
1023 void mono_gc_scan_for_specific_ref (MonoObject *key);
1025 static void init_stats (void);
1027 //#define BINARY_PROTOCOL
1028 #include "sgen-protocol.c"
1029 #include "sgen-pinning.c"
1030 #include "sgen-pinning-stats.c"
1031 #include "sgen-gray.c"
1034 * ######################################################################
1035 * ######## GC descriptors
1036 * ######################################################################
1037 * Used to quickly get the info the GC needs about an object: size and
1038 * where the references are held.
1040 /* objects are aligned to 8 bytes boundaries
1041 * A descriptor is a pointer in MonoVTable, so 32 or 64 bits of size.
1042 * The low 3 bits define the type of the descriptor. The other bits
1043 * depend on the type.
1044 * As a general rule the 13 remaining low bits define the size, either
1045 * of the whole object or of the elements in the arrays. While for objects
1046 * the size is already in bytes, for arrays we need to shift, because
1047 * array elements might be smaller than 8 bytes. In case of arrays, we
1048 * use two bits to describe what the additional high bits represents,
1049 * so the default behaviour can handle element sizes less than 2048 bytes.
1050 * The high 16 bits, if 0 it means the object is pointer-free.
1051 * This design should make it easy and fast to skip over ptr-free data.
1052 * The first 4 types should cover >95% of the objects.
1053 * Note that since the size of objects is limited to 64K, larger objects
1054 * will be allocated in the large object heap.
1055 * If we want 4-bytes alignment, we need to put vector and small bitmap
1059 DESC_TYPE_RUN_LENGTH, /* 16 bits aligned byte size | 1-3 (offset, numptr) bytes tuples */
1060 DESC_TYPE_SMALL_BITMAP, /* 16 bits aligned byte size | 16-48 bit bitmap */
1061 DESC_TYPE_STRING, /* nothing */
1062 DESC_TYPE_COMPLEX, /* index for bitmap into complex_descriptors */
1063 DESC_TYPE_VECTOR, /* 10 bits element size | 1 bit array | 2 bits desc | element desc */
1064 DESC_TYPE_ARRAY, /* 10 bits element size | 1 bit array | 2 bits desc | element desc */
1065 DESC_TYPE_LARGE_BITMAP, /* | 29-61 bitmap bits */
1066 DESC_TYPE_COMPLEX_ARR, /* index for bitmap into complex_descriptors */
1067 /* subtypes for arrays and vectors */
1068 DESC_TYPE_V_PTRFREE = 0,/* there are no refs: keep first so it has a zero value */
1069 DESC_TYPE_V_REFS, /* all the array elements are refs */
1070 DESC_TYPE_V_RUN_LEN, /* elements are run-length encoded as DESC_TYPE_RUN_LENGTH */
1071 DESC_TYPE_V_BITMAP /* elements are as the bitmap in DESC_TYPE_SMALL_BITMAP */
1074 #define OBJECT_HEADER_WORDS (sizeof(MonoObject)/sizeof(gpointer))
1075 #define LOW_TYPE_BITS 3
1076 #define SMALL_BITMAP_SHIFT 16
1077 #define SMALL_BITMAP_SIZE (GC_BITS_PER_WORD - SMALL_BITMAP_SHIFT)
1078 #define VECTOR_INFO_SHIFT 14
1079 #define VECTOR_ELSIZE_SHIFT 3
1080 #define LARGE_BITMAP_SIZE (GC_BITS_PER_WORD - LOW_TYPE_BITS)
1081 #define MAX_ELEMENT_SIZE 0x3ff
1082 #define VECTOR_SUBTYPE_PTRFREE (DESC_TYPE_V_PTRFREE << VECTOR_INFO_SHIFT)
1083 #define VECTOR_SUBTYPE_REFS (DESC_TYPE_V_REFS << VECTOR_INFO_SHIFT)
1084 #define VECTOR_SUBTYPE_RUN_LEN (DESC_TYPE_V_RUN_LEN << VECTOR_INFO_SHIFT)
1085 #define VECTOR_SUBTYPE_BITMAP (DESC_TYPE_V_BITMAP << VECTOR_INFO_SHIFT)
1088 /* Root bitmap descriptors are simpler: the lower three bits describe the type
1089 * and we either have 30/62 bitmap bits or nibble-based run-length,
1090 * or a complex descriptor, or a user defined marker function.
1093 ROOT_DESC_CONSERVATIVE, /* 0, so matches NULL value */
1098 ROOT_DESC_TYPE_MASK = 0x7,
1099 ROOT_DESC_TYPE_SHIFT = 3,
1102 #define MAKE_ROOT_DESC(type,val) ((type) | ((val) << ROOT_DESC_TYPE_SHIFT))
1104 #define MAX_USER_DESCRIPTORS 16
1106 static gsize* complex_descriptors = NULL;
1107 static int complex_descriptors_size = 0;
1108 static int complex_descriptors_next = 0;
1109 static MonoGCRootMarkFunc user_descriptors [MAX_USER_DESCRIPTORS];
1110 static int user_descriptors_next = 0;
1113 alloc_complex_descriptor (gsize *bitmap, int numbits)
1117 numbits = ALIGN_TO (numbits, GC_BITS_PER_WORD);
1118 nwords = numbits / GC_BITS_PER_WORD + 1;
1121 res = complex_descriptors_next;
1122 /* linear search, so we don't have duplicates with domain load/unload
1123 * this should not be performance critical or we'd have bigger issues
1124 * (the number and size of complex descriptors should be small).
1126 for (i = 0; i < complex_descriptors_next; ) {
1127 if (complex_descriptors [i] == nwords) {
1128 int j, found = TRUE;
1129 for (j = 0; j < nwords - 1; ++j) {
1130 if (complex_descriptors [i + 1 + j] != bitmap [j]) {
1140 i += complex_descriptors [i];
1142 if (complex_descriptors_next + nwords > complex_descriptors_size) {
1143 int new_size = complex_descriptors_size * 2 + nwords;
1144 complex_descriptors = g_realloc (complex_descriptors, new_size * sizeof (gsize));
1145 complex_descriptors_size = new_size;
1147 DEBUG (6, fprintf (gc_debug_file, "Complex descriptor %d, size: %d (total desc memory: %d)\n", res, nwords, complex_descriptors_size));
1148 complex_descriptors_next += nwords;
1149 complex_descriptors [res] = nwords;
1150 for (i = 0; i < nwords - 1; ++i) {
1151 complex_descriptors [res + 1 + i] = bitmap [i];
1152 DEBUG (6, fprintf (gc_debug_file, "\tvalue: %p\n", (void*)complex_descriptors [res + 1 + i]));
1159 * Descriptor builders.
1162 mono_gc_make_descr_for_string (gsize *bitmap, int numbits)
1164 return (void*) DESC_TYPE_STRING;
1168 mono_gc_make_descr_for_object (gsize *bitmap, int numbits, size_t obj_size)
1170 int first_set = -1, num_set = 0, last_set = -1, i;
1172 size_t stored_size = obj_size;
1173 stored_size += ALLOC_ALIGN - 1;
1174 stored_size &= ~(ALLOC_ALIGN - 1);
1175 for (i = 0; i < numbits; ++i) {
1176 if (bitmap [i / GC_BITS_PER_WORD] & ((gsize)1 << (i % GC_BITS_PER_WORD))) {
1183 if (stored_size <= MAX_SMALL_OBJ_SIZE) {
1184 /* check run-length encoding first: one byte offset, one byte number of pointers
1185 * on 64 bit archs, we can have 3 runs, just one on 32.
1186 * It may be better to use nibbles.
1188 if (first_set < 0) {
1189 desc = DESC_TYPE_RUN_LENGTH | stored_size;
1190 DEBUG (6, fprintf (gc_debug_file, "Ptrfree descriptor %p, size: %zd\n", (void*)desc, stored_size));
1191 return (void*) desc;
1192 } else if (first_set < 256 && num_set < 256 && (first_set + num_set == last_set + 1)) {
1193 desc = DESC_TYPE_RUN_LENGTH | stored_size | (first_set << 16) | (num_set << 24);
1194 DEBUG (6, fprintf (gc_debug_file, "Runlen descriptor %p, size: %zd, first set: %d, num set: %d\n", (void*)desc, stored_size, first_set, num_set));
1195 return (void*) desc;
1197 /* we know the 2-word header is ptr-free */
1198 if (last_set < SMALL_BITMAP_SIZE + OBJECT_HEADER_WORDS) {
1199 desc = DESC_TYPE_SMALL_BITMAP | stored_size | ((*bitmap >> OBJECT_HEADER_WORDS) << SMALL_BITMAP_SHIFT);
1200 DEBUG (6, fprintf (gc_debug_file, "Smallbitmap descriptor %p, size: %zd, last set: %d\n", (void*)desc, stored_size, last_set));
1201 return (void*) desc;
1204 /* we know the 2-word header is ptr-free */
1205 if (last_set < LARGE_BITMAP_SIZE + OBJECT_HEADER_WORDS) {
1206 desc = DESC_TYPE_LARGE_BITMAP | ((*bitmap >> OBJECT_HEADER_WORDS) << LOW_TYPE_BITS);
1207 DEBUG (6, fprintf (gc_debug_file, "Largebitmap descriptor %p, size: %zd, last set: %d\n", (void*)desc, stored_size, last_set));
1208 return (void*) desc;
1210 /* it's a complex object ... */
1211 desc = DESC_TYPE_COMPLEX | (alloc_complex_descriptor (bitmap, last_set + 1) << LOW_TYPE_BITS);
1212 return (void*) desc;
1215 /* If the array holds references, numbits == 1 and the first bit is set in elem_bitmap */
1217 mono_gc_make_descr_for_array (int vector, gsize *elem_bitmap, int numbits, size_t elem_size)
1219 int first_set = -1, num_set = 0, last_set = -1, i;
1220 mword desc = vector? DESC_TYPE_VECTOR: DESC_TYPE_ARRAY;
1221 for (i = 0; i < numbits; ++i) {
1222 if (elem_bitmap [i / GC_BITS_PER_WORD] & ((gsize)1 << (i % GC_BITS_PER_WORD))) {
1229 if (elem_size <= MAX_ELEMENT_SIZE) {
1230 desc |= elem_size << VECTOR_ELSIZE_SHIFT;
1232 return (void*)(desc | VECTOR_SUBTYPE_PTRFREE);
1234 /* Note: we also handle structs with just ref fields */
1235 if (num_set * sizeof (gpointer) == elem_size) {
1236 return (void*)(desc | VECTOR_SUBTYPE_REFS | ((gssize)(-1) << 16));
1238 /* FIXME: try run-len first */
1239 /* Note: we can't skip the object header here, because it's not present */
1240 if (last_set <= SMALL_BITMAP_SIZE) {
1241 return (void*)(desc | VECTOR_SUBTYPE_BITMAP | (*elem_bitmap << 16));
1244 /* it's am array of complex structs ... */
1245 desc = DESC_TYPE_COMPLEX_ARR;
1246 desc |= alloc_complex_descriptor (elem_bitmap, last_set + 1) << LOW_TYPE_BITS;
1247 return (void*) desc;
1250 /* Return the bitmap encoded by a descriptor */
1252 mono_gc_get_bitmap_for_descr (void *descr, int *numbits)
1254 mword d = (mword)descr;
1258 case DESC_TYPE_RUN_LENGTH: {
1259 int first_set = (d >> 16) & 0xff;
1260 int num_set = (d >> 24) & 0xff;
1263 bitmap = g_new0 (gsize, (first_set + num_set + 7) / 8);
1265 for (i = first_set; i < first_set + num_set; ++i)
1266 bitmap [i / GC_BITS_PER_WORD] |= ((gsize)1 << (i % GC_BITS_PER_WORD));
1268 *numbits = first_set + num_set;
1272 case DESC_TYPE_SMALL_BITMAP:
1273 bitmap = g_new0 (gsize, 1);
1275 bitmap [0] = (d >> SMALL_BITMAP_SHIFT) << OBJECT_HEADER_WORDS;
1277 *numbits = GC_BITS_PER_WORD;
1281 g_assert_not_reached ();
1285 /* helper macros to scan and traverse objects, macros because we resue them in many functions */
1286 #define STRING_SIZE(size,str) do { \
1287 (size) = sizeof (MonoString) + 2 * mono_string_length ((MonoString*)(str)) + 2; \
1288 (size) += (ALLOC_ALIGN - 1); \
1289 (size) &= ~(ALLOC_ALIGN - 1); \
1292 #define OBJ_RUN_LEN_SIZE(size,desc,obj) do { \
1293 (size) = (desc) & 0xfff8; \
1296 #define OBJ_BITMAP_SIZE(size,desc,obj) do { \
1297 (size) = (desc) & 0xfff8; \
1300 //#define PREFETCH(addr) __asm__ __volatile__ (" prefetchnta %0": : "m"(*(char *)(addr)))
1301 #define PREFETCH(addr)
1303 /* code using these macros must define a HANDLE_PTR(ptr) macro that does the work */
1304 #define OBJ_RUN_LEN_FOREACH_PTR(desc,obj) do { \
1305 if ((desc) & 0xffff0000) { \
1306 /* there are pointers */ \
1307 void **_objptr_end; \
1308 void **_objptr = (void**)(obj); \
1309 _objptr += ((desc) >> 16) & 0xff; \
1310 _objptr_end = _objptr + (((desc) >> 24) & 0xff); \
1311 while (_objptr < _objptr_end) { \
1312 HANDLE_PTR (_objptr, (obj)); \
1318 /* a bitmap desc means that there are pointer references or we'd have
1319 * choosen run-length, instead: add an assert to check.
1321 #define OBJ_BITMAP_FOREACH_PTR(desc,obj) do { \
1322 /* there are pointers */ \
1323 void **_objptr = (void**)(obj); \
1324 gsize _bmap = (desc) >> 16; \
1325 _objptr += OBJECT_HEADER_WORDS; \
1327 if ((_bmap & 1)) { \
1328 HANDLE_PTR (_objptr, (obj)); \
1335 #define OBJ_LARGE_BITMAP_FOREACH_PTR(vt,obj) do { \
1336 /* there are pointers */ \
1337 void **_objptr = (void**)(obj); \
1338 gsize _bmap = (vt)->desc >> LOW_TYPE_BITS; \
1339 _objptr += OBJECT_HEADER_WORDS; \
1341 if ((_bmap & 1)) { \
1342 HANDLE_PTR (_objptr, (obj)); \
1349 #define OBJ_COMPLEX_FOREACH_PTR(vt,obj) do { \
1350 /* there are pointers */ \
1351 void **_objptr = (void**)(obj); \
1352 gsize *bitmap_data = complex_descriptors + ((vt)->desc >> LOW_TYPE_BITS); \
1353 int bwords = (*bitmap_data) - 1; \
1354 void **start_run = _objptr; \
1357 MonoObject *myobj = (MonoObject*)obj; \
1358 g_print ("found %d at %p (0x%zx): %s.%s\n", bwords, (obj), (vt)->desc, myobj->vtable->klass->name_space, myobj->vtable->klass->name); \
1360 while (bwords-- > 0) { \
1361 gsize _bmap = *bitmap_data++; \
1362 _objptr = start_run; \
1363 /*g_print ("bitmap: 0x%x/%d at %p\n", _bmap, bwords, _objptr);*/ \
1365 if ((_bmap & 1)) { \
1366 HANDLE_PTR (_objptr, (obj)); \
1371 start_run += GC_BITS_PER_WORD; \
1375 /* this one is untested */
1376 #define OBJ_COMPLEX_ARR_FOREACH_PTR(vt,obj) do { \
1377 /* there are pointers */ \
1378 gsize *mbitmap_data = complex_descriptors + ((vt)->desc >> LOW_TYPE_BITS); \
1379 int mbwords = (*mbitmap_data++) - 1; \
1380 int el_size = mono_array_element_size (((MonoObject*)(obj))->vtable->klass); \
1381 char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
1382 char *e_end = e_start + el_size * mono_array_length ((MonoArray*)(obj)); \
1384 MonoObject *myobj = (MonoObject*)start; \
1385 g_print ("found %d at %p (0x%zx): %s.%s\n", mbwords, (obj), (vt)->desc, myobj->vtable->klass->name_space, myobj->vtable->klass->name); \
1387 while (e_start < e_end) { \
1388 void **_objptr = (void**)e_start; \
1389 gsize *bitmap_data = mbitmap_data; \
1390 unsigned int bwords = mbwords; \
1391 while (bwords-- > 0) { \
1392 gsize _bmap = *bitmap_data++; \
1393 void **start_run = _objptr; \
1394 /*g_print ("bitmap: 0x%x\n", _bmap);*/ \
1396 if ((_bmap & 1)) { \
1397 HANDLE_PTR (_objptr, (obj)); \
1402 _objptr = start_run + GC_BITS_PER_WORD; \
1404 e_start += el_size; \
1408 #define OBJ_VECTOR_FOREACH_PTR(vt,obj) do { \
1409 /* note: 0xffffc000 excludes DESC_TYPE_V_PTRFREE */ \
1410 if ((vt)->desc & 0xffffc000) { \
1411 int el_size = ((vt)->desc >> 3) & MAX_ELEMENT_SIZE; \
1412 /* there are pointers */ \
1413 int etype = (vt)->desc & 0xc000; \
1414 if (etype == (DESC_TYPE_V_REFS << 14)) { \
1415 void **p = (void**)((char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector)); \
1416 void **end_refs = (void**)((char*)p + el_size * mono_array_length ((MonoArray*)(obj))); \
1417 /* Note: this code can handle also arrays of struct with only references in them */ \
1418 while (p < end_refs) { \
1419 HANDLE_PTR (p, (obj)); \
1422 } else if (etype == DESC_TYPE_V_RUN_LEN << 14) { \
1423 int offset = ((vt)->desc >> 16) & 0xff; \
1424 int num_refs = ((vt)->desc >> 24) & 0xff; \
1425 char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
1426 char *e_end = e_start + el_size * mono_array_length ((MonoArray*)(obj)); \
1427 while (e_start < e_end) { \
1428 void **p = (void**)e_start; \
1431 for (i = 0; i < num_refs; ++i) { \
1432 HANDLE_PTR (p + i, (obj)); \
1434 e_start += el_size; \
1436 } else if (etype == DESC_TYPE_V_BITMAP << 14) { \
1437 char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
1438 char *e_end = e_start + el_size * mono_array_length ((MonoArray*)(obj)); \
1439 while (e_start < e_end) { \
1440 void **p = (void**)e_start; \
1441 gsize _bmap = (vt)->desc >> 16; \
1442 /* Note: there is no object header here to skip */ \
1444 if ((_bmap & 1)) { \
1445 HANDLE_PTR (p, (obj)); \
1450 e_start += el_size; \
1456 //#include "sgen-major-copying.c"
1457 #include "sgen-marksweep.c"
1460 is_xdomain_ref_allowed (gpointer *ptr, char *obj, MonoDomain *domain)
1462 MonoObject *o = (MonoObject*)(obj);
1463 MonoObject *ref = (MonoObject*)*(ptr);
1464 int offset = (char*)(ptr) - (char*)o;
1466 if (o->vtable->klass == mono_defaults.thread_class && offset == G_STRUCT_OFFSET (MonoThread, internal_thread))
1468 if (o->vtable->klass == mono_defaults.internal_thread_class && offset == G_STRUCT_OFFSET (MonoInternalThread, current_appcontext))
1470 if (mono_class_has_parent (o->vtable->klass, mono_defaults.real_proxy_class) &&
1471 offset == G_STRUCT_OFFSET (MonoRealProxy, unwrapped_server))
1473 /* Thread.cached_culture_info */
1474 if (!strcmp (ref->vtable->klass->name_space, "System.Globalization") &&
1475 !strcmp (ref->vtable->klass->name, "CultureInfo") &&
1476 !strcmp(o->vtable->klass->name_space, "System") &&
1477 !strcmp(o->vtable->klass->name, "Object[]"))
1480 * at System.IO.MemoryStream.InternalConstructor (byte[],int,int,bool,bool) [0x0004d] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.IO/MemoryStream.cs:121
1481 * at System.IO.MemoryStream..ctor (byte[]) [0x00017] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.IO/MemoryStream.cs:81
1482 * at (wrapper remoting-invoke-with-check) System.IO.MemoryStream..ctor (byte[]) <IL 0x00020, 0xffffffff>
1483 * at System.Runtime.Remoting.Messaging.CADMethodCallMessage.GetArguments () [0x0000d] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Messaging/CADMessages.cs:327
1484 * at System.Runtime.Remoting.Messaging.MethodCall..ctor (System.Runtime.Remoting.Messaging.CADMethodCallMessage) [0x00017] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Messaging/MethodCall.cs:87
1485 * at System.AppDomain.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage,byte[]&,System.Runtime.Remoting.Messaging.CADMethodReturnMessage&) [0x00018] in /home/schani/Work/novell/trunk/mcs/class/corlib/System/AppDomain.cs:1213
1486 * at (wrapper remoting-invoke-with-check) System.AppDomain.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage,byte[]&,System.Runtime.Remoting.Messaging.CADMethodReturnMessage&) <IL 0x0003d, 0xffffffff>
1487 * at System.Runtime.Remoting.Channels.CrossAppDomainSink.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage) [0x00008] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Channels/CrossAppDomainChannel.cs:198
1488 * at (wrapper runtime-invoke) object.runtime_invoke_CrossAppDomainSink/ProcessMessageRes_object_object (object,intptr,intptr,intptr) <IL 0x0004c, 0xffffffff>
1490 if (!strcmp (ref->vtable->klass->name_space, "System") &&
1491 !strcmp (ref->vtable->klass->name, "Byte[]") &&
1492 !strcmp (o->vtable->klass->name_space, "System.IO") &&
1493 !strcmp (o->vtable->klass->name, "MemoryStream"))
1495 /* append_job() in threadpool.c */
1496 if (!strcmp (ref->vtable->klass->name_space, "System.Runtime.Remoting.Messaging") &&
1497 !strcmp (ref->vtable->klass->name, "AsyncResult") &&
1498 !strcmp (o->vtable->klass->name_space, "System") &&
1499 !strcmp (o->vtable->klass->name, "Object[]") &&
1500 mono_thread_pool_is_queue_array ((MonoArray*) o))
1506 check_reference_for_xdomain (gpointer *ptr, char *obj, MonoDomain *domain)
1508 MonoObject *o = (MonoObject*)(obj);
1509 MonoObject *ref = (MonoObject*)*(ptr);
1510 int offset = (char*)(ptr) - (char*)o;
1512 MonoClassField *field;
1515 if (!ref || ref->vtable->domain == domain)
1517 if (is_xdomain_ref_allowed (ptr, obj, domain))
1521 for (class = o->vtable->klass; class; class = class->parent) {
1524 for (i = 0; i < class->field.count; ++i) {
1525 if (class->fields[i].offset == offset) {
1526 field = &class->fields[i];
1534 if (ref->vtable->klass == mono_defaults.string_class)
1535 str = mono_string_to_utf8 ((MonoString*)ref);
1538 g_print ("xdomain reference in %p (%s.%s) at offset %d (%s) to %p (%s.%s) (%s) - pointed to by:\n",
1539 o, o->vtable->klass->name_space, o->vtable->klass->name,
1540 offset, field ? field->name : "",
1541 ref, ref->vtable->klass->name_space, ref->vtable->klass->name, str ? str : "");
1542 mono_gc_scan_for_specific_ref (o);
1548 #define HANDLE_PTR(ptr,obj) check_reference_for_xdomain ((ptr), (obj), domain)
1551 scan_object_for_xdomain_refs (char *start)
1553 MonoDomain *domain = ((MonoObject*)start)->vtable->domain;
1555 #include "sgen-scan-object.h"
1561 scan_area_for_xdomain_refs (char *start, char *end)
1563 while (start < end) {
1564 if (!*(void**)start) {
1565 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
1569 start = scan_object_for_xdomain_refs (start);
1574 #define HANDLE_PTR(ptr,obj) do { \
1575 if ((MonoObject*)*(ptr) == key) { \
1576 g_print ("found ref to %p in object %p (%s) at offset %zd\n", \
1577 key, (obj), safe_name ((obj)), ((char*)(ptr) - (char*)(obj))); \
1582 scan_object_for_specific_ref (char *start, MonoObject *key)
1584 #include "sgen-scan-object.h"
1590 scan_area_with_callback (char *start, char *end, IterateObjectCallbackFunc callback, void *data)
1592 while (start < end) {
1594 if (!*(void**)start) {
1595 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
1599 size = safe_object_get_size ((MonoObject*) start);
1600 size += ALLOC_ALIGN - 1;
1601 size &= ~(ALLOC_ALIGN - 1);
1603 callback (start, size, data);
1610 scan_object_for_specific_ref_callback (char *obj, size_t size, MonoObject *key)
1612 scan_object_for_specific_ref (obj, key);
1616 check_root_obj_specific_ref (RootRecord *root, MonoObject *key, MonoObject *obj)
1620 g_print ("found ref to %p in root record %p\n", key, root);
1623 static MonoObject *check_key = NULL;
1624 static RootRecord *check_root = NULL;
1627 check_root_obj_specific_ref_from_marker (void **obj)
1629 check_root_obj_specific_ref (check_root, check_key, *obj);
1633 scan_roots_for_specific_ref (MonoObject *key, int root_type)
1638 for (i = 0; i < roots_hash_size [root_type]; ++i) {
1639 for (root = roots_hash [root_type][i]; root; root = root->next) {
1640 void **start_root = (void**)root->start_root;
1641 mword desc = root->root_desc;
1645 switch (desc & ROOT_DESC_TYPE_MASK) {
1646 case ROOT_DESC_BITMAP:
1647 desc >>= ROOT_DESC_TYPE_SHIFT;
1650 check_root_obj_specific_ref (root, key, *start_root);
1655 case ROOT_DESC_COMPLEX: {
1656 gsize *bitmap_data = complex_descriptors + (desc >> ROOT_DESC_TYPE_SHIFT);
1657 int bwords = (*bitmap_data) - 1;
1658 void **start_run = start_root;
1660 while (bwords-- > 0) {
1661 gsize bmap = *bitmap_data++;
1662 void **objptr = start_run;
1665 check_root_obj_specific_ref (root, key, *objptr);
1669 start_run += GC_BITS_PER_WORD;
1673 case ROOT_DESC_USER: {
1674 MonoGCRootMarkFunc marker = user_descriptors [desc >> ROOT_DESC_TYPE_SHIFT];
1675 marker (start_root, check_root_obj_specific_ref_from_marker);
1678 case ROOT_DESC_RUN_LEN:
1679 g_assert_not_reached ();
1681 g_assert_not_reached ();
1690 mono_gc_scan_for_specific_ref (MonoObject *key)
1696 scan_area_with_callback (nursery_section->data, nursery_section->end_data,
1697 (IterateObjectCallbackFunc)scan_object_for_specific_ref_callback, key);
1699 major_iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)scan_object_for_specific_ref_callback, key);
1701 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1702 scan_object_for_specific_ref (bigobj->data, key);
1704 scan_roots_for_specific_ref (key, ROOT_TYPE_NORMAL);
1705 scan_roots_for_specific_ref (key, ROOT_TYPE_WBARRIER);
1707 for (i = 0; i < roots_hash_size [ROOT_TYPE_PINNED]; ++i) {
1708 for (root = roots_hash [ROOT_TYPE_PINNED][i]; root; root = root->next) {
1709 void **ptr = (void**)root->start_root;
1711 while (ptr < (void**)root->end_root) {
1712 check_root_obj_specific_ref (root, *ptr, key);
1719 /* Clear all remaining nursery fragments */
1721 clear_nursery_fragments (char *next)
1724 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION) {
1725 g_assert (next <= nursery_frag_real_end);
1726 memset (next, 0, nursery_frag_real_end - next);
1727 for (frag = nursery_fragments; frag; frag = frag->next) {
1728 memset (frag->fragment_start, 0, frag->fragment_end - frag->fragment_start);
1734 need_remove_object_for_domain (char *start, MonoDomain *domain)
1736 if (mono_object_domain (start) == domain) {
1737 DEBUG (4, fprintf (gc_debug_file, "Need to cleanup object %p\n", start));
1738 binary_protocol_cleanup (start, (gpointer)LOAD_VTABLE (start), safe_object_get_size ((MonoObject*)start));
1745 process_object_for_domain_clearing (char *start, MonoDomain *domain)
1747 GCVTable *vt = (GCVTable*)LOAD_VTABLE (start);
1748 if (vt->klass == mono_defaults.internal_thread_class)
1749 g_assert (mono_object_domain (start) == mono_get_root_domain ());
1750 /* The object could be a proxy for an object in the domain
1752 if (mono_class_has_parent (vt->klass, mono_defaults.real_proxy_class)) {
1753 MonoObject *server = ((MonoRealProxy*)start)->unwrapped_server;
1755 /* The server could already have been zeroed out, so
1756 we need to check for that, too. */
1757 if (server && (!LOAD_VTABLE (server) || mono_object_domain (server) == domain)) {
1758 DEBUG (4, fprintf (gc_debug_file, "Cleaning up remote pointer in %p to object %p\n",
1760 ((MonoRealProxy*)start)->unwrapped_server = NULL;
1765 static MonoDomain *check_domain = NULL;
1768 check_obj_not_in_domain (void **o)
1770 g_assert (((MonoObject*)(*o))->vtable->domain != check_domain);
1774 scan_for_registered_roots_in_domain (MonoDomain *domain, int root_type)
1778 check_domain = domain;
1779 for (i = 0; i < roots_hash_size [root_type]; ++i) {
1780 for (root = roots_hash [root_type][i]; root; root = root->next) {
1781 void **start_root = (void**)root->start_root;
1782 mword desc = root->root_desc;
1784 /* The MonoDomain struct is allowed to hold
1785 references to objects in its own domain. */
1786 if (start_root == (void**)domain)
1789 switch (desc & ROOT_DESC_TYPE_MASK) {
1790 case ROOT_DESC_BITMAP:
1791 desc >>= ROOT_DESC_TYPE_SHIFT;
1793 if ((desc & 1) && *start_root)
1794 check_obj_not_in_domain (*start_root);
1799 case ROOT_DESC_COMPLEX: {
1800 gsize *bitmap_data = complex_descriptors + (desc >> ROOT_DESC_TYPE_SHIFT);
1801 int bwords = (*bitmap_data) - 1;
1802 void **start_run = start_root;
1804 while (bwords-- > 0) {
1805 gsize bmap = *bitmap_data++;
1806 void **objptr = start_run;
1808 if ((bmap & 1) && *objptr)
1809 check_obj_not_in_domain (*objptr);
1813 start_run += GC_BITS_PER_WORD;
1817 case ROOT_DESC_USER: {
1818 MonoGCRootMarkFunc marker = user_descriptors [desc >> ROOT_DESC_TYPE_SHIFT];
1819 marker (start_root, check_obj_not_in_domain);
1822 case ROOT_DESC_RUN_LEN:
1823 g_assert_not_reached ();
1825 g_assert_not_reached ();
1829 check_domain = NULL;
1833 scan_pinned_object_for_xdomain_refs_callback (char *obj, size_t size, gpointer dummy)
1835 scan_object_for_xdomain_refs (obj);
1839 check_for_xdomain_refs (void)
1843 scan_area_for_xdomain_refs (nursery_section->data, nursery_section->end_data);
1845 major_iterate_objects (TRUE, TRUE, scan_pinned_object_for_xdomain_refs_callback, NULL);
1847 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1848 scan_object_for_xdomain_refs (bigobj->data);
1852 clear_domain_process_object (char *obj, MonoDomain *domain)
1856 process_object_for_domain_clearing (obj, domain);
1857 remove = need_remove_object_for_domain (obj, domain);
1859 if (remove && ((MonoObject*)obj)->synchronisation) {
1860 void **dislink = mono_monitor_get_object_monitor_weak_link ((MonoObject*)obj);
1862 mono_gc_register_disappearing_link (NULL, dislink, FALSE);
1869 clear_domain_process_minor_object_callback (char *obj, size_t size, MonoDomain *domain)
1871 if (clear_domain_process_object (obj, domain))
1872 memset (obj, 0, size);
1876 clear_domain_process_major_object_callback (char *obj, size_t size, MonoDomain *domain)
1878 clear_domain_process_object (obj, domain);
1882 clear_domain_free_major_non_pinned_object_callback (char *obj, size_t size, MonoDomain *domain)
1884 if (need_remove_object_for_domain (obj, domain))
1885 major_free_non_pinned_object (obj, size);
1889 clear_domain_free_major_pinned_object_callback (char *obj, size_t size, MonoDomain *domain)
1891 if (need_remove_object_for_domain (obj, domain))
1892 free_pinned_object (obj, size);
1896 * When appdomains are unloaded we can easily remove objects that have finalizers,
1897 * but all the others could still be present in random places on the heap.
1898 * We need a sweep to get rid of them even though it's going to be costly
1900 * The reason we need to remove them is because we access the vtable and class
1901 * structures to know the object size and the reference bitmap: once the domain is
1902 * unloaded the point to random memory.
1905 mono_gc_clear_domain (MonoDomain * domain)
1907 LOSObject *bigobj, *prev;
1912 clear_nursery_fragments (nursery_next);
1914 if (xdomain_checks && domain != mono_get_root_domain ()) {
1915 scan_for_registered_roots_in_domain (domain, ROOT_TYPE_NORMAL);
1916 scan_for_registered_roots_in_domain (domain, ROOT_TYPE_WBARRIER);
1917 check_for_xdomain_refs ();
1920 scan_area_with_callback (nursery_section->data, nursery_section->end_data,
1921 (IterateObjectCallbackFunc)clear_domain_process_minor_object_callback, domain);
1923 /* We need two passes over major and large objects because
1924 freeing such objects might give their memory back to the OS
1925 (in the case of large objects) or obliterate its vtable
1926 (pinned objects with major-copying or pinned and non-pinned
1927 objects with major-mark&sweep), but we might need to
1928 dereference a pointer from an object to another object if
1929 the first object is a proxy. */
1930 major_iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)clear_domain_process_major_object_callback, domain);
1931 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1932 clear_domain_process_object (bigobj->data, domain);
1935 for (bigobj = los_object_list; bigobj;) {
1936 if (need_remove_object_for_domain (bigobj->data, domain)) {
1937 LOSObject *to_free = bigobj;
1939 prev->next = bigobj->next;
1941 los_object_list = bigobj->next;
1942 bigobj = bigobj->next;
1943 DEBUG (4, fprintf (gc_debug_file, "Freeing large object %p\n",
1945 free_large_object (to_free);
1949 bigobj = bigobj->next;
1951 major_iterate_objects (TRUE, FALSE, (IterateObjectCallbackFunc)clear_domain_free_major_non_pinned_object_callback, domain);
1952 major_iterate_objects (FALSE, TRUE, (IterateObjectCallbackFunc)clear_domain_free_major_pinned_object_callback, domain);
1954 for (i = GENERATION_NURSERY; i < GENERATION_MAX; ++i)
1955 null_links_for_domain (domain, i);
1961 * add_to_global_remset:
1963 * The global remset contains locations which point into newspace after
1964 * a minor collection. This can happen if the objects they point to are pinned.
1967 add_to_global_remset (gpointer ptr, gboolean root)
1971 DEBUG (8, fprintf (gc_debug_file, "Adding global remset for %p\n", ptr));
1972 binary_protocol_global_remset (ptr, *(gpointer*)ptr, (gpointer)LOAD_VTABLE (*(gpointer*)ptr));
1975 g_assert (!ptr_in_nursery (ptr) && ptr_in_nursery (*(gpointer*)ptr));
1977 HEAVY_STAT (++stat_global_remsets_added);
1980 * FIXME: If an object remains pinned, we need to add it at every minor collection.
1981 * To avoid uncontrolled growth of the global remset, only add each pointer once.
1983 if (global_remset->store_next + 3 < global_remset->end_set) {
1985 *(global_remset->store_next++) = (mword)ptr | REMSET_OTHER;
1986 *(global_remset->store_next++) = (mword)REMSET_ROOT_LOCATION;
1988 *(global_remset->store_next++) = (mword)ptr;
1992 rs = alloc_remset (global_remset->end_set - global_remset->data, NULL);
1993 rs->next = global_remset;
1996 *(global_remset->store_next++) = (mword)ptr | REMSET_OTHER;
1997 *(global_remset->store_next++) = (mword)REMSET_ROOT_LOCATION;
1999 *(global_remset->store_next++) = (mword)ptr;
2003 int global_rs_size = 0;
2005 for (rs = global_remset; rs; rs = rs->next) {
2006 global_rs_size += rs->store_next - rs->data;
2008 DEBUG (4, fprintf (gc_debug_file, "Global remset now has size %d\n", global_rs_size));
2013 * FIXME: allocate before calling this function and pass the
2014 * destination address.
2017 copy_object_no_checks (void *obj)
2019 static const void *copy_labels [] = { &&LAB_0, &&LAB_1, &&LAB_2, &&LAB_3, &&LAB_4, &&LAB_5, &&LAB_6, &&LAB_7, &&LAB_8 };
2025 objsize = safe_object_get_size ((MonoObject*)obj);
2026 objsize += ALLOC_ALIGN - 1;
2027 objsize &= ~(ALLOC_ALIGN - 1);
2029 MAJOR_GET_COPY_OBJECT_SPACE (destination, objsize);
2031 DEBUG (9, fprintf (gc_debug_file, " (to %p, %s size: %zd)\n", destination, ((MonoObject*)obj)->vtable->klass->name, objsize));
2032 binary_protocol_copy (obj, destination, ((MonoObject*)obj)->vtable, objsize);
2034 HEAVY_STAT (++num_objects_copied);
2036 if (objsize <= sizeof (gpointer) * 8) {
2037 mword *dest = (mword*)destination;
2038 goto *copy_labels [objsize / sizeof (gpointer)];
2040 (dest) [7] = ((mword*)obj) [7];
2042 (dest) [6] = ((mword*)obj) [6];
2044 (dest) [5] = ((mword*)obj) [5];
2046 (dest) [4] = ((mword*)obj) [4];
2048 (dest) [3] = ((mword*)obj) [3];
2050 (dest) [2] = ((mword*)obj) [2];
2052 (dest) [1] = ((mword*)obj) [1];
2054 (dest) [0] = ((mword*)obj) [0];
2062 char* edi = destination;
2063 __asm__ __volatile__(
2065 : "=&c" (ecx), "=&D" (edi), "=&S" (esi)
2066 : "0" (objsize/4), "1" (edi),"2" (esi)
2071 memcpy (destination, obj, objsize);
2074 /* adjust array->bounds */
2075 vt = ((MonoObject*)obj)->vtable;
2076 DEBUG (9, g_assert (vt->gc_descr));
2077 if (G_UNLIKELY (vt->rank && ((MonoArray*)obj)->bounds)) {
2078 MonoArray *array = (MonoArray*)destination;
2079 array->bounds = (MonoArrayBounds*)((char*)destination + ((char*)((MonoArray*)obj)->bounds - (char*)obj));
2080 DEBUG (9, fprintf (gc_debug_file, "Array instance %p: size: %zd, rank: %d, length: %d\n", array, objsize, vt->rank, mono_array_length (array)));
2082 /* set the forwarding pointer */
2083 forward_object (obj, destination);
2084 if (G_UNLIKELY (mono_profiler_events & MONO_PROFILE_GC_MOVES)) {
2085 if (moved_objects_idx == MOVED_OBJECTS_NUM) {
2086 mono_profiler_gc_moves (moved_objects, moved_objects_idx);
2087 moved_objects_idx = 0;
2089 moved_objects [moved_objects_idx++] = obj;
2090 moved_objects [moved_objects_idx++] = destination;
2093 DEBUG (9, fprintf (gc_debug_file, "Enqueuing gray object %p (%s)\n", obj, safe_name (obj)));
2094 GRAY_OBJECT_ENQUEUE (obj);
2099 * This is how the copying happens from the nursery to the old generation.
2100 * We assume that at this time all the pinned objects have been identified and
2102 * We run scan_object() for each pinned object so that each referenced
2103 * objects if possible are copied. The new gray objects created can have
2104 * scan_object() run on them right away, too.
2105 * Then we run copy_object() for the precisely tracked roots. At this point
2106 * all the roots are either gray or black. We run scan_object() on the gray
2107 * objects until no more gray objects are created.
2108 * At the end of the process we walk again the pinned list and we unmark
2109 * the pinned flag. As we go we also create the list of free space for use
2110 * in the next allocation runs.
2112 * We need to remember objects from the old generation that point to the new one
2113 * (or just addresses?).
2115 * copy_object could be made into a macro once debugged (use inline for now).
2118 static void __attribute__((noinline))
2119 copy_object (void **obj_slot)
2122 char *obj = *obj_slot;
2124 DEBUG (9, g_assert (current_collection_generation == GENERATION_NURSERY));
2126 HEAVY_STAT (++num_copy_object_called);
2128 if (!ptr_in_nursery (obj)) {
2129 HEAVY_STAT (++stat_copy_object_failed_from_space);
2133 DEBUG (9, fprintf (gc_debug_file, "Precise copy of %p from %p", obj, obj_slot));
2136 * Before we can copy the object we must make sure that we are
2137 * allowed to, i.e. that the object not pinned or not already
2141 if ((forwarded = object_is_forwarded (obj))) {
2142 DEBUG (9, g_assert (((MonoVTable*)LOAD_VTABLE(obj))->gc_descr));
2143 DEBUG (9, fprintf (gc_debug_file, " (already forwarded to %p)\n", forwarded));
2144 HEAVY_STAT (++stat_copy_object_failed_forwarded);
2145 *obj_slot = forwarded;
2148 if (object_is_pinned (obj)) {
2149 DEBUG (9, g_assert (((MonoVTable*)LOAD_VTABLE(obj))->gc_descr));
2150 DEBUG (9, fprintf (gc_debug_file, " (pinned, no change)\n"));
2151 HEAVY_STAT (++stat_copy_object_failed_pinned);
2155 *obj_slot = copy_object_no_checks (obj);
2159 #define HANDLE_PTR(ptr,obj) do { \
2160 void *__old = *(ptr); \
2163 copy_object ((ptr)); \
2165 DEBUG (9, if (__old != __copy) fprintf (gc_debug_file, "Overwrote field at %p with %p (was: %p)\n", (ptr), *(ptr), __old)); \
2166 if (G_UNLIKELY (ptr_in_nursery (__copy) && !ptr_in_nursery ((ptr)))) \
2167 add_to_global_remset ((ptr), FALSE); \
2172 * Scan the object pointed to by @start for references to
2173 * other objects between @from_start and @from_end and copy
2174 * them to the gray_objects area.
2175 * Returns a pointer to the end of the object.
2178 scan_object (char *start)
2180 #include "sgen-scan-object.h"
2188 * Scan the valuetype pointed to by START, described by DESC for references to
2189 * other objects between @from_start and @from_end and copy them to the gray_objects area.
2190 * Returns a pointer to the end of the object.
2193 scan_vtype (char *start, mword desc, char* from_start, char* from_end)
2197 /* The descriptors include info about the MonoObject header as well */
2198 start -= sizeof (MonoObject);
2200 switch (desc & 0x7) {
2201 case DESC_TYPE_RUN_LENGTH:
2202 OBJ_RUN_LEN_FOREACH_PTR (desc,start);
2203 OBJ_RUN_LEN_SIZE (skip_size, desc, start);
2204 g_assert (skip_size);
2205 return start + skip_size;
2206 case DESC_TYPE_SMALL_BITMAP:
2207 OBJ_BITMAP_FOREACH_PTR (desc,start);
2208 OBJ_BITMAP_SIZE (skip_size, desc, start);
2209 return start + skip_size;
2210 case DESC_TYPE_LARGE_BITMAP:
2211 case DESC_TYPE_COMPLEX:
2213 g_assert_not_reached ();
2216 // The other descriptors can't happen with vtypes
2217 g_assert_not_reached ();
2224 #define HANDLE_PTR(ptr,obj) do { \
2225 void *__old = *(ptr); \
2228 major_copy_or_mark_object ((ptr)); \
2230 DEBUG (9, if (__old != __copy) fprintf (gc_debug_file, "Overwrote field at %p with %p (was: %p)\n", (ptr), *(ptr), __old)); \
2231 if (G_UNLIKELY (ptr_in_nursery (__copy) && !ptr_in_nursery ((ptr)))) \
2232 add_to_global_remset ((ptr), FALSE); \
2237 major_scan_object (char *start)
2239 #include "sgen-scan-object.h"
2247 * Scan objects in the gray stack until the stack is empty. This should be called
2248 * frequently after each object is copied, to achieve better locality and cache
2252 drain_gray_stack (void)
2256 if (current_collection_generation == GENERATION_NURSERY) {
2258 GRAY_OBJECT_DEQUEUE (obj);
2261 DEBUG (9, fprintf (gc_debug_file, "Precise gray object scan %p (%s)\n", obj, safe_name (obj)));
2266 GRAY_OBJECT_DEQUEUE (obj);
2269 DEBUG (9, fprintf (gc_debug_file, "Precise gray object scan %p (%s)\n", obj, safe_name (obj)));
2270 major_scan_object (obj);
2276 * Addresses from start to end are already sorted. This function finds
2277 * the object header for each address and pins the object. The
2278 * addresses must be inside the passed section. The (start of the)
2279 * address array is overwritten with the addresses of the actually
2280 * pinned objects. Return the number of pinned objects.
2283 pin_objects_from_addresses (GCMemSection *section, void **start, void **end, void *start_nursery, void *end_nursery)
2288 void *last_obj = NULL;
2289 size_t last_obj_size = 0;
2292 void **definitely_pinned = start;
2293 while (start < end) {
2295 /* the range check should be reduntant */
2296 if (addr != last && addr >= start_nursery && addr < end_nursery) {
2297 DEBUG (5, fprintf (gc_debug_file, "Considering pinning addr %p\n", addr));
2298 /* multiple pointers to the same object */
2299 if (addr >= last_obj && (char*)addr < (char*)last_obj + last_obj_size) {
2303 idx = ((char*)addr - (char*)section->data) / SCAN_START_SIZE;
2304 g_assert (idx < section->num_scan_start);
2305 search_start = (void*)section->scan_starts [idx];
2306 if (!search_start || search_start > addr) {
2309 search_start = section->scan_starts [idx];
2310 if (search_start && search_start <= addr)
2313 if (!search_start || search_start > addr)
2314 search_start = start_nursery;
2316 if (search_start < last_obj)
2317 search_start = (char*)last_obj + last_obj_size;
2318 /* now addr should be in an object a short distance from search_start
2319 * Note that search_start must point to zeroed mem or point to an object.
2322 if (!*(void**)search_start) {
2323 mword p = (mword)search_start;
2324 p += sizeof (gpointer);
2325 p += ALLOC_ALIGN - 1;
2326 p &= ~(ALLOC_ALIGN - 1);
2327 search_start = (void*)p;
2330 last_obj = search_start;
2331 last_obj_size = safe_object_get_size ((MonoObject*)search_start);
2332 last_obj_size += ALLOC_ALIGN - 1;
2333 last_obj_size &= ~(ALLOC_ALIGN - 1);
2334 DEBUG (8, fprintf (gc_debug_file, "Pinned try match %p (%s), size %zd\n", last_obj, safe_name (last_obj), last_obj_size));
2335 if (addr >= search_start && (char*)addr < (char*)last_obj + last_obj_size) {
2336 DEBUG (4, fprintf (gc_debug_file, "Pinned object %p, vtable %p (%s), count %d\n", search_start, *(void**)search_start, safe_name (search_start), count));
2337 binary_protocol_pin (search_start, (gpointer)LOAD_VTABLE (search_start), safe_object_get_size (search_start));
2338 pin_object (search_start);
2339 GRAY_OBJECT_ENQUEUE (search_start);
2341 pin_stats_register_object (search_start, last_obj_size);
2342 definitely_pinned [count] = search_start;
2346 /* skip to the next object */
2347 search_start = (void*)((char*)search_start + last_obj_size);
2348 } while (search_start <= addr);
2349 /* we either pinned the correct object or we ignored the addr because
2350 * it points to unused zeroed memory.
2356 //printf ("effective pinned: %d (at the end: %d)\n", count, (char*)end_nursery - (char*)last);
2361 pin_objects_in_section (GCMemSection *section)
2363 int start = section->pin_queue_start;
2364 int end = section->pin_queue_end;
2367 reduced_to = pin_objects_from_addresses (section, pin_queue + start, pin_queue + end,
2368 section->data, section->next_data);
2369 section->pin_queue_start = start;
2370 section->pin_queue_end = start + reduced_to;
2377 gap = (gap * 10) / 13;
2378 if (gap == 9 || gap == 10)
2387 compare_addr (const void *a, const void *b)
2389 return *(const void **)a - *(const void **)b;
2393 /* sort the addresses in array in increasing order */
2395 sort_addresses (void **array, int size)
2398 * qsort is slower as predicted.
2399 * qsort (array, size, sizeof (gpointer), compare_addr);
2406 gap = new_gap (gap);
2409 for (i = 0; i < end; i++) {
2411 if (array [i] > array [j]) {
2412 void* val = array [i];
2413 array [i] = array [j];
2418 if (gap == 1 && !swapped)
2423 static G_GNUC_UNUSED void
2424 print_nursery_gaps (void* start_nursery, void *end_nursery)
2427 gpointer first = start_nursery;
2429 for (i = 0; i < next_pin_slot; ++i) {
2430 next = pin_queue [i];
2431 fprintf (gc_debug_file, "Nursery range: %p-%p, size: %zd\n", first, next, (char*)next-(char*)first);
2435 fprintf (gc_debug_file, "Nursery range: %p-%p, size: %zd\n", first, next, (char*)next-(char*)first);
2438 /* reduce the info in the pin queue, removing duplicate pointers and sorting them */
2440 optimize_pin_queue (int start_slot)
2442 void **start, **cur, **end;
2443 /* sort and uniq pin_queue: we just sort and we let the rest discard multiple values */
2444 /* it may be better to keep ranges of pinned memory instead of individually pinning objects */
2445 DEBUG (5, fprintf (gc_debug_file, "Sorting pin queue, size: %d\n", next_pin_slot));
2446 if ((next_pin_slot - start_slot) > 1)
2447 sort_addresses (pin_queue + start_slot, next_pin_slot - start_slot);
2448 start = cur = pin_queue + start_slot;
2449 end = pin_queue + next_pin_slot;
2452 while (*start == *cur && cur < end)
2456 next_pin_slot = start - pin_queue;
2457 DEBUG (5, fprintf (gc_debug_file, "Pin queue reduced to size: %d\n", next_pin_slot));
2458 //DEBUG (6, print_nursery_gaps (start_nursery, end_nursery));
2463 * Scan the memory between start and end and queue values which could be pointers
2464 * to the area between start_nursery and end_nursery for later consideration.
2465 * Typically used for thread stacks.
2468 conservatively_pin_objects_from (void **start, void **end, void *start_nursery, void *end_nursery, int pin_type)
2471 while (start < end) {
2472 if (*start >= start_nursery && *start < end_nursery) {
2474 * *start can point to the middle of an object
2475 * note: should we handle pointing at the end of an object?
2476 * pinning in C# code disallows pointing at the end of an object
2477 * but there is some small chance that an optimizing C compiler
2478 * may keep the only reference to an object by pointing
2479 * at the end of it. We ignore this small chance for now.
2480 * Pointers to the end of an object are indistinguishable
2481 * from pointers to the start of the next object in memory
2482 * so if we allow that we'd need to pin two objects...
2483 * We queue the pointer in an array, the
2484 * array will then be sorted and uniqued. This way
2485 * we can coalesce several pinning pointers and it should
2486 * be faster since we'd do a memory scan with increasing
2487 * addresses. Note: we can align the address to the allocation
2488 * alignment, so the unique process is more effective.
2490 mword addr = (mword)*start;
2491 addr &= ~(ALLOC_ALIGN - 1);
2492 if (addr >= (mword)start_nursery && addr < (mword)end_nursery)
2493 pin_stage_ptr ((void*)addr);
2495 pin_stats_register_address ((char*)addr, pin_type);
2496 DEBUG (6, if (count) fprintf (gc_debug_file, "Pinning address %p\n", (void*)addr));
2501 DEBUG (7, if (count) fprintf (gc_debug_file, "found %d potential pinned heap pointers\n", count));
2505 * Debugging function: find in the conservative roots where @obj is being pinned.
2507 static G_GNUC_UNUSED void
2508 find_pinning_reference (char *obj, size_t size)
2512 char *endobj = obj + size;
2513 for (i = 0; i < roots_hash_size [0]; ++i) {
2514 for (root = roots_hash [0][i]; root; root = root->next) {
2515 /* if desc is non-null it has precise info */
2516 if (!root->root_desc) {
2517 char ** start = (char**)root->start_root;
2518 while (start < (char**)root->end_root) {
2519 if (*start >= obj && *start < endobj) {
2520 DEBUG (0, fprintf (gc_debug_file, "Object %p referenced in pinned roots %p-%p (at %p in record %p)\n", obj, root->start_root, root->end_root, start, root));
2527 find_pinning_ref_from_thread (obj, size);
2531 * The first thing we do in a collection is to identify pinned objects.
2532 * This function considers all the areas of memory that need to be
2533 * conservatively scanned.
2536 pin_from_roots (void *start_nursery, void *end_nursery)
2540 DEBUG (2, fprintf (gc_debug_file, "Scanning pinned roots (%d bytes, %d/%d entries)\n", (int)roots_size, num_roots_entries [ROOT_TYPE_NORMAL], num_roots_entries [ROOT_TYPE_PINNED]));
2541 /* objects pinned from the API are inside these roots */
2542 for (i = 0; i < roots_hash_size [ROOT_TYPE_PINNED]; ++i) {
2543 for (root = roots_hash [ROOT_TYPE_PINNED][i]; root; root = root->next) {
2544 DEBUG (6, fprintf (gc_debug_file, "Pinned roots %p-%p\n", root->start_root, root->end_root));
2545 conservatively_pin_objects_from ((void**)root->start_root, (void**)root->end_root, start_nursery, end_nursery, PIN_TYPE_OTHER);
2548 /* now deal with the thread stacks
2549 * in the future we should be able to conservatively scan only:
2550 * *) the cpu registers
2551 * *) the unmanaged stack frames
2552 * *) the _last_ managed stack frame
2553 * *) pointers slots in managed frames
2555 scan_thread_data (start_nursery, end_nursery, FALSE);
2557 evacuate_pin_staging_area ();
2561 * The memory area from start_root to end_root contains pointers to objects.
2562 * Their position is precisely described by @desc (this means that the pointer
2563 * can be either NULL or the pointer to the start of an object).
2564 * This functions copies them to to_space updates them.
2567 precisely_scan_objects_from (CopyOrMarkObjectFunc copy_func, void** start_root, void** end_root, char* n_start, char *n_end, mword desc)
2569 switch (desc & ROOT_DESC_TYPE_MASK) {
2570 case ROOT_DESC_BITMAP:
2571 desc >>= ROOT_DESC_TYPE_SHIFT;
2573 if ((desc & 1) && *start_root) {
2574 copy_func (start_root);
2575 DEBUG (9, fprintf (gc_debug_file, "Overwrote root at %p with %p\n", start_root, *start_root));
2576 drain_gray_stack ();
2582 case ROOT_DESC_COMPLEX: {
2583 gsize *bitmap_data = complex_descriptors + (desc >> ROOT_DESC_TYPE_SHIFT);
2584 int bwords = (*bitmap_data) - 1;
2585 void **start_run = start_root;
2587 while (bwords-- > 0) {
2588 gsize bmap = *bitmap_data++;
2589 void **objptr = start_run;
2591 if ((bmap & 1) && *objptr) {
2593 DEBUG (9, fprintf (gc_debug_file, "Overwrote root at %p with %p\n", objptr, *objptr));
2594 drain_gray_stack ();
2599 start_run += GC_BITS_PER_WORD;
2603 case ROOT_DESC_USER: {
2604 MonoGCRootMarkFunc marker = user_descriptors [desc >> ROOT_DESC_TYPE_SHIFT];
2605 marker (start_root, copy_func);
2608 case ROOT_DESC_RUN_LEN:
2609 g_assert_not_reached ();
2611 g_assert_not_reached ();
2616 alloc_fragment (void)
2618 Fragment *frag = fragment_freelist;
2620 fragment_freelist = frag->next;
2624 frag = get_internal_mem (sizeof (Fragment), INTERNAL_MEM_FRAGMENT);
2629 /* size must be a power of 2 */
2631 get_os_memory_aligned (mword size, mword alignment, gboolean activate)
2633 /* Allocate twice the memory to be able to put the block on an aligned address */
2634 char *mem = get_os_memory (size + alignment, activate);
2639 aligned = (char*)((mword)(mem + (alignment - 1)) & ~(alignment - 1));
2640 g_assert (aligned >= mem && aligned + size <= mem + size + alignment && !((mword)aligned & (alignment - 1)));
2643 free_os_memory (mem, aligned - mem);
2644 if (aligned + size < mem + size + alignment)
2645 free_os_memory (aligned + size, (mem + size + alignment) - (aligned + size));
2651 * Allocate and setup the data structures needed to be able to allocate objects
2652 * in the nursery. The nursery is stored in nursery_section.
2655 alloc_nursery (void)
2657 GCMemSection *section;
2663 if (nursery_section)
2665 DEBUG (2, fprintf (gc_debug_file, "Allocating nursery size: %zd\n", nursery_size));
2666 /* later we will alloc a larger area for the nursery but only activate
2667 * what we need. The rest will be used as expansion if we have too many pinned
2668 * objects in the existing nursery.
2670 /* FIXME: handle OOM */
2671 section = get_internal_mem (SIZEOF_GC_MEM_SECTION, INTERNAL_MEM_SECTION);
2673 g_assert (nursery_size == DEFAULT_NURSERY_SIZE);
2674 alloc_size = nursery_size;
2675 #ifdef ALIGN_NURSERY
2676 data = get_os_memory_aligned (alloc_size, alloc_size, TRUE);
2678 data = get_os_memory (alloc_size, TRUE);
2680 nursery_start = data;
2681 nursery_real_end = nursery_start + nursery_size;
2682 UPDATE_HEAP_BOUNDARIES (nursery_start, nursery_real_end);
2683 nursery_next = nursery_start;
2684 total_alloc += alloc_size;
2685 DEBUG (4, fprintf (gc_debug_file, "Expanding nursery size (%p-%p): %zd, total: %zd\n", data, data + alloc_size, nursery_size, total_alloc));
2686 section->data = section->next_data = data;
2687 section->size = alloc_size;
2688 section->end_data = nursery_real_end;
2689 scan_starts = (alloc_size + SCAN_START_SIZE - 1) / SCAN_START_SIZE;
2690 section->scan_starts = get_internal_mem (sizeof (char*) * scan_starts, INTERNAL_MEM_SCAN_STARTS);
2691 section->num_scan_start = scan_starts;
2692 section->block.role = MEMORY_ROLE_GEN0;
2693 section->block.next = NULL;
2695 nursery_section = section;
2697 /* Setup the single first large fragment */
2698 frag = alloc_fragment ();
2699 frag->fragment_start = nursery_start;
2700 frag->fragment_limit = nursery_start;
2701 frag->fragment_end = nursery_real_end;
2702 nursery_frag_real_end = nursery_real_end;
2703 /* FIXME: frag here is lost */
2707 scan_finalizer_entries (CopyOrMarkObjectFunc copy_func, FinalizeEntry *list) {
2710 for (fin = list; fin; fin = fin->next) {
2713 DEBUG (5, fprintf (gc_debug_file, "Scan of fin ready object: %p (%s)\n", fin->object, safe_name (fin->object)));
2714 copy_func (&fin->object);
2718 static mword fragment_total = 0;
2720 * We found a fragment of free memory in the nursery: memzero it and if
2721 * it is big enough, add it to the list of fragments that can be used for
2725 add_nursery_frag (size_t frag_size, char* frag_start, char* frag_end)
2728 DEBUG (4, fprintf (gc_debug_file, "Found empty fragment: %p-%p, size: %zd\n", frag_start, frag_end, frag_size));
2729 binary_protocol_empty (frag_start, frag_size);
2730 /* memsetting just the first chunk start is bound to provide better cache locality */
2731 if (nursery_clear_policy == CLEAR_AT_GC)
2732 memset (frag_start, 0, frag_size);
2733 /* Not worth dealing with smaller fragments: need to tune */
2734 if (frag_size >= FRAGMENT_MIN_SIZE) {
2735 fragment = alloc_fragment ();
2736 fragment->fragment_start = frag_start;
2737 fragment->fragment_limit = frag_start;
2738 fragment->fragment_end = frag_end;
2739 fragment->next = nursery_fragments;
2740 nursery_fragments = fragment;
2741 fragment_total += frag_size;
2743 /* Clear unused fragments, pinning depends on this */
2744 memset (frag_start, 0, frag_size);
2749 generation_name (int generation)
2751 switch (generation) {
2752 case GENERATION_NURSERY: return "nursery";
2753 case GENERATION_OLD: return "old";
2754 default: g_assert_not_reached ();
2758 static DisappearingLinkHashTable*
2759 get_dislink_hash_table (int generation)
2761 switch (generation) {
2762 case GENERATION_NURSERY: return &minor_disappearing_link_hash;
2763 case GENERATION_OLD: return &major_disappearing_link_hash;
2764 default: g_assert_not_reached ();
2768 static FinalizeEntryHashTable*
2769 get_finalize_entry_hash_table (int generation)
2771 switch (generation) {
2772 case GENERATION_NURSERY: return &minor_finalizable_hash;
2773 case GENERATION_OLD: return &major_finalizable_hash;
2774 default: g_assert_not_reached ();
2779 finish_gray_stack (char *start_addr, char *end_addr, int generation)
2784 CopyOrMarkObjectFunc copy_func = current_collection_generation == GENERATION_NURSERY ? copy_object : major_copy_or_mark_object;
2787 * We copied all the reachable objects. Now it's the time to copy
2788 * the objects that were not referenced by the roots, but by the copied objects.
2789 * we built a stack of objects pointed to by gray_start: they are
2790 * additional roots and we may add more items as we go.
2791 * We loop until gray_start == gray_objects which means no more objects have
2792 * been added. Note this is iterative: no recursion is involved.
2793 * We need to walk the LO list as well in search of marked big objects
2794 * (use a flag since this is needed only on major collections). We need to loop
2795 * here as well, so keep a counter of marked LO (increasing it in copy_object).
2796 * To achieve better cache locality and cache usage, we drain the gray stack
2797 * frequently, after each object is copied, and just finish the work here.
2799 drain_gray_stack ();
2801 DEBUG (2, fprintf (gc_debug_file, "%s generation done\n", generation_name (generation)));
2802 /* walk the finalization queue and move also the objects that need to be
2803 * finalized: use the finalized objects as new roots so the objects they depend
2804 * on are also not reclaimed. As with the roots above, only objects in the nursery
2805 * are marked/copied.
2806 * We need a loop here, since objects ready for finalizers may reference other objects
2807 * that are fin-ready. Speedup with a flag?
2810 fin_ready = num_ready_finalizers;
2811 finalize_in_range (copy_func, start_addr, end_addr, generation);
2812 if (generation == GENERATION_OLD)
2813 finalize_in_range (copy_func, nursery_start, nursery_real_end, GENERATION_NURSERY);
2815 /* drain the new stack that might have been created */
2816 DEBUG (6, fprintf (gc_debug_file, "Precise scan of gray area post fin\n"));
2817 drain_gray_stack ();
2818 } while (fin_ready != num_ready_finalizers);
2820 DEBUG (2, fprintf (gc_debug_file, "Finalize queue handling scan for %s generation: %d usecs\n", generation_name (generation), TV_ELAPSED (atv, btv)));
2823 * handle disappearing links
2824 * Note we do this after checking the finalization queue because if an object
2825 * survives (at least long enough to be finalized) we don't clear the link.
2826 * This also deals with a possible issue with the monitor reclamation: with the Boehm
2827 * GC a finalized object my lose the monitor because it is cleared before the finalizer is
2830 g_assert (gray_object_queue_is_empty ());
2832 null_link_in_range (copy_func, start_addr, end_addr, generation);
2833 if (generation == GENERATION_OLD)
2834 null_link_in_range (copy_func, start_addr, end_addr, GENERATION_NURSERY);
2835 if (gray_object_queue_is_empty ())
2837 drain_gray_stack ();
2840 g_assert (gray_object_queue_is_empty ());
2844 check_section_scan_starts (GCMemSection *section)
2847 for (i = 0; i < section->num_scan_start; ++i) {
2848 if (section->scan_starts [i]) {
2849 guint size = safe_object_get_size ((MonoObject*) section->scan_starts [i]);
2850 g_assert (size >= sizeof (MonoObject) && size <= MAX_SMALL_OBJ_SIZE);
2856 check_scan_starts (void)
2858 if (!do_scan_starts_check)
2860 check_section_scan_starts (nursery_section);
2861 major_check_scan_starts ();
2864 static int last_num_pinned = 0;
2867 build_nursery_fragments (int start_pin, int end_pin)
2869 char *frag_start, *frag_end;
2873 while (nursery_fragments) {
2874 Fragment *next = nursery_fragments->next;
2875 nursery_fragments->next = fragment_freelist;
2876 fragment_freelist = nursery_fragments;
2877 nursery_fragments = next;
2879 frag_start = nursery_start;
2881 /* clear scan starts */
2882 memset (nursery_section->scan_starts, 0, nursery_section->num_scan_start * sizeof (gpointer));
2883 for (i = start_pin; i < end_pin; ++i) {
2884 frag_end = pin_queue [i];
2885 /* remove the pin bit from pinned objects */
2886 unpin_object (frag_end);
2887 nursery_section->scan_starts [((char*)frag_end - (char*)nursery_section->data)/SCAN_START_SIZE] = frag_end;
2888 frag_size = frag_end - frag_start;
2890 add_nursery_frag (frag_size, frag_start, frag_end);
2891 frag_size = safe_object_get_size ((MonoObject*)pin_queue [i]);
2892 frag_size += ALLOC_ALIGN - 1;
2893 frag_size &= ~(ALLOC_ALIGN - 1);
2894 frag_start = (char*)pin_queue [i] + frag_size;
2896 nursery_last_pinned_end = frag_start;
2897 frag_end = nursery_real_end;
2898 frag_size = frag_end - frag_start;
2900 add_nursery_frag (frag_size, frag_start, frag_end);
2901 if (!nursery_fragments) {
2902 DEBUG (1, fprintf (gc_debug_file, "Nursery fully pinned (%d)\n", end_pin - start_pin));
2903 for (i = start_pin; i < end_pin; ++i) {
2904 DEBUG (3, fprintf (gc_debug_file, "Bastard pinning obj %p (%s), size: %d\n", pin_queue [i], safe_name (pin_queue [i]), safe_object_get_size (pin_queue [i])));
2909 nursery_next = nursery_frag_real_end = NULL;
2911 /* Clear TLABs for all threads */
2916 scan_from_registered_roots (CopyOrMarkObjectFunc copy_func, char *addr_start, char *addr_end, int root_type)
2920 for (i = 0; i < roots_hash_size [root_type]; ++i) {
2921 for (root = roots_hash [root_type][i]; root; root = root->next) {
2922 DEBUG (6, fprintf (gc_debug_file, "Precise root scan %p-%p (desc: %p)\n", root->start_root, root->end_root, (void*)root->root_desc));
2923 precisely_scan_objects_from (copy_func, (void**)root->start_root, (void**)root->end_root, addr_start, addr_end, root->root_desc);
2929 dump_occupied (char *start, char *end, char *section_start)
2931 fprintf (heap_dump_file, "<occupied offset=\"%zd\" size=\"%zd\"/>\n", start - section_start, end - start);
2935 dump_section (GCMemSection *section, const char *type)
2937 char *start = section->data;
2938 char *end = section->data + section->size;
2939 char *occ_start = NULL;
2941 char *old_start = NULL; /* just for debugging */
2943 fprintf (heap_dump_file, "<section type=\"%s\" size=\"%zu\">\n", type, section->size);
2945 while (start < end) {
2949 if (!*(void**)start) {
2951 dump_occupied (occ_start, start, section->data);
2954 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
2957 g_assert (start < section->next_data);
2962 vt = (GCVTable*)LOAD_VTABLE (start);
2965 size = safe_object_get_size ((MonoObject*) start);
2966 size += ALLOC_ALIGN - 1;
2967 size &= ~(ALLOC_ALIGN - 1);
2970 fprintf (heap_dump_file, "<object offset=\"%d\" class=\"%s.%s\" size=\"%d\"/>\n",
2971 start - section->data,
2972 vt->klass->name_space, vt->klass->name,
2980 dump_occupied (occ_start, start, section->data);
2982 fprintf (heap_dump_file, "</section>\n");
2986 dump_object (MonoObject *obj, gboolean dump_location)
2988 static char class_name [1024];
2990 MonoClass *class = mono_object_class (obj);
2994 * Python's XML parser is too stupid to parse angle brackets
2995 * in strings, so we just ignore them;
2998 while (class->name [i] && j < sizeof (class_name) - 1) {
2999 if (!strchr ("<>\"", class->name [i]))
3000 class_name [j++] = class->name [i];
3003 g_assert (j < sizeof (class_name));
3006 fprintf (heap_dump_file, "<object class=\"%s.%s\" size=\"%d\"",
3007 class->name_space, class_name,
3008 safe_object_get_size (obj));
3009 if (dump_location) {
3010 const char *location;
3011 if (ptr_in_nursery (obj))
3012 location = "nursery";
3013 else if (safe_object_get_size (obj) <= MAX_SMALL_OBJ_SIZE)
3017 fprintf (heap_dump_file, " location=\"%s\"", location);
3019 fprintf (heap_dump_file, "/>\n");
3023 dump_heap (const char *type, int num, const char *reason)
3025 static char const *internal_mem_names [] = { "pin-queue", "fragment", "section", "scan-starts",
3026 "fin-table", "finalize-entry", "dislink-table",
3027 "dislink", "roots-table", "root-record", "statistics",
3028 "remset", "gray-queue", "store-remset" };
3034 fprintf (heap_dump_file, "<collection type=\"%s\" num=\"%d\"", type, num);
3036 fprintf (heap_dump_file, " reason=\"%s\"", reason);
3037 fprintf (heap_dump_file, ">\n");
3038 fprintf (heap_dump_file, "<other-mem-usage type=\"pinned-chunks\" size=\"%ld\"/>\n", pinned_chunk_bytes_alloced);
3039 fprintf (heap_dump_file, "<other-mem-usage type=\"large-internal\" size=\"%ld\"/>\n", large_internal_bytes_alloced);
3040 fprintf (heap_dump_file, "<other-mem-usage type=\"mempools\" size=\"%ld\"/>\n", mono_mempool_get_bytes_allocated ());
3041 for (i = 0; i < INTERNAL_MEM_MAX; ++i)
3042 fprintf (heap_dump_file, "<other-mem-usage type=\"%s\" size=\"%ld\"/>\n", internal_mem_names [i], small_internal_mem_bytes [i]);
3043 fprintf (heap_dump_file, "<pinned type=\"stack\" bytes=\"%zu\"/>\n", pinned_byte_counts [PIN_TYPE_STACK]);
3044 /* fprintf (heap_dump_file, "<pinned type=\"static-data\" bytes=\"%d\"/>\n", pinned_byte_counts [PIN_TYPE_STATIC_DATA]); */
3045 fprintf (heap_dump_file, "<pinned type=\"other\" bytes=\"%zu\"/>\n", pinned_byte_counts [PIN_TYPE_OTHER]);
3047 fprintf (heap_dump_file, "<pinned-objects>\n");
3048 for (list = pinned_objects; list; list = list->next)
3049 dump_object (list->obj, TRUE);
3050 fprintf (heap_dump_file, "</pinned-objects>\n");
3052 dump_section (nursery_section, "nursery");
3054 major_dump_non_pinned_sections ();
3056 fprintf (heap_dump_file, "<los>\n");
3057 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
3058 dump_object ((MonoObject*)bigobj->data, FALSE);
3059 fprintf (heap_dump_file, "</los>\n");
3061 fprintf (heap_dump_file, "</collection>\n");
3067 static gboolean inited = FALSE;
3069 #ifdef HEAVY_STATISTICS
3070 num_copy_object_called = 0;
3071 num_objects_copied = 0;
3077 mono_counters_register ("Minor fragment clear", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_pre_collection_fragment_clear);
3078 mono_counters_register ("Minor pinning", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_pinning);
3079 mono_counters_register ("Minor scan remsets", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_scan_remsets);
3080 mono_counters_register ("Minor scan pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_scan_pinned);
3081 mono_counters_register ("Minor scan registered roots", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_scan_registered_roots);
3082 mono_counters_register ("Minor scan thread data", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_scan_thread_data);
3083 mono_counters_register ("Minor finish gray stack", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_finish_gray_stack);
3084 mono_counters_register ("Minor fragment creation", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_fragment_creation);
3086 mono_counters_register ("Major fragment clear", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_pre_collection_fragment_clear);
3087 mono_counters_register ("Major pinning", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_pinning);
3088 mono_counters_register ("Major scan pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_pinned);
3089 mono_counters_register ("Major scan registered roots", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_registered_roots);
3090 mono_counters_register ("Major scan thread data", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_thread_data);
3091 mono_counters_register ("Major scan alloc_pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_alloc_pinned);
3092 mono_counters_register ("Major scan finalized", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_finalized);
3093 mono_counters_register ("Major scan big objects", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_big_objects);
3094 mono_counters_register ("Major finish gray stack", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_finish_gray_stack);
3095 mono_counters_register ("Major sweep", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_sweep);
3096 mono_counters_register ("Major fragment creation", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_fragment_creation);
3098 #ifdef HEAVY_STATISTICS
3099 mono_counters_register ("WBarrier set field", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_field);
3100 mono_counters_register ("WBarrier set arrayref", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_arrayref);
3101 mono_counters_register ("WBarrier arrayref copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_arrayref_copy);
3102 mono_counters_register ("WBarrier generic store called", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_generic_store);
3103 mono_counters_register ("WBarrier generic store stored", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_generic_store_remset);
3104 mono_counters_register ("WBarrier set root", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_root);
3105 mono_counters_register ("WBarrier value copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_value_copy);
3106 mono_counters_register ("WBarrier object copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_object_copy);
3108 mono_counters_register ("# objects allocated", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_alloced);
3109 mono_counters_register ("bytes allocated", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_bytes_alloced);
3110 mono_counters_register ("# objects allocated degraded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_alloced_degraded);
3111 mono_counters_register ("bytes allocated degraded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_bytes_alloced_degraded);
3112 mono_counters_register ("bytes allocated in LOS", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_bytes_alloced_los);
3113 mono_counters_register ("# copy_object() called (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_called_nursery);
3114 mono_counters_register ("# objects copied (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_copied_nursery);
3115 mono_counters_register ("# copy_object() called (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_called_major);
3116 mono_counters_register ("# objects copied (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_copied_major);
3118 mono_counters_register ("# copy_object() failed from space", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_failed_from_space);
3119 mono_counters_register ("# copy_object() failed forwarded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_failed_forwarded);
3120 mono_counters_register ("# copy_object() failed pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_failed_pinned);
3121 mono_counters_register ("# copy_object() failed large or pinned chunk", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_failed_large_pinned);
3122 mono_counters_register ("# copy_object() failed to space", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_failed_to_space);
3124 mono_counters_register ("Store remsets", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_store_remsets);
3125 mono_counters_register ("Unique store remsets", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_store_remsets_unique);
3126 mono_counters_register ("Saved remsets 1", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_saved_remsets_1);
3127 mono_counters_register ("Saved remsets 2", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_saved_remsets_2);
3128 mono_counters_register ("Global remsets added", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_added);
3129 mono_counters_register ("Global remsets processed", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_processed);
3136 commit_stats (int generation)
3138 #ifdef HEAVY_STATISTICS
3139 if (generation == GENERATION_NURSERY) {
3140 stat_copy_object_called_nursery += num_copy_object_called;
3141 stat_objects_copied_nursery += num_objects_copied;
3143 g_assert (generation == GENERATION_OLD);
3144 stat_copy_object_called_major += num_copy_object_called;
3145 stat_objects_copied_major += num_objects_copied;
3151 * Collect objects in the nursery. Returns whether to trigger a major
3155 collect_nursery (size_t requested_size)
3157 size_t max_garbage_amount;
3158 char *orig_nursery_next;
3159 TV_DECLARE (all_atv);
3160 TV_DECLARE (all_btv);
3164 current_collection_generation = GENERATION_NURSERY;
3167 binary_protocol_collection (GENERATION_NURSERY);
3168 check_scan_starts ();
3171 orig_nursery_next = nursery_next;
3172 nursery_next = MAX (nursery_next, nursery_last_pinned_end);
3173 /* FIXME: optimize later to use the higher address where an object can be present */
3174 nursery_next = MAX (nursery_next, nursery_real_end);
3176 DEBUG (1, fprintf (gc_debug_file, "Start nursery collection %d %p-%p, size: %d\n", num_minor_gcs, nursery_start, nursery_next, (int)(nursery_next - nursery_start)));
3177 max_garbage_amount = nursery_next - nursery_start;
3178 g_assert (nursery_section->size >= max_garbage_amount);
3180 /* world must be stopped already */
3181 TV_GETTIME (all_atv);
3184 /* Pinning depends on this */
3185 clear_nursery_fragments (orig_nursery_next);
3188 time_minor_pre_collection_fragment_clear += TV_ELAPSED_MS (atv, btv);
3191 check_for_xdomain_refs ();
3193 nursery_section->next_data = nursery_next;
3195 major_start_nursery_collection ();
3197 gray_object_queue_init ();
3200 mono_stats.minor_gc_count ++;
3201 /* pin from pinned handles */
3203 pin_from_roots (nursery_start, nursery_next);
3204 /* identify pinned objects */
3205 optimize_pin_queue (0);
3206 next_pin_slot = pin_objects_from_addresses (nursery_section, pin_queue, pin_queue + next_pin_slot, nursery_start, nursery_next);
3207 nursery_section->pin_queue_start = 0;
3208 nursery_section->pin_queue_end = next_pin_slot;
3210 time_minor_pinning += TV_ELAPSED_MS (btv, atv);
3211 DEBUG (2, fprintf (gc_debug_file, "Finding pinned pointers: %d in %d usecs\n", next_pin_slot, TV_ELAPSED (btv, atv)));
3212 DEBUG (4, fprintf (gc_debug_file, "Start scan with %d pinned objects\n", next_pin_slot));
3214 if (consistency_check_at_minor_collection)
3215 check_consistency ();
3218 * walk all the roots and copy the young objects to the old generation,
3219 * starting from to_space
3222 scan_from_remsets (nursery_start, nursery_next);
3223 /* we don't have complete write barrier yet, so we scan all the old generation sections */
3225 time_minor_scan_remsets += TV_ELAPSED_MS (atv, btv);
3226 DEBUG (2, fprintf (gc_debug_file, "Old generation scan: %d usecs\n", TV_ELAPSED (atv, btv)));
3228 drain_gray_stack ();
3231 time_minor_scan_pinned += TV_ELAPSED_MS (btv, atv);
3232 /* registered roots, this includes static fields */
3233 scan_from_registered_roots (copy_object, nursery_start, nursery_next, ROOT_TYPE_NORMAL);
3234 scan_from_registered_roots (copy_object, nursery_start, nursery_next, ROOT_TYPE_WBARRIER);
3236 time_minor_scan_registered_roots += TV_ELAPSED_MS (atv, btv);
3238 scan_thread_data (nursery_start, nursery_next, TRUE);
3240 time_minor_scan_thread_data += TV_ELAPSED_MS (btv, atv);
3243 finish_gray_stack (nursery_start, nursery_next, GENERATION_NURSERY);
3245 time_minor_finish_gray_stack += TV_ELAPSED_MS (btv, atv);
3247 /* walk the pin_queue, build up the fragment list of free memory, unmark
3248 * pinned objects as we go, memzero() the empty fragments so they are ready for the
3251 build_nursery_fragments (0, next_pin_slot);
3253 time_minor_fragment_creation += TV_ELAPSED_MS (atv, btv);
3254 DEBUG (2, fprintf (gc_debug_file, "Fragment creation: %d usecs, %zd bytes available\n", TV_ELAPSED (atv, btv), fragment_total));
3256 major_finish_nursery_collection ();
3258 TV_GETTIME (all_btv);
3259 mono_stats.minor_gc_time_usecs += TV_ELAPSED (all_atv, all_btv);
3262 dump_heap ("minor", num_minor_gcs - 1, NULL);
3264 /* prepare the pin queue for the next collection */
3265 last_num_pinned = next_pin_slot;
3267 if (fin_ready_list || critical_fin_list) {
3268 DEBUG (4, fprintf (gc_debug_file, "Finalizer-thread wakeup: ready %d\n", num_ready_finalizers));
3269 mono_gc_finalize_notify ();
3273 g_assert (gray_object_queue_is_empty ());
3275 commit_stats (GENERATION_NURSERY);
3277 check_scan_starts ();
3279 current_collection_generation = -1;
3281 return major_need_major_collection ();
3285 major_collection (const char *reason)
3287 if (g_getenv ("MONO_GC_NO_MAJOR")) {
3288 collect_nursery (0);
3292 current_collection_generation = GENERATION_OLD;
3293 major_do_collection (reason);
3294 current_collection_generation = -1;
3298 * When deciding if it's better to collect or to expand, keep track
3299 * of how much garbage was reclaimed with the last collection: if it's too
3301 * This is called when we could not allocate a small object.
3303 static void __attribute__((noinline))
3304 minor_collect_or_expand_inner (size_t size)
3306 int do_minor_collection = 1;
3308 if (!nursery_section) {
3312 if (do_minor_collection) {
3314 if (collect_nursery (size))
3315 major_collection ("minor overflow");
3316 DEBUG (2, fprintf (gc_debug_file, "Heap size: %zd, LOS size: %zd\n", total_alloc, los_memory_usage));
3318 /* this also sets the proper pointers for the next allocation */
3319 if (!search_fragment_for_size (size)) {
3321 /* TypeBuilder and MonoMethod are killing mcs with fragmentation */
3322 DEBUG (1, fprintf (gc_debug_file, "nursery collection didn't find enough room for %zd alloc (%d pinned)\n", size, last_num_pinned));
3323 for (i = 0; i < last_num_pinned; ++i) {
3324 DEBUG (3, fprintf (gc_debug_file, "Bastard pinning obj %p (%s), size: %d\n", pin_queue [i], safe_name (pin_queue [i]), safe_object_get_size (pin_queue [i])));
3329 //report_internal_mem_usage ();
3333 * ######################################################################
3334 * ######## Memory allocation from the OS
3335 * ######################################################################
3336 * This section of code deals with getting memory from the OS and
3337 * allocating memory for GC-internal data structures.
3338 * Internal memory can be handled with a freelist for small objects.
3342 * Allocate a big chunk of memory from the OS (usually 64KB to several megabytes).
3343 * This must not require any lock.
3346 get_os_memory (size_t size, int activate)
3349 unsigned long prot_flags = activate? MONO_MMAP_READ|MONO_MMAP_WRITE: MONO_MMAP_NONE;
3351 prot_flags |= MONO_MMAP_PRIVATE | MONO_MMAP_ANON;
3352 size += pagesize - 1;
3353 size &= ~(pagesize - 1);
3354 ptr = mono_valloc (0, size, prot_flags);
3359 * Free the memory returned by get_os_memory (), returning it to the OS.
3362 free_os_memory (void *addr, size_t size)
3364 mono_vfree (addr, size);
3371 report_pinned_chunk (PinnedChunk *chunk, int seq) {
3373 int i, free_pages, num_free, free_mem;
3375 for (i = 0; i < chunk->num_pages; ++i) {
3376 if (!chunk->page_sizes [i])
3379 printf ("Pinned chunk %d at %p, size: %d, pages: %d, free: %d\n", seq, chunk, chunk->num_pages * FREELIST_PAGESIZE, chunk->num_pages, free_pages);
3380 free_mem = FREELIST_PAGESIZE * free_pages;
3381 for (i = 0; i < FREELIST_NUM_SLOTS; ++i) {
3382 if (!chunk->free_list [i])
3385 p = chunk->free_list [i];
3390 printf ("\tfree list of size %d, %d items\n", freelist_sizes [i], num_free);
3391 free_mem += freelist_sizes [i] * num_free;
3393 printf ("\tfree memory in chunk: %d\n", free_mem);
3399 static G_GNUC_UNUSED void
3400 report_internal_mem_usage (void) {
3403 printf ("Internal memory usage:\n");
3405 for (chunk = internal_chunk_list; chunk; chunk = chunk->block.next) {
3406 report_pinned_chunk (chunk, i++);
3408 printf ("Pinned memory usage:\n");
3409 major_report_pinned_memory_usage ();
3413 * Find the slot number in the freelist for memory chunks that
3414 * can contain @size objects.
3417 slot_for_size (size_t size)
3420 /* do a binary search or lookup table later. */
3421 for (slot = 0; slot < FREELIST_NUM_SLOTS; ++slot) {
3422 if (freelist_sizes [slot] >= size)
3425 g_assert_not_reached ();
3430 * Build a free list for @size memory chunks from the memory area between
3431 * start_page and end_page.
3434 build_freelist (PinnedChunk *chunk, int slot, int size, char *start_page, char *end_page)
3438 /*g_print ("building freelist for slot %d, size %d in %p\n", slot, size, chunk);*/
3439 p = (void**)start_page;
3440 end = (void**)(end_page - size);
3441 g_assert (!chunk->free_list [slot]);
3442 chunk->free_list [slot] = p;
3443 while ((char*)p + size <= (char*)end) {
3445 *p = (void*)((char*)p + size);
3449 /*g_print ("%d items created, max: %d\n", count, (end_page - start_page) / size);*/
3453 alloc_pinned_chunk (void)
3457 int size = PINNED_CHUNK_SIZE;
3459 chunk = get_os_memory_aligned (size, size, TRUE);
3460 chunk->block.role = MEMORY_ROLE_PINNED;
3462 UPDATE_HEAP_BOUNDARIES (chunk, ((char*)chunk + size));
3463 total_alloc += size;
3464 pinned_chunk_bytes_alloced += size;
3466 /* setup the bookeeping fields */
3467 chunk->num_pages = size / FREELIST_PAGESIZE;
3468 offset = G_STRUCT_OFFSET (PinnedChunk, data);
3469 chunk->page_sizes = (void*)((char*)chunk + offset);
3470 offset += sizeof (int) * chunk->num_pages;
3471 offset += ALLOC_ALIGN - 1;
3472 offset &= ~(ALLOC_ALIGN - 1);
3473 chunk->free_list = (void*)((char*)chunk + offset);
3474 offset += sizeof (void*) * FREELIST_NUM_SLOTS;
3475 offset += ALLOC_ALIGN - 1;
3476 offset &= ~(ALLOC_ALIGN - 1);
3477 chunk->start_data = (void*)((char*)chunk + offset);
3479 /* allocate the first page to the freelist */
3480 chunk->page_sizes [0] = PINNED_FIRST_SLOT_SIZE;
3481 build_freelist (chunk, slot_for_size (PINNED_FIRST_SLOT_SIZE), PINNED_FIRST_SLOT_SIZE, chunk->start_data, ((char*)chunk + FREELIST_PAGESIZE));
3482 DEBUG (4, fprintf (gc_debug_file, "Allocated pinned chunk %p, size: %d\n", chunk, size));
3486 /* assumes freelist for slot is empty, so try to alloc a new page */
3488 get_chunk_freelist (PinnedChunk *chunk, int slot)
3492 p = chunk->free_list [slot];
3494 chunk->free_list [slot] = *p;
3497 for (i = 0; i < chunk->num_pages; ++i) {
3499 if (chunk->page_sizes [i])
3501 size = freelist_sizes [slot];
3502 chunk->page_sizes [i] = size;
3503 build_freelist (chunk, slot, size, (char*)chunk + FREELIST_PAGESIZE * i, (char*)chunk + FREELIST_PAGESIZE * (i + 1));
3507 p = chunk->free_list [slot];
3509 chunk->free_list [slot] = *p;
3515 /* used for the GC-internal data structures */
3517 get_internal_mem (size_t size, int type)
3521 PinnedChunk *pchunk;
3523 if (size > freelist_sizes [FREELIST_NUM_SLOTS - 1]) {
3524 LargeInternalMemHeader *mh;
3526 size += sizeof (LargeInternalMemHeader);
3527 mh = get_os_memory (size, TRUE);
3528 mh->magic = LARGE_INTERNAL_MEM_HEADER_MAGIC;
3531 large_internal_bytes_alloced += size;
3536 slot = slot_for_size (size);
3537 g_assert (size <= freelist_sizes [slot]);
3539 small_internal_mem_bytes [type] += freelist_sizes [slot];
3541 for (pchunk = internal_chunk_list; pchunk; pchunk = pchunk->block.next) {
3542 void **p = pchunk->free_list [slot];
3544 pchunk->free_list [slot] = *p;
3545 memset (p, 0, size);
3549 for (pchunk = internal_chunk_list; pchunk; pchunk = pchunk->block.next) {
3550 res = get_chunk_freelist (pchunk, slot);
3552 memset (res, 0, size);
3556 pchunk = alloc_pinned_chunk ();
3557 /* FIXME: handle OOM */
3558 pchunk->block.next = internal_chunk_list;
3559 internal_chunk_list = pchunk;
3560 res = get_chunk_freelist (pchunk, slot);
3561 memset (res, 0, size);
3566 free_internal_mem (void *addr, int type)
3568 PinnedChunk *pchunk;
3569 LargeInternalMemHeader *mh;
3572 for (pchunk = internal_chunk_list; pchunk; pchunk = pchunk->block.next) {
3573 /*printf ("trying to free %p in %p (pages: %d)\n", addr, pchunk, pchunk->num_pages);*/
3574 if (addr >= (void*)pchunk && (char*)addr < (char*)pchunk + pchunk->num_pages * FREELIST_PAGESIZE) {
3575 int offset = (char*)addr - (char*)pchunk;
3576 int page = offset / FREELIST_PAGESIZE;
3577 int slot = slot_for_size (pchunk->page_sizes [page]);
3579 *p = pchunk->free_list [slot];
3580 pchunk->free_list [slot] = p;
3582 small_internal_mem_bytes [type] -= freelist_sizes [slot];
3587 mh = (LargeInternalMemHeader*)((char*)addr - G_STRUCT_OFFSET (LargeInternalMemHeader, data));
3588 g_assert (mh->magic == LARGE_INTERNAL_MEM_HEADER_MAGIC);
3589 large_internal_bytes_alloced -= mh->size;
3590 free_os_memory (mh, mh->size);
3594 * ######################################################################
3595 * ######## Object allocation
3596 * ######################################################################
3597 * This section of code deals with allocating memory for objects.
3598 * There are several ways:
3599 * *) allocate large objects
3600 * *) allocate normal objects
3601 * *) fast lock-free allocation
3602 * *) allocation of pinned objects
3606 free_large_object (LOSObject *obj)
3608 size_t size = obj->size;
3609 DEBUG (4, fprintf (gc_debug_file, "Freed large object %p, size %zd\n", obj->data, obj->size));
3610 binary_protocol_empty (obj->data, obj->size);
3612 los_memory_usage -= size;
3613 size += sizeof (LOSObject);
3614 size += pagesize - 1;
3615 size &= ~(pagesize - 1);
3616 total_alloc -= size;
3618 free_os_memory (obj, size);
3622 * Objects with size >= 64KB are allocated in the large object space.
3623 * They are currently kept track of with a linked list.
3624 * They don't move, so there is no need to pin them during collection
3625 * and we avoid the memcpy overhead.
3627 static void* __attribute__((noinline))
3628 alloc_large_inner (MonoVTable *vtable, size_t size)
3634 g_assert (size > MAX_SMALL_OBJ_SIZE);
3636 if (los_memory_usage > next_los_collection) {
3637 static mword last_los_memory_usage = 0;
3639 mword los_memory_alloced;
3640 mword old_los_memory_usage;
3641 mword los_memory_saved;
3643 mword allowance_target;
3646 DEBUG (4, fprintf (gc_debug_file, "Should trigger major collection: req size %zd (los already: %zu, limit: %zu)\n", size, los_memory_usage, next_los_collection));
3649 g_assert (los_memory_usage >= last_los_memory_usage);
3650 los_memory_alloced = los_memory_usage - last_los_memory_usage;
3651 old_los_memory_usage = los_memory_usage;
3653 major_collection ("LOS overflow");
3655 los_memory_saved = MAX (old_los_memory_usage - los_memory_usage, 1);
3656 save_target = los_memory_usage / 2;
3658 * see the comment at the end of major_collection()
3659 * for the explanation for this calculation.
3661 allowance_target = (mword)((double)save_target * (double)los_memory_alloced / (double)los_memory_saved);
3662 allowance = MAX (MIN (allowance_target, los_memory_usage), MIN_LOS_ALLOWANCE);
3663 next_los_collection = los_memory_usage + allowance;
3665 last_los_memory_usage = los_memory_usage;
3670 alloc_size += sizeof (LOSObject);
3671 alloc_size += pagesize - 1;
3672 alloc_size &= ~(pagesize - 1);
3673 /* FIXME: handle OOM */
3674 obj = get_os_memory (alloc_size, TRUE);
3675 g_assert (!((mword)obj->data & (ALLOC_ALIGN - 1)));
3677 vtslot = (void**)obj->data;
3679 total_alloc += alloc_size;
3680 UPDATE_HEAP_BOUNDARIES (obj->data, (char*)obj->data + size);
3681 obj->next = los_object_list;
3682 los_object_list = obj;
3683 los_memory_usage += size;
3685 DEBUG (4, fprintf (gc_debug_file, "Allocated large object %p, vtable: %p (%s), size: %zd\n", obj->data, vtable, vtable->klass->name, size));
3686 binary_protocol_alloc (obj->data, vtable, size);
3690 /* check if we have a suitable fragment in nursery_fragments to be able to allocate
3691 * an object of size @size
3692 * Return FALSE if not found (which means we need a collection)
3695 search_fragment_for_size (size_t size)
3697 Fragment *frag, *prev;
3698 DEBUG (4, fprintf (gc_debug_file, "Searching nursery fragment %p, size: %zd\n", nursery_frag_real_end, size));
3700 if (nursery_frag_real_end > nursery_next && nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
3701 /* Clear the remaining space, pinning depends on this */
3702 memset (nursery_next, 0, nursery_frag_real_end - nursery_next);
3705 for (frag = nursery_fragments; frag; frag = frag->next) {
3706 if (size <= (frag->fragment_end - frag->fragment_start)) {
3707 /* remove from the list */
3709 prev->next = frag->next;
3711 nursery_fragments = frag->next;
3712 nursery_next = frag->fragment_start;
3713 nursery_frag_real_end = frag->fragment_end;
3715 DEBUG (4, fprintf (gc_debug_file, "Using nursery fragment %p-%p, size: %zd (req: %zd)\n", nursery_next, nursery_frag_real_end, nursery_frag_real_end - nursery_next, size));
3716 frag->next = fragment_freelist;
3717 fragment_freelist = frag;
3726 * Provide a variant that takes just the vtable for small fixed-size objects.
3727 * The aligned size is already computed and stored in vt->gc_descr.
3728 * Note: every SCAN_START_SIZE or so we are given the chance to do some special
3729 * processing. We can keep track of where objects start, for example,
3730 * so when we scan the thread stacks for pinned objects, we can start
3731 * a search for the pinned object in SCAN_START_SIZE chunks.
3734 mono_gc_alloc_obj_nolock (MonoVTable *vtable, size_t size)
3736 /* FIXME: handle OOM */
3742 HEAVY_STAT (++stat_objects_alloced);
3743 if (size <= MAX_SMALL_OBJ_SIZE)
3744 HEAVY_STAT (stat_bytes_alloced += size);
3746 HEAVY_STAT (stat_bytes_alloced_los += size);
3748 size += ALLOC_ALIGN - 1;
3749 size &= ~(ALLOC_ALIGN - 1);
3751 g_assert (vtable->gc_descr);
3753 if (G_UNLIKELY (collect_before_allocs)) {
3754 if (nursery_section) {
3756 collect_nursery (0);
3758 if (!degraded_mode && !search_fragment_for_size (size)) {
3760 g_assert_not_reached ();
3766 * We must already have the lock here instead of after the
3767 * fast path because we might be interrupted in the fast path
3768 * (after confirming that new_next < TLAB_TEMP_END) by the GC,
3769 * and we'll end up allocating an object in a fragment which
3770 * no longer belongs to us.
3772 * The managed allocator does not do this, but it's treated
3773 * specially by the world-stopping code.
3776 if (size > MAX_SMALL_OBJ_SIZE) {
3777 p = alloc_large_inner (vtable, size);
3779 /* tlab_next and tlab_temp_end are TLS vars so accessing them might be expensive */
3781 p = (void**)TLAB_NEXT;
3782 /* FIXME: handle overflow */
3783 new_next = (char*)p + size;
3784 TLAB_NEXT = new_next;
3786 if (G_LIKELY (new_next < TLAB_TEMP_END)) {
3790 * FIXME: We might need a memory barrier here so the change to tlab_next is
3791 * visible before the vtable store.
3794 DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
3795 binary_protocol_alloc (p , vtable, size);
3796 g_assert (*p == NULL);
3799 g_assert (TLAB_NEXT == new_next);
3806 /* there are two cases: the object is too big or we run out of space in the TLAB */
3807 /* we also reach here when the thread does its first allocation after a minor
3808 * collection, since the tlab_ variables are initialized to NULL.
3809 * there can be another case (from ORP), if we cooperate with the runtime a bit:
3810 * objects that need finalizers can have the high bit set in their size
3811 * so the above check fails and we can readily add the object to the queue.
3812 * This avoids taking again the GC lock when registering, but this is moot when
3813 * doing thread-local allocation, so it may not be a good idea.
3815 g_assert (TLAB_NEXT == new_next);
3816 if (TLAB_NEXT >= TLAB_REAL_END) {
3818 * Run out of space in the TLAB. When this happens, some amount of space
3819 * remains in the TLAB, but not enough to satisfy the current allocation
3820 * request. Currently, we retire the TLAB in all cases, later we could
3821 * keep it if the remaining space is above a treshold, and satisfy the
3822 * allocation directly from the nursery.
3825 /* when running in degraded mode, we continue allocing that way
3826 * for a while, to decrease the number of useless nursery collections.
3828 if (degraded_mode && degraded_mode < DEFAULT_NURSERY_SIZE) {
3829 p = alloc_degraded (vtable, size);
3833 if (size > tlab_size) {
3834 /* Allocate directly from the nursery */
3835 if (nursery_next + size >= nursery_frag_real_end) {
3836 if (!search_fragment_for_size (size)) {
3837 minor_collect_or_expand_inner (size);
3838 if (degraded_mode) {
3839 p = alloc_degraded (vtable, size);
3845 p = (void*)nursery_next;
3846 nursery_next += size;
3847 if (nursery_next > nursery_frag_real_end) {
3852 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
3853 memset (p, 0, size);
3856 DEBUG (3, fprintf (gc_debug_file, "Retire TLAB: %p-%p [%ld]\n", TLAB_START, TLAB_REAL_END, (long)(TLAB_REAL_END - TLAB_NEXT - size)));
3858 if (nursery_next + tlab_size >= nursery_frag_real_end) {
3859 res = search_fragment_for_size (tlab_size);
3861 minor_collect_or_expand_inner (tlab_size);
3862 if (degraded_mode) {
3863 p = alloc_degraded (vtable, size);
3869 /* Allocate a new TLAB from the current nursery fragment */
3870 TLAB_START = nursery_next;
3871 nursery_next += tlab_size;
3872 TLAB_NEXT = TLAB_START;
3873 TLAB_REAL_END = TLAB_START + tlab_size;
3874 TLAB_TEMP_END = TLAB_START + MIN (SCAN_START_SIZE, tlab_size);
3876 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
3877 memset (TLAB_START, 0, tlab_size);
3879 /* Allocate from the TLAB */
3880 p = (void*)TLAB_NEXT;
3882 g_assert (TLAB_NEXT <= TLAB_REAL_END);
3884 nursery_section->scan_starts [((char*)p - (char*)nursery_section->data)/SCAN_START_SIZE] = (char*)p;
3887 /* Reached tlab_temp_end */
3889 /* record the scan start so we can find pinned objects more easily */
3890 nursery_section->scan_starts [((char*)p - (char*)nursery_section->data)/SCAN_START_SIZE] = (char*)p;
3891 /* we just bump tlab_temp_end as well */
3892 TLAB_TEMP_END = MIN (TLAB_REAL_END, TLAB_NEXT + SCAN_START_SIZE);
3893 DEBUG (5, fprintf (gc_debug_file, "Expanding local alloc: %p-%p\n", TLAB_NEXT, TLAB_TEMP_END));
3897 DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
3898 binary_protocol_alloc (p, vtable, size);
3905 mono_gc_try_alloc_obj_nolock (MonoVTable *vtable, size_t size)
3911 size += ALLOC_ALIGN - 1;
3912 size &= ~(ALLOC_ALIGN - 1);
3914 g_assert (vtable->gc_descr);
3915 if (size <= MAX_SMALL_OBJ_SIZE) {
3916 /* tlab_next and tlab_temp_end are TLS vars so accessing them might be expensive */
3918 p = (void**)TLAB_NEXT;
3919 /* FIXME: handle overflow */
3920 new_next = (char*)p + size;
3921 TLAB_NEXT = new_next;
3923 if (G_LIKELY (new_next < TLAB_TEMP_END)) {
3927 * FIXME: We might need a memory barrier here so the change to tlab_next is
3928 * visible before the vtable store.
3931 HEAVY_STAT (++stat_objects_alloced);
3932 HEAVY_STAT (stat_bytes_alloced += size);
3934 DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
3935 binary_protocol_alloc (p, vtable, size);
3936 g_assert (*p == NULL);
3939 g_assert (TLAB_NEXT == new_next);
3948 mono_gc_alloc_obj (MonoVTable *vtable, size_t size)
3951 #ifndef DISABLE_CRITICAL_REGION
3953 ENTER_CRITICAL_REGION;
3954 res = mono_gc_try_alloc_obj_nolock (vtable, size);
3956 EXIT_CRITICAL_REGION;
3959 EXIT_CRITICAL_REGION;
3962 res = mono_gc_alloc_obj_nolock (vtable, size);
3968 mono_gc_alloc_vector (MonoVTable *vtable, size_t size, uintptr_t max_length)
3971 #ifndef DISABLE_CRITICAL_REGION
3973 ENTER_CRITICAL_REGION;
3974 arr = mono_gc_try_alloc_obj_nolock (vtable, size);
3976 arr->max_length = max_length;
3977 EXIT_CRITICAL_REGION;
3980 EXIT_CRITICAL_REGION;
3985 arr = mono_gc_alloc_obj_nolock (vtable, size);
3986 arr->max_length = max_length;
3994 mono_gc_alloc_array (MonoVTable *vtable, size_t size, uintptr_t max_length, uintptr_t bounds_size)
3997 MonoArrayBounds *bounds;
4001 arr = mono_gc_alloc_obj_nolock (vtable, size);
4002 arr->max_length = max_length;
4004 bounds = (MonoArrayBounds*)((char*)arr + size - bounds_size);
4005 arr->bounds = bounds;
4013 mono_gc_alloc_string (MonoVTable *vtable, size_t size, gint32 len)
4016 #ifndef DISABLE_CRITICAL_REGION
4018 ENTER_CRITICAL_REGION;
4019 str = mono_gc_try_alloc_obj_nolock (vtable, size);
4022 EXIT_CRITICAL_REGION;
4025 EXIT_CRITICAL_REGION;
4030 str = mono_gc_alloc_obj_nolock (vtable, size);
4039 * To be used for interned strings and possibly MonoThread, reflection handles.
4040 * We may want to explicitly free these objects.
4043 mono_gc_alloc_pinned_obj (MonoVTable *vtable, size_t size)
4045 /* FIXME: handle OOM */
4047 size += ALLOC_ALIGN - 1;
4048 size &= ~(ALLOC_ALIGN - 1);
4050 if (size > MAX_SMALL_OBJ_SIZE) {
4051 /* large objects are always pinned anyway */
4052 p = alloc_large_inner (vtable, size);
4054 p = major_alloc_small_pinned_obj (size);
4056 DEBUG (6, fprintf (gc_debug_file, "Allocated pinned object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
4057 binary_protocol_alloc (p, vtable, size);
4064 * ######################################################################
4065 * ######## Finalization support
4066 * ######################################################################
4070 * this is valid for the nursery: if the object has been forwarded it means it's
4071 * still refrenced from a root. If it is pinned it's still alive as well.
4072 * Return TRUE if @obj is ready to be finalized.
4074 #define object_is_fin_ready(obj) (!object_is_pinned (obj) && !object_is_forwarded (obj))
4077 is_critical_finalizer (FinalizeEntry *entry)
4082 if (!mono_defaults.critical_finalizer_object)
4085 obj = entry->object;
4086 class = ((MonoVTable*)LOAD_VTABLE (obj))->klass;
4088 return mono_class_has_parent (class, mono_defaults.critical_finalizer_object);
4092 queue_finalization_entry (FinalizeEntry *entry) {
4093 if (is_critical_finalizer (entry)) {
4094 entry->next = critical_fin_list;
4095 critical_fin_list = entry;
4097 entry->next = fin_ready_list;
4098 fin_ready_list = entry;
4102 /* LOCKING: requires that the GC lock is held */
4104 rehash_fin_table (FinalizeEntryHashTable *hash_table)
4106 FinalizeEntry **finalizable_hash = hash_table->table;
4107 mword finalizable_hash_size = hash_table->size;
4110 FinalizeEntry **new_hash;
4111 FinalizeEntry *entry, *next;
4112 int new_size = g_spaced_primes_closest (hash_table->num_registered);
4114 new_hash = get_internal_mem (new_size * sizeof (FinalizeEntry*), INTERNAL_MEM_FIN_TABLE);
4115 for (i = 0; i < finalizable_hash_size; ++i) {
4116 for (entry = finalizable_hash [i]; entry; entry = next) {
4117 hash = mono_object_hash (entry->object) % new_size;
4119 entry->next = new_hash [hash];
4120 new_hash [hash] = entry;
4123 free_internal_mem (finalizable_hash, INTERNAL_MEM_FIN_TABLE);
4124 hash_table->table = new_hash;
4125 hash_table->size = new_size;
4128 /* LOCKING: requires that the GC lock is held */
4130 rehash_fin_table_if_necessary (FinalizeEntryHashTable *hash_table)
4132 if (hash_table->num_registered >= hash_table->size * 2)
4133 rehash_fin_table (hash_table);
4136 /* LOCKING: requires that the GC lock is held */
4138 finalize_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, int generation)
4140 FinalizeEntryHashTable *hash_table = get_finalize_entry_hash_table (generation);
4141 FinalizeEntry *entry, *prev;
4143 FinalizeEntry **finalizable_hash = hash_table->table;
4144 mword finalizable_hash_size = hash_table->size;
4148 for (i = 0; i < finalizable_hash_size; ++i) {
4150 for (entry = finalizable_hash [i]; entry;) {
4151 if ((char*)entry->object >= start && (char*)entry->object < end && !major_is_object_live (entry->object)) {
4152 gboolean is_fin_ready = object_is_fin_ready (entry->object);
4153 char *copy = entry->object;
4154 copy_func ((void**)©);
4157 FinalizeEntry *next;
4158 /* remove and put in fin_ready_list */
4160 prev->next = entry->next;
4162 finalizable_hash [i] = entry->next;
4164 num_ready_finalizers++;
4165 hash_table->num_registered--;
4166 queue_finalization_entry (entry);
4167 /* Make it survive */
4168 from = entry->object;
4169 entry->object = copy;
4170 DEBUG (5, fprintf (gc_debug_file, "Queueing object for finalization: %p (%s) (was at %p) (%d/%d)\n", entry->object, safe_name (entry->object), from, num_ready_finalizers, hash_table->num_registered));
4174 char *from = entry->object;
4175 if (hash_table == &minor_finalizable_hash && !ptr_in_nursery (copy)) {
4176 FinalizeEntry *next = entry->next;
4177 unsigned int major_hash;
4178 /* remove from the list */
4180 prev->next = entry->next;
4182 finalizable_hash [i] = entry->next;
4183 hash_table->num_registered--;
4185 entry->object = copy;
4187 /* insert it into the major hash */
4188 rehash_fin_table_if_necessary (&major_finalizable_hash);
4189 major_hash = mono_object_hash ((MonoObject*) copy) %
4190 major_finalizable_hash.size;
4191 entry->next = major_finalizable_hash.table [major_hash];
4192 major_finalizable_hash.table [major_hash] = entry;
4193 major_finalizable_hash.num_registered++;
4195 DEBUG (5, fprintf (gc_debug_file, "Promoting finalization of object %p (%s) (was at %p) to major table\n", copy, safe_name (copy), from));
4200 /* update pointer */
4201 DEBUG (5, fprintf (gc_debug_file, "Updating object for finalization: %p (%s) (was at %p)\n", entry->object, safe_name (entry->object), from));
4202 entry->object = copy;
4207 entry = entry->next;
4212 /* LOCKING: requires that the GC lock is held */
4214 null_link_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, int generation)
4216 DisappearingLinkHashTable *hash = get_dislink_hash_table (generation);
4217 DisappearingLink **disappearing_link_hash = hash->table;
4218 int disappearing_link_hash_size = hash->size;
4219 DisappearingLink *entry, *prev;
4221 if (!hash->num_links)
4223 for (i = 0; i < disappearing_link_hash_size; ++i) {
4225 for (entry = disappearing_link_hash [i]; entry;) {
4226 char *object = DISLINK_OBJECT (entry);
4227 if (object >= start && object < end && !major_is_object_live (object)) {
4228 gboolean track = DISLINK_TRACK (entry);
4229 if (!track && object_is_fin_ready (object)) {
4230 void **p = entry->link;
4231 DisappearingLink *old;
4233 /* remove from list */
4235 prev->next = entry->next;
4237 disappearing_link_hash [i] = entry->next;
4238 DEBUG (5, fprintf (gc_debug_file, "Dislink nullified at %p to GCed object %p\n", p, object));
4240 free_internal_mem (entry, INTERNAL_MEM_DISLINK);
4245 char *copy = object;
4246 copy_func ((void**)©);
4248 /* Update pointer if it's moved. If the object
4249 * has been moved out of the nursery, we need to
4250 * remove the link from the minor hash table to
4253 * FIXME: what if an object is moved earlier?
4256 if (hash == &minor_disappearing_link_hash && !ptr_in_nursery (copy)) {
4257 void **link = entry->link;
4258 DisappearingLink *old;
4259 /* remove from list */
4261 prev->next = entry->next;
4263 disappearing_link_hash [i] = entry->next;
4265 free_internal_mem (entry, INTERNAL_MEM_DISLINK);
4269 add_or_remove_disappearing_link ((MonoObject*)copy, link,
4270 track, GENERATION_OLD);
4272 DEBUG (5, fprintf (gc_debug_file, "Upgraded dislink at %p to major because object %p moved to %p\n", link, object, copy));
4276 /* We set the track resurrection bit to
4277 * FALSE if the object is to be finalized
4278 * so that the object can be collected in
4279 * the next cycle (i.e. after it was
4282 *entry->link = HIDE_POINTER (copy,
4283 object_is_fin_ready (object) ? FALSE : track);
4284 DEBUG (5, fprintf (gc_debug_file, "Updated dislink at %p to %p\n", entry->link, DISLINK_OBJECT (entry)));
4289 entry = entry->next;
4294 /* LOCKING: requires that the GC lock is held */
4296 null_links_for_domain (MonoDomain *domain, int generation)
4298 DisappearingLinkHashTable *hash = get_dislink_hash_table (generation);
4299 DisappearingLink **disappearing_link_hash = hash->table;
4300 int disappearing_link_hash_size = hash->size;
4301 DisappearingLink *entry, *prev;
4303 for (i = 0; i < disappearing_link_hash_size; ++i) {
4305 for (entry = disappearing_link_hash [i]; entry; ) {
4306 char *object = DISLINK_OBJECT (entry);
4307 /* FIXME: actually there should be no object
4308 left in the domain with a non-null vtable
4309 (provided we remove the Thread special
4311 if (object && (!((MonoObject*)object)->vtable || mono_object_domain (object) == domain)) {
4312 DisappearingLink *next = entry->next;
4317 disappearing_link_hash [i] = next;
4319 if (*(entry->link)) {
4320 *(entry->link) = NULL;
4321 g_warning ("Disappearing link %p not freed", entry->link);
4323 free_internal_mem (entry, INTERNAL_MEM_DISLINK);
4330 entry = entry->next;
4335 /* LOCKING: requires that the GC lock is held */
4337 finalizers_for_domain (MonoDomain *domain, MonoObject **out_array, int out_size,
4338 FinalizeEntryHashTable *hash_table)
4340 FinalizeEntry **finalizable_hash = hash_table->table;
4341 mword finalizable_hash_size = hash_table->size;
4342 FinalizeEntry *entry, *prev;
4345 if (no_finalize || !out_size || !out_array)
4348 for (i = 0; i < finalizable_hash_size; ++i) {
4350 for (entry = finalizable_hash [i]; entry;) {
4351 if (mono_object_domain (entry->object) == domain) {
4352 FinalizeEntry *next;
4353 /* remove and put in out_array */
4355 prev->next = entry->next;
4357 finalizable_hash [i] = entry->next;
4359 hash_table->num_registered--;
4360 out_array [count ++] = entry->object;
4361 DEBUG (5, fprintf (gc_debug_file, "Collecting object for finalization: %p (%s) (%d/%d)\n", entry->object, safe_name (entry->object), num_ready_finalizers, hash_table->num_registered));
4363 if (count == out_size)
4368 entry = entry->next;
4375 * mono_gc_finalizers_for_domain:
4376 * @domain: the unloading appdomain
4377 * @out_array: output array
4378 * @out_size: size of output array
4380 * Store inside @out_array up to @out_size objects that belong to the unloading
4381 * appdomain @domain. Returns the number of stored items. Can be called repeteadly
4382 * until it returns 0.
4383 * The items are removed from the finalizer data structure, so the caller is supposed
4385 * @out_array should be on the stack to allow the GC to know the objects are still alive.
4388 mono_gc_finalizers_for_domain (MonoDomain *domain, MonoObject **out_array, int out_size)
4393 result = finalizers_for_domain (domain, out_array, out_size, &minor_finalizable_hash);
4394 if (result < out_size) {
4395 result += finalizers_for_domain (domain, out_array + result, out_size - result,
4396 &major_finalizable_hash);
4404 register_for_finalization (MonoObject *obj, void *user_data, int generation)
4406 FinalizeEntryHashTable *hash_table = get_finalize_entry_hash_table (generation);
4407 FinalizeEntry **finalizable_hash;
4408 mword finalizable_hash_size;
4409 FinalizeEntry *entry, *prev;
4413 g_assert (user_data == NULL || user_data == mono_gc_run_finalize);
4414 hash = mono_object_hash (obj);
4416 rehash_fin_table_if_necessary (hash_table);
4417 finalizable_hash = hash_table->table;
4418 finalizable_hash_size = hash_table->size;
4419 hash %= finalizable_hash_size;
4421 for (entry = finalizable_hash [hash]; entry; entry = entry->next) {
4422 if (entry->object == obj) {
4424 /* remove from the list */
4426 prev->next = entry->next;
4428 finalizable_hash [hash] = entry->next;
4429 hash_table->num_registered--;
4430 DEBUG (5, fprintf (gc_debug_file, "Removed finalizer %p for object: %p (%s) (%d)\n", entry, obj, obj->vtable->klass->name, hash_table->num_registered));
4431 free_internal_mem (entry, INTERNAL_MEM_FINALIZE_ENTRY);
4439 /* request to deregister, but already out of the list */
4443 entry = get_internal_mem (sizeof (FinalizeEntry), INTERNAL_MEM_FINALIZE_ENTRY);
4444 entry->object = obj;
4445 entry->next = finalizable_hash [hash];
4446 finalizable_hash [hash] = entry;
4447 hash_table->num_registered++;
4448 DEBUG (5, fprintf (gc_debug_file, "Added finalizer %p for object: %p (%s) (%d) to %s table\n", entry, obj, obj->vtable->klass->name, hash_table->num_registered, generation_name (generation)));
4453 mono_gc_register_for_finalization (MonoObject *obj, void *user_data)
4455 if (ptr_in_nursery (obj))
4456 register_for_finalization (obj, user_data, GENERATION_NURSERY);
4458 register_for_finalization (obj, user_data, GENERATION_OLD);
4462 rehash_dislink (DisappearingLinkHashTable *hash_table)
4464 DisappearingLink **disappearing_link_hash = hash_table->table;
4465 int disappearing_link_hash_size = hash_table->size;
4468 DisappearingLink **new_hash;
4469 DisappearingLink *entry, *next;
4470 int new_size = g_spaced_primes_closest (hash_table->num_links);
4472 new_hash = get_internal_mem (new_size * sizeof (DisappearingLink*), INTERNAL_MEM_DISLINK_TABLE);
4473 for (i = 0; i < disappearing_link_hash_size; ++i) {
4474 for (entry = disappearing_link_hash [i]; entry; entry = next) {
4475 hash = mono_aligned_addr_hash (entry->link) % new_size;
4477 entry->next = new_hash [hash];
4478 new_hash [hash] = entry;
4481 free_internal_mem (disappearing_link_hash, INTERNAL_MEM_DISLINK_TABLE);
4482 hash_table->table = new_hash;
4483 hash_table->size = new_size;
4486 /* LOCKING: assumes the GC lock is held */
4488 add_or_remove_disappearing_link (MonoObject *obj, void **link, gboolean track, int generation)
4490 DisappearingLinkHashTable *hash_table = get_dislink_hash_table (generation);
4491 DisappearingLink *entry, *prev;
4493 DisappearingLink **disappearing_link_hash = hash_table->table;
4494 int disappearing_link_hash_size = hash_table->size;
4496 if (hash_table->num_links >= disappearing_link_hash_size * 2) {
4497 rehash_dislink (hash_table);
4498 disappearing_link_hash = hash_table->table;
4499 disappearing_link_hash_size = hash_table->size;
4501 /* FIXME: add check that link is not in the heap */
4502 hash = mono_aligned_addr_hash (link) % disappearing_link_hash_size;
4503 entry = disappearing_link_hash [hash];
4505 for (; entry; entry = entry->next) {
4506 /* link already added */
4507 if (link == entry->link) {
4508 /* NULL obj means remove */
4511 prev->next = entry->next;
4513 disappearing_link_hash [hash] = entry->next;
4514 hash_table->num_links--;
4515 DEBUG (5, fprintf (gc_debug_file, "Removed dislink %p (%d) from %s table\n", entry, hash_table->num_links, generation_name (generation)));
4516 free_internal_mem (entry, INTERNAL_MEM_DISLINK);
4519 *link = HIDE_POINTER (obj, track); /* we allow the change of object */
4527 entry = get_internal_mem (sizeof (DisappearingLink), INTERNAL_MEM_DISLINK);
4528 *link = HIDE_POINTER (obj, track);
4530 entry->next = disappearing_link_hash [hash];
4531 disappearing_link_hash [hash] = entry;
4532 hash_table->num_links++;
4533 DEBUG (5, fprintf (gc_debug_file, "Added dislink %p for object: %p (%s) at %p to %s table\n", entry, obj, obj->vtable->klass->name, link, generation_name (generation)));
4536 /* LOCKING: assumes the GC lock is held */
4538 mono_gc_register_disappearing_link (MonoObject *obj, void **link, gboolean track)
4540 add_or_remove_disappearing_link (NULL, link, FALSE, GENERATION_NURSERY);
4541 add_or_remove_disappearing_link (NULL, link, FALSE, GENERATION_OLD);
4543 if (ptr_in_nursery (obj))
4544 add_or_remove_disappearing_link (obj, link, track, GENERATION_NURSERY);
4546 add_or_remove_disappearing_link (obj, link, track, GENERATION_OLD);
4551 mono_gc_invoke_finalizers (void)
4553 FinalizeEntry *entry = NULL;
4554 gboolean entry_is_critical = FALSE;
4557 /* FIXME: batch to reduce lock contention */
4558 while (fin_ready_list || critical_fin_list) {
4562 FinalizeEntry **list = entry_is_critical ? &critical_fin_list : &fin_ready_list;
4564 /* We have finalized entry in the last
4565 interation, now we need to remove it from
4568 *list = entry->next;
4570 FinalizeEntry *e = *list;
4571 while (e->next != entry)
4573 e->next = entry->next;
4575 free_internal_mem (entry, INTERNAL_MEM_FINALIZE_ENTRY);
4579 /* Now look for the first non-null entry. */
4580 for (entry = fin_ready_list; entry && !entry->object; entry = entry->next)
4583 entry_is_critical = FALSE;
4585 entry_is_critical = TRUE;
4586 for (entry = critical_fin_list; entry && !entry->object; entry = entry->next)
4591 g_assert (entry->object);
4592 num_ready_finalizers--;
4593 obj = entry->object;
4594 entry->object = NULL;
4595 DEBUG (7, fprintf (gc_debug_file, "Finalizing object %p (%s)\n", obj, safe_name (obj)));
4603 g_assert (entry->object == NULL);
4605 /* the object is on the stack so it is pinned */
4606 /*g_print ("Calling finalizer for object: %p (%s)\n", entry->object, safe_name (entry->object));*/
4607 mono_gc_run_finalize (obj, NULL);
4614 mono_gc_pending_finalizers (void)
4616 return fin_ready_list || critical_fin_list;
4619 /* Negative value to remove */
4621 mono_gc_add_memory_pressure (gint64 value)
4623 /* FIXME: Use interlocked functions */
4625 memory_pressure += value;
4630 * ######################################################################
4631 * ######## registered roots support
4632 * ######################################################################
4636 rehash_roots (gboolean pinned)
4640 RootRecord **new_hash;
4641 RootRecord *entry, *next;
4644 new_size = g_spaced_primes_closest (num_roots_entries [pinned]);
4645 new_hash = get_internal_mem (new_size * sizeof (RootRecord*), INTERNAL_MEM_ROOTS_TABLE);
4646 for (i = 0; i < roots_hash_size [pinned]; ++i) {
4647 for (entry = roots_hash [pinned][i]; entry; entry = next) {
4648 hash = mono_aligned_addr_hash (entry->start_root) % new_size;
4650 entry->next = new_hash [hash];
4651 new_hash [hash] = entry;
4654 free_internal_mem (roots_hash [pinned], INTERNAL_MEM_ROOTS_TABLE);
4655 roots_hash [pinned] = new_hash;
4656 roots_hash_size [pinned] = new_size;
4660 find_root (int root_type, char *start, guint32 addr_hash)
4662 RootRecord *new_root;
4664 guint32 hash = addr_hash % roots_hash_size [root_type];
4665 for (new_root = roots_hash [root_type][hash]; new_root; new_root = new_root->next) {
4666 /* we allow changing the size and the descriptor (for thread statics etc) */
4667 if (new_root->start_root == start) {
4676 * We do not coalesce roots.
4679 mono_gc_register_root_inner (char *start, size_t size, void *descr, int root_type)
4681 RootRecord *new_root;
4682 unsigned int hash, addr_hash = mono_aligned_addr_hash (start);
4685 for (i = 0; i < ROOT_TYPE_NUM; ++i) {
4686 if (num_roots_entries [i] >= roots_hash_size [i] * 2)
4689 for (i = 0; i < ROOT_TYPE_NUM; ++i) {
4690 new_root = find_root (i, start, addr_hash);
4691 /* we allow changing the size and the descriptor (for thread statics etc) */
4693 size_t old_size = new_root->end_root - new_root->start_root;
4694 new_root->end_root = new_root->start_root + size;
4695 g_assert (((new_root->root_desc != 0) && (descr != NULL)) ||
4696 ((new_root->root_desc == 0) && (descr == NULL)));
4697 new_root->root_desc = (mword)descr;
4699 roots_size -= old_size;
4704 new_root = get_internal_mem (sizeof (RootRecord), INTERNAL_MEM_ROOT_RECORD);
4706 new_root->start_root = start;
4707 new_root->end_root = new_root->start_root + size;
4708 new_root->root_desc = (mword)descr;
4710 hash = addr_hash % roots_hash_size [root_type];
4711 num_roots_entries [root_type]++;
4712 new_root->next = roots_hash [root_type] [hash];
4713 roots_hash [root_type][hash] = new_root;
4714 DEBUG (3, fprintf (gc_debug_file, "Added root %p for range: %p-%p, descr: %p (%d/%d bytes)\n", new_root, new_root->start_root, new_root->end_root, descr, (int)size, (int)roots_size));
4724 mono_gc_register_root (char *start, size_t size, void *descr)
4726 return mono_gc_register_root_inner (start, size, descr, descr ? ROOT_TYPE_NORMAL : ROOT_TYPE_PINNED);
4730 mono_gc_register_root_wbarrier (char *start, size_t size, void *descr)
4732 return mono_gc_register_root_inner (start, size, descr, ROOT_TYPE_WBARRIER);
4736 mono_gc_deregister_root (char* addr)
4738 RootRecord *tmp, *prev;
4739 unsigned int hash, addr_hash = mono_aligned_addr_hash (addr);
4743 for (root_type = 0; root_type < ROOT_TYPE_NUM; ++root_type) {
4744 hash = addr_hash % roots_hash_size [root_type];
4745 tmp = roots_hash [root_type][hash];
4748 if (tmp->start_root == (char*)addr) {
4750 prev->next = tmp->next;
4752 roots_hash [root_type][hash] = tmp->next;
4753 roots_size -= (tmp->end_root - tmp->start_root);
4754 num_roots_entries [root_type]--;
4755 DEBUG (3, fprintf (gc_debug_file, "Removed root %p for range: %p-%p\n", tmp, tmp->start_root, tmp->end_root));
4756 free_internal_mem (tmp, INTERNAL_MEM_ROOT_RECORD);
4767 * ######################################################################
4768 * ######## Thread handling (stop/start code)
4769 * ######################################################################
4772 /* FIXME: handle large/small config */
4773 #define THREAD_HASH_SIZE 11
4774 #define HASH_PTHREAD_T(id) (((unsigned int)(id) >> 4) * 2654435761u)
4776 static SgenThreadInfo* thread_table [THREAD_HASH_SIZE];
4778 #if USE_SIGNAL_BASED_START_STOP_WORLD
4780 static MonoSemType suspend_ack_semaphore;
4781 static MonoSemType *suspend_ack_semaphore_ptr;
4782 static unsigned int global_stop_count = 0;
4784 static int suspend_signal_num = SIGXFSZ;
4786 static int suspend_signal_num = SIGPWR;
4788 static int restart_signal_num = SIGXCPU;
4789 static sigset_t suspend_signal_mask;
4790 static mword cur_thread_regs [ARCH_NUM_REGS] = {0};
4792 /* LOCKING: assumes the GC lock is held */
4793 static SgenThreadInfo*
4794 thread_info_lookup (ARCH_THREAD_TYPE id)
4796 unsigned int hash = HASH_PTHREAD_T (id) % THREAD_HASH_SIZE;
4797 SgenThreadInfo *info;
4799 info = thread_table [hash];
4800 while (info && !ARCH_THREAD_EQUALS (info->id, id)) {
4807 update_current_thread_stack (void *start)
4809 void *ptr = cur_thread_regs;
4810 SgenThreadInfo *info = thread_info_lookup (ARCH_GET_THREAD ());
4812 info->stack_start = align_pointer (&ptr);
4813 g_assert (info->stack_start >= info->stack_start_limit && info->stack_start < info->stack_end);
4814 ARCH_STORE_REGS (ptr);
4815 info->stopped_regs = ptr;
4816 if (gc_callbacks.thread_suspend_func)
4817 gc_callbacks.thread_suspend_func (info->runtime_data, NULL);
4821 signal_desc (int signum)
4823 if (signum == suspend_signal_num)
4825 if (signum == restart_signal_num)
4831 * Define this and use the "xdomain-checks" MONO_GC_DEBUG option to
4832 * have cross-domain checks in the write barrier.
4834 //#define XDOMAIN_CHECKS_IN_WBARRIER
4836 #ifndef BINARY_PROTOCOL
4837 #ifndef HEAVY_STATISTICS
4838 #define MANAGED_ALLOCATION
4839 #ifndef XDOMAIN_CHECKS_IN_WBARRIER
4840 #define MANAGED_WBARRIER
4846 is_ip_in_managed_allocator (MonoDomain *domain, gpointer ip);
4849 wait_for_suspend_ack (int count)
4853 for (i = 0; i < count; ++i) {
4854 while ((result = MONO_SEM_WAIT (suspend_ack_semaphore_ptr)) != 0) {
4855 if (errno != EINTR) {
4856 g_error ("sem_wait ()");
4862 /* LOCKING: assumes the GC lock is held */
4864 thread_handshake (int signum)
4866 int count, i, result;
4867 SgenThreadInfo *info;
4868 pthread_t me = pthread_self ();
4871 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
4872 for (info = thread_table [i]; info; info = info->next) {
4873 DEBUG (4, fprintf (gc_debug_file, "considering thread %p for signal %d (%s)\n", info, signum, signal_desc (signum)));
4874 if (ARCH_THREAD_EQUALS (info->id, me)) {
4875 DEBUG (4, fprintf (gc_debug_file, "Skip (equal): %p, %p\n", (void*)me, (void*)info->id));
4878 /*if (signum == suspend_signal_num && info->stop_count == global_stop_count)
4880 result = pthread_kill (info->id, signum);
4882 DEBUG (4, fprintf (gc_debug_file, "thread %p signal sent\n", info));
4885 DEBUG (4, fprintf (gc_debug_file, "thread %p signal failed: %d (%s)\n", (void*)info->id, result, strerror (result)));
4891 wait_for_suspend_ack (count);
4897 restart_threads_until_none_in_managed_allocator (void)
4899 SgenThreadInfo *info;
4900 int i, result, num_threads_died = 0;
4901 int sleep_duration = -1;
4904 int restart_count = 0, restarted_count = 0;
4905 /* restart all threads that stopped in the
4907 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
4908 for (info = thread_table [i]; info; info = info->next) {
4911 if (!info->stack_start || info->in_critical_region ||
4912 is_ip_in_managed_allocator (info->stopped_domain, info->stopped_ip)) {
4913 binary_protocol_thread_restart ((gpointer)info->id);
4914 result = pthread_kill (info->id, restart_signal_num);
4921 /* we set the stopped_ip to
4922 NULL for threads which
4923 we're not restarting so
4924 that we can easily identify
4926 info->stopped_ip = NULL;
4927 info->stopped_domain = NULL;
4931 /* if no threads were restarted, we're done */
4932 if (restart_count == 0)
4935 /* wait for the threads to signal their restart */
4936 wait_for_suspend_ack (restart_count);
4938 if (sleep_duration < 0) {
4942 g_usleep (sleep_duration);
4943 sleep_duration += 10;
4946 /* stop them again */
4947 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
4948 for (info = thread_table [i]; info; info = info->next) {
4949 if (info->skip || info->stopped_ip == NULL)
4951 result = pthread_kill (info->id, suspend_signal_num);
4959 /* some threads might have died */
4960 num_threads_died += restart_count - restarted_count;
4961 /* wait for the threads to signal their suspension
4963 wait_for_suspend_ack (restart_count);
4966 return num_threads_died;
4969 /* LOCKING: assumes the GC lock is held (by the stopping thread) */
4971 suspend_handler (int sig, siginfo_t *siginfo, void *context)
4973 SgenThreadInfo *info;
4976 int old_errno = errno;
4977 gpointer regs [ARCH_NUM_REGS];
4978 gpointer stack_start;
4980 id = pthread_self ();
4981 info = thread_info_lookup (id);
4982 info->stopped_domain = mono_domain_get ();
4983 info->stopped_ip = (gpointer) ARCH_SIGCTX_IP (context);
4984 stop_count = global_stop_count;
4985 /* duplicate signal */
4986 if (0 && info->stop_count == stop_count) {
4990 #ifdef HAVE_KW_THREAD
4991 /* update the remset info in the thread data structure */
4992 info->remset = remembered_set;
4994 stack_start = (char*) ARCH_SIGCTX_SP (context) - REDZONE_SIZE;
4995 /* If stack_start is not within the limits, then don't set it
4996 in info and we will be restarted. */
4997 if (stack_start >= info->stack_start_limit && info->stack_start <= info->stack_end) {
4998 info->stack_start = stack_start;
5000 ARCH_COPY_SIGCTX_REGS (regs, context);
5001 info->stopped_regs = regs;
5003 g_assert (!info->stack_start);
5006 /* Notify the JIT */
5007 if (gc_callbacks.thread_suspend_func)
5008 gc_callbacks.thread_suspend_func (info->runtime_data, context);
5010 DEBUG (4, fprintf (gc_debug_file, "Posting suspend_ack_semaphore for suspend from %p %p\n", info, (gpointer)ARCH_GET_THREAD ()));
5011 /* notify the waiting thread */
5012 MONO_SEM_POST (suspend_ack_semaphore_ptr);
5013 info->stop_count = stop_count;
5015 /* wait until we receive the restart signal */
5018 sigsuspend (&suspend_signal_mask);
5019 } while (info->signal != restart_signal_num);
5021 DEBUG (4, fprintf (gc_debug_file, "Posting suspend_ack_semaphore for resume from %p %p\n", info, (gpointer)ARCH_GET_THREAD ()));
5022 /* notify the waiting thread */
5023 MONO_SEM_POST (suspend_ack_semaphore_ptr);
5029 restart_handler (int sig)
5031 SgenThreadInfo *info;
5032 int old_errno = errno;
5034 info = thread_info_lookup (pthread_self ());
5035 info->signal = restart_signal_num;
5036 DEBUG (4, fprintf (gc_debug_file, "Restart handler in %p %p\n", info, (gpointer)ARCH_GET_THREAD ()));
5042 acquire_gc_locks (void)
5048 release_gc_locks (void)
5050 UNLOCK_INTERRUPTION;
5053 static TV_DECLARE (stop_world_time);
5054 static unsigned long max_pause_usec = 0;
5056 /* LOCKING: assumes the GC lock is held */
5062 acquire_gc_locks ();
5064 update_current_thread_stack (&count);
5066 global_stop_count++;
5067 DEBUG (3, fprintf (gc_debug_file, "stopping world n %d from %p %p\n", global_stop_count, thread_info_lookup (ARCH_GET_THREAD ()), (gpointer)ARCH_GET_THREAD ()));
5068 TV_GETTIME (stop_world_time);
5069 count = thread_handshake (suspend_signal_num);
5070 count -= restart_threads_until_none_in_managed_allocator ();
5071 g_assert (count >= 0);
5072 DEBUG (3, fprintf (gc_debug_file, "world stopped %d thread(s)\n", count));
5076 /* LOCKING: assumes the GC lock is held */
5078 restart_world (void)
5081 SgenThreadInfo *info;
5082 TV_DECLARE (end_sw);
5085 /* notify the profiler of the leftovers */
5086 if (G_UNLIKELY (mono_profiler_events & MONO_PROFILE_GC_MOVES)) {
5087 if (moved_objects_idx) {
5088 mono_profiler_gc_moves (moved_objects, moved_objects_idx);
5089 moved_objects_idx = 0;
5092 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5093 for (info = thread_table [i]; info; info = info->next) {
5094 info->stack_start = NULL;
5095 info->stopped_regs = NULL;
5099 release_gc_locks ();
5101 count = thread_handshake (restart_signal_num);
5102 TV_GETTIME (end_sw);
5103 usec = TV_ELAPSED (stop_world_time, end_sw);
5104 max_pause_usec = MAX (usec, max_pause_usec);
5105 DEBUG (2, fprintf (gc_debug_file, "restarted %d thread(s) (pause time: %d usec, max: %d)\n", count, (int)usec, (int)max_pause_usec));
5109 #endif /* USE_SIGNAL_BASED_START_STOP_WORLD */
5112 mono_gc_set_gc_callbacks (MonoGCCallbacks *callbacks)
5114 gc_callbacks = *callbacks;
5117 /* Variables holding start/end nursery so it won't have to be passed at every call */
5118 static void *scan_area_arg_start, *scan_area_arg_end;
5121 mono_gc_conservatively_scan_area (void *start, void *end)
5123 conservatively_pin_objects_from (start, end, scan_area_arg_start, scan_area_arg_end, PIN_TYPE_STACK);
5127 mono_gc_scan_object (void *obj)
5129 if (current_collection_generation == GENERATION_NURSERY)
5132 major_copy_or_mark_object (&obj);
5137 * Mark from thread stacks and registers.
5140 scan_thread_data (void *start_nursery, void *end_nursery, gboolean precise)
5143 SgenThreadInfo *info;
5145 scan_area_arg_start = start_nursery;
5146 scan_area_arg_end = end_nursery;
5148 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5149 for (info = thread_table [i]; info; info = info->next) {
5151 DEBUG (3, fprintf (gc_debug_file, "Skipping dead thread %p, range: %p-%p, size: %zd\n", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start));
5154 DEBUG (3, fprintf (gc_debug_file, "Scanning thread %p, range: %p-%p, size: %zd, pinned=%d\n", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start, next_pin_slot));
5155 if (gc_callbacks.thread_mark_func && !conservative_stack_mark)
5156 gc_callbacks.thread_mark_func (info->runtime_data, info->stack_start, info->stack_end, precise);
5158 conservatively_pin_objects_from (info->stack_start, info->stack_end, start_nursery, end_nursery, PIN_TYPE_STACK);
5161 conservatively_pin_objects_from (info->stopped_regs, info->stopped_regs + ARCH_NUM_REGS,
5162 start_nursery, end_nursery, PIN_TYPE_STACK);
5168 find_pinning_ref_from_thread (char *obj, size_t size)
5171 SgenThreadInfo *info;
5172 char *endobj = obj + size;
5174 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5175 for (info = thread_table [i]; info; info = info->next) {
5176 char **start = (char**)info->stack_start;
5179 while (start < (char**)info->stack_end) {
5180 if (*start >= obj && *start < endobj) {
5181 DEBUG (0, fprintf (gc_debug_file, "Object %p referenced in thread %p (id %p) at %p, stack: %p-%p\n", obj, info, (gpointer)info->id, start, info->stack_start, info->stack_end));
5186 /* FIXME: check info->stopped_regs */
5192 ptr_on_stack (void *ptr)
5194 gpointer stack_start = &stack_start;
5195 SgenThreadInfo *info = thread_info_lookup (ARCH_GET_THREAD ());
5197 if (ptr >= stack_start && ptr < (gpointer)info->stack_end)
5203 handle_remset (mword *p, void *start_nursery, void *end_nursery, gboolean global)
5210 HEAVY_STAT (++stat_global_remsets_processed);
5212 /* FIXME: exclude stack locations */
5213 switch ((*p) & REMSET_TYPE_MASK) {
5214 case REMSET_LOCATION:
5216 //__builtin_prefetch (ptr);
5217 if (((void*)ptr < start_nursery || (void*)ptr >= end_nursery)) {
5218 gpointer old = *ptr;
5220 DEBUG (9, fprintf (gc_debug_file, "Overwrote remset at %p with %p\n", ptr, *ptr));
5222 binary_protocol_ptr_update (ptr, old, *ptr, (gpointer)LOAD_VTABLE (*ptr), safe_object_get_size (*ptr));
5223 if (!global && *ptr >= start_nursery && *ptr < end_nursery) {
5225 * If the object is pinned, each reference to it from nonpinned objects
5226 * becomes part of the global remset, which can grow very large.
5228 DEBUG (9, fprintf (gc_debug_file, "Add to global remset because of pinning %p (%p %s)\n", ptr, *ptr, safe_name (*ptr)));
5229 add_to_global_remset (ptr, FALSE);
5232 DEBUG (9, fprintf (gc_debug_file, "Skipping remset at %p holding %p\n", ptr, *ptr));
5236 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
5237 if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery))
5240 while (count-- > 0) {
5242 DEBUG (9, fprintf (gc_debug_file, "Overwrote remset at %p with %p (count: %d)\n", ptr, *ptr, (int)count));
5243 if (!global && *ptr >= start_nursery && *ptr < end_nursery)
5244 add_to_global_remset (ptr, FALSE);
5249 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
5250 if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery))
5252 scan_object ((char*)ptr);
5254 case REMSET_OTHER: {
5255 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
5259 if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery))
5264 ptr = (void**) scan_vtype ((char*)ptr, desc, start_nursery, end_nursery);
5266 case REMSET_ROOT_LOCATION:
5267 /* Same as REMSET_LOCATION, but the address is not required to be in the heap */
5269 DEBUG (9, fprintf (gc_debug_file, "Overwrote root location remset at %p with %p\n", ptr, *ptr));
5270 if (!global && *ptr >= start_nursery && *ptr < end_nursery) {
5272 * If the object is pinned, each reference to it from nonpinned objects
5273 * becomes part of the global remset, which can grow very large.
5275 DEBUG (9, fprintf (gc_debug_file, "Add to global remset because of pinning %p (%p %s)\n", ptr, *ptr, safe_name (*ptr)));
5276 add_to_global_remset (ptr, TRUE);
5280 g_assert_not_reached ();
5285 g_assert_not_reached ();
5290 #ifdef HEAVY_STATISTICS
5292 collect_store_remsets (RememberedSet *remset, mword *bumper)
5294 mword *p = remset->data;
5299 while (p < remset->store_next) {
5300 switch ((*p) & REMSET_TYPE_MASK) {
5301 case REMSET_LOCATION:
5304 ++stat_saved_remsets_1;
5306 if (*p == last1 || *p == last2) {
5307 ++stat_saved_remsets_2;
5325 case REMSET_ROOT_LOCATION:
5329 g_assert_not_reached ();
5333 g_assert_not_reached ();
5343 RememberedSet *remset;
5345 SgenThreadInfo *info;
5347 mword *addresses, *bumper, *p, *r;
5349 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5350 for (info = thread_table [i]; info; info = info->next) {
5351 for (remset = info->remset; remset; remset = remset->next)
5352 size += remset->store_next - remset->data;
5355 for (remset = freed_thread_remsets; remset; remset = remset->next)
5356 size += remset->store_next - remset->data;
5357 for (remset = global_remset; remset; remset = remset->next)
5358 size += remset->store_next - remset->data;
5360 bumper = addresses = get_internal_mem (sizeof (mword) * size, INTERNAL_MEM_STATISTICS);
5362 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5363 for (info = thread_table [i]; info; info = info->next) {
5364 for (remset = info->remset; remset; remset = remset->next)
5365 bumper = collect_store_remsets (remset, bumper);
5368 for (remset = global_remset; remset; remset = remset->next)
5369 bumper = collect_store_remsets (remset, bumper);
5370 for (remset = freed_thread_remsets; remset; remset = remset->next)
5371 bumper = collect_store_remsets (remset, bumper);
5373 g_assert (bumper <= addresses + size);
5375 stat_store_remsets += bumper - addresses;
5377 sort_addresses ((void**)addresses, bumper - addresses);
5380 while (r < bumper) {
5386 stat_store_remsets_unique += p - addresses;
5388 free_internal_mem (addresses, INTERNAL_MEM_STATISTICS);
5393 clear_thread_store_remset_buffer (SgenThreadInfo *info)
5395 *info->store_remset_buffer_index_addr = 0;
5396 memset (*info->store_remset_buffer_addr, 0, sizeof (gpointer) * STORE_REMSET_BUFFER_SIZE);
5400 scan_from_remsets (void *start_nursery, void *end_nursery)
5403 SgenThreadInfo *info;
5404 RememberedSet *remset;
5405 GenericStoreRememberedSet *store_remset;
5406 mword *p, *next_p, *store_pos;
5408 #ifdef HEAVY_STATISTICS
5412 /* the global one */
5413 for (remset = global_remset; remset; remset = remset->next) {
5414 DEBUG (4, fprintf (gc_debug_file, "Scanning global remset range: %p-%p, size: %zd\n", remset->data, remset->store_next, remset->store_next - remset->data));
5415 store_pos = remset->data;
5416 for (p = remset->data; p < remset->store_next; p = next_p) {
5419 next_p = handle_remset (p, start_nursery, end_nursery, TRUE);
5422 * Clear global remsets of locations which no longer point to the
5423 * nursery. Otherwise, they could grow indefinitely between major
5426 ptr = (p [0] & ~REMSET_TYPE_MASK);
5427 if ((p [0] & REMSET_TYPE_MASK) == REMSET_LOCATION) {
5428 if (ptr_in_nursery (*(void**)ptr))
5429 *store_pos ++ = p [0];
5431 g_assert ((p [0] & REMSET_TYPE_MASK) == REMSET_OTHER);
5432 g_assert (p [1] == REMSET_ROOT_LOCATION);
5433 if (ptr_in_nursery (*(void**)ptr)) {
5434 *store_pos ++ = p [0];
5435 *store_pos ++ = p [1];
5440 /* Truncate the remset */
5441 remset->store_next = store_pos;
5444 /* the generic store ones */
5445 store_remset = generic_store_remsets;
5446 while (store_remset) {
5447 GenericStoreRememberedSet *next = store_remset->next;
5449 for (i = 0; i < STORE_REMSET_BUFFER_SIZE - 1; ++i) {
5450 gpointer addr = store_remset->data [i];
5452 handle_remset ((mword*)&addr, start_nursery, end_nursery, FALSE);
5455 free_internal_mem (store_remset, INTERNAL_MEM_STORE_REMSET);
5457 store_remset = next;
5459 generic_store_remsets = NULL;
5461 /* the per-thread ones */
5462 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5463 for (info = thread_table [i]; info; info = info->next) {
5464 RememberedSet *next;
5466 for (remset = info->remset; remset; remset = next) {
5467 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for thread %p, range: %p-%p, size: %zd\n", info, remset->data, remset->store_next, remset->store_next - remset->data));
5468 for (p = remset->data; p < remset->store_next;) {
5469 p = handle_remset (p, start_nursery, end_nursery, FALSE);
5471 remset->store_next = remset->data;
5472 next = remset->next;
5473 remset->next = NULL;
5474 if (remset != info->remset) {
5475 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
5476 free_internal_mem (remset, INTERNAL_MEM_REMSET);
5479 for (j = 0; j < *info->store_remset_buffer_index_addr; ++j)
5480 handle_remset ((mword*)*info->store_remset_buffer_addr + j + 1, start_nursery, end_nursery, FALSE);
5481 clear_thread_store_remset_buffer (info);
5485 /* the freed thread ones */
5486 while (freed_thread_remsets) {
5487 RememberedSet *next;
5488 remset = freed_thread_remsets;
5489 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for freed thread, range: %p-%p, size: %zd\n", remset->data, remset->store_next, remset->store_next - remset->data));
5490 for (p = remset->data; p < remset->store_next;) {
5491 p = handle_remset (p, start_nursery, end_nursery, FALSE);
5493 next = remset->next;
5494 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
5495 free_internal_mem (remset, INTERNAL_MEM_REMSET);
5496 freed_thread_remsets = next;
5501 * Clear the info in the remembered sets: we're doing a major collection, so
5502 * the per-thread ones are not needed and the global ones will be reconstructed
5506 clear_remsets (void)
5509 SgenThreadInfo *info;
5510 RememberedSet *remset, *next;
5512 /* the global list */
5513 for (remset = global_remset; remset; remset = next) {
5514 remset->store_next = remset->data;
5515 next = remset->next;
5516 remset->next = NULL;
5517 if (remset != global_remset) {
5518 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
5519 free_internal_mem (remset, INTERNAL_MEM_REMSET);
5522 /* the generic store ones */
5523 while (generic_store_remsets) {
5524 GenericStoreRememberedSet *gs_next = generic_store_remsets->next;
5525 free_internal_mem (generic_store_remsets, INTERNAL_MEM_STORE_REMSET);
5526 generic_store_remsets = gs_next;
5528 /* the per-thread ones */
5529 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5530 for (info = thread_table [i]; info; info = info->next) {
5531 for (remset = info->remset; remset; remset = next) {
5532 remset->store_next = remset->data;
5533 next = remset->next;
5534 remset->next = NULL;
5535 if (remset != info->remset) {
5536 DEBUG (3, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
5537 free_internal_mem (remset, INTERNAL_MEM_REMSET);
5540 clear_thread_store_remset_buffer (info);
5544 /* the freed thread ones */
5545 while (freed_thread_remsets) {
5546 next = freed_thread_remsets->next;
5547 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", freed_thread_remsets->data));
5548 free_internal_mem (freed_thread_remsets, INTERNAL_MEM_REMSET);
5549 freed_thread_remsets = next;
5554 * Clear the thread local TLAB variables for all threads.
5559 SgenThreadInfo *info;
5562 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5563 for (info = thread_table [i]; info; info = info->next) {
5564 /* A new TLAB will be allocated when the thread does its first allocation */
5565 *info->tlab_start_addr = NULL;
5566 *info->tlab_next_addr = NULL;
5567 *info->tlab_temp_end_addr = NULL;
5568 *info->tlab_real_end_addr = NULL;
5573 /* LOCKING: assumes the GC lock is held */
5574 static SgenThreadInfo*
5575 gc_register_current_thread (void *addr)
5578 SgenThreadInfo* info = malloc (sizeof (SgenThreadInfo));
5579 #ifndef HAVE_KW_THREAD
5580 SgenThreadInfo *__thread_info__ = info;
5586 memset (info, 0, sizeof (SgenThreadInfo));
5587 #ifndef HAVE_KW_THREAD
5588 info->tlab_start = info->tlab_next = info->tlab_temp_end = info->tlab_real_end = NULL;
5590 g_assert (!pthread_getspecific (thread_info_key));
5591 pthread_setspecific (thread_info_key, info);
5596 info->id = ARCH_GET_THREAD ();
5597 info->stop_count = -1;
5600 info->stack_start = NULL;
5601 info->tlab_start_addr = &TLAB_START;
5602 info->tlab_next_addr = &TLAB_NEXT;
5603 info->tlab_temp_end_addr = &TLAB_TEMP_END;
5604 info->tlab_real_end_addr = &TLAB_REAL_END;
5605 info->store_remset_buffer_addr = &STORE_REMSET_BUFFER;
5606 info->store_remset_buffer_index_addr = &STORE_REMSET_BUFFER_INDEX;
5607 info->stopped_ip = NULL;
5608 info->stopped_domain = NULL;
5609 info->stopped_regs = NULL;
5611 binary_protocol_thread_register ((gpointer)info->id);
5613 #ifdef HAVE_KW_THREAD
5614 tlab_next_addr = &tlab_next;
5615 store_remset_buffer_index_addr = &store_remset_buffer_index;
5618 /* try to get it with attributes first */
5619 #if defined(HAVE_PTHREAD_GETATTR_NP) && defined(HAVE_PTHREAD_ATTR_GETSTACK)
5623 pthread_attr_t attr;
5624 pthread_getattr_np (pthread_self (), &attr);
5625 pthread_attr_getstack (&attr, &sstart, &size);
5626 info->stack_start_limit = sstart;
5627 info->stack_end = (char*)sstart + size;
5628 pthread_attr_destroy (&attr);
5630 #elif defined(HAVE_PTHREAD_GET_STACKSIZE_NP) && defined(HAVE_PTHREAD_GET_STACKADDR_NP)
5631 info->stack_end = (char*)pthread_get_stackaddr_np (pthread_self ());
5632 info->stack_start_limit = (char*)info->stack_end - pthread_get_stacksize_np (pthread_self ());
5635 /* FIXME: we assume the stack grows down */
5636 gsize stack_bottom = (gsize)addr;
5637 stack_bottom += 4095;
5638 stack_bottom &= ~4095;
5639 info->stack_end = (char*)stack_bottom;
5643 #ifdef HAVE_KW_THREAD
5644 stack_end = info->stack_end;
5647 /* hash into the table */
5648 hash = HASH_PTHREAD_T (info->id) % THREAD_HASH_SIZE;
5649 info->next = thread_table [hash];
5650 thread_table [hash] = info;
5652 info->remset = alloc_remset (DEFAULT_REMSET_SIZE, info);
5653 pthread_setspecific (remembered_set_key, info->remset);
5654 #ifdef HAVE_KW_THREAD
5655 remembered_set = info->remset;
5658 STORE_REMSET_BUFFER = get_internal_mem (sizeof (gpointer) * STORE_REMSET_BUFFER_SIZE, INTERNAL_MEM_STORE_REMSET);
5659 STORE_REMSET_BUFFER_INDEX = 0;
5661 DEBUG (3, fprintf (gc_debug_file, "registered thread %p (%p) (hash: %d)\n", info, (gpointer)info->id, hash));
5663 if (gc_callbacks.thread_attach_func)
5664 info->runtime_data = gc_callbacks.thread_attach_func ();
5670 add_generic_store_remset_from_buffer (gpointer *buffer)
5672 GenericStoreRememberedSet *remset = get_internal_mem (sizeof (GenericStoreRememberedSet), INTERNAL_MEM_STORE_REMSET);
5673 memcpy (remset->data, buffer + 1, sizeof (gpointer) * (STORE_REMSET_BUFFER_SIZE - 1));
5674 remset->next = generic_store_remsets;
5675 generic_store_remsets = remset;
5679 unregister_current_thread (void)
5682 SgenThreadInfo *prev = NULL;
5684 RememberedSet *rset;
5685 ARCH_THREAD_TYPE id = ARCH_GET_THREAD ();
5687 binary_protocol_thread_unregister ((gpointer)id);
5689 hash = HASH_PTHREAD_T (id) % THREAD_HASH_SIZE;
5690 p = thread_table [hash];
5692 DEBUG (3, fprintf (gc_debug_file, "unregister thread %p (%p)\n", p, (gpointer)p->id));
5693 while (!ARCH_THREAD_EQUALS (p->id, id)) {
5698 thread_table [hash] = p->next;
5700 prev->next = p->next;
5703 if (freed_thread_remsets) {
5704 for (rset = p->remset; rset->next; rset = rset->next)
5706 rset->next = freed_thread_remsets;
5707 freed_thread_remsets = p->remset;
5709 freed_thread_remsets = p->remset;
5712 if (*p->store_remset_buffer_index_addr)
5713 add_generic_store_remset_from_buffer (*p->store_remset_buffer_addr);
5714 free_internal_mem (*p->store_remset_buffer_addr, INTERNAL_MEM_STORE_REMSET);
5719 unregister_thread (void *k)
5721 g_assert (!mono_domain_get ());
5723 unregister_current_thread ();
5728 mono_gc_register_thread (void *baseptr)
5730 SgenThreadInfo *info;
5734 info = thread_info_lookup (ARCH_GET_THREAD ());
5736 info = gc_register_current_thread (baseptr);
5738 return info != NULL;
5741 #if USE_PTHREAD_INTERCEPT
5743 #undef pthread_create
5745 #undef pthread_detach
5748 void *(*start_routine) (void *);
5751 MonoSemType registered;
5752 } SgenThreadStartInfo;
5755 gc_start_thread (void *arg)
5757 SgenThreadStartInfo *start_info = arg;
5758 SgenThreadInfo* info;
5759 void *t_arg = start_info->arg;
5760 void *(*start_func) (void*) = start_info->start_routine;
5765 info = gc_register_current_thread (&result);
5767 post_result = MONO_SEM_POST (&(start_info->registered));
5768 g_assert (!post_result);
5769 result = start_func (t_arg);
5770 g_assert (!mono_domain_get ());
5772 * this is done by the pthread key dtor
5774 unregister_current_thread ();
5782 mono_gc_pthread_create (pthread_t *new_thread, const pthread_attr_t *attr, void *(*start_routine)(void *), void *arg)
5784 SgenThreadStartInfo *start_info;
5787 start_info = malloc (sizeof (SgenThreadStartInfo));
5790 result = MONO_SEM_INIT (&(start_info->registered), 0);
5792 start_info->arg = arg;
5793 start_info->start_routine = start_routine;
5795 result = pthread_create (new_thread, attr, gc_start_thread, start_info);
5797 while (MONO_SEM_WAIT (&(start_info->registered)) != 0) {
5798 /*if (EINTR != errno) ABORT("sem_wait failed"); */
5801 MONO_SEM_DESTROY (&(start_info->registered));
5807 mono_gc_pthread_join (pthread_t thread, void **retval)
5809 return pthread_join (thread, retval);
5813 mono_gc_pthread_detach (pthread_t thread)
5815 return pthread_detach (thread);
5818 #endif /* USE_PTHREAD_INTERCEPT */
5821 * ######################################################################
5822 * ######## Write barriers
5823 * ######################################################################
5826 static RememberedSet*
5827 alloc_remset (int size, gpointer id) {
5828 RememberedSet* res = get_internal_mem (sizeof (RememberedSet) + (size * sizeof (gpointer)), INTERNAL_MEM_REMSET);
5829 res->store_next = res->data;
5830 res->end_set = res->data + size;
5832 DEBUG (4, fprintf (gc_debug_file, "Allocated remset size %d at %p for %p\n", size, res->data, id));
5837 * Note: the write barriers first do the needed GC work and then do the actual store:
5838 * this way the value is visible to the conservative GC scan after the write barrier
5839 * itself. If a GC interrupts the barrier in the middle, value will be kept alive by
5840 * the conservative scan, otherwise by the remembered set scan.
5843 mono_gc_wbarrier_set_field (MonoObject *obj, gpointer field_ptr, MonoObject* value)
5847 HEAVY_STAT (++stat_wbarrier_set_field);
5848 if (ptr_in_nursery (field_ptr)) {
5849 *(void**)field_ptr = value;
5852 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", field_ptr));
5854 rs = REMEMBERED_SET;
5855 if (rs->store_next < rs->end_set) {
5856 *(rs->store_next++) = (mword)field_ptr;
5857 *(void**)field_ptr = value;
5861 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
5862 rs->next = REMEMBERED_SET;
5863 REMEMBERED_SET = rs;
5864 #ifdef HAVE_KW_THREAD
5865 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
5867 *(rs->store_next++) = (mword)field_ptr;
5868 *(void**)field_ptr = value;
5873 mono_gc_wbarrier_set_arrayref (MonoArray *arr, gpointer slot_ptr, MonoObject* value)
5877 HEAVY_STAT (++stat_wbarrier_set_arrayref);
5878 if (ptr_in_nursery (slot_ptr)) {
5879 *(void**)slot_ptr = value;
5882 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", slot_ptr));
5884 rs = REMEMBERED_SET;
5885 if (rs->store_next < rs->end_set) {
5886 *(rs->store_next++) = (mword)slot_ptr;
5887 *(void**)slot_ptr = value;
5891 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
5892 rs->next = REMEMBERED_SET;
5893 REMEMBERED_SET = rs;
5894 #ifdef HAVE_KW_THREAD
5895 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
5897 *(rs->store_next++) = (mword)slot_ptr;
5898 *(void**)slot_ptr = value;
5903 mono_gc_wbarrier_arrayref_copy (gpointer dest_ptr, gpointer src_ptr, int count)
5907 HEAVY_STAT (++stat_wbarrier_arrayref_copy);
5909 memmove (dest_ptr, src_ptr, count * sizeof (gpointer));
5910 if (ptr_in_nursery (dest_ptr)) {
5914 rs = REMEMBERED_SET;
5915 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p, %d\n", dest_ptr, count));
5916 if (rs->store_next + 1 < rs->end_set) {
5917 *(rs->store_next++) = (mword)dest_ptr | REMSET_RANGE;
5918 *(rs->store_next++) = count;
5922 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
5923 rs->next = REMEMBERED_SET;
5924 REMEMBERED_SET = rs;
5925 #ifdef HAVE_KW_THREAD
5926 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
5928 *(rs->store_next++) = (mword)dest_ptr | REMSET_RANGE;
5929 *(rs->store_next++) = count;
5934 find_object_for_ptr_in_area (char *ptr, char *start, char *end)
5936 while (start < end) {
5939 if (!*(void**)start) {
5940 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
5946 #define SCAN_OBJECT_NOSCAN
5947 #include "sgen-scan-object.h"
5949 if (ptr >= old_start && ptr < start)
5956 static char *found_obj;
5959 find_object_for_ptr_callback (char *obj, size_t size, char *ptr)
5961 if (ptr >= obj && ptr < obj + size) {
5962 g_assert (!found_obj);
5967 /* for use in the debugger */
5968 char* find_object_for_ptr (char *ptr);
5970 find_object_for_ptr (char *ptr)
5974 if (ptr >= nursery_section->data && ptr < nursery_section->end_data)
5975 return find_object_for_ptr_in_area (ptr, nursery_section->data, nursery_section->end_data);
5977 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
5978 if (ptr >= bigobj->data && ptr < bigobj->data + bigobj->size)
5979 return bigobj->data;
5983 * Very inefficient, but this is debugging code, supposed to
5984 * be called from gdb, so we don't care.
5987 major_iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)find_object_for_ptr_callback, ptr);
5992 evacuate_remset_buffer (void)
5997 buffer = STORE_REMSET_BUFFER;
5999 add_generic_store_remset_from_buffer (buffer);
6000 memset (buffer, 0, sizeof (gpointer) * STORE_REMSET_BUFFER_SIZE);
6002 STORE_REMSET_BUFFER_INDEX = 0;
6006 mono_gc_wbarrier_generic_nostore (gpointer ptr)
6012 HEAVY_STAT (++stat_wbarrier_generic_store);
6014 #ifdef XDOMAIN_CHECKS_IN_WBARRIER
6015 /* FIXME: ptr_in_heap must be called with the GC lock held */
6016 if (xdomain_checks && *(MonoObject**)ptr && ptr_in_heap (ptr)) {
6017 char *start = find_object_for_ptr (ptr);
6018 MonoObject *value = *(MonoObject**)ptr;
6022 MonoObject *obj = (MonoObject*)start;
6023 if (obj->vtable->domain != value->vtable->domain)
6024 g_assert (is_xdomain_ref_allowed (ptr, start, obj->vtable->domain));
6032 if (*(gpointer*)ptr)
6033 binary_protocol_wbarrier (ptr, *(gpointer*)ptr, (gpointer)LOAD_VTABLE (*(gpointer*)ptr));
6035 if (ptr_in_nursery (ptr) || ptr_on_stack (ptr) || !ptr_in_nursery (*(gpointer*)ptr)) {
6036 DEBUG (8, fprintf (gc_debug_file, "Skipping remset at %p\n", ptr));
6041 buffer = STORE_REMSET_BUFFER;
6042 index = STORE_REMSET_BUFFER_INDEX;
6043 /* This simple optimization eliminates a sizable portion of
6044 entries. Comparing it to the last but one entry as well
6045 doesn't eliminate significantly more entries. */
6046 if (buffer [index] == ptr) {
6051 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", ptr));
6052 HEAVY_STAT (++stat_wbarrier_generic_store_remset);
6055 if (index >= STORE_REMSET_BUFFER_SIZE) {
6056 evacuate_remset_buffer ();
6057 index = STORE_REMSET_BUFFER_INDEX;
6058 g_assert (index == 0);
6061 buffer [index] = ptr;
6062 STORE_REMSET_BUFFER_INDEX = index;
6068 mono_gc_wbarrier_generic_store (gpointer ptr, MonoObject* value)
6070 DEBUG (8, fprintf (gc_debug_file, "Wbarrier store at %p to %p (%s)\n", ptr, value, value ? safe_name (value) : "null"));
6071 *(void**)ptr = value;
6072 if (ptr_in_nursery (value))
6073 mono_gc_wbarrier_generic_nostore (ptr);
6077 mono_gc_wbarrier_set_root (gpointer ptr, MonoObject *value)
6081 HEAVY_STAT (++stat_wbarrier_set_root);
6082 if (ptr_in_nursery (ptr))
6084 DEBUG (8, fprintf (gc_debug_file, "Adding root remset at %p (%s)\n", ptr, value ? safe_name (value) : "null"));
6086 rs = REMEMBERED_SET;
6087 if (rs->store_next + 2 < rs->end_set) {
6088 *(rs->store_next++) = (mword)ptr | REMSET_OTHER;
6089 *(rs->store_next++) = (mword)REMSET_ROOT_LOCATION;
6090 *(void**)ptr = value;
6093 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6094 rs->next = REMEMBERED_SET;
6095 REMEMBERED_SET = rs;
6096 #ifdef HAVE_KW_THREAD
6097 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6099 *(rs->store_next++) = (mword)ptr | REMSET_OTHER;
6100 *(rs->store_next++) = (mword)REMSET_ROOT_LOCATION;
6102 *(void**)ptr = value;
6106 mono_gc_wbarrier_value_copy (gpointer dest, gpointer src, int count, MonoClass *klass)
6110 HEAVY_STAT (++stat_wbarrier_value_copy);
6111 g_assert (klass->valuetype);
6113 memmove (dest, src, count * mono_class_value_size (klass, NULL));
6114 rs = REMEMBERED_SET;
6115 if (ptr_in_nursery (dest) || ptr_on_stack (dest)) {
6119 g_assert (klass->gc_descr_inited);
6120 DEBUG (8, fprintf (gc_debug_file, "Adding value remset at %p, count %d, descr %p for class %s (%p)\n", dest, count, klass->gc_descr, klass->name, klass));
6122 if (rs->store_next + 3 < rs->end_set) {
6123 *(rs->store_next++) = (mword)dest | REMSET_OTHER;
6124 *(rs->store_next++) = (mword)REMSET_VTYPE;
6125 *(rs->store_next++) = (mword)klass->gc_descr;
6126 *(rs->store_next++) = (mword)count;
6130 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6131 rs->next = REMEMBERED_SET;
6132 REMEMBERED_SET = rs;
6133 #ifdef HAVE_KW_THREAD
6134 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6136 *(rs->store_next++) = (mword)dest | REMSET_OTHER;
6137 *(rs->store_next++) = (mword)REMSET_VTYPE;
6138 *(rs->store_next++) = (mword)klass->gc_descr;
6139 *(rs->store_next++) = (mword)count;
6144 * mono_gc_wbarrier_object_copy:
6146 * Write barrier to call when obj is the result of a clone or copy of an object.
6149 mono_gc_wbarrier_object_copy (MonoObject* obj, MonoObject *src)
6155 HEAVY_STAT (++stat_wbarrier_object_copy);
6156 rs = REMEMBERED_SET;
6157 DEBUG (6, fprintf (gc_debug_file, "Adding object remset for %p\n", obj));
6158 size = mono_object_class (obj)->instance_size;
6160 /* do not copy the sync state */
6161 memcpy ((char*)obj + sizeof (MonoObject), (char*)src + sizeof (MonoObject),
6162 size - sizeof (MonoObject));
6163 if (ptr_in_nursery (obj) || ptr_on_stack (obj)) {
6167 if (rs->store_next < rs->end_set) {
6168 *(rs->store_next++) = (mword)obj | REMSET_OBJECT;
6172 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6173 rs->next = REMEMBERED_SET;
6174 REMEMBERED_SET = rs;
6175 #ifdef HAVE_KW_THREAD
6176 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6178 *(rs->store_next++) = (mword)obj | REMSET_OBJECT;
6183 * ######################################################################
6184 * ######## Collector debugging
6185 * ######################################################################
6188 const char*descriptor_types [] = {
6200 describe_ptr (char *ptr)
6206 if (ptr_in_nursery (ptr)) {
6207 printf ("Pointer inside nursery.\n");
6209 if (major_ptr_is_in_non_pinned_space (ptr)) {
6210 printf ("Pointer inside oldspace.\n");
6211 } else if (obj_is_from_pinned_alloc (ptr)) {
6212 printf ("Pointer is inside a pinned chunk.\n");
6214 printf ("Pointer unknown.\n");
6219 if (object_is_pinned (ptr))
6220 printf ("Object is pinned.\n");
6222 if (object_is_forwarded (ptr))
6223 printf ("Object is forwared.\n");
6225 // FIXME: Handle pointers to the inside of objects
6226 vtable = (MonoVTable*)LOAD_VTABLE (ptr);
6228 printf ("VTable: %p\n", vtable);
6229 if (vtable == NULL) {
6230 printf ("VTable is invalid (empty).\n");
6233 if (ptr_in_nursery (vtable)) {
6234 printf ("VTable is invalid (points inside nursery).\n");
6237 printf ("Class: %s\n", vtable->klass->name);
6239 desc = ((GCVTable*)vtable)->desc;
6240 printf ("Descriptor: %lx\n", (long)desc);
6243 printf ("Descriptor type: %d (%s)\n", type, descriptor_types [type]);
6247 find_in_remset_loc (mword *p, char *addr, gboolean *found)
6253 switch ((*p) & REMSET_TYPE_MASK) {
6254 case REMSET_LOCATION:
6255 if (*p == (mword)addr)
6259 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
6261 if ((void**)addr >= ptr && (void**)addr < ptr + count)
6265 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
6266 count = safe_object_get_size ((MonoObject*)ptr);
6267 count += (ALLOC_ALIGN - 1);
6268 count &= (ALLOC_ALIGN - 1);
6269 count /= sizeof (mword);
6270 if ((void**)addr >= ptr && (void**)addr < ptr + count)
6273 case REMSET_OTHER: {
6276 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
6280 switch (desc & 0x7) {
6281 case DESC_TYPE_RUN_LENGTH:
6282 OBJ_RUN_LEN_SIZE (skip_size, desc, ptr);
6284 case DESC_TYPE_SMALL_BITMAP:
6285 OBJ_BITMAP_SIZE (skip_size, desc, start);
6289 g_assert_not_reached ();
6292 /* The descriptor includes the size of MonoObject */
6293 skip_size -= sizeof (MonoObject);
6295 if ((void**)addr >= ptr && (void**)addr < ptr + (skip_size / sizeof (gpointer)))
6299 case REMSET_ROOT_LOCATION:
6302 g_assert_not_reached ();
6307 g_assert_not_reached ();
6313 * Return whenever ADDR occurs in the remembered sets
6316 find_in_remsets (char *addr)
6319 SgenThreadInfo *info;
6320 RememberedSet *remset;
6321 GenericStoreRememberedSet *store_remset;
6323 gboolean found = FALSE;
6325 /* the global one */
6326 for (remset = global_remset; remset; remset = remset->next) {
6327 DEBUG (4, fprintf (gc_debug_file, "Scanning global remset range: %p-%p, size: %zd\n", remset->data, remset->store_next, remset->store_next - remset->data));
6328 for (p = remset->data; p < remset->store_next;) {
6329 p = find_in_remset_loc (p, addr, &found);
6335 /* the generic store ones */
6336 for (store_remset = generic_store_remsets; store_remset; store_remset = store_remset->next) {
6337 for (i = 0; i < STORE_REMSET_BUFFER_SIZE - 1; ++i) {
6338 if (store_remset->data [i] == addr)
6343 /* the per-thread ones */
6344 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
6345 for (info = thread_table [i]; info; info = info->next) {
6347 for (remset = info->remset; remset; remset = remset->next) {
6348 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for thread %p, range: %p-%p, size: %zd\n", info, remset->data, remset->store_next, remset->store_next - remset->data));
6349 for (p = remset->data; p < remset->store_next;) {
6350 p = find_in_remset_loc (p, addr, &found);
6355 for (j = 0; j < *info->store_remset_buffer_index_addr; ++j) {
6356 if ((*info->store_remset_buffer_addr) [j + 1] == addr)
6362 /* the freed thread ones */
6363 for (remset = freed_thread_remsets; remset; remset = remset->next) {
6364 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for freed thread, range: %p-%p, size: %zd\n", remset->data, remset->store_next, remset->store_next - remset->data));
6365 for (p = remset->data; p < remset->store_next;) {
6366 p = find_in_remset_loc (p, addr, &found);
6375 static gboolean missing_remsets;
6378 * We let a missing remset slide if the target object is pinned,
6379 * because the store might have happened but the remset not yet added,
6380 * but in that case the target must be pinned. We might theoretically
6381 * miss some missing remsets this way, but it's very unlikely.
6384 #define HANDLE_PTR(ptr,obj) do { \
6385 if (*(ptr) && (char*)*(ptr) >= nursery_start && (char*)*(ptr) < nursery_next) { \
6386 if (!find_in_remsets ((char*)(ptr))) { \
6387 fprintf (gc_debug_file, "Oldspace->newspace reference %p at offset %zd in object %p (%s.%s) not found in remsets.\n", *(ptr), (char*)(ptr) - (char*)(obj), (obj), ((MonoObject*)(obj))->vtable->klass->name_space, ((MonoObject*)(obj))->vtable->klass->name); \
6388 binary_protocol_missing_remset ((obj), (gpointer)LOAD_VTABLE ((obj)), (char*)(ptr) - (char*)(obj), *(ptr), (gpointer)LOAD_VTABLE(*(ptr)), object_is_pinned (*(ptr))); \
6389 if (!object_is_pinned (*(ptr))) \
6390 missing_remsets = TRUE; \
6396 * Check that each object reference which points into the nursery can
6397 * be found in the remembered sets.
6400 check_consistency_callback (char *start, size_t size, void *dummy)
6402 GCVTable *vt = (GCVTable*)LOAD_VTABLE (start);
6403 DEBUG (8, fprintf (gc_debug_file, "Scanning object %p, vtable: %p (%s)\n", start, vt, vt->klass->name));
6405 #define SCAN_OBJECT_ACTION
6406 #include "sgen-scan-object.h"
6410 * Perform consistency check of the heap.
6412 * Assumes the world is stopped.
6415 check_consistency (void)
6417 // Need to add more checks
6419 missing_remsets = FALSE;
6421 DEBUG (1, fprintf (gc_debug_file, "Begin heap consistency check...\n"));
6423 // Check that oldspace->newspace pointers are registered with the collector
6424 major_iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)check_consistency_callback, NULL);
6426 DEBUG (1, fprintf (gc_debug_file, "Heap consistency check done.\n"));
6428 #ifdef BINARY_PROTOCOL
6429 if (!binary_protocol_file)
6431 g_assert (!missing_remsets);
6434 /* Check that the reference is valid */
6436 #define HANDLE_PTR(ptr,obj) do { \
6438 g_assert (safe_name (*(ptr)) != NULL); \
6445 * Perform consistency check on an object. Currently we only check that the
6446 * reference fields are valid.
6449 check_object (char *start)
6454 #include "sgen-scan-object.h"
6460 * ######################################################################
6461 * ######## Other mono public interface functions.
6462 * ######################################################################
6466 mono_gc_collect (int generation)
6470 if (generation == 0) {
6471 collect_nursery (0);
6473 major_collection ("user request");
6480 mono_gc_max_generation (void)
6486 mono_gc_collection_count (int generation)
6488 if (generation == 0)
6489 return num_minor_gcs;
6490 return num_major_gcs;
6494 mono_gc_get_used_size (void)
6498 tot = los_memory_usage;
6499 tot += nursery_section->next_data - nursery_section->data;
6500 tot += major_get_used_size ();
6501 /* FIXME: account for pinned objects */
6507 mono_gc_get_heap_size (void)
6513 mono_gc_disable (void)
6521 mono_gc_enable (void)
6529 mono_gc_get_los_limit (void)
6531 return MAX_SMALL_OBJ_SIZE;
6535 mono_object_is_alive (MonoObject* o)
6541 mono_gc_get_generation (MonoObject *obj)
6543 if (ptr_in_nursery (obj))
6549 mono_gc_enable_events (void)
6554 mono_gc_weak_link_add (void **link_addr, MonoObject *obj, gboolean track)
6557 mono_gc_register_disappearing_link (obj, link_addr, track);
6562 mono_gc_weak_link_remove (void **link_addr)
6565 mono_gc_register_disappearing_link (NULL, link_addr, FALSE);
6570 mono_gc_weak_link_get (void **link_addr)
6574 return (MonoObject*) REVEAL_POINTER (*link_addr);
6578 mono_gc_make_descr_from_bitmap (gsize *bitmap, int numbits)
6580 if (numbits < ((sizeof (*bitmap) * 8) - ROOT_DESC_TYPE_SHIFT)) {
6581 return (void*)MAKE_ROOT_DESC (ROOT_DESC_BITMAP, bitmap [0]);
6583 mword complex = alloc_complex_descriptor (bitmap, numbits);
6584 return (void*)MAKE_ROOT_DESC (ROOT_DESC_COMPLEX, complex);
6589 mono_gc_make_root_descr_user (MonoGCRootMarkFunc marker)
6593 g_assert (user_descriptors_next < MAX_USER_DESCRIPTORS);
6594 descr = (void*)MAKE_ROOT_DESC (ROOT_DESC_USER, (mword)user_descriptors_next);
6595 user_descriptors [user_descriptors_next ++] = marker;
6601 mono_gc_alloc_fixed (size_t size, void *descr)
6603 /* FIXME: do a single allocation */
6604 void *res = calloc (1, size);
6607 if (!mono_gc_register_root (res, size, descr)) {
6615 mono_gc_free_fixed (void* addr)
6617 mono_gc_deregister_root (addr);
6622 mono_gc_invoke_with_gc_lock (MonoGCLockedCallbackFunc func, void *data)
6626 result = func (data);
6627 UNLOCK_INTERRUPTION;
6632 mono_gc_is_gc_thread (void)
6636 result = thread_info_lookup (ARCH_GET_THREAD ()) != NULL;
6643 /* Tries to extract a number from the passed string, taking in to account m, k
6646 parse_environment_string_extract_number (gchar *str, glong *out)
6649 int len = strlen (str), shift = 0;
6651 gboolean is_suffix = FALSE;
6654 switch (str [len - 1]) {
6665 suffix = str [len - 1];
6670 val = strtol (str, &endptr, 10);
6672 if ((errno == ERANGE && (val == LONG_MAX || val == LONG_MIN))
6673 || (errno != 0 && val == 0) || (endptr == str))
6677 if (*(endptr + 1)) /* Invalid string. */
6689 mono_gc_base_init (void)
6693 struct sigaction sinfo;
6695 LOCK_INIT (gc_mutex);
6697 if (gc_initialized) {
6701 pagesize = mono_pagesize ();
6702 gc_debug_file = stderr;
6706 if ((env = getenv ("MONO_GC_PARAMS"))) {
6707 if (g_str_has_prefix (env, "nursery-size")) {
6710 while (env [index] && env [index++] != '=')
6712 if (env [index] && parse_environment_string_extract_number (env
6714 default_nursery_size = val;
6715 #ifdef ALIGN_NURSERY
6716 if ((val & (val - 1))) {
6717 fprintf (stderr, "The nursery size must be a power of two.\n");
6721 default_nursery_bits = 0;
6722 while (1 << (++ default_nursery_bits) != default_nursery_size)
6726 fprintf (stderr, "nursery-size must be an integer.\n");
6730 fprintf (stderr, "MONO_GC_PARAMS must be of the form 'nursery-size=N' (where N is an integer, possibly with a k, m or a g suffix).\n");
6737 nursery_size = DEFAULT_NURSERY_SIZE;
6741 if ((env = getenv ("MONO_GC_DEBUG"))) {
6742 opts = g_strsplit (env, ",", -1);
6743 for (ptr = opts; ptr && *ptr; ptr ++) {
6745 if (opt [0] >= '0' && opt [0] <= '9') {
6746 gc_debug_level = atoi (opt);
6751 char *rf = g_strdup_printf ("%s.%d", opt, getpid ());
6752 gc_debug_file = fopen (rf, "wb");
6754 gc_debug_file = stderr;
6757 } else if (!strcmp (opt, "collect-before-allocs")) {
6758 collect_before_allocs = TRUE;
6759 } else if (!strcmp (opt, "check-at-minor-collections")) {
6760 consistency_check_at_minor_collection = TRUE;
6761 } else if (!strcmp (opt, "xdomain-checks")) {
6762 xdomain_checks = TRUE;
6763 } else if (!strcmp (opt, "clear-at-gc")) {
6764 nursery_clear_policy = CLEAR_AT_GC;
6765 } else if (!strcmp (opt, "conservative-stack-mark")) {
6766 conservative_stack_mark = TRUE;
6767 } else if (!strcmp (opt, "check-scan-starts")) {
6768 do_scan_starts_check = TRUE;
6769 } else if (g_str_has_prefix (opt, "heap-dump=")) {
6770 char *filename = strchr (opt, '=') + 1;
6771 nursery_clear_policy = CLEAR_AT_GC;
6772 heap_dump_file = fopen (filename, "w");
6774 fprintf (heap_dump_file, "<sgen-dump>\n");
6775 #ifdef BINARY_PROTOCOL
6776 } else if (g_str_has_prefix (opt, "binary-protocol=")) {
6777 char *filename = strchr (opt, '=') + 1;
6778 binary_protocol_file = fopen (filename, "w");
6781 fprintf (stderr, "Invalid format for the MONO_GC_DEBUG env variable: '%s'\n", env);
6782 fprintf (stderr, "The format is: MONO_GC_DEBUG=[l[:filename]|<option>]+ where l is a debug level 0-9.\n");
6783 fprintf (stderr, "Valid options are: collect-before-allocs, check-at-minor-collections, xdomain-checks, clear-at-gc.\n");
6790 suspend_ack_semaphore_ptr = &suspend_ack_semaphore;
6791 MONO_SEM_INIT (&suspend_ack_semaphore, 0);
6793 sigfillset (&sinfo.sa_mask);
6794 sinfo.sa_flags = SA_RESTART | SA_SIGINFO;
6795 sinfo.sa_sigaction = suspend_handler;
6796 if (sigaction (suspend_signal_num, &sinfo, NULL) != 0) {
6797 g_error ("failed sigaction");
6800 sinfo.sa_handler = restart_handler;
6801 if (sigaction (restart_signal_num, &sinfo, NULL) != 0) {
6802 g_error ("failed sigaction");
6805 sigfillset (&suspend_signal_mask);
6806 sigdelset (&suspend_signal_mask, restart_signal_num);
6808 global_remset = alloc_remset (1024, NULL);
6809 global_remset->next = NULL;
6811 pthread_key_create (&remembered_set_key, unregister_thread);
6813 #ifndef HAVE_KW_THREAD
6814 pthread_key_create (&thread_info_key, NULL);
6817 gc_initialized = TRUE;
6819 mono_gc_register_thread (&sinfo);
6823 mono_gc_get_suspend_signal (void)
6825 return suspend_signal_num;
6835 #ifdef HAVE_KW_THREAD
6836 #define EMIT_TLS_ACCESS(mb,dummy,offset) do { \
6837 mono_mb_emit_byte ((mb), MONO_CUSTOM_PREFIX); \
6838 mono_mb_emit_byte ((mb), CEE_MONO_TLS); \
6839 mono_mb_emit_i4 ((mb), (offset)); \
6842 #define EMIT_TLS_ACCESS(mb,member,dummy) do { \
6843 mono_mb_emit_byte ((mb), MONO_CUSTOM_PREFIX); \
6844 mono_mb_emit_byte ((mb), CEE_MONO_TLS); \
6845 mono_mb_emit_i4 ((mb), thread_info_key); \
6846 mono_mb_emit_icon ((mb), G_STRUCT_OFFSET (SgenThreadInfo, member)); \
6847 mono_mb_emit_byte ((mb), CEE_ADD); \
6848 mono_mb_emit_byte ((mb), CEE_LDIND_I); \
6852 #ifdef MANAGED_ALLOCATION
6853 /* FIXME: Do this in the JIT, where specialized allocation sequences can be created
6854 * for each class. This is currently not easy to do, as it is hard to generate basic
6855 * blocks + branches, but it is easy with the linear IL codebase.
6857 * For this to work we'd need to solve the TLAB race, first. Now we
6858 * require the allocator to be in a few known methods to make sure
6859 * that they are executed atomically via the restart mechanism.
6862 create_allocator (int atype)
6864 int p_var, size_var;
6865 guint32 slowpath_branch, max_size_branch;
6866 MonoMethodBuilder *mb;
6868 MonoMethodSignature *csig;
6869 static gboolean registered = FALSE;
6870 int tlab_next_addr_var, new_next_var;
6872 const char *name = NULL;
6873 AllocatorWrapperInfo *info;
6875 #ifdef HAVE_KW_THREAD
6876 int tlab_next_addr_offset = -1;
6877 int tlab_temp_end_offset = -1;
6879 MONO_THREAD_VAR_OFFSET (tlab_next_addr, tlab_next_addr_offset);
6880 MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
6882 g_assert (tlab_next_addr_offset != -1);
6883 g_assert (tlab_temp_end_offset != -1);
6887 mono_register_jit_icall (mono_gc_alloc_obj, "mono_gc_alloc_obj", mono_create_icall_signature ("object ptr int"), FALSE);
6888 mono_register_jit_icall (mono_gc_alloc_vector, "mono_gc_alloc_vector", mono_create_icall_signature ("object ptr int int"), FALSE);
6892 if (atype == ATYPE_SMALL) {
6894 name = "AllocSmall";
6895 } else if (atype == ATYPE_NORMAL) {
6898 } else if (atype == ATYPE_VECTOR) {
6900 name = "AllocVector";
6902 g_assert_not_reached ();
6905 csig = mono_metadata_signature_alloc (mono_defaults.corlib, num_params);
6906 csig->ret = &mono_defaults.object_class->byval_arg;
6907 for (i = 0; i < num_params; ++i)
6908 csig->params [i] = &mono_defaults.int_class->byval_arg;
6910 mb = mono_mb_new (mono_defaults.object_class, name, MONO_WRAPPER_ALLOC);
6911 size_var = mono_mb_add_local (mb, &mono_defaults.int32_class->byval_arg);
6912 if (atype == ATYPE_NORMAL || atype == ATYPE_SMALL) {
6913 /* size = vtable->klass->instance_size; */
6914 mono_mb_emit_ldarg (mb, 0);
6915 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoVTable, klass));
6916 mono_mb_emit_byte (mb, CEE_ADD);
6917 mono_mb_emit_byte (mb, CEE_LDIND_I);
6918 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoClass, instance_size));
6919 mono_mb_emit_byte (mb, CEE_ADD);
6920 /* FIXME: assert instance_size stays a 4 byte integer */
6921 mono_mb_emit_byte (mb, CEE_LDIND_U4);
6922 mono_mb_emit_stloc (mb, size_var);
6923 } else if (atype == ATYPE_VECTOR) {
6924 MonoExceptionClause *clause;
6926 MonoClass *oom_exc_class;
6929 /* n > MONO_ARRAY_MAX_INDEX -> OverflowException */
6930 mono_mb_emit_ldarg (mb, 1);
6931 mono_mb_emit_icon (mb, MONO_ARRAY_MAX_INDEX);
6932 pos = mono_mb_emit_short_branch (mb, CEE_BLE_UN_S);
6933 mono_mb_emit_exception (mb, "OverflowException", NULL);
6934 mono_mb_patch_short_branch (mb, pos);
6936 clause = mono_image_alloc0 (mono_defaults.corlib, sizeof (MonoExceptionClause));
6937 clause->try_offset = mono_mb_get_label (mb);
6939 /* vtable->klass->sizes.element_size */
6940 mono_mb_emit_ldarg (mb, 0);
6941 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoVTable, klass));
6942 mono_mb_emit_byte (mb, CEE_ADD);
6943 mono_mb_emit_byte (mb, CEE_LDIND_I);
6944 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoClass, sizes.element_size));
6945 mono_mb_emit_byte (mb, CEE_ADD);
6946 mono_mb_emit_byte (mb, CEE_LDIND_U4);
6949 mono_mb_emit_ldarg (mb, 1);
6950 mono_mb_emit_byte (mb, CEE_MUL_OVF_UN);
6951 /* + sizeof (MonoArray) */
6952 mono_mb_emit_icon (mb, sizeof (MonoArray));
6953 mono_mb_emit_byte (mb, CEE_ADD_OVF_UN);
6954 mono_mb_emit_stloc (mb, size_var);
6956 pos_leave = mono_mb_emit_branch (mb, CEE_LEAVE);
6959 clause->flags = MONO_EXCEPTION_CLAUSE_NONE;
6960 clause->try_len = mono_mb_get_pos (mb) - clause->try_offset;
6961 clause->data.catch_class = mono_class_from_name (mono_defaults.corlib,
6962 "System", "OverflowException");
6963 g_assert (clause->data.catch_class);
6964 clause->handler_offset = mono_mb_get_label (mb);
6966 oom_exc_class = mono_class_from_name (mono_defaults.corlib,
6967 "System", "OutOfMemoryException");
6968 g_assert (oom_exc_class);
6969 ctor = mono_class_get_method_from_name (oom_exc_class, ".ctor", 0);
6972 mono_mb_emit_byte (mb, CEE_POP);
6973 mono_mb_emit_op (mb, CEE_NEWOBJ, ctor);
6974 mono_mb_emit_byte (mb, CEE_THROW);
6976 clause->handler_len = mono_mb_get_pos (mb) - clause->handler_offset;
6977 mono_mb_set_clauses (mb, 1, clause);
6978 mono_mb_patch_branch (mb, pos_leave);
6981 g_assert_not_reached ();
6984 /* size += ALLOC_ALIGN - 1; */
6985 mono_mb_emit_ldloc (mb, size_var);
6986 mono_mb_emit_icon (mb, ALLOC_ALIGN - 1);
6987 mono_mb_emit_byte (mb, CEE_ADD);
6988 /* size &= ~(ALLOC_ALIGN - 1); */
6989 mono_mb_emit_icon (mb, ~(ALLOC_ALIGN - 1));
6990 mono_mb_emit_byte (mb, CEE_AND);
6991 mono_mb_emit_stloc (mb, size_var);
6993 /* if (size > MAX_SMALL_OBJ_SIZE) goto slowpath */
6994 if (atype != ATYPE_SMALL) {
6995 mono_mb_emit_ldloc (mb, size_var);
6996 mono_mb_emit_icon (mb, MAX_SMALL_OBJ_SIZE);
6997 max_size_branch = mono_mb_emit_short_branch (mb, MONO_CEE_BGT_S);
7001 * We need to modify tlab_next, but the JIT only supports reading, so we read
7002 * another tls var holding its address instead.
7005 /* tlab_next_addr (local) = tlab_next_addr (TLS var) */
7006 tlab_next_addr_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7007 EMIT_TLS_ACCESS (mb, tlab_next_addr, tlab_next_addr_offset);
7008 mono_mb_emit_stloc (mb, tlab_next_addr_var);
7010 /* p = (void**)tlab_next; */
7011 p_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7012 mono_mb_emit_ldloc (mb, tlab_next_addr_var);
7013 mono_mb_emit_byte (mb, CEE_LDIND_I);
7014 mono_mb_emit_stloc (mb, p_var);
7016 /* new_next = (char*)p + size; */
7017 new_next_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7018 mono_mb_emit_ldloc (mb, p_var);
7019 mono_mb_emit_ldloc (mb, size_var);
7020 mono_mb_emit_byte (mb, CEE_CONV_I);
7021 mono_mb_emit_byte (mb, CEE_ADD);
7022 mono_mb_emit_stloc (mb, new_next_var);
7024 /* tlab_next = new_next */
7025 mono_mb_emit_ldloc (mb, tlab_next_addr_var);
7026 mono_mb_emit_ldloc (mb, new_next_var);
7027 mono_mb_emit_byte (mb, CEE_STIND_I);
7029 /* if (G_LIKELY (new_next < tlab_temp_end)) */
7030 mono_mb_emit_ldloc (mb, new_next_var);
7031 EMIT_TLS_ACCESS (mb, tlab_temp_end, tlab_temp_end_offset);
7032 slowpath_branch = mono_mb_emit_short_branch (mb, MONO_CEE_BLT_UN_S);
7035 if (atype != ATYPE_SMALL)
7036 mono_mb_patch_short_branch (mb, max_size_branch);
7038 mono_mb_emit_byte (mb, MONO_CUSTOM_PREFIX);
7039 mono_mb_emit_byte (mb, CEE_MONO_NOT_TAKEN);
7041 /* FIXME: mono_gc_alloc_obj takes a 'size_t' as an argument, not an int32 */
7042 mono_mb_emit_ldarg (mb, 0);
7043 mono_mb_emit_ldloc (mb, size_var);
7044 if (atype == ATYPE_NORMAL || atype == ATYPE_SMALL) {
7045 mono_mb_emit_icall (mb, mono_gc_alloc_obj);
7046 } else if (atype == ATYPE_VECTOR) {
7047 mono_mb_emit_ldarg (mb, 1);
7048 mono_mb_emit_icall (mb, mono_gc_alloc_vector);
7050 g_assert_not_reached ();
7052 mono_mb_emit_byte (mb, CEE_RET);
7055 mono_mb_patch_short_branch (mb, slowpath_branch);
7057 /* FIXME: Memory barrier */
7060 mono_mb_emit_ldloc (mb, p_var);
7061 mono_mb_emit_ldarg (mb, 0);
7062 mono_mb_emit_byte (mb, CEE_STIND_I);
7064 if (atype == ATYPE_VECTOR) {
7065 /* arr->max_length = max_length; */
7066 mono_mb_emit_ldloc (mb, p_var);
7067 mono_mb_emit_ldflda (mb, G_STRUCT_OFFSET (MonoArray, max_length));
7068 mono_mb_emit_ldarg (mb, 1);
7069 mono_mb_emit_byte (mb, CEE_STIND_I);
7073 mono_mb_emit_ldloc (mb, p_var);
7074 mono_mb_emit_byte (mb, CEE_RET);
7076 res = mono_mb_create_method (mb, csig, 8);
7078 mono_method_get_header (res)->init_locals = FALSE;
7080 info = mono_image_alloc0 (mono_defaults.corlib, sizeof (AllocatorWrapperInfo));
7081 info->alloc_type = atype;
7082 mono_marshal_set_wrapper_info (res, info);
7088 static MonoMethod* alloc_method_cache [ATYPE_NUM];
7089 static MonoMethod *write_barrier_method;
7092 is_ip_in_managed_allocator (MonoDomain *domain, gpointer ip)
7100 ji = mono_jit_info_table_find (domain, ip);
7103 method = ji->method;
7105 if (method == write_barrier_method)
7107 for (i = 0; i < ATYPE_NUM; ++i)
7108 if (method == alloc_method_cache [i])
7114 * Generate an allocator method implementing the fast path of mono_gc_alloc_obj ().
7115 * The signature of the called method is:
7116 * object allocate (MonoVTable *vtable)
7119 mono_gc_get_managed_allocator (MonoVTable *vtable, gboolean for_box)
7121 #ifdef MANAGED_ALLOCATION
7122 MonoClass *klass = vtable->klass;
7124 #ifdef HAVE_KW_THREAD
7125 int tlab_next_offset = -1;
7126 int tlab_temp_end_offset = -1;
7127 MONO_THREAD_VAR_OFFSET (tlab_next, tlab_next_offset);
7128 MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
7130 if (tlab_next_offset == -1 || tlab_temp_end_offset == -1)
7134 if (!mono_runtime_has_tls_get ())
7136 if (klass->instance_size > tlab_size)
7138 if (klass->has_finalize || klass->marshalbyref || (mono_profiler_get_events () & MONO_PROFILE_ALLOCATIONS))
7142 if (klass->byval_arg.type == MONO_TYPE_STRING)
7144 if (collect_before_allocs)
7147 if (ALIGN_TO (klass->instance_size, ALLOC_ALIGN) < MAX_SMALL_OBJ_SIZE)
7148 return mono_gc_get_managed_allocator_by_type (ATYPE_SMALL);
7150 return mono_gc_get_managed_allocator_by_type (ATYPE_NORMAL);
7157 mono_gc_get_managed_array_allocator (MonoVTable *vtable, int rank)
7159 #ifdef MANAGED_ALLOCATION
7160 MonoClass *klass = vtable->klass;
7162 #ifdef HAVE_KW_THREAD
7163 int tlab_next_offset = -1;
7164 int tlab_temp_end_offset = -1;
7165 MONO_THREAD_VAR_OFFSET (tlab_next, tlab_next_offset);
7166 MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
7168 if (tlab_next_offset == -1 || tlab_temp_end_offset == -1)
7174 if (!mono_runtime_has_tls_get ())
7176 if (mono_profiler_get_events () & MONO_PROFILE_ALLOCATIONS)
7178 if (collect_before_allocs)
7180 g_assert (!klass->has_finalize && !klass->marshalbyref);
7182 return mono_gc_get_managed_allocator_by_type (ATYPE_VECTOR);
7189 mono_gc_get_managed_allocator_by_type (int atype)
7191 #ifdef MANAGED_ALLOCATION
7194 if (!mono_runtime_has_tls_get ())
7197 mono_loader_lock ();
7198 res = alloc_method_cache [atype];
7200 res = alloc_method_cache [atype] = create_allocator (atype);
7201 mono_loader_unlock ();
7209 mono_gc_get_managed_allocator_types (void)
7216 mono_gc_get_write_barrier (void)
7219 MonoMethodBuilder *mb;
7220 MonoMethodSignature *sig;
7221 #ifdef MANAGED_WBARRIER
7222 int label_no_wb_1, label_no_wb_2, label_no_wb_3, label_no_wb_4, label_need_wb, label_slow_path;
7223 #ifndef ALIGN_NURSERY
7224 int label_continue_1, label_continue_2, label_no_wb_5;
7225 int dereferenced_var;
7227 int buffer_var, buffer_index_var, dummy_var;
7229 #ifdef HAVE_KW_THREAD
7230 int stack_end_offset = -1, store_remset_buffer_offset = -1;
7231 int store_remset_buffer_index_offset = -1, store_remset_buffer_index_addr_offset = -1;
7233 MONO_THREAD_VAR_OFFSET (stack_end, stack_end_offset);
7234 g_assert (stack_end_offset != -1);
7235 MONO_THREAD_VAR_OFFSET (store_remset_buffer, store_remset_buffer_offset);
7236 g_assert (store_remset_buffer_offset != -1);
7237 MONO_THREAD_VAR_OFFSET (store_remset_buffer_index, store_remset_buffer_index_offset);
7238 g_assert (store_remset_buffer_index_offset != -1);
7239 MONO_THREAD_VAR_OFFSET (store_remset_buffer_index_addr, store_remset_buffer_index_addr_offset);
7240 g_assert (store_remset_buffer_index_addr_offset != -1);
7244 // FIXME: Maybe create a separate version for ctors (the branch would be
7245 // correctly predicted more times)
7246 if (write_barrier_method)
7247 return write_barrier_method;
7249 /* Create the IL version of mono_gc_barrier_generic_store () */
7250 sig = mono_metadata_signature_alloc (mono_defaults.corlib, 1);
7251 sig->ret = &mono_defaults.void_class->byval_arg;
7252 sig->params [0] = &mono_defaults.int_class->byval_arg;
7254 mb = mono_mb_new (mono_defaults.object_class, "wbarrier", MONO_WRAPPER_WRITE_BARRIER);
7256 #ifdef MANAGED_WBARRIER
7257 if (mono_runtime_has_tls_get ()) {
7258 #ifdef ALIGN_NURSERY
7259 // if (ptr_in_nursery (ptr)) return;
7261 * Masking out the bits might be faster, but we would have to use 64 bit
7262 * immediates, which might be slower.
7264 mono_mb_emit_ldarg (mb, 0);
7265 mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
7266 mono_mb_emit_byte (mb, CEE_SHR_UN);
7267 mono_mb_emit_icon (mb, (mword)nursery_start >> DEFAULT_NURSERY_BITS);
7268 label_no_wb_1 = mono_mb_emit_branch (mb, CEE_BEQ);
7270 // if (!ptr_in_nursery (*ptr)) return;
7271 mono_mb_emit_ldarg (mb, 0);
7272 mono_mb_emit_byte (mb, CEE_LDIND_I);
7273 mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
7274 mono_mb_emit_byte (mb, CEE_SHR_UN);
7275 mono_mb_emit_icon (mb, (mword)nursery_start >> DEFAULT_NURSERY_BITS);
7276 label_no_wb_2 = mono_mb_emit_branch (mb, CEE_BNE_UN);
7279 // if (ptr < (nursery_start)) goto continue;
7280 mono_mb_emit_ldarg (mb, 0);
7281 mono_mb_emit_ptr (mb, (gpointer) nursery_start);
7282 label_continue_1 = mono_mb_emit_branch (mb, CEE_BLT);
7284 // if (ptr >= nursery_real_end)) goto continue;
7285 mono_mb_emit_ldarg (mb, 0);
7286 mono_mb_emit_ptr (mb, (gpointer) nursery_real_end);
7287 label_continue_2 = mono_mb_emit_branch (mb, CEE_BGE);
7290 label_no_wb_1 = mono_mb_emit_branch (mb, CEE_BR);
7293 mono_mb_patch_branch (mb, label_continue_1);
7294 mono_mb_patch_branch (mb, label_continue_2);
7296 // Dereference and store in local var
7297 dereferenced_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7298 mono_mb_emit_ldarg (mb, 0);
7299 mono_mb_emit_byte (mb, CEE_LDIND_I);
7300 mono_mb_emit_stloc (mb, dereferenced_var);
7302 // if (*ptr < nursery_start) return;
7303 mono_mb_emit_ldloc (mb, dereferenced_var);
7304 mono_mb_emit_ptr (mb, (gpointer) nursery_start);
7305 label_no_wb_2 = mono_mb_emit_branch (mb, CEE_BLT);
7307 // if (*ptr >= nursery_end) return;
7308 mono_mb_emit_ldloc (mb, dereferenced_var);
7309 mono_mb_emit_ptr (mb, (gpointer) nursery_real_end);
7310 label_no_wb_5 = mono_mb_emit_branch (mb, CEE_BGE);
7313 // if (ptr >= stack_end) goto need_wb;
7314 mono_mb_emit_ldarg (mb, 0);
7315 EMIT_TLS_ACCESS (mb, stack_end, stack_end_offset);
7316 label_need_wb = mono_mb_emit_branch (mb, CEE_BGE_UN);
7318 // if (ptr >= stack_start) return;
7319 dummy_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7320 mono_mb_emit_ldarg (mb, 0);
7321 mono_mb_emit_ldloc_addr (mb, dummy_var);
7322 label_no_wb_3 = mono_mb_emit_branch (mb, CEE_BGE_UN);
7325 mono_mb_patch_branch (mb, label_need_wb);
7327 // buffer = STORE_REMSET_BUFFER;
7328 buffer_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7329 EMIT_TLS_ACCESS (mb, store_remset_buffer, store_remset_buffer_offset);
7330 mono_mb_emit_stloc (mb, buffer_var);
7332 // buffer_index = STORE_REMSET_BUFFER_INDEX;
7333 buffer_index_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7334 EMIT_TLS_ACCESS (mb, store_remset_buffer_index, store_remset_buffer_index_offset);
7335 mono_mb_emit_stloc (mb, buffer_index_var);
7337 // if (buffer [buffer_index] == ptr) return;
7338 mono_mb_emit_ldloc (mb, buffer_var);
7339 mono_mb_emit_ldloc (mb, buffer_index_var);
7340 g_assert (sizeof (gpointer) == 4 || sizeof (gpointer) == 8);
7341 mono_mb_emit_icon (mb, sizeof (gpointer) == 4 ? 2 : 3);
7342 mono_mb_emit_byte (mb, CEE_SHL);
7343 mono_mb_emit_byte (mb, CEE_ADD);
7344 mono_mb_emit_byte (mb, CEE_LDIND_I);
7345 mono_mb_emit_ldarg (mb, 0);
7346 label_no_wb_4 = mono_mb_emit_branch (mb, CEE_BEQ);
7349 mono_mb_emit_ldloc (mb, buffer_index_var);
7350 mono_mb_emit_icon (mb, 1);
7351 mono_mb_emit_byte (mb, CEE_ADD);
7352 mono_mb_emit_stloc (mb, buffer_index_var);
7354 // if (buffer_index >= STORE_REMSET_BUFFER_SIZE) goto slow_path;
7355 mono_mb_emit_ldloc (mb, buffer_index_var);
7356 mono_mb_emit_icon (mb, STORE_REMSET_BUFFER_SIZE);
7357 label_slow_path = mono_mb_emit_branch (mb, CEE_BGE);
7359 // buffer [buffer_index] = ptr;
7360 mono_mb_emit_ldloc (mb, buffer_var);
7361 mono_mb_emit_ldloc (mb, buffer_index_var);
7362 g_assert (sizeof (gpointer) == 4 || sizeof (gpointer) == 8);
7363 mono_mb_emit_icon (mb, sizeof (gpointer) == 4 ? 2 : 3);
7364 mono_mb_emit_byte (mb, CEE_SHL);
7365 mono_mb_emit_byte (mb, CEE_ADD);
7366 mono_mb_emit_ldarg (mb, 0);
7367 mono_mb_emit_byte (mb, CEE_STIND_I);
7369 // STORE_REMSET_BUFFER_INDEX = buffer_index;
7370 EMIT_TLS_ACCESS (mb, store_remset_buffer_index_addr, store_remset_buffer_index_addr_offset);
7371 mono_mb_emit_ldloc (mb, buffer_index_var);
7372 mono_mb_emit_byte (mb, CEE_STIND_I);
7375 mono_mb_patch_branch (mb, label_no_wb_1);
7376 mono_mb_patch_branch (mb, label_no_wb_2);
7377 mono_mb_patch_branch (mb, label_no_wb_3);
7378 mono_mb_patch_branch (mb, label_no_wb_4);
7379 #ifndef ALIGN_NURSERY
7380 mono_mb_patch_branch (mb, label_no_wb_5);
7382 mono_mb_emit_byte (mb, CEE_RET);
7385 mono_mb_patch_branch (mb, label_slow_path);
7389 mono_mb_emit_ldarg (mb, 0);
7390 mono_mb_emit_icall (mb, mono_gc_wbarrier_generic_nostore);
7391 mono_mb_emit_byte (mb, CEE_RET);
7393 res = mono_mb_create_method (mb, sig, 16);
7396 mono_loader_lock ();
7397 if (write_barrier_method) {
7398 /* Already created */
7399 mono_free_method (res);
7401 /* double-checked locking */
7402 mono_memory_barrier ();
7403 write_barrier_method = res;
7405 mono_loader_unlock ();
7407 return write_barrier_method;
7410 #endif /* HAVE_SGEN_GC */