2 * sgen-gc.c: Simple generational GC.
5 * Paolo Molaro (lupus@ximian.com)
7 * Copyright 2005-2009 Novell, Inc (http://www.novell.com)
9 * Thread start/stop adapted from Boehm's GC:
10 * Copyright (c) 1994 by Xerox Corporation. All rights reserved.
11 * Copyright (c) 1996 by Silicon Graphics. All rights reserved.
12 * Copyright (c) 1998 by Fergus Henderson. All rights reserved.
13 * Copyright (c) 2000-2004 by Hewlett-Packard Company. All rights reserved.
15 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
16 * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
18 * Permission is hereby granted to use or copy this program
19 * for any purpose, provided the above notices are retained on all copies.
20 * Permission to modify the code and to distribute modified code is granted,
21 * provided the above notices are retained, and a notice that the code was
22 * modified is included with the above copyright notice.
24 * All the rest of the code is LGPL.
26 * Important: allocation provides always zeroed memory, having to do
27 * a memset after allocation is deadly for performance.
28 * Memory usage at startup is currently as follows:
30 * 64 KB internal space
32 * We should provide a small memory config with half the sizes
34 * We currently try to make as few mono assumptions as possible:
35 * 1) 2-word header with no GC pointers in it (first vtable, second to store the
37 * 2) gc descriptor is the second word in the vtable (first word in the class)
38 * 3) 8 byte alignment is the minimum and enough (not true for special structures, FIXME)
39 * 4) there is a function to get an object's size and the number of
40 * elements in an array.
41 * 5) we know the special way bounds are allocated for complex arrays
43 * Always try to keep stack usage to a minimum: no recursive behaviour
44 * and no large stack allocs.
46 * General description.
47 * Objects are initially allocated in a nursery using a fast bump-pointer technique.
48 * When the nursery is full we start a nursery collection: this is performed with a
50 * When the old generation is full we start a copying GC of the old generation as well:
51 * this will be changed to mark/compact in the future.
52 * The things that complicate this description are:
53 * *) pinned objects: we can't move them so we need to keep track of them
54 * *) no precise info of the thread stacks and registers: we need to be able to
55 * quickly find the objects that may be referenced conservatively and pin them
56 * (this makes the first issues more important)
57 * *) large objects are too expensive to be dealt with using copying GC: we handle them
58 * with mark/sweep during major collections
59 * *) some objects need to not move even if they are small (interned strings, Type handles):
60 * we use mark/sweep for them, too: they are not allocated in the nursery, but inside
61 * PinnedChunks regions
66 *) change the jit to emit write barrier calls when needed (we
67 can have specialized write barriers): done with icalls, still need to
68 use some specialized barriers
69 *) we could have a function pointer in MonoClass to implement
70 customized write barriers for value types
71 *) the write barrier code could be isolated in a couple of functions: when a
72 thread is stopped if it's inside the barrier it is let go again
73 until we stop outside of them (not really needed, see below GC-safe points)
74 *) investigate the stuff needed to advance a thread to a GC-safe
75 point (single-stepping, read from unmapped memory etc) and implement it
76 Not needed yet: since we treat the objects reachable from the stack/regs as
77 roots, we store the ptr and exec the write barrier so there is no race.
78 We may need this to solve the issue with setting the length of arrays and strings.
79 We may need this also for handling precise info on stacks, even simple things
80 as having uninitialized data on the stack and having to wait for the prolog
81 to zero it. Not an issue for the last frame that we scan conservatively.
82 We could always not trust the value in the slots anyway.
83 *) make the jit info table lock free
84 *) modify the jit to save info about references in stack locations:
85 this can be done just for locals as a start, so that at least
86 part of the stack is handled precisely.
87 *) Make the debug printf stuff thread and signal safe.
88 *) test/fix 64 bit issues
89 *) test/fix endianess issues
91 *) add batch moving profile info
92 *) add more timing info
93 *) there is a possible race when an array or string is created: the vtable is set,
94 but the length is set only later so if the GC needs to scan the object in that window,
95 it won't get the correct size for the object. The object can't have references and it will
96 be pinned, but a free memory fragment may be created that overlaps with it.
97 We should change the array max_length field to be at the same offset as the string length:
98 this way we can have a single special alloc function for them that sets the length.
99 Multi-dim arrays have the same issue for rank == 1 for the bounds data.
100 *) implement a card table as the write barrier instead of remembered sets?
101 *) some sort of blacklist support?
102 *) fin_ready_list and critical_fin_list are part of the root set, too
103 *) consider lowering the large object min size to 16/32KB or so and benchmark
104 *) once mark-compact is implemented we could still keep the
105 copying collector for the old generation and use it if we think
106 it is better (small heaps and no pinning object in the old
108 *) avoid the memory store from copy_object when not needed.
109 *) optimize the write barriers fastpath to happen in managed code
110 *) add an option to mmap the whole heap in one chunk: it makes for many
111 simplifications in the checks (put the nursery at the top and just use a single
112 check for inclusion/exclusion): the issue this has is that on 32 bit systems it's
113 not flexible (too much of the address space may be used by default or we can't
114 increase the heap as needed) and we'd need a race-free mechanism to return memory
115 back to the system (mprotect(PROT_NONE) will still keep the memory allocated if it
116 was written to, munmap is needed, but the following mmap may not find the same segment
118 *) memzero the fragments after restarting the world and optionally a smaller chunk at a time
119 *) an additional strategy to realloc/expand the nursery when fully pinned is to start
120 allocating objects in the old generation. This means that we can't optimize away write
121 barrier calls in ctors (but that is not valid for other reasons, too).
122 *) add write barriers to the Clone methods
130 #include <semaphore.h>
134 #include "metadata/metadata-internals.h"
135 #include "metadata/class-internals.h"
136 #include "metadata/gc-internal.h"
137 #include "metadata/object-internals.h"
138 #include "metadata/threads.h"
139 #include "metadata/sgen-gc.h"
140 #include "metadata/sgen-archdep.h"
141 #include "metadata/mono-gc.h"
142 #include "metadata/method-builder.h"
143 #include "metadata/profiler-private.h"
144 #include "metadata/monitor.h"
145 #include "metadata/threadpool-internals.h"
146 #include "metadata/mempool-internals.h"
147 #include "metadata/marshal.h"
148 #include "utils/mono-mmap.h"
149 #include "utils/mono-time.h"
150 #include "utils/mono-semaphore.h"
151 #include "utils/mono-counters.h"
153 #include <mono/utils/memcheck.h>
155 #define OPDEF(a,b,c,d,e,f,g,h,i,j) \
159 #include "mono/cil/opcode.def"
166 * ######################################################################
167 * ######## Types and constants used by the GC.
168 * ######################################################################
170 #if SIZEOF_VOID_P == 4
171 typedef guint32 mword;
173 typedef guint64 mword;
176 static int gc_initialized = 0;
177 static int gc_debug_level = 0;
178 static FILE* gc_debug_file;
179 /* If set, do a minor collection before every allocation */
180 static gboolean collect_before_allocs = FALSE;
181 /* If set, do a heap consistency check before each minor collection */
182 static gboolean consistency_check_at_minor_collection = FALSE;
183 /* If set, check that there are no references to the domain left at domain unload */
184 static gboolean xdomain_checks = FALSE;
185 /* If not null, dump the heap after each collection into this file */
186 static FILE *heap_dump_file = NULL;
187 /* If set, mark stacks conservatively, even if precise marking is possible */
188 static gboolean conservative_stack_mark = FALSE;
189 /* If set, do a plausibility check on the scan_starts before and after
191 static gboolean do_scan_starts_check = FALSE;
194 * Turning on heavy statistics will turn off the managed allocator and
195 * the managed write barrier.
197 //#define HEAVY_STATISTICS
199 #ifdef HEAVY_STATISTICS
200 #define HEAVY_STAT(x) x
202 #define HEAVY_STAT(x)
205 #ifdef HEAVY_STATISTICS
206 static long stat_objects_alloced = 0;
207 static long stat_copy_object_called_nursery = 0;
208 static long stat_objects_copied_nursery = 0;
209 static long stat_copy_object_called_major = 0;
210 static long stat_objects_copied_major = 0;
212 static long stat_copy_object_failed_from_space = 0;
213 static long stat_copy_object_failed_forwarded = 0;
214 static long stat_copy_object_failed_pinned = 0;
215 static long stat_copy_object_failed_large_pinned = 0;
216 static long stat_copy_object_failed_to_space = 0;
218 static long stat_store_remsets = 0;
219 static long stat_store_remsets_unique = 0;
220 static long stat_saved_remsets_1 = 0;
221 static long stat_saved_remsets_2 = 0;
222 static long stat_global_remsets_added = 0;
223 static long stat_global_remsets_processed = 0;
225 static long num_copy_object_called = 0;
226 static long num_objects_copied = 0;
228 static int stat_wbarrier_set_field = 0;
229 static int stat_wbarrier_set_arrayref = 0;
230 static int stat_wbarrier_arrayref_copy = 0;
231 static int stat_wbarrier_generic_store = 0;
232 static int stat_wbarrier_generic_store_remset = 0;
233 static int stat_wbarrier_set_root = 0;
234 static int stat_wbarrier_value_copy = 0;
235 static int stat_wbarrier_object_copy = 0;
238 static long pinned_chunk_bytes_alloced = 0;
239 static long large_internal_bytes_alloced = 0;
242 INTERNAL_MEM_PIN_QUEUE,
243 INTERNAL_MEM_FRAGMENT,
244 INTERNAL_MEM_SECTION,
245 INTERNAL_MEM_SCAN_STARTS,
246 INTERNAL_MEM_FIN_TABLE,
247 INTERNAL_MEM_FINALIZE_ENTRY,
248 INTERNAL_MEM_DISLINK_TABLE,
249 INTERNAL_MEM_DISLINK,
250 INTERNAL_MEM_ROOTS_TABLE,
251 INTERNAL_MEM_ROOT_RECORD,
252 INTERNAL_MEM_STATISTICS,
254 INTERNAL_MEM_GRAY_QUEUE,
255 INTERNAL_MEM_STORE_REMSET,
259 static long small_internal_mem_bytes [INTERNAL_MEM_MAX];
263 mono_gc_flush_info (void)
265 fflush (gc_debug_file);
269 #define MAX_DEBUG_LEVEL 8
270 #define DEBUG(level,a) do {if (G_UNLIKELY ((level) <= MAX_DEBUG_LEVEL && (level) <= gc_debug_level)) a;} while (0)
272 #define TV_DECLARE(name) gint64 name
273 #define TV_GETTIME(tv) tv = mono_100ns_ticks ()
274 #define TV_ELAPSED(start,end) (int)((end-start) / 10)
276 #define ALIGN_TO(val,align) ((((guint64)val) + ((align) - 1)) & ~((align) - 1))
278 #define GC_BITS_PER_WORD (sizeof (mword) * 8)
286 typedef struct _Block Block;
292 /* each request from the OS ends up in a GCMemSection */
293 typedef struct _GCMemSection GCMemSection;
294 struct _GCMemSection {
298 /* pointer where more data could be allocated if it fits */
302 * scan starts is an array of pointers to objects equally spaced in the allocation area
303 * They let use quickly find pinned objects from pinning pointers.
306 /* in major collections indexes in the pin_queue for objects that pin this section */
309 unsigned short num_scan_start;
310 gboolean is_to_space;
313 #define SIZEOF_GC_MEM_SECTION ((sizeof (GCMemSection) + 7) & ~7)
315 /* large object space struct: 64+ KB */
316 /* we could make this limit much smaller to avoid memcpy copy
317 * and potentially have more room in the GC descriptor: need to measure
318 * This also means that such small OS objects will need to be
319 * allocated in a different way (using pinned chunks).
320 * We may want to put large but smaller than 64k objects in the fixed space
321 * when we move the object from one generation to another (to limit the
322 * pig in the snake effect).
323 * Note: it may be worth to have an optimized copy function, since we can
324 * assume that objects are aligned and have a multiple of 8 size.
325 * FIXME: This structure needs to be a multiple of 8 bytes in size: this is not
326 * true if MONO_ZERO_LEN_ARRAY is nonzero.
328 typedef struct _LOSObject LOSObject;
331 mword size; /* this is the object size */
332 int dummy; /* to have a sizeof (LOSObject) a multiple of ALLOC_ALIGN and data starting at same alignment */
335 char data [MONO_ZERO_LEN_ARRAY];
338 /* Pinned objects are allocated in the LOS space if bigger than half a page
339 * or from freelists otherwise. We assume that pinned objects are relatively few
340 * and they have a slow dying speed (like interned strings, thread objects).
341 * As such they will be collected only at major collections.
342 * free lists are not global: when we need memory we allocate a PinnedChunk.
343 * Each pinned chunk is made of several pages, the first of wich is used
344 * internally for bookeeping (here think of a page as 4KB). The bookeeping
345 * includes the freelists vectors and info about the object size of each page
346 * in the pinned chunk. So, when needed, a free page is found in a pinned chunk,
347 * a size is assigned to it, the page is divided in the proper chunks and each
348 * chunk is added to the freelist. To not waste space, the remaining space in the
349 * first page is used as objects of size 16 or 32 (need to measure which are more
351 * We use this same structure to allocate memory used internally by the GC, so
352 * we never use malloc/free if we need to alloc during collection: the world is stopped
353 * and malloc/free will deadlock.
354 * When we want to iterate over pinned objects, we just scan a page at a time
355 * linearly according to the size of objects in the page: the next pointer used to link
356 * the items in the freelist uses the same word as the vtable. Since we keep freelists
357 * for each pinned chunk, if the word points outside the pinned chunk it means
359 * We could avoid this expensive scanning in creative ways. We could have a policy
360 * of putting in the pinned space only objects we know about that have no struct fields
361 * with references and we can easily use a even expensive write barrier for them,
362 * since pointer writes on such objects should be rare.
363 * The best compromise is to just alloc interned strings and System.MonoType in them.
364 * It would be nice to allocate MonoThread in it, too: must check that we properly
365 * use write barriers so we don't have to do any expensive scanning of the whole pinned
366 * chunk list during minor collections. We can avoid it now because we alloc in it only
367 * reference-free objects.
369 #define PINNED_FIRST_SLOT_SIZE (sizeof (gpointer) * 4)
370 #define MAX_FREELIST_SIZE 2048
371 #define PINNED_PAGE_SIZE (4096)
372 #define PINNED_CHUNK_MIN_SIZE (4096*8)
373 typedef struct _PinnedChunk PinnedChunk;
374 struct _PinnedChunk {
377 int *page_sizes; /* a 0 means the page is still unused */
380 void *data [1]; /* page sizes and free lists are stored here */
383 /* The method used to clear the nursery */
384 /* Clearing at nursery collections is the safest, but has bad interactions with caches.
385 * Clearing at TLAB creation is much faster, but more complex and it might expose hard
390 CLEAR_AT_TLAB_CREATION
391 } NurseryClearPolicy;
393 static NurseryClearPolicy nursery_clear_policy = CLEAR_AT_TLAB_CREATION;
396 * If this is set, the nursery is aligned to an address aligned to its size, ie.
397 * a 1MB nursery will be aligned to an address divisible by 1MB. This allows us to
398 * speed up ptr_in_nursery () checks which are very frequent. This requires the
399 * nursery size to be a compile time constant.
401 #define ALIGN_NURSERY 1
404 * The young generation is divided into fragments. This is because
405 * we can hand one fragments to a thread for lock-less fast alloc and
406 * because the young generation ends up fragmented anyway by pinned objects.
407 * Once a collection is done, a list of fragments is created. When doing
408 * thread local alloc we use smallish nurseries so we allow new threads to
409 * allocate memory from gen0 without triggering a collection. Threads that
410 * are found to allocate lots of memory are given bigger fragments. This
411 * should make the finalizer thread use little nursery memory after a while.
412 * We should start assigning threads very small fragments: if there are many
413 * threads the nursery will be full of reserved space that the threads may not
414 * use at all, slowing down allocation speed.
415 * Thread local allocation is done from areas of memory Hotspot calls Thread Local
416 * Allocation Buffers (TLABs).
418 typedef struct _Fragment Fragment;
422 char *fragment_start;
423 char *fragment_limit; /* the current soft limit for allocation */
427 /* the runtime can register areas of memory as roots: we keep two lists of roots,
428 * a pinned root set for conservatively scanned roots and a normal one for
429 * precisely scanned roots (currently implemented as a single list).
431 typedef struct _RootRecord RootRecord;
439 /* for use with write barriers */
440 typedef struct _RememberedSet RememberedSet;
441 struct _RememberedSet {
445 mword data [MONO_ZERO_LEN_ARRAY];
449 * We're never actually using the first element. It's always set to
450 * NULL to simplify the elimination of consecutive duplicate
453 #define STORE_REMSET_BUFFER_SIZE 1024
455 typedef struct _GenericStoreRememberedSet GenericStoreRememberedSet;
456 struct _GenericStoreRememberedSet {
457 GenericStoreRememberedSet *next;
458 /* We need one entry less because the first entry of store
459 remset buffers is always a dummy and we don't copy it. */
460 gpointer data [STORE_REMSET_BUFFER_SIZE - 1];
463 /* we have 4 possible values in the low 2 bits */
465 REMSET_LOCATION, /* just a pointer to the exact location */
466 REMSET_RANGE, /* range of pointer fields */
467 REMSET_OBJECT, /* mark all the object for scanning */
468 REMSET_OTHER, /* all others */
469 REMSET_TYPE_MASK = 0x3
472 /* Subtypes of REMSET_OTHER */
474 REMSET_VTYPE, /* a valuetype array described by a gc descriptor and a count */
475 REMSET_ROOT_LOCATION, /* a location inside a root */
478 #ifdef HAVE_KW_THREAD
479 static __thread RememberedSet *remembered_set MONO_TLS_FAST;
481 static pthread_key_t remembered_set_key;
482 static RememberedSet *global_remset;
483 static RememberedSet *freed_thread_remsets;
484 //static int store_to_global_remset = 0;
485 static GenericStoreRememberedSet *generic_store_remsets = NULL;
487 /* FIXME: later choose a size that takes into account the RememberedSet struct
488 * and doesn't waste any alloc paddin space.
490 #define DEFAULT_REMSET_SIZE 1024
491 static RememberedSet* alloc_remset (int size, gpointer id);
493 /* Structure that corresponds to a MonoVTable: desc is a mword so requires
494 * no cast from a pointer to an integer
501 /* these bits are set in the object vtable: we could merge them since an object can be
502 * either pinned or forwarded but not both.
503 * We store them in the vtable slot because the bits are used in the sync block for
504 * other purposes: if we merge them and alloc the sync blocks aligned to 8 bytes, we can change
505 * this and use bit 3 in the syncblock (with the lower two bits both set for forwarded, that
506 * would be an invalid combination for the monitor and hash code).
507 * The values are already shifted.
508 * The forwarding address is stored in the sync block.
510 #define FORWARDED_BIT 1
512 #define VTABLE_BITS_MASK 0x3
514 /* returns NULL if not forwarded, or the forwarded address */
515 #define object_is_forwarded(obj) (((mword*)(obj))[0] & FORWARDED_BIT? (void*)(((mword*)(obj))[1]): NULL)
516 /* set the forwarded address fw_addr for object obj */
517 #define forward_object(obj,fw_addr) do { \
518 ((mword*)(obj))[0] |= FORWARDED_BIT; \
519 ((mword*)(obj))[1] = (mword)(fw_addr); \
522 #define object_is_pinned(obj) (((mword*)(obj))[0] & PINNED_BIT)
523 #define pin_object(obj) do { \
524 ((mword*)(obj))[0] |= PINNED_BIT; \
526 #define unpin_object(obj) do { \
527 ((mword*)(obj))[0] &= ~PINNED_BIT; \
531 #define ptr_in_nursery(ptr) (((mword)(ptr) & ~((1 << DEFAULT_NURSERY_BITS) - 1)) == (mword)nursery_start)
533 #define ptr_in_nursery(ptr) ((char*)(ptr) >= nursery_start && (char*)(ptr) < nursery_real_end)
537 * Since we set bits in the vtable, use the macro to load it from the pointer to
538 * an object that is potentially pinned.
540 #define LOAD_VTABLE(addr) ((*(mword*)(addr)) & ~VTABLE_BITS_MASK)
543 safe_name (void* obj)
545 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (obj);
546 return vt->klass->name;
550 safe_object_get_size (MonoObject* o)
552 MonoClass *klass = ((MonoVTable*)LOAD_VTABLE (o))->klass;
553 if (klass == mono_defaults.string_class) {
554 return sizeof (MonoString) + 2 * mono_string_length ((MonoString*) o) + 2;
555 } else if (klass->rank) {
556 MonoArray *array = (MonoArray*)o;
557 size_t size = sizeof (MonoArray) + mono_array_element_size (klass) * mono_array_length (array);
558 if (G_UNLIKELY (array->bounds)) {
559 size += sizeof (mono_array_size_t) - 1;
560 size &= ~(sizeof (mono_array_size_t) - 1);
561 size += sizeof (MonoArrayBounds) * klass->rank;
565 /* from a created object: the class must be inited already */
566 return klass->instance_size;
571 * ######################################################################
572 * ######## Global data.
573 * ######################################################################
575 static LOCK_DECLARE (gc_mutex);
576 static int gc_disabled = 0;
577 static int num_minor_gcs = 0;
578 static int num_major_gcs = 0;
580 /* good sizes are 512KB-1MB: larger ones increase a lot memzeroing time */
581 //#define DEFAULT_NURSERY_SIZE (1024*512*125+4096*118)
582 #define DEFAULT_NURSERY_SIZE (1024*512*2)
583 /* The number of trailing 0 bits in DEFAULT_NURSERY_SIZE */
584 #define DEFAULT_NURSERY_BITS 20
585 #define MAJOR_SECTION_SIZE (128*1024)
586 #define BLOCK_FOR_OBJECT(o) ((Block*)(((mword)(o)) & ~(MAJOR_SECTION_SIZE - 1)))
587 #define MAJOR_SECTION_FOR_OBJECT(o) ((GCMemSection*)BLOCK_FOR_OBJECT ((o)))
588 #define MIN_MINOR_COLLECTION_SECTION_ALLOWANCE (DEFAULT_NURSERY_SIZE * 3 / MAJOR_SECTION_SIZE)
589 #define DEFAULT_LOS_COLLECTION_TARGET (DEFAULT_NURSERY_SIZE * 2)
590 /* to quickly find the head of an object pinned by a conservative address
591 * we keep track of the objects allocated for each SCAN_START_SIZE memory
592 * chunk in the nursery or other memory sections. Larger values have less
593 * memory overhead and bigger runtime cost. 4-8 KB are reasonable values.
595 #define SCAN_START_SIZE (4096*2)
596 /* the minimum size of a fragment that we consider useful for allocation */
597 #define FRAGMENT_MIN_SIZE (512)
598 /* This is a fixed value used for pinned chunks, not the system pagesize */
599 #define FREELIST_PAGESIZE 4096
601 static mword pagesize = 4096;
602 static mword nursery_size = DEFAULT_NURSERY_SIZE;
603 static int degraded_mode = 0;
605 static int minor_collection_section_allowance = MIN_MINOR_COLLECTION_SECTION_ALLOWANCE;
606 static int minor_collection_sections_alloced = 0;
607 static int num_major_sections = 0;
609 static LOSObject *los_object_list = NULL;
610 static mword los_memory_usage = 0;
611 static mword los_num_objects = 0;
612 static mword next_los_collection = 2*1024*1024; /* 2 MB, need to tune */
613 static mword total_alloc = 0;
614 /* use this to tune when to do a major/minor collection */
615 static mword memory_pressure = 0;
617 static GCMemSection *section_list = NULL;
618 static GCMemSection *nursery_section = NULL;
619 static mword lowest_heap_address = ~(mword)0;
620 static mword highest_heap_address = 0;
622 typedef struct _FinalizeEntry FinalizeEntry;
623 struct _FinalizeEntry {
628 typedef struct _FinalizeEntryHashTable FinalizeEntryHashTable;
629 struct _FinalizeEntryHashTable {
630 FinalizeEntry **table;
635 typedef struct _DisappearingLink DisappearingLink;
636 struct _DisappearingLink {
637 DisappearingLink *next;
641 typedef struct _DisappearingLinkHashTable DisappearingLinkHashTable;
642 struct _DisappearingLinkHashTable {
643 DisappearingLink **table;
648 #define LARGE_INTERNAL_MEM_HEADER_MAGIC 0x7d289f3a
650 typedef struct _LargeInternalMemHeader LargeInternalMemHeader;
651 struct _LargeInternalMemHeader {
664 * The link pointer is hidden by negating each bit. We use the lowest
665 * bit of the link (before negation) to store whether it needs
666 * resurrection tracking.
668 #define HIDE_POINTER(p,t) ((gpointer)(~((gulong)(p)|((t)?1:0))))
669 #define REVEAL_POINTER(p) ((gpointer)((~(gulong)(p))&~3L))
671 #define DISLINK_OBJECT(d) (REVEAL_POINTER (*(d)->link))
672 #define DISLINK_TRACK(d) ((~(gulong)(*(d)->link)) & 1)
675 * The finalizable hash has the object as the key, the
676 * disappearing_link hash, has the link address as key.
678 static FinalizeEntryHashTable minor_finalizable_hash;
679 static FinalizeEntryHashTable major_finalizable_hash;
680 /* objects that are ready to be finalized */
681 static FinalizeEntry *fin_ready_list = NULL;
682 static FinalizeEntry *critical_fin_list = NULL;
684 static DisappearingLinkHashTable minor_disappearing_link_hash;
685 static DisappearingLinkHashTable major_disappearing_link_hash;
687 static int num_ready_finalizers = 0;
688 static int no_finalize = 0;
690 /* keep each size a multiple of ALLOC_ALIGN */
691 /* on 64 bit systems 8 is likely completely unused. */
692 static const int freelist_sizes [] = {
693 8, 16, 24, 32, 40, 48, 64, 80,
694 96, 128, 160, 192, 224, 256, 320, 384,
695 448, 512, 584, 680, 816, 1024, 1360, 2048};
696 #define FREELIST_NUM_SLOTS (sizeof (freelist_sizes) / sizeof (freelist_sizes [0]))
698 static char* max_pinned_chunk_addr = NULL;
699 static char* min_pinned_chunk_addr = (char*)-1;
700 /* pinned_chunk_list is used for allocations of objects that are never moved */
701 static PinnedChunk *pinned_chunk_list = NULL;
702 /* internal_chunk_list is used for allocating structures needed by the GC */
703 static PinnedChunk *internal_chunk_list = NULL;
706 obj_is_from_pinned_alloc (char *p)
708 return BLOCK_FOR_OBJECT (p)->role == MEMORY_ROLE_PINNED;
711 static int slot_for_size (size_t size);
714 free_pinned_object (PinnedChunk *chunk, char *obj, size_t size)
716 void **p = (void**)obj;
717 int slot = slot_for_size (size);
719 g_assert (obj >= (char*)chunk->start_data && obj < ((char*)chunk + chunk->num_pages * FREELIST_PAGESIZE));
720 *p = chunk->free_list [slot];
721 chunk->free_list [slot] = p;
725 ROOT_TYPE_NORMAL = 0, /* "normal" roots */
726 ROOT_TYPE_PINNED = 1, /* roots without a GC descriptor */
727 ROOT_TYPE_WBARRIER = 2, /* roots with a write barrier */
731 /* registered roots: the key to the hash is the root start address */
733 * Different kinds of roots are kept separate to speed up pin_from_roots () for example.
735 static RootRecord **roots_hash [ROOT_TYPE_NUM] = { NULL, NULL };
736 static int roots_hash_size [ROOT_TYPE_NUM] = { 0, 0, 0 };
737 static mword roots_size = 0; /* amount of memory in the root set */
738 static int num_roots_entries [ROOT_TYPE_NUM] = { 0, 0, 0 };
741 * The current allocation cursors
742 * We allocate objects in the nursery.
743 * The nursery is the area between nursery_start and nursery_real_end.
744 * Allocation is done from a Thread Local Allocation Buffer (TLAB). TLABs are allocated
745 * from nursery fragments.
746 * tlab_next is the pointer to the space inside the TLAB where the next object will
748 * tlab_temp_end is the pointer to the end of the temporary space reserved for
749 * the allocation: it allows us to set the scan starts at reasonable intervals.
750 * tlab_real_end points to the end of the TLAB.
751 * nursery_frag_real_end points to the end of the currently used nursery fragment.
752 * nursery_first_pinned_start points to the start of the first pinned object in the nursery
753 * nursery_last_pinned_end points to the end of the last pinned object in the nursery
754 * At the next allocation, the area of the nursery where objects can be present is
755 * between MIN(nursery_first_pinned_start, first_fragment_start) and
756 * MAX(nursery_last_pinned_end, nursery_frag_real_end)
758 static char *nursery_start = NULL;
760 /* eventually share with MonoThread? */
761 typedef struct _SgenThreadInfo SgenThreadInfo;
763 struct _SgenThreadInfo {
764 SgenThreadInfo *next;
766 unsigned int stop_count; /* to catch duplicate signals */
769 volatile int in_critical_region;
772 void *stack_start_limit;
773 char **tlab_next_addr;
774 char **tlab_start_addr;
775 char **tlab_temp_end_addr;
776 char **tlab_real_end_addr;
777 gpointer **store_remset_buffer_addr;
778 long *store_remset_buffer_index_addr;
779 RememberedSet *remset;
780 gpointer runtime_data;
781 gpointer stopped_ip; /* only valid if the thread is stopped */
782 MonoDomain *stopped_domain; /* ditto */
783 gpointer *stopped_regs; /* ditto */
784 #ifndef HAVE_KW_THREAD
789 gpointer *store_remset_buffer;
790 long store_remset_buffer_index;
794 #ifdef HAVE_KW_THREAD
795 #define TLAB_ACCESS_INIT
796 #define TLAB_START tlab_start
797 #define TLAB_NEXT tlab_next
798 #define TLAB_TEMP_END tlab_temp_end
799 #define TLAB_REAL_END tlab_real_end
800 #define REMEMBERED_SET remembered_set
801 #define STORE_REMSET_BUFFER store_remset_buffer
802 #define STORE_REMSET_BUFFER_INDEX store_remset_buffer_index
803 #define IN_CRITICAL_REGION thread_info->in_critical_region
805 static pthread_key_t thread_info_key;
806 #define TLAB_ACCESS_INIT SgenThreadInfo *__thread_info__ = pthread_getspecific (thread_info_key)
807 #define TLAB_START (__thread_info__->tlab_start)
808 #define TLAB_NEXT (__thread_info__->tlab_next)
809 #define TLAB_TEMP_END (__thread_info__->tlab_temp_end)
810 #define TLAB_REAL_END (__thread_info__->tlab_real_end)
811 #define REMEMBERED_SET (__thread_info__->remset)
812 #define STORE_REMSET_BUFFER (__thread_info__->store_remset_buffer)
813 #define STORE_REMSET_BUFFER_INDEX (__thread_info__->store_remset_buffer_index)
814 #define IN_CRITICAL_REGION (__thread_info__->in_critical_region)
817 /* we use the memory barrier only to prevent compiler reordering (a memory constraint may be enough) */
818 #define ENTER_CRITICAL_REGION do {IN_CRITICAL_REGION = 1;mono_memory_barrier ();} while (0)
819 #define EXIT_CRITICAL_REGION do {IN_CRITICAL_REGION = 0;mono_memory_barrier ();} while (0)
822 * FIXME: What is faster, a TLS variable pointing to a structure, or separate TLS
823 * variables for next+temp_end ?
825 #ifdef HAVE_KW_THREAD
826 static __thread SgenThreadInfo *thread_info;
827 static __thread char *tlab_start;
828 static __thread char *tlab_next;
829 static __thread char *tlab_temp_end;
830 static __thread char *tlab_real_end;
831 static __thread gpointer *store_remset_buffer;
832 static __thread long store_remset_buffer_index;
833 /* Used by the managed allocator/wbarrier */
834 static __thread char **tlab_next_addr;
835 static __thread char *stack_end;
836 static __thread long *store_remset_buffer_index_addr;
838 static char *nursery_next = NULL;
839 static char *nursery_frag_real_end = NULL;
840 static char *nursery_real_end = NULL;
841 //static char *nursery_first_pinned_start = NULL;
842 static char *nursery_last_pinned_end = NULL;
844 /* The size of a TLAB */
845 /* The bigger the value, the less often we have to go to the slow path to allocate a new
846 * one, but the more space is wasted by threads not allocating much memory.
848 * FIXME: Make this self-tuning for each thread.
850 static guint32 tlab_size = (1024 * 4);
852 /* fragments that are free and ready to be used for allocation */
853 static Fragment *nursery_fragments = NULL;
854 /* freeelist of fragment structures */
855 static Fragment *fragment_freelist = NULL;
858 * used when moving the objects
860 static char *to_space_bumper = NULL;
861 static char *to_space_top = NULL;
862 static GCMemSection *to_space_section = NULL;
864 /* objects bigger then this go into the large object space */
865 #define MAX_SMALL_OBJ_SIZE MAX_FREELIST_SIZE
867 /* Functions supplied by the runtime to be called by the GC */
868 static MonoGCCallbacks gc_callbacks;
871 * ######################################################################
872 * ######## Macros and function declarations.
873 * ######################################################################
876 #define UPDATE_HEAP_BOUNDARIES(low,high) do { \
877 if ((mword)(low) < lowest_heap_address) \
878 lowest_heap_address = (mword)(low); \
879 if ((mword)(high) > highest_heap_address) \
880 highest_heap_address = (mword)(high); \
882 #define ADDR_IN_HEAP_BOUNDARIES(addr) ((p) >= lowest_heap_address && (p) < highest_heap_address)
885 align_pointer (void *ptr)
887 mword p = (mword)ptr;
888 p += sizeof (gpointer) - 1;
889 p &= ~ (sizeof (gpointer) - 1);
893 /* forward declarations */
894 static void* get_internal_mem (size_t size, int type);
895 static void free_internal_mem (void *addr, int type);
896 static void* get_os_memory (size_t size, int activate);
897 static void free_os_memory (void *addr, size_t size);
898 static G_GNUC_UNUSED void report_internal_mem_usage (void);
900 static int stop_world (void);
901 static int restart_world (void);
902 static void scan_thread_data (void *start_nursery, void *end_nursery, gboolean precise);
903 static void scan_from_remsets (void *start_nursery, void *end_nursery);
904 static void find_pinning_ref_from_thread (char *obj, size_t size);
905 static void update_current_thread_stack (void *start);
906 static GCMemSection* alloc_major_section (void);
907 static void finalize_in_range (char *start, char *end, int generation);
908 static void add_or_remove_disappearing_link (MonoObject *obj, void **link, gboolean track, int generation);
909 static void null_link_in_range (char *start, char *end, int generation);
910 static void null_links_for_domain (MonoDomain *domain, int generation);
911 static gboolean search_fragment_for_size (size_t size);
912 static void mark_pinned_from_addresses (PinnedChunk *chunk, void **start, void **end);
913 static void clear_remsets (void);
914 static void clear_tlabs (void);
915 typedef void (*ScanPinnedObjectCallbackFunc) (PinnedChunk*, char*, size_t, void*);
916 static void scan_pinned_objects (ScanPinnedObjectCallbackFunc callback, void *callback_data);
917 static void sweep_pinned_objects (void);
918 static void scan_from_pinned_objects (char *addr_start, char *addr_end);
919 static void free_large_object (LOSObject *obj);
920 static void free_major_section (GCMemSection *section);
921 static void to_space_expand (void);
923 static void mono_gc_register_disappearing_link (MonoObject *obj, void **link, gboolean track);
925 void describe_ptr (char *ptr);
926 void check_consistency (void);
927 char* check_object (char *start);
929 void mono_gc_scan_for_specific_ref (MonoObject *key);
932 * ######################################################################
933 * ######## GC descriptors
934 * ######################################################################
935 * Used to quickly get the info the GC needs about an object: size and
936 * where the references are held.
938 /* objects are aligned to 8 bytes boundaries
939 * A descriptor is a pointer in MonoVTable, so 32 or 64 bits of size.
940 * The low 3 bits define the type of the descriptor. The other bits
941 * depend on the type.
942 * As a general rule the 13 remaining low bits define the size, either
943 * of the whole object or of the elements in the arrays. While for objects
944 * the size is already in bytes, for arrays we need to shift, because
945 * array elements might be smaller than 8 bytes. In case of arrays, we
946 * use two bits to describe what the additional high bits represents,
947 * so the default behaviour can handle element sizes less than 2048 bytes.
948 * The high 16 bits, if 0 it means the object is pointer-free.
949 * This design should make it easy and fast to skip over ptr-free data.
950 * The first 4 types should cover >95% of the objects.
951 * Note that since the size of objects is limited to 64K, larger objects
952 * will be allocated in the large object heap.
953 * If we want 4-bytes alignment, we need to put vector and small bitmap
957 DESC_TYPE_RUN_LENGTH, /* 16 bits aligned byte size | 1-3 (offset, numptr) bytes tuples */
958 DESC_TYPE_SMALL_BITMAP, /* 16 bits aligned byte size | 16-48 bit bitmap */
959 DESC_TYPE_STRING, /* nothing */
960 DESC_TYPE_COMPLEX, /* index for bitmap into complex_descriptors */
961 DESC_TYPE_VECTOR, /* 10 bits element size | 1 bit array | 2 bits desc | element desc */
962 DESC_TYPE_ARRAY, /* 10 bits element size | 1 bit array | 2 bits desc | element desc */
963 DESC_TYPE_LARGE_BITMAP, /* | 29-61 bitmap bits */
964 DESC_TYPE_COMPLEX_ARR, /* index for bitmap into complex_descriptors */
965 /* subtypes for arrays and vectors */
966 DESC_TYPE_V_PTRFREE = 0,/* there are no refs: keep first so it has a zero value */
967 DESC_TYPE_V_REFS, /* all the array elements are refs */
968 DESC_TYPE_V_RUN_LEN, /* elements are run-length encoded as DESC_TYPE_RUN_LENGTH */
969 DESC_TYPE_V_BITMAP /* elements are as the bitmap in DESC_TYPE_SMALL_BITMAP */
972 #define OBJECT_HEADER_WORDS (sizeof(MonoObject)/sizeof(gpointer))
973 #define LOW_TYPE_BITS 3
974 #define SMALL_BITMAP_SHIFT 16
975 #define SMALL_BITMAP_SIZE (GC_BITS_PER_WORD - SMALL_BITMAP_SHIFT)
976 #define VECTOR_INFO_SHIFT 14
977 #define VECTOR_ELSIZE_SHIFT 3
978 #define LARGE_BITMAP_SIZE (GC_BITS_PER_WORD - LOW_TYPE_BITS)
979 #define MAX_SMALL_SIZE ((1 << SMALL_BITMAP_SHIFT) - 1)
980 #define SMALL_SIZE_MASK 0xfff8
981 #define MAX_ELEMENT_SIZE 0x3ff
982 #define ELEMENT_SIZE_MASK (0x3ff << LOW_TYPE_BITS)
983 #define VECTOR_SUBTYPE_PTRFREE (DESC_TYPE_V_PTRFREE << VECTOR_INFO_SHIFT)
984 #define VECTOR_SUBTYPE_REFS (DESC_TYPE_V_REFS << VECTOR_INFO_SHIFT)
985 #define VECTOR_SUBTYPE_RUN_LEN (DESC_TYPE_V_RUN_LEN << VECTOR_INFO_SHIFT)
986 #define VECTOR_SUBTYPE_BITMAP (DESC_TYPE_V_BITMAP << VECTOR_INFO_SHIFT)
988 #define ALLOC_ALIGN 8
991 /* Root bitmap descriptors are simpler: the lower three bits describe the type
992 * and we either have 30/62 bitmap bits or nibble-based run-length,
993 * or a complex descriptor, or a user defined marker function.
996 ROOT_DESC_CONSERVATIVE, /* 0, so matches NULL value */
1001 ROOT_DESC_TYPE_MASK = 0x7,
1002 ROOT_DESC_TYPE_SHIFT = 3,
1005 #define MAKE_ROOT_DESC(type,val) ((type) | ((val) << ROOT_DESC_TYPE_SHIFT))
1007 #define MAX_USER_DESCRIPTORS 16
1009 static gsize* complex_descriptors = NULL;
1010 static int complex_descriptors_size = 0;
1011 static int complex_descriptors_next = 0;
1012 static MonoGCMarkFunc user_descriptors [MAX_USER_DESCRIPTORS];
1013 static int user_descriptors_next = 0;
1016 alloc_complex_descriptor (gsize *bitmap, int numbits)
1018 int nwords = numbits/GC_BITS_PER_WORD + 2;
1023 res = complex_descriptors_next;
1024 /* linear search, so we don't have duplicates with domain load/unload
1025 * this should not be performance critical or we'd have bigger issues
1026 * (the number and size of complex descriptors should be small).
1028 for (i = 0; i < complex_descriptors_next; ) {
1029 if (complex_descriptors [i] == nwords) {
1030 int j, found = TRUE;
1031 for (j = 0; j < nwords - 1; ++j) {
1032 if (complex_descriptors [i + 1 + j] != bitmap [j]) {
1042 i += complex_descriptors [i];
1044 if (complex_descriptors_next + nwords > complex_descriptors_size) {
1045 int new_size = complex_descriptors_size * 2 + nwords;
1046 complex_descriptors = g_realloc (complex_descriptors, new_size * sizeof (gsize));
1047 complex_descriptors_size = new_size;
1049 DEBUG (6, fprintf (gc_debug_file, "Complex descriptor %d, size: %d (total desc memory: %d)\n", res, nwords, complex_descriptors_size));
1050 complex_descriptors_next += nwords;
1051 complex_descriptors [res] = nwords;
1052 for (i = 0; i < nwords - 1; ++i) {
1053 complex_descriptors [res + 1 + i] = bitmap [i];
1054 DEBUG (6, fprintf (gc_debug_file, "\tvalue: %p\n", (void*)complex_descriptors [res + 1 + i]));
1061 * Descriptor builders.
1064 mono_gc_make_descr_for_string (gsize *bitmap, int numbits)
1066 return (void*) DESC_TYPE_STRING;
1070 mono_gc_make_descr_for_object (gsize *bitmap, int numbits, size_t obj_size)
1072 int first_set = -1, num_set = 0, last_set = -1, i;
1074 size_t stored_size = obj_size;
1075 stored_size += ALLOC_ALIGN - 1;
1076 stored_size &= ~(ALLOC_ALIGN - 1);
1077 for (i = 0; i < numbits; ++i) {
1078 if (bitmap [i / GC_BITS_PER_WORD] & ((gsize)1 << (i % GC_BITS_PER_WORD))) {
1085 if (stored_size <= MAX_SMALL_OBJ_SIZE) {
1086 /* check run-length encoding first: one byte offset, one byte number of pointers
1087 * on 64 bit archs, we can have 3 runs, just one on 32.
1088 * It may be better to use nibbles.
1090 if (first_set < 0) {
1091 desc = DESC_TYPE_RUN_LENGTH | stored_size;
1092 DEBUG (6, fprintf (gc_debug_file, "Ptrfree descriptor %p, size: %zd\n", (void*)desc, stored_size));
1093 return (void*) desc;
1094 } else if (first_set < 256 && num_set < 256 && (first_set + num_set == last_set + 1)) {
1095 desc = DESC_TYPE_RUN_LENGTH | stored_size | (first_set << 16) | (num_set << 24);
1096 DEBUG (6, fprintf (gc_debug_file, "Runlen descriptor %p, size: %zd, first set: %d, num set: %d\n", (void*)desc, stored_size, first_set, num_set));
1097 return (void*) desc;
1099 /* we know the 2-word header is ptr-free */
1100 if (last_set < SMALL_BITMAP_SIZE + OBJECT_HEADER_WORDS) {
1101 desc = DESC_TYPE_SMALL_BITMAP | stored_size | ((*bitmap >> OBJECT_HEADER_WORDS) << SMALL_BITMAP_SHIFT);
1102 DEBUG (6, fprintf (gc_debug_file, "Smallbitmap descriptor %p, size: %zd, last set: %d\n", (void*)desc, stored_size, last_set));
1103 return (void*) desc;
1106 /* we know the 2-word header is ptr-free */
1107 if (last_set < LARGE_BITMAP_SIZE + OBJECT_HEADER_WORDS) {
1108 desc = DESC_TYPE_LARGE_BITMAP | ((*bitmap >> OBJECT_HEADER_WORDS) << LOW_TYPE_BITS);
1109 DEBUG (6, fprintf (gc_debug_file, "Largebitmap descriptor %p, size: %zd, last set: %d\n", (void*)desc, stored_size, last_set));
1110 return (void*) desc;
1112 /* it's a complex object ... */
1113 desc = DESC_TYPE_COMPLEX | (alloc_complex_descriptor (bitmap, last_set + 1) << LOW_TYPE_BITS);
1114 return (void*) desc;
1117 /* If the array holds references, numbits == 1 and the first bit is set in elem_bitmap */
1119 mono_gc_make_descr_for_array (int vector, gsize *elem_bitmap, int numbits, size_t elem_size)
1121 int first_set = -1, num_set = 0, last_set = -1, i;
1122 mword desc = vector? DESC_TYPE_VECTOR: DESC_TYPE_ARRAY;
1123 for (i = 0; i < numbits; ++i) {
1124 if (elem_bitmap [i / GC_BITS_PER_WORD] & ((gsize)1 << (i % GC_BITS_PER_WORD))) {
1131 if (elem_size <= MAX_ELEMENT_SIZE) {
1132 desc |= elem_size << VECTOR_ELSIZE_SHIFT;
1134 return (void*)(desc | VECTOR_SUBTYPE_PTRFREE);
1136 /* Note: we also handle structs with just ref fields */
1137 if (num_set * sizeof (gpointer) == elem_size) {
1138 return (void*)(desc | VECTOR_SUBTYPE_REFS | ((gssize)(-1) << 16));
1140 /* FIXME: try run-len first */
1141 /* Note: we can't skip the object header here, because it's not present */
1142 if (last_set <= SMALL_BITMAP_SIZE) {
1143 return (void*)(desc | VECTOR_SUBTYPE_BITMAP | (*elem_bitmap << 16));
1146 /* it's am array of complex structs ... */
1147 desc = DESC_TYPE_COMPLEX_ARR;
1148 desc |= alloc_complex_descriptor (elem_bitmap, last_set + 1) << LOW_TYPE_BITS;
1149 return (void*) desc;
1152 /* Return the bitmap encoded by a descriptor */
1154 mono_gc_get_bitmap_for_descr (void *descr, int *numbits)
1156 mword d = (mword)descr;
1160 case DESC_TYPE_RUN_LENGTH: {
1161 int first_set = (d >> 16) & 0xff;
1162 int num_set = (d >> 24) & 0xff;
1165 bitmap = g_new0 (gsize, (first_set + num_set + 7) / 8);
1167 for (i = first_set; i < first_set + num_set; ++i)
1168 bitmap [i / GC_BITS_PER_WORD] |= ((gsize)1 << (i % GC_BITS_PER_WORD));
1170 *numbits = first_set + num_set;
1174 case DESC_TYPE_SMALL_BITMAP:
1175 bitmap = g_new0 (gsize, 1);
1177 bitmap [0] = (d >> SMALL_BITMAP_SHIFT) << OBJECT_HEADER_WORDS;
1179 *numbits = GC_BITS_PER_WORD;
1183 g_assert_not_reached ();
1187 /* helper macros to scan and traverse objects, macros because we resue them in many functions */
1188 #define STRING_SIZE(size,str) do { \
1189 (size) = sizeof (MonoString) + 2 * (mono_string_length ((MonoString*)(str)) + 1); \
1190 (size) += (ALLOC_ALIGN - 1); \
1191 (size) &= ~(ALLOC_ALIGN - 1); \
1194 #define OBJ_RUN_LEN_SIZE(size,desc,obj) do { \
1195 (size) = (desc) & 0xfff8; \
1198 #define OBJ_BITMAP_SIZE(size,desc,obj) do { \
1199 (size) = (desc) & 0xfff8; \
1202 //#define PREFETCH(addr) __asm__ __volatile__ (" prefetchnta %0": : "m"(*(char *)(addr)))
1203 #define PREFETCH(addr)
1205 /* code using these macros must define a HANDLE_PTR(ptr) macro that does the work */
1206 #define OBJ_RUN_LEN_FOREACH_PTR(desc,obj) do { \
1207 if ((desc) & 0xffff0000) { \
1208 /* there are pointers */ \
1209 void **_objptr_end; \
1210 void **_objptr = (void**)(obj); \
1211 _objptr += ((desc) >> 16) & 0xff; \
1212 _objptr_end = _objptr + (((desc) >> 24) & 0xff); \
1213 while (_objptr < _objptr_end) { \
1214 HANDLE_PTR (_objptr, (obj)); \
1220 /* a bitmap desc means that there are pointer references or we'd have
1221 * choosen run-length, instead: add an assert to check.
1223 #define OBJ_BITMAP_FOREACH_PTR(desc,obj) do { \
1224 /* there are pointers */ \
1225 void **_objptr = (void**)(obj); \
1226 gsize _bmap = (desc) >> 16; \
1227 _objptr += OBJECT_HEADER_WORDS; \
1229 if ((_bmap & 1)) { \
1230 HANDLE_PTR (_objptr, (obj)); \
1237 #define OBJ_LARGE_BITMAP_FOREACH_PTR(vt,obj) do { \
1238 /* there are pointers */ \
1239 void **_objptr = (void**)(obj); \
1240 gsize _bmap = (vt)->desc >> LOW_TYPE_BITS; \
1241 _objptr += OBJECT_HEADER_WORDS; \
1243 if ((_bmap & 1)) { \
1244 HANDLE_PTR (_objptr, (obj)); \
1251 #define OBJ_COMPLEX_FOREACH_PTR(vt,obj) do { \
1252 /* there are pointers */ \
1253 void **_objptr = (void**)(obj); \
1254 gsize *bitmap_data = complex_descriptors + ((vt)->desc >> LOW_TYPE_BITS); \
1255 int bwords = (*bitmap_data) - 1; \
1256 void **start_run = _objptr; \
1259 MonoObject *myobj = (MonoObject*)obj; \
1260 g_print ("found %d at %p (0x%zx): %s.%s\n", bwords, (obj), (vt)->desc, myobj->vtable->klass->name_space, myobj->vtable->klass->name); \
1262 while (bwords-- > 0) { \
1263 gsize _bmap = *bitmap_data++; \
1264 _objptr = start_run; \
1265 /*g_print ("bitmap: 0x%x/%d at %p\n", _bmap, bwords, _objptr);*/ \
1267 if ((_bmap & 1)) { \
1268 HANDLE_PTR (_objptr, (obj)); \
1273 start_run += GC_BITS_PER_WORD; \
1277 /* this one is untested */
1278 #define OBJ_COMPLEX_ARR_FOREACH_PTR(vt,obj) do { \
1279 /* there are pointers */ \
1280 gsize *mbitmap_data = complex_descriptors + ((vt)->desc >> LOW_TYPE_BITS); \
1281 int mbwords = (*mbitmap_data++) - 1; \
1282 int el_size = mono_array_element_size (((MonoObject*)(obj))->vtable->klass); \
1283 char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
1284 char *e_end = e_start + el_size * mono_array_length ((MonoArray*)(obj)); \
1286 MonoObject *myobj = (MonoObject*)start; \
1287 g_print ("found %d at %p (0x%zx): %s.%s\n", mbwords, (obj), (vt)->desc, myobj->vtable->klass->name_space, myobj->vtable->klass->name); \
1289 while (e_start < e_end) { \
1290 void **_objptr = (void**)e_start; \
1291 gsize *bitmap_data = mbitmap_data; \
1292 unsigned int bwords = mbwords; \
1293 while (bwords-- > 0) { \
1294 gsize _bmap = *bitmap_data++; \
1295 void **start_run = _objptr; \
1296 /*g_print ("bitmap: 0x%x\n", _bmap);*/ \
1298 if ((_bmap & 1)) { \
1299 HANDLE_PTR (_objptr, (obj)); \
1304 _objptr = start_run + GC_BITS_PER_WORD; \
1306 e_start += el_size; \
1310 #define OBJ_VECTOR_FOREACH_PTR(vt,obj) do { \
1311 /* note: 0xffffc000 excludes DESC_TYPE_V_PTRFREE */ \
1312 if ((vt)->desc & 0xffffc000) { \
1313 int el_size = ((vt)->desc >> 3) & MAX_ELEMENT_SIZE; \
1314 /* there are pointers */ \
1315 int etype = (vt)->desc & 0xc000; \
1316 if (etype == (DESC_TYPE_V_REFS << 14)) { \
1317 void **p = (void**)((char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector)); \
1318 void **end_refs = (void**)((char*)p + el_size * mono_array_length ((MonoArray*)(obj))); \
1319 /* Note: this code can handle also arrays of struct with only references in them */ \
1320 while (p < end_refs) { \
1321 HANDLE_PTR (p, (obj)); \
1324 } else if (etype == DESC_TYPE_V_RUN_LEN << 14) { \
1325 int offset = ((vt)->desc >> 16) & 0xff; \
1326 int num_refs = ((vt)->desc >> 24) & 0xff; \
1327 char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
1328 char *e_end = e_start + el_size * mono_array_length ((MonoArray*)(obj)); \
1329 while (e_start < e_end) { \
1330 void **p = (void**)e_start; \
1333 for (i = 0; i < num_refs; ++i) { \
1334 HANDLE_PTR (p + i, (obj)); \
1336 e_start += el_size; \
1338 } else if (etype == DESC_TYPE_V_BITMAP << 14) { \
1339 char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
1340 char *e_end = e_start + el_size * mono_array_length ((MonoArray*)(obj)); \
1341 while (e_start < e_end) { \
1342 void **p = (void**)e_start; \
1343 gsize _bmap = (vt)->desc >> 16; \
1344 /* Note: there is no object header here to skip */ \
1346 if ((_bmap & 1)) { \
1347 HANDLE_PTR (p, (obj)); \
1352 e_start += el_size; \
1358 #define COUNT_OBJECT_TYPES do { \
1359 switch (desc & 0x7) { \
1360 case DESC_TYPE_STRING: type_str++; break; \
1361 case DESC_TYPE_RUN_LENGTH: type_rlen++; break; \
1362 case DESC_TYPE_ARRAY: case DESC_TYPE_VECTOR: type_vector++; break; \
1363 case DESC_TYPE_SMALL_BITMAP: type_bitmap++; break; \
1364 case DESC_TYPE_LARGE_BITMAP: type_lbit++; break; \
1365 case DESC_TYPE_COMPLEX: type_complex++; break; \
1366 case DESC_TYPE_COMPLEX_ARR: type_complex++; break; \
1367 default: g_assert_not_reached (); \
1373 * ######################################################################
1374 * ######## Detecting and removing garbage.
1375 * ######################################################################
1376 * This section of code deals with detecting the objects no longer in use
1377 * and reclaiming the memory.
1381 static mword new_obj_references = 0;
1382 static mword obj_references_checked = 0;
1385 #define HANDLE_PTR(ptr,obj) do { \
1386 if (*(ptr) && (char*)*(ptr) >= nursery_start && (char*)*(ptr) < nursery_next) { \
1387 new_obj_references++; \
1388 /*printf ("bogus ptr %p found at %p in object %p (%s.%s)\n", *(ptr), (ptr), o, o->vtable->klass->name_space, o->vtable->klass->name);*/ \
1390 obj_references_checked++; \
1394 static void __attribute__((noinline))
1395 scan_area (char *start, char *end)
1398 int type_str = 0, type_rlen = 0, type_bitmap = 0, type_vector = 0, type_lbit = 0, type_complex = 0;
1399 new_obj_references = 0;
1400 obj_references_checked = 0;
1401 while (start < end) {
1402 if (!*(void**)start) {
1403 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
1406 vt = (GCVTable*)LOAD_VTABLE (start);
1407 DEBUG (8, fprintf (gc_debug_file, "Scanning object %p, vtable: %p (%s)\n", start, vt, vt->klass->name));
1409 MonoObject *obj = (MonoObject*)start;
1410 g_print ("found at %p (0x%zx): %s.%s\n", start, vt->desc, obj->vtable->klass->name_space, obj->vtable->klass->name);
1413 #define SCAN_OBJECT_ACTION COUNT_OBJECT_TYPES
1414 #include "sgen-scan-object.h"
1416 /*printf ("references to new nursery %p-%p (size: %dk): %d, checked: %d\n", old_start, end, (end-old_start)/1024, new_obj_references, obj_references_checked);
1417 printf ("\tstrings: %d, runl: %d, vector: %d, bitmaps: %d, lbitmaps: %d, complex: %d\n",
1418 type_str, type_rlen, type_vector, type_bitmap, type_lbit, type_complex);*/
1423 is_xdomain_ref_allowed (gpointer *ptr, char *obj, MonoDomain *domain)
1425 MonoObject *o = (MonoObject*)(obj);
1426 MonoObject *ref = (MonoObject*)*(ptr);
1427 int offset = (char*)(ptr) - (char*)o;
1429 if (o->vtable->klass == mono_defaults.thread_class && offset == G_STRUCT_OFFSET (MonoThread, internal_thread))
1431 if (o->vtable->klass == mono_defaults.internal_thread_class && offset == G_STRUCT_OFFSET (MonoInternalThread, current_appcontext))
1433 if (mono_class_has_parent (o->vtable->klass, mono_defaults.real_proxy_class) &&
1434 offset == G_STRUCT_OFFSET (MonoRealProxy, unwrapped_server))
1436 /* Thread.cached_culture_info */
1437 if (!strcmp (ref->vtable->klass->name_space, "System.Globalization") &&
1438 !strcmp (ref->vtable->klass->name, "CultureInfo") &&
1439 !strcmp(o->vtable->klass->name_space, "System") &&
1440 !strcmp(o->vtable->klass->name, "Object[]"))
1443 * at System.IO.MemoryStream.InternalConstructor (byte[],int,int,bool,bool) [0x0004d] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.IO/MemoryStream.cs:121
1444 * at System.IO.MemoryStream..ctor (byte[]) [0x00017] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.IO/MemoryStream.cs:81
1445 * at (wrapper remoting-invoke-with-check) System.IO.MemoryStream..ctor (byte[]) <IL 0x00020, 0xffffffff>
1446 * at System.Runtime.Remoting.Messaging.CADMethodCallMessage.GetArguments () [0x0000d] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Messaging/CADMessages.cs:327
1447 * at System.Runtime.Remoting.Messaging.MethodCall..ctor (System.Runtime.Remoting.Messaging.CADMethodCallMessage) [0x00017] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Messaging/MethodCall.cs:87
1448 * at System.AppDomain.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage,byte[]&,System.Runtime.Remoting.Messaging.CADMethodReturnMessage&) [0x00018] in /home/schani/Work/novell/trunk/mcs/class/corlib/System/AppDomain.cs:1213
1449 * at (wrapper remoting-invoke-with-check) System.AppDomain.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage,byte[]&,System.Runtime.Remoting.Messaging.CADMethodReturnMessage&) <IL 0x0003d, 0xffffffff>
1450 * at System.Runtime.Remoting.Channels.CrossAppDomainSink.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage) [0x00008] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Channels/CrossAppDomainChannel.cs:198
1451 * at (wrapper runtime-invoke) object.runtime_invoke_CrossAppDomainSink/ProcessMessageRes_object_object (object,intptr,intptr,intptr) <IL 0x0004c, 0xffffffff>
1453 if (!strcmp (ref->vtable->klass->name_space, "System") &&
1454 !strcmp (ref->vtable->klass->name, "Byte[]") &&
1455 !strcmp (o->vtable->klass->name_space, "System.IO") &&
1456 !strcmp (o->vtable->klass->name, "MemoryStream"))
1458 /* append_job() in threadpool.c */
1459 if (!strcmp (ref->vtable->klass->name_space, "System.Runtime.Remoting.Messaging") &&
1460 !strcmp (ref->vtable->klass->name, "AsyncResult") &&
1461 !strcmp (o->vtable->klass->name_space, "System") &&
1462 !strcmp (o->vtable->klass->name, "Object[]") &&
1463 mono_thread_pool_is_queue_array ((MonoArray*) o))
1469 check_reference_for_xdomain (gpointer *ptr, char *obj, MonoDomain *domain)
1471 MonoObject *o = (MonoObject*)(obj);
1472 MonoObject *ref = (MonoObject*)*(ptr);
1473 int offset = (char*)(ptr) - (char*)o;
1475 MonoClassField *field;
1478 if (!ref || ref->vtable->domain == domain)
1480 if (is_xdomain_ref_allowed (ptr, obj, domain))
1484 for (class = o->vtable->klass; class; class = class->parent) {
1487 for (i = 0; i < class->field.count; ++i) {
1488 if (class->fields[i].offset == offset) {
1489 field = &class->fields[i];
1497 if (ref->vtable->klass == mono_defaults.string_class)
1498 str = mono_string_to_utf8 ((MonoString*)ref);
1501 g_print ("xdomain reference in %p (%s.%s) at offset %d (%s) to %p (%s.%s) (%s) - pointed to by:\n",
1502 o, o->vtable->klass->name_space, o->vtable->klass->name,
1503 offset, field ? field->name : "",
1504 ref, ref->vtable->klass->name_space, ref->vtable->klass->name, str ? str : "");
1505 mono_gc_scan_for_specific_ref (o);
1511 #define HANDLE_PTR(ptr,obj) check_reference_for_xdomain ((ptr), (obj), domain)
1514 scan_object_for_xdomain_refs (char *start)
1516 MonoDomain *domain = ((MonoObject*)start)->vtable->domain;
1518 #include "sgen-scan-object.h"
1524 scan_area_for_xdomain_refs (char *start, char *end)
1526 while (start < end) {
1527 if (!*(void**)start) {
1528 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
1532 start = scan_object_for_xdomain_refs (start);
1537 #define HANDLE_PTR(ptr,obj) do { \
1538 if ((MonoObject*)*(ptr) == key) { \
1539 g_print ("found ref to %p in object %p (%s) at offset %zd\n", \
1540 key, (obj), safe_name ((obj)), ((char*)(ptr) - (char*)(obj))); \
1545 scan_object_for_specific_ref (char *start, MonoObject *key)
1547 #include "sgen-scan-object.h"
1553 scan_area_for_specific_ref (char *start, char *end, MonoObject *key)
1555 while (start < end) {
1556 if (!*(void**)start) {
1557 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
1561 start = scan_object_for_specific_ref (start, key);
1566 scan_pinned_object_for_specific_ref_callback (PinnedChunk *chunk, char *obj, size_t size, MonoObject *key)
1568 scan_object_for_specific_ref (obj, key);
1572 check_root_obj_specific_ref (RootRecord *root, MonoObject *key, MonoObject *obj)
1576 g_print ("found ref to %p in root record %p\n", key, root);
1579 static MonoObject *check_key = NULL;
1580 static RootRecord *check_root = NULL;
1583 check_root_obj_specific_ref_from_marker (void *obj)
1585 check_root_obj_specific_ref (check_root, check_key, obj);
1590 scan_roots_for_specific_ref (MonoObject *key, int root_type)
1595 for (i = 0; i < roots_hash_size [root_type]; ++i) {
1596 for (root = roots_hash [root_type][i]; root; root = root->next) {
1597 void **start_root = (void**)root->start_root;
1598 mword desc = root->root_desc;
1602 switch (desc & ROOT_DESC_TYPE_MASK) {
1603 case ROOT_DESC_BITMAP:
1604 desc >>= ROOT_DESC_TYPE_SHIFT;
1607 check_root_obj_specific_ref (root, key, *start_root);
1612 case ROOT_DESC_COMPLEX: {
1613 gsize *bitmap_data = complex_descriptors + (desc >> ROOT_DESC_TYPE_SHIFT);
1614 int bwords = (*bitmap_data) - 1;
1615 void **start_run = start_root;
1617 while (bwords-- > 0) {
1618 gsize bmap = *bitmap_data++;
1619 void **objptr = start_run;
1622 check_root_obj_specific_ref (root, key, *objptr);
1626 start_run += GC_BITS_PER_WORD;
1630 case ROOT_DESC_USER: {
1631 MonoGCMarkFunc marker = user_descriptors [desc >> ROOT_DESC_TYPE_SHIFT];
1632 marker (start_root, check_root_obj_specific_ref_from_marker);
1635 case ROOT_DESC_RUN_LEN:
1636 g_assert_not_reached ();
1638 g_assert_not_reached ();
1647 mono_gc_scan_for_specific_ref (MonoObject *key)
1649 GCMemSection *section;
1654 for (section = section_list; section; section = section->block.next)
1655 scan_area_for_specific_ref (section->data, section->end_data, key);
1657 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1658 scan_object_for_specific_ref (bigobj->data, key);
1660 scan_pinned_objects ((ScanPinnedObjectCallbackFunc)scan_pinned_object_for_specific_ref_callback, key);
1662 scan_roots_for_specific_ref (key, ROOT_TYPE_NORMAL);
1663 scan_roots_for_specific_ref (key, ROOT_TYPE_WBARRIER);
1665 for (i = 0; i < roots_hash_size [ROOT_TYPE_PINNED]; ++i) {
1666 for (root = roots_hash [ROOT_TYPE_PINNED][i]; root; root = root->next) {
1667 void **ptr = (void**)root->start_root;
1669 while (ptr < (void**)root->end_root) {
1670 check_root_obj_specific_ref (root, *ptr, key);
1678 need_remove_object_for_domain (char *start, MonoDomain *domain)
1680 if (mono_object_domain (start) == domain) {
1681 DEBUG (1, fprintf (gc_debug_file, "Need to cleanup object %p\n", start));
1688 process_object_for_domain_clearing (char *start, MonoDomain *domain)
1690 GCVTable *vt = (GCVTable*)LOAD_VTABLE (start);
1691 if (vt->klass == mono_defaults.internal_thread_class)
1692 g_assert (mono_object_domain (start) == mono_get_root_domain ());
1693 /* The object could be a proxy for an object in the domain
1695 if (mono_class_has_parent (vt->klass, mono_defaults.real_proxy_class)) {
1696 MonoObject *server = ((MonoRealProxy*)start)->unwrapped_server;
1698 /* The server could already have been zeroed out, so
1699 we need to check for that, too. */
1700 if (server && (!LOAD_VTABLE (server) || mono_object_domain (server) == domain)) {
1701 DEBUG (1, fprintf (gc_debug_file, "Cleaning up remote pointer in %p to object %p\n",
1703 ((MonoRealProxy*)start)->unwrapped_server = NULL;
1708 static void __attribute__((noinline))
1709 scan_area_for_domain (MonoDomain *domain, char *start, char *end)
1714 while (start < end) {
1715 if (!*(void**)start) {
1716 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
1719 vt = (GCVTable*)LOAD_VTABLE (start);
1720 process_object_for_domain_clearing (start, domain);
1721 remove = need_remove_object_for_domain (start, domain);
1722 if (remove && ((MonoObject*)start)->synchronisation) {
1723 void **dislink = mono_monitor_get_object_monitor_weak_link ((MonoObject*)start);
1725 mono_gc_register_disappearing_link (NULL, dislink, FALSE);
1728 #define SCAN_OBJECT_NOSCAN
1729 #define SCAN_OBJECT_ACTION do { \
1730 if (remove) memset (start, 0, skip_size); \
1732 #include "sgen-scan-object.h"
1736 static MonoDomain *check_domain = NULL;
1739 check_obj_not_in_domain (void *o)
1741 g_assert (((MonoObject*)o)->vtable->domain != check_domain);
1746 scan_for_registered_roots_in_domain (MonoDomain *domain, int root_type)
1750 check_domain = domain;
1751 for (i = 0; i < roots_hash_size [root_type]; ++i) {
1752 for (root = roots_hash [root_type][i]; root; root = root->next) {
1753 void **start_root = (void**)root->start_root;
1754 mword desc = root->root_desc;
1756 /* The MonoDomain struct is allowed to hold
1757 references to objects in its own domain. */
1758 if (start_root == (void**)domain)
1761 switch (desc & ROOT_DESC_TYPE_MASK) {
1762 case ROOT_DESC_BITMAP:
1763 desc >>= ROOT_DESC_TYPE_SHIFT;
1765 if ((desc & 1) && *start_root)
1766 check_obj_not_in_domain (*start_root);
1771 case ROOT_DESC_COMPLEX: {
1772 gsize *bitmap_data = complex_descriptors + (desc >> ROOT_DESC_TYPE_SHIFT);
1773 int bwords = (*bitmap_data) - 1;
1774 void **start_run = start_root;
1776 while (bwords-- > 0) {
1777 gsize bmap = *bitmap_data++;
1778 void **objptr = start_run;
1780 if ((bmap & 1) && *objptr)
1781 check_obj_not_in_domain (*objptr);
1785 start_run += GC_BITS_PER_WORD;
1789 case ROOT_DESC_USER: {
1790 MonoGCMarkFunc marker = user_descriptors [desc >> ROOT_DESC_TYPE_SHIFT];
1791 marker (start_root, check_obj_not_in_domain);
1794 case ROOT_DESC_RUN_LEN:
1795 g_assert_not_reached ();
1797 g_assert_not_reached ();
1801 check_domain = NULL;
1805 clear_domain_process_pinned_object_callback (PinnedChunk *chunk, char *obj, size_t size, MonoDomain *domain)
1807 process_object_for_domain_clearing (obj, domain);
1811 clear_domain_free_pinned_object_callback (PinnedChunk *chunk, char *obj, size_t size, MonoDomain *domain)
1813 if (need_remove_object_for_domain (obj, domain))
1814 free_pinned_object (chunk, obj, size);
1818 scan_pinned_object_for_xdomain_refs_callback (PinnedChunk *chunk, char *obj, size_t size, gpointer dummy)
1820 scan_object_for_xdomain_refs (obj);
1824 check_for_xdomain_refs (void)
1826 GCMemSection *section;
1829 for (section = section_list; section; section = section->block.next)
1830 scan_area_for_xdomain_refs (section->data, section->end_data);
1832 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1833 scan_object_for_xdomain_refs (bigobj->data);
1835 scan_pinned_objects (scan_pinned_object_for_xdomain_refs_callback, NULL);
1839 * When appdomains are unloaded we can easily remove objects that have finalizers,
1840 * but all the others could still be present in random places on the heap.
1841 * We need a sweep to get rid of them even though it's going to be costly
1843 * The reason we need to remove them is because we access the vtable and class
1844 * structures to know the object size and the reference bitmap: once the domain is
1845 * unloaded the point to random memory.
1848 mono_gc_clear_domain (MonoDomain * domain)
1850 GCMemSection *section;
1851 LOSObject *bigobj, *prev;
1856 /* Clear all remaining nursery fragments */
1857 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION) {
1858 g_assert (nursery_next <= nursery_frag_real_end);
1859 memset (nursery_next, 0, nursery_frag_real_end - nursery_next);
1860 for (frag = nursery_fragments; frag; frag = frag->next) {
1861 memset (frag->fragment_start, 0, frag->fragment_end - frag->fragment_start);
1865 if (xdomain_checks && domain != mono_get_root_domain ()) {
1866 scan_for_registered_roots_in_domain (domain, ROOT_TYPE_NORMAL);
1867 scan_for_registered_roots_in_domain (domain, ROOT_TYPE_WBARRIER);
1868 check_for_xdomain_refs ();
1871 for (section = section_list; section; section = section->block.next) {
1872 scan_area_for_domain (domain, section->data, section->end_data);
1875 /* We need two passes over pinned and large objects because
1876 freeing such an object gives its memory back to the OS (in
1877 the case of large objects) or obliterates its vtable
1878 (pinned objects), but we might need to dereference a
1879 pointer from an object to another object if the first
1880 object is a proxy. */
1881 scan_pinned_objects ((ScanPinnedObjectCallbackFunc)clear_domain_process_pinned_object_callback, domain);
1882 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1883 process_object_for_domain_clearing (bigobj->data, domain);
1886 for (bigobj = los_object_list; bigobj;) {
1887 if (need_remove_object_for_domain (bigobj->data, domain)) {
1888 LOSObject *to_free = bigobj;
1890 prev->next = bigobj->next;
1892 los_object_list = bigobj->next;
1893 bigobj = bigobj->next;
1894 DEBUG (1, fprintf (gc_debug_file, "Freeing large object %p\n",
1896 free_large_object (to_free);
1900 bigobj = bigobj->next;
1902 scan_pinned_objects ((ScanPinnedObjectCallbackFunc)clear_domain_free_pinned_object_callback, domain);
1904 for (i = GENERATION_NURSERY; i < GENERATION_MAX; ++i)
1905 null_links_for_domain (domain, i);
1911 * add_to_global_remset:
1913 * The global remset contains locations which point into newspace after
1914 * a minor collection. This can happen if the objects they point to are pinned.
1917 add_to_global_remset (gpointer ptr, gboolean root)
1921 DEBUG (8, fprintf (gc_debug_file, "Adding global remset for %p\n", ptr));
1924 g_assert (!ptr_in_nursery (ptr) && ptr_in_nursery (*(gpointer*)ptr));
1926 HEAVY_STAT (++stat_global_remsets_added);
1929 * FIXME: If an object remains pinned, we need to add it at every minor collection.
1930 * To avoid uncontrolled growth of the global remset, only add each pointer once.
1932 if (global_remset->store_next + 3 < global_remset->end_set) {
1934 *(global_remset->store_next++) = (mword)ptr | REMSET_OTHER;
1935 *(global_remset->store_next++) = (mword)REMSET_ROOT_LOCATION;
1937 *(global_remset->store_next++) = (mword)ptr;
1941 rs = alloc_remset (global_remset->end_set - global_remset->data, NULL);
1942 rs->next = global_remset;
1945 *(global_remset->store_next++) = (mword)ptr | REMSET_OTHER;
1946 *(global_remset->store_next++) = (mword)REMSET_ROOT_LOCATION;
1948 *(global_remset->store_next++) = (mword)ptr;
1952 int global_rs_size = 0;
1954 for (rs = global_remset; rs; rs = rs->next) {
1955 global_rs_size += rs->store_next - rs->data;
1957 DEBUG (4, fprintf (gc_debug_file, "Global remset now has size %d\n", global_rs_size));
1961 #include "sgen-gray.c"
1964 * This is how the copying happens from the nursery to the old generation.
1965 * We assume that at this time all the pinned objects have been identified and
1967 * We run scan_object() for each pinned object so that each referenced
1968 * objects if possible are copied. The new gray objects created can have
1969 * scan_object() run on them right away, too.
1970 * Then we run copy_object() for the precisely tracked roots. At this point
1971 * all the roots are either gray or black. We run scan_object() on the gray
1972 * objects until no more gray objects are created.
1973 * At the end of the process we walk again the pinned list and we unmark
1974 * the pinned flag. As we go we also create the list of free space for use
1975 * in the next allocation runs.
1977 * We need to remember objects from the old generation that point to the new one
1978 * (or just addresses?).
1980 * copy_object could be made into a macro once debugged (use inline for now).
1983 static char* __attribute__((noinline))
1984 copy_object (char *obj, char *from_space_start, char *from_space_end)
1986 static void *copy_labels [] = { &&LAB_0, &&LAB_1, &&LAB_2, &&LAB_3, &&LAB_4, &&LAB_5, &&LAB_6, &&LAB_7, &&LAB_8 };
1992 HEAVY_STAT (++num_copy_object_called);
1994 if (!(obj >= from_space_start && obj < from_space_end)) {
1995 DEBUG (9, fprintf (gc_debug_file, "Not copying %p because it's not in from space (%p-%p)\n",
1996 obj, from_space_start, from_space_end));
1997 HEAVY_STAT (++stat_copy_object_failed_from_space);
2001 DEBUG (9, fprintf (gc_debug_file, "Precise copy of %p", obj));
2004 * obj must belong to one of:
2009 * 4. a non-to-space section of the major heap
2010 * 5. a to-space section of the major heap
2012 * In addition, objects in 1, 2 and 4 might also be pinned.
2013 * Objects in 1 and 4 might be forwarded.
2015 * Before we can copy the object we must make sure that we are
2016 * allowed to, i.e. that the object not pinned, not already
2017 * forwarded and doesn't belong to the LOS, a pinned chunk, or
2018 * a to-space section.
2020 * We are usually called for to-space objects (5) when we have
2021 * two remset entries for the same reference. The first entry
2022 * copies the object and updates the reference and the second
2023 * calls us with the updated reference that points into
2024 * to-space. There might also be other circumstances where we
2025 * get to-space objects.
2028 if ((forwarded = object_is_forwarded (obj))) {
2029 g_assert (((MonoVTable*)LOAD_VTABLE(obj))->gc_descr);
2030 DEBUG (9, fprintf (gc_debug_file, " (already forwarded to %p)\n", forwarded));
2031 HEAVY_STAT (++stat_copy_object_failed_forwarded);
2034 if (object_is_pinned (obj)) {
2035 g_assert (((MonoVTable*)LOAD_VTABLE(obj))->gc_descr);
2036 DEBUG (9, fprintf (gc_debug_file, " (pinned, no change)\n"));
2037 HEAVY_STAT (++stat_copy_object_failed_pinned);
2041 objsize = safe_object_get_size ((MonoObject*)obj);
2042 objsize += ALLOC_ALIGN - 1;
2043 objsize &= ~(ALLOC_ALIGN - 1);
2045 if (ptr_in_nursery (obj))
2049 * At this point we know obj is not pinned, not forwarded and
2050 * belongs to 2, 3, 4, or 5.
2052 * LOS object (2) are simple, at least until we always follow
2053 * the rule: if objsize > MAX_SMALL_OBJ_SIZE, pin the object
2054 * and return it. At the end of major collections, we walk
2055 * the los list and if the object is pinned, it is marked,
2056 * otherwise it can be freed.
2058 * Pinned chunks (3) and major heap sections (4, 5) both
2059 * reside in blocks, which are always aligned, so once we've
2060 * eliminated LOS objects, we can just access the block and
2061 * see whether it's a pinned chunk or a major heap section.
2063 if (G_UNLIKELY (objsize > MAX_SMALL_OBJ_SIZE || obj_is_from_pinned_alloc (obj))) {
2064 DEBUG (9, fprintf (gc_debug_file, " (marked LOS/Pinned %p (%s), size: %zd)\n", obj, safe_name (obj), objsize));
2066 HEAVY_STAT (++stat_copy_object_failed_large_pinned);
2071 * Now we know the object is in a major heap section. All we
2072 * need to do is check whether it's already in to-space (5) or
2075 if (MAJOR_SECTION_FOR_OBJECT (obj)->is_to_space) {
2076 g_assert (objsize <= MAX_SMALL_OBJ_SIZE);
2077 DEBUG (9, fprintf (gc_debug_file, " (already copied)\n"));
2078 HEAVY_STAT (++stat_copy_object_failed_to_space);
2083 DEBUG (9, fprintf (gc_debug_file, " (to %p, %s size: %zd)\n", to_space_bumper, ((MonoObject*)obj)->vtable->klass->name, objsize));
2085 HEAVY_STAT (++num_objects_copied);
2087 /* Make sure we have enough space available */
2088 if (to_space_bumper + objsize > to_space_top) {
2090 g_assert (to_space_bumper + objsize <= to_space_top);
2093 if (objsize <= sizeof (gpointer) * 8) {
2094 mword *dest = (mword*)to_space_bumper;
2095 goto *copy_labels [objsize / sizeof (gpointer)];
2097 (dest) [7] = ((mword*)obj) [7];
2099 (dest) [6] = ((mword*)obj) [6];
2101 (dest) [5] = ((mword*)obj) [5];
2103 (dest) [4] = ((mword*)obj) [4];
2105 (dest) [3] = ((mword*)obj) [3];
2107 (dest) [2] = ((mword*)obj) [2];
2109 (dest) [1] = ((mword*)obj) [1];
2111 (dest) [0] = ((mword*)obj) [0];
2119 char* edi = to_space_bumper;
2120 __asm__ __volatile__(
2122 : "=&c" (ecx), "=&D" (edi), "=&S" (esi)
2123 : "0" (objsize/4), "1" (edi),"2" (esi)
2128 memcpy (to_space_bumper, obj, objsize);
2131 /* adjust array->bounds */
2132 vt = ((MonoObject*)obj)->vtable;
2133 g_assert (vt->gc_descr);
2134 if (G_UNLIKELY (vt->rank && ((MonoArray*)obj)->bounds)) {
2135 MonoArray *array = (MonoArray*)to_space_bumper;
2136 array->bounds = (MonoArrayBounds*)((char*)to_space_bumper + ((char*)((MonoArray*)obj)->bounds - (char*)obj));
2137 DEBUG (9, fprintf (gc_debug_file, "Array instance %p: size: %zd, rank: %d, length: %d\n", array, objsize, vt->rank, mono_array_length (array)));
2139 /* set the forwarding pointer */
2140 forward_object (obj, to_space_bumper);
2141 obj = to_space_bumper;
2142 to_space_section->scan_starts [((char*)obj - (char*)to_space_section->data)/SCAN_START_SIZE] = obj;
2143 to_space_bumper += objsize;
2144 DEBUG (9, fprintf (gc_debug_file, "Enqueuing gray object %p (%s)\n", obj, safe_name (obj)));
2145 gray_object_enqueue (obj);
2146 DEBUG (8, g_assert (to_space_bumper <= to_space_top));
2151 #define HANDLE_PTR(ptr,obj) do { \
2152 void *__old = *(ptr); \
2155 *(ptr) = __copy = copy_object (__old, from_start, from_end); \
2156 DEBUG (9, if (__old != __copy) fprintf (gc_debug_file, "Overwrote field at %p with %p (was: %p)\n", (ptr), *(ptr), __old)); \
2157 if (G_UNLIKELY (ptr_in_nursery (__copy) && !ptr_in_nursery ((ptr)))) \
2158 add_to_global_remset ((ptr), FALSE); \
2163 * Scan the object pointed to by @start for references to
2164 * other objects between @from_start and @from_end and copy
2165 * them to the gray_objects area.
2166 * Returns a pointer to the end of the object.
2169 scan_object (char *start, char* from_start, char* from_end)
2171 #include "sgen-scan-object.h"
2179 * Scan objects in the gray stack until the stack is empty. This should be called
2180 * frequently after each object is copied, to achieve better locality and cache
2184 drain_gray_stack (char *start_addr, char *end_addr)
2188 while ((obj = gray_object_dequeue ())) {
2189 DEBUG (9, fprintf (gc_debug_file, "Precise gray object scan %p (%s)\n", obj, safe_name (obj)));
2190 scan_object (obj, start_addr, end_addr);
2197 * Scan the valuetype pointed to by START, described by DESC for references to
2198 * other objects between @from_start and @from_end and copy them to the gray_objects area.
2199 * Returns a pointer to the end of the object.
2202 scan_vtype (char *start, mword desc, char* from_start, char* from_end)
2206 /* The descriptors include info about the MonoObject header as well */
2207 start -= sizeof (MonoObject);
2209 switch (desc & 0x7) {
2210 case DESC_TYPE_RUN_LENGTH:
2211 OBJ_RUN_LEN_FOREACH_PTR (desc,start);
2212 OBJ_RUN_LEN_SIZE (skip_size, desc, start);
2213 g_assert (skip_size);
2214 return start + skip_size;
2215 case DESC_TYPE_SMALL_BITMAP:
2216 OBJ_BITMAP_FOREACH_PTR (desc,start);
2217 OBJ_BITMAP_SIZE (skip_size, desc, start);
2218 return start + skip_size;
2219 case DESC_TYPE_LARGE_BITMAP:
2220 case DESC_TYPE_COMPLEX:
2222 g_assert_not_reached ();
2225 // The other descriptors can't happen with vtypes
2226 g_assert_not_reached ();
2232 #include "sgen-pinning-stats.c"
2235 * Addresses from start to end are already sorted. This function finds
2236 * the object header for each address and pins the object. The
2237 * addresses must be inside the passed section. The (start of the)
2238 * address array is overwritten with the addresses of the actually
2239 * pinned objects. Return the number of pinned objects.
2242 pin_objects_from_addresses (GCMemSection *section, void **start, void **end, void *start_nursery, void *end_nursery)
2247 void *last_obj = NULL;
2248 size_t last_obj_size = 0;
2251 void **definitely_pinned = start;
2252 while (start < end) {
2254 /* the range check should be reduntant */
2255 if (addr != last && addr >= start_nursery && addr < end_nursery) {
2256 DEBUG (5, fprintf (gc_debug_file, "Considering pinning addr %p\n", addr));
2257 /* multiple pointers to the same object */
2258 if (addr >= last_obj && (char*)addr < (char*)last_obj + last_obj_size) {
2262 idx = ((char*)addr - (char*)section->data) / SCAN_START_SIZE;
2263 g_assert (idx < section->num_scan_start);
2264 search_start = (void*)section->scan_starts [idx];
2265 if (!search_start || search_start > addr) {
2268 search_start = section->scan_starts [idx];
2269 if (search_start && search_start <= addr)
2272 if (!search_start || search_start > addr)
2273 search_start = start_nursery;
2275 if (search_start < last_obj)
2276 search_start = (char*)last_obj + last_obj_size;
2277 /* now addr should be in an object a short distance from search_start
2278 * Note that search_start must point to zeroed mem or point to an object.
2281 if (!*(void**)search_start) {
2282 mword p = (mword)search_start;
2283 p += sizeof (gpointer);
2284 p += ALLOC_ALIGN - 1;
2285 p &= ~(ALLOC_ALIGN - 1);
2286 search_start = (void*)p;
2289 last_obj = search_start;
2290 last_obj_size = safe_object_get_size ((MonoObject*)search_start);
2291 last_obj_size += ALLOC_ALIGN - 1;
2292 last_obj_size &= ~(ALLOC_ALIGN - 1);
2293 DEBUG (8, fprintf (gc_debug_file, "Pinned try match %p (%s), size %zd\n", last_obj, safe_name (last_obj), last_obj_size));
2294 if (addr >= search_start && (char*)addr < (char*)last_obj + last_obj_size) {
2295 DEBUG (4, fprintf (gc_debug_file, "Pinned object %p, vtable %p (%s), count %d\n", search_start, *(void**)search_start, safe_name (search_start), count));
2296 pin_object (search_start);
2298 pin_stats_register_object (search_start, last_obj_size);
2299 definitely_pinned [count] = search_start;
2303 /* skip to the next object */
2304 search_start = (void*)((char*)search_start + last_obj_size);
2305 } while (search_start <= addr);
2306 /* we either pinned the correct object or we ignored the addr because
2307 * it points to unused zeroed memory.
2313 //printf ("effective pinned: %d (at the end: %d)\n", count, (char*)end_nursery - (char*)last);
2317 static void** pin_queue;
2318 static int pin_queue_size = 0;
2319 static int next_pin_slot = 0;
2324 gap = (gap * 10) / 13;
2325 if (gap == 9 || gap == 10)
2334 compare_addr (const void *a, const void *b)
2336 return *(const void **)a - *(const void **)b;
2340 /* sort the addresses in array in increasing order */
2342 sort_addresses (void **array, int size)
2345 * qsort is slower as predicted.
2346 * qsort (array, size, sizeof (gpointer), compare_addr);
2353 gap = new_gap (gap);
2356 for (i = 0; i < end; i++) {
2358 if (array [i] > array [j]) {
2359 void* val = array [i];
2360 array [i] = array [j];
2365 if (gap == 1 && !swapped)
2370 static G_GNUC_UNUSED void
2371 print_nursery_gaps (void* start_nursery, void *end_nursery)
2374 gpointer first = start_nursery;
2376 for (i = 0; i < next_pin_slot; ++i) {
2377 next = pin_queue [i];
2378 fprintf (gc_debug_file, "Nursery range: %p-%p, size: %zd\n", first, next, (char*)next-(char*)first);
2382 fprintf (gc_debug_file, "Nursery range: %p-%p, size: %zd\n", first, next, (char*)next-(char*)first);
2385 /* reduce the info in the pin queue, removing duplicate pointers and sorting them */
2387 optimize_pin_queue (int start_slot)
2389 void **start, **cur, **end;
2390 /* sort and uniq pin_queue: we just sort and we let the rest discard multiple values */
2391 /* it may be better to keep ranges of pinned memory instead of individually pinning objects */
2392 DEBUG (5, fprintf (gc_debug_file, "Sorting pin queue, size: %d\n", next_pin_slot));
2393 if ((next_pin_slot - start_slot) > 1)
2394 sort_addresses (pin_queue + start_slot, next_pin_slot - start_slot);
2395 start = cur = pin_queue + start_slot;
2396 end = pin_queue + next_pin_slot;
2399 while (*start == *cur && cur < end)
2403 next_pin_slot = start - pin_queue;
2404 DEBUG (5, fprintf (gc_debug_file, "Pin queue reduced to size: %d\n", next_pin_slot));
2405 //DEBUG (6, print_nursery_gaps (start_nursery, end_nursery));
2410 optimized_pin_queue_search (void *addr)
2412 int first = 0, last = next_pin_slot;
2413 while (first < last) {
2414 int middle = first + ((last - first) >> 1);
2415 if (addr <= pin_queue [middle])
2420 g_assert (first == last);
2425 find_optimized_pin_queue_area (void *start, void *end, int *first, int *last)
2427 *first = optimized_pin_queue_search (start);
2428 *last = optimized_pin_queue_search (end);
2432 realloc_pin_queue (void)
2434 int new_size = pin_queue_size? pin_queue_size + pin_queue_size/2: 1024;
2435 void **new_pin = get_internal_mem (sizeof (void*) * new_size, INTERNAL_MEM_PIN_QUEUE);
2436 memcpy (new_pin, pin_queue, sizeof (void*) * next_pin_slot);
2437 free_internal_mem (pin_queue, INTERNAL_MEM_PIN_QUEUE);
2438 pin_queue = new_pin;
2439 pin_queue_size = new_size;
2440 DEBUG (4, fprintf (gc_debug_file, "Reallocated pin queue to size: %d\n", new_size));
2443 #include "sgen-pinning.c"
2446 * Scan the memory between start and end and queue values which could be pointers
2447 * to the area between start_nursery and end_nursery for later consideration.
2448 * Typically used for thread stacks.
2451 conservatively_pin_objects_from (void **start, void **end, void *start_nursery, void *end_nursery, int pin_type)
2454 while (start < end) {
2455 if (*start >= start_nursery && *start < end_nursery) {
2457 * *start can point to the middle of an object
2458 * note: should we handle pointing at the end of an object?
2459 * pinning in C# code disallows pointing at the end of an object
2460 * but there is some small chance that an optimizing C compiler
2461 * may keep the only reference to an object by pointing
2462 * at the end of it. We ignore this small chance for now.
2463 * Pointers to the end of an object are indistinguishable
2464 * from pointers to the start of the next object in memory
2465 * so if we allow that we'd need to pin two objects...
2466 * We queue the pointer in an array, the
2467 * array will then be sorted and uniqued. This way
2468 * we can coalesce several pinning pointers and it should
2469 * be faster since we'd do a memory scan with increasing
2470 * addresses. Note: we can align the address to the allocation
2471 * alignment, so the unique process is more effective.
2473 mword addr = (mword)*start;
2474 addr &= ~(ALLOC_ALIGN - 1);
2475 if (addr >= (mword)start_nursery && addr < (mword)end_nursery)
2476 pin_stage_ptr ((void*)addr);
2478 pin_stats_register_address ((char*)addr, pin_type);
2479 DEBUG (6, if (count) fprintf (gc_debug_file, "Pinning address %p\n", (void*)addr));
2484 DEBUG (7, if (count) fprintf (gc_debug_file, "found %d potential pinned heap pointers\n", count));
2488 * If generation is 0, just mark objects in the nursery, the others we don't care,
2489 * since they are not going to move anyway.
2490 * There are different areas that are scanned for pinned pointers:
2491 * *) the thread stacks (when jit support is ready only the unmanaged frames)
2492 * *) the pinned handle table
2493 * *) the pinned roots
2495 * Note: when we'll use a write barrier for old to new gen references, we need to
2496 * keep track of old gen objects that point to pinned new gen objects because in that
2497 * case the referenced object will be moved maybe at the next collection, but there
2498 * is no write in the old generation area where the pinned object is referenced
2499 * and we may not consider it as reachable.
2501 static G_GNUC_UNUSED void
2502 mark_pinned_objects (int generation)
2507 * Debugging function: find in the conservative roots where @obj is being pinned.
2509 static G_GNUC_UNUSED void
2510 find_pinning_reference (char *obj, size_t size)
2514 char *endobj = obj + size;
2515 for (i = 0; i < roots_hash_size [0]; ++i) {
2516 for (root = roots_hash [0][i]; root; root = root->next) {
2517 /* if desc is non-null it has precise info */
2518 if (!root->root_desc) {
2519 char ** start = (char**)root->start_root;
2520 while (start < (char**)root->end_root) {
2521 if (*start >= obj && *start < endobj) {
2522 DEBUG (0, fprintf (gc_debug_file, "Object %p referenced in pinned roots %p-%p (at %p in record %p)\n", obj, root->start_root, root->end_root, start, root));
2529 find_pinning_ref_from_thread (obj, size);
2533 * The first thing we do in a collection is to identify pinned objects.
2534 * This function considers all the areas of memory that need to be
2535 * conservatively scanned.
2538 pin_from_roots (void *start_nursery, void *end_nursery)
2542 DEBUG (2, fprintf (gc_debug_file, "Scanning pinned roots (%d bytes, %d/%d entries)\n", (int)roots_size, num_roots_entries [ROOT_TYPE_NORMAL], num_roots_entries [ROOT_TYPE_PINNED]));
2543 /* objects pinned from the API are inside these roots */
2544 for (i = 0; i < roots_hash_size [ROOT_TYPE_PINNED]; ++i) {
2545 for (root = roots_hash [ROOT_TYPE_PINNED][i]; root; root = root->next) {
2546 DEBUG (6, fprintf (gc_debug_file, "Pinned roots %p-%p\n", root->start_root, root->end_root));
2547 conservatively_pin_objects_from ((void**)root->start_root, (void**)root->end_root, start_nursery, end_nursery, PIN_TYPE_OTHER);
2550 /* now deal with the thread stacks
2551 * in the future we should be able to conservatively scan only:
2552 * *) the cpu registers
2553 * *) the unmanaged stack frames
2554 * *) the _last_ managed stack frame
2555 * *) pointers slots in managed frames
2557 scan_thread_data (start_nursery, end_nursery, FALSE);
2559 evacuate_pin_staging_area ();
2562 /* Copy function called from user defined mark functions */
2563 static char *user_copy_n_start;
2564 static char *user_copy_n_end;
2567 user_copy (void *addr)
2570 return copy_object (addr, user_copy_n_start, user_copy_n_end);
2576 * The memory area from start_root to end_root contains pointers to objects.
2577 * Their position is precisely described by @desc (this means that the pointer
2578 * can be either NULL or the pointer to the start of an object).
2579 * This functions copies them to to_space updates them.
2582 precisely_scan_objects_from (void** start_root, void** end_root, char* n_start, char *n_end, mword desc)
2584 switch (desc & ROOT_DESC_TYPE_MASK) {
2585 case ROOT_DESC_BITMAP:
2586 desc >>= ROOT_DESC_TYPE_SHIFT;
2588 if ((desc & 1) && *start_root) {
2589 *start_root = copy_object (*start_root, n_start, n_end);
2590 DEBUG (9, fprintf (gc_debug_file, "Overwrote root at %p with %p\n", start_root, *start_root));
2591 drain_gray_stack (n_start, n_end);
2597 case ROOT_DESC_COMPLEX: {
2598 gsize *bitmap_data = complex_descriptors + (desc >> ROOT_DESC_TYPE_SHIFT);
2599 int bwords = (*bitmap_data) - 1;
2600 void **start_run = start_root;
2602 while (bwords-- > 0) {
2603 gsize bmap = *bitmap_data++;
2604 void **objptr = start_run;
2606 if ((bmap & 1) && *objptr) {
2607 *objptr = copy_object (*objptr, n_start, n_end);
2608 DEBUG (9, fprintf (gc_debug_file, "Overwrote root at %p with %p\n", objptr, *objptr));
2609 drain_gray_stack (n_start, n_end);
2614 start_run += GC_BITS_PER_WORD;
2618 case ROOT_DESC_USER: {
2619 MonoGCMarkFunc marker = user_descriptors [desc >> ROOT_DESC_TYPE_SHIFT];
2621 user_copy_n_start = n_start;
2622 user_copy_n_end = n_end;
2623 marker (start_root, user_copy);
2626 case ROOT_DESC_RUN_LEN:
2627 g_assert_not_reached ();
2629 g_assert_not_reached ();
2634 alloc_fragment (void)
2636 Fragment *frag = fragment_freelist;
2638 fragment_freelist = frag->next;
2642 frag = get_internal_mem (sizeof (Fragment), INTERNAL_MEM_FRAGMENT);
2647 /* size must be a power of 2 */
2649 get_os_memory_aligned (mword size, gboolean activate)
2651 /* Allocate twice the memory to be able to put the block on an aligned address */
2652 char *mem = get_os_memory (size * 2, activate);
2657 aligned = (char*)((mword)(mem + (size - 1)) & ~(size - 1));
2658 g_assert (aligned >= mem && aligned + size <= mem + size * 2 && !((mword)aligned & (size - 1)));
2661 free_os_memory (mem, aligned - mem);
2662 if (aligned + size < mem + size * 2)
2663 free_os_memory (aligned + size, (mem + size * 2) - (aligned + size));
2669 * Allocate and setup the data structures needed to be able to allocate objects
2670 * in the nursery. The nursery is stored in nursery_section.
2673 alloc_nursery (void)
2675 GCMemSection *section;
2681 if (nursery_section)
2683 DEBUG (2, fprintf (gc_debug_file, "Allocating nursery size: %zd\n", nursery_size));
2684 /* later we will alloc a larger area for the nursery but only activate
2685 * what we need. The rest will be used as expansion if we have too many pinned
2686 * objects in the existing nursery.
2688 /* FIXME: handle OOM */
2689 section = get_internal_mem (SIZEOF_GC_MEM_SECTION, INTERNAL_MEM_SECTION);
2691 g_assert (nursery_size == DEFAULT_NURSERY_SIZE);
2692 alloc_size = nursery_size;
2693 #ifdef ALIGN_NURSERY
2694 data = get_os_memory_aligned (alloc_size, TRUE);
2696 data = get_os_memory (alloc_size, TRUE);
2698 nursery_start = data;
2699 nursery_real_end = nursery_start + nursery_size;
2700 UPDATE_HEAP_BOUNDARIES (nursery_start, nursery_real_end);
2701 nursery_next = nursery_start;
2702 total_alloc += alloc_size;
2703 DEBUG (4, fprintf (gc_debug_file, "Expanding nursery size (%p-%p): %zd, total: %zd\n", data, data + alloc_size, nursery_size, total_alloc));
2704 section->data = section->next_data = data;
2705 section->size = alloc_size;
2706 section->end_data = nursery_real_end;
2707 scan_starts = (alloc_size + SCAN_START_SIZE - 1) / SCAN_START_SIZE;
2708 section->scan_starts = get_internal_mem (sizeof (char*) * scan_starts, INTERNAL_MEM_SCAN_STARTS);
2709 section->num_scan_start = scan_starts;
2710 section->block.role = MEMORY_ROLE_GEN0;
2712 /* add to the section list */
2713 section->block.next = section_list;
2714 section_list = section;
2716 nursery_section = section;
2718 /* Setup the single first large fragment */
2719 frag = alloc_fragment ();
2720 frag->fragment_start = nursery_start;
2721 frag->fragment_limit = nursery_start;
2722 frag->fragment_end = nursery_real_end;
2723 nursery_frag_real_end = nursery_real_end;
2724 /* FIXME: frag here is lost */
2728 scan_finalizer_entries (FinalizeEntry *list, char *start, char *end) {
2731 for (fin = list; fin; fin = fin->next) {
2734 DEBUG (5, fprintf (gc_debug_file, "Scan of fin ready object: %p (%s)\n", fin->object, safe_name (fin->object)));
2735 fin->object = copy_object (fin->object, start, end);
2740 * Update roots in the old generation. Since we currently don't have the
2741 * info from the write barriers, we just scan all the objects.
2743 static G_GNUC_UNUSED void
2744 scan_old_generation (char *start, char* end)
2746 GCMemSection *section;
2747 LOSObject *big_object;
2750 for (section = section_list; section; section = section->block.next) {
2751 if (section == nursery_section)
2753 DEBUG (2, fprintf (gc_debug_file, "Scan of old section: %p-%p, size: %d\n", section->data, section->next_data, (int)(section->next_data - section->data)));
2754 /* we have to deal with zeroed holes in old generation (truncated strings ...) */
2756 while (p < section->next_data) {
2761 DEBUG (8, fprintf (gc_debug_file, "Precise old object scan of %p (%s)\n", p, safe_name (p)));
2762 p = scan_object (p, start, end);
2765 /* scan the old object space, too */
2766 for (big_object = los_object_list; big_object; big_object = big_object->next) {
2767 DEBUG (5, fprintf (gc_debug_file, "Scan of big object: %p (%s), size: %zd\n", big_object->data, safe_name (big_object->data), big_object->size));
2768 scan_object (big_object->data, start, end);
2770 /* scan the list of objects ready for finalization */
2771 scan_finalizer_entries (fin_ready_list, start, end);
2772 scan_finalizer_entries (critical_fin_list, start, end);
2775 static mword fragment_total = 0;
2777 * We found a fragment of free memory in the nursery: memzero it and if
2778 * it is big enough, add it to the list of fragments that can be used for
2782 add_nursery_frag (size_t frag_size, char* frag_start, char* frag_end)
2785 DEBUG (4, fprintf (gc_debug_file, "Found empty fragment: %p-%p, size: %zd\n", frag_start, frag_end, frag_size));
2786 /* memsetting just the first chunk start is bound to provide better cache locality */
2787 if (nursery_clear_policy == CLEAR_AT_GC)
2788 memset (frag_start, 0, frag_size);
2789 /* Not worth dealing with smaller fragments: need to tune */
2790 if (frag_size >= FRAGMENT_MIN_SIZE) {
2791 fragment = alloc_fragment ();
2792 fragment->fragment_start = frag_start;
2793 fragment->fragment_limit = frag_start;
2794 fragment->fragment_end = frag_end;
2795 fragment->next = nursery_fragments;
2796 nursery_fragments = fragment;
2797 fragment_total += frag_size;
2799 /* Clear unused fragments, pinning depends on this */
2800 memset (frag_start, 0, frag_size);
2805 scan_needed_big_objects (char *start_addr, char *end_addr)
2807 LOSObject *big_object;
2809 for (big_object = los_object_list; big_object; big_object = big_object->next) {
2810 if (!big_object->scanned && object_is_pinned (big_object->data)) {
2811 DEBUG (5, fprintf (gc_debug_file, "Scan of big object: %p (%s), size: %zd\n", big_object->data, safe_name (big_object->data), big_object->size));
2812 scan_object (big_object->data, start_addr, end_addr);
2813 big_object->scanned = TRUE;
2821 generation_name (int generation)
2823 switch (generation) {
2824 case GENERATION_NURSERY: return "nursery";
2825 case GENERATION_OLD: return "old";
2826 default: g_assert_not_reached ();
2830 static DisappearingLinkHashTable*
2831 get_dislink_hash_table (int generation)
2833 switch (generation) {
2834 case GENERATION_NURSERY: return &minor_disappearing_link_hash;
2835 case GENERATION_OLD: return &major_disappearing_link_hash;
2836 default: g_assert_not_reached ();
2840 static FinalizeEntryHashTable*
2841 get_finalize_entry_hash_table (int generation)
2843 switch (generation) {
2844 case GENERATION_NURSERY: return &minor_finalizable_hash;
2845 case GENERATION_OLD: return &major_finalizable_hash;
2846 default: g_assert_not_reached ();
2851 new_to_space_section (void)
2853 /* FIXME: if the current to_space_section is empty, we don't
2854 have to allocate a new one */
2856 to_space_section = alloc_major_section ();
2857 to_space_bumper = to_space_section->next_data;
2858 to_space_top = to_space_section->end_data;
2862 to_space_set_next_data (void)
2864 g_assert (to_space_bumper >= to_space_section->next_data && to_space_bumper <= to_space_section->end_data);
2865 to_space_section->next_data = to_space_bumper;
2869 to_space_expand (void)
2871 if (to_space_section) {
2872 g_assert (to_space_top == to_space_section->end_data);
2873 to_space_set_next_data ();
2876 new_to_space_section ();
2880 unset_to_space (void)
2882 /* between collections the to_space_bumper is invalidated
2883 because degraded allocations might occur, so we set it to
2884 NULL, just to make it explicit */
2885 to_space_bumper = NULL;
2887 /* don't unset to_space_section if we implement the FIXME in
2888 new_to_space_section */
2889 to_space_section = NULL;
2893 object_is_in_to_space (char *obj)
2898 if (ptr_in_nursery (obj))
2901 objsize = safe_object_get_size ((MonoObject*)obj);
2902 objsize += ALLOC_ALIGN - 1;
2903 objsize &= ~(ALLOC_ALIGN - 1);
2906 if (objsize > MAX_SMALL_OBJ_SIZE)
2910 if (obj_is_from_pinned_alloc (obj))
2913 /* now we know it's in a major heap section */
2914 return MAJOR_SECTION_FOR_OBJECT (obj)->is_to_space;
2918 finish_gray_stack (char *start_addr, char *end_addr, int generation)
2922 int fin_ready, bigo_scanned_num;
2925 * We copied all the reachable objects. Now it's the time to copy
2926 * the objects that were not referenced by the roots, but by the copied objects.
2927 * we built a stack of objects pointed to by gray_start: they are
2928 * additional roots and we may add more items as we go.
2929 * We loop until gray_start == gray_objects which means no more objects have
2930 * been added. Note this is iterative: no recursion is involved.
2931 * We need to walk the LO list as well in search of marked big objects
2932 * (use a flag since this is needed only on major collections). We need to loop
2933 * here as well, so keep a counter of marked LO (increasing it in copy_object).
2934 * To achieve better cache locality and cache usage, we drain the gray stack
2935 * frequently, after each object is copied, and just finish the work here.
2937 drain_gray_stack (start_addr, end_addr);
2939 //scan_old_generation (start_addr, end_addr);
2940 DEBUG (2, fprintf (gc_debug_file, "%s generation done\n", generation_name (generation)));
2941 /* walk the finalization queue and move also the objects that need to be
2942 * finalized: use the finalized objects as new roots so the objects they depend
2943 * on are also not reclaimed. As with the roots above, only objects in the nursery
2944 * are marked/copied.
2945 * We need a loop here, since objects ready for finalizers may reference other objects
2946 * that are fin-ready. Speedup with a flag?
2949 fin_ready = num_ready_finalizers;
2950 finalize_in_range (start_addr, end_addr, generation);
2951 if (generation == GENERATION_OLD)
2952 finalize_in_range (nursery_start, nursery_real_end, GENERATION_NURSERY);
2953 bigo_scanned_num = scan_needed_big_objects (start_addr, end_addr);
2955 /* drain the new stack that might have been created */
2956 DEBUG (6, fprintf (gc_debug_file, "Precise scan of gray area post fin\n"));
2957 drain_gray_stack (start_addr, end_addr);
2958 } while (fin_ready != num_ready_finalizers || bigo_scanned_num);
2960 DEBUG (2, fprintf (gc_debug_file, "Finalize queue handling scan for %s generation: %d usecs\n", generation_name (generation), TV_ELAPSED (atv, btv)));
2963 * handle disappearing links
2964 * Note we do this after checking the finalization queue because if an object
2965 * survives (at least long enough to be finalized) we don't clear the link.
2966 * This also deals with a possible issue with the monitor reclamation: with the Boehm
2967 * GC a finalized object my lose the monitor because it is cleared before the finalizer is
2970 g_assert (gray_object_queue_is_empty ());
2972 null_link_in_range (start_addr, end_addr, generation);
2973 if (generation == GENERATION_OLD)
2974 null_link_in_range (start_addr, end_addr, GENERATION_NURSERY);
2975 if (gray_object_queue_is_empty ())
2977 drain_gray_stack (start_addr, end_addr);
2980 g_assert (gray_object_queue_is_empty ());
2981 /* DEBUG (2, fprintf (gc_debug_file, "Copied from %s to old space: %d bytes (%p-%p)\n", generation_name (generation), (int)(to_space_bumper - to_space), to_space, to_space_bumper)); */
2982 to_space_set_next_data ();
2986 check_scan_starts (void)
2988 GCMemSection *section;
2990 if (!do_scan_starts_check)
2992 for (section = section_list; section; section = section->block.next) {
2993 for (i = 0; i < section->num_scan_start; ++i) {
2994 if (section->scan_starts [i]) {
2995 guint size = safe_object_get_size ((MonoObject*) section->scan_starts [i]);
2996 g_assert (size >= sizeof (MonoObject) && size <= MAX_SMALL_OBJ_SIZE);
3002 static int last_num_pinned = 0;
3005 build_nursery_fragments (int start_pin, int end_pin)
3007 char *frag_start, *frag_end;
3011 while (nursery_fragments) {
3012 Fragment *next = nursery_fragments->next;
3013 nursery_fragments->next = fragment_freelist;
3014 fragment_freelist = nursery_fragments;
3015 nursery_fragments = next;
3017 frag_start = nursery_start;
3019 /* clear scan starts */
3020 memset (nursery_section->scan_starts, 0, nursery_section->num_scan_start * sizeof (gpointer));
3021 for (i = start_pin; i < end_pin; ++i) {
3022 frag_end = pin_queue [i];
3023 /* remove the pin bit from pinned objects */
3024 unpin_object (frag_end);
3025 nursery_section->scan_starts [((char*)frag_end - (char*)nursery_section->data)/SCAN_START_SIZE] = frag_end;
3026 frag_size = frag_end - frag_start;
3028 add_nursery_frag (frag_size, frag_start, frag_end);
3029 frag_size = safe_object_get_size ((MonoObject*)pin_queue [i]);
3030 frag_size += ALLOC_ALIGN - 1;
3031 frag_size &= ~(ALLOC_ALIGN - 1);
3032 frag_start = (char*)pin_queue [i] + frag_size;
3034 nursery_last_pinned_end = frag_start;
3035 frag_end = nursery_real_end;
3036 frag_size = frag_end - frag_start;
3038 add_nursery_frag (frag_size, frag_start, frag_end);
3039 if (!nursery_fragments) {
3040 DEBUG (1, fprintf (gc_debug_file, "Nursery fully pinned (%d)\n", end_pin - start_pin));
3041 for (i = start_pin; i < end_pin; ++i) {
3042 DEBUG (3, fprintf (gc_debug_file, "Bastard pinning obj %p (%s), size: %d\n", pin_queue [i], safe_name (pin_queue [i]), safe_object_get_size (pin_queue [i])));
3047 nursery_next = nursery_frag_real_end = NULL;
3049 /* Clear TLABs for all threads */
3053 /* FIXME: later reduce code duplication here with the above
3054 * We don't keep track of section fragments for non-nursery sections yet, so
3058 build_section_fragments (GCMemSection *section)
3061 char *frag_start, *frag_end;
3064 /* clear scan starts */
3065 memset (section->scan_starts, 0, section->num_scan_start * sizeof (gpointer));
3066 frag_start = section->data;
3067 section->next_data = section->data;
3068 for (i = section->pin_queue_start; i < section->pin_queue_end; ++i) {
3069 frag_end = pin_queue [i];
3070 /* remove the pin bit from pinned objects */
3071 unpin_object (frag_end);
3072 if (frag_end >= section->data + section->size) {
3073 frag_end = section->data + section->size;
3075 section->scan_starts [((char*)frag_end - (char*)section->data)/SCAN_START_SIZE] = frag_end;
3077 frag_size = frag_end - frag_start;
3079 memset (frag_start, 0, frag_size);
3080 frag_size = safe_object_get_size ((MonoObject*)pin_queue [i]);
3081 frag_size += ALLOC_ALIGN - 1;
3082 frag_size &= ~(ALLOC_ALIGN - 1);
3083 frag_start = (char*)pin_queue [i] + frag_size;
3084 section->next_data = MAX (section->next_data, frag_start);
3086 frag_end = section->end_data;
3087 frag_size = frag_end - frag_start;
3089 memset (frag_start, 0, frag_size);
3093 scan_from_registered_roots (char *addr_start, char *addr_end, int root_type)
3097 for (i = 0; i < roots_hash_size [root_type]; ++i) {
3098 for (root = roots_hash [root_type][i]; root; root = root->next) {
3099 DEBUG (6, fprintf (gc_debug_file, "Precise root scan %p-%p (desc: %p)\n", root->start_root, root->end_root, (void*)root->root_desc));
3100 precisely_scan_objects_from ((void**)root->start_root, (void**)root->end_root, addr_start, addr_end, root->root_desc);
3106 dump_occupied (char *start, char *end, char *section_start)
3108 fprintf (heap_dump_file, "<occupied offset=\"%zd\" size=\"%zd\"/>\n", start - section_start, end - start);
3112 dump_section (GCMemSection *section, const char *type)
3114 char *start = section->data;
3115 char *end = section->data + section->size;
3116 char *occ_start = NULL;
3118 char *old_start = NULL; /* just for debugging */
3120 fprintf (heap_dump_file, "<section type=\"%s\" size=\"%zu\">\n", type, section->size);
3122 while (start < end) {
3126 if (!*(void**)start) {
3128 dump_occupied (occ_start, start, section->data);
3131 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
3134 g_assert (start < section->next_data);
3139 vt = (GCVTable*)LOAD_VTABLE (start);
3142 size = safe_object_get_size ((MonoObject*) start);
3143 size += ALLOC_ALIGN - 1;
3144 size &= ~(ALLOC_ALIGN - 1);
3147 fprintf (heap_dump_file, "<object offset=\"%d\" class=\"%s.%s\" size=\"%d\"/>\n",
3148 start - section->data,
3149 vt->klass->name_space, vt->klass->name,
3157 dump_occupied (occ_start, start, section->data);
3159 fprintf (heap_dump_file, "</section>\n");
3163 dump_heap (const char *type, int num, const char *reason)
3165 static char const *internal_mem_names [] = { "pin-queue", "fragment", "section", "scan-starts",
3166 "fin-table", "finalize-entry", "dislink-table",
3167 "dislink", "roots-table", "root-record", "statistics",
3168 "remset", "gray-queue", "store-remset" };
3170 GCMemSection *section;
3174 fprintf (heap_dump_file, "<collection type=\"%s\" num=\"%d\"", type, num);
3176 fprintf (heap_dump_file, " reason=\"%s\"", reason);
3177 fprintf (heap_dump_file, ">\n");
3178 fprintf (heap_dump_file, "<other-mem-usage type=\"pinned-chunks\" size=\"%ld\"/>\n", pinned_chunk_bytes_alloced);
3179 fprintf (heap_dump_file, "<other-mem-usage type=\"large-internal\" size=\"%ld\"/>\n", large_internal_bytes_alloced);
3180 fprintf (heap_dump_file, "<other-mem-usage type=\"mempools\" size=\"%ld\"/>\n", mono_mempool_get_bytes_allocated ());
3181 for (i = 0; i < INTERNAL_MEM_MAX; ++i)
3182 fprintf (heap_dump_file, "<other-mem-usage type=\"%s\" size=\"%ld\"/>\n", internal_mem_names [i], small_internal_mem_bytes [i]);
3183 fprintf (heap_dump_file, "<pinned type=\"stack\" bytes=\"%zu\"/>\n", pinned_byte_counts [PIN_TYPE_STACK]);
3184 /* fprintf (heap_dump_file, "<pinned type=\"static-data\" bytes=\"%d\"/>\n", pinned_byte_counts [PIN_TYPE_STATIC_DATA]); */
3185 fprintf (heap_dump_file, "<pinned type=\"other\" bytes=\"%zu\"/>\n", pinned_byte_counts [PIN_TYPE_OTHER]);
3187 dump_section (nursery_section, "nursery");
3189 for (section = section_list; section; section = section->block.next) {
3190 if (section != nursery_section)
3191 dump_section (section, "old");
3194 fprintf (heap_dump_file, "<los>\n");
3195 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
3196 MonoObject *obj = (MonoObject*) bigobj->data;
3197 MonoClass *class = mono_object_class (obj);
3199 fprintf (heap_dump_file, "<object class=\"%s.%s\" size=\"%d\"/>\n",
3200 class->name_space, class->name,
3201 safe_object_get_size (obj));
3203 fprintf (heap_dump_file, "</los>\n");
3205 fprintf (heap_dump_file, "</collection>\n");
3211 static gboolean inited = FALSE;
3213 #ifdef HEAVY_STATISTICS
3214 num_copy_object_called = 0;
3215 num_objects_copied = 0;
3221 #ifdef HEAVY_STATISTICS
3222 mono_counters_register ("WBarrier set field", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_field);
3223 mono_counters_register ("WBarrier set arrayref", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_arrayref);
3224 mono_counters_register ("WBarrier arrayref copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_arrayref_copy);
3225 mono_counters_register ("WBarrier generic store called", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_generic_store);
3226 mono_counters_register ("WBarrier generic store stored", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_generic_store_remset);
3227 mono_counters_register ("WBarrier set root", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_root);
3228 mono_counters_register ("WBarrier value copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_value_copy);
3229 mono_counters_register ("WBarrier object copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_object_copy);
3231 mono_counters_register ("# objects allocated", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_alloced);
3232 mono_counters_register ("# copy_object() called (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_called_nursery);
3233 mono_counters_register ("# objects copied (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_copied_nursery);
3234 mono_counters_register ("# copy_object() called (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_called_major);
3235 mono_counters_register ("# objects copied (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_copied_major);
3237 mono_counters_register ("# copy_object() failed from space", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_failed_from_space);
3238 mono_counters_register ("# copy_object() failed forwarded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_failed_forwarded);
3239 mono_counters_register ("# copy_object() failed pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_failed_pinned);
3240 mono_counters_register ("# copy_object() failed large or pinned chunk", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_failed_large_pinned);
3241 mono_counters_register ("# copy_object() failed to space", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_failed_to_space);
3243 mono_counters_register ("Store remsets", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_store_remsets);
3244 mono_counters_register ("Unique store remsets", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_store_remsets_unique);
3245 mono_counters_register ("Saved remsets 1", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_saved_remsets_1);
3246 mono_counters_register ("Saved remsets 2", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_saved_remsets_2);
3247 mono_counters_register ("Global remsets added", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_added);
3248 mono_counters_register ("Global remsets processed", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_processed);
3255 commit_stats (int generation)
3257 #ifdef HEAVY_STATISTICS
3258 if (generation == GENERATION_NURSERY) {
3259 stat_copy_object_called_nursery += num_copy_object_called;
3260 stat_objects_copied_nursery += num_objects_copied;
3262 g_assert (generation == GENERATION_OLD);
3263 stat_copy_object_called_major += num_copy_object_called;
3264 stat_objects_copied_major += num_objects_copied;
3270 * Collect objects in the nursery. Returns whether to trigger a major
3274 collect_nursery (size_t requested_size)
3276 size_t max_garbage_amount;
3278 char *orig_nursery_next;
3280 GCMemSection *section;
3281 int old_num_major_sections = num_major_sections;
3282 int sections_alloced;
3283 TV_DECLARE (all_atv);
3284 TV_DECLARE (all_btv);
3289 check_scan_starts ();
3292 orig_nursery_next = nursery_next;
3293 nursery_next = MAX (nursery_next, nursery_last_pinned_end);
3294 /* FIXME: optimize later to use the higher address where an object can be present */
3295 nursery_next = MAX (nursery_next, nursery_real_end);
3297 DEBUG (1, fprintf (gc_debug_file, "Start nursery collection %d %p-%p, size: %d\n", num_minor_gcs, nursery_start, nursery_next, (int)(nursery_next - nursery_start)));
3298 max_garbage_amount = nursery_next - nursery_start;
3299 g_assert (nursery_section->size >= max_garbage_amount);
3301 /* Clear all remaining nursery fragments, pinning depends on this */
3302 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION) {
3303 g_assert (orig_nursery_next <= nursery_frag_real_end);
3304 memset (orig_nursery_next, 0, nursery_frag_real_end - orig_nursery_next);
3305 for (frag = nursery_fragments; frag; frag = frag->next) {
3306 memset (frag->fragment_start, 0, frag->fragment_end - frag->fragment_start);
3311 check_for_xdomain_refs ();
3313 nursery_section->next_data = nursery_next;
3315 if (!to_space_section) {
3316 new_to_space_section ();
3318 /* we might have done degraded allocation since the
3320 g_assert (to_space_bumper <= to_space_section->next_data);
3321 to_space_bumper = to_space_section->next_data;
3323 to_space_section->is_to_space = TRUE;
3325 gray_object_queue_init ();
3328 mono_stats.minor_gc_count ++;
3329 /* world must be stopped already */
3330 TV_GETTIME (all_atv);
3332 /* pin from pinned handles */
3334 pin_from_roots (nursery_start, nursery_next);
3335 /* identify pinned objects */
3336 optimize_pin_queue (0);
3337 next_pin_slot = pin_objects_from_addresses (nursery_section, pin_queue, pin_queue + next_pin_slot, nursery_start, nursery_next);
3339 DEBUG (2, fprintf (gc_debug_file, "Finding pinned pointers: %d in %d usecs\n", next_pin_slot, TV_ELAPSED (atv, btv)));
3340 DEBUG (4, fprintf (gc_debug_file, "Start scan with %d pinned objects\n", next_pin_slot));
3342 if (consistency_check_at_minor_collection)
3343 check_consistency ();
3346 * walk all the roots and copy the young objects to the old generation,
3347 * starting from to_space
3350 scan_from_remsets (nursery_start, nursery_next);
3351 /* we don't have complete write barrier yet, so we scan all the old generation sections */
3353 DEBUG (2, fprintf (gc_debug_file, "Old generation scan: %d usecs\n", TV_ELAPSED (btv, atv)));
3355 /* the pinned objects are roots */
3356 for (i = 0; i < next_pin_slot; ++i) {
3357 DEBUG (6, fprintf (gc_debug_file, "Precise object scan %d of pinned %p (%s)\n", i, pin_queue [i], safe_name (pin_queue [i])));
3358 scan_object (pin_queue [i], nursery_start, nursery_next);
3360 /* registered roots, this includes static fields */
3361 scan_from_registered_roots (nursery_start, nursery_next, ROOT_TYPE_NORMAL);
3362 scan_from_registered_roots (nursery_start, nursery_next, ROOT_TYPE_WBARRIER);
3363 scan_thread_data (nursery_start, nursery_next, TRUE);
3364 /* alloc_pinned objects */
3365 scan_from_pinned_objects (nursery_start, nursery_next);
3367 DEBUG (2, fprintf (gc_debug_file, "Root scan: %d usecs\n", TV_ELAPSED (atv, btv)));
3369 finish_gray_stack (nursery_start, nursery_next, GENERATION_NURSERY);
3371 /* walk the pin_queue, build up the fragment list of free memory, unmark
3372 * pinned objects as we go, memzero() the empty fragments so they are ready for the
3375 build_nursery_fragments (0, next_pin_slot);
3377 DEBUG (2, fprintf (gc_debug_file, "Fragment creation: %d usecs, %zd bytes available\n", TV_ELAPSED (btv, atv), fragment_total));
3379 for (section = section_list; section; section = section->block.next) {
3380 if (section->is_to_space)
3381 section->is_to_space = FALSE;
3384 TV_GETTIME (all_btv);
3385 mono_stats.minor_gc_time_usecs += TV_ELAPSED (all_atv, all_btv);
3388 dump_heap ("minor", num_minor_gcs - 1, NULL);
3390 /* prepare the pin queue for the next collection */
3391 last_num_pinned = next_pin_slot;
3393 if (fin_ready_list || critical_fin_list) {
3394 DEBUG (4, fprintf (gc_debug_file, "Finalizer-thread wakeup: ready %d\n", num_ready_finalizers));
3395 mono_gc_finalize_notify ();
3399 g_assert (gray_object_queue_is_empty ());
3401 commit_stats (GENERATION_NURSERY);
3403 check_scan_starts ();
3405 sections_alloced = num_major_sections - old_num_major_sections;
3406 minor_collection_sections_alloced += sections_alloced;
3408 return minor_collection_sections_alloced > minor_collection_section_allowance;
3412 scan_from_pinned_chunk_if_marked (PinnedChunk *chunk, char *obj, size_t size, void *dummy)
3414 if (object_is_pinned (obj))
3415 scan_object (obj, NULL, (char*)-1);
3419 major_collection (const char *reason)
3421 GCMemSection *section, *prev_section;
3422 LOSObject *bigobj, *prevbo;
3426 TV_DECLARE (all_atv);
3427 TV_DECLARE (all_btv);
3430 /* FIXME: only use these values for the precise scan
3431 * note that to_space pointers should be excluded anyway...
3433 char *heap_start = NULL;
3434 char *heap_end = (char*)-1;
3435 size_t copy_space_required = 0;
3436 int old_num_major_sections = num_major_sections;
3437 int num_major_sections_saved, save_target, allowance_target;
3440 check_scan_starts ();
3443 DEBUG (1, fprintf (gc_debug_file, "Start major collection %d\n", num_major_gcs));
3445 mono_stats.major_gc_count ++;
3447 /* Clear all remaining nursery fragments, pinning depends on this */
3448 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION) {
3449 g_assert (nursery_next <= nursery_frag_real_end);
3450 memset (nursery_next, 0, nursery_frag_real_end - nursery_next);
3451 for (frag = nursery_fragments; frag; frag = frag->next) {
3452 memset (frag->fragment_start, 0, frag->fragment_end - frag->fragment_start);
3457 check_for_xdomain_refs ();
3460 * FIXME: implement Mark/Compact
3461 * Until that is done, we can just apply mostly the same alg as for the nursery:
3462 * this means we need a big section to potentially copy all the other sections, so
3463 * it is not ideal specially with large heaps.
3465 if (g_getenv ("MONO_GC_NO_MAJOR")) {
3466 collect_nursery (0);
3469 TV_GETTIME (all_atv);
3470 /* FIXME: make sure the nursery next_data ptr is updated */
3471 nursery_section->next_data = nursery_real_end;
3472 /* we should also coalesce scanning from sections close to each other
3473 * and deal with pointers outside of the sections later.
3475 /* The remsets are not useful for a major collection */
3477 /* world must be stopped already */
3480 DEBUG (6, fprintf (gc_debug_file, "Collecting pinned addresses\n"));
3481 pin_from_roots ((void*)lowest_heap_address, (void*)highest_heap_address);
3482 optimize_pin_queue (0);
3485 * pin_queue now contains all candidate pointers, sorted and
3486 * uniqued. We must do two passes now to figure out which
3487 * objects are pinned.
3489 * The first is to find within the pin_queue the area for each
3490 * section. This requires that the pin_queue be sorted. We
3491 * also process the LOS objects and pinned chunks here.
3493 * The second, destructive, pass is to reduce the section
3494 * areas to pointers to the actually pinned objects.
3496 DEBUG (6, fprintf (gc_debug_file, "Pinning from sections\n"));
3497 /* first pass for the sections */
3498 for (section = section_list; section; section = section->block.next) {
3500 DEBUG (6, fprintf (gc_debug_file, "Pinning from section %p (%p-%p)\n", section, section->data, section->end_data));
3501 find_optimized_pin_queue_area (section->data, section->end_data, &start, &end);
3502 DEBUG (6, fprintf (gc_debug_file, "Found %d pinning addresses in section %p (%d-%d)\n",
3503 end - start, section, start, end));
3504 section->pin_queue_start = start;
3505 section->pin_queue_end = end;
3507 /* identify possible pointers to the insize of large objects */
3508 DEBUG (6, fprintf (gc_debug_file, "Pinning from large objects\n"));
3509 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
3511 find_optimized_pin_queue_area (bigobj->data, (char*)bigobj->data + bigobj->size, &start, &end);
3513 pin_object (bigobj->data);
3515 pin_stats_register_object ((char*) bigobj->data, safe_object_get_size ((MonoObject*) bigobj->data));
3516 DEBUG (6, fprintf (gc_debug_file, "Marked large object %p (%s) size: %zd from roots\n", bigobj->data, safe_name (bigobj->data), bigobj->size));
3519 /* look for pinned addresses for pinned-alloc objects */
3520 DEBUG (6, fprintf (gc_debug_file, "Pinning from pinned-alloc objects\n"));
3521 for (chunk = pinned_chunk_list; chunk; chunk = chunk->block.next) {
3523 find_optimized_pin_queue_area (chunk->start_data, (char*)chunk + chunk->num_pages * FREELIST_PAGESIZE, &start, &end);
3525 mark_pinned_from_addresses (chunk, pin_queue + start, pin_queue + end);
3527 /* second pass for the sections */
3528 for (section = section_list; section; section = section->block.next) {
3529 int start = section->pin_queue_start;
3530 int end = section->pin_queue_end;
3533 reduced_to = pin_objects_from_addresses (section, pin_queue + start, pin_queue + end,
3534 section->data, section->next_data);
3535 section->pin_queue_start = start;
3536 section->pin_queue_end = start + reduced_to;
3538 copy_space_required += (char*)section->next_data - (char*)section->data;
3542 DEBUG (2, fprintf (gc_debug_file, "Finding pinned pointers: %d in %d usecs\n", next_pin_slot, TV_ELAPSED (atv, btv)));
3543 DEBUG (4, fprintf (gc_debug_file, "Start scan with %d pinned objects\n", next_pin_slot));
3545 new_to_space_section ();
3546 gray_object_queue_init ();
3548 /* the old generation doesn't need to be scanned (no remembered sets or card
3549 * table needed either): the only objects that must survive are those pinned and
3550 * those referenced by the precise roots.
3551 * mark any section without pinned objects, so we can free it since we will be able to
3552 * move all the objects.
3554 /* the pinned objects are roots (big objects are included in this list, too) */
3555 for (section = section_list; section; section = section->block.next) {
3556 for (i = section->pin_queue_start; i < section->pin_queue_end; ++i) {
3557 DEBUG (6, fprintf (gc_debug_file, "Precise object scan %d of pinned %p (%s)\n",
3558 i, pin_queue [i], safe_name (pin_queue [i])));
3559 scan_object (pin_queue [i], heap_start, heap_end);
3562 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
3563 if (object_is_pinned (bigobj->data)) {
3564 DEBUG (6, fprintf (gc_debug_file, "Precise object scan pinned LOS object %p (%s)\n",
3565 bigobj->data, safe_name (bigobj->data)));
3566 scan_object (bigobj->data, heap_start, heap_end);
3569 scan_pinned_objects (scan_from_pinned_chunk_if_marked, NULL);
3570 /* registered roots, this includes static fields */
3571 scan_from_registered_roots (heap_start, heap_end, ROOT_TYPE_NORMAL);
3572 scan_from_registered_roots (heap_start, heap_end, ROOT_TYPE_WBARRIER);
3574 scan_thread_data (heap_start, heap_end, TRUE);
3575 /* alloc_pinned objects */
3576 scan_from_pinned_objects (heap_start, heap_end);
3577 /* scan the list of objects ready for finalization */
3578 scan_finalizer_entries (fin_ready_list, heap_start, heap_end);
3579 scan_finalizer_entries (critical_fin_list, heap_start, heap_end);
3581 DEBUG (2, fprintf (gc_debug_file, "Root scan: %d usecs\n", TV_ELAPSED (btv, atv)));
3583 /* we need to go over the big object list to see if any was marked and scan it
3584 * And we need to make this in a loop, considering that objects referenced by finalizable
3585 * objects could reference big objects (this happens in finish_gray_stack ())
3587 scan_needed_big_objects (heap_start, heap_end);
3588 /* all the objects in the heap */
3589 finish_gray_stack (heap_start, heap_end, GENERATION_OLD);
3593 /* sweep the big objects list */
3595 for (bigobj = los_object_list; bigobj;) {
3596 if (object_is_pinned (bigobj->data)) {
3597 unpin_object (bigobj->data);
3598 bigobj->scanned = FALSE;
3601 /* not referenced anywhere, so we can free it */
3603 prevbo->next = bigobj->next;
3605 los_object_list = bigobj->next;
3607 bigobj = bigobj->next;
3608 free_large_object (to_free);
3612 bigobj = bigobj->next;
3614 /* unpin objects from the pinned chunks and free the unmarked ones */
3615 sweep_pinned_objects ();
3617 /* free the unused sections */
3618 prev_section = NULL;
3619 for (section = section_list; section;) {
3620 /* to_space doesn't need handling here and the nursery is special */
3621 if (section->is_to_space || section == nursery_section) {
3622 if (section->is_to_space)
3623 section->is_to_space = FALSE;
3624 prev_section = section;
3625 section = section->block.next;
3628 /* no pinning object, so the section is free */
3629 if (section->pin_queue_start == section->pin_queue_end) {
3630 GCMemSection *to_free;
3632 prev_section->block.next = section->block.next;
3634 section_list = section->block.next;
3636 section = section->block.next;
3637 free_major_section (to_free);
3640 DEBUG (6, fprintf (gc_debug_file, "Section %p has still pinned objects (%d)\n", section, section->pin_queue_end - section->pin_queue_start));
3641 build_section_fragments (section);
3643 prev_section = section;
3644 section = section->block.next;
3647 /* walk the pin_queue, build up the fragment list of free memory, unmark
3648 * pinned objects as we go, memzero() the empty fragments so they are ready for the
3651 build_nursery_fragments (nursery_section->pin_queue_start, nursery_section->pin_queue_end);
3653 TV_GETTIME (all_btv);
3654 mono_stats.major_gc_time_usecs += TV_ELAPSED (all_atv, all_btv);
3657 dump_heap ("major", num_major_gcs - 1, reason);
3659 /* prepare the pin queue for the next collection */
3661 if (fin_ready_list || critical_fin_list) {
3662 DEBUG (4, fprintf (gc_debug_file, "Finalizer-thread wakeup: ready %d\n", num_ready_finalizers));
3663 mono_gc_finalize_notify ();
3667 g_assert (gray_object_queue_is_empty ());
3669 commit_stats (GENERATION_OLD);
3671 num_major_sections_saved = MAX (old_num_major_sections - num_major_sections, 1);
3673 save_target = num_major_sections / 2;
3674 allowance_target = save_target * minor_collection_sections_alloced / num_major_sections_saved;
3676 minor_collection_section_allowance = MAX (MIN (allowance_target, num_major_sections), MIN_MINOR_COLLECTION_SECTION_ALLOWANCE);
3679 printf ("alloced %d saved %d target %d allowance %d\n",
3680 minor_collection_sections_alloced, num_major_sections_saved, allowance_target,
3681 minor_collection_section_allowance);
3684 minor_collection_sections_alloced = 0;
3686 check_scan_starts ();
3690 * Allocate a new section of memory to be used as old generation.
3692 static GCMemSection*
3693 alloc_major_section (void)
3695 GCMemSection *section;
3698 section = get_os_memory_aligned (MAJOR_SECTION_SIZE, TRUE);
3699 section->next_data = section->data = (char*)section + SIZEOF_GC_MEM_SECTION;
3700 g_assert (!((mword)section->data & 7));
3701 section->size = MAJOR_SECTION_SIZE - SIZEOF_GC_MEM_SECTION;
3702 section->end_data = section->data + section->size;
3703 UPDATE_HEAP_BOUNDARIES (section->data, section->end_data);
3704 total_alloc += section->size;
3705 DEBUG (3, fprintf (gc_debug_file, "New major heap section: (%p-%p), total: %zd\n", section->data, section->end_data, total_alloc));
3706 scan_starts = (section->size + SCAN_START_SIZE - 1) / SCAN_START_SIZE;
3707 section->scan_starts = get_internal_mem (sizeof (char*) * scan_starts, INTERNAL_MEM_SCAN_STARTS);
3708 section->num_scan_start = scan_starts;
3709 section->block.role = MEMORY_ROLE_GEN1;
3710 section->is_to_space = TRUE;
3712 /* add to the section list */
3713 section->block.next = section_list;
3714 section_list = section;
3716 ++num_major_sections;
3722 free_major_section (GCMemSection *section)
3724 DEBUG (3, fprintf (gc_debug_file, "Freed major section %p (%p-%p)\n", section, section->data, section->end_data));
3725 free_internal_mem (section->scan_starts, INTERNAL_MEM_SCAN_STARTS);
3726 free_os_memory (section, MAJOR_SECTION_SIZE);
3727 total_alloc -= MAJOR_SECTION_SIZE - SIZEOF_GC_MEM_SECTION;
3729 --num_major_sections;
3733 * When deciding if it's better to collect or to expand, keep track
3734 * of how much garbage was reclaimed with the last collection: if it's too
3736 * This is called when we could not allocate a small object.
3738 static void __attribute__((noinline))
3739 minor_collect_or_expand_inner (size_t size)
3741 int do_minor_collection = 1;
3743 if (!nursery_section) {
3747 if (do_minor_collection) {
3749 if (collect_nursery (size))
3750 major_collection ("minor overflow");
3751 DEBUG (2, fprintf (gc_debug_file, "Heap size: %zd, LOS size: %zd\n", total_alloc, los_memory_usage));
3753 /* this also sets the proper pointers for the next allocation */
3754 if (!search_fragment_for_size (size)) {
3756 /* TypeBuilder and MonoMethod are killing mcs with fragmentation */
3757 DEBUG (1, fprintf (gc_debug_file, "nursery collection didn't find enough room for %zd alloc (%d pinned)\n", size, last_num_pinned));
3758 for (i = 0; i < last_num_pinned; ++i) {
3759 DEBUG (3, fprintf (gc_debug_file, "Bastard pinning obj %p (%s), size: %d\n", pin_queue [i], safe_name (pin_queue [i]), safe_object_get_size (pin_queue [i])));
3764 //report_internal_mem_usage ();
3768 * ######################################################################
3769 * ######## Memory allocation from the OS
3770 * ######################################################################
3771 * This section of code deals with getting memory from the OS and
3772 * allocating memory for GC-internal data structures.
3773 * Internal memory can be handled with a freelist for small objects.
3777 * Allocate a big chunk of memory from the OS (usually 64KB to several megabytes).
3778 * This must not require any lock.
3781 get_os_memory (size_t size, int activate)
3784 unsigned long prot_flags = activate? MONO_MMAP_READ|MONO_MMAP_WRITE: MONO_MMAP_NONE;
3786 prot_flags |= MONO_MMAP_PRIVATE | MONO_MMAP_ANON;
3787 size += pagesize - 1;
3788 size &= ~(pagesize - 1);
3789 ptr = mono_valloc (0, size, prot_flags);
3794 * Free the memory returned by get_os_memory (), returning it to the OS.
3797 free_os_memory (void *addr, size_t size)
3799 mono_vfree (addr, size);
3806 report_pinned_chunk (PinnedChunk *chunk, int seq) {
3808 int i, free_pages, num_free, free_mem;
3810 for (i = 0; i < chunk->num_pages; ++i) {
3811 if (!chunk->page_sizes [i])
3814 printf ("Pinned chunk %d at %p, size: %d, pages: %d, free: %d\n", seq, chunk, chunk->num_pages * FREELIST_PAGESIZE, chunk->num_pages, free_pages);
3815 free_mem = FREELIST_PAGESIZE * free_pages;
3816 for (i = 0; i < FREELIST_NUM_SLOTS; ++i) {
3817 if (!chunk->free_list [i])
3820 p = chunk->free_list [i];
3825 printf ("\tfree list of size %d, %d items\n", freelist_sizes [i], num_free);
3826 free_mem += freelist_sizes [i] * num_free;
3828 printf ("\tfree memory in chunk: %d\n", free_mem);
3834 static G_GNUC_UNUSED void
3835 report_internal_mem_usage (void) {
3838 printf ("Internal memory usage:\n");
3840 for (chunk = internal_chunk_list; chunk; chunk = chunk->block.next) {
3841 report_pinned_chunk (chunk, i++);
3843 printf ("Pinned memory usage:\n");
3845 for (chunk = pinned_chunk_list; chunk; chunk = chunk->block.next) {
3846 report_pinned_chunk (chunk, i++);
3851 * the array of pointers from @start to @end contains conservative
3852 * pointers to objects inside @chunk: mark each referenced object
3856 mark_pinned_from_addresses (PinnedChunk *chunk, void **start, void **end)
3858 for (; start < end; start++) {
3859 char *addr = *start;
3860 int offset = (char*)addr - (char*)chunk;
3861 int page = offset / FREELIST_PAGESIZE;
3862 int obj_offset = page == 0? offset - ((char*)chunk->start_data - (char*)chunk): offset % FREELIST_PAGESIZE;
3863 int slot_size = chunk->page_sizes [page];
3865 /* the page is not allocated */
3868 /* would be faster if we restrict the sizes to power of two,
3869 * but that's a waste of memory: need to measure. it could reduce
3870 * fragmentation since there are less pages needed, if for example
3871 * someone interns strings of each size we end up with one page per
3872 * interned string (still this is just ~40 KB): with more fine-grained sizes
3873 * this increases the number of used pages.
3876 obj_offset /= slot_size;
3877 obj_offset *= slot_size;
3878 addr = (char*)chunk->start_data + obj_offset;
3880 obj_offset /= slot_size;
3881 obj_offset *= slot_size;
3882 addr = (char*)chunk + page * FREELIST_PAGESIZE + obj_offset;
3885 /* if the vtable is inside the chunk it's on the freelist, so skip */
3886 if (*ptr && (*ptr < (void*)chunk->start_data || *ptr > (void*)((char*)chunk + chunk->num_pages * FREELIST_PAGESIZE))) {
3889 pin_stats_register_object ((char*) addr, safe_object_get_size ((MonoObject*) addr));
3890 DEBUG (6, fprintf (gc_debug_file, "Marked pinned object %p (%s) from roots\n", addr, safe_name (addr)));
3896 scan_pinned_objects (ScanPinnedObjectCallbackFunc callback, void *callback_data)
3903 for (chunk = pinned_chunk_list; chunk; chunk = chunk->block.next) {
3904 end_chunk = (char*)chunk + chunk->num_pages * FREELIST_PAGESIZE;
3905 DEBUG (6, fprintf (gc_debug_file, "Scanning pinned chunk %p (range: %p-%p)\n", chunk, chunk->start_data, end_chunk));
3906 for (i = 0; i < chunk->num_pages; ++i) {
3907 obj_size = chunk->page_sizes [i];
3910 p = i? (char*)chunk + i * FREELIST_PAGESIZE: chunk->start_data;
3911 endp = i? p + FREELIST_PAGESIZE: (char*)chunk + FREELIST_PAGESIZE;
3912 DEBUG (6, fprintf (gc_debug_file, "Page %d (size: %d, range: %p-%p)\n", i, obj_size, p, endp));
3913 while (p + obj_size <= endp) {
3915 DEBUG (9, fprintf (gc_debug_file, "Considering %p (vtable: %p)\n", ptr, *ptr));
3916 /* if the first word (the vtable) is outside the chunk we have an object */
3917 if (*ptr && (*ptr < (void*)chunk || *ptr >= end_chunk))
3918 callback (chunk, (char*)ptr, obj_size, callback_data);
3926 sweep_pinned_objects_callback (PinnedChunk *chunk, char *ptr, size_t size, void *data)
3928 if (object_is_pinned (ptr)) {
3930 DEBUG (6, fprintf (gc_debug_file, "Unmarked pinned object %p (%s)\n", ptr, safe_name (ptr)));
3932 DEBUG (6, fprintf (gc_debug_file, "Freeing unmarked pinned object %p (%s)\n", ptr, safe_name (ptr)));
3933 free_pinned_object (chunk, ptr, size);
3938 sweep_pinned_objects (void)
3940 scan_pinned_objects (sweep_pinned_objects_callback, NULL);
3944 scan_object_callback (PinnedChunk *chunk, char *ptr, size_t size, char **data)
3946 DEBUG (6, fprintf (gc_debug_file, "Precise object scan of alloc_pinned %p (%s)\n", ptr, safe_name (ptr)));
3947 /* FIXME: Put objects without references into separate chunks
3948 which do not need to be scanned */
3949 scan_object (ptr, data [0], data [1]);
3953 scan_from_pinned_objects (char *addr_start, char *addr_end)
3955 char *data [2] = { addr_start, addr_end };
3956 scan_pinned_objects ((ScanPinnedObjectCallbackFunc)scan_object_callback, data);
3960 * Find the slot number in the freelist for memory chunks that
3961 * can contain @size objects.
3964 slot_for_size (size_t size)
3967 /* do a binary search or lookup table later. */
3968 for (slot = 0; slot < FREELIST_NUM_SLOTS; ++slot) {
3969 if (freelist_sizes [slot] >= size)
3972 g_assert_not_reached ();
3977 * Build a free list for @size memory chunks from the memory area between
3978 * start_page and end_page.
3981 build_freelist (PinnedChunk *chunk, int slot, int size, char *start_page, char *end_page)
3985 /*g_print ("building freelist for slot %d, size %d in %p\n", slot, size, chunk);*/
3986 p = (void**)start_page;
3987 end = (void**)(end_page - size);
3988 g_assert (!chunk->free_list [slot]);
3989 chunk->free_list [slot] = p;
3990 while ((char*)p + size <= (char*)end) {
3992 *p = (void*)((char*)p + size);
3996 /*g_print ("%d items created, max: %d\n", count, (end_page - start_page) / size);*/
4000 alloc_pinned_chunk (void)
4004 int size = MAJOR_SECTION_SIZE;
4006 chunk = get_os_memory_aligned (size, TRUE);
4007 chunk->block.role = MEMORY_ROLE_PINNED;
4009 UPDATE_HEAP_BOUNDARIES (chunk, ((char*)chunk + size));
4010 total_alloc += size;
4011 pinned_chunk_bytes_alloced += size;
4013 /* setup the bookeeping fields */
4014 chunk->num_pages = size / FREELIST_PAGESIZE;
4015 offset = G_STRUCT_OFFSET (PinnedChunk, data);
4016 chunk->page_sizes = (void*)((char*)chunk + offset);
4017 offset += sizeof (int) * chunk->num_pages;
4018 offset += ALLOC_ALIGN - 1;
4019 offset &= ~(ALLOC_ALIGN - 1);
4020 chunk->free_list = (void*)((char*)chunk + offset);
4021 offset += sizeof (void*) * FREELIST_NUM_SLOTS;
4022 offset += ALLOC_ALIGN - 1;
4023 offset &= ~(ALLOC_ALIGN - 1);
4024 chunk->start_data = (void*)((char*)chunk + offset);
4026 /* allocate the first page to the freelist */
4027 chunk->page_sizes [0] = PINNED_FIRST_SLOT_SIZE;
4028 build_freelist (chunk, slot_for_size (PINNED_FIRST_SLOT_SIZE), PINNED_FIRST_SLOT_SIZE, chunk->start_data, ((char*)chunk + FREELIST_PAGESIZE));
4029 DEBUG (4, fprintf (gc_debug_file, "Allocated pinned chunk %p, size: %d\n", chunk, size));
4030 min_pinned_chunk_addr = MIN (min_pinned_chunk_addr, (char*)chunk->start_data);
4031 max_pinned_chunk_addr = MAX (max_pinned_chunk_addr, ((char*)chunk + size));
4035 /* assumes freelist for slot is empty, so try to alloc a new page */
4037 get_chunk_freelist (PinnedChunk *chunk, int slot)
4041 p = chunk->free_list [slot];
4043 chunk->free_list [slot] = *p;
4046 for (i = 0; i < chunk->num_pages; ++i) {
4048 if (chunk->page_sizes [i])
4050 size = freelist_sizes [slot];
4051 chunk->page_sizes [i] = size;
4052 build_freelist (chunk, slot, size, (char*)chunk + FREELIST_PAGESIZE * i, (char*)chunk + FREELIST_PAGESIZE * (i + 1));
4056 p = chunk->free_list [slot];
4058 chunk->free_list [slot] = *p;
4065 alloc_from_freelist (size_t size)
4069 PinnedChunk *pchunk;
4070 slot = slot_for_size (size);
4071 /*g_print ("using slot %d for size %d (slot size: %d)\n", slot, size, freelist_sizes [slot]);*/
4072 g_assert (size <= freelist_sizes [slot]);
4073 for (pchunk = pinned_chunk_list; pchunk; pchunk = pchunk->block.next) {
4074 void **p = pchunk->free_list [slot];
4076 /*g_print ("found freelist for slot %d in chunk %p, returning %p, next %p\n", slot, pchunk, p, *p);*/
4077 pchunk->free_list [slot] = *p;
4081 for (pchunk = pinned_chunk_list; pchunk; pchunk = pchunk->block.next) {
4082 res = get_chunk_freelist (pchunk, slot);
4086 pchunk = alloc_pinned_chunk ();
4087 /* FIXME: handle OOM */
4088 pchunk->block.next = pinned_chunk_list;
4089 pinned_chunk_list = pchunk;
4090 res = get_chunk_freelist (pchunk, slot);
4094 /* used for the GC-internal data structures */
4095 /* FIXME: add support for bigger sizes by allocating more than one page
4099 get_internal_mem (size_t size, int type)
4103 PinnedChunk *pchunk;
4105 if (size > freelist_sizes [FREELIST_NUM_SLOTS - 1]) {
4106 LargeInternalMemHeader *mh;
4108 size += sizeof (LargeInternalMemHeader);
4109 mh = get_os_memory (size, TRUE);
4110 mh->magic = LARGE_INTERNAL_MEM_HEADER_MAGIC;
4113 large_internal_bytes_alloced += size;
4118 slot = slot_for_size (size);
4119 g_assert (size <= freelist_sizes [slot]);
4121 small_internal_mem_bytes [type] += freelist_sizes [slot];
4123 for (pchunk = internal_chunk_list; pchunk; pchunk = pchunk->block.next) {
4124 void **p = pchunk->free_list [slot];
4126 pchunk->free_list [slot] = *p;
4127 memset (p, 0, size);
4131 for (pchunk = internal_chunk_list; pchunk; pchunk = pchunk->block.next) {
4132 res = get_chunk_freelist (pchunk, slot);
4134 memset (res, 0, size);
4138 pchunk = alloc_pinned_chunk ();
4139 /* FIXME: handle OOM */
4140 pchunk->block.next = internal_chunk_list;
4141 internal_chunk_list = pchunk;
4142 res = get_chunk_freelist (pchunk, slot);
4143 memset (res, 0, size);
4148 free_internal_mem (void *addr, int type)
4150 PinnedChunk *pchunk;
4151 LargeInternalMemHeader *mh;
4154 for (pchunk = internal_chunk_list; pchunk; pchunk = pchunk->block.next) {
4155 /*printf ("trying to free %p in %p (pages: %d)\n", addr, pchunk, pchunk->num_pages);*/
4156 if (addr >= (void*)pchunk && (char*)addr < (char*)pchunk + pchunk->num_pages * FREELIST_PAGESIZE) {
4157 int offset = (char*)addr - (char*)pchunk;
4158 int page = offset / FREELIST_PAGESIZE;
4159 int slot = slot_for_size (pchunk->page_sizes [page]);
4161 *p = pchunk->free_list [slot];
4162 pchunk->free_list [slot] = p;
4164 small_internal_mem_bytes [type] -= freelist_sizes [slot];
4169 mh = (LargeInternalMemHeader*)((char*)addr - G_STRUCT_OFFSET (LargeInternalMemHeader, data));
4170 g_assert (mh->magic == LARGE_INTERNAL_MEM_HEADER_MAGIC);
4171 large_internal_bytes_alloced -= mh->size;
4172 free_os_memory (mh, mh->size);
4176 * ######################################################################
4177 * ######## Object allocation
4178 * ######################################################################
4179 * This section of code deals with allocating memory for objects.
4180 * There are several ways:
4181 * *) allocate large objects
4182 * *) allocate normal objects
4183 * *) fast lock-free allocation
4184 * *) allocation of pinned objects
4188 free_large_object (LOSObject *obj)
4190 size_t size = obj->size;
4191 DEBUG (4, fprintf (gc_debug_file, "Freed large object %p, size %zd\n", obj->data, obj->size));
4193 los_memory_usage -= size;
4194 size += sizeof (LOSObject);
4195 size += pagesize - 1;
4196 size &= ~(pagesize - 1);
4197 total_alloc -= size;
4199 free_os_memory (obj, size);
4203 * Objects with size >= 64KB are allocated in the large object space.
4204 * They are currently kept track of with a linked list.
4205 * They don't move, so there is no need to pin them during collection
4206 * and we avoid the memcpy overhead.
4208 static void* __attribute__((noinline))
4209 alloc_large_inner (MonoVTable *vtable, size_t size)
4214 int just_did_major_gc = FALSE;
4216 g_assert (size > MAX_SMALL_OBJ_SIZE);
4218 if (los_memory_usage > next_los_collection) {
4219 DEBUG (4, fprintf (gc_debug_file, "Should trigger major collection: req size %zd (los already: %zu, limit: %zu)\n", size, los_memory_usage, next_los_collection));
4220 just_did_major_gc = TRUE;
4222 major_collection ("LOS overflow");
4224 /* later increase based on a percent of the heap size */
4225 next_los_collection = los_memory_usage + 5*1024*1024;
4228 alloc_size += sizeof (LOSObject);
4229 alloc_size += pagesize - 1;
4230 alloc_size &= ~(pagesize - 1);
4231 /* FIXME: handle OOM */
4232 obj = get_os_memory (alloc_size, TRUE);
4234 vtslot = (void**)obj->data;
4236 total_alloc += alloc_size;
4237 UPDATE_HEAP_BOUNDARIES (obj->data, (char*)obj->data + size);
4238 obj->next = los_object_list;
4239 los_object_list = obj;
4240 los_memory_usage += size;
4242 DEBUG (4, fprintf (gc_debug_file, "Allocated large object %p, vtable: %p (%s), size: %zd\n", obj->data, vtable, vtable->klass->name, size));
4246 /* check if we have a suitable fragment in nursery_fragments to be able to allocate
4247 * an object of size @size
4248 * Return FALSE if not found (which means we need a collection)
4251 search_fragment_for_size (size_t size)
4253 Fragment *frag, *prev;
4254 DEBUG (4, fprintf (gc_debug_file, "Searching nursery fragment %p, size: %zd\n", nursery_frag_real_end, size));
4256 if (nursery_frag_real_end > nursery_next && nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
4257 /* Clear the remaining space, pinning depends on this */
4258 memset (nursery_next, 0, nursery_frag_real_end - nursery_next);
4261 for (frag = nursery_fragments; frag; frag = frag->next) {
4262 if (size <= (frag->fragment_end - frag->fragment_start)) {
4263 /* remove from the list */
4265 prev->next = frag->next;
4267 nursery_fragments = frag->next;
4268 nursery_next = frag->fragment_start;
4269 nursery_frag_real_end = frag->fragment_end;
4271 DEBUG (4, fprintf (gc_debug_file, "Using nursery fragment %p-%p, size: %zd (req: %zd)\n", nursery_next, nursery_frag_real_end, nursery_frag_real_end - nursery_next, size));
4272 frag->next = fragment_freelist;
4273 fragment_freelist = frag;
4282 * size is already rounded up and we hold the GC lock.
4285 alloc_degraded (MonoVTable *vtable, size_t size)
4287 GCMemSection *section;
4289 g_assert (size <= MAX_SMALL_OBJ_SIZE);
4290 for (section = section_list; section; section = section->block.next) {
4291 if (section != nursery_section && (section->end_data - section->next_data) >= size) {
4292 p = (void**)section->next_data;
4297 section = alloc_major_section ();
4298 section->is_to_space = FALSE;
4299 /* FIXME: handle OOM */
4300 p = (void**)section->next_data;
4302 section->next_data += size;
4303 degraded_mode += size;
4304 DEBUG (3, fprintf (gc_debug_file, "Allocated (degraded) object %p, vtable: %p (%s), size: %zd in section %p\n", p, vtable, vtable->klass->name, size, section));
4310 * Provide a variant that takes just the vtable for small fixed-size objects.
4311 * The aligned size is already computed and stored in vt->gc_descr.
4312 * Note: every SCAN_START_SIZE or so we are given the chance to do some special
4313 * processing. We can keep track of where objects start, for example,
4314 * so when we scan the thread stacks for pinned objects, we can start
4315 * a search for the pinned object in SCAN_START_SIZE chunks.
4318 mono_gc_alloc_obj_nolock (MonoVTable *vtable, size_t size)
4320 /* FIXME: handle OOM */
4326 HEAVY_STAT (++stat_objects_alloced);
4328 size += ALLOC_ALIGN - 1;
4329 size &= ~(ALLOC_ALIGN - 1);
4331 g_assert (vtable->gc_descr);
4333 if (G_UNLIKELY (collect_before_allocs)) {
4334 if (nursery_section) {
4336 collect_nursery (0);
4338 if (!degraded_mode && !search_fragment_for_size (size)) {
4340 g_assert_not_reached ();
4346 * We must already have the lock here instead of after the
4347 * fast path because we might be interrupted in the fast path
4348 * (after confirming that new_next < TLAB_TEMP_END) by the GC,
4349 * and we'll end up allocating an object in a fragment which
4350 * no longer belongs to us.
4352 * The managed allocator does not do this, but it's treated
4353 * specially by the world-stopping code.
4356 if (size > MAX_SMALL_OBJ_SIZE) {
4357 p = alloc_large_inner (vtable, size);
4359 /* tlab_next and tlab_temp_end are TLS vars so accessing them might be expensive */
4361 p = (void**)TLAB_NEXT;
4362 /* FIXME: handle overflow */
4363 new_next = (char*)p + size;
4364 TLAB_NEXT = new_next;
4366 if (G_LIKELY (new_next < TLAB_TEMP_END)) {
4370 * FIXME: We might need a memory barrier here so the change to tlab_next is
4371 * visible before the vtable store.
4374 DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
4375 g_assert (*p == NULL);
4378 g_assert (TLAB_NEXT == new_next);
4385 /* there are two cases: the object is too big or we run out of space in the TLAB */
4386 /* we also reach here when the thread does its first allocation after a minor
4387 * collection, since the tlab_ variables are initialized to NULL.
4388 * there can be another case (from ORP), if we cooperate with the runtime a bit:
4389 * objects that need finalizers can have the high bit set in their size
4390 * so the above check fails and we can readily add the object to the queue.
4391 * This avoids taking again the GC lock when registering, but this is moot when
4392 * doing thread-local allocation, so it may not be a good idea.
4394 g_assert (TLAB_NEXT == new_next);
4395 if (TLAB_NEXT >= TLAB_REAL_END) {
4397 * Run out of space in the TLAB. When this happens, some amount of space
4398 * remains in the TLAB, but not enough to satisfy the current allocation
4399 * request. Currently, we retire the TLAB in all cases, later we could
4400 * keep it if the remaining space is above a treshold, and satisfy the
4401 * allocation directly from the nursery.
4404 /* when running in degraded mode, we continue allocing that way
4405 * for a while, to decrease the number of useless nursery collections.
4407 if (degraded_mode && degraded_mode < DEFAULT_NURSERY_SIZE) {
4408 p = alloc_degraded (vtable, size);
4412 if (size > tlab_size) {
4413 /* Allocate directly from the nursery */
4414 if (nursery_next + size >= nursery_frag_real_end) {
4415 if (!search_fragment_for_size (size)) {
4416 minor_collect_or_expand_inner (size);
4417 if (degraded_mode) {
4418 p = alloc_degraded (vtable, size);
4424 p = (void*)nursery_next;
4425 nursery_next += size;
4426 if (nursery_next > nursery_frag_real_end) {
4431 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
4432 memset (p, 0, size);
4435 DEBUG (3, fprintf (gc_debug_file, "Retire TLAB: %p-%p [%ld]\n", TLAB_START, TLAB_REAL_END, (long)(TLAB_REAL_END - TLAB_NEXT - size)));
4437 if (nursery_next + tlab_size >= nursery_frag_real_end) {
4438 res = search_fragment_for_size (tlab_size);
4440 minor_collect_or_expand_inner (tlab_size);
4441 if (degraded_mode) {
4442 p = alloc_degraded (vtable, size);
4448 /* Allocate a new TLAB from the current nursery fragment */
4449 TLAB_START = nursery_next;
4450 nursery_next += tlab_size;
4451 TLAB_NEXT = TLAB_START;
4452 TLAB_REAL_END = TLAB_START + tlab_size;
4453 TLAB_TEMP_END = TLAB_START + MIN (SCAN_START_SIZE, tlab_size);
4455 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
4456 memset (TLAB_START, 0, tlab_size);
4458 /* Allocate from the TLAB */
4459 p = (void*)TLAB_NEXT;
4461 g_assert (TLAB_NEXT <= TLAB_REAL_END);
4463 nursery_section->scan_starts [((char*)p - (char*)nursery_section->data)/SCAN_START_SIZE] = (char*)p;
4466 /* Reached tlab_temp_end */
4468 /* record the scan start so we can find pinned objects more easily */
4469 nursery_section->scan_starts [((char*)p - (char*)nursery_section->data)/SCAN_START_SIZE] = (char*)p;
4470 /* we just bump tlab_temp_end as well */
4471 TLAB_TEMP_END = MIN (TLAB_REAL_END, TLAB_NEXT + SCAN_START_SIZE);
4472 DEBUG (5, fprintf (gc_debug_file, "Expanding local alloc: %p-%p\n", TLAB_NEXT, TLAB_TEMP_END));
4476 DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
4483 mono_gc_try_alloc_obj_nolock (MonoVTable *vtable, size_t size)
4489 size += ALLOC_ALIGN - 1;
4490 size &= ~(ALLOC_ALIGN - 1);
4492 g_assert (vtable->gc_descr);
4493 if (size <= MAX_SMALL_OBJ_SIZE) {
4494 /* tlab_next and tlab_temp_end are TLS vars so accessing them might be expensive */
4496 p = (void**)TLAB_NEXT;
4497 /* FIXME: handle overflow */
4498 new_next = (char*)p + size;
4499 TLAB_NEXT = new_next;
4501 if (G_LIKELY (new_next < TLAB_TEMP_END)) {
4505 * FIXME: We might need a memory barrier here so the change to tlab_next is
4506 * visible before the vtable store.
4509 HEAVY_STAT (++stat_objects_alloced);
4511 DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
4512 g_assert (*p == NULL);
4515 g_assert (TLAB_NEXT == new_next);
4524 mono_gc_alloc_obj (MonoVTable *vtable, size_t size)
4527 #ifndef DISABLE_CRITICAL_REGION
4529 ENTER_CRITICAL_REGION;
4530 res = mono_gc_try_alloc_obj_nolock (vtable, size);
4532 EXIT_CRITICAL_REGION;
4535 EXIT_CRITICAL_REGION;
4538 res = mono_gc_alloc_obj_nolock (vtable, size);
4544 mono_gc_alloc_vector (MonoVTable *vtable, size_t size, mono_array_size_t max_length)
4547 #ifndef DISABLE_CRITICAL_REGION
4549 ENTER_CRITICAL_REGION;
4550 arr = mono_gc_try_alloc_obj_nolock (vtable, size);
4552 arr->max_length = max_length;
4553 EXIT_CRITICAL_REGION;
4556 EXIT_CRITICAL_REGION;
4561 arr = mono_gc_alloc_obj_nolock (vtable, size);
4562 arr->max_length = max_length;
4570 mono_gc_alloc_array (MonoVTable *vtable, size_t size, mono_array_size_t max_length, mono_array_size_t bounds_size)
4573 MonoArrayBounds *bounds;
4577 arr = mono_gc_alloc_obj_nolock (vtable, size);
4578 arr->max_length = max_length;
4580 bounds = (MonoArrayBounds*)((char*)arr + size - bounds_size);
4581 arr->bounds = bounds;
4589 mono_gc_alloc_string (MonoVTable *vtable, size_t size, gint32 len)
4592 #ifndef DISABLE_CRITICAL_REGION
4594 ENTER_CRITICAL_REGION;
4595 str = mono_gc_try_alloc_obj_nolock (vtable, size);
4598 EXIT_CRITICAL_REGION;
4601 EXIT_CRITICAL_REGION;
4606 str = mono_gc_alloc_obj_nolock (vtable, size);
4615 * To be used for interned strings and possibly MonoThread, reflection handles.
4616 * We may want to explicitly free these objects.
4619 mono_gc_alloc_pinned_obj (MonoVTable *vtable, size_t size)
4621 /* FIXME: handle OOM */
4623 size += ALLOC_ALIGN - 1;
4624 size &= ~(ALLOC_ALIGN - 1);
4626 if (size > MAX_FREELIST_SIZE) {
4627 /* large objects are always pinned anyway */
4628 p = alloc_large_inner (vtable, size);
4630 p = alloc_from_freelist (size);
4631 memset (p, 0, size);
4633 DEBUG (6, fprintf (gc_debug_file, "Allocated pinned object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
4640 * ######################################################################
4641 * ######## Finalization support
4642 * ######################################################################
4646 * this is valid for the nursery: if the object has been forwarded it means it's
4647 * still refrenced from a root. If it is pinned it's still alive as well.
4648 * Return TRUE if @obj is ready to be finalized.
4650 #define object_is_fin_ready(obj) (!object_is_pinned (obj) && !object_is_forwarded (obj))
4653 is_critical_finalizer (FinalizeEntry *entry)
4658 if (!mono_defaults.critical_finalizer_object)
4661 obj = entry->object;
4662 class = ((MonoVTable*)LOAD_VTABLE (obj))->klass;
4664 return mono_class_has_parent (class, mono_defaults.critical_finalizer_object);
4668 queue_finalization_entry (FinalizeEntry *entry) {
4669 if (is_critical_finalizer (entry)) {
4670 entry->next = critical_fin_list;
4671 critical_fin_list = entry;
4673 entry->next = fin_ready_list;
4674 fin_ready_list = entry;
4678 /* LOCKING: requires that the GC lock is held */
4680 rehash_fin_table (FinalizeEntryHashTable *hash_table)
4682 FinalizeEntry **finalizable_hash = hash_table->table;
4683 mword finalizable_hash_size = hash_table->size;
4686 FinalizeEntry **new_hash;
4687 FinalizeEntry *entry, *next;
4688 int new_size = g_spaced_primes_closest (hash_table->num_registered);
4690 new_hash = get_internal_mem (new_size * sizeof (FinalizeEntry*), INTERNAL_MEM_FIN_TABLE);
4691 for (i = 0; i < finalizable_hash_size; ++i) {
4692 for (entry = finalizable_hash [i]; entry; entry = next) {
4693 hash = mono_object_hash (entry->object) % new_size;
4695 entry->next = new_hash [hash];
4696 new_hash [hash] = entry;
4699 free_internal_mem (finalizable_hash, INTERNAL_MEM_FIN_TABLE);
4700 hash_table->table = new_hash;
4701 hash_table->size = new_size;
4704 /* LOCKING: requires that the GC lock is held */
4706 rehash_fin_table_if_necessary (FinalizeEntryHashTable *hash_table)
4708 if (hash_table->num_registered >= hash_table->size * 2)
4709 rehash_fin_table (hash_table);
4712 /* LOCKING: requires that the GC lock is held */
4714 finalize_in_range (char *start, char *end, int generation)
4716 FinalizeEntryHashTable *hash_table = get_finalize_entry_hash_table (generation);
4717 FinalizeEntry *entry, *prev;
4719 FinalizeEntry **finalizable_hash = hash_table->table;
4720 mword finalizable_hash_size = hash_table->size;
4724 for (i = 0; i < finalizable_hash_size; ++i) {
4726 for (entry = finalizable_hash [i]; entry;) {
4727 if ((char*)entry->object >= start && (char*)entry->object < end && !object_is_in_to_space (entry->object)) {
4728 gboolean is_fin_ready = object_is_fin_ready (entry->object);
4729 char *copy = copy_object (entry->object, start, end);
4732 FinalizeEntry *next;
4733 /* remove and put in fin_ready_list */
4735 prev->next = entry->next;
4737 finalizable_hash [i] = entry->next;
4739 num_ready_finalizers++;
4740 hash_table->num_registered--;
4741 queue_finalization_entry (entry);
4742 /* Make it survive */
4743 from = entry->object;
4744 entry->object = copy;
4745 DEBUG (5, fprintf (gc_debug_file, "Queueing object for finalization: %p (%s) (was at %p) (%d/%d)\n", entry->object, safe_name (entry->object), from, num_ready_finalizers, hash_table->num_registered));
4749 char *from = entry->object;
4750 if (hash_table == &minor_finalizable_hash && !ptr_in_nursery (copy)) {
4751 FinalizeEntry *next = entry->next;
4752 unsigned int major_hash;
4753 /* remove from the list */
4755 prev->next = entry->next;
4757 finalizable_hash [i] = entry->next;
4758 hash_table->num_registered--;
4760 entry->object = copy;
4762 /* insert it into the major hash */
4763 rehash_fin_table_if_necessary (&major_finalizable_hash);
4764 major_hash = mono_object_hash ((MonoObject*) copy) %
4765 major_finalizable_hash.size;
4766 entry->next = major_finalizable_hash.table [major_hash];
4767 major_finalizable_hash.table [major_hash] = entry;
4768 major_finalizable_hash.num_registered++;
4770 DEBUG (5, fprintf (gc_debug_file, "Promoting finalization of object %p (%s) (was at %p) to major table\n", copy, safe_name (copy), from));
4775 /* update pointer */
4776 DEBUG (5, fprintf (gc_debug_file, "Updating object for finalization: %p (%s) (was at %p)\n", entry->object, safe_name (entry->object), from));
4777 entry->object = copy;
4782 entry = entry->next;
4787 /* LOCKING: requires that the GC lock is held */
4789 null_link_in_range (char *start, char *end, int generation)
4791 DisappearingLinkHashTable *hash = get_dislink_hash_table (generation);
4792 DisappearingLink **disappearing_link_hash = hash->table;
4793 int disappearing_link_hash_size = hash->size;
4794 DisappearingLink *entry, *prev;
4796 if (!hash->num_links)
4798 for (i = 0; i < disappearing_link_hash_size; ++i) {
4800 for (entry = disappearing_link_hash [i]; entry;) {
4801 char *object = DISLINK_OBJECT (entry);
4802 if (object >= start && object < end && !object_is_in_to_space (object)) {
4803 gboolean track = DISLINK_TRACK (entry);
4804 if (!track && object_is_fin_ready (object)) {
4805 void **p = entry->link;
4806 DisappearingLink *old;
4808 /* remove from list */
4810 prev->next = entry->next;
4812 disappearing_link_hash [i] = entry->next;
4813 DEBUG (5, fprintf (gc_debug_file, "Dislink nullified at %p to GCed object %p\n", p, object));
4815 free_internal_mem (entry, INTERNAL_MEM_DISLINK);
4820 char *copy = copy_object (object, start, end);
4822 /* Update pointer if it's moved. If the object
4823 * has been moved out of the nursery, we need to
4824 * remove the link from the minor hash table to
4827 * FIXME: what if an object is moved earlier?
4830 if (hash == &minor_disappearing_link_hash && !ptr_in_nursery (copy)) {
4831 void **link = entry->link;
4832 DisappearingLink *old;
4833 /* remove from list */
4835 prev->next = entry->next;
4837 disappearing_link_hash [i] = entry->next;
4839 free_internal_mem (entry, INTERNAL_MEM_DISLINK);
4843 add_or_remove_disappearing_link ((MonoObject*)copy, link,
4844 track, GENERATION_OLD);
4846 DEBUG (5, fprintf (gc_debug_file, "Upgraded dislink at %p to major because object %p moved to %p\n", link, object, copy));
4850 /* We set the track resurrection bit to
4851 * FALSE if the object is to be finalized
4852 * so that the object can be collected in
4853 * the next cycle (i.e. after it was
4856 *entry->link = HIDE_POINTER (copy,
4857 object_is_fin_ready (object) ? FALSE : track);
4858 DEBUG (5, fprintf (gc_debug_file, "Updated dislink at %p to %p\n", entry->link, DISLINK_OBJECT (entry)));
4863 entry = entry->next;
4868 /* LOCKING: requires that the GC lock is held */
4870 null_links_for_domain (MonoDomain *domain, int generation)
4872 DisappearingLinkHashTable *hash = get_dislink_hash_table (generation);
4873 DisappearingLink **disappearing_link_hash = hash->table;
4874 int disappearing_link_hash_size = hash->size;
4875 DisappearingLink *entry, *prev;
4877 for (i = 0; i < disappearing_link_hash_size; ++i) {
4879 for (entry = disappearing_link_hash [i]; entry; ) {
4880 char *object = DISLINK_OBJECT (entry);
4881 /* FIXME: actually there should be no object
4882 left in the domain with a non-null vtable
4883 (provided we remove the Thread special
4885 if (object && (!((MonoObject*)object)->vtable || mono_object_domain (object) == domain)) {
4886 DisappearingLink *next = entry->next;
4891 disappearing_link_hash [i] = next;
4893 if (*(entry->link)) {
4894 *(entry->link) = NULL;
4895 g_warning ("Disappearing link %p not freed", entry->link);
4897 free_internal_mem (entry, INTERNAL_MEM_DISLINK);
4904 entry = entry->next;
4909 /* LOCKING: requires that the GC lock is held */
4911 finalizers_for_domain (MonoDomain *domain, MonoObject **out_array, int out_size,
4912 FinalizeEntryHashTable *hash_table)
4914 FinalizeEntry **finalizable_hash = hash_table->table;
4915 mword finalizable_hash_size = hash_table->size;
4916 FinalizeEntry *entry, *prev;
4919 if (no_finalize || !out_size || !out_array)
4922 for (i = 0; i < finalizable_hash_size; ++i) {
4924 for (entry = finalizable_hash [i]; entry;) {
4925 if (mono_object_domain (entry->object) == domain) {
4926 FinalizeEntry *next;
4927 /* remove and put in out_array */
4929 prev->next = entry->next;
4931 finalizable_hash [i] = entry->next;
4933 hash_table->num_registered--;
4934 out_array [count ++] = entry->object;
4935 DEBUG (5, fprintf (gc_debug_file, "Collecting object for finalization: %p (%s) (%d/%d)\n", entry->object, safe_name (entry->object), num_ready_finalizers, hash_table->num_registered));
4937 if (count == out_size)
4942 entry = entry->next;
4949 * mono_gc_finalizers_for_domain:
4950 * @domain: the unloading appdomain
4951 * @out_array: output array
4952 * @out_size: size of output array
4954 * Store inside @out_array up to @out_size objects that belong to the unloading
4955 * appdomain @domain. Returns the number of stored items. Can be called repeteadly
4956 * until it returns 0.
4957 * The items are removed from the finalizer data structure, so the caller is supposed
4959 * @out_array should be on the stack to allow the GC to know the objects are still alive.
4962 mono_gc_finalizers_for_domain (MonoDomain *domain, MonoObject **out_array, int out_size)
4967 result = finalizers_for_domain (domain, out_array, out_size, &minor_finalizable_hash);
4968 if (result < out_size) {
4969 result += finalizers_for_domain (domain, out_array + result, out_size - result,
4970 &major_finalizable_hash);
4978 register_for_finalization (MonoObject *obj, void *user_data, int generation)
4980 FinalizeEntryHashTable *hash_table = get_finalize_entry_hash_table (generation);
4981 FinalizeEntry **finalizable_hash;
4982 mword finalizable_hash_size;
4983 FinalizeEntry *entry, *prev;
4987 g_assert (user_data == NULL || user_data == mono_gc_run_finalize);
4988 hash = mono_object_hash (obj);
4990 rehash_fin_table_if_necessary (hash_table);
4991 finalizable_hash = hash_table->table;
4992 finalizable_hash_size = hash_table->size;
4993 hash %= finalizable_hash_size;
4995 for (entry = finalizable_hash [hash]; entry; entry = entry->next) {
4996 if (entry->object == obj) {
4998 /* remove from the list */
5000 prev->next = entry->next;
5002 finalizable_hash [hash] = entry->next;
5003 hash_table->num_registered--;
5004 DEBUG (5, fprintf (gc_debug_file, "Removed finalizer %p for object: %p (%s) (%d)\n", entry, obj, obj->vtable->klass->name, hash_table->num_registered));
5005 free_internal_mem (entry, INTERNAL_MEM_FINALIZE_ENTRY);
5013 /* request to deregister, but already out of the list */
5017 entry = get_internal_mem (sizeof (FinalizeEntry), INTERNAL_MEM_FINALIZE_ENTRY);
5018 entry->object = obj;
5019 entry->next = finalizable_hash [hash];
5020 finalizable_hash [hash] = entry;
5021 hash_table->num_registered++;
5022 DEBUG (5, fprintf (gc_debug_file, "Added finalizer %p for object: %p (%s) (%d) to %s table\n", entry, obj, obj->vtable->klass->name, hash_table->num_registered, generation_name (generation)));
5027 mono_gc_register_for_finalization (MonoObject *obj, void *user_data)
5029 if (ptr_in_nursery (obj))
5030 register_for_finalization (obj, user_data, GENERATION_NURSERY);
5032 register_for_finalization (obj, user_data, GENERATION_OLD);
5036 rehash_dislink (DisappearingLinkHashTable *hash_table)
5038 DisappearingLink **disappearing_link_hash = hash_table->table;
5039 int disappearing_link_hash_size = hash_table->size;
5042 DisappearingLink **new_hash;
5043 DisappearingLink *entry, *next;
5044 int new_size = g_spaced_primes_closest (hash_table->num_links);
5046 new_hash = get_internal_mem (new_size * sizeof (DisappearingLink*), INTERNAL_MEM_DISLINK_TABLE);
5047 for (i = 0; i < disappearing_link_hash_size; ++i) {
5048 for (entry = disappearing_link_hash [i]; entry; entry = next) {
5049 hash = mono_aligned_addr_hash (entry->link) % new_size;
5051 entry->next = new_hash [hash];
5052 new_hash [hash] = entry;
5055 free_internal_mem (disappearing_link_hash, INTERNAL_MEM_DISLINK_TABLE);
5056 hash_table->table = new_hash;
5057 hash_table->size = new_size;
5060 /* LOCKING: assumes the GC lock is held */
5062 add_or_remove_disappearing_link (MonoObject *obj, void **link, gboolean track, int generation)
5064 DisappearingLinkHashTable *hash_table = get_dislink_hash_table (generation);
5065 DisappearingLink *entry, *prev;
5067 DisappearingLink **disappearing_link_hash = hash_table->table;
5068 int disappearing_link_hash_size = hash_table->size;
5070 if (hash_table->num_links >= disappearing_link_hash_size * 2) {
5071 rehash_dislink (hash_table);
5072 disappearing_link_hash = hash_table->table;
5073 disappearing_link_hash_size = hash_table->size;
5075 /* FIXME: add check that link is not in the heap */
5076 hash = mono_aligned_addr_hash (link) % disappearing_link_hash_size;
5077 entry = disappearing_link_hash [hash];
5079 for (; entry; entry = entry->next) {
5080 /* link already added */
5081 if (link == entry->link) {
5082 /* NULL obj means remove */
5085 prev->next = entry->next;
5087 disappearing_link_hash [hash] = entry->next;
5088 hash_table->num_links--;
5089 DEBUG (5, fprintf (gc_debug_file, "Removed dislink %p (%d) from %s table\n", entry, hash_table->num_links, generation_name (generation)));
5090 free_internal_mem (entry, INTERNAL_MEM_DISLINK);
5093 *link = HIDE_POINTER (obj, track); /* we allow the change of object */
5101 entry = get_internal_mem (sizeof (DisappearingLink), INTERNAL_MEM_DISLINK);
5102 *link = HIDE_POINTER (obj, track);
5104 entry->next = disappearing_link_hash [hash];
5105 disappearing_link_hash [hash] = entry;
5106 hash_table->num_links++;
5107 DEBUG (5, fprintf (gc_debug_file, "Added dislink %p for object: %p (%s) at %p to %s table\n", entry, obj, obj->vtable->klass->name, link, generation_name (generation)));
5110 /* LOCKING: assumes the GC lock is held */
5112 mono_gc_register_disappearing_link (MonoObject *obj, void **link, gboolean track)
5114 add_or_remove_disappearing_link (NULL, link, FALSE, GENERATION_NURSERY);
5115 add_or_remove_disappearing_link (NULL, link, FALSE, GENERATION_OLD);
5117 if (ptr_in_nursery (obj))
5118 add_or_remove_disappearing_link (obj, link, track, GENERATION_NURSERY);
5120 add_or_remove_disappearing_link (obj, link, track, GENERATION_OLD);
5125 mono_gc_invoke_finalizers (void)
5127 FinalizeEntry *entry = NULL;
5128 gboolean entry_is_critical = FALSE;
5131 /* FIXME: batch to reduce lock contention */
5132 while (fin_ready_list || critical_fin_list) {
5136 FinalizeEntry **list = entry_is_critical ? &critical_fin_list : &fin_ready_list;
5138 /* We have finalized entry in the last
5139 interation, now we need to remove it from
5142 *list = entry->next;
5144 FinalizeEntry *e = *list;
5145 while (e->next != entry)
5147 e->next = entry->next;
5149 free_internal_mem (entry, INTERNAL_MEM_FINALIZE_ENTRY);
5153 /* Now look for the first non-null entry. */
5154 for (entry = fin_ready_list; entry && !entry->object; entry = entry->next)
5157 entry_is_critical = FALSE;
5159 entry_is_critical = TRUE;
5160 for (entry = critical_fin_list; entry && !entry->object; entry = entry->next)
5165 g_assert (entry->object);
5166 num_ready_finalizers--;
5167 obj = entry->object;
5168 entry->object = NULL;
5169 DEBUG (7, fprintf (gc_debug_file, "Finalizing object %p (%s)\n", obj, safe_name (obj)));
5177 g_assert (entry->object == NULL);
5179 /* the object is on the stack so it is pinned */
5180 /*g_print ("Calling finalizer for object: %p (%s)\n", entry->object, safe_name (entry->object));*/
5181 mono_gc_run_finalize (obj, NULL);
5188 mono_gc_pending_finalizers (void)
5190 return fin_ready_list || critical_fin_list;
5193 /* Negative value to remove */
5195 mono_gc_add_memory_pressure (gint64 value)
5197 /* FIXME: Use interlocked functions */
5199 memory_pressure += value;
5204 * ######################################################################
5205 * ######## registered roots support
5206 * ######################################################################
5210 rehash_roots (gboolean pinned)
5214 RootRecord **new_hash;
5215 RootRecord *entry, *next;
5218 new_size = g_spaced_primes_closest (num_roots_entries [pinned]);
5219 new_hash = get_internal_mem (new_size * sizeof (RootRecord*), INTERNAL_MEM_ROOTS_TABLE);
5220 for (i = 0; i < roots_hash_size [pinned]; ++i) {
5221 for (entry = roots_hash [pinned][i]; entry; entry = next) {
5222 hash = mono_aligned_addr_hash (entry->start_root) % new_size;
5224 entry->next = new_hash [hash];
5225 new_hash [hash] = entry;
5228 free_internal_mem (roots_hash [pinned], INTERNAL_MEM_ROOTS_TABLE);
5229 roots_hash [pinned] = new_hash;
5230 roots_hash_size [pinned] = new_size;
5234 find_root (int root_type, char *start, guint32 addr_hash)
5236 RootRecord *new_root;
5238 guint32 hash = addr_hash % roots_hash_size [root_type];
5239 for (new_root = roots_hash [root_type][hash]; new_root; new_root = new_root->next) {
5240 /* we allow changing the size and the descriptor (for thread statics etc) */
5241 if (new_root->start_root == start) {
5250 * We do not coalesce roots.
5253 mono_gc_register_root_inner (char *start, size_t size, void *descr, int root_type)
5255 RootRecord *new_root;
5256 unsigned int hash, addr_hash = mono_aligned_addr_hash (start);
5259 for (i = 0; i < ROOT_TYPE_NUM; ++i) {
5260 if (num_roots_entries [i] >= roots_hash_size [i] * 2)
5263 for (i = 0; i < ROOT_TYPE_NUM; ++i) {
5264 new_root = find_root (i, start, addr_hash);
5265 /* we allow changing the size and the descriptor (for thread statics etc) */
5267 size_t old_size = new_root->end_root - new_root->start_root;
5268 new_root->end_root = new_root->start_root + size;
5269 g_assert (((new_root->root_desc != 0) && (descr != NULL)) ||
5270 ((new_root->root_desc == 0) && (descr == NULL)));
5271 new_root->root_desc = (mword)descr;
5273 roots_size -= old_size;
5278 new_root = get_internal_mem (sizeof (RootRecord), INTERNAL_MEM_ROOT_RECORD);
5280 new_root->start_root = start;
5281 new_root->end_root = new_root->start_root + size;
5282 new_root->root_desc = (mword)descr;
5284 hash = addr_hash % roots_hash_size [root_type];
5285 num_roots_entries [root_type]++;
5286 new_root->next = roots_hash [root_type] [hash];
5287 roots_hash [root_type][hash] = new_root;
5288 DEBUG (3, fprintf (gc_debug_file, "Added root %p for range: %p-%p, descr: %p (%d/%d bytes)\n", new_root, new_root->start_root, new_root->end_root, descr, (int)size, (int)roots_size));
5298 mono_gc_register_root (char *start, size_t size, void *descr)
5300 return mono_gc_register_root_inner (start, size, descr, descr ? ROOT_TYPE_NORMAL : ROOT_TYPE_PINNED);
5304 mono_gc_register_root_wbarrier (char *start, size_t size, void *descr)
5306 return mono_gc_register_root_inner (start, size, descr, ROOT_TYPE_WBARRIER);
5310 mono_gc_deregister_root (char* addr)
5312 RootRecord *tmp, *prev;
5313 unsigned int hash, addr_hash = mono_aligned_addr_hash (addr);
5317 for (root_type = 0; root_type < ROOT_TYPE_NUM; ++root_type) {
5318 hash = addr_hash % roots_hash_size [root_type];
5319 tmp = roots_hash [root_type][hash];
5322 if (tmp->start_root == (char*)addr) {
5324 prev->next = tmp->next;
5326 roots_hash [root_type][hash] = tmp->next;
5327 roots_size -= (tmp->end_root - tmp->start_root);
5328 num_roots_entries [root_type]--;
5329 DEBUG (3, fprintf (gc_debug_file, "Removed root %p for range: %p-%p\n", tmp, tmp->start_root, tmp->end_root));
5330 free_internal_mem (tmp, INTERNAL_MEM_ROOT_RECORD);
5341 * ######################################################################
5342 * ######## Thread handling (stop/start code)
5343 * ######################################################################
5346 /* FIXME: handle large/small config */
5347 #define THREAD_HASH_SIZE 11
5348 #define HASH_PTHREAD_T(id) (((unsigned int)(id) >> 4) * 2654435761u)
5350 static SgenThreadInfo* thread_table [THREAD_HASH_SIZE];
5352 #if USE_SIGNAL_BASED_START_STOP_WORLD
5354 static MonoSemType suspend_ack_semaphore;
5355 static MonoSemType *suspend_ack_semaphore_ptr;
5356 static unsigned int global_stop_count = 0;
5358 static int suspend_signal_num = SIGXFSZ;
5360 static int suspend_signal_num = SIGPWR;
5362 static int restart_signal_num = SIGXCPU;
5363 static sigset_t suspend_signal_mask;
5364 static mword cur_thread_regs [ARCH_NUM_REGS] = {0};
5366 /* LOCKING: assumes the GC lock is held */
5367 static SgenThreadInfo*
5368 thread_info_lookup (ARCH_THREAD_TYPE id)
5370 unsigned int hash = HASH_PTHREAD_T (id) % THREAD_HASH_SIZE;
5371 SgenThreadInfo *info;
5373 info = thread_table [hash];
5374 while (info && !ARCH_THREAD_EQUALS (info->id, id)) {
5381 update_current_thread_stack (void *start)
5383 void *ptr = cur_thread_regs;
5384 SgenThreadInfo *info = thread_info_lookup (ARCH_GET_THREAD ());
5385 info->stack_start = align_pointer (&ptr);
5386 g_assert (info->stack_start >= info->stack_start_limit && info->stack_start < info->stack_end);
5387 ARCH_STORE_REGS (ptr);
5388 info->stopped_regs = ptr;
5389 if (gc_callbacks.thread_suspend_func)
5390 gc_callbacks.thread_suspend_func (info->runtime_data, NULL);
5394 signal_desc (int signum)
5396 if (signum == suspend_signal_num)
5398 if (signum == restart_signal_num)
5404 * Define this and use the "xdomain-checks" MONO_GC_DEBUG option to
5405 * have cross-domain checks in the write barrier.
5407 //#define XDOMAIN_CHECKS_IN_WBARRIER
5409 #ifndef HEAVY_STATISTICS
5410 #define MANAGED_ALLOCATION
5411 #ifndef XDOMAIN_CHECKS_IN_WBARRIER
5412 #define MANAGED_WBARRIER
5417 is_ip_in_managed_allocator (MonoDomain *domain, gpointer ip);
5420 wait_for_suspend_ack (int count)
5424 for (i = 0; i < count; ++i) {
5425 while ((result = MONO_SEM_WAIT (suspend_ack_semaphore_ptr)) != 0) {
5426 if (errno != EINTR) {
5427 g_error ("sem_wait ()");
5433 /* LOCKING: assumes the GC lock is held */
5435 thread_handshake (int signum)
5437 int count, i, result;
5438 SgenThreadInfo *info;
5439 pthread_t me = pthread_self ();
5442 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5443 for (info = thread_table [i]; info; info = info->next) {
5444 DEBUG (4, fprintf (gc_debug_file, "considering thread %p for signal %d (%s)\n", info, signum, signal_desc (signum)));
5445 if (ARCH_THREAD_EQUALS (info->id, me)) {
5446 DEBUG (4, fprintf (gc_debug_file, "Skip (equal): %p, %p\n", (void*)me, (void*)info->id));
5449 /*if (signum == suspend_signal_num && info->stop_count == global_stop_count)
5451 result = pthread_kill (info->id, signum);
5453 DEBUG (4, fprintf (gc_debug_file, "thread %p signal sent\n", info));
5456 DEBUG (4, fprintf (gc_debug_file, "thread %p signal failed: %d (%s)\n", (void*)info->id, result, strerror (result)));
5462 wait_for_suspend_ack (count);
5468 restart_threads_until_none_in_managed_allocator (void)
5470 SgenThreadInfo *info;
5471 int i, result, num_threads_died = 0;
5472 int sleep_duration = -1;
5475 int restart_count = 0, restarted_count = 0;
5476 /* restart all threads that stopped in the
5478 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5479 for (info = thread_table [i]; info; info = info->next) {
5482 if (!info->stack_start || info->in_critical_region ||
5483 is_ip_in_managed_allocator (info->stopped_domain, info->stopped_ip)) {
5484 result = pthread_kill (info->id, restart_signal_num);
5491 /* we set the stopped_ip to
5492 NULL for threads which
5493 we're not restarting so
5494 that we can easily identify
5496 info->stopped_ip = NULL;
5497 info->stopped_domain = NULL;
5501 /* if no threads were restarted, we're done */
5502 if (restart_count == 0)
5505 /* wait for the threads to signal their restart */
5506 wait_for_suspend_ack (restart_count);
5508 if (sleep_duration < 0) {
5512 g_usleep (sleep_duration);
5513 sleep_duration += 10;
5516 /* stop them again */
5517 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5518 for (info = thread_table [i]; info; info = info->next) {
5519 if (info->skip || info->stopped_ip == NULL)
5521 result = pthread_kill (info->id, suspend_signal_num);
5529 /* some threads might have died */
5530 num_threads_died += restart_count - restarted_count;
5531 /* wait for the threads to signal their suspension
5533 wait_for_suspend_ack (restart_count);
5536 return num_threads_died;
5539 /* LOCKING: assumes the GC lock is held (by the stopping thread) */
5541 suspend_handler (int sig, siginfo_t *siginfo, void *context)
5543 SgenThreadInfo *info;
5546 int old_errno = errno;
5547 gpointer regs [ARCH_NUM_REGS];
5548 gpointer stack_start;
5550 id = pthread_self ();
5551 info = thread_info_lookup (id);
5552 info->stopped_domain = mono_domain_get ();
5553 info->stopped_ip = (gpointer) ARCH_SIGCTX_IP (context);
5554 stop_count = global_stop_count;
5555 /* duplicate signal */
5556 if (0 && info->stop_count == stop_count) {
5560 #ifdef HAVE_KW_THREAD
5561 /* update the remset info in the thread data structure */
5562 info->remset = remembered_set;
5564 stack_start = (char*) ARCH_SIGCTX_SP (context) - REDZONE_SIZE;
5565 /* If stack_start is not within the limits, then don't set it
5566 in info and we will be restarted. */
5567 if (stack_start >= info->stack_start_limit && info->stack_start <= info->stack_end) {
5568 info->stack_start = stack_start;
5570 ARCH_COPY_SIGCTX_REGS (regs, context);
5571 info->stopped_regs = regs;
5573 g_assert (!info->stack_start);
5576 /* Notify the JIT */
5577 if (gc_callbacks.thread_suspend_func)
5578 gc_callbacks.thread_suspend_func (info->runtime_data, context);
5580 DEBUG (4, fprintf (gc_debug_file, "Posting suspend_ack_semaphore for suspend from %p %p\n", info, (gpointer)ARCH_GET_THREAD ()));
5581 /* notify the waiting thread */
5582 MONO_SEM_POST (suspend_ack_semaphore_ptr);
5583 info->stop_count = stop_count;
5585 /* wait until we receive the restart signal */
5588 sigsuspend (&suspend_signal_mask);
5589 } while (info->signal != restart_signal_num);
5591 DEBUG (4, fprintf (gc_debug_file, "Posting suspend_ack_semaphore for resume from %p %p\n", info, (gpointer)ARCH_GET_THREAD ()));
5592 /* notify the waiting thread */
5593 MONO_SEM_POST (suspend_ack_semaphore_ptr);
5599 restart_handler (int sig)
5601 SgenThreadInfo *info;
5602 int old_errno = errno;
5604 info = thread_info_lookup (pthread_self ());
5605 info->signal = restart_signal_num;
5606 DEBUG (4, fprintf (gc_debug_file, "Restart handler in %p %p\n", info, (gpointer)ARCH_GET_THREAD ()));
5611 static TV_DECLARE (stop_world_time);
5612 static unsigned long max_pause_usec = 0;
5614 /* LOCKING: assumes the GC lock is held */
5620 update_current_thread_stack (&count);
5622 global_stop_count++;
5623 DEBUG (3, fprintf (gc_debug_file, "stopping world n %d from %p %p\n", global_stop_count, thread_info_lookup (ARCH_GET_THREAD ()), (gpointer)ARCH_GET_THREAD ()));
5624 TV_GETTIME (stop_world_time);
5625 count = thread_handshake (suspend_signal_num);
5626 count -= restart_threads_until_none_in_managed_allocator ();
5627 g_assert (count >= 0);
5628 DEBUG (3, fprintf (gc_debug_file, "world stopped %d thread(s)\n", count));
5632 /* LOCKING: assumes the GC lock is held */
5634 restart_world (void)
5637 SgenThreadInfo *info;
5638 TV_DECLARE (end_sw);
5641 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5642 for (info = thread_table [i]; info; info = info->next) {
5643 info->stack_start = NULL;
5644 info->stopped_regs = NULL;
5648 count = thread_handshake (restart_signal_num);
5649 TV_GETTIME (end_sw);
5650 usec = TV_ELAPSED (stop_world_time, end_sw);
5651 max_pause_usec = MAX (usec, max_pause_usec);
5652 DEBUG (2, fprintf (gc_debug_file, "restarted %d thread(s) (pause time: %d usec, max: %d)\n", count, (int)usec, (int)max_pause_usec));
5656 #endif /* USE_SIGNAL_BASED_START_STOP_WORLD */
5659 mono_gc_set_gc_callbacks (MonoGCCallbacks *callbacks)
5661 gc_callbacks = *callbacks;
5664 /* Variables holding start/end nursery so it won't have to be passed at every call */
5665 static void *scan_area_arg_start, *scan_area_arg_end;
5668 mono_gc_conservatively_scan_area (void *start, void *end)
5670 conservatively_pin_objects_from (start, end, scan_area_arg_start, scan_area_arg_end, PIN_TYPE_STACK);
5674 mono_gc_scan_object (void *obj)
5676 return copy_object (obj, scan_area_arg_start, scan_area_arg_end);
5680 * Mark from thread stacks and registers.
5683 scan_thread_data (void *start_nursery, void *end_nursery, gboolean precise)
5686 SgenThreadInfo *info;
5688 scan_area_arg_start = start_nursery;
5689 scan_area_arg_end = end_nursery;
5691 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5692 for (info = thread_table [i]; info; info = info->next) {
5694 DEBUG (2, fprintf (gc_debug_file, "Skipping dead thread %p, range: %p-%p, size: %zd\n", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start));
5697 DEBUG (3, fprintf (gc_debug_file, "Scanning thread %p, range: %p-%p, size: %zd, pinned=%d\n", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start, next_pin_slot));
5698 if (gc_callbacks.thread_mark_func && !conservative_stack_mark)
5699 gc_callbacks.thread_mark_func (info->runtime_data, info->stack_start, info->stack_end, precise);
5701 conservatively_pin_objects_from (info->stack_start, info->stack_end, start_nursery, end_nursery, PIN_TYPE_STACK);
5704 conservatively_pin_objects_from (info->stopped_regs, info->stopped_regs + ARCH_NUM_REGS,
5705 start_nursery, end_nursery, PIN_TYPE_STACK);
5711 find_pinning_ref_from_thread (char *obj, size_t size)
5714 SgenThreadInfo *info;
5715 char *endobj = obj + size;
5717 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5718 for (info = thread_table [i]; info; info = info->next) {
5719 char **start = (char**)info->stack_start;
5722 while (start < (char**)info->stack_end) {
5723 if (*start >= obj && *start < endobj) {
5724 DEBUG (0, fprintf (gc_debug_file, "Object %p referenced in thread %p (id %p) at %p, stack: %p-%p\n", obj, info, (gpointer)info->id, start, info->stack_start, info->stack_end));
5729 /* FIXME: check info->stopped_regs */
5735 ptr_on_stack (void *ptr)
5737 gpointer stack_start = &stack_start;
5738 SgenThreadInfo *info = thread_info_lookup (ARCH_GET_THREAD ());
5740 if (ptr >= stack_start && ptr < (gpointer)info->stack_end)
5746 handle_remset (mword *p, void *start_nursery, void *end_nursery, gboolean global)
5753 HEAVY_STAT (++stat_global_remsets_processed);
5755 /* FIXME: exclude stack locations */
5756 switch ((*p) & REMSET_TYPE_MASK) {
5757 case REMSET_LOCATION:
5759 //__builtin_prefetch (ptr);
5760 if (((void*)ptr < start_nursery || (void*)ptr >= end_nursery)) {
5761 *ptr = copy_object (*ptr, start_nursery, end_nursery);
5762 DEBUG (9, fprintf (gc_debug_file, "Overwrote remset at %p with %p\n", ptr, *ptr));
5763 if (!global && *ptr >= start_nursery && *ptr < end_nursery) {
5765 * If the object is pinned, each reference to it from nonpinned objects
5766 * becomes part of the global remset, which can grow very large.
5768 DEBUG (9, fprintf (gc_debug_file, "Add to global remset because of pinning %p (%p %s)\n", ptr, *ptr, safe_name (*ptr)));
5769 add_to_global_remset (ptr, FALSE);
5772 DEBUG (9, fprintf (gc_debug_file, "Skipping remset at %p holding %p\n", ptr, *ptr));
5776 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
5777 if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery))
5780 while (count-- > 0) {
5781 *ptr = copy_object (*ptr, start_nursery, end_nursery);
5782 DEBUG (9, fprintf (gc_debug_file, "Overwrote remset at %p with %p (count: %d)\n", ptr, *ptr, (int)count));
5783 if (!global && *ptr >= start_nursery && *ptr < end_nursery)
5784 add_to_global_remset (ptr, FALSE);
5789 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
5790 if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery))
5792 scan_object ((char*)ptr, start_nursery, end_nursery);
5794 case REMSET_OTHER: {
5795 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
5799 if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery))
5804 ptr = (void**) scan_vtype ((char*)ptr, desc, start_nursery, end_nursery);
5806 case REMSET_ROOT_LOCATION:
5807 /* Same as REMSET_LOCATION, but the address is not required to be in the heap */
5808 *ptr = copy_object (*ptr, start_nursery, end_nursery);
5809 DEBUG (9, fprintf (gc_debug_file, "Overwrote root location remset at %p with %p\n", ptr, *ptr));
5810 if (!global && *ptr >= start_nursery && *ptr < end_nursery) {
5812 * If the object is pinned, each reference to it from nonpinned objects
5813 * becomes part of the global remset, which can grow very large.
5815 DEBUG (9, fprintf (gc_debug_file, "Add to global remset because of pinning %p (%p %s)\n", ptr, *ptr, safe_name (*ptr)));
5816 add_to_global_remset (ptr, TRUE);
5820 g_assert_not_reached ();
5825 g_assert_not_reached ();
5830 #ifdef HEAVY_STATISTICS
5832 collect_store_remsets (RememberedSet *remset, mword *bumper)
5834 mword *p = remset->data;
5839 while (p < remset->store_next) {
5840 switch ((*p) & REMSET_TYPE_MASK) {
5841 case REMSET_LOCATION:
5844 ++stat_saved_remsets_1;
5846 if (*p == last1 || *p == last2) {
5847 ++stat_saved_remsets_2;
5865 case REMSET_ROOT_LOCATION:
5869 g_assert_not_reached ();
5873 g_assert_not_reached ();
5883 RememberedSet *remset;
5885 SgenThreadInfo *info;
5887 mword *addresses, *bumper, *p, *r;
5889 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5890 for (info = thread_table [i]; info; info = info->next) {
5891 for (remset = info->remset; remset; remset = remset->next)
5892 size += remset->store_next - remset->data;
5895 for (remset = freed_thread_remsets; remset; remset = remset->next)
5896 size += remset->store_next - remset->data;
5897 for (remset = global_remset; remset; remset = remset->next)
5898 size += remset->store_next - remset->data;
5900 bumper = addresses = get_internal_mem (sizeof (mword) * size, INTERNAL_MEM_STATISTICS);
5902 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5903 for (info = thread_table [i]; info; info = info->next) {
5904 for (remset = info->remset; remset; remset = remset->next)
5905 bumper = collect_store_remsets (remset, bumper);
5908 for (remset = global_remset; remset; remset = remset->next)
5909 bumper = collect_store_remsets (remset, bumper);
5910 for (remset = freed_thread_remsets; remset; remset = remset->next)
5911 bumper = collect_store_remsets (remset, bumper);
5913 g_assert (bumper <= addresses + size);
5915 stat_store_remsets += bumper - addresses;
5917 sort_addresses ((void**)addresses, bumper - addresses);
5920 while (r < bumper) {
5926 stat_store_remsets_unique += p - addresses;
5928 free_internal_mem (addresses, INTERNAL_MEM_STATISTICS);
5933 clear_thread_store_remset_buffer (SgenThreadInfo *info)
5935 *info->store_remset_buffer_index_addr = 0;
5936 memset (*info->store_remset_buffer_addr, 0, sizeof (gpointer) * STORE_REMSET_BUFFER_SIZE);
5940 scan_from_remsets (void *start_nursery, void *end_nursery)
5943 SgenThreadInfo *info;
5944 RememberedSet *remset;
5945 GenericStoreRememberedSet *store_remset;
5946 mword *p, *next_p, *store_pos;
5948 #ifdef HEAVY_STATISTICS
5952 /* the global one */
5953 for (remset = global_remset; remset; remset = remset->next) {
5954 DEBUG (4, fprintf (gc_debug_file, "Scanning global remset range: %p-%p, size: %zd\n", remset->data, remset->store_next, remset->store_next - remset->data));
5955 store_pos = remset->data;
5956 for (p = remset->data; p < remset->store_next; p = next_p) {
5959 next_p = handle_remset (p, start_nursery, end_nursery, TRUE);
5962 * Clear global remsets of locations which no longer point to the
5963 * nursery. Otherwise, they could grow indefinitely between major
5966 ptr = (p [0] & ~REMSET_TYPE_MASK);
5967 if ((p [0] & REMSET_TYPE_MASK) == REMSET_LOCATION) {
5968 if (ptr_in_nursery (*(void**)ptr))
5969 *store_pos ++ = p [0];
5971 g_assert ((p [0] & REMSET_TYPE_MASK) == REMSET_OTHER);
5972 g_assert (p [1] == REMSET_ROOT_LOCATION);
5973 if (ptr_in_nursery (*(void**)ptr)) {
5974 *store_pos ++ = p [0];
5975 *store_pos ++ = p [1];
5980 /* Truncate the remset */
5981 remset->store_next = store_pos;
5984 /* the generic store ones */
5985 store_remset = generic_store_remsets;
5986 while (store_remset) {
5987 GenericStoreRememberedSet *next = store_remset->next;
5989 for (i = 0; i < STORE_REMSET_BUFFER_SIZE - 1; ++i) {
5990 gpointer addr = store_remset->data [i];
5992 handle_remset ((mword*)&addr, start_nursery, end_nursery, FALSE);
5995 free_internal_mem (store_remset, INTERNAL_MEM_STORE_REMSET);
5997 store_remset = next;
5999 generic_store_remsets = NULL;
6001 /* the per-thread ones */
6002 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
6003 for (info = thread_table [i]; info; info = info->next) {
6004 RememberedSet *next;
6006 for (remset = info->remset; remset; remset = next) {
6007 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for thread %p, range: %p-%p, size: %zd\n", info, remset->data, remset->store_next, remset->store_next - remset->data));
6008 for (p = remset->data; p < remset->store_next;) {
6009 p = handle_remset (p, start_nursery, end_nursery, FALSE);
6011 remset->store_next = remset->data;
6012 next = remset->next;
6013 remset->next = NULL;
6014 if (remset != info->remset) {
6015 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
6016 free_internal_mem (remset, INTERNAL_MEM_REMSET);
6019 for (j = 0; j < *info->store_remset_buffer_index_addr; ++j)
6020 handle_remset ((mword*)*info->store_remset_buffer_addr + j + 1, start_nursery, end_nursery, FALSE);
6021 clear_thread_store_remset_buffer (info);
6025 /* the freed thread ones */
6026 while (freed_thread_remsets) {
6027 RememberedSet *next;
6028 remset = freed_thread_remsets;
6029 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for freed thread, range: %p-%p, size: %zd\n", remset->data, remset->store_next, remset->store_next - remset->data));
6030 for (p = remset->data; p < remset->store_next;) {
6031 p = handle_remset (p, start_nursery, end_nursery, FALSE);
6033 next = remset->next;
6034 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
6035 free_internal_mem (remset, INTERNAL_MEM_REMSET);
6036 freed_thread_remsets = next;
6041 * Clear the info in the remembered sets: we're doing a major collection, so
6042 * the per-thread ones are not needed and the global ones will be reconstructed
6046 clear_remsets (void)
6049 SgenThreadInfo *info;
6050 RememberedSet *remset, *next;
6052 /* the global list */
6053 for (remset = global_remset; remset; remset = next) {
6054 remset->store_next = remset->data;
6055 next = remset->next;
6056 remset->next = NULL;
6057 if (remset != global_remset) {
6058 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
6059 free_internal_mem (remset, INTERNAL_MEM_REMSET);
6062 /* the generic store ones */
6063 while (generic_store_remsets) {
6064 GenericStoreRememberedSet *gs_next = generic_store_remsets->next;
6065 free_internal_mem (generic_store_remsets, INTERNAL_MEM_STORE_REMSET);
6066 generic_store_remsets = gs_next;
6068 /* the per-thread ones */
6069 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
6070 for (info = thread_table [i]; info; info = info->next) {
6071 for (remset = info->remset; remset; remset = next) {
6072 remset->store_next = remset->data;
6073 next = remset->next;
6074 remset->next = NULL;
6075 if (remset != info->remset) {
6076 DEBUG (1, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
6077 free_internal_mem (remset, INTERNAL_MEM_REMSET);
6080 clear_thread_store_remset_buffer (info);
6084 /* the freed thread ones */
6085 while (freed_thread_remsets) {
6086 next = freed_thread_remsets->next;
6087 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", freed_thread_remsets->data));
6088 free_internal_mem (freed_thread_remsets, INTERNAL_MEM_REMSET);
6089 freed_thread_remsets = next;
6094 * Clear the thread local TLAB variables for all threads.
6099 SgenThreadInfo *info;
6102 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
6103 for (info = thread_table [i]; info; info = info->next) {
6104 /* A new TLAB will be allocated when the thread does its first allocation */
6105 *info->tlab_start_addr = NULL;
6106 *info->tlab_next_addr = NULL;
6107 *info->tlab_temp_end_addr = NULL;
6108 *info->tlab_real_end_addr = NULL;
6113 /* LOCKING: assumes the GC lock is held */
6114 static SgenThreadInfo*
6115 gc_register_current_thread (void *addr)
6118 SgenThreadInfo* info = malloc (sizeof (SgenThreadInfo));
6119 #ifndef HAVE_KW_THREAD
6120 SgenThreadInfo *__thread_info__ = info;
6126 memset (info, 0, sizeof (SgenThreadInfo));
6127 #ifndef HAVE_KW_THREAD
6128 info->tlab_start = info->tlab_next = info->tlab_temp_end = info->tlab_real_end = NULL;
6130 g_assert (!pthread_getspecific (thread_info_key));
6131 pthread_setspecific (thread_info_key, info);
6136 info->id = ARCH_GET_THREAD ();
6137 info->stop_count = -1;
6140 info->stack_start = NULL;
6141 info->tlab_start_addr = &TLAB_START;
6142 info->tlab_next_addr = &TLAB_NEXT;
6143 info->tlab_temp_end_addr = &TLAB_TEMP_END;
6144 info->tlab_real_end_addr = &TLAB_REAL_END;
6145 info->store_remset_buffer_addr = &STORE_REMSET_BUFFER;
6146 info->store_remset_buffer_index_addr = &STORE_REMSET_BUFFER_INDEX;
6147 info->stopped_ip = NULL;
6148 info->stopped_domain = NULL;
6149 info->stopped_regs = NULL;
6151 #ifdef HAVE_KW_THREAD
6152 tlab_next_addr = &tlab_next;
6153 store_remset_buffer_index_addr = &store_remset_buffer_index;
6156 /* try to get it with attributes first */
6157 #if defined(HAVE_PTHREAD_GETATTR_NP) && defined(HAVE_PTHREAD_ATTR_GETSTACK)
6161 pthread_attr_t attr;
6162 pthread_getattr_np (pthread_self (), &attr);
6163 pthread_attr_getstack (&attr, &sstart, &size);
6164 info->stack_start_limit = sstart;
6165 info->stack_end = (char*)sstart + size;
6166 pthread_attr_destroy (&attr);
6168 #elif defined(HAVE_PTHREAD_GET_STACKSIZE_NP) && defined(HAVE_PTHREAD_GET_STACKADDR_NP)
6169 info->stack_end = (char*)pthread_get_stackaddr_np (pthread_self ());
6172 /* FIXME: we assume the stack grows down */
6173 gsize stack_bottom = (gsize)addr;
6174 stack_bottom += 4095;
6175 stack_bottom &= ~4095;
6176 info->stack_end = (char*)stack_bottom;
6180 #ifdef HAVE_KW_THREAD
6181 stack_end = info->stack_end;
6184 /* hash into the table */
6185 hash = HASH_PTHREAD_T (info->id) % THREAD_HASH_SIZE;
6186 info->next = thread_table [hash];
6187 thread_table [hash] = info;
6189 info->remset = alloc_remset (DEFAULT_REMSET_SIZE, info);
6190 pthread_setspecific (remembered_set_key, info->remset);
6191 #ifdef HAVE_KW_THREAD
6192 remembered_set = info->remset;
6195 STORE_REMSET_BUFFER = get_internal_mem (sizeof (gpointer) * STORE_REMSET_BUFFER_SIZE, INTERNAL_MEM_STORE_REMSET);
6196 STORE_REMSET_BUFFER_INDEX = 0;
6198 DEBUG (3, fprintf (gc_debug_file, "registered thread %p (%p) (hash: %d)\n", info, (gpointer)info->id, hash));
6200 if (gc_callbacks.thread_attach_func)
6201 info->runtime_data = gc_callbacks.thread_attach_func ();
6207 add_generic_store_remset_from_buffer (gpointer *buffer)
6209 GenericStoreRememberedSet *remset = get_internal_mem (sizeof (GenericStoreRememberedSet), INTERNAL_MEM_STORE_REMSET);
6210 memcpy (remset->data, buffer + 1, sizeof (gpointer) * (STORE_REMSET_BUFFER_SIZE - 1));
6211 remset->next = generic_store_remsets;
6212 generic_store_remsets = remset;
6216 unregister_current_thread (void)
6219 SgenThreadInfo *prev = NULL;
6221 RememberedSet *rset;
6222 ARCH_THREAD_TYPE id = ARCH_GET_THREAD ();
6224 hash = HASH_PTHREAD_T (id) % THREAD_HASH_SIZE;
6225 p = thread_table [hash];
6227 DEBUG (3, fprintf (gc_debug_file, "unregister thread %p (%p)\n", p, (gpointer)p->id));
6228 while (!ARCH_THREAD_EQUALS (p->id, id)) {
6233 thread_table [hash] = p->next;
6235 prev->next = p->next;
6238 if (freed_thread_remsets) {
6239 for (rset = p->remset; rset->next; rset = rset->next)
6241 rset->next = freed_thread_remsets;
6242 freed_thread_remsets = p->remset;
6244 freed_thread_remsets = p->remset;
6247 if (*p->store_remset_buffer_index_addr)
6248 add_generic_store_remset_from_buffer (*p->store_remset_buffer_addr);
6249 free_internal_mem (*p->store_remset_buffer_addr, INTERNAL_MEM_STORE_REMSET);
6254 unregister_thread (void *k)
6256 g_assert (!mono_domain_get ());
6258 unregister_current_thread ();
6263 mono_gc_register_thread (void *baseptr)
6265 SgenThreadInfo *info;
6269 info = thread_info_lookup (ARCH_GET_THREAD ());
6271 info = gc_register_current_thread (baseptr);
6273 return info != NULL;
6276 #if USE_PTHREAD_INTERCEPT
6278 #undef pthread_create
6280 #undef pthread_detach
6283 void *(*start_routine) (void *);
6286 MonoSemType registered;
6287 } SgenThreadStartInfo;
6290 gc_start_thread (void *arg)
6292 SgenThreadStartInfo *start_info = arg;
6293 SgenThreadInfo* info;
6294 void *t_arg = start_info->arg;
6295 void *(*start_func) (void*) = start_info->start_routine;
6300 info = gc_register_current_thread (&result);
6302 post_result = MONO_SEM_POST (&(start_info->registered));
6303 g_assert (!post_result);
6304 result = start_func (t_arg);
6305 g_assert (!mono_domain_get ());
6307 * this is done by the pthread key dtor
6309 unregister_current_thread ();
6317 mono_gc_pthread_create (pthread_t *new_thread, const pthread_attr_t *attr, void *(*start_routine)(void *), void *arg)
6319 SgenThreadStartInfo *start_info;
6322 start_info = malloc (sizeof (SgenThreadStartInfo));
6325 result = MONO_SEM_INIT (&(start_info->registered), 0);
6327 start_info->arg = arg;
6328 start_info->start_routine = start_routine;
6330 result = pthread_create (new_thread, attr, gc_start_thread, start_info);
6332 while (MONO_SEM_WAIT (&(start_info->registered)) != 0) {
6333 /*if (EINTR != errno) ABORT("sem_wait failed"); */
6336 MONO_SEM_DESTROY (&(start_info->registered));
6342 mono_gc_pthread_join (pthread_t thread, void **retval)
6344 return pthread_join (thread, retval);
6348 mono_gc_pthread_detach (pthread_t thread)
6350 return pthread_detach (thread);
6353 #endif /* USE_PTHREAD_INTERCEPT */
6356 * ######################################################################
6357 * ######## Write barriers
6358 * ######################################################################
6361 static RememberedSet*
6362 alloc_remset (int size, gpointer id) {
6363 RememberedSet* res = get_internal_mem (sizeof (RememberedSet) + (size * sizeof (gpointer)), INTERNAL_MEM_REMSET);
6364 res->store_next = res->data;
6365 res->end_set = res->data + size;
6367 DEBUG (4, fprintf (gc_debug_file, "Allocated remset size %d at %p for %p\n", size, res->data, id));
6372 * Note: the write barriers first do the needed GC work and then do the actual store:
6373 * this way the value is visible to the conservative GC scan after the write barrier
6374 * itself. If a GC interrupts the barrier in the middle, value will be kept alive by
6375 * the conservative scan, otherwise by the remembered set scan. FIXME: figure out what
6376 * happens when we need to record which pointers contain references to the new generation.
6377 * The write barrier will be executed, but the pointer is still not stored.
6380 mono_gc_wbarrier_set_field (MonoObject *obj, gpointer field_ptr, MonoObject* value)
6384 HEAVY_STAT (++stat_wbarrier_set_field);
6385 if (ptr_in_nursery (field_ptr)) {
6386 *(void**)field_ptr = value;
6389 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", field_ptr));
6391 rs = REMEMBERED_SET;
6392 if (rs->store_next < rs->end_set) {
6393 *(rs->store_next++) = (mword)field_ptr;
6394 *(void**)field_ptr = value;
6398 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6399 rs->next = REMEMBERED_SET;
6400 REMEMBERED_SET = rs;
6401 #ifdef HAVE_KW_THREAD
6402 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6404 *(rs->store_next++) = (mword)field_ptr;
6405 *(void**)field_ptr = value;
6410 mono_gc_wbarrier_set_arrayref (MonoArray *arr, gpointer slot_ptr, MonoObject* value)
6414 HEAVY_STAT (++stat_wbarrier_set_arrayref);
6415 if (ptr_in_nursery (slot_ptr)) {
6416 *(void**)slot_ptr = value;
6419 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", slot_ptr));
6421 rs = REMEMBERED_SET;
6422 if (rs->store_next < rs->end_set) {
6423 *(rs->store_next++) = (mword)slot_ptr;
6424 *(void**)slot_ptr = value;
6428 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6429 rs->next = REMEMBERED_SET;
6430 REMEMBERED_SET = rs;
6431 #ifdef HAVE_KW_THREAD
6432 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6434 *(rs->store_next++) = (mword)slot_ptr;
6435 *(void**)slot_ptr = value;
6440 mono_gc_wbarrier_arrayref_copy (gpointer dest_ptr, gpointer src_ptr, int count)
6444 HEAVY_STAT (++stat_wbarrier_arrayref_copy);
6446 memmove (dest_ptr, src_ptr, count * sizeof (gpointer));
6447 if (ptr_in_nursery (dest_ptr)) {
6451 rs = REMEMBERED_SET;
6452 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p, %d\n", dest_ptr, count));
6453 if (rs->store_next + 1 < rs->end_set) {
6454 *(rs->store_next++) = (mword)dest_ptr | REMSET_RANGE;
6455 *(rs->store_next++) = count;
6459 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6460 rs->next = REMEMBERED_SET;
6461 REMEMBERED_SET = rs;
6462 #ifdef HAVE_KW_THREAD
6463 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6465 *(rs->store_next++) = (mword)dest_ptr | REMSET_RANGE;
6466 *(rs->store_next++) = count;
6471 find_object_for_ptr_in_area (char *ptr, char *start, char *end)
6473 while (start < end) {
6476 if (!*(void**)start) {
6477 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
6483 #define SCAN_OBJECT_NOSCAN
6484 #include "sgen-scan-object.h"
6486 if (ptr >= old_start && ptr < start)
6493 static char *found_obj;
6496 find_object_for_ptr_in_pinned_chunk_callback (PinnedChunk *chunk, char *obj, size_t size, char *ptr)
6498 if (ptr >= obj && ptr < obj + size) {
6499 g_assert (!found_obj);
6504 /* for use in the debugger */
6505 char* find_object_for_ptr (char *ptr);
6507 find_object_for_ptr (char *ptr)
6509 GCMemSection *section;
6512 for (section = section_list; section; section = section->block.next) {
6513 if (ptr >= section->data && ptr < section->end_data)
6514 return find_object_for_ptr_in_area (ptr, section->data, section->end_data);
6517 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
6518 if (ptr >= bigobj->data && ptr < bigobj->data + bigobj->size)
6519 return bigobj->data;
6523 scan_pinned_objects ((ScanPinnedObjectCallbackFunc)find_object_for_ptr_in_pinned_chunk_callback, ptr);
6528 evacuate_remset_buffer (void)
6533 buffer = STORE_REMSET_BUFFER;
6535 add_generic_store_remset_from_buffer (buffer);
6536 memset (buffer, 0, sizeof (gpointer) * STORE_REMSET_BUFFER_SIZE);
6538 STORE_REMSET_BUFFER_INDEX = 0;
6542 mono_gc_wbarrier_generic_nostore (gpointer ptr)
6548 HEAVY_STAT (++stat_wbarrier_generic_store);
6550 #ifdef XDOMAIN_CHECKS_IN_WBARRIER
6551 /* FIXME: ptr_in_heap must be called with the GC lock held */
6552 if (xdomain_checks && *(MonoObject**)ptr && ptr_in_heap (ptr)) {
6553 char *start = find_object_for_ptr (ptr);
6554 MonoObject *value = *(MonoObject**)ptr;
6558 MonoObject *obj = (MonoObject*)start;
6559 if (obj->vtable->domain != value->vtable->domain)
6560 g_assert (is_xdomain_ref_allowed (ptr, start, obj->vtable->domain));
6567 if (ptr_in_nursery (ptr) || ptr_on_stack (ptr) || !ptr_in_nursery (*(gpointer*)ptr)) {
6568 DEBUG (8, fprintf (gc_debug_file, "Skipping remset at %p\n", ptr));
6573 buffer = STORE_REMSET_BUFFER;
6574 index = STORE_REMSET_BUFFER_INDEX;
6575 /* This simple optimization eliminates a sizable portion of
6576 entries. Comparing it to the last but one entry as well
6577 doesn't eliminate significantly more entries. */
6578 if (buffer [index] == ptr) {
6583 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", ptr));
6584 HEAVY_STAT (++stat_wbarrier_generic_store_remset);
6587 if (index >= STORE_REMSET_BUFFER_SIZE) {
6588 evacuate_remset_buffer ();
6589 index = STORE_REMSET_BUFFER_INDEX;
6590 g_assert (index == 0);
6593 buffer [index] = ptr;
6594 STORE_REMSET_BUFFER_INDEX = index;
6600 mono_gc_wbarrier_generic_store (gpointer ptr, MonoObject* value)
6602 DEBUG (8, fprintf (gc_debug_file, "Wbarrier store at %p to %p (%s)\n", ptr, value, value ? safe_name (value) : "null"));
6603 *(void**)ptr = value;
6604 if (ptr_in_nursery (value))
6605 mono_gc_wbarrier_generic_nostore (ptr);
6609 mono_gc_wbarrier_set_root (gpointer ptr, MonoObject *value)
6613 HEAVY_STAT (++stat_wbarrier_set_root);
6614 if (ptr_in_nursery (ptr))
6616 DEBUG (8, fprintf (gc_debug_file, "Adding root remset at %p (%s)\n", ptr, value ? safe_name (value) : "null"));
6618 rs = REMEMBERED_SET;
6619 if (rs->store_next + 2 < rs->end_set) {
6620 *(rs->store_next++) = (mword)ptr | REMSET_OTHER;
6621 *(rs->store_next++) = (mword)REMSET_ROOT_LOCATION;
6622 *(void**)ptr = value;
6625 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6626 rs->next = REMEMBERED_SET;
6627 REMEMBERED_SET = rs;
6628 #ifdef HAVE_KW_THREAD
6629 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6631 *(rs->store_next++) = (mword)ptr | REMSET_OTHER;
6632 *(rs->store_next++) = (mword)REMSET_ROOT_LOCATION;
6634 *(void**)ptr = value;
6638 mono_gc_wbarrier_value_copy (gpointer dest, gpointer src, int count, MonoClass *klass)
6642 HEAVY_STAT (++stat_wbarrier_value_copy);
6643 g_assert (klass->valuetype);
6645 memmove (dest, src, count * mono_class_value_size (klass, NULL));
6646 rs = REMEMBERED_SET;
6647 if (ptr_in_nursery (dest) || ptr_on_stack (dest)) {
6651 g_assert (klass->gc_descr_inited);
6652 DEBUG (8, fprintf (gc_debug_file, "Adding value remset at %p, count %d, descr %p for class %s (%p)\n", dest, count, klass->gc_descr, klass->name, klass));
6654 if (rs->store_next + 3 < rs->end_set) {
6655 *(rs->store_next++) = (mword)dest | REMSET_OTHER;
6656 *(rs->store_next++) = (mword)REMSET_VTYPE;
6657 *(rs->store_next++) = (mword)klass->gc_descr;
6658 *(rs->store_next++) = (mword)count;
6662 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6663 rs->next = REMEMBERED_SET;
6664 REMEMBERED_SET = rs;
6665 #ifdef HAVE_KW_THREAD
6666 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6668 *(rs->store_next++) = (mword)dest | REMSET_OTHER;
6669 *(rs->store_next++) = (mword)REMSET_VTYPE;
6670 *(rs->store_next++) = (mword)klass->gc_descr;
6671 *(rs->store_next++) = (mword)count;
6676 * mono_gc_wbarrier_object_copy:
6678 * Write barrier to call when obj is the result of a clone or copy of an object.
6681 mono_gc_wbarrier_object_copy (MonoObject* obj, MonoObject *src)
6687 HEAVY_STAT (++stat_wbarrier_object_copy);
6688 rs = REMEMBERED_SET;
6689 DEBUG (1, fprintf (gc_debug_file, "Adding object remset for %p\n", obj));
6690 size = mono_object_class (obj)->instance_size;
6692 /* do not copy the sync state */
6693 memcpy ((char*)obj + sizeof (MonoObject), (char*)src + sizeof (MonoObject),
6694 size - sizeof (MonoObject));
6695 if (ptr_in_nursery (obj) || ptr_on_stack (obj)) {
6699 if (rs->store_next < rs->end_set) {
6700 *(rs->store_next++) = (mword)obj | REMSET_OBJECT;
6704 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6705 rs->next = REMEMBERED_SET;
6706 REMEMBERED_SET = rs;
6707 #ifdef HAVE_KW_THREAD
6708 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6710 *(rs->store_next++) = (mword)obj | REMSET_OBJECT;
6715 * ######################################################################
6716 * ######## Collector debugging
6717 * ######################################################################
6720 const char*descriptor_types [] = {
6732 describe_ptr (char *ptr)
6734 GCMemSection *section;
6739 if (ptr_in_nursery (ptr)) {
6740 printf ("Pointer inside nursery.\n");
6742 for (section = section_list; section;) {
6743 if (ptr >= section->data && ptr < section->data + section->size)
6745 section = section->block.next;
6749 printf ("Pointer inside oldspace.\n");
6750 } else if (obj_is_from_pinned_alloc (ptr)) {
6751 printf ("Pointer is inside a pinned chunk.\n");
6753 printf ("Pointer unknown.\n");
6758 if (object_is_pinned (ptr))
6759 printf ("Object is pinned.\n");
6761 if (object_is_forwarded (ptr))
6762 printf ("Object is forwared.\n");
6764 // FIXME: Handle pointers to the inside of objects
6765 vtable = (MonoVTable*)LOAD_VTABLE (ptr);
6767 printf ("VTable: %p\n", vtable);
6768 if (vtable == NULL) {
6769 printf ("VTable is invalid (empty).\n");
6772 if (ptr_in_nursery (vtable)) {
6773 printf ("VTable is invalid (points inside nursery).\n");
6776 printf ("Class: %s\n", vtable->klass->name);
6778 desc = ((GCVTable*)vtable)->desc;
6779 printf ("Descriptor: %lx\n", (long)desc);
6782 printf ("Descriptor type: %d (%s)\n", type, descriptor_types [type]);
6786 find_in_remset_loc (mword *p, char *addr, gboolean *found)
6792 switch ((*p) & REMSET_TYPE_MASK) {
6793 case REMSET_LOCATION:
6794 if (*p == (mword)addr)
6798 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
6800 if ((void**)addr >= ptr && (void**)addr < ptr + count)
6804 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
6805 count = safe_object_get_size ((MonoObject*)ptr);
6806 count += (ALLOC_ALIGN - 1);
6807 count &= (ALLOC_ALIGN - 1);
6808 count /= sizeof (mword);
6809 if ((void**)addr >= ptr && (void**)addr < ptr + count)
6812 case REMSET_OTHER: {
6815 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
6819 switch (desc & 0x7) {
6820 case DESC_TYPE_RUN_LENGTH:
6821 OBJ_RUN_LEN_SIZE (skip_size, desc, ptr);
6823 case DESC_TYPE_SMALL_BITMAP:
6824 OBJ_BITMAP_SIZE (skip_size, desc, start);
6828 g_assert_not_reached ();
6831 /* The descriptor includes the size of MonoObject */
6832 skip_size -= sizeof (MonoObject);
6834 if ((void**)addr >= ptr && (void**)addr < ptr + (skip_size / sizeof (gpointer)))
6838 case REMSET_ROOT_LOCATION:
6841 g_assert_not_reached ();
6846 g_assert_not_reached ();
6852 * Return whenever ADDR occurs in the remembered sets
6855 find_in_remsets (char *addr)
6858 SgenThreadInfo *info;
6859 RememberedSet *remset;
6860 GenericStoreRememberedSet *store_remset;
6862 gboolean found = FALSE;
6864 /* the global one */
6865 for (remset = global_remset; remset; remset = remset->next) {
6866 DEBUG (4, fprintf (gc_debug_file, "Scanning global remset range: %p-%p, size: %zd\n", remset->data, remset->store_next, remset->store_next - remset->data));
6867 for (p = remset->data; p < remset->store_next;) {
6868 p = find_in_remset_loc (p, addr, &found);
6874 /* the generic store ones */
6875 for (store_remset = generic_store_remsets; store_remset; store_remset = store_remset->next) {
6876 for (i = 0; i < STORE_REMSET_BUFFER_SIZE - 1; ++i) {
6877 if (store_remset->data [i] == addr)
6882 /* the per-thread ones */
6883 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
6884 for (info = thread_table [i]; info; info = info->next) {
6886 for (remset = info->remset; remset; remset = remset->next) {
6887 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for thread %p, range: %p-%p, size: %zd\n", info, remset->data, remset->store_next, remset->store_next - remset->data));
6888 for (p = remset->data; p < remset->store_next;) {
6889 p = find_in_remset_loc (p, addr, &found);
6894 for (j = 0; j < *info->store_remset_buffer_index_addr; ++j) {
6895 if ((*info->store_remset_buffer_addr) [j + 1] == addr)
6901 /* the freed thread ones */
6902 for (remset = freed_thread_remsets; remset; remset = remset->next) {
6903 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for freed thread, range: %p-%p, size: %zd\n", remset->data, remset->store_next, remset->store_next - remset->data));
6904 for (p = remset->data; p < remset->store_next;) {
6905 p = find_in_remset_loc (p, addr, &found);
6914 static gboolean missing_remsets;
6917 * We let a missing remset slide if the target object is pinned,
6918 * because the store might have happened but the remset not yet added,
6919 * but in that case the target must be pinned. We might theoretically
6920 * miss some missing remsets this way, but it's very unlikely.
6923 #define HANDLE_PTR(ptr,obj) do { \
6924 if (*(ptr) && (char*)*(ptr) >= nursery_start && (char*)*(ptr) < nursery_next) { \
6925 if (!object_is_pinned (*(ptr)) && !find_in_remsets ((char*)(ptr))) { \
6926 fprintf (gc_debug_file, "Oldspace->newspace reference %p at offset %zd in object %p (%s.%s) not found in remsets.\n", *(ptr), (char*)(ptr) - (char*)(obj), (obj), ((MonoObject*)(obj))->vtable->klass->name_space, ((MonoObject*)(obj))->vtable->klass->name); \
6927 missing_remsets = TRUE; \
6933 * Check that each object reference inside the area which points into the nursery
6934 * can be found in the remembered sets.
6936 static void __attribute__((noinline))
6937 check_remsets_for_area (char *start, char *end)
6940 int type_str = 0, type_rlen = 0, type_bitmap = 0, type_vector = 0, type_lbit = 0, type_complex = 0;
6941 while (start < end) {
6942 if (!*(void**)start) {
6943 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
6946 vt = (GCVTable*)LOAD_VTABLE (start);
6947 DEBUG (8, fprintf (gc_debug_file, "Scanning object %p, vtable: %p (%s)\n", start, vt, vt->klass->name));
6949 MonoObject *obj = (MonoObject*)start;
6950 g_print ("found at %p (0x%lx): %s.%s\n", start, (long)vt->desc, obj->vtable->klass->name_space, obj->vtable->klass->name);
6953 #define SCAN_OBJECT_ACTION COUNT_OBJECT_TYPES
6954 #include "sgen-scan-object.h"
6959 * Perform consistency check of the heap.
6961 * Assumes the world is stopped.
6964 check_consistency (void)
6966 GCMemSection *section;
6968 // Need to add more checks
6969 // FIXME: Create a general heap enumeration function and use that
6971 missing_remsets = FALSE;
6973 DEBUG (1, fprintf (gc_debug_file, "Begin heap consistency check...\n"));
6975 // Check that oldspace->newspace pointers are registered with the collector
6976 for (section = section_list; section; section = section->block.next) {
6977 if (section->block.role == MEMORY_ROLE_GEN0)
6979 DEBUG (2, fprintf (gc_debug_file, "Scan of old section: %p-%p, size: %d\n", section->data, section->next_data, (int)(section->next_data - section->data)));
6980 check_remsets_for_area (section->data, section->next_data);
6983 DEBUG (1, fprintf (gc_debug_file, "Heap consistency check done.\n"));
6985 g_assert (!missing_remsets);
6988 /* Check that the reference is valid */
6990 #define HANDLE_PTR(ptr,obj) do { \
6992 g_assert (safe_name (*(ptr)) != NULL); \
6999 * Perform consistency check on an object. Currently we only check that the
7000 * reference fields are valid.
7003 check_object (char *start)
7008 #include "sgen-scan-object.h"
7014 * ######################################################################
7015 * ######## Other mono public interface functions.
7016 * ######################################################################
7020 mono_gc_collect (int generation)
7024 if (generation == 0) {
7025 collect_nursery (0);
7027 major_collection ("user request");
7034 mono_gc_max_generation (void)
7040 mono_gc_collection_count (int generation)
7042 if (generation == 0)
7043 return num_minor_gcs;
7044 return num_major_gcs;
7048 mono_gc_get_used_size (void)
7051 GCMemSection *section;
7053 tot = los_memory_usage;
7054 for (section = section_list; section; section = section->block.next) {
7055 /* this is approximate... */
7056 tot += section->next_data - section->data;
7058 /* FIXME: account for pinned objects */
7064 mono_gc_get_heap_size (void)
7070 mono_gc_disable (void)
7078 mono_gc_enable (void)
7086 mono_gc_get_los_limit (void)
7088 return MAX_SMALL_OBJ_SIZE;
7092 mono_object_is_alive (MonoObject* o)
7098 mono_gc_get_generation (MonoObject *obj)
7100 if (ptr_in_nursery (obj))
7106 mono_gc_enable_events (void)
7111 mono_gc_weak_link_add (void **link_addr, MonoObject *obj, gboolean track)
7114 mono_gc_register_disappearing_link (obj, link_addr, track);
7119 mono_gc_weak_link_remove (void **link_addr)
7122 mono_gc_register_disappearing_link (NULL, link_addr, FALSE);
7127 mono_gc_weak_link_get (void **link_addr)
7131 return (MonoObject*) REVEAL_POINTER (*link_addr);
7135 mono_gc_make_descr_from_bitmap (gsize *bitmap, int numbits)
7137 if (numbits < ((sizeof (*bitmap) * 8) - ROOT_DESC_TYPE_SHIFT)) {
7138 return (void*)MAKE_ROOT_DESC (ROOT_DESC_BITMAP, bitmap [0]);
7140 mword complex = alloc_complex_descriptor (bitmap, numbits + 1);
7141 return (void*)MAKE_ROOT_DESC (ROOT_DESC_COMPLEX, complex);
7146 mono_gc_make_root_descr_user (MonoGCMarkFunc marker)
7150 g_assert (user_descriptors_next < MAX_USER_DESCRIPTORS);
7151 descr = (void*)MAKE_ROOT_DESC (ROOT_DESC_USER, (mword)user_descriptors_next);
7152 user_descriptors [user_descriptors_next ++] = marker;
7158 mono_gc_alloc_fixed (size_t size, void *descr)
7160 /* FIXME: do a single allocation */
7161 void *res = calloc (1, size);
7164 if (!mono_gc_register_root (res, size, descr)) {
7172 mono_gc_free_fixed (void* addr)
7174 mono_gc_deregister_root (addr);
7179 mono_gc_is_gc_thread (void)
7183 result = thread_info_lookup (ARCH_GET_THREAD ()) != NULL;
7189 mono_gc_base_init (void)
7193 struct sigaction sinfo;
7195 LOCK_INIT (gc_mutex);
7197 if (gc_initialized) {
7201 pagesize = mono_pagesize ();
7202 gc_debug_file = stderr;
7203 if ((env = getenv ("MONO_GC_DEBUG"))) {
7204 opts = g_strsplit (env, ",", -1);
7205 for (ptr = opts; ptr && *ptr; ptr ++) {
7207 if (opt [0] >= '0' && opt [0] <= '9') {
7208 gc_debug_level = atoi (opt);
7213 char *rf = g_strdup_printf ("%s.%d", opt, getpid ());
7214 gc_debug_file = fopen (rf, "wb");
7216 gc_debug_file = stderr;
7219 } else if (!strcmp (opt, "collect-before-allocs")) {
7220 collect_before_allocs = TRUE;
7221 } else if (!strcmp (opt, "check-at-minor-collections")) {
7222 consistency_check_at_minor_collection = TRUE;
7223 } else if (!strcmp (opt, "xdomain-checks")) {
7224 xdomain_checks = TRUE;
7225 } else if (!strcmp (opt, "clear-at-gc")) {
7226 nursery_clear_policy = CLEAR_AT_GC;
7227 } else if (!strcmp (opt, "conservative-stack-mark")) {
7228 conservative_stack_mark = TRUE;
7229 } else if (!strcmp (opt, "check-scan-starts")) {
7230 do_scan_starts_check = TRUE;
7231 } else if (g_str_has_prefix (opt, "heap-dump=")) {
7232 char *filename = strchr (opt, '=') + 1;
7233 nursery_clear_policy = CLEAR_AT_GC;
7234 heap_dump_file = fopen (filename, "w");
7236 fprintf (heap_dump_file, "<sgen-dump>\n");
7238 fprintf (stderr, "Invalid format for the MONO_GC_DEBUG env variable: '%s'\n", env);
7239 fprintf (stderr, "The format is: MONO_GC_DEBUG=[l[:filename]|<option>]+ where l is a debug level 0-9.\n");
7240 fprintf (stderr, "Valid options are: collect-before-allocs, check-at-minor-collections, xdomain-checks, clear-at-gc.\n");
7247 suspend_ack_semaphore_ptr = &suspend_ack_semaphore;
7248 MONO_SEM_INIT (&suspend_ack_semaphore, 0);
7250 sigfillset (&sinfo.sa_mask);
7251 sinfo.sa_flags = SA_RESTART | SA_SIGINFO;
7252 sinfo.sa_sigaction = suspend_handler;
7253 if (sigaction (suspend_signal_num, &sinfo, NULL) != 0) {
7254 g_error ("failed sigaction");
7257 sinfo.sa_handler = restart_handler;
7258 if (sigaction (restart_signal_num, &sinfo, NULL) != 0) {
7259 g_error ("failed sigaction");
7262 sigfillset (&suspend_signal_mask);
7263 sigdelset (&suspend_signal_mask, restart_signal_num);
7265 global_remset = alloc_remset (1024, NULL);
7266 global_remset->next = NULL;
7268 pthread_key_create (&remembered_set_key, unregister_thread);
7270 #ifndef HAVE_KW_THREAD
7271 pthread_key_create (&thread_info_key, NULL);
7274 gc_initialized = TRUE;
7276 mono_gc_register_thread (&sinfo);
7280 mono_gc_get_suspend_signal (void)
7282 return suspend_signal_num;
7292 #ifdef HAVE_KW_THREAD
7293 #define EMIT_TLS_ACCESS(mb,dummy,offset) do { \
7294 mono_mb_emit_byte ((mb), MONO_CUSTOM_PREFIX); \
7295 mono_mb_emit_byte ((mb), CEE_MONO_TLS); \
7296 mono_mb_emit_i4 ((mb), (offset)); \
7299 #define EMIT_TLS_ACCESS(mb,member,dummy) do { \
7300 mono_mb_emit_byte ((mb), MONO_CUSTOM_PREFIX); \
7301 mono_mb_emit_byte ((mb), CEE_MONO_TLS); \
7302 mono_mb_emit_i4 ((mb), thread_info_key); \
7303 mono_mb_emit_icon ((mb), G_STRUCT_OFFSET (SgenThreadInfo, member)); \
7304 mono_mb_emit_byte ((mb), CEE_ADD); \
7305 mono_mb_emit_byte ((mb), CEE_LDIND_I); \
7309 #ifdef MANAGED_ALLOCATION
7310 /* FIXME: Do this in the JIT, where specialized allocation sequences can be created
7311 * for each class. This is currently not easy to do, as it is hard to generate basic
7312 * blocks + branches, but it is easy with the linear IL codebase.
7314 * For this to work we'd need to solve the TLAB race, first. Now we
7315 * require the allocator to be in a few known methods to make sure
7316 * that they are executed atomically via the restart mechanism.
7319 create_allocator (int atype)
7321 int p_var, size_var;
7322 guint32 slowpath_branch, max_size_branch;
7323 MonoMethodBuilder *mb;
7325 MonoMethodSignature *csig;
7326 static gboolean registered = FALSE;
7327 int tlab_next_addr_var, new_next_var;
7329 const char *name = NULL;
7330 AllocatorWrapperInfo *info;
7332 #ifdef HAVE_KW_THREAD
7333 int tlab_next_addr_offset = -1;
7334 int tlab_temp_end_offset = -1;
7336 MONO_THREAD_VAR_OFFSET (tlab_next_addr, tlab_next_addr_offset);
7337 MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
7339 g_assert (tlab_next_addr_offset != -1);
7340 g_assert (tlab_temp_end_offset != -1);
7344 mono_register_jit_icall (mono_gc_alloc_obj, "mono_gc_alloc_obj", mono_create_icall_signature ("object ptr int"), FALSE);
7345 mono_register_jit_icall (mono_gc_alloc_vector, "mono_gc_alloc_vector", mono_create_icall_signature ("object ptr int int"), FALSE);
7349 if (atype == ATYPE_SMALL) {
7351 name = "AllocSmall";
7352 } else if (atype == ATYPE_NORMAL) {
7355 } else if (atype == ATYPE_VECTOR) {
7357 name = "AllocVector";
7359 g_assert_not_reached ();
7362 csig = mono_metadata_signature_alloc (mono_defaults.corlib, num_params);
7363 csig->ret = &mono_defaults.object_class->byval_arg;
7364 for (i = 0; i < num_params; ++i)
7365 csig->params [i] = &mono_defaults.int_class->byval_arg;
7367 mb = mono_mb_new (mono_defaults.object_class, name, MONO_WRAPPER_ALLOC);
7368 size_var = mono_mb_add_local (mb, &mono_defaults.int32_class->byval_arg);
7369 if (atype == ATYPE_NORMAL || atype == ATYPE_SMALL) {
7370 /* size = vtable->klass->instance_size; */
7371 mono_mb_emit_ldarg (mb, 0);
7372 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoVTable, klass));
7373 mono_mb_emit_byte (mb, CEE_ADD);
7374 mono_mb_emit_byte (mb, CEE_LDIND_I);
7375 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoClass, instance_size));
7376 mono_mb_emit_byte (mb, CEE_ADD);
7377 /* FIXME: assert instance_size stays a 4 byte integer */
7378 mono_mb_emit_byte (mb, CEE_LDIND_U4);
7379 mono_mb_emit_stloc (mb, size_var);
7380 } else if (atype == ATYPE_VECTOR) {
7381 MonoExceptionClause *clause;
7383 MonoClass *oom_exc_class;
7386 /* n > MONO_ARRAY_MAX_INDEX -> OverflowException */
7387 mono_mb_emit_ldarg (mb, 1);
7388 mono_mb_emit_icon (mb, MONO_ARRAY_MAX_INDEX);
7389 pos = mono_mb_emit_short_branch (mb, CEE_BLE_UN_S);
7390 mono_mb_emit_exception (mb, "OverflowException", NULL);
7391 mono_mb_patch_short_branch (mb, pos);
7393 clause = mono_image_alloc0 (mono_defaults.corlib, sizeof (MonoExceptionClause));
7394 clause->try_offset = mono_mb_get_label (mb);
7396 /* vtable->klass->sizes.element_size */
7397 mono_mb_emit_ldarg (mb, 0);
7398 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoVTable, klass));
7399 mono_mb_emit_byte (mb, CEE_ADD);
7400 mono_mb_emit_byte (mb, CEE_LDIND_I);
7401 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoClass, sizes.element_size));
7402 mono_mb_emit_byte (mb, CEE_ADD);
7403 mono_mb_emit_byte (mb, CEE_LDIND_U4);
7406 mono_mb_emit_ldarg (mb, 1);
7407 mono_mb_emit_byte (mb, CEE_MUL_OVF_UN);
7408 /* + sizeof (MonoArray) */
7409 mono_mb_emit_icon (mb, sizeof (MonoArray));
7410 mono_mb_emit_byte (mb, CEE_ADD_OVF_UN);
7411 mono_mb_emit_stloc (mb, size_var);
7413 pos_leave = mono_mb_emit_branch (mb, CEE_LEAVE);
7416 clause->flags = MONO_EXCEPTION_CLAUSE_NONE;
7417 clause->try_len = mono_mb_get_pos (mb) - clause->try_offset;
7418 clause->data.catch_class = mono_class_from_name (mono_defaults.corlib,
7419 "System", "OverflowException");
7420 g_assert (clause->data.catch_class);
7421 clause->handler_offset = mono_mb_get_label (mb);
7423 oom_exc_class = mono_class_from_name (mono_defaults.corlib,
7424 "System", "OutOfMemoryException");
7425 g_assert (oom_exc_class);
7426 ctor = mono_class_get_method_from_name (oom_exc_class, ".ctor", 0);
7429 mono_mb_emit_byte (mb, CEE_POP);
7430 mono_mb_emit_op (mb, CEE_NEWOBJ, ctor);
7431 mono_mb_emit_byte (mb, CEE_THROW);
7433 clause->handler_len = mono_mb_get_pos (mb) - clause->handler_offset;
7434 mono_mb_set_clauses (mb, 1, clause);
7435 mono_mb_patch_branch (mb, pos_leave);
7438 g_assert_not_reached ();
7441 /* size += ALLOC_ALIGN - 1; */
7442 mono_mb_emit_ldloc (mb, size_var);
7443 mono_mb_emit_icon (mb, ALLOC_ALIGN - 1);
7444 mono_mb_emit_byte (mb, CEE_ADD);
7445 /* size &= ~(ALLOC_ALIGN - 1); */
7446 mono_mb_emit_icon (mb, ~(ALLOC_ALIGN - 1));
7447 mono_mb_emit_byte (mb, CEE_AND);
7448 mono_mb_emit_stloc (mb, size_var);
7450 /* if (size > MAX_SMALL_OBJ_SIZE) goto slowpath */
7451 if (atype != ATYPE_SMALL) {
7452 mono_mb_emit_ldloc (mb, size_var);
7453 mono_mb_emit_icon (mb, MAX_SMALL_OBJ_SIZE);
7454 max_size_branch = mono_mb_emit_short_branch (mb, MONO_CEE_BGT_S);
7458 * We need to modify tlab_next, but the JIT only supports reading, so we read
7459 * another tls var holding its address instead.
7462 /* tlab_next_addr (local) = tlab_next_addr (TLS var) */
7463 tlab_next_addr_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7464 EMIT_TLS_ACCESS (mb, tlab_next_addr, tlab_next_addr_offset);
7465 mono_mb_emit_stloc (mb, tlab_next_addr_var);
7467 /* p = (void**)tlab_next; */
7468 p_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7469 mono_mb_emit_ldloc (mb, tlab_next_addr_var);
7470 mono_mb_emit_byte (mb, CEE_LDIND_I);
7471 mono_mb_emit_stloc (mb, p_var);
7473 /* new_next = (char*)p + size; */
7474 new_next_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7475 mono_mb_emit_ldloc (mb, p_var);
7476 mono_mb_emit_ldloc (mb, size_var);
7477 mono_mb_emit_byte (mb, CEE_CONV_I);
7478 mono_mb_emit_byte (mb, CEE_ADD);
7479 mono_mb_emit_stloc (mb, new_next_var);
7481 /* tlab_next = new_next */
7482 mono_mb_emit_ldloc (mb, tlab_next_addr_var);
7483 mono_mb_emit_ldloc (mb, new_next_var);
7484 mono_mb_emit_byte (mb, CEE_STIND_I);
7486 /* if (G_LIKELY (new_next < tlab_temp_end)) */
7487 mono_mb_emit_ldloc (mb, new_next_var);
7488 EMIT_TLS_ACCESS (mb, tlab_temp_end, tlab_temp_end_offset);
7489 slowpath_branch = mono_mb_emit_short_branch (mb, MONO_CEE_BLT_UN_S);
7492 if (atype != ATYPE_SMALL)
7493 mono_mb_patch_short_branch (mb, max_size_branch);
7495 mono_mb_emit_byte (mb, MONO_CUSTOM_PREFIX);
7496 mono_mb_emit_byte (mb, CEE_MONO_NOT_TAKEN);
7498 /* FIXME: mono_gc_alloc_obj takes a 'size_t' as an argument, not an int32 */
7499 mono_mb_emit_ldarg (mb, 0);
7500 mono_mb_emit_ldloc (mb, size_var);
7501 if (atype == ATYPE_NORMAL || atype == ATYPE_SMALL) {
7502 mono_mb_emit_icall (mb, mono_gc_alloc_obj);
7503 } else if (atype == ATYPE_VECTOR) {
7504 mono_mb_emit_ldarg (mb, 1);
7505 mono_mb_emit_icall (mb, mono_gc_alloc_vector);
7507 g_assert_not_reached ();
7509 mono_mb_emit_byte (mb, CEE_RET);
7512 mono_mb_patch_short_branch (mb, slowpath_branch);
7514 /* FIXME: Memory barrier */
7517 mono_mb_emit_ldloc (mb, p_var);
7518 mono_mb_emit_ldarg (mb, 0);
7519 mono_mb_emit_byte (mb, CEE_STIND_I);
7521 if (atype == ATYPE_VECTOR) {
7522 /* arr->max_length = max_length; */
7523 mono_mb_emit_ldloc (mb, p_var);
7524 mono_mb_emit_ldflda (mb, G_STRUCT_OFFSET (MonoArray, max_length));
7525 mono_mb_emit_ldarg (mb, 1);
7526 mono_mb_emit_byte (mb, CEE_STIND_I);
7530 mono_mb_emit_ldloc (mb, p_var);
7531 mono_mb_emit_byte (mb, CEE_RET);
7533 res = mono_mb_create_method (mb, csig, 8);
7535 mono_method_get_header (res)->init_locals = FALSE;
7537 info = mono_image_alloc0 (mono_defaults.corlib, sizeof (AllocatorWrapperInfo));
7538 info->alloc_type = atype;
7539 mono_marshal_set_wrapper_info (res, info);
7545 static MonoMethod* alloc_method_cache [ATYPE_NUM];
7546 static MonoMethod *write_barrier_method;
7549 is_ip_in_managed_allocator (MonoDomain *domain, gpointer ip)
7557 ji = mono_jit_info_table_find (domain, ip);
7560 method = ji->method;
7562 if (method == write_barrier_method)
7564 for (i = 0; i < ATYPE_NUM; ++i)
7565 if (method == alloc_method_cache [i])
7571 * Generate an allocator method implementing the fast path of mono_gc_alloc_obj ().
7572 * The signature of the called method is:
7573 * object allocate (MonoVTable *vtable)
7576 mono_gc_get_managed_allocator (MonoVTable *vtable, gboolean for_box)
7578 #ifdef MANAGED_ALLOCATION
7579 MonoClass *klass = vtable->klass;
7581 #ifdef HAVE_KW_THREAD
7582 int tlab_next_offset = -1;
7583 int tlab_temp_end_offset = -1;
7584 MONO_THREAD_VAR_OFFSET (tlab_next, tlab_next_offset);
7585 MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
7587 if (tlab_next_offset == -1 || tlab_temp_end_offset == -1)
7591 if (!mono_runtime_has_tls_get ())
7593 if (klass->instance_size > tlab_size)
7595 if (klass->has_finalize || klass->marshalbyref || (mono_profiler_get_events () & MONO_PROFILE_ALLOCATIONS))
7599 if (klass->byval_arg.type == MONO_TYPE_STRING)
7601 if (collect_before_allocs)
7604 if (ALIGN_TO (klass->instance_size, ALLOC_ALIGN) < MAX_SMALL_OBJ_SIZE)
7605 return mono_gc_get_managed_allocator_by_type (ATYPE_SMALL);
7607 return mono_gc_get_managed_allocator_by_type (ATYPE_NORMAL);
7614 mono_gc_get_managed_array_allocator (MonoVTable *vtable, int rank)
7616 #ifdef MANAGED_ALLOCATION
7617 MonoClass *klass = vtable->klass;
7619 #ifdef HAVE_KW_THREAD
7620 int tlab_next_offset = -1;
7621 int tlab_temp_end_offset = -1;
7622 MONO_THREAD_VAR_OFFSET (tlab_next, tlab_next_offset);
7623 MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
7625 if (tlab_next_offset == -1 || tlab_temp_end_offset == -1)
7631 if (!mono_runtime_has_tls_get ())
7633 if (mono_profiler_get_events () & MONO_PROFILE_ALLOCATIONS)
7635 if (collect_before_allocs)
7637 g_assert (!klass->has_finalize && !klass->marshalbyref);
7639 return mono_gc_get_managed_allocator_by_type (ATYPE_VECTOR);
7646 mono_gc_get_managed_allocator_by_type (int atype)
7648 #ifdef MANAGED_ALLOCATION
7651 if (!mono_runtime_has_tls_get ())
7654 mono_loader_lock ();
7655 res = alloc_method_cache [atype];
7657 res = alloc_method_cache [atype] = create_allocator (atype);
7658 mono_loader_unlock ();
7666 mono_gc_get_managed_allocator_types (void)
7673 mono_gc_get_write_barrier (void)
7676 MonoMethodBuilder *mb;
7677 MonoMethodSignature *sig;
7678 #ifdef MANAGED_WBARRIER
7679 int label_no_wb_1, label_no_wb_2, label_no_wb_3, label_no_wb_4, label_need_wb, label_slow_path;
7680 int buffer_var, buffer_index_var, dummy_var;
7682 #ifdef HAVE_KW_THREAD
7683 int stack_end_offset = -1, store_remset_buffer_offset = -1;
7684 int store_remset_buffer_index_offset = -1, store_remset_buffer_index_addr_offset = -1;
7686 MONO_THREAD_VAR_OFFSET (stack_end, stack_end_offset);
7687 g_assert (stack_end_offset != -1);
7688 MONO_THREAD_VAR_OFFSET (store_remset_buffer, store_remset_buffer_offset);
7689 g_assert (store_remset_buffer_offset != -1);
7690 MONO_THREAD_VAR_OFFSET (store_remset_buffer_index, store_remset_buffer_index_offset);
7691 g_assert (store_remset_buffer_index_offset != -1);
7692 MONO_THREAD_VAR_OFFSET (store_remset_buffer_index_addr, store_remset_buffer_index_addr_offset);
7693 g_assert (store_remset_buffer_index_addr_offset != -1);
7697 // FIXME: Maybe create a separate version for ctors (the branch would be
7698 // correctly predicted more times)
7699 if (write_barrier_method)
7700 return write_barrier_method;
7702 /* Create the IL version of mono_gc_barrier_generic_store () */
7703 sig = mono_metadata_signature_alloc (mono_defaults.corlib, 1);
7704 sig->ret = &mono_defaults.void_class->byval_arg;
7705 sig->params [0] = &mono_defaults.int_class->byval_arg;
7707 mb = mono_mb_new (mono_defaults.object_class, "wbarrier", MONO_WRAPPER_WRITE_BARRIER);
7709 #ifdef MANAGED_WBARRIER
7710 if (mono_runtime_has_tls_get ()) {
7711 #ifdef ALIGN_NURSERY
7712 // if (ptr_in_nursery (ptr)) return;
7714 * Masking out the bits might be faster, but we would have to use 64 bit
7715 * immediates, which might be slower.
7717 mono_mb_emit_ldarg (mb, 0);
7718 mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
7719 mono_mb_emit_byte (mb, CEE_SHR_UN);
7720 mono_mb_emit_icon (mb, (mword)nursery_start >> DEFAULT_NURSERY_BITS);
7721 label_no_wb_1 = mono_mb_emit_branch (mb, CEE_BEQ);
7723 // if (!ptr_in_nursery (*ptr)) return;
7724 mono_mb_emit_ldarg (mb, 0);
7725 mono_mb_emit_byte (mb, CEE_LDIND_I);
7726 mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
7727 mono_mb_emit_byte (mb, CEE_SHR_UN);
7728 mono_mb_emit_icon (mb, (mword)nursery_start >> DEFAULT_NURSERY_BITS);
7729 label_no_wb_2 = mono_mb_emit_branch (mb, CEE_BNE_UN);
7732 g_assert_not_reached ();
7735 // if (ptr >= stack_end) goto need_wb;
7736 mono_mb_emit_ldarg (mb, 0);
7737 EMIT_TLS_ACCESS (mb, stack_end, stack_end_offset);
7738 label_need_wb = mono_mb_emit_branch (mb, CEE_BGE_UN);
7740 // if (ptr >= stack_start) return;
7741 dummy_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7742 mono_mb_emit_ldarg (mb, 0);
7743 mono_mb_emit_ldloc_addr (mb, dummy_var);
7744 label_no_wb_3 = mono_mb_emit_branch (mb, CEE_BGE_UN);
7747 mono_mb_patch_branch (mb, label_need_wb);
7749 // buffer = STORE_REMSET_BUFFER;
7750 buffer_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7751 EMIT_TLS_ACCESS (mb, store_remset_buffer, store_remset_buffer_offset);
7752 mono_mb_emit_stloc (mb, buffer_var);
7754 // buffer_index = STORE_REMSET_BUFFER_INDEX;
7755 buffer_index_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7756 EMIT_TLS_ACCESS (mb, store_remset_buffer_index, store_remset_buffer_index_offset);
7757 mono_mb_emit_stloc (mb, buffer_index_var);
7759 // if (buffer [buffer_index] == ptr) return;
7760 mono_mb_emit_ldloc (mb, buffer_var);
7761 mono_mb_emit_ldloc (mb, buffer_index_var);
7762 g_assert (sizeof (gpointer) == 4 || sizeof (gpointer) == 8);
7763 mono_mb_emit_icon (mb, sizeof (gpointer) == 4 ? 2 : 3);
7764 mono_mb_emit_byte (mb, CEE_SHL);
7765 mono_mb_emit_byte (mb, CEE_ADD);
7766 mono_mb_emit_byte (mb, CEE_LDIND_I);
7767 mono_mb_emit_ldarg (mb, 0);
7768 label_no_wb_4 = mono_mb_emit_branch (mb, CEE_BEQ);
7771 mono_mb_emit_ldloc (mb, buffer_index_var);
7772 mono_mb_emit_icon (mb, 1);
7773 mono_mb_emit_byte (mb, CEE_ADD);
7774 mono_mb_emit_stloc (mb, buffer_index_var);
7776 // if (buffer_index >= STORE_REMSET_BUFFER_SIZE) goto slow_path;
7777 mono_mb_emit_ldloc (mb, buffer_index_var);
7778 mono_mb_emit_icon (mb, STORE_REMSET_BUFFER_SIZE);
7779 label_slow_path = mono_mb_emit_branch (mb, CEE_BGE);
7781 // buffer [buffer_index] = ptr;
7782 mono_mb_emit_ldloc (mb, buffer_var);
7783 mono_mb_emit_ldloc (mb, buffer_index_var);
7784 g_assert (sizeof (gpointer) == 4 || sizeof (gpointer) == 8);
7785 mono_mb_emit_icon (mb, sizeof (gpointer) == 4 ? 2 : 3);
7786 mono_mb_emit_byte (mb, CEE_SHL);
7787 mono_mb_emit_byte (mb, CEE_ADD);
7788 mono_mb_emit_ldarg (mb, 0);
7789 mono_mb_emit_byte (mb, CEE_STIND_I);
7791 // STORE_REMSET_BUFFER_INDEX = buffer_index;
7792 EMIT_TLS_ACCESS (mb, store_remset_buffer_index_addr, store_remset_buffer_index_addr_offset);
7793 mono_mb_emit_ldloc (mb, buffer_index_var);
7794 mono_mb_emit_byte (mb, CEE_STIND_I);
7797 mono_mb_patch_branch (mb, label_no_wb_1);
7798 mono_mb_patch_branch (mb, label_no_wb_2);
7799 mono_mb_patch_branch (mb, label_no_wb_3);
7800 mono_mb_patch_branch (mb, label_no_wb_4);
7801 mono_mb_emit_byte (mb, CEE_RET);
7804 mono_mb_patch_branch (mb, label_slow_path);
7808 mono_mb_emit_ldarg (mb, 0);
7809 mono_mb_emit_icall (mb, mono_gc_wbarrier_generic_nostore);
7810 mono_mb_emit_byte (mb, CEE_RET);
7812 res = mono_mb_create_method (mb, sig, 16);
7815 mono_loader_lock ();
7816 if (write_barrier_method) {
7817 /* Already created */
7818 mono_free_method (res);
7820 /* double-checked locking */
7821 mono_memory_barrier ();
7822 write_barrier_method = res;
7824 mono_loader_unlock ();
7826 return write_barrier_method;
7829 #endif /* HAVE_SGEN_GC */