2 * sgen-gc.c: Simple generational GC.
5 * Paolo Molaro (lupus@ximian.com)
7 * Copyright 2005-2009 Novell, Inc (http://www.novell.com)
9 * Thread start/stop adapted from Boehm's GC:
10 * Copyright (c) 1994 by Xerox Corporation. All rights reserved.
11 * Copyright (c) 1996 by Silicon Graphics. All rights reserved.
12 * Copyright (c) 1998 by Fergus Henderson. All rights reserved.
13 * Copyright (c) 2000-2004 by Hewlett-Packard Company. All rights reserved.
15 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
16 * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
18 * Permission is hereby granted to use or copy this program
19 * for any purpose, provided the above notices are retained on all copies.
20 * Permission to modify the code and to distribute modified code is granted,
21 * provided the above notices are retained, and a notice that the code was
22 * modified is included with the above copyright notice.
24 * All the rest of the code is LGPL.
26 * Important: allocation provides always zeroed memory, having to do
27 * a memset after allocation is deadly for performance.
28 * Memory usage at startup is currently as follows:
30 * 64 KB internal space
32 * We should provide a small memory config with half the sizes
34 * We currently try to make as few mono assumptions as possible:
35 * 1) 2-word header with no GC pointers in it (first vtable, second to store the
37 * 2) gc descriptor is the second word in the vtable (first word in the class)
38 * 3) 8 byte alignment is the minimum and enough (not true for special structures, FIXME)
39 * 4) there is a function to get an object's size and the number of
40 * elements in an array.
41 * 5) we know the special way bounds are allocated for complex arrays
43 * Always try to keep stack usage to a minimum: no recursive behaviour
44 * and no large stack allocs.
46 * General description.
47 * Objects are initially allocated in a nursery using a fast bump-pointer technique.
48 * When the nursery is full we start a nursery collection: this is performed with a
50 * When the old generation is full we start a copying GC of the old generation as well:
51 * this will be changed to mark/compact in the future.
52 * The things that complicate this description are:
53 * *) pinned objects: we can't move them so we need to keep track of them
54 * *) no precise info of the thread stacks and registers: we need to be able to
55 * quickly find the objects that may be referenced conservatively and pin them
56 * (this makes the first issues more important)
57 * *) large objects are too expensive to be dealt with using copying GC: we handle them
58 * with mark/sweep during major collections
59 * *) some objects need to not move even if they are small (interned strings, Type handles):
60 * we use mark/sweep for them, too: they are not allocated in the nursery, but inside
61 * PinnedChunks regions
66 *) change the jit to emit write barrier calls when needed (we
67 can have specialized write barriers): done with icalls, still need to
68 use some specialized barriers
69 *) we could have a function pointer in MonoClass to implement
70 customized write barriers for value types
71 *) the write barrier code could be isolated in a couple of functions: when a
72 thread is stopped if it's inside the barrier it is let go again
73 until we stop outside of them (not really needed, see below GC-safe points)
74 *) investigate the stuff needed to advance a thread to a GC-safe
75 point (single-stepping, read from unmapped memory etc) and implement it
76 Not needed yet: since we treat the objects reachable from the stack/regs as
77 roots, we store the ptr and exec the write barrier so there is no race.
78 We may need this to solve the issue with setting the length of arrays and strings.
79 We may need this also for handling precise info on stacks, even simple things
80 as having uninitialized data on the stack and having to wait for the prolog
81 to zero it. Not an issue for the last frame that we scan conservatively.
82 We could always not trust the value in the slots anyway.
83 *) make the jit info table lock free
84 *) modify the jit to save info about references in stack locations:
85 this can be done just for locals as a start, so that at least
86 part of the stack is handled precisely.
87 *) Make the debug printf stuff thread and signal safe.
88 *) test/fix 64 bit issues
89 *) test/fix endianess issues
91 *) add batch moving profile info
92 *) add more timing info
93 *) there is a possible race when an array or string is created: the vtable is set,
94 but the length is set only later so if the GC needs to scan the object in that window,
95 it won't get the correct size for the object. The object can't have references and it will
96 be pinned, but a free memory fragment may be created that overlaps with it.
97 We should change the array max_length field to be at the same offset as the string length:
98 this way we can have a single special alloc function for them that sets the length.
99 Multi-dim arrays have the same issue for rank == 1 for the bounds data.
100 *) implement a card table as the write barrier instead of remembered sets?
101 *) some sort of blacklist support?
102 *) fin_ready_list and critical_fin_list are part of the root set, too
103 *) consider lowering the large object min size to 16/32KB or so and benchmark
104 *) once mark-compact is implemented we could still keep the
105 copying collector for the old generation and use it if we think
106 it is better (small heaps and no pinning object in the old
108 *) avoid the memory store from copy_object when not needed.
109 *) optimize the write barriers fastpath to happen in managed code
110 *) add an option to mmap the whole heap in one chunk: it makes for many
111 simplifications in the checks (put the nursery at the top and just use a single
112 check for inclusion/exclusion): the issue this has is that on 32 bit systems it's
113 not flexible (too much of the address space may be used by default or we can't
114 increase the heap as needed) and we'd need a race-free mechanism to return memory
115 back to the system (mprotect(PROT_NONE) will still keep the memory allocated if it
116 was written to, munmap is needed, but the following mmap may not find the same segment
118 *) memzero the fragments after restarting the world and optionally a smaller chunk at a time
119 *) an additional strategy to realloc/expand the nursery when fully pinned is to start
120 allocating objects in the old generation. This means that we can't optimize away write
121 barrier calls in ctors (but that is not valid for other reasons, too).
122 *) add write barriers to the Clone methods
130 #include <semaphore.h>
134 #include "metadata/metadata-internals.h"
135 #include "metadata/class-internals.h"
136 #include "metadata/gc-internal.h"
137 #include "metadata/object-internals.h"
138 #include "metadata/threads.h"
139 #include "metadata/sgen-gc.h"
140 #include "metadata/sgen-archdep.h"
141 #include "metadata/mono-gc.h"
142 #include "metadata/method-builder.h"
143 #include "metadata/profiler-private.h"
144 #include "metadata/monitor.h"
145 #include "metadata/threadpool-internals.h"
146 #include "metadata/mempool-internals.h"
147 #include "metadata/marshal.h"
148 #include "utils/mono-mmap.h"
149 #include "utils/mono-time.h"
150 #include "utils/mono-semaphore.h"
151 #include "utils/mono-counters.h"
153 #include <mono/utils/memcheck.h>
155 #define OPDEF(a,b,c,d,e,f,g,h,i,j) \
159 #include "mono/cil/opcode.def"
166 * ######################################################################
167 * ######## Types and constants used by the GC.
168 * ######################################################################
170 #if SIZEOF_VOID_P == 4
171 typedef guint32 mword;
173 typedef guint64 mword;
176 static int gc_initialized = 0;
177 static int gc_debug_level = 0;
178 static FILE* gc_debug_file;
179 /* If set, do a minor collection before every allocation */
180 static gboolean collect_before_allocs = FALSE;
181 /* If set, do a heap consistency check before each minor collection */
182 static gboolean consistency_check_at_minor_collection = FALSE;
183 /* If set, check that there are no references to the domain left at domain unload */
184 static gboolean xdomain_checks = FALSE;
185 /* If not null, dump the heap after each collection into this file */
186 static FILE *heap_dump_file = NULL;
187 /* If set, mark stacks conservatively, even if precise marking is possible */
188 static gboolean conservative_stack_mark = FALSE;
189 /* If set, do a plausibility check on the scan_starts before and after
191 static gboolean do_scan_starts_check = FALSE;
194 * Turning on heavy statistics will turn off the managed allocator and
195 * the managed write barrier.
197 //#define HEAVY_STATISTICS
199 #ifdef HEAVY_STATISTICS
200 #define HEAVY_STAT(x) x
202 #define HEAVY_STAT(x)
205 #ifdef HEAVY_STATISTICS
206 static long stat_objects_alloced = 0;
207 static long stat_objects_alloced_degraded = 0;
208 static long stat_copy_object_called_nursery = 0;
209 static long stat_objects_copied_nursery = 0;
210 static long stat_copy_object_called_major = 0;
211 static long stat_objects_copied_major = 0;
213 static long stat_copy_object_failed_from_space = 0;
214 static long stat_copy_object_failed_forwarded = 0;
215 static long stat_copy_object_failed_pinned = 0;
216 static long stat_copy_object_failed_large_pinned = 0;
217 static long stat_copy_object_failed_to_space = 0;
219 static long stat_store_remsets = 0;
220 static long stat_store_remsets_unique = 0;
221 static long stat_saved_remsets_1 = 0;
222 static long stat_saved_remsets_2 = 0;
223 static long stat_global_remsets_added = 0;
224 static long stat_global_remsets_processed = 0;
226 static long num_copy_object_called = 0;
227 static long num_objects_copied = 0;
229 static int stat_wbarrier_set_field = 0;
230 static int stat_wbarrier_set_arrayref = 0;
231 static int stat_wbarrier_arrayref_copy = 0;
232 static int stat_wbarrier_generic_store = 0;
233 static int stat_wbarrier_generic_store_remset = 0;
234 static int stat_wbarrier_set_root = 0;
235 static int stat_wbarrier_value_copy = 0;
236 static int stat_wbarrier_object_copy = 0;
239 static long pinned_chunk_bytes_alloced = 0;
240 static long large_internal_bytes_alloced = 0;
243 INTERNAL_MEM_PIN_QUEUE,
244 INTERNAL_MEM_FRAGMENT,
245 INTERNAL_MEM_SECTION,
246 INTERNAL_MEM_SCAN_STARTS,
247 INTERNAL_MEM_FIN_TABLE,
248 INTERNAL_MEM_FINALIZE_ENTRY,
249 INTERNAL_MEM_DISLINK_TABLE,
250 INTERNAL_MEM_DISLINK,
251 INTERNAL_MEM_ROOTS_TABLE,
252 INTERNAL_MEM_ROOT_RECORD,
253 INTERNAL_MEM_STATISTICS,
255 INTERNAL_MEM_GRAY_QUEUE,
256 INTERNAL_MEM_STORE_REMSET,
260 static long small_internal_mem_bytes [INTERNAL_MEM_MAX];
264 mono_gc_flush_info (void)
266 fflush (gc_debug_file);
270 #define MAX_DEBUG_LEVEL 8
271 #define DEBUG(level,a) do {if (G_UNLIKELY ((level) <= MAX_DEBUG_LEVEL && (level) <= gc_debug_level)) a;} while (0)
273 #define TV_DECLARE(name) gint64 name
274 #define TV_GETTIME(tv) tv = mono_100ns_ticks ()
275 #define TV_ELAPSED(start,end) (int)((end-start) / 10)
277 #define ALIGN_TO(val,align) ((((guint64)val) + ((align) - 1)) & ~((align) - 1))
279 #define GC_BITS_PER_WORD (sizeof (mword) * 8)
287 typedef struct _Block Block;
293 /* each request from the OS ends up in a GCMemSection */
294 typedef struct _GCMemSection GCMemSection;
295 struct _GCMemSection {
299 /* pointer where more data could be allocated if it fits */
303 * scan starts is an array of pointers to objects equally spaced in the allocation area
304 * They let use quickly find pinned objects from pinning pointers.
307 /* in major collections indexes in the pin_queue for objects that pin this section */
310 unsigned short num_scan_start;
311 gboolean is_to_space;
314 #define SIZEOF_GC_MEM_SECTION ((sizeof (GCMemSection) + 7) & ~7)
316 /* large object space struct: 64+ KB */
317 /* we could make this limit much smaller to avoid memcpy copy
318 * and potentially have more room in the GC descriptor: need to measure
319 * This also means that such small OS objects will need to be
320 * allocated in a different way (using pinned chunks).
321 * We may want to put large but smaller than 64k objects in the fixed space
322 * when we move the object from one generation to another (to limit the
323 * pig in the snake effect).
324 * Note: it may be worth to have an optimized copy function, since we can
325 * assume that objects are aligned and have a multiple of 8 size.
326 * FIXME: This structure needs to be a multiple of 8 bytes in size: this is not
327 * true if MONO_ZERO_LEN_ARRAY is nonzero.
329 typedef struct _LOSObject LOSObject;
332 mword size; /* this is the object size */
333 int dummy; /* to have a sizeof (LOSObject) a multiple of ALLOC_ALIGN and data starting at same alignment */
336 char data [MONO_ZERO_LEN_ARRAY];
339 /* Pinned objects are allocated in the LOS space if bigger than half a page
340 * or from freelists otherwise. We assume that pinned objects are relatively few
341 * and they have a slow dying speed (like interned strings, thread objects).
342 * As such they will be collected only at major collections.
343 * free lists are not global: when we need memory we allocate a PinnedChunk.
344 * Each pinned chunk is made of several pages, the first of wich is used
345 * internally for bookeeping (here think of a page as 4KB). The bookeeping
346 * includes the freelists vectors and info about the object size of each page
347 * in the pinned chunk. So, when needed, a free page is found in a pinned chunk,
348 * a size is assigned to it, the page is divided in the proper chunks and each
349 * chunk is added to the freelist. To not waste space, the remaining space in the
350 * first page is used as objects of size 16 or 32 (need to measure which are more
352 * We use this same structure to allocate memory used internally by the GC, so
353 * we never use malloc/free if we need to alloc during collection: the world is stopped
354 * and malloc/free will deadlock.
355 * When we want to iterate over pinned objects, we just scan a page at a time
356 * linearly according to the size of objects in the page: the next pointer used to link
357 * the items in the freelist uses the same word as the vtable. Since we keep freelists
358 * for each pinned chunk, if the word points outside the pinned chunk it means
360 * We could avoid this expensive scanning in creative ways. We could have a policy
361 * of putting in the pinned space only objects we know about that have no struct fields
362 * with references and we can easily use a even expensive write barrier for them,
363 * since pointer writes on such objects should be rare.
364 * The best compromise is to just alloc interned strings and System.MonoType in them.
365 * It would be nice to allocate MonoThread in it, too: must check that we properly
366 * use write barriers so we don't have to do any expensive scanning of the whole pinned
367 * chunk list during minor collections. We can avoid it now because we alloc in it only
368 * reference-free objects.
370 #define PINNED_FIRST_SLOT_SIZE (sizeof (gpointer) * 4)
371 #define MAX_FREELIST_SIZE 2048
372 #define PINNED_PAGE_SIZE (4096)
373 #define PINNED_CHUNK_MIN_SIZE (4096*8)
374 typedef struct _PinnedChunk PinnedChunk;
375 struct _PinnedChunk {
378 int *page_sizes; /* a 0 means the page is still unused */
381 void *data [1]; /* page sizes and free lists are stored here */
384 /* The method used to clear the nursery */
385 /* Clearing at nursery collections is the safest, but has bad interactions with caches.
386 * Clearing at TLAB creation is much faster, but more complex and it might expose hard
391 CLEAR_AT_TLAB_CREATION
392 } NurseryClearPolicy;
394 static NurseryClearPolicy nursery_clear_policy = CLEAR_AT_TLAB_CREATION;
397 * If this is set, the nursery is aligned to an address aligned to its size, ie.
398 * a 1MB nursery will be aligned to an address divisible by 1MB. This allows us to
399 * speed up ptr_in_nursery () checks which are very frequent. This requires the
400 * nursery size to be a compile time constant.
402 #define ALIGN_NURSERY 1
405 * The young generation is divided into fragments. This is because
406 * we can hand one fragments to a thread for lock-less fast alloc and
407 * because the young generation ends up fragmented anyway by pinned objects.
408 * Once a collection is done, a list of fragments is created. When doing
409 * thread local alloc we use smallish nurseries so we allow new threads to
410 * allocate memory from gen0 without triggering a collection. Threads that
411 * are found to allocate lots of memory are given bigger fragments. This
412 * should make the finalizer thread use little nursery memory after a while.
413 * We should start assigning threads very small fragments: if there are many
414 * threads the nursery will be full of reserved space that the threads may not
415 * use at all, slowing down allocation speed.
416 * Thread local allocation is done from areas of memory Hotspot calls Thread Local
417 * Allocation Buffers (TLABs).
419 typedef struct _Fragment Fragment;
423 char *fragment_start;
424 char *fragment_limit; /* the current soft limit for allocation */
428 /* the runtime can register areas of memory as roots: we keep two lists of roots,
429 * a pinned root set for conservatively scanned roots and a normal one for
430 * precisely scanned roots (currently implemented as a single list).
432 typedef struct _RootRecord RootRecord;
440 /* for use with write barriers */
441 typedef struct _RememberedSet RememberedSet;
442 struct _RememberedSet {
446 mword data [MONO_ZERO_LEN_ARRAY];
450 * We're never actually using the first element. It's always set to
451 * NULL to simplify the elimination of consecutive duplicate
454 #define STORE_REMSET_BUFFER_SIZE 1024
456 typedef struct _GenericStoreRememberedSet GenericStoreRememberedSet;
457 struct _GenericStoreRememberedSet {
458 GenericStoreRememberedSet *next;
459 /* We need one entry less because the first entry of store
460 remset buffers is always a dummy and we don't copy it. */
461 gpointer data [STORE_REMSET_BUFFER_SIZE - 1];
464 /* we have 4 possible values in the low 2 bits */
466 REMSET_LOCATION, /* just a pointer to the exact location */
467 REMSET_RANGE, /* range of pointer fields */
468 REMSET_OBJECT, /* mark all the object for scanning */
469 REMSET_OTHER, /* all others */
470 REMSET_TYPE_MASK = 0x3
473 /* Subtypes of REMSET_OTHER */
475 REMSET_VTYPE, /* a valuetype array described by a gc descriptor and a count */
476 REMSET_ROOT_LOCATION, /* a location inside a root */
479 #ifdef HAVE_KW_THREAD
480 static __thread RememberedSet *remembered_set MONO_TLS_FAST;
482 static pthread_key_t remembered_set_key;
483 static RememberedSet *global_remset;
484 static RememberedSet *freed_thread_remsets;
485 //static int store_to_global_remset = 0;
486 static GenericStoreRememberedSet *generic_store_remsets = NULL;
488 /* FIXME: later choose a size that takes into account the RememberedSet struct
489 * and doesn't waste any alloc paddin space.
491 #define DEFAULT_REMSET_SIZE 1024
492 static RememberedSet* alloc_remset (int size, gpointer id);
494 /* Structure that corresponds to a MonoVTable: desc is a mword so requires
495 * no cast from a pointer to an integer
502 /* these bits are set in the object vtable: we could merge them since an object can be
503 * either pinned or forwarded but not both.
504 * We store them in the vtable slot because the bits are used in the sync block for
505 * other purposes: if we merge them and alloc the sync blocks aligned to 8 bytes, we can change
506 * this and use bit 3 in the syncblock (with the lower two bits both set for forwarded, that
507 * would be an invalid combination for the monitor and hash code).
508 * The values are already shifted.
509 * The forwarding address is stored in the sync block.
511 #define FORWARDED_BIT 1
513 #define VTABLE_BITS_MASK 0x3
515 /* returns NULL if not forwarded, or the forwarded address */
516 #define object_is_forwarded(obj) (((mword*)(obj))[0] & FORWARDED_BIT? (void*)(((mword*)(obj))[1]): NULL)
517 /* set the forwarded address fw_addr for object obj */
518 #define forward_object(obj,fw_addr) do { \
519 ((mword*)(obj))[0] |= FORWARDED_BIT; \
520 ((mword*)(obj))[1] = (mword)(fw_addr); \
523 #define object_is_pinned(obj) (((mword*)(obj))[0] & PINNED_BIT)
524 #define pin_object(obj) do { \
525 ((mword*)(obj))[0] |= PINNED_BIT; \
527 #define unpin_object(obj) do { \
528 ((mword*)(obj))[0] &= ~PINNED_BIT; \
532 #define ptr_in_nursery(ptr) (((mword)(ptr) & ~((1 << DEFAULT_NURSERY_BITS) - 1)) == (mword)nursery_start)
534 #define ptr_in_nursery(ptr) ((char*)(ptr) >= nursery_start && (char*)(ptr) < nursery_real_end)
538 * Since we set bits in the vtable, use the macro to load it from the pointer to
539 * an object that is potentially pinned.
541 #define LOAD_VTABLE(addr) ((*(mword*)(addr)) & ~VTABLE_BITS_MASK)
544 safe_name (void* obj)
546 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (obj);
547 return vt->klass->name;
551 safe_object_get_size (MonoObject* o)
553 MonoClass *klass = ((MonoVTable*)LOAD_VTABLE (o))->klass;
554 if (klass == mono_defaults.string_class) {
555 return sizeof (MonoString) + 2 * mono_string_length ((MonoString*) o) + 2;
556 } else if (klass->rank) {
557 MonoArray *array = (MonoArray*)o;
558 size_t size = sizeof (MonoArray) + klass->sizes.element_size * mono_array_length (array);
559 if (G_UNLIKELY (array->bounds)) {
560 size += sizeof (mono_array_size_t) - 1;
561 size &= ~(sizeof (mono_array_size_t) - 1);
562 size += sizeof (MonoArrayBounds) * klass->rank;
566 /* from a created object: the class must be inited already */
567 return klass->instance_size;
572 * ######################################################################
573 * ######## Global data.
574 * ######################################################################
576 static LOCK_DECLARE (gc_mutex);
577 static int gc_disabled = 0;
578 static int num_minor_gcs = 0;
579 static int num_major_gcs = 0;
581 /* good sizes are 512KB-1MB: larger ones increase a lot memzeroing time */
582 //#define DEFAULT_NURSERY_SIZE (1024*512*125+4096*118)
583 #define DEFAULT_NURSERY_SIZE (1024*512*2)
584 /* The number of trailing 0 bits in DEFAULT_NURSERY_SIZE */
585 #define DEFAULT_NURSERY_BITS 20
586 #define MAJOR_SECTION_SIZE (128*1024)
587 #define BLOCK_FOR_OBJECT(o) ((Block*)(((mword)(o)) & ~(MAJOR_SECTION_SIZE - 1)))
588 #define MAJOR_SECTION_FOR_OBJECT(o) ((GCMemSection*)BLOCK_FOR_OBJECT ((o)))
589 #define MIN_MINOR_COLLECTION_SECTION_ALLOWANCE (DEFAULT_NURSERY_SIZE * 3 / MAJOR_SECTION_SIZE)
590 #define DEFAULT_LOS_COLLECTION_TARGET (DEFAULT_NURSERY_SIZE * 2)
591 /* to quickly find the head of an object pinned by a conservative address
592 * we keep track of the objects allocated for each SCAN_START_SIZE memory
593 * chunk in the nursery or other memory sections. Larger values have less
594 * memory overhead and bigger runtime cost. 4-8 KB are reasonable values.
596 #define SCAN_START_SIZE (4096*2)
597 /* the minimum size of a fragment that we consider useful for allocation */
598 #define FRAGMENT_MIN_SIZE (512)
599 /* This is a fixed value used for pinned chunks, not the system pagesize */
600 #define FREELIST_PAGESIZE 4096
602 static mword pagesize = 4096;
603 static mword nursery_size = DEFAULT_NURSERY_SIZE;
604 static int degraded_mode = 0;
606 static int minor_collection_section_allowance = MIN_MINOR_COLLECTION_SECTION_ALLOWANCE;
607 static int minor_collection_sections_alloced = 0;
608 static int num_major_sections = 0;
610 static LOSObject *los_object_list = NULL;
611 static mword los_memory_usage = 0;
612 static mword los_num_objects = 0;
613 static mword next_los_collection = 2*1024*1024; /* 2 MB, need to tune */
614 static mword total_alloc = 0;
615 /* use this to tune when to do a major/minor collection */
616 static mword memory_pressure = 0;
618 static GCMemSection *section_list = NULL;
619 static GCMemSection *nursery_section = NULL;
620 static mword lowest_heap_address = ~(mword)0;
621 static mword highest_heap_address = 0;
623 typedef struct _FinalizeEntry FinalizeEntry;
624 struct _FinalizeEntry {
629 typedef struct _FinalizeEntryHashTable FinalizeEntryHashTable;
630 struct _FinalizeEntryHashTable {
631 FinalizeEntry **table;
636 typedef struct _DisappearingLink DisappearingLink;
637 struct _DisappearingLink {
638 DisappearingLink *next;
642 typedef struct _DisappearingLinkHashTable DisappearingLinkHashTable;
643 struct _DisappearingLinkHashTable {
644 DisappearingLink **table;
649 #define LARGE_INTERNAL_MEM_HEADER_MAGIC 0x7d289f3a
651 typedef struct _LargeInternalMemHeader LargeInternalMemHeader;
652 struct _LargeInternalMemHeader {
665 * The link pointer is hidden by negating each bit. We use the lowest
666 * bit of the link (before negation) to store whether it needs
667 * resurrection tracking.
669 #define HIDE_POINTER(p,t) ((gpointer)(~((gulong)(p)|((t)?1:0))))
670 #define REVEAL_POINTER(p) ((gpointer)((~(gulong)(p))&~3L))
672 #define DISLINK_OBJECT(d) (REVEAL_POINTER (*(d)->link))
673 #define DISLINK_TRACK(d) ((~(gulong)(*(d)->link)) & 1)
676 * The finalizable hash has the object as the key, the
677 * disappearing_link hash, has the link address as key.
679 static FinalizeEntryHashTable minor_finalizable_hash;
680 static FinalizeEntryHashTable major_finalizable_hash;
681 /* objects that are ready to be finalized */
682 static FinalizeEntry *fin_ready_list = NULL;
683 static FinalizeEntry *critical_fin_list = NULL;
685 static DisappearingLinkHashTable minor_disappearing_link_hash;
686 static DisappearingLinkHashTable major_disappearing_link_hash;
688 static int num_ready_finalizers = 0;
689 static int no_finalize = 0;
691 /* keep each size a multiple of ALLOC_ALIGN */
692 /* on 64 bit systems 8 is likely completely unused. */
693 static const int freelist_sizes [] = {
694 8, 16, 24, 32, 40, 48, 64, 80,
695 96, 128, 160, 192, 224, 256, 320, 384,
696 448, 512, 584, 680, 816, 1024, 1360, 2048};
697 #define FREELIST_NUM_SLOTS (sizeof (freelist_sizes) / sizeof (freelist_sizes [0]))
699 static char* max_pinned_chunk_addr = NULL;
700 static char* min_pinned_chunk_addr = (char*)-1;
701 /* pinned_chunk_list is used for allocations of objects that are never moved */
702 static PinnedChunk *pinned_chunk_list = NULL;
703 /* internal_chunk_list is used for allocating structures needed by the GC */
704 static PinnedChunk *internal_chunk_list = NULL;
707 obj_is_from_pinned_alloc (char *p)
709 return BLOCK_FOR_OBJECT (p)->role == MEMORY_ROLE_PINNED;
712 static int slot_for_size (size_t size);
715 free_pinned_object (PinnedChunk *chunk, char *obj, size_t size)
717 void **p = (void**)obj;
718 int slot = slot_for_size (size);
720 g_assert (obj >= (char*)chunk->start_data && obj < ((char*)chunk + chunk->num_pages * FREELIST_PAGESIZE));
721 *p = chunk->free_list [slot];
722 chunk->free_list [slot] = p;
726 ROOT_TYPE_NORMAL = 0, /* "normal" roots */
727 ROOT_TYPE_PINNED = 1, /* roots without a GC descriptor */
728 ROOT_TYPE_WBARRIER = 2, /* roots with a write barrier */
732 /* registered roots: the key to the hash is the root start address */
734 * Different kinds of roots are kept separate to speed up pin_from_roots () for example.
736 static RootRecord **roots_hash [ROOT_TYPE_NUM] = { NULL, NULL };
737 static int roots_hash_size [ROOT_TYPE_NUM] = { 0, 0, 0 };
738 static mword roots_size = 0; /* amount of memory in the root set */
739 static int num_roots_entries [ROOT_TYPE_NUM] = { 0, 0, 0 };
742 * The current allocation cursors
743 * We allocate objects in the nursery.
744 * The nursery is the area between nursery_start and nursery_real_end.
745 * Allocation is done from a Thread Local Allocation Buffer (TLAB). TLABs are allocated
746 * from nursery fragments.
747 * tlab_next is the pointer to the space inside the TLAB where the next object will
749 * tlab_temp_end is the pointer to the end of the temporary space reserved for
750 * the allocation: it allows us to set the scan starts at reasonable intervals.
751 * tlab_real_end points to the end of the TLAB.
752 * nursery_frag_real_end points to the end of the currently used nursery fragment.
753 * nursery_first_pinned_start points to the start of the first pinned object in the nursery
754 * nursery_last_pinned_end points to the end of the last pinned object in the nursery
755 * At the next allocation, the area of the nursery where objects can be present is
756 * between MIN(nursery_first_pinned_start, first_fragment_start) and
757 * MAX(nursery_last_pinned_end, nursery_frag_real_end)
759 static char *nursery_start = NULL;
761 /* eventually share with MonoThread? */
762 typedef struct _SgenThreadInfo SgenThreadInfo;
764 struct _SgenThreadInfo {
765 SgenThreadInfo *next;
767 unsigned int stop_count; /* to catch duplicate signals */
770 volatile int in_critical_region;
773 void *stack_start_limit;
774 char **tlab_next_addr;
775 char **tlab_start_addr;
776 char **tlab_temp_end_addr;
777 char **tlab_real_end_addr;
778 gpointer **store_remset_buffer_addr;
779 long *store_remset_buffer_index_addr;
780 RememberedSet *remset;
781 gpointer runtime_data;
782 gpointer stopped_ip; /* only valid if the thread is stopped */
783 MonoDomain *stopped_domain; /* ditto */
784 gpointer *stopped_regs; /* ditto */
785 #ifndef HAVE_KW_THREAD
790 gpointer *store_remset_buffer;
791 long store_remset_buffer_index;
795 #ifdef HAVE_KW_THREAD
796 #define TLAB_ACCESS_INIT
797 #define TLAB_START tlab_start
798 #define TLAB_NEXT tlab_next
799 #define TLAB_TEMP_END tlab_temp_end
800 #define TLAB_REAL_END tlab_real_end
801 #define REMEMBERED_SET remembered_set
802 #define STORE_REMSET_BUFFER store_remset_buffer
803 #define STORE_REMSET_BUFFER_INDEX store_remset_buffer_index
804 #define IN_CRITICAL_REGION thread_info->in_critical_region
806 static pthread_key_t thread_info_key;
807 #define TLAB_ACCESS_INIT SgenThreadInfo *__thread_info__ = pthread_getspecific (thread_info_key)
808 #define TLAB_START (__thread_info__->tlab_start)
809 #define TLAB_NEXT (__thread_info__->tlab_next)
810 #define TLAB_TEMP_END (__thread_info__->tlab_temp_end)
811 #define TLAB_REAL_END (__thread_info__->tlab_real_end)
812 #define REMEMBERED_SET (__thread_info__->remset)
813 #define STORE_REMSET_BUFFER (__thread_info__->store_remset_buffer)
814 #define STORE_REMSET_BUFFER_INDEX (__thread_info__->store_remset_buffer_index)
815 #define IN_CRITICAL_REGION (__thread_info__->in_critical_region)
818 /* we use the memory barrier only to prevent compiler reordering (a memory constraint may be enough) */
819 #define ENTER_CRITICAL_REGION do {IN_CRITICAL_REGION = 1;mono_memory_barrier ();} while (0)
820 #define EXIT_CRITICAL_REGION do {IN_CRITICAL_REGION = 0;mono_memory_barrier ();} while (0)
823 * FIXME: What is faster, a TLS variable pointing to a structure, or separate TLS
824 * variables for next+temp_end ?
826 #ifdef HAVE_KW_THREAD
827 static __thread SgenThreadInfo *thread_info;
828 static __thread char *tlab_start;
829 static __thread char *tlab_next;
830 static __thread char *tlab_temp_end;
831 static __thread char *tlab_real_end;
832 static __thread gpointer *store_remset_buffer;
833 static __thread long store_remset_buffer_index;
834 /* Used by the managed allocator/wbarrier */
835 static __thread char **tlab_next_addr;
836 static __thread char *stack_end;
837 static __thread long *store_remset_buffer_index_addr;
839 static char *nursery_next = NULL;
840 static char *nursery_frag_real_end = NULL;
841 static char *nursery_real_end = NULL;
842 //static char *nursery_first_pinned_start = NULL;
843 static char *nursery_last_pinned_end = NULL;
845 /* The size of a TLAB */
846 /* The bigger the value, the less often we have to go to the slow path to allocate a new
847 * one, but the more space is wasted by threads not allocating much memory.
849 * FIXME: Make this self-tuning for each thread.
851 static guint32 tlab_size = (1024 * 4);
853 /* fragments that are free and ready to be used for allocation */
854 static Fragment *nursery_fragments = NULL;
855 /* freeelist of fragment structures */
856 static Fragment *fragment_freelist = NULL;
859 * used when moving the objects
861 static char *to_space_bumper = NULL;
862 static char *to_space_top = NULL;
863 static GCMemSection *to_space_section = NULL;
865 /* objects bigger then this go into the large object space */
866 #define MAX_SMALL_OBJ_SIZE MAX_FREELIST_SIZE
868 /* Functions supplied by the runtime to be called by the GC */
869 static MonoGCCallbacks gc_callbacks;
872 * ######################################################################
873 * ######## Macros and function declarations.
874 * ######################################################################
877 #define UPDATE_HEAP_BOUNDARIES(low,high) do { \
878 if ((mword)(low) < lowest_heap_address) \
879 lowest_heap_address = (mword)(low); \
880 if ((mword)(high) > highest_heap_address) \
881 highest_heap_address = (mword)(high); \
883 #define ADDR_IN_HEAP_BOUNDARIES(addr) ((p) >= lowest_heap_address && (p) < highest_heap_address)
886 align_pointer (void *ptr)
888 mword p = (mword)ptr;
889 p += sizeof (gpointer) - 1;
890 p &= ~ (sizeof (gpointer) - 1);
894 /* forward declarations */
895 static void* get_internal_mem (size_t size, int type);
896 static void free_internal_mem (void *addr, int type);
897 static void* get_os_memory (size_t size, int activate);
898 static void free_os_memory (void *addr, size_t size);
899 static G_GNUC_UNUSED void report_internal_mem_usage (void);
901 static int stop_world (void);
902 static int restart_world (void);
903 static void scan_thread_data (void *start_nursery, void *end_nursery, gboolean precise);
904 static void scan_from_remsets (void *start_nursery, void *end_nursery);
905 static void find_pinning_ref_from_thread (char *obj, size_t size);
906 static void update_current_thread_stack (void *start);
907 static GCMemSection* alloc_major_section (void);
908 static void finalize_in_range (char *start, char *end, int generation);
909 static void add_or_remove_disappearing_link (MonoObject *obj, void **link, gboolean track, int generation);
910 static void null_link_in_range (char *start, char *end, int generation);
911 static void null_links_for_domain (MonoDomain *domain, int generation);
912 static gboolean search_fragment_for_size (size_t size);
913 static void mark_pinned_from_addresses (PinnedChunk *chunk, void **start, void **end);
914 static void clear_remsets (void);
915 static void clear_tlabs (void);
916 typedef void (*ScanPinnedObjectCallbackFunc) (PinnedChunk*, char*, size_t, void*);
917 static void scan_pinned_objects (ScanPinnedObjectCallbackFunc callback, void *callback_data);
918 static void sweep_pinned_objects (void);
919 static void scan_from_pinned_objects (char *addr_start, char *addr_end);
920 static void free_large_object (LOSObject *obj);
921 static void free_major_section (GCMemSection *section);
922 static void to_space_expand (void);
924 static void mono_gc_register_disappearing_link (MonoObject *obj, void **link, gboolean track);
926 void describe_ptr (char *ptr);
927 void check_consistency (void);
928 char* check_object (char *start);
930 void mono_gc_scan_for_specific_ref (MonoObject *key);
933 * ######################################################################
934 * ######## GC descriptors
935 * ######################################################################
936 * Used to quickly get the info the GC needs about an object: size and
937 * where the references are held.
939 /* objects are aligned to 8 bytes boundaries
940 * A descriptor is a pointer in MonoVTable, so 32 or 64 bits of size.
941 * The low 3 bits define the type of the descriptor. The other bits
942 * depend on the type.
943 * As a general rule the 13 remaining low bits define the size, either
944 * of the whole object or of the elements in the arrays. While for objects
945 * the size is already in bytes, for arrays we need to shift, because
946 * array elements might be smaller than 8 bytes. In case of arrays, we
947 * use two bits to describe what the additional high bits represents,
948 * so the default behaviour can handle element sizes less than 2048 bytes.
949 * The high 16 bits, if 0 it means the object is pointer-free.
950 * This design should make it easy and fast to skip over ptr-free data.
951 * The first 4 types should cover >95% of the objects.
952 * Note that since the size of objects is limited to 64K, larger objects
953 * will be allocated in the large object heap.
954 * If we want 4-bytes alignment, we need to put vector and small bitmap
958 DESC_TYPE_RUN_LENGTH, /* 16 bits aligned byte size | 1-3 (offset, numptr) bytes tuples */
959 DESC_TYPE_SMALL_BITMAP, /* 16 bits aligned byte size | 16-48 bit bitmap */
960 DESC_TYPE_STRING, /* nothing */
961 DESC_TYPE_COMPLEX, /* index for bitmap into complex_descriptors */
962 DESC_TYPE_VECTOR, /* 10 bits element size | 1 bit array | 2 bits desc | element desc */
963 DESC_TYPE_ARRAY, /* 10 bits element size | 1 bit array | 2 bits desc | element desc */
964 DESC_TYPE_LARGE_BITMAP, /* | 29-61 bitmap bits */
965 DESC_TYPE_COMPLEX_ARR, /* index for bitmap into complex_descriptors */
966 /* subtypes for arrays and vectors */
967 DESC_TYPE_V_PTRFREE = 0,/* there are no refs: keep first so it has a zero value */
968 DESC_TYPE_V_REFS, /* all the array elements are refs */
969 DESC_TYPE_V_RUN_LEN, /* elements are run-length encoded as DESC_TYPE_RUN_LENGTH */
970 DESC_TYPE_V_BITMAP /* elements are as the bitmap in DESC_TYPE_SMALL_BITMAP */
973 #define OBJECT_HEADER_WORDS (sizeof(MonoObject)/sizeof(gpointer))
974 #define LOW_TYPE_BITS 3
975 #define SMALL_BITMAP_SHIFT 16
976 #define SMALL_BITMAP_SIZE (GC_BITS_PER_WORD - SMALL_BITMAP_SHIFT)
977 #define VECTOR_INFO_SHIFT 14
978 #define VECTOR_ELSIZE_SHIFT 3
979 #define LARGE_BITMAP_SIZE (GC_BITS_PER_WORD - LOW_TYPE_BITS)
980 #define MAX_SMALL_SIZE ((1 << SMALL_BITMAP_SHIFT) - 1)
981 #define SMALL_SIZE_MASK 0xfff8
982 #define MAX_ELEMENT_SIZE 0x3ff
983 #define ELEMENT_SIZE_MASK (0x3ff << LOW_TYPE_BITS)
984 #define VECTOR_SUBTYPE_PTRFREE (DESC_TYPE_V_PTRFREE << VECTOR_INFO_SHIFT)
985 #define VECTOR_SUBTYPE_REFS (DESC_TYPE_V_REFS << VECTOR_INFO_SHIFT)
986 #define VECTOR_SUBTYPE_RUN_LEN (DESC_TYPE_V_RUN_LEN << VECTOR_INFO_SHIFT)
987 #define VECTOR_SUBTYPE_BITMAP (DESC_TYPE_V_BITMAP << VECTOR_INFO_SHIFT)
989 #define ALLOC_ALIGN 8
992 /* Root bitmap descriptors are simpler: the lower three bits describe the type
993 * and we either have 30/62 bitmap bits or nibble-based run-length,
994 * or a complex descriptor, or a user defined marker function.
997 ROOT_DESC_CONSERVATIVE, /* 0, so matches NULL value */
1002 ROOT_DESC_TYPE_MASK = 0x7,
1003 ROOT_DESC_TYPE_SHIFT = 3,
1006 #define MAKE_ROOT_DESC(type,val) ((type) | ((val) << ROOT_DESC_TYPE_SHIFT))
1008 #define MAX_USER_DESCRIPTORS 16
1010 static gsize* complex_descriptors = NULL;
1011 static int complex_descriptors_size = 0;
1012 static int complex_descriptors_next = 0;
1013 static MonoGCMarkFunc user_descriptors [MAX_USER_DESCRIPTORS];
1014 static int user_descriptors_next = 0;
1017 alloc_complex_descriptor (gsize *bitmap, int numbits)
1021 numbits = ALIGN_TO (numbits, GC_BITS_PER_WORD);
1022 nwords = numbits / GC_BITS_PER_WORD + 1;
1025 res = complex_descriptors_next;
1026 /* linear search, so we don't have duplicates with domain load/unload
1027 * this should not be performance critical or we'd have bigger issues
1028 * (the number and size of complex descriptors should be small).
1030 for (i = 0; i < complex_descriptors_next; ) {
1031 if (complex_descriptors [i] == nwords) {
1032 int j, found = TRUE;
1033 for (j = 0; j < nwords - 1; ++j) {
1034 if (complex_descriptors [i + 1 + j] != bitmap [j]) {
1044 i += complex_descriptors [i];
1046 if (complex_descriptors_next + nwords > complex_descriptors_size) {
1047 int new_size = complex_descriptors_size * 2 + nwords;
1048 complex_descriptors = g_realloc (complex_descriptors, new_size * sizeof (gsize));
1049 complex_descriptors_size = new_size;
1051 DEBUG (6, fprintf (gc_debug_file, "Complex descriptor %d, size: %d (total desc memory: %d)\n", res, nwords, complex_descriptors_size));
1052 complex_descriptors_next += nwords;
1053 complex_descriptors [res] = nwords;
1054 for (i = 0; i < nwords - 1; ++i) {
1055 complex_descriptors [res + 1 + i] = bitmap [i];
1056 DEBUG (6, fprintf (gc_debug_file, "\tvalue: %p\n", (void*)complex_descriptors [res + 1 + i]));
1063 * Descriptor builders.
1066 mono_gc_make_descr_for_string (gsize *bitmap, int numbits)
1068 return (void*) DESC_TYPE_STRING;
1072 mono_gc_make_descr_for_object (gsize *bitmap, int numbits, size_t obj_size)
1074 int first_set = -1, num_set = 0, last_set = -1, i;
1076 size_t stored_size = obj_size;
1077 stored_size += ALLOC_ALIGN - 1;
1078 stored_size &= ~(ALLOC_ALIGN - 1);
1079 for (i = 0; i < numbits; ++i) {
1080 if (bitmap [i / GC_BITS_PER_WORD] & ((gsize)1 << (i % GC_BITS_PER_WORD))) {
1087 if (stored_size <= MAX_SMALL_OBJ_SIZE) {
1088 /* check run-length encoding first: one byte offset, one byte number of pointers
1089 * on 64 bit archs, we can have 3 runs, just one on 32.
1090 * It may be better to use nibbles.
1092 if (first_set < 0) {
1093 desc = DESC_TYPE_RUN_LENGTH | stored_size;
1094 DEBUG (6, fprintf (gc_debug_file, "Ptrfree descriptor %p, size: %zd\n", (void*)desc, stored_size));
1095 return (void*) desc;
1096 } else if (first_set < 256 && num_set < 256 && (first_set + num_set == last_set + 1)) {
1097 desc = DESC_TYPE_RUN_LENGTH | stored_size | (first_set << 16) | (num_set << 24);
1098 DEBUG (6, fprintf (gc_debug_file, "Runlen descriptor %p, size: %zd, first set: %d, num set: %d\n", (void*)desc, stored_size, first_set, num_set));
1099 return (void*) desc;
1101 /* we know the 2-word header is ptr-free */
1102 if (last_set < SMALL_BITMAP_SIZE + OBJECT_HEADER_WORDS) {
1103 desc = DESC_TYPE_SMALL_BITMAP | stored_size | ((*bitmap >> OBJECT_HEADER_WORDS) << SMALL_BITMAP_SHIFT);
1104 DEBUG (6, fprintf (gc_debug_file, "Smallbitmap descriptor %p, size: %zd, last set: %d\n", (void*)desc, stored_size, last_set));
1105 return (void*) desc;
1108 /* we know the 2-word header is ptr-free */
1109 if (last_set < LARGE_BITMAP_SIZE + OBJECT_HEADER_WORDS) {
1110 desc = DESC_TYPE_LARGE_BITMAP | ((*bitmap >> OBJECT_HEADER_WORDS) << LOW_TYPE_BITS);
1111 DEBUG (6, fprintf (gc_debug_file, "Largebitmap descriptor %p, size: %zd, last set: %d\n", (void*)desc, stored_size, last_set));
1112 return (void*) desc;
1114 /* it's a complex object ... */
1115 desc = DESC_TYPE_COMPLEX | (alloc_complex_descriptor (bitmap, last_set + 1) << LOW_TYPE_BITS);
1116 return (void*) desc;
1119 /* If the array holds references, numbits == 1 and the first bit is set in elem_bitmap */
1121 mono_gc_make_descr_for_array (int vector, gsize *elem_bitmap, int numbits, size_t elem_size)
1123 int first_set = -1, num_set = 0, last_set = -1, i;
1124 mword desc = vector? DESC_TYPE_VECTOR: DESC_TYPE_ARRAY;
1125 for (i = 0; i < numbits; ++i) {
1126 if (elem_bitmap [i / GC_BITS_PER_WORD] & ((gsize)1 << (i % GC_BITS_PER_WORD))) {
1133 if (elem_size <= MAX_ELEMENT_SIZE) {
1134 desc |= elem_size << VECTOR_ELSIZE_SHIFT;
1136 return (void*)(desc | VECTOR_SUBTYPE_PTRFREE);
1138 /* Note: we also handle structs with just ref fields */
1139 if (num_set * sizeof (gpointer) == elem_size) {
1140 return (void*)(desc | VECTOR_SUBTYPE_REFS | ((gssize)(-1) << 16));
1142 /* FIXME: try run-len first */
1143 /* Note: we can't skip the object header here, because it's not present */
1144 if (last_set <= SMALL_BITMAP_SIZE) {
1145 return (void*)(desc | VECTOR_SUBTYPE_BITMAP | (*elem_bitmap << 16));
1148 /* it's am array of complex structs ... */
1149 desc = DESC_TYPE_COMPLEX_ARR;
1150 desc |= alloc_complex_descriptor (elem_bitmap, last_set + 1) << LOW_TYPE_BITS;
1151 return (void*) desc;
1154 /* Return the bitmap encoded by a descriptor */
1156 mono_gc_get_bitmap_for_descr (void *descr, int *numbits)
1158 mword d = (mword)descr;
1162 case DESC_TYPE_RUN_LENGTH: {
1163 int first_set = (d >> 16) & 0xff;
1164 int num_set = (d >> 24) & 0xff;
1167 bitmap = g_new0 (gsize, (first_set + num_set + 7) / 8);
1169 for (i = first_set; i < first_set + num_set; ++i)
1170 bitmap [i / GC_BITS_PER_WORD] |= ((gsize)1 << (i % GC_BITS_PER_WORD));
1172 *numbits = first_set + num_set;
1176 case DESC_TYPE_SMALL_BITMAP:
1177 bitmap = g_new0 (gsize, 1);
1179 bitmap [0] = (d >> SMALL_BITMAP_SHIFT) << OBJECT_HEADER_WORDS;
1181 *numbits = GC_BITS_PER_WORD;
1185 g_assert_not_reached ();
1189 /* helper macros to scan and traverse objects, macros because we resue them in many functions */
1190 #define STRING_SIZE(size,str) do { \
1191 (size) = sizeof (MonoString) + 2 * mono_string_length ((MonoString*)(str)) + 2; \
1192 (size) += (ALLOC_ALIGN - 1); \
1193 (size) &= ~(ALLOC_ALIGN - 1); \
1196 #define OBJ_RUN_LEN_SIZE(size,desc,obj) do { \
1197 (size) = (desc) & 0xfff8; \
1200 #define OBJ_BITMAP_SIZE(size,desc,obj) do { \
1201 (size) = (desc) & 0xfff8; \
1204 //#define PREFETCH(addr) __asm__ __volatile__ (" prefetchnta %0": : "m"(*(char *)(addr)))
1205 #define PREFETCH(addr)
1207 /* code using these macros must define a HANDLE_PTR(ptr) macro that does the work */
1208 #define OBJ_RUN_LEN_FOREACH_PTR(desc,obj) do { \
1209 if ((desc) & 0xffff0000) { \
1210 /* there are pointers */ \
1211 void **_objptr_end; \
1212 void **_objptr = (void**)(obj); \
1213 _objptr += ((desc) >> 16) & 0xff; \
1214 _objptr_end = _objptr + (((desc) >> 24) & 0xff); \
1215 while (_objptr < _objptr_end) { \
1216 HANDLE_PTR (_objptr, (obj)); \
1222 /* a bitmap desc means that there are pointer references or we'd have
1223 * choosen run-length, instead: add an assert to check.
1225 #define OBJ_BITMAP_FOREACH_PTR(desc,obj) do { \
1226 /* there are pointers */ \
1227 void **_objptr = (void**)(obj); \
1228 gsize _bmap = (desc) >> 16; \
1229 _objptr += OBJECT_HEADER_WORDS; \
1231 if ((_bmap & 1)) { \
1232 HANDLE_PTR (_objptr, (obj)); \
1239 #define OBJ_LARGE_BITMAP_FOREACH_PTR(vt,obj) do { \
1240 /* there are pointers */ \
1241 void **_objptr = (void**)(obj); \
1242 gsize _bmap = (vt)->desc >> LOW_TYPE_BITS; \
1243 _objptr += OBJECT_HEADER_WORDS; \
1245 if ((_bmap & 1)) { \
1246 HANDLE_PTR (_objptr, (obj)); \
1253 #define OBJ_COMPLEX_FOREACH_PTR(vt,obj) do { \
1254 /* there are pointers */ \
1255 void **_objptr = (void**)(obj); \
1256 gsize *bitmap_data = complex_descriptors + ((vt)->desc >> LOW_TYPE_BITS); \
1257 int bwords = (*bitmap_data) - 1; \
1258 void **start_run = _objptr; \
1261 MonoObject *myobj = (MonoObject*)obj; \
1262 g_print ("found %d at %p (0x%zx): %s.%s\n", bwords, (obj), (vt)->desc, myobj->vtable->klass->name_space, myobj->vtable->klass->name); \
1264 while (bwords-- > 0) { \
1265 gsize _bmap = *bitmap_data++; \
1266 _objptr = start_run; \
1267 /*g_print ("bitmap: 0x%x/%d at %p\n", _bmap, bwords, _objptr);*/ \
1269 if ((_bmap & 1)) { \
1270 HANDLE_PTR (_objptr, (obj)); \
1275 start_run += GC_BITS_PER_WORD; \
1279 /* this one is untested */
1280 #define OBJ_COMPLEX_ARR_FOREACH_PTR(vt,obj) do { \
1281 /* there are pointers */ \
1282 gsize *mbitmap_data = complex_descriptors + ((vt)->desc >> LOW_TYPE_BITS); \
1283 int mbwords = (*mbitmap_data++) - 1; \
1284 int el_size = mono_array_element_size (((MonoObject*)(obj))->vtable->klass); \
1285 char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
1286 char *e_end = e_start + el_size * mono_array_length ((MonoArray*)(obj)); \
1288 MonoObject *myobj = (MonoObject*)start; \
1289 g_print ("found %d at %p (0x%zx): %s.%s\n", mbwords, (obj), (vt)->desc, myobj->vtable->klass->name_space, myobj->vtable->klass->name); \
1291 while (e_start < e_end) { \
1292 void **_objptr = (void**)e_start; \
1293 gsize *bitmap_data = mbitmap_data; \
1294 unsigned int bwords = mbwords; \
1295 while (bwords-- > 0) { \
1296 gsize _bmap = *bitmap_data++; \
1297 void **start_run = _objptr; \
1298 /*g_print ("bitmap: 0x%x\n", _bmap);*/ \
1300 if ((_bmap & 1)) { \
1301 HANDLE_PTR (_objptr, (obj)); \
1306 _objptr = start_run + GC_BITS_PER_WORD; \
1308 e_start += el_size; \
1312 #define OBJ_VECTOR_FOREACH_PTR(vt,obj) do { \
1313 /* note: 0xffffc000 excludes DESC_TYPE_V_PTRFREE */ \
1314 if ((vt)->desc & 0xffffc000) { \
1315 int el_size = ((vt)->desc >> 3) & MAX_ELEMENT_SIZE; \
1316 /* there are pointers */ \
1317 int etype = (vt)->desc & 0xc000; \
1318 if (etype == (DESC_TYPE_V_REFS << 14)) { \
1319 void **p = (void**)((char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector)); \
1320 void **end_refs = (void**)((char*)p + el_size * mono_array_length ((MonoArray*)(obj))); \
1321 /* Note: this code can handle also arrays of struct with only references in them */ \
1322 while (p < end_refs) { \
1323 HANDLE_PTR (p, (obj)); \
1326 } else if (etype == DESC_TYPE_V_RUN_LEN << 14) { \
1327 int offset = ((vt)->desc >> 16) & 0xff; \
1328 int num_refs = ((vt)->desc >> 24) & 0xff; \
1329 char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
1330 char *e_end = e_start + el_size * mono_array_length ((MonoArray*)(obj)); \
1331 while (e_start < e_end) { \
1332 void **p = (void**)e_start; \
1335 for (i = 0; i < num_refs; ++i) { \
1336 HANDLE_PTR (p + i, (obj)); \
1338 e_start += el_size; \
1340 } else if (etype == DESC_TYPE_V_BITMAP << 14) { \
1341 char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
1342 char *e_end = e_start + el_size * mono_array_length ((MonoArray*)(obj)); \
1343 while (e_start < e_end) { \
1344 void **p = (void**)e_start; \
1345 gsize _bmap = (vt)->desc >> 16; \
1346 /* Note: there is no object header here to skip */ \
1348 if ((_bmap & 1)) { \
1349 HANDLE_PTR (p, (obj)); \
1354 e_start += el_size; \
1360 #define COUNT_OBJECT_TYPES do { \
1361 switch (desc & 0x7) { \
1362 case DESC_TYPE_STRING: type_str++; break; \
1363 case DESC_TYPE_RUN_LENGTH: type_rlen++; break; \
1364 case DESC_TYPE_ARRAY: case DESC_TYPE_VECTOR: type_vector++; break; \
1365 case DESC_TYPE_SMALL_BITMAP: type_bitmap++; break; \
1366 case DESC_TYPE_LARGE_BITMAP: type_lbit++; break; \
1367 case DESC_TYPE_COMPLEX: type_complex++; break; \
1368 case DESC_TYPE_COMPLEX_ARR: type_complex++; break; \
1369 default: g_assert_not_reached (); \
1375 * ######################################################################
1376 * ######## Detecting and removing garbage.
1377 * ######################################################################
1378 * This section of code deals with detecting the objects no longer in use
1379 * and reclaiming the memory.
1383 static mword new_obj_references = 0;
1384 static mword obj_references_checked = 0;
1387 #define HANDLE_PTR(ptr,obj) do { \
1388 if (*(ptr) && (char*)*(ptr) >= nursery_start && (char*)*(ptr) < nursery_next) { \
1389 new_obj_references++; \
1390 /*printf ("bogus ptr %p found at %p in object %p (%s.%s)\n", *(ptr), (ptr), o, o->vtable->klass->name_space, o->vtable->klass->name);*/ \
1392 obj_references_checked++; \
1396 static void __attribute__((noinline))
1397 scan_area (char *start, char *end)
1400 int type_str = 0, type_rlen = 0, type_bitmap = 0, type_vector = 0, type_lbit = 0, type_complex = 0;
1401 new_obj_references = 0;
1402 obj_references_checked = 0;
1403 while (start < end) {
1404 if (!*(void**)start) {
1405 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
1408 vt = (GCVTable*)LOAD_VTABLE (start);
1409 DEBUG (8, fprintf (gc_debug_file, "Scanning object %p, vtable: %p (%s)\n", start, vt, vt->klass->name));
1411 MonoObject *obj = (MonoObject*)start;
1412 g_print ("found at %p (0x%zx): %s.%s\n", start, vt->desc, obj->vtable->klass->name_space, obj->vtable->klass->name);
1415 #define SCAN_OBJECT_ACTION COUNT_OBJECT_TYPES
1416 #include "sgen-scan-object.h"
1418 /*printf ("references to new nursery %p-%p (size: %dk): %d, checked: %d\n", old_start, end, (end-old_start)/1024, new_obj_references, obj_references_checked);
1419 printf ("\tstrings: %d, runl: %d, vector: %d, bitmaps: %d, lbitmaps: %d, complex: %d\n",
1420 type_str, type_rlen, type_vector, type_bitmap, type_lbit, type_complex);*/
1425 is_xdomain_ref_allowed (gpointer *ptr, char *obj, MonoDomain *domain)
1427 MonoObject *o = (MonoObject*)(obj);
1428 MonoObject *ref = (MonoObject*)*(ptr);
1429 int offset = (char*)(ptr) - (char*)o;
1431 if (o->vtable->klass == mono_defaults.thread_class && offset == G_STRUCT_OFFSET (MonoThread, internal_thread))
1433 if (o->vtable->klass == mono_defaults.internal_thread_class && offset == G_STRUCT_OFFSET (MonoInternalThread, current_appcontext))
1435 if (mono_class_has_parent (o->vtable->klass, mono_defaults.real_proxy_class) &&
1436 offset == G_STRUCT_OFFSET (MonoRealProxy, unwrapped_server))
1438 /* Thread.cached_culture_info */
1439 if (!strcmp (ref->vtable->klass->name_space, "System.Globalization") &&
1440 !strcmp (ref->vtable->klass->name, "CultureInfo") &&
1441 !strcmp(o->vtable->klass->name_space, "System") &&
1442 !strcmp(o->vtable->klass->name, "Object[]"))
1445 * at System.IO.MemoryStream.InternalConstructor (byte[],int,int,bool,bool) [0x0004d] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.IO/MemoryStream.cs:121
1446 * at System.IO.MemoryStream..ctor (byte[]) [0x00017] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.IO/MemoryStream.cs:81
1447 * at (wrapper remoting-invoke-with-check) System.IO.MemoryStream..ctor (byte[]) <IL 0x00020, 0xffffffff>
1448 * at System.Runtime.Remoting.Messaging.CADMethodCallMessage.GetArguments () [0x0000d] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Messaging/CADMessages.cs:327
1449 * at System.Runtime.Remoting.Messaging.MethodCall..ctor (System.Runtime.Remoting.Messaging.CADMethodCallMessage) [0x00017] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Messaging/MethodCall.cs:87
1450 * at System.AppDomain.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage,byte[]&,System.Runtime.Remoting.Messaging.CADMethodReturnMessage&) [0x00018] in /home/schani/Work/novell/trunk/mcs/class/corlib/System/AppDomain.cs:1213
1451 * at (wrapper remoting-invoke-with-check) System.AppDomain.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage,byte[]&,System.Runtime.Remoting.Messaging.CADMethodReturnMessage&) <IL 0x0003d, 0xffffffff>
1452 * at System.Runtime.Remoting.Channels.CrossAppDomainSink.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage) [0x00008] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Channels/CrossAppDomainChannel.cs:198
1453 * at (wrapper runtime-invoke) object.runtime_invoke_CrossAppDomainSink/ProcessMessageRes_object_object (object,intptr,intptr,intptr) <IL 0x0004c, 0xffffffff>
1455 if (!strcmp (ref->vtable->klass->name_space, "System") &&
1456 !strcmp (ref->vtable->klass->name, "Byte[]") &&
1457 !strcmp (o->vtable->klass->name_space, "System.IO") &&
1458 !strcmp (o->vtable->klass->name, "MemoryStream"))
1460 /* append_job() in threadpool.c */
1461 if (!strcmp (ref->vtable->klass->name_space, "System.Runtime.Remoting.Messaging") &&
1462 !strcmp (ref->vtable->klass->name, "AsyncResult") &&
1463 !strcmp (o->vtable->klass->name_space, "System") &&
1464 !strcmp (o->vtable->klass->name, "Object[]") &&
1465 mono_thread_pool_is_queue_array ((MonoArray*) o))
1471 check_reference_for_xdomain (gpointer *ptr, char *obj, MonoDomain *domain)
1473 MonoObject *o = (MonoObject*)(obj);
1474 MonoObject *ref = (MonoObject*)*(ptr);
1475 int offset = (char*)(ptr) - (char*)o;
1477 MonoClassField *field;
1480 if (!ref || ref->vtable->domain == domain)
1482 if (is_xdomain_ref_allowed (ptr, obj, domain))
1486 for (class = o->vtable->klass; class; class = class->parent) {
1489 for (i = 0; i < class->field.count; ++i) {
1490 if (class->fields[i].offset == offset) {
1491 field = &class->fields[i];
1499 if (ref->vtable->klass == mono_defaults.string_class)
1500 str = mono_string_to_utf8 ((MonoString*)ref);
1503 g_print ("xdomain reference in %p (%s.%s) at offset %d (%s) to %p (%s.%s) (%s) - pointed to by:\n",
1504 o, o->vtable->klass->name_space, o->vtable->klass->name,
1505 offset, field ? field->name : "",
1506 ref, ref->vtable->klass->name_space, ref->vtable->klass->name, str ? str : "");
1507 mono_gc_scan_for_specific_ref (o);
1513 #define HANDLE_PTR(ptr,obj) check_reference_for_xdomain ((ptr), (obj), domain)
1516 scan_object_for_xdomain_refs (char *start)
1518 MonoDomain *domain = ((MonoObject*)start)->vtable->domain;
1520 #include "sgen-scan-object.h"
1526 scan_area_for_xdomain_refs (char *start, char *end)
1528 while (start < end) {
1529 if (!*(void**)start) {
1530 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
1534 start = scan_object_for_xdomain_refs (start);
1539 #define HANDLE_PTR(ptr,obj) do { \
1540 if ((MonoObject*)*(ptr) == key) { \
1541 g_print ("found ref to %p in object %p (%s) at offset %zd\n", \
1542 key, (obj), safe_name ((obj)), ((char*)(ptr) - (char*)(obj))); \
1547 scan_object_for_specific_ref (char *start, MonoObject *key)
1549 #include "sgen-scan-object.h"
1555 scan_area_for_specific_ref (char *start, char *end, MonoObject *key)
1557 while (start < end) {
1558 if (!*(void**)start) {
1559 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
1563 start = scan_object_for_specific_ref (start, key);
1568 scan_pinned_object_for_specific_ref_callback (PinnedChunk *chunk, char *obj, size_t size, MonoObject *key)
1570 scan_object_for_specific_ref (obj, key);
1574 check_root_obj_specific_ref (RootRecord *root, MonoObject *key, MonoObject *obj)
1578 g_print ("found ref to %p in root record %p\n", key, root);
1581 static MonoObject *check_key = NULL;
1582 static RootRecord *check_root = NULL;
1585 check_root_obj_specific_ref_from_marker (void *obj)
1587 check_root_obj_specific_ref (check_root, check_key, obj);
1592 scan_roots_for_specific_ref (MonoObject *key, int root_type)
1597 for (i = 0; i < roots_hash_size [root_type]; ++i) {
1598 for (root = roots_hash [root_type][i]; root; root = root->next) {
1599 void **start_root = (void**)root->start_root;
1600 mword desc = root->root_desc;
1604 switch (desc & ROOT_DESC_TYPE_MASK) {
1605 case ROOT_DESC_BITMAP:
1606 desc >>= ROOT_DESC_TYPE_SHIFT;
1609 check_root_obj_specific_ref (root, key, *start_root);
1614 case ROOT_DESC_COMPLEX: {
1615 gsize *bitmap_data = complex_descriptors + (desc >> ROOT_DESC_TYPE_SHIFT);
1616 int bwords = (*bitmap_data) - 1;
1617 void **start_run = start_root;
1619 while (bwords-- > 0) {
1620 gsize bmap = *bitmap_data++;
1621 void **objptr = start_run;
1624 check_root_obj_specific_ref (root, key, *objptr);
1628 start_run += GC_BITS_PER_WORD;
1632 case ROOT_DESC_USER: {
1633 MonoGCMarkFunc marker = user_descriptors [desc >> ROOT_DESC_TYPE_SHIFT];
1634 marker (start_root, check_root_obj_specific_ref_from_marker);
1637 case ROOT_DESC_RUN_LEN:
1638 g_assert_not_reached ();
1640 g_assert_not_reached ();
1649 mono_gc_scan_for_specific_ref (MonoObject *key)
1651 GCMemSection *section;
1656 for (section = section_list; section; section = section->block.next)
1657 scan_area_for_specific_ref (section->data, section->end_data, key);
1659 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1660 scan_object_for_specific_ref (bigobj->data, key);
1662 scan_pinned_objects ((ScanPinnedObjectCallbackFunc)scan_pinned_object_for_specific_ref_callback, key);
1664 scan_roots_for_specific_ref (key, ROOT_TYPE_NORMAL);
1665 scan_roots_for_specific_ref (key, ROOT_TYPE_WBARRIER);
1667 for (i = 0; i < roots_hash_size [ROOT_TYPE_PINNED]; ++i) {
1668 for (root = roots_hash [ROOT_TYPE_PINNED][i]; root; root = root->next) {
1669 void **ptr = (void**)root->start_root;
1671 while (ptr < (void**)root->end_root) {
1672 check_root_obj_specific_ref (root, *ptr, key);
1679 //#define BINARY_PROTOCOL
1680 #include "sgen-protocol.c"
1683 need_remove_object_for_domain (char *start, MonoDomain *domain)
1685 if (mono_object_domain (start) == domain) {
1686 DEBUG (1, fprintf (gc_debug_file, "Need to cleanup object %p\n", start));
1687 binary_protocol_cleanup (start, (gpointer)LOAD_VTABLE (start), safe_object_get_size ((MonoObject*)start));
1694 process_object_for_domain_clearing (char *start, MonoDomain *domain)
1696 GCVTable *vt = (GCVTable*)LOAD_VTABLE (start);
1697 if (vt->klass == mono_defaults.internal_thread_class)
1698 g_assert (mono_object_domain (start) == mono_get_root_domain ());
1699 /* The object could be a proxy for an object in the domain
1701 if (mono_class_has_parent (vt->klass, mono_defaults.real_proxy_class)) {
1702 MonoObject *server = ((MonoRealProxy*)start)->unwrapped_server;
1704 /* The server could already have been zeroed out, so
1705 we need to check for that, too. */
1706 if (server && (!LOAD_VTABLE (server) || mono_object_domain (server) == domain)) {
1707 DEBUG (1, fprintf (gc_debug_file, "Cleaning up remote pointer in %p to object %p\n",
1709 ((MonoRealProxy*)start)->unwrapped_server = NULL;
1714 static void __attribute__((noinline))
1715 scan_area_for_domain (MonoDomain *domain, char *start, char *end)
1720 while (start < end) {
1721 if (!*(void**)start) {
1722 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
1725 vt = (GCVTable*)LOAD_VTABLE (start);
1726 process_object_for_domain_clearing (start, domain);
1727 remove = need_remove_object_for_domain (start, domain);
1728 if (remove && ((MonoObject*)start)->synchronisation) {
1729 void **dislink = mono_monitor_get_object_monitor_weak_link ((MonoObject*)start);
1731 mono_gc_register_disappearing_link (NULL, dislink, FALSE);
1734 #define SCAN_OBJECT_NOSCAN
1735 #define SCAN_OBJECT_ACTION do { \
1736 if (remove) memset (start, 0, skip_size); \
1738 #include "sgen-scan-object.h"
1742 static MonoDomain *check_domain = NULL;
1745 check_obj_not_in_domain (void *o)
1747 g_assert (((MonoObject*)o)->vtable->domain != check_domain);
1752 scan_for_registered_roots_in_domain (MonoDomain *domain, int root_type)
1756 check_domain = domain;
1757 for (i = 0; i < roots_hash_size [root_type]; ++i) {
1758 for (root = roots_hash [root_type][i]; root; root = root->next) {
1759 void **start_root = (void**)root->start_root;
1760 mword desc = root->root_desc;
1762 /* The MonoDomain struct is allowed to hold
1763 references to objects in its own domain. */
1764 if (start_root == (void**)domain)
1767 switch (desc & ROOT_DESC_TYPE_MASK) {
1768 case ROOT_DESC_BITMAP:
1769 desc >>= ROOT_DESC_TYPE_SHIFT;
1771 if ((desc & 1) && *start_root)
1772 check_obj_not_in_domain (*start_root);
1777 case ROOT_DESC_COMPLEX: {
1778 gsize *bitmap_data = complex_descriptors + (desc >> ROOT_DESC_TYPE_SHIFT);
1779 int bwords = (*bitmap_data) - 1;
1780 void **start_run = start_root;
1782 while (bwords-- > 0) {
1783 gsize bmap = *bitmap_data++;
1784 void **objptr = start_run;
1786 if ((bmap & 1) && *objptr)
1787 check_obj_not_in_domain (*objptr);
1791 start_run += GC_BITS_PER_WORD;
1795 case ROOT_DESC_USER: {
1796 MonoGCMarkFunc marker = user_descriptors [desc >> ROOT_DESC_TYPE_SHIFT];
1797 marker (start_root, check_obj_not_in_domain);
1800 case ROOT_DESC_RUN_LEN:
1801 g_assert_not_reached ();
1803 g_assert_not_reached ();
1807 check_domain = NULL;
1811 clear_domain_process_pinned_object_callback (PinnedChunk *chunk, char *obj, size_t size, MonoDomain *domain)
1813 process_object_for_domain_clearing (obj, domain);
1817 clear_domain_free_pinned_object_callback (PinnedChunk *chunk, char *obj, size_t size, MonoDomain *domain)
1819 if (need_remove_object_for_domain (obj, domain))
1820 free_pinned_object (chunk, obj, size);
1824 scan_pinned_object_for_xdomain_refs_callback (PinnedChunk *chunk, char *obj, size_t size, gpointer dummy)
1826 scan_object_for_xdomain_refs (obj);
1830 check_for_xdomain_refs (void)
1832 GCMemSection *section;
1835 for (section = section_list; section; section = section->block.next)
1836 scan_area_for_xdomain_refs (section->data, section->end_data);
1838 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1839 scan_object_for_xdomain_refs (bigobj->data);
1841 scan_pinned_objects (scan_pinned_object_for_xdomain_refs_callback, NULL);
1845 * When appdomains are unloaded we can easily remove objects that have finalizers,
1846 * but all the others could still be present in random places on the heap.
1847 * We need a sweep to get rid of them even though it's going to be costly
1849 * The reason we need to remove them is because we access the vtable and class
1850 * structures to know the object size and the reference bitmap: once the domain is
1851 * unloaded the point to random memory.
1854 mono_gc_clear_domain (MonoDomain * domain)
1856 GCMemSection *section;
1857 LOSObject *bigobj, *prev;
1862 /* Clear all remaining nursery fragments */
1863 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION) {
1864 g_assert (nursery_next <= nursery_frag_real_end);
1865 memset (nursery_next, 0, nursery_frag_real_end - nursery_next);
1866 for (frag = nursery_fragments; frag; frag = frag->next) {
1867 memset (frag->fragment_start, 0, frag->fragment_end - frag->fragment_start);
1871 if (xdomain_checks && domain != mono_get_root_domain ()) {
1872 scan_for_registered_roots_in_domain (domain, ROOT_TYPE_NORMAL);
1873 scan_for_registered_roots_in_domain (domain, ROOT_TYPE_WBARRIER);
1874 check_for_xdomain_refs ();
1877 for (section = section_list; section; section = section->block.next) {
1878 scan_area_for_domain (domain, section->data, section->end_data);
1881 /* We need two passes over pinned and large objects because
1882 freeing such an object gives its memory back to the OS (in
1883 the case of large objects) or obliterates its vtable
1884 (pinned objects), but we might need to dereference a
1885 pointer from an object to another object if the first
1886 object is a proxy. */
1887 scan_pinned_objects ((ScanPinnedObjectCallbackFunc)clear_domain_process_pinned_object_callback, domain);
1888 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1889 process_object_for_domain_clearing (bigobj->data, domain);
1892 for (bigobj = los_object_list; bigobj;) {
1893 if (need_remove_object_for_domain (bigobj->data, domain)) {
1894 LOSObject *to_free = bigobj;
1896 prev->next = bigobj->next;
1898 los_object_list = bigobj->next;
1899 bigobj = bigobj->next;
1900 DEBUG (1, fprintf (gc_debug_file, "Freeing large object %p\n",
1902 free_large_object (to_free);
1906 bigobj = bigobj->next;
1908 scan_pinned_objects ((ScanPinnedObjectCallbackFunc)clear_domain_free_pinned_object_callback, domain);
1910 for (i = GENERATION_NURSERY; i < GENERATION_MAX; ++i)
1911 null_links_for_domain (domain, i);
1917 * add_to_global_remset:
1919 * The global remset contains locations which point into newspace after
1920 * a minor collection. This can happen if the objects they point to are pinned.
1923 add_to_global_remset (gpointer ptr, gboolean root)
1927 DEBUG (8, fprintf (gc_debug_file, "Adding global remset for %p\n", ptr));
1928 binary_protocol_global_remset (ptr, *(gpointer*)ptr, (gpointer)LOAD_VTABLE (*(gpointer*)ptr));
1931 g_assert (!ptr_in_nursery (ptr) && ptr_in_nursery (*(gpointer*)ptr));
1933 HEAVY_STAT (++stat_global_remsets_added);
1936 * FIXME: If an object remains pinned, we need to add it at every minor collection.
1937 * To avoid uncontrolled growth of the global remset, only add each pointer once.
1939 if (global_remset->store_next + 3 < global_remset->end_set) {
1941 *(global_remset->store_next++) = (mword)ptr | REMSET_OTHER;
1942 *(global_remset->store_next++) = (mword)REMSET_ROOT_LOCATION;
1944 *(global_remset->store_next++) = (mword)ptr;
1948 rs = alloc_remset (global_remset->end_set - global_remset->data, NULL);
1949 rs->next = global_remset;
1952 *(global_remset->store_next++) = (mword)ptr | REMSET_OTHER;
1953 *(global_remset->store_next++) = (mword)REMSET_ROOT_LOCATION;
1955 *(global_remset->store_next++) = (mword)ptr;
1959 int global_rs_size = 0;
1961 for (rs = global_remset; rs; rs = rs->next) {
1962 global_rs_size += rs->store_next - rs->data;
1964 DEBUG (4, fprintf (gc_debug_file, "Global remset now has size %d\n", global_rs_size));
1968 #define MOVED_OBJECTS_NUM 64
1969 static void *moved_objects [MOVED_OBJECTS_NUM];
1970 static int moved_objects_idx = 0;
1972 #include "sgen-gray.c"
1975 * This is how the copying happens from the nursery to the old generation.
1976 * We assume that at this time all the pinned objects have been identified and
1978 * We run scan_object() for each pinned object so that each referenced
1979 * objects if possible are copied. The new gray objects created can have
1980 * scan_object() run on them right away, too.
1981 * Then we run copy_object() for the precisely tracked roots. At this point
1982 * all the roots are either gray or black. We run scan_object() on the gray
1983 * objects until no more gray objects are created.
1984 * At the end of the process we walk again the pinned list and we unmark
1985 * the pinned flag. As we go we also create the list of free space for use
1986 * in the next allocation runs.
1988 * We need to remember objects from the old generation that point to the new one
1989 * (or just addresses?).
1991 * copy_object could be made into a macro once debugged (use inline for now).
1994 static char* __attribute__((noinline))
1995 copy_object (char *obj, char *from_space_start, char *from_space_end)
1997 static void *copy_labels [] = { &&LAB_0, &&LAB_1, &&LAB_2, &&LAB_3, &&LAB_4, &&LAB_5, &&LAB_6, &&LAB_7, &&LAB_8 };
2003 HEAVY_STAT (++num_copy_object_called);
2005 if (!(obj >= from_space_start && obj < from_space_end)) {
2006 DEBUG (9, fprintf (gc_debug_file, "Not copying %p because it's not in from space (%p-%p)\n",
2007 obj, from_space_start, from_space_end));
2008 HEAVY_STAT (++stat_copy_object_failed_from_space);
2012 DEBUG (9, fprintf (gc_debug_file, "Precise copy of %p", obj));
2015 * obj must belong to one of:
2020 * 4. a non-to-space section of the major heap
2021 * 5. a to-space section of the major heap
2023 * In addition, objects in 1, 2 and 4 might also be pinned.
2024 * Objects in 1 and 4 might be forwarded.
2026 * Before we can copy the object we must make sure that we are
2027 * allowed to, i.e. that the object not pinned, not already
2028 * forwarded and doesn't belong to the LOS, a pinned chunk, or
2029 * a to-space section.
2031 * We are usually called for to-space objects (5) when we have
2032 * two remset entries for the same reference. The first entry
2033 * copies the object and updates the reference and the second
2034 * calls us with the updated reference that points into
2035 * to-space. There might also be other circumstances where we
2036 * get to-space objects.
2039 if ((forwarded = object_is_forwarded (obj))) {
2040 g_assert (((MonoVTable*)LOAD_VTABLE(obj))->gc_descr);
2041 DEBUG (9, fprintf (gc_debug_file, " (already forwarded to %p)\n", forwarded));
2042 HEAVY_STAT (++stat_copy_object_failed_forwarded);
2045 if (object_is_pinned (obj)) {
2046 g_assert (((MonoVTable*)LOAD_VTABLE(obj))->gc_descr);
2047 DEBUG (9, fprintf (gc_debug_file, " (pinned, no change)\n"));
2048 HEAVY_STAT (++stat_copy_object_failed_pinned);
2052 objsize = safe_object_get_size ((MonoObject*)obj);
2053 objsize += ALLOC_ALIGN - 1;
2054 objsize &= ~(ALLOC_ALIGN - 1);
2056 if (ptr_in_nursery (obj))
2060 * At this point we know obj is not pinned, not forwarded and
2061 * belongs to 2, 3, 4, or 5.
2063 * LOS object (2) are simple, at least until we always follow
2064 * the rule: if objsize > MAX_SMALL_OBJ_SIZE, pin the object
2065 * and return it. At the end of major collections, we walk
2066 * the los list and if the object is pinned, it is marked,
2067 * otherwise it can be freed.
2069 * Pinned chunks (3) and major heap sections (4, 5) both
2070 * reside in blocks, which are always aligned, so once we've
2071 * eliminated LOS objects, we can just access the block and
2072 * see whether it's a pinned chunk or a major heap section.
2074 if (G_UNLIKELY (objsize > MAX_SMALL_OBJ_SIZE || obj_is_from_pinned_alloc (obj))) {
2075 DEBUG (9, fprintf (gc_debug_file, " (marked LOS/Pinned %p (%s), size: %zd)\n", obj, safe_name (obj), objsize));
2076 binary_protocol_pin (obj, (gpointer)LOAD_VTABLE (obj), safe_object_get_size ((MonoObject*)obj));
2078 HEAVY_STAT (++stat_copy_object_failed_large_pinned);
2083 * Now we know the object is in a major heap section. All we
2084 * need to do is check whether it's already in to-space (5) or
2087 if (MAJOR_SECTION_FOR_OBJECT (obj)->is_to_space) {
2088 g_assert (objsize <= MAX_SMALL_OBJ_SIZE);
2089 DEBUG (9, fprintf (gc_debug_file, " (already copied)\n"));
2090 HEAVY_STAT (++stat_copy_object_failed_to_space);
2095 DEBUG (9, fprintf (gc_debug_file, " (to %p, %s size: %zd)\n", to_space_bumper, ((MonoObject*)obj)->vtable->klass->name, objsize));
2096 binary_protocol_copy (obj, to_space_bumper, ((MonoObject*)obj)->vtable, objsize);
2098 HEAVY_STAT (++num_objects_copied);
2100 /* Make sure we have enough space available */
2101 if (to_space_bumper + objsize > to_space_top) {
2103 g_assert (to_space_bumper + objsize <= to_space_top);
2106 if (objsize <= sizeof (gpointer) * 8) {
2107 mword *dest = (mword*)to_space_bumper;
2108 goto *copy_labels [objsize / sizeof (gpointer)];
2110 (dest) [7] = ((mword*)obj) [7];
2112 (dest) [6] = ((mword*)obj) [6];
2114 (dest) [5] = ((mword*)obj) [5];
2116 (dest) [4] = ((mword*)obj) [4];
2118 (dest) [3] = ((mword*)obj) [3];
2120 (dest) [2] = ((mword*)obj) [2];
2122 (dest) [1] = ((mword*)obj) [1];
2124 (dest) [0] = ((mword*)obj) [0];
2132 char* edi = to_space_bumper;
2133 __asm__ __volatile__(
2135 : "=&c" (ecx), "=&D" (edi), "=&S" (esi)
2136 : "0" (objsize/4), "1" (edi),"2" (esi)
2141 memcpy (to_space_bumper, obj, objsize);
2144 /* adjust array->bounds */
2145 vt = ((MonoObject*)obj)->vtable;
2146 g_assert (vt->gc_descr);
2147 if (G_UNLIKELY (vt->rank && ((MonoArray*)obj)->bounds)) {
2148 MonoArray *array = (MonoArray*)to_space_bumper;
2149 array->bounds = (MonoArrayBounds*)((char*)to_space_bumper + ((char*)((MonoArray*)obj)->bounds - (char*)obj));
2150 DEBUG (9, fprintf (gc_debug_file, "Array instance %p: size: %zd, rank: %d, length: %d\n", array, objsize, vt->rank, mono_array_length (array)));
2152 /* set the forwarding pointer */
2153 forward_object (obj, to_space_bumper);
2154 if (G_UNLIKELY (mono_profiler_events & MONO_PROFILE_GC_MOVES)) {
2155 if (moved_objects_idx == MOVED_OBJECTS_NUM) {
2156 mono_profiler_gc_moves (moved_objects, moved_objects_idx);
2157 moved_objects_idx = 0;
2159 moved_objects [moved_objects_idx++] = obj;
2160 moved_objects [moved_objects_idx++] = to_space_bumper;
2162 obj = to_space_bumper;
2163 to_space_section->scan_starts [((char*)obj - (char*)to_space_section->data)/SCAN_START_SIZE] = obj;
2164 to_space_bumper += objsize;
2165 DEBUG (9, fprintf (gc_debug_file, "Enqueuing gray object %p (%s)\n", obj, safe_name (obj)));
2166 gray_object_enqueue (obj);
2167 DEBUG (8, g_assert (to_space_bumper <= to_space_top));
2172 #define HANDLE_PTR(ptr,obj) do { \
2173 void *__old = *(ptr); \
2176 *(ptr) = __copy = copy_object (__old, from_start, from_end); \
2177 DEBUG (9, if (__old != __copy) fprintf (gc_debug_file, "Overwrote field at %p with %p (was: %p)\n", (ptr), *(ptr), __old)); \
2178 if (G_UNLIKELY (ptr_in_nursery (__copy) && !ptr_in_nursery ((ptr)))) \
2179 add_to_global_remset ((ptr), FALSE); \
2184 * Scan the object pointed to by @start for references to
2185 * other objects between @from_start and @from_end and copy
2186 * them to the gray_objects area.
2187 * Returns a pointer to the end of the object.
2190 scan_object (char *start, char* from_start, char* from_end)
2192 #include "sgen-scan-object.h"
2200 * Scan objects in the gray stack until the stack is empty. This should be called
2201 * frequently after each object is copied, to achieve better locality and cache
2205 drain_gray_stack (char *start_addr, char *end_addr)
2209 while ((obj = gray_object_dequeue ())) {
2210 DEBUG (9, fprintf (gc_debug_file, "Precise gray object scan %p (%s)\n", obj, safe_name (obj)));
2211 scan_object (obj, start_addr, end_addr);
2218 * Scan the valuetype pointed to by START, described by DESC for references to
2219 * other objects between @from_start and @from_end and copy them to the gray_objects area.
2220 * Returns a pointer to the end of the object.
2223 scan_vtype (char *start, mword desc, char* from_start, char* from_end)
2227 /* The descriptors include info about the MonoObject header as well */
2228 start -= sizeof (MonoObject);
2230 switch (desc & 0x7) {
2231 case DESC_TYPE_RUN_LENGTH:
2232 OBJ_RUN_LEN_FOREACH_PTR (desc,start);
2233 OBJ_RUN_LEN_SIZE (skip_size, desc, start);
2234 g_assert (skip_size);
2235 return start + skip_size;
2236 case DESC_TYPE_SMALL_BITMAP:
2237 OBJ_BITMAP_FOREACH_PTR (desc,start);
2238 OBJ_BITMAP_SIZE (skip_size, desc, start);
2239 return start + skip_size;
2240 case DESC_TYPE_LARGE_BITMAP:
2241 case DESC_TYPE_COMPLEX:
2243 g_assert_not_reached ();
2246 // The other descriptors can't happen with vtypes
2247 g_assert_not_reached ();
2253 #include "sgen-pinning-stats.c"
2256 * Addresses from start to end are already sorted. This function finds
2257 * the object header for each address and pins the object. The
2258 * addresses must be inside the passed section. The (start of the)
2259 * address array is overwritten with the addresses of the actually
2260 * pinned objects. Return the number of pinned objects.
2263 pin_objects_from_addresses (GCMemSection *section, void **start, void **end, void *start_nursery, void *end_nursery)
2268 void *last_obj = NULL;
2269 size_t last_obj_size = 0;
2272 void **definitely_pinned = start;
2273 while (start < end) {
2275 /* the range check should be reduntant */
2276 if (addr != last && addr >= start_nursery && addr < end_nursery) {
2277 DEBUG (5, fprintf (gc_debug_file, "Considering pinning addr %p\n", addr));
2278 /* multiple pointers to the same object */
2279 if (addr >= last_obj && (char*)addr < (char*)last_obj + last_obj_size) {
2283 idx = ((char*)addr - (char*)section->data) / SCAN_START_SIZE;
2284 g_assert (idx < section->num_scan_start);
2285 search_start = (void*)section->scan_starts [idx];
2286 if (!search_start || search_start > addr) {
2289 search_start = section->scan_starts [idx];
2290 if (search_start && search_start <= addr)
2293 if (!search_start || search_start > addr)
2294 search_start = start_nursery;
2296 if (search_start < last_obj)
2297 search_start = (char*)last_obj + last_obj_size;
2298 /* now addr should be in an object a short distance from search_start
2299 * Note that search_start must point to zeroed mem or point to an object.
2302 if (!*(void**)search_start) {
2303 mword p = (mword)search_start;
2304 p += sizeof (gpointer);
2305 p += ALLOC_ALIGN - 1;
2306 p &= ~(ALLOC_ALIGN - 1);
2307 search_start = (void*)p;
2310 last_obj = search_start;
2311 last_obj_size = safe_object_get_size ((MonoObject*)search_start);
2312 last_obj_size += ALLOC_ALIGN - 1;
2313 last_obj_size &= ~(ALLOC_ALIGN - 1);
2314 DEBUG (8, fprintf (gc_debug_file, "Pinned try match %p (%s), size %zd\n", last_obj, safe_name (last_obj), last_obj_size));
2315 if (addr >= search_start && (char*)addr < (char*)last_obj + last_obj_size) {
2316 DEBUG (4, fprintf (gc_debug_file, "Pinned object %p, vtable %p (%s), count %d\n", search_start, *(void**)search_start, safe_name (search_start), count));
2317 binary_protocol_pin (search_start, (gpointer)LOAD_VTABLE (search_start), safe_object_get_size (search_start));
2318 pin_object (search_start);
2320 pin_stats_register_object (search_start, last_obj_size);
2321 definitely_pinned [count] = search_start;
2325 /* skip to the next object */
2326 search_start = (void*)((char*)search_start + last_obj_size);
2327 } while (search_start <= addr);
2328 /* we either pinned the correct object or we ignored the addr because
2329 * it points to unused zeroed memory.
2335 //printf ("effective pinned: %d (at the end: %d)\n", count, (char*)end_nursery - (char*)last);
2339 static void** pin_queue;
2340 static int pin_queue_size = 0;
2341 static int next_pin_slot = 0;
2346 gap = (gap * 10) / 13;
2347 if (gap == 9 || gap == 10)
2356 compare_addr (const void *a, const void *b)
2358 return *(const void **)a - *(const void **)b;
2362 /* sort the addresses in array in increasing order */
2364 sort_addresses (void **array, int size)
2367 * qsort is slower as predicted.
2368 * qsort (array, size, sizeof (gpointer), compare_addr);
2375 gap = new_gap (gap);
2378 for (i = 0; i < end; i++) {
2380 if (array [i] > array [j]) {
2381 void* val = array [i];
2382 array [i] = array [j];
2387 if (gap == 1 && !swapped)
2392 static G_GNUC_UNUSED void
2393 print_nursery_gaps (void* start_nursery, void *end_nursery)
2396 gpointer first = start_nursery;
2398 for (i = 0; i < next_pin_slot; ++i) {
2399 next = pin_queue [i];
2400 fprintf (gc_debug_file, "Nursery range: %p-%p, size: %zd\n", first, next, (char*)next-(char*)first);
2404 fprintf (gc_debug_file, "Nursery range: %p-%p, size: %zd\n", first, next, (char*)next-(char*)first);
2407 /* reduce the info in the pin queue, removing duplicate pointers and sorting them */
2409 optimize_pin_queue (int start_slot)
2411 void **start, **cur, **end;
2412 /* sort and uniq pin_queue: we just sort and we let the rest discard multiple values */
2413 /* it may be better to keep ranges of pinned memory instead of individually pinning objects */
2414 DEBUG (5, fprintf (gc_debug_file, "Sorting pin queue, size: %d\n", next_pin_slot));
2415 if ((next_pin_slot - start_slot) > 1)
2416 sort_addresses (pin_queue + start_slot, next_pin_slot - start_slot);
2417 start = cur = pin_queue + start_slot;
2418 end = pin_queue + next_pin_slot;
2421 while (*start == *cur && cur < end)
2425 next_pin_slot = start - pin_queue;
2426 DEBUG (5, fprintf (gc_debug_file, "Pin queue reduced to size: %d\n", next_pin_slot));
2427 //DEBUG (6, print_nursery_gaps (start_nursery, end_nursery));
2432 optimized_pin_queue_search (void *addr)
2434 int first = 0, last = next_pin_slot;
2435 while (first < last) {
2436 int middle = first + ((last - first) >> 1);
2437 if (addr <= pin_queue [middle])
2442 g_assert (first == last);
2447 find_optimized_pin_queue_area (void *start, void *end, int *first, int *last)
2449 *first = optimized_pin_queue_search (start);
2450 *last = optimized_pin_queue_search (end);
2454 realloc_pin_queue (void)
2456 int new_size = pin_queue_size? pin_queue_size + pin_queue_size/2: 1024;
2457 void **new_pin = get_internal_mem (sizeof (void*) * new_size, INTERNAL_MEM_PIN_QUEUE);
2458 memcpy (new_pin, pin_queue, sizeof (void*) * next_pin_slot);
2459 free_internal_mem (pin_queue, INTERNAL_MEM_PIN_QUEUE);
2460 pin_queue = new_pin;
2461 pin_queue_size = new_size;
2462 DEBUG (4, fprintf (gc_debug_file, "Reallocated pin queue to size: %d\n", new_size));
2465 #include "sgen-pinning.c"
2468 * Scan the memory between start and end and queue values which could be pointers
2469 * to the area between start_nursery and end_nursery for later consideration.
2470 * Typically used for thread stacks.
2473 conservatively_pin_objects_from (void **start, void **end, void *start_nursery, void *end_nursery, int pin_type)
2476 while (start < end) {
2477 if (*start >= start_nursery && *start < end_nursery) {
2479 * *start can point to the middle of an object
2480 * note: should we handle pointing at the end of an object?
2481 * pinning in C# code disallows pointing at the end of an object
2482 * but there is some small chance that an optimizing C compiler
2483 * may keep the only reference to an object by pointing
2484 * at the end of it. We ignore this small chance for now.
2485 * Pointers to the end of an object are indistinguishable
2486 * from pointers to the start of the next object in memory
2487 * so if we allow that we'd need to pin two objects...
2488 * We queue the pointer in an array, the
2489 * array will then be sorted and uniqued. This way
2490 * we can coalesce several pinning pointers and it should
2491 * be faster since we'd do a memory scan with increasing
2492 * addresses. Note: we can align the address to the allocation
2493 * alignment, so the unique process is more effective.
2495 mword addr = (mword)*start;
2496 addr &= ~(ALLOC_ALIGN - 1);
2497 if (addr >= (mword)start_nursery && addr < (mword)end_nursery)
2498 pin_stage_ptr ((void*)addr);
2500 pin_stats_register_address ((char*)addr, pin_type);
2501 DEBUG (6, if (count) fprintf (gc_debug_file, "Pinning address %p\n", (void*)addr));
2506 DEBUG (7, if (count) fprintf (gc_debug_file, "found %d potential pinned heap pointers\n", count));
2510 * If generation is 0, just mark objects in the nursery, the others we don't care,
2511 * since they are not going to move anyway.
2512 * There are different areas that are scanned for pinned pointers:
2513 * *) the thread stacks (when jit support is ready only the unmanaged frames)
2514 * *) the pinned handle table
2515 * *) the pinned roots
2517 * Note: when we'll use a write barrier for old to new gen references, we need to
2518 * keep track of old gen objects that point to pinned new gen objects because in that
2519 * case the referenced object will be moved maybe at the next collection, but there
2520 * is no write in the old generation area where the pinned object is referenced
2521 * and we may not consider it as reachable.
2523 static G_GNUC_UNUSED void
2524 mark_pinned_objects (int generation)
2529 * Debugging function: find in the conservative roots where @obj is being pinned.
2531 static G_GNUC_UNUSED void
2532 find_pinning_reference (char *obj, size_t size)
2536 char *endobj = obj + size;
2537 for (i = 0; i < roots_hash_size [0]; ++i) {
2538 for (root = roots_hash [0][i]; root; root = root->next) {
2539 /* if desc is non-null it has precise info */
2540 if (!root->root_desc) {
2541 char ** start = (char**)root->start_root;
2542 while (start < (char**)root->end_root) {
2543 if (*start >= obj && *start < endobj) {
2544 DEBUG (0, fprintf (gc_debug_file, "Object %p referenced in pinned roots %p-%p (at %p in record %p)\n", obj, root->start_root, root->end_root, start, root));
2551 find_pinning_ref_from_thread (obj, size);
2555 * The first thing we do in a collection is to identify pinned objects.
2556 * This function considers all the areas of memory that need to be
2557 * conservatively scanned.
2560 pin_from_roots (void *start_nursery, void *end_nursery)
2564 DEBUG (2, fprintf (gc_debug_file, "Scanning pinned roots (%d bytes, %d/%d entries)\n", (int)roots_size, num_roots_entries [ROOT_TYPE_NORMAL], num_roots_entries [ROOT_TYPE_PINNED]));
2565 /* objects pinned from the API are inside these roots */
2566 for (i = 0; i < roots_hash_size [ROOT_TYPE_PINNED]; ++i) {
2567 for (root = roots_hash [ROOT_TYPE_PINNED][i]; root; root = root->next) {
2568 DEBUG (6, fprintf (gc_debug_file, "Pinned roots %p-%p\n", root->start_root, root->end_root));
2569 conservatively_pin_objects_from ((void**)root->start_root, (void**)root->end_root, start_nursery, end_nursery, PIN_TYPE_OTHER);
2572 /* now deal with the thread stacks
2573 * in the future we should be able to conservatively scan only:
2574 * *) the cpu registers
2575 * *) the unmanaged stack frames
2576 * *) the _last_ managed stack frame
2577 * *) pointers slots in managed frames
2579 scan_thread_data (start_nursery, end_nursery, FALSE);
2581 evacuate_pin_staging_area ();
2584 /* Copy function called from user defined mark functions */
2585 static char *user_copy_n_start;
2586 static char *user_copy_n_end;
2589 user_copy (void *addr)
2592 return copy_object (addr, user_copy_n_start, user_copy_n_end);
2598 * The memory area from start_root to end_root contains pointers to objects.
2599 * Their position is precisely described by @desc (this means that the pointer
2600 * can be either NULL or the pointer to the start of an object).
2601 * This functions copies them to to_space updates them.
2604 precisely_scan_objects_from (void** start_root, void** end_root, char* n_start, char *n_end, mword desc)
2606 switch (desc & ROOT_DESC_TYPE_MASK) {
2607 case ROOT_DESC_BITMAP:
2608 desc >>= ROOT_DESC_TYPE_SHIFT;
2610 if ((desc & 1) && *start_root) {
2611 *start_root = copy_object (*start_root, n_start, n_end);
2612 DEBUG (9, fprintf (gc_debug_file, "Overwrote root at %p with %p\n", start_root, *start_root));
2613 drain_gray_stack (n_start, n_end);
2619 case ROOT_DESC_COMPLEX: {
2620 gsize *bitmap_data = complex_descriptors + (desc >> ROOT_DESC_TYPE_SHIFT);
2621 int bwords = (*bitmap_data) - 1;
2622 void **start_run = start_root;
2624 while (bwords-- > 0) {
2625 gsize bmap = *bitmap_data++;
2626 void **objptr = start_run;
2628 if ((bmap & 1) && *objptr) {
2629 *objptr = copy_object (*objptr, n_start, n_end);
2630 DEBUG (9, fprintf (gc_debug_file, "Overwrote root at %p with %p\n", objptr, *objptr));
2631 drain_gray_stack (n_start, n_end);
2636 start_run += GC_BITS_PER_WORD;
2640 case ROOT_DESC_USER: {
2641 MonoGCMarkFunc marker = user_descriptors [desc >> ROOT_DESC_TYPE_SHIFT];
2643 user_copy_n_start = n_start;
2644 user_copy_n_end = n_end;
2645 marker (start_root, user_copy);
2648 case ROOT_DESC_RUN_LEN:
2649 g_assert_not_reached ();
2651 g_assert_not_reached ();
2656 alloc_fragment (void)
2658 Fragment *frag = fragment_freelist;
2660 fragment_freelist = frag->next;
2664 frag = get_internal_mem (sizeof (Fragment), INTERNAL_MEM_FRAGMENT);
2669 /* size must be a power of 2 */
2671 get_os_memory_aligned (mword size, gboolean activate)
2673 /* Allocate twice the memory to be able to put the block on an aligned address */
2674 char *mem = get_os_memory (size * 2, activate);
2679 aligned = (char*)((mword)(mem + (size - 1)) & ~(size - 1));
2680 g_assert (aligned >= mem && aligned + size <= mem + size * 2 && !((mword)aligned & (size - 1)));
2683 free_os_memory (mem, aligned - mem);
2684 if (aligned + size < mem + size * 2)
2685 free_os_memory (aligned + size, (mem + size * 2) - (aligned + size));
2691 * Allocate and setup the data structures needed to be able to allocate objects
2692 * in the nursery. The nursery is stored in nursery_section.
2695 alloc_nursery (void)
2697 GCMemSection *section;
2703 if (nursery_section)
2705 DEBUG (2, fprintf (gc_debug_file, "Allocating nursery size: %zd\n", nursery_size));
2706 /* later we will alloc a larger area for the nursery but only activate
2707 * what we need. The rest will be used as expansion if we have too many pinned
2708 * objects in the existing nursery.
2710 /* FIXME: handle OOM */
2711 section = get_internal_mem (SIZEOF_GC_MEM_SECTION, INTERNAL_MEM_SECTION);
2713 g_assert (nursery_size == DEFAULT_NURSERY_SIZE);
2714 alloc_size = nursery_size;
2715 #ifdef ALIGN_NURSERY
2716 data = get_os_memory_aligned (alloc_size, TRUE);
2718 data = get_os_memory (alloc_size, TRUE);
2720 nursery_start = data;
2721 nursery_real_end = nursery_start + nursery_size;
2722 UPDATE_HEAP_BOUNDARIES (nursery_start, nursery_real_end);
2723 nursery_next = nursery_start;
2724 total_alloc += alloc_size;
2725 DEBUG (4, fprintf (gc_debug_file, "Expanding nursery size (%p-%p): %zd, total: %zd\n", data, data + alloc_size, nursery_size, total_alloc));
2726 section->data = section->next_data = data;
2727 section->size = alloc_size;
2728 section->end_data = nursery_real_end;
2729 scan_starts = (alloc_size + SCAN_START_SIZE - 1) / SCAN_START_SIZE;
2730 section->scan_starts = get_internal_mem (sizeof (char*) * scan_starts, INTERNAL_MEM_SCAN_STARTS);
2731 section->num_scan_start = scan_starts;
2732 section->block.role = MEMORY_ROLE_GEN0;
2734 /* add to the section list */
2735 section->block.next = section_list;
2736 section_list = section;
2738 nursery_section = section;
2740 /* Setup the single first large fragment */
2741 frag = alloc_fragment ();
2742 frag->fragment_start = nursery_start;
2743 frag->fragment_limit = nursery_start;
2744 frag->fragment_end = nursery_real_end;
2745 nursery_frag_real_end = nursery_real_end;
2746 /* FIXME: frag here is lost */
2750 scan_finalizer_entries (FinalizeEntry *list, char *start, char *end) {
2753 for (fin = list; fin; fin = fin->next) {
2756 DEBUG (5, fprintf (gc_debug_file, "Scan of fin ready object: %p (%s)\n", fin->object, safe_name (fin->object)));
2757 fin->object = copy_object (fin->object, start, end);
2762 * Update roots in the old generation. Since we currently don't have the
2763 * info from the write barriers, we just scan all the objects.
2765 static G_GNUC_UNUSED void
2766 scan_old_generation (char *start, char* end)
2768 GCMemSection *section;
2769 LOSObject *big_object;
2772 for (section = section_list; section; section = section->block.next) {
2773 if (section == nursery_section)
2775 DEBUG (2, fprintf (gc_debug_file, "Scan of old section: %p-%p, size: %d\n", section->data, section->next_data, (int)(section->next_data - section->data)));
2776 /* we have to deal with zeroed holes in old generation (truncated strings ...) */
2778 while (p < section->next_data) {
2783 DEBUG (8, fprintf (gc_debug_file, "Precise old object scan of %p (%s)\n", p, safe_name (p)));
2784 p = scan_object (p, start, end);
2787 /* scan the old object space, too */
2788 for (big_object = los_object_list; big_object; big_object = big_object->next) {
2789 DEBUG (5, fprintf (gc_debug_file, "Scan of big object: %p (%s), size: %zd\n", big_object->data, safe_name (big_object->data), big_object->size));
2790 scan_object (big_object->data, start, end);
2792 /* scan the list of objects ready for finalization */
2793 scan_finalizer_entries (fin_ready_list, start, end);
2794 scan_finalizer_entries (critical_fin_list, start, end);
2797 static mword fragment_total = 0;
2799 * We found a fragment of free memory in the nursery: memzero it and if
2800 * it is big enough, add it to the list of fragments that can be used for
2804 add_nursery_frag (size_t frag_size, char* frag_start, char* frag_end)
2807 DEBUG (4, fprintf (gc_debug_file, "Found empty fragment: %p-%p, size: %zd\n", frag_start, frag_end, frag_size));
2808 binary_protocol_empty (frag_start, frag_size);
2809 /* memsetting just the first chunk start is bound to provide better cache locality */
2810 if (nursery_clear_policy == CLEAR_AT_GC)
2811 memset (frag_start, 0, frag_size);
2812 /* Not worth dealing with smaller fragments: need to tune */
2813 if (frag_size >= FRAGMENT_MIN_SIZE) {
2814 fragment = alloc_fragment ();
2815 fragment->fragment_start = frag_start;
2816 fragment->fragment_limit = frag_start;
2817 fragment->fragment_end = frag_end;
2818 fragment->next = nursery_fragments;
2819 nursery_fragments = fragment;
2820 fragment_total += frag_size;
2822 /* Clear unused fragments, pinning depends on this */
2823 memset (frag_start, 0, frag_size);
2828 scan_needed_big_objects (char *start_addr, char *end_addr)
2830 LOSObject *big_object;
2832 for (big_object = los_object_list; big_object; big_object = big_object->next) {
2833 if (!big_object->scanned && object_is_pinned (big_object->data)) {
2834 DEBUG (5, fprintf (gc_debug_file, "Scan of big object: %p (%s), size: %zd\n", big_object->data, safe_name (big_object->data), big_object->size));
2835 scan_object (big_object->data, start_addr, end_addr);
2836 drain_gray_stack (start_addr, end_addr);
2837 big_object->scanned = TRUE;
2845 generation_name (int generation)
2847 switch (generation) {
2848 case GENERATION_NURSERY: return "nursery";
2849 case GENERATION_OLD: return "old";
2850 default: g_assert_not_reached ();
2854 static DisappearingLinkHashTable*
2855 get_dislink_hash_table (int generation)
2857 switch (generation) {
2858 case GENERATION_NURSERY: return &minor_disappearing_link_hash;
2859 case GENERATION_OLD: return &major_disappearing_link_hash;
2860 default: g_assert_not_reached ();
2864 static FinalizeEntryHashTable*
2865 get_finalize_entry_hash_table (int generation)
2867 switch (generation) {
2868 case GENERATION_NURSERY: return &minor_finalizable_hash;
2869 case GENERATION_OLD: return &major_finalizable_hash;
2870 default: g_assert_not_reached ();
2875 new_to_space_section (void)
2877 /* FIXME: if the current to_space_section is empty, we don't
2878 have to allocate a new one */
2880 to_space_section = alloc_major_section ();
2881 to_space_bumper = to_space_section->next_data;
2882 to_space_top = to_space_section->end_data;
2886 to_space_set_next_data (void)
2888 g_assert (to_space_bumper >= to_space_section->next_data && to_space_bumper <= to_space_section->end_data);
2889 to_space_section->next_data = to_space_bumper;
2893 to_space_expand (void)
2895 if (to_space_section) {
2896 g_assert (to_space_top == to_space_section->end_data);
2897 to_space_set_next_data ();
2900 new_to_space_section ();
2904 unset_to_space (void)
2906 /* between collections the to_space_bumper is invalidated
2907 because degraded allocations might occur, so we set it to
2908 NULL, just to make it explicit */
2909 to_space_bumper = NULL;
2911 /* don't unset to_space_section if we implement the FIXME in
2912 new_to_space_section */
2913 to_space_section = NULL;
2917 object_is_in_to_space (char *obj)
2922 if (ptr_in_nursery (obj))
2925 objsize = safe_object_get_size ((MonoObject*)obj);
2926 objsize += ALLOC_ALIGN - 1;
2927 objsize &= ~(ALLOC_ALIGN - 1);
2930 if (objsize > MAX_SMALL_OBJ_SIZE)
2934 if (obj_is_from_pinned_alloc (obj))
2937 /* now we know it's in a major heap section */
2938 return MAJOR_SECTION_FOR_OBJECT (obj)->is_to_space;
2942 finish_gray_stack (char *start_addr, char *end_addr, int generation)
2946 int fin_ready, bigo_scanned_num;
2949 * We copied all the reachable objects. Now it's the time to copy
2950 * the objects that were not referenced by the roots, but by the copied objects.
2951 * we built a stack of objects pointed to by gray_start: they are
2952 * additional roots and we may add more items as we go.
2953 * We loop until gray_start == gray_objects which means no more objects have
2954 * been added. Note this is iterative: no recursion is involved.
2955 * We need to walk the LO list as well in search of marked big objects
2956 * (use a flag since this is needed only on major collections). We need to loop
2957 * here as well, so keep a counter of marked LO (increasing it in copy_object).
2958 * To achieve better cache locality and cache usage, we drain the gray stack
2959 * frequently, after each object is copied, and just finish the work here.
2961 drain_gray_stack (start_addr, end_addr);
2963 //scan_old_generation (start_addr, end_addr);
2964 DEBUG (2, fprintf (gc_debug_file, "%s generation done\n", generation_name (generation)));
2965 /* walk the finalization queue and move also the objects that need to be
2966 * finalized: use the finalized objects as new roots so the objects they depend
2967 * on are also not reclaimed. As with the roots above, only objects in the nursery
2968 * are marked/copied.
2969 * We need a loop here, since objects ready for finalizers may reference other objects
2970 * that are fin-ready. Speedup with a flag?
2973 fin_ready = num_ready_finalizers;
2974 finalize_in_range (start_addr, end_addr, generation);
2975 if (generation == GENERATION_OLD)
2976 finalize_in_range (nursery_start, nursery_real_end, GENERATION_NURSERY);
2977 bigo_scanned_num = scan_needed_big_objects (start_addr, end_addr);
2979 /* drain the new stack that might have been created */
2980 DEBUG (6, fprintf (gc_debug_file, "Precise scan of gray area post fin\n"));
2981 drain_gray_stack (start_addr, end_addr);
2982 } while (fin_ready != num_ready_finalizers || bigo_scanned_num);
2984 DEBUG (2, fprintf (gc_debug_file, "Finalize queue handling scan for %s generation: %d usecs\n", generation_name (generation), TV_ELAPSED (atv, btv)));
2987 * handle disappearing links
2988 * Note we do this after checking the finalization queue because if an object
2989 * survives (at least long enough to be finalized) we don't clear the link.
2990 * This also deals with a possible issue with the monitor reclamation: with the Boehm
2991 * GC a finalized object my lose the monitor because it is cleared before the finalizer is
2994 g_assert (gray_object_queue_is_empty ());
2996 null_link_in_range (start_addr, end_addr, generation);
2997 if (generation == GENERATION_OLD)
2998 null_link_in_range (start_addr, end_addr, GENERATION_NURSERY);
2999 if (gray_object_queue_is_empty ())
3001 drain_gray_stack (start_addr, end_addr);
3004 g_assert (gray_object_queue_is_empty ());
3005 /* DEBUG (2, fprintf (gc_debug_file, "Copied from %s to old space: %d bytes (%p-%p)\n", generation_name (generation), (int)(to_space_bumper - to_space), to_space, to_space_bumper)); */
3006 to_space_set_next_data ();
3010 check_scan_starts (void)
3012 GCMemSection *section;
3014 if (!do_scan_starts_check)
3016 for (section = section_list; section; section = section->block.next) {
3017 for (i = 0; i < section->num_scan_start; ++i) {
3018 if (section->scan_starts [i]) {
3019 guint size = safe_object_get_size ((MonoObject*) section->scan_starts [i]);
3020 g_assert (size >= sizeof (MonoObject) && size <= MAX_SMALL_OBJ_SIZE);
3026 static int last_num_pinned = 0;
3029 build_nursery_fragments (int start_pin, int end_pin)
3031 char *frag_start, *frag_end;
3035 while (nursery_fragments) {
3036 Fragment *next = nursery_fragments->next;
3037 nursery_fragments->next = fragment_freelist;
3038 fragment_freelist = nursery_fragments;
3039 nursery_fragments = next;
3041 frag_start = nursery_start;
3043 /* clear scan starts */
3044 memset (nursery_section->scan_starts, 0, nursery_section->num_scan_start * sizeof (gpointer));
3045 for (i = start_pin; i < end_pin; ++i) {
3046 frag_end = pin_queue [i];
3047 /* remove the pin bit from pinned objects */
3048 unpin_object (frag_end);
3049 nursery_section->scan_starts [((char*)frag_end - (char*)nursery_section->data)/SCAN_START_SIZE] = frag_end;
3050 frag_size = frag_end - frag_start;
3052 add_nursery_frag (frag_size, frag_start, frag_end);
3053 frag_size = safe_object_get_size ((MonoObject*)pin_queue [i]);
3054 frag_size += ALLOC_ALIGN - 1;
3055 frag_size &= ~(ALLOC_ALIGN - 1);
3056 frag_start = (char*)pin_queue [i] + frag_size;
3058 nursery_last_pinned_end = frag_start;
3059 frag_end = nursery_real_end;
3060 frag_size = frag_end - frag_start;
3062 add_nursery_frag (frag_size, frag_start, frag_end);
3063 if (!nursery_fragments) {
3064 DEBUG (1, fprintf (gc_debug_file, "Nursery fully pinned (%d)\n", end_pin - start_pin));
3065 for (i = start_pin; i < end_pin; ++i) {
3066 DEBUG (3, fprintf (gc_debug_file, "Bastard pinning obj %p (%s), size: %d\n", pin_queue [i], safe_name (pin_queue [i]), safe_object_get_size (pin_queue [i])));
3071 nursery_next = nursery_frag_real_end = NULL;
3073 /* Clear TLABs for all threads */
3077 /* FIXME: later reduce code duplication here with the above
3078 * We don't keep track of section fragments for non-nursery sections yet, so
3082 build_section_fragments (GCMemSection *section)
3085 char *frag_start, *frag_end;
3088 /* clear scan starts */
3089 memset (section->scan_starts, 0, section->num_scan_start * sizeof (gpointer));
3090 frag_start = section->data;
3091 section->next_data = section->data;
3092 for (i = section->pin_queue_start; i < section->pin_queue_end; ++i) {
3093 frag_end = pin_queue [i];
3094 /* remove the pin bit from pinned objects */
3095 unpin_object (frag_end);
3096 if (frag_end >= section->data + section->size) {
3097 frag_end = section->data + section->size;
3099 section->scan_starts [((char*)frag_end - (char*)section->data)/SCAN_START_SIZE] = frag_end;
3101 frag_size = frag_end - frag_start;
3103 binary_protocol_empty (frag_start, frag_size);
3104 memset (frag_start, 0, frag_size);
3106 frag_size = safe_object_get_size ((MonoObject*)pin_queue [i]);
3107 frag_size += ALLOC_ALIGN - 1;
3108 frag_size &= ~(ALLOC_ALIGN - 1);
3109 frag_start = (char*)pin_queue [i] + frag_size;
3110 section->next_data = MAX (section->next_data, frag_start);
3112 frag_end = section->end_data;
3113 frag_size = frag_end - frag_start;
3115 binary_protocol_empty (frag_start, frag_size);
3116 memset (frag_start, 0, frag_size);
3121 scan_from_registered_roots (char *addr_start, char *addr_end, int root_type)
3125 for (i = 0; i < roots_hash_size [root_type]; ++i) {
3126 for (root = roots_hash [root_type][i]; root; root = root->next) {
3127 DEBUG (6, fprintf (gc_debug_file, "Precise root scan %p-%p (desc: %p)\n", root->start_root, root->end_root, (void*)root->root_desc));
3128 precisely_scan_objects_from ((void**)root->start_root, (void**)root->end_root, addr_start, addr_end, root->root_desc);
3134 dump_occupied (char *start, char *end, char *section_start)
3136 fprintf (heap_dump_file, "<occupied offset=\"%zd\" size=\"%zd\"/>\n", start - section_start, end - start);
3140 dump_section (GCMemSection *section, const char *type)
3142 char *start = section->data;
3143 char *end = section->data + section->size;
3144 char *occ_start = NULL;
3146 char *old_start = NULL; /* just for debugging */
3148 fprintf (heap_dump_file, "<section type=\"%s\" size=\"%zu\">\n", type, section->size);
3150 while (start < end) {
3154 if (!*(void**)start) {
3156 dump_occupied (occ_start, start, section->data);
3159 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
3162 g_assert (start < section->next_data);
3167 vt = (GCVTable*)LOAD_VTABLE (start);
3170 size = safe_object_get_size ((MonoObject*) start);
3171 size += ALLOC_ALIGN - 1;
3172 size &= ~(ALLOC_ALIGN - 1);
3175 fprintf (heap_dump_file, "<object offset=\"%d\" class=\"%s.%s\" size=\"%d\"/>\n",
3176 start - section->data,
3177 vt->klass->name_space, vt->klass->name,
3185 dump_occupied (occ_start, start, section->data);
3187 fprintf (heap_dump_file, "</section>\n");
3191 dump_heap (const char *type, int num, const char *reason)
3193 static char const *internal_mem_names [] = { "pin-queue", "fragment", "section", "scan-starts",
3194 "fin-table", "finalize-entry", "dislink-table",
3195 "dislink", "roots-table", "root-record", "statistics",
3196 "remset", "gray-queue", "store-remset" };
3198 GCMemSection *section;
3202 fprintf (heap_dump_file, "<collection type=\"%s\" num=\"%d\"", type, num);
3204 fprintf (heap_dump_file, " reason=\"%s\"", reason);
3205 fprintf (heap_dump_file, ">\n");
3206 fprintf (heap_dump_file, "<other-mem-usage type=\"pinned-chunks\" size=\"%ld\"/>\n", pinned_chunk_bytes_alloced);
3207 fprintf (heap_dump_file, "<other-mem-usage type=\"large-internal\" size=\"%ld\"/>\n", large_internal_bytes_alloced);
3208 fprintf (heap_dump_file, "<other-mem-usage type=\"mempools\" size=\"%ld\"/>\n", mono_mempool_get_bytes_allocated ());
3209 for (i = 0; i < INTERNAL_MEM_MAX; ++i)
3210 fprintf (heap_dump_file, "<other-mem-usage type=\"%s\" size=\"%ld\"/>\n", internal_mem_names [i], small_internal_mem_bytes [i]);
3211 fprintf (heap_dump_file, "<pinned type=\"stack\" bytes=\"%zu\"/>\n", pinned_byte_counts [PIN_TYPE_STACK]);
3212 /* fprintf (heap_dump_file, "<pinned type=\"static-data\" bytes=\"%d\"/>\n", pinned_byte_counts [PIN_TYPE_STATIC_DATA]); */
3213 fprintf (heap_dump_file, "<pinned type=\"other\" bytes=\"%zu\"/>\n", pinned_byte_counts [PIN_TYPE_OTHER]);
3215 dump_section (nursery_section, "nursery");
3217 for (section = section_list; section; section = section->block.next) {
3218 if (section != nursery_section)
3219 dump_section (section, "old");
3222 fprintf (heap_dump_file, "<los>\n");
3223 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
3224 MonoObject *obj = (MonoObject*) bigobj->data;
3225 MonoClass *class = mono_object_class (obj);
3227 fprintf (heap_dump_file, "<object class=\"%s.%s\" size=\"%d\"/>\n",
3228 class->name_space, class->name,
3229 safe_object_get_size (obj));
3231 fprintf (heap_dump_file, "</los>\n");
3233 fprintf (heap_dump_file, "</collection>\n");
3239 static gboolean inited = FALSE;
3241 #ifdef HEAVY_STATISTICS
3242 num_copy_object_called = 0;
3243 num_objects_copied = 0;
3249 #ifdef HEAVY_STATISTICS
3250 mono_counters_register ("WBarrier set field", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_field);
3251 mono_counters_register ("WBarrier set arrayref", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_arrayref);
3252 mono_counters_register ("WBarrier arrayref copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_arrayref_copy);
3253 mono_counters_register ("WBarrier generic store called", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_generic_store);
3254 mono_counters_register ("WBarrier generic store stored", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_generic_store_remset);
3255 mono_counters_register ("WBarrier set root", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_root);
3256 mono_counters_register ("WBarrier value copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_value_copy);
3257 mono_counters_register ("WBarrier object copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_object_copy);
3259 mono_counters_register ("# objects allocated", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_alloced);
3260 mono_counters_register ("# objects allocated degraded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_alloced_degraded);
3261 mono_counters_register ("# copy_object() called (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_called_nursery);
3262 mono_counters_register ("# objects copied (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_copied_nursery);
3263 mono_counters_register ("# copy_object() called (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_called_major);
3264 mono_counters_register ("# objects copied (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_copied_major);
3266 mono_counters_register ("# copy_object() failed from space", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_failed_from_space);
3267 mono_counters_register ("# copy_object() failed forwarded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_failed_forwarded);
3268 mono_counters_register ("# copy_object() failed pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_failed_pinned);
3269 mono_counters_register ("# copy_object() failed large or pinned chunk", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_failed_large_pinned);
3270 mono_counters_register ("# copy_object() failed to space", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_failed_to_space);
3272 mono_counters_register ("Store remsets", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_store_remsets);
3273 mono_counters_register ("Unique store remsets", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_store_remsets_unique);
3274 mono_counters_register ("Saved remsets 1", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_saved_remsets_1);
3275 mono_counters_register ("Saved remsets 2", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_saved_remsets_2);
3276 mono_counters_register ("Global remsets added", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_added);
3277 mono_counters_register ("Global remsets processed", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_processed);
3284 commit_stats (int generation)
3286 #ifdef HEAVY_STATISTICS
3287 if (generation == GENERATION_NURSERY) {
3288 stat_copy_object_called_nursery += num_copy_object_called;
3289 stat_objects_copied_nursery += num_objects_copied;
3291 g_assert (generation == GENERATION_OLD);
3292 stat_copy_object_called_major += num_copy_object_called;
3293 stat_objects_copied_major += num_objects_copied;
3299 * Collect objects in the nursery. Returns whether to trigger a major
3303 collect_nursery (size_t requested_size)
3305 size_t max_garbage_amount;
3307 char *orig_nursery_next;
3309 GCMemSection *section;
3310 int old_num_major_sections = num_major_sections;
3311 int sections_alloced;
3312 TV_DECLARE (all_atv);
3313 TV_DECLARE (all_btv);
3318 binary_protocol_collection (GENERATION_NURSERY);
3319 check_scan_starts ();
3322 orig_nursery_next = nursery_next;
3323 nursery_next = MAX (nursery_next, nursery_last_pinned_end);
3324 /* FIXME: optimize later to use the higher address where an object can be present */
3325 nursery_next = MAX (nursery_next, nursery_real_end);
3327 DEBUG (1, fprintf (gc_debug_file, "Start nursery collection %d %p-%p, size: %d\n", num_minor_gcs, nursery_start, nursery_next, (int)(nursery_next - nursery_start)));
3328 max_garbage_amount = nursery_next - nursery_start;
3329 g_assert (nursery_section->size >= max_garbage_amount);
3331 /* Clear all remaining nursery fragments, pinning depends on this */
3332 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION) {
3333 g_assert (orig_nursery_next <= nursery_frag_real_end);
3334 memset (orig_nursery_next, 0, nursery_frag_real_end - orig_nursery_next);
3335 for (frag = nursery_fragments; frag; frag = frag->next) {
3336 memset (frag->fragment_start, 0, frag->fragment_end - frag->fragment_start);
3341 check_for_xdomain_refs ();
3343 nursery_section->next_data = nursery_next;
3345 if (!to_space_section) {
3346 new_to_space_section ();
3348 /* we might have done degraded allocation since the
3350 g_assert (to_space_bumper <= to_space_section->next_data);
3351 to_space_bumper = to_space_section->next_data;
3353 to_space_section->is_to_space = TRUE;
3355 gray_object_queue_init ();
3358 mono_stats.minor_gc_count ++;
3359 /* world must be stopped already */
3360 TV_GETTIME (all_atv);
3362 /* pin from pinned handles */
3364 pin_from_roots (nursery_start, nursery_next);
3365 /* identify pinned objects */
3366 optimize_pin_queue (0);
3367 next_pin_slot = pin_objects_from_addresses (nursery_section, pin_queue, pin_queue + next_pin_slot, nursery_start, nursery_next);
3369 DEBUG (2, fprintf (gc_debug_file, "Finding pinned pointers: %d in %d usecs\n", next_pin_slot, TV_ELAPSED (atv, btv)));
3370 DEBUG (4, fprintf (gc_debug_file, "Start scan with %d pinned objects\n", next_pin_slot));
3372 if (consistency_check_at_minor_collection)
3373 check_consistency ();
3376 * walk all the roots and copy the young objects to the old generation,
3377 * starting from to_space
3380 scan_from_remsets (nursery_start, nursery_next);
3381 /* we don't have complete write barrier yet, so we scan all the old generation sections */
3383 DEBUG (2, fprintf (gc_debug_file, "Old generation scan: %d usecs\n", TV_ELAPSED (btv, atv)));
3385 /* the pinned objects are roots */
3386 for (i = 0; i < next_pin_slot; ++i) {
3387 DEBUG (6, fprintf (gc_debug_file, "Precise object scan %d of pinned %p (%s)\n", i, pin_queue [i], safe_name (pin_queue [i])));
3388 scan_object (pin_queue [i], nursery_start, nursery_next);
3390 /* registered roots, this includes static fields */
3391 scan_from_registered_roots (nursery_start, nursery_next, ROOT_TYPE_NORMAL);
3392 scan_from_registered_roots (nursery_start, nursery_next, ROOT_TYPE_WBARRIER);
3393 scan_thread_data (nursery_start, nursery_next, TRUE);
3394 /* alloc_pinned objects */
3395 scan_from_pinned_objects (nursery_start, nursery_next);
3397 DEBUG (2, fprintf (gc_debug_file, "Root scan: %d usecs\n", TV_ELAPSED (atv, btv)));
3399 finish_gray_stack (nursery_start, nursery_next, GENERATION_NURSERY);
3401 /* walk the pin_queue, build up the fragment list of free memory, unmark
3402 * pinned objects as we go, memzero() the empty fragments so they are ready for the
3405 build_nursery_fragments (0, next_pin_slot);
3407 DEBUG (2, fprintf (gc_debug_file, "Fragment creation: %d usecs, %zd bytes available\n", TV_ELAPSED (btv, atv), fragment_total));
3409 for (section = section_list; section; section = section->block.next) {
3410 if (section->is_to_space)
3411 section->is_to_space = FALSE;
3414 TV_GETTIME (all_btv);
3415 mono_stats.minor_gc_time_usecs += TV_ELAPSED (all_atv, all_btv);
3418 dump_heap ("minor", num_minor_gcs - 1, NULL);
3420 /* prepare the pin queue for the next collection */
3421 last_num_pinned = next_pin_slot;
3423 if (fin_ready_list || critical_fin_list) {
3424 DEBUG (4, fprintf (gc_debug_file, "Finalizer-thread wakeup: ready %d\n", num_ready_finalizers));
3425 mono_gc_finalize_notify ();
3429 g_assert (gray_object_queue_is_empty ());
3431 commit_stats (GENERATION_NURSERY);
3433 check_scan_starts ();
3435 sections_alloced = num_major_sections - old_num_major_sections;
3436 minor_collection_sections_alloced += sections_alloced;
3438 return minor_collection_sections_alloced > minor_collection_section_allowance;
3442 scan_from_pinned_chunk_if_marked (PinnedChunk *chunk, char *obj, size_t size, void *dummy)
3444 if (object_is_pinned (obj))
3445 scan_object (obj, NULL, (char*)-1);
3449 major_collection (const char *reason)
3451 GCMemSection *section, *prev_section;
3452 LOSObject *bigobj, *prevbo;
3456 TV_DECLARE (all_atv);
3457 TV_DECLARE (all_btv);
3460 /* FIXME: only use these values for the precise scan
3461 * note that to_space pointers should be excluded anyway...
3463 char *heap_start = NULL;
3464 char *heap_end = (char*)-1;
3465 size_t copy_space_required = 0;
3466 int old_num_major_sections = num_major_sections;
3467 int num_major_sections_saved, save_target, allowance_target;
3470 binary_protocol_collection (GENERATION_OLD);
3471 check_scan_starts ();
3474 DEBUG (1, fprintf (gc_debug_file, "Start major collection %d\n", num_major_gcs));
3476 mono_stats.major_gc_count ++;
3478 /* Clear all remaining nursery fragments, pinning depends on this */
3479 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION) {
3480 g_assert (nursery_next <= nursery_frag_real_end);
3481 memset (nursery_next, 0, nursery_frag_real_end - nursery_next);
3482 for (frag = nursery_fragments; frag; frag = frag->next) {
3483 memset (frag->fragment_start, 0, frag->fragment_end - frag->fragment_start);
3488 check_for_xdomain_refs ();
3491 * FIXME: implement Mark/Compact
3492 * Until that is done, we can just apply mostly the same alg as for the nursery:
3493 * this means we need a big section to potentially copy all the other sections, so
3494 * it is not ideal specially with large heaps.
3496 if (g_getenv ("MONO_GC_NO_MAJOR")) {
3497 collect_nursery (0);
3500 TV_GETTIME (all_atv);
3501 /* FIXME: make sure the nursery next_data ptr is updated */
3502 nursery_section->next_data = nursery_real_end;
3503 /* we should also coalesce scanning from sections close to each other
3504 * and deal with pointers outside of the sections later.
3506 /* The remsets are not useful for a major collection */
3508 /* world must be stopped already */
3511 DEBUG (6, fprintf (gc_debug_file, "Collecting pinned addresses\n"));
3512 pin_from_roots ((void*)lowest_heap_address, (void*)highest_heap_address);
3513 optimize_pin_queue (0);
3516 * pin_queue now contains all candidate pointers, sorted and
3517 * uniqued. We must do two passes now to figure out which
3518 * objects are pinned.
3520 * The first is to find within the pin_queue the area for each
3521 * section. This requires that the pin_queue be sorted. We
3522 * also process the LOS objects and pinned chunks here.
3524 * The second, destructive, pass is to reduce the section
3525 * areas to pointers to the actually pinned objects.
3527 DEBUG (6, fprintf (gc_debug_file, "Pinning from sections\n"));
3528 /* first pass for the sections */
3529 for (section = section_list; section; section = section->block.next) {
3531 DEBUG (6, fprintf (gc_debug_file, "Pinning from section %p (%p-%p)\n", section, section->data, section->end_data));
3532 find_optimized_pin_queue_area (section->data, section->end_data, &start, &end);
3533 DEBUG (6, fprintf (gc_debug_file, "Found %d pinning addresses in section %p (%d-%d)\n",
3534 end - start, section, start, end));
3535 section->pin_queue_start = start;
3536 section->pin_queue_end = end;
3538 /* identify possible pointers to the insize of large objects */
3539 DEBUG (6, fprintf (gc_debug_file, "Pinning from large objects\n"));
3540 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
3542 find_optimized_pin_queue_area (bigobj->data, (char*)bigobj->data + bigobj->size, &start, &end);
3544 pin_object (bigobj->data);
3546 pin_stats_register_object ((char*) bigobj->data, safe_object_get_size ((MonoObject*) bigobj->data));
3547 DEBUG (6, fprintf (gc_debug_file, "Marked large object %p (%s) size: %zd from roots\n", bigobj->data, safe_name (bigobj->data), bigobj->size));
3550 /* look for pinned addresses for pinned-alloc objects */
3551 DEBUG (6, fprintf (gc_debug_file, "Pinning from pinned-alloc objects\n"));
3552 for (chunk = pinned_chunk_list; chunk; chunk = chunk->block.next) {
3554 find_optimized_pin_queue_area (chunk->start_data, (char*)chunk + chunk->num_pages * FREELIST_PAGESIZE, &start, &end);
3556 mark_pinned_from_addresses (chunk, pin_queue + start, pin_queue + end);
3558 /* second pass for the sections */
3559 for (section = section_list; section; section = section->block.next) {
3560 int start = section->pin_queue_start;
3561 int end = section->pin_queue_end;
3564 reduced_to = pin_objects_from_addresses (section, pin_queue + start, pin_queue + end,
3565 section->data, section->next_data);
3566 section->pin_queue_start = start;
3567 section->pin_queue_end = start + reduced_to;
3569 copy_space_required += (char*)section->next_data - (char*)section->data;
3573 DEBUG (2, fprintf (gc_debug_file, "Finding pinned pointers: %d in %d usecs\n", next_pin_slot, TV_ELAPSED (atv, btv)));
3574 DEBUG (4, fprintf (gc_debug_file, "Start scan with %d pinned objects\n", next_pin_slot));
3576 new_to_space_section ();
3577 gray_object_queue_init ();
3579 /* the old generation doesn't need to be scanned (no remembered sets or card
3580 * table needed either): the only objects that must survive are those pinned and
3581 * those referenced by the precise roots.
3582 * mark any section without pinned objects, so we can free it since we will be able to
3583 * move all the objects.
3585 /* the pinned objects are roots (big objects are included in this list, too) */
3586 for (section = section_list; section; section = section->block.next) {
3587 for (i = section->pin_queue_start; i < section->pin_queue_end; ++i) {
3588 DEBUG (6, fprintf (gc_debug_file, "Precise object scan %d of pinned %p (%s)\n",
3589 i, pin_queue [i], safe_name (pin_queue [i])));
3590 scan_object (pin_queue [i], heap_start, heap_end);
3593 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
3594 if (object_is_pinned (bigobj->data)) {
3595 DEBUG (6, fprintf (gc_debug_file, "Precise object scan pinned LOS object %p (%s)\n",
3596 bigobj->data, safe_name (bigobj->data)));
3597 scan_object (bigobj->data, heap_start, heap_end);
3600 scan_pinned_objects (scan_from_pinned_chunk_if_marked, NULL);
3601 /* registered roots, this includes static fields */
3602 scan_from_registered_roots (heap_start, heap_end, ROOT_TYPE_NORMAL);
3603 scan_from_registered_roots (heap_start, heap_end, ROOT_TYPE_WBARRIER);
3605 scan_thread_data (heap_start, heap_end, TRUE);
3606 /* alloc_pinned objects */
3607 scan_from_pinned_objects (heap_start, heap_end);
3608 /* scan the list of objects ready for finalization */
3609 scan_finalizer_entries (fin_ready_list, heap_start, heap_end);
3610 scan_finalizer_entries (critical_fin_list, heap_start, heap_end);
3612 DEBUG (2, fprintf (gc_debug_file, "Root scan: %d usecs\n", TV_ELAPSED (btv, atv)));
3614 /* we need to go over the big object list to see if any was marked and scan it
3615 * And we need to make this in a loop, considering that objects referenced by finalizable
3616 * objects could reference big objects (this happens in finish_gray_stack ())
3618 scan_needed_big_objects (heap_start, heap_end);
3619 /* all the objects in the heap */
3620 finish_gray_stack (heap_start, heap_end, GENERATION_OLD);
3624 /* sweep the big objects list */
3626 for (bigobj = los_object_list; bigobj;) {
3627 if (object_is_pinned (bigobj->data)) {
3628 unpin_object (bigobj->data);
3629 bigobj->scanned = FALSE;
3632 /* not referenced anywhere, so we can free it */
3634 prevbo->next = bigobj->next;
3636 los_object_list = bigobj->next;
3638 bigobj = bigobj->next;
3639 free_large_object (to_free);
3643 bigobj = bigobj->next;
3645 /* unpin objects from the pinned chunks and free the unmarked ones */
3646 sweep_pinned_objects ();
3648 /* free the unused sections */
3649 prev_section = NULL;
3650 for (section = section_list; section;) {
3651 /* to_space doesn't need handling here and the nursery is special */
3652 if (section->is_to_space || section == nursery_section) {
3653 if (section->is_to_space)
3654 section->is_to_space = FALSE;
3655 prev_section = section;
3656 section = section->block.next;
3659 /* no pinning object, so the section is free */
3660 if (section->pin_queue_start == section->pin_queue_end) {
3661 GCMemSection *to_free;
3663 prev_section->block.next = section->block.next;
3665 section_list = section->block.next;
3667 section = section->block.next;
3668 free_major_section (to_free);
3671 DEBUG (6, fprintf (gc_debug_file, "Section %p has still pinned objects (%d)\n", section, section->pin_queue_end - section->pin_queue_start));
3672 build_section_fragments (section);
3674 prev_section = section;
3675 section = section->block.next;
3678 /* walk the pin_queue, build up the fragment list of free memory, unmark
3679 * pinned objects as we go, memzero() the empty fragments so they are ready for the
3682 build_nursery_fragments (nursery_section->pin_queue_start, nursery_section->pin_queue_end);
3684 TV_GETTIME (all_btv);
3685 mono_stats.major_gc_time_usecs += TV_ELAPSED (all_atv, all_btv);
3688 dump_heap ("major", num_major_gcs - 1, reason);
3690 /* prepare the pin queue for the next collection */
3692 if (fin_ready_list || critical_fin_list) {
3693 DEBUG (4, fprintf (gc_debug_file, "Finalizer-thread wakeup: ready %d\n", num_ready_finalizers));
3694 mono_gc_finalize_notify ();
3698 g_assert (gray_object_queue_is_empty ());
3700 commit_stats (GENERATION_OLD);
3702 num_major_sections_saved = MAX (old_num_major_sections - num_major_sections, 1);
3704 save_target = num_major_sections / 2;
3705 allowance_target = save_target * minor_collection_sections_alloced / num_major_sections_saved;
3707 minor_collection_section_allowance = MAX (MIN (allowance_target, num_major_sections), MIN_MINOR_COLLECTION_SECTION_ALLOWANCE);
3710 printf ("alloced %d saved %d target %d allowance %d\n",
3711 minor_collection_sections_alloced, num_major_sections_saved, allowance_target,
3712 minor_collection_section_allowance);
3715 minor_collection_sections_alloced = 0;
3717 check_scan_starts ();
3721 * Allocate a new section of memory to be used as old generation.
3723 static GCMemSection*
3724 alloc_major_section (void)
3726 GCMemSection *section;
3729 section = get_os_memory_aligned (MAJOR_SECTION_SIZE, TRUE);
3730 section->next_data = section->data = (char*)section + SIZEOF_GC_MEM_SECTION;
3731 g_assert (!((mword)section->data & 7));
3732 section->size = MAJOR_SECTION_SIZE - SIZEOF_GC_MEM_SECTION;
3733 section->end_data = section->data + section->size;
3734 UPDATE_HEAP_BOUNDARIES (section->data, section->end_data);
3735 total_alloc += section->size;
3736 DEBUG (3, fprintf (gc_debug_file, "New major heap section: (%p-%p), total: %zd\n", section->data, section->end_data, total_alloc));
3737 scan_starts = (section->size + SCAN_START_SIZE - 1) / SCAN_START_SIZE;
3738 section->scan_starts = get_internal_mem (sizeof (char*) * scan_starts, INTERNAL_MEM_SCAN_STARTS);
3739 section->num_scan_start = scan_starts;
3740 section->block.role = MEMORY_ROLE_GEN1;
3741 section->is_to_space = TRUE;
3743 /* add to the section list */
3744 section->block.next = section_list;
3745 section_list = section;
3747 ++num_major_sections;
3753 free_major_section (GCMemSection *section)
3755 DEBUG (3, fprintf (gc_debug_file, "Freed major section %p (%p-%p)\n", section, section->data, section->end_data));
3756 free_internal_mem (section->scan_starts, INTERNAL_MEM_SCAN_STARTS);
3757 free_os_memory (section, MAJOR_SECTION_SIZE);
3758 total_alloc -= MAJOR_SECTION_SIZE - SIZEOF_GC_MEM_SECTION;
3760 --num_major_sections;
3764 * When deciding if it's better to collect or to expand, keep track
3765 * of how much garbage was reclaimed with the last collection: if it's too
3767 * This is called when we could not allocate a small object.
3769 static void __attribute__((noinline))
3770 minor_collect_or_expand_inner (size_t size)
3772 int do_minor_collection = 1;
3774 if (!nursery_section) {
3778 if (do_minor_collection) {
3780 if (collect_nursery (size))
3781 major_collection ("minor overflow");
3782 DEBUG (2, fprintf (gc_debug_file, "Heap size: %zd, LOS size: %zd\n", total_alloc, los_memory_usage));
3784 /* this also sets the proper pointers for the next allocation */
3785 if (!search_fragment_for_size (size)) {
3787 /* TypeBuilder and MonoMethod are killing mcs with fragmentation */
3788 DEBUG (1, fprintf (gc_debug_file, "nursery collection didn't find enough room for %zd alloc (%d pinned)\n", size, last_num_pinned));
3789 for (i = 0; i < last_num_pinned; ++i) {
3790 DEBUG (3, fprintf (gc_debug_file, "Bastard pinning obj %p (%s), size: %d\n", pin_queue [i], safe_name (pin_queue [i]), safe_object_get_size (pin_queue [i])));
3795 //report_internal_mem_usage ();
3799 * ######################################################################
3800 * ######## Memory allocation from the OS
3801 * ######################################################################
3802 * This section of code deals with getting memory from the OS and
3803 * allocating memory for GC-internal data structures.
3804 * Internal memory can be handled with a freelist for small objects.
3808 * Allocate a big chunk of memory from the OS (usually 64KB to several megabytes).
3809 * This must not require any lock.
3812 get_os_memory (size_t size, int activate)
3815 unsigned long prot_flags = activate? MONO_MMAP_READ|MONO_MMAP_WRITE: MONO_MMAP_NONE;
3817 prot_flags |= MONO_MMAP_PRIVATE | MONO_MMAP_ANON;
3818 size += pagesize - 1;
3819 size &= ~(pagesize - 1);
3820 ptr = mono_valloc (0, size, prot_flags);
3825 * Free the memory returned by get_os_memory (), returning it to the OS.
3828 free_os_memory (void *addr, size_t size)
3830 mono_vfree (addr, size);
3837 report_pinned_chunk (PinnedChunk *chunk, int seq) {
3839 int i, free_pages, num_free, free_mem;
3841 for (i = 0; i < chunk->num_pages; ++i) {
3842 if (!chunk->page_sizes [i])
3845 printf ("Pinned chunk %d at %p, size: %d, pages: %d, free: %d\n", seq, chunk, chunk->num_pages * FREELIST_PAGESIZE, chunk->num_pages, free_pages);
3846 free_mem = FREELIST_PAGESIZE * free_pages;
3847 for (i = 0; i < FREELIST_NUM_SLOTS; ++i) {
3848 if (!chunk->free_list [i])
3851 p = chunk->free_list [i];
3856 printf ("\tfree list of size %d, %d items\n", freelist_sizes [i], num_free);
3857 free_mem += freelist_sizes [i] * num_free;
3859 printf ("\tfree memory in chunk: %d\n", free_mem);
3865 static G_GNUC_UNUSED void
3866 report_internal_mem_usage (void) {
3869 printf ("Internal memory usage:\n");
3871 for (chunk = internal_chunk_list; chunk; chunk = chunk->block.next) {
3872 report_pinned_chunk (chunk, i++);
3874 printf ("Pinned memory usage:\n");
3876 for (chunk = pinned_chunk_list; chunk; chunk = chunk->block.next) {
3877 report_pinned_chunk (chunk, i++);
3882 * the array of pointers from @start to @end contains conservative
3883 * pointers to objects inside @chunk: mark each referenced object
3887 mark_pinned_from_addresses (PinnedChunk *chunk, void **start, void **end)
3889 for (; start < end; start++) {
3890 char *addr = *start;
3891 int offset = (char*)addr - (char*)chunk;
3892 int page = offset / FREELIST_PAGESIZE;
3893 int obj_offset = page == 0? offset - ((char*)chunk->start_data - (char*)chunk): offset % FREELIST_PAGESIZE;
3894 int slot_size = chunk->page_sizes [page];
3896 /* the page is not allocated */
3899 /* would be faster if we restrict the sizes to power of two,
3900 * but that's a waste of memory: need to measure. it could reduce
3901 * fragmentation since there are less pages needed, if for example
3902 * someone interns strings of each size we end up with one page per
3903 * interned string (still this is just ~40 KB): with more fine-grained sizes
3904 * this increases the number of used pages.
3907 obj_offset /= slot_size;
3908 obj_offset *= slot_size;
3909 addr = (char*)chunk->start_data + obj_offset;
3911 obj_offset /= slot_size;
3912 obj_offset *= slot_size;
3913 addr = (char*)chunk + page * FREELIST_PAGESIZE + obj_offset;
3916 /* if the vtable is inside the chunk it's on the freelist, so skip */
3917 if (*ptr && (*ptr < (void*)chunk->start_data || *ptr > (void*)((char*)chunk + chunk->num_pages * FREELIST_PAGESIZE))) {
3918 binary_protocol_pin (addr, (gpointer)LOAD_VTABLE (addr), safe_object_get_size ((MonoObject*)addr));
3921 pin_stats_register_object ((char*) addr, safe_object_get_size ((MonoObject*) addr));
3922 DEBUG (6, fprintf (gc_debug_file, "Marked pinned object %p (%s) from roots\n", addr, safe_name (addr)));
3928 scan_pinned_objects (ScanPinnedObjectCallbackFunc callback, void *callback_data)
3935 for (chunk = pinned_chunk_list; chunk; chunk = chunk->block.next) {
3936 end_chunk = (char*)chunk + chunk->num_pages * FREELIST_PAGESIZE;
3937 DEBUG (6, fprintf (gc_debug_file, "Scanning pinned chunk %p (range: %p-%p)\n", chunk, chunk->start_data, end_chunk));
3938 for (i = 0; i < chunk->num_pages; ++i) {
3939 obj_size = chunk->page_sizes [i];
3942 p = i? (char*)chunk + i * FREELIST_PAGESIZE: chunk->start_data;
3943 endp = i? p + FREELIST_PAGESIZE: (char*)chunk + FREELIST_PAGESIZE;
3944 DEBUG (6, fprintf (gc_debug_file, "Page %d (size: %d, range: %p-%p)\n", i, obj_size, p, endp));
3945 while (p + obj_size <= endp) {
3947 DEBUG (9, fprintf (gc_debug_file, "Considering %p (vtable: %p)\n", ptr, *ptr));
3948 /* if the first word (the vtable) is outside the chunk we have an object */
3949 if (*ptr && (*ptr < (void*)chunk || *ptr >= end_chunk))
3950 callback (chunk, (char*)ptr, obj_size, callback_data);
3958 sweep_pinned_objects_callback (PinnedChunk *chunk, char *ptr, size_t size, void *data)
3960 if (object_is_pinned (ptr)) {
3962 DEBUG (6, fprintf (gc_debug_file, "Unmarked pinned object %p (%s)\n", ptr, safe_name (ptr)));
3964 DEBUG (6, fprintf (gc_debug_file, "Freeing unmarked pinned object %p (%s)\n", ptr, safe_name (ptr)));
3965 free_pinned_object (chunk, ptr, size);
3970 sweep_pinned_objects (void)
3972 scan_pinned_objects (sweep_pinned_objects_callback, NULL);
3976 scan_object_callback (PinnedChunk *chunk, char *ptr, size_t size, char **data)
3978 DEBUG (6, fprintf (gc_debug_file, "Precise object scan of alloc_pinned %p (%s)\n", ptr, safe_name (ptr)));
3979 /* FIXME: Put objects without references into separate chunks
3980 which do not need to be scanned */
3981 scan_object (ptr, data [0], data [1]);
3985 scan_from_pinned_objects (char *addr_start, char *addr_end)
3987 char *data [2] = { addr_start, addr_end };
3988 scan_pinned_objects ((ScanPinnedObjectCallbackFunc)scan_object_callback, data);
3992 * Find the slot number in the freelist for memory chunks that
3993 * can contain @size objects.
3996 slot_for_size (size_t size)
3999 /* do a binary search or lookup table later. */
4000 for (slot = 0; slot < FREELIST_NUM_SLOTS; ++slot) {
4001 if (freelist_sizes [slot] >= size)
4004 g_assert_not_reached ();
4009 * Build a free list for @size memory chunks from the memory area between
4010 * start_page and end_page.
4013 build_freelist (PinnedChunk *chunk, int slot, int size, char *start_page, char *end_page)
4017 /*g_print ("building freelist for slot %d, size %d in %p\n", slot, size, chunk);*/
4018 p = (void**)start_page;
4019 end = (void**)(end_page - size);
4020 g_assert (!chunk->free_list [slot]);
4021 chunk->free_list [slot] = p;
4022 while ((char*)p + size <= (char*)end) {
4024 *p = (void*)((char*)p + size);
4028 /*g_print ("%d items created, max: %d\n", count, (end_page - start_page) / size);*/
4032 alloc_pinned_chunk (void)
4036 int size = MAJOR_SECTION_SIZE;
4038 chunk = get_os_memory_aligned (size, TRUE);
4039 chunk->block.role = MEMORY_ROLE_PINNED;
4041 UPDATE_HEAP_BOUNDARIES (chunk, ((char*)chunk + size));
4042 total_alloc += size;
4043 pinned_chunk_bytes_alloced += size;
4045 /* setup the bookeeping fields */
4046 chunk->num_pages = size / FREELIST_PAGESIZE;
4047 offset = G_STRUCT_OFFSET (PinnedChunk, data);
4048 chunk->page_sizes = (void*)((char*)chunk + offset);
4049 offset += sizeof (int) * chunk->num_pages;
4050 offset += ALLOC_ALIGN - 1;
4051 offset &= ~(ALLOC_ALIGN - 1);
4052 chunk->free_list = (void*)((char*)chunk + offset);
4053 offset += sizeof (void*) * FREELIST_NUM_SLOTS;
4054 offset += ALLOC_ALIGN - 1;
4055 offset &= ~(ALLOC_ALIGN - 1);
4056 chunk->start_data = (void*)((char*)chunk + offset);
4058 /* allocate the first page to the freelist */
4059 chunk->page_sizes [0] = PINNED_FIRST_SLOT_SIZE;
4060 build_freelist (chunk, slot_for_size (PINNED_FIRST_SLOT_SIZE), PINNED_FIRST_SLOT_SIZE, chunk->start_data, ((char*)chunk + FREELIST_PAGESIZE));
4061 DEBUG (4, fprintf (gc_debug_file, "Allocated pinned chunk %p, size: %d\n", chunk, size));
4062 min_pinned_chunk_addr = MIN (min_pinned_chunk_addr, (char*)chunk->start_data);
4063 max_pinned_chunk_addr = MAX (max_pinned_chunk_addr, ((char*)chunk + size));
4067 /* assumes freelist for slot is empty, so try to alloc a new page */
4069 get_chunk_freelist (PinnedChunk *chunk, int slot)
4073 p = chunk->free_list [slot];
4075 chunk->free_list [slot] = *p;
4078 for (i = 0; i < chunk->num_pages; ++i) {
4080 if (chunk->page_sizes [i])
4082 size = freelist_sizes [slot];
4083 chunk->page_sizes [i] = size;
4084 build_freelist (chunk, slot, size, (char*)chunk + FREELIST_PAGESIZE * i, (char*)chunk + FREELIST_PAGESIZE * (i + 1));
4088 p = chunk->free_list [slot];
4090 chunk->free_list [slot] = *p;
4097 alloc_from_freelist (size_t size)
4101 PinnedChunk *pchunk;
4102 slot = slot_for_size (size);
4103 /*g_print ("using slot %d for size %d (slot size: %d)\n", slot, size, freelist_sizes [slot]);*/
4104 g_assert (size <= freelist_sizes [slot]);
4105 for (pchunk = pinned_chunk_list; pchunk; pchunk = pchunk->block.next) {
4106 void **p = pchunk->free_list [slot];
4108 /*g_print ("found freelist for slot %d in chunk %p, returning %p, next %p\n", slot, pchunk, p, *p);*/
4109 pchunk->free_list [slot] = *p;
4113 for (pchunk = pinned_chunk_list; pchunk; pchunk = pchunk->block.next) {
4114 res = get_chunk_freelist (pchunk, slot);
4118 pchunk = alloc_pinned_chunk ();
4119 /* FIXME: handle OOM */
4120 pchunk->block.next = pinned_chunk_list;
4121 pinned_chunk_list = pchunk;
4122 res = get_chunk_freelist (pchunk, slot);
4126 /* used for the GC-internal data structures */
4127 /* FIXME: add support for bigger sizes by allocating more than one page
4131 get_internal_mem (size_t size, int type)
4135 PinnedChunk *pchunk;
4137 if (size > freelist_sizes [FREELIST_NUM_SLOTS - 1]) {
4138 LargeInternalMemHeader *mh;
4140 size += sizeof (LargeInternalMemHeader);
4141 mh = get_os_memory (size, TRUE);
4142 mh->magic = LARGE_INTERNAL_MEM_HEADER_MAGIC;
4145 large_internal_bytes_alloced += size;
4150 slot = slot_for_size (size);
4151 g_assert (size <= freelist_sizes [slot]);
4153 small_internal_mem_bytes [type] += freelist_sizes [slot];
4155 for (pchunk = internal_chunk_list; pchunk; pchunk = pchunk->block.next) {
4156 void **p = pchunk->free_list [slot];
4158 pchunk->free_list [slot] = *p;
4159 memset (p, 0, size);
4163 for (pchunk = internal_chunk_list; pchunk; pchunk = pchunk->block.next) {
4164 res = get_chunk_freelist (pchunk, slot);
4166 memset (res, 0, size);
4170 pchunk = alloc_pinned_chunk ();
4171 /* FIXME: handle OOM */
4172 pchunk->block.next = internal_chunk_list;
4173 internal_chunk_list = pchunk;
4174 res = get_chunk_freelist (pchunk, slot);
4175 memset (res, 0, size);
4180 free_internal_mem (void *addr, int type)
4182 PinnedChunk *pchunk;
4183 LargeInternalMemHeader *mh;
4186 for (pchunk = internal_chunk_list; pchunk; pchunk = pchunk->block.next) {
4187 /*printf ("trying to free %p in %p (pages: %d)\n", addr, pchunk, pchunk->num_pages);*/
4188 if (addr >= (void*)pchunk && (char*)addr < (char*)pchunk + pchunk->num_pages * FREELIST_PAGESIZE) {
4189 int offset = (char*)addr - (char*)pchunk;
4190 int page = offset / FREELIST_PAGESIZE;
4191 int slot = slot_for_size (pchunk->page_sizes [page]);
4193 *p = pchunk->free_list [slot];
4194 pchunk->free_list [slot] = p;
4196 small_internal_mem_bytes [type] -= freelist_sizes [slot];
4201 mh = (LargeInternalMemHeader*)((char*)addr - G_STRUCT_OFFSET (LargeInternalMemHeader, data));
4202 g_assert (mh->magic == LARGE_INTERNAL_MEM_HEADER_MAGIC);
4203 large_internal_bytes_alloced -= mh->size;
4204 free_os_memory (mh, mh->size);
4208 * ######################################################################
4209 * ######## Object allocation
4210 * ######################################################################
4211 * This section of code deals with allocating memory for objects.
4212 * There are several ways:
4213 * *) allocate large objects
4214 * *) allocate normal objects
4215 * *) fast lock-free allocation
4216 * *) allocation of pinned objects
4220 free_large_object (LOSObject *obj)
4222 size_t size = obj->size;
4223 DEBUG (4, fprintf (gc_debug_file, "Freed large object %p, size %zd\n", obj->data, obj->size));
4224 binary_protocol_empty (obj->data, obj->size);
4226 los_memory_usage -= size;
4227 size += sizeof (LOSObject);
4228 size += pagesize - 1;
4229 size &= ~(pagesize - 1);
4230 total_alloc -= size;
4232 free_os_memory (obj, size);
4236 * Objects with size >= 64KB are allocated in the large object space.
4237 * They are currently kept track of with a linked list.
4238 * They don't move, so there is no need to pin them during collection
4239 * and we avoid the memcpy overhead.
4241 static void* __attribute__((noinline))
4242 alloc_large_inner (MonoVTable *vtable, size_t size)
4247 int just_did_major_gc = FALSE;
4249 g_assert (size > MAX_SMALL_OBJ_SIZE);
4251 if (los_memory_usage > next_los_collection) {
4252 DEBUG (4, fprintf (gc_debug_file, "Should trigger major collection: req size %zd (los already: %zu, limit: %zu)\n", size, los_memory_usage, next_los_collection));
4253 just_did_major_gc = TRUE;
4255 major_collection ("LOS overflow");
4257 /* later increase based on a percent of the heap size */
4258 next_los_collection = los_memory_usage + 5*1024*1024;
4261 alloc_size += sizeof (LOSObject);
4262 alloc_size += pagesize - 1;
4263 alloc_size &= ~(pagesize - 1);
4264 /* FIXME: handle OOM */
4265 obj = get_os_memory (alloc_size, TRUE);
4267 vtslot = (void**)obj->data;
4269 total_alloc += alloc_size;
4270 UPDATE_HEAP_BOUNDARIES (obj->data, (char*)obj->data + size);
4271 obj->next = los_object_list;
4272 los_object_list = obj;
4273 los_memory_usage += size;
4275 DEBUG (4, fprintf (gc_debug_file, "Allocated large object %p, vtable: %p (%s), size: %zd\n", obj->data, vtable, vtable->klass->name, size));
4276 binary_protocol_alloc (obj->data, vtable, size);
4280 /* check if we have a suitable fragment in nursery_fragments to be able to allocate
4281 * an object of size @size
4282 * Return FALSE if not found (which means we need a collection)
4285 search_fragment_for_size (size_t size)
4287 Fragment *frag, *prev;
4288 DEBUG (4, fprintf (gc_debug_file, "Searching nursery fragment %p, size: %zd\n", nursery_frag_real_end, size));
4290 if (nursery_frag_real_end > nursery_next && nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
4291 /* Clear the remaining space, pinning depends on this */
4292 memset (nursery_next, 0, nursery_frag_real_end - nursery_next);
4295 for (frag = nursery_fragments; frag; frag = frag->next) {
4296 if (size <= (frag->fragment_end - frag->fragment_start)) {
4297 /* remove from the list */
4299 prev->next = frag->next;
4301 nursery_fragments = frag->next;
4302 nursery_next = frag->fragment_start;
4303 nursery_frag_real_end = frag->fragment_end;
4305 DEBUG (4, fprintf (gc_debug_file, "Using nursery fragment %p-%p, size: %zd (req: %zd)\n", nursery_next, nursery_frag_real_end, nursery_frag_real_end - nursery_next, size));
4306 frag->next = fragment_freelist;
4307 fragment_freelist = frag;
4316 * size is already rounded up and we hold the GC lock.
4319 alloc_degraded (MonoVTable *vtable, size_t size)
4321 GCMemSection *section;
4323 g_assert (size <= MAX_SMALL_OBJ_SIZE);
4324 HEAVY_STAT (++stat_objects_alloced_degraded);
4325 for (section = section_list; section; section = section->block.next) {
4326 if (section != nursery_section && (section->end_data - section->next_data) >= size) {
4327 p = (void**)section->next_data;
4332 section = alloc_major_section ();
4333 section->is_to_space = FALSE;
4334 /* FIXME: handle OOM */
4335 p = (void**)section->next_data;
4337 section->next_data += size;
4338 degraded_mode += size;
4339 DEBUG (3, fprintf (gc_debug_file, "Allocated (degraded) object %p, vtable: %p (%s), size: %zd in section %p\n", p, vtable, vtable->klass->name, size, section));
4345 * Provide a variant that takes just the vtable for small fixed-size objects.
4346 * The aligned size is already computed and stored in vt->gc_descr.
4347 * Note: every SCAN_START_SIZE or so we are given the chance to do some special
4348 * processing. We can keep track of where objects start, for example,
4349 * so when we scan the thread stacks for pinned objects, we can start
4350 * a search for the pinned object in SCAN_START_SIZE chunks.
4353 mono_gc_alloc_obj_nolock (MonoVTable *vtable, size_t size)
4355 /* FIXME: handle OOM */
4361 HEAVY_STAT (++stat_objects_alloced);
4363 size += ALLOC_ALIGN - 1;
4364 size &= ~(ALLOC_ALIGN - 1);
4366 g_assert (vtable->gc_descr);
4368 if (G_UNLIKELY (collect_before_allocs)) {
4369 if (nursery_section) {
4371 collect_nursery (0);
4373 if (!degraded_mode && !search_fragment_for_size (size)) {
4375 g_assert_not_reached ();
4381 * We must already have the lock here instead of after the
4382 * fast path because we might be interrupted in the fast path
4383 * (after confirming that new_next < TLAB_TEMP_END) by the GC,
4384 * and we'll end up allocating an object in a fragment which
4385 * no longer belongs to us.
4387 * The managed allocator does not do this, but it's treated
4388 * specially by the world-stopping code.
4391 if (size > MAX_SMALL_OBJ_SIZE) {
4392 p = alloc_large_inner (vtable, size);
4394 /* tlab_next and tlab_temp_end are TLS vars so accessing them might be expensive */
4396 p = (void**)TLAB_NEXT;
4397 /* FIXME: handle overflow */
4398 new_next = (char*)p + size;
4399 TLAB_NEXT = new_next;
4401 if (G_LIKELY (new_next < TLAB_TEMP_END)) {
4405 * FIXME: We might need a memory barrier here so the change to tlab_next is
4406 * visible before the vtable store.
4409 DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
4410 binary_protocol_alloc (p , vtable, size);
4411 g_assert (*p == NULL);
4414 g_assert (TLAB_NEXT == new_next);
4421 /* there are two cases: the object is too big or we run out of space in the TLAB */
4422 /* we also reach here when the thread does its first allocation after a minor
4423 * collection, since the tlab_ variables are initialized to NULL.
4424 * there can be another case (from ORP), if we cooperate with the runtime a bit:
4425 * objects that need finalizers can have the high bit set in their size
4426 * so the above check fails and we can readily add the object to the queue.
4427 * This avoids taking again the GC lock when registering, but this is moot when
4428 * doing thread-local allocation, so it may not be a good idea.
4430 g_assert (TLAB_NEXT == new_next);
4431 if (TLAB_NEXT >= TLAB_REAL_END) {
4433 * Run out of space in the TLAB. When this happens, some amount of space
4434 * remains in the TLAB, but not enough to satisfy the current allocation
4435 * request. Currently, we retire the TLAB in all cases, later we could
4436 * keep it if the remaining space is above a treshold, and satisfy the
4437 * allocation directly from the nursery.
4440 /* when running in degraded mode, we continue allocing that way
4441 * for a while, to decrease the number of useless nursery collections.
4443 if (degraded_mode && degraded_mode < DEFAULT_NURSERY_SIZE) {
4444 p = alloc_degraded (vtable, size);
4448 if (size > tlab_size) {
4449 /* Allocate directly from the nursery */
4450 if (nursery_next + size >= nursery_frag_real_end) {
4451 if (!search_fragment_for_size (size)) {
4452 minor_collect_or_expand_inner (size);
4453 if (degraded_mode) {
4454 p = alloc_degraded (vtable, size);
4460 p = (void*)nursery_next;
4461 nursery_next += size;
4462 if (nursery_next > nursery_frag_real_end) {
4467 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
4468 memset (p, 0, size);
4471 DEBUG (3, fprintf (gc_debug_file, "Retire TLAB: %p-%p [%ld]\n", TLAB_START, TLAB_REAL_END, (long)(TLAB_REAL_END - TLAB_NEXT - size)));
4473 if (nursery_next + tlab_size >= nursery_frag_real_end) {
4474 res = search_fragment_for_size (tlab_size);
4476 minor_collect_or_expand_inner (tlab_size);
4477 if (degraded_mode) {
4478 p = alloc_degraded (vtable, size);
4484 /* Allocate a new TLAB from the current nursery fragment */
4485 TLAB_START = nursery_next;
4486 nursery_next += tlab_size;
4487 TLAB_NEXT = TLAB_START;
4488 TLAB_REAL_END = TLAB_START + tlab_size;
4489 TLAB_TEMP_END = TLAB_START + MIN (SCAN_START_SIZE, tlab_size);
4491 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
4492 memset (TLAB_START, 0, tlab_size);
4494 /* Allocate from the TLAB */
4495 p = (void*)TLAB_NEXT;
4497 g_assert (TLAB_NEXT <= TLAB_REAL_END);
4499 nursery_section->scan_starts [((char*)p - (char*)nursery_section->data)/SCAN_START_SIZE] = (char*)p;
4502 /* Reached tlab_temp_end */
4504 /* record the scan start so we can find pinned objects more easily */
4505 nursery_section->scan_starts [((char*)p - (char*)nursery_section->data)/SCAN_START_SIZE] = (char*)p;
4506 /* we just bump tlab_temp_end as well */
4507 TLAB_TEMP_END = MIN (TLAB_REAL_END, TLAB_NEXT + SCAN_START_SIZE);
4508 DEBUG (5, fprintf (gc_debug_file, "Expanding local alloc: %p-%p\n", TLAB_NEXT, TLAB_TEMP_END));
4512 DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
4513 binary_protocol_alloc (p, vtable, size);
4520 mono_gc_try_alloc_obj_nolock (MonoVTable *vtable, size_t size)
4526 size += ALLOC_ALIGN - 1;
4527 size &= ~(ALLOC_ALIGN - 1);
4529 g_assert (vtable->gc_descr);
4530 if (size <= MAX_SMALL_OBJ_SIZE) {
4531 /* tlab_next and tlab_temp_end are TLS vars so accessing them might be expensive */
4533 p = (void**)TLAB_NEXT;
4534 /* FIXME: handle overflow */
4535 new_next = (char*)p + size;
4536 TLAB_NEXT = new_next;
4538 if (G_LIKELY (new_next < TLAB_TEMP_END)) {
4542 * FIXME: We might need a memory barrier here so the change to tlab_next is
4543 * visible before the vtable store.
4546 HEAVY_STAT (++stat_objects_alloced);
4548 DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
4549 binary_protocol_alloc (p, vtable, size);
4550 g_assert (*p == NULL);
4553 g_assert (TLAB_NEXT == new_next);
4562 mono_gc_alloc_obj (MonoVTable *vtable, size_t size)
4565 #ifndef DISABLE_CRITICAL_REGION
4567 ENTER_CRITICAL_REGION;
4568 res = mono_gc_try_alloc_obj_nolock (vtable, size);
4570 EXIT_CRITICAL_REGION;
4573 EXIT_CRITICAL_REGION;
4576 res = mono_gc_alloc_obj_nolock (vtable, size);
4582 mono_gc_alloc_vector (MonoVTable *vtable, size_t size, uintptr_t max_length)
4585 #ifndef DISABLE_CRITICAL_REGION
4587 ENTER_CRITICAL_REGION;
4588 arr = mono_gc_try_alloc_obj_nolock (vtable, size);
4590 arr->max_length = max_length;
4591 EXIT_CRITICAL_REGION;
4594 EXIT_CRITICAL_REGION;
4599 arr = mono_gc_alloc_obj_nolock (vtable, size);
4600 arr->max_length = max_length;
4608 mono_gc_alloc_array (MonoVTable *vtable, size_t size, uintptr_t max_length, uintptr_t bounds_size)
4611 MonoArrayBounds *bounds;
4615 arr = mono_gc_alloc_obj_nolock (vtable, size);
4616 arr->max_length = max_length;
4618 bounds = (MonoArrayBounds*)((char*)arr + size - bounds_size);
4619 arr->bounds = bounds;
4627 mono_gc_alloc_string (MonoVTable *vtable, size_t size, gint32 len)
4630 #ifndef DISABLE_CRITICAL_REGION
4632 ENTER_CRITICAL_REGION;
4633 str = mono_gc_try_alloc_obj_nolock (vtable, size);
4636 EXIT_CRITICAL_REGION;
4639 EXIT_CRITICAL_REGION;
4644 str = mono_gc_alloc_obj_nolock (vtable, size);
4653 * To be used for interned strings and possibly MonoThread, reflection handles.
4654 * We may want to explicitly free these objects.
4657 mono_gc_alloc_pinned_obj (MonoVTable *vtable, size_t size)
4659 /* FIXME: handle OOM */
4661 size += ALLOC_ALIGN - 1;
4662 size &= ~(ALLOC_ALIGN - 1);
4664 if (size > MAX_FREELIST_SIZE) {
4665 /* large objects are always pinned anyway */
4666 p = alloc_large_inner (vtable, size);
4668 p = alloc_from_freelist (size);
4669 memset (p, 0, size);
4671 DEBUG (6, fprintf (gc_debug_file, "Allocated pinned object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
4672 binary_protocol_alloc (p, vtable, size);
4679 * ######################################################################
4680 * ######## Finalization support
4681 * ######################################################################
4685 * this is valid for the nursery: if the object has been forwarded it means it's
4686 * still refrenced from a root. If it is pinned it's still alive as well.
4687 * Return TRUE if @obj is ready to be finalized.
4689 #define object_is_fin_ready(obj) (!object_is_pinned (obj) && !object_is_forwarded (obj))
4692 is_critical_finalizer (FinalizeEntry *entry)
4697 if (!mono_defaults.critical_finalizer_object)
4700 obj = entry->object;
4701 class = ((MonoVTable*)LOAD_VTABLE (obj))->klass;
4703 return mono_class_has_parent (class, mono_defaults.critical_finalizer_object);
4707 queue_finalization_entry (FinalizeEntry *entry) {
4708 if (is_critical_finalizer (entry)) {
4709 entry->next = critical_fin_list;
4710 critical_fin_list = entry;
4712 entry->next = fin_ready_list;
4713 fin_ready_list = entry;
4717 /* LOCKING: requires that the GC lock is held */
4719 rehash_fin_table (FinalizeEntryHashTable *hash_table)
4721 FinalizeEntry **finalizable_hash = hash_table->table;
4722 mword finalizable_hash_size = hash_table->size;
4725 FinalizeEntry **new_hash;
4726 FinalizeEntry *entry, *next;
4727 int new_size = g_spaced_primes_closest (hash_table->num_registered);
4729 new_hash = get_internal_mem (new_size * sizeof (FinalizeEntry*), INTERNAL_MEM_FIN_TABLE);
4730 for (i = 0; i < finalizable_hash_size; ++i) {
4731 for (entry = finalizable_hash [i]; entry; entry = next) {
4732 hash = mono_object_hash (entry->object) % new_size;
4734 entry->next = new_hash [hash];
4735 new_hash [hash] = entry;
4738 free_internal_mem (finalizable_hash, INTERNAL_MEM_FIN_TABLE);
4739 hash_table->table = new_hash;
4740 hash_table->size = new_size;
4743 /* LOCKING: requires that the GC lock is held */
4745 rehash_fin_table_if_necessary (FinalizeEntryHashTable *hash_table)
4747 if (hash_table->num_registered >= hash_table->size * 2)
4748 rehash_fin_table (hash_table);
4751 /* LOCKING: requires that the GC lock is held */
4753 finalize_in_range (char *start, char *end, int generation)
4755 FinalizeEntryHashTable *hash_table = get_finalize_entry_hash_table (generation);
4756 FinalizeEntry *entry, *prev;
4758 FinalizeEntry **finalizable_hash = hash_table->table;
4759 mword finalizable_hash_size = hash_table->size;
4763 for (i = 0; i < finalizable_hash_size; ++i) {
4765 for (entry = finalizable_hash [i]; entry;) {
4766 if ((char*)entry->object >= start && (char*)entry->object < end && !object_is_in_to_space (entry->object)) {
4767 gboolean is_fin_ready = object_is_fin_ready (entry->object);
4768 char *copy = copy_object (entry->object, start, end);
4771 FinalizeEntry *next;
4772 /* remove and put in fin_ready_list */
4774 prev->next = entry->next;
4776 finalizable_hash [i] = entry->next;
4778 num_ready_finalizers++;
4779 hash_table->num_registered--;
4780 queue_finalization_entry (entry);
4781 /* Make it survive */
4782 from = entry->object;
4783 entry->object = copy;
4784 DEBUG (5, fprintf (gc_debug_file, "Queueing object for finalization: %p (%s) (was at %p) (%d/%d)\n", entry->object, safe_name (entry->object), from, num_ready_finalizers, hash_table->num_registered));
4788 char *from = entry->object;
4789 if (hash_table == &minor_finalizable_hash && !ptr_in_nursery (copy)) {
4790 FinalizeEntry *next = entry->next;
4791 unsigned int major_hash;
4792 /* remove from the list */
4794 prev->next = entry->next;
4796 finalizable_hash [i] = entry->next;
4797 hash_table->num_registered--;
4799 entry->object = copy;
4801 /* insert it into the major hash */
4802 rehash_fin_table_if_necessary (&major_finalizable_hash);
4803 major_hash = mono_object_hash ((MonoObject*) copy) %
4804 major_finalizable_hash.size;
4805 entry->next = major_finalizable_hash.table [major_hash];
4806 major_finalizable_hash.table [major_hash] = entry;
4807 major_finalizable_hash.num_registered++;
4809 DEBUG (5, fprintf (gc_debug_file, "Promoting finalization of object %p (%s) (was at %p) to major table\n", copy, safe_name (copy), from));
4814 /* update pointer */
4815 DEBUG (5, fprintf (gc_debug_file, "Updating object for finalization: %p (%s) (was at %p)\n", entry->object, safe_name (entry->object), from));
4816 entry->object = copy;
4821 entry = entry->next;
4826 /* LOCKING: requires that the GC lock is held */
4828 null_link_in_range (char *start, char *end, int generation)
4830 DisappearingLinkHashTable *hash = get_dislink_hash_table (generation);
4831 DisappearingLink **disappearing_link_hash = hash->table;
4832 int disappearing_link_hash_size = hash->size;
4833 DisappearingLink *entry, *prev;
4835 if (!hash->num_links)
4837 for (i = 0; i < disappearing_link_hash_size; ++i) {
4839 for (entry = disappearing_link_hash [i]; entry;) {
4840 char *object = DISLINK_OBJECT (entry);
4841 if (object >= start && object < end && !object_is_in_to_space (object)) {
4842 gboolean track = DISLINK_TRACK (entry);
4843 if (!track && object_is_fin_ready (object)) {
4844 void **p = entry->link;
4845 DisappearingLink *old;
4847 /* remove from list */
4849 prev->next = entry->next;
4851 disappearing_link_hash [i] = entry->next;
4852 DEBUG (5, fprintf (gc_debug_file, "Dislink nullified at %p to GCed object %p\n", p, object));
4854 free_internal_mem (entry, INTERNAL_MEM_DISLINK);
4859 char *copy = copy_object (object, start, end);
4861 /* Update pointer if it's moved. If the object
4862 * has been moved out of the nursery, we need to
4863 * remove the link from the minor hash table to
4866 * FIXME: what if an object is moved earlier?
4869 if (hash == &minor_disappearing_link_hash && !ptr_in_nursery (copy)) {
4870 void **link = entry->link;
4871 DisappearingLink *old;
4872 /* remove from list */
4874 prev->next = entry->next;
4876 disappearing_link_hash [i] = entry->next;
4878 free_internal_mem (entry, INTERNAL_MEM_DISLINK);
4882 add_or_remove_disappearing_link ((MonoObject*)copy, link,
4883 track, GENERATION_OLD);
4885 DEBUG (5, fprintf (gc_debug_file, "Upgraded dislink at %p to major because object %p moved to %p\n", link, object, copy));
4889 /* We set the track resurrection bit to
4890 * FALSE if the object is to be finalized
4891 * so that the object can be collected in
4892 * the next cycle (i.e. after it was
4895 *entry->link = HIDE_POINTER (copy,
4896 object_is_fin_ready (object) ? FALSE : track);
4897 DEBUG (5, fprintf (gc_debug_file, "Updated dislink at %p to %p\n", entry->link, DISLINK_OBJECT (entry)));
4902 entry = entry->next;
4907 /* LOCKING: requires that the GC lock is held */
4909 null_links_for_domain (MonoDomain *domain, int generation)
4911 DisappearingLinkHashTable *hash = get_dislink_hash_table (generation);
4912 DisappearingLink **disappearing_link_hash = hash->table;
4913 int disappearing_link_hash_size = hash->size;
4914 DisappearingLink *entry, *prev;
4916 for (i = 0; i < disappearing_link_hash_size; ++i) {
4918 for (entry = disappearing_link_hash [i]; entry; ) {
4919 char *object = DISLINK_OBJECT (entry);
4920 /* FIXME: actually there should be no object
4921 left in the domain with a non-null vtable
4922 (provided we remove the Thread special
4924 if (object && (!((MonoObject*)object)->vtable || mono_object_domain (object) == domain)) {
4925 DisappearingLink *next = entry->next;
4930 disappearing_link_hash [i] = next;
4932 if (*(entry->link)) {
4933 *(entry->link) = NULL;
4934 g_warning ("Disappearing link %p not freed", entry->link);
4936 free_internal_mem (entry, INTERNAL_MEM_DISLINK);
4943 entry = entry->next;
4948 /* LOCKING: requires that the GC lock is held */
4950 finalizers_for_domain (MonoDomain *domain, MonoObject **out_array, int out_size,
4951 FinalizeEntryHashTable *hash_table)
4953 FinalizeEntry **finalizable_hash = hash_table->table;
4954 mword finalizable_hash_size = hash_table->size;
4955 FinalizeEntry *entry, *prev;
4958 if (no_finalize || !out_size || !out_array)
4961 for (i = 0; i < finalizable_hash_size; ++i) {
4963 for (entry = finalizable_hash [i]; entry;) {
4964 if (mono_object_domain (entry->object) == domain) {
4965 FinalizeEntry *next;
4966 /* remove and put in out_array */
4968 prev->next = entry->next;
4970 finalizable_hash [i] = entry->next;
4972 hash_table->num_registered--;
4973 out_array [count ++] = entry->object;
4974 DEBUG (5, fprintf (gc_debug_file, "Collecting object for finalization: %p (%s) (%d/%d)\n", entry->object, safe_name (entry->object), num_ready_finalizers, hash_table->num_registered));
4976 if (count == out_size)
4981 entry = entry->next;
4988 * mono_gc_finalizers_for_domain:
4989 * @domain: the unloading appdomain
4990 * @out_array: output array
4991 * @out_size: size of output array
4993 * Store inside @out_array up to @out_size objects that belong to the unloading
4994 * appdomain @domain. Returns the number of stored items. Can be called repeteadly
4995 * until it returns 0.
4996 * The items are removed from the finalizer data structure, so the caller is supposed
4998 * @out_array should be on the stack to allow the GC to know the objects are still alive.
5001 mono_gc_finalizers_for_domain (MonoDomain *domain, MonoObject **out_array, int out_size)
5006 result = finalizers_for_domain (domain, out_array, out_size, &minor_finalizable_hash);
5007 if (result < out_size) {
5008 result += finalizers_for_domain (domain, out_array + result, out_size - result,
5009 &major_finalizable_hash);
5017 register_for_finalization (MonoObject *obj, void *user_data, int generation)
5019 FinalizeEntryHashTable *hash_table = get_finalize_entry_hash_table (generation);
5020 FinalizeEntry **finalizable_hash;
5021 mword finalizable_hash_size;
5022 FinalizeEntry *entry, *prev;
5026 g_assert (user_data == NULL || user_data == mono_gc_run_finalize);
5027 hash = mono_object_hash (obj);
5029 rehash_fin_table_if_necessary (hash_table);
5030 finalizable_hash = hash_table->table;
5031 finalizable_hash_size = hash_table->size;
5032 hash %= finalizable_hash_size;
5034 for (entry = finalizable_hash [hash]; entry; entry = entry->next) {
5035 if (entry->object == obj) {
5037 /* remove from the list */
5039 prev->next = entry->next;
5041 finalizable_hash [hash] = entry->next;
5042 hash_table->num_registered--;
5043 DEBUG (5, fprintf (gc_debug_file, "Removed finalizer %p for object: %p (%s) (%d)\n", entry, obj, obj->vtable->klass->name, hash_table->num_registered));
5044 free_internal_mem (entry, INTERNAL_MEM_FINALIZE_ENTRY);
5052 /* request to deregister, but already out of the list */
5056 entry = get_internal_mem (sizeof (FinalizeEntry), INTERNAL_MEM_FINALIZE_ENTRY);
5057 entry->object = obj;
5058 entry->next = finalizable_hash [hash];
5059 finalizable_hash [hash] = entry;
5060 hash_table->num_registered++;
5061 DEBUG (5, fprintf (gc_debug_file, "Added finalizer %p for object: %p (%s) (%d) to %s table\n", entry, obj, obj->vtable->klass->name, hash_table->num_registered, generation_name (generation)));
5066 mono_gc_register_for_finalization (MonoObject *obj, void *user_data)
5068 if (ptr_in_nursery (obj))
5069 register_for_finalization (obj, user_data, GENERATION_NURSERY);
5071 register_for_finalization (obj, user_data, GENERATION_OLD);
5075 rehash_dislink (DisappearingLinkHashTable *hash_table)
5077 DisappearingLink **disappearing_link_hash = hash_table->table;
5078 int disappearing_link_hash_size = hash_table->size;
5081 DisappearingLink **new_hash;
5082 DisappearingLink *entry, *next;
5083 int new_size = g_spaced_primes_closest (hash_table->num_links);
5085 new_hash = get_internal_mem (new_size * sizeof (DisappearingLink*), INTERNAL_MEM_DISLINK_TABLE);
5086 for (i = 0; i < disappearing_link_hash_size; ++i) {
5087 for (entry = disappearing_link_hash [i]; entry; entry = next) {
5088 hash = mono_aligned_addr_hash (entry->link) % new_size;
5090 entry->next = new_hash [hash];
5091 new_hash [hash] = entry;
5094 free_internal_mem (disappearing_link_hash, INTERNAL_MEM_DISLINK_TABLE);
5095 hash_table->table = new_hash;
5096 hash_table->size = new_size;
5099 /* LOCKING: assumes the GC lock is held */
5101 add_or_remove_disappearing_link (MonoObject *obj, void **link, gboolean track, int generation)
5103 DisappearingLinkHashTable *hash_table = get_dislink_hash_table (generation);
5104 DisappearingLink *entry, *prev;
5106 DisappearingLink **disappearing_link_hash = hash_table->table;
5107 int disappearing_link_hash_size = hash_table->size;
5109 if (hash_table->num_links >= disappearing_link_hash_size * 2) {
5110 rehash_dislink (hash_table);
5111 disappearing_link_hash = hash_table->table;
5112 disappearing_link_hash_size = hash_table->size;
5114 /* FIXME: add check that link is not in the heap */
5115 hash = mono_aligned_addr_hash (link) % disappearing_link_hash_size;
5116 entry = disappearing_link_hash [hash];
5118 for (; entry; entry = entry->next) {
5119 /* link already added */
5120 if (link == entry->link) {
5121 /* NULL obj means remove */
5124 prev->next = entry->next;
5126 disappearing_link_hash [hash] = entry->next;
5127 hash_table->num_links--;
5128 DEBUG (5, fprintf (gc_debug_file, "Removed dislink %p (%d) from %s table\n", entry, hash_table->num_links, generation_name (generation)));
5129 free_internal_mem (entry, INTERNAL_MEM_DISLINK);
5132 *link = HIDE_POINTER (obj, track); /* we allow the change of object */
5140 entry = get_internal_mem (sizeof (DisappearingLink), INTERNAL_MEM_DISLINK);
5141 *link = HIDE_POINTER (obj, track);
5143 entry->next = disappearing_link_hash [hash];
5144 disappearing_link_hash [hash] = entry;
5145 hash_table->num_links++;
5146 DEBUG (5, fprintf (gc_debug_file, "Added dislink %p for object: %p (%s) at %p to %s table\n", entry, obj, obj->vtable->klass->name, link, generation_name (generation)));
5149 /* LOCKING: assumes the GC lock is held */
5151 mono_gc_register_disappearing_link (MonoObject *obj, void **link, gboolean track)
5153 add_or_remove_disappearing_link (NULL, link, FALSE, GENERATION_NURSERY);
5154 add_or_remove_disappearing_link (NULL, link, FALSE, GENERATION_OLD);
5156 if (ptr_in_nursery (obj))
5157 add_or_remove_disappearing_link (obj, link, track, GENERATION_NURSERY);
5159 add_or_remove_disappearing_link (obj, link, track, GENERATION_OLD);
5164 mono_gc_invoke_finalizers (void)
5166 FinalizeEntry *entry = NULL;
5167 gboolean entry_is_critical = FALSE;
5170 /* FIXME: batch to reduce lock contention */
5171 while (fin_ready_list || critical_fin_list) {
5175 FinalizeEntry **list = entry_is_critical ? &critical_fin_list : &fin_ready_list;
5177 /* We have finalized entry in the last
5178 interation, now we need to remove it from
5181 *list = entry->next;
5183 FinalizeEntry *e = *list;
5184 while (e->next != entry)
5186 e->next = entry->next;
5188 free_internal_mem (entry, INTERNAL_MEM_FINALIZE_ENTRY);
5192 /* Now look for the first non-null entry. */
5193 for (entry = fin_ready_list; entry && !entry->object; entry = entry->next)
5196 entry_is_critical = FALSE;
5198 entry_is_critical = TRUE;
5199 for (entry = critical_fin_list; entry && !entry->object; entry = entry->next)
5204 g_assert (entry->object);
5205 num_ready_finalizers--;
5206 obj = entry->object;
5207 entry->object = NULL;
5208 DEBUG (7, fprintf (gc_debug_file, "Finalizing object %p (%s)\n", obj, safe_name (obj)));
5216 g_assert (entry->object == NULL);
5218 /* the object is on the stack so it is pinned */
5219 /*g_print ("Calling finalizer for object: %p (%s)\n", entry->object, safe_name (entry->object));*/
5220 mono_gc_run_finalize (obj, NULL);
5227 mono_gc_pending_finalizers (void)
5229 return fin_ready_list || critical_fin_list;
5232 /* Negative value to remove */
5234 mono_gc_add_memory_pressure (gint64 value)
5236 /* FIXME: Use interlocked functions */
5238 memory_pressure += value;
5243 * ######################################################################
5244 * ######## registered roots support
5245 * ######################################################################
5249 rehash_roots (gboolean pinned)
5253 RootRecord **new_hash;
5254 RootRecord *entry, *next;
5257 new_size = g_spaced_primes_closest (num_roots_entries [pinned]);
5258 new_hash = get_internal_mem (new_size * sizeof (RootRecord*), INTERNAL_MEM_ROOTS_TABLE);
5259 for (i = 0; i < roots_hash_size [pinned]; ++i) {
5260 for (entry = roots_hash [pinned][i]; entry; entry = next) {
5261 hash = mono_aligned_addr_hash (entry->start_root) % new_size;
5263 entry->next = new_hash [hash];
5264 new_hash [hash] = entry;
5267 free_internal_mem (roots_hash [pinned], INTERNAL_MEM_ROOTS_TABLE);
5268 roots_hash [pinned] = new_hash;
5269 roots_hash_size [pinned] = new_size;
5273 find_root (int root_type, char *start, guint32 addr_hash)
5275 RootRecord *new_root;
5277 guint32 hash = addr_hash % roots_hash_size [root_type];
5278 for (new_root = roots_hash [root_type][hash]; new_root; new_root = new_root->next) {
5279 /* we allow changing the size and the descriptor (for thread statics etc) */
5280 if (new_root->start_root == start) {
5289 * We do not coalesce roots.
5292 mono_gc_register_root_inner (char *start, size_t size, void *descr, int root_type)
5294 RootRecord *new_root;
5295 unsigned int hash, addr_hash = mono_aligned_addr_hash (start);
5298 for (i = 0; i < ROOT_TYPE_NUM; ++i) {
5299 if (num_roots_entries [i] >= roots_hash_size [i] * 2)
5302 for (i = 0; i < ROOT_TYPE_NUM; ++i) {
5303 new_root = find_root (i, start, addr_hash);
5304 /* we allow changing the size and the descriptor (for thread statics etc) */
5306 size_t old_size = new_root->end_root - new_root->start_root;
5307 new_root->end_root = new_root->start_root + size;
5308 g_assert (((new_root->root_desc != 0) && (descr != NULL)) ||
5309 ((new_root->root_desc == 0) && (descr == NULL)));
5310 new_root->root_desc = (mword)descr;
5312 roots_size -= old_size;
5317 new_root = get_internal_mem (sizeof (RootRecord), INTERNAL_MEM_ROOT_RECORD);
5319 new_root->start_root = start;
5320 new_root->end_root = new_root->start_root + size;
5321 new_root->root_desc = (mword)descr;
5323 hash = addr_hash % roots_hash_size [root_type];
5324 num_roots_entries [root_type]++;
5325 new_root->next = roots_hash [root_type] [hash];
5326 roots_hash [root_type][hash] = new_root;
5327 DEBUG (3, fprintf (gc_debug_file, "Added root %p for range: %p-%p, descr: %p (%d/%d bytes)\n", new_root, new_root->start_root, new_root->end_root, descr, (int)size, (int)roots_size));
5337 mono_gc_register_root (char *start, size_t size, void *descr)
5339 return mono_gc_register_root_inner (start, size, descr, descr ? ROOT_TYPE_NORMAL : ROOT_TYPE_PINNED);
5343 mono_gc_register_root_wbarrier (char *start, size_t size, void *descr)
5345 return mono_gc_register_root_inner (start, size, descr, ROOT_TYPE_WBARRIER);
5349 mono_gc_deregister_root (char* addr)
5351 RootRecord *tmp, *prev;
5352 unsigned int hash, addr_hash = mono_aligned_addr_hash (addr);
5356 for (root_type = 0; root_type < ROOT_TYPE_NUM; ++root_type) {
5357 hash = addr_hash % roots_hash_size [root_type];
5358 tmp = roots_hash [root_type][hash];
5361 if (tmp->start_root == (char*)addr) {
5363 prev->next = tmp->next;
5365 roots_hash [root_type][hash] = tmp->next;
5366 roots_size -= (tmp->end_root - tmp->start_root);
5367 num_roots_entries [root_type]--;
5368 DEBUG (3, fprintf (gc_debug_file, "Removed root %p for range: %p-%p\n", tmp, tmp->start_root, tmp->end_root));
5369 free_internal_mem (tmp, INTERNAL_MEM_ROOT_RECORD);
5380 * ######################################################################
5381 * ######## Thread handling (stop/start code)
5382 * ######################################################################
5385 /* FIXME: handle large/small config */
5386 #define THREAD_HASH_SIZE 11
5387 #define HASH_PTHREAD_T(id) (((unsigned int)(id) >> 4) * 2654435761u)
5389 static SgenThreadInfo* thread_table [THREAD_HASH_SIZE];
5391 #if USE_SIGNAL_BASED_START_STOP_WORLD
5393 static MonoSemType suspend_ack_semaphore;
5394 static MonoSemType *suspend_ack_semaphore_ptr;
5395 static unsigned int global_stop_count = 0;
5397 static int suspend_signal_num = SIGXFSZ;
5399 static int suspend_signal_num = SIGPWR;
5401 static int restart_signal_num = SIGXCPU;
5402 static sigset_t suspend_signal_mask;
5403 static mword cur_thread_regs [ARCH_NUM_REGS] = {0};
5405 /* LOCKING: assumes the GC lock is held */
5406 static SgenThreadInfo*
5407 thread_info_lookup (ARCH_THREAD_TYPE id)
5409 unsigned int hash = HASH_PTHREAD_T (id) % THREAD_HASH_SIZE;
5410 SgenThreadInfo *info;
5412 info = thread_table [hash];
5413 while (info && !ARCH_THREAD_EQUALS (info->id, id)) {
5420 update_current_thread_stack (void *start)
5422 void *ptr = cur_thread_regs;
5423 SgenThreadInfo *info = thread_info_lookup (ARCH_GET_THREAD ());
5425 info->stack_start = align_pointer (&ptr);
5426 g_assert (info->stack_start >= info->stack_start_limit && info->stack_start < info->stack_end);
5427 ARCH_STORE_REGS (ptr);
5428 info->stopped_regs = ptr;
5429 if (gc_callbacks.thread_suspend_func)
5430 gc_callbacks.thread_suspend_func (info->runtime_data, NULL);
5434 signal_desc (int signum)
5436 if (signum == suspend_signal_num)
5438 if (signum == restart_signal_num)
5444 * Define this and use the "xdomain-checks" MONO_GC_DEBUG option to
5445 * have cross-domain checks in the write barrier.
5447 //#define XDOMAIN_CHECKS_IN_WBARRIER
5449 #ifndef HEAVY_STATISTICS
5450 #define MANAGED_ALLOCATION
5451 #ifndef XDOMAIN_CHECKS_IN_WBARRIER
5452 #define MANAGED_WBARRIER
5457 is_ip_in_managed_allocator (MonoDomain *domain, gpointer ip);
5460 wait_for_suspend_ack (int count)
5464 for (i = 0; i < count; ++i) {
5465 while ((result = MONO_SEM_WAIT (suspend_ack_semaphore_ptr)) != 0) {
5466 if (errno != EINTR) {
5467 g_error ("sem_wait ()");
5473 /* LOCKING: assumes the GC lock is held */
5475 thread_handshake (int signum)
5477 int count, i, result;
5478 SgenThreadInfo *info;
5479 pthread_t me = pthread_self ();
5482 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5483 for (info = thread_table [i]; info; info = info->next) {
5484 DEBUG (4, fprintf (gc_debug_file, "considering thread %p for signal %d (%s)\n", info, signum, signal_desc (signum)));
5485 if (ARCH_THREAD_EQUALS (info->id, me)) {
5486 DEBUG (4, fprintf (gc_debug_file, "Skip (equal): %p, %p\n", (void*)me, (void*)info->id));
5489 /*if (signum == suspend_signal_num && info->stop_count == global_stop_count)
5491 result = pthread_kill (info->id, signum);
5493 DEBUG (4, fprintf (gc_debug_file, "thread %p signal sent\n", info));
5496 DEBUG (4, fprintf (gc_debug_file, "thread %p signal failed: %d (%s)\n", (void*)info->id, result, strerror (result)));
5502 wait_for_suspend_ack (count);
5508 restart_threads_until_none_in_managed_allocator (void)
5510 SgenThreadInfo *info;
5511 int i, result, num_threads_died = 0;
5512 int sleep_duration = -1;
5515 int restart_count = 0, restarted_count = 0;
5516 /* restart all threads that stopped in the
5518 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5519 for (info = thread_table [i]; info; info = info->next) {
5522 if (!info->stack_start || info->in_critical_region ||
5523 is_ip_in_managed_allocator (info->stopped_domain, info->stopped_ip)) {
5524 binary_protocol_thread_restart ((gpointer)info->id);
5525 result = pthread_kill (info->id, restart_signal_num);
5532 /* we set the stopped_ip to
5533 NULL for threads which
5534 we're not restarting so
5535 that we can easily identify
5537 info->stopped_ip = NULL;
5538 info->stopped_domain = NULL;
5542 /* if no threads were restarted, we're done */
5543 if (restart_count == 0)
5546 /* wait for the threads to signal their restart */
5547 wait_for_suspend_ack (restart_count);
5549 if (sleep_duration < 0) {
5553 g_usleep (sleep_duration);
5554 sleep_duration += 10;
5557 /* stop them again */
5558 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5559 for (info = thread_table [i]; info; info = info->next) {
5560 if (info->skip || info->stopped_ip == NULL)
5562 result = pthread_kill (info->id, suspend_signal_num);
5570 /* some threads might have died */
5571 num_threads_died += restart_count - restarted_count;
5572 /* wait for the threads to signal their suspension
5574 wait_for_suspend_ack (restart_count);
5577 return num_threads_died;
5580 /* LOCKING: assumes the GC lock is held (by the stopping thread) */
5582 suspend_handler (int sig, siginfo_t *siginfo, void *context)
5584 SgenThreadInfo *info;
5587 int old_errno = errno;
5588 gpointer regs [ARCH_NUM_REGS];
5589 gpointer stack_start;
5591 id = pthread_self ();
5592 info = thread_info_lookup (id);
5593 info->stopped_domain = mono_domain_get ();
5594 info->stopped_ip = (gpointer) ARCH_SIGCTX_IP (context);
5595 stop_count = global_stop_count;
5596 /* duplicate signal */
5597 if (0 && info->stop_count == stop_count) {
5601 #ifdef HAVE_KW_THREAD
5602 /* update the remset info in the thread data structure */
5603 info->remset = remembered_set;
5605 stack_start = (char*) ARCH_SIGCTX_SP (context) - REDZONE_SIZE;
5606 /* If stack_start is not within the limits, then don't set it
5607 in info and we will be restarted. */
5608 if (stack_start >= info->stack_start_limit && info->stack_start <= info->stack_end) {
5609 info->stack_start = stack_start;
5611 ARCH_COPY_SIGCTX_REGS (regs, context);
5612 info->stopped_regs = regs;
5614 g_assert (!info->stack_start);
5617 /* Notify the JIT */
5618 if (gc_callbacks.thread_suspend_func)
5619 gc_callbacks.thread_suspend_func (info->runtime_data, context);
5621 DEBUG (4, fprintf (gc_debug_file, "Posting suspend_ack_semaphore for suspend from %p %p\n", info, (gpointer)ARCH_GET_THREAD ()));
5622 /* notify the waiting thread */
5623 MONO_SEM_POST (suspend_ack_semaphore_ptr);
5624 info->stop_count = stop_count;
5626 /* wait until we receive the restart signal */
5629 sigsuspend (&suspend_signal_mask);
5630 } while (info->signal != restart_signal_num);
5632 DEBUG (4, fprintf (gc_debug_file, "Posting suspend_ack_semaphore for resume from %p %p\n", info, (gpointer)ARCH_GET_THREAD ()));
5633 /* notify the waiting thread */
5634 MONO_SEM_POST (suspend_ack_semaphore_ptr);
5640 restart_handler (int sig)
5642 SgenThreadInfo *info;
5643 int old_errno = errno;
5645 info = thread_info_lookup (pthread_self ());
5646 info->signal = restart_signal_num;
5647 DEBUG (4, fprintf (gc_debug_file, "Restart handler in %p %p\n", info, (gpointer)ARCH_GET_THREAD ()));
5652 static TV_DECLARE (stop_world_time);
5653 static unsigned long max_pause_usec = 0;
5655 /* LOCKING: assumes the GC lock is held */
5661 update_current_thread_stack (&count);
5663 global_stop_count++;
5664 DEBUG (3, fprintf (gc_debug_file, "stopping world n %d from %p %p\n", global_stop_count, thread_info_lookup (ARCH_GET_THREAD ()), (gpointer)ARCH_GET_THREAD ()));
5665 TV_GETTIME (stop_world_time);
5666 count = thread_handshake (suspend_signal_num);
5667 count -= restart_threads_until_none_in_managed_allocator ();
5668 g_assert (count >= 0);
5669 DEBUG (3, fprintf (gc_debug_file, "world stopped %d thread(s)\n", count));
5673 /* LOCKING: assumes the GC lock is held */
5675 restart_world (void)
5678 SgenThreadInfo *info;
5679 TV_DECLARE (end_sw);
5682 /* notify the profiler of the leftovers */
5683 if (G_UNLIKELY (mono_profiler_events & MONO_PROFILE_GC_MOVES)) {
5684 if (moved_objects_idx) {
5685 mono_profiler_gc_moves (moved_objects, moved_objects_idx);
5686 moved_objects_idx = 0;
5689 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5690 for (info = thread_table [i]; info; info = info->next) {
5691 info->stack_start = NULL;
5692 info->stopped_regs = NULL;
5696 count = thread_handshake (restart_signal_num);
5697 TV_GETTIME (end_sw);
5698 usec = TV_ELAPSED (stop_world_time, end_sw);
5699 max_pause_usec = MAX (usec, max_pause_usec);
5700 DEBUG (2, fprintf (gc_debug_file, "restarted %d thread(s) (pause time: %d usec, max: %d)\n", count, (int)usec, (int)max_pause_usec));
5704 #endif /* USE_SIGNAL_BASED_START_STOP_WORLD */
5707 mono_gc_set_gc_callbacks (MonoGCCallbacks *callbacks)
5709 gc_callbacks = *callbacks;
5712 /* Variables holding start/end nursery so it won't have to be passed at every call */
5713 static void *scan_area_arg_start, *scan_area_arg_end;
5716 mono_gc_conservatively_scan_area (void *start, void *end)
5718 conservatively_pin_objects_from (start, end, scan_area_arg_start, scan_area_arg_end, PIN_TYPE_STACK);
5722 mono_gc_scan_object (void *obj)
5724 return copy_object (obj, scan_area_arg_start, scan_area_arg_end);
5728 * Mark from thread stacks and registers.
5731 scan_thread_data (void *start_nursery, void *end_nursery, gboolean precise)
5734 SgenThreadInfo *info;
5736 scan_area_arg_start = start_nursery;
5737 scan_area_arg_end = end_nursery;
5739 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5740 for (info = thread_table [i]; info; info = info->next) {
5742 DEBUG (2, fprintf (gc_debug_file, "Skipping dead thread %p, range: %p-%p, size: %zd\n", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start));
5745 DEBUG (3, fprintf (gc_debug_file, "Scanning thread %p, range: %p-%p, size: %zd, pinned=%d\n", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start, next_pin_slot));
5746 if (gc_callbacks.thread_mark_func && !conservative_stack_mark)
5747 gc_callbacks.thread_mark_func (info->runtime_data, info->stack_start, info->stack_end, precise);
5749 conservatively_pin_objects_from (info->stack_start, info->stack_end, start_nursery, end_nursery, PIN_TYPE_STACK);
5752 conservatively_pin_objects_from (info->stopped_regs, info->stopped_regs + ARCH_NUM_REGS,
5753 start_nursery, end_nursery, PIN_TYPE_STACK);
5759 find_pinning_ref_from_thread (char *obj, size_t size)
5762 SgenThreadInfo *info;
5763 char *endobj = obj + size;
5765 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5766 for (info = thread_table [i]; info; info = info->next) {
5767 char **start = (char**)info->stack_start;
5770 while (start < (char**)info->stack_end) {
5771 if (*start >= obj && *start < endobj) {
5772 DEBUG (0, fprintf (gc_debug_file, "Object %p referenced in thread %p (id %p) at %p, stack: %p-%p\n", obj, info, (gpointer)info->id, start, info->stack_start, info->stack_end));
5777 /* FIXME: check info->stopped_regs */
5783 ptr_on_stack (void *ptr)
5785 gpointer stack_start = &stack_start;
5786 SgenThreadInfo *info = thread_info_lookup (ARCH_GET_THREAD ());
5788 if (ptr >= stack_start && ptr < (gpointer)info->stack_end)
5794 handle_remset (mword *p, void *start_nursery, void *end_nursery, gboolean global)
5801 HEAVY_STAT (++stat_global_remsets_processed);
5803 /* FIXME: exclude stack locations */
5804 switch ((*p) & REMSET_TYPE_MASK) {
5805 case REMSET_LOCATION:
5807 //__builtin_prefetch (ptr);
5808 if (((void*)ptr < start_nursery || (void*)ptr >= end_nursery)) {
5809 gpointer old = *ptr;
5810 *ptr = copy_object (*ptr, start_nursery, end_nursery);
5811 DEBUG (9, fprintf (gc_debug_file, "Overwrote remset at %p with %p\n", ptr, *ptr));
5813 binary_protocol_ptr_update (ptr, old, *ptr, (gpointer)LOAD_VTABLE (*ptr), safe_object_get_size (*ptr));
5814 if (!global && *ptr >= start_nursery && *ptr < end_nursery) {
5816 * If the object is pinned, each reference to it from nonpinned objects
5817 * becomes part of the global remset, which can grow very large.
5819 DEBUG (9, fprintf (gc_debug_file, "Add to global remset because of pinning %p (%p %s)\n", ptr, *ptr, safe_name (*ptr)));
5820 add_to_global_remset (ptr, FALSE);
5823 DEBUG (9, fprintf (gc_debug_file, "Skipping remset at %p holding %p\n", ptr, *ptr));
5827 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
5828 if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery))
5831 while (count-- > 0) {
5832 *ptr = copy_object (*ptr, start_nursery, end_nursery);
5833 DEBUG (9, fprintf (gc_debug_file, "Overwrote remset at %p with %p (count: %d)\n", ptr, *ptr, (int)count));
5834 if (!global && *ptr >= start_nursery && *ptr < end_nursery)
5835 add_to_global_remset (ptr, FALSE);
5840 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
5841 if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery))
5843 scan_object ((char*)ptr, start_nursery, end_nursery);
5845 case REMSET_OTHER: {
5846 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
5850 if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery))
5855 ptr = (void**) scan_vtype ((char*)ptr, desc, start_nursery, end_nursery);
5857 case REMSET_ROOT_LOCATION:
5858 /* Same as REMSET_LOCATION, but the address is not required to be in the heap */
5859 *ptr = copy_object (*ptr, start_nursery, end_nursery);
5860 DEBUG (9, fprintf (gc_debug_file, "Overwrote root location remset at %p with %p\n", ptr, *ptr));
5861 if (!global && *ptr >= start_nursery && *ptr < end_nursery) {
5863 * If the object is pinned, each reference to it from nonpinned objects
5864 * becomes part of the global remset, which can grow very large.
5866 DEBUG (9, fprintf (gc_debug_file, "Add to global remset because of pinning %p (%p %s)\n", ptr, *ptr, safe_name (*ptr)));
5867 add_to_global_remset (ptr, TRUE);
5871 g_assert_not_reached ();
5876 g_assert_not_reached ();
5881 #ifdef HEAVY_STATISTICS
5883 collect_store_remsets (RememberedSet *remset, mword *bumper)
5885 mword *p = remset->data;
5890 while (p < remset->store_next) {
5891 switch ((*p) & REMSET_TYPE_MASK) {
5892 case REMSET_LOCATION:
5895 ++stat_saved_remsets_1;
5897 if (*p == last1 || *p == last2) {
5898 ++stat_saved_remsets_2;
5916 case REMSET_ROOT_LOCATION:
5920 g_assert_not_reached ();
5924 g_assert_not_reached ();
5934 RememberedSet *remset;
5936 SgenThreadInfo *info;
5938 mword *addresses, *bumper, *p, *r;
5940 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5941 for (info = thread_table [i]; info; info = info->next) {
5942 for (remset = info->remset; remset; remset = remset->next)
5943 size += remset->store_next - remset->data;
5946 for (remset = freed_thread_remsets; remset; remset = remset->next)
5947 size += remset->store_next - remset->data;
5948 for (remset = global_remset; remset; remset = remset->next)
5949 size += remset->store_next - remset->data;
5951 bumper = addresses = get_internal_mem (sizeof (mword) * size, INTERNAL_MEM_STATISTICS);
5953 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5954 for (info = thread_table [i]; info; info = info->next) {
5955 for (remset = info->remset; remset; remset = remset->next)
5956 bumper = collect_store_remsets (remset, bumper);
5959 for (remset = global_remset; remset; remset = remset->next)
5960 bumper = collect_store_remsets (remset, bumper);
5961 for (remset = freed_thread_remsets; remset; remset = remset->next)
5962 bumper = collect_store_remsets (remset, bumper);
5964 g_assert (bumper <= addresses + size);
5966 stat_store_remsets += bumper - addresses;
5968 sort_addresses ((void**)addresses, bumper - addresses);
5971 while (r < bumper) {
5977 stat_store_remsets_unique += p - addresses;
5979 free_internal_mem (addresses, INTERNAL_MEM_STATISTICS);
5984 clear_thread_store_remset_buffer (SgenThreadInfo *info)
5986 *info->store_remset_buffer_index_addr = 0;
5987 memset (*info->store_remset_buffer_addr, 0, sizeof (gpointer) * STORE_REMSET_BUFFER_SIZE);
5991 scan_from_remsets (void *start_nursery, void *end_nursery)
5994 SgenThreadInfo *info;
5995 RememberedSet *remset;
5996 GenericStoreRememberedSet *store_remset;
5997 mword *p, *next_p, *store_pos;
5999 #ifdef HEAVY_STATISTICS
6003 /* the global one */
6004 for (remset = global_remset; remset; remset = remset->next) {
6005 DEBUG (4, fprintf (gc_debug_file, "Scanning global remset range: %p-%p, size: %zd\n", remset->data, remset->store_next, remset->store_next - remset->data));
6006 store_pos = remset->data;
6007 for (p = remset->data; p < remset->store_next; p = next_p) {
6010 next_p = handle_remset (p, start_nursery, end_nursery, TRUE);
6013 * Clear global remsets of locations which no longer point to the
6014 * nursery. Otherwise, they could grow indefinitely between major
6017 ptr = (p [0] & ~REMSET_TYPE_MASK);
6018 if ((p [0] & REMSET_TYPE_MASK) == REMSET_LOCATION) {
6019 if (ptr_in_nursery (*(void**)ptr))
6020 *store_pos ++ = p [0];
6022 g_assert ((p [0] & REMSET_TYPE_MASK) == REMSET_OTHER);
6023 g_assert (p [1] == REMSET_ROOT_LOCATION);
6024 if (ptr_in_nursery (*(void**)ptr)) {
6025 *store_pos ++ = p [0];
6026 *store_pos ++ = p [1];
6031 /* Truncate the remset */
6032 remset->store_next = store_pos;
6035 /* the generic store ones */
6036 store_remset = generic_store_remsets;
6037 while (store_remset) {
6038 GenericStoreRememberedSet *next = store_remset->next;
6040 for (i = 0; i < STORE_REMSET_BUFFER_SIZE - 1; ++i) {
6041 gpointer addr = store_remset->data [i];
6043 handle_remset ((mword*)&addr, start_nursery, end_nursery, FALSE);
6046 free_internal_mem (store_remset, INTERNAL_MEM_STORE_REMSET);
6048 store_remset = next;
6050 generic_store_remsets = NULL;
6052 /* the per-thread ones */
6053 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
6054 for (info = thread_table [i]; info; info = info->next) {
6055 RememberedSet *next;
6057 for (remset = info->remset; remset; remset = next) {
6058 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for thread %p, range: %p-%p, size: %zd\n", info, remset->data, remset->store_next, remset->store_next - remset->data));
6059 for (p = remset->data; p < remset->store_next;) {
6060 p = handle_remset (p, start_nursery, end_nursery, FALSE);
6062 remset->store_next = remset->data;
6063 next = remset->next;
6064 remset->next = NULL;
6065 if (remset != info->remset) {
6066 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
6067 free_internal_mem (remset, INTERNAL_MEM_REMSET);
6070 for (j = 0; j < *info->store_remset_buffer_index_addr; ++j)
6071 handle_remset ((mword*)*info->store_remset_buffer_addr + j + 1, start_nursery, end_nursery, FALSE);
6072 clear_thread_store_remset_buffer (info);
6076 /* the freed thread ones */
6077 while (freed_thread_remsets) {
6078 RememberedSet *next;
6079 remset = freed_thread_remsets;
6080 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for freed thread, range: %p-%p, size: %zd\n", remset->data, remset->store_next, remset->store_next - remset->data));
6081 for (p = remset->data; p < remset->store_next;) {
6082 p = handle_remset (p, start_nursery, end_nursery, FALSE);
6084 next = remset->next;
6085 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
6086 free_internal_mem (remset, INTERNAL_MEM_REMSET);
6087 freed_thread_remsets = next;
6092 * Clear the info in the remembered sets: we're doing a major collection, so
6093 * the per-thread ones are not needed and the global ones will be reconstructed
6097 clear_remsets (void)
6100 SgenThreadInfo *info;
6101 RememberedSet *remset, *next;
6103 /* the global list */
6104 for (remset = global_remset; remset; remset = next) {
6105 remset->store_next = remset->data;
6106 next = remset->next;
6107 remset->next = NULL;
6108 if (remset != global_remset) {
6109 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
6110 free_internal_mem (remset, INTERNAL_MEM_REMSET);
6113 /* the generic store ones */
6114 while (generic_store_remsets) {
6115 GenericStoreRememberedSet *gs_next = generic_store_remsets->next;
6116 free_internal_mem (generic_store_remsets, INTERNAL_MEM_STORE_REMSET);
6117 generic_store_remsets = gs_next;
6119 /* the per-thread ones */
6120 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
6121 for (info = thread_table [i]; info; info = info->next) {
6122 for (remset = info->remset; remset; remset = next) {
6123 remset->store_next = remset->data;
6124 next = remset->next;
6125 remset->next = NULL;
6126 if (remset != info->remset) {
6127 DEBUG (1, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
6128 free_internal_mem (remset, INTERNAL_MEM_REMSET);
6131 clear_thread_store_remset_buffer (info);
6135 /* the freed thread ones */
6136 while (freed_thread_remsets) {
6137 next = freed_thread_remsets->next;
6138 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", freed_thread_remsets->data));
6139 free_internal_mem (freed_thread_remsets, INTERNAL_MEM_REMSET);
6140 freed_thread_remsets = next;
6145 * Clear the thread local TLAB variables for all threads.
6150 SgenThreadInfo *info;
6153 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
6154 for (info = thread_table [i]; info; info = info->next) {
6155 /* A new TLAB will be allocated when the thread does its first allocation */
6156 *info->tlab_start_addr = NULL;
6157 *info->tlab_next_addr = NULL;
6158 *info->tlab_temp_end_addr = NULL;
6159 *info->tlab_real_end_addr = NULL;
6164 /* LOCKING: assumes the GC lock is held */
6165 static SgenThreadInfo*
6166 gc_register_current_thread (void *addr)
6169 SgenThreadInfo* info = malloc (sizeof (SgenThreadInfo));
6170 #ifndef HAVE_KW_THREAD
6171 SgenThreadInfo *__thread_info__ = info;
6177 memset (info, 0, sizeof (SgenThreadInfo));
6178 #ifndef HAVE_KW_THREAD
6179 info->tlab_start = info->tlab_next = info->tlab_temp_end = info->tlab_real_end = NULL;
6181 g_assert (!pthread_getspecific (thread_info_key));
6182 pthread_setspecific (thread_info_key, info);
6187 info->id = ARCH_GET_THREAD ();
6188 info->stop_count = -1;
6191 info->stack_start = NULL;
6192 info->tlab_start_addr = &TLAB_START;
6193 info->tlab_next_addr = &TLAB_NEXT;
6194 info->tlab_temp_end_addr = &TLAB_TEMP_END;
6195 info->tlab_real_end_addr = &TLAB_REAL_END;
6196 info->store_remset_buffer_addr = &STORE_REMSET_BUFFER;
6197 info->store_remset_buffer_index_addr = &STORE_REMSET_BUFFER_INDEX;
6198 info->stopped_ip = NULL;
6199 info->stopped_domain = NULL;
6200 info->stopped_regs = NULL;
6202 binary_protocol_thread_register ((gpointer)info->id);
6204 #ifdef HAVE_KW_THREAD
6205 tlab_next_addr = &tlab_next;
6206 store_remset_buffer_index_addr = &store_remset_buffer_index;
6209 /* try to get it with attributes first */
6210 #if defined(HAVE_PTHREAD_GETATTR_NP) && defined(HAVE_PTHREAD_ATTR_GETSTACK)
6214 pthread_attr_t attr;
6215 pthread_getattr_np (pthread_self (), &attr);
6216 pthread_attr_getstack (&attr, &sstart, &size);
6217 info->stack_start_limit = sstart;
6218 info->stack_end = (char*)sstart + size;
6219 pthread_attr_destroy (&attr);
6221 #elif defined(HAVE_PTHREAD_GET_STACKSIZE_NP) && defined(HAVE_PTHREAD_GET_STACKADDR_NP)
6222 info->stack_end = (char*)pthread_get_stackaddr_np (pthread_self ());
6225 /* FIXME: we assume the stack grows down */
6226 gsize stack_bottom = (gsize)addr;
6227 stack_bottom += 4095;
6228 stack_bottom &= ~4095;
6229 info->stack_end = (char*)stack_bottom;
6233 #ifdef HAVE_KW_THREAD
6234 stack_end = info->stack_end;
6237 /* hash into the table */
6238 hash = HASH_PTHREAD_T (info->id) % THREAD_HASH_SIZE;
6239 info->next = thread_table [hash];
6240 thread_table [hash] = info;
6242 info->remset = alloc_remset (DEFAULT_REMSET_SIZE, info);
6243 pthread_setspecific (remembered_set_key, info->remset);
6244 #ifdef HAVE_KW_THREAD
6245 remembered_set = info->remset;
6248 STORE_REMSET_BUFFER = get_internal_mem (sizeof (gpointer) * STORE_REMSET_BUFFER_SIZE, INTERNAL_MEM_STORE_REMSET);
6249 STORE_REMSET_BUFFER_INDEX = 0;
6251 DEBUG (3, fprintf (gc_debug_file, "registered thread %p (%p) (hash: %d)\n", info, (gpointer)info->id, hash));
6253 if (gc_callbacks.thread_attach_func)
6254 info->runtime_data = gc_callbacks.thread_attach_func ();
6260 add_generic_store_remset_from_buffer (gpointer *buffer)
6262 GenericStoreRememberedSet *remset = get_internal_mem (sizeof (GenericStoreRememberedSet), INTERNAL_MEM_STORE_REMSET);
6263 memcpy (remset->data, buffer + 1, sizeof (gpointer) * (STORE_REMSET_BUFFER_SIZE - 1));
6264 remset->next = generic_store_remsets;
6265 generic_store_remsets = remset;
6269 unregister_current_thread (void)
6272 SgenThreadInfo *prev = NULL;
6274 RememberedSet *rset;
6275 ARCH_THREAD_TYPE id = ARCH_GET_THREAD ();
6277 binary_protocol_thread_unregister ((gpointer)id);
6279 hash = HASH_PTHREAD_T (id) % THREAD_HASH_SIZE;
6280 p = thread_table [hash];
6282 DEBUG (3, fprintf (gc_debug_file, "unregister thread %p (%p)\n", p, (gpointer)p->id));
6283 while (!ARCH_THREAD_EQUALS (p->id, id)) {
6288 thread_table [hash] = p->next;
6290 prev->next = p->next;
6293 if (freed_thread_remsets) {
6294 for (rset = p->remset; rset->next; rset = rset->next)
6296 rset->next = freed_thread_remsets;
6297 freed_thread_remsets = p->remset;
6299 freed_thread_remsets = p->remset;
6302 if (*p->store_remset_buffer_index_addr)
6303 add_generic_store_remset_from_buffer (*p->store_remset_buffer_addr);
6304 free_internal_mem (*p->store_remset_buffer_addr, INTERNAL_MEM_STORE_REMSET);
6309 unregister_thread (void *k)
6311 g_assert (!mono_domain_get ());
6313 unregister_current_thread ();
6318 mono_gc_register_thread (void *baseptr)
6320 SgenThreadInfo *info;
6324 info = thread_info_lookup (ARCH_GET_THREAD ());
6326 info = gc_register_current_thread (baseptr);
6328 return info != NULL;
6331 #if USE_PTHREAD_INTERCEPT
6333 #undef pthread_create
6335 #undef pthread_detach
6338 void *(*start_routine) (void *);
6341 MonoSemType registered;
6342 } SgenThreadStartInfo;
6345 gc_start_thread (void *arg)
6347 SgenThreadStartInfo *start_info = arg;
6348 SgenThreadInfo* info;
6349 void *t_arg = start_info->arg;
6350 void *(*start_func) (void*) = start_info->start_routine;
6355 info = gc_register_current_thread (&result);
6357 post_result = MONO_SEM_POST (&(start_info->registered));
6358 g_assert (!post_result);
6359 result = start_func (t_arg);
6360 g_assert (!mono_domain_get ());
6362 * this is done by the pthread key dtor
6364 unregister_current_thread ();
6372 mono_gc_pthread_create (pthread_t *new_thread, const pthread_attr_t *attr, void *(*start_routine)(void *), void *arg)
6374 SgenThreadStartInfo *start_info;
6377 start_info = malloc (sizeof (SgenThreadStartInfo));
6380 result = MONO_SEM_INIT (&(start_info->registered), 0);
6382 start_info->arg = arg;
6383 start_info->start_routine = start_routine;
6385 result = pthread_create (new_thread, attr, gc_start_thread, start_info);
6387 while (MONO_SEM_WAIT (&(start_info->registered)) != 0) {
6388 /*if (EINTR != errno) ABORT("sem_wait failed"); */
6391 MONO_SEM_DESTROY (&(start_info->registered));
6397 mono_gc_pthread_join (pthread_t thread, void **retval)
6399 return pthread_join (thread, retval);
6403 mono_gc_pthread_detach (pthread_t thread)
6405 return pthread_detach (thread);
6408 #endif /* USE_PTHREAD_INTERCEPT */
6411 * ######################################################################
6412 * ######## Write barriers
6413 * ######################################################################
6416 static RememberedSet*
6417 alloc_remset (int size, gpointer id) {
6418 RememberedSet* res = get_internal_mem (sizeof (RememberedSet) + (size * sizeof (gpointer)), INTERNAL_MEM_REMSET);
6419 res->store_next = res->data;
6420 res->end_set = res->data + size;
6422 DEBUG (4, fprintf (gc_debug_file, "Allocated remset size %d at %p for %p\n", size, res->data, id));
6427 * Note: the write barriers first do the needed GC work and then do the actual store:
6428 * this way the value is visible to the conservative GC scan after the write barrier
6429 * itself. If a GC interrupts the barrier in the middle, value will be kept alive by
6430 * the conservative scan, otherwise by the remembered set scan. FIXME: figure out what
6431 * happens when we need to record which pointers contain references to the new generation.
6432 * The write barrier will be executed, but the pointer is still not stored.
6435 mono_gc_wbarrier_set_field (MonoObject *obj, gpointer field_ptr, MonoObject* value)
6439 HEAVY_STAT (++stat_wbarrier_set_field);
6440 if (ptr_in_nursery (field_ptr)) {
6441 *(void**)field_ptr = value;
6444 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", field_ptr));
6446 rs = REMEMBERED_SET;
6447 if (rs->store_next < rs->end_set) {
6448 *(rs->store_next++) = (mword)field_ptr;
6449 *(void**)field_ptr = value;
6453 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6454 rs->next = REMEMBERED_SET;
6455 REMEMBERED_SET = rs;
6456 #ifdef HAVE_KW_THREAD
6457 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6459 *(rs->store_next++) = (mword)field_ptr;
6460 *(void**)field_ptr = value;
6465 mono_gc_wbarrier_set_arrayref (MonoArray *arr, gpointer slot_ptr, MonoObject* value)
6469 HEAVY_STAT (++stat_wbarrier_set_arrayref);
6470 if (ptr_in_nursery (slot_ptr)) {
6471 *(void**)slot_ptr = value;
6474 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", slot_ptr));
6476 rs = REMEMBERED_SET;
6477 if (rs->store_next < rs->end_set) {
6478 *(rs->store_next++) = (mword)slot_ptr;
6479 *(void**)slot_ptr = value;
6483 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6484 rs->next = REMEMBERED_SET;
6485 REMEMBERED_SET = rs;
6486 #ifdef HAVE_KW_THREAD
6487 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6489 *(rs->store_next++) = (mword)slot_ptr;
6490 *(void**)slot_ptr = value;
6495 mono_gc_wbarrier_arrayref_copy (gpointer dest_ptr, gpointer src_ptr, int count)
6499 HEAVY_STAT (++stat_wbarrier_arrayref_copy);
6501 memmove (dest_ptr, src_ptr, count * sizeof (gpointer));
6502 if (ptr_in_nursery (dest_ptr)) {
6506 rs = REMEMBERED_SET;
6507 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p, %d\n", dest_ptr, count));
6508 if (rs->store_next + 1 < rs->end_set) {
6509 *(rs->store_next++) = (mword)dest_ptr | REMSET_RANGE;
6510 *(rs->store_next++) = count;
6514 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6515 rs->next = REMEMBERED_SET;
6516 REMEMBERED_SET = rs;
6517 #ifdef HAVE_KW_THREAD
6518 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6520 *(rs->store_next++) = (mword)dest_ptr | REMSET_RANGE;
6521 *(rs->store_next++) = count;
6526 find_object_for_ptr_in_area (char *ptr, char *start, char *end)
6528 while (start < end) {
6531 if (!*(void**)start) {
6532 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
6538 #define SCAN_OBJECT_NOSCAN
6539 #include "sgen-scan-object.h"
6541 if (ptr >= old_start && ptr < start)
6548 static char *found_obj;
6551 find_object_for_ptr_in_pinned_chunk_callback (PinnedChunk *chunk, char *obj, size_t size, char *ptr)
6553 if (ptr >= obj && ptr < obj + size) {
6554 g_assert (!found_obj);
6559 /* for use in the debugger */
6560 char* find_object_for_ptr (char *ptr);
6562 find_object_for_ptr (char *ptr)
6564 GCMemSection *section;
6567 for (section = section_list; section; section = section->block.next) {
6568 if (ptr >= section->data && ptr < section->end_data)
6569 return find_object_for_ptr_in_area (ptr, section->data, section->end_data);
6572 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
6573 if (ptr >= bigobj->data && ptr < bigobj->data + bigobj->size)
6574 return bigobj->data;
6578 scan_pinned_objects ((ScanPinnedObjectCallbackFunc)find_object_for_ptr_in_pinned_chunk_callback, ptr);
6583 evacuate_remset_buffer (void)
6588 buffer = STORE_REMSET_BUFFER;
6590 add_generic_store_remset_from_buffer (buffer);
6591 memset (buffer, 0, sizeof (gpointer) * STORE_REMSET_BUFFER_SIZE);
6593 STORE_REMSET_BUFFER_INDEX = 0;
6597 mono_gc_wbarrier_generic_nostore (gpointer ptr)
6603 HEAVY_STAT (++stat_wbarrier_generic_store);
6605 #ifdef XDOMAIN_CHECKS_IN_WBARRIER
6606 /* FIXME: ptr_in_heap must be called with the GC lock held */
6607 if (xdomain_checks && *(MonoObject**)ptr && ptr_in_heap (ptr)) {
6608 char *start = find_object_for_ptr (ptr);
6609 MonoObject *value = *(MonoObject**)ptr;
6613 MonoObject *obj = (MonoObject*)start;
6614 if (obj->vtable->domain != value->vtable->domain)
6615 g_assert (is_xdomain_ref_allowed (ptr, start, obj->vtable->domain));
6623 if (*(gpointer*)ptr)
6624 binary_protocol_wbarrier (ptr, *(gpointer*)ptr, (gpointer)LOAD_VTABLE (*(gpointer*)ptr));
6626 if (ptr_in_nursery (ptr) || ptr_on_stack (ptr) || !ptr_in_nursery (*(gpointer*)ptr)) {
6627 DEBUG (8, fprintf (gc_debug_file, "Skipping remset at %p\n", ptr));
6632 buffer = STORE_REMSET_BUFFER;
6633 index = STORE_REMSET_BUFFER_INDEX;
6634 /* This simple optimization eliminates a sizable portion of
6635 entries. Comparing it to the last but one entry as well
6636 doesn't eliminate significantly more entries. */
6637 if (buffer [index] == ptr) {
6642 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", ptr));
6643 HEAVY_STAT (++stat_wbarrier_generic_store_remset);
6646 if (index >= STORE_REMSET_BUFFER_SIZE) {
6647 evacuate_remset_buffer ();
6648 index = STORE_REMSET_BUFFER_INDEX;
6649 g_assert (index == 0);
6652 buffer [index] = ptr;
6653 STORE_REMSET_BUFFER_INDEX = index;
6659 mono_gc_wbarrier_generic_store (gpointer ptr, MonoObject* value)
6661 DEBUG (8, fprintf (gc_debug_file, "Wbarrier store at %p to %p (%s)\n", ptr, value, value ? safe_name (value) : "null"));
6662 *(void**)ptr = value;
6663 if (ptr_in_nursery (value))
6664 mono_gc_wbarrier_generic_nostore (ptr);
6668 mono_gc_wbarrier_set_root (gpointer ptr, MonoObject *value)
6672 HEAVY_STAT (++stat_wbarrier_set_root);
6673 if (ptr_in_nursery (ptr))
6675 DEBUG (8, fprintf (gc_debug_file, "Adding root remset at %p (%s)\n", ptr, value ? safe_name (value) : "null"));
6677 rs = REMEMBERED_SET;
6678 if (rs->store_next + 2 < rs->end_set) {
6679 *(rs->store_next++) = (mword)ptr | REMSET_OTHER;
6680 *(rs->store_next++) = (mword)REMSET_ROOT_LOCATION;
6681 *(void**)ptr = value;
6684 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6685 rs->next = REMEMBERED_SET;
6686 REMEMBERED_SET = rs;
6687 #ifdef HAVE_KW_THREAD
6688 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6690 *(rs->store_next++) = (mword)ptr | REMSET_OTHER;
6691 *(rs->store_next++) = (mword)REMSET_ROOT_LOCATION;
6693 *(void**)ptr = value;
6697 mono_gc_wbarrier_value_copy (gpointer dest, gpointer src, int count, MonoClass *klass)
6701 HEAVY_STAT (++stat_wbarrier_value_copy);
6702 g_assert (klass->valuetype);
6704 memmove (dest, src, count * mono_class_value_size (klass, NULL));
6705 rs = REMEMBERED_SET;
6706 if (ptr_in_nursery (dest) || ptr_on_stack (dest)) {
6710 g_assert (klass->gc_descr_inited);
6711 DEBUG (8, fprintf (gc_debug_file, "Adding value remset at %p, count %d, descr %p for class %s (%p)\n", dest, count, klass->gc_descr, klass->name, klass));
6713 if (rs->store_next + 3 < rs->end_set) {
6714 *(rs->store_next++) = (mword)dest | REMSET_OTHER;
6715 *(rs->store_next++) = (mword)REMSET_VTYPE;
6716 *(rs->store_next++) = (mword)klass->gc_descr;
6717 *(rs->store_next++) = (mword)count;
6721 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6722 rs->next = REMEMBERED_SET;
6723 REMEMBERED_SET = rs;
6724 #ifdef HAVE_KW_THREAD
6725 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6727 *(rs->store_next++) = (mword)dest | REMSET_OTHER;
6728 *(rs->store_next++) = (mword)REMSET_VTYPE;
6729 *(rs->store_next++) = (mword)klass->gc_descr;
6730 *(rs->store_next++) = (mword)count;
6735 * mono_gc_wbarrier_object_copy:
6737 * Write barrier to call when obj is the result of a clone or copy of an object.
6740 mono_gc_wbarrier_object_copy (MonoObject* obj, MonoObject *src)
6746 HEAVY_STAT (++stat_wbarrier_object_copy);
6747 rs = REMEMBERED_SET;
6748 DEBUG (1, fprintf (gc_debug_file, "Adding object remset for %p\n", obj));
6749 size = mono_object_class (obj)->instance_size;
6751 /* do not copy the sync state */
6752 memcpy ((char*)obj + sizeof (MonoObject), (char*)src + sizeof (MonoObject),
6753 size - sizeof (MonoObject));
6754 if (ptr_in_nursery (obj) || ptr_on_stack (obj)) {
6758 if (rs->store_next < rs->end_set) {
6759 *(rs->store_next++) = (mword)obj | REMSET_OBJECT;
6763 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6764 rs->next = REMEMBERED_SET;
6765 REMEMBERED_SET = rs;
6766 #ifdef HAVE_KW_THREAD
6767 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6769 *(rs->store_next++) = (mword)obj | REMSET_OBJECT;
6774 * ######################################################################
6775 * ######## Collector debugging
6776 * ######################################################################
6779 const char*descriptor_types [] = {
6791 describe_ptr (char *ptr)
6793 GCMemSection *section;
6798 if (ptr_in_nursery (ptr)) {
6799 printf ("Pointer inside nursery.\n");
6801 for (section = section_list; section;) {
6802 if (ptr >= section->data && ptr < section->data + section->size)
6804 section = section->block.next;
6808 printf ("Pointer inside oldspace.\n");
6809 } else if (obj_is_from_pinned_alloc (ptr)) {
6810 printf ("Pointer is inside a pinned chunk.\n");
6812 printf ("Pointer unknown.\n");
6817 if (object_is_pinned (ptr))
6818 printf ("Object is pinned.\n");
6820 if (object_is_forwarded (ptr))
6821 printf ("Object is forwared.\n");
6823 // FIXME: Handle pointers to the inside of objects
6824 vtable = (MonoVTable*)LOAD_VTABLE (ptr);
6826 printf ("VTable: %p\n", vtable);
6827 if (vtable == NULL) {
6828 printf ("VTable is invalid (empty).\n");
6831 if (ptr_in_nursery (vtable)) {
6832 printf ("VTable is invalid (points inside nursery).\n");
6835 printf ("Class: %s\n", vtable->klass->name);
6837 desc = ((GCVTable*)vtable)->desc;
6838 printf ("Descriptor: %lx\n", (long)desc);
6841 printf ("Descriptor type: %d (%s)\n", type, descriptor_types [type]);
6845 find_in_remset_loc (mword *p, char *addr, gboolean *found)
6851 switch ((*p) & REMSET_TYPE_MASK) {
6852 case REMSET_LOCATION:
6853 if (*p == (mword)addr)
6857 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
6859 if ((void**)addr >= ptr && (void**)addr < ptr + count)
6863 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
6864 count = safe_object_get_size ((MonoObject*)ptr);
6865 count += (ALLOC_ALIGN - 1);
6866 count &= (ALLOC_ALIGN - 1);
6867 count /= sizeof (mword);
6868 if ((void**)addr >= ptr && (void**)addr < ptr + count)
6871 case REMSET_OTHER: {
6874 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
6878 switch (desc & 0x7) {
6879 case DESC_TYPE_RUN_LENGTH:
6880 OBJ_RUN_LEN_SIZE (skip_size, desc, ptr);
6882 case DESC_TYPE_SMALL_BITMAP:
6883 OBJ_BITMAP_SIZE (skip_size, desc, start);
6887 g_assert_not_reached ();
6890 /* The descriptor includes the size of MonoObject */
6891 skip_size -= sizeof (MonoObject);
6893 if ((void**)addr >= ptr && (void**)addr < ptr + (skip_size / sizeof (gpointer)))
6897 case REMSET_ROOT_LOCATION:
6900 g_assert_not_reached ();
6905 g_assert_not_reached ();
6911 * Return whenever ADDR occurs in the remembered sets
6914 find_in_remsets (char *addr)
6917 SgenThreadInfo *info;
6918 RememberedSet *remset;
6919 GenericStoreRememberedSet *store_remset;
6921 gboolean found = FALSE;
6923 /* the global one */
6924 for (remset = global_remset; remset; remset = remset->next) {
6925 DEBUG (4, fprintf (gc_debug_file, "Scanning global remset range: %p-%p, size: %zd\n", remset->data, remset->store_next, remset->store_next - remset->data));
6926 for (p = remset->data; p < remset->store_next;) {
6927 p = find_in_remset_loc (p, addr, &found);
6933 /* the generic store ones */
6934 for (store_remset = generic_store_remsets; store_remset; store_remset = store_remset->next) {
6935 for (i = 0; i < STORE_REMSET_BUFFER_SIZE - 1; ++i) {
6936 if (store_remset->data [i] == addr)
6941 /* the per-thread ones */
6942 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
6943 for (info = thread_table [i]; info; info = info->next) {
6945 for (remset = info->remset; remset; remset = remset->next) {
6946 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for thread %p, range: %p-%p, size: %zd\n", info, remset->data, remset->store_next, remset->store_next - remset->data));
6947 for (p = remset->data; p < remset->store_next;) {
6948 p = find_in_remset_loc (p, addr, &found);
6953 for (j = 0; j < *info->store_remset_buffer_index_addr; ++j) {
6954 if ((*info->store_remset_buffer_addr) [j + 1] == addr)
6960 /* the freed thread ones */
6961 for (remset = freed_thread_remsets; remset; remset = remset->next) {
6962 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for freed thread, range: %p-%p, size: %zd\n", remset->data, remset->store_next, remset->store_next - remset->data));
6963 for (p = remset->data; p < remset->store_next;) {
6964 p = find_in_remset_loc (p, addr, &found);
6973 static gboolean missing_remsets;
6976 * We let a missing remset slide if the target object is pinned,
6977 * because the store might have happened but the remset not yet added,
6978 * but in that case the target must be pinned. We might theoretically
6979 * miss some missing remsets this way, but it's very unlikely.
6982 #define HANDLE_PTR(ptr,obj) do { \
6983 if (*(ptr) && (char*)*(ptr) >= nursery_start && (char*)*(ptr) < nursery_next) { \
6984 if (!find_in_remsets ((char*)(ptr))) { \
6985 fprintf (gc_debug_file, "Oldspace->newspace reference %p at offset %zd in object %p (%s.%s) not found in remsets.\n", *(ptr), (char*)(ptr) - (char*)(obj), (obj), ((MonoObject*)(obj))->vtable->klass->name_space, ((MonoObject*)(obj))->vtable->klass->name); \
6986 binary_protocol_missing_remset ((obj), (gpointer)LOAD_VTABLE ((obj)), (char*)(ptr) - (char*)(obj), *(ptr), (gpointer)LOAD_VTABLE(*(ptr)), object_is_pinned (*(ptr))); \
6987 if (!object_is_pinned (*(ptr))) \
6988 missing_remsets = TRUE; \
6994 * Check that each object reference inside the area which points into the nursery
6995 * can be found in the remembered sets.
6997 static void __attribute__((noinline))
6998 check_remsets_for_area (char *start, char *end)
7001 int type_str = 0, type_rlen = 0, type_bitmap = 0, type_vector = 0, type_lbit = 0, type_complex = 0;
7002 while (start < end) {
7003 if (!*(void**)start) {
7004 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
7007 vt = (GCVTable*)LOAD_VTABLE (start);
7008 DEBUG (8, fprintf (gc_debug_file, "Scanning object %p, vtable: %p (%s)\n", start, vt, vt->klass->name));
7010 MonoObject *obj = (MonoObject*)start;
7011 g_print ("found at %p (0x%lx): %s.%s\n", start, (long)vt->desc, obj->vtable->klass->name_space, obj->vtable->klass->name);
7014 #define SCAN_OBJECT_ACTION COUNT_OBJECT_TYPES
7015 #include "sgen-scan-object.h"
7020 * Perform consistency check of the heap.
7022 * Assumes the world is stopped.
7025 check_consistency (void)
7027 GCMemSection *section;
7029 // Need to add more checks
7030 // FIXME: Create a general heap enumeration function and use that
7032 missing_remsets = FALSE;
7034 DEBUG (1, fprintf (gc_debug_file, "Begin heap consistency check...\n"));
7036 // Check that oldspace->newspace pointers are registered with the collector
7037 for (section = section_list; section; section = section->block.next) {
7038 if (section->block.role == MEMORY_ROLE_GEN0)
7040 DEBUG (2, fprintf (gc_debug_file, "Scan of old section: %p-%p, size: %d\n", section->data, section->next_data, (int)(section->next_data - section->data)));
7041 check_remsets_for_area (section->data, section->next_data);
7044 DEBUG (1, fprintf (gc_debug_file, "Heap consistency check done.\n"));
7046 #ifdef BINARY_PROTOCOL
7047 if (!binary_protocol_file)
7049 g_assert (!missing_remsets);
7052 /* Check that the reference is valid */
7054 #define HANDLE_PTR(ptr,obj) do { \
7056 g_assert (safe_name (*(ptr)) != NULL); \
7063 * Perform consistency check on an object. Currently we only check that the
7064 * reference fields are valid.
7067 check_object (char *start)
7072 #include "sgen-scan-object.h"
7078 * ######################################################################
7079 * ######## Other mono public interface functions.
7080 * ######################################################################
7084 mono_gc_collect (int generation)
7088 if (generation == 0) {
7089 collect_nursery (0);
7091 major_collection ("user request");
7098 mono_gc_max_generation (void)
7104 mono_gc_collection_count (int generation)
7106 if (generation == 0)
7107 return num_minor_gcs;
7108 return num_major_gcs;
7112 mono_gc_get_used_size (void)
7115 GCMemSection *section;
7117 tot = los_memory_usage;
7118 for (section = section_list; section; section = section->block.next) {
7119 /* this is approximate... */
7120 tot += section->next_data - section->data;
7122 /* FIXME: account for pinned objects */
7128 mono_gc_get_heap_size (void)
7134 mono_gc_disable (void)
7142 mono_gc_enable (void)
7150 mono_gc_get_los_limit (void)
7152 return MAX_SMALL_OBJ_SIZE;
7156 mono_object_is_alive (MonoObject* o)
7162 mono_gc_get_generation (MonoObject *obj)
7164 if (ptr_in_nursery (obj))
7170 mono_gc_enable_events (void)
7175 mono_gc_weak_link_add (void **link_addr, MonoObject *obj, gboolean track)
7178 mono_gc_register_disappearing_link (obj, link_addr, track);
7183 mono_gc_weak_link_remove (void **link_addr)
7186 mono_gc_register_disappearing_link (NULL, link_addr, FALSE);
7191 mono_gc_weak_link_get (void **link_addr)
7195 return (MonoObject*) REVEAL_POINTER (*link_addr);
7199 mono_gc_make_descr_from_bitmap (gsize *bitmap, int numbits)
7201 if (numbits < ((sizeof (*bitmap) * 8) - ROOT_DESC_TYPE_SHIFT)) {
7202 return (void*)MAKE_ROOT_DESC (ROOT_DESC_BITMAP, bitmap [0]);
7204 mword complex = alloc_complex_descriptor (bitmap, numbits);
7205 return (void*)MAKE_ROOT_DESC (ROOT_DESC_COMPLEX, complex);
7210 mono_gc_make_root_descr_user (MonoGCMarkFunc marker)
7214 g_assert (user_descriptors_next < MAX_USER_DESCRIPTORS);
7215 descr = (void*)MAKE_ROOT_DESC (ROOT_DESC_USER, (mword)user_descriptors_next);
7216 user_descriptors [user_descriptors_next ++] = marker;
7222 mono_gc_alloc_fixed (size_t size, void *descr)
7224 /* FIXME: do a single allocation */
7225 void *res = calloc (1, size);
7228 if (!mono_gc_register_root (res, size, descr)) {
7236 mono_gc_free_fixed (void* addr)
7238 mono_gc_deregister_root (addr);
7243 mono_gc_is_gc_thread (void)
7247 result = thread_info_lookup (ARCH_GET_THREAD ()) != NULL;
7253 mono_gc_base_init (void)
7257 struct sigaction sinfo;
7259 LOCK_INIT (gc_mutex);
7261 if (gc_initialized) {
7265 pagesize = mono_pagesize ();
7266 gc_debug_file = stderr;
7267 if ((env = getenv ("MONO_GC_DEBUG"))) {
7268 opts = g_strsplit (env, ",", -1);
7269 for (ptr = opts; ptr && *ptr; ptr ++) {
7271 if (opt [0] >= '0' && opt [0] <= '9') {
7272 gc_debug_level = atoi (opt);
7277 char *rf = g_strdup_printf ("%s.%d", opt, getpid ());
7278 gc_debug_file = fopen (rf, "wb");
7280 gc_debug_file = stderr;
7283 } else if (!strcmp (opt, "collect-before-allocs")) {
7284 collect_before_allocs = TRUE;
7285 } else if (!strcmp (opt, "check-at-minor-collections")) {
7286 consistency_check_at_minor_collection = TRUE;
7287 } else if (!strcmp (opt, "xdomain-checks")) {
7288 xdomain_checks = TRUE;
7289 } else if (!strcmp (opt, "clear-at-gc")) {
7290 nursery_clear_policy = CLEAR_AT_GC;
7291 } else if (!strcmp (opt, "conservative-stack-mark")) {
7292 conservative_stack_mark = TRUE;
7293 } else if (!strcmp (opt, "check-scan-starts")) {
7294 do_scan_starts_check = TRUE;
7295 } else if (g_str_has_prefix (opt, "heap-dump=")) {
7296 char *filename = strchr (opt, '=') + 1;
7297 nursery_clear_policy = CLEAR_AT_GC;
7298 heap_dump_file = fopen (filename, "w");
7300 fprintf (heap_dump_file, "<sgen-dump>\n");
7301 #ifdef BINARY_PROTOCOL
7302 } else if (g_str_has_prefix (opt, "binary-protocol=")) {
7303 char *filename = strchr (opt, '=') + 1;
7304 binary_protocol_file = fopen (filename, "w");
7307 fprintf (stderr, "Invalid format for the MONO_GC_DEBUG env variable: '%s'\n", env);
7308 fprintf (stderr, "The format is: MONO_GC_DEBUG=[l[:filename]|<option>]+ where l is a debug level 0-9.\n");
7309 fprintf (stderr, "Valid options are: collect-before-allocs, check-at-minor-collections, xdomain-checks, clear-at-gc.\n");
7316 suspend_ack_semaphore_ptr = &suspend_ack_semaphore;
7317 MONO_SEM_INIT (&suspend_ack_semaphore, 0);
7319 sigfillset (&sinfo.sa_mask);
7320 sinfo.sa_flags = SA_RESTART | SA_SIGINFO;
7321 sinfo.sa_sigaction = suspend_handler;
7322 if (sigaction (suspend_signal_num, &sinfo, NULL) != 0) {
7323 g_error ("failed sigaction");
7326 sinfo.sa_handler = restart_handler;
7327 if (sigaction (restart_signal_num, &sinfo, NULL) != 0) {
7328 g_error ("failed sigaction");
7331 sigfillset (&suspend_signal_mask);
7332 sigdelset (&suspend_signal_mask, restart_signal_num);
7334 global_remset = alloc_remset (1024, NULL);
7335 global_remset->next = NULL;
7337 pthread_key_create (&remembered_set_key, unregister_thread);
7339 #ifndef HAVE_KW_THREAD
7340 pthread_key_create (&thread_info_key, NULL);
7343 gc_initialized = TRUE;
7345 mono_gc_register_thread (&sinfo);
7349 mono_gc_get_suspend_signal (void)
7351 return suspend_signal_num;
7361 #ifdef HAVE_KW_THREAD
7362 #define EMIT_TLS_ACCESS(mb,dummy,offset) do { \
7363 mono_mb_emit_byte ((mb), MONO_CUSTOM_PREFIX); \
7364 mono_mb_emit_byte ((mb), CEE_MONO_TLS); \
7365 mono_mb_emit_i4 ((mb), (offset)); \
7368 #define EMIT_TLS_ACCESS(mb,member,dummy) do { \
7369 mono_mb_emit_byte ((mb), MONO_CUSTOM_PREFIX); \
7370 mono_mb_emit_byte ((mb), CEE_MONO_TLS); \
7371 mono_mb_emit_i4 ((mb), thread_info_key); \
7372 mono_mb_emit_icon ((mb), G_STRUCT_OFFSET (SgenThreadInfo, member)); \
7373 mono_mb_emit_byte ((mb), CEE_ADD); \
7374 mono_mb_emit_byte ((mb), CEE_LDIND_I); \
7378 #ifdef MANAGED_ALLOCATION
7379 /* FIXME: Do this in the JIT, where specialized allocation sequences can be created
7380 * for each class. This is currently not easy to do, as it is hard to generate basic
7381 * blocks + branches, but it is easy with the linear IL codebase.
7383 * For this to work we'd need to solve the TLAB race, first. Now we
7384 * require the allocator to be in a few known methods to make sure
7385 * that they are executed atomically via the restart mechanism.
7388 create_allocator (int atype)
7390 int p_var, size_var;
7391 guint32 slowpath_branch, max_size_branch;
7392 MonoMethodBuilder *mb;
7394 MonoMethodSignature *csig;
7395 static gboolean registered = FALSE;
7396 int tlab_next_addr_var, new_next_var;
7398 const char *name = NULL;
7399 AllocatorWrapperInfo *info;
7401 #ifdef HAVE_KW_THREAD
7402 int tlab_next_addr_offset = -1;
7403 int tlab_temp_end_offset = -1;
7405 MONO_THREAD_VAR_OFFSET (tlab_next_addr, tlab_next_addr_offset);
7406 MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
7408 g_assert (tlab_next_addr_offset != -1);
7409 g_assert (tlab_temp_end_offset != -1);
7413 mono_register_jit_icall (mono_gc_alloc_obj, "mono_gc_alloc_obj", mono_create_icall_signature ("object ptr int"), FALSE);
7414 mono_register_jit_icall (mono_gc_alloc_vector, "mono_gc_alloc_vector", mono_create_icall_signature ("object ptr int int"), FALSE);
7418 if (atype == ATYPE_SMALL) {
7420 name = "AllocSmall";
7421 } else if (atype == ATYPE_NORMAL) {
7424 } else if (atype == ATYPE_VECTOR) {
7426 name = "AllocVector";
7428 g_assert_not_reached ();
7431 csig = mono_metadata_signature_alloc (mono_defaults.corlib, num_params);
7432 csig->ret = &mono_defaults.object_class->byval_arg;
7433 for (i = 0; i < num_params; ++i)
7434 csig->params [i] = &mono_defaults.int_class->byval_arg;
7436 mb = mono_mb_new (mono_defaults.object_class, name, MONO_WRAPPER_ALLOC);
7437 size_var = mono_mb_add_local (mb, &mono_defaults.int32_class->byval_arg);
7438 if (atype == ATYPE_NORMAL || atype == ATYPE_SMALL) {
7439 /* size = vtable->klass->instance_size; */
7440 mono_mb_emit_ldarg (mb, 0);
7441 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoVTable, klass));
7442 mono_mb_emit_byte (mb, CEE_ADD);
7443 mono_mb_emit_byte (mb, CEE_LDIND_I);
7444 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoClass, instance_size));
7445 mono_mb_emit_byte (mb, CEE_ADD);
7446 /* FIXME: assert instance_size stays a 4 byte integer */
7447 mono_mb_emit_byte (mb, CEE_LDIND_U4);
7448 mono_mb_emit_stloc (mb, size_var);
7449 } else if (atype == ATYPE_VECTOR) {
7450 MonoExceptionClause *clause;
7452 MonoClass *oom_exc_class;
7455 /* n > MONO_ARRAY_MAX_INDEX -> OverflowException */
7456 mono_mb_emit_ldarg (mb, 1);
7457 mono_mb_emit_icon (mb, MONO_ARRAY_MAX_INDEX);
7458 pos = mono_mb_emit_short_branch (mb, CEE_BLE_UN_S);
7459 mono_mb_emit_exception (mb, "OverflowException", NULL);
7460 mono_mb_patch_short_branch (mb, pos);
7462 clause = mono_image_alloc0 (mono_defaults.corlib, sizeof (MonoExceptionClause));
7463 clause->try_offset = mono_mb_get_label (mb);
7465 /* vtable->klass->sizes.element_size */
7466 mono_mb_emit_ldarg (mb, 0);
7467 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoVTable, klass));
7468 mono_mb_emit_byte (mb, CEE_ADD);
7469 mono_mb_emit_byte (mb, CEE_LDIND_I);
7470 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoClass, sizes.element_size));
7471 mono_mb_emit_byte (mb, CEE_ADD);
7472 mono_mb_emit_byte (mb, CEE_LDIND_U4);
7475 mono_mb_emit_ldarg (mb, 1);
7476 mono_mb_emit_byte (mb, CEE_MUL_OVF_UN);
7477 /* + sizeof (MonoArray) */
7478 mono_mb_emit_icon (mb, sizeof (MonoArray));
7479 mono_mb_emit_byte (mb, CEE_ADD_OVF_UN);
7480 mono_mb_emit_stloc (mb, size_var);
7482 pos_leave = mono_mb_emit_branch (mb, CEE_LEAVE);
7485 clause->flags = MONO_EXCEPTION_CLAUSE_NONE;
7486 clause->try_len = mono_mb_get_pos (mb) - clause->try_offset;
7487 clause->data.catch_class = mono_class_from_name (mono_defaults.corlib,
7488 "System", "OverflowException");
7489 g_assert (clause->data.catch_class);
7490 clause->handler_offset = mono_mb_get_label (mb);
7492 oom_exc_class = mono_class_from_name (mono_defaults.corlib,
7493 "System", "OutOfMemoryException");
7494 g_assert (oom_exc_class);
7495 ctor = mono_class_get_method_from_name (oom_exc_class, ".ctor", 0);
7498 mono_mb_emit_byte (mb, CEE_POP);
7499 mono_mb_emit_op (mb, CEE_NEWOBJ, ctor);
7500 mono_mb_emit_byte (mb, CEE_THROW);
7502 clause->handler_len = mono_mb_get_pos (mb) - clause->handler_offset;
7503 mono_mb_set_clauses (mb, 1, clause);
7504 mono_mb_patch_branch (mb, pos_leave);
7507 g_assert_not_reached ();
7510 /* size += ALLOC_ALIGN - 1; */
7511 mono_mb_emit_ldloc (mb, size_var);
7512 mono_mb_emit_icon (mb, ALLOC_ALIGN - 1);
7513 mono_mb_emit_byte (mb, CEE_ADD);
7514 /* size &= ~(ALLOC_ALIGN - 1); */
7515 mono_mb_emit_icon (mb, ~(ALLOC_ALIGN - 1));
7516 mono_mb_emit_byte (mb, CEE_AND);
7517 mono_mb_emit_stloc (mb, size_var);
7519 /* if (size > MAX_SMALL_OBJ_SIZE) goto slowpath */
7520 if (atype != ATYPE_SMALL) {
7521 mono_mb_emit_ldloc (mb, size_var);
7522 mono_mb_emit_icon (mb, MAX_SMALL_OBJ_SIZE);
7523 max_size_branch = mono_mb_emit_short_branch (mb, MONO_CEE_BGT_S);
7527 * We need to modify tlab_next, but the JIT only supports reading, so we read
7528 * another tls var holding its address instead.
7531 /* tlab_next_addr (local) = tlab_next_addr (TLS var) */
7532 tlab_next_addr_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7533 EMIT_TLS_ACCESS (mb, tlab_next_addr, tlab_next_addr_offset);
7534 mono_mb_emit_stloc (mb, tlab_next_addr_var);
7536 /* p = (void**)tlab_next; */
7537 p_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7538 mono_mb_emit_ldloc (mb, tlab_next_addr_var);
7539 mono_mb_emit_byte (mb, CEE_LDIND_I);
7540 mono_mb_emit_stloc (mb, p_var);
7542 /* new_next = (char*)p + size; */
7543 new_next_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7544 mono_mb_emit_ldloc (mb, p_var);
7545 mono_mb_emit_ldloc (mb, size_var);
7546 mono_mb_emit_byte (mb, CEE_CONV_I);
7547 mono_mb_emit_byte (mb, CEE_ADD);
7548 mono_mb_emit_stloc (mb, new_next_var);
7550 /* tlab_next = new_next */
7551 mono_mb_emit_ldloc (mb, tlab_next_addr_var);
7552 mono_mb_emit_ldloc (mb, new_next_var);
7553 mono_mb_emit_byte (mb, CEE_STIND_I);
7555 /* if (G_LIKELY (new_next < tlab_temp_end)) */
7556 mono_mb_emit_ldloc (mb, new_next_var);
7557 EMIT_TLS_ACCESS (mb, tlab_temp_end, tlab_temp_end_offset);
7558 slowpath_branch = mono_mb_emit_short_branch (mb, MONO_CEE_BLT_UN_S);
7561 if (atype != ATYPE_SMALL)
7562 mono_mb_patch_short_branch (mb, max_size_branch);
7564 mono_mb_emit_byte (mb, MONO_CUSTOM_PREFIX);
7565 mono_mb_emit_byte (mb, CEE_MONO_NOT_TAKEN);
7567 /* FIXME: mono_gc_alloc_obj takes a 'size_t' as an argument, not an int32 */
7568 mono_mb_emit_ldarg (mb, 0);
7569 mono_mb_emit_ldloc (mb, size_var);
7570 if (atype == ATYPE_NORMAL || atype == ATYPE_SMALL) {
7571 mono_mb_emit_icall (mb, mono_gc_alloc_obj);
7572 } else if (atype == ATYPE_VECTOR) {
7573 mono_mb_emit_ldarg (mb, 1);
7574 mono_mb_emit_icall (mb, mono_gc_alloc_vector);
7576 g_assert_not_reached ();
7578 mono_mb_emit_byte (mb, CEE_RET);
7581 mono_mb_patch_short_branch (mb, slowpath_branch);
7583 /* FIXME: Memory barrier */
7586 mono_mb_emit_ldloc (mb, p_var);
7587 mono_mb_emit_ldarg (mb, 0);
7588 mono_mb_emit_byte (mb, CEE_STIND_I);
7590 if (atype == ATYPE_VECTOR) {
7591 /* arr->max_length = max_length; */
7592 mono_mb_emit_ldloc (mb, p_var);
7593 mono_mb_emit_ldflda (mb, G_STRUCT_OFFSET (MonoArray, max_length));
7594 mono_mb_emit_ldarg (mb, 1);
7595 mono_mb_emit_byte (mb, CEE_STIND_I);
7599 mono_mb_emit_ldloc (mb, p_var);
7600 mono_mb_emit_byte (mb, CEE_RET);
7602 res = mono_mb_create_method (mb, csig, 8);
7604 mono_method_get_header (res)->init_locals = FALSE;
7606 info = mono_image_alloc0 (mono_defaults.corlib, sizeof (AllocatorWrapperInfo));
7607 info->alloc_type = atype;
7608 mono_marshal_set_wrapper_info (res, info);
7614 static MonoMethod* alloc_method_cache [ATYPE_NUM];
7615 static MonoMethod *write_barrier_method;
7618 is_ip_in_managed_allocator (MonoDomain *domain, gpointer ip)
7626 ji = mono_jit_info_table_find (domain, ip);
7629 method = ji->method;
7631 if (method == write_barrier_method)
7633 for (i = 0; i < ATYPE_NUM; ++i)
7634 if (method == alloc_method_cache [i])
7640 * Generate an allocator method implementing the fast path of mono_gc_alloc_obj ().
7641 * The signature of the called method is:
7642 * object allocate (MonoVTable *vtable)
7645 mono_gc_get_managed_allocator (MonoVTable *vtable, gboolean for_box)
7647 #ifdef MANAGED_ALLOCATION
7648 MonoClass *klass = vtable->klass;
7650 #ifdef HAVE_KW_THREAD
7651 int tlab_next_offset = -1;
7652 int tlab_temp_end_offset = -1;
7653 MONO_THREAD_VAR_OFFSET (tlab_next, tlab_next_offset);
7654 MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
7656 if (tlab_next_offset == -1 || tlab_temp_end_offset == -1)
7660 if (!mono_runtime_has_tls_get ())
7662 if (klass->instance_size > tlab_size)
7664 if (klass->has_finalize || klass->marshalbyref || (mono_profiler_get_events () & MONO_PROFILE_ALLOCATIONS))
7668 if (klass->byval_arg.type == MONO_TYPE_STRING)
7670 if (collect_before_allocs)
7673 if (ALIGN_TO (klass->instance_size, ALLOC_ALIGN) < MAX_SMALL_OBJ_SIZE)
7674 return mono_gc_get_managed_allocator_by_type (ATYPE_SMALL);
7676 return mono_gc_get_managed_allocator_by_type (ATYPE_NORMAL);
7683 mono_gc_get_managed_array_allocator (MonoVTable *vtable, int rank)
7685 #ifdef MANAGED_ALLOCATION
7686 MonoClass *klass = vtable->klass;
7688 #ifdef HAVE_KW_THREAD
7689 int tlab_next_offset = -1;
7690 int tlab_temp_end_offset = -1;
7691 MONO_THREAD_VAR_OFFSET (tlab_next, tlab_next_offset);
7692 MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
7694 if (tlab_next_offset == -1 || tlab_temp_end_offset == -1)
7700 if (!mono_runtime_has_tls_get ())
7702 if (mono_profiler_get_events () & MONO_PROFILE_ALLOCATIONS)
7704 if (collect_before_allocs)
7706 g_assert (!klass->has_finalize && !klass->marshalbyref);
7708 return mono_gc_get_managed_allocator_by_type (ATYPE_VECTOR);
7715 mono_gc_get_managed_allocator_by_type (int atype)
7717 #ifdef MANAGED_ALLOCATION
7720 if (!mono_runtime_has_tls_get ())
7723 mono_loader_lock ();
7724 res = alloc_method_cache [atype];
7726 res = alloc_method_cache [atype] = create_allocator (atype);
7727 mono_loader_unlock ();
7735 mono_gc_get_managed_allocator_types (void)
7742 mono_gc_get_write_barrier (void)
7745 MonoMethodBuilder *mb;
7746 MonoMethodSignature *sig;
7747 #ifdef MANAGED_WBARRIER
7748 int label_no_wb_1, label_no_wb_2, label_no_wb_3, label_no_wb_4, label_need_wb, label_slow_path;
7749 int buffer_var, buffer_index_var, dummy_var;
7751 #ifdef HAVE_KW_THREAD
7752 int stack_end_offset = -1, store_remset_buffer_offset = -1;
7753 int store_remset_buffer_index_offset = -1, store_remset_buffer_index_addr_offset = -1;
7755 MONO_THREAD_VAR_OFFSET (stack_end, stack_end_offset);
7756 g_assert (stack_end_offset != -1);
7757 MONO_THREAD_VAR_OFFSET (store_remset_buffer, store_remset_buffer_offset);
7758 g_assert (store_remset_buffer_offset != -1);
7759 MONO_THREAD_VAR_OFFSET (store_remset_buffer_index, store_remset_buffer_index_offset);
7760 g_assert (store_remset_buffer_index_offset != -1);
7761 MONO_THREAD_VAR_OFFSET (store_remset_buffer_index_addr, store_remset_buffer_index_addr_offset);
7762 g_assert (store_remset_buffer_index_addr_offset != -1);
7766 // FIXME: Maybe create a separate version for ctors (the branch would be
7767 // correctly predicted more times)
7768 if (write_barrier_method)
7769 return write_barrier_method;
7771 /* Create the IL version of mono_gc_barrier_generic_store () */
7772 sig = mono_metadata_signature_alloc (mono_defaults.corlib, 1);
7773 sig->ret = &mono_defaults.void_class->byval_arg;
7774 sig->params [0] = &mono_defaults.int_class->byval_arg;
7776 mb = mono_mb_new (mono_defaults.object_class, "wbarrier", MONO_WRAPPER_WRITE_BARRIER);
7778 #ifdef MANAGED_WBARRIER
7779 if (mono_runtime_has_tls_get ()) {
7780 #ifdef ALIGN_NURSERY
7781 // if (ptr_in_nursery (ptr)) return;
7783 * Masking out the bits might be faster, but we would have to use 64 bit
7784 * immediates, which might be slower.
7786 mono_mb_emit_ldarg (mb, 0);
7787 mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
7788 mono_mb_emit_byte (mb, CEE_SHR_UN);
7789 mono_mb_emit_icon (mb, (mword)nursery_start >> DEFAULT_NURSERY_BITS);
7790 label_no_wb_1 = mono_mb_emit_branch (mb, CEE_BEQ);
7792 // if (!ptr_in_nursery (*ptr)) return;
7793 mono_mb_emit_ldarg (mb, 0);
7794 mono_mb_emit_byte (mb, CEE_LDIND_I);
7795 mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
7796 mono_mb_emit_byte (mb, CEE_SHR_UN);
7797 mono_mb_emit_icon (mb, (mword)nursery_start >> DEFAULT_NURSERY_BITS);
7798 label_no_wb_2 = mono_mb_emit_branch (mb, CEE_BNE_UN);
7801 g_assert_not_reached ();
7804 // if (ptr >= stack_end) goto need_wb;
7805 mono_mb_emit_ldarg (mb, 0);
7806 EMIT_TLS_ACCESS (mb, stack_end, stack_end_offset);
7807 label_need_wb = mono_mb_emit_branch (mb, CEE_BGE_UN);
7809 // if (ptr >= stack_start) return;
7810 dummy_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7811 mono_mb_emit_ldarg (mb, 0);
7812 mono_mb_emit_ldloc_addr (mb, dummy_var);
7813 label_no_wb_3 = mono_mb_emit_branch (mb, CEE_BGE_UN);
7816 mono_mb_patch_branch (mb, label_need_wb);
7818 // buffer = STORE_REMSET_BUFFER;
7819 buffer_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7820 EMIT_TLS_ACCESS (mb, store_remset_buffer, store_remset_buffer_offset);
7821 mono_mb_emit_stloc (mb, buffer_var);
7823 // buffer_index = STORE_REMSET_BUFFER_INDEX;
7824 buffer_index_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7825 EMIT_TLS_ACCESS (mb, store_remset_buffer_index, store_remset_buffer_index_offset);
7826 mono_mb_emit_stloc (mb, buffer_index_var);
7828 // if (buffer [buffer_index] == ptr) return;
7829 mono_mb_emit_ldloc (mb, buffer_var);
7830 mono_mb_emit_ldloc (mb, buffer_index_var);
7831 g_assert (sizeof (gpointer) == 4 || sizeof (gpointer) == 8);
7832 mono_mb_emit_icon (mb, sizeof (gpointer) == 4 ? 2 : 3);
7833 mono_mb_emit_byte (mb, CEE_SHL);
7834 mono_mb_emit_byte (mb, CEE_ADD);
7835 mono_mb_emit_byte (mb, CEE_LDIND_I);
7836 mono_mb_emit_ldarg (mb, 0);
7837 label_no_wb_4 = mono_mb_emit_branch (mb, CEE_BEQ);
7840 mono_mb_emit_ldloc (mb, buffer_index_var);
7841 mono_mb_emit_icon (mb, 1);
7842 mono_mb_emit_byte (mb, CEE_ADD);
7843 mono_mb_emit_stloc (mb, buffer_index_var);
7845 // if (buffer_index >= STORE_REMSET_BUFFER_SIZE) goto slow_path;
7846 mono_mb_emit_ldloc (mb, buffer_index_var);
7847 mono_mb_emit_icon (mb, STORE_REMSET_BUFFER_SIZE);
7848 label_slow_path = mono_mb_emit_branch (mb, CEE_BGE);
7850 // buffer [buffer_index] = ptr;
7851 mono_mb_emit_ldloc (mb, buffer_var);
7852 mono_mb_emit_ldloc (mb, buffer_index_var);
7853 g_assert (sizeof (gpointer) == 4 || sizeof (gpointer) == 8);
7854 mono_mb_emit_icon (mb, sizeof (gpointer) == 4 ? 2 : 3);
7855 mono_mb_emit_byte (mb, CEE_SHL);
7856 mono_mb_emit_byte (mb, CEE_ADD);
7857 mono_mb_emit_ldarg (mb, 0);
7858 mono_mb_emit_byte (mb, CEE_STIND_I);
7860 // STORE_REMSET_BUFFER_INDEX = buffer_index;
7861 EMIT_TLS_ACCESS (mb, store_remset_buffer_index_addr, store_remset_buffer_index_addr_offset);
7862 mono_mb_emit_ldloc (mb, buffer_index_var);
7863 mono_mb_emit_byte (mb, CEE_STIND_I);
7866 mono_mb_patch_branch (mb, label_no_wb_1);
7867 mono_mb_patch_branch (mb, label_no_wb_2);
7868 mono_mb_patch_branch (mb, label_no_wb_3);
7869 mono_mb_patch_branch (mb, label_no_wb_4);
7870 mono_mb_emit_byte (mb, CEE_RET);
7873 mono_mb_patch_branch (mb, label_slow_path);
7877 mono_mb_emit_ldarg (mb, 0);
7878 mono_mb_emit_icall (mb, mono_gc_wbarrier_generic_nostore);
7879 mono_mb_emit_byte (mb, CEE_RET);
7881 res = mono_mb_create_method (mb, sig, 16);
7884 mono_loader_lock ();
7885 if (write_barrier_method) {
7886 /* Already created */
7887 mono_free_method (res);
7889 /* double-checked locking */
7890 mono_memory_barrier ();
7891 write_barrier_method = res;
7893 mono_loader_unlock ();
7895 return write_barrier_method;
7898 #endif /* HAVE_SGEN_GC */