2 * sgen-gc.c: Simple generational GC.
5 * Paolo Molaro (lupus@ximian.com)
7 * Copyright 2005-2009 Novell, Inc (http://www.novell.com)
9 * Thread start/stop adapted from Boehm's GC:
10 * Copyright (c) 1994 by Xerox Corporation. All rights reserved.
11 * Copyright (c) 1996 by Silicon Graphics. All rights reserved.
12 * Copyright (c) 1998 by Fergus Henderson. All rights reserved.
13 * Copyright (c) 2000-2004 by Hewlett-Packard Company. All rights reserved.
15 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
16 * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
18 * Permission is hereby granted to use or copy this program
19 * for any purpose, provided the above notices are retained on all copies.
20 * Permission to modify the code and to distribute modified code is granted,
21 * provided the above notices are retained, and a notice that the code was
22 * modified is included with the above copyright notice.
24 * All the rest of the code is LGPL.
26 * Important: allocation provides always zeroed memory, having to do
27 * a memset after allocation is deadly for performance.
28 * Memory usage at startup is currently as follows:
30 * 64 KB internal space
32 * We should provide a small memory config with half the sizes
34 * We currently try to make as few mono assumptions as possible:
35 * 1) 2-word header with no GC pointers in it (first vtable, second to store the
37 * 2) gc descriptor is the second word in the vtable (first word in the class)
38 * 3) 8 byte alignment is the minimum and enough (not true for special structures, FIXME)
39 * 4) there is a function to get an object's size and the number of
40 * elements in an array.
41 * 5) we know the special way bounds are allocated for complex arrays
43 * Always try to keep stack usage to a minimum: no recursive behaviour
44 * and no large stack allocs.
46 * General description.
47 * Objects are initially allocated in a nursery using a fast bump-pointer technique.
48 * When the nursery is full we start a nursery collection: this is performed with a
50 * When the old generation is full we start a copying GC of the old generation as well:
51 * this will be changed to mark/compact in the future.
52 * The things that complicate this description are:
53 * *) pinned objects: we can't move them so we need to keep track of them
54 * *) no precise info of the thread stacks and registers: we need to be able to
55 * quickly find the objects that may be referenced conservatively and pin them
56 * (this makes the first issues more important)
57 * *) large objects are too expensive to be dealt with using copying GC: we handle them
58 * with mark/sweep during major collections
59 * *) some objects need to not move even if they are small (interned strings, Type handles):
60 * we use mark/sweep for them, too: they are not allocated in the nursery, but inside
61 * PinnedChunks regions
66 *) change the jit to emit write barrier calls when needed (we
67 can have specialized write barriers): done with icalls, still need to
68 use some specialized barriers
69 *) we could have a function pointer in MonoClass to implement
70 customized write barriers for value types
71 *) the write barrier code could be isolated in a couple of functions: when a
72 thread is stopped if it's inside the barrier it is let go again
73 until we stop outside of them (not really needed, see below GC-safe points)
74 *) investigate the stuff needed to advance a thread to a GC-safe
75 point (single-stepping, read from unmapped memory etc) and implement it
76 Not needed yet: since we treat the objects reachable from the stack/regs as
77 roots, we store the ptr and exec the write barrier so there is no race.
78 We may need this to solve the issue with setting the length of arrays and strings.
79 We may need this also for handling precise info on stacks, even simple things
80 as having uninitialized data on the stack and having to wait for the prolog
81 to zero it. Not an issue for the last frame that we scan conservatively.
82 We could always not trust the value in the slots anyway.
83 *) make the jit info table lock free
84 *) modify the jit to save info about references in stack locations:
85 this can be done just for locals as a start, so that at least
86 part of the stack is handled precisely.
87 *) Make the debug printf stuff thread and signal safe.
88 *) test/fix 64 bit issues
89 *) test/fix endianess issues
91 *) add batch moving profile info
92 *) add more timing info
93 *) there is a possible race when an array or string is created: the vtable is set,
94 but the length is set only later so if the GC needs to scan the object in that window,
95 it won't get the correct size for the object. The object can't have references and it will
96 be pinned, but a free memory fragment may be created that overlaps with it.
97 We should change the array max_length field to be at the same offset as the string length:
98 this way we can have a single special alloc function for them that sets the length.
99 Multi-dim arrays have the same issue for rank == 1 for the bounds data.
100 *) implement a card table as the write barrier instead of remembered sets?
101 *) some sort of blacklist support?
102 *) fin_ready_list and critical_fin_list are part of the root set, too
103 *) consider lowering the large object min size to 16/32KB or so and benchmark
104 *) once mark-compact is implemented we could still keep the
105 copying collector for the old generation and use it if we think
106 it is better (small heaps and no pinning object in the old
108 *) avoid the memory store from copy_object when not needed.
109 *) optimize the write barriers fastpath to happen in managed code
110 *) add an option to mmap the whole heap in one chunk: it makes for many
111 simplifications in the checks (put the nursery at the top and just use a single
112 check for inclusion/exclusion): the issue this has is that on 32 bit systems it's
113 not flexible (too much of the address space may be used by default or we can't
114 increase the heap as needed) and we'd need a race-free mechanism to return memory
115 back to the system (mprotect(PROT_NONE) will still keep the memory allocated if it
116 was written to, munmap is needed, but the following mmap may not find the same segment
118 *) memzero the fragments after restarting the world and optionally a smaller chunk at a time
119 *) an additional strategy to realloc/expand the nursery when fully pinned is to start
120 allocating objects in the old generation. This means that we can't optimize away write
121 barrier calls in ctors (but that is not valid for other reasons, too).
122 *) add write barriers to the Clone methods
130 #include <semaphore.h>
134 #include "metadata/metadata-internals.h"
135 #include "metadata/class-internals.h"
136 #include "metadata/gc-internal.h"
137 #include "metadata/object-internals.h"
138 #include "metadata/threads.h"
139 #include "metadata/sgen-gc.h"
140 #include "metadata/sgen-archdep.h"
141 #include "metadata/mono-gc.h"
142 #include "metadata/method-builder.h"
143 #include "metadata/profiler-private.h"
144 #include "metadata/monitor.h"
145 #include "metadata/threadpool-internals.h"
146 #include "metadata/mempool-internals.h"
147 #include "metadata/marshal.h"
148 #include "utils/mono-mmap.h"
149 #include "utils/mono-time.h"
150 #include "utils/mono-semaphore.h"
151 #include "utils/mono-counters.h"
153 #include <mono/utils/memcheck.h>
155 #define OPDEF(a,b,c,d,e,f,g,h,i,j) \
159 #include "mono/cil/opcode.def"
166 * ######################################################################
167 * ######## Types and constants used by the GC.
168 * ######################################################################
170 #if SIZEOF_VOID_P == 4
171 typedef guint32 mword;
173 typedef guint64 mword;
176 static int gc_initialized = 0;
177 static int gc_debug_level = 0;
178 static FILE* gc_debug_file;
179 /* If set, do a minor collection before every allocation */
180 static gboolean collect_before_allocs = FALSE;
181 /* If set, do a heap consistency check before each minor collection */
182 static gboolean consistency_check_at_minor_collection = FALSE;
183 /* If set, check that there are no references to the domain left at domain unload */
184 static gboolean xdomain_checks = FALSE;
185 /* If not null, dump the heap after each collection into this file */
186 static FILE *heap_dump_file = NULL;
187 /* If set, mark stacks conservatively, even if precise marking is possible */
188 static gboolean conservative_stack_mark = FALSE;
189 /* If set, do a plausibility check on the scan_starts before and after
191 static gboolean do_scan_starts_check = FALSE;
194 * Turning on heavy statistics will turn off the managed allocator and
195 * the managed write barrier.
197 //#define HEAVY_STATISTICS
199 #ifdef HEAVY_STATISTICS
200 #define HEAVY_STAT(x) x
202 #define HEAVY_STAT(x)
205 #ifdef HEAVY_STATISTICS
206 static long stat_objects_alloced = 0;
207 static long stat_copy_object_called_nursery = 0;
208 static long stat_objects_copied_nursery = 0;
209 static long stat_copy_object_called_major = 0;
210 static long stat_objects_copied_major = 0;
212 static long stat_copy_object_failed_from_space = 0;
213 static long stat_copy_object_failed_forwarded = 0;
214 static long stat_copy_object_failed_pinned = 0;
215 static long stat_copy_object_failed_large_pinned = 0;
216 static long stat_copy_object_failed_to_space = 0;
218 static long stat_store_remsets = 0;
219 static long stat_store_remsets_unique = 0;
220 static long stat_saved_remsets_1 = 0;
221 static long stat_saved_remsets_2 = 0;
222 static long stat_global_remsets_added = 0;
223 static long stat_global_remsets_processed = 0;
225 static long num_copy_object_called = 0;
226 static long num_objects_copied = 0;
228 static int stat_wbarrier_set_field = 0;
229 static int stat_wbarrier_set_arrayref = 0;
230 static int stat_wbarrier_arrayref_copy = 0;
231 static int stat_wbarrier_generic_store = 0;
232 static int stat_wbarrier_generic_store_remset = 0;
233 static int stat_wbarrier_set_root = 0;
234 static int stat_wbarrier_value_copy = 0;
235 static int stat_wbarrier_object_copy = 0;
238 static long pinned_chunk_bytes_alloced = 0;
239 static long large_internal_bytes_alloced = 0;
242 INTERNAL_MEM_PIN_QUEUE,
243 INTERNAL_MEM_FRAGMENT,
244 INTERNAL_MEM_SECTION,
245 INTERNAL_MEM_SCAN_STARTS,
246 INTERNAL_MEM_FIN_TABLE,
247 INTERNAL_MEM_FINALIZE_ENTRY,
248 INTERNAL_MEM_DISLINK_TABLE,
249 INTERNAL_MEM_DISLINK,
250 INTERNAL_MEM_ROOTS_TABLE,
251 INTERNAL_MEM_ROOT_RECORD,
252 INTERNAL_MEM_STATISTICS,
254 INTERNAL_MEM_GRAY_QUEUE,
255 INTERNAL_MEM_STORE_REMSET,
259 static long small_internal_mem_bytes [INTERNAL_MEM_MAX];
263 mono_gc_flush_info (void)
265 fflush (gc_debug_file);
269 #define MAX_DEBUG_LEVEL 8
270 #define DEBUG(level,a) do {if (G_UNLIKELY ((level) <= MAX_DEBUG_LEVEL && (level) <= gc_debug_level)) a;} while (0)
272 #define TV_DECLARE(name) gint64 name
273 #define TV_GETTIME(tv) tv = mono_100ns_ticks ()
274 #define TV_ELAPSED(start,end) (int)((end-start) / 10)
276 #define ALIGN_TO(val,align) ((((guint64)val) + ((align) - 1)) & ~((align) - 1))
278 #define GC_BITS_PER_WORD (sizeof (mword) * 8)
286 typedef struct _Block Block;
292 /* each request from the OS ends up in a GCMemSection */
293 typedef struct _GCMemSection GCMemSection;
294 struct _GCMemSection {
298 /* pointer where more data could be allocated if it fits */
302 * scan starts is an array of pointers to objects equally spaced in the allocation area
303 * They let use quickly find pinned objects from pinning pointers.
306 /* in major collections indexes in the pin_queue for objects that pin this section */
309 unsigned short num_scan_start;
310 gboolean is_to_space;
313 #define SIZEOF_GC_MEM_SECTION ((sizeof (GCMemSection) + 7) & ~7)
315 /* large object space struct: 64+ KB */
316 /* we could make this limit much smaller to avoid memcpy copy
317 * and potentially have more room in the GC descriptor: need to measure
318 * This also means that such small OS objects will need to be
319 * allocated in a different way (using pinned chunks).
320 * We may want to put large but smaller than 64k objects in the fixed space
321 * when we move the object from one generation to another (to limit the
322 * pig in the snake effect).
323 * Note: it may be worth to have an optimized copy function, since we can
324 * assume that objects are aligned and have a multiple of 8 size.
325 * FIXME: This structure needs to be a multiple of 8 bytes in size: this is not
326 * true if MONO_ZERO_LEN_ARRAY is nonzero.
328 typedef struct _LOSObject LOSObject;
331 mword size; /* this is the object size */
332 int dummy; /* to have a sizeof (LOSObject) a multiple of ALLOC_ALIGN and data starting at same alignment */
335 char data [MONO_ZERO_LEN_ARRAY];
338 /* Pinned objects are allocated in the LOS space if bigger than half a page
339 * or from freelists otherwise. We assume that pinned objects are relatively few
340 * and they have a slow dying speed (like interned strings, thread objects).
341 * As such they will be collected only at major collections.
342 * free lists are not global: when we need memory we allocate a PinnedChunk.
343 * Each pinned chunk is made of several pages, the first of wich is used
344 * internally for bookeeping (here think of a page as 4KB). The bookeeping
345 * includes the freelists vectors and info about the object size of each page
346 * in the pinned chunk. So, when needed, a free page is found in a pinned chunk,
347 * a size is assigned to it, the page is divided in the proper chunks and each
348 * chunk is added to the freelist. To not waste space, the remaining space in the
349 * first page is used as objects of size 16 or 32 (need to measure which are more
351 * We use this same structure to allocate memory used internally by the GC, so
352 * we never use malloc/free if we need to alloc during collection: the world is stopped
353 * and malloc/free will deadlock.
354 * When we want to iterate over pinned objects, we just scan a page at a time
355 * linearly according to the size of objects in the page: the next pointer used to link
356 * the items in the freelist uses the same word as the vtable. Since we keep freelists
357 * for each pinned chunk, if the word points outside the pinned chunk it means
359 * We could avoid this expensive scanning in creative ways. We could have a policy
360 * of putting in the pinned space only objects we know about that have no struct fields
361 * with references and we can easily use a even expensive write barrier for them,
362 * since pointer writes on such objects should be rare.
363 * The best compromise is to just alloc interned strings and System.MonoType in them.
364 * It would be nice to allocate MonoThread in it, too: must check that we properly
365 * use write barriers so we don't have to do any expensive scanning of the whole pinned
366 * chunk list during minor collections. We can avoid it now because we alloc in it only
367 * reference-free objects.
369 #define PINNED_FIRST_SLOT_SIZE (sizeof (gpointer) * 4)
370 #define MAX_FREELIST_SIZE 2048
371 #define PINNED_PAGE_SIZE (4096)
372 #define PINNED_CHUNK_MIN_SIZE (4096*8)
373 typedef struct _PinnedChunk PinnedChunk;
374 struct _PinnedChunk {
377 int *page_sizes; /* a 0 means the page is still unused */
380 void *data [1]; /* page sizes and free lists are stored here */
383 /* The method used to clear the nursery */
384 /* Clearing at nursery collections is the safest, but has bad interactions with caches.
385 * Clearing at TLAB creation is much faster, but more complex and it might expose hard
390 CLEAR_AT_TLAB_CREATION
391 } NurseryClearPolicy;
393 static NurseryClearPolicy nursery_clear_policy = CLEAR_AT_TLAB_CREATION;
396 * If this is set, the nursery is aligned to an address aligned to its size, ie.
397 * a 1MB nursery will be aligned to an address divisible by 1MB. This allows us to
398 * speed up ptr_in_nursery () checks which are very frequent. This requires the
399 * nursery size to be a compile time constant.
401 #define ALIGN_NURSERY 1
404 * The young generation is divided into fragments. This is because
405 * we can hand one fragments to a thread for lock-less fast alloc and
406 * because the young generation ends up fragmented anyway by pinned objects.
407 * Once a collection is done, a list of fragments is created. When doing
408 * thread local alloc we use smallish nurseries so we allow new threads to
409 * allocate memory from gen0 without triggering a collection. Threads that
410 * are found to allocate lots of memory are given bigger fragments. This
411 * should make the finalizer thread use little nursery memory after a while.
412 * We should start assigning threads very small fragments: if there are many
413 * threads the nursery will be full of reserved space that the threads may not
414 * use at all, slowing down allocation speed.
415 * Thread local allocation is done from areas of memory Hotspot calls Thread Local
416 * Allocation Buffers (TLABs).
418 typedef struct _Fragment Fragment;
422 char *fragment_start;
423 char *fragment_limit; /* the current soft limit for allocation */
427 /* the runtime can register areas of memory as roots: we keep two lists of roots,
428 * a pinned root set for conservatively scanned roots and a normal one for
429 * precisely scanned roots (currently implemented as a single list).
431 typedef struct _RootRecord RootRecord;
439 /* for use with write barriers */
440 typedef struct _RememberedSet RememberedSet;
441 struct _RememberedSet {
445 mword data [MONO_ZERO_LEN_ARRAY];
449 * We're never actually using the first element. It's always set to
450 * NULL to simplify the elimination of consecutive duplicate
453 #define STORE_REMSET_BUFFER_SIZE 1024
455 typedef struct _GenericStoreRememberedSet GenericStoreRememberedSet;
456 struct _GenericStoreRememberedSet {
457 GenericStoreRememberedSet *next;
458 /* We need one entry less because the first entry of store
459 remset buffers is always a dummy and we don't copy it. */
460 gpointer data [STORE_REMSET_BUFFER_SIZE - 1];
463 /* we have 4 possible values in the low 2 bits */
465 REMSET_LOCATION, /* just a pointer to the exact location */
466 REMSET_RANGE, /* range of pointer fields */
467 REMSET_OBJECT, /* mark all the object for scanning */
468 REMSET_OTHER, /* all others */
469 REMSET_TYPE_MASK = 0x3
472 /* Subtypes of REMSET_OTHER */
474 REMSET_VTYPE, /* a valuetype array described by a gc descriptor and a count */
475 REMSET_ROOT_LOCATION, /* a location inside a root */
478 #ifdef HAVE_KW_THREAD
479 static __thread RememberedSet *remembered_set MONO_TLS_FAST;
481 static pthread_key_t remembered_set_key;
482 static RememberedSet *global_remset;
483 static RememberedSet *freed_thread_remsets;
484 //static int store_to_global_remset = 0;
485 static GenericStoreRememberedSet *generic_store_remsets = NULL;
487 /* FIXME: later choose a size that takes into account the RememberedSet struct
488 * and doesn't waste any alloc paddin space.
490 #define DEFAULT_REMSET_SIZE 1024
491 static RememberedSet* alloc_remset (int size, gpointer id);
493 /* Structure that corresponds to a MonoVTable: desc is a mword so requires
494 * no cast from a pointer to an integer
501 /* these bits are set in the object vtable: we could merge them since an object can be
502 * either pinned or forwarded but not both.
503 * We store them in the vtable slot because the bits are used in the sync block for
504 * other purposes: if we merge them and alloc the sync blocks aligned to 8 bytes, we can change
505 * this and use bit 3 in the syncblock (with the lower two bits both set for forwarded, that
506 * would be an invalid combination for the monitor and hash code).
507 * The values are already shifted.
508 * The forwarding address is stored in the sync block.
510 #define FORWARDED_BIT 1
512 #define VTABLE_BITS_MASK 0x3
514 /* returns NULL if not forwarded, or the forwarded address */
515 #define object_is_forwarded(obj) (((mword*)(obj))[0] & FORWARDED_BIT? (void*)(((mword*)(obj))[1]): NULL)
516 /* set the forwarded address fw_addr for object obj */
517 #define forward_object(obj,fw_addr) do { \
518 ((mword*)(obj))[0] |= FORWARDED_BIT; \
519 ((mword*)(obj))[1] = (mword)(fw_addr); \
522 #define object_is_pinned(obj) (((mword*)(obj))[0] & PINNED_BIT)
523 #define pin_object(obj) do { \
524 ((mword*)(obj))[0] |= PINNED_BIT; \
526 #define unpin_object(obj) do { \
527 ((mword*)(obj))[0] &= ~PINNED_BIT; \
531 #define ptr_in_nursery(ptr) (((mword)(ptr) & ~((1 << DEFAULT_NURSERY_BITS) - 1)) == (mword)nursery_start)
533 #define ptr_in_nursery(ptr) ((char*)(ptr) >= nursery_start && (char*)(ptr) < nursery_real_end)
537 * Since we set bits in the vtable, use the macro to load it from the pointer to
538 * an object that is potentially pinned.
540 #define LOAD_VTABLE(addr) ((*(mword*)(addr)) & ~VTABLE_BITS_MASK)
543 safe_name (void* obj)
545 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (obj);
546 return vt->klass->name;
550 safe_object_get_size (MonoObject* o)
552 MonoClass *klass = ((MonoVTable*)LOAD_VTABLE (o))->klass;
553 if (klass == mono_defaults.string_class) {
554 return sizeof (MonoString) + 2 * mono_string_length ((MonoString*) o) + 2;
555 } else if (klass->rank) {
556 MonoArray *array = (MonoArray*)o;
557 size_t size = sizeof (MonoArray) + mono_array_element_size (klass) * mono_array_length (array);
558 if (G_UNLIKELY (array->bounds)) {
559 size += sizeof (mono_array_size_t) - 1;
560 size &= ~(sizeof (mono_array_size_t) - 1);
561 size += sizeof (MonoArrayBounds) * klass->rank;
565 /* from a created object: the class must be inited already */
566 return klass->instance_size;
571 * ######################################################################
572 * ######## Global data.
573 * ######################################################################
575 static LOCK_DECLARE (gc_mutex);
576 static int gc_disabled = 0;
577 static int num_minor_gcs = 0;
578 static int num_major_gcs = 0;
580 /* good sizes are 512KB-1MB: larger ones increase a lot memzeroing time */
581 //#define DEFAULT_NURSERY_SIZE (1024*512*125+4096*118)
582 #define DEFAULT_NURSERY_SIZE (1024*512*2)
583 /* The number of trailing 0 bits in DEFAULT_NURSERY_SIZE */
584 #define DEFAULT_NURSERY_BITS 20
585 #define MAJOR_SECTION_SIZE (128*1024)
586 #define BLOCK_FOR_OBJECT(o) ((Block*)(((mword)(o)) & ~(MAJOR_SECTION_SIZE - 1)))
587 #define MAJOR_SECTION_FOR_OBJECT(o) ((GCMemSection*)BLOCK_FOR_OBJECT ((o)))
588 #define MIN_MINOR_COLLECTION_SECTION_ALLOWANCE (DEFAULT_NURSERY_SIZE * 3 / MAJOR_SECTION_SIZE)
589 #define DEFAULT_LOS_COLLECTION_TARGET (DEFAULT_NURSERY_SIZE * 2)
590 /* to quickly find the head of an object pinned by a conservative address
591 * we keep track of the objects allocated for each SCAN_START_SIZE memory
592 * chunk in the nursery or other memory sections. Larger values have less
593 * memory overhead and bigger runtime cost. 4-8 KB are reasonable values.
595 #define SCAN_START_SIZE (4096*2)
596 /* the minimum size of a fragment that we consider useful for allocation */
597 #define FRAGMENT_MIN_SIZE (512)
598 /* This is a fixed value used for pinned chunks, not the system pagesize */
599 #define FREELIST_PAGESIZE 4096
601 static mword pagesize = 4096;
602 static mword nursery_size = DEFAULT_NURSERY_SIZE;
603 static int degraded_mode = 0;
605 static int minor_collection_section_allowance = MIN_MINOR_COLLECTION_SECTION_ALLOWANCE;
606 static int minor_collection_sections_alloced = 0;
607 static int num_major_sections = 0;
609 static LOSObject *los_object_list = NULL;
610 static mword los_memory_usage = 0;
611 static mword los_num_objects = 0;
612 static mword next_los_collection = 2*1024*1024; /* 2 MB, need to tune */
613 static mword total_alloc = 0;
614 /* use this to tune when to do a major/minor collection */
615 static mword memory_pressure = 0;
617 static GCMemSection *section_list = NULL;
618 static GCMemSection *nursery_section = NULL;
619 static mword lowest_heap_address = ~(mword)0;
620 static mword highest_heap_address = 0;
622 typedef struct _FinalizeEntry FinalizeEntry;
623 struct _FinalizeEntry {
628 typedef struct _FinalizeEntryHashTable FinalizeEntryHashTable;
629 struct _FinalizeEntryHashTable {
630 FinalizeEntry **table;
635 typedef struct _DisappearingLink DisappearingLink;
636 struct _DisappearingLink {
637 DisappearingLink *next;
641 typedef struct _DisappearingLinkHashTable DisappearingLinkHashTable;
642 struct _DisappearingLinkHashTable {
643 DisappearingLink **table;
648 #define LARGE_INTERNAL_MEM_HEADER_MAGIC 0x7d289f3a
650 typedef struct _LargeInternalMemHeader LargeInternalMemHeader;
651 struct _LargeInternalMemHeader {
664 * The link pointer is hidden by negating each bit. We use the lowest
665 * bit of the link (before negation) to store whether it needs
666 * resurrection tracking.
668 #define HIDE_POINTER(p,t) ((gpointer)(~((gulong)(p)|((t)?1:0))))
669 #define REVEAL_POINTER(p) ((gpointer)((~(gulong)(p))&~3L))
671 #define DISLINK_OBJECT(d) (REVEAL_POINTER (*(d)->link))
672 #define DISLINK_TRACK(d) ((~(gulong)(*(d)->link)) & 1)
675 * The finalizable hash has the object as the key, the
676 * disappearing_link hash, has the link address as key.
678 static FinalizeEntryHashTable minor_finalizable_hash;
679 static FinalizeEntryHashTable major_finalizable_hash;
680 /* objects that are ready to be finalized */
681 static FinalizeEntry *fin_ready_list = NULL;
682 static FinalizeEntry *critical_fin_list = NULL;
684 static DisappearingLinkHashTable minor_disappearing_link_hash;
685 static DisappearingLinkHashTable major_disappearing_link_hash;
687 static int num_ready_finalizers = 0;
688 static int no_finalize = 0;
690 /* keep each size a multiple of ALLOC_ALIGN */
691 /* on 64 bit systems 8 is likely completely unused. */
692 static const int freelist_sizes [] = {
693 8, 16, 24, 32, 40, 48, 64, 80,
694 96, 128, 160, 192, 224, 256, 320, 384,
695 448, 512, 584, 680, 816, 1024, 1360, 2048};
696 #define FREELIST_NUM_SLOTS (sizeof (freelist_sizes) / sizeof (freelist_sizes [0]))
698 static char* max_pinned_chunk_addr = NULL;
699 static char* min_pinned_chunk_addr = (char*)-1;
700 /* pinned_chunk_list is used for allocations of objects that are never moved */
701 static PinnedChunk *pinned_chunk_list = NULL;
702 /* internal_chunk_list is used for allocating structures needed by the GC */
703 static PinnedChunk *internal_chunk_list = NULL;
706 obj_is_from_pinned_alloc (char *p)
708 return BLOCK_FOR_OBJECT (p)->role == MEMORY_ROLE_PINNED;
711 static int slot_for_size (size_t size);
714 free_pinned_object (PinnedChunk *chunk, char *obj, size_t size)
716 void **p = (void**)obj;
717 int slot = slot_for_size (size);
719 g_assert (obj >= (char*)chunk->start_data && obj < ((char*)chunk + chunk->num_pages * FREELIST_PAGESIZE));
720 *p = chunk->free_list [slot];
721 chunk->free_list [slot] = p;
725 ROOT_TYPE_NORMAL = 0, /* "normal" roots */
726 ROOT_TYPE_PINNED = 1, /* roots without a GC descriptor */
727 ROOT_TYPE_WBARRIER = 2, /* roots with a write barrier */
731 /* registered roots: the key to the hash is the root start address */
733 * Different kinds of roots are kept separate to speed up pin_from_roots () for example.
735 static RootRecord **roots_hash [ROOT_TYPE_NUM] = { NULL, NULL };
736 static int roots_hash_size [ROOT_TYPE_NUM] = { 0, 0, 0 };
737 static mword roots_size = 0; /* amount of memory in the root set */
738 static int num_roots_entries [ROOT_TYPE_NUM] = { 0, 0, 0 };
741 * The current allocation cursors
742 * We allocate objects in the nursery.
743 * The nursery is the area between nursery_start and nursery_real_end.
744 * Allocation is done from a Thread Local Allocation Buffer (TLAB). TLABs are allocated
745 * from nursery fragments.
746 * tlab_next is the pointer to the space inside the TLAB where the next object will
748 * tlab_temp_end is the pointer to the end of the temporary space reserved for
749 * the allocation: it allows us to set the scan starts at reasonable intervals.
750 * tlab_real_end points to the end of the TLAB.
751 * nursery_frag_real_end points to the end of the currently used nursery fragment.
752 * nursery_first_pinned_start points to the start of the first pinned object in the nursery
753 * nursery_last_pinned_end points to the end of the last pinned object in the nursery
754 * At the next allocation, the area of the nursery where objects can be present is
755 * between MIN(nursery_first_pinned_start, first_fragment_start) and
756 * MAX(nursery_last_pinned_end, nursery_frag_real_end)
758 static char *nursery_start = NULL;
760 /* eventually share with MonoThread? */
761 typedef struct _SgenThreadInfo SgenThreadInfo;
763 struct _SgenThreadInfo {
764 SgenThreadInfo *next;
766 unsigned int stop_count; /* to catch duplicate signals */
769 volatile int in_critical_region;
772 void *stack_start_limit;
773 char **tlab_next_addr;
774 char **tlab_start_addr;
775 char **tlab_temp_end_addr;
776 char **tlab_real_end_addr;
777 gpointer **store_remset_buffer_addr;
778 long *store_remset_buffer_index_addr;
779 RememberedSet *remset;
780 gpointer runtime_data;
781 gpointer stopped_ip; /* only valid if the thread is stopped */
782 MonoDomain *stopped_domain; /* ditto */
783 gpointer *stopped_regs; /* ditto */
784 #ifndef HAVE_KW_THREAD
789 gpointer *store_remset_buffer;
790 long store_remset_buffer_index;
794 #ifdef HAVE_KW_THREAD
795 #define TLAB_ACCESS_INIT
796 #define TLAB_START tlab_start
797 #define TLAB_NEXT tlab_next
798 #define TLAB_TEMP_END tlab_temp_end
799 #define TLAB_REAL_END tlab_real_end
800 #define REMEMBERED_SET remembered_set
801 #define STORE_REMSET_BUFFER store_remset_buffer
802 #define STORE_REMSET_BUFFER_INDEX store_remset_buffer_index
803 #define IN_CRITICAL_REGION thread_info->in_critical_region
805 static pthread_key_t thread_info_key;
806 #define TLAB_ACCESS_INIT SgenThreadInfo *__thread_info__ = pthread_getspecific (thread_info_key)
807 #define TLAB_START (__thread_info__->tlab_start)
808 #define TLAB_NEXT (__thread_info__->tlab_next)
809 #define TLAB_TEMP_END (__thread_info__->tlab_temp_end)
810 #define TLAB_REAL_END (__thread_info__->tlab_real_end)
811 #define REMEMBERED_SET (__thread_info__->remset)
812 #define STORE_REMSET_BUFFER (__thread_info__->store_remset_buffer)
813 #define STORE_REMSET_BUFFER_INDEX (__thread_info__->store_remset_buffer_index)
814 #define IN_CRITICAL_REGION (__thread_info__->in_critical_region)
817 /* we use the memory barrier only to prevent compiler reordering (a memory constraint may be enough) */
818 #define ENTER_CRITICAL_REGION do {IN_CRITICAL_REGION = 1;mono_memory_barrier ();} while (0)
819 #define EXIT_CRITICAL_REGION do {IN_CRITICAL_REGION = 0;mono_memory_barrier ();} while (0)
822 * FIXME: What is faster, a TLS variable pointing to a structure, or separate TLS
823 * variables for next+temp_end ?
825 #ifdef HAVE_KW_THREAD
826 static __thread SgenThreadInfo *thread_info;
827 static __thread char *tlab_start;
828 static __thread char *tlab_next;
829 static __thread char *tlab_temp_end;
830 static __thread char *tlab_real_end;
831 static __thread gpointer *store_remset_buffer;
832 static __thread long store_remset_buffer_index;
833 /* Used by the managed allocator/wbarrier */
834 static __thread char **tlab_next_addr;
835 static __thread char *stack_end;
836 static __thread long *store_remset_buffer_index_addr;
838 static char *nursery_next = NULL;
839 static char *nursery_frag_real_end = NULL;
840 static char *nursery_real_end = NULL;
841 //static char *nursery_first_pinned_start = NULL;
842 static char *nursery_last_pinned_end = NULL;
844 /* The size of a TLAB */
845 /* The bigger the value, the less often we have to go to the slow path to allocate a new
846 * one, but the more space is wasted by threads not allocating much memory.
848 * FIXME: Make this self-tuning for each thread.
850 static guint32 tlab_size = (1024 * 4);
852 /* fragments that are free and ready to be used for allocation */
853 static Fragment *nursery_fragments = NULL;
854 /* freeelist of fragment structures */
855 static Fragment *fragment_freelist = NULL;
858 * used when moving the objects
860 static char *to_space_bumper = NULL;
861 static char *to_space_top = NULL;
862 static GCMemSection *to_space_section = NULL;
864 /* objects bigger then this go into the large object space */
865 #define MAX_SMALL_OBJ_SIZE MAX_FREELIST_SIZE
867 /* Functions supplied by the runtime to be called by the GC */
868 static MonoGCCallbacks gc_callbacks;
871 * ######################################################################
872 * ######## Macros and function declarations.
873 * ######################################################################
876 #define UPDATE_HEAP_BOUNDARIES(low,high) do { \
877 if ((mword)(low) < lowest_heap_address) \
878 lowest_heap_address = (mword)(low); \
879 if ((mword)(high) > highest_heap_address) \
880 highest_heap_address = (mword)(high); \
882 #define ADDR_IN_HEAP_BOUNDARIES(addr) ((p) >= lowest_heap_address && (p) < highest_heap_address)
885 align_pointer (void *ptr)
887 mword p = (mword)ptr;
888 p += sizeof (gpointer) - 1;
889 p &= ~ (sizeof (gpointer) - 1);
893 /* forward declarations */
894 static void* get_internal_mem (size_t size, int type);
895 static void free_internal_mem (void *addr, int type);
896 static void* get_os_memory (size_t size, int activate);
897 static void free_os_memory (void *addr, size_t size);
898 static G_GNUC_UNUSED void report_internal_mem_usage (void);
900 static int stop_world (void);
901 static int restart_world (void);
902 static void scan_thread_data (void *start_nursery, void *end_nursery, gboolean precise);
903 static void scan_from_remsets (void *start_nursery, void *end_nursery);
904 static void find_pinning_ref_from_thread (char *obj, size_t size);
905 static void update_current_thread_stack (void *start);
906 static GCMemSection* alloc_major_section (void);
907 static void finalize_in_range (char *start, char *end, int generation);
908 static void add_or_remove_disappearing_link (MonoObject *obj, void **link, gboolean track, int generation);
909 static void null_link_in_range (char *start, char *end, int generation);
910 static void null_links_for_domain (MonoDomain *domain, int generation);
911 static gboolean search_fragment_for_size (size_t size);
912 static void mark_pinned_from_addresses (PinnedChunk *chunk, void **start, void **end);
913 static void clear_remsets (void);
914 static void clear_tlabs (void);
915 typedef void (*ScanPinnedObjectCallbackFunc) (PinnedChunk*, char*, size_t, void*);
916 static void scan_pinned_objects (ScanPinnedObjectCallbackFunc callback, void *callback_data);
917 static void sweep_pinned_objects (void);
918 static void scan_from_pinned_objects (char *addr_start, char *addr_end);
919 static void free_large_object (LOSObject *obj);
920 static void free_major_section (GCMemSection *section);
921 static void to_space_expand (void);
923 static void mono_gc_register_disappearing_link (MonoObject *obj, void **link, gboolean track);
925 void describe_ptr (char *ptr);
926 void check_consistency (void);
927 char* check_object (char *start);
929 void mono_gc_scan_for_specific_ref (MonoObject *key);
932 * ######################################################################
933 * ######## GC descriptors
934 * ######################################################################
935 * Used to quickly get the info the GC needs about an object: size and
936 * where the references are held.
938 /* objects are aligned to 8 bytes boundaries
939 * A descriptor is a pointer in MonoVTable, so 32 or 64 bits of size.
940 * The low 3 bits define the type of the descriptor. The other bits
941 * depend on the type.
942 * As a general rule the 13 remaining low bits define the size, either
943 * of the whole object or of the elements in the arrays. While for objects
944 * the size is already in bytes, for arrays we need to shift, because
945 * array elements might be smaller than 8 bytes. In case of arrays, we
946 * use two bits to describe what the additional high bits represents,
947 * so the default behaviour can handle element sizes less than 2048 bytes.
948 * The high 16 bits, if 0 it means the object is pointer-free.
949 * This design should make it easy and fast to skip over ptr-free data.
950 * The first 4 types should cover >95% of the objects.
951 * Note that since the size of objects is limited to 64K, larger objects
952 * will be allocated in the large object heap.
953 * If we want 4-bytes alignment, we need to put vector and small bitmap
957 DESC_TYPE_RUN_LENGTH, /* 16 bits aligned byte size | 1-3 (offset, numptr) bytes tuples */
958 DESC_TYPE_SMALL_BITMAP, /* 16 bits aligned byte size | 16-48 bit bitmap */
959 DESC_TYPE_STRING, /* nothing */
960 DESC_TYPE_COMPLEX, /* index for bitmap into complex_descriptors */
961 DESC_TYPE_VECTOR, /* 10 bits element size | 1 bit array | 2 bits desc | element desc */
962 DESC_TYPE_ARRAY, /* 10 bits element size | 1 bit array | 2 bits desc | element desc */
963 DESC_TYPE_LARGE_BITMAP, /* | 29-61 bitmap bits */
964 DESC_TYPE_COMPLEX_ARR, /* index for bitmap into complex_descriptors */
965 /* subtypes for arrays and vectors */
966 DESC_TYPE_V_PTRFREE = 0,/* there are no refs: keep first so it has a zero value */
967 DESC_TYPE_V_REFS, /* all the array elements are refs */
968 DESC_TYPE_V_RUN_LEN, /* elements are run-length encoded as DESC_TYPE_RUN_LENGTH */
969 DESC_TYPE_V_BITMAP /* elements are as the bitmap in DESC_TYPE_SMALL_BITMAP */
972 #define OBJECT_HEADER_WORDS (sizeof(MonoObject)/sizeof(gpointer))
973 #define LOW_TYPE_BITS 3
974 #define SMALL_BITMAP_SHIFT 16
975 #define SMALL_BITMAP_SIZE (GC_BITS_PER_WORD - SMALL_BITMAP_SHIFT)
976 #define VECTOR_INFO_SHIFT 14
977 #define VECTOR_ELSIZE_SHIFT 3
978 #define LARGE_BITMAP_SIZE (GC_BITS_PER_WORD - LOW_TYPE_BITS)
979 #define MAX_SMALL_SIZE ((1 << SMALL_BITMAP_SHIFT) - 1)
980 #define SMALL_SIZE_MASK 0xfff8
981 #define MAX_ELEMENT_SIZE 0x3ff
982 #define ELEMENT_SIZE_MASK (0x3ff << LOW_TYPE_BITS)
983 #define VECTOR_SUBTYPE_PTRFREE (DESC_TYPE_V_PTRFREE << VECTOR_INFO_SHIFT)
984 #define VECTOR_SUBTYPE_REFS (DESC_TYPE_V_REFS << VECTOR_INFO_SHIFT)
985 #define VECTOR_SUBTYPE_RUN_LEN (DESC_TYPE_V_RUN_LEN << VECTOR_INFO_SHIFT)
986 #define VECTOR_SUBTYPE_BITMAP (DESC_TYPE_V_BITMAP << VECTOR_INFO_SHIFT)
988 #define ALLOC_ALIGN 8
991 /* Root bitmap descriptors are simpler: the lower three bits describe the type
992 * and we either have 30/62 bitmap bits or nibble-based run-length,
993 * or a complex descriptor, or a user defined marker function.
996 ROOT_DESC_CONSERVATIVE, /* 0, so matches NULL value */
1001 ROOT_DESC_TYPE_MASK = 0x7,
1002 ROOT_DESC_TYPE_SHIFT = 3,
1005 #define MAKE_ROOT_DESC(type,val) ((type) | ((val) << ROOT_DESC_TYPE_SHIFT))
1007 #define MAX_USER_DESCRIPTORS 16
1009 static gsize* complex_descriptors = NULL;
1010 static int complex_descriptors_size = 0;
1011 static int complex_descriptors_next = 0;
1012 static MonoGCMarkFunc user_descriptors [MAX_USER_DESCRIPTORS];
1013 static int user_descriptors_next = 0;
1016 alloc_complex_descriptor (gsize *bitmap, int numbits)
1020 numbits = ALIGN_TO (numbits, GC_BITS_PER_WORD);
1021 nwords = numbits / GC_BITS_PER_WORD + 1;
1024 res = complex_descriptors_next;
1025 /* linear search, so we don't have duplicates with domain load/unload
1026 * this should not be performance critical or we'd have bigger issues
1027 * (the number and size of complex descriptors should be small).
1029 for (i = 0; i < complex_descriptors_next; ) {
1030 if (complex_descriptors [i] == nwords) {
1031 int j, found = TRUE;
1032 for (j = 0; j < nwords - 1; ++j) {
1033 if (complex_descriptors [i + 1 + j] != bitmap [j]) {
1043 i += complex_descriptors [i];
1045 if (complex_descriptors_next + nwords > complex_descriptors_size) {
1046 int new_size = complex_descriptors_size * 2 + nwords;
1047 complex_descriptors = g_realloc (complex_descriptors, new_size * sizeof (gsize));
1048 complex_descriptors_size = new_size;
1050 DEBUG (6, fprintf (gc_debug_file, "Complex descriptor %d, size: %d (total desc memory: %d)\n", res, nwords, complex_descriptors_size));
1051 complex_descriptors_next += nwords;
1052 complex_descriptors [res] = nwords;
1053 for (i = 0; i < nwords - 1; ++i) {
1054 complex_descriptors [res + 1 + i] = bitmap [i];
1055 DEBUG (6, fprintf (gc_debug_file, "\tvalue: %p\n", (void*)complex_descriptors [res + 1 + i]));
1062 * Descriptor builders.
1065 mono_gc_make_descr_for_string (gsize *bitmap, int numbits)
1067 return (void*) DESC_TYPE_STRING;
1071 mono_gc_make_descr_for_object (gsize *bitmap, int numbits, size_t obj_size)
1073 int first_set = -1, num_set = 0, last_set = -1, i;
1075 size_t stored_size = obj_size;
1076 stored_size += ALLOC_ALIGN - 1;
1077 stored_size &= ~(ALLOC_ALIGN - 1);
1078 for (i = 0; i < numbits; ++i) {
1079 if (bitmap [i / GC_BITS_PER_WORD] & ((gsize)1 << (i % GC_BITS_PER_WORD))) {
1086 if (stored_size <= MAX_SMALL_OBJ_SIZE) {
1087 /* check run-length encoding first: one byte offset, one byte number of pointers
1088 * on 64 bit archs, we can have 3 runs, just one on 32.
1089 * It may be better to use nibbles.
1091 if (first_set < 0) {
1092 desc = DESC_TYPE_RUN_LENGTH | stored_size;
1093 DEBUG (6, fprintf (gc_debug_file, "Ptrfree descriptor %p, size: %zd\n", (void*)desc, stored_size));
1094 return (void*) desc;
1095 } else if (first_set < 256 && num_set < 256 && (first_set + num_set == last_set + 1)) {
1096 desc = DESC_TYPE_RUN_LENGTH | stored_size | (first_set << 16) | (num_set << 24);
1097 DEBUG (6, fprintf (gc_debug_file, "Runlen descriptor %p, size: %zd, first set: %d, num set: %d\n", (void*)desc, stored_size, first_set, num_set));
1098 return (void*) desc;
1100 /* we know the 2-word header is ptr-free */
1101 if (last_set < SMALL_BITMAP_SIZE + OBJECT_HEADER_WORDS) {
1102 desc = DESC_TYPE_SMALL_BITMAP | stored_size | ((*bitmap >> OBJECT_HEADER_WORDS) << SMALL_BITMAP_SHIFT);
1103 DEBUG (6, fprintf (gc_debug_file, "Smallbitmap descriptor %p, size: %zd, last set: %d\n", (void*)desc, stored_size, last_set));
1104 return (void*) desc;
1107 /* we know the 2-word header is ptr-free */
1108 if (last_set < LARGE_BITMAP_SIZE + OBJECT_HEADER_WORDS) {
1109 desc = DESC_TYPE_LARGE_BITMAP | ((*bitmap >> OBJECT_HEADER_WORDS) << LOW_TYPE_BITS);
1110 DEBUG (6, fprintf (gc_debug_file, "Largebitmap descriptor %p, size: %zd, last set: %d\n", (void*)desc, stored_size, last_set));
1111 return (void*) desc;
1113 /* it's a complex object ... */
1114 desc = DESC_TYPE_COMPLEX | (alloc_complex_descriptor (bitmap, last_set + 1) << LOW_TYPE_BITS);
1115 return (void*) desc;
1118 /* If the array holds references, numbits == 1 and the first bit is set in elem_bitmap */
1120 mono_gc_make_descr_for_array (int vector, gsize *elem_bitmap, int numbits, size_t elem_size)
1122 int first_set = -1, num_set = 0, last_set = -1, i;
1123 mword desc = vector? DESC_TYPE_VECTOR: DESC_TYPE_ARRAY;
1124 for (i = 0; i < numbits; ++i) {
1125 if (elem_bitmap [i / GC_BITS_PER_WORD] & ((gsize)1 << (i % GC_BITS_PER_WORD))) {
1132 if (elem_size <= MAX_ELEMENT_SIZE) {
1133 desc |= elem_size << VECTOR_ELSIZE_SHIFT;
1135 return (void*)(desc | VECTOR_SUBTYPE_PTRFREE);
1137 /* Note: we also handle structs with just ref fields */
1138 if (num_set * sizeof (gpointer) == elem_size) {
1139 return (void*)(desc | VECTOR_SUBTYPE_REFS | ((gssize)(-1) << 16));
1141 /* FIXME: try run-len first */
1142 /* Note: we can't skip the object header here, because it's not present */
1143 if (last_set <= SMALL_BITMAP_SIZE) {
1144 return (void*)(desc | VECTOR_SUBTYPE_BITMAP | (*elem_bitmap << 16));
1147 /* it's am array of complex structs ... */
1148 desc = DESC_TYPE_COMPLEX_ARR;
1149 desc |= alloc_complex_descriptor (elem_bitmap, last_set + 1) << LOW_TYPE_BITS;
1150 return (void*) desc;
1153 /* Return the bitmap encoded by a descriptor */
1155 mono_gc_get_bitmap_for_descr (void *descr, int *numbits)
1157 mword d = (mword)descr;
1161 case DESC_TYPE_RUN_LENGTH: {
1162 int first_set = (d >> 16) & 0xff;
1163 int num_set = (d >> 24) & 0xff;
1166 bitmap = g_new0 (gsize, (first_set + num_set + 7) / 8);
1168 for (i = first_set; i < first_set + num_set; ++i)
1169 bitmap [i / GC_BITS_PER_WORD] |= ((gsize)1 << (i % GC_BITS_PER_WORD));
1171 *numbits = first_set + num_set;
1175 case DESC_TYPE_SMALL_BITMAP:
1176 bitmap = g_new0 (gsize, 1);
1178 bitmap [0] = (d >> SMALL_BITMAP_SHIFT) << OBJECT_HEADER_WORDS;
1180 *numbits = GC_BITS_PER_WORD;
1184 g_assert_not_reached ();
1188 /* helper macros to scan and traverse objects, macros because we resue them in many functions */
1189 #define STRING_SIZE(size,str) do { \
1190 (size) = sizeof (MonoString) + 2 * (mono_string_length ((MonoString*)(str)) + 1); \
1191 (size) += (ALLOC_ALIGN - 1); \
1192 (size) &= ~(ALLOC_ALIGN - 1); \
1195 #define OBJ_RUN_LEN_SIZE(size,desc,obj) do { \
1196 (size) = (desc) & 0xfff8; \
1199 #define OBJ_BITMAP_SIZE(size,desc,obj) do { \
1200 (size) = (desc) & 0xfff8; \
1203 //#define PREFETCH(addr) __asm__ __volatile__ (" prefetchnta %0": : "m"(*(char *)(addr)))
1204 #define PREFETCH(addr)
1206 /* code using these macros must define a HANDLE_PTR(ptr) macro that does the work */
1207 #define OBJ_RUN_LEN_FOREACH_PTR(desc,obj) do { \
1208 if ((desc) & 0xffff0000) { \
1209 /* there are pointers */ \
1210 void **_objptr_end; \
1211 void **_objptr = (void**)(obj); \
1212 _objptr += ((desc) >> 16) & 0xff; \
1213 _objptr_end = _objptr + (((desc) >> 24) & 0xff); \
1214 while (_objptr < _objptr_end) { \
1215 HANDLE_PTR (_objptr, (obj)); \
1221 /* a bitmap desc means that there are pointer references or we'd have
1222 * choosen run-length, instead: add an assert to check.
1224 #define OBJ_BITMAP_FOREACH_PTR(desc,obj) do { \
1225 /* there are pointers */ \
1226 void **_objptr = (void**)(obj); \
1227 gsize _bmap = (desc) >> 16; \
1228 _objptr += OBJECT_HEADER_WORDS; \
1230 if ((_bmap & 1)) { \
1231 HANDLE_PTR (_objptr, (obj)); \
1238 #define OBJ_LARGE_BITMAP_FOREACH_PTR(vt,obj) do { \
1239 /* there are pointers */ \
1240 void **_objptr = (void**)(obj); \
1241 gsize _bmap = (vt)->desc >> LOW_TYPE_BITS; \
1242 _objptr += OBJECT_HEADER_WORDS; \
1244 if ((_bmap & 1)) { \
1245 HANDLE_PTR (_objptr, (obj)); \
1252 #define OBJ_COMPLEX_FOREACH_PTR(vt,obj) do { \
1253 /* there are pointers */ \
1254 void **_objptr = (void**)(obj); \
1255 gsize *bitmap_data = complex_descriptors + ((vt)->desc >> LOW_TYPE_BITS); \
1256 int bwords = (*bitmap_data) - 1; \
1257 void **start_run = _objptr; \
1260 MonoObject *myobj = (MonoObject*)obj; \
1261 g_print ("found %d at %p (0x%zx): %s.%s\n", bwords, (obj), (vt)->desc, myobj->vtable->klass->name_space, myobj->vtable->klass->name); \
1263 while (bwords-- > 0) { \
1264 gsize _bmap = *bitmap_data++; \
1265 _objptr = start_run; \
1266 /*g_print ("bitmap: 0x%x/%d at %p\n", _bmap, bwords, _objptr);*/ \
1268 if ((_bmap & 1)) { \
1269 HANDLE_PTR (_objptr, (obj)); \
1274 start_run += GC_BITS_PER_WORD; \
1278 /* this one is untested */
1279 #define OBJ_COMPLEX_ARR_FOREACH_PTR(vt,obj) do { \
1280 /* there are pointers */ \
1281 gsize *mbitmap_data = complex_descriptors + ((vt)->desc >> LOW_TYPE_BITS); \
1282 int mbwords = (*mbitmap_data++) - 1; \
1283 int el_size = mono_array_element_size (((MonoObject*)(obj))->vtable->klass); \
1284 char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
1285 char *e_end = e_start + el_size * mono_array_length ((MonoArray*)(obj)); \
1287 MonoObject *myobj = (MonoObject*)start; \
1288 g_print ("found %d at %p (0x%zx): %s.%s\n", mbwords, (obj), (vt)->desc, myobj->vtable->klass->name_space, myobj->vtable->klass->name); \
1290 while (e_start < e_end) { \
1291 void **_objptr = (void**)e_start; \
1292 gsize *bitmap_data = mbitmap_data; \
1293 unsigned int bwords = mbwords; \
1294 while (bwords-- > 0) { \
1295 gsize _bmap = *bitmap_data++; \
1296 void **start_run = _objptr; \
1297 /*g_print ("bitmap: 0x%x\n", _bmap);*/ \
1299 if ((_bmap & 1)) { \
1300 HANDLE_PTR (_objptr, (obj)); \
1305 _objptr = start_run + GC_BITS_PER_WORD; \
1307 e_start += el_size; \
1311 #define OBJ_VECTOR_FOREACH_PTR(vt,obj) do { \
1312 /* note: 0xffffc000 excludes DESC_TYPE_V_PTRFREE */ \
1313 if ((vt)->desc & 0xffffc000) { \
1314 int el_size = ((vt)->desc >> 3) & MAX_ELEMENT_SIZE; \
1315 /* there are pointers */ \
1316 int etype = (vt)->desc & 0xc000; \
1317 if (etype == (DESC_TYPE_V_REFS << 14)) { \
1318 void **p = (void**)((char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector)); \
1319 void **end_refs = (void**)((char*)p + el_size * mono_array_length ((MonoArray*)(obj))); \
1320 /* Note: this code can handle also arrays of struct with only references in them */ \
1321 while (p < end_refs) { \
1322 HANDLE_PTR (p, (obj)); \
1325 } else if (etype == DESC_TYPE_V_RUN_LEN << 14) { \
1326 int offset = ((vt)->desc >> 16) & 0xff; \
1327 int num_refs = ((vt)->desc >> 24) & 0xff; \
1328 char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
1329 char *e_end = e_start + el_size * mono_array_length ((MonoArray*)(obj)); \
1330 while (e_start < e_end) { \
1331 void **p = (void**)e_start; \
1334 for (i = 0; i < num_refs; ++i) { \
1335 HANDLE_PTR (p + i, (obj)); \
1337 e_start += el_size; \
1339 } else if (etype == DESC_TYPE_V_BITMAP << 14) { \
1340 char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
1341 char *e_end = e_start + el_size * mono_array_length ((MonoArray*)(obj)); \
1342 while (e_start < e_end) { \
1343 void **p = (void**)e_start; \
1344 gsize _bmap = (vt)->desc >> 16; \
1345 /* Note: there is no object header here to skip */ \
1347 if ((_bmap & 1)) { \
1348 HANDLE_PTR (p, (obj)); \
1353 e_start += el_size; \
1359 #define COUNT_OBJECT_TYPES do { \
1360 switch (desc & 0x7) { \
1361 case DESC_TYPE_STRING: type_str++; break; \
1362 case DESC_TYPE_RUN_LENGTH: type_rlen++; break; \
1363 case DESC_TYPE_ARRAY: case DESC_TYPE_VECTOR: type_vector++; break; \
1364 case DESC_TYPE_SMALL_BITMAP: type_bitmap++; break; \
1365 case DESC_TYPE_LARGE_BITMAP: type_lbit++; break; \
1366 case DESC_TYPE_COMPLEX: type_complex++; break; \
1367 case DESC_TYPE_COMPLEX_ARR: type_complex++; break; \
1368 default: g_assert_not_reached (); \
1374 * ######################################################################
1375 * ######## Detecting and removing garbage.
1376 * ######################################################################
1377 * This section of code deals with detecting the objects no longer in use
1378 * and reclaiming the memory.
1382 static mword new_obj_references = 0;
1383 static mword obj_references_checked = 0;
1386 #define HANDLE_PTR(ptr,obj) do { \
1387 if (*(ptr) && (char*)*(ptr) >= nursery_start && (char*)*(ptr) < nursery_next) { \
1388 new_obj_references++; \
1389 /*printf ("bogus ptr %p found at %p in object %p (%s.%s)\n", *(ptr), (ptr), o, o->vtable->klass->name_space, o->vtable->klass->name);*/ \
1391 obj_references_checked++; \
1395 static void __attribute__((noinline))
1396 scan_area (char *start, char *end)
1399 int type_str = 0, type_rlen = 0, type_bitmap = 0, type_vector = 0, type_lbit = 0, type_complex = 0;
1400 new_obj_references = 0;
1401 obj_references_checked = 0;
1402 while (start < end) {
1403 if (!*(void**)start) {
1404 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
1407 vt = (GCVTable*)LOAD_VTABLE (start);
1408 DEBUG (8, fprintf (gc_debug_file, "Scanning object %p, vtable: %p (%s)\n", start, vt, vt->klass->name));
1410 MonoObject *obj = (MonoObject*)start;
1411 g_print ("found at %p (0x%zx): %s.%s\n", start, vt->desc, obj->vtable->klass->name_space, obj->vtable->klass->name);
1414 #define SCAN_OBJECT_ACTION COUNT_OBJECT_TYPES
1415 #include "sgen-scan-object.h"
1417 /*printf ("references to new nursery %p-%p (size: %dk): %d, checked: %d\n", old_start, end, (end-old_start)/1024, new_obj_references, obj_references_checked);
1418 printf ("\tstrings: %d, runl: %d, vector: %d, bitmaps: %d, lbitmaps: %d, complex: %d\n",
1419 type_str, type_rlen, type_vector, type_bitmap, type_lbit, type_complex);*/
1424 is_xdomain_ref_allowed (gpointer *ptr, char *obj, MonoDomain *domain)
1426 MonoObject *o = (MonoObject*)(obj);
1427 MonoObject *ref = (MonoObject*)*(ptr);
1428 int offset = (char*)(ptr) - (char*)o;
1430 if (o->vtable->klass == mono_defaults.thread_class && offset == G_STRUCT_OFFSET (MonoThread, internal_thread))
1432 if (o->vtable->klass == mono_defaults.internal_thread_class && offset == G_STRUCT_OFFSET (MonoInternalThread, current_appcontext))
1434 if (mono_class_has_parent (o->vtable->klass, mono_defaults.real_proxy_class) &&
1435 offset == G_STRUCT_OFFSET (MonoRealProxy, unwrapped_server))
1437 /* Thread.cached_culture_info */
1438 if (!strcmp (ref->vtable->klass->name_space, "System.Globalization") &&
1439 !strcmp (ref->vtable->klass->name, "CultureInfo") &&
1440 !strcmp(o->vtable->klass->name_space, "System") &&
1441 !strcmp(o->vtable->klass->name, "Object[]"))
1444 * at System.IO.MemoryStream.InternalConstructor (byte[],int,int,bool,bool) [0x0004d] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.IO/MemoryStream.cs:121
1445 * at System.IO.MemoryStream..ctor (byte[]) [0x00017] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.IO/MemoryStream.cs:81
1446 * at (wrapper remoting-invoke-with-check) System.IO.MemoryStream..ctor (byte[]) <IL 0x00020, 0xffffffff>
1447 * at System.Runtime.Remoting.Messaging.CADMethodCallMessage.GetArguments () [0x0000d] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Messaging/CADMessages.cs:327
1448 * at System.Runtime.Remoting.Messaging.MethodCall..ctor (System.Runtime.Remoting.Messaging.CADMethodCallMessage) [0x00017] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Messaging/MethodCall.cs:87
1449 * at System.AppDomain.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage,byte[]&,System.Runtime.Remoting.Messaging.CADMethodReturnMessage&) [0x00018] in /home/schani/Work/novell/trunk/mcs/class/corlib/System/AppDomain.cs:1213
1450 * at (wrapper remoting-invoke-with-check) System.AppDomain.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage,byte[]&,System.Runtime.Remoting.Messaging.CADMethodReturnMessage&) <IL 0x0003d, 0xffffffff>
1451 * at System.Runtime.Remoting.Channels.CrossAppDomainSink.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage) [0x00008] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Channels/CrossAppDomainChannel.cs:198
1452 * at (wrapper runtime-invoke) object.runtime_invoke_CrossAppDomainSink/ProcessMessageRes_object_object (object,intptr,intptr,intptr) <IL 0x0004c, 0xffffffff>
1454 if (!strcmp (ref->vtable->klass->name_space, "System") &&
1455 !strcmp (ref->vtable->klass->name, "Byte[]") &&
1456 !strcmp (o->vtable->klass->name_space, "System.IO") &&
1457 !strcmp (o->vtable->klass->name, "MemoryStream"))
1459 /* append_job() in threadpool.c */
1460 if (!strcmp (ref->vtable->klass->name_space, "System.Runtime.Remoting.Messaging") &&
1461 !strcmp (ref->vtable->klass->name, "AsyncResult") &&
1462 !strcmp (o->vtable->klass->name_space, "System") &&
1463 !strcmp (o->vtable->klass->name, "Object[]") &&
1464 mono_thread_pool_is_queue_array ((MonoArray*) o))
1470 check_reference_for_xdomain (gpointer *ptr, char *obj, MonoDomain *domain)
1472 MonoObject *o = (MonoObject*)(obj);
1473 MonoObject *ref = (MonoObject*)*(ptr);
1474 int offset = (char*)(ptr) - (char*)o;
1476 MonoClassField *field;
1479 if (!ref || ref->vtable->domain == domain)
1481 if (is_xdomain_ref_allowed (ptr, obj, domain))
1485 for (class = o->vtable->klass; class; class = class->parent) {
1488 for (i = 0; i < class->field.count; ++i) {
1489 if (class->fields[i].offset == offset) {
1490 field = &class->fields[i];
1498 if (ref->vtable->klass == mono_defaults.string_class)
1499 str = mono_string_to_utf8 ((MonoString*)ref);
1502 g_print ("xdomain reference in %p (%s.%s) at offset %d (%s) to %p (%s.%s) (%s) - pointed to by:\n",
1503 o, o->vtable->klass->name_space, o->vtable->klass->name,
1504 offset, field ? field->name : "",
1505 ref, ref->vtable->klass->name_space, ref->vtable->klass->name, str ? str : "");
1506 mono_gc_scan_for_specific_ref (o);
1512 #define HANDLE_PTR(ptr,obj) check_reference_for_xdomain ((ptr), (obj), domain)
1515 scan_object_for_xdomain_refs (char *start)
1517 MonoDomain *domain = ((MonoObject*)start)->vtable->domain;
1519 #include "sgen-scan-object.h"
1525 scan_area_for_xdomain_refs (char *start, char *end)
1527 while (start < end) {
1528 if (!*(void**)start) {
1529 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
1533 start = scan_object_for_xdomain_refs (start);
1538 #define HANDLE_PTR(ptr,obj) do { \
1539 if ((MonoObject*)*(ptr) == key) { \
1540 g_print ("found ref to %p in object %p (%s) at offset %zd\n", \
1541 key, (obj), safe_name ((obj)), ((char*)(ptr) - (char*)(obj))); \
1546 scan_object_for_specific_ref (char *start, MonoObject *key)
1548 #include "sgen-scan-object.h"
1554 scan_area_for_specific_ref (char *start, char *end, MonoObject *key)
1556 while (start < end) {
1557 if (!*(void**)start) {
1558 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
1562 start = scan_object_for_specific_ref (start, key);
1567 scan_pinned_object_for_specific_ref_callback (PinnedChunk *chunk, char *obj, size_t size, MonoObject *key)
1569 scan_object_for_specific_ref (obj, key);
1573 check_root_obj_specific_ref (RootRecord *root, MonoObject *key, MonoObject *obj)
1577 g_print ("found ref to %p in root record %p\n", key, root);
1580 static MonoObject *check_key = NULL;
1581 static RootRecord *check_root = NULL;
1584 check_root_obj_specific_ref_from_marker (void *obj)
1586 check_root_obj_specific_ref (check_root, check_key, obj);
1591 scan_roots_for_specific_ref (MonoObject *key, int root_type)
1596 for (i = 0; i < roots_hash_size [root_type]; ++i) {
1597 for (root = roots_hash [root_type][i]; root; root = root->next) {
1598 void **start_root = (void**)root->start_root;
1599 mword desc = root->root_desc;
1603 switch (desc & ROOT_DESC_TYPE_MASK) {
1604 case ROOT_DESC_BITMAP:
1605 desc >>= ROOT_DESC_TYPE_SHIFT;
1608 check_root_obj_specific_ref (root, key, *start_root);
1613 case ROOT_DESC_COMPLEX: {
1614 gsize *bitmap_data = complex_descriptors + (desc >> ROOT_DESC_TYPE_SHIFT);
1615 int bwords = (*bitmap_data) - 1;
1616 void **start_run = start_root;
1618 while (bwords-- > 0) {
1619 gsize bmap = *bitmap_data++;
1620 void **objptr = start_run;
1623 check_root_obj_specific_ref (root, key, *objptr);
1627 start_run += GC_BITS_PER_WORD;
1631 case ROOT_DESC_USER: {
1632 MonoGCMarkFunc marker = user_descriptors [desc >> ROOT_DESC_TYPE_SHIFT];
1633 marker (start_root, check_root_obj_specific_ref_from_marker);
1636 case ROOT_DESC_RUN_LEN:
1637 g_assert_not_reached ();
1639 g_assert_not_reached ();
1648 mono_gc_scan_for_specific_ref (MonoObject *key)
1650 GCMemSection *section;
1655 for (section = section_list; section; section = section->block.next)
1656 scan_area_for_specific_ref (section->data, section->end_data, key);
1658 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1659 scan_object_for_specific_ref (bigobj->data, key);
1661 scan_pinned_objects ((ScanPinnedObjectCallbackFunc)scan_pinned_object_for_specific_ref_callback, key);
1663 scan_roots_for_specific_ref (key, ROOT_TYPE_NORMAL);
1664 scan_roots_for_specific_ref (key, ROOT_TYPE_WBARRIER);
1666 for (i = 0; i < roots_hash_size [ROOT_TYPE_PINNED]; ++i) {
1667 for (root = roots_hash [ROOT_TYPE_PINNED][i]; root; root = root->next) {
1668 void **ptr = (void**)root->start_root;
1670 while (ptr < (void**)root->end_root) {
1671 check_root_obj_specific_ref (root, *ptr, key);
1678 //#define BINARY_PROTOCOL
1679 #include "sgen-protocol.c"
1682 need_remove_object_for_domain (char *start, MonoDomain *domain)
1684 if (mono_object_domain (start) == domain) {
1685 DEBUG (1, fprintf (gc_debug_file, "Need to cleanup object %p\n", start));
1686 binary_protocol_cleanup (start, (gpointer)LOAD_VTABLE (start), safe_object_get_size ((MonoObject*)start));
1693 process_object_for_domain_clearing (char *start, MonoDomain *domain)
1695 GCVTable *vt = (GCVTable*)LOAD_VTABLE (start);
1696 if (vt->klass == mono_defaults.internal_thread_class)
1697 g_assert (mono_object_domain (start) == mono_get_root_domain ());
1698 /* The object could be a proxy for an object in the domain
1700 if (mono_class_has_parent (vt->klass, mono_defaults.real_proxy_class)) {
1701 MonoObject *server = ((MonoRealProxy*)start)->unwrapped_server;
1703 /* The server could already have been zeroed out, so
1704 we need to check for that, too. */
1705 if (server && (!LOAD_VTABLE (server) || mono_object_domain (server) == domain)) {
1706 DEBUG (1, fprintf (gc_debug_file, "Cleaning up remote pointer in %p to object %p\n",
1708 ((MonoRealProxy*)start)->unwrapped_server = NULL;
1713 static void __attribute__((noinline))
1714 scan_area_for_domain (MonoDomain *domain, char *start, char *end)
1719 while (start < end) {
1720 if (!*(void**)start) {
1721 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
1724 vt = (GCVTable*)LOAD_VTABLE (start);
1725 process_object_for_domain_clearing (start, domain);
1726 remove = need_remove_object_for_domain (start, domain);
1727 if (remove && ((MonoObject*)start)->synchronisation) {
1728 void **dislink = mono_monitor_get_object_monitor_weak_link ((MonoObject*)start);
1730 mono_gc_register_disappearing_link (NULL, dislink, FALSE);
1733 #define SCAN_OBJECT_NOSCAN
1734 #define SCAN_OBJECT_ACTION do { \
1735 if (remove) memset (start, 0, skip_size); \
1737 #include "sgen-scan-object.h"
1741 static MonoDomain *check_domain = NULL;
1744 check_obj_not_in_domain (void *o)
1746 g_assert (((MonoObject*)o)->vtable->domain != check_domain);
1751 scan_for_registered_roots_in_domain (MonoDomain *domain, int root_type)
1755 check_domain = domain;
1756 for (i = 0; i < roots_hash_size [root_type]; ++i) {
1757 for (root = roots_hash [root_type][i]; root; root = root->next) {
1758 void **start_root = (void**)root->start_root;
1759 mword desc = root->root_desc;
1761 /* The MonoDomain struct is allowed to hold
1762 references to objects in its own domain. */
1763 if (start_root == (void**)domain)
1766 switch (desc & ROOT_DESC_TYPE_MASK) {
1767 case ROOT_DESC_BITMAP:
1768 desc >>= ROOT_DESC_TYPE_SHIFT;
1770 if ((desc & 1) && *start_root)
1771 check_obj_not_in_domain (*start_root);
1776 case ROOT_DESC_COMPLEX: {
1777 gsize *bitmap_data = complex_descriptors + (desc >> ROOT_DESC_TYPE_SHIFT);
1778 int bwords = (*bitmap_data) - 1;
1779 void **start_run = start_root;
1781 while (bwords-- > 0) {
1782 gsize bmap = *bitmap_data++;
1783 void **objptr = start_run;
1785 if ((bmap & 1) && *objptr)
1786 check_obj_not_in_domain (*objptr);
1790 start_run += GC_BITS_PER_WORD;
1794 case ROOT_DESC_USER: {
1795 MonoGCMarkFunc marker = user_descriptors [desc >> ROOT_DESC_TYPE_SHIFT];
1796 marker (start_root, check_obj_not_in_domain);
1799 case ROOT_DESC_RUN_LEN:
1800 g_assert_not_reached ();
1802 g_assert_not_reached ();
1806 check_domain = NULL;
1810 clear_domain_process_pinned_object_callback (PinnedChunk *chunk, char *obj, size_t size, MonoDomain *domain)
1812 process_object_for_domain_clearing (obj, domain);
1816 clear_domain_free_pinned_object_callback (PinnedChunk *chunk, char *obj, size_t size, MonoDomain *domain)
1818 if (need_remove_object_for_domain (obj, domain))
1819 free_pinned_object (chunk, obj, size);
1823 scan_pinned_object_for_xdomain_refs_callback (PinnedChunk *chunk, char *obj, size_t size, gpointer dummy)
1825 scan_object_for_xdomain_refs (obj);
1829 check_for_xdomain_refs (void)
1831 GCMemSection *section;
1834 for (section = section_list; section; section = section->block.next)
1835 scan_area_for_xdomain_refs (section->data, section->end_data);
1837 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1838 scan_object_for_xdomain_refs (bigobj->data);
1840 scan_pinned_objects (scan_pinned_object_for_xdomain_refs_callback, NULL);
1844 * When appdomains are unloaded we can easily remove objects that have finalizers,
1845 * but all the others could still be present in random places on the heap.
1846 * We need a sweep to get rid of them even though it's going to be costly
1848 * The reason we need to remove them is because we access the vtable and class
1849 * structures to know the object size and the reference bitmap: once the domain is
1850 * unloaded the point to random memory.
1853 mono_gc_clear_domain (MonoDomain * domain)
1855 GCMemSection *section;
1856 LOSObject *bigobj, *prev;
1861 /* Clear all remaining nursery fragments */
1862 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION) {
1863 g_assert (nursery_next <= nursery_frag_real_end);
1864 memset (nursery_next, 0, nursery_frag_real_end - nursery_next);
1865 for (frag = nursery_fragments; frag; frag = frag->next) {
1866 memset (frag->fragment_start, 0, frag->fragment_end - frag->fragment_start);
1870 if (xdomain_checks && domain != mono_get_root_domain ()) {
1871 scan_for_registered_roots_in_domain (domain, ROOT_TYPE_NORMAL);
1872 scan_for_registered_roots_in_domain (domain, ROOT_TYPE_WBARRIER);
1873 check_for_xdomain_refs ();
1876 for (section = section_list; section; section = section->block.next) {
1877 scan_area_for_domain (domain, section->data, section->end_data);
1880 /* We need two passes over pinned and large objects because
1881 freeing such an object gives its memory back to the OS (in
1882 the case of large objects) or obliterates its vtable
1883 (pinned objects), but we might need to dereference a
1884 pointer from an object to another object if the first
1885 object is a proxy. */
1886 scan_pinned_objects ((ScanPinnedObjectCallbackFunc)clear_domain_process_pinned_object_callback, domain);
1887 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1888 process_object_for_domain_clearing (bigobj->data, domain);
1891 for (bigobj = los_object_list; bigobj;) {
1892 if (need_remove_object_for_domain (bigobj->data, domain)) {
1893 LOSObject *to_free = bigobj;
1895 prev->next = bigobj->next;
1897 los_object_list = bigobj->next;
1898 bigobj = bigobj->next;
1899 DEBUG (1, fprintf (gc_debug_file, "Freeing large object %p\n",
1901 free_large_object (to_free);
1905 bigobj = bigobj->next;
1907 scan_pinned_objects ((ScanPinnedObjectCallbackFunc)clear_domain_free_pinned_object_callback, domain);
1909 for (i = GENERATION_NURSERY; i < GENERATION_MAX; ++i)
1910 null_links_for_domain (domain, i);
1916 * add_to_global_remset:
1918 * The global remset contains locations which point into newspace after
1919 * a minor collection. This can happen if the objects they point to are pinned.
1922 add_to_global_remset (gpointer ptr, gboolean root)
1926 DEBUG (8, fprintf (gc_debug_file, "Adding global remset for %p\n", ptr));
1927 binary_protocol_global_remset (ptr, *(gpointer*)ptr, (gpointer)LOAD_VTABLE (*(gpointer*)ptr));
1930 g_assert (!ptr_in_nursery (ptr) && ptr_in_nursery (*(gpointer*)ptr));
1932 HEAVY_STAT (++stat_global_remsets_added);
1935 * FIXME: If an object remains pinned, we need to add it at every minor collection.
1936 * To avoid uncontrolled growth of the global remset, only add each pointer once.
1938 if (global_remset->store_next + 3 < global_remset->end_set) {
1940 *(global_remset->store_next++) = (mword)ptr | REMSET_OTHER;
1941 *(global_remset->store_next++) = (mword)REMSET_ROOT_LOCATION;
1943 *(global_remset->store_next++) = (mword)ptr;
1947 rs = alloc_remset (global_remset->end_set - global_remset->data, NULL);
1948 rs->next = global_remset;
1951 *(global_remset->store_next++) = (mword)ptr | REMSET_OTHER;
1952 *(global_remset->store_next++) = (mword)REMSET_ROOT_LOCATION;
1954 *(global_remset->store_next++) = (mword)ptr;
1958 int global_rs_size = 0;
1960 for (rs = global_remset; rs; rs = rs->next) {
1961 global_rs_size += rs->store_next - rs->data;
1963 DEBUG (4, fprintf (gc_debug_file, "Global remset now has size %d\n", global_rs_size));
1967 #define MOVED_OBJECTS_NUM 64
1968 static void *moved_objects [MOVED_OBJECTS_NUM];
1969 static int moved_objects_idx = 0;
1971 #include "sgen-gray.c"
1974 * This is how the copying happens from the nursery to the old generation.
1975 * We assume that at this time all the pinned objects have been identified and
1977 * We run scan_object() for each pinned object so that each referenced
1978 * objects if possible are copied. The new gray objects created can have
1979 * scan_object() run on them right away, too.
1980 * Then we run copy_object() for the precisely tracked roots. At this point
1981 * all the roots are either gray or black. We run scan_object() on the gray
1982 * objects until no more gray objects are created.
1983 * At the end of the process we walk again the pinned list and we unmark
1984 * the pinned flag. As we go we also create the list of free space for use
1985 * in the next allocation runs.
1987 * We need to remember objects from the old generation that point to the new one
1988 * (or just addresses?).
1990 * copy_object could be made into a macro once debugged (use inline for now).
1993 static char* __attribute__((noinline))
1994 copy_object (char *obj, char *from_space_start, char *from_space_end)
1996 static void *copy_labels [] = { &&LAB_0, &&LAB_1, &&LAB_2, &&LAB_3, &&LAB_4, &&LAB_5, &&LAB_6, &&LAB_7, &&LAB_8 };
2002 HEAVY_STAT (++num_copy_object_called);
2004 if (!(obj >= from_space_start && obj < from_space_end)) {
2005 DEBUG (9, fprintf (gc_debug_file, "Not copying %p because it's not in from space (%p-%p)\n",
2006 obj, from_space_start, from_space_end));
2007 HEAVY_STAT (++stat_copy_object_failed_from_space);
2011 DEBUG (9, fprintf (gc_debug_file, "Precise copy of %p", obj));
2014 * obj must belong to one of:
2019 * 4. a non-to-space section of the major heap
2020 * 5. a to-space section of the major heap
2022 * In addition, objects in 1, 2 and 4 might also be pinned.
2023 * Objects in 1 and 4 might be forwarded.
2025 * Before we can copy the object we must make sure that we are
2026 * allowed to, i.e. that the object not pinned, not already
2027 * forwarded and doesn't belong to the LOS, a pinned chunk, or
2028 * a to-space section.
2030 * We are usually called for to-space objects (5) when we have
2031 * two remset entries for the same reference. The first entry
2032 * copies the object and updates the reference and the second
2033 * calls us with the updated reference that points into
2034 * to-space. There might also be other circumstances where we
2035 * get to-space objects.
2038 if ((forwarded = object_is_forwarded (obj))) {
2039 g_assert (((MonoVTable*)LOAD_VTABLE(obj))->gc_descr);
2040 DEBUG (9, fprintf (gc_debug_file, " (already forwarded to %p)\n", forwarded));
2041 HEAVY_STAT (++stat_copy_object_failed_forwarded);
2044 if (object_is_pinned (obj)) {
2045 g_assert (((MonoVTable*)LOAD_VTABLE(obj))->gc_descr);
2046 DEBUG (9, fprintf (gc_debug_file, " (pinned, no change)\n"));
2047 HEAVY_STAT (++stat_copy_object_failed_pinned);
2051 objsize = safe_object_get_size ((MonoObject*)obj);
2052 objsize += ALLOC_ALIGN - 1;
2053 objsize &= ~(ALLOC_ALIGN - 1);
2055 if (ptr_in_nursery (obj))
2059 * At this point we know obj is not pinned, not forwarded and
2060 * belongs to 2, 3, 4, or 5.
2062 * LOS object (2) are simple, at least until we always follow
2063 * the rule: if objsize > MAX_SMALL_OBJ_SIZE, pin the object
2064 * and return it. At the end of major collections, we walk
2065 * the los list and if the object is pinned, it is marked,
2066 * otherwise it can be freed.
2068 * Pinned chunks (3) and major heap sections (4, 5) both
2069 * reside in blocks, which are always aligned, so once we've
2070 * eliminated LOS objects, we can just access the block and
2071 * see whether it's a pinned chunk or a major heap section.
2073 if (G_UNLIKELY (objsize > MAX_SMALL_OBJ_SIZE || obj_is_from_pinned_alloc (obj))) {
2074 DEBUG (9, fprintf (gc_debug_file, " (marked LOS/Pinned %p (%s), size: %zd)\n", obj, safe_name (obj), objsize));
2075 binary_protocol_pin (obj, (gpointer)LOAD_VTABLE (obj), safe_object_get_size ((MonoObject*)obj));
2077 HEAVY_STAT (++stat_copy_object_failed_large_pinned);
2082 * Now we know the object is in a major heap section. All we
2083 * need to do is check whether it's already in to-space (5) or
2086 if (MAJOR_SECTION_FOR_OBJECT (obj)->is_to_space) {
2087 g_assert (objsize <= MAX_SMALL_OBJ_SIZE);
2088 DEBUG (9, fprintf (gc_debug_file, " (already copied)\n"));
2089 HEAVY_STAT (++stat_copy_object_failed_to_space);
2094 DEBUG (9, fprintf (gc_debug_file, " (to %p, %s size: %zd)\n", to_space_bumper, ((MonoObject*)obj)->vtable->klass->name, objsize));
2095 binary_protocol_copy (obj, to_space_bumper, ((MonoObject*)obj)->vtable, objsize);
2097 HEAVY_STAT (++num_objects_copied);
2099 /* Make sure we have enough space available */
2100 if (to_space_bumper + objsize > to_space_top) {
2102 g_assert (to_space_bumper + objsize <= to_space_top);
2105 if (objsize <= sizeof (gpointer) * 8) {
2106 mword *dest = (mword*)to_space_bumper;
2107 goto *copy_labels [objsize / sizeof (gpointer)];
2109 (dest) [7] = ((mword*)obj) [7];
2111 (dest) [6] = ((mword*)obj) [6];
2113 (dest) [5] = ((mword*)obj) [5];
2115 (dest) [4] = ((mword*)obj) [4];
2117 (dest) [3] = ((mword*)obj) [3];
2119 (dest) [2] = ((mword*)obj) [2];
2121 (dest) [1] = ((mword*)obj) [1];
2123 (dest) [0] = ((mword*)obj) [0];
2131 char* edi = to_space_bumper;
2132 __asm__ __volatile__(
2134 : "=&c" (ecx), "=&D" (edi), "=&S" (esi)
2135 : "0" (objsize/4), "1" (edi),"2" (esi)
2140 memcpy (to_space_bumper, obj, objsize);
2143 /* adjust array->bounds */
2144 vt = ((MonoObject*)obj)->vtable;
2145 g_assert (vt->gc_descr);
2146 if (G_UNLIKELY (vt->rank && ((MonoArray*)obj)->bounds)) {
2147 MonoArray *array = (MonoArray*)to_space_bumper;
2148 array->bounds = (MonoArrayBounds*)((char*)to_space_bumper + ((char*)((MonoArray*)obj)->bounds - (char*)obj));
2149 DEBUG (9, fprintf (gc_debug_file, "Array instance %p: size: %zd, rank: %d, length: %d\n", array, objsize, vt->rank, mono_array_length (array)));
2151 /* set the forwarding pointer */
2152 forward_object (obj, to_space_bumper);
2153 if (G_UNLIKELY (mono_profiler_events & MONO_PROFILE_GC_MOVES)) {
2154 if (moved_objects_idx == MOVED_OBJECTS_NUM) {
2155 mono_profiler_gc_moves (moved_objects, moved_objects_idx);
2156 moved_objects_idx = 0;
2158 moved_objects [moved_objects_idx++] = obj;
2159 moved_objects [moved_objects_idx++] = to_space_bumper;
2161 obj = to_space_bumper;
2162 to_space_section->scan_starts [((char*)obj - (char*)to_space_section->data)/SCAN_START_SIZE] = obj;
2163 to_space_bumper += objsize;
2164 DEBUG (9, fprintf (gc_debug_file, "Enqueuing gray object %p (%s)\n", obj, safe_name (obj)));
2165 gray_object_enqueue (obj);
2166 DEBUG (8, g_assert (to_space_bumper <= to_space_top));
2171 #define HANDLE_PTR(ptr,obj) do { \
2172 void *__old = *(ptr); \
2175 *(ptr) = __copy = copy_object (__old, from_start, from_end); \
2176 DEBUG (9, if (__old != __copy) fprintf (gc_debug_file, "Overwrote field at %p with %p (was: %p)\n", (ptr), *(ptr), __old)); \
2177 if (G_UNLIKELY (ptr_in_nursery (__copy) && !ptr_in_nursery ((ptr)))) \
2178 add_to_global_remset ((ptr), FALSE); \
2183 * Scan the object pointed to by @start for references to
2184 * other objects between @from_start and @from_end and copy
2185 * them to the gray_objects area.
2186 * Returns a pointer to the end of the object.
2189 scan_object (char *start, char* from_start, char* from_end)
2191 #include "sgen-scan-object.h"
2199 * Scan objects in the gray stack until the stack is empty. This should be called
2200 * frequently after each object is copied, to achieve better locality and cache
2204 drain_gray_stack (char *start_addr, char *end_addr)
2208 while ((obj = gray_object_dequeue ())) {
2209 DEBUG (9, fprintf (gc_debug_file, "Precise gray object scan %p (%s)\n", obj, safe_name (obj)));
2210 scan_object (obj, start_addr, end_addr);
2217 * Scan the valuetype pointed to by START, described by DESC for references to
2218 * other objects between @from_start and @from_end and copy them to the gray_objects area.
2219 * Returns a pointer to the end of the object.
2222 scan_vtype (char *start, mword desc, char* from_start, char* from_end)
2226 /* The descriptors include info about the MonoObject header as well */
2227 start -= sizeof (MonoObject);
2229 switch (desc & 0x7) {
2230 case DESC_TYPE_RUN_LENGTH:
2231 OBJ_RUN_LEN_FOREACH_PTR (desc,start);
2232 OBJ_RUN_LEN_SIZE (skip_size, desc, start);
2233 g_assert (skip_size);
2234 return start + skip_size;
2235 case DESC_TYPE_SMALL_BITMAP:
2236 OBJ_BITMAP_FOREACH_PTR (desc,start);
2237 OBJ_BITMAP_SIZE (skip_size, desc, start);
2238 return start + skip_size;
2239 case DESC_TYPE_LARGE_BITMAP:
2240 case DESC_TYPE_COMPLEX:
2242 g_assert_not_reached ();
2245 // The other descriptors can't happen with vtypes
2246 g_assert_not_reached ();
2252 #include "sgen-pinning-stats.c"
2255 * Addresses from start to end are already sorted. This function finds
2256 * the object header for each address and pins the object. The
2257 * addresses must be inside the passed section. The (start of the)
2258 * address array is overwritten with the addresses of the actually
2259 * pinned objects. Return the number of pinned objects.
2262 pin_objects_from_addresses (GCMemSection *section, void **start, void **end, void *start_nursery, void *end_nursery)
2267 void *last_obj = NULL;
2268 size_t last_obj_size = 0;
2271 void **definitely_pinned = start;
2272 while (start < end) {
2274 /* the range check should be reduntant */
2275 if (addr != last && addr >= start_nursery && addr < end_nursery) {
2276 DEBUG (5, fprintf (gc_debug_file, "Considering pinning addr %p\n", addr));
2277 /* multiple pointers to the same object */
2278 if (addr >= last_obj && (char*)addr < (char*)last_obj + last_obj_size) {
2282 idx = ((char*)addr - (char*)section->data) / SCAN_START_SIZE;
2283 g_assert (idx < section->num_scan_start);
2284 search_start = (void*)section->scan_starts [idx];
2285 if (!search_start || search_start > addr) {
2288 search_start = section->scan_starts [idx];
2289 if (search_start && search_start <= addr)
2292 if (!search_start || search_start > addr)
2293 search_start = start_nursery;
2295 if (search_start < last_obj)
2296 search_start = (char*)last_obj + last_obj_size;
2297 /* now addr should be in an object a short distance from search_start
2298 * Note that search_start must point to zeroed mem or point to an object.
2301 if (!*(void**)search_start) {
2302 mword p = (mword)search_start;
2303 p += sizeof (gpointer);
2304 p += ALLOC_ALIGN - 1;
2305 p &= ~(ALLOC_ALIGN - 1);
2306 search_start = (void*)p;
2309 last_obj = search_start;
2310 last_obj_size = safe_object_get_size ((MonoObject*)search_start);
2311 last_obj_size += ALLOC_ALIGN - 1;
2312 last_obj_size &= ~(ALLOC_ALIGN - 1);
2313 DEBUG (8, fprintf (gc_debug_file, "Pinned try match %p (%s), size %zd\n", last_obj, safe_name (last_obj), last_obj_size));
2314 if (addr >= search_start && (char*)addr < (char*)last_obj + last_obj_size) {
2315 DEBUG (4, fprintf (gc_debug_file, "Pinned object %p, vtable %p (%s), count %d\n", search_start, *(void**)search_start, safe_name (search_start), count));
2316 binary_protocol_pin (search_start, (gpointer)LOAD_VTABLE (search_start), safe_object_get_size (search_start));
2317 pin_object (search_start);
2319 pin_stats_register_object (search_start, last_obj_size);
2320 definitely_pinned [count] = search_start;
2324 /* skip to the next object */
2325 search_start = (void*)((char*)search_start + last_obj_size);
2326 } while (search_start <= addr);
2327 /* we either pinned the correct object or we ignored the addr because
2328 * it points to unused zeroed memory.
2334 //printf ("effective pinned: %d (at the end: %d)\n", count, (char*)end_nursery - (char*)last);
2338 static void** pin_queue;
2339 static int pin_queue_size = 0;
2340 static int next_pin_slot = 0;
2345 gap = (gap * 10) / 13;
2346 if (gap == 9 || gap == 10)
2355 compare_addr (const void *a, const void *b)
2357 return *(const void **)a - *(const void **)b;
2361 /* sort the addresses in array in increasing order */
2363 sort_addresses (void **array, int size)
2366 * qsort is slower as predicted.
2367 * qsort (array, size, sizeof (gpointer), compare_addr);
2374 gap = new_gap (gap);
2377 for (i = 0; i < end; i++) {
2379 if (array [i] > array [j]) {
2380 void* val = array [i];
2381 array [i] = array [j];
2386 if (gap == 1 && !swapped)
2391 static G_GNUC_UNUSED void
2392 print_nursery_gaps (void* start_nursery, void *end_nursery)
2395 gpointer first = start_nursery;
2397 for (i = 0; i < next_pin_slot; ++i) {
2398 next = pin_queue [i];
2399 fprintf (gc_debug_file, "Nursery range: %p-%p, size: %zd\n", first, next, (char*)next-(char*)first);
2403 fprintf (gc_debug_file, "Nursery range: %p-%p, size: %zd\n", first, next, (char*)next-(char*)first);
2406 /* reduce the info in the pin queue, removing duplicate pointers and sorting them */
2408 optimize_pin_queue (int start_slot)
2410 void **start, **cur, **end;
2411 /* sort and uniq pin_queue: we just sort and we let the rest discard multiple values */
2412 /* it may be better to keep ranges of pinned memory instead of individually pinning objects */
2413 DEBUG (5, fprintf (gc_debug_file, "Sorting pin queue, size: %d\n", next_pin_slot));
2414 if ((next_pin_slot - start_slot) > 1)
2415 sort_addresses (pin_queue + start_slot, next_pin_slot - start_slot);
2416 start = cur = pin_queue + start_slot;
2417 end = pin_queue + next_pin_slot;
2420 while (*start == *cur && cur < end)
2424 next_pin_slot = start - pin_queue;
2425 DEBUG (5, fprintf (gc_debug_file, "Pin queue reduced to size: %d\n", next_pin_slot));
2426 //DEBUG (6, print_nursery_gaps (start_nursery, end_nursery));
2431 optimized_pin_queue_search (void *addr)
2433 int first = 0, last = next_pin_slot;
2434 while (first < last) {
2435 int middle = first + ((last - first) >> 1);
2436 if (addr <= pin_queue [middle])
2441 g_assert (first == last);
2446 find_optimized_pin_queue_area (void *start, void *end, int *first, int *last)
2448 *first = optimized_pin_queue_search (start);
2449 *last = optimized_pin_queue_search (end);
2453 realloc_pin_queue (void)
2455 int new_size = pin_queue_size? pin_queue_size + pin_queue_size/2: 1024;
2456 void **new_pin = get_internal_mem (sizeof (void*) * new_size, INTERNAL_MEM_PIN_QUEUE);
2457 memcpy (new_pin, pin_queue, sizeof (void*) * next_pin_slot);
2458 free_internal_mem (pin_queue, INTERNAL_MEM_PIN_QUEUE);
2459 pin_queue = new_pin;
2460 pin_queue_size = new_size;
2461 DEBUG (4, fprintf (gc_debug_file, "Reallocated pin queue to size: %d\n", new_size));
2464 #include "sgen-pinning.c"
2467 * Scan the memory between start and end and queue values which could be pointers
2468 * to the area between start_nursery and end_nursery for later consideration.
2469 * Typically used for thread stacks.
2472 conservatively_pin_objects_from (void **start, void **end, void *start_nursery, void *end_nursery, int pin_type)
2475 while (start < end) {
2476 if (*start >= start_nursery && *start < end_nursery) {
2478 * *start can point to the middle of an object
2479 * note: should we handle pointing at the end of an object?
2480 * pinning in C# code disallows pointing at the end of an object
2481 * but there is some small chance that an optimizing C compiler
2482 * may keep the only reference to an object by pointing
2483 * at the end of it. We ignore this small chance for now.
2484 * Pointers to the end of an object are indistinguishable
2485 * from pointers to the start of the next object in memory
2486 * so if we allow that we'd need to pin two objects...
2487 * We queue the pointer in an array, the
2488 * array will then be sorted and uniqued. This way
2489 * we can coalesce several pinning pointers and it should
2490 * be faster since we'd do a memory scan with increasing
2491 * addresses. Note: we can align the address to the allocation
2492 * alignment, so the unique process is more effective.
2494 mword addr = (mword)*start;
2495 addr &= ~(ALLOC_ALIGN - 1);
2496 if (addr >= (mword)start_nursery && addr < (mword)end_nursery)
2497 pin_stage_ptr ((void*)addr);
2499 pin_stats_register_address ((char*)addr, pin_type);
2500 DEBUG (6, if (count) fprintf (gc_debug_file, "Pinning address %p\n", (void*)addr));
2505 DEBUG (7, if (count) fprintf (gc_debug_file, "found %d potential pinned heap pointers\n", count));
2509 * If generation is 0, just mark objects in the nursery, the others we don't care,
2510 * since they are not going to move anyway.
2511 * There are different areas that are scanned for pinned pointers:
2512 * *) the thread stacks (when jit support is ready only the unmanaged frames)
2513 * *) the pinned handle table
2514 * *) the pinned roots
2516 * Note: when we'll use a write barrier for old to new gen references, we need to
2517 * keep track of old gen objects that point to pinned new gen objects because in that
2518 * case the referenced object will be moved maybe at the next collection, but there
2519 * is no write in the old generation area where the pinned object is referenced
2520 * and we may not consider it as reachable.
2522 static G_GNUC_UNUSED void
2523 mark_pinned_objects (int generation)
2528 * Debugging function: find in the conservative roots where @obj is being pinned.
2530 static G_GNUC_UNUSED void
2531 find_pinning_reference (char *obj, size_t size)
2535 char *endobj = obj + size;
2536 for (i = 0; i < roots_hash_size [0]; ++i) {
2537 for (root = roots_hash [0][i]; root; root = root->next) {
2538 /* if desc is non-null it has precise info */
2539 if (!root->root_desc) {
2540 char ** start = (char**)root->start_root;
2541 while (start < (char**)root->end_root) {
2542 if (*start >= obj && *start < endobj) {
2543 DEBUG (0, fprintf (gc_debug_file, "Object %p referenced in pinned roots %p-%p (at %p in record %p)\n", obj, root->start_root, root->end_root, start, root));
2550 find_pinning_ref_from_thread (obj, size);
2554 * The first thing we do in a collection is to identify pinned objects.
2555 * This function considers all the areas of memory that need to be
2556 * conservatively scanned.
2559 pin_from_roots (void *start_nursery, void *end_nursery)
2563 DEBUG (2, fprintf (gc_debug_file, "Scanning pinned roots (%d bytes, %d/%d entries)\n", (int)roots_size, num_roots_entries [ROOT_TYPE_NORMAL], num_roots_entries [ROOT_TYPE_PINNED]));
2564 /* objects pinned from the API are inside these roots */
2565 for (i = 0; i < roots_hash_size [ROOT_TYPE_PINNED]; ++i) {
2566 for (root = roots_hash [ROOT_TYPE_PINNED][i]; root; root = root->next) {
2567 DEBUG (6, fprintf (gc_debug_file, "Pinned roots %p-%p\n", root->start_root, root->end_root));
2568 conservatively_pin_objects_from ((void**)root->start_root, (void**)root->end_root, start_nursery, end_nursery, PIN_TYPE_OTHER);
2571 /* now deal with the thread stacks
2572 * in the future we should be able to conservatively scan only:
2573 * *) the cpu registers
2574 * *) the unmanaged stack frames
2575 * *) the _last_ managed stack frame
2576 * *) pointers slots in managed frames
2578 scan_thread_data (start_nursery, end_nursery, FALSE);
2580 evacuate_pin_staging_area ();
2583 /* Copy function called from user defined mark functions */
2584 static char *user_copy_n_start;
2585 static char *user_copy_n_end;
2588 user_copy (void *addr)
2591 return copy_object (addr, user_copy_n_start, user_copy_n_end);
2597 * The memory area from start_root to end_root contains pointers to objects.
2598 * Their position is precisely described by @desc (this means that the pointer
2599 * can be either NULL or the pointer to the start of an object).
2600 * This functions copies them to to_space updates them.
2603 precisely_scan_objects_from (void** start_root, void** end_root, char* n_start, char *n_end, mword desc)
2605 switch (desc & ROOT_DESC_TYPE_MASK) {
2606 case ROOT_DESC_BITMAP:
2607 desc >>= ROOT_DESC_TYPE_SHIFT;
2609 if ((desc & 1) && *start_root) {
2610 *start_root = copy_object (*start_root, n_start, n_end);
2611 DEBUG (9, fprintf (gc_debug_file, "Overwrote root at %p with %p\n", start_root, *start_root));
2612 drain_gray_stack (n_start, n_end);
2618 case ROOT_DESC_COMPLEX: {
2619 gsize *bitmap_data = complex_descriptors + (desc >> ROOT_DESC_TYPE_SHIFT);
2620 int bwords = (*bitmap_data) - 1;
2621 void **start_run = start_root;
2623 while (bwords-- > 0) {
2624 gsize bmap = *bitmap_data++;
2625 void **objptr = start_run;
2627 if ((bmap & 1) && *objptr) {
2628 *objptr = copy_object (*objptr, n_start, n_end);
2629 DEBUG (9, fprintf (gc_debug_file, "Overwrote root at %p with %p\n", objptr, *objptr));
2630 drain_gray_stack (n_start, n_end);
2635 start_run += GC_BITS_PER_WORD;
2639 case ROOT_DESC_USER: {
2640 MonoGCMarkFunc marker = user_descriptors [desc >> ROOT_DESC_TYPE_SHIFT];
2642 user_copy_n_start = n_start;
2643 user_copy_n_end = n_end;
2644 marker (start_root, user_copy);
2647 case ROOT_DESC_RUN_LEN:
2648 g_assert_not_reached ();
2650 g_assert_not_reached ();
2655 alloc_fragment (void)
2657 Fragment *frag = fragment_freelist;
2659 fragment_freelist = frag->next;
2663 frag = get_internal_mem (sizeof (Fragment), INTERNAL_MEM_FRAGMENT);
2668 /* size must be a power of 2 */
2670 get_os_memory_aligned (mword size, gboolean activate)
2672 /* Allocate twice the memory to be able to put the block on an aligned address */
2673 char *mem = get_os_memory (size * 2, activate);
2678 aligned = (char*)((mword)(mem + (size - 1)) & ~(size - 1));
2679 g_assert (aligned >= mem && aligned + size <= mem + size * 2 && !((mword)aligned & (size - 1)));
2682 free_os_memory (mem, aligned - mem);
2683 if (aligned + size < mem + size * 2)
2684 free_os_memory (aligned + size, (mem + size * 2) - (aligned + size));
2690 * Allocate and setup the data structures needed to be able to allocate objects
2691 * in the nursery. The nursery is stored in nursery_section.
2694 alloc_nursery (void)
2696 GCMemSection *section;
2702 if (nursery_section)
2704 DEBUG (2, fprintf (gc_debug_file, "Allocating nursery size: %zd\n", nursery_size));
2705 /* later we will alloc a larger area for the nursery but only activate
2706 * what we need. The rest will be used as expansion if we have too many pinned
2707 * objects in the existing nursery.
2709 /* FIXME: handle OOM */
2710 section = get_internal_mem (SIZEOF_GC_MEM_SECTION, INTERNAL_MEM_SECTION);
2712 g_assert (nursery_size == DEFAULT_NURSERY_SIZE);
2713 alloc_size = nursery_size;
2714 #ifdef ALIGN_NURSERY
2715 data = get_os_memory_aligned (alloc_size, TRUE);
2717 data = get_os_memory (alloc_size, TRUE);
2719 nursery_start = data;
2720 nursery_real_end = nursery_start + nursery_size;
2721 UPDATE_HEAP_BOUNDARIES (nursery_start, nursery_real_end);
2722 nursery_next = nursery_start;
2723 total_alloc += alloc_size;
2724 DEBUG (4, fprintf (gc_debug_file, "Expanding nursery size (%p-%p): %zd, total: %zd\n", data, data + alloc_size, nursery_size, total_alloc));
2725 section->data = section->next_data = data;
2726 section->size = alloc_size;
2727 section->end_data = nursery_real_end;
2728 scan_starts = (alloc_size + SCAN_START_SIZE - 1) / SCAN_START_SIZE;
2729 section->scan_starts = get_internal_mem (sizeof (char*) * scan_starts, INTERNAL_MEM_SCAN_STARTS);
2730 section->num_scan_start = scan_starts;
2731 section->block.role = MEMORY_ROLE_GEN0;
2733 /* add to the section list */
2734 section->block.next = section_list;
2735 section_list = section;
2737 nursery_section = section;
2739 /* Setup the single first large fragment */
2740 frag = alloc_fragment ();
2741 frag->fragment_start = nursery_start;
2742 frag->fragment_limit = nursery_start;
2743 frag->fragment_end = nursery_real_end;
2744 nursery_frag_real_end = nursery_real_end;
2745 /* FIXME: frag here is lost */
2749 scan_finalizer_entries (FinalizeEntry *list, char *start, char *end) {
2752 for (fin = list; fin; fin = fin->next) {
2755 DEBUG (5, fprintf (gc_debug_file, "Scan of fin ready object: %p (%s)\n", fin->object, safe_name (fin->object)));
2756 fin->object = copy_object (fin->object, start, end);
2761 * Update roots in the old generation. Since we currently don't have the
2762 * info from the write barriers, we just scan all the objects.
2764 static G_GNUC_UNUSED void
2765 scan_old_generation (char *start, char* end)
2767 GCMemSection *section;
2768 LOSObject *big_object;
2771 for (section = section_list; section; section = section->block.next) {
2772 if (section == nursery_section)
2774 DEBUG (2, fprintf (gc_debug_file, "Scan of old section: %p-%p, size: %d\n", section->data, section->next_data, (int)(section->next_data - section->data)));
2775 /* we have to deal with zeroed holes in old generation (truncated strings ...) */
2777 while (p < section->next_data) {
2782 DEBUG (8, fprintf (gc_debug_file, "Precise old object scan of %p (%s)\n", p, safe_name (p)));
2783 p = scan_object (p, start, end);
2786 /* scan the old object space, too */
2787 for (big_object = los_object_list; big_object; big_object = big_object->next) {
2788 DEBUG (5, fprintf (gc_debug_file, "Scan of big object: %p (%s), size: %zd\n", big_object->data, safe_name (big_object->data), big_object->size));
2789 scan_object (big_object->data, start, end);
2791 /* scan the list of objects ready for finalization */
2792 scan_finalizer_entries (fin_ready_list, start, end);
2793 scan_finalizer_entries (critical_fin_list, start, end);
2796 static mword fragment_total = 0;
2798 * We found a fragment of free memory in the nursery: memzero it and if
2799 * it is big enough, add it to the list of fragments that can be used for
2803 add_nursery_frag (size_t frag_size, char* frag_start, char* frag_end)
2806 DEBUG (4, fprintf (gc_debug_file, "Found empty fragment: %p-%p, size: %zd\n", frag_start, frag_end, frag_size));
2807 binary_protocol_empty (frag_start, frag_size);
2808 /* memsetting just the first chunk start is bound to provide better cache locality */
2809 if (nursery_clear_policy == CLEAR_AT_GC)
2810 memset (frag_start, 0, frag_size);
2811 /* Not worth dealing with smaller fragments: need to tune */
2812 if (frag_size >= FRAGMENT_MIN_SIZE) {
2813 fragment = alloc_fragment ();
2814 fragment->fragment_start = frag_start;
2815 fragment->fragment_limit = frag_start;
2816 fragment->fragment_end = frag_end;
2817 fragment->next = nursery_fragments;
2818 nursery_fragments = fragment;
2819 fragment_total += frag_size;
2821 /* Clear unused fragments, pinning depends on this */
2822 memset (frag_start, 0, frag_size);
2827 scan_needed_big_objects (char *start_addr, char *end_addr)
2829 LOSObject *big_object;
2831 for (big_object = los_object_list; big_object; big_object = big_object->next) {
2832 if (!big_object->scanned && object_is_pinned (big_object->data)) {
2833 DEBUG (5, fprintf (gc_debug_file, "Scan of big object: %p (%s), size: %zd\n", big_object->data, safe_name (big_object->data), big_object->size));
2834 scan_object (big_object->data, start_addr, end_addr);
2835 big_object->scanned = TRUE;
2843 generation_name (int generation)
2845 switch (generation) {
2846 case GENERATION_NURSERY: return "nursery";
2847 case GENERATION_OLD: return "old";
2848 default: g_assert_not_reached ();
2852 static DisappearingLinkHashTable*
2853 get_dislink_hash_table (int generation)
2855 switch (generation) {
2856 case GENERATION_NURSERY: return &minor_disappearing_link_hash;
2857 case GENERATION_OLD: return &major_disappearing_link_hash;
2858 default: g_assert_not_reached ();
2862 static FinalizeEntryHashTable*
2863 get_finalize_entry_hash_table (int generation)
2865 switch (generation) {
2866 case GENERATION_NURSERY: return &minor_finalizable_hash;
2867 case GENERATION_OLD: return &major_finalizable_hash;
2868 default: g_assert_not_reached ();
2873 new_to_space_section (void)
2875 /* FIXME: if the current to_space_section is empty, we don't
2876 have to allocate a new one */
2878 to_space_section = alloc_major_section ();
2879 to_space_bumper = to_space_section->next_data;
2880 to_space_top = to_space_section->end_data;
2884 to_space_set_next_data (void)
2886 g_assert (to_space_bumper >= to_space_section->next_data && to_space_bumper <= to_space_section->end_data);
2887 to_space_section->next_data = to_space_bumper;
2891 to_space_expand (void)
2893 if (to_space_section) {
2894 g_assert (to_space_top == to_space_section->end_data);
2895 to_space_set_next_data ();
2898 new_to_space_section ();
2902 unset_to_space (void)
2904 /* between collections the to_space_bumper is invalidated
2905 because degraded allocations might occur, so we set it to
2906 NULL, just to make it explicit */
2907 to_space_bumper = NULL;
2909 /* don't unset to_space_section if we implement the FIXME in
2910 new_to_space_section */
2911 to_space_section = NULL;
2915 object_is_in_to_space (char *obj)
2920 if (ptr_in_nursery (obj))
2923 objsize = safe_object_get_size ((MonoObject*)obj);
2924 objsize += ALLOC_ALIGN - 1;
2925 objsize &= ~(ALLOC_ALIGN - 1);
2928 if (objsize > MAX_SMALL_OBJ_SIZE)
2932 if (obj_is_from_pinned_alloc (obj))
2935 /* now we know it's in a major heap section */
2936 return MAJOR_SECTION_FOR_OBJECT (obj)->is_to_space;
2940 finish_gray_stack (char *start_addr, char *end_addr, int generation)
2944 int fin_ready, bigo_scanned_num;
2947 * We copied all the reachable objects. Now it's the time to copy
2948 * the objects that were not referenced by the roots, but by the copied objects.
2949 * we built a stack of objects pointed to by gray_start: they are
2950 * additional roots and we may add more items as we go.
2951 * We loop until gray_start == gray_objects which means no more objects have
2952 * been added. Note this is iterative: no recursion is involved.
2953 * We need to walk the LO list as well in search of marked big objects
2954 * (use a flag since this is needed only on major collections). We need to loop
2955 * here as well, so keep a counter of marked LO (increasing it in copy_object).
2956 * To achieve better cache locality and cache usage, we drain the gray stack
2957 * frequently, after each object is copied, and just finish the work here.
2959 drain_gray_stack (start_addr, end_addr);
2961 //scan_old_generation (start_addr, end_addr);
2962 DEBUG (2, fprintf (gc_debug_file, "%s generation done\n", generation_name (generation)));
2963 /* walk the finalization queue and move also the objects that need to be
2964 * finalized: use the finalized objects as new roots so the objects they depend
2965 * on are also not reclaimed. As with the roots above, only objects in the nursery
2966 * are marked/copied.
2967 * We need a loop here, since objects ready for finalizers may reference other objects
2968 * that are fin-ready. Speedup with a flag?
2971 fin_ready = num_ready_finalizers;
2972 finalize_in_range (start_addr, end_addr, generation);
2973 if (generation == GENERATION_OLD)
2974 finalize_in_range (nursery_start, nursery_real_end, GENERATION_NURSERY);
2975 bigo_scanned_num = scan_needed_big_objects (start_addr, end_addr);
2977 /* drain the new stack that might have been created */
2978 DEBUG (6, fprintf (gc_debug_file, "Precise scan of gray area post fin\n"));
2979 drain_gray_stack (start_addr, end_addr);
2980 } while (fin_ready != num_ready_finalizers || bigo_scanned_num);
2982 DEBUG (2, fprintf (gc_debug_file, "Finalize queue handling scan for %s generation: %d usecs\n", generation_name (generation), TV_ELAPSED (atv, btv)));
2985 * handle disappearing links
2986 * Note we do this after checking the finalization queue because if an object
2987 * survives (at least long enough to be finalized) we don't clear the link.
2988 * This also deals with a possible issue with the monitor reclamation: with the Boehm
2989 * GC a finalized object my lose the monitor because it is cleared before the finalizer is
2992 g_assert (gray_object_queue_is_empty ());
2994 null_link_in_range (start_addr, end_addr, generation);
2995 if (generation == GENERATION_OLD)
2996 null_link_in_range (start_addr, end_addr, GENERATION_NURSERY);
2997 if (gray_object_queue_is_empty ())
2999 drain_gray_stack (start_addr, end_addr);
3002 g_assert (gray_object_queue_is_empty ());
3003 /* DEBUG (2, fprintf (gc_debug_file, "Copied from %s to old space: %d bytes (%p-%p)\n", generation_name (generation), (int)(to_space_bumper - to_space), to_space, to_space_bumper)); */
3004 to_space_set_next_data ();
3008 check_scan_starts (void)
3010 GCMemSection *section;
3012 if (!do_scan_starts_check)
3014 for (section = section_list; section; section = section->block.next) {
3015 for (i = 0; i < section->num_scan_start; ++i) {
3016 if (section->scan_starts [i]) {
3017 guint size = safe_object_get_size ((MonoObject*) section->scan_starts [i]);
3018 g_assert (size >= sizeof (MonoObject) && size <= MAX_SMALL_OBJ_SIZE);
3024 static int last_num_pinned = 0;
3027 build_nursery_fragments (int start_pin, int end_pin)
3029 char *frag_start, *frag_end;
3033 while (nursery_fragments) {
3034 Fragment *next = nursery_fragments->next;
3035 nursery_fragments->next = fragment_freelist;
3036 fragment_freelist = nursery_fragments;
3037 nursery_fragments = next;
3039 frag_start = nursery_start;
3041 /* clear scan starts */
3042 memset (nursery_section->scan_starts, 0, nursery_section->num_scan_start * sizeof (gpointer));
3043 for (i = start_pin; i < end_pin; ++i) {
3044 frag_end = pin_queue [i];
3045 /* remove the pin bit from pinned objects */
3046 unpin_object (frag_end);
3047 nursery_section->scan_starts [((char*)frag_end - (char*)nursery_section->data)/SCAN_START_SIZE] = frag_end;
3048 frag_size = frag_end - frag_start;
3050 add_nursery_frag (frag_size, frag_start, frag_end);
3051 frag_size = safe_object_get_size ((MonoObject*)pin_queue [i]);
3052 frag_size += ALLOC_ALIGN - 1;
3053 frag_size &= ~(ALLOC_ALIGN - 1);
3054 frag_start = (char*)pin_queue [i] + frag_size;
3056 nursery_last_pinned_end = frag_start;
3057 frag_end = nursery_real_end;
3058 frag_size = frag_end - frag_start;
3060 add_nursery_frag (frag_size, frag_start, frag_end);
3061 if (!nursery_fragments) {
3062 DEBUG (1, fprintf (gc_debug_file, "Nursery fully pinned (%d)\n", end_pin - start_pin));
3063 for (i = start_pin; i < end_pin; ++i) {
3064 DEBUG (3, fprintf (gc_debug_file, "Bastard pinning obj %p (%s), size: %d\n", pin_queue [i], safe_name (pin_queue [i]), safe_object_get_size (pin_queue [i])));
3069 nursery_next = nursery_frag_real_end = NULL;
3071 /* Clear TLABs for all threads */
3075 /* FIXME: later reduce code duplication here with the above
3076 * We don't keep track of section fragments for non-nursery sections yet, so
3080 build_section_fragments (GCMemSection *section)
3083 char *frag_start, *frag_end;
3086 /* clear scan starts */
3087 memset (section->scan_starts, 0, section->num_scan_start * sizeof (gpointer));
3088 frag_start = section->data;
3089 section->next_data = section->data;
3090 for (i = section->pin_queue_start; i < section->pin_queue_end; ++i) {
3091 frag_end = pin_queue [i];
3092 /* remove the pin bit from pinned objects */
3093 unpin_object (frag_end);
3094 if (frag_end >= section->data + section->size) {
3095 frag_end = section->data + section->size;
3097 section->scan_starts [((char*)frag_end - (char*)section->data)/SCAN_START_SIZE] = frag_end;
3099 frag_size = frag_end - frag_start;
3101 binary_protocol_empty (frag_start, frag_size);
3102 memset (frag_start, 0, frag_size);
3104 frag_size = safe_object_get_size ((MonoObject*)pin_queue [i]);
3105 frag_size += ALLOC_ALIGN - 1;
3106 frag_size &= ~(ALLOC_ALIGN - 1);
3107 frag_start = (char*)pin_queue [i] + frag_size;
3108 section->next_data = MAX (section->next_data, frag_start);
3110 frag_end = section->end_data;
3111 frag_size = frag_end - frag_start;
3113 binary_protocol_empty (frag_start, frag_size);
3114 memset (frag_start, 0, frag_size);
3119 scan_from_registered_roots (char *addr_start, char *addr_end, int root_type)
3123 for (i = 0; i < roots_hash_size [root_type]; ++i) {
3124 for (root = roots_hash [root_type][i]; root; root = root->next) {
3125 DEBUG (6, fprintf (gc_debug_file, "Precise root scan %p-%p (desc: %p)\n", root->start_root, root->end_root, (void*)root->root_desc));
3126 precisely_scan_objects_from ((void**)root->start_root, (void**)root->end_root, addr_start, addr_end, root->root_desc);
3132 dump_occupied (char *start, char *end, char *section_start)
3134 fprintf (heap_dump_file, "<occupied offset=\"%zd\" size=\"%zd\"/>\n", start - section_start, end - start);
3138 dump_section (GCMemSection *section, const char *type)
3140 char *start = section->data;
3141 char *end = section->data + section->size;
3142 char *occ_start = NULL;
3144 char *old_start = NULL; /* just for debugging */
3146 fprintf (heap_dump_file, "<section type=\"%s\" size=\"%zu\">\n", type, section->size);
3148 while (start < end) {
3152 if (!*(void**)start) {
3154 dump_occupied (occ_start, start, section->data);
3157 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
3160 g_assert (start < section->next_data);
3165 vt = (GCVTable*)LOAD_VTABLE (start);
3168 size = safe_object_get_size ((MonoObject*) start);
3169 size += ALLOC_ALIGN - 1;
3170 size &= ~(ALLOC_ALIGN - 1);
3173 fprintf (heap_dump_file, "<object offset=\"%d\" class=\"%s.%s\" size=\"%d\"/>\n",
3174 start - section->data,
3175 vt->klass->name_space, vt->klass->name,
3183 dump_occupied (occ_start, start, section->data);
3185 fprintf (heap_dump_file, "</section>\n");
3189 dump_heap (const char *type, int num, const char *reason)
3191 static char const *internal_mem_names [] = { "pin-queue", "fragment", "section", "scan-starts",
3192 "fin-table", "finalize-entry", "dislink-table",
3193 "dislink", "roots-table", "root-record", "statistics",
3194 "remset", "gray-queue", "store-remset" };
3196 GCMemSection *section;
3200 fprintf (heap_dump_file, "<collection type=\"%s\" num=\"%d\"", type, num);
3202 fprintf (heap_dump_file, " reason=\"%s\"", reason);
3203 fprintf (heap_dump_file, ">\n");
3204 fprintf (heap_dump_file, "<other-mem-usage type=\"pinned-chunks\" size=\"%ld\"/>\n", pinned_chunk_bytes_alloced);
3205 fprintf (heap_dump_file, "<other-mem-usage type=\"large-internal\" size=\"%ld\"/>\n", large_internal_bytes_alloced);
3206 fprintf (heap_dump_file, "<other-mem-usage type=\"mempools\" size=\"%ld\"/>\n", mono_mempool_get_bytes_allocated ());
3207 for (i = 0; i < INTERNAL_MEM_MAX; ++i)
3208 fprintf (heap_dump_file, "<other-mem-usage type=\"%s\" size=\"%ld\"/>\n", internal_mem_names [i], small_internal_mem_bytes [i]);
3209 fprintf (heap_dump_file, "<pinned type=\"stack\" bytes=\"%zu\"/>\n", pinned_byte_counts [PIN_TYPE_STACK]);
3210 /* fprintf (heap_dump_file, "<pinned type=\"static-data\" bytes=\"%d\"/>\n", pinned_byte_counts [PIN_TYPE_STATIC_DATA]); */
3211 fprintf (heap_dump_file, "<pinned type=\"other\" bytes=\"%zu\"/>\n", pinned_byte_counts [PIN_TYPE_OTHER]);
3213 dump_section (nursery_section, "nursery");
3215 for (section = section_list; section; section = section->block.next) {
3216 if (section != nursery_section)
3217 dump_section (section, "old");
3220 fprintf (heap_dump_file, "<los>\n");
3221 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
3222 MonoObject *obj = (MonoObject*) bigobj->data;
3223 MonoClass *class = mono_object_class (obj);
3225 fprintf (heap_dump_file, "<object class=\"%s.%s\" size=\"%d\"/>\n",
3226 class->name_space, class->name,
3227 safe_object_get_size (obj));
3229 fprintf (heap_dump_file, "</los>\n");
3231 fprintf (heap_dump_file, "</collection>\n");
3237 static gboolean inited = FALSE;
3239 #ifdef HEAVY_STATISTICS
3240 num_copy_object_called = 0;
3241 num_objects_copied = 0;
3247 #ifdef HEAVY_STATISTICS
3248 mono_counters_register ("WBarrier set field", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_field);
3249 mono_counters_register ("WBarrier set arrayref", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_arrayref);
3250 mono_counters_register ("WBarrier arrayref copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_arrayref_copy);
3251 mono_counters_register ("WBarrier generic store called", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_generic_store);
3252 mono_counters_register ("WBarrier generic store stored", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_generic_store_remset);
3253 mono_counters_register ("WBarrier set root", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_root);
3254 mono_counters_register ("WBarrier value copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_value_copy);
3255 mono_counters_register ("WBarrier object copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_object_copy);
3257 mono_counters_register ("# objects allocated", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_alloced);
3258 mono_counters_register ("# copy_object() called (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_called_nursery);
3259 mono_counters_register ("# objects copied (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_copied_nursery);
3260 mono_counters_register ("# copy_object() called (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_called_major);
3261 mono_counters_register ("# objects copied (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_copied_major);
3263 mono_counters_register ("# copy_object() failed from space", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_failed_from_space);
3264 mono_counters_register ("# copy_object() failed forwarded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_failed_forwarded);
3265 mono_counters_register ("# copy_object() failed pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_failed_pinned);
3266 mono_counters_register ("# copy_object() failed large or pinned chunk", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_failed_large_pinned);
3267 mono_counters_register ("# copy_object() failed to space", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_failed_to_space);
3269 mono_counters_register ("Store remsets", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_store_remsets);
3270 mono_counters_register ("Unique store remsets", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_store_remsets_unique);
3271 mono_counters_register ("Saved remsets 1", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_saved_remsets_1);
3272 mono_counters_register ("Saved remsets 2", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_saved_remsets_2);
3273 mono_counters_register ("Global remsets added", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_added);
3274 mono_counters_register ("Global remsets processed", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_processed);
3281 commit_stats (int generation)
3283 #ifdef HEAVY_STATISTICS
3284 if (generation == GENERATION_NURSERY) {
3285 stat_copy_object_called_nursery += num_copy_object_called;
3286 stat_objects_copied_nursery += num_objects_copied;
3288 g_assert (generation == GENERATION_OLD);
3289 stat_copy_object_called_major += num_copy_object_called;
3290 stat_objects_copied_major += num_objects_copied;
3296 * Collect objects in the nursery. Returns whether to trigger a major
3300 collect_nursery (size_t requested_size)
3302 size_t max_garbage_amount;
3304 char *orig_nursery_next;
3306 GCMemSection *section;
3307 int old_num_major_sections = num_major_sections;
3308 int sections_alloced;
3309 TV_DECLARE (all_atv);
3310 TV_DECLARE (all_btv);
3315 binary_protocol_collection (GENERATION_NURSERY);
3316 check_scan_starts ();
3319 orig_nursery_next = nursery_next;
3320 nursery_next = MAX (nursery_next, nursery_last_pinned_end);
3321 /* FIXME: optimize later to use the higher address where an object can be present */
3322 nursery_next = MAX (nursery_next, nursery_real_end);
3324 DEBUG (1, fprintf (gc_debug_file, "Start nursery collection %d %p-%p, size: %d\n", num_minor_gcs, nursery_start, nursery_next, (int)(nursery_next - nursery_start)));
3325 max_garbage_amount = nursery_next - nursery_start;
3326 g_assert (nursery_section->size >= max_garbage_amount);
3328 /* Clear all remaining nursery fragments, pinning depends on this */
3329 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION) {
3330 g_assert (orig_nursery_next <= nursery_frag_real_end);
3331 memset (orig_nursery_next, 0, nursery_frag_real_end - orig_nursery_next);
3332 for (frag = nursery_fragments; frag; frag = frag->next) {
3333 memset (frag->fragment_start, 0, frag->fragment_end - frag->fragment_start);
3338 check_for_xdomain_refs ();
3340 nursery_section->next_data = nursery_next;
3342 if (!to_space_section) {
3343 new_to_space_section ();
3345 /* we might have done degraded allocation since the
3347 g_assert (to_space_bumper <= to_space_section->next_data);
3348 to_space_bumper = to_space_section->next_data;
3350 to_space_section->is_to_space = TRUE;
3352 gray_object_queue_init ();
3355 mono_stats.minor_gc_count ++;
3356 /* world must be stopped already */
3357 TV_GETTIME (all_atv);
3359 /* pin from pinned handles */
3361 pin_from_roots (nursery_start, nursery_next);
3362 /* identify pinned objects */
3363 optimize_pin_queue (0);
3364 next_pin_slot = pin_objects_from_addresses (nursery_section, pin_queue, pin_queue + next_pin_slot, nursery_start, nursery_next);
3366 DEBUG (2, fprintf (gc_debug_file, "Finding pinned pointers: %d in %d usecs\n", next_pin_slot, TV_ELAPSED (atv, btv)));
3367 DEBUG (4, fprintf (gc_debug_file, "Start scan with %d pinned objects\n", next_pin_slot));
3369 if (consistency_check_at_minor_collection)
3370 check_consistency ();
3373 * walk all the roots and copy the young objects to the old generation,
3374 * starting from to_space
3377 scan_from_remsets (nursery_start, nursery_next);
3378 /* we don't have complete write barrier yet, so we scan all the old generation sections */
3380 DEBUG (2, fprintf (gc_debug_file, "Old generation scan: %d usecs\n", TV_ELAPSED (btv, atv)));
3382 /* the pinned objects are roots */
3383 for (i = 0; i < next_pin_slot; ++i) {
3384 DEBUG (6, fprintf (gc_debug_file, "Precise object scan %d of pinned %p (%s)\n", i, pin_queue [i], safe_name (pin_queue [i])));
3385 scan_object (pin_queue [i], nursery_start, nursery_next);
3387 /* registered roots, this includes static fields */
3388 scan_from_registered_roots (nursery_start, nursery_next, ROOT_TYPE_NORMAL);
3389 scan_from_registered_roots (nursery_start, nursery_next, ROOT_TYPE_WBARRIER);
3390 scan_thread_data (nursery_start, nursery_next, TRUE);
3391 /* alloc_pinned objects */
3392 scan_from_pinned_objects (nursery_start, nursery_next);
3394 DEBUG (2, fprintf (gc_debug_file, "Root scan: %d usecs\n", TV_ELAPSED (atv, btv)));
3396 finish_gray_stack (nursery_start, nursery_next, GENERATION_NURSERY);
3398 /* walk the pin_queue, build up the fragment list of free memory, unmark
3399 * pinned objects as we go, memzero() the empty fragments so they are ready for the
3402 build_nursery_fragments (0, next_pin_slot);
3404 DEBUG (2, fprintf (gc_debug_file, "Fragment creation: %d usecs, %zd bytes available\n", TV_ELAPSED (btv, atv), fragment_total));
3406 for (section = section_list; section; section = section->block.next) {
3407 if (section->is_to_space)
3408 section->is_to_space = FALSE;
3411 TV_GETTIME (all_btv);
3412 mono_stats.minor_gc_time_usecs += TV_ELAPSED (all_atv, all_btv);
3415 dump_heap ("minor", num_minor_gcs - 1, NULL);
3417 /* prepare the pin queue for the next collection */
3418 last_num_pinned = next_pin_slot;
3420 if (fin_ready_list || critical_fin_list) {
3421 DEBUG (4, fprintf (gc_debug_file, "Finalizer-thread wakeup: ready %d\n", num_ready_finalizers));
3422 mono_gc_finalize_notify ();
3426 g_assert (gray_object_queue_is_empty ());
3428 commit_stats (GENERATION_NURSERY);
3430 check_scan_starts ();
3432 sections_alloced = num_major_sections - old_num_major_sections;
3433 minor_collection_sections_alloced += sections_alloced;
3435 return minor_collection_sections_alloced > minor_collection_section_allowance;
3439 scan_from_pinned_chunk_if_marked (PinnedChunk *chunk, char *obj, size_t size, void *dummy)
3441 if (object_is_pinned (obj))
3442 scan_object (obj, NULL, (char*)-1);
3446 major_collection (const char *reason)
3448 GCMemSection *section, *prev_section;
3449 LOSObject *bigobj, *prevbo;
3453 TV_DECLARE (all_atv);
3454 TV_DECLARE (all_btv);
3457 /* FIXME: only use these values for the precise scan
3458 * note that to_space pointers should be excluded anyway...
3460 char *heap_start = NULL;
3461 char *heap_end = (char*)-1;
3462 size_t copy_space_required = 0;
3463 int old_num_major_sections = num_major_sections;
3464 int num_major_sections_saved, save_target, allowance_target;
3467 binary_protocol_collection (GENERATION_OLD);
3468 check_scan_starts ();
3471 DEBUG (1, fprintf (gc_debug_file, "Start major collection %d\n", num_major_gcs));
3473 mono_stats.major_gc_count ++;
3475 /* Clear all remaining nursery fragments, pinning depends on this */
3476 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION) {
3477 g_assert (nursery_next <= nursery_frag_real_end);
3478 memset (nursery_next, 0, nursery_frag_real_end - nursery_next);
3479 for (frag = nursery_fragments; frag; frag = frag->next) {
3480 memset (frag->fragment_start, 0, frag->fragment_end - frag->fragment_start);
3485 check_for_xdomain_refs ();
3488 * FIXME: implement Mark/Compact
3489 * Until that is done, we can just apply mostly the same alg as for the nursery:
3490 * this means we need a big section to potentially copy all the other sections, so
3491 * it is not ideal specially with large heaps.
3493 if (g_getenv ("MONO_GC_NO_MAJOR")) {
3494 collect_nursery (0);
3497 TV_GETTIME (all_atv);
3498 /* FIXME: make sure the nursery next_data ptr is updated */
3499 nursery_section->next_data = nursery_real_end;
3500 /* we should also coalesce scanning from sections close to each other
3501 * and deal with pointers outside of the sections later.
3503 /* The remsets are not useful for a major collection */
3505 /* world must be stopped already */
3508 DEBUG (6, fprintf (gc_debug_file, "Collecting pinned addresses\n"));
3509 pin_from_roots ((void*)lowest_heap_address, (void*)highest_heap_address);
3510 optimize_pin_queue (0);
3513 * pin_queue now contains all candidate pointers, sorted and
3514 * uniqued. We must do two passes now to figure out which
3515 * objects are pinned.
3517 * The first is to find within the pin_queue the area for each
3518 * section. This requires that the pin_queue be sorted. We
3519 * also process the LOS objects and pinned chunks here.
3521 * The second, destructive, pass is to reduce the section
3522 * areas to pointers to the actually pinned objects.
3524 DEBUG (6, fprintf (gc_debug_file, "Pinning from sections\n"));
3525 /* first pass for the sections */
3526 for (section = section_list; section; section = section->block.next) {
3528 DEBUG (6, fprintf (gc_debug_file, "Pinning from section %p (%p-%p)\n", section, section->data, section->end_data));
3529 find_optimized_pin_queue_area (section->data, section->end_data, &start, &end);
3530 DEBUG (6, fprintf (gc_debug_file, "Found %d pinning addresses in section %p (%d-%d)\n",
3531 end - start, section, start, end));
3532 section->pin_queue_start = start;
3533 section->pin_queue_end = end;
3535 /* identify possible pointers to the insize of large objects */
3536 DEBUG (6, fprintf (gc_debug_file, "Pinning from large objects\n"));
3537 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
3539 find_optimized_pin_queue_area (bigobj->data, (char*)bigobj->data + bigobj->size, &start, &end);
3541 pin_object (bigobj->data);
3543 pin_stats_register_object ((char*) bigobj->data, safe_object_get_size ((MonoObject*) bigobj->data));
3544 DEBUG (6, fprintf (gc_debug_file, "Marked large object %p (%s) size: %zd from roots\n", bigobj->data, safe_name (bigobj->data), bigobj->size));
3547 /* look for pinned addresses for pinned-alloc objects */
3548 DEBUG (6, fprintf (gc_debug_file, "Pinning from pinned-alloc objects\n"));
3549 for (chunk = pinned_chunk_list; chunk; chunk = chunk->block.next) {
3551 find_optimized_pin_queue_area (chunk->start_data, (char*)chunk + chunk->num_pages * FREELIST_PAGESIZE, &start, &end);
3553 mark_pinned_from_addresses (chunk, pin_queue + start, pin_queue + end);
3555 /* second pass for the sections */
3556 for (section = section_list; section; section = section->block.next) {
3557 int start = section->pin_queue_start;
3558 int end = section->pin_queue_end;
3561 reduced_to = pin_objects_from_addresses (section, pin_queue + start, pin_queue + end,
3562 section->data, section->next_data);
3563 section->pin_queue_start = start;
3564 section->pin_queue_end = start + reduced_to;
3566 copy_space_required += (char*)section->next_data - (char*)section->data;
3570 DEBUG (2, fprintf (gc_debug_file, "Finding pinned pointers: %d in %d usecs\n", next_pin_slot, TV_ELAPSED (atv, btv)));
3571 DEBUG (4, fprintf (gc_debug_file, "Start scan with %d pinned objects\n", next_pin_slot));
3573 new_to_space_section ();
3574 gray_object_queue_init ();
3576 /* the old generation doesn't need to be scanned (no remembered sets or card
3577 * table needed either): the only objects that must survive are those pinned and
3578 * those referenced by the precise roots.
3579 * mark any section without pinned objects, so we can free it since we will be able to
3580 * move all the objects.
3582 /* the pinned objects are roots (big objects are included in this list, too) */
3583 for (section = section_list; section; section = section->block.next) {
3584 for (i = section->pin_queue_start; i < section->pin_queue_end; ++i) {
3585 DEBUG (6, fprintf (gc_debug_file, "Precise object scan %d of pinned %p (%s)\n",
3586 i, pin_queue [i], safe_name (pin_queue [i])));
3587 scan_object (pin_queue [i], heap_start, heap_end);
3590 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
3591 if (object_is_pinned (bigobj->data)) {
3592 DEBUG (6, fprintf (gc_debug_file, "Precise object scan pinned LOS object %p (%s)\n",
3593 bigobj->data, safe_name (bigobj->data)));
3594 scan_object (bigobj->data, heap_start, heap_end);
3597 scan_pinned_objects (scan_from_pinned_chunk_if_marked, NULL);
3598 /* registered roots, this includes static fields */
3599 scan_from_registered_roots (heap_start, heap_end, ROOT_TYPE_NORMAL);
3600 scan_from_registered_roots (heap_start, heap_end, ROOT_TYPE_WBARRIER);
3602 scan_thread_data (heap_start, heap_end, TRUE);
3603 /* alloc_pinned objects */
3604 scan_from_pinned_objects (heap_start, heap_end);
3605 /* scan the list of objects ready for finalization */
3606 scan_finalizer_entries (fin_ready_list, heap_start, heap_end);
3607 scan_finalizer_entries (critical_fin_list, heap_start, heap_end);
3609 DEBUG (2, fprintf (gc_debug_file, "Root scan: %d usecs\n", TV_ELAPSED (btv, atv)));
3611 /* we need to go over the big object list to see if any was marked and scan it
3612 * And we need to make this in a loop, considering that objects referenced by finalizable
3613 * objects could reference big objects (this happens in finish_gray_stack ())
3615 scan_needed_big_objects (heap_start, heap_end);
3616 /* all the objects in the heap */
3617 finish_gray_stack (heap_start, heap_end, GENERATION_OLD);
3621 /* sweep the big objects list */
3623 for (bigobj = los_object_list; bigobj;) {
3624 if (object_is_pinned (bigobj->data)) {
3625 unpin_object (bigobj->data);
3626 bigobj->scanned = FALSE;
3629 /* not referenced anywhere, so we can free it */
3631 prevbo->next = bigobj->next;
3633 los_object_list = bigobj->next;
3635 bigobj = bigobj->next;
3636 free_large_object (to_free);
3640 bigobj = bigobj->next;
3642 /* unpin objects from the pinned chunks and free the unmarked ones */
3643 sweep_pinned_objects ();
3645 /* free the unused sections */
3646 prev_section = NULL;
3647 for (section = section_list; section;) {
3648 /* to_space doesn't need handling here and the nursery is special */
3649 if (section->is_to_space || section == nursery_section) {
3650 if (section->is_to_space)
3651 section->is_to_space = FALSE;
3652 prev_section = section;
3653 section = section->block.next;
3656 /* no pinning object, so the section is free */
3657 if (section->pin_queue_start == section->pin_queue_end) {
3658 GCMemSection *to_free;
3660 prev_section->block.next = section->block.next;
3662 section_list = section->block.next;
3664 section = section->block.next;
3665 free_major_section (to_free);
3668 DEBUG (6, fprintf (gc_debug_file, "Section %p has still pinned objects (%d)\n", section, section->pin_queue_end - section->pin_queue_start));
3669 build_section_fragments (section);
3671 prev_section = section;
3672 section = section->block.next;
3675 /* walk the pin_queue, build up the fragment list of free memory, unmark
3676 * pinned objects as we go, memzero() the empty fragments so they are ready for the
3679 build_nursery_fragments (nursery_section->pin_queue_start, nursery_section->pin_queue_end);
3681 TV_GETTIME (all_btv);
3682 mono_stats.major_gc_time_usecs += TV_ELAPSED (all_atv, all_btv);
3685 dump_heap ("major", num_major_gcs - 1, reason);
3687 /* prepare the pin queue for the next collection */
3689 if (fin_ready_list || critical_fin_list) {
3690 DEBUG (4, fprintf (gc_debug_file, "Finalizer-thread wakeup: ready %d\n", num_ready_finalizers));
3691 mono_gc_finalize_notify ();
3695 g_assert (gray_object_queue_is_empty ());
3697 commit_stats (GENERATION_OLD);
3699 num_major_sections_saved = MAX (old_num_major_sections - num_major_sections, 1);
3701 save_target = num_major_sections / 2;
3702 allowance_target = save_target * minor_collection_sections_alloced / num_major_sections_saved;
3704 minor_collection_section_allowance = MAX (MIN (allowance_target, num_major_sections), MIN_MINOR_COLLECTION_SECTION_ALLOWANCE);
3707 printf ("alloced %d saved %d target %d allowance %d\n",
3708 minor_collection_sections_alloced, num_major_sections_saved, allowance_target,
3709 minor_collection_section_allowance);
3712 minor_collection_sections_alloced = 0;
3714 check_scan_starts ();
3718 * Allocate a new section of memory to be used as old generation.
3720 static GCMemSection*
3721 alloc_major_section (void)
3723 GCMemSection *section;
3726 section = get_os_memory_aligned (MAJOR_SECTION_SIZE, TRUE);
3727 section->next_data = section->data = (char*)section + SIZEOF_GC_MEM_SECTION;
3728 g_assert (!((mword)section->data & 7));
3729 section->size = MAJOR_SECTION_SIZE - SIZEOF_GC_MEM_SECTION;
3730 section->end_data = section->data + section->size;
3731 UPDATE_HEAP_BOUNDARIES (section->data, section->end_data);
3732 total_alloc += section->size;
3733 DEBUG (3, fprintf (gc_debug_file, "New major heap section: (%p-%p), total: %zd\n", section->data, section->end_data, total_alloc));
3734 scan_starts = (section->size + SCAN_START_SIZE - 1) / SCAN_START_SIZE;
3735 section->scan_starts = get_internal_mem (sizeof (char*) * scan_starts, INTERNAL_MEM_SCAN_STARTS);
3736 section->num_scan_start = scan_starts;
3737 section->block.role = MEMORY_ROLE_GEN1;
3738 section->is_to_space = TRUE;
3740 /* add to the section list */
3741 section->block.next = section_list;
3742 section_list = section;
3744 ++num_major_sections;
3750 free_major_section (GCMemSection *section)
3752 DEBUG (3, fprintf (gc_debug_file, "Freed major section %p (%p-%p)\n", section, section->data, section->end_data));
3753 free_internal_mem (section->scan_starts, INTERNAL_MEM_SCAN_STARTS);
3754 free_os_memory (section, MAJOR_SECTION_SIZE);
3755 total_alloc -= MAJOR_SECTION_SIZE - SIZEOF_GC_MEM_SECTION;
3757 --num_major_sections;
3761 * When deciding if it's better to collect or to expand, keep track
3762 * of how much garbage was reclaimed with the last collection: if it's too
3764 * This is called when we could not allocate a small object.
3766 static void __attribute__((noinline))
3767 minor_collect_or_expand_inner (size_t size)
3769 int do_minor_collection = 1;
3771 if (!nursery_section) {
3775 if (do_minor_collection) {
3777 if (collect_nursery (size))
3778 major_collection ("minor overflow");
3779 DEBUG (2, fprintf (gc_debug_file, "Heap size: %zd, LOS size: %zd\n", total_alloc, los_memory_usage));
3781 /* this also sets the proper pointers for the next allocation */
3782 if (!search_fragment_for_size (size)) {
3784 /* TypeBuilder and MonoMethod are killing mcs with fragmentation */
3785 DEBUG (1, fprintf (gc_debug_file, "nursery collection didn't find enough room for %zd alloc (%d pinned)\n", size, last_num_pinned));
3786 for (i = 0; i < last_num_pinned; ++i) {
3787 DEBUG (3, fprintf (gc_debug_file, "Bastard pinning obj %p (%s), size: %d\n", pin_queue [i], safe_name (pin_queue [i]), safe_object_get_size (pin_queue [i])));
3792 //report_internal_mem_usage ();
3796 * ######################################################################
3797 * ######## Memory allocation from the OS
3798 * ######################################################################
3799 * This section of code deals with getting memory from the OS and
3800 * allocating memory for GC-internal data structures.
3801 * Internal memory can be handled with a freelist for small objects.
3805 * Allocate a big chunk of memory from the OS (usually 64KB to several megabytes).
3806 * This must not require any lock.
3809 get_os_memory (size_t size, int activate)
3812 unsigned long prot_flags = activate? MONO_MMAP_READ|MONO_MMAP_WRITE: MONO_MMAP_NONE;
3814 prot_flags |= MONO_MMAP_PRIVATE | MONO_MMAP_ANON;
3815 size += pagesize - 1;
3816 size &= ~(pagesize - 1);
3817 ptr = mono_valloc (0, size, prot_flags);
3822 * Free the memory returned by get_os_memory (), returning it to the OS.
3825 free_os_memory (void *addr, size_t size)
3827 mono_vfree (addr, size);
3834 report_pinned_chunk (PinnedChunk *chunk, int seq) {
3836 int i, free_pages, num_free, free_mem;
3838 for (i = 0; i < chunk->num_pages; ++i) {
3839 if (!chunk->page_sizes [i])
3842 printf ("Pinned chunk %d at %p, size: %d, pages: %d, free: %d\n", seq, chunk, chunk->num_pages * FREELIST_PAGESIZE, chunk->num_pages, free_pages);
3843 free_mem = FREELIST_PAGESIZE * free_pages;
3844 for (i = 0; i < FREELIST_NUM_SLOTS; ++i) {
3845 if (!chunk->free_list [i])
3848 p = chunk->free_list [i];
3853 printf ("\tfree list of size %d, %d items\n", freelist_sizes [i], num_free);
3854 free_mem += freelist_sizes [i] * num_free;
3856 printf ("\tfree memory in chunk: %d\n", free_mem);
3862 static G_GNUC_UNUSED void
3863 report_internal_mem_usage (void) {
3866 printf ("Internal memory usage:\n");
3868 for (chunk = internal_chunk_list; chunk; chunk = chunk->block.next) {
3869 report_pinned_chunk (chunk, i++);
3871 printf ("Pinned memory usage:\n");
3873 for (chunk = pinned_chunk_list; chunk; chunk = chunk->block.next) {
3874 report_pinned_chunk (chunk, i++);
3879 * the array of pointers from @start to @end contains conservative
3880 * pointers to objects inside @chunk: mark each referenced object
3884 mark_pinned_from_addresses (PinnedChunk *chunk, void **start, void **end)
3886 for (; start < end; start++) {
3887 char *addr = *start;
3888 int offset = (char*)addr - (char*)chunk;
3889 int page = offset / FREELIST_PAGESIZE;
3890 int obj_offset = page == 0? offset - ((char*)chunk->start_data - (char*)chunk): offset % FREELIST_PAGESIZE;
3891 int slot_size = chunk->page_sizes [page];
3893 /* the page is not allocated */
3896 /* would be faster if we restrict the sizes to power of two,
3897 * but that's a waste of memory: need to measure. it could reduce
3898 * fragmentation since there are less pages needed, if for example
3899 * someone interns strings of each size we end up with one page per
3900 * interned string (still this is just ~40 KB): with more fine-grained sizes
3901 * this increases the number of used pages.
3904 obj_offset /= slot_size;
3905 obj_offset *= slot_size;
3906 addr = (char*)chunk->start_data + obj_offset;
3908 obj_offset /= slot_size;
3909 obj_offset *= slot_size;
3910 addr = (char*)chunk + page * FREELIST_PAGESIZE + obj_offset;
3913 /* if the vtable is inside the chunk it's on the freelist, so skip */
3914 if (*ptr && (*ptr < (void*)chunk->start_data || *ptr > (void*)((char*)chunk + chunk->num_pages * FREELIST_PAGESIZE))) {
3915 binary_protocol_pin (addr, (gpointer)LOAD_VTABLE (addr), safe_object_get_size ((MonoObject*)addr));
3918 pin_stats_register_object ((char*) addr, safe_object_get_size ((MonoObject*) addr));
3919 DEBUG (6, fprintf (gc_debug_file, "Marked pinned object %p (%s) from roots\n", addr, safe_name (addr)));
3925 scan_pinned_objects (ScanPinnedObjectCallbackFunc callback, void *callback_data)
3932 for (chunk = pinned_chunk_list; chunk; chunk = chunk->block.next) {
3933 end_chunk = (char*)chunk + chunk->num_pages * FREELIST_PAGESIZE;
3934 DEBUG (6, fprintf (gc_debug_file, "Scanning pinned chunk %p (range: %p-%p)\n", chunk, chunk->start_data, end_chunk));
3935 for (i = 0; i < chunk->num_pages; ++i) {
3936 obj_size = chunk->page_sizes [i];
3939 p = i? (char*)chunk + i * FREELIST_PAGESIZE: chunk->start_data;
3940 endp = i? p + FREELIST_PAGESIZE: (char*)chunk + FREELIST_PAGESIZE;
3941 DEBUG (6, fprintf (gc_debug_file, "Page %d (size: %d, range: %p-%p)\n", i, obj_size, p, endp));
3942 while (p + obj_size <= endp) {
3944 DEBUG (9, fprintf (gc_debug_file, "Considering %p (vtable: %p)\n", ptr, *ptr));
3945 /* if the first word (the vtable) is outside the chunk we have an object */
3946 if (*ptr && (*ptr < (void*)chunk || *ptr >= end_chunk))
3947 callback (chunk, (char*)ptr, obj_size, callback_data);
3955 sweep_pinned_objects_callback (PinnedChunk *chunk, char *ptr, size_t size, void *data)
3957 if (object_is_pinned (ptr)) {
3959 DEBUG (6, fprintf (gc_debug_file, "Unmarked pinned object %p (%s)\n", ptr, safe_name (ptr)));
3961 DEBUG (6, fprintf (gc_debug_file, "Freeing unmarked pinned object %p (%s)\n", ptr, safe_name (ptr)));
3962 free_pinned_object (chunk, ptr, size);
3967 sweep_pinned_objects (void)
3969 scan_pinned_objects (sweep_pinned_objects_callback, NULL);
3973 scan_object_callback (PinnedChunk *chunk, char *ptr, size_t size, char **data)
3975 DEBUG (6, fprintf (gc_debug_file, "Precise object scan of alloc_pinned %p (%s)\n", ptr, safe_name (ptr)));
3976 /* FIXME: Put objects without references into separate chunks
3977 which do not need to be scanned */
3978 scan_object (ptr, data [0], data [1]);
3982 scan_from_pinned_objects (char *addr_start, char *addr_end)
3984 char *data [2] = { addr_start, addr_end };
3985 scan_pinned_objects ((ScanPinnedObjectCallbackFunc)scan_object_callback, data);
3989 * Find the slot number in the freelist for memory chunks that
3990 * can contain @size objects.
3993 slot_for_size (size_t size)
3996 /* do a binary search or lookup table later. */
3997 for (slot = 0; slot < FREELIST_NUM_SLOTS; ++slot) {
3998 if (freelist_sizes [slot] >= size)
4001 g_assert_not_reached ();
4006 * Build a free list for @size memory chunks from the memory area between
4007 * start_page and end_page.
4010 build_freelist (PinnedChunk *chunk, int slot, int size, char *start_page, char *end_page)
4014 /*g_print ("building freelist for slot %d, size %d in %p\n", slot, size, chunk);*/
4015 p = (void**)start_page;
4016 end = (void**)(end_page - size);
4017 g_assert (!chunk->free_list [slot]);
4018 chunk->free_list [slot] = p;
4019 while ((char*)p + size <= (char*)end) {
4021 *p = (void*)((char*)p + size);
4025 /*g_print ("%d items created, max: %d\n", count, (end_page - start_page) / size);*/
4029 alloc_pinned_chunk (void)
4033 int size = MAJOR_SECTION_SIZE;
4035 chunk = get_os_memory_aligned (size, TRUE);
4036 chunk->block.role = MEMORY_ROLE_PINNED;
4038 UPDATE_HEAP_BOUNDARIES (chunk, ((char*)chunk + size));
4039 total_alloc += size;
4040 pinned_chunk_bytes_alloced += size;
4042 /* setup the bookeeping fields */
4043 chunk->num_pages = size / FREELIST_PAGESIZE;
4044 offset = G_STRUCT_OFFSET (PinnedChunk, data);
4045 chunk->page_sizes = (void*)((char*)chunk + offset);
4046 offset += sizeof (int) * chunk->num_pages;
4047 offset += ALLOC_ALIGN - 1;
4048 offset &= ~(ALLOC_ALIGN - 1);
4049 chunk->free_list = (void*)((char*)chunk + offset);
4050 offset += sizeof (void*) * FREELIST_NUM_SLOTS;
4051 offset += ALLOC_ALIGN - 1;
4052 offset &= ~(ALLOC_ALIGN - 1);
4053 chunk->start_data = (void*)((char*)chunk + offset);
4055 /* allocate the first page to the freelist */
4056 chunk->page_sizes [0] = PINNED_FIRST_SLOT_SIZE;
4057 build_freelist (chunk, slot_for_size (PINNED_FIRST_SLOT_SIZE), PINNED_FIRST_SLOT_SIZE, chunk->start_data, ((char*)chunk + FREELIST_PAGESIZE));
4058 DEBUG (4, fprintf (gc_debug_file, "Allocated pinned chunk %p, size: %d\n", chunk, size));
4059 min_pinned_chunk_addr = MIN (min_pinned_chunk_addr, (char*)chunk->start_data);
4060 max_pinned_chunk_addr = MAX (max_pinned_chunk_addr, ((char*)chunk + size));
4064 /* assumes freelist for slot is empty, so try to alloc a new page */
4066 get_chunk_freelist (PinnedChunk *chunk, int slot)
4070 p = chunk->free_list [slot];
4072 chunk->free_list [slot] = *p;
4075 for (i = 0; i < chunk->num_pages; ++i) {
4077 if (chunk->page_sizes [i])
4079 size = freelist_sizes [slot];
4080 chunk->page_sizes [i] = size;
4081 build_freelist (chunk, slot, size, (char*)chunk + FREELIST_PAGESIZE * i, (char*)chunk + FREELIST_PAGESIZE * (i + 1));
4085 p = chunk->free_list [slot];
4087 chunk->free_list [slot] = *p;
4094 alloc_from_freelist (size_t size)
4098 PinnedChunk *pchunk;
4099 slot = slot_for_size (size);
4100 /*g_print ("using slot %d for size %d (slot size: %d)\n", slot, size, freelist_sizes [slot]);*/
4101 g_assert (size <= freelist_sizes [slot]);
4102 for (pchunk = pinned_chunk_list; pchunk; pchunk = pchunk->block.next) {
4103 void **p = pchunk->free_list [slot];
4105 /*g_print ("found freelist for slot %d in chunk %p, returning %p, next %p\n", slot, pchunk, p, *p);*/
4106 pchunk->free_list [slot] = *p;
4110 for (pchunk = pinned_chunk_list; pchunk; pchunk = pchunk->block.next) {
4111 res = get_chunk_freelist (pchunk, slot);
4115 pchunk = alloc_pinned_chunk ();
4116 /* FIXME: handle OOM */
4117 pchunk->block.next = pinned_chunk_list;
4118 pinned_chunk_list = pchunk;
4119 res = get_chunk_freelist (pchunk, slot);
4123 /* used for the GC-internal data structures */
4124 /* FIXME: add support for bigger sizes by allocating more than one page
4128 get_internal_mem (size_t size, int type)
4132 PinnedChunk *pchunk;
4134 if (size > freelist_sizes [FREELIST_NUM_SLOTS - 1]) {
4135 LargeInternalMemHeader *mh;
4137 size += sizeof (LargeInternalMemHeader);
4138 mh = get_os_memory (size, TRUE);
4139 mh->magic = LARGE_INTERNAL_MEM_HEADER_MAGIC;
4142 large_internal_bytes_alloced += size;
4147 slot = slot_for_size (size);
4148 g_assert (size <= freelist_sizes [slot]);
4150 small_internal_mem_bytes [type] += freelist_sizes [slot];
4152 for (pchunk = internal_chunk_list; pchunk; pchunk = pchunk->block.next) {
4153 void **p = pchunk->free_list [slot];
4155 pchunk->free_list [slot] = *p;
4156 memset (p, 0, size);
4160 for (pchunk = internal_chunk_list; pchunk; pchunk = pchunk->block.next) {
4161 res = get_chunk_freelist (pchunk, slot);
4163 memset (res, 0, size);
4167 pchunk = alloc_pinned_chunk ();
4168 /* FIXME: handle OOM */
4169 pchunk->block.next = internal_chunk_list;
4170 internal_chunk_list = pchunk;
4171 res = get_chunk_freelist (pchunk, slot);
4172 memset (res, 0, size);
4177 free_internal_mem (void *addr, int type)
4179 PinnedChunk *pchunk;
4180 LargeInternalMemHeader *mh;
4183 for (pchunk = internal_chunk_list; pchunk; pchunk = pchunk->block.next) {
4184 /*printf ("trying to free %p in %p (pages: %d)\n", addr, pchunk, pchunk->num_pages);*/
4185 if (addr >= (void*)pchunk && (char*)addr < (char*)pchunk + pchunk->num_pages * FREELIST_PAGESIZE) {
4186 int offset = (char*)addr - (char*)pchunk;
4187 int page = offset / FREELIST_PAGESIZE;
4188 int slot = slot_for_size (pchunk->page_sizes [page]);
4190 *p = pchunk->free_list [slot];
4191 pchunk->free_list [slot] = p;
4193 small_internal_mem_bytes [type] -= freelist_sizes [slot];
4198 mh = (LargeInternalMemHeader*)((char*)addr - G_STRUCT_OFFSET (LargeInternalMemHeader, data));
4199 g_assert (mh->magic == LARGE_INTERNAL_MEM_HEADER_MAGIC);
4200 large_internal_bytes_alloced -= mh->size;
4201 free_os_memory (mh, mh->size);
4205 * ######################################################################
4206 * ######## Object allocation
4207 * ######################################################################
4208 * This section of code deals with allocating memory for objects.
4209 * There are several ways:
4210 * *) allocate large objects
4211 * *) allocate normal objects
4212 * *) fast lock-free allocation
4213 * *) allocation of pinned objects
4217 free_large_object (LOSObject *obj)
4219 size_t size = obj->size;
4220 DEBUG (4, fprintf (gc_debug_file, "Freed large object %p, size %zd\n", obj->data, obj->size));
4221 binary_protocol_empty (obj->data, obj->size);
4223 los_memory_usage -= size;
4224 size += sizeof (LOSObject);
4225 size += pagesize - 1;
4226 size &= ~(pagesize - 1);
4227 total_alloc -= size;
4229 free_os_memory (obj, size);
4233 * Objects with size >= 64KB are allocated in the large object space.
4234 * They are currently kept track of with a linked list.
4235 * They don't move, so there is no need to pin them during collection
4236 * and we avoid the memcpy overhead.
4238 static void* __attribute__((noinline))
4239 alloc_large_inner (MonoVTable *vtable, size_t size)
4244 int just_did_major_gc = FALSE;
4246 g_assert (size > MAX_SMALL_OBJ_SIZE);
4248 if (los_memory_usage > next_los_collection) {
4249 DEBUG (4, fprintf (gc_debug_file, "Should trigger major collection: req size %zd (los already: %zu, limit: %zu)\n", size, los_memory_usage, next_los_collection));
4250 just_did_major_gc = TRUE;
4252 major_collection ("LOS overflow");
4254 /* later increase based on a percent of the heap size */
4255 next_los_collection = los_memory_usage + 5*1024*1024;
4258 alloc_size += sizeof (LOSObject);
4259 alloc_size += pagesize - 1;
4260 alloc_size &= ~(pagesize - 1);
4261 /* FIXME: handle OOM */
4262 obj = get_os_memory (alloc_size, TRUE);
4264 vtslot = (void**)obj->data;
4266 total_alloc += alloc_size;
4267 UPDATE_HEAP_BOUNDARIES (obj->data, (char*)obj->data + size);
4268 obj->next = los_object_list;
4269 los_object_list = obj;
4270 los_memory_usage += size;
4272 DEBUG (4, fprintf (gc_debug_file, "Allocated large object %p, vtable: %p (%s), size: %zd\n", obj->data, vtable, vtable->klass->name, size));
4273 binary_protocol_alloc (obj->data, vtable, size);
4277 /* check if we have a suitable fragment in nursery_fragments to be able to allocate
4278 * an object of size @size
4279 * Return FALSE if not found (which means we need a collection)
4282 search_fragment_for_size (size_t size)
4284 Fragment *frag, *prev;
4285 DEBUG (4, fprintf (gc_debug_file, "Searching nursery fragment %p, size: %zd\n", nursery_frag_real_end, size));
4287 if (nursery_frag_real_end > nursery_next && nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
4288 /* Clear the remaining space, pinning depends on this */
4289 memset (nursery_next, 0, nursery_frag_real_end - nursery_next);
4292 for (frag = nursery_fragments; frag; frag = frag->next) {
4293 if (size <= (frag->fragment_end - frag->fragment_start)) {
4294 /* remove from the list */
4296 prev->next = frag->next;
4298 nursery_fragments = frag->next;
4299 nursery_next = frag->fragment_start;
4300 nursery_frag_real_end = frag->fragment_end;
4302 DEBUG (4, fprintf (gc_debug_file, "Using nursery fragment %p-%p, size: %zd (req: %zd)\n", nursery_next, nursery_frag_real_end, nursery_frag_real_end - nursery_next, size));
4303 frag->next = fragment_freelist;
4304 fragment_freelist = frag;
4313 * size is already rounded up and we hold the GC lock.
4316 alloc_degraded (MonoVTable *vtable, size_t size)
4318 GCMemSection *section;
4320 g_assert (size <= MAX_SMALL_OBJ_SIZE);
4321 for (section = section_list; section; section = section->block.next) {
4322 if (section != nursery_section && (section->end_data - section->next_data) >= size) {
4323 p = (void**)section->next_data;
4328 section = alloc_major_section ();
4329 section->is_to_space = FALSE;
4330 /* FIXME: handle OOM */
4331 p = (void**)section->next_data;
4333 section->next_data += size;
4334 degraded_mode += size;
4335 DEBUG (3, fprintf (gc_debug_file, "Allocated (degraded) object %p, vtable: %p (%s), size: %zd in section %p\n", p, vtable, vtable->klass->name, size, section));
4341 * Provide a variant that takes just the vtable for small fixed-size objects.
4342 * The aligned size is already computed and stored in vt->gc_descr.
4343 * Note: every SCAN_START_SIZE or so we are given the chance to do some special
4344 * processing. We can keep track of where objects start, for example,
4345 * so when we scan the thread stacks for pinned objects, we can start
4346 * a search for the pinned object in SCAN_START_SIZE chunks.
4349 mono_gc_alloc_obj_nolock (MonoVTable *vtable, size_t size)
4351 /* FIXME: handle OOM */
4357 HEAVY_STAT (++stat_objects_alloced);
4359 size += ALLOC_ALIGN - 1;
4360 size &= ~(ALLOC_ALIGN - 1);
4362 g_assert (vtable->gc_descr);
4364 if (G_UNLIKELY (collect_before_allocs)) {
4365 if (nursery_section) {
4367 collect_nursery (0);
4369 if (!degraded_mode && !search_fragment_for_size (size)) {
4371 g_assert_not_reached ();
4377 * We must already have the lock here instead of after the
4378 * fast path because we might be interrupted in the fast path
4379 * (after confirming that new_next < TLAB_TEMP_END) by the GC,
4380 * and we'll end up allocating an object in a fragment which
4381 * no longer belongs to us.
4383 * The managed allocator does not do this, but it's treated
4384 * specially by the world-stopping code.
4387 if (size > MAX_SMALL_OBJ_SIZE) {
4388 p = alloc_large_inner (vtable, size);
4390 /* tlab_next and tlab_temp_end are TLS vars so accessing them might be expensive */
4392 p = (void**)TLAB_NEXT;
4393 /* FIXME: handle overflow */
4394 new_next = (char*)p + size;
4395 TLAB_NEXT = new_next;
4397 if (G_LIKELY (new_next < TLAB_TEMP_END)) {
4401 * FIXME: We might need a memory barrier here so the change to tlab_next is
4402 * visible before the vtable store.
4405 DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
4406 binary_protocol_alloc (p , vtable, size);
4407 g_assert (*p == NULL);
4410 g_assert (TLAB_NEXT == new_next);
4417 /* there are two cases: the object is too big or we run out of space in the TLAB */
4418 /* we also reach here when the thread does its first allocation after a minor
4419 * collection, since the tlab_ variables are initialized to NULL.
4420 * there can be another case (from ORP), if we cooperate with the runtime a bit:
4421 * objects that need finalizers can have the high bit set in their size
4422 * so the above check fails and we can readily add the object to the queue.
4423 * This avoids taking again the GC lock when registering, but this is moot when
4424 * doing thread-local allocation, so it may not be a good idea.
4426 g_assert (TLAB_NEXT == new_next);
4427 if (TLAB_NEXT >= TLAB_REAL_END) {
4429 * Run out of space in the TLAB. When this happens, some amount of space
4430 * remains in the TLAB, but not enough to satisfy the current allocation
4431 * request. Currently, we retire the TLAB in all cases, later we could
4432 * keep it if the remaining space is above a treshold, and satisfy the
4433 * allocation directly from the nursery.
4436 /* when running in degraded mode, we continue allocing that way
4437 * for a while, to decrease the number of useless nursery collections.
4439 if (degraded_mode && degraded_mode < DEFAULT_NURSERY_SIZE) {
4440 p = alloc_degraded (vtable, size);
4444 if (size > tlab_size) {
4445 /* Allocate directly from the nursery */
4446 if (nursery_next + size >= nursery_frag_real_end) {
4447 if (!search_fragment_for_size (size)) {
4448 minor_collect_or_expand_inner (size);
4449 if (degraded_mode) {
4450 p = alloc_degraded (vtable, size);
4456 p = (void*)nursery_next;
4457 nursery_next += size;
4458 if (nursery_next > nursery_frag_real_end) {
4463 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
4464 memset (p, 0, size);
4467 DEBUG (3, fprintf (gc_debug_file, "Retire TLAB: %p-%p [%ld]\n", TLAB_START, TLAB_REAL_END, (long)(TLAB_REAL_END - TLAB_NEXT - size)));
4469 if (nursery_next + tlab_size >= nursery_frag_real_end) {
4470 res = search_fragment_for_size (tlab_size);
4472 minor_collect_or_expand_inner (tlab_size);
4473 if (degraded_mode) {
4474 p = alloc_degraded (vtable, size);
4480 /* Allocate a new TLAB from the current nursery fragment */
4481 TLAB_START = nursery_next;
4482 nursery_next += tlab_size;
4483 TLAB_NEXT = TLAB_START;
4484 TLAB_REAL_END = TLAB_START + tlab_size;
4485 TLAB_TEMP_END = TLAB_START + MIN (SCAN_START_SIZE, tlab_size);
4487 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
4488 memset (TLAB_START, 0, tlab_size);
4490 /* Allocate from the TLAB */
4491 p = (void*)TLAB_NEXT;
4493 g_assert (TLAB_NEXT <= TLAB_REAL_END);
4495 nursery_section->scan_starts [((char*)p - (char*)nursery_section->data)/SCAN_START_SIZE] = (char*)p;
4498 /* Reached tlab_temp_end */
4500 /* record the scan start so we can find pinned objects more easily */
4501 nursery_section->scan_starts [((char*)p - (char*)nursery_section->data)/SCAN_START_SIZE] = (char*)p;
4502 /* we just bump tlab_temp_end as well */
4503 TLAB_TEMP_END = MIN (TLAB_REAL_END, TLAB_NEXT + SCAN_START_SIZE);
4504 DEBUG (5, fprintf (gc_debug_file, "Expanding local alloc: %p-%p\n", TLAB_NEXT, TLAB_TEMP_END));
4508 DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
4509 binary_protocol_alloc (p, vtable, size);
4516 mono_gc_try_alloc_obj_nolock (MonoVTable *vtable, size_t size)
4522 size += ALLOC_ALIGN - 1;
4523 size &= ~(ALLOC_ALIGN - 1);
4525 g_assert (vtable->gc_descr);
4526 if (size <= MAX_SMALL_OBJ_SIZE) {
4527 /* tlab_next and tlab_temp_end are TLS vars so accessing them might be expensive */
4529 p = (void**)TLAB_NEXT;
4530 /* FIXME: handle overflow */
4531 new_next = (char*)p + size;
4532 TLAB_NEXT = new_next;
4534 if (G_LIKELY (new_next < TLAB_TEMP_END)) {
4538 * FIXME: We might need a memory barrier here so the change to tlab_next is
4539 * visible before the vtable store.
4542 HEAVY_STAT (++stat_objects_alloced);
4544 DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
4545 binary_protocol_alloc (p, vtable, size);
4546 g_assert (*p == NULL);
4549 g_assert (TLAB_NEXT == new_next);
4558 mono_gc_alloc_obj (MonoVTable *vtable, size_t size)
4561 #ifndef DISABLE_CRITICAL_REGION
4563 ENTER_CRITICAL_REGION;
4564 res = mono_gc_try_alloc_obj_nolock (vtable, size);
4566 EXIT_CRITICAL_REGION;
4569 EXIT_CRITICAL_REGION;
4572 res = mono_gc_alloc_obj_nolock (vtable, size);
4578 mono_gc_alloc_vector (MonoVTable *vtable, size_t size, mono_array_size_t max_length)
4581 #ifndef DISABLE_CRITICAL_REGION
4583 ENTER_CRITICAL_REGION;
4584 arr = mono_gc_try_alloc_obj_nolock (vtable, size);
4586 arr->max_length = max_length;
4587 EXIT_CRITICAL_REGION;
4590 EXIT_CRITICAL_REGION;
4595 arr = mono_gc_alloc_obj_nolock (vtable, size);
4596 arr->max_length = max_length;
4604 mono_gc_alloc_array (MonoVTable *vtable, size_t size, mono_array_size_t max_length, mono_array_size_t bounds_size)
4607 MonoArrayBounds *bounds;
4611 arr = mono_gc_alloc_obj_nolock (vtable, size);
4612 arr->max_length = max_length;
4614 bounds = (MonoArrayBounds*)((char*)arr + size - bounds_size);
4615 arr->bounds = bounds;
4623 mono_gc_alloc_string (MonoVTable *vtable, size_t size, gint32 len)
4626 #ifndef DISABLE_CRITICAL_REGION
4628 ENTER_CRITICAL_REGION;
4629 str = mono_gc_try_alloc_obj_nolock (vtable, size);
4632 EXIT_CRITICAL_REGION;
4635 EXIT_CRITICAL_REGION;
4640 str = mono_gc_alloc_obj_nolock (vtable, size);
4649 * To be used for interned strings and possibly MonoThread, reflection handles.
4650 * We may want to explicitly free these objects.
4653 mono_gc_alloc_pinned_obj (MonoVTable *vtable, size_t size)
4655 /* FIXME: handle OOM */
4657 size += ALLOC_ALIGN - 1;
4658 size &= ~(ALLOC_ALIGN - 1);
4660 if (size > MAX_FREELIST_SIZE) {
4661 /* large objects are always pinned anyway */
4662 p = alloc_large_inner (vtable, size);
4664 p = alloc_from_freelist (size);
4665 memset (p, 0, size);
4667 DEBUG (6, fprintf (gc_debug_file, "Allocated pinned object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
4668 binary_protocol_alloc (p, vtable, size);
4675 * ######################################################################
4676 * ######## Finalization support
4677 * ######################################################################
4681 * this is valid for the nursery: if the object has been forwarded it means it's
4682 * still refrenced from a root. If it is pinned it's still alive as well.
4683 * Return TRUE if @obj is ready to be finalized.
4685 #define object_is_fin_ready(obj) (!object_is_pinned (obj) && !object_is_forwarded (obj))
4688 is_critical_finalizer (FinalizeEntry *entry)
4693 if (!mono_defaults.critical_finalizer_object)
4696 obj = entry->object;
4697 class = ((MonoVTable*)LOAD_VTABLE (obj))->klass;
4699 return mono_class_has_parent (class, mono_defaults.critical_finalizer_object);
4703 queue_finalization_entry (FinalizeEntry *entry) {
4704 if (is_critical_finalizer (entry)) {
4705 entry->next = critical_fin_list;
4706 critical_fin_list = entry;
4708 entry->next = fin_ready_list;
4709 fin_ready_list = entry;
4713 /* LOCKING: requires that the GC lock is held */
4715 rehash_fin_table (FinalizeEntryHashTable *hash_table)
4717 FinalizeEntry **finalizable_hash = hash_table->table;
4718 mword finalizable_hash_size = hash_table->size;
4721 FinalizeEntry **new_hash;
4722 FinalizeEntry *entry, *next;
4723 int new_size = g_spaced_primes_closest (hash_table->num_registered);
4725 new_hash = get_internal_mem (new_size * sizeof (FinalizeEntry*), INTERNAL_MEM_FIN_TABLE);
4726 for (i = 0; i < finalizable_hash_size; ++i) {
4727 for (entry = finalizable_hash [i]; entry; entry = next) {
4728 hash = mono_object_hash (entry->object) % new_size;
4730 entry->next = new_hash [hash];
4731 new_hash [hash] = entry;
4734 free_internal_mem (finalizable_hash, INTERNAL_MEM_FIN_TABLE);
4735 hash_table->table = new_hash;
4736 hash_table->size = new_size;
4739 /* LOCKING: requires that the GC lock is held */
4741 rehash_fin_table_if_necessary (FinalizeEntryHashTable *hash_table)
4743 if (hash_table->num_registered >= hash_table->size * 2)
4744 rehash_fin_table (hash_table);
4747 /* LOCKING: requires that the GC lock is held */
4749 finalize_in_range (char *start, char *end, int generation)
4751 FinalizeEntryHashTable *hash_table = get_finalize_entry_hash_table (generation);
4752 FinalizeEntry *entry, *prev;
4754 FinalizeEntry **finalizable_hash = hash_table->table;
4755 mword finalizable_hash_size = hash_table->size;
4759 for (i = 0; i < finalizable_hash_size; ++i) {
4761 for (entry = finalizable_hash [i]; entry;) {
4762 if ((char*)entry->object >= start && (char*)entry->object < end && !object_is_in_to_space (entry->object)) {
4763 gboolean is_fin_ready = object_is_fin_ready (entry->object);
4764 char *copy = copy_object (entry->object, start, end);
4767 FinalizeEntry *next;
4768 /* remove and put in fin_ready_list */
4770 prev->next = entry->next;
4772 finalizable_hash [i] = entry->next;
4774 num_ready_finalizers++;
4775 hash_table->num_registered--;
4776 queue_finalization_entry (entry);
4777 /* Make it survive */
4778 from = entry->object;
4779 entry->object = copy;
4780 DEBUG (5, fprintf (gc_debug_file, "Queueing object for finalization: %p (%s) (was at %p) (%d/%d)\n", entry->object, safe_name (entry->object), from, num_ready_finalizers, hash_table->num_registered));
4784 char *from = entry->object;
4785 if (hash_table == &minor_finalizable_hash && !ptr_in_nursery (copy)) {
4786 FinalizeEntry *next = entry->next;
4787 unsigned int major_hash;
4788 /* remove from the list */
4790 prev->next = entry->next;
4792 finalizable_hash [i] = entry->next;
4793 hash_table->num_registered--;
4795 entry->object = copy;
4797 /* insert it into the major hash */
4798 rehash_fin_table_if_necessary (&major_finalizable_hash);
4799 major_hash = mono_object_hash ((MonoObject*) copy) %
4800 major_finalizable_hash.size;
4801 entry->next = major_finalizable_hash.table [major_hash];
4802 major_finalizable_hash.table [major_hash] = entry;
4803 major_finalizable_hash.num_registered++;
4805 DEBUG (5, fprintf (gc_debug_file, "Promoting finalization of object %p (%s) (was at %p) to major table\n", copy, safe_name (copy), from));
4810 /* update pointer */
4811 DEBUG (5, fprintf (gc_debug_file, "Updating object for finalization: %p (%s) (was at %p)\n", entry->object, safe_name (entry->object), from));
4812 entry->object = copy;
4817 entry = entry->next;
4822 /* LOCKING: requires that the GC lock is held */
4824 null_link_in_range (char *start, char *end, int generation)
4826 DisappearingLinkHashTable *hash = get_dislink_hash_table (generation);
4827 DisappearingLink **disappearing_link_hash = hash->table;
4828 int disappearing_link_hash_size = hash->size;
4829 DisappearingLink *entry, *prev;
4831 if (!hash->num_links)
4833 for (i = 0; i < disappearing_link_hash_size; ++i) {
4835 for (entry = disappearing_link_hash [i]; entry;) {
4836 char *object = DISLINK_OBJECT (entry);
4837 if (object >= start && object < end && !object_is_in_to_space (object)) {
4838 gboolean track = DISLINK_TRACK (entry);
4839 if (!track && object_is_fin_ready (object)) {
4840 void **p = entry->link;
4841 DisappearingLink *old;
4843 /* remove from list */
4845 prev->next = entry->next;
4847 disappearing_link_hash [i] = entry->next;
4848 DEBUG (5, fprintf (gc_debug_file, "Dislink nullified at %p to GCed object %p\n", p, object));
4850 free_internal_mem (entry, INTERNAL_MEM_DISLINK);
4855 char *copy = copy_object (object, start, end);
4857 /* Update pointer if it's moved. If the object
4858 * has been moved out of the nursery, we need to
4859 * remove the link from the minor hash table to
4862 * FIXME: what if an object is moved earlier?
4865 if (hash == &minor_disappearing_link_hash && !ptr_in_nursery (copy)) {
4866 void **link = entry->link;
4867 DisappearingLink *old;
4868 /* remove from list */
4870 prev->next = entry->next;
4872 disappearing_link_hash [i] = entry->next;
4874 free_internal_mem (entry, INTERNAL_MEM_DISLINK);
4878 add_or_remove_disappearing_link ((MonoObject*)copy, link,
4879 track, GENERATION_OLD);
4881 DEBUG (5, fprintf (gc_debug_file, "Upgraded dislink at %p to major because object %p moved to %p\n", link, object, copy));
4885 /* We set the track resurrection bit to
4886 * FALSE if the object is to be finalized
4887 * so that the object can be collected in
4888 * the next cycle (i.e. after it was
4891 *entry->link = HIDE_POINTER (copy,
4892 object_is_fin_ready (object) ? FALSE : track);
4893 DEBUG (5, fprintf (gc_debug_file, "Updated dislink at %p to %p\n", entry->link, DISLINK_OBJECT (entry)));
4898 entry = entry->next;
4903 /* LOCKING: requires that the GC lock is held */
4905 null_links_for_domain (MonoDomain *domain, int generation)
4907 DisappearingLinkHashTable *hash = get_dislink_hash_table (generation);
4908 DisappearingLink **disappearing_link_hash = hash->table;
4909 int disappearing_link_hash_size = hash->size;
4910 DisappearingLink *entry, *prev;
4912 for (i = 0; i < disappearing_link_hash_size; ++i) {
4914 for (entry = disappearing_link_hash [i]; entry; ) {
4915 char *object = DISLINK_OBJECT (entry);
4916 /* FIXME: actually there should be no object
4917 left in the domain with a non-null vtable
4918 (provided we remove the Thread special
4920 if (object && (!((MonoObject*)object)->vtable || mono_object_domain (object) == domain)) {
4921 DisappearingLink *next = entry->next;
4926 disappearing_link_hash [i] = next;
4928 if (*(entry->link)) {
4929 *(entry->link) = NULL;
4930 g_warning ("Disappearing link %p not freed", entry->link);
4932 free_internal_mem (entry, INTERNAL_MEM_DISLINK);
4939 entry = entry->next;
4944 /* LOCKING: requires that the GC lock is held */
4946 finalizers_for_domain (MonoDomain *domain, MonoObject **out_array, int out_size,
4947 FinalizeEntryHashTable *hash_table)
4949 FinalizeEntry **finalizable_hash = hash_table->table;
4950 mword finalizable_hash_size = hash_table->size;
4951 FinalizeEntry *entry, *prev;
4954 if (no_finalize || !out_size || !out_array)
4957 for (i = 0; i < finalizable_hash_size; ++i) {
4959 for (entry = finalizable_hash [i]; entry;) {
4960 if (mono_object_domain (entry->object) == domain) {
4961 FinalizeEntry *next;
4962 /* remove and put in out_array */
4964 prev->next = entry->next;
4966 finalizable_hash [i] = entry->next;
4968 hash_table->num_registered--;
4969 out_array [count ++] = entry->object;
4970 DEBUG (5, fprintf (gc_debug_file, "Collecting object for finalization: %p (%s) (%d/%d)\n", entry->object, safe_name (entry->object), num_ready_finalizers, hash_table->num_registered));
4972 if (count == out_size)
4977 entry = entry->next;
4984 * mono_gc_finalizers_for_domain:
4985 * @domain: the unloading appdomain
4986 * @out_array: output array
4987 * @out_size: size of output array
4989 * Store inside @out_array up to @out_size objects that belong to the unloading
4990 * appdomain @domain. Returns the number of stored items. Can be called repeteadly
4991 * until it returns 0.
4992 * The items are removed from the finalizer data structure, so the caller is supposed
4994 * @out_array should be on the stack to allow the GC to know the objects are still alive.
4997 mono_gc_finalizers_for_domain (MonoDomain *domain, MonoObject **out_array, int out_size)
5002 result = finalizers_for_domain (domain, out_array, out_size, &minor_finalizable_hash);
5003 if (result < out_size) {
5004 result += finalizers_for_domain (domain, out_array + result, out_size - result,
5005 &major_finalizable_hash);
5013 register_for_finalization (MonoObject *obj, void *user_data, int generation)
5015 FinalizeEntryHashTable *hash_table = get_finalize_entry_hash_table (generation);
5016 FinalizeEntry **finalizable_hash;
5017 mword finalizable_hash_size;
5018 FinalizeEntry *entry, *prev;
5022 g_assert (user_data == NULL || user_data == mono_gc_run_finalize);
5023 hash = mono_object_hash (obj);
5025 rehash_fin_table_if_necessary (hash_table);
5026 finalizable_hash = hash_table->table;
5027 finalizable_hash_size = hash_table->size;
5028 hash %= finalizable_hash_size;
5030 for (entry = finalizable_hash [hash]; entry; entry = entry->next) {
5031 if (entry->object == obj) {
5033 /* remove from the list */
5035 prev->next = entry->next;
5037 finalizable_hash [hash] = entry->next;
5038 hash_table->num_registered--;
5039 DEBUG (5, fprintf (gc_debug_file, "Removed finalizer %p for object: %p (%s) (%d)\n", entry, obj, obj->vtable->klass->name, hash_table->num_registered));
5040 free_internal_mem (entry, INTERNAL_MEM_FINALIZE_ENTRY);
5048 /* request to deregister, but already out of the list */
5052 entry = get_internal_mem (sizeof (FinalizeEntry), INTERNAL_MEM_FINALIZE_ENTRY);
5053 entry->object = obj;
5054 entry->next = finalizable_hash [hash];
5055 finalizable_hash [hash] = entry;
5056 hash_table->num_registered++;
5057 DEBUG (5, fprintf (gc_debug_file, "Added finalizer %p for object: %p (%s) (%d) to %s table\n", entry, obj, obj->vtable->klass->name, hash_table->num_registered, generation_name (generation)));
5062 mono_gc_register_for_finalization (MonoObject *obj, void *user_data)
5064 if (ptr_in_nursery (obj))
5065 register_for_finalization (obj, user_data, GENERATION_NURSERY);
5067 register_for_finalization (obj, user_data, GENERATION_OLD);
5071 rehash_dislink (DisappearingLinkHashTable *hash_table)
5073 DisappearingLink **disappearing_link_hash = hash_table->table;
5074 int disappearing_link_hash_size = hash_table->size;
5077 DisappearingLink **new_hash;
5078 DisappearingLink *entry, *next;
5079 int new_size = g_spaced_primes_closest (hash_table->num_links);
5081 new_hash = get_internal_mem (new_size * sizeof (DisappearingLink*), INTERNAL_MEM_DISLINK_TABLE);
5082 for (i = 0; i < disappearing_link_hash_size; ++i) {
5083 for (entry = disappearing_link_hash [i]; entry; entry = next) {
5084 hash = mono_aligned_addr_hash (entry->link) % new_size;
5086 entry->next = new_hash [hash];
5087 new_hash [hash] = entry;
5090 free_internal_mem (disappearing_link_hash, INTERNAL_MEM_DISLINK_TABLE);
5091 hash_table->table = new_hash;
5092 hash_table->size = new_size;
5095 /* LOCKING: assumes the GC lock is held */
5097 add_or_remove_disappearing_link (MonoObject *obj, void **link, gboolean track, int generation)
5099 DisappearingLinkHashTable *hash_table = get_dislink_hash_table (generation);
5100 DisappearingLink *entry, *prev;
5102 DisappearingLink **disappearing_link_hash = hash_table->table;
5103 int disappearing_link_hash_size = hash_table->size;
5105 if (hash_table->num_links >= disappearing_link_hash_size * 2) {
5106 rehash_dislink (hash_table);
5107 disappearing_link_hash = hash_table->table;
5108 disappearing_link_hash_size = hash_table->size;
5110 /* FIXME: add check that link is not in the heap */
5111 hash = mono_aligned_addr_hash (link) % disappearing_link_hash_size;
5112 entry = disappearing_link_hash [hash];
5114 for (; entry; entry = entry->next) {
5115 /* link already added */
5116 if (link == entry->link) {
5117 /* NULL obj means remove */
5120 prev->next = entry->next;
5122 disappearing_link_hash [hash] = entry->next;
5123 hash_table->num_links--;
5124 DEBUG (5, fprintf (gc_debug_file, "Removed dislink %p (%d) from %s table\n", entry, hash_table->num_links, generation_name (generation)));
5125 free_internal_mem (entry, INTERNAL_MEM_DISLINK);
5128 *link = HIDE_POINTER (obj, track); /* we allow the change of object */
5136 entry = get_internal_mem (sizeof (DisappearingLink), INTERNAL_MEM_DISLINK);
5137 *link = HIDE_POINTER (obj, track);
5139 entry->next = disappearing_link_hash [hash];
5140 disappearing_link_hash [hash] = entry;
5141 hash_table->num_links++;
5142 DEBUG (5, fprintf (gc_debug_file, "Added dislink %p for object: %p (%s) at %p to %s table\n", entry, obj, obj->vtable->klass->name, link, generation_name (generation)));
5145 /* LOCKING: assumes the GC lock is held */
5147 mono_gc_register_disappearing_link (MonoObject *obj, void **link, gboolean track)
5149 add_or_remove_disappearing_link (NULL, link, FALSE, GENERATION_NURSERY);
5150 add_or_remove_disappearing_link (NULL, link, FALSE, GENERATION_OLD);
5152 if (ptr_in_nursery (obj))
5153 add_or_remove_disappearing_link (obj, link, track, GENERATION_NURSERY);
5155 add_or_remove_disappearing_link (obj, link, track, GENERATION_OLD);
5160 mono_gc_invoke_finalizers (void)
5162 FinalizeEntry *entry = NULL;
5163 gboolean entry_is_critical = FALSE;
5166 /* FIXME: batch to reduce lock contention */
5167 while (fin_ready_list || critical_fin_list) {
5171 FinalizeEntry **list = entry_is_critical ? &critical_fin_list : &fin_ready_list;
5173 /* We have finalized entry in the last
5174 interation, now we need to remove it from
5177 *list = entry->next;
5179 FinalizeEntry *e = *list;
5180 while (e->next != entry)
5182 e->next = entry->next;
5184 free_internal_mem (entry, INTERNAL_MEM_FINALIZE_ENTRY);
5188 /* Now look for the first non-null entry. */
5189 for (entry = fin_ready_list; entry && !entry->object; entry = entry->next)
5192 entry_is_critical = FALSE;
5194 entry_is_critical = TRUE;
5195 for (entry = critical_fin_list; entry && !entry->object; entry = entry->next)
5200 g_assert (entry->object);
5201 num_ready_finalizers--;
5202 obj = entry->object;
5203 entry->object = NULL;
5204 DEBUG (7, fprintf (gc_debug_file, "Finalizing object %p (%s)\n", obj, safe_name (obj)));
5212 g_assert (entry->object == NULL);
5214 /* the object is on the stack so it is pinned */
5215 /*g_print ("Calling finalizer for object: %p (%s)\n", entry->object, safe_name (entry->object));*/
5216 mono_gc_run_finalize (obj, NULL);
5223 mono_gc_pending_finalizers (void)
5225 return fin_ready_list || critical_fin_list;
5228 /* Negative value to remove */
5230 mono_gc_add_memory_pressure (gint64 value)
5232 /* FIXME: Use interlocked functions */
5234 memory_pressure += value;
5239 * ######################################################################
5240 * ######## registered roots support
5241 * ######################################################################
5245 rehash_roots (gboolean pinned)
5249 RootRecord **new_hash;
5250 RootRecord *entry, *next;
5253 new_size = g_spaced_primes_closest (num_roots_entries [pinned]);
5254 new_hash = get_internal_mem (new_size * sizeof (RootRecord*), INTERNAL_MEM_ROOTS_TABLE);
5255 for (i = 0; i < roots_hash_size [pinned]; ++i) {
5256 for (entry = roots_hash [pinned][i]; entry; entry = next) {
5257 hash = mono_aligned_addr_hash (entry->start_root) % new_size;
5259 entry->next = new_hash [hash];
5260 new_hash [hash] = entry;
5263 free_internal_mem (roots_hash [pinned], INTERNAL_MEM_ROOTS_TABLE);
5264 roots_hash [pinned] = new_hash;
5265 roots_hash_size [pinned] = new_size;
5269 find_root (int root_type, char *start, guint32 addr_hash)
5271 RootRecord *new_root;
5273 guint32 hash = addr_hash % roots_hash_size [root_type];
5274 for (new_root = roots_hash [root_type][hash]; new_root; new_root = new_root->next) {
5275 /* we allow changing the size and the descriptor (for thread statics etc) */
5276 if (new_root->start_root == start) {
5285 * We do not coalesce roots.
5288 mono_gc_register_root_inner (char *start, size_t size, void *descr, int root_type)
5290 RootRecord *new_root;
5291 unsigned int hash, addr_hash = mono_aligned_addr_hash (start);
5294 for (i = 0; i < ROOT_TYPE_NUM; ++i) {
5295 if (num_roots_entries [i] >= roots_hash_size [i] * 2)
5298 for (i = 0; i < ROOT_TYPE_NUM; ++i) {
5299 new_root = find_root (i, start, addr_hash);
5300 /* we allow changing the size and the descriptor (for thread statics etc) */
5302 size_t old_size = new_root->end_root - new_root->start_root;
5303 new_root->end_root = new_root->start_root + size;
5304 g_assert (((new_root->root_desc != 0) && (descr != NULL)) ||
5305 ((new_root->root_desc == 0) && (descr == NULL)));
5306 new_root->root_desc = (mword)descr;
5308 roots_size -= old_size;
5313 new_root = get_internal_mem (sizeof (RootRecord), INTERNAL_MEM_ROOT_RECORD);
5315 new_root->start_root = start;
5316 new_root->end_root = new_root->start_root + size;
5317 new_root->root_desc = (mword)descr;
5319 hash = addr_hash % roots_hash_size [root_type];
5320 num_roots_entries [root_type]++;
5321 new_root->next = roots_hash [root_type] [hash];
5322 roots_hash [root_type][hash] = new_root;
5323 DEBUG (3, fprintf (gc_debug_file, "Added root %p for range: %p-%p, descr: %p (%d/%d bytes)\n", new_root, new_root->start_root, new_root->end_root, descr, (int)size, (int)roots_size));
5333 mono_gc_register_root (char *start, size_t size, void *descr)
5335 return mono_gc_register_root_inner (start, size, descr, descr ? ROOT_TYPE_NORMAL : ROOT_TYPE_PINNED);
5339 mono_gc_register_root_wbarrier (char *start, size_t size, void *descr)
5341 return mono_gc_register_root_inner (start, size, descr, ROOT_TYPE_WBARRIER);
5345 mono_gc_deregister_root (char* addr)
5347 RootRecord *tmp, *prev;
5348 unsigned int hash, addr_hash = mono_aligned_addr_hash (addr);
5352 for (root_type = 0; root_type < ROOT_TYPE_NUM; ++root_type) {
5353 hash = addr_hash % roots_hash_size [root_type];
5354 tmp = roots_hash [root_type][hash];
5357 if (tmp->start_root == (char*)addr) {
5359 prev->next = tmp->next;
5361 roots_hash [root_type][hash] = tmp->next;
5362 roots_size -= (tmp->end_root - tmp->start_root);
5363 num_roots_entries [root_type]--;
5364 DEBUG (3, fprintf (gc_debug_file, "Removed root %p for range: %p-%p\n", tmp, tmp->start_root, tmp->end_root));
5365 free_internal_mem (tmp, INTERNAL_MEM_ROOT_RECORD);
5376 * ######################################################################
5377 * ######## Thread handling (stop/start code)
5378 * ######################################################################
5381 /* FIXME: handle large/small config */
5382 #define THREAD_HASH_SIZE 11
5383 #define HASH_PTHREAD_T(id) (((unsigned int)(id) >> 4) * 2654435761u)
5385 static SgenThreadInfo* thread_table [THREAD_HASH_SIZE];
5387 #if USE_SIGNAL_BASED_START_STOP_WORLD
5389 static MonoSemType suspend_ack_semaphore;
5390 static MonoSemType *suspend_ack_semaphore_ptr;
5391 static unsigned int global_stop_count = 0;
5393 static int suspend_signal_num = SIGXFSZ;
5395 static int suspend_signal_num = SIGPWR;
5397 static int restart_signal_num = SIGXCPU;
5398 static sigset_t suspend_signal_mask;
5399 static mword cur_thread_regs [ARCH_NUM_REGS] = {0};
5401 /* LOCKING: assumes the GC lock is held */
5402 static SgenThreadInfo*
5403 thread_info_lookup (ARCH_THREAD_TYPE id)
5405 unsigned int hash = HASH_PTHREAD_T (id) % THREAD_HASH_SIZE;
5406 SgenThreadInfo *info;
5408 info = thread_table [hash];
5409 while (info && !ARCH_THREAD_EQUALS (info->id, id)) {
5416 update_current_thread_stack (void *start)
5418 void *ptr = cur_thread_regs;
5419 SgenThreadInfo *info = thread_info_lookup (ARCH_GET_THREAD ());
5421 info->stack_start = align_pointer (&ptr);
5422 g_assert (info->stack_start >= info->stack_start_limit && info->stack_start < info->stack_end);
5423 ARCH_STORE_REGS (ptr);
5424 info->stopped_regs = ptr;
5425 if (gc_callbacks.thread_suspend_func)
5426 gc_callbacks.thread_suspend_func (info->runtime_data, NULL);
5430 signal_desc (int signum)
5432 if (signum == suspend_signal_num)
5434 if (signum == restart_signal_num)
5440 * Define this and use the "xdomain-checks" MONO_GC_DEBUG option to
5441 * have cross-domain checks in the write barrier.
5443 //#define XDOMAIN_CHECKS_IN_WBARRIER
5445 #ifndef HEAVY_STATISTICS
5446 #define MANAGED_ALLOCATION
5447 #ifndef XDOMAIN_CHECKS_IN_WBARRIER
5448 #define MANAGED_WBARRIER
5453 is_ip_in_managed_allocator (MonoDomain *domain, gpointer ip);
5456 wait_for_suspend_ack (int count)
5460 for (i = 0; i < count; ++i) {
5461 while ((result = MONO_SEM_WAIT (suspend_ack_semaphore_ptr)) != 0) {
5462 if (errno != EINTR) {
5463 g_error ("sem_wait ()");
5469 /* LOCKING: assumes the GC lock is held */
5471 thread_handshake (int signum)
5473 int count, i, result;
5474 SgenThreadInfo *info;
5475 pthread_t me = pthread_self ();
5478 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5479 for (info = thread_table [i]; info; info = info->next) {
5480 DEBUG (4, fprintf (gc_debug_file, "considering thread %p for signal %d (%s)\n", info, signum, signal_desc (signum)));
5481 if (ARCH_THREAD_EQUALS (info->id, me)) {
5482 DEBUG (4, fprintf (gc_debug_file, "Skip (equal): %p, %p\n", (void*)me, (void*)info->id));
5485 /*if (signum == suspend_signal_num && info->stop_count == global_stop_count)
5487 result = pthread_kill (info->id, signum);
5489 DEBUG (4, fprintf (gc_debug_file, "thread %p signal sent\n", info));
5492 DEBUG (4, fprintf (gc_debug_file, "thread %p signal failed: %d (%s)\n", (void*)info->id, result, strerror (result)));
5498 wait_for_suspend_ack (count);
5504 restart_threads_until_none_in_managed_allocator (void)
5506 SgenThreadInfo *info;
5507 int i, result, num_threads_died = 0;
5508 int sleep_duration = -1;
5511 int restart_count = 0, restarted_count = 0;
5512 /* restart all threads that stopped in the
5514 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5515 for (info = thread_table [i]; info; info = info->next) {
5518 if (!info->stack_start || info->in_critical_region ||
5519 is_ip_in_managed_allocator (info->stopped_domain, info->stopped_ip)) {
5520 binary_protocol_thread_restart ((gpointer)info->id);
5521 result = pthread_kill (info->id, restart_signal_num);
5528 /* we set the stopped_ip to
5529 NULL for threads which
5530 we're not restarting so
5531 that we can easily identify
5533 info->stopped_ip = NULL;
5534 info->stopped_domain = NULL;
5538 /* if no threads were restarted, we're done */
5539 if (restart_count == 0)
5542 /* wait for the threads to signal their restart */
5543 wait_for_suspend_ack (restart_count);
5545 if (sleep_duration < 0) {
5549 g_usleep (sleep_duration);
5550 sleep_duration += 10;
5553 /* stop them again */
5554 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5555 for (info = thread_table [i]; info; info = info->next) {
5556 if (info->skip || info->stopped_ip == NULL)
5558 result = pthread_kill (info->id, suspend_signal_num);
5566 /* some threads might have died */
5567 num_threads_died += restart_count - restarted_count;
5568 /* wait for the threads to signal their suspension
5570 wait_for_suspend_ack (restart_count);
5573 return num_threads_died;
5576 /* LOCKING: assumes the GC lock is held (by the stopping thread) */
5578 suspend_handler (int sig, siginfo_t *siginfo, void *context)
5580 SgenThreadInfo *info;
5583 int old_errno = errno;
5584 gpointer regs [ARCH_NUM_REGS];
5585 gpointer stack_start;
5587 id = pthread_self ();
5588 info = thread_info_lookup (id);
5589 info->stopped_domain = mono_domain_get ();
5590 info->stopped_ip = (gpointer) ARCH_SIGCTX_IP (context);
5591 stop_count = global_stop_count;
5592 /* duplicate signal */
5593 if (0 && info->stop_count == stop_count) {
5597 #ifdef HAVE_KW_THREAD
5598 /* update the remset info in the thread data structure */
5599 info->remset = remembered_set;
5601 stack_start = (char*) ARCH_SIGCTX_SP (context) - REDZONE_SIZE;
5602 /* If stack_start is not within the limits, then don't set it
5603 in info and we will be restarted. */
5604 if (stack_start >= info->stack_start_limit && info->stack_start <= info->stack_end) {
5605 info->stack_start = stack_start;
5607 ARCH_COPY_SIGCTX_REGS (regs, context);
5608 info->stopped_regs = regs;
5610 g_assert (!info->stack_start);
5613 /* Notify the JIT */
5614 if (gc_callbacks.thread_suspend_func)
5615 gc_callbacks.thread_suspend_func (info->runtime_data, context);
5617 DEBUG (4, fprintf (gc_debug_file, "Posting suspend_ack_semaphore for suspend from %p %p\n", info, (gpointer)ARCH_GET_THREAD ()));
5618 /* notify the waiting thread */
5619 MONO_SEM_POST (suspend_ack_semaphore_ptr);
5620 info->stop_count = stop_count;
5622 /* wait until we receive the restart signal */
5625 sigsuspend (&suspend_signal_mask);
5626 } while (info->signal != restart_signal_num);
5628 DEBUG (4, fprintf (gc_debug_file, "Posting suspend_ack_semaphore for resume from %p %p\n", info, (gpointer)ARCH_GET_THREAD ()));
5629 /* notify the waiting thread */
5630 MONO_SEM_POST (suspend_ack_semaphore_ptr);
5636 restart_handler (int sig)
5638 SgenThreadInfo *info;
5639 int old_errno = errno;
5641 info = thread_info_lookup (pthread_self ());
5642 info->signal = restart_signal_num;
5643 DEBUG (4, fprintf (gc_debug_file, "Restart handler in %p %p\n", info, (gpointer)ARCH_GET_THREAD ()));
5648 static TV_DECLARE (stop_world_time);
5649 static unsigned long max_pause_usec = 0;
5651 /* LOCKING: assumes the GC lock is held */
5657 update_current_thread_stack (&count);
5659 global_stop_count++;
5660 DEBUG (3, fprintf (gc_debug_file, "stopping world n %d from %p %p\n", global_stop_count, thread_info_lookup (ARCH_GET_THREAD ()), (gpointer)ARCH_GET_THREAD ()));
5661 TV_GETTIME (stop_world_time);
5662 count = thread_handshake (suspend_signal_num);
5663 count -= restart_threads_until_none_in_managed_allocator ();
5664 g_assert (count >= 0);
5665 DEBUG (3, fprintf (gc_debug_file, "world stopped %d thread(s)\n", count));
5669 /* LOCKING: assumes the GC lock is held */
5671 restart_world (void)
5674 SgenThreadInfo *info;
5675 TV_DECLARE (end_sw);
5678 /* notify the profiler of the leftovers */
5679 if (G_UNLIKELY (mono_profiler_events & MONO_PROFILE_GC_MOVES)) {
5680 if (moved_objects_idx) {
5681 mono_profiler_gc_moves (moved_objects, moved_objects_idx);
5682 moved_objects_idx = 0;
5685 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5686 for (info = thread_table [i]; info; info = info->next) {
5687 info->stack_start = NULL;
5688 info->stopped_regs = NULL;
5692 count = thread_handshake (restart_signal_num);
5693 TV_GETTIME (end_sw);
5694 usec = TV_ELAPSED (stop_world_time, end_sw);
5695 max_pause_usec = MAX (usec, max_pause_usec);
5696 DEBUG (2, fprintf (gc_debug_file, "restarted %d thread(s) (pause time: %d usec, max: %d)\n", count, (int)usec, (int)max_pause_usec));
5700 #endif /* USE_SIGNAL_BASED_START_STOP_WORLD */
5703 mono_gc_set_gc_callbacks (MonoGCCallbacks *callbacks)
5705 gc_callbacks = *callbacks;
5708 /* Variables holding start/end nursery so it won't have to be passed at every call */
5709 static void *scan_area_arg_start, *scan_area_arg_end;
5712 mono_gc_conservatively_scan_area (void *start, void *end)
5714 conservatively_pin_objects_from (start, end, scan_area_arg_start, scan_area_arg_end, PIN_TYPE_STACK);
5718 mono_gc_scan_object (void *obj)
5720 return copy_object (obj, scan_area_arg_start, scan_area_arg_end);
5724 * Mark from thread stacks and registers.
5727 scan_thread_data (void *start_nursery, void *end_nursery, gboolean precise)
5730 SgenThreadInfo *info;
5732 scan_area_arg_start = start_nursery;
5733 scan_area_arg_end = end_nursery;
5735 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5736 for (info = thread_table [i]; info; info = info->next) {
5738 DEBUG (2, fprintf (gc_debug_file, "Skipping dead thread %p, range: %p-%p, size: %zd\n", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start));
5741 DEBUG (3, fprintf (gc_debug_file, "Scanning thread %p, range: %p-%p, size: %zd, pinned=%d\n", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start, next_pin_slot));
5742 if (gc_callbacks.thread_mark_func && !conservative_stack_mark)
5743 gc_callbacks.thread_mark_func (info->runtime_data, info->stack_start, info->stack_end, precise);
5745 conservatively_pin_objects_from (info->stack_start, info->stack_end, start_nursery, end_nursery, PIN_TYPE_STACK);
5748 conservatively_pin_objects_from (info->stopped_regs, info->stopped_regs + ARCH_NUM_REGS,
5749 start_nursery, end_nursery, PIN_TYPE_STACK);
5755 find_pinning_ref_from_thread (char *obj, size_t size)
5758 SgenThreadInfo *info;
5759 char *endobj = obj + size;
5761 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5762 for (info = thread_table [i]; info; info = info->next) {
5763 char **start = (char**)info->stack_start;
5766 while (start < (char**)info->stack_end) {
5767 if (*start >= obj && *start < endobj) {
5768 DEBUG (0, fprintf (gc_debug_file, "Object %p referenced in thread %p (id %p) at %p, stack: %p-%p\n", obj, info, (gpointer)info->id, start, info->stack_start, info->stack_end));
5773 /* FIXME: check info->stopped_regs */
5779 ptr_on_stack (void *ptr)
5781 gpointer stack_start = &stack_start;
5782 SgenThreadInfo *info = thread_info_lookup (ARCH_GET_THREAD ());
5784 if (ptr >= stack_start && ptr < (gpointer)info->stack_end)
5790 handle_remset (mword *p, void *start_nursery, void *end_nursery, gboolean global)
5797 HEAVY_STAT (++stat_global_remsets_processed);
5799 /* FIXME: exclude stack locations */
5800 switch ((*p) & REMSET_TYPE_MASK) {
5801 case REMSET_LOCATION:
5803 //__builtin_prefetch (ptr);
5804 if (((void*)ptr < start_nursery || (void*)ptr >= end_nursery)) {
5805 gpointer old = *ptr;
5806 *ptr = copy_object (*ptr, start_nursery, end_nursery);
5807 DEBUG (9, fprintf (gc_debug_file, "Overwrote remset at %p with %p\n", ptr, *ptr));
5809 binary_protocol_ptr_update (ptr, old, *ptr, (gpointer)LOAD_VTABLE (*ptr), safe_object_get_size (*ptr));
5810 if (!global && *ptr >= start_nursery && *ptr < end_nursery) {
5812 * If the object is pinned, each reference to it from nonpinned objects
5813 * becomes part of the global remset, which can grow very large.
5815 DEBUG (9, fprintf (gc_debug_file, "Add to global remset because of pinning %p (%p %s)\n", ptr, *ptr, safe_name (*ptr)));
5816 add_to_global_remset (ptr, FALSE);
5819 DEBUG (9, fprintf (gc_debug_file, "Skipping remset at %p holding %p\n", ptr, *ptr));
5823 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
5824 if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery))
5827 while (count-- > 0) {
5828 *ptr = copy_object (*ptr, start_nursery, end_nursery);
5829 DEBUG (9, fprintf (gc_debug_file, "Overwrote remset at %p with %p (count: %d)\n", ptr, *ptr, (int)count));
5830 if (!global && *ptr >= start_nursery && *ptr < end_nursery)
5831 add_to_global_remset (ptr, FALSE);
5836 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
5837 if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery))
5839 scan_object ((char*)ptr, start_nursery, end_nursery);
5841 case REMSET_OTHER: {
5842 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
5846 if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery))
5851 ptr = (void**) scan_vtype ((char*)ptr, desc, start_nursery, end_nursery);
5853 case REMSET_ROOT_LOCATION:
5854 /* Same as REMSET_LOCATION, but the address is not required to be in the heap */
5855 *ptr = copy_object (*ptr, start_nursery, end_nursery);
5856 DEBUG (9, fprintf (gc_debug_file, "Overwrote root location remset at %p with %p\n", ptr, *ptr));
5857 if (!global && *ptr >= start_nursery && *ptr < end_nursery) {
5859 * If the object is pinned, each reference to it from nonpinned objects
5860 * becomes part of the global remset, which can grow very large.
5862 DEBUG (9, fprintf (gc_debug_file, "Add to global remset because of pinning %p (%p %s)\n", ptr, *ptr, safe_name (*ptr)));
5863 add_to_global_remset (ptr, TRUE);
5867 g_assert_not_reached ();
5872 g_assert_not_reached ();
5877 #ifdef HEAVY_STATISTICS
5879 collect_store_remsets (RememberedSet *remset, mword *bumper)
5881 mword *p = remset->data;
5886 while (p < remset->store_next) {
5887 switch ((*p) & REMSET_TYPE_MASK) {
5888 case REMSET_LOCATION:
5891 ++stat_saved_remsets_1;
5893 if (*p == last1 || *p == last2) {
5894 ++stat_saved_remsets_2;
5912 case REMSET_ROOT_LOCATION:
5916 g_assert_not_reached ();
5920 g_assert_not_reached ();
5930 RememberedSet *remset;
5932 SgenThreadInfo *info;
5934 mword *addresses, *bumper, *p, *r;
5936 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5937 for (info = thread_table [i]; info; info = info->next) {
5938 for (remset = info->remset; remset; remset = remset->next)
5939 size += remset->store_next - remset->data;
5942 for (remset = freed_thread_remsets; remset; remset = remset->next)
5943 size += remset->store_next - remset->data;
5944 for (remset = global_remset; remset; remset = remset->next)
5945 size += remset->store_next - remset->data;
5947 bumper = addresses = get_internal_mem (sizeof (mword) * size, INTERNAL_MEM_STATISTICS);
5949 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5950 for (info = thread_table [i]; info; info = info->next) {
5951 for (remset = info->remset; remset; remset = remset->next)
5952 bumper = collect_store_remsets (remset, bumper);
5955 for (remset = global_remset; remset; remset = remset->next)
5956 bumper = collect_store_remsets (remset, bumper);
5957 for (remset = freed_thread_remsets; remset; remset = remset->next)
5958 bumper = collect_store_remsets (remset, bumper);
5960 g_assert (bumper <= addresses + size);
5962 stat_store_remsets += bumper - addresses;
5964 sort_addresses ((void**)addresses, bumper - addresses);
5967 while (r < bumper) {
5973 stat_store_remsets_unique += p - addresses;
5975 free_internal_mem (addresses, INTERNAL_MEM_STATISTICS);
5980 clear_thread_store_remset_buffer (SgenThreadInfo *info)
5982 *info->store_remset_buffer_index_addr = 0;
5983 memset (*info->store_remset_buffer_addr, 0, sizeof (gpointer) * STORE_REMSET_BUFFER_SIZE);
5987 scan_from_remsets (void *start_nursery, void *end_nursery)
5990 SgenThreadInfo *info;
5991 RememberedSet *remset;
5992 GenericStoreRememberedSet *store_remset;
5993 mword *p, *next_p, *store_pos;
5995 #ifdef HEAVY_STATISTICS
5999 /* the global one */
6000 for (remset = global_remset; remset; remset = remset->next) {
6001 DEBUG (4, fprintf (gc_debug_file, "Scanning global remset range: %p-%p, size: %zd\n", remset->data, remset->store_next, remset->store_next - remset->data));
6002 store_pos = remset->data;
6003 for (p = remset->data; p < remset->store_next; p = next_p) {
6006 next_p = handle_remset (p, start_nursery, end_nursery, TRUE);
6009 * Clear global remsets of locations which no longer point to the
6010 * nursery. Otherwise, they could grow indefinitely between major
6013 ptr = (p [0] & ~REMSET_TYPE_MASK);
6014 if ((p [0] & REMSET_TYPE_MASK) == REMSET_LOCATION) {
6015 if (ptr_in_nursery (*(void**)ptr))
6016 *store_pos ++ = p [0];
6018 g_assert ((p [0] & REMSET_TYPE_MASK) == REMSET_OTHER);
6019 g_assert (p [1] == REMSET_ROOT_LOCATION);
6020 if (ptr_in_nursery (*(void**)ptr)) {
6021 *store_pos ++ = p [0];
6022 *store_pos ++ = p [1];
6027 /* Truncate the remset */
6028 remset->store_next = store_pos;
6031 /* the generic store ones */
6032 store_remset = generic_store_remsets;
6033 while (store_remset) {
6034 GenericStoreRememberedSet *next = store_remset->next;
6036 for (i = 0; i < STORE_REMSET_BUFFER_SIZE - 1; ++i) {
6037 gpointer addr = store_remset->data [i];
6039 handle_remset ((mword*)&addr, start_nursery, end_nursery, FALSE);
6042 free_internal_mem (store_remset, INTERNAL_MEM_STORE_REMSET);
6044 store_remset = next;
6046 generic_store_remsets = NULL;
6048 /* the per-thread ones */
6049 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
6050 for (info = thread_table [i]; info; info = info->next) {
6051 RememberedSet *next;
6053 for (remset = info->remset; remset; remset = next) {
6054 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for thread %p, range: %p-%p, size: %zd\n", info, remset->data, remset->store_next, remset->store_next - remset->data));
6055 for (p = remset->data; p < remset->store_next;) {
6056 p = handle_remset (p, start_nursery, end_nursery, FALSE);
6058 remset->store_next = remset->data;
6059 next = remset->next;
6060 remset->next = NULL;
6061 if (remset != info->remset) {
6062 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
6063 free_internal_mem (remset, INTERNAL_MEM_REMSET);
6066 for (j = 0; j < *info->store_remset_buffer_index_addr; ++j)
6067 handle_remset ((mword*)*info->store_remset_buffer_addr + j + 1, start_nursery, end_nursery, FALSE);
6068 clear_thread_store_remset_buffer (info);
6072 /* the freed thread ones */
6073 while (freed_thread_remsets) {
6074 RememberedSet *next;
6075 remset = freed_thread_remsets;
6076 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for freed thread, range: %p-%p, size: %zd\n", remset->data, remset->store_next, remset->store_next - remset->data));
6077 for (p = remset->data; p < remset->store_next;) {
6078 p = handle_remset (p, start_nursery, end_nursery, FALSE);
6080 next = remset->next;
6081 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
6082 free_internal_mem (remset, INTERNAL_MEM_REMSET);
6083 freed_thread_remsets = next;
6088 * Clear the info in the remembered sets: we're doing a major collection, so
6089 * the per-thread ones are not needed and the global ones will be reconstructed
6093 clear_remsets (void)
6096 SgenThreadInfo *info;
6097 RememberedSet *remset, *next;
6099 /* the global list */
6100 for (remset = global_remset; remset; remset = next) {
6101 remset->store_next = remset->data;
6102 next = remset->next;
6103 remset->next = NULL;
6104 if (remset != global_remset) {
6105 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
6106 free_internal_mem (remset, INTERNAL_MEM_REMSET);
6109 /* the generic store ones */
6110 while (generic_store_remsets) {
6111 GenericStoreRememberedSet *gs_next = generic_store_remsets->next;
6112 free_internal_mem (generic_store_remsets, INTERNAL_MEM_STORE_REMSET);
6113 generic_store_remsets = gs_next;
6115 /* the per-thread ones */
6116 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
6117 for (info = thread_table [i]; info; info = info->next) {
6118 for (remset = info->remset; remset; remset = next) {
6119 remset->store_next = remset->data;
6120 next = remset->next;
6121 remset->next = NULL;
6122 if (remset != info->remset) {
6123 DEBUG (1, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
6124 free_internal_mem (remset, INTERNAL_MEM_REMSET);
6127 clear_thread_store_remset_buffer (info);
6131 /* the freed thread ones */
6132 while (freed_thread_remsets) {
6133 next = freed_thread_remsets->next;
6134 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", freed_thread_remsets->data));
6135 free_internal_mem (freed_thread_remsets, INTERNAL_MEM_REMSET);
6136 freed_thread_remsets = next;
6141 * Clear the thread local TLAB variables for all threads.
6146 SgenThreadInfo *info;
6149 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
6150 for (info = thread_table [i]; info; info = info->next) {
6151 /* A new TLAB will be allocated when the thread does its first allocation */
6152 *info->tlab_start_addr = NULL;
6153 *info->tlab_next_addr = NULL;
6154 *info->tlab_temp_end_addr = NULL;
6155 *info->tlab_real_end_addr = NULL;
6160 /* LOCKING: assumes the GC lock is held */
6161 static SgenThreadInfo*
6162 gc_register_current_thread (void *addr)
6165 SgenThreadInfo* info = malloc (sizeof (SgenThreadInfo));
6166 #ifndef HAVE_KW_THREAD
6167 SgenThreadInfo *__thread_info__ = info;
6173 memset (info, 0, sizeof (SgenThreadInfo));
6174 #ifndef HAVE_KW_THREAD
6175 info->tlab_start = info->tlab_next = info->tlab_temp_end = info->tlab_real_end = NULL;
6177 g_assert (!pthread_getspecific (thread_info_key));
6178 pthread_setspecific (thread_info_key, info);
6183 info->id = ARCH_GET_THREAD ();
6184 info->stop_count = -1;
6187 info->stack_start = NULL;
6188 info->tlab_start_addr = &TLAB_START;
6189 info->tlab_next_addr = &TLAB_NEXT;
6190 info->tlab_temp_end_addr = &TLAB_TEMP_END;
6191 info->tlab_real_end_addr = &TLAB_REAL_END;
6192 info->store_remset_buffer_addr = &STORE_REMSET_BUFFER;
6193 info->store_remset_buffer_index_addr = &STORE_REMSET_BUFFER_INDEX;
6194 info->stopped_ip = NULL;
6195 info->stopped_domain = NULL;
6196 info->stopped_regs = NULL;
6198 binary_protocol_thread_register ((gpointer)info->id);
6200 #ifdef HAVE_KW_THREAD
6201 tlab_next_addr = &tlab_next;
6202 store_remset_buffer_index_addr = &store_remset_buffer_index;
6205 /* try to get it with attributes first */
6206 #if defined(HAVE_PTHREAD_GETATTR_NP) && defined(HAVE_PTHREAD_ATTR_GETSTACK)
6210 pthread_attr_t attr;
6211 pthread_getattr_np (pthread_self (), &attr);
6212 pthread_attr_getstack (&attr, &sstart, &size);
6213 info->stack_start_limit = sstart;
6214 info->stack_end = (char*)sstart + size;
6215 pthread_attr_destroy (&attr);
6217 #elif defined(HAVE_PTHREAD_GET_STACKSIZE_NP) && defined(HAVE_PTHREAD_GET_STACKADDR_NP)
6218 info->stack_end = (char*)pthread_get_stackaddr_np (pthread_self ());
6221 /* FIXME: we assume the stack grows down */
6222 gsize stack_bottom = (gsize)addr;
6223 stack_bottom += 4095;
6224 stack_bottom &= ~4095;
6225 info->stack_end = (char*)stack_bottom;
6229 #ifdef HAVE_KW_THREAD
6230 stack_end = info->stack_end;
6233 /* hash into the table */
6234 hash = HASH_PTHREAD_T (info->id) % THREAD_HASH_SIZE;
6235 info->next = thread_table [hash];
6236 thread_table [hash] = info;
6238 info->remset = alloc_remset (DEFAULT_REMSET_SIZE, info);
6239 pthread_setspecific (remembered_set_key, info->remset);
6240 #ifdef HAVE_KW_THREAD
6241 remembered_set = info->remset;
6244 STORE_REMSET_BUFFER = get_internal_mem (sizeof (gpointer) * STORE_REMSET_BUFFER_SIZE, INTERNAL_MEM_STORE_REMSET);
6245 STORE_REMSET_BUFFER_INDEX = 0;
6247 DEBUG (3, fprintf (gc_debug_file, "registered thread %p (%p) (hash: %d)\n", info, (gpointer)info->id, hash));
6249 if (gc_callbacks.thread_attach_func)
6250 info->runtime_data = gc_callbacks.thread_attach_func ();
6256 add_generic_store_remset_from_buffer (gpointer *buffer)
6258 GenericStoreRememberedSet *remset = get_internal_mem (sizeof (GenericStoreRememberedSet), INTERNAL_MEM_STORE_REMSET);
6259 memcpy (remset->data, buffer + 1, sizeof (gpointer) * (STORE_REMSET_BUFFER_SIZE - 1));
6260 remset->next = generic_store_remsets;
6261 generic_store_remsets = remset;
6265 unregister_current_thread (void)
6268 SgenThreadInfo *prev = NULL;
6270 RememberedSet *rset;
6271 ARCH_THREAD_TYPE id = ARCH_GET_THREAD ();
6273 binary_protocol_thread_unregister ((gpointer)id);
6275 hash = HASH_PTHREAD_T (id) % THREAD_HASH_SIZE;
6276 p = thread_table [hash];
6278 DEBUG (3, fprintf (gc_debug_file, "unregister thread %p (%p)\n", p, (gpointer)p->id));
6279 while (!ARCH_THREAD_EQUALS (p->id, id)) {
6284 thread_table [hash] = p->next;
6286 prev->next = p->next;
6289 if (freed_thread_remsets) {
6290 for (rset = p->remset; rset->next; rset = rset->next)
6292 rset->next = freed_thread_remsets;
6293 freed_thread_remsets = p->remset;
6295 freed_thread_remsets = p->remset;
6298 if (*p->store_remset_buffer_index_addr)
6299 add_generic_store_remset_from_buffer (*p->store_remset_buffer_addr);
6300 free_internal_mem (*p->store_remset_buffer_addr, INTERNAL_MEM_STORE_REMSET);
6305 unregister_thread (void *k)
6307 g_assert (!mono_domain_get ());
6309 unregister_current_thread ();
6314 mono_gc_register_thread (void *baseptr)
6316 SgenThreadInfo *info;
6320 info = thread_info_lookup (ARCH_GET_THREAD ());
6322 info = gc_register_current_thread (baseptr);
6324 return info != NULL;
6327 #if USE_PTHREAD_INTERCEPT
6329 #undef pthread_create
6331 #undef pthread_detach
6334 void *(*start_routine) (void *);
6337 MonoSemType registered;
6338 } SgenThreadStartInfo;
6341 gc_start_thread (void *arg)
6343 SgenThreadStartInfo *start_info = arg;
6344 SgenThreadInfo* info;
6345 void *t_arg = start_info->arg;
6346 void *(*start_func) (void*) = start_info->start_routine;
6351 info = gc_register_current_thread (&result);
6353 post_result = MONO_SEM_POST (&(start_info->registered));
6354 g_assert (!post_result);
6355 result = start_func (t_arg);
6356 g_assert (!mono_domain_get ());
6358 * this is done by the pthread key dtor
6360 unregister_current_thread ();
6368 mono_gc_pthread_create (pthread_t *new_thread, const pthread_attr_t *attr, void *(*start_routine)(void *), void *arg)
6370 SgenThreadStartInfo *start_info;
6373 start_info = malloc (sizeof (SgenThreadStartInfo));
6376 result = MONO_SEM_INIT (&(start_info->registered), 0);
6378 start_info->arg = arg;
6379 start_info->start_routine = start_routine;
6381 result = pthread_create (new_thread, attr, gc_start_thread, start_info);
6383 while (MONO_SEM_WAIT (&(start_info->registered)) != 0) {
6384 /*if (EINTR != errno) ABORT("sem_wait failed"); */
6387 MONO_SEM_DESTROY (&(start_info->registered));
6393 mono_gc_pthread_join (pthread_t thread, void **retval)
6395 return pthread_join (thread, retval);
6399 mono_gc_pthread_detach (pthread_t thread)
6401 return pthread_detach (thread);
6404 #endif /* USE_PTHREAD_INTERCEPT */
6407 * ######################################################################
6408 * ######## Write barriers
6409 * ######################################################################
6412 static RememberedSet*
6413 alloc_remset (int size, gpointer id) {
6414 RememberedSet* res = get_internal_mem (sizeof (RememberedSet) + (size * sizeof (gpointer)), INTERNAL_MEM_REMSET);
6415 res->store_next = res->data;
6416 res->end_set = res->data + size;
6418 DEBUG (4, fprintf (gc_debug_file, "Allocated remset size %d at %p for %p\n", size, res->data, id));
6423 * Note: the write barriers first do the needed GC work and then do the actual store:
6424 * this way the value is visible to the conservative GC scan after the write barrier
6425 * itself. If a GC interrupts the barrier in the middle, value will be kept alive by
6426 * the conservative scan, otherwise by the remembered set scan. FIXME: figure out what
6427 * happens when we need to record which pointers contain references to the new generation.
6428 * The write barrier will be executed, but the pointer is still not stored.
6431 mono_gc_wbarrier_set_field (MonoObject *obj, gpointer field_ptr, MonoObject* value)
6435 HEAVY_STAT (++stat_wbarrier_set_field);
6436 if (ptr_in_nursery (field_ptr)) {
6437 *(void**)field_ptr = value;
6440 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", field_ptr));
6442 rs = REMEMBERED_SET;
6443 if (rs->store_next < rs->end_set) {
6444 *(rs->store_next++) = (mword)field_ptr;
6445 *(void**)field_ptr = value;
6449 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6450 rs->next = REMEMBERED_SET;
6451 REMEMBERED_SET = rs;
6452 #ifdef HAVE_KW_THREAD
6453 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6455 *(rs->store_next++) = (mword)field_ptr;
6456 *(void**)field_ptr = value;
6461 mono_gc_wbarrier_set_arrayref (MonoArray *arr, gpointer slot_ptr, MonoObject* value)
6465 HEAVY_STAT (++stat_wbarrier_set_arrayref);
6466 if (ptr_in_nursery (slot_ptr)) {
6467 *(void**)slot_ptr = value;
6470 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", slot_ptr));
6472 rs = REMEMBERED_SET;
6473 if (rs->store_next < rs->end_set) {
6474 *(rs->store_next++) = (mword)slot_ptr;
6475 *(void**)slot_ptr = value;
6479 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6480 rs->next = REMEMBERED_SET;
6481 REMEMBERED_SET = rs;
6482 #ifdef HAVE_KW_THREAD
6483 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6485 *(rs->store_next++) = (mword)slot_ptr;
6486 *(void**)slot_ptr = value;
6491 mono_gc_wbarrier_arrayref_copy (gpointer dest_ptr, gpointer src_ptr, int count)
6495 HEAVY_STAT (++stat_wbarrier_arrayref_copy);
6497 memmove (dest_ptr, src_ptr, count * sizeof (gpointer));
6498 if (ptr_in_nursery (dest_ptr)) {
6502 rs = REMEMBERED_SET;
6503 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p, %d\n", dest_ptr, count));
6504 if (rs->store_next + 1 < rs->end_set) {
6505 *(rs->store_next++) = (mword)dest_ptr | REMSET_RANGE;
6506 *(rs->store_next++) = count;
6510 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6511 rs->next = REMEMBERED_SET;
6512 REMEMBERED_SET = rs;
6513 #ifdef HAVE_KW_THREAD
6514 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6516 *(rs->store_next++) = (mword)dest_ptr | REMSET_RANGE;
6517 *(rs->store_next++) = count;
6522 find_object_for_ptr_in_area (char *ptr, char *start, char *end)
6524 while (start < end) {
6527 if (!*(void**)start) {
6528 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
6534 #define SCAN_OBJECT_NOSCAN
6535 #include "sgen-scan-object.h"
6537 if (ptr >= old_start && ptr < start)
6544 static char *found_obj;
6547 find_object_for_ptr_in_pinned_chunk_callback (PinnedChunk *chunk, char *obj, size_t size, char *ptr)
6549 if (ptr >= obj && ptr < obj + size) {
6550 g_assert (!found_obj);
6555 /* for use in the debugger */
6556 char* find_object_for_ptr (char *ptr);
6558 find_object_for_ptr (char *ptr)
6560 GCMemSection *section;
6563 for (section = section_list; section; section = section->block.next) {
6564 if (ptr >= section->data && ptr < section->end_data)
6565 return find_object_for_ptr_in_area (ptr, section->data, section->end_data);
6568 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
6569 if (ptr >= bigobj->data && ptr < bigobj->data + bigobj->size)
6570 return bigobj->data;
6574 scan_pinned_objects ((ScanPinnedObjectCallbackFunc)find_object_for_ptr_in_pinned_chunk_callback, ptr);
6579 evacuate_remset_buffer (void)
6584 buffer = STORE_REMSET_BUFFER;
6586 add_generic_store_remset_from_buffer (buffer);
6587 memset (buffer, 0, sizeof (gpointer) * STORE_REMSET_BUFFER_SIZE);
6589 STORE_REMSET_BUFFER_INDEX = 0;
6593 mono_gc_wbarrier_generic_nostore (gpointer ptr)
6599 HEAVY_STAT (++stat_wbarrier_generic_store);
6601 #ifdef XDOMAIN_CHECKS_IN_WBARRIER
6602 /* FIXME: ptr_in_heap must be called with the GC lock held */
6603 if (xdomain_checks && *(MonoObject**)ptr && ptr_in_heap (ptr)) {
6604 char *start = find_object_for_ptr (ptr);
6605 MonoObject *value = *(MonoObject**)ptr;
6609 MonoObject *obj = (MonoObject*)start;
6610 if (obj->vtable->domain != value->vtable->domain)
6611 g_assert (is_xdomain_ref_allowed (ptr, start, obj->vtable->domain));
6619 if (*(gpointer*)ptr)
6620 binary_protocol_wbarrier (ptr, *(gpointer*)ptr, (gpointer)LOAD_VTABLE (*(gpointer*)ptr));
6622 if (ptr_in_nursery (ptr) || ptr_on_stack (ptr) || !ptr_in_nursery (*(gpointer*)ptr)) {
6623 DEBUG (8, fprintf (gc_debug_file, "Skipping remset at %p\n", ptr));
6628 buffer = STORE_REMSET_BUFFER;
6629 index = STORE_REMSET_BUFFER_INDEX;
6630 /* This simple optimization eliminates a sizable portion of
6631 entries. Comparing it to the last but one entry as well
6632 doesn't eliminate significantly more entries. */
6633 if (buffer [index] == ptr) {
6638 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", ptr));
6639 HEAVY_STAT (++stat_wbarrier_generic_store_remset);
6642 if (index >= STORE_REMSET_BUFFER_SIZE) {
6643 evacuate_remset_buffer ();
6644 index = STORE_REMSET_BUFFER_INDEX;
6645 g_assert (index == 0);
6648 buffer [index] = ptr;
6649 STORE_REMSET_BUFFER_INDEX = index;
6655 mono_gc_wbarrier_generic_store (gpointer ptr, MonoObject* value)
6657 DEBUG (8, fprintf (gc_debug_file, "Wbarrier store at %p to %p (%s)\n", ptr, value, value ? safe_name (value) : "null"));
6658 *(void**)ptr = value;
6659 if (ptr_in_nursery (value))
6660 mono_gc_wbarrier_generic_nostore (ptr);
6664 mono_gc_wbarrier_set_root (gpointer ptr, MonoObject *value)
6668 HEAVY_STAT (++stat_wbarrier_set_root);
6669 if (ptr_in_nursery (ptr))
6671 DEBUG (8, fprintf (gc_debug_file, "Adding root remset at %p (%s)\n", ptr, value ? safe_name (value) : "null"));
6673 rs = REMEMBERED_SET;
6674 if (rs->store_next + 2 < rs->end_set) {
6675 *(rs->store_next++) = (mword)ptr | REMSET_OTHER;
6676 *(rs->store_next++) = (mword)REMSET_ROOT_LOCATION;
6677 *(void**)ptr = value;
6680 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6681 rs->next = REMEMBERED_SET;
6682 REMEMBERED_SET = rs;
6683 #ifdef HAVE_KW_THREAD
6684 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6686 *(rs->store_next++) = (mword)ptr | REMSET_OTHER;
6687 *(rs->store_next++) = (mword)REMSET_ROOT_LOCATION;
6689 *(void**)ptr = value;
6693 mono_gc_wbarrier_value_copy (gpointer dest, gpointer src, int count, MonoClass *klass)
6697 HEAVY_STAT (++stat_wbarrier_value_copy);
6698 g_assert (klass->valuetype);
6700 memmove (dest, src, count * mono_class_value_size (klass, NULL));
6701 rs = REMEMBERED_SET;
6702 if (ptr_in_nursery (dest) || ptr_on_stack (dest)) {
6706 g_assert (klass->gc_descr_inited);
6707 DEBUG (8, fprintf (gc_debug_file, "Adding value remset at %p, count %d, descr %p for class %s (%p)\n", dest, count, klass->gc_descr, klass->name, klass));
6709 if (rs->store_next + 3 < rs->end_set) {
6710 *(rs->store_next++) = (mword)dest | REMSET_OTHER;
6711 *(rs->store_next++) = (mword)REMSET_VTYPE;
6712 *(rs->store_next++) = (mword)klass->gc_descr;
6713 *(rs->store_next++) = (mword)count;
6717 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6718 rs->next = REMEMBERED_SET;
6719 REMEMBERED_SET = rs;
6720 #ifdef HAVE_KW_THREAD
6721 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6723 *(rs->store_next++) = (mword)dest | REMSET_OTHER;
6724 *(rs->store_next++) = (mword)REMSET_VTYPE;
6725 *(rs->store_next++) = (mword)klass->gc_descr;
6726 *(rs->store_next++) = (mword)count;
6731 * mono_gc_wbarrier_object_copy:
6733 * Write barrier to call when obj is the result of a clone or copy of an object.
6736 mono_gc_wbarrier_object_copy (MonoObject* obj, MonoObject *src)
6742 HEAVY_STAT (++stat_wbarrier_object_copy);
6743 rs = REMEMBERED_SET;
6744 DEBUG (1, fprintf (gc_debug_file, "Adding object remset for %p\n", obj));
6745 size = mono_object_class (obj)->instance_size;
6747 /* do not copy the sync state */
6748 memcpy ((char*)obj + sizeof (MonoObject), (char*)src + sizeof (MonoObject),
6749 size - sizeof (MonoObject));
6750 if (ptr_in_nursery (obj) || ptr_on_stack (obj)) {
6754 if (rs->store_next < rs->end_set) {
6755 *(rs->store_next++) = (mword)obj | REMSET_OBJECT;
6759 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6760 rs->next = REMEMBERED_SET;
6761 REMEMBERED_SET = rs;
6762 #ifdef HAVE_KW_THREAD
6763 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6765 *(rs->store_next++) = (mword)obj | REMSET_OBJECT;
6770 * ######################################################################
6771 * ######## Collector debugging
6772 * ######################################################################
6775 const char*descriptor_types [] = {
6787 describe_ptr (char *ptr)
6789 GCMemSection *section;
6794 if (ptr_in_nursery (ptr)) {
6795 printf ("Pointer inside nursery.\n");
6797 for (section = section_list; section;) {
6798 if (ptr >= section->data && ptr < section->data + section->size)
6800 section = section->block.next;
6804 printf ("Pointer inside oldspace.\n");
6805 } else if (obj_is_from_pinned_alloc (ptr)) {
6806 printf ("Pointer is inside a pinned chunk.\n");
6808 printf ("Pointer unknown.\n");
6813 if (object_is_pinned (ptr))
6814 printf ("Object is pinned.\n");
6816 if (object_is_forwarded (ptr))
6817 printf ("Object is forwared.\n");
6819 // FIXME: Handle pointers to the inside of objects
6820 vtable = (MonoVTable*)LOAD_VTABLE (ptr);
6822 printf ("VTable: %p\n", vtable);
6823 if (vtable == NULL) {
6824 printf ("VTable is invalid (empty).\n");
6827 if (ptr_in_nursery (vtable)) {
6828 printf ("VTable is invalid (points inside nursery).\n");
6831 printf ("Class: %s\n", vtable->klass->name);
6833 desc = ((GCVTable*)vtable)->desc;
6834 printf ("Descriptor: %lx\n", (long)desc);
6837 printf ("Descriptor type: %d (%s)\n", type, descriptor_types [type]);
6841 find_in_remset_loc (mword *p, char *addr, gboolean *found)
6847 switch ((*p) & REMSET_TYPE_MASK) {
6848 case REMSET_LOCATION:
6849 if (*p == (mword)addr)
6853 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
6855 if ((void**)addr >= ptr && (void**)addr < ptr + count)
6859 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
6860 count = safe_object_get_size ((MonoObject*)ptr);
6861 count += (ALLOC_ALIGN - 1);
6862 count &= (ALLOC_ALIGN - 1);
6863 count /= sizeof (mword);
6864 if ((void**)addr >= ptr && (void**)addr < ptr + count)
6867 case REMSET_OTHER: {
6870 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
6874 switch (desc & 0x7) {
6875 case DESC_TYPE_RUN_LENGTH:
6876 OBJ_RUN_LEN_SIZE (skip_size, desc, ptr);
6878 case DESC_TYPE_SMALL_BITMAP:
6879 OBJ_BITMAP_SIZE (skip_size, desc, start);
6883 g_assert_not_reached ();
6886 /* The descriptor includes the size of MonoObject */
6887 skip_size -= sizeof (MonoObject);
6889 if ((void**)addr >= ptr && (void**)addr < ptr + (skip_size / sizeof (gpointer)))
6893 case REMSET_ROOT_LOCATION:
6896 g_assert_not_reached ();
6901 g_assert_not_reached ();
6907 * Return whenever ADDR occurs in the remembered sets
6910 find_in_remsets (char *addr)
6913 SgenThreadInfo *info;
6914 RememberedSet *remset;
6915 GenericStoreRememberedSet *store_remset;
6917 gboolean found = FALSE;
6919 /* the global one */
6920 for (remset = global_remset; remset; remset = remset->next) {
6921 DEBUG (4, fprintf (gc_debug_file, "Scanning global remset range: %p-%p, size: %zd\n", remset->data, remset->store_next, remset->store_next - remset->data));
6922 for (p = remset->data; p < remset->store_next;) {
6923 p = find_in_remset_loc (p, addr, &found);
6929 /* the generic store ones */
6930 for (store_remset = generic_store_remsets; store_remset; store_remset = store_remset->next) {
6931 for (i = 0; i < STORE_REMSET_BUFFER_SIZE - 1; ++i) {
6932 if (store_remset->data [i] == addr)
6937 /* the per-thread ones */
6938 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
6939 for (info = thread_table [i]; info; info = info->next) {
6941 for (remset = info->remset; remset; remset = remset->next) {
6942 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for thread %p, range: %p-%p, size: %zd\n", info, remset->data, remset->store_next, remset->store_next - remset->data));
6943 for (p = remset->data; p < remset->store_next;) {
6944 p = find_in_remset_loc (p, addr, &found);
6949 for (j = 0; j < *info->store_remset_buffer_index_addr; ++j) {
6950 if ((*info->store_remset_buffer_addr) [j + 1] == addr)
6956 /* the freed thread ones */
6957 for (remset = freed_thread_remsets; remset; remset = remset->next) {
6958 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for freed thread, range: %p-%p, size: %zd\n", remset->data, remset->store_next, remset->store_next - remset->data));
6959 for (p = remset->data; p < remset->store_next;) {
6960 p = find_in_remset_loc (p, addr, &found);
6969 static gboolean missing_remsets;
6972 * We let a missing remset slide if the target object is pinned,
6973 * because the store might have happened but the remset not yet added,
6974 * but in that case the target must be pinned. We might theoretically
6975 * miss some missing remsets this way, but it's very unlikely.
6978 #define HANDLE_PTR(ptr,obj) do { \
6979 if (*(ptr) && (char*)*(ptr) >= nursery_start && (char*)*(ptr) < nursery_next) { \
6980 if (!find_in_remsets ((char*)(ptr))) { \
6981 fprintf (gc_debug_file, "Oldspace->newspace reference %p at offset %zd in object %p (%s.%s) not found in remsets.\n", *(ptr), (char*)(ptr) - (char*)(obj), (obj), ((MonoObject*)(obj))->vtable->klass->name_space, ((MonoObject*)(obj))->vtable->klass->name); \
6982 binary_protocol_missing_remset ((obj), (gpointer)LOAD_VTABLE ((obj)), (char*)(ptr) - (char*)(obj), *(ptr), (gpointer)LOAD_VTABLE(*(ptr)), object_is_pinned (*(ptr))); \
6983 if (!object_is_pinned (*(ptr))) \
6984 missing_remsets = TRUE; \
6990 * Check that each object reference inside the area which points into the nursery
6991 * can be found in the remembered sets.
6993 static void __attribute__((noinline))
6994 check_remsets_for_area (char *start, char *end)
6997 int type_str = 0, type_rlen = 0, type_bitmap = 0, type_vector = 0, type_lbit = 0, type_complex = 0;
6998 while (start < end) {
6999 if (!*(void**)start) {
7000 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
7003 vt = (GCVTable*)LOAD_VTABLE (start);
7004 DEBUG (8, fprintf (gc_debug_file, "Scanning object %p, vtable: %p (%s)\n", start, vt, vt->klass->name));
7006 MonoObject *obj = (MonoObject*)start;
7007 g_print ("found at %p (0x%lx): %s.%s\n", start, (long)vt->desc, obj->vtable->klass->name_space, obj->vtable->klass->name);
7010 #define SCAN_OBJECT_ACTION COUNT_OBJECT_TYPES
7011 #include "sgen-scan-object.h"
7016 * Perform consistency check of the heap.
7018 * Assumes the world is stopped.
7021 check_consistency (void)
7023 GCMemSection *section;
7025 // Need to add more checks
7026 // FIXME: Create a general heap enumeration function and use that
7028 missing_remsets = FALSE;
7030 DEBUG (1, fprintf (gc_debug_file, "Begin heap consistency check...\n"));
7032 // Check that oldspace->newspace pointers are registered with the collector
7033 for (section = section_list; section; section = section->block.next) {
7034 if (section->block.role == MEMORY_ROLE_GEN0)
7036 DEBUG (2, fprintf (gc_debug_file, "Scan of old section: %p-%p, size: %d\n", section->data, section->next_data, (int)(section->next_data - section->data)));
7037 check_remsets_for_area (section->data, section->next_data);
7040 DEBUG (1, fprintf (gc_debug_file, "Heap consistency check done.\n"));
7042 #ifdef BINARY_PROTOCOL
7043 if (!binary_protocol_file)
7045 g_assert (!missing_remsets);
7048 /* Check that the reference is valid */
7050 #define HANDLE_PTR(ptr,obj) do { \
7052 g_assert (safe_name (*(ptr)) != NULL); \
7059 * Perform consistency check on an object. Currently we only check that the
7060 * reference fields are valid.
7063 check_object (char *start)
7068 #include "sgen-scan-object.h"
7074 * ######################################################################
7075 * ######## Other mono public interface functions.
7076 * ######################################################################
7080 mono_gc_collect (int generation)
7084 if (generation == 0) {
7085 collect_nursery (0);
7087 major_collection ("user request");
7094 mono_gc_max_generation (void)
7100 mono_gc_collection_count (int generation)
7102 if (generation == 0)
7103 return num_minor_gcs;
7104 return num_major_gcs;
7108 mono_gc_get_used_size (void)
7111 GCMemSection *section;
7113 tot = los_memory_usage;
7114 for (section = section_list; section; section = section->block.next) {
7115 /* this is approximate... */
7116 tot += section->next_data - section->data;
7118 /* FIXME: account for pinned objects */
7124 mono_gc_get_heap_size (void)
7130 mono_gc_disable (void)
7138 mono_gc_enable (void)
7146 mono_gc_get_los_limit (void)
7148 return MAX_SMALL_OBJ_SIZE;
7152 mono_object_is_alive (MonoObject* o)
7158 mono_gc_get_generation (MonoObject *obj)
7160 if (ptr_in_nursery (obj))
7166 mono_gc_enable_events (void)
7171 mono_gc_weak_link_add (void **link_addr, MonoObject *obj, gboolean track)
7174 mono_gc_register_disappearing_link (obj, link_addr, track);
7179 mono_gc_weak_link_remove (void **link_addr)
7182 mono_gc_register_disappearing_link (NULL, link_addr, FALSE);
7187 mono_gc_weak_link_get (void **link_addr)
7191 return (MonoObject*) REVEAL_POINTER (*link_addr);
7195 mono_gc_make_descr_from_bitmap (gsize *bitmap, int numbits)
7197 if (numbits < ((sizeof (*bitmap) * 8) - ROOT_DESC_TYPE_SHIFT)) {
7198 return (void*)MAKE_ROOT_DESC (ROOT_DESC_BITMAP, bitmap [0]);
7200 mword complex = alloc_complex_descriptor (bitmap, numbits);
7201 return (void*)MAKE_ROOT_DESC (ROOT_DESC_COMPLEX, complex);
7206 mono_gc_make_root_descr_user (MonoGCMarkFunc marker)
7210 g_assert (user_descriptors_next < MAX_USER_DESCRIPTORS);
7211 descr = (void*)MAKE_ROOT_DESC (ROOT_DESC_USER, (mword)user_descriptors_next);
7212 user_descriptors [user_descriptors_next ++] = marker;
7218 mono_gc_alloc_fixed (size_t size, void *descr)
7220 /* FIXME: do a single allocation */
7221 void *res = calloc (1, size);
7224 if (!mono_gc_register_root (res, size, descr)) {
7232 mono_gc_free_fixed (void* addr)
7234 mono_gc_deregister_root (addr);
7239 mono_gc_is_gc_thread (void)
7243 result = thread_info_lookup (ARCH_GET_THREAD ()) != NULL;
7249 mono_gc_base_init (void)
7253 struct sigaction sinfo;
7255 LOCK_INIT (gc_mutex);
7257 if (gc_initialized) {
7261 pagesize = mono_pagesize ();
7262 gc_debug_file = stderr;
7263 if ((env = getenv ("MONO_GC_DEBUG"))) {
7264 opts = g_strsplit (env, ",", -1);
7265 for (ptr = opts; ptr && *ptr; ptr ++) {
7267 if (opt [0] >= '0' && opt [0] <= '9') {
7268 gc_debug_level = atoi (opt);
7273 char *rf = g_strdup_printf ("%s.%d", opt, getpid ());
7274 gc_debug_file = fopen (rf, "wb");
7276 gc_debug_file = stderr;
7279 } else if (!strcmp (opt, "collect-before-allocs")) {
7280 collect_before_allocs = TRUE;
7281 } else if (!strcmp (opt, "check-at-minor-collections")) {
7282 consistency_check_at_minor_collection = TRUE;
7283 } else if (!strcmp (opt, "xdomain-checks")) {
7284 xdomain_checks = TRUE;
7285 } else if (!strcmp (opt, "clear-at-gc")) {
7286 nursery_clear_policy = CLEAR_AT_GC;
7287 } else if (!strcmp (opt, "conservative-stack-mark")) {
7288 conservative_stack_mark = TRUE;
7289 } else if (!strcmp (opt, "check-scan-starts")) {
7290 do_scan_starts_check = TRUE;
7291 } else if (g_str_has_prefix (opt, "heap-dump=")) {
7292 char *filename = strchr (opt, '=') + 1;
7293 nursery_clear_policy = CLEAR_AT_GC;
7294 heap_dump_file = fopen (filename, "w");
7296 fprintf (heap_dump_file, "<sgen-dump>\n");
7297 #ifdef BINARY_PROTOCOL
7298 } else if (g_str_has_prefix (opt, "binary-protocol=")) {
7299 char *filename = strchr (opt, '=') + 1;
7300 binary_protocol_file = fopen (filename, "w");
7303 fprintf (stderr, "Invalid format for the MONO_GC_DEBUG env variable: '%s'\n", env);
7304 fprintf (stderr, "The format is: MONO_GC_DEBUG=[l[:filename]|<option>]+ where l is a debug level 0-9.\n");
7305 fprintf (stderr, "Valid options are: collect-before-allocs, check-at-minor-collections, xdomain-checks, clear-at-gc.\n");
7312 suspend_ack_semaphore_ptr = &suspend_ack_semaphore;
7313 MONO_SEM_INIT (&suspend_ack_semaphore, 0);
7315 sigfillset (&sinfo.sa_mask);
7316 sinfo.sa_flags = SA_RESTART | SA_SIGINFO;
7317 sinfo.sa_sigaction = suspend_handler;
7318 if (sigaction (suspend_signal_num, &sinfo, NULL) != 0) {
7319 g_error ("failed sigaction");
7322 sinfo.sa_handler = restart_handler;
7323 if (sigaction (restart_signal_num, &sinfo, NULL) != 0) {
7324 g_error ("failed sigaction");
7327 sigfillset (&suspend_signal_mask);
7328 sigdelset (&suspend_signal_mask, restart_signal_num);
7330 global_remset = alloc_remset (1024, NULL);
7331 global_remset->next = NULL;
7333 pthread_key_create (&remembered_set_key, unregister_thread);
7335 #ifndef HAVE_KW_THREAD
7336 pthread_key_create (&thread_info_key, NULL);
7339 gc_initialized = TRUE;
7341 mono_gc_register_thread (&sinfo);
7345 mono_gc_get_suspend_signal (void)
7347 return suspend_signal_num;
7357 #ifdef HAVE_KW_THREAD
7358 #define EMIT_TLS_ACCESS(mb,dummy,offset) do { \
7359 mono_mb_emit_byte ((mb), MONO_CUSTOM_PREFIX); \
7360 mono_mb_emit_byte ((mb), CEE_MONO_TLS); \
7361 mono_mb_emit_i4 ((mb), (offset)); \
7364 #define EMIT_TLS_ACCESS(mb,member,dummy) do { \
7365 mono_mb_emit_byte ((mb), MONO_CUSTOM_PREFIX); \
7366 mono_mb_emit_byte ((mb), CEE_MONO_TLS); \
7367 mono_mb_emit_i4 ((mb), thread_info_key); \
7368 mono_mb_emit_icon ((mb), G_STRUCT_OFFSET (SgenThreadInfo, member)); \
7369 mono_mb_emit_byte ((mb), CEE_ADD); \
7370 mono_mb_emit_byte ((mb), CEE_LDIND_I); \
7374 #ifdef MANAGED_ALLOCATION
7375 /* FIXME: Do this in the JIT, where specialized allocation sequences can be created
7376 * for each class. This is currently not easy to do, as it is hard to generate basic
7377 * blocks + branches, but it is easy with the linear IL codebase.
7379 * For this to work we'd need to solve the TLAB race, first. Now we
7380 * require the allocator to be in a few known methods to make sure
7381 * that they are executed atomically via the restart mechanism.
7384 create_allocator (int atype)
7386 int p_var, size_var;
7387 guint32 slowpath_branch, max_size_branch;
7388 MonoMethodBuilder *mb;
7390 MonoMethodSignature *csig;
7391 static gboolean registered = FALSE;
7392 int tlab_next_addr_var, new_next_var;
7394 const char *name = NULL;
7395 AllocatorWrapperInfo *info;
7397 #ifdef HAVE_KW_THREAD
7398 int tlab_next_addr_offset = -1;
7399 int tlab_temp_end_offset = -1;
7401 MONO_THREAD_VAR_OFFSET (tlab_next_addr, tlab_next_addr_offset);
7402 MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
7404 g_assert (tlab_next_addr_offset != -1);
7405 g_assert (tlab_temp_end_offset != -1);
7409 mono_register_jit_icall (mono_gc_alloc_obj, "mono_gc_alloc_obj", mono_create_icall_signature ("object ptr int"), FALSE);
7410 mono_register_jit_icall (mono_gc_alloc_vector, "mono_gc_alloc_vector", mono_create_icall_signature ("object ptr int int"), FALSE);
7414 if (atype == ATYPE_SMALL) {
7416 name = "AllocSmall";
7417 } else if (atype == ATYPE_NORMAL) {
7420 } else if (atype == ATYPE_VECTOR) {
7422 name = "AllocVector";
7424 g_assert_not_reached ();
7427 csig = mono_metadata_signature_alloc (mono_defaults.corlib, num_params);
7428 csig->ret = &mono_defaults.object_class->byval_arg;
7429 for (i = 0; i < num_params; ++i)
7430 csig->params [i] = &mono_defaults.int_class->byval_arg;
7432 mb = mono_mb_new (mono_defaults.object_class, name, MONO_WRAPPER_ALLOC);
7433 size_var = mono_mb_add_local (mb, &mono_defaults.int32_class->byval_arg);
7434 if (atype == ATYPE_NORMAL || atype == ATYPE_SMALL) {
7435 /* size = vtable->klass->instance_size; */
7436 mono_mb_emit_ldarg (mb, 0);
7437 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoVTable, klass));
7438 mono_mb_emit_byte (mb, CEE_ADD);
7439 mono_mb_emit_byte (mb, CEE_LDIND_I);
7440 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoClass, instance_size));
7441 mono_mb_emit_byte (mb, CEE_ADD);
7442 /* FIXME: assert instance_size stays a 4 byte integer */
7443 mono_mb_emit_byte (mb, CEE_LDIND_U4);
7444 mono_mb_emit_stloc (mb, size_var);
7445 } else if (atype == ATYPE_VECTOR) {
7446 MonoExceptionClause *clause;
7448 MonoClass *oom_exc_class;
7451 /* n > MONO_ARRAY_MAX_INDEX -> OverflowException */
7452 mono_mb_emit_ldarg (mb, 1);
7453 mono_mb_emit_icon (mb, MONO_ARRAY_MAX_INDEX);
7454 pos = mono_mb_emit_short_branch (mb, CEE_BLE_UN_S);
7455 mono_mb_emit_exception (mb, "OverflowException", NULL);
7456 mono_mb_patch_short_branch (mb, pos);
7458 clause = mono_image_alloc0 (mono_defaults.corlib, sizeof (MonoExceptionClause));
7459 clause->try_offset = mono_mb_get_label (mb);
7461 /* vtable->klass->sizes.element_size */
7462 mono_mb_emit_ldarg (mb, 0);
7463 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoVTable, klass));
7464 mono_mb_emit_byte (mb, CEE_ADD);
7465 mono_mb_emit_byte (mb, CEE_LDIND_I);
7466 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoClass, sizes.element_size));
7467 mono_mb_emit_byte (mb, CEE_ADD);
7468 mono_mb_emit_byte (mb, CEE_LDIND_U4);
7471 mono_mb_emit_ldarg (mb, 1);
7472 mono_mb_emit_byte (mb, CEE_MUL_OVF_UN);
7473 /* + sizeof (MonoArray) */
7474 mono_mb_emit_icon (mb, sizeof (MonoArray));
7475 mono_mb_emit_byte (mb, CEE_ADD_OVF_UN);
7476 mono_mb_emit_stloc (mb, size_var);
7478 pos_leave = mono_mb_emit_branch (mb, CEE_LEAVE);
7481 clause->flags = MONO_EXCEPTION_CLAUSE_NONE;
7482 clause->try_len = mono_mb_get_pos (mb) - clause->try_offset;
7483 clause->data.catch_class = mono_class_from_name (mono_defaults.corlib,
7484 "System", "OverflowException");
7485 g_assert (clause->data.catch_class);
7486 clause->handler_offset = mono_mb_get_label (mb);
7488 oom_exc_class = mono_class_from_name (mono_defaults.corlib,
7489 "System", "OutOfMemoryException");
7490 g_assert (oom_exc_class);
7491 ctor = mono_class_get_method_from_name (oom_exc_class, ".ctor", 0);
7494 mono_mb_emit_byte (mb, CEE_POP);
7495 mono_mb_emit_op (mb, CEE_NEWOBJ, ctor);
7496 mono_mb_emit_byte (mb, CEE_THROW);
7498 clause->handler_len = mono_mb_get_pos (mb) - clause->handler_offset;
7499 mono_mb_set_clauses (mb, 1, clause);
7500 mono_mb_patch_branch (mb, pos_leave);
7503 g_assert_not_reached ();
7506 /* size += ALLOC_ALIGN - 1; */
7507 mono_mb_emit_ldloc (mb, size_var);
7508 mono_mb_emit_icon (mb, ALLOC_ALIGN - 1);
7509 mono_mb_emit_byte (mb, CEE_ADD);
7510 /* size &= ~(ALLOC_ALIGN - 1); */
7511 mono_mb_emit_icon (mb, ~(ALLOC_ALIGN - 1));
7512 mono_mb_emit_byte (mb, CEE_AND);
7513 mono_mb_emit_stloc (mb, size_var);
7515 /* if (size > MAX_SMALL_OBJ_SIZE) goto slowpath */
7516 if (atype != ATYPE_SMALL) {
7517 mono_mb_emit_ldloc (mb, size_var);
7518 mono_mb_emit_icon (mb, MAX_SMALL_OBJ_SIZE);
7519 max_size_branch = mono_mb_emit_short_branch (mb, MONO_CEE_BGT_S);
7523 * We need to modify tlab_next, but the JIT only supports reading, so we read
7524 * another tls var holding its address instead.
7527 /* tlab_next_addr (local) = tlab_next_addr (TLS var) */
7528 tlab_next_addr_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7529 EMIT_TLS_ACCESS (mb, tlab_next_addr, tlab_next_addr_offset);
7530 mono_mb_emit_stloc (mb, tlab_next_addr_var);
7532 /* p = (void**)tlab_next; */
7533 p_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7534 mono_mb_emit_ldloc (mb, tlab_next_addr_var);
7535 mono_mb_emit_byte (mb, CEE_LDIND_I);
7536 mono_mb_emit_stloc (mb, p_var);
7538 /* new_next = (char*)p + size; */
7539 new_next_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7540 mono_mb_emit_ldloc (mb, p_var);
7541 mono_mb_emit_ldloc (mb, size_var);
7542 mono_mb_emit_byte (mb, CEE_CONV_I);
7543 mono_mb_emit_byte (mb, CEE_ADD);
7544 mono_mb_emit_stloc (mb, new_next_var);
7546 /* tlab_next = new_next */
7547 mono_mb_emit_ldloc (mb, tlab_next_addr_var);
7548 mono_mb_emit_ldloc (mb, new_next_var);
7549 mono_mb_emit_byte (mb, CEE_STIND_I);
7551 /* if (G_LIKELY (new_next < tlab_temp_end)) */
7552 mono_mb_emit_ldloc (mb, new_next_var);
7553 EMIT_TLS_ACCESS (mb, tlab_temp_end, tlab_temp_end_offset);
7554 slowpath_branch = mono_mb_emit_short_branch (mb, MONO_CEE_BLT_UN_S);
7557 if (atype != ATYPE_SMALL)
7558 mono_mb_patch_short_branch (mb, max_size_branch);
7560 mono_mb_emit_byte (mb, MONO_CUSTOM_PREFIX);
7561 mono_mb_emit_byte (mb, CEE_MONO_NOT_TAKEN);
7563 /* FIXME: mono_gc_alloc_obj takes a 'size_t' as an argument, not an int32 */
7564 mono_mb_emit_ldarg (mb, 0);
7565 mono_mb_emit_ldloc (mb, size_var);
7566 if (atype == ATYPE_NORMAL || atype == ATYPE_SMALL) {
7567 mono_mb_emit_icall (mb, mono_gc_alloc_obj);
7568 } else if (atype == ATYPE_VECTOR) {
7569 mono_mb_emit_ldarg (mb, 1);
7570 mono_mb_emit_icall (mb, mono_gc_alloc_vector);
7572 g_assert_not_reached ();
7574 mono_mb_emit_byte (mb, CEE_RET);
7577 mono_mb_patch_short_branch (mb, slowpath_branch);
7579 /* FIXME: Memory barrier */
7582 mono_mb_emit_ldloc (mb, p_var);
7583 mono_mb_emit_ldarg (mb, 0);
7584 mono_mb_emit_byte (mb, CEE_STIND_I);
7586 if (atype == ATYPE_VECTOR) {
7587 /* arr->max_length = max_length; */
7588 mono_mb_emit_ldloc (mb, p_var);
7589 mono_mb_emit_ldflda (mb, G_STRUCT_OFFSET (MonoArray, max_length));
7590 mono_mb_emit_ldarg (mb, 1);
7591 mono_mb_emit_byte (mb, CEE_STIND_I);
7595 mono_mb_emit_ldloc (mb, p_var);
7596 mono_mb_emit_byte (mb, CEE_RET);
7598 res = mono_mb_create_method (mb, csig, 8);
7600 mono_method_get_header (res)->init_locals = FALSE;
7602 info = mono_image_alloc0 (mono_defaults.corlib, sizeof (AllocatorWrapperInfo));
7603 info->alloc_type = atype;
7604 mono_marshal_set_wrapper_info (res, info);
7610 static MonoMethod* alloc_method_cache [ATYPE_NUM];
7611 static MonoMethod *write_barrier_method;
7614 is_ip_in_managed_allocator (MonoDomain *domain, gpointer ip)
7622 ji = mono_jit_info_table_find (domain, ip);
7625 method = ji->method;
7627 if (method == write_barrier_method)
7629 for (i = 0; i < ATYPE_NUM; ++i)
7630 if (method == alloc_method_cache [i])
7636 * Generate an allocator method implementing the fast path of mono_gc_alloc_obj ().
7637 * The signature of the called method is:
7638 * object allocate (MonoVTable *vtable)
7641 mono_gc_get_managed_allocator (MonoVTable *vtable, gboolean for_box)
7643 #ifdef MANAGED_ALLOCATION
7644 MonoClass *klass = vtable->klass;
7646 #ifdef HAVE_KW_THREAD
7647 int tlab_next_offset = -1;
7648 int tlab_temp_end_offset = -1;
7649 MONO_THREAD_VAR_OFFSET (tlab_next, tlab_next_offset);
7650 MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
7652 if (tlab_next_offset == -1 || tlab_temp_end_offset == -1)
7656 if (!mono_runtime_has_tls_get ())
7658 if (klass->instance_size > tlab_size)
7660 if (klass->has_finalize || klass->marshalbyref || (mono_profiler_get_events () & MONO_PROFILE_ALLOCATIONS))
7664 if (klass->byval_arg.type == MONO_TYPE_STRING)
7666 if (collect_before_allocs)
7669 if (ALIGN_TO (klass->instance_size, ALLOC_ALIGN) < MAX_SMALL_OBJ_SIZE)
7670 return mono_gc_get_managed_allocator_by_type (ATYPE_SMALL);
7672 return mono_gc_get_managed_allocator_by_type (ATYPE_NORMAL);
7679 mono_gc_get_managed_array_allocator (MonoVTable *vtable, int rank)
7681 #ifdef MANAGED_ALLOCATION
7682 MonoClass *klass = vtable->klass;
7684 #ifdef HAVE_KW_THREAD
7685 int tlab_next_offset = -1;
7686 int tlab_temp_end_offset = -1;
7687 MONO_THREAD_VAR_OFFSET (tlab_next, tlab_next_offset);
7688 MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
7690 if (tlab_next_offset == -1 || tlab_temp_end_offset == -1)
7696 if (!mono_runtime_has_tls_get ())
7698 if (mono_profiler_get_events () & MONO_PROFILE_ALLOCATIONS)
7700 if (collect_before_allocs)
7702 g_assert (!klass->has_finalize && !klass->marshalbyref);
7704 return mono_gc_get_managed_allocator_by_type (ATYPE_VECTOR);
7711 mono_gc_get_managed_allocator_by_type (int atype)
7713 #ifdef MANAGED_ALLOCATION
7716 if (!mono_runtime_has_tls_get ())
7719 mono_loader_lock ();
7720 res = alloc_method_cache [atype];
7722 res = alloc_method_cache [atype] = create_allocator (atype);
7723 mono_loader_unlock ();
7731 mono_gc_get_managed_allocator_types (void)
7738 mono_gc_get_write_barrier (void)
7741 MonoMethodBuilder *mb;
7742 MonoMethodSignature *sig;
7743 #ifdef MANAGED_WBARRIER
7744 int label_no_wb_1, label_no_wb_2, label_no_wb_3, label_no_wb_4, label_need_wb, label_slow_path;
7745 int buffer_var, buffer_index_var, dummy_var;
7747 #ifdef HAVE_KW_THREAD
7748 int stack_end_offset = -1, store_remset_buffer_offset = -1;
7749 int store_remset_buffer_index_offset = -1, store_remset_buffer_index_addr_offset = -1;
7751 MONO_THREAD_VAR_OFFSET (stack_end, stack_end_offset);
7752 g_assert (stack_end_offset != -1);
7753 MONO_THREAD_VAR_OFFSET (store_remset_buffer, store_remset_buffer_offset);
7754 g_assert (store_remset_buffer_offset != -1);
7755 MONO_THREAD_VAR_OFFSET (store_remset_buffer_index, store_remset_buffer_index_offset);
7756 g_assert (store_remset_buffer_index_offset != -1);
7757 MONO_THREAD_VAR_OFFSET (store_remset_buffer_index_addr, store_remset_buffer_index_addr_offset);
7758 g_assert (store_remset_buffer_index_addr_offset != -1);
7762 // FIXME: Maybe create a separate version for ctors (the branch would be
7763 // correctly predicted more times)
7764 if (write_barrier_method)
7765 return write_barrier_method;
7767 /* Create the IL version of mono_gc_barrier_generic_store () */
7768 sig = mono_metadata_signature_alloc (mono_defaults.corlib, 1);
7769 sig->ret = &mono_defaults.void_class->byval_arg;
7770 sig->params [0] = &mono_defaults.int_class->byval_arg;
7772 mb = mono_mb_new (mono_defaults.object_class, "wbarrier", MONO_WRAPPER_WRITE_BARRIER);
7774 #ifdef MANAGED_WBARRIER
7775 if (mono_runtime_has_tls_get ()) {
7776 #ifdef ALIGN_NURSERY
7777 // if (ptr_in_nursery (ptr)) return;
7779 * Masking out the bits might be faster, but we would have to use 64 bit
7780 * immediates, which might be slower.
7782 mono_mb_emit_ldarg (mb, 0);
7783 mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
7784 mono_mb_emit_byte (mb, CEE_SHR_UN);
7785 mono_mb_emit_icon (mb, (mword)nursery_start >> DEFAULT_NURSERY_BITS);
7786 label_no_wb_1 = mono_mb_emit_branch (mb, CEE_BEQ);
7788 // if (!ptr_in_nursery (*ptr)) return;
7789 mono_mb_emit_ldarg (mb, 0);
7790 mono_mb_emit_byte (mb, CEE_LDIND_I);
7791 mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
7792 mono_mb_emit_byte (mb, CEE_SHR_UN);
7793 mono_mb_emit_icon (mb, (mword)nursery_start >> DEFAULT_NURSERY_BITS);
7794 label_no_wb_2 = mono_mb_emit_branch (mb, CEE_BNE_UN);
7797 g_assert_not_reached ();
7800 // if (ptr >= stack_end) goto need_wb;
7801 mono_mb_emit_ldarg (mb, 0);
7802 EMIT_TLS_ACCESS (mb, stack_end, stack_end_offset);
7803 label_need_wb = mono_mb_emit_branch (mb, CEE_BGE_UN);
7805 // if (ptr >= stack_start) return;
7806 dummy_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7807 mono_mb_emit_ldarg (mb, 0);
7808 mono_mb_emit_ldloc_addr (mb, dummy_var);
7809 label_no_wb_3 = mono_mb_emit_branch (mb, CEE_BGE_UN);
7812 mono_mb_patch_branch (mb, label_need_wb);
7814 // buffer = STORE_REMSET_BUFFER;
7815 buffer_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7816 EMIT_TLS_ACCESS (mb, store_remset_buffer, store_remset_buffer_offset);
7817 mono_mb_emit_stloc (mb, buffer_var);
7819 // buffer_index = STORE_REMSET_BUFFER_INDEX;
7820 buffer_index_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7821 EMIT_TLS_ACCESS (mb, store_remset_buffer_index, store_remset_buffer_index_offset);
7822 mono_mb_emit_stloc (mb, buffer_index_var);
7824 // if (buffer [buffer_index] == ptr) return;
7825 mono_mb_emit_ldloc (mb, buffer_var);
7826 mono_mb_emit_ldloc (mb, buffer_index_var);
7827 g_assert (sizeof (gpointer) == 4 || sizeof (gpointer) == 8);
7828 mono_mb_emit_icon (mb, sizeof (gpointer) == 4 ? 2 : 3);
7829 mono_mb_emit_byte (mb, CEE_SHL);
7830 mono_mb_emit_byte (mb, CEE_ADD);
7831 mono_mb_emit_byte (mb, CEE_LDIND_I);
7832 mono_mb_emit_ldarg (mb, 0);
7833 label_no_wb_4 = mono_mb_emit_branch (mb, CEE_BEQ);
7836 mono_mb_emit_ldloc (mb, buffer_index_var);
7837 mono_mb_emit_icon (mb, 1);
7838 mono_mb_emit_byte (mb, CEE_ADD);
7839 mono_mb_emit_stloc (mb, buffer_index_var);
7841 // if (buffer_index >= STORE_REMSET_BUFFER_SIZE) goto slow_path;
7842 mono_mb_emit_ldloc (mb, buffer_index_var);
7843 mono_mb_emit_icon (mb, STORE_REMSET_BUFFER_SIZE);
7844 label_slow_path = mono_mb_emit_branch (mb, CEE_BGE);
7846 // buffer [buffer_index] = ptr;
7847 mono_mb_emit_ldloc (mb, buffer_var);
7848 mono_mb_emit_ldloc (mb, buffer_index_var);
7849 g_assert (sizeof (gpointer) == 4 || sizeof (gpointer) == 8);
7850 mono_mb_emit_icon (mb, sizeof (gpointer) == 4 ? 2 : 3);
7851 mono_mb_emit_byte (mb, CEE_SHL);
7852 mono_mb_emit_byte (mb, CEE_ADD);
7853 mono_mb_emit_ldarg (mb, 0);
7854 mono_mb_emit_byte (mb, CEE_STIND_I);
7856 // STORE_REMSET_BUFFER_INDEX = buffer_index;
7857 EMIT_TLS_ACCESS (mb, store_remset_buffer_index_addr, store_remset_buffer_index_addr_offset);
7858 mono_mb_emit_ldloc (mb, buffer_index_var);
7859 mono_mb_emit_byte (mb, CEE_STIND_I);
7862 mono_mb_patch_branch (mb, label_no_wb_1);
7863 mono_mb_patch_branch (mb, label_no_wb_2);
7864 mono_mb_patch_branch (mb, label_no_wb_3);
7865 mono_mb_patch_branch (mb, label_no_wb_4);
7866 mono_mb_emit_byte (mb, CEE_RET);
7869 mono_mb_patch_branch (mb, label_slow_path);
7873 mono_mb_emit_ldarg (mb, 0);
7874 mono_mb_emit_icall (mb, mono_gc_wbarrier_generic_nostore);
7875 mono_mb_emit_byte (mb, CEE_RET);
7877 res = mono_mb_create_method (mb, sig, 16);
7880 mono_loader_lock ();
7881 if (write_barrier_method) {
7882 /* Already created */
7883 mono_free_method (res);
7885 /* double-checked locking */
7886 mono_memory_barrier ();
7887 write_barrier_method = res;
7889 mono_loader_unlock ();
7891 return write_barrier_method;
7894 #endif /* HAVE_SGEN_GC */