2 * sgen-gc.c: Simple generational GC.
5 * Paolo Molaro (lupus@ximian.com)
7 * Copyright 2005-2009 Novell, Inc (http://www.novell.com)
9 * Thread start/stop adapted from Boehm's GC:
10 * Copyright (c) 1994 by Xerox Corporation. All rights reserved.
11 * Copyright (c) 1996 by Silicon Graphics. All rights reserved.
12 * Copyright (c) 1998 by Fergus Henderson. All rights reserved.
13 * Copyright (c) 2000-2004 by Hewlett-Packard Company. All rights reserved.
15 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
16 * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
18 * Permission is hereby granted to use or copy this program
19 * for any purpose, provided the above notices are retained on all copies.
20 * Permission to modify the code and to distribute modified code is granted,
21 * provided the above notices are retained, and a notice that the code was
22 * modified is included with the above copyright notice.
24 * All the rest of the code is LGPL.
26 * Important: allocation provides always zeroed memory, having to do
27 * a memset after allocation is deadly for performance.
28 * Memory usage at startup is currently as follows:
30 * 64 KB internal space
32 * We should provide a small memory config with half the sizes
34 * We currently try to make as few mono assumptions as possible:
35 * 1) 2-word header with no GC pointers in it (first vtable, second to store the
37 * 2) gc descriptor is the second word in the vtable (first word in the class)
38 * 3) 8 byte alignment is the minimum and enough (not true for special structures, FIXME)
39 * 4) there is a function to get an object's size and the number of
40 * elements in an array.
41 * 5) we know the special way bounds are allocated for complex arrays
43 * Always try to keep stack usage to a minimum: no recursive behaviour
44 * and no large stack allocs.
46 * General description.
47 * Objects are initially allocated in a nursery using a fast bump-pointer technique.
48 * When the nursery is full we start a nursery collection: this is performed with a
50 * When the old generation is full we start a copying GC of the old generation as well:
51 * this will be changed to mark/compact in the future.
52 * The things that complicate this description are:
53 * *) pinned objects: we can't move them so we need to keep track of them
54 * *) no precise info of the thread stacks and registers: we need to be able to
55 * quickly find the objects that may be referenced conservatively and pin them
56 * (this makes the first issues more important)
57 * *) large objects are too expensive to be dealt with using copying GC: we handle them
58 * with mark/sweep during major collections
59 * *) some objects need to not move even if they are small (interned strings, Type handles):
60 * we use mark/sweep for them, too: they are not allocated in the nursery, but inside
61 * PinnedChunks regions
66 *) change the jit to emit write barrier calls when needed (we
67 can have specialized write barriers): done with icalls, still need to
68 use some specialized barriers
69 *) we could have a function pointer in MonoClass to implement
70 customized write barriers for value types
71 *) the write barrier code could be isolated in a couple of functions: when a
72 thread is stopped if it's inside the barrier it is let go again
73 until we stop outside of them (not really needed, see below GC-safe points)
74 *) investigate the stuff needed to advance a thread to a GC-safe
75 point (single-stepping, read from unmapped memory etc) and implement it
76 Not needed yet: since we treat the objects reachable from the stack/regs as
77 roots, we store the ptr and exec the write barrier so there is no race.
78 We may need this to solve the issue with setting the length of arrays and strings.
79 We may need this also for handling precise info on stacks, even simple things
80 as having uninitialized data on the stack and having to wait for the prolog
81 to zero it. Not an issue for the last frame that we scan conservatively.
82 We could always not trust the value in the slots anyway.
83 *) make the jit info table lock free
84 *) modify the jit to save info about references in stack locations:
85 this can be done just for locals as a start, so that at least
86 part of the stack is handled precisely.
87 *) Make the debug printf stuff thread and signal safe.
88 *) test/fix 64 bit issues
89 *) test/fix endianess issues
91 *) add batch moving profile info
92 *) add more timing info
93 *) there is a possible race when an array or string is created: the vtable is set,
94 but the length is set only later so if the GC needs to scan the object in that window,
95 it won't get the correct size for the object. The object can't have references and it will
96 be pinned, but a free memory fragment may be created that overlaps with it.
97 We should change the array max_length field to be at the same offset as the string length:
98 this way we can have a single special alloc function for them that sets the length.
99 Multi-dim arrays have the same issue for rank == 1 for the bounds data.
100 *) implement a card table as the write barrier instead of remembered sets?
101 *) some sort of blacklist support?
102 *) fin_ready_list and critical_fin_list are part of the root set, too
103 *) consider lowering the large object min size to 16/32KB or so and benchmark
104 *) once mark-compact is implemented we could still keep the
105 copying collector for the old generation and use it if we think
106 it is better (small heaps and no pinning object in the old
108 *) avoid the memory store from copy_object when not needed.
109 *) optimize the write barriers fastpath to happen in managed code
110 *) add an option to mmap the whole heap in one chunk: it makes for many
111 simplifications in the checks (put the nursery at the top and just use a single
112 check for inclusion/exclusion): the issue this has is that on 32 bit systems it's
113 not flexible (too much of the address space may be used by default or we can't
114 increase the heap as needed) and we'd need a race-free mechanism to return memory
115 back to the system (mprotect(PROT_NONE) will still keep the memory allocated if it
116 was written to, munmap is needed, but the following mmap may not find the same segment
118 *) memzero the fragments after restarting the world and optionally a smaller chunk at a time
119 *) an additional strategy to realloc/expand the nursery when fully pinned is to start
120 allocating objects in the old generation. This means that we can't optimize away write
121 barrier calls in ctors (but that is not valid for other reasons, too).
122 *) add write barriers to the Clone methods
130 #include <semaphore.h>
134 #include <sys/types.h>
135 #include <sys/stat.h>
136 #include <sys/mman.h>
137 #include <sys/time.h>
140 #include "metadata/metadata-internals.h"
141 #include "metadata/class-internals.h"
142 #include "metadata/gc-internal.h"
143 #include "metadata/object-internals.h"
144 #include "metadata/threads.h"
145 #include "metadata/sgen-gc.h"
146 #include "metadata/sgen-archdep.h"
147 #include "metadata/mono-gc.h"
148 #include "metadata/method-builder.h"
149 #include "metadata/profiler-private.h"
150 #include "metadata/monitor.h"
151 #include "utils/mono-mmap.h"
153 #ifdef HAVE_VALGRIND_MEMCHECK_H
154 #include <valgrind/memcheck.h>
157 #define OPDEF(a,b,c,d,e,f,g,h,i,j) \
161 #include "mono/cil/opcode.def"
168 * ######################################################################
169 * ######## Types and constants used by the GC.
170 * ######################################################################
172 #if SIZEOF_VOID_P == 4
173 typedef guint32 mword;
175 typedef guint64 mword;
178 static int gc_initialized = 0;
179 static int gc_debug_level = 0;
180 static FILE* gc_debug_file;
181 /* If set, do a minor collection before every allocation */
182 static gboolean collect_before_allocs = FALSE;
183 /* If set, do a heap consistency check before each minor collection */
184 static gboolean consistency_check_at_minor_collection = FALSE;
188 mono_gc_flush_info (void)
190 fflush (gc_debug_file);
194 #define MAX_DEBUG_LEVEL 8
195 #define DEBUG(level,a) do {if (G_UNLIKELY ((level) <= MAX_DEBUG_LEVEL && (level) <= gc_debug_level)) a;} while (0)
197 #define TV_DECLARE(name) struct timeval name
198 #define TV_GETTIME(tv) gettimeofday (&(tv), NULL)
199 #define TV_ELAPSED(start,end) (int)((((end).tv_sec - (start).tv_sec) * 1000000) + end.tv_usec - start.tv_usec)
201 #define GC_BITS_PER_WORD (sizeof (mword) * 8)
211 /* each request from the OS ends up in a GCMemSection */
212 typedef struct _GCMemSection GCMemSection;
213 struct _GCMemSection {
217 /* pointer where more data could be allocated if it fits */
221 * scan starts is an array of pointers to objects equally spaced in the allocation area
222 * They let use quickly find pinned objects from pinning pointers.
225 /* in major collections indexes in the pin_queue for objects that pin this section */
228 unsigned short num_scan_start;
232 /* large object space struct: 64+ KB */
233 /* we could make this limit much smaller to avoid memcpy copy
234 * and potentially have more room in the GC descriptor: need to measure
235 * This also means that such small OS objects will need to be
236 * allocated in a different way (using pinned chunks).
237 * We may want to put large but smaller than 64k objects in the fixed space
238 * when we move the object from one generation to another (to limit the
239 * pig in the snake effect).
240 * Note: it may be worth to have an optimized copy function, since we can
241 * assume that objects are aligned and have a multiple of 8 size.
242 * FIXME: This structure needs to be a multiple of 8 bytes in size: this is not
243 * true if MONO_ZERO_LEN_ARRAY is nonzero.
245 typedef struct _LOSObject LOSObject;
248 mword size; /* this is the object size */
249 int dummy; /* to have a sizeof (LOSObject) a multiple of ALLOC_ALIGN and data starting at same alignment */
252 char data [MONO_ZERO_LEN_ARRAY];
255 /* Pinned objects are allocated in the LOS space if bigger than half a page
256 * or from freelists otherwise. We assume that pinned objects are relatively few
257 * and they have a slow dying speed (like interned strings, thread objects).
258 * As such they will be collected only at major collections.
259 * free lists are not global: when we need memory we allocate a PinnedChunk.
260 * Each pinned chunk is made of several pages, the first of wich is used
261 * internally for bookeeping (here think of a page as 4KB). The bookeeping
262 * includes the freelists vectors and info about the object size of each page
263 * in the pinned chunk. So, when needed, a free page is found in a pinned chunk,
264 * a size is assigned to it, the page is divided in the proper chunks and each
265 * chunk is added to the freelist. To not waste space, the remaining space in the
266 * first page is used as objects of size 16 or 32 (need to measure which are more
268 * We use this same structure to allocate memory used internally by the GC, so
269 * we never use malloc/free if we need to alloc during collection: the world is stopped
270 * and malloc/free will deadlock.
271 * When we want to iterate over pinned objects, we just scan a page at a time
272 * linearly according to the size of objects in the page: the next pointer used to link
273 * the items in the freelist uses the same word as the vtable. Since we keep freelists
274 * for each pinned chunk, if the word points outside the pinned chunk it means
276 * We could avoid this expensive scanning in creative ways. We could have a policy
277 * of putting in the pinned space only objects we know about that have no struct fields
278 * with references and we can easily use a even expensive write barrier for them,
279 * since pointer writes on such objects should be rare.
280 * The best compromise is to just alloc interned strings and System.MonoType in them.
281 * It would be nice to allocate MonoThread in it, too: must check that we properly
282 * use write barriers so we don't have to do any expensive scanning of the whole pinned
283 * chunk list during minor collections. We can avoid it now because we alloc in it only
284 * reference-free objects.
286 #define PINNED_FIRST_SLOT_SIZE (sizeof (gpointer) * 4)
287 #define MAX_FREELIST_SIZE 2048
288 #define PINNED_PAGE_SIZE (4096)
289 #define PINNED_CHUNK_MIN_SIZE (4096*8)
290 typedef struct _PinnedChunk PinnedChunk;
291 struct _PinnedChunk {
294 int *page_sizes; /* a 0 means the page is still unused */
297 void *data [1]; /* page sizes and free lists are stored here */
300 /* The method used to clear the nursery */
301 /* Clearing at nursery collections is the safest, but has bad interactions with caches.
302 * Clearing at TLAB creation is much faster, but more complex and it might expose hard
307 CLEAR_AT_TLAB_CREATION
308 } NurseryClearPolicy;
310 static NurseryClearPolicy nursery_clear_policy = CLEAR_AT_TLAB_CREATION;
313 * If this is set, the nursery is aligned to an address aligned to its size, ie.
314 * a 1MB nursery will be aligned to an address divisible by 1MB. This allows us to
315 * speed up ptr_in_nursery () checks which are very frequent. This requires the
316 * nursery size to be a compile time constant.
318 #define ALIGN_NURSERY 1
321 * The young generation is divided into fragments. This is because
322 * we can hand one fragments to a thread for lock-less fast alloc and
323 * because the young generation ends up fragmented anyway by pinned objects.
324 * Once a collection is done, a list of fragments is created. When doing
325 * thread local alloc we use smallish nurseries so we allow new threads to
326 * allocate memory from gen0 without triggering a collection. Threads that
327 * are found to allocate lots of memory are given bigger fragments. This
328 * should make the finalizer thread use little nursery memory after a while.
329 * We should start assigning threads very small fragments: if there are many
330 * threads the nursery will be full of reserved space that the threads may not
331 * use at all, slowing down allocation speed.
332 * Thread local allocation is done from areas of memory Hotspot calls Thread Local
333 * Allocation Buffers (TLABs).
335 typedef struct _Fragment Fragment;
339 char *fragment_start;
340 char *fragment_limit; /* the current soft limit for allocation */
344 /* the runtime can register areas of memory as roots: we keep two lists of roots,
345 * a pinned root set for conservatively scanned roots and a normal one for
346 * precisely scanned roots (currently implemented as a single list).
348 typedef struct _RootRecord RootRecord;
356 /* for use with write barriers */
357 typedef struct _RememberedSet RememberedSet;
358 struct _RememberedSet {
362 mword data [MONO_ZERO_LEN_ARRAY];
365 /* we have 4 possible values in the low 2 bits */
367 REMSET_LOCATION, /* just a pointer to the exact location */
368 REMSET_RANGE, /* range of pointer fields */
369 REMSET_OBJECT, /* mark all the object for scanning */
370 REMSET_OTHER, /* all others */
371 REMSET_TYPE_MASK = 0x3
374 /* Subtypes of REMSET_OTHER */
376 REMSET_VTYPE, /* a valuetype array described by a gc descriptor and a count */
377 REMSET_ROOT_LOCATION, /* a location inside a root */
380 #ifdef HAVE_KW_THREAD
381 static __thread RememberedSet *remembered_set MONO_TLS_FAST;
383 static pthread_key_t remembered_set_key;
384 static RememberedSet *global_remset;
385 static RememberedSet *freed_thread_remsets;
386 //static int store_to_global_remset = 0;
388 /* FIXME: later choose a size that takes into account the RememberedSet struct
389 * and doesn't waste any alloc paddin space.
391 #define DEFAULT_REMSET_SIZE 1024
392 static RememberedSet* alloc_remset (int size, gpointer id);
394 /* Structure that corresponds to a MonoVTable: desc is a mword so requires
395 * no cast from a pointer to an integer
402 /* these bits are set in the object vtable: we could merge them since an object can be
403 * either pinned or forwarded but not both.
404 * We store them in the vtable slot because the bits are used in the sync block for
405 * other purposes: if we merge them and alloc the sync blocks aligned to 8 bytes, we can change
406 * this and use bit 3 in the syncblock (with the lower two bits both set for forwarded, that
407 * would be an invalid combination for the monitor and hash code).
408 * The values are already shifted.
409 * The forwarding address is stored in the sync block.
411 #define FORWARDED_BIT 1
413 #define VTABLE_BITS_MASK 0x3
415 /* returns NULL if not forwarded, or the forwarded address */
416 #define object_is_forwarded(obj) (((mword*)(obj))[0] & FORWARDED_BIT? (void*)(((mword*)(obj))[1]): NULL)
417 /* set the forwarded address fw_addr for object obj */
418 #define forward_object(obj,fw_addr) do { \
419 ((mword*)(obj))[0] |= FORWARDED_BIT; \
420 ((mword*)(obj))[1] = (mword)(fw_addr); \
423 #define object_is_pinned(obj) (((mword*)(obj))[0] & PINNED_BIT)
424 #define pin_object(obj) do { \
425 ((mword*)(obj))[0] |= PINNED_BIT; \
427 #define unpin_object(obj) do { \
428 ((mword*)(obj))[0] &= ~PINNED_BIT; \
432 #define ptr_in_nursery(ptr) (((mword)(ptr) & ~((1 << DEFAULT_NURSERY_BITS) - 1)) == (mword)nursery_start)
434 #define ptr_in_nursery(ptr) ((char*)(ptr) >= nursery_start && (char*)(ptr) < nursery_real_end)
438 * Since we set bits in the vtable, use the macro to load it from the pointer to
439 * an object that is potentially pinned.
441 #define LOAD_VTABLE(addr) ((*(mword*)(addr)) & ~VTABLE_BITS_MASK)
444 safe_name (void* obj)
446 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (obj);
447 return vt->klass->name;
451 safe_object_get_size (MonoObject* o)
453 MonoClass *klass = ((MonoVTable*)LOAD_VTABLE (o))->klass;
454 if (klass == mono_defaults.string_class) {
455 return sizeof (MonoString) + 2 * mono_string_length ((MonoString*) o) + 2;
456 } else if (klass->rank) {
457 MonoArray *array = (MonoArray*)o;
458 size_t size = sizeof (MonoArray) + mono_array_element_size (klass) * mono_array_length (array);
459 if (G_UNLIKELY (array->bounds)) {
460 size += sizeof (mono_array_size_t) - 1;
461 size &= ~(sizeof (mono_array_size_t) - 1);
462 size += sizeof (MonoArrayBounds) * klass->rank;
466 /* from a created object: the class must be inited already */
467 return klass->instance_size;
471 static inline gboolean
472 is_maybe_half_constructed (MonoObject *o)
476 klass = ((MonoVTable*)LOAD_VTABLE (o))->klass;
477 if ((klass == mono_defaults.string_class && mono_string_length ((MonoString*)o) == 0) ||
478 (klass->rank && mono_array_length ((MonoArray*)o) == 0))
485 * ######################################################################
486 * ######## Global data.
487 * ######################################################################
489 static LOCK_DECLARE (gc_mutex);
490 static int gc_disabled = 0;
491 static int num_minor_gcs = 0;
492 static int num_major_gcs = 0;
494 /* good sizes are 512KB-1MB: larger ones increase a lot memzeroing time */
495 //#define DEFAULT_NURSERY_SIZE (1024*512*125+4096*118)
496 #define DEFAULT_NURSERY_SIZE (1024*512*2)
497 /* The number of trailing 0 bits in DEFAULT_NURSERY_SIZE */
498 #define DEFAULT_NURSERY_BITS 20
499 #define DEFAULT_MAX_SECTION (DEFAULT_NURSERY_SIZE * 16)
500 #define DEFAULT_LOS_COLLECTION_TARGET (DEFAULT_NURSERY_SIZE * 2)
501 /* to quickly find the head of an object pinned by a conservative address
502 * we keep track of the objects allocated for each SCAN_START_SIZE memory
503 * chunk in the nursery or other memory sections. Larger values have less
504 * memory overhead and bigger runtime cost. 4-8 KB are reasonable values.
506 #define SCAN_START_SIZE (4096*2)
507 /* the minimum size of a fragment that we consider useful for allocation */
508 #define FRAGMENT_MIN_SIZE (512)
509 /* This is a fixed value used for pinned chunks, not the system pagesize */
510 #define FREELIST_PAGESIZE 4096
512 static mword pagesize = 4096;
513 static mword nursery_size = DEFAULT_NURSERY_SIZE;
514 static mword next_section_size = DEFAULT_NURSERY_SIZE * 4;
515 static mword max_section_size = DEFAULT_MAX_SECTION;
516 static int section_size_used = 0;
517 static int degraded_mode = 0;
519 static LOSObject *los_object_list = NULL;
520 static mword los_memory_usage = 0;
521 static mword los_num_objects = 0;
522 static mword next_los_collection = 2*1024*1024; /* 2 MB, need to tune */
523 static mword total_alloc = 0;
524 /* use this to tune when to do a major/minor collection */
525 static mword memory_pressure = 0;
527 static GCMemSection *section_list = NULL;
528 static GCMemSection *nursery_section = NULL;
529 static mword lowest_heap_address = ~(mword)0;
530 static mword highest_heap_address = 0;
532 typedef struct _FinalizeEntry FinalizeEntry;
533 struct _FinalizeEntry {
538 typedef struct _FinalizeEntryHashTable FinalizeEntryHashTable;
539 struct _FinalizeEntryHashTable {
540 FinalizeEntry **table;
545 typedef struct _DisappearingLink DisappearingLink;
546 struct _DisappearingLink {
547 DisappearingLink *next;
551 typedef struct _DisappearingLinkHashTable DisappearingLinkHashTable;
552 struct _DisappearingLinkHashTable {
553 DisappearingLink **table;
558 #define LARGE_INTERNAL_MEM_HEADER_MAGIC 0x7d289f3a
560 typedef struct _LargeInternalMemHeader LargeInternalMemHeader;
561 struct _LargeInternalMemHeader {
574 * The link pointer is hidden by negating each bit. We use the lowest
575 * bit of the link (before negation) to store whether it needs
576 * resurrection tracking.
578 #define HIDE_POINTER(p,t) ((gpointer)(~((gulong)(p)|((t)?1:0))))
579 #define REVEAL_POINTER(p) ((gpointer)((~(gulong)(p))&~3L))
581 #define DISLINK_OBJECT(d) (REVEAL_POINTER (*(d)->link))
582 #define DISLINK_TRACK(d) ((~(gulong)(*(d)->link)) & 1)
585 * The finalizable hash has the object as the key, the
586 * disappearing_link hash, has the link address as key.
588 static FinalizeEntryHashTable minor_finalizable_hash;
589 static FinalizeEntryHashTable major_finalizable_hash;
590 /* objects that are ready to be finalized */
591 static FinalizeEntry *fin_ready_list = NULL;
592 static FinalizeEntry *critical_fin_list = NULL;
594 static DisappearingLinkHashTable minor_disappearing_link_hash;
595 static DisappearingLinkHashTable major_disappearing_link_hash;
597 static int num_ready_finalizers = 0;
598 static int no_finalize = 0;
600 /* keep each size a multiple of ALLOC_ALIGN */
601 /* on 64 bit systems 8 is likely completely unused. */
602 static const int freelist_sizes [] = {
603 8, 16, 24, 32, 40, 48, 64, 80,
604 96, 128, 160, 192, 224, 256, 320, 384,
605 448, 512, 584, 680, 816, 1024, 1360, 2048};
606 #define FREELIST_NUM_SLOTS (sizeof (freelist_sizes) / sizeof (freelist_sizes [0]))
608 static char* max_pinned_chunk_addr = NULL;
609 static char* min_pinned_chunk_addr = (char*)-1;
610 /* pinned_chunk_list is used for allocations of objects that are never moved */
611 static PinnedChunk *pinned_chunk_list = NULL;
612 /* internal_chunk_list is used for allocating structures needed by the GC */
613 static PinnedChunk *internal_chunk_list = NULL;
616 obj_is_from_pinned_alloc (char *p)
618 PinnedChunk *chunk = pinned_chunk_list;
619 for (; chunk; chunk = chunk->next) {
620 if (p >= (char*)chunk->start_data && p < ((char*)chunk + chunk->num_pages * FREELIST_PAGESIZE))
626 static int slot_for_size (size_t size);
629 free_pinned_object (PinnedChunk *chunk, char *obj, size_t size)
631 void **p = (void**)obj;
632 int slot = slot_for_size (size);
634 g_assert (obj >= (char*)chunk->start_data && obj < ((char*)chunk + chunk->num_pages * FREELIST_PAGESIZE));
635 *p = chunk->free_list [slot];
636 chunk->free_list [slot] = p;
640 ROOT_TYPE_NORMAL = 0, /* "normal" roots */
641 ROOT_TYPE_PINNED = 1, /* roots without a GC descriptor */
642 ROOT_TYPE_WBARRIER = 2, /* roots with a write barrier */
646 /* registered roots: the key to the hash is the root start address */
648 * Different kinds of roots are kept separate to speed up pin_from_roots () for example.
650 static RootRecord **roots_hash [ROOT_TYPE_NUM] = { NULL, NULL };
651 static int roots_hash_size [ROOT_TYPE_NUM] = { 0, 0, 0 };
652 static mword roots_size = 0; /* amount of memory in the root set */
653 static int num_roots_entries [ROOT_TYPE_NUM] = { 0, 0, 0 };
656 * The current allocation cursors
657 * We allocate objects in the nursery.
658 * The nursery is the area between nursery_start and nursery_real_end.
659 * Allocation is done from a Thread Local Allocation Buffer (TLAB). TLABs are allocated
660 * from nursery fragments.
661 * tlab_next is the pointer to the space inside the TLAB where the next object will
663 * tlab_temp_end is the pointer to the end of the temporary space reserved for
664 * the allocation: it allows us to set the scan starts at reasonable intervals.
665 * tlab_real_end points to the end of the TLAB.
666 * nursery_frag_real_end points to the end of the currently used nursery fragment.
667 * nursery_first_pinned_start points to the start of the first pinned object in the nursery
668 * nursery_last_pinned_end points to the end of the last pinned object in the nursery
669 * At the next allocation, the area of the nursery where objects can be present is
670 * between MIN(nursery_first_pinned_start, first_fragment_start) and
671 * MAX(nursery_last_pinned_end, nursery_frag_real_end)
673 static char *nursery_start = NULL;
675 /* eventually share with MonoThread? */
676 typedef struct _SgenThreadInfo SgenThreadInfo;
678 struct _SgenThreadInfo {
679 SgenThreadInfo *next;
681 unsigned int stop_count; /* to catch duplicate signals */
686 void *stack_start_limit;
687 char **tlab_next_addr;
688 char **tlab_start_addr;
689 char **tlab_temp_end_addr;
690 char **tlab_real_end_addr;
691 RememberedSet *remset;
692 gpointer runtime_data;
693 gpointer stopped_ip; /* only valid if the thread is stopped */
694 MonoDomain *stopped_domain; /* ditto */
695 gpointer *stopped_regs; /* ditto */
696 #ifndef HAVE_KW_THREAD
704 #ifdef HAVE_KW_THREAD
705 #define TLAB_ACCESS_INIT
706 #define TLAB_START tlab_start
707 #define TLAB_NEXT tlab_next
708 #define TLAB_TEMP_END tlab_temp_end
709 #define TLAB_REAL_END tlab_real_end
710 #define REMEMBERED_SET remembered_set
712 static pthread_key_t thread_info_key;
713 #define TLAB_ACCESS_INIT SgenThreadInfo *__thread_info__ = pthread_getspecific (thread_info_key)
714 #define TLAB_START (__thread_info__->tlab_start)
715 #define TLAB_NEXT (__thread_info__->tlab_next)
716 #define TLAB_TEMP_END (__thread_info__->tlab_temp_end)
717 #define TLAB_REAL_END (__thread_info__->tlab_real_end)
718 #define REMEMBERED_SET (__thread_info__->remset)
722 * FIXME: What is faster, a TLS variable pointing to a structure, or separate TLS
723 * variables for next+temp_end ?
725 #ifdef HAVE_KW_THREAD
726 static __thread char *tlab_start;
727 static __thread char *tlab_next;
728 static __thread char *tlab_temp_end;
729 static __thread char *tlab_real_end;
730 /* Used by the managed allocator */
731 static __thread char **tlab_next_addr;
732 static __thread char *stack_end;
734 static char *nursery_next = NULL;
735 static char *nursery_frag_real_end = NULL;
736 static char *nursery_real_end = NULL;
737 //static char *nursery_first_pinned_start = NULL;
738 static char *nursery_last_pinned_end = NULL;
740 /* The size of a TLAB */
741 /* The bigger the value, the less often we have to go to the slow path to allocate a new
742 * one, but the more space is wasted by threads not allocating much memory.
744 * FIXME: Make this self-tuning for each thread.
746 static guint32 tlab_size = (1024 * 4);
748 /* fragments that are free and ready to be used for allocation */
749 static Fragment *nursery_fragments = NULL;
750 /* freeelist of fragment structures */
751 static Fragment *fragment_freelist = NULL;
754 * used when moving the objects
755 * When the nursery is collected, objects are copied to to_space.
756 * The area between gray_first and gray_objects is used as a stack
757 * of objects that need their fields checked for more references
759 * We should optimize somehow this mechanism to avoid rescanning
760 * ptr-free objects. The order is also probably not optimal: need to
761 * test cache misses and other graph traversal orders.
763 static char *to_space = NULL;
764 static char *gray_first = NULL;
765 static char *gray_objects = NULL;
766 static char *to_space_end = NULL;
767 static GCMemSection *to_space_section = NULL;
769 /* objects bigger then this go into the large object space */
770 #define MAX_SMALL_OBJ_SIZE 0xffff
772 /* Functions supplied by the runtime to be called by the GC */
773 static MonoGCCallbacks gc_callbacks;
776 * ######################################################################
777 * ######## Macros and function declarations.
778 * ######################################################################
781 #define UPDATE_HEAP_BOUNDARIES(low,high) do { \
782 if ((mword)(low) < lowest_heap_address) \
783 lowest_heap_address = (mword)(low); \
784 if ((mword)(high) > highest_heap_address) \
785 highest_heap_address = (mword)(high); \
789 align_pointer (void *ptr)
791 mword p = (mword)ptr;
792 p += sizeof (gpointer) - 1;
793 p &= ~ (sizeof (gpointer) - 1);
797 /* forward declarations */
798 static void* get_internal_mem (size_t size);
799 static void free_internal_mem (void *addr);
800 static void* get_os_memory (size_t size, int activate);
801 static void free_os_memory (void *addr, size_t size);
802 static G_GNUC_UNUSED void report_internal_mem_usage (void);
804 static int stop_world (void);
805 static int restart_world (void);
806 static void scan_thread_data (void *start_nursery, void *end_nursery, gboolean precise);
807 static void scan_from_remsets (void *start_nursery, void *end_nursery);
808 static void find_pinning_ref_from_thread (char *obj, size_t size);
809 static void update_current_thread_stack (void *start);
810 static GCMemSection* alloc_section (size_t size);
811 static void finalize_in_range (char *start, char *end, int generation);
812 static void add_or_remove_disappearing_link (MonoObject *obj, void **link, gboolean track, int generation);
813 static void null_link_in_range (char *start, char *end, int generation);
814 static void null_links_for_domain (MonoDomain *domain, int generation);
815 static gboolean search_fragment_for_size (size_t size);
816 static void mark_pinned_from_addresses (PinnedChunk *chunk, void **start, void **end);
817 static void clear_remsets (void);
818 static void clear_tlabs (void);
819 static char *find_tlab_next_from_address (char *addr);
820 static void scan_pinned_objects (void (*callback) (PinnedChunk*, char*, size_t, void*), void *callback_data);
821 static void sweep_pinned_objects (void);
822 static void scan_from_pinned_objects (char *addr_start, char *addr_end);
823 static void free_large_object (LOSObject *obj);
824 static void free_mem_section (GCMemSection *section);
826 static void mono_gc_register_disappearing_link (MonoObject *obj, void **link, gboolean track);
828 void describe_ptr (char *ptr);
829 void check_consistency (void);
830 char* check_object (char *start);
833 * ######################################################################
834 * ######## GC descriptors
835 * ######################################################################
836 * Used to quickly get the info the GC needs about an object: size and
837 * where the references are held.
839 /* objects are aligned to 8 bytes boundaries
840 * A descriptor is a pointer in MonoVTable, so 32 or 64 bits of size.
841 * The low 3 bits define the type of the descriptor. The other bits
842 * depend on the type.
843 * As a general rule the 13 remaining low bits define the size, either
844 * of the whole object or of the elements in the arrays. While for objects
845 * the size is already in bytes, for arrays we need to shift, because
846 * array elements might be smaller than 8 bytes. In case of arrays, we
847 * use two bits to describe what the additional high bits represents,
848 * so the default behaviour can handle element sizes less than 2048 bytes.
849 * The high 16 bits, if 0 it means the object is pointer-free.
850 * This design should make it easy and fast to skip over ptr-free data.
851 * The first 4 types should cover >95% of the objects.
852 * Note that since the size of objects is limited to 64K, larger objects
853 * will be allocated in the large object heap.
854 * If we want 4-bytes alignment, we need to put vector and small bitmap
858 DESC_TYPE_RUN_LENGTH, /* 16 bits aligned byte size | 1-3 (offset, numptr) bytes tuples */
859 DESC_TYPE_SMALL_BITMAP, /* 16 bits aligned byte size | 16-48 bit bitmap */
860 DESC_TYPE_STRING, /* nothing */
861 DESC_TYPE_COMPLEX, /* index for bitmap into complex_descriptors */
862 DESC_TYPE_VECTOR, /* 10 bits element size | 1 bit array | 2 bits desc | element desc */
863 DESC_TYPE_ARRAY, /* 10 bits element size | 1 bit array | 2 bits desc | element desc */
864 DESC_TYPE_LARGE_BITMAP, /* | 29-61 bitmap bits */
865 DESC_TYPE_COMPLEX_ARR, /* index for bitmap into complex_descriptors */
866 /* subtypes for arrays and vectors */
867 DESC_TYPE_V_PTRFREE = 0,/* there are no refs: keep first so it has a zero value */
868 DESC_TYPE_V_REFS, /* all the array elements are refs */
869 DESC_TYPE_V_RUN_LEN, /* elements are run-length encoded as DESC_TYPE_RUN_LENGTH */
870 DESC_TYPE_V_BITMAP /* elements are as the bitmap in DESC_TYPE_SMALL_BITMAP */
873 #define OBJECT_HEADER_WORDS (sizeof(MonoObject)/sizeof(gpointer))
874 #define LOW_TYPE_BITS 3
875 #define SMALL_BITMAP_SHIFT 16
876 #define SMALL_BITMAP_SIZE (GC_BITS_PER_WORD - SMALL_BITMAP_SHIFT)
877 #define VECTOR_INFO_SHIFT 14
878 #define VECTOR_ELSIZE_SHIFT 3
879 #define LARGE_BITMAP_SIZE (GC_BITS_PER_WORD - LOW_TYPE_BITS)
880 #define MAX_SMALL_SIZE ((1 << SMALL_BITMAP_SHIFT) - 1)
881 #define SMALL_SIZE_MASK 0xfff8
882 #define MAX_ELEMENT_SIZE 0x3ff
883 #define ELEMENT_SIZE_MASK (0x3ff << LOW_TYPE_BITS)
884 #define VECTOR_SUBTYPE_PTRFREE (DESC_TYPE_V_PTRFREE << VECTOR_INFO_SHIFT)
885 #define VECTOR_SUBTYPE_REFS (DESC_TYPE_V_REFS << VECTOR_INFO_SHIFT)
886 #define VECTOR_SUBTYPE_RUN_LEN (DESC_TYPE_V_RUN_LEN << VECTOR_INFO_SHIFT)
887 #define VECTOR_SUBTYPE_BITMAP (DESC_TYPE_V_BITMAP << VECTOR_INFO_SHIFT)
889 #define ALLOC_ALIGN 8
892 /* Root bitmap descriptors are simpler: the lower three bits describe the type
893 * and we either have 30/62 bitmap bits or nibble-based run-length,
894 * or a complex descriptor, or a user defined marker function.
897 ROOT_DESC_CONSERVATIVE, /* 0, so matches NULL value */
902 ROOT_DESC_TYPE_MASK = 0x7,
903 ROOT_DESC_TYPE_SHIFT = 3,
906 #define MAKE_ROOT_DESC(type,val) ((type) | ((val) << ROOT_DESC_TYPE_SHIFT))
908 #define MAX_USER_DESCRIPTORS 16
910 static gsize* complex_descriptors = NULL;
911 static int complex_descriptors_size = 0;
912 static int complex_descriptors_next = 0;
913 static MonoGCMarkFunc user_descriptors [MAX_USER_DESCRIPTORS];
914 static int user_descriptors_next = 0;
917 alloc_complex_descriptor (gsize *bitmap, int numbits)
919 int nwords = numbits/GC_BITS_PER_WORD + 2;
924 res = complex_descriptors_next;
925 /* linear search, so we don't have duplicates with domain load/unload
926 * this should not be performance critical or we'd have bigger issues
927 * (the number and size of complex descriptors should be small).
929 for (i = 0; i < complex_descriptors_next; ) {
930 if (complex_descriptors [i] == nwords) {
932 for (j = 0; j < nwords - 1; ++j) {
933 if (complex_descriptors [i + 1 + j] != bitmap [j]) {
943 i += complex_descriptors [i];
945 if (complex_descriptors_next + nwords > complex_descriptors_size) {
946 int new_size = complex_descriptors_size * 2 + nwords;
947 complex_descriptors = g_realloc (complex_descriptors, new_size * sizeof (gsize));
948 complex_descriptors_size = new_size;
950 DEBUG (6, fprintf (gc_debug_file, "Complex descriptor %d, size: %d (total desc memory: %d)\n", res, nwords, complex_descriptors_size));
951 complex_descriptors_next += nwords;
952 complex_descriptors [res] = nwords;
953 for (i = 0; i < nwords - 1; ++i) {
954 complex_descriptors [res + 1 + i] = bitmap [i];
955 DEBUG (6, fprintf (gc_debug_file, "\tvalue: %p\n", (void*)complex_descriptors [res + 1 + i]));
962 * Descriptor builders.
965 mono_gc_make_descr_for_string (gsize *bitmap, int numbits)
967 return (void*) DESC_TYPE_STRING;
971 mono_gc_make_descr_for_object (gsize *bitmap, int numbits, size_t obj_size)
973 int first_set = -1, num_set = 0, last_set = -1, i;
975 size_t stored_size = obj_size;
976 stored_size += ALLOC_ALIGN - 1;
977 stored_size &= ~(ALLOC_ALIGN - 1);
978 for (i = 0; i < numbits; ++i) {
979 if (bitmap [i / GC_BITS_PER_WORD] & ((gsize)1 << (i % GC_BITS_PER_WORD))) {
986 if (stored_size <= MAX_SMALL_OBJ_SIZE) {
987 /* check run-length encoding first: one byte offset, one byte number of pointers
988 * on 64 bit archs, we can have 3 runs, just one on 32.
989 * It may be better to use nibbles.
992 desc = DESC_TYPE_RUN_LENGTH | stored_size;
993 DEBUG (6, fprintf (gc_debug_file, "Ptrfree descriptor %p, size: %zd\n", (void*)desc, stored_size));
995 } else if (first_set < 256 && num_set < 256 && (first_set + num_set == last_set + 1)) {
996 desc = DESC_TYPE_RUN_LENGTH | stored_size | (first_set << 16) | (num_set << 24);
997 DEBUG (6, fprintf (gc_debug_file, "Runlen descriptor %p, size: %zd, first set: %d, num set: %d\n", (void*)desc, stored_size, first_set, num_set));
1000 /* we know the 2-word header is ptr-free */
1001 if (last_set < SMALL_BITMAP_SIZE + OBJECT_HEADER_WORDS) {
1002 desc = DESC_TYPE_SMALL_BITMAP | stored_size | ((*bitmap >> OBJECT_HEADER_WORDS) << SMALL_BITMAP_SHIFT);
1003 DEBUG (6, fprintf (gc_debug_file, "Smallbitmap descriptor %p, size: %zd, last set: %d\n", (void*)desc, stored_size, last_set));
1004 return (void*) desc;
1007 /* we know the 2-word header is ptr-free */
1008 if (last_set < LARGE_BITMAP_SIZE + OBJECT_HEADER_WORDS) {
1009 desc = DESC_TYPE_LARGE_BITMAP | ((*bitmap >> OBJECT_HEADER_WORDS) << LOW_TYPE_BITS);
1010 DEBUG (6, fprintf (gc_debug_file, "Largebitmap descriptor %p, size: %zd, last set: %d\n", (void*)desc, stored_size, last_set));
1011 return (void*) desc;
1013 /* it's a complex object ... */
1014 desc = DESC_TYPE_COMPLEX | (alloc_complex_descriptor (bitmap, last_set + 1) << LOW_TYPE_BITS);
1015 return (void*) desc;
1018 /* If the array holds references, numbits == 1 and the first bit is set in elem_bitmap */
1020 mono_gc_make_descr_for_array (int vector, gsize *elem_bitmap, int numbits, size_t elem_size)
1022 int first_set = -1, num_set = 0, last_set = -1, i;
1023 mword desc = vector? DESC_TYPE_VECTOR: DESC_TYPE_ARRAY;
1024 for (i = 0; i < numbits; ++i) {
1025 if (elem_bitmap [i / GC_BITS_PER_WORD] & ((gsize)1 << (i % GC_BITS_PER_WORD))) {
1032 if (elem_size <= MAX_ELEMENT_SIZE) {
1033 desc |= elem_size << VECTOR_ELSIZE_SHIFT;
1035 return (void*)(desc | VECTOR_SUBTYPE_PTRFREE);
1037 /* Note: we also handle structs with just ref fields */
1038 if (num_set * sizeof (gpointer) == elem_size) {
1039 return (void*)(desc | VECTOR_SUBTYPE_REFS | ((gssize)(-1) << 16));
1041 /* FIXME: try run-len first */
1042 /* Note: we can't skip the object header here, because it's not present */
1043 if (last_set <= SMALL_BITMAP_SIZE) {
1044 return (void*)(desc | VECTOR_SUBTYPE_BITMAP | (*elem_bitmap << 16));
1047 /* it's am array of complex structs ... */
1048 desc = DESC_TYPE_COMPLEX_ARR;
1049 desc |= alloc_complex_descriptor (elem_bitmap, last_set + 1) << LOW_TYPE_BITS;
1050 return (void*) desc;
1053 /* Return the bitmap encoded by a descriptor */
1055 mono_gc_get_bitmap_for_descr (void *descr, int *numbits)
1057 mword d = (mword)descr;
1061 case DESC_TYPE_RUN_LENGTH: {
1062 int first_set = (d >> 16) & 0xff;
1063 int num_set = (d >> 16) & 0xff;
1066 bitmap = g_new0 (gsize, (first_set + num_set + 7) / 8);
1068 for (i = first_set; i < first_set + num_set; ++i)
1069 bitmap [i / GC_BITS_PER_WORD] |= ((gsize)1 << (i % GC_BITS_PER_WORD));
1071 *numbits = first_set + num_set;
1075 case DESC_TYPE_SMALL_BITMAP:
1076 bitmap = g_new0 (gsize, 1);
1078 bitmap [0] = (d >> SMALL_BITMAP_SHIFT) << OBJECT_HEADER_WORDS;
1080 *numbits = GC_BITS_PER_WORD;
1084 g_assert_not_reached ();
1088 /* helper macros to scan and traverse objects, macros because we resue them in many functions */
1089 #define STRING_SIZE(size,str) do { \
1090 (size) = sizeof (MonoString) + 2 * (mono_string_length ((MonoString*)(str)) + 1); \
1091 (size) += (ALLOC_ALIGN - 1); \
1092 (size) &= ~(ALLOC_ALIGN - 1); \
1095 #define OBJ_RUN_LEN_SIZE(size,desc,obj) do { \
1096 (size) = (desc) & 0xfff8; \
1099 #define OBJ_BITMAP_SIZE(size,desc,obj) do { \
1100 (size) = (desc) & 0xfff8; \
1103 //#define PREFETCH(addr) __asm__ __volatile__ (" prefetchnta %0": : "m"(*(char *)(addr)))
1104 #define PREFETCH(addr)
1106 /* code using these macros must define a HANDLE_PTR(ptr) macro that does the work */
1107 #define OBJ_RUN_LEN_FOREACH_PTR(desc,obj) do { \
1108 if ((desc) & 0xffff0000) { \
1109 /* there are pointers */ \
1110 void **_objptr_end; \
1111 void **_objptr = (void**)(obj); \
1112 _objptr += ((desc) >> 16) & 0xff; \
1113 _objptr_end = _objptr + (((desc) >> 24) & 0xff); \
1114 while (_objptr < _objptr_end) { \
1115 HANDLE_PTR (_objptr, (obj)); \
1121 /* a bitmap desc means that there are pointer references or we'd have
1122 * choosen run-length, instead: add an assert to check.
1124 #define OBJ_BITMAP_FOREACH_PTR(desc,obj) do { \
1125 /* there are pointers */ \
1126 void **_objptr = (void**)(obj); \
1127 gsize _bmap = (desc) >> 16; \
1128 _objptr += OBJECT_HEADER_WORDS; \
1130 if ((_bmap & 1)) { \
1131 HANDLE_PTR (_objptr, (obj)); \
1138 #define OBJ_LARGE_BITMAP_FOREACH_PTR(vt,obj) do { \
1139 /* there are pointers */ \
1140 void **_objptr = (void**)(obj); \
1141 gsize _bmap = (vt)->desc >> LOW_TYPE_BITS; \
1142 _objptr += OBJECT_HEADER_WORDS; \
1144 if ((_bmap & 1)) { \
1145 HANDLE_PTR (_objptr, (obj)); \
1152 #define OBJ_COMPLEX_FOREACH_PTR(vt,obj) do { \
1153 /* there are pointers */ \
1154 void **_objptr = (void**)(obj); \
1155 gsize *bitmap_data = complex_descriptors + ((vt)->desc >> LOW_TYPE_BITS); \
1156 int bwords = (*bitmap_data) - 1; \
1157 void **start_run = _objptr; \
1160 MonoObject *myobj = (MonoObject*)obj; \
1161 g_print ("found %d at %p (0x%zx): %s.%s\n", bwords, (obj), (vt)->desc, myobj->vtable->klass->name_space, myobj->vtable->klass->name); \
1163 while (bwords-- > 0) { \
1164 gsize _bmap = *bitmap_data++; \
1165 _objptr = start_run; \
1166 /*g_print ("bitmap: 0x%x/%d at %p\n", _bmap, bwords, _objptr);*/ \
1168 if ((_bmap & 1)) { \
1169 HANDLE_PTR (_objptr, (obj)); \
1174 start_run += GC_BITS_PER_WORD; \
1178 /* this one is untested */
1179 #define OBJ_COMPLEX_ARR_FOREACH_PTR(vt,obj) do { \
1180 /* there are pointers */ \
1181 gsize *mbitmap_data = complex_descriptors + ((vt)->desc >> LOW_TYPE_BITS); \
1182 int mbwords = (*mbitmap_data++) - 1; \
1183 int el_size = mono_array_element_size (((MonoObject*)(obj))->vtable->klass); \
1184 char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
1185 char *e_end = e_start + el_size * mono_array_length ((MonoArray*)(obj)); \
1187 MonoObject *myobj = (MonoObject*)start; \
1188 g_print ("found %d at %p (0x%zx): %s.%s\n", mbwords, (obj), (vt)->desc, myobj->vtable->klass->name_space, myobj->vtable->klass->name); \
1190 while (e_start < e_end) { \
1191 void **_objptr = (void**)e_start; \
1192 gsize *bitmap_data = mbitmap_data; \
1193 unsigned int bwords = mbwords; \
1194 while (bwords-- > 0) { \
1195 gsize _bmap = *bitmap_data++; \
1196 void **start_run = _objptr; \
1197 /*g_print ("bitmap: 0x%x\n", _bmap);*/ \
1199 if ((_bmap & 1)) { \
1200 HANDLE_PTR (_objptr, (obj)); \
1205 _objptr = start_run + GC_BITS_PER_WORD; \
1207 e_start += el_size; \
1211 #define OBJ_VECTOR_FOREACH_PTR(vt,obj) do { \
1212 /* note: 0xffffc000 excludes DESC_TYPE_V_PTRFREE */ \
1213 if ((vt)->desc & 0xffffc000) { \
1214 int el_size = ((vt)->desc >> 3) & MAX_ELEMENT_SIZE; \
1215 /* there are pointers */ \
1216 int etype = (vt)->desc & 0xc000; \
1217 if (etype == (DESC_TYPE_V_REFS << 14)) { \
1218 void **p = (void**)((char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector)); \
1219 void **end_refs = (void**)((char*)p + el_size * mono_array_length ((MonoArray*)(obj))); \
1220 /* Note: this code can handle also arrays of struct with only references in them */ \
1221 while (p < end_refs) { \
1222 HANDLE_PTR (p, (obj)); \
1225 } else if (etype == DESC_TYPE_V_RUN_LEN << 14) { \
1226 int offset = ((vt)->desc >> 16) & 0xff; \
1227 int num_refs = ((vt)->desc >> 24) & 0xff; \
1228 char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
1229 char *e_end = e_start + el_size * mono_array_length ((MonoArray*)(obj)); \
1230 while (e_start < e_end) { \
1231 void **p = (void**)e_start; \
1234 for (i = 0; i < num_refs; ++i) { \
1235 HANDLE_PTR (p + i, (obj)); \
1237 e_start += el_size; \
1239 } else if (etype == DESC_TYPE_V_BITMAP << 14) { \
1240 char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
1241 char *e_end = e_start + el_size * mono_array_length ((MonoArray*)(obj)); \
1242 while (e_start < e_end) { \
1243 void **p = (void**)e_start; \
1244 gsize _bmap = (vt)->desc >> 16; \
1245 /* Note: there is no object header here to skip */ \
1247 if ((_bmap & 1)) { \
1248 HANDLE_PTR (p, (obj)); \
1253 e_start += el_size; \
1259 static mword new_obj_references = 0;
1260 static mword obj_references_checked = 0;
1263 #define HANDLE_PTR(ptr,obj) do { \
1264 if (*(ptr) && (char*)*(ptr) >= nursery_start && (char*)*(ptr) < nursery_next) { \
1265 new_obj_references++; \
1266 /*printf ("bogus ptr %p found at %p in object %p (%s.%s)\n", *(ptr), (ptr), o, o->vtable->klass->name_space, o->vtable->klass->name);*/ \
1268 obj_references_checked++; \
1273 * ######################################################################
1274 * ######## Detecting and removing garbage.
1275 * ######################################################################
1276 * This section of code deals with detecting the objects no longer in use
1277 * and reclaiming the memory.
1280 #define COUNT_OBJECT_TYPES do { \
1281 switch (desc & 0x7) { \
1282 case DESC_TYPE_STRING: type_str++; break; \
1283 case DESC_TYPE_RUN_LENGTH: type_rlen++; break; \
1284 case DESC_TYPE_ARRAY: case DESC_TYPE_VECTOR: type_vector++; break; \
1285 case DESC_TYPE_SMALL_BITMAP: type_bitmap++; break; \
1286 case DESC_TYPE_LARGE_BITMAP: type_lbit++; break; \
1287 case DESC_TYPE_COMPLEX: type_complex++; break; \
1288 case DESC_TYPE_COMPLEX_ARR: type_complex++; break; \
1289 default: g_assert_not_reached (); \
1293 static void __attribute__((noinline))
1294 scan_area (char *start, char *end)
1297 int type_str = 0, type_rlen = 0, type_bitmap = 0, type_vector = 0, type_lbit = 0, type_complex = 0;
1298 new_obj_references = 0;
1299 obj_references_checked = 0;
1300 while (start < end) {
1301 if (!*(void**)start) {
1302 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
1305 vt = (GCVTable*)LOAD_VTABLE (start);
1306 DEBUG (8, fprintf (gc_debug_file, "Scanning object %p, vtable: %p (%s)\n", start, vt, vt->klass->name));
1308 MonoObject *obj = (MonoObject*)start;
1309 g_print ("found at %p (0x%zx): %s.%s\n", start, vt->desc, obj->vtable->klass->name_space, obj->vtable->klass->name);
1312 #define SCAN_OBJECT_ACTION COUNT_OBJECT_TYPES
1313 #include "sgen-scan-object.h"
1315 /*printf ("references to new nursery %p-%p (size: %dk): %d, checked: %d\n", old_start, end, (end-old_start)/1024, new_obj_references, obj_references_checked);
1316 printf ("\tstrings: %d, runl: %d, vector: %d, bitmaps: %d, lbitmaps: %d, complex: %d\n",
1317 type_str, type_rlen, type_vector, type_bitmap, type_lbit, type_complex);*/
1321 need_remove_object_for_domain (char *start, MonoDomain *domain)
1323 GCVTable *vt = (GCVTable*)LOAD_VTABLE (start);
1324 if (mono_object_domain (start) == domain) {
1325 DEBUG (1, fprintf (gc_debug_file, "Need to cleanup object %p, (%s)\n", start, safe_name (start)));
1332 process_object_for_domain_clearing (char *start, MonoDomain *domain)
1334 GCVTable *vt = (GCVTable*)LOAD_VTABLE (start);
1335 /* The object could be a proxy for an object in the domain
1337 if (mono_class_has_parent (vt->klass, mono_defaults.real_proxy_class)) {
1338 MonoObject *server = ((MonoRealProxy*)start)->unwrapped_server;
1340 /* The server could already have been zeroed out, so
1341 we need to check for that, too. */
1342 if (server && (!LOAD_VTABLE (server) || mono_object_domain (server) == domain)) {
1343 DEBUG (1, fprintf (gc_debug_file, "Cleaning up remote pointer in %p to object %p (%s)\n",
1344 start, server, LOAD_VTABLE (server) ? safe_name (server) : "null"));
1345 ((MonoRealProxy*)start)->unwrapped_server = NULL;
1350 static void __attribute__((noinline))
1351 scan_area_for_domain (MonoDomain *domain, char *start, char *end)
1356 while (start < end) {
1357 if (!*(void**)start) {
1358 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
1361 vt = (GCVTable*)LOAD_VTABLE (start);
1362 process_object_for_domain_clearing (start, domain);
1363 remove = need_remove_object_for_domain (start, domain);
1364 if (remove && ((MonoObject*)start)->synchronisation) {
1365 void **dislink = mono_monitor_get_object_monitor_weak_link ((MonoObject*)start);
1367 mono_gc_register_disappearing_link (NULL, dislink, FALSE);
1370 #define SCAN_OBJECT_NOSCAN
1371 #define SCAN_OBJECT_ACTION do { if (remove) memset (start, 0, skip_size); } while (0)
1372 #include "sgen-scan-object.h"
1377 clear_domain_process_pinned_object_callback (PinnedChunk *chunk, char *obj, size_t size, MonoDomain *domain)
1379 process_object_for_domain_clearing (obj, domain);
1383 clear_domain_free_pinned_object_callback (PinnedChunk *chunk, char *obj, size_t size, MonoDomain *domain)
1385 if (need_remove_object_for_domain (obj, domain))
1386 free_pinned_object (chunk, obj, size);
1390 * When appdomains are unloaded we can easily remove objects that have finalizers,
1391 * but all the others could still be present in random places on the heap.
1392 * We need a sweep to get rid of them even though it's going to be costly
1394 * The reason we need to remove them is because we access the vtable and class
1395 * structures to know the object size and the reference bitmap: once the domain is
1396 * unloaded the point to random memory.
1399 mono_gc_clear_domain (MonoDomain * domain)
1401 GCMemSection *section;
1402 LOSObject *bigobj, *prev;
1407 /* Clear all remaining nursery fragments */
1408 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION) {
1409 g_assert (nursery_next <= nursery_frag_real_end);
1410 memset (nursery_next, 0, nursery_frag_real_end - nursery_next);
1411 for (frag = nursery_fragments; frag; frag = frag->next) {
1412 memset (frag->fragment_start, 0, frag->fragment_end - frag->fragment_start);
1416 for (section = section_list; section; section = section->next) {
1417 scan_area_for_domain (domain, section->data, section->end_data);
1420 /* We need two passes over pinned and large objects because
1421 freeing such an object gives its memory back to the OS (in
1422 the case of large objects) or obliterates its vtable
1423 (pinned objects), but we might need to dereference a
1424 pointer from an object to another object if the first
1425 object is a proxy. */
1426 scan_pinned_objects (clear_domain_process_pinned_object_callback, domain);
1427 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1428 process_object_for_domain_clearing (bigobj->data, domain);
1431 for (bigobj = los_object_list; bigobj;) {
1432 if (need_remove_object_for_domain (bigobj->data, domain)) {
1433 LOSObject *to_free = bigobj;
1435 prev->next = bigobj->next;
1437 los_object_list = bigobj->next;
1438 bigobj = bigobj->next;
1439 DEBUG (1, fprintf (gc_debug_file, "Freeing large object %p (%s)\n",
1440 bigobj->data, safe_name (bigobj->data)));
1441 free_large_object (to_free);
1445 bigobj = bigobj->next;
1447 scan_pinned_objects (clear_domain_free_pinned_object_callback, domain);
1449 for (i = GENERATION_NURSERY; i < GENERATION_MAX; ++i)
1450 null_links_for_domain (domain, i);
1456 * add_to_global_remset:
1458 * The global remset contains locations which point into newspace after
1459 * a minor collection. This can happen if the objects they point to are pinned.
1462 add_to_global_remset (gpointer ptr, gboolean root)
1466 DEBUG (8, fprintf (gc_debug_file, "Adding global remset for %p\n", ptr));
1469 * FIXME: If an object remains pinned, we need to add it at every minor collection.
1470 * To avoid uncontrolled growth of the global remset, only add each pointer once.
1472 if (global_remset->store_next + 3 < global_remset->end_set) {
1474 *(global_remset->store_next++) = (mword)ptr | REMSET_OTHER;
1475 *(global_remset->store_next++) = (mword)REMSET_ROOT_LOCATION;
1477 *(global_remset->store_next++) = (mword)ptr;
1481 rs = alloc_remset (global_remset->end_set - global_remset->data, NULL);
1482 rs->next = global_remset;
1485 *(global_remset->store_next++) = (mword)ptr | REMSET_OTHER;
1486 *(global_remset->store_next++) = (mword)REMSET_ROOT_LOCATION;
1488 *(global_remset->store_next++) = (mword)ptr;
1492 int global_rs_size = 0;
1494 for (rs = global_remset; rs; rs = rs->next) {
1495 global_rs_size += rs->store_next - rs->data;
1497 DEBUG (4, fprintf (gc_debug_file, "Global remset now has size %d\n", global_rs_size));
1502 * This is how the copying happens from the nursery to the old generation.
1503 * We assume that at this time all the pinned objects have been identified and
1505 * We run scan_object() for each pinned object so that each referenced
1506 * objects if possible are copied. The new gray objects created can have
1507 * scan_object() run on them right away, too.
1508 * Then we run copy_object() for the precisely tracked roots. At this point
1509 * all the roots are either gray or black. We run scan_object() on the gray
1510 * objects until no more gray objects are created.
1511 * At the end of the process we walk again the pinned list and we unmark
1512 * the pinned flag. As we go we also create the list of free space for use
1513 * in the next allocation runs.
1515 * We need to remember objects from the old generation that point to the new one
1516 * (or just addresses?).
1518 * copy_object could be made into a macro once debugged (use inline for now).
1521 static char* __attribute__((noinline))
1522 copy_object (char *obj, char *from_space_start, char *from_space_end)
1524 static void *copy_labels [] = { &&LAB_0, &&LAB_1, &&LAB_2, &&LAB_3, &&LAB_4, &&LAB_5, &&LAB_6, &&LAB_7, &&LAB_8 };
1527 * FIXME: The second set of checks is only needed if we are called for tospace
1530 if (obj >= from_space_start && obj < from_space_end && (obj < to_space || obj >= to_space_end)) {
1534 DEBUG (9, fprintf (gc_debug_file, "Precise copy of %p", obj));
1535 if ((forwarded = object_is_forwarded (obj))) {
1536 g_assert (((MonoVTable*)LOAD_VTABLE(obj))->gc_descr);
1537 DEBUG (9, fprintf (gc_debug_file, " (already forwarded to %p)\n", forwarded));
1540 if (object_is_pinned (obj)) {
1541 g_assert (((MonoVTable*)LOAD_VTABLE(obj))->gc_descr);
1542 DEBUG (9, fprintf (gc_debug_file, " (pinned, no change)\n"));
1545 objsize = safe_object_get_size ((MonoObject*)obj);
1546 objsize += ALLOC_ALIGN - 1;
1547 objsize &= ~(ALLOC_ALIGN - 1);
1548 DEBUG (9, fprintf (gc_debug_file, " (to %p, %s size: %zd)\n", gray_objects, ((MonoObject*)obj)->vtable->klass->name, objsize));
1549 /* FIXME: handle pinned allocs:
1550 * Large objects are simple, at least until we always follow the rule:
1551 * if objsize >= MAX_SMALL_OBJ_SIZE, pin the object and return it.
1552 * At the end of major collections, we walk the los list and if
1553 * the object is pinned, it is marked, otherwise it can be freed.
1555 if (G_UNLIKELY (objsize >= MAX_SMALL_OBJ_SIZE || (obj >= min_pinned_chunk_addr && obj < max_pinned_chunk_addr && obj_is_from_pinned_alloc (obj)))) {
1556 DEBUG (9, fprintf (gc_debug_file, "Marked LOS/Pinned %p (%s), size: %zd\n", obj, safe_name (obj), objsize));
1560 /* ok, the object is not pinned, we can move it */
1561 /* use a optimized memcpy here */
1562 if (objsize <= sizeof (gpointer) * 8) {
1563 mword *dest = (mword*)gray_objects;
1564 goto *copy_labels [objsize / sizeof (gpointer)];
1566 (dest) [7] = ((mword*)obj) [7];
1568 (dest) [6] = ((mword*)obj) [6];
1570 (dest) [5] = ((mword*)obj) [5];
1572 (dest) [4] = ((mword*)obj) [4];
1574 (dest) [3] = ((mword*)obj) [3];
1576 (dest) [2] = ((mword*)obj) [2];
1578 (dest) [1] = ((mword*)obj) [1];
1580 (dest) [0] = ((mword*)obj) [0];
1588 char* edi = gray_objects;
1589 __asm__ __volatile__(
1591 : "=&c" (ecx), "=&D" (edi), "=&S" (esi)
1592 : "0" (objsize/4), "1" (edi),"2" (esi)
1597 memcpy (gray_objects, obj, objsize);
1600 /* adjust array->bounds */
1601 vt = ((MonoObject*)obj)->vtable;
1602 g_assert (vt->gc_descr);
1603 if (G_UNLIKELY (vt->rank && ((MonoArray*)obj)->bounds)) {
1604 MonoArray *array = (MonoArray*)gray_objects;
1605 array->bounds = (MonoArrayBounds*)((char*)gray_objects + ((char*)((MonoArray*)obj)->bounds - (char*)obj));
1606 DEBUG (9, fprintf (gc_debug_file, "Array instance %p: size: %zd, rank: %d, length: %d\n", array, objsize, vt->rank, mono_array_length (array)));
1608 /* set the forwarding pointer */
1609 forward_object (obj, gray_objects);
1611 to_space_section->scan_starts [((char*)obj - (char*)to_space_section->data)/SCAN_START_SIZE] = obj;
1612 gray_objects += objsize;
1613 DEBUG (8, g_assert (gray_objects <= to_space_end));
1620 #define HANDLE_PTR(ptr,obj) do { \
1621 void *__old = *(ptr); \
1623 *(ptr) = copy_object (__old, from_start, from_end); \
1624 DEBUG (9, if (__old != *(ptr)) fprintf (gc_debug_file, "Overwrote field at %p with %p (was: %p)\n", (ptr), *(ptr), __old)); \
1625 if (G_UNLIKELY (*(ptr) >= (void*)from_start && *(ptr) < (void*)from_end) && !ptr_in_nursery (ptr)) \
1626 add_to_global_remset ((ptr), FALSE); \
1631 * Scan the object pointed to by @start for references to
1632 * other objects between @from_start and @from_end and copy
1633 * them to the gray_objects area.
1634 * Returns a pointer to the end of the object.
1637 scan_object (char *start, char* from_start, char* from_end)
1639 #include "sgen-scan-object.h"
1647 * Scan objects in the gray stack until the stack is empty. This should be called
1648 * frequently after each object is copied, to achieve better locality and cache
1652 drain_gray_stack (char *start_addr, char *end_addr)
1654 char *gray_start = gray_first;
1656 while (gray_start < gray_objects) {
1657 DEBUG (9, fprintf (gc_debug_file, "Precise gray object scan %p (%s)\n", gray_start, safe_name (gray_start)));
1658 gray_start = scan_object (gray_start, start_addr, end_addr);
1661 gray_first = gray_start;
1667 * Scan the valuetype pointed to by START, described by DESC for references to
1668 * other objects between @from_start and @from_end and copy them to the gray_objects area.
1669 * Returns a pointer to the end of the object.
1672 scan_vtype (char *start, mword desc, char* from_start, char* from_end)
1676 /* The descriptors include info about the MonoObject header as well */
1677 start -= sizeof (MonoObject);
1679 switch (desc & 0x7) {
1680 case DESC_TYPE_RUN_LENGTH:
1681 OBJ_RUN_LEN_FOREACH_PTR (desc,start);
1682 OBJ_RUN_LEN_SIZE (skip_size, desc, start);
1683 g_assert (skip_size);
1684 return start + skip_size;
1685 case DESC_TYPE_SMALL_BITMAP:
1686 OBJ_BITMAP_FOREACH_PTR (desc,start);
1687 OBJ_BITMAP_SIZE (skip_size, desc, start);
1688 return start + skip_size;
1689 case DESC_TYPE_LARGE_BITMAP:
1690 case DESC_TYPE_COMPLEX:
1692 g_assert_not_reached ();
1695 // The other descriptors can't happen with vtypes
1696 g_assert_not_reached ();
1703 * Addresses from start to end are already sorted. This function finds the object header
1704 * for each address and pins the object. The addresses must be inside the passed section.
1705 * Return the number of pinned objects.
1708 pin_objects_from_addresses (GCMemSection *section, void **start, void **end, void *start_nursery, void *end_nursery)
1713 void *last_obj = NULL;
1714 size_t last_obj_size = 0;
1717 void **definitely_pinned = start;
1718 while (start < end) {
1720 /* the range check should be reduntant */
1721 if (addr != last && addr >= start_nursery && addr < end_nursery) {
1722 DEBUG (5, fprintf (gc_debug_file, "Considering pinning addr %p\n", addr));
1723 /* multiple pointers to the same object */
1724 if (addr >= last_obj && (char*)addr < (char*)last_obj + last_obj_size) {
1728 idx = ((char*)addr - (char*)section->data) / SCAN_START_SIZE;
1729 search_start = (void*)section->scan_starts [idx];
1730 if (!search_start || search_start > addr) {
1733 search_start = section->scan_starts [idx];
1734 if (search_start && search_start <= addr)
1737 if (!search_start || search_start > addr)
1738 search_start = start_nursery;
1740 if (search_start < last_obj)
1741 search_start = (char*)last_obj + last_obj_size;
1742 /* now addr should be in an object a short distance from search_start
1743 * Note that search_start must point to zeroed mem or point to an object.
1746 if (!*(void**)search_start) {
1747 mword p = (mword)search_start;
1748 p += sizeof (gpointer);
1749 p += ALLOC_ALIGN - 1;
1750 p &= ~(ALLOC_ALIGN - 1);
1751 search_start = (void*)p;
1754 last_obj = search_start;
1755 last_obj_size = safe_object_get_size ((MonoObject*)search_start);
1756 last_obj_size += ALLOC_ALIGN - 1;
1757 last_obj_size &= ~(ALLOC_ALIGN - 1);
1758 DEBUG (8, fprintf (gc_debug_file, "Pinned try match %p (%s), size %zd\n", last_obj, safe_name (last_obj), last_obj_size));
1759 if (addr >= search_start && (char*)addr < (char*)last_obj + last_obj_size) {
1760 DEBUG (4, fprintf (gc_debug_file, "Pinned object %p, vtable %p (%s), count %d\n", search_start, *(void**)search_start, safe_name (search_start), count));
1761 pin_object (search_start);
1762 definitely_pinned [count] = search_start;
1766 /* skip to the next object */
1767 search_start = (void*)((char*)search_start + last_obj_size);
1768 } while (search_start <= addr);
1769 /* we either pinned the correct object or we ignored the addr because
1770 * it points to unused zeroed memory.
1776 //printf ("effective pinned: %d (at the end: %d)\n", count, (char*)end_nursery - (char*)last);
1780 static void** pin_queue;
1781 static int pin_queue_size = 0;
1782 static int next_pin_slot = 0;
1787 gap = (gap * 10) / 13;
1788 if (gap == 9 || gap == 10)
1797 compare_addr (const void *a, const void *b)
1799 return *(const void **)a - *(const void **)b;
1803 /* sort the addresses in array in increasing order */
1805 sort_addresses (void **array, int size)
1808 * qsort is slower as predicted.
1809 * qsort (array, size, sizeof (gpointer), compare_addr);
1816 gap = new_gap (gap);
1819 for (i = 0; i < end; i++) {
1821 if (array [i] > array [j]) {
1822 void* val = array [i];
1823 array [i] = array [j];
1828 if (gap == 1 && !swapped)
1833 static G_GNUC_UNUSED void
1834 print_nursery_gaps (void* start_nursery, void *end_nursery)
1837 gpointer first = start_nursery;
1839 for (i = 0; i < next_pin_slot; ++i) {
1840 next = pin_queue [i];
1841 fprintf (gc_debug_file, "Nursery range: %p-%p, size: %zd\n", first, next, (char*)next-(char*)first);
1845 fprintf (gc_debug_file, "Nursery range: %p-%p, size: %zd\n", first, next, (char*)next-(char*)first);
1848 /* reduce the info in the pin queue, removing duplicate pointers and sorting them */
1850 optimize_pin_queue (int start_slot)
1852 void **start, **cur, **end;
1853 /* sort and uniq pin_queue: we just sort and we let the rest discard multiple values */
1854 /* it may be better to keep ranges of pinned memory instead of individually pinning objects */
1855 DEBUG (5, fprintf (gc_debug_file, "Sorting pin queue, size: %d\n", next_pin_slot));
1856 if ((next_pin_slot - start_slot) > 1)
1857 sort_addresses (pin_queue + start_slot, next_pin_slot - start_slot);
1858 start = cur = pin_queue + start_slot;
1859 end = pin_queue + next_pin_slot;
1862 while (*start == *cur && cur < end)
1866 next_pin_slot = start - pin_queue;
1867 DEBUG (5, fprintf (gc_debug_file, "Pin queue reduced to size: %d\n", next_pin_slot));
1868 //DEBUG (6, print_nursery_gaps (start_nursery, end_nursery));
1873 realloc_pin_queue (void)
1875 int new_size = pin_queue_size? pin_queue_size + pin_queue_size/2: 1024;
1876 void **new_pin = get_internal_mem (sizeof (void*) * new_size);
1877 memcpy (new_pin, pin_queue, sizeof (void*) * next_pin_slot);
1878 free_internal_mem (pin_queue);
1879 pin_queue = new_pin;
1880 pin_queue_size = new_size;
1881 DEBUG (4, fprintf (gc_debug_file, "Reallocated pin queue to size: %d\n", new_size));
1885 * Scan the memory between start and end and queue values which could be pointers
1886 * to the area between start_nursery and end_nursery for later consideration.
1887 * Typically used for thread stacks.
1890 conservatively_pin_objects_from (void **start, void **end, void *start_nursery, void *end_nursery)
1893 while (start < end) {
1894 if (*start >= start_nursery && *start < end_nursery) {
1896 * *start can point to the middle of an object
1897 * note: should we handle pointing at the end of an object?
1898 * pinning in C# code disallows pointing at the end of an object
1899 * but there is some small chance that an optimizing C compiler
1900 * may keep the only reference to an object by pointing
1901 * at the end of it. We ignore this small chance for now.
1902 * Pointers to the end of an object are indistinguishable
1903 * from pointers to the start of the next object in memory
1904 * so if we allow that we'd need to pin two objects...
1905 * We queue the pointer in an array, the
1906 * array will then be sorted and uniqued. This way
1907 * we can coalesce several pinning pointers and it should
1908 * be faster since we'd do a memory scan with increasing
1909 * addresses. Note: we can align the address to the allocation
1910 * alignment, so the unique process is more effective.
1912 mword addr = (mword)*start;
1913 addr &= ~(ALLOC_ALIGN - 1);
1914 if (next_pin_slot >= pin_queue_size)
1915 realloc_pin_queue ();
1916 pin_queue [next_pin_slot++] = (void*)addr;
1917 DEBUG (6, if (count) fprintf (gc_debug_file, "Pinning address %p\n", (void*)addr));
1922 DEBUG (7, if (count) fprintf (gc_debug_file, "found %d potential pinned heap pointers\n", count));
1924 #ifdef HAVE_VALGRIND_MEMCHECK_H
1926 * The pinning addresses might come from undefined memory, this is normal. Since they
1927 * are used in lots of functions, we make the memory defined here instead of having
1928 * to add a supression for those functions.
1930 VALGRIND_MAKE_MEM_DEFINED (pin_queue, next_pin_slot * sizeof (pin_queue [0]));
1935 * If generation is 0, just mark objects in the nursery, the others we don't care,
1936 * since they are not going to move anyway.
1937 * There are different areas that are scanned for pinned pointers:
1938 * *) the thread stacks (when jit support is ready only the unmanaged frames)
1939 * *) the pinned handle table
1940 * *) the pinned roots
1942 * Note: when we'll use a write barrier for old to new gen references, we need to
1943 * keep track of old gen objects that point to pinned new gen objects because in that
1944 * case the referenced object will be moved maybe at the next collection, but there
1945 * is no write in the old generation area where the pinned object is referenced
1946 * and we may not consider it as reachable.
1948 static G_GNUC_UNUSED void
1949 mark_pinned_objects (int generation)
1954 * Debugging function: find in the conservative roots where @obj is being pinned.
1956 static G_GNUC_UNUSED void
1957 find_pinning_reference (char *obj, size_t size)
1961 char *endobj = obj + size;
1962 for (i = 0; i < roots_hash_size [0]; ++i) {
1963 for (root = roots_hash [0][i]; root; root = root->next) {
1964 /* if desc is non-null it has precise info */
1965 if (!root->root_desc) {
1966 char ** start = (char**)root->start_root;
1967 while (start < (char**)root->end_root) {
1968 if (*start >= obj && *start < endobj) {
1969 DEBUG (0, fprintf (gc_debug_file, "Object %p referenced in pinned roots %p-%p (at %p in record %p)\n", obj, root->start_root, root->end_root, start, root));
1976 find_pinning_ref_from_thread (obj, size);
1980 * The first thing we do in a collection is to identify pinned objects.
1981 * This function considers all the areas of memory that need to be
1982 * conservatively scanned.
1985 pin_from_roots (void *start_nursery, void *end_nursery)
1989 DEBUG (2, fprintf (gc_debug_file, "Scanning pinned roots (%d bytes, %d/%d entries)\n", (int)roots_size, num_roots_entries [ROOT_TYPE_NORMAL], num_roots_entries [ROOT_TYPE_PINNED]));
1990 /* objects pinned from the API are inside these roots */
1991 for (i = 0; i < roots_hash_size [ROOT_TYPE_PINNED]; ++i) {
1992 for (root = roots_hash [ROOT_TYPE_PINNED][i]; root; root = root->next) {
1993 DEBUG (6, fprintf (gc_debug_file, "Pinned roots %p-%p\n", root->start_root, root->end_root));
1994 conservatively_pin_objects_from ((void**)root->start_root, (void**)root->end_root, start_nursery, end_nursery);
1997 /* now deal with the thread stacks
1998 * in the future we should be able to conservatively scan only:
1999 * *) the cpu registers
2000 * *) the unmanaged stack frames
2001 * *) the _last_ managed stack frame
2002 * *) pointers slots in managed frames
2004 scan_thread_data (start_nursery, end_nursery, FALSE);
2007 /* Copy function called from user defined mark functions */
2008 static char *user_copy_n_start;
2009 static char *user_copy_n_end;
2012 user_copy (void *addr)
2015 return copy_object (addr, user_copy_n_start, user_copy_n_end);
2021 * The memory area from start_root to end_root contains pointers to objects.
2022 * Their position is precisely described by @desc (this means that the pointer
2023 * can be either NULL or the pointer to the start of an object).
2024 * This functions copies them to to_space updates them.
2027 precisely_scan_objects_from (void** start_root, void** end_root, char* n_start, char *n_end, mword desc)
2029 switch (desc & ROOT_DESC_TYPE_MASK) {
2030 case ROOT_DESC_BITMAP:
2031 desc >>= ROOT_DESC_TYPE_SHIFT;
2033 if ((desc & 1) && *start_root) {
2034 *start_root = copy_object (*start_root, n_start, n_end);
2035 DEBUG (9, fprintf (gc_debug_file, "Overwrote root at %p with %p\n", start_root, *start_root));
2036 drain_gray_stack (n_start, n_end);
2042 case ROOT_DESC_COMPLEX: {
2043 gsize *bitmap_data = complex_descriptors + (desc >> ROOT_DESC_TYPE_SHIFT);
2044 int bwords = (*bitmap_data) - 1;
2045 void **start_run = start_root;
2047 while (bwords-- > 0) {
2048 gsize bmap = *bitmap_data++;
2049 void **objptr = start_run;
2051 if ((bmap & 1) && *objptr) {
2052 *objptr = copy_object (*objptr, n_start, n_end);
2053 DEBUG (9, fprintf (gc_debug_file, "Overwrote root at %p with %p\n", objptr, *objptr));
2054 drain_gray_stack (n_start, n_end);
2059 start_run += GC_BITS_PER_WORD;
2063 case ROOT_DESC_USER: {
2064 MonoGCMarkFunc marker = user_descriptors [desc >> ROOT_DESC_TYPE_SHIFT];
2066 user_copy_n_start = n_start;
2067 user_copy_n_end = n_end;
2068 marker (start_root, user_copy);
2071 case ROOT_DESC_RUN_LEN:
2072 g_assert_not_reached ();
2074 g_assert_not_reached ();
2079 alloc_fragment (void)
2081 Fragment *frag = fragment_freelist;
2083 fragment_freelist = frag->next;
2087 frag = get_internal_mem (sizeof (Fragment));
2093 * Allocate and setup the data structures needed to be able to allocate objects
2094 * in the nursery. The nursery is stored in nursery_section.
2097 alloc_nursery (void)
2099 GCMemSection *section;
2105 if (nursery_section)
2107 DEBUG (2, fprintf (gc_debug_file, "Allocating nursery size: %zd\n", nursery_size));
2108 /* later we will alloc a larger area for the nursery but only activate
2109 * what we need. The rest will be used as expansion if we have too many pinned
2110 * objects in the existing nursery.
2112 /* FIXME: handle OOM */
2113 section = get_internal_mem (sizeof (GCMemSection));
2115 #ifdef ALIGN_NURSERY
2116 /* Allocate twice the memory to be able to put the nursery at an aligned address */
2117 g_assert (nursery_size == DEFAULT_NURSERY_SIZE);
2119 alloc_size = nursery_size * 2;
2120 data = get_os_memory (alloc_size, TRUE);
2121 nursery_start = (void*)(((mword)data + (1 << DEFAULT_NURSERY_BITS) - 1) & ~((1 << DEFAULT_NURSERY_BITS) - 1));
2122 g_assert ((char*)nursery_start + nursery_size <= ((char*)data + alloc_size));
2123 /* FIXME: Use the remaining size for something else, if it is big enough */
2125 alloc_size = nursery_size;
2126 data = get_os_memory (alloc_size, TRUE);
2127 nursery_start = data;
2129 nursery_real_end = nursery_start + nursery_size;
2130 UPDATE_HEAP_BOUNDARIES (nursery_start, nursery_real_end);
2131 nursery_next = nursery_start;
2132 total_alloc += alloc_size;
2133 DEBUG (4, fprintf (gc_debug_file, "Expanding nursery size (%p-%p): %zd, total: %zd\n", data, data + alloc_size, nursery_size, total_alloc));
2134 section->data = section->next_data = data;
2135 section->size = alloc_size;
2136 section->end_data = nursery_real_end;
2137 scan_starts = alloc_size / SCAN_START_SIZE;
2138 section->scan_starts = get_internal_mem (sizeof (char*) * scan_starts);
2139 section->num_scan_start = scan_starts;
2140 section->role = MEMORY_ROLE_GEN0;
2142 /* add to the section list */
2143 section->next = section_list;
2144 section_list = section;
2146 nursery_section = section;
2148 /* Setup the single first large fragment */
2149 frag = alloc_fragment ();
2150 frag->fragment_start = nursery_start;
2151 frag->fragment_limit = nursery_start;
2152 frag->fragment_end = nursery_real_end;
2153 nursery_frag_real_end = nursery_real_end;
2154 /* FIXME: frag here is lost */
2158 scan_finalizer_entries (FinalizeEntry *list, char *start, char *end) {
2161 for (fin = list; fin; fin = fin->next) {
2164 DEBUG (5, fprintf (gc_debug_file, "Scan of fin ready object: %p (%s)\n", fin->object, safe_name (fin->object)));
2165 fin->object = copy_object (fin->object, start, end);
2170 * Update roots in the old generation. Since we currently don't have the
2171 * info from the write barriers, we just scan all the objects.
2173 static G_GNUC_UNUSED void
2174 scan_old_generation (char *start, char* end)
2176 GCMemSection *section;
2177 LOSObject *big_object;
2180 for (section = section_list; section; section = section->next) {
2181 if (section == nursery_section)
2183 DEBUG (2, fprintf (gc_debug_file, "Scan of old section: %p-%p, size: %d\n", section->data, section->next_data, (int)(section->next_data - section->data)));
2184 /* we have to deal with zeroed holes in old generation (truncated strings ...) */
2186 while (p < section->next_data) {
2191 DEBUG (8, fprintf (gc_debug_file, "Precise old object scan of %p (%s)\n", p, safe_name (p)));
2192 p = scan_object (p, start, end);
2195 /* scan the old object space, too */
2196 for (big_object = los_object_list; big_object; big_object = big_object->next) {
2197 DEBUG (5, fprintf (gc_debug_file, "Scan of big object: %p (%s), size: %zd\n", big_object->data, safe_name (big_object->data), big_object->size));
2198 scan_object (big_object->data, start, end);
2200 /* scan the list of objects ready for finalization */
2201 scan_finalizer_entries (fin_ready_list, start, end);
2202 scan_finalizer_entries (critical_fin_list, start, end);
2205 static mword fragment_total = 0;
2207 * We found a fragment of free memory in the nursery: memzero it and if
2208 * it is big enough, add it to the list of fragments that can be used for
2212 add_nursery_frag (size_t frag_size, char* frag_start, char* frag_end)
2215 DEBUG (4, fprintf (gc_debug_file, "Found empty fragment: %p-%p, size: %zd\n", frag_start, frag_end, frag_size));
2216 /* memsetting just the first chunk start is bound to provide better cache locality */
2217 if (nursery_clear_policy == CLEAR_AT_GC)
2218 memset (frag_start, 0, frag_size);
2219 /* Not worth dealing with smaller fragments: need to tune */
2220 if (frag_size >= FRAGMENT_MIN_SIZE) {
2221 fragment = alloc_fragment ();
2222 fragment->fragment_start = frag_start;
2223 fragment->fragment_limit = frag_start;
2224 fragment->fragment_end = frag_end;
2225 fragment->next = nursery_fragments;
2226 nursery_fragments = fragment;
2227 fragment_total += frag_size;
2229 /* Clear unused fragments, pinning depends on this */
2230 memset (frag_start, 0, frag_size);
2235 scan_needed_big_objects (char *start_addr, char *end_addr)
2237 LOSObject *big_object;
2239 for (big_object = los_object_list; big_object; big_object = big_object->next) {
2240 if (!big_object->scanned && object_is_pinned (big_object->data)) {
2241 DEBUG (5, fprintf (gc_debug_file, "Scan of big object: %p (%s), size: %zd\n", big_object->data, safe_name (big_object->data), big_object->size));
2242 scan_object (big_object->data, start_addr, end_addr);
2243 big_object->scanned = TRUE;
2251 generation_name (int generation)
2253 switch (generation) {
2254 case GENERATION_NURSERY: return "nursery";
2255 case GENERATION_OLD: return "old";
2256 default: g_assert_not_reached ();
2260 static DisappearingLinkHashTable*
2261 get_dislink_hash_table (int generation)
2263 switch (generation) {
2264 case GENERATION_NURSERY: return &minor_disappearing_link_hash;
2265 case GENERATION_OLD: return &major_disappearing_link_hash;
2266 default: g_assert_not_reached ();
2270 static FinalizeEntryHashTable*
2271 get_finalize_entry_hash_table (int generation)
2273 switch (generation) {
2274 case GENERATION_NURSERY: return &minor_finalizable_hash;
2275 case GENERATION_OLD: return &major_finalizable_hash;
2276 default: g_assert_not_reached ();
2281 finish_gray_stack (char *start_addr, char *end_addr, int generation)
2285 int fin_ready, bigo_scanned_num;
2289 * We copied all the reachable objects. Now it's the time to copy
2290 * the objects that were not referenced by the roots, but by the copied objects.
2291 * we built a stack of objects pointed to by gray_start: they are
2292 * additional roots and we may add more items as we go.
2293 * We loop until gray_start == gray_objects which means no more objects have
2294 * been added. Note this is iterative: no recursion is involved.
2295 * We need to walk the LO list as well in search of marked big objects
2296 * (use a flag since this is needed only on major collections). We need to loop
2297 * here as well, so keep a counter of marked LO (increasing it in copy_object).
2298 * To achieve better cache locality and cache usage, we drain the gray stack
2299 * frequently, after each object is copied, and just finish the work here.
2301 gray_start = gray_first;
2302 while (gray_start < gray_objects) {
2303 DEBUG (9, fprintf (gc_debug_file, "Precise gray object scan %p (%s)\n", gray_start, safe_name (gray_start)));
2304 gray_start = scan_object (gray_start, start_addr, end_addr);
2307 //scan_old_generation (start_addr, end_addr);
2308 DEBUG (2, fprintf (gc_debug_file, "%s generation done\n", generation_name (generation)));
2309 /* walk the finalization queue and move also the objects that need to be
2310 * finalized: use the finalized objects as new roots so the objects they depend
2311 * on are also not reclaimed. As with the roots above, only objects in the nursery
2312 * are marked/copied.
2313 * We need a loop here, since objects ready for finalizers may reference other objects
2314 * that are fin-ready. Speedup with a flag?
2317 fin_ready = num_ready_finalizers;
2318 finalize_in_range (start_addr, end_addr, generation);
2319 if (generation == GENERATION_OLD)
2320 finalize_in_range (nursery_start, nursery_real_end, GENERATION_NURSERY);
2321 bigo_scanned_num = scan_needed_big_objects (start_addr, end_addr);
2323 /* drain the new stack that might have been created */
2324 DEBUG (6, fprintf (gc_debug_file, "Precise scan of gray area post fin: %p-%p, size: %d\n", gray_start, gray_objects, (int)(gray_objects - gray_start)));
2325 while (gray_start < gray_objects) {
2326 DEBUG (9, fprintf (gc_debug_file, "Precise gray object scan %p (%s)\n", gray_start, safe_name (gray_start)));
2327 gray_start = scan_object (gray_start, start_addr, end_addr);
2329 } while (fin_ready != num_ready_finalizers || bigo_scanned_num);
2331 DEBUG (2, fprintf (gc_debug_file, "Finalize queue handling scan for %s generation: %d usecs\n", generation_name (generation), TV_ELAPSED (atv, btv)));
2334 * handle disappearing links
2335 * Note we do this after checking the finalization queue because if an object
2336 * survives (at least long enough to be finalized) we don't clear the link.
2337 * This also deals with a possible issue with the monitor reclamation: with the Boehm
2338 * GC a finalized object my lose the monitor because it is cleared before the finalizer is
2341 g_assert (gray_start == gray_objects);
2343 null_link_in_range (start_addr, end_addr, generation);
2344 if (generation == GENERATION_OLD)
2345 null_link_in_range (start_addr, end_addr, GENERATION_NURSERY);
2346 if (gray_start == gray_objects)
2348 while (gray_start < gray_objects) {
2349 DEBUG (9, fprintf (gc_debug_file, "Precise gray object scan %p (%s)\n", gray_start, safe_name (gray_start)));
2350 gray_start = scan_object (gray_start, start_addr, end_addr);
2354 g_assert (gray_start == gray_objects);
2355 DEBUG (2, fprintf (gc_debug_file, "Copied from %s to old space: %d bytes (%p-%p)\n", generation_name (generation), (int)(gray_objects - to_space), to_space, gray_objects));
2356 to_space = gray_start;
2357 to_space_section->next_data = to_space;
2360 static int last_num_pinned = 0;
2363 build_nursery_fragments (int start_pin, int end_pin)
2365 char *frag_start, *frag_end;
2369 /* FIXME: handle non-NULL fragment_freelist */
2370 fragment_freelist = nursery_fragments;
2371 nursery_fragments = NULL;
2372 frag_start = nursery_start;
2374 /* clear scan starts */
2375 memset (nursery_section->scan_starts, 0, nursery_section->num_scan_start * sizeof (gpointer));
2376 for (i = start_pin; i < end_pin; ++i) {
2377 frag_end = pin_queue [i];
2378 /* remove the pin bit from pinned objects */
2379 unpin_object (frag_end);
2380 nursery_section->scan_starts [((char*)frag_end - (char*)nursery_section->data)/SCAN_START_SIZE] = frag_end;
2381 frag_size = frag_end - frag_start;
2383 add_nursery_frag (frag_size, frag_start, frag_end);
2384 frag_size = safe_object_get_size ((MonoObject*)pin_queue [i]);
2385 frag_size += ALLOC_ALIGN - 1;
2386 frag_size &= ~(ALLOC_ALIGN - 1);
2387 frag_start = (char*)pin_queue [i] + frag_size;
2389 * pin_queue [i] might point to a half-constructed string or vector whose
2390 * length field is not set. In that case, frag_start points inside the
2391 * (zero initialized) object. Find the end of the object by scanning forward.
2394 if (is_maybe_half_constructed (pin_queue [i])) {
2397 /* This is also hit for zero length arrays/strings */
2399 /* Find the end of the TLAB which contained this allocation */
2400 tlab_end = find_tlab_next_from_address (pin_queue [i]);
2403 while ((frag_start < tlab_end) && *(mword*)frag_start == 0)
2404 frag_start += sizeof (mword);
2407 * FIXME: The object is either not allocated in a TLAB, or it isn't a
2408 * half constructed object.
2413 nursery_last_pinned_end = frag_start;
2414 frag_end = nursery_real_end;
2415 frag_size = frag_end - frag_start;
2417 add_nursery_frag (frag_size, frag_start, frag_end);
2418 if (!nursery_fragments) {
2419 DEBUG (1, fprintf (gc_debug_file, "Nursery fully pinned (%d)\n", end_pin - start_pin));
2420 for (i = start_pin; i < end_pin; ++i) {
2421 DEBUG (3, fprintf (gc_debug_file, "Bastard pinning obj %p (%s), size: %d\n", pin_queue [i], safe_name (pin_queue [i]), safe_object_get_size (pin_queue [i])));
2426 nursery_next = nursery_frag_real_end = NULL;
2428 /* Clear TLABs for all threads */
2432 /* FIXME: later reduce code duplication here with the above
2433 * We don't keep track of section fragments for non-nursery sections yet, so
2437 build_section_fragments (GCMemSection *section)
2440 char *frag_start, *frag_end;
2443 /* clear scan starts */
2444 memset (section->scan_starts, 0, section->num_scan_start * sizeof (gpointer));
2445 frag_start = section->data;
2446 section->next_data = section->data;
2447 for (i = section->pin_queue_start; i < section->pin_queue_end; ++i) {
2448 frag_end = pin_queue [i];
2449 /* remove the pin bit from pinned objects */
2450 unpin_object (frag_end);
2451 if (frag_end >= section->data + section->size) {
2452 frag_end = section->data + section->size;
2454 section->scan_starts [((char*)frag_end - (char*)section->data)/SCAN_START_SIZE] = frag_end;
2456 frag_size = frag_end - frag_start;
2458 memset (frag_start, 0, frag_size);
2459 frag_size = safe_object_get_size ((MonoObject*)pin_queue [i]);
2460 frag_size += ALLOC_ALIGN - 1;
2461 frag_size &= ~(ALLOC_ALIGN - 1);
2462 frag_start = (char*)pin_queue [i] + frag_size;
2463 section->next_data = MAX (section->next_data, frag_start);
2465 frag_end = section->end_data;
2466 frag_size = frag_end - frag_start;
2468 memset (frag_start, 0, frag_size);
2472 scan_from_registered_roots (char *addr_start, char *addr_end, int root_type)
2476 for (i = 0; i < roots_hash_size [root_type]; ++i) {
2477 for (root = roots_hash [root_type][i]; root; root = root->next) {
2478 DEBUG (6, fprintf (gc_debug_file, "Precise root scan %p-%p (desc: %p)\n", root->start_root, root->end_root, (void*)root->root_desc));
2479 precisely_scan_objects_from ((void**)root->start_root, (void**)root->end_root, addr_start, addr_end, root->root_desc);
2485 * Collect objects in the nursery. Returns whether to trigger a major
2489 collect_nursery (size_t requested_size)
2491 GCMemSection *section;
2492 size_t max_garbage_amount;
2494 char *orig_nursery_next;
2496 gboolean invoke_major_gc = FALSE;
2497 TV_DECLARE (all_atv);
2498 TV_DECLARE (all_btv);
2503 orig_nursery_next = nursery_next;
2504 nursery_next = MAX (nursery_next, nursery_last_pinned_end);
2505 /* FIXME: optimize later to use the higher address where an object can be present */
2506 nursery_next = MAX (nursery_next, nursery_real_end);
2508 if (consistency_check_at_minor_collection)
2509 check_consistency ();
2511 DEBUG (1, fprintf (gc_debug_file, "Start nursery collection %d %p-%p, size: %d\n", num_minor_gcs, nursery_start, nursery_next, (int)(nursery_next - nursery_start)));
2512 max_garbage_amount = nursery_next - nursery_start;
2514 /* Clear all remaining nursery fragments, pinning depends on this */
2515 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION) {
2516 g_assert (orig_nursery_next <= nursery_frag_real_end);
2517 memset (orig_nursery_next, 0, nursery_frag_real_end - orig_nursery_next);
2518 for (frag = nursery_fragments; frag; frag = frag->next) {
2519 memset (frag->fragment_start, 0, frag->fragment_end - frag->fragment_start);
2524 * not enough room in the old generation to store all the possible data from
2525 * the nursery in a single continuous space.
2526 * We reset to_space if we allocated objects in degraded mode.
2528 if (to_space_section)
2529 to_space = gray_objects = gray_first = to_space_section->next_data;
2530 if ((to_space_end - to_space) < max_garbage_amount) {
2531 section = alloc_section (nursery_section->size * 4);
2532 g_assert (nursery_section->size >= max_garbage_amount);
2533 to_space = gray_objects = gray_first = section->next_data;
2534 to_space_end = section->end_data;
2535 to_space_section = section;
2536 invoke_major_gc = TRUE;
2538 DEBUG (2, fprintf (gc_debug_file, "To space setup: %p-%p in section %p\n", to_space, to_space_end, to_space_section));
2539 nursery_section->next_data = nursery_next;
2542 mono_stats.minor_gc_count ++;
2543 /* world must be stopped already */
2544 TV_GETTIME (all_atv);
2546 /* pin from pinned handles */
2547 pin_from_roots (nursery_start, nursery_next);
2548 /* identify pinned objects */
2549 optimize_pin_queue (0);
2550 next_pin_slot = pin_objects_from_addresses (nursery_section, pin_queue, pin_queue + next_pin_slot, nursery_start, nursery_next);
2552 DEBUG (2, fprintf (gc_debug_file, "Finding pinned pointers: %d in %d usecs\n", next_pin_slot, TV_ELAPSED (atv, btv)));
2553 DEBUG (4, fprintf (gc_debug_file, "Start scan with %d pinned objects\n", next_pin_slot));
2556 * walk all the roots and copy the young objects to the old generation,
2557 * starting from to_space
2560 scan_from_remsets (nursery_start, nursery_next);
2561 /* we don't have complete write barrier yet, so we scan all the old generation sections */
2563 DEBUG (2, fprintf (gc_debug_file, "Old generation scan: %d usecs\n", TV_ELAPSED (btv, atv)));
2565 /* the pinned objects are roots */
2566 for (i = 0; i < next_pin_slot; ++i) {
2567 DEBUG (6, fprintf (gc_debug_file, "Precise object scan %d of pinned %p (%s)\n", i, pin_queue [i], safe_name (pin_queue [i])));
2568 scan_object (pin_queue [i], nursery_start, nursery_next);
2570 /* registered roots, this includes static fields */
2571 scan_from_registered_roots (nursery_start, nursery_next, ROOT_TYPE_NORMAL);
2572 scan_from_registered_roots (nursery_start, nursery_next, ROOT_TYPE_WBARRIER);
2573 scan_thread_data (nursery_start, nursery_next, TRUE);
2574 /* alloc_pinned objects */
2575 scan_from_pinned_objects (nursery_start, nursery_next);
2577 DEBUG (2, fprintf (gc_debug_file, "Root scan: %d usecs\n", TV_ELAPSED (atv, btv)));
2579 finish_gray_stack (nursery_start, nursery_next, GENERATION_NURSERY);
2581 /* walk the pin_queue, build up the fragment list of free memory, unmark
2582 * pinned objects as we go, memzero() the empty fragments so they are ready for the
2585 build_nursery_fragments (0, next_pin_slot);
2587 DEBUG (2, fprintf (gc_debug_file, "Fragment creation: %d usecs, %zd bytes available\n", TV_ELAPSED (btv, atv), fragment_total));
2589 TV_GETTIME (all_btv);
2590 mono_stats.minor_gc_time_usecs += TV_ELAPSED (all_atv, all_btv);
2592 /* prepare the pin queue for the next collection */
2593 last_num_pinned = next_pin_slot;
2595 if (fin_ready_list || critical_fin_list) {
2596 DEBUG (4, fprintf (gc_debug_file, "Finalizer-thread wakeup: ready %d\n", num_ready_finalizers));
2597 mono_gc_finalize_notify ();
2600 return invoke_major_gc;
2604 major_collection (void)
2606 GCMemSection *section, *prev_section;
2607 LOSObject *bigobj, *prevbo;
2612 TV_DECLARE (all_atv);
2613 TV_DECLARE (all_btv);
2616 /* FIXME: only use these values for the precise scan
2617 * note that to_space pointers should be excluded anyway...
2619 char *heap_start = NULL;
2620 char *heap_end = (char*)-1;
2621 size_t copy_space_required = 0;
2624 DEBUG (1, fprintf (gc_debug_file, "Start major collection %d\n", num_major_gcs));
2626 mono_stats.major_gc_count ++;
2628 /* Clear all remaining nursery fragments, pinning depends on this */
2629 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION) {
2630 g_assert (nursery_next <= nursery_frag_real_end);
2631 memset (nursery_next, 0, nursery_frag_real_end - nursery_next);
2632 for (frag = nursery_fragments; frag; frag = frag->next) {
2633 memset (frag->fragment_start, 0, frag->fragment_end - frag->fragment_start);
2638 * FIXME: implement Mark/Compact
2639 * Until that is done, we can just apply mostly the same alg as for the nursery:
2640 * this means we need a big section to potentially copy all the other sections, so
2641 * it is not ideal specially with large heaps.
2643 if (g_getenv ("MONO_GC_NO_MAJOR")) {
2644 collect_nursery (0);
2647 TV_GETTIME (all_atv);
2648 /* FIXME: make sure the nursery next_data ptr is updated */
2649 nursery_section->next_data = nursery_real_end;
2650 /* we should also coalesce scanning from sections close to each other
2651 * and deal with pointers outside of the sections later.
2653 /* The remsets are not useful for a major collection */
2655 /* world must be stopped already */
2657 DEBUG (6, fprintf (gc_debug_file, "Pinning from sections\n"));
2658 for (section = section_list; section; section = section->next) {
2659 section->pin_queue_start = count = section->pin_queue_end = next_pin_slot;
2660 pin_from_roots (section->data, section->next_data);
2661 if (count != next_pin_slot) {
2663 optimize_pin_queue (count);
2664 DEBUG (6, fprintf (gc_debug_file, "Found %d pinning addresses in section %p (%d-%d)\n", next_pin_slot - count, section, count, next_pin_slot));
2665 reduced_to = pin_objects_from_addresses (section, pin_queue + count, pin_queue + next_pin_slot, section->data, section->next_data);
2666 section->pin_queue_end = next_pin_slot = count + reduced_to;
2668 copy_space_required += (char*)section->next_data - (char*)section->data;
2670 /* identify possible pointers to the insize of large objects */
2671 DEBUG (6, fprintf (gc_debug_file, "Pinning from large objects\n"));
2672 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
2673 count = next_pin_slot;
2674 pin_from_roots (bigobj->data, (char*)bigobj->data + bigobj->size);
2675 /* FIXME: this is only valid until we don't optimize the pin queue midway */
2676 if (next_pin_slot != count) {
2677 next_pin_slot = count;
2678 pin_object (bigobj->data);
2679 DEBUG (6, fprintf (gc_debug_file, "Marked large object %p (%s) size: %zd from roots\n", bigobj->data, safe_name (bigobj->data), bigobj->size));
2682 /* look for pinned addresses for pinned-alloc objects */
2683 DEBUG (6, fprintf (gc_debug_file, "Pinning from pinned-alloc objects\n"));
2684 for (chunk = pinned_chunk_list; chunk; chunk = chunk->next) {
2685 count = next_pin_slot;
2686 pin_from_roots (chunk->start_data, (char*)chunk + chunk->num_pages * FREELIST_PAGESIZE);
2687 /* FIXME: this is only valid until we don't optimize the pin queue midway */
2688 if (next_pin_slot != count) {
2689 mark_pinned_from_addresses (chunk, pin_queue + count, pin_queue + next_pin_slot);
2690 next_pin_slot = count;
2695 DEBUG (2, fprintf (gc_debug_file, "Finding pinned pointers: %d in %d usecs\n", next_pin_slot, TV_ELAPSED (atv, btv)));
2696 DEBUG (4, fprintf (gc_debug_file, "Start scan with %d pinned objects\n", next_pin_slot));
2698 /* allocate the big to space */
2699 DEBUG (4, fprintf (gc_debug_file, "Allocate tospace for size: %zd\n", copy_space_required));
2700 section = alloc_section (copy_space_required);
2701 to_space = gray_objects = gray_first = section->next_data;
2702 to_space_end = section->end_data;
2703 to_space_section = section;
2705 /* the old generation doesn't need to be scanned (no remembered sets or card
2706 * table needed either): the only objects that must survive are those pinned and
2707 * those referenced by the precise roots.
2708 * mark any section without pinned objects, so we can free it since we will be able to
2709 * move all the objects.
2711 /* the pinned objects are roots (big objects are included in this list, too) */
2712 for (i = 0; i < next_pin_slot; ++i) {
2713 DEBUG (6, fprintf (gc_debug_file, "Precise object scan %d of pinned %p (%s)\n", i, pin_queue [i], safe_name (pin_queue [i])));
2714 scan_object (pin_queue [i], heap_start, heap_end);
2716 /* registered roots, this includes static fields */
2717 scan_from_registered_roots (heap_start, heap_end, ROOT_TYPE_NORMAL);
2718 scan_from_registered_roots (heap_start, heap_end, ROOT_TYPE_WBARRIER);
2720 scan_thread_data (heap_start, heap_end, TRUE);
2721 /* alloc_pinned objects */
2722 scan_from_pinned_objects (heap_start, heap_end);
2723 /* scan the list of objects ready for finalization */
2724 scan_finalizer_entries (fin_ready_list, heap_start, heap_end);
2725 scan_finalizer_entries (critical_fin_list, heap_start, heap_end);
2727 DEBUG (2, fprintf (gc_debug_file, "Root scan: %d usecs\n", TV_ELAPSED (btv, atv)));
2729 /* we need to go over the big object list to see if any was marked and scan it
2730 * And we need to make this in a loop, considering that objects referenced by finalizable
2731 * objects could reference big objects (this happens in finish_gray_stack ())
2733 scan_needed_big_objects (heap_start, heap_end);
2734 /* all the objects in the heap */
2735 finish_gray_stack (heap_start, heap_end, GENERATION_OLD);
2737 /* sweep the big objects list */
2739 for (bigobj = los_object_list; bigobj;) {
2740 if (object_is_pinned (bigobj->data)) {
2741 unpin_object (bigobj->data);
2742 bigobj->scanned = FALSE;
2745 /* not referenced anywhere, so we can free it */
2747 prevbo->next = bigobj->next;
2749 los_object_list = bigobj->next;
2751 bigobj = bigobj->next;
2752 free_large_object (to_free);
2756 bigobj = bigobj->next;
2758 /* unpin objects from the pinned chunks and free the unmarked ones */
2759 sweep_pinned_objects ();
2761 /* free the unused sections */
2762 prev_section = NULL;
2763 for (section = section_list; section;) {
2764 /* to_space doesn't need handling here and the nursery is special */
2765 if (section == to_space_section || section == nursery_section) {
2766 prev_section = section;
2767 section = section->next;
2770 /* no pinning object, so the section is free */
2771 if (section->pin_queue_start == section->pin_queue_end) {
2772 GCMemSection *to_free;
2774 prev_section->next = section->next;
2776 section_list = section->next;
2778 section = section->next;
2779 free_mem_section (to_free);
2782 DEBUG (6, fprintf (gc_debug_file, "Section %p has still pinned objects (%d)\n", section, section->pin_queue_end - section->pin_queue_start));
2783 build_section_fragments (section);
2785 prev_section = section;
2786 section = section->next;
2789 /* walk the pin_queue, build up the fragment list of free memory, unmark
2790 * pinned objects as we go, memzero() the empty fragments so they are ready for the
2793 build_nursery_fragments (nursery_section->pin_queue_start, nursery_section->pin_queue_end);
2795 TV_GETTIME (all_btv);
2796 mono_stats.major_gc_time_usecs += TV_ELAPSED (all_atv, all_btv);
2797 /* prepare the pin queue for the next collection */
2799 if (fin_ready_list || critical_fin_list) {
2800 DEBUG (4, fprintf (gc_debug_file, "Finalizer-thread wakeup: ready %d\n", num_ready_finalizers));
2801 mono_gc_finalize_notify ();
2806 * Allocate a new section of memory to be used as old generation.
2808 static GCMemSection*
2809 alloc_section (size_t size)
2811 GCMemSection *section;
2814 size_t new_size = next_section_size;
2816 if (size > next_section_size) {
2818 new_size += pagesize - 1;
2819 new_size &= ~(pagesize - 1);
2821 section_size_used++;
2822 if (section_size_used > 3) {
2823 section_size_used = 0;
2824 next_section_size *= 2;
2825 if (next_section_size > max_section_size)
2826 next_section_size = max_section_size;
2828 section = get_internal_mem (sizeof (GCMemSection));
2829 data = get_os_memory (new_size, TRUE);
2830 section->data = section->next_data = data;
2831 section->size = new_size;
2832 section->end_data = data + new_size;
2833 UPDATE_HEAP_BOUNDARIES (data, section->end_data);
2834 total_alloc += new_size;
2835 DEBUG (2, fprintf (gc_debug_file, "Expanding heap size: %zd (%p-%p), total: %zd\n", new_size, data, data + new_size, total_alloc));
2836 section->data = data;
2837 section->size = new_size;
2838 scan_starts = new_size / SCAN_START_SIZE;
2839 section->scan_starts = get_internal_mem (sizeof (char*) * scan_starts);
2840 section->num_scan_start = scan_starts;
2841 section->role = MEMORY_ROLE_GEN1;
2843 /* add to the section list */
2844 section->next = section_list;
2845 section_list = section;
2851 free_mem_section (GCMemSection *section)
2853 char *data = section->data;
2854 size_t size = section->size;
2855 DEBUG (2, fprintf (gc_debug_file, "Freed section %p, size %zd\n", data, size));
2856 free_os_memory (data, size);
2857 free_internal_mem (section);
2858 total_alloc -= size;
2862 * When deciding if it's better to collect or to expand, keep track
2863 * of how much garbage was reclaimed with the last collection: if it's too
2865 * This is called when we could not allocate a small object.
2867 static void __attribute__((noinline))
2868 minor_collect_or_expand_inner (size_t size)
2870 int do_minor_collection = 1;
2872 if (!nursery_section) {
2876 if (do_minor_collection) {
2878 if (collect_nursery (size))
2879 major_collection ();
2880 DEBUG (2, fprintf (gc_debug_file, "Heap size: %zd, LOS size: %zd\n", total_alloc, los_memory_usage));
2882 /* this also sets the proper pointers for the next allocation */
2883 if (!search_fragment_for_size (size)) {
2885 /* TypeBuilder and MonoMethod are killing mcs with fragmentation */
2886 DEBUG (1, fprintf (gc_debug_file, "nursery collection didn't find enough room for %zd alloc (%d pinned)\n", size, last_num_pinned));
2887 for (i = 0; i < last_num_pinned; ++i) {
2888 DEBUG (3, fprintf (gc_debug_file, "Bastard pinning obj %p (%s), size: %d\n", pin_queue [i], safe_name (pin_queue [i]), safe_object_get_size (pin_queue [i])));
2893 //report_internal_mem_usage ();
2897 * ######################################################################
2898 * ######## Memory allocation from the OS
2899 * ######################################################################
2900 * This section of code deals with getting memory from the OS and
2901 * allocating memory for GC-internal data structures.
2902 * Internal memory can be handled with a freelist for small objects.
2906 * Allocate a big chunk of memory from the OS (usually 64KB to several megabytes).
2907 * This must not require any lock.
2910 get_os_memory (size_t size, int activate)
2913 unsigned long prot_flags = activate? MONO_MMAP_READ|MONO_MMAP_WRITE: MONO_MMAP_NONE;
2915 prot_flags |= MONO_MMAP_PRIVATE | MONO_MMAP_ANON;
2916 size += pagesize - 1;
2917 size &= ~(pagesize - 1);
2918 ptr = mono_valloc (0, size, prot_flags);
2923 * Free the memory returned by get_os_memory (), returning it to the OS.
2926 free_os_memory (void *addr, size_t size)
2928 munmap (addr, size);
2935 report_pinned_chunk (PinnedChunk *chunk, int seq) {
2937 int i, free_pages, num_free, free_mem;
2939 for (i = 0; i < chunk->num_pages; ++i) {
2940 if (!chunk->page_sizes [i])
2943 printf ("Pinned chunk %d at %p, size: %d, pages: %d, free: %d\n", seq, chunk, chunk->num_pages * FREELIST_PAGESIZE, chunk->num_pages, free_pages);
2944 free_mem = FREELIST_PAGESIZE * free_pages;
2945 for (i = 0; i < FREELIST_NUM_SLOTS; ++i) {
2946 if (!chunk->free_list [i])
2949 p = chunk->free_list [i];
2954 printf ("\tfree list of size %d, %d items\n", freelist_sizes [i], num_free);
2955 free_mem += freelist_sizes [i] * num_free;
2957 printf ("\tfree memory in chunk: %d\n", free_mem);
2963 static G_GNUC_UNUSED void
2964 report_internal_mem_usage (void) {
2967 printf ("Internal memory usage:\n");
2969 for (chunk = internal_chunk_list; chunk; chunk = chunk->next) {
2970 report_pinned_chunk (chunk, i++);
2972 printf ("Pinned memory usage:\n");
2974 for (chunk = pinned_chunk_list; chunk; chunk = chunk->next) {
2975 report_pinned_chunk (chunk, i++);
2980 * the array of pointers from @start to @end contains conservative
2981 * pointers to objects inside @chunk: mark each referenced object
2985 mark_pinned_from_addresses (PinnedChunk *chunk, void **start, void **end)
2987 for (; start < end; start++) {
2988 char *addr = *start;
2989 int offset = (char*)addr - (char*)chunk;
2990 int page = offset / FREELIST_PAGESIZE;
2991 int obj_offset = page == 0? offset - ((char*)chunk->start_data - (char*)chunk): offset % FREELIST_PAGESIZE;
2992 int slot_size = chunk->page_sizes [page];
2994 /* the page is not allocated */
2997 /* would be faster if we restrict the sizes to power of two,
2998 * but that's a waste of memory: need to measure. it could reduce
2999 * fragmentation since there are less pages needed, if for example
3000 * someone interns strings of each size we end up with one page per
3001 * interned string (still this is just ~40 KB): with more fine-grained sizes
3002 * this increases the number of used pages.
3005 obj_offset /= slot_size;
3006 obj_offset *= slot_size;
3007 addr = (char*)chunk->start_data + obj_offset;
3009 obj_offset /= slot_size;
3010 obj_offset *= slot_size;
3011 addr = (char*)chunk + page * FREELIST_PAGESIZE + obj_offset;
3014 /* if the vtable is inside the chunk it's on the freelist, so skip */
3015 if (*ptr && (*ptr < (void*)chunk->start_data || *ptr > (void*)((char*)chunk + chunk->num_pages * FREELIST_PAGESIZE))) {
3017 DEBUG (6, fprintf (gc_debug_file, "Marked pinned object %p (%s) from roots\n", addr, safe_name (addr)));
3023 scan_pinned_objects (void (*callback) (PinnedChunk*, char*, size_t, void*), void *callback_data)
3030 for (chunk = pinned_chunk_list; chunk; chunk = chunk->next) {
3031 end_chunk = (char*)chunk + chunk->num_pages * FREELIST_PAGESIZE;
3032 DEBUG (6, fprintf (gc_debug_file, "Scanning pinned chunk %p (range: %p-%p)\n", chunk, chunk->start_data, end_chunk));
3033 for (i = 0; i < chunk->num_pages; ++i) {
3034 obj_size = chunk->page_sizes [i];
3037 p = i? (char*)chunk + i * FREELIST_PAGESIZE: chunk->start_data;
3038 endp = i? p + FREELIST_PAGESIZE: (char*)chunk + FREELIST_PAGESIZE;
3039 DEBUG (6, fprintf (gc_debug_file, "Page %d (size: %d, range: %p-%p)\n", i, obj_size, p, endp));
3040 while (p + obj_size <= endp) {
3042 DEBUG (9, fprintf (gc_debug_file, "Considering %p (vtable: %p)\n", ptr, *ptr));
3043 /* if the first word (the vtable) is outside the chunk we have an object */
3044 if (*ptr && (*ptr < (void*)chunk || *ptr >= end_chunk))
3045 callback (chunk, (char*)ptr, obj_size, callback_data);
3053 sweep_pinned_objects_callback (PinnedChunk *chunk, char *ptr, size_t size, void *data)
3055 if (object_is_pinned (ptr)) {
3057 DEBUG (6, fprintf (gc_debug_file, "Unmarked pinned object %p (%s)\n", ptr, safe_name (ptr)));
3059 DEBUG (6, fprintf (gc_debug_file, "Freeing unmarked pinned object %p (%s)\n", ptr, safe_name (ptr)));
3060 free_pinned_object (chunk, ptr, size);
3065 sweep_pinned_objects (void)
3067 scan_pinned_objects (sweep_pinned_objects_callback, NULL);
3071 scan_object_callback (PinnedChunk *chunk, char *ptr, size_t size, char **data)
3073 DEBUG (6, fprintf (gc_debug_file, "Precise object scan of alloc_pinned %p (%s)\n", ptr, safe_name (ptr)));
3074 /* FIXME: Put objects without references into separate chunks
3075 which do not need to be scanned */
3076 scan_object (ptr, data [0], data [1]);
3080 scan_from_pinned_objects (char *addr_start, char *addr_end)
3082 char *data [2] = { addr_start, addr_end };
3083 scan_pinned_objects (scan_object_callback, data);
3087 * Find the slot number in the freelist for memory chunks that
3088 * can contain @size objects.
3091 slot_for_size (size_t size)
3094 /* do a binary search or lookup table later. */
3095 for (slot = 0; slot < FREELIST_NUM_SLOTS; ++slot) {
3096 if (freelist_sizes [slot] >= size)
3099 g_assert_not_reached ();
3104 * Build a free list for @size memory chunks from the memory area between
3105 * start_page and end_page.
3108 build_freelist (PinnedChunk *chunk, int slot, int size, char *start_page, char *end_page)
3112 /*g_print ("building freelist for slot %d, size %d in %p\n", slot, size, chunk);*/
3113 p = (void**)start_page;
3114 end = (void**)(end_page - size);
3115 g_assert (!chunk->free_list [slot]);
3116 chunk->free_list [slot] = p;
3117 while ((char*)p + size <= (char*)end) {
3119 *p = (void*)((char*)p + size);
3123 /*g_print ("%d items created, max: %d\n", count, (end_page - start_page) / size);*/
3127 alloc_pinned_chunk (size_t size)
3132 size += pagesize; /* at least one page */
3133 size += pagesize - 1;
3134 size &= ~(pagesize - 1);
3135 if (size < PINNED_CHUNK_MIN_SIZE * 2)
3136 size = PINNED_CHUNK_MIN_SIZE * 2;
3137 chunk = get_os_memory (size, TRUE);
3138 UPDATE_HEAP_BOUNDARIES (chunk, ((char*)chunk + size));
3139 total_alloc += size;
3141 /* setup the bookeeping fields */
3142 chunk->num_pages = size / FREELIST_PAGESIZE;
3143 offset = G_STRUCT_OFFSET (PinnedChunk, data);
3144 chunk->page_sizes = (void*)((char*)chunk + offset);
3145 offset += sizeof (int) * chunk->num_pages;
3146 offset += ALLOC_ALIGN - 1;
3147 offset &= ~(ALLOC_ALIGN - 1);
3148 chunk->free_list = (void*)((char*)chunk + offset);
3149 offset += sizeof (void*) * FREELIST_NUM_SLOTS;
3150 offset += ALLOC_ALIGN - 1;
3151 offset &= ~(ALLOC_ALIGN - 1);
3152 chunk->start_data = (void*)((char*)chunk + offset);
3154 /* allocate the first page to the freelist */
3155 chunk->page_sizes [0] = PINNED_FIRST_SLOT_SIZE;
3156 build_freelist (chunk, slot_for_size (PINNED_FIRST_SLOT_SIZE), PINNED_FIRST_SLOT_SIZE, chunk->start_data, ((char*)chunk + FREELIST_PAGESIZE));
3157 DEBUG (4, fprintf (gc_debug_file, "Allocated pinned chunk %p, size: %zd\n", chunk, size));
3158 min_pinned_chunk_addr = MIN (min_pinned_chunk_addr, (char*)chunk->start_data);
3159 max_pinned_chunk_addr = MAX (max_pinned_chunk_addr, ((char*)chunk + size));
3163 /* assumes freelist for slot is empty, so try to alloc a new page */
3165 get_chunk_freelist (PinnedChunk *chunk, int slot)
3169 p = chunk->free_list [slot];
3171 chunk->free_list [slot] = *p;
3174 for (i = 0; i < chunk->num_pages; ++i) {
3176 if (chunk->page_sizes [i])
3178 size = freelist_sizes [slot];
3179 chunk->page_sizes [i] = size;
3180 build_freelist (chunk, slot, size, (char*)chunk + FREELIST_PAGESIZE * i, (char*)chunk + FREELIST_PAGESIZE * (i + 1));
3184 p = chunk->free_list [slot];
3186 chunk->free_list [slot] = *p;
3193 alloc_from_freelist (size_t size)
3197 PinnedChunk *pchunk;
3198 slot = slot_for_size (size);
3199 /*g_print ("using slot %d for size %d (slot size: %d)\n", slot, size, freelist_sizes [slot]);*/
3200 g_assert (size <= freelist_sizes [slot]);
3201 for (pchunk = pinned_chunk_list; pchunk; pchunk = pchunk->next) {
3202 void **p = pchunk->free_list [slot];
3204 /*g_print ("found freelist for slot %d in chunk %p, returning %p, next %p\n", slot, pchunk, p, *p);*/
3205 pchunk->free_list [slot] = *p;
3209 for (pchunk = pinned_chunk_list; pchunk; pchunk = pchunk->next) {
3210 res = get_chunk_freelist (pchunk, slot);
3214 pchunk = alloc_pinned_chunk (size);
3215 /* FIXME: handle OOM */
3216 pchunk->next = pinned_chunk_list;
3217 pinned_chunk_list = pchunk;
3218 res = get_chunk_freelist (pchunk, slot);
3222 /* used for the GC-internal data structures */
3223 /* FIXME: add support for bigger sizes by allocating more than one page
3227 get_internal_mem (size_t size)
3231 PinnedChunk *pchunk;
3233 if (size > freelist_sizes [FREELIST_NUM_SLOTS - 1]) {
3234 LargeInternalMemHeader *mh;
3236 size += sizeof (LargeInternalMemHeader);
3237 mh = get_os_memory (size, TRUE);
3238 mh->magic = LARGE_INTERNAL_MEM_HEADER_MAGIC;
3243 slot = slot_for_size (size);
3244 g_assert (size <= freelist_sizes [slot]);
3245 for (pchunk = internal_chunk_list; pchunk; pchunk = pchunk->next) {
3246 void **p = pchunk->free_list [slot];
3248 pchunk->free_list [slot] = *p;
3249 memset (p, 0, size);
3253 for (pchunk = internal_chunk_list; pchunk; pchunk = pchunk->next) {
3254 res = get_chunk_freelist (pchunk, slot);
3256 memset (res, 0, size);
3260 pchunk = alloc_pinned_chunk (size);
3261 /* FIXME: handle OOM */
3262 pchunk->next = internal_chunk_list;
3263 internal_chunk_list = pchunk;
3264 res = get_chunk_freelist (pchunk, slot);
3265 memset (res, 0, size);
3270 free_internal_mem (void *addr)
3272 PinnedChunk *pchunk;
3273 LargeInternalMemHeader *mh;
3276 for (pchunk = internal_chunk_list; pchunk; pchunk = pchunk->next) {
3277 /*printf ("trying to free %p in %p (pages: %d)\n", addr, pchunk, pchunk->num_pages);*/
3278 if (addr >= (void*)pchunk && (char*)addr < (char*)pchunk + pchunk->num_pages * FREELIST_PAGESIZE) {
3279 int offset = (char*)addr - (char*)pchunk;
3280 int page = offset / FREELIST_PAGESIZE;
3281 int slot = slot_for_size (pchunk->page_sizes [page]);
3283 *p = pchunk->free_list [slot];
3284 pchunk->free_list [slot] = p;
3288 mh = (LargeInternalMemHeader*)((char*)addr - G_STRUCT_OFFSET (LargeInternalMemHeader, data));
3289 g_assert (mh->magic == LARGE_INTERNAL_MEM_HEADER_MAGIC);
3290 free_os_memory (mh, mh->size);
3294 * ######################################################################
3295 * ######## Object allocation
3296 * ######################################################################
3297 * This section of code deals with allocating memory for objects.
3298 * There are several ways:
3299 * *) allocate large objects
3300 * *) allocate normal objects
3301 * *) fast lock-free allocation
3302 * *) allocation of pinned objects
3306 free_large_object (LOSObject *obj)
3308 size_t size = obj->size;
3309 DEBUG (4, fprintf (gc_debug_file, "Freed large object %p, size %zd\n", obj->data, obj->size));
3311 los_memory_usage -= size;
3312 size += sizeof (LOSObject);
3313 size += pagesize - 1;
3314 size &= ~(pagesize - 1);
3315 total_alloc -= size;
3317 free_os_memory (obj, size);
3321 * Objects with size >= 64KB are allocated in the large object space.
3322 * They are currently kept track of with a linked list.
3323 * They don't move, so there is no need to pin them during collection
3324 * and we avoid the memcpy overhead.
3326 static void* __attribute__((noinline))
3327 alloc_large_inner (MonoVTable *vtable, size_t size)
3332 int just_did_major_gc = FALSE;
3334 if (los_memory_usage > next_los_collection) {
3335 DEBUG (4, fprintf (gc_debug_file, "Should trigger major collection: req size %zd (los already: %zu, limit: %zu)\n", size, los_memory_usage, next_los_collection));
3336 just_did_major_gc = TRUE;
3338 major_collection ();
3340 /* later increase based on a percent of the heap size */
3341 next_los_collection = los_memory_usage + 5*1024*1024;
3344 alloc_size += sizeof (LOSObject);
3345 alloc_size += pagesize - 1;
3346 alloc_size &= ~(pagesize - 1);
3347 /* FIXME: handle OOM */
3348 obj = get_os_memory (alloc_size, TRUE);
3350 vtslot = (void**)obj->data;
3352 total_alloc += alloc_size;
3353 UPDATE_HEAP_BOUNDARIES (obj->data, (char*)obj->data + size);
3354 obj->next = los_object_list;
3355 los_object_list = obj;
3356 los_memory_usage += size;
3358 DEBUG (4, fprintf (gc_debug_file, "Allocated large object %p, vtable: %p (%s), size: %zd\n", obj->data, vtable, vtable->klass->name, size));
3362 /* check if we have a suitable fragment in nursery_fragments to be able to allocate
3363 * an object of size @size
3364 * Return FALSE if not found (which means we need a collection)
3367 search_fragment_for_size (size_t size)
3369 Fragment *frag, *prev;
3370 DEBUG (4, fprintf (gc_debug_file, "Searching nursery fragment %p, size: %zd\n", nursery_frag_real_end, size));
3372 if (nursery_frag_real_end > nursery_next && nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
3373 /* Clear the remaining space, pinning depends on this */
3374 memset (nursery_next, 0, nursery_frag_real_end - nursery_next);
3377 for (frag = nursery_fragments; frag; frag = frag->next) {
3378 if (size <= (frag->fragment_end - frag->fragment_start)) {
3379 /* remove from the list */
3381 prev->next = frag->next;
3383 nursery_fragments = frag->next;
3384 nursery_next = frag->fragment_start;
3385 nursery_frag_real_end = frag->fragment_end;
3387 DEBUG (4, fprintf (gc_debug_file, "Using nursery fragment %p-%p, size: %zd (req: %zd)\n", nursery_next, nursery_frag_real_end, nursery_frag_real_end - nursery_next, size));
3388 frag->next = fragment_freelist;
3389 fragment_freelist = frag;
3398 * size is already rounded up and we hold the GC lock.
3401 alloc_degraded (MonoVTable *vtable, size_t size)
3403 GCMemSection *section;
3405 for (section = section_list; section; section = section->next) {
3406 if (section != nursery_section && (section->end_data - section->next_data) >= size) {
3407 p = (void**)section->next_data;
3412 section = alloc_section (nursery_section->size * 4);
3413 /* FIXME: handle OOM */
3414 p = (void**)section->next_data;
3416 section->next_data += size;
3417 degraded_mode += size;
3418 DEBUG (3, fprintf (gc_debug_file, "Allocated (degraded) object %p, vtable: %p (%s), size: %zd in section %p\n", p, vtable, vtable->klass->name, size, section));
3424 * Provide a variant that takes just the vtable for small fixed-size objects.
3425 * The aligned size is already computed and stored in vt->gc_descr.
3426 * Note: every SCAN_START_SIZE or so we are given the chance to do some special
3427 * processing. We can keep track of where objects start, for example,
3428 * so when we scan the thread stacks for pinned objects, we can start
3429 * a search for the pinned object in SCAN_START_SIZE chunks.
3432 mono_gc_alloc_obj (MonoVTable *vtable, size_t size)
3434 /* FIXME: handle OOM */
3441 size += ALLOC_ALIGN - 1;
3442 size &= ~(ALLOC_ALIGN - 1);
3444 g_assert (vtable->gc_descr);
3446 if (G_UNLIKELY (collect_before_allocs)) {
3449 if (nursery_section) {
3453 collect_nursery (0);
3455 if (!degraded_mode && !search_fragment_for_size (size)) {
3457 g_assert_not_reached ();
3464 * We need to lock here instead of after the fast path because
3465 * we might be interrupted in the fast path (after confirming
3466 * that new_next < TLAB_TEMP_END) by the GC, and we'll end up
3467 * allocating an object in a fragment which no longer belongs
3470 * The managed allocator does not do this, but it's treated
3471 * specially by the world-stopping code.
3475 /* tlab_next and tlab_temp_end are TLS vars so accessing them might be expensive */
3477 p = (void**)TLAB_NEXT;
3478 /* FIXME: handle overflow */
3479 new_next = (char*)p + size;
3480 TLAB_NEXT = new_next;
3482 if (G_LIKELY (new_next < TLAB_TEMP_END)) {
3486 * FIXME: We might need a memory barrier here so the change to tlab_next is
3487 * visible before the vtable store.
3490 DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
3491 g_assert (*p == NULL);
3494 g_assert (TLAB_NEXT == new_next);
3503 /* there are two cases: the object is too big or we run out of space in the TLAB */
3504 /* we also reach here when the thread does its first allocation after a minor
3505 * collection, since the tlab_ variables are initialized to NULL.
3506 * there can be another case (from ORP), if we cooperate with the runtime a bit:
3507 * objects that need finalizers can have the high bit set in their size
3508 * so the above check fails and we can readily add the object to the queue.
3509 * This avoids taking again the GC lock when registering, but this is moot when
3510 * doing thread-local allocation, so it may not be a good idea.
3512 g_assert (TLAB_NEXT == new_next);
3513 if (size > MAX_SMALL_OBJ_SIZE) {
3515 p = alloc_large_inner (vtable, size);
3517 if (TLAB_NEXT >= TLAB_REAL_END) {
3519 * Run out of space in the TLAB. When this happens, some amount of space
3520 * remains in the TLAB, but not enough to satisfy the current allocation
3521 * request. Currently, we retire the TLAB in all cases, later we could
3522 * keep it if the remaining space is above a treshold, and satisfy the
3523 * allocation directly from the nursery.
3526 /* when running in degraded mode, we continue allocing that way
3527 * for a while, to decrease the number of useless nursery collections.
3529 if (degraded_mode && degraded_mode < DEFAULT_NURSERY_SIZE) {
3530 p = alloc_degraded (vtable, size);
3535 if (size > tlab_size) {
3536 /* Allocate directly from the nursery */
3537 if (nursery_next + size >= nursery_frag_real_end) {
3538 if (!search_fragment_for_size (size)) {
3539 minor_collect_or_expand_inner (size);
3540 if (degraded_mode) {
3541 p = alloc_degraded (vtable, size);
3548 p = (void*)nursery_next;
3549 nursery_next += size;
3550 if (nursery_next > nursery_frag_real_end) {
3555 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
3556 memset (p, 0, size);
3559 DEBUG (3, fprintf (gc_debug_file, "Retire TLAB: %p-%p [%ld]\n", TLAB_START, TLAB_REAL_END, (long)(TLAB_REAL_END - TLAB_NEXT - size)));
3561 if (nursery_next + tlab_size >= nursery_frag_real_end) {
3562 res = search_fragment_for_size (tlab_size);
3564 minor_collect_or_expand_inner (tlab_size);
3565 if (degraded_mode) {
3566 p = alloc_degraded (vtable, size);
3573 /* Allocate a new TLAB from the current nursery fragment */
3574 TLAB_START = nursery_next;
3575 nursery_next += tlab_size;
3576 TLAB_NEXT = TLAB_START;
3577 TLAB_REAL_END = TLAB_START + tlab_size;
3578 TLAB_TEMP_END = TLAB_START + MIN (SCAN_START_SIZE, tlab_size);
3580 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
3581 memset (TLAB_START, 0, tlab_size);
3583 /* Allocate from the TLAB */
3584 p = (void*)TLAB_NEXT;
3586 g_assert (TLAB_NEXT <= TLAB_REAL_END);
3588 nursery_section->scan_starts [((char*)p - (char*)nursery_section->data)/SCAN_START_SIZE] = (char*)p;
3591 /* Reached tlab_temp_end */
3593 /* record the scan start so we can find pinned objects more easily */
3594 nursery_section->scan_starts [((char*)p - (char*)nursery_section->data)/SCAN_START_SIZE] = (char*)p;
3595 /* we just bump tlab_temp_end as well */
3596 TLAB_TEMP_END = MIN (TLAB_REAL_END, TLAB_NEXT + SCAN_START_SIZE);
3597 DEBUG (5, fprintf (gc_debug_file, "Expanding local alloc: %p-%p\n", TLAB_NEXT, TLAB_TEMP_END));
3601 DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
3610 * To be used for interned strings and possibly MonoThread, reflection handles.
3611 * We may want to explicitly free these objects.
3614 mono_gc_alloc_pinned_obj (MonoVTable *vtable, size_t size)
3616 /* FIXME: handle OOM */
3618 size += ALLOC_ALIGN - 1;
3619 size &= ~(ALLOC_ALIGN - 1);
3621 if (size > MAX_FREELIST_SIZE) {
3622 /* large objects are always pinned anyway */
3623 p = alloc_large_inner (vtable, size);
3625 p = alloc_from_freelist (size);
3626 memset (p, 0, size);
3628 DEBUG (6, fprintf (gc_debug_file, "Allocated pinned object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
3635 * ######################################################################
3636 * ######## Finalization support
3637 * ######################################################################
3641 * this is valid for the nursery: if the object has been forwarded it means it's
3642 * still refrenced from a root. If it is pinned it's still alive as well.
3643 * Return TRUE if @obj is ready to be finalized.
3645 #define object_is_fin_ready(obj) (!object_is_pinned (obj) && !object_is_forwarded (obj))
3648 is_critical_finalizer (FinalizeEntry *entry)
3653 if (!mono_defaults.critical_finalizer_object)
3656 obj = entry->object;
3657 class = ((MonoVTable*)LOAD_VTABLE (obj))->klass;
3659 return mono_class_has_parent (class, mono_defaults.critical_finalizer_object);
3663 queue_finalization_entry (FinalizeEntry *entry) {
3664 if (is_critical_finalizer (entry)) {
3665 entry->next = critical_fin_list;
3666 critical_fin_list = entry;
3668 entry->next = fin_ready_list;
3669 fin_ready_list = entry;
3673 /* LOCKING: requires that the GC lock is held */
3675 rehash_fin_table (FinalizeEntryHashTable *hash_table)
3677 FinalizeEntry **finalizable_hash = hash_table->table;
3678 mword finalizable_hash_size = hash_table->size;
3681 FinalizeEntry **new_hash;
3682 FinalizeEntry *entry, *next;
3683 int new_size = g_spaced_primes_closest (hash_table->num_registered);
3685 new_hash = get_internal_mem (new_size * sizeof (FinalizeEntry*));
3686 for (i = 0; i < finalizable_hash_size; ++i) {
3687 for (entry = finalizable_hash [i]; entry; entry = next) {
3688 hash = mono_object_hash (entry->object) % new_size;
3690 entry->next = new_hash [hash];
3691 new_hash [hash] = entry;
3694 free_internal_mem (finalizable_hash);
3695 hash_table->table = new_hash;
3696 hash_table->size = new_size;
3699 /* LOCKING: requires that the GC lock is held */
3701 rehash_fin_table_if_necessary (FinalizeEntryHashTable *hash_table)
3703 if (hash_table->num_registered >= hash_table->size * 2)
3704 rehash_fin_table (hash_table);
3707 /* LOCKING: requires that the GC lock is held */
3709 finalize_in_range (char *start, char *end, int generation)
3711 FinalizeEntryHashTable *hash_table = get_finalize_entry_hash_table (generation);
3712 FinalizeEntry *entry, *prev;
3714 FinalizeEntry **finalizable_hash = hash_table->table;
3715 mword finalizable_hash_size = hash_table->size;
3719 for (i = 0; i < finalizable_hash_size; ++i) {
3721 for (entry = finalizable_hash [i]; entry;) {
3722 if ((char*)entry->object >= start && (char*)entry->object < end && ((char*)entry->object < to_space || (char*)entry->object >= to_space_end)) {
3723 gboolean is_fin_ready = object_is_fin_ready (entry->object);
3724 char *copy = copy_object (entry->object, start, end);
3727 FinalizeEntry *next;
3728 /* remove and put in fin_ready_list */
3730 prev->next = entry->next;
3732 finalizable_hash [i] = entry->next;
3734 num_ready_finalizers++;
3735 hash_table->num_registered--;
3736 queue_finalization_entry (entry);
3737 /* Make it survive */
3738 from = entry->object;
3739 entry->object = copy;
3740 DEBUG (5, fprintf (gc_debug_file, "Queueing object for finalization: %p (%s) (was at %p) (%d/%d)\n", entry->object, safe_name (entry->object), from, num_ready_finalizers, hash_table->num_registered));
3744 char *from = entry->object;
3745 if (hash_table == &minor_finalizable_hash && !ptr_in_nursery (copy)) {
3746 FinalizeEntry *next = entry->next;
3747 unsigned int major_hash;
3748 /* remove from the list */
3750 prev->next = entry->next;
3752 finalizable_hash [i] = entry->next;
3753 hash_table->num_registered--;
3755 entry->object = copy;
3757 /* insert it into the major hash */
3758 rehash_fin_table_if_necessary (&major_finalizable_hash);
3759 major_hash = mono_object_hash ((MonoObject*) copy) %
3760 major_finalizable_hash.size;
3761 entry->next = major_finalizable_hash.table [major_hash];
3762 major_finalizable_hash.table [major_hash] = entry;
3763 major_finalizable_hash.num_registered++;
3765 DEBUG (5, fprintf (gc_debug_file, "Promoting finalization of object %p (%s) (was at %p) to major table\n", copy, safe_name (copy), from));
3770 /* update pointer */
3771 DEBUG (5, fprintf (gc_debug_file, "Updating object for finalization: %p (%s) (was at %p)\n", entry->object, safe_name (entry->object), from));
3772 entry->object = copy;
3777 entry = entry->next;
3782 /* LOCKING: requires that the GC lock is held */
3784 null_link_in_range (char *start, char *end, int generation)
3786 DisappearingLinkHashTable *hash = get_dislink_hash_table (generation);
3787 DisappearingLink **disappearing_link_hash = hash->table;
3788 int disappearing_link_hash_size = hash->size;
3789 DisappearingLink *entry, *prev;
3791 if (!hash->num_links)
3793 for (i = 0; i < disappearing_link_hash_size; ++i) {
3795 for (entry = disappearing_link_hash [i]; entry;) {
3796 char *object = DISLINK_OBJECT (entry);
3797 if (object >= start && object < end && (object < to_space || object >= to_space_end)) {
3798 gboolean track = DISLINK_TRACK (entry);
3799 if (!track && object_is_fin_ready (object)) {
3800 void **p = entry->link;
3801 DisappearingLink *old;
3803 /* remove from list */
3805 prev->next = entry->next;
3807 disappearing_link_hash [i] = entry->next;
3808 DEBUG (5, fprintf (gc_debug_file, "Dislink nullified at %p to GCed object %p\n", p, object));
3810 free_internal_mem (entry);
3815 char *copy = copy_object (object, start, end);
3817 /* Update pointer if it's moved. If the object
3818 * has been moved out of the nursery, we need to
3819 * remove the link from the minor hash table to
3822 * FIXME: what if an object is moved earlier?
3825 if (hash == &minor_disappearing_link_hash && !ptr_in_nursery (copy)) {
3826 void **link = entry->link;
3827 DisappearingLink *old;
3828 /* remove from list */
3830 prev->next = entry->next;
3832 disappearing_link_hash [i] = entry->next;
3834 free_internal_mem (entry);
3838 add_or_remove_disappearing_link ((MonoObject*)copy, link,
3839 track, GENERATION_OLD);
3841 DEBUG (5, fprintf (gc_debug_file, "Upgraded dislink at %p to major because object %p moved to %p\n", link, object, copy));
3845 /* We set the track resurrection bit to
3846 * FALSE if the object is to be finalized
3847 * so that the object can be collected in
3848 * the next cycle (i.e. after it was
3851 *entry->link = HIDE_POINTER (copy,
3852 object_is_fin_ready (object) ? FALSE : track);
3853 DEBUG (5, fprintf (gc_debug_file, "Updated dislink at %p to %p\n", entry->link, DISLINK_OBJECT (entry)));
3858 entry = entry->next;
3863 /* LOCKING: requires that the GC lock is held */
3865 null_links_for_domain (MonoDomain *domain, int generation)
3867 DisappearingLinkHashTable *hash = get_dislink_hash_table (generation);
3868 DisappearingLink **disappearing_link_hash = hash->table;
3869 int disappearing_link_hash_size = hash->size;
3870 DisappearingLink *entry, *prev;
3872 for (i = 0; i < disappearing_link_hash_size; ++i) {
3874 for (entry = disappearing_link_hash [i]; entry; ) {
3875 char *object = DISLINK_OBJECT (entry);
3876 /* FIXME: actually there should be no object
3877 left in the domain with a non-null vtable
3878 (provided we remove the Thread special
3880 if (object && (!((MonoObject*)object)->vtable || mono_object_domain (object) == domain)) {
3881 DisappearingLink *next = entry->next;
3886 disappearing_link_hash [i] = next;
3888 if (*(entry->link)) {
3889 *(entry->link) = NULL;
3890 g_warning ("Disappearing link %p not freed", entry->link);
3892 free_internal_mem (entry);
3899 entry = entry->next;
3904 /* LOCKING: requires that the GC lock is held */
3906 finalizers_for_domain (MonoDomain *domain, MonoObject **out_array, int out_size,
3907 FinalizeEntryHashTable *hash_table)
3909 FinalizeEntry **finalizable_hash = hash_table->table;
3910 mword finalizable_hash_size = hash_table->size;
3911 FinalizeEntry *entry, *prev;
3914 if (no_finalize || !out_size || !out_array)
3917 for (i = 0; i < finalizable_hash_size; ++i) {
3919 for (entry = finalizable_hash [i]; entry;) {
3920 if (mono_object_domain (entry->object) == domain) {
3921 FinalizeEntry *next;
3922 /* remove and put in out_array */
3924 prev->next = entry->next;
3926 finalizable_hash [i] = entry->next;
3928 hash_table->num_registered--;
3929 out_array [count ++] = entry->object;
3930 DEBUG (5, fprintf (gc_debug_file, "Collecting object for finalization: %p (%s) (%d/%d)\n", entry->object, safe_name (entry->object), num_ready_finalizers, hash_table->num_registered));
3932 if (count == out_size)
3937 entry = entry->next;
3944 * mono_gc_finalizers_for_domain:
3945 * @domain: the unloading appdomain
3946 * @out_array: output array
3947 * @out_size: size of output array
3949 * Store inside @out_array up to @out_size objects that belong to the unloading
3950 * appdomain @domain. Returns the number of stored items. Can be called repeteadly
3951 * until it returns 0.
3952 * The items are removed from the finalizer data structure, so the caller is supposed
3954 * @out_array should be on the stack to allow the GC to know the objects are still alive.
3957 mono_gc_finalizers_for_domain (MonoDomain *domain, MonoObject **out_array, int out_size)
3962 result = finalizers_for_domain (domain, out_array, out_size, &minor_finalizable_hash);
3963 if (result < out_size) {
3964 result += finalizers_for_domain (domain, out_array + result, out_size - result,
3965 &major_finalizable_hash);
3973 register_for_finalization (MonoObject *obj, void *user_data, int generation)
3975 FinalizeEntryHashTable *hash_table = get_finalize_entry_hash_table (generation);
3976 FinalizeEntry **finalizable_hash;
3977 mword finalizable_hash_size;
3978 FinalizeEntry *entry, *prev;
3982 g_assert (user_data == NULL || user_data == mono_gc_run_finalize);
3983 hash = mono_object_hash (obj);
3985 rehash_fin_table_if_necessary (hash_table);
3986 finalizable_hash = hash_table->table;
3987 finalizable_hash_size = hash_table->size;
3988 hash %= finalizable_hash_size;
3990 for (entry = finalizable_hash [hash]; entry; entry = entry->next) {
3991 if (entry->object == obj) {
3993 /* remove from the list */
3995 prev->next = entry->next;
3997 finalizable_hash [hash] = entry->next;
3998 hash_table->num_registered--;
3999 DEBUG (5, fprintf (gc_debug_file, "Removed finalizer %p for object: %p (%s) (%d)\n", entry, obj, obj->vtable->klass->name, hash_table->num_registered));
4000 free_internal_mem (entry);
4008 /* request to deregister, but already out of the list */
4012 entry = get_internal_mem (sizeof (FinalizeEntry));
4013 entry->object = obj;
4014 entry->next = finalizable_hash [hash];
4015 finalizable_hash [hash] = entry;
4016 hash_table->num_registered++;
4017 DEBUG (5, fprintf (gc_debug_file, "Added finalizer %p for object: %p (%s) (%d) to %s table\n", entry, obj, obj->vtable->klass->name, hash_table->num_registered, generation_name (generation)));
4022 mono_gc_register_for_finalization (MonoObject *obj, void *user_data)
4024 if (ptr_in_nursery (obj))
4025 register_for_finalization (obj, user_data, GENERATION_NURSERY);
4027 register_for_finalization (obj, user_data, GENERATION_OLD);
4031 rehash_dislink (DisappearingLinkHashTable *hash_table)
4033 DisappearingLink **disappearing_link_hash = hash_table->table;
4034 int disappearing_link_hash_size = hash_table->size;
4037 DisappearingLink **new_hash;
4038 DisappearingLink *entry, *next;
4039 int new_size = g_spaced_primes_closest (hash_table->num_links);
4041 new_hash = get_internal_mem (new_size * sizeof (DisappearingLink*));
4042 for (i = 0; i < disappearing_link_hash_size; ++i) {
4043 for (entry = disappearing_link_hash [i]; entry; entry = next) {
4044 hash = mono_aligned_addr_hash (entry->link) % new_size;
4046 entry->next = new_hash [hash];
4047 new_hash [hash] = entry;
4050 free_internal_mem (disappearing_link_hash);
4051 hash_table->table = new_hash;
4052 hash_table->size = new_size;
4055 /* LOCKING: assumes the GC lock is held */
4057 add_or_remove_disappearing_link (MonoObject *obj, void **link, gboolean track, int generation)
4059 DisappearingLinkHashTable *hash_table = get_dislink_hash_table (generation);
4060 DisappearingLink *entry, *prev;
4062 DisappearingLink **disappearing_link_hash = hash_table->table;
4063 int disappearing_link_hash_size = hash_table->size;
4065 if (hash_table->num_links >= disappearing_link_hash_size * 2) {
4066 rehash_dislink (hash_table);
4067 disappearing_link_hash = hash_table->table;
4068 disappearing_link_hash_size = hash_table->size;
4070 /* FIXME: add check that link is not in the heap */
4071 hash = mono_aligned_addr_hash (link) % disappearing_link_hash_size;
4072 entry = disappearing_link_hash [hash];
4074 for (; entry; entry = entry->next) {
4075 /* link already added */
4076 if (link == entry->link) {
4077 /* NULL obj means remove */
4080 prev->next = entry->next;
4082 disappearing_link_hash [hash] = entry->next;
4083 hash_table->num_links--;
4084 DEBUG (5, fprintf (gc_debug_file, "Removed dislink %p (%d) from %s table\n", entry, hash_table->num_links, generation_name (generation)));
4085 free_internal_mem (entry);
4088 *link = HIDE_POINTER (obj, track); /* we allow the change of object */
4096 entry = get_internal_mem (sizeof (DisappearingLink));
4097 *link = HIDE_POINTER (obj, track);
4099 entry->next = disappearing_link_hash [hash];
4100 disappearing_link_hash [hash] = entry;
4101 hash_table->num_links++;
4102 DEBUG (5, fprintf (gc_debug_file, "Added dislink %p for object: %p (%s) at %p to %s table\n", entry, obj, obj->vtable->klass->name, link, generation_name (generation)));
4105 /* LOCKING: assumes the GC lock is held */
4107 mono_gc_register_disappearing_link (MonoObject *obj, void **link, gboolean track)
4109 add_or_remove_disappearing_link (NULL, link, FALSE, GENERATION_NURSERY);
4110 add_or_remove_disappearing_link (NULL, link, FALSE, GENERATION_OLD);
4112 if (ptr_in_nursery (obj))
4113 add_or_remove_disappearing_link (obj, link, track, GENERATION_NURSERY);
4115 add_or_remove_disappearing_link (obj, link, track, GENERATION_OLD);
4120 mono_gc_invoke_finalizers (void)
4122 FinalizeEntry *entry = NULL;
4123 gboolean entry_is_critical;
4126 /* FIXME: batch to reduce lock contention */
4127 while (fin_ready_list || critical_fin_list) {
4131 FinalizeEntry **list = entry_is_critical ? &critical_fin_list : &fin_ready_list;
4133 /* We have finalized entry in the last
4134 interation, now we need to remove it from
4137 *list = entry->next;
4139 FinalizeEntry *e = *list;
4140 while (e->next != entry)
4142 e->next = entry->next;
4144 free_internal_mem (entry);
4148 /* Now look for the first non-null entry. */
4149 for (entry = fin_ready_list; entry && !entry->object; entry = entry->next)
4152 entry_is_critical = FALSE;
4154 entry_is_critical = TRUE;
4155 for (entry = critical_fin_list; entry && !entry->object; entry = entry->next)
4160 g_assert (entry->object);
4161 num_ready_finalizers--;
4162 obj = entry->object;
4163 entry->object = NULL;
4164 DEBUG (7, fprintf (gc_debug_file, "Finalizing object %p (%s)\n", obj, safe_name (obj)));
4172 g_assert (entry->object == NULL);
4174 /* the object is on the stack so it is pinned */
4175 /*g_print ("Calling finalizer for object: %p (%s)\n", entry->object, safe_name (entry->object));*/
4176 mono_gc_run_finalize (obj, NULL);
4183 mono_gc_pending_finalizers (void)
4185 return fin_ready_list || critical_fin_list;
4188 /* Negative value to remove */
4190 mono_gc_add_memory_pressure (gint64 value)
4192 /* FIXME: Use interlocked functions */
4194 memory_pressure += value;
4199 * ######################################################################
4200 * ######## registered roots support
4201 * ######################################################################
4205 rehash_roots (gboolean pinned)
4209 RootRecord **new_hash;
4210 RootRecord *entry, *next;
4213 new_size = g_spaced_primes_closest (num_roots_entries [pinned]);
4214 new_hash = get_internal_mem (new_size * sizeof (RootRecord*));
4215 for (i = 0; i < roots_hash_size [pinned]; ++i) {
4216 for (entry = roots_hash [pinned][i]; entry; entry = next) {
4217 hash = mono_aligned_addr_hash (entry->start_root) % new_size;
4219 entry->next = new_hash [hash];
4220 new_hash [hash] = entry;
4223 free_internal_mem (roots_hash [pinned]);
4224 roots_hash [pinned] = new_hash;
4225 roots_hash_size [pinned] = new_size;
4229 find_root (int root_type, char *start, guint32 addr_hash)
4231 RootRecord *new_root;
4233 guint32 hash = addr_hash % roots_hash_size [root_type];
4234 for (new_root = roots_hash [root_type][hash]; new_root; new_root = new_root->next) {
4235 /* we allow changing the size and the descriptor (for thread statics etc) */
4236 if (new_root->start_root == start) {
4245 * We do not coalesce roots.
4248 mono_gc_register_root_inner (char *start, size_t size, void *descr, int root_type)
4250 RootRecord *new_root;
4251 unsigned int hash, addr_hash = mono_aligned_addr_hash (start);
4254 for (i = 0; i < ROOT_TYPE_NUM; ++i) {
4255 if (num_roots_entries [i] >= roots_hash_size [i] * 2)
4258 for (i = 0; i < ROOT_TYPE_NUM; ++i) {
4259 new_root = find_root (i, start, addr_hash);
4260 /* we allow changing the size and the descriptor (for thread statics etc) */
4262 size_t old_size = new_root->end_root - new_root->start_root;
4263 new_root->end_root = new_root->start_root + size;
4264 g_assert (((new_root->root_desc != 0) && (descr != NULL)) ||
4265 ((new_root->root_desc == 0) && (descr == NULL)));
4266 new_root->root_desc = (mword)descr;
4268 roots_size -= old_size;
4273 new_root = get_internal_mem (sizeof (RootRecord));
4275 new_root->start_root = start;
4276 new_root->end_root = new_root->start_root + size;
4277 new_root->root_desc = (mword)descr;
4279 hash = addr_hash % roots_hash_size [root_type];
4280 num_roots_entries [root_type]++;
4281 new_root->next = roots_hash [root_type] [hash];
4282 roots_hash [root_type][hash] = new_root;
4283 DEBUG (3, fprintf (gc_debug_file, "Added root %p for range: %p-%p, descr: %p (%d/%d bytes)\n", new_root, new_root->start_root, new_root->end_root, descr, (int)size, (int)roots_size));
4293 mono_gc_register_root (char *start, size_t size, void *descr)
4295 return mono_gc_register_root_inner (start, size, descr, descr ? ROOT_TYPE_NORMAL : ROOT_TYPE_PINNED);
4299 mono_gc_register_root_wbarrier (char *start, size_t size, void *descr)
4301 return mono_gc_register_root_inner (start, size, descr, ROOT_TYPE_WBARRIER);
4305 mono_gc_deregister_root (char* addr)
4307 RootRecord *tmp, *prev;
4308 unsigned int hash, addr_hash = mono_aligned_addr_hash (addr);
4312 for (root_type = 0; root_type < ROOT_TYPE_NUM; ++root_type) {
4313 hash = addr_hash % roots_hash_size [root_type];
4314 tmp = roots_hash [root_type][hash];
4317 if (tmp->start_root == (char*)addr) {
4319 prev->next = tmp->next;
4321 roots_hash [root_type][hash] = tmp->next;
4322 roots_size -= (tmp->end_root - tmp->start_root);
4323 num_roots_entries [root_type]--;
4324 DEBUG (3, fprintf (gc_debug_file, "Removed root %p for range: %p-%p\n", tmp, tmp->start_root, tmp->end_root));
4325 free_internal_mem (tmp);
4336 * ######################################################################
4337 * ######## Thread handling (stop/start code)
4338 * ######################################################################
4341 /* FIXME: handle large/small config */
4342 #define THREAD_HASH_SIZE 11
4343 #define HASH_PTHREAD_T(id) (((unsigned int)(id) >> 4) * 2654435761u)
4345 static SgenThreadInfo* thread_table [THREAD_HASH_SIZE];
4347 #if USE_SIGNAL_BASED_START_STOP_WORLD
4350 static sem_t suspend_ack_semaphore;
4352 static sem_t *suspend_ack_semaphore_ptr;
4353 static unsigned int global_stop_count = 0;
4355 static int suspend_signal_num = SIGXFSZ;
4357 static int suspend_signal_num = SIGPWR;
4359 static int restart_signal_num = SIGXCPU;
4360 static sigset_t suspend_signal_mask;
4361 static mword cur_thread_regs [ARCH_NUM_REGS] = {0};
4363 /* LOCKING: assumes the GC lock is held */
4364 static SgenThreadInfo*
4365 thread_info_lookup (ARCH_THREAD_TYPE id)
4367 unsigned int hash = HASH_PTHREAD_T (id) % THREAD_HASH_SIZE;
4368 SgenThreadInfo *info;
4370 info = thread_table [hash];
4371 while (info && !ARCH_THREAD_EQUALS (info->id, id)) {
4378 update_current_thread_stack (void *start)
4380 void *ptr = cur_thread_regs;
4381 SgenThreadInfo *info = thread_info_lookup (ARCH_GET_THREAD ());
4382 info->stack_start = align_pointer (&ptr);
4383 g_assert (info->stack_start >= info->stack_start_limit && info->stack_start < info->stack_end);
4384 ARCH_STORE_REGS (ptr);
4385 info->stopped_regs = ptr;
4386 if (gc_callbacks.thread_suspend_func)
4387 gc_callbacks.thread_suspend_func (info->runtime_data, NULL);
4391 signal_desc (int signum)
4393 if (signum == suspend_signal_num)
4395 if (signum == restart_signal_num)
4400 #define MANAGED_ALLOCATION
4401 #define MANAGED_WBARRIER
4403 #ifdef MANAGED_ALLOCATION
4405 is_ip_in_managed_allocator (MonoDomain *domain, gpointer ip);
4408 is_ip_in_managed_allocator (MonoDomain *domain, gpointer ip)
4415 wait_for_suspend_ack (int count)
4419 for (i = 0; i < count; ++i) {
4420 while ((result = sem_wait (suspend_ack_semaphore_ptr)) != 0) {
4421 if (errno != EINTR) {
4422 g_error ("sem_wait ()");
4428 /* LOCKING: assumes the GC lock is held */
4430 thread_handshake (int signum)
4432 int count, i, result;
4433 SgenThreadInfo *info;
4434 pthread_t me = pthread_self ();
4437 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
4438 for (info = thread_table [i]; info; info = info->next) {
4439 DEBUG (4, fprintf (gc_debug_file, "considering thread %p for signal %d (%s)\n", info, signum, signal_desc (signum)));
4440 if (ARCH_THREAD_EQUALS (info->id, me)) {
4441 DEBUG (4, fprintf (gc_debug_file, "Skip (equal): %p, %p\n", (void*)me, (void*)info->id));
4444 /*if (signum == suspend_signal_num && info->stop_count == global_stop_count)
4446 result = pthread_kill (info->id, signum);
4448 DEBUG (4, fprintf (gc_debug_file, "thread %p signal sent\n", info));
4451 DEBUG (4, fprintf (gc_debug_file, "thread %p signal failed: %d (%s)\n", (void*)info->id, result, strerror (result)));
4457 wait_for_suspend_ack (count);
4463 restart_threads_until_none_in_managed_allocator (void)
4465 SgenThreadInfo *info;
4466 int i, result, num_threads_died = 0;
4467 int sleep_duration = -1;
4470 int restart_count = 0, restarted_count = 0;
4471 /* restart all threads that stopped in the
4473 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
4474 for (info = thread_table [i]; info; info = info->next) {
4477 if (!info->stack_start ||
4478 is_ip_in_managed_allocator (info->stopped_domain, info->stopped_ip)) {
4479 result = pthread_kill (info->id, restart_signal_num);
4486 /* we set the stopped_ip to
4487 NULL for threads which
4488 we're not restarting so
4489 that we can easily identify
4491 info->stopped_ip = NULL;
4492 info->stopped_domain = NULL;
4496 /* if no threads were restarted, we're done */
4497 if (restart_count == 0)
4500 /* wait for the threads to signal their restart */
4501 wait_for_suspend_ack (restart_count);
4503 if (sleep_duration < 0) {
4507 g_usleep (sleep_duration);
4508 sleep_duration += 10;
4511 /* stop them again */
4512 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
4513 for (info = thread_table [i]; info; info = info->next) {
4514 if (info->skip || info->stopped_ip == NULL)
4516 result = pthread_kill (info->id, suspend_signal_num);
4524 /* some threads might have died */
4525 num_threads_died += restart_count - restarted_count;
4526 /* wait for the threads to signal their suspension
4528 wait_for_suspend_ack (restart_count);
4531 return num_threads_died;
4534 /* LOCKING: assumes the GC lock is held (by the stopping thread) */
4536 suspend_handler (int sig, siginfo_t *siginfo, void *context)
4538 SgenThreadInfo *info;
4541 int old_errno = errno;
4542 gpointer regs [ARCH_NUM_REGS];
4543 gpointer stack_start;
4545 id = pthread_self ();
4546 info = thread_info_lookup (id);
4547 info->stopped_domain = mono_domain_get ();
4548 info->stopped_ip = (gpointer) ARCH_SIGCTX_IP (context);
4549 stop_count = global_stop_count;
4550 /* duplicate signal */
4551 if (0 && info->stop_count == stop_count) {
4555 #ifdef HAVE_KW_THREAD
4556 /* update the remset info in the thread data structure */
4557 info->remset = remembered_set;
4559 stack_start = (char*) ARCH_SIGCTX_SP (context) - REDZONE_SIZE;
4560 /* If stack_start is not within the limits, then don't set it
4561 in info and we will be restarted. */
4562 if (stack_start >= info->stack_start_limit && info->stack_start <= info->stack_end) {
4563 info->stack_start = stack_start;
4565 ARCH_COPY_SIGCTX_REGS (regs, context);
4566 info->stopped_regs = regs;
4568 g_assert (!info->stack_start);
4571 /* Notify the JIT */
4572 if (gc_callbacks.thread_suspend_func)
4573 gc_callbacks.thread_suspend_func (info->runtime_data, context);
4575 /* notify the waiting thread */
4576 sem_post (suspend_ack_semaphore_ptr);
4577 info->stop_count = stop_count;
4579 /* wait until we receive the restart signal */
4582 sigsuspend (&suspend_signal_mask);
4583 } while (info->signal != restart_signal_num);
4585 /* notify the waiting thread */
4586 sem_post (suspend_ack_semaphore_ptr);
4592 restart_handler (int sig)
4594 SgenThreadInfo *info;
4595 int old_errno = errno;
4597 info = thread_info_lookup (pthread_self ());
4598 info->signal = restart_signal_num;
4603 static TV_DECLARE (stop_world_time);
4604 static unsigned long max_pause_usec = 0;
4606 /* LOCKING: assumes the GC lock is held */
4612 update_current_thread_stack (&count);
4614 global_stop_count++;
4615 DEBUG (3, fprintf (gc_debug_file, "stopping world n %d from %p %p\n", global_stop_count, thread_info_lookup (ARCH_GET_THREAD ()), (gpointer)ARCH_GET_THREAD ()));
4616 TV_GETTIME (stop_world_time);
4617 count = thread_handshake (suspend_signal_num);
4618 count -= restart_threads_until_none_in_managed_allocator ();
4619 g_assert (count >= 0);
4620 DEBUG (3, fprintf (gc_debug_file, "world stopped %d thread(s)\n", count));
4624 /* LOCKING: assumes the GC lock is held */
4626 restart_world (void)
4629 SgenThreadInfo *info;
4630 TV_DECLARE (end_sw);
4633 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
4634 for (info = thread_table [i]; info; info = info->next) {
4635 info->stack_start = NULL;
4636 info->stopped_regs = NULL;
4640 count = thread_handshake (restart_signal_num);
4641 TV_GETTIME (end_sw);
4642 usec = TV_ELAPSED (stop_world_time, end_sw);
4643 max_pause_usec = MAX (usec, max_pause_usec);
4644 DEBUG (2, fprintf (gc_debug_file, "restarted %d thread(s) (pause time: %d usec, max: %d)\n", count, (int)usec, (int)max_pause_usec));
4648 #endif /* USE_SIGNAL_BASED_START_STOP_WORLD */
4651 mono_gc_set_gc_callbacks (MonoGCCallbacks *callbacks)
4653 gc_callbacks = *callbacks;
4656 /* Variables holding start/end nursery so it won't have to be passed at every call */
4657 static void *scan_area_arg_start, *scan_area_arg_end;
4660 mono_gc_conservatively_scan_area (void *start, void *end)
4662 conservatively_pin_objects_from (start, end, scan_area_arg_start, scan_area_arg_end);
4666 mono_gc_scan_object (void *obj)
4668 return copy_object (obj, scan_area_arg_start, scan_area_arg_end);
4672 * Mark from thread stacks and registers.
4675 scan_thread_data (void *start_nursery, void *end_nursery, gboolean precise)
4678 SgenThreadInfo *info;
4680 scan_area_arg_start = start_nursery;
4681 scan_area_arg_end = end_nursery;
4683 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
4684 for (info = thread_table [i]; info; info = info->next) {
4686 DEBUG (2, fprintf (gc_debug_file, "Skipping dead thread %p, range: %p-%p, size: %zd\n", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start));
4689 DEBUG (2, fprintf (gc_debug_file, "Scanning thread %p, range: %p-%p, size: %zd, pinned=%d\n", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start, next_pin_slot));
4690 if (gc_callbacks.thread_mark_func)
4691 gc_callbacks.thread_mark_func (info->runtime_data, info->stack_start, info->stack_end, precise);
4693 conservatively_pin_objects_from (info->stack_start, info->stack_end, start_nursery, end_nursery);
4696 conservatively_pin_objects_from (info->stopped_regs, info->stopped_regs + ARCH_NUM_REGS,
4697 start_nursery, end_nursery);
4703 find_pinning_ref_from_thread (char *obj, size_t size)
4706 SgenThreadInfo *info;
4707 char *endobj = obj + size;
4709 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
4710 for (info = thread_table [i]; info; info = info->next) {
4711 char **start = (char**)info->stack_start;
4714 while (start < (char**)info->stack_end) {
4715 if (*start >= obj && *start < endobj) {
4716 DEBUG (0, fprintf (gc_debug_file, "Object %p referenced in thread %p (id %p) at %p, stack: %p-%p\n", obj, info, (gpointer)info->id, start, info->stack_start, info->stack_end));
4721 /* FIXME: check info->stopped_regs */
4727 ptr_on_stack (void *ptr)
4730 gpointer stack_start = &stack_start;
4731 SgenThreadInfo *info = thread_info_lookup (ARCH_GET_THREAD ());
4733 if (ptr >= stack_start && ptr < (gpointer)info->stack_end)
4738 /* return TRUE if ptr points inside the managed heap */
4740 ptr_in_heap (void* ptr)
4742 mword p = (mword)ptr;
4744 GCMemSection *section;
4746 if (p < lowest_heap_address || p >= highest_heap_address)
4749 if (ptr_in_nursery (ptr))
4752 if (ptr_on_stack (ptr))
4755 for (section = section_list; section; section = section->next) {
4756 if (ptr >= (gpointer)section->data && ptr < (gpointer)(section->data + section->size))
4760 if (obj_is_from_pinned_alloc (ptr))
4763 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
4764 if (ptr >= (gpointer)bigobj->data && ptr < (gpointer)(bigobj->data + bigobj->size))
4772 handle_remset (mword *p, void *start_nursery, void *end_nursery, gboolean global)
4778 /* FIXME: exclude stack locations */
4779 switch ((*p) & REMSET_TYPE_MASK) {
4780 case REMSET_LOCATION:
4782 //__builtin_prefetch (ptr);
4783 if (((void*)ptr < start_nursery || (void*)ptr >= end_nursery) && ptr_in_heap (ptr)) {
4784 *ptr = copy_object (*ptr, start_nursery, end_nursery);
4785 DEBUG (9, fprintf (gc_debug_file, "Overwrote remset at %p with %p\n", ptr, *ptr));
4786 if (!global && *ptr >= start_nursery && *ptr < end_nursery) {
4788 * If the object is pinned, each reference to it from nonpinned objects
4789 * becomes part of the global remset, which can grow very large.
4791 DEBUG (9, fprintf (gc_debug_file, "Add to global remset because of pinning %p (%p %s)\n", ptr, *ptr, safe_name (*ptr)));
4792 add_to_global_remset (ptr, FALSE);
4795 DEBUG (9, fprintf (gc_debug_file, "Skipping remset at %p holding %p\n", ptr, *ptr));
4799 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
4800 if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery) || !ptr_in_heap (ptr))
4803 while (count-- > 0) {
4804 *ptr = copy_object (*ptr, start_nursery, end_nursery);
4805 DEBUG (9, fprintf (gc_debug_file, "Overwrote remset at %p with %p (count: %d)\n", ptr, *ptr, (int)count));
4806 if (!global && *ptr >= start_nursery && *ptr < end_nursery)
4807 add_to_global_remset (ptr, FALSE);
4812 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
4813 if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery) || !ptr_in_heap (ptr))
4815 scan_object ((char*)ptr, start_nursery, end_nursery);
4817 case REMSET_OTHER: {
4818 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
4822 if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery) || !ptr_in_heap (ptr))
4827 ptr = (void**) scan_vtype ((char*)ptr, desc, start_nursery, end_nursery);
4829 case REMSET_ROOT_LOCATION:
4830 /* Same as REMSET_LOCATION, but the address is not required to be in the heap */
4831 *ptr = copy_object (*ptr, start_nursery, end_nursery);
4832 DEBUG (9, fprintf (gc_debug_file, "Overwrote root location remset at %p with %p\n", ptr, *ptr));
4833 if (!global && *ptr >= start_nursery && *ptr < end_nursery) {
4835 * If the object is pinned, each reference to it from nonpinned objects
4836 * becomes part of the global remset, which can grow very large.
4838 DEBUG (9, fprintf (gc_debug_file, "Add to global remset because of pinning %p (%p %s)\n", ptr, *ptr, safe_name (*ptr)));
4839 add_to_global_remset (ptr, TRUE);
4843 g_assert_not_reached ();
4848 g_assert_not_reached ();
4854 scan_from_remsets (void *start_nursery, void *end_nursery)
4857 SgenThreadInfo *info;
4858 RememberedSet *remset, *next;
4859 mword *p, *next_p, *store_pos;
4861 /* the global one */
4862 for (remset = global_remset; remset; remset = remset->next) {
4863 DEBUG (4, fprintf (gc_debug_file, "Scanning global remset range: %p-%p, size: %zd\n", remset->data, remset->store_next, remset->store_next - remset->data));
4864 store_pos = remset->data;
4865 for (p = remset->data; p < remset->store_next; p = next_p) {
4868 next_p = handle_remset (p, start_nursery, end_nursery, TRUE);
4871 * Clear global remsets of locations which no longer point to the
4872 * nursery. Otherwise, they could grow indefinitely between major
4875 ptr = (p [0] & ~REMSET_TYPE_MASK);
4876 if ((p [0] & REMSET_TYPE_MASK) == REMSET_LOCATION) {
4877 if (ptr_in_nursery (*(void**)ptr))
4878 *store_pos ++ = p [0];
4880 g_assert ((p [0] & REMSET_TYPE_MASK) == REMSET_OTHER);
4881 g_assert (p [1] == REMSET_ROOT_LOCATION);
4882 if (ptr_in_nursery (*(void**)ptr)) {
4883 *store_pos ++ = p [0];
4884 *store_pos ++ = p [1];
4889 /* Truncate the remset */
4890 remset->store_next = store_pos;
4893 /* the per-thread ones */
4894 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
4895 for (info = thread_table [i]; info; info = info->next) {
4896 for (remset = info->remset; remset; remset = next) {
4897 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for thread %p, range: %p-%p, size: %zd\n", info, remset->data, remset->store_next, remset->store_next - remset->data));
4898 for (p = remset->data; p < remset->store_next;) {
4899 p = handle_remset (p, start_nursery, end_nursery, FALSE);
4901 remset->store_next = remset->data;
4902 next = remset->next;
4903 remset->next = NULL;
4904 if (remset != info->remset) {
4905 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
4906 free_internal_mem (remset);
4912 /* the freed thread ones */
4913 while (freed_thread_remsets) {
4914 remset = freed_thread_remsets;
4915 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for freed thread, range: %p-%p, size: %zd\n", remset->data, remset->store_next, remset->store_next - remset->data));
4916 for (p = remset->data; p < remset->store_next;) {
4917 p = handle_remset (p, start_nursery, end_nursery, FALSE);
4919 next = remset->next;
4920 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
4921 free_internal_mem (remset);
4922 freed_thread_remsets = next;
4927 * Clear the info in the remembered sets: we're doing a major collection, so
4928 * the per-thread ones are not needed and the global ones will be reconstructed
4932 clear_remsets (void)
4935 SgenThreadInfo *info;
4936 RememberedSet *remset, *next;
4938 /* the global list */
4939 for (remset = global_remset; remset; remset = next) {
4940 remset->store_next = remset->data;
4941 next = remset->next;
4942 remset->next = NULL;
4943 if (remset != global_remset) {
4944 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
4945 free_internal_mem (remset);
4948 /* the per-thread ones */
4949 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
4950 for (info = thread_table [i]; info; info = info->next) {
4951 for (remset = info->remset; remset; remset = next) {
4952 remset->store_next = remset->data;
4953 next = remset->next;
4954 remset->next = NULL;
4955 if (remset != info->remset) {
4956 DEBUG (1, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
4957 free_internal_mem (remset);
4963 /* the freed thread ones */
4964 while (freed_thread_remsets) {
4965 next = freed_thread_remsets->next;
4966 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", freed_thread_remsets->data));
4967 free_internal_mem (freed_thread_remsets);
4968 freed_thread_remsets = next;
4973 * Clear the thread local TLAB variables for all threads.
4978 SgenThreadInfo *info;
4981 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
4982 for (info = thread_table [i]; info; info = info->next) {
4983 /* A new TLAB will be allocated when the thread does its first allocation */
4984 *info->tlab_start_addr = NULL;
4985 *info->tlab_next_addr = NULL;
4986 *info->tlab_temp_end_addr = NULL;
4987 *info->tlab_real_end_addr = NULL;
4993 * Find the tlab_next value of the TLAB which contains ADDR.
4996 find_tlab_next_from_address (char *addr)
4998 SgenThreadInfo *info;
5001 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5002 for (info = thread_table [i]; info; info = info->next) {
5003 if (addr >= *info->tlab_start_addr && addr < *info->tlab_next_addr)
5004 return *info->tlab_next_addr;
5011 /* LOCKING: assumes the GC lock is held */
5012 static SgenThreadInfo*
5013 gc_register_current_thread (void *addr)
5016 SgenThreadInfo* info = malloc (sizeof (SgenThreadInfo));
5017 SgenThreadInfo *__thread_info__ = info;
5022 #ifndef HAVE_KW_THREAD
5023 info->tlab_start = info->tlab_next = info->tlab_temp_end = info->tlab_real_end = NULL;
5025 g_assert (!pthread_getspecific (thread_info_key));
5026 pthread_setspecific (thread_info_key, info);
5029 info->id = ARCH_GET_THREAD ();
5030 info->stop_count = -1;
5033 info->stack_start = NULL;
5034 info->tlab_start_addr = &TLAB_START;
5035 info->tlab_next_addr = &TLAB_NEXT;
5036 info->tlab_temp_end_addr = &TLAB_TEMP_END;
5037 info->tlab_real_end_addr = &TLAB_REAL_END;
5038 info->stopped_ip = NULL;
5039 info->stopped_domain = NULL;
5040 info->stopped_regs = NULL;
5042 #ifdef HAVE_KW_THREAD
5043 tlab_next_addr = &tlab_next;
5046 /* try to get it with attributes first */
5047 #if defined(HAVE_PTHREAD_GETATTR_NP) && defined(HAVE_PTHREAD_ATTR_GETSTACK)
5051 pthread_attr_t attr;
5052 pthread_getattr_np (pthread_self (), &attr);
5053 pthread_attr_getstack (&attr, &sstart, &size);
5054 info->stack_start_limit = sstart;
5055 info->stack_end = (char*)sstart + size;
5056 pthread_attr_destroy (&attr);
5058 #elif defined(HAVE_PTHREAD_GET_STACKSIZE_NP) && defined(HAVE_PTHREAD_GET_STACKADDR_NP)
5059 info->stack_end = (char*)pthread_get_stackaddr_np (pthread_self ());
5062 /* FIXME: we assume the stack grows down */
5063 gsize stack_bottom = (gsize)addr;
5064 stack_bottom += 4095;
5065 stack_bottom &= ~4095;
5066 info->stack_end = (char*)stack_bottom;
5070 #ifdef HAVE_KW_THREAD
5071 stack_end = info->stack_end;
5074 /* hash into the table */
5075 hash = HASH_PTHREAD_T (info->id) % THREAD_HASH_SIZE;
5076 info->next = thread_table [hash];
5077 thread_table [hash] = info;
5079 info->remset = alloc_remset (DEFAULT_REMSET_SIZE, info);
5080 pthread_setspecific (remembered_set_key, info->remset);
5081 #ifdef HAVE_KW_THREAD
5082 remembered_set = info->remset;
5084 DEBUG (3, fprintf (gc_debug_file, "registered thread %p (%p) (hash: %d)\n", info, (gpointer)info->id, hash));
5086 if (gc_callbacks.thread_attach_func)
5087 info->runtime_data = gc_callbacks.thread_attach_func ();
5093 unregister_current_thread (void)
5096 SgenThreadInfo *prev = NULL;
5098 RememberedSet *rset;
5099 ARCH_THREAD_TYPE id = ARCH_GET_THREAD ();
5101 hash = HASH_PTHREAD_T (id) % THREAD_HASH_SIZE;
5102 p = thread_table [hash];
5104 DEBUG (3, fprintf (gc_debug_file, "unregister thread %p (%p)\n", p, (gpointer)p->id));
5105 while (!ARCH_THREAD_EQUALS (p->id, id)) {
5110 thread_table [hash] = p->next;
5112 prev->next = p->next;
5115 if (freed_thread_remsets) {
5116 for (rset = p->remset; rset->next; rset = rset->next)
5118 rset->next = freed_thread_remsets;
5119 freed_thread_remsets = p->remset;
5121 freed_thread_remsets = p->remset;
5128 unregister_thread (void *k)
5131 unregister_current_thread ();
5136 mono_gc_register_thread (void *baseptr)
5138 SgenThreadInfo *info;
5140 info = thread_info_lookup (ARCH_GET_THREAD ());
5142 info = gc_register_current_thread (baseptr);
5144 return info != NULL;
5147 #if USE_PTHREAD_INTERCEPT
5149 #undef pthread_create
5151 #undef pthread_detach
5154 void *(*start_routine) (void *);
5158 pthread_mutex_t registered_mutex;
5159 pthread_cond_t registered;
5163 } SgenThreadStartInfo;
5166 gc_start_thread (void *arg)
5168 SgenThreadStartInfo *start_info = arg;
5169 SgenThreadInfo* info;
5170 void *t_arg = start_info->arg;
5171 void *(*start_func) (void*) = start_info->start_routine;
5176 info = gc_register_current_thread (&result);
5179 pthread_mutex_lock (&start_info->registered_mutex);
5180 post_result = pthread_cond_signal (&start_info->registered);
5181 pthread_mutex_unlock (&start_info->registered_mutex);
5183 post_result = sem_post (&(start_info->registered));
5185 g_assert (!post_result);
5186 result = start_func (t_arg);
5188 * this is done by the pthread key dtor
5190 unregister_current_thread ();
5198 mono_gc_pthread_create (pthread_t *new_thread, const pthread_attr_t *attr, void *(*start_routine)(void *), void *arg)
5200 SgenThreadStartInfo *start_info;
5203 start_info = malloc (sizeof (SgenThreadStartInfo));
5207 pthread_mutex_init (&start_info->registered_mutex, NULL);
5208 pthread_mutex_lock (&start_info->registered_mutex);
5209 pthread_cond_init (&start_info->registered, NULL);
5211 result = sem_init (&(start_info->registered), 0, 0);
5214 start_info->arg = arg;
5215 start_info->start_routine = start_routine;
5217 result = pthread_create (new_thread, attr, gc_start_thread, start_info);
5220 result = pthread_cond_wait (&start_info->registered, &start_info->registered_mutex);
5223 while (sem_wait (&(start_info->registered)) != 0) {
5224 /*if (EINTR != errno) ABORT("sem_wait failed"); */
5229 pthread_mutex_unlock (&start_info->registered_mutex);
5230 pthread_mutex_destroy (&start_info->registered_mutex);
5231 pthread_cond_destroy (&start_info->registered);
5233 sem_destroy (&(start_info->registered));
5240 mono_gc_pthread_join (pthread_t thread, void **retval)
5242 return pthread_join (thread, retval);
5246 mono_gc_pthread_detach (pthread_t thread)
5248 return pthread_detach (thread);
5251 #endif /* USE_PTHREAD_INTERCEPT */
5254 * ######################################################################
5255 * ######## Write barriers
5256 * ######################################################################
5259 static RememberedSet*
5260 alloc_remset (int size, gpointer id) {
5261 RememberedSet* res = get_internal_mem (sizeof (RememberedSet) + (size * sizeof (gpointer)));
5262 res->store_next = res->data;
5263 res->end_set = res->data + size;
5265 DEBUG (4, fprintf (gc_debug_file, "Allocated remset size %d at %p for %p\n", size, res->data, id));
5270 * Note: the write barriers first do the needed GC work and then do the actual store:
5271 * this way the value is visible to the conservative GC scan after the write barrier
5272 * itself. If a GC interrupts the barrier in the middle, value will be kept alive by
5273 * the conservative scan, otherwise by the remembered set scan. FIXME: figure out what
5274 * happens when we need to record which pointers contain references to the new generation.
5275 * The write barrier will be executed, but the pointer is still not stored.
5278 mono_gc_wbarrier_set_field (MonoObject *obj, gpointer field_ptr, MonoObject* value)
5282 if (ptr_in_nursery (field_ptr)) {
5283 *(void**)field_ptr = value;
5286 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", field_ptr));
5287 rs = REMEMBERED_SET;
5288 if (rs->store_next < rs->end_set) {
5289 *(rs->store_next++) = (mword)field_ptr;
5290 *(void**)field_ptr = value;
5293 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
5294 rs->next = REMEMBERED_SET;
5295 REMEMBERED_SET = rs;
5296 #ifdef HAVE_KW_THREAD
5297 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
5299 *(rs->store_next++) = (mword)field_ptr;
5300 *(void**)field_ptr = value;
5304 mono_gc_wbarrier_set_arrayref (MonoArray *arr, gpointer slot_ptr, MonoObject* value)
5308 rs = REMEMBERED_SET;
5309 if (ptr_in_nursery (slot_ptr)) {
5310 *(void**)slot_ptr = value;
5313 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", slot_ptr));
5314 if (rs->store_next < rs->end_set) {
5315 *(rs->store_next++) = (mword)slot_ptr;
5316 *(void**)slot_ptr = value;
5319 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
5320 rs->next = REMEMBERED_SET;
5321 REMEMBERED_SET = rs;
5322 #ifdef HAVE_KW_THREAD
5323 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
5325 *(rs->store_next++) = (mword)slot_ptr;
5326 *(void**)slot_ptr = value;
5330 mono_gc_wbarrier_arrayref_copy (MonoArray *arr, gpointer slot_ptr, int count)
5334 rs = REMEMBERED_SET;
5335 if (ptr_in_nursery (slot_ptr))
5337 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p, %d\n", slot_ptr, count));
5338 if (rs->store_next + 1 < rs->end_set) {
5339 *(rs->store_next++) = (mword)slot_ptr | REMSET_RANGE;
5340 *(rs->store_next++) = count;
5343 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
5344 rs->next = REMEMBERED_SET;
5345 REMEMBERED_SET = rs;
5346 #ifdef HAVE_KW_THREAD
5347 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
5349 *(rs->store_next++) = (mword)slot_ptr | REMSET_RANGE;
5350 *(rs->store_next++) = count;
5354 mono_gc_wbarrier_generic_nostore (gpointer ptr)
5358 if (ptr_in_nursery (ptr) || !ptr_in_heap (ptr)) {
5359 DEBUG (8, fprintf (gc_debug_file, "Skipping remset at %p\n", ptr));
5362 rs = REMEMBERED_SET;
5363 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", ptr));
5364 if (rs->store_next < rs->end_set) {
5365 *(rs->store_next++) = (mword)ptr;
5368 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
5369 rs->next = REMEMBERED_SET;
5370 REMEMBERED_SET = rs;
5371 #ifdef HAVE_KW_THREAD
5372 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
5374 *(rs->store_next++) = (mword)ptr;
5378 mono_gc_wbarrier_generic_store (gpointer ptr, MonoObject* value)
5380 DEBUG (8, fprintf (gc_debug_file, "Wbarrier store at %p to %p (%s)\n", ptr, value, value ? safe_name (value) : "null"));
5381 *(void**)ptr = value;
5382 mono_gc_wbarrier_generic_nostore (ptr);
5386 mono_gc_wbarrier_set_root (gpointer ptr, MonoObject *value)
5390 rs = REMEMBERED_SET;
5391 if (ptr_in_nursery (ptr))
5393 DEBUG (8, fprintf (gc_debug_file, "Adding root remset at %p (%s)\n", ptr, value ? safe_name (value) : "null"));
5395 if (rs->store_next + 2 < rs->end_set) {
5396 *(rs->store_next++) = (mword)ptr | REMSET_OTHER;
5397 *(rs->store_next++) = (mword)REMSET_ROOT_LOCATION;
5398 *(void**)ptr = value;
5401 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
5402 rs->next = REMEMBERED_SET;
5403 REMEMBERED_SET = rs;
5404 #ifdef HAVE_KW_THREAD
5405 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
5407 *(rs->store_next++) = (mword)ptr | REMSET_OTHER;
5408 *(rs->store_next++) = (mword)REMSET_ROOT_LOCATION;
5410 *(void**)ptr = value;
5414 mono_gc_wbarrier_value_copy (gpointer dest, gpointer src, int count, MonoClass *klass)
5418 rs = REMEMBERED_SET;
5419 if (ptr_in_nursery (dest) || !ptr_in_heap (dest))
5421 g_assert (klass->gc_descr_inited);
5422 DEBUG (8, fprintf (gc_debug_file, "Adding value remset at %p, count %d, descr %p for class %s (%p)\n", dest, count, klass->gc_descr, klass->name, klass));
5424 if (rs->store_next + 3 < rs->end_set) {
5425 *(rs->store_next++) = (mword)dest | REMSET_OTHER;
5426 *(rs->store_next++) = (mword)REMSET_VTYPE;
5427 *(rs->store_next++) = (mword)klass->gc_descr;
5428 *(rs->store_next++) = (mword)count;
5431 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
5432 rs->next = REMEMBERED_SET;
5433 REMEMBERED_SET = rs;
5434 #ifdef HAVE_KW_THREAD
5435 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
5437 *(rs->store_next++) = (mword)dest | REMSET_OTHER;
5438 *(rs->store_next++) = (mword)REMSET_VTYPE;
5439 *(rs->store_next++) = (mword)klass->gc_descr;
5440 *(rs->store_next++) = (mword)count;
5444 * mono_gc_wbarrier_object:
5446 * Write barrier to call when obj is the result of a clone or copy of an object.
5449 mono_gc_wbarrier_object (MonoObject* obj)
5453 rs = REMEMBERED_SET;
5454 DEBUG (1, fprintf (gc_debug_file, "Adding object remset for %p\n", obj));
5455 if (rs->store_next < rs->end_set) {
5456 *(rs->store_next++) = (mword)obj | REMSET_OBJECT;
5459 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
5460 rs->next = REMEMBERED_SET;
5461 REMEMBERED_SET = rs;
5462 #ifdef HAVE_KW_THREAD
5463 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
5465 *(rs->store_next++) = (mword)obj | REMSET_OBJECT;
5469 * ######################################################################
5470 * ######## Collector debugging
5471 * ######################################################################
5474 const char*descriptor_types [] = {
5486 describe_ptr (char *ptr)
5488 GCMemSection *section;
5493 if (ptr_in_nursery (ptr)) {
5494 printf ("Pointer inside nursery.\n");
5496 for (section = section_list; section;) {
5497 if (ptr >= section->data && ptr < section->data + section->size)
5499 section = section->next;
5503 printf ("Pointer inside oldspace.\n");
5504 } else if (obj_is_from_pinned_alloc (ptr)) {
5505 printf ("Pointer is inside a pinned chunk.\n");
5507 printf ("Pointer unknown.\n");
5512 if (object_is_pinned (ptr))
5513 printf ("Object is pinned.\n");
5515 if (object_is_forwarded (ptr))
5516 printf ("Object is forwared.\n");
5518 // FIXME: Handle pointers to the inside of objects
5519 vtable = (MonoVTable*)LOAD_VTABLE (ptr);
5521 printf ("VTable: %p\n", vtable);
5522 if (vtable == NULL) {
5523 printf ("VTable is invalid (empty).\n");
5526 if (ptr_in_nursery (vtable)) {
5527 printf ("VTable is invalid (points inside nursery).\n");
5530 printf ("Class: %s\n", vtable->klass->name);
5532 desc = ((GCVTable*)vtable)->desc;
5533 printf ("Descriptor: %lx\n", desc);
5536 printf ("Descriptor type: %d (%s)\n", type, descriptor_types [type]);
5540 find_in_remset_loc (mword *p, char *addr, gboolean *found)
5546 switch ((*p) & REMSET_TYPE_MASK) {
5547 case REMSET_LOCATION:
5548 if (*p == (mword)addr)
5552 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
5554 if ((void**)addr >= ptr && (void**)addr < ptr + count)
5558 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
5559 count = safe_object_get_size ((MonoObject*)ptr);
5560 count += (ALLOC_ALIGN - 1);
5561 count &= (ALLOC_ALIGN - 1);
5562 count /= sizeof (mword);
5563 if ((void**)addr >= ptr && (void**)addr < ptr + count)
5566 case REMSET_OTHER: {
5569 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
5573 switch (desc & 0x7) {
5574 case DESC_TYPE_RUN_LENGTH:
5575 OBJ_RUN_LEN_SIZE (skip_size, desc, ptr);
5576 /* The descriptor includes the size of MonoObject */
5577 skip_size -= sizeof (MonoObject);
5579 if ((void**)addr >= ptr && (void**)addr < ptr + (skip_size / sizeof (gpointer)))
5584 g_assert_not_reached ();
5588 case REMSET_ROOT_LOCATION:
5591 g_assert_not_reached ();
5596 g_assert_not_reached ();
5602 * Return whenever ADDR occurs in the remembered sets
5605 find_in_remsets (char *addr)
5608 SgenThreadInfo *info;
5609 RememberedSet *remset;
5611 gboolean found = FALSE;
5613 /* the global one */
5614 for (remset = global_remset; remset; remset = remset->next) {
5615 DEBUG (4, fprintf (gc_debug_file, "Scanning global remset range: %p-%p, size: %zd\n", remset->data, remset->store_next, remset->store_next - remset->data));
5616 for (p = remset->data; p < remset->store_next;) {
5617 p = find_in_remset_loc (p, addr, &found);
5622 /* the per-thread ones */
5623 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5624 for (info = thread_table [i]; info; info = info->next) {
5625 for (remset = info->remset; remset; remset = remset->next) {
5626 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for thread %p, range: %p-%p, size: %zd\n", info, remset->data, remset->store_next, remset->store_next - remset->data));
5627 for (p = remset->data; p < remset->store_next;) {
5628 p = find_in_remset_loc (p, addr, &found);
5636 /* the freed thread ones */
5637 for (remset = freed_thread_remsets; remset; remset = remset->next) {
5638 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for freed thread, range: %p-%p, size: %zd\n", remset->data, remset->store_next, remset->store_next - remset->data));
5639 for (p = remset->data; p < remset->store_next;) {
5640 p = find_in_remset_loc (p, addr, &found);
5650 #define HANDLE_PTR(ptr,obj) do { \
5651 if (*(ptr) && (char*)*(ptr) >= nursery_start && (char*)*(ptr) < nursery_next) { \
5652 if (!find_in_remsets ((char*)(ptr))) { \
5653 fprintf (gc_debug_file, "Oldspace->newspace reference %p at offset %zd in object %p (%s.%s) not found in remsets.\n", *(ptr), (char*)(ptr) - (char*)(obj), (obj), ((MonoObject*)(obj))->vtable->klass->name_space, ((MonoObject*)(obj))->vtable->klass->name); \
5654 g_assert_not_reached (); \
5660 * Check that each object reference inside the area which points into the nursery
5661 * can be found in the remembered sets.
5663 static void __attribute__((noinline))
5664 check_remsets_for_area (char *start, char *end)
5667 int type_str = 0, type_rlen = 0, type_bitmap = 0, type_vector = 0, type_lbit = 0, type_complex = 0;
5668 new_obj_references = 0;
5669 obj_references_checked = 0;
5670 while (start < end) {
5671 if (!*(void**)start) {
5672 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
5675 vt = (GCVTable*)LOAD_VTABLE (start);
5676 DEBUG (8, fprintf (gc_debug_file, "Scanning object %p, vtable: %p (%s)\n", start, vt, vt->klass->name));
5678 MonoObject *obj = (MonoObject*)start;
5679 g_print ("found at %p (0x%lx): %s.%s\n", start, (long)vt->desc, obj->vtable->klass->name_space, obj->vtable->klass->name);
5682 #define SCAN_OBJECT_ACTION COUNT_OBJECT_TYPES
5683 #include "sgen-scan-object.h"
5688 * Perform consistency check of the heap.
5690 * Assumes the world is stopped.
5693 check_consistency (void)
5695 GCMemSection *section;
5697 // Need to add more checks
5698 // FIXME: Create a general heap enumeration function and use that
5700 DEBUG (1, fprintf (gc_debug_file, "Begin heap consistency check...\n"));
5702 // Check that oldspace->newspace pointers are registered with the collector
5703 for (section = section_list; section; section = section->next) {
5704 if (section->role == MEMORY_ROLE_GEN0)
5706 DEBUG (2, fprintf (gc_debug_file, "Scan of old section: %p-%p, size: %d\n", section->data, section->next_data, (int)(section->next_data - section->data)));
5707 check_remsets_for_area (section->data, section->next_data);
5710 DEBUG (1, fprintf (gc_debug_file, "Heap consistency check done.\n"));
5713 /* Check that the reference is valid */
5715 #define HANDLE_PTR(ptr,obj) do { \
5717 g_assert (safe_name (*(ptr)) != NULL); \
5724 * Perform consistency check on an object. Currently we only check that the
5725 * reference fields are valid.
5728 check_object (char *start)
5733 #include "sgen-scan-object.h"
5739 * ######################################################################
5740 * ######## Other mono public interface functions.
5741 * ######################################################################
5745 mono_gc_collect (int generation)
5749 if (generation == 0) {
5750 collect_nursery (0);
5752 major_collection ();
5759 mono_gc_max_generation (void)
5765 mono_gc_collection_count (int generation)
5767 if (generation == 0)
5768 return num_minor_gcs;
5769 return num_major_gcs;
5773 mono_gc_get_used_size (void)
5776 GCMemSection *section;
5778 tot = los_memory_usage;
5779 for (section = section_list; section; section = section->next) {
5780 /* this is approximate... */
5781 tot += section->next_data - section->data;
5783 /* FIXME: account for pinned objects */
5789 mono_gc_get_heap_size (void)
5795 mono_gc_disable (void)
5803 mono_gc_enable (void)
5811 mono_object_is_alive (MonoObject* o)
5817 mono_gc_get_generation (MonoObject *obj)
5819 if (ptr_in_nursery (obj))
5825 mono_gc_enable_events (void)
5830 mono_gc_weak_link_add (void **link_addr, MonoObject *obj, gboolean track)
5833 mono_gc_register_disappearing_link (obj, link_addr, track);
5838 mono_gc_weak_link_remove (void **link_addr)
5841 mono_gc_register_disappearing_link (NULL, link_addr, FALSE);
5846 mono_gc_weak_link_get (void **link_addr)
5850 return (MonoObject*) REVEAL_POINTER (*link_addr);
5854 mono_gc_make_descr_from_bitmap (gsize *bitmap, int numbits)
5856 if (numbits < ((sizeof (*bitmap) * 8) - ROOT_DESC_TYPE_SHIFT)) {
5857 return (void*)MAKE_ROOT_DESC (ROOT_DESC_BITMAP, bitmap [0]);
5859 mword complex = alloc_complex_descriptor (bitmap, numbits + 1);
5860 return (void*)MAKE_ROOT_DESC (ROOT_DESC_COMPLEX, complex);
5865 mono_gc_make_root_descr_user (MonoGCMarkFunc marker)
5869 g_assert (user_descriptors_next < MAX_USER_DESCRIPTORS);
5870 descr = (void*)MAKE_ROOT_DESC (ROOT_DESC_USER, (mword)user_descriptors_next);
5871 user_descriptors [user_descriptors_next ++] = marker;
5877 mono_gc_alloc_fixed (size_t size, void *descr)
5879 /* FIXME: do a single allocation */
5880 void *res = calloc (1, size);
5883 if (!mono_gc_register_root (res, size, descr)) {
5891 mono_gc_free_fixed (void* addr)
5893 mono_gc_deregister_root (addr);
5898 mono_gc_is_gc_thread (void)
5902 result = thread_info_lookup (ARCH_GET_THREAD ()) != NULL;
5908 mono_gc_base_init (void)
5912 struct sigaction sinfo;
5914 LOCK_INIT (gc_mutex);
5916 if (gc_initialized) {
5920 pagesize = mono_pagesize ();
5921 gc_debug_file = stderr;
5922 if ((env = getenv ("MONO_GC_DEBUG"))) {
5923 opts = g_strsplit (env, ",", -1);
5924 for (ptr = opts; ptr && *ptr; ptr ++) {
5926 if (opt [0] >= '0' && opt [0] <= '9') {
5927 gc_debug_level = atoi (opt);
5932 char *rf = g_strdup_printf ("%s.%d", opt, getpid ());
5933 gc_debug_file = fopen (rf, "wb");
5935 gc_debug_file = stderr;
5938 } else if (!strcmp (opt, "collect-before-allocs")) {
5939 collect_before_allocs = TRUE;
5940 } else if (!strcmp (opt, "check-at-minor-collections")) {
5941 consistency_check_at_minor_collection = TRUE;
5942 } else if (!strcmp (opt, "clear-at-gc")) {
5943 nursery_clear_policy = CLEAR_AT_GC;
5945 fprintf (stderr, "Invalid format for the MONO_GC_DEBUG env variable: '%s'\n", env);
5946 fprintf (stderr, "The format is: MONO_GC_DEBUG=[l[:filename]|<option>]+ where l is a debug level 0-9.\n");
5947 fprintf (stderr, "Valid options are: collect-before-allocs, check-at-minor-collections, clear-at-gc.\n");
5956 char *name = g_strdup_printf ("/mono/%d/suspacksem", getpid ());
5957 suspend_ack_semaphore_ptr = sem_open (name, O_CREAT | O_EXCL, S_IRWXU, 0);
5958 if (suspend_ack_semaphore_ptr == SEM_FAILED)
5959 g_error ("sem_open");
5964 suspend_ack_semaphore_ptr = &suspend_ack_semaphore;
5965 sem_init (&suspend_ack_semaphore, 0, 0);
5968 sigfillset (&sinfo.sa_mask);
5969 sinfo.sa_flags = SA_RESTART | SA_SIGINFO;
5970 sinfo.sa_sigaction = suspend_handler;
5971 if (sigaction (suspend_signal_num, &sinfo, NULL) != 0) {
5972 g_error ("failed sigaction");
5975 sinfo.sa_handler = restart_handler;
5976 if (sigaction (restart_signal_num, &sinfo, NULL) != 0) {
5977 g_error ("failed sigaction");
5980 sigfillset (&suspend_signal_mask);
5981 sigdelset (&suspend_signal_mask, restart_signal_num);
5983 global_remset = alloc_remset (1024, NULL);
5984 global_remset->next = NULL;
5986 pthread_key_create (&remembered_set_key, unregister_thread);
5988 #ifndef HAVE_KW_THREAD
5989 pthread_key_create (&thread_info_key, NULL);
5992 gc_initialized = TRUE;
5994 mono_gc_register_thread (&sinfo);
6002 #ifdef HAVE_KW_THREAD
6003 #define EMIT_TLS_ACCESS(mb,dummy,offset) do { \
6004 mono_mb_emit_byte ((mb), MONO_CUSTOM_PREFIX); \
6005 mono_mb_emit_byte ((mb), CEE_MONO_TLS); \
6006 mono_mb_emit_i4 ((mb), (offset)); \
6009 #define EMIT_TLS_ACCESS(mb,member,dummy) do { \
6010 mono_mb_emit_byte ((mb), MONO_CUSTOM_PREFIX); \
6011 mono_mb_emit_byte ((mb), CEE_MONO_TLS); \
6012 mono_mb_emit_i4 ((mb), thread_info_key); \
6013 mono_mb_emit_icon ((mb), G_STRUCT_OFFSET (SgenThreadInfo, member)); \
6014 mono_mb_emit_byte ((mb), CEE_ADD); \
6015 mono_mb_emit_byte ((mb), CEE_LDIND_I); \
6019 #ifdef MANAGED_ALLOCATION
6020 /* FIXME: Do this in the JIT, where specialized allocation sequences can be created
6021 * for each class. This is currently not easy to do, as it is hard to generate basic
6022 * blocks + branches, but it is easy with the linear IL codebase.
6025 create_allocator (int atype)
6027 int p_var, size_var;
6028 guint32 slowpath_branch;
6029 MonoMethodBuilder *mb;
6031 MonoMethodSignature *csig;
6032 static gboolean registered = FALSE;
6033 int tlab_next_addr_var, new_next_var;
6035 #ifdef HAVE_KW_THREAD
6036 int tlab_next_addr_offset = -1;
6037 int tlab_temp_end_offset = -1;
6039 MONO_THREAD_VAR_OFFSET (tlab_next_addr, tlab_next_addr_offset);
6040 MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
6042 g_assert (tlab_next_addr_offset != -1);
6043 g_assert (tlab_temp_end_offset != -1);
6046 g_assert (atype == ATYPE_NORMAL);
6049 mono_register_jit_icall (mono_gc_alloc_obj, "mono_gc_alloc_obj", mono_create_icall_signature ("object ptr int"), FALSE);
6053 csig = mono_metadata_signature_alloc (mono_defaults.corlib, 1);
6054 csig->ret = &mono_defaults.object_class->byval_arg;
6055 csig->params [0] = &mono_defaults.int_class->byval_arg;
6057 mb = mono_mb_new (mono_defaults.object_class, "Alloc", MONO_WRAPPER_ALLOC);
6058 size_var = mono_mb_add_local (mb, &mono_defaults.int32_class->byval_arg);
6059 /* size = vtable->klass->instance_size; */
6060 mono_mb_emit_ldarg (mb, 0);
6061 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoVTable, klass));
6062 mono_mb_emit_byte (mb, CEE_ADD);
6063 mono_mb_emit_byte (mb, CEE_LDIND_I);
6064 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoClass, instance_size));
6065 mono_mb_emit_byte (mb, CEE_ADD);
6066 /* FIXME: assert instance_size stays a 4 byte integer */
6067 mono_mb_emit_byte (mb, CEE_LDIND_U4);
6068 mono_mb_emit_stloc (mb, size_var);
6070 /* size += ALLOC_ALIGN - 1; */
6071 mono_mb_emit_ldloc (mb, size_var);
6072 mono_mb_emit_icon (mb, ALLOC_ALIGN - 1);
6073 mono_mb_emit_byte (mb, CEE_ADD);
6074 /* size &= ~(ALLOC_ALIGN - 1); */
6075 mono_mb_emit_icon (mb, ~(ALLOC_ALIGN - 1));
6076 mono_mb_emit_byte (mb, CEE_AND);
6077 mono_mb_emit_stloc (mb, size_var);
6080 * We need to modify tlab_next, but the JIT only supports reading, so we read
6081 * another tls var holding its address instead.
6084 /* tlab_next_addr (local) = tlab_next_addr (TLS var) */
6085 tlab_next_addr_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
6086 EMIT_TLS_ACCESS (mb, tlab_next_addr, tlab_next_addr_offset);
6087 mono_mb_emit_stloc (mb, tlab_next_addr_var);
6089 /* p = (void**)tlab_next; */
6090 p_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
6091 mono_mb_emit_ldloc (mb, tlab_next_addr_var);
6092 mono_mb_emit_byte (mb, CEE_LDIND_I);
6093 mono_mb_emit_stloc (mb, p_var);
6095 /* new_next = (char*)p + size; */
6096 new_next_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
6097 mono_mb_emit_ldloc (mb, p_var);
6098 mono_mb_emit_ldloc (mb, size_var);
6099 mono_mb_emit_byte (mb, CEE_CONV_I);
6100 mono_mb_emit_byte (mb, CEE_ADD);
6101 mono_mb_emit_stloc (mb, new_next_var);
6103 /* tlab_next = new_next */
6104 mono_mb_emit_ldloc (mb, tlab_next_addr_var);
6105 mono_mb_emit_ldloc (mb, new_next_var);
6106 mono_mb_emit_byte (mb, CEE_STIND_I);
6108 /* if (G_LIKELY (new_next < tlab_temp_end)) */
6109 mono_mb_emit_ldloc (mb, new_next_var);
6110 EMIT_TLS_ACCESS (mb, tlab_temp_end, tlab_temp_end_offset);
6111 slowpath_branch = mono_mb_emit_short_branch (mb, MONO_CEE_BLT_UN_S);
6115 mono_mb_emit_byte (mb, MONO_CUSTOM_PREFIX);
6116 mono_mb_emit_byte (mb, CEE_MONO_NOT_TAKEN);
6118 /* FIXME: mono_gc_alloc_obj takes a 'size_t' as an argument, not an int32 */
6119 mono_mb_emit_ldarg (mb, 0);
6120 mono_mb_emit_ldloc (mb, size_var);
6121 mono_mb_emit_icall (mb, mono_gc_alloc_obj);
6122 mono_mb_emit_byte (mb, CEE_RET);
6125 mono_mb_patch_short_branch (mb, slowpath_branch);
6127 /* FIXME: Memory barrier */
6130 mono_mb_emit_ldloc (mb, p_var);
6131 mono_mb_emit_ldarg (mb, 0);
6132 mono_mb_emit_byte (mb, CEE_STIND_I);
6135 mono_mb_emit_ldloc (mb, p_var);
6136 mono_mb_emit_byte (mb, CEE_RET);
6138 res = mono_mb_create_method (mb, csig, 8);
6140 mono_method_get_header (res)->init_locals = FALSE;
6144 static MonoMethod* alloc_method_cache [ATYPE_NUM];
6147 is_ip_in_managed_allocator (MonoDomain *domain, gpointer ip)
6155 ji = mono_jit_info_table_find (domain, ip);
6158 method = ji->method;
6160 for (i = 0; i < ATYPE_NUM; ++i)
6161 if (method == alloc_method_cache [i])
6168 * Generate an allocator method implementing the fast path of mono_gc_alloc_obj ().
6169 * The signature of the called method is:
6170 * object allocate (MonoVTable *vtable)
6173 mono_gc_get_managed_allocator (MonoVTable *vtable, gboolean for_box)
6175 #ifdef MANAGED_ALLOCATION
6176 MonoClass *klass = vtable->klass;
6178 #ifdef HAVE_KW_THREAD
6179 int tlab_next_offset = -1;
6180 int tlab_temp_end_offset = -1;
6181 MONO_THREAD_VAR_OFFSET (tlab_next, tlab_next_offset);
6182 MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
6184 if (tlab_next_offset == -1 || tlab_temp_end_offset == -1)
6188 if (!mono_runtime_has_tls_get ())
6190 if (klass->instance_size > tlab_size)
6192 if (klass->has_finalize || klass->marshalbyref || (mono_profiler_get_events () & MONO_PROFILE_ALLOCATIONS))
6196 if (klass->byval_arg.type == MONO_TYPE_STRING)
6198 if (collect_before_allocs)
6201 return mono_gc_get_managed_allocator_by_type (0);
6208 mono_gc_get_managed_allocator_type (MonoMethod *managed_alloc)
6214 mono_gc_get_managed_allocator_by_type (int atype)
6216 #ifdef MANAGED_ALLOCATION
6219 if (!mono_runtime_has_tls_get ())
6222 mono_loader_lock ();
6223 res = alloc_method_cache [atype];
6225 res = alloc_method_cache [atype] = create_allocator (atype);
6226 mono_loader_unlock ();
6234 mono_gc_get_managed_allocator_types (void)
6239 static MonoMethod *write_barrier_method;
6242 mono_gc_get_write_barrier (void)
6245 MonoMethodBuilder *mb;
6246 MonoMethodSignature *sig;
6247 #ifdef MANAGED_WBARRIER
6248 int label_no_wb, label_need_wb_1, label_need_wb_2, label2;
6249 int remset_var, next_var, dummy_var;
6251 #ifdef HAVE_KW_THREAD
6252 int remset_offset = -1, stack_end_offset = -1;
6254 MONO_THREAD_VAR_OFFSET (remembered_set, remset_offset);
6255 MONO_THREAD_VAR_OFFSET (stack_end, stack_end_offset);
6256 g_assert (remset_offset != -1 && stack_end_offset != -1);
6260 // FIXME: Maybe create a separate version for ctors (the branch would be
6261 // correctly predicted more times)
6262 if (write_barrier_method)
6263 return write_barrier_method;
6265 /* Create the IL version of mono_gc_barrier_generic_store () */
6266 sig = mono_metadata_signature_alloc (mono_defaults.corlib, 1);
6267 sig->ret = &mono_defaults.void_class->byval_arg;
6268 sig->params [0] = &mono_defaults.int_class->byval_arg;
6270 mb = mono_mb_new (mono_defaults.object_class, "wbarrier", MONO_WRAPPER_WRITE_BARRIER);
6272 #ifdef MANAGED_WBARRIER
6273 if (mono_runtime_has_tls_get ()) {
6274 /* ptr_in_nursery () check */
6275 #ifdef ALIGN_NURSERY
6277 * Masking out the bits might be faster, but we would have to use 64 bit
6278 * immediates, which might be slower.
6280 mono_mb_emit_ldarg (mb, 0);
6281 mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
6282 mono_mb_emit_byte (mb, CEE_SHR_UN);
6283 mono_mb_emit_icon (mb, (mword)nursery_start >> DEFAULT_NURSERY_BITS);
6284 label_no_wb = mono_mb_emit_branch (mb, CEE_BEQ);
6287 g_assert_not_reached ();
6290 /* Need write barrier if ptr >= stack_end */
6291 mono_mb_emit_ldarg (mb, 0);
6292 EMIT_TLS_ACCESS (mb, stack_end, stack_end_offset);
6293 label_need_wb_1 = mono_mb_emit_branch (mb, CEE_BGE_UN);
6295 /* Need write barrier if ptr < stack_start */
6296 dummy_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
6297 mono_mb_emit_ldarg (mb, 0);
6298 mono_mb_emit_ldloc_addr (mb, dummy_var);
6299 label_need_wb_2 = mono_mb_emit_branch (mb, CEE_BLE_UN);
6301 /* Don't need write barrier case */
6302 mono_mb_patch_branch (mb, label_no_wb);
6304 mono_mb_emit_byte (mb, CEE_RET);
6306 /* Need write barrier case */
6307 mono_mb_patch_branch (mb, label_need_wb_1);
6308 mono_mb_patch_branch (mb, label_need_wb_2);
6310 // remset_var = remembered_set;
6311 remset_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
6312 EMIT_TLS_ACCESS (mb, remset, remset_offset);
6313 mono_mb_emit_stloc (mb, remset_var);
6315 // next_var = rs->store_next
6316 next_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
6317 mono_mb_emit_ldloc (mb, remset_var);
6318 mono_mb_emit_ldflda (mb, G_STRUCT_OFFSET (RememberedSet, store_next));
6319 mono_mb_emit_byte (mb, CEE_LDIND_I);
6320 mono_mb_emit_stloc (mb, next_var);
6322 // if (rs->store_next < rs->end_set) {
6323 mono_mb_emit_ldloc (mb, next_var);
6324 mono_mb_emit_ldloc (mb, remset_var);
6325 mono_mb_emit_ldflda (mb, G_STRUCT_OFFSET (RememberedSet, end_set));
6326 mono_mb_emit_byte (mb, CEE_LDIND_I);
6327 label2 = mono_mb_emit_branch (mb, CEE_BGE);
6329 /* write barrier fast path */
6330 // *(rs->store_next++) = (mword)ptr;
6331 mono_mb_emit_ldloc (mb, next_var);
6332 mono_mb_emit_ldarg (mb, 0);
6333 mono_mb_emit_byte (mb, CEE_STIND_I);
6335 mono_mb_emit_ldloc (mb, next_var);
6336 mono_mb_emit_icon (mb, sizeof (gpointer));
6337 mono_mb_emit_byte (mb, CEE_ADD);
6338 mono_mb_emit_stloc (mb, next_var);
6340 mono_mb_emit_ldloc (mb, remset_var);
6341 mono_mb_emit_ldflda (mb, G_STRUCT_OFFSET (RememberedSet, store_next));
6342 mono_mb_emit_ldloc (mb, next_var);
6343 mono_mb_emit_byte (mb, CEE_STIND_I);
6345 /* write barrier slow path */
6346 mono_mb_patch_branch (mb, label2);
6350 mono_mb_emit_ldarg (mb, 0);
6351 mono_mb_emit_icall (mb, mono_gc_wbarrier_generic_nostore);
6352 mono_mb_emit_byte (mb, CEE_RET);
6354 res = mono_mb_create_method (mb, sig, 16);
6357 mono_loader_lock ();
6358 if (write_barrier_method) {
6359 /* Already created */
6360 mono_free_method (res);
6362 /* double-checked locking */
6363 mono_memory_barrier ();
6364 write_barrier_method = res;
6366 mono_loader_unlock ();
6368 return write_barrier_method;
6371 #endif /* HAVE_SGEN_GC */