2 * sgen-gc.c: Simple generational GC.
5 * Paolo Molaro (lupus@ximian.com)
7 * Copyright 2005-2009 Novell, Inc (http://www.novell.com)
9 * Thread start/stop adapted from Boehm's GC:
10 * Copyright (c) 1994 by Xerox Corporation. All rights reserved.
11 * Copyright (c) 1996 by Silicon Graphics. All rights reserved.
12 * Copyright (c) 1998 by Fergus Henderson. All rights reserved.
13 * Copyright (c) 2000-2004 by Hewlett-Packard Company. All rights reserved.
15 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
16 * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
18 * Permission is hereby granted to use or copy this program
19 * for any purpose, provided the above notices are retained on all copies.
20 * Permission to modify the code and to distribute modified code is granted,
21 * provided the above notices are retained, and a notice that the code was
22 * modified is included with the above copyright notice.
24 * All the rest of the code is LGPL.
26 * Important: allocation provides always zeroed memory, having to do
27 * a memset after allocation is deadly for performance.
28 * Memory usage at startup is currently as follows:
30 * 64 KB internal space
32 * We should provide a small memory config with half the sizes
34 * We currently try to make as few mono assumptions as possible:
35 * 1) 2-word header with no GC pointers in it (first vtable, second to store the
37 * 2) gc descriptor is the second word in the vtable (first word in the class)
38 * 3) 8 byte alignment is the minimum and enough (not true for special structures, FIXME)
39 * 4) there is a function to get an object's size and the number of
40 * elements in an array.
41 * 5) we know the special way bounds are allocated for complex arrays
43 * Always try to keep stack usage to a minimum: no recursive behaviour
44 * and no large stack allocs.
46 * General description.
47 * Objects are initially allocated in a nursery using a fast bump-pointer technique.
48 * When the nursery is full we start a nursery collection: this is performed with a
50 * When the old generation is full we start a copying GC of the old generation as well:
51 * this will be changed to mark/compact in the future.
52 * The things that complicate this description are:
53 * *) pinned objects: we can't move them so we need to keep track of them
54 * *) no precise info of the thread stacks and registers: we need to be able to
55 * quickly find the objects that may be referenced conservatively and pin them
56 * (this makes the first issues more important)
57 * *) large objects are too expensive to be dealt with using copying GC: we handle them
58 * with mark/sweep during major collections
59 * *) some objects need to not move even if they are small (interned strings, Type handles):
60 * we use mark/sweep for them, too: they are not allocated in the nursery, but inside
61 * PinnedChunks regions
66 *) change the jit to emit write barrier calls when needed (we
67 can have specialized write barriers): done with icalls, still need to
68 use some specialized barriers
69 *) we could have a function pointer in MonoClass to implement
70 customized write barriers for value types
71 *) the write barrier code could be isolated in a couple of functions: when a
72 thread is stopped if it's inside the barrier it is let go again
73 until we stop outside of them (not really needed, see below GC-safe points)
74 *) investigate the stuff needed to advance a thread to a GC-safe
75 point (single-stepping, read from unmapped memory etc) and implement it
76 Not needed yet: since we treat the objects reachable from the stack/regs as
77 roots, we store the ptr and exec the write barrier so there is no race.
78 We may need this to solve the issue with setting the length of arrays and strings.
79 We may need this also for handling precise info on stacks, even simple things
80 as having uninitialized data on the stack and having to wait for the prolog
81 to zero it. Not an issue for the last frame that we scan conservatively.
82 We could always not trust the value in the slots anyway.
83 *) make the jit info table lock free
84 *) modify the jit to save info about references in stack locations:
85 this can be done just for locals as a start, so that at least
86 part of the stack is handled precisely.
87 *) Make the debug printf stuff thread and signal safe.
88 *) test/fix 64 bit issues
89 *) test/fix endianess issues
91 *) add batch moving profile info
92 *) add more timing info
93 *) there is a possible race when an array or string is created: the vtable is set,
94 but the length is set only later so if the GC needs to scan the object in that window,
95 it won't get the correct size for the object. The object can't have references and it will
96 be pinned, but a free memory fragment may be created that overlaps with it.
97 We should change the array max_length field to be at the same offset as the string length:
98 this way we can have a single special alloc function for them that sets the length.
99 Multi-dim arrays have the same issue for rank == 1 for the bounds data.
100 *) implement a card table as the write barrier instead of remembered sets?
101 *) some sort of blacklist support?
102 *) fin_ready_list and critical_fin_list are part of the root set, too
103 *) consider lowering the large object min size to 16/32KB or so and benchmark
104 *) once mark-compact is implemented we could still keep the
105 copying collector for the old generation and use it if we think
106 it is better (small heaps and no pinning object in the old
108 *) avoid the memory store from copy_object when not needed.
109 *) optimize the write barriers fastpath to happen in managed code
110 *) add an option to mmap the whole heap in one chunk: it makes for many
111 simplifications in the checks (put the nursery at the top and just use a single
112 check for inclusion/exclusion): the issue this has is that on 32 bit systems it's
113 not flexible (too much of the address space may be used by default or we can't
114 increase the heap as needed) and we'd need a race-free mechanism to return memory
115 back to the system (mprotect(PROT_NONE) will still keep the memory allocated if it
116 was written to, munmap is needed, but the following mmap may not find the same segment
118 *) memzero the fragments after restarting the world and optionally a smaller chunk at a time
119 *) an additional strategy to realloc/expand the nursery when fully pinned is to start
120 allocating objects in the old generation. This means that we can't optimize away write
121 barrier calls in ctors (but that is not valid for other reasons, too).
122 *) add write barriers to the Clone methods
130 #include <semaphore.h>
134 #include <sys/types.h>
135 #include <sys/stat.h>
136 #include <sys/mman.h>
137 #include <sys/time.h>
140 #include "metadata/metadata-internals.h"
141 #include "metadata/class-internals.h"
142 #include "metadata/gc-internal.h"
143 #include "metadata/object-internals.h"
144 #include "metadata/threads.h"
145 #include "metadata/sgen-gc.h"
146 #include "metadata/sgen-archdep.h"
147 #include "metadata/mono-gc.h"
148 #include "metadata/method-builder.h"
149 #include "metadata/profiler-private.h"
150 #include "metadata/monitor.h"
151 #include "metadata/threadpool-internals.h"
152 #include "metadata/mempool-internals.h"
153 #include "utils/mono-mmap.h"
154 #include "utils/mono-semaphore.h"
155 #include "utils/mono-counters.h"
157 #include <mono/utils/memcheck.h>
159 #define OPDEF(a,b,c,d,e,f,g,h,i,j) \
163 #include "mono/cil/opcode.def"
170 * ######################################################################
171 * ######## Types and constants used by the GC.
172 * ######################################################################
174 #if SIZEOF_VOID_P == 4
175 typedef guint32 mword;
177 typedef guint64 mword;
180 static int gc_initialized = 0;
181 static int gc_debug_level = 0;
182 static FILE* gc_debug_file;
183 /* If set, do a minor collection before every allocation */
184 static gboolean collect_before_allocs = FALSE;
185 /* If set, do a heap consistency check before each minor collection */
186 static gboolean consistency_check_at_minor_collection = FALSE;
187 /* If set, check that there are no references to the domain left at domain unload */
188 static gboolean xdomain_checks = FALSE;
189 /* If not null, dump the heap after each collection into this file */
190 static FILE *heap_dump_file = NULL;
193 * Turning on heavy statistics will turn off the managed allocator and
194 * the managed write barrier.
196 //#define HEAVY_STATISTICS
198 #ifdef HEAVY_STATISTICS
199 #define HEAVY_STAT(x) x
201 #define HEAVY_STAT(x)
204 #ifdef HEAVY_STATISTICS
205 static long stat_objects_alloced = 0;
206 static long stat_copy_object_called_nursery = 0;
207 static long stat_objects_copied_nursery = 0;
208 static long stat_copy_object_called_major = 0;
209 static long stat_objects_copied_major = 0;
211 static long stat_copy_object_failed_from_space = 0;
212 static long stat_copy_object_failed_forwarded = 0;
213 static long stat_copy_object_failed_pinned = 0;
214 static long stat_copy_object_failed_large_pinned = 0;
215 static long stat_copy_object_failed_to_space = 0;
217 static long stat_store_remsets = 0;
218 static long stat_store_remsets_unique = 0;
219 static long stat_saved_remsets_1 = 0;
220 static long stat_saved_remsets_2 = 0;
221 static long stat_global_remsets_added = 0;
222 static long stat_global_remsets_processed = 0;
224 static long num_copy_object_called = 0;
225 static long num_objects_copied = 0;
227 static int stat_wbarrier_set_field = 0;
228 static int stat_wbarrier_set_arrayref = 0;
229 static int stat_wbarrier_arrayref_copy = 0;
230 static int stat_wbarrier_generic_store = 0;
231 static int stat_wbarrier_generic_store_remset = 0;
232 static int stat_wbarrier_set_root = 0;
233 static int stat_wbarrier_value_copy = 0;
234 static int stat_wbarrier_object_copy = 0;
237 static long pinned_chunk_bytes_alloced = 0;
238 static long large_internal_bytes_alloced = 0;
241 INTERNAL_MEM_PIN_QUEUE,
242 INTERNAL_MEM_FRAGMENT,
243 INTERNAL_MEM_SECTION,
244 INTERNAL_MEM_SCAN_STARTS,
245 INTERNAL_MEM_FIN_TABLE,
246 INTERNAL_MEM_FINALIZE_ENTRY,
247 INTERNAL_MEM_DISLINK_TABLE,
248 INTERNAL_MEM_DISLINK,
249 INTERNAL_MEM_ROOTS_TABLE,
250 INTERNAL_MEM_ROOT_RECORD,
251 INTERNAL_MEM_STATISTICS,
253 INTERNAL_MEM_GRAY_QUEUE,
254 INTERNAL_MEM_STORE_REMSET,
258 static long small_internal_mem_bytes [INTERNAL_MEM_MAX];
262 mono_gc_flush_info (void)
264 fflush (gc_debug_file);
268 #define MAX_DEBUG_LEVEL 8
269 #define DEBUG(level,a) do {if (G_UNLIKELY ((level) <= MAX_DEBUG_LEVEL && (level) <= gc_debug_level)) a;} while (0)
271 #define TV_DECLARE(name) struct timeval name
272 #define TV_GETTIME(tv) gettimeofday (&(tv), NULL)
273 #define TV_ELAPSED(start,end) (int)((((end).tv_sec - (start).tv_sec) * 1000000) + end.tv_usec - start.tv_usec)
275 #define GC_BITS_PER_WORD (sizeof (mword) * 8)
283 typedef struct _Block Block;
289 /* each request from the OS ends up in a GCMemSection */
290 typedef struct _GCMemSection GCMemSection;
291 struct _GCMemSection {
295 /* pointer where more data could be allocated if it fits */
299 * scan starts is an array of pointers to objects equally spaced in the allocation area
300 * They let use quickly find pinned objects from pinning pointers.
303 /* in major collections indexes in the pin_queue for objects that pin this section */
306 unsigned short num_scan_start;
307 gboolean is_to_space;
310 #define SIZEOF_GC_MEM_SECTION ((sizeof (GCMemSection) + 7) & ~7)
312 /* large object space struct: 64+ KB */
313 /* we could make this limit much smaller to avoid memcpy copy
314 * and potentially have more room in the GC descriptor: need to measure
315 * This also means that such small OS objects will need to be
316 * allocated in a different way (using pinned chunks).
317 * We may want to put large but smaller than 64k objects in the fixed space
318 * when we move the object from one generation to another (to limit the
319 * pig in the snake effect).
320 * Note: it may be worth to have an optimized copy function, since we can
321 * assume that objects are aligned and have a multiple of 8 size.
322 * FIXME: This structure needs to be a multiple of 8 bytes in size: this is not
323 * true if MONO_ZERO_LEN_ARRAY is nonzero.
325 typedef struct _LOSObject LOSObject;
328 mword size; /* this is the object size */
329 int dummy; /* to have a sizeof (LOSObject) a multiple of ALLOC_ALIGN and data starting at same alignment */
332 char data [MONO_ZERO_LEN_ARRAY];
335 /* Pinned objects are allocated in the LOS space if bigger than half a page
336 * or from freelists otherwise. We assume that pinned objects are relatively few
337 * and they have a slow dying speed (like interned strings, thread objects).
338 * As such they will be collected only at major collections.
339 * free lists are not global: when we need memory we allocate a PinnedChunk.
340 * Each pinned chunk is made of several pages, the first of wich is used
341 * internally for bookeeping (here think of a page as 4KB). The bookeeping
342 * includes the freelists vectors and info about the object size of each page
343 * in the pinned chunk. So, when needed, a free page is found in a pinned chunk,
344 * a size is assigned to it, the page is divided in the proper chunks and each
345 * chunk is added to the freelist. To not waste space, the remaining space in the
346 * first page is used as objects of size 16 or 32 (need to measure which are more
348 * We use this same structure to allocate memory used internally by the GC, so
349 * we never use malloc/free if we need to alloc during collection: the world is stopped
350 * and malloc/free will deadlock.
351 * When we want to iterate over pinned objects, we just scan a page at a time
352 * linearly according to the size of objects in the page: the next pointer used to link
353 * the items in the freelist uses the same word as the vtable. Since we keep freelists
354 * for each pinned chunk, if the word points outside the pinned chunk it means
356 * We could avoid this expensive scanning in creative ways. We could have a policy
357 * of putting in the pinned space only objects we know about that have no struct fields
358 * with references and we can easily use a even expensive write barrier for them,
359 * since pointer writes on such objects should be rare.
360 * The best compromise is to just alloc interned strings and System.MonoType in them.
361 * It would be nice to allocate MonoThread in it, too: must check that we properly
362 * use write barriers so we don't have to do any expensive scanning of the whole pinned
363 * chunk list during minor collections. We can avoid it now because we alloc in it only
364 * reference-free objects.
366 #define PINNED_FIRST_SLOT_SIZE (sizeof (gpointer) * 4)
367 #define MAX_FREELIST_SIZE 2048
368 #define PINNED_PAGE_SIZE (4096)
369 #define PINNED_CHUNK_MIN_SIZE (4096*8)
370 typedef struct _PinnedChunk PinnedChunk;
371 struct _PinnedChunk {
374 int *page_sizes; /* a 0 means the page is still unused */
377 void *data [1]; /* page sizes and free lists are stored here */
380 /* The method used to clear the nursery */
381 /* Clearing at nursery collections is the safest, but has bad interactions with caches.
382 * Clearing at TLAB creation is much faster, but more complex and it might expose hard
387 CLEAR_AT_TLAB_CREATION
388 } NurseryClearPolicy;
390 static NurseryClearPolicy nursery_clear_policy = CLEAR_AT_TLAB_CREATION;
393 * If this is set, the nursery is aligned to an address aligned to its size, ie.
394 * a 1MB nursery will be aligned to an address divisible by 1MB. This allows us to
395 * speed up ptr_in_nursery () checks which are very frequent. This requires the
396 * nursery size to be a compile time constant.
398 #define ALIGN_NURSERY 1
401 * The young generation is divided into fragments. This is because
402 * we can hand one fragments to a thread for lock-less fast alloc and
403 * because the young generation ends up fragmented anyway by pinned objects.
404 * Once a collection is done, a list of fragments is created. When doing
405 * thread local alloc we use smallish nurseries so we allow new threads to
406 * allocate memory from gen0 without triggering a collection. Threads that
407 * are found to allocate lots of memory are given bigger fragments. This
408 * should make the finalizer thread use little nursery memory after a while.
409 * We should start assigning threads very small fragments: if there are many
410 * threads the nursery will be full of reserved space that the threads may not
411 * use at all, slowing down allocation speed.
412 * Thread local allocation is done from areas of memory Hotspot calls Thread Local
413 * Allocation Buffers (TLABs).
415 typedef struct _Fragment Fragment;
419 char *fragment_start;
420 char *fragment_limit; /* the current soft limit for allocation */
424 /* the runtime can register areas of memory as roots: we keep two lists of roots,
425 * a pinned root set for conservatively scanned roots and a normal one for
426 * precisely scanned roots (currently implemented as a single list).
428 typedef struct _RootRecord RootRecord;
436 /* for use with write barriers */
437 typedef struct _RememberedSet RememberedSet;
438 struct _RememberedSet {
442 mword data [MONO_ZERO_LEN_ARRAY];
446 * We're never actually using the first element. It's always set to
447 * NULL to simplify the elimination of consecutive duplicate
450 #define STORE_REMSET_BUFFER_SIZE 1024
452 typedef struct _GenericStoreRememberedSet GenericStoreRememberedSet;
453 struct _GenericStoreRememberedSet {
454 GenericStoreRememberedSet *next;
455 /* We need one entry less because the first entry of store
456 remset buffers is always a dummy and we don't copy it. */
457 gpointer data [STORE_REMSET_BUFFER_SIZE - 1];
460 /* we have 4 possible values in the low 2 bits */
462 REMSET_LOCATION, /* just a pointer to the exact location */
463 REMSET_RANGE, /* range of pointer fields */
464 REMSET_OBJECT, /* mark all the object for scanning */
465 REMSET_OTHER, /* all others */
466 REMSET_TYPE_MASK = 0x3
469 /* Subtypes of REMSET_OTHER */
471 REMSET_VTYPE, /* a valuetype array described by a gc descriptor and a count */
472 REMSET_ROOT_LOCATION, /* a location inside a root */
475 #ifdef HAVE_KW_THREAD
476 static __thread RememberedSet *remembered_set MONO_TLS_FAST;
478 static pthread_key_t remembered_set_key;
479 static RememberedSet *global_remset;
480 static RememberedSet *freed_thread_remsets;
481 //static int store_to_global_remset = 0;
482 static GenericStoreRememberedSet *generic_store_remsets = NULL;
484 /* FIXME: later choose a size that takes into account the RememberedSet struct
485 * and doesn't waste any alloc paddin space.
487 #define DEFAULT_REMSET_SIZE 1024
488 static RememberedSet* alloc_remset (int size, gpointer id);
490 /* Structure that corresponds to a MonoVTable: desc is a mword so requires
491 * no cast from a pointer to an integer
498 /* these bits are set in the object vtable: we could merge them since an object can be
499 * either pinned or forwarded but not both.
500 * We store them in the vtable slot because the bits are used in the sync block for
501 * other purposes: if we merge them and alloc the sync blocks aligned to 8 bytes, we can change
502 * this and use bit 3 in the syncblock (with the lower two bits both set for forwarded, that
503 * would be an invalid combination for the monitor and hash code).
504 * The values are already shifted.
505 * The forwarding address is stored in the sync block.
507 #define FORWARDED_BIT 1
509 #define VTABLE_BITS_MASK 0x3
511 /* returns NULL if not forwarded, or the forwarded address */
512 #define object_is_forwarded(obj) (((mword*)(obj))[0] & FORWARDED_BIT? (void*)(((mword*)(obj))[1]): NULL)
513 /* set the forwarded address fw_addr for object obj */
514 #define forward_object(obj,fw_addr) do { \
515 ((mword*)(obj))[0] |= FORWARDED_BIT; \
516 ((mword*)(obj))[1] = (mword)(fw_addr); \
519 #define object_is_pinned(obj) (((mword*)(obj))[0] & PINNED_BIT)
520 #define pin_object(obj) do { \
521 ((mword*)(obj))[0] |= PINNED_BIT; \
523 #define unpin_object(obj) do { \
524 ((mword*)(obj))[0] &= ~PINNED_BIT; \
528 #define ptr_in_nursery(ptr) (((mword)(ptr) & ~((1 << DEFAULT_NURSERY_BITS) - 1)) == (mword)nursery_start)
530 #define ptr_in_nursery(ptr) ((char*)(ptr) >= nursery_start && (char*)(ptr) < nursery_real_end)
534 * Since we set bits in the vtable, use the macro to load it from the pointer to
535 * an object that is potentially pinned.
537 #define LOAD_VTABLE(addr) ((*(mword*)(addr)) & ~VTABLE_BITS_MASK)
540 safe_name (void* obj)
542 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (obj);
543 return vt->klass->name;
547 safe_object_get_size (MonoObject* o)
549 MonoClass *klass = ((MonoVTable*)LOAD_VTABLE (o))->klass;
550 if (klass == mono_defaults.string_class) {
551 return sizeof (MonoString) + 2 * mono_string_length ((MonoString*) o) + 2;
552 } else if (klass->rank) {
553 MonoArray *array = (MonoArray*)o;
554 size_t size = sizeof (MonoArray) + mono_array_element_size (klass) * mono_array_length (array);
555 if (G_UNLIKELY (array->bounds)) {
556 size += sizeof (mono_array_size_t) - 1;
557 size &= ~(sizeof (mono_array_size_t) - 1);
558 size += sizeof (MonoArrayBounds) * klass->rank;
562 /* from a created object: the class must be inited already */
563 return klass->instance_size;
567 static inline gboolean
568 is_maybe_half_constructed (MonoObject *o)
572 klass = ((MonoVTable*)LOAD_VTABLE (o))->klass;
573 if ((klass == mono_defaults.string_class && mono_string_length ((MonoString*)o) == 0) ||
574 (klass->rank && mono_array_length ((MonoArray*)o) == 0))
581 * ######################################################################
582 * ######## Global data.
583 * ######################################################################
585 static LOCK_DECLARE (gc_mutex);
586 static int gc_disabled = 0;
587 static int num_minor_gcs = 0;
588 static int num_major_gcs = 0;
590 /* good sizes are 512KB-1MB: larger ones increase a lot memzeroing time */
591 //#define DEFAULT_NURSERY_SIZE (1024*512*125+4096*118)
592 #define DEFAULT_NURSERY_SIZE (1024*512*2)
593 /* The number of trailing 0 bits in DEFAULT_NURSERY_SIZE */
594 #define DEFAULT_NURSERY_BITS 20
595 #define MAJOR_SECTION_SIZE (128*1024)
596 #define BLOCK_FOR_OBJECT(o) ((Block*)(((mword)(o)) & ~(MAJOR_SECTION_SIZE - 1)))
597 #define MAJOR_SECTION_FOR_OBJECT(o) ((GCMemSection*)BLOCK_FOR_OBJECT ((o)))
598 #define DEFAULT_MINOR_COLLECTION_SECTION_ALLOWANCE (DEFAULT_NURSERY_SIZE * 3 / MAJOR_SECTION_SIZE)
599 #define DEFAULT_LOS_COLLECTION_TARGET (DEFAULT_NURSERY_SIZE * 2)
600 /* to quickly find the head of an object pinned by a conservative address
601 * we keep track of the objects allocated for each SCAN_START_SIZE memory
602 * chunk in the nursery or other memory sections. Larger values have less
603 * memory overhead and bigger runtime cost. 4-8 KB are reasonable values.
605 #define SCAN_START_SIZE (4096*2)
606 /* the minimum size of a fragment that we consider useful for allocation */
607 #define FRAGMENT_MIN_SIZE (512)
608 /* This is a fixed value used for pinned chunks, not the system pagesize */
609 #define FREELIST_PAGESIZE 4096
611 static mword pagesize = 4096;
612 static mword nursery_size = DEFAULT_NURSERY_SIZE;
613 static int section_size_used = 0;
614 static int degraded_mode = 0;
616 static int minor_collection_section_allowance = DEFAULT_MINOR_COLLECTION_SECTION_ALLOWANCE;
617 static int minor_collection_sections_alloced = 0;
618 static int sections_alloced = 0; /* will be reset frequently */
620 static LOSObject *los_object_list = NULL;
621 static mword los_memory_usage = 0;
622 static mword los_num_objects = 0;
623 static mword next_los_collection = 2*1024*1024; /* 2 MB, need to tune */
624 static mword total_alloc = 0;
625 /* use this to tune when to do a major/minor collection */
626 static mword memory_pressure = 0;
628 static GCMemSection *section_list = NULL;
629 static GCMemSection *nursery_section = NULL;
630 static mword lowest_heap_address = ~(mword)0;
631 static mword highest_heap_address = 0;
633 typedef struct _FinalizeEntry FinalizeEntry;
634 struct _FinalizeEntry {
639 typedef struct _FinalizeEntryHashTable FinalizeEntryHashTable;
640 struct _FinalizeEntryHashTable {
641 FinalizeEntry **table;
646 typedef struct _DisappearingLink DisappearingLink;
647 struct _DisappearingLink {
648 DisappearingLink *next;
652 typedef struct _DisappearingLinkHashTable DisappearingLinkHashTable;
653 struct _DisappearingLinkHashTable {
654 DisappearingLink **table;
659 #define LARGE_INTERNAL_MEM_HEADER_MAGIC 0x7d289f3a
661 typedef struct _LargeInternalMemHeader LargeInternalMemHeader;
662 struct _LargeInternalMemHeader {
675 * The link pointer is hidden by negating each bit. We use the lowest
676 * bit of the link (before negation) to store whether it needs
677 * resurrection tracking.
679 #define HIDE_POINTER(p,t) ((gpointer)(~((gulong)(p)|((t)?1:0))))
680 #define REVEAL_POINTER(p) ((gpointer)((~(gulong)(p))&~3L))
682 #define DISLINK_OBJECT(d) (REVEAL_POINTER (*(d)->link))
683 #define DISLINK_TRACK(d) ((~(gulong)(*(d)->link)) & 1)
686 * The finalizable hash has the object as the key, the
687 * disappearing_link hash, has the link address as key.
689 static FinalizeEntryHashTable minor_finalizable_hash;
690 static FinalizeEntryHashTable major_finalizable_hash;
691 /* objects that are ready to be finalized */
692 static FinalizeEntry *fin_ready_list = NULL;
693 static FinalizeEntry *critical_fin_list = NULL;
695 static DisappearingLinkHashTable minor_disappearing_link_hash;
696 static DisappearingLinkHashTable major_disappearing_link_hash;
698 static int num_ready_finalizers = 0;
699 static int no_finalize = 0;
701 /* keep each size a multiple of ALLOC_ALIGN */
702 /* on 64 bit systems 8 is likely completely unused. */
703 static const int freelist_sizes [] = {
704 8, 16, 24, 32, 40, 48, 64, 80,
705 96, 128, 160, 192, 224, 256, 320, 384,
706 448, 512, 584, 680, 816, 1024, 1360, 2048};
707 #define FREELIST_NUM_SLOTS (sizeof (freelist_sizes) / sizeof (freelist_sizes [0]))
709 static char* max_pinned_chunk_addr = NULL;
710 static char* min_pinned_chunk_addr = (char*)-1;
711 /* pinned_chunk_list is used for allocations of objects that are never moved */
712 static PinnedChunk *pinned_chunk_list = NULL;
713 /* internal_chunk_list is used for allocating structures needed by the GC */
714 static PinnedChunk *internal_chunk_list = NULL;
717 obj_is_from_pinned_alloc (char *p)
719 return BLOCK_FOR_OBJECT (p)->role == MEMORY_ROLE_PINNED;
722 static int slot_for_size (size_t size);
725 free_pinned_object (PinnedChunk *chunk, char *obj, size_t size)
727 void **p = (void**)obj;
728 int slot = slot_for_size (size);
730 g_assert (obj >= (char*)chunk->start_data && obj < ((char*)chunk + chunk->num_pages * FREELIST_PAGESIZE));
731 *p = chunk->free_list [slot];
732 chunk->free_list [slot] = p;
736 ROOT_TYPE_NORMAL = 0, /* "normal" roots */
737 ROOT_TYPE_PINNED = 1, /* roots without a GC descriptor */
738 ROOT_TYPE_WBARRIER = 2, /* roots with a write barrier */
742 /* registered roots: the key to the hash is the root start address */
744 * Different kinds of roots are kept separate to speed up pin_from_roots () for example.
746 static RootRecord **roots_hash [ROOT_TYPE_NUM] = { NULL, NULL };
747 static int roots_hash_size [ROOT_TYPE_NUM] = { 0, 0, 0 };
748 static mword roots_size = 0; /* amount of memory in the root set */
749 static int num_roots_entries [ROOT_TYPE_NUM] = { 0, 0, 0 };
752 * The current allocation cursors
753 * We allocate objects in the nursery.
754 * The nursery is the area between nursery_start and nursery_real_end.
755 * Allocation is done from a Thread Local Allocation Buffer (TLAB). TLABs are allocated
756 * from nursery fragments.
757 * tlab_next is the pointer to the space inside the TLAB where the next object will
759 * tlab_temp_end is the pointer to the end of the temporary space reserved for
760 * the allocation: it allows us to set the scan starts at reasonable intervals.
761 * tlab_real_end points to the end of the TLAB.
762 * nursery_frag_real_end points to the end of the currently used nursery fragment.
763 * nursery_first_pinned_start points to the start of the first pinned object in the nursery
764 * nursery_last_pinned_end points to the end of the last pinned object in the nursery
765 * At the next allocation, the area of the nursery where objects can be present is
766 * between MIN(nursery_first_pinned_start, first_fragment_start) and
767 * MAX(nursery_last_pinned_end, nursery_frag_real_end)
769 static char *nursery_start = NULL;
771 /* eventually share with MonoThread? */
772 typedef struct _SgenThreadInfo SgenThreadInfo;
774 struct _SgenThreadInfo {
775 SgenThreadInfo *next;
777 unsigned int stop_count; /* to catch duplicate signals */
782 void *stack_start_limit;
783 char **tlab_next_addr;
784 char **tlab_start_addr;
785 char **tlab_temp_end_addr;
786 char **tlab_real_end_addr;
787 gpointer **store_remset_buffer_addr;
788 int *store_remset_buffer_index_addr;
789 RememberedSet *remset;
790 gpointer runtime_data;
791 gpointer stopped_ip; /* only valid if the thread is stopped */
792 MonoDomain *stopped_domain; /* ditto */
793 gpointer *stopped_regs; /* ditto */
794 #ifndef HAVE_KW_THREAD
799 gpointer *store_remset_buffer;
800 int store_remset_buffer_index;
804 #ifdef HAVE_KW_THREAD
805 #define TLAB_ACCESS_INIT
806 #define TLAB_START tlab_start
807 #define TLAB_NEXT tlab_next
808 #define TLAB_TEMP_END tlab_temp_end
809 #define TLAB_REAL_END tlab_real_end
810 #define REMEMBERED_SET remembered_set
811 #define STORE_REMSET_BUFFER store_remset_buffer
812 #define STORE_REMSET_BUFFER_INDEX store_remset_buffer_index
814 static pthread_key_t thread_info_key;
815 #define TLAB_ACCESS_INIT SgenThreadInfo *__thread_info__ = pthread_getspecific (thread_info_key)
816 #define TLAB_START (__thread_info__->tlab_start)
817 #define TLAB_NEXT (__thread_info__->tlab_next)
818 #define TLAB_TEMP_END (__thread_info__->tlab_temp_end)
819 #define TLAB_REAL_END (__thread_info__->tlab_real_end)
820 #define REMEMBERED_SET (__thread_info__->remset)
821 #define STORE_REMSET_BUFFER (__thread_info__->store_remset_buffer)
822 #define STORE_REMSET_BUFFER_INDEX (__thread_info__->store_remset_buffer_index)
826 * FIXME: What is faster, a TLS variable pointing to a structure, or separate TLS
827 * variables for next+temp_end ?
829 #ifdef HAVE_KW_THREAD
830 static __thread char *tlab_start;
831 static __thread char *tlab_next;
832 static __thread char *tlab_temp_end;
833 static __thread char *tlab_real_end;
834 static __thread gpointer *store_remset_buffer;
835 static __thread int store_remset_buffer_index;
836 /* Used by the managed allocator/wbarrier */
837 static __thread char **tlab_next_addr;
838 static __thread char *stack_end;
839 static __thread int *store_remset_buffer_index_addr;
841 static char *nursery_next = NULL;
842 static char *nursery_frag_real_end = NULL;
843 static char *nursery_real_end = NULL;
844 //static char *nursery_first_pinned_start = NULL;
845 static char *nursery_last_pinned_end = NULL;
847 /* The size of a TLAB */
848 /* The bigger the value, the less often we have to go to the slow path to allocate a new
849 * one, but the more space is wasted by threads not allocating much memory.
851 * FIXME: Make this self-tuning for each thread.
853 static guint32 tlab_size = (1024 * 4);
855 /* fragments that are free and ready to be used for allocation */
856 static Fragment *nursery_fragments = NULL;
857 /* freeelist of fragment structures */
858 static Fragment *fragment_freelist = NULL;
861 * used when moving the objects
863 static char *to_space_bumper = NULL;
864 static char *to_space_top = NULL;
865 static GCMemSection *to_space_section = NULL;
867 /* objects bigger then this go into the large object space */
868 #define MAX_SMALL_OBJ_SIZE MAX_FREELIST_SIZE
870 /* Functions supplied by the runtime to be called by the GC */
871 static MonoGCCallbacks gc_callbacks;
874 * ######################################################################
875 * ######## Macros and function declarations.
876 * ######################################################################
879 #define UPDATE_HEAP_BOUNDARIES(low,high) do { \
880 if ((mword)(low) < lowest_heap_address) \
881 lowest_heap_address = (mword)(low); \
882 if ((mword)(high) > highest_heap_address) \
883 highest_heap_address = (mword)(high); \
885 #define ADDR_IN_HEAP_BOUNDARIES(addr) ((p) >= lowest_heap_address && (p) < highest_heap_address)
888 align_pointer (void *ptr)
890 mword p = (mword)ptr;
891 p += sizeof (gpointer) - 1;
892 p &= ~ (sizeof (gpointer) - 1);
896 /* forward declarations */
897 static void* get_internal_mem (size_t size, int type);
898 static void free_internal_mem (void *addr, int type);
899 static void* get_os_memory (size_t size, int activate);
900 static void free_os_memory (void *addr, size_t size);
901 static G_GNUC_UNUSED void report_internal_mem_usage (void);
903 static int stop_world (void);
904 static int restart_world (void);
905 static void scan_thread_data (void *start_nursery, void *end_nursery, gboolean precise);
906 static void scan_from_remsets (void *start_nursery, void *end_nursery);
907 static void find_pinning_ref_from_thread (char *obj, size_t size);
908 static void update_current_thread_stack (void *start);
909 static GCMemSection* alloc_major_section (void);
910 static void finalize_in_range (char *start, char *end, int generation);
911 static void add_or_remove_disappearing_link (MonoObject *obj, void **link, gboolean track, int generation);
912 static void null_link_in_range (char *start, char *end, int generation);
913 static void null_links_for_domain (MonoDomain *domain, int generation);
914 static gboolean search_fragment_for_size (size_t size);
915 static void mark_pinned_from_addresses (PinnedChunk *chunk, void **start, void **end);
916 static void clear_remsets (void);
917 static void clear_tlabs (void);
918 static char *find_tlab_next_from_address (char *addr);
919 typedef void (*ScanPinnedObjectCallbackFunc) (PinnedChunk*, char*, size_t, void*);
920 static void scan_pinned_objects (ScanPinnedObjectCallbackFunc callback, void *callback_data);
921 static void sweep_pinned_objects (void);
922 static void scan_from_pinned_objects (char *addr_start, char *addr_end);
923 static void free_large_object (LOSObject *obj);
924 static void free_major_section (GCMemSection *section);
925 static void to_space_expand (void);
927 static void mono_gc_register_disappearing_link (MonoObject *obj, void **link, gboolean track);
929 void describe_ptr (char *ptr);
930 void check_consistency (void);
931 char* check_object (char *start);
933 void mono_gc_scan_for_specific_ref (MonoObject *key);
936 * ######################################################################
937 * ######## GC descriptors
938 * ######################################################################
939 * Used to quickly get the info the GC needs about an object: size and
940 * where the references are held.
942 /* objects are aligned to 8 bytes boundaries
943 * A descriptor is a pointer in MonoVTable, so 32 or 64 bits of size.
944 * The low 3 bits define the type of the descriptor. The other bits
945 * depend on the type.
946 * As a general rule the 13 remaining low bits define the size, either
947 * of the whole object or of the elements in the arrays. While for objects
948 * the size is already in bytes, for arrays we need to shift, because
949 * array elements might be smaller than 8 bytes. In case of arrays, we
950 * use two bits to describe what the additional high bits represents,
951 * so the default behaviour can handle element sizes less than 2048 bytes.
952 * The high 16 bits, if 0 it means the object is pointer-free.
953 * This design should make it easy and fast to skip over ptr-free data.
954 * The first 4 types should cover >95% of the objects.
955 * Note that since the size of objects is limited to 64K, larger objects
956 * will be allocated in the large object heap.
957 * If we want 4-bytes alignment, we need to put vector and small bitmap
961 DESC_TYPE_RUN_LENGTH, /* 16 bits aligned byte size | 1-3 (offset, numptr) bytes tuples */
962 DESC_TYPE_SMALL_BITMAP, /* 16 bits aligned byte size | 16-48 bit bitmap */
963 DESC_TYPE_STRING, /* nothing */
964 DESC_TYPE_COMPLEX, /* index for bitmap into complex_descriptors */
965 DESC_TYPE_VECTOR, /* 10 bits element size | 1 bit array | 2 bits desc | element desc */
966 DESC_TYPE_ARRAY, /* 10 bits element size | 1 bit array | 2 bits desc | element desc */
967 DESC_TYPE_LARGE_BITMAP, /* | 29-61 bitmap bits */
968 DESC_TYPE_COMPLEX_ARR, /* index for bitmap into complex_descriptors */
969 /* subtypes for arrays and vectors */
970 DESC_TYPE_V_PTRFREE = 0,/* there are no refs: keep first so it has a zero value */
971 DESC_TYPE_V_REFS, /* all the array elements are refs */
972 DESC_TYPE_V_RUN_LEN, /* elements are run-length encoded as DESC_TYPE_RUN_LENGTH */
973 DESC_TYPE_V_BITMAP /* elements are as the bitmap in DESC_TYPE_SMALL_BITMAP */
976 #define OBJECT_HEADER_WORDS (sizeof(MonoObject)/sizeof(gpointer))
977 #define LOW_TYPE_BITS 3
978 #define SMALL_BITMAP_SHIFT 16
979 #define SMALL_BITMAP_SIZE (GC_BITS_PER_WORD - SMALL_BITMAP_SHIFT)
980 #define VECTOR_INFO_SHIFT 14
981 #define VECTOR_ELSIZE_SHIFT 3
982 #define LARGE_BITMAP_SIZE (GC_BITS_PER_WORD - LOW_TYPE_BITS)
983 #define MAX_SMALL_SIZE ((1 << SMALL_BITMAP_SHIFT) - 1)
984 #define SMALL_SIZE_MASK 0xfff8
985 #define MAX_ELEMENT_SIZE 0x3ff
986 #define ELEMENT_SIZE_MASK (0x3ff << LOW_TYPE_BITS)
987 #define VECTOR_SUBTYPE_PTRFREE (DESC_TYPE_V_PTRFREE << VECTOR_INFO_SHIFT)
988 #define VECTOR_SUBTYPE_REFS (DESC_TYPE_V_REFS << VECTOR_INFO_SHIFT)
989 #define VECTOR_SUBTYPE_RUN_LEN (DESC_TYPE_V_RUN_LEN << VECTOR_INFO_SHIFT)
990 #define VECTOR_SUBTYPE_BITMAP (DESC_TYPE_V_BITMAP << VECTOR_INFO_SHIFT)
992 #define ALLOC_ALIGN 8
995 /* Root bitmap descriptors are simpler: the lower three bits describe the type
996 * and we either have 30/62 bitmap bits or nibble-based run-length,
997 * or a complex descriptor, or a user defined marker function.
1000 ROOT_DESC_CONSERVATIVE, /* 0, so matches NULL value */
1005 ROOT_DESC_TYPE_MASK = 0x7,
1006 ROOT_DESC_TYPE_SHIFT = 3,
1009 #define MAKE_ROOT_DESC(type,val) ((type) | ((val) << ROOT_DESC_TYPE_SHIFT))
1011 #define MAX_USER_DESCRIPTORS 16
1013 static gsize* complex_descriptors = NULL;
1014 static int complex_descriptors_size = 0;
1015 static int complex_descriptors_next = 0;
1016 static MonoGCMarkFunc user_descriptors [MAX_USER_DESCRIPTORS];
1017 static int user_descriptors_next = 0;
1020 alloc_complex_descriptor (gsize *bitmap, int numbits)
1022 int nwords = numbits/GC_BITS_PER_WORD + 2;
1027 res = complex_descriptors_next;
1028 /* linear search, so we don't have duplicates with domain load/unload
1029 * this should not be performance critical or we'd have bigger issues
1030 * (the number and size of complex descriptors should be small).
1032 for (i = 0; i < complex_descriptors_next; ) {
1033 if (complex_descriptors [i] == nwords) {
1034 int j, found = TRUE;
1035 for (j = 0; j < nwords - 1; ++j) {
1036 if (complex_descriptors [i + 1 + j] != bitmap [j]) {
1046 i += complex_descriptors [i];
1048 if (complex_descriptors_next + nwords > complex_descriptors_size) {
1049 int new_size = complex_descriptors_size * 2 + nwords;
1050 complex_descriptors = g_realloc (complex_descriptors, new_size * sizeof (gsize));
1051 complex_descriptors_size = new_size;
1053 DEBUG (6, fprintf (gc_debug_file, "Complex descriptor %d, size: %d (total desc memory: %d)\n", res, nwords, complex_descriptors_size));
1054 complex_descriptors_next += nwords;
1055 complex_descriptors [res] = nwords;
1056 for (i = 0; i < nwords - 1; ++i) {
1057 complex_descriptors [res + 1 + i] = bitmap [i];
1058 DEBUG (6, fprintf (gc_debug_file, "\tvalue: %p\n", (void*)complex_descriptors [res + 1 + i]));
1065 * Descriptor builders.
1068 mono_gc_make_descr_for_string (gsize *bitmap, int numbits)
1070 return (void*) DESC_TYPE_STRING;
1074 mono_gc_make_descr_for_object (gsize *bitmap, int numbits, size_t obj_size)
1076 int first_set = -1, num_set = 0, last_set = -1, i;
1078 size_t stored_size = obj_size;
1079 stored_size += ALLOC_ALIGN - 1;
1080 stored_size &= ~(ALLOC_ALIGN - 1);
1081 for (i = 0; i < numbits; ++i) {
1082 if (bitmap [i / GC_BITS_PER_WORD] & ((gsize)1 << (i % GC_BITS_PER_WORD))) {
1089 if (stored_size <= MAX_SMALL_OBJ_SIZE) {
1090 /* check run-length encoding first: one byte offset, one byte number of pointers
1091 * on 64 bit archs, we can have 3 runs, just one on 32.
1092 * It may be better to use nibbles.
1094 if (first_set < 0) {
1095 desc = DESC_TYPE_RUN_LENGTH | stored_size;
1096 DEBUG (6, fprintf (gc_debug_file, "Ptrfree descriptor %p, size: %zd\n", (void*)desc, stored_size));
1097 return (void*) desc;
1098 } else if (first_set < 256 && num_set < 256 && (first_set + num_set == last_set + 1)) {
1099 desc = DESC_TYPE_RUN_LENGTH | stored_size | (first_set << 16) | (num_set << 24);
1100 DEBUG (6, fprintf (gc_debug_file, "Runlen descriptor %p, size: %zd, first set: %d, num set: %d\n", (void*)desc, stored_size, first_set, num_set));
1101 return (void*) desc;
1103 /* we know the 2-word header is ptr-free */
1104 if (last_set < SMALL_BITMAP_SIZE + OBJECT_HEADER_WORDS) {
1105 desc = DESC_TYPE_SMALL_BITMAP | stored_size | ((*bitmap >> OBJECT_HEADER_WORDS) << SMALL_BITMAP_SHIFT);
1106 DEBUG (6, fprintf (gc_debug_file, "Smallbitmap descriptor %p, size: %zd, last set: %d\n", (void*)desc, stored_size, last_set));
1107 return (void*) desc;
1110 /* we know the 2-word header is ptr-free */
1111 if (last_set < LARGE_BITMAP_SIZE + OBJECT_HEADER_WORDS) {
1112 desc = DESC_TYPE_LARGE_BITMAP | ((*bitmap >> OBJECT_HEADER_WORDS) << LOW_TYPE_BITS);
1113 DEBUG (6, fprintf (gc_debug_file, "Largebitmap descriptor %p, size: %zd, last set: %d\n", (void*)desc, stored_size, last_set));
1114 return (void*) desc;
1116 /* it's a complex object ... */
1117 desc = DESC_TYPE_COMPLEX | (alloc_complex_descriptor (bitmap, last_set + 1) << LOW_TYPE_BITS);
1118 return (void*) desc;
1121 /* If the array holds references, numbits == 1 and the first bit is set in elem_bitmap */
1123 mono_gc_make_descr_for_array (int vector, gsize *elem_bitmap, int numbits, size_t elem_size)
1125 int first_set = -1, num_set = 0, last_set = -1, i;
1126 mword desc = vector? DESC_TYPE_VECTOR: DESC_TYPE_ARRAY;
1127 for (i = 0; i < numbits; ++i) {
1128 if (elem_bitmap [i / GC_BITS_PER_WORD] & ((gsize)1 << (i % GC_BITS_PER_WORD))) {
1135 if (elem_size <= MAX_ELEMENT_SIZE) {
1136 desc |= elem_size << VECTOR_ELSIZE_SHIFT;
1138 return (void*)(desc | VECTOR_SUBTYPE_PTRFREE);
1140 /* Note: we also handle structs with just ref fields */
1141 if (num_set * sizeof (gpointer) == elem_size) {
1142 return (void*)(desc | VECTOR_SUBTYPE_REFS | ((gssize)(-1) << 16));
1144 /* FIXME: try run-len first */
1145 /* Note: we can't skip the object header here, because it's not present */
1146 if (last_set <= SMALL_BITMAP_SIZE) {
1147 return (void*)(desc | VECTOR_SUBTYPE_BITMAP | (*elem_bitmap << 16));
1150 /* it's am array of complex structs ... */
1151 desc = DESC_TYPE_COMPLEX_ARR;
1152 desc |= alloc_complex_descriptor (elem_bitmap, last_set + 1) << LOW_TYPE_BITS;
1153 return (void*) desc;
1156 /* Return the bitmap encoded by a descriptor */
1158 mono_gc_get_bitmap_for_descr (void *descr, int *numbits)
1160 mword d = (mword)descr;
1164 case DESC_TYPE_RUN_LENGTH: {
1165 int first_set = (d >> 16) & 0xff;
1166 int num_set = (d >> 16) & 0xff;
1169 bitmap = g_new0 (gsize, (first_set + num_set + 7) / 8);
1171 for (i = first_set; i < first_set + num_set; ++i)
1172 bitmap [i / GC_BITS_PER_WORD] |= ((gsize)1 << (i % GC_BITS_PER_WORD));
1174 *numbits = first_set + num_set;
1178 case DESC_TYPE_SMALL_BITMAP:
1179 bitmap = g_new0 (gsize, 1);
1181 bitmap [0] = (d >> SMALL_BITMAP_SHIFT) << OBJECT_HEADER_WORDS;
1183 *numbits = GC_BITS_PER_WORD;
1187 g_assert_not_reached ();
1191 /* helper macros to scan and traverse objects, macros because we resue them in many functions */
1192 #define STRING_SIZE(size,str) do { \
1193 (size) = sizeof (MonoString) + 2 * (mono_string_length ((MonoString*)(str)) + 1); \
1194 (size) += (ALLOC_ALIGN - 1); \
1195 (size) &= ~(ALLOC_ALIGN - 1); \
1198 #define OBJ_RUN_LEN_SIZE(size,desc,obj) do { \
1199 (size) = (desc) & 0xfff8; \
1202 #define OBJ_BITMAP_SIZE(size,desc,obj) do { \
1203 (size) = (desc) & 0xfff8; \
1206 //#define PREFETCH(addr) __asm__ __volatile__ (" prefetchnta %0": : "m"(*(char *)(addr)))
1207 #define PREFETCH(addr)
1209 /* code using these macros must define a HANDLE_PTR(ptr) macro that does the work */
1210 #define OBJ_RUN_LEN_FOREACH_PTR(desc,obj) do { \
1211 if ((desc) & 0xffff0000) { \
1212 /* there are pointers */ \
1213 void **_objptr_end; \
1214 void **_objptr = (void**)(obj); \
1215 _objptr += ((desc) >> 16) & 0xff; \
1216 _objptr_end = _objptr + (((desc) >> 24) & 0xff); \
1217 while (_objptr < _objptr_end) { \
1218 HANDLE_PTR (_objptr, (obj)); \
1224 /* a bitmap desc means that there are pointer references or we'd have
1225 * choosen run-length, instead: add an assert to check.
1227 #define OBJ_BITMAP_FOREACH_PTR(desc,obj) do { \
1228 /* there are pointers */ \
1229 void **_objptr = (void**)(obj); \
1230 gsize _bmap = (desc) >> 16; \
1231 _objptr += OBJECT_HEADER_WORDS; \
1233 if ((_bmap & 1)) { \
1234 HANDLE_PTR (_objptr, (obj)); \
1241 #define OBJ_LARGE_BITMAP_FOREACH_PTR(vt,obj) do { \
1242 /* there are pointers */ \
1243 void **_objptr = (void**)(obj); \
1244 gsize _bmap = (vt)->desc >> LOW_TYPE_BITS; \
1245 _objptr += OBJECT_HEADER_WORDS; \
1247 if ((_bmap & 1)) { \
1248 HANDLE_PTR (_objptr, (obj)); \
1255 #define OBJ_COMPLEX_FOREACH_PTR(vt,obj) do { \
1256 /* there are pointers */ \
1257 void **_objptr = (void**)(obj); \
1258 gsize *bitmap_data = complex_descriptors + ((vt)->desc >> LOW_TYPE_BITS); \
1259 int bwords = (*bitmap_data) - 1; \
1260 void **start_run = _objptr; \
1263 MonoObject *myobj = (MonoObject*)obj; \
1264 g_print ("found %d at %p (0x%zx): %s.%s\n", bwords, (obj), (vt)->desc, myobj->vtable->klass->name_space, myobj->vtable->klass->name); \
1266 while (bwords-- > 0) { \
1267 gsize _bmap = *bitmap_data++; \
1268 _objptr = start_run; \
1269 /*g_print ("bitmap: 0x%x/%d at %p\n", _bmap, bwords, _objptr);*/ \
1271 if ((_bmap & 1)) { \
1272 HANDLE_PTR (_objptr, (obj)); \
1277 start_run += GC_BITS_PER_WORD; \
1281 /* this one is untested */
1282 #define OBJ_COMPLEX_ARR_FOREACH_PTR(vt,obj) do { \
1283 /* there are pointers */ \
1284 gsize *mbitmap_data = complex_descriptors + ((vt)->desc >> LOW_TYPE_BITS); \
1285 int mbwords = (*mbitmap_data++) - 1; \
1286 int el_size = mono_array_element_size (((MonoObject*)(obj))->vtable->klass); \
1287 char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
1288 char *e_end = e_start + el_size * mono_array_length ((MonoArray*)(obj)); \
1290 MonoObject *myobj = (MonoObject*)start; \
1291 g_print ("found %d at %p (0x%zx): %s.%s\n", mbwords, (obj), (vt)->desc, myobj->vtable->klass->name_space, myobj->vtable->klass->name); \
1293 while (e_start < e_end) { \
1294 void **_objptr = (void**)e_start; \
1295 gsize *bitmap_data = mbitmap_data; \
1296 unsigned int bwords = mbwords; \
1297 while (bwords-- > 0) { \
1298 gsize _bmap = *bitmap_data++; \
1299 void **start_run = _objptr; \
1300 /*g_print ("bitmap: 0x%x\n", _bmap);*/ \
1302 if ((_bmap & 1)) { \
1303 HANDLE_PTR (_objptr, (obj)); \
1308 _objptr = start_run + GC_BITS_PER_WORD; \
1310 e_start += el_size; \
1314 #define OBJ_VECTOR_FOREACH_PTR(vt,obj) do { \
1315 /* note: 0xffffc000 excludes DESC_TYPE_V_PTRFREE */ \
1316 if ((vt)->desc & 0xffffc000) { \
1317 int el_size = ((vt)->desc >> 3) & MAX_ELEMENT_SIZE; \
1318 /* there are pointers */ \
1319 int etype = (vt)->desc & 0xc000; \
1320 if (etype == (DESC_TYPE_V_REFS << 14)) { \
1321 void **p = (void**)((char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector)); \
1322 void **end_refs = (void**)((char*)p + el_size * mono_array_length ((MonoArray*)(obj))); \
1323 /* Note: this code can handle also arrays of struct with only references in them */ \
1324 while (p < end_refs) { \
1325 HANDLE_PTR (p, (obj)); \
1328 } else if (etype == DESC_TYPE_V_RUN_LEN << 14) { \
1329 int offset = ((vt)->desc >> 16) & 0xff; \
1330 int num_refs = ((vt)->desc >> 24) & 0xff; \
1331 char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
1332 char *e_end = e_start + el_size * mono_array_length ((MonoArray*)(obj)); \
1333 while (e_start < e_end) { \
1334 void **p = (void**)e_start; \
1337 for (i = 0; i < num_refs; ++i) { \
1338 HANDLE_PTR (p + i, (obj)); \
1340 e_start += el_size; \
1342 } else if (etype == DESC_TYPE_V_BITMAP << 14) { \
1343 char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
1344 char *e_end = e_start + el_size * mono_array_length ((MonoArray*)(obj)); \
1345 while (e_start < e_end) { \
1346 void **p = (void**)e_start; \
1347 gsize _bmap = (vt)->desc >> 16; \
1348 /* Note: there is no object header here to skip */ \
1350 if ((_bmap & 1)) { \
1351 HANDLE_PTR (p, (obj)); \
1356 e_start += el_size; \
1362 static mword new_obj_references = 0;
1363 static mword obj_references_checked = 0;
1366 #define HANDLE_PTR(ptr,obj) do { \
1367 if (*(ptr) && (char*)*(ptr) >= nursery_start && (char*)*(ptr) < nursery_next) { \
1368 new_obj_references++; \
1369 /*printf ("bogus ptr %p found at %p in object %p (%s.%s)\n", *(ptr), (ptr), o, o->vtable->klass->name_space, o->vtable->klass->name);*/ \
1371 obj_references_checked++; \
1376 * ######################################################################
1377 * ######## Detecting and removing garbage.
1378 * ######################################################################
1379 * This section of code deals with detecting the objects no longer in use
1380 * and reclaiming the memory.
1383 #define COUNT_OBJECT_TYPES do { \
1384 switch (desc & 0x7) { \
1385 case DESC_TYPE_STRING: type_str++; break; \
1386 case DESC_TYPE_RUN_LENGTH: type_rlen++; break; \
1387 case DESC_TYPE_ARRAY: case DESC_TYPE_VECTOR: type_vector++; break; \
1388 case DESC_TYPE_SMALL_BITMAP: type_bitmap++; break; \
1389 case DESC_TYPE_LARGE_BITMAP: type_lbit++; break; \
1390 case DESC_TYPE_COMPLEX: type_complex++; break; \
1391 case DESC_TYPE_COMPLEX_ARR: type_complex++; break; \
1392 default: g_assert_not_reached (); \
1396 static void __attribute__((noinline))
1397 scan_area (char *start, char *end)
1400 int type_str = 0, type_rlen = 0, type_bitmap = 0, type_vector = 0, type_lbit = 0, type_complex = 0;
1401 new_obj_references = 0;
1402 obj_references_checked = 0;
1403 while (start < end) {
1404 if (!*(void**)start) {
1405 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
1408 vt = (GCVTable*)LOAD_VTABLE (start);
1409 DEBUG (8, fprintf (gc_debug_file, "Scanning object %p, vtable: %p (%s)\n", start, vt, vt->klass->name));
1411 MonoObject *obj = (MonoObject*)start;
1412 g_print ("found at %p (0x%zx): %s.%s\n", start, vt->desc, obj->vtable->klass->name_space, obj->vtable->klass->name);
1415 #define SCAN_OBJECT_ACTION COUNT_OBJECT_TYPES
1416 #include "sgen-scan-object.h"
1418 /*printf ("references to new nursery %p-%p (size: %dk): %d, checked: %d\n", old_start, end, (end-old_start)/1024, new_obj_references, obj_references_checked);
1419 printf ("\tstrings: %d, runl: %d, vector: %d, bitmaps: %d, lbitmaps: %d, complex: %d\n",
1420 type_str, type_rlen, type_vector, type_bitmap, type_lbit, type_complex);*/
1424 is_xdomain_ref_allowed (gpointer *ptr, char *obj, MonoDomain *domain)
1426 MonoObject *o = (MonoObject*)(obj);
1427 MonoObject *ref = (MonoObject*)*(ptr);
1428 int offset = (char*)(ptr) - (char*)o;
1430 if (o->vtable->klass == mono_defaults.thread_class && offset == G_STRUCT_OFFSET (MonoThread, internal_thread))
1432 if (o->vtable->klass == mono_defaults.internal_thread_class && offset == G_STRUCT_OFFSET (MonoInternalThread, current_appcontext))
1434 if (mono_class_has_parent (o->vtable->klass, mono_defaults.real_proxy_class) &&
1435 offset == G_STRUCT_OFFSET (MonoRealProxy, unwrapped_server))
1437 /* Thread.cached_culture_info */
1438 if (!strcmp (ref->vtable->klass->name_space, "System.Globalization") &&
1439 !strcmp (ref->vtable->klass->name, "CultureInfo") &&
1440 !strcmp(o->vtable->klass->name_space, "System") &&
1441 !strcmp(o->vtable->klass->name, "Object[]"))
1444 * at System.IO.MemoryStream.InternalConstructor (byte[],int,int,bool,bool) [0x0004d] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.IO/MemoryStream.cs:121
1445 * at System.IO.MemoryStream..ctor (byte[]) [0x00017] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.IO/MemoryStream.cs:81
1446 * at (wrapper remoting-invoke-with-check) System.IO.MemoryStream..ctor (byte[]) <IL 0x00020, 0xffffffff>
1447 * at System.Runtime.Remoting.Messaging.CADMethodCallMessage.GetArguments () [0x0000d] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Messaging/CADMessages.cs:327
1448 * at System.Runtime.Remoting.Messaging.MethodCall..ctor (System.Runtime.Remoting.Messaging.CADMethodCallMessage) [0x00017] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Messaging/MethodCall.cs:87
1449 * at System.AppDomain.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage,byte[]&,System.Runtime.Remoting.Messaging.CADMethodReturnMessage&) [0x00018] in /home/schani/Work/novell/trunk/mcs/class/corlib/System/AppDomain.cs:1213
1450 * at (wrapper remoting-invoke-with-check) System.AppDomain.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage,byte[]&,System.Runtime.Remoting.Messaging.CADMethodReturnMessage&) <IL 0x0003d, 0xffffffff>
1451 * at System.Runtime.Remoting.Channels.CrossAppDomainSink.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage) [0x00008] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Channels/CrossAppDomainChannel.cs:198
1452 * at (wrapper runtime-invoke) object.runtime_invoke_CrossAppDomainSink/ProcessMessageRes_object_object (object,intptr,intptr,intptr) <IL 0x0004c, 0xffffffff>
1454 if (!strcmp (ref->vtable->klass->name_space, "System") &&
1455 !strcmp (ref->vtable->klass->name, "Byte[]") &&
1456 !strcmp (o->vtable->klass->name_space, "System.IO") &&
1457 !strcmp (o->vtable->klass->name, "MemoryStream"))
1459 /* append_job() in threadpool.c */
1460 if (!strcmp (ref->vtable->klass->name_space, "System.Runtime.Remoting.Messaging") &&
1461 !strcmp (ref->vtable->klass->name, "AsyncResult") &&
1462 !strcmp (o->vtable->klass->name_space, "System") &&
1463 !strcmp (o->vtable->klass->name, "Object[]") &&
1464 mono_thread_pool_is_queue_array ((MonoArray*) o))
1470 check_reference_for_xdomain (gpointer *ptr, char *obj, MonoDomain *domain)
1472 MonoObject *o = (MonoObject*)(obj);
1473 MonoObject *ref = (MonoObject*)*(ptr);
1474 int offset = (char*)(ptr) - (char*)o;
1476 MonoClassField *field;
1479 if (!ref || ref->vtable->domain == domain)
1481 if (is_xdomain_ref_allowed (ptr, obj, domain))
1485 for (class = o->vtable->klass; class; class = class->parent) {
1488 for (i = 0; i < class->field.count; ++i) {
1489 if (class->fields[i].offset == offset) {
1490 field = &class->fields[i];
1498 if (ref->vtable->klass == mono_defaults.string_class)
1499 str = mono_string_to_utf8 ((MonoString*)ref);
1502 g_print ("xdomain reference in %p (%s.%s) at offset %d (%s) to %p (%s.%s) (%s) - pointed to by:\n",
1503 o, o->vtable->klass->name_space, o->vtable->klass->name,
1504 offset, field ? field->name : "",
1505 ref, ref->vtable->klass->name_space, ref->vtable->klass->name, str ? str : "");
1506 mono_gc_scan_for_specific_ref (o);
1512 #define HANDLE_PTR(ptr,obj) check_reference_for_xdomain ((ptr), (obj), domain)
1515 scan_object_for_xdomain_refs (char *start)
1517 MonoDomain *domain = ((MonoObject*)start)->vtable->domain;
1519 #include "sgen-scan-object.h"
1525 scan_area_for_xdomain_refs (char *start, char *end)
1527 while (start < end) {
1528 if (!*(void**)start) {
1529 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
1533 start = scan_object_for_xdomain_refs (start);
1538 #define HANDLE_PTR(ptr,obj) do { \
1539 if ((MonoObject*)*(ptr) == key) { \
1540 g_print ("found ref to %p in object %p (%s) at offset %d\n", \
1541 key, (obj), safe_name ((obj)), ((char*)(ptr) - (char*)(obj))); \
1546 scan_object_for_specific_ref (char *start, MonoObject *key)
1548 #include "sgen-scan-object.h"
1554 scan_area_for_specific_ref (char *start, char *end, MonoObject *key)
1556 while (start < end) {
1557 if (!*(void**)start) {
1558 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
1562 start = scan_object_for_specific_ref (start, key);
1567 scan_pinned_object_for_specific_ref_callback (PinnedChunk *chunk, char *obj, size_t size, MonoObject *key)
1569 scan_object_for_specific_ref (obj, key);
1573 check_root_obj_specific_ref (RootRecord *root, MonoObject *key, MonoObject *obj)
1577 g_print ("found ref to %p in root record %p\n", key, root);
1580 static MonoObject *check_key = NULL;
1581 static RootRecord *check_root = NULL;
1584 check_root_obj_specific_ref_from_marker (void *obj)
1586 check_root_obj_specific_ref (check_root, check_key, obj);
1591 scan_roots_for_specific_ref (MonoObject *key, int root_type)
1596 for (i = 0; i < roots_hash_size [root_type]; ++i) {
1597 for (root = roots_hash [root_type][i]; root; root = root->next) {
1598 void **start_root = (void**)root->start_root;
1599 mword desc = root->root_desc;
1603 switch (desc & ROOT_DESC_TYPE_MASK) {
1604 case ROOT_DESC_BITMAP:
1605 desc >>= ROOT_DESC_TYPE_SHIFT;
1608 check_root_obj_specific_ref (root, key, *start_root);
1613 case ROOT_DESC_COMPLEX: {
1614 gsize *bitmap_data = complex_descriptors + (desc >> ROOT_DESC_TYPE_SHIFT);
1615 int bwords = (*bitmap_data) - 1;
1616 void **start_run = start_root;
1618 while (bwords-- > 0) {
1619 gsize bmap = *bitmap_data++;
1620 void **objptr = start_run;
1623 check_root_obj_specific_ref (root, key, *objptr);
1627 start_run += GC_BITS_PER_WORD;
1631 case ROOT_DESC_USER: {
1632 MonoGCMarkFunc marker = user_descriptors [desc >> ROOT_DESC_TYPE_SHIFT];
1633 marker (start_root, check_root_obj_specific_ref_from_marker);
1636 case ROOT_DESC_RUN_LEN:
1637 g_assert_not_reached ();
1639 g_assert_not_reached ();
1648 mono_gc_scan_for_specific_ref (MonoObject *key)
1650 GCMemSection *section;
1655 for (section = section_list; section; section = section->block.next)
1656 scan_area_for_specific_ref (section->data, section->end_data, key);
1658 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1659 scan_object_for_specific_ref (bigobj->data, key);
1661 scan_pinned_objects ((ScanPinnedObjectCallbackFunc)scan_pinned_object_for_specific_ref_callback, key);
1663 scan_roots_for_specific_ref (key, ROOT_TYPE_NORMAL);
1664 scan_roots_for_specific_ref (key, ROOT_TYPE_WBARRIER);
1666 for (i = 0; i < roots_hash_size [ROOT_TYPE_PINNED]; ++i) {
1667 for (root = roots_hash [ROOT_TYPE_PINNED][i]; root; root = root->next) {
1668 void **ptr = (void**)root->start_root;
1670 while (ptr < (void**)root->end_root) {
1671 check_root_obj_specific_ref (root, *ptr, key);
1679 need_remove_object_for_domain (char *start, MonoDomain *domain)
1681 if (mono_object_domain (start) == domain) {
1682 DEBUG (1, fprintf (gc_debug_file, "Need to cleanup object %p\n", start));
1689 process_object_for_domain_clearing (char *start, MonoDomain *domain)
1691 GCVTable *vt = (GCVTable*)LOAD_VTABLE (start);
1692 if (vt->klass == mono_defaults.internal_thread_class)
1693 g_assert (mono_object_domain (start) == mono_get_root_domain ());
1694 /* The object could be a proxy for an object in the domain
1696 if (mono_class_has_parent (vt->klass, mono_defaults.real_proxy_class)) {
1697 MonoObject *server = ((MonoRealProxy*)start)->unwrapped_server;
1699 /* The server could already have been zeroed out, so
1700 we need to check for that, too. */
1701 if (server && (!LOAD_VTABLE (server) || mono_object_domain (server) == domain)) {
1702 DEBUG (1, fprintf (gc_debug_file, "Cleaning up remote pointer in %p to object %p\n",
1704 ((MonoRealProxy*)start)->unwrapped_server = NULL;
1709 static void __attribute__((noinline))
1710 scan_area_for_domain (MonoDomain *domain, char *start, char *end)
1715 while (start < end) {
1716 if (!*(void**)start) {
1717 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
1720 vt = (GCVTable*)LOAD_VTABLE (start);
1721 process_object_for_domain_clearing (start, domain);
1722 remove = need_remove_object_for_domain (start, domain);
1723 if (remove && ((MonoObject*)start)->synchronisation) {
1724 void **dislink = mono_monitor_get_object_monitor_weak_link ((MonoObject*)start);
1726 mono_gc_register_disappearing_link (NULL, dislink, FALSE);
1729 #define SCAN_OBJECT_NOSCAN
1730 #define SCAN_OBJECT_ACTION do { \
1731 if (remove) memset (start, 0, skip_size); \
1733 #include "sgen-scan-object.h"
1737 static MonoDomain *check_domain = NULL;
1740 check_obj_not_in_domain (void *o)
1742 g_assert (((MonoObject*)o)->vtable->domain != check_domain);
1747 scan_for_registered_roots_in_domain (MonoDomain *domain, int root_type)
1751 check_domain = domain;
1752 for (i = 0; i < roots_hash_size [root_type]; ++i) {
1753 for (root = roots_hash [root_type][i]; root; root = root->next) {
1754 void **start_root = (void**)root->start_root;
1755 mword desc = root->root_desc;
1757 /* The MonoDomain struct is allowed to hold
1758 references to objects in its own domain. */
1759 if (start_root == (void**)domain)
1762 switch (desc & ROOT_DESC_TYPE_MASK) {
1763 case ROOT_DESC_BITMAP:
1764 desc >>= ROOT_DESC_TYPE_SHIFT;
1766 if ((desc & 1) && *start_root)
1767 check_obj_not_in_domain (*start_root);
1772 case ROOT_DESC_COMPLEX: {
1773 gsize *bitmap_data = complex_descriptors + (desc >> ROOT_DESC_TYPE_SHIFT);
1774 int bwords = (*bitmap_data) - 1;
1775 void **start_run = start_root;
1777 while (bwords-- > 0) {
1778 gsize bmap = *bitmap_data++;
1779 void **objptr = start_run;
1781 if ((bmap & 1) && *objptr)
1782 check_obj_not_in_domain (*objptr);
1786 start_run += GC_BITS_PER_WORD;
1790 case ROOT_DESC_USER: {
1791 MonoGCMarkFunc marker = user_descriptors [desc >> ROOT_DESC_TYPE_SHIFT];
1792 marker (start_root, check_obj_not_in_domain);
1795 case ROOT_DESC_RUN_LEN:
1796 g_assert_not_reached ();
1798 g_assert_not_reached ();
1802 check_domain = NULL;
1806 clear_domain_process_pinned_object_callback (PinnedChunk *chunk, char *obj, size_t size, MonoDomain *domain)
1808 process_object_for_domain_clearing (obj, domain);
1812 clear_domain_free_pinned_object_callback (PinnedChunk *chunk, char *obj, size_t size, MonoDomain *domain)
1814 if (need_remove_object_for_domain (obj, domain))
1815 free_pinned_object (chunk, obj, size);
1819 scan_pinned_object_for_xdomain_refs_callback (PinnedChunk *chunk, char *obj, size_t size, gpointer dummy)
1821 scan_object_for_xdomain_refs (obj);
1825 check_for_xdomain_refs (void)
1827 GCMemSection *section;
1830 for (section = section_list; section; section = section->block.next)
1831 scan_area_for_xdomain_refs (section->data, section->end_data);
1833 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1834 scan_object_for_xdomain_refs (bigobj->data);
1836 scan_pinned_objects (scan_pinned_object_for_xdomain_refs_callback, NULL);
1840 * When appdomains are unloaded we can easily remove objects that have finalizers,
1841 * but all the others could still be present in random places on the heap.
1842 * We need a sweep to get rid of them even though it's going to be costly
1844 * The reason we need to remove them is because we access the vtable and class
1845 * structures to know the object size and the reference bitmap: once the domain is
1846 * unloaded the point to random memory.
1849 mono_gc_clear_domain (MonoDomain * domain)
1851 GCMemSection *section;
1852 LOSObject *bigobj, *prev;
1857 /* Clear all remaining nursery fragments */
1858 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION) {
1859 g_assert (nursery_next <= nursery_frag_real_end);
1860 memset (nursery_next, 0, nursery_frag_real_end - nursery_next);
1861 for (frag = nursery_fragments; frag; frag = frag->next) {
1862 memset (frag->fragment_start, 0, frag->fragment_end - frag->fragment_start);
1866 if (xdomain_checks && domain != mono_get_root_domain ()) {
1867 scan_for_registered_roots_in_domain (domain, ROOT_TYPE_NORMAL);
1868 scan_for_registered_roots_in_domain (domain, ROOT_TYPE_WBARRIER);
1869 check_for_xdomain_refs ();
1872 for (section = section_list; section; section = section->block.next) {
1873 scan_area_for_domain (domain, section->data, section->end_data);
1876 /* We need two passes over pinned and large objects because
1877 freeing such an object gives its memory back to the OS (in
1878 the case of large objects) or obliterates its vtable
1879 (pinned objects), but we might need to dereference a
1880 pointer from an object to another object if the first
1881 object is a proxy. */
1882 scan_pinned_objects ((ScanPinnedObjectCallbackFunc)clear_domain_process_pinned_object_callback, domain);
1883 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1884 process_object_for_domain_clearing (bigobj->data, domain);
1887 for (bigobj = los_object_list; bigobj;) {
1888 if (need_remove_object_for_domain (bigobj->data, domain)) {
1889 LOSObject *to_free = bigobj;
1891 prev->next = bigobj->next;
1893 los_object_list = bigobj->next;
1894 bigobj = bigobj->next;
1895 DEBUG (1, fprintf (gc_debug_file, "Freeing large object %p\n",
1897 free_large_object (to_free);
1901 bigobj = bigobj->next;
1903 scan_pinned_objects ((ScanPinnedObjectCallbackFunc)clear_domain_free_pinned_object_callback, domain);
1905 for (i = GENERATION_NURSERY; i < GENERATION_MAX; ++i)
1906 null_links_for_domain (domain, i);
1912 * add_to_global_remset:
1914 * The global remset contains locations which point into newspace after
1915 * a minor collection. This can happen if the objects they point to are pinned.
1918 add_to_global_remset (gpointer ptr, gboolean root)
1922 DEBUG (8, fprintf (gc_debug_file, "Adding global remset for %p\n", ptr));
1925 g_assert (!ptr_in_nursery (ptr) && ptr_in_nursery (*(gpointer*)ptr));
1927 HEAVY_STAT (++stat_global_remsets_added);
1930 * FIXME: If an object remains pinned, we need to add it at every minor collection.
1931 * To avoid uncontrolled growth of the global remset, only add each pointer once.
1933 if (global_remset->store_next + 3 < global_remset->end_set) {
1935 *(global_remset->store_next++) = (mword)ptr | REMSET_OTHER;
1936 *(global_remset->store_next++) = (mword)REMSET_ROOT_LOCATION;
1938 *(global_remset->store_next++) = (mword)ptr;
1942 rs = alloc_remset (global_remset->end_set - global_remset->data, NULL);
1943 rs->next = global_remset;
1946 *(global_remset->store_next++) = (mword)ptr | REMSET_OTHER;
1947 *(global_remset->store_next++) = (mword)REMSET_ROOT_LOCATION;
1949 *(global_remset->store_next++) = (mword)ptr;
1953 int global_rs_size = 0;
1955 for (rs = global_remset; rs; rs = rs->next) {
1956 global_rs_size += rs->store_next - rs->data;
1958 DEBUG (4, fprintf (gc_debug_file, "Global remset now has size %d\n", global_rs_size));
1962 #include "sgen-gray.c"
1965 * This is how the copying happens from the nursery to the old generation.
1966 * We assume that at this time all the pinned objects have been identified and
1968 * We run scan_object() for each pinned object so that each referenced
1969 * objects if possible are copied. The new gray objects created can have
1970 * scan_object() run on them right away, too.
1971 * Then we run copy_object() for the precisely tracked roots. At this point
1972 * all the roots are either gray or black. We run scan_object() on the gray
1973 * objects until no more gray objects are created.
1974 * At the end of the process we walk again the pinned list and we unmark
1975 * the pinned flag. As we go we also create the list of free space for use
1976 * in the next allocation runs.
1978 * We need to remember objects from the old generation that point to the new one
1979 * (or just addresses?).
1981 * copy_object could be made into a macro once debugged (use inline for now).
1984 static char* __attribute__((noinline))
1985 copy_object (char *obj, char *from_space_start, char *from_space_end)
1987 static void *copy_labels [] = { &&LAB_0, &&LAB_1, &&LAB_2, &&LAB_3, &&LAB_4, &&LAB_5, &&LAB_6, &&LAB_7, &&LAB_8 };
1993 HEAVY_STAT (++num_copy_object_called);
1995 if (!(obj >= from_space_start && obj < from_space_end)) {
1996 DEBUG (9, fprintf (gc_debug_file, "Not copying %p because it's not in from space (%p-%p)\n",
1997 obj, from_space_start, from_space_end));
1998 HEAVY_STAT (++stat_copy_object_failed_from_space);
2002 DEBUG (9, fprintf (gc_debug_file, "Precise copy of %p", obj));
2005 * obj must belong to one of:
2010 * 4. a non-to-space section of the major heap
2011 * 5. a to-space section of the major heap
2013 * In addition, objects in 1, 2 and 4 might also be pinned.
2014 * Objects in 1 and 4 might be forwarded.
2016 * Before we can copy the object we must make sure that we are
2017 * allowed to, i.e. that the object not pinned, not already
2018 * forwarded and doesn't belong to the LOS, a pinned chunk, or
2019 * a to-space section.
2021 * We are usually called for to-space objects (5) when we have
2022 * two remset entries for the same reference. The first entry
2023 * copies the object and updates the reference and the second
2024 * calls us with the updated reference that points into
2025 * to-space. There might also be other circumstances where we
2026 * get to-space objects.
2029 if ((forwarded = object_is_forwarded (obj))) {
2030 g_assert (((MonoVTable*)LOAD_VTABLE(obj))->gc_descr);
2031 DEBUG (9, fprintf (gc_debug_file, " (already forwarded to %p)\n", forwarded));
2032 HEAVY_STAT (++stat_copy_object_failed_forwarded);
2035 if (object_is_pinned (obj)) {
2036 g_assert (((MonoVTable*)LOAD_VTABLE(obj))->gc_descr);
2037 DEBUG (9, fprintf (gc_debug_file, " (pinned, no change)\n"));
2038 HEAVY_STAT (++stat_copy_object_failed_pinned);
2042 objsize = safe_object_get_size ((MonoObject*)obj);
2043 objsize += ALLOC_ALIGN - 1;
2044 objsize &= ~(ALLOC_ALIGN - 1);
2046 if (ptr_in_nursery (obj))
2050 * At this point we know obj is not pinned, not forwarded and
2051 * belongs to 2, 3, 4, or 5.
2053 * LOS object (2) are simple, at least until we always follow
2054 * the rule: if objsize > MAX_SMALL_OBJ_SIZE, pin the object
2055 * and return it. At the end of major collections, we walk
2056 * the los list and if the object is pinned, it is marked,
2057 * otherwise it can be freed.
2059 * Pinned chunks (3) and major heap sections (4, 5) both
2060 * reside in blocks, which are always aligned, so once we've
2061 * eliminated LOS objects, we can just access the block and
2062 * see whether it's a pinned chunk or a major heap section.
2064 if (G_UNLIKELY (objsize > MAX_SMALL_OBJ_SIZE || obj_is_from_pinned_alloc (obj))) {
2065 DEBUG (9, fprintf (gc_debug_file, " (marked LOS/Pinned %p (%s), size: %zd)\n", obj, safe_name (obj), objsize));
2067 HEAVY_STAT (++stat_copy_object_failed_large_pinned);
2072 * Now we know the object is in a major heap section. All we
2073 * need to do is check whether it's already in to-space (5) or
2076 if (MAJOR_SECTION_FOR_OBJECT (obj)->is_to_space) {
2077 g_assert (objsize <= MAX_SMALL_OBJ_SIZE);
2078 DEBUG (9, fprintf (gc_debug_file, " (already copied)\n"));
2079 HEAVY_STAT (++stat_copy_object_failed_to_space);
2084 DEBUG (9, fprintf (gc_debug_file, " (to %p, %s size: %zd)\n", to_space_bumper, ((MonoObject*)obj)->vtable->klass->name, objsize));
2086 HEAVY_STAT (++num_objects_copied);
2088 /* Make sure we have enough space available */
2089 if (to_space_bumper + objsize > to_space_top) {
2091 g_assert (to_space_bumper + objsize <= to_space_top);
2094 if (objsize <= sizeof (gpointer) * 8) {
2095 mword *dest = (mword*)to_space_bumper;
2096 goto *copy_labels [objsize / sizeof (gpointer)];
2098 (dest) [7] = ((mword*)obj) [7];
2100 (dest) [6] = ((mword*)obj) [6];
2102 (dest) [5] = ((mword*)obj) [5];
2104 (dest) [4] = ((mword*)obj) [4];
2106 (dest) [3] = ((mword*)obj) [3];
2108 (dest) [2] = ((mword*)obj) [2];
2110 (dest) [1] = ((mword*)obj) [1];
2112 (dest) [0] = ((mword*)obj) [0];
2120 char* edi = to_space_bumper;
2121 __asm__ __volatile__(
2123 : "=&c" (ecx), "=&D" (edi), "=&S" (esi)
2124 : "0" (objsize/4), "1" (edi),"2" (esi)
2129 memcpy (to_space_bumper, obj, objsize);
2132 /* adjust array->bounds */
2133 vt = ((MonoObject*)obj)->vtable;
2134 g_assert (vt->gc_descr);
2135 if (G_UNLIKELY (vt->rank && ((MonoArray*)obj)->bounds)) {
2136 MonoArray *array = (MonoArray*)to_space_bumper;
2137 array->bounds = (MonoArrayBounds*)((char*)to_space_bumper + ((char*)((MonoArray*)obj)->bounds - (char*)obj));
2138 DEBUG (9, fprintf (gc_debug_file, "Array instance %p: size: %zd, rank: %d, length: %d\n", array, objsize, vt->rank, mono_array_length (array)));
2140 /* set the forwarding pointer */
2141 forward_object (obj, to_space_bumper);
2142 obj = to_space_bumper;
2143 to_space_section->scan_starts [((char*)obj - (char*)to_space_section->data)/SCAN_START_SIZE] = obj;
2144 to_space_bumper += objsize;
2145 DEBUG (9, fprintf (gc_debug_file, "Enqueuing gray object %p (%s)\n", obj, safe_name (obj)));
2146 gray_object_enqueue (obj);
2147 DEBUG (8, g_assert (to_space_bumper <= to_space_top));
2152 #define HANDLE_PTR(ptr,obj) do { \
2153 void *__old = *(ptr); \
2156 *(ptr) = __copy = copy_object (__old, from_start, from_end); \
2157 DEBUG (9, if (__old != __copy) fprintf (gc_debug_file, "Overwrote field at %p with %p (was: %p)\n", (ptr), *(ptr), __old)); \
2158 if (G_UNLIKELY (ptr_in_nursery (__copy) && !ptr_in_nursery ((ptr)))) \
2159 add_to_global_remset ((ptr), FALSE); \
2164 * Scan the object pointed to by @start for references to
2165 * other objects between @from_start and @from_end and copy
2166 * them to the gray_objects area.
2167 * Returns a pointer to the end of the object.
2170 scan_object (char *start, char* from_start, char* from_end)
2172 #include "sgen-scan-object.h"
2180 * Scan objects in the gray stack until the stack is empty. This should be called
2181 * frequently after each object is copied, to achieve better locality and cache
2185 drain_gray_stack (char *start_addr, char *end_addr)
2189 while ((obj = gray_object_dequeue ())) {
2190 DEBUG (9, fprintf (gc_debug_file, "Precise gray object scan %p (%s)\n", obj, safe_name (obj)));
2191 scan_object (obj, start_addr, end_addr);
2198 * Scan the valuetype pointed to by START, described by DESC for references to
2199 * other objects between @from_start and @from_end and copy them to the gray_objects area.
2200 * Returns a pointer to the end of the object.
2203 scan_vtype (char *start, mword desc, char* from_start, char* from_end)
2207 /* The descriptors include info about the MonoObject header as well */
2208 start -= sizeof (MonoObject);
2210 switch (desc & 0x7) {
2211 case DESC_TYPE_RUN_LENGTH:
2212 OBJ_RUN_LEN_FOREACH_PTR (desc,start);
2213 OBJ_RUN_LEN_SIZE (skip_size, desc, start);
2214 g_assert (skip_size);
2215 return start + skip_size;
2216 case DESC_TYPE_SMALL_BITMAP:
2217 OBJ_BITMAP_FOREACH_PTR (desc,start);
2218 OBJ_BITMAP_SIZE (skip_size, desc, start);
2219 return start + skip_size;
2220 case DESC_TYPE_LARGE_BITMAP:
2221 case DESC_TYPE_COMPLEX:
2223 g_assert_not_reached ();
2226 // The other descriptors can't happen with vtypes
2227 g_assert_not_reached ();
2233 #include "sgen-pinning-stats.c"
2236 * Addresses from start to end are already sorted. This function finds
2237 * the object header for each address and pins the object. The
2238 * addresses must be inside the passed section. The (start of the)
2239 * address array is overwritten with the addresses of the actually
2240 * pinned objects. Return the number of pinned objects.
2243 pin_objects_from_addresses (GCMemSection *section, void **start, void **end, void *start_nursery, void *end_nursery)
2248 void *last_obj = NULL;
2249 size_t last_obj_size = 0;
2252 void **definitely_pinned = start;
2253 while (start < end) {
2255 /* the range check should be reduntant */
2256 if (addr != last && addr >= start_nursery && addr < end_nursery) {
2257 DEBUG (5, fprintf (gc_debug_file, "Considering pinning addr %p\n", addr));
2258 /* multiple pointers to the same object */
2259 if (addr >= last_obj && (char*)addr < (char*)last_obj + last_obj_size) {
2263 idx = ((char*)addr - (char*)section->data) / SCAN_START_SIZE;
2264 search_start = (void*)section->scan_starts [idx];
2265 if (!search_start || search_start > addr) {
2268 search_start = section->scan_starts [idx];
2269 if (search_start && search_start <= addr)
2272 if (!search_start || search_start > addr)
2273 search_start = start_nursery;
2275 if (search_start < last_obj)
2276 search_start = (char*)last_obj + last_obj_size;
2277 /* now addr should be in an object a short distance from search_start
2278 * Note that search_start must point to zeroed mem or point to an object.
2281 if (!*(void**)search_start) {
2282 mword p = (mword)search_start;
2283 p += sizeof (gpointer);
2284 p += ALLOC_ALIGN - 1;
2285 p &= ~(ALLOC_ALIGN - 1);
2286 search_start = (void*)p;
2289 last_obj = search_start;
2290 last_obj_size = safe_object_get_size ((MonoObject*)search_start);
2291 last_obj_size += ALLOC_ALIGN - 1;
2292 last_obj_size &= ~(ALLOC_ALIGN - 1);
2293 DEBUG (8, fprintf (gc_debug_file, "Pinned try match %p (%s), size %zd\n", last_obj, safe_name (last_obj), last_obj_size));
2294 if (addr >= search_start && (char*)addr < (char*)last_obj + last_obj_size) {
2295 DEBUG (4, fprintf (gc_debug_file, "Pinned object %p, vtable %p (%s), count %d\n", search_start, *(void**)search_start, safe_name (search_start), count));
2296 pin_object (search_start);
2298 pin_stats_register_object (search_start, last_obj_size);
2299 definitely_pinned [count] = search_start;
2303 /* skip to the next object */
2304 search_start = (void*)((char*)search_start + last_obj_size);
2305 } while (search_start <= addr);
2306 /* we either pinned the correct object or we ignored the addr because
2307 * it points to unused zeroed memory.
2313 //printf ("effective pinned: %d (at the end: %d)\n", count, (char*)end_nursery - (char*)last);
2317 static void** pin_queue;
2318 static int pin_queue_size = 0;
2319 static int next_pin_slot = 0;
2324 gap = (gap * 10) / 13;
2325 if (gap == 9 || gap == 10)
2334 compare_addr (const void *a, const void *b)
2336 return *(const void **)a - *(const void **)b;
2340 /* sort the addresses in array in increasing order */
2342 sort_addresses (void **array, int size)
2345 * qsort is slower as predicted.
2346 * qsort (array, size, sizeof (gpointer), compare_addr);
2353 gap = new_gap (gap);
2356 for (i = 0; i < end; i++) {
2358 if (array [i] > array [j]) {
2359 void* val = array [i];
2360 array [i] = array [j];
2365 if (gap == 1 && !swapped)
2370 static G_GNUC_UNUSED void
2371 print_nursery_gaps (void* start_nursery, void *end_nursery)
2374 gpointer first = start_nursery;
2376 for (i = 0; i < next_pin_slot; ++i) {
2377 next = pin_queue [i];
2378 fprintf (gc_debug_file, "Nursery range: %p-%p, size: %zd\n", first, next, (char*)next-(char*)first);
2382 fprintf (gc_debug_file, "Nursery range: %p-%p, size: %zd\n", first, next, (char*)next-(char*)first);
2385 /* reduce the info in the pin queue, removing duplicate pointers and sorting them */
2387 optimize_pin_queue (int start_slot)
2389 void **start, **cur, **end;
2390 /* sort and uniq pin_queue: we just sort and we let the rest discard multiple values */
2391 /* it may be better to keep ranges of pinned memory instead of individually pinning objects */
2392 DEBUG (5, fprintf (gc_debug_file, "Sorting pin queue, size: %d\n", next_pin_slot));
2393 if ((next_pin_slot - start_slot) > 1)
2394 sort_addresses (pin_queue + start_slot, next_pin_slot - start_slot);
2395 start = cur = pin_queue + start_slot;
2396 end = pin_queue + next_pin_slot;
2399 while (*start == *cur && cur < end)
2403 next_pin_slot = start - pin_queue;
2404 DEBUG (5, fprintf (gc_debug_file, "Pin queue reduced to size: %d\n", next_pin_slot));
2405 //DEBUG (6, print_nursery_gaps (start_nursery, end_nursery));
2410 optimized_pin_queue_search (void *addr)
2412 int first = 0, last = next_pin_slot;
2413 while (first < last) {
2414 int middle = first + ((last - first) >> 1);
2415 if (addr <= pin_queue [middle])
2420 g_assert (first == last);
2425 find_optimized_pin_queue_area (void *start, void *end, int *first, int *last)
2427 *first = optimized_pin_queue_search (start);
2428 *last = optimized_pin_queue_search (end);
2432 realloc_pin_queue (void)
2434 int new_size = pin_queue_size? pin_queue_size + pin_queue_size/2: 1024;
2435 void **new_pin = get_internal_mem (sizeof (void*) * new_size, INTERNAL_MEM_PIN_QUEUE);
2436 memcpy (new_pin, pin_queue, sizeof (void*) * next_pin_slot);
2437 free_internal_mem (pin_queue, INTERNAL_MEM_PIN_QUEUE);
2438 pin_queue = new_pin;
2439 pin_queue_size = new_size;
2440 DEBUG (4, fprintf (gc_debug_file, "Reallocated pin queue to size: %d\n", new_size));
2443 #include "sgen-pinning.c"
2446 * Scan the memory between start and end and queue values which could be pointers
2447 * to the area between start_nursery and end_nursery for later consideration.
2448 * Typically used for thread stacks.
2451 conservatively_pin_objects_from (void **start, void **end, void *start_nursery, void *end_nursery, int pin_type)
2454 while (start < end) {
2455 if (*start >= start_nursery && *start < end_nursery) {
2457 * *start can point to the middle of an object
2458 * note: should we handle pointing at the end of an object?
2459 * pinning in C# code disallows pointing at the end of an object
2460 * but there is some small chance that an optimizing C compiler
2461 * may keep the only reference to an object by pointing
2462 * at the end of it. We ignore this small chance for now.
2463 * Pointers to the end of an object are indistinguishable
2464 * from pointers to the start of the next object in memory
2465 * so if we allow that we'd need to pin two objects...
2466 * We queue the pointer in an array, the
2467 * array will then be sorted and uniqued. This way
2468 * we can coalesce several pinning pointers and it should
2469 * be faster since we'd do a memory scan with increasing
2470 * addresses. Note: we can align the address to the allocation
2471 * alignment, so the unique process is more effective.
2473 mword addr = (mword)*start;
2474 addr &= ~(ALLOC_ALIGN - 1);
2475 if (addr >= (mword)start_nursery && addr < (mword)end_nursery)
2476 pin_stage_ptr ((void*)addr);
2478 pin_stats_register_address ((char*)addr, pin_type);
2479 DEBUG (6, if (count) fprintf (gc_debug_file, "Pinning address %p\n", (void*)addr));
2484 DEBUG (7, if (count) fprintf (gc_debug_file, "found %d potential pinned heap pointers\n", count));
2488 * If generation is 0, just mark objects in the nursery, the others we don't care,
2489 * since they are not going to move anyway.
2490 * There are different areas that are scanned for pinned pointers:
2491 * *) the thread stacks (when jit support is ready only the unmanaged frames)
2492 * *) the pinned handle table
2493 * *) the pinned roots
2495 * Note: when we'll use a write barrier for old to new gen references, we need to
2496 * keep track of old gen objects that point to pinned new gen objects because in that
2497 * case the referenced object will be moved maybe at the next collection, but there
2498 * is no write in the old generation area where the pinned object is referenced
2499 * and we may not consider it as reachable.
2501 static G_GNUC_UNUSED void
2502 mark_pinned_objects (int generation)
2507 * Debugging function: find in the conservative roots where @obj is being pinned.
2509 static G_GNUC_UNUSED void
2510 find_pinning_reference (char *obj, size_t size)
2514 char *endobj = obj + size;
2515 for (i = 0; i < roots_hash_size [0]; ++i) {
2516 for (root = roots_hash [0][i]; root; root = root->next) {
2517 /* if desc is non-null it has precise info */
2518 if (!root->root_desc) {
2519 char ** start = (char**)root->start_root;
2520 while (start < (char**)root->end_root) {
2521 if (*start >= obj && *start < endobj) {
2522 DEBUG (0, fprintf (gc_debug_file, "Object %p referenced in pinned roots %p-%p (at %p in record %p)\n", obj, root->start_root, root->end_root, start, root));
2529 find_pinning_ref_from_thread (obj, size);
2533 * The first thing we do in a collection is to identify pinned objects.
2534 * This function considers all the areas of memory that need to be
2535 * conservatively scanned.
2538 pin_from_roots (void *start_nursery, void *end_nursery)
2542 DEBUG (2, fprintf (gc_debug_file, "Scanning pinned roots (%d bytes, %d/%d entries)\n", (int)roots_size, num_roots_entries [ROOT_TYPE_NORMAL], num_roots_entries [ROOT_TYPE_PINNED]));
2543 /* objects pinned from the API are inside these roots */
2544 for (i = 0; i < roots_hash_size [ROOT_TYPE_PINNED]; ++i) {
2545 for (root = roots_hash [ROOT_TYPE_PINNED][i]; root; root = root->next) {
2546 DEBUG (6, fprintf (gc_debug_file, "Pinned roots %p-%p\n", root->start_root, root->end_root));
2547 conservatively_pin_objects_from ((void**)root->start_root, (void**)root->end_root, start_nursery, end_nursery, PIN_TYPE_OTHER);
2550 /* now deal with the thread stacks
2551 * in the future we should be able to conservatively scan only:
2552 * *) the cpu registers
2553 * *) the unmanaged stack frames
2554 * *) the _last_ managed stack frame
2555 * *) pointers slots in managed frames
2557 scan_thread_data (start_nursery, end_nursery, FALSE);
2559 evacuate_pin_staging_area ();
2562 /* Copy function called from user defined mark functions */
2563 static char *user_copy_n_start;
2564 static char *user_copy_n_end;
2567 user_copy (void *addr)
2570 return copy_object (addr, user_copy_n_start, user_copy_n_end);
2576 * The memory area from start_root to end_root contains pointers to objects.
2577 * Their position is precisely described by @desc (this means that the pointer
2578 * can be either NULL or the pointer to the start of an object).
2579 * This functions copies them to to_space updates them.
2582 precisely_scan_objects_from (void** start_root, void** end_root, char* n_start, char *n_end, mword desc)
2584 switch (desc & ROOT_DESC_TYPE_MASK) {
2585 case ROOT_DESC_BITMAP:
2586 desc >>= ROOT_DESC_TYPE_SHIFT;
2588 if ((desc & 1) && *start_root) {
2589 *start_root = copy_object (*start_root, n_start, n_end);
2590 DEBUG (9, fprintf (gc_debug_file, "Overwrote root at %p with %p\n", start_root, *start_root));
2591 drain_gray_stack (n_start, n_end);
2597 case ROOT_DESC_COMPLEX: {
2598 gsize *bitmap_data = complex_descriptors + (desc >> ROOT_DESC_TYPE_SHIFT);
2599 int bwords = (*bitmap_data) - 1;
2600 void **start_run = start_root;
2602 while (bwords-- > 0) {
2603 gsize bmap = *bitmap_data++;
2604 void **objptr = start_run;
2606 if ((bmap & 1) && *objptr) {
2607 *objptr = copy_object (*objptr, n_start, n_end);
2608 DEBUG (9, fprintf (gc_debug_file, "Overwrote root at %p with %p\n", objptr, *objptr));
2609 drain_gray_stack (n_start, n_end);
2614 start_run += GC_BITS_PER_WORD;
2618 case ROOT_DESC_USER: {
2619 MonoGCMarkFunc marker = user_descriptors [desc >> ROOT_DESC_TYPE_SHIFT];
2621 user_copy_n_start = n_start;
2622 user_copy_n_end = n_end;
2623 marker (start_root, user_copy);
2626 case ROOT_DESC_RUN_LEN:
2627 g_assert_not_reached ();
2629 g_assert_not_reached ();
2634 alloc_fragment (void)
2636 Fragment *frag = fragment_freelist;
2638 fragment_freelist = frag->next;
2642 frag = get_internal_mem (sizeof (Fragment), INTERNAL_MEM_FRAGMENT);
2647 /* size must be a power of 2 */
2649 get_os_memory_aligned (mword size, gboolean activate)
2651 /* Allocate twice the memory to be able to put the block on an aligned address */
2652 char *mem = get_os_memory (size * 2, activate);
2657 aligned = (char*)((mword)(mem + (size - 1)) & ~(size - 1));
2658 g_assert (aligned >= mem && aligned + size <= mem + size * 2 && !((mword)aligned & (size - 1)));
2661 free_os_memory (mem, aligned - mem);
2662 if (aligned + size < mem + size * 2)
2663 free_os_memory (aligned + size, (mem + size * 2) - (aligned + size));
2669 * Allocate and setup the data structures needed to be able to allocate objects
2670 * in the nursery. The nursery is stored in nursery_section.
2673 alloc_nursery (void)
2675 GCMemSection *section;
2681 if (nursery_section)
2683 DEBUG (2, fprintf (gc_debug_file, "Allocating nursery size: %zd\n", nursery_size));
2684 /* later we will alloc a larger area for the nursery but only activate
2685 * what we need. The rest will be used as expansion if we have too many pinned
2686 * objects in the existing nursery.
2688 /* FIXME: handle OOM */
2689 section = get_internal_mem (SIZEOF_GC_MEM_SECTION, INTERNAL_MEM_SECTION);
2691 g_assert (nursery_size == DEFAULT_NURSERY_SIZE);
2692 alloc_size = nursery_size;
2693 #ifdef ALIGN_NURSERY
2694 data = get_os_memory_aligned (alloc_size, TRUE);
2696 data = get_os_memory (alloc_size, TRUE);
2698 nursery_start = data;
2699 nursery_real_end = nursery_start + nursery_size;
2700 UPDATE_HEAP_BOUNDARIES (nursery_start, nursery_real_end);
2701 nursery_next = nursery_start;
2702 total_alloc += alloc_size;
2703 DEBUG (4, fprintf (gc_debug_file, "Expanding nursery size (%p-%p): %zd, total: %zd\n", data, data + alloc_size, nursery_size, total_alloc));
2704 section->data = section->next_data = data;
2705 section->size = alloc_size;
2706 section->end_data = nursery_real_end;
2707 scan_starts = alloc_size / SCAN_START_SIZE;
2708 section->scan_starts = get_internal_mem (sizeof (char*) * scan_starts, INTERNAL_MEM_SCAN_STARTS);
2709 section->num_scan_start = scan_starts;
2710 section->block.role = MEMORY_ROLE_GEN0;
2712 /* add to the section list */
2713 section->block.next = section_list;
2714 section_list = section;
2716 nursery_section = section;
2718 /* Setup the single first large fragment */
2719 frag = alloc_fragment ();
2720 frag->fragment_start = nursery_start;
2721 frag->fragment_limit = nursery_start;
2722 frag->fragment_end = nursery_real_end;
2723 nursery_frag_real_end = nursery_real_end;
2724 /* FIXME: frag here is lost */
2728 scan_finalizer_entries (FinalizeEntry *list, char *start, char *end) {
2731 for (fin = list; fin; fin = fin->next) {
2734 DEBUG (5, fprintf (gc_debug_file, "Scan of fin ready object: %p (%s)\n", fin->object, safe_name (fin->object)));
2735 fin->object = copy_object (fin->object, start, end);
2740 * Update roots in the old generation. Since we currently don't have the
2741 * info from the write barriers, we just scan all the objects.
2743 static G_GNUC_UNUSED void
2744 scan_old_generation (char *start, char* end)
2746 GCMemSection *section;
2747 LOSObject *big_object;
2750 for (section = section_list; section; section = section->block.next) {
2751 if (section == nursery_section)
2753 DEBUG (2, fprintf (gc_debug_file, "Scan of old section: %p-%p, size: %d\n", section->data, section->next_data, (int)(section->next_data - section->data)));
2754 /* we have to deal with zeroed holes in old generation (truncated strings ...) */
2756 while (p < section->next_data) {
2761 DEBUG (8, fprintf (gc_debug_file, "Precise old object scan of %p (%s)\n", p, safe_name (p)));
2762 p = scan_object (p, start, end);
2765 /* scan the old object space, too */
2766 for (big_object = los_object_list; big_object; big_object = big_object->next) {
2767 DEBUG (5, fprintf (gc_debug_file, "Scan of big object: %p (%s), size: %zd\n", big_object->data, safe_name (big_object->data), big_object->size));
2768 scan_object (big_object->data, start, end);
2770 /* scan the list of objects ready for finalization */
2771 scan_finalizer_entries (fin_ready_list, start, end);
2772 scan_finalizer_entries (critical_fin_list, start, end);
2775 static mword fragment_total = 0;
2777 * We found a fragment of free memory in the nursery: memzero it and if
2778 * it is big enough, add it to the list of fragments that can be used for
2782 add_nursery_frag (size_t frag_size, char* frag_start, char* frag_end)
2785 DEBUG (4, fprintf (gc_debug_file, "Found empty fragment: %p-%p, size: %zd\n", frag_start, frag_end, frag_size));
2786 /* memsetting just the first chunk start is bound to provide better cache locality */
2787 if (nursery_clear_policy == CLEAR_AT_GC)
2788 memset (frag_start, 0, frag_size);
2789 /* Not worth dealing with smaller fragments: need to tune */
2790 if (frag_size >= FRAGMENT_MIN_SIZE) {
2791 fragment = alloc_fragment ();
2792 fragment->fragment_start = frag_start;
2793 fragment->fragment_limit = frag_start;
2794 fragment->fragment_end = frag_end;
2795 fragment->next = nursery_fragments;
2796 nursery_fragments = fragment;
2797 fragment_total += frag_size;
2799 /* Clear unused fragments, pinning depends on this */
2800 memset (frag_start, 0, frag_size);
2805 scan_needed_big_objects (char *start_addr, char *end_addr)
2807 LOSObject *big_object;
2809 for (big_object = los_object_list; big_object; big_object = big_object->next) {
2810 if (!big_object->scanned && object_is_pinned (big_object->data)) {
2811 DEBUG (5, fprintf (gc_debug_file, "Scan of big object: %p (%s), size: %zd\n", big_object->data, safe_name (big_object->data), big_object->size));
2812 scan_object (big_object->data, start_addr, end_addr);
2813 big_object->scanned = TRUE;
2821 generation_name (int generation)
2823 switch (generation) {
2824 case GENERATION_NURSERY: return "nursery";
2825 case GENERATION_OLD: return "old";
2826 default: g_assert_not_reached ();
2830 static DisappearingLinkHashTable*
2831 get_dislink_hash_table (int generation)
2833 switch (generation) {
2834 case GENERATION_NURSERY: return &minor_disappearing_link_hash;
2835 case GENERATION_OLD: return &major_disappearing_link_hash;
2836 default: g_assert_not_reached ();
2840 static FinalizeEntryHashTable*
2841 get_finalize_entry_hash_table (int generation)
2843 switch (generation) {
2844 case GENERATION_NURSERY: return &minor_finalizable_hash;
2845 case GENERATION_OLD: return &major_finalizable_hash;
2846 default: g_assert_not_reached ();
2851 new_to_space_section (void)
2853 /* FIXME: if the current to_space_section is empty, we don't
2854 have to allocate a new one */
2856 to_space_section = alloc_major_section ();
2857 to_space_bumper = to_space_section->next_data;
2858 to_space_top = to_space_section->end_data;
2862 to_space_expand (void)
2864 if (to_space_section) {
2865 g_assert (to_space_top == to_space_section->end_data);
2866 g_assert (to_space_bumper >= to_space_section->next_data && to_space_bumper <= to_space_top);
2868 to_space_section->next_data = to_space_bumper;
2871 new_to_space_section ();
2875 to_space_set_next_data (void)
2877 g_assert (to_space_bumper >= to_space_section->next_data && to_space_bumper < to_space_section->end_data);
2878 to_space_section->next_data = to_space_bumper;
2882 unset_to_space (void)
2884 /* between collections the to_space_bumper is invalidated
2885 because degraded allocations might occur, so we set it to
2886 NULL, just to make it explicit */
2887 to_space_bumper = NULL;
2889 /* don't unset to_space_section if we implement the FIXME in
2890 new_to_space_section */
2891 to_space_section = NULL;
2895 object_is_in_to_space (char *obj)
2900 if (ptr_in_nursery (obj))
2903 objsize = safe_object_get_size ((MonoObject*)obj);
2904 objsize += ALLOC_ALIGN - 1;
2905 objsize &= ~(ALLOC_ALIGN - 1);
2908 if (objsize > MAX_SMALL_OBJ_SIZE)
2912 if (obj_is_from_pinned_alloc (obj))
2915 /* now we know it's in a major heap section */
2916 return MAJOR_SECTION_FOR_OBJECT (obj)->is_to_space;
2920 finish_gray_stack (char *start_addr, char *end_addr, int generation)
2924 int fin_ready, bigo_scanned_num;
2927 * We copied all the reachable objects. Now it's the time to copy
2928 * the objects that were not referenced by the roots, but by the copied objects.
2929 * we built a stack of objects pointed to by gray_start: they are
2930 * additional roots and we may add more items as we go.
2931 * We loop until gray_start == gray_objects which means no more objects have
2932 * been added. Note this is iterative: no recursion is involved.
2933 * We need to walk the LO list as well in search of marked big objects
2934 * (use a flag since this is needed only on major collections). We need to loop
2935 * here as well, so keep a counter of marked LO (increasing it in copy_object).
2936 * To achieve better cache locality and cache usage, we drain the gray stack
2937 * frequently, after each object is copied, and just finish the work here.
2939 drain_gray_stack (start_addr, end_addr);
2941 //scan_old_generation (start_addr, end_addr);
2942 DEBUG (2, fprintf (gc_debug_file, "%s generation done\n", generation_name (generation)));
2943 /* walk the finalization queue and move also the objects that need to be
2944 * finalized: use the finalized objects as new roots so the objects they depend
2945 * on are also not reclaimed. As with the roots above, only objects in the nursery
2946 * are marked/copied.
2947 * We need a loop here, since objects ready for finalizers may reference other objects
2948 * that are fin-ready. Speedup with a flag?
2951 fin_ready = num_ready_finalizers;
2952 finalize_in_range (start_addr, end_addr, generation);
2953 if (generation == GENERATION_OLD)
2954 finalize_in_range (nursery_start, nursery_real_end, GENERATION_NURSERY);
2955 bigo_scanned_num = scan_needed_big_objects (start_addr, end_addr);
2957 /* drain the new stack that might have been created */
2958 DEBUG (6, fprintf (gc_debug_file, "Precise scan of gray area post fin\n"));
2959 drain_gray_stack (start_addr, end_addr);
2960 } while (fin_ready != num_ready_finalizers || bigo_scanned_num);
2962 DEBUG (2, fprintf (gc_debug_file, "Finalize queue handling scan for %s generation: %d usecs\n", generation_name (generation), TV_ELAPSED (atv, btv)));
2965 * handle disappearing links
2966 * Note we do this after checking the finalization queue because if an object
2967 * survives (at least long enough to be finalized) we don't clear the link.
2968 * This also deals with a possible issue with the monitor reclamation: with the Boehm
2969 * GC a finalized object my lose the monitor because it is cleared before the finalizer is
2972 g_assert (gray_object_queue_is_empty ());
2974 null_link_in_range (start_addr, end_addr, generation);
2975 if (generation == GENERATION_OLD)
2976 null_link_in_range (start_addr, end_addr, GENERATION_NURSERY);
2977 if (gray_object_queue_is_empty ())
2979 drain_gray_stack (start_addr, end_addr);
2982 g_assert (gray_object_queue_is_empty ());
2983 /* DEBUG (2, fprintf (gc_debug_file, "Copied from %s to old space: %d bytes (%p-%p)\n", generation_name (generation), (int)(to_space_bumper - to_space), to_space, to_space_bumper)); */
2984 to_space_set_next_data ();
2987 static int last_num_pinned = 0;
2990 build_nursery_fragments (int start_pin, int end_pin)
2992 char *frag_start, *frag_end;
2996 while (nursery_fragments) {
2997 Fragment *next = nursery_fragments->next;
2998 nursery_fragments->next = fragment_freelist;
2999 fragment_freelist = nursery_fragments;
3000 nursery_fragments = next;
3002 frag_start = nursery_start;
3004 /* clear scan starts */
3005 memset (nursery_section->scan_starts, 0, nursery_section->num_scan_start * sizeof (gpointer));
3006 for (i = start_pin; i < end_pin; ++i) {
3007 frag_end = pin_queue [i];
3008 /* remove the pin bit from pinned objects */
3009 unpin_object (frag_end);
3010 nursery_section->scan_starts [((char*)frag_end - (char*)nursery_section->data)/SCAN_START_SIZE] = frag_end;
3011 frag_size = frag_end - frag_start;
3013 add_nursery_frag (frag_size, frag_start, frag_end);
3014 frag_size = safe_object_get_size ((MonoObject*)pin_queue [i]);
3015 frag_size += ALLOC_ALIGN - 1;
3016 frag_size &= ~(ALLOC_ALIGN - 1);
3017 frag_start = (char*)pin_queue [i] + frag_size;
3019 * pin_queue [i] might point to a half-constructed string or vector whose
3020 * length field is not set. In that case, frag_start points inside the
3021 * (zero initialized) object. Find the end of the object by scanning forward.
3024 if (is_maybe_half_constructed (pin_queue [i])) {
3027 /* This is also hit for zero length arrays/strings */
3029 /* Find the end of the TLAB which contained this allocation */
3030 tlab_end = find_tlab_next_from_address (pin_queue [i]);
3033 while ((frag_start < tlab_end) && *(mword*)frag_start == 0)
3034 frag_start += sizeof (mword);
3037 * FIXME: The object is either not allocated in a TLAB, or it isn't a
3038 * half constructed object.
3043 nursery_last_pinned_end = frag_start;
3044 frag_end = nursery_real_end;
3045 frag_size = frag_end - frag_start;
3047 add_nursery_frag (frag_size, frag_start, frag_end);
3048 if (!nursery_fragments) {
3049 DEBUG (1, fprintf (gc_debug_file, "Nursery fully pinned (%d)\n", end_pin - start_pin));
3050 for (i = start_pin; i < end_pin; ++i) {
3051 DEBUG (3, fprintf (gc_debug_file, "Bastard pinning obj %p (%s), size: %d\n", pin_queue [i], safe_name (pin_queue [i]), safe_object_get_size (pin_queue [i])));
3056 nursery_next = nursery_frag_real_end = NULL;
3058 /* Clear TLABs for all threads */
3062 /* FIXME: later reduce code duplication here with the above
3063 * We don't keep track of section fragments for non-nursery sections yet, so
3067 build_section_fragments (GCMemSection *section)
3070 char *frag_start, *frag_end;
3073 /* clear scan starts */
3074 memset (section->scan_starts, 0, section->num_scan_start * sizeof (gpointer));
3075 frag_start = section->data;
3076 section->next_data = section->data;
3077 for (i = section->pin_queue_start; i < section->pin_queue_end; ++i) {
3078 frag_end = pin_queue [i];
3079 /* remove the pin bit from pinned objects */
3080 unpin_object (frag_end);
3081 if (frag_end >= section->data + section->size) {
3082 frag_end = section->data + section->size;
3084 section->scan_starts [((char*)frag_end - (char*)section->data)/SCAN_START_SIZE] = frag_end;
3086 frag_size = frag_end - frag_start;
3088 memset (frag_start, 0, frag_size);
3089 frag_size = safe_object_get_size ((MonoObject*)pin_queue [i]);
3090 frag_size += ALLOC_ALIGN - 1;
3091 frag_size &= ~(ALLOC_ALIGN - 1);
3092 frag_start = (char*)pin_queue [i] + frag_size;
3093 section->next_data = MAX (section->next_data, frag_start);
3095 frag_end = section->end_data;
3096 frag_size = frag_end - frag_start;
3098 memset (frag_start, 0, frag_size);
3102 scan_from_registered_roots (char *addr_start, char *addr_end, int root_type)
3106 for (i = 0; i < roots_hash_size [root_type]; ++i) {
3107 for (root = roots_hash [root_type][i]; root; root = root->next) {
3108 DEBUG (6, fprintf (gc_debug_file, "Precise root scan %p-%p (desc: %p)\n", root->start_root, root->end_root, (void*)root->root_desc));
3109 precisely_scan_objects_from ((void**)root->start_root, (void**)root->end_root, addr_start, addr_end, root->root_desc);
3115 dump_occupied (char *start, char *end, char *section_start)
3117 fprintf (heap_dump_file, "<occupied offset=\"%d\" size=\"%d\"/>\n", start - section_start, end - start);
3121 dump_section (GCMemSection *section, const char *type)
3123 char *start = section->data;
3124 char *end = section->data + section->size;
3125 char *occ_start = NULL;
3128 char *old_start = NULL; /* just for debugging */
3130 fprintf (heap_dump_file, "<section type=\"%s\" size=\"%d\">\n", type, section->size);
3132 while (start < end) {
3136 if (!*(void**)start) {
3138 dump_occupied (occ_start, start, section->data);
3141 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
3144 g_assert (start < section->next_data);
3149 vt = (GCVTable*)LOAD_VTABLE (start);
3152 size = safe_object_get_size ((MonoObject*) start);
3153 size += ALLOC_ALIGN - 1;
3154 size &= ~(ALLOC_ALIGN - 1);
3157 fprintf (heap_dump_file, "<object offset=\"%d\" class=\"%s.%s\" size=\"%d\"/>\n",
3158 start - section->data,
3159 vt->klass->name_space, vt->klass->name,
3167 dump_occupied (occ_start, start, section->data);
3169 fprintf (heap_dump_file, "</section>\n");
3173 dump_heap (const char *type, int num, const char *reason)
3175 static char *internal_mem_names [] = { "pin-queue", "fragment", "section", "scan-starts",
3176 "fin-table", "finalize-entry", "dislink-table",
3177 "dislink", "roots-table", "root-record", "statistics",
3178 "remset", "gray-queue", "store-remset" };
3180 GCMemSection *section;
3184 fprintf (heap_dump_file, "<collection type=\"%s\" num=\"%d\"", type, num);
3186 fprintf (heap_dump_file, " reason=\"%s\"", reason);
3187 fprintf (heap_dump_file, ">\n");
3188 fprintf (heap_dump_file, "<other-mem-usage type=\"pinned-chunks\" size=\"%ld\"/>\n", pinned_chunk_bytes_alloced);
3189 fprintf (heap_dump_file, "<other-mem-usage type=\"large-internal\" size=\"%ld\"/>\n", large_internal_bytes_alloced);
3190 fprintf (heap_dump_file, "<other-mem-usage type=\"mempools\" size=\"%ld\"/>\n", mono_mempool_get_bytes_allocated ());
3191 for (i = 0; i < INTERNAL_MEM_MAX; ++i)
3192 fprintf (heap_dump_file, "<other-mem-usage type=\"%s\" size=\"%ld\"/>\n", internal_mem_names [i], small_internal_mem_bytes [i]);
3193 fprintf (heap_dump_file, "<pinned type=\"stack\" bytes=\"%d\"/>\n", pinned_byte_counts [PIN_TYPE_STACK]);
3194 /* fprintf (heap_dump_file, "<pinned type=\"static-data\" bytes=\"%d\"/>\n", pinned_byte_counts [PIN_TYPE_STATIC_DATA]); */
3195 fprintf (heap_dump_file, "<pinned type=\"other\" bytes=\"%d\"/>\n", pinned_byte_counts [PIN_TYPE_OTHER]);
3197 dump_section (nursery_section, "nursery");
3199 for (section = section_list; section; section = section->block.next) {
3200 if (section != nursery_section)
3201 dump_section (section, "old");
3204 fprintf (heap_dump_file, "<los>\n");
3205 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
3206 MonoObject *obj = (MonoObject*) bigobj->data;
3207 MonoClass *class = mono_object_class (obj);
3209 fprintf (heap_dump_file, "<object class=\"%s.%s\" size=\"%d\"/>\n",
3210 class->name_space, class->name,
3211 safe_object_get_size (obj));
3213 fprintf (heap_dump_file, "</los>\n");
3215 fprintf (heap_dump_file, "</collection>\n");
3221 static gboolean inited = FALSE;
3223 #ifdef HEAVY_STATISTICS
3224 num_copy_object_called = 0;
3225 num_objects_copied = 0;
3231 #ifdef HEAVY_STATISTICS
3232 mono_counters_register ("WBarrier set field", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_field);
3233 mono_counters_register ("WBarrier set arrayref", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_arrayref);
3234 mono_counters_register ("WBarrier arrayref copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_arrayref_copy);
3235 mono_counters_register ("WBarrier generic store called", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_generic_store);
3236 mono_counters_register ("WBarrier generic store stored", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_generic_store_remset);
3237 mono_counters_register ("WBarrier set root", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_root);
3238 mono_counters_register ("WBarrier value copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_value_copy);
3239 mono_counters_register ("WBarrier object copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_object_copy);
3241 mono_counters_register ("# objects allocated", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_alloced);
3242 mono_counters_register ("# copy_object() called (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_called_nursery);
3243 mono_counters_register ("# objects copied (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_copied_nursery);
3244 mono_counters_register ("# copy_object() called (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_called_major);
3245 mono_counters_register ("# objects copied (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_copied_major);
3247 mono_counters_register ("# copy_object() failed from space", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_failed_from_space);
3248 mono_counters_register ("# copy_object() failed forwarded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_failed_forwarded);
3249 mono_counters_register ("# copy_object() failed pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_failed_pinned);
3250 mono_counters_register ("# copy_object() failed large or pinned chunk", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_failed_large_pinned);
3251 mono_counters_register ("# copy_object() failed to space", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_failed_to_space);
3253 mono_counters_register ("Store remsets", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_store_remsets);
3254 mono_counters_register ("Unique store remsets", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_store_remsets_unique);
3255 mono_counters_register ("Saved remsets 1", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_saved_remsets_1);
3256 mono_counters_register ("Saved remsets 2", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_saved_remsets_2);
3257 mono_counters_register ("Global remsets added", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_added);
3258 mono_counters_register ("Global remsets processed", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_processed);
3265 commit_stats (int generation)
3267 #ifdef HEAVY_STATISTICS
3268 if (generation == GENERATION_NURSERY) {
3269 stat_copy_object_called_nursery += num_copy_object_called;
3270 stat_objects_copied_nursery += num_objects_copied;
3272 g_assert (generation == GENERATION_OLD);
3273 stat_copy_object_called_major += num_copy_object_called;
3274 stat_objects_copied_major += num_objects_copied;
3280 * Collect objects in the nursery. Returns whether to trigger a major
3284 collect_nursery (size_t requested_size)
3286 size_t max_garbage_amount;
3288 char *orig_nursery_next;
3290 GCMemSection *section;
3291 TV_DECLARE (all_atv);
3292 TV_DECLARE (all_btv);
3299 orig_nursery_next = nursery_next;
3300 nursery_next = MAX (nursery_next, nursery_last_pinned_end);
3301 /* FIXME: optimize later to use the higher address where an object can be present */
3302 nursery_next = MAX (nursery_next, nursery_real_end);
3304 if (consistency_check_at_minor_collection)
3305 check_consistency ();
3307 DEBUG (1, fprintf (gc_debug_file, "Start nursery collection %d %p-%p, size: %d\n", num_minor_gcs, nursery_start, nursery_next, (int)(nursery_next - nursery_start)));
3308 max_garbage_amount = nursery_next - nursery_start;
3309 g_assert (nursery_section->size >= max_garbage_amount);
3311 /* Clear all remaining nursery fragments, pinning depends on this */
3312 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION) {
3313 g_assert (orig_nursery_next <= nursery_frag_real_end);
3314 memset (orig_nursery_next, 0, nursery_frag_real_end - orig_nursery_next);
3315 for (frag = nursery_fragments; frag; frag = frag->next) {
3316 memset (frag->fragment_start, 0, frag->fragment_end - frag->fragment_start);
3321 check_for_xdomain_refs ();
3323 nursery_section->next_data = nursery_next;
3325 sections_alloced = 0;
3327 new_to_space_section ();
3328 gray_object_queue_init ();
3331 mono_stats.minor_gc_count ++;
3332 /* world must be stopped already */
3333 TV_GETTIME (all_atv);
3335 /* pin from pinned handles */
3337 pin_from_roots (nursery_start, nursery_next);
3338 /* identify pinned objects */
3339 optimize_pin_queue (0);
3340 next_pin_slot = pin_objects_from_addresses (nursery_section, pin_queue, pin_queue + next_pin_slot, nursery_start, nursery_next);
3342 DEBUG (2, fprintf (gc_debug_file, "Finding pinned pointers: %d in %d usecs\n", next_pin_slot, TV_ELAPSED (atv, btv)));
3343 DEBUG (4, fprintf (gc_debug_file, "Start scan with %d pinned objects\n", next_pin_slot));
3346 * walk all the roots and copy the young objects to the old generation,
3347 * starting from to_space
3350 scan_from_remsets (nursery_start, nursery_next);
3351 /* we don't have complete write barrier yet, so we scan all the old generation sections */
3353 DEBUG (2, fprintf (gc_debug_file, "Old generation scan: %d usecs\n", TV_ELAPSED (btv, atv)));
3355 /* the pinned objects are roots */
3356 for (i = 0; i < next_pin_slot; ++i) {
3357 DEBUG (6, fprintf (gc_debug_file, "Precise object scan %d of pinned %p (%s)\n", i, pin_queue [i], safe_name (pin_queue [i])));
3358 scan_object (pin_queue [i], nursery_start, nursery_next);
3360 /* registered roots, this includes static fields */
3361 scan_from_registered_roots (nursery_start, nursery_next, ROOT_TYPE_NORMAL);
3362 scan_from_registered_roots (nursery_start, nursery_next, ROOT_TYPE_WBARRIER);
3363 scan_thread_data (nursery_start, nursery_next, TRUE);
3364 /* alloc_pinned objects */
3365 scan_from_pinned_objects (nursery_start, nursery_next);
3367 DEBUG (2, fprintf (gc_debug_file, "Root scan: %d usecs\n", TV_ELAPSED (atv, btv)));
3369 finish_gray_stack (nursery_start, nursery_next, GENERATION_NURSERY);
3373 /* walk the pin_queue, build up the fragment list of free memory, unmark
3374 * pinned objects as we go, memzero() the empty fragments so they are ready for the
3377 build_nursery_fragments (0, next_pin_slot);
3379 DEBUG (2, fprintf (gc_debug_file, "Fragment creation: %d usecs, %zd bytes available\n", TV_ELAPSED (btv, atv), fragment_total));
3381 for (section = section_list; section; section = section->block.next) {
3382 if (section->is_to_space)
3383 section->is_to_space = FALSE;
3386 TV_GETTIME (all_btv);
3387 mono_stats.minor_gc_time_usecs += TV_ELAPSED (all_atv, all_btv);
3390 dump_heap ("minor", num_minor_gcs - 1, NULL);
3392 /* prepare the pin queue for the next collection */
3393 last_num_pinned = next_pin_slot;
3395 if (fin_ready_list || critical_fin_list) {
3396 DEBUG (4, fprintf (gc_debug_file, "Finalizer-thread wakeup: ready %d\n", num_ready_finalizers));
3397 mono_gc_finalize_notify ();
3401 g_assert (gray_object_queue_is_empty ());
3403 commit_stats (GENERATION_NURSERY);
3405 minor_collection_sections_alloced += sections_alloced;
3407 return minor_collection_sections_alloced > minor_collection_section_allowance;
3411 count_major_sections (void)
3413 GCMemSection *section;
3416 for (section = section_list; section; section = section->block.next)
3417 if (section != nursery_section)
3423 scan_from_pinned_chunk_if_marked (PinnedChunk *chunk, char *obj, void *dummy)
3425 if (object_is_pinned (obj))
3426 scan_object (obj, NULL, (char*)-1);
3430 major_collection (const char *reason)
3432 GCMemSection *section, *prev_section;
3433 LOSObject *bigobj, *prevbo;
3438 TV_DECLARE (all_atv);
3439 TV_DECLARE (all_btv);
3442 /* FIXME: only use these values for the precise scan
3443 * note that to_space pointers should be excluded anyway...
3445 char *heap_start = NULL;
3446 char *heap_end = (char*)-1;
3447 size_t copy_space_required = 0;
3452 DEBUG (1, fprintf (gc_debug_file, "Start major collection %d\n", num_major_gcs));
3454 mono_stats.major_gc_count ++;
3456 /* Clear all remaining nursery fragments, pinning depends on this */
3457 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION) {
3458 g_assert (nursery_next <= nursery_frag_real_end);
3459 memset (nursery_next, 0, nursery_frag_real_end - nursery_next);
3460 for (frag = nursery_fragments; frag; frag = frag->next) {
3461 memset (frag->fragment_start, 0, frag->fragment_end - frag->fragment_start);
3466 check_for_xdomain_refs ();
3469 * FIXME: implement Mark/Compact
3470 * Until that is done, we can just apply mostly the same alg as for the nursery:
3471 * this means we need a big section to potentially copy all the other sections, so
3472 * it is not ideal specially with large heaps.
3474 if (g_getenv ("MONO_GC_NO_MAJOR")) {
3475 collect_nursery (0);
3478 TV_GETTIME (all_atv);
3479 /* FIXME: make sure the nursery next_data ptr is updated */
3480 nursery_section->next_data = nursery_real_end;
3481 /* we should also coalesce scanning from sections close to each other
3482 * and deal with pointers outside of the sections later.
3484 /* The remsets are not useful for a major collection */
3486 /* world must be stopped already */
3489 DEBUG (6, fprintf (gc_debug_file, "Collecting pinned addresses\n"));
3490 pin_from_roots (lowest_heap_address, highest_heap_address);
3491 optimize_pin_queue (0);
3494 * pin_queue now contains all candidate pointers, sorted and
3495 * uniqued. We must do two passes now to figure out which
3496 * objects are pinned.
3498 * The first is to find within the pin_queue the area for each
3499 * section. This requires that the pin_queue be sorted. We
3500 * also process the LOS objects and pinned chunks here.
3502 * The second, destructive, pass is to reduce the section
3503 * areas to pointers to the actually pinned objects.
3505 DEBUG (6, fprintf (gc_debug_file, "Pinning from sections\n"));
3506 /* first pass for the sections */
3507 for (section = section_list; section; section = section->block.next) {
3509 DEBUG (6, fprintf (gc_debug_file, "Pinning from section %p (%p-%p)\n", section, section->data, section->end_data));
3510 find_optimized_pin_queue_area (section->data, section->end_data, &start, &end);
3511 DEBUG (6, fprintf (gc_debug_file, "Found %d pinning addresses in section %p (%d-%d)\n",
3512 end - start, section, start, end));
3513 section->pin_queue_start = start;
3514 section->pin_queue_end = end;
3516 /* identify possible pointers to the insize of large objects */
3517 DEBUG (6, fprintf (gc_debug_file, "Pinning from large objects\n"));
3518 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
3520 find_optimized_pin_queue_area (bigobj->data, (char*)bigobj->data + bigobj->size, &start, &end);
3522 pin_object (bigobj->data);
3524 pin_stats_register_object ((char*) bigobj->data, safe_object_get_size (bigobj->data));
3525 DEBUG (6, fprintf (gc_debug_file, "Marked large object %p (%s) size: %zd from roots\n", bigobj->data, safe_name (bigobj->data), bigobj->size));
3528 /* look for pinned addresses for pinned-alloc objects */
3529 DEBUG (6, fprintf (gc_debug_file, "Pinning from pinned-alloc objects\n"));
3530 for (chunk = pinned_chunk_list; chunk; chunk = chunk->block.next) {
3532 find_optimized_pin_queue_area (chunk->start_data, (char*)chunk + chunk->num_pages * FREELIST_PAGESIZE, &start, &end);
3534 mark_pinned_from_addresses (chunk, pin_queue + start, pin_queue + end);
3536 /* second pass for the sections */
3537 for (section = section_list; section; section = section->block.next) {
3538 int start = section->pin_queue_start;
3539 int end = section->pin_queue_end;
3542 reduced_to = pin_objects_from_addresses (section, pin_queue + start, pin_queue + end,
3543 section->data, section->next_data);
3544 section->pin_queue_start = start;
3545 section->pin_queue_end = start + reduced_to;
3547 copy_space_required += (char*)section->next_data - (char*)section->data;
3551 DEBUG (2, fprintf (gc_debug_file, "Finding pinned pointers: %d in %d usecs\n", next_pin_slot, TV_ELAPSED (atv, btv)));
3552 DEBUG (4, fprintf (gc_debug_file, "Start scan with %d pinned objects\n", next_pin_slot));
3554 new_to_space_section ();
3555 gray_object_queue_init ();
3557 /* the old generation doesn't need to be scanned (no remembered sets or card
3558 * table needed either): the only objects that must survive are those pinned and
3559 * those referenced by the precise roots.
3560 * mark any section without pinned objects, so we can free it since we will be able to
3561 * move all the objects.
3563 /* the pinned objects are roots (big objects are included in this list, too) */
3564 for (section = section_list; section; section = section->block.next) {
3565 for (i = section->pin_queue_start; i < section->pin_queue_end; ++i) {
3566 DEBUG (6, fprintf (gc_debug_file, "Precise object scan %d of pinned %p (%s)\n",
3567 i, pin_queue [i], safe_name (pin_queue [i])));
3568 scan_object (pin_queue [i], heap_start, heap_end);
3571 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
3572 if (object_is_pinned (bigobj->data)) {
3573 DEBUG (6, fprintf (gc_debug_file, "Precise object scan pinned LOS object %p (%s)\n",
3574 bigobj->data, safe_name (bigobj->data)));
3575 scan_object (bigobj->data, heap_start, heap_end);
3578 scan_pinned_objects (scan_from_pinned_chunk_if_marked, NULL);
3579 /* registered roots, this includes static fields */
3580 scan_from_registered_roots (heap_start, heap_end, ROOT_TYPE_NORMAL);
3581 scan_from_registered_roots (heap_start, heap_end, ROOT_TYPE_WBARRIER);
3583 scan_thread_data (heap_start, heap_end, TRUE);
3584 /* alloc_pinned objects */
3585 scan_from_pinned_objects (heap_start, heap_end);
3586 /* scan the list of objects ready for finalization */
3587 scan_finalizer_entries (fin_ready_list, heap_start, heap_end);
3588 scan_finalizer_entries (critical_fin_list, heap_start, heap_end);
3590 DEBUG (2, fprintf (gc_debug_file, "Root scan: %d usecs\n", TV_ELAPSED (btv, atv)));
3592 /* we need to go over the big object list to see if any was marked and scan it
3593 * And we need to make this in a loop, considering that objects referenced by finalizable
3594 * objects could reference big objects (this happens in finish_gray_stack ())
3596 scan_needed_big_objects (heap_start, heap_end);
3597 /* all the objects in the heap */
3598 finish_gray_stack (heap_start, heap_end, GENERATION_OLD);
3602 /* sweep the big objects list */
3604 for (bigobj = los_object_list; bigobj;) {
3605 if (object_is_pinned (bigobj->data)) {
3606 unpin_object (bigobj->data);
3607 bigobj->scanned = FALSE;
3610 /* not referenced anywhere, so we can free it */
3612 prevbo->next = bigobj->next;
3614 los_object_list = bigobj->next;
3616 bigobj = bigobj->next;
3617 free_large_object (to_free);
3621 bigobj = bigobj->next;
3623 /* unpin objects from the pinned chunks and free the unmarked ones */
3624 sweep_pinned_objects ();
3626 /* free the unused sections */
3627 prev_section = NULL;
3628 for (section = section_list; section;) {
3629 /* to_space doesn't need handling here and the nursery is special */
3630 if (section->is_to_space || section == nursery_section) {
3631 if (section->is_to_space)
3632 section->is_to_space = FALSE;
3633 prev_section = section;
3634 section = section->block.next;
3637 /* no pinning object, so the section is free */
3638 if (section->pin_queue_start == section->pin_queue_end) {
3639 GCMemSection *to_free;
3641 prev_section->block.next = section->block.next;
3643 section_list = section->block.next;
3645 section = section->block.next;
3646 free_major_section (to_free);
3649 DEBUG (6, fprintf (gc_debug_file, "Section %p has still pinned objects (%d)\n", section, section->pin_queue_end - section->pin_queue_start));
3650 build_section_fragments (section);
3652 prev_section = section;
3653 section = section->block.next;
3656 /* walk the pin_queue, build up the fragment list of free memory, unmark
3657 * pinned objects as we go, memzero() the empty fragments so they are ready for the
3660 build_nursery_fragments (nursery_section->pin_queue_start, nursery_section->pin_queue_end);
3662 TV_GETTIME (all_btv);
3663 mono_stats.major_gc_time_usecs += TV_ELAPSED (all_atv, all_btv);
3666 dump_heap ("major", num_major_gcs - 1, reason);
3668 /* prepare the pin queue for the next collection */
3670 if (fin_ready_list || critical_fin_list) {
3671 DEBUG (4, fprintf (gc_debug_file, "Finalizer-thread wakeup: ready %d\n", num_ready_finalizers));
3672 mono_gc_finalize_notify ();
3676 g_assert (gray_object_queue_is_empty ());
3678 commit_stats (GENERATION_OLD);
3680 minor_collection_sections_alloced = 0;
3681 minor_collection_section_allowance = MAX (DEFAULT_MINOR_COLLECTION_SECTION_ALLOWANCE, count_major_sections () / 3);
3685 * Allocate a new section of memory to be used as old generation.
3687 static GCMemSection*
3688 alloc_major_section (void)
3690 GCMemSection *section;
3693 section = get_os_memory_aligned (MAJOR_SECTION_SIZE, TRUE);
3694 section->next_data = section->data = (char*)section + SIZEOF_GC_MEM_SECTION;
3695 g_assert (!((mword)section->data & 7));
3696 section->size = MAJOR_SECTION_SIZE - SIZEOF_GC_MEM_SECTION;
3697 section->end_data = section->data + section->size;
3698 UPDATE_HEAP_BOUNDARIES (section->data, section->end_data);
3699 total_alloc += section->size;
3700 DEBUG (3, fprintf (gc_debug_file, "New major heap section: (%p-%p), total: %zd\n", section->data, section->end_data, total_alloc));
3701 scan_starts = section->size / SCAN_START_SIZE;
3702 section->scan_starts = get_internal_mem (sizeof (char*) * scan_starts, INTERNAL_MEM_SCAN_STARTS);
3703 section->num_scan_start = scan_starts;
3704 section->block.role = MEMORY_ROLE_GEN1;
3705 section->is_to_space = TRUE;
3707 /* add to the section list */
3708 section->block.next = section_list;
3709 section_list = section;
3717 free_major_section (GCMemSection *section)
3719 DEBUG (3, fprintf (gc_debug_file, "Freed major section %p (%p-%p)\n", section, section->data, section->end_data));
3720 free_internal_mem (section->scan_starts, INTERNAL_MEM_SCAN_STARTS);
3721 free_os_memory (section, MAJOR_SECTION_SIZE);
3722 total_alloc -= MAJOR_SECTION_SIZE - SIZEOF_GC_MEM_SECTION;
3726 * When deciding if it's better to collect or to expand, keep track
3727 * of how much garbage was reclaimed with the last collection: if it's too
3729 * This is called when we could not allocate a small object.
3731 static void __attribute__((noinline))
3732 minor_collect_or_expand_inner (size_t size)
3734 int do_minor_collection = 1;
3736 if (!nursery_section) {
3740 if (do_minor_collection) {
3742 if (collect_nursery (size))
3743 major_collection ("minor overflow");
3744 DEBUG (2, fprintf (gc_debug_file, "Heap size: %zd, LOS size: %zd\n", total_alloc, los_memory_usage));
3746 /* this also sets the proper pointers for the next allocation */
3747 if (!search_fragment_for_size (size)) {
3749 /* TypeBuilder and MonoMethod are killing mcs with fragmentation */
3750 DEBUG (1, fprintf (gc_debug_file, "nursery collection didn't find enough room for %zd alloc (%d pinned)\n", size, last_num_pinned));
3751 for (i = 0; i < last_num_pinned; ++i) {
3752 DEBUG (3, fprintf (gc_debug_file, "Bastard pinning obj %p (%s), size: %d\n", pin_queue [i], safe_name (pin_queue [i]), safe_object_get_size (pin_queue [i])));
3757 //report_internal_mem_usage ();
3761 * ######################################################################
3762 * ######## Memory allocation from the OS
3763 * ######################################################################
3764 * This section of code deals with getting memory from the OS and
3765 * allocating memory for GC-internal data structures.
3766 * Internal memory can be handled with a freelist for small objects.
3770 * Allocate a big chunk of memory from the OS (usually 64KB to several megabytes).
3771 * This must not require any lock.
3774 get_os_memory (size_t size, int activate)
3777 unsigned long prot_flags = activate? MONO_MMAP_READ|MONO_MMAP_WRITE: MONO_MMAP_NONE;
3779 prot_flags |= MONO_MMAP_PRIVATE | MONO_MMAP_ANON;
3780 size += pagesize - 1;
3781 size &= ~(pagesize - 1);
3782 ptr = mono_valloc (0, size, prot_flags);
3787 * Free the memory returned by get_os_memory (), returning it to the OS.
3790 free_os_memory (void *addr, size_t size)
3792 munmap (addr, size);
3799 report_pinned_chunk (PinnedChunk *chunk, int seq) {
3801 int i, free_pages, num_free, free_mem;
3803 for (i = 0; i < chunk->num_pages; ++i) {
3804 if (!chunk->page_sizes [i])
3807 printf ("Pinned chunk %d at %p, size: %d, pages: %d, free: %d\n", seq, chunk, chunk->num_pages * FREELIST_PAGESIZE, chunk->num_pages, free_pages);
3808 free_mem = FREELIST_PAGESIZE * free_pages;
3809 for (i = 0; i < FREELIST_NUM_SLOTS; ++i) {
3810 if (!chunk->free_list [i])
3813 p = chunk->free_list [i];
3818 printf ("\tfree list of size %d, %d items\n", freelist_sizes [i], num_free);
3819 free_mem += freelist_sizes [i] * num_free;
3821 printf ("\tfree memory in chunk: %d\n", free_mem);
3827 static G_GNUC_UNUSED void
3828 report_internal_mem_usage (void) {
3831 printf ("Internal memory usage:\n");
3833 for (chunk = internal_chunk_list; chunk; chunk = chunk->block.next) {
3834 report_pinned_chunk (chunk, i++);
3836 printf ("Pinned memory usage:\n");
3838 for (chunk = pinned_chunk_list; chunk; chunk = chunk->block.next) {
3839 report_pinned_chunk (chunk, i++);
3844 * the array of pointers from @start to @end contains conservative
3845 * pointers to objects inside @chunk: mark each referenced object
3849 mark_pinned_from_addresses (PinnedChunk *chunk, void **start, void **end)
3851 for (; start < end; start++) {
3852 char *addr = *start;
3853 int offset = (char*)addr - (char*)chunk;
3854 int page = offset / FREELIST_PAGESIZE;
3855 int obj_offset = page == 0? offset - ((char*)chunk->start_data - (char*)chunk): offset % FREELIST_PAGESIZE;
3856 int slot_size = chunk->page_sizes [page];
3858 /* the page is not allocated */
3861 /* would be faster if we restrict the sizes to power of two,
3862 * but that's a waste of memory: need to measure. it could reduce
3863 * fragmentation since there are less pages needed, if for example
3864 * someone interns strings of each size we end up with one page per
3865 * interned string (still this is just ~40 KB): with more fine-grained sizes
3866 * this increases the number of used pages.
3869 obj_offset /= slot_size;
3870 obj_offset *= slot_size;
3871 addr = (char*)chunk->start_data + obj_offset;
3873 obj_offset /= slot_size;
3874 obj_offset *= slot_size;
3875 addr = (char*)chunk + page * FREELIST_PAGESIZE + obj_offset;
3878 /* if the vtable is inside the chunk it's on the freelist, so skip */
3879 if (*ptr && (*ptr < (void*)chunk->start_data || *ptr > (void*)((char*)chunk + chunk->num_pages * FREELIST_PAGESIZE))) {
3882 pin_stats_register_object ((char*) addr, safe_object_get_size (addr));
3883 DEBUG (6, fprintf (gc_debug_file, "Marked pinned object %p (%s) from roots\n", addr, safe_name (addr)));
3889 scan_pinned_objects (ScanPinnedObjectCallbackFunc callback, void *callback_data)
3896 for (chunk = pinned_chunk_list; chunk; chunk = chunk->block.next) {
3897 end_chunk = (char*)chunk + chunk->num_pages * FREELIST_PAGESIZE;
3898 DEBUG (6, fprintf (gc_debug_file, "Scanning pinned chunk %p (range: %p-%p)\n", chunk, chunk->start_data, end_chunk));
3899 for (i = 0; i < chunk->num_pages; ++i) {
3900 obj_size = chunk->page_sizes [i];
3903 p = i? (char*)chunk + i * FREELIST_PAGESIZE: chunk->start_data;
3904 endp = i? p + FREELIST_PAGESIZE: (char*)chunk + FREELIST_PAGESIZE;
3905 DEBUG (6, fprintf (gc_debug_file, "Page %d (size: %d, range: %p-%p)\n", i, obj_size, p, endp));
3906 while (p + obj_size <= endp) {
3908 DEBUG (9, fprintf (gc_debug_file, "Considering %p (vtable: %p)\n", ptr, *ptr));
3909 /* if the first word (the vtable) is outside the chunk we have an object */
3910 if (*ptr && (*ptr < (void*)chunk || *ptr >= end_chunk))
3911 callback (chunk, (char*)ptr, obj_size, callback_data);
3919 sweep_pinned_objects_callback (PinnedChunk *chunk, char *ptr, size_t size, void *data)
3921 if (object_is_pinned (ptr)) {
3923 DEBUG (6, fprintf (gc_debug_file, "Unmarked pinned object %p (%s)\n", ptr, safe_name (ptr)));
3925 DEBUG (6, fprintf (gc_debug_file, "Freeing unmarked pinned object %p (%s)\n", ptr, safe_name (ptr)));
3926 free_pinned_object (chunk, ptr, size);
3931 sweep_pinned_objects (void)
3933 scan_pinned_objects (sweep_pinned_objects_callback, NULL);
3937 scan_object_callback (PinnedChunk *chunk, char *ptr, size_t size, char **data)
3939 DEBUG (6, fprintf (gc_debug_file, "Precise object scan of alloc_pinned %p (%s)\n", ptr, safe_name (ptr)));
3940 /* FIXME: Put objects without references into separate chunks
3941 which do not need to be scanned */
3942 scan_object (ptr, data [0], data [1]);
3946 scan_from_pinned_objects (char *addr_start, char *addr_end)
3948 char *data [2] = { addr_start, addr_end };
3949 scan_pinned_objects ((ScanPinnedObjectCallbackFunc)scan_object_callback, data);
3953 * Find the slot number in the freelist for memory chunks that
3954 * can contain @size objects.
3957 slot_for_size (size_t size)
3960 /* do a binary search or lookup table later. */
3961 for (slot = 0; slot < FREELIST_NUM_SLOTS; ++slot) {
3962 if (freelist_sizes [slot] >= size)
3965 g_assert_not_reached ();
3970 * Build a free list for @size memory chunks from the memory area between
3971 * start_page and end_page.
3974 build_freelist (PinnedChunk *chunk, int slot, int size, char *start_page, char *end_page)
3978 /*g_print ("building freelist for slot %d, size %d in %p\n", slot, size, chunk);*/
3979 p = (void**)start_page;
3980 end = (void**)(end_page - size);
3981 g_assert (!chunk->free_list [slot]);
3982 chunk->free_list [slot] = p;
3983 while ((char*)p + size <= (char*)end) {
3985 *p = (void*)((char*)p + size);
3989 /*g_print ("%d items created, max: %d\n", count, (end_page - start_page) / size);*/
3993 alloc_pinned_chunk (void)
3997 int size = MAJOR_SECTION_SIZE;
3999 chunk = get_os_memory_aligned (size, TRUE);
4000 chunk->block.role = MEMORY_ROLE_PINNED;
4002 UPDATE_HEAP_BOUNDARIES (chunk, ((char*)chunk + size));
4003 total_alloc += size;
4004 pinned_chunk_bytes_alloced += size;
4006 /* setup the bookeeping fields */
4007 chunk->num_pages = size / FREELIST_PAGESIZE;
4008 offset = G_STRUCT_OFFSET (PinnedChunk, data);
4009 chunk->page_sizes = (void*)((char*)chunk + offset);
4010 offset += sizeof (int) * chunk->num_pages;
4011 offset += ALLOC_ALIGN - 1;
4012 offset &= ~(ALLOC_ALIGN - 1);
4013 chunk->free_list = (void*)((char*)chunk + offset);
4014 offset += sizeof (void*) * FREELIST_NUM_SLOTS;
4015 offset += ALLOC_ALIGN - 1;
4016 offset &= ~(ALLOC_ALIGN - 1);
4017 chunk->start_data = (void*)((char*)chunk + offset);
4019 /* allocate the first page to the freelist */
4020 chunk->page_sizes [0] = PINNED_FIRST_SLOT_SIZE;
4021 build_freelist (chunk, slot_for_size (PINNED_FIRST_SLOT_SIZE), PINNED_FIRST_SLOT_SIZE, chunk->start_data, ((char*)chunk + FREELIST_PAGESIZE));
4022 DEBUG (4, fprintf (gc_debug_file, "Allocated pinned chunk %p, size: %zd\n", chunk, size));
4023 min_pinned_chunk_addr = MIN (min_pinned_chunk_addr, (char*)chunk->start_data);
4024 max_pinned_chunk_addr = MAX (max_pinned_chunk_addr, ((char*)chunk + size));
4028 /* assumes freelist for slot is empty, so try to alloc a new page */
4030 get_chunk_freelist (PinnedChunk *chunk, int slot)
4034 p = chunk->free_list [slot];
4036 chunk->free_list [slot] = *p;
4039 for (i = 0; i < chunk->num_pages; ++i) {
4041 if (chunk->page_sizes [i])
4043 size = freelist_sizes [slot];
4044 chunk->page_sizes [i] = size;
4045 build_freelist (chunk, slot, size, (char*)chunk + FREELIST_PAGESIZE * i, (char*)chunk + FREELIST_PAGESIZE * (i + 1));
4049 p = chunk->free_list [slot];
4051 chunk->free_list [slot] = *p;
4058 alloc_from_freelist (size_t size)
4062 PinnedChunk *pchunk;
4063 slot = slot_for_size (size);
4064 /*g_print ("using slot %d for size %d (slot size: %d)\n", slot, size, freelist_sizes [slot]);*/
4065 g_assert (size <= freelist_sizes [slot]);
4066 for (pchunk = pinned_chunk_list; pchunk; pchunk = pchunk->block.next) {
4067 void **p = pchunk->free_list [slot];
4069 /*g_print ("found freelist for slot %d in chunk %p, returning %p, next %p\n", slot, pchunk, p, *p);*/
4070 pchunk->free_list [slot] = *p;
4074 for (pchunk = pinned_chunk_list; pchunk; pchunk = pchunk->block.next) {
4075 res = get_chunk_freelist (pchunk, slot);
4079 pchunk = alloc_pinned_chunk ();
4080 /* FIXME: handle OOM */
4081 pchunk->block.next = pinned_chunk_list;
4082 pinned_chunk_list = pchunk;
4083 res = get_chunk_freelist (pchunk, slot);
4087 /* used for the GC-internal data structures */
4088 /* FIXME: add support for bigger sizes by allocating more than one page
4092 get_internal_mem (size_t size, int type)
4096 PinnedChunk *pchunk;
4098 if (size > freelist_sizes [FREELIST_NUM_SLOTS - 1]) {
4099 LargeInternalMemHeader *mh;
4101 size += sizeof (LargeInternalMemHeader);
4102 mh = get_os_memory (size, TRUE);
4103 mh->magic = LARGE_INTERNAL_MEM_HEADER_MAGIC;
4106 large_internal_bytes_alloced += size;
4111 slot = slot_for_size (size);
4112 g_assert (size <= freelist_sizes [slot]);
4114 small_internal_mem_bytes [type] += freelist_sizes [slot];
4116 for (pchunk = internal_chunk_list; pchunk; pchunk = pchunk->block.next) {
4117 void **p = pchunk->free_list [slot];
4119 pchunk->free_list [slot] = *p;
4120 memset (p, 0, size);
4124 for (pchunk = internal_chunk_list; pchunk; pchunk = pchunk->block.next) {
4125 res = get_chunk_freelist (pchunk, slot);
4127 memset (res, 0, size);
4131 pchunk = alloc_pinned_chunk ();
4132 /* FIXME: handle OOM */
4133 pchunk->block.next = internal_chunk_list;
4134 internal_chunk_list = pchunk;
4135 res = get_chunk_freelist (pchunk, slot);
4136 memset (res, 0, size);
4141 free_internal_mem (void *addr, int type)
4143 PinnedChunk *pchunk;
4144 LargeInternalMemHeader *mh;
4147 for (pchunk = internal_chunk_list; pchunk; pchunk = pchunk->block.next) {
4148 /*printf ("trying to free %p in %p (pages: %d)\n", addr, pchunk, pchunk->num_pages);*/
4149 if (addr >= (void*)pchunk && (char*)addr < (char*)pchunk + pchunk->num_pages * FREELIST_PAGESIZE) {
4150 int offset = (char*)addr - (char*)pchunk;
4151 int page = offset / FREELIST_PAGESIZE;
4152 int slot = slot_for_size (pchunk->page_sizes [page]);
4154 *p = pchunk->free_list [slot];
4155 pchunk->free_list [slot] = p;
4157 small_internal_mem_bytes [type] -= freelist_sizes [slot];
4162 mh = (LargeInternalMemHeader*)((char*)addr - G_STRUCT_OFFSET (LargeInternalMemHeader, data));
4163 g_assert (mh->magic == LARGE_INTERNAL_MEM_HEADER_MAGIC);
4164 large_internal_bytes_alloced -= mh->size;
4165 free_os_memory (mh, mh->size);
4169 * ######################################################################
4170 * ######## Object allocation
4171 * ######################################################################
4172 * This section of code deals with allocating memory for objects.
4173 * There are several ways:
4174 * *) allocate large objects
4175 * *) allocate normal objects
4176 * *) fast lock-free allocation
4177 * *) allocation of pinned objects
4181 free_large_object (LOSObject *obj)
4183 size_t size = obj->size;
4184 DEBUG (4, fprintf (gc_debug_file, "Freed large object %p, size %zd\n", obj->data, obj->size));
4186 los_memory_usage -= size;
4187 size += sizeof (LOSObject);
4188 size += pagesize - 1;
4189 size &= ~(pagesize - 1);
4190 total_alloc -= size;
4192 free_os_memory (obj, size);
4196 * Objects with size >= 64KB are allocated in the large object space.
4197 * They are currently kept track of with a linked list.
4198 * They don't move, so there is no need to pin them during collection
4199 * and we avoid the memcpy overhead.
4201 static void* __attribute__((noinline))
4202 alloc_large_inner (MonoVTable *vtable, size_t size)
4207 int just_did_major_gc = FALSE;
4209 g_assert (size > MAX_SMALL_OBJ_SIZE);
4211 if (los_memory_usage > next_los_collection) {
4212 DEBUG (4, fprintf (gc_debug_file, "Should trigger major collection: req size %zd (los already: %zu, limit: %zu)\n", size, los_memory_usage, next_los_collection));
4213 just_did_major_gc = TRUE;
4215 major_collection ("LOS overflow");
4217 /* later increase based on a percent of the heap size */
4218 next_los_collection = los_memory_usage + 5*1024*1024;
4221 alloc_size += sizeof (LOSObject);
4222 alloc_size += pagesize - 1;
4223 alloc_size &= ~(pagesize - 1);
4224 /* FIXME: handle OOM */
4225 obj = get_os_memory (alloc_size, TRUE);
4227 vtslot = (void**)obj->data;
4229 total_alloc += alloc_size;
4230 UPDATE_HEAP_BOUNDARIES (obj->data, (char*)obj->data + size);
4231 obj->next = los_object_list;
4232 los_object_list = obj;
4233 los_memory_usage += size;
4235 DEBUG (4, fprintf (gc_debug_file, "Allocated large object %p, vtable: %p (%s), size: %zd\n", obj->data, vtable, vtable->klass->name, size));
4239 /* check if we have a suitable fragment in nursery_fragments to be able to allocate
4240 * an object of size @size
4241 * Return FALSE if not found (which means we need a collection)
4244 search_fragment_for_size (size_t size)
4246 Fragment *frag, *prev;
4247 DEBUG (4, fprintf (gc_debug_file, "Searching nursery fragment %p, size: %zd\n", nursery_frag_real_end, size));
4249 if (nursery_frag_real_end > nursery_next && nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
4250 /* Clear the remaining space, pinning depends on this */
4251 memset (nursery_next, 0, nursery_frag_real_end - nursery_next);
4254 for (frag = nursery_fragments; frag; frag = frag->next) {
4255 if (size <= (frag->fragment_end - frag->fragment_start)) {
4256 /* remove from the list */
4258 prev->next = frag->next;
4260 nursery_fragments = frag->next;
4261 nursery_next = frag->fragment_start;
4262 nursery_frag_real_end = frag->fragment_end;
4264 DEBUG (4, fprintf (gc_debug_file, "Using nursery fragment %p-%p, size: %zd (req: %zd)\n", nursery_next, nursery_frag_real_end, nursery_frag_real_end - nursery_next, size));
4265 frag->next = fragment_freelist;
4266 fragment_freelist = frag;
4275 * size is already rounded up and we hold the GC lock.
4278 alloc_degraded (MonoVTable *vtable, size_t size)
4280 GCMemSection *section;
4282 g_assert (size <= MAX_SMALL_OBJ_SIZE);
4283 for (section = section_list; section; section = section->block.next) {
4284 if (section != nursery_section && (section->end_data - section->next_data) >= size) {
4285 p = (void**)section->next_data;
4290 section = alloc_major_section ();
4291 section->is_to_space = FALSE;
4292 /* FIXME: handle OOM */
4293 p = (void**)section->next_data;
4295 section->next_data += size;
4296 degraded_mode += size;
4297 DEBUG (3, fprintf (gc_debug_file, "Allocated (degraded) object %p, vtable: %p (%s), size: %zd in section %p\n", p, vtable, vtable->klass->name, size, section));
4303 * Provide a variant that takes just the vtable for small fixed-size objects.
4304 * The aligned size is already computed and stored in vt->gc_descr.
4305 * Note: every SCAN_START_SIZE or so we are given the chance to do some special
4306 * processing. We can keep track of where objects start, for example,
4307 * so when we scan the thread stacks for pinned objects, we can start
4308 * a search for the pinned object in SCAN_START_SIZE chunks.
4311 mono_gc_alloc_obj_nolock (MonoVTable *vtable, size_t size)
4313 /* FIXME: handle OOM */
4319 HEAVY_STAT (++stat_objects_alloced);
4321 size += ALLOC_ALIGN - 1;
4322 size &= ~(ALLOC_ALIGN - 1);
4324 g_assert (vtable->gc_descr);
4326 if (G_UNLIKELY (collect_before_allocs)) {
4327 if (nursery_section) {
4329 collect_nursery (0);
4331 if (!degraded_mode && !search_fragment_for_size (size)) {
4333 g_assert_not_reached ();
4339 * We must already have the lock here instead of after the
4340 * fast path because we might be interrupted in the fast path
4341 * (after confirming that new_next < TLAB_TEMP_END) by the GC,
4342 * and we'll end up allocating an object in a fragment which
4343 * no longer belongs to us.
4345 * The managed allocator does not do this, but it's treated
4346 * specially by the world-stopping code.
4349 if (size > MAX_SMALL_OBJ_SIZE) {
4350 p = alloc_large_inner (vtable, size);
4352 /* tlab_next and tlab_temp_end are TLS vars so accessing them might be expensive */
4354 p = (void**)TLAB_NEXT;
4355 /* FIXME: handle overflow */
4356 new_next = (char*)p + size;
4357 TLAB_NEXT = new_next;
4359 if (G_LIKELY (new_next < TLAB_TEMP_END)) {
4363 * FIXME: We might need a memory barrier here so the change to tlab_next is
4364 * visible before the vtable store.
4367 DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
4368 g_assert (*p == NULL);
4371 g_assert (TLAB_NEXT == new_next);
4378 /* there are two cases: the object is too big or we run out of space in the TLAB */
4379 /* we also reach here when the thread does its first allocation after a minor
4380 * collection, since the tlab_ variables are initialized to NULL.
4381 * there can be another case (from ORP), if we cooperate with the runtime a bit:
4382 * objects that need finalizers can have the high bit set in their size
4383 * so the above check fails and we can readily add the object to the queue.
4384 * This avoids taking again the GC lock when registering, but this is moot when
4385 * doing thread-local allocation, so it may not be a good idea.
4387 g_assert (TLAB_NEXT == new_next);
4388 if (TLAB_NEXT >= TLAB_REAL_END) {
4390 * Run out of space in the TLAB. When this happens, some amount of space
4391 * remains in the TLAB, but not enough to satisfy the current allocation
4392 * request. Currently, we retire the TLAB in all cases, later we could
4393 * keep it if the remaining space is above a treshold, and satisfy the
4394 * allocation directly from the nursery.
4397 /* when running in degraded mode, we continue allocing that way
4398 * for a while, to decrease the number of useless nursery collections.
4400 if (degraded_mode && degraded_mode < DEFAULT_NURSERY_SIZE) {
4401 p = alloc_degraded (vtable, size);
4405 if (size > tlab_size) {
4406 /* Allocate directly from the nursery */
4407 if (nursery_next + size >= nursery_frag_real_end) {
4408 if (!search_fragment_for_size (size)) {
4409 minor_collect_or_expand_inner (size);
4410 if (degraded_mode) {
4411 p = alloc_degraded (vtable, size);
4417 p = (void*)nursery_next;
4418 nursery_next += size;
4419 if (nursery_next > nursery_frag_real_end) {
4424 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
4425 memset (p, 0, size);
4428 DEBUG (3, fprintf (gc_debug_file, "Retire TLAB: %p-%p [%ld]\n", TLAB_START, TLAB_REAL_END, (long)(TLAB_REAL_END - TLAB_NEXT - size)));
4430 if (nursery_next + tlab_size >= nursery_frag_real_end) {
4431 res = search_fragment_for_size (tlab_size);
4433 minor_collect_or_expand_inner (tlab_size);
4434 if (degraded_mode) {
4435 p = alloc_degraded (vtable, size);
4441 /* Allocate a new TLAB from the current nursery fragment */
4442 TLAB_START = nursery_next;
4443 nursery_next += tlab_size;
4444 TLAB_NEXT = TLAB_START;
4445 TLAB_REAL_END = TLAB_START + tlab_size;
4446 TLAB_TEMP_END = TLAB_START + MIN (SCAN_START_SIZE, tlab_size);
4448 if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
4449 memset (TLAB_START, 0, tlab_size);
4451 /* Allocate from the TLAB */
4452 p = (void*)TLAB_NEXT;
4454 g_assert (TLAB_NEXT <= TLAB_REAL_END);
4456 nursery_section->scan_starts [((char*)p - (char*)nursery_section->data)/SCAN_START_SIZE] = (char*)p;
4459 /* Reached tlab_temp_end */
4461 /* record the scan start so we can find pinned objects more easily */
4462 nursery_section->scan_starts [((char*)p - (char*)nursery_section->data)/SCAN_START_SIZE] = (char*)p;
4463 /* we just bump tlab_temp_end as well */
4464 TLAB_TEMP_END = MIN (TLAB_REAL_END, TLAB_NEXT + SCAN_START_SIZE);
4465 DEBUG (5, fprintf (gc_debug_file, "Expanding local alloc: %p-%p\n", TLAB_NEXT, TLAB_TEMP_END));
4469 DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
4476 mono_gc_alloc_obj (MonoVTable *vtable, size_t size)
4480 res = mono_gc_alloc_obj_nolock (vtable, size);
4486 mono_gc_alloc_vector (MonoVTable *vtable, size_t size, mono_array_size_t max_length)
4492 arr = mono_gc_alloc_obj_nolock (vtable, size);
4493 arr->max_length = max_length;
4501 mono_gc_alloc_array (MonoVTable *vtable, size_t size, mono_array_size_t max_length, mono_array_size_t bounds_size)
4504 MonoArrayBounds *bounds;
4508 arr = mono_gc_alloc_obj_nolock (vtable, size);
4509 arr->max_length = max_length;
4511 bounds = (MonoArrayBounds*)((char*)arr + size - bounds_size);
4512 arr->bounds = bounds;
4520 * To be used for interned strings and possibly MonoThread, reflection handles.
4521 * We may want to explicitly free these objects.
4524 mono_gc_alloc_pinned_obj (MonoVTable *vtable, size_t size)
4526 /* FIXME: handle OOM */
4528 size += ALLOC_ALIGN - 1;
4529 size &= ~(ALLOC_ALIGN - 1);
4531 if (size > MAX_FREELIST_SIZE) {
4532 /* large objects are always pinned anyway */
4533 p = alloc_large_inner (vtable, size);
4535 p = alloc_from_freelist (size);
4536 memset (p, 0, size);
4538 DEBUG (6, fprintf (gc_debug_file, "Allocated pinned object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
4545 * ######################################################################
4546 * ######## Finalization support
4547 * ######################################################################
4551 * this is valid for the nursery: if the object has been forwarded it means it's
4552 * still refrenced from a root. If it is pinned it's still alive as well.
4553 * Return TRUE if @obj is ready to be finalized.
4555 #define object_is_fin_ready(obj) (!object_is_pinned (obj) && !object_is_forwarded (obj))
4558 is_critical_finalizer (FinalizeEntry *entry)
4563 if (!mono_defaults.critical_finalizer_object)
4566 obj = entry->object;
4567 class = ((MonoVTable*)LOAD_VTABLE (obj))->klass;
4569 return mono_class_has_parent (class, mono_defaults.critical_finalizer_object);
4573 queue_finalization_entry (FinalizeEntry *entry) {
4574 if (is_critical_finalizer (entry)) {
4575 entry->next = critical_fin_list;
4576 critical_fin_list = entry;
4578 entry->next = fin_ready_list;
4579 fin_ready_list = entry;
4583 /* LOCKING: requires that the GC lock is held */
4585 rehash_fin_table (FinalizeEntryHashTable *hash_table)
4587 FinalizeEntry **finalizable_hash = hash_table->table;
4588 mword finalizable_hash_size = hash_table->size;
4591 FinalizeEntry **new_hash;
4592 FinalizeEntry *entry, *next;
4593 int new_size = g_spaced_primes_closest (hash_table->num_registered);
4595 new_hash = get_internal_mem (new_size * sizeof (FinalizeEntry*), INTERNAL_MEM_FIN_TABLE);
4596 for (i = 0; i < finalizable_hash_size; ++i) {
4597 for (entry = finalizable_hash [i]; entry; entry = next) {
4598 hash = mono_object_hash (entry->object) % new_size;
4600 entry->next = new_hash [hash];
4601 new_hash [hash] = entry;
4604 free_internal_mem (finalizable_hash, INTERNAL_MEM_FIN_TABLE);
4605 hash_table->table = new_hash;
4606 hash_table->size = new_size;
4609 /* LOCKING: requires that the GC lock is held */
4611 rehash_fin_table_if_necessary (FinalizeEntryHashTable *hash_table)
4613 if (hash_table->num_registered >= hash_table->size * 2)
4614 rehash_fin_table (hash_table);
4617 /* LOCKING: requires that the GC lock is held */
4619 finalize_in_range (char *start, char *end, int generation)
4621 FinalizeEntryHashTable *hash_table = get_finalize_entry_hash_table (generation);
4622 FinalizeEntry *entry, *prev;
4624 FinalizeEntry **finalizable_hash = hash_table->table;
4625 mword finalizable_hash_size = hash_table->size;
4629 for (i = 0; i < finalizable_hash_size; ++i) {
4631 for (entry = finalizable_hash [i]; entry;) {
4632 if ((char*)entry->object >= start && (char*)entry->object < end && !object_is_in_to_space (entry->object)) {
4633 gboolean is_fin_ready = object_is_fin_ready (entry->object);
4634 char *copy = copy_object (entry->object, start, end);
4637 FinalizeEntry *next;
4638 /* remove and put in fin_ready_list */
4640 prev->next = entry->next;
4642 finalizable_hash [i] = entry->next;
4644 num_ready_finalizers++;
4645 hash_table->num_registered--;
4646 queue_finalization_entry (entry);
4647 /* Make it survive */
4648 from = entry->object;
4649 entry->object = copy;
4650 DEBUG (5, fprintf (gc_debug_file, "Queueing object for finalization: %p (%s) (was at %p) (%d/%d)\n", entry->object, safe_name (entry->object), from, num_ready_finalizers, hash_table->num_registered));
4654 char *from = entry->object;
4655 if (hash_table == &minor_finalizable_hash && !ptr_in_nursery (copy)) {
4656 FinalizeEntry *next = entry->next;
4657 unsigned int major_hash;
4658 /* remove from the list */
4660 prev->next = entry->next;
4662 finalizable_hash [i] = entry->next;
4663 hash_table->num_registered--;
4665 entry->object = copy;
4667 /* insert it into the major hash */
4668 rehash_fin_table_if_necessary (&major_finalizable_hash);
4669 major_hash = mono_object_hash ((MonoObject*) copy) %
4670 major_finalizable_hash.size;
4671 entry->next = major_finalizable_hash.table [major_hash];
4672 major_finalizable_hash.table [major_hash] = entry;
4673 major_finalizable_hash.num_registered++;
4675 DEBUG (5, fprintf (gc_debug_file, "Promoting finalization of object %p (%s) (was at %p) to major table\n", copy, safe_name (copy), from));
4680 /* update pointer */
4681 DEBUG (5, fprintf (gc_debug_file, "Updating object for finalization: %p (%s) (was at %p)\n", entry->object, safe_name (entry->object), from));
4682 entry->object = copy;
4687 entry = entry->next;
4692 /* LOCKING: requires that the GC lock is held */
4694 null_link_in_range (char *start, char *end, int generation)
4696 DisappearingLinkHashTable *hash = get_dislink_hash_table (generation);
4697 DisappearingLink **disappearing_link_hash = hash->table;
4698 int disappearing_link_hash_size = hash->size;
4699 DisappearingLink *entry, *prev;
4701 if (!hash->num_links)
4703 for (i = 0; i < disappearing_link_hash_size; ++i) {
4705 for (entry = disappearing_link_hash [i]; entry;) {
4706 char *object = DISLINK_OBJECT (entry);
4707 if (object >= start && object < end && !object_is_in_to_space (object)) {
4708 gboolean track = DISLINK_TRACK (entry);
4709 if (!track && object_is_fin_ready (object)) {
4710 void **p = entry->link;
4711 DisappearingLink *old;
4713 /* remove from list */
4715 prev->next = entry->next;
4717 disappearing_link_hash [i] = entry->next;
4718 DEBUG (5, fprintf (gc_debug_file, "Dislink nullified at %p to GCed object %p\n", p, object));
4720 free_internal_mem (entry, INTERNAL_MEM_DISLINK);
4725 char *copy = copy_object (object, start, end);
4727 /* Update pointer if it's moved. If the object
4728 * has been moved out of the nursery, we need to
4729 * remove the link from the minor hash table to
4732 * FIXME: what if an object is moved earlier?
4735 if (hash == &minor_disappearing_link_hash && !ptr_in_nursery (copy)) {
4736 void **link = entry->link;
4737 DisappearingLink *old;
4738 /* remove from list */
4740 prev->next = entry->next;
4742 disappearing_link_hash [i] = entry->next;
4744 free_internal_mem (entry, INTERNAL_MEM_DISLINK);
4748 add_or_remove_disappearing_link ((MonoObject*)copy, link,
4749 track, GENERATION_OLD);
4751 DEBUG (5, fprintf (gc_debug_file, "Upgraded dislink at %p to major because object %p moved to %p\n", link, object, copy));
4755 /* We set the track resurrection bit to
4756 * FALSE if the object is to be finalized
4757 * so that the object can be collected in
4758 * the next cycle (i.e. after it was
4761 *entry->link = HIDE_POINTER (copy,
4762 object_is_fin_ready (object) ? FALSE : track);
4763 DEBUG (5, fprintf (gc_debug_file, "Updated dislink at %p to %p\n", entry->link, DISLINK_OBJECT (entry)));
4768 entry = entry->next;
4773 /* LOCKING: requires that the GC lock is held */
4775 null_links_for_domain (MonoDomain *domain, int generation)
4777 DisappearingLinkHashTable *hash = get_dislink_hash_table (generation);
4778 DisappearingLink **disappearing_link_hash = hash->table;
4779 int disappearing_link_hash_size = hash->size;
4780 DisappearingLink *entry, *prev;
4782 for (i = 0; i < disappearing_link_hash_size; ++i) {
4784 for (entry = disappearing_link_hash [i]; entry; ) {
4785 char *object = DISLINK_OBJECT (entry);
4786 /* FIXME: actually there should be no object
4787 left in the domain with a non-null vtable
4788 (provided we remove the Thread special
4790 if (object && (!((MonoObject*)object)->vtable || mono_object_domain (object) == domain)) {
4791 DisappearingLink *next = entry->next;
4796 disappearing_link_hash [i] = next;
4798 if (*(entry->link)) {
4799 *(entry->link) = NULL;
4800 g_warning ("Disappearing link %p not freed", entry->link);
4802 free_internal_mem (entry, INTERNAL_MEM_DISLINK);
4809 entry = entry->next;
4814 /* LOCKING: requires that the GC lock is held */
4816 finalizers_for_domain (MonoDomain *domain, MonoObject **out_array, int out_size,
4817 FinalizeEntryHashTable *hash_table)
4819 FinalizeEntry **finalizable_hash = hash_table->table;
4820 mword finalizable_hash_size = hash_table->size;
4821 FinalizeEntry *entry, *prev;
4824 if (no_finalize || !out_size || !out_array)
4827 for (i = 0; i < finalizable_hash_size; ++i) {
4829 for (entry = finalizable_hash [i]; entry;) {
4830 if (mono_object_domain (entry->object) == domain) {
4831 FinalizeEntry *next;
4832 /* remove and put in out_array */
4834 prev->next = entry->next;
4836 finalizable_hash [i] = entry->next;
4838 hash_table->num_registered--;
4839 out_array [count ++] = entry->object;
4840 DEBUG (5, fprintf (gc_debug_file, "Collecting object for finalization: %p (%s) (%d/%d)\n", entry->object, safe_name (entry->object), num_ready_finalizers, hash_table->num_registered));
4842 if (count == out_size)
4847 entry = entry->next;
4854 * mono_gc_finalizers_for_domain:
4855 * @domain: the unloading appdomain
4856 * @out_array: output array
4857 * @out_size: size of output array
4859 * Store inside @out_array up to @out_size objects that belong to the unloading
4860 * appdomain @domain. Returns the number of stored items. Can be called repeteadly
4861 * until it returns 0.
4862 * The items are removed from the finalizer data structure, so the caller is supposed
4864 * @out_array should be on the stack to allow the GC to know the objects are still alive.
4867 mono_gc_finalizers_for_domain (MonoDomain *domain, MonoObject **out_array, int out_size)
4872 result = finalizers_for_domain (domain, out_array, out_size, &minor_finalizable_hash);
4873 if (result < out_size) {
4874 result += finalizers_for_domain (domain, out_array + result, out_size - result,
4875 &major_finalizable_hash);
4883 register_for_finalization (MonoObject *obj, void *user_data, int generation)
4885 FinalizeEntryHashTable *hash_table = get_finalize_entry_hash_table (generation);
4886 FinalizeEntry **finalizable_hash;
4887 mword finalizable_hash_size;
4888 FinalizeEntry *entry, *prev;
4892 g_assert (user_data == NULL || user_data == mono_gc_run_finalize);
4893 hash = mono_object_hash (obj);
4895 rehash_fin_table_if_necessary (hash_table);
4896 finalizable_hash = hash_table->table;
4897 finalizable_hash_size = hash_table->size;
4898 hash %= finalizable_hash_size;
4900 for (entry = finalizable_hash [hash]; entry; entry = entry->next) {
4901 if (entry->object == obj) {
4903 /* remove from the list */
4905 prev->next = entry->next;
4907 finalizable_hash [hash] = entry->next;
4908 hash_table->num_registered--;
4909 DEBUG (5, fprintf (gc_debug_file, "Removed finalizer %p for object: %p (%s) (%d)\n", entry, obj, obj->vtable->klass->name, hash_table->num_registered));
4910 free_internal_mem (entry, INTERNAL_MEM_FINALIZE_ENTRY);
4918 /* request to deregister, but already out of the list */
4922 entry = get_internal_mem (sizeof (FinalizeEntry), INTERNAL_MEM_FINALIZE_ENTRY);
4923 entry->object = obj;
4924 entry->next = finalizable_hash [hash];
4925 finalizable_hash [hash] = entry;
4926 hash_table->num_registered++;
4927 DEBUG (5, fprintf (gc_debug_file, "Added finalizer %p for object: %p (%s) (%d) to %s table\n", entry, obj, obj->vtable->klass->name, hash_table->num_registered, generation_name (generation)));
4932 mono_gc_register_for_finalization (MonoObject *obj, void *user_data)
4934 if (ptr_in_nursery (obj))
4935 register_for_finalization (obj, user_data, GENERATION_NURSERY);
4937 register_for_finalization (obj, user_data, GENERATION_OLD);
4941 rehash_dislink (DisappearingLinkHashTable *hash_table)
4943 DisappearingLink **disappearing_link_hash = hash_table->table;
4944 int disappearing_link_hash_size = hash_table->size;
4947 DisappearingLink **new_hash;
4948 DisappearingLink *entry, *next;
4949 int new_size = g_spaced_primes_closest (hash_table->num_links);
4951 new_hash = get_internal_mem (new_size * sizeof (DisappearingLink*), INTERNAL_MEM_DISLINK_TABLE);
4952 for (i = 0; i < disappearing_link_hash_size; ++i) {
4953 for (entry = disappearing_link_hash [i]; entry; entry = next) {
4954 hash = mono_aligned_addr_hash (entry->link) % new_size;
4956 entry->next = new_hash [hash];
4957 new_hash [hash] = entry;
4960 free_internal_mem (disappearing_link_hash, INTERNAL_MEM_DISLINK_TABLE);
4961 hash_table->table = new_hash;
4962 hash_table->size = new_size;
4965 /* LOCKING: assumes the GC lock is held */
4967 add_or_remove_disappearing_link (MonoObject *obj, void **link, gboolean track, int generation)
4969 DisappearingLinkHashTable *hash_table = get_dislink_hash_table (generation);
4970 DisappearingLink *entry, *prev;
4972 DisappearingLink **disappearing_link_hash = hash_table->table;
4973 int disappearing_link_hash_size = hash_table->size;
4975 if (hash_table->num_links >= disappearing_link_hash_size * 2) {
4976 rehash_dislink (hash_table);
4977 disappearing_link_hash = hash_table->table;
4978 disappearing_link_hash_size = hash_table->size;
4980 /* FIXME: add check that link is not in the heap */
4981 hash = mono_aligned_addr_hash (link) % disappearing_link_hash_size;
4982 entry = disappearing_link_hash [hash];
4984 for (; entry; entry = entry->next) {
4985 /* link already added */
4986 if (link == entry->link) {
4987 /* NULL obj means remove */
4990 prev->next = entry->next;
4992 disappearing_link_hash [hash] = entry->next;
4993 hash_table->num_links--;
4994 DEBUG (5, fprintf (gc_debug_file, "Removed dislink %p (%d) from %s table\n", entry, hash_table->num_links, generation_name (generation)));
4995 free_internal_mem (entry, INTERNAL_MEM_DISLINK);
4998 *link = HIDE_POINTER (obj, track); /* we allow the change of object */
5006 entry = get_internal_mem (sizeof (DisappearingLink), INTERNAL_MEM_DISLINK);
5007 *link = HIDE_POINTER (obj, track);
5009 entry->next = disappearing_link_hash [hash];
5010 disappearing_link_hash [hash] = entry;
5011 hash_table->num_links++;
5012 DEBUG (5, fprintf (gc_debug_file, "Added dislink %p for object: %p (%s) at %p to %s table\n", entry, obj, obj->vtable->klass->name, link, generation_name (generation)));
5015 /* LOCKING: assumes the GC lock is held */
5017 mono_gc_register_disappearing_link (MonoObject *obj, void **link, gboolean track)
5019 add_or_remove_disappearing_link (NULL, link, FALSE, GENERATION_NURSERY);
5020 add_or_remove_disappearing_link (NULL, link, FALSE, GENERATION_OLD);
5022 if (ptr_in_nursery (obj))
5023 add_or_remove_disappearing_link (obj, link, track, GENERATION_NURSERY);
5025 add_or_remove_disappearing_link (obj, link, track, GENERATION_OLD);
5030 mono_gc_invoke_finalizers (void)
5032 FinalizeEntry *entry = NULL;
5033 gboolean entry_is_critical;
5036 /* FIXME: batch to reduce lock contention */
5037 while (fin_ready_list || critical_fin_list) {
5041 FinalizeEntry **list = entry_is_critical ? &critical_fin_list : &fin_ready_list;
5043 /* We have finalized entry in the last
5044 interation, now we need to remove it from
5047 *list = entry->next;
5049 FinalizeEntry *e = *list;
5050 while (e->next != entry)
5052 e->next = entry->next;
5054 free_internal_mem (entry, INTERNAL_MEM_FINALIZE_ENTRY);
5058 /* Now look for the first non-null entry. */
5059 for (entry = fin_ready_list; entry && !entry->object; entry = entry->next)
5062 entry_is_critical = FALSE;
5064 entry_is_critical = TRUE;
5065 for (entry = critical_fin_list; entry && !entry->object; entry = entry->next)
5070 g_assert (entry->object);
5071 num_ready_finalizers--;
5072 obj = entry->object;
5073 entry->object = NULL;
5074 DEBUG (7, fprintf (gc_debug_file, "Finalizing object %p (%s)\n", obj, safe_name (obj)));
5082 g_assert (entry->object == NULL);
5084 /* the object is on the stack so it is pinned */
5085 /*g_print ("Calling finalizer for object: %p (%s)\n", entry->object, safe_name (entry->object));*/
5086 mono_gc_run_finalize (obj, NULL);
5093 mono_gc_pending_finalizers (void)
5095 return fin_ready_list || critical_fin_list;
5098 /* Negative value to remove */
5100 mono_gc_add_memory_pressure (gint64 value)
5102 /* FIXME: Use interlocked functions */
5104 memory_pressure += value;
5109 * ######################################################################
5110 * ######## registered roots support
5111 * ######################################################################
5115 rehash_roots (gboolean pinned)
5119 RootRecord **new_hash;
5120 RootRecord *entry, *next;
5123 new_size = g_spaced_primes_closest (num_roots_entries [pinned]);
5124 new_hash = get_internal_mem (new_size * sizeof (RootRecord*), INTERNAL_MEM_ROOTS_TABLE);
5125 for (i = 0; i < roots_hash_size [pinned]; ++i) {
5126 for (entry = roots_hash [pinned][i]; entry; entry = next) {
5127 hash = mono_aligned_addr_hash (entry->start_root) % new_size;
5129 entry->next = new_hash [hash];
5130 new_hash [hash] = entry;
5133 free_internal_mem (roots_hash [pinned], INTERNAL_MEM_ROOTS_TABLE);
5134 roots_hash [pinned] = new_hash;
5135 roots_hash_size [pinned] = new_size;
5139 find_root (int root_type, char *start, guint32 addr_hash)
5141 RootRecord *new_root;
5143 guint32 hash = addr_hash % roots_hash_size [root_type];
5144 for (new_root = roots_hash [root_type][hash]; new_root; new_root = new_root->next) {
5145 /* we allow changing the size and the descriptor (for thread statics etc) */
5146 if (new_root->start_root == start) {
5155 * We do not coalesce roots.
5158 mono_gc_register_root_inner (char *start, size_t size, void *descr, int root_type)
5160 RootRecord *new_root;
5161 unsigned int hash, addr_hash = mono_aligned_addr_hash (start);
5164 for (i = 0; i < ROOT_TYPE_NUM; ++i) {
5165 if (num_roots_entries [i] >= roots_hash_size [i] * 2)
5168 for (i = 0; i < ROOT_TYPE_NUM; ++i) {
5169 new_root = find_root (i, start, addr_hash);
5170 /* we allow changing the size and the descriptor (for thread statics etc) */
5172 size_t old_size = new_root->end_root - new_root->start_root;
5173 new_root->end_root = new_root->start_root + size;
5174 g_assert (((new_root->root_desc != 0) && (descr != NULL)) ||
5175 ((new_root->root_desc == 0) && (descr == NULL)));
5176 new_root->root_desc = (mword)descr;
5178 roots_size -= old_size;
5183 new_root = get_internal_mem (sizeof (RootRecord), INTERNAL_MEM_ROOT_RECORD);
5185 new_root->start_root = start;
5186 new_root->end_root = new_root->start_root + size;
5187 new_root->root_desc = (mword)descr;
5189 hash = addr_hash % roots_hash_size [root_type];
5190 num_roots_entries [root_type]++;
5191 new_root->next = roots_hash [root_type] [hash];
5192 roots_hash [root_type][hash] = new_root;
5193 DEBUG (3, fprintf (gc_debug_file, "Added root %p for range: %p-%p, descr: %p (%d/%d bytes)\n", new_root, new_root->start_root, new_root->end_root, descr, (int)size, (int)roots_size));
5203 mono_gc_register_root (char *start, size_t size, void *descr)
5205 return mono_gc_register_root_inner (start, size, descr, descr ? ROOT_TYPE_NORMAL : ROOT_TYPE_PINNED);
5209 mono_gc_register_root_wbarrier (char *start, size_t size, void *descr)
5211 return mono_gc_register_root_inner (start, size, descr, ROOT_TYPE_WBARRIER);
5215 mono_gc_deregister_root (char* addr)
5217 RootRecord *tmp, *prev;
5218 unsigned int hash, addr_hash = mono_aligned_addr_hash (addr);
5222 for (root_type = 0; root_type < ROOT_TYPE_NUM; ++root_type) {
5223 hash = addr_hash % roots_hash_size [root_type];
5224 tmp = roots_hash [root_type][hash];
5227 if (tmp->start_root == (char*)addr) {
5229 prev->next = tmp->next;
5231 roots_hash [root_type][hash] = tmp->next;
5232 roots_size -= (tmp->end_root - tmp->start_root);
5233 num_roots_entries [root_type]--;
5234 DEBUG (3, fprintf (gc_debug_file, "Removed root %p for range: %p-%p\n", tmp, tmp->start_root, tmp->end_root));
5235 free_internal_mem (tmp, INTERNAL_MEM_ROOT_RECORD);
5246 * ######################################################################
5247 * ######## Thread handling (stop/start code)
5248 * ######################################################################
5251 /* FIXME: handle large/small config */
5252 #define THREAD_HASH_SIZE 11
5253 #define HASH_PTHREAD_T(id) (((unsigned int)(id) >> 4) * 2654435761u)
5255 static SgenThreadInfo* thread_table [THREAD_HASH_SIZE];
5257 #if USE_SIGNAL_BASED_START_STOP_WORLD
5259 static MonoSemType suspend_ack_semaphore;
5260 static MonoSemType *suspend_ack_semaphore_ptr;
5261 static unsigned int global_stop_count = 0;
5263 static int suspend_signal_num = SIGXFSZ;
5265 static int suspend_signal_num = SIGPWR;
5267 static int restart_signal_num = SIGXCPU;
5268 static sigset_t suspend_signal_mask;
5269 static mword cur_thread_regs [ARCH_NUM_REGS] = {0};
5271 /* LOCKING: assumes the GC lock is held */
5272 static SgenThreadInfo*
5273 thread_info_lookup (ARCH_THREAD_TYPE id)
5275 unsigned int hash = HASH_PTHREAD_T (id) % THREAD_HASH_SIZE;
5276 SgenThreadInfo *info;
5278 info = thread_table [hash];
5279 while (info && !ARCH_THREAD_EQUALS (info->id, id)) {
5286 update_current_thread_stack (void *start)
5288 void *ptr = cur_thread_regs;
5289 SgenThreadInfo *info = thread_info_lookup (ARCH_GET_THREAD ());
5290 info->stack_start = align_pointer (&ptr);
5291 g_assert (info->stack_start >= info->stack_start_limit && info->stack_start < info->stack_end);
5292 ARCH_STORE_REGS (ptr);
5293 info->stopped_regs = ptr;
5294 if (gc_callbacks.thread_suspend_func)
5295 gc_callbacks.thread_suspend_func (info->runtime_data, NULL);
5299 signal_desc (int signum)
5301 if (signum == suspend_signal_num)
5303 if (signum == restart_signal_num)
5309 * Define this and use the "xdomain-checks" MONO_GC_DEBUG option to
5310 * have cross-domain checks in the write barrier.
5312 //#define XDOMAIN_CHECKS_IN_WBARRIER
5314 #ifndef HEAVY_STATISTICS
5315 #define MANAGED_ALLOCATION
5316 #ifndef XDOMAIN_CHECKS_IN_WBARRIER
5317 #define MANAGED_WBARRIER
5322 is_ip_in_managed_allocator (MonoDomain *domain, gpointer ip);
5325 wait_for_suspend_ack (int count)
5329 for (i = 0; i < count; ++i) {
5330 while ((result = MONO_SEM_WAIT (suspend_ack_semaphore_ptr)) != 0) {
5331 if (errno != EINTR) {
5332 g_error ("sem_wait ()");
5338 /* LOCKING: assumes the GC lock is held */
5340 thread_handshake (int signum)
5342 int count, i, result;
5343 SgenThreadInfo *info;
5344 pthread_t me = pthread_self ();
5347 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5348 for (info = thread_table [i]; info; info = info->next) {
5349 DEBUG (4, fprintf (gc_debug_file, "considering thread %p for signal %d (%s)\n", info, signum, signal_desc (signum)));
5350 if (ARCH_THREAD_EQUALS (info->id, me)) {
5351 DEBUG (4, fprintf (gc_debug_file, "Skip (equal): %p, %p\n", (void*)me, (void*)info->id));
5354 /*if (signum == suspend_signal_num && info->stop_count == global_stop_count)
5356 result = pthread_kill (info->id, signum);
5358 DEBUG (4, fprintf (gc_debug_file, "thread %p signal sent\n", info));
5361 DEBUG (4, fprintf (gc_debug_file, "thread %p signal failed: %d (%s)\n", (void*)info->id, result, strerror (result)));
5367 wait_for_suspend_ack (count);
5373 restart_threads_until_none_in_managed_allocator (void)
5375 SgenThreadInfo *info;
5376 int i, result, num_threads_died = 0;
5377 int sleep_duration = -1;
5380 int restart_count = 0, restarted_count = 0;
5381 /* restart all threads that stopped in the
5383 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5384 for (info = thread_table [i]; info; info = info->next) {
5387 if (!info->stack_start ||
5388 is_ip_in_managed_allocator (info->stopped_domain, info->stopped_ip)) {
5389 result = pthread_kill (info->id, restart_signal_num);
5396 /* we set the stopped_ip to
5397 NULL for threads which
5398 we're not restarting so
5399 that we can easily identify
5401 info->stopped_ip = NULL;
5402 info->stopped_domain = NULL;
5406 /* if no threads were restarted, we're done */
5407 if (restart_count == 0)
5410 /* wait for the threads to signal their restart */
5411 wait_for_suspend_ack (restart_count);
5413 if (sleep_duration < 0) {
5417 g_usleep (sleep_duration);
5418 sleep_duration += 10;
5421 /* stop them again */
5422 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5423 for (info = thread_table [i]; info; info = info->next) {
5424 if (info->skip || info->stopped_ip == NULL)
5426 result = pthread_kill (info->id, suspend_signal_num);
5434 /* some threads might have died */
5435 num_threads_died += restart_count - restarted_count;
5436 /* wait for the threads to signal their suspension
5438 wait_for_suspend_ack (restart_count);
5441 return num_threads_died;
5444 /* LOCKING: assumes the GC lock is held (by the stopping thread) */
5446 suspend_handler (int sig, siginfo_t *siginfo, void *context)
5448 SgenThreadInfo *info;
5451 int old_errno = errno;
5452 gpointer regs [ARCH_NUM_REGS];
5453 gpointer stack_start;
5455 id = pthread_self ();
5456 info = thread_info_lookup (id);
5457 info->stopped_domain = mono_domain_get ();
5458 info->stopped_ip = (gpointer) ARCH_SIGCTX_IP (context);
5459 stop_count = global_stop_count;
5460 /* duplicate signal */
5461 if (0 && info->stop_count == stop_count) {
5465 #ifdef HAVE_KW_THREAD
5466 /* update the remset info in the thread data structure */
5467 info->remset = remembered_set;
5469 stack_start = (char*) ARCH_SIGCTX_SP (context) - REDZONE_SIZE;
5470 /* If stack_start is not within the limits, then don't set it
5471 in info and we will be restarted. */
5472 if (stack_start >= info->stack_start_limit && info->stack_start <= info->stack_end) {
5473 info->stack_start = stack_start;
5475 ARCH_COPY_SIGCTX_REGS (regs, context);
5476 info->stopped_regs = regs;
5478 g_assert (!info->stack_start);
5481 /* Notify the JIT */
5482 if (gc_callbacks.thread_suspend_func)
5483 gc_callbacks.thread_suspend_func (info->runtime_data, context);
5485 /* notify the waiting thread */
5486 MONO_SEM_POST (suspend_ack_semaphore_ptr);
5487 info->stop_count = stop_count;
5489 /* wait until we receive the restart signal */
5492 sigsuspend (&suspend_signal_mask);
5493 } while (info->signal != restart_signal_num);
5495 /* notify the waiting thread */
5496 MONO_SEM_POST (suspend_ack_semaphore_ptr);
5502 restart_handler (int sig)
5504 SgenThreadInfo *info;
5505 int old_errno = errno;
5507 info = thread_info_lookup (pthread_self ());
5508 info->signal = restart_signal_num;
5513 static TV_DECLARE (stop_world_time);
5514 static unsigned long max_pause_usec = 0;
5516 /* LOCKING: assumes the GC lock is held */
5522 update_current_thread_stack (&count);
5524 global_stop_count++;
5525 DEBUG (3, fprintf (gc_debug_file, "stopping world n %d from %p %p\n", global_stop_count, thread_info_lookup (ARCH_GET_THREAD ()), (gpointer)ARCH_GET_THREAD ()));
5526 TV_GETTIME (stop_world_time);
5527 count = thread_handshake (suspend_signal_num);
5528 count -= restart_threads_until_none_in_managed_allocator ();
5529 g_assert (count >= 0);
5530 DEBUG (3, fprintf (gc_debug_file, "world stopped %d thread(s)\n", count));
5534 /* LOCKING: assumes the GC lock is held */
5536 restart_world (void)
5539 SgenThreadInfo *info;
5540 TV_DECLARE (end_sw);
5543 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5544 for (info = thread_table [i]; info; info = info->next) {
5545 info->stack_start = NULL;
5546 info->stopped_regs = NULL;
5550 count = thread_handshake (restart_signal_num);
5551 TV_GETTIME (end_sw);
5552 usec = TV_ELAPSED (stop_world_time, end_sw);
5553 max_pause_usec = MAX (usec, max_pause_usec);
5554 DEBUG (2, fprintf (gc_debug_file, "restarted %d thread(s) (pause time: %d usec, max: %d)\n", count, (int)usec, (int)max_pause_usec));
5558 #endif /* USE_SIGNAL_BASED_START_STOP_WORLD */
5561 mono_gc_set_gc_callbacks (MonoGCCallbacks *callbacks)
5563 gc_callbacks = *callbacks;
5566 /* Variables holding start/end nursery so it won't have to be passed at every call */
5567 static void *scan_area_arg_start, *scan_area_arg_end;
5570 mono_gc_conservatively_scan_area (void *start, void *end)
5572 g_assert_not_reached ();
5573 conservatively_pin_objects_from (start, end, scan_area_arg_start, scan_area_arg_end, PIN_TYPE_OTHER);
5577 mono_gc_scan_object (void *obj)
5579 return copy_object (obj, scan_area_arg_start, scan_area_arg_end);
5583 * Mark from thread stacks and registers.
5586 scan_thread_data (void *start_nursery, void *end_nursery, gboolean precise)
5589 SgenThreadInfo *info;
5591 scan_area_arg_start = start_nursery;
5592 scan_area_arg_end = end_nursery;
5594 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5595 for (info = thread_table [i]; info; info = info->next) {
5597 DEBUG (2, fprintf (gc_debug_file, "Skipping dead thread %p, range: %p-%p, size: %zd\n", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start));
5600 DEBUG (3, fprintf (gc_debug_file, "Scanning thread %p, range: %p-%p, size: %zd, pinned=%d\n", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start, next_pin_slot));
5601 if (gc_callbacks.thread_mark_func)
5602 gc_callbacks.thread_mark_func (info->runtime_data, info->stack_start, info->stack_end, precise);
5604 conservatively_pin_objects_from (info->stack_start, info->stack_end, start_nursery, end_nursery, PIN_TYPE_STACK);
5607 conservatively_pin_objects_from (info->stopped_regs, info->stopped_regs + ARCH_NUM_REGS,
5608 start_nursery, end_nursery, PIN_TYPE_STACK);
5614 find_pinning_ref_from_thread (char *obj, size_t size)
5617 SgenThreadInfo *info;
5618 char *endobj = obj + size;
5620 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5621 for (info = thread_table [i]; info; info = info->next) {
5622 char **start = (char**)info->stack_start;
5625 while (start < (char**)info->stack_end) {
5626 if (*start >= obj && *start < endobj) {
5627 DEBUG (0, fprintf (gc_debug_file, "Object %p referenced in thread %p (id %p) at %p, stack: %p-%p\n", obj, info, (gpointer)info->id, start, info->stack_start, info->stack_end));
5632 /* FIXME: check info->stopped_regs */
5638 ptr_on_stack (void *ptr)
5640 gpointer stack_start = &stack_start;
5641 SgenThreadInfo *info = thread_info_lookup (ARCH_GET_THREAD ());
5643 if (ptr >= stack_start && ptr < (gpointer)info->stack_end)
5648 /* return TRUE if ptr points inside the managed heap */
5650 ptr_in_heap (void* ptr)
5652 mword p = (mword)ptr;
5654 GCMemSection *section;
5656 if (!ADDR_IN_HEAP_BOUNDARIES (p))
5659 if (ptr_in_nursery (ptr))
5662 if (ptr_on_stack (ptr))
5665 for (section = section_list; section; section = section->block.next) {
5666 if (ptr >= (gpointer)section->data && ptr < (gpointer)(section->data + section->size))
5670 if (obj_is_from_pinned_alloc (ptr))
5673 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
5674 if (ptr >= (gpointer)bigobj->data && ptr < (gpointer)(bigobj->data + bigobj->size))
5682 handle_remset (mword *p, void *start_nursery, void *end_nursery, gboolean global)
5689 HEAVY_STAT (++stat_global_remsets_processed);
5691 /* FIXME: exclude stack locations */
5692 switch ((*p) & REMSET_TYPE_MASK) {
5693 case REMSET_LOCATION:
5695 //__builtin_prefetch (ptr);
5696 if (((void*)ptr < start_nursery || (void*)ptr >= end_nursery)) {
5697 *ptr = copy_object (*ptr, start_nursery, end_nursery);
5698 DEBUG (9, fprintf (gc_debug_file, "Overwrote remset at %p with %p\n", ptr, *ptr));
5699 if (!global && *ptr >= start_nursery && *ptr < end_nursery) {
5701 * If the object is pinned, each reference to it from nonpinned objects
5702 * becomes part of the global remset, which can grow very large.
5704 DEBUG (9, fprintf (gc_debug_file, "Add to global remset because of pinning %p (%p %s)\n", ptr, *ptr, safe_name (*ptr)));
5705 add_to_global_remset (ptr, FALSE);
5708 DEBUG (9, fprintf (gc_debug_file, "Skipping remset at %p holding %p\n", ptr, *ptr));
5712 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
5713 if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery))
5716 while (count-- > 0) {
5717 *ptr = copy_object (*ptr, start_nursery, end_nursery);
5718 DEBUG (9, fprintf (gc_debug_file, "Overwrote remset at %p with %p (count: %d)\n", ptr, *ptr, (int)count));
5719 if (!global && *ptr >= start_nursery && *ptr < end_nursery)
5720 add_to_global_remset (ptr, FALSE);
5725 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
5726 if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery))
5728 scan_object ((char*)ptr, start_nursery, end_nursery);
5730 case REMSET_OTHER: {
5731 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
5735 if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery))
5740 ptr = (void**) scan_vtype ((char*)ptr, desc, start_nursery, end_nursery);
5742 case REMSET_ROOT_LOCATION:
5743 /* Same as REMSET_LOCATION, but the address is not required to be in the heap */
5744 *ptr = copy_object (*ptr, start_nursery, end_nursery);
5745 DEBUG (9, fprintf (gc_debug_file, "Overwrote root location remset at %p with %p\n", ptr, *ptr));
5746 if (!global && *ptr >= start_nursery && *ptr < end_nursery) {
5748 * If the object is pinned, each reference to it from nonpinned objects
5749 * becomes part of the global remset, which can grow very large.
5751 DEBUG (9, fprintf (gc_debug_file, "Add to global remset because of pinning %p (%p %s)\n", ptr, *ptr, safe_name (*ptr)));
5752 add_to_global_remset (ptr, TRUE);
5756 g_assert_not_reached ();
5761 g_assert_not_reached ();
5766 #ifdef HEAVY_STATISTICS
5768 collect_store_remsets (RememberedSet *remset, mword *bumper)
5770 mword *p = remset->data;
5775 while (p < remset->store_next) {
5776 switch ((*p) & REMSET_TYPE_MASK) {
5777 case REMSET_LOCATION:
5780 ++stat_saved_remsets_1;
5782 if (*p == last1 || *p == last2) {
5783 ++stat_saved_remsets_2;
5801 case REMSET_ROOT_LOCATION:
5805 g_assert_not_reached ();
5809 g_assert_not_reached ();
5819 RememberedSet *remset;
5821 SgenThreadInfo *info;
5823 mword *addresses, *bumper, *p, *r;
5825 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5826 for (info = thread_table [i]; info; info = info->next) {
5827 for (remset = info->remset; remset; remset = remset->next)
5828 size += remset->store_next - remset->data;
5831 for (remset = freed_thread_remsets; remset; remset = remset->next)
5832 size += remset->store_next - remset->data;
5833 for (remset = global_remset; remset; remset = remset->next)
5834 size += remset->store_next - remset->data;
5836 bumper = addresses = get_internal_mem (sizeof (mword) * size, INTERNAL_MEM_STATISTICS);
5838 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5839 for (info = thread_table [i]; info; info = info->next) {
5840 for (remset = info->remset; remset; remset = remset->next)
5841 bumper = collect_store_remsets (remset, bumper);
5844 for (remset = global_remset; remset; remset = remset->next)
5845 bumper = collect_store_remsets (remset, bumper);
5846 for (remset = freed_thread_remsets; remset; remset = remset->next)
5847 bumper = collect_store_remsets (remset, bumper);
5849 g_assert (bumper <= addresses + size);
5851 stat_store_remsets += bumper - addresses;
5853 sort_addresses ((void**)addresses, bumper - addresses);
5856 while (r < bumper) {
5862 stat_store_remsets_unique += p - addresses;
5864 free_internal_mem (addresses, INTERNAL_MEM_STATISTICS);
5869 clear_thread_store_remset_buffer (SgenThreadInfo *info)
5871 *info->store_remset_buffer_index_addr = 0;
5872 memset (*info->store_remset_buffer_addr, 0, sizeof (gpointer) * STORE_REMSET_BUFFER_SIZE);
5876 scan_from_remsets (void *start_nursery, void *end_nursery)
5879 SgenThreadInfo *info;
5880 RememberedSet *remset;
5881 GenericStoreRememberedSet *store_remset;
5882 mword *p, *next_p, *store_pos;
5884 #ifdef HEAVY_STATISTICS
5888 /* the global one */
5889 for (remset = global_remset; remset; remset = remset->next) {
5890 DEBUG (4, fprintf (gc_debug_file, "Scanning global remset range: %p-%p, size: %zd\n", remset->data, remset->store_next, remset->store_next - remset->data));
5891 store_pos = remset->data;
5892 for (p = remset->data; p < remset->store_next; p = next_p) {
5895 next_p = handle_remset (p, start_nursery, end_nursery, TRUE);
5898 * Clear global remsets of locations which no longer point to the
5899 * nursery. Otherwise, they could grow indefinitely between major
5902 ptr = (p [0] & ~REMSET_TYPE_MASK);
5903 if ((p [0] & REMSET_TYPE_MASK) == REMSET_LOCATION) {
5904 if (ptr_in_nursery (*(void**)ptr))
5905 *store_pos ++ = p [0];
5907 g_assert ((p [0] & REMSET_TYPE_MASK) == REMSET_OTHER);
5908 g_assert (p [1] == REMSET_ROOT_LOCATION);
5909 if (ptr_in_nursery (*(void**)ptr)) {
5910 *store_pos ++ = p [0];
5911 *store_pos ++ = p [1];
5916 /* Truncate the remset */
5917 remset->store_next = store_pos;
5920 /* the generic store ones */
5921 store_remset = generic_store_remsets;
5922 while (store_remset) {
5923 GenericStoreRememberedSet *next = store_remset->next;
5925 for (i = 0; i < STORE_REMSET_BUFFER_SIZE - 1; ++i) {
5926 gpointer addr = store_remset->data [i];
5928 handle_remset ((mword*)&addr, start_nursery, end_nursery, FALSE);
5931 free_internal_mem (store_remset, INTERNAL_MEM_STORE_REMSET);
5933 store_remset = next;
5935 generic_store_remsets = NULL;
5937 /* the per-thread ones */
5938 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
5939 for (info = thread_table [i]; info; info = info->next) {
5940 RememberedSet *next;
5942 for (remset = info->remset; remset; remset = next) {
5943 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for thread %p, range: %p-%p, size: %zd\n", info, remset->data, remset->store_next, remset->store_next - remset->data));
5944 for (p = remset->data; p < remset->store_next;) {
5945 p = handle_remset (p, start_nursery, end_nursery, FALSE);
5947 remset->store_next = remset->data;
5948 next = remset->next;
5949 remset->next = NULL;
5950 if (remset != info->remset) {
5951 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
5952 free_internal_mem (remset, INTERNAL_MEM_REMSET);
5955 for (j = 0; j < *info->store_remset_buffer_index_addr; ++j)
5956 handle_remset ((mword*)*info->store_remset_buffer_addr + j + 1, start_nursery, end_nursery, FALSE);
5957 clear_thread_store_remset_buffer (info);
5961 /* the freed thread ones */
5962 while (freed_thread_remsets) {
5963 RememberedSet *next;
5964 remset = freed_thread_remsets;
5965 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for freed thread, range: %p-%p, size: %zd\n", remset->data, remset->store_next, remset->store_next - remset->data));
5966 for (p = remset->data; p < remset->store_next;) {
5967 p = handle_remset (p, start_nursery, end_nursery, FALSE);
5969 next = remset->next;
5970 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
5971 free_internal_mem (remset, INTERNAL_MEM_REMSET);
5972 freed_thread_remsets = next;
5977 * Clear the info in the remembered sets: we're doing a major collection, so
5978 * the per-thread ones are not needed and the global ones will be reconstructed
5982 clear_remsets (void)
5985 SgenThreadInfo *info;
5986 RememberedSet *remset, *next;
5988 /* the global list */
5989 for (remset = global_remset; remset; remset = next) {
5990 remset->store_next = remset->data;
5991 next = remset->next;
5992 remset->next = NULL;
5993 if (remset != global_remset) {
5994 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
5995 free_internal_mem (remset, INTERNAL_MEM_REMSET);
5998 /* the generic store ones */
5999 while (generic_store_remsets) {
6000 GenericStoreRememberedSet *gs_next = generic_store_remsets->next;
6001 free_internal_mem (generic_store_remsets, INTERNAL_MEM_STORE_REMSET);
6002 generic_store_remsets = gs_next;
6004 /* the per-thread ones */
6005 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
6006 for (info = thread_table [i]; info; info = info->next) {
6007 for (remset = info->remset; remset; remset = next) {
6008 remset->store_next = remset->data;
6009 next = remset->next;
6010 remset->next = NULL;
6011 if (remset != info->remset) {
6012 DEBUG (1, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
6013 free_internal_mem (remset, INTERNAL_MEM_REMSET);
6016 clear_thread_store_remset_buffer (info);
6020 /* the freed thread ones */
6021 while (freed_thread_remsets) {
6022 next = freed_thread_remsets->next;
6023 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", freed_thread_remsets->data));
6024 free_internal_mem (freed_thread_remsets, INTERNAL_MEM_REMSET);
6025 freed_thread_remsets = next;
6030 * Clear the thread local TLAB variables for all threads.
6035 SgenThreadInfo *info;
6038 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
6039 for (info = thread_table [i]; info; info = info->next) {
6040 /* A new TLAB will be allocated when the thread does its first allocation */
6041 *info->tlab_start_addr = NULL;
6042 *info->tlab_next_addr = NULL;
6043 *info->tlab_temp_end_addr = NULL;
6044 *info->tlab_real_end_addr = NULL;
6050 * Find the tlab_next value of the TLAB which contains ADDR.
6053 find_tlab_next_from_address (char *addr)
6055 SgenThreadInfo *info;
6058 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
6059 for (info = thread_table [i]; info; info = info->next) {
6061 * The allocator increments tlab_next before
6062 * checking whether that address is still in
6063 * the TLAB, so we have to check here.
6065 char *next_addr = *info->tlab_next_addr;
6066 char *end_addr = *info->tlab_real_end_addr;
6067 if (next_addr > end_addr)
6068 next_addr = end_addr;
6069 if (addr >= *info->tlab_start_addr && addr < next_addr)
6077 /* LOCKING: assumes the GC lock is held */
6078 static SgenThreadInfo*
6079 gc_register_current_thread (void *addr)
6082 SgenThreadInfo* info = malloc (sizeof (SgenThreadInfo));
6083 #ifndef HAVE_KW_THREAD
6084 SgenThreadInfo *__thread_info__ = info;
6090 #ifndef HAVE_KW_THREAD
6091 info->tlab_start = info->tlab_next = info->tlab_temp_end = info->tlab_real_end = NULL;
6093 g_assert (!pthread_getspecific (thread_info_key));
6094 pthread_setspecific (thread_info_key, info);
6097 info->id = ARCH_GET_THREAD ();
6098 info->stop_count = -1;
6101 info->stack_start = NULL;
6102 info->tlab_start_addr = &TLAB_START;
6103 info->tlab_next_addr = &TLAB_NEXT;
6104 info->tlab_temp_end_addr = &TLAB_TEMP_END;
6105 info->tlab_real_end_addr = &TLAB_REAL_END;
6106 info->store_remset_buffer_addr = &STORE_REMSET_BUFFER;
6107 info->store_remset_buffer_index_addr = &STORE_REMSET_BUFFER_INDEX;
6108 info->stopped_ip = NULL;
6109 info->stopped_domain = NULL;
6110 info->stopped_regs = NULL;
6112 #ifdef HAVE_KW_THREAD
6113 tlab_next_addr = &tlab_next;
6114 store_remset_buffer_index_addr = &store_remset_buffer_index;
6117 /* try to get it with attributes first */
6118 #if defined(HAVE_PTHREAD_GETATTR_NP) && defined(HAVE_PTHREAD_ATTR_GETSTACK)
6122 pthread_attr_t attr;
6123 pthread_getattr_np (pthread_self (), &attr);
6124 pthread_attr_getstack (&attr, &sstart, &size);
6125 info->stack_start_limit = sstart;
6126 info->stack_end = (char*)sstart + size;
6127 pthread_attr_destroy (&attr);
6129 #elif defined(HAVE_PTHREAD_GET_STACKSIZE_NP) && defined(HAVE_PTHREAD_GET_STACKADDR_NP)
6130 info->stack_end = (char*)pthread_get_stackaddr_np (pthread_self ());
6133 /* FIXME: we assume the stack grows down */
6134 gsize stack_bottom = (gsize)addr;
6135 stack_bottom += 4095;
6136 stack_bottom &= ~4095;
6137 info->stack_end = (char*)stack_bottom;
6141 #ifdef HAVE_KW_THREAD
6142 stack_end = info->stack_end;
6145 /* hash into the table */
6146 hash = HASH_PTHREAD_T (info->id) % THREAD_HASH_SIZE;
6147 info->next = thread_table [hash];
6148 thread_table [hash] = info;
6150 info->remset = alloc_remset (DEFAULT_REMSET_SIZE, info);
6151 pthread_setspecific (remembered_set_key, info->remset);
6152 #ifdef HAVE_KW_THREAD
6153 remembered_set = info->remset;
6156 STORE_REMSET_BUFFER = get_internal_mem (sizeof (gpointer) * STORE_REMSET_BUFFER_SIZE, INTERNAL_MEM_STORE_REMSET);
6157 STORE_REMSET_BUFFER_INDEX = 0;
6159 DEBUG (3, fprintf (gc_debug_file, "registered thread %p (%p) (hash: %d)\n", info, (gpointer)info->id, hash));
6161 if (gc_callbacks.thread_attach_func)
6162 info->runtime_data = gc_callbacks.thread_attach_func ();
6168 add_generic_store_remset_from_buffer (gpointer *buffer)
6170 GenericStoreRememberedSet *remset = get_internal_mem (sizeof (GenericStoreRememberedSet), INTERNAL_MEM_STORE_REMSET);
6171 memcpy (remset->data, buffer + 1, sizeof (gpointer) * (STORE_REMSET_BUFFER_SIZE - 1));
6172 remset->next = generic_store_remsets;
6173 generic_store_remsets = remset;
6177 unregister_current_thread (void)
6180 SgenThreadInfo *prev = NULL;
6182 RememberedSet *rset;
6183 ARCH_THREAD_TYPE id = ARCH_GET_THREAD ();
6185 hash = HASH_PTHREAD_T (id) % THREAD_HASH_SIZE;
6186 p = thread_table [hash];
6188 DEBUG (3, fprintf (gc_debug_file, "unregister thread %p (%p)\n", p, (gpointer)p->id));
6189 while (!ARCH_THREAD_EQUALS (p->id, id)) {
6194 thread_table [hash] = p->next;
6196 prev->next = p->next;
6199 if (freed_thread_remsets) {
6200 for (rset = p->remset; rset->next; rset = rset->next)
6202 rset->next = freed_thread_remsets;
6203 freed_thread_remsets = p->remset;
6205 freed_thread_remsets = p->remset;
6208 if (*p->store_remset_buffer_index_addr)
6209 add_generic_store_remset_from_buffer (*p->store_remset_buffer_addr);
6210 free_internal_mem (*p->store_remset_buffer_addr, INTERNAL_MEM_STORE_REMSET);
6215 unregister_thread (void *k)
6217 g_assert (!mono_domain_get ());
6219 unregister_current_thread ();
6224 mono_gc_register_thread (void *baseptr)
6226 SgenThreadInfo *info;
6230 info = thread_info_lookup (ARCH_GET_THREAD ());
6232 info = gc_register_current_thread (baseptr);
6234 return info != NULL;
6237 #if USE_PTHREAD_INTERCEPT
6239 #undef pthread_create
6241 #undef pthread_detach
6244 void *(*start_routine) (void *);
6247 MonoSemType registered;
6248 } SgenThreadStartInfo;
6251 gc_start_thread (void *arg)
6253 SgenThreadStartInfo *start_info = arg;
6254 SgenThreadInfo* info;
6255 void *t_arg = start_info->arg;
6256 void *(*start_func) (void*) = start_info->start_routine;
6261 info = gc_register_current_thread (&result);
6263 post_result = MONO_SEM_POST (&(start_info->registered));
6264 g_assert (!post_result);
6265 result = start_func (t_arg);
6266 g_assert (!mono_domain_get ());
6268 * this is done by the pthread key dtor
6270 unregister_current_thread ();
6278 mono_gc_pthread_create (pthread_t *new_thread, const pthread_attr_t *attr, void *(*start_routine)(void *), void *arg)
6280 SgenThreadStartInfo *start_info;
6283 start_info = malloc (sizeof (SgenThreadStartInfo));
6286 result = MONO_SEM_INIT (&(start_info->registered), 0);
6288 start_info->arg = arg;
6289 start_info->start_routine = start_routine;
6291 result = pthread_create (new_thread, attr, gc_start_thread, start_info);
6293 while (MONO_SEM_WAIT (&(start_info->registered)) != 0) {
6294 /*if (EINTR != errno) ABORT("sem_wait failed"); */
6297 MONO_SEM_DESTROY (&(start_info->registered));
6303 mono_gc_pthread_join (pthread_t thread, void **retval)
6305 return pthread_join (thread, retval);
6309 mono_gc_pthread_detach (pthread_t thread)
6311 return pthread_detach (thread);
6314 #endif /* USE_PTHREAD_INTERCEPT */
6317 * ######################################################################
6318 * ######## Write barriers
6319 * ######################################################################
6322 static RememberedSet*
6323 alloc_remset (int size, gpointer id) {
6324 RememberedSet* res = get_internal_mem (sizeof (RememberedSet) + (size * sizeof (gpointer)), INTERNAL_MEM_REMSET);
6325 res->store_next = res->data;
6326 res->end_set = res->data + size;
6328 DEBUG (4, fprintf (gc_debug_file, "Allocated remset size %d at %p for %p\n", size, res->data, id));
6333 * Note: the write barriers first do the needed GC work and then do the actual store:
6334 * this way the value is visible to the conservative GC scan after the write barrier
6335 * itself. If a GC interrupts the barrier in the middle, value will be kept alive by
6336 * the conservative scan, otherwise by the remembered set scan. FIXME: figure out what
6337 * happens when we need to record which pointers contain references to the new generation.
6338 * The write barrier will be executed, but the pointer is still not stored.
6341 mono_gc_wbarrier_set_field (MonoObject *obj, gpointer field_ptr, MonoObject* value)
6345 HEAVY_STAT (++stat_wbarrier_set_field);
6346 if (ptr_in_nursery (field_ptr)) {
6347 *(void**)field_ptr = value;
6350 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", field_ptr));
6352 rs = REMEMBERED_SET;
6353 if (rs->store_next < rs->end_set) {
6354 *(rs->store_next++) = (mword)field_ptr;
6355 *(void**)field_ptr = value;
6359 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6360 rs->next = REMEMBERED_SET;
6361 REMEMBERED_SET = rs;
6362 #ifdef HAVE_KW_THREAD
6363 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6365 *(rs->store_next++) = (mword)field_ptr;
6366 *(void**)field_ptr = value;
6371 mono_gc_wbarrier_set_arrayref (MonoArray *arr, gpointer slot_ptr, MonoObject* value)
6375 HEAVY_STAT (++stat_wbarrier_set_arrayref);
6376 if (ptr_in_nursery (slot_ptr)) {
6377 *(void**)slot_ptr = value;
6380 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", slot_ptr));
6382 rs = REMEMBERED_SET;
6383 if (rs->store_next < rs->end_set) {
6384 *(rs->store_next++) = (mword)slot_ptr;
6385 *(void**)slot_ptr = value;
6389 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6390 rs->next = REMEMBERED_SET;
6391 REMEMBERED_SET = rs;
6392 #ifdef HAVE_KW_THREAD
6393 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6395 *(rs->store_next++) = (mword)slot_ptr;
6396 *(void**)slot_ptr = value;
6401 mono_gc_wbarrier_arrayref_copy (gpointer dest_ptr, gpointer src_ptr, int count)
6405 HEAVY_STAT (++stat_wbarrier_arrayref_copy);
6407 memmove (dest_ptr, src_ptr, count * sizeof (gpointer));
6408 if (ptr_in_nursery (dest_ptr)) {
6412 rs = REMEMBERED_SET;
6413 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p, %d\n", dest_ptr, count));
6414 if (rs->store_next + 1 < rs->end_set) {
6415 *(rs->store_next++) = (mword)dest_ptr | REMSET_RANGE;
6416 *(rs->store_next++) = count;
6420 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6421 rs->next = REMEMBERED_SET;
6422 REMEMBERED_SET = rs;
6423 #ifdef HAVE_KW_THREAD
6424 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6426 *(rs->store_next++) = (mword)dest_ptr | REMSET_RANGE;
6427 *(rs->store_next++) = count;
6432 find_object_for_ptr_in_area (char *ptr, char *start, char *end)
6434 while (start < end) {
6437 if (!*(void**)start) {
6438 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
6444 #define SCAN_OBJECT_NOSCAN
6445 #include "sgen-scan-object.h"
6447 if (ptr >= old_start && ptr < start)
6454 static char *found_obj;
6457 find_object_for_ptr_in_pinned_chunk_callback (PinnedChunk *chunk, char *obj, size_t size, char *ptr)
6459 if (ptr >= obj && ptr < obj + size) {
6460 g_assert (!found_obj);
6466 find_object_for_ptr (char *ptr)
6468 GCMemSection *section;
6471 for (section = section_list; section; section = section->block.next) {
6472 if (ptr >= section->data && ptr < section->end_data)
6473 return find_object_for_ptr_in_area (ptr, section->data, section->end_data);
6476 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
6477 if (ptr >= bigobj->data && ptr < bigobj->data + bigobj->size)
6478 return bigobj->data;
6482 scan_pinned_objects ((ScanPinnedObjectCallbackFunc)find_object_for_ptr_in_pinned_chunk_callback, ptr);
6487 evacuate_remset_buffer (void)
6492 buffer = STORE_REMSET_BUFFER;
6494 add_generic_store_remset_from_buffer (buffer);
6495 memset (buffer, 0, sizeof (gpointer) * STORE_REMSET_BUFFER_SIZE);
6497 STORE_REMSET_BUFFER_INDEX = 0;
6501 mono_gc_wbarrier_generic_nostore (gpointer ptr)
6507 HEAVY_STAT (++stat_wbarrier_generic_store);
6509 #ifdef XDOMAIN_CHECKS_IN_WBARRIER
6510 /* FIXME: ptr_in_heap must be called with the GC lock held */
6511 if (xdomain_checks && *(MonoObject**)ptr && ptr_in_heap (ptr)) {
6512 char *start = find_object_for_ptr (ptr);
6513 MonoObject *value = *(MonoObject**)ptr;
6517 MonoObject *obj = (MonoObject*)start;
6518 if (obj->vtable->domain != value->vtable->domain)
6519 g_assert (is_xdomain_ref_allowed (ptr, start, obj->vtable->domain));
6526 if (ptr_in_nursery (ptr) || ptr_on_stack (ptr) || !ptr_in_nursery (*(gpointer*)ptr)) {
6527 DEBUG (8, fprintf (gc_debug_file, "Skipping remset at %p\n", ptr));
6532 buffer = STORE_REMSET_BUFFER;
6533 index = STORE_REMSET_BUFFER_INDEX;
6534 /* This simple optimization eliminates a sizable portion of
6535 entries. Comparing it to the last but one entry as well
6536 doesn't eliminate significantly more entries. */
6537 if (buffer [index] == ptr) {
6542 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", ptr));
6543 HEAVY_STAT (++stat_wbarrier_generic_store_remset);
6546 if (index >= STORE_REMSET_BUFFER_SIZE) {
6547 evacuate_remset_buffer ();
6548 index = STORE_REMSET_BUFFER_INDEX;
6549 g_assert (index == 0);
6552 buffer [index] = ptr;
6553 STORE_REMSET_BUFFER_INDEX = index;
6559 mono_gc_wbarrier_generic_store (gpointer ptr, MonoObject* value)
6561 DEBUG (8, fprintf (gc_debug_file, "Wbarrier store at %p to %p (%s)\n", ptr, value, value ? safe_name (value) : "null"));
6562 *(void**)ptr = value;
6563 if (ptr_in_nursery (value))
6564 mono_gc_wbarrier_generic_nostore (ptr);
6568 mono_gc_wbarrier_set_root (gpointer ptr, MonoObject *value)
6572 HEAVY_STAT (++stat_wbarrier_set_root);
6573 if (ptr_in_nursery (ptr))
6575 DEBUG (8, fprintf (gc_debug_file, "Adding root remset at %p (%s)\n", ptr, value ? safe_name (value) : "null"));
6577 rs = REMEMBERED_SET;
6578 if (rs->store_next + 2 < rs->end_set) {
6579 *(rs->store_next++) = (mword)ptr | REMSET_OTHER;
6580 *(rs->store_next++) = (mword)REMSET_ROOT_LOCATION;
6581 *(void**)ptr = value;
6584 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6585 rs->next = REMEMBERED_SET;
6586 REMEMBERED_SET = rs;
6587 #ifdef HAVE_KW_THREAD
6588 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6590 *(rs->store_next++) = (mword)ptr | REMSET_OTHER;
6591 *(rs->store_next++) = (mword)REMSET_ROOT_LOCATION;
6593 *(void**)ptr = value;
6597 mono_gc_wbarrier_value_copy (gpointer dest, gpointer src, int count, MonoClass *klass)
6601 HEAVY_STAT (++stat_wbarrier_value_copy);
6602 g_assert (klass->valuetype);
6604 memmove (dest, src, count * mono_class_value_size (klass, NULL));
6605 rs = REMEMBERED_SET;
6606 if (ptr_in_nursery (dest) || ptr_on_stack (dest)) {
6610 g_assert (klass->gc_descr_inited);
6611 DEBUG (8, fprintf (gc_debug_file, "Adding value remset at %p, count %d, descr %p for class %s (%p)\n", dest, count, klass->gc_descr, klass->name, klass));
6613 if (rs->store_next + 3 < rs->end_set) {
6614 *(rs->store_next++) = (mword)dest | REMSET_OTHER;
6615 *(rs->store_next++) = (mword)REMSET_VTYPE;
6616 *(rs->store_next++) = (mword)klass->gc_descr;
6617 *(rs->store_next++) = (mword)count;
6621 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6622 rs->next = REMEMBERED_SET;
6623 REMEMBERED_SET = rs;
6624 #ifdef HAVE_KW_THREAD
6625 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6627 *(rs->store_next++) = (mword)dest | REMSET_OTHER;
6628 *(rs->store_next++) = (mword)REMSET_VTYPE;
6629 *(rs->store_next++) = (mword)klass->gc_descr;
6630 *(rs->store_next++) = (mword)count;
6635 * mono_gc_wbarrier_object_copy:
6637 * Write barrier to call when obj is the result of a clone or copy of an object.
6640 mono_gc_wbarrier_object_copy (MonoObject* obj, MonoObject *src)
6646 HEAVY_STAT (++stat_wbarrier_object_copy);
6647 rs = REMEMBERED_SET;
6648 DEBUG (1, fprintf (gc_debug_file, "Adding object remset for %p\n", obj));
6649 size = mono_object_class (obj)->instance_size;
6651 /* do not copy the sync state */
6652 memcpy ((char*)obj + sizeof (MonoObject), (char*)src + sizeof (MonoObject),
6653 size - sizeof (MonoObject));
6654 if (ptr_in_nursery (obj) || ptr_on_stack (obj)) {
6658 if (rs->store_next < rs->end_set) {
6659 *(rs->store_next++) = (mword)obj | REMSET_OBJECT;
6663 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
6664 rs->next = REMEMBERED_SET;
6665 REMEMBERED_SET = rs;
6666 #ifdef HAVE_KW_THREAD
6667 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
6669 *(rs->store_next++) = (mword)obj | REMSET_OBJECT;
6674 * ######################################################################
6675 * ######## Collector debugging
6676 * ######################################################################
6679 const char*descriptor_types [] = {
6691 describe_ptr (char *ptr)
6693 GCMemSection *section;
6698 if (ptr_in_nursery (ptr)) {
6699 printf ("Pointer inside nursery.\n");
6701 for (section = section_list; section;) {
6702 if (ptr >= section->data && ptr < section->data + section->size)
6704 section = section->block.next;
6708 printf ("Pointer inside oldspace.\n");
6709 } else if (obj_is_from_pinned_alloc (ptr)) {
6710 printf ("Pointer is inside a pinned chunk.\n");
6712 printf ("Pointer unknown.\n");
6717 if (object_is_pinned (ptr))
6718 printf ("Object is pinned.\n");
6720 if (object_is_forwarded (ptr))
6721 printf ("Object is forwared.\n");
6723 // FIXME: Handle pointers to the inside of objects
6724 vtable = (MonoVTable*)LOAD_VTABLE (ptr);
6726 printf ("VTable: %p\n", vtable);
6727 if (vtable == NULL) {
6728 printf ("VTable is invalid (empty).\n");
6731 if (ptr_in_nursery (vtable)) {
6732 printf ("VTable is invalid (points inside nursery).\n");
6735 printf ("Class: %s\n", vtable->klass->name);
6737 desc = ((GCVTable*)vtable)->desc;
6738 printf ("Descriptor: %lx\n", (long)desc);
6741 printf ("Descriptor type: %d (%s)\n", type, descriptor_types [type]);
6745 find_in_remset_loc (mword *p, char *addr, gboolean *found)
6751 switch ((*p) & REMSET_TYPE_MASK) {
6752 case REMSET_LOCATION:
6753 if (*p == (mword)addr)
6757 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
6759 if ((void**)addr >= ptr && (void**)addr < ptr + count)
6763 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
6764 count = safe_object_get_size ((MonoObject*)ptr);
6765 count += (ALLOC_ALIGN - 1);
6766 count &= (ALLOC_ALIGN - 1);
6767 count /= sizeof (mword);
6768 if ((void**)addr >= ptr && (void**)addr < ptr + count)
6771 case REMSET_OTHER: {
6774 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
6778 switch (desc & 0x7) {
6779 case DESC_TYPE_RUN_LENGTH:
6780 OBJ_RUN_LEN_SIZE (skip_size, desc, ptr);
6781 /* The descriptor includes the size of MonoObject */
6782 skip_size -= sizeof (MonoObject);
6784 if ((void**)addr >= ptr && (void**)addr < ptr + (skip_size / sizeof (gpointer)))
6789 g_assert_not_reached ();
6793 case REMSET_ROOT_LOCATION:
6796 g_assert_not_reached ();
6801 g_assert_not_reached ();
6807 * Return whenever ADDR occurs in the remembered sets
6810 find_in_remsets (char *addr)
6813 SgenThreadInfo *info;
6814 RememberedSet *remset;
6815 GenericStoreRememberedSet *store_remset;
6817 gboolean found = FALSE;
6819 /* the global one */
6820 for (remset = global_remset; remset; remset = remset->next) {
6821 DEBUG (4, fprintf (gc_debug_file, "Scanning global remset range: %p-%p, size: %zd\n", remset->data, remset->store_next, remset->store_next - remset->data));
6822 for (p = remset->data; p < remset->store_next;) {
6823 p = find_in_remset_loc (p, addr, &found);
6829 /* the generic store ones */
6830 for (store_remset = generic_store_remsets; store_remset; store_remset = store_remset->next) {
6831 for (i = 0; i < STORE_REMSET_BUFFER_SIZE - 1; ++i) {
6832 if (store_remset->data [i] == addr)
6837 /* the per-thread ones */
6838 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
6839 for (info = thread_table [i]; info; info = info->next) {
6841 for (remset = info->remset; remset; remset = remset->next) {
6842 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for thread %p, range: %p-%p, size: %zd\n", info, remset->data, remset->store_next, remset->store_next - remset->data));
6843 for (p = remset->data; p < remset->store_next;) {
6844 p = find_in_remset_loc (p, addr, &found);
6849 for (j = 0; j < *info->store_remset_buffer_index_addr; ++j) {
6850 if ((*info->store_remset_buffer_addr) [j + 1] == addr)
6856 /* the freed thread ones */
6857 for (remset = freed_thread_remsets; remset; remset = remset->next) {
6858 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for freed thread, range: %p-%p, size: %zd\n", remset->data, remset->store_next, remset->store_next - remset->data));
6859 for (p = remset->data; p < remset->store_next;) {
6860 p = find_in_remset_loc (p, addr, &found);
6870 #define HANDLE_PTR(ptr,obj) do { \
6871 if (*(ptr) && (char*)*(ptr) >= nursery_start && (char*)*(ptr) < nursery_next) { \
6872 if (!find_in_remsets ((char*)(ptr))) { \
6873 fprintf (gc_debug_file, "Oldspace->newspace reference %p at offset %zd in object %p (%s.%s) not found in remsets.\n", *(ptr), (char*)(ptr) - (char*)(obj), (obj), ((MonoObject*)(obj))->vtable->klass->name_space, ((MonoObject*)(obj))->vtable->klass->name); \
6874 g_assert_not_reached (); \
6880 * Check that each object reference inside the area which points into the nursery
6881 * can be found in the remembered sets.
6883 static void __attribute__((noinline))
6884 check_remsets_for_area (char *start, char *end)
6887 int type_str = 0, type_rlen = 0, type_bitmap = 0, type_vector = 0, type_lbit = 0, type_complex = 0;
6888 new_obj_references = 0;
6889 obj_references_checked = 0;
6890 while (start < end) {
6891 if (!*(void**)start) {
6892 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
6895 vt = (GCVTable*)LOAD_VTABLE (start);
6896 DEBUG (8, fprintf (gc_debug_file, "Scanning object %p, vtable: %p (%s)\n", start, vt, vt->klass->name));
6898 MonoObject *obj = (MonoObject*)start;
6899 g_print ("found at %p (0x%lx): %s.%s\n", start, (long)vt->desc, obj->vtable->klass->name_space, obj->vtable->klass->name);
6902 #define SCAN_OBJECT_ACTION COUNT_OBJECT_TYPES
6903 #include "sgen-scan-object.h"
6908 * Perform consistency check of the heap.
6910 * Assumes the world is stopped.
6913 check_consistency (void)
6915 GCMemSection *section;
6917 // Need to add more checks
6918 // FIXME: Create a general heap enumeration function and use that
6920 DEBUG (1, fprintf (gc_debug_file, "Begin heap consistency check...\n"));
6922 // Check that oldspace->newspace pointers are registered with the collector
6923 for (section = section_list; section; section = section->block.next) {
6924 if (section->block.role == MEMORY_ROLE_GEN0)
6926 DEBUG (2, fprintf (gc_debug_file, "Scan of old section: %p-%p, size: %d\n", section->data, section->next_data, (int)(section->next_data - section->data)));
6927 check_remsets_for_area (section->data, section->next_data);
6930 DEBUG (1, fprintf (gc_debug_file, "Heap consistency check done.\n"));
6933 /* Check that the reference is valid */
6935 #define HANDLE_PTR(ptr,obj) do { \
6937 g_assert (safe_name (*(ptr)) != NULL); \
6944 * Perform consistency check on an object. Currently we only check that the
6945 * reference fields are valid.
6948 check_object (char *start)
6953 #include "sgen-scan-object.h"
6959 * ######################################################################
6960 * ######## Other mono public interface functions.
6961 * ######################################################################
6965 mono_gc_collect (int generation)
6969 if (generation == 0) {
6970 collect_nursery (0);
6972 major_collection ("user request");
6979 mono_gc_max_generation (void)
6985 mono_gc_collection_count (int generation)
6987 if (generation == 0)
6988 return num_minor_gcs;
6989 return num_major_gcs;
6993 mono_gc_get_used_size (void)
6996 GCMemSection *section;
6998 tot = los_memory_usage;
6999 for (section = section_list; section; section = section->block.next) {
7000 /* this is approximate... */
7001 tot += section->next_data - section->data;
7003 /* FIXME: account for pinned objects */
7009 mono_gc_get_heap_size (void)
7015 mono_gc_disable (void)
7023 mono_gc_enable (void)
7031 mono_gc_get_los_limit (void)
7033 return MAX_SMALL_OBJ_SIZE;
7037 mono_object_is_alive (MonoObject* o)
7043 mono_gc_get_generation (MonoObject *obj)
7045 if (ptr_in_nursery (obj))
7051 mono_gc_enable_events (void)
7056 mono_gc_weak_link_add (void **link_addr, MonoObject *obj, gboolean track)
7059 mono_gc_register_disappearing_link (obj, link_addr, track);
7064 mono_gc_weak_link_remove (void **link_addr)
7067 mono_gc_register_disappearing_link (NULL, link_addr, FALSE);
7072 mono_gc_weak_link_get (void **link_addr)
7076 return (MonoObject*) REVEAL_POINTER (*link_addr);
7080 mono_gc_make_descr_from_bitmap (gsize *bitmap, int numbits)
7082 if (numbits < ((sizeof (*bitmap) * 8) - ROOT_DESC_TYPE_SHIFT)) {
7083 return (void*)MAKE_ROOT_DESC (ROOT_DESC_BITMAP, bitmap [0]);
7085 mword complex = alloc_complex_descriptor (bitmap, numbits + 1);
7086 return (void*)MAKE_ROOT_DESC (ROOT_DESC_COMPLEX, complex);
7091 mono_gc_make_root_descr_user (MonoGCMarkFunc marker)
7095 g_assert (user_descriptors_next < MAX_USER_DESCRIPTORS);
7096 descr = (void*)MAKE_ROOT_DESC (ROOT_DESC_USER, (mword)user_descriptors_next);
7097 user_descriptors [user_descriptors_next ++] = marker;
7103 mono_gc_alloc_fixed (size_t size, void *descr)
7105 /* FIXME: do a single allocation */
7106 void *res = calloc (1, size);
7109 if (!mono_gc_register_root (res, size, descr)) {
7117 mono_gc_free_fixed (void* addr)
7119 mono_gc_deregister_root (addr);
7124 mono_gc_is_gc_thread (void)
7128 result = thread_info_lookup (ARCH_GET_THREAD ()) != NULL;
7134 mono_gc_base_init (void)
7138 struct sigaction sinfo;
7140 LOCK_INIT (gc_mutex);
7142 if (gc_initialized) {
7146 pagesize = mono_pagesize ();
7147 gc_debug_file = stderr;
7148 if ((env = getenv ("MONO_GC_DEBUG"))) {
7149 opts = g_strsplit (env, ",", -1);
7150 for (ptr = opts; ptr && *ptr; ptr ++) {
7152 if (opt [0] >= '0' && opt [0] <= '9') {
7153 gc_debug_level = atoi (opt);
7158 char *rf = g_strdup_printf ("%s.%d", opt, getpid ());
7159 gc_debug_file = fopen (rf, "wb");
7161 gc_debug_file = stderr;
7164 } else if (!strcmp (opt, "collect-before-allocs")) {
7165 collect_before_allocs = TRUE;
7166 } else if (!strcmp (opt, "check-at-minor-collections")) {
7167 consistency_check_at_minor_collection = TRUE;
7168 } else if (!strcmp (opt, "xdomain-checks")) {
7169 xdomain_checks = TRUE;
7170 } else if (!strcmp (opt, "clear-at-gc")) {
7171 nursery_clear_policy = CLEAR_AT_GC;
7172 } else if (g_str_has_prefix (opt, "heap-dump=")) {
7173 char *filename = strchr (opt, '=') + 1;
7174 nursery_clear_policy = CLEAR_AT_GC;
7175 heap_dump_file = fopen (filename, "w");
7177 fprintf (heap_dump_file, "<sgen-dump>\n");
7179 fprintf (stderr, "Invalid format for the MONO_GC_DEBUG env variable: '%s'\n", env);
7180 fprintf (stderr, "The format is: MONO_GC_DEBUG=[l[:filename]|<option>]+ where l is a debug level 0-9.\n");
7181 fprintf (stderr, "Valid options are: collect-before-allocs, check-at-minor-collections, xdomain-checks, clear-at-gc.\n");
7188 suspend_ack_semaphore_ptr = &suspend_ack_semaphore;
7189 MONO_SEM_INIT (&suspend_ack_semaphore, 0);
7191 sigfillset (&sinfo.sa_mask);
7192 sinfo.sa_flags = SA_RESTART | SA_SIGINFO;
7193 sinfo.sa_sigaction = suspend_handler;
7194 if (sigaction (suspend_signal_num, &sinfo, NULL) != 0) {
7195 g_error ("failed sigaction");
7198 sinfo.sa_handler = restart_handler;
7199 if (sigaction (restart_signal_num, &sinfo, NULL) != 0) {
7200 g_error ("failed sigaction");
7203 sigfillset (&suspend_signal_mask);
7204 sigdelset (&suspend_signal_mask, restart_signal_num);
7206 global_remset = alloc_remset (1024, NULL);
7207 global_remset->next = NULL;
7209 pthread_key_create (&remembered_set_key, unregister_thread);
7211 #ifndef HAVE_KW_THREAD
7212 pthread_key_create (&thread_info_key, NULL);
7215 gc_initialized = TRUE;
7217 mono_gc_register_thread (&sinfo);
7221 mono_gc_get_suspend_signal (void)
7223 return suspend_signal_num;
7232 #ifdef HAVE_KW_THREAD
7233 #define EMIT_TLS_ACCESS(mb,dummy,offset) do { \
7234 mono_mb_emit_byte ((mb), MONO_CUSTOM_PREFIX); \
7235 mono_mb_emit_byte ((mb), CEE_MONO_TLS); \
7236 mono_mb_emit_i4 ((mb), (offset)); \
7239 #define EMIT_TLS_ACCESS(mb,member,dummy) do { \
7240 mono_mb_emit_byte ((mb), MONO_CUSTOM_PREFIX); \
7241 mono_mb_emit_byte ((mb), CEE_MONO_TLS); \
7242 mono_mb_emit_i4 ((mb), thread_info_key); \
7243 mono_mb_emit_icon ((mb), G_STRUCT_OFFSET (SgenThreadInfo, member)); \
7244 mono_mb_emit_byte ((mb), CEE_ADD); \
7245 mono_mb_emit_byte ((mb), CEE_LDIND_I); \
7249 #ifdef MANAGED_ALLOCATION
7250 /* FIXME: Do this in the JIT, where specialized allocation sequences can be created
7251 * for each class. This is currently not easy to do, as it is hard to generate basic
7252 * blocks + branches, but it is easy with the linear IL codebase.
7254 * For this to work we'd need to solve the TLAB race, first. Now we
7255 * require the allocator to be in a few known methods to make sure
7256 * that they are executed atomically via the restart mechanism.
7259 create_allocator (int atype)
7261 int p_var, size_var;
7262 guint32 slowpath_branch, max_size_branch;
7263 MonoMethodBuilder *mb;
7265 MonoMethodSignature *csig;
7266 static gboolean registered = FALSE;
7267 int tlab_next_addr_var, new_next_var;
7270 #ifdef HAVE_KW_THREAD
7271 int tlab_next_addr_offset = -1;
7272 int tlab_temp_end_offset = -1;
7274 MONO_THREAD_VAR_OFFSET (tlab_next_addr, tlab_next_addr_offset);
7275 MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
7277 g_assert (tlab_next_addr_offset != -1);
7278 g_assert (tlab_temp_end_offset != -1);
7282 mono_register_jit_icall (mono_gc_alloc_obj, "mono_gc_alloc_obj", mono_create_icall_signature ("object ptr int"), FALSE);
7283 mono_register_jit_icall (mono_gc_alloc_vector, "mono_gc_alloc_vector", mono_create_icall_signature ("object ptr int int"), FALSE);
7287 if (atype == ATYPE_NORMAL)
7289 else if (atype == ATYPE_VECTOR)
7292 g_assert_not_reached ();
7294 csig = mono_metadata_signature_alloc (mono_defaults.corlib, num_params);
7295 csig->ret = &mono_defaults.object_class->byval_arg;
7296 for (i = 0; i < num_params; ++i)
7297 csig->params [i] = &mono_defaults.int_class->byval_arg;
7299 mb = mono_mb_new (mono_defaults.object_class, "Alloc", MONO_WRAPPER_ALLOC);
7300 size_var = mono_mb_add_local (mb, &mono_defaults.int32_class->byval_arg);
7301 if (atype == ATYPE_NORMAL) {
7302 /* size = vtable->klass->instance_size; */
7303 mono_mb_emit_ldarg (mb, 0);
7304 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoVTable, klass));
7305 mono_mb_emit_byte (mb, CEE_ADD);
7306 mono_mb_emit_byte (mb, CEE_LDIND_I);
7307 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoClass, instance_size));
7308 mono_mb_emit_byte (mb, CEE_ADD);
7309 /* FIXME: assert instance_size stays a 4 byte integer */
7310 mono_mb_emit_byte (mb, CEE_LDIND_U4);
7311 mono_mb_emit_stloc (mb, size_var);
7312 } else if (atype == ATYPE_VECTOR) {
7313 MonoExceptionClause *clause;
7315 MonoClass *oom_exc_class;
7318 clause = mono_image_alloc0 (mono_defaults.corlib, sizeof (MonoExceptionClause));
7319 clause->try_offset = mono_mb_get_label (mb);
7321 /* vtable->klass->sizes.element_size */
7322 mono_mb_emit_ldarg (mb, 0);
7323 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoVTable, klass));
7324 mono_mb_emit_byte (mb, CEE_ADD);
7325 mono_mb_emit_byte (mb, CEE_LDIND_I);
7326 mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoClass, sizes.element_size));
7327 mono_mb_emit_byte (mb, CEE_ADD);
7328 mono_mb_emit_byte (mb, CEE_LDIND_U4);
7331 mono_mb_emit_ldarg (mb, 1);
7332 mono_mb_emit_byte (mb, CEE_MUL_OVF_UN);
7333 /* + sizeof (MonoArray) */
7334 mono_mb_emit_icon (mb, sizeof (MonoArray));
7335 mono_mb_emit_byte (mb, CEE_ADD_OVF_UN);
7336 mono_mb_emit_stloc (mb, size_var);
7338 pos_leave = mono_mb_emit_branch (mb, CEE_LEAVE);
7341 clause->flags = MONO_EXCEPTION_CLAUSE_NONE;
7342 clause->try_len = mono_mb_get_pos (mb) - clause->try_offset;
7343 clause->data.catch_class = mono_class_from_name (mono_defaults.corlib,
7344 "System", "OverflowException");
7345 g_assert (clause->data.catch_class);
7346 clause->handler_offset = mono_mb_get_label (mb);
7348 oom_exc_class = mono_class_from_name (mono_defaults.corlib,
7349 "System", "OutOfMemoryException");
7350 g_assert (oom_exc_class);
7351 ctor = mono_class_get_method_from_name (oom_exc_class, ".ctor", 0);
7354 mono_mb_emit_op (mb, CEE_NEWOBJ, ctor);
7355 mono_mb_emit_byte (mb, CEE_THROW);
7357 clause->handler_len = mono_mb_get_pos (mb) - clause->handler_offset;
7358 mono_mb_set_clauses (mb, 1, clause);
7359 mono_mb_patch_branch (mb, pos_leave);
7362 g_assert_not_reached ();
7365 /* size += ALLOC_ALIGN - 1; */
7366 mono_mb_emit_ldloc (mb, size_var);
7367 mono_mb_emit_icon (mb, ALLOC_ALIGN - 1);
7368 mono_mb_emit_byte (mb, CEE_ADD);
7369 /* size &= ~(ALLOC_ALIGN - 1); */
7370 mono_mb_emit_icon (mb, ~(ALLOC_ALIGN - 1));
7371 mono_mb_emit_byte (mb, CEE_AND);
7372 mono_mb_emit_stloc (mb, size_var);
7374 /* if (size > MAX_SMALL_OBJ_SIZE) goto slowpath */
7375 mono_mb_emit_ldloc (mb, size_var);
7376 mono_mb_emit_icon (mb, MAX_SMALL_OBJ_SIZE);
7377 max_size_branch = mono_mb_emit_short_branch (mb, MONO_CEE_BGT_S);
7380 * We need to modify tlab_next, but the JIT only supports reading, so we read
7381 * another tls var holding its address instead.
7384 /* tlab_next_addr (local) = tlab_next_addr (TLS var) */
7385 tlab_next_addr_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7386 EMIT_TLS_ACCESS (mb, tlab_next_addr, tlab_next_addr_offset);
7387 mono_mb_emit_stloc (mb, tlab_next_addr_var);
7389 /* p = (void**)tlab_next; */
7390 p_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7391 mono_mb_emit_ldloc (mb, tlab_next_addr_var);
7392 mono_mb_emit_byte (mb, CEE_LDIND_I);
7393 mono_mb_emit_stloc (mb, p_var);
7395 /* new_next = (char*)p + size; */
7396 new_next_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7397 mono_mb_emit_ldloc (mb, p_var);
7398 mono_mb_emit_ldloc (mb, size_var);
7399 mono_mb_emit_byte (mb, CEE_CONV_I);
7400 mono_mb_emit_byte (mb, CEE_ADD);
7401 mono_mb_emit_stloc (mb, new_next_var);
7403 /* tlab_next = new_next */
7404 mono_mb_emit_ldloc (mb, tlab_next_addr_var);
7405 mono_mb_emit_ldloc (mb, new_next_var);
7406 mono_mb_emit_byte (mb, CEE_STIND_I);
7408 /* if (G_LIKELY (new_next < tlab_temp_end)) */
7409 mono_mb_emit_ldloc (mb, new_next_var);
7410 EMIT_TLS_ACCESS (mb, tlab_temp_end, tlab_temp_end_offset);
7411 slowpath_branch = mono_mb_emit_short_branch (mb, MONO_CEE_BLT_UN_S);
7415 mono_mb_patch_short_branch (mb, max_size_branch);
7417 mono_mb_emit_byte (mb, MONO_CUSTOM_PREFIX);
7418 mono_mb_emit_byte (mb, CEE_MONO_NOT_TAKEN);
7420 /* FIXME: mono_gc_alloc_obj takes a 'size_t' as an argument, not an int32 */
7421 mono_mb_emit_ldarg (mb, 0);
7422 mono_mb_emit_ldloc (mb, size_var);
7423 if (atype == ATYPE_NORMAL) {
7424 mono_mb_emit_icall (mb, mono_gc_alloc_obj);
7425 } else if (atype == ATYPE_VECTOR) {
7426 mono_mb_emit_ldarg (mb, 1);
7427 mono_mb_emit_icall (mb, mono_gc_alloc_vector);
7429 g_assert_not_reached ();
7431 mono_mb_emit_byte (mb, CEE_RET);
7434 mono_mb_patch_short_branch (mb, slowpath_branch);
7436 /* FIXME: Memory barrier */
7439 mono_mb_emit_ldloc (mb, p_var);
7440 mono_mb_emit_ldarg (mb, 0);
7441 mono_mb_emit_byte (mb, CEE_STIND_I);
7443 if (atype == ATYPE_VECTOR) {
7444 /* arr->max_length = max_length; */
7445 mono_mb_emit_ldloc (mb, p_var);
7446 mono_mb_emit_ldflda (mb, G_STRUCT_OFFSET (MonoArray, max_length));
7447 mono_mb_emit_ldarg (mb, 1);
7448 mono_mb_emit_byte (mb, CEE_STIND_I);
7452 mono_mb_emit_ldloc (mb, p_var);
7453 mono_mb_emit_byte (mb, CEE_RET);
7455 res = mono_mb_create_method (mb, csig, 8);
7457 mono_method_get_header (res)->init_locals = FALSE;
7462 static MonoMethod* alloc_method_cache [ATYPE_NUM];
7463 static MonoMethod *write_barrier_method;
7466 is_ip_in_managed_allocator (MonoDomain *domain, gpointer ip)
7474 ji = mono_jit_info_table_find (domain, ip);
7477 method = ji->method;
7479 if (method == write_barrier_method)
7481 for (i = 0; i < ATYPE_NUM; ++i)
7482 if (method == alloc_method_cache [i])
7488 * Generate an allocator method implementing the fast path of mono_gc_alloc_obj ().
7489 * The signature of the called method is:
7490 * object allocate (MonoVTable *vtable)
7493 mono_gc_get_managed_allocator (MonoVTable *vtable, gboolean for_box)
7495 #ifdef MANAGED_ALLOCATION
7496 MonoClass *klass = vtable->klass;
7498 #ifdef HAVE_KW_THREAD
7499 int tlab_next_offset = -1;
7500 int tlab_temp_end_offset = -1;
7501 MONO_THREAD_VAR_OFFSET (tlab_next, tlab_next_offset);
7502 MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
7504 if (tlab_next_offset == -1 || tlab_temp_end_offset == -1)
7508 if (!mono_runtime_has_tls_get ())
7510 if (klass->instance_size > tlab_size)
7512 if (klass->has_finalize || klass->marshalbyref || (mono_profiler_get_events () & MONO_PROFILE_ALLOCATIONS))
7516 if (klass->byval_arg.type == MONO_TYPE_STRING)
7518 if (collect_before_allocs)
7521 return mono_gc_get_managed_allocator_by_type (ATYPE_NORMAL);
7528 mono_gc_get_managed_array_allocator (MonoVTable *vtable, int rank)
7530 #ifdef MANAGED_ALLOCATION
7531 MonoClass *klass = vtable->klass;
7533 #ifdef HAVE_KW_THREAD
7534 int tlab_next_offset = -1;
7535 int tlab_temp_end_offset = -1;
7536 MONO_THREAD_VAR_OFFSET (tlab_next, tlab_next_offset);
7537 MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
7539 if (tlab_next_offset == -1 || tlab_temp_end_offset == -1)
7545 if (!mono_runtime_has_tls_get ())
7547 if (mono_profiler_get_events () & MONO_PROFILE_ALLOCATIONS)
7549 if (collect_before_allocs)
7551 g_assert (!klass->has_finalize && !klass->marshalbyref);
7553 return mono_gc_get_managed_allocator_by_type (ATYPE_VECTOR);
7560 mono_gc_get_managed_allocator_type (MonoMethod *managed_alloc)
7562 #ifdef MANAGED_ALLOCATION
7565 for (i = 0; i < ATYPE_NUM; ++i)
7566 if (managed_alloc == alloc_method_cache [i])
7569 g_assert_not_reached ();
7574 mono_gc_get_managed_allocator_by_type (int atype)
7576 #ifdef MANAGED_ALLOCATION
7579 if (!mono_runtime_has_tls_get ())
7582 mono_loader_lock ();
7583 res = alloc_method_cache [atype];
7585 res = alloc_method_cache [atype] = create_allocator (atype);
7586 mono_loader_unlock ();
7594 mono_gc_get_managed_allocator_types (void)
7601 mono_gc_get_write_barrier (void)
7604 MonoMethodBuilder *mb;
7605 MonoMethodSignature *sig;
7606 #ifdef MANAGED_WBARRIER
7607 int label_no_wb_1, label_no_wb_2, label_no_wb_3, label_no_wb_4, label_need_wb, label_slow_path;
7608 int buffer_var, buffer_index_var, dummy_var;
7610 #ifdef HAVE_KW_THREAD
7611 int stack_end_offset = -1, store_remset_buffer_offset = -1;
7612 int store_remset_buffer_index_offset = -1, store_remset_buffer_index_addr_offset = -1;
7614 MONO_THREAD_VAR_OFFSET (stack_end, stack_end_offset);
7615 g_assert (stack_end_offset != -1);
7616 MONO_THREAD_VAR_OFFSET (store_remset_buffer, store_remset_buffer_offset);
7617 g_assert (store_remset_buffer_offset != -1);
7618 MONO_THREAD_VAR_OFFSET (store_remset_buffer_index, store_remset_buffer_index_offset);
7619 g_assert (store_remset_buffer_index_offset != -1);
7620 MONO_THREAD_VAR_OFFSET (store_remset_buffer_index_addr, store_remset_buffer_index_addr_offset);
7621 g_assert (store_remset_buffer_index_addr_offset != -1);
7625 // FIXME: Maybe create a separate version for ctors (the branch would be
7626 // correctly predicted more times)
7627 if (write_barrier_method)
7628 return write_barrier_method;
7630 /* Create the IL version of mono_gc_barrier_generic_store () */
7631 sig = mono_metadata_signature_alloc (mono_defaults.corlib, 1);
7632 sig->ret = &mono_defaults.void_class->byval_arg;
7633 sig->params [0] = &mono_defaults.int_class->byval_arg;
7635 mb = mono_mb_new (mono_defaults.object_class, "wbarrier", MONO_WRAPPER_WRITE_BARRIER);
7637 #ifdef MANAGED_WBARRIER
7638 if (mono_runtime_has_tls_get ()) {
7639 #ifdef ALIGN_NURSERY
7640 // if (ptr_in_nursery (ptr)) return;
7642 * Masking out the bits might be faster, but we would have to use 64 bit
7643 * immediates, which might be slower.
7645 mono_mb_emit_ldarg (mb, 0);
7646 mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
7647 mono_mb_emit_byte (mb, CEE_SHR_UN);
7648 mono_mb_emit_icon (mb, (mword)nursery_start >> DEFAULT_NURSERY_BITS);
7649 label_no_wb_1 = mono_mb_emit_branch (mb, CEE_BEQ);
7651 // if (!ptr_in_nursery (*ptr)) return;
7652 mono_mb_emit_ldarg (mb, 0);
7653 mono_mb_emit_byte (mb, CEE_LDIND_I);
7654 mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
7655 mono_mb_emit_byte (mb, CEE_SHR_UN);
7656 mono_mb_emit_icon (mb, (mword)nursery_start >> DEFAULT_NURSERY_BITS);
7657 label_no_wb_2 = mono_mb_emit_branch (mb, CEE_BNE_UN);
7660 g_assert_not_reached ();
7663 // if (ptr >= stack_end) goto need_wb;
7664 mono_mb_emit_ldarg (mb, 0);
7665 EMIT_TLS_ACCESS (mb, stack_end, stack_end_offset);
7666 label_need_wb = mono_mb_emit_branch (mb, CEE_BGE_UN);
7668 // if (ptr >= stack_start) return;
7669 dummy_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7670 mono_mb_emit_ldarg (mb, 0);
7671 mono_mb_emit_ldloc_addr (mb, dummy_var);
7672 label_no_wb_3 = mono_mb_emit_branch (mb, CEE_BGE_UN);
7675 mono_mb_patch_branch (mb, label_need_wb);
7677 // buffer = STORE_REMSET_BUFFER;
7678 buffer_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
7679 EMIT_TLS_ACCESS (mb, store_remset_buffer, store_remset_buffer_offset);
7680 mono_mb_emit_stloc (mb, buffer_var);
7682 // buffer_index = STORE_REMSET_BUFFER_INDEX;
7683 buffer_index_var = mono_mb_add_local (mb, &mono_defaults.int32_class->byval_arg);
7684 EMIT_TLS_ACCESS (mb, store_remset_buffer_index, store_remset_buffer_index_offset);
7685 mono_mb_emit_stloc (mb, buffer_index_var);
7687 // if (buffer [buffer_index] == ptr) return;
7688 mono_mb_emit_ldloc (mb, buffer_var);
7689 mono_mb_emit_ldloc (mb, buffer_index_var);
7690 g_assert (sizeof (gpointer) == 4 || sizeof (gpointer) == 8);
7691 mono_mb_emit_icon (mb, sizeof (gpointer) == 4 ? 2 : 3);
7692 mono_mb_emit_byte (mb, CEE_SHL);
7693 mono_mb_emit_byte (mb, CEE_ADD);
7694 mono_mb_emit_byte (mb, CEE_LDIND_I);
7695 mono_mb_emit_ldarg (mb, 0);
7696 label_no_wb_4 = mono_mb_emit_branch (mb, CEE_BEQ);
7699 mono_mb_emit_ldloc (mb, buffer_index_var);
7700 mono_mb_emit_icon (mb, 1);
7701 mono_mb_emit_byte (mb, CEE_ADD);
7702 mono_mb_emit_stloc (mb, buffer_index_var);
7704 // if (buffer_index >= STORE_REMSET_BUFFER_SIZE) goto slow_path;
7705 mono_mb_emit_ldloc (mb, buffer_index_var);
7706 mono_mb_emit_icon (mb, STORE_REMSET_BUFFER_SIZE);
7707 label_slow_path = mono_mb_emit_branch (mb, CEE_BGE);
7709 // buffer [buffer_index] = ptr;
7710 mono_mb_emit_ldloc (mb, buffer_var);
7711 mono_mb_emit_ldloc (mb, buffer_index_var);
7712 g_assert (sizeof (gpointer) == 4 || sizeof (gpointer) == 8);
7713 mono_mb_emit_icon (mb, sizeof (gpointer) == 4 ? 2 : 3);
7714 mono_mb_emit_byte (mb, CEE_SHL);
7715 mono_mb_emit_byte (mb, CEE_ADD);
7716 mono_mb_emit_ldarg (mb, 0);
7717 mono_mb_emit_byte (mb, CEE_STIND_I);
7719 // STORE_REMSET_BUFFER_INDEX = buffer_index;
7720 EMIT_TLS_ACCESS (mb, store_remset_buffer_index_addr, store_remset_buffer_index_addr_offset);
7721 mono_mb_emit_ldloc (mb, buffer_index_var);
7722 mono_mb_emit_byte (mb, CEE_STIND_I4);
7725 mono_mb_patch_branch (mb, label_no_wb_1);
7726 mono_mb_patch_branch (mb, label_no_wb_2);
7727 mono_mb_patch_branch (mb, label_no_wb_3);
7728 mono_mb_patch_branch (mb, label_no_wb_4);
7729 mono_mb_emit_byte (mb, CEE_RET);
7732 mono_mb_patch_branch (mb, label_slow_path);
7736 mono_mb_emit_ldarg (mb, 0);
7737 mono_mb_emit_icall (mb, mono_gc_wbarrier_generic_nostore);
7738 mono_mb_emit_byte (mb, CEE_RET);
7740 res = mono_mb_create_method (mb, sig, 16);
7743 mono_loader_lock ();
7744 if (write_barrier_method) {
7745 /* Already created */
7746 mono_free_method (res);
7748 /* double-checked locking */
7749 mono_memory_barrier ();
7750 write_barrier_method = res;
7752 mono_loader_unlock ();
7754 return write_barrier_method;
7757 #endif /* HAVE_SGEN_GC */