2 * sgen-gc.c: Simple generational GC.
5 * Paolo Molaro (lupus@ximian.com)
7 * Copyright (C) 2005-2006 Novell, Inc
9 * Thread start/stop adapted from Boehm's GC:
10 * Copyright (c) 1994 by Xerox Corporation. All rights reserved.
11 * Copyright (c) 1996 by Silicon Graphics. All rights reserved.
12 * Copyright (c) 1998 by Fergus Henderson. All rights reserved.
13 * Copyright (c) 2000-2004 by Hewlett-Packard Company. All rights reserved.
15 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
16 * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
18 * Permission is hereby granted to use or copy this program
19 * for any purpose, provided the above notices are retained on all copies.
20 * Permission to modify the code and to distribute modified code is granted,
21 * provided the above notices are retained, and a notice that the code was
22 * modified is included with the above copyright notice.
24 * All the rest of the code is LGPL.
26 * Important: allocation provides always zeroed memory, having to do
27 * a memset after allocation is deadly for performance.
28 * Memory usage at startup is currently as follows:
30 * 64 KB internal space
32 * We should provide a small memory config with half the sizes
34 * We currently try to make as few mono assumptions as possible:
35 * 1) 2-word header with no GC pointers in it (firts vtable, second to store the
37 * 2) gc descriptor is the second word in the vtable (first word in the class)
38 * 3) 8 byte alignment is the minimum and enough (not true for special structures, FIXME)
39 * 4) there is a function to get an object's size and the number of
40 * elements in an array.
41 * 5) we know the special way bounds are allocated for complex arrays
43 * Always try to keep stack usage to a minimum: no recursive behaviour
44 * and no large stack allocs.
46 * General description.
47 * Objects are initially allocated in a nursery using a fast bump-pointer technique.
48 * When the nursery is full we start a nursery collection: this is performed with a
50 * When the old generation is full we start a copying GC of the old generation as well:
51 * this will be changed to mark/compact in the future.
52 * The things that complicate this description are:
53 * *) pinned objects: we can't move them so we need to keep track of them
54 * *) no precise info of the thread stacks and registers: we need to be able to
55 * quickly find the objects that may be referenced conservatively and pin them
56 * (this makes the first issues more important)
57 * *) large objects are too expensive to be dealt with using copying GC: we handle them
58 * with mark/sweep during major collections
59 * *) some objects need to not move even if they are small (interned strings, Type handles):
60 * we use mark/sweep for them, too: they are not allocated in the nursery, but inside
61 * PinnedChunks regions
66 *) change the jit to emit write barrier calls when needed (we
67 can have specialized write barriers): done with icalls, still need to
68 use some specialized barriers
69 *) we could have a function pointer in MonoClass to implement
70 customized write barriers for value types
71 *) the write barrier code could be isolated in a couple of functions: when a
72 thread is stopped if it's inside the barrier it is let go again
73 until we stop outside of them (not really needed, see below GC-safe points)
74 *) investigate the stuff needed to advance a thread to a GC-safe
75 point (single-stepping, read from unmapped memory etc) and implement it
76 Not needed yet: since we treat the objects reachable from the stack/regs as
77 roots, we store the ptr and exec the write barrier so there is no race.
78 We may need this to solve the issue with setting the length of arrays and strings.
79 We may need this also for handling precise info on stacks, even simple things
80 as having uninitialized data on the stack and having to wait for the prolog
81 to zero it. Not an issue for the last frame that we scan conservatively.
82 We could always not trust the value in the slots anyway.
83 *) make the jit info table lock free
84 *) modify the jit to save info about references in stack locations:
85 this can be done just for locals as a start, so that at least
86 part of the stack is handled precisely.
87 *) Make the debug printf stuff thread and signal safe.
88 *) test/fix 64 bit issues
89 *) test/fix endianess issues
91 *) add batch moving profile info
92 *) add more timing info
93 *) there is a possible race when an array or string is created: the vtable is set,
94 but the length is set only later so if the GC needs to scan the object in that window,
95 it won't get the correct size for the object. The object can't have references and it will
96 be pinned, but a free memory fragment may be created that overlaps with it.
97 We should change the array max_length field to be at the same offset as the string length:
98 this way we can have a single special alloc function for them that sets the length.
99 Multi-dim arrays have the same issue for rank == 1 for the bounds data.
100 *) implement a card table as the write barrier instead of remembered sets?
101 *) some sort of blacklist support?
102 *) fin_ready_list is part of the root set, too
103 *) consider lowering the large object min size to 16/32KB or so and benchmark
104 *) once mark-compact is implemented we could still keep the
105 copying collector for the old generation and use it if we think
106 it is better (small heaps and no pinning object in the old
108 *) avoid the memory store from copy_object when not needed.
109 *) optimize the write barriers fastpath to happen in managed code
110 *) add an option to mmap the whole heap in one chunk: it makes for many
111 simplifications in the checks (put the nursery at the top and just use a single
112 check for inclusion/exclusion): the issue this has is that on 32 bit systems it's
113 not flexible (too much of the address space may be used by default or we can't
114 increase the heap as needed) and we'd need a race-free mechanism to return memory
115 back to the system (mprotect(PROT_NONE) will still keep the memory allocated if it
116 was written to, munmap is needed, but the following mmap may not find the same segment
118 *) memzero the fragments after restarting the world and optionally a smaller chunk at a time
119 *) an additional strategy to realloc/expand the nursery when fully pinned is to start
120 allocating objects in the old generation. This means that we can't optimize away write
121 barrier calls in ctors (but that is not valid for other reasons, too).
122 *) add write barriers to the Clone methods
130 #include <semaphore.h>
134 #include <sys/types.h>
135 #include <sys/stat.h>
136 #include <sys/mman.h>
137 #include <sys/time.h>
140 #include "metadata/metadata-internals.h"
141 #include "metadata/class-internals.h"
142 #include "metadata/gc-internal.h"
143 #include "metadata/object-internals.h"
144 #include "metadata/threads.h"
145 #include "metadata/sgen-gc.h"
146 #include "metadata/mono-gc.h"
147 #include "utils/mono-mmap.h"
151 * ######################################################################
152 * ######## Types and constants used by the GC.
153 * ######################################################################
155 #if SIZEOF_VOID_P == 4
156 typedef guint32 mword;
158 typedef guint64 mword;
161 static int gc_initialized = 0;
162 static int gc_debug_level = 0;
163 static FILE* gc_debug_file;
166 mono_gc_flush_info (void)
168 fflush (gc_debug_file);
171 #define MAX_DEBUG_LEVEL 9
172 #define DEBUG(level,a) do {if ((level) <= MAX_DEBUG_LEVEL && (level) <= gc_debug_level) a;} while (0)
174 #define TV_DECLARE(name) struct timeval name
175 #define TV_GETTIME(tv) gettimeofday (&(tv), NULL)
176 #define TV_ELAPSED(start,end) (int)((((end).tv_sec - (start).tv_sec) * 1000000) + end.tv_usec - start.tv_usec)
178 #define GC_BITS_PER_WORD (sizeof (mword) * 8)
188 /* each request from the OS ends up in a GCMemSection */
189 typedef struct _GCMemSection GCMemSection;
190 struct _GCMemSection {
194 /* pointer where more data could be allocated if it fits */
198 * scan starts is an array of pointers to objects equally spaced in the allocation area
199 * They let use quickly find pinned objects from pinning pointers.
202 /* in major collections indexes in the pin_queue for objects that pin this section */
205 unsigned short num_scan_start;
209 /* large object space struct: 64+ KB */
210 /* we could make this limit much smaller to avoid memcpy copy
211 * and potentially have more room in the GC descriptor: need to measure
212 * This also means that such small OS objects will need to be
213 * allocated in a different way (using pinned chunks).
214 * We may want to put large but smaller than 64k objects in the fixed space
215 * when we move the object from one generation to another (to limit the
216 * pig in the snake effect).
217 * Note: it may be worth to have an optimized copy function, since we can
218 * assume that objects are aligned and have a multiple of 8 size.
219 * FIXME: This structure needs to be a multiple of 8 bytes in size: this is not
220 * true if MONO_ZERO_LEN_ARRAY is nonzero.
222 typedef struct _LOSObject LOSObject;
225 mword size; /* this is the object size */
226 int dummy; /* to have a sizeof (LOSObject) a multiple of ALLOC_ALIGN and data starting at same alignment */
228 char data [MONO_ZERO_LEN_ARRAY];
231 /* Pinned objects are allocated in the LOS space if bigger than half a page
232 * or from freelists otherwise. We assume that pinned objects are relatively few
233 * and they have a slow dying speed (like interned strings, thread objects).
234 * As such they will be collected only at major collections.
235 * free lists are not global: when we need memory we allocate a PinnedChunk.
236 * Each pinned chunk is made of several pages, the first of wich is used
237 * internally for bookeeping (here think of a page as 4KB). The bookeeping
238 * includes the freelists vectors and info about the object size of each page
239 * in the pinned chunk. So, when needed, a free page is found in a pinned chunk,
240 * a size is assigned to it, the page is divided in the proper chunks and each
241 * chunk is added to the freelist. To not waste space, the remaining space in the
242 * first page is used as objects of size 16 or 32 (need to measure which are more
244 * We use this same structure to allocate memory used internally by the GC, so
245 * we never use malloc/free if we need to alloc during collection: the world is stopped
246 * and malloc/free will deadlock.
247 * When we want to iterate over pinned objects, we just scan a page at a time
248 * linearly according to the size of objects in the page: the next pointer used to link
249 * the items in the freelist uses the same word as the vtable. Since we keep freelists
250 * for each pinned chunk, if the word points outside the pinned chunk it means
252 * We could avoid this expensive scanning in creative ways. We could have a policy
253 * of putting in the pinned space only objects we know about that have no struct fields
254 * with references and we can easily use a even expensive write barrier for them,
255 * since pointer writes on such objects should be rare.
256 * The best compromise is to just alloc interned strings and System.MonoType in them.
257 * It would be nice to allocate MonoThread in it, too: must check that we properly
258 * use write barriers so we don't have to do any expensive scanning of the whole pinned
259 * chunk list during minor collections. We can avoid it now because we alloc in it only
260 * reference-free objects.
262 #define PINNED_FIRST_SLOT_SIZE (sizeof (gpointer) * 4)
263 #define MAX_FREELIST_SIZE 2048
264 #define PINNED_PAGE_SIZE (4096)
265 #define PINNED_CHUNK_MIN_SIZE (4096*8)
266 typedef struct _PinnedChunk PinnedChunk;
267 struct _PinnedChunk {
270 int *page_sizes; /* a 0 means the page is still unused */
273 void *data [1]; /* page sizes and free lists are stored here */
277 * The young generation is divided into fragments. This is because
278 * we can hand one fragments to a thread for lock-less fast alloc and
279 * because the young generation ends up fragmented anyway by pinned objects.
280 * Once a collection is done, a list of fragments is created. When doing
281 * thread local alloc we use smallish nurseries so we allow new threads to
282 * allocate memory from gen0 without triggering a collection. Threads that
283 * are found to allocate lots of memory are given bigger fragments. This
284 * should make the finalizer thread use little nursery memory after a while.
285 * We should start assigning threads very small fragments: if there are many
286 * threads the nursery will be full of reserved space that the threads may not
287 * use at all, slowing down allocation speed.
289 typedef struct _Fragment Fragment;
293 char *fragment_start;
294 char *fragment_limit; /* the current soft limit for allocation */
298 /* the runtime can register areas of memory as roots: we keep two lists of roots,
299 * a pinned root set for conservatively scanned roots and a normal one for
300 * precisely scanned roots (currently implemented as a single list).
302 typedef struct _RootRecord RootRecord;
310 /* for use with write barriers */
311 typedef struct _RememberedSet RememberedSet;
312 struct _RememberedSet {
316 mword data [MONO_ZERO_LEN_ARRAY];
319 /* we have 4 possible values in the low 2 bits */
321 REMSET_LOCATION, /* just a pointer to the exact location */
322 REMSET_RANGE, /* range of pointer fields */
323 REMSET_OBJECT, /* mark all the object for scanning */
324 REMSET_TYPE_MASK = 0x3
327 static __thread RememberedSet *remembered_set MONO_TLS_FAST;
328 static pthread_key_t remembered_set_key;
329 static RememberedSet *global_remset;
330 static int store_to_global_remset = 0;
332 /* FIXME: later choose a size that takes into account the RememberedSet struct
333 * and doesn't waste any alloc paddin space.
335 #define DEFAULT_REMSET_SIZE 1024
336 static RememberedSet* alloc_remset (int size, gpointer id);
338 /* Structure that corresponds to a MonoVTable: desc is a mword so requires
339 * no cast from a pointer to an integer
346 /* these bits are set in the object vtable: we could merge them since an object can be
347 * either pinned or forwarded but not both.
348 * We store them in the vtable slot because the bits are used in the sync block for
349 * other purpouses: if we merge them and alloc the sync blocks aligned to 8 bytes, we can change
350 * this and use bit 3 in the syncblock (with the lower two bits both set for forwarded, that
351 * would be an invalid combination for the monitor and hash code).
352 * The values are already shifted.
353 * The forwarding address is stored in the sync block.
355 #define FORWARDED_BIT 1
357 #define VTABLE_BITS_MASK 0x3
359 /* returns NULL if not forwarded, or the forwarded address */
360 #define object_is_forwarded(obj) (((mword*)(obj))[0] & FORWARDED_BIT? (void*)(((mword*)(obj))[1]): NULL)
361 /* set the forwarded address fw_addr for object obj */
362 #define forward_object(obj,fw_addr) do { \
363 ((mword*)(obj))[0] |= FORWARDED_BIT; \
364 ((mword*)(obj))[1] = (mword)(fw_addr); \
367 #define object_is_pinned(obj) (((mword*)(obj))[0] & PINNED_BIT)
368 #define pin_object(obj) do { \
369 ((mword*)(obj))[0] |= PINNED_BIT; \
371 #define unpin_object(obj) do { \
372 ((mword*)(obj))[0] &= ~PINNED_BIT; \
377 * Since we set bits in the vtable, use the macro to load it from the pointer to
378 * an object that is potentially pinned.
380 #define LOAD_VTABLE(addr) ((*(mword*)(addr)) & ~VTABLE_BITS_MASK)
383 safe_name (void* obj)
385 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (obj);
386 return vt->klass->name;
390 safe_object_get_size (MonoObject* o)
392 MonoClass *klass = ((MonoVTable*)LOAD_VTABLE (o))->klass;
393 if (klass == mono_defaults.string_class) {
394 return sizeof (MonoString) + 2 * mono_string_length ((MonoString*) o) + 2;
395 } else if (klass->rank) {
396 MonoArray *array = (MonoArray*)o;
397 size_t size = sizeof (MonoArray) + mono_array_element_size (klass) * mono_array_length (array);
401 size += sizeof (MonoArrayBounds) * klass->rank;
405 /* from a created object: the class must be inited already */
406 return klass->instance_size;
411 * ######################################################################
412 * ######## Global data.
413 * ######################################################################
415 static LOCK_DECLARE (gc_mutex);
416 static int gc_disabled = 0;
417 static int num_minor_gcs = 0;
418 static int num_major_gcs = 0;
420 /* good sizes are 512KB-1MB: larger ones increase a lot memzeroing time */
421 //#define DEFAULT_NURSERY_SIZE (1024*512*125+4096*118)
422 #define DEFAULT_NURSERY_SIZE (1024*512*2)
423 #define DEFAULT_MAX_SECTION (DEFAULT_NURSERY_SIZE * 16)
424 #define DEFAULT_LOS_COLLECTION_TARGET (DEFAULT_NURSERY_SIZE * 2)
425 /* to quickly find the heard of an object pinned by a conservative address
426 * we keep track of the objects allocated for each SCAN_START_SIZE memory
427 * chunk in the nursery or other memory sections. Larger values have less
428 * memory overhead and bigger runtime cost. 4-8 KB are reasonable values.
430 #define SCAN_START_SIZE (4096*2)
431 /* the minimum size of a fragment that we consider useful for allocation */
432 #define FRAGMENT_MIN_SIZE (512)
433 /* This is a fixed value used for pinned chunks, not the system pagesize */
434 #define FREELIST_PAGESIZE 4096
436 static mword pagesize = 4096;
437 static mword nursery_size = DEFAULT_NURSERY_SIZE;
438 static mword next_section_size = DEFAULT_NURSERY_SIZE * 4;
439 static mword max_section_size = DEFAULT_MAX_SECTION;
440 static int section_size_used = 0;
441 static int degraded_mode = 0;
443 static LOSObject *los_object_list = NULL;
444 static mword los_memory_usage = 0;
445 static mword los_num_objects = 0;
446 static mword next_los_collection = 2*1024*1024; /* 2 MB, need to tune */
447 static mword total_alloc = 0;
448 /* use this to tune when to do a major/minor collection */
449 static mword memory_pressure = 0;
451 static GCMemSection *section_list = NULL;
452 static GCMemSection *nursery_section = NULL;
453 static mword lowest_heap_address = ~(mword)0;
454 static mword highest_heap_address = 0;
456 typedef struct _FinalizeEntry FinalizeEntry;
457 struct _FinalizeEntry {
460 void *data; /* can be a disappearing link or the data for the finalizer */
461 /* Note we could use just one pointer if we don't support multiple callbacks
462 * for finalizers and per-finalizer data and if we store the obj pointers
463 * in the link like libgc does
468 * The finalizable hash has the object as the key, the
469 * disappearing_link hash, has the link address as key.
471 static FinalizeEntry **finalizable_hash = NULL;
472 /* objects that are ready to be finalized */
473 static FinalizeEntry *fin_ready_list = NULL;
474 /* disappearing links use the same structure but a different list */
475 static FinalizeEntry **disappearing_link_hash = NULL;
476 static mword disappearing_link_hash_size = 0;
477 static mword finalizable_hash_size = 0;
479 static mword num_registered_finalizers = 0;
480 static mword num_ready_finalizers = 0;
481 static mword num_disappearing_links = 0;
482 static int no_finalize = 0;
484 /* keep each size a multiple of ALLOC_ALIGN */
485 /* on 64 bit systems 8 is likely completely unused. */
486 static const int freelist_sizes [] = {
487 8, 16, 24, 32, 40, 48, 64, 80,
488 96, 128, 160, 192, 224, 256, 320, 384,
489 448, 512, 584, 680, 816, 1024, 1360, 2048};
490 #define FREELIST_NUM_SLOTS (sizeof (freelist_sizes) / sizeof (freelist_sizes [0]))
492 static char* max_pinned_chunk_addr = NULL;
493 static char* min_pinned_chunk_addr = (char*)-1;
494 /* pinned_chunk_list is used for allocations of objects that are never moved */
495 static PinnedChunk *pinned_chunk_list = NULL;
496 /* internal_chunk_list is used for allocating structures needed by the GC */
497 static PinnedChunk *internal_chunk_list = NULL;
500 obj_is_from_pinned_alloc (char *p)
502 PinnedChunk *chunk = pinned_chunk_list;
503 for (; chunk; chunk = chunk->next) {
504 if (p >= (char*)chunk->start_data && p < ((char*)chunk + chunk->num_pages * FREELIST_PAGESIZE))
510 /* registered roots: the key to the hash is the root start address */
511 static RootRecord **roots_hash = NULL;
512 static int roots_hash_size = 0;
513 static mword roots_size = 0; /* amount of memory in the root set */
514 static mword num_roots_entries = 0;
517 * The current allocation cursors
518 * We allocate objects in the nursery.
519 * The nursery is the area between nursery_start and nursery_real_end.
520 * nursery_next is the pointer to the space where the next object will be allocated.
521 * nursery_temp_end is the pointer to the end of the temporary space reserved for
522 * the allocation: this allows us to allow allocations inside the fragments of the
523 * nursery (the empty holes between pinned objects) and it allows us to set the
524 * scan starts at reasonable intervals.
525 * nursery_next and nursery_temp_end will become per-thread vars to allow lock-free
527 * nursery_first_pinned_start points to the start of the first pinned object in the nursery
528 * nursery_last_pinned_end points to the end of the last pinned object in the nursery
529 * At the next allocation, the area of the nursery where objects can be present is
530 * between MIN(nursery_first_pinned_start, first_fragment_start) and
531 * MAX(nursery_last_pinned_end, nursery_temp_end)
533 static char *nursery_start = NULL;
534 static char *nursery_next = NULL;
535 static char *nursery_temp_end = NULL;
536 static char *nursery_real_end = NULL;
537 static char *nursery_frag_real_end = NULL;
538 static char *nursery_first_pinned_start = NULL;
539 static char *nursery_last_pinned_end = NULL;
541 /* fragments that are free and ready to be used for allocation */
542 static Fragment *nursery_fragments = NULL;
543 /* freeelist of fragment structures */
544 static Fragment *fragment_freelist = NULL;
547 * used when moving the objects
548 * When the nursery is collected, objects are copied to to_space.
549 * The area between to_space and gray_objects is used as a stack
550 * of objects that need their fields checked for more references
552 * We should optimize somehow this mechanism to avoid rescanning
553 * ptr-free objects. The order is also probably not optimal: need to
554 * test cache misses and other graph traversal orders.
556 static char *to_space = NULL;
557 static char *gray_objects = NULL;
558 static char *to_space_end = NULL;
559 static GCMemSection *to_space_section = NULL;
561 /* objects bigger then this go into the large object space */
562 #define MAX_SMALL_OBJ_SIZE 0xffff
565 * ######################################################################
566 * ######## Macros and function declarations.
567 * ######################################################################
570 #define UPDATE_HEAP_BOUNDARIES(low,high) do { \
571 if ((mword)(low) < lowest_heap_address) \
572 lowest_heap_address = (mword)(low); \
573 if ((mword)(high) > highest_heap_address) \
574 highest_heap_address = (mword)(high); \
578 align_pointer (void *ptr)
580 mword p = (mword)ptr;
581 p += sizeof (gpointer) - 1;
582 p &= ~ (sizeof (gpointer) - 1);
586 /* forward declarations */
587 static void* get_internal_mem (size_t size);
588 static void free_internal_mem (void *addr);
589 static void* get_os_memory (size_t size, int activate);
590 static void free_os_memory (void *addr, size_t size);
591 static void report_internal_mem_usage (void);
593 static int stop_world (void);
594 static int restart_world (void);
595 static void pin_thread_data (void *start_nursery, void *end_nursery);
596 static void scan_from_remsets (void *start_nursery, void *end_nursery);
597 static void find_pinning_ref_from_thread (char *obj, size_t size);
598 static void update_current_thread_stack (void *start);
599 static GCMemSection* alloc_section (size_t size);
600 static void finalize_in_range (void **start, void **end);
601 static void null_link_in_range (void **start, void **end);
602 static gboolean search_fragment_for_size (size_t size);
603 static void mark_pinned_from_addresses (PinnedChunk *chunk, void **start, void **end);
604 static void clear_remsets (void);
605 static void sweep_pinned_objects (void);
606 static void free_large_object (LOSObject *obj);
607 static void free_mem_section (GCMemSection *section);
610 * ######################################################################
611 * ######## GC descriptors
612 * ######################################################################
613 * Used to quickly get the info the GC needs about an object: size and
614 * where the references are held.
616 /* objects are aligned to 8 bytes boundaries
617 * A descriptor is a pointer in MonoVTable, so 32 or 64 bits of size.
618 * The low 3 bits define the type of the descriptor. The other bits
619 * depend on the type.
620 * As a general rule the 13 remaining low bits define the size, either
621 * of the whole object or of the elements in the arrays. While for objects
622 * the size is already in bytes, for arrays we need to shift, because
623 * array elements might be smaller than 8 bytes. In case of arrays, we
624 * use two bits to describe what the additional high bits represents,
625 * so the default behaviour can handle element sizes less than 2048 bytes.
626 * The high 16 bits, if 0 it means the object is pointer-free.
627 * This design should make it easy and fast to skip over ptr-free data.
628 * The first 4 types should cover >95% of the objects.
629 * Note that since the size of objects is limited to 64K, larger objects
630 * will be allocated in the large object heap.
631 * If we want 4-bytes alignment, we need to put vector and small bitmap
635 DESC_TYPE_RUN_LENGTH, /* 16 bits aligned byte size | 1-3 (offset, numptr) bytes tuples */
636 DESC_TYPE_SMALL_BITMAP, /* 16 bits aligned byte size | 16-48 bit bitmap */
637 DESC_TYPE_STRING, /* nothing */
638 DESC_TYPE_COMPLEX, /* index for bitmap into complex_descriptors */
639 DESC_TYPE_VECTOR, /* 10 bits element size | 1 bit array | 2 bits desc | element desc */
640 DESC_TYPE_ARRAY, /* 10 bits element size | 1 bit array | 2 bits desc | element desc */
641 DESC_TYPE_LARGE_BITMAP, /* | 29-61 bitmap bits */
642 DESC_TYPE_COMPLEX_ARR, /* index for bitmap into complex_descriptors */
643 /* subtypes for arrays and vectors */
644 DESC_TYPE_V_PTRFREE = 0,/* there are no refs: keep first so it has a zero value */
645 DESC_TYPE_V_REFS, /* all the array elements are refs */
646 DESC_TYPE_V_RUN_LEN, /* elements are run-length encoded as DESC_TYPE_RUN_LENGTH */
647 DESC_TYPE_V_BITMAP /* elements are as the bitmap in DESC_TYPE_SMALL_BITMAP */
650 #define OBJECT_HEADER_WORDS (sizeof(MonoObject)/sizeof(gpointer))
651 #define LOW_TYPE_BITS 3
652 #define SMALL_BITMAP_SHIFT 16
653 #define SMALL_BITMAP_SIZE (GC_BITS_PER_WORD - SMALL_BITMAP_SHIFT)
654 #define VECTOR_INFO_SHIFT 14
655 #define VECTOR_ELSIZE_SHIFT 3
656 #define LARGE_BITMAP_SIZE (GC_BITS_PER_WORD - LOW_TYPE_BITS)
657 #define MAX_SMALL_SIZE ((1 << SMALL_BITMAP_SHIFT) - 1)
658 #define SMALL_SIZE_MASK 0xfff8
659 #define MAX_ELEMENT_SIZE 0x3ff
660 #define ELEMENT_SIZE_MASK (0x3ff << LOW_TYPE_BITS)
661 #define VECTOR_SUBTYPE_PTRFREE (DESC_TYPE_V_PTRFREE << VECTOR_INFO_SHIFT)
662 #define VECTOR_SUBTYPE_REFS (DESC_TYPE_V_REFS << VECTOR_INFO_SHIFT)
663 #define VECTOR_SUBTYPE_RUN_LEN (DESC_TYPE_V_RUN_LEN << VECTOR_INFO_SHIFT)
664 #define VECTOR_SUBTYPE_BITMAP (DESC_TYPE_V_BITMAP << VECTOR_INFO_SHIFT)
666 #define ALLOC_ALIGN 8
669 /* Root bitmap descriptors are simpler: the lower two bits describe the type
670 * and we either have 30/62 bitmap bits or nibble-based run-length,
671 * or a complex descriptor
674 ROOT_DESC_CONSERVATIVE, /* 0, so matches NULL value */
677 ROOT_DESC_LARGE_BITMAP,
678 ROOT_DESC_TYPE_MASK = 0x3,
679 ROOT_DESC_TYPE_SHIFT = 2,
682 static gsize* complex_descriptors = NULL;
683 static int complex_descriptors_size = 0;
684 static int complex_descriptors_next = 0;
687 alloc_complex_descriptor (gsize *bitmap, int numbits)
689 int nwords = numbits/GC_BITS_PER_WORD + 2;
694 res = complex_descriptors_next;
695 /* linear search, so we don't have duplicates with domain load/unload
696 * this should not be performance critical or we'd have bigger issues
697 * (the number and size of complex descriptors should be small).
699 for (i = 0; i < complex_descriptors_next; ) {
700 if (complex_descriptors [i] == nwords) {
702 for (j = 0; j < nwords - 1; ++j) {
703 if (complex_descriptors [i + 1 + j] != bitmap [j]) {
713 i += complex_descriptors [i];
715 if (complex_descriptors_next + nwords > complex_descriptors_size) {
716 int new_size = complex_descriptors_size * 2 + nwords;
717 complex_descriptors = g_realloc (complex_descriptors, new_size * sizeof (gsize));
718 complex_descriptors_size = new_size;
720 DEBUG (6, fprintf (gc_debug_file, "Complex descriptor %d, size: %d (total desc memory: %d)\n", res, nwords, complex_descriptors_size));
721 complex_descriptors_next += nwords;
722 complex_descriptors [res] = nwords;
723 for (i = 0; i < nwords - 1; ++i) {
724 complex_descriptors [res + 1 + i] = bitmap [i];
725 DEBUG (6, fprintf (gc_debug_file, "\tvalue: %p\n", (void*)complex_descriptors [res + 1 + i]));
732 * Descriptor builders.
735 mono_gc_make_descr_for_string (void)
737 return (void*) DESC_TYPE_STRING;
741 mono_gc_make_descr_for_object (gsize *bitmap, int numbits, size_t obj_size)
743 int first_set = -1, num_set = 0, last_set = -1, i;
745 size_t stored_size = obj_size;
746 stored_size += ALLOC_ALIGN - 1;
747 stored_size &= ~(ALLOC_ALIGN - 1);
748 for (i = 0; i < numbits; ++i) {
749 if (bitmap [i / GC_BITS_PER_WORD] & (1 << (i % GC_BITS_PER_WORD))) {
756 if (stored_size <= MAX_SMALL_OBJ_SIZE) {
757 /* check run-length encoding first: one byte offset, one byte number of pointers
758 * on 64 bit archs, we can have 3 runs, just one on 32.
759 * It may be better to use nibbles.
762 desc = DESC_TYPE_RUN_LENGTH | stored_size;
763 DEBUG (6, fprintf (gc_debug_file, "Ptrfree descriptor %p, size: %d\n", (void*)desc, stored_size));
765 } else if (first_set < 256 && num_set < 256 && (first_set + num_set == last_set + 1)) {
766 desc = DESC_TYPE_RUN_LENGTH | stored_size | (first_set << 16) | (num_set << 24);
767 DEBUG (6, fprintf (gc_debug_file, "Runlen descriptor %p, size: %d, first set: %d, num set: %d\n", (void*)desc, stored_size, first_set, num_set));
770 /* we know the 2-word header is ptr-free */
771 if (last_set < SMALL_BITMAP_SIZE + OBJECT_HEADER_WORDS) {
772 desc = DESC_TYPE_SMALL_BITMAP | stored_size | ((*bitmap >> OBJECT_HEADER_WORDS) << SMALL_BITMAP_SHIFT);
773 DEBUG (6, fprintf (gc_debug_file, "Smallbitmap descriptor %p, size: %d, last set: %d\n", (void*)desc, stored_size, last_set));
777 /* we know the 2-word header is ptr-free */
778 if (last_set < LARGE_BITMAP_SIZE + OBJECT_HEADER_WORDS) {
779 desc = DESC_TYPE_LARGE_BITMAP | ((*bitmap >> OBJECT_HEADER_WORDS) << LOW_TYPE_BITS);
780 DEBUG (6, fprintf (gc_debug_file, "Largebitmap descriptor %p, size: %d, last set: %d\n", (void*)desc, stored_size, last_set));
783 /* it's a complex object ... */
784 desc = DESC_TYPE_COMPLEX | (alloc_complex_descriptor (bitmap, last_set + 1) << LOW_TYPE_BITS);
788 /* If the array holds references, numbits == 1 and the first bit is set in elem_bitmap */
790 mono_gc_make_descr_for_array (int vector, gsize *elem_bitmap, int numbits, size_t elem_size)
792 int first_set = -1, num_set = 0, last_set = -1, i;
793 mword desc = vector? DESC_TYPE_VECTOR: DESC_TYPE_ARRAY;
794 for (i = 0; i < numbits; ++i) {
795 if (elem_bitmap [i / GC_BITS_PER_WORD] & (1 << (i % GC_BITS_PER_WORD))) {
802 if (elem_size <= MAX_ELEMENT_SIZE) {
803 desc |= elem_size << VECTOR_ELSIZE_SHIFT;
805 return (void*)(desc | VECTOR_SUBTYPE_PTRFREE);
807 /* Note: we also handle structs with just ref fields */
808 if (num_set * sizeof (gpointer) == elem_size) {
809 return (void*)(desc | VECTOR_SUBTYPE_REFS | ((-1LL) << 16));
811 /* FIXME: try run-len first */
812 /* Note: we can't skip the object header here, because it's not present */
813 if (last_set <= SMALL_BITMAP_SIZE) {
814 return (void*)(desc | VECTOR_SUBTYPE_BITMAP | (*elem_bitmap << 16));
817 /* it's am array of complex structs ... */
818 desc = DESC_TYPE_COMPLEX_ARR;
819 desc |= alloc_complex_descriptor (elem_bitmap, last_set + 1) << LOW_TYPE_BITS;
823 /* helper macros to scan and traverse objects, macros because we resue them in many functions */
824 #define STRING_SIZE(size,str) do { \
825 (size) = sizeof (MonoString) + 2 * (mono_string_length ((MonoString*)(str)) + 1); \
826 (size) += (ALLOC_ALIGN - 1); \
827 (size) &= ~(ALLOC_ALIGN - 1); \
830 #define OBJ_RUN_LEN_SIZE(size,vt,obj) do { \
831 (size) = (vt)->desc & 0xfff8; \
834 #define OBJ_BITMAP_SIZE(size,vt,obj) do { \
835 (size) = (vt)->desc & 0xfff8; \
838 //#define PREFETCH(addr) __asm__ __volatile__ (" prefetchnta %0": : "m"(*(char *)(addr)))
839 #define PREFETCH(addr)
841 /* code using these macros must define a HANDLE_PTR(ptr) macro that does the work */
842 #define OBJ_RUN_LEN_FOREACH_PTR(vt,obj) do { \
843 if ((vt)->desc & 0xffff0000) { \
844 /* there are pointers */ \
845 void **_objptr_end; \
846 void **_objptr = (void**)(obj); \
847 _objptr += ((vt)->desc >> 16) & 0xff; \
848 _objptr_end = _objptr + (((vt)->desc >> 24) & 0xff); \
849 while (_objptr < _objptr_end) { \
850 HANDLE_PTR (_objptr, (obj)); \
856 /* a bitmap desc means that there are pointer references or we'd have
857 * choosen run-length, instead: add an assert to check.
859 #define OBJ_BITMAP_FOREACH_PTR(vt,obj) do { \
860 /* there are pointers */ \
861 void **_objptr = (void**)(obj); \
862 gsize _bmap = (vt)->desc >> 16; \
863 _objptr += OBJECT_HEADER_WORDS; \
866 HANDLE_PTR (_objptr, (obj)); \
873 #define OBJ_LARGE_BITMAP_FOREACH_PTR(vt,obj) do { \
874 /* there are pointers */ \
875 void **_objptr = (void**)(obj); \
876 gsize _bmap = (vt)->desc >> LOW_TYPE_BITS; \
877 _objptr += OBJECT_HEADER_WORDS; \
880 HANDLE_PTR (_objptr, (obj)); \
887 #define OBJ_COMPLEX_FOREACH_PTR(vt,obj) do { \
888 /* there are pointers */ \
889 void **_objptr = (void**)(obj); \
890 gsize *bitmap_data = complex_descriptors + ((vt)->desc >> LOW_TYPE_BITS); \
891 int bwords = (*bitmap_data) - 1; \
892 void **start_run = _objptr; \
895 MonoObject *myobj = (MonoObject*)obj; \
896 g_print ("found %d at %p (0x%x): %s.%s\n", bwords, (obj), (vt)->desc, myobj->vtable->klass->name_space, myobj->vtable->klass->name); \
898 while (bwords-- > 0) { \
899 gsize _bmap = *bitmap_data++; \
900 _objptr = start_run; \
901 /*g_print ("bitmap: 0x%x/%d at %p\n", _bmap, bwords, _objptr);*/ \
904 HANDLE_PTR (_objptr, (obj)); \
909 start_run += GC_BITS_PER_WORD; \
913 /* this one is untested */
914 #define OBJ_COMPLEX_ARR_FOREACH_PTR(vt,obj) do { \
915 /* there are pointers */ \
916 gsize *mbitmap_data = complex_descriptors + ((vt)->desc >> LOW_TYPE_BITS); \
917 int mbwords = (*mbitmap_data++) - 1; \
918 int el_size = mono_array_element_size (((MonoObject*)(obj))->vtable->klass); \
919 char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
920 char *e_end = e_start + el_size * mono_array_length ((MonoArray*)(obj)); \
922 MonoObject *myobj = (MonoObject*)start; \
923 g_print ("found %d at %p (0x%x): %s.%s\n", mbwords, (obj), (vt)->desc, myobj->vtable->klass->name_space, myobj->vtable->klass->name); \
925 while (e_start < e_end) { \
926 void **_objptr = (void**)e_start; \
927 gsize *bitmap_data = mbitmap_data; \
928 unsigned int bwords = mbwords; \
929 while (bwords-- > 0) { \
930 gsize _bmap = *bitmap_data++; \
931 void **start_run = _objptr; \
932 /*g_print ("bitmap: 0x%x\n", _bmap);*/ \
935 HANDLE_PTR (_objptr, (obj)); \
940 _objptr = start_run + GC_BITS_PER_WORD; \
942 e_start += el_size; \
946 #define OBJ_VECTOR_FOREACH_PTR(vt,obj) do { \
947 /* note: 0xffffc000 excludes DESC_TYPE_V_PTRFREE */ \
948 if ((vt)->desc & 0xffffc000) { \
949 int el_size = ((vt)->desc >> 3) & MAX_ELEMENT_SIZE; \
950 /* there are pointers */ \
951 int etype = (vt)->desc & 0xc000; \
952 if (etype == (DESC_TYPE_V_REFS << 14)) { \
953 void **p = (void**)((char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector)); \
954 void **end_refs = (void**)((char*)p + el_size * mono_array_length ((MonoArray*)(obj))); \
955 /* Note: this code can handle also arrays of struct with only references in them */ \
956 while (p < end_refs) { \
957 HANDLE_PTR (p, (obj)); \
960 } else if (etype == DESC_TYPE_V_RUN_LEN << 14) { \
961 int offset = ((vt)->desc >> 16) & 0xff; \
962 int num_refs = ((vt)->desc >> 24) & 0xff; \
963 char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
964 char *e_end = e_start + el_size * mono_array_length ((MonoArray*)(obj)); \
965 while (e_start < e_end) { \
966 void **p = (void**)e_start; \
969 for (i = 0; i < num_refs; ++i) { \
970 HANDLE_PTR (p + i, (obj)); \
972 e_start += el_size; \
974 } else if (etype == DESC_TYPE_V_BITMAP << 14) { \
975 char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
976 char *e_end = e_start + el_size * mono_array_length ((MonoArray*)(obj)); \
977 while (e_start < e_end) { \
978 void **p = (void**)e_start; \
979 gsize _bmap = (vt)->desc >> 16; \
980 /* Note: there is no object header here to skip */ \
983 HANDLE_PTR (p, (obj)); \
988 e_start += el_size; \
994 static mword new_obj_references = 0;
995 static mword obj_references_checked = 0;
998 #define HANDLE_PTR(ptr,obj) do { \
999 if (*(ptr) && (char*)*(ptr) >= nursery_start && (char*)*(ptr) < nursery_next) { \
1000 new_obj_references++; \
1001 /*printf ("bogus ptr %p found at %p in object %p (%s.%s)\n", *(ptr), (ptr), o, o->vtable->klass->name_space, o->vtable->klass->name);*/ \
1003 obj_references_checked++; \
1008 * ######################################################################
1009 * ######## Detecting and removing garbage.
1010 * ######################################################################
1011 * This section of code deals with detecting the objects no longer in use
1012 * and reclaiming the memory.
1014 static void __attribute__((noinline))
1015 scan_area (char *start, char *end)
1020 int type_str = 0, type_rlen = 0, type_bitmap = 0, type_vector = 0, type_lbit = 0, type_complex = 0;
1021 new_obj_references = 0;
1022 obj_references_checked = 0;
1023 while (start < end) {
1024 if (!*(void**)start) {
1025 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
1028 vt = (GCVTable*)LOAD_VTABLE (start);
1029 DEBUG (8, fprintf (gc_debug_file, "Scanning object %p, vtable: %p (%s)\n", start, vt, vt->klass->name));
1031 MonoObject *obj = (MonoObject*)start;
1032 g_print ("found at %p (0x%x): %s.%s\n", start, vt->desc, obj->vtable->klass->name_space, obj->vtable->klass->name);
1034 type = vt->desc & 0x7;
1035 if (type == DESC_TYPE_STRING) {
1036 STRING_SIZE (skip_size, start);
1040 } else if (type == DESC_TYPE_RUN_LENGTH) {
1041 OBJ_RUN_LEN_SIZE (skip_size, vt, start);
1042 g_assert (skip_size);
1043 OBJ_RUN_LEN_FOREACH_PTR (vt,start);
1047 } else if (type == DESC_TYPE_VECTOR) { // includes ARRAY, too
1048 skip_size = (vt->desc >> LOW_TYPE_BITS) & MAX_ELEMENT_SIZE;
1049 skip_size *= mono_array_length ((MonoArray*)start);
1050 skip_size += sizeof (MonoArray);
1051 skip_size += (ALLOC_ALIGN - 1);
1052 skip_size &= ~(ALLOC_ALIGN - 1);
1053 OBJ_VECTOR_FOREACH_PTR (vt, start);
1054 if (type == DESC_TYPE_ARRAY) {
1055 /* account for the bounds */
1060 } else if (type == DESC_TYPE_SMALL_BITMAP) {
1061 OBJ_BITMAP_SIZE (skip_size, vt, start);
1062 g_assert (skip_size);
1063 OBJ_BITMAP_FOREACH_PTR (vt,start);
1067 } else if (type == DESC_TYPE_LARGE_BITMAP) {
1068 skip_size = safe_object_get_size ((MonoObject*)start);
1069 skip_size += (ALLOC_ALIGN - 1);
1070 skip_size &= ~(ALLOC_ALIGN - 1);
1071 OBJ_LARGE_BITMAP_FOREACH_PTR (vt,start);
1075 } else if (type == DESC_TYPE_COMPLEX) {
1076 /* this is a complex object */
1077 skip_size = safe_object_get_size ((MonoObject*)start);
1078 skip_size += (ALLOC_ALIGN - 1);
1079 skip_size &= ~(ALLOC_ALIGN - 1);
1080 OBJ_COMPLEX_FOREACH_PTR (vt, start);
1084 } else if (type == DESC_TYPE_COMPLEX_ARR) {
1085 /* this is an array of complex structs */
1086 skip_size = mono_array_element_size (((MonoVTable*)vt)->klass);
1087 skip_size *= mono_array_length ((MonoArray*)start);
1088 skip_size += sizeof (MonoArray);
1089 skip_size += (ALLOC_ALIGN - 1);
1090 skip_size &= ~(ALLOC_ALIGN - 1);
1091 OBJ_COMPLEX_ARR_FOREACH_PTR (vt, start);
1092 if (type == DESC_TYPE_ARRAY) {
1093 /* account for the bounds */
1102 /*printf ("references to new nursery %p-%p (size: %dk): %d, checked: %d\n", old_start, end, (end-old_start)/1024, new_obj_references, obj_references_checked);
1103 printf ("\tstrings: %d, runl: %d, vector: %d, bitmaps: %d, lbitmaps: %d, complex: %d\n",
1104 type_str, type_rlen, type_vector, type_bitmap, type_lbit, type_complex);*/
1107 static void __attribute__((noinline))
1108 scan_area_for_domain (MonoDomain *domain, char *start, char *end)
1113 while (start < end) {
1114 if (!*(void**)start) {
1115 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
1118 vt = (GCVTable*)LOAD_VTABLE (start);
1119 /* handle threads someway (maybe insert the root domain vtable?) */
1120 if (mono_object_domain (start) == domain && vt->klass != mono_defaults.thread_class) {
1121 DEBUG (1, fprintf (gc_debug_file, "Need to cleanup object %p, (%s)\n", start, safe_name (start)));
1126 type = vt->desc & 0x7;
1127 if (type == DESC_TYPE_STRING) {
1128 STRING_SIZE (skip_size, start);
1129 if (remove) memset (start, 0, skip_size);
1132 } else if (type == DESC_TYPE_RUN_LENGTH) {
1133 OBJ_RUN_LEN_SIZE (skip_size, vt, start);
1134 g_assert (skip_size);
1135 if (remove) memset (start, 0, skip_size);
1138 } else if (type == DESC_TYPE_VECTOR) { // includes ARRAY, too
1139 skip_size = (vt->desc >> LOW_TYPE_BITS) & MAX_ELEMENT_SIZE;
1140 skip_size *= mono_array_length ((MonoArray*)start);
1141 skip_size += sizeof (MonoArray);
1142 skip_size += (ALLOC_ALIGN - 1);
1143 skip_size &= ~(ALLOC_ALIGN - 1);
1144 if (type == DESC_TYPE_ARRAY) {
1145 /* account for the bounds */
1147 if (remove) memset (start, 0, skip_size);
1150 } else if (type == DESC_TYPE_SMALL_BITMAP) {
1151 OBJ_BITMAP_SIZE (skip_size, vt, start);
1152 g_assert (skip_size);
1153 if (remove) memset (start, 0, skip_size);
1156 } else if (type == DESC_TYPE_LARGE_BITMAP) {
1157 skip_size = safe_object_get_size ((MonoObject*)start);
1158 skip_size += (ALLOC_ALIGN - 1);
1159 skip_size &= ~(ALLOC_ALIGN - 1);
1160 if (remove) memset (start, 0, skip_size);
1163 } else if (type == DESC_TYPE_COMPLEX) {
1164 /* this is a complex object */
1165 skip_size = safe_object_get_size ((MonoObject*)start);
1166 skip_size += (ALLOC_ALIGN - 1);
1167 skip_size &= ~(ALLOC_ALIGN - 1);
1168 if (remove) memset (start, 0, skip_size);
1171 } else if (type == DESC_TYPE_COMPLEX_ARR) {
1172 /* this is an array of complex structs */
1173 skip_size = mono_array_element_size (((MonoVTable*)vt)->klass);
1174 skip_size *= mono_array_length ((MonoArray*)start);
1175 skip_size += sizeof (MonoArray);
1176 skip_size += (ALLOC_ALIGN - 1);
1177 skip_size &= ~(ALLOC_ALIGN - 1);
1178 if (type == DESC_TYPE_ARRAY) {
1179 /* account for the bounds */
1181 if (remove) memset (start, 0, skip_size);
1191 * When appdomains are unloaded we can easily remove objects that have finalizers,
1192 * but all the others could still be present in random places on the heap.
1193 * We need a sweep to get rid of them even though it's going to be costly
1195 * The reason we need to remove them is because we access the vtable and class
1196 * structures to know the object size and the reference bitmap: once the domain is
1197 * unloaded the point to random memory.
1200 mono_gc_clear_domain (MonoDomain * domain)
1202 GCMemSection *section;
1204 for (section = section_list; section; section = section->next) {
1205 scan_area_for_domain (domain, section->data, section->end_data);
1207 /* FIXME: handle big and fixed objects (we remove, don't clear in this case) */
1212 add_to_global_remset (gpointer ptr)
1215 DEBUG (8, fprintf (gc_debug_file, "Adding global remset for %p\n", ptr));
1216 if (global_remset->store_next < global_remset->end_set) {
1217 *(global_remset->store_next++) = (mword)ptr;
1220 rs = alloc_remset (global_remset->end_set - global_remset->data, NULL);
1221 rs->next = global_remset;
1223 *(global_remset->store_next++) = (mword)ptr;
1227 * This is how the copying happens from the nursery to the old generation.
1228 * We assume that at this time all the pinned objects have been identified and
1230 * We run scan_object() for each pinned object so that each referenced
1231 * objects if possible are copied. The new gray objects created can have
1232 * scan_object() run on them right away, too.
1233 * Then we run copy_object() for the precisely tracked roots. At this point
1234 * all the roots are either gray or black. We run scan_object() on the gray
1235 * objects until no more gray objects are created.
1236 * At the end of the process we walk again the pinned list and we unmark
1237 * the pinned flag. As we go we also create the list of free space for use
1238 * in the next allocation runs.
1240 * We need to remember objects from the old generation that point to the new one
1241 * (or just addresses?).
1243 * copy_object could be made into a macro once debugged (use inline for now).
1246 static char* __attribute__((noinline))
1247 copy_object (char *obj, char *from_space_start, char *from_space_end)
1249 if (obj >= from_space_start && obj < from_space_end) {
1253 DEBUG (9, fprintf (gc_debug_file, "Precise copy of %p", obj));
1254 if ((forwarded = object_is_forwarded (obj))) {
1255 g_assert (((MonoVTable*)LOAD_VTABLE(obj))->gc_descr);
1256 DEBUG (9, fprintf (gc_debug_file, " (already forwarded to %p)\n", forwarded));
1259 if (object_is_pinned (obj)) {
1260 g_assert (((MonoVTable*)LOAD_VTABLE(obj))->gc_descr);
1261 DEBUG (9, fprintf (gc_debug_file, " (pinned, no change)\n"));
1264 objsize = safe_object_get_size ((MonoObject*)obj);
1265 objsize += ALLOC_ALIGN - 1;
1266 objsize &= ~(ALLOC_ALIGN - 1);
1267 DEBUG (9, fprintf (gc_debug_file, " (to %p, %s size: %d)\n", gray_objects, ((MonoObject*)obj)->vtable->klass->name, objsize));
1268 /* FIXME: handle pinned allocs:
1269 * Large objects are simple, at least until we always follow the rule:
1270 * if objsize >= MAX_SMALL_OBJ_SIZE, pin the object and return it.
1271 * At the end of major collections, we walk the los list and if
1272 * the object is pinned, it is marked, otherwise it can be freed.
1274 if (objsize >= MAX_SMALL_OBJ_SIZE || (obj >= min_pinned_chunk_addr && obj < max_pinned_chunk_addr && obj_is_from_pinned_alloc (obj))) {
1275 DEBUG (9, fprintf (gc_debug_file, "Marked LOS/Pinned %p (%s), size: %d\n", obj, safe_name (obj), objsize));
1279 /* ok, the object is not pinned, we can move it */
1280 /* use a optimized memcpy here */
1285 char* edi = gray_objects;
1286 __asm__ __volatile__(
1288 : "=&c" (ecx), "=&D" (edi), "=&S" (esi)
1289 : "0" (objsize/4), "1" (edi),"2" (esi)
1294 memcpy (gray_objects, obj, objsize);
1296 /* adjust array->bounds */
1297 vt = ((MonoObject*)obj)->vtable;
1298 g_assert (vt->gc_descr);
1299 if (vt->rank && ((MonoArray*)obj)->bounds) {
1300 MonoArray *array = (MonoArray*)gray_objects;
1301 array->bounds = (MonoArrayBounds*)((char*)gray_objects + ((char*)((MonoArray*)obj)->bounds - (char*)obj));
1302 DEBUG (9, fprintf (gc_debug_file, "Array instance %p: size: %d, rank: %d, length: %d\n", array, objsize, vt->rank, mono_array_length (array)));
1304 /* set the forwarding pointer */
1305 forward_object (obj, gray_objects);
1307 to_space_section->scan_starts [((char*)obj - (char*)to_space_section->data)/SCAN_START_SIZE] = obj;
1308 gray_objects += objsize;
1309 DEBUG (8, g_assert (gray_objects <= to_space_end));
1316 #define HANDLE_PTR(ptr,obj) do { \
1318 void *__old = *(ptr); \
1319 *(ptr) = copy_object (*(ptr), from_start, from_end); \
1320 DEBUG (9, if (__old != *(ptr)) fprintf (gc_debug_file, "Overwrote field at %p with %p (was: %p)\n", (ptr), *(ptr), __old)); \
1321 if (*(ptr) >= (void*)from_start && *(ptr) < (void*)from_end) \
1322 add_to_global_remset ((ptr)); \
1327 * Scan the object pointed to by @start for references to
1328 * other objects between @from_start and @from_end and copy
1329 * them to the gray_objects area.
1330 * Returns a pointer to the end of the object.
1333 scan_object (char *start, char* from_start, char* from_end)
1338 vt = (GCVTable*)LOAD_VTABLE (start);
1339 //type = vt->desc & 0x7;
1341 /* gcc should be smart enough to remove the bounds check, but it isn't:( */
1342 switch (vt->desc & 0x7) {
1343 //if (type == DESC_TYPE_STRING) {
1344 case DESC_TYPE_STRING:
1345 STRING_SIZE (skip_size, start);
1346 return start + skip_size;
1347 //} else if (type == DESC_TYPE_RUN_LENGTH) {
1348 case DESC_TYPE_RUN_LENGTH:
1349 OBJ_RUN_LEN_FOREACH_PTR (vt,start);
1350 OBJ_RUN_LEN_SIZE (skip_size, vt, start);
1351 g_assert (skip_size);
1352 return start + skip_size;
1353 //} else if (type == DESC_TYPE_VECTOR) { // includes ARRAY, too
1354 case DESC_TYPE_ARRAY:
1355 case DESC_TYPE_VECTOR:
1356 OBJ_VECTOR_FOREACH_PTR (vt, start);
1357 skip_size = safe_object_get_size ((MonoObject*)start);
1359 skip_size = (vt->desc >> LOW_TYPE_BITS) & MAX_ELEMENT_SIZE;
1360 skip_size *= mono_array_length ((MonoArray*)start);
1361 skip_size += sizeof (MonoArray);
1363 skip_size += (ALLOC_ALIGN - 1);
1364 skip_size &= ~(ALLOC_ALIGN - 1);
1365 return start + skip_size;
1366 //} else if (type == DESC_TYPE_SMALL_BITMAP) {
1367 case DESC_TYPE_SMALL_BITMAP:
1368 OBJ_BITMAP_FOREACH_PTR (vt,start);
1369 OBJ_BITMAP_SIZE (skip_size, vt, start);
1370 return start + skip_size;
1371 //} else if (type == DESC_TYPE_LARGE_BITMAP) {
1372 case DESC_TYPE_LARGE_BITMAP:
1373 OBJ_LARGE_BITMAP_FOREACH_PTR (vt,start);
1374 skip_size = safe_object_get_size ((MonoObject*)start);
1375 skip_size += (ALLOC_ALIGN - 1);
1376 skip_size &= ~(ALLOC_ALIGN - 1);
1377 return start + skip_size;
1378 //} else if (type == DESC_TYPE_COMPLEX) {
1379 case DESC_TYPE_COMPLEX:
1380 OBJ_COMPLEX_FOREACH_PTR (vt, start);
1381 /* this is a complex object */
1382 skip_size = safe_object_get_size ((MonoObject*)start);
1383 skip_size += (ALLOC_ALIGN - 1);
1384 skip_size &= ~(ALLOC_ALIGN - 1);
1385 return start + skip_size;
1386 //} else if (type == DESC_TYPE_COMPLEX_ARR) {
1387 case DESC_TYPE_COMPLEX_ARR:
1388 OBJ_COMPLEX_ARR_FOREACH_PTR (vt, start);
1389 /* this is an array of complex structs */
1390 skip_size = safe_object_get_size ((MonoObject*)start);
1392 skip_size = mono_array_element_size (((MonoObject*)start)->vtable->klass);
1393 skip_size *= mono_array_length ((MonoArray*)start);
1394 skip_size += sizeof (MonoArray);
1396 skip_size += (ALLOC_ALIGN - 1);
1397 skip_size &= ~(ALLOC_ALIGN - 1);
1398 return start + skip_size;
1400 g_assert_not_reached ();
1405 * Addresses from start to end are already sorted. This function finds the object header
1406 * for each address and pins the object. The addresses must be inside the passed section.
1407 * Return the number of pinned objects.
1410 pin_objects_from_addresses (GCMemSection *section, void **start, void **end, void *start_nursery, void *end_nursery)
1415 void *last_obj = NULL;
1416 size_t last_obj_size = 0;
1419 void **definitely_pinned = start;
1420 while (start < end) {
1422 /* the range check should be reduntant */
1423 if (addr != last && addr >= start_nursery && addr < end_nursery) {
1424 DEBUG (5, fprintf (gc_debug_file, "Considering pinning addr %p\n", addr));
1425 /* multiple pointers to the same object */
1426 if (addr >= last_obj && (char*)addr < (char*)last_obj + last_obj_size) {
1430 idx = ((char*)addr - (char*)section->data) / SCAN_START_SIZE;
1431 search_start = (void*)section->scan_starts [idx];
1432 if (!search_start || search_start > addr) {
1435 search_start = section->scan_starts [idx];
1436 if (search_start && search_start <= addr)
1439 if (!search_start || search_start > addr)
1440 search_start = start_nursery;
1442 if (search_start < last_obj)
1443 search_start = (char*)last_obj + last_obj_size;
1444 /* now addr should be in an object a short distance from search_start
1445 * Note that search_start must point to zeroed mem or point to an object.
1448 if (!*(void**)search_start) {
1449 mword p = (mword)search_start;
1450 p += sizeof (gpointer);
1451 p += ALLOC_ALIGN - 1;
1452 p &= ~(ALLOC_ALIGN - 1);
1453 search_start = (void*)p;
1456 last_obj = search_start;
1457 last_obj_size = safe_object_get_size ((MonoObject*)search_start);
1458 last_obj_size += ALLOC_ALIGN - 1;
1459 last_obj_size &= ~(ALLOC_ALIGN - 1);
1460 DEBUG (8, fprintf (gc_debug_file, "Pinned try match %p (%s), size %d\n", last_obj, safe_name (last_obj), last_obj_size));
1461 if (addr >= search_start && (char*)addr < (char*)last_obj + last_obj_size) {
1462 DEBUG (4, fprintf (gc_debug_file, "Pinned object %p, vtable %p (%s), count %d\n", search_start, *(void**)search_start, safe_name (search_start), count));
1463 pin_object (search_start);
1464 definitely_pinned [count] = search_start;
1468 /* skip to the next object */
1469 search_start = (void*)((char*)search_start + last_obj_size);
1470 } while (search_start <= addr);
1471 /* we either pinned the correct object or we ignored the addr because
1472 * it points to unused zeroed memory.
1478 //printf ("effective pinned: %d (at the end: %d)\n", count, (char*)end_nursery - (char*)last);
1482 static void** pin_queue;
1483 static int pin_queue_size = 0;
1484 static int next_pin_slot = 0;
1489 gap = (gap * 10) / 13;
1490 if (gap == 9 || gap == 10)
1499 compare_addr (const void *a, const void *b)
1501 return *(const void **)a - *(const void **)b;
1505 /* sort the addresses in array in increasing order */
1507 sort_addresses (void **array, int size)
1510 * qsort is slower as predicted.
1511 * qsort (array, size, sizeof (gpointer), compare_addr);
1518 gap = new_gap (gap);
1521 for (i = 0; i < end; i++) {
1523 if (array [i] > array [j]) {
1524 void* val = array [i];
1525 array [i] = array [j];
1530 if (gap == 1 && !swapped)
1536 print_nursery_gaps (void* start_nursery, void *end_nursery)
1539 gpointer first = start_nursery;
1541 for (i = 0; i < next_pin_slot; ++i) {
1542 next = pin_queue [i];
1543 fprintf (gc_debug_file, "Nursery range: %p-%p, size: %d\n", first, next, (char*)next-(char*)first);
1547 fprintf (gc_debug_file, "Nursery range: %p-%p, size: %d\n", first, next, (char*)next-(char*)first);
1550 /* reduce the info in the pin queue, removing duplicate pointers and sorting them */
1552 optimize_pin_queue (int start_slot)
1554 void **start, **cur, **end;
1555 /* sort and uniq pin_queue: we just sort and we let the rest discard multiple values */
1556 /* it may be better to keep ranges of pinned memory instead of individually pinning objects */
1557 DEBUG (5, fprintf (gc_debug_file, "Sorting pin queue, size: %d\n", next_pin_slot));
1558 if ((next_pin_slot - start_slot) > 1)
1559 sort_addresses (pin_queue + start_slot, next_pin_slot - start_slot);
1560 start = cur = pin_queue + start_slot;
1561 end = pin_queue + next_pin_slot;
1564 while (*start == *cur && cur < end)
1568 next_pin_slot = start - pin_queue;
1569 DEBUG (5, fprintf (gc_debug_file, "Pin queue reduced to size: %d\n", next_pin_slot));
1570 //DEBUG (6, print_nursery_gaps (start_nursery, end_nursery));
1575 realloc_pin_queue (void)
1577 int new_size = pin_queue_size? pin_queue_size + pin_queue_size/2: 1024;
1578 void **new_pin = get_internal_mem (sizeof (void*) * new_size);
1579 memcpy (new_pin, pin_queue, sizeof (void*) * next_pin_slot);
1580 free_internal_mem (pin_queue);
1581 pin_queue = new_pin;
1582 pin_queue_size = new_size;
1583 DEBUG (4, fprintf (gc_debug_file, "Reallocated pin queue to size: %d\n", new_size));
1587 * Scan the memory between start and end and queue values which could be pointers
1588 * to the area between start_nursery and end_nursery for later consideration.
1589 * Typically used for thread stacks.
1592 conservatively_pin_objects_from (void **start, void **end, void *start_nursery, void *end_nursery)
1595 while (start < end) {
1596 if (*start >= start_nursery && *start < end_nursery) {
1598 * *start can point to the middle of an object
1599 * note: should we handle pointing at the end of an object?
1600 * pinning in C# code disallows pointing at the end of an object
1601 * but there is some small chance that an optimizing C compiler
1602 * may keep the only reference to an object by pointing
1603 * at the end of it. We ignore this small chance for now.
1604 * Pointers to the end of an object are indistinguishable
1605 * from pointers to the start of the next object in memory
1606 * so if we allow that we'd need to pin two objects...
1607 * We queue the pointer in an array, the
1608 * array will then be sorted and uniqued. This way
1609 * we can coalesce several pinning pointers and it should
1610 * be faster since we'd do a memory scan with increasing
1611 * addresses. Note: we can align the address to the allocation
1612 * alignment, so the unique process is more effective.
1614 mword addr = (mword)*start;
1615 addr &= ~(ALLOC_ALIGN - 1);
1616 if (next_pin_slot >= pin_queue_size)
1617 realloc_pin_queue ();
1618 pin_queue [next_pin_slot++] = (void*)addr;
1619 DEBUG (6, if (count) fprintf (gc_debug_file, "Pinning address %p\n", (void*)addr));
1624 DEBUG (7, if (count) fprintf (gc_debug_file, "found %d potential pinned heap pointers\n", count));
1628 * If generation is 0, just mark objects in the nursery, the others we don't care,
1629 * since they are not going to move anyway.
1630 * There are different areas that are scanned for pinned pointers:
1631 * *) the thread stacks (when jit support is ready only the unmanaged frames)
1632 * *) the pinned handle table
1633 * *) the pinned roots
1635 * Note: when we'll use a write barrier for old to new gen references, we need to
1636 * keep track of old gen objects that point to pinned new gen objects because in that
1637 * case the referenced object will be moved maybe at the next collection, but there
1638 * is no write in the old generation area where the pinned object is referenced
1639 * and we may not consider it as reachable.
1642 mark_pinned_objects (int generation)
1647 * Debugging function: find in the conservative roots where @obj is being pinned.
1650 find_pinning_reference (char *obj, size_t size)
1654 char *endobj = obj + size;
1655 for (i = 0; i < roots_hash_size; ++i) {
1656 for (root = roots_hash [i]; root; root = root->next) {
1657 /* if desc is non-null it has precise info */
1658 if (!root->root_desc) {
1659 char ** start = (char**)root->start_root;
1660 while (start < (char**)root->end_root) {
1661 if (*start >= obj && *start < endobj) {
1662 DEBUG (0, fprintf (gc_debug_file, "Object %p referenced in pinned roots %p-%p (at %p in record %p)\n", obj, root->start_root, root->end_root, start, root));
1669 find_pinning_ref_from_thread (obj, size);
1673 * The first thing we do in a collection is to identify pinned objects.
1674 * This function considers all the areas of memory that need to be
1675 * conservatively scanned.
1678 pin_from_roots (void *start_nursery, void *end_nursery)
1682 DEBUG (3, fprintf (gc_debug_file, "Scanning pinned roots (%d bytes, %d entries)\n", (int)roots_size, num_roots_entries));
1683 /* objects pinned from the API are inside these roots */
1684 for (i = 0; i < roots_hash_size; ++i) {
1685 for (root = roots_hash [i]; root; root = root->next) {
1686 /* if desc is non-null it has precise info */
1687 if (root->root_desc)
1689 DEBUG (6, fprintf (gc_debug_file, "Pinned roots %p-%p\n", root->start_root, root->end_root));
1690 conservatively_pin_objects_from ((void**)root->start_root, (void**)root->end_root, start_nursery, end_nursery);
1693 /* now deal with the thread stacks
1694 * in the future we should be able to conservatively scan only:
1695 * *) the cpu registers
1696 * *) the unmanaged stack frames
1697 * *) the _last_ managed stack frame
1698 * *) pointers slots in managed frames
1700 pin_thread_data (start_nursery, end_nursery);
1704 * The memory area from start_root to end_root contains pointers to objects.
1705 * Their position is precisely described by @desc (this means that the pointer
1706 * can be either NULL or the pointer to the start of an object).
1707 * This functions copies them to to_space updates them.
1710 precisely_scan_objects_from (void** start_root, void** end_root, char* n_start, char *n_end, mword desc)
1712 switch (desc & ROOT_DESC_TYPE_MASK) {
1713 case ROOT_DESC_BITMAP:
1714 desc >>= ROOT_DESC_TYPE_SHIFT;
1716 if ((desc & 1) && *start_root) {
1717 *start_root = copy_object (*start_root, n_start, n_end);
1718 DEBUG (9, fprintf (gc_debug_file, "Overwrote root at %p with %p\n", start_root, *start_root)); \
1724 case ROOT_DESC_RUN_LEN:
1725 case ROOT_DESC_LARGE_BITMAP:
1726 g_assert_not_reached ();
1731 alloc_fragment (void)
1733 Fragment *frag = fragment_freelist;
1735 fragment_freelist = frag->next;
1739 frag = get_internal_mem (sizeof (Fragment));
1745 * Allocate and setup the data structures needed to be able to allocate objects
1746 * in the nursery. The nursery is stored in nursery_section.
1749 alloc_nursery (void)
1751 GCMemSection *section;
1756 if (nursery_section)
1758 DEBUG (2, fprintf (gc_debug_file, "Allocating nursery size: %d\n", nursery_size));
1759 /* later we will alloc a larger area for the nursery but only activate
1760 * what we need. The rest will be used as expansion if we have too many pinned
1761 * objects in the existing nursery.
1763 /* FIXME: handle OOM */
1764 section = get_internal_mem (sizeof (GCMemSection));
1765 data = get_os_memory (nursery_size, TRUE);
1766 nursery_start = nursery_next = data;
1767 nursery_real_end = data + nursery_size;
1768 nursery_temp_end = data + SCAN_START_SIZE;
1769 UPDATE_HEAP_BOUNDARIES (nursery_start, nursery_real_end);
1770 total_alloc += nursery_size;
1771 DEBUG (4, fprintf (gc_debug_file, "Expanding heap size: %d, total: %d\n", nursery_size, total_alloc));
1772 section->data = section->next_data = data;
1773 section->size = nursery_size;
1774 section->end_data = nursery_real_end;
1775 scan_starts = nursery_size / SCAN_START_SIZE;
1776 section->scan_starts = get_internal_mem (sizeof (char*) * scan_starts);
1777 section->num_scan_start = scan_starts;
1779 /* add to the section list */
1780 section->next = section_list;
1781 section_list = section;
1783 nursery_section = section;
1785 /* Setup the single first large fragment */
1786 frag = alloc_fragment ();
1787 frag->fragment_start = nursery_start;
1788 frag->fragment_limit = nursery_start;
1789 frag->fragment_end = nursery_real_end;
1790 nursery_frag_real_end = nursery_real_end;
1791 /* FIXME: frag here is lost */
1795 * Update roots in the old generation. Since we currently don't have the
1796 * info from the write barriers, we just scan all the objects.
1799 scan_old_generation (char *start, char* end)
1801 GCMemSection *section;
1803 LOSObject *big_object;
1806 for (section = section_list; section; section = section->next) {
1807 if (section == nursery_section)
1809 DEBUG (2, fprintf (gc_debug_file, "Scan of old section: %p-%p, size: %d\n", section->data, section->next_data, (int)(section->next_data - section->data)));
1810 /* we have to deal with zeroed holes in old generation (truncated strings ...) */
1812 while (p < section->next_data) {
1817 DEBUG (8, fprintf (gc_debug_file, "Precise old object scan of %p (%s)\n", p, safe_name (p)));
1818 p = scan_object (p, start, end);
1821 /* scan the old object space, too */
1822 for (big_object = los_object_list; big_object; big_object = big_object->next) {
1823 DEBUG (5, fprintf (gc_debug_file, "Scan of big object: %p (%s), size: %d\n", big_object->data, safe_name (big_object->data), big_object->size));
1824 scan_object (big_object->data, start, end);
1826 /* scan the list of objects ready for finalization */
1827 for (fin = fin_ready_list; fin; fin = fin->next) {
1828 DEBUG (5, fprintf (gc_debug_file, "Scan of fin ready object: %p (%s)\n", fin->object, safe_name (fin->object)));
1829 fin->object = copy_object (fin->object, start, end);
1833 static mword fragment_total = 0;
1835 * We found a fragment of free memory in the nursery: memzero it and if
1836 * it is big enough, add it to the list of fragments that can be used for
1840 add_nursery_frag (size_t frag_size, char* frag_start, char* frag_end)
1843 DEBUG (4, fprintf (gc_debug_file, "Found empty fragment: %p-%p, size: %d\n", frag_start, frag_end, frag_size));
1844 /* memsetting just the first chunk start is bound to provide better cache locality */
1845 memset (frag_start, 0, frag_size);
1846 /* Not worth dealing with smaller fragments: need to tune */
1847 if (frag_size >= FRAGMENT_MIN_SIZE) {
1848 fragment = alloc_fragment ();
1849 fragment->fragment_start = frag_start;
1850 fragment->fragment_limit = frag_start;
1851 fragment->fragment_end = frag_end;
1852 fragment->next = nursery_fragments;
1853 nursery_fragments = fragment;
1854 fragment_total += frag_size;
1859 drain_gray_stack (char *start_addr, char *end_addr)
1867 * We copied all the reachable objects. Now it's the time to copy
1868 * the objects that were not referenced by the roots, but by the copied objects.
1869 * we built a stack of objects pointed to by gray_start: they are
1870 * additional roots and we may add more items as we go.
1871 * We loop until gray_start == gray_objects which means no more objects have
1872 * been added. Note this is iterative: no recursion is involved.
1873 * We need to walk the LO list as well in search of marked big objects
1874 * (use a flag since this is needed only on major collections). We need to loop
1875 * here as well, so keep a counter of marked LO (increasing it in copy_object).
1878 gray_start = to_space;
1879 DEBUG (6, fprintf (gc_debug_file, "Precise scan of gray area: %p-%p, size: %d\n", gray_start, gray_objects, (int)(gray_objects - gray_start)));
1880 while (gray_start < gray_objects) {
1881 DEBUG (9, fprintf (gc_debug_file, "Precise gray object scan %p (%s)\n", gray_start, safe_name (gray_start)));
1882 gray_start = scan_object (gray_start, start_addr, end_addr);
1885 DEBUG (2, fprintf (gc_debug_file, "Gray stack scan: %d usecs\n", TV_ELAPSED (btv, atv)));
1886 //scan_old_generation (start_addr, end_addr);
1887 DEBUG (2, fprintf (gc_debug_file, "Old generation done\n"));
1888 /* walk the finalization queue and move also the objects that need to be
1889 * finalized: use the finalized objects as new roots so the objects they depend
1890 * on are also not reclaimed. As with the roots above, only objects in the nursery
1891 * are marked/copied.
1892 * We need a loop here, since objects ready for finalizers may reference other objects
1893 * that are fin-ready. Speedup with a flag?
1896 fin_ready = num_ready_finalizers;
1897 finalize_in_range ((void**)start_addr, (void**)end_addr);
1899 /* drain the new stack that might have been created */
1900 DEBUG (6, fprintf (gc_debug_file, "Precise scan of gray area post fin: %p-%p, size: %d\n", gray_start, gray_objects, (int)(gray_objects - gray_start)));
1901 while (gray_start < gray_objects) {
1902 DEBUG (9, fprintf (gc_debug_file, "Precise gray object scan %p (%s)\n", gray_start, safe_name (gray_start)));
1903 gray_start = scan_object (gray_start, start_addr, end_addr);
1905 } while (fin_ready != num_ready_finalizers);
1907 DEBUG (2, fprintf (gc_debug_file, "Copied to old space: %d bytes\n", (int)(gray_objects - to_space)));
1908 to_space = gray_start;
1909 to_space_section->next_data = to_space;
1912 * handle disappearing links
1913 * Note we do this after checking the finalization queue because if an object
1914 * survives (at least long enough to be finalized) we don't clear the link.
1915 * This also deals with a possible issue with the monitor reclamation: with the Boehm
1916 * GC a finalized object my lose the monitor because it is cleared before the finalizer is
1919 null_link_in_range ((void**)start_addr, (void**)end_addr);
1921 DEBUG (2, fprintf (gc_debug_file, "Finalize queue handling scan: %d usecs\n", TV_ELAPSED (atv, btv)));
1924 static int last_num_pinned = 0;
1927 build_nursery_fragments (int start_pin, int end_pin)
1929 char *frag_start, *frag_end;
1933 /* FIXME: handle non-NULL fragment_freelist */
1934 fragment_freelist = nursery_fragments;
1935 nursery_fragments = NULL;
1936 frag_start = nursery_start;
1938 /* clear scan starts */
1939 memset (nursery_section->scan_starts, 0, nursery_section->num_scan_start * sizeof (gpointer));
1940 for (i = start_pin; i < end_pin; ++i) {
1941 frag_end = pin_queue [i];
1942 /* remove the pin bit from pinned objects */
1943 unpin_object (frag_end);
1944 nursery_section->scan_starts [((char*)frag_end - (char*)nursery_section->data)/SCAN_START_SIZE] = frag_end;
1945 frag_size = frag_end - frag_start;
1947 add_nursery_frag (frag_size, frag_start, frag_end);
1948 frag_size = safe_object_get_size ((MonoObject*)pin_queue [i]);
1949 frag_size += ALLOC_ALIGN - 1;
1950 frag_size &= ~(ALLOC_ALIGN - 1);
1951 frag_start = (char*)pin_queue [i] + frag_size;
1953 nursery_last_pinned_end = frag_start;
1954 frag_end = nursery_real_end;
1955 frag_size = frag_end - frag_start;
1957 add_nursery_frag (frag_size, frag_start, frag_end);
1958 if (!nursery_fragments) {
1959 DEBUG (1, fprintf (gc_debug_file, "Nursery fully pinned (%d)\n", end_pin - start_pin));
1960 for (i = start_pin; i < end_pin; ++i) {
1961 DEBUG (3, fprintf (gc_debug_file, "Bastard pinning obj %p (%s), size: %d\n", pin_queue [i], safe_name (pin_queue [i]), safe_object_get_size (pin_queue [i])));
1967 /* FIXME: later reduce code duplication here with the above
1968 * We don't keep track of section fragments for non-nursery sections yet, so
1972 build_section_fragments (GCMemSection *section)
1975 char *frag_start, *frag_end;
1978 /* clear scan starts */
1979 memset (section->scan_starts, 0, section->num_scan_start * sizeof (gpointer));
1980 frag_start = section->data;
1981 section->next_data = section->data;
1982 for (i = section->pin_queue_start; i < section->pin_queue_end; ++i) {
1983 frag_end = pin_queue [i];
1984 /* remove the pin bit from pinned objects */
1985 unpin_object (frag_end);
1986 if (frag_end >= section->data + section->size) {
1987 frag_end = section->data + section->size;
1989 section->scan_starts [((char*)frag_end - (char*)section->data)/SCAN_START_SIZE] = frag_end;
1991 frag_size = frag_end - frag_start;
1993 memset (frag_start, 0, frag_size);
1994 frag_size = safe_object_get_size ((MonoObject*)pin_queue [i]);
1995 frag_size += ALLOC_ALIGN - 1;
1996 frag_size &= ~(ALLOC_ALIGN - 1);
1997 frag_start = (char*)pin_queue [i] + frag_size;
1998 section->next_data = MAX (section->next_data, frag_start);
2000 frag_end = section->end_data;
2001 frag_size = frag_end - frag_start;
2003 memset (frag_start, 0, frag_size);
2007 scan_from_registered_roots (char *addr_start, char *addr_end)
2011 for (i = 0; i < roots_hash_size; ++i) {
2012 for (root = roots_hash [i]; root; root = root->next) {
2013 /* if desc is non-null it has precise info */
2014 if (!root->root_desc)
2016 DEBUG (6, fprintf (gc_debug_file, "Precise root scan %p-%p (desc: %p)\n", root->start_root, root->end_root, (void*)root->root_desc));
2017 precisely_scan_objects_from ((void**)root->start_root, root->end_root, addr_start, addr_end, root->root_desc);
2023 * Collect objects in the nursery.
2026 collect_nursery (size_t requested_size)
2028 GCMemSection *section;
2029 size_t max_garbage_amount;
2035 nursery_next = MAX (nursery_next, nursery_last_pinned_end);
2036 /* FIXME: optimize later to use the higher address where an object can be present */
2037 nursery_next = MAX (nursery_next, nursery_real_end);
2039 DEBUG (1, fprintf (gc_debug_file, "Start nursery collection %d %p-%p, size: %d\n", num_minor_gcs, nursery_start, nursery_next, (int)(nursery_next - nursery_start)));
2040 max_garbage_amount = nursery_next - nursery_start;
2042 * not enough room in the old generation to store all the possible data from
2043 * the nursery in a single continuous space.
2044 * We reset to_space if we allocated objects in degraded mode.
2046 if (to_space_section)
2047 to_space = gray_objects = to_space_section->next_data;
2048 if ((to_space_end - to_space) < max_garbage_amount) {
2049 section = alloc_section (nursery_section->size * 4);
2050 g_assert (nursery_section->size >= max_garbage_amount);
2051 to_space = gray_objects = section->next_data;
2052 to_space_end = section->end_data;
2053 to_space_section = section;
2055 DEBUG (2, fprintf (gc_debug_file, "To space setup: %p-%p in section %p\n", to_space, to_space_end, to_space_section));
2056 nursery_section->next_data = nursery_next;
2059 /* world must be stopped already */
2061 /* pin from pinned handles */
2062 pin_from_roots (nursery_start, nursery_next);
2063 /* identify pinned objects */
2064 optimize_pin_queue (0);
2065 next_pin_slot = pin_objects_from_addresses (nursery_section, pin_queue, pin_queue + next_pin_slot, nursery_start, nursery_next);
2067 DEBUG (2, fprintf (gc_debug_file, "Finding pinned pointers: %d in %d usecs\n", next_pin_slot, TV_ELAPSED (atv, btv)));
2068 DEBUG (4, fprintf (gc_debug_file, "Start scan with %d pinned objects\n", next_pin_slot));
2071 * walk all the roots and copy the young objects to the old generation,
2072 * starting from to_space
2075 scan_from_remsets (nursery_start, nursery_next);
2076 /* we don't have complete write barrier yet, so we scan all the old generation sections */
2078 DEBUG (2, fprintf (gc_debug_file, "Old generation scan: %d usecs\n", TV_ELAPSED (btv, atv)));
2079 /* FIXME: later scan also alloc_pinned objects */
2081 /* the pinned objects are roots */
2082 for (i = 0; i < next_pin_slot; ++i) {
2083 DEBUG (6, fprintf (gc_debug_file, "Precise object scan %d of pinned %p (%s)\n", i, pin_queue [i], safe_name (pin_queue [i])));
2084 scan_object (pin_queue [i], nursery_start, nursery_next);
2086 /* registered roots, this includes static fields */
2087 scan_from_registered_roots (nursery_start, nursery_next);
2089 DEBUG (2, fprintf (gc_debug_file, "Root scan: %d usecs\n", TV_ELAPSED (atv, btv)));
2091 drain_gray_stack (nursery_start, nursery_next);
2093 /* walk the pin_queue, build up the fragment list of free memory, unmark
2094 * pinned objects as we go, memzero() the empty fragments so they are ready for the
2097 build_nursery_fragments (0, next_pin_slot);
2099 DEBUG (2, fprintf (gc_debug_file, "Fragment creation: %d usecs, %d bytes available\n", TV_ELAPSED (btv, atv), fragment_total));
2101 /* prepare the pin queue for the next collection */
2102 last_num_pinned = next_pin_slot;
2104 if (fin_ready_list) {
2105 DEBUG (4, fprintf (gc_debug_file, "Finalizer-thread wakeup: ready %d\n", num_ready_finalizers));
2106 mono_gc_finalize_notify ();
2111 major_collection (void)
2113 GCMemSection *section, *prev_section;
2114 LOSObject *bigobj, *prevbo;
2121 /* FIXME: only use these values for the precise scan
2122 * note that to_space pointers should be excluded anyway...
2124 char *heap_start = NULL;
2125 char *heap_end = (char*)-1;
2126 size_t copy_space_required = 0;
2129 DEBUG (1, fprintf (gc_debug_file, "Start major collection %d\n", num_major_gcs));
2132 * FIXME: implement Mark/Compact
2133 * Until that is done, we can just apply mostly the same alg as for the nursery:
2134 * this means we need a big section to potentially copy all the other sections, so
2135 * it is not ideal specially with large heaps.
2137 if (g_getenv ("MONO_GC_NO_MAJOR")) {
2138 collect_nursery (0);
2141 /* FIXME: make sure the nursery next_data ptr is updated */
2142 nursery_section->next_data = nursery_real_end;
2143 /* we should also coalesce scanning from sections close to each other
2144 * and deal with pointers outside of the sections later.
2146 /* The remsets are not useful for a major collection */
2148 /* world must be stopped already */
2150 DEBUG (6, fprintf (gc_debug_file, "Pinning from sections\n"));
2151 for (section = section_list; section; section = section->next) {
2152 section->pin_queue_start = count = section->pin_queue_end = next_pin_slot;
2153 pin_from_roots (section->data, section->next_data);
2154 if (count != next_pin_slot) {
2156 optimize_pin_queue (count);
2157 DEBUG (6, fprintf (gc_debug_file, "Found %d pinning addresses in section %p (%d-%d)\n", next_pin_slot - count, section, count, next_pin_slot));
2158 reduced_to = pin_objects_from_addresses (section, pin_queue + count, pin_queue + next_pin_slot, section->data, section->next_data);
2159 section->pin_queue_end = next_pin_slot = count + reduced_to;
2161 copy_space_required += (char*)section->next_data - (char*)section->data;
2163 /* identify possible pointers to the insize of large objects */
2164 DEBUG (6, fprintf (gc_debug_file, "Pinning from large objects\n"));
2165 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
2166 count = next_pin_slot;
2167 pin_from_roots (bigobj->data, (char*)bigobj->data + bigobj->size);
2168 /* FIXME: this is only valid until we don't optimize the pin queue midway */
2169 if (next_pin_slot != count) {
2170 next_pin_slot = count;
2171 pin_object (bigobj->data);
2172 DEBUG (6, fprintf (gc_debug_file, "Marked large object %p (%s) size: %d from roots\n", bigobj->data, safe_name (bigobj->data), bigobj->size));
2175 /* look for pinned addresses for pinned-alloc objects */
2176 DEBUG (6, fprintf (gc_debug_file, "Pinning from pinned-alloc objects\n"));
2177 for (chunk = pinned_chunk_list; chunk; chunk = chunk->next) {
2178 count = next_pin_slot;
2179 pin_from_roots (chunk->start_data, (char*)chunk + chunk->num_pages * FREELIST_PAGESIZE);
2180 /* FIXME: this is only valid until we don't optimize the pin queue midway */
2181 if (next_pin_slot != count) {
2182 mark_pinned_from_addresses (chunk, pin_queue + count, pin_queue + next_pin_slot);
2183 next_pin_slot = count;
2188 DEBUG (2, fprintf (gc_debug_file, "Finding pinned pointers: %d in %d usecs\n", next_pin_slot, TV_ELAPSED (atv, btv)));
2189 DEBUG (4, fprintf (gc_debug_file, "Start scan with %d pinned objects\n", next_pin_slot));
2191 /* allocate the big to space */
2192 DEBUG (4, fprintf (gc_debug_file, "Allocate tospace for size: %d\n", copy_space_required));
2193 section = alloc_section (copy_space_required);
2194 to_space = gray_objects = section->next_data;
2195 to_space_end = section->end_data;
2196 to_space_section = section;
2198 /* the old generation doesn't need to be scanned (no remembered sets or card
2199 * table needed either): the only objects that must survive are those pinned and
2200 * those referenced by the precise roots.
2201 * mark any section without pinned objects, so we can free it since we will be able to
2202 * move all the objects.
2204 /* the pinned objects are roots (big objects are included in this list, too) */
2205 for (i = 0; i < next_pin_slot; ++i) {
2206 DEBUG (6, fprintf (gc_debug_file, "Precise object scan %d of pinned %p (%s)\n", i, pin_queue [i], safe_name (pin_queue [i])));
2207 scan_object (pin_queue [i], heap_start, heap_end);
2209 /* registered roots, this includes static fields */
2210 scan_from_registered_roots (heap_start, heap_end);
2212 /* scan the list of objects ready for finalization */
2213 for (fin = fin_ready_list; fin; fin = fin->next) {
2214 DEBUG (5, fprintf (gc_debug_file, "Scan of fin ready object: %p (%s)\n", fin->object, safe_name (fin->object)));
2215 fin->object = copy_object (fin->object, heap_start, heap_end);
2218 DEBUG (2, fprintf (gc_debug_file, "Root scan: %d usecs\n", TV_ELAPSED (btv, atv)));
2220 /* all the objects in the heap */
2221 drain_gray_stack (heap_start, heap_end);
2223 /* sweep the big objects list */
2225 for (bigobj = los_object_list; bigobj;) {
2226 if (object_is_pinned (bigobj->data)) {
2227 unpin_object (bigobj->data);
2230 /* not referenced anywhere, so we can free it */
2232 prevbo->next = bigobj->next;
2234 los_object_list = bigobj->next;
2236 bigobj = bigobj->next;
2237 free_large_object (to_free);
2241 bigobj = bigobj->next;
2243 /* unpin objects from the pinned chunks and free the unmarked ones */
2244 sweep_pinned_objects ();
2246 /* free the unused sections */
2247 prev_section = NULL;
2248 for (section = section_list; section;) {
2249 /* to_space doesn't need handling here and the nursery is special */
2250 if (section == to_space_section || section == nursery_section) {
2251 prev_section = section;
2252 section = section->next;
2255 /* no pinning object, so the section is free */
2256 if (section->pin_queue_start == section->pin_queue_end) {
2257 GCMemSection *to_free;
2259 prev_section->next = section->next;
2261 section_list = section->next;
2263 section = section->next;
2264 free_mem_section (to_free);
2267 DEBUG (6, fprintf (gc_debug_file, "Section %p has still pinned objects (%d)\n", section, section->pin_queue_end - section->pin_queue_start));
2268 build_section_fragments (section);
2270 prev_section = section;
2271 section = section->next;
2274 /* walk the pin_queue, build up the fragment list of free memory, unmark
2275 * pinned objects as we go, memzero() the empty fragments so they are ready for the
2278 build_nursery_fragments (nursery_section->pin_queue_start, nursery_section->pin_queue_end);
2280 /* prepare the pin queue for the next collection */
2282 if (fin_ready_list) {
2283 DEBUG (4, fprintf (gc_debug_file, "Finalizer-thread wakeup: ready %d\n", num_ready_finalizers));
2284 mono_gc_finalize_notify ();
2289 * Allocate a new section of memory to be used as old generation.
2291 static GCMemSection*
2292 alloc_section (size_t size)
2294 GCMemSection *section;
2297 size_t new_size = next_section_size;
2299 if (size > next_section_size) {
2301 new_size += pagesize - 1;
2302 new_size &= ~(pagesize - 1);
2304 section_size_used++;
2305 if (section_size_used > 3) {
2306 section_size_used = 0;
2307 next_section_size *= 2;
2308 if (next_section_size > max_section_size)
2309 next_section_size = max_section_size;
2311 section = get_internal_mem (sizeof (GCMemSection));
2312 data = get_os_memory (new_size, TRUE);
2313 section->data = section->next_data = data;
2314 section->size = new_size;
2315 section->end_data = data + new_size;
2316 UPDATE_HEAP_BOUNDARIES (data, section->end_data);
2317 total_alloc += new_size;
2318 DEBUG (2, fprintf (gc_debug_file, "Expanding heap size: %d, total: %d\n", new_size, total_alloc));
2319 section->data = data;
2320 section->size = new_size;
2321 scan_starts = new_size / SCAN_START_SIZE;
2322 section->scan_starts = get_internal_mem (sizeof (char*) * scan_starts);
2323 section->num_scan_start = scan_starts;
2325 /* add to the section list */
2326 section->next = section_list;
2327 section_list = section;
2333 free_mem_section (GCMemSection *section)
2335 char *data = section->data;
2336 size_t size = section->size;
2337 DEBUG (2, fprintf (gc_debug_file, "Freed section %p, size %d\n", data, size));
2338 free_os_memory (data, size);
2339 free_internal_mem (section);
2340 total_alloc -= size;
2344 * When deciding if it's better to collect or to expand, keep track
2345 * of how much garbage was reclaimed with the last collection: if it's too
2347 * This is called when we could not allocate a small object.
2349 static void __attribute__((noinline))
2350 minor_collect_or_expand_inner (size_t size)
2352 int do_minor_collection = 1;
2354 if (!nursery_section) {
2358 if (do_minor_collection) {
2360 collect_nursery (size);
2361 DEBUG (2, fprintf (gc_debug_file, "Heap size: %d, LOS size: %d\n", total_alloc, los_memory_usage));
2363 /* this also sets the proper pointers for the next allocation */
2364 if (!search_fragment_for_size (size)) {
2366 /* TypeBuilder and MonoMethod are killing mcs with fragmentation */
2367 DEBUG (1, fprintf (gc_debug_file, "nursery collection didn't find enough room for %d alloc (%d pinned)", size, last_num_pinned));
2368 for (i = 0; i < last_num_pinned; ++i) {
2369 DEBUG (3, fprintf (gc_debug_file, "Bastard pinning obj %p (%s), size: %d\n", pin_queue [i], safe_name (pin_queue [i]), safe_object_get_size (pin_queue [i])));
2374 //report_internal_mem_usage ();
2378 * ######################################################################
2379 * ######## Memory allocation from the OS
2380 * ######################################################################
2381 * This section of code deals with getting memory from the OS and
2382 * allocating memory for GC-internal data structures.
2383 * Internal memory can be handled with a freelist for small objects.
2386 #ifndef MAP_ANONYMOUS
2387 #define MAP_ANONYMOUS MAP_ANON
2391 * Allocate a big chunk of memory from the OS (usually 64KB to several megabytes).
2392 * This must not require any lock.
2395 get_os_memory (size_t size, int activate)
2398 unsigned long prot_flags = activate? MONO_MMAP_READ|MONO_MMAP_WRITE: MONO_MMAP_NONE;
2400 prot_flags |= MONO_MMAP_PRIVATE | MONO_MMAP_ANON;
2401 size += pagesize - 1;
2402 size &= ~(pagesize - 1);
2403 ptr = mono_valloc (0, size, prot_flags);
2408 * Free the memory returned by get_os_memory (), returning it to the OS.
2411 free_os_memory (void *addr, size_t size)
2413 munmap (addr, size);
2420 report_pinned_chunk (PinnedChunk *chunk, int seq) {
2422 int i, free_pages, num_free, free_mem;
2424 for (i = 0; i < chunk->num_pages; ++i) {
2425 if (!chunk->page_sizes [i])
2428 printf ("Pinned chunk %d at %p, size: %d, pages: %d, free: %d\n", seq, chunk, chunk->num_pages * FREELIST_PAGESIZE, chunk->num_pages, free_pages);
2429 free_mem = FREELIST_PAGESIZE * free_pages;
2430 for (i = 0; i < FREELIST_NUM_SLOTS; ++i) {
2431 if (!chunk->free_list [i])
2434 p = chunk->free_list [i];
2439 printf ("\tfree list of size %d, %d items\n", freelist_sizes [i], num_free);
2440 free_mem += freelist_sizes [i] * num_free;
2442 printf ("\tfree memory in chunk: %d\n", free_mem);
2449 report_internal_mem_usage (void) {
2452 printf ("Internal memory usage:\n");
2454 for (chunk = internal_chunk_list; chunk; chunk = chunk->next) {
2455 report_pinned_chunk (chunk, i++);
2457 printf ("Pinned memory usage:\n");
2459 for (chunk = pinned_chunk_list; chunk; chunk = chunk->next) {
2460 report_pinned_chunk (chunk, i++);
2465 * the array of pointers from @start to @end contains conservative
2466 * pointers to objects inside @chunk: mark each referenced object
2470 mark_pinned_from_addresses (PinnedChunk *chunk, void **start, void **end)
2472 for (; start < end; start++) {
2473 char *addr = *start;
2474 int offset = (char*)addr - (char*)chunk;
2475 int page = offset / FREELIST_PAGESIZE;
2476 int obj_offset = page == 0? offset - ((char*)chunk->start_data - (char*)chunk): offset % FREELIST_PAGESIZE;
2477 int slot_size = chunk->page_sizes [page];
2479 /* the page is not allocated */
2482 /* would be faster if we restrict the sizes to power of two,
2483 * but that's a waste of memory: need to measure. it could reduce
2484 * fragmentation since there are less pages needed, if for example
2485 * someone interns strings of each size we end up with one page per
2486 * interned string (still this is just ~40 KB): with more fine-grained sizes
2487 * this increases the number of used pages.
2490 obj_offset /= slot_size;
2491 obj_offset *= slot_size;
2492 addr = (char*)chunk->start_data + obj_offset;
2494 obj_offset /= slot_size;
2495 obj_offset *= slot_size;
2496 addr = (char*)chunk + page * FREELIST_PAGESIZE + obj_offset;
2499 /* if the vtable is inside the chunk it's on the freelist, so skip */
2500 if (*ptr && (*ptr < (void*)chunk->start_data || *ptr > (void*)((char*)chunk + chunk->num_pages * FREELIST_PAGESIZE))) {
2502 DEBUG (6, fprintf (gc_debug_file, "Marked pinned object %p (%s) from roots\n", addr, safe_name (addr)));
2508 sweep_pinned_objects (void)
2515 for (chunk = pinned_chunk_list; chunk; chunk = chunk->next) {
2516 end_chunk = (char*)chunk + chunk->num_pages * FREELIST_PAGESIZE;
2517 DEBUG (6, fprintf (gc_debug_file, "Sweeping pinned chunk %p (ranhe: %p-%p)\n", chunk, chunk->start_data, end_chunk));
2518 for (i = 0; i < chunk->num_pages; ++i) {
2519 obj_size = chunk->page_sizes [i];
2522 p = i? (char*)chunk + i * FREELIST_PAGESIZE: chunk->start_data;
2523 endp = i? p + FREELIST_PAGESIZE: (char*)chunk + FREELIST_PAGESIZE;
2524 DEBUG (6, fprintf (gc_debug_file, "Page %d (size: %d, range: %p-%p)\n", i, obj_size, p, endp));
2525 while (p + obj_size <= endp) {
2527 DEBUG (9, fprintf (gc_debug_file, "Considering %p (vtable: %p)\n", ptr, *ptr));
2528 /* if the first word (the vtable) is outside the chunk we have an object */
2529 if (*ptr && (*ptr < (void*)chunk || *ptr >= end_chunk)) {
2530 if (object_is_pinned (ptr)) {
2532 DEBUG (6, fprintf (gc_debug_file, "Unmarked pinned object %p (%s)\n", ptr, safe_name (ptr)));
2534 /* FIXME: add to freelist */
2535 DEBUG (6, fprintf (gc_debug_file, "Going to free unmarked pinned object %p (%s)\n", ptr, safe_name (ptr)));
2545 * Find the slot number in the freelist for memory chunks that
2546 * can contain @size objects.
2549 slot_for_size (size_t size)
2552 /* do a binary search or lookup table later. */
2553 for (slot = 0; slot < FREELIST_NUM_SLOTS; ++slot) {
2554 if (freelist_sizes [slot] >= size)
2557 g_assert_not_reached ();
2562 * Build a free list for @size memory chunks from the memory area between
2563 * start_page and end_page.
2566 build_freelist (PinnedChunk *chunk, int slot, int size, char *start_page, char *end_page)
2570 /*g_print ("building freelist for slot %d, size %d in %p\n", slot, size, chunk);*/
2572 end = (void**)(end_page - size);
2573 g_assert (!chunk->free_list [slot]);
2574 chunk->free_list [slot] = p;
2575 while ((char*)p + size <= end) {
2577 *p = (void*)((char*)p + size);
2581 /*g_print ("%d items created, max: %d\n", count, (end_page - start_page) / size);*/
2585 alloc_pinned_chunk (size_t size)
2590 size += pagesize; /* at least one page */
2591 size += pagesize - 1;
2592 size &= ~(pagesize - 1);
2593 if (size < PINNED_CHUNK_MIN_SIZE * 2)
2594 size = PINNED_CHUNK_MIN_SIZE * 2;
2595 chunk = get_os_memory (size, TRUE);
2596 UPDATE_HEAP_BOUNDARIES (chunk, ((char*)chunk + size));
2597 total_alloc += size;
2599 /* setup the bookeeping fields */
2600 chunk->num_pages = size / FREELIST_PAGESIZE;
2601 offset = G_STRUCT_OFFSET (PinnedChunk, data);
2602 chunk->page_sizes = (void*)((char*)chunk + offset);
2603 offset += sizeof (int) * chunk->num_pages;
2604 offset += ALLOC_ALIGN - 1;
2605 offset &= ~(ALLOC_ALIGN - 1);
2606 chunk->free_list = (void*)((char*)chunk + offset);
2607 offset += sizeof (void*) * FREELIST_NUM_SLOTS;
2608 offset += ALLOC_ALIGN - 1;
2609 offset &= ~(ALLOC_ALIGN - 1);
2610 chunk->start_data = (void*)((char*)chunk + offset);
2612 /* allocate the first page to the freelist */
2613 chunk->page_sizes [0] = PINNED_FIRST_SLOT_SIZE;
2614 build_freelist (chunk, slot_for_size (PINNED_FIRST_SLOT_SIZE), PINNED_FIRST_SLOT_SIZE, chunk->start_data, ((char*)chunk + FREELIST_PAGESIZE));
2615 DEBUG (4, fprintf (gc_debug_file, "Allocated pinned chunk %p, size: %d\n", chunk, size));
2616 min_pinned_chunk_addr = MIN (min_pinned_chunk_addr, (char*)chunk->start_data);
2617 max_pinned_chunk_addr = MAX (max_pinned_chunk_addr, ((char*)chunk + size));
2621 /* assumes freelist for slot is empty, so try to alloc a new page */
2623 get_chunk_freelist (PinnedChunk *chunk, int slot)
2627 p = chunk->free_list [slot];
2629 chunk->free_list [slot] = *p;
2632 for (i = 0; i < chunk->num_pages; ++i) {
2634 if (chunk->page_sizes [i])
2636 size = freelist_sizes [slot];
2637 chunk->page_sizes [i] = size;
2638 build_freelist (chunk, slot, size, (char*)chunk + FREELIST_PAGESIZE * i, (char*)chunk + FREELIST_PAGESIZE * (i + 1));
2642 p = chunk->free_list [slot];
2644 chunk->free_list [slot] = *p;
2651 alloc_from_freelist (size_t size)
2655 PinnedChunk *pchunk;
2656 slot = slot_for_size (size);
2657 /*g_print ("using slot %d for size %d (slot size: %d)\n", slot, size, freelist_sizes [slot]);*/
2658 g_assert (size <= freelist_sizes [slot]);
2659 for (pchunk = pinned_chunk_list; pchunk; pchunk = pchunk->next) {
2660 void **p = pchunk->free_list [slot];
2662 /*g_print ("found freelist for slot %d in chunk %p, returning %p, next %p\n", slot, pchunk, p, *p);*/
2663 pchunk->free_list [slot] = *p;
2667 for (pchunk = pinned_chunk_list; pchunk; pchunk = pchunk->next) {
2668 res = get_chunk_freelist (pchunk, slot);
2672 pchunk = alloc_pinned_chunk (size);
2673 /* FIXME: handle OOM */
2674 pchunk->next = pinned_chunk_list;
2675 pinned_chunk_list = pchunk;
2676 res = get_chunk_freelist (pchunk, slot);
2680 /* used for the GC-internal data structures */
2681 /* FIXME: add support for bigger sizes by allocating more than one page
2685 get_internal_mem (size_t size)
2687 return calloc (1, size);
2690 PinnedChunk *pchunk;
2691 slot = slot_for_size (size);
2692 g_assert (size <= freelist_sizes [slot]);
2693 for (pchunk = internal_chunk_list; pchunk; pchunk = pchunk->next) {
2694 void **p = pchunk->free_list [slot];
2696 pchunk->free_list [slot] = *p;
2700 for (pchunk = internal_chunk_list; pchunk; pchunk = pchunk->next) {
2701 res = get_chunk_freelist (pchunk, slot);
2705 pchunk = alloc_pinned_chunk (size);
2706 /* FIXME: handle OOM */
2707 pchunk->next = internal_chunk_list;
2708 internal_chunk_list = pchunk;
2709 res = get_chunk_freelist (pchunk, slot);
2714 free_internal_mem (void *addr)
2718 PinnedChunk *pchunk;
2719 for (pchunk = internal_chunk_list; pchunk; pchunk = pchunk->next) {
2720 /*printf ("trying to free %p in %p (pages: %d)\n", addr, pchunk, pchunk->num_pages);*/
2721 if (addr >= (void*)pchunk && (char*)addr < (char*)pchunk + pchunk->num_pages * FREELIST_PAGESIZE) {
2722 int offset = (char*)addr - (char*)pchunk;
2723 int page = offset / FREELIST_PAGESIZE;
2724 int slot = slot_for_size (pchunk->page_sizes [page]);
2726 *p = pchunk->free_list [slot];
2727 pchunk->free_list [slot] = p;
2731 printf ("free of %p failed\n", addr);
2732 g_assert_not_reached ();
2736 * ######################################################################
2737 * ######## Object allocation
2738 * ######################################################################
2739 * This section of code deals with allocating memory for objects.
2740 * There are several ways:
2741 * *) allocate large objects
2742 * *) allocate normal objects
2743 * *) fast lock-free allocation
2744 * *) allocation of pinned objects
2748 free_large_object (LOSObject *obj)
2750 size_t size = obj->size;
2751 DEBUG (4, fprintf (gc_debug_file, "Freed large object %p, size %d\n", obj->data, obj->size));
2753 los_memory_usage -= size;
2754 size += sizeof (LOSObject);
2755 size += pagesize - 1;
2756 size &= ~(pagesize - 1);
2757 total_alloc -= size;
2759 free_os_memory (obj, size);
2763 * Objects with size >= 64KB are allocated in the large object space.
2764 * They are currently kept track of with a linked list.
2765 * They don't move, so there is no need to pin them during collection
2766 * and we avoid the memcpy overhead.
2768 static void* __attribute__((noinline))
2769 alloc_large_inner (MonoVTable *vtable, size_t size)
2774 int just_did_major_gc = FALSE;
2776 if (los_memory_usage > next_los_collection) {
2777 DEBUG (4, fprintf (gc_debug_file, "Should trigger major collection: req size %d (los already: %u, limit: %u)\n", size, los_memory_usage, next_los_collection));
2778 just_did_major_gc = TRUE;
2780 major_collection ();
2782 /* later increase based on a percent of the heap size */
2783 next_los_collection = los_memory_usage + 5*1024*1024;
2786 alloc_size += sizeof (LOSObject);
2787 alloc_size += pagesize - 1;
2788 alloc_size &= ~(pagesize - 1);
2789 /* FIXME: handle OOM */
2790 obj = get_os_memory (alloc_size, TRUE);
2792 vtslot = (void**)obj->data;
2794 total_alloc += alloc_size;
2795 UPDATE_HEAP_BOUNDARIES (obj->data, (char*)obj->data + size);
2796 obj->next = los_object_list;
2797 los_object_list = obj;
2798 los_memory_usage += size;
2800 DEBUG (4, fprintf (gc_debug_file, "Allocated large object %p, vtable: %p (%s), size: %d\n", obj->data, vtable, vtable->klass->name, size));
2804 /* check if we have a suitable fragment in nursery_fragments to be able to allocate
2805 * an object of size @size
2806 * Return FALSE if not found (which means we need a collection)
2809 search_fragment_for_size (size_t size)
2811 Fragment *frag, *prev;
2812 DEBUG (4, fprintf (gc_debug_file, "Searching nursery fragment %p, size: %d\n", nursery_frag_real_end, size));
2814 for (frag = nursery_fragments; frag; frag = frag->next) {
2815 if (size <= (frag->fragment_end - frag->fragment_start)) {
2816 /* remove from the list */
2818 prev->next = frag->next;
2820 nursery_fragments = frag->next;
2821 nursery_next = frag->fragment_start;
2822 nursery_frag_real_end = frag->fragment_end;
2823 nursery_temp_end = MIN (nursery_frag_real_end, nursery_next + size + SCAN_START_SIZE);
2825 DEBUG (4, fprintf (gc_debug_file, "Using nursery fragment %p-%p, size: %d (req: %d)\n", nursery_next, nursery_frag_real_end, nursery_frag_real_end - nursery_next, size));
2826 frag->next = fragment_freelist;
2827 fragment_freelist = frag;
2836 * size is already rounded up.
2839 alloc_degraded (MonoVTable *vtable, size_t size)
2841 GCMemSection *section;
2843 for (section = section_list; section; section = section->next) {
2844 if (section != nursery_section && (section->end_data - section->next_data) >= size) {
2845 p = section->next_data;
2850 section = alloc_section (nursery_section->size * 4);
2851 /* FIXME: handle OOM */
2852 p = section->next_data;
2854 section->next_data += size;
2855 degraded_mode += size;
2856 DEBUG (3, fprintf (gc_debug_file, "Allocated (degraded) object %p, vtable: %p (%s), size: %d in section %p\n", p, vtable, vtable->klass->name, size, section));
2862 * Provide a variant that takes just the vtable for small fixed-size objects.
2863 * The aligned size is already computed and stored in vt->gc_descr.
2864 * Note: every SCAN_START_SIZE or so we are given the chance to do some special
2865 * processing. We can keep track of where objects start, for example,
2866 * so when we scan the thread stacks for pinned objects, we can start
2867 * a search for the pinned object in SCAN_START_SIZE chunks.
2870 mono_gc_alloc_obj (MonoVTable *vtable, size_t size)
2872 /* FIXME: handle OOM */
2875 size += ALLOC_ALIGN - 1;
2876 size &= ~(ALLOC_ALIGN - 1);
2878 g_assert (vtable->gc_descr);
2881 p = (void**)nursery_next;
2882 /* FIXME: handle overflow */
2883 nursery_next += size;
2884 if (nursery_next >= nursery_temp_end) {
2885 /* there are two cases: the object is too big or we need to collect */
2886 /* there can be another case (from ORP), if we cooperate with the runtime a bit:
2887 * objects that need finalizers can have the high bit set in their size
2888 * so the above check fails and we can readily add the object to the queue.
2889 * This avoids taking again the GC lock when registering, but this is moot when
2890 * doing thread-local allocation, so it may not be a good idea.
2892 if (size > MAX_SMALL_OBJ_SIZE) {
2893 /* get ready for possible collection */
2894 update_current_thread_stack (&dummy);
2895 nursery_next -= size;
2896 p = alloc_large_inner (vtable, size);
2898 if (nursery_next >= nursery_frag_real_end) {
2899 nursery_next -= size;
2900 /* when running in degraded mode, we continue allocing that way
2901 * for a while, to decrease the number of useless nursery collections.
2903 if (degraded_mode && degraded_mode < DEFAULT_NURSERY_SIZE) {
2904 p = alloc_degraded (vtable, size);
2908 if (!search_fragment_for_size (size)) {
2909 /* get ready for possible collection */
2910 update_current_thread_stack (&dummy);
2911 minor_collect_or_expand_inner (size);
2912 if (degraded_mode) {
2913 p = alloc_degraded (vtable, size);
2918 /* nursery_next changed by minor_collect_or_expand_inner () */
2919 p = (void*)nursery_next;
2920 nursery_next += size;
2921 if (nursery_next > nursery_temp_end) {
2926 /* record the scan start so we can find pinned objects more easily */
2927 nursery_section->scan_starts [((char*)p - (char*)nursery_section->data)/SCAN_START_SIZE] = p;
2928 /* we just bump nursery_temp_end as well */
2929 nursery_temp_end = MIN (nursery_frag_real_end, nursery_next + SCAN_START_SIZE);
2930 DEBUG (5, fprintf (gc_debug_file, "Expanding local alloc: %p-%p\n", nursery_next, nursery_temp_end));
2934 DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %d\n", p, vtable, vtable->klass->name, size));
2943 * To be used for interned strings and possibly MonoThread, reflection handles.
2944 * We may want to explicitly free these objects.
2947 mono_gc_alloc_pinned_obj (MonoVTable *vtable, size_t size)
2949 /* FIXME: handle OOM */
2951 size += ALLOC_ALIGN - 1;
2952 size &= ~(ALLOC_ALIGN - 1);
2954 if (size > MAX_FREELIST_SIZE) {
2955 update_current_thread_stack (&p);
2956 /* large objects are always pinned anyway */
2957 p = alloc_large_inner (vtable, size);
2959 p = alloc_from_freelist (size);
2960 memset (p, 0, size);
2962 DEBUG (6, fprintf (gc_debug_file, "Allocated pinned object %p, vtable: %p (%s), size: %d\n", p, vtable, vtable->klass->name, size));
2969 * ######################################################################
2970 * ######## Finalization support
2971 * ######################################################################
2975 * this is valid for the nursery: if the object has been forwarded it means it's
2976 * still refrenced from a root. If it is pinned it's still alive as well.
2977 * Return TRUE if @obj is ready to be finalized.
2979 #define object_is_fin_ready(obj) (!object_is_pinned (obj) && !object_is_forwarded (obj))
2982 finalize_in_range (void **start, void **end)
2984 FinalizeEntry *entry, *prev;
2988 for (i = 0; i < finalizable_hash_size; ++i) {
2990 for (entry = finalizable_hash [i]; entry;) {
2991 if (entry->object >= start && entry->object < end) {
2992 if (object_is_fin_ready (entry->object)) {
2994 FinalizeEntry *next;
2995 /* remove and put in fin_ready_list */
2997 prev->next = entry->next;
2999 finalizable_hash [i] = entry->next;
3001 num_ready_finalizers++;
3002 num_registered_finalizers--;
3003 entry->next = fin_ready_list;
3004 fin_ready_list = entry;
3005 /* Make it survive */
3006 from = entry->object;
3007 entry->object = copy_object (entry->object, start, end);
3008 DEBUG (5, fprintf (gc_debug_file, "Queueing object for finalization: %p (%s) (was at %p) (%d/%d)\n", entry->object, safe_name (entry->object), from, num_ready_finalizers, num_registered_finalizers));
3012 /* update pointer */
3013 DEBUG (5, fprintf (gc_debug_file, "Updating object for finalization: %p (%s)\n", entry->object, safe_name (entry->object)));
3014 entry->object = copy_object (entry->object, start, end);
3018 entry = entry->next;
3024 null_link_in_range (void **start, void **end)
3026 FinalizeEntry *entry, *prev;
3028 for (i = 0; i < disappearing_link_hash_size; ++i) {
3030 for (entry = disappearing_link_hash [i]; entry;) {
3031 if (entry->object >= start && entry->object < end) {
3032 if (object_is_fin_ready (entry->object)) {
3033 void **p = entry->data;
3036 /* remove from list */
3038 prev->next = entry->next;
3040 disappearing_link_hash [i] = entry->next;
3041 DEBUG (5, fprintf (gc_debug_file, "Dislink nullified at %p to GCed object %p\n", p, entry->object));
3043 free_internal_mem (entry);
3045 num_disappearing_links--;
3049 /* update pointer if it's moved
3050 * FIXME: what if an object is moved earlier?
3052 entry->object = copy_object (entry->object, start, end);
3053 DEBUG (5, fprintf (gc_debug_file, "Updated dislink at %p to %p\n", entry->data, entry->object));
3055 *link = entry->object;
3059 entry = entry->next;
3065 * mono_gc_finalizers_for_domain:
3066 * @domain: the unloading appdomain
3067 * @out_array: output array
3068 * @out_size: size of output array
3070 * Store inside @out_array up to @out_size objects that belong to the unloading
3071 * appdomain @domain. Returns the number of stored items. Can be called repeteadly
3072 * until it returns 0.
3073 * The items are removed from the finalizer data structure, so the caller is supposed
3075 * @out_array should be on the stack to allow the GC to know the objects are still alive.
3078 mono_gc_finalizers_for_domain (MonoDomain *domain, MonoObject **out_array, int out_size)
3080 FinalizeEntry *entry, *prev;
3082 if (no_finalize || !out_size || !out_array)
3086 for (i = 0; i < finalizable_hash_size; ++i) {
3088 for (entry = finalizable_hash [i]; entry;) {
3089 if (mono_object_domain (entry->object) == domain) {
3090 FinalizeEntry *next;
3091 /* remove and put in out_array */
3093 prev->next = entry->next;
3095 finalizable_hash [i] = entry->next;
3097 num_registered_finalizers--;
3098 out_array [count ++] = entry->object;
3099 DEBUG (5, fprintf (gc_debug_file, "Collecting object for finalization: %p (%s) (%d/%d)\n", entry->object, safe_name (entry->object), num_ready_finalizers, num_registered_finalizers));
3101 if (count == out_size) {
3108 entry = entry->next;
3116 rehash_fin_table (void)
3120 FinalizeEntry **new_hash;
3121 FinalizeEntry *entry, *next;
3122 int new_size = g_spaced_primes_closest (num_registered_finalizers);
3124 new_hash = get_internal_mem (new_size * sizeof (FinalizeEntry*));
3125 for (i = 0; i < finalizable_hash_size; ++i) {
3126 for (entry = finalizable_hash [i]; entry; entry = next) {
3127 hash = mono_object_hash (entry->object) % new_size;
3129 entry->next = new_hash [hash];
3130 new_hash [hash] = entry;
3133 free_internal_mem (finalizable_hash);
3134 finalizable_hash = new_hash;
3135 finalizable_hash_size = new_size;
3139 mono_gc_register_for_finalization (MonoObject *obj, void *user_data)
3141 FinalizeEntry *entry, *prev;
3145 hash = mono_object_hash (obj);
3147 if (num_registered_finalizers >= finalizable_hash_size * 2)
3148 rehash_fin_table ();
3149 hash %= finalizable_hash_size;
3151 for (entry = finalizable_hash [hash]; entry; entry = entry->next) {
3152 if (entry->object == obj) {
3154 entry->data = user_data;
3156 /* remove from the list */
3158 prev->next = entry->next;
3160 finalizable_hash [hash] = entry->next;
3161 num_registered_finalizers--;
3162 DEBUG (5, fprintf (gc_debug_file, "Removed finalizer %p for object: %p (%s) (%d)\n", entry, obj, obj->vtable->klass->name, num_registered_finalizers));
3163 free_internal_mem (entry);
3171 /* request to deregister, but already out of the list */
3175 entry = get_internal_mem (sizeof (FinalizeEntry));
3176 entry->object = obj;
3177 entry->data = user_data;
3178 entry->next = finalizable_hash [hash];
3179 finalizable_hash [hash] = entry;
3180 num_registered_finalizers++;
3181 DEBUG (5, fprintf (gc_debug_file, "Added finalizer %p for object: %p (%s) (%d)\n", entry, obj, obj->vtable->klass->name, num_registered_finalizers));
3186 rehash_dislink (void)
3190 FinalizeEntry **new_hash;
3191 FinalizeEntry *entry, *next;
3192 int new_size = g_spaced_primes_closest (num_disappearing_links);
3194 new_hash = get_internal_mem (new_size * sizeof (FinalizeEntry*));
3195 for (i = 0; i < disappearing_link_hash_size; ++i) {
3196 for (entry = disappearing_link_hash [i]; entry; entry = next) {
3197 hash = mono_aligned_addr_hash (entry->data) % new_size;
3199 entry->next = new_hash [hash];
3200 new_hash [hash] = entry;
3203 free_internal_mem (disappearing_link_hash);
3204 disappearing_link_hash = new_hash;
3205 disappearing_link_hash_size = new_size;
3209 mono_gc_register_disappearing_link (MonoObject *obj, void *link)
3211 FinalizeEntry *entry, *prev;
3215 if (num_disappearing_links >= disappearing_link_hash_size * 2)
3217 /* FIXME: add check that link is not in the heap */
3218 hash = mono_aligned_addr_hash (link) % disappearing_link_hash_size;
3219 entry = disappearing_link_hash [hash];
3221 for (; entry; entry = entry->next) {
3222 /* link already added */
3223 if (link == entry->data) {
3224 /* NULL obj means remove */
3227 prev->next = entry->next;
3229 disappearing_link_hash [hash] = entry->next;
3230 num_disappearing_links--;
3231 DEBUG (5, fprintf (gc_debug_file, "Removed dislink %p (%d)\n", entry, num_disappearing_links));
3232 free_internal_mem (entry);
3234 entry->object = obj; /* we allow the change of object */
3241 entry = get_internal_mem (sizeof (FinalizeEntry));
3242 entry->object = obj;
3244 entry->next = disappearing_link_hash [hash];
3245 disappearing_link_hash [hash] = entry;
3246 num_disappearing_links++;
3247 DEBUG (5, fprintf (gc_debug_file, "Added dislink %p for object: %p (%s) at %p\n", entry, obj, obj->vtable->klass->name, link));
3252 mono_gc_invoke_finalizers (void)
3254 FinalizeEntry *entry;
3257 /* FIXME: batch to reduce lock contention */
3258 while (fin_ready_list) {
3260 entry = fin_ready_list;
3262 fin_ready_list = entry->next;
3263 num_ready_finalizers--;
3264 obj = entry->object;
3265 DEBUG (7, fprintf (gc_debug_file, "Finalizing object %p (%s)\n", obj, safe_name (obj)));
3269 void (*callback)(void *, void*) = entry->data;
3271 obj = entry->object;
3273 /* the object is on the stack so it is pinned */
3274 /*g_print ("Calling finalizer for object: %p (%s)\n", entry->object, safe_name (entry->object));*/
3275 callback (obj, NULL);
3276 free_internal_mem (entry);
3283 mono_gc_pending_finalizers (void)
3285 return fin_ready_list != NULL;
3288 /* Negative value to remove */
3290 mono_gc_add_memory_pressure (gint64 value)
3292 /* FIXME: Use interlocked functions */
3294 memory_pressure += value;
3299 * ######################################################################
3300 * ######## registered roots support
3301 * ######################################################################
3309 RootRecord **new_hash;
3310 RootRecord *entry, *next;
3311 int new_size = g_spaced_primes_closest (num_roots_entries);
3313 new_hash = get_internal_mem (new_size * sizeof (RootRecord*));
3314 for (i = 0; i < roots_hash_size; ++i) {
3315 for (entry = roots_hash [i]; entry; entry = next) {
3316 hash = mono_aligned_addr_hash (entry->start_root) % new_size;
3318 entry->next = new_hash [hash];
3319 new_hash [hash] = entry;
3322 free_internal_mem (roots_hash);
3323 roots_hash = new_hash;
3324 roots_hash_size = new_size;
3328 * We do not coalesce roots.
3331 mono_gc_register_root (char *start, size_t size, void *descr)
3333 RootRecord *new_root;
3334 unsigned int hash = mono_aligned_addr_hash (start);
3336 if (num_roots_entries >= roots_hash_size * 2)
3338 hash %= roots_hash_size;
3339 for (new_root = roots_hash [hash]; new_root; new_root = new_root->next) {
3340 /* we allow changing the size and the descriptor (for thread statics etc) */
3341 if (new_root->start_root == start) {
3342 size_t old_size = new_root->end_root - new_root->start_root;
3343 new_root->end_root = new_root->start_root + size;
3344 new_root->root_desc = (mword)descr;
3346 roots_size -= old_size;
3351 new_root = get_internal_mem (sizeof (RootRecord));
3353 new_root->start_root = start;
3354 new_root->end_root = new_root->start_root + size;
3355 new_root->root_desc = (mword)descr;
3357 num_roots_entries++;
3358 new_root->next = roots_hash [hash];
3359 roots_hash [hash] = new_root;
3360 DEBUG (3, fprintf (gc_debug_file, "Added root %p for range: %p-%p, descr: %p (%d/%d bytes)\n", new_root, new_root->start_root, new_root->end_root, descr, (int)size, (int)roots_size));
3370 mono_gc_deregister_root (char* addr)
3372 RootRecord *tmp, *prev = NULL;
3373 unsigned int hash = mono_aligned_addr_hash (addr);
3375 hash %= roots_hash_size;
3376 tmp = roots_hash [hash];
3378 if (tmp->start_root == (char*)addr) {
3380 prev->next = tmp->next;
3382 roots_hash [hash] = tmp->next;
3383 roots_size -= (tmp->end_root - tmp->start_root);
3384 num_roots_entries--;
3385 DEBUG (3, fprintf (gc_debug_file, "Removed root %p for range: %p-%p\n", tmp, tmp->start_root, tmp->end_root));
3386 free_internal_mem (tmp);
3396 * ######################################################################
3397 * ######## Thread handling (stop/start code)
3398 * ######################################################################
3401 /* eventually share with MonoThread? */
3402 typedef struct _SgenThreadInfo SgenThreadInfo;
3404 struct _SgenThreadInfo {
3405 SgenThreadInfo *next;
3406 ARCH_THREAD_TYPE id;
3407 unsigned int stop_count; /* to catch duplicate signals */
3412 RememberedSet **remset;
3415 /* FIXME: handle large/small config */
3416 #define THREAD_HASH_SIZE 11
3417 #define HASH_PTHREAD_T(id) (((unsigned int)(id) >> 4) * 2654435761u)
3419 static SgenThreadInfo* thread_table [THREAD_HASH_SIZE];
3421 #if USE_SIGNAL_BASED_START_STOP_WORLD
3423 static sem_t suspend_ack_semaphore;
3424 static unsigned int global_stop_count = 0;
3425 static int suspend_signal_num = SIGPWR;
3426 static int restart_signal_num = SIGXCPU;
3427 static sigset_t suspend_signal_mask;
3428 static mword cur_thread_regs [ARCH_NUM_REGS] = {0};
3430 /* LOCKING: assumes the GC lock is held */
3431 static SgenThreadInfo*
3432 thread_info_lookup (ARCH_THREAD_TYPE id)
3434 unsigned int hash = HASH_PTHREAD_T (id) % THREAD_HASH_SIZE;
3435 SgenThreadInfo *info;
3437 info = thread_table [hash];
3438 while (info && !ARCH_THREAD_EQUALS (info->id, id)) {
3445 update_current_thread_stack (void *start)
3447 void *ptr = cur_thread_regs;
3448 SgenThreadInfo *info = thread_info_lookup (ARCH_GET_THREAD ());
3449 info->stack_start = align_pointer (&ptr);
3450 ARCH_STORE_REGS (ptr);
3454 signal_desc (int signum)
3456 if (signum == suspend_signal_num)
3458 if (signum == restart_signal_num)
3463 /* LOCKING: assumes the GC lock is held */
3465 thread_handshake (int signum)
3467 int count, i, result;
3468 SgenThreadInfo *info;
3469 pthread_t me = pthread_self ();
3472 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
3473 for (info = thread_table [i]; info; info = info->next) {
3474 DEBUG (4, fprintf (gc_debug_file, "considering thread %p for signal %d (%s)\n", info, signum, signal_desc (signum)));
3475 if (ARCH_THREAD_EQUALS (info->id, me)) {
3476 DEBUG (4, fprintf (gc_debug_file, "Skip (equal): %p, %p\n", (void*)me, (void*)info->id));
3479 /*if (signum == suspend_signal_num && info->stop_count == global_stop_count)
3481 result = pthread_kill (info->id, signum);
3483 DEBUG (4, fprintf (gc_debug_file, "thread %p signal sent\n", info));
3486 DEBUG (4, fprintf (gc_debug_file, "thread %p signal failed: %d (%s)\n", (void*)info->id, result, strerror (result)));
3492 for (i = 0; i < count; ++i) {
3493 while ((result = sem_wait (&suspend_ack_semaphore)) != 0) {
3494 if (errno != EINTR) {
3495 g_error ("sem_wait ()");
3502 /* LOCKING: assumes the GC lock is held (by the stopping thread) */
3504 suspend_handler (int sig)
3506 SgenThreadInfo *info;
3509 int old_errno = errno;
3511 id = pthread_self ();
3512 info = thread_info_lookup (id);
3513 stop_count = global_stop_count;
3514 /* duplicate signal */
3515 if (0 && info->stop_count == stop_count) {
3519 /* update the remset info in the thread data structure */
3520 info->remset = remembered_set;
3522 * this includes the register values that the kernel put on the stack.
3523 * Write arch-specific code to only push integer regs and a more accurate
3526 info->stack_start = align_pointer (&id);
3528 /* notify the waiting thread */
3529 sem_post (&suspend_ack_semaphore);
3530 info->stop_count = stop_count;
3532 /* wait until we receive the restart signal */
3535 sigsuspend (&suspend_signal_mask);
3536 } while (info->signal != restart_signal_num);
3538 /* notify the waiting thread */
3539 sem_post (&suspend_ack_semaphore);
3545 restart_handler (int sig)
3547 SgenThreadInfo *info;
3548 int old_errno = errno;
3550 info = thread_info_lookup (pthread_self ());
3551 info->signal = restart_signal_num;
3556 static TV_DECLARE (stop_world_time);
3557 static unsigned long max_pause_usec = 0;
3559 /* LOCKING: assumes the GC lock is held */
3565 global_stop_count++;
3566 DEBUG (3, fprintf (gc_debug_file, "stopping world n %d from %p %p\n", global_stop_count, thread_info_lookup (ARCH_GET_THREAD ()), (gpointer)ARCH_GET_THREAD ()));
3567 TV_GETTIME (stop_world_time);
3568 count = thread_handshake (suspend_signal_num);
3569 DEBUG (3, fprintf (gc_debug_file, "world stopped %d thread(s)\n", count));
3573 /* LOCKING: assumes the GC lock is held */
3575 restart_world (void)
3578 TV_DECLARE (end_sw);
3581 count = thread_handshake (restart_signal_num);
3582 TV_GETTIME (end_sw);
3583 usec = TV_ELAPSED (stop_world_time, end_sw);
3584 max_pause_usec = MAX (usec, max_pause_usec);
3585 DEBUG (2, fprintf (gc_debug_file, "restarted %d thread(s) (pause time: %d usec, max: %d)\n", count, (int)usec, (int)max_pause_usec));
3589 #endif /* USE_SIGNAL_BASED_START_STOP_WORLD */
3592 * Identify objects pinned in a thread stack and its registers.
3595 pin_thread_data (void *start_nursery, void *end_nursery)
3598 SgenThreadInfo *info;
3600 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
3601 for (info = thread_table [i]; info; info = info->next) {
3603 DEBUG (2, fprintf (gc_debug_file, "Skipping dead thread %p, range: %p-%p, size: %d\n", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start));
3606 DEBUG (2, fprintf (gc_debug_file, "Scanning thread %p, range: %p-%p, size: %d\n", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start));
3607 conservatively_pin_objects_from (info->stack_start, info->stack_end, start_nursery, end_nursery);
3610 DEBUG (2, fprintf (gc_debug_file, "Scanning current thread registers\n"));
3611 conservatively_pin_objects_from (cur_thread_regs, cur_thread_regs + ARCH_NUM_REGS, start_nursery, end_nursery);
3615 find_pinning_ref_from_thread (char *obj, size_t size)
3618 SgenThreadInfo *info;
3619 char *endobj = obj + size;
3621 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
3622 for (info = thread_table [i]; info; info = info->next) {
3623 char **start = (char**)info->stack_start;
3626 while (start < (char**)info->stack_end) {
3627 if (*start >= obj && *start < endobj) {
3628 DEBUG (0, fprintf (gc_debug_file, "Object %p referenced in thread %p (id %p) at %p, stack: %p-%p\n", obj, info, info->id, start, info->stack_start, info->stack_end));
3634 /* FIXME: check register */
3637 /* return TRUE if ptr points inside the managed heap */
3639 ptr_in_heap (void* ptr)
3641 mword p = (mword)ptr;
3642 if (p < lowest_heap_address || p >= highest_heap_address)
3644 /* FIXME: more checks */
3649 handle_remset (mword *p, void *start_nursery, void *end_nursery, gboolean global)
3654 /* FIXME: exclude stack locations */
3655 switch ((*p) & REMSET_TYPE_MASK) {
3656 case REMSET_LOCATION:
3658 if ((ptr < start_nursery || ptr >= end_nursery) && ptr_in_heap (ptr)) {
3659 *ptr = copy_object (*ptr, start_nursery, end_nursery);
3660 DEBUG (9, fprintf (gc_debug_file, "Overwrote remset at %p with %p\n", ptr, *ptr));
3661 if (!global && *ptr >= start_nursery && *ptr < end_nursery)
3662 add_to_global_remset (ptr);
3664 DEBUG (9, fprintf (gc_debug_file, "Skipping remset at %p holding %p\n", ptr, *ptr));
3668 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
3669 if ((ptr >= start_nursery && ptr < end_nursery) || !ptr_in_heap (ptr))
3672 while (count-- > 0) {
3673 *ptr = copy_object (*ptr, start_nursery, end_nursery);
3674 DEBUG (9, fprintf (gc_debug_file, "Overwrote remset at %p with %p (count: %d)\n", ptr, *ptr, (int)count));
3675 if (!global && *ptr >= start_nursery && *ptr < end_nursery)
3676 add_to_global_remset (ptr);
3681 ptr = (void**)(*p & ~REMSET_TYPE_MASK);
3682 if ((ptr >= start_nursery && ptr < end_nursery) || !ptr_in_heap (ptr))
3684 scan_object (*ptr, start_nursery, end_nursery);
3687 g_assert_not_reached ();
3693 scan_from_remsets (void *start_nursery, void *end_nursery)
3696 SgenThreadInfo *info;
3697 RememberedSet *remset, *next;
3700 /* the global one */
3701 for (remset = global_remset; remset; remset = remset->next) {
3702 DEBUG (4, fprintf (gc_debug_file, "Scanning global remset range: %p-%p, size: %d\n", remset->data, remset->store_next, remset->store_next - remset->data));
3703 for (p = remset->data; p < remset->store_next;) {
3704 p = handle_remset (p, start_nursery, end_nursery, TRUE);
3707 /* the per-thread ones */
3708 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
3709 for (info = thread_table [i]; info; info = info->next) {
3710 for (remset = info->remset; remset; remset = next) {
3711 DEBUG (4, fprintf (gc_debug_file, "Scanning remset for thread %p, range: %p-%p, size: %d\n", info, remset->data, remset->store_next, remset->store_next - remset->data));
3712 for (p = remset->data; p < remset->store_next;) {
3713 p = handle_remset (p, start_nursery, end_nursery, FALSE);
3715 remset->store_next = remset->data;
3716 next = remset->next;
3717 remset->next = NULL;
3718 if (remset != info->remset) {
3719 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
3720 free_internal_mem (remset);
3728 * Clear the info in the remembered sets: we're doing a major collection, so
3729 * the per-thread ones are not needed and the global ones will be reconstructed
3733 clear_remsets (void)
3736 SgenThreadInfo *info;
3737 RememberedSet *remset, *next;
3739 /* the global list */
3740 for (remset = global_remset; remset; remset = next) {
3741 remset->store_next = remset->data;
3742 next = remset->next;
3743 remset->next = NULL;
3744 if (remset != global_remset) {
3745 DEBUG (4, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
3746 free_internal_mem (remset);
3749 /* the per-thread ones */
3750 for (i = 0; i < THREAD_HASH_SIZE; ++i) {
3751 for (info = thread_table [i]; info; info = info->next) {
3752 for (remset = info->remset; remset; remset = next) {
3753 remset->store_next = remset->data;
3754 next = remset->next;
3755 remset->next = NULL;
3756 if (remset != info->remset) {
3757 DEBUG (1, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
3758 free_internal_mem (remset);
3765 /* LOCKING: assumes the GC lock is held */
3766 static SgenThreadInfo*
3767 gc_register_current_thread (void *addr)
3770 SgenThreadInfo* info = malloc (sizeof (SgenThreadInfo));
3773 info->id = ARCH_GET_THREAD ();
3774 info->stop_count = -1;
3777 info->stack_start = NULL;
3779 /* try to get it with attributes first */
3780 #if defined(HAVE_PTHREAD_GETATTR_NP) && defined(HAVE_PTHREAD_ATTR_GETSTACK)
3784 pthread_attr_t attr;
3785 pthread_getattr_np (pthread_self (), &attr);
3786 pthread_attr_getstack (&attr, &sstart, &size);
3787 info->stack_end = (char*)sstart + size;
3788 pthread_attr_destroy (&attr);
3790 #elif defined(HAVE_PTHREAD_GET_STACKSIZE_NP) && defined(HAVE_PTHREAD_GET_STACKADDR_NP)
3791 info->stack_end = (char*)pthread_get_stackaddr_np (pthread_self ());
3794 /* FIXME: we assume the stack grows down */
3795 gsize stack_bottom = (gsize)addr;
3796 stack_bottom += 4095;
3797 stack_bottom &= ~4095;
3798 info->stack_end = (char*)stack_bottom;
3802 /* hash into the table */
3803 hash = HASH_PTHREAD_T (info->id) % THREAD_HASH_SIZE;
3804 info->next = thread_table [hash];
3805 thread_table [hash] = info;
3807 remembered_set = info->remset = alloc_remset (DEFAULT_REMSET_SIZE, info);
3808 pthread_setspecific (remembered_set_key, remembered_set);
3809 DEBUG (3, fprintf (gc_debug_file, "registered thread %p (%p) (hash: %d)\n", info, (gpointer)info->id, hash));
3814 unregister_current_thread (void)
3817 SgenThreadInfo *prev = NULL;
3819 RememberedSet *rset;
3820 ARCH_THREAD_TYPE id = ARCH_GET_THREAD ();
3822 hash = HASH_PTHREAD_T (id) % THREAD_HASH_SIZE;
3823 p = thread_table [hash];
3825 DEBUG (3, fprintf (gc_debug_file, "unregister thread %p (%p)\n", p, (gpointer)p->id));
3826 while (!ARCH_THREAD_EQUALS (p->id, id)) {
3831 thread_table [hash] = p->next;
3833 prev->next = p->next;
3836 /* FIXME: transfer remsets if any */
3838 RememberedSet *next = rset->next;
3839 free_internal_mem (rset);
3846 unregister_thread (void *k)
3849 unregister_current_thread ();
3854 mono_gc_register_thread (void *baseptr)
3856 SgenThreadInfo *info;
3858 info = thread_info_lookup (ARCH_GET_THREAD ());
3860 info = gc_register_current_thread (baseptr);
3862 return info != NULL;
3865 #if USE_PTHREAD_INTERCEPT
3867 #undef pthread_create
3869 #undef pthread_detach
3872 void *(*start_routine) (void *);
3876 } SgenThreadStartInfo;
3879 gc_start_thread (void *arg)
3881 SgenThreadStartInfo *start_info = arg;
3882 SgenThreadInfo* info;
3883 void *t_arg = start_info->arg;
3884 void *(*start_func) (void*) = start_info->start_routine;
3888 info = gc_register_current_thread (&result);
3890 sem_post (&(start_info->registered));
3891 result = start_func (t_arg);
3893 * this is done by the pthread key dtor
3895 unregister_current_thread ();
3903 mono_gc_pthread_create (pthread_t *new_thread, const pthread_attr_t *attr, void *(*start_routine)(void *), void *arg)
3905 SgenThreadStartInfo *start_info;
3908 start_info = malloc (sizeof (SgenThreadStartInfo));
3911 sem_init (&(start_info->registered), 0, 0);
3912 start_info->arg = arg;
3913 start_info->start_routine = start_routine;
3915 result = pthread_create (new_thread, attr, gc_start_thread, start_info);
3917 while (sem_wait (&(start_info->registered)) != 0) {
3918 /*if (EINTR != errno) ABORT("sem_wait failed"); */
3921 sem_destroy (&(start_info->registered));
3927 mono_gc_pthread_join (pthread_t thread, void **retval)
3929 return pthread_join (thread, retval);
3933 mono_gc_pthread_detach (pthread_t thread)
3935 return pthread_detach (thread);
3938 #endif /* USE_PTHREAD_INTERCEPT */
3941 * ######################################################################
3942 * ######## Write barriers
3943 * ######################################################################
3946 static RememberedSet*
3947 alloc_remset (int size, gpointer id) {
3948 RememberedSet* res = get_internal_mem (sizeof (RememberedSet) + (size * sizeof (gpointer)));
3949 res->store_next = res->data;
3950 res->end_set = res->data + size;
3952 DEBUG (4, fprintf (gc_debug_file, "Allocated remset size %d at %p for %p\n", size, res->data, id));
3957 * Note: the write barriers first do the needed GC work and then do the actual store:
3958 * this way the value is visible to the conservative GC scan after the write barrier
3959 * itself. If a GC interrupts the barrier in the middle, value will be kept alive by
3960 * the conservative scan, otherwise by the remembered set scan. FIXME: figure out what
3961 * happens when we need to record which pointers contain references to the new generation.
3962 * The write barrier will be executed, but the pointer is still not stored.
3965 mono_gc_wbarrier_set_field (MonoObject *obj, gpointer field_ptr, MonoObject* value)
3968 if (field_ptr >= nursery_start && field_ptr < nursery_real_end) {
3969 *(void**)field_ptr = value;
3972 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", field_ptr));
3973 rs = remembered_set;
3974 if (rs->store_next < rs->end_set) {
3975 *(rs->store_next++) = (mword)field_ptr;
3976 *(void**)field_ptr = value;
3979 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
3980 rs->next = remembered_set;
3981 remembered_set = rs;
3982 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
3983 *(rs->store_next++) = (mword)field_ptr;
3984 *(void**)field_ptr = value;
3988 mono_gc_wbarrier_set_arrayref (MonoArray *arr, gpointer slot_ptr, MonoObject* value)
3990 RememberedSet *rs = remembered_set;
3991 if (slot_ptr >= nursery_start && slot_ptr < nursery_real_end) {
3992 *(void**)slot_ptr = value;
3995 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", slot_ptr));
3996 if (rs->store_next < rs->end_set) {
3997 *(rs->store_next++) = (mword)slot_ptr;
3998 *(void**)slot_ptr = value;
4001 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
4002 rs->next = remembered_set;
4003 remembered_set = rs;
4004 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
4005 *(rs->store_next++) = (mword)slot_ptr;
4006 *(void**)slot_ptr = value;
4010 mono_gc_wbarrier_arrayref_copy (MonoArray *arr, gpointer slot_ptr, int count)
4012 RememberedSet *rs = remembered_set;
4013 if (slot_ptr >= nursery_start && slot_ptr < nursery_real_end)
4015 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p, %d\n", slot_ptr, count));
4016 if (rs->store_next + 1 < rs->end_set) {
4017 *(rs->store_next++) = (mword)slot_ptr | REMSET_RANGE;
4018 *(rs->store_next++) = count;
4021 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
4022 rs->next = remembered_set;
4023 remembered_set = rs;
4024 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
4025 *(rs->store_next++) = (mword)slot_ptr | REMSET_RANGE;
4026 *(rs->store_next++) = count;
4030 mono_gc_wbarrier_generic_store (gpointer ptr, MonoObject* value)
4032 RememberedSet *rs = remembered_set;
4033 if (ptr >= nursery_start && ptr < nursery_real_end) {
4034 DEBUG (8, fprintf (gc_debug_file, "Skipping remset at %p\n", ptr));
4035 *(void**)ptr = value;
4038 DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", ptr));
4039 /* FIXME: ensure it is on the heap */
4040 if (rs->store_next < rs->end_set) {
4041 *(rs->store_next++) = (mword)ptr;
4042 *(void**)ptr = value;
4045 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
4046 rs->next = remembered_set;
4047 remembered_set = rs;
4048 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
4049 *(rs->store_next++) = (mword)ptr;
4050 *(void**)ptr = value;
4054 mono_gc_wbarrier_value_copy (gpointer dest, gpointer src, int count, MonoClass *klass)
4056 if (dest >= nursery_start && dest < nursery_real_end) {
4059 DEBUG (1, fprintf (gc_debug_file, "Adding value remset at %p, count %d for class %s\n", dest, count, klass->name));
4063 * mono_gc_wbarrier_object:
4065 * Write barrier to call when obj is the result of a clone or copy of an object.
4068 mono_gc_wbarrier_object (MonoObject* obj)
4070 RememberedSet *rs = remembered_set;
4071 DEBUG (1, fprintf (gc_debug_file, "Adding object remset for %p\n", obj));
4072 if (rs->store_next < rs->end_set) {
4073 *(rs->store_next++) = (mword)obj | REMSET_OBJECT;
4076 rs = alloc_remset (rs->end_set - rs->data, (void*)1);
4077 rs->next = remembered_set;
4078 remembered_set = rs;
4079 thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
4080 *(rs->store_next++) = (mword)obj | REMSET_OBJECT;
4084 * ######################################################################
4085 * ######## Other mono public interface functions.
4086 * ######################################################################
4090 mono_gc_collect (int generation)
4093 update_current_thread_stack (&generation);
4095 if (generation == 0) {
4096 collect_nursery (0);
4098 major_collection ();
4105 mono_gc_max_generation (void)
4111 mono_gc_collection_count (int generation)
4113 if (generation == 0)
4114 return num_minor_gcs;
4115 return num_major_gcs;
4119 mono_gc_get_used_size (void)
4122 GCMemSection *section;
4124 tot = los_memory_usage;
4125 for (section = section_list; section; section = section->next) {
4126 /* this is approximate... */
4127 tot += section->next_data - section->data;
4129 /* FIXME: account for pinned objects */
4135 mono_gc_get_heap_size (void)
4141 mono_gc_disable (void)
4149 mono_gc_enable (void)
4157 mono_object_is_alive (MonoObject* o)
4163 mono_gc_get_generation (MonoObject *obj)
4165 if ((char*)obj >= nursery_start && (char*)obj < nursery_real_end)
4171 mono_gc_enable_events (void)
4176 mono_gc_weak_link_add (void **link_addr, MonoObject *obj)
4178 mono_gc_register_disappearing_link (obj, link_addr);
4183 mono_gc_weak_link_remove (void **link_addr)
4185 mono_gc_register_disappearing_link (NULL, link_addr);
4190 mono_gc_weak_link_get (void **link_addr)
4196 mono_gc_make_descr_from_bitmap (gsize *bitmap, int numbits)
4198 if (numbits < ((sizeof (*bitmap) * 8) - ROOT_DESC_TYPE_SHIFT)) {
4199 mword desc = ROOT_DESC_BITMAP | (bitmap [0] << ROOT_DESC_TYPE_SHIFT);
4202 /* conservative scanning */
4203 DEBUG (3, fprintf (gc_debug_file, "Conservative root descr for size: %d\n", numbits));
4208 mono_gc_alloc_fixed (size_t size, void *descr)
4210 /* FIXME: do a single allocation */
4211 void *res = calloc (1, size);
4214 if (!mono_gc_register_root (res, size, descr)) {
4222 mono_gc_free_fixed (void* addr)
4224 mono_gc_deregister_root (addr);
4229 mono_gc_is_gc_thread (void)
4233 result = thread_info_lookup (ARCH_GET_THREAD ()) != NULL;
4239 mono_gc_base_init (void)
4242 struct sigaction sinfo;
4244 LOCK_INIT (gc_mutex);
4246 if (gc_initialized) {
4250 pagesize = mono_pagesize ();
4251 gc_debug_file = stderr;
4252 /* format: MONO_GC_DEBUG=l[,filename] where l is a debug level 0-9 */
4253 if ((env = getenv ("MONO_GC_DEBUG"))) {
4254 if (env [0] >= '0' && env [0] <= '9') {
4255 gc_debug_level = atoi (env);
4261 char *rf = g_strdup_printf ("%s.%d", env, getpid ());
4262 gc_debug_file = fopen (rf, "wb");
4264 gc_debug_file = stderr;
4269 sem_init (&suspend_ack_semaphore, 0, 0);
4271 sigfillset (&sinfo.sa_mask);
4272 sinfo.sa_flags = SA_RESTART | SA_SIGINFO;
4273 sinfo.sa_handler = suspend_handler;
4274 if (sigaction (suspend_signal_num, &sinfo, NULL) != 0) {
4275 g_error ("failed sigaction");
4278 sinfo.sa_handler = restart_handler;
4279 if (sigaction (restart_signal_num, &sinfo, NULL) != 0) {
4280 g_error ("failed sigaction");
4283 sigfillset (&suspend_signal_mask);
4284 sigdelset (&suspend_signal_mask, restart_signal_num);
4286 global_remset = alloc_remset (1024, NULL);
4287 global_remset->next = NULL;
4289 pthread_key_create (&remembered_set_key, unregister_thread);
4290 gc_initialized = TRUE;
4292 mono_gc_register_thread (&sinfo);
4295 #endif /* HAVE_SGEN_GC */