2 * sgen-gc.c: Simple generational GC.
5 * Paolo Molaro (lupus@ximian.com)
6 * Rodrigo Kumpera (kumpera@gmail.com)
8 * Copyright 2005-2011 Novell, Inc (http://www.novell.com)
9 * Copyright 2011 Xamarin Inc (http://www.xamarin.com)
11 * Thread start/stop adapted from Boehm's GC:
12 * Copyright (c) 1994 by Xerox Corporation. All rights reserved.
13 * Copyright (c) 1996 by Silicon Graphics. All rights reserved.
14 * Copyright (c) 1998 by Fergus Henderson. All rights reserved.
15 * Copyright (c) 2000-2004 by Hewlett-Packard Company. All rights reserved.
16 * Copyright 2001-2003 Ximian, Inc
17 * Copyright 2003-2010 Novell, Inc.
18 * Copyright 2011 Xamarin, Inc.
19 * Copyright (C) 2012 Xamarin Inc
21 * This library is free software; you can redistribute it and/or
22 * modify it under the terms of the GNU Library General Public
23 * License 2.0 as published by the Free Software Foundation;
25 * This library is distributed in the hope that it will be useful,
26 * but WITHOUT ANY WARRANTY; without even the implied warranty of
27 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
28 * Library General Public License for more details.
30 * You should have received a copy of the GNU Library General Public
31 * License 2.0 along with this library; if not, write to the Free
32 * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
34 * Important: allocation provides always zeroed memory, having to do
35 * a memset after allocation is deadly for performance.
36 * Memory usage at startup is currently as follows:
38 * 64 KB internal space
40 * We should provide a small memory config with half the sizes
42 * We currently try to make as few mono assumptions as possible:
43 * 1) 2-word header with no GC pointers in it (first vtable, second to store the
45 * 2) gc descriptor is the second word in the vtable (first word in the class)
46 * 3) 8 byte alignment is the minimum and enough (not true for special structures (SIMD), FIXME)
47 * 4) there is a function to get an object's size and the number of
48 * elements in an array.
49 * 5) we know the special way bounds are allocated for complex arrays
50 * 6) we know about proxies and how to treat them when domains are unloaded
52 * Always try to keep stack usage to a minimum: no recursive behaviour
53 * and no large stack allocs.
55 * General description.
56 * Objects are initially allocated in a nursery using a fast bump-pointer technique.
57 * When the nursery is full we start a nursery collection: this is performed with a
59 * When the old generation is full we start a copying GC of the old generation as well:
60 * this will be changed to mark&sweep with copying when fragmentation becomes to severe
61 * in the future. Maybe we'll even do both during the same collection like IMMIX.
63 * The things that complicate this description are:
64 * *) pinned objects: we can't move them so we need to keep track of them
65 * *) no precise info of the thread stacks and registers: we need to be able to
66 * quickly find the objects that may be referenced conservatively and pin them
67 * (this makes the first issues more important)
68 * *) large objects are too expensive to be dealt with using copying GC: we handle them
69 * with mark/sweep during major collections
70 * *) some objects need to not move even if they are small (interned strings, Type handles):
71 * we use mark/sweep for them, too: they are not allocated in the nursery, but inside
72 * PinnedChunks regions
78 *) we could have a function pointer in MonoClass to implement
79 customized write barriers for value types
81 *) investigate the stuff needed to advance a thread to a GC-safe
82 point (single-stepping, read from unmapped memory etc) and implement it.
83 This would enable us to inline allocations and write barriers, for example,
84 or at least parts of them, like the write barrier checks.
85 We may need this also for handling precise info on stacks, even simple things
86 as having uninitialized data on the stack and having to wait for the prolog
87 to zero it. Not an issue for the last frame that we scan conservatively.
88 We could always not trust the value in the slots anyway.
90 *) modify the jit to save info about references in stack locations:
91 this can be done just for locals as a start, so that at least
92 part of the stack is handled precisely.
94 *) test/fix endianess issues
96 *) Implement a card table as the write barrier instead of remembered
97 sets? Card tables are not easy to implement with our current
98 memory layout. We have several different kinds of major heap
99 objects: Small objects in regular blocks, small objects in pinned
100 chunks and LOS objects. If we just have a pointer we have no way
101 to tell which kind of object it points into, therefore we cannot
102 know where its card table is. The least we have to do to make
103 this happen is to get rid of write barriers for indirect stores.
106 *) Get rid of write barriers for indirect stores. We can do this by
107 telling the GC to wbarrier-register an object once we do an ldloca
108 or ldelema on it, and to unregister it once it's not used anymore
109 (it can only travel downwards on the stack). The problem with
110 unregistering is that it needs to happen eventually no matter
111 what, even if exceptions are thrown, the thread aborts, etc.
112 Rodrigo suggested that we could do only the registering part and
113 let the collector find out (pessimistically) when it's safe to
114 unregister, namely when the stack pointer of the thread that
115 registered the object is higher than it was when the registering
116 happened. This might make for a good first implementation to get
117 some data on performance.
119 *) Some sort of blacklist support? Blacklists is a concept from the
120 Boehm GC: if during a conservative scan we find pointers to an
121 area which we might use as heap, we mark that area as unusable, so
122 pointer retention by random pinning pointers is reduced.
124 *) experiment with max small object size (very small right now - 2kb,
125 because it's tied to the max freelist size)
127 *) add an option to mmap the whole heap in one chunk: it makes for many
128 simplifications in the checks (put the nursery at the top and just use a single
129 check for inclusion/exclusion): the issue this has is that on 32 bit systems it's
130 not flexible (too much of the address space may be used by default or we can't
131 increase the heap as needed) and we'd need a race-free mechanism to return memory
132 back to the system (mprotect(PROT_NONE) will still keep the memory allocated if it
133 was written to, munmap is needed, but the following mmap may not find the same segment
136 *) memzero the major fragments after restarting the world and optionally a smaller
139 *) investigate having fragment zeroing threads
141 *) separate locks for finalization and other minor stuff to reduce
144 *) try a different copying order to improve memory locality
146 *) a thread abort after a store but before the write barrier will
147 prevent the write barrier from executing
149 *) specialized dynamically generated markers/copiers
151 *) Dynamically adjust TLAB size to the number of threads. If we have
152 too many threads that do allocation, we might need smaller TLABs,
153 and we might get better performance with larger TLABs if we only
154 have a handful of threads. We could sum up the space left in all
155 assigned TLABs and if that's more than some percentage of the
156 nursery size, reduce the TLAB size.
158 *) Explore placing unreachable objects on unused nursery memory.
159 Instead of memset'ng a region to zero, place an int[] covering it.
160 A good place to start is add_nursery_frag. The tricky thing here is
161 placing those objects atomically outside of a collection.
163 *) Allocation should use asymmetric Dekker synchronization:
164 http://blogs.oracle.com/dave/resource/Asymmetric-Dekker-Synchronization.txt
165 This should help weak consistency archs.
172 #define _XOPEN_SOURCE
173 #define _DARWIN_C_SOURCE
179 #ifdef HAVE_PTHREAD_H
182 #ifdef HAVE_PTHREAD_NP_H
183 #include <pthread_np.h>
185 #ifdef HAVE_SEMAPHORE_H
186 #include <semaphore.h>
194 #include "metadata/sgen-gc.h"
195 #include "metadata/metadata-internals.h"
196 #include "metadata/class-internals.h"
197 #include "metadata/gc-internal.h"
198 #include "metadata/object-internals.h"
199 #include "metadata/threads.h"
200 #include "metadata/sgen-cardtable.h"
201 #include "metadata/sgen-protocol.h"
202 #include "metadata/sgen-archdep.h"
203 #include "metadata/sgen-bridge.h"
204 #include "metadata/sgen-memory-governor.h"
205 #include "metadata/sgen-hash-table.h"
206 #include "metadata/mono-gc.h"
207 #include "metadata/method-builder.h"
208 #include "metadata/profiler-private.h"
209 #include "metadata/monitor.h"
210 #include "metadata/mempool-internals.h"
211 #include "metadata/marshal.h"
212 #include "metadata/runtime.h"
213 #include "metadata/sgen-cardtable.h"
214 #include "metadata/sgen-pinning.h"
215 #include "metadata/sgen-workers.h"
216 #include "metadata/sgen-layout-stats.h"
217 #include "utils/mono-mmap.h"
218 #include "utils/mono-time.h"
219 #include "utils/mono-semaphore.h"
220 #include "utils/mono-counters.h"
221 #include "utils/mono-proclib.h"
222 #include "utils/mono-memory-model.h"
223 #include "utils/mono-logger-internal.h"
224 #include "utils/dtrace.h"
226 #include <mono/utils/mono-logger-internal.h>
227 #include <mono/utils/memcheck.h>
229 #if defined(__MACH__)
230 #include "utils/mach-support.h"
233 #define OPDEF(a,b,c,d,e,f,g,h,i,j) \
237 #include "mono/cil/opcode.def"
243 #undef pthread_create
245 #undef pthread_detach
248 * ######################################################################
249 * ######## Types and constants used by the GC.
250 * ######################################################################
253 /* 0 means not initialized, 1 is initialized, -1 means in progress */
254 static int gc_initialized = 0;
255 /* If set, check if we need to do something every X allocations */
256 gboolean has_per_allocation_action;
257 /* If set, do a heap check every X allocation */
258 guint32 verify_before_allocs = 0;
259 /* If set, do a minor collection before every X allocation */
260 guint32 collect_before_allocs = 0;
261 /* If set, do a whole heap check before each collection */
262 static gboolean whole_heap_check_before_collection = FALSE;
263 /* If set, do a heap consistency check before each minor collection */
264 static gboolean consistency_check_at_minor_collection = FALSE;
265 /* If set, do a mod union consistency check before each finishing collection pause */
266 static gboolean mod_union_consistency_check = FALSE;
267 /* If set, check whether mark bits are consistent after major collections */
268 static gboolean check_mark_bits_after_major_collection = FALSE;
269 /* If set, check that all nursery objects are pinned/not pinned, depending on context */
270 static gboolean check_nursery_objects_pinned = FALSE;
271 /* If set, do a few checks when the concurrent collector is used */
272 static gboolean do_concurrent_checks = FALSE;
273 /* If set, check that there are no references to the domain left at domain unload */
274 static gboolean xdomain_checks = FALSE;
275 /* If not null, dump the heap after each collection into this file */
276 static FILE *heap_dump_file = NULL;
277 /* If set, mark stacks conservatively, even if precise marking is possible */
278 static gboolean conservative_stack_mark = FALSE;
279 /* If set, do a plausibility check on the scan_starts before and after
281 static gboolean do_scan_starts_check = FALSE;
283 * If the major collector is concurrent and this is FALSE, we will
284 * never initiate a synchronous major collection, unless requested via
287 static gboolean allow_synchronous_major = TRUE;
288 static gboolean disable_minor_collections = FALSE;
289 static gboolean disable_major_collections = FALSE;
290 gboolean do_pin_stats = FALSE;
291 static gboolean do_verify_nursery = FALSE;
292 static gboolean do_dump_nursery_content = FALSE;
293 static gboolean enable_nursery_canaries = FALSE;
295 #ifdef HEAVY_STATISTICS
296 guint64 stat_objects_alloced_degraded = 0;
297 guint64 stat_bytes_alloced_degraded = 0;
299 guint64 stat_copy_object_called_nursery = 0;
300 guint64 stat_objects_copied_nursery = 0;
301 guint64 stat_copy_object_called_major = 0;
302 guint64 stat_objects_copied_major = 0;
304 guint64 stat_scan_object_called_nursery = 0;
305 guint64 stat_scan_object_called_major = 0;
307 guint64 stat_slots_allocated_in_vain;
309 guint64 stat_nursery_copy_object_failed_from_space = 0;
310 guint64 stat_nursery_copy_object_failed_forwarded = 0;
311 guint64 stat_nursery_copy_object_failed_pinned = 0;
312 guint64 stat_nursery_copy_object_failed_to_space = 0;
314 static int stat_wbarrier_add_to_global_remset = 0;
315 static int stat_wbarrier_set_field = 0;
316 static int stat_wbarrier_set_arrayref = 0;
317 static int stat_wbarrier_arrayref_copy = 0;
318 static int stat_wbarrier_generic_store = 0;
319 static int stat_wbarrier_generic_store_atomic = 0;
320 static int stat_wbarrier_set_root = 0;
321 static int stat_wbarrier_value_copy = 0;
322 static int stat_wbarrier_object_copy = 0;
325 static guint64 stat_pinned_objects = 0;
327 static guint64 time_minor_pre_collection_fragment_clear = 0;
328 static guint64 time_minor_pinning = 0;
329 static guint64 time_minor_scan_remsets = 0;
330 static guint64 time_minor_scan_pinned = 0;
331 static guint64 time_minor_scan_registered_roots = 0;
332 static guint64 time_minor_scan_thread_data = 0;
333 static guint64 time_minor_finish_gray_stack = 0;
334 static guint64 time_minor_fragment_creation = 0;
336 static guint64 time_major_pre_collection_fragment_clear = 0;
337 static guint64 time_major_pinning = 0;
338 static guint64 time_major_scan_pinned = 0;
339 static guint64 time_major_scan_registered_roots = 0;
340 static guint64 time_major_scan_thread_data = 0;
341 static guint64 time_major_scan_alloc_pinned = 0;
342 static guint64 time_major_scan_finalized = 0;
343 static guint64 time_major_scan_big_objects = 0;
344 static guint64 time_major_finish_gray_stack = 0;
345 static guint64 time_major_free_bigobjs = 0;
346 static guint64 time_major_los_sweep = 0;
347 static guint64 time_major_sweep = 0;
348 static guint64 time_major_fragment_creation = 0;
350 static guint64 time_max = 0;
352 static SGEN_TV_DECLARE (time_major_conc_collection_start);
353 static SGEN_TV_DECLARE (time_major_conc_collection_end);
355 static SGEN_TV_DECLARE (last_minor_collection_start_tv);
356 static SGEN_TV_DECLARE (last_minor_collection_end_tv);
358 int gc_debug_level = 0;
361 static MonoGCFinalizerCallbacks fin_callbacks;
365 mono_gc_flush_info (void)
367 fflush (gc_debug_file);
371 #define TV_DECLARE SGEN_TV_DECLARE
372 #define TV_GETTIME SGEN_TV_GETTIME
373 #define TV_ELAPSED SGEN_TV_ELAPSED
375 SGEN_TV_DECLARE (sgen_init_timestamp);
377 #define ALIGN_TO(val,align) ((((guint64)val) + ((align) - 1)) & ~((align) - 1))
379 NurseryClearPolicy nursery_clear_policy = CLEAR_AT_TLAB_CREATION;
381 #define object_is_forwarded SGEN_OBJECT_IS_FORWARDED
382 #define object_is_pinned SGEN_OBJECT_IS_PINNED
383 #define pin_object SGEN_PIN_OBJECT
385 #define ptr_in_nursery sgen_ptr_in_nursery
387 #define LOAD_VTABLE SGEN_LOAD_VTABLE
390 safe_name (void* obj)
392 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (obj);
393 return vt->klass->name;
397 nursery_canaries_enabled (void)
399 return enable_nursery_canaries;
402 #define safe_object_get_size sgen_safe_object_get_size
405 sgen_safe_name (void* obj)
407 return safe_name (obj);
411 * ######################################################################
412 * ######## Global data.
413 * ######################################################################
415 LOCK_DECLARE (gc_mutex);
416 gboolean sgen_try_free_some_memory;
418 #define SCAN_START_SIZE SGEN_SCAN_START_SIZE
420 static mword pagesize = 4096;
421 size_t degraded_mode = 0;
423 static mword bytes_pinned_from_failed_allocation = 0;
425 GCMemSection *nursery_section = NULL;
426 static volatile mword lowest_heap_address = ~(mword)0;
427 static volatile mword highest_heap_address = 0;
429 LOCK_DECLARE (sgen_interruption_mutex);
431 typedef struct _FinalizeReadyEntry FinalizeReadyEntry;
432 struct _FinalizeReadyEntry {
433 FinalizeReadyEntry *next;
437 typedef struct _EphemeronLinkNode EphemeronLinkNode;
439 struct _EphemeronLinkNode {
440 EphemeronLinkNode *next;
449 int current_collection_generation = -1;
450 volatile gboolean concurrent_collection_in_progress = FALSE;
452 /* objects that are ready to be finalized */
453 static FinalizeReadyEntry *fin_ready_list = NULL;
454 static FinalizeReadyEntry *critical_fin_list = NULL;
456 static EphemeronLinkNode *ephemeron_list;
458 /* registered roots: the key to the hash is the root start address */
460 * Different kinds of roots are kept separate to speed up pin_from_roots () for example.
462 SgenHashTable roots_hash [ROOT_TYPE_NUM] = {
463 SGEN_HASH_TABLE_INIT (INTERNAL_MEM_ROOTS_TABLE, INTERNAL_MEM_ROOT_RECORD, sizeof (RootRecord), mono_aligned_addr_hash, NULL),
464 SGEN_HASH_TABLE_INIT (INTERNAL_MEM_ROOTS_TABLE, INTERNAL_MEM_ROOT_RECORD, sizeof (RootRecord), mono_aligned_addr_hash, NULL),
465 SGEN_HASH_TABLE_INIT (INTERNAL_MEM_ROOTS_TABLE, INTERNAL_MEM_ROOT_RECORD, sizeof (RootRecord), mono_aligned_addr_hash, NULL)
467 static mword roots_size = 0; /* amount of memory in the root set */
469 #define GC_ROOT_NUM 32
471 int count; /* must be the first field */
472 void *objects [GC_ROOT_NUM];
473 int root_types [GC_ROOT_NUM];
474 uintptr_t extra_info [GC_ROOT_NUM];
478 notify_gc_roots (GCRootReport *report)
482 mono_profiler_gc_roots (report->count, report->objects, report->root_types, report->extra_info);
487 add_profile_gc_root (GCRootReport *report, void *object, int rtype, uintptr_t extra_info)
489 if (report->count == GC_ROOT_NUM)
490 notify_gc_roots (report);
491 report->objects [report->count] = object;
492 report->root_types [report->count] = rtype;
493 report->extra_info [report->count++] = (uintptr_t)((MonoVTable*)LOAD_VTABLE (object))->klass;
496 MonoNativeTlsKey thread_info_key;
498 #ifdef HAVE_KW_THREAD
499 __thread SgenThreadInfo *sgen_thread_info;
500 __thread char *stack_end;
503 /* The size of a TLAB */
504 /* The bigger the value, the less often we have to go to the slow path to allocate a new
505 * one, but the more space is wasted by threads not allocating much memory.
507 * FIXME: Make this self-tuning for each thread.
509 guint32 tlab_size = (1024 * 4);
511 #define MAX_SMALL_OBJ_SIZE SGEN_MAX_SMALL_OBJ_SIZE
513 /* Functions supplied by the runtime to be called by the GC */
514 static MonoGCCallbacks gc_callbacks;
516 #define ALLOC_ALIGN SGEN_ALLOC_ALIGN
518 #define ALIGN_UP SGEN_ALIGN_UP
520 #define MOVED_OBJECTS_NUM 64
521 static void *moved_objects [MOVED_OBJECTS_NUM];
522 static int moved_objects_idx = 0;
524 /* Vtable of the objects used to fill out nursery fragments before a collection */
525 static MonoVTable *array_fill_vtable;
527 #ifdef SGEN_DEBUG_INTERNAL_ALLOC
528 MonoNativeThreadId main_gc_thread = NULL;
531 /*Object was pinned during the current collection*/
532 static mword objects_pinned;
535 * ######################################################################
536 * ######## Macros and function declarations.
537 * ######################################################################
541 align_pointer (void *ptr)
543 mword p = (mword)ptr;
544 p += sizeof (gpointer) - 1;
545 p &= ~ (sizeof (gpointer) - 1);
549 typedef SgenGrayQueue GrayQueue;
551 /* forward declarations */
552 static void scan_thread_data (void *start_nursery, void *end_nursery, gboolean precise, GrayQueue *queue);
553 static void scan_from_registered_roots (char *addr_start, char *addr_end, int root_type, ScanCopyContext ctx);
554 static void scan_finalizer_entries (FinalizeReadyEntry *list, ScanCopyContext ctx);
555 static void report_finalizer_roots (void);
556 static void report_registered_roots (void);
558 static void pin_from_roots (void *start_nursery, void *end_nursery, GrayQueue *queue);
559 static void finish_gray_stack (int generation, GrayQueue *queue);
561 void mono_gc_scan_for_specific_ref (MonoObject *key, gboolean precise);
564 static void init_stats (void);
566 static int mark_ephemerons_in_range (ScanCopyContext ctx);
567 static void clear_unreachable_ephemerons (ScanCopyContext ctx);
568 static void null_ephemerons_for_domain (MonoDomain *domain);
570 static gboolean major_update_or_finish_concurrent_collection (gboolean force_finish);
572 SgenObjectOperations current_object_ops;
573 SgenMajorCollector major_collector;
574 SgenMinorCollector sgen_minor_collector;
575 static GrayQueue gray_queue;
577 static SgenRemeberedSet remset;
579 /* The gray queue to use from the main collection thread. */
580 #define WORKERS_DISTRIBUTE_GRAY_QUEUE (&gray_queue)
583 * The gray queue a worker job must use. If we're not parallel or
584 * concurrent, we use the main gray queue.
586 static SgenGrayQueue*
587 sgen_workers_get_job_gray_queue (WorkerData *worker_data)
589 return worker_data ? &worker_data->private_gray_queue : WORKERS_DISTRIBUTE_GRAY_QUEUE;
593 gray_queue_redirect (SgenGrayQueue *queue)
595 gboolean wake = FALSE;
598 GrayQueueSection *section = sgen_gray_object_dequeue_section (queue);
601 sgen_section_gray_queue_enqueue (queue->alloc_prepare_data, section);
606 g_assert (concurrent_collection_in_progress);
607 if (sgen_workers_have_started ()) {
608 sgen_workers_ensure_awake ();
610 if (concurrent_collection_in_progress)
611 g_assert (current_collection_generation == -1);
617 gray_queue_enable_redirect (SgenGrayQueue *queue)
619 if (!concurrent_collection_in_progress)
622 sgen_gray_queue_set_alloc_prepare (queue, gray_queue_redirect, sgen_workers_get_distribute_section_gray_queue ());
623 gray_queue_redirect (queue);
627 sgen_scan_area_with_callback (char *start, char *end, IterateObjectCallbackFunc callback, void *data, gboolean allow_flags)
629 while (start < end) {
633 if (!*(void**)start) {
634 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
639 if (!(obj = SGEN_OBJECT_IS_FORWARDED (start)))
645 if ((MonoVTable*)SGEN_LOAD_VTABLE (obj) != array_fill_vtable) {
646 CHECK_CANARY_FOR_OBJECT (obj);
647 size = ALIGN_UP (safe_object_get_size ((MonoObject*)obj));
648 callback (obj, size, data);
649 CANARIFY_SIZE (size);
651 size = ALIGN_UP (safe_object_get_size ((MonoObject*)obj));
659 need_remove_object_for_domain (char *start, MonoDomain *domain)
661 if (mono_object_domain (start) == domain) {
662 SGEN_LOG (4, "Need to cleanup object %p", start);
663 binary_protocol_cleanup (start, (gpointer)LOAD_VTABLE (start), safe_object_get_size ((MonoObject*)start));
670 process_object_for_domain_clearing (char *start, MonoDomain *domain)
672 GCVTable *vt = (GCVTable*)LOAD_VTABLE (start);
673 if (vt->klass == mono_defaults.internal_thread_class)
674 g_assert (mono_object_domain (start) == mono_get_root_domain ());
675 /* The object could be a proxy for an object in the domain
677 #ifndef DISABLE_REMOTING
678 if (mono_defaults.real_proxy_class->supertypes && mono_class_has_parent_fast (vt->klass, mono_defaults.real_proxy_class)) {
679 MonoObject *server = ((MonoRealProxy*)start)->unwrapped_server;
681 /* The server could already have been zeroed out, so
682 we need to check for that, too. */
683 if (server && (!LOAD_VTABLE (server) || mono_object_domain (server) == domain)) {
684 SGEN_LOG (4, "Cleaning up remote pointer in %p to object %p", start, server);
685 ((MonoRealProxy*)start)->unwrapped_server = NULL;
692 clear_domain_process_object (char *obj, MonoDomain *domain)
696 process_object_for_domain_clearing (obj, domain);
697 remove = need_remove_object_for_domain (obj, domain);
699 if (remove && ((MonoObject*)obj)->synchronisation) {
700 void **dislink = mono_monitor_get_object_monitor_weak_link ((MonoObject*)obj);
702 sgen_register_disappearing_link (NULL, dislink, FALSE, TRUE);
709 clear_domain_process_minor_object_callback (char *obj, size_t size, MonoDomain *domain)
711 if (clear_domain_process_object (obj, domain)) {
712 CANARIFY_SIZE (size);
713 memset (obj, 0, size);
718 clear_domain_process_major_object_callback (char *obj, size_t size, MonoDomain *domain)
720 clear_domain_process_object (obj, domain);
724 clear_domain_free_major_non_pinned_object_callback (char *obj, size_t size, MonoDomain *domain)
726 if (need_remove_object_for_domain (obj, domain))
727 major_collector.free_non_pinned_object (obj, size);
731 clear_domain_free_major_pinned_object_callback (char *obj, size_t size, MonoDomain *domain)
733 if (need_remove_object_for_domain (obj, domain))
734 major_collector.free_pinned_object (obj, size);
738 * When appdomains are unloaded we can easily remove objects that have finalizers,
739 * but all the others could still be present in random places on the heap.
740 * We need a sweep to get rid of them even though it's going to be costly
742 * The reason we need to remove them is because we access the vtable and class
743 * structures to know the object size and the reference bitmap: once the domain is
744 * unloaded the point to random memory.
747 mono_gc_clear_domain (MonoDomain * domain)
749 LOSObject *bigobj, *prev;
754 binary_protocol_domain_unload_begin (domain);
758 if (concurrent_collection_in_progress)
759 sgen_perform_collection (0, GENERATION_OLD, "clear domain", TRUE);
760 g_assert (!concurrent_collection_in_progress);
762 sgen_process_fin_stage_entries ();
763 sgen_process_dislink_stage_entries ();
765 sgen_clear_nursery_fragments ();
767 if (xdomain_checks && domain != mono_get_root_domain ()) {
768 sgen_scan_for_registered_roots_in_domain (domain, ROOT_TYPE_NORMAL);
769 sgen_scan_for_registered_roots_in_domain (domain, ROOT_TYPE_WBARRIER);
770 sgen_check_for_xdomain_refs ();
773 /*Ephemerons and dislinks must be processed before LOS since they might end up pointing
774 to memory returned to the OS.*/
775 null_ephemerons_for_domain (domain);
777 for (i = GENERATION_NURSERY; i < GENERATION_MAX; ++i)
778 sgen_null_links_for_domain (domain, i);
780 for (i = GENERATION_NURSERY; i < GENERATION_MAX; ++i)
781 sgen_remove_finalizers_for_domain (domain, i);
783 sgen_scan_area_with_callback (nursery_section->data, nursery_section->end_data,
784 (IterateObjectCallbackFunc)clear_domain_process_minor_object_callback, domain, FALSE);
786 /* We need two passes over major and large objects because
787 freeing such objects might give their memory back to the OS
788 (in the case of large objects) or obliterate its vtable
789 (pinned objects with major-copying or pinned and non-pinned
790 objects with major-mark&sweep), but we might need to
791 dereference a pointer from an object to another object if
792 the first object is a proxy. */
793 major_collector.iterate_objects (ITERATE_OBJECTS_SWEEP_ALL, (IterateObjectCallbackFunc)clear_domain_process_major_object_callback, domain);
794 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
795 clear_domain_process_object (bigobj->data, domain);
798 for (bigobj = los_object_list; bigobj;) {
799 if (need_remove_object_for_domain (bigobj->data, domain)) {
800 LOSObject *to_free = bigobj;
802 prev->next = bigobj->next;
804 los_object_list = bigobj->next;
805 bigobj = bigobj->next;
806 SGEN_LOG (4, "Freeing large object %p", bigobj->data);
807 sgen_los_free_object (to_free);
811 bigobj = bigobj->next;
813 major_collector.iterate_objects (ITERATE_OBJECTS_SWEEP_NON_PINNED, (IterateObjectCallbackFunc)clear_domain_free_major_non_pinned_object_callback, domain);
814 major_collector.iterate_objects (ITERATE_OBJECTS_SWEEP_PINNED, (IterateObjectCallbackFunc)clear_domain_free_major_pinned_object_callback, domain);
816 if (domain == mono_get_root_domain ()) {
817 if (G_UNLIKELY (do_pin_stats))
818 sgen_pin_stats_print_class_stats ();
819 sgen_object_layout_dump (stdout);
822 sgen_restart_world (0, NULL);
824 binary_protocol_domain_unload_end (domain);
825 binary_protocol_flush_buffers (FALSE);
831 * sgen_add_to_global_remset:
833 * The global remset contains locations which point into newspace after
834 * a minor collection. This can happen if the objects they point to are pinned.
836 * LOCKING: If called from a parallel collector, the global remset
837 * lock must be held. For serial collectors that is not necessary.
840 sgen_add_to_global_remset (gpointer ptr, gpointer obj)
842 SGEN_ASSERT (5, sgen_ptr_in_nursery (obj), "Target pointer of global remset must be in the nursery");
844 HEAVY_STAT (++stat_wbarrier_add_to_global_remset);
846 if (!major_collector.is_concurrent) {
847 SGEN_ASSERT (5, current_collection_generation != -1, "Global remsets can only be added during collections");
849 if (current_collection_generation == -1)
850 SGEN_ASSERT (5, sgen_concurrent_collection_in_progress (), "Global remsets outside of collection pauses can only be added by the concurrent collector");
853 if (!object_is_pinned (obj))
854 SGEN_ASSERT (5, sgen_minor_collector.is_split || sgen_concurrent_collection_in_progress (), "Non-pinned objects can only remain in nursery if it is a split nursery");
855 else if (sgen_cement_lookup_or_register (obj))
858 remset.record_pointer (ptr);
860 if (G_UNLIKELY (do_pin_stats))
861 sgen_pin_stats_register_global_remset (obj);
863 SGEN_LOG (8, "Adding global remset for %p", ptr);
864 binary_protocol_global_remset (ptr, obj, (gpointer)SGEN_LOAD_VTABLE (obj));
868 if (G_UNLIKELY (MONO_GC_GLOBAL_REMSET_ADD_ENABLED ())) {
869 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (obj);
870 MONO_GC_GLOBAL_REMSET_ADD ((mword)ptr, (mword)obj, sgen_safe_object_get_size (obj),
871 vt->klass->name_space, vt->klass->name);
877 * sgen_drain_gray_stack:
879 * Scan objects in the gray stack until the stack is empty. This should be called
880 * frequently after each object is copied, to achieve better locality and cache
883 * max_objs is the maximum number of objects to scan, or -1 to scan until the stack is
887 sgen_drain_gray_stack (int max_objs, ScanCopyContext ctx)
889 ScanObjectFunc scan_func = ctx.scan_func;
890 GrayQueue *queue = ctx.queue;
892 if (current_collection_generation == GENERATION_OLD && major_collector.drain_gray_stack)
893 return major_collector.drain_gray_stack (ctx);
897 for (i = 0; i != max_objs; ++i) {
900 GRAY_OBJECT_DEQUEUE (queue, &obj, &desc);
903 SGEN_LOG (9, "Precise gray object scan %p (%s)", obj, safe_name (obj));
904 scan_func (obj, desc, queue);
906 } while (max_objs < 0);
911 * Addresses in the pin queue are already sorted. This function finds
912 * the object header for each address and pins the object. The
913 * addresses must be inside the nursery section. The (start of the)
914 * address array is overwritten with the addresses of the actually
915 * pinned objects. Return the number of pinned objects.
918 pin_objects_from_nursery_pin_queue (ScanCopyContext ctx)
920 GCMemSection *section = nursery_section;
921 void **start = sgen_pinning_get_entry (section->pin_queue_first_entry);
922 void **end = sgen_pinning_get_entry (section->pin_queue_last_entry);
923 void *start_nursery = section->data;
924 void *end_nursery = section->next_data;
929 void *pinning_front = start_nursery;
931 void **definitely_pinned = start;
932 ScanObjectFunc scan_func = ctx.scan_func;
933 SgenGrayQueue *queue = ctx.queue;
935 sgen_nursery_allocator_prepare_for_pinning ();
937 while (start < end) {
938 void *obj_to_pin = NULL;
939 size_t obj_to_pin_size = 0;
944 SGEN_ASSERT (0, addr >= start_nursery && addr < end_nursery, "Potential pinning address out of range");
945 SGEN_ASSERT (0, addr >= last, "Pin queue not sorted");
952 SGEN_LOG (5, "Considering pinning addr %p", addr);
953 /* We've already processed everything up to pinning_front. */
954 if (addr < pinning_front) {
960 * Find the closest scan start <= addr. We might search backward in the
961 * scan_starts array because entries might be NULL. In the worst case we
962 * start at start_nursery.
964 idx = ((char*)addr - (char*)section->data) / SCAN_START_SIZE;
965 SGEN_ASSERT (0, idx < section->num_scan_start, "Scan start index out of range");
966 search_start = (void*)section->scan_starts [idx];
967 if (!search_start || search_start > addr) {
970 search_start = section->scan_starts [idx];
971 if (search_start && search_start <= addr)
974 if (!search_start || search_start > addr)
975 search_start = start_nursery;
979 * If the pinning front is closer than the scan start we found, start
980 * searching at the front.
982 if (search_start < pinning_front)
983 search_start = pinning_front;
986 * Now addr should be in an object a short distance from search_start.
988 * search_start must point to zeroed mem or point to an object.
994 if (!*(void**)search_start) {
995 search_start = (void*)ALIGN_UP ((mword)search_start + sizeof (gpointer));
996 /* The loop condition makes sure we don't overrun addr. */
1000 obj_size = ALIGN_UP (safe_object_get_size ((MonoObject*)search_start));
1002 if (addr >= search_start && (char*)addr < (char*)search_start + obj_size) {
1003 /* This is the object we're looking for. */
1004 obj_to_pin = search_start;
1005 obj_to_pin_size = obj_size;
1009 /* Skip to the next object */
1010 if (((MonoObject*)search_start)->synchronisation != GINT_TO_POINTER (-1)) {
1011 CHECK_CANARY_FOR_OBJECT (search_start);
1012 CANARIFY_SIZE (obj_size);
1013 CANARIFY_SIZE (obj_to_pin_size);
1015 search_start = (void*)((char*)search_start + obj_size);
1016 } while (search_start <= addr);
1018 /* We've searched past the address we were looking for. */
1020 pinning_front = search_start;
1021 goto next_pin_queue_entry;
1025 * We've found an object to pin. It might still be a dummy array, but we
1026 * can advance the pinning front in any case.
1028 pinning_front = (char*)obj_to_pin + obj_to_pin_size;
1031 * If this is a dummy array marking the beginning of a nursery
1032 * fragment, we don't pin it.
1034 if (((MonoObject*)obj_to_pin)->synchronisation == GINT_TO_POINTER (-1))
1035 goto next_pin_queue_entry;
1038 * Finally - pin the object!
1040 desc = sgen_obj_get_descriptor_safe (obj_to_pin);
1042 scan_func (obj_to_pin, desc, queue);
1044 SGEN_LOG (4, "Pinned object %p, vtable %p (%s), count %d\n",
1045 obj_to_pin, *(void**)obj_to_pin, safe_name (obj_to_pin), count);
1046 binary_protocol_pin (obj_to_pin,
1047 (gpointer)LOAD_VTABLE (obj_to_pin),
1048 safe_object_get_size (obj_to_pin));
1050 #ifdef ENABLE_DTRACE
1051 if (G_UNLIKELY (MONO_GC_OBJ_PINNED_ENABLED ())) {
1052 int gen = sgen_ptr_in_nursery (obj_to_pin) ? GENERATION_NURSERY : GENERATION_OLD;
1053 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (obj_to_pin);
1054 MONO_GC_OBJ_PINNED ((mword)obj_to_pin,
1055 sgen_safe_object_get_size (obj_to_pin),
1056 vt->klass->name_space, vt->klass->name, gen);
1060 pin_object (obj_to_pin);
1061 GRAY_OBJECT_ENQUEUE (queue, obj_to_pin, desc);
1062 if (G_UNLIKELY (do_pin_stats))
1063 sgen_pin_stats_register_object (obj_to_pin, obj_to_pin_size);
1064 definitely_pinned [count] = obj_to_pin;
1068 next_pin_queue_entry:
1072 //printf ("effective pinned: %d (at the end: %d)\n", count, (char*)end_nursery - (char*)last);
1073 if (mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS) {
1074 GCRootReport report;
1076 for (idx = 0; idx < count; ++idx)
1077 add_profile_gc_root (&report, definitely_pinned [idx], MONO_PROFILE_GC_ROOT_PINNING | MONO_PROFILE_GC_ROOT_MISC, 0);
1078 notify_gc_roots (&report);
1080 stat_pinned_objects += count;
1085 pin_objects_in_nursery (ScanCopyContext ctx)
1089 if (nursery_section->pin_queue_first_entry == nursery_section->pin_queue_last_entry)
1092 reduced_to = pin_objects_from_nursery_pin_queue (ctx);
1093 nursery_section->pin_queue_last_entry = nursery_section->pin_queue_first_entry + reduced_to;
1098 sgen_pin_object (void *object, GrayQueue *queue)
1100 SGEN_PIN_OBJECT (object);
1101 sgen_pin_stage_ptr (object);
1103 if (G_UNLIKELY (do_pin_stats))
1104 sgen_pin_stats_register_object (object, safe_object_get_size (object));
1106 GRAY_OBJECT_ENQUEUE (queue, object, sgen_obj_get_descriptor_safe (object));
1107 binary_protocol_pin (object, (gpointer)LOAD_VTABLE (object), safe_object_get_size (object));
1109 #ifdef ENABLE_DTRACE
1110 if (G_UNLIKELY (MONO_GC_OBJ_PINNED_ENABLED ())) {
1111 int gen = sgen_ptr_in_nursery (object) ? GENERATION_NURSERY : GENERATION_OLD;
1112 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (object);
1113 MONO_GC_OBJ_PINNED ((mword)object, sgen_safe_object_get_size (object), vt->klass->name_space, vt->klass->name, gen);
1119 sgen_parallel_pin_or_update (void **ptr, void *obj, MonoVTable *vt, SgenGrayQueue *queue)
1123 gboolean major_pinned = FALSE;
1125 if (sgen_ptr_in_nursery (obj)) {
1126 if (SGEN_CAS_PTR (obj, SGEN_POINTER_TAG_PINNED (vt), vt) == vt) {
1127 sgen_pin_object (obj, queue);
1131 major_collector.pin_major_object (obj, queue);
1132 major_pinned = TRUE;
1135 vtable_word = *(mword*)obj;
1136 /*someone else forwarded it, update the pointer and bail out*/
1137 if (SGEN_POINTER_IS_TAGGED_FORWARDED (vtable_word)) {
1138 *ptr = SGEN_POINTER_UNTAG_VTABLE (vtable_word);
1142 /*someone pinned it, nothing to do.*/
1143 if (SGEN_POINTER_IS_TAGGED_PINNED (vtable_word) || major_pinned)
1148 /* Sort the addresses in array in increasing order.
1149 * Done using a by-the book heap sort. Which has decent and stable performance, is pretty cache efficient.
1152 sgen_sort_addresses (void **array, size_t size)
1157 for (i = 1; i < size; ++i) {
1160 size_t parent = (child - 1) / 2;
1162 if (array [parent] >= array [child])
1165 tmp = array [parent];
1166 array [parent] = array [child];
1167 array [child] = tmp;
1173 for (i = size - 1; i > 0; --i) {
1176 array [i] = array [0];
1182 while (root * 2 + 1 <= end) {
1183 size_t child = root * 2 + 1;
1185 if (child < end && array [child] < array [child + 1])
1187 if (array [root] >= array [child])
1191 array [root] = array [child];
1192 array [child] = tmp;
1200 * Scan the memory between start and end and queue values which could be pointers
1201 * to the area between start_nursery and end_nursery for later consideration.
1202 * Typically used for thread stacks.
1205 conservatively_pin_objects_from (void **start, void **end, void *start_nursery, void *end_nursery, int pin_type)
1209 #ifdef VALGRIND_MAKE_MEM_DEFINED_IF_ADDRESSABLE
1210 VALGRIND_MAKE_MEM_DEFINED_IF_ADDRESSABLE (start, (char*)end - (char*)start);
1213 while (start < end) {
1214 if (*start >= start_nursery && *start < end_nursery) {
1216 * *start can point to the middle of an object
1217 * note: should we handle pointing at the end of an object?
1218 * pinning in C# code disallows pointing at the end of an object
1219 * but there is some small chance that an optimizing C compiler
1220 * may keep the only reference to an object by pointing
1221 * at the end of it. We ignore this small chance for now.
1222 * Pointers to the end of an object are indistinguishable
1223 * from pointers to the start of the next object in memory
1224 * so if we allow that we'd need to pin two objects...
1225 * We queue the pointer in an array, the
1226 * array will then be sorted and uniqued. This way
1227 * we can coalesce several pinning pointers and it should
1228 * be faster since we'd do a memory scan with increasing
1229 * addresses. Note: we can align the address to the allocation
1230 * alignment, so the unique process is more effective.
1232 mword addr = (mword)*start;
1233 addr &= ~(ALLOC_ALIGN - 1);
1234 if (addr >= (mword)start_nursery && addr < (mword)end_nursery) {
1235 SGEN_LOG (6, "Pinning address %p from %p", (void*)addr, start);
1236 sgen_pin_stage_ptr ((void*)addr);
1237 binary_protocol_pin_stage (start, (void*)addr);
1240 if (G_UNLIKELY (do_pin_stats)) {
1241 if (ptr_in_nursery ((void*)addr))
1242 sgen_pin_stats_register_address ((char*)addr, pin_type);
1248 SGEN_LOG (7, "found %d potential pinned heap pointers", count);
1252 * The first thing we do in a collection is to identify pinned objects.
1253 * This function considers all the areas of memory that need to be
1254 * conservatively scanned.
1257 pin_from_roots (void *start_nursery, void *end_nursery, GrayQueue *queue)
1261 SGEN_LOG (2, "Scanning pinned roots (%d bytes, %d/%d entries)", (int)roots_size, roots_hash [ROOT_TYPE_NORMAL].num_entries, roots_hash [ROOT_TYPE_PINNED].num_entries);
1262 /* objects pinned from the API are inside these roots */
1263 SGEN_HASH_TABLE_FOREACH (&roots_hash [ROOT_TYPE_PINNED], start_root, root) {
1264 SGEN_LOG (6, "Pinned roots %p-%p", start_root, root->end_root);
1265 conservatively_pin_objects_from (start_root, (void**)root->end_root, start_nursery, end_nursery, PIN_TYPE_OTHER);
1266 } SGEN_HASH_TABLE_FOREACH_END;
1267 /* now deal with the thread stacks
1268 * in the future we should be able to conservatively scan only:
1269 * *) the cpu registers
1270 * *) the unmanaged stack frames
1271 * *) the _last_ managed stack frame
1272 * *) pointers slots in managed frames
1274 scan_thread_data (start_nursery, end_nursery, FALSE, queue);
1278 unpin_objects_from_queue (SgenGrayQueue *queue)
1283 GRAY_OBJECT_DEQUEUE (queue, &addr, &desc);
1286 g_assert (SGEN_OBJECT_IS_PINNED (addr));
1287 SGEN_UNPIN_OBJECT (addr);
1292 CopyOrMarkObjectFunc func;
1294 } UserCopyOrMarkData;
1297 single_arg_user_copy_or_mark (void **obj, void *gc_data)
1299 UserCopyOrMarkData *data = gc_data;
1301 data->func (obj, data->queue);
1305 * The memory area from start_root to end_root contains pointers to objects.
1306 * Their position is precisely described by @desc (this means that the pointer
1307 * can be either NULL or the pointer to the start of an object).
1308 * This functions copies them to to_space updates them.
1310 * This function is not thread-safe!
1313 precisely_scan_objects_from (void** start_root, void** end_root, char* n_start, char *n_end, mword desc, ScanCopyContext ctx)
1315 CopyOrMarkObjectFunc copy_func = ctx.copy_func;
1316 SgenGrayQueue *queue = ctx.queue;
1318 switch (desc & ROOT_DESC_TYPE_MASK) {
1319 case ROOT_DESC_BITMAP:
1320 desc >>= ROOT_DESC_TYPE_SHIFT;
1322 if ((desc & 1) && *start_root) {
1323 copy_func (start_root, queue);
1324 SGEN_LOG (9, "Overwrote root at %p with %p", start_root, *start_root);
1330 case ROOT_DESC_COMPLEX: {
1331 gsize *bitmap_data = sgen_get_complex_descriptor_bitmap (desc);
1332 gsize bwords = (*bitmap_data) - 1;
1333 void **start_run = start_root;
1335 while (bwords-- > 0) {
1336 gsize bmap = *bitmap_data++;
1337 void **objptr = start_run;
1339 if ((bmap & 1) && *objptr) {
1340 copy_func (objptr, queue);
1341 SGEN_LOG (9, "Overwrote root at %p with %p", objptr, *objptr);
1346 start_run += GC_BITS_PER_WORD;
1350 case ROOT_DESC_USER: {
1351 UserCopyOrMarkData data = { copy_func, queue };
1352 MonoGCRootMarkFunc marker = sgen_get_user_descriptor_func (desc);
1353 marker (start_root, single_arg_user_copy_or_mark, &data);
1356 case ROOT_DESC_RUN_LEN:
1357 g_assert_not_reached ();
1359 g_assert_not_reached ();
1364 reset_heap_boundaries (void)
1366 lowest_heap_address = ~(mword)0;
1367 highest_heap_address = 0;
1371 sgen_update_heap_boundaries (mword low, mword high)
1376 old = lowest_heap_address;
1379 } while (SGEN_CAS_PTR ((gpointer*)&lowest_heap_address, (gpointer)low, (gpointer)old) != (gpointer)old);
1382 old = highest_heap_address;
1385 } while (SGEN_CAS_PTR ((gpointer*)&highest_heap_address, (gpointer)high, (gpointer)old) != (gpointer)old);
1389 * Allocate and setup the data structures needed to be able to allocate objects
1390 * in the nursery. The nursery is stored in nursery_section.
1393 alloc_nursery (void)
1395 GCMemSection *section;
1400 if (nursery_section)
1402 SGEN_LOG (2, "Allocating nursery size: %zu", (size_t)sgen_nursery_size);
1403 /* later we will alloc a larger area for the nursery but only activate
1404 * what we need. The rest will be used as expansion if we have too many pinned
1405 * objects in the existing nursery.
1407 /* FIXME: handle OOM */
1408 section = sgen_alloc_internal (INTERNAL_MEM_SECTION);
1410 alloc_size = sgen_nursery_size;
1412 /* If there isn't enough space even for the nursery we should simply abort. */
1413 g_assert (sgen_memgov_try_alloc_space (alloc_size, SPACE_NURSERY));
1415 #ifdef SGEN_ALIGN_NURSERY
1416 data = major_collector.alloc_heap (alloc_size, alloc_size, DEFAULT_NURSERY_BITS);
1418 data = major_collector.alloc_heap (alloc_size, 0, DEFAULT_NURSERY_BITS);
1420 sgen_update_heap_boundaries ((mword)data, (mword)(data + sgen_nursery_size));
1421 SGEN_LOG (4, "Expanding nursery size (%p-%p): %lu, total: %lu", data, data + alloc_size, (unsigned long)sgen_nursery_size, (unsigned long)mono_gc_get_heap_size ());
1422 section->data = section->next_data = data;
1423 section->size = alloc_size;
1424 section->end_data = data + sgen_nursery_size;
1425 scan_starts = (alloc_size + SCAN_START_SIZE - 1) / SCAN_START_SIZE;
1426 section->scan_starts = sgen_alloc_internal_dynamic (sizeof (char*) * scan_starts, INTERNAL_MEM_SCAN_STARTS, TRUE);
1427 section->num_scan_start = scan_starts;
1429 nursery_section = section;
1431 sgen_nursery_allocator_set_nursery_bounds (data, data + sgen_nursery_size);
1435 mono_gc_get_nursery (int *shift_bits, size_t *size)
1437 *size = sgen_nursery_size;
1438 #ifdef SGEN_ALIGN_NURSERY
1439 *shift_bits = DEFAULT_NURSERY_BITS;
1443 return sgen_get_nursery_start ();
1447 mono_gc_set_current_thread_appdomain (MonoDomain *domain)
1449 SgenThreadInfo *info = mono_thread_info_current ();
1451 /* Could be called from sgen_thread_unregister () with a NULL info */
1454 info->stopped_domain = domain;
1459 mono_gc_precise_stack_mark_enabled (void)
1461 return !conservative_stack_mark;
1465 mono_gc_get_logfile (void)
1467 return gc_debug_file;
1471 report_finalizer_roots_list (FinalizeReadyEntry *list)
1473 GCRootReport report;
1474 FinalizeReadyEntry *fin;
1477 for (fin = list; fin; fin = fin->next) {
1480 add_profile_gc_root (&report, fin->object, MONO_PROFILE_GC_ROOT_FINALIZER, 0);
1482 notify_gc_roots (&report);
1486 report_finalizer_roots (void)
1488 report_finalizer_roots_list (fin_ready_list);
1489 report_finalizer_roots_list (critical_fin_list);
1492 static GCRootReport *root_report;
1495 single_arg_report_root (void **obj, void *gc_data)
1498 add_profile_gc_root (root_report, *obj, MONO_PROFILE_GC_ROOT_OTHER, 0);
1502 precisely_report_roots_from (GCRootReport *report, void** start_root, void** end_root, mword desc)
1504 switch (desc & ROOT_DESC_TYPE_MASK) {
1505 case ROOT_DESC_BITMAP:
1506 desc >>= ROOT_DESC_TYPE_SHIFT;
1508 if ((desc & 1) && *start_root) {
1509 add_profile_gc_root (report, *start_root, MONO_PROFILE_GC_ROOT_OTHER, 0);
1515 case ROOT_DESC_COMPLEX: {
1516 gsize *bitmap_data = sgen_get_complex_descriptor_bitmap (desc);
1517 gsize bwords = (*bitmap_data) - 1;
1518 void **start_run = start_root;
1520 while (bwords-- > 0) {
1521 gsize bmap = *bitmap_data++;
1522 void **objptr = start_run;
1524 if ((bmap & 1) && *objptr) {
1525 add_profile_gc_root (report, *objptr, MONO_PROFILE_GC_ROOT_OTHER, 0);
1530 start_run += GC_BITS_PER_WORD;
1534 case ROOT_DESC_USER: {
1535 MonoGCRootMarkFunc marker = sgen_get_user_descriptor_func (desc);
1536 root_report = report;
1537 marker (start_root, single_arg_report_root, NULL);
1540 case ROOT_DESC_RUN_LEN:
1541 g_assert_not_reached ();
1543 g_assert_not_reached ();
1548 report_registered_roots_by_type (int root_type)
1550 GCRootReport report;
1554 SGEN_HASH_TABLE_FOREACH (&roots_hash [root_type], start_root, root) {
1555 SGEN_LOG (6, "Precise root scan %p-%p (desc: %p)", start_root, root->end_root, (void*)root->root_desc);
1556 precisely_report_roots_from (&report, start_root, (void**)root->end_root, root->root_desc);
1557 } SGEN_HASH_TABLE_FOREACH_END;
1558 notify_gc_roots (&report);
1562 report_registered_roots (void)
1564 report_registered_roots_by_type (ROOT_TYPE_NORMAL);
1565 report_registered_roots_by_type (ROOT_TYPE_WBARRIER);
1569 scan_finalizer_entries (FinalizeReadyEntry *list, ScanCopyContext ctx)
1571 CopyOrMarkObjectFunc copy_func = ctx.copy_func;
1572 SgenGrayQueue *queue = ctx.queue;
1573 FinalizeReadyEntry *fin;
1575 for (fin = list; fin; fin = fin->next) {
1578 SGEN_LOG (5, "Scan of fin ready object: %p (%s)\n", fin->object, safe_name (fin->object));
1579 copy_func (&fin->object, queue);
1584 generation_name (int generation)
1586 switch (generation) {
1587 case GENERATION_NURSERY: return "nursery";
1588 case GENERATION_OLD: return "old";
1589 default: g_assert_not_reached ();
1594 sgen_generation_name (int generation)
1596 return generation_name (generation);
1599 SgenObjectOperations *
1600 sgen_get_current_object_ops (void){
1601 return ¤t_object_ops;
1606 finish_gray_stack (int generation, GrayQueue *queue)
1610 int done_with_ephemerons, ephemeron_rounds = 0;
1611 CopyOrMarkObjectFunc copy_func = current_object_ops.copy_or_mark_object;
1612 ScanObjectFunc scan_func = current_object_ops.scan_object;
1613 ScanCopyContext ctx = { scan_func, copy_func, queue };
1614 char *start_addr = generation == GENERATION_NURSERY ? sgen_get_nursery_start () : NULL;
1615 char *end_addr = generation == GENERATION_NURSERY ? sgen_get_nursery_end () : (char*)-1;
1618 * We copied all the reachable objects. Now it's the time to copy
1619 * the objects that were not referenced by the roots, but by the copied objects.
1620 * we built a stack of objects pointed to by gray_start: they are
1621 * additional roots and we may add more items as we go.
1622 * We loop until gray_start == gray_objects which means no more objects have
1623 * been added. Note this is iterative: no recursion is involved.
1624 * We need to walk the LO list as well in search of marked big objects
1625 * (use a flag since this is needed only on major collections). We need to loop
1626 * here as well, so keep a counter of marked LO (increasing it in copy_object).
1627 * To achieve better cache locality and cache usage, we drain the gray stack
1628 * frequently, after each object is copied, and just finish the work here.
1630 sgen_drain_gray_stack (-1, ctx);
1632 SGEN_LOG (2, "%s generation done", generation_name (generation));
1635 Reset bridge data, we might have lingering data from a previous collection if this is a major
1636 collection trigged by minor overflow.
1638 We must reset the gathered bridges since their original block might be evacuated due to major
1639 fragmentation in the meanwhile and the bridge code should not have to deal with that.
1641 if (sgen_need_bridge_processing ())
1642 sgen_bridge_reset_data ();
1645 * Walk the ephemeron tables marking all values with reachable keys. This must be completely done
1646 * before processing finalizable objects and non-tracking weak links to avoid finalizing/clearing
1647 * objects that are in fact reachable.
1649 done_with_ephemerons = 0;
1651 done_with_ephemerons = mark_ephemerons_in_range (ctx);
1652 sgen_drain_gray_stack (-1, ctx);
1654 } while (!done_with_ephemerons);
1656 sgen_mark_togglerefs (start_addr, end_addr, ctx);
1658 if (sgen_need_bridge_processing ()) {
1659 /*Make sure the gray stack is empty before we process bridge objects so we get liveness right*/
1660 sgen_drain_gray_stack (-1, ctx);
1661 sgen_collect_bridge_objects (generation, ctx);
1662 if (generation == GENERATION_OLD)
1663 sgen_collect_bridge_objects (GENERATION_NURSERY, ctx);
1666 Do the first bridge step here, as the collector liveness state will become useless after that.
1668 An important optimization is to only proccess the possibly dead part of the object graph and skip
1669 over all live objects as we transitively know everything they point must be alive too.
1671 The above invariant is completely wrong if we let the gray queue be drained and mark/copy everything.
1673 This has the unfortunate side effect of making overflow collections perform the first step twice, but
1674 given we now have heuristics that perform major GC in anticipation of minor overflows this should not
1677 sgen_bridge_processing_stw_step ();
1681 Make sure we drain the gray stack before processing disappearing links and finalizers.
1682 If we don't make sure it is empty we might wrongly see a live object as dead.
1684 sgen_drain_gray_stack (-1, ctx);
1687 We must clear weak links that don't track resurrection before processing object ready for
1688 finalization so they can be cleared before that.
1690 sgen_null_link_in_range (generation, TRUE, ctx);
1691 if (generation == GENERATION_OLD)
1692 sgen_null_link_in_range (GENERATION_NURSERY, TRUE, ctx);
1695 /* walk the finalization queue and move also the objects that need to be
1696 * finalized: use the finalized objects as new roots so the objects they depend
1697 * on are also not reclaimed. As with the roots above, only objects in the nursery
1698 * are marked/copied.
1700 sgen_finalize_in_range (generation, ctx);
1701 if (generation == GENERATION_OLD)
1702 sgen_finalize_in_range (GENERATION_NURSERY, ctx);
1703 /* drain the new stack that might have been created */
1704 SGEN_LOG (6, "Precise scan of gray area post fin");
1705 sgen_drain_gray_stack (-1, ctx);
1708 * This must be done again after processing finalizable objects since CWL slots are cleared only after the key is finalized.
1710 done_with_ephemerons = 0;
1712 done_with_ephemerons = mark_ephemerons_in_range (ctx);
1713 sgen_drain_gray_stack (-1, ctx);
1715 } while (!done_with_ephemerons);
1718 * Clear ephemeron pairs with unreachable keys.
1719 * We pass the copy func so we can figure out if an array was promoted or not.
1721 clear_unreachable_ephemerons (ctx);
1724 * We clear togglerefs only after all possible chances of revival are done.
1725 * This is semantically more inline with what users expect and it allows for
1726 * user finalizers to correctly interact with TR objects.
1728 sgen_clear_togglerefs (start_addr, end_addr, ctx);
1731 SGEN_LOG (2, "Finalize queue handling scan for %s generation: %d usecs %d ephemeron rounds", generation_name (generation), TV_ELAPSED (atv, btv), ephemeron_rounds);
1734 * handle disappearing links
1735 * Note we do this after checking the finalization queue because if an object
1736 * survives (at least long enough to be finalized) we don't clear the link.
1737 * This also deals with a possible issue with the monitor reclamation: with the Boehm
1738 * GC a finalized object my lose the monitor because it is cleared before the finalizer is
1741 g_assert (sgen_gray_object_queue_is_empty (queue));
1743 sgen_null_link_in_range (generation, FALSE, ctx);
1744 if (generation == GENERATION_OLD)
1745 sgen_null_link_in_range (GENERATION_NURSERY, FALSE, ctx);
1746 if (sgen_gray_object_queue_is_empty (queue))
1748 sgen_drain_gray_stack (-1, ctx);
1751 g_assert (sgen_gray_object_queue_is_empty (queue));
1753 sgen_gray_object_queue_trim_free_list (queue);
1757 sgen_check_section_scan_starts (GCMemSection *section)
1760 for (i = 0; i < section->num_scan_start; ++i) {
1761 if (section->scan_starts [i]) {
1762 mword size = safe_object_get_size ((MonoObject*) section->scan_starts [i]);
1763 g_assert (size >= sizeof (MonoObject) && size <= MAX_SMALL_OBJ_SIZE);
1769 check_scan_starts (void)
1771 if (!do_scan_starts_check)
1773 sgen_check_section_scan_starts (nursery_section);
1774 major_collector.check_scan_starts ();
1778 scan_from_registered_roots (char *addr_start, char *addr_end, int root_type, ScanCopyContext ctx)
1782 SGEN_HASH_TABLE_FOREACH (&roots_hash [root_type], start_root, root) {
1783 SGEN_LOG (6, "Precise root scan %p-%p (desc: %p)", start_root, root->end_root, (void*)root->root_desc);
1784 precisely_scan_objects_from (start_root, (void**)root->end_root, addr_start, addr_end, root->root_desc, ctx);
1785 } SGEN_HASH_TABLE_FOREACH_END;
1789 sgen_dump_occupied (char *start, char *end, char *section_start)
1791 fprintf (heap_dump_file, "<occupied offset=\"%td\" size=\"%td\"/>\n", start - section_start, end - start);
1795 sgen_dump_section (GCMemSection *section, const char *type)
1797 char *start = section->data;
1798 char *end = section->data + section->size;
1799 char *occ_start = NULL;
1801 char *old_start = NULL; /* just for debugging */
1803 fprintf (heap_dump_file, "<section type=\"%s\" size=\"%lu\">\n", type, (unsigned long)section->size);
1805 while (start < end) {
1809 if (!*(void**)start) {
1811 sgen_dump_occupied (occ_start, start, section->data);
1814 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
1817 g_assert (start < section->next_data);
1822 vt = (GCVTable*)LOAD_VTABLE (start);
1825 size = ALIGN_UP (safe_object_get_size ((MonoObject*) start));
1828 fprintf (heap_dump_file, "<object offset=\"%d\" class=\"%s.%s\" size=\"%d\"/>\n",
1829 start - section->data,
1830 vt->klass->name_space, vt->klass->name,
1838 sgen_dump_occupied (occ_start, start, section->data);
1840 fprintf (heap_dump_file, "</section>\n");
1844 dump_object (MonoObject *obj, gboolean dump_location)
1846 static char class_name [1024];
1848 MonoClass *class = mono_object_class (obj);
1852 * Python's XML parser is too stupid to parse angle brackets
1853 * in strings, so we just ignore them;
1856 while (class->name [i] && j < sizeof (class_name) - 1) {
1857 if (!strchr ("<>\"", class->name [i]))
1858 class_name [j++] = class->name [i];
1861 g_assert (j < sizeof (class_name));
1864 fprintf (heap_dump_file, "<object class=\"%s.%s\" size=\"%zd\"",
1865 class->name_space, class_name,
1866 safe_object_get_size (obj));
1867 if (dump_location) {
1868 const char *location;
1869 if (ptr_in_nursery (obj))
1870 location = "nursery";
1871 else if (safe_object_get_size (obj) <= MAX_SMALL_OBJ_SIZE)
1875 fprintf (heap_dump_file, " location=\"%s\"", location);
1877 fprintf (heap_dump_file, "/>\n");
1881 dump_heap (const char *type, int num, const char *reason)
1886 fprintf (heap_dump_file, "<collection type=\"%s\" num=\"%d\"", type, num);
1888 fprintf (heap_dump_file, " reason=\"%s\"", reason);
1889 fprintf (heap_dump_file, ">\n");
1890 fprintf (heap_dump_file, "<other-mem-usage type=\"mempools\" size=\"%ld\"/>\n", mono_mempool_get_bytes_allocated ());
1891 sgen_dump_internal_mem_usage (heap_dump_file);
1892 fprintf (heap_dump_file, "<pinned type=\"stack\" bytes=\"%zu\"/>\n", sgen_pin_stats_get_pinned_byte_count (PIN_TYPE_STACK));
1893 /* fprintf (heap_dump_file, "<pinned type=\"static-data\" bytes=\"%d\"/>\n", pinned_byte_counts [PIN_TYPE_STATIC_DATA]); */
1894 fprintf (heap_dump_file, "<pinned type=\"other\" bytes=\"%zu\"/>\n", sgen_pin_stats_get_pinned_byte_count (PIN_TYPE_OTHER));
1896 fprintf (heap_dump_file, "<pinned-objects>\n");
1897 for (list = sgen_pin_stats_get_object_list (); list; list = list->next)
1898 dump_object (list->obj, TRUE);
1899 fprintf (heap_dump_file, "</pinned-objects>\n");
1901 sgen_dump_section (nursery_section, "nursery");
1903 major_collector.dump_heap (heap_dump_file);
1905 fprintf (heap_dump_file, "<los>\n");
1906 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1907 dump_object ((MonoObject*)bigobj->data, FALSE);
1908 fprintf (heap_dump_file, "</los>\n");
1910 fprintf (heap_dump_file, "</collection>\n");
1914 sgen_register_moved_object (void *obj, void *destination)
1916 g_assert (mono_profiler_events & MONO_PROFILE_GC_MOVES);
1918 if (moved_objects_idx == MOVED_OBJECTS_NUM) {
1919 mono_profiler_gc_moves (moved_objects, moved_objects_idx);
1920 moved_objects_idx = 0;
1922 moved_objects [moved_objects_idx++] = obj;
1923 moved_objects [moved_objects_idx++] = destination;
1929 static gboolean inited = FALSE;
1934 mono_counters_register ("Collection max time", MONO_COUNTER_GC | MONO_COUNTER_ULONG | MONO_COUNTER_TIME | MONO_COUNTER_MONOTONIC, &time_max);
1936 mono_counters_register ("Minor fragment clear", MONO_COUNTER_GC | MONO_COUNTER_ULONG | MONO_COUNTER_TIME, &time_minor_pre_collection_fragment_clear);
1937 mono_counters_register ("Minor pinning", MONO_COUNTER_GC | MONO_COUNTER_ULONG | MONO_COUNTER_TIME, &time_minor_pinning);
1938 mono_counters_register ("Minor scan remembered set", MONO_COUNTER_GC | MONO_COUNTER_ULONG | MONO_COUNTER_TIME, &time_minor_scan_remsets);
1939 mono_counters_register ("Minor scan pinned", MONO_COUNTER_GC | MONO_COUNTER_ULONG | MONO_COUNTER_TIME, &time_minor_scan_pinned);
1940 mono_counters_register ("Minor scan registered roots", MONO_COUNTER_GC | MONO_COUNTER_ULONG | MONO_COUNTER_TIME, &time_minor_scan_registered_roots);
1941 mono_counters_register ("Minor scan thread data", MONO_COUNTER_GC | MONO_COUNTER_ULONG | MONO_COUNTER_TIME, &time_minor_scan_thread_data);
1942 mono_counters_register ("Minor finish gray stack", MONO_COUNTER_GC | MONO_COUNTER_ULONG | MONO_COUNTER_TIME, &time_minor_finish_gray_stack);
1943 mono_counters_register ("Minor fragment creation", MONO_COUNTER_GC | MONO_COUNTER_ULONG | MONO_COUNTER_TIME, &time_minor_fragment_creation);
1945 mono_counters_register ("Major fragment clear", MONO_COUNTER_GC | MONO_COUNTER_ULONG | MONO_COUNTER_TIME, &time_major_pre_collection_fragment_clear);
1946 mono_counters_register ("Major pinning", MONO_COUNTER_GC | MONO_COUNTER_ULONG | MONO_COUNTER_TIME, &time_major_pinning);
1947 mono_counters_register ("Major scan pinned", MONO_COUNTER_GC | MONO_COUNTER_ULONG | MONO_COUNTER_TIME, &time_major_scan_pinned);
1948 mono_counters_register ("Major scan registered roots", MONO_COUNTER_GC | MONO_COUNTER_ULONG | MONO_COUNTER_TIME, &time_major_scan_registered_roots);
1949 mono_counters_register ("Major scan thread data", MONO_COUNTER_GC | MONO_COUNTER_ULONG | MONO_COUNTER_TIME, &time_major_scan_thread_data);
1950 mono_counters_register ("Major scan alloc_pinned", MONO_COUNTER_GC | MONO_COUNTER_ULONG | MONO_COUNTER_TIME, &time_major_scan_alloc_pinned);
1951 mono_counters_register ("Major scan finalized", MONO_COUNTER_GC | MONO_COUNTER_ULONG | MONO_COUNTER_TIME, &time_major_scan_finalized);
1952 mono_counters_register ("Major scan big objects", MONO_COUNTER_GC | MONO_COUNTER_ULONG | MONO_COUNTER_TIME, &time_major_scan_big_objects);
1953 mono_counters_register ("Major finish gray stack", MONO_COUNTER_GC | MONO_COUNTER_ULONG | MONO_COUNTER_TIME, &time_major_finish_gray_stack);
1954 mono_counters_register ("Major free big objects", MONO_COUNTER_GC | MONO_COUNTER_ULONG | MONO_COUNTER_TIME, &time_major_free_bigobjs);
1955 mono_counters_register ("Major LOS sweep", MONO_COUNTER_GC | MONO_COUNTER_ULONG | MONO_COUNTER_TIME, &time_major_los_sweep);
1956 mono_counters_register ("Major sweep", MONO_COUNTER_GC | MONO_COUNTER_ULONG | MONO_COUNTER_TIME, &time_major_sweep);
1957 mono_counters_register ("Major fragment creation", MONO_COUNTER_GC | MONO_COUNTER_ULONG | MONO_COUNTER_TIME, &time_major_fragment_creation);
1959 mono_counters_register ("Number of pinned objects", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &stat_pinned_objects);
1961 #ifdef HEAVY_STATISTICS
1962 mono_counters_register ("WBarrier remember pointer", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_add_to_global_remset);
1963 mono_counters_register ("WBarrier set field", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_field);
1964 mono_counters_register ("WBarrier set arrayref", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_arrayref);
1965 mono_counters_register ("WBarrier arrayref copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_arrayref_copy);
1966 mono_counters_register ("WBarrier generic store called", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_generic_store);
1967 mono_counters_register ("WBarrier generic atomic store called", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_generic_store_atomic);
1968 mono_counters_register ("WBarrier set root", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_root);
1969 mono_counters_register ("WBarrier value copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_value_copy);
1970 mono_counters_register ("WBarrier object copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_object_copy);
1972 mono_counters_register ("# objects allocated degraded", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &stat_objects_alloced_degraded);
1973 mono_counters_register ("bytes allocated degraded", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &stat_bytes_alloced_degraded);
1975 mono_counters_register ("# copy_object() called (nursery)", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &stat_copy_object_called_nursery);
1976 mono_counters_register ("# objects copied (nursery)", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &stat_objects_copied_nursery);
1977 mono_counters_register ("# copy_object() called (major)", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &stat_copy_object_called_major);
1978 mono_counters_register ("# objects copied (major)", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &stat_objects_copied_major);
1980 mono_counters_register ("# scan_object() called (nursery)", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &stat_scan_object_called_nursery);
1981 mono_counters_register ("# scan_object() called (major)", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &stat_scan_object_called_major);
1983 mono_counters_register ("Slots allocated in vain", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &stat_slots_allocated_in_vain);
1985 mono_counters_register ("# nursery copy_object() failed from space", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &stat_nursery_copy_object_failed_from_space);
1986 mono_counters_register ("# nursery copy_object() failed forwarded", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &stat_nursery_copy_object_failed_forwarded);
1987 mono_counters_register ("# nursery copy_object() failed pinned", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &stat_nursery_copy_object_failed_pinned);
1988 mono_counters_register ("# nursery copy_object() failed to space", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &stat_nursery_copy_object_failed_to_space);
1990 sgen_nursery_allocator_init_heavy_stats ();
1991 sgen_alloc_init_heavy_stats ();
1999 reset_pinned_from_failed_allocation (void)
2001 bytes_pinned_from_failed_allocation = 0;
2005 sgen_set_pinned_from_failed_allocation (mword objsize)
2007 bytes_pinned_from_failed_allocation += objsize;
2011 sgen_collection_is_concurrent (void)
2013 switch (current_collection_generation) {
2014 case GENERATION_NURSERY:
2016 case GENERATION_OLD:
2017 return concurrent_collection_in_progress;
2019 g_error ("Invalid current generation %d", current_collection_generation);
2024 sgen_concurrent_collection_in_progress (void)
2026 return concurrent_collection_in_progress;
2033 } FinishRememberedSetScanJobData;
2036 job_finish_remembered_set_scan (WorkerData *worker_data, void *job_data_untyped)
2038 FinishRememberedSetScanJobData *job_data = job_data_untyped;
2040 remset.finish_scan_remsets (job_data->heap_start, job_data->heap_end, sgen_workers_get_job_gray_queue (worker_data));
2041 sgen_free_internal_dynamic (job_data, sizeof (FinishRememberedSetScanJobData), INTERNAL_MEM_WORKER_JOB_DATA);
2046 CopyOrMarkObjectFunc copy_or_mark_func;
2047 ScanObjectFunc scan_func;
2051 } ScanFromRegisteredRootsJobData;
2054 job_scan_from_registered_roots (WorkerData *worker_data, void *job_data_untyped)
2056 ScanFromRegisteredRootsJobData *job_data = job_data_untyped;
2057 ScanCopyContext ctx = { job_data->scan_func, job_data->copy_or_mark_func,
2058 sgen_workers_get_job_gray_queue (worker_data) };
2060 scan_from_registered_roots (job_data->heap_start, job_data->heap_end, job_data->root_type, ctx);
2061 sgen_free_internal_dynamic (job_data, sizeof (ScanFromRegisteredRootsJobData), INTERNAL_MEM_WORKER_JOB_DATA);
2068 } ScanThreadDataJobData;
2071 job_scan_thread_data (WorkerData *worker_data, void *job_data_untyped)
2073 ScanThreadDataJobData *job_data = job_data_untyped;
2075 scan_thread_data (job_data->heap_start, job_data->heap_end, TRUE,
2076 sgen_workers_get_job_gray_queue (worker_data));
2077 sgen_free_internal_dynamic (job_data, sizeof (ScanThreadDataJobData), INTERNAL_MEM_WORKER_JOB_DATA);
2081 job_scan_finalizer_entries (WorkerData *worker_data, void *job_data_untyped)
2083 FinalizeReadyEntry *list = job_data_untyped;
2084 ScanCopyContext ctx = { NULL, current_object_ops.copy_or_mark_object, sgen_workers_get_job_gray_queue (worker_data) };
2086 scan_finalizer_entries (list, ctx);
2090 job_scan_major_mod_union_cardtable (WorkerData *worker_data, void *job_data_untyped)
2092 g_assert (concurrent_collection_in_progress);
2093 major_collector.scan_card_table (TRUE, sgen_workers_get_job_gray_queue (worker_data));
2097 job_scan_los_mod_union_cardtable (WorkerData *worker_data, void *job_data_untyped)
2099 g_assert (concurrent_collection_in_progress);
2100 sgen_los_scan_card_table (TRUE, sgen_workers_get_job_gray_queue (worker_data));
2104 verify_scan_starts (char *start, char *end)
2108 for (i = 0; i < nursery_section->num_scan_start; ++i) {
2109 char *addr = nursery_section->scan_starts [i];
2110 if (addr > start && addr < end)
2111 SGEN_LOG (1, "NFC-BAD SCAN START [%zu] %p for obj [%p %p]", i, addr, start, end);
2116 verify_nursery (void)
2118 char *start, *end, *cur, *hole_start;
2120 if (!do_verify_nursery)
2123 if (nursery_canaries_enabled ())
2124 SGEN_LOG (1, "Checking nursery canaries...");
2126 /*This cleans up unused fragments */
2127 sgen_nursery_allocator_prepare_for_pinning ();
2129 hole_start = start = cur = sgen_get_nursery_start ();
2130 end = sgen_get_nursery_end ();
2135 if (!*(void**)cur) {
2136 cur += sizeof (void*);
2140 if (object_is_forwarded (cur))
2141 SGEN_LOG (1, "FORWARDED OBJ %p", cur);
2142 else if (object_is_pinned (cur))
2143 SGEN_LOG (1, "PINNED OBJ %p", cur);
2145 ss = safe_object_get_size ((MonoObject*)cur);
2146 size = ALIGN_UP (safe_object_get_size ((MonoObject*)cur));
2147 verify_scan_starts (cur, cur + size);
2148 if (do_dump_nursery_content) {
2149 if (cur > hole_start)
2150 SGEN_LOG (1, "HOLE [%p %p %d]", hole_start, cur, (int)(cur - hole_start));
2151 SGEN_LOG (1, "OBJ [%p %p %d %d %s %d]", cur, cur + size, (int)size, (int)ss, sgen_safe_name ((MonoObject*)cur), (gpointer)LOAD_VTABLE (cur) == sgen_get_array_fill_vtable ());
2153 if (nursery_canaries_enabled () && (MonoVTable*)SGEN_LOAD_VTABLE (cur) != array_fill_vtable) {
2154 CHECK_CANARY_FOR_OBJECT (cur);
2155 CANARIFY_SIZE (size);
2163 * Checks that no objects in the nursery are fowarded or pinned. This
2164 * is a precondition to restarting the mutator while doing a
2165 * concurrent collection. Note that we don't clear fragments because
2166 * we depend on that having happened earlier.
2169 check_nursery_is_clean (void)
2171 char *start, *end, *cur;
2173 start = cur = sgen_get_nursery_start ();
2174 end = sgen_get_nursery_end ();
2179 if (!*(void**)cur) {
2180 cur += sizeof (void*);
2184 g_assert (!object_is_forwarded (cur));
2185 g_assert (!object_is_pinned (cur));
2187 ss = safe_object_get_size ((MonoObject*)cur);
2188 size = ALIGN_UP (safe_object_get_size ((MonoObject*)cur));
2189 verify_scan_starts (cur, cur + size);
2196 init_gray_queue (void)
2198 if (sgen_collection_is_concurrent ())
2199 sgen_workers_init_distribute_gray_queue ();
2200 sgen_gray_object_queue_init (&gray_queue, NULL);
2204 * Perform a nursery collection.
2206 * Return whether any objects were late-pinned due to being out of memory.
2209 collect_nursery (SgenGrayQueue *unpin_queue, gboolean finish_up_concurrent_mark)
2211 gboolean needs_major;
2212 size_t max_garbage_amount;
2214 FinishRememberedSetScanJobData *frssjd;
2215 ScanFromRegisteredRootsJobData *scrrjd_normal, *scrrjd_wbarrier;
2216 ScanThreadDataJobData *stdjd;
2217 mword fragment_total;
2218 ScanCopyContext ctx;
2222 if (disable_minor_collections)
2225 TV_GETTIME (last_minor_collection_start_tv);
2226 atv = last_minor_collection_start_tv;
2228 MONO_GC_BEGIN (GENERATION_NURSERY);
2229 binary_protocol_collection_begin (gc_stats.minor_gc_count, GENERATION_NURSERY);
2233 #ifndef DISABLE_PERFCOUNTERS
2234 mono_perfcounters->gc_collections0++;
2237 current_collection_generation = GENERATION_NURSERY;
2238 current_object_ops = sgen_minor_collector.serial_ops;
2240 reset_pinned_from_failed_allocation ();
2242 check_scan_starts ();
2244 sgen_nursery_alloc_prepare_for_minor ();
2248 nursery_next = sgen_nursery_alloc_get_upper_alloc_bound ();
2249 /* FIXME: optimize later to use the higher address where an object can be present */
2250 nursery_next = MAX (nursery_next, sgen_get_nursery_end ());
2252 SGEN_LOG (1, "Start nursery collection %d %p-%p, size: %d", gc_stats.minor_gc_count, sgen_get_nursery_start (), nursery_next, (int)(nursery_next - sgen_get_nursery_start ()));
2253 max_garbage_amount = nursery_next - sgen_get_nursery_start ();
2254 g_assert (nursery_section->size >= max_garbage_amount);
2256 /* world must be stopped already */
2258 time_minor_pre_collection_fragment_clear += TV_ELAPSED (atv, btv);
2260 if (xdomain_checks) {
2261 sgen_clear_nursery_fragments ();
2262 sgen_check_for_xdomain_refs ();
2265 nursery_section->next_data = nursery_next;
2267 major_collector.start_nursery_collection ();
2269 sgen_memgov_minor_collection_start ();
2273 gc_stats.minor_gc_count ++;
2275 if (whole_heap_check_before_collection) {
2276 sgen_clear_nursery_fragments ();
2277 sgen_check_whole_heap (finish_up_concurrent_mark);
2279 if (consistency_check_at_minor_collection)
2280 sgen_check_consistency ();
2282 MONO_GC_CHECKPOINT_1 (GENERATION_NURSERY);
2284 sgen_process_fin_stage_entries ();
2285 sgen_process_dislink_stage_entries ();
2287 MONO_GC_CHECKPOINT_2 (GENERATION_NURSERY);
2289 /* pin from pinned handles */
2290 sgen_init_pinning ();
2291 mono_profiler_gc_event (MONO_GC_EVENT_MARK_START, 0);
2292 pin_from_roots (sgen_get_nursery_start (), nursery_next, WORKERS_DISTRIBUTE_GRAY_QUEUE);
2293 /* pin cemented objects */
2294 sgen_pin_cemented_objects ();
2295 /* identify pinned objects */
2296 sgen_optimize_pin_queue ();
2297 sgen_pinning_setup_section (nursery_section);
2298 ctx.scan_func = NULL;
2299 ctx.copy_func = NULL;
2300 ctx.queue = WORKERS_DISTRIBUTE_GRAY_QUEUE;
2301 pin_objects_in_nursery (ctx);
2302 sgen_pinning_trim_queue_to_section (nursery_section);
2305 time_minor_pinning += TV_ELAPSED (btv, atv);
2306 SGEN_LOG (2, "Finding pinned pointers: %zd in %d usecs", sgen_get_pinned_count (), TV_ELAPSED (btv, atv));
2307 SGEN_LOG (4, "Start scan with %zd pinned objects", sgen_get_pinned_count ());
2309 MONO_GC_CHECKPOINT_3 (GENERATION_NURSERY);
2311 frssjd = sgen_alloc_internal_dynamic (sizeof (FinishRememberedSetScanJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2312 frssjd->heap_start = sgen_get_nursery_start ();
2313 frssjd->heap_end = nursery_next;
2314 sgen_workers_enqueue_job (job_finish_remembered_set_scan, frssjd);
2316 /* we don't have complete write barrier yet, so we scan all the old generation sections */
2318 time_minor_scan_remsets += TV_ELAPSED (atv, btv);
2319 SGEN_LOG (2, "Old generation scan: %d usecs", TV_ELAPSED (atv, btv));
2321 MONO_GC_CHECKPOINT_4 (GENERATION_NURSERY);
2323 /* FIXME: why is this here? */
2324 ctx.scan_func = current_object_ops.scan_object;
2325 ctx.copy_func = NULL;
2326 ctx.queue = &gray_queue;
2327 sgen_drain_gray_stack (-1, ctx);
2329 if (mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS)
2330 report_registered_roots ();
2331 if (mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS)
2332 report_finalizer_roots ();
2334 time_minor_scan_pinned += TV_ELAPSED (btv, atv);
2336 MONO_GC_CHECKPOINT_5 (GENERATION_NURSERY);
2338 /* registered roots, this includes static fields */
2339 scrrjd_normal = sgen_alloc_internal_dynamic (sizeof (ScanFromRegisteredRootsJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2340 scrrjd_normal->copy_or_mark_func = current_object_ops.copy_or_mark_object;
2341 scrrjd_normal->scan_func = current_object_ops.scan_object;
2342 scrrjd_normal->heap_start = sgen_get_nursery_start ();
2343 scrrjd_normal->heap_end = nursery_next;
2344 scrrjd_normal->root_type = ROOT_TYPE_NORMAL;
2345 sgen_workers_enqueue_job (job_scan_from_registered_roots, scrrjd_normal);
2347 scrrjd_wbarrier = sgen_alloc_internal_dynamic (sizeof (ScanFromRegisteredRootsJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2348 scrrjd_wbarrier->copy_or_mark_func = current_object_ops.copy_or_mark_object;
2349 scrrjd_wbarrier->scan_func = current_object_ops.scan_object;
2350 scrrjd_wbarrier->heap_start = sgen_get_nursery_start ();
2351 scrrjd_wbarrier->heap_end = nursery_next;
2352 scrrjd_wbarrier->root_type = ROOT_TYPE_WBARRIER;
2353 sgen_workers_enqueue_job (job_scan_from_registered_roots, scrrjd_wbarrier);
2356 time_minor_scan_registered_roots += TV_ELAPSED (atv, btv);
2358 MONO_GC_CHECKPOINT_6 (GENERATION_NURSERY);
2361 stdjd = sgen_alloc_internal_dynamic (sizeof (ScanThreadDataJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2362 stdjd->heap_start = sgen_get_nursery_start ();
2363 stdjd->heap_end = nursery_next;
2364 sgen_workers_enqueue_job (job_scan_thread_data, stdjd);
2367 time_minor_scan_thread_data += TV_ELAPSED (btv, atv);
2370 MONO_GC_CHECKPOINT_7 (GENERATION_NURSERY);
2372 g_assert (!sgen_collection_is_concurrent ());
2374 /* Scan the list of objects ready for finalization. If */
2375 sgen_workers_enqueue_job (job_scan_finalizer_entries, fin_ready_list);
2376 sgen_workers_enqueue_job (job_scan_finalizer_entries, critical_fin_list);
2378 MONO_GC_CHECKPOINT_8 (GENERATION_NURSERY);
2380 finish_gray_stack (GENERATION_NURSERY, &gray_queue);
2382 time_minor_finish_gray_stack += TV_ELAPSED (btv, atv);
2383 mono_profiler_gc_event (MONO_GC_EVENT_MARK_END, 0);
2385 MONO_GC_CHECKPOINT_9 (GENERATION_NURSERY);
2388 * The (single-threaded) finalization code might have done
2389 * some copying/marking so we can only reset the GC thread's
2390 * worker data here instead of earlier when we joined the
2393 sgen_workers_reset_data ();
2395 if (objects_pinned) {
2396 sgen_optimize_pin_queue ();
2397 sgen_pinning_setup_section (nursery_section);
2400 /* walk the pin_queue, build up the fragment list of free memory, unmark
2401 * pinned objects as we go, memzero() the empty fragments so they are ready for the
2404 mono_profiler_gc_event (MONO_GC_EVENT_RECLAIM_START, 0);
2405 fragment_total = sgen_build_nursery_fragments (nursery_section, unpin_queue);
2406 if (!fragment_total)
2409 /* Clear TLABs for all threads */
2410 sgen_clear_tlabs ();
2412 mono_profiler_gc_event (MONO_GC_EVENT_RECLAIM_END, 0);
2414 time_minor_fragment_creation += TV_ELAPSED (atv, btv);
2415 SGEN_LOG (2, "Fragment creation: %d usecs, %lu bytes available", TV_ELAPSED (atv, btv), (unsigned long)fragment_total);
2417 if (consistency_check_at_minor_collection)
2418 sgen_check_major_refs ();
2420 major_collector.finish_nursery_collection ();
2422 TV_GETTIME (last_minor_collection_end_tv);
2423 gc_stats.minor_gc_time += TV_ELAPSED (last_minor_collection_start_tv, last_minor_collection_end_tv);
2426 dump_heap ("minor", gc_stats.minor_gc_count - 1, NULL);
2428 /* prepare the pin queue for the next collection */
2429 sgen_finish_pinning ();
2430 if (fin_ready_list || critical_fin_list) {
2431 SGEN_LOG (4, "Finalizer-thread wakeup: ready %d", num_ready_finalizers);
2432 mono_gc_finalize_notify ();
2434 sgen_pin_stats_reset ();
2435 /* clear cemented hash */
2436 sgen_cement_clear_below_threshold ();
2438 g_assert (sgen_gray_object_queue_is_empty (&gray_queue));
2440 remset.finish_minor_collection ();
2442 check_scan_starts ();
2444 binary_protocol_flush_buffers (FALSE);
2446 sgen_memgov_minor_collection_end ();
2448 /*objects are late pinned because of lack of memory, so a major is a good call*/
2449 needs_major = objects_pinned > 0;
2450 current_collection_generation = -1;
2453 MONO_GC_END (GENERATION_NURSERY);
2454 binary_protocol_collection_end (gc_stats.minor_gc_count - 1, GENERATION_NURSERY, 0, 0);
2456 if (check_nursery_objects_pinned && !sgen_minor_collector.is_split)
2457 sgen_check_nursery_objects_pinned (unpin_queue != NULL);
2463 scan_nursery_objects_callback (char *obj, size_t size, ScanCopyContext *ctx)
2466 * This is called on all objects in the nursery, including pinned ones, so we need
2467 * to use sgen_obj_get_descriptor_safe(), which masks out the vtable tag bits.
2469 ctx->scan_func (obj, sgen_obj_get_descriptor_safe (obj), ctx->queue);
2473 scan_nursery_objects (ScanCopyContext ctx)
2475 sgen_scan_area_with_callback (nursery_section->data, nursery_section->end_data,
2476 (IterateObjectCallbackFunc)scan_nursery_objects_callback, (void*)&ctx, FALSE);
2480 major_copy_or_mark_from_roots (size_t *old_next_pin_slot, gboolean start_concurrent_mark, gboolean finish_up_concurrent_mark, gboolean scan_mod_union, gboolean scan_whole_nursery)
2485 /* FIXME: only use these values for the precise scan
2486 * note that to_space pointers should be excluded anyway...
2488 char *heap_start = NULL;
2489 char *heap_end = (char*)-1;
2490 gboolean profile_roots = mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS;
2491 GCRootReport root_report = { 0 };
2492 ScanFromRegisteredRootsJobData *scrrjd_normal, *scrrjd_wbarrier;
2493 ScanThreadDataJobData *stdjd;
2494 ScanCopyContext ctx;
2496 if (concurrent_collection_in_progress) {
2497 /*This cleans up unused fragments */
2498 sgen_nursery_allocator_prepare_for_pinning ();
2500 if (do_concurrent_checks)
2501 check_nursery_is_clean ();
2503 /* The concurrent collector doesn't touch the nursery. */
2504 sgen_nursery_alloc_prepare_for_major ();
2511 /* Pinning depends on this */
2512 sgen_clear_nursery_fragments ();
2514 if (whole_heap_check_before_collection)
2515 sgen_check_whole_heap (finish_up_concurrent_mark);
2518 time_major_pre_collection_fragment_clear += TV_ELAPSED (atv, btv);
2520 if (!sgen_collection_is_concurrent ())
2521 nursery_section->next_data = sgen_get_nursery_end ();
2522 /* we should also coalesce scanning from sections close to each other
2523 * and deal with pointers outside of the sections later.
2527 *major_collector.have_swept = FALSE;
2529 if (xdomain_checks) {
2530 sgen_clear_nursery_fragments ();
2531 sgen_check_for_xdomain_refs ();
2534 if (!concurrent_collection_in_progress) {
2535 /* Remsets are not useful for a major collection */
2536 remset.prepare_for_major_collection ();
2539 sgen_process_fin_stage_entries ();
2540 sgen_process_dislink_stage_entries ();
2543 sgen_init_pinning ();
2544 SGEN_LOG (6, "Collecting pinned addresses");
2545 pin_from_roots ((void*)lowest_heap_address, (void*)highest_heap_address, WORKERS_DISTRIBUTE_GRAY_QUEUE);
2547 if (!concurrent_collection_in_progress || finish_up_concurrent_mark) {
2548 if (major_collector.is_concurrent) {
2550 * The concurrent major collector cannot evict
2551 * yet, so we need to pin cemented objects to
2552 * not break some asserts.
2554 * FIXME: We could evict now!
2556 sgen_pin_cemented_objects ();
2559 if (!concurrent_collection_in_progress)
2560 sgen_cement_reset ();
2563 sgen_optimize_pin_queue ();
2566 * pin_queue now contains all candidate pointers, sorted and
2567 * uniqued. We must do two passes now to figure out which
2568 * objects are pinned.
2570 * The first is to find within the pin_queue the area for each
2571 * section. This requires that the pin_queue be sorted. We
2572 * also process the LOS objects and pinned chunks here.
2574 * The second, destructive, pass is to reduce the section
2575 * areas to pointers to the actually pinned objects.
2577 SGEN_LOG (6, "Pinning from sections");
2578 /* first pass for the sections */
2579 sgen_find_section_pin_queue_start_end (nursery_section);
2580 major_collector.find_pin_queue_start_ends (WORKERS_DISTRIBUTE_GRAY_QUEUE);
2581 /* identify possible pointers to the insize of large objects */
2582 SGEN_LOG (6, "Pinning from large objects");
2583 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
2585 if (sgen_find_optimized_pin_queue_area (bigobj->data, (char*)bigobj->data + sgen_los_object_size (bigobj), &dummy, &dummy)) {
2586 binary_protocol_pin (bigobj->data, (gpointer)LOAD_VTABLE (bigobj->data), safe_object_get_size (((MonoObject*)(bigobj->data))));
2588 #ifdef ENABLE_DTRACE
2589 if (G_UNLIKELY (MONO_GC_OBJ_PINNED_ENABLED ())) {
2590 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (bigobj->data);
2591 MONO_GC_OBJ_PINNED ((mword)bigobj->data, sgen_safe_object_get_size ((MonoObject*)bigobj->data), vt->klass->name_space, vt->klass->name, GENERATION_OLD);
2595 if (sgen_los_object_is_pinned (bigobj->data)) {
2596 g_assert (finish_up_concurrent_mark);
2599 sgen_los_pin_object (bigobj->data);
2600 if (SGEN_OBJECT_HAS_REFERENCES (bigobj->data))
2601 GRAY_OBJECT_ENQUEUE (WORKERS_DISTRIBUTE_GRAY_QUEUE, bigobj->data, sgen_obj_get_descriptor (bigobj->data));
2602 if (G_UNLIKELY (do_pin_stats))
2603 sgen_pin_stats_register_object ((char*) bigobj->data, safe_object_get_size ((MonoObject*) bigobj->data));
2604 SGEN_LOG (6, "Marked large object %p (%s) size: %lu from roots", bigobj->data, safe_name (bigobj->data), (unsigned long)sgen_los_object_size (bigobj));
2607 add_profile_gc_root (&root_report, bigobj->data, MONO_PROFILE_GC_ROOT_PINNING | MONO_PROFILE_GC_ROOT_MISC, 0);
2611 notify_gc_roots (&root_report);
2612 /* second pass for the sections */
2613 ctx.scan_func = concurrent_collection_in_progress ? current_object_ops.scan_object : NULL;
2614 ctx.copy_func = NULL;
2615 ctx.queue = WORKERS_DISTRIBUTE_GRAY_QUEUE;
2618 * Concurrent mark never follows references into the nursery. In the start and
2619 * finish pauses we must scan live nursery objects, though.
2621 * In the finish pause we do this conservatively by scanning all nursery objects.
2622 * Previously we would only scan pinned objects here. We assumed that all objects
2623 * that were pinned during the nursery collection immediately preceding this finish
2624 * mark would be pinned again here. Due to the way we get the stack end for the GC
2625 * thread, however, that's not necessarily the case: we scan part of the stack used
2626 * by the GC itself, which changes constantly, so pinning isn't entirely
2629 * The split nursery also complicates things because non-pinned objects can survive
2630 * in the nursery. That's why we need to do a full scan of the nursery for it, too.
2632 * In the future we shouldn't do a preceding nursery collection at all and instead
2633 * do the finish pause with promotion from the nursery.
2635 * A further complication arises when we have late-pinned objects from the preceding
2636 * nursery collection. Those are the result of being out of memory when trying to
2637 * evacuate objects. They won't be found from the roots, so we just scan the whole
2640 * Non-concurrent mark evacuates from the nursery, so it's
2641 * sufficient to just scan pinned nursery objects.
2643 if (scan_whole_nursery || finish_up_concurrent_mark || (concurrent_collection_in_progress && sgen_minor_collector.is_split)) {
2644 scan_nursery_objects (ctx);
2646 pin_objects_in_nursery (ctx);
2647 if (check_nursery_objects_pinned && !sgen_minor_collector.is_split)
2648 sgen_check_nursery_objects_pinned (!concurrent_collection_in_progress || finish_up_concurrent_mark);
2651 major_collector.pin_objects (WORKERS_DISTRIBUTE_GRAY_QUEUE);
2652 if (old_next_pin_slot)
2653 *old_next_pin_slot = sgen_get_pinned_count ();
2656 time_major_pinning += TV_ELAPSED (atv, btv);
2657 SGEN_LOG (2, "Finding pinned pointers: %zd in %d usecs", sgen_get_pinned_count (), TV_ELAPSED (atv, btv));
2658 SGEN_LOG (4, "Start scan with %zd pinned objects", sgen_get_pinned_count ());
2660 major_collector.init_to_space ();
2663 * The concurrent collector doesn't move objects, neither on
2664 * the major heap nor in the nursery, so we can mark even
2665 * before pinning has finished. For the non-concurrent
2666 * collector we start the workers after pinning.
2668 if (start_concurrent_mark) {
2669 sgen_workers_start_all_workers ();
2670 gray_queue_enable_redirect (WORKERS_DISTRIBUTE_GRAY_QUEUE);
2673 #ifdef SGEN_DEBUG_INTERNAL_ALLOC
2674 main_gc_thread = mono_native_thread_self ();
2677 if (mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS)
2678 report_registered_roots ();
2680 time_major_scan_pinned += TV_ELAPSED (btv, atv);
2682 /* registered roots, this includes static fields */
2683 scrrjd_normal = sgen_alloc_internal_dynamic (sizeof (ScanFromRegisteredRootsJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2684 scrrjd_normal->copy_or_mark_func = current_object_ops.copy_or_mark_object;
2685 scrrjd_normal->scan_func = current_object_ops.scan_object;
2686 scrrjd_normal->heap_start = heap_start;
2687 scrrjd_normal->heap_end = heap_end;
2688 scrrjd_normal->root_type = ROOT_TYPE_NORMAL;
2689 sgen_workers_enqueue_job (job_scan_from_registered_roots, scrrjd_normal);
2691 scrrjd_wbarrier = sgen_alloc_internal_dynamic (sizeof (ScanFromRegisteredRootsJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2692 scrrjd_wbarrier->copy_or_mark_func = current_object_ops.copy_or_mark_object;
2693 scrrjd_wbarrier->scan_func = current_object_ops.scan_object;
2694 scrrjd_wbarrier->heap_start = heap_start;
2695 scrrjd_wbarrier->heap_end = heap_end;
2696 scrrjd_wbarrier->root_type = ROOT_TYPE_WBARRIER;
2697 sgen_workers_enqueue_job (job_scan_from_registered_roots, scrrjd_wbarrier);
2700 time_major_scan_registered_roots += TV_ELAPSED (atv, btv);
2703 stdjd = sgen_alloc_internal_dynamic (sizeof (ScanThreadDataJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2704 stdjd->heap_start = heap_start;
2705 stdjd->heap_end = heap_end;
2706 sgen_workers_enqueue_job (job_scan_thread_data, stdjd);
2709 time_major_scan_thread_data += TV_ELAPSED (btv, atv);
2712 time_major_scan_alloc_pinned += TV_ELAPSED (atv, btv);
2714 if (mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS)
2715 report_finalizer_roots ();
2717 /* scan the list of objects ready for finalization */
2718 sgen_workers_enqueue_job (job_scan_finalizer_entries, fin_ready_list);
2719 sgen_workers_enqueue_job (job_scan_finalizer_entries, critical_fin_list);
2721 if (scan_mod_union) {
2722 g_assert (finish_up_concurrent_mark);
2724 /* Mod union card table */
2725 sgen_workers_enqueue_job (job_scan_major_mod_union_cardtable, NULL);
2726 sgen_workers_enqueue_job (job_scan_los_mod_union_cardtable, NULL);
2730 time_major_scan_finalized += TV_ELAPSED (btv, atv);
2731 SGEN_LOG (2, "Root scan: %d usecs", TV_ELAPSED (btv, atv));
2734 time_major_scan_big_objects += TV_ELAPSED (atv, btv);
2736 if (concurrent_collection_in_progress) {
2737 /* prepare the pin queue for the next collection */
2738 sgen_finish_pinning ();
2740 sgen_pin_stats_reset ();
2742 if (do_concurrent_checks)
2743 check_nursery_is_clean ();
2748 major_start_collection (gboolean concurrent, size_t *old_next_pin_slot)
2750 MONO_GC_BEGIN (GENERATION_OLD);
2751 binary_protocol_collection_begin (gc_stats.major_gc_count, GENERATION_OLD);
2753 current_collection_generation = GENERATION_OLD;
2754 #ifndef DISABLE_PERFCOUNTERS
2755 mono_perfcounters->gc_collections1++;
2758 g_assert (sgen_section_gray_queue_is_empty (sgen_workers_get_distribute_section_gray_queue ()));
2761 g_assert (major_collector.is_concurrent);
2762 concurrent_collection_in_progress = TRUE;
2764 sgen_cement_concurrent_start ();
2766 current_object_ops = major_collector.major_concurrent_ops;
2768 current_object_ops = major_collector.major_ops;
2771 reset_pinned_from_failed_allocation ();
2773 sgen_memgov_major_collection_start ();
2775 //count_ref_nonref_objs ();
2776 //consistency_check ();
2778 check_scan_starts ();
2781 SGEN_LOG (1, "Start major collection %d", gc_stats.major_gc_count);
2782 gc_stats.major_gc_count ++;
2784 if (major_collector.start_major_collection)
2785 major_collector.start_major_collection ();
2787 major_copy_or_mark_from_roots (old_next_pin_slot, concurrent, FALSE, FALSE, FALSE);
2791 wait_for_workers_to_finish (void)
2793 while (!sgen_workers_all_done ())
2798 major_finish_collection (const char *reason, size_t old_next_pin_slot, gboolean scan_mod_union, gboolean scan_whole_nursery)
2800 ScannedObjectCounts counts;
2801 LOSObject *bigobj, *prevbo;
2807 if (concurrent_collection_in_progress) {
2808 sgen_workers_signal_start_nursery_collection_and_wait ();
2810 current_object_ops = major_collector.major_concurrent_ops;
2812 major_copy_or_mark_from_roots (NULL, FALSE, TRUE, scan_mod_union, scan_whole_nursery);
2814 sgen_workers_signal_finish_nursery_collection ();
2815 gray_queue_enable_redirect (WORKERS_DISTRIBUTE_GRAY_QUEUE);
2817 sgen_workers_join ();
2819 SGEN_ASSERT (0, sgen_gray_object_queue_is_empty (&gray_queue), "Why is the gray queue not empty after workers have finished working?");
2821 #ifdef SGEN_DEBUG_INTERNAL_ALLOC
2822 main_gc_thread = NULL;
2825 if (do_concurrent_checks)
2826 check_nursery_is_clean ();
2828 SGEN_ASSERT (0, !scan_whole_nursery, "scan_whole_nursery only applies to concurrent collections");
2829 current_object_ops = major_collector.major_ops;
2833 * The workers have stopped so we need to finish gray queue
2834 * work that might result from finalization in the main GC
2835 * thread. Redirection must therefore be turned off.
2837 sgen_gray_object_queue_disable_alloc_prepare (&gray_queue);
2838 g_assert (sgen_section_gray_queue_is_empty (sgen_workers_get_distribute_section_gray_queue ()));
2840 /* all the objects in the heap */
2841 finish_gray_stack (GENERATION_OLD, &gray_queue);
2843 time_major_finish_gray_stack += TV_ELAPSED (btv, atv);
2845 SGEN_ASSERT (0, sgen_workers_all_done (), "Can't have workers working after joining");
2848 * The (single-threaded) finalization code might have done
2849 * some copying/marking so we can only reset the GC thread's
2850 * worker data here instead of earlier when we joined the
2853 sgen_workers_reset_data ();
2855 if (objects_pinned) {
2856 g_assert (!concurrent_collection_in_progress);
2859 * This is slow, but we just OOM'd.
2861 * See comment at `sgen_pin_queue_clear_discarded_entries` for how the pin
2862 * queue is laid out at this point.
2864 sgen_pin_queue_clear_discarded_entries (nursery_section, old_next_pin_slot);
2866 * We need to reestablish all pinned nursery objects in the pin queue
2867 * because they're needed for fragment creation. Unpinning happens by
2868 * walking the whole queue, so it's not necessary to reestablish where major
2869 * heap block pins are - all we care is that they're still in there
2872 sgen_optimize_pin_queue ();
2873 sgen_find_section_pin_queue_start_end (nursery_section);
2877 reset_heap_boundaries ();
2878 sgen_update_heap_boundaries ((mword)sgen_get_nursery_start (), (mword)sgen_get_nursery_end ());
2880 if (!concurrent_collection_in_progress) {
2881 /* walk the pin_queue, build up the fragment list of free memory, unmark
2882 * pinned objects as we go, memzero() the empty fragments so they are ready for the
2885 if (!sgen_build_nursery_fragments (nursery_section, NULL))
2888 /* prepare the pin queue for the next collection */
2889 sgen_finish_pinning ();
2891 /* Clear TLABs for all threads */
2892 sgen_clear_tlabs ();
2894 sgen_pin_stats_reset ();
2897 if (concurrent_collection_in_progress)
2898 sgen_cement_concurrent_finish ();
2899 sgen_cement_clear_below_threshold ();
2901 if (check_mark_bits_after_major_collection)
2902 sgen_check_heap_marked (concurrent_collection_in_progress);
2905 time_major_fragment_creation += TV_ELAPSED (atv, btv);
2908 MONO_GC_SWEEP_BEGIN (GENERATION_OLD, !major_collector.sweeps_lazily);
2910 /* sweep the big objects list */
2912 for (bigobj = los_object_list; bigobj;) {
2913 g_assert (!object_is_pinned (bigobj->data));
2914 if (sgen_los_object_is_pinned (bigobj->data)) {
2915 sgen_los_unpin_object (bigobj->data);
2916 sgen_update_heap_boundaries ((mword)bigobj->data, (mword)bigobj->data + sgen_los_object_size (bigobj));
2919 /* not referenced anywhere, so we can free it */
2921 prevbo->next = bigobj->next;
2923 los_object_list = bigobj->next;
2925 bigobj = bigobj->next;
2926 sgen_los_free_object (to_free);
2930 bigobj = bigobj->next;
2934 time_major_free_bigobjs += TV_ELAPSED (btv, atv);
2939 time_major_los_sweep += TV_ELAPSED (atv, btv);
2941 major_collector.sweep ();
2943 MONO_GC_SWEEP_END (GENERATION_OLD, !major_collector.sweeps_lazily);
2946 time_major_sweep += TV_ELAPSED (btv, atv);
2949 dump_heap ("major", gc_stats.major_gc_count - 1, reason);
2951 if (fin_ready_list || critical_fin_list) {
2952 SGEN_LOG (4, "Finalizer-thread wakeup: ready %d", num_ready_finalizers);
2953 mono_gc_finalize_notify ();
2956 g_assert (sgen_gray_object_queue_is_empty (&gray_queue));
2958 sgen_memgov_major_collection_end ();
2959 current_collection_generation = -1;
2961 memset (&counts, 0, sizeof (ScannedObjectCounts));
2962 major_collector.finish_major_collection (&counts);
2964 g_assert (sgen_section_gray_queue_is_empty (sgen_workers_get_distribute_section_gray_queue ()));
2966 SGEN_ASSERT (0, sgen_workers_all_done (), "Can't have workers working after major collection has finished");
2967 if (concurrent_collection_in_progress)
2968 concurrent_collection_in_progress = FALSE;
2970 check_scan_starts ();
2972 binary_protocol_flush_buffers (FALSE);
2974 //consistency_check ();
2976 MONO_GC_END (GENERATION_OLD);
2977 binary_protocol_collection_end (gc_stats.major_gc_count - 1, GENERATION_OLD, counts.num_scanned_objects, counts.num_unique_scanned_objects);
2981 major_do_collection (const char *reason)
2983 TV_DECLARE (time_start);
2984 TV_DECLARE (time_end);
2985 size_t old_next_pin_slot;
2987 if (disable_major_collections)
2990 if (major_collector.get_and_reset_num_major_objects_marked) {
2991 long long num_marked = major_collector.get_and_reset_num_major_objects_marked ();
2992 g_assert (!num_marked);
2995 /* world must be stopped already */
2996 TV_GETTIME (time_start);
2998 major_start_collection (FALSE, &old_next_pin_slot);
2999 major_finish_collection (reason, old_next_pin_slot, FALSE, FALSE);
3001 TV_GETTIME (time_end);
3002 gc_stats.major_gc_time += TV_ELAPSED (time_start, time_end);
3004 /* FIXME: also report this to the user, preferably in gc-end. */
3005 if (major_collector.get_and_reset_num_major_objects_marked)
3006 major_collector.get_and_reset_num_major_objects_marked ();
3008 return bytes_pinned_from_failed_allocation > 0;
3012 major_start_concurrent_collection (const char *reason)
3014 TV_DECLARE (time_start);
3015 TV_DECLARE (time_end);
3016 long long num_objects_marked;
3018 if (disable_major_collections)
3021 TV_GETTIME (time_start);
3022 SGEN_TV_GETTIME (time_major_conc_collection_start);
3024 num_objects_marked = major_collector.get_and_reset_num_major_objects_marked ();
3025 g_assert (num_objects_marked == 0);
3027 MONO_GC_CONCURRENT_START_BEGIN (GENERATION_OLD);
3028 binary_protocol_concurrent_start ();
3030 // FIXME: store reason and pass it when finishing
3031 major_start_collection (TRUE, NULL);
3033 gray_queue_redirect (&gray_queue);
3034 sgen_workers_wait_for_jobs_finished ();
3036 num_objects_marked = major_collector.get_and_reset_num_major_objects_marked ();
3037 MONO_GC_CONCURRENT_START_END (GENERATION_OLD, num_objects_marked);
3039 TV_GETTIME (time_end);
3040 gc_stats.major_gc_time += TV_ELAPSED (time_start, time_end);
3042 current_collection_generation = -1;
3046 * Returns whether the major collection has finished.
3049 major_should_finish_concurrent_collection (void)
3051 SGEN_ASSERT (0, sgen_gray_object_queue_is_empty (&gray_queue), "Why is the gray queue not empty before we have started doing anything?");
3052 return sgen_workers_all_done ();
3056 major_update_concurrent_collection (void)
3058 TV_DECLARE (total_start);
3059 TV_DECLARE (total_end);
3061 TV_GETTIME (total_start);
3063 MONO_GC_CONCURRENT_UPDATE_FINISH_BEGIN (GENERATION_OLD, major_collector.get_and_reset_num_major_objects_marked ());
3064 binary_protocol_concurrent_update ();
3066 major_collector.update_cardtable_mod_union ();
3067 sgen_los_update_cardtable_mod_union ();
3069 MONO_GC_CONCURRENT_UPDATE_END (GENERATION_OLD, major_collector.get_and_reset_num_major_objects_marked ());
3071 TV_GETTIME (total_end);
3072 gc_stats.major_gc_time += TV_ELAPSED (total_start, total_end);
3076 major_finish_concurrent_collection (void)
3078 TV_DECLARE (total_start);
3079 TV_DECLARE (total_end);
3080 gboolean late_pinned;
3081 SgenGrayQueue unpin_queue;
3082 memset (&unpin_queue, 0, sizeof (unpin_queue));
3084 TV_GETTIME (total_start);
3086 MONO_GC_CONCURRENT_UPDATE_FINISH_BEGIN (GENERATION_OLD, major_collector.get_and_reset_num_major_objects_marked ());
3087 binary_protocol_concurrent_finish ();
3090 * The major collector can add global remsets which are processed in the finishing
3091 * nursery collection, below. That implies that the workers must have finished
3092 * marking before the nursery collection is allowed to run, otherwise we might miss
3095 wait_for_workers_to_finish ();
3097 SGEN_TV_GETTIME (time_major_conc_collection_end);
3098 gc_stats.major_gc_time_concurrent += SGEN_TV_ELAPSED (time_major_conc_collection_start, time_major_conc_collection_end);
3100 major_collector.update_cardtable_mod_union ();
3101 sgen_los_update_cardtable_mod_union ();
3103 late_pinned = collect_nursery (&unpin_queue, TRUE);
3105 if (mod_union_consistency_check)
3106 sgen_check_mod_union_consistency ();
3108 current_collection_generation = GENERATION_OLD;
3109 major_finish_collection ("finishing", -1, TRUE, late_pinned);
3111 if (whole_heap_check_before_collection)
3112 sgen_check_whole_heap (FALSE);
3114 unpin_objects_from_queue (&unpin_queue);
3115 sgen_gray_object_queue_deinit (&unpin_queue);
3117 MONO_GC_CONCURRENT_FINISH_END (GENERATION_OLD, major_collector.get_and_reset_num_major_objects_marked ());
3119 TV_GETTIME (total_end);
3120 gc_stats.major_gc_time += TV_ELAPSED (total_start, total_end) - TV_ELAPSED (last_minor_collection_start_tv, last_minor_collection_end_tv);
3122 current_collection_generation = -1;
3126 * Ensure an allocation request for @size will succeed by freeing enough memory.
3128 * LOCKING: The GC lock MUST be held.
3131 sgen_ensure_free_space (size_t size)
3133 int generation_to_collect = -1;
3134 const char *reason = NULL;
3137 if (size > SGEN_MAX_SMALL_OBJ_SIZE) {
3138 if (sgen_need_major_collection (size)) {
3139 reason = "LOS overflow";
3140 generation_to_collect = GENERATION_OLD;
3143 if (degraded_mode) {
3144 if (sgen_need_major_collection (size)) {
3145 reason = "Degraded mode overflow";
3146 generation_to_collect = GENERATION_OLD;
3148 } else if (sgen_need_major_collection (size)) {
3149 reason = "Minor allowance";
3150 generation_to_collect = GENERATION_OLD;
3152 generation_to_collect = GENERATION_NURSERY;
3153 reason = "Nursery full";
3157 if (generation_to_collect == -1) {
3158 if (concurrent_collection_in_progress && sgen_workers_all_done ()) {
3159 generation_to_collect = GENERATION_OLD;
3160 reason = "Finish concurrent collection";
3164 if (generation_to_collect == -1)
3166 sgen_perform_collection (size, generation_to_collect, reason, FALSE);
3170 * LOCKING: Assumes the GC lock is held.
3173 sgen_perform_collection (size_t requested_size, int generation_to_collect, const char *reason, gboolean wait_to_finish)
3175 TV_DECLARE (gc_start);
3176 TV_DECLARE (gc_end);
3177 TV_DECLARE (gc_total_start);
3178 TV_DECLARE (gc_total_end);
3179 GGTimingInfo infos [2];
3180 int overflow_generation_to_collect = -1;
3181 int oldest_generation_collected = generation_to_collect;
3182 const char *overflow_reason = NULL;
3184 MONO_GC_REQUESTED (generation_to_collect, requested_size, wait_to_finish ? 1 : 0);
3186 binary_protocol_collection_force (generation_to_collect);
3188 SGEN_ASSERT (0, generation_to_collect == GENERATION_NURSERY || generation_to_collect == GENERATION_OLD, "What generation is this?");
3190 mono_profiler_gc_event (MONO_GC_EVENT_START, generation_to_collect);
3192 TV_GETTIME (gc_start);
3194 sgen_stop_world (generation_to_collect);
3196 TV_GETTIME (gc_total_start);
3198 if (concurrent_collection_in_progress) {
3200 * We update the concurrent collection. If it finished, we're done. If
3201 * not, and we've been asked to do a nursery collection, we do that.
3203 gboolean finish = major_should_finish_concurrent_collection () || (wait_to_finish && generation_to_collect == GENERATION_OLD);
3206 major_finish_concurrent_collection ();
3207 oldest_generation_collected = GENERATION_OLD;
3209 sgen_workers_signal_start_nursery_collection_and_wait ();
3211 major_update_concurrent_collection ();
3212 if (generation_to_collect == GENERATION_NURSERY)
3213 collect_nursery (NULL, FALSE);
3215 sgen_workers_signal_finish_nursery_collection ();
3222 * If we've been asked to do a major collection, and the major collector wants to
3223 * run synchronously (to evacuate), we set the flag to do that.
3225 if (generation_to_collect == GENERATION_OLD &&
3226 allow_synchronous_major &&
3227 major_collector.want_synchronous_collection &&
3228 *major_collector.want_synchronous_collection) {
3229 wait_to_finish = TRUE;
3232 SGEN_ASSERT (0, !concurrent_collection_in_progress, "Why did this not get handled above?");
3235 * There's no concurrent collection in progress. Collect the generation we're asked
3236 * to collect. If the major collector is concurrent and we're not forced to wait,
3237 * start a concurrent collection.
3239 // FIXME: extract overflow reason
3240 if (generation_to_collect == GENERATION_NURSERY) {
3241 if (collect_nursery (NULL, FALSE)) {
3242 overflow_generation_to_collect = GENERATION_OLD;
3243 overflow_reason = "Minor overflow";
3246 if (major_collector.is_concurrent && !wait_to_finish) {
3247 collect_nursery (NULL, FALSE);
3248 major_start_concurrent_collection (reason);
3249 // FIXME: set infos[0] properly
3253 if (major_do_collection (reason)) {
3254 overflow_generation_to_collect = GENERATION_NURSERY;
3255 overflow_reason = "Excessive pinning";
3259 TV_GETTIME (gc_end);
3261 memset (infos, 0, sizeof (infos));
3262 infos [0].generation = generation_to_collect;
3263 infos [0].reason = reason;
3264 infos [0].is_overflow = FALSE;
3265 infos [1].generation = -1;
3266 infos [0].total_time = SGEN_TV_ELAPSED (gc_start, gc_end);
3268 SGEN_ASSERT (0, !concurrent_collection_in_progress, "Why did this not get handled above?");
3270 if (overflow_generation_to_collect != -1) {
3272 * We need to do an overflow collection, either because we ran out of memory
3273 * or the nursery is fully pinned.
3276 mono_profiler_gc_event (MONO_GC_EVENT_START, overflow_generation_to_collect);
3277 infos [1].generation = overflow_generation_to_collect;
3278 infos [1].reason = overflow_reason;
3279 infos [1].is_overflow = TRUE;
3280 infos [1].total_time = gc_end;
3282 if (overflow_generation_to_collect == GENERATION_NURSERY)
3283 collect_nursery (NULL, FALSE);
3285 major_do_collection (overflow_reason);
3287 TV_GETTIME (gc_end);
3288 infos [1].total_time = SGEN_TV_ELAPSED (infos [1].total_time, gc_end);
3290 /* keep events symmetric */
3291 mono_profiler_gc_event (MONO_GC_EVENT_END, overflow_generation_to_collect);
3293 oldest_generation_collected = MAX (oldest_generation_collected, overflow_generation_to_collect);
3296 SGEN_LOG (2, "Heap size: %lu, LOS size: %lu", (unsigned long)mono_gc_get_heap_size (), (unsigned long)los_memory_usage);
3298 /* this also sets the proper pointers for the next allocation */
3299 if (generation_to_collect == GENERATION_NURSERY && !sgen_can_alloc_size (requested_size)) {
3300 /* TypeBuilder and MonoMethod are killing mcs with fragmentation */
3301 SGEN_LOG (1, "nursery collection didn't find enough room for %zd alloc (%zd pinned)", requested_size, sgen_get_pinned_count ());
3302 sgen_dump_pin_queue ();
3307 g_assert (sgen_gray_object_queue_is_empty (&gray_queue));
3309 TV_GETTIME (gc_total_end);
3310 time_max = MAX (time_max, TV_ELAPSED (gc_total_start, gc_total_end));
3312 sgen_restart_world (oldest_generation_collected, infos);
3314 mono_profiler_gc_event (MONO_GC_EVENT_END, generation_to_collect);
3318 * ######################################################################
3319 * ######## Memory allocation from the OS
3320 * ######################################################################
3321 * This section of code deals with getting memory from the OS and
3322 * allocating memory for GC-internal data structures.
3323 * Internal memory can be handled with a freelist for small objects.
3329 G_GNUC_UNUSED static void
3330 report_internal_mem_usage (void)
3332 printf ("Internal memory usage:\n");
3333 sgen_report_internal_mem_usage ();
3334 printf ("Pinned memory usage:\n");
3335 major_collector.report_pinned_memory_usage ();
3339 * ######################################################################
3340 * ######## Finalization support
3341 * ######################################################################
3344 static inline gboolean
3345 sgen_major_is_object_alive (void *object)
3349 /* Oldgen objects can be pinned and forwarded too */
3350 if (SGEN_OBJECT_IS_PINNED (object) || SGEN_OBJECT_IS_FORWARDED (object))
3354 * FIXME: major_collector.is_object_live() also calculates the
3355 * size. Avoid the double calculation.
3357 objsize = SGEN_ALIGN_UP (sgen_safe_object_get_size ((MonoObject*)object));
3358 if (objsize > SGEN_MAX_SMALL_OBJ_SIZE)
3359 return sgen_los_object_is_pinned (object);
3361 return major_collector.is_object_live (object);
3365 * If the object has been forwarded it means it's still referenced from a root.
3366 * If it is pinned it's still alive as well.
3367 * A LOS object is only alive if we have pinned it.
3368 * Return TRUE if @obj is ready to be finalized.
3370 static inline gboolean
3371 sgen_is_object_alive (void *object)
3373 if (ptr_in_nursery (object))
3374 return sgen_nursery_is_object_alive (object);
3376 return sgen_major_is_object_alive (object);
3380 * This function returns true if @object is either alive or it belongs to the old gen
3381 * and we're currently doing a minor collection.
3384 sgen_is_object_alive_for_current_gen (char *object)
3386 if (ptr_in_nursery (object))
3387 return sgen_nursery_is_object_alive (object);
3389 if (current_collection_generation == GENERATION_NURSERY)
3392 return sgen_major_is_object_alive (object);
3396 * This function returns true if @object is either alive and belongs to the
3397 * current collection - major collections are full heap, so old gen objects
3398 * are never alive during a minor collection.
3401 sgen_is_object_alive_and_on_current_collection (char *object)
3403 if (ptr_in_nursery (object))
3404 return sgen_nursery_is_object_alive (object);
3406 if (current_collection_generation == GENERATION_NURSERY)
3409 return sgen_major_is_object_alive (object);
3414 sgen_gc_is_object_ready_for_finalization (void *object)
3416 return !sgen_is_object_alive (object);
3420 has_critical_finalizer (MonoObject *obj)
3424 if (!mono_defaults.critical_finalizer_object)
3427 class = ((MonoVTable*)LOAD_VTABLE (obj))->klass;
3429 return mono_class_has_parent_fast (class, mono_defaults.critical_finalizer_object);
3433 is_finalization_aware (MonoObject *obj)
3435 MonoVTable *vt = ((MonoVTable*)LOAD_VTABLE (obj));
3436 return (vt->gc_bits & SGEN_GC_BIT_FINALIZER_AWARE) == SGEN_GC_BIT_FINALIZER_AWARE;
3440 sgen_queue_finalization_entry (MonoObject *obj)
3442 FinalizeReadyEntry *entry = sgen_alloc_internal (INTERNAL_MEM_FINALIZE_READY_ENTRY);
3443 gboolean critical = has_critical_finalizer (obj);
3444 entry->object = obj;
3446 entry->next = critical_fin_list;
3447 critical_fin_list = entry;
3449 entry->next = fin_ready_list;
3450 fin_ready_list = entry;
3453 if (fin_callbacks.object_queued_for_finalization && is_finalization_aware (obj))
3454 fin_callbacks.object_queued_for_finalization (obj);
3456 #ifdef ENABLE_DTRACE
3457 if (G_UNLIKELY (MONO_GC_FINALIZE_ENQUEUE_ENABLED ())) {
3458 int gen = sgen_ptr_in_nursery (obj) ? GENERATION_NURSERY : GENERATION_OLD;
3459 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (obj);
3460 MONO_GC_FINALIZE_ENQUEUE ((mword)obj, sgen_safe_object_get_size (obj),
3461 vt->klass->name_space, vt->klass->name, gen, critical);
3467 sgen_object_is_live (void *obj)
3469 return sgen_is_object_alive_and_on_current_collection (obj);
3472 /* LOCKING: requires that the GC lock is held */
3474 null_ephemerons_for_domain (MonoDomain *domain)
3476 EphemeronLinkNode *current = ephemeron_list, *prev = NULL;
3479 MonoObject *object = (MonoObject*)current->array;
3481 if (object && !object->vtable) {
3482 EphemeronLinkNode *tmp = current;
3485 prev->next = current->next;
3487 ephemeron_list = current->next;
3489 current = current->next;
3490 sgen_free_internal (tmp, INTERNAL_MEM_EPHEMERON_LINK);
3493 current = current->next;
3498 /* LOCKING: requires that the GC lock is held */
3500 clear_unreachable_ephemerons (ScanCopyContext ctx)
3502 CopyOrMarkObjectFunc copy_func = ctx.copy_func;
3503 GrayQueue *queue = ctx.queue;
3504 EphemeronLinkNode *current = ephemeron_list, *prev = NULL;
3506 Ephemeron *cur, *array_end;
3510 char *object = current->array;
3512 if (!sgen_is_object_alive_for_current_gen (object)) {
3513 EphemeronLinkNode *tmp = current;
3515 SGEN_LOG (5, "Dead Ephemeron array at %p", object);
3518 prev->next = current->next;
3520 ephemeron_list = current->next;
3522 current = current->next;
3523 sgen_free_internal (tmp, INTERNAL_MEM_EPHEMERON_LINK);
3528 copy_func ((void**)&object, queue);
3529 current->array = object;
3531 SGEN_LOG (5, "Clearing unreachable entries for ephemeron array at %p", object);
3533 array = (MonoArray*)object;
3534 cur = mono_array_addr (array, Ephemeron, 0);
3535 array_end = cur + mono_array_length_fast (array);
3536 tombstone = (char*)((MonoVTable*)LOAD_VTABLE (object))->domain->ephemeron_tombstone;
3538 for (; cur < array_end; ++cur) {
3539 char *key = (char*)cur->key;
3541 if (!key || key == tombstone)
3544 SGEN_LOG (5, "[%td] key %p (%s) value %p (%s)", cur - mono_array_addr (array, Ephemeron, 0),
3545 key, sgen_is_object_alive_for_current_gen (key) ? "reachable" : "unreachable",
3546 cur->value, cur->value && sgen_is_object_alive_for_current_gen (cur->value) ? "reachable" : "unreachable");
3548 if (!sgen_is_object_alive_for_current_gen (key)) {
3549 cur->key = tombstone;
3555 current = current->next;
3560 LOCKING: requires that the GC lock is held
3562 Limitations: We scan all ephemerons on every collection since the current design doesn't allow for a simple nursery/mature split.
3565 mark_ephemerons_in_range (ScanCopyContext ctx)
3567 CopyOrMarkObjectFunc copy_func = ctx.copy_func;
3568 GrayQueue *queue = ctx.queue;
3569 int nothing_marked = 1;
3570 EphemeronLinkNode *current = ephemeron_list;
3572 Ephemeron *cur, *array_end;
3575 for (current = ephemeron_list; current; current = current->next) {
3576 char *object = current->array;
3577 SGEN_LOG (5, "Ephemeron array at %p", object);
3579 /*It has to be alive*/
3580 if (!sgen_is_object_alive_for_current_gen (object)) {
3581 SGEN_LOG (5, "\tnot reachable");
3585 copy_func ((void**)&object, queue);
3587 array = (MonoArray*)object;
3588 cur = mono_array_addr (array, Ephemeron, 0);
3589 array_end = cur + mono_array_length_fast (array);
3590 tombstone = (char*)((MonoVTable*)LOAD_VTABLE (object))->domain->ephemeron_tombstone;
3592 for (; cur < array_end; ++cur) {
3593 char *key = cur->key;
3595 if (!key || key == tombstone)
3598 SGEN_LOG (5, "[%td] key %p (%s) value %p (%s)", cur - mono_array_addr (array, Ephemeron, 0),
3599 key, sgen_is_object_alive_for_current_gen (key) ? "reachable" : "unreachable",
3600 cur->value, cur->value && sgen_is_object_alive_for_current_gen (cur->value) ? "reachable" : "unreachable");
3602 if (sgen_is_object_alive_for_current_gen (key)) {
3603 char *value = cur->value;
3605 copy_func ((void**)&cur->key, queue);
3607 if (!sgen_is_object_alive_for_current_gen (value))
3609 copy_func ((void**)&cur->value, queue);
3615 SGEN_LOG (5, "Ephemeron run finished. Is it done %d", nothing_marked);
3616 return nothing_marked;
3620 mono_gc_invoke_finalizers (void)
3622 FinalizeReadyEntry *entry = NULL;
3623 gboolean entry_is_critical = FALSE;
3626 /* FIXME: batch to reduce lock contention */
3627 while (fin_ready_list || critical_fin_list) {
3631 FinalizeReadyEntry **list = entry_is_critical ? &critical_fin_list : &fin_ready_list;
3633 /* We have finalized entry in the last
3634 interation, now we need to remove it from
3637 *list = entry->next;
3639 FinalizeReadyEntry *e = *list;
3640 while (e->next != entry)
3642 e->next = entry->next;
3644 sgen_free_internal (entry, INTERNAL_MEM_FINALIZE_READY_ENTRY);
3648 /* Now look for the first non-null entry. */
3649 for (entry = fin_ready_list; entry && !entry->object; entry = entry->next)
3652 entry_is_critical = FALSE;
3654 entry_is_critical = TRUE;
3655 for (entry = critical_fin_list; entry && !entry->object; entry = entry->next)
3660 g_assert (entry->object);
3661 num_ready_finalizers--;
3662 obj = entry->object;
3663 entry->object = NULL;
3664 SGEN_LOG (7, "Finalizing object %p (%s)", obj, safe_name (obj));
3672 g_assert (entry->object == NULL);
3674 /* the object is on the stack so it is pinned */
3675 /*g_print ("Calling finalizer for object: %p (%s)\n", entry->object, safe_name (entry->object));*/
3676 mono_gc_run_finalize (obj, NULL);
3683 mono_gc_pending_finalizers (void)
3685 return fin_ready_list || critical_fin_list;
3689 * ######################################################################
3690 * ######## registered roots support
3691 * ######################################################################
3695 * We do not coalesce roots.
3698 mono_gc_register_root_inner (char *start, size_t size, void *descr, int root_type)
3700 RootRecord new_root;
3703 for (i = 0; i < ROOT_TYPE_NUM; ++i) {
3704 RootRecord *root = sgen_hash_table_lookup (&roots_hash [i], start);
3705 /* we allow changing the size and the descriptor (for thread statics etc) */
3707 size_t old_size = root->end_root - start;
3708 root->end_root = start + size;
3709 g_assert (((root->root_desc != 0) && (descr != NULL)) ||
3710 ((root->root_desc == 0) && (descr == NULL)));
3711 root->root_desc = (mword)descr;
3713 roots_size -= old_size;
3719 new_root.end_root = start + size;
3720 new_root.root_desc = (mword)descr;
3722 sgen_hash_table_replace (&roots_hash [root_type], start, &new_root, NULL);
3725 SGEN_LOG (3, "Added root for range: %p-%p, descr: %p (%d/%d bytes)", start, new_root.end_root, descr, (int)size, (int)roots_size);
3732 mono_gc_register_root (char *start, size_t size, void *descr)
3734 return mono_gc_register_root_inner (start, size, descr, descr ? ROOT_TYPE_NORMAL : ROOT_TYPE_PINNED);
3738 mono_gc_register_root_wbarrier (char *start, size_t size, void *descr)
3740 return mono_gc_register_root_inner (start, size, descr, ROOT_TYPE_WBARRIER);
3744 mono_gc_deregister_root (char* addr)
3750 for (root_type = 0; root_type < ROOT_TYPE_NUM; ++root_type) {
3751 if (sgen_hash_table_remove (&roots_hash [root_type], addr, &root))
3752 roots_size -= (root.end_root - addr);
3758 * ######################################################################
3759 * ######## Thread handling (stop/start code)
3760 * ######################################################################
3763 unsigned int sgen_global_stop_count = 0;
3766 sgen_get_current_collection_generation (void)
3768 return current_collection_generation;
3772 mono_gc_set_gc_callbacks (MonoGCCallbacks *callbacks)
3774 gc_callbacks = *callbacks;
3778 mono_gc_get_gc_callbacks ()
3780 return &gc_callbacks;
3783 /* Variables holding start/end nursery so it won't have to be passed at every call */
3784 static void *scan_area_arg_start, *scan_area_arg_end;
3787 mono_gc_conservatively_scan_area (void *start, void *end)
3789 conservatively_pin_objects_from (start, end, scan_area_arg_start, scan_area_arg_end, PIN_TYPE_STACK);
3793 mono_gc_scan_object (void *obj, void *gc_data)
3795 UserCopyOrMarkData *data = gc_data;
3796 current_object_ops.copy_or_mark_object (&obj, data->queue);
3801 * Mark from thread stacks and registers.
3804 scan_thread_data (void *start_nursery, void *end_nursery, gboolean precise, GrayQueue *queue)
3806 SgenThreadInfo *info;
3808 scan_area_arg_start = start_nursery;
3809 scan_area_arg_end = end_nursery;
3811 FOREACH_THREAD (info) {
3813 SGEN_LOG (3, "Skipping dead thread %p, range: %p-%p, size: %td", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start);
3816 if (info->gc_disabled) {
3817 SGEN_LOG (3, "GC disabled for thread %p, range: %p-%p, size: %td", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start);
3820 if (mono_thread_info_run_state (info) != STATE_RUNNING) {
3821 SGEN_LOG (3, "Skipping non-running thread %p, range: %p-%p, size: %td (state %d)", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start, mono_thread_info_run_state (info));
3824 SGEN_LOG (3, "Scanning thread %p, range: %p-%p, size: %td, pinned=%zd", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start, sgen_get_pinned_count ());
3825 if (gc_callbacks.thread_mark_func && !conservative_stack_mark) {
3826 UserCopyOrMarkData data = { NULL, queue };
3827 gc_callbacks.thread_mark_func (info->runtime_data, info->stack_start, info->stack_end, precise, &data);
3828 } else if (!precise) {
3829 if (!conservative_stack_mark) {
3830 fprintf (stderr, "Precise stack mark not supported - disabling.\n");
3831 conservative_stack_mark = TRUE;
3833 conservatively_pin_objects_from (info->stack_start, info->stack_end, start_nursery, end_nursery, PIN_TYPE_STACK);
3838 conservatively_pin_objects_from ((void**)&info->ctx, (void**)&info->ctx + ARCH_NUM_REGS,
3839 start_nursery, end_nursery, PIN_TYPE_STACK);
3841 conservatively_pin_objects_from ((void**)&info->regs, (void**)&info->regs + ARCH_NUM_REGS,
3842 start_nursery, end_nursery, PIN_TYPE_STACK);
3845 } END_FOREACH_THREAD
3849 ptr_on_stack (void *ptr)
3851 gpointer stack_start = &stack_start;
3852 SgenThreadInfo *info = mono_thread_info_current ();
3854 if (ptr >= stack_start && ptr < (gpointer)info->stack_end)
3860 sgen_thread_register (SgenThreadInfo* info, void *addr)
3863 guint8 *staddr = NULL;
3865 #ifndef HAVE_KW_THREAD
3866 info->tlab_start = info->tlab_next = info->tlab_temp_end = info->tlab_real_end = NULL;
3868 g_assert (!mono_native_tls_get_value (thread_info_key));
3869 mono_native_tls_set_value (thread_info_key, info);
3871 sgen_thread_info = info;
3874 #ifdef SGEN_POSIX_STW
3875 info->stop_count = -1;
3879 info->stack_start = NULL;
3880 info->stopped_ip = NULL;
3881 info->stopped_domain = NULL;
3883 memset (&info->ctx, 0, sizeof (MonoContext));
3885 memset (&info->regs, 0, sizeof (info->regs));
3888 sgen_init_tlab_info (info);
3890 binary_protocol_thread_register ((gpointer)mono_thread_info_get_tid (info));
3892 /* On win32, stack_start_limit should be 0, since the stack can grow dynamically */
3893 mono_thread_info_get_stack_bounds (&staddr, &stsize);
3896 info->stack_start_limit = staddr;
3898 info->stack_end = staddr + stsize;
3900 gsize stack_bottom = (gsize)addr;
3901 stack_bottom += 4095;
3902 stack_bottom &= ~4095;
3903 info->stack_end = (char*)stack_bottom;
3906 #ifdef HAVE_KW_THREAD
3907 stack_end = info->stack_end;
3910 SGEN_LOG (3, "registered thread %p (%p) stack end %p", info, (gpointer)mono_thread_info_get_tid (info), info->stack_end);
3912 if (gc_callbacks.thread_attach_func)
3913 info->runtime_data = gc_callbacks.thread_attach_func ();
3918 sgen_thread_detach (SgenThreadInfo *p)
3920 /* If a delegate is passed to native code and invoked on a thread we dont
3921 * know about, the jit will register it with mono_jit_thread_attach, but
3922 * we have no way of knowing when that thread goes away. SGen has a TSD
3923 * so we assume that if the domain is still registered, we can detach
3926 if (mono_domain_get ())
3927 mono_thread_detach_internal (mono_thread_internal_current ());
3931 sgen_thread_unregister (SgenThreadInfo *p)
3933 MonoNativeThreadId tid;
3935 tid = mono_thread_info_get_tid (p);
3936 binary_protocol_thread_unregister ((gpointer)tid);
3937 SGEN_LOG (3, "unregister thread %p (%p)", p, (gpointer)tid);
3939 #ifndef HAVE_KW_THREAD
3940 mono_native_tls_set_value (thread_info_key, NULL);
3942 sgen_thread_info = NULL;
3945 if (p->info.runtime_thread)
3946 mono_threads_add_joinable_thread ((gpointer)tid);
3948 if (gc_callbacks.thread_detach_func) {
3949 gc_callbacks.thread_detach_func (p->runtime_data);
3950 p->runtime_data = NULL;
3956 sgen_thread_attach (SgenThreadInfo *info)
3959 /*this is odd, can we get attached before the gc is inited?*/
3963 if (gc_callbacks.thread_attach_func && !info->runtime_data)
3964 info->runtime_data = gc_callbacks.thread_attach_func ();
3967 mono_gc_register_thread (void *baseptr)
3969 return mono_thread_info_attach (baseptr) != NULL;
3973 * mono_gc_set_stack_end:
3975 * Set the end of the current threads stack to STACK_END. The stack space between
3976 * STACK_END and the real end of the threads stack will not be scanned during collections.
3979 mono_gc_set_stack_end (void *stack_end)
3981 SgenThreadInfo *info;
3984 info = mono_thread_info_current ();
3986 g_assert (stack_end < info->stack_end);
3987 info->stack_end = stack_end;
3992 #if USE_PTHREAD_INTERCEPT
3996 mono_gc_pthread_create (pthread_t *new_thread, const pthread_attr_t *attr, void *(*start_routine)(void *), void *arg)
3998 return pthread_create (new_thread, attr, start_routine, arg);
4002 mono_gc_pthread_join (pthread_t thread, void **retval)
4004 return pthread_join (thread, retval);
4008 mono_gc_pthread_detach (pthread_t thread)
4010 return pthread_detach (thread);
4014 mono_gc_pthread_exit (void *retval)
4016 mono_thread_info_detach ();
4017 pthread_exit (retval);
4018 g_assert_not_reached ();
4021 #endif /* USE_PTHREAD_INTERCEPT */
4024 * ######################################################################
4025 * ######## Write barriers
4026 * ######################################################################
4030 * Note: the write barriers first do the needed GC work and then do the actual store:
4031 * this way the value is visible to the conservative GC scan after the write barrier
4032 * itself. If a GC interrupts the barrier in the middle, value will be kept alive by
4033 * the conservative scan, otherwise by the remembered set scan.
4036 mono_gc_wbarrier_set_field (MonoObject *obj, gpointer field_ptr, MonoObject* value)
4038 HEAVY_STAT (++stat_wbarrier_set_field);
4039 if (ptr_in_nursery (field_ptr)) {
4040 *(void**)field_ptr = value;
4043 SGEN_LOG (8, "Adding remset at %p", field_ptr);
4045 binary_protocol_wbarrier (field_ptr, value, value->vtable);
4047 remset.wbarrier_set_field (obj, field_ptr, value);
4051 mono_gc_wbarrier_set_arrayref (MonoArray *arr, gpointer slot_ptr, MonoObject* value)
4053 HEAVY_STAT (++stat_wbarrier_set_arrayref);
4054 if (ptr_in_nursery (slot_ptr)) {
4055 *(void**)slot_ptr = value;
4058 SGEN_LOG (8, "Adding remset at %p", slot_ptr);
4060 binary_protocol_wbarrier (slot_ptr, value, value->vtable);
4062 remset.wbarrier_set_arrayref (arr, slot_ptr, value);
4066 mono_gc_wbarrier_arrayref_copy (gpointer dest_ptr, gpointer src_ptr, int count)
4068 HEAVY_STAT (++stat_wbarrier_arrayref_copy);
4069 /*This check can be done without taking a lock since dest_ptr array is pinned*/
4070 if (ptr_in_nursery (dest_ptr) || count <= 0) {
4071 mono_gc_memmove_aligned (dest_ptr, src_ptr, count * sizeof (gpointer));
4075 #ifdef SGEN_HEAVY_BINARY_PROTOCOL
4076 if (binary_protocol_is_heavy_enabled ()) {
4078 for (i = 0; i < count; ++i) {
4079 gpointer dest = (gpointer*)dest_ptr + i;
4080 gpointer obj = *((gpointer*)src_ptr + i);
4082 binary_protocol_wbarrier (dest, obj, (gpointer)LOAD_VTABLE (obj));
4087 remset.wbarrier_arrayref_copy (dest_ptr, src_ptr, count);
4090 static char *found_obj;
4093 find_object_for_ptr_callback (char *obj, size_t size, void *user_data)
4095 char *ptr = user_data;
4097 if (ptr >= obj && ptr < obj + size) {
4098 g_assert (!found_obj);
4103 /* for use in the debugger */
4104 char* find_object_for_ptr (char *ptr);
4106 find_object_for_ptr (char *ptr)
4108 if (ptr >= nursery_section->data && ptr < nursery_section->end_data) {
4110 sgen_scan_area_with_callback (nursery_section->data, nursery_section->end_data,
4111 find_object_for_ptr_callback, ptr, TRUE);
4117 sgen_los_iterate_objects (find_object_for_ptr_callback, ptr);
4122 * Very inefficient, but this is debugging code, supposed to
4123 * be called from gdb, so we don't care.
4126 major_collector.iterate_objects (ITERATE_OBJECTS_SWEEP_ALL, find_object_for_ptr_callback, ptr);
4131 mono_gc_wbarrier_generic_nostore (gpointer ptr)
4135 HEAVY_STAT (++stat_wbarrier_generic_store);
4137 #ifdef XDOMAIN_CHECKS_IN_WBARRIER
4138 /* FIXME: ptr_in_heap must be called with the GC lock held */
4139 if (xdomain_checks && *(MonoObject**)ptr && ptr_in_heap (ptr)) {
4140 char *start = find_object_for_ptr (ptr);
4141 MonoObject *value = *(MonoObject**)ptr;
4145 MonoObject *obj = (MonoObject*)start;
4146 if (obj->vtable->domain != value->vtable->domain)
4147 g_assert (is_xdomain_ref_allowed (ptr, start, obj->vtable->domain));
4153 obj = *(gpointer*)ptr;
4155 binary_protocol_wbarrier (ptr, obj, (gpointer)LOAD_VTABLE (obj));
4157 if (ptr_in_nursery (ptr) || ptr_on_stack (ptr)) {
4158 SGEN_LOG (8, "Skipping remset at %p", ptr);
4163 * We need to record old->old pointer locations for the
4164 * concurrent collector.
4166 if (!ptr_in_nursery (obj) && !concurrent_collection_in_progress) {
4167 SGEN_LOG (8, "Skipping remset at %p", ptr);
4171 SGEN_LOG (8, "Adding remset at %p", ptr);
4173 remset.wbarrier_generic_nostore (ptr);
4177 mono_gc_wbarrier_generic_store (gpointer ptr, MonoObject* value)
4179 SGEN_LOG (8, "Wbarrier store at %p to %p (%s)", ptr, value, value ? safe_name (value) : "null");
4180 SGEN_UPDATE_REFERENCE_ALLOW_NULL (ptr, value);
4181 if (ptr_in_nursery (value))
4182 mono_gc_wbarrier_generic_nostore (ptr);
4183 sgen_dummy_use (value);
4186 /* Same as mono_gc_wbarrier_generic_store () but performs the store
4187 * as an atomic operation with release semantics.
4190 mono_gc_wbarrier_generic_store_atomic (gpointer ptr, MonoObject *value)
4192 HEAVY_STAT (++stat_wbarrier_generic_store_atomic);
4194 SGEN_LOG (8, "Wbarrier atomic store at %p to %p (%s)", ptr, value, value ? safe_name (value) : "null");
4196 InterlockedWritePointer (ptr, value);
4198 if (ptr_in_nursery (value))
4199 mono_gc_wbarrier_generic_nostore (ptr);
4201 sgen_dummy_use (value);
4204 void mono_gc_wbarrier_value_copy_bitmap (gpointer _dest, gpointer _src, int size, unsigned bitmap)
4206 mword *dest = _dest;
4211 mono_gc_wbarrier_generic_store (dest, (MonoObject*)*src);
4213 SGEN_UPDATE_REFERENCE_ALLOW_NULL (dest, *src);
4216 size -= SIZEOF_VOID_P;
4221 #ifdef SGEN_HEAVY_BINARY_PROTOCOL
4223 #define HANDLE_PTR(ptr,obj) do { \
4224 gpointer o = *(gpointer*)(ptr); \
4226 gpointer d = ((char*)dest) + ((char*)(ptr) - (char*)(obj)); \
4227 binary_protocol_wbarrier (d, o, (gpointer) LOAD_VTABLE (o)); \
4232 scan_object_for_binary_protocol_copy_wbarrier (gpointer dest, char *start, mword desc)
4234 #define SCAN_OBJECT_NOVTABLE
4235 #include "sgen-scan-object.h"
4240 mono_gc_wbarrier_value_copy (gpointer dest, gpointer src, int count, MonoClass *klass)
4242 HEAVY_STAT (++stat_wbarrier_value_copy);
4243 g_assert (klass->valuetype);
4245 SGEN_LOG (8, "Adding value remset at %p, count %d, descr %p for class %s (%p)", dest, count, klass->gc_descr, klass->name, klass);
4247 if (ptr_in_nursery (dest) || ptr_on_stack (dest) || !SGEN_CLASS_HAS_REFERENCES (klass)) {
4248 size_t element_size = mono_class_value_size (klass, NULL);
4249 size_t size = count * element_size;
4250 mono_gc_memmove_atomic (dest, src, size);
4254 #ifdef SGEN_HEAVY_BINARY_PROTOCOL
4255 if (binary_protocol_is_heavy_enabled ()) {
4256 size_t element_size = mono_class_value_size (klass, NULL);
4258 for (i = 0; i < count; ++i) {
4259 scan_object_for_binary_protocol_copy_wbarrier ((char*)dest + i * element_size,
4260 (char*)src + i * element_size - sizeof (MonoObject),
4261 (mword) klass->gc_descr);
4266 remset.wbarrier_value_copy (dest, src, count, klass);
4270 * mono_gc_wbarrier_object_copy:
4272 * Write barrier to call when obj is the result of a clone or copy of an object.
4275 mono_gc_wbarrier_object_copy (MonoObject* obj, MonoObject *src)
4279 HEAVY_STAT (++stat_wbarrier_object_copy);
4281 if (ptr_in_nursery (obj) || ptr_on_stack (obj)) {
4282 size = mono_object_class (obj)->instance_size;
4283 mono_gc_memmove_aligned ((char*)obj + sizeof (MonoObject), (char*)src + sizeof (MonoObject),
4284 size - sizeof (MonoObject));
4288 #ifdef SGEN_HEAVY_BINARY_PROTOCOL
4289 if (binary_protocol_is_heavy_enabled ())
4290 scan_object_for_binary_protocol_copy_wbarrier (obj, (char*)src, (mword) src->vtable->gc_descr);
4293 remset.wbarrier_object_copy (obj, src);
4298 * ######################################################################
4299 * ######## Other mono public interface functions.
4300 * ######################################################################
4303 #define REFS_SIZE 128
4306 MonoGCReferences callback;
4310 MonoObject *refs [REFS_SIZE];
4311 uintptr_t offsets [REFS_SIZE];
4315 #define HANDLE_PTR(ptr,obj) do { \
4317 if (hwi->count == REFS_SIZE) { \
4318 hwi->callback ((MonoObject*)start, mono_object_class (start), hwi->called? 0: size, hwi->count, hwi->refs, hwi->offsets, hwi->data); \
4322 hwi->offsets [hwi->count] = (char*)(ptr)-(char*)start; \
4323 hwi->refs [hwi->count++] = *(ptr); \
4328 collect_references (HeapWalkInfo *hwi, char *start, size_t size)
4330 mword desc = sgen_obj_get_descriptor (start);
4332 #include "sgen-scan-object.h"
4336 walk_references (char *start, size_t size, void *data)
4338 HeapWalkInfo *hwi = data;
4341 collect_references (hwi, start, size);
4342 if (hwi->count || !hwi->called)
4343 hwi->callback ((MonoObject*)start, mono_object_class (start), hwi->called? 0: size, hwi->count, hwi->refs, hwi->offsets, hwi->data);
4347 * mono_gc_walk_heap:
4348 * @flags: flags for future use
4349 * @callback: a function pointer called for each object in the heap
4350 * @data: a user data pointer that is passed to callback
4352 * This function can be used to iterate over all the live objects in the heap:
4353 * for each object, @callback is invoked, providing info about the object's
4354 * location in memory, its class, its size and the objects it references.
4355 * For each referenced object it's offset from the object address is
4356 * reported in the offsets array.
4357 * The object references may be buffered, so the callback may be invoked
4358 * multiple times for the same object: in all but the first call, the size
4359 * argument will be zero.
4360 * Note that this function can be only called in the #MONO_GC_EVENT_PRE_START_WORLD
4361 * profiler event handler.
4363 * Returns: a non-zero value if the GC doesn't support heap walking
4366 mono_gc_walk_heap (int flags, MonoGCReferences callback, void *data)
4371 hwi.callback = callback;
4374 sgen_clear_nursery_fragments ();
4375 sgen_scan_area_with_callback (nursery_section->data, nursery_section->end_data, walk_references, &hwi, FALSE);
4377 major_collector.iterate_objects (ITERATE_OBJECTS_SWEEP_ALL, walk_references, &hwi);
4378 sgen_los_iterate_objects (walk_references, &hwi);
4384 mono_gc_collect (int generation)
4389 sgen_perform_collection (0, generation, "user request", TRUE);
4394 mono_gc_max_generation (void)
4400 mono_gc_collection_count (int generation)
4402 if (generation == 0)
4403 return gc_stats.minor_gc_count;
4404 return gc_stats.major_gc_count;
4408 mono_gc_get_used_size (void)
4412 tot = los_memory_usage;
4413 tot += nursery_section->next_data - nursery_section->data;
4414 tot += major_collector.get_used_size ();
4415 /* FIXME: account for pinned objects */
4421 mono_gc_get_los_limit (void)
4423 return MAX_SMALL_OBJ_SIZE;
4427 mono_gc_set_string_length (MonoString *str, gint32 new_length)
4429 mono_unichar2 *new_end = str->chars + new_length;
4431 /* zero the discarded string. This null-delimits the string and allows
4432 * the space to be reclaimed by SGen. */
4434 if (nursery_canaries_enabled () && sgen_ptr_in_nursery (str)) {
4435 CHECK_CANARY_FOR_OBJECT (str);
4436 memset (new_end, 0, (str->length - new_length + 1) * sizeof (mono_unichar2) + CANARY_SIZE);
4437 memcpy (new_end + 1 , CANARY_STRING, CANARY_SIZE);
4439 memset (new_end, 0, (str->length - new_length + 1) * sizeof (mono_unichar2));
4442 str->length = new_length;
4446 mono_gc_user_markers_supported (void)
4452 mono_object_is_alive (MonoObject* o)
4458 mono_gc_get_generation (MonoObject *obj)
4460 if (ptr_in_nursery (obj))
4466 mono_gc_enable_events (void)
4471 mono_gc_weak_link_add (void **link_addr, MonoObject *obj, gboolean track)
4473 sgen_register_disappearing_link (obj, link_addr, track, FALSE);
4477 mono_gc_weak_link_remove (void **link_addr, gboolean track)
4479 sgen_register_disappearing_link (NULL, link_addr, track, FALSE);
4483 mono_gc_weak_link_get (void **link_addr)
4485 void * volatile *link_addr_volatile;
4489 link_addr_volatile = link_addr;
4490 ptr = (void*)*link_addr_volatile;
4492 * At this point we have a hidden pointer. If the GC runs
4493 * here, it will not recognize the hidden pointer as a
4494 * reference, and if the object behind it is not referenced
4495 * elsewhere, it will be freed. Once the world is restarted
4496 * we reveal the pointer, giving us a pointer to a freed
4497 * object. To make sure we don't return it, we load the
4498 * hidden pointer again. If it's still the same, we can be
4499 * sure the object reference is valid.
4502 obj = (MonoObject*) REVEAL_POINTER (ptr);
4506 mono_memory_barrier ();
4509 * During the second bridge processing step the world is
4510 * running again. That step processes all weak links once
4511 * more to null those that refer to dead objects. Before that
4512 * is completed, those links must not be followed, so we
4513 * conservatively wait for bridge processing when any weak
4514 * link is dereferenced.
4516 if (G_UNLIKELY (bridge_processing_in_progress))
4517 mono_gc_wait_for_bridge_processing ();
4519 if ((void*)*link_addr_volatile != ptr)
4526 mono_gc_ephemeron_array_add (MonoObject *obj)
4528 EphemeronLinkNode *node;
4532 node = sgen_alloc_internal (INTERNAL_MEM_EPHEMERON_LINK);
4537 node->array = (char*)obj;
4538 node->next = ephemeron_list;
4539 ephemeron_list = node;
4541 SGEN_LOG (5, "Registered ephemeron array %p", obj);
4548 mono_gc_set_allow_synchronous_major (gboolean flag)
4550 if (!major_collector.is_concurrent)
4553 allow_synchronous_major = flag;
4558 mono_gc_invoke_with_gc_lock (MonoGCLockedCallbackFunc func, void *data)
4562 result = func (data);
4563 UNLOCK_INTERRUPTION;
4568 mono_gc_is_gc_thread (void)
4572 result = mono_thread_info_current () != NULL;
4578 is_critical_method (MonoMethod *method)
4580 return mono_runtime_is_critical_method (method) || sgen_is_critical_method (method);
4584 sgen_env_var_error (const char *env_var, const char *fallback, const char *description_format, ...)
4588 va_start (ap, description_format);
4590 fprintf (stderr, "Warning: In environment variable `%s': ", env_var);
4591 vfprintf (stderr, description_format, ap);
4593 fprintf (stderr, " - %s", fallback);
4594 fprintf (stderr, "\n");
4600 parse_double_in_interval (const char *env_var, const char *opt_name, const char *opt, double min, double max, double *result)
4603 double val = strtod (opt, &endptr);
4604 if (endptr == opt) {
4605 sgen_env_var_error (env_var, "Using default value.", "`%s` must be a number.", opt_name);
4608 else if (val < min || val > max) {
4609 sgen_env_var_error (env_var, "Using default value.", "`%s` must be between %.2f - %.2f.", opt_name, min, max);
4617 mono_gc_base_init (void)
4619 MonoThreadInfoCallbacks cb;
4622 char *major_collector_opt = NULL;
4623 char *minor_collector_opt = NULL;
4624 size_t max_heap = 0;
4625 size_t soft_limit = 0;
4628 gboolean debug_print_allowance = FALSE;
4629 double allowance_ratio = 0, save_target = 0;
4630 gboolean have_split_nursery = FALSE;
4631 gboolean cement_enabled = TRUE;
4633 mono_counters_init ();
4636 result = InterlockedCompareExchange (&gc_initialized, -1, 0);
4639 /* already inited */
4642 /* being inited by another thread */
4646 /* we will init it */
4649 g_assert_not_reached ();
4651 } while (result != 0);
4653 SGEN_TV_GETTIME (sgen_init_timestamp);
4655 LOCK_INIT (gc_mutex);
4657 pagesize = mono_pagesize ();
4658 gc_debug_file = stderr;
4660 cb.thread_register = sgen_thread_register;
4661 cb.thread_detach = sgen_thread_detach;
4662 cb.thread_unregister = sgen_thread_unregister;
4663 cb.thread_attach = sgen_thread_attach;
4664 cb.mono_method_is_critical = (gpointer)is_critical_method;
4666 cb.thread_exit = mono_gc_pthread_exit;
4667 cb.mono_gc_pthread_create = (gpointer)mono_gc_pthread_create;
4670 mono_threads_init (&cb, sizeof (SgenThreadInfo));
4672 LOCK_INIT (sgen_interruption_mutex);
4674 if ((env = g_getenv (MONO_GC_PARAMS_NAME))) {
4675 opts = g_strsplit (env, ",", -1);
4676 for (ptr = opts; *ptr; ++ptr) {
4678 if (g_str_has_prefix (opt, "major=")) {
4679 opt = strchr (opt, '=') + 1;
4680 major_collector_opt = g_strdup (opt);
4681 } else if (g_str_has_prefix (opt, "minor=")) {
4682 opt = strchr (opt, '=') + 1;
4683 minor_collector_opt = g_strdup (opt);
4691 sgen_init_internal_allocator ();
4692 sgen_init_nursery_allocator ();
4693 sgen_init_fin_weak_hash ();
4695 sgen_init_hash_table ();
4696 sgen_init_descriptors ();
4697 sgen_init_gray_queues ();
4699 sgen_register_fixed_internal_mem_type (INTERNAL_MEM_SECTION, SGEN_SIZEOF_GC_MEM_SECTION);
4700 sgen_register_fixed_internal_mem_type (INTERNAL_MEM_FINALIZE_READY_ENTRY, sizeof (FinalizeReadyEntry));
4701 sgen_register_fixed_internal_mem_type (INTERNAL_MEM_GRAY_QUEUE, sizeof (GrayQueueSection));
4702 sgen_register_fixed_internal_mem_type (INTERNAL_MEM_EPHEMERON_LINK, sizeof (EphemeronLinkNode));
4704 #ifndef HAVE_KW_THREAD
4705 mono_native_tls_alloc (&thread_info_key, NULL);
4706 #if defined(__APPLE__) || defined (HOST_WIN32)
4708 * CEE_MONO_TLS requires the tls offset, not the key, so the code below only works on darwin,
4709 * where the two are the same.
4711 mono_tls_key_set_offset (TLS_KEY_SGEN_THREAD_INFO, thread_info_key);
4715 int tls_offset = -1;
4716 MONO_THREAD_VAR_OFFSET (sgen_thread_info, tls_offset);
4717 mono_tls_key_set_offset (TLS_KEY_SGEN_THREAD_INFO, tls_offset);
4722 * This needs to happen before any internal allocations because
4723 * it inits the small id which is required for hazard pointer
4728 mono_thread_info_attach (&dummy);
4730 if (!minor_collector_opt) {
4731 sgen_simple_nursery_init (&sgen_minor_collector);
4733 if (!strcmp (minor_collector_opt, "simple")) {
4735 sgen_simple_nursery_init (&sgen_minor_collector);
4736 } else if (!strcmp (minor_collector_opt, "split")) {
4737 sgen_split_nursery_init (&sgen_minor_collector);
4738 have_split_nursery = TRUE;
4740 sgen_env_var_error (MONO_GC_PARAMS_NAME, "Using `simple` instead.", "Unknown minor collector `%s'.", minor_collector_opt);
4741 goto use_simple_nursery;
4745 if (!major_collector_opt || !strcmp (major_collector_opt, "marksweep")) {
4746 use_marksweep_major:
4747 sgen_marksweep_init (&major_collector);
4748 } else if (!major_collector_opt || !strcmp (major_collector_opt, "marksweep-conc")) {
4749 sgen_marksweep_conc_init (&major_collector);
4751 sgen_env_var_error (MONO_GC_PARAMS_NAME, "Using `marksweep` instead.", "Unknown major collector `%s'.", major_collector_opt);
4752 goto use_marksweep_major;
4755 ///* Keep this the default for now */
4756 /* Precise marking is broken on all supported targets. Disable until fixed. */
4757 conservative_stack_mark = TRUE;
4759 sgen_nursery_size = DEFAULT_NURSERY_SIZE;
4762 gboolean usage_printed = FALSE;
4764 for (ptr = opts; *ptr; ++ptr) {
4766 if (!strcmp (opt, ""))
4768 if (g_str_has_prefix (opt, "major="))
4770 if (g_str_has_prefix (opt, "minor="))
4772 if (g_str_has_prefix (opt, "max-heap-size=")) {
4773 size_t max_heap_candidate = 0;
4774 opt = strchr (opt, '=') + 1;
4775 if (*opt && mono_gc_parse_environment_string_extract_number (opt, &max_heap_candidate)) {
4776 max_heap = (max_heap_candidate + mono_pagesize () - 1) & ~(size_t)(mono_pagesize () - 1);
4777 if (max_heap != max_heap_candidate)
4778 sgen_env_var_error (MONO_GC_PARAMS_NAME, "Rounding up.", "`max-heap-size` size must be a multiple of %d.", mono_pagesize ());
4780 sgen_env_var_error (MONO_GC_PARAMS_NAME, NULL, "`max-heap-size` must be an integer.");
4784 if (g_str_has_prefix (opt, "soft-heap-limit=")) {
4785 opt = strchr (opt, '=') + 1;
4786 if (*opt && mono_gc_parse_environment_string_extract_number (opt, &soft_limit)) {
4787 if (soft_limit <= 0) {
4788 sgen_env_var_error (MONO_GC_PARAMS_NAME, NULL, "`soft-heap-limit` must be positive.");
4792 sgen_env_var_error (MONO_GC_PARAMS_NAME, NULL, "`soft-heap-limit` must be an integer.");
4796 if (g_str_has_prefix (opt, "stack-mark=")) {
4797 opt = strchr (opt, '=') + 1;
4798 if (!strcmp (opt, "precise")) {
4799 conservative_stack_mark = FALSE;
4800 } else if (!strcmp (opt, "conservative")) {
4801 conservative_stack_mark = TRUE;
4803 sgen_env_var_error (MONO_GC_PARAMS_NAME, conservative_stack_mark ? "Using `conservative`." : "Using `precise`.",
4804 "Invalid value `%s` for `stack-mark` option, possible values are: `precise`, `conservative`.", opt);
4808 if (g_str_has_prefix (opt, "bridge-implementation=")) {
4809 opt = strchr (opt, '=') + 1;
4810 sgen_set_bridge_implementation (opt);
4813 if (g_str_has_prefix (opt, "toggleref-test")) {
4814 sgen_register_test_toggleref_callback ();
4819 if (g_str_has_prefix (opt, "nursery-size=")) {
4821 opt = strchr (opt, '=') + 1;
4822 if (*opt && mono_gc_parse_environment_string_extract_number (opt, &val)) {
4823 #ifdef SGEN_ALIGN_NURSERY
4824 if ((val & (val - 1))) {
4825 sgen_env_var_error (MONO_GC_PARAMS_NAME, "Using default value.", "`nursery-size` must be a power of two.");
4829 if (val < SGEN_MAX_NURSERY_WASTE) {
4830 sgen_env_var_error (MONO_GC_PARAMS_NAME, "Using default value.",
4831 "`nursery-size` must be at least %d bytes.", SGEN_MAX_NURSERY_WASTE);
4835 sgen_nursery_size = val;
4836 sgen_nursery_bits = 0;
4837 while (ONE_P << (++ sgen_nursery_bits) != sgen_nursery_size)
4840 sgen_nursery_size = val;
4843 sgen_env_var_error (MONO_GC_PARAMS_NAME, "Using default value.", "`nursery-size` must be an integer.");
4849 if (g_str_has_prefix (opt, "save-target-ratio=")) {
4851 opt = strchr (opt, '=') + 1;
4852 if (parse_double_in_interval (MONO_GC_PARAMS_NAME, "save-target-ratio", opt,
4853 SGEN_MIN_SAVE_TARGET_RATIO, SGEN_MAX_SAVE_TARGET_RATIO, &val)) {
4858 if (g_str_has_prefix (opt, "default-allowance-ratio=")) {
4860 opt = strchr (opt, '=') + 1;
4861 if (parse_double_in_interval (MONO_GC_PARAMS_NAME, "default-allowance-ratio", opt,
4862 SGEN_MIN_ALLOWANCE_NURSERY_SIZE_RATIO, SGEN_MIN_ALLOWANCE_NURSERY_SIZE_RATIO, &val)) {
4863 allowance_ratio = val;
4867 if (g_str_has_prefix (opt, "allow-synchronous-major=")) {
4868 if (!major_collector.is_concurrent) {
4869 sgen_env_var_error (MONO_GC_PARAMS_NAME, "Ignoring.", "`allow-synchronous-major` is only valid for the concurrent major collector.");
4873 opt = strchr (opt, '=') + 1;
4875 if (!strcmp (opt, "yes")) {
4876 allow_synchronous_major = TRUE;
4877 } else if (!strcmp (opt, "no")) {
4878 allow_synchronous_major = FALSE;
4880 sgen_env_var_error (MONO_GC_PARAMS_NAME, "Using default value.", "`allow-synchronous-major` must be either `yes' or `no'.");
4885 if (!strcmp (opt, "cementing")) {
4886 cement_enabled = TRUE;
4889 if (!strcmp (opt, "no-cementing")) {
4890 cement_enabled = FALSE;
4894 if (major_collector.handle_gc_param && major_collector.handle_gc_param (opt))
4897 if (sgen_minor_collector.handle_gc_param && sgen_minor_collector.handle_gc_param (opt))
4900 sgen_env_var_error (MONO_GC_PARAMS_NAME, "Ignoring.", "Unknown option `%s`.", opt);
4905 fprintf (stderr, "\n%s must be a comma-delimited list of one or more of the following:\n", MONO_GC_PARAMS_NAME);
4906 fprintf (stderr, " max-heap-size=N (where N is an integer, possibly with a k, m or a g suffix)\n");
4907 fprintf (stderr, " soft-heap-limit=n (where N is an integer, possibly with a k, m or a g suffix)\n");
4908 fprintf (stderr, " nursery-size=N (where N is an integer, possibly with a k, m or a g suffix)\n");
4909 fprintf (stderr, " major=COLLECTOR (where COLLECTOR is `marksweep', `marksweep-conc', `marksweep-par')\n");
4910 fprintf (stderr, " minor=COLLECTOR (where COLLECTOR is `simple' or `split')\n");
4911 fprintf (stderr, " wbarrier=WBARRIER (where WBARRIER is `remset' or `cardtable')\n");
4912 fprintf (stderr, " stack-mark=MARK-METHOD (where MARK-METHOD is 'precise' or 'conservative')\n");
4913 fprintf (stderr, " [no-]cementing\n");
4914 if (major_collector.is_concurrent)
4915 fprintf (stderr, " allow-synchronous-major=FLAG (where FLAG is `yes' or `no')\n");
4916 if (major_collector.print_gc_param_usage)
4917 major_collector.print_gc_param_usage ();
4918 if (sgen_minor_collector.print_gc_param_usage)
4919 sgen_minor_collector.print_gc_param_usage ();
4920 fprintf (stderr, " Experimental options:\n");
4921 fprintf (stderr, " save-target-ratio=R (where R must be between %.2f - %.2f).\n", SGEN_MIN_SAVE_TARGET_RATIO, SGEN_MAX_SAVE_TARGET_RATIO);
4922 fprintf (stderr, " default-allowance-ratio=R (where R must be between %.2f - %.2f).\n", SGEN_MIN_ALLOWANCE_NURSERY_SIZE_RATIO, SGEN_MAX_ALLOWANCE_NURSERY_SIZE_RATIO);
4923 fprintf (stderr, "\n");
4925 usage_printed = TRUE;
4930 if (major_collector.is_concurrent)
4931 sgen_workers_init (1);
4933 if (major_collector_opt)
4934 g_free (major_collector_opt);
4936 if (minor_collector_opt)
4937 g_free (minor_collector_opt);
4941 sgen_cement_init (cement_enabled);
4943 if ((env = g_getenv (MONO_GC_DEBUG_NAME))) {
4944 gboolean usage_printed = FALSE;
4946 opts = g_strsplit (env, ",", -1);
4947 for (ptr = opts; ptr && *ptr; ptr ++) {
4949 if (!strcmp (opt, ""))
4951 if (opt [0] >= '0' && opt [0] <= '9') {
4952 gc_debug_level = atoi (opt);
4957 char *rf = g_strdup_printf ("%s.%d", opt, mono_process_current_pid ());
4958 gc_debug_file = fopen (rf, "wb");
4960 gc_debug_file = stderr;
4963 } else if (!strcmp (opt, "print-allowance")) {
4964 debug_print_allowance = TRUE;
4965 } else if (!strcmp (opt, "print-pinning")) {
4966 do_pin_stats = TRUE;
4967 } else if (!strcmp (opt, "verify-before-allocs")) {
4968 verify_before_allocs = 1;
4969 has_per_allocation_action = TRUE;
4970 } else if (g_str_has_prefix (opt, "verify-before-allocs=")) {
4971 char *arg = strchr (opt, '=') + 1;
4972 verify_before_allocs = atoi (arg);
4973 has_per_allocation_action = TRUE;
4974 } else if (!strcmp (opt, "collect-before-allocs")) {
4975 collect_before_allocs = 1;
4976 has_per_allocation_action = TRUE;
4977 } else if (g_str_has_prefix (opt, "collect-before-allocs=")) {
4978 char *arg = strchr (opt, '=') + 1;
4979 has_per_allocation_action = TRUE;
4980 collect_before_allocs = atoi (arg);
4981 } else if (!strcmp (opt, "verify-before-collections")) {
4982 whole_heap_check_before_collection = TRUE;
4983 } else if (!strcmp (opt, "check-at-minor-collections")) {
4984 consistency_check_at_minor_collection = TRUE;
4985 nursery_clear_policy = CLEAR_AT_GC;
4986 } else if (!strcmp (opt, "mod-union-consistency-check")) {
4987 if (!major_collector.is_concurrent) {
4988 sgen_env_var_error (MONO_GC_DEBUG_NAME, "Ignoring.", "`mod-union-consistency-check` only works with concurrent major collector.");
4991 mod_union_consistency_check = TRUE;
4992 } else if (!strcmp (opt, "check-mark-bits")) {
4993 check_mark_bits_after_major_collection = TRUE;
4994 } else if (!strcmp (opt, "check-nursery-pinned")) {
4995 check_nursery_objects_pinned = TRUE;
4996 } else if (!strcmp (opt, "xdomain-checks")) {
4997 xdomain_checks = TRUE;
4998 } else if (!strcmp (opt, "clear-at-gc")) {
4999 nursery_clear_policy = CLEAR_AT_GC;
5000 } else if (!strcmp (opt, "clear-nursery-at-gc")) {
5001 nursery_clear_policy = CLEAR_AT_GC;
5002 } else if (!strcmp (opt, "clear-at-tlab-creation")) {
5003 nursery_clear_policy = CLEAR_AT_TLAB_CREATION;
5004 } else if (!strcmp (opt, "debug-clear-at-tlab-creation")) {
5005 nursery_clear_policy = CLEAR_AT_TLAB_CREATION_DEBUG;
5006 } else if (!strcmp (opt, "check-scan-starts")) {
5007 do_scan_starts_check = TRUE;
5008 } else if (!strcmp (opt, "verify-nursery-at-minor-gc")) {
5009 do_verify_nursery = TRUE;
5010 } else if (!strcmp (opt, "check-concurrent")) {
5011 if (!major_collector.is_concurrent) {
5012 sgen_env_var_error (MONO_GC_DEBUG_NAME, "Ignoring.", "`check-concurrent` only works with concurrent major collectors.");
5015 do_concurrent_checks = TRUE;
5016 } else if (!strcmp (opt, "dump-nursery-at-minor-gc")) {
5017 do_dump_nursery_content = TRUE;
5018 } else if (!strcmp (opt, "no-managed-allocator")) {
5019 sgen_set_use_managed_allocator (FALSE);
5020 } else if (!strcmp (opt, "disable-minor")) {
5021 disable_minor_collections = TRUE;
5022 } else if (!strcmp (opt, "disable-major")) {
5023 disable_major_collections = TRUE;
5024 } else if (g_str_has_prefix (opt, "heap-dump=")) {
5025 char *filename = strchr (opt, '=') + 1;
5026 nursery_clear_policy = CLEAR_AT_GC;
5027 heap_dump_file = fopen (filename, "w");
5028 if (heap_dump_file) {
5029 fprintf (heap_dump_file, "<sgen-dump>\n");
5030 do_pin_stats = TRUE;
5032 } else if (g_str_has_prefix (opt, "binary-protocol=")) {
5033 char *filename = strchr (opt, '=') + 1;
5034 char *colon = strrchr (filename, ':');
5037 if (!mono_gc_parse_environment_string_extract_number (colon + 1, &limit)) {
5038 sgen_env_var_error (MONO_GC_DEBUG_NAME, "Ignoring limit.", "Binary protocol file size limit must be an integer.");
5043 binary_protocol_init (filename, (long long)limit);
5044 } else if (!strcmp (opt, "nursery-canaries")) {
5045 do_verify_nursery = TRUE;
5046 sgen_set_use_managed_allocator (FALSE);
5047 enable_nursery_canaries = TRUE;
5048 } else if (!sgen_bridge_handle_gc_debug (opt)) {
5049 sgen_env_var_error (MONO_GC_DEBUG_NAME, "Ignoring.", "Unknown option `%s`.", opt);
5054 fprintf (stderr, "\n%s must be of the format [<l>[:<filename>]|<option>]+ where <l> is a debug level 0-9.\n", MONO_GC_DEBUG_NAME);
5055 fprintf (stderr, "Valid <option>s are:\n");
5056 fprintf (stderr, " collect-before-allocs[=<n>]\n");
5057 fprintf (stderr, " verify-before-allocs[=<n>]\n");
5058 fprintf (stderr, " check-at-minor-collections\n");
5059 fprintf (stderr, " check-mark-bits\n");
5060 fprintf (stderr, " check-nursery-pinned\n");
5061 fprintf (stderr, " verify-before-collections\n");
5062 fprintf (stderr, " verify-nursery-at-minor-gc\n");
5063 fprintf (stderr, " dump-nursery-at-minor-gc\n");
5064 fprintf (stderr, " disable-minor\n");
5065 fprintf (stderr, " disable-major\n");
5066 fprintf (stderr, " xdomain-checks\n");
5067 fprintf (stderr, " check-concurrent\n");
5068 fprintf (stderr, " clear-[nursery-]at-gc\n");
5069 fprintf (stderr, " clear-at-tlab-creation\n");
5070 fprintf (stderr, " debug-clear-at-tlab-creation\n");
5071 fprintf (stderr, " check-scan-starts\n");
5072 fprintf (stderr, " no-managed-allocator\n");
5073 fprintf (stderr, " print-allowance\n");
5074 fprintf (stderr, " print-pinning\n");
5075 fprintf (stderr, " heap-dump=<filename>\n");
5076 fprintf (stderr, " binary-protocol=<filename>[:<file-size-limit>]\n");
5077 fprintf (stderr, " nursery-canaries\n");
5078 sgen_bridge_print_gc_debug_usage ();
5079 fprintf (stderr, "\n");
5081 usage_printed = TRUE;
5087 if (check_mark_bits_after_major_collection)
5088 nursery_clear_policy = CLEAR_AT_GC;
5090 if (major_collector.post_param_init)
5091 major_collector.post_param_init (&major_collector);
5093 sgen_memgov_init (max_heap, soft_limit, debug_print_allowance, allowance_ratio, save_target);
5095 memset (&remset, 0, sizeof (remset));
5097 sgen_card_table_init (&remset);
5103 mono_gc_get_gc_name (void)
5108 static MonoMethod *write_barrier_method;
5111 sgen_is_critical_method (MonoMethod *method)
5113 return (method == write_barrier_method || sgen_is_managed_allocator (method));
5117 sgen_has_critical_method (void)
5119 return write_barrier_method || sgen_has_managed_allocator ();
5125 emit_nursery_check (MonoMethodBuilder *mb, int *nursery_check_return_labels)
5127 memset (nursery_check_return_labels, 0, sizeof (int) * 3);
5128 #ifdef SGEN_ALIGN_NURSERY
5129 // if (ptr_in_nursery (ptr)) return;
5131 * Masking out the bits might be faster, but we would have to use 64 bit
5132 * immediates, which might be slower.
5134 mono_mb_emit_ldarg (mb, 0);
5135 mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
5136 mono_mb_emit_byte (mb, CEE_SHR_UN);
5137 mono_mb_emit_ptr (mb, (gpointer)((mword)sgen_get_nursery_start () >> DEFAULT_NURSERY_BITS));
5138 nursery_check_return_labels [0] = mono_mb_emit_branch (mb, CEE_BEQ);
5140 if (!major_collector.is_concurrent) {
5141 // if (!ptr_in_nursery (*ptr)) return;
5142 mono_mb_emit_ldarg (mb, 0);
5143 mono_mb_emit_byte (mb, CEE_LDIND_I);
5144 mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
5145 mono_mb_emit_byte (mb, CEE_SHR_UN);
5146 mono_mb_emit_ptr (mb, (gpointer)((mword)sgen_get_nursery_start () >> DEFAULT_NURSERY_BITS));
5147 nursery_check_return_labels [1] = mono_mb_emit_branch (mb, CEE_BNE_UN);
5150 int label_continue1, label_continue2;
5151 int dereferenced_var;
5153 // if (ptr < (sgen_get_nursery_start ())) goto continue;
5154 mono_mb_emit_ldarg (mb, 0);
5155 mono_mb_emit_ptr (mb, (gpointer) sgen_get_nursery_start ());
5156 label_continue_1 = mono_mb_emit_branch (mb, CEE_BLT);
5158 // if (ptr >= sgen_get_nursery_end ())) goto continue;
5159 mono_mb_emit_ldarg (mb, 0);
5160 mono_mb_emit_ptr (mb, (gpointer) sgen_get_nursery_end ());
5161 label_continue_2 = mono_mb_emit_branch (mb, CEE_BGE);
5164 nursery_check_return_labels [0] = mono_mb_emit_branch (mb, CEE_BR);
5167 mono_mb_patch_branch (mb, label_continue_1);
5168 mono_mb_patch_branch (mb, label_continue_2);
5170 // Dereference and store in local var
5171 dereferenced_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
5172 mono_mb_emit_ldarg (mb, 0);
5173 mono_mb_emit_byte (mb, CEE_LDIND_I);
5174 mono_mb_emit_stloc (mb, dereferenced_var);
5176 if (!major_collector.is_concurrent) {
5177 // if (*ptr < sgen_get_nursery_start ()) return;
5178 mono_mb_emit_ldloc (mb, dereferenced_var);
5179 mono_mb_emit_ptr (mb, (gpointer) sgen_get_nursery_start ());
5180 nursery_check_return_labels [1] = mono_mb_emit_branch (mb, CEE_BLT);
5182 // if (*ptr >= sgen_get_nursery_end ()) return;
5183 mono_mb_emit_ldloc (mb, dereferenced_var);
5184 mono_mb_emit_ptr (mb, (gpointer) sgen_get_nursery_end ());
5185 nursery_check_return_labels [2] = mono_mb_emit_branch (mb, CEE_BGE);
5192 mono_gc_get_write_barrier (void)
5195 MonoMethodBuilder *mb;
5196 MonoMethodSignature *sig;
5197 #ifdef MANAGED_WBARRIER
5198 int i, nursery_check_labels [3];
5200 #ifdef HAVE_KW_THREAD
5201 int stack_end_offset = -1;
5203 MONO_THREAD_VAR_OFFSET (stack_end, stack_end_offset);
5204 g_assert (stack_end_offset != -1);
5208 // FIXME: Maybe create a separate version for ctors (the branch would be
5209 // correctly predicted more times)
5210 if (write_barrier_method)
5211 return write_barrier_method;
5213 /* Create the IL version of mono_gc_barrier_generic_store () */
5214 sig = mono_metadata_signature_alloc (mono_defaults.corlib, 1);
5215 sig->ret = &mono_defaults.void_class->byval_arg;
5216 sig->params [0] = &mono_defaults.int_class->byval_arg;
5218 mb = mono_mb_new (mono_defaults.object_class, "wbarrier", MONO_WRAPPER_WRITE_BARRIER);
5221 #ifdef MANAGED_WBARRIER
5222 emit_nursery_check (mb, nursery_check_labels);
5224 addr = sgen_cardtable + ((address >> CARD_BITS) & CARD_MASK)
5228 LDC_PTR sgen_cardtable
5230 address >> CARD_BITS
5234 if (SGEN_HAVE_OVERLAPPING_CARDS) {
5235 LDC_PTR card_table_mask
5242 mono_mb_emit_ptr (mb, sgen_cardtable);
5243 mono_mb_emit_ldarg (mb, 0);
5244 mono_mb_emit_icon (mb, CARD_BITS);
5245 mono_mb_emit_byte (mb, CEE_SHR_UN);
5246 #ifdef SGEN_HAVE_OVERLAPPING_CARDS
5247 mono_mb_emit_ptr (mb, (gpointer)CARD_MASK);
5248 mono_mb_emit_byte (mb, CEE_AND);
5250 mono_mb_emit_byte (mb, CEE_ADD);
5251 mono_mb_emit_icon (mb, 1);
5252 mono_mb_emit_byte (mb, CEE_STIND_I1);
5255 for (i = 0; i < 3; ++i) {
5256 if (nursery_check_labels [i])
5257 mono_mb_patch_branch (mb, nursery_check_labels [i]);
5259 mono_mb_emit_byte (mb, CEE_RET);
5261 mono_mb_emit_ldarg (mb, 0);
5262 mono_mb_emit_icall (mb, mono_gc_wbarrier_generic_nostore);
5263 mono_mb_emit_byte (mb, CEE_RET);
5266 res = mono_mb_create_method (mb, sig, 16);
5270 if (write_barrier_method) {
5271 /* Already created */
5272 mono_free_method (res);
5274 /* double-checked locking */
5275 mono_memory_barrier ();
5276 write_barrier_method = res;
5280 return write_barrier_method;
5284 mono_gc_get_description (void)
5286 return g_strdup ("sgen");
5290 mono_gc_set_desktop_mode (void)
5295 mono_gc_is_moving (void)
5301 mono_gc_is_disabled (void)
5307 BOOL APIENTRY mono_gc_dllmain (HMODULE module_handle, DWORD reason, LPVOID reserved)
5314 sgen_get_nursery_clear_policy (void)
5316 return nursery_clear_policy;
5320 sgen_get_array_fill_vtable (void)
5322 if (!array_fill_vtable) {
5323 static MonoClass klass;
5324 static MonoVTable vtable;
5327 MonoDomain *domain = mono_get_root_domain ();
5330 klass.element_class = mono_defaults.byte_class;
5332 klass.instance_size = sizeof (MonoArray);
5333 klass.sizes.element_size = 1;
5334 klass.name = "array_filler_type";
5336 vtable.klass = &klass;
5338 vtable.gc_descr = mono_gc_make_descr_for_array (TRUE, &bmap, 0, 1);
5341 array_fill_vtable = &vtable;
5343 return array_fill_vtable;
5353 sgen_gc_unlock (void)
5355 gboolean try_free = sgen_try_free_some_memory;
5356 sgen_try_free_some_memory = FALSE;
5357 mono_mutex_unlock (&gc_mutex);
5358 MONO_GC_UNLOCKED ();
5360 mono_thread_hazardous_try_free_some ();
5364 sgen_major_collector_iterate_live_block_ranges (sgen_cardtable_block_callback callback)
5366 major_collector.iterate_live_block_ranges (callback);
5370 sgen_major_collector_scan_card_table (SgenGrayQueue *queue)
5372 major_collector.scan_card_table (FALSE, queue);
5376 sgen_get_major_collector (void)
5378 return &major_collector;
5381 void mono_gc_set_skip_thread (gboolean skip)
5383 SgenThreadInfo *info = mono_thread_info_current ();
5386 info->gc_disabled = skip;
5391 sgen_get_remset (void)
5397 mono_gc_get_vtable_bits (MonoClass *class)
5400 /* FIXME move this to the bridge code */
5401 if (sgen_need_bridge_processing ()) {
5402 switch (sgen_bridge_class_kind (class)) {
5403 case GC_BRIDGE_TRANSPARENT_BRIDGE_CLASS:
5404 case GC_BRIDGE_OPAQUE_BRIDGE_CLASS:
5405 res = SGEN_GC_BIT_BRIDGE_OBJECT;
5407 case GC_BRIDGE_OPAQUE_CLASS:
5408 res = SGEN_GC_BIT_BRIDGE_OPAQUE_OBJECT;
5412 if (fin_callbacks.is_class_finalization_aware) {
5413 if (fin_callbacks.is_class_finalization_aware (class))
5414 res |= SGEN_GC_BIT_FINALIZER_AWARE;
5420 mono_gc_register_altstack (gpointer stack, gint32 stack_size, gpointer altstack, gint32 altstack_size)
5427 sgen_check_whole_heap_stw (void)
5429 sgen_stop_world (0);
5430 sgen_clear_nursery_fragments ();
5431 sgen_check_whole_heap (FALSE);
5432 sgen_restart_world (0, NULL);
5436 sgen_gc_event_moves (void)
5438 if (moved_objects_idx) {
5439 mono_profiler_gc_moves (moved_objects, moved_objects_idx);
5440 moved_objects_idx = 0;
5445 sgen_timestamp (void)
5447 SGEN_TV_DECLARE (timestamp);
5448 SGEN_TV_GETTIME (timestamp);
5449 return SGEN_TV_ELAPSED (sgen_init_timestamp, timestamp);
5453 mono_gc_register_finalizer_callbacks (MonoGCFinalizerCallbacks *callbacks)
5455 if (callbacks->version != MONO_GC_FINALIZER_EXTENSION_VERSION)
5456 g_error ("Invalid finalizer callback version. Expected %d but got %d\n", MONO_GC_FINALIZER_EXTENSION_VERSION, callbacks->version);
5458 fin_callbacks = *callbacks;
5465 #endif /* HAVE_SGEN_GC */