2 * sgen-gc.c: Simple generational GC.
5 * Paolo Molaro (lupus@ximian.com)
6 * Rodrigo Kumpera (kumpera@gmail.com)
8 * Copyright 2005-2011 Novell, Inc (http://www.novell.com)
9 * Copyright 2011 Xamarin Inc (http://www.xamarin.com)
11 * Thread start/stop adapted from Boehm's GC:
12 * Copyright (c) 1994 by Xerox Corporation. All rights reserved.
13 * Copyright (c) 1996 by Silicon Graphics. All rights reserved.
14 * Copyright (c) 1998 by Fergus Henderson. All rights reserved.
15 * Copyright (c) 2000-2004 by Hewlett-Packard Company. All rights reserved.
16 * Copyright 2001-2003 Ximian, Inc
17 * Copyright 2003-2010 Novell, Inc.
18 * Copyright 2011 Xamarin, Inc.
19 * Copyright (C) 2012 Xamarin Inc
21 * This library is free software; you can redistribute it and/or
22 * modify it under the terms of the GNU Library General Public
23 * License 2.0 as published by the Free Software Foundation;
25 * This library is distributed in the hope that it will be useful,
26 * but WITHOUT ANY WARRANTY; without even the implied warranty of
27 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
28 * Library General Public License for more details.
30 * You should have received a copy of the GNU Library General Public
31 * License 2.0 along with this library; if not, write to the Free
32 * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
34 * Important: allocation provides always zeroed memory, having to do
35 * a memset after allocation is deadly for performance.
36 * Memory usage at startup is currently as follows:
38 * 64 KB internal space
40 * We should provide a small memory config with half the sizes
42 * We currently try to make as few mono assumptions as possible:
43 * 1) 2-word header with no GC pointers in it (first vtable, second to store the
45 * 2) gc descriptor is the second word in the vtable (first word in the class)
46 * 3) 8 byte alignment is the minimum and enough (not true for special structures (SIMD), FIXME)
47 * 4) there is a function to get an object's size and the number of
48 * elements in an array.
49 * 5) we know the special way bounds are allocated for complex arrays
50 * 6) we know about proxies and how to treat them when domains are unloaded
52 * Always try to keep stack usage to a minimum: no recursive behaviour
53 * and no large stack allocs.
55 * General description.
56 * Objects are initially allocated in a nursery using a fast bump-pointer technique.
57 * When the nursery is full we start a nursery collection: this is performed with a
59 * When the old generation is full we start a copying GC of the old generation as well:
60 * this will be changed to mark&sweep with copying when fragmentation becomes to severe
61 * in the future. Maybe we'll even do both during the same collection like IMMIX.
63 * The things that complicate this description are:
64 * *) pinned objects: we can't move them so we need to keep track of them
65 * *) no precise info of the thread stacks and registers: we need to be able to
66 * quickly find the objects that may be referenced conservatively and pin them
67 * (this makes the first issues more important)
68 * *) large objects are too expensive to be dealt with using copying GC: we handle them
69 * with mark/sweep during major collections
70 * *) some objects need to not move even if they are small (interned strings, Type handles):
71 * we use mark/sweep for them, too: they are not allocated in the nursery, but inside
72 * PinnedChunks regions
78 *) we could have a function pointer in MonoClass to implement
79 customized write barriers for value types
81 *) investigate the stuff needed to advance a thread to a GC-safe
82 point (single-stepping, read from unmapped memory etc) and implement it.
83 This would enable us to inline allocations and write barriers, for example,
84 or at least parts of them, like the write barrier checks.
85 We may need this also for handling precise info on stacks, even simple things
86 as having uninitialized data on the stack and having to wait for the prolog
87 to zero it. Not an issue for the last frame that we scan conservatively.
88 We could always not trust the value in the slots anyway.
90 *) modify the jit to save info about references in stack locations:
91 this can be done just for locals as a start, so that at least
92 part of the stack is handled precisely.
94 *) test/fix endianess issues
96 *) Implement a card table as the write barrier instead of remembered
97 sets? Card tables are not easy to implement with our current
98 memory layout. We have several different kinds of major heap
99 objects: Small objects in regular blocks, small objects in pinned
100 chunks and LOS objects. If we just have a pointer we have no way
101 to tell which kind of object it points into, therefore we cannot
102 know where its card table is. The least we have to do to make
103 this happen is to get rid of write barriers for indirect stores.
106 *) Get rid of write barriers for indirect stores. We can do this by
107 telling the GC to wbarrier-register an object once we do an ldloca
108 or ldelema on it, and to unregister it once it's not used anymore
109 (it can only travel downwards on the stack). The problem with
110 unregistering is that it needs to happen eventually no matter
111 what, even if exceptions are thrown, the thread aborts, etc.
112 Rodrigo suggested that we could do only the registering part and
113 let the collector find out (pessimistically) when it's safe to
114 unregister, namely when the stack pointer of the thread that
115 registered the object is higher than it was when the registering
116 happened. This might make for a good first implementation to get
117 some data on performance.
119 *) Some sort of blacklist support? Blacklists is a concept from the
120 Boehm GC: if during a conservative scan we find pointers to an
121 area which we might use as heap, we mark that area as unusable, so
122 pointer retention by random pinning pointers is reduced.
124 *) experiment with max small object size (very small right now - 2kb,
125 because it's tied to the max freelist size)
127 *) add an option to mmap the whole heap in one chunk: it makes for many
128 simplifications in the checks (put the nursery at the top and just use a single
129 check for inclusion/exclusion): the issue this has is that on 32 bit systems it's
130 not flexible (too much of the address space may be used by default or we can't
131 increase the heap as needed) and we'd need a race-free mechanism to return memory
132 back to the system (mprotect(PROT_NONE) will still keep the memory allocated if it
133 was written to, munmap is needed, but the following mmap may not find the same segment
136 *) memzero the major fragments after restarting the world and optionally a smaller
139 *) investigate having fragment zeroing threads
141 *) separate locks for finalization and other minor stuff to reduce
144 *) try a different copying order to improve memory locality
146 *) a thread abort after a store but before the write barrier will
147 prevent the write barrier from executing
149 *) specialized dynamically generated markers/copiers
151 *) Dynamically adjust TLAB size to the number of threads. If we have
152 too many threads that do allocation, we might need smaller TLABs,
153 and we might get better performance with larger TLABs if we only
154 have a handful of threads. We could sum up the space left in all
155 assigned TLABs and if that's more than some percentage of the
156 nursery size, reduce the TLAB size.
158 *) Explore placing unreachable objects on unused nursery memory.
159 Instead of memset'ng a region to zero, place an int[] covering it.
160 A good place to start is add_nursery_frag. The tricky thing here is
161 placing those objects atomically outside of a collection.
163 *) Allocation should use asymmetric Dekker synchronization:
164 http://blogs.oracle.com/dave/resource/Asymmetric-Dekker-Synchronization.txt
165 This should help weak consistency archs.
172 #define _XOPEN_SOURCE
173 #define _DARWIN_C_SOURCE
179 #ifdef HAVE_PTHREAD_H
182 #ifdef HAVE_PTHREAD_NP_H
183 #include <pthread_np.h>
185 #ifdef HAVE_SEMAPHORE_H
186 #include <semaphore.h>
194 #include "metadata/sgen-gc.h"
195 #include "metadata/metadata-internals.h"
196 #include "metadata/class-internals.h"
197 #include "metadata/gc-internal.h"
198 #include "metadata/object-internals.h"
199 #include "metadata/threads.h"
200 #include "metadata/sgen-cardtable.h"
201 #include "metadata/sgen-protocol.h"
202 #include "metadata/sgen-archdep.h"
203 #include "metadata/sgen-bridge.h"
204 #include "metadata/sgen-memory-governor.h"
205 #include "metadata/sgen-hash-table.h"
206 #include "metadata/mono-gc.h"
207 #include "metadata/method-builder.h"
208 #include "metadata/profiler-private.h"
209 #include "metadata/monitor.h"
210 #include "metadata/mempool-internals.h"
211 #include "metadata/marshal.h"
212 #include "metadata/runtime.h"
213 #include "metadata/sgen-cardtable.h"
214 #include "metadata/sgen-pinning.h"
215 #include "metadata/sgen-workers.h"
216 #include "metadata/sgen-layout-stats.h"
217 #include "utils/mono-mmap.h"
218 #include "utils/mono-time.h"
219 #include "utils/mono-semaphore.h"
220 #include "utils/mono-counters.h"
221 #include "utils/mono-proclib.h"
222 #include "utils/mono-memory-model.h"
223 #include "utils/mono-logger-internal.h"
224 #include "utils/dtrace.h"
226 #include <mono/utils/mono-logger-internal.h>
227 #include <mono/utils/memcheck.h>
229 #if defined(__MACH__)
230 #include "utils/mach-support.h"
233 #define OPDEF(a,b,c,d,e,f,g,h,i,j) \
237 #include "mono/cil/opcode.def"
243 #undef pthread_create
245 #undef pthread_detach
248 * ######################################################################
249 * ######## Types and constants used by the GC.
250 * ######################################################################
253 /* 0 means not initialized, 1 is initialized, -1 means in progress */
254 static int gc_initialized = 0;
255 /* If set, check if we need to do something every X allocations */
256 gboolean has_per_allocation_action;
257 /* If set, do a heap check every X allocation */
258 guint32 verify_before_allocs = 0;
259 /* If set, do a minor collection before every X allocation */
260 guint32 collect_before_allocs = 0;
261 /* If set, do a whole heap check before each collection */
262 static gboolean whole_heap_check_before_collection = FALSE;
263 /* If set, do a heap consistency check before each minor collection */
264 static gboolean consistency_check_at_minor_collection = FALSE;
265 /* If set, do a mod union consistency check before each finishing collection pause */
266 static gboolean mod_union_consistency_check = FALSE;
267 /* If set, check whether mark bits are consistent after major collections */
268 static gboolean check_mark_bits_after_major_collection = FALSE;
269 /* If set, check that all nursery objects are pinned/not pinned, depending on context */
270 static gboolean check_nursery_objects_pinned = FALSE;
271 /* If set, do a few checks when the concurrent collector is used */
272 static gboolean do_concurrent_checks = FALSE;
273 /* If set, check that there are no references to the domain left at domain unload */
274 static gboolean xdomain_checks = FALSE;
275 /* If not null, dump the heap after each collection into this file */
276 static FILE *heap_dump_file = NULL;
277 /* If set, mark stacks conservatively, even if precise marking is possible */
278 static gboolean conservative_stack_mark = FALSE;
279 /* If set, do a plausibility check on the scan_starts before and after
281 static gboolean do_scan_starts_check = FALSE;
283 * If the major collector is concurrent and this is FALSE, we will
284 * never initiate a synchronous major collection, unless requested via
287 static gboolean allow_synchronous_major = TRUE;
288 static gboolean disable_minor_collections = FALSE;
289 static gboolean disable_major_collections = FALSE;
290 gboolean do_pin_stats = FALSE;
291 static gboolean do_verify_nursery = FALSE;
292 static gboolean do_dump_nursery_content = FALSE;
294 #ifdef HEAVY_STATISTICS
295 long long stat_objects_alloced_degraded = 0;
296 long long stat_bytes_alloced_degraded = 0;
298 long long stat_copy_object_called_nursery = 0;
299 long long stat_objects_copied_nursery = 0;
300 long long stat_copy_object_called_major = 0;
301 long long stat_objects_copied_major = 0;
303 long long stat_scan_object_called_nursery = 0;
304 long long stat_scan_object_called_major = 0;
306 long long stat_slots_allocated_in_vain;
308 long long stat_nursery_copy_object_failed_from_space = 0;
309 long long stat_nursery_copy_object_failed_forwarded = 0;
310 long long stat_nursery_copy_object_failed_pinned = 0;
311 long long stat_nursery_copy_object_failed_to_space = 0;
313 static int stat_wbarrier_add_to_global_remset = 0;
314 static int stat_wbarrier_set_field = 0;
315 static int stat_wbarrier_set_arrayref = 0;
316 static int stat_wbarrier_arrayref_copy = 0;
317 static int stat_wbarrier_generic_store = 0;
318 static int stat_wbarrier_generic_store_atomic = 0;
319 static int stat_wbarrier_set_root = 0;
320 static int stat_wbarrier_value_copy = 0;
321 static int stat_wbarrier_object_copy = 0;
324 static long long stat_pinned_objects = 0;
326 static long long time_minor_pre_collection_fragment_clear = 0;
327 static long long time_minor_pinning = 0;
328 static long long time_minor_scan_remsets = 0;
329 static long long time_minor_scan_pinned = 0;
330 static long long time_minor_scan_registered_roots = 0;
331 static long long time_minor_scan_thread_data = 0;
332 static long long time_minor_finish_gray_stack = 0;
333 static long long time_minor_fragment_creation = 0;
335 static long long time_major_pre_collection_fragment_clear = 0;
336 static long long time_major_pinning = 0;
337 static long long time_major_scan_pinned = 0;
338 static long long time_major_scan_registered_roots = 0;
339 static long long time_major_scan_thread_data = 0;
340 static long long time_major_scan_alloc_pinned = 0;
341 static long long time_major_scan_finalized = 0;
342 static long long time_major_scan_big_objects = 0;
343 static long long time_major_finish_gray_stack = 0;
344 static long long time_major_free_bigobjs = 0;
345 static long long time_major_los_sweep = 0;
346 static long long time_major_sweep = 0;
347 static long long time_major_fragment_creation = 0;
349 static SGEN_TV_DECLARE (time_major_conc_collection_start);
350 static SGEN_TV_DECLARE (time_major_conc_collection_end);
352 static SGEN_TV_DECLARE (last_minor_collection_start_tv);
353 static SGEN_TV_DECLARE (last_minor_collection_end_tv);
355 int gc_debug_level = 0;
358 static MonoGCFinalizerCallbacks fin_callbacks;
362 mono_gc_flush_info (void)
364 fflush (gc_debug_file);
368 #define TV_DECLARE SGEN_TV_DECLARE
369 #define TV_GETTIME SGEN_TV_GETTIME
370 #define TV_ELAPSED SGEN_TV_ELAPSED
371 #define TV_ELAPSED_MS SGEN_TV_ELAPSED_MS
373 SGEN_TV_DECLARE (sgen_init_timestamp);
375 #define ALIGN_TO(val,align) ((((guint64)val) + ((align) - 1)) & ~((align) - 1))
377 NurseryClearPolicy nursery_clear_policy = CLEAR_AT_TLAB_CREATION;
379 #define object_is_forwarded SGEN_OBJECT_IS_FORWARDED
380 #define object_is_pinned SGEN_OBJECT_IS_PINNED
381 #define pin_object SGEN_PIN_OBJECT
382 #define unpin_object SGEN_UNPIN_OBJECT
384 #define ptr_in_nursery sgen_ptr_in_nursery
386 #define LOAD_VTABLE SGEN_LOAD_VTABLE
389 safe_name (void* obj)
391 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (obj);
392 return vt->klass->name;
395 #define safe_object_get_size sgen_safe_object_get_size
398 sgen_safe_name (void* obj)
400 return safe_name (obj);
404 * ######################################################################
405 * ######## Global data.
406 * ######################################################################
408 LOCK_DECLARE (gc_mutex);
409 gboolean sgen_try_free_some_memory;
411 #define SCAN_START_SIZE SGEN_SCAN_START_SIZE
413 static mword pagesize = 4096;
414 size_t degraded_mode = 0;
416 static mword bytes_pinned_from_failed_allocation = 0;
418 GCMemSection *nursery_section = NULL;
419 static mword lowest_heap_address = ~(mword)0;
420 static mword highest_heap_address = 0;
422 LOCK_DECLARE (sgen_interruption_mutex);
423 static LOCK_DECLARE (pin_queue_mutex);
425 #define LOCK_PIN_QUEUE mono_mutex_lock (&pin_queue_mutex)
426 #define UNLOCK_PIN_QUEUE mono_mutex_unlock (&pin_queue_mutex)
428 typedef struct _FinalizeReadyEntry FinalizeReadyEntry;
429 struct _FinalizeReadyEntry {
430 FinalizeReadyEntry *next;
434 typedef struct _EphemeronLinkNode EphemeronLinkNode;
436 struct _EphemeronLinkNode {
437 EphemeronLinkNode *next;
446 int current_collection_generation = -1;
447 volatile gboolean concurrent_collection_in_progress = FALSE;
449 /* objects that are ready to be finalized */
450 static FinalizeReadyEntry *fin_ready_list = NULL;
451 static FinalizeReadyEntry *critical_fin_list = NULL;
453 static EphemeronLinkNode *ephemeron_list;
455 /* registered roots: the key to the hash is the root start address */
457 * Different kinds of roots are kept separate to speed up pin_from_roots () for example.
459 SgenHashTable roots_hash [ROOT_TYPE_NUM] = {
460 SGEN_HASH_TABLE_INIT (INTERNAL_MEM_ROOTS_TABLE, INTERNAL_MEM_ROOT_RECORD, sizeof (RootRecord), mono_aligned_addr_hash, NULL),
461 SGEN_HASH_TABLE_INIT (INTERNAL_MEM_ROOTS_TABLE, INTERNAL_MEM_ROOT_RECORD, sizeof (RootRecord), mono_aligned_addr_hash, NULL),
462 SGEN_HASH_TABLE_INIT (INTERNAL_MEM_ROOTS_TABLE, INTERNAL_MEM_ROOT_RECORD, sizeof (RootRecord), mono_aligned_addr_hash, NULL)
464 static mword roots_size = 0; /* amount of memory in the root set */
466 #define GC_ROOT_NUM 32
468 int count; /* must be the first field */
469 void *objects [GC_ROOT_NUM];
470 int root_types [GC_ROOT_NUM];
471 uintptr_t extra_info [GC_ROOT_NUM];
475 notify_gc_roots (GCRootReport *report)
479 mono_profiler_gc_roots (report->count, report->objects, report->root_types, report->extra_info);
484 add_profile_gc_root (GCRootReport *report, void *object, int rtype, uintptr_t extra_info)
486 if (report->count == GC_ROOT_NUM)
487 notify_gc_roots (report);
488 report->objects [report->count] = object;
489 report->root_types [report->count] = rtype;
490 report->extra_info [report->count++] = (uintptr_t)((MonoVTable*)LOAD_VTABLE (object))->klass;
493 MonoNativeTlsKey thread_info_key;
495 #ifdef HAVE_KW_THREAD
496 __thread SgenThreadInfo *sgen_thread_info;
497 __thread char *stack_end;
500 /* The size of a TLAB */
501 /* The bigger the value, the less often we have to go to the slow path to allocate a new
502 * one, but the more space is wasted by threads not allocating much memory.
504 * FIXME: Make this self-tuning for each thread.
506 guint32 tlab_size = (1024 * 4);
508 #define MAX_SMALL_OBJ_SIZE SGEN_MAX_SMALL_OBJ_SIZE
510 /* Functions supplied by the runtime to be called by the GC */
511 static MonoGCCallbacks gc_callbacks;
513 #define ALLOC_ALIGN SGEN_ALLOC_ALIGN
514 #define ALLOC_ALIGN_BITS SGEN_ALLOC_ALIGN_BITS
516 #define ALIGN_UP SGEN_ALIGN_UP
518 #define MOVED_OBJECTS_NUM 64
519 static void *moved_objects [MOVED_OBJECTS_NUM];
520 static int moved_objects_idx = 0;
522 /* Vtable of the objects used to fill out nursery fragments before a collection */
523 static MonoVTable *array_fill_vtable;
525 #ifdef SGEN_DEBUG_INTERNAL_ALLOC
526 MonoNativeThreadId main_gc_thread = NULL;
529 /*Object was pinned during the current collection*/
530 static mword objects_pinned;
533 * ######################################################################
534 * ######## Macros and function declarations.
535 * ######################################################################
539 align_pointer (void *ptr)
541 mword p = (mword)ptr;
542 p += sizeof (gpointer) - 1;
543 p &= ~ (sizeof (gpointer) - 1);
547 typedef SgenGrayQueue GrayQueue;
549 /* forward declarations */
550 static void scan_thread_data (void *start_nursery, void *end_nursery, gboolean precise, GrayQueue *queue);
551 static void scan_from_registered_roots (char *addr_start, char *addr_end, int root_type, ScanCopyContext ctx);
552 static void scan_finalizer_entries (FinalizeReadyEntry *list, ScanCopyContext ctx);
553 static void report_finalizer_roots (void);
554 static void report_registered_roots (void);
556 static void pin_from_roots (void *start_nursery, void *end_nursery, GrayQueue *queue);
557 static void finish_gray_stack (int generation, GrayQueue *queue);
559 void mono_gc_scan_for_specific_ref (MonoObject *key, gboolean precise);
562 static void init_stats (void);
564 static int mark_ephemerons_in_range (ScanCopyContext ctx);
565 static void clear_unreachable_ephemerons (ScanCopyContext ctx);
566 static void null_ephemerons_for_domain (MonoDomain *domain);
568 static gboolean major_update_or_finish_concurrent_collection (gboolean force_finish);
570 SgenObjectOperations current_object_ops;
571 SgenMajorCollector major_collector;
572 SgenMinorCollector sgen_minor_collector;
573 static GrayQueue gray_queue;
575 static SgenRemeberedSet remset;
577 /* The gray queue to use from the main collection thread. */
578 #define WORKERS_DISTRIBUTE_GRAY_QUEUE (&gray_queue)
581 * The gray queue a worker job must use. If we're not parallel or
582 * concurrent, we use the main gray queue.
584 static SgenGrayQueue*
585 sgen_workers_get_job_gray_queue (WorkerData *worker_data)
587 return worker_data ? &worker_data->private_gray_queue : WORKERS_DISTRIBUTE_GRAY_QUEUE;
591 gray_queue_redirect (SgenGrayQueue *queue)
593 gboolean wake = FALSE;
597 GrayQueueSection *section = sgen_gray_object_dequeue_section (queue);
600 sgen_section_gray_queue_enqueue (queue->alloc_prepare_data, section);
605 g_assert (concurrent_collection_in_progress);
606 if (sgen_workers_have_started ()) {
607 sgen_workers_wake_up_all ();
609 if (concurrent_collection_in_progress)
610 g_assert (current_collection_generation == -1);
616 sgen_scan_area_with_callback (char *start, char *end, IterateObjectCallbackFunc callback, void *data, gboolean allow_flags)
618 while (start < end) {
622 if (!*(void**)start) {
623 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
628 if (!(obj = SGEN_OBJECT_IS_FORWARDED (start)))
634 size = ALIGN_UP (safe_object_get_size ((MonoObject*)obj));
636 if ((MonoVTable*)SGEN_LOAD_VTABLE (obj) != array_fill_vtable)
637 callback (obj, size, data);
644 need_remove_object_for_domain (char *start, MonoDomain *domain)
646 if (mono_object_domain (start) == domain) {
647 SGEN_LOG (4, "Need to cleanup object %p", start);
648 binary_protocol_cleanup (start, (gpointer)LOAD_VTABLE (start), safe_object_get_size ((MonoObject*)start));
655 process_object_for_domain_clearing (char *start, MonoDomain *domain)
657 GCVTable *vt = (GCVTable*)LOAD_VTABLE (start);
658 if (vt->klass == mono_defaults.internal_thread_class)
659 g_assert (mono_object_domain (start) == mono_get_root_domain ());
660 /* The object could be a proxy for an object in the domain
662 #ifndef DISABLE_REMOTING
663 if (mono_defaults.real_proxy_class->supertypes && mono_class_has_parent_fast (vt->klass, mono_defaults.real_proxy_class)) {
664 MonoObject *server = ((MonoRealProxy*)start)->unwrapped_server;
666 /* The server could already have been zeroed out, so
667 we need to check for that, too. */
668 if (server && (!LOAD_VTABLE (server) || mono_object_domain (server) == domain)) {
669 SGEN_LOG (4, "Cleaning up remote pointer in %p to object %p", start, server);
670 ((MonoRealProxy*)start)->unwrapped_server = NULL;
677 clear_domain_process_object (char *obj, MonoDomain *domain)
681 process_object_for_domain_clearing (obj, domain);
682 remove = need_remove_object_for_domain (obj, domain);
684 if (remove && ((MonoObject*)obj)->synchronisation) {
685 void **dislink = mono_monitor_get_object_monitor_weak_link ((MonoObject*)obj);
687 sgen_register_disappearing_link (NULL, dislink, FALSE, TRUE);
694 clear_domain_process_minor_object_callback (char *obj, size_t size, MonoDomain *domain)
696 if (clear_domain_process_object (obj, domain))
697 memset (obj, 0, size);
701 clear_domain_process_major_object_callback (char *obj, size_t size, MonoDomain *domain)
703 clear_domain_process_object (obj, domain);
707 clear_domain_free_major_non_pinned_object_callback (char *obj, size_t size, MonoDomain *domain)
709 if (need_remove_object_for_domain (obj, domain))
710 major_collector.free_non_pinned_object (obj, size);
714 clear_domain_free_major_pinned_object_callback (char *obj, size_t size, MonoDomain *domain)
716 if (need_remove_object_for_domain (obj, domain))
717 major_collector.free_pinned_object (obj, size);
721 * When appdomains are unloaded we can easily remove objects that have finalizers,
722 * but all the others could still be present in random places on the heap.
723 * We need a sweep to get rid of them even though it's going to be costly
725 * The reason we need to remove them is because we access the vtable and class
726 * structures to know the object size and the reference bitmap: once the domain is
727 * unloaded the point to random memory.
730 mono_gc_clear_domain (MonoDomain * domain)
732 LOSObject *bigobj, *prev;
737 binary_protocol_domain_unload_begin (domain);
741 if (concurrent_collection_in_progress)
742 sgen_perform_collection (0, GENERATION_OLD, "clear domain", TRUE);
743 g_assert (!concurrent_collection_in_progress);
745 sgen_process_fin_stage_entries ();
746 sgen_process_dislink_stage_entries ();
748 sgen_clear_nursery_fragments ();
750 if (xdomain_checks && domain != mono_get_root_domain ()) {
751 sgen_scan_for_registered_roots_in_domain (domain, ROOT_TYPE_NORMAL);
752 sgen_scan_for_registered_roots_in_domain (domain, ROOT_TYPE_WBARRIER);
753 sgen_check_for_xdomain_refs ();
756 /*Ephemerons and dislinks must be processed before LOS since they might end up pointing
757 to memory returned to the OS.*/
758 null_ephemerons_for_domain (domain);
760 for (i = GENERATION_NURSERY; i < GENERATION_MAX; ++i)
761 sgen_null_links_for_domain (domain, i);
763 for (i = GENERATION_NURSERY; i < GENERATION_MAX; ++i)
764 sgen_remove_finalizers_for_domain (domain, i);
766 sgen_scan_area_with_callback (nursery_section->data, nursery_section->end_data,
767 (IterateObjectCallbackFunc)clear_domain_process_minor_object_callback, domain, FALSE);
769 /* We need two passes over major and large objects because
770 freeing such objects might give their memory back to the OS
771 (in the case of large objects) or obliterate its vtable
772 (pinned objects with major-copying or pinned and non-pinned
773 objects with major-mark&sweep), but we might need to
774 dereference a pointer from an object to another object if
775 the first object is a proxy. */
776 major_collector.iterate_objects (ITERATE_OBJECTS_SWEEP_ALL, (IterateObjectCallbackFunc)clear_domain_process_major_object_callback, domain);
777 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
778 clear_domain_process_object (bigobj->data, domain);
781 for (bigobj = los_object_list; bigobj;) {
782 if (need_remove_object_for_domain (bigobj->data, domain)) {
783 LOSObject *to_free = bigobj;
785 prev->next = bigobj->next;
787 los_object_list = bigobj->next;
788 bigobj = bigobj->next;
789 SGEN_LOG (4, "Freeing large object %p", bigobj->data);
790 sgen_los_free_object (to_free);
794 bigobj = bigobj->next;
796 major_collector.iterate_objects (ITERATE_OBJECTS_SWEEP_NON_PINNED, (IterateObjectCallbackFunc)clear_domain_free_major_non_pinned_object_callback, domain);
797 major_collector.iterate_objects (ITERATE_OBJECTS_SWEEP_PINNED, (IterateObjectCallbackFunc)clear_domain_free_major_pinned_object_callback, domain);
799 if (domain == mono_get_root_domain ()) {
800 if (G_UNLIKELY (do_pin_stats))
801 sgen_pin_stats_print_class_stats ();
802 sgen_object_layout_dump (stdout);
805 sgen_restart_world (0, NULL);
807 binary_protocol_domain_unload_end (domain);
813 * sgen_add_to_global_remset:
815 * The global remset contains locations which point into newspace after
816 * a minor collection. This can happen if the objects they point to are pinned.
818 * LOCKING: If called from a parallel collector, the global remset
819 * lock must be held. For serial collectors that is not necessary.
822 sgen_add_to_global_remset (gpointer ptr, gpointer obj)
824 SGEN_ASSERT (5, sgen_ptr_in_nursery (obj), "Target pointer of global remset must be in the nursery");
826 HEAVY_STAT (++stat_wbarrier_add_to_global_remset);
828 if (!major_collector.is_concurrent) {
829 SGEN_ASSERT (5, current_collection_generation != -1, "Global remsets can only be added during collections");
831 if (current_collection_generation == -1)
832 SGEN_ASSERT (5, sgen_concurrent_collection_in_progress (), "Global remsets outside of collection pauses can only be added by the concurrent collector");
835 if (!object_is_pinned (obj))
836 SGEN_ASSERT (5, sgen_minor_collector.is_split || sgen_concurrent_collection_in_progress (), "Non-pinned objects can only remain in nursery if it is a split nursery");
837 else if (sgen_cement_lookup_or_register (obj))
840 remset.record_pointer (ptr);
842 if (G_UNLIKELY (do_pin_stats))
843 sgen_pin_stats_register_global_remset (obj);
845 SGEN_LOG (8, "Adding global remset for %p", ptr);
846 binary_protocol_global_remset (ptr, obj, (gpointer)SGEN_LOAD_VTABLE (obj));
850 if (G_UNLIKELY (MONO_GC_GLOBAL_REMSET_ADD_ENABLED ())) {
851 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (obj);
852 MONO_GC_GLOBAL_REMSET_ADD ((mword)ptr, (mword)obj, sgen_safe_object_get_size (obj),
853 vt->klass->name_space, vt->klass->name);
859 * sgen_drain_gray_stack:
861 * Scan objects in the gray stack until the stack is empty. This should be called
862 * frequently after each object is copied, to achieve better locality and cache
865 * max_objs is the maximum number of objects to scan, or -1 to scan until the stack is
869 sgen_drain_gray_stack (int max_objs, ScanCopyContext ctx)
871 ScanObjectFunc scan_func = ctx.scan_func;
872 GrayQueue *queue = ctx.queue;
876 for (i = 0; i != max_objs; ++i) {
879 GRAY_OBJECT_DEQUEUE (queue, &obj, &desc);
882 SGEN_LOG (9, "Precise gray object scan %p (%s)", obj, safe_name (obj));
883 scan_func (obj, desc, queue);
885 } while (max_objs < 0);
890 * Addresses in the pin queue are already sorted. This function finds
891 * the object header for each address and pins the object. The
892 * addresses must be inside the nursery section. The (start of the)
893 * address array is overwritten with the addresses of the actually
894 * pinned objects. Return the number of pinned objects.
897 pin_objects_from_nursery_pin_queue (ScanCopyContext ctx)
899 GCMemSection *section = nursery_section;
900 void **start = section->pin_queue_start;
901 void **end = start + section->pin_queue_num_entries;
902 void *start_nursery = section->data;
903 void *end_nursery = section->next_data;
908 void *pinning_front = start_nursery;
910 void **definitely_pinned = start;
911 ScanObjectFunc scan_func = ctx.scan_func;
912 SgenGrayQueue *queue = ctx.queue;
914 sgen_nursery_allocator_prepare_for_pinning ();
916 while (start < end) {
917 void *obj_to_pin = NULL;
918 size_t obj_to_pin_size = 0;
923 SGEN_ASSERT (0, addr >= start_nursery && addr < end_nursery, "Potential pinning address out of range");
924 SGEN_ASSERT (0, addr >= last, "Pin queue not sorted");
931 SGEN_LOG (5, "Considering pinning addr %p", addr);
932 /* We've already processed everything up to pinning_front. */
933 if (addr < pinning_front) {
939 * Find the closest scan start <= addr. We might search backward in the
940 * scan_starts array because entries might be NULL. In the worst case we
941 * start at start_nursery.
943 idx = ((char*)addr - (char*)section->data) / SCAN_START_SIZE;
944 SGEN_ASSERT (0, idx < section->num_scan_start, "Scan start index out of range");
945 search_start = (void*)section->scan_starts [idx];
946 if (!search_start || search_start > addr) {
949 search_start = section->scan_starts [idx];
950 if (search_start && search_start <= addr)
953 if (!search_start || search_start > addr)
954 search_start = start_nursery;
958 * If the pinning front is closer than the scan start we found, start
959 * searching at the front.
961 if (search_start < pinning_front)
962 search_start = pinning_front;
965 * Now addr should be in an object a short distance from search_start.
967 * search_start must point to zeroed mem or point to an object.
973 if (!*(void**)search_start) {
974 search_start = (void*)ALIGN_UP ((mword)search_start + sizeof (gpointer));
975 /* The loop condition makes sure we don't overrun addr. */
979 obj_size = ALIGN_UP (safe_object_get_size ((MonoObject*)search_start));
981 if (addr >= search_start && (char*)addr < (char*)search_start + obj_size) {
982 /* This is the object we're looking for. */
983 obj_to_pin = search_start;
984 obj_to_pin_size = obj_size;
988 /* Skip to the next object */
989 search_start = (void*)((char*)search_start + obj_size);
990 } while (search_start <= addr);
992 /* We've searched past the address we were looking for. */
994 pinning_front = search_start;
995 goto next_pin_queue_entry;
999 * We've found an object to pin. It might still be a dummy array, but we
1000 * can advance the pinning front in any case.
1002 pinning_front = (char*)obj_to_pin + obj_to_pin_size;
1005 * If this is a dummy array marking the beginning of a nursery
1006 * fragment, we don't pin it.
1008 if (((MonoObject*)obj_to_pin)->synchronisation == GINT_TO_POINTER (-1))
1009 goto next_pin_queue_entry;
1012 * Finally - pin the object!
1014 desc = sgen_obj_get_descriptor_safe (obj_to_pin);
1016 scan_func (obj_to_pin, desc, queue);
1018 SGEN_LOG (4, "Pinned object %p, vtable %p (%s), count %d\n",
1019 obj_to_pin, *(void**)obj_to_pin, safe_name (obj_to_pin), count);
1020 binary_protocol_pin (obj_to_pin,
1021 (gpointer)LOAD_VTABLE (obj_to_pin),
1022 safe_object_get_size (obj_to_pin));
1024 #ifdef ENABLE_DTRACE
1025 if (G_UNLIKELY (MONO_GC_OBJ_PINNED_ENABLED ())) {
1026 int gen = sgen_ptr_in_nursery (obj_to_pin) ? GENERATION_NURSERY : GENERATION_OLD;
1027 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (obj_to_pin);
1028 MONO_GC_OBJ_PINNED ((mword)obj_to_pin,
1029 sgen_safe_object_get_size (obj_to_pin),
1030 vt->klass->name_space, vt->klass->name, gen);
1034 pin_object (obj_to_pin);
1035 GRAY_OBJECT_ENQUEUE (queue, obj_to_pin, desc);
1036 if (G_UNLIKELY (do_pin_stats))
1037 sgen_pin_stats_register_object (obj_to_pin, obj_to_pin_size);
1038 definitely_pinned [count] = obj_to_pin;
1042 next_pin_queue_entry:
1046 //printf ("effective pinned: %d (at the end: %d)\n", count, (char*)end_nursery - (char*)last);
1047 if (mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS) {
1048 GCRootReport report;
1050 for (idx = 0; idx < count; ++idx)
1051 add_profile_gc_root (&report, definitely_pinned [idx], MONO_PROFILE_GC_ROOT_PINNING | MONO_PROFILE_GC_ROOT_MISC, 0);
1052 notify_gc_roots (&report);
1054 stat_pinned_objects += count;
1059 pin_objects_in_nursery (ScanCopyContext ctx)
1063 if (!nursery_section->pin_queue_num_entries)
1066 reduced_to = pin_objects_from_nursery_pin_queue (ctx);
1067 nursery_section->pin_queue_num_entries = reduced_to;
1069 nursery_section->pin_queue_start = NULL;
1074 sgen_pin_object (void *object, GrayQueue *queue)
1076 g_assert (!concurrent_collection_in_progress);
1078 SGEN_PIN_OBJECT (object);
1079 sgen_pin_stage_ptr (object);
1081 if (G_UNLIKELY (do_pin_stats))
1082 sgen_pin_stats_register_object (object, safe_object_get_size (object));
1084 GRAY_OBJECT_ENQUEUE (queue, object, sgen_obj_get_descriptor (object));
1085 binary_protocol_pin (object, (gpointer)LOAD_VTABLE (object), safe_object_get_size (object));
1087 #ifdef ENABLE_DTRACE
1088 if (G_UNLIKELY (MONO_GC_OBJ_PINNED_ENABLED ())) {
1089 int gen = sgen_ptr_in_nursery (object) ? GENERATION_NURSERY : GENERATION_OLD;
1090 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (object);
1091 MONO_GC_OBJ_PINNED ((mword)object, sgen_safe_object_get_size (object), vt->klass->name_space, vt->klass->name, gen);
1097 sgen_parallel_pin_or_update (void **ptr, void *obj, MonoVTable *vt, SgenGrayQueue *queue)
1101 gboolean major_pinned = FALSE;
1103 if (sgen_ptr_in_nursery (obj)) {
1104 if (SGEN_CAS_PTR (obj, SGEN_POINTER_TAG_PINNED (vt), vt) == vt) {
1105 sgen_pin_object (obj, queue);
1109 major_collector.pin_major_object (obj, queue);
1110 major_pinned = TRUE;
1113 vtable_word = *(mword*)obj;
1114 /*someone else forwarded it, update the pointer and bail out*/
1115 if (SGEN_POINTER_IS_TAGGED_FORWARDED (vtable_word)) {
1116 *ptr = SGEN_POINTER_UNTAG_VTABLE (vtable_word);
1120 /*someone pinned it, nothing to do.*/
1121 if (SGEN_POINTER_IS_TAGGED_PINNED (vtable_word) || major_pinned)
1126 /* Sort the addresses in array in increasing order.
1127 * Done using a by-the book heap sort. Which has decent and stable performance, is pretty cache efficient.
1130 sgen_sort_addresses (void **array, size_t size)
1135 for (i = 1; i < size; ++i) {
1138 size_t parent = (child - 1) / 2;
1140 if (array [parent] >= array [child])
1143 tmp = array [parent];
1144 array [parent] = array [child];
1145 array [child] = tmp;
1151 for (i = size - 1; i > 0; --i) {
1154 array [i] = array [0];
1160 while (root * 2 + 1 <= end) {
1161 size_t child = root * 2 + 1;
1163 if (child < end && array [child] < array [child + 1])
1165 if (array [root] >= array [child])
1169 array [root] = array [child];
1170 array [child] = tmp;
1178 * Scan the memory between start and end and queue values which could be pointers
1179 * to the area between start_nursery and end_nursery for later consideration.
1180 * Typically used for thread stacks.
1183 conservatively_pin_objects_from (void **start, void **end, void *start_nursery, void *end_nursery, int pin_type)
1187 #ifdef VALGRIND_MAKE_MEM_DEFINED_IF_ADDRESSABLE
1188 VALGRIND_MAKE_MEM_DEFINED_IF_ADDRESSABLE (start, (char*)end - (char*)start);
1191 while (start < end) {
1192 if (*start >= start_nursery && *start < end_nursery) {
1194 * *start can point to the middle of an object
1195 * note: should we handle pointing at the end of an object?
1196 * pinning in C# code disallows pointing at the end of an object
1197 * but there is some small chance that an optimizing C compiler
1198 * may keep the only reference to an object by pointing
1199 * at the end of it. We ignore this small chance for now.
1200 * Pointers to the end of an object are indistinguishable
1201 * from pointers to the start of the next object in memory
1202 * so if we allow that we'd need to pin two objects...
1203 * We queue the pointer in an array, the
1204 * array will then be sorted and uniqued. This way
1205 * we can coalesce several pinning pointers and it should
1206 * be faster since we'd do a memory scan with increasing
1207 * addresses. Note: we can align the address to the allocation
1208 * alignment, so the unique process is more effective.
1210 mword addr = (mword)*start;
1211 addr &= ~(ALLOC_ALIGN - 1);
1212 if (addr >= (mword)start_nursery && addr < (mword)end_nursery) {
1213 SGEN_LOG (6, "Pinning address %p from %p", (void*)addr, start);
1214 sgen_pin_stage_ptr ((void*)addr);
1217 if (G_UNLIKELY (do_pin_stats)) {
1218 if (ptr_in_nursery ((void*)addr))
1219 sgen_pin_stats_register_address ((char*)addr, pin_type);
1225 SGEN_LOG (7, "found %d potential pinned heap pointers", count);
1229 * The first thing we do in a collection is to identify pinned objects.
1230 * This function considers all the areas of memory that need to be
1231 * conservatively scanned.
1234 pin_from_roots (void *start_nursery, void *end_nursery, GrayQueue *queue)
1238 SGEN_LOG (2, "Scanning pinned roots (%d bytes, %d/%d entries)", (int)roots_size, roots_hash [ROOT_TYPE_NORMAL].num_entries, roots_hash [ROOT_TYPE_PINNED].num_entries);
1239 /* objects pinned from the API are inside these roots */
1240 SGEN_HASH_TABLE_FOREACH (&roots_hash [ROOT_TYPE_PINNED], start_root, root) {
1241 SGEN_LOG (6, "Pinned roots %p-%p", start_root, root->end_root);
1242 conservatively_pin_objects_from (start_root, (void**)root->end_root, start_nursery, end_nursery, PIN_TYPE_OTHER);
1243 } SGEN_HASH_TABLE_FOREACH_END;
1244 /* now deal with the thread stacks
1245 * in the future we should be able to conservatively scan only:
1246 * *) the cpu registers
1247 * *) the unmanaged stack frames
1248 * *) the _last_ managed stack frame
1249 * *) pointers slots in managed frames
1251 scan_thread_data (start_nursery, end_nursery, FALSE, queue);
1255 unpin_objects_from_queue (SgenGrayQueue *queue)
1260 GRAY_OBJECT_DEQUEUE (queue, &addr, &desc);
1263 g_assert (SGEN_OBJECT_IS_PINNED (addr));
1264 SGEN_UNPIN_OBJECT (addr);
1269 CopyOrMarkObjectFunc func;
1271 } UserCopyOrMarkData;
1274 single_arg_user_copy_or_mark (void **obj, void *gc_data)
1276 UserCopyOrMarkData *data = gc_data;
1278 data->func (obj, data->queue);
1282 * The memory area from start_root to end_root contains pointers to objects.
1283 * Their position is precisely described by @desc (this means that the pointer
1284 * can be either NULL or the pointer to the start of an object).
1285 * This functions copies them to to_space updates them.
1287 * This function is not thread-safe!
1290 precisely_scan_objects_from (void** start_root, void** end_root, char* n_start, char *n_end, mword desc, ScanCopyContext ctx)
1292 CopyOrMarkObjectFunc copy_func = ctx.copy_func;
1293 SgenGrayQueue *queue = ctx.queue;
1295 switch (desc & ROOT_DESC_TYPE_MASK) {
1296 case ROOT_DESC_BITMAP:
1297 desc >>= ROOT_DESC_TYPE_SHIFT;
1299 if ((desc & 1) && *start_root) {
1300 copy_func (start_root, queue);
1301 SGEN_LOG (9, "Overwrote root at %p with %p", start_root, *start_root);
1302 sgen_drain_gray_stack (-1, ctx);
1308 case ROOT_DESC_COMPLEX: {
1309 gsize *bitmap_data = sgen_get_complex_descriptor_bitmap (desc);
1310 gsize bwords = (*bitmap_data) - 1;
1311 void **start_run = start_root;
1313 while (bwords-- > 0) {
1314 gsize bmap = *bitmap_data++;
1315 void **objptr = start_run;
1317 if ((bmap & 1) && *objptr) {
1318 copy_func (objptr, queue);
1319 SGEN_LOG (9, "Overwrote root at %p with %p", objptr, *objptr);
1320 sgen_drain_gray_stack (-1, ctx);
1325 start_run += GC_BITS_PER_WORD;
1329 case ROOT_DESC_USER: {
1330 UserCopyOrMarkData data = { copy_func, queue };
1331 MonoGCRootMarkFunc marker = sgen_get_user_descriptor_func (desc);
1332 marker (start_root, single_arg_user_copy_or_mark, &data);
1335 case ROOT_DESC_RUN_LEN:
1336 g_assert_not_reached ();
1338 g_assert_not_reached ();
1343 reset_heap_boundaries (void)
1345 lowest_heap_address = ~(mword)0;
1346 highest_heap_address = 0;
1350 sgen_update_heap_boundaries (mword low, mword high)
1355 old = lowest_heap_address;
1358 } while (SGEN_CAS_PTR ((gpointer*)&lowest_heap_address, (gpointer)low, (gpointer)old) != (gpointer)old);
1361 old = highest_heap_address;
1364 } while (SGEN_CAS_PTR ((gpointer*)&highest_heap_address, (gpointer)high, (gpointer)old) != (gpointer)old);
1368 * Allocate and setup the data structures needed to be able to allocate objects
1369 * in the nursery. The nursery is stored in nursery_section.
1372 alloc_nursery (void)
1374 GCMemSection *section;
1379 if (nursery_section)
1381 SGEN_LOG (2, "Allocating nursery size: %zu", (size_t)sgen_nursery_size);
1382 /* later we will alloc a larger area for the nursery but only activate
1383 * what we need. The rest will be used as expansion if we have too many pinned
1384 * objects in the existing nursery.
1386 /* FIXME: handle OOM */
1387 section = sgen_alloc_internal (INTERNAL_MEM_SECTION);
1389 alloc_size = sgen_nursery_size;
1391 /* If there isn't enough space even for the nursery we should simply abort. */
1392 g_assert (sgen_memgov_try_alloc_space (alloc_size, SPACE_NURSERY));
1394 #ifdef SGEN_ALIGN_NURSERY
1395 data = major_collector.alloc_heap (alloc_size, alloc_size, DEFAULT_NURSERY_BITS);
1397 data = major_collector.alloc_heap (alloc_size, 0, DEFAULT_NURSERY_BITS);
1399 sgen_update_heap_boundaries ((mword)data, (mword)(data + sgen_nursery_size));
1400 SGEN_LOG (4, "Expanding nursery size (%p-%p): %lu, total: %lu", data, data + alloc_size, (unsigned long)sgen_nursery_size, (unsigned long)mono_gc_get_heap_size ());
1401 section->data = section->next_data = data;
1402 section->size = alloc_size;
1403 section->end_data = data + sgen_nursery_size;
1404 scan_starts = (alloc_size + SCAN_START_SIZE - 1) / SCAN_START_SIZE;
1405 section->scan_starts = sgen_alloc_internal_dynamic (sizeof (char*) * scan_starts, INTERNAL_MEM_SCAN_STARTS, TRUE);
1406 section->num_scan_start = scan_starts;
1408 nursery_section = section;
1410 sgen_nursery_allocator_set_nursery_bounds (data, data + sgen_nursery_size);
1414 mono_gc_get_nursery (int *shift_bits, size_t *size)
1416 *size = sgen_nursery_size;
1417 #ifdef SGEN_ALIGN_NURSERY
1418 *shift_bits = DEFAULT_NURSERY_BITS;
1422 return sgen_get_nursery_start ();
1426 mono_gc_set_current_thread_appdomain (MonoDomain *domain)
1428 SgenThreadInfo *info = mono_thread_info_current ();
1430 /* Could be called from sgen_thread_unregister () with a NULL info */
1433 info->stopped_domain = domain;
1438 mono_gc_precise_stack_mark_enabled (void)
1440 return !conservative_stack_mark;
1444 mono_gc_get_logfile (void)
1446 return gc_debug_file;
1450 report_finalizer_roots_list (FinalizeReadyEntry *list)
1452 GCRootReport report;
1453 FinalizeReadyEntry *fin;
1456 for (fin = list; fin; fin = fin->next) {
1459 add_profile_gc_root (&report, fin->object, MONO_PROFILE_GC_ROOT_FINALIZER, 0);
1461 notify_gc_roots (&report);
1465 report_finalizer_roots (void)
1467 report_finalizer_roots_list (fin_ready_list);
1468 report_finalizer_roots_list (critical_fin_list);
1471 static GCRootReport *root_report;
1474 single_arg_report_root (void **obj, void *gc_data)
1477 add_profile_gc_root (root_report, *obj, MONO_PROFILE_GC_ROOT_OTHER, 0);
1481 precisely_report_roots_from (GCRootReport *report, void** start_root, void** end_root, mword desc)
1483 switch (desc & ROOT_DESC_TYPE_MASK) {
1484 case ROOT_DESC_BITMAP:
1485 desc >>= ROOT_DESC_TYPE_SHIFT;
1487 if ((desc & 1) && *start_root) {
1488 add_profile_gc_root (report, *start_root, MONO_PROFILE_GC_ROOT_OTHER, 0);
1494 case ROOT_DESC_COMPLEX: {
1495 gsize *bitmap_data = sgen_get_complex_descriptor_bitmap (desc);
1496 gsize bwords = (*bitmap_data) - 1;
1497 void **start_run = start_root;
1499 while (bwords-- > 0) {
1500 gsize bmap = *bitmap_data++;
1501 void **objptr = start_run;
1503 if ((bmap & 1) && *objptr) {
1504 add_profile_gc_root (report, *objptr, MONO_PROFILE_GC_ROOT_OTHER, 0);
1509 start_run += GC_BITS_PER_WORD;
1513 case ROOT_DESC_USER: {
1514 MonoGCRootMarkFunc marker = sgen_get_user_descriptor_func (desc);
1515 root_report = report;
1516 marker (start_root, single_arg_report_root, NULL);
1519 case ROOT_DESC_RUN_LEN:
1520 g_assert_not_reached ();
1522 g_assert_not_reached ();
1527 report_registered_roots_by_type (int root_type)
1529 GCRootReport report;
1533 SGEN_HASH_TABLE_FOREACH (&roots_hash [root_type], start_root, root) {
1534 SGEN_LOG (6, "Precise root scan %p-%p (desc: %p)", start_root, root->end_root, (void*)root->root_desc);
1535 precisely_report_roots_from (&report, start_root, (void**)root->end_root, root->root_desc);
1536 } SGEN_HASH_TABLE_FOREACH_END;
1537 notify_gc_roots (&report);
1541 report_registered_roots (void)
1543 report_registered_roots_by_type (ROOT_TYPE_NORMAL);
1544 report_registered_roots_by_type (ROOT_TYPE_WBARRIER);
1548 scan_finalizer_entries (FinalizeReadyEntry *list, ScanCopyContext ctx)
1550 CopyOrMarkObjectFunc copy_func = ctx.copy_func;
1551 SgenGrayQueue *queue = ctx.queue;
1552 FinalizeReadyEntry *fin;
1554 for (fin = list; fin; fin = fin->next) {
1557 SGEN_LOG (5, "Scan of fin ready object: %p (%s)\n", fin->object, safe_name (fin->object));
1558 copy_func (&fin->object, queue);
1563 generation_name (int generation)
1565 switch (generation) {
1566 case GENERATION_NURSERY: return "nursery";
1567 case GENERATION_OLD: return "old";
1568 default: g_assert_not_reached ();
1573 sgen_generation_name (int generation)
1575 return generation_name (generation);
1578 SgenObjectOperations *
1579 sgen_get_current_object_ops (void){
1580 return ¤t_object_ops;
1585 finish_gray_stack (int generation, GrayQueue *queue)
1589 int done_with_ephemerons, ephemeron_rounds = 0;
1590 CopyOrMarkObjectFunc copy_func = current_object_ops.copy_or_mark_object;
1591 ScanObjectFunc scan_func = current_object_ops.scan_object;
1592 ScanCopyContext ctx = { scan_func, copy_func, queue };
1593 char *start_addr = generation == GENERATION_NURSERY ? sgen_get_nursery_start () : NULL;
1594 char *end_addr = generation == GENERATION_NURSERY ? sgen_get_nursery_end () : (char*)-1;
1597 * We copied all the reachable objects. Now it's the time to copy
1598 * the objects that were not referenced by the roots, but by the copied objects.
1599 * we built a stack of objects pointed to by gray_start: they are
1600 * additional roots and we may add more items as we go.
1601 * We loop until gray_start == gray_objects which means no more objects have
1602 * been added. Note this is iterative: no recursion is involved.
1603 * We need to walk the LO list as well in search of marked big objects
1604 * (use a flag since this is needed only on major collections). We need to loop
1605 * here as well, so keep a counter of marked LO (increasing it in copy_object).
1606 * To achieve better cache locality and cache usage, we drain the gray stack
1607 * frequently, after each object is copied, and just finish the work here.
1609 sgen_drain_gray_stack (-1, ctx);
1611 SGEN_LOG (2, "%s generation done", generation_name (generation));
1614 Reset bridge data, we might have lingering data from a previous collection if this is a major
1615 collection trigged by minor overflow.
1617 We must reset the gathered bridges since their original block might be evacuated due to major
1618 fragmentation in the meanwhile and the bridge code should not have to deal with that.
1620 if (sgen_need_bridge_processing ())
1621 sgen_bridge_reset_data ();
1624 * Walk the ephemeron tables marking all values with reachable keys. This must be completely done
1625 * before processing finalizable objects and non-tracking weak links to avoid finalizing/clearing
1626 * objects that are in fact reachable.
1628 done_with_ephemerons = 0;
1630 done_with_ephemerons = mark_ephemerons_in_range (ctx);
1631 sgen_drain_gray_stack (-1, ctx);
1633 } while (!done_with_ephemerons);
1635 sgen_mark_togglerefs (start_addr, end_addr, ctx);
1637 if (sgen_need_bridge_processing ()) {
1638 /*Make sure the gray stack is empty before we process bridge objects so we get liveness right*/
1639 sgen_drain_gray_stack (-1, ctx);
1640 sgen_collect_bridge_objects (generation, ctx);
1641 if (generation == GENERATION_OLD)
1642 sgen_collect_bridge_objects (GENERATION_NURSERY, ctx);
1645 Do the first bridge step here, as the collector liveness state will become useless after that.
1647 An important optimization is to only proccess the possibly dead part of the object graph and skip
1648 over all live objects as we transitively know everything they point must be alive too.
1650 The above invariant is completely wrong if we let the gray queue be drained and mark/copy everything.
1652 This has the unfortunate side effect of making overflow collections perform the first step twice, but
1653 given we now have heuristics that perform major GC in anticipation of minor overflows this should not
1656 sgen_bridge_processing_stw_step ();
1660 Make sure we drain the gray stack before processing disappearing links and finalizers.
1661 If we don't make sure it is empty we might wrongly see a live object as dead.
1663 sgen_drain_gray_stack (-1, ctx);
1666 We must clear weak links that don't track resurrection before processing object ready for
1667 finalization so they can be cleared before that.
1669 sgen_null_link_in_range (generation, TRUE, ctx);
1670 if (generation == GENERATION_OLD)
1671 sgen_null_link_in_range (GENERATION_NURSERY, TRUE, ctx);
1674 /* walk the finalization queue and move also the objects that need to be
1675 * finalized: use the finalized objects as new roots so the objects they depend
1676 * on are also not reclaimed. As with the roots above, only objects in the nursery
1677 * are marked/copied.
1679 sgen_finalize_in_range (generation, ctx);
1680 if (generation == GENERATION_OLD)
1681 sgen_finalize_in_range (GENERATION_NURSERY, ctx);
1682 /* drain the new stack that might have been created */
1683 SGEN_LOG (6, "Precise scan of gray area post fin");
1684 sgen_drain_gray_stack (-1, ctx);
1687 * This must be done again after processing finalizable objects since CWL slots are cleared only after the key is finalized.
1689 done_with_ephemerons = 0;
1691 done_with_ephemerons = mark_ephemerons_in_range (ctx);
1692 sgen_drain_gray_stack (-1, ctx);
1694 } while (!done_with_ephemerons);
1697 * Clear ephemeron pairs with unreachable keys.
1698 * We pass the copy func so we can figure out if an array was promoted or not.
1700 clear_unreachable_ephemerons (ctx);
1703 * We clear togglerefs only after all possible chances of revival are done.
1704 * This is semantically more inline with what users expect and it allows for
1705 * user finalizers to correctly interact with TR objects.
1707 sgen_clear_togglerefs (start_addr, end_addr, ctx);
1710 SGEN_LOG (2, "Finalize queue handling scan for %s generation: %d usecs %d ephemeron rounds", generation_name (generation), TV_ELAPSED (atv, btv), ephemeron_rounds);
1713 * handle disappearing links
1714 * Note we do this after checking the finalization queue because if an object
1715 * survives (at least long enough to be finalized) we don't clear the link.
1716 * This also deals with a possible issue with the monitor reclamation: with the Boehm
1717 * GC a finalized object my lose the monitor because it is cleared before the finalizer is
1720 g_assert (sgen_gray_object_queue_is_empty (queue));
1722 sgen_null_link_in_range (generation, FALSE, ctx);
1723 if (generation == GENERATION_OLD)
1724 sgen_null_link_in_range (GENERATION_NURSERY, FALSE, ctx);
1725 if (sgen_gray_object_queue_is_empty (queue))
1727 sgen_drain_gray_stack (-1, ctx);
1730 g_assert (sgen_gray_object_queue_is_empty (queue));
1732 sgen_gray_object_queue_trim_free_list (queue);
1736 sgen_check_section_scan_starts (GCMemSection *section)
1739 for (i = 0; i < section->num_scan_start; ++i) {
1740 if (section->scan_starts [i]) {
1741 mword size = safe_object_get_size ((MonoObject*) section->scan_starts [i]);
1742 g_assert (size >= sizeof (MonoObject) && size <= MAX_SMALL_OBJ_SIZE);
1748 check_scan_starts (void)
1750 if (!do_scan_starts_check)
1752 sgen_check_section_scan_starts (nursery_section);
1753 major_collector.check_scan_starts ();
1757 scan_from_registered_roots (char *addr_start, char *addr_end, int root_type, ScanCopyContext ctx)
1761 SGEN_HASH_TABLE_FOREACH (&roots_hash [root_type], start_root, root) {
1762 SGEN_LOG (6, "Precise root scan %p-%p (desc: %p)", start_root, root->end_root, (void*)root->root_desc);
1763 precisely_scan_objects_from (start_root, (void**)root->end_root, addr_start, addr_end, root->root_desc, ctx);
1764 } SGEN_HASH_TABLE_FOREACH_END;
1768 sgen_dump_occupied (char *start, char *end, char *section_start)
1770 fprintf (heap_dump_file, "<occupied offset=\"%td\" size=\"%td\"/>\n", start - section_start, end - start);
1774 sgen_dump_section (GCMemSection *section, const char *type)
1776 char *start = section->data;
1777 char *end = section->data + section->size;
1778 char *occ_start = NULL;
1780 char *old_start = NULL; /* just for debugging */
1782 fprintf (heap_dump_file, "<section type=\"%s\" size=\"%lu\">\n", type, (unsigned long)section->size);
1784 while (start < end) {
1788 if (!*(void**)start) {
1790 sgen_dump_occupied (occ_start, start, section->data);
1793 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
1796 g_assert (start < section->next_data);
1801 vt = (GCVTable*)LOAD_VTABLE (start);
1804 size = ALIGN_UP (safe_object_get_size ((MonoObject*) start));
1807 fprintf (heap_dump_file, "<object offset=\"%d\" class=\"%s.%s\" size=\"%d\"/>\n",
1808 start - section->data,
1809 vt->klass->name_space, vt->klass->name,
1817 sgen_dump_occupied (occ_start, start, section->data);
1819 fprintf (heap_dump_file, "</section>\n");
1823 dump_object (MonoObject *obj, gboolean dump_location)
1825 static char class_name [1024];
1827 MonoClass *class = mono_object_class (obj);
1831 * Python's XML parser is too stupid to parse angle brackets
1832 * in strings, so we just ignore them;
1835 while (class->name [i] && j < sizeof (class_name) - 1) {
1836 if (!strchr ("<>\"", class->name [i]))
1837 class_name [j++] = class->name [i];
1840 g_assert (j < sizeof (class_name));
1843 fprintf (heap_dump_file, "<object class=\"%s.%s\" size=\"%zd\"",
1844 class->name_space, class_name,
1845 safe_object_get_size (obj));
1846 if (dump_location) {
1847 const char *location;
1848 if (ptr_in_nursery (obj))
1849 location = "nursery";
1850 else if (safe_object_get_size (obj) <= MAX_SMALL_OBJ_SIZE)
1854 fprintf (heap_dump_file, " location=\"%s\"", location);
1856 fprintf (heap_dump_file, "/>\n");
1860 dump_heap (const char *type, int num, const char *reason)
1865 fprintf (heap_dump_file, "<collection type=\"%s\" num=\"%d\"", type, num);
1867 fprintf (heap_dump_file, " reason=\"%s\"", reason);
1868 fprintf (heap_dump_file, ">\n");
1869 fprintf (heap_dump_file, "<other-mem-usage type=\"mempools\" size=\"%ld\"/>\n", mono_mempool_get_bytes_allocated ());
1870 sgen_dump_internal_mem_usage (heap_dump_file);
1871 fprintf (heap_dump_file, "<pinned type=\"stack\" bytes=\"%zu\"/>\n", sgen_pin_stats_get_pinned_byte_count (PIN_TYPE_STACK));
1872 /* fprintf (heap_dump_file, "<pinned type=\"static-data\" bytes=\"%d\"/>\n", pinned_byte_counts [PIN_TYPE_STATIC_DATA]); */
1873 fprintf (heap_dump_file, "<pinned type=\"other\" bytes=\"%zu\"/>\n", sgen_pin_stats_get_pinned_byte_count (PIN_TYPE_OTHER));
1875 fprintf (heap_dump_file, "<pinned-objects>\n");
1876 for (list = sgen_pin_stats_get_object_list (); list; list = list->next)
1877 dump_object (list->obj, TRUE);
1878 fprintf (heap_dump_file, "</pinned-objects>\n");
1880 sgen_dump_section (nursery_section, "nursery");
1882 major_collector.dump_heap (heap_dump_file);
1884 fprintf (heap_dump_file, "<los>\n");
1885 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1886 dump_object ((MonoObject*)bigobj->data, FALSE);
1887 fprintf (heap_dump_file, "</los>\n");
1889 fprintf (heap_dump_file, "</collection>\n");
1893 sgen_register_moved_object (void *obj, void *destination)
1895 g_assert (mono_profiler_events & MONO_PROFILE_GC_MOVES);
1897 if (moved_objects_idx == MOVED_OBJECTS_NUM) {
1898 mono_profiler_gc_moves (moved_objects, moved_objects_idx);
1899 moved_objects_idx = 0;
1901 moved_objects [moved_objects_idx++] = obj;
1902 moved_objects [moved_objects_idx++] = destination;
1908 static gboolean inited = FALSE;
1913 mono_counters_register ("Minor fragment clear", MONO_COUNTER_GC | MONO_COUNTER_LONG | MONO_COUNTER_TIME, &time_minor_pre_collection_fragment_clear);
1914 mono_counters_register ("Minor pinning", MONO_COUNTER_GC | MONO_COUNTER_LONG | MONO_COUNTER_TIME, &time_minor_pinning);
1915 mono_counters_register ("Minor scan remembered set", MONO_COUNTER_GC | MONO_COUNTER_LONG | MONO_COUNTER_TIME, &time_minor_scan_remsets);
1916 mono_counters_register ("Minor scan pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG | MONO_COUNTER_TIME, &time_minor_scan_pinned);
1917 mono_counters_register ("Minor scan registered roots", MONO_COUNTER_GC | MONO_COUNTER_LONG | MONO_COUNTER_TIME, &time_minor_scan_registered_roots);
1918 mono_counters_register ("Minor scan thread data", MONO_COUNTER_GC | MONO_COUNTER_LONG | MONO_COUNTER_TIME, &time_minor_scan_thread_data);
1919 mono_counters_register ("Minor finish gray stack", MONO_COUNTER_GC | MONO_COUNTER_LONG | MONO_COUNTER_TIME, &time_minor_finish_gray_stack);
1920 mono_counters_register ("Minor fragment creation", MONO_COUNTER_GC | MONO_COUNTER_LONG | MONO_COUNTER_TIME, &time_minor_fragment_creation);
1922 mono_counters_register ("Major fragment clear", MONO_COUNTER_GC | MONO_COUNTER_LONG | MONO_COUNTER_TIME, &time_major_pre_collection_fragment_clear);
1923 mono_counters_register ("Major pinning", MONO_COUNTER_GC | MONO_COUNTER_LONG | MONO_COUNTER_TIME, &time_major_pinning);
1924 mono_counters_register ("Major scan pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG | MONO_COUNTER_TIME, &time_major_scan_pinned);
1925 mono_counters_register ("Major scan registered roots", MONO_COUNTER_GC | MONO_COUNTER_LONG | MONO_COUNTER_TIME, &time_major_scan_registered_roots);
1926 mono_counters_register ("Major scan thread data", MONO_COUNTER_GC | MONO_COUNTER_LONG | MONO_COUNTER_TIME, &time_major_scan_thread_data);
1927 mono_counters_register ("Major scan alloc_pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG | MONO_COUNTER_TIME, &time_major_scan_alloc_pinned);
1928 mono_counters_register ("Major scan finalized", MONO_COUNTER_GC | MONO_COUNTER_LONG | MONO_COUNTER_TIME, &time_major_scan_finalized);
1929 mono_counters_register ("Major scan big objects", MONO_COUNTER_GC | MONO_COUNTER_LONG | MONO_COUNTER_TIME, &time_major_scan_big_objects);
1930 mono_counters_register ("Major finish gray stack", MONO_COUNTER_GC | MONO_COUNTER_LONG | MONO_COUNTER_TIME, &time_major_finish_gray_stack);
1931 mono_counters_register ("Major free big objects", MONO_COUNTER_GC | MONO_COUNTER_LONG | MONO_COUNTER_TIME, &time_major_free_bigobjs);
1932 mono_counters_register ("Major LOS sweep", MONO_COUNTER_GC | MONO_COUNTER_LONG | MONO_COUNTER_TIME, &time_major_los_sweep);
1933 mono_counters_register ("Major sweep", MONO_COUNTER_GC | MONO_COUNTER_LONG | MONO_COUNTER_TIME, &time_major_sweep);
1934 mono_counters_register ("Major fragment creation", MONO_COUNTER_GC | MONO_COUNTER_LONG | MONO_COUNTER_TIME, &time_major_fragment_creation);
1936 mono_counters_register ("Number of pinned objects", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_pinned_objects);
1938 #ifdef HEAVY_STATISTICS
1939 mono_counters_register ("WBarrier remember pointer", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_add_to_global_remset);
1940 mono_counters_register ("WBarrier set field", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_field);
1941 mono_counters_register ("WBarrier set arrayref", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_arrayref);
1942 mono_counters_register ("WBarrier arrayref copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_arrayref_copy);
1943 mono_counters_register ("WBarrier generic store called", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_generic_store);
1944 mono_counters_register ("WBarrier generic atomic store called", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_generic_store_atomic);
1945 mono_counters_register ("WBarrier set root", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_root);
1946 mono_counters_register ("WBarrier value copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_value_copy);
1947 mono_counters_register ("WBarrier object copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_object_copy);
1949 mono_counters_register ("# objects allocated degraded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_alloced_degraded);
1950 mono_counters_register ("bytes allocated degraded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_bytes_alloced_degraded);
1952 mono_counters_register ("# copy_object() called (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_called_nursery);
1953 mono_counters_register ("# objects copied (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_copied_nursery);
1954 mono_counters_register ("# copy_object() called (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_called_major);
1955 mono_counters_register ("# objects copied (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_copied_major);
1957 mono_counters_register ("# scan_object() called (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_scan_object_called_nursery);
1958 mono_counters_register ("# scan_object() called (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_scan_object_called_major);
1960 mono_counters_register ("Slots allocated in vain", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_slots_allocated_in_vain);
1962 mono_counters_register ("# nursery copy_object() failed from space", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_from_space);
1963 mono_counters_register ("# nursery copy_object() failed forwarded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_forwarded);
1964 mono_counters_register ("# nursery copy_object() failed pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_pinned);
1965 mono_counters_register ("# nursery copy_object() failed to space", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_to_space);
1967 sgen_nursery_allocator_init_heavy_stats ();
1968 sgen_alloc_init_heavy_stats ();
1976 reset_pinned_from_failed_allocation (void)
1978 bytes_pinned_from_failed_allocation = 0;
1982 sgen_set_pinned_from_failed_allocation (mword objsize)
1984 bytes_pinned_from_failed_allocation += objsize;
1988 sgen_collection_is_concurrent (void)
1990 switch (current_collection_generation) {
1991 case GENERATION_NURSERY:
1993 case GENERATION_OLD:
1994 return concurrent_collection_in_progress;
1996 g_error ("Invalid current generation %d", current_collection_generation);
2001 sgen_concurrent_collection_in_progress (void)
2003 return concurrent_collection_in_progress;
2010 } FinishRememberedSetScanJobData;
2013 job_finish_remembered_set_scan (WorkerData *worker_data, void *job_data_untyped)
2015 FinishRememberedSetScanJobData *job_data = job_data_untyped;
2017 remset.finish_scan_remsets (job_data->heap_start, job_data->heap_end, sgen_workers_get_job_gray_queue (worker_data));
2018 sgen_free_internal_dynamic (job_data, sizeof (FinishRememberedSetScanJobData), INTERNAL_MEM_WORKER_JOB_DATA);
2023 CopyOrMarkObjectFunc copy_or_mark_func;
2024 ScanObjectFunc scan_func;
2028 } ScanFromRegisteredRootsJobData;
2031 job_scan_from_registered_roots (WorkerData *worker_data, void *job_data_untyped)
2033 ScanFromRegisteredRootsJobData *job_data = job_data_untyped;
2034 ScanCopyContext ctx = { job_data->scan_func, job_data->copy_or_mark_func,
2035 sgen_workers_get_job_gray_queue (worker_data) };
2037 scan_from_registered_roots (job_data->heap_start, job_data->heap_end, job_data->root_type, ctx);
2038 sgen_free_internal_dynamic (job_data, sizeof (ScanFromRegisteredRootsJobData), INTERNAL_MEM_WORKER_JOB_DATA);
2045 } ScanThreadDataJobData;
2048 job_scan_thread_data (WorkerData *worker_data, void *job_data_untyped)
2050 ScanThreadDataJobData *job_data = job_data_untyped;
2052 scan_thread_data (job_data->heap_start, job_data->heap_end, TRUE,
2053 sgen_workers_get_job_gray_queue (worker_data));
2054 sgen_free_internal_dynamic (job_data, sizeof (ScanThreadDataJobData), INTERNAL_MEM_WORKER_JOB_DATA);
2058 job_scan_finalizer_entries (WorkerData *worker_data, void *job_data_untyped)
2060 FinalizeReadyEntry *list = job_data_untyped;
2061 ScanCopyContext ctx = { NULL, current_object_ops.copy_or_mark_object, sgen_workers_get_job_gray_queue (worker_data) };
2063 scan_finalizer_entries (list, ctx);
2067 job_scan_major_mod_union_cardtable (WorkerData *worker_data, void *job_data_untyped)
2069 g_assert (concurrent_collection_in_progress);
2070 major_collector.scan_card_table (TRUE, sgen_workers_get_job_gray_queue (worker_data));
2074 job_scan_los_mod_union_cardtable (WorkerData *worker_data, void *job_data_untyped)
2076 g_assert (concurrent_collection_in_progress);
2077 sgen_los_scan_card_table (TRUE, sgen_workers_get_job_gray_queue (worker_data));
2081 verify_scan_starts (char *start, char *end)
2085 for (i = 0; i < nursery_section->num_scan_start; ++i) {
2086 char *addr = nursery_section->scan_starts [i];
2087 if (addr > start && addr < end)
2088 SGEN_LOG (1, "NFC-BAD SCAN START [%zu] %p for obj [%p %p]", i, addr, start, end);
2093 verify_nursery (void)
2095 char *start, *end, *cur, *hole_start;
2097 if (!do_verify_nursery)
2100 /*This cleans up unused fragments */
2101 sgen_nursery_allocator_prepare_for_pinning ();
2103 hole_start = start = cur = sgen_get_nursery_start ();
2104 end = sgen_get_nursery_end ();
2109 if (!*(void**)cur) {
2110 cur += sizeof (void*);
2114 if (object_is_forwarded (cur))
2115 SGEN_LOG (1, "FORWARDED OBJ %p", cur);
2116 else if (object_is_pinned (cur))
2117 SGEN_LOG (1, "PINNED OBJ %p", cur);
2119 ss = safe_object_get_size ((MonoObject*)cur);
2120 size = ALIGN_UP (safe_object_get_size ((MonoObject*)cur));
2121 verify_scan_starts (cur, cur + size);
2122 if (do_dump_nursery_content) {
2123 if (cur > hole_start)
2124 SGEN_LOG (1, "HOLE [%p %p %d]", hole_start, cur, (int)(cur - hole_start));
2125 SGEN_LOG (1, "OBJ [%p %p %d %d %s %d]", cur, cur + size, (int)size, (int)ss, sgen_safe_name ((MonoObject*)cur), (gpointer)LOAD_VTABLE (cur) == sgen_get_array_fill_vtable ());
2133 * Checks that no objects in the nursery are fowarded or pinned. This
2134 * is a precondition to restarting the mutator while doing a
2135 * concurrent collection. Note that we don't clear fragments because
2136 * we depend on that having happened earlier.
2139 check_nursery_is_clean (void)
2141 char *start, *end, *cur;
2143 start = cur = sgen_get_nursery_start ();
2144 end = sgen_get_nursery_end ();
2149 if (!*(void**)cur) {
2150 cur += sizeof (void*);
2154 g_assert (!object_is_forwarded (cur));
2155 g_assert (!object_is_pinned (cur));
2157 ss = safe_object_get_size ((MonoObject*)cur);
2158 size = ALIGN_UP (safe_object_get_size ((MonoObject*)cur));
2159 verify_scan_starts (cur, cur + size);
2166 init_gray_queue (void)
2168 if (sgen_collection_is_concurrent ()) {
2169 sgen_workers_init_distribute_gray_queue ();
2170 sgen_gray_object_queue_init_with_alloc_prepare (&gray_queue, NULL,
2171 gray_queue_redirect, sgen_workers_get_distribute_section_gray_queue ());
2173 sgen_gray_object_queue_init (&gray_queue, NULL);
2178 * Collect objects in the nursery. Returns whether to trigger a major
2182 collect_nursery (SgenGrayQueue *unpin_queue, gboolean finish_up_concurrent_mark)
2184 gboolean needs_major;
2185 size_t max_garbage_amount;
2187 FinishRememberedSetScanJobData *frssjd;
2188 ScanFromRegisteredRootsJobData *scrrjd_normal, *scrrjd_wbarrier;
2189 ScanThreadDataJobData *stdjd;
2190 mword fragment_total;
2191 ScanCopyContext ctx;
2195 if (disable_minor_collections)
2198 TV_GETTIME (last_minor_collection_start_tv);
2199 atv = last_minor_collection_start_tv;
2201 MONO_GC_BEGIN (GENERATION_NURSERY);
2202 binary_protocol_collection_begin (gc_stats.minor_gc_count, GENERATION_NURSERY);
2206 #ifndef DISABLE_PERFCOUNTERS
2207 mono_perfcounters->gc_collections0++;
2210 current_collection_generation = GENERATION_NURSERY;
2211 current_object_ops = sgen_minor_collector.serial_ops;
2213 reset_pinned_from_failed_allocation ();
2215 check_scan_starts ();
2217 sgen_nursery_alloc_prepare_for_minor ();
2221 nursery_next = sgen_nursery_alloc_get_upper_alloc_bound ();
2222 /* FIXME: optimize later to use the higher address where an object can be present */
2223 nursery_next = MAX (nursery_next, sgen_get_nursery_end ());
2225 SGEN_LOG (1, "Start nursery collection %d %p-%p, size: %d", gc_stats.minor_gc_count, sgen_get_nursery_start (), nursery_next, (int)(nursery_next - sgen_get_nursery_start ()));
2226 max_garbage_amount = nursery_next - sgen_get_nursery_start ();
2227 g_assert (nursery_section->size >= max_garbage_amount);
2229 /* world must be stopped already */
2231 time_minor_pre_collection_fragment_clear += TV_ELAPSED (atv, btv);
2233 if (xdomain_checks) {
2234 sgen_clear_nursery_fragments ();
2235 sgen_check_for_xdomain_refs ();
2238 nursery_section->next_data = nursery_next;
2240 major_collector.start_nursery_collection ();
2242 sgen_memgov_minor_collection_start ();
2246 gc_stats.minor_gc_count ++;
2248 MONO_GC_CHECKPOINT_1 (GENERATION_NURSERY);
2250 sgen_process_fin_stage_entries ();
2251 sgen_process_dislink_stage_entries ();
2253 MONO_GC_CHECKPOINT_2 (GENERATION_NURSERY);
2255 /* pin from pinned handles */
2256 sgen_init_pinning ();
2257 mono_profiler_gc_event (MONO_GC_EVENT_MARK_START, 0);
2258 pin_from_roots (sgen_get_nursery_start (), nursery_next, WORKERS_DISTRIBUTE_GRAY_QUEUE);
2259 /* pin cemented objects */
2260 sgen_pin_cemented_objects ();
2261 /* identify pinned objects */
2262 sgen_optimize_pin_queue ();
2263 sgen_pinning_setup_section (nursery_section);
2264 ctx.scan_func = NULL;
2265 ctx.copy_func = NULL;
2266 ctx.queue = WORKERS_DISTRIBUTE_GRAY_QUEUE;
2267 pin_objects_in_nursery (ctx);
2268 sgen_pinning_trim_queue_to_section (nursery_section);
2271 time_minor_pinning += TV_ELAPSED (btv, atv);
2272 SGEN_LOG (2, "Finding pinned pointers: %zd in %d usecs", sgen_get_pinned_count (), TV_ELAPSED (btv, atv));
2273 SGEN_LOG (4, "Start scan with %zd pinned objects", sgen_get_pinned_count ());
2275 MONO_GC_CHECKPOINT_3 (GENERATION_NURSERY);
2277 if (whole_heap_check_before_collection) {
2278 sgen_clear_nursery_fragments ();
2279 sgen_check_whole_heap (finish_up_concurrent_mark);
2281 if (consistency_check_at_minor_collection)
2282 sgen_check_consistency ();
2284 sgen_workers_start_all_workers ();
2285 sgen_workers_start_marking ();
2287 frssjd = sgen_alloc_internal_dynamic (sizeof (FinishRememberedSetScanJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2288 frssjd->heap_start = sgen_get_nursery_start ();
2289 frssjd->heap_end = nursery_next;
2290 sgen_workers_enqueue_job (job_finish_remembered_set_scan, frssjd);
2292 /* we don't have complete write barrier yet, so we scan all the old generation sections */
2294 time_minor_scan_remsets += TV_ELAPSED (atv, btv);
2295 SGEN_LOG (2, "Old generation scan: %d usecs", TV_ELAPSED (atv, btv));
2297 MONO_GC_CHECKPOINT_4 (GENERATION_NURSERY);
2299 /* FIXME: why is this here? */
2300 ctx.scan_func = current_object_ops.scan_object;
2301 ctx.copy_func = NULL;
2302 ctx.queue = &gray_queue;
2303 sgen_drain_gray_stack (-1, ctx);
2305 if (mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS)
2306 report_registered_roots ();
2307 if (mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS)
2308 report_finalizer_roots ();
2310 time_minor_scan_pinned += TV_ELAPSED (btv, atv);
2312 MONO_GC_CHECKPOINT_5 (GENERATION_NURSERY);
2314 /* registered roots, this includes static fields */
2315 scrrjd_normal = sgen_alloc_internal_dynamic (sizeof (ScanFromRegisteredRootsJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2316 scrrjd_normal->copy_or_mark_func = current_object_ops.copy_or_mark_object;
2317 scrrjd_normal->scan_func = current_object_ops.scan_object;
2318 scrrjd_normal->heap_start = sgen_get_nursery_start ();
2319 scrrjd_normal->heap_end = nursery_next;
2320 scrrjd_normal->root_type = ROOT_TYPE_NORMAL;
2321 sgen_workers_enqueue_job (job_scan_from_registered_roots, scrrjd_normal);
2323 scrrjd_wbarrier = sgen_alloc_internal_dynamic (sizeof (ScanFromRegisteredRootsJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2324 scrrjd_wbarrier->copy_or_mark_func = current_object_ops.copy_or_mark_object;
2325 scrrjd_wbarrier->scan_func = current_object_ops.scan_object;
2326 scrrjd_wbarrier->heap_start = sgen_get_nursery_start ();
2327 scrrjd_wbarrier->heap_end = nursery_next;
2328 scrrjd_wbarrier->root_type = ROOT_TYPE_WBARRIER;
2329 sgen_workers_enqueue_job (job_scan_from_registered_roots, scrrjd_wbarrier);
2332 time_minor_scan_registered_roots += TV_ELAPSED (atv, btv);
2334 MONO_GC_CHECKPOINT_6 (GENERATION_NURSERY);
2337 stdjd = sgen_alloc_internal_dynamic (sizeof (ScanThreadDataJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2338 stdjd->heap_start = sgen_get_nursery_start ();
2339 stdjd->heap_end = nursery_next;
2340 sgen_workers_enqueue_job (job_scan_thread_data, stdjd);
2343 time_minor_scan_thread_data += TV_ELAPSED (btv, atv);
2346 MONO_GC_CHECKPOINT_7 (GENERATION_NURSERY);
2348 g_assert (!sgen_collection_is_concurrent ());
2350 /* Scan the list of objects ready for finalization. If */
2351 sgen_workers_enqueue_job (job_scan_finalizer_entries, fin_ready_list);
2352 sgen_workers_enqueue_job (job_scan_finalizer_entries, critical_fin_list);
2354 MONO_GC_CHECKPOINT_8 (GENERATION_NURSERY);
2356 finish_gray_stack (GENERATION_NURSERY, &gray_queue);
2358 time_minor_finish_gray_stack += TV_ELAPSED (btv, atv);
2359 mono_profiler_gc_event (MONO_GC_EVENT_MARK_END, 0);
2361 MONO_GC_CHECKPOINT_9 (GENERATION_NURSERY);
2364 * The (single-threaded) finalization code might have done
2365 * some copying/marking so we can only reset the GC thread's
2366 * worker data here instead of earlier when we joined the
2369 sgen_workers_reset_data ();
2371 if (objects_pinned) {
2372 sgen_optimize_pin_queue ();
2373 sgen_pinning_setup_section (nursery_section);
2376 /* walk the pin_queue, build up the fragment list of free memory, unmark
2377 * pinned objects as we go, memzero() the empty fragments so they are ready for the
2380 mono_profiler_gc_event (MONO_GC_EVENT_RECLAIM_START, 0);
2381 fragment_total = sgen_build_nursery_fragments (nursery_section,
2382 nursery_section->pin_queue_start, nursery_section->pin_queue_num_entries,
2384 if (!fragment_total)
2387 /* Clear TLABs for all threads */
2388 sgen_clear_tlabs ();
2390 mono_profiler_gc_event (MONO_GC_EVENT_RECLAIM_END, 0);
2392 time_minor_fragment_creation += TV_ELAPSED (atv, btv);
2393 SGEN_LOG (2, "Fragment creation: %d usecs, %lu bytes available", TV_ELAPSED (atv, btv), (unsigned long)fragment_total);
2395 if (consistency_check_at_minor_collection)
2396 sgen_check_major_refs ();
2398 major_collector.finish_nursery_collection ();
2400 TV_GETTIME (last_minor_collection_end_tv);
2401 gc_stats.minor_gc_time += TV_ELAPSED (last_minor_collection_start_tv, last_minor_collection_end_tv);
2404 dump_heap ("minor", gc_stats.minor_gc_count - 1, NULL);
2406 /* prepare the pin queue for the next collection */
2407 sgen_finish_pinning ();
2408 if (fin_ready_list || critical_fin_list) {
2409 SGEN_LOG (4, "Finalizer-thread wakeup: ready %d", num_ready_finalizers);
2410 mono_gc_finalize_notify ();
2412 sgen_pin_stats_reset ();
2413 /* clear cemented hash */
2414 sgen_cement_clear_below_threshold ();
2416 g_assert (sgen_gray_object_queue_is_empty (&gray_queue));
2418 remset.finish_minor_collection ();
2420 check_scan_starts ();
2422 binary_protocol_flush_buffers (FALSE);
2424 sgen_memgov_minor_collection_end ();
2426 /*objects are late pinned because of lack of memory, so a major is a good call*/
2427 needs_major = objects_pinned > 0;
2428 current_collection_generation = -1;
2431 MONO_GC_END (GENERATION_NURSERY);
2432 binary_protocol_collection_end (gc_stats.minor_gc_count - 1, GENERATION_NURSERY);
2434 if (check_nursery_objects_pinned && !sgen_minor_collector.is_split)
2435 sgen_check_nursery_objects_pinned (unpin_queue != NULL);
2441 scan_nursery_objects_callback (char *obj, size_t size, ScanCopyContext *ctx)
2443 ctx->scan_func (obj, sgen_obj_get_descriptor (obj), ctx->queue);
2447 scan_nursery_objects (ScanCopyContext ctx)
2449 sgen_scan_area_with_callback (nursery_section->data, nursery_section->end_data,
2450 (IterateObjectCallbackFunc)scan_nursery_objects_callback, (void*)&ctx, FALSE);
2454 major_copy_or_mark_from_roots (size_t *old_next_pin_slot, gboolean finish_up_concurrent_mark, gboolean scan_mod_union)
2459 /* FIXME: only use these values for the precise scan
2460 * note that to_space pointers should be excluded anyway...
2462 char *heap_start = NULL;
2463 char *heap_end = (char*)-1;
2464 gboolean profile_roots = mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS;
2465 GCRootReport root_report = { 0 };
2466 ScanFromRegisteredRootsJobData *scrrjd_normal, *scrrjd_wbarrier;
2467 ScanThreadDataJobData *stdjd;
2468 ScanCopyContext ctx;
2470 if (concurrent_collection_in_progress) {
2471 /*This cleans up unused fragments */
2472 sgen_nursery_allocator_prepare_for_pinning ();
2474 if (do_concurrent_checks)
2475 check_nursery_is_clean ();
2477 /* The concurrent collector doesn't touch the nursery. */
2478 sgen_nursery_alloc_prepare_for_major ();
2485 /* Pinning depends on this */
2486 sgen_clear_nursery_fragments ();
2488 if (whole_heap_check_before_collection)
2489 sgen_check_whole_heap (finish_up_concurrent_mark);
2492 time_major_pre_collection_fragment_clear += TV_ELAPSED (atv, btv);
2494 if (!sgen_collection_is_concurrent ())
2495 nursery_section->next_data = sgen_get_nursery_end ();
2496 /* we should also coalesce scanning from sections close to each other
2497 * and deal with pointers outside of the sections later.
2501 *major_collector.have_swept = FALSE;
2503 if (xdomain_checks) {
2504 sgen_clear_nursery_fragments ();
2505 sgen_check_for_xdomain_refs ();
2508 if (!concurrent_collection_in_progress) {
2509 /* Remsets are not useful for a major collection */
2510 remset.prepare_for_major_collection ();
2513 sgen_process_fin_stage_entries ();
2514 sgen_process_dislink_stage_entries ();
2517 sgen_init_pinning ();
2518 SGEN_LOG (6, "Collecting pinned addresses");
2519 pin_from_roots ((void*)lowest_heap_address, (void*)highest_heap_address, WORKERS_DISTRIBUTE_GRAY_QUEUE);
2521 if (!concurrent_collection_in_progress || finish_up_concurrent_mark) {
2522 if (major_collector.is_concurrent) {
2524 * The concurrent major collector cannot evict
2525 * yet, so we need to pin cemented objects to
2526 * not break some asserts.
2528 * FIXME: We could evict now!
2530 sgen_pin_cemented_objects ();
2533 if (!concurrent_collection_in_progress)
2534 sgen_cement_reset ();
2537 sgen_optimize_pin_queue ();
2540 * The concurrent collector doesn't move objects, neither on
2541 * the major heap nor in the nursery, so we can mark even
2542 * before pinning has finished. For the non-concurrent
2543 * collector we start the workers after pinning.
2545 if (concurrent_collection_in_progress) {
2546 sgen_workers_start_all_workers ();
2547 sgen_workers_start_marking ();
2551 * pin_queue now contains all candidate pointers, sorted and
2552 * uniqued. We must do two passes now to figure out which
2553 * objects are pinned.
2555 * The first is to find within the pin_queue the area for each
2556 * section. This requires that the pin_queue be sorted. We
2557 * also process the LOS objects and pinned chunks here.
2559 * The second, destructive, pass is to reduce the section
2560 * areas to pointers to the actually pinned objects.
2562 SGEN_LOG (6, "Pinning from sections");
2563 /* first pass for the sections */
2564 sgen_find_section_pin_queue_start_end (nursery_section);
2565 major_collector.find_pin_queue_start_ends (WORKERS_DISTRIBUTE_GRAY_QUEUE);
2566 /* identify possible pointers to the insize of large objects */
2567 SGEN_LOG (6, "Pinning from large objects");
2568 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
2570 if (sgen_find_optimized_pin_queue_area (bigobj->data, (char*)bigobj->data + sgen_los_object_size (bigobj), &dummy)) {
2571 binary_protocol_pin (bigobj->data, (gpointer)LOAD_VTABLE (bigobj->data), safe_object_get_size (((MonoObject*)(bigobj->data))));
2573 #ifdef ENABLE_DTRACE
2574 if (G_UNLIKELY (MONO_GC_OBJ_PINNED_ENABLED ())) {
2575 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (bigobj->data);
2576 MONO_GC_OBJ_PINNED ((mword)bigobj->data, sgen_safe_object_get_size ((MonoObject*)bigobj->data), vt->klass->name_space, vt->klass->name, GENERATION_OLD);
2580 if (sgen_los_object_is_pinned (bigobj->data)) {
2581 g_assert (finish_up_concurrent_mark);
2584 sgen_los_pin_object (bigobj->data);
2585 if (SGEN_OBJECT_HAS_REFERENCES (bigobj->data))
2586 GRAY_OBJECT_ENQUEUE (WORKERS_DISTRIBUTE_GRAY_QUEUE, bigobj->data, sgen_obj_get_descriptor (bigobj->data));
2587 if (G_UNLIKELY (do_pin_stats))
2588 sgen_pin_stats_register_object ((char*) bigobj->data, safe_object_get_size ((MonoObject*) bigobj->data));
2589 SGEN_LOG (6, "Marked large object %p (%s) size: %lu from roots", bigobj->data, safe_name (bigobj->data), (unsigned long)sgen_los_object_size (bigobj));
2592 add_profile_gc_root (&root_report, bigobj->data, MONO_PROFILE_GC_ROOT_PINNING | MONO_PROFILE_GC_ROOT_MISC, 0);
2596 notify_gc_roots (&root_report);
2597 /* second pass for the sections */
2598 ctx.scan_func = concurrent_collection_in_progress ? current_object_ops.scan_object : NULL;
2599 ctx.copy_func = NULL;
2600 ctx.queue = WORKERS_DISTRIBUTE_GRAY_QUEUE;
2603 * Concurrent mark never follows references into the nursery.
2604 * In the start and finish pauses we must scan live nursery
2605 * objects, though. We could simply scan all nursery objects,
2606 * but that would be conservative. The easiest way is to do a
2607 * nursery collection, which copies all live nursery objects
2608 * (except pinned ones, with the simple nursery) to the major
2609 * heap. Scanning the mod union table later will then scan
2610 * those promoted objects, provided they're reachable. Pinned
2611 * objects in the nursery - which we can trivially find in the
2612 * pinning queue - are treated as roots in the mark pauses.
2614 * The split nursery complicates the latter part because
2615 * non-pinned objects can survive in the nursery. That's why
2616 * we need to do a full front-to-back scan of the nursery,
2617 * marking all objects.
2619 * Non-concurrent mark evacuates from the nursery, so it's
2620 * sufficient to just scan pinned nursery objects.
2622 if (concurrent_collection_in_progress && sgen_minor_collector.is_split) {
2623 scan_nursery_objects (ctx);
2625 pin_objects_in_nursery (ctx);
2626 if (check_nursery_objects_pinned && !sgen_minor_collector.is_split)
2627 sgen_check_nursery_objects_pinned (!concurrent_collection_in_progress || finish_up_concurrent_mark);
2630 major_collector.pin_objects (WORKERS_DISTRIBUTE_GRAY_QUEUE);
2631 if (old_next_pin_slot)
2632 *old_next_pin_slot = sgen_get_pinned_count ();
2635 time_major_pinning += TV_ELAPSED (atv, btv);
2636 SGEN_LOG (2, "Finding pinned pointers: %zd in %d usecs", sgen_get_pinned_count (), TV_ELAPSED (atv, btv));
2637 SGEN_LOG (4, "Start scan with %zd pinned objects", sgen_get_pinned_count ());
2639 major_collector.init_to_space ();
2641 #ifdef SGEN_DEBUG_INTERNAL_ALLOC
2642 main_gc_thread = mono_native_thread_self ();
2645 if (mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS)
2646 report_registered_roots ();
2648 time_major_scan_pinned += TV_ELAPSED (btv, atv);
2650 /* registered roots, this includes static fields */
2651 scrrjd_normal = sgen_alloc_internal_dynamic (sizeof (ScanFromRegisteredRootsJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2652 scrrjd_normal->copy_or_mark_func = current_object_ops.copy_or_mark_object;
2653 scrrjd_normal->scan_func = current_object_ops.scan_object;
2654 scrrjd_normal->heap_start = heap_start;
2655 scrrjd_normal->heap_end = heap_end;
2656 scrrjd_normal->root_type = ROOT_TYPE_NORMAL;
2657 sgen_workers_enqueue_job (job_scan_from_registered_roots, scrrjd_normal);
2659 scrrjd_wbarrier = sgen_alloc_internal_dynamic (sizeof (ScanFromRegisteredRootsJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2660 scrrjd_wbarrier->copy_or_mark_func = current_object_ops.copy_or_mark_object;
2661 scrrjd_wbarrier->scan_func = current_object_ops.scan_object;
2662 scrrjd_wbarrier->heap_start = heap_start;
2663 scrrjd_wbarrier->heap_end = heap_end;
2664 scrrjd_wbarrier->root_type = ROOT_TYPE_WBARRIER;
2665 sgen_workers_enqueue_job (job_scan_from_registered_roots, scrrjd_wbarrier);
2668 time_major_scan_registered_roots += TV_ELAPSED (atv, btv);
2671 stdjd = sgen_alloc_internal_dynamic (sizeof (ScanThreadDataJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2672 stdjd->heap_start = heap_start;
2673 stdjd->heap_end = heap_end;
2674 sgen_workers_enqueue_job (job_scan_thread_data, stdjd);
2677 time_major_scan_thread_data += TV_ELAPSED (btv, atv);
2680 time_major_scan_alloc_pinned += TV_ELAPSED (atv, btv);
2682 if (mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS)
2683 report_finalizer_roots ();
2685 /* scan the list of objects ready for finalization */
2686 sgen_workers_enqueue_job (job_scan_finalizer_entries, fin_ready_list);
2687 sgen_workers_enqueue_job (job_scan_finalizer_entries, critical_fin_list);
2689 if (scan_mod_union) {
2690 g_assert (finish_up_concurrent_mark);
2692 /* Mod union card table */
2693 sgen_workers_enqueue_job (job_scan_major_mod_union_cardtable, NULL);
2694 sgen_workers_enqueue_job (job_scan_los_mod_union_cardtable, NULL);
2698 time_major_scan_finalized += TV_ELAPSED (btv, atv);
2699 SGEN_LOG (2, "Root scan: %d usecs", TV_ELAPSED (btv, atv));
2702 time_major_scan_big_objects += TV_ELAPSED (atv, btv);
2704 if (concurrent_collection_in_progress) {
2705 /* prepare the pin queue for the next collection */
2706 sgen_finish_pinning ();
2708 sgen_pin_stats_reset ();
2710 if (do_concurrent_checks)
2711 check_nursery_is_clean ();
2716 major_start_collection (gboolean concurrent, size_t *old_next_pin_slot)
2718 MONO_GC_BEGIN (GENERATION_OLD);
2719 binary_protocol_collection_begin (gc_stats.major_gc_count, GENERATION_OLD);
2721 current_collection_generation = GENERATION_OLD;
2722 #ifndef DISABLE_PERFCOUNTERS
2723 mono_perfcounters->gc_collections1++;
2726 g_assert (sgen_section_gray_queue_is_empty (sgen_workers_get_distribute_section_gray_queue ()));
2729 g_assert (major_collector.is_concurrent);
2730 concurrent_collection_in_progress = TRUE;
2732 sgen_cement_concurrent_start ();
2734 current_object_ops = major_collector.major_concurrent_ops;
2736 current_object_ops = major_collector.major_ops;
2739 reset_pinned_from_failed_allocation ();
2741 sgen_memgov_major_collection_start ();
2743 //count_ref_nonref_objs ();
2744 //consistency_check ();
2746 check_scan_starts ();
2749 SGEN_LOG (1, "Start major collection %d", gc_stats.major_gc_count);
2750 gc_stats.major_gc_count ++;
2752 if (major_collector.start_major_collection)
2753 major_collector.start_major_collection ();
2755 major_copy_or_mark_from_roots (old_next_pin_slot, FALSE, FALSE);
2759 wait_for_workers_to_finish (void)
2761 while (!sgen_workers_all_done ())
2768 if (concurrent_collection_in_progress) {
2769 gray_queue_redirect (&gray_queue);
2770 sgen_workers_join ();
2773 g_assert (sgen_gray_object_queue_is_empty (&gray_queue));
2775 #ifdef SGEN_DEBUG_INTERNAL_ALLOC
2776 main_gc_thread = NULL;
2781 major_finish_collection (const char *reason, size_t old_next_pin_slot, gboolean scan_mod_union)
2783 LOSObject *bigobj, *prevbo;
2789 if (concurrent_collection_in_progress)
2792 if (concurrent_collection_in_progress) {
2793 current_object_ops = major_collector.major_concurrent_ops;
2795 major_copy_or_mark_from_roots (NULL, TRUE, scan_mod_union);
2798 g_assert (sgen_gray_object_queue_is_empty (&gray_queue));
2800 if (do_concurrent_checks)
2801 check_nursery_is_clean ();
2803 current_object_ops = major_collector.major_ops;
2807 * The workers have stopped so we need to finish gray queue
2808 * work that might result from finalization in the main GC
2809 * thread. Redirection must therefore be turned off.
2811 sgen_gray_object_queue_disable_alloc_prepare (&gray_queue);
2812 g_assert (sgen_section_gray_queue_is_empty (sgen_workers_get_distribute_section_gray_queue ()));
2814 /* all the objects in the heap */
2815 finish_gray_stack (GENERATION_OLD, &gray_queue);
2817 time_major_finish_gray_stack += TV_ELAPSED (btv, atv);
2820 * The (single-threaded) finalization code might have done
2821 * some copying/marking so we can only reset the GC thread's
2822 * worker data here instead of earlier when we joined the
2825 sgen_workers_reset_data ();
2827 if (objects_pinned) {
2828 g_assert (!concurrent_collection_in_progress);
2830 /*This is slow, but we just OOM'd*/
2831 sgen_pin_queue_clear_discarded_entries (nursery_section, old_next_pin_slot);
2832 sgen_optimize_pin_queue ();
2833 sgen_find_section_pin_queue_start_end (nursery_section);
2837 reset_heap_boundaries ();
2838 sgen_update_heap_boundaries ((mword)sgen_get_nursery_start (), (mword)sgen_get_nursery_end ());
2840 if (check_mark_bits_after_major_collection)
2841 sgen_check_major_heap_marked ();
2843 MONO_GC_SWEEP_BEGIN (GENERATION_OLD, !major_collector.sweeps_lazily);
2845 /* sweep the big objects list */
2847 for (bigobj = los_object_list; bigobj;) {
2848 g_assert (!object_is_pinned (bigobj->data));
2849 if (sgen_los_object_is_pinned (bigobj->data)) {
2850 sgen_los_unpin_object (bigobj->data);
2851 sgen_update_heap_boundaries ((mword)bigobj->data, (mword)bigobj->data + sgen_los_object_size (bigobj));
2854 /* not referenced anywhere, so we can free it */
2856 prevbo->next = bigobj->next;
2858 los_object_list = bigobj->next;
2860 bigobj = bigobj->next;
2861 sgen_los_free_object (to_free);
2865 bigobj = bigobj->next;
2869 time_major_free_bigobjs += TV_ELAPSED (atv, btv);
2874 time_major_los_sweep += TV_ELAPSED (btv, atv);
2876 major_collector.sweep ();
2878 MONO_GC_SWEEP_END (GENERATION_OLD, !major_collector.sweeps_lazily);
2881 time_major_sweep += TV_ELAPSED (atv, btv);
2883 if (!concurrent_collection_in_progress) {
2884 /* walk the pin_queue, build up the fragment list of free memory, unmark
2885 * pinned objects as we go, memzero() the empty fragments so they are ready for the
2888 if (!sgen_build_nursery_fragments (nursery_section, nursery_section->pin_queue_start, nursery_section->pin_queue_num_entries, NULL))
2891 /* prepare the pin queue for the next collection */
2892 sgen_finish_pinning ();
2894 /* Clear TLABs for all threads */
2895 sgen_clear_tlabs ();
2897 sgen_pin_stats_reset ();
2900 if (concurrent_collection_in_progress)
2901 sgen_cement_concurrent_finish ();
2902 sgen_cement_clear_below_threshold ();
2905 time_major_fragment_creation += TV_ELAPSED (btv, atv);
2908 dump_heap ("major", gc_stats.major_gc_count - 1, reason);
2910 if (fin_ready_list || critical_fin_list) {
2911 SGEN_LOG (4, "Finalizer-thread wakeup: ready %d", num_ready_finalizers);
2912 mono_gc_finalize_notify ();
2915 g_assert (sgen_gray_object_queue_is_empty (&gray_queue));
2917 sgen_memgov_major_collection_end ();
2918 current_collection_generation = -1;
2920 major_collector.finish_major_collection ();
2922 g_assert (sgen_section_gray_queue_is_empty (sgen_workers_get_distribute_section_gray_queue ()));
2924 if (concurrent_collection_in_progress)
2925 concurrent_collection_in_progress = FALSE;
2927 check_scan_starts ();
2929 binary_protocol_flush_buffers (FALSE);
2931 //consistency_check ();
2933 MONO_GC_END (GENERATION_OLD);
2934 binary_protocol_collection_end (gc_stats.major_gc_count - 1, GENERATION_OLD);
2938 major_do_collection (const char *reason)
2940 TV_DECLARE (time_start);
2941 TV_DECLARE (time_end);
2942 size_t old_next_pin_slot;
2944 if (disable_major_collections)
2947 if (major_collector.get_and_reset_num_major_objects_marked) {
2948 long long num_marked = major_collector.get_and_reset_num_major_objects_marked ();
2949 g_assert (!num_marked);
2952 /* world must be stopped already */
2953 TV_GETTIME (time_start);
2955 major_start_collection (FALSE, &old_next_pin_slot);
2956 major_finish_collection (reason, old_next_pin_slot, FALSE);
2958 TV_GETTIME (time_end);
2959 gc_stats.major_gc_time += TV_ELAPSED (time_start, time_end);
2961 /* FIXME: also report this to the user, preferably in gc-end. */
2962 if (major_collector.get_and_reset_num_major_objects_marked)
2963 major_collector.get_and_reset_num_major_objects_marked ();
2965 return bytes_pinned_from_failed_allocation > 0;
2969 major_start_concurrent_collection (const char *reason)
2971 TV_DECLARE (time_start);
2972 TV_DECLARE (time_end);
2973 long long num_objects_marked;
2975 if (disable_major_collections)
2978 TV_GETTIME (time_start);
2979 SGEN_TV_GETTIME (time_major_conc_collection_start);
2981 num_objects_marked = major_collector.get_and_reset_num_major_objects_marked ();
2982 g_assert (num_objects_marked == 0);
2984 MONO_GC_CONCURRENT_START_BEGIN (GENERATION_OLD);
2985 binary_protocol_concurrent_start ();
2987 // FIXME: store reason and pass it when finishing
2988 major_start_collection (TRUE, NULL);
2990 gray_queue_redirect (&gray_queue);
2991 sgen_workers_wait_for_jobs ();
2993 num_objects_marked = major_collector.get_and_reset_num_major_objects_marked ();
2994 MONO_GC_CONCURRENT_START_END (GENERATION_OLD, num_objects_marked);
2996 TV_GETTIME (time_end);
2997 gc_stats.major_gc_time += TV_ELAPSED (time_start, time_end);
2999 current_collection_generation = -1;
3003 major_update_or_finish_concurrent_collection (gboolean force_finish)
3005 TV_DECLARE (total_start);
3006 TV_DECLARE (total_end);
3007 SgenGrayQueue unpin_queue;
3008 memset (&unpin_queue, 0, sizeof (unpin_queue));
3010 TV_GETTIME (total_start);
3012 MONO_GC_CONCURRENT_UPDATE_FINISH_BEGIN (GENERATION_OLD, major_collector.get_and_reset_num_major_objects_marked ());
3013 binary_protocol_concurrent_update_finish ();
3015 g_assert (sgen_gray_object_queue_is_empty (&gray_queue));
3017 if (!force_finish && !sgen_workers_all_done ()) {
3018 major_collector.update_cardtable_mod_union ();
3019 sgen_los_update_cardtable_mod_union ();
3021 MONO_GC_CONCURRENT_UPDATE_END (GENERATION_OLD, major_collector.get_and_reset_num_major_objects_marked ());
3023 TV_GETTIME (total_end);
3024 gc_stats.major_gc_time += TV_ELAPSED (total_start, total_end);
3030 * The major collector can add global remsets which are processed in the finishing
3031 * nursery collection, below. That implies that the workers must have finished
3032 * marking before the nursery collection is allowed to run, otherwise we might miss
3035 wait_for_workers_to_finish ();
3037 SGEN_TV_GETTIME (time_major_conc_collection_end);
3038 gc_stats.major_gc_time_concurrent += SGEN_TV_ELAPSED (time_major_conc_collection_start, time_major_conc_collection_end);
3040 major_collector.update_cardtable_mod_union ();
3041 sgen_los_update_cardtable_mod_union ();
3043 collect_nursery (&unpin_queue, TRUE);
3045 if (mod_union_consistency_check)
3046 sgen_check_mod_union_consistency ();
3048 current_collection_generation = GENERATION_OLD;
3049 major_finish_collection ("finishing", -1, TRUE);
3051 if (whole_heap_check_before_collection)
3052 sgen_check_whole_heap (FALSE);
3054 unpin_objects_from_queue (&unpin_queue);
3055 sgen_gray_object_queue_deinit (&unpin_queue);
3057 MONO_GC_CONCURRENT_FINISH_END (GENERATION_OLD, major_collector.get_and_reset_num_major_objects_marked ());
3059 TV_GETTIME (total_end);
3060 gc_stats.major_gc_time += TV_ELAPSED (total_start, total_end) - TV_ELAPSED (last_minor_collection_start_tv, last_minor_collection_end_tv);
3062 current_collection_generation = -1;
3068 * Ensure an allocation request for @size will succeed by freeing enough memory.
3070 * LOCKING: The GC lock MUST be held.
3073 sgen_ensure_free_space (size_t size)
3075 int generation_to_collect = -1;
3076 const char *reason = NULL;
3079 if (size > SGEN_MAX_SMALL_OBJ_SIZE) {
3080 if (sgen_need_major_collection (size)) {
3081 reason = "LOS overflow";
3082 generation_to_collect = GENERATION_OLD;
3085 if (degraded_mode) {
3086 if (sgen_need_major_collection (size)) {
3087 reason = "Degraded mode overflow";
3088 generation_to_collect = GENERATION_OLD;
3090 } else if (sgen_need_major_collection (size)) {
3091 reason = "Minor allowance";
3092 generation_to_collect = GENERATION_OLD;
3094 generation_to_collect = GENERATION_NURSERY;
3095 reason = "Nursery full";
3099 if (generation_to_collect == -1) {
3100 if (concurrent_collection_in_progress && sgen_workers_all_done ()) {
3101 generation_to_collect = GENERATION_OLD;
3102 reason = "Finish concurrent collection";
3106 if (generation_to_collect == -1)
3108 sgen_perform_collection (size, generation_to_collect, reason, FALSE);
3112 * LOCKING: Assumes the GC lock is held.
3115 sgen_perform_collection (size_t requested_size, int generation_to_collect, const char *reason, gboolean wait_to_finish)
3117 TV_DECLARE (gc_end);
3118 GGTimingInfo infos [2];
3119 int overflow_generation_to_collect = -1;
3120 int oldest_generation_collected = generation_to_collect;
3121 const char *overflow_reason = NULL;
3123 MONO_GC_REQUESTED (generation_to_collect, requested_size, wait_to_finish ? 1 : 0);
3125 binary_protocol_collection_force (generation_to_collect);
3127 g_assert (generation_to_collect == GENERATION_NURSERY || generation_to_collect == GENERATION_OLD);
3129 memset (infos, 0, sizeof (infos));
3130 mono_profiler_gc_event (MONO_GC_EVENT_START, generation_to_collect);
3132 infos [0].generation = generation_to_collect;
3133 infos [0].reason = reason;
3134 infos [0].is_overflow = FALSE;
3135 TV_GETTIME (infos [0].total_time);
3136 infos [1].generation = -1;
3138 sgen_stop_world (generation_to_collect);
3140 if (concurrent_collection_in_progress) {
3141 if (major_update_or_finish_concurrent_collection (wait_to_finish && generation_to_collect == GENERATION_OLD)) {
3142 oldest_generation_collected = GENERATION_OLD;
3145 if (generation_to_collect == GENERATION_OLD)
3148 if (generation_to_collect == GENERATION_OLD &&
3149 allow_synchronous_major &&
3150 major_collector.want_synchronous_collection &&
3151 *major_collector.want_synchronous_collection) {
3152 wait_to_finish = TRUE;
3156 //FIXME extract overflow reason
3157 if (generation_to_collect == GENERATION_NURSERY) {
3158 if (collect_nursery (NULL, FALSE)) {
3159 overflow_generation_to_collect = GENERATION_OLD;
3160 overflow_reason = "Minor overflow";
3163 if (major_collector.is_concurrent) {
3164 g_assert (!concurrent_collection_in_progress);
3165 if (!wait_to_finish)
3166 collect_nursery (NULL, FALSE);
3169 if (major_collector.is_concurrent && !wait_to_finish) {
3170 major_start_concurrent_collection (reason);
3171 // FIXME: set infos[0] properly
3174 if (major_do_collection (reason)) {
3175 overflow_generation_to_collect = GENERATION_NURSERY;
3176 overflow_reason = "Excessive pinning";
3181 TV_GETTIME (gc_end);
3182 infos [0].total_time = SGEN_TV_ELAPSED (infos [0].total_time, gc_end);
3185 if (!major_collector.is_concurrent && overflow_generation_to_collect != -1) {
3186 mono_profiler_gc_event (MONO_GC_EVENT_START, overflow_generation_to_collect);
3187 infos [1].generation = overflow_generation_to_collect;
3188 infos [1].reason = overflow_reason;
3189 infos [1].is_overflow = TRUE;
3190 infos [1].total_time = gc_end;
3192 if (overflow_generation_to_collect == GENERATION_NURSERY)
3193 collect_nursery (NULL, FALSE);
3195 major_do_collection (overflow_reason);
3197 TV_GETTIME (gc_end);
3198 infos [1].total_time = SGEN_TV_ELAPSED (infos [1].total_time, gc_end);
3200 /* keep events symmetric */
3201 mono_profiler_gc_event (MONO_GC_EVENT_END, overflow_generation_to_collect);
3203 oldest_generation_collected = MAX (oldest_generation_collected, overflow_generation_to_collect);
3206 SGEN_LOG (2, "Heap size: %lu, LOS size: %lu", (unsigned long)mono_gc_get_heap_size (), (unsigned long)los_memory_usage);
3208 /* this also sets the proper pointers for the next allocation */
3209 if (generation_to_collect == GENERATION_NURSERY && !sgen_can_alloc_size (requested_size)) {
3210 /* TypeBuilder and MonoMethod are killing mcs with fragmentation */
3211 SGEN_LOG (1, "nursery collection didn't find enough room for %zd alloc (%zd pinned)", requested_size, sgen_get_pinned_count ());
3212 sgen_dump_pin_queue ();
3217 g_assert (sgen_gray_object_queue_is_empty (&gray_queue));
3219 sgen_restart_world (oldest_generation_collected, infos);
3221 mono_profiler_gc_event (MONO_GC_EVENT_END, generation_to_collect);
3225 * ######################################################################
3226 * ######## Memory allocation from the OS
3227 * ######################################################################
3228 * This section of code deals with getting memory from the OS and
3229 * allocating memory for GC-internal data structures.
3230 * Internal memory can be handled with a freelist for small objects.
3236 G_GNUC_UNUSED static void
3237 report_internal_mem_usage (void)
3239 printf ("Internal memory usage:\n");
3240 sgen_report_internal_mem_usage ();
3241 printf ("Pinned memory usage:\n");
3242 major_collector.report_pinned_memory_usage ();
3246 * ######################################################################
3247 * ######## Finalization support
3248 * ######################################################################
3251 static inline gboolean
3252 sgen_major_is_object_alive (void *object)
3256 /* Oldgen objects can be pinned and forwarded too */
3257 if (SGEN_OBJECT_IS_PINNED (object) || SGEN_OBJECT_IS_FORWARDED (object))
3261 * FIXME: major_collector.is_object_live() also calculates the
3262 * size. Avoid the double calculation.
3264 objsize = SGEN_ALIGN_UP (sgen_safe_object_get_size ((MonoObject*)object));
3265 if (objsize > SGEN_MAX_SMALL_OBJ_SIZE)
3266 return sgen_los_object_is_pinned (object);
3268 return major_collector.is_object_live (object);
3272 * If the object has been forwarded it means it's still referenced from a root.
3273 * If it is pinned it's still alive as well.
3274 * A LOS object is only alive if we have pinned it.
3275 * Return TRUE if @obj is ready to be finalized.
3277 static inline gboolean
3278 sgen_is_object_alive (void *object)
3280 if (ptr_in_nursery (object))
3281 return sgen_nursery_is_object_alive (object);
3283 return sgen_major_is_object_alive (object);
3287 * This function returns true if @object is either alive or it belongs to the old gen
3288 * and we're currently doing a minor collection.
3291 sgen_is_object_alive_for_current_gen (char *object)
3293 if (ptr_in_nursery (object))
3294 return sgen_nursery_is_object_alive (object);
3296 if (current_collection_generation == GENERATION_NURSERY)
3299 return sgen_major_is_object_alive (object);
3303 * This function returns true if @object is either alive and belongs to the
3304 * current collection - major collections are full heap, so old gen objects
3305 * are never alive during a minor collection.
3308 sgen_is_object_alive_and_on_current_collection (char *object)
3310 if (ptr_in_nursery (object))
3311 return sgen_nursery_is_object_alive (object);
3313 if (current_collection_generation == GENERATION_NURSERY)
3316 return sgen_major_is_object_alive (object);
3321 sgen_gc_is_object_ready_for_finalization (void *object)
3323 return !sgen_is_object_alive (object);
3327 has_critical_finalizer (MonoObject *obj)
3331 if (!mono_defaults.critical_finalizer_object)
3334 class = ((MonoVTable*)LOAD_VTABLE (obj))->klass;
3336 return mono_class_has_parent_fast (class, mono_defaults.critical_finalizer_object);
3340 is_finalization_aware (MonoObject *obj)
3342 MonoVTable *vt = ((MonoVTable*)LOAD_VTABLE (obj));
3343 return (vt->gc_bits & SGEN_GC_BIT_FINALIZER_AWARE) == SGEN_GC_BIT_FINALIZER_AWARE;
3347 sgen_queue_finalization_entry (MonoObject *obj)
3349 FinalizeReadyEntry *entry = sgen_alloc_internal (INTERNAL_MEM_FINALIZE_READY_ENTRY);
3350 gboolean critical = has_critical_finalizer (obj);
3351 entry->object = obj;
3353 entry->next = critical_fin_list;
3354 critical_fin_list = entry;
3356 entry->next = fin_ready_list;
3357 fin_ready_list = entry;
3360 if (fin_callbacks.object_queued_for_finalization && is_finalization_aware (obj))
3361 fin_callbacks.object_queued_for_finalization (obj);
3363 #ifdef ENABLE_DTRACE
3364 if (G_UNLIKELY (MONO_GC_FINALIZE_ENQUEUE_ENABLED ())) {
3365 int gen = sgen_ptr_in_nursery (obj) ? GENERATION_NURSERY : GENERATION_OLD;
3366 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (obj);
3367 MONO_GC_FINALIZE_ENQUEUE ((mword)obj, sgen_safe_object_get_size (obj),
3368 vt->klass->name_space, vt->klass->name, gen, critical);
3374 sgen_object_is_live (void *obj)
3376 return sgen_is_object_alive_and_on_current_collection (obj);
3379 /* LOCKING: requires that the GC lock is held */
3381 null_ephemerons_for_domain (MonoDomain *domain)
3383 EphemeronLinkNode *current = ephemeron_list, *prev = NULL;
3386 MonoObject *object = (MonoObject*)current->array;
3388 if (object && !object->vtable) {
3389 EphemeronLinkNode *tmp = current;
3392 prev->next = current->next;
3394 ephemeron_list = current->next;
3396 current = current->next;
3397 sgen_free_internal (tmp, INTERNAL_MEM_EPHEMERON_LINK);
3400 current = current->next;
3405 /* LOCKING: requires that the GC lock is held */
3407 clear_unreachable_ephemerons (ScanCopyContext ctx)
3409 CopyOrMarkObjectFunc copy_func = ctx.copy_func;
3410 GrayQueue *queue = ctx.queue;
3411 EphemeronLinkNode *current = ephemeron_list, *prev = NULL;
3413 Ephemeron *cur, *array_end;
3417 char *object = current->array;
3419 if (!sgen_is_object_alive_for_current_gen (object)) {
3420 EphemeronLinkNode *tmp = current;
3422 SGEN_LOG (5, "Dead Ephemeron array at %p", object);
3425 prev->next = current->next;
3427 ephemeron_list = current->next;
3429 current = current->next;
3430 sgen_free_internal (tmp, INTERNAL_MEM_EPHEMERON_LINK);
3435 copy_func ((void**)&object, queue);
3436 current->array = object;
3438 SGEN_LOG (5, "Clearing unreachable entries for ephemeron array at %p", object);
3440 array = (MonoArray*)object;
3441 cur = mono_array_addr (array, Ephemeron, 0);
3442 array_end = cur + mono_array_length_fast (array);
3443 tombstone = (char*)((MonoVTable*)LOAD_VTABLE (object))->domain->ephemeron_tombstone;
3445 for (; cur < array_end; ++cur) {
3446 char *key = (char*)cur->key;
3448 if (!key || key == tombstone)
3451 SGEN_LOG (5, "[%td] key %p (%s) value %p (%s)", cur - mono_array_addr (array, Ephemeron, 0),
3452 key, sgen_is_object_alive_for_current_gen (key) ? "reachable" : "unreachable",
3453 cur->value, cur->value && sgen_is_object_alive_for_current_gen (cur->value) ? "reachable" : "unreachable");
3455 if (!sgen_is_object_alive_for_current_gen (key)) {
3456 cur->key = tombstone;
3462 current = current->next;
3467 LOCKING: requires that the GC lock is held
3469 Limitations: We scan all ephemerons on every collection since the current design doesn't allow for a simple nursery/mature split.
3472 mark_ephemerons_in_range (ScanCopyContext ctx)
3474 CopyOrMarkObjectFunc copy_func = ctx.copy_func;
3475 GrayQueue *queue = ctx.queue;
3476 int nothing_marked = 1;
3477 EphemeronLinkNode *current = ephemeron_list;
3479 Ephemeron *cur, *array_end;
3482 for (current = ephemeron_list; current; current = current->next) {
3483 char *object = current->array;
3484 SGEN_LOG (5, "Ephemeron array at %p", object);
3486 /*It has to be alive*/
3487 if (!sgen_is_object_alive_for_current_gen (object)) {
3488 SGEN_LOG (5, "\tnot reachable");
3492 copy_func ((void**)&object, queue);
3494 array = (MonoArray*)object;
3495 cur = mono_array_addr (array, Ephemeron, 0);
3496 array_end = cur + mono_array_length_fast (array);
3497 tombstone = (char*)((MonoVTable*)LOAD_VTABLE (object))->domain->ephemeron_tombstone;
3499 for (; cur < array_end; ++cur) {
3500 char *key = cur->key;
3502 if (!key || key == tombstone)
3505 SGEN_LOG (5, "[%td] key %p (%s) value %p (%s)", cur - mono_array_addr (array, Ephemeron, 0),
3506 key, sgen_is_object_alive_for_current_gen (key) ? "reachable" : "unreachable",
3507 cur->value, cur->value && sgen_is_object_alive_for_current_gen (cur->value) ? "reachable" : "unreachable");
3509 if (sgen_is_object_alive_for_current_gen (key)) {
3510 char *value = cur->value;
3512 copy_func ((void**)&cur->key, queue);
3514 if (!sgen_is_object_alive_for_current_gen (value))
3516 copy_func ((void**)&cur->value, queue);
3522 SGEN_LOG (5, "Ephemeron run finished. Is it done %d", nothing_marked);
3523 return nothing_marked;
3527 mono_gc_invoke_finalizers (void)
3529 FinalizeReadyEntry *entry = NULL;
3530 gboolean entry_is_critical = FALSE;
3533 /* FIXME: batch to reduce lock contention */
3534 while (fin_ready_list || critical_fin_list) {
3538 FinalizeReadyEntry **list = entry_is_critical ? &critical_fin_list : &fin_ready_list;
3540 /* We have finalized entry in the last
3541 interation, now we need to remove it from
3544 *list = entry->next;
3546 FinalizeReadyEntry *e = *list;
3547 while (e->next != entry)
3549 e->next = entry->next;
3551 sgen_free_internal (entry, INTERNAL_MEM_FINALIZE_READY_ENTRY);
3555 /* Now look for the first non-null entry. */
3556 for (entry = fin_ready_list; entry && !entry->object; entry = entry->next)
3559 entry_is_critical = FALSE;
3561 entry_is_critical = TRUE;
3562 for (entry = critical_fin_list; entry && !entry->object; entry = entry->next)
3567 g_assert (entry->object);
3568 num_ready_finalizers--;
3569 obj = entry->object;
3570 entry->object = NULL;
3571 SGEN_LOG (7, "Finalizing object %p (%s)", obj, safe_name (obj));
3579 g_assert (entry->object == NULL);
3581 /* the object is on the stack so it is pinned */
3582 /*g_print ("Calling finalizer for object: %p (%s)\n", entry->object, safe_name (entry->object));*/
3583 mono_gc_run_finalize (obj, NULL);
3590 mono_gc_pending_finalizers (void)
3592 return fin_ready_list || critical_fin_list;
3596 * ######################################################################
3597 * ######## registered roots support
3598 * ######################################################################
3602 * We do not coalesce roots.
3605 mono_gc_register_root_inner (char *start, size_t size, void *descr, int root_type)
3607 RootRecord new_root;
3610 for (i = 0; i < ROOT_TYPE_NUM; ++i) {
3611 RootRecord *root = sgen_hash_table_lookup (&roots_hash [i], start);
3612 /* we allow changing the size and the descriptor (for thread statics etc) */
3614 size_t old_size = root->end_root - start;
3615 root->end_root = start + size;
3616 g_assert (((root->root_desc != 0) && (descr != NULL)) ||
3617 ((root->root_desc == 0) && (descr == NULL)));
3618 root->root_desc = (mword)descr;
3620 roots_size -= old_size;
3626 new_root.end_root = start + size;
3627 new_root.root_desc = (mword)descr;
3629 sgen_hash_table_replace (&roots_hash [root_type], start, &new_root, NULL);
3632 SGEN_LOG (3, "Added root for range: %p-%p, descr: %p (%d/%d bytes)", start, new_root.end_root, descr, (int)size, (int)roots_size);
3639 mono_gc_register_root (char *start, size_t size, void *descr)
3641 return mono_gc_register_root_inner (start, size, descr, descr ? ROOT_TYPE_NORMAL : ROOT_TYPE_PINNED);
3645 mono_gc_register_root_wbarrier (char *start, size_t size, void *descr)
3647 return mono_gc_register_root_inner (start, size, descr, ROOT_TYPE_WBARRIER);
3651 mono_gc_deregister_root (char* addr)
3657 for (root_type = 0; root_type < ROOT_TYPE_NUM; ++root_type) {
3658 if (sgen_hash_table_remove (&roots_hash [root_type], addr, &root))
3659 roots_size -= (root.end_root - addr);
3665 * ######################################################################
3666 * ######## Thread handling (stop/start code)
3667 * ######################################################################
3670 unsigned int sgen_global_stop_count = 0;
3673 sgen_get_current_collection_generation (void)
3675 return current_collection_generation;
3679 mono_gc_set_gc_callbacks (MonoGCCallbacks *callbacks)
3681 gc_callbacks = *callbacks;
3685 mono_gc_get_gc_callbacks ()
3687 return &gc_callbacks;
3690 /* Variables holding start/end nursery so it won't have to be passed at every call */
3691 static void *scan_area_arg_start, *scan_area_arg_end;
3694 mono_gc_conservatively_scan_area (void *start, void *end)
3696 conservatively_pin_objects_from (start, end, scan_area_arg_start, scan_area_arg_end, PIN_TYPE_STACK);
3700 mono_gc_scan_object (void *obj, void *gc_data)
3702 UserCopyOrMarkData *data = gc_data;
3703 current_object_ops.copy_or_mark_object (&obj, data->queue);
3708 * Mark from thread stacks and registers.
3711 scan_thread_data (void *start_nursery, void *end_nursery, gboolean precise, GrayQueue *queue)
3713 SgenThreadInfo *info;
3715 scan_area_arg_start = start_nursery;
3716 scan_area_arg_end = end_nursery;
3718 FOREACH_THREAD (info) {
3720 SGEN_LOG (3, "Skipping dead thread %p, range: %p-%p, size: %td", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start);
3723 if (info->gc_disabled) {
3724 SGEN_LOG (3, "GC disabled for thread %p, range: %p-%p, size: %td", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start);
3727 if (mono_thread_info_run_state (info) != STATE_RUNNING) {
3728 SGEN_LOG (3, "Skipping non-running thread %p, range: %p-%p, size: %td (state %d)", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start, mono_thread_info_run_state (info));
3731 SGEN_LOG (3, "Scanning thread %p, range: %p-%p, size: %td, pinned=%zd", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start, sgen_get_pinned_count ());
3732 if (gc_callbacks.thread_mark_func && !conservative_stack_mark) {
3733 UserCopyOrMarkData data = { NULL, queue };
3734 gc_callbacks.thread_mark_func (info->runtime_data, info->stack_start, info->stack_end, precise, &data);
3735 } else if (!precise) {
3736 if (!conservative_stack_mark) {
3737 fprintf (stderr, "Precise stack mark not supported - disabling.\n");
3738 conservative_stack_mark = TRUE;
3740 conservatively_pin_objects_from (info->stack_start, info->stack_end, start_nursery, end_nursery, PIN_TYPE_STACK);
3745 conservatively_pin_objects_from ((void**)&info->ctx, (void**)&info->ctx + ARCH_NUM_REGS,
3746 start_nursery, end_nursery, PIN_TYPE_STACK);
3748 conservatively_pin_objects_from ((void**)&info->regs, (void**)&info->regs + ARCH_NUM_REGS,
3749 start_nursery, end_nursery, PIN_TYPE_STACK);
3752 } END_FOREACH_THREAD
3756 ptr_on_stack (void *ptr)
3758 gpointer stack_start = &stack_start;
3759 SgenThreadInfo *info = mono_thread_info_current ();
3761 if (ptr >= stack_start && ptr < (gpointer)info->stack_end)
3767 sgen_thread_register (SgenThreadInfo* info, void *addr)
3770 guint8 *staddr = NULL;
3772 #ifndef HAVE_KW_THREAD
3773 info->tlab_start = info->tlab_next = info->tlab_temp_end = info->tlab_real_end = NULL;
3775 g_assert (!mono_native_tls_get_value (thread_info_key));
3776 mono_native_tls_set_value (thread_info_key, info);
3778 sgen_thread_info = info;
3781 #ifdef SGEN_POSIX_STW
3782 info->stop_count = -1;
3786 info->stack_start = NULL;
3787 info->stopped_ip = NULL;
3788 info->stopped_domain = NULL;
3790 memset (&info->ctx, 0, sizeof (MonoContext));
3792 memset (&info->regs, 0, sizeof (info->regs));
3795 sgen_init_tlab_info (info);
3797 binary_protocol_thread_register ((gpointer)mono_thread_info_get_tid (info));
3799 /* On win32, stack_start_limit should be 0, since the stack can grow dynamically */
3800 mono_thread_info_get_stack_bounds (&staddr, &stsize);
3803 info->stack_start_limit = staddr;
3805 info->stack_end = staddr + stsize;
3807 gsize stack_bottom = (gsize)addr;
3808 stack_bottom += 4095;
3809 stack_bottom &= ~4095;
3810 info->stack_end = (char*)stack_bottom;
3813 #ifdef HAVE_KW_THREAD
3814 stack_end = info->stack_end;
3817 SGEN_LOG (3, "registered thread %p (%p) stack end %p", info, (gpointer)mono_thread_info_get_tid (info), info->stack_end);
3819 if (gc_callbacks.thread_attach_func)
3820 info->runtime_data = gc_callbacks.thread_attach_func ();
3825 sgen_thread_detach (SgenThreadInfo *p)
3827 /* If a delegate is passed to native code and invoked on a thread we dont
3828 * know about, the jit will register it with mono_jit_thread_attach, but
3829 * we have no way of knowing when that thread goes away. SGen has a TSD
3830 * so we assume that if the domain is still registered, we can detach
3833 if (mono_domain_get ())
3834 mono_thread_detach_internal (mono_thread_internal_current ());
3838 sgen_thread_unregister (SgenThreadInfo *p)
3840 MonoNativeThreadId tid;
3842 tid = mono_thread_info_get_tid (p);
3843 binary_protocol_thread_unregister ((gpointer)tid);
3844 SGEN_LOG (3, "unregister thread %p (%p)", p, (gpointer)tid);
3846 #ifndef HAVE_KW_THREAD
3847 mono_native_tls_set_value (thread_info_key, NULL);
3849 sgen_thread_info = NULL;
3852 if (p->info.runtime_thread)
3853 mono_threads_add_joinable_thread ((gpointer)tid);
3855 if (gc_callbacks.thread_detach_func) {
3856 gc_callbacks.thread_detach_func (p->runtime_data);
3857 p->runtime_data = NULL;
3863 sgen_thread_attach (SgenThreadInfo *info)
3866 /*this is odd, can we get attached before the gc is inited?*/
3870 if (gc_callbacks.thread_attach_func && !info->runtime_data)
3871 info->runtime_data = gc_callbacks.thread_attach_func ();
3874 mono_gc_register_thread (void *baseptr)
3876 return mono_thread_info_attach (baseptr) != NULL;
3880 * mono_gc_set_stack_end:
3882 * Set the end of the current threads stack to STACK_END. The stack space between
3883 * STACK_END and the real end of the threads stack will not be scanned during collections.
3886 mono_gc_set_stack_end (void *stack_end)
3888 SgenThreadInfo *info;
3891 info = mono_thread_info_current ();
3893 g_assert (stack_end < info->stack_end);
3894 info->stack_end = stack_end;
3899 #if USE_PTHREAD_INTERCEPT
3903 mono_gc_pthread_create (pthread_t *new_thread, const pthread_attr_t *attr, void *(*start_routine)(void *), void *arg)
3905 return pthread_create (new_thread, attr, start_routine, arg);
3909 mono_gc_pthread_join (pthread_t thread, void **retval)
3911 return pthread_join (thread, retval);
3915 mono_gc_pthread_detach (pthread_t thread)
3917 return pthread_detach (thread);
3921 mono_gc_pthread_exit (void *retval)
3923 mono_thread_info_detach ();
3924 pthread_exit (retval);
3925 g_assert_not_reached ();
3928 #endif /* USE_PTHREAD_INTERCEPT */
3931 * ######################################################################
3932 * ######## Write barriers
3933 * ######################################################################
3937 * Note: the write barriers first do the needed GC work and then do the actual store:
3938 * this way the value is visible to the conservative GC scan after the write barrier
3939 * itself. If a GC interrupts the barrier in the middle, value will be kept alive by
3940 * the conservative scan, otherwise by the remembered set scan.
3943 mono_gc_wbarrier_set_field (MonoObject *obj, gpointer field_ptr, MonoObject* value)
3945 HEAVY_STAT (++stat_wbarrier_set_field);
3946 if (ptr_in_nursery (field_ptr)) {
3947 *(void**)field_ptr = value;
3950 SGEN_LOG (8, "Adding remset at %p", field_ptr);
3952 binary_protocol_wbarrier (field_ptr, value, value->vtable);
3954 remset.wbarrier_set_field (obj, field_ptr, value);
3958 mono_gc_wbarrier_set_arrayref (MonoArray *arr, gpointer slot_ptr, MonoObject* value)
3960 HEAVY_STAT (++stat_wbarrier_set_arrayref);
3961 if (ptr_in_nursery (slot_ptr)) {
3962 *(void**)slot_ptr = value;
3965 SGEN_LOG (8, "Adding remset at %p", slot_ptr);
3967 binary_protocol_wbarrier (slot_ptr, value, value->vtable);
3969 remset.wbarrier_set_arrayref (arr, slot_ptr, value);
3973 mono_gc_wbarrier_arrayref_copy (gpointer dest_ptr, gpointer src_ptr, int count)
3975 HEAVY_STAT (++stat_wbarrier_arrayref_copy);
3976 /*This check can be done without taking a lock since dest_ptr array is pinned*/
3977 if (ptr_in_nursery (dest_ptr) || count <= 0) {
3978 mono_gc_memmove_aligned (dest_ptr, src_ptr, count * sizeof (gpointer));
3982 #ifdef SGEN_HEAVY_BINARY_PROTOCOL
3983 if (binary_protocol_is_heavy_enabled ()) {
3985 for (i = 0; i < count; ++i) {
3986 gpointer dest = (gpointer*)dest_ptr + i;
3987 gpointer obj = *((gpointer*)src_ptr + i);
3989 binary_protocol_wbarrier (dest, obj, (gpointer)LOAD_VTABLE (obj));
3994 remset.wbarrier_arrayref_copy (dest_ptr, src_ptr, count);
3997 static char *found_obj;
4000 find_object_for_ptr_callback (char *obj, size_t size, void *user_data)
4002 char *ptr = user_data;
4004 if (ptr >= obj && ptr < obj + size) {
4005 g_assert (!found_obj);
4010 /* for use in the debugger */
4011 char* find_object_for_ptr (char *ptr);
4013 find_object_for_ptr (char *ptr)
4015 if (ptr >= nursery_section->data && ptr < nursery_section->end_data) {
4017 sgen_scan_area_with_callback (nursery_section->data, nursery_section->end_data,
4018 find_object_for_ptr_callback, ptr, TRUE);
4024 sgen_los_iterate_objects (find_object_for_ptr_callback, ptr);
4029 * Very inefficient, but this is debugging code, supposed to
4030 * be called from gdb, so we don't care.
4033 major_collector.iterate_objects (ITERATE_OBJECTS_SWEEP_ALL, find_object_for_ptr_callback, ptr);
4038 mono_gc_wbarrier_generic_nostore (gpointer ptr)
4042 HEAVY_STAT (++stat_wbarrier_generic_store);
4044 #ifdef XDOMAIN_CHECKS_IN_WBARRIER
4045 /* FIXME: ptr_in_heap must be called with the GC lock held */
4046 if (xdomain_checks && *(MonoObject**)ptr && ptr_in_heap (ptr)) {
4047 char *start = find_object_for_ptr (ptr);
4048 MonoObject *value = *(MonoObject**)ptr;
4052 MonoObject *obj = (MonoObject*)start;
4053 if (obj->vtable->domain != value->vtable->domain)
4054 g_assert (is_xdomain_ref_allowed (ptr, start, obj->vtable->domain));
4060 obj = *(gpointer*)ptr;
4062 binary_protocol_wbarrier (ptr, obj, (gpointer)LOAD_VTABLE (obj));
4064 if (ptr_in_nursery (ptr) || ptr_on_stack (ptr)) {
4065 SGEN_LOG (8, "Skipping remset at %p", ptr);
4070 * We need to record old->old pointer locations for the
4071 * concurrent collector.
4073 if (!ptr_in_nursery (obj) && !concurrent_collection_in_progress) {
4074 SGEN_LOG (8, "Skipping remset at %p", ptr);
4078 SGEN_LOG (8, "Adding remset at %p", ptr);
4080 remset.wbarrier_generic_nostore (ptr);
4084 mono_gc_wbarrier_generic_store (gpointer ptr, MonoObject* value)
4086 SGEN_LOG (8, "Wbarrier store at %p to %p (%s)", ptr, value, value ? safe_name (value) : "null");
4087 *(void**)ptr = value;
4088 if (ptr_in_nursery (value))
4089 mono_gc_wbarrier_generic_nostore (ptr);
4090 sgen_dummy_use (value);
4093 /* Same as mono_gc_wbarrier_generic_store () but performs the store
4094 * as an atomic operation with release semantics.
4097 mono_gc_wbarrier_generic_store_atomic (gpointer ptr, MonoObject *value)
4099 HEAVY_STAT (++stat_wbarrier_generic_store_atomic);
4101 SGEN_LOG (8, "Wbarrier atomic store at %p to %p (%s)", ptr, value, value ? safe_name (value) : "null");
4103 InterlockedWritePointer (ptr, value);
4105 if (ptr_in_nursery (value))
4106 mono_gc_wbarrier_generic_nostore (ptr);
4108 sgen_dummy_use (value);
4111 void mono_gc_wbarrier_value_copy_bitmap (gpointer _dest, gpointer _src, int size, unsigned bitmap)
4113 mword *dest = _dest;
4118 mono_gc_wbarrier_generic_store (dest, (MonoObject*)*src);
4123 size -= SIZEOF_VOID_P;
4128 #ifdef SGEN_HEAVY_BINARY_PROTOCOL
4130 #define HANDLE_PTR(ptr,obj) do { \
4131 gpointer o = *(gpointer*)(ptr); \
4133 gpointer d = ((char*)dest) + ((char*)(ptr) - (char*)(obj)); \
4134 binary_protocol_wbarrier (d, o, (gpointer) LOAD_VTABLE (o)); \
4139 scan_object_for_binary_protocol_copy_wbarrier (gpointer dest, char *start, mword desc)
4141 #define SCAN_OBJECT_NOVTABLE
4142 #include "sgen-scan-object.h"
4147 mono_gc_wbarrier_value_copy (gpointer dest, gpointer src, int count, MonoClass *klass)
4149 HEAVY_STAT (++stat_wbarrier_value_copy);
4150 g_assert (klass->valuetype);
4152 SGEN_LOG (8, "Adding value remset at %p, count %d, descr %p for class %s (%p)", dest, count, klass->gc_descr, klass->name, klass);
4154 if (ptr_in_nursery (dest) || ptr_on_stack (dest) || !SGEN_CLASS_HAS_REFERENCES (klass)) {
4155 size_t element_size = mono_class_value_size (klass, NULL);
4156 size_t size = count * element_size;
4157 mono_gc_memmove_atomic (dest, src, size);
4161 #ifdef SGEN_HEAVY_BINARY_PROTOCOL
4162 if (binary_protocol_is_heavy_enabled ()) {
4163 size_t element_size = mono_class_value_size (klass, NULL);
4165 for (i = 0; i < count; ++i) {
4166 scan_object_for_binary_protocol_copy_wbarrier ((char*)dest + i * element_size,
4167 (char*)src + i * element_size - sizeof (MonoObject),
4168 (mword) klass->gc_descr);
4173 remset.wbarrier_value_copy (dest, src, count, klass);
4177 * mono_gc_wbarrier_object_copy:
4179 * Write barrier to call when obj is the result of a clone or copy of an object.
4182 mono_gc_wbarrier_object_copy (MonoObject* obj, MonoObject *src)
4186 HEAVY_STAT (++stat_wbarrier_object_copy);
4188 if (ptr_in_nursery (obj) || ptr_on_stack (obj)) {
4189 size = mono_object_class (obj)->instance_size;
4190 mono_gc_memmove_aligned ((char*)obj + sizeof (MonoObject), (char*)src + sizeof (MonoObject),
4191 size - sizeof (MonoObject));
4195 #ifdef SGEN_HEAVY_BINARY_PROTOCOL
4196 if (binary_protocol_is_heavy_enabled ())
4197 scan_object_for_binary_protocol_copy_wbarrier (obj, (char*)src, (mword) src->vtable->gc_descr);
4200 remset.wbarrier_object_copy (obj, src);
4205 * ######################################################################
4206 * ######## Other mono public interface functions.
4207 * ######################################################################
4210 #define REFS_SIZE 128
4213 MonoGCReferences callback;
4217 MonoObject *refs [REFS_SIZE];
4218 uintptr_t offsets [REFS_SIZE];
4222 #define HANDLE_PTR(ptr,obj) do { \
4224 if (hwi->count == REFS_SIZE) { \
4225 hwi->callback ((MonoObject*)start, mono_object_class (start), hwi->called? 0: size, hwi->count, hwi->refs, hwi->offsets, hwi->data); \
4229 hwi->offsets [hwi->count] = (char*)(ptr)-(char*)start; \
4230 hwi->refs [hwi->count++] = *(ptr); \
4235 collect_references (HeapWalkInfo *hwi, char *start, size_t size)
4237 mword desc = sgen_obj_get_descriptor (start);
4239 #include "sgen-scan-object.h"
4243 walk_references (char *start, size_t size, void *data)
4245 HeapWalkInfo *hwi = data;
4248 collect_references (hwi, start, size);
4249 if (hwi->count || !hwi->called)
4250 hwi->callback ((MonoObject*)start, mono_object_class (start), hwi->called? 0: size, hwi->count, hwi->refs, hwi->offsets, hwi->data);
4254 * mono_gc_walk_heap:
4255 * @flags: flags for future use
4256 * @callback: a function pointer called for each object in the heap
4257 * @data: a user data pointer that is passed to callback
4259 * This function can be used to iterate over all the live objects in the heap:
4260 * for each object, @callback is invoked, providing info about the object's
4261 * location in memory, its class, its size and the objects it references.
4262 * For each referenced object it's offset from the object address is
4263 * reported in the offsets array.
4264 * The object references may be buffered, so the callback may be invoked
4265 * multiple times for the same object: in all but the first call, the size
4266 * argument will be zero.
4267 * Note that this function can be only called in the #MONO_GC_EVENT_PRE_START_WORLD
4268 * profiler event handler.
4270 * Returns: a non-zero value if the GC doesn't support heap walking
4273 mono_gc_walk_heap (int flags, MonoGCReferences callback, void *data)
4278 hwi.callback = callback;
4281 sgen_clear_nursery_fragments ();
4282 sgen_scan_area_with_callback (nursery_section->data, nursery_section->end_data, walk_references, &hwi, FALSE);
4284 major_collector.iterate_objects (ITERATE_OBJECTS_SWEEP_ALL, walk_references, &hwi);
4285 sgen_los_iterate_objects (walk_references, &hwi);
4291 mono_gc_collect (int generation)
4296 sgen_perform_collection (0, generation, "user request", TRUE);
4301 mono_gc_max_generation (void)
4307 mono_gc_collection_count (int generation)
4309 if (generation == 0)
4310 return gc_stats.minor_gc_count;
4311 return gc_stats.major_gc_count;
4315 mono_gc_get_used_size (void)
4319 tot = los_memory_usage;
4320 tot += nursery_section->next_data - nursery_section->data;
4321 tot += major_collector.get_used_size ();
4322 /* FIXME: account for pinned objects */
4328 mono_gc_get_los_limit (void)
4330 return MAX_SMALL_OBJ_SIZE;
4334 mono_gc_user_markers_supported (void)
4340 mono_object_is_alive (MonoObject* o)
4346 mono_gc_get_generation (MonoObject *obj)
4348 if (ptr_in_nursery (obj))
4354 mono_gc_enable_events (void)
4359 mono_gc_weak_link_add (void **link_addr, MonoObject *obj, gboolean track)
4361 sgen_register_disappearing_link (obj, link_addr, track, FALSE);
4365 mono_gc_weak_link_remove (void **link_addr, gboolean track)
4367 sgen_register_disappearing_link (NULL, link_addr, track, FALSE);
4371 mono_gc_weak_link_get (void **link_addr)
4373 void * volatile *link_addr_volatile;
4377 link_addr_volatile = link_addr;
4378 ptr = (void*)*link_addr_volatile;
4380 * At this point we have a hidden pointer. If the GC runs
4381 * here, it will not recognize the hidden pointer as a
4382 * reference, and if the object behind it is not referenced
4383 * elsewhere, it will be freed. Once the world is restarted
4384 * we reveal the pointer, giving us a pointer to a freed
4385 * object. To make sure we don't return it, we load the
4386 * hidden pointer again. If it's still the same, we can be
4387 * sure the object reference is valid.
4390 obj = (MonoObject*) REVEAL_POINTER (ptr);
4394 mono_memory_barrier ();
4397 * During the second bridge processing step the world is
4398 * running again. That step processes all weak links once
4399 * more to null those that refer to dead objects. Before that
4400 * is completed, those links must not be followed, so we
4401 * conservatively wait for bridge processing when any weak
4402 * link is dereferenced.
4404 if (G_UNLIKELY (bridge_processing_in_progress))
4405 mono_gc_wait_for_bridge_processing ();
4407 if ((void*)*link_addr_volatile != ptr)
4414 mono_gc_ephemeron_array_add (MonoObject *obj)
4416 EphemeronLinkNode *node;
4420 node = sgen_alloc_internal (INTERNAL_MEM_EPHEMERON_LINK);
4425 node->array = (char*)obj;
4426 node->next = ephemeron_list;
4427 ephemeron_list = node;
4429 SGEN_LOG (5, "Registered ephemeron array %p", obj);
4436 mono_gc_set_allow_synchronous_major (gboolean flag)
4438 if (!major_collector.is_concurrent)
4441 allow_synchronous_major = flag;
4446 mono_gc_invoke_with_gc_lock (MonoGCLockedCallbackFunc func, void *data)
4450 result = func (data);
4451 UNLOCK_INTERRUPTION;
4456 mono_gc_is_gc_thread (void)
4460 result = mono_thread_info_current () != NULL;
4466 is_critical_method (MonoMethod *method)
4468 return mono_runtime_is_critical_method (method) || sgen_is_critical_method (method);
4472 sgen_env_var_error (const char *env_var, const char *fallback, const char *description_format, ...)
4476 va_start (ap, description_format);
4478 fprintf (stderr, "Warning: In environment variable `%s': ", env_var);
4479 vfprintf (stderr, description_format, ap);
4481 fprintf (stderr, " - %s", fallback);
4482 fprintf (stderr, "\n");
4488 parse_double_in_interval (const char *env_var, const char *opt_name, const char *opt, double min, double max, double *result)
4491 double val = strtod (opt, &endptr);
4492 if (endptr == opt) {
4493 sgen_env_var_error (env_var, "Using default value.", "`%s` must be a number.", opt_name);
4496 else if (val < min || val > max) {
4497 sgen_env_var_error (env_var, "Using default value.", "`%s` must be between %.2f - %.2f.", opt_name, min, max);
4505 mono_gc_base_init (void)
4507 MonoThreadInfoCallbacks cb;
4510 char *major_collector_opt = NULL;
4511 char *minor_collector_opt = NULL;
4512 size_t max_heap = 0;
4513 size_t soft_limit = 0;
4516 gboolean debug_print_allowance = FALSE;
4517 double allowance_ratio = 0, save_target = 0;
4518 gboolean have_split_nursery = FALSE;
4519 gboolean cement_enabled = TRUE;
4522 result = InterlockedCompareExchange (&gc_initialized, -1, 0);
4525 /* already inited */
4528 /* being inited by another thread */
4532 /* we will init it */
4535 g_assert_not_reached ();
4537 } while (result != 0);
4539 SGEN_TV_GETTIME (sgen_init_timestamp);
4541 LOCK_INIT (gc_mutex);
4543 pagesize = mono_pagesize ();
4544 gc_debug_file = stderr;
4546 cb.thread_register = sgen_thread_register;
4547 cb.thread_detach = sgen_thread_detach;
4548 cb.thread_unregister = sgen_thread_unregister;
4549 cb.thread_attach = sgen_thread_attach;
4550 cb.mono_method_is_critical = (gpointer)is_critical_method;
4552 cb.thread_exit = mono_gc_pthread_exit;
4553 cb.mono_gc_pthread_create = (gpointer)mono_gc_pthread_create;
4556 mono_threads_init (&cb, sizeof (SgenThreadInfo));
4558 LOCK_INIT (sgen_interruption_mutex);
4559 LOCK_INIT (pin_queue_mutex);
4561 if ((env = g_getenv (MONO_GC_PARAMS_NAME))) {
4562 opts = g_strsplit (env, ",", -1);
4563 for (ptr = opts; *ptr; ++ptr) {
4565 if (g_str_has_prefix (opt, "major=")) {
4566 opt = strchr (opt, '=') + 1;
4567 major_collector_opt = g_strdup (opt);
4568 } else if (g_str_has_prefix (opt, "minor=")) {
4569 opt = strchr (opt, '=') + 1;
4570 minor_collector_opt = g_strdup (opt);
4578 sgen_init_internal_allocator ();
4579 sgen_init_nursery_allocator ();
4580 sgen_init_fin_weak_hash ();
4582 sgen_init_hash_table ();
4583 sgen_init_descriptors ();
4584 sgen_init_gray_queues ();
4586 sgen_register_fixed_internal_mem_type (INTERNAL_MEM_SECTION, SGEN_SIZEOF_GC_MEM_SECTION);
4587 sgen_register_fixed_internal_mem_type (INTERNAL_MEM_FINALIZE_READY_ENTRY, sizeof (FinalizeReadyEntry));
4588 sgen_register_fixed_internal_mem_type (INTERNAL_MEM_GRAY_QUEUE, sizeof (GrayQueueSection));
4589 sgen_register_fixed_internal_mem_type (INTERNAL_MEM_EPHEMERON_LINK, sizeof (EphemeronLinkNode));
4591 #ifndef HAVE_KW_THREAD
4592 mono_native_tls_alloc (&thread_info_key, NULL);
4593 #if defined(__APPLE__) || defined (HOST_WIN32)
4595 * CEE_MONO_TLS requires the tls offset, not the key, so the code below only works on darwin,
4596 * where the two are the same.
4598 mono_tls_key_set_offset (TLS_KEY_SGEN_THREAD_INFO, thread_info_key);
4602 int tls_offset = -1;
4603 MONO_THREAD_VAR_OFFSET (sgen_thread_info, tls_offset);
4604 mono_tls_key_set_offset (TLS_KEY_SGEN_THREAD_INFO, tls_offset);
4609 * This needs to happen before any internal allocations because
4610 * it inits the small id which is required for hazard pointer
4615 mono_thread_info_attach (&dummy);
4617 if (!minor_collector_opt) {
4618 sgen_simple_nursery_init (&sgen_minor_collector);
4620 if (!strcmp (minor_collector_opt, "simple")) {
4622 sgen_simple_nursery_init (&sgen_minor_collector);
4623 } else if (!strcmp (minor_collector_opt, "split")) {
4624 sgen_split_nursery_init (&sgen_minor_collector);
4625 have_split_nursery = TRUE;
4627 sgen_env_var_error (MONO_GC_PARAMS_NAME, "Using `simple` instead.", "Unknown minor collector `%s'.", minor_collector_opt);
4628 goto use_simple_nursery;
4632 if (!major_collector_opt || !strcmp (major_collector_opt, "marksweep")) {
4633 use_marksweep_major:
4634 sgen_marksweep_init (&major_collector);
4635 } else if (!major_collector_opt || !strcmp (major_collector_opt, "marksweep-conc")) {
4636 sgen_marksweep_conc_init (&major_collector);
4638 sgen_env_var_error (MONO_GC_PARAMS_NAME, "Using `marksweep` instead.", "Unknown major collector `%s'.", major_collector_opt);
4639 goto use_marksweep_major;
4642 ///* Keep this the default for now */
4643 /* Precise marking is broken on all supported targets. Disable until fixed. */
4644 conservative_stack_mark = TRUE;
4646 sgen_nursery_size = DEFAULT_NURSERY_SIZE;
4649 gboolean usage_printed = FALSE;
4651 for (ptr = opts; *ptr; ++ptr) {
4653 if (!strcmp (opt, ""))
4655 if (g_str_has_prefix (opt, "major="))
4657 if (g_str_has_prefix (opt, "minor="))
4659 if (g_str_has_prefix (opt, "max-heap-size=")) {
4660 size_t max_heap_candidate = 0;
4661 opt = strchr (opt, '=') + 1;
4662 if (*opt && mono_gc_parse_environment_string_extract_number (opt, &max_heap_candidate)) {
4663 max_heap = (max_heap_candidate + mono_pagesize () - 1) & ~(size_t)(mono_pagesize () - 1);
4664 if (max_heap != max_heap_candidate)
4665 sgen_env_var_error (MONO_GC_PARAMS_NAME, "Rounding up.", "`max-heap-size` size must be a multiple of %d.", mono_pagesize ());
4667 sgen_env_var_error (MONO_GC_PARAMS_NAME, NULL, "`max-heap-size` must be an integer.");
4671 if (g_str_has_prefix (opt, "soft-heap-limit=")) {
4672 opt = strchr (opt, '=') + 1;
4673 if (*opt && mono_gc_parse_environment_string_extract_number (opt, &soft_limit)) {
4674 if (soft_limit <= 0) {
4675 sgen_env_var_error (MONO_GC_PARAMS_NAME, NULL, "`soft-heap-limit` must be positive.");
4679 sgen_env_var_error (MONO_GC_PARAMS_NAME, NULL, "`soft-heap-limit` must be an integer.");
4683 if (g_str_has_prefix (opt, "stack-mark=")) {
4684 opt = strchr (opt, '=') + 1;
4685 if (!strcmp (opt, "precise")) {
4686 conservative_stack_mark = FALSE;
4687 } else if (!strcmp (opt, "conservative")) {
4688 conservative_stack_mark = TRUE;
4690 sgen_env_var_error (MONO_GC_PARAMS_NAME, conservative_stack_mark ? "Using `conservative`." : "Using `precise`.",
4691 "Invalid value `%s` for `stack-mark` option, possible values are: `precise`, `conservative`.", opt);
4695 if (g_str_has_prefix (opt, "bridge-implementation=")) {
4696 opt = strchr (opt, '=') + 1;
4697 sgen_set_bridge_implementation (opt);
4700 if (g_str_has_prefix (opt, "toggleref-test")) {
4701 sgen_register_test_toggleref_callback ();
4706 if (g_str_has_prefix (opt, "nursery-size=")) {
4708 opt = strchr (opt, '=') + 1;
4709 if (*opt && mono_gc_parse_environment_string_extract_number (opt, &val)) {
4710 #ifdef SGEN_ALIGN_NURSERY
4711 if ((val & (val - 1))) {
4712 sgen_env_var_error (MONO_GC_PARAMS_NAME, "Using default value.", "`nursery-size` must be a power of two.");
4716 if (val < SGEN_MAX_NURSERY_WASTE) {
4717 sgen_env_var_error (MONO_GC_PARAMS_NAME, "Using default value.",
4718 "`nursery-size` must be at least %d bytes.", SGEN_MAX_NURSERY_WASTE);
4722 sgen_nursery_size = val;
4723 sgen_nursery_bits = 0;
4724 while (ONE_P << (++ sgen_nursery_bits) != sgen_nursery_size)
4727 sgen_nursery_size = val;
4730 sgen_env_var_error (MONO_GC_PARAMS_NAME, "Using default value.", "`nursery-size` must be an integer.");
4736 if (g_str_has_prefix (opt, "save-target-ratio=")) {
4738 opt = strchr (opt, '=') + 1;
4739 if (parse_double_in_interval (MONO_GC_PARAMS_NAME, "save-target-ratio", opt,
4740 SGEN_MIN_SAVE_TARGET_RATIO, SGEN_MAX_SAVE_TARGET_RATIO, &val)) {
4745 if (g_str_has_prefix (opt, "default-allowance-ratio=")) {
4747 opt = strchr (opt, '=') + 1;
4748 if (parse_double_in_interval (MONO_GC_PARAMS_NAME, "default-allowance-ratio", opt,
4749 SGEN_MIN_ALLOWANCE_NURSERY_SIZE_RATIO, SGEN_MIN_ALLOWANCE_NURSERY_SIZE_RATIO, &val)) {
4750 allowance_ratio = val;
4754 if (g_str_has_prefix (opt, "allow-synchronous-major=")) {
4755 if (!major_collector.is_concurrent) {
4756 sgen_env_var_error (MONO_GC_PARAMS_NAME, "Ignoring.", "`allow-synchronous-major` is only valid for the concurrent major collector.");
4760 opt = strchr (opt, '=') + 1;
4762 if (!strcmp (opt, "yes")) {
4763 allow_synchronous_major = TRUE;
4764 } else if (!strcmp (opt, "no")) {
4765 allow_synchronous_major = FALSE;
4767 sgen_env_var_error (MONO_GC_PARAMS_NAME, "Using default value.", "`allow-synchronous-major` must be either `yes' or `no'.");
4772 if (!strcmp (opt, "cementing")) {
4773 cement_enabled = TRUE;
4776 if (!strcmp (opt, "no-cementing")) {
4777 cement_enabled = FALSE;
4781 if (major_collector.handle_gc_param && major_collector.handle_gc_param (opt))
4784 if (sgen_minor_collector.handle_gc_param && sgen_minor_collector.handle_gc_param (opt))
4787 sgen_env_var_error (MONO_GC_PARAMS_NAME, "Ignoring.", "Unknown option `%s`.", opt);
4792 fprintf (stderr, "\n%s must be a comma-delimited list of one or more of the following:\n", MONO_GC_PARAMS_NAME);
4793 fprintf (stderr, " max-heap-size=N (where N is an integer, possibly with a k, m or a g suffix)\n");
4794 fprintf (stderr, " soft-heap-limit=n (where N is an integer, possibly with a k, m or a g suffix)\n");
4795 fprintf (stderr, " nursery-size=N (where N is an integer, possibly with a k, m or a g suffix)\n");
4796 fprintf (stderr, " major=COLLECTOR (where COLLECTOR is `marksweep', `marksweep-conc', `marksweep-par')\n");
4797 fprintf (stderr, " minor=COLLECTOR (where COLLECTOR is `simple' or `split')\n");
4798 fprintf (stderr, " wbarrier=WBARRIER (where WBARRIER is `remset' or `cardtable')\n");
4799 fprintf (stderr, " stack-mark=MARK-METHOD (where MARK-METHOD is 'precise' or 'conservative')\n");
4800 fprintf (stderr, " [no-]cementing\n");
4801 if (major_collector.is_concurrent)
4802 fprintf (stderr, " allow-synchronous-major=FLAG (where FLAG is `yes' or `no')\n");
4803 if (major_collector.print_gc_param_usage)
4804 major_collector.print_gc_param_usage ();
4805 if (sgen_minor_collector.print_gc_param_usage)
4806 sgen_minor_collector.print_gc_param_usage ();
4807 fprintf (stderr, " Experimental options:\n");
4808 fprintf (stderr, " save-target-ratio=R (where R must be between %.2f - %.2f).\n", SGEN_MIN_SAVE_TARGET_RATIO, SGEN_MAX_SAVE_TARGET_RATIO);
4809 fprintf (stderr, " default-allowance-ratio=R (where R must be between %.2f - %.2f).\n", SGEN_MIN_ALLOWANCE_NURSERY_SIZE_RATIO, SGEN_MAX_ALLOWANCE_NURSERY_SIZE_RATIO);
4810 fprintf (stderr, "\n");
4812 usage_printed = TRUE;
4817 if (major_collector.is_concurrent)
4818 sgen_workers_init (1);
4820 if (major_collector_opt)
4821 g_free (major_collector_opt);
4823 if (minor_collector_opt)
4824 g_free (minor_collector_opt);
4828 sgen_cement_init (cement_enabled);
4830 if ((env = g_getenv (MONO_GC_DEBUG_NAME))) {
4831 gboolean usage_printed = FALSE;
4833 opts = g_strsplit (env, ",", -1);
4834 for (ptr = opts; ptr && *ptr; ptr ++) {
4836 if (!strcmp (opt, ""))
4838 if (opt [0] >= '0' && opt [0] <= '9') {
4839 gc_debug_level = atoi (opt);
4845 char *rf = g_strdup_printf ("%s.%d", opt, GetCurrentProcessId ());
4847 char *rf = g_strdup_printf ("%s.%d", opt, getpid ());
4849 gc_debug_file = fopen (rf, "wb");
4851 gc_debug_file = stderr;
4854 } else if (!strcmp (opt, "print-allowance")) {
4855 debug_print_allowance = TRUE;
4856 } else if (!strcmp (opt, "print-pinning")) {
4857 do_pin_stats = TRUE;
4858 } else if (!strcmp (opt, "verify-before-allocs")) {
4859 verify_before_allocs = 1;
4860 has_per_allocation_action = TRUE;
4861 } else if (g_str_has_prefix (opt, "verify-before-allocs=")) {
4862 char *arg = strchr (opt, '=') + 1;
4863 verify_before_allocs = atoi (arg);
4864 has_per_allocation_action = TRUE;
4865 } else if (!strcmp (opt, "collect-before-allocs")) {
4866 collect_before_allocs = 1;
4867 has_per_allocation_action = TRUE;
4868 } else if (g_str_has_prefix (opt, "collect-before-allocs=")) {
4869 char *arg = strchr (opt, '=') + 1;
4870 has_per_allocation_action = TRUE;
4871 collect_before_allocs = atoi (arg);
4872 } else if (!strcmp (opt, "verify-before-collections")) {
4873 whole_heap_check_before_collection = TRUE;
4874 } else if (!strcmp (opt, "check-at-minor-collections")) {
4875 consistency_check_at_minor_collection = TRUE;
4876 nursery_clear_policy = CLEAR_AT_GC;
4877 } else if (!strcmp (opt, "mod-union-consistency-check")) {
4878 if (!major_collector.is_concurrent) {
4879 sgen_env_var_error (MONO_GC_DEBUG_NAME, "Ignoring.", "`mod-union-consistency-check` only works with concurrent major collector.");
4882 mod_union_consistency_check = TRUE;
4883 } else if (!strcmp (opt, "check-mark-bits")) {
4884 check_mark_bits_after_major_collection = TRUE;
4885 } else if (!strcmp (opt, "check-nursery-pinned")) {
4886 check_nursery_objects_pinned = TRUE;
4887 } else if (!strcmp (opt, "xdomain-checks")) {
4888 xdomain_checks = TRUE;
4889 } else if (!strcmp (opt, "clear-at-gc")) {
4890 nursery_clear_policy = CLEAR_AT_GC;
4891 } else if (!strcmp (opt, "clear-nursery-at-gc")) {
4892 nursery_clear_policy = CLEAR_AT_GC;
4893 } else if (!strcmp (opt, "clear-at-tlab-creation")) {
4894 nursery_clear_policy = CLEAR_AT_TLAB_CREATION;
4895 } else if (!strcmp (opt, "debug-clear-at-tlab-creation")) {
4896 nursery_clear_policy = CLEAR_AT_TLAB_CREATION_DEBUG;
4897 } else if (!strcmp (opt, "check-scan-starts")) {
4898 do_scan_starts_check = TRUE;
4899 } else if (!strcmp (opt, "verify-nursery-at-minor-gc")) {
4900 do_verify_nursery = TRUE;
4901 } else if (!strcmp (opt, "check-concurrent")) {
4902 if (!major_collector.is_concurrent) {
4903 sgen_env_var_error (MONO_GC_DEBUG_NAME, "Ignoring.", "`check-concurrent` only works with concurrent major collectors.");
4906 do_concurrent_checks = TRUE;
4907 } else if (!strcmp (opt, "dump-nursery-at-minor-gc")) {
4908 do_dump_nursery_content = TRUE;
4909 } else if (!strcmp (opt, "no-managed-allocator")) {
4910 sgen_set_use_managed_allocator (FALSE);
4911 } else if (!strcmp (opt, "disable-minor")) {
4912 disable_minor_collections = TRUE;
4913 } else if (!strcmp (opt, "disable-major")) {
4914 disable_major_collections = TRUE;
4915 } else if (g_str_has_prefix (opt, "heap-dump=")) {
4916 char *filename = strchr (opt, '=') + 1;
4917 nursery_clear_policy = CLEAR_AT_GC;
4918 heap_dump_file = fopen (filename, "w");
4919 if (heap_dump_file) {
4920 fprintf (heap_dump_file, "<sgen-dump>\n");
4921 do_pin_stats = TRUE;
4923 } else if (g_str_has_prefix (opt, "binary-protocol=")) {
4924 char *filename = strchr (opt, '=') + 1;
4925 char *colon = strrchr (filename, ':');
4928 if (!mono_gc_parse_environment_string_extract_number (colon + 1, &limit)) {
4929 sgen_env_var_error (MONO_GC_DEBUG_NAME, "Ignoring limit.", "Binary protocol file size limit must be an integer.");
4934 binary_protocol_init (filename, (long long)limit);
4935 } else if (!sgen_bridge_handle_gc_debug (opt)) {
4936 sgen_env_var_error (MONO_GC_DEBUG_NAME, "Ignoring.", "Unknown option `%s`.", opt);
4941 fprintf (stderr, "\n%s must be of the format [<l>[:<filename>]|<option>]+ where <l> is a debug level 0-9.\n", MONO_GC_DEBUG_NAME);
4942 fprintf (stderr, "Valid <option>s are:\n");
4943 fprintf (stderr, " collect-before-allocs[=<n>]\n");
4944 fprintf (stderr, " verify-before-allocs[=<n>]\n");
4945 fprintf (stderr, " check-at-minor-collections\n");
4946 fprintf (stderr, " check-mark-bits\n");
4947 fprintf (stderr, " check-nursery-pinned\n");
4948 fprintf (stderr, " verify-before-collections\n");
4949 fprintf (stderr, " verify-nursery-at-minor-gc\n");
4950 fprintf (stderr, " dump-nursery-at-minor-gc\n");
4951 fprintf (stderr, " disable-minor\n");
4952 fprintf (stderr, " disable-major\n");
4953 fprintf (stderr, " xdomain-checks\n");
4954 fprintf (stderr, " check-concurrent\n");
4955 fprintf (stderr, " clear-[nursery-]at-gc\n");
4956 fprintf (stderr, " clear-at-tlab-creation\n");
4957 fprintf (stderr, " debug-clear-at-tlab-creation\n");
4958 fprintf (stderr, " check-scan-starts\n");
4959 fprintf (stderr, " no-managed-allocator\n");
4960 fprintf (stderr, " print-allowance\n");
4961 fprintf (stderr, " print-pinning\n");
4962 fprintf (stderr, " heap-dump=<filename>\n");
4963 fprintf (stderr, " binary-protocol=<filename>[:<file-size-limit>]\n");
4964 sgen_bridge_print_gc_debug_usage ();
4965 fprintf (stderr, "\n");
4967 usage_printed = TRUE;
4973 if (major_collector.post_param_init)
4974 major_collector.post_param_init (&major_collector);
4976 sgen_memgov_init (max_heap, soft_limit, debug_print_allowance, allowance_ratio, save_target);
4978 memset (&remset, 0, sizeof (remset));
4980 sgen_card_table_init (&remset);
4986 mono_gc_get_gc_name (void)
4991 static MonoMethod *write_barrier_method;
4994 sgen_is_critical_method (MonoMethod *method)
4996 return (method == write_barrier_method || sgen_is_managed_allocator (method));
5000 sgen_has_critical_method (void)
5002 return write_barrier_method || sgen_has_managed_allocator ();
5008 emit_nursery_check (MonoMethodBuilder *mb, int *nursery_check_return_labels)
5010 memset (nursery_check_return_labels, 0, sizeof (int) * 3);
5011 #ifdef SGEN_ALIGN_NURSERY
5012 // if (ptr_in_nursery (ptr)) return;
5014 * Masking out the bits might be faster, but we would have to use 64 bit
5015 * immediates, which might be slower.
5017 mono_mb_emit_ldarg (mb, 0);
5018 mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
5019 mono_mb_emit_byte (mb, CEE_SHR_UN);
5020 mono_mb_emit_ptr (mb, (gpointer)((mword)sgen_get_nursery_start () >> DEFAULT_NURSERY_BITS));
5021 nursery_check_return_labels [0] = mono_mb_emit_branch (mb, CEE_BEQ);
5023 if (!major_collector.is_concurrent) {
5024 // if (!ptr_in_nursery (*ptr)) return;
5025 mono_mb_emit_ldarg (mb, 0);
5026 mono_mb_emit_byte (mb, CEE_LDIND_I);
5027 mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
5028 mono_mb_emit_byte (mb, CEE_SHR_UN);
5029 mono_mb_emit_ptr (mb, (gpointer)((mword)sgen_get_nursery_start () >> DEFAULT_NURSERY_BITS));
5030 nursery_check_return_labels [1] = mono_mb_emit_branch (mb, CEE_BNE_UN);
5033 int label_continue1, label_continue2;
5034 int dereferenced_var;
5036 // if (ptr < (sgen_get_nursery_start ())) goto continue;
5037 mono_mb_emit_ldarg (mb, 0);
5038 mono_mb_emit_ptr (mb, (gpointer) sgen_get_nursery_start ());
5039 label_continue_1 = mono_mb_emit_branch (mb, CEE_BLT);
5041 // if (ptr >= sgen_get_nursery_end ())) goto continue;
5042 mono_mb_emit_ldarg (mb, 0);
5043 mono_mb_emit_ptr (mb, (gpointer) sgen_get_nursery_end ());
5044 label_continue_2 = mono_mb_emit_branch (mb, CEE_BGE);
5047 nursery_check_return_labels [0] = mono_mb_emit_branch (mb, CEE_BR);
5050 mono_mb_patch_branch (mb, label_continue_1);
5051 mono_mb_patch_branch (mb, label_continue_2);
5053 // Dereference and store in local var
5054 dereferenced_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
5055 mono_mb_emit_ldarg (mb, 0);
5056 mono_mb_emit_byte (mb, CEE_LDIND_I);
5057 mono_mb_emit_stloc (mb, dereferenced_var);
5059 if (!major_collector.is_concurrent) {
5060 // if (*ptr < sgen_get_nursery_start ()) return;
5061 mono_mb_emit_ldloc (mb, dereferenced_var);
5062 mono_mb_emit_ptr (mb, (gpointer) sgen_get_nursery_start ());
5063 nursery_check_return_labels [1] = mono_mb_emit_branch (mb, CEE_BLT);
5065 // if (*ptr >= sgen_get_nursery_end ()) return;
5066 mono_mb_emit_ldloc (mb, dereferenced_var);
5067 mono_mb_emit_ptr (mb, (gpointer) sgen_get_nursery_end ());
5068 nursery_check_return_labels [2] = mono_mb_emit_branch (mb, CEE_BGE);
5075 mono_gc_get_write_barrier (void)
5078 MonoMethodBuilder *mb;
5079 MonoMethodSignature *sig;
5080 #ifdef MANAGED_WBARRIER
5081 int i, nursery_check_labels [3];
5083 #ifdef HAVE_KW_THREAD
5084 int stack_end_offset = -1;
5086 MONO_THREAD_VAR_OFFSET (stack_end, stack_end_offset);
5087 g_assert (stack_end_offset != -1);
5091 // FIXME: Maybe create a separate version for ctors (the branch would be
5092 // correctly predicted more times)
5093 if (write_barrier_method)
5094 return write_barrier_method;
5096 /* Create the IL version of mono_gc_barrier_generic_store () */
5097 sig = mono_metadata_signature_alloc (mono_defaults.corlib, 1);
5098 sig->ret = &mono_defaults.void_class->byval_arg;
5099 sig->params [0] = &mono_defaults.int_class->byval_arg;
5101 mb = mono_mb_new (mono_defaults.object_class, "wbarrier", MONO_WRAPPER_WRITE_BARRIER);
5104 #ifdef MANAGED_WBARRIER
5105 emit_nursery_check (mb, nursery_check_labels);
5107 addr = sgen_cardtable + ((address >> CARD_BITS) & CARD_MASK)
5111 LDC_PTR sgen_cardtable
5113 address >> CARD_BITS
5117 if (SGEN_HAVE_OVERLAPPING_CARDS) {
5118 LDC_PTR card_table_mask
5125 mono_mb_emit_ptr (mb, sgen_cardtable);
5126 mono_mb_emit_ldarg (mb, 0);
5127 mono_mb_emit_icon (mb, CARD_BITS);
5128 mono_mb_emit_byte (mb, CEE_SHR_UN);
5129 #ifdef SGEN_HAVE_OVERLAPPING_CARDS
5130 mono_mb_emit_ptr (mb, (gpointer)CARD_MASK);
5131 mono_mb_emit_byte (mb, CEE_AND);
5133 mono_mb_emit_byte (mb, CEE_ADD);
5134 mono_mb_emit_icon (mb, 1);
5135 mono_mb_emit_byte (mb, CEE_STIND_I1);
5138 for (i = 0; i < 3; ++i) {
5139 if (nursery_check_labels [i])
5140 mono_mb_patch_branch (mb, nursery_check_labels [i]);
5142 mono_mb_emit_byte (mb, CEE_RET);
5144 mono_mb_emit_ldarg (mb, 0);
5145 mono_mb_emit_icall (mb, mono_gc_wbarrier_generic_nostore);
5146 mono_mb_emit_byte (mb, CEE_RET);
5149 res = mono_mb_create_method (mb, sig, 16);
5153 if (write_barrier_method) {
5154 /* Already created */
5155 mono_free_method (res);
5157 /* double-checked locking */
5158 mono_memory_barrier ();
5159 write_barrier_method = res;
5163 return write_barrier_method;
5167 mono_gc_get_description (void)
5169 return g_strdup ("sgen");
5173 mono_gc_set_desktop_mode (void)
5178 mono_gc_is_moving (void)
5184 mono_gc_is_disabled (void)
5190 BOOL APIENTRY mono_gc_dllmain (HMODULE module_handle, DWORD reason, LPVOID reserved)
5197 sgen_get_nursery_clear_policy (void)
5199 return nursery_clear_policy;
5203 sgen_get_array_fill_vtable (void)
5205 if (!array_fill_vtable) {
5206 static MonoClass klass;
5207 static MonoVTable vtable;
5210 MonoDomain *domain = mono_get_root_domain ();
5213 klass.element_class = mono_defaults.byte_class;
5215 klass.instance_size = sizeof (MonoArray);
5216 klass.sizes.element_size = 1;
5217 klass.name = "array_filler_type";
5219 vtable.klass = &klass;
5221 vtable.gc_descr = mono_gc_make_descr_for_array (TRUE, &bmap, 0, 1);
5224 array_fill_vtable = &vtable;
5226 return array_fill_vtable;
5236 sgen_gc_unlock (void)
5238 gboolean try_free = sgen_try_free_some_memory;
5239 sgen_try_free_some_memory = FALSE;
5240 mono_mutex_unlock (&gc_mutex);
5241 MONO_GC_UNLOCKED ();
5243 mono_thread_hazardous_try_free_some ();
5247 sgen_major_collector_iterate_live_block_ranges (sgen_cardtable_block_callback callback)
5249 major_collector.iterate_live_block_ranges (callback);
5253 sgen_major_collector_scan_card_table (SgenGrayQueue *queue)
5255 major_collector.scan_card_table (FALSE, queue);
5259 sgen_get_major_collector (void)
5261 return &major_collector;
5264 void mono_gc_set_skip_thread (gboolean skip)
5266 SgenThreadInfo *info = mono_thread_info_current ();
5269 info->gc_disabled = skip;
5274 sgen_get_remset (void)
5280 mono_gc_get_vtable_bits (MonoClass *class)
5283 /* FIXME move this to the bridge code */
5284 if (sgen_need_bridge_processing ()) {
5285 switch (sgen_bridge_class_kind (class)) {
5286 case GC_BRIDGE_TRANSPARENT_BRIDGE_CLASS:
5287 case GC_BRIDGE_OPAQUE_BRIDGE_CLASS:
5288 res = SGEN_GC_BIT_BRIDGE_OBJECT;
5290 case GC_BRIDGE_OPAQUE_CLASS:
5291 res = SGEN_GC_BIT_BRIDGE_OPAQUE_OBJECT;
5295 if (fin_callbacks.is_class_finalization_aware) {
5296 if (fin_callbacks.is_class_finalization_aware (class))
5297 res |= SGEN_GC_BIT_FINALIZER_AWARE;
5303 mono_gc_register_altstack (gpointer stack, gint32 stack_size, gpointer altstack, gint32 altstack_size)
5310 sgen_check_whole_heap_stw (void)
5312 sgen_stop_world (0);
5313 sgen_clear_nursery_fragments ();
5314 sgen_check_whole_heap (FALSE);
5315 sgen_restart_world (0, NULL);
5319 sgen_gc_event_moves (void)
5321 if (moved_objects_idx) {
5322 mono_profiler_gc_moves (moved_objects, moved_objects_idx);
5323 moved_objects_idx = 0;
5328 sgen_timestamp (void)
5330 SGEN_TV_DECLARE (timestamp);
5331 SGEN_TV_GETTIME (timestamp);
5332 return SGEN_TV_ELAPSED (sgen_init_timestamp, timestamp);
5336 mono_gc_register_finalizer_callbacks (MonoGCFinalizerCallbacks *callbacks)
5338 if (callbacks->version != MONO_GC_FINALIZER_EXTENSION_VERSION)
5339 g_error ("Invalid finalizer callback version. Expected %d but got %d\n", MONO_GC_FINALIZER_EXTENSION_VERSION, callbacks->version);
5341 fin_callbacks = *callbacks;
5344 #endif /* HAVE_SGEN_GC */